2023-10-01T08:39:21.480 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-01T08:39:21.480 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407954 --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --name teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi --verbose --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407954/orig.config.yaml 2023-10-01T08:39:21.578 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-01T08:39:21.903 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407954 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7407954' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: fc1e90a395a6a5631142eb70fd9bf3d22740a9da owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 3147 sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_fc1e90a395a6a5631142eb70fd9bf3d22740a9da/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: fc1e90a395a6a5631142eb70fd9bf3d22740a9da targets: smithi138.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi190.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-30_14:23:11 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-01T08:39:21.905 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_fc1e90a395a6a5631142eb70fd9bf3d22740a9da/qa; will attempt to use it 2023-10-01T08:39:21.905 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_fc1e90a395a6a5631142eb70fd9bf3d22740a9da/qa/tasks 2023-10-01T08:39:21.906 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-01T08:39:21.909 INFO:teuthology.task.internal:Checking packages... 2023-10-01T08:39:21.941 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'b56822191ad0416d0c05ed59720df03ad9c2a44e' 2023-10-01T08:39:21.941 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-01T08:39:21.941 INFO:teuthology.packaging:ref: None 2023-10-01T08:39:21.941 INFO:teuthology.packaging:tag: None 2023-10-01T08:39:21.941 INFO:teuthology.packaging:branch: reef 2023-10-01T08:39:21.941 INFO:teuthology.packaging:sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T08:39:21.941 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-10-01T08:39:22.325 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-501.gfc1e90a3 2023-10-01T08:39:22.326 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-01T08:39:22.340 INFO:teuthology.task.internal:no buildpackages task found 2023-10-01T08:39:22.340 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-01T08:39:22.347 INFO:teuthology.task.internal:Saving configuration 2023-10-01T08:39:22.379 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-01T08:39:22.388 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-01T08:39:22.416 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi138.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407954', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-01 08:34:16.615934', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-01T08:39:22.443 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi190.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407954', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-01 08:34:16.614776', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-01T08:39:22.443 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-01T08:39:22.453 INFO:teuthology.task.internal:roles: ubuntu@smithi138.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-10-01T08:39:22.455 INFO:teuthology.task.internal:roles: ubuntu@smithi190.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-10-01T08:39:22.455 INFO:teuthology.run_tasks:Running task console_log... 2023-10-01T08:39:22.553 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f0a7ee47af0>, signals=[15]) 2023-10-01T08:39:22.554 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-01T08:39:22.565 INFO:teuthology.task.internal:Opening connections... 2023-10-01T08:39:22.565 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi138.front.sepia.ceph.com 2023-10-01T08:39:22.567 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T08:39:22.681 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi190.front.sepia.ceph.com 2023-10-01T08:39:22.682 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi190.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T08:39:22.760 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-01T08:39:22.770 DEBUG:teuthology.orchestra.run.smithi138:> uname -m 2023-10-01T08:39:22.795 INFO:teuthology.orchestra.run.smithi138.stdout:x86_64 2023-10-01T08:39:22.795 DEBUG:teuthology.orchestra.run.smithi138:> cat /etc/os-release 2023-10-01T08:39:22.859 INFO:teuthology.orchestra.run.smithi138.stdout:NAME="Red Hat Enterprise Linux" 2023-10-01T08:39:22.859 INFO:teuthology.orchestra.run.smithi138.stdout:VERSION="8.6 (Ootpa)" 2023-10-01T08:39:22.859 INFO:teuthology.orchestra.run.smithi138.stdout:ID="rhel" 2023-10-01T08:39:22.859 INFO:teuthology.orchestra.run.smithi138.stdout:ID_LIKE="fedora" 2023-10-01T08:39:22.859 INFO:teuthology.orchestra.run.smithi138.stdout:VERSION_ID="8.6" 2023-10-01T08:39:22.860 INFO:teuthology.orchestra.run.smithi138.stdout:PLATFORM_ID="platform:el8" 2023-10-01T08:39:22.860 INFO:teuthology.orchestra.run.smithi138.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-10-01T08:39:22.860 INFO:teuthology.orchestra.run.smithi138.stdout:ANSI_COLOR="0;31" 2023-10-01T08:39:22.860 INFO:teuthology.orchestra.run.smithi138.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-10-01T08:39:22.860 INFO:teuthology.orchestra.run.smithi138.stdout:HOME_URL="https://www.redhat.com/" 2023-10-01T08:39:22.860 INFO:teuthology.orchestra.run.smithi138.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-10-01T08:39:22.860 INFO:teuthology.orchestra.run.smithi138.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-01T08:39:22.860 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:39:22.860 INFO:teuthology.orchestra.run.smithi138.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-01T08:39:22.860 INFO:teuthology.orchestra.run.smithi138.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-10-01T08:39:22.860 INFO:teuthology.orchestra.run.smithi138.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-10-01T08:39:22.861 INFO:teuthology.orchestra.run.smithi138.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-10-01T08:39:22.861 INFO:teuthology.lock.ops:Updating smithi138.front.sepia.ceph.com on lock server 2023-10-01T08:39:22.887 DEBUG:teuthology.orchestra.run.smithi190:> uname -m 2023-10-01T08:39:22.911 INFO:teuthology.orchestra.run.smithi190.stdout:x86_64 2023-10-01T08:39:22.912 DEBUG:teuthology.orchestra.run.smithi190:> cat /etc/os-release 2023-10-01T08:39:22.973 INFO:teuthology.orchestra.run.smithi190.stdout:NAME="Red Hat Enterprise Linux" 2023-10-01T08:39:22.973 INFO:teuthology.orchestra.run.smithi190.stdout:VERSION="8.6 (Ootpa)" 2023-10-01T08:39:22.974 INFO:teuthology.orchestra.run.smithi190.stdout:ID="rhel" 2023-10-01T08:39:22.974 INFO:teuthology.orchestra.run.smithi190.stdout:ID_LIKE="fedora" 2023-10-01T08:39:22.974 INFO:teuthology.orchestra.run.smithi190.stdout:VERSION_ID="8.6" 2023-10-01T08:39:22.974 INFO:teuthology.orchestra.run.smithi190.stdout:PLATFORM_ID="platform:el8" 2023-10-01T08:39:22.974 INFO:teuthology.orchestra.run.smithi190.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-10-01T08:39:22.974 INFO:teuthology.orchestra.run.smithi190.stdout:ANSI_COLOR="0;31" 2023-10-01T08:39:22.975 INFO:teuthology.orchestra.run.smithi190.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-10-01T08:39:22.975 INFO:teuthology.orchestra.run.smithi190.stdout:HOME_URL="https://www.redhat.com/" 2023-10-01T08:39:22.975 INFO:teuthology.orchestra.run.smithi190.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-10-01T08:39:22.975 INFO:teuthology.orchestra.run.smithi190.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-10-01T08:39:22.975 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:39:22.975 INFO:teuthology.orchestra.run.smithi190.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-10-01T08:39:22.976 INFO:teuthology.orchestra.run.smithi190.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-10-01T08:39:22.976 INFO:teuthology.orchestra.run.smithi190.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-10-01T08:39:22.977 INFO:teuthology.orchestra.run.smithi190.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-10-01T08:39:22.977 INFO:teuthology.lock.ops:Updating smithi190.front.sepia.ceph.com on lock server 2023-10-01T08:39:22.995 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-01T08:39:23.005 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-01T08:39:23.012 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-01T08:39:23.013 DEBUG:teuthology.orchestra.run.smithi138:> test '!' -e /home/ubuntu/cephtest 2023-10-01T08:39:23.018 DEBUG:teuthology.orchestra.run.smithi190:> test '!' -e /home/ubuntu/cephtest 2023-10-01T08:39:23.037 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-01T08:39:23.045 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-01T08:39:23.045 DEBUG:teuthology.orchestra.run.smithi138:> test -z $(ls -A /var/lib/ceph) 2023-10-01T08:39:23.076 DEBUG:teuthology.orchestra.run.smithi190:> test -z $(ls -A /var/lib/ceph) 2023-10-01T08:39:23.110 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-01T08:39:23.165 INFO:teuthology.run_tasks:Running task kernel... 2023-10-01T08:39:23.182 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-01T08:39:23.182 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-10-01T08:39:23.182 DEBUG:teuthology.orchestra.run.smithi138:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-01T08:39:23.183 DEBUG:teuthology.orchestra.run.smithi190:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-01T08:39:23.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T08:39:23.202 DEBUG:teuthology.orchestra.run.smithi138:> uname -r 2023-10-01T08:39:23.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T08:39:23.204 DEBUG:teuthology.orchestra.run.smithi190:> uname -r 2023-10-01T08:39:23.262 INFO:teuthology.orchestra.run.smithi138.stdout:4.18.0-372.9.1.el8.x86_64 2023-10-01T08:39:23.262 INFO:teuthology.task.kernel:Running kernel on smithi138: 4.18.0-372.9.1.el8.x86_64 2023-10-01T08:39:23.262 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum install -y kernel 2023-10-01T08:39:23.263 INFO:teuthology.orchestra.run.smithi190.stdout:4.18.0-372.9.1.el8.x86_64 2023-10-01T08:39:23.263 INFO:teuthology.task.kernel:Running kernel on smithi190: 4.18.0-372.9.1.el8.x86_64 2023-10-01T08:39:23.264 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum install -y kernel 2023-10-01T08:39:24.394 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:39:24.394 INFO:teuthology.orchestra.run.smithi190.stdout:Unable to read consumer identity 2023-10-01T08:39:24.534 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:39:24.535 INFO:teuthology.orchestra.run.smithi138.stdout:Unable to read consumer identity 2023-10-01T08:39:25.978 INFO:teuthology.orchestra.run.smithi190.stdout:Copr repo for python3-asyncssh owned by ceph 12 kB/s | 3.5 kB 00:00 2023-10-01T08:39:26.061 INFO:teuthology.orchestra.run.smithi138.stdout:Copr repo for python3-asyncssh owned by ceph 19 kB/s | 3.5 kB 00:00 2023-10-01T08:39:27.204 INFO:teuthology.orchestra.run.smithi190.stdout:Extra Packages for Enterprise Linux 13 MB/s | 16 MB 00:01 2023-10-01T08:39:28.003 INFO:teuthology.orchestra.run.smithi138.stdout:Extra Packages for Enterprise Linux 8.3 MB/s | 16 MB 00:01 2023-10-01T08:39:30.729 INFO:teuthology.orchestra.run.smithi190.stdout:lab-extras 360 kB/s | 24 kB 00:00 2023-10-01T08:39:31.423 INFO:teuthology.orchestra.run.smithi190.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-10-01T08:39:31.470 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:39:31.471 INFO:teuthology.orchestra.run.smithi190.stdout:Nothing to do. 2023-10-01T08:39:31.471 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:39:31.561 INFO:teuthology.orchestra.run.smithi138.stdout:lab-extras 459 kB/s | 24 kB 00:00 2023-10-01T08:39:31.587 DEBUG:teuthology.orchestra.run.smithi190:> echo no | sudo yum reinstall kernel || true 2023-10-01T08:39:31.959 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:39:31.959 INFO:teuthology.orchestra.run.smithi190.stdout:Unable to read consumer identity 2023-10-01T08:39:32.221 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:00:02 ago on Sun 01 Oct 2023 08:39:30 AM UTC. 2023-10-01T08:39:32.263 INFO:teuthology.orchestra.run.smithi190.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-01T08:39:32.264 INFO:teuthology.orchestra.run.smithi138.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-10-01T08:39:32.312 INFO:teuthology.orchestra.run.smithi190.stderr:Error: No packages marked for reinstall. 2023-10-01T08:39:32.323 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:39:32.323 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-10-01T08:39:32.323 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:39:32.359 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum reinstall -y kernel || true 2023-10-01T08:39:32.480 DEBUG:teuthology.orchestra.run.smithi138:> echo no | sudo yum reinstall kernel || true 2023-10-01T08:39:32.689 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:39:32.689 INFO:teuthology.orchestra.run.smithi190.stdout:Unable to read consumer identity 2023-10-01T08:39:32.813 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:39:32.813 INFO:teuthology.orchestra.run.smithi138.stdout:Unable to read consumer identity 2023-10-01T08:39:32.945 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:00:02 ago on Sun 01 Oct 2023 08:39:30 AM UTC. 2023-10-01T08:39:32.984 INFO:teuthology.orchestra.run.smithi190.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-01T08:39:33.017 INFO:teuthology.orchestra.run.smithi190.stderr:Error: No packages marked for reinstall. 2023-10-01T08:39:33.063 DEBUG:teuthology.orchestra.run.smithi190:> rpm -q kernel | sort -rV | head -n 1 2023-10-01T08:39:33.076 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:00:02 ago on Sun 01 Oct 2023 08:39:31 AM UTC. 2023-10-01T08:39:33.117 INFO:teuthology.orchestra.run.smithi138.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-01T08:39:33.144 INFO:teuthology.orchestra.run.smithi190.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-10-01T08:39:33.144 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-10-01T08:39:33.144 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-10-01T08:39:33.145 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-10-01T08:39:33.145 DEBUG:teuthology.orchestra.run.smithi190:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-01T08:39:33.170 INFO:teuthology.orchestra.run.smithi138.stderr:Error: No packages marked for reinstall. 2023-10-01T08:39:33.180 INFO:teuthology.orchestra.run.smithi190.stdout:ttyS1 2023-10-01T08:39:33.199 DEBUG:teuthology.parallel:result is None 2023-10-01T08:39:33.219 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum reinstall -y kernel || true 2023-10-01T08:39:33.554 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:39:33.555 INFO:teuthology.orchestra.run.smithi138.stdout:Unable to read consumer identity 2023-10-01T08:39:33.809 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:00:02 ago on Sun 01 Oct 2023 08:39:31 AM UTC. 2023-10-01T08:39:33.851 INFO:teuthology.orchestra.run.smithi138.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-10-01T08:39:33.880 INFO:teuthology.orchestra.run.smithi138.stderr:Error: No packages marked for reinstall. 2023-10-01T08:39:33.932 DEBUG:teuthology.orchestra.run.smithi138:> rpm -q kernel | sort -rV | head -n 1 2023-10-01T08:39:34.012 INFO:teuthology.orchestra.run.smithi138.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-10-01T08:39:34.012 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-10-01T08:39:34.013 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-10-01T08:39:34.013 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-10-01T08:39:34.013 DEBUG:teuthology.orchestra.run.smithi138:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-01T08:39:34.048 INFO:teuthology.orchestra.run.smithi138.stdout:ttyS1 2023-10-01T08:39:34.060 DEBUG:teuthology.parallel:result is None 2023-10-01T08:39:34.060 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-01T08:39:34.067 INFO:teuthology.task.internal:Creating test directory... 2023-10-01T08:39:34.068 DEBUG:teuthology.orchestra.run.smithi138:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-01T08:39:34.102 DEBUG:teuthology.orchestra.run.smithi190:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-01T08:39:34.122 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-01T08:39:34.128 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-01T08:39:34.137 INFO:teuthology.task.internal:Creating archive directory... 2023-10-01T08:39:34.137 DEBUG:teuthology.orchestra.run.smithi138:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-01T08:39:34.159 DEBUG:teuthology.orchestra.run.smithi190:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-01T08:39:34.187 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-01T08:39:34.195 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-01T08:39:34.195 DEBUG:teuthology.orchestra.run.smithi138:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-01T08:39:34.216 DEBUG:teuthology.orchestra.run.smithi190:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-01T08:39:34.240 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-01T08:39:34.250 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-01T08:39:34.273 INFO:teuthology.orchestra.run.smithi190.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-01T08:39:34.284 INFO:teuthology.orchestra.run.smithi190.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-01T08:39:34.285 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-01T08:39:34.295 INFO:teuthology.task.internal:Configuring sudo... 2023-10-01T08:39:34.295 DEBUG:teuthology.orchestra.run.smithi138:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-01T08:39:34.297 DEBUG:teuthology.orchestra.run.smithi190:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-01T08:39:34.325 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-01T08:39:34.336 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-01T08:39:34.336 DEBUG:teuthology.orchestra.run.smithi138:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-01T08:39:34.364 DEBUG:teuthology.orchestra.run.smithi190:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-01T08:39:34.382 DEBUG:teuthology.orchestra.run.smithi138:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-01T08:39:34.462 DEBUG:teuthology.orchestra.run.smithi138:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-01T08:39:34.516 DEBUG:teuthology.orchestra.run.smithi138:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-01T08:39:34.602 DEBUG:teuthology.orchestra.run.smithi138:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-01T08:39:34.642 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T08:39:34.642 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-01T08:39:34.710 DEBUG:teuthology.orchestra.run.smithi190:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-01T08:39:34.755 DEBUG:teuthology.orchestra.run.smithi190:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-01T08:39:34.807 DEBUG:teuthology.orchestra.run.smithi190:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-01T08:39:34.892 DEBUG:teuthology.orchestra.run.smithi190:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-01T08:39:34.934 DEBUG:teuthology.orchestra.run.smithi190:> set -ex 2023-10-01T08:39:34.934 DEBUG:teuthology.orchestra.run.smithi190:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-01T08:39:35.001 DEBUG:teuthology.orchestra.run.smithi138:> sudo service rsyslog restart 2023-10-01T08:39:35.003 DEBUG:teuthology.orchestra.run.smithi190:> sudo service rsyslog restart 2023-10-01T08:39:35.057 INFO:teuthology.orchestra.run.smithi138.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-01T08:39:35.104 INFO:teuthology.orchestra.run.smithi190.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-01T08:39:35.569 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-01T08:39:35.578 INFO:teuthology.task.internal:Starting timer... 2023-10-01T08:39:35.579 INFO:teuthology.run_tasks:Running task pcp... 2023-10-01T08:39:35.597 INFO:teuthology.run_tasks:Running task selinux... 2023-10-01T08:39:35.610 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-10-01T08:39:35.611 DEBUG:teuthology.orchestra.run.smithi138:> sudo service auditd rotate 2023-10-01T08:39:35.672 INFO:teuthology.orchestra.run.smithi138.stdout:Rotating logs: 2023-10-01T08:39:35.673 DEBUG:teuthology.orchestra.run.smithi190:> sudo service auditd rotate 2023-10-01T08:39:35.726 INFO:teuthology.orchestra.run.smithi190.stdout:Rotating logs: 2023-10-01T08:39:35.727 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-01T08:39:35.727 DEBUG:teuthology.orchestra.run.smithi138:> /usr/sbin/getenforce 2023-10-01T08:39:35.745 INFO:teuthology.orchestra.run.smithi138.stdout:Permissive 2023-10-01T08:39:35.745 DEBUG:teuthology.orchestra.run.smithi190:> /usr/sbin/getenforce 2023-10-01T08:39:35.763 INFO:teuthology.orchestra.run.smithi190.stdout:Permissive 2023-10-01T08:39:35.763 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi138.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi190.front.sepia.ceph.com': 'permissive'} 2023-10-01T08:39:35.764 DEBUG:teuthology.orchestra.run.smithi138:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-01T08:39:35.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T08:39:35.812 DEBUG:teuthology.orchestra.run.smithi190:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-01T08:39:35.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T08:39:35.839 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-01T08:39:35.839 DEBUG:teuthology.orchestra.run.smithi138:> sudo /usr/sbin/setenforce permissive 2023-10-01T08:39:35.879 DEBUG:teuthology.orchestra.run.smithi190:> sudo /usr/sbin/setenforce permissive 2023-10-01T08:39:35.907 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-01T08:39:35.916 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-10-01T08:39:35.921 INFO:teuthology.repo_utils:Fetching main from origin 2023-10-01T08:39:36.037 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-01T08:39:36.052 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-01T08:39:36.053 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi138.front.sepia.ceph.com,smithi190.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-01T08:48:45.153 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi138.front.sepia.ceph.com'), Remote(name='ubuntu@smithi190.front.sepia.ceph.com')] 2023-10-01T08:48:45.154 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi138.front.sepia.ceph.com' 2023-10-01T08:48:45.155 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T08:48:45.223 DEBUG:teuthology.orchestra.run.smithi138:> true 2023-10-01T08:48:45.296 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi138.front.sepia.ceph.com' 2023-10-01T08:48:45.297 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi190.front.sepia.ceph.com' 2023-10-01T08:48:45.297 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi190.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T08:48:45.365 DEBUG:teuthology.orchestra.run.smithi190:> true 2023-10-01T08:48:45.431 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi190.front.sepia.ceph.com' 2023-10-01T08:48:45.432 INFO:teuthology.run_tasks:Running task clock... 2023-10-01T08:48:45.443 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-01T08:48:45.443 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-01T08:48:45.443 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-01T08:48:45.446 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-01T08:48:45.446 DEBUG:teuthology.orchestra.run.smithi190:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-01T08:48:45.475 INFO:teuthology.orchestra.run.smithi138.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-01T08:48:45.490 INFO:teuthology.orchestra.run.smithi138.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-01T08:48:45.504 INFO:teuthology.orchestra.run.smithi190.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-10-01T08:48:45.519 INFO:teuthology.orchestra.run.smithi190.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-10-01T08:48:45.562 INFO:teuthology.orchestra.run.smithi138.stderr:sudo: ntpd: command not found 2023-10-01T08:48:45.574 INFO:teuthology.orchestra.run.smithi138.stdout:506 Cannot talk to daemon 2023-10-01T08:48:45.588 INFO:teuthology.orchestra.run.smithi138.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-01T08:48:45.598 INFO:teuthology.orchestra.run.smithi190.stderr:sudo: ntpd: command not found 2023-10-01T08:48:45.602 INFO:teuthology.orchestra.run.smithi138.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-01T08:48:45.609 INFO:teuthology.orchestra.run.smithi190.stdout:506 Cannot talk to daemon 2023-10-01T08:48:45.623 INFO:teuthology.orchestra.run.smithi190.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-10-01T08:48:45.638 INFO:teuthology.orchestra.run.smithi190.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-10-01T08:48:45.674 INFO:teuthology.orchestra.run.smithi138.stderr:bash: ntpq: command not found 2023-10-01T08:48:45.677 INFO:teuthology.orchestra.run.smithi138.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-01T08:48:45.677 INFO:teuthology.orchestra.run.smithi138.stdout:=============================================================================== 2023-10-01T08:48:45.677 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T08:48:45.678 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T08:48:45.678 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T08:48:45.678 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T08:48:45.707 INFO:teuthology.orchestra.run.smithi190.stderr:bash: ntpq: command not found 2023-10-01T08:48:45.711 INFO:teuthology.orchestra.run.smithi190.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-01T08:48:45.711 INFO:teuthology.orchestra.run.smithi190.stdout:=============================================================================== 2023-10-01T08:48:45.711 INFO:teuthology.orchestra.run.smithi190.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T08:48:45.711 INFO:teuthology.orchestra.run.smithi190.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T08:48:45.711 INFO:teuthology.orchestra.run.smithi190.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T08:48:45.711 INFO:teuthology.orchestra.run.smithi190.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T08:48:45.712 INFO:teuthology.run_tasks:Running task pexec... 2023-10-01T08:48:45.721 INFO:teuthology.task.pexec:Executing custom commands... 2023-10-01T08:48:45.722 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi138.front.sepia.ceph.com 2023-10-01T08:48:45.722 DEBUG:teuthology.orchestra.run.smithi138:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-01T08:48:45.722 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi190.front.sepia.ceph.com 2023-10-01T08:48:45.722 DEBUG:teuthology.orchestra.run.smithi190:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-10-01T08:48:46.091 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:48:46.123 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:48:47.753 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:42 ago on Sun 01 Oct 2023 08:47:05 AM UTC. 2023-10-01T08:48:47.759 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:01:42 ago on Sun 01 Oct 2023 08:47:05 AM UTC. 2023-10-01T08:48:49.488 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:48:49.489 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:48:49.489 INFO:teuthology.orchestra.run.smithi190.stdout: Package Architecture Version Repository Size 2023-10-01T08:48:49.489 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:48:49.489 INFO:teuthology.orchestra.run.smithi190.stdout:Resetting modules: 2023-10-01T08:48:49.489 INFO:teuthology.orchestra.run.smithi190.stdout: container-tools 2023-10-01T08:48:49.489 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:48:49.489 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:48:49.489 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:48:49.490 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:48:49.630 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:48:49.631 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:48:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: Package Architecture Version Repository Size 2023-10-01T08:48:49.631 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:48:49.631 INFO:teuthology.orchestra.run.smithi138.stdout:Resetting modules: 2023-10-01T08:48:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: container-tools 2023-10-01T08:48:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:48:49.631 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:48:49.631 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:48:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:48:50.447 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:01:45 ago on Sun 01 Oct 2023 08:47:05 AM UTC. 2023-10-01T08:48:50.617 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:45 ago on Sun 01 Oct 2023 08:47:05 AM UTC. 2023-10-01T08:48:59.920 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:49:00.378 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:49:00.819 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:49:01.276 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:49:01.901 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:01:56 ago on Sun 01 Oct 2023 08:47:05 AM UTC. 2023-10-01T08:49:02.811 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:57 ago on Sun 01 Oct 2023 08:47:05 AM UTC. 2023-10-01T08:49:03.571 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:49:03.662 INFO:teuthology.orchestra.run.smithi190.stdout:========================================================================================================================== 2023-10-01T08:49:03.662 INFO:teuthology.orchestra.run.smithi190.stdout: Package Arch Version Repository Size 2023-10-01T08:49:03.662 INFO:teuthology.orchestra.run.smithi190.stdout:========================================================================================================================== 2023-10-01T08:49:03.662 INFO:teuthology.orchestra.run.smithi190.stdout:Installing group/module packages: 2023-10-01T08:49:03.663 INFO:teuthology.orchestra.run.smithi190.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-10-01T08:49:03.663 INFO:teuthology.orchestra.run.smithi190.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-10-01T08:49:03.663 INFO:teuthology.orchestra.run.smithi190.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-10-01T08:49:03.663 INFO:teuthology.orchestra.run.smithi190.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-10-01T08:49:03.663 INFO:teuthology.orchestra.run.smithi190.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-10-01T08:49:03.663 INFO:teuthology.orchestra.run.smithi190.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-01T08:49:03.663 INFO:teuthology.orchestra.run.smithi190.stdout:Installing dependencies: 2023-10-01T08:49:03.664 INFO:teuthology.orchestra.run.smithi190.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-10-01T08:49:03.664 INFO:teuthology.orchestra.run.smithi190.stdout:Downgrading: 2023-10-01T08:49:03.664 INFO:teuthology.orchestra.run.smithi190.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-01T08:49:03.664 INFO:teuthology.orchestra.run.smithi190.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-10-01T08:49:03.664 INFO:teuthology.orchestra.run.smithi190.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-10-01T08:49:03.664 INFO:teuthology.orchestra.run.smithi190.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-10-01T08:49:03.664 INFO:teuthology.orchestra.run.smithi190.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-10-01T08:49:03.665 INFO:teuthology.orchestra.run.smithi190.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-10-01T08:49:03.665 INFO:teuthology.orchestra.run.smithi190.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-10-01T08:49:03.665 INFO:teuthology.orchestra.run.smithi190.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-10-01T08:49:03.665 INFO:teuthology.orchestra.run.smithi190.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-10-01T08:49:03.665 INFO:teuthology.orchestra.run.smithi190.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-10-01T08:49:03.665 INFO:teuthology.orchestra.run.smithi190.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-10-01T08:49:03.665 INFO:teuthology.orchestra.run.smithi190.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-01T08:49:03.665 INFO:teuthology.orchestra.run.smithi190.stdout:Installing module profiles: 2023-10-01T08:49:03.666 INFO:teuthology.orchestra.run.smithi190.stdout: container-tools/common 2023-10-01T08:49:03.666 INFO:teuthology.orchestra.run.smithi190.stdout:Enabling module streams: 2023-10-01T08:49:03.666 INFO:teuthology.orchestra.run.smithi190.stdout: container-tools 3.0 2023-10-01T08:49:03.666 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:49:03.666 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:49:03.666 INFO:teuthology.orchestra.run.smithi190.stdout:========================================================================================================================== 2023-10-01T08:49:03.667 INFO:teuthology.orchestra.run.smithi190.stdout:Install 7 Packages 2023-10-01T08:49:03.667 INFO:teuthology.orchestra.run.smithi190.stdout:Downgrade 12 Packages 2023-10-01T08:49:03.667 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:49:03.667 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 51 M 2023-10-01T08:49:03.668 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:49:03.841 INFO:teuthology.orchestra.run.smithi190.stdout:(1/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 294 kB/s | 51 kB 00:00 2023-10-01T08:49:03.858 INFO:teuthology.orchestra.run.smithi190.stdout:(2/19): container-selinux-2.178.0-2.module+el8. 276 kB/s | 53 kB 00:00 2023-10-01T08:49:03.883 INFO:teuthology.orchestra.run.smithi190.stdout:(3/19): containers-common-1.2.4-1.module+el8.6. 419 kB/s | 91 kB 00:00 2023-10-01T08:49:04.008 INFO:teuthology.orchestra.run.smithi190.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 3.0 MB/s | 511 kB 00:00 2023-10-01T08:49:04.033 INFO:teuthology.orchestra.run.smithi190.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 394 kB/s | 69 kB 00:00 2023-10-01T08:49:04.109 INFO:teuthology.orchestra.run.smithi190.stdout:(6/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 13 MB/s | 2.9 MB 00:00 2023-10-01T08:49:04.134 INFO:teuthology.orchestra.run.smithi190.stdout:(7/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 574 kB/s | 72 kB 00:00 2023-10-01T08:49:04.334 INFO:teuthology.orchestra.run.smithi190.stdout:(8/19): podman-3.0.1-9.module+el8.6.0+14874+644 36 MB/s | 11 MB 00:00 2023-10-01T08:49:04.360 INFO:teuthology.orchestra.run.smithi190.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 226 kB/s | 51 kB 00:00 2023-10-01T08:49:04.385 INFO:teuthology.orchestra.run.smithi190.stdout:(10/19): podman-docker-3.0.1-9.module+el8.6.0+1 203 kB/s | 56 kB 00:00 2023-10-01T08:49:04.497 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:49:04.501 INFO:teuthology.orchestra.run.smithi190.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 349 kB/s | 41 kB 00:00 2023-10-01T08:49:04.527 INFO:teuthology.orchestra.run.smithi190.stdout:(12/19): podman-catatonit-3.0.1-9.module+el8.6. 1.9 MB/s | 322 kB 00:00 2023-10-01T08:49:04.589 INFO:teuthology.orchestra.run.smithi138.stdout:========================================================================================================================== 2023-10-01T08:49:04.589 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-10-01T08:49:04.589 INFO:teuthology.orchestra.run.smithi138.stdout:========================================================================================================================== 2023-10-01T08:49:04.589 INFO:teuthology.orchestra.run.smithi138.stdout:Installing group/module packages: 2023-10-01T08:49:04.589 INFO:teuthology.orchestra.run.smithi138.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-10-01T08:49:04.589 INFO:teuthology.orchestra.run.smithi138.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-10-01T08:49:04.589 INFO:teuthology.orchestra.run.smithi138.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-10-01T08:49:04.590 INFO:teuthology.orchestra.run.smithi138.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-10-01T08:49:04.590 INFO:teuthology.orchestra.run.smithi138.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-10-01T08:49:04.590 INFO:teuthology.orchestra.run.smithi138.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-01T08:49:04.590 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-10-01T08:49:04.590 INFO:teuthology.orchestra.run.smithi138.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-10-01T08:49:04.590 INFO:teuthology.orchestra.run.smithi138.stdout:Downgrading: 2023-10-01T08:49:04.590 INFO:teuthology.orchestra.run.smithi138.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-01T08:49:04.590 INFO:teuthology.orchestra.run.smithi138.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-10-01T08:49:04.590 INFO:teuthology.orchestra.run.smithi138.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-10-01T08:49:04.591 INFO:teuthology.orchestra.run.smithi138.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-10-01T08:49:04.591 INFO:teuthology.orchestra.run.smithi138.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-10-01T08:49:04.591 INFO:teuthology.orchestra.run.smithi138.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-10-01T08:49:04.591 INFO:teuthology.orchestra.run.smithi138.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-10-01T08:49:04.591 INFO:teuthology.orchestra.run.smithi138.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-10-01T08:49:04.591 INFO:teuthology.orchestra.run.smithi138.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-10-01T08:49:04.591 INFO:teuthology.orchestra.run.smithi138.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-10-01T08:49:04.591 INFO:teuthology.orchestra.run.smithi138.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-10-01T08:49:04.591 INFO:teuthology.orchestra.run.smithi138.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-10-01T08:49:04.591 INFO:teuthology.orchestra.run.smithi138.stdout:Installing module profiles: 2023-10-01T08:49:04.592 INFO:teuthology.orchestra.run.smithi138.stdout: container-tools/common 2023-10-01T08:49:04.592 INFO:teuthology.orchestra.run.smithi138.stdout:Enabling module streams: 2023-10-01T08:49:04.592 INFO:teuthology.orchestra.run.smithi138.stdout: container-tools 3.0 2023-10-01T08:49:04.592 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:49:04.592 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:49:04.592 INFO:teuthology.orchestra.run.smithi138.stdout:========================================================================================================================== 2023-10-01T08:49:04.592 INFO:teuthology.orchestra.run.smithi138.stdout:Install 7 Packages 2023-10-01T08:49:04.592 INFO:teuthology.orchestra.run.smithi138.stdout:Downgrade 12 Packages 2023-10-01T08:49:04.592 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:49:04.593 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 51 M 2023-10-01T08:49:04.593 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:49:04.702 INFO:teuthology.orchestra.run.smithi190.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 11 MB/s | 2.2 MB 00:00 2023-10-01T08:49:04.726 INFO:teuthology.orchestra.run.smithi138.stdout:(1/19): containers-common-1.2.4-1.module+el8.6. 681 kB/s | 91 kB 00:00 2023-10-01T08:49:04.760 INFO:teuthology.orchestra.run.smithi138.stdout:(2/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 307 kB/s | 51 kB 00:00 2023-10-01T08:49:04.794 INFO:teuthology.orchestra.run.smithi138.stdout:(3/19): container-selinux-2.178.0-2.module+el8. 263 kB/s | 53 kB 00:00 2023-10-01T08:49:04.861 INFO:teuthology.orchestra.run.smithi138.stdout:(4/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 689 kB/s | 69 kB 00:00 2023-10-01T08:49:04.902 INFO:teuthology.orchestra.run.smithi138.stdout:(5/19): criu-3.15-1.module+el8.6.0+14874+644362 2.8 MB/s | 511 kB 00:00 2023-10-01T08:49:04.903 INFO:teuthology.orchestra.run.smithi190.stdout:(14/19): containernetworking-plugins-0.9.1-1.mo 33 MB/s | 19 MB 00:00 2023-10-01T08:49:04.993 INFO:teuthology.orchestra.run.smithi138.stdout:(6/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 15 MB/s | 2.9 MB 00:00 2023-10-01T08:49:05.011 INFO:teuthology.orchestra.run.smithi138.stdout:(7/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 478 kB/s | 72 kB 00:00 2023-10-01T08:49:05.045 INFO:teuthology.orchestra.run.smithi190.stdout:(15/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 12 MB/s | 6.4 MB 00:00 2023-10-01T08:49:05.086 INFO:teuthology.orchestra.run.smithi190.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 5.8 MB/s | 1.1 MB 00:00 2023-10-01T08:49:05.087 INFO:teuthology.orchestra.run.smithi138.stdout:(8/19): conmon-2.0.26-1.module+el8.6.0+14874+64 678 kB/s | 51 kB 00:00 2023-10-01T08:49:05.111 INFO:teuthology.orchestra.run.smithi190.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 2.7 MB/s | 185 kB 00:00 2023-10-01T08:49:05.129 INFO:teuthology.orchestra.run.smithi138.stdout:(9/19): podman-docker-3.0.1-9.module+el8.6.0+14 411 kB/s | 56 kB 00:00 2023-10-01T08:49:05.139 INFO:teuthology.orchestra.run.smithi190.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 1.0 MB/s | 51 kB 00:00 2023-10-01T08:49:05.221 INFO:teuthology.orchestra.run.smithi190.stdout:(19/19): buildah-1.19.9-3.module+el8.6.0+14874+ 13 MB/s | 6.8 MB 00:00 2023-10-01T08:49:05.221 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:49:05.221 INFO:teuthology.orchestra.run.smithi190.stdout:Total 33 MB/s | 51 MB 00:01 2023-10-01T08:49:05.346 INFO:teuthology.orchestra.run.smithi138.stdout:(10/19): podman-3.0.1-9.module+el8.6.0+14874+64 25 MB/s | 11 MB 00:00 2023-10-01T08:49:05.371 INFO:teuthology.orchestra.run.smithi138.stdout:(11/19): podman-catatonit-3.0.1-9.module+el8.6. 1.3 MB/s | 322 kB 00:00 2023-10-01T08:49:05.692 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:49:05.696 INFO:teuthology.orchestra.run.smithi138.stdout:(12/19): containernetworking-plugins-0.9.1-1.mo 31 MB/s | 19 MB 00:00 2023-10-01T08:49:05.713 INFO:teuthology.orchestra.run.smithi138.stdout:(13/19): yajl-2.1.0-10.el8.x86_64.rpm 111 kB/s | 41 kB 00:00 2023-10-01T08:49:05.734 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:49:05.734 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:49:05.788 INFO:teuthology.orchestra.run.smithi138.stdout:(14/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 5.3 MB/s | 2.2 MB 00:00 2023-10-01T08:49:05.964 INFO:teuthology.orchestra.run.smithi138.stdout:(15/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 24 MB/s | 6.4 MB 00:00 2023-10-01T08:49:06.081 INFO:teuthology.orchestra.run.smithi138.stdout:(16/19): buildah-1.19.9-3.module+el8.6.0+14874+ 18 MB/s | 6.8 MB 00:00 2023-10-01T08:49:06.123 INFO:teuthology.orchestra.run.smithi138.stdout:(17/19): cockpit-podman-29-2.module+el8.6.0+148 3.2 MB/s | 1.1 MB 00:00 2023-10-01T08:49:06.148 INFO:teuthology.orchestra.run.smithi138.stdout:(18/19): crun-0.18-3.module+el8.6.0+14874+64436 1.0 MB/s | 185 kB 00:00 2023-10-01T08:49:06.168 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:49:06.172 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:49:06.281 INFO:teuthology.orchestra.run.smithi138.stdout:(19/19): udica-0.2.4-1.module+el8.6.0+14874+644 252 kB/s | 51 kB 00:00 2023-10-01T08:49:06.282 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:49:06.282 INFO:teuthology.orchestra.run.smithi138.stdout:Total 30 MB/s | 51 MB 00:01 2023-10-01T08:49:06.762 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:49:06.804 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:49:06.805 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:49:07.214 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:49:07.218 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:49:07.328 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:49:07.382 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-10-01T08:49:07.565 INFO:teuthology.orchestra.run.smithi190.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-01T08:49:07.573 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-01T08:49:07.597 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-01T08:49:07.773 INFO:teuthology.orchestra.run.smithi190.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-01T08:49:08.438 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:49:08.493 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-10-01T08:49:08.722 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-01T08:49:08.731 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-10-01T08:49:08.754 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-01T08:49:08.946 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-01T08:49:24.919 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-01T08:49:26.118 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-10-01T08:49:26.674 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-10-01T08:49:26.886 INFO:teuthology.orchestra.run.smithi190.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-10-01T08:49:27.083 INFO:teuthology.orchestra.run.smithi190.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-10-01T08:49:27.243 INFO:teuthology.orchestra.run.smithi190.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-10-01T08:49:27.463 INFO:teuthology.orchestra.run.smithi190.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-10-01T08:49:27.463 INFO:teuthology.orchestra.run.smithi190.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-10-01T08:49:27.464 INFO:teuthology.orchestra.run.smithi190.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-10-01T08:49:27.464 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:49:27.895 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-10-01T08:49:28.163 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-10-01T08:49:28.273 INFO:teuthology.orchestra.run.smithi190.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-10-01T08:49:28.401 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-10-01T08:49:28.453 INFO:teuthology.orchestra.run.smithi190.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-10-01T08:49:28.625 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-10-01T08:49:28.843 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-10-01T08:49:28.843 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-10-01T08:49:28.844 INFO:teuthology.orchestra.run.smithi138.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-10-01T08:49:28.844 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:49:29.716 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-10-01T08:49:29.923 INFO:teuthology.orchestra.run.smithi190.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-10-01T08:49:30.001 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-10-01T08:49:30.205 INFO:teuthology.orchestra.run.smithi190.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-10-01T08:49:30.603 INFO:teuthology.orchestra.run.smithi190.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-10-01T08:49:30.869 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-10-01T08:49:31.473 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-10-01T08:49:31.604 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-10-01T08:49:31.804 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-10-01T08:49:32.201 INFO:teuthology.orchestra.run.smithi138.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-10-01T08:49:32.326 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-10-01T08:49:32.509 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-10-01T08:49:32.560 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-10-01T08:49:32.752 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-10-01T08:49:32.920 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-10-01T08:49:33.076 INFO:teuthology.orchestra.run.smithi190.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-10-01T08:49:33.076 INFO:teuthology.orchestra.run.smithi190.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-01T08:49:33.268 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-10-01T08:49:33.380 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-01T08:49:33.554 INFO:teuthology.orchestra.run.smithi190.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-10-01T08:49:33.555 INFO:teuthology.orchestra.run.smithi190.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-01T08:49:33.720 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-01T08:49:33.861 INFO:teuthology.orchestra.run.smithi190.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-01T08:49:34.012 INFO:teuthology.orchestra.run.smithi190.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-10-01T08:49:34.023 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-10-01T08:49:34.162 INFO:teuthology.orchestra.run.smithi190.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-10-01T08:49:34.259 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-10-01T08:49:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-10-01T08:49:34.445 INFO:teuthology.orchestra.run.smithi190.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-10-01T08:49:34.491 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-10-01T08:49:34.578 INFO:teuthology.orchestra.run.smithi190.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-10-01T08:49:34.700 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-10-01T08:49:34.721 INFO:teuthology.orchestra.run.smithi190.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-10-01T08:49:34.721 INFO:teuthology.orchestra.run.smithi190.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-01T08:49:34.832 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-10-01T08:49:34.833 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-01T08:49:34.904 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-01T08:49:35.187 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-10-01T08:49:35.393 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-10-01T08:49:35.394 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-01T08:49:35.423 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-10-01T08:49:35.552 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-10-01T08:49:35.718 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-01T08:49:35.821 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-01T08:49:35.821 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-10-01T08:49:35.822 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-10-01T08:49:35.822 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-10-01T08:49:35.822 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-10-01T08:49:35.822 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-10-01T08:49:35.822 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-10-01T08:49:35.822 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-10-01T08:49:35.822 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-10-01T08:49:35.822 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-10-01T08:49:35.824 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-10-01T08:49:35.824 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-10-01T08:49:35.824 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-10-01T08:49:35.825 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-10-01T08:49:35.825 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-10-01T08:49:35.825 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-10-01T08:49:35.825 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-10-01T08:49:35.825 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-10-01T08:49:35.825 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-10-01T08:49:35.825 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-10-01T08:49:35.825 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-10-01T08:49:35.825 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-10-01T08:49:35.825 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-10-01T08:49:35.825 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-10-01T08:49:35.825 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-01T08:49:35.826 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-10-01T08:49:35.826 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-10-01T08:49:35.826 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-10-01T08:49:35.826 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-10-01T08:49:35.826 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-10-01T08:49:35.826 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-10-01T08:49:35.869 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-10-01T08:49:36.043 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-10-01T08:49:36.210 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-10-01T08:49:36.377 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-10-01T08:49:36.518 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-10-01T08:49:36.717 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-10-01T08:49:36.718 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-01T08:49:36.909 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-01T08:49:37.224 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-10-01T08:49:37.224 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:02:31 ago on Sun 01 Oct 2023 08:47:05 AM UTC. 2023-10-01T08:49:37.443 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-10-01T08:49:37.825 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-10-01T08:49:37.825 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-10-01T08:49:37.825 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-10-01T08:49:37.825 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-10-01T08:49:37.825 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-10-01T08:49:37.825 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-10-01T08:49:37.825 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-10-01T08:49:37.826 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-10-01T08:49:37.826 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-10-01T08:49:37.826 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-10-01T08:49:37.826 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-10-01T08:49:37.826 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-10-01T08:49:37.826 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-10-01T08:49:37.826 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-10-01T08:49:37.826 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-10-01T08:49:37.826 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-10-01T08:49:37.826 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-10-01T08:49:37.827 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-10-01T08:49:37.827 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-10-01T08:49:37.827 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-10-01T08:49:37.827 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-10-01T08:49:37.827 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-10-01T08:49:37.827 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-10-01T08:49:37.827 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-10-01T08:49:37.827 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-10-01T08:49:37.827 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-10-01T08:49:37.827 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-10-01T08:49:37.828 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-10-01T08:49:37.829 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-10-01T08:49:37.829 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-10-01T08:49:37.830 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-10-01T08:49:39.276 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-10-01T08:49:39.276 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:33 ago on Sun 01 Oct 2023 08:47:05 AM UTC. 2023-10-01T08:49:44.952 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:49:45.144 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:49:45.144 INFO:teuthology.orchestra.run.smithi190.stdout:Downgraded: 2023-10-01T08:49:45.145 INFO:teuthology.orchestra.run.smithi190.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:45.145 INFO:teuthology.orchestra.run.smithi190.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-10-01T08:49:45.145 INFO:teuthology.orchestra.run.smithi190.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:45.145 INFO:teuthology.orchestra.run.smithi190.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:45.145 INFO:teuthology.orchestra.run.smithi190.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:45.145 INFO:teuthology.orchestra.run.smithi190.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:45.145 INFO:teuthology.orchestra.run.smithi190.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:45.145 INFO:teuthology.orchestra.run.smithi190.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:45.145 INFO:teuthology.orchestra.run.smithi190.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:45.145 INFO:teuthology.orchestra.run.smithi190.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-10-01T08:49:45.146 INFO:teuthology.orchestra.run.smithi190.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:45.146 INFO:teuthology.orchestra.run.smithi190.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:45.146 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:49:45.146 INFO:teuthology.orchestra.run.smithi190.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:45.146 INFO:teuthology.orchestra.run.smithi190.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-10-01T08:49:45.146 INFO:teuthology.orchestra.run.smithi190.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:45.146 INFO:teuthology.orchestra.run.smithi190.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:45.146 INFO:teuthology.orchestra.run.smithi190.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:45.146 INFO:teuthology.orchestra.run.smithi190.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-10-01T08:49:45.146 INFO:teuthology.orchestra.run.smithi190.stdout: yajl-2.1.0-10.el8.x86_64 2023-10-01T08:49:45.146 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:49:45.147 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:49:45.341 DEBUG:teuthology.parallel:result is None 2023-10-01T08:49:47.428 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:49:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:49:47.616 INFO:teuthology.orchestra.run.smithi138.stdout:Downgraded: 2023-10-01T08:49:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-10-01T08:49:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-10-01T08:49:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:47.617 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:49:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-10-01T08:49:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-10-01T08:49:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-10-01T08:49:47.618 INFO:teuthology.orchestra.run.smithi138.stdout: yajl-2.1.0-10.el8.x86_64 2023-10-01T08:49:47.618 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:49:47.618 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:49:47.814 DEBUG:teuthology.parallel:result is None 2023-10-01T08:49:47.814 INFO:teuthology.run_tasks:Running task install... 2023-10-01T08:49:47.825 DEBUG:teuthology.task.install:project ceph 2023-10-01T08:49:47.826 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b56822191ad0416d0c05ed59720df03ad9c2a44e'}} 2023-10-01T08:49:47.826 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b56822191ad0416d0c05ed59720df03ad9c2a44e'} 2023-10-01T08:49:47.826 INFO:teuthology.task.install:Using flavor: default 2023-10-01T08:49:47.836 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-10-01T08:49:47.837 INFO:teuthology.task.install:extra packages: [] 2023-10-01T08:49:47.837 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'pacific', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b56822191ad0416d0c05ed59720df03ad9c2a44e', 'tag': None, 'wait_for_package': False} 2023-10-01T08:49:47.837 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-01T08:49:47.837 INFO:teuthology.packaging:ref: None 2023-10-01T08:49:47.838 INFO:teuthology.packaging:tag: None 2023-10-01T08:49:47.838 INFO:teuthology.packaging:branch: pacific 2023-10-01T08:49:47.838 INFO:teuthology.packaging:sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T08:49:47.838 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-10-01T08:49:47.840 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'pacific', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b56822191ad0416d0c05ed59720df03ad9c2a44e', 'tag': None, 'wait_for_package': False} 2023-10-01T08:49:47.840 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-01T08:49:47.840 INFO:teuthology.packaging:ref: None 2023-10-01T08:49:47.840 INFO:teuthology.packaging:tag: None 2023-10-01T08:49:47.840 INFO:teuthology.packaging:branch: pacific 2023-10-01T08:49:47.840 INFO:teuthology.packaging:sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T08:49:47.840 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-10-01T08:49:48.070 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/centos/8/flavors/default/ 2023-10-01T08:49:48.070 INFO:teuthology.task.install.rpm:Package version is 16.2.14-44.geeb13ac9 2023-10-01T08:49:48.152 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/centos/8/flavors/default/ 2023-10-01T08:49:48.152 INFO:teuthology.task.install.rpm:Package version is 16.2.14-44.geeb13ac9 2023-10-01T08:49:48.190 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-01T08:49:48.191 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T08:49:48.191 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-01T08:49:48.225 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-01T08:49:48.225 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-01T08:49:48.225 INFO:teuthology.packaging:ref: None 2023-10-01T08:49:48.225 INFO:teuthology.packaging:tag: None 2023-10-01T08:49:48.225 INFO:teuthology.packaging:branch: pacific 2023-10-01T08:49:48.225 INFO:teuthology.packaging:sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T08:49:48.225 DEBUG:teuthology.orchestra.run.smithi138:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/pacific/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-01T08:49:48.293 DEBUG:teuthology.orchestra.run.smithi138:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-01T08:49:48.321 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-10-01T08:49:48.321 DEBUG:teuthology.orchestra.run.smithi190:> set -ex 2023-10-01T08:49:48.322 DEBUG:teuthology.orchestra.run.smithi190:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-10-01T08:49:48.354 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-10-01T08:49:48.354 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-01T08:49:48.354 INFO:teuthology.packaging:ref: None 2023-10-01T08:49:48.354 INFO:teuthology.packaging:tag: None 2023-10-01T08:49:48.354 INFO:teuthology.packaging:branch: pacific 2023-10-01T08:49:48.354 INFO:teuthology.packaging:sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T08:49:48.355 DEBUG:teuthology.orchestra.run.smithi190:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/pacific/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-10-01T08:49:48.370 DEBUG:teuthology.orchestra.run.smithi138:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-01T08:49:48.425 DEBUG:teuthology.orchestra.run.smithi190:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-10-01T08:49:48.435 INFO:teuthology.orchestra.run.smithi138.stdout:check_obsoletes = 1 2023-10-01T08:49:48.437 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum clean all 2023-10-01T08:49:48.505 DEBUG:teuthology.orchestra.run.smithi190:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-10-01T08:49:48.571 INFO:teuthology.orchestra.run.smithi190.stdout:check_obsoletes = 1 2023-10-01T08:49:48.573 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum clean all 2023-10-01T08:49:48.808 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:49:48.945 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:49:49.760 INFO:teuthology.orchestra.run.smithi138.stdout:44 files removed 2023-10-01T08:49:49.814 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-radosgw 2023-10-01T08:49:49.871 INFO:teuthology.orchestra.run.smithi190.stdout:44 files removed 2023-10-01T08:49:49.926 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install ceph-radosgw 2023-10-01T08:49:50.146 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:49:50.256 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:49:51.533 INFO:teuthology.orchestra.run.smithi190.stdout:ceph packages for x86_64 302 kB/s | 76 kB 00:00 2023-10-01T08:49:51.579 INFO:teuthology.orchestra.run.smithi138.stdout:ceph packages for x86_64 230 kB/s | 76 kB 00:00 2023-10-01T08:49:51.834 INFO:teuthology.orchestra.run.smithi190.stdout:ceph noarch packages 57 kB/s | 15 kB 00:00 2023-10-01T08:49:51.930 INFO:teuthology.orchestra.run.smithi138.stdout:ceph noarch packages 48 kB/s | 15 kB 00:00 2023-10-01T08:49:52.126 INFO:teuthology.orchestra.run.smithi190.stdout:ceph source packages 5.3 kB/s | 1.4 kB 00:00 2023-10-01T08:49:52.269 INFO:teuthology.orchestra.run.smithi190.stdout:Copr repo for python3-asyncssh owned by ceph 28 kB/s | 3.5 kB 00:00 2023-10-01T08:49:52.272 INFO:teuthology.orchestra.run.smithi138.stdout:ceph source packages 4.5 kB/s | 1.4 kB 00:00 2023-10-01T08:49:52.415 INFO:teuthology.orchestra.run.smithi138.stdout:Copr repo for python3-asyncssh owned by ceph 28 kB/s | 3.5 kB 00:00 2023-10-01T08:49:53.371 INFO:teuthology.orchestra.run.smithi190.stdout:Extra Packages for Enterprise Linux 15 MB/s | 16 MB 00:01 2023-10-01T08:49:53.584 INFO:teuthology.orchestra.run.smithi138.stdout:Extra Packages for Enterprise Linux 14 MB/s | 16 MB 00:01 2023-10-01T08:49:56.837 INFO:teuthology.orchestra.run.smithi190.stdout:lab-extras 473 kB/s | 24 kB 00:00 2023-10-01T08:49:57.105 INFO:teuthology.orchestra.run.smithi138.stdout:lab-extras 288 kB/s | 24 kB 00:00 2023-10-01T08:49:58.013 INFO:teuthology.orchestra.run.smithi190.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 48 MB/s | 53 MB 00:01 2023-10-01T08:49:58.284 INFO:teuthology.orchestra.run.smithi138.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 48 MB/s | 53 MB 00:01 2023-10-01T08:50:06.857 INFO:teuthology.orchestra.run.smithi190.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 31 MB/s | 47 MB 00:01 2023-10-01T08:50:06.863 INFO:teuthology.orchestra.run.smithi138.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 39 MB/s | 47 MB 00:01 2023-10-01T08:50:17.753 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:50:17.756 INFO:teuthology.orchestra.run.smithi138.stdout:=================================================================================================== 2023-10-01T08:50:17.756 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-10-01T08:50:17.756 INFO:teuthology.orchestra.run.smithi138.stdout:=================================================================================================== 2023-10-01T08:50:17.756 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:50:17.756 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-radosgw x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 12 M 2023-10-01T08:50:17.756 INFO:teuthology.orchestra.run.smithi138.stdout:Upgrading: 2023-10-01T08:50:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: librados2 x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 3.7 M 2023-10-01T08:50:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: librbd1 x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 4.2 M 2023-10-01T08:50:17.757 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-10-01T08:50:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-base x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 6.1 M 2023-10-01T08:50:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-common x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 24 M 2023-10-01T08:50:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-selinux x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 26 k 2023-10-01T08:50:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-01T08:50:17.757 INFO:teuthology.orchestra.run.smithi138.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-10-01T08:50:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: libcephfs2 x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 824 k 2023-10-01T08:50:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-01T08:50:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-10-01T08:50:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: libradosstriper1 x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 493 k 2023-10-01T08:50:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-10-01T08:50:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: librgw2 x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 3.8 M 2023-10-01T08:50:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-01T08:50:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-10-01T08:50:17.758 INFO:teuthology.orchestra.run.smithi138.stdout: python3-ceph-argparse x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 45 k 2023-10-01T08:50:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: python3-ceph-common x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 99 k 2023-10-01T08:50:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cephfs x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 215 k 2023-10-01T08:50:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-10-01T08:50:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rados x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 390 k 2023-10-01T08:50:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rbd x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 369 k 2023-10-01T08:50:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rgw x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 114 k 2023-10-01T08:50:17.759 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:50:17.759 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:50:17.760 INFO:teuthology.orchestra.run.smithi138.stdout:=================================================================================================== 2023-10-01T08:50:17.760 INFO:teuthology.orchestra.run.smithi138.stdout:Install 21 Packages 2023-10-01T08:50:17.760 INFO:teuthology.orchestra.run.smithi138.stdout:Upgrade 2 Packages 2023-10-01T08:50:17.760 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:50:17.760 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 57 M 2023-10-01T08:50:17.760 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:50:17.859 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:50:17.862 INFO:teuthology.orchestra.run.smithi190.stdout:=================================================================================================== 2023-10-01T08:50:17.862 INFO:teuthology.orchestra.run.smithi190.stdout: Package Arch Version Repository Size 2023-10-01T08:50:17.862 INFO:teuthology.orchestra.run.smithi190.stdout:=================================================================================================== 2023-10-01T08:50:17.862 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:50:17.862 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-radosgw x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 12 M 2023-10-01T08:50:17.862 INFO:teuthology.orchestra.run.smithi190.stdout:Upgrading: 2023-10-01T08:50:17.862 INFO:teuthology.orchestra.run.smithi190.stdout: librados2 x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 3.7 M 2023-10-01T08:50:17.862 INFO:teuthology.orchestra.run.smithi190.stdout: librbd1 x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 4.2 M 2023-10-01T08:50:17.862 INFO:teuthology.orchestra.run.smithi190.stdout:Installing dependencies: 2023-10-01T08:50:17.863 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-base x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 6.1 M 2023-10-01T08:50:17.863 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-common x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 24 M 2023-10-01T08:50:17.863 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-selinux x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 26 k 2023-10-01T08:50:17.863 INFO:teuthology.orchestra.run.smithi190.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-10-01T08:50:17.863 INFO:teuthology.orchestra.run.smithi190.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-10-01T08:50:17.863 INFO:teuthology.orchestra.run.smithi190.stdout: libcephfs2 x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 824 k 2023-10-01T08:50:17.864 INFO:teuthology.orchestra.run.smithi190.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-10-01T08:50:17.864 INFO:teuthology.orchestra.run.smithi190.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-10-01T08:50:17.864 INFO:teuthology.orchestra.run.smithi190.stdout: libradosstriper1 x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 493 k 2023-10-01T08:50:17.864 INFO:teuthology.orchestra.run.smithi190.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-10-01T08:50:17.864 INFO:teuthology.orchestra.run.smithi190.stdout: librgw2 x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 3.8 M 2023-10-01T08:50:17.864 INFO:teuthology.orchestra.run.smithi190.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-10-01T08:50:17.864 INFO:teuthology.orchestra.run.smithi190.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-10-01T08:50:17.864 INFO:teuthology.orchestra.run.smithi190.stdout: python3-ceph-argparse x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 45 k 2023-10-01T08:50:17.864 INFO:teuthology.orchestra.run.smithi190.stdout: python3-ceph-common x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 99 k 2023-10-01T08:50:17.864 INFO:teuthology.orchestra.run.smithi190.stdout: python3-cephfs x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 215 k 2023-10-01T08:50:17.864 INFO:teuthology.orchestra.run.smithi190.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-10-01T08:50:17.865 INFO:teuthology.orchestra.run.smithi190.stdout: python3-rados x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 390 k 2023-10-01T08:50:17.865 INFO:teuthology.orchestra.run.smithi190.stdout: python3-rbd x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 369 k 2023-10-01T08:50:17.865 INFO:teuthology.orchestra.run.smithi190.stdout: python3-rgw x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 114 k 2023-10-01T08:50:17.865 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:50:17.865 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:50:17.865 INFO:teuthology.orchestra.run.smithi190.stdout:=================================================================================================== 2023-10-01T08:50:17.865 INFO:teuthology.orchestra.run.smithi190.stdout:Install 21 Packages 2023-10-01T08:50:17.866 INFO:teuthology.orchestra.run.smithi190.stdout:Upgrade 2 Packages 2023-10-01T08:50:17.866 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:50:17.866 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 57 M 2023-10-01T08:50:17.866 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:50:18.453 INFO:teuthology.orchestra.run.smithi138.stdout:(1/23): ceph-base-16.2.14-44.geeb13ac9.el8.x86_ 9.2 MB/s | 6.1 MB 00:00 2023-10-01T08:50:18.511 INFO:teuthology.orchestra.run.smithi138.stdout:(2/23): ceph-selinux-16.2.14-44.geeb13ac9.el8.x 441 kB/s | 26 kB 00:00 2023-10-01T08:50:18.620 INFO:teuthology.orchestra.run.smithi138.stdout:(3/23): libcephfs2-16.2.14-44.geeb13ac9.el8.x86 7.4 MB/s | 824 kB 00:00 2023-10-01T08:50:18.740 INFO:teuthology.orchestra.run.smithi138.stdout:(4/23): libradosstriper1-16.2.14-44.geeb13ac9.e 4.0 MB/s | 493 kB 00:00 2023-10-01T08:50:19.145 INFO:teuthology.orchestra.run.smithi138.stdout:(5/23): ceph-radosgw-16.2.14-44.geeb13ac9.el8.x 8.5 MB/s | 12 MB 00:01 2023-10-01T08:50:19.213 INFO:teuthology.orchestra.run.smithi138.stdout:(6/23): librgw2-16.2.14-44.geeb13ac9.el8.x86_64 8.0 MB/s | 3.8 MB 00:00 2023-10-01T08:50:19.238 INFO:teuthology.orchestra.run.smithi138.stdout:(7/23): python3-ceph-argparse-16.2.14-44.geeb13 490 kB/s | 45 kB 00:00 2023-10-01T08:50:19.271 INFO:teuthology.orchestra.run.smithi138.stdout:(8/23): python3-ceph-common-16.2.14-44.geeb13ac 1.6 MB/s | 99 kB 00:00 2023-10-01T08:50:19.305 INFO:teuthology.orchestra.run.smithi138.stdout:(9/23): python3-cephfs-16.2.14-44.geeb13ac9.el8 3.1 MB/s | 215 kB 00:00 2023-10-01T08:50:19.356 INFO:teuthology.orchestra.run.smithi138.stdout:(10/23): python3-rados-16.2.14-44.geeb13ac9.el8 4.6 MB/s | 390 kB 00:00 2023-10-01T08:50:19.398 INFO:teuthology.orchestra.run.smithi138.stdout:(11/23): python3-rbd-16.2.14-44.geeb13ac9.el8.x 3.9 MB/s | 369 kB 00:00 2023-10-01T08:50:19.463 INFO:teuthology.orchestra.run.smithi138.stdout:(12/23): python3-rgw-16.2.14-44.geeb13ac9.el8.x 1.0 MB/s | 114 kB 00:00 2023-10-01T08:50:19.914 INFO:teuthology.orchestra.run.smithi190.stdout:(1/23): ceph-radosgw-16.2.14-44.geeb13ac9.el8.x 6.7 MB/s | 12 MB 00:01 2023-10-01T08:50:19.964 INFO:teuthology.orchestra.run.smithi190.stdout:(2/23): ceph-selinux-16.2.14-44.geeb13ac9.el8.x 513 kB/s | 26 kB 00:00 2023-10-01T08:50:20.024 INFO:teuthology.orchestra.run.smithi138.stdout:(13/23): gperftools-libs-2.7-9.el8.x86_64.rpm 490 kB/s | 306 kB 00:00 2023-10-01T08:50:20.049 INFO:teuthology.orchestra.run.smithi138.stdout:(14/23): leveldb-1.22-1.el8.x86_64.rpm 309 kB/s | 181 kB 00:00 2023-10-01T08:50:20.081 INFO:teuthology.orchestra.run.smithi190.stdout:(3/23): libcephfs2-16.2.14-44.geeb13ac9.el8.x86 6.9 MB/s | 824 kB 00:00 2023-10-01T08:50:20.148 INFO:teuthology.orchestra.run.smithi190.stdout:(4/23): libradosstriper1-16.2.14-44.geeb13ac9.e 7.2 MB/s | 493 kB 00:00 2023-10-01T08:50:20.149 INFO:teuthology.orchestra.run.smithi138.stdout:(15/23): liboath-2.6.2-3.el8.x86_64.rpm 470 kB/s | 59 kB 00:00 2023-10-01T08:50:20.174 INFO:teuthology.orchestra.run.smithi138.stdout:(16/23): libunwind-1.3.1-3.el8.x86_64.rpm 600 kB/s | 75 kB 00:00 2023-10-01T08:50:20.308 INFO:teuthology.orchestra.run.smithi138.stdout:(17/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 295 kB/s | 47 kB 00:00 2023-10-01T08:50:20.358 INFO:teuthology.orchestra.run.smithi138.stdout:(18/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 1.4 MB/s | 259 kB 00:00 2023-10-01T08:50:20.417 INFO:teuthology.orchestra.run.smithi138.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 405 kB/s | 44 kB 00:00 2023-10-01T08:50:20.490 INFO:teuthology.orchestra.run.smithi190.stdout:(5/23): librgw2-16.2.14-44.geeb13ac9.el8.x86_64 11 MB/s | 3.8 MB 00:00 2023-10-01T08:50:20.509 INFO:teuthology.orchestra.run.smithi138.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 2.3 MB/s | 354 kB 00:00 2023-10-01T08:50:20.549 INFO:teuthology.orchestra.run.smithi190.stdout:(6/23): python3-ceph-argparse-16.2.14-44.geeb13 771 kB/s | 45 kB 00:00 2023-10-01T08:50:20.624 INFO:teuthology.orchestra.run.smithi190.stdout:(7/23): python3-ceph-common-16.2.14-44.geeb13ac 1.3 MB/s | 99 kB 00:00 2023-10-01T08:50:20.732 INFO:teuthology.orchestra.run.smithi190.stdout:(8/23): python3-cephfs-16.2.14-44.geeb13ac9.el8 1.9 MB/s | 215 kB 00:00 2023-10-01T08:50:20.951 INFO:teuthology.orchestra.run.smithi138.stdout:(21/23): librados2-16.2.14-44.geeb13ac9.el8.x86 7.0 MB/s | 3.7 MB 00:00 2023-10-01T08:50:21.199 INFO:teuthology.orchestra.run.smithi190.stdout:(9/23): python3-rados-16.2.14-44.geeb13ac9.el8. 837 kB/s | 390 kB 00:00 2023-10-01T08:50:21.283 INFO:teuthology.orchestra.run.smithi190.stdout:(10/23): python3-rbd-16.2.14-44.geeb13ac9.el8.x 4.3 MB/s | 369 kB 00:00 2023-10-01T08:50:21.350 INFO:teuthology.orchestra.run.smithi190.stdout:(11/23): python3-rgw-16.2.14-44.geeb13ac9.el8.x 1.7 MB/s | 114 kB 00:00 2023-10-01T08:50:21.450 INFO:teuthology.orchestra.run.smithi190.stdout:(12/23): ceph-base-16.2.14-44.geeb13ac9.el8.x86 1.9 MB/s | 6.1 MB 00:03 2023-10-01T08:50:21.551 INFO:teuthology.orchestra.run.smithi138.stdout:(22/23): librbd1-16.2.14-44.geeb13ac9.el8.x86_6 4.0 MB/s | 4.2 MB 00:01 2023-10-01T08:50:21.842 INFO:teuthology.orchestra.run.smithi190.stdout:(13/23): gperftools-libs-2.7-9.el8.x86_64.rpm 622 kB/s | 306 kB 00:00 2023-10-01T08:50:21.876 INFO:teuthology.orchestra.run.smithi190.stdout:(14/23): leveldb-1.22-1.el8.x86_64.rpm 425 kB/s | 181 kB 00:00 2023-10-01T08:50:21.935 INFO:teuthology.orchestra.run.smithi190.stdout:(15/23): liboath-2.6.2-3.el8.x86_64.rpm 640 kB/s | 59 kB 00:00 2023-10-01T08:50:21.977 INFO:teuthology.orchestra.run.smithi190.stdout:(16/23): libunwind-1.3.1-3.el8.x86_64.rpm 749 kB/s | 75 kB 00:00 2023-10-01T08:50:22.077 INFO:teuthology.orchestra.run.smithi190.stdout:(17/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 330 kB/s | 47 kB 00:00 2023-10-01T08:50:22.202 INFO:teuthology.orchestra.run.smithi190.stdout:(18/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 1.2 MB/s | 259 kB 00:00 2023-10-01T08:50:22.219 INFO:teuthology.orchestra.run.smithi190.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 309 kB/s | 44 kB 00:00 2023-10-01T08:50:22.311 INFO:teuthology.orchestra.run.smithi190.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 3.2 MB/s | 354 kB 00:00 2023-10-01T08:50:23.612 INFO:teuthology.orchestra.run.smithi190.stdout:(21/23): librbd1-16.2.14-44.geeb13ac9.el8.x86_6 3.2 MB/s | 4.2 MB 00:01 2023-10-01T08:50:24.287 INFO:teuthology.orchestra.run.smithi190.stdout:(22/23): librados2-16.2.14-44.geeb13ac9.el8.x86 1.8 MB/s | 3.7 MB 00:02 2023-10-01T08:50:24.737 INFO:teuthology.orchestra.run.smithi190.stdout:(23/23): ceph-common-16.2.14-44.geeb13ac9.el8.x 3.7 MB/s | 24 MB 00:06 2023-10-01T08:50:24.739 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:50:24.739 INFO:teuthology.orchestra.run.smithi190.stdout:Total 8.3 MB/s | 57 MB 00:06 2023-10-01T08:50:24.786 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:50:24.832 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:50:24.832 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:50:25.170 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:50:25.174 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:50:26.186 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:50:26.361 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-01T08:50:26.749 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-01T08:50:26.903 INFO:teuthology.orchestra.run.smithi190.stdout: Upgrading : librados2-2:16.2.14-44.geeb13ac9.el8.x86_64 2/25 2023-10-01T08:50:27.004 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: librados2-2:16.2.14-44.geeb13ac9.el8.x86_64 2/25 2023-10-01T08:50:27.192 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-rados-2:16.2.14-44.geeb13ac9.el8.x86_64 3/25 2023-10-01T08:50:27.626 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-10-01T08:50:27.786 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : libcephfs2-2:16.2.14-44.geeb13ac9.el8.x86_64 5/25 2023-10-01T08:50:28.203 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: libcephfs2-2:16.2.14-44.geeb13ac9.el8.x86_64 5/25 2023-10-01T08:50:28.427 INFO:teuthology.orchestra.run.smithi190.stdout: Upgrading : librbd1-2:16.2.14-44.geeb13ac9.el8.x86_64 6/25 2023-10-01T08:50:28.500 INFO:teuthology.orchestra.run.smithi138.stdout:(23/23): ceph-common-16.2.14-44.geeb13ac9.el8.x 2.3 MB/s | 24 MB 00:10 2023-10-01T08:50:28.505 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:50:28.505 INFO:teuthology.orchestra.run.smithi138.stdout:Total 5.3 MB/s | 57 MB 00:10 2023-10-01T08:50:28.541 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: librbd1-2:16.2.14-44.geeb13ac9.el8.x86_64 6/25 2023-10-01T08:50:28.552 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:50:28.599 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:50:28.599 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:50:28.710 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-01T08:50:28.776 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-01T08:50:28.930 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:50:28.934 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:50:29.246 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-10-01T08:50:29.418 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : librgw2-2:16.2.14-44.geeb13ac9.el8.x86_64 9/25 2023-10-01T08:50:29.499 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: librgw2-2:16.2.14-44.geeb13ac9.el8.x86_64 9/25 2023-10-01T08:50:29.682 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-10-01T08:50:29.806 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-ceph-argparse-2:16.2.14-44.geeb13ac9.el8.x 11/25 2023-10-01T08:50:29.971 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-cephfs-2:16.2.14-44.geeb13ac9.el8.x86_64 12/25 2023-10-01T08:50:30.006 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:50:30.224 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-rgw-2:16.2.14-44.geeb13ac9.el8.x86_64 13/25 2023-10-01T08:50:30.233 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-01T08:50:30.426 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-rbd-2:16.2.14-44.geeb13ac9.el8.x86_64 14/25 2023-10-01T08:50:30.577 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : libradosstriper1-2:16.2.14-44.geeb13ac9.el8.x86_64 15/25 2023-10-01T08:50:30.620 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-10-01T08:50:30.643 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: libradosstriper1-2:16.2.14-44.geeb13ac9.el8.x86_64 15/25 2023-10-01T08:50:30.787 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-01T08:50:30.833 INFO:teuthology.orchestra.run.smithi138.stdout: Upgrading : librados2-2:16.2.14-44.geeb13ac9.el8.x86_64 2/25 2023-10-01T08:50:30.934 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: librados2-2:16.2.14-44.geeb13ac9.el8.x86_64 2/25 2023-10-01T08:50:30.987 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-01T08:50:31.139 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-rados-2:16.2.14-44.geeb13ac9.el8.x86_64 3/25 2023-10-01T08:50:31.237 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-01T08:50:31.369 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-ceph-common-2:16.2.14-44.geeb13ac9.el8.x86 19/25 2023-10-01T08:50:31.372 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-10-01T08:50:31.583 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : libcephfs2-2:16.2.14-44.geeb13ac9.el8.x86_64 5/25 2023-10-01T08:50:31.983 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: libcephfs2-2:16.2.14-44.geeb13ac9.el8.x86_64 5/25 2023-10-01T08:50:32.225 INFO:teuthology.orchestra.run.smithi138.stdout: Upgrading : librbd1-2:16.2.14-44.geeb13ac9.el8.x86_64 6/25 2023-10-01T08:50:32.303 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: librbd1-2:16.2.14-44.geeb13ac9.el8.x86_64 6/25 2023-10-01T08:50:32.491 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-01T08:50:32.557 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-10-01T08:50:33.036 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-10-01T08:50:33.232 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : librgw2-2:16.2.14-44.geeb13ac9.el8.x86_64 9/25 2023-10-01T08:50:33.314 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: librgw2-2:16.2.14-44.geeb13ac9.el8.x86_64 9/25 2023-10-01T08:50:33.530 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-10-01T08:50:33.746 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-ceph-argparse-2:16.2.14-44.geeb13ac9.el8.x 11/25 2023-10-01T08:50:34.020 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-cephfs-2:16.2.14-44.geeb13ac9.el8.x86_64 12/25 2023-10-01T08:50:34.259 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-rgw-2:16.2.14-44.geeb13ac9.el8.x86_64 13/25 2023-10-01T08:50:34.457 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-rbd-2:16.2.14-44.geeb13ac9.el8.x86_64 14/25 2023-10-01T08:50:34.632 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : libradosstriper1-2:16.2.14-44.geeb13ac9.el8.x86_64 15/25 2023-10-01T08:50:34.692 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: libradosstriper1-2:16.2.14-44.geeb13ac9.el8.x86_64 15/25 2023-10-01T08:50:34.860 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-10-01T08:50:34.988 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-common-2:16.2.14-44.geeb13ac9.el8.x86_64 20/25 2023-10-01T08:50:35.068 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-01T08:50:35.227 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-common-2:16.2.14-44.geeb13ac9.el8.x86_64 20/25 2023-10-01T08:50:35.322 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-10-01T08:50:35.516 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-ceph-common-2:16.2.14-44.geeb13ac9.el8.x86 19/25 2023-10-01T08:50:35.924 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-common-2:16.2.14-44.geeb13ac9.el8.x86_64 20/25 2023-10-01T08:50:36.203 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-base-2:16.2.14-44.geeb13ac9.el8.x86_64 21/25 2023-10-01T08:50:36.283 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-base-2:16.2.14-44.geeb13ac9.el8.x86_64 21/25 2023-10-01T08:50:36.409 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-selinux-2:16.2.14-44.geeb13ac9.el8.x86_64 22/25 2023-10-01T08:50:39.145 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-common-2:16.2.14-44.geeb13ac9.el8.x86_64 20/25 2023-10-01T08:50:39.608 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-common-2:16.2.14-44.geeb13ac9.el8.x86_64 20/25 2023-10-01T08:50:40.305 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-common-2:16.2.14-44.geeb13ac9.el8.x86_64 20/25 2023-10-01T08:50:40.649 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-base-2:16.2.14-44.geeb13ac9.el8.x86_64 21/25 2023-10-01T08:50:40.736 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-base-2:16.2.14-44.geeb13ac9.el8.x86_64 21/25 2023-10-01T08:50:40.889 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-selinux-2:16.2.14-44.geeb13ac9.el8.x86_64 22/25 2023-10-01T08:50:54.417 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-selinux-2:16.2.14-44.geeb13ac9.el8.x86_64 22/25 2023-10-01T08:50:54.417 INFO:teuthology.orchestra.run.smithi190.stdout:skipping the directory /sys 2023-10-01T08:50:54.417 INFO:teuthology.orchestra.run.smithi190.stdout:skipping the directory /proc 2023-10-01T08:50:54.417 INFO:teuthology.orchestra.run.smithi190.stdout:skipping the directory /dev 2023-10-01T08:50:54.417 INFO:teuthology.orchestra.run.smithi190.stdout:skipping the directory /run 2023-10-01T08:50:54.417 INFO:teuthology.orchestra.run.smithi190.stdout:skipping the directory /mnt 2023-10-01T08:50:54.418 INFO:teuthology.orchestra.run.smithi190.stdout:skipping the directory /var/tmp 2023-10-01T08:50:54.418 INFO:teuthology.orchestra.run.smithi190.stdout:skipping the directory /home 2023-10-01T08:50:54.418 INFO:teuthology.orchestra.run.smithi190.stdout:skipping the directory /tmp 2023-10-01T08:50:54.418 INFO:teuthology.orchestra.run.smithi190.stdout:skipping the directory /dev 2023-10-01T08:50:54.418 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:50:55.574 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-radosgw-2:16.2.14-44.geeb13ac9.el8.x86_64 23/25 2023-10-01T08:50:55.640 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-radosgw-2:16.2.14-44.geeb13ac9.el8.x86_64 23/25 2023-10-01T08:50:55.641 INFO:teuthology.orchestra.run.smithi190.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-01T08:50:55.825 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-01T08:50:55.825 INFO:teuthology.orchestra.run.smithi190.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-01T08:50:56.418 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-01T08:50:56.418 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-base-2:16.2.14-44.geeb13ac9.el8.x86_64 1/25 2023-10-01T08:50:56.418 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-common-2:16.2.14-44.geeb13ac9.el8.x86_64 2/25 2023-10-01T08:50:56.418 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-radosgw-2:16.2.14-44.geeb13ac9.el8.x86_64 3/25 2023-10-01T08:50:56.418 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-selinux-2:16.2.14-44.geeb13ac9.el8.x86_64 4/25 2023-10-01T08:50:56.418 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : libcephfs2-2:16.2.14-44.geeb13ac9.el8.x86_64 5/25 2023-10-01T08:50:56.419 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : libradosstriper1-2:16.2.14-44.geeb13ac9.el8.x86_64 6/25 2023-10-01T08:50:56.419 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : librgw2-2:16.2.14-44.geeb13ac9.el8.x86_64 7/25 2023-10-01T08:50:56.419 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-ceph-argparse-2:16.2.14-44.geeb13ac9.el8.x 8/25 2023-10-01T08:50:56.419 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-ceph-common-2:16.2.14-44.geeb13ac9.el8.x86 9/25 2023-10-01T08:50:56.419 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-cephfs-2:16.2.14-44.geeb13ac9.el8.x86_64 10/25 2023-10-01T08:50:56.419 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-rados-2:16.2.14-44.geeb13ac9.el8.x86_64 11/25 2023-10-01T08:50:56.419 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-rbd-2:16.2.14-44.geeb13ac9.el8.x86_64 12/25 2023-10-01T08:50:56.419 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-rgw-2:16.2.14-44.geeb13ac9.el8.x86_64 13/25 2023-10-01T08:50:56.419 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-10-01T08:50:56.419 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : leveldb-1.22-1.el8.x86_64 15/25 2023-10-01T08:50:56.419 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 16/25 2023-10-01T08:50:56.420 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-01T08:50:56.420 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-10-01T08:50:56.420 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-10-01T08:50:56.420 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-10-01T08:50:56.420 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-10-01T08:50:56.420 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : librados2-2:16.2.14-44.geeb13ac9.el8.x86_64 22/25 2023-10-01T08:50:56.420 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-01T08:50:56.420 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : librbd1-2:16.2.14-44.geeb13ac9.el8.x86_64 24/25 2023-10-01T08:50:57.756 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-01T08:50:57.756 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:00:52 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:50:59.070 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-selinux-2:16.2.14-44.geeb13ac9.el8.x86_64 22/25 2023-10-01T08:50:59.070 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /sys 2023-10-01T08:50:59.070 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /proc 2023-10-01T08:50:59.070 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /dev 2023-10-01T08:50:59.071 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /run 2023-10-01T08:50:59.071 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /mnt 2023-10-01T08:50:59.071 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /var/tmp 2023-10-01T08:50:59.071 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /home 2023-10-01T08:50:59.071 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /tmp 2023-10-01T08:50:59.071 INFO:teuthology.orchestra.run.smithi138.stdout:skipping the directory /dev 2023-10-01T08:50:59.071 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:51:00.246 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-radosgw-2:16.2.14-44.geeb13ac9.el8.x86_64 23/25 2023-10-01T08:51:00.319 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-radosgw-2:16.2.14-44.geeb13ac9.el8.x86_64 23/25 2023-10-01T08:51:00.319 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-01T08:51:00.547 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-10-01T08:51:00.547 INFO:teuthology.orchestra.run.smithi138.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-01T08:51:01.191 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-10-01T08:51:01.192 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-base-2:16.2.14-44.geeb13ac9.el8.x86_64 1/25 2023-10-01T08:51:01.192 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-common-2:16.2.14-44.geeb13ac9.el8.x86_64 2/25 2023-10-01T08:51:01.192 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-radosgw-2:16.2.14-44.geeb13ac9.el8.x86_64 3/25 2023-10-01T08:51:01.192 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-selinux-2:16.2.14-44.geeb13ac9.el8.x86_64 4/25 2023-10-01T08:51:01.192 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libcephfs2-2:16.2.14-44.geeb13ac9.el8.x86_64 5/25 2023-10-01T08:51:01.192 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libradosstriper1-2:16.2.14-44.geeb13ac9.el8.x86_64 6/25 2023-10-01T08:51:01.192 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librgw2-2:16.2.14-44.geeb13ac9.el8.x86_64 7/25 2023-10-01T08:51:01.192 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-ceph-argparse-2:16.2.14-44.geeb13ac9.el8.x 8/25 2023-10-01T08:51:01.192 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-ceph-common-2:16.2.14-44.geeb13ac9.el8.x86 9/25 2023-10-01T08:51:01.192 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-cephfs-2:16.2.14-44.geeb13ac9.el8.x86_64 10/25 2023-10-01T08:51:01.192 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-rados-2:16.2.14-44.geeb13ac9.el8.x86_64 11/25 2023-10-01T08:51:01.193 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-rbd-2:16.2.14-44.geeb13ac9.el8.x86_64 12/25 2023-10-01T08:51:01.193 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-rgw-2:16.2.14-44.geeb13ac9.el8.x86_64 13/25 2023-10-01T08:51:01.193 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-10-01T08:51:01.193 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : leveldb-1.22-1.el8.x86_64 15/25 2023-10-01T08:51:01.193 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 16/25 2023-10-01T08:51:01.193 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 17/25 2023-10-01T08:51:01.193 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-10-01T08:51:01.193 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-10-01T08:51:01.193 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-10-01T08:51:01.193 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-10-01T08:51:01.193 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librados2-2:16.2.14-44.geeb13ac9.el8.x86_64 22/25 2023-10-01T08:51:01.193 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-10-01T08:51:01.194 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librbd1-2:16.2.14-44.geeb13ac9.el8.x86_64 24/25 2023-10-01T08:51:02.628 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-10-01T08:51:02.628 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:00:57 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:51:08.903 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:51:09.090 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:51:09.090 INFO:teuthology.orchestra.run.smithi190.stdout:Upgraded: 2023-10-01T08:51:09.090 INFO:teuthology.orchestra.run.smithi190.stdout: librados2-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.090 INFO:teuthology.orchestra.run.smithi190.stdout: librbd1-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.090 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:51:09.091 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-base-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.091 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-common-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.091 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-radosgw-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.091 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-selinux-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.091 INFO:teuthology.orchestra.run.smithi190.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-01T08:51:09.091 INFO:teuthology.orchestra.run.smithi190.stdout: leveldb-1.22-1.el8.x86_64 2023-10-01T08:51:09.091 INFO:teuthology.orchestra.run.smithi190.stdout: libcephfs2-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.091 INFO:teuthology.orchestra.run.smithi190.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-01T08:51:09.091 INFO:teuthology.orchestra.run.smithi190.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-10-01T08:51:09.092 INFO:teuthology.orchestra.run.smithi190.stdout: libradosstriper1-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.092 INFO:teuthology.orchestra.run.smithi190.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-01T08:51:09.092 INFO:teuthology.orchestra.run.smithi190.stdout: librgw2-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.092 INFO:teuthology.orchestra.run.smithi190.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-01T08:51:09.092 INFO:teuthology.orchestra.run.smithi190.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-01T08:51:09.092 INFO:teuthology.orchestra.run.smithi190.stdout: python3-ceph-argparse-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.092 INFO:teuthology.orchestra.run.smithi190.stdout: python3-ceph-common-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.092 INFO:teuthology.orchestra.run.smithi190.stdout: python3-cephfs-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.092 INFO:teuthology.orchestra.run.smithi190.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-01T08:51:09.093 INFO:teuthology.orchestra.run.smithi190.stdout: python3-rados-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.093 INFO:teuthology.orchestra.run.smithi190.stdout: python3-rbd-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.093 INFO:teuthology.orchestra.run.smithi190.stdout: python3-rgw-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:09.093 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:51:09.093 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:51:09.256 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install ceph-test 2023-10-01T08:51:09.604 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:51:11.270 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:01:06 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:51:12.257 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:51:12.445 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:51:12.445 INFO:teuthology.orchestra.run.smithi138.stdout:Upgraded: 2023-10-01T08:51:12.445 INFO:teuthology.orchestra.run.smithi138.stdout: librados2-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.445 INFO:teuthology.orchestra.run.smithi138.stdout: librbd1-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.445 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:51:12.445 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-base-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.445 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-common-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.445 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-radosgw-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.445 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-selinux-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.446 INFO:teuthology.orchestra.run.smithi138.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-10-01T08:51:12.446 INFO:teuthology.orchestra.run.smithi138.stdout: leveldb-1.22-1.el8.x86_64 2023-10-01T08:51:12.446 INFO:teuthology.orchestra.run.smithi138.stdout: libcephfs2-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.446 INFO:teuthology.orchestra.run.smithi138.stdout: liboath-2.6.2-3.el8.x86_64 2023-10-01T08:51:12.446 INFO:teuthology.orchestra.run.smithi138.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-10-01T08:51:12.446 INFO:teuthology.orchestra.run.smithi138.stdout: libradosstriper1-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.446 INFO:teuthology.orchestra.run.smithi138.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-10-01T08:51:12.446 INFO:teuthology.orchestra.run.smithi138.stdout: librgw2-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.446 INFO:teuthology.orchestra.run.smithi138.stdout: libunwind-1.3.1-3.el8.x86_64 2023-10-01T08:51:12.446 INFO:teuthology.orchestra.run.smithi138.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-10-01T08:51:12.446 INFO:teuthology.orchestra.run.smithi138.stdout: python3-ceph-argparse-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.447 INFO:teuthology.orchestra.run.smithi138.stdout: python3-ceph-common-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.447 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cephfs-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.447 INFO:teuthology.orchestra.run.smithi138.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-10-01T08:51:12.447 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rados-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.447 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rbd-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.447 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rgw-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:12.447 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:51:12.447 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:51:12.619 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-test 2023-10-01T08:51:12.630 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:51:12.631 INFO:teuthology.orchestra.run.smithi190.stdout:=========================================================================================== 2023-10-01T08:51:12.631 INFO:teuthology.orchestra.run.smithi190.stdout: Package Arch Version Repository Size 2023-10-01T08:51:12.631 INFO:teuthology.orchestra.run.smithi190.stdout:=========================================================================================== 2023-10-01T08:51:12.631 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:51:12.631 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-test x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 52 M 2023-10-01T08:51:12.631 INFO:teuthology.orchestra.run.smithi190.stdout:Installing dependencies: 2023-10-01T08:51:12.631 INFO:teuthology.orchestra.run.smithi190.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-10-01T08:51:12.632 INFO:teuthology.orchestra.run.smithi190.stdout: libcephsqlite x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 174 k 2023-10-01T08:51:12.632 INFO:teuthology.orchestra.run.smithi190.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-10-01T08:51:12.632 INFO:teuthology.orchestra.run.smithi190.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-10-01T08:51:12.632 INFO:teuthology.orchestra.run.smithi190.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-10-01T08:51:12.632 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:51:12.632 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:51:12.632 INFO:teuthology.orchestra.run.smithi190.stdout:=========================================================================================== 2023-10-01T08:51:12.632 INFO:teuthology.orchestra.run.smithi190.stdout:Install 6 Packages 2023-10-01T08:51:12.632 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:51:12.633 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 53 M 2023-10-01T08:51:12.633 INFO:teuthology.orchestra.run.smithi190.stdout:Installed size: 195 M 2023-10-01T08:51:12.633 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:51:12.681 INFO:teuthology.orchestra.run.smithi190.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.4 MB/s | 70 kB 00:00 2023-10-01T08:51:12.856 INFO:teuthology.orchestra.run.smithi190.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 1.9 MB/s | 323 kB 00:00 2023-10-01T08:51:12.898 INFO:teuthology.orchestra.run.smithi190.stdout:(3/6): libcephsqlite-16.2.14-44.geeb13ac9.el8.x 657 kB/s | 174 kB 00:00 2023-10-01T08:51:12.965 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:51:12.998 INFO:teuthology.orchestra.run.smithi190.stdout:(4/6): oniguruma-6.8.2-2.el8.x86_64.rpm 1.3 MB/s | 187 kB 00:00 2023-10-01T08:51:13.057 INFO:teuthology.orchestra.run.smithi190.stdout:(5/6): jq-1.6-3.el8.x86_64.rpm 1.2 MB/s | 202 kB 00:00 2023-10-01T08:51:14.397 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:09 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:51:15.448 INFO:teuthology.orchestra.run.smithi190.stdout:(6/6): ceph-test-16.2.14-44.geeb13ac9.el8.x86_6 18 MB/s | 52 MB 00:02 2023-10-01T08:51:15.450 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:51:15.451 INFO:teuthology.orchestra.run.smithi190.stdout:Total 19 MB/s | 53 MB 00:02 2023-10-01T08:51:15.488 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:51:15.509 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:51:15.509 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:51:15.782 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:51:15.786 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:51:15.857 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:51:15.858 INFO:teuthology.orchestra.run.smithi138.stdout:=========================================================================================== 2023-10-01T08:51:15.858 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-10-01T08:51:15.859 INFO:teuthology.orchestra.run.smithi138.stdout:=========================================================================================== 2023-10-01T08:51:15.859 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:51:15.859 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-test x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 52 M 2023-10-01T08:51:15.859 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-10-01T08:51:15.859 INFO:teuthology.orchestra.run.smithi138.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-10-01T08:51:15.859 INFO:teuthology.orchestra.run.smithi138.stdout: libcephsqlite x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 174 k 2023-10-01T08:51:15.859 INFO:teuthology.orchestra.run.smithi138.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-10-01T08:51:15.859 INFO:teuthology.orchestra.run.smithi138.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-10-01T08:51:15.859 INFO:teuthology.orchestra.run.smithi138.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-10-01T08:51:15.859 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:51:15.860 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:51:15.860 INFO:teuthology.orchestra.run.smithi138.stdout:=========================================================================================== 2023-10-01T08:51:15.860 INFO:teuthology.orchestra.run.smithi138.stdout:Install 6 Packages 2023-10-01T08:51:15.860 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:51:15.860 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 53 M 2023-10-01T08:51:15.860 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 195 M 2023-10-01T08:51:15.861 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:51:15.904 INFO:teuthology.orchestra.run.smithi138.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.6 MB/s | 70 kB 00:00 2023-10-01T08:51:16.079 INFO:teuthology.orchestra.run.smithi138.stdout:(2/6): libcephsqlite-16.2.14-44.geeb13ac9.el8.x 796 kB/s | 174 kB 00:00 2023-10-01T08:51:16.113 INFO:teuthology.orchestra.run.smithi138.stdout:(3/6): socat-1.7.4.1-1.el8.x86_64.rpm 1.6 MB/s | 323 kB 00:00 2023-10-01T08:51:16.180 INFO:teuthology.orchestra.run.smithi138.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 3.2 MB/s | 202 kB 00:00 2023-10-01T08:51:16.234 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:51:16.280 INFO:teuthology.orchestra.run.smithi138.stdout:(5/6): oniguruma-6.8.2-2.el8.x86_64.rpm 936 kB/s | 187 kB 00:00 2023-10-01T08:51:16.378 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-01T08:51:16.459 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-01T08:51:16.673 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-10-01T08:51:16.820 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-10-01T08:51:16.968 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-10-01T08:51:17.111 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : libcephsqlite-2:16.2.14-44.geeb13ac9.el8.x86_64 5/6 2023-10-01T08:51:19.305 INFO:teuthology.orchestra.run.smithi138.stdout:(6/6): ceph-test-16.2.14-44.geeb13ac9.el8.x86_6 15 MB/s | 52 MB 00:03 2023-10-01T08:51:19.306 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:51:19.307 INFO:teuthology.orchestra.run.smithi138.stdout:Total 15 MB/s | 53 MB 00:03 2023-10-01T08:51:19.344 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:51:19.365 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:51:19.365 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:51:19.637 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:51:19.641 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:51:20.091 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:51:20.217 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-01T08:51:20.291 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-10-01T08:51:20.513 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-10-01T08:51:20.692 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-10-01T08:51:20.848 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-10-01T08:51:21.017 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : libcephsqlite-2:16.2.14-44.geeb13ac9.el8.x86_64 5/6 2023-10-01T08:51:21.654 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: libcephsqlite-2:16.2.14-44.geeb13ac9.el8.x86_64 5/6 2023-10-01T08:51:21.874 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-test-2:16.2.14-44.geeb13ac9.el8.x86_64 6/6 2023-10-01T08:51:22.117 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-test-2:16.2.14-44.geeb13ac9.el8.x86_64 6/6 2023-10-01T08:51:22.117 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-test-2:16.2.14-44.geeb13ac9.el8.x86_64 1/6 2023-10-01T08:51:22.117 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : libcephsqlite-2:16.2.14-44.geeb13ac9.el8.x86_64 2/6 2023-10-01T08:51:22.117 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-10-01T08:51:22.118 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-10-01T08:51:22.118 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-10-01T08:51:23.224 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-10-01T08:51:23.224 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:01:18 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:51:25.566 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: libcephsqlite-2:16.2.14-44.geeb13ac9.el8.x86_64 5/6 2023-10-01T08:51:25.821 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-test-2:16.2.14-44.geeb13ac9.el8.x86_64 6/6 2023-10-01T08:51:26.038 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-test-2:16.2.14-44.geeb13ac9.el8.x86_64 6/6 2023-10-01T08:51:26.039 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-test-2:16.2.14-44.geeb13ac9.el8.x86_64 1/6 2023-10-01T08:51:26.039 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libcephsqlite-2:16.2.14-44.geeb13ac9.el8.x86_64 2/6 2023-10-01T08:51:26.039 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-10-01T08:51:26.039 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-10-01T08:51:26.039 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-10-01T08:51:27.161 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-10-01T08:51:27.162 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:22 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:51:31.839 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:51:32.028 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:51:32.028 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:51:32.028 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-test-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:32.028 INFO:teuthology.orchestra.run.smithi190.stdout: jq-1.6-3.el8.x86_64 2023-10-01T08:51:32.029 INFO:teuthology.orchestra.run.smithi190.stdout: libcephsqlite-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:32.029 INFO:teuthology.orchestra.run.smithi190.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-01T08:51:32.029 INFO:teuthology.orchestra.run.smithi190.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-01T08:51:32.029 INFO:teuthology.orchestra.run.smithi190.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-01T08:51:32.029 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:51:32.029 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:51:32.168 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install ceph 2023-10-01T08:51:32.503 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:51:34.099 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:01:29 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:51:35.530 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:51:35.533 INFO:teuthology.orchestra.run.smithi190.stdout:====================================================================================================== 2023-10-01T08:51:35.533 INFO:teuthology.orchestra.run.smithi190.stdout: Package Arch Version Repository Size 2023-10-01T08:51:35.533 INFO:teuthology.orchestra.run.smithi190.stdout:====================================================================================================== 2023-10-01T08:51:35.534 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:51:35.534 INFO:teuthology.orchestra.run.smithi190.stdout: ceph x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 6.4 k 2023-10-01T08:51:35.534 INFO:teuthology.orchestra.run.smithi190.stdout:Installing dependencies: 2023-10-01T08:51:35.534 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mds x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 2.4 M 2023-10-01T08:51:35.534 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mgr x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 1.6 M 2023-10-01T08:51:35.534 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mgr-modules-core noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 244 k 2023-10-01T08:51:35.534 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mon x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 5.0 M 2023-10-01T08:51:35.534 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-osd x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 18 M 2023-10-01T08:51:35.535 INFO:teuthology.orchestra.run.smithi190.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-01T08:51:35.535 INFO:teuthology.orchestra.run.smithi190.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-01T08:51:35.535 INFO:teuthology.orchestra.run.smithi190.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-10-01T08:51:35.535 INFO:teuthology.orchestra.run.smithi190.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-01T08:51:35.535 INFO:teuthology.orchestra.run.smithi190.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-01T08:51:35.535 INFO:teuthology.orchestra.run.smithi190.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-10-01T08:51:35.535 INFO:teuthology.orchestra.run.smithi190.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-01T08:51:35.535 INFO:teuthology.orchestra.run.smithi190.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-01T08:51:35.535 INFO:teuthology.orchestra.run.smithi190.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-01T08:51:35.535 INFO:teuthology.orchestra.run.smithi190.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-10-01T08:51:35.535 INFO:teuthology.orchestra.run.smithi190.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-10-01T08:51:35.535 INFO:teuthology.orchestra.run.smithi190.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-01T08:51:35.536 INFO:teuthology.orchestra.run.smithi190.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-01T08:51:35.536 INFO:teuthology.orchestra.run.smithi190.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-01T08:51:35.536 INFO:teuthology.orchestra.run.smithi190.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-10-01T08:51:35.536 INFO:teuthology.orchestra.run.smithi190.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-10-01T08:51:35.536 INFO:teuthology.orchestra.run.smithi190.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-10-01T08:51:35.536 INFO:teuthology.orchestra.run.smithi190.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-01T08:51:35.536 INFO:teuthology.orchestra.run.smithi190.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-01T08:51:35.536 INFO:teuthology.orchestra.run.smithi190.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-01T08:51:35.536 INFO:teuthology.orchestra.run.smithi190.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-01T08:51:35.536 INFO:teuthology.orchestra.run.smithi190.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-01T08:51:35.536 INFO:teuthology.orchestra.run.smithi190.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-01T08:51:35.537 INFO:teuthology.orchestra.run.smithi190.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-01T08:51:35.537 INFO:teuthology.orchestra.run.smithi190.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-10-01T08:51:35.537 INFO:teuthology.orchestra.run.smithi190.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-01T08:51:35.537 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:51:35.537 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:51:35.537 INFO:teuthology.orchestra.run.smithi190.stdout:====================================================================================================== 2023-10-01T08:51:35.537 INFO:teuthology.orchestra.run.smithi190.stdout:Install 32 Packages 2023-10-01T08:51:35.537 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:51:35.538 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 31 M 2023-10-01T08:51:35.538 INFO:teuthology.orchestra.run.smithi190.stdout:Installed size: 101 M 2023-10-01T08:51:35.538 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:51:35.791 INFO:teuthology.orchestra.run.smithi190.stdout:(1/32): ceph-16.2.14-44.geeb13ac9.el8.x86_64.rp 33 kB/s | 6.4 kB 00:00 2023-10-01T08:51:36.091 INFO:teuthology.orchestra.run.smithi190.stdout:(2/32): ceph-mds-16.2.14-44.geeb13ac9.el8.x86_6 5.0 MB/s | 2.4 MB 00:00 2023-10-01T08:51:36.133 INFO:teuthology.orchestra.run.smithi190.stdout:(3/32): ceph-mgr-16.2.14-44.geeb13ac9.el8.x86_6 3.0 MB/s | 1.6 MB 00:00 2023-10-01T08:51:36.225 INFO:teuthology.orchestra.run.smithi190.stdout:(4/32): ceph-mgr-modules-core-16.2.14-44.geeb13 2.6 MB/s | 244 kB 00:00 2023-10-01T08:51:36.359 INFO:teuthology.orchestra.run.smithi190.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 327 kB/s | 44 kB 00:00 2023-10-01T08:51:36.440 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:51:36.442 INFO:teuthology.orchestra.run.smithi190.stdout:(6/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 2.2 MB/s | 185 kB 00:00 2023-10-01T08:51:36.526 INFO:teuthology.orchestra.run.smithi190.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 2.0 MB/s | 173 kB 00:00 2023-10-01T08:51:36.622 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:51:36.623 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:51:36.623 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-test-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:36.623 INFO:teuthology.orchestra.run.smithi138.stdout: jq-1.6-3.el8.x86_64 2023-10-01T08:51:36.623 INFO:teuthology.orchestra.run.smithi138.stdout: libcephsqlite-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:36.623 INFO:teuthology.orchestra.run.smithi138.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-10-01T08:51:36.623 INFO:teuthology.orchestra.run.smithi138.stdout: socat-1.7.4.1-1.el8.x86_64 2023-10-01T08:51:36.623 INFO:teuthology.orchestra.run.smithi138.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-10-01T08:51:36.623 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:51:36.623 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:51:36.651 INFO:teuthology.orchestra.run.smithi190.stdout:(8/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 3.0 MB/s | 384 kB 00:00 2023-10-01T08:51:36.694 INFO:teuthology.orchestra.run.smithi190.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 266 kB/s | 11 kB 00:00 2023-10-01T08:51:36.735 INFO:teuthology.orchestra.run.smithi190.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 423 kB/s | 18 kB 00:00 2023-10-01T08:51:36.773 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph 2023-10-01T08:51:36.777 INFO:teuthology.orchestra.run.smithi190.stdout:(11/32): python3-logutils-0.3.5-11.el8.noarch.r 1.1 MB/s | 49 kB 00:00 2023-10-01T08:51:36.819 INFO:teuthology.orchestra.run.smithi190.stdout:(12/32): python3-more-itertools-7.2.0-3.el8.noa 1.4 MB/s | 59 kB 00:00 2023-10-01T08:51:37.045 INFO:teuthology.orchestra.run.smithi190.stdout:(13/32): ceph-osd-16.2.14-44.geeb13ac9.el8.x86_ 19 MB/s | 18 MB 00:00 2023-10-01T08:51:37.070 INFO:teuthology.orchestra.run.smithi190.stdout:(14/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.1 MB/s | 283 kB 00:00 2023-10-01T08:51:37.106 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:51:37.137 INFO:teuthology.orchestra.run.smithi190.stdout:(15/32): ceph-mon-16.2.14-44.geeb13ac9.el8.x86_ 3.7 MB/s | 5.0 MB 00:01 2023-10-01T08:51:37.162 INFO:teuthology.orchestra.run.smithi190.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 213 kB/s | 19 kB 00:00 2023-10-01T08:51:37.212 INFO:teuthology.orchestra.run.smithi190.stdout:(17/32): python3-tempora-1.14.1-5.el8.noarch.rp 643 kB/s | 29 kB 00:00 2023-10-01T08:51:37.238 INFO:teuthology.orchestra.run.smithi190.stdout:(18/32): python3-portend-2.6-1.el8.noarch.rpm 81 kB/s | 16 kB 00:00 2023-10-01T08:51:37.263 INFO:teuthology.orchestra.run.smithi190.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 548 kB/s | 27 kB 00:00 2023-10-01T08:51:37.288 INFO:teuthology.orchestra.run.smithi190.stdout:(20/32): python3-singledispatch-3.4.0.3-18.el8. 159 kB/s | 24 kB 00:00 2023-10-01T08:51:37.347 INFO:teuthology.orchestra.run.smithi190.stdout:(21/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.4 MB/s | 85 kB 00:00 2023-10-01T08:51:37.372 INFO:teuthology.orchestra.run.smithi190.stdout:(22/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.9 MB/s | 255 kB 00:00 2023-10-01T08:51:37.397 INFO:teuthology.orchestra.run.smithi190.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 459 kB/s | 23 kB 00:00 2023-10-01T08:51:37.423 INFO:teuthology.orchestra.run.smithi190.stdout:(24/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.5 MB/s | 251 kB 00:00 2023-10-01T08:51:37.556 INFO:teuthology.orchestra.run.smithi190.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 595 kB/s | 109 kB 00:00 2023-10-01T08:51:37.581 INFO:teuthology.orchestra.run.smithi190.stdout:(26/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.5 MB/s | 238 kB 00:00 2023-10-01T08:51:37.615 INFO:teuthology.orchestra.run.smithi190.stdout:(27/32): python3-cryptography-3.2.1-5.el8.x86_6 2.5 MB/s | 559 kB 00:00 2023-10-01T08:51:37.724 INFO:teuthology.orchestra.run.smithi190.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 274 kB/s | 39 kB 00:00 2023-10-01T08:51:37.749 INFO:teuthology.orchestra.run.smithi190.stdout:(29/32): python3-pytz-2017.2-9.el8.noarch.rpm 281 kB/s | 54 kB 00:00 2023-10-01T08:51:37.791 INFO:teuthology.orchestra.run.smithi190.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 895 kB/s | 157 kB 00:00 2023-10-01T08:51:37.891 INFO:teuthology.orchestra.run.smithi190.stdout:(31/32): python3-werkzeug-0.12.2-4.el8.noarch.r 2.7 MB/s | 457 kB 00:00 2023-10-01T08:51:37.924 INFO:teuthology.orchestra.run.smithi190.stdout:(32/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 591 kB/s | 103 kB 00:00 2023-10-01T08:51:37.929 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:51:37.929 INFO:teuthology.orchestra.run.smithi190.stdout:Total 13 MB/s | 31 MB 00:02 2023-10-01T08:51:38.019 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:51:38.057 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:51:38.057 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:51:38.272 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:51:38.276 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:51:38.659 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:33 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:51:39.353 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:51:39.562 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-01T08:51:39.848 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-01T08:51:40.015 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:51:40.019 INFO:teuthology.orchestra.run.smithi138.stdout:====================================================================================================== 2023-10-01T08:51:40.019 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-10-01T08:51:40.019 INFO:teuthology.orchestra.run.smithi138.stdout:====================================================================================================== 2023-10-01T08:51:40.019 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:51:40.019 INFO:teuthology.orchestra.run.smithi138.stdout: ceph x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 6.4 k 2023-10-01T08:51:40.019 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-10-01T08:51:40.019 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mds x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 2.4 M 2023-10-01T08:51:40.020 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 1.6 M 2023-10-01T08:51:40.020 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-modules-core noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 244 k 2023-10-01T08:51:40.020 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mon x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 5.0 M 2023-10-01T08:51:40.020 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-osd x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 18 M 2023-10-01T08:51:40.020 INFO:teuthology.orchestra.run.smithi138.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-10-01T08:51:40.020 INFO:teuthology.orchestra.run.smithi138.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-10-01T08:51:40.020 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-10-01T08:51:40.020 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-10-01T08:51:40.020 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-10-01T08:51:40.020 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-10-01T08:51:40.021 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-10-01T08:51:40.021 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-10-01T08:51:40.021 INFO:teuthology.orchestra.run.smithi138.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-10-01T08:51:40.021 INFO:teuthology.orchestra.run.smithi138.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-10-01T08:51:40.021 INFO:teuthology.orchestra.run.smithi138.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-10-01T08:51:40.021 INFO:teuthology.orchestra.run.smithi138.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-10-01T08:51:40.021 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-10-01T08:51:40.021 INFO:teuthology.orchestra.run.smithi138.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-10-01T08:51:40.021 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-10-01T08:51:40.021 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-10-01T08:51:40.021 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-10-01T08:51:40.021 INFO:teuthology.orchestra.run.smithi138.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-10-01T08:51:40.022 INFO:teuthology.orchestra.run.smithi138.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-10-01T08:51:40.022 INFO:teuthology.orchestra.run.smithi138.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-10-01T08:51:40.022 INFO:teuthology.orchestra.run.smithi138.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-10-01T08:51:40.022 INFO:teuthology.orchestra.run.smithi138.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-10-01T08:51:40.022 INFO:teuthology.orchestra.run.smithi138.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-10-01T08:51:40.022 INFO:teuthology.orchestra.run.smithi138.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-10-01T08:51:40.022 INFO:teuthology.orchestra.run.smithi138.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-10-01T08:51:40.022 INFO:teuthology.orchestra.run.smithi138.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-10-01T08:51:40.022 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:51:40.022 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:51:40.022 INFO:teuthology.orchestra.run.smithi138.stdout:====================================================================================================== 2023-10-01T08:51:40.023 INFO:teuthology.orchestra.run.smithi138.stdout:Install 32 Packages 2023-10-01T08:51:40.023 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:51:40.023 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 31 M 2023-10-01T08:51:40.023 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 101 M 2023-10-01T08:51:40.023 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:51:40.119 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-10-01T08:51:40.205 INFO:teuthology.orchestra.run.smithi138.stdout:(1/32): ceph-16.2.14-44.geeb13ac9.el8.x86_64.rp 42 kB/s | 6.4 kB 00:00 2023-10-01T08:51:40.350 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-10-01T08:51:40.380 INFO:teuthology.orchestra.run.smithi138.stdout:(2/32): ceph-mgr-16.2.14-44.geeb13ac9.el8.x86_6 4.9 MB/s | 1.6 MB 00:00 2023-10-01T08:51:40.431 INFO:teuthology.orchestra.run.smithi138.stdout:(3/32): ceph-mds-16.2.14-44.geeb13ac9.el8.x86_6 6.5 MB/s | 2.4 MB 00:00 2023-10-01T08:51:40.531 INFO:teuthology.orchestra.run.smithi138.stdout:(4/32): ceph-mgr-modules-core-16.2.14-44.geeb13 2.4 MB/s | 244 kB 00:00 2023-10-01T08:51:40.572 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-10-01T08:51:40.632 INFO:teuthology.orchestra.run.smithi138.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 436 kB/s | 44 kB 00:00 2023-10-01T08:51:40.715 INFO:teuthology.orchestra.run.smithi138.stdout:(6/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 2.2 MB/s | 185 kB 00:00 2023-10-01T08:51:40.752 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-10-01T08:51:40.765 INFO:teuthology.orchestra.run.smithi138.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 3.4 MB/s | 173 kB 00:00 2023-10-01T08:51:40.849 INFO:teuthology.orchestra.run.smithi138.stdout:(8/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 4.5 MB/s | 384 kB 00:00 2023-10-01T08:51:40.891 INFO:teuthology.orchestra.run.smithi138.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 266 kB/s | 11 kB 00:00 2023-10-01T08:51:40.933 INFO:teuthology.orchestra.run.smithi138.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 421 kB/s | 18 kB 00:00 2023-10-01T08:51:40.967 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-10-01T08:51:41.025 INFO:teuthology.orchestra.run.smithi138.stdout:(11/32): ceph-mon-16.2.14-44.geeb13ac9.el8.x86_ 6.1 MB/s | 5.0 MB 00:00 2023-10-01T08:51:41.050 INFO:teuthology.orchestra.run.smithi138.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 416 kB/s | 49 kB 00:00 2023-10-01T08:51:41.100 INFO:teuthology.orchestra.run.smithi138.stdout:(13/32): python3-pecan-1.3.2-9.el8.noarch.rpm 5.5 MB/s | 283 kB 00:00 2023-10-01T08:51:41.165 INFO:teuthology.orchestra.run.smithi138.stdout:(14/32): python3-more-itertools-7.2.0-3.el8.noa 419 kB/s | 59 kB 00:00 2023-10-01T08:51:41.201 INFO:teuthology.orchestra.run.smithi138.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 155 kB/s | 16 kB 00:00 2023-10-01T08:51:41.218 INFO:teuthology.orchestra.run.smithi138.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 357 kB/s | 19 kB 00:00 2023-10-01T08:51:41.243 INFO:teuthology.orchestra.run.smithi138.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 568 kB/s | 24 kB 00:00 2023-10-01T08:51:41.269 INFO:teuthology.orchestra.run.smithi138.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 580 kB/s | 29 kB 00:00 2023-10-01T08:51:41.285 INFO:teuthology.orchestra.run.smithi138.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 658 kB/s | 27 kB 00:00 2023-10-01T08:51:41.320 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-10-01T08:51:41.461 INFO:teuthology.orchestra.run.smithi138.stdout:(20/32): ceph-osd-16.2.14-44.geeb13ac9.el8.x86_ 17 MB/s | 18 MB 00:01 2023-10-01T08:51:41.519 INFO:teuthology.orchestra.run.smithi138.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.0 MB/s | 255 kB 00:00 2023-10-01T08:51:41.544 INFO:teuthology.orchestra.run.smithi138.stdout:(22/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 970 kB/s | 251 kB 00:00 2023-10-01T08:51:41.577 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-10-01T08:51:41.578 INFO:teuthology.orchestra.run.smithi138.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 692 kB/s | 23 kB 00:00 2023-10-01T08:51:41.645 INFO:teuthology.orchestra.run.smithi138.stdout:(24/32): python3-webtest-2.0.33-1.el8.noarch.rp 464 kB/s | 85 kB 00:00 2023-10-01T08:51:41.670 INFO:teuthology.orchestra.run.smithi138.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 870 kB/s | 109 kB 00:00 2023-10-01T08:51:41.704 INFO:teuthology.orchestra.run.smithi138.stdout:(26/32): python3-cryptography-3.2.1-5.el8.x86_6 4.3 MB/s | 559 kB 00:00 2023-10-01T08:51:41.746 INFO:teuthology.orchestra.run.smithi138.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 775 kB/s | 54 kB 00:00 2023-10-01T08:51:41.765 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-10-01T08:51:41.832 INFO:teuthology.orchestra.run.smithi138.stdout:(28/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.2 MB/s | 238 kB 00:00 2023-10-01T08:51:41.863 INFO:teuthology.orchestra.run.smithi138.stdout:(29/32): python3-markupsafe-0.23-19.el8.x86_64. 244 kB/s | 39 kB 00:00 2023-10-01T08:51:41.888 INFO:teuthology.orchestra.run.smithi138.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.1 MB/s | 157 kB 00:00 2023-10-01T08:51:41.957 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-10-01T08:51:41.997 INFO:teuthology.orchestra.run.smithi138.stdout:(31/32): python3-werkzeug-0.12.2-4.el8.noarch.r 2.7 MB/s | 457 kB 00:00 2023-10-01T08:51:42.081 INFO:teuthology.orchestra.run.smithi138.stdout:(32/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 477 kB/s | 103 kB 00:00 2023-10-01T08:51:42.082 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:51:42.082 INFO:teuthology.orchestra.run.smithi138.stdout:Total 15 MB/s | 31 MB 00:02 2023-10-01T08:51:42.157 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-10-01T08:51:42.169 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:51:42.206 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:51:42.206 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:51:42.365 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-10-01T08:51:42.419 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:51:42.423 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:51:42.538 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-10-01T08:51:42.728 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-10-01T08:51:42.882 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-10-01T08:51:43.068 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-10-01T08:51:43.204 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-10-01T08:51:43.398 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-10-01T08:51:43.581 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:51:43.618 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-10-01T08:51:43.787 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-01T08:51:43.827 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-10-01T08:51:44.042 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-01T08:51:44.146 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-10-01T08:51:44.303 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-01T08:51:44.392 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-10-01T08:51:44.491 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-10-01T08:51:44.682 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-10-01T08:51:44.770 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-10-01T08:51:44.936 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-10-01T08:51:45.068 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-01T08:51:45.133 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-10-01T08:51:45.423 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-10-01T08:51:45.626 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-mgr-modules-core-2:16.2.14-44.geeb13ac9.el8.n 27/32 2023-10-01T08:51:45.783 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-mgr-2:16.2.14-44.geeb13ac9.el8.x86_64 28/32 2023-10-01T08:51:45.809 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-10-01T08:51:46.107 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-10-01T08:51:46.304 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-10-01T08:51:46.513 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-10-01T08:51:46.763 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-10-01T08:51:46.985 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-10-01T08:51:47.178 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-10-01T08:51:47.278 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-mgr-2:16.2.14-44.geeb13ac9.el8.x86_64 28/32 2023-10-01T08:51:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-10-01T08:51:47.490 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-osd-2:16.2.14-44.geeb13ac9.el8.x86_64 29/32 2023-10-01T08:51:47.657 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-10-01T08:51:47.849 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-10-01T08:51:47.932 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-osd-2:16.2.14-44.geeb13ac9.el8.x86_64 29/32 2023-10-01T08:51:48.043 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-10-01T08:51:48.073 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-mon-2:16.2.14-44.geeb13ac9.el8.x86_64 30/32 2023-10-01T08:51:48.283 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-10-01T08:51:48.314 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-mon-2:16.2.14-44.geeb13ac9.el8.x86_64 30/32 2023-10-01T08:51:48.465 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-mds-2:16.2.14-44.geeb13ac9.el8.x86_64 31/32 2023-10-01T08:51:48.483 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-mds-2:16.2.14-44.geeb13ac9.el8.x86_64 31/32 2023-10-01T08:51:48.507 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-10-01T08:51:48.604 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-2:16.2.14-44.geeb13ac9.el8.x86_64 32/32 2023-10-01T08:51:48.734 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-10-01T08:51:48.982 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-2:16.2.14-44.geeb13ac9.el8.x86_64 32/32 2023-10-01T08:51:48.982 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-2:16.2.14-44.geeb13ac9.el8.x86_64 1/32 2023-10-01T08:51:48.982 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-mds-2:16.2.14-44.geeb13ac9.el8.x86_64 2/32 2023-10-01T08:51:48.982 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-mgr-2:16.2.14-44.geeb13ac9.el8.x86_64 3/32 2023-10-01T08:51:48.982 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-mon-2:16.2.14-44.geeb13ac9.el8.x86_64 4/32 2023-10-01T08:51:48.982 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-osd-2:16.2.14-44.geeb13ac9.el8.x86_64 5/32 2023-10-01T08:51:48.983 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-mgr-modules-core-2:16.2.14-44.geeb13ac9.el8.n 6/32 2023-10-01T08:51:48.983 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-10-01T08:51:48.983 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-10-01T08:51:48.983 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-10-01T08:51:48.983 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-10-01T08:51:48.983 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-10-01T08:51:48.983 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-10-01T08:51:48.983 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-10-01T08:51:48.983 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-10-01T08:51:48.983 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-10-01T08:51:48.984 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-10-01T08:51:48.984 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-10-01T08:51:48.984 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-10-01T08:51:48.984 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-10-01T08:51:48.984 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-10-01T08:51:48.984 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-10-01T08:51:48.984 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-10-01T08:51:48.984 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-10-01T08:51:48.984 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-10-01T08:51:48.984 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-10-01T08:51:48.984 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-10-01T08:51:48.985 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-10-01T08:51:48.986 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-10-01T08:51:48.986 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-10-01T08:51:48.986 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-10-01T08:51:48.986 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-10-01T08:51:49.057 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-10-01T08:51:49.393 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-10-01T08:51:49.606 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-10-01T08:51:49.910 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-10-01T08:51:50.290 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-10-01T08:51:50.380 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-10-01T08:51:50.380 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:01:45 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:51:50.623 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mgr-modules-core-2:16.2.14-44.geeb13ac9.el8.n 27/32 2023-10-01T08:51:50.780 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mgr-2:16.2.14-44.geeb13ac9.el8.x86_64 28/32 2023-10-01T08:51:52.284 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-mgr-2:16.2.14-44.geeb13ac9.el8.x86_64 28/32 2023-10-01T08:51:52.487 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-osd-2:16.2.14-44.geeb13ac9.el8.x86_64 29/32 2023-10-01T08:51:52.943 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-osd-2:16.2.14-44.geeb13ac9.el8.x86_64 29/32 2023-10-01T08:51:53.095 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mon-2:16.2.14-44.geeb13ac9.el8.x86_64 30/32 2023-10-01T08:51:53.325 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-mon-2:16.2.14-44.geeb13ac9.el8.x86_64 30/32 2023-10-01T08:51:53.470 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mds-2:16.2.14-44.geeb13ac9.el8.x86_64 31/32 2023-10-01T08:51:53.488 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-mds-2:16.2.14-44.geeb13ac9.el8.x86_64 31/32 2023-10-01T08:51:53.651 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-2:16.2.14-44.geeb13ac9.el8.x86_64 32/32 2023-10-01T08:51:54.012 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-2:16.2.14-44.geeb13ac9.el8.x86_64 32/32 2023-10-01T08:51:54.012 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-2:16.2.14-44.geeb13ac9.el8.x86_64 1/32 2023-10-01T08:51:54.013 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mds-2:16.2.14-44.geeb13ac9.el8.x86_64 2/32 2023-10-01T08:51:54.013 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mgr-2:16.2.14-44.geeb13ac9.el8.x86_64 3/32 2023-10-01T08:51:54.013 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mon-2:16.2.14-44.geeb13ac9.el8.x86_64 4/32 2023-10-01T08:51:54.013 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-osd-2:16.2.14-44.geeb13ac9.el8.x86_64 5/32 2023-10-01T08:51:54.013 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mgr-modules-core-2:16.2.14-44.geeb13ac9.el8.n 6/32 2023-10-01T08:51:54.013 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-10-01T08:51:54.013 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-10-01T08:51:54.013 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-10-01T08:51:54.013 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-10-01T08:51:54.013 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-10-01T08:51:54.013 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-10-01T08:51:54.014 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-10-01T08:51:54.014 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-10-01T08:51:54.014 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-10-01T08:51:54.014 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-10-01T08:51:54.014 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-10-01T08:51:54.014 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-10-01T08:51:54.014 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-10-01T08:51:54.014 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-10-01T08:51:54.014 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-10-01T08:51:54.014 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-10-01T08:51:54.014 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-10-01T08:51:54.014 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-10-01T08:51:54.015 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-10-01T08:51:54.015 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-10-01T08:51:54.015 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-10-01T08:51:54.015 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-10-01T08:51:54.015 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-10-01T08:51:54.016 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-10-01T08:51:54.016 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-10-01T08:51:55.578 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-10-01T08:51:55.578 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:50 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:51:58.861 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:51:59.046 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:51:59.046 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:51:59.046 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:59.046 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mds-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:59.047 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mgr-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:59.047 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mgr-modules-core-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:51:59.047 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mon-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:59.047 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-osd-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:51:59.047 INFO:teuthology.orchestra.run.smithi190.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-01T08:51:59.047 INFO:teuthology.orchestra.run.smithi190.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-01T08:51:59.047 INFO:teuthology.orchestra.run.smithi190.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-10-01T08:51:59.047 INFO:teuthology.orchestra.run.smithi190.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-01T08:51:59.047 INFO:teuthology.orchestra.run.smithi190.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-01T08:51:59.047 INFO:teuthology.orchestra.run.smithi190.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-10-01T08:51:59.048 INFO:teuthology.orchestra.run.smithi190.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-01T08:51:59.048 INFO:teuthology.orchestra.run.smithi190.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-01T08:51:59.048 INFO:teuthology.orchestra.run.smithi190.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-01T08:51:59.048 INFO:teuthology.orchestra.run.smithi190.stdout: python3-mako-1.0.6-13.el8.noarch 2023-10-01T08:51:59.048 INFO:teuthology.orchestra.run.smithi190.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-01T08:51:59.048 INFO:teuthology.orchestra.run.smithi190.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-01T08:51:59.048 INFO:teuthology.orchestra.run.smithi190.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-01T08:51:59.048 INFO:teuthology.orchestra.run.smithi190.stdout: python3-portend-2.6-1.el8.noarch 2023-10-01T08:51:59.048 INFO:teuthology.orchestra.run.smithi190.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-01T08:51:59.049 INFO:teuthology.orchestra.run.smithi190.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-01T08:51:59.049 INFO:teuthology.orchestra.run.smithi190.stdout: python3-pytz-2017.2-9.el8.noarch 2023-10-01T08:51:59.049 INFO:teuthology.orchestra.run.smithi190.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-01T08:51:59.049 INFO:teuthology.orchestra.run.smithi190.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-01T08:51:59.049 INFO:teuthology.orchestra.run.smithi190.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-01T08:51:59.049 INFO:teuthology.orchestra.run.smithi190.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-01T08:51:59.049 INFO:teuthology.orchestra.run.smithi190.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-01T08:51:59.049 INFO:teuthology.orchestra.run.smithi190.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-01T08:51:59.049 INFO:teuthology.orchestra.run.smithi190.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-01T08:51:59.049 INFO:teuthology.orchestra.run.smithi190.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-01T08:51:59.050 INFO:teuthology.orchestra.run.smithi190.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-01T08:51:59.050 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:51:59.050 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:51:59.230 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install ceph-base 2023-10-01T08:51:59.574 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:52:00.990 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:01:55 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:01.250 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:52:01.441 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:01.441 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:52:01.441 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:52:01.441 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mds-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:52:01.441 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:52:01.441 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-modules-core-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:52:01.442 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mon-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:52:01.442 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-osd-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:52:01.442 INFO:teuthology.orchestra.run.smithi138.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-10-01T08:52:01.442 INFO:teuthology.orchestra.run.smithi138.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-10-01T08:52:01.442 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-10-01T08:52:01.442 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-10-01T08:52:01.442 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-10-01T08:52:01.442 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-10-01T08:52:01.442 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jaraco-6.2-6.el8.noarch 2023-10-01T08:52:01.442 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-10-01T08:52:01.442 INFO:teuthology.orchestra.run.smithi138.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-10-01T08:52:01.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-mako-1.0.6-13.el8.noarch 2023-10-01T08:52:01.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-10-01T08:52:01.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-10-01T08:52:01.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-10-01T08:52:01.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-portend-2.6-1.el8.noarch 2023-10-01T08:52:01.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-10-01T08:52:01.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pycparser-2.14-14.el8.noarch 2023-10-01T08:52:01.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pytz-2017.2-9.el8.noarch 2023-10-01T08:52:01.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-10-01T08:52:01.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-10-01T08:52:01.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-10-01T08:52:01.444 INFO:teuthology.orchestra.run.smithi138.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-10-01T08:52:01.444 INFO:teuthology.orchestra.run.smithi138.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-10-01T08:52:01.444 INFO:teuthology.orchestra.run.smithi138.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-10-01T08:52:01.444 INFO:teuthology.orchestra.run.smithi138.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-10-01T08:52:01.444 INFO:teuthology.orchestra.run.smithi138.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-10-01T08:52:01.444 INFO:teuthology.orchestra.run.smithi138.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-10-01T08:52:01.444 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:01.444 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:52:01.603 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-base 2023-10-01T08:52:01.949 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:52:02.246 INFO:teuthology.orchestra.run.smithi190.stdout:Package ceph-base-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:52:02.353 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:52:02.355 INFO:teuthology.orchestra.run.smithi190.stdout:Nothing to do. 2023-10-01T08:52:02.355 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:52:02.477 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install cephadm 2023-10-01T08:52:02.819 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:52:03.365 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:01:58 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:04.457 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:01:59 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:04.625 INFO:teuthology.orchestra.run.smithi138.stdout:Package ceph-base-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:52:04.734 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:52:04.735 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-10-01T08:52:04.736 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:52:04.859 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install cephadm 2023-10-01T08:52:05.203 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:52:05.880 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:52:05.881 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:52:05.881 INFO:teuthology.orchestra.run.smithi190.stdout: Package Arch Version Repository Size 2023-10-01T08:52:05.881 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:52:05.881 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:52:05.881 INFO:teuthology.orchestra.run.smithi190.stdout: cephadm noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 84 k 2023-10-01T08:52:05.881 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:52:05.882 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:52:05.882 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:52:05.882 INFO:teuthology.orchestra.run.smithi190.stdout:Install 1 Package 2023-10-01T08:52:05.882 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:52:05.882 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 84 k 2023-10-01T08:52:05.882 INFO:teuthology.orchestra.run.smithi190.stdout:Installed size: 348 k 2023-10-01T08:52:05.882 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:52:06.062 INFO:teuthology.orchestra.run.smithi190.stdout:cephadm-16.2.14-44.geeb13ac9.el8.noarch.rpm 468 kB/s | 84 kB 00:00 2023-10-01T08:52:06.062 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:52:06.063 INFO:teuthology.orchestra.run.smithi190.stdout:Total 465 kB/s | 84 kB 00:00 2023-10-01T08:52:06.063 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:52:06.069 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:52:06.070 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:52:06.071 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:52:06.075 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:52:06.114 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:52:06.628 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:01 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:07.765 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 1/1 2023-10-01T08:52:07.917 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 1/1 2023-10-01T08:52:07.980 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:52:07.980 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:52:07.980 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-10-01T08:52:07.980 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:52:07.980 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:52:07.981 INFO:teuthology.orchestra.run.smithi138.stdout: cephadm noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 84 k 2023-10-01T08:52:07.981 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:07.981 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:52:07.981 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:52:07.981 INFO:teuthology.orchestra.run.smithi138.stdout:Install 1 Package 2023-10-01T08:52:07.981 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:07.982 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 84 k 2023-10-01T08:52:07.982 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 348 k 2023-10-01T08:52:07.982 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:52:08.085 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 1/1 2023-10-01T08:52:08.229 INFO:teuthology.orchestra.run.smithi138.stdout:cephadm-16.2.14-44.geeb13ac9.el8.noarch.rpm 341 kB/s | 84 kB 00:00 2023-10-01T08:52:08.229 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:52:08.230 INFO:teuthology.orchestra.run.smithi138.stdout:Total 339 kB/s | 84 kB 00:00 2023-10-01T08:52:08.230 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:52:08.237 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:52:08.237 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:52:08.239 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:52:08.242 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:52:08.314 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:52:09.171 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 1/1 2023-10-01T08:52:09.171 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:02:04 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:10.031 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 1/1 2023-10-01T08:52:10.180 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 1/1 2023-10-01T08:52:10.351 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 1/1 2023-10-01T08:52:11.440 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 1/1 2023-10-01T08:52:11.440 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:06 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:14.257 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:52:14.439 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:52:14.439 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:52:14.439 INFO:teuthology.orchestra.run.smithi190.stdout: cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:52:14.439 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:52:14.439 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:52:14.569 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install ceph-immutable-object-cache 2023-10-01T08:52:14.926 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:52:16.425 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:02:11 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:17.538 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:52:17.721 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:17.721 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:52:17.721 INFO:teuthology.orchestra.run.smithi138.stdout: cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:52:17.721 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:17.721 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:52:17.787 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:52:17.788 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:52:17.788 INFO:teuthology.orchestra.run.smithi190.stdout: Package Arch Version Repo Size 2023-10-01T08:52:17.788 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:52:17.788 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:52:17.788 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-immutable-object-cache x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 161 k 2023-10-01T08:52:17.788 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:52:17.788 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:52:17.788 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:52:17.788 INFO:teuthology.orchestra.run.smithi190.stdout:Install 1 Package 2023-10-01T08:52:17.789 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:52:17.789 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 161 k 2023-10-01T08:52:17.789 INFO:teuthology.orchestra.run.smithi190.stdout:Installed size: 470 k 2023-10-01T08:52:17.789 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:52:17.860 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-immutable-object-cache 2023-10-01T08:52:18.217 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:52:18.528 INFO:teuthology.orchestra.run.smithi190.stdout:ceph-immutable-object-cache-16.2.14-44.geeb13ac 218 kB/s | 161 kB 00:00 2023-10-01T08:52:18.528 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:52:18.529 INFO:teuthology.orchestra.run.smithi190.stdout:Total 218 kB/s | 161 kB 00:00 2023-10-01T08:52:18.529 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:52:18.541 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:52:18.542 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:52:18.598 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:52:18.602 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:52:18.888 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:52:19.060 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-immutable-object-cache-2:16.2.14-44.geeb13ac9.e 1/1 2023-10-01T08:52:19.404 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.14-44.geeb13ac9.e 1/1 2023-10-01T08:52:19.830 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:14 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:20.427 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-immutable-object-cache-2:16.2.14-44.geeb13ac9.e 1/1 2023-10-01T08:52:20.427 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:02:15 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:21.197 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:52:21.197 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:52:21.197 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repo Size 2023-10-01T08:52:21.197 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:52:21.197 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:52:21.198 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-immutable-object-cache x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 161 k 2023-10-01T08:52:21.198 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:21.198 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:52:21.198 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:52:21.198 INFO:teuthology.orchestra.run.smithi138.stdout:Install 1 Package 2023-10-01T08:52:21.198 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:21.199 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 161 k 2023-10-01T08:52:21.199 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 470 k 2023-10-01T08:52:21.199 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:52:21.394 INFO:teuthology.orchestra.run.smithi138.stdout:ceph-immutable-object-cache-16.2.14-44.geeb13ac 827 kB/s | 161 kB 00:00 2023-10-01T08:52:21.394 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:52:21.394 INFO:teuthology.orchestra.run.smithi138.stdout:Total 822 kB/s | 161 kB 00:00 2023-10-01T08:52:21.395 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:52:21.405 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:52:21.405 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:52:21.456 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:52:21.459 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:52:21.605 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:52:21.840 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-immutable-object-cache-2:16.2.14-44.geeb13ac9.e 1/1 2023-10-01T08:52:22.203 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.14-44.geeb13ac9.e 1/1 2023-10-01T08:52:23.243 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-immutable-object-cache-2:16.2.14-44.geeb13ac9.e 1/1 2023-10-01T08:52:23.244 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:18 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:25.075 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:52:25.259 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:52:25.259 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:52:25.259 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-immutable-object-cache-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:52:25.259 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:52:25.259 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:52:25.399 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install ceph-mgr 2023-10-01T08:52:25.748 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:52:27.130 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:02:22 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:28.106 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:52:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:28.288 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:52:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-immutable-object-cache-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:52:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:28.288 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:52:28.402 INFO:teuthology.orchestra.run.smithi190.stdout:Package ceph-mgr-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:52:28.456 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-mgr 2023-10-01T08:52:28.511 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:52:28.512 INFO:teuthology.orchestra.run.smithi190.stdout:Nothing to do. 2023-10-01T08:52:28.512 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:52:28.693 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install ceph-mgr-dashboard 2023-10-01T08:52:28.800 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:52:29.035 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:52:30.408 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:25 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:30.555 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:02:25 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:31.658 INFO:teuthology.orchestra.run.smithi138.stdout:Package ceph-mgr-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:52:31.766 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:52:31.767 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-10-01T08:52:31.767 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:52:31.891 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-mgr-dashboard 2023-10-01T08:52:31.917 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:52:31.918 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================================== 2023-10-01T08:52:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: Package Arch Version Repository Size 2023-10-01T08:52:31.918 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================================== 2023-10-01T08:52:31.918 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:52:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mgr-dashboard noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 3.1 M 2023-10-01T08:52:31.919 INFO:teuthology.orchestra.run.smithi190.stdout:Installing dependencies: 2023-10-01T08:52:31.919 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-grafana-dashboards noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 24 k 2023-10-01T08:52:31.919 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-prometheus-alerts noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 15 k 2023-10-01T08:52:31.919 INFO:teuthology.orchestra.run.smithi190.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-01T08:52:31.919 INFO:teuthology.orchestra.run.smithi190.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-01T08:52:31.919 INFO:teuthology.orchestra.run.smithi190.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-01T08:52:31.919 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:52:31.919 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:52:31.919 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================================== 2023-10-01T08:52:31.919 INFO:teuthology.orchestra.run.smithi190.stdout:Install 6 Packages 2023-10-01T08:52:31.919 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:52:31.920 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 3.4 M 2023-10-01T08:52:31.920 INFO:teuthology.orchestra.run.smithi190.stdout:Installed size: 22 M 2023-10-01T08:52:31.920 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:52:32.222 INFO:teuthology.orchestra.run.smithi190.stdout:(1/6): ceph-prometheus-alerts-16.2.14-44.geeb13 107 kB/s | 15 kB 00:00 2023-10-01T08:52:32.234 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:52:32.322 INFO:teuthology.orchestra.run.smithi190.stdout:(2/6): ceph-grafana-dashboards-16.2.14-44.geeb1 102 kB/s | 24 kB 00:00 2023-10-01T08:52:32.347 INFO:teuthology.orchestra.run.smithi190.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 269 kB/s | 34 kB 00:00 2023-10-01T08:52:32.539 INFO:teuthology.orchestra.run.smithi190.stdout:(4/6): python3-routes-2.4.1-12.el8.noarch.rpm 905 kB/s | 196 kB 00:00 2023-10-01T08:52:32.564 INFO:teuthology.orchestra.run.smithi190.stdout:(5/6): python3-jwt-1.6.1-2.el8.noarch.rpm 196 kB/s | 43 kB 00:00 2023-10-01T08:52:32.623 INFO:teuthology.orchestra.run.smithi190.stdout:(6/6): ceph-mgr-dashboard-16.2.14-44.geeb13ac9. 5.8 MB/s | 3.1 MB 00:00 2023-10-01T08:52:32.624 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:52:32.625 INFO:teuthology.orchestra.run.smithi190.stdout:Total 4.9 MB/s | 3.4 MB 00:00 2023-10-01T08:52:32.636 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:52:32.647 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:52:32.647 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:52:32.715 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:52:32.719 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:52:33.017 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:52:33.175 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-10-01T08:52:33.355 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-10-01T08:52:33.543 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-10-01T08:52:33.672 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-prometheus-alerts-2:16.2.14-44.geeb13ac9.el8.no 4/6 2023-10-01T08:52:33.715 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:28 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:34.707 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-grafana-dashboards-2:16.2.14-44.geeb13ac9.el8.n 5/6 2023-10-01T08:52:34.950 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-mgr-dashboard-2:16.2.14-44.geeb13ac9.el8.noarch 6/6 2023-10-01T08:52:35.156 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:52:35.157 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================================== 2023-10-01T08:52:35.158 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-10-01T08:52:35.158 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================================== 2023-10-01T08:52:35.158 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:52:35.158 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-dashboard noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 3.1 M 2023-10-01T08:52:35.158 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-10-01T08:52:35.158 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-grafana-dashboards noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 24 k 2023-10-01T08:52:35.158 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-prometheus-alerts noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 15 k 2023-10-01T08:52:35.158 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-01T08:52:35.158 INFO:teuthology.orchestra.run.smithi138.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-10-01T08:52:35.158 INFO:teuthology.orchestra.run.smithi138.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-10-01T08:52:35.159 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:35.159 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:52:35.159 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================================== 2023-10-01T08:52:35.159 INFO:teuthology.orchestra.run.smithi138.stdout:Install 6 Packages 2023-10-01T08:52:35.159 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:35.159 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 3.4 M 2023-10-01T08:52:35.160 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 22 M 2023-10-01T08:52:35.160 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:52:35.194 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.14-44.geeb13ac9.el8.noarch 6/6 2023-10-01T08:52:35.195 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-grafana-dashboards-2:16.2.14-44.geeb13ac9.el8.n 1/6 2023-10-01T08:52:35.195 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-mgr-dashboard-2:16.2.14-44.geeb13ac9.el8.noarch 2/6 2023-10-01T08:52:35.195 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-prometheus-alerts-2:16.2.14-44.geeb13ac9.el8.no 3/6 2023-10-01T08:52:35.195 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-10-01T08:52:35.195 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-10-01T08:52:35.340 INFO:teuthology.orchestra.run.smithi138.stdout:(1/6): ceph-prometheus-alerts-16.2.14-44.geeb13 99 kB/s | 15 kB 00:00 2023-10-01T08:52:35.440 INFO:teuthology.orchestra.run.smithi138.stdout:(2/6): ceph-grafana-dashboards-16.2.14-44.geeb1 97 kB/s | 24 kB 00:00 2023-10-01T08:52:35.466 INFO:teuthology.orchestra.run.smithi138.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 268 kB/s | 34 kB 00:00 2023-10-01T08:52:35.524 INFO:teuthology.orchestra.run.smithi138.stdout:(4/6): ceph-mgr-dashboard-16.2.14-44.geeb13ac9. 9.4 MB/s | 3.1 MB 00:00 2023-10-01T08:52:35.658 INFO:teuthology.orchestra.run.smithi138.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 904 kB/s | 196 kB 00:00 2023-10-01T08:52:35.691 INFO:teuthology.orchestra.run.smithi138.stdout:(6/6): python3-jwt-1.6.1-2.el8.noarch.rpm 192 kB/s | 43 kB 00:00 2023-10-01T08:52:35.694 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:52:35.695 INFO:teuthology.orchestra.run.smithi138.stdout:Total 6.4 MB/s | 3.4 MB 00:00 2023-10-01T08:52:35.706 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:52:35.717 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:52:35.717 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:52:35.786 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:52:35.790 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:52:36.147 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:52:36.245 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-10-01T08:52:36.245 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:02:31 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:36.364 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-10-01T08:52:36.561 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-10-01T08:52:36.799 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-10-01T08:52:36.953 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-prometheus-alerts-2:16.2.14-44.geeb13ac9.el8.no 4/6 2023-10-01T08:52:37.976 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-grafana-dashboards-2:16.2.14-44.geeb13ac9.el8.n 5/6 2023-10-01T08:52:38.285 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mgr-dashboard-2:16.2.14-44.geeb13ac9.el8.noarch 6/6 2023-10-01T08:52:38.526 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.14-44.geeb13ac9.el8.noarch 6/6 2023-10-01T08:52:38.526 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-grafana-dashboards-2:16.2.14-44.geeb13ac9.el8.n 1/6 2023-10-01T08:52:38.526 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mgr-dashboard-2:16.2.14-44.geeb13ac9.el8.noarch 2/6 2023-10-01T08:52:38.526 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-prometheus-alerts-2:16.2.14-44.geeb13ac9.el8.no 3/6 2023-10-01T08:52:38.526 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-10-01T08:52:38.526 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-10-01T08:52:39.615 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-10-01T08:52:39.615 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:34 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:45.492 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:52:45.677 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:52:45.677 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:52:45.677 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-grafana-dashboards-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:52:45.677 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mgr-dashboard-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:52:45.677 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-prometheus-alerts-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:52:45.677 INFO:teuthology.orchestra.run.smithi190.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-01T08:52:45.677 INFO:teuthology.orchestra.run.smithi190.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-01T08:52:45.677 INFO:teuthology.orchestra.run.smithi190.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-01T08:52:45.678 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:52:45.678 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:52:45.819 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-01T08:52:46.164 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:52:46.926 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:52:47.107 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:47.107 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:52:47.107 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-grafana-dashboards-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:52:47.107 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-dashboard-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:52:47.107 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-prometheus-alerts-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:52:47.107 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-10-01T08:52:47.107 INFO:teuthology.orchestra.run.smithi138.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-10-01T08:52:47.107 INFO:teuthology.orchestra.run.smithi138.stdout: python3-routes-2.4.1-12.el8.noarch 2023-10-01T08:52:47.107 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:47.107 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:52:47.271 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-mgr-diskprediction-local 2023-10-01T08:52:47.618 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:52:47.754 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:02:42 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:49.111 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:44 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:49.121 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:52:49.122 INFO:teuthology.orchestra.run.smithi190.stdout:======================================================================================================================= 2023-10-01T08:52:49.123 INFO:teuthology.orchestra.run.smithi190.stdout: Package Arch Version Repository Size 2023-10-01T08:52:49.123 INFO:teuthology.orchestra.run.smithi190.stdout:======================================================================================================================= 2023-10-01T08:52:49.123 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:52:49.123 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 8.6 M 2023-10-01T08:52:49.123 INFO:teuthology.orchestra.run.smithi190.stdout:Installing dependencies: 2023-10-01T08:52:49.123 INFO:teuthology.orchestra.run.smithi190.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-10-01T08:52:49.123 INFO:teuthology.orchestra.run.smithi190.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-10-01T08:52:49.124 INFO:teuthology.orchestra.run.smithi190.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-10-01T08:52:49.124 INFO:teuthology.orchestra.run.smithi190.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-10-01T08:52:49.124 INFO:teuthology.orchestra.run.smithi190.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-10-01T08:52:49.124 INFO:teuthology.orchestra.run.smithi190.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-10-01T08:52:49.124 INFO:teuthology.orchestra.run.smithi190.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-10-01T08:52:49.124 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:52:49.124 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:52:49.124 INFO:teuthology.orchestra.run.smithi190.stdout:======================================================================================================================= 2023-10-01T08:52:49.125 INFO:teuthology.orchestra.run.smithi190.stdout:Install 8 Packages 2023-10-01T08:52:49.125 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:52:49.126 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 37 M 2023-10-01T08:52:49.126 INFO:teuthology.orchestra.run.smithi190.stdout:Installed size: 210 M 2023-10-01T08:52:49.126 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:52:49.350 INFO:teuthology.orchestra.run.smithi190.stdout:(1/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 760 kB/s | 171 kB 00:00 2023-10-01T08:52:49.400 INFO:teuthology.orchestra.run.smithi190.stdout:(2/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 2.3 MB/s | 644 kB 00:00 2023-10-01T08:52:49.692 INFO:teuthology.orchestra.run.smithi190.stdout:(3/8): python3-scipy-1.0.0-21.module+el8.5.0+10 40 MB/s | 14 MB 00:00 2023-10-01T08:52:49.776 INFO:teuthology.orchestra.run.smithi190.stdout:(4/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 9.9 MB/s | 3.7 MB 00:00 2023-10-01T08:52:49.826 INFO:teuthology.orchestra.run.smithi190.stdout:(5/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 1.6 MB/s | 225 kB 00:00 2023-10-01T08:52:49.968 INFO:teuthology.orchestra.run.smithi190.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 25 MB/s | 4.9 MB 00:00 2023-10-01T08:52:50.060 INFO:teuthology.orchestra.run.smithi190.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 20 MB/s | 4.7 MB 00:00 2023-10-01T08:52:50.352 INFO:teuthology.orchestra.run.smithi190.stdout:(8/8): ceph-mgr-diskprediction-local-16.2.14-44 7.0 MB/s | 8.6 MB 00:01 2023-10-01T08:52:50.353 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:52:50.353 INFO:teuthology.orchestra.run.smithi190.stdout:Total 30 MB/s | 37 MB 00:01 2023-10-01T08:52:50.483 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:52:50.485 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================================= 2023-10-01T08:52:50.485 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-10-01T08:52:50.485 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================================= 2023-10-01T08:52:50.485 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:52:50.485 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 8.6 M 2023-10-01T08:52:50.485 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-10-01T08:52:50.485 INFO:teuthology.orchestra.run.smithi138.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-10-01T08:52:50.485 INFO:teuthology.orchestra.run.smithi138.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-10-01T08:52:50.485 INFO:teuthology.orchestra.run.smithi138.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-10-01T08:52:50.485 INFO:teuthology.orchestra.run.smithi138.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-10-01T08:52:50.486 INFO:teuthology.orchestra.run.smithi138.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-10-01T08:52:50.486 INFO:teuthology.orchestra.run.smithi138.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-10-01T08:52:50.486 INFO:teuthology.orchestra.run.smithi138.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-10-01T08:52:50.486 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:50.486 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:52:50.486 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================================= 2023-10-01T08:52:50.486 INFO:teuthology.orchestra.run.smithi138.stdout:Install 8 Packages 2023-10-01T08:52:50.486 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:52:50.487 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 37 M 2023-10-01T08:52:50.487 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 210 M 2023-10-01T08:52:50.487 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:52:50.586 INFO:teuthology.orchestra.run.smithi138.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 6.4 MB/s | 644 kB 00:00 2023-10-01T08:52:50.589 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:52:50.607 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:52:50.607 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:52:50.645 INFO:teuthology.orchestra.run.smithi138.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.1 MB/s | 171 kB 00:00 2023-10-01T08:52:50.906 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:52:50.910 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:52:50.962 INFO:teuthology.orchestra.run.smithi138.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 12 MB/s | 3.7 MB 00:00 2023-10-01T08:52:51.229 INFO:teuthology.orchestra.run.smithi138.stdout:(4/8): ceph-mgr-diskprediction-local-16.2.14-44 12 MB/s | 8.6 MB 00:00 2023-10-01T08:52:51.254 INFO:teuthology.orchestra.run.smithi138.stdout:(5/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 771 kB/s | 225 kB 00:00 2023-10-01T08:52:51.457 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:52:51.562 INFO:teuthology.orchestra.run.smithi138.stdout:(6/8): python3-scipy-1.0.0-21.module+el8.5.0+10 14 MB/s | 14 MB 00:00 2023-10-01T08:52:51.588 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-01T08:52:51.688 INFO:teuthology.orchestra.run.smithi138.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 11 MB/s | 4.7 MB 00:00 2023-10-01T08:52:51.734 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-01T08:52:51.781 INFO:teuthology.orchestra.run.smithi138.stdout:(8/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 8.8 MB/s | 4.9 MB 00:00 2023-10-01T08:52:51.782 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:52:51.783 INFO:teuthology.orchestra.run.smithi138.stdout:Total 28 MB/s | 37 MB 00:01 2023-10-01T08:52:51.904 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-01T08:52:52.022 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:52:52.040 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:52:52.040 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:52:52.339 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:52:52.343 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:52:52.517 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-01T08:52:52.696 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-01T08:52:52.928 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:52:53.052 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-01T08:52:53.203 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-10-01T08:52:53.291 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-01T08:52:53.344 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-01T08:52:53.488 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-10-01T08:52:53.961 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-01T08:52:54.144 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-01T08:52:54.152 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-10-01T08:52:54.439 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-01T08:52:54.663 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-01T08:52:54.744 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-10-01T08:52:54.943 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-10-01T08:52:55.621 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-10-01T08:52:55.947 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-01T08:52:56.177 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-01T08:52:56.384 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-01T08:52:57.669 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-10-01T08:52:57.821 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.14-44.geeb13ac9 8/8 2023-10-01T08:52:57.912 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-01T08:52:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.14-44.geeb13ac9 8/8 2023-10-01T08:52:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.14-44.geeb13ac9 1/8 2023-10-01T08:52:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-01T08:52:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-10-01T08:52:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-10-01T08:52:58.074 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-01T08:52:58.074 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-01T08:52:58.074 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-10-01T08:52:59.163 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-10-01T08:52:59.164 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:02:54 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:52:59.258 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-10-01T08:52:59.501 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.14-44.geeb13ac9 8/8 2023-10-01T08:52:59.753 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.14-44.geeb13ac9 8/8 2023-10-01T08:52:59.753 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.14-44.geeb13ac9 1/8 2023-10-01T08:52:59.753 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-10-01T08:52:59.753 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-10-01T08:52:59.754 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-10-01T08:52:59.754 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-10-01T08:52:59.754 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-10-01T08:52:59.754 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-10-01T08:53:00.841 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-10-01T08:53:00.842 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:02:55 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:53:07.378 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:53:07.564 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:53:07.564 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:53:07.564 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-diskprediction-local-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:53:07.564 INFO:teuthology.orchestra.run.smithi138.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-10-01T08:53:07.564 INFO:teuthology.orchestra.run.smithi138.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-10-01T08:53:07.564 INFO:teuthology.orchestra.run.smithi138.stdout: openblas-0.3.15-3.el8.x86_64 2023-10-01T08:53:07.564 INFO:teuthology.orchestra.run.smithi138.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-10-01T08:53:07.565 INFO:teuthology.orchestra.run.smithi138.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-01T08:53:07.565 INFO:teuthology.orchestra.run.smithi138.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-01T08:53:07.565 INFO:teuthology.orchestra.run.smithi138.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-10-01T08:53:07.565 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:53:07.565 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:53:07.704 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-mgr-rook 2023-10-01T08:53:08.050 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:53:09.714 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:04 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:53:10.047 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:53:10.231 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:53:10.231 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:53:10.231 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mgr-diskprediction-local-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:53:10.232 INFO:teuthology.orchestra.run.smithi190.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-10-01T08:53:10.232 INFO:teuthology.orchestra.run.smithi190.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-10-01T08:53:10.232 INFO:teuthology.orchestra.run.smithi190.stdout: openblas-0.3.15-3.el8.x86_64 2023-10-01T08:53:10.232 INFO:teuthology.orchestra.run.smithi190.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-10-01T08:53:10.232 INFO:teuthology.orchestra.run.smithi190.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-10-01T08:53:10.232 INFO:teuthology.orchestra.run.smithi190.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-10-01T08:53:10.232 INFO:teuthology.orchestra.run.smithi190.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-10-01T08:53:10.232 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:53:10.232 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:53:10.367 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install ceph-mgr-rook 2023-10-01T08:53:10.713 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:53:11.106 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:53:11.108 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================= 2023-10-01T08:53:11.108 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-10-01T08:53:11.108 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================= 2023-10-01T08:53:11.108 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:53:11.108 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-rook noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 27 k 2023-10-01T08:53:11.108 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-10-01T08:53:11.108 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-01T08:53:11.109 INFO:teuthology.orchestra.run.smithi138.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-01T08:53:11.109 INFO:teuthology.orchestra.run.smithi138.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-01T08:53:11.109 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-10-01T08:53:11.109 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-10-01T08:53:11.109 INFO:teuthology.orchestra.run.smithi138.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-01T08:53:11.109 INFO:teuthology.orchestra.run.smithi138.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-10-01T08:53:11.109 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-10-01T08:53:11.109 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-10-01T08:53:11.109 INFO:teuthology.orchestra.run.smithi138.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-01T08:53:11.109 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-01T08:53:11.109 INFO:teuthology.orchestra.run.smithi138.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-01T08:53:11.110 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:53:11.110 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:53:11.110 INFO:teuthology.orchestra.run.smithi138.stdout:======================================================================================================= 2023-10-01T08:53:11.110 INFO:teuthology.orchestra.run.smithi138.stdout:Install 13 Packages 2023-10-01T08:53:11.110 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:53:11.110 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 1.9 M 2023-10-01T08:53:11.111 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 24 M 2023-10-01T08:53:11.111 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:53:11.279 INFO:teuthology.orchestra.run.smithi138.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 205 kB/s | 16 kB 00:00 2023-10-01T08:53:11.323 INFO:teuthology.orchestra.run.smithi138.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 266 kB/s | 33 kB 00:00 2023-10-01T08:53:11.354 INFO:teuthology.orchestra.run.smithi138.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 1.1 MB/s | 81 kB 00:00 2023-10-01T08:53:11.388 INFO:teuthology.orchestra.run.smithi138.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 1.9 MB/s | 64 kB 00:00 2023-10-01T08:53:11.430 INFO:teuthology.orchestra.run.smithi138.stdout:(5/13): python3-kubernetes-11.0.0-6.el8.noarch. 11 MB/s | 1.2 MB 00:00 2023-10-01T08:53:11.455 INFO:teuthology.orchestra.run.smithi138.stdout:(6/13): python3-websocket-client-0.56.0-5.el8.n 915 kB/s | 61 kB 00:00 2023-10-01T08:53:11.555 INFO:teuthology.orchestra.run.smithi138.stdout:(7/13): ceph-mgr-rook-16.2.14-44.geeb13ac9.el8. 77 kB/s | 27 kB 00:00 2023-10-01T08:53:11.580 INFO:teuthology.orchestra.run.smithi138.stdout:(8/13): python3-requests-oauthlib-1.0.0-1.el8.n 284 kB/s | 43 kB 00:00 2023-10-01T08:53:11.630 INFO:teuthology.orchestra.run.smithi138.stdout:(9/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 882 kB/s | 155 kB 00:00 2023-10-01T08:53:11.798 INFO:teuthology.orchestra.run.smithi138.stdout:(10/13): python3-pyasn1-modules-0.3.7-6.el8.noa 454 kB/s | 110 kB 00:00 2023-10-01T08:53:11.823 INFO:teuthology.orchestra.run.smithi138.stdout:(11/13): python3-jsonpatch-1.21-2.el8.noarch.rp 112 kB/s | 27 kB 00:00 2023-10-01T08:53:11.848 INFO:teuthology.orchestra.run.smithi138.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 581 kB/s | 126 kB 00:00 2023-10-01T08:53:12.065 INFO:teuthology.orchestra.run.smithi138.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 75 kB/s | 20 kB 00:00 2023-10-01T08:53:12.065 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:53:12.066 INFO:teuthology.orchestra.run.smithi138.stdout:Total 2.0 MB/s | 1.9 MB 00:00 2023-10-01T08:53:12.123 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:53:12.138 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:53:12.138 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:53:12.208 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:53:12.212 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:53:12.340 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:03:07 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:53:12.709 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:53:12.931 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-01T08:53:13.201 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-10-01T08:53:13.405 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-10-01T08:53:13.581 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-10-01T08:53:13.794 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:53:13.795 INFO:teuthology.orchestra.run.smithi190.stdout:======================================================================================================= 2023-10-01T08:53:13.795 INFO:teuthology.orchestra.run.smithi190.stdout: Package Arch Version Repository Size 2023-10-01T08:53:13.795 INFO:teuthology.orchestra.run.smithi190.stdout:======================================================================================================= 2023-10-01T08:53:13.796 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:53:13.796 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mgr-rook noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 27 k 2023-10-01T08:53:13.796 INFO:teuthology.orchestra.run.smithi190.stdout:Installing dependencies: 2023-10-01T08:53:13.796 INFO:teuthology.orchestra.run.smithi190.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-10-01T08:53:13.796 INFO:teuthology.orchestra.run.smithi190.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-10-01T08:53:13.796 INFO:teuthology.orchestra.run.smithi190.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-10-01T08:53:13.796 INFO:teuthology.orchestra.run.smithi190.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-10-01T08:53:13.796 INFO:teuthology.orchestra.run.smithi190.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-10-01T08:53:13.796 INFO:teuthology.orchestra.run.smithi190.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-10-01T08:53:13.797 INFO:teuthology.orchestra.run.smithi190.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-10-01T08:53:13.797 INFO:teuthology.orchestra.run.smithi190.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-10-01T08:53:13.797 INFO:teuthology.orchestra.run.smithi190.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-10-01T08:53:13.797 INFO:teuthology.orchestra.run.smithi190.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-10-01T08:53:13.797 INFO:teuthology.orchestra.run.smithi190.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-10-01T08:53:13.797 INFO:teuthology.orchestra.run.smithi190.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-10-01T08:53:13.797 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:53:13.797 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:53:13.797 INFO:teuthology.orchestra.run.smithi190.stdout:======================================================================================================= 2023-10-01T08:53:13.797 INFO:teuthology.orchestra.run.smithi190.stdout:Install 13 Packages 2023-10-01T08:53:13.798 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:53:13.798 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 1.9 M 2023-10-01T08:53:13.798 INFO:teuthology.orchestra.run.smithi190.stdout:Installed size: 24 M 2023-10-01T08:53:13.798 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:53:13.835 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-10-01T08:53:13.933 INFO:teuthology.orchestra.run.smithi190.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 226 kB/s | 16 kB 00:00 2023-10-01T08:53:13.958 INFO:teuthology.orchestra.run.smithi190.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 339 kB/s | 33 kB 00:00 2023-10-01T08:53:14.000 INFO:teuthology.orchestra.run.smithi190.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 1.2 MB/s | 81 kB 00:00 2023-10-01T08:53:14.050 INFO:teuthology.orchestra.run.smithi190.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 1.3 MB/s | 64 kB 00:00 2023-10-01T08:53:14.101 INFO:teuthology.orchestra.run.smithi190.stdout:(5/13): python3-kubernetes-11.0.0-6.el8.noarch. 8.2 MB/s | 1.2 MB 00:00 2023-10-01T08:53:14.103 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-10-01T08:53:14.126 INFO:teuthology.orchestra.run.smithi190.stdout:(6/13): ceph-mgr-rook-16.2.14-44.geeb13ac9.el8. 103 kB/s | 27 kB 00:00 2023-10-01T08:53:14.151 INFO:teuthology.orchestra.run.smithi190.stdout:(7/13): python3-websocket-client-0.56.0-5.el8.n 610 kB/s | 61 kB 00:00 2023-10-01T08:53:14.309 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-10-01T08:53:14.318 INFO:teuthology.orchestra.run.smithi190.stdout:(8/13): python3-requests-oauthlib-1.0.0-1.el8.n 197 kB/s | 43 kB 00:00 2023-10-01T08:53:14.343 INFO:teuthology.orchestra.run.smithi190.stdout:(9/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 714 kB/s | 155 kB 00:00 2023-10-01T08:53:14.368 INFO:teuthology.orchestra.run.smithi190.stdout:(10/13): python3-pyasn1-modules-0.3.7-6.el8.noa 509 kB/s | 110 kB 00:00 2023-10-01T08:53:14.469 INFO:teuthology.orchestra.run.smithi190.stdout:(11/13): python3-jsonpatch-1.21-2.el8.noarch.rp 176 kB/s | 27 kB 00:00 2023-10-01T08:53:14.494 INFO:teuthology.orchestra.run.smithi190.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 839 kB/s | 126 kB 00:00 2023-10-01T08:53:14.511 INFO:teuthology.orchestra.run.smithi190.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 141 kB/s | 20 kB 00:00 2023-10-01T08:53:14.511 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:53:14.512 INFO:teuthology.orchestra.run.smithi190.stdout:Total 2.7 MB/s | 1.9 MB 00:00 2023-10-01T08:53:14.531 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-10-01T08:53:14.569 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:53:14.585 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:53:14.585 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:53:14.655 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:53:14.659 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:53:14.746 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-01T08:53:14.952 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-10-01T08:53:15.118 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:53:15.308 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-10-01T08:53:15.553 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-10-01T08:53:15.749 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-10-01T08:53:15.916 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-10-01T08:53:16.074 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-10-01T08:53:16.129 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-10-01T08:53:16.322 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-10-01T08:53:16.363 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-10-01T08:53:16.515 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mgr-rook-2:16.2.14-44.geeb13ac9.el8.noarch 13/13 2023-10-01T08:53:16.537 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-10-01T08:53:16.708 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-10-01T08:53:16.738 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.14-44.geeb13ac9.el8.noarch 13/13 2023-10-01T08:53:16.739 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mgr-rook-2:16.2.14-44.geeb13ac9.el8.noarch 1/13 2023-10-01T08:53:16.739 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-10-01T08:53:16.739 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-10-01T08:53:16.739 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-10-01T08:53:16.739 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-10-01T08:53:16.739 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-10-01T08:53:16.739 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-10-01T08:53:16.739 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-10-01T08:53:16.739 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-10-01T08:53:16.739 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-10-01T08:53:16.739 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-10-01T08:53:16.740 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-10-01T08:53:16.882 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-10-01T08:53:17.073 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-10-01T08:53:17.872 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-10-01T08:53:17.872 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:12 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:53:18.155 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-10-01T08:53:18.383 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-10-01T08:53:18.534 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-mgr-rook-2:16.2.14-44.geeb13ac9.el8.noarch 13/13 2023-10-01T08:53:18.801 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.14-44.geeb13ac9.el8.noarch 13/13 2023-10-01T08:53:18.802 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-mgr-rook-2:16.2.14-44.geeb13ac9.el8.noarch 1/13 2023-10-01T08:53:18.802 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-10-01T08:53:18.802 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-10-01T08:53:18.802 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-10-01T08:53:18.802 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-10-01T08:53:18.802 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-10-01T08:53:18.802 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-10-01T08:53:18.802 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-10-01T08:53:18.803 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-10-01T08:53:18.803 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-10-01T08:53:18.803 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-10-01T08:53:18.803 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-10-01T08:53:19.921 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-10-01T08:53:19.921 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:03:14 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:53:28.515 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:53:28.699 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:53:28.700 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:53:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-rook-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:53:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-01T08:53:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-01T08:53:28.700 INFO:teuthology.orchestra.run.smithi138.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-01T08:53:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-01T08:53:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-01T08:53:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-01T08:53:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-01T08:53:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-01T08:53:28.701 INFO:teuthology.orchestra.run.smithi138.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-01T08:53:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-01T08:53:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-01T08:53:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-01T08:53:28.702 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:53:28.702 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:53:28.849 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-mgr-cephadm 2023-10-01T08:53:28.979 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:53:29.165 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:53:29.165 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:53:29.165 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mgr-rook-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:53:29.166 INFO:teuthology.orchestra.run.smithi190.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-10-01T08:53:29.166 INFO:teuthology.orchestra.run.smithi190.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-10-01T08:53:29.166 INFO:teuthology.orchestra.run.smithi190.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-10-01T08:53:29.166 INFO:teuthology.orchestra.run.smithi190.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-10-01T08:53:29.166 INFO:teuthology.orchestra.run.smithi190.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-10-01T08:53:29.166 INFO:teuthology.orchestra.run.smithi190.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-10-01T08:53:29.166 INFO:teuthology.orchestra.run.smithi190.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-10-01T08:53:29.166 INFO:teuthology.orchestra.run.smithi190.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-10-01T08:53:29.166 INFO:teuthology.orchestra.run.smithi190.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-10-01T08:53:29.166 INFO:teuthology.orchestra.run.smithi190.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-10-01T08:53:29.166 INFO:teuthology.orchestra.run.smithi190.stdout: python3-rsa-4.9-2.el8.noarch 2023-10-01T08:53:29.167 INFO:teuthology.orchestra.run.smithi190.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-10-01T08:53:29.167 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:53:29.167 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:53:29.204 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:53:29.358 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install ceph-mgr-cephadm 2023-10-01T08:53:29.711 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:53:30.833 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:25 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:53:31.366 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:03:26 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:53:32.204 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:53:32.205 INFO:teuthology.orchestra.run.smithi138.stdout:============================================================================================== 2023-10-01T08:53:32.206 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-10-01T08:53:32.206 INFO:teuthology.orchestra.run.smithi138.stdout:============================================================================================== 2023-10-01T08:53:32.206 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:53:32.206 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-cephadm noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 116 k 2023-10-01T08:53:32.206 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-10-01T08:53:32.206 INFO:teuthology.orchestra.run.smithi138.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-10-01T08:53:32.206 INFO:teuthology.orchestra.run.smithi138.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-10-01T08:53:32.206 INFO:teuthology.orchestra.run.smithi138.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-10-01T08:53:32.206 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-10-01T08:53:32.206 INFO:teuthology.orchestra.run.smithi138.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-10-01T08:53:32.207 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:53:32.207 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:53:32.207 INFO:teuthology.orchestra.run.smithi138.stdout:============================================================================================== 2023-10-01T08:53:32.207 INFO:teuthology.orchestra.run.smithi138.stdout:Install 6 Packages 2023-10-01T08:53:32.207 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:53:32.207 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 5.8 M 2023-10-01T08:53:32.207 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 24 M 2023-10-01T08:53:32.208 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:53:32.707 INFO:teuthology.orchestra.run.smithi138.stdout:(1/6): python3-apipkg-1.5-6.el8.noarch.rpm 70 kB/s | 21 kB 00:00 2023-10-01T08:53:32.777 INFO:teuthology.orchestra.run.smithi138.stdout:(2/6): ceph-mgr-cephadm-16.2.14-44.geeb13ac9.el 313 kB/s | 116 kB 00:00 2023-10-01T08:53:32.781 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:53:32.782 INFO:teuthology.orchestra.run.smithi190.stdout:============================================================================================== 2023-10-01T08:53:32.782 INFO:teuthology.orchestra.run.smithi190.stdout: Package Arch Version Repository Size 2023-10-01T08:53:32.782 INFO:teuthology.orchestra.run.smithi190.stdout:============================================================================================== 2023-10-01T08:53:32.782 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:53:32.782 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mgr-cephadm noarch 2:16.2.14-44.geeb13ac9.el8 ceph-noarch 116 k 2023-10-01T08:53:32.782 INFO:teuthology.orchestra.run.smithi190.stdout:Installing dependencies: 2023-10-01T08:53:32.783 INFO:teuthology.orchestra.run.smithi190.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-10-01T08:53:32.783 INFO:teuthology.orchestra.run.smithi190.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-10-01T08:53:32.783 INFO:teuthology.orchestra.run.smithi190.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-10-01T08:53:32.783 INFO:teuthology.orchestra.run.smithi190.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-10-01T08:53:32.783 INFO:teuthology.orchestra.run.smithi190.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-10-01T08:53:32.783 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:53:32.783 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:53:32.783 INFO:teuthology.orchestra.run.smithi190.stdout:============================================================================================== 2023-10-01T08:53:32.783 INFO:teuthology.orchestra.run.smithi190.stdout:Install 6 Packages 2023-10-01T08:53:32.783 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:53:32.784 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 5.8 M 2023-10-01T08:53:32.784 INFO:teuthology.orchestra.run.smithi190.stdout:Installed size: 24 M 2023-10-01T08:53:32.784 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:53:32.835 INFO:teuthology.orchestra.run.smithi138.stdout:(3/6): python3-remoto-1.2.1-1.el8.noarch.rpm 380 kB/s | 49 kB 00:00 2023-10-01T08:53:32.894 INFO:teuthology.orchestra.run.smithi138.stdout:(4/6): python3-execnet-1.7.1-1.el8.noarch.rpm 728 kB/s | 355 kB 00:00 2023-10-01T08:53:32.972 INFO:teuthology.orchestra.run.smithi190.stdout:(1/6): python3-apipkg-1.5-6.el8.noarch.rpm 164 kB/s | 21 kB 00:00 2023-10-01T08:53:33.011 INFO:teuthology.orchestra.run.smithi138.stdout:(5/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 2.2 MB/s | 538 kB 00:00 2023-10-01T08:53:33.022 INFO:teuthology.orchestra.run.smithi190.stdout:(2/6): python3-remoto-1.2.1-1.el8.noarch.rpm 981 kB/s | 49 kB 00:00 2023-10-01T08:53:33.114 INFO:teuthology.orchestra.run.smithi190.stdout:(3/6): python3-execnet-1.7.1-1.el8.noarch.rpm 1.3 MB/s | 355 kB 00:00 2023-10-01T08:53:33.144 INFO:teuthology.orchestra.run.smithi138.stdout:(6/6): python3-babel-2.5.1-7.el8.noarch.rpm 16 MB/s | 4.8 MB 00:00 2023-10-01T08:53:33.146 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:53:33.146 INFO:teuthology.orchestra.run.smithi138.stdout:Total 6.2 MB/s | 5.8 MB 00:00 2023-10-01T08:53:33.172 INFO:teuthology.orchestra.run.smithi190.stdout:(4/6): ceph-mgr-cephadm-16.2.14-44.geeb13ac9.el 352 kB/s | 116 kB 00:00 2023-10-01T08:53:33.200 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:53:33.206 INFO:teuthology.orchestra.run.smithi190.stdout:(5/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 2.9 MB/s | 538 kB 00:00 2023-10-01T08:53:33.210 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:53:33.210 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:53:33.289 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:53:33.292 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:53:33.465 INFO:teuthology.orchestra.run.smithi190.stdout:(6/6): python3-babel-2.5.1-7.el8.noarch.rpm 14 MB/s | 4.8 MB 00:00 2023-10-01T08:53:33.466 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:53:33.466 INFO:teuthology.orchestra.run.smithi190.stdout:Total 8.6 MB/s | 5.8 MB 00:00 2023-10-01T08:53:33.519 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:53:33.529 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:53:33.529 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:53:33.608 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:53:33.612 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:53:34.300 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:53:34.573 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:53:34.725 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/6 2023-10-01T08:53:34.968 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/6 2023-10-01T08:53:34.976 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/6 2023-10-01T08:53:35.204 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/6 2023-10-01T08:53:35.280 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 3/6 2023-10-01T08:53:35.455 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 3/6 2023-10-01T08:53:35.557 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 4/6 2023-10-01T08:53:35.700 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 4/6 2023-10-01T08:53:35.848 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 5/6 2023-10-01T08:53:35.922 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 5/6 2023-10-01T08:53:36.020 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-mgr-cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 6/6 2023-10-01T08:53:36.098 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-mgr-cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 6/6 2023-10-01T08:53:36.307 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 6/6 2023-10-01T08:53:36.307 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-mgr-cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 1/6 2023-10-01T08:53:36.307 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 2/6 2023-10-01T08:53:36.307 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 3/6 2023-10-01T08:53:36.308 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 4/6 2023-10-01T08:53:36.308 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-10-01T08:53:36.336 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 6/6 2023-10-01T08:53:36.336 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-mgr-cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 1/6 2023-10-01T08:53:36.336 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 2/6 2023-10-01T08:53:36.337 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 3/6 2023-10-01T08:53:36.337 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 4/6 2023-10-01T08:53:36.337 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-10-01T08:53:37.373 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 6/6 2023-10-01T08:53:37.373 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:32 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:53:37.382 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 6/6 2023-10-01T08:53:37.382 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:03:32 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:53:46.958 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:53:47.141 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:53:47.141 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:53:47.141 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-mgr-cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:53:47.141 INFO:teuthology.orchestra.run.smithi190.stdout: python3-apipkg-1.5-6.el8.noarch 2023-10-01T08:53:47.141 INFO:teuthology.orchestra.run.smithi190.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-01T08:53:47.142 INFO:teuthology.orchestra.run.smithi190.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-10-01T08:53:47.142 INFO:teuthology.orchestra.run.smithi190.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-01T08:53:47.142 INFO:teuthology.orchestra.run.smithi190.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-10-01T08:53:47.142 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:53:47.142 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:53:47.256 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:53:47.279 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install ceph-fuse 2023-10-01T08:53:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:53:47.442 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:53:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-mgr-cephadm-2:16.2.14-44.geeb13ac9.el8.noarch 2023-10-01T08:53:47.442 INFO:teuthology.orchestra.run.smithi138.stdout: python3-apipkg-1.5-6.el8.noarch 2023-10-01T08:53:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-babel-2.5.1-7.el8.noarch 2023-10-01T08:53:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-10-01T08:53:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-10-01T08:53:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-10-01T08:53:47.443 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:53:47.443 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:53:47.633 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:53:48.020 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install ceph-fuse 2023-10-01T08:53:48.384 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:53:49.402 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:03:44 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:53:49.890 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:44 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:53:50.758 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:53:50.759 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:53:50.759 INFO:teuthology.orchestra.run.smithi190.stdout: Package Arch Version Repository Size 2023-10-01T08:53:50.759 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:53:50.759 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:53:50.759 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-fuse 2023-10-01T08:53:50.759 INFO:teuthology.orchestra.run.smithi190.stdout: x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 910 k 2023-10-01T08:53:50.759 INFO:teuthology.orchestra.run.smithi190.stdout:Installing dependencies: 2023-10-01T08:53:50.759 INFO:teuthology.orchestra.run.smithi190.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-10-01T08:53:50.759 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:53:50.760 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:53:50.760 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:53:50.760 INFO:teuthology.orchestra.run.smithi190.stdout:Install 2 Packages 2023-10-01T08:53:50.760 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:53:50.761 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 993 k 2023-10-01T08:53:50.761 INFO:teuthology.orchestra.run.smithi190.stdout:Installed size: 2.8 M 2023-10-01T08:53:50.761 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:53:50.983 INFO:teuthology.orchestra.run.smithi190.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 372 kB/s | 83 kB 00:00 2023-10-01T08:53:51.092 INFO:teuthology.orchestra.run.smithi190.stdout:(2/2): ceph-fuse-16.2.14-44.geeb13ac9.el8.x86_6 2.7 MB/s | 910 kB 00:00 2023-10-01T08:53:51.092 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:53:51.092 INFO:teuthology.orchestra.run.smithi190.stdout:Total 2.9 MB/s | 993 kB 00:00 2023-10-01T08:53:51.104 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:53:51.124 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:53:51.125 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:53:51.199 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:53:51.203 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:53:51.252 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:53:51.253 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:53:51.253 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repository Size 2023-10-01T08:53:51.253 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:53:51.253 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:53:51.253 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-fuse 2023-10-01T08:53:51.253 INFO:teuthology.orchestra.run.smithi138.stdout: x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 910 k 2023-10-01T08:53:51.254 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2023-10-01T08:53:51.254 INFO:teuthology.orchestra.run.smithi138.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-10-01T08:53:51.254 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:53:51.254 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:53:51.254 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:53:51.254 INFO:teuthology.orchestra.run.smithi138.stdout:Install 2 Packages 2023-10-01T08:53:51.254 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:53:51.255 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 993 k 2023-10-01T08:53:51.255 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 2.8 M 2023-10-01T08:53:51.255 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:53:51.363 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:53:51.406 INFO:teuthology.orchestra.run.smithi138.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 549 kB/s | 83 kB 00:00 2023-10-01T08:53:51.506 INFO:teuthology.orchestra.run.smithi138.stdout:(2/2): ceph-fuse-16.2.14-44.geeb13ac9.el8.x86_6 3.5 MB/s | 910 kB 00:00 2023-10-01T08:53:51.506 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:53:51.507 INFO:teuthology.orchestra.run.smithi138.stdout:Total 3.8 MB/s | 993 kB 00:00 2023-10-01T08:53:51.517 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:53:51.537 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:53:51.538 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:53:51.611 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:53:51.615 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:53:51.619 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-10-01T08:53:51.777 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : ceph-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 2/2 2023-10-01T08:53:51.803 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:53:52.101 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-10-01T08:53:52.119 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: ceph-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 2/2 2023-10-01T08:53:52.119 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : ceph-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 1/2 2023-10-01T08:53:52.266 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : ceph-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 2/2 2023-10-01T08:53:52.636 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: ceph-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 2/2 2023-10-01T08:53:52.636 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : ceph-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 1/2 2023-10-01T08:53:53.171 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-10-01T08:53:53.171 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:03:48 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:53:53.736 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-10-01T08:53:53.736 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:48 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:53:59.804 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:53:59.882 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:53:59.988 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:53:59.988 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:53:59.988 INFO:teuthology.orchestra.run.smithi190.stdout: ceph-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-10-01T08:53:59.988 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:53:59.989 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:54:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:54:00.065 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:54:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: ceph-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-10-01T08:54:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:54:00.065 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:54:00.125 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install librados-devel 2023-10-01T08:54:00.215 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install librados-devel 2023-10-01T08:54:00.481 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:00.571 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:02.066 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:03:57 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:02.156 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:03:57 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:03.513 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:54:03.514 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:54:03.514 INFO:teuthology.orchestra.run.smithi190.stdout: Package Arch Version Repo Size 2023-10-01T08:54:03.514 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:54:03.514 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:54:03.514 INFO:teuthology.orchestra.run.smithi190.stdout: librados-devel x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 131 k 2023-10-01T08:54:03.514 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:54:03.514 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:54:03.514 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:54:03.515 INFO:teuthology.orchestra.run.smithi190.stdout:Install 1 Package 2023-10-01T08:54:03.515 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:54:03.515 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 131 k 2023-10-01T08:54:03.515 INFO:teuthology.orchestra.run.smithi190.stdout:Installed size: 432 k 2023-10-01T08:54:03.515 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:54:03.522 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:54:03.523 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:54:03.523 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repo Size 2023-10-01T08:54:03.523 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:54:03.523 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:54:03.523 INFO:teuthology.orchestra.run.smithi138.stdout: librados-devel x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 131 k 2023-10-01T08:54:03.524 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:54:03.524 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:54:03.524 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:54:03.524 INFO:teuthology.orchestra.run.smithi138.stdout:Install 1 Package 2023-10-01T08:54:03.524 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:54:03.524 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 131 k 2023-10-01T08:54:03.525 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 432 k 2023-10-01T08:54:03.525 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:54:03.710 INFO:teuthology.orchestra.run.smithi138.stdout:librados-devel-16.2.14-44.geeb13ac9.el8.x86_64. 708 kB/s | 131 kB 00:00 2023-10-01T08:54:03.710 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:54:03.710 INFO:teuthology.orchestra.run.smithi138.stdout:Total 704 kB/s | 131 kB 00:00 2023-10-01T08:54:03.711 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:54:03.716 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:54:03.716 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:54:03.771 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:54:03.778 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:54:03.886 INFO:teuthology.orchestra.run.smithi190.stdout:librados-devel-16.2.14-44.geeb13ac9.el8.x86_64. 353 kB/s | 131 kB 00:00 2023-10-01T08:54:03.887 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:54:03.887 INFO:teuthology.orchestra.run.smithi190.stdout:Total 352 kB/s | 131 kB 00:00 2023-10-01T08:54:03.887 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:54:03.892 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:54:03.892 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:54:03.920 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:54:03.948 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:54:03.951 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:54:04.072 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:54:04.088 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : librados-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:04.249 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : librados-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:04.331 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: librados-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:04.545 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: librados-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:05.445 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : librados-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:05.445 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:04:00 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:05.609 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : librados-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:05.609 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:04:00 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:12.723 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:54:12.905 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:54:12.905 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:54:12.905 INFO:teuthology.orchestra.run.smithi190.stdout: librados-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:54:12.905 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:54:12.905 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:54:13.054 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install libcephfs2 2023-10-01T08:54:13.407 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:13.925 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:54:14.105 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:54:14.105 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:54:14.105 INFO:teuthology.orchestra.run.smithi138.stdout: librados-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:54:14.105 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:54:14.105 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:54:14.273 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install libcephfs2 2023-10-01T08:54:14.628 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:14.976 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:04:09 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:16.152 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:04:11 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:16.256 INFO:teuthology.orchestra.run.smithi190.stdout:Package libcephfs2-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:54:16.364 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:54:16.365 INFO:teuthology.orchestra.run.smithi190.stdout:Nothing to do. 2023-10-01T08:54:16.365 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:54:16.486 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install libcephfs-devel 2023-10-01T08:54:16.840 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:17.416 INFO:teuthology.orchestra.run.smithi138.stdout:Package libcephfs2-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:54:17.528 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:54:17.529 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-10-01T08:54:17.529 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:54:17.666 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install libcephfs-devel 2023-10-01T08:54:18.019 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:18.421 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:04:13 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:19.529 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:04:14 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:19.789 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:54:19.789 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:54:19.789 INFO:teuthology.orchestra.run.smithi190.stdout: Package Arch Version Repo Size 2023-10-01T08:54:19.790 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:54:19.790 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:54:19.790 INFO:teuthology.orchestra.run.smithi190.stdout: libcephfs-devel x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 25 k 2023-10-01T08:54:19.790 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:54:19.790 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:54:19.790 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:54:19.790 INFO:teuthology.orchestra.run.smithi190.stdout:Install 1 Package 2023-10-01T08:54:19.790 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:54:19.791 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 25 k 2023-10-01T08:54:19.791 INFO:teuthology.orchestra.run.smithi190.stdout:Installed size: 107 k 2023-10-01T08:54:19.791 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:54:19.939 INFO:teuthology.orchestra.run.smithi190.stdout:libcephfs-devel-16.2.14-44.geeb13ac9.el8.x86_64 172 kB/s | 25 kB 00:00 2023-10-01T08:54:19.940 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:54:19.940 INFO:teuthology.orchestra.run.smithi190.stdout:Total 170 kB/s | 25 kB 00:00 2023-10-01T08:54:19.940 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:54:19.944 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:54:19.944 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:54:19.946 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:54:19.950 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:54:19.997 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:54:20.163 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : libcephfs-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:20.418 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: libcephfs-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:20.891 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:54:20.891 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:54:20.891 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repo Size 2023-10-01T08:54:20.892 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:54:20.892 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:54:20.892 INFO:teuthology.orchestra.run.smithi138.stdout: libcephfs-devel x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 25 k 2023-10-01T08:54:20.892 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:54:20.892 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:54:20.892 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:54:20.892 INFO:teuthology.orchestra.run.smithi138.stdout:Install 1 Package 2023-10-01T08:54:20.892 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:54:20.893 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 25 k 2023-10-01T08:54:20.893 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 107 k 2023-10-01T08:54:20.893 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:54:21.064 INFO:teuthology.orchestra.run.smithi138.stdout:libcephfs-devel-16.2.14-44.geeb13ac9.el8.x86_64 149 kB/s | 25 kB 00:00 2023-10-01T08:54:21.064 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:54:21.064 INFO:teuthology.orchestra.run.smithi138.stdout:Total 148 kB/s | 25 kB 00:00 2023-10-01T08:54:21.065 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:54:21.068 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:54:21.069 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:54:21.071 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:54:21.075 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:54:21.147 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:54:21.335 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : libcephfs-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:21.429 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : libcephfs-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:21.429 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:04:16 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:21.535 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: libcephfs-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : libcephfs-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:22.585 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:04:17 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:26.920 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:54:27.104 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:54:27.104 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:54:27.104 INFO:teuthology.orchestra.run.smithi190.stdout: libcephfs-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:54:27.104 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:54:27.104 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:54:27.250 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install librados2 2023-10-01T08:54:27.604 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:29.203 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:04:24 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:29.801 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:54:29.984 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:54:29.984 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:54:29.984 INFO:teuthology.orchestra.run.smithi138.stdout: libcephfs-devel-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:54:29.984 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:54:29.985 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:54:30.134 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install librados2 2023-10-01T08:54:30.457 INFO:teuthology.orchestra.run.smithi190.stdout:Package librados2-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:54:30.490 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:30.567 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:54:30.568 INFO:teuthology.orchestra.run.smithi190.stdout:Nothing to do. 2023-10-01T08:54:30.568 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:54:30.699 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install librbd1 2023-10-01T08:54:31.054 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:32.099 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:04:27 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:32.582 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:04:27 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:33.430 INFO:teuthology.orchestra.run.smithi138.stdout:Package librados2-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:54:33.541 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:54:33.542 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-10-01T08:54:33.542 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:54:33.674 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install librbd1 2023-10-01T08:54:33.833 INFO:teuthology.orchestra.run.smithi190.stdout:Package librbd1-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:54:33.939 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:54:33.940 INFO:teuthology.orchestra.run.smithi190.stdout:Nothing to do. 2023-10-01T08:54:33.940 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:54:34.029 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:34.082 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install python3-rados 2023-10-01T08:54:34.434 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:35.535 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:04:30 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:35.877 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:04:30 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:36.862 INFO:teuthology.orchestra.run.smithi138.stdout:Package librbd1-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:54:36.972 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:54:36.973 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-10-01T08:54:36.974 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:54:37.097 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install python3-rados 2023-10-01T08:54:37.125 INFO:teuthology.orchestra.run.smithi190.stdout:Package python3-rados-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:54:37.234 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:54:37.236 INFO:teuthology.orchestra.run.smithi190.stdout:Nothing to do. 2023-10-01T08:54:37.236 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:54:37.448 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:37.492 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install python3-rgw 2023-10-01T08:54:37.846 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:38.999 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:04:33 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:39.358 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:04:34 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:40.257 INFO:teuthology.orchestra.run.smithi138.stdout:Package python3-rados-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:54:40.366 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:54:40.367 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-10-01T08:54:40.367 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:54:40.490 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install python3-rgw 2023-10-01T08:54:40.615 INFO:teuthology.orchestra.run.smithi190.stdout:Package python3-rgw-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:54:40.727 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:54:40.728 INFO:teuthology.orchestra.run.smithi190.stdout:Nothing to do. 2023-10-01T08:54:40.729 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:54:40.844 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:40.941 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install python3-cephfs 2023-10-01T08:54:41.293 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:42.420 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:04:37 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:42.738 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:04:37 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:43.746 INFO:teuthology.orchestra.run.smithi138.stdout:Package python3-rgw-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:54:43.856 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:54:43.857 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-10-01T08:54:43.857 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:54:44.014 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install python3-cephfs 2023-10-01T08:54:44.057 INFO:teuthology.orchestra.run.smithi190.stdout:Package python3-cephfs-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:54:44.168 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:54:44.169 INFO:teuthology.orchestra.run.smithi190.stdout:Nothing to do. 2023-10-01T08:54:44.169 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:54:44.366 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:44.374 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install python3-rbd 2023-10-01T08:54:44.726 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:45.787 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:04:40 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:46.174 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:04:41 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:47.061 INFO:teuthology.orchestra.run.smithi138.stdout:Package python3-cephfs-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:54:47.171 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:54:47.172 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-10-01T08:54:47.172 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:54:47.305 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install python3-rbd 2023-10-01T08:54:47.464 INFO:teuthology.orchestra.run.smithi190.stdout:Package python3-rbd-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:54:47.572 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:54:47.573 INFO:teuthology.orchestra.run.smithi190.stdout:Nothing to do. 2023-10-01T08:54:47.573 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:54:47.658 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:47.772 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install rbd-fuse 2023-10-01T08:54:48.125 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:49.173 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:04:44 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:49.705 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:04:44 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:50.507 INFO:teuthology.orchestra.run.smithi138.stdout:Package python3-rbd-2:16.2.14-44.geeb13ac9.el8.x86_64 is already installed. 2023-10-01T08:54:50.617 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:54:50.618 INFO:teuthology.orchestra.run.smithi138.stdout:Nothing to do. 2023-10-01T08:54:50.618 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:54:50.746 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install rbd-fuse 2023-10-01T08:54:51.106 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:54:51.146 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:54:51.146 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:54:51.146 INFO:teuthology.orchestra.run.smithi190.stdout: Package Architecture Version Repository Size 2023-10-01T08:54:51.147 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:54:51.147 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:54:51.147 INFO:teuthology.orchestra.run.smithi190.stdout: rbd-fuse x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 88 k 2023-10-01T08:54:51.147 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:54:51.147 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:54:51.147 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:54:51.147 INFO:teuthology.orchestra.run.smithi190.stdout:Install 1 Package 2023-10-01T08:54:51.147 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:54:51.147 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 88 k 2023-10-01T08:54:51.147 INFO:teuthology.orchestra.run.smithi190.stdout:Installed size: 240 k 2023-10-01T08:54:51.147 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:54:51.499 INFO:teuthology.orchestra.run.smithi190.stdout:rbd-fuse-16.2.14-44.geeb13ac9.el8.x86_64.rpm 250 kB/s | 88 kB 00:00 2023-10-01T08:54:51.500 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:54:51.500 INFO:teuthology.orchestra.run.smithi190.stdout:Total 249 kB/s | 88 kB 00:00 2023-10-01T08:54:51.500 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:54:51.510 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:54:51.510 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:54:51.566 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:54:51.569 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:54:51.683 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:54:51.833 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : rbd-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:52.057 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: rbd-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:52.746 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:04:47 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:53.149 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : rbd-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:53.149 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:04:48 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:54:54.181 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:54:54.182 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:54:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: Package Architecture Version Repository Size 2023-10-01T08:54:54.182 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:54:54.182 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:54:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: rbd-fuse x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 88 k 2023-10-01T08:54:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:54:54.182 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:54:54.182 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:54:54.183 INFO:teuthology.orchestra.run.smithi138.stdout:Install 1 Package 2023-10-01T08:54:54.183 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:54:54.183 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 88 k 2023-10-01T08:54:54.183 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 240 k 2023-10-01T08:54:54.183 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:54:54.440 INFO:teuthology.orchestra.run.smithi138.stdout:rbd-fuse-16.2.14-44.geeb13ac9.el8.x86_64.rpm 344 kB/s | 88 kB 00:00 2023-10-01T08:54:54.441 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:54:54.441 INFO:teuthology.orchestra.run.smithi138.stdout:Total 342 kB/s | 88 kB 00:00 2023-10-01T08:54:54.441 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:54:54.450 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:54:54.451 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:54:54.504 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:54:54.508 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:54:54.614 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:54:54.805 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : rbd-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:55.029 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: rbd-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:56.146 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : rbd-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:54:56.147 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:04:51 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:55:01.493 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:55:01.678 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:55:01.679 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:55:01.679 INFO:teuthology.orchestra.run.smithi190.stdout: rbd-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:55:01.679 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:55:01.679 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:55:01.849 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install rbd-mirror 2023-10-01T08:55:02.204 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:55:03.451 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:55:03.634 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:55:03.634 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:55:03.634 INFO:teuthology.orchestra.run.smithi138.stdout: rbd-fuse-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:55:03.634 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:55:03.634 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:55:03.704 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:04:58 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:55:03.783 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install rbd-mirror 2023-10-01T08:55:04.138 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:55:05.068 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:55:05.068 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:55:05.068 INFO:teuthology.orchestra.run.smithi190.stdout: Package Arch Version Repo Size 2023-10-01T08:55:05.068 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:55:05.068 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:55:05.069 INFO:teuthology.orchestra.run.smithi190.stdout: rbd-mirror x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 3.7 M 2023-10-01T08:55:05.069 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:55:05.069 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:55:05.069 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:55:05.069 INFO:teuthology.orchestra.run.smithi190.stdout:Install 1 Package 2023-10-01T08:55:05.069 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:55:05.070 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 3.7 M 2023-10-01T08:55:05.070 INFO:teuthology.orchestra.run.smithi190.stdout:Installed size: 15 M 2023-10-01T08:55:05.070 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:55:05.443 INFO:teuthology.orchestra.run.smithi190.stdout:rbd-mirror-16.2.14-44.geeb13ac9.el8.x86_64.rpm 9.8 MB/s | 3.7 MB 00:00 2023-10-01T08:55:05.443 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:55:05.443 INFO:teuthology.orchestra.run.smithi190.stdout:Total 9.8 MB/s | 3.7 MB 00:00 2023-10-01T08:55:05.444 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:55:05.455 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:55:05.455 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:55:05.521 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:55:05.525 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:55:05.660 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:05:00 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:55:05.961 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:55:06.125 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : rbd-mirror-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:55:06.478 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: rbd-mirror-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:55:07.127 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:55:07.128 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:55:07.128 INFO:teuthology.orchestra.run.smithi138.stdout: Package Arch Version Repo Size 2023-10-01T08:55:07.128 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:55:07.128 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:55:07.128 INFO:teuthology.orchestra.run.smithi138.stdout: rbd-mirror x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 3.7 M 2023-10-01T08:55:07.128 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:55:07.128 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:55:07.128 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:55:07.128 INFO:teuthology.orchestra.run.smithi138.stdout:Install 1 Package 2023-10-01T08:55:07.128 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:55:07.129 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 3.7 M 2023-10-01T08:55:07.129 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 15 M 2023-10-01T08:55:07.129 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:55:07.521 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : rbd-mirror-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:55:07.521 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:05:02 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:55:07.522 INFO:teuthology.orchestra.run.smithi138.stdout:rbd-mirror-16.2.14-44.geeb13ac9.el8.x86_64.rpm 9.3 MB/s | 3.7 MB 00:00 2023-10-01T08:55:07.523 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:55:07.523 INFO:teuthology.orchestra.run.smithi138.stdout:Total 9.3 MB/s | 3.7 MB 00:00 2023-10-01T08:55:07.524 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:55:07.535 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:55:07.535 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:55:07.603 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:55:07.607 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:55:08.053 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:55:08.239 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : rbd-mirror-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:55:08.597 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: rbd-mirror-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:55:09.674 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : rbd-mirror-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:55:09.675 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:05:04 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:55:15.541 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:55:15.693 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:55:15.723 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:55:15.724 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:55:15.724 INFO:teuthology.orchestra.run.smithi190.stdout: rbd-mirror-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:55:15.724 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:55:15.724 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:55:15.875 DEBUG:teuthology.orchestra.run.smithi190:> sudo yum -y install rbd-nbd 2023-10-01T08:55:15.877 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:55:15.877 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:55:15.877 INFO:teuthology.orchestra.run.smithi138.stdout: rbd-mirror-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:55:15.877 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:55:15.877 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:55:16.016 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum -y install rbd-nbd 2023-10-01T08:55:16.230 INFO:teuthology.orchestra.run.smithi190.stdout:Updating Subscription Management repositories. 2023-10-01T08:55:16.374 INFO:teuthology.orchestra.run.smithi138.stdout:Updating Subscription Management repositories. 2023-10-01T08:55:17.701 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:05:12 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:55:17.880 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:05:12 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:55:19.059 INFO:teuthology.orchestra.run.smithi190.stdout:Dependencies resolved. 2023-10-01T08:55:19.060 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:55:19.060 INFO:teuthology.orchestra.run.smithi190.stdout: Package Architecture Version Repository Size 2023-10-01T08:55:19.060 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:55:19.060 INFO:teuthology.orchestra.run.smithi190.stdout:Installing: 2023-10-01T08:55:19.061 INFO:teuthology.orchestra.run.smithi190.stdout: rbd-nbd x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 173 k 2023-10-01T08:55:19.061 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:55:19.061 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction Summary 2023-10-01T08:55:19.061 INFO:teuthology.orchestra.run.smithi190.stdout:================================================================================ 2023-10-01T08:55:19.061 INFO:teuthology.orchestra.run.smithi190.stdout:Install 1 Package 2023-10-01T08:55:19.061 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:55:19.062 INFO:teuthology.orchestra.run.smithi190.stdout:Total download size: 173 k 2023-10-01T08:55:19.062 INFO:teuthology.orchestra.run.smithi190.stdout:Installed size: 515 k 2023-10-01T08:55:19.063 INFO:teuthology.orchestra.run.smithi190.stdout:Downloading Packages: 2023-10-01T08:55:19.315 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2023-10-01T08:55:19.316 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:55:19.316 INFO:teuthology.orchestra.run.smithi138.stdout: Package Architecture Version Repository Size 2023-10-01T08:55:19.316 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:55:19.316 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2023-10-01T08:55:19.316 INFO:teuthology.orchestra.run.smithi138.stdout: rbd-nbd x86_64 2:16.2.14-44.geeb13ac9.el8 ceph 173 k 2023-10-01T08:55:19.316 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:55:19.316 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2023-10-01T08:55:19.316 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2023-10-01T08:55:19.317 INFO:teuthology.orchestra.run.smithi138.stdout:Install 1 Package 2023-10-01T08:55:19.317 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:55:19.317 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 173 k 2023-10-01T08:55:19.317 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 515 k 2023-10-01T08:55:19.317 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2023-10-01T08:55:19.577 INFO:teuthology.orchestra.run.smithi190.stdout:rbd-nbd-16.2.14-44.geeb13ac9.el8.x86_64.rpm 336 kB/s | 173 kB 00:00 2023-10-01T08:55:19.578 INFO:teuthology.orchestra.run.smithi190.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:55:19.578 INFO:teuthology.orchestra.run.smithi190.stdout:Total 335 kB/s | 173 kB 00:00 2023-10-01T08:55:19.578 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction check 2023-10-01T08:55:19.589 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction check succeeded. 2023-10-01T08:55:19.589 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction test 2023-10-01T08:55:19.595 INFO:teuthology.orchestra.run.smithi138.stdout:rbd-nbd-16.2.14-44.geeb13ac9.el8.x86_64.rpm 623 kB/s | 173 kB 00:00 2023-10-01T08:55:19.595 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2023-10-01T08:55:19.596 INFO:teuthology.orchestra.run.smithi138.stdout:Total 621 kB/s | 173 kB 00:00 2023-10-01T08:55:19.596 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2023-10-01T08:55:19.606 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2023-10-01T08:55:19.607 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2023-10-01T08:55:19.644 INFO:teuthology.orchestra.run.smithi190.stdout:Transaction test succeeded. 2023-10-01T08:55:19.648 INFO:teuthology.orchestra.run.smithi190.stdout:Running transaction 2023-10-01T08:55:19.662 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2023-10-01T08:55:19.665 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2023-10-01T08:55:19.764 INFO:teuthology.orchestra.run.smithi190.stdout: Preparing : 1/1 2023-10-01T08:55:19.778 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2023-10-01T08:55:19.921 INFO:teuthology.orchestra.run.smithi190.stdout: Installing : rbd-nbd-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:55:19.934 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : rbd-nbd-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:55:20.141 INFO:teuthology.orchestra.run.smithi190.stdout: Running scriptlet: rbd-nbd-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:55:20.152 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: rbd-nbd-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:55:21.200 INFO:teuthology.orchestra.run.smithi190.stdout: Verifying : rbd-nbd-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:55:21.200 INFO:teuthology.orchestra.run.smithi190.stdout:Last metadata expiration check: 0:05:16 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:55:21.258 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : rbd-nbd-2:16.2.14-44.geeb13ac9.el8.x86_64 1/1 2023-10-01T08:55:21.258 INFO:teuthology.orchestra.run.smithi138.stdout:Last metadata expiration check: 0:05:16 ago on Sun 01 Oct 2023 08:50:05 AM UTC. 2023-10-01T08:55:28.938 INFO:teuthology.orchestra.run.smithi190.stdout:Installed products updated. 2023-10-01T08:55:29.121 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:55:29.121 INFO:teuthology.orchestra.run.smithi190.stdout:Installed: 2023-10-01T08:55:29.121 INFO:teuthology.orchestra.run.smithi190.stdout: rbd-nbd-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:55:29.121 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:55:29.121 INFO:teuthology.orchestra.run.smithi190.stdout:Complete! 2023-10-01T08:55:29.252 DEBUG:teuthology.parallel:result is None 2023-10-01T08:55:29.350 INFO:teuthology.orchestra.run.smithi138.stdout:Installed products updated. 2023-10-01T08:55:29.535 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:55:29.536 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2023-10-01T08:55:29.536 INFO:teuthology.orchestra.run.smithi138.stdout: rbd-nbd-2:16.2.14-44.geeb13ac9.el8.x86_64 2023-10-01T08:55:29.536 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:55:29.536 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2023-10-01T08:55:29.681 DEBUG:teuthology.parallel:result is None 2023-10-01T08:55:29.681 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-01T08:55:29.682 INFO:teuthology.packaging:ref: None 2023-10-01T08:55:29.682 INFO:teuthology.packaging:tag: None 2023-10-01T08:55:29.682 INFO:teuthology.packaging:branch: pacific 2023-10-01T08:55:29.682 INFO:teuthology.packaging:sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T08:55:29.682 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-10-01T08:55:29.983 DEBUG:teuthology.orchestra.run.smithi138:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-01T08:55:30.006 INFO:teuthology.orchestra.run.smithi138.stdout:16.2.14-44.geeb13ac9.el8 2023-10-01T08:55:30.007 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-44.geeb13ac9.el8 2023-10-01T08:55:30.007 INFO:teuthology.task.install:The correct ceph version 16.2.14-44.geeb13ac9 is installed. 2023-10-01T08:55:30.007 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-01T08:55:30.008 INFO:teuthology.packaging:ref: None 2023-10-01T08:55:30.008 INFO:teuthology.packaging:tag: None 2023-10-01T08:55:30.008 INFO:teuthology.packaging:branch: pacific 2023-10-01T08:55:30.008 INFO:teuthology.packaging:sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T08:55:30.008 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-10-01T08:55:30.210 DEBUG:teuthology.orchestra.run.smithi190:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-10-01T08:55:30.237 INFO:teuthology.orchestra.run.smithi190.stdout:16.2.14-44.geeb13ac9.el8 2023-10-01T08:55:30.237 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-44.geeb13ac9.el8 2023-10-01T08:55:30.237 INFO:teuthology.task.install:The correct ceph version 16.2.14-44.geeb13ac9 is installed. 2023-10-01T08:55:30.237 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-10-01T08:55:30.237 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T08:55:30.237 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-01T08:55:30.266 DEBUG:teuthology.orchestra.run.smithi190:> set -ex 2023-10-01T08:55:30.266 DEBUG:teuthology.orchestra.run.smithi190:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-01T08:55:30.310 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-10-01T08:55:30.310 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T08:55:30.311 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/usr/bin/daemon-helper 2023-10-01T08:55:30.338 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-01T08:55:30.405 DEBUG:teuthology.orchestra.run.smithi190:> set -ex 2023-10-01T08:55:30.406 DEBUG:teuthology.orchestra.run.smithi190:> sudo dd of=/usr/bin/daemon-helper 2023-10-01T08:55:30.436 DEBUG:teuthology.orchestra.run.smithi190:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-01T08:55:30.505 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-10-01T08:55:30.506 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T08:55:30.506 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-01T08:55:30.533 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-01T08:55:30.599 DEBUG:teuthology.orchestra.run.smithi190:> set -ex 2023-10-01T08:55:30.600 DEBUG:teuthology.orchestra.run.smithi190:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-01T08:55:30.628 DEBUG:teuthology.orchestra.run.smithi190:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-01T08:55:30.697 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-10-01T08:55:30.698 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T08:55:30.698 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/usr/bin/stdin-killer 2023-10-01T08:55:30.725 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-01T08:55:30.791 DEBUG:teuthology.orchestra.run.smithi190:> set -ex 2023-10-01T08:55:30.791 DEBUG:teuthology.orchestra.run.smithi190:> sudo dd of=/usr/bin/stdin-killer 2023-10-01T08:55:30.819 DEBUG:teuthology.orchestra.run.smithi190:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-01T08:55:30.886 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-01T08:55:30.968 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b56822191ad0416d0c05ed59720df03ad9c2a44e'} 2023-10-01T08:55:30.969 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-10-01T08:55:30.970 INFO:tasks.cephadm:Cluster fsid is 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:55:30.970 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-01T08:55:30.970 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.138', 'mon.c': '[v2:172.21.15.138:3301,v1:172.21.15.138:6790]', 'mon.b': '172.21.15.190'} 2023-10-01T08:55:30.970 INFO:tasks.cephadm:First mon is mon.a on smithi138 2023-10-01T08:55:30.970 INFO:tasks.cephadm:First mgr is y 2023-10-01T08:55:30.970 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-01T08:55:30.971 DEBUG:teuthology.orchestra.run.smithi138:> sudo hostname $(hostname -s) 2023-10-01T08:55:30.998 DEBUG:teuthology.orchestra.run.smithi190:> sudo hostname $(hostname -s) 2023-10-01T08:55:31.026 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-10-01T08:55:31.026 DEBUG:teuthology.orchestra.run.smithi138:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-01T08:55:31.185 INFO:teuthology.orchestra.run.smithi138.stdout:-rw-rw-r--. 1 ubuntu ubuntu 350873 Oct 1 08:55 /home/ubuntu/cephtest/cephadm 2023-10-01T08:55:31.185 DEBUG:teuthology.orchestra.run.smithi190:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-01T08:55:31.330 INFO:teuthology.orchestra.run.smithi190.stdout:-rw-rw-r--. 1 ubuntu ubuntu 350873 Oct 1 08:55 /home/ubuntu/cephtest/cephadm 2023-10-01T08:55:31.330 DEBUG:teuthology.orchestra.run.smithi138:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-01T08:55:31.348 DEBUG:teuthology.orchestra.run.smithi190:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-01T08:55:31.376 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-10-01T08:55:31.376 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-01T08:55:31.390 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-01T08:55:32.572 INFO:teuthology.orchestra.run.smithi138.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-01T08:55:32.611 INFO:teuthology.orchestra.run.smithi190.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-01T08:56:44.370 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T08:56:44.371 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph_version": "ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)", 2023-10-01T08:56:44.371 INFO:teuthology.orchestra.run.smithi138.stdout: "image_id": "94a6c4498547c13992866c6d194f294b2191daf4f02c7d4d133790c792e33bf5", 2023-10-01T08:56:44.371 INFO:teuthology.orchestra.run.smithi138.stdout: "repo_digests": [ 2023-10-01T08:56:44.371 INFO:teuthology.orchestra.run.smithi138.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:8e9fcccea6e0115c1572f3a4163cdf39971d9e270ff3987b3049e8aad314c194" 2023-10-01T08:56:44.371 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T08:56:44.371 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T08:56:46.681 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T08:56:46.681 INFO:teuthology.orchestra.run.smithi190.stdout: "ceph_version": "ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)", 2023-10-01T08:56:46.681 INFO:teuthology.orchestra.run.smithi190.stdout: "image_id": "94a6c4498547c13992866c6d194f294b2191daf4f02c7d4d133790c792e33bf5", 2023-10-01T08:56:46.682 INFO:teuthology.orchestra.run.smithi190.stdout: "repo_digests": [ 2023-10-01T08:56:46.682 INFO:teuthology.orchestra.run.smithi190.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:8e9fcccea6e0115c1572f3a4163cdf39971d9e270ff3987b3049e8aad314c194" 2023-10-01T08:56:46.682 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T08:56:46.682 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T08:56:46.707 DEBUG:teuthology.orchestra.run.smithi138:> sudo mkdir -p /etc/ceph 2023-10-01T08:56:46.743 DEBUG:teuthology.orchestra.run.smithi190:> sudo mkdir -p /etc/ceph 2023-10-01T08:56:46.776 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod 777 /etc/ceph 2023-10-01T08:56:46.814 DEBUG:teuthology.orchestra.run.smithi190:> sudo chmod 777 /etc/ceph 2023-10-01T08:56:46.845 INFO:tasks.cephadm:Writing seed config... 2023-10-01T08:56:46.846 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-10-01T08:56:46.846 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-10-01T08:56:46.846 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-01T08:56:46.846 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-01T08:56:46.846 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-10-01T08:56:46.846 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-10-01T08:56:46.846 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-01T08:56:46.846 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-01T08:56:46.846 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-01T08:56:46.847 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-01T08:56:46.847 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-01T08:56:46.847 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T08:56:46.847 DEBUG:teuthology.orchestra.run.smithi138:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-01T08:56:46.874 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 465870a4-6038-11ee-8db5-212e2dc638e7 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-01T08:56:46.874 DEBUG:teuthology.orchestra.run.smithi138:mon.a> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-01T08:56:46.916 DEBUG:teuthology.orchestra.run.smithi138:mgr.y> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.y.service 2023-10-01T08:56:46.945 INFO:journalctl@ceph.mon.a.smithi138.stdout:-- Logs begin at Sun 2023-10-01 08:38:26 UTC. -- 2023-10-01T08:56:46.946 INFO:tasks.cephadm:Bootstrapping... 2023-10-01T08:56:46.946 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.138 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-01T08:56:46.972 INFO:journalctl@ceph.mgr.y.smithi138.stdout:-- Logs begin at Sun 2023-10-01 08:38:26 UTC. -- 2023-10-01T08:56:47.149 INFO:teuthology.orchestra.run.smithi138.stderr:-------------------------------------------------------------------------------- 2023-10-01T08:56:47.149 INFO:teuthology.orchestra.run.smithi138.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', '465870a4-6038-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.138', '--skip-admin-label'] 2023-10-01T08:56:47.195 INFO:teuthology.orchestra.run.smithi138.stderr:/bin/podman: 3.0.1 2023-10-01T08:56:47.213 INFO:teuthology.orchestra.run.smithi138.stderr:Verifying podman|docker is present... 2023-10-01T08:56:47.257 INFO:teuthology.orchestra.run.smithi138.stderr:/bin/podman: 3.0.1 2023-10-01T08:56:47.274 INFO:teuthology.orchestra.run.smithi138.stderr:Verifying lvm2 is present... 2023-10-01T08:56:47.274 INFO:teuthology.orchestra.run.smithi138.stderr:Verifying time synchronization is in place... 2023-10-01T08:56:47.281 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-01T08:56:47.286 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: inactive 2023-10-01T08:56:47.292 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: enabled 2023-10-01T08:56:47.297 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: active 2023-10-01T08:56:47.298 INFO:teuthology.orchestra.run.smithi138.stderr:Unit chronyd.service is enabled and running 2023-10-01T08:56:47.298 INFO:teuthology.orchestra.run.smithi138.stderr:Repeating the final host check... 2023-10-01T08:56:47.340 INFO:teuthology.orchestra.run.smithi138.stderr:/bin/podman: 3.0.1 2023-10-01T08:56:47.354 INFO:teuthology.orchestra.run.smithi138.stderr:podman (/bin/podman) version 3.0.1 is present 2023-10-01T08:56:47.354 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl is present 2023-10-01T08:56:47.354 INFO:teuthology.orchestra.run.smithi138.stderr:lvcreate is present 2023-10-01T08:56:47.359 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-01T08:56:47.365 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: inactive 2023-10-01T08:56:47.371 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: enabled 2023-10-01T08:56:47.376 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: active 2023-10-01T08:56:47.377 INFO:teuthology.orchestra.run.smithi138.stderr:Unit chronyd.service is enabled and running 2023-10-01T08:56:47.377 INFO:teuthology.orchestra.run.smithi138.stderr:Host looks OK 2023-10-01T08:56:47.377 INFO:teuthology.orchestra.run.smithi138.stderr:Cluster fsid: 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:56:47.377 INFO:teuthology.orchestra.run.smithi138.stderr:Acquiring lock 140329541192168 on /run/cephadm/465870a4-6038-11ee-8db5-212e2dc638e7.lock 2023-10-01T08:56:47.377 INFO:teuthology.orchestra.run.smithi138.stderr:Lock 140329541192168 acquired on /run/cephadm/465870a4-6038-11ee-8db5-212e2dc638e7.lock 2023-10-01T08:56:47.378 INFO:teuthology.orchestra.run.smithi138.stderr:Verifying IP 172.21.15.138 port 3300 ... 2023-10-01T08:56:47.378 INFO:teuthology.orchestra.run.smithi138.stderr:Verifying IP 172.21.15.138 port 6789 ... 2023-10-01T08:56:47.378 INFO:teuthology.orchestra.run.smithi138.stderr:Base mon IP(s) is [172.21.15.138:3300, 172.21.15.138:6789], mon addrv is [v2:172.21.15.138:3300,v1:172.21.15.138:6789] 2023-10-01T08:56:47.382 INFO:teuthology.orchestra.run.smithi138.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp metric 100 2023-10-01T08:56:47.382 INFO:teuthology.orchestra.run.smithi138.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.138 metric 100 2023-10-01T08:56:47.384 INFO:teuthology.orchestra.run.smithi138.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-10-01T08:56:47.384 INFO:teuthology.orchestra.run.smithi138.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2023-10-01T08:56:47.384 INFO:teuthology.orchestra.run.smithi138.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2023-10-01T08:56:47.387 INFO:teuthology.orchestra.run.smithi138.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-01T08:56:47.387 INFO:teuthology.orchestra.run.smithi138.stderr:/sbin/ip: inet6 ::1/128 scope host 2023-10-01T08:56:47.387 INFO:teuthology.orchestra.run.smithi138.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-10-01T08:56:47.387 INFO:teuthology.orchestra.run.smithi138.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-10-01T08:56:47.387 INFO:teuthology.orchestra.run.smithi138.stderr:/sbin/ip: inet6 fe80::c511:67e3:68df:ed2/64 scope link noprefixroute 2023-10-01T08:56:47.387 INFO:teuthology.orchestra.run.smithi138.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-10-01T08:56:47.388 INFO:teuthology.orchestra.run.smithi138.stderr:Mon IP `172.21.15.138` is in CIDR network `172.21.0.0/20` 2023-10-01T08:56:47.388 INFO:teuthology.orchestra.run.smithi138.stderr:Mon IP `172.21.15.138` is in CIDR network `172.21.0.0/20` 2023-10-01T08:56:47.388 INFO:teuthology.orchestra.run.smithi138.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-10-01T08:56:47.389 INFO:teuthology.orchestra.run.smithi138.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-01T08:56:47.389 INFO:teuthology.orchestra.run.smithi138.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-01T08:56:47.460 INFO:teuthology.orchestra.run.smithi138.stderr:/bin/podman: Trying to pull quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-01T08:56:47.562 INFO:teuthology.orchestra.run.smithi138.stderr:/bin/podman: Getting image source signatures 2023-10-01T08:56:47.563 INFO:teuthology.orchestra.run.smithi138.stderr:/bin/podman: Copying blob sha256:1c2ae6d96d4402dd464b09e8ada99076c55bdd8d8215d5bf7798b31d5926335d 2023-10-01T08:56:47.563 INFO:teuthology.orchestra.run.smithi138.stderr:/bin/podman: Copying blob sha256:3f4fd5c3c800c6da9ef94691df203a05a5bbbf32c483d09d620ec06eef42ea2b 2023-10-01T08:56:47.563 INFO:teuthology.orchestra.run.smithi138.stderr:/bin/podman: Copying config sha256:94a6c4498547c13992866c6d194f294b2191daf4f02c7d4d133790c792e33bf5 2023-10-01T08:56:47.617 INFO:teuthology.orchestra.run.smithi138.stderr:/bin/podman: Writing manifest to image destination 2023-10-01T08:56:47.617 INFO:teuthology.orchestra.run.smithi138.stderr:/bin/podman: Storing signatures 2023-10-01T08:56:47.814 INFO:teuthology.orchestra.run.smithi138.stderr:/bin/podman: 94a6c4498547c13992866c6d194f294b2191daf4f02c7d4d133790c792e33bf5 2023-10-01T08:56:48.268 INFO:teuthology.orchestra.run.smithi138.stderr:ceph: ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable) 2023-10-01T08:56:48.794 INFO:teuthology.orchestra.run.smithi138.stderr:Ceph version: ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable) 2023-10-01T08:56:48.795 INFO:teuthology.orchestra.run.smithi138.stderr:Extracting ceph user uid/gid from container image... 2023-10-01T08:56:49.165 INFO:teuthology.orchestra.run.smithi138.stderr:stat: 167 167 2023-10-01T08:56:49.614 INFO:teuthology.orchestra.run.smithi138.stderr:Creating initial keys... 2023-10-01T08:56:50.006 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-authtool: AQBRNBll8cIKOxAA3GSoowvrYSG9WWVU3J5TGg== 2023-10-01T08:56:50.873 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-authtool: AQBSNBllNpLLMRAAyay3xyKTRxZ+CjXT158apg== 2023-10-01T08:56:51.722 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-authtool: AQBTNBllHo/PKRAAOIIObopZJKOnUKUozti9jQ== 2023-10-01T08:56:52.163 INFO:teuthology.orchestra.run.smithi138.stderr:Creating initial monmap... 2023-10-01T08:56:52.506 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-01T08:56:52.506 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:56:52.506 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-01T08:56:53.019 INFO:teuthology.orchestra.run.smithi138.stderr:monmaptool for a [v2:172.21.15.138:3300,v1:172.21.15.138:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-01T08:56:53.021 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/monmaptool: set fsid to 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:56:53.021 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-01T08:56:53.021 INFO:teuthology.orchestra.run.smithi138.stderr: 2023-10-01T08:56:53.021 INFO:teuthology.orchestra.run.smithi138.stderr:Creating mon... 2023-10-01T08:56:53.454 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.417+0000 7fde85e8c880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-01T08:56:53.454 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.417+0000 7fde85e8c880 1 imported monmap: 2023-10-01T08:56:53.455 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: epoch 0 2023-10-01T08:56:53.455 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: fsid 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:56:53.455 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: last_changed 2023-10-01T08:56:52.494621+0000 2023-10-01T08:56:53.455 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: created 2023-10-01T08:56:52.494621+0000 2023-10-01T08:56:53.455 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-10-01T08:56:53.455 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-10-01T08:56:53.455 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon.a 2023-10-01T08:56:53.455 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.455 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.417+0000 7fde85e8c880 0 /usr/bin/ceph-mon: set fsid to 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:56:53.455 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: RocksDB version: 6.8.1 2023-10-01T08:56:53.455 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.456 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-01T08:56:53.456 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Compile date Sep 25 2023 2023-10-01T08:56:53.456 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: DB SUMMARY 2023-10-01T08:56:53.456 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.456 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-10-01T08:56:53.456 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.456 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-10-01T08:56:53.456 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.456 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.error_if_exists: 0 2023-10-01T08:56:53.456 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.create_if_missing: 1 2023-10-01T08:56:53.456 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.paranoid_checks: 1 2023-10-01T08:56:53.457 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.env: 0x559ea1b0b080 2023-10-01T08:56:53.457 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.fs: Posix File System 2023-10-01T08:56:53.457 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.info_log: 0x559ea24adaa0 2023-10-01T08:56:53.457 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-01T08:56:53.457 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.statistics: (nil) 2023-10-01T08:56:53.457 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.use_fsync: 0 2023-10-01T08:56:53.457 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.max_log_file_size: 0 2023-10-01T08:56:53.457 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T08:56:53.457 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T08:56:53.457 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-01T08:56:53.457 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-01T08:56:53.458 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.allow_fallocate: 1 2023-10-01T08:56:53.458 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-01T08:56:53.458 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-01T08:56:53.458 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.use_direct_reads: 0 2023-10-01T08:56:53.458 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T08:56:53.458 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-01T08:56:53.458 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.db_log_dir: 2023-10-01T08:56:53.458 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-01T08:56:53.458 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T08:56:53.458 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.max_subcompactions: 1 2023-10-01T08:56:53.458 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.max_background_flushes: -1 2023-10-01T08:56:53.458 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T08:56:53.459 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T08:56:53.459 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T08:56:53.459 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T08:56:53.459 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T08:56:53.459 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-01T08:56:53.459 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-01T08:56:53.459 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.write_buffer_manager: 0x559ea24b5890 2023-10-01T08:56:53.459 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T08:56:53.459 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-01T08:56:53.459 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T08:56:53.459 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T08:56:53.460 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-01T08:56:53.460 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T08:56:53.460 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-01T08:56:53.460 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-01T08:56:53.460 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-01T08:56:53.460 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.unordered_write: 0 2023-10-01T08:56:53.460 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T08:56:53.460 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T08:56:53.460 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T08:56:53.460 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T08:56:53.460 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.row_cache: None 2023-10-01T08:56:53.460 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.wal_filter: None 2023-10-01T08:56:53.461 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T08:56:53.461 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-01T08:56:53.461 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.preserve_deletes: 0 2023-10-01T08:56:53.461 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.two_write_queues: 0 2023-10-01T08:56:53.461 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-01T08:56:53.461 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.atomic_flush: 0 2023-10-01T08:56:53.461 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T08:56:53.461 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T08:56:53.461 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T08:56:53.461 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.log_readahead_size: 0 2023-10-01T08:56:53.461 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-01T08:56:53.462 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.max_background_jobs: 2 2023-10-01T08:56:53.462 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.max_background_compactions: -1 2023-10-01T08:56:53.462 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T08:56:53.462 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T08:56:53.462 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T08:56:53.462 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-01T08:56:53.462 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T08:56:53.462 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T08:56:53.462 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T08:56:53.462 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T08:56:53.462 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.max_open_files: -1 2023-10-01T08:56:53.463 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-01T08:56:53.463 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T08:56:53.463 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T08:56:53.463 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-01T08:56:53.463 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Compression algorithms supported: 2023-10-01T08:56:53.463 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T08:56:53.463 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: kZSTD supported: 0 2023-10-01T08:56:53.463 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: kXpressCompression supported: 0 2023-10-01T08:56:53.463 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-01T08:56:53.463 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: kLZ4Compression supported: 1 2023-10-01T08:56:53.463 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: kBZip2Compression supported: 0 2023-10-01T08:56:53.463 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: kZlibCompression supported: 1 2023-10-01T08:56:53.464 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: kSnappyCompression supported: 1 2023-10-01T08:56:53.464 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.418+0000 7fde85e8c880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T08:56:53.464 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-10-01T08:56:53.464 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.464 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-10-01T08:56:53.464 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.464 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-01T08:56:53.464 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.464 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T08:56:53.464 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.merge_operator: 2023-10-01T08:56:53.464 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compaction_filter: None 2023-10-01T08:56:53.464 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-01T08:56:53.465 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T08:56:53.465 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T08:56:53.465 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559ea23bfd20) 2023-10-01T08:56:53.465 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-10-01T08:56:53.465 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T08:56:53.465 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T08:56:53.465 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-10-01T08:56:53.465 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: index_type: 0 2023-10-01T08:56:53.465 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-10-01T08:56:53.465 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-10-01T08:56:53.465 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-10-01T08:56:53.466 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-10-01T08:56:53.466 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: checksum: 1 2023-10-01T08:56:53.466 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-10-01T08:56:53.466 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: block_cache: 0x559ea23f8d10 2023-10-01T08:56:53.466 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-10-01T08:56:53.466 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-10-01T08:56:53.466 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-10-01T08:56:53.466 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-10-01T08:56:53.466 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-10-01T08:56:53.466 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-10-01T08:56:53.466 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-10-01T08:56:53.466 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-10-01T08:56:53.467 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-10-01T08:56:53.467 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-10-01T08:56:53.467 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-10-01T08:56:53.467 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-10-01T08:56:53.467 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-10-01T08:56:53.467 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-10-01T08:56:53.467 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-10-01T08:56:53.467 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-01T08:56:53.467 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-10-01T08:56:53.467 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-10-01T08:56:53.467 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-10-01T08:56:53.467 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: format_version: 2 2023-10-01T08:56:53.468 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-10-01T08:56:53.468 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: block_align: 0 2023-10-01T08:56:53.468 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.468 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-01T08:56:53.468 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-01T08:56:53.468 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compression: NoCompression 2023-10-01T08:56:53.468 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-01T08:56:53.468 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-01T08:56:53.468 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T08:56:53.468 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.num_levels: 7 2023-10-01T08:56:53.468 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T08:56:53.468 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T08:56:53.469 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T08:56:53.469 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T08:56:53.469 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T08:56:53.469 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T08:56:53.469 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T08:56:53.469 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T08:56:53.469 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T08:56:53.469 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T08:56:53.469 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-01T08:56:53.469 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-01T08:56:53.469 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T08:56:53.469 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T08:56:53.470 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-01T08:56:53.470 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T08:56:53.470 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T08:56:53.470 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T08:56:53.470 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-01T08:56:53.470 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T08:56:53.470 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T08:56:53.470 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T08:56:53.470 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T08:56:53.470 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T08:56:53.470 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T08:56:53.471 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T08:56:53.471 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T08:56:53.471 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T08:56:53.471 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T08:56:53.471 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T08:56:53.471 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T08:56:53.471 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T08:56:53.471 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-01T08:56:53.471 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T08:56:53.471 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T08:56:53.471 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-01T08:56:53.472 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-01T08:56:53.472 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T08:56:53.472 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T08:56:53.472 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T08:56:53.472 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T08:56:53.472 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T08:56:53.472 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T08:56:53.472 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T08:56:53.472 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T08:56:53.472 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T08:56:53.472 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T08:56:53.472 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.table_properties_collectors: 2023-10-01T08:56:53.473 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.inplace_update_support: 0 2023-10-01T08:56:53.473 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T08:56:53.473 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T08:56:53.473 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T08:56:53.473 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T08:56:53.473 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.bloom_locality: 0 2023-10-01T08:56:53.473 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.max_successive_merges: 0 2023-10-01T08:56:53.473 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T08:56:53.473 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-01T08:56:53.473 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-01T08:56:53.473 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-01T08:56:53.473 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.ttl: 2592000 2023-10-01T08:56:53.474 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T08:56:53.474 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T08:56:53.474 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.474 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.419+0000 7fde85e8c880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-01T08:56:53.474 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.474 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.420+0000 7fde85e8c880 4 rocksdb: DB pointer 0x559ea24c3800 2023-10-01T08:56:53.474 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.420+0000 7fde6ed15700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-01T08:56:53.474 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.420+0000 7fde6ed15700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-01T08:56:53.474 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-10-01T08:56:53.474 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T08:56:53.474 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T08:56:53.474 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T08:56:53.475 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T08:56:53.475 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T08:56:53.475 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-01T08:56:53.475 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T08:56:53.475 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.475 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-01T08:56:53.475 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:56:53.475 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:56:53.475 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-01T08:56:53.475 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-01T08:56:53.475 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.475 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-01T08:56:53.476 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:56:53.476 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:56:53.476 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T08:56:53.476 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T08:56:53.476 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T08:56:53.476 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T08:56:53.476 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T08:56:53.476 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-01T08:56:53.476 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:56:53.476 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:56:53.476 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T08:56:53.477 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.477 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-01T08:56:53.477 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.477 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-01T08:56:53.477 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:56:53.477 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:56:53.477 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-01T08:56:53.477 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-01T08:56:53.477 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.477 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-01T08:56:53.477 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:56:53.477 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:56:53.477 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T08:56:53.478 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T08:56:53.478 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T08:56:53.478 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T08:56:53.478 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T08:56:53.478 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-01T08:56:53.478 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:56:53.478 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:56:53.478 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T08:56:53.478 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.478 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-01T08:56:53.478 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: 2023-10-01T08:56:53.479 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.421+0000 7fde85e8c880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-10-01T08:56:53.479 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.421+0000 7fde85e8c880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-10-01T08:56:53.479 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph-mon: debug 2023-10-01T08:56:53.421+0000 7fde85e8c880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-10-01T08:56:53.958 INFO:teuthology.orchestra.run.smithi138.stderr:create mon.a on 2023-10-01T08:56:53.971 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-10-01T08:56:54.065 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-10-01T08:56:54.158 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-465870a4-6038-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-465870a4-6038-11ee-8db5-212e2dc638e7.target. 2023-10-01T08:56:54.158 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-465870a4-6038-11ee-8db5-212e2dc638e7.target → /etc/systemd/system/ceph-465870a4-6038-11ee-8db5-212e2dc638e7.target. 2023-10-01T08:56:54.356 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: Failed to reset failed state of unit ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-01T08:56:54.363 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: Created symlink /etc/systemd/system/ceph-465870a4-6038-11ee-8db5-212e2dc638e7.target.wants/ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-465870a4-6038-11ee-8db5-212e2dc638e7@.service. 2023-10-01T08:56:54.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:54 smithi138 systemd[1]: Starting Ceph mon.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T08:56:55.030 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: disabled 2023-10-01T08:56:55.037 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: inactive 2023-10-01T08:56:55.037 INFO:teuthology.orchestra.run.smithi138.stderr:firewalld.service is not enabled 2023-10-01T08:56:55.037 INFO:teuthology.orchestra.run.smithi138.stderr:Not possible to enable service . firewalld.service is not available 2023-10-01T08:56:55.038 INFO:teuthology.orchestra.run.smithi138.stderr:Waiting for mon to start... 2023-10-01T08:56:55.038 INFO:teuthology.orchestra.run.smithi138.stderr:Waiting for mon... 2023-10-01T08:56:55.274 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:55 smithi138 bash[99262]: 664d56ee7867af9d405cf85b68d158d7c9ca17cc1c2993859764344ba9fa63a1 2023-10-01T08:56:55.274 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:55 smithi138 systemd[1]: Started Ceph mon.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T08:56:55.274 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:55 smithi138 conmon[99303]: cluster 2023-10-01T08:56:55.039979+0000 mon.a (mon.0) 0 : [INF] mkfs 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:56:55.274 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:55 smithi138 conmon[99303]: cluster 2023-10-01T08:56:55.034856+0000 mon.a 2023-10-01T08:56:55.274 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:55 smithi138 conmon[99303]: (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-01T08:56:55.621 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: cluster: 2023-10-01T08:56:55.621 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: id: 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:56:55.622 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-10-01T08:56:55.622 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 2023-10-01T08:56:55.622 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: services: 2023-10-01T08:56:55.622 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.58126s) 2023-10-01T08:56:55.622 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: mgr: no daemons active 2023-10-01T08:56:55.622 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-10-01T08:56:55.622 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 2023-10-01T08:56:55.622 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: data: 2023-10-01T08:56:55.622 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-10-01T08:56:55.622 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-10-01T08:56:55.622 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-10-01T08:56:55.623 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: pgs: 2023-10-01T08:56:55.623 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 2023-10-01T08:56:56.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:56 smithi138 conmon[99303]: cluster 2023-10-01T08:56:55.038951+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-01T08:56:56.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:56 smithi138 conmon[99303]: cluster 2023-10-01T08:56:55.039286+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0]} removed_ranks: {} 2023-10-01T08:56:56.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:56 smithi138 conmon[99303]: cluster 2023-10-01T08:56:55.041927+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-10-01T08:56:56.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:56 smithi138 conmon[99303]: cluster 2023-10-01T08:56:55.044436+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-01T08:56:56.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:56 smithi138 conmon[99303]: cluster 2023-10-01T08:56:55.044982+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-10-01T08:56:56.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:56 smithi138 conmon[99303]: audit 2023-10-01T08:56:55.620167+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.138:0/2366574976' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-01T08:56:56.358 INFO:teuthology.orchestra.run.smithi138.stderr:mon is available 2023-10-01T08:56:56.358 INFO:teuthology.orchestra.run.smithi138.stderr:Assimilating anything we can from ceph.conf... 2023-10-01T08:56:56.918 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 2023-10-01T08:56:56.918 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: [global] 2023-10-01T08:56:56.918 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: fsid = 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:56:56.918 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-01T08:56:56.918 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.138:3300,v1:172.21.15.138:6789] 2023-10-01T08:56:56.918 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-01T08:56:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-01T08:56:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-01T08:56:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-01T08:56:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 2023-10-01T08:56:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: [mgr] 2023-10-01T08:56:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-01T08:56:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 2023-10-01T08:56:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: [osd] 2023-10-01T08:56:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-01T08:56:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-01T08:56:57.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:57 smithi138 conmon[99303]: audit 2023-10-01T08:56:56.914086+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.138:0/1549873885' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-01T08:56:57.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:57 smithi138 conmon[99303]: audit 2023-10-01T08:56:56.916728+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.138:0/1549873885' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-01T08:56:57.532 INFO:teuthology.orchestra.run.smithi138.stderr:Generating new minimal ceph.conf... 2023-10-01T08:56:58.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:58 smithi138 conmon[99303]: audit 2023-10-01T08:56:58.059077+0000 mon.a 2023-10-01T08:56:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:58 smithi138 conmon[99303]: (mon.0) 10 : audit [DBG] from='client.? 172.21.15.138:0/200397927' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:56:58.732 INFO:teuthology.orchestra.run.smithi138.stderr:Restarting the monitor... 2023-10-01T08:56:59.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:58 smithi138 systemd[1]: Stopping Ceph mon.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T08:56:59.023 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:58 smithi138 bash[99808]: Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-mon.a found: no such container 2023-10-01T08:56:59.023 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:59 smithi138 conmon[99303]: debug 2023-10-01T08:56:59.022+0000 7f4a47759700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-01T08:56:59.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:59 smithi138 conmon[99303]: debug 2023-10-01T08:56:59.022+0000 7f4a47759700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-01T08:57:00.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:59 smithi138 bash[99924]: 664d56ee7867af9d405cf85b68d158d7c9ca17cc1c2993859764344ba9fa63a1 2023-10-01T08:57:00.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:59 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-01T08:57:00.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:59 smithi138 systemd[1]: Stopped Ceph mon.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T08:57:00.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:56:59 smithi138 systemd[1]: Starting Ceph mon.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T08:57:00.411 INFO:teuthology.orchestra.run.smithi138.stderr:Setting mon public_network to 172.21.0.0/20 2023-10-01T08:57:00.653 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.397+0000 7f41e6c8e880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-01T08:57:00.654 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.397+0000 7f41e6c8e880 0 ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable), process ceph-mon, pid 7 2023-10-01T08:57:00.654 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.397+0000 7f41e6c8e880 0 pidfile_write: ignore empty --pid-file 2023-10-01T08:57:00.654 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.401+0000 7f41e6c8e880 0 load: jerasure load: lrc load: isa 2023-10-01T08:57:00.654 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: RocksDB version: 6.8.1 2023-10-01T08:57:00.654 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.654 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-01T08:57:00.655 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Compile date Sep 25 2023 2023-10-01T08:57:00.655 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: DB SUMMARY 2023-10-01T08:57:00.655 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.655 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: CURRENT file: CURRENT 2023-10-01T08:57:00.655 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.655 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-01T08:57:00.655 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.656 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-10-01T08:57:00.656 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.656 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-10-01T08:57:00.656 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.656 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86709 ; 2023-10-01T08:57:00.656 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.657 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.error_if_exists: 0 2023-10-01T08:57:00.657 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.create_if_missing: 0 2023-10-01T08:57:00.657 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.paranoid_checks: 1 2023-10-01T08:57:00.657 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.env: 0x555927cf8080 2023-10-01T08:57:00.657 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.fs: Posix File System 2023-10-01T08:57:00.657 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 bash[100248]: 476ba6abfb2c09e4f2dbb2b9301c99ede5fa33e9f07a6663cf138191f39cf5be 2023-10-01T08:57:00.658 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.info_log: 0x555929f79a80 2023-10-01T08:57:00.658 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-01T08:57:00.658 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.statistics: (nil) 2023-10-01T08:57:00.658 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.use_fsync: 0 2023-10-01T08:57:00.658 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_log_file_size: 0 2023-10-01T08:57:00.658 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T08:57:00.658 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T08:57:00.659 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-01T08:57:00.659 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-01T08:57:00.659 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.allow_fallocate: 1 2023-10-01T08:57:00.659 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-01T08:57:00.659 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-01T08:57:00.659 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.use_direct_reads: 0 2023-10-01T08:57:00.659 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T08:57:00.660 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-01T08:57:00.660 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.db_log_dir: 2023-10-01T08:57:00.660 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-01T08:57:00.660 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T08:57:00.660 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_subcompactions: 1 2023-10-01T08:57:00.660 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_background_flushes: -1 2023-10-01T08:57:00.660 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T08:57:00.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T08:57:00.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T08:57:00.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T08:57:00.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T08:57:00.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-01T08:57:00.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-01T08:57:00.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.write_buffer_manager: 0x55592ac742d0 2023-10-01T08:57:00.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T08:57:00.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-01T08:57:00.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T08:57:00.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T08:57:00.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-01T08:57:00.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T08:57:00.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-01T08:57:00.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-01T08:57:00.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-01T08:57:00.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.unordered_write: 0 2023-10-01T08:57:00.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T08:57:00.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T08:57:00.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T08:57:00.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T08:57:00.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.row_cache: None 2023-10-01T08:57:00.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.wal_filter: None 2023-10-01T08:57:00.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T08:57:00.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-01T08:57:00.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.preserve_deletes: 0 2023-10-01T08:57:00.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.two_write_queues: 0 2023-10-01T08:57:00.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-01T08:57:00.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.atomic_flush: 0 2023-10-01T08:57:00.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T08:57:00.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T08:57:00.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T08:57:00.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.log_readahead_size: 0 2023-10-01T08:57:00.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-01T08:57:00.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_background_jobs: 2 2023-10-01T08:57:00.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_background_compactions: -1 2023-10-01T08:57:00.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T08:57:00.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T08:57:00.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T08:57:00.668 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-01T08:57:00.668 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T08:57:00.668 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T08:57:00.668 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T08:57:00.668 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T08:57:00.668 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_open_files: -1 2023-10-01T08:57:00.668 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-01T08:57:00.668 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T08:57:00.669 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T08:57:00.669 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-01T08:57:00.669 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Compression algorithms supported: 2023-10-01T08:57:00.669 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T08:57:00.669 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: kZSTD supported: 0 2023-10-01T08:57:00.669 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: kXpressCompression supported: 0 2023-10-01T08:57:00.669 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-01T08:57:00.669 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: kLZ4Compression supported: 1 2023-10-01T08:57:00.670 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: kBZip2Compression supported: 0 2023-10-01T08:57:00.670 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: kZlibCompression supported: 1 2023-10-01T08:57:00.670 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: kSnappyCompression supported: 1 2023-10-01T08:57:00.670 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T08:57:00.670 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-10-01T08:57:00.670 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.670 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-01T08:57:00.670 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T08:57:00.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.merge_operator: 2023-10-01T08:57:00.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.compaction_filter: None 2023-10-01T08:57:00.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-01T08:57:00.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T08:57:00.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T08:57:00.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555929f23d18) 2023-10-01T08:57:00.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: cache_index_and_filter_blocks: 1 2023-10-01T08:57:00.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T08:57:00.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T08:57:00.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: pin_top_level_index_and_filter: 1 2023-10-01T08:57:00.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: index_type: 0 2023-10-01T08:57:00.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: data_block_index_type: 0 2023-10-01T08:57:00.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: index_shortening: 1 2023-10-01T08:57:00.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: data_block_hash_table_util_ratio: 0.750000 2023-10-01T08:57:00.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: hash_index_allow_collision: 1 2023-10-01T08:57:00.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: checksum: 1 2023-10-01T08:57:00.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: no_block_cache: 0 2023-10-01T08:57:00.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: block_cache: 0x555929f5cf10 2023-10-01T08:57:00.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: block_cache_name: BinnedLRUCache 2023-10-01T08:57:00.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: block_cache_options: 2023-10-01T08:57:00.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: capacity : 536870912 2023-10-01T08:57:00.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: num_shard_bits : 4 2023-10-01T08:57:00.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: strict_capacity_limit : 0 2023-10-01T08:57:00.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: high_pri_pool_ratio: 0.000 2023-10-01T08:57:00.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: block_cache_compressed: (nil) 2023-10-01T08:57:00.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: persistent_cache: (nil) 2023-10-01T08:57:00.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: block_size: 4096 2023-10-01T08:57:00.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: block_size_deviation: 10 2023-10-01T08:57:00.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: block_restart_interval: 16 2023-10-01T08:57:00.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: index_block_restart_interval: 1 2023-10-01T08:57:00.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: metadata_block_size: 4096 2023-10-01T08:57:00.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: partition_filters: 0 2023-10-01T08:57:00.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: use_delta_encoding: 1 2023-10-01T08:57:00.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-01T08:57:00.674 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: whole_key_filtering: 1 2023-10-01T08:57:00.674 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: verify_compression: 0 2023-10-01T08:57:00.674 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: read_amp_bytes_per_bit: 0 2023-10-01T08:57:00.674 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: format_version: 2 2023-10-01T08:57:00.674 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: enable_index_compression: 1 2023-10-01T08:57:00.674 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: block_align: 0 2023-10-01T08:57:00.674 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.674 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-01T08:57:00.674 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-01T08:57:00.674 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.compression: NoCompression 2023-10-01T08:57:00.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-01T08:57:00.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-01T08:57:00.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T08:57:00.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.num_levels: 7 2023-10-01T08:57:00.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T08:57:00.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T08:57:00.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T08:57:00.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T08:57:00.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T08:57:00.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T08:57:00.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T08:57:00.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T08:57:00.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T08:57:00.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T08:57:00.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-01T08:57:00.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-01T08:57:00.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T08:57:00.677 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T08:57:00.677 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-01T08:57:00.677 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T08:57:00.677 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T08:57:00.677 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T08:57:00.677 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-01T08:57:00.677 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T08:57:00.678 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T08:57:00.678 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T08:57:00.678 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T08:57:00.678 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T08:57:00.678 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.402+0000 7f41e6c8e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T08:57:00.678 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T08:57:00.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T08:57:00.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T08:57:00.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T08:57:00.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T08:57:00.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T08:57:00.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T08:57:00.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-01T08:57:00.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T08:57:00.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T08:57:00.680 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-01T08:57:00.680 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-01T08:57:00.680 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T08:57:00.680 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T08:57:00.680 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T08:57:00.680 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T08:57:00.680 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T08:57:00.680 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T08:57:00.680 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T08:57:00.680 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T08:57:00.680 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T08:57:00.680 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T08:57:00.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.table_properties_collectors: 2023-10-01T08:57:00.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.inplace_update_support: 0 2023-10-01T08:57:00.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T08:57:00.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T08:57:00.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T08:57:00.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T08:57:00.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.bloom_locality: 0 2023-10-01T08:57:00.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.max_successive_merges: 0 2023-10-01T08:57:00.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T08:57:00.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-01T08:57:00.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-01T08:57:00.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-01T08:57:00.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.ttl: 2592000 2023-10-01T08:57:00.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T08:57:00.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T08:57:00.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-10-01T08:57:00.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696150620404833, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-10-01T08:57:00.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.403+0000 7f41e6c8e880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-10-01T08:57:00.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.404+0000 7f41e6c8e880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-01T08:57:00.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.404+0000 7f41e6c8e880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696150620405752, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83309, "table_properties": {"data_size": 81582, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75903, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696150620, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-01T08:57:00.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.404+0000 7f41e6c8e880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-10-01T08:57:00.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.404+0000 7f41e6c8e880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696150620406118, "job": 1, "event": "recovery_finished"} 2023-10-01T08:57:00.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.405+0000 7f41e6c8e880 4 rocksdb: DB pointer 0x55592a027800 2023-10-01T08:57:00.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.405+0000 7f41cda96700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-01T08:57:00.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.405+0000 7f41cda96700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-01T08:57:00.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: ** DB Stats ** 2023-10-01T08:57:00.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T08:57:00.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T08:57:00.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T08:57:00.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T08:57:00.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T08:57:00.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-01T08:57:00.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T08:57:00.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: ** Compaction Stats [default] ** 2023-10-01T08:57:00.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:57:00.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:57:00.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: L0 2/0 83.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 124.3 0.00 0.00 1 0.001 0 0 2023-10-01T08:57:00.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Sum 2/0 83.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 124.3 0.00 0.00 1 0.001 0 0 2023-10-01T08:57:00.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 124.3 0.00 0.00 1 0.001 0 0 2023-10-01T08:57:00.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: ** Compaction Stats [default] ** 2023-10-01T08:57:00.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:57:00.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:57:00.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 124.3 0.00 0.00 1 0.001 0 0 2023-10-01T08:57:00.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T08:57:00.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T08:57:00.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T08:57:00.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T08:57:00.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T08:57:00.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: AddFile(Keys): cumulative 0, interval 0 2023-10-01T08:57:00.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Cumulative compaction: 0.00 GB write, 29.10 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:57:00.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Interval compaction: 0.00 GB write, 29.10 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:57:00.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T08:57:00.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: ** File Read Latency Histogram By Level [default] ** 2023-10-01T08:57:00.686 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.686 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: ** Compaction Stats [default] ** 2023-10-01T08:57:00.686 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:57:00.686 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:57:00.686 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: L0 2/0 83.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 124.3 0.00 0.00 1 0.001 0 0 2023-10-01T08:57:00.686 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Sum 2/0 83.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 124.3 0.00 0.00 1 0.001 0 0 2023-10-01T08:57:00.686 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-01T08:57:00.686 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.686 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: ** Compaction Stats [default] ** 2023-10-01T08:57:00.686 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:57:00.687 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:57:00.687 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 124.3 0.00 0.00 1 0.001 0 0 2023-10-01T08:57:00.687 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T08:57:00.687 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T08:57:00.687 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T08:57:00.687 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T08:57:00.687 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T08:57:00.687 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: AddFile(Keys): cumulative 0, interval 0 2023-10-01T08:57:00.687 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Cumulative compaction: 0.00 GB write, 28.82 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:57:00.687 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:57:00.687 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T08:57:00.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: ** File Read Latency Histogram By Level [default] ** 2023-10-01T08:57:00.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.406+0000 7f41e6c8e880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] at bind addrs [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:57:00.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.407+0000 7f41e6c8e880 1 mon.a@-1(???) e1 preinit fsid 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:57:00.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.407+0000 7f41e6c8e880 0 mon.a@-1(???).mds e1 new map 2023-10-01T08:57:00.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.407+0000 7f41e6c8e880 0 mon.a@-1(???).mds e1 print_map 2023-10-01T08:57:00.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: e1 2023-10-01T08:57:00.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-01T08:57:00.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T08:57:00.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: legacy client fscid: -1 2023-10-01T08:57:00.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: No filesystems configured 2023-10-01T08:57:00.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.407+0000 7f41e6c8e880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-01T08:57:00.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.407+0000 7f41e6c8e880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T08:57:00.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.407+0000 7f41e6c8e880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T08:57:00.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.407+0000 7f41e6c8e880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T08:57:00.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: audit 2023-10-01T08:57:00.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:56:58.059077 2023-10-01T08:57:00.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: +0000 mon.a (mon.0) 10 : audit 2023-10-01T08:57:00.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: [DBG] from='client.? 172.21.15.138:0/200397927' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:57:00.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: debug 2023-10-01T08:57:00.407+0000 7f41e6c8e880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-01T08:57:00.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 systemd[1]: Started Ceph mon.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T08:57:00.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: cluster 2023-10-01T08:57:00.424948+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-01T08:57:00.690 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: cluster 2023-10-01T08:57:00.425037+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0]} removed_ranks: {} 2023-10-01T08:57:00.690 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: cluster 2023-10-01T08:57:00 2023-10-01T08:57:00.690 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: .425466+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-10-01T08:57:00.690 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: cluster 2023-10-01T08:57:00.690 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 2023-10-01T08:57:00. 2023-10-01T08:57:00.690 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 425505+0000 mon.a (mon.0) 4 2023-10-01T08:57:00.690 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-01T08:57:00.690 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: cluster 2023-10-01T08: 2023-10-01T08:57:00.690 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: 57:00.425906+0000 mon.a (mon.0) 5 2023-10-01T08:57:00.690 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:00 smithi138 conmon[100289]: : cluster [DBG] mgrmap e1: no daemons active 2023-10-01T08:57:01.548 INFO:teuthology.orchestra.run.smithi138.stderr:Wrote config to /etc/ceph/ceph.conf 2023-10-01T08:57:01.549 INFO:teuthology.orchestra.run.smithi138.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-01T08:57:01.549 INFO:teuthology.orchestra.run.smithi138.stderr:Creating mgr... 2023-10-01T08:57:01.549 INFO:teuthology.orchestra.run.smithi138.stderr:Verifying port 9283 ... 2023-10-01T08:57:01.675 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: Failed to reset failed state of unit ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.y.service: Unit ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.y.service not loaded. 2023-10-01T08:57:01.682 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: Created symlink /etc/systemd/system/ceph-465870a4-6038-11ee-8db5-212e2dc638e7.target.wants/ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-465870a4-6038-11ee-8db5-212e2dc638e7@.service. 2023-10-01T08:57:01.819 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:01 smithi138 systemd[1]: Starting Ceph mgr.y for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T08:57:02.199 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:01 smithi138 conmon[100289]: audit 2023-10-01T08:57:00.921896+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.138:0/905146057' entity='client.admin' 2023-10-01T08:57:02.362 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: disabled 2023-10-01T08:57:02.369 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: inactive 2023-10-01T08:57:02.370 INFO:teuthology.orchestra.run.smithi138.stderr:firewalld.service is not enabled 2023-10-01T08:57:02.370 INFO:teuthology.orchestra.run.smithi138.stderr:Not possible to enable service . firewalld.service is not available 2023-10-01T08:57:02.377 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: disabled 2023-10-01T08:57:02.385 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: inactive 2023-10-01T08:57:02.386 INFO:teuthology.orchestra.run.smithi138.stderr:firewalld.service is not enabled 2023-10-01T08:57:02.386 INFO:teuthology.orchestra.run.smithi138.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-10-01T08:57:02.386 INFO:teuthology.orchestra.run.smithi138.stderr:Waiting for mgr to start... 2023-10-01T08:57:02.386 INFO:teuthology.orchestra.run.smithi138.stderr:Waiting for mgr... 2023-10-01T08:57:02.598 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:02 smithi138 bash[100697]: 5d4bcde0f5d467a176e0f57f0a5fec120abba85bba8bb289ceaeee759499c655 2023-10-01T08:57:02.598 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:02 smithi138 systemd[1]: Started Ceph mgr.y for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T08:57:02.598 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:02 smithi138 conmon[100738]: debug 2023-10-01T08:57:02.472+0000 7fd8a7261680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T08:57:02.598 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:02 smithi138 conmon[100738]: debug 2023-10-01T08:57:02.555+0000 7fd8a7261680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T08:57:02.960 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 2023-10-01T08:57:02.960 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: { 2023-10-01T08:57:02.960 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "fsid": "465870a4-6038-11ee-8db5-212e2dc638e7", 2023-10-01T08:57:02.960 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "health": { 2023-10-01T08:57:02.960 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-01T08:57:02.960 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "checks": {}, 2023-10-01T08:57:02.961 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "mutes": [] 2023-10-01T08:57:02.961 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:02.961 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-01T08:57:02.962 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "quorum": [ 2023-10-01T08:57:02.962 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 0 2023-10-01T08:57:02.963 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: ], 2023-10-01T08:57:02.963 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-01T08:57:02.963 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "a" 2023-10-01T08:57:02.963 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: ], 2023-10-01T08:57:02.963 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "quorum_age": 2, 2023-10-01T08:57:02.963 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "monmap": { 2023-10-01T08:57:02.963 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T08:57:02.963 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-01T08:57:02.963 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-01T08:57:02.963 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:02.963 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "osdmap": { 2023-10-01T08:57:02.963 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T08:57:02.964 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-01T08:57:02.964 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-01T08:57:02.964 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-01T08:57:02.967 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-01T08:57:02.967 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-01T08:57:02.967 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-01T08:57:02.967 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:02.967 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "pgmap": { 2023-10-01T08:57:02.967 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-01T08:57:02.968 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-01T08:57:02.968 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-01T08:57:02.968 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-01T08:57:02.968 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-01T08:57:02.968 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-01T08:57:02.968 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-01T08:57:02.968 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-01T08:57:02.968 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:02.968 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "fsmap": { 2023-10-01T08:57:02.968 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T08:57:02.968 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-01T08:57:02.968 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-01T08:57:02.969 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:02.969 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-01T08:57:02.969 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "available": false, 2023-10-01T08:57:02.969 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-01T08:57:02.969 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "modules": [ 2023-10-01T08:57:02.969 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "iostat", 2023-10-01T08:57:02.969 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "nfs", 2023-10-01T08:57:02.969 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "restful" 2023-10-01T08:57:02.969 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: ], 2023-10-01T08:57:02.969 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "services": {} 2023-10-01T08:57:02.969 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:02.970 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "servicemap": { 2023-10-01T08:57:02.970 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T08:57:02.970 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "modified": "2023-10-01T08:56:55.040381+0000", 2023-10-01T08:57:02.970 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "services": {} 2023-10-01T08:57:02.970 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:02.970 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-01T08:57:02.970 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: } 2023-10-01T08:57:03.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:03 smithi138 conmon[100289]: audit 2023-10-01T08:57:02.958784+0000 mon.a (mon.0 2023-10-01T08:57:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:03 smithi138 conmon[100289]: ) 7 : audit [DBG] from='client.? 172.21.15.138:0/2482937167' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-01T08:57:03.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:02 smithi138 conmon[100738]: debug 2023-10-01T08:57:02.945+0000 7fd8a7261680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T08:57:03.631 INFO:teuthology.orchestra.run.smithi138.stderr:mgr not available, waiting (1/15)... 2023-10-01T08:57:04.007 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:03 smithi138 conmon[100738]: debug 2023-10-01T08:57:03.714+0000 7fd8a7261680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T08:57:04.007 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:03 smithi138 conmon[100738]: debug 2023-10-01T08:57:03.797+0000 7fd8a7261680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T08:57:05.235 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:04 smithi138 conmon[100738]: debug 2023-10-01T08:57:04.005+0000 7fd8a7261680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T08:57:05.235 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:04 smithi138 conmon[100738]: debug 2023-10-01T08:57:04.944+0000 7fd8a7261680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T08:57:05.235 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:05 smithi138 conmon[100738]: debug 2023-10-01T08:57:05.069+0000 7fd8a7261680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T08:57:05.235 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:05 smithi138 conmon[100738]: debug 2023-10-01T08:57:05.146+0000 7fd8a7261680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T08:57:05.235 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:05 smithi138 conmon[100738]: debug 2023-10-01T08:57:05.233+0000 7fd8a7261680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T08:57:05.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:05 smithi138 conmon[100738]: debug 2023-10-01T08:57:05.428+0000 7fd8a7261680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T08:57:06.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:05 smithi138 conmon[100738]: debug 2023-10-01T08:57:05.714+0000 7fd8a7261680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T08:57:06.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:05 smithi138 conmon[100738]: debug 2023-10-01T08:57:05.816+0000 7fd8a7261680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T08:57:06.229 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 2023-10-01T08:57:06.229 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: { 2023-10-01T08:57:06.229 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "fsid": "465870a4-6038-11ee-8db5-212e2dc638e7", 2023-10-01T08:57:06.229 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "health": { 2023-10-01T08:57:06.229 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-01T08:57:06.229 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "checks": {}, 2023-10-01T08:57:06.229 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "mutes": [] 2023-10-01T08:57:06.229 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:06.229 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-01T08:57:06.230 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "quorum": [ 2023-10-01T08:57:06.230 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 0 2023-10-01T08:57:06.230 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: ], 2023-10-01T08:57:06.230 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-01T08:57:06.230 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "a" 2023-10-01T08:57:06.230 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: ], 2023-10-01T08:57:06.230 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "quorum_age": 5, 2023-10-01T08:57:06.230 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "monmap": { 2023-10-01T08:57:06.230 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T08:57:06.231 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-01T08:57:06.231 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-01T08:57:06.231 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:06.231 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "osdmap": { 2023-10-01T08:57:06.231 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T08:57:06.231 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-01T08:57:06.231 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-01T08:57:06.231 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-01T08:57:06.234 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-01T08:57:06.234 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-01T08:57:06.235 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-01T08:57:06.235 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:06.235 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "pgmap": { 2023-10-01T08:57:06.235 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-01T08:57:06.235 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-01T08:57:06.235 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-01T08:57:06.235 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-01T08:57:06.235 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-01T08:57:06.235 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-01T08:57:06.235 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-01T08:57:06.235 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-01T08:57:06.236 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:06.236 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "fsmap": { 2023-10-01T08:57:06.236 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T08:57:06.236 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-01T08:57:06.236 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-01T08:57:06.236 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:06.236 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-01T08:57:06.236 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "available": false, 2023-10-01T08:57:06.236 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-01T08:57:06.236 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "modules": [ 2023-10-01T08:57:06.236 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "iostat", 2023-10-01T08:57:06.236 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "nfs", 2023-10-01T08:57:06.237 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "restful" 2023-10-01T08:57:06.237 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: ], 2023-10-01T08:57:06.237 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "services": {} 2023-10-01T08:57:06.237 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:06.237 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "servicemap": { 2023-10-01T08:57:06.237 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T08:57:06.237 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "modified": "2023-10-01T08:56:55.040381+0000", 2023-10-01T08:57:06.237 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "services": {} 2023-10-01T08:57:06.237 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:06.237 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-01T08:57:06.237 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: } 2023-10-01T08:57:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:06 smithi138 conmon[100289]: audit 2023-10-01T08:57:06.226165+0000 mon.a (mon.0) 2023-10-01T08:57:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:06 smithi138 conmon[100289]: 8 : audit [DBG] from='client.? 172.21.15.138:0/3756089570' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-01T08:57:06.814 INFO:teuthology.orchestra.run.smithi138.stderr:mgr not available, waiting (2/15)... 2023-10-01T08:57:06.925 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:06 smithi138 conmon[100738]: debug 2023-10-01T08:57:06.576+0000 7fd8a7261680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T08:57:06.926 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:06 smithi138 conmon[100738]: debug 2023-10-01T08:57:06.657+0000 7fd8a7261680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T08:57:06.926 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:06 smithi138 conmon[100738]: debug 2023-10-01T08:57:06.748+0000 7fd8a7261680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T08:57:06.926 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:06 smithi138 conmon[100738]: debug 2023-10-01T08:57:06.924+0000 7fd8a7261680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T08:57:07.442 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100738]: debug 2023-10-01T08:57:07.182+0000 7fd8a7261680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T08:57:07.442 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100738]: debug 2023-10-01T08:57:07.310+0000 7fd8a7261680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T08:57:07.694 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100738]: debug 2023-10-01T08:57:07.441+0000 7fd8a7261680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T08:57:07.694 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100738]: debug 2023-10-01T08:57:07.611+0000 7fd8a7261680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T08:57:08.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100738]: debug 2023-10-01T08:57:07.693+0000 7fd8a7261680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T08:57:08.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: cluster 2023-10-01T08:57:07.707063+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2023-10-01T08:57:08.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: cluster 2023-10-01T08:57:07.709592+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00260677s) 2023-10-01T08:57:08.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: audit 2023-10-01T08:57:07. 2023-10-01T08:57:08.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: 710528+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.138:0/3654432020' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T08:57:08.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: audit 2023-10-01T08:57:07 2023-10-01T08:57:08.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: .710729+0000 mon.a (mon.0) 12 : audit 2023-10-01T08:57:08.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: [DBG] from='mgr.14100 172.21.15.138:0/3654432020' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T08:57:08.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: audit 2023-10-01T08:57:08.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: 2023-10-01T08:57:07.710858+0000 mon.a (mon. 2023-10-01T08:57:08.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: 0) 13 : audit [DBG] from='mgr.14100 172.21.15.138:0/3654432020' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T08:57:08.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: audit 2023-10-01T08: 2023-10-01T08:57:08.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: 57:07.710982+0000 mon.a (mon.0) 2023-10-01T08:57:08.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: 14 : audit [DBG] from='mgr.14100 172.21.15.138:0/3654432020' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T08:57:08.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: audit 2023-10-01T08: 2023-10-01T08:57:08.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: 57:07.711143+0000 mon.a (mon.0) 15 2023-10-01T08:57:08.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: : audit [DBG] from='mgr.14100 172.21.15.138:0/3654432020' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T08:57:08.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: cluster 2023-10-01T08:57 2023-10-01T08:57:08.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: :07.712899+0000 mon.a (mon.0) 16 2023-10-01T08:57:08.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: : cluster [INF] Manager daemon y is now available 2023-10-01T08:57:08.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: audit 2023-10-01T08:57: 2023-10-01T08:57:08.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: 07.722631+0000 mon.a (mon.0) 17 : audit 2023-10-01T08:57:08.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: [INF] from='mgr.14100 172.21.15.138:0/3654432020' entity='mgr.y' 2023-10-01T08:57:08.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: audit 2023-10-01T08:57:08.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: 2023-10-01T08:57:07.726259+0000 mon.a ( 2023-10-01T08:57:08.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.138:0/3654432020' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T08:57:08.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T08:57:08.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: 08:57:07.728813+0000 mon.a (mon.0 2023-10-01T08:57:08.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: ) 19 : audit [INF] from='mgr.14100 172.21.15.138:0/3654432020' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T08:57:08.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: audit 2023-10-01T08 2023-10-01T08:57:08.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: :57:07.734514+0000 mon.a (mon.0 2023-10-01T08:57:08.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: ) 20 : audit [INF] from='mgr.14100 172.21.15.138:0/3654432020' entity='mgr.y' 2023-10-01T08:57:08.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: audit 2023-10-01T08 2023-10-01T08:57:08.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: :57:07.739169+0000 mon.a (mon.0 2023-10-01T08:57:08.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:07 smithi138 conmon[100289]: ) 21 : audit [INF] from='mgr.14100 172.21.15.138:0/3654432020' entity='mgr.y' 2023-10-01T08:57:09.537 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 2023-10-01T08:57:09.537 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: { 2023-10-01T08:57:09.537 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "fsid": "465870a4-6038-11ee-8db5-212e2dc638e7", 2023-10-01T08:57:09.537 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "health": { 2023-10-01T08:57:09.537 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-01T08:57:09.537 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "checks": {}, 2023-10-01T08:57:09.537 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "mutes": [] 2023-10-01T08:57:09.537 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:09.539 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-01T08:57:09.539 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "quorum": [ 2023-10-01T08:57:09.539 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 0 2023-10-01T08:57:09.539 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: ], 2023-10-01T08:57:09.539 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-01T08:57:09.539 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "a" 2023-10-01T08:57:09.539 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: ], 2023-10-01T08:57:09.539 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "quorum_age": 9, 2023-10-01T08:57:09.540 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "monmap": { 2023-10-01T08:57:09.540 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T08:57:09.540 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-01T08:57:09.540 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-01T08:57:09.540 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:09.540 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "osdmap": { 2023-10-01T08:57:09.540 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T08:57:09.540 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-01T08:57:09.540 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-01T08:57:09.540 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-01T08:57:09.543 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-01T08:57:09.543 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-01T08:57:09.544 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-01T08:57:09.544 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:09.544 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "pgmap": { 2023-10-01T08:57:09.544 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-01T08:57:09.544 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-01T08:57:09.544 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-01T08:57:09.544 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-01T08:57:09.544 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-01T08:57:09.544 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-01T08:57:09.544 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-01T08:57:09.544 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-01T08:57:09.544 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:09.545 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "fsmap": { 2023-10-01T08:57:09.545 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T08:57:09.545 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-01T08:57:09.545 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-01T08:57:09.545 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:09.545 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-01T08:57:09.545 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "available": true, 2023-10-01T08:57:09.545 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-01T08:57:09.545 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "modules": [ 2023-10-01T08:57:09.545 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "iostat", 2023-10-01T08:57:09.545 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "nfs", 2023-10-01T08:57:09.545 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "restful" 2023-10-01T08:57:09.546 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: ], 2023-10-01T08:57:09.546 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "services": {} 2023-10-01T08:57:09.546 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:09.546 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "servicemap": { 2023-10-01T08:57:09.546 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T08:57:09.546 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "modified": "2023-10-01T08:56:55.040381+0000", 2023-10-01T08:57:09.546 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "services": {} 2023-10-01T08:57:09.546 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: }, 2023-10-01T08:57:09.546 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-01T08:57:09.546 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: } 2023-10-01T08:57:09.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:09 smithi138 conmon[100289]: cluster 2023-10-01T08:57:08.712970+0000 mon.a (mon.0) 22 : cluster 2023-10-01T08:57:09.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:09 smithi138 conmon[100289]: [DBG] mgrmap e3: y(active, since 1.00598s) 2023-10-01T08:57:09.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:09 smithi138 conmon[100289]: audit 2023-10-01T08:57:09.534577+0000 mon.a (mon.0) 23 : audit [DBG] from='client.? 172.21.15.138:0/1787830674' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-01T08:57:10.174 INFO:teuthology.orchestra.run.smithi138.stderr:mgr is available 2023-10-01T08:57:10.779 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 2023-10-01T08:57:10.779 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: [global] 2023-10-01T08:57:10.779 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: fsid = 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:57:10.779 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-01T08:57:10.779 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-01T08:57:10.779 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-01T08:57:10.779 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-01T08:57:10.779 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-01T08:57:10.780 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 2023-10-01T08:57:10.780 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: [mgr] 2023-10-01T08:57:10.780 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-01T08:57:10.780 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 2023-10-01T08:57:10.780 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: [osd] 2023-10-01T08:57:10.780 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-01T08:57:10.780 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-01T08:57:11.413 INFO:teuthology.orchestra.run.smithi138.stderr:Enabling cephadm module... 2023-10-01T08:57:12.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:11 smithi138 conmon[100289]: cluster 2023-10-01T08:57:10.712703+0000 mon.a (mon.0) 2023-10-01T08:57:12.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:11 smithi138 conmon[100289]: 24 : cluster [DBG] mgrmap e4: y(active, since 3s) 2023-10-01T08:57:12.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:11 smithi138 conmon[100289]: audit 2023-10-01T08:57:10.776805+0000 mon.a (mon.0) 25 : audit [INF] from='client.? 172.21.15.138:0/1494150608' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-01T08:57:13.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:12 smithi138 conmon[100289]: audit 2023-10-01T08:57:12.108094+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.138:0/2968742585' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-01T08:57:13.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:12 smithi138 conmon[100738]: ignoring --setuser ceph since I am not root 2023-10-01T08:57:13.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:12 smithi138 conmon[100738]: ignoring --setgroup ceph since I am not root 2023-10-01T08:57:13.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:12 smithi138 conmon[100738]: debug 2023-10-01T08:57:12.879+0000 7f95c2bcd680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T08:57:13.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:12 smithi138 conmon[100738]: debug 2023-10-01T08:57:12.963+0000 7f95c2bcd680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T08:57:13.604 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:13 smithi138 conmon[100738]: debug 2023-10-01T08:57:13.332+0000 7f95c2bcd680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T08:57:14.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:13 smithi138 conmon[100289]: audit 2023-10-01T08:57:12.715236+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.138:0/2968742585' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-01T08:57:14.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:13 smithi138 conmon[100289]: cluster 2023-10-01T08:57:12.715305+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 5s) 2023-10-01T08:57:14.091 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: { 2023-10-01T08:57:14.091 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "epoch": 5, 2023-10-01T08:57:14.091 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "available": true, 2023-10-01T08:57:14.091 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-01T08:57:14.091 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-01T08:57:14.091 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: } 2023-10-01T08:57:14.463 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:14 smithi138 conmon[100738]: debug 2023-10-01T08:57:14.113+0000 7f95c2bcd680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T08:57:14.463 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:14 smithi138 conmon[100738]: debug 2023-10-01T08:57:14.203+0000 7f95c2bcd680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T08:57:14.696 INFO:teuthology.orchestra.run.smithi138.stderr:Waiting for the mgr to restart... 2023-10-01T08:57:14.696 INFO:teuthology.orchestra.run.smithi138.stderr:Waiting for mgr epoch 5... 2023-10-01T08:57:14.717 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:14 smithi138 conmon[100738]: debug 2023-10-01T08:57:14.424+0000 7f95c2bcd680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T08:57:14.987 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:14 smithi138 conmon[100289]: audit 2023-10-01T08:57:14.088094+0000 2023-10-01T08:57:14.987 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:14 smithi138 conmon[100289]: mon.a (mon.0) 29 : audit [DBG] from='client.? 172.21.15.138:0/2044895854' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-01T08:57:15.691 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:15 smithi138 conmon[100738]: debug 2023-10-01T08:57:15.398+0000 7f95c2bcd680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T08:57:15.691 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:15 smithi138 conmon[100738]: debug 2023-10-01T08:57:15.525+0000 7f95c2bcd680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T08:57:15.691 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:15 smithi138 conmon[100738]: debug 2023-10-01T08:57:15.602+0000 7f95c2bcd680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T08:57:16.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:15 smithi138 conmon[100738]: debug 2023-10-01T08:57:15.690+0000 7f95c2bcd680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T08:57:16.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:15 smithi138 conmon[100738]: debug 2023-10-01T08:57:15.884+0000 7f95c2bcd680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T08:57:16.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:16 smithi138 conmon[100738]: debug 2023-10-01T08:57:16.159+0000 7f95c2bcd680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T08:57:16.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:16 smithi138 conmon[100738]: debug 2023-10-01T08:57:16.258+0000 7f95c2bcd680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T08:57:17.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:16 smithi138 conmon[100738]: debug 2023-10-01T08:57:16.976+0000 7f95c2bcd680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T08:57:17.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:17 smithi138 conmon[100738]: debug 2023-10-01T08:57:17.058+0000 7f95c2bcd680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T08:57:17.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:17 smithi138 conmon[100738]: debug 2023-10-01T08:57:17.150+0000 7f95c2bcd680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T08:57:17.597 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:17 smithi138 conmon[100738]: debug 2023-10-01T08:57:17.323+0000 7f95c2bcd680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T08:57:17.861 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:17 smithi138 conmon[100738]: debug 2023-10-01T08:57:17.596+0000 7f95c2bcd680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T08:57:17.861 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:17 smithi138 conmon[100738]: debug 2023-10-01T08:57:17.727+0000 7f95c2bcd680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T08:57:18.113 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:17 smithi138 conmon[100738]: debug 2023-10-01T08:57:17.858+0000 7f95c2bcd680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T08:57:18.113 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100738]: debug 2023-10-01T08:57:18.027+0000 7f95c2bcd680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T08:57:18.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100738]: debug 2023-10-01T08:57:18.112+0000 7f95c2bcd680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T08:57:18.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: cluster 2023-10-01T08:57:18.125968+0000 mon.a (mon.0) 30 2023-10-01T08:57:18.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: : cluster [INF] Active manager daemon y restarted 2023-10-01T08:57:18.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: cluster 2023-10-01T08:57:18.126192+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2023-10-01T08:57:18.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: cluster 2023- 2023-10-01T08:57:18.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: 10-01T08:57:18.128955+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-10-01T08:57:18.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: cluster 2023-10-01T 2023-10-01T08:57:18.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: 08:57:18.129055+0000 mon.a (mon. 2023-10-01T08:57:18.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: 0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00293912s) 2023-10-01T08:57:18.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T08:57:18.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: T08:57:18.129991+0000 mon.a (mon 2023-10-01T08:57:18.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: .0) 34 : audit [DBG] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T08:57:18.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T08:57:18.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: T08:57:18.130601+0000 mon.a (mon 2023-10-01T08:57:18.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: .0) 35 : audit [DBG] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T08:57:18.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: audit 2023-10-01T08:57:18 2023-10-01T08:57:18.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: .131269+0000 mon.a (mon.0) 36 : audit 2023-10-01T08:57:18.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: [DBG] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T08:57:18.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: audit 2023-10-01T08:57:18. 2023-10-01T08:57:18.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: 131487+0000 mon.a (mon.0) 37 : audit 2023-10-01T08:57:18.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: [DBG] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T08:57:18.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: audit 2023-10-01T08:57:18.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: 2023-10-01T08:57:18.131673+0000 mon.a ( 2023-10-01T08:57:18.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T08:57:18.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: cluster 2023-10 2023-10-01T08:57:18.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: -01T08:57:18.145565+0000 mon.a ( 2023-10-01T08:57:18.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: mon.0) 39 : cluster [INF] Manager daemon y is now available 2023-10-01T08:57:18.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: audit 2023-10 2023-10-01T08:57:18.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: -01T08:57:18.157895+0000 mon.a 2023-10-01T08:57:18.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:18 smithi138 conmon[100289]: (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' 2023-10-01T08:57:19.137 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: { 2023-10-01T08:57:19.138 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-10-01T08:57:19.138 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "initialized": true 2023-10-01T08:57:19.138 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: } 2023-10-01T08:57:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: cephadm 2023-10-01T08:57:18.155198+0000 mgr.y (mgr. 2023-10-01T08:57:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: 14116) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-10-01T08:57:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: audit 2023-10-01T08:57:18.241704+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' 2023-10-01T08:57:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: audit 2023-10-01T08:57:18.242671+0000 mon.a ( 2023-10-01T08:57:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: mon.0) 42 : audit [DBG] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:57:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T08:57:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: T08:57:18.244619+0000 mon.a (mon. 2023-10-01T08:57:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: 0) 43 : audit [DBG] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:57:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T08:57:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: 08:57:18.246295+0000 mon.a (mon.0 2023-10-01T08:57:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: ) 44 : audit [DBG] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:57:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T08:57:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: 08:57:18.258404+0000 mon.a (mon. 2023-10-01T08:57:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: 0) 45 : audit [INF] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T08:57:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T08:57:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: T08:57:18.261200+0000 mon.a (mon. 2023-10-01T08:57:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: 0) 46 : audit [INF] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T08:57:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: cluster 2023-10-01T 2023-10-01T08:57:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: 08:57:19.131177+0000 mon.a (mon.0 2023-10-01T08:57:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:19 smithi138 conmon[100289]: ) 47 : cluster [DBG] mgrmap e7: y(active, since 1.00505s) 2023-10-01T08:57:19.737 INFO:teuthology.orchestra.run.smithi138.stderr:mgr epoch 5 is available 2023-10-01T08:57:19.738 INFO:teuthology.orchestra.run.smithi138.stderr:Setting orchestrator backend to cephadm... 2023-10-01T08:57:20.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:20 smithi138 conmon[100289]: audit 2023-10-01T08:57:19.130528+0000 mgr.y (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-01T08:57:20.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:20 smithi138 conmon[100289]: audit 2023-10-01T08:57:19.137044+0000 mgr.y (mgr.14116) 3 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-01T08:57:21.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:21 smithi138 conmon[100289]: cluster 2023-10-01T08:57:20.245374+0000 2023-10-01T08:57:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:21 smithi138 conmon[100289]: mon.a (mon.0) 48 : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-10-01T08:57:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:21 smithi138 conmon[100289]: audit 2023-10-01T08:57:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:21 smithi138 conmon[100289]: 2023-10-01T08:57:20.412800+0000 mgr.y ( 2023-10-01T08:57:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:21 smithi138 conmon[100289]: mgr.14116) 4 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:57:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:21 smithi138 conmon[100289]: audit 2023 2023-10-01T08:57:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:21 smithi138 conmon[100289]: -10-01T08:57:20 2023-10-01T08:57:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:21 smithi138 conmon[100289]: .417384+0000 mon.a (mon.0) 49 : audit 2023-10-01T08:57:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:21 smithi138 conmon[100289]: [INF] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' 2023-10-01T08:57:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:21 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T08:57:21.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:21 smithi138 conmon[100289]: 01T08:57:20.468478+0000 mon.a 2023-10-01T08:57:21.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:21 smithi138 conmon[100289]: (mon.0) 50 : audit [DBG] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:57:21.640 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: value unchanged 2023-10-01T08:57:22.261 INFO:teuthology.orchestra.run.smithi138.stderr:Generating ssh key... 2023-10-01T08:57:22.516 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:22 smithi138 conmon[100289]: audit 2023-10-01T08:57:21.639574+0000 mgr.y (mgr.14116) 5 : audit [DBG] 2023-10-01T08:57:22.516 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:22 smithi138 conmon[100289]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:57:23.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: Generating public/private rsa key pair. 2023-10-01T08:57:23.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: Your identification has been saved in /tmp/tmpsn5pxf8s/key. 2023-10-01T08:57:23.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: Your public key has been saved in /tmp/tmpsn5pxf8s/key.pub. 2023-10-01T08:57:23.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: The key fingerprint is: 2023-10-01T08:57:23.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: SHA256:lEdbD/Ens5npNSHZWoUSAiOmuBmGQToXwV8PMMTupNw ceph-465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:57:23.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: The key's randomart image is: 2023-10-01T08:57:23.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: +---[RSA 3072]----+ 2023-10-01T08:57:23.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: |ooo++.o o.o =o ..| 2023-10-01T08:57:23.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: |.o.o.+o. + +.++ .| 2023-10-01T08:57:23.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: |+ =o.. oo o +=+.| 2023-10-01T08:57:23.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: | + ++ ... +O.| 2023-10-01T08:57:23.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: | .o= S .=..| 2023-10-01T08:57:23.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: | o E . ..| 2023-10-01T08:57:23.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: | . | 2023-10-01T08:57:23.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: | | 2023-10-01T08:57:23.322 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: | | 2023-10-01T08:57:23.322 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:23 smithi138 conmon[100738]: +----[SHA256]-----+ 2023-10-01T08:57:24.272 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3gXIhZS2nXIHRbmU2S3ulEotVJmYnJIIKXx6PAk84FC2r6Ck51qmj7LsqLTrNSZAyXsEvAXPfHhfqIi2UmAKVxpBhCxT6GbTfAL8T5SeuHJ864f4iqPv4213b1b6UTYx+2dZCt5MO5XgwL87xmm9MnYDJXGoLNfygmgSxUMcw5FNJqiLqKs4Wsz0mQMVDMqOffSXRM2myFF2fGiOuKEBvAtO1lxZZlGD6K6Ifqb13tYONs7MA4j63zXjItu1oWLaEum/scGb4N7M0Ce51TH90ECqXLdwtdcn7PRzSO+P8iBcRaQXLS8Wkst0zEk30mBsmNBPKIQ6zS92hARwkPrQ5Ci/6r8/uEa/ZkSv0Ro/2C53CVJrMiy95+sh6dRVA0QNxQwr+n9SzQ2ka3mvqz5vttJOCpY4gu+U2Y9bRdCIolor+U8vAOQuuywunzx9gGKkmSzd2NE2X0k2TFukIYESTX0SmpVGZdIDRY6YkSS1khyCRcsPdkFsAOVR9brtukfc= ceph-465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:57:24.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:24 smithi138 conmon[100289]: audit 2023-10-01T08:57:22.915558+0000 mgr.y (mgr.14116) 6 2023-10-01T08:57:24.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:24 smithi138 conmon[100289]: : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:57:24.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:24 smithi138 conmon[100289]: cephadm 2023-10-01T08:57:22.915795+0000 mgr.y (mgr.14116) 7 : cephadm [INF] Generating ssh key... 2023-10-01T08:57:24.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:24 smithi138 conmon[100289]: audit 2023-10-01T08:57:23.024007+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' 2023-10-01T08:57:24.302 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:24 smithi138 conmon[100289]: audit 2023-10-01T08:57:23.025810+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' 2023-10-01T08:57:24.853 INFO:teuthology.orchestra.run.smithi138.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-01T08:57:24.853 INFO:teuthology.orchestra.run.smithi138.stderr:Adding key to root@localhost authorized_keys... 2023-10-01T08:57:24.854 INFO:teuthology.orchestra.run.smithi138.stderr:Adding host smithi138... 2023-10-01T08:57:25.134 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:25 smithi138 conmon[100289]: audit 2023-10-01T08:57:24.271459+0000 mgr.y (mgr.14116) 8 : audit 2023-10-01T08:57:25.135 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:25 smithi138 conmon[100289]: [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:57:25.819 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:25 smithi138 conmon[100738]: [80B blob data] 2023-10-01T08:57:26.171 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: Added host 'smithi138' with addr '172.21.15.138' 2023-10-01T08:57:26.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:26 smithi138 conmon[100289]: audit 2023-10-01T08:57:25.511403+0000 mgr.y (mgr.14116) 9 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi138", "addr": "172.21.15.138", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:57:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:26 smithi138 conmon[100289]: audit 2023-10-01T08:57:26.170249+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' 2023-10-01T08:57:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:26 smithi138 conmon[100289]: audit 2023-10-01T08:57:26.194341+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:57:26.840 INFO:teuthology.orchestra.run.smithi138.stderr:Deploying unmanaged mon service... 2023-10-01T08:57:27.479 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: Scheduled mon update... 2023-10-01T08:57:27.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:27 smithi138 conmon[100289]: cephadm 2023-10-01T08:57:26.170530+0000 mgr.y (mgr.14116) 10 : cephadm [INF] Added host smithi138 2023-10-01T08:57:28.102 INFO:teuthology.orchestra.run.smithi138.stderr:Deploying unmanaged mgr service... 2023-10-01T08:57:28.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:28 smithi138 conmon[100289]: audit 2023-10-01T08:57:27.474922+0000 mgr.y 2023-10-01T08:57:28.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:28 smithi138 conmon[100289]: (mgr.14116) 11 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:57:28.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:28 smithi138 conmon[100289]: cephadm 2023-10-01T08:57:27.476091+0000 mgr.y (mgr.14116) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-10-01T08:57:28.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:28 smithi138 conmon[100289]: audit 2023-10-01T08:57:27.478916+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' 2023-10-01T08:57:28.855 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-10-01T08:57:30.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:29 smithi138 conmon[100289]: audit 2023-10-01T08:57:28.850840+0000 mgr.y (mgr.14116) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:57:30.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:29 smithi138 conmon[100289]: cephadm 2023-10-01T08:57:28.851754+0000 mgr.y (mgr.14116) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-10-01T08:57:30.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:29 smithi138 conmon[100289]: audit 2023-10-01T08:57:28.854081+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' 2023-10-01T08:57:31.459 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:31 smithi138 conmon[100289]: audit 2023-10-01T08:57:30.141669+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' 2023-10-01T08:57:31.459 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:31 smithi138 conmon[100289]: audit 2023-10-01T08:57:30.515009+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' 2023-10-01T08:57:31.460 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:31 smithi138 conmon[100289]: audit 2023-10-01T08:57:30.654099+0000 mon.a (mon.0) 59 : audit [INF] from='client.? 172.21.15.138:0/3828305706' entity='client.admin' 2023-10-01T08:57:32.528 INFO:teuthology.orchestra.run.smithi138.stderr:Enabling the dashboard module... 2023-10-01T08:57:33.186 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:32 smithi138 conmon[100289]: audit 2023-10-01T08:57:31.897941+0000 mon.a (mon.0) 60 : audit [INF] from='client.? 172.21.15.138:0/38916765' entity='client.admin' 2023-10-01T08:57:33.187 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:32 smithi138 conmon[100289]: audit 2023-10-01T08:57:32.506171+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.14116 172.21.15.138:0/1203996240' entity='mgr.y' 2023-10-01T08:57:34.153 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:33 smithi138 conmon[100289]: audit 2023-10-01T08:57:33.220890+0000 mon.a (mon.0) 2023-10-01T08:57:34.153 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:33 smithi138 conmon[100289]: 62 : audit [INF] from='client.? 172.21.15.138:0/4046567092' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-01T08:57:34.153 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:33 smithi138 conmon[100738]: ignoring --setuser ceph since I am not root 2023-10-01T08:57:34.153 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:33 smithi138 conmon[100738]: ignoring --setgroup ceph since I am not root 2023-10-01T08:57:34.154 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:34 smithi138 conmon[100738]: debug 2023-10-01T08:57:34.067+0000 7f3a6ec82680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T08:57:34.154 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:34 smithi138 conmon[100738]: debug 2023-10-01T08:57:34.152+0000 7f3a6ec82680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T08:57:34.725 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:34 smithi138 conmon[100738]: debug 2023-10-01T08:57:34.522+0000 7f3a6ec82680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T08:57:35.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:34 smithi138 conmon[100289]: audit 2023-10-01T08:57:33.902650+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.138:0/4046567092' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-01T08:57:35.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:34 smithi138 conmon[100289]: cluster 2023-10-01T08:57:33.902762+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e9: y(active, since 15s) 2023-10-01T08:57:35.205 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: { 2023-10-01T08:57:35.206 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "epoch": 9, 2023-10-01T08:57:35.206 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "available": true, 2023-10-01T08:57:35.206 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-01T08:57:35.206 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-01T08:57:35.206 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: } 2023-10-01T08:57:35.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:35 smithi138 conmon[100738]: debug 2023-10-01T08:57:35.298+0000 7f3a6ec82680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T08:57:35.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:35 smithi138 conmon[100738]: debug 2023-10-01T08:57:35.385+0000 7f3a6ec82680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T08:57:35.842 INFO:teuthology.orchestra.run.smithi138.stderr:Waiting for the mgr to restart... 2023-10-01T08:57:35.843 INFO:teuthology.orchestra.run.smithi138.stderr:Waiting for mgr epoch 9... 2023-10-01T08:57:35.901 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:35 smithi138 conmon[100738]: debug 2023-10-01T08:57:35.601+0000 7f3a6ec82680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T08:57:36.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:35 smithi138 conmon[100289]: audit 2023-10-01T08:57:35.204679+0000 mon.a (mon.0) 65 : audit [DBG] from='client.? 172.21.15.138:0/958761135' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-01T08:57:36.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:35 smithi138 conmon[100289]: 2023-10-01T08:57:36.864 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:36 smithi138 conmon[100738]: debug 2023-10-01T08:57:36.569+0000 7f3a6ec82680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T08:57:36.864 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:36 smithi138 conmon[100738]: debug 2023-10-01T08:57:36.695+0000 7f3a6ec82680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T08:57:36.864 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:36 smithi138 conmon[100738]: debug 2023-10-01T08:57:36.773+0000 7f3a6ec82680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T08:57:36.864 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:36 smithi138 conmon[100738]: debug 2023-10-01T08:57:36.863+0000 7f3a6ec82680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T08:57:37.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:37 smithi138 conmon[100738]: debug 2023-10-01T08:57:37.058+0000 7f3a6ec82680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T08:57:37.819 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:37 smithi138 conmon[100738]: debug 2023-10-01T08:57:37.335+0000 7f3a6ec82680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T08:57:37.819 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:37 smithi138 conmon[100738]: debug 2023-10-01T08:57:37.432+0000 7f3a6ec82680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T08:57:38.511 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:38 smithi138 conmon[100738]: debug 2023-10-01T08:57:38.165+0000 7f3a6ec82680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T08:57:38.512 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:38 smithi138 conmon[100738]: debug 2023-10-01T08:57:38.247+0000 7f3a6ec82680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T08:57:38.512 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:38 smithi138 conmon[100738]: debug 2023-10-01T08:57:38.339+0000 7f3a6ec82680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T08:57:38.770 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:38 smithi138 conmon[100738]: debug 2023-10-01T08:57:38.511+0000 7f3a6ec82680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T08:57:39.033 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:38 smithi138 conmon[100738]: debug 2023-10-01T08:57:38.769+0000 7f3a6ec82680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T08:57:39.033 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:38 smithi138 conmon[100738]: debug 2023-10-01T08:57:38.899+0000 7f3a6ec82680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T08:57:39.286 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100738]: debug 2023-10-01T08:57:39.033+0000 7f3a6ec82680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T08:57:39.287 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100738]: debug 2023-10-01T08:57:39.202+0000 7f3a6ec82680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T08:57:39.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100738]: debug 2023-10-01T08:57:39.286+0000 7f3a6ec82680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T08:57:39.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100738]: [80B blob data] 2023-10-01T08:57:39.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: cluster 2023-10-01T08:57:39.299909+0000 mon.a (mon.0) 2023-10-01T08:57:39.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: 66 : cluster [INF] Active manager daemon y restarted 2023-10-01T08:57:39.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: cluster 2023-10-01T08:57:39.300136+0000 mon.a (mon.0) 67 : cluster [INF] Activating manager daemon y 2023-10-01T08:57:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: cluster 2023-10-01T08:57:39.303170+0000 mon.a ( 2023-10-01T08:57:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: mon.0) 68 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-10-01T08:57:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: 2023-10-01T08:57:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: cluster 2023-10-01 2023-10-01T08:57:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: T08:57:39.303300+0000 2023-10-01T08:57:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: mon.a (mon.0) 69 : cluster 2023-10-01T08:57:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: [DBG] mgrmap e10: y(active, starting, since 0.00325933s) 2023-10-01T08:57:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: audit 2023-10-01T08:57:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: 2023-10-01T08:57:39 2023-10-01T08:57:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: .304120+0000 mon.a (mon.0) 2023-10-01T08:57:39.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: 70 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T08:57:39.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: audit 2023-10 2023-10-01T08:57:39.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: -01T08:57:39.304796 2023-10-01T08:57:39.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: +0000 mon.a (mon.0) 71 : audit 2023-10-01T08:57:39.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T08:57:39.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: audit 2023-10-01T08:57:39.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: 2023-10-01T08:57 2023-10-01T08:57:39.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: :39.305606+0000 mon.a (mon. 2023-10-01T08:57:39.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: 0) 72 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T08:57:39.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: audit 2023- 2023-10-01T08:57:39.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: 10-01T08:57:39.305771 2023-10-01T08:57:39.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: +0000 mon.a (mon.0) 73 : audit 2023-10-01T08:57:39.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T08:57:39.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: audit 2023 2023-10-01T08:57:39.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: -10-01T08:57:39 2023-10-01T08:57:39.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: .305923+0000 mon.a (mon.0) 2023-10-01T08:57:39.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: 74 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T08:57:39.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: 2023-10-01T08:57:39.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: cluster 2023 2023-10-01T08:57:39.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: -10-01T08:57:39 2023-10-01T08:57:39.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: .319736+0000 mon.a (mon.0) 75 : 2023-10-01T08:57:39.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:39 smithi138 conmon[100289]: cluster [INF] Manager daemon y is now available 2023-10-01T08:57:40.316 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: { 2023-10-01T08:57:40.317 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-10-01T08:57:40.317 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: "initialized": true 2023-10-01T08:57:40.317 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: } 2023-10-01T08:57:40.624 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:40 smithi138 conmon[100289]: audit 2023-10-01T08:57:39.412855+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:57:40.624 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:40 smithi138 conmon[100289]: audit 2023-10-01T08:57:39.414247+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:57:40.624 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:40 smithi138 conmon[100289]: audit 2023-10-01T08:57:39.437973+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T08:57:40.624 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:40 smithi138 conmon[100289]: audit 2023-10-01T08:57:39.440118+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T08:57:40.625 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:40 smithi138 conmon[100289]: cluster 2023-10-01T08:57:40.304951+0000 mon.a (mon.0) 80 : cluster [DBG] mgrmap e11: y(active, since 1.00491s) 2023-10-01T08:57:41.053 INFO:teuthology.orchestra.run.smithi138.stderr:mgr epoch 9 is available 2023-10-01T08:57:41.053 INFO:teuthology.orchestra.run.smithi138.stderr:Generating a dashboard self-signed certificate... 2023-10-01T08:57:41.786 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: Self-signed certificate created 2023-10-01T08:57:41.949 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:41 smithi138 conmon[100289]: audit 2023-10-01T08:57:40.304189+0000 mgr.y (mgr.14146) 1 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-01T08:57:41.949 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:41 smithi138 conmon[100289]: audit 2023-10-01T08:57:40.315817+0000 mgr.y (mgr.14146) 2 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-01T08:57:41.949 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:41 smithi138 conmon[100289]: audit 2023-10-01T08:57:40.613374+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:57:42.625 INFO:teuthology.orchestra.run.smithi138.stderr:Creating initial admin user... 2023-10-01T08:57:42.860 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:42 smithi138 conmon[100289]: cluster 2023-10-01T08:57:41.617348+0000 mon.a (mon.0 2023-10-01T08:57:42.860 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:42 smithi138 conmon[100289]: ) 82 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-10-01T08:57:42.860 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:42 smithi138 conmon[100289]: audit 2023-10-01T08:57:41.714826+0000 mgr.y (mgr.14146) 3 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:57:42.861 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:42 smithi138 conmon[100289]: audit 2023-10-01T08:57:41.782939+0000 mon.a (mon.0) 2023-10-01T08:57:42.861 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:42 smithi138 conmon[100289]: 83 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:57:42.861 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:42 smithi138 conmon[100289]: audit 2023-10-01T08: 2023-10-01T08:57:42.861 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:42 smithi138 conmon[100289]: 57:41.785381+0000 mon.a (mon.0) 84 : 2023-10-01T08:57:42.861 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:42 smithi138 conmon[100289]: audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:57:43.482 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$KrYIKsYbU1BvadGzG3EPXOyXEVINXxGAO6wMUZnD//Wb8YKCXZchq", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696150663, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-01T08:57:44.049 INFO:teuthology.orchestra.run.smithi138.stderr:Fetching dashboard port number... 2023-10-01T08:57:44.668 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: 8443 2023-10-01T08:57:44.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:44 smithi138 conmon[100289]: audit 2023-10-01T08:57:43.265808+0000 mgr.y (mgr.14146) 4 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:57:44.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:44 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T08:57:44.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:44 smithi138 conmon[100289]: 01T08:57:43.481991+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:57:45.233 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: disabled 2023-10-01T08:57:45.240 INFO:teuthology.orchestra.run.smithi138.stderr:systemctl: inactive 2023-10-01T08:57:45.241 INFO:teuthology.orchestra.run.smithi138.stderr:firewalld.service is not enabled 2023-10-01T08:57:45.241 INFO:teuthology.orchestra.run.smithi138.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-01T08:57:45.243 INFO:teuthology.orchestra.run.smithi138.stderr:Ceph Dashboard is now available at: 2023-10-01T08:57:45.243 INFO:teuthology.orchestra.run.smithi138.stderr: 2023-10-01T08:57:45.243 INFO:teuthology.orchestra.run.smithi138.stderr: URL: https://smithi138:8443/ 2023-10-01T08:57:45.243 INFO:teuthology.orchestra.run.smithi138.stderr: User: admin 2023-10-01T08:57:45.243 INFO:teuthology.orchestra.run.smithi138.stderr: Password: qca684cyqj 2023-10-01T08:57:45.243 INFO:teuthology.orchestra.run.smithi138.stderr: 2023-10-01T08:57:45.244 INFO:teuthology.orchestra.run.smithi138.stderr:Enabling autotune for osd_memory_target 2023-10-01T08:57:45.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:45 smithi138 conmon[100289]: audit 2023-10-01T08:57:44.668523+0000 mon.a (mon.0) 86 : audit [DBG] from='client.? 172.21.15.138:0/2881383592' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-01T08:57:46.754 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:46 smithi138 conmon[100289]: cluster 2023-10-01T08:57:45.484944+0000 mon.a (mon.0) 2023-10-01T08:57:46.754 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:46 smithi138 conmon[100289]: 87 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-10-01T08:57:47.204 INFO:teuthology.orchestra.run.smithi138.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-10-01T08:57:47.791 INFO:teuthology.orchestra.run.smithi138.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-01T08:57:47.791 INFO:teuthology.orchestra.run.smithi138.stderr: 2023-10-01T08:57:47.791 INFO:teuthology.orchestra.run.smithi138.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-01T08:57:47.791 INFO:teuthology.orchestra.run.smithi138.stderr: 2023-10-01T08:57:47.792 INFO:teuthology.orchestra.run.smithi138.stderr:Or, if you are only running a single cluster on this host: 2023-10-01T08:57:47.792 INFO:teuthology.orchestra.run.smithi138.stderr: 2023-10-01T08:57:47.792 INFO:teuthology.orchestra.run.smithi138.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-01T08:57:47.792 INFO:teuthology.orchestra.run.smithi138.stderr: 2023-10-01T08:57:47.792 INFO:teuthology.orchestra.run.smithi138.stderr:Please consider enabling telemetry to help improve Ceph: 2023-10-01T08:57:47.792 INFO:teuthology.orchestra.run.smithi138.stderr: 2023-10-01T08:57:47.792 INFO:teuthology.orchestra.run.smithi138.stderr: ceph telemetry on 2023-10-01T08:57:47.792 INFO:teuthology.orchestra.run.smithi138.stderr: 2023-10-01T08:57:47.792 INFO:teuthology.orchestra.run.smithi138.stderr:For more information see: 2023-10-01T08:57:47.792 INFO:teuthology.orchestra.run.smithi138.stderr: 2023-10-01T08:57:47.792 INFO:teuthology.orchestra.run.smithi138.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-10-01T08:57:47.792 INFO:teuthology.orchestra.run.smithi138.stderr: 2023-10-01T08:57:47.793 INFO:teuthology.orchestra.run.smithi138.stderr:Bootstrap complete. 2023-10-01T08:57:47.840 INFO:tasks.cephadm:Fetching config... 2023-10-01T08:57:47.840 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T08:57:47.840 DEBUG:teuthology.orchestra.run.smithi138:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-01T08:57:47.857 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-01T08:57:47.857 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T08:57:47.857 DEBUG:teuthology.orchestra.run.smithi138:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-01T08:57:47.916 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-01T08:57:47.916 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T08:57:47.916 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-01T08:57:47.983 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-01T08:57:47.983 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T08:57:47.983 DEBUG:teuthology.orchestra.run.smithi138:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-01T08:57:48.040 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-01T08:57:48.040 DEBUG:teuthology.orchestra.run.smithi138:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3gXIhZS2nXIHRbmU2S3ulEotVJmYnJIIKXx6PAk84FC2r6Ck51qmj7LsqLTrNSZAyXsEvAXPfHhfqIi2UmAKVxpBhCxT6GbTfAL8T5SeuHJ864f4iqPv4213b1b6UTYx+2dZCt5MO5XgwL87xmm9MnYDJXGoLNfygmgSxUMcw5FNJqiLqKs4Wsz0mQMVDMqOffSXRM2myFF2fGiOuKEBvAtO1lxZZlGD6K6Ifqb13tYONs7MA4j63zXjItu1oWLaEum/scGb4N7M0Ce51TH90ECqXLdwtdcn7PRzSO+P8iBcRaQXLS8Wkst0zEk30mBsmNBPKIQ6zS92hARwkPrQ5Ci/6r8/uEa/ZkSv0Ro/2C53CVJrMiy95+sh6dRVA0QNxQwr+n9SzQ2ka3mvqz5vttJOCpY4gu+U2Y9bRdCIolor+U8vAOQuuywunzx9gGKkmSzd2NE2X0k2TFukIYESTX0SmpVGZdIDRY6YkSS1khyCRcsPdkFsAOVR9brtukfc= ceph-465870a4-6038-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-01T08:57:48.117 INFO:teuthology.orchestra.run.smithi138.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3gXIhZS2nXIHRbmU2S3ulEotVJmYnJIIKXx6PAk84FC2r6Ck51qmj7LsqLTrNSZAyXsEvAXPfHhfqIi2UmAKVxpBhCxT6GbTfAL8T5SeuHJ864f4iqPv4213b1b6UTYx+2dZCt5MO5XgwL87xmm9MnYDJXGoLNfygmgSxUMcw5FNJqiLqKs4Wsz0mQMVDMqOffSXRM2myFF2fGiOuKEBvAtO1lxZZlGD6K6Ifqb13tYONs7MA4j63zXjItu1oWLaEum/scGb4N7M0Ce51TH90ECqXLdwtdcn7PRzSO+P8iBcRaQXLS8Wkst0zEk30mBsmNBPKIQ6zS92hARwkPrQ5Ci/6r8/uEa/ZkSv0Ro/2C53CVJrMiy95+sh6dRVA0QNxQwr+n9SzQ2ka3mvqz5vttJOCpY4gu+U2Y9bRdCIolor+U8vAOQuuywunzx9gGKkmSzd2NE2X0k2TFukIYESTX0SmpVGZdIDRY6YkSS1khyCRcsPdkFsAOVR9brtukfc= ceph-465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:57:48.133 DEBUG:teuthology.orchestra.run.smithi190:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3gXIhZS2nXIHRbmU2S3ulEotVJmYnJIIKXx6PAk84FC2r6Ck51qmj7LsqLTrNSZAyXsEvAXPfHhfqIi2UmAKVxpBhCxT6GbTfAL8T5SeuHJ864f4iqPv4213b1b6UTYx+2dZCt5MO5XgwL87xmm9MnYDJXGoLNfygmgSxUMcw5FNJqiLqKs4Wsz0mQMVDMqOffSXRM2myFF2fGiOuKEBvAtO1lxZZlGD6K6Ifqb13tYONs7MA4j63zXjItu1oWLaEum/scGb4N7M0Ce51TH90ECqXLdwtdcn7PRzSO+P8iBcRaQXLS8Wkst0zEk30mBsmNBPKIQ6zS92hARwkPrQ5Ci/6r8/uEa/ZkSv0Ro/2C53CVJrMiy95+sh6dRVA0QNxQwr+n9SzQ2ka3mvqz5vttJOCpY4gu+U2Y9bRdCIolor+U8vAOQuuywunzx9gGKkmSzd2NE2X0k2TFukIYESTX0SmpVGZdIDRY6YkSS1khyCRcsPdkFsAOVR9brtukfc= ceph-465870a4-6038-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-01T08:57:48.191 INFO:teuthology.orchestra.run.smithi190.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3gXIhZS2nXIHRbmU2S3ulEotVJmYnJIIKXx6PAk84FC2r6Ck51qmj7LsqLTrNSZAyXsEvAXPfHhfqIi2UmAKVxpBhCxT6GbTfAL8T5SeuHJ864f4iqPv4213b1b6UTYx+2dZCt5MO5XgwL87xmm9MnYDJXGoLNfygmgSxUMcw5FNJqiLqKs4Wsz0mQMVDMqOffSXRM2myFF2fGiOuKEBvAtO1lxZZlGD6K6Ifqb13tYONs7MA4j63zXjItu1oWLaEum/scGb4N7M0Ce51TH90ECqXLdwtdcn7PRzSO+P8iBcRaQXLS8Wkst0zEk30mBsmNBPKIQ6zS92hARwkPrQ5Ci/6r8/uEa/ZkSv0Ro/2C53CVJrMiy95+sh6dRVA0QNxQwr+n9SzQ2ka3mvqz5vttJOCpY4gu+U2Y9bRdCIolor+U8vAOQuuywunzx9gGKkmSzd2NE2X0k2TFukIYESTX0SmpVGZdIDRY6YkSS1khyCRcsPdkFsAOVR9brtukfc= ceph-465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:57:48.203 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-01T08:57:48.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:48 smithi138 conmon[100289]: audit 2023-10-01T08:57:47.201275+0000 mon.a (mon.0) 88 : audit [INF] from='client.? 172.21.15.138:0/315163851' entity='client.admin' 2023-10-01T08:57:51.123 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-01T08:57:51.123 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-01T08:57:51.789 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:51 smithi138 conmon[100289]: audit 2023-10-01T08:57:50.454704+0000 mon.a (mon.0) 89 : audit [INF] from='client.? 172.21.15.138:0/4190830472' entity='client.admin' 2023-10-01T08:57:53.215 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:52 smithi138 conmon[100289]: audit 2023-10-01T08:57:51.882667+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:57:53.215 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:52 smithi138 conmon[100289]: audit 2023-10-01T08:57:53.216 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:52 smithi138 conmon[100289]: 2023-10-01T08:57:52.480559+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi138", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:57:53.216 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:52 smithi138 conmon[100289]: audit 2023-10-01T08:57:52.483856+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:57:53.216 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:52 smithi138 conmon[100289]: audit 2023-10-01T08:57:52.485183+0000 mon.a (mon.0) 93 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:57:53.216 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:52 smithi138 conmon[100289]: audit 2023-10-01T08:57:52.488274+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:57:53.846 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi190 2023-10-01T08:57:53.846 DEBUG:teuthology.orchestra.run.smithi190:> set -ex 2023-10-01T08:57:53.846 DEBUG:teuthology.orchestra.run.smithi190:> dd of=/etc/ceph/ceph.conf 2023-10-01T08:57:53.867 DEBUG:teuthology.orchestra.run.smithi190:> set -ex 2023-10-01T08:57:53.867 DEBUG:teuthology.orchestra.run.smithi190:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-01T08:57:53.926 INFO:tasks.cephadm:Adding host smithi190 to orchestrator... 2023-10-01T08:57:53.927 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi190 2023-10-01T08:57:54.446 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:54 smithi138 conmon[100289]: audit 2023-10-01T08:57:53.193197+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:57:54.446 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:54 smithi138 conmon[100289]: audit 2023-10-01T08:57:53.204037+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:57:54.446 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:54 smithi138 conmon[100289]: audit 2023-10-01T08:57:53.205281+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:57:54.447 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:54 smithi138 conmon[100289]: audit 2023-10-01T08:57:53.205852+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:57:54.447 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:54 smithi138 conmon[100289]: cephadm 2023-10-01T08:57:53.206820+0000 mgr.y (mgr.14146) 6 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T08:57:54.447 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:54 smithi138 conmon[100289]: cephadm 2023-10-01T08:57:53.230948+0000 mgr.y (mgr.14146) 7 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T08:57:54.447 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:54 smithi138 conmon[100289]: audit 2023-10-01T08:57:53.259634+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:57:54.447 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:54 smithi138 conmon[100289]: 2023-10-01T08:57:54.447 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:54 smithi138 conmon[100289]: audit 2023-10-01T08:57:54.447 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:54 smithi138 conmon[100289]: 2023-10-01T08:57:53.262800+0000 mon.a (mon. 2023-10-01T08:57:54.448 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:54 smithi138 conmon[100289]: 0) 100 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:57:56.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:55 smithi138 conmon[100738]: [90B blob data] 2023-10-01T08:57:56.544 INFO:teuthology.orchestra.run.smithi138.stdout:Added host 'smithi190' with addr '172.21.15.190' 2023-10-01T08:57:56.765 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:56 smithi138 conmon[100289]: audit 2023-10-01T08:57:55.856910+0000 mgr.y (mgr.14146) 8 : audit [DBG] from='client.14170 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi190", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:57:56.765 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 08:57:56 smithi138 conmon[100738]: [80B blob data] 2023-10-01T08:57:57.230 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-01T08:57:57.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:57 smithi138 conmon[100289]: audit 2023-10-01T08:57:56.543783+0000 mon.a (mon 2023-10-01T08:57:57.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:57 smithi138 conmon[100289]: .0) 101 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:57:57.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:57 smithi138 conmon[100289]: cephadm 2023-10-01T08:57:56.544057+0000 mgr.y (mgr.14146) 9 : cephadm [INF] Added host smithi190 2023-10-01T08:57:57.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:57 smithi138 conmon[100289]: audit 2023-10-01T08:57:56.584109+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:57:57.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:57 smithi138 conmon[100289]: audit 2023-10-01T08:57:57.053315+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:57:57.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:57:57 smithi138 conmon[100289]: audit 2023-10-01T08:57:57.414690+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:57:59.140 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T08:57:59.141 INFO:teuthology.orchestra.run.smithi138.stdout:[{"addr": "172.21.15.138", "hostname": "smithi138", "labels": [], "status": ""}, {"addr": "172.21.15.190", "hostname": "smithi190", "labels": [], "status": ""}] 2023-10-01T08:57:59.728 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-01T08:57:59.729 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-01T08:58:00.537 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:00 smithi138 conmon[100289]: audit 2023-10-01T08:57:59.139675+0000 mgr.y (mgr.14146) 10 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T08:58:00.537 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:00 smithi138 conmon[100289]: cluster 2023-10-01T08:57:59.306865+0000 mgr.y (mgr.14146) 11 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:01.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:01 smithi138 conmon[100289]: audit 2023-10-01T08:58:00.324457+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:01.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:01 smithi138 conmon[100289]: audit 2023-10-01T08: 2023-10-01T08:58:01.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:01 smithi138 conmon[100289]: 58:01.124286+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:01.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:01 smithi138 conmon[100289]: audit 2023-10-01T08:58:01.124962+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi190", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:58:01.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:01 smithi138 conmon[100289]: audit 2023-10-01T08:58:01.125729+0000 mon.a (mon.0) 108 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:01.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:01 smithi138 conmon[100289]: audit 2023-10-01T08:58:01.126344+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:01.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:01 smithi138 conmon[100289]: audit 2023-10-01T08:58:01.180060+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:01.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:01 smithi138 conmon[100289]: audit 2023-10-01T08:58:01.185453+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:02.334 INFO:teuthology.orchestra.run.smithi138.stderr:adjusted tunables profile to default 2023-10-01T08:58:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:02 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:01.127328+0000 mgr.y (mgr.14146) 12 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T08:58:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:02 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:01.158457+0000 mgr.y (mgr.14146) 13 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T08:58:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:02 smithi138 conmon[100289]: cluster 2023-10-01T08:58:01.307139+0000 mgr.y (mgr.14146) 14 : 2023-10-01T08:58:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:02 smithi138 conmon[100289]: cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:02 smithi138 conmon[100289]: audit 2023-10-01T08: 2023-10-01T08:58:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:02 smithi138 conmon[100289]: 58:01.522449+0000 mon.a (mon.0) 112 : audit [INF] from='client.? 172.21.15.138:0/1227346057' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-01T08:58:02.990 INFO:tasks.cephadm:Adding mon.a on smithi138 2023-10-01T08:58:02.990 INFO:tasks.cephadm:Adding mon.c on smithi138 2023-10-01T08:58:02.990 INFO:tasks.cephadm:Adding mon.b on smithi190 2023-10-01T08:58:02.990 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi138:172.21.15.138=a;smithi138:[v2:172.21.15.138:3301,v1:172.21.15.138:6790]=c;smithi190:172.21.15.190=b' 2023-10-01T08:58:03.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:03 smithi138 conmon[100289]: audit 2023-10-01T08:58:02.330481+0000 mon.a (mon.0) 113 : audit [INF] from='client.? 172.21.15.138:0/1227346057' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-01T08:58:03.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:03 smithi138 conmon[100289]: cluster 2023-10-01T08:58:02.330579+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-01T08:58:04.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:04 smithi138 conmon[100289]: cluster 2023-10-01T08:58:03.307544+0000 mgr.y (mgr.14146) 15 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:04.849 INFO:teuthology.orchestra.run.smithi190.stdout:Scheduled mon update... 2023-10-01T08:58:05.375 DEBUG:teuthology.orchestra.run.smithi138:mon.c> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-01T08:58:05.378 DEBUG:teuthology.orchestra.run.smithi190:mon.b> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-01T08:58:05.380 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-01T08:58:05.381 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-01T08:58:05.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:-- Logs begin at Sun 2023-10-01 08:38:18 UTC. -- 2023-10-01T08:58:05.415 INFO:journalctl@ceph.mon.c.smithi138.stdout:-- Logs begin at Sun 2023-10-01 08:38:26 UTC. -- 2023-10-01T08:58:06.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:05 smithi138 conmon[100289]: audit 2023-10-01T08:58:04.842642+0000 mgr.y (mgr.14146) 16 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi138:172.21.15.138=a;smithi138:[v2:172.21.15.138:3301,v1:172.21.15.138:6790]=c;smithi190:172.21.15.190=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:58:06.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:05 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:04.844838+0000 mgr.y (mgr.14146) 17 : cephadm [INF] Saving service mon spec with placement smithi138:172.21.15.138=a;smithi138:[v2:172.21.15.138:3301,v1:172.21.15.138:6790]=c;smithi190:172.21.15.190=b;count:3 2023-10-01T08:58:06.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:05 smithi138 conmon[100289]: audit 2023-10-01T08:58:04.847748+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:06.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:05 smithi138 conmon[100289]: audit 2023-10-01T08:58:04.858419+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:06.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:05 smithi138 conmon[100289]: audit 2023-10-01T08:58:04.860108+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:06.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:05 smithi138 conmon[100289]: audit 2023-10-01T08:58:04.860915+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:06.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:05 smithi138 conmon[100289]: audit 2023-10-01T08:58:04.867209+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:06.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:05 smithi138 conmon[100289]: audit 2023-10-01T08:58:04.869093+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T08:58:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:05 smithi138 conmon[100289]: audit 2023-10-01T08:58:04.869843+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:05 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:04.870560+0000 mgr.y (mgr.14146) 18 : cephadm [INF] Deploying daemon mon.b on smithi190 2023-10-01T08:58:07.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:06 smithi138 conmon[100289]: cluster 2023-10-01T08:58:05.307943+0000 mgr.y (mgr.14146 2023-10-01T08:58:07.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:06 smithi138 conmon[100289]: ) 19 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:07.776 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:58:07.776 INFO:teuthology.orchestra.run.smithi190.stdout:{"epoch":1,"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","modified":"2023-10-01T08:56:52.494621Z","created":"2023-10-01T08:56:52.494621Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:3300","nonce":0},{"type":"v1","addr":"172.21.15.138:6789","nonce":0}]},"addr":"172.21.15.138:6789/0","public_addr":"172.21.15.138:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-10-01T08:58:07.778 INFO:teuthology.orchestra.run.smithi190.stderr:dumped monmap epoch 1 2023-10-01T08:58:08.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:07 smithi138 conmon[100289]: audit 2023-10-01T08:58:07.775885+0000 mon.a (mon.0) 122 : 2023-10-01T08:58:08.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:07 smithi138 conmon[100289]: audit [DBG] from='client.? 172.21.15.190:0/860426866' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T08:58:09.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:08 smithi190 systemd[1]: Starting Ceph mon.b for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T08:58:09.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:08 smithi138 conmon[100289]: cluster 2023-10-01T08:58:07.308354 2023-10-01T08:58:09.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:08 smithi138 conmon[100289]: +0000 mgr.y (mgr.14146) 20 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:09.543 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-01T08:58:09.543 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-01T08:58:09.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.408+0000 7fabba85b880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-01T08:58:09.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.408+0000 7fabba85b880 0 ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable), process ceph-mon, pid 7 2023-10-01T08:58:09.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.408+0000 7fabba85b880 0 pidfile_write: ignore empty --pid-file 2023-10-01T08:58:09.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.412+0000 7fabba85b880 0 load: jerasure load: lrc load: isa 2023-10-01T08:58:09.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: RocksDB version: 6.8.1 2023-10-01T08:58:09.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-01T08:58:09.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Compile date Sep 25 2023 2023-10-01T08:58:09.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: DB SUMMARY 2023-10-01T08:58:09.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: CURRENT file: CURRENT 2023-10-01T08:58:09.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-01T08:58:09.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-10-01T08:58:09.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-10-01T08:58:09.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000003.log size: 511 ; 2023-10-01T08:58:09.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.error_if_exists: 0 2023-10-01T08:58:09.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.create_if_missing: 0 2023-10-01T08:58:09.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.paranoid_checks: 1 2023-10-01T08:58:09.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.env: 0x55a17afe6080 2023-10-01T08:58:09.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.fs: Posix File System 2023-10-01T08:58:09.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.info_log: 0x55a17ba5da80 2023-10-01T08:58:09.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-01T08:58:09.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.statistics: (nil) 2023-10-01T08:58:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.use_fsync: 0 2023-10-01T08:58:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_log_file_size: 0 2023-10-01T08:58:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T08:58:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T08:58:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-01T08:58:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-01T08:58:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.allow_fallocate: 1 2023-10-01T08:58:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-01T08:58:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-01T08:58:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.use_direct_reads: 0 2023-10-01T08:58:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T08:58:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-01T08:58:09.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.db_log_dir: 2023-10-01T08:58:09.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-10-01T08:58:09.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T08:58:09.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_subcompactions: 1 2023-10-01T08:58:09.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_background_flushes: -1 2023-10-01T08:58:09.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T08:58:09.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T08:58:09.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T08:58:09.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T08:58:09.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T08:58:09.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-01T08:58:09.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-01T08:58:09.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.write_buffer_manager: 0x55a17c7582d0 2023-10-01T08:58:09.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T08:58:09.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-01T08:58:09.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T08:58:09.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T08:58:09.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-01T08:58:09.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T08:58:09.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-01T08:58:09.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-01T08:58:09.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-01T08:58:09.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.unordered_write: 0 2023-10-01T08:58:09.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T08:58:09.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T08:58:09.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T08:58:09.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T08:58:09.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.row_cache: None 2023-10-01T08:58:09.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.wal_filter: None 2023-10-01T08:58:09.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T08:58:09.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-01T08:58:09.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.preserve_deletes: 0 2023-10-01T08:58:09.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.two_write_queues: 0 2023-10-01T08:58:09.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-01T08:58:09.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.atomic_flush: 0 2023-10-01T08:58:09.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T08:58:09.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T08:58:09.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T08:58:09.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.log_readahead_size: 0 2023-10-01T08:58:09.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-01T08:58:09.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_background_jobs: 2 2023-10-01T08:58:09.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_background_compactions: -1 2023-10-01T08:58:09.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T08:58:09.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T08:58:09.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T08:58:09.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-01T08:58:09.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T08:58:09.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T08:58:09.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T08:58:09.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T08:58:09.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_open_files: -1 2023-10-01T08:58:09.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-01T08:58:09.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T08:58:09.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T08:58:09.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-01T08:58:09.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Compression algorithms supported: 2023-10-01T08:58:09.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T08:58:09.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: kZSTD supported: 0 2023-10-01T08:58:09.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: kXpressCompression supported: 0 2023-10-01T08:58:09.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-01T08:58:09.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: kLZ4Compression supported: 1 2023-10-01T08:58:09.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: kBZip2Compression supported: 0 2023-10-01T08:58:09.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: kZlibCompression supported: 1 2023-10-01T08:58:09.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: kSnappyCompression supported: 1 2023-10-01T08:58:09.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T08:58:09.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 2023-10-01T08:58:09.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-01T08:58:09.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T08:58:09.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.merge_operator: 2023-10-01T08:58:09.818 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compaction_filter: None 2023-10-01T08:58:09.818 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-01T08:58:09.818 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T08:58:09.818 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T08:58:09.818 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a17ba07d18) 2023-10-01T08:58:09.818 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: cache_index_and_filter_blocks: 1 2023-10-01T08:58:09.818 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T08:58:09.818 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T08:58:09.818 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: pin_top_level_index_and_filter: 1 2023-10-01T08:58:09.818 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: index_type: 0 2023-10-01T08:58:09.818 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: data_block_index_type: 0 2023-10-01T08:58:09.819 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: index_shortening: 1 2023-10-01T08:58:09.819 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: data_block_hash_table_util_ratio: 0.750000 2023-10-01T08:58:09.819 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: hash_index_allow_collision: 1 2023-10-01T08:58:09.819 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: checksum: 1 2023-10-01T08:58:09.819 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: no_block_cache: 0 2023-10-01T08:58:09.819 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: block_cache: 0x55a17ba40f10 2023-10-01T08:58:09.819 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: block_cache_name: BinnedLRUCache 2023-10-01T08:58:09.819 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: block_cache_options: 2023-10-01T08:58:09.819 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: capacity : 536870912 2023-10-01T08:58:09.819 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: num_shard_bits : 4 2023-10-01T08:58:09.819 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: strict_capacity_limit : 0 2023-10-01T08:58:09.819 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: high_pri_pool_ratio: 0.000 2023-10-01T08:58:09.819 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: block_cache_compressed: (nil) 2023-10-01T08:58:09.820 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: persistent_cache: (nil) 2023-10-01T08:58:09.820 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: block_size: 4096 2023-10-01T08:58:09.820 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: block_size_deviation: 10 2023-10-01T08:58:09.820 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: block_restart_interval: 16 2023-10-01T08:58:09.820 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: index_block_restart_interval: 1 2023-10-01T08:58:09.820 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: metadata_block_size: 4096 2023-10-01T08:58:09.820 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: partition_filters: 0 2023-10-01T08:58:09.820 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: use_delta_encoding: 1 2023-10-01T08:58:09.820 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-01T08:58:09.820 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: whole_key_filtering: 1 2023-10-01T08:58:09.820 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: verify_compression: 0 2023-10-01T08:58:09.820 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: read_amp_bytes_per_bit: 0 2023-10-01T08:58:09.821 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: format_version: 2 2023-10-01T08:58:09.821 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: enable_index_compression: 1 2023-10-01T08:58:09.821 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: block_align: 0 2023-10-01T08:58:09.821 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.821 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-01T08:58:09.821 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-01T08:58:09.821 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compression: NoCompression 2023-10-01T08:58:09.821 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-01T08:58:09.821 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-01T08:58:09.821 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T08:58:09.821 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.num_levels: 7 2023-10-01T08:58:09.821 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T08:58:09.822 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T08:58:09.822 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T08:58:09.822 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T08:58:09.822 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T08:58:09.822 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T08:58:09.822 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T08:58:09.822 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T08:58:09.822 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T08:58:09.822 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T08:58:09.822 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-01T08:58:09.822 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-01T08:58:09.822 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T08:58:09.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T08:58:09.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-01T08:58:09.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T08:58:09.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T08:58:09.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T08:58:09.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-01T08:58:09.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T08:58:09.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T08:58:09.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T08:58:09.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T08:58:09.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T08:58:09.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T08:58:09.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T08:58:09.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T08:58:09.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T08:58:09.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T08:58:09.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T08:58:09.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T08:58:09.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T08:58:09.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-01T08:58:09.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T08:58:09.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T08:58:09.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-01T08:58:09.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-01T08:58:09.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T08:58:09.825 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T08:58:09.825 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T08:58:09.825 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T08:58:09.825 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T08:58:09.825 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T08:58:09.825 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T08:58:09.825 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T08:58:09.825 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T08:58:09.825 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T08:58:09.825 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.table_properties_collectors: 2023-10-01T08:58:09.825 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.inplace_update_support: 0 2023-10-01T08:58:09.825 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T08:58:09.826 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T08:58:09.826 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T08:58:09.826 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T08:58:09.826 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.bloom_locality: 0 2023-10-01T08:58:09.826 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.max_successive_merges: 0 2023-10-01T08:58:09.826 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T08:58:09.826 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-01T08:58:09.826 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-01T08:58:09.826 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-01T08:58:09.826 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.ttl: 2592000 2023-10-01T08:58:09.826 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.413+0000 7fabba85b880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T08:58:09.826 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.414+0000 7fabba85b880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T08:58:09.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.414+0000 7fabba85b880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-01T08:58:09.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.414+0000 7fabba85b880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696150689415681, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-10-01T08:58:09.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.414+0000 7fabba85b880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-10-01T08:58:09.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.414+0000 7fabba85b880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-01T08:58:09.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.414+0000 7fabba85b880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696150689416149, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1437, "table_properties": {"data_size": 523, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696150689, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-01T08:58:09.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.414+0000 7fabba85b880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-10-01T08:58:09.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 bash[100477]: 520f260f067cbf49006b6a61421850c9196ae7274f1d7372925efbde02bc38b9 2023-10-01T08:58:09.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.415+0000 7fabba85b880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696150689416521, "job": 1, "event": "recovery_finished"} 2023-10-01T08:58:09.828 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.415+0000 7fabba85b880 4 rocksdb: DB pointer 0x55a17bb0b800 2023-10-01T08:58:09.828 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.415+0000 7faba1663700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-01T08:58:09.828 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.415+0000 7faba1663700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-01T08:58:09.828 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: ** DB Stats ** 2023-10-01T08:58:09.829 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T08:58:09.829 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T08:58:09.829 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T08:58:09.829 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T08:58:09.829 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T08:58:09.829 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-01T08:58:09.829 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T08:58:09.829 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.829 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: ** Compaction Stats [default] ** 2023-10-01T08:58:09.829 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:58:09.829 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:58:09.829 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-10-01T08:58:09.830 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-10-01T08:58:09.830 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-10-01T08:58:09.830 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.830 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: ** Compaction Stats [default] ** 2023-10-01T08:58:09.830 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:58:09.830 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:58:09.830 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-10-01T08:58:09.830 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T08:58:09.830 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T08:58:09.830 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T08:58:09.830 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T08:58:09.830 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T08:58:09.831 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: AddFile(Keys): cumulative 0, interval 0 2023-10-01T08:58:09.831 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:58:09.831 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Interval compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:58:09.831 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T08:58:09.831 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.831 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: ** File Read Latency Histogram By Level [default] ** 2023-10-01T08:58:09.831 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.831 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: ** Compaction Stats [default] ** 2023-10-01T08:58:09.831 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:58:09.831 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:58:09.831 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-10-01T08:58:09.832 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-10-01T08:58:09.832 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-01T08:58:09.832 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.832 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: ** Compaction Stats [default] ** 2023-10-01T08:58:09.832 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:58:09.832 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:58:09.832 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-10-01T08:58:09.832 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T08:58:09.832 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T08:58:09.832 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T08:58:09.832 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T08:58:09.832 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T08:58:09.832 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: AddFile(Keys): cumulative 0, interval 0 2023-10-01T08:58:09.833 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:58:09.833 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:58:09.833 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T08:58:09.833 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.833 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: ** File Read Latency Histogram By Level [default] ** 2023-10-01T08:58:09.833 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.833 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.416+0000 7fabba85b880 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2023-10-01T08:58:09.833 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.416+0000 7fabba85b880 0 using public_addr v2:172.21.15.190:0/0 -> [v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0] 2023-10-01T08:58:09.833 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.416+0000 7fabba85b880 0 starting mon.b rank -1 at public addrs [v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0] at bind addrs [v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:58:09.833 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.417+0000 7fabba85b880 1 mon.b@-1(???) e0 preinit fsid 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:58:09.833 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 systemd[1]: Started Ceph mon.b for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T08:58:09.833 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.436+0000 7faba3e68700 0 mon.b@-1(synchronizing).mds e1 new map 2023-10-01T08:58:09.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.436+0000 7faba3e68700 0 mon.b@-1(synchronizing).mds e1 print_map 2023-10-01T08:58:09.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: e1 2023-10-01T08:58:09.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-01T08:58:09.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T08:58:09.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: legacy client fscid: -1 2023-10-01T08:58:09.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: No filesystems configured 2023-10-01T08:58:09.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.436+0000 7faba3e68700 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-01T08:58:09.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.436+0000 7faba3e68700 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-01T08:58:09.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.436+0000 7faba3e68700 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-01T08:58:09.835 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.436+0000 7faba3e68700 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-01T08:58:09.835 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.436+0000 7faba3e68700 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-01T08:58:09.835 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.436+0000 7faba3e68700 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-01T08:58:09.835 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.438+0000 7faba3e68700 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-01T08:58:09.835 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.438+0000 7faba3e68700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T08:58:09.835 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.438+0000 7faba3e68700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T08:58:09.835 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.438+0000 7faba3e68700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T08:58:09.835 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: cluster 2023- 2023-10-01T08:58:09.835 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 10-01T08:58:07.308354+0000 mgr.y (mgr.14146) 20 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:09.836 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: 2023-10-01T08:58:09.836 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:09 smithi190 conmon[100518]: debug 2023-10-01T08:58:09.439+0000 7faba3e68700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-01T08:58:14.739 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:58:14.739 INFO:teuthology.orchestra.run.smithi190.stdout:{"epoch":2,"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","modified":"2023-10-01T08:58:09.444065Z","created":"2023-10-01T08:56:52.494621Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:3300","nonce":0},{"type":"v1","addr":"172.21.15.138:6789","nonce":0}]},"addr":"172.21.15.138:6789/0","public_addr":"172.21.15.138:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:3300","nonce":0},{"type":"v1","addr":"172.21.15.190:6789","nonce":0}]},"addr":"172.21.15.190:6789/0","public_addr":"172.21.15.190:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-10-01T08:58:14.741 INFO:teuthology.orchestra.run.smithi190.stderr:dumped monmap epoch 2 2023-10-01T08:58:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: cluster 2023-10-01T08:58:09.308618+0000 mgr.y (mgr.14146) 21 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: cluster 2023-10-01T08:58:09.446886+0000 mon.a (mon.0) 124 : cluster [INF] mon.a calling monitor election 2023-10-01T08:58:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: audit 2023-10-01T08:58:09.460169+0000 mon.a (mon.0) 125 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T08:58:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: audit 2023-10-01T08:58:09.460359+0000 mon.a (mon.0) 126 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: audit 2023-10-01T08:58:10.444412+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: cluster 2023-10-01T08:58:11.308899+0000 mgr.y (mgr.14146) 22 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: audit 2023-10-01T08:58:11.444075+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: cluster 2023-10-01T08:58:11.444993+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-01T08:58:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: audit 2023-10-01T08:58:12.444182+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:14.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: cluster 2023-10-01T08:58:13.309199+0000 mgr.y (mgr.14146) 23 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:14.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: audit 2023-10-01T08:58:13.444236+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:14.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: audit 2023-10-01T08:58:14.444261+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:14.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: cluster 2023-10-01T08:58:14.473659+0000 mon.a (mon.0) 132 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-01T08:58:14.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: cluster 2023-10-01T08:58:14.482456+0000 mon.a (mon.0) 133 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0]} removed_ranks: {} 2023-10-01T08:58:14.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: cluster 2023-10-01T08:58:14.494817+0000 mon.a (mon.0) 134 : cluster [DBG] fsmap 2023-10-01T08:58:14.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: cluster 2023-10-01T08:58:14.494867+0000 mon.a (mon.0) 135 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-01T08:58:14.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: cluster 2023-10-01T08:58:14.495077+0000 mon.a (mon.0) 136 : cluster [DBG] mgrmap e13: y(active, since 35s) 2023-10-01T08:58:14.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: cluster 2023-10-01T08:58:14.498656+0000 mon.a (mon.0) 137 : cluster [INF] overall HEALTH_OK 2023-10-01T08:58:14.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: audit 2023-10-01T08:58:14.507658+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:14.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: audit 2023-10-01T08:58:14.508697+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T08:58:14.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:14 smithi190 conmon[100518]: audit 2023-10-01T08:58:14.509434+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:14.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: cluster 2023-10-01T08:58:09.308618+0000 2023-10-01T08:58:14.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: mgr.y (mgr.14146) 21 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:14.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: cluster 2023-10-01T08:58:09.446886+0000 mon.a (mon.0) 124 : cluster 2023-10-01T08:58:14.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: [INF] mon.a calling monitor election 2023-10-01T08:58:14.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T08:58:14.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 01T08:58:09.460169+0000 2023-10-01T08:58:14.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: mon.a (mon.0) 125 : audit 2023-10-01T08:58:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T08:58:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: audit 2023-10-01T08:58:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 2023-10-01T08:58 2023-10-01T08:58:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: :09.460359+0000 mon.a (mon 2023-10-01T08:58:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: .0) 126 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 2023-10-01T08:58:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: audit 2023-10 2023-10-01T08:58:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: -01T08:58:10.444412 2023-10-01T08:58:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: +0000 mon.a (mon.0) 127 : 2023-10-01T08:58:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: cluster 2023-10-01T08:58:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 2023-10-01 2023-10-01T08:58:14.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: T08:58:11.308899+0000 2023-10-01T08:58:14.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: mgr.y (mgr.14146) 22 : cluster 2023-10-01T08:58:14.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:14.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: audit 2023-10-01T08:58:14.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 2023-10 2023-10-01T08:58:14.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: -01T08:58:11. 2023-10-01T08:58:14.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 444075+0000 mon.a (mon.0) 128 2023-10-01T08:58:14.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:14.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: cluster 2023- 2023-10-01T08:58:14.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 10-01T08:58:11.444993 2023-10-01T08:58:14.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: +0000 mon.b (mon.1) 1 2023-10-01T08:58:14.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: : cluster [INF] mon.b calling monitor election 2023-10-01T08:58:14.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: audit 2023-10-01T08:58:14.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 2023-10-01T08:58: 2023-10-01T08:58:14.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 12.444182+0000 mon.a (mon.0) 2023-10-01T08:58:14.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 129 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:14.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: cluster 2023- 2023-10-01T08:58:14.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 10-01T08:58:13.309199 2023-10-01T08:58:14.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: +0000 mgr.y (mgr.14146) 23 2023-10-01T08:58:14.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:14.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: audit 2023-10-01T08:58:14.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 2023-10-01T 2023-10-01T08:58:14.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 08:58:13.444236+0000 2023-10-01T08:58:14.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: mon.a (mon.0) 130 : audit 2023-10-01T08:58:14.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:14.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: audit 2023-10-01T08:58:14.827 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 2023-10- 2023-10-01T08:58:14.827 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 01T08:58:14.444261 2023-10-01T08:58:14.827 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: +0000 mon.a (mon.0) 131 : 2023-10-01T08:58:14.827 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:14.827 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: cluster 2023-10-01T08:58:14.827 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 2023-10-01 2023-10-01T08:58:14.827 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: T08:58:14.473659+0000 mon.a 2023-10-01T08:58:14.827 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: (mon.0) 132 : cluster [INF] 2023-10-01T08:58:14.827 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-01T08:58:14.827 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: cluster 2023-10-01T08:58:14.827 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 2023-10-01 2023-10-01T08:58:14.827 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: T08:58:14.482456+0000 2023-10-01T08:58:14.828 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: mon.a (mon.0) 133 : cluster 2023-10-01T08:58:14.828 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0]} removed_ranks: {} 2023-10-01T08:58:14.828 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: cluster 2023 2023-10-01T08:58:14.828 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: -10-01T08:58: 2023-10-01T08:58:14.828 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 14.494817+0000 mon.a (mon. 2023-10-01T08:58:14.828 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 0) 134 : cluster [DBG] fsmap 2023-10-01T08:58:14.828 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: cluster 2023- 2023-10-01T08:58:14.828 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 10-01T08:58:14.494867 2023-10-01T08:58:14.828 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: +0000 mon.a (mon.0) 135 : cluster 2023-10-01T08:58:14.828 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-01T08:58:14.828 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: cluster 2023-10-01T08:58:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 2023- 2023-10-01T08:58:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 10-01T08:58 2023-10-01T08:58:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: :14.495077+0000 mon.a (mon 2023-10-01T08:58:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: .0) 136 : cluster [DBG] mgrmap e13: y(active, since 35s) 2023-10-01T08:58:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 2023-10-01T08:58:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: cluster 2023-10-01T08:58:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 2023-10-01T08: 2023-10-01T08:58:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 58:14.498656+0000 mon.a ( 2023-10-01T08:58:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: mon.0) 137 : cluster [INF] 2023-10-01T08:58:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: overall HEALTH_OK 2023-10-01T08:58:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: audit 2023-10-01T08:58:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 2023-10-01T08:58: 2023-10-01T08:58:14.830 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 14.507658+0000 mon.a (mon.0 2023-10-01T08:58:14.830 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: ) 138 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:14.830 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: audit 2023 2023-10-01T08:58:14.830 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: -10-01T08:58:14. 2023-10-01T08:58:14.830 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 508697+0000 mon.a (mon.0) 2023-10-01T08:58:14.830 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 139 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T08:58:14.830 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: audit 2023 2023-10-01T08:58:14.830 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: -10-01T08:58:14 2023-10-01T08:58:14.830 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: .509434+0000 mon.a (mon.0) 2023-10-01T08:58:14.830 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:14 smithi138 conmon[100289]: 140 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:15.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:15 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:14.510094+0000 mgr.y ( 2023-10-01T08:58:15.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:15 smithi138 conmon[100289]: mgr.14146) 24 : cephadm [INF] Deploying daemon mon.c on smithi138 2023-10-01T08:58:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:15 smithi138 conmon[100289]: audit 2023-10-01T08:58:14.739098+0000 mon.a (mon.0) 141 : audit [DBG] from='client.? 172.21.15.190:0/1461532158' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T08:58:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:15 smithi138 conmon[100289]: audit 2023-10-01T08:58:15.444539+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:16.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:15 smithi190 conmon[100518]: cephadm 2023-10-01T08:58:14.510094+0000 mgr.y (mgr.14146) 24 : cephadm [INF] Deploying daemon mon.c on smithi138 2023-10-01T08:58:16.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:15 smithi190 conmon[100518]: audit 2023-10-01T08:58:14.739098+0000 mon.a (mon.0) 141 : audit [DBG] from='client.? 172.21.15.190:0/1461532158' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T08:58:16.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:15 smithi190 conmon[100518]: audit 2023-10-01T08:58:15.444539+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:16.307 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-01T08:58:16.307 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-01T08:58:16.764 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:16 smithi190 conmon[100518]: cluster 2023-10-01T08:58:15.309455+0000 mgr.y (mgr.14146) 25 : 2023-10-01T08:58:16.764 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:16 smithi190 conmon[100518]: cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:16.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:16 smithi138 conmon[100289]: cluster 2023-10-01T08:58:15.309455+0000 mgr.y (mgr.14146) 25 : cluster 2023-10-01T08:58:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:16 smithi138 conmon[100289]: [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:17.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 systemd[1]: Starting Ceph mon.c for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T08:58:18.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.686+0000 7fbdba5b4880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-01T08:58:18.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.686+0000 7fbdba5b4880 0 ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable), process ceph-mon, pid 7 2023-10-01T08:58:18.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.686+0000 7fbdba5b4880 0 pidfile_write: ignore empty --pid-file 2023-10-01T08:58:18.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.690+0000 7fbdba5b4880 0 load: jerasure load: lrc load: isa 2023-10-01T08:58:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: RocksDB version: 6.8.1 2023-10-01T08:58:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-01T08:58:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Compile date Sep 25 2023 2023-10-01T08:58:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: DB SUMMARY 2023-10-01T08:58:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: CURRENT file: CURRENT 2023-10-01T08:58:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-01T08:58:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-10-01T08:58:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-10-01T08:58:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000003.log size: 636 ; 2023-10-01T08:58:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.error_if_exists: 0 2023-10-01T08:58:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.create_if_missing: 0 2023-10-01T08:58:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.paranoid_checks: 1 2023-10-01T08:58:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.env: 0x561e97d8a080 2023-10-01T08:58:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.fs: Posix File System 2023-10-01T08:58:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.info_log: 0x561e9a3a1a80 2023-10-01T08:58:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-01T08:58:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.statistics: (nil) 2023-10-01T08:58:18.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.use_fsync: 0 2023-10-01T08:58:18.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_log_file_size: 0 2023-10-01T08:58:18.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T08:58:18.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T08:58:18.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-01T08:58:18.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-01T08:58:18.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.allow_fallocate: 1 2023-10-01T08:58:18.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-01T08:58:18.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-01T08:58:18.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.use_direct_reads: 0 2023-10-01T08:58:18.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T08:58:18.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-01T08:58:18.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.db_log_dir: 2023-10-01T08:58:18.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-10-01T08:58:18.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T08:58:18.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_subcompactions: 1 2023-10-01T08:58:18.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_background_flushes: -1 2023-10-01T08:58:18.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T08:58:18.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T08:58:18.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T08:58:18.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T08:58:18.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T08:58:18.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-01T08:58:18.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-01T08:58:18.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.write_buffer_manager: 0x561e9b09c300 2023-10-01T08:58:18.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T08:58:18.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-01T08:58:18.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T08:58:18.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T08:58:18.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-01T08:58:18.077 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T08:58:18.077 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-01T08:58:18.077 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-01T08:58:18.077 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-01T08:58:18.077 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.unordered_write: 0 2023-10-01T08:58:18.077 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T08:58:18.077 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T08:58:18.077 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T08:58:18.077 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T08:58:18.077 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.row_cache: None 2023-10-01T08:58:18.077 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.wal_filter: None 2023-10-01T08:58:18.078 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T08:58:18.078 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-01T08:58:18.078 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.preserve_deletes: 0 2023-10-01T08:58:18.078 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.two_write_queues: 0 2023-10-01T08:58:18.078 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-01T08:58:18.078 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.atomic_flush: 0 2023-10-01T08:58:18.078 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T08:58:18.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T08:58:18.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T08:58:18.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.log_readahead_size: 0 2023-10-01T08:58:18.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-01T08:58:18.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_background_jobs: 2 2023-10-01T08:58:18.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_background_compactions: -1 2023-10-01T08:58:18.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T08:58:18.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T08:58:18.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T08:58:18.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-01T08:58:18.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T08:58:18.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T08:58:18.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T08:58:18.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T08:58:18.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_open_files: -1 2023-10-01T08:58:18.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-01T08:58:18.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T08:58:18.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T08:58:18.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-01T08:58:18.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Compression algorithms supported: 2023-10-01T08:58:18.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T08:58:18.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: kZSTD supported: 0 2023-10-01T08:58:18.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: kXpressCompression supported: 0 2023-10-01T08:58:18.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-01T08:58:18.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: kLZ4Compression supported: 1 2023-10-01T08:58:18.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: kBZip2Compression supported: 0 2023-10-01T08:58:18.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: kZlibCompression supported: 1 2023-10-01T08:58:18.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: kSnappyCompression supported: 1 2023-10-01T08:58:18.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T08:58:18.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 2023-10-01T08:58:18.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-01T08:58:18.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T08:58:18.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.merge_operator: 2023-10-01T08:58:18.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compaction_filter: None 2023-10-01T08:58:18.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-01T08:58:18.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T08:58:18.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T08:58:18.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561e9a34bd18) 2023-10-01T08:58:18.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: cache_index_and_filter_blocks: 1 2023-10-01T08:58:18.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T08:58:18.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T08:58:18.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: pin_top_level_index_and_filter: 1 2023-10-01T08:58:18.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: index_type: 0 2023-10-01T08:58:18.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: data_block_index_type: 0 2023-10-01T08:58:18.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: index_shortening: 1 2023-10-01T08:58:18.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: data_block_hash_table_util_ratio: 0.750000 2023-10-01T08:58:18.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: hash_index_allow_collision: 1 2023-10-01T08:58:18.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: checksum: 1 2023-10-01T08:58:18.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: no_block_cache: 0 2023-10-01T08:58:18.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: block_cache: 0x561e9a384f10 2023-10-01T08:58:18.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: block_cache_name: BinnedLRUCache 2023-10-01T08:58:18.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: block_cache_options: 2023-10-01T08:58:18.084 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: capacity : 536870912 2023-10-01T08:58:18.084 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: num_shard_bits : 4 2023-10-01T08:58:18.084 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: strict_capacity_limit : 0 2023-10-01T08:58:18.084 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: high_pri_pool_ratio: 0.000 2023-10-01T08:58:18.084 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: block_cache_compressed: (nil) 2023-10-01T08:58:18.084 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: persistent_cache: (nil) 2023-10-01T08:58:18.084 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: block_size: 4096 2023-10-01T08:58:18.084 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: block_size_deviation: 10 2023-10-01T08:58:18.084 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: block_restart_interval: 16 2023-10-01T08:58:18.084 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: index_block_restart_interval: 1 2023-10-01T08:58:18.084 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: metadata_block_size: 4096 2023-10-01T08:58:18.084 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: partition_filters: 0 2023-10-01T08:58:18.085 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: use_delta_encoding: 1 2023-10-01T08:58:18.085 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-01T08:58:18.085 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: whole_key_filtering: 1 2023-10-01T08:58:18.085 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: verify_compression: 0 2023-10-01T08:58:18.085 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: read_amp_bytes_per_bit: 0 2023-10-01T08:58:18.085 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: format_version: 2 2023-10-01T08:58:18.085 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: enable_index_compression: 1 2023-10-01T08:58:18.085 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: block_align: 0 2023-10-01T08:58:18.085 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.085 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-01T08:58:18.085 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-01T08:58:18.085 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compression: NoCompression 2023-10-01T08:58:18.086 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-01T08:58:18.086 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-01T08:58:18.086 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T08:58:18.086 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.num_levels: 7 2023-10-01T08:58:18.086 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T08:58:18.086 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T08:58:18.086 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T08:58:18.086 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T08:58:18.086 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T08:58:18.086 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T08:58:18.086 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T08:58:18.087 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T08:58:18.087 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T08:58:18.087 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T08:58:18.087 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-01T08:58:18.087 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-01T08:58:18.087 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T08:58:18.087 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T08:58:18.087 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-01T08:58:18.087 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T08:58:18.087 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T08:58:18.087 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T08:58:18.087 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-01T08:58:18.088 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T08:58:18.088 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T08:58:18.088 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T08:58:18.088 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T08:58:18.088 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T08:58:18.088 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T08:58:18.088 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T08:58:18.088 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T08:58:18.088 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T08:58:18.088 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T08:58:18.088 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T08:58:18.088 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T08:58:18.089 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T08:58:18.089 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-01T08:58:18.089 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T08:58:18.089 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T08:58:18.089 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-01T08:58:18.089 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-01T08:58:18.089 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T08:58:18.089 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T08:58:18.089 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T08:58:18.089 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T08:58:18.089 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T08:58:18.090 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T08:58:18.090 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T08:58:18.090 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T08:58:18.090 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T08:58:18.090 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T08:58:18.090 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.table_properties_collectors: 2023-10-01T08:58:18.090 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.inplace_update_support: 0 2023-10-01T08:58:18.090 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T08:58:18.090 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T08:58:18.090 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 bash[109918]: 711c0763bf425596789306b1e0047d574ea94fe22b28af324a0eda0eef996adb 2023-10-01T08:58:18.090 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T08:58:18.091 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T08:58:18.091 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.bloom_locality: 0 2023-10-01T08:58:18.091 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.max_successive_merges: 0 2023-10-01T08:58:18.091 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T08:58:18.091 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-01T08:58:18.091 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-01T08:58:18.091 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-01T08:58:18.091 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.ttl: 2592000 2023-10-01T08:58:18.091 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.691+0000 7fbdba5b4880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T08:58:18.091 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.692+0000 7fbdba5b4880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T08:58:18.091 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.092 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.692+0000 7fbdba5b4880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-01T08:58:18.092 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.092 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.692+0000 7fbdba5b4880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696150697693811, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-10-01T08:58:18.092 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.692+0000 7fbdba5b4880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-10-01T08:58:18.092 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.692+0000 7fbdba5b4880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-01T08:58:18.092 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.693+0000 7fbdba5b4880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696150697694269, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1562, "table_properties": {"data_size": 648, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696150697, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-01T08:58:18.092 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.693+0000 7fbdba5b4880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-10-01T08:58:18.092 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.092 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.693+0000 7fbdba5b4880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696150697694652, "job": 1, "event": "recovery_finished"} 2023-10-01T08:58:18.092 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.694+0000 7fbdba5b4880 4 rocksdb: DB pointer 0x561e9a44f800 2023-10-01T08:58:18.092 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.694+0000 7fbda13bc700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-01T08:58:18.093 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.694+0000 7fbda13bc700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-01T08:58:18.093 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: ** DB Stats ** 2023-10-01T08:58:18.093 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T08:58:18.093 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T08:58:18.093 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T08:58:18.093 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T08:58:18.093 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T08:58:18.093 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-01T08:58:18.093 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T08:58:18.093 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.093 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: ** Compaction Stats [default] ** 2023-10-01T08:58:18.094 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:58:18.094 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:58:18.094 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-10-01T08:58:18.094 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-10-01T08:58:18.094 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-10-01T08:58:18.094 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.094 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: ** Compaction Stats [default] ** 2023-10-01T08:58:18.094 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:58:18.094 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:58:18.094 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-10-01T08:58:18.094 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T08:58:18.094 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T08:58:18.095 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T08:58:18.095 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T08:58:18.095 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T08:58:18.095 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: AddFile(Keys): cumulative 0, interval 0 2023-10-01T08:58:18.095 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:58:18.095 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Interval compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:58:18.095 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T08:58:18.095 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.095 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: ** File Read Latency Histogram By Level [default] ** 2023-10-01T08:58:18.095 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.095 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: ** Compaction Stats [default] ** 2023-10-01T08:58:18.096 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:58:18.096 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:58:18.096 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-10-01T08:58:18.096 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-10-01T08:58:18.096 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-01T08:58:18.096 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.096 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: ** Compaction Stats [default] ** 2023-10-01T08:58:18.096 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T08:58:18.096 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T08:58:18.096 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-10-01T08:58:18.096 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T08:58:18.096 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T08:58:18.097 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T08:58:18.097 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T08:58:18.097 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T08:58:18.097 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: AddFile(Keys): cumulative 0, interval 0 2023-10-01T08:58:18.097 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:58:18.097 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T08:58:18.097 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T08:58:18.097 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.097 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: ** File Read Latency Histogram By Level [default] ** 2023-10-01T08:58:18.097 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.097 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.694+0000 7fbdba5b4880 0 mon.c does not exist in monmap, will attempt to join an existing cluster 2023-10-01T08:58:18.097 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.694+0000 7fbdba5b4880 0 using public_addrv [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] 2023-10-01T08:58:18.098 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.695+0000 7fbdba5b4880 0 starting mon.c rank -1 at public addrs [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] at bind addrs [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:58:18.098 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.696+0000 7fbdba5b4880 1 mon.c@-1(???) e0 preinit fsid 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:58:18.098 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 systemd[1]: Started Ceph mon.c for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T08:58:18.098 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.711+0000 7fbda3bc1700 0 mon.c@-1(synchronizing).mds e1 new map 2023-10-01T08:58:18.098 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.711+0000 7fbda3bc1700 0 mon.c@-1(synchronizing).mds e1 print_map 2023-10-01T08:58:18.098 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: e1 2023-10-01T08:58:18.098 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-01T08:58:18.098 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T08:58:18.098 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: legacy client fscid: -1 2023-10-01T08:58:18.098 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.098 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: No filesystems configured 2023-10-01T08:58:18.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 2023-10-01T08:58:18.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.711+0000 7fbda3bc1700 1 mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-01T08:58:18.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.711+0000 7fbda3bc1700 1 mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-01T08:58:18.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.711+0000 7fbda3bc1700 1 mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-01T08:58:18.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.711+0000 7fbda3bc1700 1 mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-01T08:58:18.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.711+0000 7fbda3bc1700 1 mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-01T08:58:18.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.711+0000 7fbda3bc1700 1 mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-01T08:58:18.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.712+0000 7fbda3bc1700 0 mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-01T08:58:18.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.712+0000 7fbda3bc1700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T08:58:18.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.712+0000 7fbda3bc1700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T08:58:18.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.712+0000 7fbda3bc1700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T08:58:18.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: cluster 2023- 2023-10-01T08:58:18.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: 10-01T08:58:15.309455+0000 mgr.y (mgr.14146) 25 : cluster [DBG] 2023-10-01T08:58:18.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:18.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:17 smithi138 conmon[109959]: debug 2023-10-01T08:58:17.712+0000 7fbda3bc1700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-01T08:58:23.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: cluster 2023-10-01T08:58:17.309694+0000 mgr.y (mgr.14146) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:23.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: audit 2023-10-01T08:58:17.720200+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T08:58:23.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: cluster 2023-10-01T08:58:17.720388+0000 mon.a (mon.0) 145 : cluster [INF] mon.a calling monitor election 2023-10-01T08:58:23.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: cluster 2023-10-01T08:58:17.720524+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-01T08:58:23.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: audit 2023-10-01T08:58:17.733311+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:23.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: audit 2023-10-01T08:58:17.733445+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:23.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: audit 2023-10-01T08:58:18.716786+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:23.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: cluster 2023-10-01T08:58:19.310066+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:23.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: audit 2023-10-01T08:58:19.716742+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:23.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: audit 2023-10-01T08:58:20.716896+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:23.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: cluster 2023-10-01T08:58:21.310343+0000 mgr.y (mgr.14146) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:23.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: audit 2023-10-01T08:58:21.717075+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:23.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: audit 2023-10-01T08:58:22.717093+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:23.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: cluster 2023-10-01T08:58:22.733466+0000 mon.a (mon.0) 153 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-01T08:58:23.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: cluster 2023-10-01T08:58:22.736835+0000 mon.a (mon.0) 154 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T08:58:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: cluster 2023-10-01T08:58:22.748724+0000 mon.a (mon.0) 155 : cluster [DBG] fsmap 2023-10-01T08:58:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: cluster 2023-10-01T08:58:22.748769+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-01T08:58:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: cluster 2023-10-01T08:58:22.748983+0000 mon.a (mon.0) 157 : cluster [DBG] mgrmap e13: y(active, since 43s) 2023-10-01T08:58:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: cluster 2023-10-01T08:58:22.749156+0000 mon.a (mon.0) 158 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-01T08:58:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: cluster 2023-10-01T08:58:22.752507+0000 mon.a (mon.0) 159 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-01T08:58:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: cluster 2023-10-01T08:58:22.752530+0000 mon.a (mon.0) 160 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-01T08:58:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: cluster 2023-10-01T08:58:22.752539+0000 mon.a (mon.0) 161 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] is down (out of quorum) 2023-10-01T08:58:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: audit 2023-10-01T08:58:22.754830+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:22 smithi190 conmon[100518]: audit 2023-10-01T08:58:22.756242+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:23.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: cluster 2023-10-01T08:58:17.309694+0000 mgr.y (mgr.14146) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:23.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: audit 2023-10-01T08:58:17.720200+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T08:58:23.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: cluster 2023-10-01T08:58:17.720388+0000 mon.a (mon.0) 145 : cluster [INF] mon.a calling monitor election 2023-10-01T08:58:23.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: cluster 2023-10-01T08:58:17.720524+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-01T08:58:23.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: audit 2023-10-01T08:58:17.733311+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T08:58:23.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: audit 2023-10-01T08:58:17.733445+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:23.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: audit 2023-10-01T08:58:18.716786+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:23.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: cluster 2023-10-01T08:58:19.310066+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:23.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: audit 2023-10-01T08:58:19.716742+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: audit 2023-10-01T08:58:20.716896+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: cluster 2023-10-01T08:58:21.310343+0000 mgr.y (mgr.14146) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: audit 2023-10-01T08:58:21.717075+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: audit 2023-10-01T08:58:22.717093+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: cluster 2023-10-01T08:58:22.733466+0000 mon.a (mon.0) 153 : cluster 2023-10-01T08:58:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-01T08:58:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: cluster 2023-10-01T08:58:22.736835+0000 mon.a (mon.0) 154 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T08:58:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: cluster 2023-10-01T08:58:22.748724+0000 mon.a (mon.0) 155 : cluster [DBG] fsmap 2023-10-01T08:58:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: cluster 2023-10-01T08:58:22.748769+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-01T08:58:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: cluster 2023-10-01T08:58:22.748983+0000 mon.a (mon.0) 157 : cluster [DBG] mgrmap e13: y(active, since 43s) 2023-10-01T08:58:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: cluster 2023-10-01T08:58:22.749156+0000 mon.a (mon.0) 158 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-01T08:58:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: cluster 2023-10-01T08:58:22.752507+0000 mon.a (mon.0) 159 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-01T08:58:23.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: cluster 2023-10-01T08:58:22.752530+0000 mon.a (mon.0) 160 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-01T08:58:23.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: cluster 2023-10-01T08:58:22.752539+0000 mon.a (mon.0) 161 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] is down (out of quorum) 2023-10-01T08:58:23.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: audit 2023-10-01T08:58:22.754830+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:23.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:22 smithi138 conmon[100289]: audit 2023-10-01T08:58:22.756242+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:24.054 INFO:teuthology.orchestra.run.smithi190.stdout: 2023-10-01T08:58:24.055 INFO:teuthology.orchestra.run.smithi190.stdout:{"epoch":3,"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","modified":"2023-10-01T08:58:17.717120Z","created":"2023-10-01T08:56:52.494621Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:3300","nonce":0},{"type":"v1","addr":"172.21.15.138:6789","nonce":0}]},"addr":"172.21.15.138:6789/0","public_addr":"172.21.15.138:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:3300","nonce":0},{"type":"v1","addr":"172.21.15.190:6789","nonce":0}]},"addr":"172.21.15.190:6789/0","public_addr":"172.21.15.190:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:3301","nonce":0},{"type":"v1","addr":"172.21.15.138:6790","nonce":0}]},"addr":"172.21.15.138:6790/0","public_addr":"172.21.15.138:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-10-01T08:58:24.056 INFO:teuthology.orchestra.run.smithi190.stderr:dumped monmap epoch 3 2023-10-01T08:58:24.611 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-01T08:58:24.612 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-01T08:58:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: cluster 2023-10-01T08:58:23.310580+0000 mgr.y (mgr.14146) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: audit 2023-10-01T08:58:23.587984+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: audit 2023-10-01T08:58:23.717264+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: audit 2023-10-01T08:58:23.724661+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: audit 2023-10-01T08:58:23.725190+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: audit 2023-10-01T08:58:23.725801+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:23.726667+0000 mgr.y (mgr.14146) 30 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T08:58:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:23.726868+0000 mgr.y (mgr.14146) 31 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T08:58:24.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: audit 2023-10-01T08:58:23.747191+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:24.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: audit 2023-10-01T08:58:23.750314+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:24.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: audit 2023-10-01T08:58:23.753254+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:24.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:23.754229+0000 mgr.y (mgr.14146) 32 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-01T08:58:24.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: audit 2023-10-01T08:58:23.754430+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T08:58:24.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: audit 2023-10-01T08:58:23.754990+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T08:58:24.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: audit 2023-10-01T08:58:23.755491+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:24.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:23.755935+0000 mgr.y (mgr.14146) 33 : cephadm [INF] Reconfiguring daemon mon.a on smithi138 2023-10-01T08:58:24.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: audit 2023-10-01T08:58:24.054299+0000 mon.a (mon.0) 175 : audit [DBG] from='client.? 172.21.15.190:0/1235790799' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T08:58:24.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: audit 2023-10-01T08:58:24.442259+0000 mon.a (mon.0) 2023-10-01T08:58:24.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[100289]: 176 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:24.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: cluster 2023-10-01T08:58:23.310580+0000 mgr.y (mgr 2023-10-01T08:58:24.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: .14146) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:24.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: audit 2023-10-01T08:58:23.587984+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:24.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: audit 2023-10-01T08:58:23.717264+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:24.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: audit 2023-10-01T08:58:23.724661+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:24.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: audit 2023-10-01T08:58:23.725190+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:24.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: audit 2023-10-01T08:58:23.725801+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:24.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: cephadm 2023-10-01T08:58:23.726667+0000 mgr.y 2023-10-01T08:58:24.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: (mgr.14146) 30 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T08:58:24.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: cephadm 2023-10-01T08:58:23.726868+0000 mgr.y (mgr.14146) 31 : 2023-10-01T08:58:24.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: cephadm [INF] Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T08:58:24.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: audit 2023-10-01T08:58:24.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: 2023-10-01T08:58:23.747191+0000 mon.a (mon. 2023-10-01T08:58:24.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: 0) 169 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:24.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T08:58:24.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: 01T08:58:23.750314+0000 mon.a (mon 2023-10-01T08:58:24.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: .0) 170 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:24.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: audit 2023 2023-10-01T08:58:24.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: -10-01T08:58:23.753254+0000 mon.a 2023-10-01T08:58:24.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:24.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: cephadm 2023-10-01T08:58:24.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: 2023-10-01T08:58 2023-10-01T08:58:24.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: :23.754229+0000 mgr.y (mgr.14146) 32 : 2023-10-01T08:58:24.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-01T08:58:24.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T08:58:24.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: 08:58:23.754430+0000 mon.a (mon.0 2023-10-01T08:58:24.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: ) 172 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T08:58:24.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: audit 2023-10-01T08:58:24.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: 2023-10-01T08:58:23.754990+0000 2023-10-01T08:58:24.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: mon.a (mon.0) 173 : audit [DBG] 2023-10-01T08:58:24.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T08:58:24.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T08:58:24.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: T08:58:23.755491+0000 mon.a (mon. 2023-10-01T08:58:24.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: 0) 174 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:24.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: cephadm 2023-10-01 2023-10-01T08:58:24.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: T08:58:23.755935+0000 mgr.y (mgr. 2023-10-01T08:58:24.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: 14146) 33 : cephadm [INF] Reconfiguring daemon mon.a on smithi138 2023-10-01T08:58:24.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: audit 2023 2023-10-01T08:58:24.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: -10-01T08:58:24.054299+0000 2023-10-01T08:58:24.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: mon.a (mon.0) 175 : audit [DBG] from='client.? 172.21.15.190:0/1235790799' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T08:58:24.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: 2023-10-01T08:58:24.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: audit 2023-10-01T08:58:24.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: 2023-10-01T08:58:24.442259 2023-10-01T08:58:24.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: +0000 mon.a (mon.0) 176 : audit [INF] 2023-10-01T08:58:24.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:24 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:25.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: cluster 2023-10-01T08:58:23.310580+0000 mgr.y (mgr.14146) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:25.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: audit 2023-10-01T08:58:23.587984+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:25.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: audit 2023-10-01T08:58:23.717264+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:25.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: audit 2023-10-01T08:58:23.724661+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:25.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: audit 2023-10-01T08:58:23.725190+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:25.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: audit 2023-10-01T08:58:23.725801+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:25.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: cephadm 2023-10-01T08:58:23.726667+0000 mgr.y (mgr.14146) 30 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T08:58:25.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: cephadm 2023-10-01T08:58:23.726868+0000 mgr.y (mgr.14146) 31 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T08:58:25.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: audit 2023-10-01T08:58:25.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: 2023-10-01T08:58:23.747191+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:25.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T08:58:25.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: 08:58:23.750314+0000 mon.a (mon.0 2023-10-01T08:58:25.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: ) 170 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:25.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: audit 2023-10 2023-10-01T08:58:25.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: -01T08:58:23.753254+0000 mon.a 2023-10-01T08:58:25.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:25.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: cephadm 2023- 2023-10-01T08:58:25.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: 10-01T08:58:23.754229+0000 2023-10-01T08:58:25.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: mgr.y (mgr.14146) 32 : cephadm [INF] 2023-10-01T08:58:25.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: Reconfiguring mon.a (unknown last config time)... 2023-10-01T08:58:25.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: audit 2023-10-01T08:58: 2023-10-01T08:58:25.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: 23.754430+0000 mon.a (mon.0) 172 : audit 2023-10-01T08:58:25.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T08:58:25.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: audit 2023-10-01T08: 2023-10-01T08:58:25.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: 58:23.754990+0000 mon.a (mon.0) 173 2023-10-01T08:58:25.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T08:58:25.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: audit 2023-10-01T08: 2023-10-01T08:58:25.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: 58:23.755491+0000 mon.a (mon.0) 174 2023-10-01T08:58:25.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:25.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: cephadm 2023-10-01T 2023-10-01T08:58:25.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: 08:58:23.755935+0000 mgr.y (mgr. 2023-10-01T08:58:25.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: 14146) 33 : cephadm [INF] Reconfiguring daemon mon.a on smithi138 2023-10-01T08:58:25.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: audit 2023 2023-10-01T08:58:25.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: -10-01T08:58:24.054299 2023-10-01T08:58:25.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: +0000 mon.a (mon.0) 175 : audit [DBG] 2023-10-01T08:58:25.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: from='client.? 172.21.15.190:0/1235790799' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T08:58:25.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: audit 2023-10-01T08: 2023-10-01T08:58:25.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: 58:24.442259+0000 mon.a (mon.0) 2023-10-01T08:58:25.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:24 smithi190 conmon[100518]: 176 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:25.918 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: cluster 2023-10-01T08:58:19.718163+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-01T08:58:25.918 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: cluster 2023-10-01T08:58:24.735291+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-10-01T08:58:25.918 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: cluster 2023-10-01T08:58:24.735612+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-10-01T08:58:25.918 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: cluster 2023-10-01T08:58:24.735623+0000 mon.a (mon.0) 178 : cluster [INF] mon.a calling monitor election 2023-10-01T08:58:25.918 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: 2023-10-01T08:58:25.918 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: cluster 2023-10-01T08:58:24.748876+0000 mon.a (mon.0) 179 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T08:58:25.918 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: cluster 2023-10-01T08:58:24.751165+0000 mon.a (mon.0) 180 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T08:58:25.919 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: cluster 2023-10-01T08:58:24.751558+0000 mon.a (mon.0) 181 : cluster [DBG] fsmap 2023-10-01T08:58:25.919 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: cluster 2023-10-01T08:58:24.751594+0000 mon.a (mon.0) 182 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-01T08:58:25.919 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: cluster 2023-10-01T08:58:24.751740+0000 mon.a (mon.0) 183 : cluster [DBG] mgrmap e13: y(active, since 45s) 2023-10-01T08:58:25.919 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: cluster 2023-10-01T08:58:24.751926+0000 mon.a (mon.0) 184 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-01T08:58:25.919 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: cluster 2023-10-01T08:58:24.751940+0000 mon.a (mon.0) 185 : cluster [INF] Cluster is now healthy 2023-10-01T08:58:25.919 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: cluster 2023-10-01T08:58:24.754217+0000 mon.a (mon.0) 186 : cluster [INF] 2023-10-01T08:58:25.919 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: overall HEALTH_OK 2023-10-01T08:58:25.919 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: audit 2023-10-01T08:58:25.079546+0000 mon.a (mon.0) 187 2023-10-01T08:58:25.919 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:25.919 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: audit 2023-10-01T08:58:25.919 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: 2023-10 2023-10-01T08:58:25.919 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: -01T08:58:25. 2023-10-01T08:58:25.920 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: 080192+0000 mon.a (mon.0) 2023-10-01T08:58:25.920 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: 188 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T08:58:25.920 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T08:58:25.920 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: 01T08:58:25.080728+0000 mon.a (mon. 2023-10-01T08:58:25.920 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: 0) 189 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T08:58:25.920 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T08:58:25.920 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: T08:58:25.081229+0000 mon.a (mon 2023-10-01T08:58:25.920 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[100289]: .0) 190 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:25.921 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: cluster 2023-10-01T08:58:19.718163+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-01T08:58:25.921 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: cluster 2023-10-01T08:58:24.735291+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-10-01T08:58:25.921 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: cluster 2023-10-01T08:58:24.735612+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-10-01T08:58:25.921 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: cluster 2023-10-01T08:58:24.735623+0000 mon.a (mon.0) 178 : cluster [INF] mon.a calling monitor election 2023-10-01T08:58:25.921 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: cluster 2023-10-01T08:58:24.748876+0000 mon.a (mon.0) 179 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T08:58:25.921 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: cluster 2023-10-01T08:58:24.751165+0000 mon.a (mon. 2023-10-01T08:58:25.921 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: 0) 180 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T08:58:25.921 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: cluster 2023-10-01T08:58:24.751558+0000 mon.a (mon.0) 181 : cluster [DBG] fsmap 2023-10-01T08:58:25.921 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: cluster 2023-10-01T08:58:24.751594+0000 mon.a (mon.0) 182 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-01T08:58:25.921 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: cluster 2023-10-01T08:58:24.751740+0000 mon.a (mon.0) 183 : cluster [DBG] mgrmap e13: y(active, since 45s) 2023-10-01T08:58:25.921 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: cluster 2023-10-01T08:58:24.751926+0000 mon.a (mon.0) 184 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-01T08:58:25.922 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: cluster 2023-10-01T08:58:24.751940+0000 mon.a (mon.0) 185 : cluster [INF] Cluster is now healthy 2023-10-01T08:58:25.922 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: cluster 2023-10-01T08:58:24.754217+0000 mon.a (mon.0) 186 : cluster [INF] overall HEALTH_OK 2023-10-01T08:58:25.922 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: audit 2023-10-01T08:58:25.079546+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:25.922 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: audit 2023-10-01T08:58:25.080192+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T08:58:25.922 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: audit 2023-10-01T08:58:25.080728+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T08:58:25.922 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: audit 2023-10-01T08:58: 2023-10-01T08:58:25.922 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:25 smithi138 conmon[109959]: 25.081229+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: cluster 2023-10-01T08:58:19.718163+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-01T08:58:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: cluster 2023-10-01T08:58:24.735291+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-10-01T08:58:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: cluster 2023-10-01T08:58:24.735612+0000 mon.b (mon. 2023-10-01T08:58:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: 1) 3 : cluster [INF] mon.b calling monitor election 2023-10-01T08:58:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: cluster 2023-10 2023-10-01T08:58:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: -01T08:58:24.735623+0000 mon.a (mon.0) 178 : 2023-10-01T08:58:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: cluster [INF] mon.a calling monitor election 2023-10-01T08:58:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: cluster 2023-10-01T08:58 2023-10-01T08:58:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: :24.748876+0000 mon.a (mon.0) 179 : cluster 2023-10-01T08:58:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T08:58:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: cluster 2023-10-01T08:58: 2023-10-01T08:58:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: 24.751165+0000 mon.a (mon.0) 180 : cluster 2023-10-01T08:58:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T08:58:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: cluster 2023-10-01T08:58:24 2023-10-01T08:58:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: .751558+0000 mon.a (mon.0) 181 : cluster 2023-10-01T08:58:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: [DBG] fsmap 2023-10-01T08:58:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: cluster 2023-10-01T08:58:24 2023-10-01T08:58:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: .751594+0000 mon.a (mon.0) 182 : cluster 2023-10-01T08:58:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-01T08:58:26.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: cluster 2023-10-01T08:58:24.751740 2023-10-01T08:58:26.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: +0000 mon.a (mon.0) 183 : cluster [DBG] 2023-10-01T08:58:26.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: mgrmap e13: y(active, since 45s) 2023-10-01T08:58:26.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: cluster 2023-10-01T08:58:26.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: 2023-10-01T08:58:24.751926+0000 mon.a (mon.0 2023-10-01T08:58:26.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: ) 184 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-01T08:58:26.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: cluster 2023-10-01T08: 2023-10-01T08:58:26.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: 58:24.751940+0000 mon.a (mon.0) 185 2023-10-01T08:58:26.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: : cluster [INF] Cluster is now healthy 2023-10-01T08:58:26.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: cluster 2023-10-01T08:58: 2023-10-01T08:58:26.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: 24.754217+0000 mon.a (mon.0) 186 : cluster 2023-10-01T08:58:26.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: [INF] overall HEALTH_OK 2023-10-01T08:58:26.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: audit 2023-10-01T08:58:25. 2023-10-01T08:58:26.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: 079546+0000 mon.a (mon.0) 187 : audit [INF] 2023-10-01T08:58:26.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:26.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: audit 2023-10-01T08:58:26.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: 2023-10-01T08:58:25.080192+0000 mon.a (mon.0 2023-10-01T08:58:26.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: ) 188 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T08:58:26.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: audit 2023-10-01T08: 2023-10-01T08:58:26.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: 58:25.080728+0000 mon.a (mon.0) 189 2023-10-01T08:58:26.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T08:58:26.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: audit 2023-10-01T08: 2023-10-01T08:58:26.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: 58:25.081229+0000 mon.a (mon.0) 190 2023-10-01T08:58:26.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:25 smithi190 conmon[100518]: : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:26.943 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:26 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:25.079997+0000 mgr.y (mgr.14146) 34 : cephadm [INF] 2023-10-01T08:58:26.943 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:26 smithi138 conmon[100289]: Reconfiguring mon.c (monmap changed)... 2023-10-01T08:58:26.943 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:26 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:25.081679+0000 mgr.y (mgr.14146) 35 : cephadm [INF] Reconfiguring daemon mon.c on smithi138 2023-10-01T08:58:26.943 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:26 smithi138 conmon[100289]: cluster 2023-10-01T08:58:25.310822+0000 mgr.y (mgr.14146) 36 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:26.943 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:26 smithi138 conmon[100289]: audit 2023-10-01T08:58:25.717418+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:26.943 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:26 smithi138 conmon[109959]: cephadm 2023-10-01T08:58:25.079997+0000 mgr.y ( 2023-10-01T08:58:26.943 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:26 smithi138 conmon[109959]: mgr.14146) 34 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-01T08:58:26.944 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:26 smithi138 conmon[109959]: cephadm 2023-10-01T08:58:25.081679+0000 mgr.y (mgr. 2023-10-01T08:58:26.944 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:26 smithi138 conmon[109959]: 14146) 35 : cephadm [INF] Reconfiguring daemon mon.c on smithi138 2023-10-01T08:58:26.944 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:26 smithi138 conmon[109959]: cluster 2023-10-01T08:58:26.944 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:26 smithi138 conmon[109959]: 2023-10-01T08:58:25.310822+0000 2023-10-01T08:58:26.944 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:26 smithi138 conmon[109959]: mgr.y (mgr.14146) 36 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:26.944 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:26 smithi138 conmon[109959]: audit 2023-10-01T08:58 2023-10-01T08:58:26.944 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:26 smithi138 conmon[109959]: :25.717418+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:27.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:26 smithi190 conmon[100518]: cephadm 2023-10-01T08:58:25.079997+0000 mgr.y (mgr.14146) 34 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-01T08:58:27.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:26 smithi190 conmon[100518]: cephadm 2023-10-01T08:58:25.081679+0000 mgr.y (mgr.14146) 35 : cephadm [INF] Reconfiguring daemon mon.c on smithi138 2023-10-01T08:58:27.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:26 smithi190 conmon[100518]: cluster 2023 2023-10-01T08:58:27.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:26 smithi190 conmon[100518]: -10-01T08:58:25.310822+0000 mgr.y (mgr.14146) 36 : cluster [DBG] 2023-10-01T08:58:27.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:26 smithi190 conmon[100518]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:27.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:26 smithi190 conmon[100518]: audit 2023-10-01T08:58:27.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:26 smithi190 conmon[100518]: 2023-10-01T08:58:25.717418+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T08:58:27.286 INFO:teuthology.orchestra.run.smithi138.stdout:# minimal ceph.conf for 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:58:27.286 INFO:teuthology.orchestra.run.smithi138.stdout:[global] 2023-10-01T08:58:27.286 INFO:teuthology.orchestra.run.smithi138.stdout: fsid = 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T08:58:27.286 INFO:teuthology.orchestra.run.smithi138.stdout: mon_host = [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] [v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0] [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] 2023-10-01T08:58:27.794 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:27 smithi190 conmon[100518]: audit 2023-10-01T08:58:26.683693+0000 mon.a (mon.0 2023-10-01T08:58:27.794 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:27 smithi190 conmon[100518]: ) 192 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:27.794 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:27 smithi190 conmon[100518]: cephadm 2023-10-01T08:58:26.684319+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-01T08:58:27.794 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:27 smithi190 conmon[100518]: audit 2023-10-01T08:58:26.684568+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T08:58:27.794 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:27 smithi190 conmon[100518]: audit 2023-10-01T08:58:26.685219+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T08:58:27.794 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:27 smithi190 conmon[100518]: audit 2023-10-01T08:58:26.685774+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:27.794 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:27 smithi190 conmon[100518]: cephadm 2023-10-01T08:58:26.686293+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring daemon mon.b on smithi190 2023-10-01T08:58:27.795 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:27 smithi190 conmon[100518]: audit 2023-10-01T08:58:27.286009+0000 mon.a (mon.0) 196 : audit [DBG] from='client.? 172.21.15.138:0/3658942877' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:27.891 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-01T08:58:27.891 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T08:58:27.891 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/ceph/ceph.conf 2023-10-01T08:58:27.920 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T08:58:27.921 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-01T08:58:27.984 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[100289]: audit 2023-10-01T08:58:26.683693+0000 2023-10-01T08:58:27.985 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[100289]: mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:27.985 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:26.684319+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-01T08:58:27.985 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[100289]: audit 2023-10-01T08:58:26.684568+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T08:58:27.985 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[100289]: audit 2023-10-01T08:58:26.685219+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T08:58:27.985 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[100289]: audit 2023-10-01T08:58:26.685774+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:27.985 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:26.686293+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring daemon mon.b on smithi190 2023-10-01T08:58:27.985 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[100289]: audit 2023-10-01T08:58:27.286009+0000 mon.a (mon.0) 196 : audit [DBG] from='client.? 172.21.15.138:0/3658942877' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:27.985 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[109959]: audit 2023-10-01T08:58:26.683693 2023-10-01T08:58:27.985 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[109959]: +0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:27.985 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[109959]: cephadm 2023-10-01T08:58:26.684319+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-01T08:58:27.986 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[109959]: audit 2023-10-01T08:58:26.684568+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T08:58:27.986 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[109959]: audit 2023-10-01T08:58:26.685219+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T08:58:27.986 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[109959]: audit 2023-10-01T08:58:26.685774+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:27.986 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[109959]: cephadm 2023-10-01T08:58:26.686293+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring daemon mon.b on smithi190 2023-10-01T08:58:27.986 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:27 smithi138 conmon[109959]: audit 2023-10-01T08:58:27.286009+0000 mon.a (mon.0) 196 : audit [DBG] from='client.? 172.21.15.138:0/3658942877' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:27.988 DEBUG:teuthology.orchestra.run.smithi190:> set -ex 2023-10-01T08:58:27.988 DEBUG:teuthology.orchestra.run.smithi190:> sudo dd of=/etc/ceph/ceph.conf 2023-10-01T08:58:28.017 DEBUG:teuthology.orchestra.run.smithi190:> set -ex 2023-10-01T08:58:28.018 DEBUG:teuthology.orchestra.run.smithi190:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-01T08:58:28.084 INFO:tasks.cephadm:Adding mgr.y on smithi138 2023-10-01T08:58:28.085 INFO:tasks.cephadm:Adding mgr.x on smithi190 2023-10-01T08:58:28.085 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '2;smithi138=y;smithi190=x' 2023-10-01T08:58:29.254 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:28 smithi190 conmon[100518]: cluster 2023-10-01T08:58:27.311090+0000 mgr.y (mgr.14146) 39 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:29.254 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:28 smithi190 conmon[100518]: audit 2023-10-01T08:58:27.835243+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:29.254 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:28 smithi190 conmon[100518]: audit 2023-10-01T08:58:27.836179+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:29.254 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:28 smithi190 conmon[100518]: audit 2023-10-01T08:58:27.837504+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:29.254 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:28 smithi190 conmon[100518]: audit 2023-10-01T08:58:27.838187+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:29.254 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:28 smithi190 conmon[100518]: audit 2023-10-01T08:58:27.844221+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:29.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:28 smithi138 conmon[100289]: cluster 2023-10-01T08:58:27.311090+0000 mgr.y (mgr.14146) 39 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:29.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:28 smithi138 conmon[100289]: audit 2023-10-01T08:58:27.835243+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:29.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:28 smithi138 conmon[100289]: audit 2023-10-01T08:58:27.836179+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:29.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:28 smithi138 conmon[100289]: audit 2023-10-01T08:58:27.837504+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:29.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:28 smithi138 conmon[100289]: audit 2023-10-01T08:58:27.838187+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:29.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:28 smithi138 conmon[100289]: audit 2023-10-01T08:58:27.844221+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:29.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:28 smithi138 conmon[109959]: cluster 2023-10-01T08:58:27.311090+0000 mgr.y (mgr.14146) 39 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:29.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:28 smithi138 conmon[109959]: audit 2023-10-01T08:58:27.835243+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:29.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:28 smithi138 conmon[109959]: audit 2023-10-01T08:58:27.836179+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:29.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:28 smithi138 conmon[109959]: audit 2023-10-01T08:58:27.837504+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:29.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:28 smithi138 conmon[109959]: audit 2023-10-01T08:58:27.838187+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:29.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:28 smithi138 conmon[109959]: audit 2023-10-01T08:58:27.844221+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:29.843 INFO:teuthology.orchestra.run.smithi190.stdout:Scheduled mgr update... 2023-10-01T08:58:30.367 DEBUG:teuthology.orchestra.run.smithi190:mgr.x> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-01T08:58:30.369 INFO:tasks.cephadm:Deploying OSDs... 2023-10-01T08:58:30.370 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T08:58:30.370 DEBUG:teuthology.orchestra.run.smithi138:> dd if=/scratch_devs of=/dev/stdout 2023-10-01T08:58:30.391 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-01T08:58:30.391 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_1 2023-10-01T08:58:30.400 INFO:journalctl@ceph.mgr.x.smithi190.stdout:-- Logs begin at Sun 2023-10-01 08:38:18 UTC. -- 2023-10-01T08:58:30.452 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-01T08:58:30.452 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T08:58:30.452 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 6h/6d Inode: 390449 Links: 1 2023-10-01T08:58:30.452 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T08:58:30.453 INFO:teuthology.orchestra.run.smithi138.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T08:58:30.453 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2023-10-01 08:58:17.075374683 +0000 2023-10-01T08:58:30.453 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2023-10-01 08:48:12.613387710 +0000 2023-10-01T08:58:30.453 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2023-10-01 08:48:12.613387710 +0000 2023-10-01T08:58:30.453 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: - 2023-10-01T08:58:30.453 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-01T08:58:30.521 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2023-10-01T08:58:30.521 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2023-10-01T08:58:30.521 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000259757 s, 2.0 MB/s 2023-10-01T08:58:30.522 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-01T08:58:30.583 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_2 2023-10-01T08:58:30.642 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-01T08:58:30.642 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T08:58:30.642 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 6h/6d Inode: 387449 Links: 1 2023-10-01T08:58:30.642 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T08:58:30.642 INFO:teuthology.orchestra.run.smithi138.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T08:58:30.643 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2023-10-01 08:58:17.075374683 +0000 2023-10-01T08:58:30.643 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2023-10-01 08:48:13.127377040 +0000 2023-10-01T08:58:30.643 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2023-10-01 08:48:13.127377040 +0000 2023-10-01T08:58:30.643 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: - 2023-10-01T08:58:30.643 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-01T08:58:30.710 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2023-10-01T08:58:30.710 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2023-10-01T08:58:30.711 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000235698 s, 2.2 MB/s 2023-10-01T08:58:30.712 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-01T08:58:30.771 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_3 2023-10-01T08:58:30.829 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-01T08:58:30.829 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T08:58:30.829 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 6h/6d Inode: 382923 Links: 1 2023-10-01T08:58:30.829 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T08:58:30.830 INFO:teuthology.orchestra.run.smithi138.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T08:58:30.830 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2023-10-01 08:58:17.075374683 +0000 2023-10-01T08:58:30.830 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2023-10-01 08:48:13.700365146 +0000 2023-10-01T08:58:30.830 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2023-10-01 08:48:13.700365146 +0000 2023-10-01T08:58:30.830 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: - 2023-10-01T08:58:30.830 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-01T08:58:30.897 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2023-10-01T08:58:30.897 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2023-10-01T08:58:30.897 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000251424 s, 2.0 MB/s 2023-10-01T08:58:30.898 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-01T08:58:30.946 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:30 smithi190 conmon[100518]: cluster 2023-10-01T08:58:29.311413+0000 mgr.y (mgr.14146) 40 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:30.946 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:30 smithi190 conmon[100518]: audit 2023-10-01T08:58:29.837847+0000 mgr.y (mgr.14146) 41 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi138=y;smithi190=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:58:30.946 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:30 smithi190 conmon[100518]: cephadm 2023-10-01T08:58:29.839481+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Saving service mgr spec with placement smithi138=y;smithi190=x;count:2 2023-10-01T08:58:30.946 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:30 smithi190 conmon[100518]: audit 2023-10-01T08:58:29.842944+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:30.946 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:30 smithi190 conmon[100518]: audit 2023-10-01T08:58:29.864219+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:30.946 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:30 smithi190 conmon[100518]: audit 2023-10-01T08:58:29.865839+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:30.946 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:30 smithi190 conmon[100518]: audit 2023-10-01T08:58:29.866606+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:30.946 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:30 smithi190 conmon[100518]: audit 2023-10-01T08:58:29.871888+0000 mon.a (mon.0) 206 : audit 2023-10-01T08:58:30.946 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:30 smithi190 conmon[100518]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:30.947 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:30 smithi190 conmon[100518]: audit 2023-10-01T08:58:29.873346+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T08:58:30.947 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:30 smithi190 conmon[100518]: audit 2023-10-01T08:58:29.876796+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-01T08:58:30.947 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:30 smithi190 conmon[100518]: audit 2023-10-01T08:58:29.877659+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T08:58:30.947 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:30 smithi190 conmon[100518]: audit 2023-10-01T08:58:29.878521+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:30.947 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:30 smithi190 conmon[100518]: cephadm 2023-10-01T08:58:29.879166+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Deploying daemon mgr.x on smithi190 2023-10-01T08:58:30.957 DEBUG:teuthology.orchestra.run.smithi138:> stat /dev/vg_nvme/lv_4 2023-10-01T08:58:31.015 INFO:teuthology.orchestra.run.smithi138.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-01T08:58:31.015 INFO:teuthology.orchestra.run.smithi138.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T08:58:31.015 INFO:teuthology.orchestra.run.smithi138.stdout:Device: 6h/6d Inode: 392351 Links: 1 2023-10-01T08:58:31.015 INFO:teuthology.orchestra.run.smithi138.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T08:58:31.015 INFO:teuthology.orchestra.run.smithi138.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T08:58:31.015 INFO:teuthology.orchestra.run.smithi138.stdout:Access: 2023-10-01 08:58:17.075374683 +0000 2023-10-01T08:58:31.016 INFO:teuthology.orchestra.run.smithi138.stdout:Modify: 2023-10-01 08:48:14.209354580 +0000 2023-10-01T08:58:31.016 INFO:teuthology.orchestra.run.smithi138.stdout:Change: 2023-10-01 08:48:14.209354580 +0000 2023-10-01T08:58:31.016 INFO:teuthology.orchestra.run.smithi138.stdout: Birth: - 2023-10-01T08:58:31.016 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-01T08:58:31.080 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records in 2023-10-01T08:58:31.081 INFO:teuthology.orchestra.run.smithi138.stderr:1+0 records out 2023-10-01T08:58:31.081 INFO:teuthology.orchestra.run.smithi138.stderr:512 bytes copied, 0.000204216 s, 2.5 MB/s 2023-10-01T08:58:31.082 DEBUG:teuthology.orchestra.run.smithi138:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-01T08:58:31.140 DEBUG:teuthology.orchestra.run.smithi190:> set -ex 2023-10-01T08:58:31.140 DEBUG:teuthology.orchestra.run.smithi190:> dd if=/scratch_devs of=/dev/stdout 2023-10-01T08:58:31.158 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-01T08:58:31.158 DEBUG:teuthology.orchestra.run.smithi190:> stat /dev/vg_nvme/lv_1 2023-10-01T08:58:31.215 INFO:teuthology.orchestra.run.smithi190.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-01T08:58:31.215 INFO:teuthology.orchestra.run.smithi190.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T08:58:31.216 INFO:teuthology.orchestra.run.smithi190.stdout:Device: 6h/6d Inode: 384479 Links: 1 2023-10-01T08:58:31.216 INFO:teuthology.orchestra.run.smithi190.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T08:58:31.216 INFO:teuthology.orchestra.run.smithi190.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T08:58:31.216 INFO:teuthology.orchestra.run.smithi190.stdout:Access: 2023-10-01 08:58:08.846493998 +0000 2023-10-01T08:58:31.216 INFO:teuthology.orchestra.run.smithi190.stdout:Modify: 2023-10-01 08:48:14.360123265 +0000 2023-10-01T08:58:31.216 INFO:teuthology.orchestra.run.smithi190.stdout:Change: 2023-10-01 08:48:14.360123265 +0000 2023-10-01T08:58:31.216 INFO:teuthology.orchestra.run.smithi190.stdout: Birth: - 2023-10-01T08:58:31.216 DEBUG:teuthology.orchestra.run.smithi190:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-01T08:58:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: cluster 2023-10-01T08:58:29.311413+0000 mgr.y (mgr.14146) 40 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: audit 2023- 2023-10-01T08:58:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: 10-01T08:58:29.837847+0000 mgr.y (mgr.14146) 41 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi138=y;smithi190=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:58:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:29.839481+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Saving service mgr spec with placement smithi138=y;smithi190=x;count:2 2023-10-01T08:58:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: audit 2023-10-01T08:58:29.842944+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: audit 2023-10-01T08:58:29.864219+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: audit 2023-10-01T08:58:29.865839+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: audit 2023-10-01T08:58:29.866606+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:31.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: audit 2023-10-01T08:58:29.871888+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:31.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: audit 2023-10-01T08:58:29.873346+0000 mon.a (mon.0) 2023-10-01T08:58:31.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: 207 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T08:58:31.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: audit 2023-10-01T08:58:31.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: 2023-10-01T08:58:29.876796+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-01T08:58:31.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: audit 2023-10-01T08:58:29.877659 2023-10-01T08:58:31.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: +0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T08:58:31.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: audit 2023-10-01T08:58:29.878521+0000 2023-10-01T08:58:31.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:31.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:29.879166+0000 2023-10-01T08:58:31.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[100289]: mgr.y (mgr.14146) 43 : cephadm [INF] Deploying daemon mgr.x on smithi190 2023-10-01T08:58:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: cluster 2023-10-01T08:58:29.311413+0000 mgr.y ( 2023-10-01T08:58:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: mgr.14146) 40 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: audit 2023-10-01T08:58:29 2023-10-01T08:58:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: .837847+0000 mgr.y (mgr.14146) 41 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi138=y;smithi190=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:58:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: cephadm 2023-10 2023-10-01T08:58:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: -01T08:58:29.839481+0000 mgr.y 2023-10-01T08:58:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: (mgr.14146) 42 : cephadm [INF] Saving service mgr spec with placement smithi138=y;smithi190=x;count:2 2023-10-01T08:58:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T08:58:31.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: 01T08:58:29.842944+0000 mon.a 2023-10-01T08:58:31.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:31.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: audit 2023 2023-10-01T08:58:31.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: -10-01T08:58:29. 2023-10-01T08:58:31.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: 864219+0000 mon.a (mon.0) 203 : 2023-10-01T08:58:31.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:31.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T08:58:31.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: T08:58:29.865839+0000 mon.a 2023-10-01T08:58:31.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: (mon.0) 204 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:31.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T08:58:31.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: 01T08:58:29.866606+0000 2023-10-01T08:58:31.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:31.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: 2023-10-01T08:58:31.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: audit 2023-10-01T08:58:31.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: 2023-10-01T08:58:29.871888+0000 mon.a (mon.0) 206 : audit 2023-10-01T08:58:31.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:31.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T08:58:31.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: T08:58:29.873346+0000 2023-10-01T08:58:31.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T08:58:31.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: audit 2023 2023-10-01T08:58:31.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: -10-01T08:58:29 2023-10-01T08:58:31.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: .876796+0000 mon.a (mon.0) 208 : audit 2023-10-01T08:58:31.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-01T08:58:31.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: audit 2023-10-01T08 2023-10-01T08:58:31.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: :58:29.877659+0000 mon.a (mon.0 2023-10-01T08:58:31.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: ) 209 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T08:58:31.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T08:58:31.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: 01T08:58:29.878521+0000 2023-10-01T08:58:31.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:31.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: 2023-10-01T08:58:31.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: cephadm 2023-10-01T08:58:31.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: 2023-10-01T08:58:29.879166+0000 mgr.y 2023-10-01T08:58:31.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:30 smithi138 conmon[109959]: (mgr.14146) 43 : cephadm [INF] Deploying daemon mgr.x on smithi190 2023-10-01T08:58:31.362 INFO:teuthology.orchestra.run.smithi190.stderr:1+0 records in 2023-10-01T08:58:31.362 INFO:teuthology.orchestra.run.smithi190.stderr:1+0 records out 2023-10-01T08:58:31.362 INFO:teuthology.orchestra.run.smithi190.stderr:512 bytes copied, 0.000249604 s, 2.1 MB/s 2023-10-01T08:58:31.363 DEBUG:teuthology.orchestra.run.smithi190:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-01T08:58:31.381 DEBUG:teuthology.orchestra.run.smithi190:> stat /dev/vg_nvme/lv_2 2023-10-01T08:58:31.438 INFO:teuthology.orchestra.run.smithi190.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-01T08:58:31.438 INFO:teuthology.orchestra.run.smithi190.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T08:58:31.438 INFO:teuthology.orchestra.run.smithi190.stdout:Device: 6h/6d Inode: 389796 Links: 1 2023-10-01T08:58:31.438 INFO:teuthology.orchestra.run.smithi190.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T08:58:31.438 INFO:teuthology.orchestra.run.smithi190.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T08:58:31.439 INFO:teuthology.orchestra.run.smithi190.stdout:Access: 2023-10-01 08:58:31.315868570 +0000 2023-10-01T08:58:31.439 INFO:teuthology.orchestra.run.smithi190.stdout:Modify: 2023-10-01 08:48:14.876108861 +0000 2023-10-01T08:58:31.439 INFO:teuthology.orchestra.run.smithi190.stdout:Change: 2023-10-01 08:48:14.876108861 +0000 2023-10-01T08:58:31.439 INFO:teuthology.orchestra.run.smithi190.stdout: Birth: - 2023-10-01T08:58:31.439 DEBUG:teuthology.orchestra.run.smithi190:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-01T08:58:31.506 INFO:teuthology.orchestra.run.smithi190.stderr:1+0 records in 2023-10-01T08:58:31.506 INFO:teuthology.orchestra.run.smithi190.stderr:1+0 records out 2023-10-01T08:58:31.506 INFO:teuthology.orchestra.run.smithi190.stderr:512 bytes copied, 0.000209932 s, 2.4 MB/s 2023-10-01T08:58:31.507 DEBUG:teuthology.orchestra.run.smithi190:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-01T08:58:31.556 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:31 smithi190 systemd[1]: Starting Ceph mgr.x for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T08:58:31.567 DEBUG:teuthology.orchestra.run.smithi190:> stat /dev/vg_nvme/lv_3 2023-10-01T08:58:31.625 INFO:teuthology.orchestra.run.smithi190.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-01T08:58:31.625 INFO:teuthology.orchestra.run.smithi190.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T08:58:31.626 INFO:teuthology.orchestra.run.smithi190.stdout:Device: 6h/6d Inode: 383502 Links: 1 2023-10-01T08:58:31.626 INFO:teuthology.orchestra.run.smithi190.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T08:58:31.626 INFO:teuthology.orchestra.run.smithi190.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T08:58:31.626 INFO:teuthology.orchestra.run.smithi190.stdout:Access: 2023-10-01 08:58:31.436865202 +0000 2023-10-01T08:58:31.626 INFO:teuthology.orchestra.run.smithi190.stdout:Modify: 2023-10-01 08:48:15.415093816 +0000 2023-10-01T08:58:31.626 INFO:teuthology.orchestra.run.smithi190.stdout:Change: 2023-10-01 08:48:15.415093816 +0000 2023-10-01T08:58:31.626 INFO:teuthology.orchestra.run.smithi190.stdout: Birth: - 2023-10-01T08:58:31.626 DEBUG:teuthology.orchestra.run.smithi190:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-01T08:58:31.693 INFO:teuthology.orchestra.run.smithi190.stderr:1+0 records in 2023-10-01T08:58:31.693 INFO:teuthology.orchestra.run.smithi190.stderr:1+0 records out 2023-10-01T08:58:31.693 INFO:teuthology.orchestra.run.smithi190.stderr:512 bytes copied, 0.000226023 s, 2.3 MB/s 2023-10-01T08:58:31.694 DEBUG:teuthology.orchestra.run.smithi190:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-01T08:58:31.752 DEBUG:teuthology.orchestra.run.smithi190:> stat /dev/vg_nvme/lv_4 2023-10-01T08:58:31.810 INFO:teuthology.orchestra.run.smithi190.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-01T08:58:31.810 INFO:teuthology.orchestra.run.smithi190.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T08:58:31.810 INFO:teuthology.orchestra.run.smithi190.stdout:Device: 6h/6d Inode: 392345 Links: 1 2023-10-01T08:58:31.810 INFO:teuthology.orchestra.run.smithi190.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T08:58:31.810 INFO:teuthology.orchestra.run.smithi190.stdout:Context: system_u:object_r:device_t:s0 2023-10-01T08:58:31.810 INFO:teuthology.orchestra.run.smithi190.stdout:Access: 2023-10-01 08:58:31.436865202 +0000 2023-10-01T08:58:31.810 INFO:teuthology.orchestra.run.smithi190.stdout:Modify: 2023-10-01 08:48:15.920079719 +0000 2023-10-01T08:58:31.811 INFO:teuthology.orchestra.run.smithi190.stdout:Change: 2023-10-01 08:48:15.920079719 +0000 2023-10-01T08:58:31.811 INFO:teuthology.orchestra.run.smithi190.stdout: Birth: - 2023-10-01T08:58:31.811 DEBUG:teuthology.orchestra.run.smithi190:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-01T08:58:31.875 INFO:teuthology.orchestra.run.smithi190.stderr:1+0 records in 2023-10-01T08:58:31.875 INFO:teuthology.orchestra.run.smithi190.stderr:1+0 records out 2023-10-01T08:58:31.875 INFO:teuthology.orchestra.run.smithi190.stderr:512 bytes copied, 0.000197407 s, 2.6 MB/s 2023-10-01T08:58:31.876 DEBUG:teuthology.orchestra.run.smithi190:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-01T08:58:31.935 INFO:tasks.cephadm:Deploying osd.0 on smithi138 with /dev/vg_nvme/lv_4... 2023-10-01T08:58:31.935 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-01T08:58:32.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:32 smithi190 conmon[102823]: debug 2023-10-01T08:58:32.077+0000 7f28fba2b700 1 -- 172.21.15.190:0/751896110 <== mon.1 v2:172.21.15.190:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b3371544e0 con 0x55b33712b400 2023-10-01T08:58:32.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:32 smithi190 bash[102718]: 9629bd7ec45f54a9ddbd0367d9c28a8bae70be4fdc7109c9fee6fec0174451f3 2023-10-01T08:58:32.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:32 smithi190 systemd[1]: Started Ceph mgr.x for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T08:58:32.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:32 smithi190 conmon[102823]: debug 2023-10-01T08:58:32.205+0000 7f290c010680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T08:58:32.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:32 smithi190 conmon[102823]: debug 2023-10-01T08:58:32.288+0000 7f290c010680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T08:58:33.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:32 smithi190 conmon[102823]: debug 2023-10-01T08:58:32.743+0000 7f290c010680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T08:58:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:32 smithi190 conmon[100518]: cluster 2023-10-01T08:58:31.311702+0000 mgr.y (mgr.14146) 44 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:32 smithi190 conmon[100518]: 2023-10-01T08:58:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:32 smithi190 conmon[100518]: audit 2023-10-01T08:58:32.159886+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:32 smithi190 conmon[100518]: audit 2023-10-01T08:58:32.161486+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:33.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:32 smithi138 conmon[100289]: cluster 2023-10-01T08:58:31.311702+0000 mgr.y (mgr.14146) 44 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:33.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:32 smithi138 conmon[100289]: audit 2023-10-01T08:58:32.159886+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:33.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:32 smithi138 conmon[100289]: audit 2023-10-01T08:58:32.161486+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:33.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:32 smithi138 conmon[109959]: cluster 2023-10-01T08:58:31.311702+0000 mgr.y 2023-10-01T08:58:33.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:32 smithi138 conmon[109959]: (mgr.14146) 44 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:33.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:32 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T08:58:33.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:32 smithi138 conmon[109959]: 01T08:58:32.159886+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:33.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:32 smithi138 conmon[109959]: audit 2023-10-01T08:58:33.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:32 smithi138 conmon[109959]: 2023-10-01T08:58:32. 2023-10-01T08:58:33.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:32 smithi138 conmon[109959]: 161486+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:33.806 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:33 smithi190 conmon[102823]: debug 2023-10-01T08:58:33.518+0000 7f290c010680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T08:58:33.806 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:33 smithi190 conmon[102823]: debug 2023-10-01T08:58:33.601+0000 7f290c010680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T08:58:33.997 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-01T08:58:33.998 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-01T08:58:33.998 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2023-10-01T08:58:33.998 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2023-10-01T08:58:33.998 INFO:teuthology.orchestra.run.smithi138.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0112804 s, 930 MB/s 2023-10-01T08:58:33.998 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2023-10-01T08:58:34.084 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:33 smithi190 conmon[102823]: debug 2023-10-01T08:58:33.810+0000 7f290c010680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T08:58:34.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[100289]: audit 2023-10-01T08:58:33.083562+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:34.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[100289]: audit 2023-10-01T08:58:33.084140+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:34.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[100289]: audit 2023-10-01T08:58:33.084746+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:34.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[100289]: audit 2023-10-01T08:58:33.087889+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:34.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:33.089167+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-01T08:58:34.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[100289]: audit 2023-10-01T08:58:33.089380+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T08:58:34.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[100289]: audit 2023-10-01T08:58:33.089998+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T08:58:34.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[100289]: audit 2023-10-01T08:58:33.090496+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:34.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:33.090928+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring daemon mgr.y on smithi138 2023-10-01T08:58:34.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[100289]: cluster 2023-10-01T08:58:33.311990+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:34.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[109959]: audit 2023-10-01T08:58:33.083562+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:34.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[109959]: audit 2023-10-01T08:58:33.084140+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:34.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[109959]: audit 2023-10-01T08:58:33.084746+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:34.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[109959]: audit 2023-10-01T08:58:33.087889+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:34.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[109959]: cephadm 2023-10-01T08:58:33.089167+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-01T08:58:34.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T08:58:34.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[109959]: T08:58:33.089380+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T08:58:34.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[109959]: audit 2023-10-01T08:58:33.089998+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T08:58:34.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[109959]: audit 2023-10-01T08:58:33.090496+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:34.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[109959]: cephadm 2023-10-01T08:58:33.090928+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring daemon mgr.y on smithi138 2023-10-01T08:58:34.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:34 smithi138 conmon[109959]: cluster 2023-10-01T08:58:33.311990+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:34.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: audit 2023-10-01T08:58:33.083562+0000 mon.a (mon. 2023-10-01T08:58:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: 0) 213 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: audit 2023-10-01T08:58:33.084140+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: audit 2023-10-01T08:58:33. 2023-10-01T08:58:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: 084746+0000 mon.a (mon.0) 215 : audit [INF] 2023-10-01T08:58:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: audit 2023-10-01T08: 2023-10-01T08:58:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: 58:33.087889+0000 mon.a (mon.0) 2023-10-01T08:58:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: 216 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:34.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: cephadm 2023-10-01T08:58:34.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: 2023-10-01T08:58:33.089167+0000 2023-10-01T08:58:34.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-01T08:58:34.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: 2023-10-01T08:58:34.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: audit 2023-10-01T08:58: 2023-10-01T08:58:34.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: 33.089380+0000 mon.a (mon.0) 217 : audit 2023-10-01T08:58:34.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T08:58:34.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: audit 2023-10-01T08:58:33 2023-10-01T08:58:34.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: .089998+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T08:58:34.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: audit 2023-10-01T08:58:33.090496 2023-10-01T08:58:34.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: +0000 mon.a (mon.0) 219 : audit [DBG] 2023-10-01T08:58:34.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:34.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: cephadm 2023- 2023-10-01T08:58:34.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: 10-01T08:58:33. 2023-10-01T08:58:34.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: 090928+0000 mgr.y (mgr.14146) 46 2023-10-01T08:58:34.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: : cephadm [INF] Reconfiguring daemon mgr.y on smithi138 2023-10-01T08:58:34.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: cluster 2023-10 2023-10-01T08:58:34.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: -01T08:58:33.311990+0000 mgr.y ( 2023-10-01T08:58:34.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[100518]: mgr.14146) 47 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:34.955 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi138:vg_nvme/lv_4 2023-10-01T08:58:35.040 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[102823]: debug 2023-10-01T08:58:34.746+0000 7f290c010680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T08:58:35.040 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[102823]: debug 2023-10-01T08:58:34.873+0000 7f290c010680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T08:58:35.041 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:34 smithi190 conmon[102823]: debug 2023-10-01T08:58:34.950+0000 7f290c010680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T08:58:35.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[102823]: debug 2023-10-01T08:58:35.039+0000 7f290c010680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T08:58:35.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[102823]: debug 2023-10-01T08:58:35.233+0000 7f290c010680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T08:58:35.806 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[102823]: debug 2023-10-01T08:58:35.508+0000 7f290c010680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T08:58:35.807 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[102823]: debug 2023-10-01T08:58:35.605+0000 7f290c010680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T08:58:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[100518]: audit 2023-10-01T08:58:34.446664+0000 mon.a (mon.0 2023-10-01T08:58:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[100518]: ) 220 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[100518]: audit 2023-10-01T08:58:34.847268+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[100518]: audit 2023-10-01T08:58:34.848170+0000 mon.a (mon. 2023-10-01T08:58:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[100518]: 0) 222 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T08:58:35.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[100518]: 08:58:34.849353+0000 mon.a (mon.0 2023-10-01T08:58:35.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[100518]: ) 223 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:35.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T08:58:35.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[100518]: 08:58:34.849882+0000 mon.a (mon.0 2023-10-01T08:58:35.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[100518]: ) 224 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:35.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[100518]: audit 2023-10-01T08 2023-10-01T08:58:35.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[100518]: :58:34.853242+0000 mon.a (mon.0) 2023-10-01T08:58:35.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:35 smithi190 conmon[100518]: 225 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:35.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[100289]: audit 2023-10-01T08:58:34.446664+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[100289]: audit 2023-10-01T08:58:34.847268+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[100289]: audit 2023-10-01T08:58:34.848170+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[100289]: audit 2023-10-01T08:58:34.849353+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T08:58:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[100289]: 01T08:58:34.849882+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[100289]: audit 2023-10-01T08:58:34.853242+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:35.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: audit 2023-10-01T08:58:34. 2023-10-01T08:58:35.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: 446664+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T08:58:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: 08:58:34.847268+0000 mon.a (mon 2023-10-01T08:58:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: .0) 221 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: 2023-10-01T08:58:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T08:58:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: 01T08:58:34.848170+0000 mon.a (mon. 2023-10-01T08:58:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: 0) 222 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T08:58:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: T08:58:34.849353+0000 mon.a (mon. 2023-10-01T08:58:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: 0) 223 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: audit 2023- 2023-10-01T08:58:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: 10-01T08:58:34.849882+0000 2023-10-01T08:58:35.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: mon.a (mon.0) 224 : audit 2023-10-01T08:58:35.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:35.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: audit 2023- 2023-10-01T08:58:35.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: 10-01T08:58:34.853242+0000 mon.a (mon.0) 2023-10-01T08:58:35.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:35 smithi138 conmon[109959]: 225 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:36.665 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:36 smithi190 conmon[102823]: debug 2023-10-01T08:58:36.323+0000 7f290c010680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T08:58:36.666 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:36 smithi190 conmon[102823]: debug 2023-10-01T08:58:36.404+0000 7f290c010680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T08:58:36.666 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:36 smithi190 conmon[102823]: debug 2023-10-01T08:58:36.494+0000 7f290c010680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T08:58:36.925 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:36 smithi190 conmon[100518]: cluster 2023-10-01T08:58:35.312431+0000 mgr.y (mgr. 2023-10-01T08:58:36.925 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:36 smithi190 conmon[100518]: 14146) 48 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:36.926 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:36 smithi190 conmon[102823]: debug 2023-10-01T08:58:36.664+0000 7f290c010680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T08:58:37.184 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:36 smithi190 conmon[102823]: debug 2023-10-01T08:58:36.924+0000 7f290c010680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T08:58:37.184 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[102823]: debug 2023-10-01T08:58:37.053+0000 7f290c010680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T08:58:37.184 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[102823]: debug 2023-10-01T08:58:37.183+0000 7f290c010680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T08:58:37.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:36 smithi138 conmon[100289]: cluster 2023- 2023-10-01T08:58:37.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:36 smithi138 conmon[100289]: 10-01T08:58:35.312431+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:37.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:36 smithi138 conmon[109959]: cluster 2023-10-01T08:58:35.312431+0000 mgr.y (mgr 2023-10-01T08:58:37.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:36 smithi138 conmon[109959]: .14146) 48 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:37.434 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[102823]: debug 2023-10-01T08:58:37.352+0000 7f290c010680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T08:58:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[100289]: audit 2023-10-01T08:58:36.895666+0000 mgr.y (mgr.14146) 49 : audit [DBG] from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:58:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[100289]: audit 2023-10-01T08:58:36.897847+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T08:58:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[100289]: audit 2023-10-01T08:58:36.899698+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T08:58:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[100289]: audit 2023-10-01T08:58:36.900213+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[100289]: cluster 2023-10-01T08:58:37.449084+0000 mon.a (mon.0) 229 : cluster [DBG] Standby manager daemon x started 2023-10-01T08:58:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[100289]: audit 2023-10-01T08:58:37.449851+0000 mon.c (mon.2) 3 : audit [DBG] from='mgr.? 172.21.15.190:0/1879622934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T08:58:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[100289]: audit 2023-10-01T08:58:37.450248+0000 mon.c (mon.2) 4 : audit [DBG] from='mgr.? 172.21.15.190:0/1879622934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T08:58:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[100289]: audit 2023-10-01T08:58:37.451227+0000 mon.c (mon.2) 5 : audit [DBG] from='mgr.? 172.21.15.190:0/1879622934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T08:58:38.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[100289]: audit 2023-10-01T08:58:37.451540+0000 mon.c (mon.2) 6 : audit [DBG] from='mgr.? 172.21.15.190:0/1879622934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T08:58:38.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: audit 2023-10-01T08:58:36.895666+0000 mgr.y 2023-10-01T08:58:38.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: (mgr.14146) 49 : audit [DBG] from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:58:38.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: audit 2023-10-01T08:58:36.897847+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T08:58:38.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: audit 2023-10-01T08:58:36.899698+0000 mon.a (mon.0) 227 : audit [INF] 2023-10-01T08:58:38.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T08:58:38.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: audit 2023-10 2023-10-01T08:58:38.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: -01T08:58:36.900213+0000 mon.a (mon 2023-10-01T08:58:38.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: .0) 228 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:38.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: cluster 2023- 2023-10-01T08:58:38.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: 10-01T08:58:37.449084 2023-10-01T08:58:38.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: +0000 mon.a (mon.0) 229 : 2023-10-01T08:58:38.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: cluster [DBG] Standby manager daemon x started 2023-10-01T08:58:38.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: audit 2023 2023-10-01T08:58:38.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: -10-01T08:58:37.449851+0000 2023-10-01T08:58:38.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: mon.c (mon.2) 3 : audit [DBG] 2023-10-01T08:58:38.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: from='mgr.? 172.21.15.190:0/1879622934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T08:58:38.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: audit 2023-10-01T08:58 2023-10-01T08:58:38.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: :37.450248+0000 mon.c (mon.2) 4 : 2023-10-01T08:58:38.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: audit [DBG] from='mgr.? 172.21.15.190:0/1879622934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T08:58:38.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: audit 2023-10-01T08:58:38.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: 2023-10-01T08:58:37.451227 2023-10-01T08:58:38.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: +0000 mon.c (mon.2) 5 : audit [DBG] 2023-10-01T08:58:38.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: from='mgr.? 172.21.15.190:0/1879622934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T08:58:38.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: audit 2023-10-01T08:58:38.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: 2023-10-01T08:58:37.451540+0000 2023-10-01T08:58:38.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: mon.c (mon.2) 6 : audit [DBG] from='mgr.? 172.21.15.190:0/1879622934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T08:58:38.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:37 smithi138 conmon[109959]: 2023-10-01T08:58:38.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[102823]: debug 2023-10-01T08:58:37.433+0000 7f290c010680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T08:58:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: audit 2023-10-01T08:58:36.895666+0000 mgr.y (mgr.14146) 2023-10-01T08:58:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: 49 : audit [DBG] from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:58:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: audit 2023-10-01T08:58:36.897847+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T08:58:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: audit 2023-10-01T08:58:36.899698+0000 mon.a (mon.0) 227 : audit [INF] 2023-10-01T08:58:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T08:58:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: audit 2023-10-01T08:58:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: 2023-10-01T08:58:36.900213+0000 mon.a (mon.0) 2023-10-01T08:58:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: 228 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:38.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: cluster 2023-10-01T08 2023-10-01T08:58:38.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: :58:37.449084+0000 mon.a (mon.0) 2023-10-01T08:58:38.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: 229 : cluster [DBG] Standby manager daemon x started 2023-10-01T08:58:38.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: audit 2023-10-01T08: 2023-10-01T08:58:38.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: 58:37.449851+0000 mon.c (mon.2) 3 2023-10-01T08:58:38.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: : audit [DBG] from='mgr.? 172.21.15.190:0/1879622934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T08:58:38.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: audit 2023-10-01T08:58 2023-10-01T08:58:38.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: :37.450248+0000 mon.c (mon.2) 4 : 2023-10-01T08:58:38.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: audit [DBG] from='mgr.? 172.21.15.190:0/1879622934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T08:58:38.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: audit 2023-10-01T08:58: 2023-10-01T08:58:38.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: 37.451227+0000 mon.c (mon.2) 5 : audit 2023-10-01T08:58:38.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: [DBG] from='mgr.? 172.21.15.190:0/1879622934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T08:58:38.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: audit 2023-10-01T08:58: 2023-10-01T08:58:38.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: 37.451540+0000 mon.c (mon.2) 6 : audit 2023-10-01T08:58:38.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:37 smithi190 conmon[100518]: [DBG] from='mgr.? 172.21.15.190:0/1879622934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T08:58:39.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:38 smithi190 conmon[100518]: cluster 2023-10-01T08:58:37.312676+0000 mgr.y (mgr.14146) 50 2023-10-01T08:58:39.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:38 smithi190 conmon[100518]: : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:39.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:38 smithi190 conmon[100518]: cluster 2023-10-01T08:58:37.863252+0000 mon.a (mon.0) 230 : cluster [DBG] mgrmap e14: y(active, since 58s), standbys: x 2023-10-01T08:58:39.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:38 smithi190 conmon[100518]: audit 2023-10-01T08:58:37.863366+0000 mon.a (mon.0) 231 : audit 2023-10-01T08:58:39.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:38 smithi190 conmon[100518]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T08:58:39.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:38 smithi138 conmon[100289]: cluster 2023-10-01T08:58:37.312676+0000 mgr.y ( 2023-10-01T08:58:39.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:38 smithi138 conmon[100289]: mgr.14146) 50 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:39.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:38 smithi138 conmon[100289]: cluster 2023-10-01T08:58:37.863252+0000 mon.a (mon.0) 230 : cluster [DBG] mgrmap e14: y(active, since 58s), standbys: x 2023-10-01T08:58:39.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:38 smithi138 conmon[100289]: audit 2023-10-01T08:58:37.863366+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T08:58:39.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:38 smithi138 conmon[109959]: cluster 2023-10-01T08:58:37.312676+0000 mgr.y (mgr.14146) 50 2023-10-01T08:58:39.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:38 smithi138 conmon[109959]: : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:39.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:38 smithi138 conmon[109959]: cluster 2023-10-01T08:58:37.863252+0000 mon.a (mon.0) 230 : cluster [DBG] mgrmap e14: y(active, since 58s), standbys: x 2023-10-01T08:58:39.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:38 smithi138 conmon[109959]: audit 2023-10-01T08:58:37.863366+0000 mon.a (mon.0) 231 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T08:58:40.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:39 smithi190 conmon[100518]: audit 2023-10-01T08:58:39.067558+0000 mon.c (mon.2) 7 2023-10-01T08:58:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:39 smithi190 conmon[100518]: : audit [INF] from='client.? 172.21.15.138:0/3371128728' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12f71e77-6099-48ee-a89e-e931e000e9fa"}]: dispatch 2023-10-01T08:58:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:39 smithi190 conmon[100518]: audit 2023-10-01T08:58:39.067890+0000 mon.a (mon.0) 232 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12f71e77-6099-48ee-a89e-e931e000e9fa"}]: dispatch 2023-10-01T08:58:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:39 smithi190 conmon[100518]: audit 2023-10-01T08:58:39 2023-10-01T08:58:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:39 smithi190 conmon[100518]: .071663+0000 mon.a (mon.0) 233 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "12f71e77-6099-48ee-a89e-e931e000e9fa"}]': finished 2023-10-01T08:58:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:39 smithi190 conmon[100518]: cluster 2023-10-01T08: 2023-10-01T08:58:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:39 smithi190 conmon[100518]: 58:39.071742+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-01T08:58:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:39 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T08:58:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:39 smithi190 conmon[100518]: 01T08:58:39.071864+0000 mon.a (mon.0 2023-10-01T08:58:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:39 smithi190 conmon[100518]: ) 235 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:39 smithi190 conmon[100518]: audit 2023-10-01T08:58 2023-10-01T08:58:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:39 smithi190 conmon[100518]: :39.695391+0000 mon.a (mon.0) 236 : audit 2023-10-01T08:58:40.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:39 smithi190 conmon[100518]: [DBG] from='client.? 172.21.15.138:0/4022461951' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T08:58:40.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[100289]: audit 2023-10-01T08:58:39.067558+0000 mon.c (mon.2) 7 : audit 2023-10-01T08:58:40.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[100289]: [INF] from='client.? 172.21.15.138:0/3371128728' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12f71e77-6099-48ee-a89e-e931e000e9fa"}]: dispatch 2023-10-01T08:58:40.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[100289]: audit 2023-10-01T08:58:39.067890+0000 mon.a (mon.0) 232 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12f71e77-6099-48ee-a89e-e931e000e9fa"}]: dispatch 2023-10-01T08:58:40.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[100289]: audit 2023-10-01T08:58:39.071663+0000 mon.a (mon.0) 233 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "12f71e77-6099-48ee-a89e-e931e000e9fa"}]': finished 2023-10-01T08:58:40.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[100289]: cluster 2023-10-01T08:58:39.071742+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-01T08:58:40.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[100289]: audit 2023-10-01T08:58:39.071864+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:40.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T08:58:40.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[100289]: T08:58:39.695391+0000 mon.a (mon. 2023-10-01T08:58:40.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[100289]: 0) 236 : audit [DBG] from='client.? 172.21.15.138:0/4022461951' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T08:58:40.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: audit 2023-10-01T08:58:39.067558+0000 mon.c (mon.2 2023-10-01T08:58:40.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: ) 7 : audit [INF] from='client.? 172.21.15.138:0/3371128728' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12f71e77-6099-48ee-a89e-e931e000e9fa"}]: dispatch 2023-10-01T08:58:40.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T08:58:40.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: T08:58:39.067890+0000 mon.a (mon.0) 232 : audit [INF] 2023-10-01T08:58:40.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "12f71e77-6099-48ee-a89e-e931e000e9fa"}]: dispatch 2023-10-01T08:58:40.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T08:58:40.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: 08:58:39.071663+0000 mon.a (mon 2023-10-01T08:58:40.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: .0) 233 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "12f71e77-6099-48ee-a89e-e931e000e9fa"}]': finished 2023-10-01T08:58:40.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: 2023-10-01T08:58:40.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: cluster 2023-10-01T08: 2023-10-01T08:58:40.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: 58:39.071742+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-01T08:58:40.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T08:58:40.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: 01T08:58:39.071864+0000 mon.a (mon.0) 235 2023-10-01T08:58:40.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:40.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: audit 2023-10-01T08:58:39.695391+0000 2023-10-01T08:58:40.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:39 smithi138 conmon[109959]: mon.a (mon.0) 236 : audit [DBG] from='client.? 172.21.15.138:0/4022461951' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T08:58:41.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:40 smithi190 conmon[100518]: cluster 2023-10-01T08:58:39.313000+0000 mgr.y (mgr.14146) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:41.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:40 smithi138 conmon[100289]: cluster 2023-10-01T08:58:39. 2023-10-01T08:58:41.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:40 smithi138 conmon[100289]: 313000+0000 mgr.y (mgr.14146) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:41.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:40 smithi138 conmon[109959]: cluster 2023-10-01T08:58:39.313000+0000 2023-10-01T08:58:41.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:40 smithi138 conmon[109959]: mgr.y (mgr.14146) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:43.147 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:42 smithi138 conmon[100289]: cluster 2023-10-01T08:58:41.313304+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:43.148 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:42 smithi138 conmon[109959]: cluster 2023-10-01T08:58:41.313304+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:43.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:42 smithi190 conmon[100518]: cluster 2023-10-01T08:58:41.313304+0000 mgr.y (mgr. 2023-10-01T08:58:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:42 smithi190 conmon[100518]: 14146) 52 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:45.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:44 smithi138 conmon[100289]: cluster 2023-10-01T08:58:43.313623+0000 mgr.y ( 2023-10-01T08:58:45.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:44 smithi138 conmon[100289]: mgr.14146) 53 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:45.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:44 smithi138 conmon[109959]: cluster 2023-10-01T08:58:43.313623+0000 mgr.y 2023-10-01T08:58:45.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:44 smithi138 conmon[109959]: (mgr.14146) 53 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:45.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:44 smithi190 conmon[100518]: cluster 2023-10-01T08:58:43.313623+0000 mgr.y (mgr.14146) 53 2023-10-01T08:58:45.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:44 smithi190 conmon[100518]: : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:46.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:45 smithi138 conmon[109959]: audit 2023-10-01T08:58:45.236095+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T08:58:46.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:45 smithi138 conmon[109959]: audit 2023-10-01T08:58:45.236729+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:46.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:45 smithi138 conmon[100289]: audit 2023-10-01T08:58:45.236095+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T08:58:46.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:45 smithi138 conmon[100289]: audit 2023-10-01T08:58:45.236729+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:46.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:45 smithi190 conmon[100518]: audit 2023-10-01T08:58:45.236095+0000 mon.a (mon. 2023-10-01T08:58:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:45 smithi190 conmon[100518]: 0) 237 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T08:58:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:45 smithi190 conmon[100518]: audit 2023-10-01T08:58:45.236729+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:47.239 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:46 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:45.237286+0000 mgr.y (mgr.14146) 54 : cephadm 2023-10-01T08:58:47.239 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:46 smithi138 conmon[100289]: [INF] Deploying daemon osd.0 on smithi138 2023-10-01T08:58:47.239 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:46 smithi138 conmon[100289]: cluster 2023-10-01T08:58:45.313853+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:47.239 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:46 smithi138 conmon[109959]: cephadm 2023-10-01T08:58:45.237286 2023-10-01T08:58:47.239 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:46 smithi138 conmon[109959]: +0000 mgr.y (mgr.14146) 54 : cephadm [INF] Deploying daemon osd.0 on smithi138 2023-10-01T08:58:47.239 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:46 smithi138 conmon[109959]: cluster 2023-10- 2023-10-01T08:58:47.240 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:46 smithi138 conmon[109959]: 01T08:58:45.313853+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:47.240 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:46 smithi138 conmon[109959]: 2023-10-01T08:58:47.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:46 smithi190 conmon[100518]: cephadm 2023-10-01T08:58:45.237286+0000 mgr.y 2023-10-01T08:58:47.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:46 smithi190 conmon[100518]: (mgr.14146) 54 : cephadm [INF] Deploying daemon osd.0 on smithi138 2023-10-01T08:58:47.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:46 smithi190 conmon[100518]: cluster 2023-10-01T08:58:45.313853+0000 mgr.y (mgr.14146) 55 : cluster [DBG] 2023-10-01T08:58:47.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:46 smithi190 conmon[100518]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:49.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:48 smithi190 conmon[100518]: cluster 2023-10-01T08:58:47.314120+0000 mgr.y (mgr.14146) 56 2023-10-01T08:58:49.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:48 smithi190 conmon[100518]: : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:49.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:48 smithi138 conmon[109959]: cluster 2023-10-01T08:58:47.314120+0000 mgr.y (mgr.14146) 2023-10-01T08:58:49.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:48 smithi138 conmon[109959]: 56 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:48 smithi138 conmon[100289]: cluster 2023-10-01T08:58:47.314120+0000 mgr.y (mgr. 2023-10-01T08:58:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:48 smithi138 conmon[100289]: 14146) 56 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:51.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:50 smithi138 conmon[100289]: cluster 2023-10-01T08:58:49.314663+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:51.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:50 smithi138 conmon[100289]: audit 2023-10-01T08:58:50.038756+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:51.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:50 smithi138 conmon[100289]: audit 2023-10-01T08:58:50.047116+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:51.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:50 smithi138 conmon[109959]: cluster 2023-10-01T08:58:49.314663+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:51.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:50 smithi138 conmon[109959]: audit 2023-10-01T08:58:50.038756+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:51.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:50 smithi138 conmon[109959]: audit 2023-10-01T08:58:50.047116+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:51.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:50 smithi190 conmon[100518]: cluster 2023-10-01T08:58:49.314663+0000 mgr.y (mgr.14146) 2023-10-01T08:58:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:50 smithi190 conmon[100518]: 57 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:50 smithi190 conmon[100518]: audit 2023-10-01T08:58:50.038756+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:50 smithi190 conmon[100518]: audit 2023-10-01T08:58:50.047116+0000 mon.a (mon.0) 240 : audit [DBG] 2023-10-01T08:58:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:50 smithi190 conmon[100518]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:53.139 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 0 on host 'smithi138' 2023-10-01T08:58:53.290 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:52 smithi138 conmon[100289]: cluster 2023-10-01T08:58:51.315002+0000 mgr.y (mgr.14146) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:53.290 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:52 smithi138 conmon[100289]: audit 2023-10-01T08:58:51.946678+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:53.290 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:52 smithi138 conmon[100289]: audit 2023-10-01T08:58:51.947304+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:53.290 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:52 smithi138 conmon[100289]: audit 2023-10-01T08:58:51.947910+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:53.291 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:52 smithi138 conmon[100289]: audit 2023-10-01T08:58:53.291 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:52 smithi138 conmon[100289]: 2023-10-01T08:58:51.952196+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:53.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:52 smithi138 conmon[109959]: cluster 2023-10-01T08:58:51.315002+0000 mgr.y (mgr.14146) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:53.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:52 smithi138 conmon[109959]: audit 2023-10-01T08:58:51.946678+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:53.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:52 smithi138 conmon[109959]: audit 2023-10-01T08:58:51.947304+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:53.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:52 smithi138 conmon[109959]: audit 2023-10-01T08:58:51.947910+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:53.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:52 smithi138 conmon[109959]: audit 2023-10-01T08:58:51.952196+0000 mon.a 2023-10-01T08:58:53.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:52 smithi138 conmon[109959]: (mon.0) 244 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:53.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:52 smithi190 conmon[100518]: cluster 2023-10-01T08:58:51.315002+0000 mgr.y (mgr.14146) 2023-10-01T08:58:53.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:52 smithi190 conmon[100518]: 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:52 smithi190 conmon[100518]: audit 2023-10-01T08:58:51.946678+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:52 smithi190 conmon[100518]: audit 2023-10-01T08:58:51.947304+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:52 smithi190 conmon[100518]: 2023-10-01T08:58:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:52 smithi190 conmon[100518]: audit 2023-10-01T08:58:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:52 smithi190 conmon[100518]: 2023-10-01T08:58:51.947910+0000 mon.a (mon.0) 243 2023-10-01T08:58:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:52 smithi190 conmon[100518]: : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:52 smithi190 conmon[100518]: audit 2023-10-01T08:58:51 2023-10-01T08:58:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:52 smithi190 conmon[100518]: .952196+0000 mon.a (mon.0) 244 : audit [INF] 2023-10-01T08:58:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:52 smithi190 conmon[100518]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:53.752 DEBUG:teuthology.orchestra.run.smithi138:osd.0> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-01T08:58:53.754 INFO:tasks.cephadm:Deploying osd.1 on smithi138 with /dev/vg_nvme/lv_3... 2023-10-01T08:58:53.754 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-01T08:58:53.784 INFO:journalctl@ceph.osd.0.smithi138.stdout:-- Logs begin at Sun 2023-10-01 08:38:26 UTC. -- 2023-10-01T08:58:54.521 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[100289]: audit 2023-10-01T08:58:53.138261+0000 2023-10-01T08:58:54.521 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[100289]: mon.a (mon.0) 245 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:54.521 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[100289]: audit 2023-10-01T08:58:53.170126+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:54.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[100289]: audit 2023-10-01T08:58:53.293434+0000 mon.c (mon.2) 8 : audit [INF] from='osd.0 [v2:172.21.15.138:6802/635244462,v1:172.21.15.138:6803/635244462]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T08:58:54.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[100289]: audit 2023-10-01T08:58:53.293770+0000 mon.a (mon.0) 247 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T08:58:54.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[100289]: cluster 2023-10-01T08:58:53.315306+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:54.522 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[109959]: audit 2023-10-01T08:58:53.138261+0000 mon.a (mon 2023-10-01T08:58:54.522 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[109959]: .0) 245 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:54.522 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[109959]: audit 2023-10-01T08:58:53.170126+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:54.522 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[109959]: audit 2023-10-01T08 2023-10-01T08:58:54.522 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[109959]: :58:53.293434+0000 mon.c (mon.2) 8 2023-10-01T08:58:54.522 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[109959]: : audit [INF] from='osd.0 [v2:172.21.15.138:6802/635244462,v1:172.21.15.138:6803/635244462]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T08:58:54.522 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T08:58:54.523 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[109959]: 08:58:53.293770+0000 mon.a (mon.0 2023-10-01T08:58:54.523 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[109959]: ) 247 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T08:58:54.523 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[109959]: cluster 2023 2023-10-01T08:58:54.523 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[109959]: -10-01T08:58:53.315306+0000 2023-10-01T08:58:54.523 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:54 smithi138 conmon[109959]: mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:54.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:54 smithi190 conmon[100518]: audit 2023-10-01T08:58:53.138261+0000 mon.a (mon. 2023-10-01T08:58:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:54 smithi190 conmon[100518]: 0) 245 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:54 smithi190 conmon[100518]: audit 2023-10-01T08:58:53.170126+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:58:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:54 smithi190 conmon[100518]: audit 2023-10-01T08:58:53.293434+0000 mon.c (mon.2) 8 : audit [INF] 2023-10-01T08:58:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:54 smithi190 conmon[100518]: from='osd.0 [v2:172.21.15.138:6802/635244462,v1:172.21.15.138:6803/635244462]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T08:58:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:54 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T08:58:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:54 smithi190 conmon[100518]: 08:58:53.293770+0000 mon.a (mon.0) 247 2023-10-01T08:58:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:54 smithi190 conmon[100518]: : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T08:58:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:54 smithi190 conmon[100518]: cluster 2023-10-01T08:58: 2023-10-01T08:58:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:54 smithi190 conmon[100518]: 53.315306+0000 mgr.y (mgr.14146) 59 : cluster [DBG] 2023-10-01T08:58:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:54 smithi190 conmon[100518]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:55.148 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:55 smithi138 conmon[109959]: audit 2023-10-01T08:58:54.147838+0000 mon.a (mon.0) 248 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T08:58:55.526 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 08:58:55 smithi138 conmon[115078]: debug 2023-10-01T08:58:55.149+0000 7fbf800cc700 -1 osd.0 0 waiting for initial osdmap 2023-10-01T08:58:55.526 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 08:58:55 smithi138 conmon[115078]: debug 2023-10-01T08:58:55.151+0000 7fbf78840700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T08:58:55.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:55 smithi138 conmon[100289]: audit 2023-10-01T08:58:54.147838+0000 mon.a (mon.0) 248 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T08:58:55.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:55 smithi138 conmon[100289]: cluster 2023-10-01T08:58:54.147949+0000 mon.a (mon.0) 249 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-01T08:58:55.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:55 smithi138 conmon[100289]: audit 2023-10-01T08:58:54.148064+0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:55.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:55 smithi138 conmon[100289]: audit 2023-10-01T08:58:54.150486+0000 mon.c ( 2023-10-01T08:58:55.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:55 smithi138 conmon[100289]: mon.2) 9 : audit [INF] from='osd.0 [v2:172.21.15.138:6802/635244462,v1:172.21.15.138:6803/635244462]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:58:55.527 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:55 smithi138 conmon[100289]: audit 2023-10-01T08:58:54.150792+0000 mon.a (mon.0) 251 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:58:55.527 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:55 smithi138 conmon[109959]: cluster 2023-10-01T08:58:54.147949+0000 mon.a (mon.0) 249 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-01T08:58:55.527 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:55 smithi138 conmon[109959]: audit 2023-10-01T08:58:54.148064+0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:55.527 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:55 smithi138 conmon[109959]: audit 2023-10-01T08:58:54.150486+0000 mon.c (mon.2) 9 : audit [INF] from='osd.0 [v2:172.21.15.138:6802/635244462,v1:172.21.15.138:6803/635244462]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:58:55.527 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:55 smithi138 conmon[109959]: audit 2023-10-01T08:58:54.150792+0000 mon.a (mon.0) 251 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:58:55.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:55 smithi190 conmon[100518]: audit 2023-10-01T08:58:54.147838+0000 mon.a (mon.0) 248 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T08:58:55.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:55 smithi190 conmon[100518]: cluster 2023-10-01T08:58:54.147949+0000 mon.a (mon.0) 249 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-01T08:58:55.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:55 smithi190 conmon[100518]: audit 2023-10-01T08:58:54.148064+0000 mon.a (mon. 2023-10-01T08:58:55.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:55 smithi190 conmon[100518]: 0) 250 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:55.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:55 smithi190 conmon[100518]: audit 2023-10-01T08 2023-10-01T08:58:55.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:55 smithi190 conmon[100518]: :58:54.150486+0000 mon.c (mon.2) 2023-10-01T08:58:55.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:55 smithi190 conmon[100518]: 9 : audit [INF] from='osd.0 [v2:172.21.15.138:6802/635244462,v1:172.21.15.138:6803/635244462]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:58:55.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:55 smithi190 conmon[100518]: audit 2023-10-01T08:58 2023-10-01T08:58:55.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:55 smithi190 conmon[100518]: :54.150792+0000 mon.a (mon.0) 251 : 2023-10-01T08:58:55.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:55 smithi190 conmon[100518]: audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:58:56.449 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:56 smithi138 conmon[109959]: audit 2023-10-01T08:58:55.146511 2023-10-01T08:58:56.449 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:56 smithi138 conmon[109959]: +0000 mon.a (mon.0) 252 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-10-01T08:58:56.449 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:56 smithi138 conmon[109959]: cluster 2023-10-01T08:58:55.146572+0000 mon.a (mon.0) 253 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-01T08:58:56.449 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:56 smithi138 conmon[109959]: audit 2023-10-01T08:58:55.146876+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:56.450 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:56 smithi138 conmon[109959]: audit 2023-10-01T08:58:55.150728+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:56.450 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:56 smithi138 conmon[109959]: cluster 2023-10-01T08:58:55.315565+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:56.450 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:56 smithi138 conmon[100289]: audit 2023-10-01T08:58:55.146511+0000 mon.a (mon.0) 252 2023-10-01T08:58:56.450 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:56 smithi138 conmon[100289]: : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-10-01T08:58:56.450 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:56 smithi138 conmon[100289]: cluster 2023-10-01T08:58:55.146572+0000 mon.a (mon.0) 253 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-01T08:58:56.450 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:56 smithi138 conmon[100289]: audit 2023-10-01T08:58:55.146876+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:56.450 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:56 smithi138 conmon[100289]: audit 2023-10-01T08:58:55.150728+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:56.450 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:56 smithi138 conmon[100289]: cluster 2023-10-01T08:58:55.315565+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:56.503 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-01T08:58:56.504 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-01T08:58:56.504 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2023-10-01T08:58:56.504 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2023-10-01T08:58:56.504 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0133638 s, 785 MB/s 2023-10-01T08:58:56.504 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2023-10-01T08:58:56.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:56 smithi190 conmon[100518]: audit 2023-10-01T08:58:55.146511+0000 mon.a (mon.0) 252 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-10-01T08:58:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:56 smithi190 conmon[100518]: cluster 2023-10-01T08:58:55.146572+0000 mon.a (mon.0) 253 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-01T08:58:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:56 smithi190 conmon[100518]: audit 2023-10-01T08:58:55.146876+0000 mon.a (mon.0) 254 : audit 2023-10-01T08:58:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:56 smithi190 conmon[100518]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:56 smithi190 conmon[100518]: audit 2023-10-01T08:58:55.150728 2023-10-01T08:58:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:56 smithi190 conmon[100518]: +0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:56 smithi190 conmon[100518]: cluster 2023-10-01T08:58:55.315565+0000 mgr.y (mgr. 2023-10-01T08:58:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:56 smithi190 conmon[100518]: 14146) 60 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T08:58:57.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:57 smithi138 conmon[100289]: cluster 2023-10-01T08:58:54.313549+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T08:58:57.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:57 smithi138 conmon[100289]: cluster 2023-10-01T08:58:54.313616+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T08:58:57.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:57 smithi138 conmon[100289]: cluster 2023-10-01T08:58:56.151953+0000 mon.a (mon.0) 256 : cluster [INF] osd.0 [v2:172.21.15.138:6802/635244462,v1:172.21.15.138:6803/635244462] boot 2023-10-01T08:58:57.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:57 smithi138 conmon[100289]: cluster 2023-10-01T08:58:56.151999+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-01T08:58:57.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:57 smithi138 conmon[100289]: audit 2023-10-01T08:58:56.152381+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:57.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:57 smithi138 conmon[109959]: cluster 2023-10-01T08:58:54.313549+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T08:58:57.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:57 smithi138 conmon[109959]: cluster 2023-10-01T08:58:54.313616+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T08:58:57.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:57 smithi138 conmon[109959]: cluster 2023-10-01T08:58:56.151953+0000 mon.a (mon.0) 256 : cluster [INF] osd.0 [v2:172.21.15.138:6802/635244462,v1:172.21.15.138:6803/635244462] boot 2023-10-01T08:58:57.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:57 smithi138 conmon[109959]: cluster 2023-10-01T08:58:56.151999+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-01T08:58:57.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:57 smithi138 conmon[109959]: audit 2023-10-01T08:58:56.152381+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:57.394 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi138:vg_nvme/lv_3 2023-10-01T08:58:57.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:57 smithi190 conmon[100518]: cluster 2023-10-01T08:58:54.313549+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T08:58:57.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:57 smithi190 conmon[100518]: cluster 2023-10-01T08:58:54.313616+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T08:58:57.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:57 smithi190 conmon[100518]: cluster 2023 2023-10-01T08:58:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:57 smithi190 conmon[100518]: -10-01T08:58:56.151953+0000 mon.a (mon.0) 256 : cluster [INF] osd.0 [v2:172.21.15.138:6802/635244462,v1:172.21.15.138:6803/635244462] boot 2023-10-01T08:58:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:57 smithi190 conmon[100518]: cluster 2023- 2023-10-01T08:58:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:57 smithi190 conmon[100518]: 10-01T08:58:56.151999+0000 mon.a (mon 2023-10-01T08:58:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:57 smithi190 conmon[100518]: .0) 257 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-01T08:58:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:57 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T08:58:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:57 smithi190 conmon[100518]: T08:58:56.152381+0000 mon.a (mon.0) 2023-10-01T08:58:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:57 smithi190 conmon[100518]: 258 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T08:58:58.170 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[100289]: cluster 2023-10-01T08:58:57.159328+0000 mon.a (mon.0) 259 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-01T08:58:58.170 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[100289]: cluster 2023-10-01T08:58:57.315910+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:58:58.170 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:57.685532+0000 mgr.y (mgr.14146) 62 : cephadm [INF] Detected new or changed devices on smithi138 2023-10-01T08:58:58.170 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[100289]: audit 2023-10-01T08:58:57.689850+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:58.170 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[100289]: audit 2023-10-01T08:58:57.992948+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:58:58.171 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[100289]: audit 2023-10-01T08:58:57.997218+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:58.171 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[100289]: audit 2023-10-01T08:58:57.997976+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:58.171 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[100289]: audit 2023-10-01T08:58:57.998532+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:58.171 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[100289]: audit 2023-10-01T08:58:58.002462+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: cluster 2023-10-01T08:58:57.159328+0000 mon.a (mon.0) 259 2023-10-01T08:58:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-01T08:58:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: cluster 2023-10-01T08:58:57.315910+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:58:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: cephadm 2023-10-01T08:58:57.685532+0000 mgr.y (mgr.14146) 62 : cephadm [INF] Detected new or changed devices on smithi138 2023-10-01T08:58:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: 2023-10-01T08:58:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: audit 2023-10-01T08:58:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: 2023-10-01T08:58:57.689850+0000 mon.a (mon.0) 2023-10-01T08:58:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: 260 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: audit 2023-10-01T08: 2023-10-01T08:58:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: 58:57.992948+0000 mon.a (mon.0) 261 : audit 2023-10-01T08:58:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:58:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: audit 2023-10-01T08:58:57. 2023-10-01T08:58:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: 997218+0000 mon.a (mon.0) 262 : audit [INF] 2023-10-01T08:58:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: audit 2023-10-01T08:58:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: 2023-10-01T08:58:57.997976+0000 mon.a (mon.0 2023-10-01T08:58:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: ) 263 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: audit 2023-10-01T08:58 2023-10-01T08:58:58.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: :57.998532+0000 mon.a (mon.0) 264 : audit 2023-10-01T08:58:58.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:58.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: audit 2023-10-01T08:58:58 2023-10-01T08:58:58.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:58 smithi190 conmon[100518]: .002462+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:58.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[109959]: cluster 2023-10-01T08:58:57.159328+0000 mon.a (mon.0) 259 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-01T08:58:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[109959]: cluster 2023-10-01T08:58:57.315910+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:58:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[109959]: cephadm 2023-10-01T08:58:57.685532+0000 mgr.y (mgr.14146) 62 : cephadm [INF] Detected new or changed devices on smithi138 2023-10-01T08:58:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[109959]: audit 2023-10-01T08:58:57.689850+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[109959]: audit 2023-10-01T08:58:57.992948+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:58:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[109959]: audit 2023-10-01T08:58:57.997218+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[109959]: audit 2023-10-01T08:58:57.997976+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:58:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[109959]: audit 2023-10-01T08:58:57.998532+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:58:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:58 smithi138 conmon[109959]: audit 2023-10-01T08:58:58.002462+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:58:59.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:59 smithi138 conmon[100289]: cephadm 2023-10-01T08:58:57.993481+0000 2023-10-01T08:58:59.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:58:59 smithi138 conmon[100289]: mgr.y (mgr.14146) 63 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 16144M 2023-10-01T08:58:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:59 smithi138 conmon[109959]: cephadm 2023-10-01T08:58:57. 2023-10-01T08:58:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:58:59 smithi138 conmon[109959]: 993481+0000 mgr.y (mgr.14146) 63 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 16144M 2023-10-01T08:58:59.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:59 smithi190 conmon[100518]: cephadm 2023-10-01T08:58:57.993481+0000 mgr.y (mgr. 2023-10-01T08:58:59.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:58:59 smithi190 conmon[100518]: 14146) 63 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 16144M 2023-10-01T08:59:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:00 smithi190 conmon[100518]: cluster 2023-10-01T08:58:59.316215+0000 2023-10-01T08:59:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:00 smithi190 conmon[100518]: mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:00 smithi190 conmon[100518]: audit 2023-10-01T08:58:59.337985+0000 mgr.y (mgr.14146) 65 : audit [DBG] from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:59:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:00 smithi190 conmon[100518]: 2023-10-01T08:59:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:00 smithi190 conmon[100518]: audit 2023-10-01T08:58:59.339914 2023-10-01T08:59:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:00 smithi190 conmon[100518]: +0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T08:59:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:00 smithi190 conmon[100518]: audit 2023-10-01T08 2023-10-01T08:59:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:00 smithi190 conmon[100518]: :58:59.341946+0000 mon.a (mon.0) 267 2023-10-01T08:59:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:00 smithi190 conmon[100518]: : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T08:59:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:00 smithi190 conmon[100518]: audit 2023-10-01T08:58: 2023-10-01T08:59:00.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:00 smithi190 conmon[100518]: 59.342465+0000 mon.a (mon.0) 268 : audit [DBG] 2023-10-01T08:59:00.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:00 smithi190 conmon[100518]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:00.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:00 smithi138 conmon[100289]: cluster 2023-10-01T08:58:59.316215+0000 mgr.y (mgr 2023-10-01T08:59:00.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:00 smithi138 conmon[100289]: .14146) 64 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:00.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:00 smithi138 conmon[100289]: audit 2023-10-01T08:58:59.337985+0000 mgr.y (mgr.14146) 65 : audit [DBG] from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:59:00.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:00 smithi138 conmon[100289]: audit 2023-10-01T08:58:59.339914+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T08:59:00.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:00 smithi138 conmon[100289]: audit 2023-10-01T08:58:59.341946+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T08:59:00.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:00 smithi138 conmon[100289]: audit 2023-10-01T08:58:59.342465+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:00.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:00 smithi138 conmon[109959]: cluster 2023-10-01T08:58:59.316215+0000 mgr.y (mgr.14146 2023-10-01T08:59:00.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:00 smithi138 conmon[109959]: ) 64 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:00.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:00 smithi138 conmon[109959]: audit 2023-10-01T08:58:59.337985+0000 mgr.y (mgr.14146) 65 : audit [DBG] from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:59:00.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:00 smithi138 conmon[109959]: audit 2023-10-01T08:58:59.339914+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T08:59:00.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:00 smithi138 conmon[109959]: audit 2023-10-01T08:58:59.341946+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T08:59:00.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:00 smithi138 conmon[109959]: audit 2023-10-01T08:58:59.342465+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:02.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:02 smithi190 conmon[100518]: cluster 2023-10-01T08:59:01.316524+0000 mgr.y (mgr.14146) 66 : cluster [DBG] 2023-10-01T08:59:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:02 smithi190 conmon[100518]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:02 smithi190 conmon[100518]: audit 2023-10-01T08:59:01.697759+0000 mon.c (mon.2) 10 : audit [INF] from='client.? 172.21.15.138:0/901401881' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "525b5e81-4fa5-4b10-9147-83fcb549ca30"}]: dispatch 2023-10-01T08:59:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:02 smithi190 conmon[100518]: audit 2023-10-01T08:59:01.698110+0000 mon.a (mon.0) 269 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "525b5e81-4fa5-4b10-9147-83fcb549ca30"}]: dispatch 2023-10-01T08:59:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:02 smithi190 conmon[100518]: audit 2023-10-01T08:59:01.709269+0000 mon.a (mon.0) 270 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "525b5e81-4fa5-4b10-9147-83fcb549ca30"}]': finished 2023-10-01T08:59:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:02 smithi190 conmon[100518]: cluster 2023-10-01T08:59:01.709345+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-01T08:59:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:02 smithi190 conmon[100518]: audit 2023-10-01T08:59:01.709469+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:02 smithi190 conmon[100518]: audit 2023-10-01T08:59:02.297737+0000 mon.a (mon.0) 273 : audit [DBG] from='client.? 172.21.15.138:0/224571570' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T08:59:02.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[100289]: cluster 2023-10-01T08:59:01.316524+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:02.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[100289]: audit 2023-10-01T08:59:01.697759+0000 mon.c (mon.2) 10 : audit [INF] from='client.? 172.21.15.138:0/901401881' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "525b5e81-4fa5-4b10-9147-83fcb549ca30"}]: dispatch 2023-10-01T08:59:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[100289]: audit 2023-10-01T08:59:01.698110+0000 mon.a (mon.0) 269 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "525b5e81-4fa5-4b10-9147-83fcb549ca30"}]: dispatch 2023-10-01T08:59:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[100289]: audit 2023-10-01T08:59:01.709269+0000 mon.a (mon.0) 270 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "525b5e81-4fa5-4b10-9147-83fcb549ca30"}]': finished 2023-10-01T08:59:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[100289]: cluster 2023-10-01T08:59:01.709345+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-01T08:59:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[100289]: audit 2023-10-01T08:59:01.709469+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[100289]: audit 2023-10-01T08:59:02.297737+0000 mon.a (mon.0) 273 : audit [DBG] from='client.? 172.21.15.138:0/224571570' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T08:59:02.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: cluster 2023-10-01T08:59:01.316524+0000 2023-10-01T08:59:02.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:02.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: audit 2023-10-01T08:59:02.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: 2023-10-01T08:59:01.697759+0000 mon.c (mon.2) 10 : audit [INF] from='client.? 172.21.15.138:0/901401881' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "525b5e81-4fa5-4b10-9147-83fcb549ca30"}]: dispatch 2023-10-01T08:59:02.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: audit 2023-10-01T08:59:02.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: 2023-10-01T08: 2023-10-01T08:59:02.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: 59:01.698110+0000 mon.a (mon.0) 269 2023-10-01T08:59:02.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "525b5e81-4fa5-4b10-9147-83fcb549ca30"}]: dispatch 2023-10-01T08:59:02.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: audit 2023-10-01T08 2023-10-01T08:59:02.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: :59:01.709269+0000 mon.a (mon.0) 2023-10-01T08:59:02.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: 270 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "525b5e81-4fa5-4b10-9147-83fcb549ca30"}]': finished 2023-10-01T08:59:02.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: cluster 2023 2023-10-01T08:59:02.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: -10-01T08:59:01.709345+0000 2023-10-01T08:59:02.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: mon.a (mon.0) 271 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-01T08:59:02.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: audit 2023-10-01T08:59:02.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: 2023-10-01T08 2023-10-01T08:59:02.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: :59:01.709469+0000 mon.a (mon.0) 2023-10-01T08:59:02.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: 272 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:02.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: audit 2023 2023-10-01T08:59:02.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:02 smithi138 conmon[109959]: -10-01T08:59:02.297737+0000 mon.a (mon.0) 273 : audit [DBG] from='client.? 172.21.15.138:0/224571570' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T08:59:04.719 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:04 smithi138 conmon[100289]: cluster 2023-10-01T08:59:03.316827+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:04.720 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:04 smithi138 conmon[109959]: cluster 2023-10-01T08:59:03.316827+0000 mgr.y (mgr. 2023-10-01T08:59:04.720 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:04 smithi138 conmon[109959]: 14146) 67 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:04.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:04 smithi190 conmon[100518]: cluster 2023-10-01T08:59:03.316827+0000 mgr.y (mgr.14146 2023-10-01T08:59:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:04 smithi190 conmon[100518]: ) 67 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:06.778 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:06 smithi138 conmon[100289]: cluster 2023-10-01T08:59:05.317089+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:06.778 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:06 smithi138 conmon[109959]: cluster 2023-10-01T08:59:05.317089+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:06.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:06 smithi190 conmon[100518]: cluster 2023-10-01T08:59:05.317089+0000 mgr.y (mgr.14146) 2023-10-01T08:59:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:06 smithi190 conmon[100518]: 68 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:08.768 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:08 smithi138 conmon[109959]: cluster 2023-10-01T08:59:07.317456+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:08.768 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:08 smithi138 conmon[109959]: audit 2023-10-01T08:59:07.884254+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T08:59:08.768 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:08 smithi138 conmon[109959]: audit 2023-10-01T08:59:07.884854+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:08.768 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:08 smithi138 conmon[109959]: cephadm 2023-10-01T08:59:07.885345+0000 mgr.y (mgr.14146) 70 : cephadm [INF] Deploying daemon osd.1 on smithi138 2023-10-01T08:59:08.769 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:08 smithi138 conmon[100289]: cluster 2023-10-01T08:59:07. 2023-10-01T08:59:08.769 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:08 smithi138 conmon[100289]: 317456+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:08.769 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:08 smithi138 conmon[100289]: audit 2023-10-01T08:59:07.884254+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T08:59:08.769 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:08 smithi138 conmon[100289]: audit 2023-10-01T08:59:07.884854+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:08.769 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:08 smithi138 conmon[100289]: cephadm 2023-10-01T08:59:07.885345+0000 mgr.y (mgr.14146) 70 : cephadm [INF] Deploying daemon osd.1 on smithi138 2023-10-01T08:59:08.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:08 smithi190 conmon[100518]: cluster 2023-10-01T08:59:07.317456+0000 mgr.y (mgr. 2023-10-01T08:59:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:08 smithi190 conmon[100518]: 14146) 69 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:08 smithi190 conmon[100518]: audit 2023-10-01T08:59:07.884254+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T08:59:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:08 smithi190 conmon[100518]: audit 2023-10-01T08:59:07.884854+0000 mon.a (mon.0 2023-10-01T08:59:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:08 smithi190 conmon[100518]: ) 275 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:08 smithi190 conmon[100518]: cephadm 2023-10-01T08:59: 2023-10-01T08:59:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:08 smithi190 conmon[100518]: 07.885345+0000 mgr.y (mgr.14146) 70 : cephadm 2023-10-01T08:59:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:08 smithi190 conmon[100518]: [INF] Deploying daemon osd.1 on smithi138 2023-10-01T08:59:10.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:10 smithi138 conmon[100289]: cluster 2023-10-01T08:59:09.317972+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:10.522 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:10 smithi138 conmon[109959]: cluster 2023-10-01T08:59:09.317972+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:10.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:10 smithi190 conmon[100518]: cluster 2023-10-01T08:59:09.317972+0000 mgr.y (mgr.14146 2023-10-01T08:59:10.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:10 smithi190 conmon[100518]: ) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:12.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:12 smithi190 conmon[100518]: cluster 2023-10-01T08:59:11.318310+0000 mgr.y ( 2023-10-01T08:59:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:12 smithi190 conmon[100518]: mgr.14146) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:12.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:12 smithi138 conmon[100289]: cluster 2023-10-01T08:59:11.318310+0000 mgr.y (mgr. 2023-10-01T08:59:12.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:12 smithi138 conmon[100289]: 14146) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:12 smithi138 conmon[109959]: cluster 2023-10-01T08:59:11.318310+0000 mgr.y ( 2023-10-01T08:59:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:12 smithi138 conmon[109959]: mgr.14146) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:13 smithi138 conmon[100289]: audit 2023-10-01T08:59:12.611129+0000 mon.a (mon.0 2023-10-01T08:59:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:13 smithi138 conmon[100289]: ) 276 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:13 smithi138 conmon[100289]: audit 2023-10-01T08:59:12.645987+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:59:13.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:13 smithi138 conmon[109959]: audit 2023-10-01T08:59:12.611129 2023-10-01T08:59:13.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:13 smithi138 conmon[109959]: +0000 mon.a (mon.0) 276 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:13.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:13 smithi138 conmon[109959]: audit 2023-10-01T08:59:12.645987+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:59:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:13 smithi190 conmon[100518]: audit 2023-10-01T08:59:12.611129+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:13 smithi190 conmon[100518]: audit 2023-10-01T08:59:12.645987 2023-10-01T08:59:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:13 smithi190 conmon[100518]: +0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:59:14.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:14 smithi138 conmon[100289]: cluster 2023-10-01T08:59:13.318571+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:14 smithi138 conmon[109959]: cluster 2023-10-01T08:59:13.318571+0000 mgr.y (mgr. 2023-10-01T08:59:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:14 smithi138 conmon[109959]: 14146) 73 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:15.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:14 smithi190 conmon[100518]: cluster 2023-10-01T08:59:13.318571+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:15.670 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 1 on host 'smithi138' 2023-10-01T08:59:15.959 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:15 smithi138 conmon[100289]: audit 2023-10-01T08:59:14.764952+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:15.959 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:15 smithi138 conmon[100289]: audit 2023-10-01T08:59:14.765677+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:15.959 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:15 smithi138 conmon[100289]: audit 2023-10-01T08:59:14.766313+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:59:15.959 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:15 smithi138 conmon[100289]: audit 2023-10-01T08:59:14.771472+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:15.960 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:15 smithi138 conmon[100289]: audit 2023-10-01T08:59:15.670068+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:15.960 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:15 smithi138 conmon[100289]: audit 2023-10-01T08:59:15.688747+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:59:15.960 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:15 smithi138 conmon[109959]: audit 2023-10-01T08:59:14.764952+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:15.960 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:15 smithi138 conmon[109959]: audit 2023-10-01T08:59:14.765677+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:15.960 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:15 smithi138 conmon[109959]: audit 2023- 2023-10-01T08:59:15.960 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:15 smithi138 conmon[109959]: 10-01T08:59:14.766313+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:59:15.960 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:15 smithi138 conmon[109959]: audit 2023-10-01T08:59:14.771472+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:15.960 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:15 smithi138 conmon[109959]: audit 2023-10-01T08:59:15.670068+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:15.960 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:15 smithi138 conmon[109959]: audit 2023-10-01T08:59:15.688747+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:59:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:15 smithi190 conmon[100518]: audit 2023-10-01T08:59:14.764952+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:15 smithi190 conmon[100518]: audit 2023-10-01T08:59:14.765677+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:15 smithi190 conmon[100518]: audit 2023 2023-10-01T08:59:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:15 smithi190 conmon[100518]: -10-01T08:59:14.766313+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:59:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:15 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T08:59:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:15 smithi190 conmon[100518]: T08:59:14.771472+0000 mon.a (mon.0) 2023-10-01T08:59:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:15 smithi190 conmon[100518]: 281 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:15 smithi190 conmon[100518]: audit 2023-10-01T08: 2023-10-01T08:59:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:15 smithi190 conmon[100518]: 59:15.670068+0000 mon.a (mon.0) 282 2023-10-01T08:59:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:15 smithi190 conmon[100518]: : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:16.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:15 smithi190 conmon[100518]: audit 2023-10-01T08: 2023-10-01T08:59:16.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:15 smithi190 conmon[100518]: 59:15.688747+0000 mon.a (mon.0) 283 : 2023-10-01T08:59:16.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:15 smithi190 conmon[100518]: audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:59:16.388 DEBUG:teuthology.orchestra.run.smithi138:osd.1> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-01T08:59:16.390 INFO:tasks.cephadm:Deploying osd.2 on smithi138 with /dev/vg_nvme/lv_2... 2023-10-01T08:59:16.390 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-01T08:59:16.422 INFO:journalctl@ceph.osd.1.smithi138.stdout:-- Logs begin at Sun 2023-10-01 08:38:26 UTC. -- 2023-10-01T08:59:16.926 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:16 smithi138 conmon[100289]: cluster 2023-10-01T08:59:15.318917+0000 2023-10-01T08:59:16.927 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:16 smithi138 conmon[100289]: mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:16.927 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:16 smithi138 conmon[100289]: audit 2023-10-01T08:59:15.819818+0000 mon.c (mon.2) 11 : audit [INF] from='osd.1 [v2:172.21.15.138:6810/989382814,v1:172.21.15.138:6811/989382814]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T08:59:16.927 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:16 smithi138 conmon[100289]: audit 2023-10-01T08:59:15.820222+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T08:59:16.927 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:16 smithi138 conmon[109959]: cluster 2023-10-01T08:59:15.318917+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:16.927 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:16 smithi138 conmon[109959]: audit 2023-10-01T08:59:15.819818+0000 mon.c (mon.2) 11 2023-10-01T08:59:16.927 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:16 smithi138 conmon[109959]: : audit [INF] from='osd.1 [v2:172.21.15.138:6810/989382814,v1:172.21.15.138:6811/989382814]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T08:59:16.927 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:16 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T08:59:16.927 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:16 smithi138 conmon[109959]: 08:59:15.820222+0000 mon.a (mon.0) 284 : 2023-10-01T08:59:16.927 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:16 smithi138 conmon[109959]: audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T08:59:17.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:16 smithi190 conmon[100518]: cluster 2023-10-01T08:59:15.318917+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:17.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:16 smithi190 conmon[100518]: audit 2023-10-01T08:59:15.819818+0000 mon.c (mon.2) 11 : audit [INF] from='osd.1 [v2:172.21.15.138:6810/989382814,v1:172.21.15.138:6811/989382814]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T08:59:17.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:16 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T08:59:17.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:16 smithi190 conmon[100518]: 01T08:59:15.820222+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T08:59:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:17 smithi190 conmon[100518]: audit 2023-10-01T08:59:16.780852+0000 mon.a (mon.0) 285 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T08:59:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:17 smithi190 conmon[100518]: cluster 2023 2023-10-01T08:59:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:17 smithi190 conmon[100518]: -10-01T08:59:16.780923+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-01T08:59:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:17 smithi190 conmon[100518]: audit 2023-10-01T08:59:16.781044+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:17 smithi190 conmon[100518]: audit 2023-10-01T08:59:16.783756+0000 mon.c (mon.2) 2023-10-01T08:59:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:17 smithi190 conmon[100518]: 12 : audit [INF] from='osd.1 [v2:172.21.15.138:6810/989382814,v1:172.21.15.138:6811/989382814]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:59:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:17 smithi190 conmon[100518]: audit 2023-10-01T08:59:16 2023-10-01T08:59:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:17 smithi190 conmon[100518]: .784086+0000 mon.a (mon.0) 288 : audit [INF] 2023-10-01T08:59:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:17 smithi190 conmon[100518]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:59:18.069 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[120861]: debug 2023-10-01T08:59:17.792+0000 7fdb66bc5700 -1 osd.1 0 waiting for initial osdmap 2023-10-01T08:59:18.069 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[120861]: debug 2023-10-01T08:59:17.795+0000 7fdb6133d700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T08:59:18.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[100289]: audit 2023-10-01T08:59:16.780852+0000 mon.a (mon. 2023-10-01T08:59:18.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[100289]: 0) 285 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T08:59:18.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[100289]: cluster 2023-10-01T08:59:16.780923+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-01T08:59:18.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[100289]: audit 2023-10-01T08:59:16.781044+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:18.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[100289]: audit 2023-10 2023-10-01T08:59:18.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[100289]: -01T08:59:16.783756+0000 mon.c (mon.2) 12 : audit [INF] from='osd.1 [v2:172.21.15.138:6810/989382814,v1:172.21.15.138:6811/989382814]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:59:18.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[100289]: audit 2023- 2023-10-01T08:59:18.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[100289]: 10-01T08:59:16.784086 2023-10-01T08:59:18.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[100289]: +0000 mon.a (mon.0) 288 : 2023-10-01T08:59:18.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[100289]: audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:59:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[109959]: audit 2023-10-01T08:59:16.780852+0000 mon.a (mon.0) 285 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T08:59:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[109959]: cluster 2023-10-01T08:59:16.780923+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-01T08:59:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[109959]: audit 2023 2023-10-01T08:59:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[109959]: -10-01T08:59:16.781044+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[109959]: audit 2023-10-01T08:59:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[109959]: 2023-10-01T08:59:16.783756+0000 mon.c (mon. 2023-10-01T08:59:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[109959]: 2) 12 : audit [INF] from='osd.1 [v2:172.21.15.138:6810/989382814,v1:172.21.15.138:6811/989382814]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:59:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[109959]: audit 2023 2023-10-01T08:59:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[109959]: -10-01T08:59:16.784086+0000 2023-10-01T08:59:18.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:17 smithi138 conmon[109959]: mon.a (mon.0) 288 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:59:18.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:18 smithi138 conmon[100289]: cluster 2023-10-01T08:59:17. 2023-10-01T08:59:18.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:18 smithi138 conmon[100289]: 319219+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:18.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:18 smithi138 conmon[100289]: audit 2023-10-01T08:59:17.788408+0000 mon.a (mon.0) 289 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-10-01T08:59:18.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:18 smithi138 conmon[100289]: cluster 2023-10-01T08:59:17.788487+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-01T08:59:18.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:18 smithi138 conmon[100289]: audit 2023-10-01T08:59:17.788692+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:18.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:18 smithi138 conmon[100289]: audit 2023-10-01T08:59:17.793474+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:18.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:18 smithi138 conmon[109959]: cluster 2023-10-01T08:59: 2023-10-01T08:59:18.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:18 smithi138 conmon[109959]: 17.319219+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:18.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:18 smithi138 conmon[109959]: audit 2023-10-01T08:59:17.788408+0000 mon.a (mon.0) 289 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-10-01T08:59:18.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:18 smithi138 conmon[109959]: cluster 2023-10-01T08:59:17.788487+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-01T08:59:18.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:18 smithi138 conmon[109959]: audit 2023-10-01T08:59:17.788692+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:18.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:18 smithi138 conmon[109959]: audit 2023-10-01T08:59:17.793474+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:19.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:18 smithi190 conmon[100518]: cluster 2023-10-01T08:59:17.319219+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T08:59:19.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:18 smithi190 conmon[100518]: audit 2023-10-01T08:59:17.788408+0000 mon.a (mon.0) 289 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-10-01T08:59:19.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:18 smithi190 conmon[100518]: cluster 2023-10 2023-10-01T08:59:19.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:18 smithi190 conmon[100518]: -01T08:59:17.788487+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-01T08:59:19.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:18 smithi190 conmon[100518]: audit 2023-10 2023-10-01T08:59:19.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:18 smithi190 conmon[100518]: -01T08:59:17.788692+0000 mon.a (mon. 2023-10-01T08:59:19.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:18 smithi190 conmon[100518]: 0) 291 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:19.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:18 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T08:59:19.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:18 smithi190 conmon[100518]: 08:59:17.793474+0000 mon.a (mon.0) 292 2023-10-01T08:59:19.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:18 smithi190 conmon[100518]: : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:19.087 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-01T08:59:19.087 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-01T08:59:19.087 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2023-10-01T08:59:19.087 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2023-10-01T08:59:19.087 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0109581 s, 957 MB/s 2023-10-01T08:59:19.087 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2023-10-01T08:59:20.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[100289]: cluster 2023-10-01T08:59:16.837394+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T08:59:20.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[100289]: cluster 2023-10-01T08:59:16.837448+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T08:59:20.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[100289]: cluster 2023-10-01T08:59:18.792901+0000 mon.a (mon.0) 293 : cluster [INF] osd.1 [v2:172.21.15.138:6810/989382814,v1:172.21.15.138:6811/989382814] boot 2023-10-01T08:59:20.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[100289]: cluster 2023-10-01T08:59:18.792960+0000 mon.a (mon.0) 294 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-01T08:59:20.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[100289]: audit 2023-10-01T08:59:18.793423+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:20.030 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[109959]: cluster 2023-10-01T08:59:16.837394+0000 2023-10-01T08:59:20.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[109959]: osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T08:59:20.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[109959]: cluster 2023-10-01T08:59:20.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[109959]: 2023-10-01T08:59:16 2023-10-01T08:59:20.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[109959]: .837448+0000 osd.1 (osd.1) 2 : cluster 2023-10-01T08:59:20.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[109959]: [DBG] purged_snaps scrub ok 2023-10-01T08:59:20.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[109959]: cluster 2023 2023-10-01T08:59:20.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[109959]: -10-01T08:59:18.792901+0000 mon.a 2023-10-01T08:59:20.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[109959]: (mon.0) 293 : cluster [INF] osd.1 [v2:172.21.15.138:6810/989382814,v1:172.21.15.138:6811/989382814] boot 2023-10-01T08:59:20.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[109959]: cluster 2023-10-01T08:59:18.792960+0000 2023-10-01T08:59:20.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[109959]: mon.a (mon.0) 294 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-01T08:59:20.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[109959]: audit 2023-10-01T08:59:18.793423+0000 mon.a ( 2023-10-01T08:59:20.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[109959]: mon.0) 295 : audit [DBG] 2023-10-01T08:59:20.032 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:19 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:20.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:19 smithi190 conmon[100518]: cluster 2023-10-01T08:59:16.837394+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T08:59:20.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:19 smithi190 conmon[100518]: cluster 2023-10-01T08:59:16.837448+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T08:59:20.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:19 smithi190 conmon[100518]: cluster 2023-10-01T08 2023-10-01T08:59:20.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:19 smithi190 conmon[100518]: :59:18.792901+0000 mon.a (mon.0) 293 : cluster [INF] osd.1 [v2:172.21.15.138:6810/989382814,v1:172.21.15.138:6811/989382814] boot 2023-10-01T08:59:20.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:19 smithi190 conmon[100518]: cluster 2023-10-01T08: 2023-10-01T08:59:20.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:19 smithi190 conmon[100518]: 59:18.792960+0000 mon.a (mon.0) 294 : 2023-10-01T08:59:20.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:19 smithi190 conmon[100518]: cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-01T08:59:20.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:19 smithi190 conmon[100518]: audit 2023-10-01T08:59: 2023-10-01T08:59:20.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:19 smithi190 conmon[100518]: 18.793423+0000 mon.a (mon.0) 295 : audit 2023-10-01T08:59:20.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:19 smithi190 conmon[100518]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T08:59:20.128 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi138:vg_nvme/lv_2 2023-10-01T08:59:21.000 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[100289]: cluster 2023-10-01T08:59:19.319588+0000 2023-10-01T08:59:21.000 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[100289]: mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:21.000 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[100289]: cluster 2023-10-01T08:59:19.807679+0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-01T08:59:21.000 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[100289]: audit 2023-10-01T08:59:20.308392+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:21.000 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[100289]: audit 2023-10-01T08:59:20.309220+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:21.000 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[100289]: audit 2023-10-01T08:59:20.309864+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:21.000 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[100289]: audit 2023-10-01T08:59:20.313738+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:21.001 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[100289]: audit 2023-10-01T08:59:20.314533+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:21.001 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[100289]: audit 2023-10-01T08:59:20.315270+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:59:21.001 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[100289]: audit 2023-10-01T08:59:20.319517+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:21.001 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[109959]: cluster 2023-10-01T08:59:19.319588+0000 2023-10-01T08:59:21.001 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[109959]: mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:21.001 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[109959]: cluster 2023-10-01T08:59:19.807679+0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-01T08:59:21.001 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[109959]: audit 2023-10-01T08:59:20.308392+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:21.001 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[109959]: audit 2023-10-01T08:59:20.309220+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:21.001 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[109959]: audit 2023-10-01T08:59:20.309864+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:21.001 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[109959]: audit 2023-10-01T08:59:20.313738+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:21.002 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[109959]: audit 2023-10-01T08:59:20.314533+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:21.002 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[109959]: audit 2023-10-01T08:59:20.315270 2023-10-01T08:59:21.002 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[109959]: +0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:59:21.002 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:20 smithi138 conmon[109959]: audit 2023-10-01T08:59:20.319517+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:21.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: cluster 2023-10-01T08:59:19.319588+0000 2023-10-01T08:59:21.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:21.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: cluster 2023-10-01T08:59:19.807679 2023-10-01T08:59:21.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: +0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-01T08:59:21.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: audit 2023-10-01T08:59:20.308392+0000 2023-10-01T08:59:21.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: mon.a (mon.0) 297 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:21.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: audit 2023-10-01T08:59:21.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: 2023-10-01T08:59:20.309220+0000 mon.a (mon. 2023-10-01T08:59:21.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: 0) 298 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:21.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: audit 2023-10-01T08:59:21.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: 2023-10-01T08:59:20 2023-10-01T08:59:21.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: .309864+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:21.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: 2023-10-01T08:59:21.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: audit 2023-10-01T08:59:20.313738+0000 mon.a 2023-10-01T08:59:21.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:21.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: audit 2023 2023-10-01T08:59:21.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: -10-01T08:59:20.314533+0000 mon.a (mon.0 2023-10-01T08:59:21.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: ) 301 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:21.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T08:59:21.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: 01T08:59:20.315270+0000 mon.a (mon.0) 2023-10-01T08:59:21.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: 302 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:59:21.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T08:59:21.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: T08:59:20.319517+0000 mon.a (mon.0) 2023-10-01T08:59:21.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:20 smithi190 conmon[100518]: 303 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:22.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:21 smithi138 conmon[100289]: cephadm 2023-10-01T08:59:20.303765+0000 mgr.y (mgr.14146) 77 : cephadm [INF] Detected new or changed devices on smithi138 2023-10-01T08:59:22.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:21 smithi138 conmon[100289]: cephadm 2023-10-01T08:59:20.310367+0000 mgr.y (mgr.14146) 78 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 8072M 2023-10-01T08:59:22.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:21 smithi138 conmon[109959]: cephadm 2023-10-01T08:59:20.303765+0000 mgr.y (mgr.14146) 77 : cephadm [INF] Detected new or changed devices on smithi138 2023-10-01T08:59:22.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:21 smithi138 conmon[109959]: cephadm 2023-10-01T08:59:20.310367+0000 mgr.y (mgr.14146) 78 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 8072M 2023-10-01T08:59:22.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:21 smithi190 conmon[100518]: cephadm 2023-10-01T08:59:20.303765+0000 mgr.y (mgr 2023-10-01T08:59:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:21 smithi190 conmon[100518]: .14146) 77 : cephadm [INF] Detected new or changed devices on smithi138 2023-10-01T08:59:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:21 smithi190 conmon[100518]: cephadm 2023-10-01T08:59:20.310367+0000 mgr.y (mgr.14146 2023-10-01T08:59:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:21 smithi190 conmon[100518]: ) 78 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 8072M 2023-10-01T08:59:22.933 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:22 smithi138 conmon[100289]: cluster 2023-10-01T08:59:21.320048+0000 mgr.y ( 2023-10-01T08:59:22.933 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:22 smithi138 conmon[100289]: mgr.14146) 79 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:22.933 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:22 smithi138 conmon[100289]: audit 2023-10-01T08:59:22.037026+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T08:59:22.933 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:22 smithi138 conmon[100289]: audit 2023-10-01T08:59:22.038921+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T08:59:22.933 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:22 smithi138 conmon[100289]: audit 2023-10-01T08:59:22.039438+0000 mon.a (mon.0) 306 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:22.934 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:22 smithi138 conmon[109959]: cluster 2023-10-01T08:59:21.320048+0000 mgr.y ( 2023-10-01T08:59:22.934 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:22 smithi138 conmon[109959]: mgr.14146) 79 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:22.934 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:22 smithi138 conmon[109959]: audit 2023-10-01T08:59:22.037026+0000 mon.a (mon.0) 304 : audit 2023-10-01T08:59:22.934 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:22 smithi138 conmon[109959]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T08:59:22.934 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:22 smithi138 conmon[109959]: audit 2023-10-01T08:59:22.038921+0000 mon.a (mon.0) 305 2023-10-01T08:59:22.934 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:22 smithi138 conmon[109959]: : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T08:59:22.934 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:22 smithi138 conmon[109959]: audit 2023-10-01T08:59:22.039438+0000 mon.a (mon.0) 2023-10-01T08:59:22.934 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:22 smithi138 conmon[109959]: 306 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:22 smithi190 conmon[100518]: cluster 2023-10-01T08:59:21.320048+0000 2023-10-01T08:59:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:22 smithi190 conmon[100518]: mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:22 smithi190 conmon[100518]: audit 2023-10-01T08:59 2023-10-01T08:59:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:22 smithi190 conmon[100518]: :22.037026+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T08:59:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:22 smithi190 conmon[100518]: audit 2023-10-01T08:59:22. 2023-10-01T08:59:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:22 smithi190 conmon[100518]: 038921+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T08:59:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:22 smithi190 conmon[100518]: audit 2023-10-01T08:59:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:22 smithi190 conmon[100518]: 2023-10-01T08:59:22.039438+0000 mon.a (mon. 2023-10-01T08:59:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:22 smithi190 conmon[100518]: 0) 306 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:24.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:23 smithi138 conmon[100289]: audit 2023-10-01T08:59:22.035080+0000 mgr.y (mgr.14146) 80 : audit 2023-10-01T08:59:24.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:23 smithi138 conmon[100289]: [DBG] from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:59:24.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:23 smithi138 conmon[109959]: audit 2023-10-01T08:59:22.035080+0000 mgr.y (mgr.14146) 2023-10-01T08:59:24.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:23 smithi138 conmon[109959]: 80 : audit [DBG] from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:59:24.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:23 smithi190 conmon[100518]: audit 2023-10-01T08:59:22.035080+0000 2023-10-01T08:59:24.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:23 smithi190 conmon[100518]: mgr.y (mgr.14146) 80 : audit [DBG] from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:59:25.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:24 smithi138 conmon[100289]: cluster 2023-10-01T08:59:23.320416+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:25.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:24 smithi138 conmon[100289]: audit 2023-10-01T08:59:24.343453+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.138:0/2537806577' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97443e05-2513-4832-bb54-f65493df5967"}]: dispatch 2023-10-01T08:59:25.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:24 smithi138 conmon[100289]: audit 2023-10-01T08:59:24.343779+0000 mon.a (mon.0) 307 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97443e05-2513-4832-bb54-f65493df5967"}]: dispatch 2023-10-01T08:59:25.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:24 smithi138 conmon[100289]: audit 2023-10-01T08:59:24.347218+0000 mon.a (mon.0) 308 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "97443e05-2513-4832-bb54-f65493df5967"}]': finished 2023-10-01T08:59:25.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:24 smithi138 conmon[100289]: cluster 2023-10-01T08:59:24.347279+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-01T08:59:25.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:24 smithi138 conmon[100289]: audit 2023-10-01T08:59:24.347396+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:24 smithi138 conmon[109959]: cluster 2023-10-01T08:59:23.320416+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:24 smithi138 conmon[109959]: audit 2023-10-01T08:59:24.343453+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.138:0/2537806577' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97443e05-2513-4832-bb54-f65493df5967"}]: dispatch 2023-10-01T08:59:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:24 smithi138 conmon[109959]: audit 2023-10-01T08:59:24.343779+0000 mon.a (mon.0) 307 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97443e05-2513-4832-bb54-f65493df5967"}]: dispatch 2023-10-01T08:59:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:24 smithi138 conmon[109959]: audit 2023-10-01T08:59:24.347218+0000 mon.a (mon.0) 308 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "97443e05-2513-4832-bb54-f65493df5967"}]': finished 2023-10-01T08:59:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:24 smithi138 conmon[109959]: cluster 2023-10-01T08:59:24.347279+0000 mon.a (mon.0) 309 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-01T08:59:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:24 smithi138 conmon[109959]: audit 2023-10-01T08:59:24.347396+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:25.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: cluster 2023-10-01T08:59:23.320416+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:25.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: audit 2023-10-01T08:59:24.343453+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.138:0/2537806577' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97443e05-2513-4832-bb54-f65493df5967"}]: dispatch 2023-10-01T08:59:25.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: audit 2023-10-01T08:59:25.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: 2023-10-01T08:59:24.343779+0000 mon.a (mon.0) 2023-10-01T08:59:25.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: 307 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97443e05-2513-4832-bb54-f65493df5967"}]: dispatch 2023-10-01T08:59:25.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: audit 2023- 2023-10-01T08:59:25.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: 10-01T08:59:24.347218+0000 mon.a 2023-10-01T08:59:25.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: (mon.0) 308 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "97443e05-2513-4832-bb54-f65493df5967"}]': finished 2023-10-01T08:59:25.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: 2023-10-01T08:59:25.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: cluster 2023-10-01 2023-10-01T08:59:25.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: T08:59:24.347279+0000 mon.a ( 2023-10-01T08:59:25.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: mon.0) 309 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-01T08:59:25.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: 2023-10-01T08:59:25.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: audit 2023-10-01T08:59:25.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: 2023-10-01T08:59: 2023-10-01T08:59:25.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: 24.347396+0000 mon.a (mon.0 2023-10-01T08:59:25.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:24 smithi190 conmon[100518]: ) 310 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:26.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:25 smithi190 conmon[100518]: audit 2023-10-01T08:59:24.927805+0000 mon.c (mon.2) 14 : audit 2023-10-01T08:59:26.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:25 smithi190 conmon[100518]: [DBG] from='client.? 172.21.15.138:0/3755235110' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T08:59:26.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:25 smithi138 conmon[100289]: audit 2023-10-01T08:59:24.927805+0000 mon.c (mon.2) 14 : audit 2023-10-01T08:59:26.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:25 smithi138 conmon[100289]: [DBG] from='client.? 172.21.15.138:0/3755235110' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T08:59:26.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:25 smithi138 conmon[109959]: audit 2023-10-01T08:59:24.927805+0000 mon.c (mon.2 2023-10-01T08:59:26.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:25 smithi138 conmon[109959]: ) 14 : audit [DBG] from='client.? 172.21.15.138:0/3755235110' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T08:59:27.205 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:26 smithi138 conmon[100289]: cluster 2023-10-01T08:59:25.320917+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:27.205 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:26 smithi138 conmon[109959]: cluster 2023-10-01T08:59:25.320917+0000 mgr.y (mgr.14146) 82 2023-10-01T08:59:27.205 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:26 smithi138 conmon[109959]: : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:27.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:26 smithi190 conmon[100518]: cluster 2023-10-01T08:59:25.320917+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:29.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:28 smithi138 conmon[109959]: cluster 2023-10-01T08:59:27.321342+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:29.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:28 smithi138 conmon[100289]: cluster 2023-10-01T08:59 2023-10-01T08:59:29.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:28 smithi138 conmon[100289]: :27.321342+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:29.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:28 smithi190 conmon[100518]: cluster 2023-10-01T08:59:27.321342+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:31.116 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:30 smithi138 conmon[100289]: cluster 2023-10-01T08:59:29.321599+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:31.116 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:30 smithi138 conmon[100289]: audit 2023-10-01T08:59:30.538024+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T08:59:31.116 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:30 smithi138 conmon[100289]: audit 2023-10-01T08:59:30.538619+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:31.116 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:30 smithi138 conmon[109959]: cluster 2023-10-01T08:59:29.321599+0000 mgr.y ( 2023-10-01T08:59:31.116 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:30 smithi138 conmon[109959]: mgr.14146) 84 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:31.116 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:30 smithi138 conmon[109959]: audit 2023-10-01T08:59:30.538024+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T08:59:31.116 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:30 smithi138 conmon[109959]: audit 2023-10-01T08:59:30.538619+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:31.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:30 smithi190 conmon[100518]: cluster 2023-10-01T08:59:29.321599+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:31.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:30 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T08:59:31.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:30 smithi190 conmon[100518]: T08:59:30.538024+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T08:59:31.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:30 smithi190 conmon[100518]: audit 2023-10-01T08:59:30.538619+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:32.116 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:31 smithi138 conmon[100289]: cephadm 2023-10-01T08:59:30.539097+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Deploying daemon osd.2 on smithi138 2023-10-01T08:59:32.116 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:31 smithi138 conmon[109959]: cephadm 2023-10-01T08:59:30.539097+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Deploying daemon osd.2 on smithi138 2023-10-01T08:59:32.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:31 smithi190 conmon[100518]: cephadm 2023-10-01T08:59:30.539097+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Deploying daemon osd.2 on smithi138 2023-10-01T08:59:33.142 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:32 smithi138 conmon[100289]: cluster 2023-10-01T08:59:31.321859 2023-10-01T08:59:33.142 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:32 smithi138 conmon[100289]: +0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:33.142 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:32 smithi138 conmon[109959]: cluster 2023-10-01T08:59:31.321859+0000 mgr.y (mgr.14146 2023-10-01T08:59:33.143 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:32 smithi138 conmon[109959]: ) 86 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:33.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:32 smithi190 conmon[100518]: cluster 2023-10-01T08:59:31.321859+0000 mgr.y 2023-10-01T08:59:33.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:32 smithi190 conmon[100518]: (mgr.14146) 86 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:35.279 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:34 smithi138 conmon[100289]: cluster 2023-10-01T08:59:33.322180+0000 2023-10-01T08:59:35.279 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:34 smithi138 conmon[100289]: mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:35.280 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:34 smithi138 conmon[109959]: cluster 2023-10-01T08:59:33.322180+0000 mgr.y ( 2023-10-01T08:59:35.280 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:34 smithi138 conmon[109959]: mgr.14146) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:34 smithi190 conmon[100518]: cluster 2023-10-01T08:59:33.322180+0000 mgr.y (mgr.14146) 2023-10-01T08:59:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:34 smithi190 conmon[100518]: 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:36.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:36 smithi138 conmon[100289]: cluster 2023-10-01T08:59:35.322780+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:36.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:36 smithi138 conmon[100289]: audit 2023-10-01T08:59:35.424029+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:36.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:36 smithi138 conmon[100289]: audit 2023-10-01T08:59:35.467096+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:59:36.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:36 smithi138 conmon[109959]: cluster 2023-10-01T08:59:35.322780+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:36.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:36 smithi138 conmon[109959]: audit 2023-10-01T08:59:35.424029+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:36.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:36 smithi138 conmon[109959]: audit 2023-10-01T08:59:35.467096+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:59:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:36 smithi190 conmon[100518]: cluster 2023-10-01T08:59:35.322780+0000 mgr.y (mgr.14146) 88 : cluster [DBG] 2023-10-01T08:59:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:36 smithi190 conmon[100518]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:36 smithi190 conmon[100518]: audit 2023-10-01T08:59:35.424029+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:36 smithi190 conmon[100518]: audit 2023-10-01T08:59:35.467096+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:59:38.593 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 2 on host 'smithi138' 2023-10-01T08:59:38.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[100289]: cluster 2023-10-01T08:59:37.323076+0000 mgr.y (mgr 2023-10-01T08:59:38.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[100289]: .14146) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:38.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[100289]: audit 2023-10-01T08:59:37.455440+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:38.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[100289]: audit 2023-10-01T08:59:37.456211+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:38.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[100289]: audit 2023-10-01T08:59:37.456926+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:59:38.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[100289]: audit 2023-10-01T08:59:37.461365+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:38.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[109959]: cluster 2023-10-01T08:59:37. 2023-10-01T08:59:38.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[109959]: 323076+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:38.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T08:59:38.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[109959]: 08:59:37.455440+0000 mon.a ( 2023-10-01T08:59:38.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[109959]: mon.0) 315 : audit [INF] 2023-10-01T08:59:38.793 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:38.793 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[109959]: audit 2023-10-01T08: 2023-10-01T08:59:38.793 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[109959]: 59:37.456211+0000 mon.a (mon.0) 2023-10-01T08:59:38.793 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[109959]: 316 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:38.793 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[109959]: audit 2023-10-01T08:59:37.456926+0000 mon.a 2023-10-01T08:59:38.793 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[109959]: (mon.0) 317 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:59:38.793 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[109959]: audit 2023-10-01T08:59:37.461365+0000 2023-10-01T08:59:38.793 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:38 smithi138 conmon[109959]: mon.a (mon.0) 318 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:38.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:38 smithi190 conmon[100518]: cluster 2023-10-01T08:59:37.323076+0000 mgr.y (mgr. 2023-10-01T08:59:38.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:38 smithi190 conmon[100518]: 14146) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:38 smithi190 conmon[100518]: audit 2023-10-01T08:59:37.455440+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:38 smithi190 conmon[100518]: audit 2023-10-01T08:59:37.456211+0000 mon.a (mon.0) 316 : audit 2023-10-01T08:59:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:38 smithi190 conmon[100518]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:38 smithi190 conmon[100518]: audit 2023-10-01T08:59:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:38 smithi190 conmon[100518]: 2023-10-01T08:59:37.456926+0000 mon.a (mon.0 2023-10-01T08:59:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:38 smithi190 conmon[100518]: ) 317 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:59:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:38 smithi190 conmon[100518]: audit 2023-10-01T08:59:37.461365+0000 mon.a ( 2023-10-01T08:59:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:38 smithi190 conmon[100518]: mon.0) 318 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:39.201 DEBUG:teuthology.orchestra.run.smithi138:osd.2> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-01T08:59:39.203 INFO:tasks.cephadm:Deploying osd.3 on smithi138 with /dev/vg_nvme/lv_1... 2023-10-01T08:59:39.203 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-01T08:59:39.237 INFO:journalctl@ceph.osd.2.smithi138.stdout:-- Logs begin at Sun 2023-10-01 08:38:26 UTC. -- 2023-10-01T08:59:39.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:39 smithi138 conmon[100289]: audit 2023-10-01T08:59:38. 2023-10-01T08:59:39.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:39 smithi138 conmon[100289]: 590532+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:39.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:39 smithi138 conmon[100289]: audit 2023-10-01T08:59:38.621354+0000 mon.a (mon.0) 320 : audit [INF] from='osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T08:59:39.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:39 smithi138 conmon[100289]: audit 2023-10-01T08:59:38.629002+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:59:39.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:39 smithi138 conmon[109959]: audit 2023-10-01T08:59:38. 2023-10-01T08:59:39.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:39 smithi138 conmon[109959]: 590532+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:39.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:39 smithi138 conmon[109959]: audit 2023-10-01T08:59:38.621354+0000 mon.a (mon.0) 320 : audit [INF] from='osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T08:59:39.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:39 smithi138 conmon[109959]: audit 2023-10-01T08:59:38.629002+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:59:40.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:39 smithi190 conmon[100518]: audit 2023-10-01T08:59:38.590532 2023-10-01T08:59:40.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:39 smithi190 conmon[100518]: +0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:40.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:39 smithi190 conmon[100518]: audit 2023- 2023-10-01T08:59:40.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:39 smithi190 conmon[100518]: 10-01T08:59:38.621354+0000 mon.a (mon.0) 320 : audit 2023-10-01T08:59:40.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:39 smithi190 conmon[100518]: [INF] from='osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T08:59:40.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:39 smithi190 conmon[100518]: audit 2023-10-01T08:59:38.629002 2023-10-01T08:59:40.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:39 smithi190 conmon[100518]: +0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T08:59:40.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:39 smithi190 conmon[100518]: 2023-10-01T08:59:40.603 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 08:59:40 smithi138 conmon[126645]: debug 2023-10-01T08:59:40.600+0000 7f371201f700 -1 osd.2 0 waiting for initial osdmap 2023-10-01T08:59:40.603 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:40 smithi138 conmon[100289]: cluster 2023-10-01T08:59:39.323344+0000 mgr.y (mgr.14146) 90 : cluster 2023-10-01T08:59:40.603 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:40 smithi138 conmon[100289]: [DBG] pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:40.604 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:40 smithi138 conmon[100289]: audit 2023-10-01T08:59:39.597825+0000 mon.a (mon.0) 322 : audit [INF] from='osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T08:59:40.604 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:40 smithi138 conmon[100289]: cluster 2023-10-01T08:59:39.597897+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-01T08:59:40.604 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:40 smithi138 conmon[100289]: audit 2023-10-01T08:59:39.598117+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:41.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:40 smithi190 conmon[100518]: cluster 2023-10-01T08:59:39.323344+0000 mgr.y (mgr.14146) 2023-10-01T08:59:41.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:40 smithi190 conmon[100518]: 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:41.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:40 smithi190 conmon[100518]: audit 2023-10-01T08:59:39.597825+0000 mon.a (mon.0) 322 : audit [INF] from='osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T08:59:41.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:40 smithi190 conmon[100518]: cluster 2023-10-01T08:59:39.597897+0000 mon.a (mon.0) 323 : cluster [DBG] 2023-10-01T08:59:41.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:40 smithi190 conmon[100518]: osdmap e16: 3 total, 2 up, 3 in 2023-10-01T08:59:41.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:40 smithi190 conmon[100518]: audit 2023-10-01T08:59:39.598117+0000 2023-10-01T08:59:41.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:40 smithi190 conmon[100518]: mon.a (mon.0) 324 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:41.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:40 smithi190 conmon[100518]: audit 2023-10 2023-10-01T08:59:41.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:40 smithi190 conmon[100518]: -01T08:59:39.598443+0000 mon.a (mon. 2023-10-01T08:59:41.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:40 smithi190 conmon[100518]: 0) 325 : audit [INF] from='osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:59:41.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:40 smithi138 conmon[100289]: audit 2023-10-01T08:59:39.598443+0000 mon.a (mon.0) 325 : audit [INF] from='osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:59:41.070 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 08:59:40 smithi138 conmon[126645]: debug 2023-10-01T08:59:40.608+0000 7f370cf98700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T08:59:41.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:40 smithi138 conmon[109959]: cluster 2023-10-01T08:59:39.323344+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:41.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:40 smithi138 conmon[109959]: audit 2023-10-01T08:59:39.597825+0000 mon.a (mon.0) 322 : audit [INF] from='osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T08:59:41.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:40 smithi138 conmon[109959]: cluster 2023-10-01T08:59:39.597897+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-01T08:59:41.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:40 smithi138 conmon[109959]: audit 2023-10-01T08:59:39.598117+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:41.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:40 smithi138 conmon[109959]: audit 2023-10-01T08:59:39.598443+0000 mon.a (mon.0) 325 : audit [INF] from='osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T08:59:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:41 smithi138 conmon[109959]: audit 2023-10-01T08:59:40.598344+0000 mon.a (mon.0) 326 : audit [INF] from='osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-10-01T08:59:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:41 smithi138 conmon[109959]: cluster 2023-10-01T08:59:40.598412+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-01T08:59:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:41 smithi138 conmon[109959]: audit 2023-10-01T08:59:40.598873+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:41 smithi138 conmon[109959]: audit 2023-10-01T08:59:40.605900+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:42.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:41 smithi138 conmon[100289]: audit 2023-10-01T08:59:40.598344+0000 mon.a (mon.0) 326 : audit [INF] from='osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-10-01T08:59:42.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:41 smithi138 conmon[100289]: cluster 2023-10-01T08:59:40.598412+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-01T08:59:42.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:41 smithi138 conmon[100289]: audit 2023-10-01T08:59:40.598873+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:42.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:41 smithi138 conmon[100289]: audit 2023-10-01T08:59:40.605900+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:42.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:41 smithi190 conmon[100518]: audit 2023-10-01T08:59:40.598344+0000 mon.a (mon.0) 2023-10-01T08:59:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:41 smithi190 conmon[100518]: 326 : audit [INF] from='osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-10-01T08:59:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:41 smithi190 conmon[100518]: cluster 2023-10-01T08:59:40.598412+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-01T08:59:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:41 smithi190 conmon[100518]: audit 2023-10-01T08:59:40.598873+0000 mon.a (mon.0) 328 : audit [DBG] 2023-10-01T08:59:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:41 smithi190 conmon[100518]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:41 smithi190 conmon[100518]: audit 2023-10-01T08:59:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:41 smithi190 conmon[100518]: 2023-10-01T08:59:40.605900+0000 mon.a (mon.0 2023-10-01T08:59:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:41 smithi190 conmon[100518]: ) 329 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:42.335 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-01T08:59:42.336 INFO:teuthology.orchestra.run.smithi138.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-01T08:59:42.336 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10+0 records in 2023-10-01T08:59:42.336 INFO:teuthology.orchestra.run.smithi138.stderr:10+0 records out 2023-10-01T08:59:42.336 INFO:teuthology.orchestra.run.smithi138.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110674 s, 947 MB/s 2023-10-01T08:59:42.336 INFO:teuthology.orchestra.run.smithi138.stderr:--> Zapping successful for: 2023-10-01T08:59:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: cluster 2023-10-01T08:59:39.610813+0000 osd.2 (osd.2) 2023-10-01T08:59:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T08:59:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: cluster 2023-10-01T08:59:39.610863+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T08:59:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: cluster 2023-10-01T08:59:41.323674+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: audit 2023-10-01T08:59:41.602528+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: cluster 2023-10-01T08:59:41.605235+0000 mon.a (mon.0) 331 : cluster [INF] osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573] boot 2023-10-01T08:59:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: cluster 2023-10-01T08:59:41.605277+0000 mon.a (mon.0) 332 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-01T08:59:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: audit 2023-10-01T08:59:41.606278+0000 mon.a (mon.0) 333 : audit [DBG] 2023-10-01T08:59:43.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:43.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: audit 2023-10-01T08:59:41.606596+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T08:59:43.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: audit 2023-10-01T08:59:42.609719+0000 mon.a (mon.0) 335 2023-10-01T08:59:43.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-01T08:59:43.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: cluster 2023-10-01T08:59:42.609786+0000 mon.a ( 2023-10-01T08:59:43.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: mon.0) 336 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-01T08:59:43.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T08:59:43.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[100289]: 08:59:42.610645+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T08:59:43.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: cluster 2023-10-01T08:59:39.610813+0000 2023-10-01T08:59:43.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T08:59:43.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: cluster 2023-10-01T08:59:39.610863+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T08:59:43.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: cluster 2023-10-01T08:59:41.323674+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:43.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: audit 2023-10-01T08:59:41.602528+0000 mon.a (mon.0 2023-10-01T08:59:43.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: ) 330 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:43.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: cluster 2023-10- 2023-10-01T08:59:43.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: 01T08:59:41.605235+0000 mon.a (mon. 2023-10-01T08:59:43.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: 0) 331 : cluster [INF] osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573] boot 2023-10-01T08:59:43.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: cluster 2023-10-01 2023-10-01T08:59:43.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: T08:59:41.605277+0000 mon.a (mon. 2023-10-01T08:59:43.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: 0) 332 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-01T08:59:43.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T08:59:43.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: 01T08:59:41.606278+0000 mon.a (mon. 2023-10-01T08:59:43.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: 0) 333 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:43.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: audit 2023-10-01T08:59:43.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: 2023-10-01T08:59 2023-10-01T08:59:43.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: :41.606596+0000 mon.a (mon.0) 334 2023-10-01T08:59:43.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T08:59:43.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: audit 2023-10-01T08:59:43.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: 2023-10-01T08:59:42.609719+0000 2023-10-01T08:59:43.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-01T08:59:43.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: 2023-10-01T08:59:43.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: cluster 2023-10-01T08:59:43.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: 2023-10-01T08:59:42.609786 2023-10-01T08:59:43.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: +0000 mon.a (mon.0) 336 : cluster [DBG] 2023-10-01T08:59:43.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: osdmap e19: 3 total, 3 up, 3 in 2023-10-01T08:59:43.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: audit 2023-10-01T08:59:42 2023-10-01T08:59:43.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: .610645+0000 mon.a (mon.0) 337 : 2023-10-01T08:59:43.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:42 smithi138 conmon[109959]: audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T08:59:43.131 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi138:vg_nvme/lv_1 2023-10-01T08:59:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: cluster 2023-10-01T08:59:39.610813+0000 osd.2 (osd. 2023-10-01T08:59:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: 2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T08:59:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: cluster 2023-10-01T08:59:39.610863+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T08:59:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: cluster 2023-10-01T08:59:41.323674+0000 mgr.y (mgr 2023-10-01T08:59:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: .14146) 91 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T08:59:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: audit 2023-10-01T08: 2023-10-01T08:59:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: 59:41.602528+0000 mon.a (mon.0) 330 : audit 2023-10-01T08:59:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: cluster 2023-10-01T08:59:41. 2023-10-01T08:59:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: 605235+0000 mon.a (mon.0) 331 : cluster [INF] 2023-10-01T08:59:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: osd.2 [v2:172.21.15.138:6818/3174817573,v1:172.21.15.138:6819/3174817573] boot 2023-10-01T08:59:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: cluster 2023-10-01T08:59:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: 2023-10-01T08:59:41.605277+0000 mon.a (mon. 2023-10-01T08:59:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: 0) 332 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-01T08:59:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: audit 2023-10-01T08 2023-10-01T08:59:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: :59:41.606278+0000 mon.a (mon.0) 333 : 2023-10-01T08:59:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T08:59:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: audit 2023-10-01T08:59: 2023-10-01T08:59:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: 41.606596+0000 mon.a (mon.0) 334 : audit 2023-10-01T08:59:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T08:59:43.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: audit 2023-10-01T08:59:42.609719 2023-10-01T08:59:43.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: +0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-01T08:59:43.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: 2023-10-01T08:59:43.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: cluster 2023- 2023-10-01T08:59:43.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: 10-01T08:59:42.609786+0000 mon.a (mon.0) 336 2023-10-01T08:59:43.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-01T08:59:43.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: audit 2023-10-01T08:59 2023-10-01T08:59:43.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: :42.610645+0000 mon.a (mon.0) 337 : audit 2023-10-01T08:59:43.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:42 smithi190 conmon[100518]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T08:59:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[100289]: cephadm 2023-10-01T08:59:43.171010+0000 mgr.y (mgr.14146) 92 : cephadm [INF] Detected new or changed devices on smithi138 2023-10-01T08:59:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[100289]: audit 2023-10-01T08:59:43.177232+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[100289]: audit 2023-10-01T08:59:43.178113+0000 mon.a ( 2023-10-01T08:59:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[100289]: mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[100289]: audit 2023-10-01T08:59:43.178723+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[100289]: audit 2023-10-01T08:59:43.179269+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[100289]: cephadm 2023-10-01T08:59:43.179684+0000 mgr.y (mgr.14146) 93 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 5381M 2023-10-01T08:59:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[100289]: audit 2023-10-01T08:59:43.185263+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[100289]: audit 2023-10-01T08:59:43.186120+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:44.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[100289]: audit 2023-10-01T08:59:43.186701+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:59:44.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[100289]: audit 2023-10-01T08:59:43.192714+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:44.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[100289]: cluster 2023-10-01T08:59:43.324002+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v71: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:44.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[100289]: audit 2023-10-01T08:59:43.611122+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-01T08:59:44.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[100289]: cluster 2023-10-01T08:59:43.611182+0000 mon.a (mon.0) 347 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-01T08:59:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[109959]: cephadm 2023-10-01T08:59:43. 2023-10-01T08:59:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[109959]: 171010+0000 mgr.y (mgr.14146) 92 : cephadm [INF] Detected new or changed devices on smithi138 2023-10-01T08:59:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[109959]: audit 2023-10-01T08:59:43.177232+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[109959]: audit 2023-10-01T08:59:43.178113+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[109959]: audit 2023-10-01T08:59:43.178723+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:44.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[109959]: audit 2023-10-01T08:59:43.179269+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:44.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[109959]: cephadm 2023-10-01T08:59:43.179684+0000 mgr.y (mgr.14146) 93 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 5381M 2023-10-01T08:59:44.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[109959]: audit 2023-10-01T08:59:43.185263+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:44.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[109959]: audit 2023-10-01T08:59:43.186120+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:44.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[109959]: audit 2023-10-01T08:59:43.186701+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:59:44.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[109959]: audit 2023-10-01T08:59:43.192714+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:44.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[109959]: cluster 2023-10-01T08:59:43.324002+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v71: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:44.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[109959]: audit 2023-10-01T08:59:43.611122+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-01T08:59:44.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:44 smithi138 conmon[109959]: cluster 2023-10-01T08:59:43.611182+0000 mon.a (mon.0) 347 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-01T08:59:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: cephadm 2023-10-01T08:59:43.171010+0000 mgr.y (mgr.14146) 92 : 2023-10-01T08:59:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: cephadm [INF] Detected new or changed devices on smithi138 2023-10-01T08:59:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: audit 2023-10-01T08:59:43.177232+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: audit 2023-10-01T08:59:43.178113+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: 2023-10-01T08:59:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: audit 2023-10-01T08:59:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: 2023-10-01T08:59:43.178723+0000 mon.a (mon.0) 340 2023-10-01T08:59:44.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:44.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: audit 2023-10-01T08:59:43 2023-10-01T08:59:44.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: .179269+0000 mon.a (mon.0) 341 : audit [INF] 2023-10-01T08:59:44.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T08:59:44.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: cephadm 2023-10-01T08:59:44.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: 2023-10-01T08:59:43.179684+0000 mgr.y (mgr.14146) 93 2023-10-01T08:59:44.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: : cephadm [INF] Adjusting osd_memory_target on smithi138 to 5381M 2023-10-01T08:59:44.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: audit 2023-10-01T08: 2023-10-01T08:59:44.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: 59:43.185263+0000 mon.a (mon.0) 342 : 2023-10-01T08:59:44.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:44.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: audit 2023-10-01T08:59: 2023-10-01T08:59:44.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: 43.186120+0000 mon.a (mon.0) 343 : audit 2023-10-01T08:59:44.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:44.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: audit 2023-10-01T08:59:43 2023-10-01T08:59:44.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: .186701+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T08:59:44.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: 2023-10-01T08:59:44.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: audit 2023-10-01T08:59:44.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: 2023-10-01T08:59:43.192714+0000 mon.a (mon. 2023-10-01T08:59:44.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: 0) 345 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T08:59:44.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: cluster 2023-10-01 2023-10-01T08:59:44.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: T08:59:43.324002+0000 mgr.y (mgr. 2023-10-01T08:59:44.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: 14146) 94 : cluster [DBG] pgmap v71: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:44.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T08:59:44.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: T08:59:43.611122+0000 mon.a (mon. 2023-10-01T08:59:44.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: 0) 346 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-01T08:59:44.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: cluster 2023-10- 2023-10-01T08:59:44.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: 01T08:59:43.611182+0000 mon.a (mon 2023-10-01T08:59:44.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:44 smithi190 conmon[100518]: .0) 347 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-01T08:59:45.996 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:45 smithi138 conmon[100289]: cluster 2023-10-01T08:59:44.618722+0000 mon.a (mon.0) 348 : 2023-10-01T08:59:45.996 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:45 smithi138 conmon[100289]: cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-01T08:59:45.996 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:45 smithi138 conmon[100289]: audit 2023-10-01T08:59:45.070071+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T08:59:45.996 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:45 smithi138 conmon[100289]: audit 2023-10-01T08:59:45.071968+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T08:59:45.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:45 smithi138 conmon[100289]: audit 2023-10-01T08:59:45.072510+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:45.997 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:45 smithi138 conmon[109959]: cluster 2023-10-01T08:59:44.618722+0000 mon.a (mon.0) 2023-10-01T08:59:45.997 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:45 smithi138 conmon[109959]: 348 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-01T08:59:45.997 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:45 smithi138 conmon[109959]: audit 2023-10-01T08:59:45.070071+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T08:59:45.997 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:45 smithi138 conmon[109959]: audit 2023-10-01T08:59:45.071968+0000 mon.a (mon.0) 350 : 2023-10-01T08:59:45.997 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:45 smithi138 conmon[109959]: audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T08:59:45.997 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:45 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T08:59:45.997 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:45 smithi138 conmon[109959]: 08:59:45.072510+0000 mon.a (mon.0) 351 : audit 2023-10-01T08:59:45.997 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:45 smithi138 conmon[109959]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:46.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:45 smithi190 conmon[100518]: cluster 2023-10-01T08:59:44.618722+0000 mon.a (mon.0) 348 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-01T08:59:46.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:45 smithi190 conmon[100518]: audit 2023-10-01T08:59:45.070071+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T08:59:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:45 smithi190 conmon[100518]: audit 2023-10-01T08:59:45.071968+0000 mon.a 2023-10-01T08:59:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:45 smithi190 conmon[100518]: (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T08:59:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:45 smithi190 conmon[100518]: audit 2023-10-01T08:59:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:45 smithi190 conmon[100518]: 2023-10-01T08:59:45.072510+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:46.723 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:46 smithi138 conmon[100289]: audit 2023-10-01T08:59:45.068079+0000 mgr.y ( 2023-10-01T08:59:46.723 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:46 smithi138 conmon[100289]: mgr.14146) 95 : audit [DBG] from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:59:46.723 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:46 smithi138 conmon[100289]: cluster 2023-10-01T08:59:45.324367+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v74: 1 pgs: 1 creating+activating; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:46.723 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:46 smithi138 conmon[109959]: audit 2023-10-01T08:59:45.068079+0000 mgr.y (mgr.14146) 95 2023-10-01T08:59:46.723 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:46 smithi138 conmon[109959]: : audit [DBG] from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:59:46.723 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:46 smithi138 conmon[109959]: cluster 2023-10-01T08:59:45.324367+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v74: 1 pgs: 1 creating+activating; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:47.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:46 smithi190 conmon[100518]: audit 2023-10-01T08:59:45.068079+0000 mgr.y (mgr.14146 2023-10-01T08:59:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:46 smithi190 conmon[100518]: ) 95 : audit [DBG] from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi138:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T08:59:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:46 smithi190 conmon[100518]: cluster 2023-10-01T08:59:45.324367+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v74: 1 pgs: 1 creating+activating; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:48.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:47 smithi190 conmon[100518]: audit 2023-10-01T08:59:47.440874+0000 mon.a (mon.0) 2023-10-01T08:59:48.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:47 smithi190 conmon[100518]: 352 : audit [INF] from='client.? 172.21.15.138:0/941116766' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac51c165-e007-4390-b95b-26eef984d637"}]: dispatch 2023-10-01T08:59:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:47 smithi190 conmon[100518]: audit 2023-10-01T08:59:47.444812+0000 mon.a (mon.0) 353 : audit [INF] from='client.? 172.21.15.138:0/941116766' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ac51c165-e007-4390-b95b-26eef984d637"}]': finished 2023-10-01T08:59:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:47 smithi190 conmon[100518]: cluster 2023-10-01T08:59:47.444878+0000 mon.a (mon.0) 354 2023-10-01T08:59:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:47 smithi190 conmon[100518]: : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-01T08:59:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:47 smithi190 conmon[100518]: audit 2023-10-01T08:59 2023-10-01T08:59:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:47 smithi190 conmon[100518]: :47.445015+0000 mon.a (mon.0) 355 : audit 2023-10-01T08:59:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:47 smithi190 conmon[100518]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T08:59:48.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:47 smithi138 conmon[100289]: audit 2023-10-01T08:59:47.440874+0000 mon.a ( 2023-10-01T08:59:48.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:47 smithi138 conmon[100289]: mon.0) 352 : audit [INF] from='client.? 172.21.15.138:0/941116766' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac51c165-e007-4390-b95b-26eef984d637"}]: dispatch 2023-10-01T08:59:48.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:47 smithi138 conmon[100289]: audit 2023-10-01T08:59:47.444812+0000 mon.a (mon.0) 353 : audit [INF] from='client.? 172.21.15.138:0/941116766' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ac51c165-e007-4390-b95b-26eef984d637"}]': finished 2023-10-01T08:59:48.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:47 smithi138 conmon[100289]: cluster 2023-10-01T08:59:47.444878+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-01T08:59:48.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:47 smithi138 conmon[100289]: audit 2023-10-01T08:59:47.445015+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T08:59:48.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:47 smithi138 conmon[109959]: audit 2023-10-01T08:59:47.440874+0000 mon.a (mon. 2023-10-01T08:59:48.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:47 smithi138 conmon[109959]: 0) 352 : audit [INF] from='client.? 172.21.15.138:0/941116766' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac51c165-e007-4390-b95b-26eef984d637"}]: dispatch 2023-10-01T08:59:48.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:47 smithi138 conmon[109959]: audit 2023-10-01T08:59:47.444812+0000 mon.a (mon.0) 353 : audit [INF] from='client.? 172.21.15.138:0/941116766' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ac51c165-e007-4390-b95b-26eef984d637"}]': finished 2023-10-01T08:59:48.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:47 smithi138 conmon[109959]: cluster 2023-10-01T08:59:47.444878+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-01T08:59:48.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:47 smithi138 conmon[109959]: audit 2023-10-01T08:59:47.445015+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T08:59:49.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:48 smithi190 conmon[100518]: cluster 2023-10-01T08:59:47.324693+0000 mgr.y (mgr.14146) 97 2023-10-01T08:59:49.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:48 smithi190 conmon[100518]: : cluster [DBG] pgmap v75: 1 pgs: 1 creating+activating; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:49.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:48 smithi190 conmon[100518]: audit 2023-10-01T08:59:48.060685+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.138:0/1053722344' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T08:59:49.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:48 smithi138 conmon[100289]: cluster 2023-10-01T08:59:47.324693 2023-10-01T08:59:49.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:48 smithi138 conmon[100289]: +0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v75: 1 pgs: 1 creating+activating; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:49.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:48 smithi138 conmon[100289]: audit 2023-10-01T08:59:48.060685+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.138:0/1053722344' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T08:59:49.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:48 smithi138 conmon[109959]: cluster 2023-10-01T08:59:47.324693+0000 mgr.y (mgr.14146) 97 : cluster 2023-10-01T08:59:49.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:48 smithi138 conmon[109959]: [DBG] pgmap v75: 1 pgs: 1 creating+activating; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:49.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:48 smithi138 conmon[109959]: audit 2023-10-01T08:59:48.060685+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.138:0/1053722344' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T08:59:51.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:50 smithi190 conmon[100518]: cluster 2023-10-01T08:59:49.325032+0000 mgr.y (mgr.14146) 98 2023-10-01T08:59:51.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:50 smithi190 conmon[100518]: : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:51.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:50 smithi138 conmon[100289]: cluster 2023-10-01T08:59:49.325032+0000 2023-10-01T08:59:51.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:50 smithi138 conmon[100289]: mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:51.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:50 smithi138 conmon[109959]: cluster 2023-10-01T08:59:49.325032+0000 mgr.y ( 2023-10-01T08:59:51.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:50 smithi138 conmon[109959]: mgr.14146) 98 : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:52.687 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:52 smithi138 conmon[100289]: cluster 2023-10-01T08:59:51.325305+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:52.688 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:52 smithi138 conmon[109959]: cluster 2023-10-01T08:59:51.325305+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:53.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:52 smithi190 conmon[100518]: cluster 2023-10-01T08:59:51.325305+0000 mgr.y (mgr.14146) 2023-10-01T08:59:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:52 smithi190 conmon[100518]: 99 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:55.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:54 smithi190 conmon[100518]: cluster 2023-10-01T08:59:53.325630+0000 mgr.y (mgr. 2023-10-01T08:59:55.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:54 smithi190 conmon[100518]: 14146) 100 : cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:55.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:54 smithi190 conmon[100518]: audit 2023-10-01T08:59:53.751845+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T08:59:55.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:54 smithi190 conmon[100518]: audit 2023-10-01T08:59:53.752459+0000 mon.a (mon.0) 357 : audit 2023-10-01T08:59:55.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:54 smithi190 conmon[100518]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:55.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:54 smithi190 conmon[100518]: cephadm 2023-10-01T08:59:53.752947+0000 2023-10-01T08:59:55.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:54 smithi190 conmon[100518]: mgr.y (mgr.14146) 101 : cephadm [INF] Deploying daemon osd.3 on smithi138 2023-10-01T08:59:55.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:54 smithi138 conmon[109959]: cluster 2023-10-01T08:59:53.325630+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:55.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:54 smithi138 conmon[109959]: audit 2023-10-01T08:59:53.751845+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T08:59:55.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:54 smithi138 conmon[109959]: audit 2023-10-01T08:59:53.752459+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:55.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:54 smithi138 conmon[109959]: cephadm 2023-10-01T08:59:53.752947+0000 mgr.y (mgr.14146) 101 : cephadm [INF] Deploying daemon osd.3 on smithi138 2023-10-01T08:59:55.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:54 smithi138 conmon[100289]: cluster 2023-10-01T08:59:53.325630+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:55.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:54 smithi138 conmon[100289]: audit 2023-10-01T08:59:53.751845+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T08:59:55.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:54 smithi138 conmon[100289]: audit 2023-10-01T08:59:53.752459+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T08:59:55.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:54 smithi138 conmon[100289]: cephadm 2023-10-01T08:59:53.752947+0000 mgr.y (mgr.14146) 101 : cephadm [INF] Deploying daemon osd.3 on smithi138 2023-10-01T08:59:56.639 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:56 smithi138 conmon[100289]: cluster 2023-10-01T08:59:55.326083+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:57.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:56 smithi190 conmon[100518]: cluster 2023-10-01T08:59:55.326083+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:57.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:56 smithi138 conmon[109959]: cluster 2023-10-01T08:59:55.326083+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:58.902 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:58 smithi138 conmon[109959]: cluster 2023-10-01T08:59:57.326376+0000 mgr.y 2023-10-01T08:59:58.903 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:58 smithi138 conmon[109959]: (mgr.14146) 103 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:58.903 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:58 smithi138 conmon[100289]: cluster 2023-10-01T08:59:57.326376+0000 mgr.y (mgr 2023-10-01T08:59:58.903 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:58 smithi138 conmon[100289]: .14146) 103 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T08:59:59.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:58 smithi190 conmon[100518]: cluster 2023-10-01T08:59:57.326376+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T09:00:00.161 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:59 smithi138 conmon[109959]: audit 2023-10-01T08:59:58.963603+0000 2023-10-01T09:00:00.161 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:59 smithi138 conmon[109959]: mon.a (mon.0) 358 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:00.161 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 08:59:59 smithi138 conmon[109959]: audit 2023-10-01T08:59:58.987195+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:00.161 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:59 smithi138 conmon[100289]: audit 2023-10-01T08:59:58.963603+0000 2023-10-01T09:00:00.162 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:59 smithi138 conmon[100289]: mon.a (mon.0) 358 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:00.162 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 08:59:59 smithi138 conmon[100289]: audit 2023-10-01T08:59:58.987195+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:00.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:59 smithi190 conmon[100518]: audit 2023-10-01T08:59:58.963603+0000 2023-10-01T09:00:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:59 smithi190 conmon[100518]: mon.a (mon.0) 358 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:59 smithi190 conmon[100518]: audit 2023-10-01T08:59:58.987195+0000 mon.a (mon.0) 359 : audit 2023-10-01T09:00:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 08:59:59 smithi190 conmon[100518]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:01.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:00 smithi190 conmon[100518]: cluster 2023-10-01T08:59:59.326668+0000 2023-10-01T09:00:01.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:00 smithi190 conmon[100518]: mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T09:00:01.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:00 smithi190 conmon[100518]: cluster 2023-10-01T09:00:00.000164+0000 mon.a (mon 2023-10-01T09:00:01.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:00 smithi190 conmon[100518]: .0) 360 : cluster [INF] overall HEALTH_OK 2023-10-01T09:00:01.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:00 smithi138 conmon[100289]: cluster 2023-10-01T08:59:59.326668+0000 2023-10-01T09:00:01.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:00 smithi138 conmon[100289]: mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T09:00:01.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:00 smithi138 conmon[100289]: cluster 2023-10-01T09:00:00.000164+0000 mon.a (mon.0) 360 : cluster [INF] overall HEALTH_OK 2023-10-01T09:00:01.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:00 smithi138 conmon[109959]: cluster 2023-10-01T08:59:59.326668+0000 2023-10-01T09:00:01.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:00 smithi138 conmon[109959]: mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T09:00:01.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:00 smithi138 conmon[109959]: cluster 2023-10-01T09:00:00.000164+0000 mon.a (mon.0) 360 : cluster [INF] overall HEALTH_OK 2023-10-01T09:00:02.080 INFO:teuthology.orchestra.run.smithi138.stdout:Created osd(s) 3 on host 'smithi138' 2023-10-01T09:00:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: audit 2023-10-01T09:00:01.197749+0000 mon.a (mon.0) 361 : audit 2023-10-01T09:00:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: cluster 2023-10-01T09:00:01.326981+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T09:00:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: audit 2023-10-01T09:00:01.526166+0000 mon.a (mon.0) 362 : audit [DBG] 2023-10-01T09:00:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: audit 2023-10-01T09:00:01.526876 2023-10-01T09:00:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: +0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: audit 2023-10-01T09:00:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: 2023-10-01T09:00:01.531659+0000 mon.a 2023-10-01T09:00:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: (mon.0) 364 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:00:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: 10-01T09:00:02.078768+0000 mon.a 2023-10-01T09:00:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: (mon.0) 365 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: audit 2023-10-01T09:00:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: 2023-10-01T09:00:02.099003+0000 2023-10-01T09:00:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: mon.a (mon.0) 366 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:00:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: 09:00:02.169643+0000 mon.c (mon.2) 2023-10-01T09:00:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: 16 : audit [INF] from='osd.3 [v2:172.21.15.138:6826/1517480090,v1:172.21.15.138:6827/1517480090]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T09:00:02.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:00:02.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: :00:02.170030+0000 mon.a (mon.0) 367 : audit 2023-10-01T09:00:02.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[100289]: [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T09:00:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: audit 2023-10-01T09:00:01.197749+0000 mon.a ( 2023-10-01T09:00:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: mon.0) 361 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: cluster 2023-10-01T09:00 2023-10-01T09:00:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: :01.326981+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T09:00:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: audit 2023 2023-10-01T09:00:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: -10-01T09:00:01.526166+0000 mon.a ( 2023-10-01T09:00:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: mon.0) 362 : audit [DBG] 2023-10-01T09:00:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: audit 2023 2023-10-01T09:00:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: -10-01T09:00:01.526876+0000 mon.a 2023-10-01T09:00:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: (mon.0) 363 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: audit 2023-10-01T09:00:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: 2023-10-01T09:00:01. 2023-10-01T09:00:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: 531659+0000 mon.a (mon.0) 364 2023-10-01T09:00:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:02.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:00:02.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: 09:00:02.078768+0000 mon.a (mon.0) 365 : audit [INF] 2023-10-01T09:00:02.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:02.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: audit 2023-10-01T09:00:02.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: 2023-10-01T09:00:02.099003+0000 mon.a (mon.0) 366 : audit [DBG] 2023-10-01T09:00:02.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:02.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: audit 2023-10-01T09:00: 2023-10-01T09:00:02.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: 02.169643+0000 mon.c (mon.2) 16 : audit 2023-10-01T09:00:02.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: [INF] from='osd.3 [v2:172.21.15.138:6826/1517480090,v1:172.21.15.138:6827/1517480090]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T09:00:02.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:00:02.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: 09:00:02.170030+0000 mon.a (mon.0) 367 2023-10-01T09:00:02.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:02 smithi138 conmon[109959]: : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T09:00:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: audit 2023-10-01T09:00:01.197749+0000 mon.a (mon.0) 2023-10-01T09:00:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: 361 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: cluster 2023-10-01T09:00:01.326981+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T09:00:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: audit 2023-10-01T09: 2023-10-01T09:00:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: 00:01.526166+0000 mon.a (mon.0) 2023-10-01T09:00:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: 362 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T09:00:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: T09:00:01.526876+0000 mon.a (mon. 2023-10-01T09:00:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: 0) 363 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:00:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: -01T09:00:01.531659+0000 mon.a 2023-10-01T09:00:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: (mon.0) 364 : audit [INF] 2023-10-01T09:00:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: audit 2023-10-01T09:00:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: 2023-10-01T09:00:02 2023-10-01T09:00:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: .078768+0000 mon.a (mon.0) 2023-10-01T09:00:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: 365 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:00:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: 10-01T09:00:02.099003+0000 2023-10-01T09:00:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: mon.a (mon.0) 366 : audit 2023-10-01T09:00:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:02.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:00:02.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: 10-01T09:00:02.169643+0000 2023-10-01T09:00:02.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: mon.c (mon.2) 16 : audit 2023-10-01T09:00:02.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: [INF] from='osd.3 [v2:172.21.15.138:6826/1517480090,v1:172.21.15.138:6827/1517480090]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T09:00:02.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: audit 2023 2023-10-01T09:00:02.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: -10-01T09:00:02.170030+0000 2023-10-01T09:00:02.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: mon.a (mon.0) 367 : audit 2023-10-01T09:00:02.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:02 smithi190 conmon[100518]: [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T09:00:02.703 DEBUG:teuthology.orchestra.run.smithi138:osd.3> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-01T09:00:02.705 INFO:tasks.cephadm:Deploying osd.4 on smithi190 with /dev/vg_nvme/lv_4... 2023-10-01T09:00:02.705 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-01T09:00:02.738 INFO:journalctl@ceph.osd.3.smithi138.stdout:-- Logs begin at Sun 2023-10-01 08:38:26 UTC. -- 2023-10-01T09:00:04.156 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:04 smithi190 conmon[100518]: audit 2023-10-01T09:00:03.080698+0000 mon.a (mon.0) 368 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T09:00:04.156 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:04 smithi190 conmon[100518]: cluster 2023-10-01T09:00:03.080797+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-01T09:00:04.156 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:04 smithi190 conmon[100518]: audit 2023-10-01T09:00:03.080910+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:04.157 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:04 smithi190 conmon[100518]: audit 2023-10-01T09:00:03.081326+0000 mon.c (mon.2) 17 : audit [INF] from='osd.3 [v2:172.21.15.138:6826/1517480090,v1:172.21.15.138:6827/1517480090]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:00:04.157 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:04 smithi190 conmon[100518]: audit 2023-10-01T09:00:03.081624+0000 mon.a (mon.0) 371 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:00:04.157 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:04 smithi190 conmon[100518]: cluster 2023-10-01T09:00:03.327342+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T09:00:04.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[100289]: audit 2023-10-01T09:00:03.080698+0000 mon.a (mon.0) 368 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T09:00:04.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[100289]: cluster 2023-10-01T09:00:03.080797+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-01T09:00:04.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[100289]: audit 2023-10-01T09:00:03.080910+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:04.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[100289]: audit 2023-10-01T09:00:03.081326+0000 mon.c (mon.2) 17 : audit [INF] from='osd.3 [v2:172.21.15.138:6826/1517480090,v1:172.21.15.138:6827/1517480090]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:00:04.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[100289]: audit 2023-10-01T09:00:03.081624+0000 mon.a (mon.0) 371 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:00:04.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[100289]: cluster 2023-10-01T09:00:03.327342+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T09:00:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[109959]: audit 2023-10-01T09:00:03.080698+0000 mon.a (mon.0) 368 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T09:00:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[109959]: cluster 2023-10-01T09:00:03.080797+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-01T09:00:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[109959]: audit 2023-10-01T09:00:03.080910+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[109959]: audit 2023-10-01T09:00:03.081326+0000 mon.c (mon.2) 17 : audit [INF] from='osd.3 [v2:172.21.15.138:6826/1517480090,v1:172.21.15.138:6827/1517480090]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:00:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[109959]: audit 2023-10-01T09:00:03.081624+0000 mon.a (mon.0) 371 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:00:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[109959]: cluster 2023 2023-10-01T09:00:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[109959]: -10-01T09:00:03.327342+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T09:00:04.323 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[132596]: debug 2023-10-01T09:00:04.087+0000 7f9232aa0700 -1 osd.3 0 waiting for initial osdmap 2023-10-01T09:00:04.323 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:00:04 smithi138 conmon[132596]: debug 2023-10-01T09:00:04.091+0000 7f922ca17700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T09:00:04.583 INFO:teuthology.orchestra.run.smithi190.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-01T09:00:04.583 INFO:teuthology.orchestra.run.smithi190.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-01T09:00:04.583 INFO:teuthology.orchestra.run.smithi190.stderr: stderr: 10+0 records in 2023-10-01T09:00:04.583 INFO:teuthology.orchestra.run.smithi190.stderr:10+0 records out 2023-10-01T09:00:04.583 INFO:teuthology.orchestra.run.smithi190.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0128241 s, 818 MB/s 2023-10-01T09:00:04.583 INFO:teuthology.orchestra.run.smithi190.stderr:--> Zapping successful for: 2023-10-01T09:00:05.164 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi190:vg_nvme/lv_4 2023-10-01T09:00:05.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:05 smithi138 conmon[109959]: audit 2023-10-01T09:00:04.082256+0000 2023-10-01T09:00:05.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:05 smithi138 conmon[109959]: mon.a (mon.0) 372 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-10-01T09:00:05.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:05 smithi138 conmon[109959]: cluster 2023-10-01T09:00:04.082354+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-01T09:00:05.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:05 smithi138 conmon[109959]: audit 2023-10-01T09:00:04.082744+0000 mon.a (mon.0) 374 : audit [DBG] 2023-10-01T09:00:05.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:05 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:05.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:05 smithi138 conmon[109959]: audit 2023-10-01T09:00 2023-10-01T09:00:05.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:05 smithi138 conmon[109959]: :04.087819+0000 mon.a (mon.0) 375 : audit 2023-10-01T09:00:05.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:05 smithi138 conmon[109959]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:05.453 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:05 smithi138 conmon[100289]: audit 2023-10-01T09:00:04.082256+0000 mon.a (mon.0) 372 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-10-01T09:00:05.453 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:05 smithi138 conmon[100289]: cluster 2023-10-01T09:00:05.453 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:05 smithi138 conmon[100289]: 2023-10-01T09:00:04.082354+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-01T09:00:05.453 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:05 smithi138 conmon[100289]: audit 2023-10-01T09:00:04.082744+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:05.453 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:05 smithi138 conmon[100289]: audit 2023-10-01T09:00:04.087819+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:05.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:05 smithi190 conmon[100518]: audit 2023-10-01T09:00:04.082256+0000 mon.a (mon.0) 372 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]': finished 2023-10-01T09:00:05.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:05 smithi190 conmon[100518]: cluster 2023-10-01T09:00:04.082354+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-01T09:00:05.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:05 smithi190 conmon[100518]: audit 2023-10-01T09:00:04.082744+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:05.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:05 smithi190 conmon[100518]: audit 2023-10-01T09:00:04.087819+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: cluster 2023-10-01T09:00:03.127830+0000 osd.3 (osd.3) 2023-10-01T09:00:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T09:00:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: cluster 2023-10-01T09:00:03.127911+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T09:00:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: audit 2023-10-01T09:00:05.086158+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: cluster 2023-10-01T09:00:05.092032+0000 mon.a (mon.0) 377 : cluster [INF] osd.3 [v2:172.21.15.138:6826/1517480090,v1:172.21.15.138:6827/1517480090] boot 2023-10-01T09:00:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: cluster 2023-10-01T09:00:05.092065+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-01T09:00:06.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: audit 2023-10-01T09:00:05.092562+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:06.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: cluster 2023-10-01T09:00:05.327610+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T09:00:06.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: cephadm 2023-10-01T09:00:05.709592+0000 mgr.y (mgr.14146) 108 : cephadm [INF] Detected new or changed devices on smithi138 2023-10-01T09:00:06.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: audit 2023-10-01T09:00:05.714520+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:06.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: audit 2023-10-01T09:00:05.715372+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:06.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: audit 2023-10-01T09:00:05.715907+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:06.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: audit 2023-10-01T09:00:05.716385+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:06.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: audit 2023-10-01T09:00:05.716849+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:06.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: cephadm 2023-10-01T09:00:05.717214+0000 mgr.y (mgr.14146) 109 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 4036M 2023-10-01T09:00:06.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: audit 2023-10-01T09:00:05.720297+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:06.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: audit 2023-10-01T09:00:05.721296+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:06.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: audit 2023-10-01T09:00:05.721895+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:06.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:06 smithi190 conmon[100518]: audit 2023-10-01T09:00:05.728116+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: cluster 2023-10-01T09:00:03.127830+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T09:00:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: 2023-10-01T09:00:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: cluster 2023-10-01T09:00:03.127911+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T09:00:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: audit 2023-10-01T09:00:05.086158+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: cluster 2023-10-01T09:00:05.092032+0000 mon.a (mon.0) 377 : cluster [INF] osd.3 [v2:172.21.15.138:6826/1517480090,v1:172.21.15.138:6827/1517480090] boot 2023-10-01T09:00:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: cluster 2023-10-01T09:00:05.092065+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-01T09:00:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: audit 2023-10-01T09:00:05.092562+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: cluster 2023-10-01T 2023-10-01T09:00:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: 09:00:05.327610+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T09:00:06.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: cephadm 2023-10-01T09:00:06.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: 2023-10-01T09:00:05.709592+0000 mgr.y (mgr.14146) 108 : cephadm [INF] Detected new or changed devices on smithi138 2023-10-01T09:00:06.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:00:06.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: 00:05.714520+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:06.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:00:06.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: 09:00:05.715372+0000 mon.a (mon.0) 381 : audit [INF] 2023-10-01T09:00:06.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:06.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: audit 2023-10 2023-10-01T09:00:06.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: -01T09:00:05.715907+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:06.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: audit 2023-10-01T09:00:05. 2023-10-01T09:00:06.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: 716385+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: audit 2023-10-01T09:00: 2023-10-01T09:00:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: 05.716849+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: cephadm 2023-10-01T09:00:05 2023-10-01T09:00:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: .717214+0000 mgr.y (mgr.14146) 109 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 4036M 2023-10-01T09:00:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: audit 2023-10-01T09:00: 2023-10-01T09:00:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: 05.720297+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:00:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: :00:05.721296+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: 2023-10-01T09:00:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:00:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: 09:00:05.721895+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: audit 2023-10-01T09:00:05.728116+0000 2023-10-01T09:00:06.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[100289]: mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:06.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: cluster 2023-10-01T09:00:03.127830+0000 osd.3 (osd 2023-10-01T09:00:06.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: .3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T09:00:06.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: cluster 2023-10-01T09:00:03.127911+0000 osd.3 (osd.3) 2 2023-10-01T09:00:06.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: : cluster [DBG] purged_snaps scrub ok 2023-10-01T09:00:06.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:00:06.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 09:00:05.086158+0000 mon.a (mon.0) 376 2023-10-01T09:00:06.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: cluster 2023-10-01T09:00:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 2023-10-01T09:00:05.092032+0000 mon.a ( 2023-10-01T09:00:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: mon.0) 377 : cluster [INF] osd.3 [v2:172.21.15.138:6826/1517480090,v1:172.21.15.138:6827/1517480090] boot 2023-10-01T09:00:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: cluster 2023- 2023-10-01T09:00:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 10-01T09:00:05.092065 2023-10-01T09:00:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: +0000 mon.a (mon.0) 378 : 2023-10-01T09:00:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-01T09:00:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: audit 2023-10-01T09:00:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 2023-10-01T09 2023-10-01T09:00:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: :00:05.092562+0000 mon.a (mon.0) 379 2023-10-01T09:00:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:00:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: cluster 2023-10-01T09:00:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 2023-10-01T09:00:05. 2023-10-01T09:00:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 327610+0000 mgr.y (mgr.14146) 107 2023-10-01T09:00:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T09:00:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: cephadm 2023 2023-10-01T09:00:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: -10-01T09:00:05. 2023-10-01T09:00:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 709592+0000 mgr.y (mgr.14146) 108 2023-10-01T09:00:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: : cephadm [INF] Detected new or changed devices on smithi138 2023-10-01T09:00:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:00:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 10-01T09:00:05.714520+0000 2023-10-01T09:00:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: mon.a (mon.0) 380 : audit 2023-10-01T09:00:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:06.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: audit 2023 2023-10-01T09:00:06.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: -10-01T09:00:05 2023-10-01T09:00:06.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: .715372+0000 mon.a (mon.0 2023-10-01T09:00:06.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: ) 381 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:06.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:00:06.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: -01T09:00:05.715907+0000 mon.a (mon.0) 382 : 2023-10-01T09:00:06.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:06.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: audit 2023-10-01T09:00:05. 2023-10-01T09:00:06.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 716385+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:06.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 2023-10-01T09:00:06.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: audit 2023-10-01T09:00:06.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 2023-10-01T09:00:05.716849+0000 mon.a (mon.0) 2023-10-01T09:00:06.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 384 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:06.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: cephadm 2023-10-01T09: 2023-10-01T09:00:06.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 00:05.717214+0000 mgr.y (mgr.14146) 109 : cephadm 2023-10-01T09:00:06.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: [INF] Adjusting osd_memory_target on smithi138 to 4036M 2023-10-01T09:00:06.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:00:06.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 00:05.720297+0000 mon.a (mon.0) 385 2023-10-01T09:00:06.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:06.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:00:06.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: T09:00:05.721296+0000 mon.a (mon.0) 2023-10-01T09:00:06.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 386 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:06.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: audit 2023-10-01T09:00:05. 2023-10-01T09:00:06.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 721895+0000 mon.a (mon.0) 387 : audit 2023-10-01T09:00:06.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:06.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:00:06.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: 01T09:00:05.728116 2023-10-01T09:00:06.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: +0000 mon.a (mon.0) 388 2023-10-01T09:00:06.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:06 smithi138 conmon[109959]: : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:07.421 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:07 smithi190 conmon[100518]: cluster 2023-10-01T09:00:06.102673+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-01T09:00:07.421 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:07 smithi190 conmon[100518]: audit 2023-10-01T09:00:06.926041+0000 mgr.y (mgr.14146) 110 : audit [DBG] from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi190:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:00:07.421 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:07 smithi190 conmon[100518]: audit 2023-10-01T09:00:06.928584+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T09:00:07.421 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:07 smithi190 conmon[100518]: audit 2023-10-01T09:00:06.931127+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T09:00:07.421 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:07 smithi190 conmon[100518]: audit 2023-10-01T09:00:06.931854+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:07.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[100289]: cluster 2023-10-01T09:00:06.102673+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-01T09:00:07.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[100289]: audit 2023-10-01T09:00: 2023-10-01T09:00:07.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[100289]: 06.926041+0000 mgr.y (mgr.14146) 110 : audit [DBG] from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi190:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:00:07.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[100289]: audit 2023-10-01T09:00:06.928584+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T09:00:07.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[100289]: audit 2023-10-01T09:00:06.931127+0000 mon.a (mon.0) 391 : audit 2023-10-01T09:00:07.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[100289]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T09:00:07.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[100289]: audit 2023-10-01T09:00:06.931854+0000 mon.a (mon.0) 2023-10-01T09:00:07.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[100289]: 392 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:07.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[109959]: cluster 2023-10-01T09:00:06.102673+0000 mon.a (mon.0) 389 : cluster [DBG] 2023-10-01T09:00:07.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[109959]: osdmap e26: 4 total, 4 up, 4 in 2023-10-01T09:00:07.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[109959]: audit 2023-10-01T09:00:06.926041+0000 mgr.y (mgr.14146) 110 : audit [DBG] from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi190:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:00:07.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[109959]: audit 2023-10-01T09:00:06.928584+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T09:00:07.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[109959]: 2023-10-01T09:00:07.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[109959]: audit 2023-10-01T09:00:06.931127+0000 mon.a (mon 2023-10-01T09:00:07.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[109959]: .0) 391 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T09:00:07.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:00:07.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:07 smithi138 conmon[109959]: 09:00:06.931854+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:08.382 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:08 smithi190 conmon[100518]: cluster 2023-10-01T09:00:07.327993+0000 mgr.y (mgr. 2023-10-01T09:00:08.382 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:08 smithi190 conmon[100518]: 14146) 111 : cluster [DBG] pgmap v90: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:08 smithi138 conmon[100289]: cluster 2023-10-01T09:00:07.327993+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v90: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:08 smithi138 conmon[109959]: cluster 2023-10-01T09:00:07.327993 2023-10-01T09:00:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:08 smithi138 conmon[109959]: +0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v90: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:09.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:09 smithi190 conmon[100518]: audit 2023-10-01T09:00:09.075908+0000 mon.b (mon 2023-10-01T09:00:09.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:09 smithi190 conmon[100518]: .1) 4 : audit [INF] from='client.? 172.21.15.190:0/2924517429' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bef630c-bff4-41a3-a932-de0501541f78"}]: dispatch 2023-10-01T09:00:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:09 smithi190 conmon[100518]: audit 2023-10-01T09:00:09.076566+0000 mon.a (mon.0) 393 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bef630c-bff4-41a3-a932-de0501541f78"}]: dispatch 2023-10-01T09:00:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:09 smithi190 conmon[100518]: audit 2023-10-01T09:00:09.084351+0000 mon.a (mon.0) 394 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4bef630c-bff4-41a3-a932-de0501541f78"}]': finished 2023-10-01T09:00:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:09 smithi190 conmon[100518]: cluster 2023-10-01T09:00:09.084457+0000 mon.a (mon.0) 395 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-01T09:00:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:09 smithi190 conmon[100518]: audit 2023-10-01T09:00:09.084630+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[100289]: audit 2023-10-01T09:00:09.075908+0000 mon.b (mon.1) 4 2023-10-01T09:00:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[100289]: : audit [INF] from='client.? 172.21.15.190:0/2924517429' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bef630c-bff4-41a3-a932-de0501541f78"}]: dispatch 2023-10-01T09:00:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[100289]: audit 2023-10-01T09:00:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[100289]: 2023-10-01T09:00:09.076566+0000 mon.a (mon.0 2023-10-01T09:00:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[100289]: ) 393 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bef630c-bff4-41a3-a932-de0501541f78"}]: dispatch 2023-10-01T09:00:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:00:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[100289]: 01T09:00:09.084351+0000 mon.a (mon.0) 394 : audit 2023-10-01T09:00:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[100289]: [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4bef630c-bff4-41a3-a932-de0501541f78"}]': finished 2023-10-01T09:00:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[100289]: cluster 2023-10 2023-10-01T09:00:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[100289]: -01T09:00:09.084457+0000 mon.a (mon.0) 395 : cluster 2023-10-01T09:00:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[100289]: [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-01T09:00:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:00:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[100289]: 09:00:09.084630+0000 mon.a (mon.0) 396 : audit 2023-10-01T09:00:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[100289]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:09.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[109959]: audit 2023-10-01T09:00:09.075908 2023-10-01T09:00:09.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[109959]: +0000 mon.b (mon.1) 4 : audit [INF] from='client.? 172.21.15.190:0/2924517429' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bef630c-bff4-41a3-a932-de0501541f78"}]: dispatch 2023-10-01T09:00:09.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[109959]: audit 2023-10-01T09:00:09.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[109959]: 2023-10-01T09:00:09.076566+0000 mon.a 2023-10-01T09:00:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[109959]: (mon.0) 393 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4bef630c-bff4-41a3-a932-de0501541f78"}]: dispatch 2023-10-01T09:00:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[109959]: audit 2023 2023-10-01T09:00:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[109959]: -10-01T09:00:09.084351+0000 mon.a (mon.0 2023-10-01T09:00:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[109959]: ) 394 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4bef630c-bff4-41a3-a932-de0501541f78"}]': finished 2023-10-01T09:00:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[109959]: cluster 2023-10-01T09:00:09. 2023-10-01T09:00:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[109959]: 084457+0000 mon.a (mon.0) 395 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-01T09:00:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[109959]: audit 2023-10-01T09:00:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[109959]: 2023-10-01T09:00:09.084630+0000 2023-10-01T09:00:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[109959]: mon.a (mon.0) 396 : audit [DBG] 2023-10-01T09:00:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:09 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:10 smithi190 conmon[100518]: cluster 2023-10-01T09:00:09.328362+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v92: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:10 smithi190 conmon[100518]: audit 2023-10-01T09:00:09.631954+0000 mon.b (mon.1) 5 : audit [DBG] from='client.? 172.21.15.190:0/1110472061' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T09:00:10.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:10 smithi138 conmon[100289]: cluster 2023-10-01T09:00:09.328362+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v92: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:10.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:10 smithi138 conmon[100289]: audit 2023-10-01T09:00:09.631954+0000 mon.b (mon.1) 5 : audit [DBG] from='client.? 172.21.15.190:0/1110472061' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T09:00:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:10 smithi138 conmon[109959]: cluster 2023-10-01T09:00:09.328362+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v92: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:10 smithi138 conmon[109959]: audit 2023-10-01T09:00:09.631954+0000 mon.b (mon.1) 5 : audit [DBG] from='client.? 172.21.15.190:0/1110472061' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T09:00:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:12 smithi190 conmon[100518]: cluster 2023-10-01T09:00:11.328806+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:12.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:12 smithi138 conmon[100289]: cluster 2023-10-01T09:00:11.328806+0000 mgr.y (mgr.14146) 113 : cluster 2023-10-01T09:00:12.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:12 smithi138 conmon[100289]: [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:12 smithi138 conmon[109959]: cluster 2023-10-01T09:00:11.328806 2023-10-01T09:00:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:12 smithi138 conmon[109959]: +0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:14 smithi190 conmon[100518]: cluster 2023-10-01T09:00:13.329334+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:14.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:14 smithi138 conmon[100289]: cluster 2023-10-01T09:00:13.329334+0000 mgr.y (mgr 2023-10-01T09:00:14.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:14 smithi138 conmon[100289]: .14146) 114 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:14 smithi138 conmon[109959]: cluster 2023-10-01T09:00:13.329334+0000 mgr.y (mgr.14146) 2023-10-01T09:00:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:14 smithi138 conmon[109959]: 114 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:15.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:15 smithi190 conmon[100518]: audit 2023-10-01T09:00:15.010796+0000 mon.a (mon.0) 397 : audit 2023-10-01T09:00:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:15 smithi190 conmon[100518]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T09:00:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:15 smithi190 conmon[100518]: audit 2023-10-01T09:00:15.011635+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:15.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:15 smithi138 conmon[109959]: audit 2023-10-01T09:00:15.010796+0000 mon.a (mon.0) 397 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T09:00:15.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:15 smithi138 conmon[109959]: audit 2023-10-01T09:00:15.011635+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:15 smithi138 conmon[100289]: audit 2023-10-01T09:00:15.010796+0000 mon.a (mon.0) 397 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T09:00:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:15 smithi138 conmon[100289]: audit 2023-10-01T09:00:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:15 smithi138 conmon[100289]: 2023-10-01T09:00:15.011635+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:16.467 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:16 smithi190 conmon[100518]: cephadm 2023-10-01T09:00:15.012323+0000 mgr.y (mgr. 2023-10-01T09:00:16.467 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:16 smithi190 conmon[100518]: 14146) 115 : cephadm [INF] Deploying daemon osd.4 on smithi190 2023-10-01T09:00:16.757 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:16 smithi190 conmon[100518]: cluster 2023-10-01T09:00:15.329842+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:16.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:16 smithi138 conmon[109959]: cephadm 2023-10-01T09:00:15.012323+0000 mgr.y (mgr.14146) 115 : cephadm [INF] Deploying daemon osd.4 on smithi190 2023-10-01T09:00:16.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:16 smithi138 conmon[109959]: cluster 2023-10-01T09:00:15.329842+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:16 smithi138 conmon[100289]: cephadm 2023-10-01T09:00:15.012323+0000 mgr.y (mgr 2023-10-01T09:00:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:16 smithi138 conmon[100289]: .14146) 115 : cephadm [INF] Deploying daemon osd.4 on smithi190 2023-10-01T09:00:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:16 smithi138 conmon[100289]: cluster 2023-10-01T09:00:15.329842+0000 mgr.y (mgr.14146) 116 : cluster [DBG] 2023-10-01T09:00:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:16 smithi138 conmon[100289]: pgmap v95: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:18 smithi190 conmon[100518]: cluster 2023-10-01T09:00:17.330311+0000 mgr.y (mgr.14146) 117 2023-10-01T09:00:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:18 smithi190 conmon[100518]: : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:18.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:18 smithi138 conmon[100289]: cluster 2023-10-01T09:00:17.330311+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:18.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:18 smithi138 conmon[109959]: cluster 2023-10-01T09:00:17.330311+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:18.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:18 smithi138 conmon[109959]: 2023-10-01T09:00:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:20 smithi190 conmon[100518]: cluster 2023-10-01T09:00:19.330726+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:20 smithi190 conmon[100518]: audit 2023-10-01T09:00:20.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:20 smithi190 conmon[100518]: 2023-10-01T09:00:19.679175+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:20.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:20 smithi190 conmon[100518]: audit 2023-10-01T09:00:19.679450+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:20 smithi138 conmon[100289]: cluster 2023-10-01T09:00:19.330726+0000 mgr.y (mgr.14146) 118 : cluster [DBG] 2023-10-01T09:00:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:20 smithi138 conmon[100289]: pgmap v97: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:20 smithi138 conmon[100289]: audit 2023-10-01T09:00:19.679175+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:20 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:00:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:20 smithi138 conmon[100289]: :00:19.679450+0000 mon.a (mon. 2023-10-01T09:00:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:20 smithi138 conmon[100289]: 0) 400 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:21.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:20 smithi138 conmon[109959]: cluster 2023-10-01T09:00:19.330726+0000 mgr.y (mgr.14146) 118 2023-10-01T09:00:21.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:20 smithi138 conmon[109959]: : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:21.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:20 smithi138 conmon[109959]: audit 2023-10-01T09:00:19.679175+0000 mon.a (mon.0) 2023-10-01T09:00:21.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:20 smithi138 conmon[109959]: 399 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:21.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:20 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:00:21.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:20 smithi138 conmon[109959]: :00:19.679450+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:22.633 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:22 smithi190 conmon[100518]: cluster 2023-10-01T09:00:21.331059+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:22.633 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:22 smithi190 conmon[100518]: audit 2023-10-01T09:00:21.371400+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:22.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:22 smithi190 conmon[100518]: audit 2023-10-01T09:00:21.691756+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:22.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:22 smithi190 conmon[100518]: audit 2023-10-01T09:00:21.692766+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:22.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:22 smithi190 conmon[100518]: audit 2023-10-01T09:00:21.697412+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:22.656 INFO:teuthology.orchestra.run.smithi190.stdout:Created osd(s) 4 on host 'smithi190' 2023-10-01T09:00:22.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[100289]: cluster 2023-10-01T09:00:21.331059+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[100289]: 2023-10-01T09:00:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[100289]: audit 2023-10-01T09:00:21.371400+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[100289]: audit 2023-10-01T09:00:21.691756+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[100289]: audit 2023-10-01T09:00:21.692766+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[100289]: audit 2023-10-01T09:00:21.697412 2023-10-01T09:00:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[100289]: +0000 mon.a (mon.0) 404 : audit [INF] 2023-10-01T09:00:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[100289]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[109959]: cluster 2023-10-01T09:00:21.331059+0000 mgr.y (mgr.14146) 119 2023-10-01T09:00:22.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[109959]: : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:22.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[109959]: audit 2023-10-01T09:00:21.371400+0000 mon.a (mon.0) 401 : audit 2023-10-01T09:00:22.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[109959]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:22.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:00:22.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[109959]: 10-01T09:00:21.691756+0000 mon.a (mon.0) 402 : 2023-10-01T09:00:22.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[109959]: audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:22.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[109959]: audit 2023 2023-10-01T09:00:22.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[109959]: -10-01T09:00:21.692766+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:22.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:00:22.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:22 smithi138 conmon[109959]: :00:21.697412+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:23.175 DEBUG:teuthology.orchestra.run.smithi190:osd.4> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-01T09:00:23.177 INFO:tasks.cephadm:Deploying osd.5 on smithi190 with /dev/vg_nvme/lv_3... 2023-10-01T09:00:23.177 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-01T09:00:23.207 INFO:journalctl@ceph.osd.4.smithi190.stdout:-- Logs begin at Sun 2023-10-01 08:38:18 UTC. -- 2023-10-01T09:00:23.744 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:23 smithi190 conmon[100518]: audit 2023-10-01T09:00:22.655359+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:23.744 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:23 smithi190 conmon[100518]: audit 2023-10-01T09:00:22.665915+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:23.744 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:23 smithi190 conmon[100518]: audit 2023-10-01T09:00:22.882284+0000 mon.b (mon.1) 6 : audit [INF] from='osd.4 [v2:172.21.15.190:6800/743095145,v1:172.21.15.190:6801/743095145]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T09:00:23.745 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:23 smithi190 conmon[100518]: audit 2023-10-01T09:00:22.882727+0000 mon.a (mon.0) 407 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T09:00:24.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:23 smithi138 conmon[109959]: audit 2023-10-01T09:00:22.655359+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:24.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:23 smithi138 conmon[109959]: audit 2023-10-01T09:00:22.665915+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:24.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:23 smithi138 conmon[109959]: audit 2023-10-01T09:00:22.882284+0000 mon.b (mon.1) 6 : audit [INF] from='osd.4 [v2:172.21.15.190:6800/743095145,v1:172.21.15.190:6801/743095145]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T09:00:24.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:23 smithi138 conmon[109959]: audit 2023-10-01T09:00:22.882727+0000 mon.a (mon.0) 407 : audit 2023-10-01T09:00:24.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:23 smithi138 conmon[109959]: [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T09:00:24.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:23 smithi138 conmon[100289]: audit 2023-10-01T09:00:22.655359+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:24.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:23 smithi138 conmon[100289]: 2023-10-01T09:00:24.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:23 smithi138 conmon[100289]: audit 2023-10-01T09:00:22.665915+0000 mon.a (mon. 2023-10-01T09:00:24.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:23 smithi138 conmon[100289]: 0) 406 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:24.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:23 smithi138 conmon[100289]: audit 2023-10-01T09:00:24.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:23 smithi138 conmon[100289]: 2023-10-01T09:00:22.882284+0000 mon.b (mon.1) 6 : audit [INF] from='osd.4 [v2:172.21.15.190:6800/743095145,v1:172.21.15.190:6801/743095145]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T09:00:24.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:23 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T09:00:24.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:23 smithi138 conmon[100289]: T09:00:22.882727+0000 mon.a (mon.0) 407 : audit [INF] 2023-10-01T09:00:24.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:23 smithi138 conmon[100289]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T09:00:24.980 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:24 smithi190 conmon[100518]: cluster 2023-10-01T09:00:23.331492+0000 mgr.y (mgr.14146) 120 : cluster [DBG] 2023-10-01T09:00:24.980 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:24 smithi190 conmon[100518]: pgmap v99: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:24.981 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:24 smithi190 conmon[100518]: audit 2023-10-01T09:00:23.665070+0000 mon.a (mon.0) 408 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-01T09:00:24.981 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:24 smithi190 conmon[100518]: cluster 2023-10-01T09:00:23.665162+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-01T09:00:24.981 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:24 smithi190 conmon[100518]: audit 2023-10-01T09:00:23.665471+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:24.981 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:24 smithi190 conmon[100518]: audit 2023-10-01T09:00:23.665988+0000 mon.b (mon.1) 7 : audit [INF] from='osd.4 [v2:172.21.15.190:6800/743095145,v1:172.21.15.190:6801/743095145]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:00:24.981 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:24 smithi190 conmon[100518]: audit 2023-10-01T09:00:23.666342+0000 mon.a (mon.0) 411 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:00:24.981 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:00:24 smithi190 conmon[106808]: debug 2023-10-01T09:00:24.668+0000 7fee3b1f7700 -1 osd.4 0 waiting for initial osdmap 2023-10-01T09:00:24.981 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:00:24 smithi190 conmon[106808]: debug 2023-10-01T09:00:24.673+0000 7fee3516e700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T09:00:25.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[100289]: cluster 2023-10-01T09:00:23.331492+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:25.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[100289]: audit 2023-10-01T09:00:23.665070+0000 mon.a (mon.0) 408 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-01T09:00:25.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[100289]: cluster 2023-10-01T09:00:23.665162+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-01T09:00:25.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[100289]: audit 2023 2023-10-01T09:00:25.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[100289]: -10-01T09:00:23.665471+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:25.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[100289]: audit 2023 2023-10-01T09:00:25.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[100289]: -10-01T09:00:23.665988+0000 mon.b (mon.1) 7 : audit [INF] from='osd.4 [v2:172.21.15.190:6800/743095145,v1:172.21.15.190:6801/743095145]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:00:25.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[100289]: audit 2023-10-01T09:00:23.666342+0000 mon.a (mon.0) 411 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:00:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: cluster 2023-10-01T09:00:23.331492+0000 mgr.y (mgr.14146) 120 : cluster [DBG] 2023-10-01T09:00:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: pgmap v99: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: audit 2023-10-01T09:00:23.665070+0000 mon.a (mon.0) 408 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-01T09:00:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: cluster 2023-10 2023-10-01T09:00:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: -01T09:00:23.665162+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-01T09:00:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:00:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: 09:00:23.665471+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: audit 2023-10-01T09:00:25.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: 2023-10-01T09:00:23 2023-10-01T09:00:25.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: .665988+0000 mon.b (mon.1) 2023-10-01T09:00:25.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: 7 : audit [INF] from='osd.4 [v2:172.21.15.190:6800/743095145,v1:172.21.15.190:6801/743095145]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:00:25.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: audit 2023 2023-10-01T09:00:25.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: -10-01T09:00:23. 2023-10-01T09:00:25.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: 666342+0000 mon.a (mon.0) 411 : audit [INF] 2023-10-01T09:00:25.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:24 smithi138 conmon[109959]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:00:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:25 smithi190 conmon[100518]: audit 2023-10-01T09:00:24.664732+0000 mon.a (mon.0) 412 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]': finished 2023-10-01T09:00:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:25 smithi190 conmon[100518]: cluster 2023-10-01T09:00:24.664835+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-01T09:00:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:25 smithi190 conmon[100518]: audit 2023-10-01T09:00:24.665313+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:25 smithi190 conmon[100518]: audit 2023-10-01T09:00:24.670164+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:25.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:25 smithi190 conmon[100518]: cluster 2023-10-01T09:00:25.458469+0000 mon.a (mon.0) 416 : cluster [INF] osd.4 [v2:172.21.15.190:6800/743095145,v1:172.21.15.190:6801/743095145] boot 2023-10-01T09:00:25.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:25 smithi190 conmon[100518]: cluster 2023-10-01T09:00:25.458523+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-01T09:00:25.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:25 smithi190 conmon[100518]: audit 2023-10-01T09:00:25.458937+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:25.936 INFO:teuthology.orchestra.run.smithi190.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-01T09:00:25.936 INFO:teuthology.orchestra.run.smithi190.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-01T09:00:25.936 INFO:teuthology.orchestra.run.smithi190.stderr: stderr: 10+0 records in 2023-10-01T09:00:25.936 INFO:teuthology.orchestra.run.smithi190.stderr:10+0 records out 2023-10-01T09:00:25.936 INFO:teuthology.orchestra.run.smithi190.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110332 s, 950 MB/s 2023-10-01T09:00:25.936 INFO:teuthology.orchestra.run.smithi190.stderr:--> Zapping successful for: 2023-10-01T09:00:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[100289]: audit 2023-10-01T09:00:24.664732+0000 mon.a (mon.0) 412 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]': finished 2023-10-01T09:00:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[100289]: cluster 2023-10-01T09:00:24.664835+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-01T09:00:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[100289]: audit 2023-10-01T09:00:24.665313+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[100289]: audit 2023-10-01T09:00:24.670164+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[100289]: cluster 2023-10-01T09:00: 2023-10-01T09:00:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[100289]: 25.458469+0000 mon.a (mon.0) 416 : cluster [INF] 2023-10-01T09:00:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[100289]: osd.4 [v2:172.21.15.190:6800/743095145,v1:172.21.15.190:6801/743095145] boot 2023-10-01T09:00:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[100289]: cluster 2023-10-01T09:00:26.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[100289]: 2023-10-01T09:00:25.458523+0000 mon.a (mon.0 2023-10-01T09:00:26.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[100289]: ) 417 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-01T09:00:26.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[100289]: audit 2023-10-01T09:00:25.458937+0000 2023-10-01T09:00:26.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[100289]: mon.a (mon.0) 418 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:26.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[109959]: audit 2023-10-01T09:00:24.664732+0000 mon.a (mon.0) 2023-10-01T09:00:26.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[109959]: 412 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]': finished 2023-10-01T09:00:26.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[109959]: cluster 2023-10-01T09:00:24.664835+0000 mon.a (mon 2023-10-01T09:00:26.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[109959]: .0) 413 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-01T09:00:26.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:00:26.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[109959]: 10-01T09:00:24.665313+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:26.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[109959]: audit 2023-10-01T09:00:24.670164+0000 mon.a (mon.0 2023-10-01T09:00:26.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[109959]: ) 415 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:26.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[109959]: cluster 2023-10-01T09:00:25 2023-10-01T09:00:26.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[109959]: .458469+0000 mon.a (mon.0) 416 : cluster [INF] osd.4 [v2:172.21.15.190:6800/743095145,v1:172.21.15.190:6801/743095145] boot 2023-10-01T09:00:26.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[109959]: cluster 2023-10-01T09:00:25.458523+0000 mon.a ( 2023-10-01T09:00:26.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[109959]: mon.0) 417 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-01T09:00:26.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:00:26.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:25 smithi138 conmon[109959]: 10-01T09:00:25.458937+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:00:26.685 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi190:vg_nvme/lv_3 2023-10-01T09:00:27.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:26 smithi190 conmon[100518]: cluster 2023-10-01T09:00:23 2023-10-01T09:00:27.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:26 smithi190 conmon[100518]: .838408+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T09:00:27.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:26 smithi190 conmon[100518]: cluster 2023-10-01T09:00:23.838458+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T09:00:27.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:26 smithi190 conmon[100518]: cluster 2023-10-01T09:00:25.331898 2023-10-01T09:00:27.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:26 smithi190 conmon[100518]: +0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:27.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:26 smithi190 conmon[100518]: cluster 2023-10-01T09:00:26. 2023-10-01T09:00:27.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:26 smithi190 conmon[100518]: 464078+0000 mon.a (mon.0) 419 : cluster [DBG] 2023-10-01T09:00:27.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:26 smithi190 conmon[100518]: osdmap e31: 5 total, 5 up, 5 in 2023-10-01T09:00:27.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:26 smithi138 conmon[100289]: cluster 2023-10-01T09:00:23.838408+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T09:00:27.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:26 smithi138 conmon[100289]: cluster 2023-10-01T09:00:23.838458+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T09:00:27.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:26 smithi138 conmon[100289]: cluster 2023-10-01T09:00:25.331898+0000 mgr.y (mgr.14146) 121 2023-10-01T09:00:27.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:26 smithi138 conmon[100289]: : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:27.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:26 smithi138 conmon[100289]: cluster 2023-10-01T09:00:26.464078+0000 mon.a (mon. 2023-10-01T09:00:27.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:26 smithi138 conmon[100289]: 0) 419 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-01T09:00:27.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:26 smithi138 conmon[109959]: cluster 2023-10-01T09:00:23.838408+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T09:00:27.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:26 smithi138 conmon[109959]: cluster 2023-10-01T09:00:23.838458+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T09:00:27.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:26 smithi138 conmon[109959]: cluster 2023-10-01T09:00:25.331898+0000 mgr.y (mgr 2023-10-01T09:00:27.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:26 smithi138 conmon[109959]: .14146) 121 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T09:00:27.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:26 smithi138 conmon[109959]: cluster 2023-10-01T09:00:26.464078+0000 mon.a 2023-10-01T09:00:27.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:26 smithi138 conmon[109959]: (mon.0) 419 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-01T09:00:28.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:27 smithi190 conmon[100518]: cephadm 2023-10-01T09:00:26.748040+0000 mgr.y (mgr. 2023-10-01T09:00:28.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:27 smithi190 conmon[100518]: 14146) 122 : cephadm [INF] Detected new or changed devices on smithi190 2023-10-01T09:00:28.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:27 smithi190 conmon[100518]: audit 2023-10-01T09:00:26.753480+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:28.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:27 smithi190 conmon[100518]: audit 2023-10-01T09:00:26.754415+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:28.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:27 smithi190 conmon[100518]: cephadm 2023-10-01T09:00:26.755037+0000 mgr.y (mgr.14146) 123 : cephadm [INF] Adjusting osd_memory_target on smithi190 to 17168M 2023-10-01T09:00:28.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:27 smithi190 conmon[100518]: audit 2023-10-01T09:00:26.759017+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:28.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:27 smithi190 conmon[100518]: audit 2023-10-01T09:00:26.760067+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:28.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:27 smithi190 conmon[100518]: audit 2023-10-01T09:00:26.760877+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:28.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:27 smithi190 conmon[100518]: audit 2023-10-01T09:00:26.765304+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:28.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:27 smithi190 conmon[100518]: cluster 2023-10-01T09:00:27.468156+0000 mon.a (mon.0) 426 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-01T09:00:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: cephadm 2023-10-01T09:00:26.748040+0000 mgr.y (mgr 2023-10-01T09:00:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: .14146) 122 : cephadm [INF] Detected new or changed devices on smithi190 2023-10-01T09:00:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: audit 2023-10-01T09:00:26.753480+0000 mon.a (mon.0) 420 : audit 2023-10-01T09:00:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: audit 2023-10-01T09:00:26.754415+0000 mon.a (mon.0) 2023-10-01T09:00:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: 421 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: cephadm 2023-10 2023-10-01T09:00:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: -01T09:00:26.755037+0000 mgr.y (mgr.14146) 123 : cephadm [INF] Adjusting osd_memory_target on smithi190 to 17168M 2023-10-01T09:00:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: audit 2023-10-01T09:00:26.759017+0000 mon.a (mon.0) 422 2023-10-01T09:00:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: audit 2023-10-01T09:00:26.760067+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:28.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: 2023-10-01T09:00:28.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:00:28.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: 01T09:00:26.760877+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:28.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: audit 2023-10-01T09:00:26.765304+0000 mon.a ( 2023-10-01T09:00:28.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:28.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: cluster 2023-10-01T09:00:28.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: 2023-10-01T09:00:27.468156+0000 mon.a (mon.0 2023-10-01T09:00:28.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[109959]: ) 426 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-01T09:00:28.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: cephadm 2023-10-01T09:00:26.748040+0000 mgr.y (mgr.14146) 122 : cephadm [INF] 2023-10-01T09:00:28.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: Detected new or changed devices on smithi190 2023-10-01T09:00:28.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: audit 2023-10-01T09:00:26.753480+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:28.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: audit 2023-10-01T09:00:26.754415+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:28.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: cephadm 2023-10-01T09:00:28.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: 2023-10-01T09:00:26.755037 2023-10-01T09:00:28.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: +0000 mgr.y (mgr.14146) 123 : cephadm [INF] Adjusting osd_memory_target on smithi190 to 17168M 2023-10-01T09:00:28.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: 2023-10-01T09:00:28.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: audit 2023-10-01T09:00:26 2023-10-01T09:00:28.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: .759017+0000 mon.a (mon.0) 422 : audit [INF] 2023-10-01T09:00:28.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:28.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:00:28.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: 00:26.760067+0000 mon.a (mon.0) 423 : 2023-10-01T09:00:28.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:28.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:00:28.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: 01T09:00:26.760877+0000 mon.a (mon.0 2023-10-01T09:00:28.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: ) 424 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:28.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: audit 2023 2023-10-01T09:00:28.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: -10-01T09:00:26.765304+0000 mon.a (mon 2023-10-01T09:00:28.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: .0) 425 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:28.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: cluster 2023- 2023-10-01T09:00:28.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: 10-01T09:00:27.468156+0000 mon.a (mon 2023-10-01T09:00:28.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:27 smithi138 conmon[100289]: .0) 426 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-01T09:00:29.035 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:28 smithi190 conmon[100518]: cluster 2023-10-01T09:00:27.332240+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:29.035 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:28 smithi190 conmon[100518]: audit 2023-10-01T09:00:28.505289+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T09:00:29.035 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:28 smithi190 conmon[100518]: audit 2023-10-01T09:00:28.507793+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T09:00:29.035 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:28 smithi190 conmon[100518]: audit 2023-10-01T09:00:28.508544+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:29.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[100289]: cluster 2023-10-01T09:00:27.332240+0000 mgr.y (mgr.14146) 124 : cluster [DBG] 2023-10-01T09:00:29.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[100289]: pgmap v105: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:29.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[100289]: audit 2023-10-01T09:00:28.505289+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T09:00:29.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[100289]: audit 2023-10-01T09:00:28 2023-10-01T09:00:29.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[100289]: .507793+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T09:00:29.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T09:00:29.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[100289]: T09:00:28.508544+0000 mon.a (mon.0) 429 : audit [DBG] 2023-10-01T09:00:29.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[100289]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:29.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[109959]: cluster 2023-10-01T09:00:27.332240+0000 mgr.y (mgr.14146) 124 : cluster 2023-10-01T09:00:29.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[109959]: [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:29.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[109959]: audit 2023-10-01T09:00:28.505289+0000 mon.a (mon.0) 427 : audit [DBG] 2023-10-01T09:00:29.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T09:00:29.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[109959]: audit 2023-10-01T09:00:28.507793 2023-10-01T09:00:29.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[109959]: +0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T09:00:29.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[109959]: audit 2023-10-01T09:00:28. 2023-10-01T09:00:29.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:28 smithi138 conmon[109959]: 508544+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:30.042 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:29 smithi190 conmon[100518]: audit 2023-10-01T09:00:28.502850+0000 mgr.y (mgr.14146) 125 : audit 2023-10-01T09:00:30.042 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:29 smithi190 conmon[100518]: [DBG] from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi190:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:00:30.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:29 smithi138 conmon[109959]: audit 2023-10-01T09:00:28.502850+0000 mgr.y (mgr.14146) 125 : audit [DBG] from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi190:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:00:30.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:29 smithi138 conmon[100289]: audit 2023-10-01T09:00:28.502850+0000 mgr.y (mgr.14146) 125 : 2023-10-01T09:00:30.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:29 smithi138 conmon[100289]: audit [DBG] from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi190:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:00:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:30 smithi190 conmon[100518]: cluster 2023-10-01T09:00:29.332634+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:30 smithi190 conmon[100518]: audit 2023-10-01T09:00:30.720610+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.190:0/3991159519' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac6e16ed-681d-4e9d-87a3-d20d50e01437"}]: dispatch 2023-10-01T09:00:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:30 smithi190 conmon[100518]: audit 2023-10-01T09:00:30.721238+0000 mon.a (mon.0) 430 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac6e16ed-681d-4e9d-87a3-d20d50e01437"}]: dispatch 2023-10-01T09:00:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:30 smithi190 conmon[100518]: audit 2023-10-01T09:00:30.726789+0000 mon.a (mon.0) 431 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ac6e16ed-681d-4e9d-87a3-d20d50e01437"}]': finished 2023-10-01T09:00:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:30 smithi190 conmon[100518]: cluster 2023-10-01T09:00:30.726901+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-01T09:00:31.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:30 smithi190 conmon[100518]: audit 2023-10-01T09:00:30.727073+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:31.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: cluster 2023-10-01T09:00:29.332634+0000 mgr.y (mgr. 2023-10-01T09:00:31.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: 14146) 126 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:31.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: audit 2023-10-01T09:00:30.720610+0000 mon.b (mon.1) 2023-10-01T09:00:31.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: 8 : audit [INF] from='client.? 172.21.15.190:0/3991159519' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac6e16ed-681d-4e9d-87a3-d20d50e01437"}]: dispatch 2023-10-01T09:00:31.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: audit 2023-10-01T09:00:31.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: 2023-10-01T09:00: 2023-10-01T09:00:31.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: 30.721238+0000 mon.a (mon.0) 2023-10-01T09:00:31.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: 430 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac6e16ed-681d-4e9d-87a3-d20d50e01437"}]: dispatch 2023-10-01T09:00:31.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:00:31.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: 01T09:00:30.726789+0000 mon.a (mon 2023-10-01T09:00:31.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: .0) 431 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ac6e16ed-681d-4e9d-87a3-d20d50e01437"}]': finished 2023-10-01T09:00:31.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: cluster 2023- 2023-10-01T09:00:31.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: 10-01T09:00:30.726901+0000 2023-10-01T09:00:31.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: mon.a (mon.0) 432 : cluster [DBG] 2023-10-01T09:00:31.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: osdmap e33: 6 total, 5 up, 6 in 2023-10-01T09:00:31.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: audit 2023 2023-10-01T09:00:31.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: -10-01T09:00:30.727073 2023-10-01T09:00:31.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: +0000 mon.a (mon.0) 433 : audit 2023-10-01T09:00:31.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[100289]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:31.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: cluster 2023-10-01T09:00:29.332634+0000 mgr.y ( 2023-10-01T09:00:31.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: mgr.14146) 126 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:31.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: audit 2023-10-01T09:00:31.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: 2023-10-01T09:00:30.720610+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.190:0/3991159519' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac6e16ed-681d-4e9d-87a3-d20d50e01437"}]: dispatch 2023-10-01T09:00:31.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: audit 2023 2023-10-01T09:00:31.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: -10-01T09:00:30.721238+0000 mon.a (mon. 2023-10-01T09:00:31.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: 0) 430 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ac6e16ed-681d-4e9d-87a3-d20d50e01437"}]: dispatch 2023-10-01T09:00:31.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: audit 2023-10-01T09:00:31.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: 2023-10-01T09: 2023-10-01T09:00:31.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: 00:30.726789+0000 mon.a (mon 2023-10-01T09:00:31.076 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: .0) 431 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ac6e16ed-681d-4e9d-87a3-d20d50e01437"}]': finished 2023-10-01T09:00:31.076 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: cluster 2023-10- 2023-10-01T09:00:31.076 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: 01T09:00:30.726901+0000 mon.a (mon.0) 432 : cluster 2023-10-01T09:00:31.076 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-01T09:00:31.076 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:00:31.076 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: 01T09:00:30.727073+0000 mon.a (mon.0) 433 2023-10-01T09:00:31.077 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:30 smithi138 conmon[109959]: : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:32.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:31 smithi190 conmon[100518]: audit 2023-10-01T09:00:31.322122+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.190:0/602679496' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T09:00:32.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:31 smithi138 conmon[100289]: audit 2023-10-01T09:00:31.322122+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.190:0/602679496' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T09:00:32.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:31 smithi138 conmon[109959]: audit 2023-10-01T09:00:31.322122+0000 mon.b (mon.1) 9 2023-10-01T09:00:32.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:31 smithi138 conmon[109959]: : audit [DBG] from='client.? 172.21.15.190:0/602679496' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T09:00:33.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:32 smithi190 conmon[100518]: cluster 2023-10-01T09:00:31.333128+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:33.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:32 smithi138 conmon[100289]: cluster 2023-10-01T09:00:31.333128+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:33.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:32 smithi138 conmon[109959]: cluster 2023-10-01T09:00:31.333128+0000 mgr.y (mgr.14146) 127 2023-10-01T09:00:33.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:32 smithi138 conmon[109959]: : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:35.018 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:34 smithi190 conmon[100518]: cluster 2023-10-01T09:00:33.333598+0000 mgr.y (mgr. 2023-10-01T09:00:35.018 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:34 smithi190 conmon[100518]: 14146) 128 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:35.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:34 smithi138 conmon[100289]: cluster 2023-10-01T09:00:33.333598+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:35.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:34 smithi138 conmon[109959]: cluster 2023-10-01T09:00:33.333598+0000 2023-10-01T09:00:35.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:34 smithi138 conmon[109959]: mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:37.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:36 smithi190 conmon[100518]: cluster 2023-10-01T09:00:35.334085+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:37.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:36 smithi190 conmon[100518]: audit 2023-10-01T09:00:36.641739+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T09:00:37.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:36 smithi190 conmon[100518]: audit 2023-10-01T09:00:36.642565+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:37.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:36 smithi138 conmon[109959]: cluster 2023-10-01T09:00:35.334085+0000 mgr.y (mgr.14146) 129 : 2023-10-01T09:00:37.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:36 smithi138 conmon[109959]: cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:37.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:36 smithi138 conmon[109959]: audit 2023-10-01T09:00:36.641739+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T09:00:37.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:36 smithi138 conmon[109959]: audit 2023-10-01T09:00:36.642565+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:37.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:36 smithi138 conmon[100289]: cluster 2023-10-01T09:00:35.334085+0000 mgr.y (mgr.14146) 129 2023-10-01T09:00:37.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:36 smithi138 conmon[100289]: : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:37.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:36 smithi138 conmon[100289]: audit 2023-10-01T09:00:36.641739+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T09:00:37.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:36 smithi138 conmon[100289]: audit 2023-10-01T09:00:36.642565+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:38.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:37 smithi190 conmon[100518]: cephadm 2023-10-01T09:00:36.643229+0000 mgr.y (mgr.14146) 2023-10-01T09:00:38.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:37 smithi190 conmon[100518]: 130 : cephadm [INF] Deploying daemon osd.5 on smithi190 2023-10-01T09:00:38.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:37 smithi138 conmon[100289]: cephadm 2023-10-01T09:00:36.643229+0000 mgr.y (mgr.14146) 130 : cephadm [INF] 2023-10-01T09:00:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:37 smithi138 conmon[100289]: Deploying daemon osd.5 on smithi190 2023-10-01T09:00:38.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:37 smithi138 conmon[109959]: cephadm 2023-10-01T09:00:36.643229+0000 mgr.y (mgr.14146) 2023-10-01T09:00:38.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:37 smithi138 conmon[109959]: 130 : cephadm [INF] Deploying daemon osd.5 on smithi190 2023-10-01T09:00:39.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:38 smithi190 conmon[100518]: cluster 2023-10-01T09:00:37.334452+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:39.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:38 smithi138 conmon[109959]: cluster 2023-10-01T09:00:37.334452+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:38 smithi138 conmon[100289]: cluster 2023-10-01T09:00:37.334452+0000 mgr.y (mgr.14146 2023-10-01T09:00:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:38 smithi138 conmon[100289]: ) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:41.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:40 smithi190 conmon[100518]: cluster 2023-10-01T09:00:39.334843+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:41.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:40 smithi138 conmon[100289]: cluster 2023-10-01T09:00:39.334843+0000 mgr.y (mgr.14146 2023-10-01T09:00:41.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:40 smithi138 conmon[100289]: ) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:41.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:40 smithi138 conmon[109959]: cluster 2023-10-01T09:00:39.334843+0000 mgr.y (mgr.14146) 132 : 2023-10-01T09:00:41.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:40 smithi138 conmon[109959]: cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:42 smithi190 conmon[100518]: cluster 2023-10-01T09:00:41.335304+0000 mgr.y (mgr.14146 2023-10-01T09:00:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:42 smithi190 conmon[100518]: ) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:42 smithi190 conmon[100518]: audit 2023-10-01T09:00:41.385882+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:42 smithi190 conmon[100518]: audit 2023-10-01T09:00:41.419437+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:42.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:42 smithi138 conmon[100289]: cluster 2023-10-01T09:00:41.335304+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:42.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:42 smithi138 conmon[100289]: audit 2023-10-01T09:00:41.385882+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:42.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:42 smithi138 conmon[100289]: audit 2023-10-01T09:00:41.419437+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:42.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:42 smithi138 conmon[109959]: cluster 2023-10-01T09:00:41.335304+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:42.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:42 smithi138 conmon[109959]: audit 2023-10-01T09:00:41.385882+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:42.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:42 smithi138 conmon[109959]: audit 2023-10-01T09:00 2023-10-01T09:00:42.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:42 smithi138 conmon[109959]: :41.419437+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:44.236 INFO:teuthology.orchestra.run.smithi190.stdout:Created osd(s) 5 on host 'smithi190' 2023-10-01T09:00:44.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:44 smithi190 conmon[100518]: audit 2023-10-01T09:00:43.020213+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:44.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:44 smithi190 conmon[100518]: audit 2023-10-01T09:00:43.021020+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:44.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:44 smithi190 conmon[100518]: audit 2023-10-01T09:00:43.021939+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:44.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:44 smithi190 conmon[100518]: audit 2023-10-01T09:00:43.026581+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:44.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:44 smithi190 conmon[100518]: cluster 2023-10-01T09:00:43.335692+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[100289]: audit 2023-10-01T09:00:43.020213+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[100289]: audit 2023-10-01T09:00:43.021020+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[100289]: audit 2023-10-01T09:00:43.021939+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[100289]: audit 2023-10-01T09:00:43.026581+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[100289]: cluster 2023-10-01T09:00:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[100289]: 2023-10-01T09:00:43.335692+0000 mgr.y ( 2023-10-01T09:00:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[100289]: mgr.14146) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:44.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[109959]: audit 2023-10-01T09:00:43.020213+0000 mon.a (mon.0) 438 : audit [INF] 2023-10-01T09:00:44.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[109959]: audit 2023-10-01T09:00:43.021020+0000 mon.a (mon.0) 2023-10-01T09:00:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[109959]: 439 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:00:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[109959]: 01T09:00:43.021939+0000 mon.a 2023-10-01T09:00:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[109959]: (mon.0) 440 : audit [INF] 2023-10-01T09:00:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[109959]: audit 2023-10-01T09:00:43. 2023-10-01T09:00:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[109959]: 026581+0000 mon.a (mon.0) 441 : audit [INF] 2023-10-01T09:00:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[109959]: cluster 2023-10-01T 2023-10-01T09:00:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[109959]: 09:00:43.335692+0000 mgr.y (mgr.14146) 134 : 2023-10-01T09:00:44.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:44 smithi138 conmon[109959]: cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:44.868 DEBUG:teuthology.orchestra.run.smithi190:osd.5> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-01T09:00:44.870 INFO:tasks.cephadm:Deploying osd.6 on smithi190 with /dev/vg_nvme/lv_2... 2023-10-01T09:00:44.870 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-01T09:00:44.902 INFO:journalctl@ceph.osd.5.smithi190.stdout:-- Logs begin at Sun 2023-10-01 08:38:18 UTC. -- 2023-10-01T09:00:45.442 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:45 smithi190 conmon[100518]: audit 2023-10-01T09:00:44.235296+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:45.442 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:45 smithi190 conmon[100518]: audit 2023-10-01T09:00:44.245899+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:45.442 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:45 smithi190 conmon[100518]: audit 2023-10-01T09:00:44.616324+0000 mon.b (mon.1) 10 : audit [INF] from='osd.5 [v2:172.21.15.190:6808/4080051847,v1:172.21.15.190:6809/4080051847]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:00:45.442 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:45 smithi190 conmon[100518]: audit 2023-10-01T09:00:44.616778+0000 mon.a (mon.0) 444 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:00:45.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[109959]: audit 2023-10-01T09:00:44.235296+0000 mon.a (mon.0) 2023-10-01T09:00:45.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[109959]: 442 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:45.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[109959]: audit 2023-10-01T09:00:44.245899+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:45.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[109959]: audit 2023-10-01T09:00:44.616324+0000 mon.b (mon. 2023-10-01T09:00:45.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[109959]: 1) 10 : audit [INF] from='osd.5 [v2:172.21.15.190:6808/4080051847,v1:172.21.15.190:6809/4080051847]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:00:45.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:00:45.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[109959]: 10-01T09:00:44.616778+0000 mon.a ( 2023-10-01T09:00:45.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[109959]: mon.0) 444 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:00:45.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[100289]: audit 2023-10-01T09:00:44.235296+0000 mon.a (mon.0) 442 : audit [INF] 2023-10-01T09:00:45.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[100289]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:45.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[100289]: audit 2023-10-01T09:00:44.245899+0000 mon.a (mon.0) 443 : audit 2023-10-01T09:00:45.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[100289]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:00:45.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:00:45.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[100289]: :00:44.616324+0000 mon.b (mon. 2023-10-01T09:00:45.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[100289]: 1) 10 : audit [INF] from='osd.5 [v2:172.21.15.190:6808/4080051847,v1:172.21.15.190:6809/4080051847]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:00:45.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:00:45.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[100289]: 09:00:44.616778+0000 mon.a (mon.0) 2023-10-01T09:00:45.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:45 smithi138 conmon[100289]: 444 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:00:46.554 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:00:46 smithi190 conmon[112665]: debug 2023-10-01T09:00:46.249+0000 7f8db98b4700 -1 osd.5 0 waiting for initial osdmap 2023-10-01T09:00:46.554 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:00:46 smithi190 conmon[112665]: debug 2023-10-01T09:00:46.255+0000 7f8db2028700 -1 osd.5 35 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T09:00:46.554 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:46 smithi190 conmon[100518]: audit 2023-10-01T09:00:45.242108+0000 mon.a (mon.0) 445 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T09:00:46.555 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:46 smithi190 conmon[100518]: 2023-10-01T09:00:46.555 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:46 smithi190 conmon[100518]: cluster 2023-10-01T09:00:45.242193+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-01T09:00:46.555 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:46 smithi190 conmon[100518]: audit 2023-10-01T09:00:45.242376+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:46.555 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:46 smithi190 conmon[100518]: audit 2023-10-01T09:00:45.242792+0000 mon.b (mon.1) 11 : audit [INF] from='osd.5 [v2:172.21.15.190:6808/4080051847,v1:172.21.15.190:6809/4080051847]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:00:46.555 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:46 smithi190 conmon[100518]: audit 2023-10-01T09:00:45.243080+0000 mon.a (mon.0) 448 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:00:46.555 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:46 smithi190 conmon[100518]: cluster 2023-10-01T09:00:45.336140+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[100289]: audit 2023-10-01T09:00:45.242108+0000 mon.a (mon.0) 445 : audit 2023-10-01T09:00:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[100289]: [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T09:00:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[100289]: cluster 2023-10-01T09:00:45.242193+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-01T09:00:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[100289]: audit 2023-10-01T09:00:45.242376 2023-10-01T09:00:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[100289]: +0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:00:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[100289]: 10-01T09:00:45.242792+0000 mon.b (mon.1) 11 : audit [INF] from='osd.5 [v2:172.21.15.190:6808/4080051847,v1:172.21.15.190:6809/4080051847]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:00:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[100289]: audit 2023-10-01T09:00:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[100289]: 2023-10-01T09:00:45.243080+0000 mon.a (mon. 2023-10-01T09:00:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[100289]: 0) 448 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:00:46.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[100289]: cluster 2023-10-01 2023-10-01T09:00:46.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[100289]: T09:00:45.336140+0000 mgr.y (mgr.14146) 135 2023-10-01T09:00:46.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[100289]: : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:46.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[109959]: audit 2023-10-01T09:00:45.242108+0000 mon.a (mon.0) 445 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T09:00:46.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[109959]: cluster 2023-10-01T09:00:45.242193+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-01T09:00:46.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[109959]: audit 2023-10-01T09:00 2023-10-01T09:00:46.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[109959]: :45.242376+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:46.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[109959]: 2023-10-01T09:00:46.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:00:46.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[109959]: 10-01T09:00:45.242792+0000 mon.b (mon.1) 11 : audit 2023-10-01T09:00:46.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[109959]: [INF] from='osd.5 [v2:172.21.15.190:6808/4080051847,v1:172.21.15.190:6809/4080051847]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:00:46.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[109959]: audit 2023-10-01T09:00:45 2023-10-01T09:00:46.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[109959]: .243080+0000 mon.a (mon.0) 448 : audit [INF] 2023-10-01T09:00:46.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[109959]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:00:46.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[109959]: cluster 2023-10-01T09:00:46.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[109959]: 2023-10-01T09:00:45.336140+0000 mgr.y (mgr.14146 2023-10-01T09:00:46.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:46 smithi138 conmon[109959]: ) 135 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:47.254 INFO:teuthology.orchestra.run.smithi190.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-01T09:00:47.254 INFO:teuthology.orchestra.run.smithi190.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-01T09:00:47.254 INFO:teuthology.orchestra.run.smithi190.stderr: stderr: 10+0 records in 2023-10-01T09:00:47.254 INFO:teuthology.orchestra.run.smithi190.stderr:10+0 records out 2023-10-01T09:00:47.254 INFO:teuthology.orchestra.run.smithi190.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112104 s, 935 MB/s 2023-10-01T09:00:47.254 INFO:teuthology.orchestra.run.smithi190.stderr:--> Zapping successful for: 2023-10-01T09:00:47.295 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:47 smithi190 conmon[100518]: audit 2023-10-01T09:00:46.247368+0000 mon.a (mon.0) 449 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]': finished 2023-10-01T09:00:47.295 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:47 smithi190 conmon[100518]: cluster 2023-10-01T09:00:46.247468+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-01T09:00:47.295 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:47 smithi190 conmon[100518]: audit 2023-10-01T09:00:46.247834+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:47.295 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:47 smithi190 conmon[100518]: audit 2023-10-01T09:00:46.250612+0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:47.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:47 smithi138 conmon[100289]: audit 2023-10-01T09:00:46.247368+0000 mon.a (mon.0) 449 : audit 2023-10-01T09:00:47.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:47 smithi138 conmon[100289]: [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]': finished 2023-10-01T09:00:47.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:47 smithi138 conmon[100289]: cluster 2023-10-01T09:00:46.247468+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-01T09:00:47.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:47 smithi138 conmon[100289]: audit 2023-10-01T09:00:46.247834+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:47.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:47 smithi138 conmon[100289]: audit 2023 2023-10-01T09:00:47.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:47 smithi138 conmon[100289]: -10-01T09:00:46.250612+0000 mon.a (mon.0) 2023-10-01T09:00:47.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:47 smithi138 conmon[100289]: 452 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:47 smithi138 conmon[109959]: audit 2023-10-01T09:00:46.247368+0000 mon.a (mon.0) 449 2023-10-01T09:00:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:47 smithi138 conmon[109959]: : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]': finished 2023-10-01T09:00:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:47 smithi138 conmon[109959]: cluster 2023-10-01T09:00:46.247468+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-01T09:00:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:47 smithi138 conmon[109959]: audit 2023-10-01T09:00:46.247834+0000 mon.a (mon.0) 451 2023-10-01T09:00:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:47 smithi138 conmon[109959]: : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:47 smithi138 conmon[109959]: audit 2023 2023-10-01T09:00:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:47 smithi138 conmon[109959]: -10-01T09:00:46.250612+0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:48.013 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi190:vg_nvme/lv_2 2023-10-01T09:00:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:48 smithi190 conmon[100518]: cluster 2023-10-01T09:00:45.582207+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T09:00:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:48 smithi190 conmon[100518]: cluster 2023-10-01T09:00:45.582290+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T09:00:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:48 smithi190 conmon[100518]: audit 2023-10-01T09:00:47.250522+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:48 smithi190 conmon[100518]: cluster 2023-10-01T09:00:47.256005+0000 mon.a (mon.0) 454 : cluster [INF] osd.5 [v2:172.21.15.190:6808/4080051847,v1:172.21.15.190:6809/4080051847] boot 2023-10-01T09:00:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:48 smithi190 conmon[100518]: cluster 2023-10-01T09:00:47.256053+0000 mon.a (mon.0) 455 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-01T09:00:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:48 smithi190 conmon[100518]: audit 2023-10-01T09:00:47.256255+0000 mon.a ( 2023-10-01T09:00:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:48 smithi190 conmon[100518]: mon.0) 456 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:48 smithi190 conmon[100518]: cluster 2023-10-01T09:00:47.336681+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v120: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[100289]: cluster 2023-10-01T09:00:45.582207+0000 osd.5 (osd.5) 2023-10-01T09:00:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[100289]: 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T09:00:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[100289]: cluster 2023-10-01T09:00:45.582290+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T09:00:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[100289]: audit 2023-10-01T09:00:47.250522+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[100289]: cluster 2023-10-01T09:00:47.256005+0000 mon.a ( 2023-10-01T09:00:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[100289]: mon.0) 454 : cluster [INF] osd.5 [v2:172.21.15.190:6808/4080051847,v1:172.21.15.190:6809/4080051847] boot 2023-10-01T09:00:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[100289]: cluster 2023-10-01T09:00:48.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[100289]: 2023-10-01T09:00:47.256053+0000 mon.a (mon.0) 455 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-01T09:00:48.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[100289]: audit 2023-10-01T09:00:47.256255+0000 mon.a (mon 2023-10-01T09:00:48.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[100289]: .0) 456 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:48.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[100289]: cluster 2023-10-01 2023-10-01T09:00:48.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[100289]: T09:00:47.336681+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v120: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:48.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: cluster 2023-10-01T09:00:45.582207+0000 osd.5 ( 2023-10-01T09:00:48.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T09:00:48.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: cluster 2023-10-01T09:00:45.582290+0000 osd.5 (osd. 2023-10-01T09:00:48.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: 5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T09:00:48.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: audit 2023-10-01T09:00:47.250522+0000 mon.a ( 2023-10-01T09:00:48.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: mon.0) 453 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:48.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: cluster 2023-10-01T09:00:47.256005+0000 mon.a (mon.0) 454 : cluster [INF] 2023-10-01T09:00:48.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: osd.5 [v2:172.21.15.190:6808/4080051847,v1:172.21.15.190:6809/4080051847] boot 2023-10-01T09:00:48.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: cluster 2023-10- 2023-10-01T09:00:48.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: 01T09:00:47.256053+0000 mon.a (mon.0) 455 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-01T09:00:48.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: audit 2023-10-01T09:00:48.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: 2023-10-01T09:00:47.256255+0000 mon.a (mon.0) 456 : audit 2023-10-01T09:00:48.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:00:48.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: cluster 2023-10-01T09:00:48.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:48 smithi138 conmon[109959]: 2023-10-01T09:00:47.336681+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v120: 1 pgs: 1 peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T09:00:49.524 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:49 smithi190 conmon[100518]: cluster 2023-10-01T09:00:48.262112+0000 mon.a (mon.0) 2023-10-01T09:00:49.524 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:49 smithi190 conmon[100518]: 457 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-01T09:00:49.524 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:49 smithi190 conmon[100518]: cephadm 2023-10-01T09:00:48.607460+0000 mgr.y (mgr.14146) 137 : cephadm [INF] Detected new or changed devices on smithi190 2023-10-01T09:00:49.524 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:49 smithi190 conmon[100518]: audit 2023-10-01T09:00:48.612296+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:49.525 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:49 smithi190 conmon[100518]: audit 2023-10-01T09:00:48.613371+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:49.525 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:49 smithi190 conmon[100518]: audit 2023-10-01T09:00:48.614192+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:49.525 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:49 smithi190 conmon[100518]: cephadm 2023-10-01T09:00:48.614833+0000 mgr.y (mgr.14146) 138 : cephadm [INF] Adjusting osd_memory_target on smithi190 to 8584M 2023-10-01T09:00:49.525 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:49 smithi190 conmon[100518]: audit 2023-10-01T09:00:48.620040+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:49.525 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:49 smithi190 conmon[100518]: audit 2023-10-01T09:00:48.621024+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:49.525 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:49 smithi190 conmon[100518]: audit 2023-10-01T09:00:48.621798+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:49.525 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:49 smithi190 conmon[100518]: audit 2023-10-01T09:00:48.626473+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:49.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: cluster 2023-10-01T09:00:48.262112+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-01T09:00:49.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: cephadm 2023-10-01T09:00:48.607460+0000 mgr.y (mgr.14146) 137 : cephadm [INF] Detected new or changed devices on smithi190 2023-10-01T09:00:49.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: audit 2023-10-01T09:00:48.612296+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:49.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T09:00:49.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: T09:00:48.613371+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:49.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:00:49.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: 00:48.614192+0000 mon.a (mon.0) 2023-10-01T09:00:49.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: 460 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:49.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: cephadm 2023-10-01T09:00:49.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: 2023-10-01T09 2023-10-01T09:00:49.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: :00:48.614833+0000 mgr.y (mgr. 2023-10-01T09:00:49.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: 14146) 138 : cephadm [INF] Adjusting osd_memory_target on smithi190 to 8584M 2023-10-01T09:00:49.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: audit 2023-10-01T09:00:49.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: 2023-10-01T09:00 2023-10-01T09:00:49.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: :48.620040+0000 mon.a (mon.0 2023-10-01T09:00:49.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: ) 461 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:49.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: audit 2023-10-01T09:00:49.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: 2023-10-01T09:00: 2023-10-01T09:00:49.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: 48.621024+0000 mon.a (mon.0) 2023-10-01T09:00:49.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: 462 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:49.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: audit 2023-10 2023-10-01T09:00:49.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: -01T09:00:48.621798+0000 mon.a 2023-10-01T09:00:49.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:49.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:00:49.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: 10-01T09:00:48.626473+0000 mon.a 2023-10-01T09:00:49.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:49.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[100289]: 2023-10-01T09:00:49.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: cluster 2023-10-01T09:00:48.262112+0000 mon.a (mon.0) 457 : cluster [DBG] 2023-10-01T09:00:49.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: osdmap e37: 6 total, 6 up, 6 in 2023-10-01T09:00:49.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: cephadm 2023-10-01T09:00:48.607460+0000 mgr.y (mgr.14146) 137 : cephadm [INF] Detected new or changed devices on smithi190 2023-10-01T09:00:49.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: audit 2023-10-01T09:00:48.612296+0000 mon.a (mon.0) 458 : audit [INF] 2023-10-01T09:00:49.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:49.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: audit 2023-10-01T09:00:49.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: 2023-10-01T09:00:48.613371+0000 mon.a (mon.0) 2023-10-01T09:00:49.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: 459 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:49.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: audit 2023-10-01T09:00:48 2023-10-01T09:00:49.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: .614192+0000 mon.a (mon.0) 460 : audit [INF] 2023-10-01T09:00:49.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:00:49.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: cephadm 2023-10-01T09:00:48.614833+0000 2023-10-01T09:00:49.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: mgr.y (mgr.14146) 138 : cephadm [INF] Adjusting osd_memory_target on smithi190 to 8584M 2023-10-01T09:00:49.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:00:49.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: -01T09:00:48.620040+0000 mon.a (mon 2023-10-01T09:00:49.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: .0) 461 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:49.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:00:49.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: T09:00:48.621024+0000 mon.a (mon.0 2023-10-01T09:00:49.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: ) 462 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:49.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: audit 2023-10-01T09:00 2023-10-01T09:00:49.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: :48.621798+0000 mon.a (mon.0) 463 : audit 2023-10-01T09:00:49.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:00:49.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: audit 2023-10-01T09:00:48.626473 2023-10-01T09:00:49.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: +0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:00:49.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:49 smithi138 conmon[109959]: 2023-10-01T09:00:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:50 smithi190 conmon[100518]: cluster 2023-10-01T09:00:49.266387+0000 2023-10-01T09:00:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:50 smithi190 conmon[100518]: mon.a (mon.0) 465 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-01T09:00:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:50 smithi190 conmon[100518]: cluster 2023-10-01T09:00:49.337040+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v123: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:50 smithi190 conmon[100518]: audit 2023-10-01T09:00:49.876148+0000 mgr.y (mgr.14146) 140 : audit [DBG] from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi190:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:00:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:50 smithi190 conmon[100518]: audit 2023-10-01T09:00:49.878482+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T09:00:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:50 smithi190 conmon[100518]: audit 2023-10-01T09:00:49.880954+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T09:00:50.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:50 smithi190 conmon[100518]: audit 2023-10-01T09:00:49.881745+0000 mon.a (mon.0) 468 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:50.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[100289]: cluster 2023-10-01T09:00:49.266387+0000 mon.a (mon.0) 465 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-01T09:00:50.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[100289]: 2023-10-01T09:00:50.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[100289]: cluster 2023-10-01T09:00:49.337040+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v123: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:50.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[100289]: audit 2023-10-01T09:00:49 2023-10-01T09:00:50.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[100289]: .876148+0000 mgr.y (mgr.14146) 140 : audit [DBG] from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi190:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:00:50.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:00:50.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[100289]: 10-01T09:00:49.878482+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T09:00:50.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[100289]: 2023-10-01T09:00:50.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[100289]: audit 2023-10-01T09:00:50.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[100289]: 2023-10-01T09:00:49.880954+0000 mon.a (mon.0) 467 2023-10-01T09:00:50.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[100289]: : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T09:00:50.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[100289]: audit 2023-10-01T09:00:49 2023-10-01T09:00:50.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[100289]: .881745+0000 mon.a (mon.0) 468 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:50.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[100289]: 2023-10-01T09:00:50.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[109959]: cluster 2023-10-01T09:00:49.266387+0000 mon.a ( 2023-10-01T09:00:50.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[109959]: mon.0) 465 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-01T09:00:50.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[109959]: cluster 2023-10-01T09:00:49. 2023-10-01T09:00:50.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[109959]: 337040+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v123: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:50.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:00:50.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[109959]: T09:00:49.876148+0000 mgr.y (mgr.14146) 140 : audit [DBG] from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi190:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:00:50.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:00:50.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[109959]: T09:00:49.878482+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T09:00:50.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[109959]: audit 2023-10-01T09:00:50.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[109959]: 2023-10-01T09:00:49.880954+0000 mon.a (mon.0) 467 : audit [INF] 2023-10-01T09:00:50.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T09:00:50.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[109959]: audit 2023-10-01T09:00:50.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:50 smithi138 conmon[109959]: 2023-10-01T09:00:49.881745+0000 mon.a (mon.0) 468 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:52 smithi190 conmon[100518]: cluster 2023-10-01T09:00:51.337537+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v124: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:52 smithi190 conmon[100518]: audit 2023-10-01T09:00:52.118238+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.190:0/2720307180' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a6f65a5-16dc-45de-854d-751d9151e85b"}]: dispatch 2023-10-01T09:00:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:52 smithi190 conmon[100518]: audit 2023-10-01T09:00:52.118793+0000 mon.a (mon.0) 469 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a6f65a5-16dc-45de-854d-751d9151e85b"}]: dispatch 2023-10-01T09:00:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:52 smithi190 conmon[100518]: audit 2023-10-01T09:00:52.123105+0000 mon.a (mon.0) 470 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7a6f65a5-16dc-45de-854d-751d9151e85b"}]': finished 2023-10-01T09:00:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:52 smithi190 conmon[100518]: cluster 2023-10-01T09:00:52.123195+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-01T09:00:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:52 smithi190 conmon[100518]: audit 2023-10-01T09:00:52.123381+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:00:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[109959]: cluster 2023-10-01T09:00:51.337537 2023-10-01T09:00:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[109959]: +0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v124: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[109959]: audit 2023-10-01T09:00:52 2023-10-01T09:00:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[109959]: .118238+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.190:0/2720307180' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a6f65a5-16dc-45de-854d-751d9151e85b"}]: dispatch 2023-10-01T09:00:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[109959]: audit 2023-10-01T09:00:52.118793+0000 mon.a (mon.0) 469 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a6f65a5-16dc-45de-854d-751d9151e85b"}]: dispatch 2023-10-01T09:00:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[109959]: audit 2023-10-01T09:00:52.123105+0000 mon.a (mon.0) 470 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7a6f65a5-16dc-45de-854d-751d9151e85b"}]': finished 2023-10-01T09:00:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[109959]: cluster 2023-10- 2023-10-01T09:00:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[109959]: 01T09:00:52.123195+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-01T09:00:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[109959]: audit 2023-10-01T09:00:52.123381+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:00:52.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: cluster 2023-10-01T09:00:51.337537+0000 mgr.y (mgr.14146) 141 : cluster [DBG] 2023-10-01T09:00:52.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: pgmap v124: 1 pgs: 1 peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:52.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: audit 2023-10-01T09:00:52.118238+0000 mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.190:0/2720307180' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a6f65a5-16dc-45de-854d-751d9151e85b"}]: dispatch 2023-10-01T09:00:52.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: audit 2023-10-01T09:00:52 2023-10-01T09:00:52.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: .118793+0000 mon.a (mon.0) 469 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a6f65a5-16dc-45de-854d-751d9151e85b"}]: dispatch 2023-10-01T09:00:52.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:00:52.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: 10-01T09:00:52.123105+0000 mon.a (mon.0) 470 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7a6f65a5-16dc-45de-854d-751d9151e85b"}]': finished 2023-10-01T09:00:52.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: 2023-10-01T09:00:52.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: cluster 2023-10-01T09:00:52.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: 2023-10 2023-10-01T09:00:52.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: -01T09:00:52.123195+0000 mon.a (mon.0 2023-10-01T09:00:52.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: ) 471 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-01T09:00:52.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: audit 2023-10-01T09:00:52.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: 2023-10-01T09:00:52.123381+0000 mon.a ( 2023-10-01T09:00:52.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:52 smithi138 conmon[100289]: mon.0) 472 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:00:53.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:53 smithi190 conmon[100518]: audit 2023-10-01T09:00:52.702573+0000 mon.b (mon.1) 13 : audit 2023-10-01T09:00:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:53 smithi190 conmon[100518]: [DBG] from='client.? 172.21.15.190:0/394073325' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T09:00:53.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:53 smithi138 conmon[100289]: audit 2023-10-01T09:00:52.702573+0000 mon.b (mon.1) 13 2023-10-01T09:00:53.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:53 smithi138 conmon[100289]: : audit [DBG] from='client.? 172.21.15.190:0/394073325' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T09:00:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:53 smithi138 conmon[109959]: audit 2023-10-01T09:00:52.702573+0000 mon.b (mon.1 2023-10-01T09:00:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:53 smithi138 conmon[109959]: ) 13 : audit [DBG] from='client.? 172.21.15.190:0/394073325' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T09:00:54.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:54 smithi190 conmon[100518]: cluster 2023-10-01T09:00:53.337984+0000 mgr.y (mgr.14146) 142 2023-10-01T09:00:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:54 smithi190 conmon[100518]: : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:54.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:54 smithi138 conmon[100289]: cluster 2023-10-01T09:00:53.337984+0000 mgr.y (mgr.14146) 142 2023-10-01T09:00:54.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:54 smithi138 conmon[100289]: : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:54 smithi138 conmon[109959]: cluster 2023-10-01T09:00:53.337984+0000 2023-10-01T09:00:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:54 smithi138 conmon[109959]: mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:56 smithi190 conmon[100518]: cluster 2023-10-01T09:00:55.338293+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:56.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:56 smithi138 conmon[100289]: cluster 2023-10-01T09:00:55.338293+0000 mgr.y (mgr 2023-10-01T09:00:56.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:56 smithi138 conmon[100289]: .14146) 143 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:56.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:56 smithi138 conmon[109959]: cluster 2023-10-01T09:00:55.338293+0000 mgr.y (mgr.14146) 2023-10-01T09:00:56.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:56 smithi138 conmon[109959]: 143 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:58.759 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:58 smithi190 conmon[100518]: cluster 2023-10-01T09:00:57.338696+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:58.759 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:58 smithi190 conmon[100518]: audit 2023-10-01T09:00:58.038741+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T09:00:58.759 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:58 smithi190 conmon[100518]: audit 2023-10-01T09:00:58.039560+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:58.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:58 smithi138 conmon[100289]: cluster 2023-10-01T09:00:57.338696+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:58 smithi138 conmon[100289]: 2023-10-01T09:00:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:58 smithi138 conmon[100289]: audit 2023-10-01T09:00:58.038741+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T09:00:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:58 smithi138 conmon[100289]: audit 2023-10-01T09:00 2023-10-01T09:00:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:58 smithi138 conmon[100289]: :58.039560+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:58.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:58 smithi138 conmon[109959]: cluster 2023-10-01T09:00:57.338696+0000 mgr.y (mgr.14146) 144 : 2023-10-01T09:00:58.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:58 smithi138 conmon[109959]: cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:00:58.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:58 smithi138 conmon[109959]: audit 2023-10-01T09:00:58.038741+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T09:00:58.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:58 smithi138 conmon[109959]: audit 2023-10-01T09:00:58.039560+0000 mon.a ( 2023-10-01T09:00:58.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:58 smithi138 conmon[109959]: mon.0) 474 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:00:59.554 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:59 smithi190 conmon[100518]: cephadm 2023-10-01T09:00:58.040224+0000 mgr.y (mgr.14146) 2023-10-01T09:00:59.554 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:00:59 smithi190 conmon[100518]: 145 : cephadm [INF] Deploying daemon osd.6 on smithi190 2023-10-01T09:00:59.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:59 smithi138 conmon[100289]: cephadm 2023-10-01T09:00:58.040224+0000 mgr.y (mgr.14146) 145 : cephadm 2023-10-01T09:00:59.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:00:59 smithi138 conmon[100289]: [INF] Deploying daemon osd.6 on smithi190 2023-10-01T09:00:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:59 smithi138 conmon[109959]: cephadm 2023-10-01T09:00:58.040224+0000 mgr.y (mgr.14146) 145 : cephadm [INF] 2023-10-01T09:00:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:00:59 smithi138 conmon[109959]: Deploying daemon osd.6 on smithi190 2023-10-01T09:01:00.660 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:00 smithi190 conmon[100518]: cluster 2023-10-01T09:00:59.339078+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:00.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:00 smithi138 conmon[100289]: cluster 2023-10-01T09:00:59.339078+0000 mgr.y (mgr.14146) 146 : cluster [DBG] 2023-10-01T09:01:00.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:00 smithi138 conmon[100289]: pgmap v129: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:00.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:00 smithi138 conmon[109959]: cluster 2023-10-01T09:00:59.339078+0000 mgr.y (mgr.14146) 2023-10-01T09:01:00.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:00 smithi138 conmon[109959]: 146 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:02.795 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:02 smithi190 conmon[100518]: cluster 2023-10-01T09:01:01.339462+0000 mgr.y (mgr.14146) 2023-10-01T09:01:02.796 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:02 smithi190 conmon[100518]: 147 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:02.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:02 smithi138 conmon[109959]: cluster 2023-10-01T09:01:01.339462+0000 mgr.y (mgr. 2023-10-01T09:01:02.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:02 smithi138 conmon[109959]: 14146) 147 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:02 smithi138 conmon[100289]: cluster 2023-10-01T09:01:01.339462+0000 mgr.y (mgr.14146) 147 : cluster [DBG] 2023-10-01T09:01:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:02 smithi138 conmon[100289]: pgmap v130: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:03 smithi190 conmon[100518]: audit 2023-10-01T09:01:02.865627+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:03 smithi190 conmon[100518]: audit 2023-10-01T09:01:02.877582+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:03 smithi138 conmon[100289]: audit 2023-10-01T09:01:02.865627+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:03 smithi138 conmon[100289]: audit 2023-10-01T09:01:02. 2023-10-01T09:01:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:03 smithi138 conmon[100289]: 877582+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:03 smithi138 conmon[109959]: audit 2023-10-01T09:01:02.865627+0000 mon.a (mon. 2023-10-01T09:01:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:03 smithi138 conmon[109959]: 0) 475 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:03 smithi138 conmon[109959]: audit 2023 2023-10-01T09:01:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:03 smithi138 conmon[109959]: -10-01T09 2023-10-01T09:01:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:03 smithi138 conmon[109959]: :01:02.877582+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:05.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:04 smithi190 conmon[100518]: cluster 2023-10-01T09:01:03.339820+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:04 smithi190 conmon[100518]: audit 2023-10-01T09:01:04.735930+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:04 smithi190 conmon[100518]: audit 2023-10-01T09:01:04.736883+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:04 smithi190 conmon[100518]: audit 2023-10-01T09:01:04.737757+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:04 smithi190 conmon[100518]: audit 2023-10-01T09:01:04.742370+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:05.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[109959]: cluster 2023-10-01T09:01:03.339820+0000 mgr.y ( 2023-10-01T09:01:05.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[109959]: mgr.14146) 148 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:05.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:01:05.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[109959]: 09:01:04.735930+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:05.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[109959]: audit 2023-10-01T09:01:05.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[109959]: 2023-10-01T09:01:04.736883+0000 mon.a 2023-10-01T09:01:05.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[109959]: (mon.0) 478 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:05.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[109959]: audit 2023-10-01T09:01:05.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[109959]: 2023-10-01T09:01:04 2023-10-01T09:01:05.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[109959]: .737757+0000 mon.a (mon.0) 479 : audit [INF] 2023-10-01T09:01:05.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:05.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[109959]: audit 2023-10-01T09:01:04.742370 2023-10-01T09:01:05.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[109959]: +0000 mon.a (mon.0) 480 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:05.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[109959]: 2023-10-01T09:01:05.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[100289]: cluster 2023-10-01T09:01:03.339820+0000 mgr.y (mgr.14146) 148 2023-10-01T09:01:05.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[100289]: : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:05.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[100289]: audit 2023-10-01T09:01:04.735930+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:05.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:01:05.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[100289]: 01:04.736883+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:05.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:01:05.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[100289]: 01T09:01:04.737757+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:05.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[100289]: audit 2023 2023-10-01T09:01:05.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[100289]: -10-01T09:01:04.742370+0000 mon.a (mon.0 2023-10-01T09:01:05.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:04 smithi138 conmon[100289]: ) 480 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:05.733 INFO:teuthology.orchestra.run.smithi190.stdout:Created osd(s) 6 on host 'smithi190' 2023-10-01T09:01:06.391 DEBUG:teuthology.orchestra.run.smithi190:osd.6> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-01T09:01:06.393 INFO:tasks.cephadm:Deploying osd.7 on smithi190 with /dev/vg_nvme/lv_1... 2023-10-01T09:01:06.394 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-01T09:01:06.425 INFO:journalctl@ceph.osd.6.smithi190.stdout:-- Logs begin at Sun 2023-10-01 08:38:18 UTC. -- 2023-10-01T09:01:07.030 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:06 smithi190 conmon[100518]: cluster 2023-10-01T09:01:05.340144+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:07.030 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:06 smithi190 conmon[100518]: audit 2023-10-01T09:01:05.732074+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:07.030 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:06 smithi190 conmon[100518]: audit 2023-10-01T09:01:05.760806+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:07.030 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:06 smithi190 conmon[100518]: audit 2023-10-01T09:01:06.080687+0000 mon.b (mon.1) 14 : audit [INF] from='osd.6 [v2:172.21.15.190:6816/797186189,v1:172.21.15.190:6817/797186189]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T09:01:07.030 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:06 smithi190 conmon[100518]: audit 2023-10-01T09:01:06.081082+0000 mon.a (mon.0) 483 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T09:01:07.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[100289]: cluster 2023-10-01T09:01:05.340144+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:07.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[100289]: 2023-10-01T09:01:07.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[100289]: audit 2023-10-01T09:01:05.732074+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:07.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[100289]: audit 2023-10-01T09:01:07.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[100289]: 2023-10-01T09:01:05.760806+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:07.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[100289]: audit 2023-10-01T09:01 2023-10-01T09:01:07.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[100289]: :06.080687+0000 mon.b (mon.1) 14 : audit [INF] 2023-10-01T09:01:07.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[100289]: from='osd.6 [v2:172.21.15.190:6816/797186189,v1:172.21.15.190:6817/797186189]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T09:01:07.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[100289]: audit 2023-10-01T09:01:06.081082 2023-10-01T09:01:07.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[100289]: +0000 mon.a (mon.0) 483 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T09:01:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[109959]: cluster 2023-10-01T09:01:05.340144+0000 mgr.y (mgr. 2023-10-01T09:01:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[109959]: 14146) 149 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[109959]: audit 2023-10-01T09:01:05.732074+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[109959]: audit 2023-10-01T09:01:05.760806+0000 mon.a (mon.0) 482 : audit [DBG] 2023-10-01T09:01:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[109959]: audit 2023-10-01T09:01:06.080687+0000 2023-10-01T09:01:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[109959]: mon.b (mon.1) 14 : audit [INF] from='osd.6 [v2:172.21.15.190:6816/797186189,v1:172.21.15.190:6817/797186189]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T09:01:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[109959]: audit 2023-10-01T09:01:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[109959]: 2023-10-01T09:01:06.081082+0000 mon.a 2023-10-01T09:01:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:06 smithi138 conmon[109959]: (mon.0) 483 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T09:01:08.306 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:01:07 smithi190 conmon[118570]: debug 2023-10-01T09:01:07.881+0000 7fc68e3b6700 -1 osd.6 0 waiting for initial osdmap 2023-10-01T09:01:08.307 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:01:07 smithi190 conmon[118570]: debug 2023-10-01T09:01:07.887+0000 7fc68a331700 -1 osd.6 41 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T09:01:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:07 smithi190 conmon[100518]: audit 2023-10-01T09:01:06.875024+0000 mon.a (mon.0) 484 : audit [INF] 2023-10-01T09:01:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:07 smithi190 conmon[100518]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-01T09:01:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:07 smithi190 conmon[100518]: cluster 2023-10-01T09:01:06.875118+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-01T09:01:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:07 smithi190 conmon[100518]: audit 2023-10-01T09:01:06.875295+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:08.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:07 smithi190 conmon[100518]: audit 2023-10-01T09:01:06.877651+0000 mon.b (mon.1) 15 : audit [INF] from='osd.6 [v2:172.21.15.190:6816/797186189,v1:172.21.15.190:6817/797186189]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:01:08.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:07 smithi190 conmon[100518]: audit 2023-10-01T09:01:06.878001+0000 mon.a (mon.0) 487 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:01:08.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[109959]: audit 2023-10-01T09:01:06.875024+0000 mon.a (mon.0) 484 : audit 2023-10-01T09:01:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[109959]: [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-01T09:01:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[109959]: cluster 2023-10-01T09:01:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[109959]: 2023-10-01T09:01:06.875118+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-01T09:01:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[109959]: audit 2023-10-01T09:01:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[109959]: 2023-10-01T09:01:06.875295+0000 mon.a ( 2023-10-01T09:01:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[109959]: mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:01:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[109959]: -01T09:01:06.877651+0000 mon.b (mon.1) 15 : 2023-10-01T09:01:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[109959]: audit [INF] from='osd.6 [v2:172.21.15.190:6816/797186189,v1:172.21.15.190:6817/797186189]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:01:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:01:08.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[109959]: :01:06.878001+0000 mon.a (mon.0) 2023-10-01T09:01:08.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[109959]: 487 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:01:08.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[100289]: audit 2023-10-01T09:01:06.875024+0000 mon.a (mon.0) 484 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-01T09:01:08.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[100289]: cluster 2023-10-01T09:01:06.875118+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-01T09:01:08.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[100289]: audit 2023-10-01T09:01:06.875295+0000 mon.a (mon. 2023-10-01T09:01:08.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[100289]: 0) 486 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:08.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[100289]: audit 2023-10-01T09:01:06. 2023-10-01T09:01:08.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[100289]: 877651+0000 mon.b (mon.1) 15 : audit [INF] from='osd.6 [v2:172.21.15.190:6816/797186189,v1:172.21.15.190:6817/797186189]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:01:08.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[100289]: audit 2023-10-01T09:01:06.878001+0000 2023-10-01T09:01:08.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:07 smithi138 conmon[100289]: mon.a (mon.0) 487 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:01:08.886 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:08 smithi190 conmon[100518]: cluster 2023-10-01T09:01:07.340616+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:08.886 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:08 smithi190 conmon[100518]: audit 2023-10-01T09:01:07.877583+0000 mon.a (mon.0) 488 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]': finished 2023-10-01T09:01:09.131 INFO:teuthology.orchestra.run.smithi190.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-01T09:01:09.131 INFO:teuthology.orchestra.run.smithi190.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-01T09:01:09.131 INFO:teuthology.orchestra.run.smithi190.stderr: stderr: 10+0 records in 2023-10-01T09:01:09.131 INFO:teuthology.orchestra.run.smithi190.stderr:10+0 records out 2023-10-01T09:01:09.131 INFO:teuthology.orchestra.run.smithi190.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.011065 s, 948 MB/s 2023-10-01T09:01:09.131 INFO:teuthology.orchestra.run.smithi190.stderr:--> Zapping successful for: 2023-10-01T09:01:09.167 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:08 smithi190 conmon[100518]: cluster 2023-10-01T09:01:07.877702+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-01T09:01:09.167 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:08 smithi190 conmon[100518]: audit 2023-10-01T09:01:07.878238+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:09.167 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:08 smithi190 conmon[100518]: audit 2023-10-01T09:01:07.882383+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:09.167 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:08 smithi190 conmon[100518]: cluster 2023-10-01T09:01:09.167 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:08 smithi190 conmon[100518]: 2023-10-01T09:01:08.878031+0000 mon.a (mon.0) 492 : cluster [INF] osd.6 [v2:172.21.15.190:6816/797186189,v1:172.21.15.190:6817/797186189] boot 2023-10-01T09:01:09.167 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:08 smithi190 conmon[100518]: cluster 2023-10-01T09:01:08.878132+0000 mon.a (mon.0) 493 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-01T09:01:09.168 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:08 smithi190 conmon[100518]: audit 2023-10-01T09:01:08.878349+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:09.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: cluster 2023-10-01T09:01:07.340616+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:09.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: audit 2023-10-01T09:01:07.877583+0000 mon.a (mon.0) 488 : 2023-10-01T09:01:09.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]': finished 2023-10-01T09:01:09.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: cluster 2023-10-01T09:01:07.877702+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-01T09:01:09.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: audit 2023-10-01T09:01:07.878238+0000 mon.a (mon.0) 490 : audit 2023-10-01T09:01:09.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:09.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: audit 2023-10-01T09:01:07.882383+0000 mon.a (mon.0 2023-10-01T09:01:09.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: ) 491 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:09.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: cluster 2023-10 2023-10-01T09:01:09.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: -01T09:01:08.878031+0000 mon.a (mon.0) 492 : cluster [INF] osd.6 [v2:172.21.15.190:6816/797186189,v1:172.21.15.190:6817/797186189] boot 2023-10-01T09:01:09.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: 2023-10-01T09:01:09.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: cluster 2023-10-01 2023-10-01T09:01:09.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: T09:01:08.878132+0000 mon.a (mon.0) 2023-10-01T09:01:09.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: 493 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-01T09:01:09.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: audit 2023-10-01T09:01: 2023-10-01T09:01:09.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: 08.878349+0000 mon.a (mon.0) 494 : audit [DBG] 2023-10-01T09:01:09.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[100289]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:09.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: cluster 2023-10-01T09:01:07.340616+0000 mgr.y (mgr.14146) 150 : cluster [DBG] 2023-10-01T09:01:09.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T09:01:09.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: audit 2023-10-01T09:01:07.877583+0000 mon.a (mon.0) 2023-10-01T09:01:09.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: 488 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]': finished 2023-10-01T09:01:09.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: cluster 2023-10-01T09:01:07 2023-10-01T09:01:09.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: .877702+0000 mon.a (mon.0) 489 : cluster [DBG] 2023-10-01T09:01:09.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: osdmap e41: 7 total, 6 up, 7 in 2023-10-01T09:01:09.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:01:09.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: :01:07.878238+0000 mon.a (mon.0 2023-10-01T09:01:09.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: ) 490 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:09.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:01:09.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: 10-01T09:01:07.882383+0000 2023-10-01T09:01:09.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: mon.a (mon.0) 491 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:09.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: 2023-10-01T09:01:09.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: cluster 2023-10-01T09:01:09.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: 2023-10-01T09:01:08.878031 2023-10-01T09:01:09.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: +0000 mon.a (mon.0) 492 : cluster [INF] 2023-10-01T09:01:09.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: osd.6 [v2:172.21.15.190:6816/797186189,v1:172.21.15.190:6817/797186189] boot 2023-10-01T09:01:09.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: cluster 2023-10-01T09 2023-10-01T09:01:09.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: :01:08.878132+0000 mon.a (mon.0 2023-10-01T09:01:09.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: ) 493 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-01T09:01:09.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:01:09.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: -01T09:01:08.878349 2023-10-01T09:01:09.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: +0000 mon.a (mon.0) 494 : 2023-10-01T09:01:09.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:08 smithi138 conmon[109959]: audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:10.045 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:09 smithi190 conmon[100518]: cluster 2023-10-01T09:01:07.078856+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T09:01:10.045 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:09 smithi190 conmon[100518]: cluster 2023-10-01T09:01:07.078957+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T09:01:10.045 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:09 smithi190 conmon[100518]: cluster 2023-10-01T09:01:09.881443+0000 mon.a (mon.0) 495 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-01T09:01:10.062 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi190:vg_nvme/lv_1 2023-10-01T09:01:10.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:09 smithi138 conmon[100289]: cluster 2023-10-01T09:01:07.078856+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T09:01:10.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:09 smithi138 conmon[100289]: cluster 2023-10-01T09:01:07.078957+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T09:01:10.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:09 smithi138 conmon[100289]: cluster 2023-10-01T09:01:09.881443+0000 mon.a (mon 2023-10-01T09:01:10.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:09 smithi138 conmon[100289]: .0) 495 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-01T09:01:10.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:09 smithi138 conmon[109959]: cluster 2023-10-01T09:01:07.078856+0000 osd.6 (osd.6) 1 : 2023-10-01T09:01:10.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:09 smithi138 conmon[109959]: cluster [DBG] purged_snaps scrub starts 2023-10-01T09:01:10.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:09 smithi138 conmon[109959]: cluster 2023-10-01T09:01:07.078957+0000 osd.6 (osd.6) 2 : cluster 2023-10-01T09:01:10.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:09 smithi138 conmon[109959]: [DBG] purged_snaps scrub ok 2023-10-01T09:01:10.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:09 smithi138 conmon[109959]: cluster 2023-10-01T09 2023-10-01T09:01:10.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:09 smithi138 conmon[109959]: :01:09.881443+0000 mon.a (mon.0) 495 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-01T09:01:10.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:09 smithi138 conmon[109959]: 2023-10-01T09:01:11.244 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:10 smithi190 conmon[100518]: cluster 2023-10-01T09:01:09.340999+0000 2023-10-01T09:01:11.244 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:10 smithi190 conmon[100518]: mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:11.244 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:10 smithi190 conmon[100518]: audit 2023-10-01T09:01:10.045761+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:11.244 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:10 smithi190 conmon[100518]: audit 2023-10-01T09:01:10.046896+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:11.244 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:10 smithi190 conmon[100518]: audit 2023-10-01T09:01:10.047789+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:11.244 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:10 smithi190 conmon[100518]: audit 2023-10-01T09:01:10.048552+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:11.245 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:10 smithi190 conmon[100518]: audit 2023-10-01T09:01:10.053098+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:11.245 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:10 smithi190 conmon[100518]: audit 2023-10-01T09:01:10.054050+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:11.245 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:10 smithi190 conmon[100518]: audit 2023-10-01T09:01:10.054878+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:11.245 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:10 smithi190 conmon[100518]: audit 2023-10-01T09:01:10.059356+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:11.245 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:10 smithi190 conmon[100518]: cluster 2023-10-01T09:01:10.879791+0000 mon.a (mon.0) 504 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-01T09:01:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[100289]: cluster 2023-10-01T09:01:09.340999+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[100289]: audit 2023-10-01T09:01:10.045761+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[100289]: audit 2023-10-01T09:01:10.046896+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:11.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[100289]: audit 2023-10-01T09:01:11.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[100289]: 2023-10-01T09:01:10.047789+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:11.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[100289]: audit 2023-10-01T09:01:10.048552+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:11.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[100289]: audit 2023-10-01T09:01:10.053098+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:11.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[100289]: audit 2023-10-01T09:01:10.054050+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:11.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[100289]: audit 2023-10-01T09:01:10.054878+0000 mon.a ( 2023-10-01T09:01:11.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[100289]: mon.0) 502 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:11.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:01:11.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[100289]: 09:01:10.059356+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:11.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[100289]: cluster 2023-10-01T09:01:11.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[100289]: 2023-10-01T09:01:10.879791+0000 mon.a (mon.0) 504 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-01T09:01:11.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: cluster 2023-10-01T09:01:09.340999+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:11.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: 2023-10-01T09:01:11.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: audit 2023-10-01T09:01:10.045761+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:11.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: audit 2023-10-01T09:01:10.046896+0000 mon.a (mon.0) 497 2023-10-01T09:01:11.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:11.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: audit 2023-10-01T09:01:10.047789+0000 mon.a (mon. 2023-10-01T09:01:11.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: 0) 498 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:11.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: audit 2023-10-01T09:01:11.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: 2023-10-01T09:01:10.048552 2023-10-01T09:01:11.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: +0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:11.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:01:11.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: 10-01T09:01:10.053098+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:11.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: audit 2023-10-01T09:01: 2023-10-01T09:01:11.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: 10.054050+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:11.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: audit 2023-10-01T09:01:11.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: 2023-10-01T09:01:10.054878 2023-10-01T09:01:11.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: +0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:11.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: audit 2023-10-01T09:01:11.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: 2023-10-01T09:01:10.059356+0000 mon.a (mon.0) 2023-10-01T09:01:11.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: 503 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:11.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: cluster 2023-10-01T09:01:10.879791+0000 2023-10-01T09:01:11.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:10 smithi138 conmon[109959]: mon.a (mon.0) 504 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-01T09:01:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:11 smithi190 conmon[100518]: cephadm 2023-10-01T09:01:10.041056+0000 mgr.y (mgr.14146) 152 : cephadm [INF] Detected new or changed devices on smithi190 2023-10-01T09:01:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:11 smithi190 conmon[100518]: cephadm 2023-10-01T09:01:10.049119+0000 mgr.y (mgr.14146) 153 : cephadm [INF] Adjusting osd_memory_target on smithi190 to 5722M 2023-10-01T09:01:12.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:11 smithi138 conmon[100289]: cephadm 2023-10-01T09:01:10.041056+0000 mgr.y (mgr.14146) 152 : cephadm [INF] Detected new or changed devices on smithi190 2023-10-01T09:01:12.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:11 smithi138 conmon[100289]: 2023-10-01T09:01:12.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:11 smithi138 conmon[100289]: cephadm 2023-10-01T09:01:10.049119+0000 mgr.y (mgr.14146) 153 : cephadm [INF] Adjusting osd_memory_target on smithi190 to 5722M 2023-10-01T09:01:12.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:11 smithi138 conmon[109959]: cephadm 2023-10-01T09:01:10.041056+0000 mgr.y (mgr.14146) 152 2023-10-01T09:01:12.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:11 smithi138 conmon[109959]: : cephadm [INF] Detected new or changed devices on smithi190 2023-10-01T09:01:12.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:11 smithi138 conmon[109959]: cephadm 2023-10-01T09:01:10.049119+0000 mgr.y (mgr.14146) 153 : cephadm [INF] Adjusting osd_memory_target on smithi190 to 5722M 2023-10-01T09:01:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:12 smithi190 conmon[100518]: cluster 2023-10-01T09:01:11.341467+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:12 smithi190 conmon[100518]: audit 2023-10-01T09:01:11.911897+0000 mgr.y (mgr.14146) 155 : audit [DBG] from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi190:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:01:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:12 smithi190 conmon[100518]: audit 2023-10-01T09:01:11.914248+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T09:01:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:12 smithi190 conmon[100518]: audit 2023-10-01T09:01:11.916600+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T09:01:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:12 smithi190 conmon[100518]: audit 2023-10-01T09:01:11.917323+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[100289]: cluster 2023-10-01T09:01:11.341467+0000 mgr.y (mgr.14146) 154 : cluster [DBG] 2023-10-01T09:01:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[100289]: pgmap v140: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[100289]: audit 2023-10-01T09:01:11.911897+0000 mgr.y (mgr.14146) 155 : audit [DBG] from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi190:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:01:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[100289]: audit 2023-10-01T09:01:11.914248+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T09:01:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T09:01:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[100289]: T09:01:11.916600+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T09:01:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[100289]: audit 2023-10-01T09:01:11. 2023-10-01T09:01:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[100289]: 917323+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:13.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[109959]: cluster 2023-10-01T09:01:11.341467+0000 mgr.y ( 2023-10-01T09:01:13.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[109959]: mgr.14146) 154 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:13.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[109959]: audit 2023-10-01T09:01:11.911897+0000 mgr.y (mgr.14146) 155 : 2023-10-01T09:01:13.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[109959]: audit [DBG] from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi190:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:01:13.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[109959]: audit 2023-10-01T09:01:11.914248+0000 mon.a (mon.0) 505 : audit 2023-10-01T09:01:13.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[109959]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T09:01:13.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[109959]: audit 2023-10-01T09:01:11.916600+0000 2023-10-01T09:01:13.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[109959]: mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T09:01:13.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[109959]: audit 2023 2023-10-01T09:01:13.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:12 smithi138 conmon[109959]: -10-01T09:01:11.917323+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:14 smithi190 conmon[100518]: cluster 2023-10-01T09:01:13.341895+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:14 smithi190 conmon[100518]: audit 2023-10-01T09:01:14.133169+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.190:0/1250143235' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09a5ad5d-d58f-4fcf-8b09-10f395bd01f0"}]: dispatch 2023-10-01T09:01:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:14 smithi190 conmon[100518]: audit 2023-10-01T09:01:14.133702+0000 mon.a (mon.0) 508 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09a5ad5d-d58f-4fcf-8b09-10f395bd01f0"}]: dispatch 2023-10-01T09:01:15.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:14 smithi190 conmon[100518]: audit 2023-10-01T09:01:14.138644+0000 mon.a (mon.0) 509 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "09a5ad5d-d58f-4fcf-8b09-10f395bd01f0"}]': finished 2023-10-01T09:01:15.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:14 smithi190 conmon[100518]: cluster 2023-10-01T09:01:14.138726+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-01T09:01:15.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:14 smithi190 conmon[100518]: audit 2023-10-01T09:01:14.138883+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:15.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:14 smithi190 conmon[100518]: audit 2023-10-01T09:01:14.712493+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.190:0/1812018023' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T09:01:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[100289]: cluster 2023-10-01T09:01:13.341895+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[100289]: audit 2023-10-01T09:01:14.133169+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.190:0/1250143235' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09a5ad5d-d58f-4fcf-8b09-10f395bd01f0"}]: dispatch 2023-10-01T09:01:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[100289]: audit 2023-10-01T09:01:14.133702+0000 mon.a (mon.0) 508 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09a5ad5d-d58f-4fcf-8b09-10f395bd01f0"}]: dispatch 2023-10-01T09:01:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:01:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[100289]: 01:14.138644+0000 mon.a (mon.0) 509 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "09a5ad5d-d58f-4fcf-8b09-10f395bd01f0"}]': finished 2023-10-01T09:01:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[100289]: cluster 2023-10-01T09:01:14.138726+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-01T09:01:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[100289]: audit 2023-10-01T09:01: 2023-10-01T09:01:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[100289]: 14.138883+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[100289]: audit 2023-10-01T09:01:14.712493+0000 mon.b (mon 2023-10-01T09:01:15.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[100289]: .1) 17 : audit [DBG] from='client.? 172.21.15.190:0/1812018023' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T09:01:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[109959]: cluster 2023-10-01T09:01:13.341895+0000 mgr.y (mgr.14146) 156 : cluster 2023-10-01T09:01:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[109959]: [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[109959]: audit 2023-10-01T09:01:14.133169+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.190:0/1250143235' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09a5ad5d-d58f-4fcf-8b09-10f395bd01f0"}]: dispatch 2023-10-01T09:01:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[109959]: audit 2023-10-01T09:01:14.133702+0000 mon.a (mon.0) 508 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09a5ad5d-d58f-4fcf-8b09-10f395bd01f0"}]: dispatch 2023-10-01T09:01:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:01:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[109959]: -01T09:01:14.138644+0000 mon.a (mon.0) 509 : audit 2023-10-01T09:01:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[109959]: [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "09a5ad5d-d58f-4fcf-8b09-10f395bd01f0"}]': finished 2023-10-01T09:01:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[109959]: cluster 2023-10-01T09:01:15.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[109959]: 2023-10-01T09:01:14.138726+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-01T09:01:15.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[109959]: 2023-10-01T09:01:15.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:01:15.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[109959]: 01T09:01:14.138883+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:15.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[109959]: audit 2023-10-01T09:01:14 2023-10-01T09:01:15.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:14 smithi138 conmon[109959]: .712493+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.190:0/1812018023' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T09:01:17.195 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:16 smithi190 conmon[100518]: cluster 2023-10-01T09:01:15.342349+0000 mgr.y (mgr.14146) 157 : cluster [DBG] 2023-10-01T09:01:17.195 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:16 smithi190 conmon[100518]: pgmap v143: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:17.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:16 smithi138 conmon[100289]: cluster 2023-10-01T09:01:15.342349+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:17.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:16 smithi138 conmon[109959]: cluster 2023-10-01T09:01:15.342349+0000 mgr.y (mgr.14146) 157 2023-10-01T09:01:17.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:16 smithi138 conmon[109959]: : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:19.179 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:18 smithi190 conmon[100518]: cluster 2023-10-01T09:01:17.342795+0000 mgr.y (mgr.14146) 2023-10-01T09:01:19.179 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:18 smithi190 conmon[100518]: 158 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:19.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:18 smithi138 conmon[100289]: cluster 2023-10-01T09:01:17.342795+0000 mgr.y (mgr.14146) 158 : 2023-10-01T09:01:19.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:18 smithi138 conmon[100289]: cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:19.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:18 smithi138 conmon[109959]: cluster 2023-10-01T09:01:17.342795+0000 2023-10-01T09:01:19.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:18 smithi138 conmon[109959]: mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:21.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:20 smithi190 conmon[100518]: cluster 2023-10-01T09:01:19.343343+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:20 smithi190 conmon[100518]: audit 2023-10-01T09:01:20.143139+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T09:01:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:20 smithi190 conmon[100518]: audit 2023-10-01T09:01:20.143986+0000 mon.a (mon.0) 513 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:21.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:20 smithi138 conmon[100289]: cluster 2023-10-01T09:01:19.343343+0000 mgr.y (mgr.14146) 159 : cluster [DBG] 2023-10-01T09:01:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:20 smithi138 conmon[100289]: pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:20 smithi138 conmon[100289]: audit 2023-10-01T09:01:20.143139+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T09:01:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:20 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:01:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:20 smithi138 conmon[100289]: 09:01:20.143986+0000 mon.a (mon.0) 513 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:21.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:20 smithi138 conmon[109959]: cluster 2023-10-01T09:01:19.343343+0000 mgr.y (mgr.14146) 159 : cluster [DBG] 2023-10-01T09:01:21.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:20 smithi138 conmon[109959]: pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:21.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:20 smithi138 conmon[109959]: audit 2023-10-01T09:01:20.143139+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T09:01:21.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:20 smithi138 conmon[109959]: audit 2023-10-01T09:01:21.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:20 smithi138 conmon[109959]: 2023-10-01T09:01:20.143986+0000 mon.a (mon.0) 513 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:21 smithi190 conmon[100518]: cephadm 2023-10-01T09:01:20.144697+0000 mgr.y (mgr.14146) 160 : cephadm 2023-10-01T09:01:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:21 smithi190 conmon[100518]: [INF] Deploying daemon osd.7 on smithi190 2023-10-01T09:01:22.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:21 smithi138 conmon[100289]: cephadm 2023-10-01T09:01:20.144697+0000 mgr.y (mgr.14146) 160 : cephadm [INF] Deploying daemon osd.7 on smithi190 2023-10-01T09:01:22.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:21 smithi138 conmon[109959]: cephadm 2023-10-01T09:01:20.144697+0000 mgr.y (mgr.14146) 2023-10-01T09:01:22.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:21 smithi138 conmon[109959]: 160 : cephadm [INF] Deploying daemon osd.7 on smithi190 2023-10-01T09:01:23.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:22 smithi190 conmon[100518]: cluster 2023-10-01T09:01:21.343769+0000 mgr.y (mgr. 2023-10-01T09:01:23.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:22 smithi190 conmon[100518]: 14146) 161 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:23.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:22 smithi138 conmon[109959]: cluster 2023-10-01T09:01:21.343769+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:23.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:22 smithi138 conmon[100289]: cluster 2023-10-01T09:01:21.343769+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:25.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:24 smithi190 conmon[100518]: cluster 2023-10-01T09:01:23.344117+0000 mgr.y (mgr.14146) 2023-10-01T09:01:25.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:24 smithi190 conmon[100518]: 162 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:25.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:24 smithi190 conmon[100518]: audit 2023-10-01T09:01:24.920468+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:25.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:24 smithi138 conmon[100289]: cluster 2023-10-01T09:01:23.344117+0000 mgr.y (mgr.14146) 2023-10-01T09:01:25.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:24 smithi138 conmon[100289]: 162 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:25.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:24 smithi138 conmon[100289]: audit 2023-10-01T09:01:24.920468+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:25.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:24 smithi138 conmon[109959]: cluster 2023-10-01T09:01:23.344117+0000 mgr.y (mgr.14146) 162 : 2023-10-01T09:01:25.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:24 smithi138 conmon[109959]: cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:25.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:24 smithi138 conmon[109959]: audit 2023-10-01T09:01:24.920468+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:26.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:25 smithi190 conmon[100518]: audit 2023-10-01T09:01:24.963350+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:26.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:25 smithi138 conmon[100289]: audit 2023-10-01T09:01:24.963350+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:26.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:25 smithi138 conmon[109959]: audit 2023-10-01T09:01:24.963350+0000 mon.a (mon.0) 515 : audit 2023-10-01T09:01:26.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:25 smithi138 conmon[109959]: [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:27.276 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:26 smithi190 conmon[100518]: cluster 2023-10-01T09:01:25.344542+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:27.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:26 smithi138 conmon[100289]: cluster 2023-10-01T09:01:25.344542+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:27.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:26 smithi138 conmon[109959]: cluster 2023-10-01T09:01:25.344542 2023-10-01T09:01:27.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:26 smithi138 conmon[109959]: +0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:27.857 INFO:teuthology.orchestra.run.smithi190.stdout:Created osd(s) 7 on host 'smithi190' 2023-10-01T09:01:28.111 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:27 smithi190 conmon[100518]: audit 2023-10-01T09:01:26.987911+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:28.112 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:27 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:01:28.112 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:27 smithi190 conmon[100518]: 01T09:01:27.316359+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:28.112 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:27 smithi190 conmon[100518]: audit 2023-10-01T09:01:27.317297+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:28.112 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:27 smithi190 conmon[100518]: audit 2023-10-01T09:01:27.322745+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:28.112 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:27 smithi190 conmon[100518]: cluster 2023-10-01T09:01:27.345004+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:28.112 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:27 smithi190 conmon[100518]: audit 2023-10-01T09:01:27.856136+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:28.112 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:27 smithi190 conmon[100518]: audit 2023-10-01T09:01:27.890362+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:28.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[100289]: audit 2023-10-01T09:01:26.987911+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:28.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[100289]: audit 2023-10-01T09:01:27.316359+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:28.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[100289]: audit 2023-10-01T09:01:27.317297 2023-10-01T09:01:28.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[100289]: +0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:28.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[100289]: audit 2023-10-01T09:01:27.322745+0000 mon.a (mon. 2023-10-01T09:01:28.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[100289]: 0) 519 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:28.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[100289]: cluster 2023-10-01T09:01:28.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[100289]: 2023-10-01T09:01:27.345004+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:28.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[100289]: audit 2023-10-01T09:01:27.856136+0000 mon.a (mon. 2023-10-01T09:01:28.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[100289]: 0) 520 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:28.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[100289]: audit 2023-10-01T09:01:27.890362+0000 mon.a (mon 2023-10-01T09:01:28.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[100289]: .0) 521 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:28.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[109959]: audit 2023-10-01T09:01:26.987911+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:28.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[109959]: audit 2023-10-01T09:01:27.316359+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:28.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[109959]: audit 2023-10-01T09:01:28.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[109959]: 2023-10-01T09:01:27.317297+0000 mon.a (mon 2023-10-01T09:01:28.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[109959]: .0) 518 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:28.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:01:28.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[109959]: 01:27.322745+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:28.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[109959]: cluster 2023-10-01 2023-10-01T09:01:28.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[109959]: T09:01:27.345004+0000 mgr.y (mgr.14146) 164 : cluster 2023-10-01T09:01:28.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[109959]: [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:28.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[109959]: audit 2023-10-01T09:01:27.856136+0000 mon.a (mon.0) 2023-10-01T09:01:28.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[109959]: 520 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:28.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:01:28.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:27 smithi138 conmon[109959]: 01T09:01:27.890362+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:28.491 DEBUG:teuthology.orchestra.run.smithi190:osd.7> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-01T09:01:28.493 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-01T09:01:28.493 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-01T09:01:28.525 INFO:journalctl@ceph.osd.7.smithi190.stdout:-- Logs begin at Sun 2023-10-01 08:38:18 UTC. -- 2023-10-01T09:01:29.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:28 smithi190 conmon[100518]: audit 2023-10-01T09:01:28.114329+0000 mon.b (mon.1) 18 : audit [INF] from='osd.7 [v2:172.21.15.190:6824/185238034,v1:172.21.15.190:6825/185238034]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T09:01:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:28 smithi190 conmon[100518]: audit 2023-10-01T09:01:28.114697+0000 mon.a (mon.0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T09:01:29.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:28 smithi138 conmon[109959]: audit 2023-10-01T09:01:28.114329+0000 mon.b (mon.1) 18 : audit [INF] from='osd.7 [v2:172.21.15.190:6824/185238034,v1:172.21.15.190:6825/185238034]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T09:01:29.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:28 smithi138 conmon[109959]: audit 2023-10-01T09:01:28.114697+0000 mon.a (mon.0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T09:01:29.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:28 smithi138 conmon[100289]: audit 2023-10-01T09:01:28.114329+0000 mon.b (mon.1) 18 : audit [INF] from='osd.7 [v2:172.21.15.190:6824/185238034,v1:172.21.15.190:6825/185238034]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T09:01:29.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:28 smithi138 conmon[100289]: audit 2023-10-01T09:01:28.114697+0000 mon.a (mon.0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T09:01:30.098 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:29 smithi138 conmon[100289]: audit 2023-10-01T09:01:28.993960+0000 mon.a (mon.0) 523 : audit [INF] 2023-10-01T09:01:30.098 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:29 smithi138 conmon[100289]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-01T09:01:30.098 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:29 smithi138 conmon[100289]: cluster 2023-10-01T09:01:28.994036+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-01T09:01:30.098 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:29 smithi138 conmon[100289]: audit 2023-10-01T09:01:28.994141+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:30.099 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:29 smithi138 conmon[100289]: audit 2023-10-01T09:01:28.994616+0000 mon.b (mon.1) 19 : audit [INF] from='osd.7 [v2:172.21.15.190:6824/185238034,v1:172.21.15.190:6825/185238034]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:01:30.099 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:29 smithi138 conmon[100289]: audit 2023-10-01T09:01:28.994887+0000 mon.a (mon.0) 526 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:01:30.099 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:29 smithi138 conmon[100289]: cluster 2023-10-01T09:01:29.345429+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:30.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: audit 2023-10-01T09:01:28.993960 2023-10-01T09:01:30.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: +0000 mon.a (mon.0) 523 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-01T09:01:30.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: cluster 2023-10-01T 2023-10-01T09:01:30.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: 09:01:28.994036+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-01T09:01:30.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:01:30.099 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: T09:01:28.994141+0000 mon.a (mon.0 2023-10-01T09:01:30.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: ) 525 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:30.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:01:30.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: T09:01:28.994616+0000 mon.b (mon. 2023-10-01T09:01:30.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: 1) 19 : audit [INF] from='osd.7 [v2:172.21.15.190:6824/185238034,v1:172.21.15.190:6825/185238034]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:01:30.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:01:30.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: 01T09:01:28.994887+0000 mon.a (mon. 2023-10-01T09:01:30.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: 0) 526 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:01:30.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: cluster 2023- 2023-10-01T09:01:30.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: 10-01T09:01:29.345429+0000 mgr.y 2023-10-01T09:01:30.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:30 smithi138 conmon[109959]: (mgr.14146) 165 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:30.307 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:01:30 smithi190 conmon[124440]: debug 2023-10-01T09:01:29.999+0000 7fb3b6455700 -1 osd.7 0 waiting for initial osdmap 2023-10-01T09:01:30.307 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:01:30 smithi190 conmon[124440]: debug 2023-10-01T09:01:30.007+0000 7fb3b13ce700 -1 osd.7 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T09:01:30.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:29 smithi190 conmon[100518]: audit 2023-10-01T09:01:28.993960+0000 mon.a (mon.0) 523 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-01T09:01:30.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:29 smithi190 conmon[100518]: cluster 2023-10-01T09:01:28.994036+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-01T09:01:30.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:29 smithi190 conmon[100518]: audit 2023-10-01T09:01:28.994141+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:30.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:29 smithi190 conmon[100518]: audit 2023-10-01T09:01:28.994616+0000 mon.b (mon.1) 19 : audit [INF] from='osd.7 [v2:172.21.15.190:6824/185238034,v1:172.21.15.190:6825/185238034]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:01:30.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:29 smithi190 conmon[100518]: audit 2023-10-01T09:01:28.994887+0000 mon.a (mon.0) 526 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:01:30.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:29 smithi190 conmon[100518]: cluster 2023-10-01T09:01:29.345429+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:01:30.478 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:01:31.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:31 smithi190 conmon[100518]: audit 2023-10-01T09:01:29.995623+0000 mon.a (mon.0) 527 2023-10-01T09:01:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:31 smithi190 conmon[100518]: : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]': finished 2023-10-01T09:01:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:31 smithi190 conmon[100518]: cluster 2023-10-01T09:01:29.995712+0000 mon.a (mon.0) 528 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-01T09:01:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:31 smithi190 conmon[100518]: audit 2023-10-01T09:01:29.996048+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:31 smithi190 conmon[100518]: audit 2023-10-01T09:01:29.998530+0000 mon.a (mon.0) 530 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:31 smithi190 conmon[100518]: audit 2023-10-01T09:01:30.470668+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.138:0/1238714698' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-01T09:01:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:31 smithi190 conmon[100518]: cluster 2023-10-01T09:01:30.478340+0000 mon.a (mon.0) 531 : cluster [INF] osd.7 [v2:172.21.15.190:6824/185238034,v1:172.21.15.190:6825/185238034] boot 2023-10-01T09:01:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:31 smithi190 conmon[100518]: cluster 2023-10-01T09:01:30.478389+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-01T09:01:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:31 smithi190 conmon[100518]: audit 2023-10-01T09:01:30.478766+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:31.177 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":48,"num_osds":8,"num_up_osds":8,"osd_up_since":1696150890,"num_in_osds":8,"osd_in_since":1696150874,"num_remapped_pgs":1} 2023-10-01T09:01:31.177 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T09:01:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[100289]: audit 2023-10-01T09:01:29.995623+0000 mon.a (mon.0) 527 : 2023-10-01T09:01:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[100289]: audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]': finished 2023-10-01T09:01:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[100289]: cluster 2023-10-01T09:01:29.995712+0000 mon.a (mon.0) 528 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-01T09:01:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[100289]: audit 2023-10-01T09:01:29.996048+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[100289]: audit 2023-10-01T09:01:29.998530+0000 mon.a (mon.0) 530 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[100289]: audit 2023-10-01T09:01:30.470668+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.138:0/1238714698' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-01T09:01:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[100289]: cluster 2023-10-01T09:01:30.478340+0000 mon.a (mon. 2023-10-01T09:01:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[100289]: 0) 531 : cluster [INF] osd.7 [v2:172.21.15.190:6824/185238034,v1:172.21.15.190:6825/185238034] boot 2023-10-01T09:01:31.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[100289]: cluster 2023-10-01T09:01:30.478389+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-01T09:01:31.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[100289]: audit 2023-10-01T09:01:30.478766+0000 mon.a (mon. 2023-10-01T09:01:31.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[100289]: 0) 533 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:31.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: audit 2023-10-01T09:01:29.995623 2023-10-01T09:01:31.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: +0000 mon.a (mon.0) 527 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]': finished 2023-10-01T09:01:31.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: cluster 2023-10-01T09:01:29.995712+0000 mon.a (mon.0) 528 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-01T09:01:31.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: audit 2023-10-01T09:01:29.996048+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:31.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: audit 2023-10-01T09:01:29.998530+0000 mon.a (mon.0) 530 2023-10-01T09:01:31.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:31.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: audit 2023-10-01T09:01:30.470668+0000 2023-10-01T09:01:31.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.138:0/1238714698' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-01T09:01:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: cluster 2023-10-01T09 2023-10-01T09:01:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: :01:30.478340+0000 mon.a (mon.0) 2023-10-01T09:01:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: 531 : cluster [INF] osd.7 [v2:172.21.15.190:6824/185238034,v1:172.21.15.190:6825/185238034] boot 2023-10-01T09:01:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: cluster 2023-10-01 2023-10-01T09:01:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: T09:01:30.478389+0000 mon.a (mon.0) 532 : cluster 2023-10-01T09:01:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-01T09:01:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: audit 2023-10-01T09:01:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: 2023-10-01T09:01:30.478766+0000 mon.a (mon.0) 533 : audit [DBG] 2023-10-01T09:01:31.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:31 smithi138 conmon[109959]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:31.517 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:01:32.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: cluster 2023-10-01T09:01:29.080802+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T09:01:32.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: cluster 2023-10-01T09:01:29.080848+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T09:01:32.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: cephadm 2023-10-01T09:01:31.023419+0000 mgr.y (mgr.14146) 166 : cephadm [INF] Detected new or changed devices on smithi190 2023-10-01T09:01:32.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: audit 2023-10-01T09:01:31.029773+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:32.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: audit 2023-10-01T09:01:31.030585+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:32.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: audit 2023-10-01T09:01:31.031165+0000 mon.a 2023-10-01T09:01:32.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:32.032 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: audit 2023-10-01T09:01:31.031704+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:32.032 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: audit 2023-10-01T09:01:31.032234+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:32.032 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: cephadm 2023-10-01T09:01:31.032635+0000 mgr.y (mgr.14146) 167 : cephadm [INF] Adjusting osd_memory_target on smithi190 to 4292M 2023-10-01T09:01:32.032 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: audit 2023-10-01T09:01:31.038421+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:32.032 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: audit 2023-10-01T09:01:31.039106+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:32.032 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: audit 2023-10-01T09:01:31.039643+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:32.032 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: audit 2023-10-01T09:01:31.045843+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:32.032 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: cluster 2023-10-01T09:01:31.345741+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:32.033 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: cluster 2023-10-01T09:01:29.080802+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T09:01:32.033 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: cluster 2023-10-01T09:01:29.080848+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T09:01:32.033 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: cephadm 2023-10-01T09:01:31.023419+0000 mgr.y (mgr.14146) 166 : cephadm [INF] Detected new or changed devices on smithi190 2023-10-01T09:01:32.033 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: audit 2023-10-01T09:01:31.029773+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:32.033 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: audit 2023-10-01T09:01:31.030585+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:32.033 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: audit 2023-10-01T09:01:31.031165+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:32.033 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: audit 2023-10-01T09:01:31.031704+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:32.033 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: audit 2023-10-01T09:01:31.032234+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:32.033 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: cephadm 2023-10-01T 2023-10-01T09:01:32.033 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: 09:01:31.032635+0000 mgr.y (mgr.14146) 167 : cephadm [INF] Adjusting osd_memory_target on smithi190 to 4292M 2023-10-01T09:01:32.033 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: audit 2023-10-01T09:01:31.038421+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:32.033 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: audit 2023-10-01T09:01:31.039106+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:32.034 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: audit 2023-10-01T09:01:31.039643+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:32.034 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: audit 2023-10-01T09:01:31.045843+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:32.034 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: cluster 2023-10-01T09:01:31.345741+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:32.034 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[100289]: cluster 2023-10-01T09:01:31.489751+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-01T09:01:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: cluster 2023-10-01T09:01:29.080802+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T09:01:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: cluster 2023-10-01T09:01:29.080848+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T09:01:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: cephadm 2023-10-01T09:01:31.023419+0000 mgr.y (mgr.14146) 166 : cephadm [INF] Detected new or changed devices on smithi190 2023-10-01T09:01:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: audit 2023-10-01T09:01:31.029773+0000 mon.a (mon.0) 2023-10-01T09:01:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: 534 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: audit 2023-10-01T09:01 2023-10-01T09:01:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: :31.030585+0000 mon.a (mon.0) 535 : 2023-10-01T09:01:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: audit 2023-10-01T09:01: 2023-10-01T09:01:32.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: 31.031165+0000 mon.a (mon.0) 536 : audit 2023-10-01T09:01:32.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:32.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: audit 2023-10-01T09:01: 2023-10-01T09:01:32.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: 31.031704+0000 mon.a (mon.0) 537 : audit 2023-10-01T09:01:32.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:32.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: audit 2023-10-01T09:01:31. 2023-10-01T09:01:32.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: 032234+0000 mon.a (mon.0) 538 : audit [INF] 2023-10-01T09:01:32.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:01:32.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: cephadm 2023-10-01T09:01:31. 2023-10-01T09:01:32.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: 032635+0000 mgr.y (mgr.14146) 167 : cephadm [INF] 2023-10-01T09:01:32.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: Adjusting osd_memory_target on smithi190 to 4292M 2023-10-01T09:01:32.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: audit 2023-10-01T09:01:32.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: 2023-10-01T09:01:31.038421+0000 mon.a (mon.0 2023-10-01T09:01:32.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: ) 539 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:32.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:01:32.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: -01T09:01:31.039106+0000 mon.a (mon 2023-10-01T09:01:32.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: .0) 540 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:32.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T09:01:32.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: 09:01:31.039643+0000 mon.a (mon.0) 2023-10-01T09:01:32.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: 541 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:32.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: audit 2023-10-01T09: 2023-10-01T09:01:32.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: 01:31.045843+0000 mon.a (mon.0) 542 : 2023-10-01T09:01:32.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:32.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: cluster 2023-10-01T09: 2023-10-01T09:01:32.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: 01:31.345741+0000 mgr.y (mgr.14146) 168 : 2023-10-01T09:01:32.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:32.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: cluster 2023-10-01T09:01: 2023-10-01T09:01:32.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: 31.489751+0000 mon.a (mon.0) 543 : cluster 2023-10-01T09:01:32.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:32 smithi190 conmon[100518]: [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-01T09:01:32.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:32 smithi138 conmon[109959]: cluster 2023-10-01T09:01:31.489751+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-01T09:01:33.227 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:01:33.227 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":50,"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","created":"2023-10-01T08:56:55.039846+0000","modified":"2023-10-01T09:01:32.484955+0000","last_up_change":"2023-10-01T09:01:30.467448+0000","last_in_change":"2023-10-01T09:01:14.134268+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-01T08:59:41.882085+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"12f71e77-6099-48ee-a89e-e931e000e9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6803","nonce":635244462}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6805","nonce":635244462}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6809","nonce":635244462}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6807","nonce":635244462}]},"public_addr":"172.21.15.138:6803/635244462","cluster_addr":"172.21.15.138:6805/635244462","heartbeat_back_addr":"172.21.15.138:6809/635244462","heartbeat_front_addr":"172.21.15.138:6807/635244462","state":["exists","up"]},{"osd":1,"uuid":"525b5e81-4fa5-4b10-9147-83fcb549ca30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6811","nonce":989382814}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6813","nonce":989382814}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6817","nonce":989382814}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6815","nonce":989382814}]},"public_addr":"172.21.15.138:6811/989382814","cluster_addr":"172.21.15.138:6813/989382814","heartbeat_back_addr":"172.21.15.138:6817/989382814","heartbeat_front_addr":"172.21.15.138:6815/989382814","state":["exists","up"]},{"osd":2,"uuid":"97443e05-2513-4832-bb54-f65493df5967","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6819","nonce":3174817573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6821","nonce":3174817573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6825","nonce":3174817573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6823","nonce":3174817573}]},"public_addr":"172.21.15.138:6819/3174817573","cluster_addr":"172.21.15.138:6821/3174817573","heartbeat_back_addr":"172.21.15.138:6825/3174817573","heartbeat_front_addr":"172.21.15.138:6823/3174817573","state":["exists","up"]},{"osd":3,"uuid":"ac51c165-e007-4390-b95b-26eef984d637","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6827","nonce":1517480090}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6829","nonce":1517480090}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6832","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6833","nonce":1517480090}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6831","nonce":1517480090}]},"public_addr":"172.21.15.138:6827/1517480090","cluster_addr":"172.21.15.138:6829/1517480090","heartbeat_back_addr":"172.21.15.138:6833/1517480090","heartbeat_front_addr":"172.21.15.138:6831/1517480090","state":["exists","up"]},{"osd":4,"uuid":"4bef630c-bff4-41a3-a932-de0501541f78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6801","nonce":743095145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6803","nonce":743095145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6807","nonce":743095145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6805","nonce":743095145}]},"public_addr":"172.21.15.190:6801/743095145","cluster_addr":"172.21.15.190:6803/743095145","heartbeat_back_addr":"172.21.15.190:6807/743095145","heartbeat_front_addr":"172.21.15.190:6805/743095145","state":["exists","up"]},{"osd":5,"uuid":"ac6e16ed-681d-4e9d-87a3-d20d50e01437","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6808","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6809","nonce":4080051847}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6810","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6811","nonce":4080051847}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6814","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6815","nonce":4080051847}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6812","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6813","nonce":4080051847}]},"public_addr":"172.21.15.190:6809/4080051847","cluster_addr":"172.21.15.190:6811/4080051847","heartbeat_back_addr":"172.21.15.190:6815/4080051847","heartbeat_front_addr":"172.21.15.190:6813/4080051847","state":["exists","up"]},{"osd":6,"uuid":"7a6f65a5-16dc-45de-854d-751d9151e85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6816","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6817","nonce":797186189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6818","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6819","nonce":797186189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6822","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6823","nonce":797186189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6820","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6821","nonce":797186189}]},"public_addr":"172.21.15.190:6817/797186189","cluster_addr":"172.21.15.190:6819/797186189","heartbeat_back_addr":"172.21.15.190:6823/797186189","heartbeat_front_addr":"172.21.15.190:6821/797186189","state":["exists","up"]},{"osd":7,"uuid":"09a5ad5d-d58f-4fcf-8b09-10f395bd01f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6824","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6825","nonce":185238034}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6826","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6827","nonce":185238034}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6830","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6831","nonce":185238034}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6828","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6829","nonce":185238034}]},"public_addr":"172.21.15.190:6825/185238034","cluster_addr":"172.21.15.190:6827/185238034","heartbeat_back_addr":"172.21.15.190:6831/185238034","heartbeat_front_addr":"172.21.15.190:6829/185238034","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:58:54.313619+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:16.837452+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:39.610865+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:03.127916+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:23.838460+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:45.582295+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:07.078976+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:29.080850+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.138:0/3940341494":"2023-10-02T08:57:39.299924+0000","172.21.15.138:6801/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.138:6800/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2054656381":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2018177063":"2023-10-02T08:57:18.125987+0000","172.21.15.138:0/3675584653":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6800/3165627846":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6801/3165627846":"2023-10-02T08:57:18.125987+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T09:01:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:33 smithi138 conmon[100289]: cluster 2023-10-01T09:01:32.490963+0000 2023-10-01T09:01:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:33 smithi138 conmon[100289]: mon.a (mon.0) 544 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-01T09:01:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:33 smithi138 conmon[100289]: audit 2023-10-01T09:01:33.225713+0000 mon.a (mon.0) 545 : audit [DBG] from='client.? 172.21.15.138:0/3180512014' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:01:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:33 smithi138 conmon[109959]: cluster 2023-10-01T09:01:32.490963+0000 2023-10-01T09:01:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:33 smithi138 conmon[109959]: mon.a (mon.0) 544 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-01T09:01:33.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:33 smithi138 conmon[109959]: audit 2023-10-01T09:01:33.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:33 smithi138 conmon[109959]: 2023-10-01T09:01:33.225713+0000 2023-10-01T09:01:33.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:33 smithi138 conmon[109959]: mon.a (mon.0) 545 : audit [DBG] from='client.? 172.21.15.138:0/3180512014' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:01:33.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:33 smithi138 conmon[109959]: 2023-10-01T09:01:33.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:33 smithi190 conmon[100518]: cluster 2023-10-01T09:01:32.490963+0000 mon.a (mon.0) 544 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-01T09:01:33.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:33 smithi190 conmon[100518]: audit 2023-10-01T09:01:33.225713+0000 mon.a (mon.0) 545 : audit [DBG] from='client.? 172.21.15.138:0/3180512014' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:01:34.046 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-10-01T08:59:41.882085+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-10-01T09:01:34.046 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool get device_health_metrics pg_num 2023-10-01T09:01:34.388 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:01:34.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:34 smithi138 conmon[100289]: cluster 2023-10-01T09:01:33.346247+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:34.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:34 smithi138 conmon[109959]: cluster 2023-10-01T09:01:33.346247+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:34.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:34 smithi190 conmon[100518]: cluster 2023-10-01T09:01:33.346247+0000 mgr.y (mgr.14146) 169 : cluster [DBG] 2023-10-01T09:01:34.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:34 smithi190 conmon[100518]: pgmap v157: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:35.993 INFO:teuthology.orchestra.run.smithi138.stdout:pg_num: 1 2023-10-01T09:01:36.652 INFO:tasks.cephadm:Adding prometheus.a on smithi190 2023-10-01T09:01:36.652 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch apply prometheus '1;smithi190=a' 2023-10-01T09:01:36.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:36 smithi190 conmon[100518]: cluster 2023-10-01T09:01:35.346579+0000 mgr.y (mgr.14146) 170 : cluster [DBG] 2023-10-01T09:01:36.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:36 smithi190 conmon[100518]: pgmap v158: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:36 smithi190 conmon[100518]: audit 2023-10-01T09:01:35.992777+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.138:0/3175030504' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-01T09:01:36.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:36 smithi138 conmon[100289]: cluster 2023-10-01T09:01:35.346579+0000 mgr.y 2023-10-01T09:01:36.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:36 smithi138 conmon[100289]: (mgr.14146) 170 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:36.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:36 smithi138 conmon[100289]: audit 2023-10-01T09:01:35.992777+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.138:0/3175030504' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-01T09:01:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:36 smithi138 conmon[109959]: cluster 2023-10-01T09:01:35.346579+0000 2023-10-01T09:01:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:36 smithi138 conmon[109959]: mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:36 smithi138 conmon[109959]: audit 2023 2023-10-01T09:01:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:36 smithi138 conmon[109959]: -10-01T09:01:35.992777+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.138:0/3175030504' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-01T09:01:38.478 INFO:teuthology.orchestra.run.smithi190.stdout:Scheduled prometheus update... 2023-10-01T09:01:38.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:38 smithi190 conmon[100518]: cluster 2023-10-01T09:01:37.346978+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:38.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:38 smithi190 conmon[100518]: audit 2023-10-01T09:01:38.471213+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:38.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:38 smithi138 conmon[100289]: cluster 2023-10-01T09:01:37.346978+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:38.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:38 smithi138 conmon[100289]: audit 2023-10-01T09:01:38.471213+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:38 smithi138 conmon[109959]: cluster 2023-10-01T09:01:37.346978+0000 mgr.y (mgr.14146) 171 : 2023-10-01T09:01:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:38 smithi138 conmon[109959]: cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:38 smithi138 conmon[109959]: audit 2023-10-01T09:01:38.471213+0000 mon.a (mon.0 2023-10-01T09:01:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:38 smithi138 conmon[109959]: ) 546 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:39.174 DEBUG:teuthology.orchestra.run.smithi190:prometheus.a> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@prometheus.a.service 2023-10-01T09:01:39.176 INFO:tasks.cephadm:Adding node-exporter.a on smithi138 2023-10-01T09:01:39.176 INFO:tasks.cephadm:Adding node-exporter.b on smithi190 2023-10-01T09:01:39.177 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi138=a;smithi190=b' 2023-10-01T09:01:39.204 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:-- Logs begin at Sun 2023-10-01 08:38:18 UTC. -- 2023-10-01T09:01:39.788 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100738]: ignoring --setuser ceph since I am not root 2023-10-01T09:01:39.788 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100738]: ignoring --setgroup ceph since I am not root 2023-10-01T09:01:39.788 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100738]: debug 2023-10-01T09:01:39.586+0000 7f2032e66700 1 -- 172.21.15.138:0/719490236 <== mon.0 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c59a7764e0 con 0x55c59a74f000 2023-10-01T09:01:39.788 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100738]: debug 2023-10-01T09:01:39.707+0000 7f204302d680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T09:01:39.788 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100738]: debug 2023-10-01T09:01:39.787+0000 7f204302d680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T09:01:39.789 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: audit 2023-10-01T09:01:38.465589+0000 mgr.y (mgr 2023-10-01T09:01:39.789 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: .14146) 172 : audit [DBG] from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi190=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:01:39.789 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: cephadm 2023- 2023-10-01T09:01:39.789 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: 10-01T09:01:38.467148+0000 mgr.y (mgr.14146) 173 : cephadm [INF] Saving service prometheus spec with placement smithi190=a;count:1 2023-10-01T09:01:39.789 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: audit 2023-10-01T09:01:39.789 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: 2023-10-01T09:01:38.526600+0000 mon.a (mon 2023-10-01T09:01:39.789 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: .0) 547 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:39.789 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:01:39.789 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: 01T09:01:38.528335+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:39.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:01:39.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: 10-01T09:01:38.529055+0000 mon.a (mon.0) 549 : 2023-10-01T09:01:39.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:39.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:01:39.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: 10-01T09:01:38.533750+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:39.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: audit 2023-10-01T09:01:39.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: 2023-10-01T09:01:38.539403+0000 mon.a 2023-10-01T09:01:39.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: (mon.0) 551 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-01T09:01:39.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[109959]: 2023-10-01T09:01:39.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: audit 2023-10-01T09:01:38.465589+0000 mgr.y (mgr.14146) 2023-10-01T09:01:39.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: 172 : audit [DBG] from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi190=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:01:39.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: cephadm 2023-10-01T09:01:38.467148+0000 mgr.y (mgr.14146) 173 : cephadm [INF] Saving service prometheus spec with placement smithi190=a;count:1 2023-10-01T09:01:39.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:01:39.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: 10-01T09:01:38.526600+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:39.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: audit 2023-10-01T09:01 2023-10-01T09:01:39.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: :38.528335+0000 mon.a (mon.0) 2023-10-01T09:01:39.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: 548 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:39.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: audit 2023-10-01T09:01:38.529055+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:39.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: audit 2023-10-01T09:01:39.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: 2023-10-01 2023-10-01T09:01:39.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: T09:01:38.533750+0000 mon.a ( 2023-10-01T09:01:39.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: mon.0) 550 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:39.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: 2023-10-01T09:01:39.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: audit 2023-10-01T09:01 2023-10-01T09:01:39.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: :38.539403+0000 mon.a (mon.0) 551 : audit 2023-10-01T09:01:39.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:39 smithi138 conmon[100289]: [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-01T09:01:39.795 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:39 smithi190 conmon[102823]: ignoring --setuser ceph since I am not root 2023-10-01T09:01:39.796 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:39 smithi190 conmon[102823]: ignoring --setgroup ceph since I am not root 2023-10-01T09:01:39.796 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:39 smithi190 conmon[102823]: debug 2023-10-01T09:01:39.580+0000 7fef7c498700 1 -- 172.21.15.190:0/554954783 <== mon.1 v2:172.21.15.190:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fe7b6f24e0 con 0x55fe7b6cac00 2023-10-01T09:01:39.796 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:39 smithi190 conmon[102823]: debug 2023-10-01T09:01:39.712+0000 7fef8c65f680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T09:01:39.796 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:39 smithi190 conmon[102823]: debug 2023-10-01T09:01:39.794+0000 7fef8c65f680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T09:01:39.796 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:39 smithi190 conmon[100518]: audit 2023-10-01T09:01:38.465589+0000 mgr.y (mgr.14146) 172 : audit [DBG] from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi190=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:01:39.796 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:39 smithi190 conmon[100518]: cephadm 2023-10-01T09:01:38.467148+0000 mgr.y (mgr.14146) 173 : cephadm [INF] Saving service prometheus spec with placement smithi190=a;count:1 2023-10-01T09:01:39.796 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:39 smithi190 conmon[100518]: audit 2023-10-01T09:01:38.526600+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:39.796 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:39 smithi190 conmon[100518]: audit 2023-10-01T09:01:38.528335+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:39.796 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:39 smithi190 conmon[100518]: audit 2023-10-01T09:01:38.529055+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:39.796 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:39 smithi190 conmon[100518]: audit 2023-10-01T09:01:38.533750+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' 2023-10-01T09:01:39.797 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:39 smithi190 conmon[100518]: audit 2023-10-01T09:01:38.539403+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-01T09:01:40.440 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:40 smithi190 conmon[102823]: debug 2023-10-01T09:01:40.179+0000 7fef8c65f680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T09:01:40.547 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:40 smithi138 conmon[100738]: debug 2023-10-01T09:01:40.145+0000 7f204302d680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T09:01:40.726 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:40 smithi190 conmon[100518]: audit 2023-10-01T09:01:39.546956+0000 mon.a ( 2023-10-01T09:01:40.726 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:40 smithi190 conmon[100518]: mon.0) 552 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-01T09:01:40.726 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:40 smithi190 conmon[100518]: cluster 2023-10-01T09:01:39.547037+0000 mon.a (mon.0) 553 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2023-10-01T09:01:40.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:40 smithi138 conmon[100289]: audit 2023-10-01T09:01:39.546956+0000 mon.a ( 2023-10-01T09:01:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:40 smithi138 conmon[100289]: mon.0) 552 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-01T09:01:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:40 smithi138 conmon[100289]: cluster 2023-10-01T09:01:39.547037+0000 mon.a (mon.0) 553 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2023-10-01T09:01:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:40 smithi138 conmon[109959]: audit 2023-10-01T09:01: 2023-10-01T09:01:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:40 smithi138 conmon[109959]: 39.546956+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14146 172.21.15.138:0/1063312022' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-01T09:01:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:40 smithi138 conmon[109959]: cluster 2023-10-01T09:01:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:40 smithi138 conmon[109959]: 2023-10-01T09:01:39 2023-10-01T09:01:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:40 smithi138 conmon[109959]: .547037+0000 mon.a (mon.0 2023-10-01T09:01:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:40 smithi138 conmon[109959]: ) 553 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2023-10-01T09:01:41.174 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:40 smithi138 conmon[100738]: debug 2023-10-01T09:01:40.880+0000 7f204302d680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T09:01:41.174 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:40 smithi138 conmon[100738]: debug 2023-10-01T09:01:40.965+0000 7f204302d680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T09:01:41.258 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:40 smithi190 conmon[102823]: debug 2023-10-01T09:01:40.962+0000 7fef8c65f680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T09:01:41.258 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:41 smithi190 conmon[102823]: debug 2023-10-01T09:01:41.047+0000 7fef8c65f680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T09:01:41.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:41 smithi138 conmon[100738]: debug 2023-10-01T09:01:41.174+0000 7f204302d680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T09:01:42.396 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:42 smithi138 conmon[100738]: debug 2023-10-01T09:01:42.101+0000 7f204302d680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T09:01:42.397 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:42 smithi138 conmon[100738]: debug 2023-10-01T09:01:42.228+0000 7f204302d680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:01:42.397 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:42 smithi138 conmon[100738]: debug 2023-10-01T09:01:42.306+0000 7f204302d680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T09:01:42.479 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:41 smithi190 conmon[102823]: debug 2023-10-01T09:01:41.257+0000 7fef8c65f680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T09:01:42.479 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:42 smithi190 conmon[102823]: debug 2023-10-01T09:01:42.183+0000 7fef8c65f680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T09:01:42.479 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:42 smithi190 conmon[102823]: debug 2023-10-01T09:01:42.311+0000 7fef8c65f680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:01:42.479 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:42 smithi190 conmon[102823]: debug 2023-10-01T09:01:42.389+0000 7fef8c65f680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T09:01:42.806 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:42 smithi190 conmon[102823]: debug 2023-10-01T09:01:42.478+0000 7fef8c65f680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T09:01:42.806 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:42 smithi190 conmon[102823]: debug 2023-10-01T09:01:42.673+0000 7fef8c65f680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T09:01:42.819 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:42 smithi138 conmon[100738]: debug 2023-10-01T09:01:42.396+0000 7f204302d680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T09:01:42.819 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:42 smithi138 conmon[100738]: debug 2023-10-01T09:01:42.591+0000 7f204302d680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T09:01:43.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:42 smithi190 conmon[102823]: debug 2023-10-01T09:01:42.949+0000 7fef8c65f680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T09:01:43.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:43 smithi190 conmon[102823]: debug 2023-10-01T09:01:43.046+0000 7fef8c65f680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T09:01:43.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:42 smithi138 conmon[100738]: debug 2023-10-01T09:01:42.868+0000 7f204302d680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T09:01:43.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:42 smithi138 conmon[100738]: debug 2023-10-01T09:01:42.966+0000 7f204302d680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T09:01:44.029 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:43 smithi138 conmon[100738]: debug 2023-10-01T09:01:43.682+0000 7f204302d680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T09:01:44.030 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:43 smithi138 conmon[100738]: debug 2023-10-01T09:01:43.763+0000 7f204302d680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T09:01:44.030 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:43 smithi138 conmon[100738]: debug 2023-10-01T09:01:43.856+0000 7f204302d680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T09:01:44.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:43 smithi190 conmon[102823]: debug 2023-10-01T09:01:43.761+0000 7fef8c65f680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T09:01:44.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:43 smithi190 conmon[102823]: debug 2023-10-01T09:01:43.843+0000 7fef8c65f680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T09:01:44.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:43 smithi190 conmon[102823]: debug 2023-10-01T09:01:43.933+0000 7fef8c65f680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T09:01:44.288 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100738]: debug 2023-10-01T09:01:44.029+0000 7f204302d680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T09:01:44.364 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[102823]: debug 2023-10-01T09:01:44.105+0000 7fef8c65f680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T09:01:44.549 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100738]: debug 2023-10-01T09:01:44.287+0000 7f204302d680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T09:01:44.549 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100738]: debug 2023-10-01T09:01:44.417+0000 7f204302d680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T09:01:44.550 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100738]: debug 2023-10-01T09:01:44.549+0000 7f204302d680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:01:44.625 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[102823]: debug 2023-10-01T09:01:44.362+0000 7fef8c65f680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T09:01:44.625 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[102823]: debug 2023-10-01T09:01:44.493+0000 7fef8c65f680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T09:01:44.804 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100738]: debug 2023-10-01T09:01:44.719+0000 7f204302d680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T09:01:44.882 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[102823]: debug 2023-10-01T09:01:44.624+0000 7fef8c65f680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:01:44.882 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[102823]: debug 2023-10-01T09:01:44.794+0000 7fef8c65f680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T09:01:44.883 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[102823]: debug 2023-10-01T09:01:44.881+0000 7fef8c65f680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T09:01:44.883 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: cluster 2023-10-01T09:01:44.817338+0000 mon.a (mon. 2023-10-01T09:01:44.883 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: 0) 554 : cluster [INF] Active manager daemon y restarted 2023-10-01T09:01:44.883 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: cluster 2023-10-01T09:01:44.817645+0000 mon.a (mon.0) 555 : cluster [INF] Activating manager daemon y 2023-10-01T09:01:44.883 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: cluster 2023-10-01T09:01:44.822712 2023-10-01T09:01:44.884 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: +0000 mon.a (mon.0) 556 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-01T09:01:44.884 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: 2023-10-01T09:01:44.884 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: cluster 2023-10-01T09:01:44 2023-10-01T09:01:44.884 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: .823094+0000 mon.a (mon.0) 557 : cluster 2023-10-01T09:01:44.884 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: [DBG] mgrmap e16: y(active, starting, since 0.00553992s), standbys: x 2023-10-01T09:01:44.884 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023-10-01T09:01: 2023-10-01T09:01:44.884 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: 44 2023-10-01T09:01:44.884 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: .826649+0000 mon.c (mon.2) 20 : audit 2023-10-01T09:01:44.884 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:01:44.884 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T09:01:44.885 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: T09:01:44.826867+0000 mon.c 2023-10-01T09:01:44.885 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: (mon.2) 21 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:01:44.885 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023 2023-10-01T09:01:44.885 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: -10-01T09:01:44.827062+0000 2023-10-01T09:01:44.885 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: mon.c (mon.2) 22 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:01:44.885 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.885 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: 2023-10-01T09:01:44.827830+0000 2023-10-01T09:01:44.885 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: mon.c (mon.2) 23 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T09:01:44.885 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023 2023-10-01T09:01:44.885 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: -10-01T09:01:44.828002+0000 mon.c 2023-10-01T09:01:44.886 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: (mon.2) 24 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T09:01:44.886 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023 2023-10-01T09:01:44.886 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: -10-01T09:01:44.828196+0000 2023-10-01T09:01:44.886 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: mon.c (mon.2) 25 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T09:01:44.886 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: 2023-10-01T09:01:44.886 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.886 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: 2023-10-01T09:01:44.828454+0000 mon.c (mon 2023-10-01T09:01:44.886 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: .2) 26 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T09:01:44.886 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:01:44.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: 10-01T09:01:44.828672+0000 mon.c 2023-10-01T09:01:44.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: (mon.2) 27 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T09:01:44.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: 2023-10-01T09:01:44.828913+0000 2023-10-01T09:01:44.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: mon.c (mon.2) 28 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:01:44.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: 2023-10-01T09:01:44.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.829223 2023-10-01T09:01:44.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: +0000 mon.c (mon.2) 29 : audit [DBG] 2023-10-01T09:01:44.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:01:44.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023-10-01T09:01:44 2023-10-01T09:01:44.888 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: .829498+0000 mon.c (mon.2) 30 : audit 2023-10-01T09:01:44.888 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:01:44.888 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023-10-01T09:01 2023-10-01T09:01:44.888 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: :44.829733+0000 mon.c (mon.2) 31 2023-10-01T09:01:44.888 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:44.888 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:01:44.888 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: -01T09:01:44.829957+0000 mon.c (mon.2) 32 2023-10-01T09:01:44.888 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:44.888 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:01:44.888 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: 10-01T09:01:44.830183+0000 mon.c 2023-10-01T09:01:44.888 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: (mon.2) 33 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T09:01:44.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: 2023-10-01T09:01:44.830373+0000 2023-10-01T09:01:44.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: mon.c (mon.2) 34 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T09:01:44.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: 2023-10-01T09:01:44.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: 2023-10-01T09:01:44.830785+0000 mon.c ( 2023-10-01T09:01:44.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: mon.2) 35 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T09:01:44.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: cluster 2023 2023-10-01T09:01:44.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: -10-01T09:01:44.845462+0000 2023-10-01T09:01:44.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: mon.a (mon.0) 558 : cluster [INF] Manager daemon y is now available 2023-10-01T09:01:44.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[100518]: 2023-10-01T09:01:45.055 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100738]: debug 2023-10-01T09:01:44.803+0000 7f204302d680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T09:01:45.055 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100738]: [01/Oct/2023:09:01:44] ENGINE Bus STARTING 2023-10-01T09:01:45.055 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100738]: [80B blob data] 2023-10-01T09:01:45.055 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100738]: [80B blob data] 2023-10-01T09:01:45.056 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: cluster 2023-10-01T09:01:44. 2023-10-01T09:01:45.056 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 817338+0000 mon.a (mon.0) 554 : cluster [INF] Active manager daemon y restarted 2023-10-01T09:01:45.056 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: cluster 2023-10-01T09:01:45.056 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 2023-10-01T09:01:44.817645+0000 mon.a (mon.0) 555 : cluster 2023-10-01T09:01:45.056 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: [INF] Activating manager daemon y 2023-10-01T09:01:45.056 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: cluster 2023 2023-10-01T09:01:45.056 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: -10-01T09:01:44.822712+0000 mon.a 2023-10-01T09:01:45.057 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: (mon.0) 556 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-01T09:01:45.057 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: cluster 2023-10-01T09:01:45.057 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 2023-10-01T09:01:44.823094+0000 mon.a 2023-10-01T09:01:45.057 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: (mon.0) 557 : cluster [DBG] mgrmap e16: y(active, starting, since 0.00553992s), standbys: x 2023-10-01T09:01:45.057 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023-10-01T09:01:45.057 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 2023-10-01T09:01:44.826649+0000 2023-10-01T09:01:45.057 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: mon.c (mon.2) 20 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:01:45.057 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 2023-10-01T09:01:45.058 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023-10-01T09:01:45.058 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 2023-10-01T09:01:44.826867+0000 mon.c ( 2023-10-01T09:01:45.058 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: mon.2) 21 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:01:45.058 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023 2023-10-01T09:01:45.058 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: -10-01T09:01:44.827062+0000 mon.c 2023-10-01T09:01:45.058 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: (mon.2) 22 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:01:45.058 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:01:45.058 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 10-01T09:01:44.827830+0000 mon.c (mon 2023-10-01T09:01:45.058 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: .2) 23 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T09:01:45.058 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023 2023-10-01T09:01:45.058 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: -10-01T09:01:44.828002+0000 2023-10-01T09:01:45.059 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: mon.c (mon.2) 24 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T09:01:45.059 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023 2023-10-01T09:01:45.059 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: -10-01T09:01:44.828196+0000 mon.c 2023-10-01T09:01:45.059 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: (mon.2) 25 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T09:01:45.059 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023-10-01T09:01:45.059 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 2023-10-01T09:01 2023-10-01T09:01:45.059 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: :44.828454+0000 mon.c (mon.2) 26 2023-10-01T09:01:45.059 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T09:01:45.059 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:01:45.059 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 09:01:44.828672+0000 mon.c (mon. 2023-10-01T09:01:45.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 2) 27 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T09:01:45.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023-10-01T09:01:45.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 2023-10-01T09:01 2023-10-01T09:01:45.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: :44.828913+0000 mon.c (mon.2) 2023-10-01T09:01:45.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 28 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:01:45.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:01:45.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 01T09:01:44.829223+0000 mon.c (mon 2023-10-01T09:01:45.060 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: cluster 2023-10-01T09:01:44.817338+0000 mon.a 2023-10-01T09:01:45.061 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: (mon.0) 554 : cluster [INF] Active manager daemon y restarted 2023-10-01T09:01:45.061 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: cluster 2023-10-01T09:01:44.817645+0000 mon.a (mon.0) 555 : cluster [INF] Activating manager daemon y 2023-10-01T09:01:45.061 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: cluster 2023-10-01T09:01: 2023-10-01T09:01:45.061 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 44.822712+0000 mon.a (mon.0) 556 : cluster [DBG] 2023-10-01T09:01:45.061 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: osdmap e51: 8 total, 8 up, 8 in 2023-10-01T09:01:45.061 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: cluster 2023-10-01T09 2023-10-01T09:01:45.061 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: :01:44.823094+0000 mon.a (mon.0) 557 2023-10-01T09:01:45.061 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: : cluster [DBG] mgrmap e16: y(active, starting, since 0.00553992s), standbys: x 2023-10-01T09:01:45.061 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:01:45.061 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 01T09:01:44.826649+0000 mon.c ( 2023-10-01T09:01:45.061 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: mon.2) 20 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:01:45.062 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023 2023-10-01T09:01:45.062 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: -10-01T09:01:44.826867+0000 2023-10-01T09:01:45.062 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: mon.c (mon.2) 21 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:01:45.062 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023 2023-10-01T09:01:45.062 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: -10-01T09:01:44.827062+0000 2023-10-01T09:01:45.062 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: mon.c (mon.2) 22 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:01:45.062 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:01:45.062 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 10-01T09:01:44.827830+0000 mon.c ( 2023-10-01T09:01:45.062 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: mon.2) 23 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T09:01:45.062 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023 2023-10-01T09:01:45.062 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: -10-01T09:01:44.828002+0000 2023-10-01T09:01:45.062 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: mon.c (mon.2) 24 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T09:01:45.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023 2023-10-01T09:01:45.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: -10-01T09:01:44.828196+0000 2023-10-01T09:01:45.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: mon.c (mon.2) 25 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T09:01:45.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 2023-10-01T09:01:45.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023-10-01T09:01:45.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 2023-10-01T09:01:44.828454+0000 mon.c (mon 2023-10-01T09:01:45.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: .2) 26 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T09:01:45.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:01:45.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 01T09:01:44.828672+0000 mon.c ( 2023-10-01T09:01:45.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: mon.2) 27 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T09:01:45.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023 2023-10-01T09:01:45.064 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: -10-01T09:01:44.828913+0000 2023-10-01T09:01:45.064 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: mon.c (mon.2) 28 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:01:45.064 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023 2023-10-01T09:01:45.064 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: -10-01T09:01:44.829223+0000 2023-10-01T09:01:45.064 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: mon.c (mon.2) 29 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:01:45.064 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023-10-01T09:01:45.064 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: .2) 29 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:01:45.064 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023-10-01T09:01:45.065 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 2023-10-01T09:01: 2023-10-01T09:01:45.065 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 44.829498+0000 mon.c (mon.2) 30 : audit 2023-10-01T09:01:45.065 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:01:45.065 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023-10-01T09:01:45.065 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 2023-10-01T09:01:44.829733 2023-10-01T09:01:45.065 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: +0000 mon.c (mon.2) 31 : audit [DBG] 2023-10-01T09:01:45.065 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:45.065 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:01:45.065 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 10-01T09:01:44.829957+0000 mon.c (mon. 2023-10-01T09:01:45.065 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 2) 32 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:45.065 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023-10-01T09:01:45.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 2023-10-01T09:01:44 2023-10-01T09:01:45.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: .830183+0000 mon.c (mon.2) 33 : audit 2023-10-01T09:01:45.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T09:01:45.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:01:45.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 01:44.830373+0000 mon.c (mon.2) 2023-10-01T09:01:45.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 34 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T09:01:45.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: audit 2023-10-01T09:01:45.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 2023-10-01T09:01:44.830785+0000 2023-10-01T09:01:45.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: mon.c (mon.2) 35 : audit [DBG] 2023-10-01T09:01:45.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T09:01:45.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: cluster 2023-10-01T09:01:45.067 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: 2023-10-01T09:01:44.845462+0000 mon.a ( 2023-10-01T09:01:45.067 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[109959]: mon.0) 558 : cluster [INF] Manager daemon y is now available 2023-10-01T09:01:45.067 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 2023-10-01T09:01:44.829498+0000 2023-10-01T09:01:45.067 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: mon.c (mon.2) 30 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:01:45.067 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 2023-10-01T09:01:45.067 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023-10-01T09:01:45.067 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 2023-10-01T09:01: 2023-10-01T09:01:45.067 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 44.829733+0000 mon.c (mon.2) 31 : 2023-10-01T09:01:45.067 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:01:45.067 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023-10-01T09:01:45.068 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 2023-10-01T09:01:44.829957+0000 2023-10-01T09:01:45.068 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: mon.c (mon.2) 32 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:01:45.068 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:01:45.068 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 09:01:44.830183+0000 mon.c (mon.2) 33 : 2023-10-01T09:01:45.068 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T09:01:45.068 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023 2023-10-01T09:01:45.068 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: -10-01T09:01:44.830373+0000 2023-10-01T09:01:45.068 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: mon.c (mon.2) 34 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T09:01:45.068 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 2023-10-01T09:01:45.068 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: audit 2023-10-01T09:01:45.068 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 2023-10-01T09:01:44.830785+0000 mon.c ( 2023-10-01T09:01:45.068 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: mon.2) 35 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T09:01:45.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: cluster 2023-10-01T09:01:45.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: 2023-10-01T09:01:44 2023-10-01T09:01:45.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: .845462+0000 mon.a (mon.0) 558 : cluster 2023-10-01T09:01:45.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:44 smithi138 conmon[100289]: [INF] Manager daemon y is now available 2023-10-01T09:01:45.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:44 smithi190 conmon[102823]: [01/Oct/2023:09:01:44] ENGINE Bus STARTING 2023-10-01T09:01:45.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[102823]: CherryPy Checker: 2023-10-01T09:01:45.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[102823]: The Application mounted at '' has an empty config. 2023-10-01T09:01:45.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[102823]: 2023-10-01T09:01:45.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[102823]: [01/Oct/2023:09:01:45] ENGINE Serving on http://:::9283 2023-10-01T09:01:45.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[102823]: [01/Oct/2023:09:01:45] ENGINE Bus STARTED 2023-10-01T09:01:45.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100738]: CherryPy Checker: 2023-10-01T09:01:45.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100738]: The Application mounted at '' has an empty config. 2023-10-01T09:01:45.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100738]: 2023-10-01T09:01:45.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100738]: [01/Oct/2023:09:01:45] ENGINE Serving on http://:::9283 2023-10-01T09:01:45.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100738]: [01/Oct/2023:09:01:45] ENGINE Bus STARTED 2023-10-01T09:01:45.837 INFO:teuthology.orchestra.run.smithi190.stdout:Scheduled node-exporter update... 2023-10-01T09:01:46.132 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: cluster 2023-10-01T09:01:44.896792+0000 mon.a (mon.0) 559 : cluster [DBG] Standby manager daemon x restarted 2023-10-01T09:01:46.132 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: cluster 2023-10-01T09:01:44.896879+0000 mon.a (mon.0) 560 : cluster [DBG] Standby manager daemon x started 2023-10-01T09:01:46.132 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.898120+0000 mon.b (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.190:0/1743631852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:01:46.132 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.898675+0000 mon.b (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.190:0/1743631852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:01:46.133 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.900589+0000 mon.b (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.190:0/1743631852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T09:01:46.133 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.901650+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.190:0/1743631852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:01:46.133 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.942180+0000 mon.c (mon.2) 36 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:46.133 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.943608+0000 mon.c (mon.2) 37 2023-10-01T09:01:46.133 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:46.133 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.977020+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:46.133 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.987442+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:01:46.133 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.987749+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:01:46.133 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.991188+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T09:01:46.133 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: audit 2023-10-01T09:01:44.991535+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T09:01:46.133 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: cluster 2023-10-01T09:01:45.827694+0000 mon.a (mon.0) 564 : cluster [DBG] mgrmap e17: y(active, since 1.01013s), standbys: x 2023-10-01T09:01:46.134 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:45 smithi190 conmon[100518]: audit 2023-10-01T09:01:45.836238+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:46.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: cluster 2023-10-01T09:01:44.896792+0000 mon.a (mon. 2023-10-01T09:01:46.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: 0) 559 : cluster [DBG] Standby manager daemon x restarted 2023-10-01T09:01:46.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: cluster 2023-10-01T09:01:44.896879+0000 mon.a (mon.0) 560 : cluster [DBG] Standby manager daemon x started 2023-10-01T09:01:46.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: audit 2023-10-01T09:01:44.898120+0000 mon.b (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.190:0/1743631852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:01:46.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: audit 2023-10-01T09:01:44.898675+0000 mon.b (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.190:0/1743631852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:01:46.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: audit 2023-10-01T09:01:44.900589+0000 mon.b (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.190:0/1743631852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T09:01:46.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: audit 2023-10-01T09:01:44.901650+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.190:0/1743631852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:01:46.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: audit 2023-10-01T09:01:44.942180+0000 mon.c (mon.2) 36 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:46.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: audit 2023-10-01T09:01:44.943608+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:46.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: audit 2023-10-01T09:01:44.977020+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:46.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: audit 2023-10-01T09:01:44.987442+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:01:46.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: audit 2023-10-01T09:01:44.987749+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:01:46.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: audit 2023-10-01T09:01:44.991188+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T09:01:46.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: audit 2023-10-01T09:01:44.991535+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T09:01:46.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: cluster 2023-10-01T09:01:45.827694+0000 mon.a (mon.0) 564 : cluster [DBG] mgrmap e17: y(active, since 1.01013s), standbys: x 2023-10-01T09:01:46.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[100289]: audit 2023-10-01T09:01:45.836238+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:46.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: cluster 2023-10-01T09:01:44.896792+0000 mon.a ( 2023-10-01T09:01:46.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: mon.0) 559 : cluster [DBG] Standby manager daemon x restarted 2023-10-01T09:01:46.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: cluster 2023-10-01T09:01:44.896879+0000 mon.a (mon.0) 560 : cluster [DBG] Standby manager daemon x started 2023-10-01T09:01:46.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: audit 2023-10-01T09:01:44.898120+0000 mon.b (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.190:0/1743631852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:01:46.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: audit 2023-10-01T09:01:44.898675+0000 mon.b (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.190:0/1743631852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:01:46.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: audit 2023-10-01T09:01:44.900589+0000 mon.b (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.190:0/1743631852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T09:01:46.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: audit 2023-10-01T09:01:44.901650+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.190:0/1743631852' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:01:46.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: audit 2023-10-01T09:01:44.942180+0000 mon.c (mon.2) 36 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:46.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: audit 2023-10-01T09:01:44.943608+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:01:46.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: audit 2023-10-01T09:01:44.977020+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:46.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: audit 2023-10-01T09:01:44.987442+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:01:46.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: audit 2023-10-01T09:01:44.987749+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:01:46.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: audit 2023-10-01T09:01:44.991188+0000 mon.c (mon.2) 39 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T09:01:46.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: audit 2023-10-01T09:01:44.991535+0000 mon.a (mon.0) 563 : audit [INF] 2023-10-01T09:01:46.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T09:01:46.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: cluster 2023-10 2023-10-01T09:01:46.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: -01T09:01:45.827694+0000 mon.a ( 2023-10-01T09:01:46.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: mon.0) 564 : cluster [DBG] mgrmap e17: y(active, since 1.01013s), standbys: x 2023-10-01T09:01:46.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: audit 2023 2023-10-01T09:01:46.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: -10-01T09:01:45. 2023-10-01T09:01:46.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: 836238+0000 mon.a (mon.0) 565 : audit [INF] 2023-10-01T09:01:46.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:45 smithi138 conmon[109959]: from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:46.569 DEBUG:teuthology.orchestra.run.smithi138:node-exporter.a> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@node-exporter.a.service 2023-10-01T09:01:46.571 DEBUG:teuthology.orchestra.run.smithi190:node-exporter.b> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@node-exporter.b.service 2023-10-01T09:01:46.573 INFO:tasks.cephadm:Adding alertmanager.a on smithi138 2023-10-01T09:01:46.574 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi138=a' 2023-10-01T09:01:46.602 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:-- Logs begin at Sun 2023-10-01 08:38:26 UTC. -- 2023-10-01T09:01:46.604 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:-- Logs begin at Sun 2023-10-01 08:38:18 UTC. -- 2023-10-01T09:01:47.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:47 smithi190 conmon[100518]: cephadm 2023-10-01T09:01:45.832679+0000 mgr.y (mgr.14433) 2 : cephadm [INF] Saving service node-exporter spec with placement smithi138=a;smithi190=b;count:2 2023-10-01T09:01:47.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:47 smithi190 conmon[100518]: cluster 2023-10-01T09:01:45.833030+0000 mgr.y (mgr.14433) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:47.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:47 smithi190 conmon[100518]: audit 2023-10-01T09:01:46.410098+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:47 smithi190 conmon[100518]: audit 2023-10-01T09:01:46.812552+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:47 smithi190 conmon[100518]: audit 2023-10-01T09:01:47.176880+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:47.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:47 smithi190 conmon[100518]: audit 2023-10-01T09:01:47.177728+0000 mon.c (mon.2) 41 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:47.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:47 smithi190 conmon[100518]: audit 2023-10-01T09:01:47.233536+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:47 smithi190 conmon[100518]: audit 2023-10-01T09:01:47.256039+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:47 smithi190 conmon[100518]: audit 2023-10-01T09:01:47.260001+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:47 smithi190 conmon[100518]: audit 2023-10-01T09:01:47.264131+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-01T09:01:47.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:47 smithi190 conmon[100518]: audit 2023-10-01T09:01:47.264438+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-01T09:01:47.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[109959]: cephadm 2023-10-01T09:01:45.832679+0000 mgr.y (mgr.14433) 2 : cephadm [INF] Saving service node-exporter spec with placement smithi138=a;smithi190=b;count:2 2023-10-01T09:01:47.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[109959]: cluster 2023-10-01T09:01:45.833030+0000 mgr.y (mgr.14433) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:47.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[109959]: audit 2023-10-01T09:01:46.410098+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[109959]: audit 2023-10-01T09:01:46.812552+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[109959]: audit 2023-10-01T09:01:47.176880+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:47.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[109959]: audit 2023-10-01T09:01:47.177728+0000 mon.c (mon.2) 41 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:47.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[109959]: audit 2023-10-01T09:01:47.233536+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[109959]: audit 2023-10-01T09:01:47.256039+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[109959]: audit 2023-10-01T09:01:47.260001+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[109959]: audit 2023-10-01T09:01:47.264131+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-01T09:01:47.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[109959]: audit 2023-10-01T09:01:47.264438+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-01T09:01:47.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[100289]: cephadm 2023-10-01T09:01:45.832679+0000 mgr.y (mgr.14433) 2 : cephadm [INF] Saving service node-exporter spec with placement smithi138=a;smithi190=b;count:2 2023-10-01T09:01:47.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[100289]: cluster 2023-10-01T09:01:45.833030+0000 mgr.y (mgr.14433) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:47.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[100289]: audit 2023-10-01T09:01:46.410098+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[100289]: audit 2023-10-01T09:01:46.812552+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[100289]: audit 2023-10-01T09:01:47.176880+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:01:47.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[100289]: audit 2023-10-01T09:01:47.177728+0000 2023-10-01T09:01:47.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[100289]: mon.c (mon.2) 41 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:01:47.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[100289]: audit 2023-10-01T09:01:47.233536+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[100289]: audit 2023-10-01T09:01:47.256039+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[100289]: audit 2023-10-01T09:01:47.260001+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:47.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[100289]: audit 2023-10-01T09:01:47.264131+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-01T09:01:47.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:47 smithi138 conmon[100289]: audit 2023-10-01T09:01:47.264438+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-01T09:01:48.516 INFO:teuthology.orchestra.run.smithi190.stdout:Scheduled alertmanager update... 2023-10-01T09:01:48.673 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:48 smithi190 conmon[100518]: cluster 2023-10-01T09:01:46 2023-10-01T09:01:48.674 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:48 smithi190 conmon[100518]: .829039+0000 mgr.y (mgr.14433) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:48.674 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:48 smithi190 conmon[100518]: cephadm 2023-10-01T09:01:47.178890+0000 mgr.y (mgr.14433) 5 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:01:48.674 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:48 smithi190 conmon[100518]: cephadm 2023-10-01T09:01:47.179109+0000 mgr.y (mgr.14433) 6 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:01:48.674 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:48 smithi190 conmon[100518]: cephadm 2023-10-01T09:01:47.202672+0000 mgr.y (mgr.14433) 7 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:01:48.674 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:48 smithi190 conmon[100518]: cephadm 2023-10-01T09:01:47.215386+0000 mgr.y (mgr.14433) 8 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:01:48.674 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:48 smithi190 conmon[100518]: audit 2023-10-01T09:01:47.834526+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-01T09:01:48.675 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:48 smithi190 conmon[100518]: cluster 2023-10-01T09:01:47.834625+0000 mon.a (mon.0) 573 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-10-01T09:01:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[109959]: cluster 2023-10-01T09:01:46.829039+0000 mgr.y (mgr.14433) 4 : 2023-10-01T09:01:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[109959]: cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[109959]: cephadm 2023-10-01T09:01:47.178890+0000 mgr.y (mgr.14433) 5 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:01:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[109959]: cephadm 2023-10-01T09:01:47.179109+0000 mgr.y (mgr.14433) 6 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:01:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[109959]: cephadm 2023-10-01T09: 2023-10-01T09:01:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[109959]: 01:47.202672+0000 mgr.y (mgr.14433) 7 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:01:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[109959]: cephadm 2023- 2023-10-01T09:01:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[109959]: 10-01T09:01:47.215386+0000 mgr.y (mgr.14433) 8 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:01:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[109959]: audit 2023-10-01T09:01:47.834526+0000 mon.a ( 2023-10-01T09:01:48.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[109959]: mon.0) 572 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-01T09:01:48.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[109959]: cluster 2023-10-01T09:01:47.834625+0000 mon.a (mon.0) 573 : cluster 2023-10-01T09:01:48.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[109959]: [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-10-01T09:01:48.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: cluster 2023-10-01T09:01:46.829039+0000 mgr.y (mgr.14433) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:48.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: 2023-10-01T09:01:48.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: cephadm 2023-10-01T09:01:47.178890+0000 mgr.y (mgr.14433) 5 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:01:48.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: cephadm 2023-10-01T09:01:47 2023-10-01T09:01:48.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: .179109+0000 mgr.y (mgr.14433) 6 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:01:48.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: cephadm 2023-10-01T09:01:48.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: 2023-10-01T09:01:47.202672+0000 mgr.y (mgr.14433) 7 : cephadm [INF] 2023-10-01T09:01:48.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:01:48.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: cephadm 2023-10-01T09:01:47.215386 2023-10-01T09:01:48.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: +0000 mgr.y (mgr.14433) 8 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:01:48.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: audit 2023-10-01T09:01:47.834526+0000 2023-10-01T09:01:48.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: mon.a (mon.0) 572 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-01T09:01:48.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: cluster 2023-10-01T09:01:48.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: 2023-10-01T09:01:47. 2023-10-01T09:01:48.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: 834625+0000 mon.a (mon.0) 573 : cluster [DBG] 2023-10-01T09:01:48.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:48 smithi138 conmon[100289]: mgrmap e18: y(active, since 3s), standbys: x 2023-10-01T09:01:49.097 DEBUG:teuthology.orchestra.run.smithi138:alertmanager.a> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@alertmanager.a.service 2023-10-01T09:01:49.100 INFO:tasks.cephadm:Adding grafana.a on smithi190 2023-10-01T09:01:49.100 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph orch apply grafana '1;smithi190=a' 2023-10-01T09:01:49.132 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:-- Logs begin at Sun 2023-10-01 08:38:26 UTC. -- 2023-10-01T09:01:49.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:49 smithi190 conmon[100518]: cephadm 2023-10-01T09:01:47.846093+0000 mgr.y (mgr 2023-10-01T09:01:49.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:49 smithi190 conmon[100518]: .14433) 9 : cephadm [INF] Deploying daemon prometheus.a on smithi190 2023-10-01T09:01:49.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:49 smithi190 conmon[100518]: audit 2023-10-01T09:01:48.510078+0000 mgr.y (mgr.14433) 10 : audit [DBG] from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:01:49.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:49 smithi190 conmon[100518]: cephadm 2023-10-01T09:01:48.511561+0000 mgr.y (mgr.14433) 11 : cephadm [INF] Saving service alertmanager spec with placement smithi138=a;count:1 2023-10-01T09:01:49.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:49 smithi190 conmon[100518]: audit 2023-10-01T09:01:48.515503+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:49.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[100289]: cephadm 2023-10-01T09:01:47.846093+0000 mgr.y (mgr.14433) 2023-10-01T09:01:49.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[100289]: 9 : cephadm [INF] Deploying daemon prometheus.a on smithi190 2023-10-01T09:01:49.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[100289]: audit 2023-10-01T09:01:48.510078+0000 mgr.y (mgr.14433) 10 : audit [DBG] from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:01:49.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[100289]: cephadm 2023-10-01T09:01:49.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[100289]: 2023-10-01T09:01:48.511561+0000 mgr.y (mgr.14433) 11 : cephadm [INF] Saving service alertmanager spec with placement smithi138=a;count:1 2023-10-01T09:01:49.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[100289]: audit 2023-10-01T09:01: 2023-10-01T09:01:49.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[100289]: 48.515503+0000 mon.a (mon.0) 574 : audit [INF] 2023-10-01T09:01:49.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[100289]: from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:49.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[109959]: cephadm 2023-10-01T09:01:47.846093+0000 2023-10-01T09:01:49.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[109959]: mgr.y (mgr.14433) 9 : cephadm [INF] Deploying daemon prometheus.a on smithi190 2023-10-01T09:01:49.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[109959]: audit 2023-10-01T09:01:49.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[109959]: 2023-10-01T09:01:48.510078+0000 mgr.y (mgr.14433 2023-10-01T09:01:49.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[109959]: ) 10 : audit [DBG] from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi138=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:01:49.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[109959]: cephadm 2023-10-01T09:01:49.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[109959]: 2023-10-01T09:01:48. 2023-10-01T09:01:49.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[109959]: 511561+0000 mgr.y (mgr.14433) 11 : cephadm [INF] Saving service alertmanager spec with placement smithi138=a;count:1 2023-10-01T09:01:49.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:01:49.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[109959]: 01T09:01:48.515503+0000 mon.a (mon.0 2023-10-01T09:01:49.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:49 smithi138 conmon[109959]: ) 574 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:50.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:50 smithi190 conmon[100518]: cluster 2023-10-01T09:01:48.829381+0000 mgr.y (mgr.14433) 12 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:50.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:50 smithi138 conmon[100289]: cluster 2023-10-01T09:01:48.829381+0000 mgr.y (mgr.14433) 12 2023-10-01T09:01:50.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:50 smithi138 conmon[100289]: : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:50 smithi138 conmon[109959]: cluster 2023-10-01T09:01:48.829381+0000 mgr.y (mgr.14433) 12 : 2023-10-01T09:01:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:50 smithi138 conmon[109959]: cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:52.000 INFO:teuthology.orchestra.run.smithi190.stdout:Scheduled grafana update... 2023-10-01T09:01:52.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:52 smithi190 conmon[100518]: cluster 2023-10-01T09:01:50.829702+0000 mgr.y (mgr 2023-10-01T09:01:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:52 smithi190 conmon[100518]: .14433) 13 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:52 smithi190 conmon[100518]: audit 2023-10-01T09:01:51.999386+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:52.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:52 smithi138 conmon[100289]: cluster 2023-10-01T09:01:50.829702+0000 mgr.y (mgr.14433 2023-10-01T09:01:52.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:52 smithi138 conmon[100289]: ) 13 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:52.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:52 smithi138 conmon[100289]: audit 2023-10-01T09:01:51.999386+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:52 smithi138 conmon[109959]: cluster 2023-10-01T09:01:50.829702+0000 mgr.y (mgr.14433) 2023-10-01T09:01:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:52 smithi138 conmon[109959]: 13 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:52 smithi138 conmon[109959]: audit 2023-10-01T09:01:51.999386+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:01:53.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:53 smithi190 conmon[100518]: audit 2023-10-01T09:01:51.993619+0000 mgr.y (mgr.14433 2023-10-01T09:01:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:53 smithi190 conmon[100518]: ) 14 : audit [DBG] from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi190=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:01:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:53 smithi190 conmon[100518]: cephadm 2023-10-01T09:01:51.994998+0000 mgr.y (mgr.14433) 15 : cephadm [INF] Saving service grafana spec with placement smithi190=a;count:1 2023-10-01T09:01:53.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:53 smithi138 conmon[100289]: audit 2023-10-01T09:01:51.993619+0000 mgr.y (mgr.14433) 14 : audit [DBG] from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi190=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:01:53.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:53 smithi138 conmon[100289]: cephadm 2023-10-01T09:01:51.994998+0000 mgr.y (mgr.14433) 15 : cephadm [INF] Saving service grafana spec with placement smithi190=a;count:1 2023-10-01T09:01:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:53 smithi138 conmon[109959]: audit 2023-10-01T09:01:51.993619+0000 mgr.y (mgr.14433) 14 2023-10-01T09:01:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:53 smithi138 conmon[109959]: : audit [DBG] from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi190=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:01:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:53 smithi138 conmon[109959]: cephadm 2023-10-01T09:01:51.994998+0000 mgr.y (mgr.14433) 15 : cephadm [INF] Saving service grafana spec with placement smithi190=a;count:1 2023-10-01T09:01:54.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:54 smithi190 conmon[100518]: cluster 2023-10-01T09:01:52.830255+0000 mgr.y (mgr.14433) 16 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:54.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:54 smithi138 conmon[109959]: cluster 2023-10-01T09:01:52.830255+0000 mgr.y (mgr.14433) 16 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:54 smithi138 conmon[109959]: 2023-10-01T09:01:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:54 smithi138 conmon[100289]: cluster 2023-10-01T09:01:52.830255+0000 mgr.y (mgr.14433) 16 : cluster [DBG] 2023-10-01T09:01:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:54 smithi138 conmon[100289]: pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:55.613 DEBUG:teuthology.orchestra.run.smithi190:grafana.a> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@grafana.a.service 2023-10-01T09:01:55.615 INFO:tasks.cephadm:Setting up client nodes... 2023-10-01T09:01:55.616 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-01T09:01:55.649 INFO:journalctl@ceph.grafana.a.smithi190.stdout:-- Logs begin at Sun 2023-10-01 08:38:18 UTC. -- 2023-10-01T09:01:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:56 smithi190 conmon[100518]: cluster 2023-10-01T09:01:54.830684+0000 mgr.y (mgr.14433) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:56.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:56 smithi138 conmon[100289]: cluster 2023-10-01T09:01:54.830684 2023-10-01T09:01:56.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:56 smithi138 conmon[100289]: +0000 mgr.y (mgr.14433) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:56.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:56 smithi138 conmon[109959]: cluster 2023-10-01T09:01:54.830684+0000 mgr.y 2023-10-01T09:01:56.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:56 smithi138 conmon[109959]: (mgr.14433) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:57.637 INFO:teuthology.orchestra.run.smithi138.stdout:[client.0] 2023-10-01T09:01:57.638 INFO:teuthology.orchestra.run.smithi138.stdout: key = AQCFNRllHkHJJRAA9+Fr3Da3tdnYozcq1qGj+Q== 2023-10-01T09:01:58.289 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T09:01:58.289 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-01T09:01:58.289 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-01T09:01:58.329 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-01T09:01:58.535 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:58 smithi190 conmon[100518]: cluster 2023-10-01T09:01:56.831261+0000 mgr.y (mgr.14433) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:58.536 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:58 smithi190 conmon[100518]: audit 2023-10-01T09:01:57.633461+0000 mon.c (mon.2) 43 : audit [INF] from='client.? 172.21.15.138:0/728111417' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T09:01:58.536 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:58 smithi190 conmon[100518]: audit 2023-10-01T09:01:57.633799+0000 mon.a (mon.0) 576 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T09:01:58.536 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:01:58 smithi190 conmon[100518]: audit 2023-10-01T09:01:57.637499+0000 mon.a (mon.0) 577 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T09:01:58.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:58 smithi138 conmon[109959]: cluster 2023-10-01T09:01:56.831261+0000 mgr.y (mgr.14433) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:58.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:58 smithi138 conmon[109959]: audit 2023-10-01T09:01:57.633461+0000 mon.c (mon.2) 43 : audit [INF] from='client.? 172.21.15.138:0/728111417' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T09:01:58.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:58 smithi138 conmon[109959]: audit 2023-10-01T09:01:57.633799+0000 mon.a (mon.0) 576 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T09:01:58.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:01:58 smithi138 conmon[109959]: audit 2023-10-01T09:01:57.637499+0000 mon.a (mon.0) 577 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T09:01:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:58 smithi138 conmon[100289]: cluster 2023-10-01T09:01:56.831261+0000 mgr.y (mgr.14433) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:01:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:58 smithi138 conmon[100289]: audit 2023-10-01T09:01:57.633461+0000 mon.c (mon.2) 43 : audit [INF] from='client.? 172.21.15.138:0/728111417' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T09:01:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:58 smithi138 conmon[100289]: audit 2023-10-01T09:01:57.633799+0000 mon.a (mon.0) 576 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T09:01:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:01:58 smithi138 conmon[100289]: audit 2023-10-01T09:01:57.637499+0000 mon.a (mon.0) 577 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T09:02:00.306 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:00 smithi190 systemd[1]: Started Ceph prometheus.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:02:00.307 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:00 smithi190 conmon[129942]: ts=2023-10-01T09:02:00.049Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-01T09:02:00.307 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:00 smithi190 conmon[129942]: ts=2023-10-01T09:02:00.049Z caller=main.go:947 level=info msg="TSDB started" 2023-10-01T09:02:00.307 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:00 smithi190 conmon[129942]: ts=2023-10-01T09:02:00.049Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-01T09:02:00.307 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:00 smithi190 conmon[129942]: ts=2023-10-01T09:02:00.069Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.413994ms db_storage=774ns remote_storage=1.8µs web_handler=749ns query_engine=742ns scrape=577.1µs scrape_sd=25.451µs notify=1.006µs notify_sd=2.011µs rules=19.455839ms 2023-10-01T09:02:00.307 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:00 smithi190 conmon[129942]: ts=2023-10-01T09:02:00.069Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-01T09:02:00.478 INFO:teuthology.orchestra.run.smithi190.stdout:[client.1] 2023-10-01T09:02:00.479 INFO:teuthology.orchestra.run.smithi190.stdout: key = AQCINRllKREXHBAA/Gk8meD8dFqylfbOYwuoIA== 2023-10-01T09:02:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:00 smithi190 conmon[100518]: cluster 2023-10-01T09:01:58.831663+0000 mgr.y (mgr. 2023-10-01T09:02:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:00 smithi190 conmon[100518]: 14433) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:00 smithi190 conmon[100518]: audit 2023-10-01T09:02:00.111776+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:00 smithi190 conmon[100518]: audit 2023-10-01T09:02:00.464623+0000 mon.b (mon.1) 24 : audit [INF] from='client.? 172.21.15.190:0/3587458442' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T09:02:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:00 smithi190 conmon[100518]: audit 2023-10-01T09:02:00.471055+0000 mon.a (mon.0) 579 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T09:02:00.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:00 smithi190 conmon[100518]: audit 2023-10-01T09:02:00.478532+0000 mon.a (mon.0) 580 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T09:02:00.820 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:00 smithi138 systemd[1]: Starting Ceph node-exporter.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:02:00.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[100289]: cluster 2023-10-01T09:01:58.831663+0000 mgr.y (mgr. 2023-10-01T09:02:00.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[100289]: 14433) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:00.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[100289]: audit 2023-10-01T09:02:00.111776+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:00.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[100289]: audit 2023-10-01T09:02:00.464623+0000 mon.b (mon.1) 24 : audit [INF] from='client.? 172.21.15.190:0/3587458442' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T09:02:00.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[100289]: audit 2023-10-01T09:02:00.471055+0000 mon.a (mon.0) 579 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T09:02:00.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[100289]: audit 2023-10-01T09:02:00.478532+0000 mon.a (mon.0) 580 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T09:02:00.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: cluster 2023-10-01T09:01:58.831663+0000 mgr.y 2023-10-01T09:02:00.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: (mgr.14433) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:00.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: audit 2023-10-01T09:02: 2023-10-01T09:02:00.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: 00.111776+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:00.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: audit 2023 2023-10-01T09:02:00.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: -10-01T09:02:00. 2023-10-01T09:02:00.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: 464623+0000 mon.b (mon.1) 24 : audit 2023-10-01T09:02:00.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: [INF] from='client.? 172.21.15.190:0/3587458442' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T09:02:00.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:02:00.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: :02:00.471055+0000 mon.a (mon. 2023-10-01T09:02:00.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: 0) 579 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T09:02:00.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: audit 2023 2023-10-01T09:02:00.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: -10-01T09:02:00 2023-10-01T09:02:00.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: .478532+0000 mon.a (mon.0) 580 : audit 2023-10-01T09:02:00.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:00 smithi138 conmon[109959]: [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T09:02:01.129 DEBUG:teuthology.orchestra.run.smithi190:> set -ex 2023-10-01T09:02:01.129 DEBUG:teuthology.orchestra.run.smithi190:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-01T09:02:01.129 DEBUG:teuthology.orchestra.run.smithi190:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-01T09:02:01.170 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-10-01T09:02:01.170 DEBUG:teuthology.orchestra.run.smithi138:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-10-01T09:02:01.319 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:00 smithi138 bash[137909]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2023-10-01T09:02:01.497 INFO:teuthology.misc.health.smithi138.stdout: 2023-10-01T09:02:01.497 INFO:teuthology.misc.health.smithi138.stdout:{"epoch":51,"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","created":"2023-10-01T08:56:55.039846+0000","modified":"2023-10-01T09:01:44.817395+0000","last_up_change":"2023-10-01T09:01:30.467448+0000","last_in_change":"2023-10-01T09:01:14.134268+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-01T08:59:41.882085+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"12f71e77-6099-48ee-a89e-e931e000e9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6803","nonce":635244462}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6805","nonce":635244462}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6809","nonce":635244462}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6807","nonce":635244462}]},"public_addr":"172.21.15.138:6803/635244462","cluster_addr":"172.21.15.138:6805/635244462","heartbeat_back_addr":"172.21.15.138:6809/635244462","heartbeat_front_addr":"172.21.15.138:6807/635244462","state":["exists","up"]},{"osd":1,"uuid":"525b5e81-4fa5-4b10-9147-83fcb549ca30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6811","nonce":989382814}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6813","nonce":989382814}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6817","nonce":989382814}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6815","nonce":989382814}]},"public_addr":"172.21.15.138:6811/989382814","cluster_addr":"172.21.15.138:6813/989382814","heartbeat_back_addr":"172.21.15.138:6817/989382814","heartbeat_front_addr":"172.21.15.138:6815/989382814","state":["exists","up"]},{"osd":2,"uuid":"97443e05-2513-4832-bb54-f65493df5967","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6819","nonce":3174817573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6821","nonce":3174817573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6825","nonce":3174817573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6823","nonce":3174817573}]},"public_addr":"172.21.15.138:6819/3174817573","cluster_addr":"172.21.15.138:6821/3174817573","heartbeat_back_addr":"172.21.15.138:6825/3174817573","heartbeat_front_addr":"172.21.15.138:6823/3174817573","state":["exists","up"]},{"osd":3,"uuid":"ac51c165-e007-4390-b95b-26eef984d637","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6827","nonce":1517480090}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6829","nonce":1517480090}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6832","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6833","nonce":1517480090}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6831","nonce":1517480090}]},"public_addr":"172.21.15.138:6827/1517480090","cluster_addr":"172.21.15.138:6829/1517480090","heartbeat_back_addr":"172.21.15.138:6833/1517480090","heartbeat_front_addr":"172.21.15.138:6831/1517480090","state":["exists","up"]},{"osd":4,"uuid":"4bef630c-bff4-41a3-a932-de0501541f78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6801","nonce":743095145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6803","nonce":743095145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6807","nonce":743095145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6805","nonce":743095145}]},"public_addr":"172.21.15.190:6801/743095145","cluster_addr":"172.21.15.190:6803/743095145","heartbeat_back_addr":"172.21.15.190:6807/743095145","heartbeat_front_addr":"172.21.15.190:6805/743095145","state":["exists","up"]},{"osd":5,"uuid":"ac6e16ed-681d-4e9d-87a3-d20d50e01437","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6808","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6809","nonce":4080051847}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6810","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6811","nonce":4080051847}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6814","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6815","nonce":4080051847}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6812","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6813","nonce":4080051847}]},"public_addr":"172.21.15.190:6809/4080051847","cluster_addr":"172.21.15.190:6811/4080051847","heartbeat_back_addr":"172.21.15.190:6815/4080051847","heartbeat_front_addr":"172.21.15.190:6813/4080051847","state":["exists","up"]},{"osd":6,"uuid":"7a6f65a5-16dc-45de-854d-751d9151e85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6816","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6817","nonce":797186189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6818","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6819","nonce":797186189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6822","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6823","nonce":797186189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6820","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6821","nonce":797186189}]},"public_addr":"172.21.15.190:6817/797186189","cluster_addr":"172.21.15.190:6819/797186189","heartbeat_back_addr":"172.21.15.190:6823/797186189","heartbeat_front_addr":"172.21.15.190:6821/797186189","state":["exists","up"]},{"osd":7,"uuid":"09a5ad5d-d58f-4fcf-8b09-10f395bd01f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6824","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6825","nonce":185238034}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6826","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6827","nonce":185238034}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6830","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6831","nonce":185238034}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6828","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6829","nonce":185238034}]},"public_addr":"172.21.15.190:6825/185238034","cluster_addr":"172.21.15.190:6827/185238034","heartbeat_back_addr":"172.21.15.190:6831/185238034","heartbeat_front_addr":"172.21.15.190:6829/185238034","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:58:54.313619+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:16.837452+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:39.610865+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:03.127916+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:23.838460+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:45.582295+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:07.078976+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:29.080850+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.138:0/2334114390":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6801/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6800/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:0/3940341494":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2575047401":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6801/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.138:6800/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2054656381":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2018177063":"2023-10-02T08:57:18.125987+0000","172.21.15.138:0/3675584653":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6800/3165627846":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6801/3165627846":"2023-10-02T08:57:18.125987+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T09:02:01.509 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-10-01T09:02:01.510 INFO:tasks.cephadm:Creating RBD pool 2023-10-01T09:02:01.510 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-10-01T09:02:01.750 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:01 smithi138 conmon[109959]: cephadm 2023-10-01T09:02:00.114314+0000 mgr.y (mgr.14433) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi138 2023-10-01T09:02:01.751 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:01 smithi138 conmon[109959]: audit 2023-10-01T09:02:01.496696+0000 mon.a (mon.0) 581 : audit [DBG] from='client.? 172.21.15.138:0/3076385942' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:02:01.751 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:01 smithi138 bash[137909]: Getting image source signatures 2023-10-01T09:02:01.751 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:01 smithi138 bash[137909]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2023-10-01T09:02:01.751 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:01 smithi138 bash[137909]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2023-10-01T09:02:01.751 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:01 smithi138 conmon[100289]: cephadm 2023-10-01T09:02:00.114314+0000 mgr.y (mgr.14433) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi138 2023-10-01T09:02:01.751 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:01 smithi138 conmon[100289]: audit 2023-10-01T09:02:01.496696+0000 mon.a (mon.0) 581 : audit [DBG] from='client.? 172.21.15.138:0/3076385942' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:02:01.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:01 smithi190 conmon[100518]: cephadm 2023-10-01T09:02:00.114314+0000 mgr.y (mgr.14433 2023-10-01T09:02:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:01 smithi190 conmon[100518]: ) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi138 2023-10-01T09:02:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:01 smithi190 conmon[100518]: audit 2023-10-01T09:02:01.496696+0000 mon.a (mon.0) 581 : audit [DBG] from='client.? 172.21.15.138:0/3076385942' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:02:02.010 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:01 smithi138 bash[137909]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2023-10-01T09:02:02.319 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:02 smithi138 bash[137909]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2023-10-01T09:02:02.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:02 smithi190 conmon[100518]: cluster 2023-10-01T09:02:00.831999+0000 mgr.y (mgr. 2023-10-01T09:02:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:02 smithi190 conmon[100518]: 14433) 21 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:02.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:02 smithi138 conmon[109959]: cluster 2023-10-01T09:02:00.831999+0000 mgr.y ( 2023-10-01T09:02:02.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:02 smithi138 conmon[109959]: mgr.14433) 21 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:02.820 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:02 smithi138 bash[137909]: Writing manifest to image destination 2023-10-01T09:02:02.820 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:02 smithi138 bash[137909]: Storing signatures 2023-10-01T09:02:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:02 smithi138 conmon[100289]: cluster 2023-10-01T09:02:00.831999+0000 mgr.y (mgr.14433) 2023-10-01T09:02:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:02 smithi138 conmon[100289]: 21 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:04.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:04 smithi190 conmon[100518]: cluster 2023-10-01T09:02:02.832360+0000 mgr.y (mgr.14433) 22 : 2023-10-01T09:02:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:04 smithi190 conmon[100518]: cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:04.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[109959]: cluster 2023-10-01T09:02:02.832360+0000 mgr.y (mgr. 2023-10-01T09:02:04.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[109959]: 14433) 22 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:04.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[100289]: cluster 2023-10-01T09:02:02.832360+0000 mgr.y (mgr. 2023-10-01T09:02:04.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[100289]: 14433) 22 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:04.820 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.653Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-01T09:02:04.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.653Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-01T09:02:04.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.653Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-01T09:02:04.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.653Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-01T09:02:04.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-01T09:02:04.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=arp 2023-10-01T09:02:04.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-01T09:02:04.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-01T09:02:04.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-01T09:02:04.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-01T09:02:04.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-01T09:02:04.822 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-01T09:02:04.822 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-01T09:02:04.822 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-01T09:02:04.822 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=edac 2023-10-01T09:02:04.822 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-01T09:02:04.822 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-01T09:02:04.822 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-01T09:02:04.822 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-01T09:02:04.822 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-01T09:02:04.822 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-01T09:02:04.822 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-01T09:02:04.822 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-01T09:02:04.823 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-01T09:02:04.823 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-01T09:02:04.823 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-01T09:02:04.823 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-01T09:02:04.823 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-01T09:02:04.823 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-01T09:02:04.823 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-01T09:02:04.823 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-01T09:02:04.823 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=os 2023-10-01T09:02:04.823 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-01T09:02:04.823 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-01T09:02:04.823 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-01T09:02:04.824 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-01T09:02:04.824 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-01T09:02:04.824 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-01T09:02:04.824 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=stat 2023-10-01T09:02:04.824 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-01T09:02:04.824 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-01T09:02:04.824 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-01T09:02:04.824 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=time 2023-10-01T09:02:04.824 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-01T09:02:04.824 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=uname 2023-10-01T09:02:04.824 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-01T09:02:04.825 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-01T09:02:04.825 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-01T09:02:04.825 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-01T09:02:04.825 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 conmon[138253]: ts=2023-10-01T09:02:04.654Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-01T09:02:04.825 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 bash[137909]: 6ec380bfea78636fe9290a596ad4c5cf358802ef7d56e7cf113f480928e4ad3d 2023-10-01T09:02:04.825 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:02:04 smithi138 systemd[1]: Started Ceph node-exporter.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:02:05.557 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:05 smithi190 systemd[1]: Starting Ceph node-exporter.b for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:02:05.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:02:05] "GET /metrics HTTP/1.1" 200 153265 "" "Prometheus/2.33.4" 2023-10-01T09:02:05.992 INFO:teuthology.orchestra.run.smithi138.stderr:pool 'rbd' created 2023-10-01T09:02:06.018 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[109959]: audit 2023-10-01T09:02:04.733173+0000 mon.a (mon.0) 582 2023-10-01T09:02:06.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[109959]: : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:06.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[109959]: cephadm 2023-10-01T09:02:04.736371+0000 mgr.y (mgr.14433) 23 : cephadm [INF] Deploying daemon node-exporter.b on smithi190 2023-10-01T09:02:06.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[109959]: cluster 2023-10-01T09:02:04.832664+0000 mgr.y (mgr.14433) 24 2023-10-01T09:02:06.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[109959]: : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:06.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[109959]: audit 2023 2023-10-01T09:02:06.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[109959]: -10-01T09:02:04.986529+0000 mon.a (mon.0) 583 : audit [INF] 2023-10-01T09:02:06.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[109959]: from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:06.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:02:06.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[109959]: T09:02:05.035020+0000 mon.c (mon.2) 44 : audit [INF] from='client.? 172.21.15.138:0/3626225825' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-01T09:02:06.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[109959]: audit 2023-10-01T09:02:05. 2023-10-01T09:02:06.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[109959]: 035470+0000 mon.a (mon.0) 584 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-01T09:02:06.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100289]: audit 2023-10-01T09:02:04.733173+0000 2023-10-01T09:02:06.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100289]: mon.a (mon.0) 582 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:06.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100289]: cephadm 2023-10-01T09:02:04.736371+0000 2023-10-01T09:02:06.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100289]: mgr.y (mgr.14433) 23 : cephadm [INF] Deploying daemon node-exporter.b on smithi190 2023-10-01T09:02:06.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100289]: cluster 2023-10-01T 2023-10-01T09:02:06.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100289]: 09:02:04.832664+0000 mgr.y (mgr.14433) 24 : cluster [DBG] 2023-10-01T09:02:06.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100289]: pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:06.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100289]: audit 2023-10-01T09:02:04.986529+0000 2023-10-01T09:02:06.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100289]: mon.a (mon.0) 583 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:06.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T09:02:06.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100289]: T09:02:05.035020+0000 mon.c (mon.2) 44 2023-10-01T09:02:06.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100289]: : audit [INF] from='client.? 172.21.15.138:0/3626225825' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-01T09:02:06.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100289]: audit 2023-10-01T09:02:05.035470+0000 2023-10-01T09:02:06.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:05 smithi138 conmon[100289]: mon.a (mon.0) 584 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-01T09:02:06.056 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:05 smithi190 bash[130430]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2023-10-01T09:02:06.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:05 smithi190 conmon[100518]: audit 2023-10-01T09:02:04.733173+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:06.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:05 smithi190 conmon[100518]: cephadm 2023-10 2023-10-01T09:02:06.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:05 smithi190 conmon[100518]: -01T09:02:04.736371+0000 mgr.y (mgr.14433) 23 : cephadm [INF] Deploying daemon node-exporter.b on smithi190 2023-10-01T09:02:06.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:05 smithi190 conmon[100518]: cluster 2023-10-01T09:02:04.832664+0000 mgr.y (mgr.14433) 24 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:06.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:05 smithi190 conmon[100518]: audit 2023-10-01T09:02:04.986529+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:06.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:05 smithi190 conmon[100518]: audit 2023-10-01T09:02:05.035020+0000 mon.c (mon.2) 44 : audit [INF] from='client.? 172.21.15.138:0/3626225825' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-01T09:02:06.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:05 smithi190 conmon[100518]: audit 2023-10-01T09:02:05.035470+0000 mon.a (mon.0) 584 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-01T09:02:06.315 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:06 smithi190 bash[130430]: Getting image source signatures 2023-10-01T09:02:06.315 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:06 smithi190 bash[130430]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2023-10-01T09:02:06.315 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:06 smithi190 bash[130430]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2023-10-01T09:02:06.565 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:06 smithi190 bash[130430]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2023-10-01T09:02:06.842 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:06 smithi190 bash[130430]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2023-10-01T09:02:06.843 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:06 smithi190 bash[130430]: Writing manifest to image destination 2023-10-01T09:02:06.843 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:06 smithi190 bash[130430]: Storing signatures 2023-10-01T09:02:06.869 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-10-01T09:02:07.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:02:06 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:02:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:02:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:06 smithi190 conmon[100518]: audit 2023-10-01T09:02:05.988350 2023-10-01T09:02:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:06 smithi190 conmon[100518]: +0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-01T09:02:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:06 smithi190 conmon[100518]: cluster 2023-10-01T09:02:05.988402+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-01T09:02:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:06 smithi190 conmon[100518]: audit 2023-10-01T09:02:05.988956+0000 mon.c (mon.2) 45 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T09:02:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:06 smithi190 conmon[100518]: audit 2023-10-01T09:02 2023-10-01T09:02:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:06 smithi190 conmon[100518]: :05.989596+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T09:02:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:06 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:02:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:06 smithi190 conmon[100518]: 01T09:02:05.990609+0000 mon.c (mon.2) 46 2023-10-01T09:02:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:06 smithi190 conmon[100518]: : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T09:02:07.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:06 smithi190 conmon[100518]: audit 2023-10-01T09:02: 2023-10-01T09:02:07.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:06 smithi190 conmon[100518]: 05.990983+0000 mon.a (mon.0) 588 : audit 2023-10-01T09:02:07.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:06 smithi190 conmon[100518]: [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T09:02:07.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:06 smithi138 conmon[100289]: audit 2023-10-01T09:02:05.988350+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-01T09:02:07.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:06 smithi138 conmon[100289]: cluster 2023-10-01T09:02:05.988402+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-01T09:02:07.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:06 smithi138 conmon[100289]: audit 2023-10-01T09:02:05.988956+0000 mon.c (mon.2) 45 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T09:02:07.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:06 smithi138 conmon[100289]: audit 2023-10-01T09:02:05.989596+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T09:02:07.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:06 smithi138 conmon[100289]: audit 2023-10-01T09:02:05.990609+0000 mon.c (mon.2) 46 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T09:02:07.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:06 smithi138 conmon[100289]: audit 2023-10-01T09:02:05.990983+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T09:02:07.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:06 smithi138 conmon[109959]: audit 2023-10-01T09:02:05.988350+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-01T09:02:07.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:06 smithi138 conmon[109959]: cluster 2023-10-01T09:02:05.988402+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-01T09:02:07.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:06 smithi138 conmon[109959]: audit 2023-10-01T09:02:05.988956+0000 mon.c (mon.2) 45 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T09:02:07.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:06 smithi138 conmon[109959]: audit 2023-10-01T09:02:05.989596+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T09:02:07.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:06 smithi138 conmon[109959]: audit 2023-10-01T09:02:05.990609+0000 mon.c (mon.2) 46 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T09:02:07.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:06 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:02:07.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:06 smithi138 conmon[109959]: 02:05.990983+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T09:02:07.994 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:07 smithi138 conmon[100289]: cluster 2023-10-01T09:02:06.833027+0000 mgr.y (mgr.14433 2023-10-01T09:02:07.994 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:07 smithi138 conmon[100289]: ) 25 : cluster [DBG] pgmap v15: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:07.995 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:07 smithi138 conmon[100289]: audit 2023-10-01T09:02:06.994954+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-01T09:02:07.995 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:07 smithi138 conmon[100289]: cluster 2023-10-01T09:02:06.995129+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-01T09:02:08.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[100518]: cluster 2023-10-01T09:02:06.833027+0000 mgr.y (mgr.14433) 25 : cluster [DBG] pgmap v15: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[100518]: audit 2023-10-01T09:02:06.994954+0000 mon.a (mon. 2023-10-01T09:02:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[100518]: 0) 589 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-01T09:02:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[100518]: cluster 2023-10-01T09:02:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[100518]: 2023-10-01 2023-10-01T09:02:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[100518]: T09:02:06.995129+0000 mon.a (mon.0 2023-10-01T09:02:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[100518]: ) 590 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-01T09:02:08.307 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.954Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-01T09:02:08.308 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.954Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-01T09:02:08.308 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-01T09:02:08.308 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-01T09:02:08.308 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-01T09:02:08.308 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=arp 2023-10-01T09:02:08.308 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-01T09:02:08.308 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-01T09:02:08.309 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-01T09:02:08.309 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-01T09:02:08.309 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-01T09:02:08.309 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-01T09:02:08.309 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-01T09:02:08.310 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-01T09:02:08.310 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=edac 2023-10-01T09:02:08.310 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-01T09:02:08.310 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-01T09:02:08.310 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-01T09:02:08.310 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-01T09:02:08.310 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-01T09:02:08.310 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-01T09:02:08.310 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-01T09:02:08.310 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-01T09:02:08.310 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-01T09:02:08.311 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-01T09:02:08.311 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-01T09:02:08.311 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-01T09:02:08.311 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-01T09:02:08.311 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-01T09:02:08.311 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-01T09:02:08.311 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-01T09:02:08.311 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=os 2023-10-01T09:02:08.311 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-01T09:02:08.311 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-01T09:02:08.311 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-01T09:02:08.311 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-01T09:02:08.312 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-01T09:02:08.312 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-01T09:02:08.312 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=stat 2023-10-01T09:02:08.312 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-01T09:02:08.312 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-01T09:02:08.312 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-01T09:02:08.312 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=time 2023-10-01T09:02:08.312 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-01T09:02:08.312 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=uname 2023-10-01T09:02:08.312 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-01T09:02:08.312 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-01T09:02:08.312 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-01T09:02:08.313 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.955Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-01T09:02:08.313 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 conmon[130506]: ts=2023-10-01T09:02:07.956Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-01T09:02:08.313 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 bash[130430]: 9a8d149372ce669cf3f80ab88d2598eb2bfbb8e74e1d912dde8132f70d4c8c66 2023-10-01T09:02:08.313 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:02:07 smithi190 systemd[1]: Started Ceph node-exporter.b for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:02:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:07 smithi138 conmon[109959]: cluster 2023-10-01T09:02:06 2023-10-01T09:02:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:07 smithi138 conmon[109959]: .833027+0000 mgr.y (mgr.14433) 25 : cluster [DBG] pgmap v15: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:07 smithi138 conmon[109959]: audit 2023-10-01T09:02: 2023-10-01T09:02:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:07 smithi138 conmon[109959]: 06.994954+0000 mon.a (mon.0) 589 : audit [INF] 2023-10-01T09:02:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:07 smithi138 conmon[109959]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-01T09:02:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:07 smithi138 conmon[109959]: cluster 2023-10-01T09:02:06 2023-10-01T09:02:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:07 smithi138 conmon[109959]: .995129+0000 mon.a (mon.0) 590 : cluster 2023-10-01T09:02:08.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:07 smithi138 conmon[109959]: [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-01T09:02:09.062 INFO:teuthology.orchestra.run.smithi138.stderr:enabled application 'rbd' on pool 'rbd' 2023-10-01T09:02:09.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:09 smithi190 conmon[100518]: cluster 2023-10-01T09:02:08.002133+0000 mon.a (mon.0) 591 2023-10-01T09:02:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:09 smithi190 conmon[100518]: : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-01T09:02:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:09 smithi190 conmon[100518]: audit 2023-10-01T09:02:08.055706+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:09 smithi190 conmon[100518]: audit 2023-10-01T09:02:08.057121+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:02:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:09 smithi190 conmon[100518]: audit 2023-10-01T09:02:08.855305+0000 mon.b (mon.1) 25 : audit [INF] from='client.? 172.21.15.138:0/17867536' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:02:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:09 smithi190 conmon[100518]: audit 2023-10-01T09:02:08.855698+0000 mon.a (mon.0) 593 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:02:09.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:09 smithi138 conmon[109959]: cluster 2023-10-01T09:02:08.002133+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-01T09:02:09.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:09 smithi138 conmon[109959]: audit 2023-10-01T09:02:08.055706+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:09.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:09 smithi138 conmon[109959]: audit 2023-10-01T09:02:08.057121+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:02:09.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:09 smithi138 conmon[109959]: audit 2023-10-01T09:02:08.855305+0000 mon.b (mon.1) 25 : audit [INF] from='client.? 172.21.15.138:0/17867536' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:02:09.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:09 smithi138 conmon[109959]: audit 2023-10-01T09:02:08.855698+0000 mon.a (mon.0) 593 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:02:09.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:09 smithi138 conmon[100289]: cluster 2023-10-01T09:02:08.002133+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-01T09:02:09.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:09 smithi138 conmon[100289]: audit 2023-10-01T09:02:08.055706+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:09.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:09 smithi138 conmon[100289]: audit 2023-10-01T09:02:08.057121+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:02:09.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:09 smithi138 conmon[100289]: audit 2023-10-01T09:02:08.855305+0000 mon.b (mon.1) 25 : audit [INF] from='client.? 172.21.15.138:0/17867536' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:02:09.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:09 smithi138 conmon[100289]: audit 2023-10-01T09:02:08.855698+0000 mon.a (mon.0) 593 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:02:09.863 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-01T09:02:09.863 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-01T09:02:09.863 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-01T09:02:10.300 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:10.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[100289]: cluster 2023-10-01T09:02:08.833366+0000 mgr.y ( 2023-10-01T09:02:10.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[100289]: mgr.14433) 26 : cluster [DBG] pgmap v18: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:10.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[100289]: audit 2023-10-01T09:02:09.058749+0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:02:10.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[100289]: cluster 2023-10-01T09:02:09.058821+0000 mon.a (mon.0) 595 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-01T09:02:10.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[100289]: audit 2023-10-01T09:02:09.993009+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:10.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[109959]: cluster 2023-10-01T09:02:08. 2023-10-01T09:02:10.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[109959]: 833366+0000 mgr.y (mgr.14433) 26 : cluster [DBG] pgmap v18: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:10.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:02:10.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[109959]: :02:09.058749+0000 mon.a (mon.0 2023-10-01T09:02:10.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[109959]: ) 594 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:02:10.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[109959]: cluster 2023 2023-10-01T09:02:10.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[109959]: -10-01T09:02:09. 2023-10-01T09:02:10.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[109959]: 058821+0000 mon.a (mon.0) 2023-10-01T09:02:10.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[109959]: 595 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-01T09:02:10.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[109959]: audit 2023 2023-10-01T09:02:10.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[109959]: -10-01T09:02:09. 2023-10-01T09:02:10.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[109959]: 993009+0000 mon.a (mon.0 2023-10-01T09:02:10.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:10 smithi138 conmon[109959]: ) 596 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:10.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:10 smithi190 conmon[100518]: cluster 2023-10-01T09:02:08.833366+0000 mgr.y (mgr.14433) 26 : cluster [DBG] 2023-10-01T09:02:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:10 smithi190 conmon[100518]: pgmap v18: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:10 smithi190 conmon[100518]: audit 2023-10-01T09:02:09.058749+0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:02:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:10 smithi190 conmon[100518]: cluster 2023-10-01T09:02:09.058821+0000 mon.a (mon.0) 595 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-01T09:02:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:10 smithi190 conmon[100518]: audit 2023-10-01T09:02:09.993009+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:11 smithi138 conmon[100289]: audit 2023-10-01T09:02:10. 2023-10-01T09:02:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:11 smithi138 conmon[100289]: 089581+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:11 smithi138 conmon[100289]: audit 2023-10-01T09:02:10.802511+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:11 smithi138 conmon[100289]: audit 2023-10-01T09:02:10.805592+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:02:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:11 smithi138 conmon[100289]: audit 2023-10-01T09:02:10.806268+0000 mon.c (mon.2) 49 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:02:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:11 smithi138 conmon[100289]: audit 2023-10-01T09:02:10.811332+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:11.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:11 smithi138 conmon[109959]: audit 2023-10-01T09:02:10.089581+0000 mon.a (mon.0) 2023-10-01T09:02:11.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:11 smithi138 conmon[109959]: 597 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:11.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:11 smithi138 conmon[109959]: audit 2023-10-01T09:02:10.802511+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:11.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:11 smithi138 conmon[109959]: audit 2023-10-01T09:02:10.805592+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:02:11.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:11 smithi138 conmon[109959]: audit 2023-10-01T09:02:10.806268+0000 mon.c (mon.2) 49 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:02:11.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:11 smithi138 conmon[109959]: audit 2023-10-01T09:02:10.811332+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:11 smithi190 conmon[100518]: audit 2023-10-01T09:02:10.089581+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:11 smithi190 conmon[100518]: 2023-10-01T09:02:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:11 smithi190 conmon[100518]: audit 2023-10-01T09:02:10.802511+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:11 smithi190 conmon[100518]: audit 2023-10-01T09:02:10.805592+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:02:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:11 smithi190 conmon[100518]: audit 2023-10-01T09:02:10.806268+0000 mon.c (mon.2) 49 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:02:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:11 smithi190 conmon[100518]: audit 2023-10-01T09:02:10 2023-10-01T09:02:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:11 smithi190 conmon[100518]: .811332+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:12.212 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:02:12.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:12 smithi138 conmon[100289]: cephadm 2023-10-01T09:02:10.818953+0000 mgr.y (mgr.14433) 27 : cephadm [INF] Deploying daemon alertmanager.a on smithi138 2023-10-01T09:02:12.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:12 smithi138 conmon[100289]: cluster 2023-10-01T09:02:10.833723+0000 mgr.y (mgr.14433) 28 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:12.517 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:12 smithi138 conmon[109959]: cephadm 2023-10-01T09:02:10.818953+0000 mgr.y (mgr.14433) 27 : cephadm [INF] Deploying daemon alertmanager.a on smithi138 2023-10-01T09:02:12.517 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:12 smithi138 conmon[109959]: cluster 2023-10-01T09:02:10.833723+0000 mgr.y (mgr.14433) 28 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:12.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:12 smithi190 conmon[100518]: cephadm 2023-10-01T09:02:10.818953+0000 mgr.y (mgr.14433) 27 : cephadm 2023-10-01T09:02:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:12 smithi190 conmon[100518]: [INF] Deploying daemon alertmanager.a on smithi138 2023-10-01T09:02:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:12 smithi190 conmon[100518]: cluster 2023-10-01T09:02:10.833723+0000 mgr.y (mgr.14433) 28 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:13.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:13 smithi190 conmon[100518]: audit 2023-10-01T09:02:12.206984+0000 mon.a (mon.0) 600 : audit [DBG] from='client.? 172.21.15.138:0/273870759' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T09:02:13.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:13 smithi138 conmon[109959]: audit 2023-10-01T09:02:12.206984+0000 mon.a (mon.0) 600 : audit [DBG] 2023-10-01T09:02:13.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:13 smithi138 conmon[109959]: from='client.? 172.21.15.138:0/273870759' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T09:02:13.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:13 smithi138 conmon[100289]: audit 2023-10-01T09:02:12.206984+0000 mon.a (mon 2023-10-01T09:02:13.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:13 smithi138 conmon[100289]: .0) 600 : audit [DBG] from='client.? 172.21.15.138:0/273870759' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T09:02:13.759 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":18,"active_gid":14433,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":974662983},{"type":"v1","addr":"172.21.15.138:6801","nonce":974662983}]},"active_addr":"172.21.15.138:6801/974662983","active_change":"2023-10-01T09:01:44.817544+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24322,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.138:0","nonce":1836785572}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.138:0","nonce":3533950111}]}]} 2023-10-01T09:02:13.763 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-01T09:02:13.763 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-01T09:02:13.763 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T09:02:14.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:14 smithi190 conmon[100518]: cluster 2023-10-01T09:02:12.834193+0000 mgr.y (mgr.14433) 29 : cluster [DBG] 2023-10-01T09:02:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:14 smithi190 conmon[100518]: pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:14.559 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:14.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:14 smithi138 conmon[109959]: cluster 2023-10-01T09:02:12.834193+0000 mgr.y (mgr.14433 2023-10-01T09:02:14.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:14 smithi138 conmon[109959]: ) 29 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:14 smithi138 conmon[100289]: cluster 2023-10-01T09:02:12.834193+0000 mgr.y (mgr. 2023-10-01T09:02:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:14 smithi138 conmon[100289]: 14433) 29 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:15.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:02:15 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:02:15] "GET /metrics HTTP/1.1" 200 155653 "" "Prometheus/2.33.4" 2023-10-01T09:02:16.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:16 smithi190 conmon[100518]: cluster 2023-10-01T09:02:14.834440+0000 mgr.y (mgr.14433) 30 : cluster [DBG] 2023-10-01T09:02:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:16 smithi190 conmon[100518]: pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:16.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:16 smithi138 conmon[100289]: cluster 2023-10-01T09:02:14.834440+0000 mgr.y (mgr.14433) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:16.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:16 smithi138 conmon[109959]: cluster 2023-10-01T09:02:14.834440+0000 mgr.y (mgr.14433) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:17.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:02:16 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:02:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:02:17.970 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:02:17.970 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":55,"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","created":"2023-10-01T08:56:55.039846+0000","modified":"2023-10-01T09:02:09.053413+0000","last_up_change":"2023-10-01T09:01:30.467448+0000","last_in_change":"2023-10-01T09:01:14.134268+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-01T08:59:41.882085+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T09:02:05.272330+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"12f71e77-6099-48ee-a89e-e931e000e9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6803","nonce":635244462}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6805","nonce":635244462}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6809","nonce":635244462}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6807","nonce":635244462}]},"public_addr":"172.21.15.138:6803/635244462","cluster_addr":"172.21.15.138:6805/635244462","heartbeat_back_addr":"172.21.15.138:6809/635244462","heartbeat_front_addr":"172.21.15.138:6807/635244462","state":["exists","up"]},{"osd":1,"uuid":"525b5e81-4fa5-4b10-9147-83fcb549ca30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6811","nonce":989382814}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6813","nonce":989382814}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6817","nonce":989382814}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6815","nonce":989382814}]},"public_addr":"172.21.15.138:6811/989382814","cluster_addr":"172.21.15.138:6813/989382814","heartbeat_back_addr":"172.21.15.138:6817/989382814","heartbeat_front_addr":"172.21.15.138:6815/989382814","state":["exists","up"]},{"osd":2,"uuid":"97443e05-2513-4832-bb54-f65493df5967","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6819","nonce":3174817573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6821","nonce":3174817573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6825","nonce":3174817573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6823","nonce":3174817573}]},"public_addr":"172.21.15.138:6819/3174817573","cluster_addr":"172.21.15.138:6821/3174817573","heartbeat_back_addr":"172.21.15.138:6825/3174817573","heartbeat_front_addr":"172.21.15.138:6823/3174817573","state":["exists","up"]},{"osd":3,"uuid":"ac51c165-e007-4390-b95b-26eef984d637","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6827","nonce":1517480090}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6829","nonce":1517480090}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6832","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6833","nonce":1517480090}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6831","nonce":1517480090}]},"public_addr":"172.21.15.138:6827/1517480090","cluster_addr":"172.21.15.138:6829/1517480090","heartbeat_back_addr":"172.21.15.138:6833/1517480090","heartbeat_front_addr":"172.21.15.138:6831/1517480090","state":["exists","up"]},{"osd":4,"uuid":"4bef630c-bff4-41a3-a932-de0501541f78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6801","nonce":743095145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6803","nonce":743095145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6807","nonce":743095145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6805","nonce":743095145}]},"public_addr":"172.21.15.190:6801/743095145","cluster_addr":"172.21.15.190:6803/743095145","heartbeat_back_addr":"172.21.15.190:6807/743095145","heartbeat_front_addr":"172.21.15.190:6805/743095145","state":["exists","up"]},{"osd":5,"uuid":"ac6e16ed-681d-4e9d-87a3-d20d50e01437","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6808","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6809","nonce":4080051847}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6810","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6811","nonce":4080051847}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6814","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6815","nonce":4080051847}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6812","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6813","nonce":4080051847}]},"public_addr":"172.21.15.190:6809/4080051847","cluster_addr":"172.21.15.190:6811/4080051847","heartbeat_back_addr":"172.21.15.190:6815/4080051847","heartbeat_front_addr":"172.21.15.190:6813/4080051847","state":["exists","up"]},{"osd":6,"uuid":"7a6f65a5-16dc-45de-854d-751d9151e85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6816","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6817","nonce":797186189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6818","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6819","nonce":797186189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6822","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6823","nonce":797186189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6820","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6821","nonce":797186189}]},"public_addr":"172.21.15.190:6817/797186189","cluster_addr":"172.21.15.190:6819/797186189","heartbeat_back_addr":"172.21.15.190:6823/797186189","heartbeat_front_addr":"172.21.15.190:6821/797186189","state":["exists","up"]},{"osd":7,"uuid":"09a5ad5d-d58f-4fcf-8b09-10f395bd01f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6824","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6825","nonce":185238034}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6826","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6827","nonce":185238034}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6830","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6831","nonce":185238034}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6828","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6829","nonce":185238034}]},"public_addr":"172.21.15.190:6825/185238034","cluster_addr":"172.21.15.190:6827/185238034","heartbeat_back_addr":"172.21.15.190:6831/185238034","heartbeat_front_addr":"172.21.15.190:6829/185238034","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:58:54.313619+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:16.837452+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:39.610865+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:03.127916+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:23.838460+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:45.582295+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:07.078976+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:29.080850+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.138:0/2334114390":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6801/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6800/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:0/3940341494":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2575047401":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6801/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.138:6800/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2054656381":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2018177063":"2023-10-02T08:57:18.125987+0000","172.21.15.138:0/3675584653":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6800/3165627846":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6801/3165627846":"2023-10-02T08:57:18.125987+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T09:02:18.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:18 smithi138 conmon[100289]: cluster 2023-10-01T09:02:16.834772+0000 mgr.y 2023-10-01T09:02:18.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:18 smithi138 conmon[100289]: (mgr.14433) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:18.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:18 smithi138 conmon[100289]: audit 2023-10-01T09:02:17.969232+0000 mon.a (mon.0) 601 : audit [DBG] from='client.? 172.21.15.138:0/3339697626' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:02:18.793 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:18 smithi138 conmon[109959]: cluster 2023-10-01T09:02:16.834772+0000 mgr.y (mgr 2023-10-01T09:02:18.793 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:18 smithi138 conmon[109959]: .14433) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:18.794 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:18 smithi138 conmon[109959]: audit 2023-10-01T09:02:17.969232+0000 mon.a (mon.0) 601 : audit [DBG] from='client.? 172.21.15.138:0/3339697626' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:02:18.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:18 smithi190 conmon[100518]: cluster 2023-10-01T09:02:16.834772+0000 mgr.y (mgr.14433) 31 : cluster [DBG] 2023-10-01T09:02:18.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:18 smithi190 conmon[100518]: pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:18.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:18 smithi190 conmon[100518]: audit 2023-10-01T09:02:17.969232+0000 mon.a (mon.0) 601 : audit [DBG] from='client.? 172.21.15.138:0/3339697626' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:02:18.991 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-01T09:02:18.992 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T09:02:19.320 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:19 smithi138 systemd[1]: Starting Ceph alertmanager.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:02:19.733 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:19.968 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:19 smithi138 conmon[140971]: level=info ts=2023-10-01T09:02:19.712Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-01T09:02:19.968 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:19 smithi138 conmon[140971]: level=info ts=2023-10-01T09:02:19.712Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-01T09:02:19.968 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:19 smithi138 conmon[140971]: level=info ts=2023-10-01T09:02:19.713Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.138 port=9094 2023-10-01T09:02:19.968 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:19 smithi138 conmon[140971]: level=info ts=2023-10-01T09:02:19.715Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-01T09:02:19.968 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:19 smithi138 bash[140851]: 20e5c74607299944e0bb16c4822523b3e5f22240437f0533b871685d2da5d761 2023-10-01T09:02:19.969 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:19 smithi138 systemd[1]: Started Ceph alertmanager.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:02:19.969 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:19 smithi138 conmon[140971]: level=info ts=2023-10-01T09:02:19.743Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-01T09:02:19.969 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:19 smithi138 conmon[140971]: level=info ts=2023-10-01T09:02:19.743Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-01T09:02:19.969 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:19 smithi138 conmon[140971]: level=info ts=2023-10-01T09:02:19.746Z caller=main.go:518 msg=Listening address=:9093 2023-10-01T09:02:19.969 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:19 smithi138 conmon[140971]: level=info ts=2023-10-01T09:02:19.746Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-01T09:02:19.970 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:02:19 smithi138 conmon[100738]: [01/Oct/2023:09:02:19] ENGINE Bus STOPPING 2023-10-01T09:02:19.970 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:02:19 smithi138 conmon[100738]: [01/Oct/2023:09:02:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T09:02:19.970 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:02:19 smithi138 conmon[100738]: [01/Oct/2023:09:02:19] ENGINE Bus STOPPED 2023-10-01T09:02:19.970 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:02:19 smithi138 conmon[100738]: [01/Oct/2023:09:02:19] ENGINE Bus STARTING 2023-10-01T09:02:20.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[100738]: [01/Oct/2023:09:02:20] ENGINE Serving on http://:::9283 2023-10-01T09:02:20.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[100738]: [01/Oct/2023:09:02:20] ENGINE Bus STARTED 2023-10-01T09:02:20.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:20 smithi190 conmon[100518]: cluster 2023-10-01T09:02:18.835092+0000 mgr.y (mgr.14433) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:20 smithi190 conmon[100518]: audit 2023-10-01T09:02:19.795042+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:20 smithi190 conmon[100518]: audit 2023-10-01T09:02:19.889959+0000 mon.a (mon.0) 603 : audit 2023-10-01T09:02:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:20 smithi190 conmon[100518]: [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:20 smithi190 conmon[100518]: audit 2023-10-01T09:02:19.895523+0000 mon.a ( 2023-10-01T09:02:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:20 smithi190 conmon[100518]: mon.0) 604 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:20 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:02:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:20 smithi190 conmon[100518]: -01T09:02:19.898066+0000 mon.c (mon.2) 50 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-01T09:02:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:20 smithi190 conmon[100518]: audit 2023-10-01T09:02:19 2023-10-01T09:02:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:20 smithi190 conmon[100518]: .902221+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:20 smithi190 conmon[100518]: audit 2023-10-01T09:02:20. 2023-10-01T09:02:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:20 smithi190 conmon[100518]: 006378+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:20.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[100289]: cluster 2023-10-01T09:02:18.835092 2023-10-01T09:02:20.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[100289]: +0000 mgr.y (mgr.14433) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:20.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[100289]: audit 2023-10-01T09:02:19.795042+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:20.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[100289]: audit 2023-10-01T09:02:19.889959+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:20.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[100289]: audit 2023-10-01T09:02:19.895523+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:20.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[100289]: audit 2023-10-01T09:02:19.898066+0000 mon.c (mon.2) 50 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-01T09:02:20.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[100289]: audit 2023-10-01T09:02: 2023-10-01T09:02:20.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[100289]: 19.902221+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:20.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[100289]: audit 2023-10-01T09:02:20.006378 2023-10-01T09:02:20.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[100289]: +0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:20.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: cluster 2023-10-01T09:02:18.835092+0000 mgr.y 2023-10-01T09:02:20.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: (mgr.14433) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:20.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: audit 2023-10-01T09:02:19.795042 2023-10-01T09:02:20.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: +0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:20.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:02:20.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: -01T09:02:19.889959+0000 2023-10-01T09:02:20.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: mon.a (mon.0) 603 : audit 2023-10-01T09:02:20.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:20.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: audit 2023-10-01T09:02:20.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: 2023-10-01T09:02 2023-10-01T09:02:20.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: :19.895523+0000 mon.a (mon 2023-10-01T09:02:20.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: .0) 604 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:20.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: 2023-10-01T09:02:20.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:02:20.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: 10-01T09:02:19. 2023-10-01T09:02:20.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: 898066+0000 mon.c (mon.2) 2023-10-01T09:02:20.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: 50 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-01T09:02:20.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:02:20.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: 01T09:02:19.902221+0000 2023-10-01T09:02:20.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: mon.a (mon.0) 605 : audit 2023-10-01T09:02:20.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:20.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: audit 2023-10-01T09:02:20.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: 2023-10-01T09:02 2023-10-01T09:02:20.816 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: :20.006378+0000 mon.a (mon 2023-10-01T09:02:20.816 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: .0) 606 : audit [INF] 2023-10-01T09:02:20.816 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:20 smithi138 conmon[109959]: from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:21.452 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:02:21.453 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":55,"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","created":"2023-10-01T08:56:55.039846+0000","modified":"2023-10-01T09:02:09.053413+0000","last_up_change":"2023-10-01T09:01:30.467448+0000","last_in_change":"2023-10-01T09:01:14.134268+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-01T08:59:41.882085+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T09:02:05.272330+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"12f71e77-6099-48ee-a89e-e931e000e9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6803","nonce":635244462}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6805","nonce":635244462}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6809","nonce":635244462}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6807","nonce":635244462}]},"public_addr":"172.21.15.138:6803/635244462","cluster_addr":"172.21.15.138:6805/635244462","heartbeat_back_addr":"172.21.15.138:6809/635244462","heartbeat_front_addr":"172.21.15.138:6807/635244462","state":["exists","up"]},{"osd":1,"uuid":"525b5e81-4fa5-4b10-9147-83fcb549ca30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6811","nonce":989382814}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6813","nonce":989382814}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6817","nonce":989382814}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6815","nonce":989382814}]},"public_addr":"172.21.15.138:6811/989382814","cluster_addr":"172.21.15.138:6813/989382814","heartbeat_back_addr":"172.21.15.138:6817/989382814","heartbeat_front_addr":"172.21.15.138:6815/989382814","state":["exists","up"]},{"osd":2,"uuid":"97443e05-2513-4832-bb54-f65493df5967","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6819","nonce":3174817573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6821","nonce":3174817573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6825","nonce":3174817573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6823","nonce":3174817573}]},"public_addr":"172.21.15.138:6819/3174817573","cluster_addr":"172.21.15.138:6821/3174817573","heartbeat_back_addr":"172.21.15.138:6825/3174817573","heartbeat_front_addr":"172.21.15.138:6823/3174817573","state":["exists","up"]},{"osd":3,"uuid":"ac51c165-e007-4390-b95b-26eef984d637","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6827","nonce":1517480090}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6829","nonce":1517480090}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6832","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6833","nonce":1517480090}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6831","nonce":1517480090}]},"public_addr":"172.21.15.138:6827/1517480090","cluster_addr":"172.21.15.138:6829/1517480090","heartbeat_back_addr":"172.21.15.138:6833/1517480090","heartbeat_front_addr":"172.21.15.138:6831/1517480090","state":["exists","up"]},{"osd":4,"uuid":"4bef630c-bff4-41a3-a932-de0501541f78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6801","nonce":743095145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6803","nonce":743095145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6807","nonce":743095145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6805","nonce":743095145}]},"public_addr":"172.21.15.190:6801/743095145","cluster_addr":"172.21.15.190:6803/743095145","heartbeat_back_addr":"172.21.15.190:6807/743095145","heartbeat_front_addr":"172.21.15.190:6805/743095145","state":["exists","up"]},{"osd":5,"uuid":"ac6e16ed-681d-4e9d-87a3-d20d50e01437","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6808","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6809","nonce":4080051847}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6810","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6811","nonce":4080051847}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6814","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6815","nonce":4080051847}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6812","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6813","nonce":4080051847}]},"public_addr":"172.21.15.190:6809/4080051847","cluster_addr":"172.21.15.190:6811/4080051847","heartbeat_back_addr":"172.21.15.190:6815/4080051847","heartbeat_front_addr":"172.21.15.190:6813/4080051847","state":["exists","up"]},{"osd":6,"uuid":"7a6f65a5-16dc-45de-854d-751d9151e85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6816","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6817","nonce":797186189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6818","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6819","nonce":797186189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6822","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6823","nonce":797186189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6820","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6821","nonce":797186189}]},"public_addr":"172.21.15.190:6817/797186189","cluster_addr":"172.21.15.190:6819/797186189","heartbeat_back_addr":"172.21.15.190:6823/797186189","heartbeat_front_addr":"172.21.15.190:6821/797186189","state":["exists","up"]},{"osd":7,"uuid":"09a5ad5d-d58f-4fcf-8b09-10f395bd01f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6824","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6825","nonce":185238034}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6826","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6827","nonce":185238034}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6830","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6831","nonce":185238034}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6828","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6829","nonce":185238034}]},"public_addr":"172.21.15.190:6825/185238034","cluster_addr":"172.21.15.190:6827/185238034","heartbeat_back_addr":"172.21.15.190:6831/185238034","heartbeat_front_addr":"172.21.15.190:6829/185238034","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:58:54.313619+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:16.837452+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:39.610865+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:03.127916+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:23.838460+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:45.582295+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:07.078976+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:29.080850+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.138:0/2334114390":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6801/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6800/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:0/3940341494":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2575047401":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6801/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.138:6800/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2054656381":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2018177063":"2023-10-02T08:57:18.125987+0000","172.21.15.138:0/3675584653":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6800/3165627846":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6801/3165627846":"2023-10-02T08:57:18.125987+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T09:02:21.496 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:21 smithi138 conmon[109959]: audit 2023-10-01T09:02:19.898430+0000 mgr.y (mgr 2023-10-01T09:02:21.496 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:21 smithi138 conmon[109959]: .14433) 33 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-01T09:02:21.496 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:21 smithi138 conmon[100289]: audit 2023-10-01T09:02:19.898430+0000 mgr.y (mgr.14433) 33 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-01T09:02:21.496 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:21 smithi138 conmon[100289]: cephadm 2023-10-01T09:02:19.911706+0000 mgr.y (mgr.14433) 34 : cephadm [INF] Deploying daemon grafana.a on smithi190 2023-10-01T09:02:21.496 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:21 smithi138 conmon[100289]: audit 2023-10-01T09:02:21.452016+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.138:0/1353582065' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:02:21.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:21 smithi190 conmon[100518]: audit 2023-10-01T09:02:19.898430+0000 mgr.y (mgr.14433 2023-10-01T09:02:21.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:21 smithi190 conmon[100518]: ) 33 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-01T09:02:21.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:21 smithi190 conmon[100518]: cephadm 2023-10-01T09:02:19.911706+0000 mgr.y (mgr.14433) 34 : cephadm [INF] Deploying daemon grafana.a on smithi190 2023-10-01T09:02:21.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:21 smithi190 conmon[100518]: audit 2023-10-01T09:02:21.452016+0000 mon.c (mon.2) 51 : audit [DBG] 2023-10-01T09:02:21.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:21 smithi190 conmon[100518]: from='client.? 172.21.15.138:0/1353582065' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:02:21.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:21 smithi138 conmon[109959]: cephadm 2023-10-01T09:02:19.911706+0000 mgr.y (mgr.14433) 34 : cephadm [INF] Deploying daemon grafana.a on smithi190 2023-10-01T09:02:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:21 smithi138 conmon[109959]: audit 2023-10-01T09:02:21.452016+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.138:0/1353582065' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:02:21.820 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:21 smithi138 conmon[140971]: level=info ts=2023-10-01T09:02:21.715Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000609053s 2023-10-01T09:02:22.113 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-01T09:02:22.114 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-01T09:02:22.114 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-01T09:02:22.115 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-01T09:02:22.115 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-01T09:02:22.115 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-01T09:02:22.116 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-01T09:02:22.116 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-01T09:02:22.506 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:22 smithi138 conmon[100289]: cluster 2023-10-01T09:02:20.835425+0000 mgr.y (mgr.14433) 35 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:22.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:22 smithi190 conmon[100518]: cluster 2023-10-01T09:02:20.835425+0000 mgr.y (mgr. 2023-10-01T09:02:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:22 smithi190 conmon[100518]: 14433) 35 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:22.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:22 smithi138 conmon[109959]: cluster 2023-10-01T09:02:20 2023-10-01T09:02:22.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:22 smithi138 conmon[109959]: .835425+0000 mgr.y (mgr.14433) 35 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:23.147 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:23.400 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:23.452 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:23.854 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:24.200 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:24.200 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:24.795 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:24 smithi138 conmon[109959]: cluster 2023-10-01T09:02:22.835710+0000 mgr.y (mgr.14433) 36 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:24.795 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:24 smithi138 conmon[100289]: cluster 2023-10-01T09:02:22.835710+0000 mgr.y (mgr.14433) 36 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:24 smithi190 conmon[100518]: cluster 2023-10-01T09:02:22.835710 2023-10-01T09:02:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:24 smithi190 conmon[100518]: +0000 mgr.y (mgr.14433) 36 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:25.257 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:25.258 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:25.440 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:02:25 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:02:25] "GET /metrics HTTP/1.1" 200 155653 "" "Prometheus/2.33.4" 2023-10-01T09:02:26.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:26 smithi138 conmon[109959]: cluster 2023-10-01T09:02:24.836047+0000 mgr.y 2023-10-01T09:02:26.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:26 smithi138 conmon[109959]: (mgr.14433) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:26.790 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:26 smithi138 conmon[100289]: cluster 2023-10-01T09:02:24.836047+0000 mgr.y (mgr.14433) 2023-10-01T09:02:26.790 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:26 smithi138 conmon[100289]: 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:26 smithi190 conmon[100518]: cluster 2023-10-01T09:02:24.836047+0000 mgr.y (mgr. 2023-10-01T09:02:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:26 smithi190 conmon[100518]: 14433) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:27.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:02:26 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:02:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:02:28.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:28 smithi138 conmon[100289]: cluster 2023-10-01T09:02:26.836461+0000 mgr.y (mgr.14433) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:28.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:28 smithi138 conmon[109959]: cluster 2023-10-01T09:02:26.836461+0000 2023-10-01T09:02:28.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:28 smithi138 conmon[109959]: mgr.y (mgr.14433) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:28.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:28 smithi190 conmon[100518]: cluster 2023-10-01T09:02:26.836461+0000 mgr.y (mgr.14433) 2023-10-01T09:02:28.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:28 smithi190 conmon[100518]: 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:29.925 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:29 smithi138 conmon[140971]: level=info ts=2023-10-01T09:02:29.718Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003401752s 2023-10-01T09:02:29.927 INFO:teuthology.orchestra.run.smithi138.stdout:180388626450 2023-10-01T09:02:29.927 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-01T09:02:30.622 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:30 smithi138 conmon[100289]: cluster 2023-10-01T09:02:28.836884+0000 mgr.y (mgr.14433) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:30.622 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:30 smithi138 conmon[109959]: cluster 2023-10-01T09:02: 2023-10-01T09:02:30.622 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:30 smithi138 conmon[109959]: 28.836884+0000 mgr.y (mgr.14433) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:30.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:30 smithi190 conmon[100518]: cluster 2023-10-01T09:02:28.836884+0000 mgr.y (mgr.14433 2023-10-01T09:02:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:30 smithi190 conmon[100518]: ) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:32.093 INFO:teuthology.orchestra.run.smithi138.stdout:77309411363 2023-10-01T09:02:32.093 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-01T09:02:32.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:32 smithi138 conmon[100289]: cluster 2023-10-01T09:02:30.837284+0000 mgr.y (mgr.14433) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:32.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:32 smithi138 conmon[109959]: cluster 2023-10-01T09:02:30.837284+0000 mgr.y (mgr.14433) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:32.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:32 smithi190 conmon[100518]: cluster 2023-10-01T09:02:30.837284+0000 mgr.y (mgr.14433) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:32.973 INFO:teuthology.orchestra.run.smithi138.stdout:34359738413 2023-10-01T09:02:32.973 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-01T09:02:33.714 INFO:teuthology.orchestra.run.smithi138.stdout:154618822679 2023-10-01T09:02:33.714 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-01T09:02:34.393 INFO:teuthology.orchestra.run.smithi138.stdout:206158430222 2023-10-01T09:02:34.393 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-01T09:02:34.646 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:34 smithi138 conmon[109959]: cluster 2023-10-01T09:02:32.837764+0000 mgr.y (mgr.14433) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:34.647 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:34 smithi138 conmon[100289]: cluster 2023-10-01T09:02:32.837764+0000 mgr.y (mgr.14433) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:34.678 INFO:teuthology.orchestra.run.smithi138.stdout:128849018907 2023-10-01T09:02:34.678 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-01T09:02:34.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:34 smithi190 conmon[100518]: cluster 2023-10-01T09:02:32.837764+0000 mgr.y (mgr.14433) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:35.253 INFO:teuthology.orchestra.run.smithi138.stdout:55834574889 2023-10-01T09:02:35.254 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-01T09:02:35.358 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:35.536 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:02:35 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:02:35] "GET /metrics HTTP/1.1" 200 155661 "" "Prometheus/2.33.4" 2023-10-01T09:02:36.083 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:36.089 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:36.090 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:36.090 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:36.125 INFO:teuthology.orchestra.run.smithi138.stdout:107374182432 2023-10-01T09:02:36.125 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-01T09:02:36.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:36 smithi190 conmon[100518]: cluster 2023-10-01T09:02:34.838100+0000 mgr.y (mgr.14433) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:36 smithi138 conmon[109959]: cluster 2023-10-01T09:02:34.838100+0000 mgr.y ( 2023-10-01T09:02:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:36 smithi138 conmon[109959]: mgr.14433) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:36.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:36 smithi138 conmon[100289]: cluster 2023-10-01T09:02:34.838100+0000 mgr.y (mgr.14433) 2023-10-01T09:02:36.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:36 smithi138 conmon[100289]: 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:37.208 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:37.208 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:02:36 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:02:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:02:37.609 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:38.307 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 systemd[1]: Starting Ceph grafana.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:02:38.526 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:38 smithi138 conmon[109959]: cluster 2023-10-01T09:02:36.838462+0000 mgr.y (mgr.14433) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:38.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:38 smithi138 conmon[100289]: cluster 2023-10-01T09:02:36.838462+0000 mgr.y (mgr.14433) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:38.789 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[100518]: cluster 2023-10-01T09:02:36.838462+0000 mgr.y (mgr.14433) 43 : cluster 2023-10-01T09:02:38.789 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[100518]: [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:38.790 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 bash[131926]: d186140d89cc59f5a74112ffec828e93ce55ab554620e70c85fff20eee7c6fd0 2023-10-01T09:02:38.790 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 systemd[1]: Started Ceph grafana.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:02:38.790 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-01T09:02:38.790 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-01T09:02:38.790 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-01T09:02:38.790 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-01T09:02:38.790 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-01T09:02:38.790 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-01T09:02:38.790 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-01T09:02:38.790 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-01T09:02:38.791 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-01T09:02:38.791 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-01T09:02:38.791 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-01T09:02:38.791 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-01T09:02:38.791 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-01T09:02:38.791 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="App mode production" logger=settings 2023-10-01T09:02:38.791 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-01T09:02:38.791 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-01T09:02:38.791 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-01T09:02:38.791 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2023-10-01T09:02:38.791 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2023-10-01T09:02:39.057 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2023-10-01T09:02:39.057 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-10-01T09:02:39.057 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-10-01T09:02:39.057 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-10-01T09:02:39.057 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-10-01T09:02:39.057 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:38 smithi190 conmon[131967]: t=2023-10-01T09:02:38+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-10-01T09:02:39.057 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-10-01T09:02:39.364 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-10-01T09:02:39.364 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-01T09:02:39.365 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-10-01T09:02:39.365 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-10-01T09:02:39.365 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-10-01T09:02:39.365 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-10-01T09:02:39.365 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-10-01T09:02:39.365 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-10-01T09:02:39.365 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-10-01T09:02:39.672 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-10-01T09:02:39.673 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-10-01T09:02:39.673 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-10-01T09:02:39.673 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-10-01T09:02:39.673 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-10-01T09:02:39.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:39 smithi138 conmon[109959]: audit 2023-10-01T09:02:38.802940+0000 mon.a (mon.0 2023-10-01T09:02:39.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:39 smithi138 conmon[109959]: ) 607 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:39.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:39 smithi138 conmon[109959]: audit 2023-10-01T09:02:38.804741+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:02:39.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:39 smithi138 conmon[109959]: cluster 2023-10-01T09:02:38.838826+0000 mgr.y (mgr.14433) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:39.948 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-10-01T09:02:39.948 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-10-01T09:02:39.948 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-10-01T09:02:39.948 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-10-01T09:02:39.948 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-10-01T09:02:39.948 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[100518]: audit 2023-10-01T09:02:38.802940+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:39.949 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[100518]: 2023-10-01T09:02:39.949 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[100518]: audit 2023-10-01T09:02:38.804741+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:02:39.949 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[100518]: cluster 2023-10-01T09:02: 2023-10-01T09:02:39.949 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[100518]: 38.838826+0000 mgr.y (mgr.14433) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:40.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:39 smithi138 conmon[100289]: audit 2023-10-01T09:02:38.802940+0000 mon.a ( 2023-10-01T09:02:40.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:39 smithi138 conmon[100289]: mon.0) 607 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:40.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:39 smithi138 conmon[100289]: audit 2023-10-01T09:02:38.804741+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:02:40.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:39 smithi138 conmon[100289]: cluster 2023-10-01T09:02:38.838826+0000 mgr.y (mgr.14433) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:40.200 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:39 smithi190 conmon[131967]: t=2023-10-01T09:02:39+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-10-01T09:02:40.200 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-10-01T09:02:40.200 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-10-01T09:02:40.200 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-10-01T09:02:40.200 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-10-01T09:02:40.200 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-10-01T09:02:40.484 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:40.485 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-10-01T09:02:40.485 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-10-01T09:02:40.486 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-10-01T09:02:40.486 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-10-01T09:02:40.486 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-10-01T09:02:40.486 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-10-01T09:02:40.595 INFO:teuthology.orchestra.run.smithi138.stdout:128849018908 2023-10-01T09:02:40.763 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-10-01T09:02:40.763 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-10-01T09:02:40.763 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-10-01T09:02:40.763 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-10-01T09:02:40.764 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-10-01T09:02:41.015 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-10-01T09:02:41.015 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-10-01T09:02:41.015 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-10-01T09:02:41.015 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:40 smithi190 conmon[131967]: t=2023-10-01T09:02:40+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-10-01T09:02:41.225 INFO:teuthology.orchestra.run.smithi138.stdout:77309411365 2023-10-01T09:02:41.306 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-10-01T09:02:41.307 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-10-01T09:02:41.307 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-10-01T09:02:41.307 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-10-01T09:02:41.307 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-10-01T09:02:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[100518]: audit 2023-10-01T09:02:40.015344+0000 2023-10-01T09:02:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[100518]: mon.a (mon.0) 608 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[100518]: audit 2023-10-01T09:02:40.547254+0000 mon.b (mon.1) 26 : audit 2023-10-01T09:02:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[100518]: [DBG] from='client.? 172.21.15.138:0/988489830' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T09:02:41.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:02:41.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[100518]: 10-01T09:02:40.595728+0000 mon.c (mon.2) 53 : audit 2023-10-01T09:02:41.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[100518]: [DBG] from='client.? 172.21.15.138:0/3292566463' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T09:02:41.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:41 smithi138 conmon[109959]: audit 2023-10-01T09:02:40.015344+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:41.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:41 smithi138 conmon[109959]: audit 2023-10-01T09:02:40.547254+0000 mon.b (mon.1) 26 : audit [DBG] from='client.? 172.21.15.138:0/988489830' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T09:02:41.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:41 smithi138 conmon[109959]: audit 2023-10-01T09:02:40.595728+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.138:0/3292566463' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T09:02:41.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:41 smithi138 conmon[100289]: audit 2023-10-01T09:02:40.015344+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:41.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:41 smithi138 conmon[100289]: audit 2023-10-01T09:02:40.547254+0000 mon.b (mon.1) 26 : audit [DBG] from='client.? 172.21.15.138:0/988489830' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T09:02:41.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:41 smithi138 conmon[100289]: audit 2023-10-01T09:02:40.595728+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.138:0/3292566463' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T09:02:41.602 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-10-01T09:02:41.602 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-10-01T09:02:41.602 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-10-01T09:02:41.602 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-10-01T09:02:41.602 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-10-01T09:02:41.602 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-10-01T09:02:41.602 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-10-01T09:02:41.887 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-10-01T09:02:41.887 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-10-01T09:02:41.887 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-10-01T09:02:41.888 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-10-01T09:02:41.888 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-10-01T09:02:42.062 INFO:teuthology.orchestra.run.smithi138.stdout:180388626453 2023-10-01T09:02:42.156 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:41 smithi190 conmon[131967]: t=2023-10-01T09:02:41+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-10-01T09:02:42.156 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-10-01T09:02:42.156 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-10-01T09:02:42.156 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-10-01T09:02:42.156 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-10-01T09:02:42.156 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-10-01T09:02:42.424 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-10-01T09:02:42.425 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-10-01T09:02:42.425 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-10-01T09:02:42.425 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-10-01T09:02:42.425 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-10-01T09:02:42.425 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-10-01T09:02:42.425 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-10-01T09:02:42.425 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-10-01T09:02:42.425 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-10-01T09:02:42.425 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-10-01T09:02:42.686 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[100518]: cluster 2023-10-01T09:02:40.839333+0000 mgr.y (mgr.14433) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:42.686 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[100518]: audit 2023-10-01T09:02:42.039880+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:42.686 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:02:42.686 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[100518]: 01T09:02:42.062044+0000 mon.c 2023-10-01T09:02:42.686 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[100518]: (mon.2) 54 : audit [DBG] 2023-10-01T09:02:42.686 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[100518]: from='client.? 172.21.15.138:0/2387602114' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T09:02:42.686 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-10-01T09:02:42.687 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-10-01T09:02:42.687 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-10-01T09:02:42.687 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-10-01T09:02:42.687 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-10-01T09:02:42.687 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-10-01T09:02:42.687 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-10-01T09:02:42.687 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-10-01T09:02:42.687 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-10-01T09:02:42.687 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-10-01T09:02:42.700 INFO:teuthology.orchestra.run.smithi138.stdout:154618822682 2023-10-01T09:02:42.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:42 smithi138 conmon[109959]: cluster 2023-10-01T09:02:40. 2023-10-01T09:02:42.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:42 smithi138 conmon[109959]: 839333+0000 mgr.y (mgr.14433) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:42.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:42 smithi138 conmon[109959]: audit 2023-10-01T09:02:42.039880+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:42.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:42 smithi138 conmon[109959]: audit 2023-10-01T09:02:42.062044+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.138:0/2387602114' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T09:02:42.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:42 smithi138 conmon[100289]: cluster 2023-10-01T09:02:40.839333+0000 2023-10-01T09:02:42.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:42 smithi138 conmon[100289]: mgr.y (mgr.14433) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:42.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:42 smithi138 conmon[100289]: audit 2023-10-01T09:02:42.039880+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:42.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:42 smithi138 conmon[100289]: audit 2023-10-01T09:02:42.062044+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.138:0/2387602114' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T09:02:42.946 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-10-01T09:02:42.946 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-10-01T09:02:42.946 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-10-01T09:02:42.946 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-10-01T09:02:42.947 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-10-01T09:02:42.947 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-10-01T09:02:42.947 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-10-01T09:02:42.947 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-10-01T09:02:42.947 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-10-01T09:02:43.206 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:42 smithi190 conmon[131967]: t=2023-10-01T09:02:42+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-01T09:02:43.206 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-10-01T09:02:43.206 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-10-01T09:02:43.207 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-10-01T09:02:43.207 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-10-01T09:02:43.207 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-10-01T09:02:43.207 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-10-01T09:02:43.207 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-10-01T09:02:43.494 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-10-01T09:02:43.494 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-10-01T09:02:43.494 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-10-01T09:02:43.494 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-10-01T09:02:43.495 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-10-01T09:02:43.495 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-10-01T09:02:43.495 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-10-01T09:02:43.495 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-10-01T09:02:43.544 INFO:teuthology.orchestra.run.smithi138.stdout:206158430224 2023-10-01T09:02:43.754 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[100518]: audit 2023-10-01T09:02:42.699834+0000 mon.c (mon.2) 55 2023-10-01T09:02:43.754 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[100518]: : audit [DBG] from='client.? 172.21.15.138:0/3802375347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T09:02:43.755 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-10-01T09:02:43.755 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-10-01T09:02:43.755 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-10-01T09:02:43.755 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-10-01T09:02:43.755 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-10-01T09:02:43.755 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-10-01T09:02:43.755 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-10-01T09:02:43.755 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-10-01T09:02:43.755 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-10-01T09:02:43.756 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-10-01T09:02:43.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:43 smithi138 conmon[109959]: audit 2023-10-01T09:02:42.699834+0000 mon.c (mon. 2023-10-01T09:02:43.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:43 smithi138 conmon[109959]: 2) 55 : audit [DBG] from='client.? 172.21.15.138:0/3802375347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T09:02:43.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:43 smithi138 conmon[100289]: audit 2023-10-01T09:02:42.699834+0000 mon.c ( 2023-10-01T09:02:43.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:43 smithi138 conmon[100289]: mon.2) 55 : audit [DBG] from='client.? 172.21.15.138:0/3802375347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T09:02:44.014 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-10-01T09:02:44.015 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-10-01T09:02:44.015 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-10-01T09:02:44.015 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-10-01T09:02:44.015 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-10-01T09:02:44.015 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-10-01T09:02:44.015 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:43 smithi190 conmon[131967]: t=2023-10-01T09:02:43+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-10-01T09:02:44.015 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-10-01T09:02:44.274 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-10-01T09:02:44.275 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-10-01T09:02:44.275 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-10-01T09:02:44.275 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-10-01T09:02:44.275 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-10-01T09:02:44.275 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-10-01T09:02:44.275 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-10-01T09:02:44.275 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-10-01T09:02:44.382 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018907 got 128849018908 for osd.4 2023-10-01T09:02:44.382 DEBUG:teuthology.parallel:result is None 2023-10-01T09:02:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[100518]: cluster 2023-10-01T09:02:42.839632+0000 mgr.y (mgr.14433) 46 : cluster 2023-10-01T09:02:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[100518]: [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[100518]: audit 2023-10-01T09:02:43.544568+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.138:0/3067200599' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T09:02:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[100518]: audit 2023-10-01T09:02:44.062602+0000 mon.c (mon.2) 57 : audit [DBG] 2023-10-01T09:02:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[100518]: from='client.? 172.21.15.138:0/4134008426' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T09:02:44.557 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-10-01T09:02:44.557 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-10-01T09:02:44.557 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-10-01T09:02:44.558 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-10-01T09:02:44.558 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-10-01T09:02:44.558 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-10-01T09:02:44.558 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-10-01T09:02:44.558 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-10-01T09:02:44.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:44 smithi138 conmon[109959]: cluster 2023-10-01T09:02:42.839632+0000 2023-10-01T09:02:44.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:44 smithi138 conmon[109959]: mgr.y (mgr.14433) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:44.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:44 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:02:44.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:44 smithi138 conmon[109959]: :02:43.544568+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.138:0/3067200599' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T09:02:44.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:44 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:02:44.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:44 smithi138 conmon[109959]: 10-01T09:02:44.062602+0000 mon.c ( 2023-10-01T09:02:44.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:44 smithi138 conmon[109959]: mon.2) 57 : audit [DBG] from='client.? 172.21.15.138:0/4134008426' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T09:02:44.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:44 smithi138 conmon[100289]: cluster 2023-10-01T09:02:42.839632+0000 mgr.y 2023-10-01T09:02:44.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:44 smithi138 conmon[100289]: (mgr.14433) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:44.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:44 smithi138 conmon[100289]: audit 2023-10-01T09:02:43.544568+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.138:0/3067200599' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T09:02:44.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:44 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T09:02:44.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:44 smithi138 conmon[100289]: T09:02:44.062602+0000 mon.c (mon.2) 2023-10-01T09:02:44.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:44 smithi138 conmon[100289]: 57 : audit [DBG] from='client.? 172.21.15.138:0/4134008426' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T09:02:44.829 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-10-01T09:02:44.829 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-10-01T09:02:44.829 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-10-01T09:02:44.829 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-10-01T09:02:44.829 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-10-01T09:02:44.830 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-10-01T09:02:44.830 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-10-01T09:02:44.830 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-10-01T09:02:44.830 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-10-01T09:02:44.830 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-10-01T09:02:44.830 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-10-01T09:02:44.830 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-10-01T09:02:44.939 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430222 got 206158430224 for osd.7 2023-10-01T09:02:44.939 DEBUG:teuthology.parallel:result is None 2023-10-01T09:02:44.972 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626450 got 180388626453 for osd.6 2023-10-01T09:02:44.972 DEBUG:teuthology.parallel:result is None 2023-10-01T09:02:45.089 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-10-01T09:02:45.089 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-10-01T09:02:45.090 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-10-01T09:02:45.090 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-10-01T09:02:45.090 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:44 smithi190 conmon[131967]: t=2023-10-01T09:02:44+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-10-01T09:02:45.090 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-10-01T09:02:45.090 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-10-01T09:02:45.090 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-10-01T09:02:45.349 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-10-01T09:02:45.350 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-10-01T09:02:45.350 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-10-01T09:02:45.350 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-10-01T09:02:45.350 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-10-01T09:02:45.350 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-10-01T09:02:45.350 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-10-01T09:02:45.551 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:02:45 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:02:45] "GET /metrics HTTP/1.1" 200 155661 "" "Prometheus/2.33.4" 2023-10-01T09:02:45.610 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-10-01T09:02:45.610 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-10-01T09:02:45.610 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-10-01T09:02:45.610 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-10-01T09:02:45.610 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-10-01T09:02:45.610 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-10-01T09:02:45.610 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-10-01T09:02:45.610 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-10-01T09:02:45.611 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-10-01T09:02:45.874 INFO:teuthology.orchestra.run.smithi138.stdout:34359738416 2023-10-01T09:02:45.878 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-10-01T09:02:45.879 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-10-01T09:02:45.879 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-10-01T09:02:45.879 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-10-01T09:02:45.879 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-10-01T09:02:45.879 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-10-01T09:02:45.879 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-10-01T09:02:46.041 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822679 got 154618822682 for osd.5 2023-10-01T09:02:46.041 DEBUG:teuthology.parallel:result is None 2023-10-01T09:02:46.072 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411363 got 77309411365 for osd.2 2023-10-01T09:02:46.073 DEBUG:teuthology.parallel:result is None 2023-10-01T09:02:46.130 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-10-01T09:02:46.130 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-10-01T09:02:46.131 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-10-01T09:02:46.131 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:45 smithi190 conmon[131967]: t=2023-10-01T09:02:45+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-10-01T09:02:46.131 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-10-01T09:02:46.131 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-10-01T09:02:46.131 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-10-01T09:02:46.131 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-10-01T09:02:46.131 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-10-01T09:02:46.399 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-10-01T09:02:46.399 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-10-01T09:02:46.399 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-10-01T09:02:46.399 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-10-01T09:02:46.399 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-10-01T09:02:46.399 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-10-01T09:02:46.399 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-10-01T09:02:46.400 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-10-01T09:02:46.400 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-10-01T09:02:46.668 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[100518]: cluster 2023-10-01T09:02:44.840139+0000 mgr.y (mgr.14433) 47 : cluster 2023-10-01T09:02:46.668 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[100518]: [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:46.668 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-10-01T09:02:46.668 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-10-01T09:02:46.669 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-10-01T09:02:46.669 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-10-01T09:02:46.669 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-10-01T09:02:46.669 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-10-01T09:02:46.669 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-10-01T09:02:46.670 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-10-01T09:02:46.670 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-10-01T09:02:46.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:46 smithi138 conmon[100289]: cluster 2023-10-01T09:02:44.840139+0000 mgr.y 2023-10-01T09:02:46.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:46 smithi138 conmon[100289]: (mgr.14433) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:46.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:46 smithi138 conmon[109959]: cluster 2023-10-01T09:02:44.840139+0000 2023-10-01T09:02:46.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:46 smithi138 conmon[109959]: mgr.y (mgr.14433) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:46.969 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:02:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:02:46.969 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-10-01T09:02:46.969 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-10-01T09:02:46.969 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-10-01T09:02:46.970 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-10-01T09:02:46.970 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-10-01T09:02:46.970 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-10-01T09:02:46.970 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:46 smithi190 conmon[131967]: t=2023-10-01T09:02:46+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-10-01T09:02:47.163 INFO:teuthology.orchestra.run.smithi138.stdout:107374182434 2023-10-01T09:02:47.230 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-10-01T09:02:47.230 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-10-01T09:02:47.230 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-10-01T09:02:47.230 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-10-01T09:02:47.230 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-10-01T09:02:47.230 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-10-01T09:02:47.230 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-10-01T09:02:47.230 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-10-01T09:02:47.287 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738413 got 34359738416 for osd.0 2023-10-01T09:02:47.287 DEBUG:teuthology.parallel:result is None 2023-10-01T09:02:47.504 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:47 smithi138 conmon[109959]: audit 2023-10-01T09:02:47.163462+0000 mon.c (mon. 2023-10-01T09:02:47.504 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:47 smithi138 conmon[109959]: 2) 58 : audit [DBG] from='client.? 172.21.15.138:0/1151282590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T09:02:47.504 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:47 smithi138 conmon[100289]: audit 2023-10-01T09:02:47 2023-10-01T09:02:47.504 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:47 smithi138 conmon[100289]: .163462+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.138:0/1151282590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T09:02:47.505 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-10-01T09:02:47.505 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-10-01T09:02:47.505 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-10-01T09:02:47.505 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-10-01T09:02:47.505 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-10-01T09:02:47.505 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-10-01T09:02:47.505 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-10-01T09:02:47.506 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-10-01T09:02:47.506 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-10-01T09:02:47.767 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[100518]: audit 2023-10-01T09:02:47.163462+0000 mon.c (mon.2) 2023-10-01T09:02:47.767 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[100518]: 58 : audit [DBG] from='client.? 172.21.15.138:0/1151282590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T09:02:47.767 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-10-01T09:02:47.767 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-10-01T09:02:47.767 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-10-01T09:02:47.768 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-10-01T09:02:47.768 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-10-01T09:02:47.768 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-10-01T09:02:47.768 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-10-01T09:02:47.768 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-10-01T09:02:47.980 INFO:teuthology.orchestra.run.smithi138.stdout:55834574892 2023-10-01T09:02:48.018 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-10-01T09:02:48.019 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-10-01T09:02:48.019 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-10-01T09:02:48.019 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-10-01T09:02:48.019 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-10-01T09:02:48.019 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-10-01T09:02:48.019 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:47 smithi190 conmon[131967]: t=2023-10-01T09:02:47+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-10-01T09:02:48.019 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-10-01T09:02:48.210 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182432 got 107374182434 for osd.3 2023-10-01T09:02:48.210 DEBUG:teuthology.parallel:result is None 2023-10-01T09:02:48.296 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-10-01T09:02:48.296 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-10-01T09:02:48.296 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-10-01T09:02:48.296 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-10-01T09:02:48.296 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-10-01T09:02:48.296 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-10-01T09:02:48.296 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-10-01T09:02:48.296 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-10-01T09:02:48.296 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-10-01T09:02:48.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[100518]: cluster 2023-10-01T09:02:46.840603+0000 mgr.y (mgr.14433) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[100518]: audit 2023-10-01T09:02:47.664952+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.138:0/2852525163' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T09:02:48.557 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-10-01T09:02:48.557 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-10-01T09:02:48.557 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-10-01T09:02:48.557 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-10-01T09:02:48.557 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-10-01T09:02:48.557 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-10-01T09:02:48.558 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-10-01T09:02:48.558 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-10-01T09:02:48.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:48 smithi138 conmon[109959]: cluster 2023-10-01T09:02:46.840603+0000 mgr.y (mgr.14433) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:48 smithi138 conmon[109959]: audit 2023-10-01T09:02:47.664952+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.138:0/2852525163' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T09:02:48.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:48 smithi138 conmon[100289]: cluster 2023-10-01T09:02:46.840603+0000 mgr.y (mgr.14433) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:48.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:48 smithi138 conmon[100289]: audit 2023-10-01T09:02:47.664952+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.138:0/2852525163' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T09:02:48.858 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-10-01T09:02:48.858 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-10-01T09:02:48.858 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-10-01T09:02:48.858 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-10-01T09:02:48.858 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-10-01T09:02:48.858 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-10-01T09:02:48.859 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-10-01T09:02:48.859 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-10-01T09:02:48.859 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-10-01T09:02:48.873 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574889 got 55834574892 for osd.1 2023-10-01T09:02:48.874 DEBUG:teuthology.parallel:result is None 2023-10-01T09:02:48.874 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-01T09:02:48.874 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:02:49.118 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-10-01T09:02:49.119 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-10-01T09:02:49.119 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:48 smithi190 conmon[131967]: t=2023-10-01T09:02:48+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-10-01T09:02:49.119 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-10-01T09:02:49.119 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-10-01T09:02:49.119 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-10-01T09:02:49.119 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-10-01T09:02:49.119 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-10-01T09:02:49.119 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-10-01T09:02:49.371 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-10-01T09:02:49.371 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-10-01T09:02:49.371 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-10-01T09:02:49.371 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-10-01T09:02:49.371 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-10-01T09:02:49.371 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-10-01T09:02:49.372 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-10-01T09:02:49.372 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-10-01T09:02:49.372 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-10-01T09:02:49.372 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-10-01T09:02:49.372 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-10-01T09:02:49.480 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:49.633 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-10-01T09:02:49.633 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-10-01T09:02:49.634 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-10-01T09:02:49.634 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-10-01T09:02:49.634 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-10-01T09:02:49.634 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-10-01T09:02:49.634 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-10-01T09:02:49.634 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-10-01T09:02:49.634 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-10-01T09:02:49.634 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-10-01T09:02:49.883 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-10-01T09:02:49.883 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-10-01T09:02:49.883 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-10-01T09:02:49.883 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-10-01T09:02:49.884 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-10-01T09:02:49.884 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-10-01T09:02:49.884 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-10-01T09:02:49.884 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-10-01T09:02:49.884 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-10-01T09:02:50.144 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-10-01T09:02:50.144 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-10-01T09:02:50.144 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-10-01T09:02:50.144 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:49 smithi190 conmon[131967]: t=2023-10-01T09:02:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-10-01T09:02:50.144 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-10-01T09:02:50.145 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-10-01T09:02:50.145 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-10-01T09:02:50.145 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-10-01T09:02:50.145 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-10-01T09:02:50.145 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-10-01T09:02:50.145 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-10-01T09:02:50.438 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-10-01T09:02:50.438 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-10-01T09:02:50.438 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-10-01T09:02:50.439 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-10-01T09:02:50.439 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-10-01T09:02:50.439 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-10-01T09:02:50.439 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-10-01T09:02:50.439 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-10-01T09:02:50.439 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-10-01T09:02:50.439 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-10-01T09:02:50.439 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-10-01T09:02:50.439 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-10-01T09:02:50.724 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[100518]: cluster 2023-10-01T09:02:48.840890+0000 mgr.y (mgr.14433 2023-10-01T09:02:50.724 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[100518]: ) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:50.724 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-10-01T09:02:50.725 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-10-01T09:02:50.725 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-10-01T09:02:50.725 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-10-01T09:02:50.725 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-10-01T09:02:50.725 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-10-01T09:02:50.725 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-10-01T09:02:50.726 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-10-01T09:02:50.726 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-10-01T09:02:50.726 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-10-01T09:02:50.726 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-10-01T09:02:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:50 smithi138 conmon[109959]: cluster 2023-10-01T09:02:48.840890+0000 mgr.y ( 2023-10-01T09:02:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:50 smithi138 conmon[109959]: mgr.14433) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:50.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:50 smithi138 conmon[100289]: cluster 2023-10-01T09:02:48.840890+0000 mgr.y (mgr 2023-10-01T09:02:50.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:50 smithi138 conmon[100289]: .14433) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:50.992 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-10-01T09:02:50.993 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-10-01T09:02:50.993 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-10-01T09:02:50.993 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-10-01T09:02:50.993 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-10-01T09:02:50.993 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-10-01T09:02:50.993 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-10-01T09:02:50.993 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:50 smithi190 conmon[131967]: t=2023-10-01T09:02:50+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-10-01T09:02:51.269 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-10-01T09:02:51.270 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-10-01T09:02:51.270 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-10-01T09:02:51.270 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-10-01T09:02:51.270 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-10-01T09:02:51.270 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-10-01T09:02:51.270 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-10-01T09:02:51.270 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-10-01T09:02:51.270 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-10-01T09:02:51.343 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:02:51.344 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:02:51.525 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-10-01T09:02:51.525 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-10-01T09:02:51.525 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-10-01T09:02:51.525 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-10-01T09:02:51.525 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=12.623622355s 2023-10-01T09:02:51.525 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-01T09:02:51.525 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-01T09:02:51.525 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-01T09:02:51.525 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-01T09:02:51.526 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-01T09:02:51.526 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-01T09:02:51.526 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-10-01T09:02:51.806 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-01T09:02:51.807 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-01T09:02:51.807 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-01T09:02:51.807 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[131967]: t=2023-10-01T09:02:51+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-01T09:02:51.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: audit 2023-10-01T09:02:50.663686+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:51.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: cluster 2023-10-01T09:02:50. 2023-10-01T09:02:51.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: 841214+0000 mgr.y (mgr.14433) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:51.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: audit 2023-10-01T09:02:51.094793+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:02:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: audit 2023-10-01T09:02: 2023-10-01T09:02:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: 51.095493+0000 mon.c (mon.2) 61 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:02:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:02:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: 10-01T09:02:51.099810+0000 mon.a (mon.0) 2023-10-01T09:02:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: 611 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: cephadm 2023-10-01T09:02: 2023-10-01T09:02:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: 51.102861+0000 mgr.y (mgr.14433) 51 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-01T09:02:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: cephadm 2023-10-01T09:02:51.105749+0000 mgr.y (mgr 2023-10-01T09:02:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: .14433) 52 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi138 2023-10-01T09:02:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T09:02:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: 09:02:51.338771+0000 mgr.y (mgr.14433 2023-10-01T09:02:51.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:51 smithi190 conmon[100518]: ) 53 : audit [DBG] from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:02:52.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[109959]: audit 2023-10-01T09:02:50.663686+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[109959]: cluster 2023-10-01T09:02:50.841214+0000 mgr.y (mgr.14433) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[109959]: audit 2023-10-01T09:02:51.094793+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:02:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[109959]: audit 2023-10-01T09:02:51.095493+0000 mon.c (mon.2) 61 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:02:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[109959]: audit 2023-10-01T09:02:51.099810+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[109959]: cephadm 2023-10-01T09:02:51.102861+0000 mgr.y (mgr.14433) 51 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-01T09:02:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[109959]: cephadm 2023-10-01T09:02:51.105749+0000 mgr.y (mgr.14433) 52 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi138 2023-10-01T09:02:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[109959]: audit 2023-10-01T09:02:51.338771+0000 mgr.y (mgr.14433) 53 : audit [DBG] from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:02:52.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[100289]: audit 2023-10-01T09:02:50.663686+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:52.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[100289]: cluster 2023-10-01T09:02:50.841214+0000 mgr.y (mgr.14433) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:52.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[100289]: audit 2023-10-01T09:02:51.094793+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:02:52.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[100289]: audit 2023-10-01T09:02:51.095493+0000 mon.c (mon.2) 61 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:02:52.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[100289]: audit 2023-10-01T09:02:51.099810+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:52.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[100289]: cephadm 2023-10-01T09:02:52.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[100289]: 2023-10-01T09:02:51.102861+0000 mgr.y (mgr.14433) 51 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-01T09:02:52.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[100289]: cephadm 2023-10-01T09:02:51.105749+0000 mgr.y (mgr.14433) 52 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi138 2023-10-01T09:02:52.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:51 smithi138 conmon[100289]: audit 2023-10-01T09:02:51.338771+0000 mgr.y (mgr.14433) 53 : audit [DBG] from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:02:52.227 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":40,"stamp":"2023-10-01T09:02:50.841030+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380976,"kb_used_data":4208,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580240,"statfs":{"total":767960285184,"available":765522165760,"internally_reserved":0,"allocated":4308992,"data_stored":1762860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T09:02:08.289224+0000","last_change":"2023-10-01T09:02:06.997887+0000","last_active":"2023-10-01T09:02:08.289224+0000","last_peered":"2023-10-01T09:02:08.289224+0000","last_clean":"2023-10-01T09:02:08.289224+0000","last_became_active":"2023-10-01T09:02:06.997597+0000","last_became_peered":"2023-10-01T09:02:06.997597+0000","last_unstale":"2023-10-01T09:02:08.289224+0000","last_undegraded":"2023-10-01T09:02:08.289224+0000","last_fullsized":"2023-10-01T09:02:08.289224+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2023-10-01T09:02:07.442384+0000","last_change":"2023-10-01T09:02:07.442083+0000","last_active":"2023-10-01T09:02:07.442384+0000","last_peered":"2023-10-01T09:02:07.442384+0000","last_clean":"2023-10-01T09:02:07.442384+0000","last_became_active":"2023-10-01T09:02:07.441757+0000","last_became_peered":"2023-10-01T09:02:07.441757+0000","last_unstale":"2023-10-01T09:02:07.442384+0000","last_undegraded":"2023-10-01T09:02:07.442384+0000","last_fullsized":"2023-10-01T09:02:07.442384+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T09:02:08.719871+0000","last_change":"2023-10-01T09:02:07.442452+0000","last_active":"2023-10-01T09:02:08.719871+0000","last_peered":"2023-10-01T09:02:08.719871+0000","last_clean":"2023-10-01T09:02:08.719871+0000","last_became_active":"2023-10-01T09:02:07.442111+0000","last_became_peered":"2023-10-01T09:02:07.442111+0000","last_unstale":"2023-10-01T09:02:08.719871+0000","last_undegraded":"2023-10-01T09:02:08.719871+0000","last_fullsized":"2023-10-01T09:02:08.719871+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T09:02:08.719805+0000","last_change":"2023-10-01T09:02:06.999043+0000","last_active":"2023-10-01T09:02:08.719805+0000","last_peered":"2023-10-01T09:02:08.719805+0000","last_clean":"2023-10-01T09:02:08.719805+0000","last_became_active":"2023-10-01T09:02:06.998671+0000","last_became_peered":"2023-10-01T09:02:06.998671+0000","last_unstale":"2023-10-01T09:02:08.719805+0000","last_undegraded":"2023-10-01T09:02:08.719805+0000","last_fullsized":"2023-10-01T09:02:08.719805+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T09:02:08.719800+0000","last_change":"2023-10-01T09:01:32.510259+0000","last_active":"2023-10-01T09:02:08.719800+0000","last_peered":"2023-10-01T09:02:08.719800+0000","last_clean":"2023-10-01T09:02:08.719800+0000","last_became_active":"2023-10-01T09:01:32.509905+0000","last_became_peered":"2023-10-01T09:01:32.509905+0000","last_unstale":"2023-10-01T09:02:08.719800+0000","last_undegraded":"2023-10-01T09:02:08.719800+0000","last_fullsized":"2023-10-01T09:02:08.719800+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2023-10-01T09:02:06.999327+0000","last_change":"2023-10-01T09:02:06.998804+0000","last_active":"2023-10-01T09:02:06.999327+0000","last_peered":"2023-10-01T09:02:06.999327+0000","last_clean":"2023-10-01T09:02:06.999327+0000","last_became_active":"2023-10-01T09:02:06.998516+0000","last_became_peered":"2023-10-01T09:02:06.998516+0000","last_unstale":"2023-10-01T09:02:06.999327+0000","last_undegraded":"2023-10-01T09:02:06.999327+0000","last_fullsized":"2023-10-01T09:02:06.999327+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T09:02:08.003399+0000","last_change":"2023-10-01T09:02:07.436492+0000","last_active":"2023-10-01T09:02:08.003399+0000","last_peered":"2023-10-01T09:02:08.003399+0000","last_clean":"2023-10-01T09:02:08.003399+0000","last_became_active":"2023-10-01T09:02:07.436146+0000","last_became_peered":"2023-10-01T09:02:07.436146+0000","last_unstale":"2023-10-01T09:02:08.003399+0000","last_undegraded":"2023-10-01T09:02:08.003399+0000","last_fullsized":"2023-10-01T09:02:08.003399+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2023-10-01T09:02:07.001885+0000","last_change":"2023-10-01T09:02:07.001644+0000","last_active":"2023-10-01T09:02:07.001885+0000","last_peered":"2023-10-01T09:02:07.001885+0000","last_clean":"2023-10-01T09:02:07.001885+0000","last_became_active":"2023-10-01T09:02:07.001197+0000","last_became_peered":"2023-10-01T09:02:07.001197+0000","last_unstale":"2023-10-01T09:02:07.001885+0000","last_undegraded":"2023-10-01T09:02:07.001885+0000","last_fullsized":"2023-10-01T09:02:07.001885+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2023-10-01T09:02:06.998506+0000","last_change":"2023-10-01T09:02:06.998188+0000","last_active":"2023-10-01T09:02:06.998506+0000","last_peered":"2023-10-01T09:02:06.998506+0000","last_clean":"2023-10-01T09:02:06.998506+0000","last_became_active":"2023-10-01T09:02:06.997855+0000","last_became_peered":"2023-10-01T09:02:06.997855+0000","last_unstale":"2023-10-01T09:02:06.998506+0000","last_undegraded":"2023-10-01T09:02:06.998506+0000","last_fullsized":"2023-10-01T09:02:06.998506+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430226,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.434},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.53000000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.64600000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.5949999999999998,"5min":5.5949999999999998,"15min":5.5949999999999998},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.504}]},{"osd":3,"last update":"Sun Oct 1 09:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":5.6210000000000004,"5min":5.6210000000000004,"15min":5.6210000000000004},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.56799999999999995}]},{"osd":4,"last update":"Sun Oct 1 09:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.5369999999999999,"5min":5.5369999999999999,"15min":5.5369999999999999},"last":0.55700000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.66900000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":5.5700000000000003,"5min":5.5700000000000003,"15min":5.5700000000000003},"last":0.58899999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626455,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.41299999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.2429999999999999,"5min":3.2429999999999999,"15min":3.2429999999999999},"last":0.438}]},{"osd":2,"last update":"Sun Oct 1 09:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.47499999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.505},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.49199999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.621}]},{"osd":5,"last update":"Sun Oct 1 09:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.66800000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.71599999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574893,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:21 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.36799999999999999,"15min":0.36599999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.67200000000000004,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.376,"5min":0.42399999999999999,"15min":0.435},"min":{"1min":0.26500000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.61299999999999999,"5min":2.2829999999999999,"15min":2.2829999999999999},"last":0.46600000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.28999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64400000000000002,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.371,"5min":0.45000000000000001,"15min":0.47599999999999998},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.63500000000000001,"5min":2.3199999999999998,"15min":2.3199999999999998},"last":0.42799999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:02:12 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.42499999999999999,"15min":0.42299999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.46800000000000003,"15min":0.47099999999999997},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.49399999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:02:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.83799999999999997,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.624,"5min":0.58799999999999997,"15min":0.58199999999999996},"min":{"1min":0.46800000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.879,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.78500000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57199999999999995,"15min":0.57799999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.42599999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.75}]},{"osd":6,"last update":"Sun Oct 1 09:02:12 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.03,"5min":1.03,"15min":1.03},"last":0.68799999999999994}]},{"osd":7,"last update":"Sun Oct 1 09:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.999,"5min":0.999,"15min":0.999},"last":0.82999999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738418,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34499999999999997,"15min":0.34300000000000003},"min":{"1min":0.245,"5min":0.22,"15min":0.22},"max":{"1min":0.433,"5min":0.52900000000000003,"15min":0.52900000000000003},"last":0.314},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.53700000000000003,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.41699999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:02:45 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.40699999999999997,"15min":0.40899999999999997},"min":{"1min":0.28399999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.499,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.377,"5min":0.38500000000000001,"15min":0.38400000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.55100000000000005,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.34100000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:02:10 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.43099999999999999,"15min":0.435},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.47399999999999998,"15min":0.47899999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.626,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.39500000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:02:28 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56599999999999995,"15min":0.56799999999999995},"min":{"1min":0.41299999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.81100000000000005,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.78400000000000003,"5min":0.87,"15min":0.87},"last":0.57699999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:01:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.625},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.53200000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:02:10 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.52200000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:02:33 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.65400000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.374,"15min":0.376},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.51500000000000001,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.246},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.371,"15min":0.36199999999999999},"min":{"1min":0.249,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.32200000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.379,"15min":0.375},"min":{"1min":0.253,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.373,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.56599999999999995,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.25700000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:02:07 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.312},{"interface":"front","average":{"1min":0.439,"5min":0.47599999999999998,"15min":0.48199999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.63900000000000001,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.30099999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.54400000000000004,"15min":0.54200000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.62},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.40699999999999997,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.68200000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:01:50 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.629},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.58799999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:02:10 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.56799999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.58499999999999996}]}]},{"osd":3,"up_from":25,"seq":107374182436,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:06 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.39200000000000002,"15min":0.39500000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.40999999999999998,"15min":0.41399999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.28999999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:02:06 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.44700000000000001,"15min":0.45300000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.42099999999999999,"15min":0.42299999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.57599999999999996,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.32100000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:02:06 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.45900000000000002,"15min":0.46300000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.45400000000000001,"15min":0.46200000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.33900000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.59599999999999997,"15min":0.59799999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.60399999999999998,"15min":0.60799999999999998},"min":{"1min":0.44500000000000001,"5min":0.436,"15min":0.436},"max":{"1min":1.1679999999999999,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.505}]},{"osd":5,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56699999999999995,"15min":0.56399999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.60199999999999998,"15min":0.59599999999999997},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.48999999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":0.47699999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:02:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.288,"5min":1.288,"15min":1.288},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.53500000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018911,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.505,"15min":0.50700000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.497,"5min":0.499,"15min":0.499},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.60699999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.374,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.66800000000000004,"5min":0.748,"15min":0.748},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.48499999999999999,"15min":0.47999999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.621}]},{"osd":2,"last update":"Sun Oct 1 09:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.504,"15min":0.503},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.51200000000000001,"15min":0.50900000000000001},"min":{"1min":0.38900000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.64000000000000001,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.47999999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.5,"15min":0.498},"min":{"1min":0.42699999999999999,"5min":0.371,"15min":0.371},"max":{"1min":0.64000000000000001,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51700000000000002,"15min":0.51600000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.56100000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:01:49 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.42299999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:02:09 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.64300000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:02:33 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.65100000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822684,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.497,"15min":0.497},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.497,"5min":0.51300000000000001,"15min":0.51600000000000001},"min":{"1min":0.34799999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.52200000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.49199999999999999,"15min":0.49399999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.55300000000000005,"15min":0.53500000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":4.0490000000000004,"5min":4.0490000000000004,"15min":4.0490000000000004},"last":0.44600000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.54100000000000004,"15min":0.54600000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.499,"15min":0.501},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.40500000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.51900000000000002,"15min":0.51700000000000002},"min":{"1min":0.379,"5min":0.377,"15min":0.377},"max":{"1min":0.67800000000000005,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.50900000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.40100000000000002,"15min":0.39300000000000002},"min":{"1min":0.32300000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.42699999999999999,"15min":0.41199999999999998},"min":{"1min":0.30299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.60199999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:02:09 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.46000000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:02:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.58899999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:02:52.230 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:02:53.126 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:54.299 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:53 smithi138 conmon[100289]: cluster 2023-10-01T09:02:52.841587+0000 mgr.y (mgr.14433) 54 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:54.300 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:53 smithi138 conmon[109959]: cluster 2023-10-01T09:02:52.841587+0000 mgr.y (mgr.14433) 54 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:54.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:53 smithi190 conmon[100518]: cluster 2023-10-01T09:02:52.841587+0000 mgr.y (mgr.14433) 54 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:54.952 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:54 smithi138 systemd[1]: Stopping Ceph alertmanager.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:02:54.952 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:54 smithi138 bash[149719]: Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-alertmanager.a found: no such container 2023-10-01T09:02:55.308 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:55 smithi138 conmon[140971]: level=info ts=2023-10-01T09:02:55.171Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-01T09:02:55.308 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:02:55 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:02:55] "GET /metrics HTTP/1.1" 200 155661 "" "Prometheus/2.33.4" 2023-10-01T09:02:55.327 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:02:55.328 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:02:56.145 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":42,"stamp":"2023-10-01T09:02:54.841733+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380976,"kb_used_data":4208,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580240,"statfs":{"total":767960285184,"available":765522165760,"internally_reserved":0,"allocated":4308992,"data_stored":1762860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T09:02:08.289224+0000","last_change":"2023-10-01T09:02:06.997887+0000","last_active":"2023-10-01T09:02:08.289224+0000","last_peered":"2023-10-01T09:02:08.289224+0000","last_clean":"2023-10-01T09:02:08.289224+0000","last_became_active":"2023-10-01T09:02:06.997597+0000","last_became_peered":"2023-10-01T09:02:06.997597+0000","last_unstale":"2023-10-01T09:02:08.289224+0000","last_undegraded":"2023-10-01T09:02:08.289224+0000","last_fullsized":"2023-10-01T09:02:08.289224+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2023-10-01T09:02:07.442384+0000","last_change":"2023-10-01T09:02:07.442083+0000","last_active":"2023-10-01T09:02:07.442384+0000","last_peered":"2023-10-01T09:02:07.442384+0000","last_clean":"2023-10-01T09:02:07.442384+0000","last_became_active":"2023-10-01T09:02:07.441757+0000","last_became_peered":"2023-10-01T09:02:07.441757+0000","last_unstale":"2023-10-01T09:02:07.442384+0000","last_undegraded":"2023-10-01T09:02:07.442384+0000","last_fullsized":"2023-10-01T09:02:07.442384+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T09:02:08.719871+0000","last_change":"2023-10-01T09:02:07.442452+0000","last_active":"2023-10-01T09:02:08.719871+0000","last_peered":"2023-10-01T09:02:08.719871+0000","last_clean":"2023-10-01T09:02:08.719871+0000","last_became_active":"2023-10-01T09:02:07.442111+0000","last_became_peered":"2023-10-01T09:02:07.442111+0000","last_unstale":"2023-10-01T09:02:08.719871+0000","last_undegraded":"2023-10-01T09:02:08.719871+0000","last_fullsized":"2023-10-01T09:02:08.719871+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T09:02:08.719805+0000","last_change":"2023-10-01T09:02:06.999043+0000","last_active":"2023-10-01T09:02:08.719805+0000","last_peered":"2023-10-01T09:02:08.719805+0000","last_clean":"2023-10-01T09:02:08.719805+0000","last_became_active":"2023-10-01T09:02:06.998671+0000","last_became_peered":"2023-10-01T09:02:06.998671+0000","last_unstale":"2023-10-01T09:02:08.719805+0000","last_undegraded":"2023-10-01T09:02:08.719805+0000","last_fullsized":"2023-10-01T09:02:08.719805+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T09:02:08.719800+0000","last_change":"2023-10-01T09:01:32.510259+0000","last_active":"2023-10-01T09:02:08.719800+0000","last_peered":"2023-10-01T09:02:08.719800+0000","last_clean":"2023-10-01T09:02:08.719800+0000","last_became_active":"2023-10-01T09:01:32.509905+0000","last_became_peered":"2023-10-01T09:01:32.509905+0000","last_unstale":"2023-10-01T09:02:08.719800+0000","last_undegraded":"2023-10-01T09:02:08.719800+0000","last_fullsized":"2023-10-01T09:02:08.719800+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2023-10-01T09:02:06.999327+0000","last_change":"2023-10-01T09:02:06.998804+0000","last_active":"2023-10-01T09:02:06.999327+0000","last_peered":"2023-10-01T09:02:06.999327+0000","last_clean":"2023-10-01T09:02:06.999327+0000","last_became_active":"2023-10-01T09:02:06.998516+0000","last_became_peered":"2023-10-01T09:02:06.998516+0000","last_unstale":"2023-10-01T09:02:06.999327+0000","last_undegraded":"2023-10-01T09:02:06.999327+0000","last_fullsized":"2023-10-01T09:02:06.999327+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T09:02:08.003399+0000","last_change":"2023-10-01T09:02:07.436492+0000","last_active":"2023-10-01T09:02:08.003399+0000","last_peered":"2023-10-01T09:02:08.003399+0000","last_clean":"2023-10-01T09:02:08.003399+0000","last_became_active":"2023-10-01T09:02:07.436146+0000","last_became_peered":"2023-10-01T09:02:07.436146+0000","last_unstale":"2023-10-01T09:02:08.003399+0000","last_undegraded":"2023-10-01T09:02:08.003399+0000","last_fullsized":"2023-10-01T09:02:08.003399+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2023-10-01T09:02:07.001885+0000","last_change":"2023-10-01T09:02:07.001644+0000","last_active":"2023-10-01T09:02:07.001885+0000","last_peered":"2023-10-01T09:02:07.001885+0000","last_clean":"2023-10-01T09:02:07.001885+0000","last_became_active":"2023-10-01T09:02:07.001197+0000","last_became_peered":"2023-10-01T09:02:07.001197+0000","last_unstale":"2023-10-01T09:02:07.001885+0000","last_undegraded":"2023-10-01T09:02:07.001885+0000","last_fullsized":"2023-10-01T09:02:07.001885+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":10,"reported_epoch":53,"state":"active+clean","last_fresh":"2023-10-01T09:02:06.998506+0000","last_change":"2023-10-01T09:02:06.998188+0000","last_active":"2023-10-01T09:02:06.998506+0000","last_peered":"2023-10-01T09:02:06.998506+0000","last_clean":"2023-10-01T09:02:06.998506+0000","last_became_active":"2023-10-01T09:02:06.997855+0000","last_became_peered":"2023-10-01T09:02:06.997855+0000","last_unstale":"2023-10-01T09:02:06.998506+0000","last_undegraded":"2023-10-01T09:02:06.998506+0000","last_fullsized":"2023-10-01T09:02:06.998506+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430226,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.434},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.53000000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.64600000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.5949999999999998,"5min":5.5949999999999998,"15min":5.5949999999999998},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.504}]},{"osd":3,"last update":"Sun Oct 1 09:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":5.6210000000000004,"5min":5.6210000000000004,"15min":5.6210000000000004},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.56799999999999995}]},{"osd":4,"last update":"Sun Oct 1 09:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.5369999999999999,"5min":5.5369999999999999,"15min":5.5369999999999999},"last":0.55700000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.66900000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:02:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":5.5700000000000003,"5min":5.5700000000000003,"15min":5.5700000000000003},"last":0.58899999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626455,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.41299999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.2429999999999999,"5min":3.2429999999999999,"15min":3.2429999999999999},"last":0.438}]},{"osd":2,"last update":"Sun Oct 1 09:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.47499999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.505},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.49199999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.621}]},{"osd":5,"last update":"Sun Oct 1 09:02:11 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.66800000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.71599999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574893,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:21 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.36799999999999999,"15min":0.36599999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.67200000000000004,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.376,"5min":0.42399999999999999,"15min":0.435},"min":{"1min":0.26500000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.61299999999999999,"5min":2.2829999999999999,"15min":2.2829999999999999},"last":0.46600000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.28999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64400000000000002,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.371,"5min":0.45000000000000001,"15min":0.47599999999999998},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.63500000000000001,"5min":2.3199999999999998,"15min":2.3199999999999998},"last":0.42799999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:02:12 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.42499999999999999,"15min":0.42299999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.46800000000000003,"15min":0.47099999999999997},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.49399999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:02:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.83799999999999997,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.624,"5min":0.58799999999999997,"15min":0.58199999999999996},"min":{"1min":0.46800000000000003,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.879,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.78500000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57199999999999995,"15min":0.57799999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.42599999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.75}]},{"osd":6,"last update":"Sun Oct 1 09:02:12 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.03,"5min":1.03,"15min":1.03},"last":0.68799999999999994}]},{"osd":7,"last update":"Sun Oct 1 09:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.999,"5min":0.999,"15min":0.999},"last":0.82999999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738418,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34499999999999997,"15min":0.34300000000000003},"min":{"1min":0.245,"5min":0.22,"15min":0.22},"max":{"1min":0.433,"5min":0.52900000000000003,"15min":0.52900000000000003},"last":0.314},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.53700000000000003,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.41699999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:02:45 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.40699999999999997,"15min":0.40899999999999997},"min":{"1min":0.28399999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.499,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.377,"5min":0.38500000000000001,"15min":0.38400000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.55100000000000005,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.34100000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:02:10 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.43099999999999999,"15min":0.435},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.47399999999999998,"15min":0.47899999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.626,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.39500000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:02:28 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56599999999999995,"15min":0.56799999999999995},"min":{"1min":0.41299999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.81100000000000005,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.78400000000000003,"5min":0.87,"15min":0.87},"last":0.57699999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:01:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.625},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.53200000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:02:10 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.52200000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:02:33 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.65400000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.374,"15min":0.376},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.51500000000000001,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.246},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.371,"15min":0.36199999999999999},"min":{"1min":0.249,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.32200000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.379,"15min":0.375},"min":{"1min":0.253,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.373,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.56599999999999995,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.25700000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:02:07 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.312},{"interface":"front","average":{"1min":0.439,"5min":0.47599999999999998,"15min":0.48199999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.63900000000000001,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.30099999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.54400000000000004,"15min":0.54200000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.62},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.40699999999999997,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.68200000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:01:50 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.629},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.58799999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:02:10 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.56799999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:02:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.58499999999999996}]}]},{"osd":3,"up_from":25,"seq":107374182436,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:06 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.39200000000000002,"15min":0.39500000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.40999999999999998,"15min":0.41399999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.28999999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:02:06 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.44700000000000001,"15min":0.45300000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.42099999999999999,"15min":0.42299999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.57599999999999996,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.32100000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:02:06 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.45900000000000002,"15min":0.46300000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.45400000000000001,"15min":0.46200000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.33900000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.59599999999999997,"15min":0.59799999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.60399999999999998,"15min":0.60799999999999998},"min":{"1min":0.44500000000000001,"5min":0.436,"15min":0.436},"max":{"1min":1.1679999999999999,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.505}]},{"osd":5,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56699999999999995,"15min":0.56399999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.60199999999999998,"15min":0.59599999999999997},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.48999999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:02:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":0.47699999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:02:34 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.288,"5min":1.288,"15min":1.288},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.53500000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018911,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.505,"15min":0.50700000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.497,"5min":0.499,"15min":0.499},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.60699999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.374,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.66800000000000004,"5min":0.748,"15min":0.748},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.48499999999999999,"15min":0.47999999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.621}]},{"osd":2,"last update":"Sun Oct 1 09:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.504,"15min":0.503},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.51200000000000001,"15min":0.50900000000000001},"min":{"1min":0.38900000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.64000000000000001,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.47999999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:02:29 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.5,"15min":0.498},"min":{"1min":0.42699999999999999,"5min":0.371,"15min":0.371},"max":{"1min":0.64000000000000001,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51700000000000002,"15min":0.51600000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.56100000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:01:49 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.42299999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:02:09 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.441,"15min":0.441},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.64300000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:02:33 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.65100000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822684,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.497,"15min":0.497},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.497,"5min":0.51300000000000001,"15min":0.51600000000000001},"min":{"1min":0.34799999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.52200000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.49199999999999999,"15min":0.49399999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.55300000000000005,"15min":0.53500000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":4.0490000000000004,"5min":4.0490000000000004,"15min":4.0490000000000004},"last":0.44600000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.54100000000000004,"15min":0.54600000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.499,"15min":0.501},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.40500000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.51900000000000002,"15min":0.51700000000000002},"min":{"1min":0.379,"5min":0.377,"15min":0.377},"max":{"1min":0.67800000000000005,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.50900000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.40100000000000002,"15min":0.39300000000000002},"min":{"1min":0.32300000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.42699999999999999,"15min":0.41199999999999998},"min":{"1min":0.30299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.60199999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:02:09 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.46000000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:02:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.58899999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:02:56.148 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-01T09:02:56.148 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-01T09:02:56.148 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-01T09:02:56.148 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-01T09:02:56.494 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:56 smithi138 bash[149859]: 20e5c74607299944e0bb16c4822523b3e5f22240437f0533b871685d2da5d761 2023-10-01T09:02:56.495 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:56 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-01T09:02:56.495 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:56 smithi138 systemd[1]: Stopped Ceph alertmanager.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:02:56.495 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:56 smithi138 systemd[1]: Starting Ceph alertmanager.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:02:56.542 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:02:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:56 smithi190 conmon[100518]: cluster 2023-10-01T09:02:54.841867+0000 mgr.y (mgr.14433) 55 : cluster [DBG] 2023-10-01T09:02:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:56 smithi190 conmon[100518]: pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:56 smithi190 conmon[100518]: audit 2023-10-01T09:02:55.323500+0000 mgr.y (mgr.14433) 56 : audit [DBG] from='client.24527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:02:56.810 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:56 smithi138 conmon[100289]: cluster 2023-10-01T09:02:54.841867+0000 mgr.y (mgr.14433) 55 : cluster 2023-10-01T09:02:56.810 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:56 smithi138 conmon[100289]: [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:56.810 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:56 smithi138 conmon[100289]: audit 2023-10-01T09:02:55.323500+0000 mgr.y (mgr.14433) 56 : audit [DBG] from='client.24527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:02:56.810 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:56 smithi138 conmon[109959]: cluster 2023-10-01T09:02:54.841867+0000 mgr.y (mgr. 2023-10-01T09:02:56.810 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:56 smithi138 conmon[109959]: 14433) 55 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:56.810 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:56 smithi138 conmon[109959]: audit 2023-10-01T09:02:55.323500+0000 mgr.y (mgr.14433) 56 : audit [DBG] from='client.24527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:02:57.069 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:57 smithi138 conmon[150478]: level=info ts=2023-10-01T09:02:57.048Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-01T09:02:57.070 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:57 smithi138 conmon[150478]: level=info ts=2023-10-01T09:02:57.048Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-01T09:02:57.070 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:57 smithi138 conmon[150478]: level=info ts=2023-10-01T09:02:57.050Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.138 port=9094 2023-10-01T09:02:57.070 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:57 smithi138 conmon[150478]: level=info ts=2023-10-01T09:02:57.051Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-01T09:02:57.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:02:56 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:02:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:02:57.570 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:57 smithi138 bash[150360]: a3bfe0be9ee50f1ea600ee982d9217d5092dce968edc2666fca96289a1a6cabb 2023-10-01T09:02:57.570 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:57 smithi138 systemd[1]: Started Ceph alertmanager.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:02:57.570 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:57 smithi138 conmon[150478]: level=info ts=2023-10-01T09:02:57.077Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-01T09:02:57.570 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:57 smithi138 conmon[150478]: level=info ts=2023-10-01T09:02:57.077Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-01T09:02:57.570 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:57 smithi138 conmon[150478]: level=info ts=2023-10-01T09:02:57.079Z caller=main.go:518 msg=Listening address=:9093 2023-10-01T09:02:57.570 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:57 smithi138 conmon[150478]: level=info ts=2023-10-01T09:02:57.080Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-01T09:02:58.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:58 smithi190 conmon[100518]: cluster 2023-10-01T09:02:56.842386+0000 2023-10-01T09:02:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:58 smithi190 conmon[100518]: mgr.y (mgr.14433) 57 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:58 smithi190 conmon[100518]: audit 2023-10-01T09:02:57.102370+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:58 smithi190 conmon[100518]: cephadm 2023-10-01T09:02:57.105739+0000 mgr.y (mgr.14433) 58 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T09:02:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:58 smithi190 conmon[100518]: cephadm 2023-10-01T09:02:57.109187+0000 mgr.y (mgr.14433 2023-10-01T09:02:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:58 smithi190 conmon[100518]: ) 59 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:02:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[100289]: cluster 2023-10-01T09:02:56.842386+0000 mgr.y (mgr.14433) 2023-10-01T09:02:58.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[100289]: 57 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:58.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[100289]: audit 2023-10-01T09:02:57.102370+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:58.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[100289]: cephadm 2023-10-01T09:02:57.105739+0000 mgr.y (mgr.14433) 58 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T09:02:58.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[100289]: cephadm 2023-10-01T09:02:57.109187+0000 mgr.y (mgr.14433) 59 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:02:58.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[109959]: cluster 2023-10-01T09:02:56.842386+0000 2023-10-01T09:02:58.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[109959]: mgr.y (mgr.14433) 57 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:02:58.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:02:58.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[109959]: :02:57.102370+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:02:58.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[109959]: cephadm 2023-10 2023-10-01T09:02:58.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[109959]: -01T09:02:57.105739+0000 mgr.y (mgr. 2023-10-01T09:02:58.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[109959]: 14433) 58 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T09:02:58.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[109959]: cephadm 2023-10 2023-10-01T09:02:58.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[109959]: -01T09:02:57.109187+0000 mgr.y (mgr 2023-10-01T09:02:58.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:58 smithi138 conmon[109959]: .14433) 59 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:02:58.572 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:02:59.232 INFO:teuthology.orchestra.run.smithi138.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-01T09:02:59.232 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-01T09:02:59.232 INFO:tasks.cephadm:Setup complete, yielding 2023-10-01T09:02:59.233 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-01T09:02:59.242 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi138.front.sepia.ceph.com 2023-10-01T09:02:59.243 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-10-01T09:02:59.319 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:02:59 smithi138 conmon[150478]: level=info ts=2023-10-01T09:02:59.051Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000144509s 2023-10-01T09:02:59.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:59 smithi138 conmon[100289]: audit 2023-10-01T09:02:58.570626+0000 mon.c (mon.2) 62 : audit 2023-10-01T09:02:59.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:02:59 smithi138 conmon[100289]: [DBG] from='client.? 172.21.15.138:0/3823141043' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T09:02:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:59 smithi138 conmon[109959]: audit 2023-10-01T09:02:58.570626 2023-10-01T09:02:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:02:59 smithi138 conmon[109959]: +0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.138:0/3823141043' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T09:02:59.544 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:59 smithi190 conmon[100518]: audit 2023-10-01T09:02:58.570626+0000 mon.c (mon.2) 62 : audit 2023-10-01T09:02:59.544 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:02:59 smithi190 conmon[100518]: [DBG] from='client.? 172.21.15.138:0/3823141043' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T09:02:59.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:59 smithi190 systemd[1]: Stopping Ceph prometheus.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:02:59.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:59 smithi190 bash[133261]: Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-prometheus.a found: no such container 2023-10-01T09:03:00.113 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:59 smithi190 conmon[129942]: ts=2023-10-01T09:02:59.862Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-01T09:03:00.113 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:59 smithi190 conmon[129942]: ts=2023-10-01T09:02:59.862Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-01T09:03:00.113 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:59 smithi190 conmon[129942]: ts=2023-10-01T09:02:59.862Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-01T09:03:00.113 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:59 smithi190 conmon[129942]: ts=2023-10-01T09:02:59.862Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-01T09:03:00.113 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:59 smithi190 conmon[129942]: ts=2023-10-01T09:02:59.862Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-01T09:03:00.113 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:59 smithi190 conmon[129942]: ts=2023-10-01T09:02:59.862Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-01T09:03:00.113 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:59 smithi190 conmon[129942]: ts=2023-10-01T09:02:59.862Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-01T09:03:00.114 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:59 smithi190 conmon[129942]: ts=2023-10-01T09:02:59.862Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-01T09:03:00.114 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:59 smithi190 conmon[129942]: ts=2023-10-01T09:02:59.862Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-01T09:03:00.114 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:59 smithi190 conmon[129942]: ts=2023-10-01T09:02:59.863Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-01T09:03:00.114 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:59 smithi190 conmon[129942]: ts=2023-10-01T09:02:59.863Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-01T09:03:00.114 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:02:59 smithi190 conmon[129942]: ts=2023-10-01T09:02:59.863Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-01T09:03:00.364 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:00 smithi190 conmon[100518]: cluster 2023-10-01T09:02:58.842751+0000 mgr.y (mgr.14433) 60 2023-10-01T09:03:00.364 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:00 smithi190 conmon[100518]: : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:00.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:00 smithi138 conmon[109959]: cluster 2023-10-01T09:02:58.842751+0000 mgr.y (mgr.14433) 60 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:00.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:00 smithi138 conmon[100289]: cluster 2023-10-01T09:02:58.842751+0000 mgr.y (mgr.14433) 60 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:00.806 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:00 smithi190 bash[133378]: a5f5e394a6702baa386e9bd182526dfb631e20b04b4547a7cce44f79ecebf309 2023-10-01T09:03:00.806 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:00 smithi190 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-01T09:03:00.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:00 smithi190 systemd[1]: Stopped Ceph prometheus.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:03:00.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:00 smithi190 systemd[1]: Starting Ceph prometheus.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:03:01.503 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Bus STOPPING 2023-10-01T09:03:01.503 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T09:03:01.503 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Bus STOPPED 2023-10-01T09:03:01.503 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Bus STARTING 2023-10-01T09:03:01.504 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Serving on http://:::9283 2023-10-01T09:03:01.504 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Bus STARTED 2023-10-01T09:03:01.504 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Bus STOPPING 2023-10-01T09:03:01.504 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T09:03:01.504 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Bus STOPPED 2023-10-01T09:03:01.504 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Bus STARTING 2023-10-01T09:03:01.504 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Serving on http://:::9283 2023-10-01T09:03:01.504 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Bus STARTED 2023-10-01T09:03:01.504 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Bus STOPPING 2023-10-01T09:03:01.504 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T09:03:01.504 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Bus STOPPED 2023-10-01T09:03:01.504 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Bus STARTING 2023-10-01T09:03:01.505 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Serving on http://:::9283 2023-10-01T09:03:01.505 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:01 smithi138 conmon[100738]: [01/Oct/2023:09:03:01] ENGINE Bus STARTED 2023-10-01T09:03:01.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 bash[133703]: a1782aa730be0f3d514bd719fe4aaf9cc9393a9be9e8700c4cd138a8086a6644 2023-10-01T09:03:01.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.104Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-01T09:03:01.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.104Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-01T09:03:01.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.104Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi190 (none))" 2023-10-01T09:03:01.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.104Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-01T09:03:01.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.105Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-01T09:03:01.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.106Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-01T09:03:01.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 systemd[1]: Started Ceph prometheus.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:03:01.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.107Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-01T09:03:01.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.108Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-01T09:03:01.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.109Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-01T09:03:01.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.109Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.538µs 2023-10-01T09:03:01.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.109Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-01T09:03:01.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.277Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-10-01T09:03:01.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.277Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-10-01T09:03:01.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.277Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=39.24µs wal_replay_duration=168.005347ms total_replay_duration=168.0577ms 2023-10-01T09:03:01.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.279Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-01T09:03:01.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.279Z caller=main.go:947 level=info msg="TSDB started" 2023-10-01T09:03:01.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.279Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-01T09:03:01.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.293Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.998608ms db_storage=770ns remote_storage=1.334µs web_handler=302ns query_engine=662ns scrape=470.626µs scrape_sd=25.029µs notify=26.158µs notify_sd=8.322µs rules=13.243759ms 2023-10-01T09:03:01.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:03:01 smithi190 conmon[133744]: ts=2023-10-01T09:03:01.293Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-01T09:03:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: cluster 2023-10-01T09:03:00.843046+0000 mgr.y (mgr. 2023-10-01T09:03:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: 14433) 61 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T09:03:01.135285+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T09:03:01.140454+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T09:03:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T09:03:01.140902+0000 mgr.y (mgr.14433) 62 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T09:03:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T09:03:01.141745+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi138:9093"}]: dispatch 2023-10-01T09:03:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T09:03:01.142010+0000 mgr.y (mgr.14433) 63 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi138:9093"}]: dispatch 2023-10-01T09:03:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T09:03:01.148075+0000 mon.a (mon.0) 614 : audit [INF] 2023-10-01T09:03:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T09:03:01.154881+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:03:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T09:03:01.155306+0000 mgr.y ( 2023-10-01T09:03:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: mgr.14433) 64 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:03:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:03:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: 10-01T09:03:01.157677+0000 mon.c 2023-10-01T09:03:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: (mon.2) 66 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T09:03:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:03:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: 10-01T09:03:01.158038+0000 mgr.y ( 2023-10-01T09:03:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: mgr.14433) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T09:03:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T09:03 2023-10-01T09:03:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: :01.164432+0000 mon.a (mon.0) 615 : audit 2023-10-01T09:03:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T09: 2023-10-01T09:03:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: 03:01.170748+0000 mon.c (mon.2) 67 2023-10-01T09:03:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T09:03:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T09:03:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: 09:03:01.171143+0000 mgr.y (mgr.14433 2023-10-01T09:03:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: ) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T09:03:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:03:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: -01T09:03:01.174071+0000 mon.c ( 2023-10-01T09:03:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: mon.2) 68 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi190.front.sepia.ceph.com:3000"}]: dispatch 2023-10-01T09:03:02.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:03:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: 10-01T09:03:01.174415+0000 mgr.y ( 2023-10-01T09:03:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: mgr.14433) 67 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi190.front.sepia.ceph.com:3000"}]: dispatch 2023-10-01T09:03:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:03:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: 10-01T09:03:01.180881+0000 mon.a 2023-10-01T09:03:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: (mon.0) 616 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T09:03:01.185560+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:03:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T09:03:01.281445+0000 mgr.y (mgr.14433) 68 : audit [DBG] from='client.24539 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:03:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T09:03:01.281933+0000 mon.c (mon.2) 70 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-01T09:03:02.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:02 smithi190 conmon[100518]: audit 2023-10-01T09:03:01.282267+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-01T09:03:02.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: cluster 2023-10-01T09:03:00 2023-10-01T09:03:02.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: .843046+0000 mgr.y (mgr.14433) 61 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:02.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10-01T09:03:02.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2023-10-01T09:03:01.135285+0000 mon.a (mon 2023-10-01T09:03:02.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: .0) 613 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:02.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2023-10-01T09:03:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10-01T09:03:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2023-10-01T09:03 2023-10-01T09:03:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: :01.140454+0000 mon.c (mon. 2023-10-01T09:03:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2) 63 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T09:03:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023 2023-10-01T09:03:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: -10-01T09:03:01.140902 2023-10-01T09:03:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: +0000 mgr.y (mgr.14433) 62 : 2023-10-01T09:03:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T09:03:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10-01T09:03:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2023-10-01T09:03 2023-10-01T09:03:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: :01.141745+0000 mon.c (mon. 2023-10-01T09:03:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2) 64 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi138:9093"}]: dispatch 2023-10-01T09:03:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023 2023-10-01T09:03:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: -10-01T09:03:01. 2023-10-01T09:03:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 142010+0000 mgr.y (mgr.14433) 63 : audit [DBG] 2023-10-01T09:03:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi138:9093"}]: dispatch 2023-10-01T09:03:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:03:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 10-01T09:03:01.148075+0000 mon.a ( 2023-10-01T09:03:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: mon.0) 614 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10-01T09:03:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2023-10-01T09:03:01 2023-10-01T09:03:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: .154881+0000 mon.c (mon.2) 2023-10-01T09:03:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 65 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:03:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:03:02.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: -01T09:03:01.155306+0000 mgr.y ( 2023-10-01T09:03:02.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: mgr.14433) 64 : audit [DBG] 2023-10-01T09:03:02.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:03:02.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:03:02.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 10-01T09:03:01.157677+0000 mon.c ( 2023-10-01T09:03:02.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: mon.2) 66 : audit [INF] 2023-10-01T09:03:02.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T09:03:02.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10-01T09:03: 2023-10-01T09:03:02.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 01.158038+0000 mgr.y (mgr.14433) 65 2023-10-01T09:03:02.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T09:03:02.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10-01T09:03:02.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2023-10-01T09:03:01.164432 2023-10-01T09:03:02.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: +0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:02.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2023-10-01T09:03:02.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:03:02.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: cluster 2023-10-01T09:03:00.843046+0000 2023-10-01T09:03:02.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: mgr.y (mgr.14433) 61 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:02.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09:03:01.135285+0000 mon.a (mon.0 2023-10-01T09:03:02.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: ) 613 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:02.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:03:02.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: 01T09:03:01.140454+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T09:03:02.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09:03:01. 2023-10-01T09:03:02.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: 140902+0000 mgr.y (mgr.14433) 62 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T09:03:02.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:03:02.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: 01T09:03:01.141745+0000 mon.c (mon.2) 2023-10-01T09:03:02.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: 64 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi138:9093"}]: dispatch 2023-10-01T09:03:02.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:03:02.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: :03:01.142010+0000 mgr.y (mgr.14433) 63 : audit 2023-10-01T09:03:02.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi138:9093"}]: dispatch 2023-10-01T09:03:02.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09:03:02.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: 2023-10-01T09:03:01.148075+0000 mon.a (mon.0) 614 2023-10-01T09:03:02.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:02.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09:03:01.154881 2023-10-01T09:03:02.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: +0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:03:02.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:03:02.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: 03:01.155306+0000 mgr.y (mgr.14433) 64 : audit 2023-10-01T09:03:02.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:03:02.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09:03:02.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: 2023-10-01T09:03:01.157677+0000 mon.c (mon.2) 66 : audit 2023-10-01T09:03:02.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T09:03:02.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09:03:02.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: 2023-10-01T09:03:01.158038+0000 mgr.y (mgr.14433 2023-10-01T09:03:02.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: ) 65 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T09:03:02.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09:03:01. 2023-10-01T09:03:02.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: 164432+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:02.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T09:03:02.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: T09:03:01.170748+0000 mon.c (mon.2) 2023-10-01T09:03:02.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: 67 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T09:03:02.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09:03:01.171143+0000 mgr.y ( 2023-10-01T09:03:02.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: mgr.14433) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T09:03:02.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09:03:01.174071+0000 2023-10-01T09:03:02.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 03:01.170748+0000 mon.c (mon.2 2023-10-01T09:03:02.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: ) 67 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T09:03:02.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10-01T09:03:02.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2023-10-01T09:03:01. 2023-10-01T09:03:02.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 171143+0000 mgr.y (mgr.14433) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T09:03:02.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2023-10-01T09:03:02.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:03:02.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: :03:01.174071+0000 mon.c (mon.2) 68 : 2023-10-01T09:03:02.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi190.front.sepia.ceph.com:3000"}]: dispatch 2023-10-01T09:03:02.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:03:02.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 01T09:03:01.174415 2023-10-01T09:03:02.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: +0000 mgr.y (mgr.14433) 67 : audit 2023-10-01T09:03:02.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi190.front.sepia.ceph.com:3000"}]: dispatch 2023-10-01T09:03:02.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:03:02.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 01T09:03:01.180881+0000 2023-10-01T09:03:02.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: mon.a (mon.0) 616 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:02.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2023-10-01T09:03:02.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10-01T09:03:02.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2023-10-01T09:03:01.185560+0000 2023-10-01T09:03:02.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: mon.c (mon.2) 69 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:03:02.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10-01T09:03:02.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2023-10-01T09:03: 2023-10-01T09:03:02.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 01.281445+0000 mgr.y (mgr.14433) 68 2023-10-01T09:03:02.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: : audit [DBG] from='client.24539 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:03:02.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:03:02.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 01T09:03:01.281933+0000 mon.c (mon. 2023-10-01T09:03:02.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 2) 70 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-01T09:03:02.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:03:02.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: 09:03:01.282267+0000 mon.a (mon 2023-10-01T09:03:02.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[109959]: .0) 617 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-01T09:03:02.580 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: mon.c (mon.2) 68 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi190.front.sepia.ceph.com:3000"}]: dispatch 2023-10-01T09:03:02.580 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:03:02.580 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: :03:01.174415+0000 mgr.y (mgr.14433) 67 : 2023-10-01T09:03:02.580 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi190.front.sepia.ceph.com:3000"}]: dispatch 2023-10-01T09:03:02.580 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09:03:02.580 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: 2023-10-01T09:03:01.180881+0000 mon.a (mon.0) 2023-10-01T09:03:02.581 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: 616 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:02.581 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09:03:01. 2023-10-01T09:03:02.581 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: 185560+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:03:02.581 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:03:02.581 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: :03:01.281445+0000 mgr.y (mgr.14433) 68 : audit [DBG] 2023-10-01T09:03:02.581 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:03:02.581 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:03:02.581 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: 09:03:01.281933+0000 mon.c (mon.2) 70 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-01T09:03:02.581 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: audit 2023-10-01T09:03:01.282267 2023-10-01T09:03:02.581 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:02 smithi138 conmon[100289]: +0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-01T09:03:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:03 smithi190 conmon[100518]: audit 2023-10-01T09:03:02.182794+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-01T09:03:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:03 smithi190 conmon[100518]: cluster 2023-10-01T09:03:02.182867+0000 mon.a (mon.0) 619 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-01T09:03:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:03 smithi190 conmon[100518]: audit 2023-10-01T09:03:02.188664+0000 mon.c (mon.2) 71 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-01T09:03:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:03 smithi190 conmon[100518]: audit 2023-10-01T09:03:02.189028+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-01T09:03:03.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:03 smithi138 conmon[109959]: audit 2023-10-01T09:03:02.182794+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-01T09:03:03.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:03 smithi138 conmon[109959]: cluster 2023-10-01T09:03:02.182867+0000 mon.a (mon.0) 619 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-01T09:03:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:03 smithi138 conmon[109959]: audit 2023-10-01T09:03:02.188664+0000 mon.c (mon.2) 71 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-01T09:03:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:03 smithi138 conmon[109959]: audit 2023-10-01T09:03:02.189028+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-01T09:03:03.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:03 smithi138 conmon[100289]: audit 2023-10-01T09:03:02.182794+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-01T09:03:03.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:03 smithi138 conmon[100289]: cluster 2023-10-01T09:03:02.182867+0000 mon.a (mon.0) 619 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-01T09:03:03.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:03 smithi138 conmon[100289]: audit 2023-10-01T09:03:02.188664+0000 mon.c (mon.2) 71 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-01T09:03:03.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:03 smithi138 conmon[100289]: audit 2023-10-01T09:03:02.189028+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-01T09:03:03.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:03 smithi138 conmon[100289]: debug 2023-10-01T09:03:03.200+0000 7f41d029b700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-01T09:03:04.005 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-10-01T09:03:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: cluster 2023-10-01T09:03:02.843384+0000 mgr.y (mgr 2023-10-01T09:03:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: .14433) 69 : cluster [DBG] pgmap v47: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023-10-01T09:03:03.187090+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-01T09:03:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: cluster 2023-10-01T09:03:03.187180+0000 mon.a (mon.0) 622 : cluster [DBG] 2023-10-01T09:03:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: osdmap e57: 8 total, 8 up, 8 in 2023-10-01T09:03:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023-10-01T09:03:03.200635+0000 mon.c (mon.2) 2023-10-01T09:03:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 72 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-01T09:03:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:03:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 10-01T09:03:03.201111+0000 mon.a (mon.0) 623 : audit [INF] 2023-10-01T09:03:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-01T09:03:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: cluster 2023- 2023-10-01T09:03:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 10-01T09:03:03.201899+0000 mon.a (mon.0) 624 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-01T09:03:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: cluster 2023-10- 2023-10-01T09:03:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 01T09:03:03.201927+0000 mon.a (mon 2023-10-01T09:03:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: .0) 625 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-01T09:03:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: cluster 2023-10-01 2023-10-01T09:03:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: T09:03:03.213192+0000 mon.a (mon. 2023-10-01T09:03:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 0) 626 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-01T09:03:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:03:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: :03:03.213297+0000 mon.a (mon.0) 2023-10-01T09:03:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 627 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-01T09:03:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: cluster 2023-10-01T09:03:03. 2023-10-01T09:03:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 213390+0000 mon.a (mon.0) 628 : cluster [DBG] 2023-10-01T09:03:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: fsmap foo:0 2023-10-01T09:03:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: cephadm 2023-10-01T09:03:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 2023-10-01T09:03:03.216600+0000 mgr.y (mgr.14433) 2023-10-01T09:03:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 70 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-01T09:03:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:03:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 03:03.223027+0000 mon.a (mon.0) 629 : 2023-10-01T09:03:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023-10-01T09:03:03. 2023-10-01T09:03:04.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 303608+0000 mon.a (mon.0) 630 : audit [INF] 2023-10-01T09:03:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023 2023-10-01T09:03:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: -10-01T09:03:03.784046+0000 mon.a (mon.0) 631 : audit [INF] 2023-10-01T09:03:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023-10-01T09:03:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 2023-10-01T09:03:04.089382+0000 mon.c (mon. 2023-10-01T09:03:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 2) 73 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:03:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 03:04.090188+0000 mon.c (mon.2) 74 : audit 2023-10-01T09:03:04.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: cluster 2023-10-01T09:03:02.843384+0000 mgr.y ( 2023-10-01T09:03:04.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: mgr.14433) 69 : cluster [DBG] pgmap v47: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:04.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10-01T09:03:03.187090+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-01T09:03:04.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: cluster 2023-10-01T09:03:03.187180+0000 mon.a (mon.0) 622 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-01T09:03:04.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10-01T09:03:03.200635+0000 mon.c (mon.2) 72 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-01T09:03:04.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10-01T09:03: 2023-10-01T09:03:04.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 03.201111+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-01T09:03:04.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: cluster 2023-10-01T09:03 2023-10-01T09:03:04.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: :03.201899+0000 mon.a (mon.0) 624 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-01T09:03:04.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: cluster 2023-10-01T09:03:04.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 2023-10-01T09:03:03 2023-10-01T09:03:04.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: .201927+0000 mon.a (mon.0) 625 : cluster 2023-10-01T09:03:04.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-01T09:03:04.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: cluster 2023-10 2023-10-01T09:03:04.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: -01T09:03:03.213192+0000 mon.a (mon 2023-10-01T09:03:04.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: .0) 626 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-01T09:03:04.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10-01T09:03:04.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 2023-10-01T09:03:03 2023-10-01T09:03:04.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: .213297+0000 mon.a (mon.0) 627 : audit [INF] 2023-10-01T09:03:04.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-01T09:03:04.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: cluster 2023-10-01T 2023-10-01T09:03:04.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 09:03:03.213390+0000 mon.a (mon.0) 628 : 2023-10-01T09:03:04.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: cluster [DBG] fsmap foo:0 2023-10-01T09:03:04.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: cephadm 2023 2023-10-01T09:03:04.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: -10-01T09:03:03.216600+0000 mgr.y ( 2023-10-01T09:03:04.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: mgr.14433) 70 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-01T09:03:04.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10-01T09:03:04.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 2023-10-01T09:03:03 2023-10-01T09:03:04.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: .223027+0000 mon.a (mon.0) 629 : audit [INF] 2023-10-01T09:03:04.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10-01T09:03:03. 2023-10-01T09:03:04.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 303608+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 2023-10-01T09:03:04.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10-01T09:03:04.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 2023-10-01T09:03:03.784046+0000 mon.a (mon.0) 2023-10-01T09:03:04.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 631 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10-01T09:03:04.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 2023-10-01T09:03:04.089382 2023-10-01T09:03:04.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: +0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:04.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10-01T09:03:04.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 2023-10-01T09:03:04. 2023-10-01T09:03:04.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 090188+0000 mon.c (mon.2) 74 : audit [INF] 2023-10-01T09:03:04.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:03:04.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10 2023-10-01T09:03:04.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: -01T09:03:04.094866+0000 mon.a (mon 2023-10-01T09:03:04.328 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: .0) 632 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.328 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10-01T09:03:04.328 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 2023-10-01T09:03:04.103186+0000 2023-10-01T09:03:04.328 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: mon.a (mon.0) 633 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.328 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10-01T09:03:04.104606+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.brhzod", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:03:04.328 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10-01T09:03:04.104895+0000 mon.a (mon.0) 634 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.brhzod", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:03:04.328 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10-01T09:03:04.108039+0000 mon.a (mon.0) 635 : audit [INF] 2023-10-01T09:03:04.328 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.brhzod", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-01T09:03:04.328 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:03:04.328 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 03:04.108975+0000 mon.c (mon.2) 2023-10-01T09:03:04.328 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[100289]: 76 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:04.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:03:04.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023-10-01T09:03:04.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 2023-10-01T09:03:04.094866+0000 mon.a (mon.0) 2023-10-01T09:03:04.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 632 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023-10-01T09:03:04.103186 2023-10-01T09:03:04.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: +0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:03:04.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: 01T09:03:04.104606+0000 mon.c (mon.2 2023-10-01T09:03:04.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: ) 75 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.brhzod", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:03:04.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023-10-01T09:03 2023-10-01T09:03:04.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: :04.104895+0000 mon.a (mon.0) 634 : audit 2023-10-01T09:03:04.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.brhzod", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:03:04.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023 2023-10-01T09:03:04.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: -10-01T09:03:04.108039+0000 mon.a (mon.0) 635 : 2023-10-01T09:03:04.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.brhzod", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-01T09:03:04.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:03:04.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: T09:03:04.108975+0000 mon.c (mon.2 2023-10-01T09:03:04.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:04 smithi138 conmon[109959]: ) 76 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: cluster 2023-10-01T09:03:02.843384+0000 mgr.y (mgr.14433) 69 2023-10-01T09:03:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: : cluster [DBG] pgmap v47: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023-10-01T09:03:03.187090+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-01T09:03:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: cluster 2023-10-01T09:03:03.187180+0000 mon.a (mon.0) 622 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-01T09:03:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023 2023-10-01T09:03:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: -10-01T09:03:03.200635+0000 mon.c (mon.2) 72 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-01T09:03:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:03:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 01T09:03:03.201111+0000 2023-10-01T09:03:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: mon.a (mon.0) 623 : audit [INF] 2023-10-01T09:03:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-01T09:03:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: cluster 2023-10-01T09:03:03.201899+0000 2023-10-01T09:03:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: mon.a (mon.0) 624 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-01T09:03:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 2023-10-01T09:03:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: cluster 2023-10-01T09:03:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 2023-10-01T09:03:03. 2023-10-01T09:03:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 201927+0000 mon.a (mon.0) 625 : cluster 2023-10-01T09:03:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-01T09:03:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: cluster 2023-10-01T 2023-10-01T09:03:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 09:03:03.213192+0000 mon.a (mon 2023-10-01T09:03:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: .0) 626 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-01T09:03:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023 2023-10-01T09:03:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: -10-01T09:03:03 2023-10-01T09:03:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: .213297+0000 mon.a (mon.0) 627 : audit 2023-10-01T09:03:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-01T09:03:04.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: cluster 2023-10-01T 2023-10-01T09:03:04.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 09:03:03.213390+0000 mon.a (mon 2023-10-01T09:03:04.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: .0) 628 : cluster [DBG] fsmap foo:0 2023-10-01T09:03:04.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: cephadm 2023 2023-10-01T09:03:04.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: -10-01T09:03:03 2023-10-01T09:03:04.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: .216600+0000 mgr.y (mgr.14433) 70 : cephadm 2023-10-01T09:03:04.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: [INF] Saving service mds.foo spec with placement count:2 2023-10-01T09:03:04.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T09:03:04.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 09:03:03.223027+0000 mon.a (mon 2023-10-01T09:03:04.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: .0) 629 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023 2023-10-01T09:03:04.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: -10-01T09:03:03. 2023-10-01T09:03:04.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 303608+0000 mon.a (mon.0) 630 : audit [INF] 2023-10-01T09:03:04.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:03:04.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: :03:03.784046+0000 mon.a (mon. 2023-10-01T09:03:04.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 0) 631 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023 2023-10-01T09:03:04.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: -10-01T09:03:04.089382 2023-10-01T09:03:04.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: +0000 mon.c (mon.2) 73 : audit [DBG] 2023-10-01T09:03:04.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:04.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023-10-01T09: 2023-10-01T09:03:04.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 03:04.090188+0000 mon.c (mon.2 2023-10-01T09:03:04.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: ) 74 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:03:04.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:03:04.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 01T09:03:04.094866+0000 mon.a 2023-10-01T09:03:04.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: (mon.0) 632 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023-10-01T09:03:04.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 2023-10-01T09:03: 2023-10-01T09:03:04.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 04.103186+0000 mon.a (mon.0) 633 : 2023-10-01T09:03:04.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:04.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T09:03:04.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: T09:03:04.104606+0000 mon.c ( 2023-10-01T09:03:04.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: mon.2) 75 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.brhzod", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:03:04.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023-10-01T09:03:04.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 2023-10-01T09:03: 2023-10-01T09:03:04.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 04.104895+0000 mon.a (mon.0) 2023-10-01T09:03:04.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: 634 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.brhzod", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:03:04.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:03:04.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: -01T09:03:04.108039+0000 2023-10-01T09:03:04.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: mon.a (mon.0) 635 : audit [INF] 2023-10-01T09:03:04.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.brhzod", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-01T09:03:04.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:03:04.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: :03:04.108975+0000 mon.c (mon.2 2023-10-01T09:03:04.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:04 smithi190 conmon[100518]: ) 76 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:05.214 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:05 smithi138 conmon[100289]: cephadm 2023-10-01T09:03:04.109657+0000 mgr.y (mgr.14433 2023-10-01T09:03:05.215 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:05 smithi138 conmon[100289]: ) 71 : cephadm [INF] Deploying daemon mds.foo.smithi138.brhzod on smithi138 2023-10-01T09:03:05.215 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:05 smithi138 conmon[100289]: cluster 2023-10-01T09:03:04.210756+0000 mon.a (mon.0) 636 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-01T09:03:05.498 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:05 smithi138 conmon[109959]: cephadm 2023-10-01T09:03:04.109657+0000 mgr.y (mgr.14433) 2023-10-01T09:03:05.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:05 smithi138 conmon[109959]: 71 : cephadm [INF] Deploying daemon mds.foo.smithi138.brhzod on smithi138 2023-10-01T09:03:05.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:05 smithi138 conmon[109959]: cluster 2023-10-01T09:03:04.210756+0000 mon.a (mon.0) 2023-10-01T09:03:05.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:05 smithi138 conmon[109959]: 636 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-01T09:03:05.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:05 smithi190 conmon[100518]: cephadm 2023-10-01T09:03:04.109657+0000 mgr.y (mgr.14433) 71 : cephadm [INF] Deploying daemon mds.foo.smithi138.brhzod on smithi138 2023-10-01T09:03:05.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:05 smithi190 conmon[100518]: 2023-10-01T09:03:05.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:05 smithi190 conmon[100518]: cluster 2023-10-01T09:03:04.210756+0000 mon.a (mon.0) 636 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-01T09:03:06.521 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:06 smithi138 conmon[109959]: cluster 2023-10-01T09:03:04.843686+0000 mgr.y (mgr.14433) 72 : cluster [DBG] pgmap v51: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:06.522 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:06 smithi138 conmon[109959]: cluster 2023-10-01T09:03:05.215184+0000 mon.a (mon.0) 637 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-01T09:03:06.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:06 smithi138 conmon[100289]: cluster 2023-10-01T09:03:04.843686+0000 mgr.y (mgr.14433) 72 : cluster [DBG] pgmap v51: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:06.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:06 smithi138 conmon[100289]: cluster 2023-10-01T09:03:05.215184+0000 mon.a (mon.0) 637 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-01T09:03:06.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:06 smithi190 conmon[100518]: cluster 2023-10-01T09:03:04.843686+0000 mgr.y (mgr.14433) 72 : 2023-10-01T09:03:06.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:06 smithi190 conmon[100518]: cluster [DBG] pgmap v51: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:06.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:06 smithi190 conmon[100518]: cluster 2023-10-01T09:03:05.215184+0000 mon.a (mon.0) 637 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-01T09:03:07.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:03:06 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:03:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:03:07.319 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:03:07 smithi138 conmon[150478]: level=info ts=2023-10-01T09:03:07.054Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003112136s 2023-10-01T09:03:07.586 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-10-01T09:03:08.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:08 smithi138 conmon[100289]: cluster 2023-10-01T09:03:06.844139+0000 mgr.y (mgr.14433) 2023-10-01T09:03:08.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:08 smithi138 conmon[100289]: 73 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:08.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:08 smithi138 conmon[100289]: audit 2023-10-01T09:03:07.731229+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:08.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:08 smithi138 conmon[100289]: audit 2023-10-01T09:03:07.732864+0000 mon.c (mon.2) 77 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi190.weynjc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:03:08.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:08 smithi138 conmon[100289]: audit 2023-10-01T09:03:07.733158+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi190.weynjc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:03:08.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:08 smithi138 conmon[100289]: audit 2023-10-01T09:03:07.736395+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi190.weynjc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-01T09:03:08.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:08 smithi138 conmon[100289]: audit 2023-10-01T09:03:07.739321+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:08.481 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:08 smithi138 conmon[109959]: cluster 2023-10-01T09:03:06.844139+0000 mgr.y (mgr.14433) 73 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:08.482 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:08 smithi138 conmon[109959]: audit 2023-10-01T09:03:07.731229+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:08.482 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:08 smithi138 conmon[109959]: audit 2023-10-01T09:03:07.732864+0000 mon.c (mon.2) 77 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi190.weynjc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:03:08.482 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:08 smithi138 conmon[109959]: audit 2023-10-01T09:03:07.733158+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi190.weynjc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:03:08.482 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:08 smithi138 conmon[109959]: audit 2023-10-01T09:03:07.736395+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi190.weynjc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-01T09:03:08.482 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:08 smithi138 conmon[109959]: audit 2023-10-01T09:03:07.739321+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:08 smithi190 conmon[100518]: cluster 2023-10-01T09:03:06.844139+0000 mgr.y (mgr.14433 2023-10-01T09:03:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:08 smithi190 conmon[100518]: ) 73 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:08 smithi190 conmon[100518]: audit 2023-10-01T09:03:07.731229+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:08 smithi190 conmon[100518]: audit 2023-10-01T09:03:07.732864+0000 mon.c (mon.2) 77 : audit 2023-10-01T09:03:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:08 smithi190 conmon[100518]: [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi190.weynjc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:03:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:08 smithi190 conmon[100518]: audit 2023-10-01T09:03:07.733158 2023-10-01T09:03:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:08 smithi190 conmon[100518]: +0000 mon.a (mon.0) 639 : audit [INF] 2023-10-01T09:03:08.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:08 smithi190 conmon[100518]: from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi190.weynjc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:03:08.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:08 smithi190 conmon[100518]: audit 2023-10-01T09:03:08.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:08 smithi190 conmon[100518]: 2023-10-01T09:03:07.736395+0000 mon.a (mon.0) 2023-10-01T09:03:08.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:08 smithi190 conmon[100518]: 640 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi190.weynjc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-01T09:03:08.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:08 smithi190 conmon[100518]: audit 2023-10-01T09:03 2023-10-01T09:03:08.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:08 smithi190 conmon[100518]: :07.739321+0000 mon.c (mon.2) 78 : 2023-10-01T09:03:08.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:08 smithi190 conmon[100518]: audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:09.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:09 smithi190 conmon[100518]: cephadm 2023-10-01T09:03:07.739903+0000 mgr.y (mgr.14433) 74 : cephadm [INF] Deploying daemon mds.foo.smithi190.weynjc on smithi190 2023-10-01T09:03:09.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:09 smithi190 conmon[100518]: cluster 2023-10-01T09:03:08.219356+0000 mon.a (mon.0) 641 : cluster [DBG] mds.? [v2:172.21.15.138:6834/416108704,v1:172.21.15.138:6835/416108704] up:boot 2023-10-01T09:03:09.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:09 smithi190 conmon[100518]: cluster 2023-10-01T09:03:08.219396+0000 mon.a (mon.0) 642 : cluster [INF] daemon mds.foo.smithi138.brhzod assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-01T09:03:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:09 smithi190 conmon[100518]: cluster 2023-10-01T09:03:08.219449+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-01T09:03:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:09 smithi190 conmon[100518]: cluster 2023-10-01T09:03:08.219471+0000 mon.a (mon.0) 644 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-01T09:03:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:09 smithi190 conmon[100518]: cluster 2023-10-01T09:03:08.219481+0000 mon.a (mon.0) 645 : cluster [INF] Cluster is now healthy 2023-10-01T09:03:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:09 smithi190 conmon[100518]: cluster 2023-10-01T09:03:08.219792+0000 mon.a (mon.0) 646 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-01T09:03:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:09 smithi190 conmon[100518]: audit 2023-10-01T09:03:08.222671+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.brhzod"}]: dispatch 2023-10-01T09:03:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:09 smithi190 conmon[100518]: cluster 2023- 2023-10-01T09:03:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:09 smithi190 conmon[100518]: 10-01T09:03:08.225839+0000 mon.a (mon.0) 647 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:creating} 2023-10-01T09:03:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:09 smithi190 conmon[100518]: cluster 2023-10-01T09:03:08.234514+0000 mon.a (mon.0) 648 : cluster [INF] daemon mds.foo.smithi138.brhzod is now active in filesystem foo as rank 0 2023-10-01T09:03:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[100289]: cephadm 2023-10-01T09:03:07.739903+0000 mgr.y (mgr.14433) 74 : 2023-10-01T09:03:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[100289]: cephadm [INF] Deploying daemon mds.foo.smithi190.weynjc on smithi190 2023-10-01T09:03:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[100289]: cluster 2023-10-01T09:03:08.219356+0000 mon.a (mon.0) 641 : cluster [DBG] mds.? [v2:172.21.15.138:6834/416108704,v1:172.21.15.138:6835/416108704] up:boot 2023-10-01T09:03:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[100289]: cluster 2023-10-01T09:03:08.219396+0000 mon.a (mon.0) 642 : cluster [INF] daemon mds.foo.smithi138.brhzod assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-01T09:03:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[100289]: cluster 2023-10-01T09:03:08.219449+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-01T09:03:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[100289]: cluster 2023-10-01T09:03:08.219471+0000 mon.a (mon.0) 644 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-01T09:03:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[100289]: cluster 2023-10-01T09:03:08.219481+0000 mon.a (mon.0) 645 : cluster [INF] Cluster is now healthy 2023-10-01T09:03:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[100289]: cluster 2023-10-01T09:03:08.219792+0000 mon.a (mon.0) 646 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-01T09:03:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[100289]: audit 2023-10-01T09:03:08.222671+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.brhzod"}]: dispatch 2023-10-01T09:03:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[100289]: cluster 2023-10-01T09:03:08.225839+0000 mon.a (mon.0) 647 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:creating} 2023-10-01T09:03:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[100289]: cluster 2023-10-01T09:03:08.234514+0000 mon.a (mon.0) 648 : cluster [INF] daemon mds.foo.smithi138.brhzod is now active in filesystem foo as rank 0 2023-10-01T09:03:09.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[109959]: cephadm 2023-10-01T09:03:07.739903+0000 2023-10-01T09:03:09.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[109959]: mgr.y (mgr.14433) 74 : cephadm [INF] Deploying daemon mds.foo.smithi190.weynjc on smithi190 2023-10-01T09:03:09.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[109959]: cluster 2023-10-01T09:03:08.219356+0000 mon.a (mon.0) 641 : cluster [DBG] mds.? [v2:172.21.15.138:6834/416108704,v1:172.21.15.138:6835/416108704] up:boot 2023-10-01T09:03:09.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[109959]: cluster 2023-10-01T09:03:08.219396+0000 mon.a (mon.0) 642 : cluster [INF] daemon mds.foo.smithi138.brhzod assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-01T09:03:09.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[109959]: cluster 2023-10-01T09:03:08.219449+0000 mon.a (mon.0) 643 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-01T09:03:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[109959]: cluster 2023-10-01T09:03:08.219471+0000 mon.a (mon.0) 644 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-01T09:03:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[109959]: cluster 2023-10-01T09:03:08.219481+0000 mon.a (mon.0) 645 : cluster [INF] Cluster is now healthy 2023-10-01T09:03:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[109959]: cluster 2023-10-01T09:03:08.219792+0000 mon.a (mon.0) 646 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-01T09:03:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[109959]: audit 2023-10-01T09:03:08.222671+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.brhzod"}]: dispatch 2023-10-01T09:03:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[109959]: cluster 2023-10-01T09:03:08.225839+0000 mon.a (mon.0) 647 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:creating} 2023-10-01T09:03:09.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:09 smithi138 conmon[109959]: cluster 2023-10-01T09:03:08.234514+0000 mon.a (mon.0) 648 : cluster [INF] daemon mds.foo.smithi138.brhzod is now active in filesystem foo as rank 0 2023-10-01T09:03:10.233 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:10 smithi190 conmon[100518]: cluster 2023-10-01T09:03:08.844504+0000 mgr.y (mgr.14433 2023-10-01T09:03:10.234 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:10 smithi190 conmon[100518]: ) 75 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:10.234 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:10 smithi190 conmon[100518]: cluster 2023-10-01T09:03:09.231084+0000 mon.a (mon.0) 649 : cluster [DBG] mds.? [v2:172.21.15.138:6834/416108704,v1:172.21.15.138:6835/416108704] up:active 2023-10-01T09:03:10.234 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:10 smithi190 conmon[100518]: cluster 2023-10-01T09:03:09.231136+0000 mon.a (mon.0) 650 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 2023-10-01T09:03:10.234 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:10 smithi190 conmon[100518]: audit 2023-10-01T09:03:10.014704+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:10.234 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:10 smithi190 conmon[100518]: audit 2023-10-01T09:03:10.016250+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:03:10.234 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:10 smithi190 conmon[100518]: audit 2023-10-01T09:03:10.029187+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:10.259 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-01T09:03:10.267 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-01T09:03:10.267 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-01T09:03:10.267 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-01T09:03:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:10 smithi138 conmon[109959]: cluster 2023-10-01T09:03:08.844504+0000 mgr.y (mgr.14433) 75 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:10 smithi138 conmon[109959]: cluster 2023-10-01T09:03:09.231084+0000 mon.a (mon.0) 649 : cluster [DBG] mds.? [v2:172.21.15.138:6834/416108704,v1:172.21.15.138:6835/416108704] up:active 2023-10-01T09:03:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:10 smithi138 conmon[109959]: cluster 2023-10-01T09:03:09.231136+0000 mon.a (mon.0) 650 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 2023-10-01T09:03:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:10 smithi138 conmon[109959]: audit 2023-10-01T09:03:10.014704+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:10 smithi138 conmon[109959]: audit 2023-10-01T09:03:10.016250+0000 mon.c (mon.2) 80 2023-10-01T09:03:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:10 smithi138 conmon[109959]: : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:03:10.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:10 smithi138 conmon[109959]: audit 2023-10-01T09:03:10.029187+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:10.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:10 smithi138 conmon[100289]: cluster 2023-10-01T09:03:08.844504+0000 mgr.y (mgr.14433) 75 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:10.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:10 smithi138 conmon[100289]: cluster 2023-10-01T09:03:09.231084+0000 mon.a (mon.0) 649 : cluster [DBG] mds.? [v2:172.21.15.138:6834/416108704,v1:172.21.15.138:6835/416108704] up:active 2023-10-01T09:03:10.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:10 smithi138 conmon[100289]: cluster 2023-10-01T09:03:09.231136+0000 mon.a (mon.0) 650 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 2023-10-01T09:03:10.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:10 smithi138 conmon[100289]: audit 2023-10-01T09:03:10.014704+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:10.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:10 smithi138 conmon[100289]: audit 2023-10-01T09:03:10.016250+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:03:10.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:10 smithi138 conmon[100289]: audit 2023-10-01T09:03:10.029187+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:10.920 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:11.488 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:11 smithi138 conmon[109959]: cluster 2023-10-01T09:03:10.238331+0000 mon.a ( 2023-10-01T09:03:11.488 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:11 smithi138 conmon[109959]: mon.0) 653 : cluster [DBG] mds.? [v2:172.21.15.190:6832/4158685175,v1:172.21.15.190:6833/4158685175] up:boot 2023-10-01T09:03:11.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:11 smithi138 conmon[109959]: cluster 2023-10-01T09:03:10.238975+0000 mon.a (mon.0) 654 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:03:11.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:11 smithi138 conmon[109959]: audit 2023-10-01T09:03:10.239083+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi190.weynjc"}]: dispatch 2023-10-01T09:03:11.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:11 smithi138 conmon[109959]: cluster 2023-10-01T09:03:10.243890+0000 mon.a (mon.0) 655 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:03:11.489 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:11 smithi138 conmon[100289]: cluster 2023-10-01T09:03:10 2023-10-01T09:03:11.489 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:11 smithi138 conmon[100289]: .238331+0000 mon.a (mon.0) 653 : cluster [DBG] mds.? [v2:172.21.15.190:6832/4158685175,v1:172.21.15.190:6833/4158685175] up:boot 2023-10-01T09:03:11.489 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:11 smithi138 conmon[100289]: cluster 2023-10-01T09:03:10.238975+0000 mon.a (mon.0) 654 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:03:11.489 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:11 smithi138 conmon[100289]: audit 2023-10-01T09:03:10.239083+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi190.weynjc"}]: dispatch 2023-10-01T09:03:11.489 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:11 smithi138 conmon[100289]: cluster 2023-10-01T09:03:10.243890+0000 mon.a (mon.0) 655 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:03:11.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:11 smithi190 conmon[100518]: cluster 2023-10-01T09:03:10.238331+0000 mon.a (mon.0 2023-10-01T09:03:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:11 smithi190 conmon[100518]: ) 653 : cluster [DBG] mds.? [v2:172.21.15.190:6832/4158685175,v1:172.21.15.190:6833/4158685175] up:boot 2023-10-01T09:03:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:11 smithi190 conmon[100518]: cluster 2023-10-01T09:03:10.238975+0000 mon.a (mon.0) 654 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:03:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:11 smithi190 conmon[100518]: audit 2023-10-01T09:03:10.239083+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi190.weynjc"}]: dispatch 2023-10-01T09:03:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:11 smithi190 conmon[100518]: cluster 2023-10-01T09:03:10.243890+0000 mon.a (mon.0) 655 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:03:12.678 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:03:12.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:12 smithi190 conmon[100518]: cluster 2023-10-01T09:03:10.844933+0000 mgr.y (mgr.14433) 76 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:12.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:12 smithi138 conmon[109959]: cluster 2023-10-01T09:03:10.844933+0000 mgr.y (mgr.14433) 76 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:12.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:12 smithi138 conmon[100289]: cluster 2023-10-01T09:03:10.844933+0000 mgr.y (mgr.14433) 76 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:13.401 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":18,"active_gid":14433,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":974662983},{"type":"v1","addr":"172.21.15.138:6801","nonce":974662983}]},"active_addr":"172.21.15.138:6801/974662983","active_change":"2023-10-01T09:01:44.817544+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24322,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.138:0","nonce":1836785572}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.138:0","nonce":3533950111}]}]} 2023-10-01T09:03:13.405 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-01T09:03:13.405 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-01T09:03:13.405 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T09:03:13.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:13 smithi138 conmon[100289]: audit 2023-10-01T09:03:12.583752+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:13.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:13 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:03:13.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:13 smithi138 conmon[100289]: 03:12.673741+0000 mon.a (mon.0) 657 : audit [DBG] from='client.? 172.21.15.138:0/3136491966' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T09:03:13.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:13 smithi138 conmon[109959]: audit 2023-10-01T09:03:12.583752+0000 mon.a (mon.0) 656 2023-10-01T09:03:13.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:13 smithi138 conmon[109959]: : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:13.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:13 smithi138 conmon[109959]: audit 2023-10-01T09:03:12.673741+0000 mon.a (mon.0) 2023-10-01T09:03:13.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:13 smithi138 conmon[109959]: 657 : audit [DBG] from='client.? 172.21.15.138:0/3136491966' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T09:03:13.978 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:14.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:13 smithi190 conmon[100518]: audit 2023-10-01T09:03:12.583752+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:14.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:13 smithi190 conmon[100518]: audit 2023-10-01T09:03:12.673741+0000 mon.a (mon.0) 657 : audit [DBG] from='client.? 172.21.15.138:0/3136491966' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T09:03:15.054 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:14 smithi138 conmon[109959]: cluster 2023-10-01T09:03:12.845296+0000 mgr.y (mgr.14433 2023-10-01T09:03:15.054 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:14 smithi138 conmon[109959]: ) 77 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:15.054 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:14 smithi138 conmon[109959]: audit 2023-10-01T09:03:13.897113+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:15.054 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:14 smithi138 conmon[109959]: audit 2023-10-01T09:03:13.898331+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:15.054 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:14 smithi138 conmon[109959]: audit 2023-10-01T09:03:13.898972+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:03:15.054 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:14 smithi138 conmon[109959]: audit 2023-10-01T09:03:13.903559+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:15.055 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:14 smithi138 conmon[109959]: audit 2023-10-01T09:03:13.908355+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:03:15.055 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:14 smithi138 conmon[100289]: cluster 2023-10-01T09:03:12.845296+0000 mgr.y (mgr.14433) 77 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:15.055 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:14 smithi138 conmon[100289]: audit 2023-10-01T09:03:13.897113+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:15.055 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:14 smithi138 conmon[100289]: audit 2023-10-01T09:03:13.898331+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:15.055 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:14 smithi138 conmon[100289]: audit 2023-10-01T09:03:13.898972+0000 mon.c (mon.2) 83 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:03:15.055 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:14 smithi138 conmon[100289]: audit 2023-10-01T09:03:13.903559+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:15.055 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:14 smithi138 conmon[100289]: audit 2023-10-01T09:03:13.908355+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:03:15.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:14 smithi190 conmon[100518]: cluster 2023-10-01T09:03:12.845296+0000 mgr.y (mgr.14433) 77 : cluster 2023-10-01T09:03:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:14 smithi190 conmon[100518]: [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:14 smithi190 conmon[100518]: audit 2023-10-01T09:03:13.897113+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:14 smithi190 conmon[100518]: audit 2023-10-01T09:03:13.898331+0000 mon.c (mon 2023-10-01T09:03:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:14 smithi190 conmon[100518]: .2) 82 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:14 smithi190 conmon[100518]: audit 2023-10-01T09:03:13.898972+0000 mon.c (mon.2) 83 : audit [INF] 2023-10-01T09:03:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:14 smithi190 conmon[100518]: from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:03:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:14 smithi190 conmon[100518]: audit 2023-10-01T09:03:13.903559+0000 2023-10-01T09:03:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:14 smithi190 conmon[100518]: mon.a (mon.0) 659 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:14 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:03:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:14 smithi190 conmon[100518]: 10-01T09:03:13.908355+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:03:15.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:15 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:03:15] "GET /metrics HTTP/1.1" 200 191322 "" "Prometheus/2.33.4" 2023-10-01T09:03:15.653 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:03:15.653 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":60,"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","created":"2023-10-01T08:56:55.039846+0000","modified":"2023-10-01T09:03:05.207950+0000","last_up_change":"2023-10-01T09:01:30.467448+0000","last_in_change":"2023-10-01T09:01:14.134268+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-01T08:59:41.882085+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T09:02:05.272330+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-01T09:03:01.537913+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-01T09:03:02.452550+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"12f71e77-6099-48ee-a89e-e931e000e9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6803","nonce":635244462}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6805","nonce":635244462}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6809","nonce":635244462}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6807","nonce":635244462}]},"public_addr":"172.21.15.138:6803/635244462","cluster_addr":"172.21.15.138:6805/635244462","heartbeat_back_addr":"172.21.15.138:6809/635244462","heartbeat_front_addr":"172.21.15.138:6807/635244462","state":["exists","up"]},{"osd":1,"uuid":"525b5e81-4fa5-4b10-9147-83fcb549ca30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6811","nonce":989382814}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6813","nonce":989382814}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6817","nonce":989382814}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6815","nonce":989382814}]},"public_addr":"172.21.15.138:6811/989382814","cluster_addr":"172.21.15.138:6813/989382814","heartbeat_back_addr":"172.21.15.138:6817/989382814","heartbeat_front_addr":"172.21.15.138:6815/989382814","state":["exists","up"]},{"osd":2,"uuid":"97443e05-2513-4832-bb54-f65493df5967","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6819","nonce":3174817573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6821","nonce":3174817573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6825","nonce":3174817573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6823","nonce":3174817573}]},"public_addr":"172.21.15.138:6819/3174817573","cluster_addr":"172.21.15.138:6821/3174817573","heartbeat_back_addr":"172.21.15.138:6825/3174817573","heartbeat_front_addr":"172.21.15.138:6823/3174817573","state":["exists","up"]},{"osd":3,"uuid":"ac51c165-e007-4390-b95b-26eef984d637","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6827","nonce":1517480090}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6829","nonce":1517480090}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6832","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6833","nonce":1517480090}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6831","nonce":1517480090}]},"public_addr":"172.21.15.138:6827/1517480090","cluster_addr":"172.21.15.138:6829/1517480090","heartbeat_back_addr":"172.21.15.138:6833/1517480090","heartbeat_front_addr":"172.21.15.138:6831/1517480090","state":["exists","up"]},{"osd":4,"uuid":"4bef630c-bff4-41a3-a932-de0501541f78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6801","nonce":743095145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6803","nonce":743095145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6807","nonce":743095145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6805","nonce":743095145}]},"public_addr":"172.21.15.190:6801/743095145","cluster_addr":"172.21.15.190:6803/743095145","heartbeat_back_addr":"172.21.15.190:6807/743095145","heartbeat_front_addr":"172.21.15.190:6805/743095145","state":["exists","up"]},{"osd":5,"uuid":"ac6e16ed-681d-4e9d-87a3-d20d50e01437","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6808","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6809","nonce":4080051847}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6810","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6811","nonce":4080051847}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6814","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6815","nonce":4080051847}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6812","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6813","nonce":4080051847}]},"public_addr":"172.21.15.190:6809/4080051847","cluster_addr":"172.21.15.190:6811/4080051847","heartbeat_back_addr":"172.21.15.190:6815/4080051847","heartbeat_front_addr":"172.21.15.190:6813/4080051847","state":["exists","up"]},{"osd":6,"uuid":"7a6f65a5-16dc-45de-854d-751d9151e85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6816","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6817","nonce":797186189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6818","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6819","nonce":797186189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6822","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6823","nonce":797186189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6820","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6821","nonce":797186189}]},"public_addr":"172.21.15.190:6817/797186189","cluster_addr":"172.21.15.190:6819/797186189","heartbeat_back_addr":"172.21.15.190:6823/797186189","heartbeat_front_addr":"172.21.15.190:6821/797186189","state":["exists","up"]},{"osd":7,"uuid":"09a5ad5d-d58f-4fcf-8b09-10f395bd01f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6824","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6825","nonce":185238034}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6826","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6827","nonce":185238034}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6830","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6831","nonce":185238034}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6828","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6829","nonce":185238034}]},"public_addr":"172.21.15.190:6825/185238034","cluster_addr":"172.21.15.190:6827/185238034","heartbeat_back_addr":"172.21.15.190:6831/185238034","heartbeat_front_addr":"172.21.15.190:6829/185238034","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:58:54.313619+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:16.837452+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:39.610865+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:03.127916+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:23.838460+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:45.582295+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:07.078976+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:29.080850+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.138:0/2334114390":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6801/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6800/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:0/3940341494":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2575047401":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6801/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.138:6800/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2054656381":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2018177063":"2023-10-02T08:57:18.125987+0000","172.21.15.138:0/3675584653":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6800/3165627846":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6801/3165627846":"2023-10-02T08:57:18.125987+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T09:03:16.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:16 smithi190 conmon[100518]: cluster 2023-10-01T09:03:14.845589+0000 mgr.y (mgr.14433) 2023-10-01T09:03:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:16 smithi190 conmon[100518]: 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:16 smithi190 conmon[100518]: audit 2023-10-01T09:03:15.037022+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:16 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:03:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:16 smithi190 conmon[100518]: 01T09:03:15.652420+0000 mon.a (mon.0) 661 : audit [DBG] from='client.? 172.21.15.138:0/937187015' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:03:16.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:16 smithi138 conmon[100289]: cluster 2023-10-01T09:03:14.845589+0000 2023-10-01T09:03:16.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:16 smithi138 conmon[100289]: mgr.y (mgr.14433) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:16.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:16 smithi138 conmon[100289]: audit 2023-10-01T09:03:15.037022+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:16.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:16 smithi138 conmon[100289]: audit 2023-10-01T09:03:15.652420+0000 mon.a (mon.0) 661 : audit [DBG] from='client.? 172.21.15.138:0/937187015' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:03:16.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:16 smithi138 conmon[109959]: cluster 2023-10-01T09:03:14.845589+0000 2023-10-01T09:03:16.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:16 smithi138 conmon[109959]: mgr.y (mgr.14433) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:16.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:16 smithi138 conmon[109959]: audit 2023-10-01T09:03:15.037022+0000 2023-10-01T09:03:16.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:16 smithi138 conmon[109959]: mon.a (mon.0) 660 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:16.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:16 smithi138 conmon[109959]: audit 2023-10-01T09:03:16.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:16 smithi138 conmon[109959]: 2023-10-01T09:03: 2023-10-01T09:03:16.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:16 smithi138 conmon[109959]: 15.652420+0000 mon.a (mon.0) 661 : audit 2023-10-01T09:03:16.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:16 smithi138 conmon[109959]: [DBG] from='client.? 172.21.15.138:0/937187015' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:03:16.380 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-01T09:03:16.380 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T09:03:16.735 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:17.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:03:16 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:03:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:03:17.544 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:17 smithi138 conmon[100289]: audit 2023-10-01T09:03:16.543706+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:17.545 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:17 smithi138 conmon[100289]: audit 2023-10-01T09:03:16.545851+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:17.545 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:17 smithi138 conmon[100289]: audit 2023-10-01T09:03:16.546612+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:03:17.545 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:17 smithi138 conmon[100289]: audit 2023-10-01T09:03:16.551434+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:17.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:17 smithi190 conmon[100518]: audit 2023-10-01T09:03:16.543706+0000 mon.a (mon.0) 662 : audit 2023-10-01T09:03:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:17 smithi190 conmon[100518]: [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:17 smithi190 conmon[100518]: audit 2023-10-01T09:03:16.545851+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:17 smithi190 conmon[100518]: audit 2023-10-01T09:03:16.546612+0000 mon.c (mon.2 2023-10-01T09:03:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:17 smithi190 conmon[100518]: ) 86 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:03:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:17 smithi190 conmon[100518]: audit 2023-10-01T09:03:16.551434+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:17 smithi138 conmon[109959]: audit 2023-10-01T09:03:16.543706+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:17 smithi138 conmon[109959]: audit 2023-10-01T09:03:16.545851+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:03:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:17 smithi138 conmon[109959]: audit 2023-10-01T09:03:16.546612+0000 mon.c (mon.2) 86 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:03:17.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:17 smithi138 conmon[109959]: audit 2023-10-01T09:03:16.551434+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:03:18.348 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:03:18.348 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":60,"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","created":"2023-10-01T08:56:55.039846+0000","modified":"2023-10-01T09:03:05.207950+0000","last_up_change":"2023-10-01T09:01:30.467448+0000","last_in_change":"2023-10-01T09:01:14.134268+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-01T08:59:41.882085+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T09:02:05.272330+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-01T09:03:01.537913+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-01T09:03:02.452550+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"12f71e77-6099-48ee-a89e-e931e000e9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6803","nonce":635244462}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6805","nonce":635244462}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6809","nonce":635244462}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":635244462},{"type":"v1","addr":"172.21.15.138:6807","nonce":635244462}]},"public_addr":"172.21.15.138:6803/635244462","cluster_addr":"172.21.15.138:6805/635244462","heartbeat_back_addr":"172.21.15.138:6809/635244462","heartbeat_front_addr":"172.21.15.138:6807/635244462","state":["exists","up"]},{"osd":1,"uuid":"525b5e81-4fa5-4b10-9147-83fcb549ca30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6811","nonce":989382814}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6813","nonce":989382814}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6817","nonce":989382814}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":989382814},{"type":"v1","addr":"172.21.15.138:6815","nonce":989382814}]},"public_addr":"172.21.15.138:6811/989382814","cluster_addr":"172.21.15.138:6813/989382814","heartbeat_back_addr":"172.21.15.138:6817/989382814","heartbeat_front_addr":"172.21.15.138:6815/989382814","state":["exists","up"]},{"osd":2,"uuid":"97443e05-2513-4832-bb54-f65493df5967","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6819","nonce":3174817573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6821","nonce":3174817573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6825","nonce":3174817573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":3174817573},{"type":"v1","addr":"172.21.15.138:6823","nonce":3174817573}]},"public_addr":"172.21.15.138:6819/3174817573","cluster_addr":"172.21.15.138:6821/3174817573","heartbeat_back_addr":"172.21.15.138:6825/3174817573","heartbeat_front_addr":"172.21.15.138:6823/3174817573","state":["exists","up"]},{"osd":3,"uuid":"ac51c165-e007-4390-b95b-26eef984d637","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6827","nonce":1517480090}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6829","nonce":1517480090}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6832","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6833","nonce":1517480090}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":1517480090},{"type":"v1","addr":"172.21.15.138:6831","nonce":1517480090}]},"public_addr":"172.21.15.138:6827/1517480090","cluster_addr":"172.21.15.138:6829/1517480090","heartbeat_back_addr":"172.21.15.138:6833/1517480090","heartbeat_front_addr":"172.21.15.138:6831/1517480090","state":["exists","up"]},{"osd":4,"uuid":"4bef630c-bff4-41a3-a932-de0501541f78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6801","nonce":743095145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6803","nonce":743095145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6807","nonce":743095145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6805","nonce":743095145}]},"public_addr":"172.21.15.190:6801/743095145","cluster_addr":"172.21.15.190:6803/743095145","heartbeat_back_addr":"172.21.15.190:6807/743095145","heartbeat_front_addr":"172.21.15.190:6805/743095145","state":["exists","up"]},{"osd":5,"uuid":"ac6e16ed-681d-4e9d-87a3-d20d50e01437","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6808","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6809","nonce":4080051847}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6810","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6811","nonce":4080051847}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6814","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6815","nonce":4080051847}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6812","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6813","nonce":4080051847}]},"public_addr":"172.21.15.190:6809/4080051847","cluster_addr":"172.21.15.190:6811/4080051847","heartbeat_back_addr":"172.21.15.190:6815/4080051847","heartbeat_front_addr":"172.21.15.190:6813/4080051847","state":["exists","up"]},{"osd":6,"uuid":"7a6f65a5-16dc-45de-854d-751d9151e85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6816","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6817","nonce":797186189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6818","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6819","nonce":797186189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6822","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6823","nonce":797186189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6820","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6821","nonce":797186189}]},"public_addr":"172.21.15.190:6817/797186189","cluster_addr":"172.21.15.190:6819/797186189","heartbeat_back_addr":"172.21.15.190:6823/797186189","heartbeat_front_addr":"172.21.15.190:6821/797186189","state":["exists","up"]},{"osd":7,"uuid":"09a5ad5d-d58f-4fcf-8b09-10f395bd01f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6824","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6825","nonce":185238034}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6826","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6827","nonce":185238034}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6830","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6831","nonce":185238034}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6828","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6829","nonce":185238034}]},"public_addr":"172.21.15.190:6825/185238034","cluster_addr":"172.21.15.190:6827/185238034","heartbeat_back_addr":"172.21.15.190:6831/185238034","heartbeat_front_addr":"172.21.15.190:6829/185238034","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:58:54.313619+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:16.837452+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:39.610865+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:03.127916+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:23.838460+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:45.582295+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:07.078976+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:29.080850+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.138:0/2334114390":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6801/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6800/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:0/3940341494":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2575047401":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6801/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.138:6800/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2054656381":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/2018177063":"2023-10-02T08:57:18.125987+0000","172.21.15.138:0/3675584653":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6800/3165627846":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6801/3165627846":"2023-10-02T08:57:18.125987+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T09:03:18.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:18 smithi190 conmon[100518]: cluster 2023-10-01T09:03:16.846222+0000 mgr.y (mgr.14433) 79 : cluster [DBG] 2023-10-01T09:03:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:18 smithi190 conmon[100518]: pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:18 smithi190 conmon[100518]: audit 2023-10-01T09:03:18.347360+0000 mon.a (mon.0) 664 : audit [DBG] from='client.? 172.21.15.138:0/184038737' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:03:18.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:18 smithi138 conmon[100289]: cluster 2023-10-01T09:03:16.846222+0000 mgr.y (mgr.14433) 79 : cluster 2023-10-01T09:03:18.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:18 smithi138 conmon[100289]: [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:18.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:18 smithi138 conmon[100289]: audit 2023-10-01T09:03:18.347360+0000 mon.a (mon.0) 664 : audit [DBG] from='client.? 172.21.15.138:0/184038737' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:03:18.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:18 smithi138 conmon[109959]: cluster 2023-10-01T09:03:16 2023-10-01T09:03:18.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:18 smithi138 conmon[109959]: .846222+0000 mgr.y (mgr.14433) 79 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:18.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:18 smithi138 conmon[109959]: audit 2023-10-01T09:03:18.347360+0000 mon.a (mon.0) 664 : audit [DBG] from='client.? 172.21.15.138:0/184038737' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:03:19.056 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-01T09:03:19.056 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-01T09:03:19.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-01T09:03:19.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-01T09:03:19.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-01T09:03:19.058 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-01T09:03:19.058 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-01T09:03:19.058 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-01T09:03:19.928 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:20.182 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:20.416 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:20.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:20 smithi138 conmon[109959]: cluster 2023-10-01T09:03:18.846527 2023-10-01T09:03:20.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:20 smithi138 conmon[109959]: +0000 mgr.y (mgr.14433) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:20.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:20 smithi138 conmon[100289]: cluster 2023-10-01T09:03:18.846527+0000 mgr.y (mgr.14433) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:21.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:20 smithi190 conmon[100518]: cluster 2023-10-01T09:03:18.846527+0000 mgr.y (mgr.14433) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:21.080 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:21.152 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:21.153 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:21.601 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:21.809 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:22.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:22 smithi138 conmon[100289]: cluster 2023-10-01T09:03:20 2023-10-01T09:03:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:22 smithi138 conmon[100289]: .846963+0000 mgr.y (mgr.14433) 81 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:22 smithi138 conmon[109959]: cluster 2023-10-01T09:03:20.846963+0000 mgr.y 2023-10-01T09:03:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:22 smithi138 conmon[109959]: (mgr.14433) 81 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:23.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:22 smithi190 conmon[100518]: cluster 2023-10-01T09:03:20.846963+0000 mgr.y (mgr.14433) 81 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:24.711 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:24 smithi138 conmon[100289]: cluster 2023-10-01T09:03:22.847358+0000 mgr.y (mgr.14433) 82 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:24.711 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:24 smithi138 conmon[109959]: cluster 2023-10-01T09:03:22.847358+0000 mgr.y (mgr.14433) 82 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:25.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:24 smithi190 conmon[100518]: cluster 2023-10-01T09:03:22.847358+0000 mgr.y (mgr.14433) 82 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:25.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:25 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:03:25] "GET /metrics HTTP/1.1" 200 191322 "" "Prometheus/2.33.4" 2023-10-01T09:03:25.820 INFO:teuthology.orchestra.run.smithi138.stdout:206158430233 2023-10-01T09:03:25.820 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-01T09:03:26.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:26 smithi138 conmon[100289]: cluster 2023-10-01T09:03:24.847931+0000 mgr.y (mgr.14433) 83 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:26 smithi138 conmon[109959]: cluster 2023-10-01T09:03:24.847931+0000 mgr.y (mgr 2023-10-01T09:03:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:26 smithi138 conmon[109959]: .14433) 83 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:26.843 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:26 smithi190 conmon[100518]: cluster 2023-10-01T09:03:24.847931+0000 mgr.y (mgr.14433) 83 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:26.843 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:26 smithi190 conmon[100518]: 2023-10-01T09:03:27.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:03:26 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:03:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:03:28.894 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:28 smithi138 conmon[100289]: cluster 2023-10-01T09:03:26.848459+0000 mgr.y (mgr.14433) 84 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:28.895 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:28 smithi138 conmon[109959]: cluster 2023-10-01T09:03:26.848459+0000 mgr.y (mgr. 2023-10-01T09:03:28.895 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:28 smithi138 conmon[109959]: 14433) 84 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:28 smithi190 conmon[100518]: cluster 2023-10-01T09:03:26.848459+0000 mgr.y (mgr.14433) 84 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T09:03:30.111 INFO:teuthology.orchestra.run.smithi138.stdout:154618822692 2023-10-01T09:03:30.111 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-01T09:03:30.610 INFO:teuthology.orchestra.run.smithi138.stdout:107374182444 2023-10-01T09:03:30.611 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-01T09:03:30.623 INFO:teuthology.orchestra.run.smithi138.stdout:128849018919 2023-10-01T09:03:30.624 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-01T09:03:30.936 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:30 smithi138 conmon[109959]: cluster 2023-10-01T09:03:28.848895+0000 mgr.y (mgr.14433) 85 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:30.937 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:30 smithi138 conmon[100289]: cluster 2023-10-01T09:03:28.848895+0000 mgr.y (mgr.14433) 85 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:31.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:30 smithi190 conmon[100518]: cluster 2023-10-01T09:03:28.848895+0000 mgr.y (mgr.14433) 85 : cluster [DBG] 2023-10-01T09:03:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:30 smithi190 conmon[100518]: pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:31.250 INFO:teuthology.orchestra.run.smithi138.stdout:77309411376 2023-10-01T09:03:31.251 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-01T09:03:31.654 INFO:teuthology.orchestra.run.smithi138.stdout:180388626463 2023-10-01T09:03:31.655 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-01T09:03:31.682 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:31.713 INFO:teuthology.orchestra.run.smithi138.stdout:34359738426 2023-10-01T09:03:31.713 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-01T09:03:32.352 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:32.385 INFO:teuthology.orchestra.run.smithi138.stdout:55834574902 2023-10-01T09:03:32.386 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-01T09:03:32.572 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:32.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:32 smithi138 conmon[109959]: cluster 2023-10-01T09:03:30 2023-10-01T09:03:32.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:32 smithi138 conmon[109959]: .849385+0000 mgr.y (mgr.14433) 86 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:32.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:32 smithi138 conmon[100289]: cluster 2023-10-01T09:03:30. 2023-10-01T09:03:32.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:32 smithi138 conmon[100289]: 849385+0000 mgr.y (mgr.14433) 86 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:33.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:32 smithi190 conmon[100518]: cluster 2023-10-01T09:03:30.849385+0000 mgr.y (mgr.14433) 86 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:32 smithi190 conmon[100518]: 2023-10-01T09:03:33.213 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:34.830 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:35.012 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:34 smithi138 conmon[100289]: cluster 2023-10-01T09:03:32.849740+0000 mgr.y (mgr.14433) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:35.012 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:34 smithi138 conmon[109959]: cluster 2023-10-01T09:03:32.849740+0000 mgr.y (mgr.14433) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:35.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:34 smithi190 conmon[100518]: cluster 2023-10-01T09:03:32.849740+0000 mgr.y (mgr.14433) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:35.096 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:35.263 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:35 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:03:35] "GET /metrics HTTP/1.1" 200 191330 "" "Prometheus/2.33.4" 2023-10-01T09:03:35.436 INFO:teuthology.orchestra.run.smithi138.stdout:77309411377 2023-10-01T09:03:35.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:35 smithi138 conmon[109959]: audit 2023-10-01T09:03:35.436629+0000 mon.a (mon. 2023-10-01T09:03:35.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:35 smithi138 conmon[109959]: 0) 665 : audit [DBG] from='client.? 172.21.15.138:0/2671439038' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T09:03:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:35 smithi138 conmon[100289]: audit 2023-10-01T09:03:35.436629+0000 mon.a (mon.0) 665 : audit 2023-10-01T09:03:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:35 smithi138 conmon[100289]: [DBG] from='client.? 172.21.15.138:0/2671439038' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T09:03:35.966 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:36.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:35 smithi190 conmon[100518]: audit 2023-10-01T09:03:35.436629+0000 mon.a (mon.0) 665 : audit [DBG] from='client.? 172.21.15.138:0/2671439038' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T09:03:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:35 smithi190 conmon[100518]: 2023-10-01T09:03:36.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:36 smithi138 conmon[100289]: cluster 2023-10-01T09:03:34.850140+0000 mgr.y (mgr.14433) 88 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:36 smithi138 conmon[109959]: cluster 2023-10-01T09:03:34.850140+0000 2023-10-01T09:03:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:36 smithi138 conmon[109959]: mgr.y (mgr.14433) 88 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:37.015 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411376 got 77309411377 for osd.2 2023-10-01T09:03:37.016 DEBUG:teuthology.parallel:result is None 2023-10-01T09:03:37.056 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:03:36 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:03:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:03:37.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:36 smithi190 conmon[100518]: cluster 2023-10-01T09:03:34.850140+0000 mgr.y (mgr.14433) 88 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:37.111 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:37.454 INFO:teuthology.orchestra.run.smithi138.stdout:154618822693 2023-10-01T09:03:37.838 INFO:teuthology.orchestra.run.smithi138.stdout:206158430236 2023-10-01T09:03:37.860 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:37 smithi138 conmon[100289]: audit 2023-10-01T09:03:37.311884+0000 mon.a 2023-10-01T09:03:37.860 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:37 smithi138 conmon[100289]: (mon.0) 666 : audit [DBG] from='client.? 172.21.15.138:0/3360078792' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T09:03:37.861 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:37 smithi138 conmon[109959]: audit 2023-10-01T09:03:37.311884+0000 mon.a (mon.0) 666 2023-10-01T09:03:37.861 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:37 smithi138 conmon[109959]: : audit [DBG] from='client.? 172.21.15.138:0/3360078792' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T09:03:38.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:37 smithi190 conmon[100518]: audit 2023-10-01T09:03:37.311884+0000 mon.a (mon.0) 666 : audit [DBG] from='client.? 172.21.15.138:0/3360078792' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T09:03:38.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:38 smithi138 conmon[100289]: cluster 2023-10-01T09:03:36.850578+0000 mgr.y (mgr.14433) 89 : cluster 2023-10-01T09:03:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:38 smithi138 conmon[100289]: [DBG] pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:38 smithi138 conmon[100289]: audit 2023-10-01T09:03:37.838509+0000 mon.a (mon.0) 667 : audit [DBG] from='client.? 172.21.15.138:0/2911796314' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T09:03:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:38 smithi138 conmon[109959]: cluster 2023-10-01T09:03:36.850578+0000 mgr.y ( 2023-10-01T09:03:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:38 smithi138 conmon[109959]: mgr.14433) 89 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:38 smithi138 conmon[109959]: audit 2023-10-01T09:03:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:38 smithi138 conmon[109959]: 2023-10-01T09:03:37.838509+0000 mon.a (mon.0) 2023-10-01T09:03:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:38 smithi138 conmon[109959]: 667 : audit [DBG] from='client.? 172.21.15.138:0/2911796314' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T09:03:39.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:38 smithi190 conmon[100518]: cluster 2023-10-01T09:03:36.850578+0000 mgr.y (mgr.14433) 89 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:39.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:38 smithi190 conmon[100518]: audit 2023-10-01T09:03:37.838509+0000 mon.a (mon.0) 667 : audit [DBG] from='client.? 172.21.15.138:0/2911796314' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T09:03:39.192 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430233 got 206158430236 for osd.7 2023-10-01T09:03:39.192 DEBUG:teuthology.parallel:result is None 2023-10-01T09:03:39.833 INFO:teuthology.orchestra.run.smithi138.stdout:34359738428 2023-10-01T09:03:40.248 INFO:teuthology.orchestra.run.smithi138.stdout:128849018921 2023-10-01T09:03:40.435 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822692 got 154618822693 for osd.5 2023-10-01T09:03:40.435 DEBUG:teuthology.parallel:result is None 2023-10-01T09:03:40.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:40 smithi138 conmon[109959]: cluster 2023-10-01T09:03:38.850922+0000 mgr.y (mgr.14433) 90 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:40 smithi138 conmon[109959]: audit 2023-10-01T09:03:39.832342+0000 mon.c (mon.2) 87 : audit [DBG] from='client.? 172.21.15.138:0/1864110734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T09:03:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:40 smithi138 conmon[109959]: audit 2023-10-01T09:03:40.248390+0000 mon.a (mon.0) 668 : audit [DBG] from='client.? 172.21.15.138:0/144264820' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T09:03:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:40 smithi138 conmon[100289]: cluster 2023-10-01T09:03:38.850922+0000 mgr.y (mgr.14433) 90 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:40 smithi138 conmon[100289]: audit 2023-10-01T09:03:39.832342+0000 mon.c (mon.2) 87 : audit [DBG] from='client.? 172.21.15.138:0/1864110734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T09:03:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:40 smithi138 conmon[100289]: audit 2023-10-01T09:03:40.248390+0000 mon.a (mon.0) 668 : audit [DBG] from='client.? 172.21.15.138:0/144264820' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T09:03:41.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:40 smithi190 conmon[100518]: cluster 2023-10-01T09:03:38.850922+0000 mgr.y (mgr.14433) 90 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:41.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:40 smithi190 conmon[100518]: audit 2023-10-01T09:03:39.832342+0000 mon.c (mon.2) 87 : audit [DBG] from='client.? 172.21.15.138:0/1864110734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T09:03:41.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:40 smithi190 conmon[100518]: audit 2023-10-01T09:03:40.248390+0000 mon.a (mon.0) 668 : audit [DBG] from='client.? 172.21.15.138:0/144264820' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T09:03:41.955 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:41 smithi138 conmon[109959]: cluster 2023-10-01T09:03:40.851699+0000 2023-10-01T09:03:41.955 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:41 smithi138 conmon[109959]: mgr.y (mgr.14433) 91 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:41.955 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:41 smithi138 conmon[100289]: cluster 2023-10-01T09:03:40.851699+0000 mgr.y (mgr.14433) 91 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:42.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:41 smithi190 conmon[100518]: cluster 2023-10-01T09:03:40.851699+0000 mgr.y (mgr.14433) 91 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:42.185 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738426 got 34359738428 for osd.0 2023-10-01T09:03:42.185 DEBUG:teuthology.parallel:result is None 2023-10-01T09:03:42.319 INFO:teuthology.orchestra.run.smithi138.stdout:107374182447 2023-10-01T09:03:42.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:42 smithi138 conmon[100289]: audit 2023-10-01T09:03:41.926880+0000 mon.a (mon.0) 669 : 2023-10-01T09:03:42.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:42 smithi138 conmon[100289]: audit [DBG] from='client.? 172.21.15.138:0/1741462686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T09:03:42.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:42 smithi138 conmon[109959]: audit 2023-10-01T09:03:41 2023-10-01T09:03:42.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:42 smithi138 conmon[109959]: .926880+0000 mon.a (mon.0) 669 : audit [DBG] from='client.? 172.21.15.138:0/1741462686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T09:03:42.891 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018919 got 128849018921 for osd.4 2023-10-01T09:03:42.891 DEBUG:teuthology.parallel:result is None 2023-10-01T09:03:43.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:42 smithi190 conmon[100518]: audit 2023-10-01T09:03:41.926880+0000 mon.a (mon.0) 669 : audit [DBG] from='client.? 172.21.15.138:0/1741462686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T09:03:43.167 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182444 got 107374182447 for osd.3 2023-10-01T09:03:43.167 DEBUG:teuthology.parallel:result is None 2023-10-01T09:03:43.495 INFO:teuthology.orchestra.run.smithi138.stdout:180388626466 2023-10-01T09:03:43.650 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:43 smithi138 conmon[109959]: cluster 2023-10-01T09:03:42.851981+0000 mgr.y (mgr.14433) 2023-10-01T09:03:43.650 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:43 smithi138 conmon[109959]: 92 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:43.651 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:43 smithi138 conmon[109959]: audit 2023-10-01T09:03:43.494688+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.138:0/2561116575' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T09:03:43.651 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:43 smithi138 conmon[100289]: cluster 2023-10-01T09:03:42.851981+0000 mgr.y (mgr. 2023-10-01T09:03:43.651 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:43 smithi138 conmon[100289]: 14433) 92 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:43.651 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:43 smithi138 conmon[100289]: audit 2023-10-01T09:03:43.494688+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.138:0/2561116575' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T09:03:43.751 INFO:teuthology.orchestra.run.smithi138.stdout:55834574904 2023-10-01T09:03:44.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:43 smithi190 conmon[100518]: cluster 2023-10-01T09:03:42.851981+0000 mgr.y (mgr.14433) 92 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:44.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:43 smithi190 conmon[100518]: audit 2023-10-01T09:03:43.494688+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.138:0/2561116575' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T09:03:44.433 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626463 got 180388626466 for osd.6 2023-10-01T09:03:44.433 DEBUG:teuthology.parallel:result is None 2023-10-01T09:03:44.665 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574902 got 55834574904 for osd.1 2023-10-01T09:03:44.665 DEBUG:teuthology.parallel:result is None 2023-10-01T09:03:44.666 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-01T09:03:44.666 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:03:45.012 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:45.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:44 smithi190 conmon[100518]: audit 2023-10-01T09:03:43.751551+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.138:0/1514352226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T09:03:45.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:44 smithi138 conmon[100289]: audit 2023-10-01T09:03:43.751551+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.138:0/1514352226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T09:03:45.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:44 smithi138 conmon[109959]: audit 2023-10-01T09:03: 2023-10-01T09:03:45.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:44 smithi138 conmon[109959]: 43.751551+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.138:0/1514352226' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T09:03:45.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:03:45] "GET /metrics HTTP/1.1" 200 191337 "" "Prometheus/2.33.4" 2023-10-01T09:03:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: audit 2023-10-01T09:03:44.850077+0000 mon.c (mon.2) 89 : 2023-10-01T09:03:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: audit 2023-10-01T09:03:44.850294+0000 mon.c (mon.2) 90 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: audit 2023-10-01T09:03:44.850516+0000 mon.a (mon.0) 671 : audit 2023-10-01T09:03:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:03:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: :03:44.850578+0000 mon.c (mon.2) 91 2023-10-01T09:03:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T09:03:46.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: T09:03:44.850994+0000 mon.a ( 2023-10-01T09:03:46.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: mon.0) 672 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: audit 2023-10-01T09:03:44.851176+0000 2023-10-01T09:03:46.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: mon.a (mon.0) 673 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: cluster 2023-10-01T09:03:46.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: 2023-10-01T09:03:44. 2023-10-01T09:03:46.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: 852277+0000 mgr.y (mgr.14433) 93 : cluster [DBG] 2023-10-01T09:03:46.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:45 smithi190 conmon[100518]: pgmap v72: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:46.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[109959]: audit 2023-10-01T09:03:44.850077+0000 mon.c (mon.2) 89 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[109959]: audit 2023-10-01T09:03:44.850294+0000 mon.c (mon.2) 90 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[109959]: audit 2023-10-01T09:03:44.850516+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:03:46.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[109959]: 03:44.850578+0000 mon.c (mon.2) 91 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[109959]: audit 2023-10-01T09:03:44.850994+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[109959]: audit 2023-10-01T09:03:44.851176+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[109959]: 2023-10-01T09:03:46.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[109959]: cluster 2023- 2023-10-01T09:03:46.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[109959]: 10-01T09:03:44.852277+0000 mgr.y (mgr.14433) 93 2023-10-01T09:03:46.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[109959]: : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:46.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[100289]: audit 2023-10-01T09:03:44.850077+0000 mon.c (mon.2) 89 : audit 2023-10-01T09:03:46.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[100289]: [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[100289]: audit 2023-10-01T09:03:44.850294+0000 mon.c (mon.2) 90 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[100289]: audit 2023-10-01T09:03:44.850516+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[100289]: audit 2023-10-01T09:03:44.850578+0000 mon.c (mon.2) 91 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[100289]: audit 2023-10-01T09:03:44.850994+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[100289]: audit 2023-10-01T09:03:44.851176+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-10-01T09:03:46.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:45 smithi138 conmon[100289]: cluster 2023-10-01T09:03:44.852277+0000 mgr.y (mgr.14433) 93 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:46.717 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:03:46.718 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:03:47.056 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:03:46 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:03:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:03:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:46 smithi190 conmon[100518]: audit 2023-10-01T09:03:45.632847+0000 mon.a (mon.0) 674 : audit [INF] 2023-10-01T09:03:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:46 smithi190 conmon[100518]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-01T09:03:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:46 smithi190 conmon[100518]: audit 2023-10-01T09:03:45.632939+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-01T09:03:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:46 smithi190 conmon[100518]: audit 2023-10-01T09:03:45.632992 2023-10-01T09:03:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:46 smithi190 conmon[100518]: +0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-01T09:03:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:46 smithi190 conmon[100518]: cluster 2023-10 2023-10-01T09:03:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:46 smithi190 conmon[100518]: -01T09:03:45.633034+0000 mon.a (mon.0) 677 : cluster 2023-10-01T09:03:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:46 smithi190 conmon[100518]: [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-01T09:03:47.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:46 smithi138 conmon[109959]: audit 2023-10-01T09:03:45.632847+0000 mon.a 2023-10-01T09:03:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:46 smithi138 conmon[109959]: (mon.0) 674 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-01T09:03:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:46 smithi138 conmon[109959]: audit 2023-10-01T09:03:45.632939+0000 mon.a (mon.0 2023-10-01T09:03:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:46 smithi138 conmon[109959]: ) 675 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-01T09:03:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:46 smithi138 conmon[109959]: audit 2023-10-01T09:03:45.632992+0000 mon.a (mon 2023-10-01T09:03:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:46 smithi138 conmon[109959]: .0) 676 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-01T09:03:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:46 smithi138 conmon[109959]: cluster 2023-10-01T09:03:45.633034+0000 mon.a (mon. 2023-10-01T09:03:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:46 smithi138 conmon[109959]: 0) 677 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-01T09:03:47.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:46 smithi138 conmon[100289]: audit 2023-10-01T09:03:45.632847+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-01T09:03:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:46 smithi138 conmon[100289]: audit 2023-10-01T09:03:45.632939+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-10-01T09:03:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:46 smithi138 conmon[100289]: audit 2023-10-01T09:03:45.632992+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-10-01T09:03:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:46 smithi138 conmon[100289]: cluster 2023-10-01T09:03:45.633034+0000 mon.a (mon.0) 677 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-01T09:03:47.350 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":74,"stamp":"2023-10-01T09:03:46.637075+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381588,"kb_used_data":4820,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579628,"statfs":{"total":767960285184,"available":765521539072,"internally_reserved":0,"allocated":4935680,"data_stored":2075578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.786818"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.216254+0000","last_change":"2023-10-01T09:03:03.223370+0000","last_active":"2023-10-01T09:03:05.216254+0000","last_peered":"2023-10-01T09:03:05.216254+0000","last_clean":"2023-10-01T09:03:05.216254+0000","last_became_active":"2023-10-01T09:03:03.223071+0000","last_became_peered":"2023-10-01T09:03:03.223071+0000","last_unstale":"2023-10-01T09:03:05.216254+0000","last_undegraded":"2023-10-01T09:03:05.216254+0000","last_fullsized":"2023-10-01T09:03:05.216254+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.232525+0000","last_change":"2023-10-01T09:03:03.202335+0000","last_active":"2023-10-01T09:03:08.232525+0000","last_peered":"2023-10-01T09:03:08.232525+0000","last_clean":"2023-10-01T09:03:08.232525+0000","last_became_active":"2023-10-01T09:03:03.201485+0000","last_became_peered":"2023-10-01T09:03:03.201485+0000","last_unstale":"2023-10-01T09:03:08.232525+0000","last_undegraded":"2023-10-01T09:03:08.232525+0000","last_fullsized":"2023-10-01T09:03:08.232525+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346371+0000","last_change":"2023-10-01T09:03:03.224058+0000","last_active":"2023-10-01T09:03:06.346371+0000","last_peered":"2023-10-01T09:03:06.346371+0000","last_clean":"2023-10-01T09:03:06.346371+0000","last_became_active":"2023-10-01T09:03:03.223598+0000","last_became_peered":"2023-10-01T09:03:03.223598+0000","last_unstale":"2023-10-01T09:03:06.346371+0000","last_undegraded":"2023-10-01T09:03:06.346371+0000","last_fullsized":"2023-10-01T09:03:06.346371+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.233101+0000","last_change":"2023-10-01T09:03:03.193542+0000","last_active":"2023-10-01T09:03:08.233101+0000","last_peered":"2023-10-01T09:03:08.233101+0000","last_clean":"2023-10-01T09:03:08.233101+0000","last_became_active":"2023-10-01T09:03:03.193245+0000","last_became_peered":"2023-10-01T09:03:03.193245+0000","last_unstale":"2023-10-01T09:03:08.233101+0000","last_undegraded":"2023-10-01T09:03:08.233101+0000","last_fullsized":"2023-10-01T09:03:08.233101+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.216607+0000","last_change":"2023-10-01T09:03:03.218020+0000","last_active":"2023-10-01T09:03:05.216607+0000","last_peered":"2023-10-01T09:03:05.216607+0000","last_clean":"2023-10-01T09:03:05.216607+0000","last_became_active":"2023-10-01T09:03:03.217717+0000","last_became_peered":"2023-10-01T09:03:03.217717+0000","last_unstale":"2023-10-01T09:03:05.216607+0000","last_undegraded":"2023-10-01T09:03:05.216607+0000","last_fullsized":"2023-10-01T09:03:05.216607+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.232617+0000","last_change":"2023-10-01T09:03:03.202063+0000","last_active":"2023-10-01T09:03:08.232617+0000","last_peered":"2023-10-01T09:03:08.232617+0000","last_clean":"2023-10-01T09:03:08.232617+0000","last_became_active":"2023-10-01T09:03:03.201769+0000","last_became_peered":"2023-10-01T09:03:03.201769+0000","last_unstale":"2023-10-01T09:03:08.232617+0000","last_undegraded":"2023-10-01T09:03:08.232617+0000","last_fullsized":"2023-10-01T09:03:08.232617+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.217294+0000","last_change":"2023-10-01T09:03:03.221313+0000","last_active":"2023-10-01T09:03:05.217294+0000","last_peered":"2023-10-01T09:03:05.217294+0000","last_clean":"2023-10-01T09:03:05.217294+0000","last_became_active":"2023-10-01T09:03:03.221080+0000","last_became_peered":"2023-10-01T09:03:03.221080+0000","last_unstale":"2023-10-01T09:03:05.217294+0000","last_undegraded":"2023-10-01T09:03:05.217294+0000","last_fullsized":"2023-10-01T09:03:05.217294+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.232352+0000","last_change":"2023-10-01T09:03:03.201659+0000","last_active":"2023-10-01T09:03:08.232352+0000","last_peered":"2023-10-01T09:03:08.232352+0000","last_clean":"2023-10-01T09:03:08.232352+0000","last_became_active":"2023-10-01T09:03:03.201348+0000","last_became_peered":"2023-10-01T09:03:03.201348+0000","last_unstale":"2023-10-01T09:03:08.232352+0000","last_undegraded":"2023-10-01T09:03:08.232352+0000","last_fullsized":"2023-10-01T09:03:08.232352+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.214523+0000","last_change":"2023-10-01T09:03:03.226772+0000","last_active":"2023-10-01T09:03:05.214523+0000","last_peered":"2023-10-01T09:03:05.214523+0000","last_clean":"2023-10-01T09:03:05.214523+0000","last_became_active":"2023-10-01T09:03:03.226446+0000","last_became_peered":"2023-10-01T09:03:03.226446+0000","last_unstale":"2023-10-01T09:03:05.214523+0000","last_undegraded":"2023-10-01T09:03:05.214523+0000","last_fullsized":"2023-10-01T09:03:05.214523+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215833+0000","last_change":"2023-10-01T09:03:03.203002+0000","last_active":"2023-10-01T09:03:05.215833+0000","last_peered":"2023-10-01T09:03:05.215833+0000","last_clean":"2023-10-01T09:03:05.215833+0000","last_became_active":"2023-10-01T09:03:03.202571+0000","last_became_peered":"2023-10-01T09:03:03.202571+0000","last_unstale":"2023-10-01T09:03:05.215833+0000","last_undegraded":"2023-10-01T09:03:05.215833+0000","last_fullsized":"2023-10-01T09:03:05.215833+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.216590+0000","last_change":"2023-10-01T09:03:03.754839+0000","last_active":"2023-10-01T09:03:05.216590+0000","last_peered":"2023-10-01T09:03:05.216590+0000","last_clean":"2023-10-01T09:03:05.216590+0000","last_became_active":"2023-10-01T09:03:03.754506+0000","last_became_peered":"2023-10-01T09:03:03.754506+0000","last_unstale":"2023-10-01T09:03:05.216590+0000","last_undegraded":"2023-10-01T09:03:05.216590+0000","last_fullsized":"2023-10-01T09:03:05.216590+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215737+0000","last_change":"2023-10-01T09:03:03.196477+0000","last_active":"2023-10-01T09:03:05.215737+0000","last_peered":"2023-10-01T09:03:05.215737+0000","last_clean":"2023-10-01T09:03:05.215737+0000","last_became_active":"2023-10-01T09:03:03.196266+0000","last_became_peered":"2023-10-01T09:03:03.196266+0000","last_unstale":"2023-10-01T09:03:05.215737+0000","last_undegraded":"2023-10-01T09:03:05.215737+0000","last_fullsized":"2023-10-01T09:03:05.215737+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.696921+0000","last_change":"2023-10-01T09:02:06.997887+0000","last_active":"2023-10-01T09:03:05.696921+0000","last_peered":"2023-10-01T09:03:05.696921+0000","last_clean":"2023-10-01T09:03:05.696921+0000","last_became_active":"2023-10-01T09:02:06.997597+0000","last_became_peered":"2023-10-01T09:02:06.997597+0000","last_unstale":"2023-10-01T09:03:05.696921+0000","last_undegraded":"2023-10-01T09:03:05.696921+0000","last_fullsized":"2023-10-01T09:03:05.696921+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.345723+0000","last_change":"2023-10-01T09:03:03.228390+0000","last_active":"2023-10-01T09:03:06.345723+0000","last_peered":"2023-10-01T09:03:06.345723+0000","last_clean":"2023-10-01T09:03:06.345723+0000","last_became_active":"2023-10-01T09:03:03.227254+0000","last_became_peered":"2023-10-01T09:03:03.227254+0000","last_unstale":"2023-10-01T09:03:06.345723+0000","last_undegraded":"2023-10-01T09:03:06.345723+0000","last_fullsized":"2023-10-01T09:03:06.345723+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346408+0000","last_change":"2023-10-01T09:02:07.442083+0000","last_active":"2023-10-01T09:03:06.346408+0000","last_peered":"2023-10-01T09:03:06.346408+0000","last_clean":"2023-10-01T09:03:06.346408+0000","last_became_active":"2023-10-01T09:02:07.441757+0000","last_became_peered":"2023-10-01T09:02:07.441757+0000","last_unstale":"2023-10-01T09:03:06.346408+0000","last_undegraded":"2023-10-01T09:03:06.346408+0000","last_fullsized":"2023-10-01T09:03:06.346408+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.233194+0000","last_change":"2023-10-01T09:03:03.198510+0000","last_active":"2023-10-01T09:03:08.233194+0000","last_peered":"2023-10-01T09:03:08.233194+0000","last_clean":"2023-10-01T09:03:08.233194+0000","last_became_active":"2023-10-01T09:03:03.198172+0000","last_became_peered":"2023-10-01T09:03:03.198172+0000","last_unstale":"2023-10-01T09:03:08.233194+0000","last_undegraded":"2023-10-01T09:03:08.233194+0000","last_fullsized":"2023-10-01T09:03:08.233194+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.209663+0000","last_change":"2023-10-01T09:03:03.754828+0000","last_active":"2023-10-01T09:03:04.209663+0000","last_peered":"2023-10-01T09:03:04.209663+0000","last_clean":"2023-10-01T09:03:04.209663+0000","last_became_active":"2023-10-01T09:03:03.754492+0000","last_became_peered":"2023-10-01T09:03:03.754492+0000","last_unstale":"2023-10-01T09:03:04.209663+0000","last_undegraded":"2023-10-01T09:03:04.209663+0000","last_fullsized":"2023-10-01T09:03:04.209663+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.347115+0000","last_change":"2023-10-01T09:03:03.203345+0000","last_active":"2023-10-01T09:03:06.347115+0000","last_peered":"2023-10-01T09:03:06.347115+0000","last_clean":"2023-10-01T09:03:06.347115+0000","last_became_active":"2023-10-01T09:03:03.203035+0000","last_became_peered":"2023-10-01T09:03:03.203035+0000","last_unstale":"2023-10-01T09:03:06.347115+0000","last_undegraded":"2023-10-01T09:03:06.347115+0000","last_fullsized":"2023-10-01T09:03:06.347115+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.244297+0000","last_change":"2023-10-01T09:02:07.442452+0000","last_active":"2023-10-01T09:03:04.244297+0000","last_peered":"2023-10-01T09:03:04.244297+0000","last_clean":"2023-10-01T09:03:04.244297+0000","last_became_active":"2023-10-01T09:02:07.442111+0000","last_became_peered":"2023-10-01T09:02:07.442111+0000","last_unstale":"2023-10-01T09:03:04.244297+0000","last_undegraded":"2023-10-01T09:03:04.244297+0000","last_fullsized":"2023-10-01T09:03:04.244297+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.211021+0000","last_change":"2023-10-01T09:03:03.223326+0000","last_active":"2023-10-01T09:03:04.211021+0000","last_peered":"2023-10-01T09:03:04.211021+0000","last_clean":"2023-10-01T09:03:04.211021+0000","last_became_active":"2023-10-01T09:03:03.222974+0000","last_became_peered":"2023-10-01T09:03:03.222974+0000","last_unstale":"2023-10-01T09:03:04.211021+0000","last_undegraded":"2023-10-01T09:03:04.211021+0000","last_fullsized":"2023-10-01T09:03:04.211021+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.210759+0000","last_change":"2023-10-01T09:03:03.195712+0000","last_active":"2023-10-01T09:03:04.210759+0000","last_peered":"2023-10-01T09:03:04.210759+0000","last_clean":"2023-10-01T09:03:04.210759+0000","last_became_active":"2023-10-01T09:03:03.195408+0000","last_became_peered":"2023-10-01T09:03:03.195408+0000","last_unstale":"2023-10-01T09:03:04.210759+0000","last_undegraded":"2023-10-01T09:03:04.210759+0000","last_fullsized":"2023-10-01T09:03:04.210759+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.244156+0000","last_change":"2023-10-01T09:02:06.999043+0000","last_active":"2023-10-01T09:03:04.244156+0000","last_peered":"2023-10-01T09:03:04.244156+0000","last_clean":"2023-10-01T09:03:04.244156+0000","last_became_active":"2023-10-01T09:02:06.998671+0000","last_became_peered":"2023-10-01T09:02:06.998671+0000","last_unstale":"2023-10-01T09:03:04.244156+0000","last_undegraded":"2023-10-01T09:03:04.244156+0000","last_fullsized":"2023-10-01T09:03:04.244156+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.696828+0000","last_change":"2023-10-01T09:03:03.227254+0000","last_active":"2023-10-01T09:03:05.696828+0000","last_peered":"2023-10-01T09:03:05.696828+0000","last_clean":"2023-10-01T09:03:05.696828+0000","last_became_active":"2023-10-01T09:03:03.226941+0000","last_became_peered":"2023-10-01T09:03:03.226941+0000","last_unstale":"2023-10-01T09:03:05.696828+0000","last_undegraded":"2023-10-01T09:03:05.696828+0000","last_fullsized":"2023-10-01T09:03:05.696828+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.345884+0000","last_change":"2023-10-01T09:03:03.198835+0000","last_active":"2023-10-01T09:03:06.345884+0000","last_peered":"2023-10-01T09:03:06.345884+0000","last_clean":"2023-10-01T09:03:06.345884+0000","last_became_active":"2023-10-01T09:03:03.198424+0000","last_became_peered":"2023-10-01T09:03:03.198424+0000","last_unstale":"2023-10-01T09:03:06.345884+0000","last_undegraded":"2023-10-01T09:03:06.345884+0000","last_fullsized":"2023-10-01T09:03:06.345884+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.244204+0000","last_change":"2023-10-01T09:01:32.510259+0000","last_active":"2023-10-01T09:03:04.244204+0000","last_peered":"2023-10-01T09:03:04.244204+0000","last_clean":"2023-10-01T09:03:04.244204+0000","last_became_active":"2023-10-01T09:01:32.509905+0000","last_became_peered":"2023-10-01T09:01:32.509905+0000","last_unstale":"2023-10-01T09:03:04.244204+0000","last_undegraded":"2023-10-01T09:03:04.244204+0000","last_fullsized":"2023-10-01T09:03:04.244204+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.237819+0000","last_change":"2023-10-01T09:02:06.998804+0000","last_active":"2023-10-01T09:03:05.237819+0000","last_peered":"2023-10-01T09:03:05.237819+0000","last_clean":"2023-10-01T09:03:05.237819+0000","last_became_active":"2023-10-01T09:02:06.998516+0000","last_became_peered":"2023-10-01T09:02:06.998516+0000","last_unstale":"2023-10-01T09:03:05.237819+0000","last_undegraded":"2023-10-01T09:03:05.237819+0000","last_fullsized":"2023-10-01T09:03:05.237819+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346495+0000","last_change":"2023-10-01T09:03:03.756021+0000","last_active":"2023-10-01T09:03:06.346495+0000","last_peered":"2023-10-01T09:03:06.346495+0000","last_clean":"2023-10-01T09:03:06.346495+0000","last_became_active":"2023-10-01T09:03:03.755659+0000","last_became_peered":"2023-10-01T09:03:03.755659+0000","last_unstale":"2023-10-01T09:03:06.346495+0000","last_undegraded":"2023-10-01T09:03:06.346495+0000","last_fullsized":"2023-10-01T09:03:06.346495+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346914+0000","last_change":"2023-10-01T09:03:03.201620+0000","last_active":"2023-10-01T09:03:06.346914+0000","last_peered":"2023-10-01T09:03:06.346914+0000","last_clean":"2023-10-01T09:03:06.346914+0000","last_became_active":"2023-10-01T09:03:03.201312+0000","last_became_peered":"2023-10-01T09:03:03.201312+0000","last_unstale":"2023-10-01T09:03:06.346914+0000","last_undegraded":"2023-10-01T09:03:06.346914+0000","last_fullsized":"2023-10-01T09:03:06.346914+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.214376+0000","last_change":"2023-10-01T09:02:07.436492+0000","last_active":"2023-10-01T09:03:05.214376+0000","last_peered":"2023-10-01T09:03:05.214376+0000","last_clean":"2023-10-01T09:03:05.214376+0000","last_became_active":"2023-10-01T09:02:07.436146+0000","last_became_peered":"2023-10-01T09:02:07.436146+0000","last_unstale":"2023-10-01T09:03:05.214376+0000","last_undegraded":"2023-10-01T09:03:05.214376+0000","last_fullsized":"2023-10-01T09:03:05.214376+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.347049+0000","last_change":"2023-10-01T09:03:03.227284+0000","last_active":"2023-10-01T09:03:06.347049+0000","last_peered":"2023-10-01T09:03:06.347049+0000","last_clean":"2023-10-01T09:03:06.347049+0000","last_became_active":"2023-10-01T09:03:03.226987+0000","last_became_peered":"2023-10-01T09:03:03.226987+0000","last_unstale":"2023-10-01T09:03:06.347049+0000","last_undegraded":"2023-10-01T09:03:06.347049+0000","last_fullsized":"2023-10-01T09:03:06.347049+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.217322+0000","last_change":"2023-10-01T09:03:03.196357+0000","last_active":"2023-10-01T09:03:05.217322+0000","last_peered":"2023-10-01T09:03:05.217322+0000","last_clean":"2023-10-01T09:03:05.217322+0000","last_became_active":"2023-10-01T09:03:03.196017+0000","last_became_peered":"2023-10-01T09:03:03.196017+0000","last_unstale":"2023-10-01T09:03:05.217322+0000","last_undegraded":"2023-10-01T09:03:05.217322+0000","last_fullsized":"2023-10-01T09:03:05.217322+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.237755+0000","last_change":"2023-10-01T09:02:07.001644+0000","last_active":"2023-10-01T09:03:05.237755+0000","last_peered":"2023-10-01T09:03:05.237755+0000","last_clean":"2023-10-01T09:03:05.237755+0000","last_became_active":"2023-10-01T09:02:07.001197+0000","last_became_peered":"2023-10-01T09:02:07.001197+0000","last_unstale":"2023-10-01T09:03:05.237755+0000","last_undegraded":"2023-10-01T09:03:05.237755+0000","last_fullsized":"2023-10-01T09:03:05.237755+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346759+0000","last_change":"2023-10-01T09:03:03.755027+0000","last_active":"2023-10-01T09:03:06.346759+0000","last_peered":"2023-10-01T09:03:06.346759+0000","last_clean":"2023-10-01T09:03:06.346759+0000","last_became_active":"2023-10-01T09:03:03.754596+0000","last_became_peered":"2023-10-01T09:03:03.754596+0000","last_unstale":"2023-10-01T09:03:06.346759+0000","last_undegraded":"2023-10-01T09:03:06.346759+0000","last_fullsized":"2023-10-01T09:03:06.346759+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.237997+0000","last_change":"2023-10-01T09:03:03.203012+0000","last_active":"2023-10-01T09:03:05.237997+0000","last_peered":"2023-10-01T09:03:05.237997+0000","last_clean":"2023-10-01T09:03:05.237997+0000","last_became_active":"2023-10-01T09:03:03.202051+0000","last_became_peered":"2023-10-01T09:03:03.202051+0000","last_unstale":"2023-10-01T09:03:05.237997+0000","last_undegraded":"2023-10-01T09:03:05.237997+0000","last_fullsized":"2023-10-01T09:03:05.237997+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346979+0000","last_change":"2023-10-01T09:02:06.998188+0000","last_active":"2023-10-01T09:03:06.346979+0000","last_peered":"2023-10-01T09:03:06.346979+0000","last_clean":"2023-10-01T09:03:06.346979+0000","last_became_active":"2023-10-01T09:02:06.997855+0000","last_became_peered":"2023-10-01T09:02:06.997855+0000","last_unstale":"2023-10-01T09:03:06.346979+0000","last_undegraded":"2023-10-01T09:03:06.346979+0000","last_fullsized":"2023-10-01T09:03:06.346979+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1,"reported_epoch":61,"state":"peering","last_fresh":"2023-10-01T09:03:45.633611+0000","last_change":"2023-10-01T09:03:45.633611+0000","last_active":"0.000000","last_peered":"0.000000","last_clean":"0.000000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:03:45.633611+0000","last_undegraded":"2023-10-01T09:03:45.633611+0000","last_fullsized":"2023-10-01T09:03:45.633611+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.345762+0000","last_change":"2023-10-01T09:03:03.200838+0000","last_active":"2023-10-01T09:03:06.345762+0000","last_peered":"2023-10-01T09:03:06.345762+0000","last_clean":"2023-10-01T09:03:06.345762+0000","last_became_active":"2023-10-01T09:03:03.199715+0000","last_became_peered":"2023-10-01T09:03:03.199715+0000","last_unstale":"2023-10-01T09:03:06.345762+0000","last_undegraded":"2023-10-01T09:03:06.345762+0000","last_fullsized":"2023-10-01T09:03:06.345762+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.217059+0000","last_change":"2023-10-01T09:03:03.221266+0000","last_active":"2023-10-01T09:03:05.217059+0000","last_peered":"2023-10-01T09:03:05.217059+0000","last_clean":"2023-10-01T09:03:05.217059+0000","last_became_active":"2023-10-01T09:03:03.220690+0000","last_became_peered":"2023-10-01T09:03:03.220690+0000","last_unstale":"2023-10-01T09:03:05.217059+0000","last_undegraded":"2023-10-01T09:03:05.217059+0000","last_fullsized":"2023-10-01T09:03:05.217059+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.216742+0000","last_change":"2023-10-01T09:03:03.199237+0000","last_active":"2023-10-01T09:03:05.216742+0000","last_peered":"2023-10-01T09:03:05.216742+0000","last_clean":"2023-10-01T09:03:05.216742+0000","last_became_active":"2023-10-01T09:03:03.198950+0000","last_became_peered":"2023-10-01T09:03:03.198950+0000","last_unstale":"2023-10-01T09:03:05.216742+0000","last_undegraded":"2023-10-01T09:03:05.216742+0000","last_fullsized":"2023-10-01T09:03:05.216742+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.217019+0000","last_change":"2023-10-01T09:03:03.223290+0000","last_active":"2023-10-01T09:03:05.217019+0000","last_peered":"2023-10-01T09:03:05.217019+0000","last_clean":"2023-10-01T09:03:05.217019+0000","last_became_active":"2023-10-01T09:03:03.223013+0000","last_became_peered":"2023-10-01T09:03:03.223013+0000","last_unstale":"2023-10-01T09:03:05.217019+0000","last_undegraded":"2023-10-01T09:03:05.217019+0000","last_fullsized":"2023-10-01T09:03:05.217019+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.696678+0000","last_change":"2023-10-01T09:03:03.198744+0000","last_active":"2023-10-01T09:03:05.696678+0000","last_peered":"2023-10-01T09:03:05.696678+0000","last_clean":"2023-10-01T09:03:05.696678+0000","last_became_active":"2023-10-01T09:03:03.198452+0000","last_became_peered":"2023-10-01T09:03:03.198452+0000","last_unstale":"2023-10-01T09:03:05.696678+0000","last_undegraded":"2023-10-01T09:03:05.696678+0000","last_fullsized":"2023-10-01T09:03:05.696678+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.216811+0000","last_change":"2023-10-01T09:03:03.218071+0000","last_active":"2023-10-01T09:03:05.216811+0000","last_peered":"2023-10-01T09:03:05.216811+0000","last_clean":"2023-10-01T09:03:05.216811+0000","last_became_active":"2023-10-01T09:03:03.217776+0000","last_became_peered":"2023-10-01T09:03:03.217776+0000","last_unstale":"2023-10-01T09:03:05.216811+0000","last_undegraded":"2023-10-01T09:03:05.216811+0000","last_fullsized":"2023-10-01T09:03:05.216811+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.233311+0000","last_change":"2023-10-01T09:03:03.200515+0000","last_active":"2023-10-01T09:03:08.233311+0000","last_peered":"2023-10-01T09:03:08.233311+0000","last_clean":"2023-10-01T09:03:08.233311+0000","last_became_active":"2023-10-01T09:03:03.199536+0000","last_became_peered":"2023-10-01T09:03:03.199536+0000","last_unstale":"2023-10-01T09:03:08.233311+0000","last_undegraded":"2023-10-01T09:03:08.233311+0000","last_fullsized":"2023-10-01T09:03:08.233311+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.209902+0000","last_change":"2023-10-01T09:03:03.224884+0000","last_active":"2023-10-01T09:03:04.209902+0000","last_peered":"2023-10-01T09:03:04.209902+0000","last_clean":"2023-10-01T09:03:04.209902+0000","last_became_active":"2023-10-01T09:03:03.224529+0000","last_became_peered":"2023-10-01T09:03:03.224529+0000","last_unstale":"2023-10-01T09:03:04.209902+0000","last_undegraded":"2023-10-01T09:03:04.209902+0000","last_fullsized":"2023-10-01T09:03:04.209902+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.237860+0000","last_change":"2023-10-01T09:03:03.203721+0000","last_active":"2023-10-01T09:03:05.237860+0000","last_peered":"2023-10-01T09:03:05.237860+0000","last_clean":"2023-10-01T09:03:05.237860+0000","last_became_active":"2023-10-01T09:03:03.203308+0000","last_became_peered":"2023-10-01T09:03:03.203308+0000","last_unstale":"2023-10-01T09:03:05.237860+0000","last_undegraded":"2023-10-01T09:03:05.237860+0000","last_fullsized":"2023-10-01T09:03:05.237860+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.696616+0000","last_change":"2023-10-01T09:03:03.227070+0000","last_active":"2023-10-01T09:03:05.696616+0000","last_peered":"2023-10-01T09:03:05.696616+0000","last_clean":"2023-10-01T09:03:05.696616+0000","last_became_active":"2023-10-01T09:03:03.226752+0000","last_became_peered":"2023-10-01T09:03:03.226752+0000","last_unstale":"2023-10-01T09:03:05.696616+0000","last_undegraded":"2023-10-01T09:03:05.696616+0000","last_fullsized":"2023-10-01T09:03:05.696616+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346853+0000","last_change":"2023-10-01T09:03:03.200972+0000","last_active":"2023-10-01T09:03:06.346853+0000","last_peered":"2023-10-01T09:03:06.346853+0000","last_clean":"2023-10-01T09:03:06.346853+0000","last_became_active":"2023-10-01T09:03:03.200695+0000","last_became_peered":"2023-10-01T09:03:03.200695+0000","last_unstale":"2023-10-01T09:03:06.346853+0000","last_undegraded":"2023-10-01T09:03:06.346853+0000","last_fullsized":"2023-10-01T09:03:06.346853+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.216573+0000","last_change":"2023-10-01T09:03:03.220741+0000","last_active":"2023-10-01T09:03:05.216573+0000","last_peered":"2023-10-01T09:03:05.216573+0000","last_clean":"2023-10-01T09:03:05.216573+0000","last_became_active":"2023-10-01T09:03:03.220481+0000","last_became_peered":"2023-10-01T09:03:03.220481+0000","last_unstale":"2023-10-01T09:03:05.216573+0000","last_undegraded":"2023-10-01T09:03:05.216573+0000","last_fullsized":"2023-10-01T09:03:05.216573+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.231170+0000","last_change":"2023-10-01T09:03:03.200367+0000","last_active":"2023-10-01T09:03:08.231170+0000","last_peered":"2023-10-01T09:03:08.231170+0000","last_clean":"2023-10-01T09:03:08.231170+0000","last_became_active":"2023-10-01T09:03:03.200068+0000","last_became_peered":"2023-10-01T09:03:03.200068+0000","last_unstale":"2023-10-01T09:03:08.231170+0000","last_undegraded":"2023-10-01T09:03:08.231170+0000","last_fullsized":"2023-10-01T09:03:08.231170+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.237727+0000","last_change":"2023-10-01T09:03:04.245105+0000","last_active":"2023-10-01T09:03:05.237727+0000","last_peered":"2023-10-01T09:03:05.237727+0000","last_clean":"2023-10-01T09:03:05.237727+0000","last_became_active":"2023-10-01T09:03:04.244807+0000","last_became_peered":"2023-10-01T09:03:04.244807+0000","last_unstale":"2023-10-01T09:03:05.237727+0000","last_undegraded":"2023-10-01T09:03:05.237727+0000","last_fullsized":"2023-10-01T09:03:05.237727+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346958+0000","last_change":"2023-10-01T09:03:03.199109+0000","last_active":"2023-10-01T09:03:06.346958+0000","last_peered":"2023-10-01T09:03:06.346958+0000","last_clean":"2023-10-01T09:03:06.346958+0000","last_became_active":"2023-10-01T09:03:03.198810+0000","last_became_peered":"2023-10-01T09:03:03.198810+0000","last_unstale":"2023-10-01T09:03:06.346958+0000","last_undegraded":"2023-10-01T09:03:06.346958+0000","last_fullsized":"2023-10-01T09:03:06.346958+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346070+0000","last_change":"2023-10-01T09:03:03.227888+0000","last_active":"2023-10-01T09:03:06.346070+0000","last_peered":"2023-10-01T09:03:06.346070+0000","last_clean":"2023-10-01T09:03:06.346070+0000","last_became_active":"2023-10-01T09:03:03.227621+0000","last_became_peered":"2023-10-01T09:03:03.227621+0000","last_unstale":"2023-10-01T09:03:06.346070+0000","last_undegraded":"2023-10-01T09:03:06.346070+0000","last_fullsized":"2023-10-01T09:03:06.346070+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":21,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.233440+0000","last_change":"2023-10-01T09:03:03.199462+0000","last_active":"2023-10-01T09:03:08.233440+0000","last_peered":"2023-10-01T09:03:08.233440+0000","last_clean":"2023-10-01T09:03:08.233440+0000","last_became_active":"2023-10-01T09:03:03.199188+0000","last_became_peered":"2023-10-01T09:03:03.199188+0000","last_unstale":"2023-10-01T09:03:08.233440+0000","last_undegraded":"2023-10-01T09:03:08.233440+0000","last_fullsized":"2023-10-01T09:03:08.233440+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.211314+0000","last_change":"2023-10-01T09:03:03.228462+0000","last_active":"2023-10-01T09:03:04.211314+0000","last_peered":"2023-10-01T09:03:04.211314+0000","last_clean":"2023-10-01T09:03:04.211314+0000","last_became_active":"2023-10-01T09:03:03.228010+0000","last_became_peered":"2023-10-01T09:03:03.228010+0000","last_unstale":"2023-10-01T09:03:04.211314+0000","last_undegraded":"2023-10-01T09:03:04.211314+0000","last_fullsized":"2023-10-01T09:03:04.211314+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.244093+0000","last_change":"2023-10-01T09:03:03.194952+0000","last_active":"2023-10-01T09:03:04.244093+0000","last_peered":"2023-10-01T09:03:04.244093+0000","last_clean":"2023-10-01T09:03:04.244093+0000","last_became_active":"2023-10-01T09:03:03.194643+0000","last_became_peered":"2023-10-01T09:03:03.194643+0000","last_unstale":"2023-10-01T09:03:04.244093+0000","last_undegraded":"2023-10-01T09:03:04.244093+0000","last_fullsized":"2023-10-01T09:03:04.244093+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.237929+0000","last_change":"2023-10-01T09:03:04.405231+0000","last_active":"2023-10-01T09:03:05.237929+0000","last_peered":"2023-10-01T09:03:05.237929+0000","last_clean":"2023-10-01T09:03:05.237929+0000","last_became_active":"2023-10-01T09:03:04.404876+0000","last_became_peered":"2023-10-01T09:03:04.404876+0000","last_unstale":"2023-10-01T09:03:05.237929+0000","last_undegraded":"2023-10-01T09:03:05.237929+0000","last_fullsized":"2023-10-01T09:03:05.237929+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.209915+0000","last_change":"2023-10-01T09:03:03.202905+0000","last_active":"2023-10-01T09:03:04.209915+0000","last_peered":"2023-10-01T09:03:04.209915+0000","last_clean":"2023-10-01T09:03:04.209915+0000","last_became_active":"2023-10-01T09:03:03.202621+0000","last_became_peered":"2023-10-01T09:03:03.202621+0000","last_unstale":"2023-10-01T09:03:04.209915+0000","last_undegraded":"2023-10-01T09:03:04.209915+0000","last_fullsized":"2023-10-01T09:03:04.209915+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346699+0000","last_change":"2023-10-01T09:03:03.228053+0000","last_active":"2023-10-01T09:03:06.346699+0000","last_peered":"2023-10-01T09:03:06.346699+0000","last_clean":"2023-10-01T09:03:06.346699+0000","last_became_active":"2023-10-01T09:03:03.227153+0000","last_became_peered":"2023-10-01T09:03:03.227153+0000","last_unstale":"2023-10-01T09:03:06.346699+0000","last_undegraded":"2023-10-01T09:03:06.346699+0000","last_fullsized":"2023-10-01T09:03:06.346699+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.233224+0000","last_change":"2023-10-01T09:03:03.195227+0000","last_active":"2023-10-01T09:03:08.233224+0000","last_peered":"2023-10-01T09:03:08.233224+0000","last_clean":"2023-10-01T09:03:08.233224+0000","last_became_active":"2023-10-01T09:03:03.194964+0000","last_became_peered":"2023-10-01T09:03:03.194964+0000","last_unstale":"2023-10-01T09:03:08.233224+0000","last_undegraded":"2023-10-01T09:03:08.233224+0000","last_fullsized":"2023-10-01T09:03:08.233224+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"peering","last_fresh":"2023-10-01T09:03:45.634299+0000","last_change":"2023-10-01T09:03:45.634584+0000","last_active":"2023-10-01T09:03:45.634299+0000","last_peered":"2023-10-01T09:03:05.217171+0000","last_clean":"2023-10-01T09:03:05.217171+0000","last_became_active":"2023-10-01T09:03:03.220629+0000","last_became_peered":"2023-10-01T09:03:03.220629+0000","last_unstale":"2023-10-01T09:03:45.634299+0000","last_undegraded":"2023-10-01T09:03:45.634299+0000","last_fullsized":"2023-10-01T09:03:45.634299+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,6],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":18,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.234272+0000","last_change":"2023-10-01T09:03:03.200261+0000","last_active":"2023-10-01T09:03:08.234272+0000","last_peered":"2023-10-01T09:03:08.234272+0000","last_clean":"2023-10-01T09:03:08.234272+0000","last_became_active":"2023-10-01T09:03:03.199944+0000","last_became_peered":"2023-10-01T09:03:03.199944+0000","last_unstale":"2023-10-01T09:03:08.234272+0000","last_undegraded":"2023-10-01T09:03:08.234272+0000","last_fullsized":"2023-10-01T09:03:08.234272+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346705+0000","last_change":"2023-10-01T09:03:03.224723+0000","last_active":"2023-10-01T09:03:06.346705+0000","last_peered":"2023-10-01T09:03:06.346705+0000","last_clean":"2023-10-01T09:03:06.346705+0000","last_became_active":"2023-10-01T09:03:03.224379+0000","last_became_peered":"2023-10-01T09:03:03.224379+0000","last_unstale":"2023-10-01T09:03:06.346705+0000","last_undegraded":"2023-10-01T09:03:06.346705+0000","last_fullsized":"2023-10-01T09:03:06.346705+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.232126+0000","last_change":"2023-10-01T09:03:03.199740+0000","last_active":"2023-10-01T09:03:08.232126+0000","last_peered":"2023-10-01T09:03:08.232126+0000","last_clean":"2023-10-01T09:03:08.232126+0000","last_became_active":"2023-10-01T09:03:03.199421+0000","last_became_peered":"2023-10-01T09:03:03.199421+0000","last_unstale":"2023-10-01T09:03:08.232126+0000","last_undegraded":"2023-10-01T09:03:08.232126+0000","last_fullsized":"2023-10-01T09:03:08.232126+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346453+0000","last_change":"2023-10-01T09:03:03.228015+0000","last_active":"2023-10-01T09:03:06.346453+0000","last_peered":"2023-10-01T09:03:06.346453+0000","last_clean":"2023-10-01T09:03:06.346453+0000","last_became_active":"2023-10-01T09:03:03.227754+0000","last_became_peered":"2023-10-01T09:03:03.227754+0000","last_unstale":"2023-10-01T09:03:06.346453+0000","last_undegraded":"2023-10-01T09:03:06.346453+0000","last_fullsized":"2023-10-01T09:03:06.346453+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.237979+0000","last_change":"2023-10-01T09:03:03.201187+0000","last_active":"2023-10-01T09:03:05.237979+0000","last_peered":"2023-10-01T09:03:05.237979+0000","last_clean":"2023-10-01T09:03:05.237979+0000","last_became_active":"2023-10-01T09:03:03.200859+0000","last_became_peered":"2023-10-01T09:03:03.200859+0000","last_unstale":"2023-10-01T09:03:05.237979+0000","last_undegraded":"2023-10-01T09:03:05.237979+0000","last_fullsized":"2023-10-01T09:03:05.237979+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.345824+0000","last_change":"2023-10-01T09:03:03.227458+0000","last_active":"2023-10-01T09:03:06.345824+0000","last_peered":"2023-10-01T09:03:06.345824+0000","last_clean":"2023-10-01T09:03:06.345824+0000","last_became_active":"2023-10-01T09:03:03.226857+0000","last_became_peered":"2023-10-01T09:03:03.226857+0000","last_unstale":"2023-10-01T09:03:06.345824+0000","last_undegraded":"2023-10-01T09:03:06.345824+0000","last_fullsized":"2023-10-01T09:03:06.345824+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.210123+0000","last_change":"2023-10-01T09:03:03.202503+0000","last_active":"2023-10-01T09:03:04.210123+0000","last_peered":"2023-10-01T09:03:04.210123+0000","last_clean":"2023-10-01T09:03:04.210123+0000","last_became_active":"2023-10-01T09:03:03.202048+0000","last_became_peered":"2023-10-01T09:03:03.202048+0000","last_unstale":"2023-10-01T09:03:04.210123+0000","last_undegraded":"2023-10-01T09:03:04.210123+0000","last_fullsized":"2023-10-01T09:03:04.210123+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"peering","last_fresh":"2023-10-01T09:03:45.635167+0000","last_change":"2023-10-01T09:03:45.635422+0000","last_active":"2023-10-01T09:03:45.635167+0000","last_peered":"2023-10-01T09:03:05.696905+0000","last_clean":"2023-10-01T09:03:05.696905+0000","last_became_active":"2023-10-01T09:03:03.754613+0000","last_became_peered":"2023-10-01T09:03:03.754613+0000","last_unstale":"2023-10-01T09:03:45.635167+0000","last_undegraded":"2023-10-01T09:03:45.635167+0000","last_fullsized":"2023-10-01T09:03:45.635167+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.345854+0000","last_change":"2023-10-01T09:03:03.198727+0000","last_active":"2023-10-01T09:03:06.345854+0000","last_peered":"2023-10-01T09:03:06.345854+0000","last_clean":"2023-10-01T09:03:06.345854+0000","last_became_active":"2023-10-01T09:03:03.198309+0000","last_became_peered":"2023-10-01T09:03:03.198309+0000","last_unstale":"2023-10-01T09:03:06.345854+0000","last_undegraded":"2023-10-01T09:03:06.345854+0000","last_fullsized":"2023-10-01T09:03:06.345854+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.210849+0000","last_change":"2023-10-01T09:03:03.228290+0000","last_active":"2023-10-01T09:03:04.210849+0000","last_peered":"2023-10-01T09:03:04.210849+0000","last_clean":"2023-10-01T09:03:04.210849+0000","last_became_active":"2023-10-01T09:03:03.227612+0000","last_became_peered":"2023-10-01T09:03:03.227612+0000","last_unstale":"2023-10-01T09:03:04.210849+0000","last_undegraded":"2023-10-01T09:03:04.210849+0000","last_fullsized":"2023-10-01T09:03:04.210849+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346251+0000","last_change":"2023-10-01T09:03:03.199888+0000","last_active":"2023-10-01T09:03:06.346251+0000","last_peered":"2023-10-01T09:03:06.346251+0000","last_clean":"2023-10-01T09:03:06.346251+0000","last_became_active":"2023-10-01T09:03:03.199579+0000","last_became_peered":"2023-10-01T09:03:03.199579+0000","last_unstale":"2023-10-01T09:03:06.346251+0000","last_undegraded":"2023-10-01T09:03:06.346251+0000","last_fullsized":"2023-10-01T09:03:06.346251+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.347011+0000","last_change":"2023-10-01T09:03:03.226961+0000","last_active":"2023-10-01T09:03:06.347011+0000","last_peered":"2023-10-01T09:03:06.347011+0000","last_clean":"2023-10-01T09:03:06.347011+0000","last_became_active":"2023-10-01T09:03:03.226660+0000","last_became_peered":"2023-10-01T09:03:03.226660+0000","last_unstale":"2023-10-01T09:03:06.347011+0000","last_undegraded":"2023-10-01T09:03:06.347011+0000","last_fullsized":"2023-10-01T09:03:06.347011+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.230824+0000","last_change":"2023-10-01T09:03:03.201578+0000","last_active":"2023-10-01T09:03:08.230824+0000","last_peered":"2023-10-01T09:03:08.230824+0000","last_clean":"2023-10-01T09:03:08.230824+0000","last_became_active":"2023-10-01T09:03:03.201137+0000","last_became_peered":"2023-10-01T09:03:03.201137+0000","last_unstale":"2023-10-01T09:03:08.230824+0000","last_undegraded":"2023-10-01T09:03:08.230824+0000","last_fullsized":"2023-10-01T09:03:08.230824+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430238,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":257891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.54600000000000004,"15min":0.52200000000000002},"min":{"1min":0.34200000000000003,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.9580000000000002,"5min":2.9580000000000002,"15min":2.9580000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.503,"5min":0.48999999999999999,"15min":0.48799999999999999},"min":{"1min":0.39600000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.63800000000000001,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.36199999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.48699999999999999,"15min":0.48299999999999998},"min":{"1min":0.38600000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.63400000000000001,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.496},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.498,"15min":0.48399999999999999},"min":{"1min":0.36099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.47999999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.71499999999999997,"15min":0.745},"min":{"1min":0.36699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.91000000000000003,"5min":5.5949999999999998,"15min":5.5949999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.52100000000000002,"15min":0.50700000000000001},"min":{"1min":0.432,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.61599999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.69799999999999995,"15min":0.72399999999999998},"min":{"1min":0.40300000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68700000000000006,"5min":5.6210000000000004,"15min":5.6210000000000004},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.54500000000000004,"15min":0.52300000000000002},"min":{"1min":0.40699999999999997,"5min":0.378,"15min":0.378},"max":{"1min":2.2050000000000001,"5min":2.2050000000000001,"15min":2.2050000000000001},"last":0.59299999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.46500000000000002,"15min":0.45900000000000002},"min":{"1min":0.30299999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.438},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.68600000000000005,"15min":0.71399999999999997},"min":{"1min":0.38800000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.72099999999999997,"5min":5.5369999999999999,"15min":5.5369999999999999},"last":0.46100000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.51100000000000001,"15min":0.50700000000000001},"min":{"1min":0.33100000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.70599999999999996,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.53200000000000003,"15min":0.52800000000000002},"min":{"1min":0.35399999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.72799999999999998,"5min":0.746,"15min":0.746},"last":0.54800000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.55400000000000005,"15min":0.55000000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.74099999999999999,"15min":0.76700000000000002},"min":{"1min":0.40000000000000002,"5min":0.37,"15min":0.37},"max":{"1min":0.78100000000000003,"5min":5.5700000000000003,"15min":5.5700000000000003},"last":0.56699999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626467,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":257115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.54600000000000004,"15min":0.54800000000000004},"min":{"1min":0.36899999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.65200000000000002,"5min":0.752,"15min":0.752},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.36099999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.64900000000000002,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.40799999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53300000000000003,"15min":0.53500000000000003},"min":{"1min":0.39600000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65300000000000002,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.439},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.629,"15min":0.64400000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.70799999999999996,"5min":3.2429999999999999,"15min":3.2429999999999999},"last":0.503}]},{"osd":2,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.51100000000000001,"15min":0.51300000000000001},"min":{"1min":0.376,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.621,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53400000000000003,"15min":0.53700000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67400000000000004,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.35399999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54200000000000004,"15min":0.54300000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.41099999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.86699999999999999,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.58299999999999996}]},{"osd":4,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.48999999999999999,"15min":0.48599999999999999},"min":{"1min":0.34599999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.47299999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.504,"15min":0.50700000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.72699999999999998,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52900000000000003,"15min":0.52800000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.76300000000000001,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.57099999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.52000000000000002,"15min":0.51600000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.497},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.56899999999999995,"15min":0.56100000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77400000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.53100000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574905,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":262760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.36399999999999999,"15min":0.36499999999999999},"min":{"1min":0.23899999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.52000000000000002,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.32500000000000001,"5min":0.40000000000000002,"15min":0.42699999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.46600000000000003,"5min":2.2829999999999999,"15min":2.2829999999999999},"last":0.28899999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.28999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64400000000000002,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.45000000000000001,"15min":0.47599999999999998},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.63500000000000001,"5min":2.3199999999999998,"15min":2.3199999999999998},"last":0.34899999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.40999999999999998,"15min":0.41799999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.46000000000000002,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.45600000000000002,"15min":0.46700000000000003},"min":{"1min":0.33600000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.58399999999999996,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.40000000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:03:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.55600000000000005,"15min":0.55500000000000005},"min":{"1min":0.40200000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.86399999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58799999999999997,"15min":0.58199999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.86899999999999999,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.46999999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57199999999999995,"15min":0.57799999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.443},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.42599999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.54500000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.60999999999999999,"15min":0.621},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.85499999999999998,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.61599999999999999,"15min":0.627},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76300000000000001,"5min":1.03,"15min":1.03},"last":0.57099999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58099999999999996,"15min":0.58299999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77400000000000002,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.61899999999999999,"15min":0.623},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.82999999999999996,"5min":0.999,"15min":0.999},"last":0.503}]}]},{"osd":0,"up_from":8,"seq":34359738430,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34499999999999997,"15min":0.34300000000000003},"min":{"1min":0.245,"5min":0.22,"15min":0.22},"max":{"1min":0.46400000000000002,"5min":0.52900000000000003,"15min":0.52900000000000003},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.245,"5min":0.221,"15min":0.221},"max":{"1min":0.53600000000000003,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.29399999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:02:45 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.40699999999999997,"15min":0.40899999999999997},"min":{"1min":0.28399999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.499,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.377,"5min":0.38500000000000001,"15min":0.38400000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.55100000000000005,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.253}]},{"osd":3,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.41799999999999998,"15min":0.42999999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.51600000000000001,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.45700000000000002,"15min":0.47299999999999998},"min":{"1min":0.28399999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56999999999999995,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.32200000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.42599999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.80000000000000004,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.41299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.77100000000000002,"5min":0.87,"15min":0.87},"last":0.70099999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:02:51 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.57299999999999995,"15min":0.57699999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.70499999999999996,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.59099999999999997,"15min":0.59699999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.56399999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.62,"15min":0.624},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.60199999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.39900000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.749,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.46000000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.64700000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411380,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.374,"15min":0.376},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.51500000000000001,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.371,"15min":0.36199999999999999},"min":{"1min":0.249,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.45200000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.379,"15min":0.375},"min":{"1min":0.253,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.373,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.56599999999999995,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.5}]},{"osd":3,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.435,"15min":0.44600000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.499,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.435},{"interface":"front","average":{"1min":0.374,"5min":0.45400000000000001,"15min":0.47499999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.56999999999999995,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.61399999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:03:27 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54000000000000004,"15min":0.54100000000000004},"min":{"1min":0.434,"5min":0.374,"15min":0.374},"max":{"1min":0.65800000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.68200000000000005,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.76300000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:02:52 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56499999999999995,"15min":0.56799999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55900000000000005,"15min":0.56200000000000006},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.73399999999999999,"5min":0.749,"15min":0.749},"last":0.55300000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.59799999999999998,"15min":0.60499999999999998},"min":{"1min":0.44400000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.69899999999999995,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.60599999999999998,"15min":0.61199999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.73699999999999999,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.69099999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.58599999999999997,"15min":0.58899999999999997},"min":{"1min":0.47899999999999998,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.71599999999999997,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.60499999999999998,"15min":0.61099999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.70199999999999996,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.71799999999999997}]}]},{"osd":3,"up_from":25,"seq":107374182448,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297688,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447464,"statfs":{"total":95995035648,"available":95690203136,"internally_reserved":0,"allocated":606208,"data_stored":256229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.51,"5min":1.51,"15min":1.51},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.8599999999999999,"5min":2.8599999999999999,"15min":2.8599999999999999},"last":0.26000000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.42999999999999999,"15min":0.44800000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.503,"5min":0.62,"15min":0.62},"last":0.313},{"interface":"front","average":{"1min":0.37,"5min":0.40999999999999998,"15min":0.41899999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.52700000000000002,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.54000000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.44600000000000001,"15min":0.45800000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.54000000000000004,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.498},{"interface":"front","average":{"1min":0.38,"5min":0.437,"15min":0.45600000000000002},"min":{"1min":0.27200000000000002,"5min":0.25,"15min":0.25},"max":{"1min":0.59199999999999997,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.35499999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:03:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59499999999999997,"15min":0.59699999999999998},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.371,"5min":1.371,"15min":1.371},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.59699999999999998,"15min":0.60499999999999998},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.76700000000000002,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.60299999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56699999999999995,"15min":0.56399999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.60199999999999998,"15min":0.59599999999999997},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.50800000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:03:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.59899999999999998,"15min":0.60699999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.84199999999999997,"5min":1.389,"15min":1.389},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.45300000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.80000000000000004,"5min":1.278,"15min":1.278},"last":0.57199999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.57299999999999995,"15min":0.56799999999999995},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60399999999999998,"15min":0.60199999999999998},"min":{"1min":0.44,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.82499999999999996,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.47599999999999998}]}]},{"osd":4,"up_from":30,"seq":128849018923,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297720,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447432,"statfs":{"total":95995035648,"available":95690170368,"internally_reserved":0,"allocated":638976,"data_stored":264757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.55900000000000005,"15min":0.52500000000000002},"min":{"1min":0.34999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.4459999999999997,"5min":5.4459999999999997,"15min":5.4459999999999997},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.57799999999999996,"15min":0.52600000000000002},"min":{"1min":0.35999999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.1890000000000001,"5min":7.1890000000000001,"15min":7.1890000000000001},"last":0.627}]},{"osd":1,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51200000000000001,"15min":0.51100000000000001},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.53100000000000003,"15min":0.495},"min":{"1min":0.372,"5min":0.315,"15min":0.315},"max":{"1min":4.2279999999999998,"5min":4.2279999999999998,"15min":4.2279999999999998},"last":0.55400000000000005}]},{"osd":2,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.55800000000000005,"15min":0.52100000000000002},"min":{"1min":0.39300000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.3310000000000004,"5min":5.3310000000000004,"15min":5.3310000000000004},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.52000000000000002,"15min":0.51200000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.51500000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.55600000000000005,"15min":0.51700000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.6429999999999998,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.52300000000000002,"15min":0.51800000000000002},"min":{"1min":0.38600000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.1279999999999999,"5min":1.1279999999999999,"15min":1.1279999999999999},"last":0.59199999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:02:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.40200000000000002,"15min":0.39000000000000001},"min":{"1min":0.35799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.5,"5min":0.39000000000000001,"15min":0.372},"min":{"1min":0.39100000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.45700000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:03:14 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.45500000000000002,"15min":0.44500000000000001},"min":{"1min":0.34599999999999997,"5min":0.313,"15min":0.313},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.40200000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.56499999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:03:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.53700000000000003,"15min":0.54100000000000004},"min":{"1min":0.33000000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.55400000000000005,"15min":0.55500000000000005},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.61299999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822696,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297700,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447452,"statfs":{"total":95995035648,"available":95690190848,"internally_reserved":0,"allocated":618496,"data_stored":263384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.497,"15min":0.497},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.441},{"interface":"front","average":{"1min":0.497,"5min":0.51300000000000001,"15min":0.51600000000000001},"min":{"1min":0.34799999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.49299999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.49199999999999999,"15min":0.49399999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.55300000000000005,"15min":0.53500000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":4.0490000000000004,"5min":4.0490000000000004,"15min":4.0490000000000004},"last":0.54100000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.54100000000000004,"15min":0.54600000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.629},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.499,"15min":0.501},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.67600000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.51900000000000002,"15min":0.51700000000000002},"min":{"1min":0.379,"5min":0.377,"15min":0.377},"max":{"1min":0.67800000000000005,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.57699999999999996}]},{"osd":4,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.40100000000000002,"15min":0.39300000000000002},"min":{"1min":0.32300000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.42699999999999999,"15min":0.41199999999999998},"min":{"1min":0.30299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.58699999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:03:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.48699999999999999,"15min":0.47899999999999998},"min":{"1min":0.34100000000000003,"5min":0.313,"15min":0.313},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.47699999999999998,"15min":0.46500000000000002},"min":{"1min":0.35999999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.65000000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:03:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54000000000000004,"15min":0.53500000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.53400000000000003,"15min":0.52700000000000002},"min":{"1min":0.36499999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.64000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:03:47.356 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:03:47.648 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:47 smithi138 conmon[100289]: cluster 2023-10-01T09:03:46.633216+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-01T09:03:47.648 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:47 smithi138 conmon[100289]: audit 2023-10-01T09:03:46.707231+0000 mgr.y (mgr.14433) 94 : audit [DBG] from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:03:47.648 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:47 smithi138 conmon[100289]: cluster 2023-10-01T09:03:46.852554+0000 mgr.y (mgr.14433) 95 : cluster [DBG] pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:47.699 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:47.898 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:47 smithi138 conmon[109959]: cluster 2023-10-01T09:03:46.633216+0000 2023-10-01T09:03:47.899 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:47 smithi138 conmon[109959]: mon.a (mon.0) 678 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-01T09:03:47.899 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:47 smithi138 conmon[109959]: audit 2023-10-01T09:03:46.707231+0000 mgr.y (mgr.14433) 2023-10-01T09:03:47.899 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:47 smithi138 conmon[109959]: 94 : audit [DBG] from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:03:47.899 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:47 smithi138 conmon[109959]: cluster 2023-10-01T09:03:47.899 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:47 smithi138 conmon[109959]: 2023-10-01T09:03:46.852554 2023-10-01T09:03:47.899 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:47 smithi138 conmon[109959]: +0000 mgr.y (mgr.14433) 95 : cluster 2023-10-01T09:03:47.899 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:47 smithi138 conmon[109959]: [DBG] pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:48.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:47 smithi190 conmon[100518]: cluster 2023-10-01T09:03:46.633216+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-01T09:03:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:47 smithi190 conmon[100518]: audit 2023-10-01T09:03:46.707231+0000 mgr.y (mgr.14433) 94 : audit [DBG] from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:03:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:47 smithi190 conmon[100518]: cluster 2023-10-01T09:03:46.852554 2023-10-01T09:03:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:47 smithi190 conmon[100518]: +0000 mgr.y (mgr.14433) 95 : cluster [DBG] pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:49.262 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:03:49.264 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:03:49.864 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":76,"stamp":"2023-10-01T09:03:48.852801+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381588,"kb_used_data":4820,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579628,"statfs":{"total":767960285184,"available":765521539072,"internally_reserved":0,"allocated":4935680,"data_stored":2075578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001632"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.216254+0000","last_change":"2023-10-01T09:03:03.223370+0000","last_active":"2023-10-01T09:03:05.216254+0000","last_peered":"2023-10-01T09:03:05.216254+0000","last_clean":"2023-10-01T09:03:05.216254+0000","last_became_active":"2023-10-01T09:03:03.223071+0000","last_became_peered":"2023-10-01T09:03:03.223071+0000","last_unstale":"2023-10-01T09:03:05.216254+0000","last_undegraded":"2023-10-01T09:03:05.216254+0000","last_fullsized":"2023-10-01T09:03:05.216254+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.232525+0000","last_change":"2023-10-01T09:03:03.202335+0000","last_active":"2023-10-01T09:03:08.232525+0000","last_peered":"2023-10-01T09:03:08.232525+0000","last_clean":"2023-10-01T09:03:08.232525+0000","last_became_active":"2023-10-01T09:03:03.201485+0000","last_became_peered":"2023-10-01T09:03:03.201485+0000","last_unstale":"2023-10-01T09:03:08.232525+0000","last_undegraded":"2023-10-01T09:03:08.232525+0000","last_fullsized":"2023-10-01T09:03:08.232525+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346371+0000","last_change":"2023-10-01T09:03:03.224058+0000","last_active":"2023-10-01T09:03:06.346371+0000","last_peered":"2023-10-01T09:03:06.346371+0000","last_clean":"2023-10-01T09:03:06.346371+0000","last_became_active":"2023-10-01T09:03:03.223598+0000","last_became_peered":"2023-10-01T09:03:03.223598+0000","last_unstale":"2023-10-01T09:03:06.346371+0000","last_undegraded":"2023-10-01T09:03:06.346371+0000","last_fullsized":"2023-10-01T09:03:06.346371+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.233101+0000","last_change":"2023-10-01T09:03:03.193542+0000","last_active":"2023-10-01T09:03:08.233101+0000","last_peered":"2023-10-01T09:03:08.233101+0000","last_clean":"2023-10-01T09:03:08.233101+0000","last_became_active":"2023-10-01T09:03:03.193245+0000","last_became_peered":"2023-10-01T09:03:03.193245+0000","last_unstale":"2023-10-01T09:03:08.233101+0000","last_undegraded":"2023-10-01T09:03:08.233101+0000","last_fullsized":"2023-10-01T09:03:08.233101+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.216607+0000","last_change":"2023-10-01T09:03:03.218020+0000","last_active":"2023-10-01T09:03:05.216607+0000","last_peered":"2023-10-01T09:03:05.216607+0000","last_clean":"2023-10-01T09:03:05.216607+0000","last_became_active":"2023-10-01T09:03:03.217717+0000","last_became_peered":"2023-10-01T09:03:03.217717+0000","last_unstale":"2023-10-01T09:03:05.216607+0000","last_undegraded":"2023-10-01T09:03:05.216607+0000","last_fullsized":"2023-10-01T09:03:05.216607+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.232617+0000","last_change":"2023-10-01T09:03:03.202063+0000","last_active":"2023-10-01T09:03:08.232617+0000","last_peered":"2023-10-01T09:03:08.232617+0000","last_clean":"2023-10-01T09:03:08.232617+0000","last_became_active":"2023-10-01T09:03:03.201769+0000","last_became_peered":"2023-10-01T09:03:03.201769+0000","last_unstale":"2023-10-01T09:03:08.232617+0000","last_undegraded":"2023-10-01T09:03:08.232617+0000","last_fullsized":"2023-10-01T09:03:08.232617+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.217294+0000","last_change":"2023-10-01T09:03:03.221313+0000","last_active":"2023-10-01T09:03:05.217294+0000","last_peered":"2023-10-01T09:03:05.217294+0000","last_clean":"2023-10-01T09:03:05.217294+0000","last_became_active":"2023-10-01T09:03:03.221080+0000","last_became_peered":"2023-10-01T09:03:03.221080+0000","last_unstale":"2023-10-01T09:03:05.217294+0000","last_undegraded":"2023-10-01T09:03:05.217294+0000","last_fullsized":"2023-10-01T09:03:05.217294+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.232352+0000","last_change":"2023-10-01T09:03:03.201659+0000","last_active":"2023-10-01T09:03:08.232352+0000","last_peered":"2023-10-01T09:03:08.232352+0000","last_clean":"2023-10-01T09:03:08.232352+0000","last_became_active":"2023-10-01T09:03:03.201348+0000","last_became_peered":"2023-10-01T09:03:03.201348+0000","last_unstale":"2023-10-01T09:03:08.232352+0000","last_undegraded":"2023-10-01T09:03:08.232352+0000","last_fullsized":"2023-10-01T09:03:08.232352+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.214523+0000","last_change":"2023-10-01T09:03:03.226772+0000","last_active":"2023-10-01T09:03:05.214523+0000","last_peered":"2023-10-01T09:03:05.214523+0000","last_clean":"2023-10-01T09:03:05.214523+0000","last_became_active":"2023-10-01T09:03:03.226446+0000","last_became_peered":"2023-10-01T09:03:03.226446+0000","last_unstale":"2023-10-01T09:03:05.214523+0000","last_undegraded":"2023-10-01T09:03:05.214523+0000","last_fullsized":"2023-10-01T09:03:05.214523+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215833+0000","last_change":"2023-10-01T09:03:03.203002+0000","last_active":"2023-10-01T09:03:05.215833+0000","last_peered":"2023-10-01T09:03:05.215833+0000","last_clean":"2023-10-01T09:03:05.215833+0000","last_became_active":"2023-10-01T09:03:03.202571+0000","last_became_peered":"2023-10-01T09:03:03.202571+0000","last_unstale":"2023-10-01T09:03:05.215833+0000","last_undegraded":"2023-10-01T09:03:05.215833+0000","last_fullsized":"2023-10-01T09:03:05.215833+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.216590+0000","last_change":"2023-10-01T09:03:03.754839+0000","last_active":"2023-10-01T09:03:05.216590+0000","last_peered":"2023-10-01T09:03:05.216590+0000","last_clean":"2023-10-01T09:03:05.216590+0000","last_became_active":"2023-10-01T09:03:03.754506+0000","last_became_peered":"2023-10-01T09:03:03.754506+0000","last_unstale":"2023-10-01T09:03:05.216590+0000","last_undegraded":"2023-10-01T09:03:05.216590+0000","last_fullsized":"2023-10-01T09:03:05.216590+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215737+0000","last_change":"2023-10-01T09:03:03.196477+0000","last_active":"2023-10-01T09:03:05.215737+0000","last_peered":"2023-10-01T09:03:05.215737+0000","last_clean":"2023-10-01T09:03:05.215737+0000","last_became_active":"2023-10-01T09:03:03.196266+0000","last_became_peered":"2023-10-01T09:03:03.196266+0000","last_unstale":"2023-10-01T09:03:05.215737+0000","last_undegraded":"2023-10-01T09:03:05.215737+0000","last_fullsized":"2023-10-01T09:03:05.215737+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.696921+0000","last_change":"2023-10-01T09:02:06.997887+0000","last_active":"2023-10-01T09:03:05.696921+0000","last_peered":"2023-10-01T09:03:05.696921+0000","last_clean":"2023-10-01T09:03:05.696921+0000","last_became_active":"2023-10-01T09:02:06.997597+0000","last_became_peered":"2023-10-01T09:02:06.997597+0000","last_unstale":"2023-10-01T09:03:05.696921+0000","last_undegraded":"2023-10-01T09:03:05.696921+0000","last_fullsized":"2023-10-01T09:03:05.696921+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.345723+0000","last_change":"2023-10-01T09:03:03.228390+0000","last_active":"2023-10-01T09:03:06.345723+0000","last_peered":"2023-10-01T09:03:06.345723+0000","last_clean":"2023-10-01T09:03:06.345723+0000","last_became_active":"2023-10-01T09:03:03.227254+0000","last_became_peered":"2023-10-01T09:03:03.227254+0000","last_unstale":"2023-10-01T09:03:06.345723+0000","last_undegraded":"2023-10-01T09:03:06.345723+0000","last_fullsized":"2023-10-01T09:03:06.345723+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346408+0000","last_change":"2023-10-01T09:02:07.442083+0000","last_active":"2023-10-01T09:03:06.346408+0000","last_peered":"2023-10-01T09:03:06.346408+0000","last_clean":"2023-10-01T09:03:06.346408+0000","last_became_active":"2023-10-01T09:02:07.441757+0000","last_became_peered":"2023-10-01T09:02:07.441757+0000","last_unstale":"2023-10-01T09:03:06.346408+0000","last_undegraded":"2023-10-01T09:03:06.346408+0000","last_fullsized":"2023-10-01T09:03:06.346408+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.233194+0000","last_change":"2023-10-01T09:03:03.198510+0000","last_active":"2023-10-01T09:03:08.233194+0000","last_peered":"2023-10-01T09:03:08.233194+0000","last_clean":"2023-10-01T09:03:08.233194+0000","last_became_active":"2023-10-01T09:03:03.198172+0000","last_became_peered":"2023-10-01T09:03:03.198172+0000","last_unstale":"2023-10-01T09:03:08.233194+0000","last_undegraded":"2023-10-01T09:03:08.233194+0000","last_fullsized":"2023-10-01T09:03:08.233194+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.209663+0000","last_change":"2023-10-01T09:03:03.754828+0000","last_active":"2023-10-01T09:03:04.209663+0000","last_peered":"2023-10-01T09:03:04.209663+0000","last_clean":"2023-10-01T09:03:04.209663+0000","last_became_active":"2023-10-01T09:03:03.754492+0000","last_became_peered":"2023-10-01T09:03:03.754492+0000","last_unstale":"2023-10-01T09:03:04.209663+0000","last_undegraded":"2023-10-01T09:03:04.209663+0000","last_fullsized":"2023-10-01T09:03:04.209663+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.347115+0000","last_change":"2023-10-01T09:03:03.203345+0000","last_active":"2023-10-01T09:03:06.347115+0000","last_peered":"2023-10-01T09:03:06.347115+0000","last_clean":"2023-10-01T09:03:06.347115+0000","last_became_active":"2023-10-01T09:03:03.203035+0000","last_became_peered":"2023-10-01T09:03:03.203035+0000","last_unstale":"2023-10-01T09:03:06.347115+0000","last_undegraded":"2023-10-01T09:03:06.347115+0000","last_fullsized":"2023-10-01T09:03:06.347115+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.244297+0000","last_change":"2023-10-01T09:02:07.442452+0000","last_active":"2023-10-01T09:03:04.244297+0000","last_peered":"2023-10-01T09:03:04.244297+0000","last_clean":"2023-10-01T09:03:04.244297+0000","last_became_active":"2023-10-01T09:02:07.442111+0000","last_became_peered":"2023-10-01T09:02:07.442111+0000","last_unstale":"2023-10-01T09:03:04.244297+0000","last_undegraded":"2023-10-01T09:03:04.244297+0000","last_fullsized":"2023-10-01T09:03:04.244297+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.211021+0000","last_change":"2023-10-01T09:03:03.223326+0000","last_active":"2023-10-01T09:03:04.211021+0000","last_peered":"2023-10-01T09:03:04.211021+0000","last_clean":"2023-10-01T09:03:04.211021+0000","last_became_active":"2023-10-01T09:03:03.222974+0000","last_became_peered":"2023-10-01T09:03:03.222974+0000","last_unstale":"2023-10-01T09:03:04.211021+0000","last_undegraded":"2023-10-01T09:03:04.211021+0000","last_fullsized":"2023-10-01T09:03:04.211021+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.210759+0000","last_change":"2023-10-01T09:03:03.195712+0000","last_active":"2023-10-01T09:03:04.210759+0000","last_peered":"2023-10-01T09:03:04.210759+0000","last_clean":"2023-10-01T09:03:04.210759+0000","last_became_active":"2023-10-01T09:03:03.195408+0000","last_became_peered":"2023-10-01T09:03:03.195408+0000","last_unstale":"2023-10-01T09:03:04.210759+0000","last_undegraded":"2023-10-01T09:03:04.210759+0000","last_fullsized":"2023-10-01T09:03:04.210759+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.244156+0000","last_change":"2023-10-01T09:02:06.999043+0000","last_active":"2023-10-01T09:03:04.244156+0000","last_peered":"2023-10-01T09:03:04.244156+0000","last_clean":"2023-10-01T09:03:04.244156+0000","last_became_active":"2023-10-01T09:02:06.998671+0000","last_became_peered":"2023-10-01T09:02:06.998671+0000","last_unstale":"2023-10-01T09:03:04.244156+0000","last_undegraded":"2023-10-01T09:03:04.244156+0000","last_fullsized":"2023-10-01T09:03:04.244156+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.696828+0000","last_change":"2023-10-01T09:03:03.227254+0000","last_active":"2023-10-01T09:03:05.696828+0000","last_peered":"2023-10-01T09:03:05.696828+0000","last_clean":"2023-10-01T09:03:05.696828+0000","last_became_active":"2023-10-01T09:03:03.226941+0000","last_became_peered":"2023-10-01T09:03:03.226941+0000","last_unstale":"2023-10-01T09:03:05.696828+0000","last_undegraded":"2023-10-01T09:03:05.696828+0000","last_fullsized":"2023-10-01T09:03:05.696828+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.345884+0000","last_change":"2023-10-01T09:03:03.198835+0000","last_active":"2023-10-01T09:03:06.345884+0000","last_peered":"2023-10-01T09:03:06.345884+0000","last_clean":"2023-10-01T09:03:06.345884+0000","last_became_active":"2023-10-01T09:03:03.198424+0000","last_became_peered":"2023-10-01T09:03:03.198424+0000","last_unstale":"2023-10-01T09:03:06.345884+0000","last_undegraded":"2023-10-01T09:03:06.345884+0000","last_fullsized":"2023-10-01T09:03:06.345884+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.244204+0000","last_change":"2023-10-01T09:01:32.510259+0000","last_active":"2023-10-01T09:03:04.244204+0000","last_peered":"2023-10-01T09:03:04.244204+0000","last_clean":"2023-10-01T09:03:04.244204+0000","last_became_active":"2023-10-01T09:01:32.509905+0000","last_became_peered":"2023-10-01T09:01:32.509905+0000","last_unstale":"2023-10-01T09:03:04.244204+0000","last_undegraded":"2023-10-01T09:03:04.244204+0000","last_fullsized":"2023-10-01T09:03:04.244204+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.237819+0000","last_change":"2023-10-01T09:02:06.998804+0000","last_active":"2023-10-01T09:03:05.237819+0000","last_peered":"2023-10-01T09:03:05.237819+0000","last_clean":"2023-10-01T09:03:05.237819+0000","last_became_active":"2023-10-01T09:02:06.998516+0000","last_became_peered":"2023-10-01T09:02:06.998516+0000","last_unstale":"2023-10-01T09:03:05.237819+0000","last_undegraded":"2023-10-01T09:03:05.237819+0000","last_fullsized":"2023-10-01T09:03:05.237819+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346495+0000","last_change":"2023-10-01T09:03:03.756021+0000","last_active":"2023-10-01T09:03:06.346495+0000","last_peered":"2023-10-01T09:03:06.346495+0000","last_clean":"2023-10-01T09:03:06.346495+0000","last_became_active":"2023-10-01T09:03:03.755659+0000","last_became_peered":"2023-10-01T09:03:03.755659+0000","last_unstale":"2023-10-01T09:03:06.346495+0000","last_undegraded":"2023-10-01T09:03:06.346495+0000","last_fullsized":"2023-10-01T09:03:06.346495+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346914+0000","last_change":"2023-10-01T09:03:03.201620+0000","last_active":"2023-10-01T09:03:06.346914+0000","last_peered":"2023-10-01T09:03:06.346914+0000","last_clean":"2023-10-01T09:03:06.346914+0000","last_became_active":"2023-10-01T09:03:03.201312+0000","last_became_peered":"2023-10-01T09:03:03.201312+0000","last_unstale":"2023-10-01T09:03:06.346914+0000","last_undegraded":"2023-10-01T09:03:06.346914+0000","last_fullsized":"2023-10-01T09:03:06.346914+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.214376+0000","last_change":"2023-10-01T09:02:07.436492+0000","last_active":"2023-10-01T09:03:05.214376+0000","last_peered":"2023-10-01T09:03:05.214376+0000","last_clean":"2023-10-01T09:03:05.214376+0000","last_became_active":"2023-10-01T09:02:07.436146+0000","last_became_peered":"2023-10-01T09:02:07.436146+0000","last_unstale":"2023-10-01T09:03:05.214376+0000","last_undegraded":"2023-10-01T09:03:05.214376+0000","last_fullsized":"2023-10-01T09:03:05.214376+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.347049+0000","last_change":"2023-10-01T09:03:03.227284+0000","last_active":"2023-10-01T09:03:06.347049+0000","last_peered":"2023-10-01T09:03:06.347049+0000","last_clean":"2023-10-01T09:03:06.347049+0000","last_became_active":"2023-10-01T09:03:03.226987+0000","last_became_peered":"2023-10-01T09:03:03.226987+0000","last_unstale":"2023-10-01T09:03:06.347049+0000","last_undegraded":"2023-10-01T09:03:06.347049+0000","last_fullsized":"2023-10-01T09:03:06.347049+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.217322+0000","last_change":"2023-10-01T09:03:03.196357+0000","last_active":"2023-10-01T09:03:05.217322+0000","last_peered":"2023-10-01T09:03:05.217322+0000","last_clean":"2023-10-01T09:03:05.217322+0000","last_became_active":"2023-10-01T09:03:03.196017+0000","last_became_peered":"2023-10-01T09:03:03.196017+0000","last_unstale":"2023-10-01T09:03:05.217322+0000","last_undegraded":"2023-10-01T09:03:05.217322+0000","last_fullsized":"2023-10-01T09:03:05.217322+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.237755+0000","last_change":"2023-10-01T09:02:07.001644+0000","last_active":"2023-10-01T09:03:05.237755+0000","last_peered":"2023-10-01T09:03:05.237755+0000","last_clean":"2023-10-01T09:03:05.237755+0000","last_became_active":"2023-10-01T09:02:07.001197+0000","last_became_peered":"2023-10-01T09:02:07.001197+0000","last_unstale":"2023-10-01T09:03:05.237755+0000","last_undegraded":"2023-10-01T09:03:05.237755+0000","last_fullsized":"2023-10-01T09:03:05.237755+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346759+0000","last_change":"2023-10-01T09:03:03.755027+0000","last_active":"2023-10-01T09:03:06.346759+0000","last_peered":"2023-10-01T09:03:06.346759+0000","last_clean":"2023-10-01T09:03:06.346759+0000","last_became_active":"2023-10-01T09:03:03.754596+0000","last_became_peered":"2023-10-01T09:03:03.754596+0000","last_unstale":"2023-10-01T09:03:06.346759+0000","last_undegraded":"2023-10-01T09:03:06.346759+0000","last_fullsized":"2023-10-01T09:03:06.346759+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.237997+0000","last_change":"2023-10-01T09:03:03.203012+0000","last_active":"2023-10-01T09:03:05.237997+0000","last_peered":"2023-10-01T09:03:05.237997+0000","last_clean":"2023-10-01T09:03:05.237997+0000","last_became_active":"2023-10-01T09:03:03.202051+0000","last_became_peered":"2023-10-01T09:03:03.202051+0000","last_unstale":"2023-10-01T09:03:05.237997+0000","last_undegraded":"2023-10-01T09:03:05.237997+0000","last_fullsized":"2023-10-01T09:03:05.237997+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346979+0000","last_change":"2023-10-01T09:02:06.998188+0000","last_active":"2023-10-01T09:03:06.346979+0000","last_peered":"2023-10-01T09:03:06.346979+0000","last_clean":"2023-10-01T09:03:06.346979+0000","last_became_active":"2023-10-01T09:02:06.997855+0000","last_became_peered":"2023-10-01T09:02:06.997855+0000","last_unstale":"2023-10-01T09:03:06.346979+0000","last_undegraded":"2023-10-01T09:03:06.346979+0000","last_fullsized":"2023-10-01T09:03:06.346979+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1,"reported_epoch":61,"state":"peering","last_fresh":"2023-10-01T09:03:45.633611+0000","last_change":"2023-10-01T09:03:45.633611+0000","last_active":"0.000000","last_peered":"0.000000","last_clean":"0.000000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:03:45.633611+0000","last_undegraded":"2023-10-01T09:03:45.633611+0000","last_fullsized":"2023-10-01T09:03:45.633611+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.345762+0000","last_change":"2023-10-01T09:03:03.200838+0000","last_active":"2023-10-01T09:03:06.345762+0000","last_peered":"2023-10-01T09:03:06.345762+0000","last_clean":"2023-10-01T09:03:06.345762+0000","last_became_active":"2023-10-01T09:03:03.199715+0000","last_became_peered":"2023-10-01T09:03:03.199715+0000","last_unstale":"2023-10-01T09:03:06.345762+0000","last_undegraded":"2023-10-01T09:03:06.345762+0000","last_fullsized":"2023-10-01T09:03:06.345762+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.217059+0000","last_change":"2023-10-01T09:03:03.221266+0000","last_active":"2023-10-01T09:03:05.217059+0000","last_peered":"2023-10-01T09:03:05.217059+0000","last_clean":"2023-10-01T09:03:05.217059+0000","last_became_active":"2023-10-01T09:03:03.220690+0000","last_became_peered":"2023-10-01T09:03:03.220690+0000","last_unstale":"2023-10-01T09:03:05.217059+0000","last_undegraded":"2023-10-01T09:03:05.217059+0000","last_fullsized":"2023-10-01T09:03:05.217059+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.216742+0000","last_change":"2023-10-01T09:03:03.199237+0000","last_active":"2023-10-01T09:03:05.216742+0000","last_peered":"2023-10-01T09:03:05.216742+0000","last_clean":"2023-10-01T09:03:05.216742+0000","last_became_active":"2023-10-01T09:03:03.198950+0000","last_became_peered":"2023-10-01T09:03:03.198950+0000","last_unstale":"2023-10-01T09:03:05.216742+0000","last_undegraded":"2023-10-01T09:03:05.216742+0000","last_fullsized":"2023-10-01T09:03:05.216742+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.217019+0000","last_change":"2023-10-01T09:03:03.223290+0000","last_active":"2023-10-01T09:03:05.217019+0000","last_peered":"2023-10-01T09:03:05.217019+0000","last_clean":"2023-10-01T09:03:05.217019+0000","last_became_active":"2023-10-01T09:03:03.223013+0000","last_became_peered":"2023-10-01T09:03:03.223013+0000","last_unstale":"2023-10-01T09:03:05.217019+0000","last_undegraded":"2023-10-01T09:03:05.217019+0000","last_fullsized":"2023-10-01T09:03:05.217019+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.696678+0000","last_change":"2023-10-01T09:03:03.198744+0000","last_active":"2023-10-01T09:03:05.696678+0000","last_peered":"2023-10-01T09:03:05.696678+0000","last_clean":"2023-10-01T09:03:05.696678+0000","last_became_active":"2023-10-01T09:03:03.198452+0000","last_became_peered":"2023-10-01T09:03:03.198452+0000","last_unstale":"2023-10-01T09:03:05.696678+0000","last_undegraded":"2023-10-01T09:03:05.696678+0000","last_fullsized":"2023-10-01T09:03:05.696678+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.216811+0000","last_change":"2023-10-01T09:03:03.218071+0000","last_active":"2023-10-01T09:03:05.216811+0000","last_peered":"2023-10-01T09:03:05.216811+0000","last_clean":"2023-10-01T09:03:05.216811+0000","last_became_active":"2023-10-01T09:03:03.217776+0000","last_became_peered":"2023-10-01T09:03:03.217776+0000","last_unstale":"2023-10-01T09:03:05.216811+0000","last_undegraded":"2023-10-01T09:03:05.216811+0000","last_fullsized":"2023-10-01T09:03:05.216811+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.233311+0000","last_change":"2023-10-01T09:03:03.200515+0000","last_active":"2023-10-01T09:03:08.233311+0000","last_peered":"2023-10-01T09:03:08.233311+0000","last_clean":"2023-10-01T09:03:08.233311+0000","last_became_active":"2023-10-01T09:03:03.199536+0000","last_became_peered":"2023-10-01T09:03:03.199536+0000","last_unstale":"2023-10-01T09:03:08.233311+0000","last_undegraded":"2023-10-01T09:03:08.233311+0000","last_fullsized":"2023-10-01T09:03:08.233311+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.209902+0000","last_change":"2023-10-01T09:03:03.224884+0000","last_active":"2023-10-01T09:03:04.209902+0000","last_peered":"2023-10-01T09:03:04.209902+0000","last_clean":"2023-10-01T09:03:04.209902+0000","last_became_active":"2023-10-01T09:03:03.224529+0000","last_became_peered":"2023-10-01T09:03:03.224529+0000","last_unstale":"2023-10-01T09:03:04.209902+0000","last_undegraded":"2023-10-01T09:03:04.209902+0000","last_fullsized":"2023-10-01T09:03:04.209902+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.237860+0000","last_change":"2023-10-01T09:03:03.203721+0000","last_active":"2023-10-01T09:03:05.237860+0000","last_peered":"2023-10-01T09:03:05.237860+0000","last_clean":"2023-10-01T09:03:05.237860+0000","last_became_active":"2023-10-01T09:03:03.203308+0000","last_became_peered":"2023-10-01T09:03:03.203308+0000","last_unstale":"2023-10-01T09:03:05.237860+0000","last_undegraded":"2023-10-01T09:03:05.237860+0000","last_fullsized":"2023-10-01T09:03:05.237860+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.696616+0000","last_change":"2023-10-01T09:03:03.227070+0000","last_active":"2023-10-01T09:03:05.696616+0000","last_peered":"2023-10-01T09:03:05.696616+0000","last_clean":"2023-10-01T09:03:05.696616+0000","last_became_active":"2023-10-01T09:03:03.226752+0000","last_became_peered":"2023-10-01T09:03:03.226752+0000","last_unstale":"2023-10-01T09:03:05.696616+0000","last_undegraded":"2023-10-01T09:03:05.696616+0000","last_fullsized":"2023-10-01T09:03:05.696616+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346853+0000","last_change":"2023-10-01T09:03:03.200972+0000","last_active":"2023-10-01T09:03:06.346853+0000","last_peered":"2023-10-01T09:03:06.346853+0000","last_clean":"2023-10-01T09:03:06.346853+0000","last_became_active":"2023-10-01T09:03:03.200695+0000","last_became_peered":"2023-10-01T09:03:03.200695+0000","last_unstale":"2023-10-01T09:03:06.346853+0000","last_undegraded":"2023-10-01T09:03:06.346853+0000","last_fullsized":"2023-10-01T09:03:06.346853+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.216573+0000","last_change":"2023-10-01T09:03:03.220741+0000","last_active":"2023-10-01T09:03:05.216573+0000","last_peered":"2023-10-01T09:03:05.216573+0000","last_clean":"2023-10-01T09:03:05.216573+0000","last_became_active":"2023-10-01T09:03:03.220481+0000","last_became_peered":"2023-10-01T09:03:03.220481+0000","last_unstale":"2023-10-01T09:03:05.216573+0000","last_undegraded":"2023-10-01T09:03:05.216573+0000","last_fullsized":"2023-10-01T09:03:05.216573+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.231170+0000","last_change":"2023-10-01T09:03:03.200367+0000","last_active":"2023-10-01T09:03:08.231170+0000","last_peered":"2023-10-01T09:03:08.231170+0000","last_clean":"2023-10-01T09:03:08.231170+0000","last_became_active":"2023-10-01T09:03:03.200068+0000","last_became_peered":"2023-10-01T09:03:03.200068+0000","last_unstale":"2023-10-01T09:03:08.231170+0000","last_undegraded":"2023-10-01T09:03:08.231170+0000","last_fullsized":"2023-10-01T09:03:08.231170+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.237727+0000","last_change":"2023-10-01T09:03:04.245105+0000","last_active":"2023-10-01T09:03:05.237727+0000","last_peered":"2023-10-01T09:03:05.237727+0000","last_clean":"2023-10-01T09:03:05.237727+0000","last_became_active":"2023-10-01T09:03:04.244807+0000","last_became_peered":"2023-10-01T09:03:04.244807+0000","last_unstale":"2023-10-01T09:03:05.237727+0000","last_undegraded":"2023-10-01T09:03:05.237727+0000","last_fullsized":"2023-10-01T09:03:05.237727+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346958+0000","last_change":"2023-10-01T09:03:03.199109+0000","last_active":"2023-10-01T09:03:06.346958+0000","last_peered":"2023-10-01T09:03:06.346958+0000","last_clean":"2023-10-01T09:03:06.346958+0000","last_became_active":"2023-10-01T09:03:03.198810+0000","last_became_peered":"2023-10-01T09:03:03.198810+0000","last_unstale":"2023-10-01T09:03:06.346958+0000","last_undegraded":"2023-10-01T09:03:06.346958+0000","last_fullsized":"2023-10-01T09:03:06.346958+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346070+0000","last_change":"2023-10-01T09:03:03.227888+0000","last_active":"2023-10-01T09:03:06.346070+0000","last_peered":"2023-10-01T09:03:06.346070+0000","last_clean":"2023-10-01T09:03:06.346070+0000","last_became_active":"2023-10-01T09:03:03.227621+0000","last_became_peered":"2023-10-01T09:03:03.227621+0000","last_unstale":"2023-10-01T09:03:06.346070+0000","last_undegraded":"2023-10-01T09:03:06.346070+0000","last_fullsized":"2023-10-01T09:03:06.346070+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":21,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.233440+0000","last_change":"2023-10-01T09:03:03.199462+0000","last_active":"2023-10-01T09:03:08.233440+0000","last_peered":"2023-10-01T09:03:08.233440+0000","last_clean":"2023-10-01T09:03:08.233440+0000","last_became_active":"2023-10-01T09:03:03.199188+0000","last_became_peered":"2023-10-01T09:03:03.199188+0000","last_unstale":"2023-10-01T09:03:08.233440+0000","last_undegraded":"2023-10-01T09:03:08.233440+0000","last_fullsized":"2023-10-01T09:03:08.233440+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.211314+0000","last_change":"2023-10-01T09:03:03.228462+0000","last_active":"2023-10-01T09:03:04.211314+0000","last_peered":"2023-10-01T09:03:04.211314+0000","last_clean":"2023-10-01T09:03:04.211314+0000","last_became_active":"2023-10-01T09:03:03.228010+0000","last_became_peered":"2023-10-01T09:03:03.228010+0000","last_unstale":"2023-10-01T09:03:04.211314+0000","last_undegraded":"2023-10-01T09:03:04.211314+0000","last_fullsized":"2023-10-01T09:03:04.211314+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.244093+0000","last_change":"2023-10-01T09:03:03.194952+0000","last_active":"2023-10-01T09:03:04.244093+0000","last_peered":"2023-10-01T09:03:04.244093+0000","last_clean":"2023-10-01T09:03:04.244093+0000","last_became_active":"2023-10-01T09:03:03.194643+0000","last_became_peered":"2023-10-01T09:03:03.194643+0000","last_unstale":"2023-10-01T09:03:04.244093+0000","last_undegraded":"2023-10-01T09:03:04.244093+0000","last_fullsized":"2023-10-01T09:03:04.244093+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.237929+0000","last_change":"2023-10-01T09:03:04.405231+0000","last_active":"2023-10-01T09:03:05.237929+0000","last_peered":"2023-10-01T09:03:05.237929+0000","last_clean":"2023-10-01T09:03:05.237929+0000","last_became_active":"2023-10-01T09:03:04.404876+0000","last_became_peered":"2023-10-01T09:03:04.404876+0000","last_unstale":"2023-10-01T09:03:05.237929+0000","last_undegraded":"2023-10-01T09:03:05.237929+0000","last_fullsized":"2023-10-01T09:03:05.237929+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.209915+0000","last_change":"2023-10-01T09:03:03.202905+0000","last_active":"2023-10-01T09:03:04.209915+0000","last_peered":"2023-10-01T09:03:04.209915+0000","last_clean":"2023-10-01T09:03:04.209915+0000","last_became_active":"2023-10-01T09:03:03.202621+0000","last_became_peered":"2023-10-01T09:03:03.202621+0000","last_unstale":"2023-10-01T09:03:04.209915+0000","last_undegraded":"2023-10-01T09:03:04.209915+0000","last_fullsized":"2023-10-01T09:03:04.209915+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346699+0000","last_change":"2023-10-01T09:03:03.228053+0000","last_active":"2023-10-01T09:03:06.346699+0000","last_peered":"2023-10-01T09:03:06.346699+0000","last_clean":"2023-10-01T09:03:06.346699+0000","last_became_active":"2023-10-01T09:03:03.227153+0000","last_became_peered":"2023-10-01T09:03:03.227153+0000","last_unstale":"2023-10-01T09:03:06.346699+0000","last_undegraded":"2023-10-01T09:03:06.346699+0000","last_fullsized":"2023-10-01T09:03:06.346699+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.233224+0000","last_change":"2023-10-01T09:03:03.195227+0000","last_active":"2023-10-01T09:03:08.233224+0000","last_peered":"2023-10-01T09:03:08.233224+0000","last_clean":"2023-10-01T09:03:08.233224+0000","last_became_active":"2023-10-01T09:03:03.194964+0000","last_became_peered":"2023-10-01T09:03:03.194964+0000","last_unstale":"2023-10-01T09:03:08.233224+0000","last_undegraded":"2023-10-01T09:03:08.233224+0000","last_fullsized":"2023-10-01T09:03:08.233224+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"peering","last_fresh":"2023-10-01T09:03:45.634299+0000","last_change":"2023-10-01T09:03:45.634584+0000","last_active":"2023-10-01T09:03:45.634299+0000","last_peered":"2023-10-01T09:03:05.217171+0000","last_clean":"2023-10-01T09:03:05.217171+0000","last_became_active":"2023-10-01T09:03:03.220629+0000","last_became_peered":"2023-10-01T09:03:03.220629+0000","last_unstale":"2023-10-01T09:03:45.634299+0000","last_undegraded":"2023-10-01T09:03:45.634299+0000","last_fullsized":"2023-10-01T09:03:45.634299+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,6],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":18,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.234272+0000","last_change":"2023-10-01T09:03:03.200261+0000","last_active":"2023-10-01T09:03:08.234272+0000","last_peered":"2023-10-01T09:03:08.234272+0000","last_clean":"2023-10-01T09:03:08.234272+0000","last_became_active":"2023-10-01T09:03:03.199944+0000","last_became_peered":"2023-10-01T09:03:03.199944+0000","last_unstale":"2023-10-01T09:03:08.234272+0000","last_undegraded":"2023-10-01T09:03:08.234272+0000","last_fullsized":"2023-10-01T09:03:08.234272+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346705+0000","last_change":"2023-10-01T09:03:03.224723+0000","last_active":"2023-10-01T09:03:06.346705+0000","last_peered":"2023-10-01T09:03:06.346705+0000","last_clean":"2023-10-01T09:03:06.346705+0000","last_became_active":"2023-10-01T09:03:03.224379+0000","last_became_peered":"2023-10-01T09:03:03.224379+0000","last_unstale":"2023-10-01T09:03:06.346705+0000","last_undegraded":"2023-10-01T09:03:06.346705+0000","last_fullsized":"2023-10-01T09:03:06.346705+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.232126+0000","last_change":"2023-10-01T09:03:03.199740+0000","last_active":"2023-10-01T09:03:08.232126+0000","last_peered":"2023-10-01T09:03:08.232126+0000","last_clean":"2023-10-01T09:03:08.232126+0000","last_became_active":"2023-10-01T09:03:03.199421+0000","last_became_peered":"2023-10-01T09:03:03.199421+0000","last_unstale":"2023-10-01T09:03:08.232126+0000","last_undegraded":"2023-10-01T09:03:08.232126+0000","last_fullsized":"2023-10-01T09:03:08.232126+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346453+0000","last_change":"2023-10-01T09:03:03.228015+0000","last_active":"2023-10-01T09:03:06.346453+0000","last_peered":"2023-10-01T09:03:06.346453+0000","last_clean":"2023-10-01T09:03:06.346453+0000","last_became_active":"2023-10-01T09:03:03.227754+0000","last_became_peered":"2023-10-01T09:03:03.227754+0000","last_unstale":"2023-10-01T09:03:06.346453+0000","last_undegraded":"2023-10-01T09:03:06.346453+0000","last_fullsized":"2023-10-01T09:03:06.346453+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.237979+0000","last_change":"2023-10-01T09:03:03.201187+0000","last_active":"2023-10-01T09:03:05.237979+0000","last_peered":"2023-10-01T09:03:05.237979+0000","last_clean":"2023-10-01T09:03:05.237979+0000","last_became_active":"2023-10-01T09:03:03.200859+0000","last_became_peered":"2023-10-01T09:03:03.200859+0000","last_unstale":"2023-10-01T09:03:05.237979+0000","last_undegraded":"2023-10-01T09:03:05.237979+0000","last_fullsized":"2023-10-01T09:03:05.237979+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.345824+0000","last_change":"2023-10-01T09:03:03.227458+0000","last_active":"2023-10-01T09:03:06.345824+0000","last_peered":"2023-10-01T09:03:06.345824+0000","last_clean":"2023-10-01T09:03:06.345824+0000","last_became_active":"2023-10-01T09:03:03.226857+0000","last_became_peered":"2023-10-01T09:03:03.226857+0000","last_unstale":"2023-10-01T09:03:06.345824+0000","last_undegraded":"2023-10-01T09:03:06.345824+0000","last_fullsized":"2023-10-01T09:03:06.345824+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.210123+0000","last_change":"2023-10-01T09:03:03.202503+0000","last_active":"2023-10-01T09:03:04.210123+0000","last_peered":"2023-10-01T09:03:04.210123+0000","last_clean":"2023-10-01T09:03:04.210123+0000","last_became_active":"2023-10-01T09:03:03.202048+0000","last_became_peered":"2023-10-01T09:03:03.202048+0000","last_unstale":"2023-10-01T09:03:04.210123+0000","last_undegraded":"2023-10-01T09:03:04.210123+0000","last_fullsized":"2023-10-01T09:03:04.210123+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"peering","last_fresh":"2023-10-01T09:03:45.635167+0000","last_change":"2023-10-01T09:03:45.635422+0000","last_active":"2023-10-01T09:03:45.635167+0000","last_peered":"2023-10-01T09:03:05.696905+0000","last_clean":"2023-10-01T09:03:05.696905+0000","last_became_active":"2023-10-01T09:03:03.754613+0000","last_became_peered":"2023-10-01T09:03:03.754613+0000","last_unstale":"2023-10-01T09:03:45.635167+0000","last_undegraded":"2023-10-01T09:03:45.635167+0000","last_fullsized":"2023-10-01T09:03:45.635167+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.345854+0000","last_change":"2023-10-01T09:03:03.198727+0000","last_active":"2023-10-01T09:03:06.345854+0000","last_peered":"2023-10-01T09:03:06.345854+0000","last_clean":"2023-10-01T09:03:06.345854+0000","last_became_active":"2023-10-01T09:03:03.198309+0000","last_became_peered":"2023-10-01T09:03:03.198309+0000","last_unstale":"2023-10-01T09:03:06.345854+0000","last_undegraded":"2023-10-01T09:03:06.345854+0000","last_fullsized":"2023-10-01T09:03:06.345854+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-10-01T09:03:04.210849+0000","last_change":"2023-10-01T09:03:03.228290+0000","last_active":"2023-10-01T09:03:04.210849+0000","last_peered":"2023-10-01T09:03:04.210849+0000","last_clean":"2023-10-01T09:03:04.210849+0000","last_became_active":"2023-10-01T09:03:03.227612+0000","last_became_peered":"2023-10-01T09:03:03.227612+0000","last_unstale":"2023-10-01T09:03:04.210849+0000","last_undegraded":"2023-10-01T09:03:04.210849+0000","last_fullsized":"2023-10-01T09:03:04.210849+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.346251+0000","last_change":"2023-10-01T09:03:03.199888+0000","last_active":"2023-10-01T09:03:06.346251+0000","last_peered":"2023-10-01T09:03:06.346251+0000","last_clean":"2023-10-01T09:03:06.346251+0000","last_became_active":"2023-10-01T09:03:03.199579+0000","last_became_peered":"2023-10-01T09:03:03.199579+0000","last_unstale":"2023-10-01T09:03:06.346251+0000","last_undegraded":"2023-10-01T09:03:06.346251+0000","last_fullsized":"2023-10-01T09:03:06.346251+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:06.347011+0000","last_change":"2023-10-01T09:03:03.226961+0000","last_active":"2023-10-01T09:03:06.347011+0000","last_peered":"2023-10-01T09:03:06.347011+0000","last_clean":"2023-10-01T09:03:06.347011+0000","last_became_active":"2023-10-01T09:03:03.226660+0000","last_became_peered":"2023-10-01T09:03:03.226660+0000","last_unstale":"2023-10-01T09:03:06.347011+0000","last_undegraded":"2023-10-01T09:03:06.347011+0000","last_fullsized":"2023-10-01T09:03:06.347011+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.230824+0000","last_change":"2023-10-01T09:03:03.201578+0000","last_active":"2023-10-01T09:03:08.230824+0000","last_peered":"2023-10-01T09:03:08.230824+0000","last_clean":"2023-10-01T09:03:08.230824+0000","last_became_active":"2023-10-01T09:03:03.201137+0000","last_became_peered":"2023-10-01T09:03:03.201137+0000","last_unstale":"2023-10-01T09:03:08.230824+0000","last_undegraded":"2023-10-01T09:03:08.230824+0000","last_fullsized":"2023-10-01T09:03:08.230824+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430238,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":257891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.54600000000000004,"15min":0.52200000000000002},"min":{"1min":0.34200000000000003,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.9580000000000002,"5min":2.9580000000000002,"15min":2.9580000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.503,"5min":0.48999999999999999,"15min":0.48799999999999999},"min":{"1min":0.39600000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.63800000000000001,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.36199999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.48699999999999999,"15min":0.48299999999999998},"min":{"1min":0.38600000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.63400000000000001,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.496},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.498,"15min":0.48399999999999999},"min":{"1min":0.36099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.47999999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.71499999999999997,"15min":0.745},"min":{"1min":0.36699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.91000000000000003,"5min":5.5949999999999998,"15min":5.5949999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.52100000000000002,"15min":0.50700000000000001},"min":{"1min":0.432,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.61599999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.69799999999999995,"15min":0.72399999999999998},"min":{"1min":0.40300000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68700000000000006,"5min":5.6210000000000004,"15min":5.6210000000000004},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.54500000000000004,"15min":0.52300000000000002},"min":{"1min":0.40699999999999997,"5min":0.378,"15min":0.378},"max":{"1min":2.2050000000000001,"5min":2.2050000000000001,"15min":2.2050000000000001},"last":0.59299999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.46500000000000002,"15min":0.45900000000000002},"min":{"1min":0.30299999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.438},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.68600000000000005,"15min":0.71399999999999997},"min":{"1min":0.38800000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.72099999999999997,"5min":5.5369999999999999,"15min":5.5369999999999999},"last":0.46100000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.51100000000000001,"15min":0.50700000000000001},"min":{"1min":0.33100000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.70599999999999996,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.53200000000000003,"15min":0.52800000000000002},"min":{"1min":0.35399999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.72799999999999998,"5min":0.746,"15min":0.746},"last":0.54800000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.55400000000000005,"15min":0.55000000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.74099999999999999,"15min":0.76700000000000002},"min":{"1min":0.40000000000000002,"5min":0.37,"15min":0.37},"max":{"1min":0.78100000000000003,"5min":5.5700000000000003,"15min":5.5700000000000003},"last":0.56699999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626467,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":257115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.54600000000000004,"15min":0.54800000000000004},"min":{"1min":0.36899999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.65200000000000002,"5min":0.752,"15min":0.752},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.36099999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.64900000000000002,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.40799999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53300000000000003,"15min":0.53500000000000003},"min":{"1min":0.39600000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65300000000000002,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.439},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.629,"15min":0.64400000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.70799999999999996,"5min":3.2429999999999999,"15min":3.2429999999999999},"last":0.503}]},{"osd":2,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.51100000000000001,"15min":0.51300000000000001},"min":{"1min":0.376,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.621,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53400000000000003,"15min":0.53700000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67400000000000004,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.35399999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54200000000000004,"15min":0.54300000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.41099999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.86699999999999999,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.58299999999999996}]},{"osd":4,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.48999999999999999,"15min":0.48599999999999999},"min":{"1min":0.34599999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.47299999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.504,"15min":0.50700000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.72699999999999998,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52900000000000003,"15min":0.52800000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.76300000000000001,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.57099999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.52000000000000002,"15min":0.51600000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.497},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.56899999999999995,"15min":0.56100000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77400000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.53100000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574905,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":262760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.36399999999999999,"15min":0.36499999999999999},"min":{"1min":0.23899999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.52000000000000002,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.32500000000000001,"5min":0.40000000000000002,"15min":0.42699999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.46600000000000003,"5min":2.2829999999999999,"15min":2.2829999999999999},"last":0.28899999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.28999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64400000000000002,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.45000000000000001,"15min":0.47599999999999998},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.63500000000000001,"5min":2.3199999999999998,"15min":2.3199999999999998},"last":0.34899999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.40999999999999998,"15min":0.41799999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.46000000000000002,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.45600000000000002,"15min":0.46700000000000003},"min":{"1min":0.33600000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.58399999999999996,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.40000000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:03:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.55600000000000005,"15min":0.55500000000000005},"min":{"1min":0.40200000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.86399999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58799999999999997,"15min":0.58199999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.86899999999999999,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.46999999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57199999999999995,"15min":0.57799999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.443},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.42599999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.54500000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.60999999999999999,"15min":0.621},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.85499999999999998,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.61599999999999999,"15min":0.627},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76300000000000001,"5min":1.03,"15min":1.03},"last":0.57099999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58099999999999996,"15min":0.58299999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77400000000000002,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.61899999999999999,"15min":0.623},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.82999999999999996,"5min":0.999,"15min":0.999},"last":0.503}]}]},{"osd":0,"up_from":8,"seq":34359738430,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34499999999999997,"15min":0.34300000000000003},"min":{"1min":0.245,"5min":0.22,"15min":0.22},"max":{"1min":0.46400000000000002,"5min":0.52900000000000003,"15min":0.52900000000000003},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.245,"5min":0.221,"15min":0.221},"max":{"1min":0.53600000000000003,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.29399999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:02:45 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.40699999999999997,"15min":0.40899999999999997},"min":{"1min":0.28399999999999997,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.499,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.377,"5min":0.38500000000000001,"15min":0.38400000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.55100000000000005,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.253}]},{"osd":3,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.41799999999999998,"15min":0.42999999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.51600000000000001,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.45700000000000002,"15min":0.47299999999999998},"min":{"1min":0.28399999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56999999999999995,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.32200000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.42599999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.80000000000000004,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.41299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.77100000000000002,"5min":0.87,"15min":0.87},"last":0.70099999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:02:51 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.57299999999999995,"15min":0.57699999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.70499999999999996,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.59099999999999997,"15min":0.59699999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.56399999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.62,"15min":0.624},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.60199999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.39900000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.749,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.46000000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.64700000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411380,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.374,"15min":0.376},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.51500000000000001,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.371,"15min":0.36199999999999999},"min":{"1min":0.249,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.45200000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.379,"15min":0.375},"min":{"1min":0.253,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.373,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.56599999999999995,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.5}]},{"osd":3,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.435,"15min":0.44600000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.499,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.435},{"interface":"front","average":{"1min":0.374,"5min":0.45400000000000001,"15min":0.47499999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.56999999999999995,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.61399999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:03:27 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54000000000000004,"15min":0.54100000000000004},"min":{"1min":0.434,"5min":0.374,"15min":0.374},"max":{"1min":0.65800000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.68200000000000005,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.76300000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:02:52 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56499999999999995,"15min":0.56799999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55900000000000005,"15min":0.56200000000000006},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.73399999999999999,"5min":0.749,"15min":0.749},"last":0.55300000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.59799999999999998,"15min":0.60499999999999998},"min":{"1min":0.44400000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.69899999999999995,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.60599999999999998,"15min":0.61199999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.73699999999999999,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.69099999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.58599999999999997,"15min":0.58899999999999997},"min":{"1min":0.47899999999999998,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.71599999999999997,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.60499999999999998,"15min":0.61099999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.70199999999999996,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.71799999999999997}]}]},{"osd":3,"up_from":25,"seq":107374182448,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297688,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447464,"statfs":{"total":95995035648,"available":95690203136,"internally_reserved":0,"allocated":606208,"data_stored":256229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.51,"5min":1.51,"15min":1.51},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.8599999999999999,"5min":2.8599999999999999,"15min":2.8599999999999999},"last":0.26000000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.42999999999999999,"15min":0.44800000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.503,"5min":0.62,"15min":0.62},"last":0.313},{"interface":"front","average":{"1min":0.37,"5min":0.40999999999999998,"15min":0.41899999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.52700000000000002,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.54000000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.44600000000000001,"15min":0.45800000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.54000000000000004,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.498},{"interface":"front","average":{"1min":0.38,"5min":0.437,"15min":0.45600000000000002},"min":{"1min":0.27200000000000002,"5min":0.25,"15min":0.25},"max":{"1min":0.59199999999999997,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.35499999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:03:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59499999999999997,"15min":0.59699999999999998},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.371,"5min":1.371,"15min":1.371},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.59699999999999998,"15min":0.60499999999999998},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.76700000000000002,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.60299999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56699999999999995,"15min":0.56399999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.60199999999999998,"15min":0.59599999999999997},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.50800000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:03:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.59899999999999998,"15min":0.60699999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.84199999999999997,"5min":1.389,"15min":1.389},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.45300000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.80000000000000004,"5min":1.278,"15min":1.278},"last":0.57199999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.57299999999999995,"15min":0.56799999999999995},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60399999999999998,"15min":0.60199999999999998},"min":{"1min":0.44,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.82499999999999996,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.47599999999999998}]}]},{"osd":4,"up_from":30,"seq":128849018923,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297720,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447432,"statfs":{"total":95995035648,"available":95690170368,"internally_reserved":0,"allocated":638976,"data_stored":264757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.55900000000000005,"15min":0.52500000000000002},"min":{"1min":0.34999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.4459999999999997,"5min":5.4459999999999997,"15min":5.4459999999999997},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.57799999999999996,"15min":0.52600000000000002},"min":{"1min":0.35999999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.1890000000000001,"5min":7.1890000000000001,"15min":7.1890000000000001},"last":0.627}]},{"osd":1,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51200000000000001,"15min":0.51100000000000001},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.53100000000000003,"15min":0.495},"min":{"1min":0.372,"5min":0.315,"15min":0.315},"max":{"1min":4.2279999999999998,"5min":4.2279999999999998,"15min":4.2279999999999998},"last":0.55400000000000005}]},{"osd":2,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.55800000000000005,"15min":0.52100000000000002},"min":{"1min":0.39300000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.3310000000000004,"5min":5.3310000000000004,"15min":5.3310000000000004},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.52000000000000002,"15min":0.51200000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.51500000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.55600000000000005,"15min":0.51700000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.6429999999999998,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.52300000000000002,"15min":0.51800000000000002},"min":{"1min":0.38600000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.1279999999999999,"5min":1.1279999999999999,"15min":1.1279999999999999},"last":0.59199999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:02:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.40200000000000002,"15min":0.39000000000000001},"min":{"1min":0.35799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.5,"5min":0.39000000000000001,"15min":0.372},"min":{"1min":0.39100000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.45700000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:03:14 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.45500000000000002,"15min":0.44500000000000001},"min":{"1min":0.34599999999999997,"5min":0.313,"15min":0.313},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.40200000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.56499999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:03:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.53700000000000003,"15min":0.54100000000000004},"min":{"1min":0.33000000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.55400000000000005,"15min":0.55500000000000005},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.61299999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822696,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297700,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447452,"statfs":{"total":95995035648,"available":95690190848,"internally_reserved":0,"allocated":618496,"data_stored":263384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.497,"15min":0.497},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.441},{"interface":"front","average":{"1min":0.497,"5min":0.51300000000000001,"15min":0.51600000000000001},"min":{"1min":0.34799999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.49299999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.49199999999999999,"15min":0.49399999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.55300000000000005,"15min":0.53500000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":4.0490000000000004,"5min":4.0490000000000004,"15min":4.0490000000000004},"last":0.54100000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.54100000000000004,"15min":0.54600000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.629},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.499,"15min":0.501},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.67600000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.51900000000000002,"15min":0.51700000000000002},"min":{"1min":0.379,"5min":0.377,"15min":0.377},"max":{"1min":0.67800000000000005,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.57699999999999996}]},{"osd":4,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.40100000000000002,"15min":0.39300000000000002},"min":{"1min":0.32300000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.42699999999999999,"15min":0.41199999999999998},"min":{"1min":0.30299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.58699999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:03:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.48699999999999999,"15min":0.47899999999999998},"min":{"1min":0.34100000000000003,"5min":0.313,"15min":0.313},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.47699999999999998,"15min":0.46500000000000002},"min":{"1min":0.35999999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.65000000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:03:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54000000000000004,"15min":0.53500000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.53400000000000003,"15min":0.52700000000000002},"min":{"1min":0.36499999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.64000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:03:49.869 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:03:50.219 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:50.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:49 smithi190 conmon[100518]: cluster 2023-10-01T09:03:48.852943+0000 mgr.y (mgr.14433) 96 : cluster [DBG] pgmap v76: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:49 smithi190 conmon[100518]: audit 2023-10-01T09:03:49.253304+0000 mgr.y (mgr.14433) 97 : audit [DBG] from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:03:50.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:49 smithi138 conmon[109959]: cluster 2023-10-01T09:03:48.852943+0000 mgr.y (mgr.14433) 96 : cluster [DBG] pgmap v76: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:50.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:49 smithi138 conmon[109959]: audit 2023-10-01T09:03:49.253304+0000 mgr.y (mgr.14433) 97 : audit [DBG] from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:03:50.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:49 smithi138 conmon[100289]: cluster 2023-10-01T09:03:48.852943+0000 mgr.y 2023-10-01T09:03:50.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:49 smithi138 conmon[100289]: (mgr.14433) 96 : cluster [DBG] pgmap v76: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:50.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:49 smithi138 conmon[100289]: audit 2023-10-01T09:03:49.253304+0000 mgr.y (mgr.14433) 97 : audit [DBG] from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:03:51.933 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:03:51.935 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:03:52.523 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":77,"stamp":"2023-10-01T09:03:50.853180+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381744,"kb_used_data":4976,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579472,"statfs":{"total":767960285184,"available":765521379328,"internally_reserved":0,"allocated":5095424,"data_stored":2165242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001336"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.633938+0000","last_change":"2023-10-01T09:03:03.223370+0000","last_active":"2023-10-01T09:03:45.633938+0000","last_peered":"2023-10-01T09:03:45.633938+0000","last_clean":"2023-10-01T09:03:45.633938+0000","last_became_active":"2023-10-01T09:03:03.223071+0000","last_became_peered":"2023-10-01T09:03:03.223071+0000","last_unstale":"2023-10-01T09:03:45.633938+0000","last_undegraded":"2023-10-01T09:03:45.633938+0000","last_fullsized":"2023-10-01T09:03:45.633938+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.232672+0000","last_change":"2023-10-01T09:03:03.202335+0000","last_active":"2023-10-01T09:03:08.232672+0000","last_peered":"2023-10-01T09:03:08.232672+0000","last_clean":"2023-10-01T09:03:08.232672+0000","last_became_active":"2023-10-01T09:03:03.201485+0000","last_became_peered":"2023-10-01T09:03:03.201485+0000","last_unstale":"2023-10-01T09:03:08.232672+0000","last_undegraded":"2023-10-01T09:03:08.232672+0000","last_fullsized":"2023-10-01T09:03:08.232672+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635131+0000","last_change":"2023-10-01T09:03:03.224058+0000","last_active":"2023-10-01T09:03:45.635131+0000","last_peered":"2023-10-01T09:03:45.635131+0000","last_clean":"2023-10-01T09:03:45.635131+0000","last_became_active":"2023-10-01T09:03:03.223598+0000","last_became_peered":"2023-10-01T09:03:03.223598+0000","last_unstale":"2023-10-01T09:03:45.635131+0000","last_undegraded":"2023-10-01T09:03:45.635131+0000","last_fullsized":"2023-10-01T09:03:45.635131+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635683+0000","last_change":"2023-10-01T09:03:03.193542+0000","last_active":"2023-10-01T09:03:45.635683+0000","last_peered":"2023-10-01T09:03:45.635683+0000","last_clean":"2023-10-01T09:03:45.635683+0000","last_became_active":"2023-10-01T09:03:03.193245+0000","last_became_peered":"2023-10-01T09:03:03.193245+0000","last_unstale":"2023-10-01T09:03:45.635683+0000","last_undegraded":"2023-10-01T09:03:45.635683+0000","last_fullsized":"2023-10-01T09:03:45.635683+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634507+0000","last_change":"2023-10-01T09:03:03.218020+0000","last_active":"2023-10-01T09:03:45.634507+0000","last_peered":"2023-10-01T09:03:45.634507+0000","last_clean":"2023-10-01T09:03:45.634507+0000","last_became_active":"2023-10-01T09:03:03.217717+0000","last_became_peered":"2023-10-01T09:03:03.217717+0000","last_unstale":"2023-10-01T09:03:45.634507+0000","last_undegraded":"2023-10-01T09:03:45.634507+0000","last_fullsized":"2023-10-01T09:03:45.634507+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.633112+0000","last_change":"2023-10-01T09:03:03.202063+0000","last_active":"2023-10-01T09:03:45.633112+0000","last_peered":"2023-10-01T09:03:45.633112+0000","last_clean":"2023-10-01T09:03:45.633112+0000","last_became_active":"2023-10-01T09:03:03.201769+0000","last_became_peered":"2023-10-01T09:03:03.201769+0000","last_unstale":"2023-10-01T09:03:45.633112+0000","last_undegraded":"2023-10-01T09:03:45.633112+0000","last_fullsized":"2023-10-01T09:03:45.633112+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635109+0000","last_change":"2023-10-01T09:03:03.221313+0000","last_active":"2023-10-01T09:03:45.635109+0000","last_peered":"2023-10-01T09:03:45.635109+0000","last_clean":"2023-10-01T09:03:45.635109+0000","last_became_active":"2023-10-01T09:03:03.221080+0000","last_became_peered":"2023-10-01T09:03:03.221080+0000","last_unstale":"2023-10-01T09:03:45.635109+0000","last_undegraded":"2023-10-01T09:03:45.635109+0000","last_fullsized":"2023-10-01T09:03:45.635109+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.633222+0000","last_change":"2023-10-01T09:03:03.201659+0000","last_active":"2023-10-01T09:03:45.633222+0000","last_peered":"2023-10-01T09:03:45.633222+0000","last_clean":"2023-10-01T09:03:45.633222+0000","last_became_active":"2023-10-01T09:03:03.201348+0000","last_became_peered":"2023-10-01T09:03:03.201348+0000","last_unstale":"2023-10-01T09:03:45.633222+0000","last_undegraded":"2023-10-01T09:03:45.633222+0000","last_fullsized":"2023-10-01T09:03:45.633222+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.631910+0000","last_change":"2023-10-01T09:03:03.226772+0000","last_active":"2023-10-01T09:03:45.631910+0000","last_peered":"2023-10-01T09:03:45.631910+0000","last_clean":"2023-10-01T09:03:45.631910+0000","last_became_active":"2023-10-01T09:03:03.226446+0000","last_became_peered":"2023-10-01T09:03:03.226446+0000","last_unstale":"2023-10-01T09:03:45.631910+0000","last_undegraded":"2023-10-01T09:03:45.631910+0000","last_fullsized":"2023-10-01T09:03:45.631910+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.231820+0000","last_change":"2023-10-01T09:03:03.203002+0000","last_active":"2023-10-01T09:03:08.231820+0000","last_peered":"2023-10-01T09:03:08.231820+0000","last_clean":"2023-10-01T09:03:08.231820+0000","last_became_active":"2023-10-01T09:03:03.202571+0000","last_became_peered":"2023-10-01T09:03:03.202571+0000","last_unstale":"2023-10-01T09:03:08.231820+0000","last_undegraded":"2023-10-01T09:03:08.231820+0000","last_fullsized":"2023-10-01T09:03:08.231820+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634224+0000","last_change":"2023-10-01T09:03:03.754839+0000","last_active":"2023-10-01T09:03:45.634224+0000","last_peered":"2023-10-01T09:03:45.634224+0000","last_clean":"2023-10-01T09:03:45.634224+0000","last_became_active":"2023-10-01T09:03:03.754506+0000","last_became_peered":"2023-10-01T09:03:03.754506+0000","last_unstale":"2023-10-01T09:03:45.634224+0000","last_undegraded":"2023-10-01T09:03:45.634224+0000","last_fullsized":"2023-10-01T09:03:45.634224+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:12.687636+0000","last_change":"2023-10-01T09:03:03.196477+0000","last_active":"2023-10-01T09:03:12.687636+0000","last_peered":"2023-10-01T09:03:12.687636+0000","last_clean":"2023-10-01T09:03:12.687636+0000","last_became_active":"2023-10-01T09:03:03.196266+0000","last_became_peered":"2023-10-01T09:03:03.196266+0000","last_unstale":"2023-10-01T09:03:12.687636+0000","last_undegraded":"2023-10-01T09:03:12.687636+0000","last_fullsized":"2023-10-01T09:03:12.687636+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.636147+0000","last_change":"2023-10-01T09:02:06.997887+0000","last_active":"2023-10-01T09:03:45.636147+0000","last_peered":"2023-10-01T09:03:45.636147+0000","last_clean":"2023-10-01T09:03:45.636147+0000","last_became_active":"2023-10-01T09:02:06.997597+0000","last_became_peered":"2023-10-01T09:02:06.997597+0000","last_unstale":"2023-10-01T09:03:45.636147+0000","last_undegraded":"2023-10-01T09:03:45.636147+0000","last_fullsized":"2023-10-01T09:03:45.636147+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635006+0000","last_change":"2023-10-01T09:03:03.228390+0000","last_active":"2023-10-01T09:03:45.635006+0000","last_peered":"2023-10-01T09:03:45.635006+0000","last_clean":"2023-10-01T09:03:45.635006+0000","last_became_active":"2023-10-01T09:03:03.227254+0000","last_became_peered":"2023-10-01T09:03:03.227254+0000","last_unstale":"2023-10-01T09:03:45.635006+0000","last_undegraded":"2023-10-01T09:03:45.635006+0000","last_fullsized":"2023-10-01T09:03:45.635006+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632276+0000","last_change":"2023-10-01T09:02:07.442083+0000","last_active":"2023-10-01T09:03:45.632276+0000","last_peered":"2023-10-01T09:03:45.632276+0000","last_clean":"2023-10-01T09:03:45.632276+0000","last_became_active":"2023-10-01T09:02:07.441757+0000","last_became_peered":"2023-10-01T09:02:07.441757+0000","last_unstale":"2023-10-01T09:03:45.632276+0000","last_undegraded":"2023-10-01T09:03:45.632276+0000","last_fullsized":"2023-10-01T09:03:45.632276+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635624+0000","last_change":"2023-10-01T09:03:03.198510+0000","last_active":"2023-10-01T09:03:45.635624+0000","last_peered":"2023-10-01T09:03:45.635624+0000","last_clean":"2023-10-01T09:03:45.635624+0000","last_became_active":"2023-10-01T09:03:03.198172+0000","last_became_peered":"2023-10-01T09:03:03.198172+0000","last_unstale":"2023-10-01T09:03:45.635624+0000","last_undegraded":"2023-10-01T09:03:45.635624+0000","last_fullsized":"2023-10-01T09:03:45.635624+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215706+0000","last_change":"2023-10-01T09:03:03.754828+0000","last_active":"2023-10-01T09:03:05.215706+0000","last_peered":"2023-10-01T09:03:05.215706+0000","last_clean":"2023-10-01T09:03:05.215706+0000","last_became_active":"2023-10-01T09:03:03.754492+0000","last_became_peered":"2023-10-01T09:03:03.754492+0000","last_unstale":"2023-10-01T09:03:05.215706+0000","last_undegraded":"2023-10-01T09:03:05.215706+0000","last_fullsized":"2023-10-01T09:03:05.215706+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632764+0000","last_change":"2023-10-01T09:03:03.203345+0000","last_active":"2023-10-01T09:03:45.632764+0000","last_peered":"2023-10-01T09:03:45.632764+0000","last_clean":"2023-10-01T09:03:45.632764+0000","last_became_active":"2023-10-01T09:03:03.203035+0000","last_became_peered":"2023-10-01T09:03:03.203035+0000","last_unstale":"2023-10-01T09:03:45.632764+0000","last_undegraded":"2023-10-01T09:03:45.632764+0000","last_fullsized":"2023-10-01T09:03:45.632764+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932484+0000","last_change":"2023-10-01T09:02:07.442452+0000","last_active":"2023-10-01T09:03:45.932484+0000","last_peered":"2023-10-01T09:03:45.932484+0000","last_clean":"2023-10-01T09:03:45.932484+0000","last_became_active":"2023-10-01T09:02:07.442111+0000","last_became_peered":"2023-10-01T09:02:07.442111+0000","last_unstale":"2023-10-01T09:03:45.932484+0000","last_undegraded":"2023-10-01T09:03:45.932484+0000","last_fullsized":"2023-10-01T09:03:45.932484+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215738+0000","last_change":"2023-10-01T09:03:03.223326+0000","last_active":"2023-10-01T09:03:05.215738+0000","last_peered":"2023-10-01T09:03:05.215738+0000","last_clean":"2023-10-01T09:03:05.215738+0000","last_became_active":"2023-10-01T09:03:03.222974+0000","last_became_peered":"2023-10-01T09:03:03.222974+0000","last_unstale":"2023-10-01T09:03:05.215738+0000","last_undegraded":"2023-10-01T09:03:05.215738+0000","last_fullsized":"2023-10-01T09:03:05.215738+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215539+0000","last_change":"2023-10-01T09:03:03.195712+0000","last_active":"2023-10-01T09:03:05.215539+0000","last_peered":"2023-10-01T09:03:05.215539+0000","last_clean":"2023-10-01T09:03:05.215539+0000","last_became_active":"2023-10-01T09:03:03.195408+0000","last_became_peered":"2023-10-01T09:03:03.195408+0000","last_unstale":"2023-10-01T09:03:05.215539+0000","last_undegraded":"2023-10-01T09:03:05.215539+0000","last_fullsized":"2023-10-01T09:03:05.215539+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932093+0000","last_change":"2023-10-01T09:02:06.999043+0000","last_active":"2023-10-01T09:03:45.932093+0000","last_peered":"2023-10-01T09:03:45.932093+0000","last_clean":"2023-10-01T09:03:45.932093+0000","last_became_active":"2023-10-01T09:02:06.998671+0000","last_became_peered":"2023-10-01T09:02:06.998671+0000","last_unstale":"2023-10-01T09:03:45.932093+0000","last_undegraded":"2023-10-01T09:03:45.932093+0000","last_fullsized":"2023-10-01T09:03:45.932093+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635816+0000","last_change":"2023-10-01T09:03:03.227254+0000","last_active":"2023-10-01T09:03:45.635816+0000","last_peered":"2023-10-01T09:03:45.635816+0000","last_clean":"2023-10-01T09:03:45.635816+0000","last_became_active":"2023-10-01T09:03:03.226941+0000","last_became_peered":"2023-10-01T09:03:03.226941+0000","last_unstale":"2023-10-01T09:03:45.635816+0000","last_undegraded":"2023-10-01T09:03:45.635816+0000","last_fullsized":"2023-10-01T09:03:45.635816+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635127+0000","last_change":"2023-10-01T09:03:03.198835+0000","last_active":"2023-10-01T09:03:45.635127+0000","last_peered":"2023-10-01T09:03:45.635127+0000","last_clean":"2023-10-01T09:03:45.635127+0000","last_became_active":"2023-10-01T09:03:03.198424+0000","last_became_peered":"2023-10-01T09:03:03.198424+0000","last_unstale":"2023-10-01T09:03:45.635127+0000","last_undegraded":"2023-10-01T09:03:45.635127+0000","last_fullsized":"2023-10-01T09:03:45.635127+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932052+0000","last_change":"2023-10-01T09:01:32.510259+0000","last_active":"2023-10-01T09:03:45.932052+0000","last_peered":"2023-10-01T09:03:45.932052+0000","last_clean":"2023-10-01T09:03:45.932052+0000","last_became_active":"2023-10-01T09:01:32.509905+0000","last_became_peered":"2023-10-01T09:01:32.509905+0000","last_unstale":"2023-10-01T09:03:45.932052+0000","last_undegraded":"2023-10-01T09:03:45.932052+0000","last_fullsized":"2023-10-01T09:03:45.932052+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632783+0000","last_change":"2023-10-01T09:02:06.998804+0000","last_active":"2023-10-01T09:03:45.632783+0000","last_peered":"2023-10-01T09:03:45.632783+0000","last_clean":"2023-10-01T09:03:45.632783+0000","last_became_active":"2023-10-01T09:02:06.998516+0000","last_became_peered":"2023-10-01T09:02:06.998516+0000","last_unstale":"2023-10-01T09:03:45.632783+0000","last_undegraded":"2023-10-01T09:03:45.632783+0000","last_fullsized":"2023-10-01T09:03:45.632783+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632512+0000","last_change":"2023-10-01T09:03:03.756021+0000","last_active":"2023-10-01T09:03:45.632512+0000","last_peered":"2023-10-01T09:03:45.632512+0000","last_clean":"2023-10-01T09:03:45.632512+0000","last_became_active":"2023-10-01T09:03:03.755659+0000","last_became_peered":"2023-10-01T09:03:03.755659+0000","last_unstale":"2023-10-01T09:03:45.632512+0000","last_undegraded":"2023-10-01T09:03:45.632512+0000","last_fullsized":"2023-10-01T09:03:45.632512+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632396+0000","last_change":"2023-10-01T09:03:03.201620+0000","last_active":"2023-10-01T09:03:45.632396+0000","last_peered":"2023-10-01T09:03:45.632396+0000","last_clean":"2023-10-01T09:03:45.632396+0000","last_became_active":"2023-10-01T09:03:03.201312+0000","last_became_peered":"2023-10-01T09:03:03.201312+0000","last_unstale":"2023-10-01T09:03:45.632396+0000","last_undegraded":"2023-10-01T09:03:45.632396+0000","last_fullsized":"2023-10-01T09:03:45.632396+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.631546+0000","last_change":"2023-10-01T09:02:07.436492+0000","last_active":"2023-10-01T09:03:45.631546+0000","last_peered":"2023-10-01T09:03:45.631546+0000","last_clean":"2023-10-01T09:03:45.631546+0000","last_became_active":"2023-10-01T09:02:07.436146+0000","last_became_peered":"2023-10-01T09:02:07.436146+0000","last_unstale":"2023-10-01T09:03:45.631546+0000","last_undegraded":"2023-10-01T09:03:45.631546+0000","last_fullsized":"2023-10-01T09:03:45.631546+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632674+0000","last_change":"2023-10-01T09:03:03.227284+0000","last_active":"2023-10-01T09:03:45.632674+0000","last_peered":"2023-10-01T09:03:45.632674+0000","last_clean":"2023-10-01T09:03:45.632674+0000","last_became_active":"2023-10-01T09:03:03.226987+0000","last_became_peered":"2023-10-01T09:03:03.226987+0000","last_unstale":"2023-10-01T09:03:45.632674+0000","last_undegraded":"2023-10-01T09:03:45.632674+0000","last_fullsized":"2023-10-01T09:03:45.632674+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634293+0000","last_change":"2023-10-01T09:03:03.196357+0000","last_active":"2023-10-01T09:03:45.634293+0000","last_peered":"2023-10-01T09:03:45.634293+0000","last_clean":"2023-10-01T09:03:45.634293+0000","last_became_active":"2023-10-01T09:03:03.196017+0000","last_became_peered":"2023-10-01T09:03:03.196017+0000","last_unstale":"2023-10-01T09:03:45.634293+0000","last_undegraded":"2023-10-01T09:03:45.634293+0000","last_fullsized":"2023-10-01T09:03:45.634293+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632885+0000","last_change":"2023-10-01T09:02:07.001644+0000","last_active":"2023-10-01T09:03:45.632885+0000","last_peered":"2023-10-01T09:03:45.632885+0000","last_clean":"2023-10-01T09:03:45.632885+0000","last_became_active":"2023-10-01T09:02:07.001197+0000","last_became_peered":"2023-10-01T09:02:07.001197+0000","last_unstale":"2023-10-01T09:03:45.632885+0000","last_undegraded":"2023-10-01T09:03:45.632885+0000","last_fullsized":"2023-10-01T09:03:45.632885+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632588+0000","last_change":"2023-10-01T09:03:03.755027+0000","last_active":"2023-10-01T09:03:45.632588+0000","last_peered":"2023-10-01T09:03:45.632588+0000","last_clean":"2023-10-01T09:03:45.632588+0000","last_became_active":"2023-10-01T09:03:03.754596+0000","last_became_peered":"2023-10-01T09:03:03.754596+0000","last_unstale":"2023-10-01T09:03:45.632588+0000","last_undegraded":"2023-10-01T09:03:45.632588+0000","last_fullsized":"2023-10-01T09:03:45.632588+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.633017+0000","last_change":"2023-10-01T09:03:03.203012+0000","last_active":"2023-10-01T09:03:45.633017+0000","last_peered":"2023-10-01T09:03:45.633017+0000","last_clean":"2023-10-01T09:03:45.633017+0000","last_became_active":"2023-10-01T09:03:03.202051+0000","last_became_peered":"2023-10-01T09:03:03.202051+0000","last_unstale":"2023-10-01T09:03:45.633017+0000","last_undegraded":"2023-10-01T09:03:45.633017+0000","last_fullsized":"2023-10-01T09:03:45.633017+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632616+0000","last_change":"2023-10-01T09:02:06.998188+0000","last_active":"2023-10-01T09:03:45.632616+0000","last_peered":"2023-10-01T09:03:45.632616+0000","last_clean":"2023-10-01T09:03:45.632616+0000","last_became_active":"2023-10-01T09:02:06.997855+0000","last_became_peered":"2023-10-01T09:02:06.997855+0000","last_unstale":"2023-10-01T09:03:45.632616+0000","last_undegraded":"2023-10-01T09:03:45.632616+0000","last_fullsized":"2023-10-01T09:03:45.632616+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T09:03:46.638938+0000","last_change":"2023-10-01T09:03:46.638938+0000","last_active":"2023-10-01T09:03:46.638938+0000","last_peered":"2023-10-01T09:03:46.638938+0000","last_clean":"2023-10-01T09:03:46.638938+0000","last_became_active":"2023-10-01T09:03:46.638713+0000","last_became_peered":"2023-10-01T09:03:46.638713+0000","last_unstale":"2023-10-01T09:03:46.638938+0000","last_undegraded":"2023-10-01T09:03:46.638938+0000","last_fullsized":"2023-10-01T09:03:46.638938+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635017+0000","last_change":"2023-10-01T09:03:03.200838+0000","last_active":"2023-10-01T09:03:45.635017+0000","last_peered":"2023-10-01T09:03:45.635017+0000","last_clean":"2023-10-01T09:03:45.635017+0000","last_became_active":"2023-10-01T09:03:03.199715+0000","last_became_peered":"2023-10-01T09:03:03.199715+0000","last_unstale":"2023-10-01T09:03:45.635017+0000","last_undegraded":"2023-10-01T09:03:45.635017+0000","last_fullsized":"2023-10-01T09:03:45.635017+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634588+0000","last_change":"2023-10-01T09:03:03.221266+0000","last_active":"2023-10-01T09:03:45.634588+0000","last_peered":"2023-10-01T09:03:45.634588+0000","last_clean":"2023-10-01T09:03:45.634588+0000","last_became_active":"2023-10-01T09:03:03.220690+0000","last_became_peered":"2023-10-01T09:03:03.220690+0000","last_unstale":"2023-10-01T09:03:45.634588+0000","last_undegraded":"2023-10-01T09:03:45.634588+0000","last_fullsized":"2023-10-01T09:03:45.634588+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634373+0000","last_change":"2023-10-01T09:03:03.199237+0000","last_active":"2023-10-01T09:03:45.634373+0000","last_peered":"2023-10-01T09:03:45.634373+0000","last_clean":"2023-10-01T09:03:45.634373+0000","last_became_active":"2023-10-01T09:03:03.198950+0000","last_became_peered":"2023-10-01T09:03:03.198950+0000","last_unstale":"2023-10-01T09:03:45.634373+0000","last_undegraded":"2023-10-01T09:03:45.634373+0000","last_fullsized":"2023-10-01T09:03:45.634373+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634678+0000","last_change":"2023-10-01T09:03:03.223290+0000","last_active":"2023-10-01T09:03:45.634678+0000","last_peered":"2023-10-01T09:03:45.634678+0000","last_clean":"2023-10-01T09:03:45.634678+0000","last_became_active":"2023-10-01T09:03:03.223013+0000","last_became_peered":"2023-10-01T09:03:03.223013+0000","last_unstale":"2023-10-01T09:03:45.634678+0000","last_undegraded":"2023-10-01T09:03:45.634678+0000","last_fullsized":"2023-10-01T09:03:45.634678+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635379+0000","last_change":"2023-10-01T09:03:03.198744+0000","last_active":"2023-10-01T09:03:45.635379+0000","last_peered":"2023-10-01T09:03:45.635379+0000","last_clean":"2023-10-01T09:03:45.635379+0000","last_became_active":"2023-10-01T09:03:03.198452+0000","last_became_peered":"2023-10-01T09:03:03.198452+0000","last_unstale":"2023-10-01T09:03:45.635379+0000","last_undegraded":"2023-10-01T09:03:45.635379+0000","last_fullsized":"2023-10-01T09:03:45.635379+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634570+0000","last_change":"2023-10-01T09:03:03.218071+0000","last_active":"2023-10-01T09:03:45.634570+0000","last_peered":"2023-10-01T09:03:45.634570+0000","last_clean":"2023-10-01T09:03:45.634570+0000","last_became_active":"2023-10-01T09:03:03.217776+0000","last_became_peered":"2023-10-01T09:03:03.217776+0000","last_unstale":"2023-10-01T09:03:45.634570+0000","last_undegraded":"2023-10-01T09:03:45.634570+0000","last_fullsized":"2023-10-01T09:03:45.634570+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635229+0000","last_change":"2023-10-01T09:03:03.200515+0000","last_active":"2023-10-01T09:03:45.635229+0000","last_peered":"2023-10-01T09:03:45.635229+0000","last_clean":"2023-10-01T09:03:45.635229+0000","last_became_active":"2023-10-01T09:03:03.199536+0000","last_became_peered":"2023-10-01T09:03:03.199536+0000","last_unstale":"2023-10-01T09:03:45.635229+0000","last_undegraded":"2023-10-01T09:03:45.635229+0000","last_fullsized":"2023-10-01T09:03:45.635229+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215903+0000","last_change":"2023-10-01T09:03:03.224884+0000","last_active":"2023-10-01T09:03:05.215903+0000","last_peered":"2023-10-01T09:03:05.215903+0000","last_clean":"2023-10-01T09:03:05.215903+0000","last_became_active":"2023-10-01T09:03:03.224529+0000","last_became_peered":"2023-10-01T09:03:03.224529+0000","last_unstale":"2023-10-01T09:03:05.215903+0000","last_undegraded":"2023-10-01T09:03:05.215903+0000","last_fullsized":"2023-10-01T09:03:05.215903+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632780+0000","last_change":"2023-10-01T09:03:03.203721+0000","last_active":"2023-10-01T09:03:45.632780+0000","last_peered":"2023-10-01T09:03:45.632780+0000","last_clean":"2023-10-01T09:03:45.632780+0000","last_became_active":"2023-10-01T09:03:03.203308+0000","last_became_peered":"2023-10-01T09:03:03.203308+0000","last_unstale":"2023-10-01T09:03:45.632780+0000","last_undegraded":"2023-10-01T09:03:45.632780+0000","last_fullsized":"2023-10-01T09:03:45.632780+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635266+0000","last_change":"2023-10-01T09:03:03.227070+0000","last_active":"2023-10-01T09:03:45.635266+0000","last_peered":"2023-10-01T09:03:45.635266+0000","last_clean":"2023-10-01T09:03:45.635266+0000","last_became_active":"2023-10-01T09:03:03.226752+0000","last_became_peered":"2023-10-01T09:03:03.226752+0000","last_unstale":"2023-10-01T09:03:45.635266+0000","last_undegraded":"2023-10-01T09:03:45.635266+0000","last_fullsized":"2023-10-01T09:03:45.635266+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932534+0000","last_change":"2023-10-01T09:03:03.200972+0000","last_active":"2023-10-01T09:03:45.932534+0000","last_peered":"2023-10-01T09:03:45.932534+0000","last_clean":"2023-10-01T09:03:45.932534+0000","last_became_active":"2023-10-01T09:03:03.200695+0000","last_became_peered":"2023-10-01T09:03:03.200695+0000","last_unstale":"2023-10-01T09:03:45.932534+0000","last_undegraded":"2023-10-01T09:03:45.932534+0000","last_fullsized":"2023-10-01T09:03:45.932534+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634095+0000","last_change":"2023-10-01T09:03:03.220741+0000","last_active":"2023-10-01T09:03:45.634095+0000","last_peered":"2023-10-01T09:03:45.634095+0000","last_clean":"2023-10-01T09:03:45.634095+0000","last_became_active":"2023-10-01T09:03:03.220481+0000","last_became_peered":"2023-10-01T09:03:03.220481+0000","last_unstale":"2023-10-01T09:03:45.634095+0000","last_undegraded":"2023-10-01T09:03:45.634095+0000","last_fullsized":"2023-10-01T09:03:45.634095+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932558+0000","last_change":"2023-10-01T09:03:03.200367+0000","last_active":"2023-10-01T09:03:45.932558+0000","last_peered":"2023-10-01T09:03:45.932558+0000","last_clean":"2023-10-01T09:03:45.932558+0000","last_became_active":"2023-10-01T09:03:03.200068+0000","last_became_peered":"2023-10-01T09:03:03.200068+0000","last_unstale":"2023-10-01T09:03:45.932558+0000","last_undegraded":"2023-10-01T09:03:45.932558+0000","last_fullsized":"2023-10-01T09:03:45.932558+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632631+0000","last_change":"2023-10-01T09:03:04.245105+0000","last_active":"2023-10-01T09:03:45.632631+0000","last_peered":"2023-10-01T09:03:45.632631+0000","last_clean":"2023-10-01T09:03:45.632631+0000","last_became_active":"2023-10-01T09:03:04.244807+0000","last_became_peered":"2023-10-01T09:03:04.244807+0000","last_unstale":"2023-10-01T09:03:45.632631+0000","last_undegraded":"2023-10-01T09:03:45.632631+0000","last_fullsized":"2023-10-01T09:03:45.632631+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932682+0000","last_change":"2023-10-01T09:03:03.199109+0000","last_active":"2023-10-01T09:03:45.932682+0000","last_peered":"2023-10-01T09:03:45.932682+0000","last_clean":"2023-10-01T09:03:45.932682+0000","last_became_active":"2023-10-01T09:03:03.198810+0000","last_became_peered":"2023-10-01T09:03:03.198810+0000","last_unstale":"2023-10-01T09:03:45.932682+0000","last_undegraded":"2023-10-01T09:03:45.932682+0000","last_fullsized":"2023-10-01T09:03:45.932682+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635430+0000","last_change":"2023-10-01T09:03:03.227888+0000","last_active":"2023-10-01T09:03:45.635430+0000","last_peered":"2023-10-01T09:03:45.635430+0000","last_clean":"2023-10-01T09:03:45.635430+0000","last_became_active":"2023-10-01T09:03:03.227621+0000","last_became_peered":"2023-10-01T09:03:03.227621+0000","last_unstale":"2023-10-01T09:03:45.635430+0000","last_undegraded":"2023-10-01T09:03:45.635430+0000","last_fullsized":"2023-10-01T09:03:45.635430+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635126+0000","last_change":"2023-10-01T09:03:03.199462+0000","last_active":"2023-10-01T09:03:45.635126+0000","last_peered":"2023-10-01T09:03:45.635126+0000","last_clean":"2023-10-01T09:03:45.635126+0000","last_became_active":"2023-10-01T09:03:03.199188+0000","last_became_peered":"2023-10-01T09:03:03.199188+0000","last_unstale":"2023-10-01T09:03:45.635126+0000","last_undegraded":"2023-10-01T09:03:45.635126+0000","last_fullsized":"2023-10-01T09:03:45.635126+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215653+0000","last_change":"2023-10-01T09:03:03.228462+0000","last_active":"2023-10-01T09:03:05.215653+0000","last_peered":"2023-10-01T09:03:05.215653+0000","last_clean":"2023-10-01T09:03:05.215653+0000","last_became_active":"2023-10-01T09:03:03.228010+0000","last_became_peered":"2023-10-01T09:03:03.228010+0000","last_unstale":"2023-10-01T09:03:05.215653+0000","last_undegraded":"2023-10-01T09:03:05.215653+0000","last_fullsized":"2023-10-01T09:03:05.215653+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932159+0000","last_change":"2023-10-01T09:03:03.194952+0000","last_active":"2023-10-01T09:03:45.932159+0000","last_peered":"2023-10-01T09:03:45.932159+0000","last_clean":"2023-10-01T09:03:45.932159+0000","last_became_active":"2023-10-01T09:03:03.194643+0000","last_became_peered":"2023-10-01T09:03:03.194643+0000","last_unstale":"2023-10-01T09:03:45.932159+0000","last_undegraded":"2023-10-01T09:03:45.932159+0000","last_fullsized":"2023-10-01T09:03:45.932159+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632819+0000","last_change":"2023-10-01T09:03:04.405231+0000","last_active":"2023-10-01T09:03:45.632819+0000","last_peered":"2023-10-01T09:03:45.632819+0000","last_clean":"2023-10-01T09:03:45.632819+0000","last_became_active":"2023-10-01T09:03:04.404876+0000","last_became_peered":"2023-10-01T09:03:04.404876+0000","last_unstale":"2023-10-01T09:03:45.632819+0000","last_undegraded":"2023-10-01T09:03:45.632819+0000","last_fullsized":"2023-10-01T09:03:45.632819+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215761+0000","last_change":"2023-10-01T09:03:03.202905+0000","last_active":"2023-10-01T09:03:05.215761+0000","last_peered":"2023-10-01T09:03:05.215761+0000","last_clean":"2023-10-01T09:03:05.215761+0000","last_became_active":"2023-10-01T09:03:03.202621+0000","last_became_peered":"2023-10-01T09:03:03.202621+0000","last_unstale":"2023-10-01T09:03:05.215761+0000","last_undegraded":"2023-10-01T09:03:05.215761+0000","last_fullsized":"2023-10-01T09:03:05.215761+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635454+0000","last_change":"2023-10-01T09:03:03.228053+0000","last_active":"2023-10-01T09:03:45.635454+0000","last_peered":"2023-10-01T09:03:45.635454+0000","last_clean":"2023-10-01T09:03:45.635454+0000","last_became_active":"2023-10-01T09:03:03.227153+0000","last_became_peered":"2023-10-01T09:03:03.227153+0000","last_unstale":"2023-10-01T09:03:45.635454+0000","last_undegraded":"2023-10-01T09:03:45.635454+0000","last_fullsized":"2023-10-01T09:03:45.635454+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932302+0000","last_change":"2023-10-01T09:03:03.195227+0000","last_active":"2023-10-01T09:03:45.932302+0000","last_peered":"2023-10-01T09:03:45.932302+0000","last_clean":"2023-10-01T09:03:45.932302+0000","last_became_active":"2023-10-01T09:03:03.194964+0000","last_became_peered":"2023-10-01T09:03:03.194964+0000","last_unstale":"2023-10-01T09:03:45.932302+0000","last_undegraded":"2023-10-01T09:03:45.932302+0000","last_fullsized":"2023-10-01T09:03:45.932302+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":21,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T09:03:46.638524+0000","last_change":"2023-10-01T09:03:46.638524+0000","last_active":"2023-10-01T09:03:46.638524+0000","last_peered":"2023-10-01T09:03:46.638524+0000","last_clean":"2023-10-01T09:03:46.638524+0000","last_became_active":"2023-10-01T09:03:46.638209+0000","last_became_peered":"2023-10-01T09:03:46.638209+0000","last_unstale":"2023-10-01T09:03:46.638524+0000","last_undegraded":"2023-10-01T09:03:46.638524+0000","last_fullsized":"2023-10-01T09:03:46.638524+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":19,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634523+0000","last_change":"2023-10-01T09:03:03.200261+0000","last_active":"2023-10-01T09:03:45.634523+0000","last_peered":"2023-10-01T09:03:45.634523+0000","last_clean":"2023-10-01T09:03:45.634523+0000","last_became_active":"2023-10-01T09:03:03.199944+0000","last_became_peered":"2023-10-01T09:03:03.199944+0000","last_unstale":"2023-10-01T09:03:45.634523+0000","last_undegraded":"2023-10-01T09:03:45.634523+0000","last_fullsized":"2023-10-01T09:03:45.634523+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632559+0000","last_change":"2023-10-01T09:03:03.224723+0000","last_active":"2023-10-01T09:03:45.632559+0000","last_peered":"2023-10-01T09:03:45.632559+0000","last_clean":"2023-10-01T09:03:45.632559+0000","last_became_active":"2023-10-01T09:03:03.224379+0000","last_became_peered":"2023-10-01T09:03:03.224379+0000","last_unstale":"2023-10-01T09:03:45.632559+0000","last_undegraded":"2023-10-01T09:03:45.632559+0000","last_fullsized":"2023-10-01T09:03:45.632559+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932421+0000","last_change":"2023-10-01T09:03:03.199740+0000","last_active":"2023-10-01T09:03:45.932421+0000","last_peered":"2023-10-01T09:03:45.932421+0000","last_clean":"2023-10-01T09:03:45.932421+0000","last_became_active":"2023-10-01T09:03:03.199421+0000","last_became_peered":"2023-10-01T09:03:03.199421+0000","last_unstale":"2023-10-01T09:03:45.932421+0000","last_undegraded":"2023-10-01T09:03:45.932421+0000","last_fullsized":"2023-10-01T09:03:45.932421+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635238+0000","last_change":"2023-10-01T09:03:03.228015+0000","last_active":"2023-10-01T09:03:45.635238+0000","last_peered":"2023-10-01T09:03:45.635238+0000","last_clean":"2023-10-01T09:03:45.635238+0000","last_became_active":"2023-10-01T09:03:03.227754+0000","last_became_peered":"2023-10-01T09:03:03.227754+0000","last_unstale":"2023-10-01T09:03:45.635238+0000","last_undegraded":"2023-10-01T09:03:45.635238+0000","last_fullsized":"2023-10-01T09:03:45.635238+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632832+0000","last_change":"2023-10-01T09:03:03.201187+0000","last_active":"2023-10-01T09:03:45.632832+0000","last_peered":"2023-10-01T09:03:45.632832+0000","last_clean":"2023-10-01T09:03:45.632832+0000","last_became_active":"2023-10-01T09:03:03.200859+0000","last_became_peered":"2023-10-01T09:03:03.200859+0000","last_unstale":"2023-10-01T09:03:45.632832+0000","last_undegraded":"2023-10-01T09:03:45.632832+0000","last_fullsized":"2023-10-01T09:03:45.632832+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635189+0000","last_change":"2023-10-01T09:03:03.227458+0000","last_active":"2023-10-01T09:03:45.635189+0000","last_peered":"2023-10-01T09:03:45.635189+0000","last_clean":"2023-10-01T09:03:45.635189+0000","last_became_active":"2023-10-01T09:03:03.226857+0000","last_became_peered":"2023-10-01T09:03:03.226857+0000","last_unstale":"2023-10-01T09:03:45.635189+0000","last_undegraded":"2023-10-01T09:03:45.635189+0000","last_fullsized":"2023-10-01T09:03:45.635189+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215921+0000","last_change":"2023-10-01T09:03:03.202503+0000","last_active":"2023-10-01T09:03:05.215921+0000","last_peered":"2023-10-01T09:03:05.215921+0000","last_clean":"2023-10-01T09:03:05.215921+0000","last_became_active":"2023-10-01T09:03:03.202048+0000","last_became_peered":"2023-10-01T09:03:03.202048+0000","last_unstale":"2023-10-01T09:03:05.215921+0000","last_undegraded":"2023-10-01T09:03:05.215921+0000","last_fullsized":"2023-10-01T09:03:05.215921+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T09:03:46.767306+0000","last_change":"2023-10-01T09:03:46.767306+0000","last_active":"2023-10-01T09:03:46.767306+0000","last_peered":"2023-10-01T09:03:46.767306+0000","last_clean":"2023-10-01T09:03:46.767306+0000","last_became_active":"2023-10-01T09:03:46.766871+0000","last_became_peered":"2023-10-01T09:03:46.766871+0000","last_unstale":"2023-10-01T09:03:46.767306+0000","last_undegraded":"2023-10-01T09:03:46.767306+0000","last_fullsized":"2023-10-01T09:03:46.767306+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635118+0000","last_change":"2023-10-01T09:03:03.198727+0000","last_active":"2023-10-01T09:03:45.635118+0000","last_peered":"2023-10-01T09:03:45.635118+0000","last_clean":"2023-10-01T09:03:45.635118+0000","last_became_active":"2023-10-01T09:03:03.198309+0000","last_became_peered":"2023-10-01T09:03:03.198309+0000","last_unstale":"2023-10-01T09:03:45.635118+0000","last_undegraded":"2023-10-01T09:03:45.635118+0000","last_fullsized":"2023-10-01T09:03:45.635118+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215683+0000","last_change":"2023-10-01T09:03:03.228290+0000","last_active":"2023-10-01T09:03:05.215683+0000","last_peered":"2023-10-01T09:03:05.215683+0000","last_clean":"2023-10-01T09:03:05.215683+0000","last_became_active":"2023-10-01T09:03:03.227612+0000","last_became_peered":"2023-10-01T09:03:03.227612+0000","last_unstale":"2023-10-01T09:03:05.215683+0000","last_undegraded":"2023-10-01T09:03:05.215683+0000","last_fullsized":"2023-10-01T09:03:05.215683+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.631923+0000","last_change":"2023-10-01T09:03:03.199888+0000","last_active":"2023-10-01T09:03:45.631923+0000","last_peered":"2023-10-01T09:03:45.631923+0000","last_clean":"2023-10-01T09:03:45.631923+0000","last_became_active":"2023-10-01T09:03:03.199579+0000","last_became_peered":"2023-10-01T09:03:03.199579+0000","last_unstale":"2023-10-01T09:03:45.631923+0000","last_undegraded":"2023-10-01T09:03:45.631923+0000","last_fullsized":"2023-10-01T09:03:45.631923+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635485+0000","last_change":"2023-10-01T09:03:03.226961+0000","last_active":"2023-10-01T09:03:45.635485+0000","last_peered":"2023-10-01T09:03:45.635485+0000","last_clean":"2023-10-01T09:03:45.635485+0000","last_became_active":"2023-10-01T09:03:03.226660+0000","last_became_peered":"2023-10-01T09:03:03.226660+0000","last_unstale":"2023-10-01T09:03:45.635485+0000","last_undegraded":"2023-10-01T09:03:45.635485+0000","last_fullsized":"2023-10-01T09:03:45.635485+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634325+0000","last_change":"2023-10-01T09:03:03.201578+0000","last_active":"2023-10-01T09:03:45.634325+0000","last_peered":"2023-10-01T09:03:45.634325+0000","last_clean":"2023-10-01T09:03:45.634325+0000","last_became_active":"2023-10-01T09:03:03.201137+0000","last_became_peered":"2023-10-01T09:03:03.201137+0000","last_unstale":"2023-10-01T09:03:45.634325+0000","last_undegraded":"2023-10-01T09:03:45.634325+0000","last_fullsized":"2023-10-01T09:03:45.634325+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430239,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297736,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447416,"statfs":{"total":95995035648,"available":95690153984,"internally_reserved":0,"allocated":655360,"data_stored":271700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.54600000000000004,"15min":0.52200000000000002},"min":{"1min":0.34200000000000003,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.9580000000000002,"5min":2.9580000000000002,"15min":2.9580000000000002},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.503,"5min":0.48999999999999999,"15min":0.48799999999999999},"min":{"1min":0.39600000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.63800000000000001,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.58799999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.48699999999999999,"15min":0.48299999999999998},"min":{"1min":0.38600000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.63400000000000001,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.498,"15min":0.48399999999999999},"min":{"1min":0.36099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.67100000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.71499999999999997,"15min":0.745},"min":{"1min":0.36699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.91000000000000003,"5min":5.5949999999999998,"15min":5.5949999999999998},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.52100000000000002,"15min":0.50700000000000001},"min":{"1min":0.432,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.65100000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.69799999999999995,"15min":0.72399999999999998},"min":{"1min":0.40300000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68700000000000006,"5min":5.6210000000000004,"15min":5.6210000000000004},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.54500000000000004,"15min":0.52300000000000002},"min":{"1min":0.40699999999999997,"5min":0.378,"15min":0.378},"max":{"1min":2.2050000000000001,"5min":2.2050000000000001,"15min":2.2050000000000001},"last":0.55500000000000005}]},{"osd":4,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.46500000000000002,"15min":0.45900000000000002},"min":{"1min":0.30299999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.68600000000000005,"15min":0.71399999999999997},"min":{"1min":0.38800000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.72099999999999997,"5min":5.5369999999999999,"15min":5.5369999999999999},"last":0.496}]},{"osd":5,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.51100000000000001,"15min":0.50700000000000001},"min":{"1min":0.33100000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.70599999999999996,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.53200000000000003,"15min":0.52800000000000002},"min":{"1min":0.35399999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.72799999999999998,"5min":0.746,"15min":0.746},"last":0.63}]},{"osd":6,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.55400000000000005,"15min":0.55000000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.74099999999999999,"15min":0.76700000000000002},"min":{"1min":0.40000000000000002,"5min":0.37,"15min":0.37},"max":{"1min":0.78100000000000003,"5min":5.5700000000000003,"15min":5.5700000000000003},"last":0.64200000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626468,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.54600000000000004,"15min":0.54800000000000004},"min":{"1min":0.36899999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.65200000000000002,"5min":0.752,"15min":0.752},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.36099999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.64900000000000002,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.53300000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53300000000000003,"15min":0.53500000000000003},"min":{"1min":0.39600000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65300000000000002,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.629,"15min":0.64400000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.70799999999999996,"5min":3.2429999999999999,"15min":3.2429999999999999},"last":0.377}]},{"osd":2,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.51100000000000001,"15min":0.51300000000000001},"min":{"1min":0.376,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.621,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.441},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53400000000000003,"15min":0.53700000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67400000000000004,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.41899999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54200000000000004,"15min":0.54300000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.41099999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.86699999999999999,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.501}]},{"osd":4,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.48999999999999999,"15min":0.48599999999999999},"min":{"1min":0.34599999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.51400000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.504,"15min":0.50700000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.72699999999999998,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52900000000000003,"15min":0.52800000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.76300000000000001,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.60899999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.52000000000000002,"15min":0.51600000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.56899999999999995,"15min":0.56100000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77400000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.63400000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574906,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297704,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447448,"statfs":{"total":95995035648,"available":95690186752,"internally_reserved":0,"allocated":622592,"data_stored":269633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.36399999999999999,"15min":0.36499999999999999},"min":{"1min":0.23899999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.52000000000000002,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.254},{"interface":"front","average":{"1min":0.32500000000000001,"5min":0.40000000000000002,"15min":0.42699999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.46600000000000003,"5min":2.2829999999999999,"15min":2.2829999999999999},"last":0.307}]},{"osd":2,"last update":"Sun Oct 1 09:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.39000000000000001,"15min":0.39600000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.64200000000000002,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.42499999999999999,"15min":0.46800000000000003},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.72099999999999997,"5min":2.3199999999999998,"15min":2.3199999999999998},"last":0.39000000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.40999999999999998,"15min":0.41799999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.46000000000000002,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.45600000000000002,"15min":0.46700000000000003},"min":{"1min":0.33600000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.58399999999999996,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.32700000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:03:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.55600000000000005,"15min":0.55500000000000005},"min":{"1min":0.40200000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.86399999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58799999999999997,"15min":0.58199999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.86899999999999999,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.70299999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57099999999999995,"15min":0.57699999999999996},"min":{"1min":0.378,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.83699999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.84699999999999998,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.69099999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.60999999999999999,"15min":0.621},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.85499999999999998,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.61599999999999999,"15min":0.627},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76300000000000001,"5min":1.03,"15min":1.03},"last":0.73299999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58099999999999996,"15min":0.58299999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77400000000000002,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.61899999999999999,"15min":0.623},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.82999999999999996,"5min":0.999,"15min":0.999},"last":0.58199999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738431,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34499999999999997,"15min":0.34300000000000003},"min":{"1min":0.245,"5min":0.22,"15min":0.22},"max":{"1min":0.46400000000000002,"5min":0.52900000000000003,"15min":0.52900000000000003},"last":0.251},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.245,"5min":0.221,"15min":0.221},"max":{"1min":0.53600000000000003,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.42999999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:03:46 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.39600000000000002,"15min":0.40600000000000003},"min":{"1min":0.26900000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.56299999999999994,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.34000000000000002,"5min":0.376,"15min":0.38100000000000001},"min":{"1min":0.253,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.47899999999999998,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.371}]},{"osd":3,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.41799999999999998,"15min":0.42999999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.51600000000000001,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.45700000000000002,"15min":0.47299999999999998},"min":{"1min":0.28399999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56999999999999995,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.35099999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.42599999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.80000000000000004,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.41299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.77100000000000002,"5min":0.87,"15min":0.87},"last":0.69099999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:02:51 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.57299999999999995,"15min":0.57699999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.70499999999999996,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.495},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.59099999999999997,"15min":0.59699999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.63700000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.62,"15min":0.624},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.62}]},{"osd":7,"last update":"Sun Oct 1 09:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.39900000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.749,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.46000000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.58499999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411381,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.374,"15min":0.376},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.51500000000000001,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.371,"15min":0.36199999999999999},"min":{"1min":0.249,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.317}]},{"osd":1,"last update":"Sun Oct 1 09:02:48 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.379,"15min":0.375},"min":{"1min":0.253,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.373,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.56599999999999995,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.40400000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.435,"15min":0.44600000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.499,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.374,"5min":0.45400000000000001,"15min":0.47499999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.56999999999999995,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.432}]},{"osd":4,"last update":"Sun Oct 1 09:03:27 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54000000000000004,"15min":0.54100000000000004},"min":{"1min":0.434,"5min":0.374,"15min":0.374},"max":{"1min":0.65800000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.68200000000000005,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.72999999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:02:52 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56499999999999995,"15min":0.56799999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55900000000000005,"15min":0.56200000000000006},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.73399999999999999,"5min":0.749,"15min":0.749},"last":0.67900000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.59799999999999998,"15min":0.60499999999999998},"min":{"1min":0.44400000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.69899999999999995,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.60599999999999998,"15min":0.61199999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.73699999999999999,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.80500000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.58599999999999997,"15min":0.58899999999999997},"min":{"1min":0.47899999999999998,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.71599999999999997,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.60499999999999998,"15min":0.61099999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.70199999999999996,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.82799999999999996}]}]},{"osd":3,"up_from":25,"seq":107374182449,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.51,"5min":1.51,"15min":1.51},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.8599999999999999,"5min":2.8599999999999999,"15min":2.8599999999999999},"last":0.373}]},{"osd":1,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.42999999999999999,"15min":0.44800000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.503,"5min":0.62,"15min":0.62},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.37,"5min":0.40999999999999998,"15min":0.41899999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.52700000000000002,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.313}]},{"osd":2,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.44600000000000001,"15min":0.45800000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.54000000000000004,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.38,"5min":0.437,"15min":0.45600000000000002},"min":{"1min":0.27200000000000002,"5min":0.25,"15min":0.25},"max":{"1min":0.59199999999999997,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.29899999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:03:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59499999999999997,"15min":0.59699999999999998},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.371,"5min":1.371,"15min":1.371},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.59699999999999998,"15min":0.60499999999999998},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.76700000000000002,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.73799999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:02:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56699999999999995,"15min":0.56399999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.60199999999999998,"15min":0.59599999999999997},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.71999999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:03:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.59899999999999998,"15min":0.60699999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.84199999999999997,"5min":1.389,"15min":1.389},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.45300000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.80000000000000004,"5min":1.278,"15min":1.278},"last":0.57499999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.57299999999999995,"15min":0.56799999999999995},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60399999999999998,"15min":0.60199999999999998},"min":{"1min":0.44,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.82499999999999996,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.56200000000000006}]}]},{"osd":4,"up_from":30,"seq":128849018924,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.55900000000000005,"15min":0.52500000000000002},"min":{"1min":0.34999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.4459999999999997,"5min":5.4459999999999997,"15min":5.4459999999999997},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.57799999999999996,"15min":0.52600000000000002},"min":{"1min":0.35999999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.1890000000000001,"5min":7.1890000000000001,"15min":7.1890000000000001},"last":0.35299999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51200000000000001,"15min":0.51100000000000001},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.53100000000000003,"15min":0.495},"min":{"1min":0.372,"5min":0.315,"15min":0.315},"max":{"1min":4.2279999999999998,"5min":4.2279999999999998,"15min":4.2279999999999998},"last":0.38400000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.55800000000000005,"15min":0.52100000000000002},"min":{"1min":0.39300000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.3310000000000004,"5min":5.3310000000000004,"15min":5.3310000000000004},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.52000000000000002,"15min":0.51200000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.47299999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.55600000000000005,"15min":0.51700000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.6429999999999998,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":0.433},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.52300000000000002,"15min":0.51800000000000002},"min":{"1min":0.38600000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.1279999999999999,"5min":1.1279999999999999,"15min":1.1279999999999999},"last":0.49399999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:02:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.40200000000000002,"15min":0.39000000000000001},"min":{"1min":0.35799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.39000000000000001,"15min":0.372},"min":{"1min":0.39100000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.58499999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:03:14 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.45500000000000002,"15min":0.44500000000000001},"min":{"1min":0.34599999999999997,"5min":0.313,"15min":0.313},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.40200000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.56299999999999994}]},{"osd":7,"last update":"Sun Oct 1 09:03:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.53700000000000003,"15min":0.54100000000000004},"min":{"1min":0.33000000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.55400000000000005,"15min":0.55500000000000005},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.61599999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822697,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.499,"15min":0.498},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.747,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.504,"5min":0.51100000000000001,"15min":0.51500000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.60799999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.49299999999999999,"15min":0.495},"min":{"1min":0.34599999999999997,"5min":0.309,"15min":0.309},"max":{"1min":0.68300000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.54500000000000004,"15min":0.53300000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.65900000000000003,"5min":4.0490000000000004,"15min":4.0490000000000004},"last":0.47799999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.53600000000000003,"15min":0.54400000000000004},"min":{"1min":0.39500000000000002,"5min":0.308,"15min":0.308},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.63},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.51300000000000001,"15min":0.50600000000000001},"min":{"1min":0.33900000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.53600000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.53500000000000003,"15min":0.52700000000000002},"min":{"1min":0.36199999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.38800000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.65500000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.42199999999999999,"15min":0.40000000000000002},"min":{"1min":0.28599999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.67500000000000004,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.46100000000000002,"15min":0.42299999999999999},"min":{"1min":0.34699999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.81399999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:03:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.48699999999999999,"15min":0.47899999999999998},"min":{"1min":0.34100000000000003,"5min":0.313,"15min":0.313},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.47699999999999998,"15min":0.46500000000000002},"min":{"1min":0.35999999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.69699999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:03:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54000000000000004,"15min":0.53500000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.747},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.53400000000000003,"15min":0.52700000000000002},"min":{"1min":0.36499999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.73399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:03:52.529 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:03:52.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:52 smithi190 conmon[100518]: cluster 2023-10-01T09:03:50.853507+0000 mgr.y (mgr.14433) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:52.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:52 smithi138 conmon[109959]: cluster 2023-10-01T09:03:50.853507 2023-10-01T09:03:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:52 smithi138 conmon[109959]: +0000 mgr.y (mgr.14433) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:52.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:52 smithi138 conmon[100289]: cluster 2023-10-01T09:03:50.853507+0000 mgr.y (mgr.14433) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:52.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:52 smithi138 conmon[100289]: 2023-10-01T09:03:52.873 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:53.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:53 smithi190 conmon[100518]: audit 2023-10-01T09:03:51.921966+0000 mgr.y (mgr.14433) 99 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:03:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:53 smithi190 conmon[100518]: 2023-10-01T09:03:53.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:53 smithi138 conmon[100289]: audit 2023-10-01T09:03:51.921966+0000 mgr.y (mgr.14433) 99 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:03:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:53 smithi138 conmon[109959]: audit 2023-10-01T09:03 2023-10-01T09:03:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:53 smithi138 conmon[109959]: :51.921966+0000 mgr.y (mgr.14433) 99 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:03:54.669 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:03:54.669 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":14,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":329,"monmap":{"epoch":3,"min_mon_release_name":"pacific","num_mons":3},"osdmap":{"epoch":62,"num_osds":8,"num_up_osds":8,"osd_up_since":1696150890,"num_in_osds":8,"osd_in_since":1696150874,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73}],"num_pgs":73,"num_pools":4,"num_objects":22,"data_bytes":2306,"bytes_used":2438905856,"bytes_avail":765521379328,"bytes_total":767960285184},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":11,"modified":"2023-10-01T09:02:46.840508+0000","services":{}},"progress_events":{}} 2023-10-01T09:03:54.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:54 smithi190 conmon[100518]: cluster 2023-10-01T09:03:52.853833+0000 mgr.y (mgr.14433) 100 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:54.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:54 smithi138 conmon[100289]: cluster 2023-10-01T09:03:52.853833+0000 mgr.y (mgr.14433) 100 2023-10-01T09:03:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:54 smithi138 conmon[100289]: : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:54 smithi138 conmon[109959]: cluster 2023-10-01T09:03 2023-10-01T09:03:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:54 smithi138 conmon[109959]: :52.853833+0000 mgr.y (mgr.14433) 100 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:55.263 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-01T09:03:55.263 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:03:55.517 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:03:55 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:03:55] "GET /metrics HTTP/1.1" 200 191337 "" "Prometheus/2.33.4" 2023-10-01T09:03:55.519 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:55 smithi138 conmon[109959]: audit 2023-10-01T09:03:54.668527+0000 mon.a (mon.0) 679 : audit [DBG] from='client.? 172.21.15.138:0/3573580018' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:03:55.612 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:03:55.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:55 smithi190 conmon[100518]: audit 2023-10-01T09:03:54.668527+0000 mon.a (mon.0) 679 : audit [DBG] from='client.? 172.21.15.138:0/3573580018' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:03:55.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:55 smithi138 conmon[100289]: audit 2023-10-01T09:03:54.668527+0000 mon.a (mon.0) 679 : audit [DBG] from='client.? 172.21.15.138:0/3573580018' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:03:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:56 smithi190 conmon[100518]: cluster 2023-10-01T09:03:54.854265+0000 mgr.y (mgr.14433) 101 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:56.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:56 smithi138 conmon[100289]: cluster 2023-10-01T09:03:54.854265+0000 mgr.y ( 2023-10-01T09:03:56.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:56 smithi138 conmon[100289]: mgr.14433) 101 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:56.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:56 smithi138 conmon[109959]: cluster 2023-10-01T09:03:54.854265+0000 mgr.y ( 2023-10-01T09:03:56.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:56 smithi138 conmon[109959]: mgr.14433) 101 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:57.256 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:03:57.257 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:03:57.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:03:56 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:03:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:03:57.930 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":80,"stamp":"2023-10-01T09:03:56.854422+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381744,"kb_used_data":4976,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579472,"statfs":{"total":767960285184,"available":765521379328,"internally_reserved":0,"allocated":5095424,"data_stored":2165242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.217347"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.633938+0000","last_change":"2023-10-01T09:03:03.223370+0000","last_active":"2023-10-01T09:03:45.633938+0000","last_peered":"2023-10-01T09:03:45.633938+0000","last_clean":"2023-10-01T09:03:45.633938+0000","last_became_active":"2023-10-01T09:03:03.223071+0000","last_became_peered":"2023-10-01T09:03:03.223071+0000","last_unstale":"2023-10-01T09:03:45.633938+0000","last_undegraded":"2023-10-01T09:03:45.633938+0000","last_fullsized":"2023-10-01T09:03:45.633938+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.232672+0000","last_change":"2023-10-01T09:03:03.202335+0000","last_active":"2023-10-01T09:03:08.232672+0000","last_peered":"2023-10-01T09:03:08.232672+0000","last_clean":"2023-10-01T09:03:08.232672+0000","last_became_active":"2023-10-01T09:03:03.201485+0000","last_became_peered":"2023-10-01T09:03:03.201485+0000","last_unstale":"2023-10-01T09:03:08.232672+0000","last_undegraded":"2023-10-01T09:03:08.232672+0000","last_fullsized":"2023-10-01T09:03:08.232672+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635131+0000","last_change":"2023-10-01T09:03:03.224058+0000","last_active":"2023-10-01T09:03:45.635131+0000","last_peered":"2023-10-01T09:03:45.635131+0000","last_clean":"2023-10-01T09:03:45.635131+0000","last_became_active":"2023-10-01T09:03:03.223598+0000","last_became_peered":"2023-10-01T09:03:03.223598+0000","last_unstale":"2023-10-01T09:03:45.635131+0000","last_undegraded":"2023-10-01T09:03:45.635131+0000","last_fullsized":"2023-10-01T09:03:45.635131+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635683+0000","last_change":"2023-10-01T09:03:03.193542+0000","last_active":"2023-10-01T09:03:45.635683+0000","last_peered":"2023-10-01T09:03:45.635683+0000","last_clean":"2023-10-01T09:03:45.635683+0000","last_became_active":"2023-10-01T09:03:03.193245+0000","last_became_peered":"2023-10-01T09:03:03.193245+0000","last_unstale":"2023-10-01T09:03:45.635683+0000","last_undegraded":"2023-10-01T09:03:45.635683+0000","last_fullsized":"2023-10-01T09:03:45.635683+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634507+0000","last_change":"2023-10-01T09:03:03.218020+0000","last_active":"2023-10-01T09:03:45.634507+0000","last_peered":"2023-10-01T09:03:45.634507+0000","last_clean":"2023-10-01T09:03:45.634507+0000","last_became_active":"2023-10-01T09:03:03.217717+0000","last_became_peered":"2023-10-01T09:03:03.217717+0000","last_unstale":"2023-10-01T09:03:45.634507+0000","last_undegraded":"2023-10-01T09:03:45.634507+0000","last_fullsized":"2023-10-01T09:03:45.634507+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.633112+0000","last_change":"2023-10-01T09:03:03.202063+0000","last_active":"2023-10-01T09:03:45.633112+0000","last_peered":"2023-10-01T09:03:45.633112+0000","last_clean":"2023-10-01T09:03:45.633112+0000","last_became_active":"2023-10-01T09:03:03.201769+0000","last_became_peered":"2023-10-01T09:03:03.201769+0000","last_unstale":"2023-10-01T09:03:45.633112+0000","last_undegraded":"2023-10-01T09:03:45.633112+0000","last_fullsized":"2023-10-01T09:03:45.633112+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635109+0000","last_change":"2023-10-01T09:03:03.221313+0000","last_active":"2023-10-01T09:03:45.635109+0000","last_peered":"2023-10-01T09:03:45.635109+0000","last_clean":"2023-10-01T09:03:45.635109+0000","last_became_active":"2023-10-01T09:03:03.221080+0000","last_became_peered":"2023-10-01T09:03:03.221080+0000","last_unstale":"2023-10-01T09:03:45.635109+0000","last_undegraded":"2023-10-01T09:03:45.635109+0000","last_fullsized":"2023-10-01T09:03:45.635109+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.633222+0000","last_change":"2023-10-01T09:03:03.201659+0000","last_active":"2023-10-01T09:03:45.633222+0000","last_peered":"2023-10-01T09:03:45.633222+0000","last_clean":"2023-10-01T09:03:45.633222+0000","last_became_active":"2023-10-01T09:03:03.201348+0000","last_became_peered":"2023-10-01T09:03:03.201348+0000","last_unstale":"2023-10-01T09:03:45.633222+0000","last_undegraded":"2023-10-01T09:03:45.633222+0000","last_fullsized":"2023-10-01T09:03:45.633222+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.631910+0000","last_change":"2023-10-01T09:03:03.226772+0000","last_active":"2023-10-01T09:03:45.631910+0000","last_peered":"2023-10-01T09:03:45.631910+0000","last_clean":"2023-10-01T09:03:45.631910+0000","last_became_active":"2023-10-01T09:03:03.226446+0000","last_became_peered":"2023-10-01T09:03:03.226446+0000","last_unstale":"2023-10-01T09:03:45.631910+0000","last_undegraded":"2023-10-01T09:03:45.631910+0000","last_fullsized":"2023-10-01T09:03:45.631910+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.231820+0000","last_change":"2023-10-01T09:03:03.203002+0000","last_active":"2023-10-01T09:03:08.231820+0000","last_peered":"2023-10-01T09:03:08.231820+0000","last_clean":"2023-10-01T09:03:08.231820+0000","last_became_active":"2023-10-01T09:03:03.202571+0000","last_became_peered":"2023-10-01T09:03:03.202571+0000","last_unstale":"2023-10-01T09:03:08.231820+0000","last_undegraded":"2023-10-01T09:03:08.231820+0000","last_fullsized":"2023-10-01T09:03:08.231820+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634224+0000","last_change":"2023-10-01T09:03:03.754839+0000","last_active":"2023-10-01T09:03:45.634224+0000","last_peered":"2023-10-01T09:03:45.634224+0000","last_clean":"2023-10-01T09:03:45.634224+0000","last_became_active":"2023-10-01T09:03:03.754506+0000","last_became_peered":"2023-10-01T09:03:03.754506+0000","last_unstale":"2023-10-01T09:03:45.634224+0000","last_undegraded":"2023-10-01T09:03:45.634224+0000","last_fullsized":"2023-10-01T09:03:45.634224+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:12.687636+0000","last_change":"2023-10-01T09:03:03.196477+0000","last_active":"2023-10-01T09:03:12.687636+0000","last_peered":"2023-10-01T09:03:12.687636+0000","last_clean":"2023-10-01T09:03:12.687636+0000","last_became_active":"2023-10-01T09:03:03.196266+0000","last_became_peered":"2023-10-01T09:03:03.196266+0000","last_unstale":"2023-10-01T09:03:12.687636+0000","last_undegraded":"2023-10-01T09:03:12.687636+0000","last_fullsized":"2023-10-01T09:03:12.687636+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.636147+0000","last_change":"2023-10-01T09:02:06.997887+0000","last_active":"2023-10-01T09:03:45.636147+0000","last_peered":"2023-10-01T09:03:45.636147+0000","last_clean":"2023-10-01T09:03:45.636147+0000","last_became_active":"2023-10-01T09:02:06.997597+0000","last_became_peered":"2023-10-01T09:02:06.997597+0000","last_unstale":"2023-10-01T09:03:45.636147+0000","last_undegraded":"2023-10-01T09:03:45.636147+0000","last_fullsized":"2023-10-01T09:03:45.636147+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635006+0000","last_change":"2023-10-01T09:03:03.228390+0000","last_active":"2023-10-01T09:03:45.635006+0000","last_peered":"2023-10-01T09:03:45.635006+0000","last_clean":"2023-10-01T09:03:45.635006+0000","last_became_active":"2023-10-01T09:03:03.227254+0000","last_became_peered":"2023-10-01T09:03:03.227254+0000","last_unstale":"2023-10-01T09:03:45.635006+0000","last_undegraded":"2023-10-01T09:03:45.635006+0000","last_fullsized":"2023-10-01T09:03:45.635006+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632276+0000","last_change":"2023-10-01T09:02:07.442083+0000","last_active":"2023-10-01T09:03:45.632276+0000","last_peered":"2023-10-01T09:03:45.632276+0000","last_clean":"2023-10-01T09:03:45.632276+0000","last_became_active":"2023-10-01T09:02:07.441757+0000","last_became_peered":"2023-10-01T09:02:07.441757+0000","last_unstale":"2023-10-01T09:03:45.632276+0000","last_undegraded":"2023-10-01T09:03:45.632276+0000","last_fullsized":"2023-10-01T09:03:45.632276+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635624+0000","last_change":"2023-10-01T09:03:03.198510+0000","last_active":"2023-10-01T09:03:45.635624+0000","last_peered":"2023-10-01T09:03:45.635624+0000","last_clean":"2023-10-01T09:03:45.635624+0000","last_became_active":"2023-10-01T09:03:03.198172+0000","last_became_peered":"2023-10-01T09:03:03.198172+0000","last_unstale":"2023-10-01T09:03:45.635624+0000","last_undegraded":"2023-10-01T09:03:45.635624+0000","last_fullsized":"2023-10-01T09:03:45.635624+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215706+0000","last_change":"2023-10-01T09:03:03.754828+0000","last_active":"2023-10-01T09:03:05.215706+0000","last_peered":"2023-10-01T09:03:05.215706+0000","last_clean":"2023-10-01T09:03:05.215706+0000","last_became_active":"2023-10-01T09:03:03.754492+0000","last_became_peered":"2023-10-01T09:03:03.754492+0000","last_unstale":"2023-10-01T09:03:05.215706+0000","last_undegraded":"2023-10-01T09:03:05.215706+0000","last_fullsized":"2023-10-01T09:03:05.215706+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632764+0000","last_change":"2023-10-01T09:03:03.203345+0000","last_active":"2023-10-01T09:03:45.632764+0000","last_peered":"2023-10-01T09:03:45.632764+0000","last_clean":"2023-10-01T09:03:45.632764+0000","last_became_active":"2023-10-01T09:03:03.203035+0000","last_became_peered":"2023-10-01T09:03:03.203035+0000","last_unstale":"2023-10-01T09:03:45.632764+0000","last_undegraded":"2023-10-01T09:03:45.632764+0000","last_fullsized":"2023-10-01T09:03:45.632764+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932484+0000","last_change":"2023-10-01T09:02:07.442452+0000","last_active":"2023-10-01T09:03:45.932484+0000","last_peered":"2023-10-01T09:03:45.932484+0000","last_clean":"2023-10-01T09:03:45.932484+0000","last_became_active":"2023-10-01T09:02:07.442111+0000","last_became_peered":"2023-10-01T09:02:07.442111+0000","last_unstale":"2023-10-01T09:03:45.932484+0000","last_undegraded":"2023-10-01T09:03:45.932484+0000","last_fullsized":"2023-10-01T09:03:45.932484+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215738+0000","last_change":"2023-10-01T09:03:03.223326+0000","last_active":"2023-10-01T09:03:05.215738+0000","last_peered":"2023-10-01T09:03:05.215738+0000","last_clean":"2023-10-01T09:03:05.215738+0000","last_became_active":"2023-10-01T09:03:03.222974+0000","last_became_peered":"2023-10-01T09:03:03.222974+0000","last_unstale":"2023-10-01T09:03:05.215738+0000","last_undegraded":"2023-10-01T09:03:05.215738+0000","last_fullsized":"2023-10-01T09:03:05.215738+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215539+0000","last_change":"2023-10-01T09:03:03.195712+0000","last_active":"2023-10-01T09:03:05.215539+0000","last_peered":"2023-10-01T09:03:05.215539+0000","last_clean":"2023-10-01T09:03:05.215539+0000","last_became_active":"2023-10-01T09:03:03.195408+0000","last_became_peered":"2023-10-01T09:03:03.195408+0000","last_unstale":"2023-10-01T09:03:05.215539+0000","last_undegraded":"2023-10-01T09:03:05.215539+0000","last_fullsized":"2023-10-01T09:03:05.215539+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932093+0000","last_change":"2023-10-01T09:02:06.999043+0000","last_active":"2023-10-01T09:03:45.932093+0000","last_peered":"2023-10-01T09:03:45.932093+0000","last_clean":"2023-10-01T09:03:45.932093+0000","last_became_active":"2023-10-01T09:02:06.998671+0000","last_became_peered":"2023-10-01T09:02:06.998671+0000","last_unstale":"2023-10-01T09:03:45.932093+0000","last_undegraded":"2023-10-01T09:03:45.932093+0000","last_fullsized":"2023-10-01T09:03:45.932093+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635816+0000","last_change":"2023-10-01T09:03:03.227254+0000","last_active":"2023-10-01T09:03:45.635816+0000","last_peered":"2023-10-01T09:03:45.635816+0000","last_clean":"2023-10-01T09:03:45.635816+0000","last_became_active":"2023-10-01T09:03:03.226941+0000","last_became_peered":"2023-10-01T09:03:03.226941+0000","last_unstale":"2023-10-01T09:03:45.635816+0000","last_undegraded":"2023-10-01T09:03:45.635816+0000","last_fullsized":"2023-10-01T09:03:45.635816+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635127+0000","last_change":"2023-10-01T09:03:03.198835+0000","last_active":"2023-10-01T09:03:45.635127+0000","last_peered":"2023-10-01T09:03:45.635127+0000","last_clean":"2023-10-01T09:03:45.635127+0000","last_became_active":"2023-10-01T09:03:03.198424+0000","last_became_peered":"2023-10-01T09:03:03.198424+0000","last_unstale":"2023-10-01T09:03:45.635127+0000","last_undegraded":"2023-10-01T09:03:45.635127+0000","last_fullsized":"2023-10-01T09:03:45.635127+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932052+0000","last_change":"2023-10-01T09:01:32.510259+0000","last_active":"2023-10-01T09:03:45.932052+0000","last_peered":"2023-10-01T09:03:45.932052+0000","last_clean":"2023-10-01T09:03:45.932052+0000","last_became_active":"2023-10-01T09:01:32.509905+0000","last_became_peered":"2023-10-01T09:01:32.509905+0000","last_unstale":"2023-10-01T09:03:45.932052+0000","last_undegraded":"2023-10-01T09:03:45.932052+0000","last_fullsized":"2023-10-01T09:03:45.932052+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632783+0000","last_change":"2023-10-01T09:02:06.998804+0000","last_active":"2023-10-01T09:03:45.632783+0000","last_peered":"2023-10-01T09:03:45.632783+0000","last_clean":"2023-10-01T09:03:45.632783+0000","last_became_active":"2023-10-01T09:02:06.998516+0000","last_became_peered":"2023-10-01T09:02:06.998516+0000","last_unstale":"2023-10-01T09:03:45.632783+0000","last_undegraded":"2023-10-01T09:03:45.632783+0000","last_fullsized":"2023-10-01T09:03:45.632783+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632512+0000","last_change":"2023-10-01T09:03:03.756021+0000","last_active":"2023-10-01T09:03:45.632512+0000","last_peered":"2023-10-01T09:03:45.632512+0000","last_clean":"2023-10-01T09:03:45.632512+0000","last_became_active":"2023-10-01T09:03:03.755659+0000","last_became_peered":"2023-10-01T09:03:03.755659+0000","last_unstale":"2023-10-01T09:03:45.632512+0000","last_undegraded":"2023-10-01T09:03:45.632512+0000","last_fullsized":"2023-10-01T09:03:45.632512+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632396+0000","last_change":"2023-10-01T09:03:03.201620+0000","last_active":"2023-10-01T09:03:45.632396+0000","last_peered":"2023-10-01T09:03:45.632396+0000","last_clean":"2023-10-01T09:03:45.632396+0000","last_became_active":"2023-10-01T09:03:03.201312+0000","last_became_peered":"2023-10-01T09:03:03.201312+0000","last_unstale":"2023-10-01T09:03:45.632396+0000","last_undegraded":"2023-10-01T09:03:45.632396+0000","last_fullsized":"2023-10-01T09:03:45.632396+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.631546+0000","last_change":"2023-10-01T09:02:07.436492+0000","last_active":"2023-10-01T09:03:45.631546+0000","last_peered":"2023-10-01T09:03:45.631546+0000","last_clean":"2023-10-01T09:03:45.631546+0000","last_became_active":"2023-10-01T09:02:07.436146+0000","last_became_peered":"2023-10-01T09:02:07.436146+0000","last_unstale":"2023-10-01T09:03:45.631546+0000","last_undegraded":"2023-10-01T09:03:45.631546+0000","last_fullsized":"2023-10-01T09:03:45.631546+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632674+0000","last_change":"2023-10-01T09:03:03.227284+0000","last_active":"2023-10-01T09:03:45.632674+0000","last_peered":"2023-10-01T09:03:45.632674+0000","last_clean":"2023-10-01T09:03:45.632674+0000","last_became_active":"2023-10-01T09:03:03.226987+0000","last_became_peered":"2023-10-01T09:03:03.226987+0000","last_unstale":"2023-10-01T09:03:45.632674+0000","last_undegraded":"2023-10-01T09:03:45.632674+0000","last_fullsized":"2023-10-01T09:03:45.632674+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634293+0000","last_change":"2023-10-01T09:03:03.196357+0000","last_active":"2023-10-01T09:03:45.634293+0000","last_peered":"2023-10-01T09:03:45.634293+0000","last_clean":"2023-10-01T09:03:45.634293+0000","last_became_active":"2023-10-01T09:03:03.196017+0000","last_became_peered":"2023-10-01T09:03:03.196017+0000","last_unstale":"2023-10-01T09:03:45.634293+0000","last_undegraded":"2023-10-01T09:03:45.634293+0000","last_fullsized":"2023-10-01T09:03:45.634293+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632885+0000","last_change":"2023-10-01T09:02:07.001644+0000","last_active":"2023-10-01T09:03:45.632885+0000","last_peered":"2023-10-01T09:03:45.632885+0000","last_clean":"2023-10-01T09:03:45.632885+0000","last_became_active":"2023-10-01T09:02:07.001197+0000","last_became_peered":"2023-10-01T09:02:07.001197+0000","last_unstale":"2023-10-01T09:03:45.632885+0000","last_undegraded":"2023-10-01T09:03:45.632885+0000","last_fullsized":"2023-10-01T09:03:45.632885+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632588+0000","last_change":"2023-10-01T09:03:03.755027+0000","last_active":"2023-10-01T09:03:45.632588+0000","last_peered":"2023-10-01T09:03:45.632588+0000","last_clean":"2023-10-01T09:03:45.632588+0000","last_became_active":"2023-10-01T09:03:03.754596+0000","last_became_peered":"2023-10-01T09:03:03.754596+0000","last_unstale":"2023-10-01T09:03:45.632588+0000","last_undegraded":"2023-10-01T09:03:45.632588+0000","last_fullsized":"2023-10-01T09:03:45.632588+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.633017+0000","last_change":"2023-10-01T09:03:03.203012+0000","last_active":"2023-10-01T09:03:45.633017+0000","last_peered":"2023-10-01T09:03:45.633017+0000","last_clean":"2023-10-01T09:03:45.633017+0000","last_became_active":"2023-10-01T09:03:03.202051+0000","last_became_peered":"2023-10-01T09:03:03.202051+0000","last_unstale":"2023-10-01T09:03:45.633017+0000","last_undegraded":"2023-10-01T09:03:45.633017+0000","last_fullsized":"2023-10-01T09:03:45.633017+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632616+0000","last_change":"2023-10-01T09:02:06.998188+0000","last_active":"2023-10-01T09:03:45.632616+0000","last_peered":"2023-10-01T09:03:45.632616+0000","last_clean":"2023-10-01T09:03:45.632616+0000","last_became_active":"2023-10-01T09:02:06.997855+0000","last_became_peered":"2023-10-01T09:02:06.997855+0000","last_unstale":"2023-10-01T09:03:45.632616+0000","last_undegraded":"2023-10-01T09:03:45.632616+0000","last_fullsized":"2023-10-01T09:03:45.632616+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T09:03:46.638938+0000","last_change":"2023-10-01T09:03:46.638938+0000","last_active":"2023-10-01T09:03:46.638938+0000","last_peered":"2023-10-01T09:03:46.638938+0000","last_clean":"2023-10-01T09:03:46.638938+0000","last_became_active":"2023-10-01T09:03:46.638713+0000","last_became_peered":"2023-10-01T09:03:46.638713+0000","last_unstale":"2023-10-01T09:03:46.638938+0000","last_undegraded":"2023-10-01T09:03:46.638938+0000","last_fullsized":"2023-10-01T09:03:46.638938+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635017+0000","last_change":"2023-10-01T09:03:03.200838+0000","last_active":"2023-10-01T09:03:45.635017+0000","last_peered":"2023-10-01T09:03:45.635017+0000","last_clean":"2023-10-01T09:03:45.635017+0000","last_became_active":"2023-10-01T09:03:03.199715+0000","last_became_peered":"2023-10-01T09:03:03.199715+0000","last_unstale":"2023-10-01T09:03:45.635017+0000","last_undegraded":"2023-10-01T09:03:45.635017+0000","last_fullsized":"2023-10-01T09:03:45.635017+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634588+0000","last_change":"2023-10-01T09:03:03.221266+0000","last_active":"2023-10-01T09:03:45.634588+0000","last_peered":"2023-10-01T09:03:45.634588+0000","last_clean":"2023-10-01T09:03:45.634588+0000","last_became_active":"2023-10-01T09:03:03.220690+0000","last_became_peered":"2023-10-01T09:03:03.220690+0000","last_unstale":"2023-10-01T09:03:45.634588+0000","last_undegraded":"2023-10-01T09:03:45.634588+0000","last_fullsized":"2023-10-01T09:03:45.634588+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634373+0000","last_change":"2023-10-01T09:03:03.199237+0000","last_active":"2023-10-01T09:03:45.634373+0000","last_peered":"2023-10-01T09:03:45.634373+0000","last_clean":"2023-10-01T09:03:45.634373+0000","last_became_active":"2023-10-01T09:03:03.198950+0000","last_became_peered":"2023-10-01T09:03:03.198950+0000","last_unstale":"2023-10-01T09:03:45.634373+0000","last_undegraded":"2023-10-01T09:03:45.634373+0000","last_fullsized":"2023-10-01T09:03:45.634373+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634678+0000","last_change":"2023-10-01T09:03:03.223290+0000","last_active":"2023-10-01T09:03:45.634678+0000","last_peered":"2023-10-01T09:03:45.634678+0000","last_clean":"2023-10-01T09:03:45.634678+0000","last_became_active":"2023-10-01T09:03:03.223013+0000","last_became_peered":"2023-10-01T09:03:03.223013+0000","last_unstale":"2023-10-01T09:03:45.634678+0000","last_undegraded":"2023-10-01T09:03:45.634678+0000","last_fullsized":"2023-10-01T09:03:45.634678+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635379+0000","last_change":"2023-10-01T09:03:03.198744+0000","last_active":"2023-10-01T09:03:45.635379+0000","last_peered":"2023-10-01T09:03:45.635379+0000","last_clean":"2023-10-01T09:03:45.635379+0000","last_became_active":"2023-10-01T09:03:03.198452+0000","last_became_peered":"2023-10-01T09:03:03.198452+0000","last_unstale":"2023-10-01T09:03:45.635379+0000","last_undegraded":"2023-10-01T09:03:45.635379+0000","last_fullsized":"2023-10-01T09:03:45.635379+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634570+0000","last_change":"2023-10-01T09:03:03.218071+0000","last_active":"2023-10-01T09:03:45.634570+0000","last_peered":"2023-10-01T09:03:45.634570+0000","last_clean":"2023-10-01T09:03:45.634570+0000","last_became_active":"2023-10-01T09:03:03.217776+0000","last_became_peered":"2023-10-01T09:03:03.217776+0000","last_unstale":"2023-10-01T09:03:45.634570+0000","last_undegraded":"2023-10-01T09:03:45.634570+0000","last_fullsized":"2023-10-01T09:03:45.634570+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635229+0000","last_change":"2023-10-01T09:03:03.200515+0000","last_active":"2023-10-01T09:03:45.635229+0000","last_peered":"2023-10-01T09:03:45.635229+0000","last_clean":"2023-10-01T09:03:45.635229+0000","last_became_active":"2023-10-01T09:03:03.199536+0000","last_became_peered":"2023-10-01T09:03:03.199536+0000","last_unstale":"2023-10-01T09:03:45.635229+0000","last_undegraded":"2023-10-01T09:03:45.635229+0000","last_fullsized":"2023-10-01T09:03:45.635229+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215903+0000","last_change":"2023-10-01T09:03:03.224884+0000","last_active":"2023-10-01T09:03:05.215903+0000","last_peered":"2023-10-01T09:03:05.215903+0000","last_clean":"2023-10-01T09:03:05.215903+0000","last_became_active":"2023-10-01T09:03:03.224529+0000","last_became_peered":"2023-10-01T09:03:03.224529+0000","last_unstale":"2023-10-01T09:03:05.215903+0000","last_undegraded":"2023-10-01T09:03:05.215903+0000","last_fullsized":"2023-10-01T09:03:05.215903+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632780+0000","last_change":"2023-10-01T09:03:03.203721+0000","last_active":"2023-10-01T09:03:45.632780+0000","last_peered":"2023-10-01T09:03:45.632780+0000","last_clean":"2023-10-01T09:03:45.632780+0000","last_became_active":"2023-10-01T09:03:03.203308+0000","last_became_peered":"2023-10-01T09:03:03.203308+0000","last_unstale":"2023-10-01T09:03:45.632780+0000","last_undegraded":"2023-10-01T09:03:45.632780+0000","last_fullsized":"2023-10-01T09:03:45.632780+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635266+0000","last_change":"2023-10-01T09:03:03.227070+0000","last_active":"2023-10-01T09:03:45.635266+0000","last_peered":"2023-10-01T09:03:45.635266+0000","last_clean":"2023-10-01T09:03:45.635266+0000","last_became_active":"2023-10-01T09:03:03.226752+0000","last_became_peered":"2023-10-01T09:03:03.226752+0000","last_unstale":"2023-10-01T09:03:45.635266+0000","last_undegraded":"2023-10-01T09:03:45.635266+0000","last_fullsized":"2023-10-01T09:03:45.635266+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932534+0000","last_change":"2023-10-01T09:03:03.200972+0000","last_active":"2023-10-01T09:03:45.932534+0000","last_peered":"2023-10-01T09:03:45.932534+0000","last_clean":"2023-10-01T09:03:45.932534+0000","last_became_active":"2023-10-01T09:03:03.200695+0000","last_became_peered":"2023-10-01T09:03:03.200695+0000","last_unstale":"2023-10-01T09:03:45.932534+0000","last_undegraded":"2023-10-01T09:03:45.932534+0000","last_fullsized":"2023-10-01T09:03:45.932534+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634095+0000","last_change":"2023-10-01T09:03:03.220741+0000","last_active":"2023-10-01T09:03:45.634095+0000","last_peered":"2023-10-01T09:03:45.634095+0000","last_clean":"2023-10-01T09:03:45.634095+0000","last_became_active":"2023-10-01T09:03:03.220481+0000","last_became_peered":"2023-10-01T09:03:03.220481+0000","last_unstale":"2023-10-01T09:03:45.634095+0000","last_undegraded":"2023-10-01T09:03:45.634095+0000","last_fullsized":"2023-10-01T09:03:45.634095+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932558+0000","last_change":"2023-10-01T09:03:03.200367+0000","last_active":"2023-10-01T09:03:45.932558+0000","last_peered":"2023-10-01T09:03:45.932558+0000","last_clean":"2023-10-01T09:03:45.932558+0000","last_became_active":"2023-10-01T09:03:03.200068+0000","last_became_peered":"2023-10-01T09:03:03.200068+0000","last_unstale":"2023-10-01T09:03:45.932558+0000","last_undegraded":"2023-10-01T09:03:45.932558+0000","last_fullsized":"2023-10-01T09:03:45.932558+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632631+0000","last_change":"2023-10-01T09:03:04.245105+0000","last_active":"2023-10-01T09:03:45.632631+0000","last_peered":"2023-10-01T09:03:45.632631+0000","last_clean":"2023-10-01T09:03:45.632631+0000","last_became_active":"2023-10-01T09:03:04.244807+0000","last_became_peered":"2023-10-01T09:03:04.244807+0000","last_unstale":"2023-10-01T09:03:45.632631+0000","last_undegraded":"2023-10-01T09:03:45.632631+0000","last_fullsized":"2023-10-01T09:03:45.632631+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932682+0000","last_change":"2023-10-01T09:03:03.199109+0000","last_active":"2023-10-01T09:03:45.932682+0000","last_peered":"2023-10-01T09:03:45.932682+0000","last_clean":"2023-10-01T09:03:45.932682+0000","last_became_active":"2023-10-01T09:03:03.198810+0000","last_became_peered":"2023-10-01T09:03:03.198810+0000","last_unstale":"2023-10-01T09:03:45.932682+0000","last_undegraded":"2023-10-01T09:03:45.932682+0000","last_fullsized":"2023-10-01T09:03:45.932682+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635430+0000","last_change":"2023-10-01T09:03:03.227888+0000","last_active":"2023-10-01T09:03:45.635430+0000","last_peered":"2023-10-01T09:03:45.635430+0000","last_clean":"2023-10-01T09:03:45.635430+0000","last_became_active":"2023-10-01T09:03:03.227621+0000","last_became_peered":"2023-10-01T09:03:03.227621+0000","last_unstale":"2023-10-01T09:03:45.635430+0000","last_undegraded":"2023-10-01T09:03:45.635430+0000","last_fullsized":"2023-10-01T09:03:45.635430+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635126+0000","last_change":"2023-10-01T09:03:03.199462+0000","last_active":"2023-10-01T09:03:45.635126+0000","last_peered":"2023-10-01T09:03:45.635126+0000","last_clean":"2023-10-01T09:03:45.635126+0000","last_became_active":"2023-10-01T09:03:03.199188+0000","last_became_peered":"2023-10-01T09:03:03.199188+0000","last_unstale":"2023-10-01T09:03:45.635126+0000","last_undegraded":"2023-10-01T09:03:45.635126+0000","last_fullsized":"2023-10-01T09:03:45.635126+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215653+0000","last_change":"2023-10-01T09:03:03.228462+0000","last_active":"2023-10-01T09:03:05.215653+0000","last_peered":"2023-10-01T09:03:05.215653+0000","last_clean":"2023-10-01T09:03:05.215653+0000","last_became_active":"2023-10-01T09:03:03.228010+0000","last_became_peered":"2023-10-01T09:03:03.228010+0000","last_unstale":"2023-10-01T09:03:05.215653+0000","last_undegraded":"2023-10-01T09:03:05.215653+0000","last_fullsized":"2023-10-01T09:03:05.215653+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932159+0000","last_change":"2023-10-01T09:03:03.194952+0000","last_active":"2023-10-01T09:03:45.932159+0000","last_peered":"2023-10-01T09:03:45.932159+0000","last_clean":"2023-10-01T09:03:45.932159+0000","last_became_active":"2023-10-01T09:03:03.194643+0000","last_became_peered":"2023-10-01T09:03:03.194643+0000","last_unstale":"2023-10-01T09:03:45.932159+0000","last_undegraded":"2023-10-01T09:03:45.932159+0000","last_fullsized":"2023-10-01T09:03:45.932159+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632819+0000","last_change":"2023-10-01T09:03:04.405231+0000","last_active":"2023-10-01T09:03:45.632819+0000","last_peered":"2023-10-01T09:03:45.632819+0000","last_clean":"2023-10-01T09:03:45.632819+0000","last_became_active":"2023-10-01T09:03:04.404876+0000","last_became_peered":"2023-10-01T09:03:04.404876+0000","last_unstale":"2023-10-01T09:03:45.632819+0000","last_undegraded":"2023-10-01T09:03:45.632819+0000","last_fullsized":"2023-10-01T09:03:45.632819+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215761+0000","last_change":"2023-10-01T09:03:03.202905+0000","last_active":"2023-10-01T09:03:05.215761+0000","last_peered":"2023-10-01T09:03:05.215761+0000","last_clean":"2023-10-01T09:03:05.215761+0000","last_became_active":"2023-10-01T09:03:03.202621+0000","last_became_peered":"2023-10-01T09:03:03.202621+0000","last_unstale":"2023-10-01T09:03:05.215761+0000","last_undegraded":"2023-10-01T09:03:05.215761+0000","last_fullsized":"2023-10-01T09:03:05.215761+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635454+0000","last_change":"2023-10-01T09:03:03.228053+0000","last_active":"2023-10-01T09:03:45.635454+0000","last_peered":"2023-10-01T09:03:45.635454+0000","last_clean":"2023-10-01T09:03:45.635454+0000","last_became_active":"2023-10-01T09:03:03.227153+0000","last_became_peered":"2023-10-01T09:03:03.227153+0000","last_unstale":"2023-10-01T09:03:45.635454+0000","last_undegraded":"2023-10-01T09:03:45.635454+0000","last_fullsized":"2023-10-01T09:03:45.635454+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932302+0000","last_change":"2023-10-01T09:03:03.195227+0000","last_active":"2023-10-01T09:03:45.932302+0000","last_peered":"2023-10-01T09:03:45.932302+0000","last_clean":"2023-10-01T09:03:45.932302+0000","last_became_active":"2023-10-01T09:03:03.194964+0000","last_became_peered":"2023-10-01T09:03:03.194964+0000","last_unstale":"2023-10-01T09:03:45.932302+0000","last_undegraded":"2023-10-01T09:03:45.932302+0000","last_fullsized":"2023-10-01T09:03:45.932302+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":21,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T09:03:46.638524+0000","last_change":"2023-10-01T09:03:46.638524+0000","last_active":"2023-10-01T09:03:46.638524+0000","last_peered":"2023-10-01T09:03:46.638524+0000","last_clean":"2023-10-01T09:03:46.638524+0000","last_became_active":"2023-10-01T09:03:46.638209+0000","last_became_peered":"2023-10-01T09:03:46.638209+0000","last_unstale":"2023-10-01T09:03:46.638524+0000","last_undegraded":"2023-10-01T09:03:46.638524+0000","last_fullsized":"2023-10-01T09:03:46.638524+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":19,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634523+0000","last_change":"2023-10-01T09:03:03.200261+0000","last_active":"2023-10-01T09:03:45.634523+0000","last_peered":"2023-10-01T09:03:45.634523+0000","last_clean":"2023-10-01T09:03:45.634523+0000","last_became_active":"2023-10-01T09:03:03.199944+0000","last_became_peered":"2023-10-01T09:03:03.199944+0000","last_unstale":"2023-10-01T09:03:45.634523+0000","last_undegraded":"2023-10-01T09:03:45.634523+0000","last_fullsized":"2023-10-01T09:03:45.634523+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632559+0000","last_change":"2023-10-01T09:03:03.224723+0000","last_active":"2023-10-01T09:03:45.632559+0000","last_peered":"2023-10-01T09:03:45.632559+0000","last_clean":"2023-10-01T09:03:45.632559+0000","last_became_active":"2023-10-01T09:03:03.224379+0000","last_became_peered":"2023-10-01T09:03:03.224379+0000","last_unstale":"2023-10-01T09:03:45.632559+0000","last_undegraded":"2023-10-01T09:03:45.632559+0000","last_fullsized":"2023-10-01T09:03:45.632559+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932421+0000","last_change":"2023-10-01T09:03:03.199740+0000","last_active":"2023-10-01T09:03:45.932421+0000","last_peered":"2023-10-01T09:03:45.932421+0000","last_clean":"2023-10-01T09:03:45.932421+0000","last_became_active":"2023-10-01T09:03:03.199421+0000","last_became_peered":"2023-10-01T09:03:03.199421+0000","last_unstale":"2023-10-01T09:03:45.932421+0000","last_undegraded":"2023-10-01T09:03:45.932421+0000","last_fullsized":"2023-10-01T09:03:45.932421+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635238+0000","last_change":"2023-10-01T09:03:03.228015+0000","last_active":"2023-10-01T09:03:45.635238+0000","last_peered":"2023-10-01T09:03:45.635238+0000","last_clean":"2023-10-01T09:03:45.635238+0000","last_became_active":"2023-10-01T09:03:03.227754+0000","last_became_peered":"2023-10-01T09:03:03.227754+0000","last_unstale":"2023-10-01T09:03:45.635238+0000","last_undegraded":"2023-10-01T09:03:45.635238+0000","last_fullsized":"2023-10-01T09:03:45.635238+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632832+0000","last_change":"2023-10-01T09:03:03.201187+0000","last_active":"2023-10-01T09:03:45.632832+0000","last_peered":"2023-10-01T09:03:45.632832+0000","last_clean":"2023-10-01T09:03:45.632832+0000","last_became_active":"2023-10-01T09:03:03.200859+0000","last_became_peered":"2023-10-01T09:03:03.200859+0000","last_unstale":"2023-10-01T09:03:45.632832+0000","last_undegraded":"2023-10-01T09:03:45.632832+0000","last_fullsized":"2023-10-01T09:03:45.632832+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635189+0000","last_change":"2023-10-01T09:03:03.227458+0000","last_active":"2023-10-01T09:03:45.635189+0000","last_peered":"2023-10-01T09:03:45.635189+0000","last_clean":"2023-10-01T09:03:45.635189+0000","last_became_active":"2023-10-01T09:03:03.226857+0000","last_became_peered":"2023-10-01T09:03:03.226857+0000","last_unstale":"2023-10-01T09:03:45.635189+0000","last_undegraded":"2023-10-01T09:03:45.635189+0000","last_fullsized":"2023-10-01T09:03:45.635189+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215921+0000","last_change":"2023-10-01T09:03:03.202503+0000","last_active":"2023-10-01T09:03:05.215921+0000","last_peered":"2023-10-01T09:03:05.215921+0000","last_clean":"2023-10-01T09:03:05.215921+0000","last_became_active":"2023-10-01T09:03:03.202048+0000","last_became_peered":"2023-10-01T09:03:03.202048+0000","last_unstale":"2023-10-01T09:03:05.215921+0000","last_undegraded":"2023-10-01T09:03:05.215921+0000","last_fullsized":"2023-10-01T09:03:05.215921+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T09:03:46.767306+0000","last_change":"2023-10-01T09:03:46.767306+0000","last_active":"2023-10-01T09:03:46.767306+0000","last_peered":"2023-10-01T09:03:46.767306+0000","last_clean":"2023-10-01T09:03:46.767306+0000","last_became_active":"2023-10-01T09:03:46.766871+0000","last_became_peered":"2023-10-01T09:03:46.766871+0000","last_unstale":"2023-10-01T09:03:46.767306+0000","last_undegraded":"2023-10-01T09:03:46.767306+0000","last_fullsized":"2023-10-01T09:03:46.767306+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635118+0000","last_change":"2023-10-01T09:03:03.198727+0000","last_active":"2023-10-01T09:03:45.635118+0000","last_peered":"2023-10-01T09:03:45.635118+0000","last_clean":"2023-10-01T09:03:45.635118+0000","last_became_active":"2023-10-01T09:03:03.198309+0000","last_became_peered":"2023-10-01T09:03:03.198309+0000","last_unstale":"2023-10-01T09:03:45.635118+0000","last_undegraded":"2023-10-01T09:03:45.635118+0000","last_fullsized":"2023-10-01T09:03:45.635118+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215683+0000","last_change":"2023-10-01T09:03:03.228290+0000","last_active":"2023-10-01T09:03:05.215683+0000","last_peered":"2023-10-01T09:03:05.215683+0000","last_clean":"2023-10-01T09:03:05.215683+0000","last_became_active":"2023-10-01T09:03:03.227612+0000","last_became_peered":"2023-10-01T09:03:03.227612+0000","last_unstale":"2023-10-01T09:03:05.215683+0000","last_undegraded":"2023-10-01T09:03:05.215683+0000","last_fullsized":"2023-10-01T09:03:05.215683+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.631923+0000","last_change":"2023-10-01T09:03:03.199888+0000","last_active":"2023-10-01T09:03:45.631923+0000","last_peered":"2023-10-01T09:03:45.631923+0000","last_clean":"2023-10-01T09:03:45.631923+0000","last_became_active":"2023-10-01T09:03:03.199579+0000","last_became_peered":"2023-10-01T09:03:03.199579+0000","last_unstale":"2023-10-01T09:03:45.631923+0000","last_undegraded":"2023-10-01T09:03:45.631923+0000","last_fullsized":"2023-10-01T09:03:45.631923+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635485+0000","last_change":"2023-10-01T09:03:03.226961+0000","last_active":"2023-10-01T09:03:45.635485+0000","last_peered":"2023-10-01T09:03:45.635485+0000","last_clean":"2023-10-01T09:03:45.635485+0000","last_became_active":"2023-10-01T09:03:03.226660+0000","last_became_peered":"2023-10-01T09:03:03.226660+0000","last_unstale":"2023-10-01T09:03:45.635485+0000","last_undegraded":"2023-10-01T09:03:45.635485+0000","last_fullsized":"2023-10-01T09:03:45.635485+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634325+0000","last_change":"2023-10-01T09:03:03.201578+0000","last_active":"2023-10-01T09:03:45.634325+0000","last_peered":"2023-10-01T09:03:45.634325+0000","last_clean":"2023-10-01T09:03:45.634325+0000","last_became_active":"2023-10-01T09:03:03.201137+0000","last_became_peered":"2023-10-01T09:03:03.201137+0000","last_unstale":"2023-10-01T09:03:45.634325+0000","last_undegraded":"2023-10-01T09:03:45.634325+0000","last_fullsized":"2023-10-01T09:03:45.634325+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430240,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297736,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447416,"statfs":{"total":95995035648,"available":95690153984,"internally_reserved":0,"allocated":655360,"data_stored":271700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.54600000000000004,"15min":0.52200000000000002},"min":{"1min":0.34200000000000003,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.9580000000000002,"5min":2.9580000000000002,"15min":2.9580000000000002},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.503,"5min":0.48999999999999999,"15min":0.48799999999999999},"min":{"1min":0.39600000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.63800000000000001,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.56499999999999995}]},{"osd":1,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.48699999999999999,"15min":0.48299999999999998},"min":{"1min":0.38600000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.63400000000000001,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.498,"15min":0.48399999999999999},"min":{"1min":0.36099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.504}]},{"osd":2,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.71499999999999997,"15min":0.745},"min":{"1min":0.36699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.91000000000000003,"5min":5.5949999999999998,"15min":5.5949999999999998},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.52100000000000002,"15min":0.50700000000000001},"min":{"1min":0.432,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.66000000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.69799999999999995,"15min":0.72399999999999998},"min":{"1min":0.40300000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68700000000000006,"5min":5.6210000000000004,"15min":5.6210000000000004},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.54500000000000004,"15min":0.52300000000000002},"min":{"1min":0.40699999999999997,"5min":0.378,"15min":0.378},"max":{"1min":2.2050000000000001,"5min":2.2050000000000001,"15min":2.2050000000000001},"last":0.60799999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.46500000000000002,"15min":0.45900000000000002},"min":{"1min":0.30299999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.68600000000000005,"15min":0.71399999999999997},"min":{"1min":0.38800000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.72099999999999997,"5min":5.5369999999999999,"15min":5.5369999999999999},"last":0.52400000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.51100000000000001,"15min":0.50700000000000001},"min":{"1min":0.33100000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.70599999999999996,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.53200000000000003,"15min":0.52800000000000002},"min":{"1min":0.35399999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.72799999999999998,"5min":0.746,"15min":0.746},"last":0.626}]},{"osd":6,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.55400000000000005,"15min":0.55000000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.74099999999999999,"15min":0.76700000000000002},"min":{"1min":0.40000000000000002,"5min":0.37,"15min":0.37},"max":{"1min":0.78100000000000003,"5min":5.5700000000000003,"15min":5.5700000000000003},"last":0.58699999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626469,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.54600000000000004,"15min":0.54800000000000004},"min":{"1min":0.36899999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.65200000000000002,"5min":0.752,"15min":0.752},"last":0.623},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.36099999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.64900000000000002,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.46999999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53300000000000003,"15min":0.53500000000000003},"min":{"1min":0.39600000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65300000000000002,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.629,"15min":0.64400000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.70799999999999996,"5min":3.2429999999999999,"15min":3.2429999999999999},"last":0.49299999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.51100000000000001,"15min":0.51300000000000001},"min":{"1min":0.376,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.621,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53400000000000003,"15min":0.53700000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67400000000000004,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.58199999999999996}]},{"osd":3,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54200000000000004,"15min":0.54300000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.41099999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.86699999999999999,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.56299999999999994}]},{"osd":4,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.48999999999999999,"15min":0.48599999999999999},"min":{"1min":0.34599999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.77500000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.504,"15min":0.50700000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.72699999999999998,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52900000000000003,"15min":0.52800000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.76300000000000001,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.80600000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.52000000000000002,"15min":0.51600000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.749},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.56899999999999995,"15min":0.56100000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77400000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.83099999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574907,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297704,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447448,"statfs":{"total":95995035648,"available":95690186752,"internally_reserved":0,"allocated":622592,"data_stored":269633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.36399999999999999,"15min":0.36499999999999999},"min":{"1min":0.23899999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.52000000000000002,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.32500000000000001,"5min":0.40000000000000002,"15min":0.42699999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.46600000000000003,"5min":2.2829999999999999,"15min":2.2829999999999999},"last":0.309}]},{"osd":2,"last update":"Sun Oct 1 09:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.39000000000000001,"15min":0.39600000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.64200000000000002,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.42499999999999999,"15min":0.46800000000000003},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.72099999999999997,"5min":2.3199999999999998,"15min":2.3199999999999998},"last":0.35399999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.40999999999999998,"15min":0.41799999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.46000000000000002,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.45600000000000002,"15min":0.46700000000000003},"min":{"1min":0.33600000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.58399999999999996,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.32000000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:03:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.55600000000000005,"15min":0.55500000000000005},"min":{"1min":0.40200000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.86399999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58799999999999997,"15min":0.58199999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.86899999999999999,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.66300000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57099999999999995,"15min":0.57699999999999996},"min":{"1min":0.378,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.83699999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.84699999999999998,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.64400000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.60999999999999999,"15min":0.621},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.85499999999999998,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.62},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.61599999999999999,"15min":0.627},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76300000000000001,"5min":1.03,"15min":1.03},"last":0.69599999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58099999999999996,"15min":0.58299999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77400000000000002,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.61899999999999999,"15min":0.623},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.82999999999999996,"5min":0.999,"15min":0.999},"last":0.52600000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738432,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34499999999999997,"15min":0.34300000000000003},"min":{"1min":0.245,"5min":0.22,"15min":0.22},"max":{"1min":0.46400000000000002,"5min":0.52900000000000003,"15min":0.52900000000000003},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.245,"5min":0.221,"15min":0.221},"max":{"1min":0.53600000000000003,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.36699999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:03:46 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.39600000000000002,"15min":0.40600000000000003},"min":{"1min":0.26900000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.56299999999999994,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.34000000000000002,"5min":0.376,"15min":0.38100000000000001},"min":{"1min":0.253,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.47899999999999998,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.31}]},{"osd":3,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.41799999999999998,"15min":0.42999999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.51600000000000001,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.45700000000000002,"15min":0.47299999999999998},"min":{"1min":0.28399999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56999999999999995,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.42799999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.42599999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.80000000000000004,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.41299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.77100000000000002,"5min":0.87,"15min":0.87},"last":0.92400000000000004}]},{"osd":5,"last update":"Sun Oct 1 09:03:52 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.56699999999999995,"15min":0.57499999999999996},"min":{"1min":0.39100000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.57699999999999996,"15min":0.59199999999999997},"min":{"1min":0.39700000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.68400000000000005,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.88500000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.62,"15min":0.624},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.89200000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.39900000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.749,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.90200000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.46000000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.71999999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411382,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":0.36099999999999999,"15min":0.371},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.44,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.23100000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.52500000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.23100000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.375,"15min":0.373},"min":{"1min":0.246,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.56499999999999995,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.251},{"interface":"front","average":{"1min":0.37,"5min":0.38900000000000001,"15min":0.39200000000000002},"min":{"1min":0.26900000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.30399999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.435,"15min":0.44600000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.499,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.374,"5min":0.45400000000000001,"15min":0.47499999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.56999999999999995,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.34999999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:03:27 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54000000000000004,"15min":0.54100000000000004},"min":{"1min":0.434,"5min":0.374,"15min":0.374},"max":{"1min":0.65800000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.68200000000000005,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.54600000000000004}]},{"osd":5,"last update":"Sun Oct 1 09:02:52 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56499999999999995,"15min":0.56799999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55900000000000005,"15min":0.56200000000000006},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.73399999999999999,"5min":0.749,"15min":0.749},"last":0.63600000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.59799999999999998,"15min":0.60499999999999998},"min":{"1min":0.44400000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.69899999999999995,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.60599999999999998,"15min":0.61199999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.73699999999999999,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.65100000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.58599999999999997,"15min":0.58899999999999997},"min":{"1min":0.47899999999999998,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.71599999999999997,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.60499999999999998,"15min":0.61099999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.70199999999999996,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.68000000000000005}]}]},{"osd":3,"up_from":25,"seq":107374182450,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.51,"5min":1.51,"15min":1.51},"last":0.248},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.8599999999999999,"5min":2.8599999999999999,"15min":2.8599999999999999},"last":0.34999999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.42999999999999999,"15min":0.44800000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.503,"5min":0.62,"15min":0.62},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.37,"5min":0.40999999999999998,"15min":0.41899999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.52700000000000002,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.33800000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.44600000000000001,"15min":0.45800000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.54000000000000004,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.38,"5min":0.437,"15min":0.45600000000000002},"min":{"1min":0.27200000000000002,"5min":0.25,"15min":0.25},"max":{"1min":0.59199999999999997,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.29799999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:03:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59499999999999997,"15min":0.59699999999999998},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.371,"5min":1.371,"15min":1.371},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.59699999999999998,"15min":0.60499999999999998},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.76700000000000002,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.60099999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:03:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.57599999999999996,"15min":0.56699999999999995},"min":{"1min":0.433,"5min":0.377,"15min":0.377},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.60099999999999998,"15min":0.59499999999999997},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.84899999999999998,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.622}]},{"osd":6,"last update":"Sun Oct 1 09:03:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.59899999999999998,"15min":0.60699999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.84199999999999997,"5min":1.389,"15min":1.389},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.45300000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.80000000000000004,"5min":1.278,"15min":1.278},"last":0.54800000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.57299999999999995,"15min":0.56799999999999995},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60399999999999998,"15min":0.60199999999999998},"min":{"1min":0.44,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.82499999999999996,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.59199999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018925,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.55900000000000005,"15min":0.52500000000000002},"min":{"1min":0.34999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.4459999999999997,"5min":5.4459999999999997,"15min":5.4459999999999997},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.57799999999999996,"15min":0.52600000000000002},"min":{"1min":0.35999999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.1890000000000001,"5min":7.1890000000000001,"15min":7.1890000000000001},"last":0.58299999999999996}]},{"osd":1,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51200000000000001,"15min":0.51100000000000001},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":0.497},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.53100000000000003,"15min":0.495},"min":{"1min":0.372,"5min":0.315,"15min":0.315},"max":{"1min":4.2279999999999998,"5min":4.2279999999999998,"15min":4.2279999999999998},"last":0.46800000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.55800000000000005,"15min":0.52100000000000002},"min":{"1min":0.39300000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.3310000000000004,"5min":5.3310000000000004,"15min":5.3310000000000004},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.52000000000000002,"15min":0.51200000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.41499999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.55600000000000005,"15min":0.51700000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.6429999999999998,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.52300000000000002,"15min":0.51800000000000002},"min":{"1min":0.38600000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.1279999999999999,"5min":1.1279999999999999,"15min":1.1279999999999999},"last":0.54700000000000004}]},{"osd":5,"last update":"Sun Oct 1 09:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.41899999999999998,"15min":0.39600000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.41599999999999998,"15min":0.38},"min":{"1min":0.31,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.66000000000000003,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.44800000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:03:14 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.45500000000000002,"15min":0.44500000000000001},"min":{"1min":0.34599999999999997,"5min":0.313,"15min":0.313},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.40200000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.53500000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:03:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.53700000000000003,"15min":0.54100000000000004},"min":{"1min":0.33000000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.55400000000000005,"15min":0.55500000000000005},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.51400000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822698,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.499,"15min":0.498},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.747,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.504,"5min":0.51100000000000001,"15min":0.51500000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.59999999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.49299999999999999,"15min":0.495},"min":{"1min":0.34599999999999997,"5min":0.309,"15min":0.309},"max":{"1min":0.68300000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.54500000000000004,"15min":0.53300000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.65900000000000003,"5min":4.0490000000000004,"15min":4.0490000000000004},"last":0.52100000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.53600000000000003,"15min":0.54400000000000004},"min":{"1min":0.39500000000000002,"5min":0.308,"15min":0.308},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.51300000000000001,"15min":0.50600000000000001},"min":{"1min":0.33900000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.628}]},{"osd":3,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.53500000000000003,"15min":0.52700000000000002},"min":{"1min":0.36199999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.38800000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.68700000000000006}]},{"osd":4,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.42199999999999999,"15min":0.40000000000000002},"min":{"1min":0.28599999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.67500000000000004,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.46100000000000002,"15min":0.42299999999999999},"min":{"1min":0.34699999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.46100000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:03:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.48699999999999999,"15min":0.47899999999999998},"min":{"1min":0.34100000000000003,"5min":0.313,"15min":0.313},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.47699999999999998,"15min":0.46500000000000002},"min":{"1min":0.35999999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.56200000000000006}]},{"osd":7,"last update":"Sun Oct 1 09:03:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54000000000000004,"15min":0.53500000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.53400000000000003,"15min":0.52700000000000002},"min":{"1min":0.36499999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.67300000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:03:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:58 smithi190 conmon[100518]: cluster 2023-10-01T09:03:56.854727+0000 mgr.y (mgr.14433) 102 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:58 smithi190 conmon[100518]: audit 2023-10-01T09:03: 2023-10-01T09:03:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:03:58 smithi190 conmon[100518]: 57.246489+0000 mgr.y (mgr.14433) 103 : audit [DBG] from='client.24701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:03:58.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:58 smithi138 conmon[109959]: cluster 2023-10-01T09:03:56.854727+0000 mgr.y (mgr.14433) 2023-10-01T09:03:58.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:58 smithi138 conmon[109959]: 102 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:58.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:58 smithi138 conmon[109959]: audit 2023-10-01T09:03:57.246489+0000 mgr.y (mgr.14433) 103 : audit [DBG] 2023-10-01T09:03:58.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:03:58 smithi138 conmon[109959]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:03:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:58 smithi138 conmon[100289]: cluster 2023-10-01T09:03:56.854727+0000 mgr.y (mgr 2023-10-01T09:03:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:58 smithi138 conmon[100289]: .14433) 102 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:03:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:03:58 smithi138 conmon[100289]: audit 2023-10-01T09:03:57.246489+0000 mgr.y (mgr.14433) 103 : audit [DBG] from='client.24701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:04:00.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:00 smithi190 conmon[100518]: cluster 2023-10-01T09:03:58.855118+0000 mgr.y (mgr.14433) 104 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:00.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:00 smithi138 conmon[109959]: cluster 2023-10-01T09:03:58.855118+0000 mgr.y (mgr.14433) 2023-10-01T09:04:00.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:00 smithi138 conmon[109959]: 104 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:00.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:00 smithi138 conmon[100289]: cluster 2023-10-01T09:03:58.855118+0000 mgr.y (mgr.14433) 104 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:00.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:00 smithi138 conmon[100289]: 2023-10-01T09:04:00.936 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:04:01.295 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:04:02.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:02 smithi138 conmon[100289]: cluster 2023-10-01T09:04:00.855627+0000 mgr.y (mgr.14433) 105 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:02.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:02 smithi138 conmon[109959]: cluster 2023-10-01T09:04:00.855627+0000 mgr.y (mgr.14433) 105 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:02.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:02 smithi190 conmon[100518]: cluster 2023-10-01T09:04:00.855627+0000 mgr.y (mgr.14433) 105 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:02 smithi190 conmon[100518]: 2023-10-01T09:04:03.142 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:04:03.144 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:04:03.786 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":83,"stamp":"2023-10-01T09:04:02.855804+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381744,"kb_used_data":4976,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579472,"statfs":{"total":767960285184,"available":765521379328,"internally_reserved":0,"allocated":5095424,"data_stored":2165242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002624"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.633938+0000","last_change":"2023-10-01T09:03:03.223370+0000","last_active":"2023-10-01T09:03:45.633938+0000","last_peered":"2023-10-01T09:03:45.633938+0000","last_clean":"2023-10-01T09:03:45.633938+0000","last_became_active":"2023-10-01T09:03:03.223071+0000","last_became_peered":"2023-10-01T09:03:03.223071+0000","last_unstale":"2023-10-01T09:03:45.633938+0000","last_undegraded":"2023-10-01T09:03:45.633938+0000","last_fullsized":"2023-10-01T09:03:45.633938+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.232672+0000","last_change":"2023-10-01T09:03:03.202335+0000","last_active":"2023-10-01T09:03:08.232672+0000","last_peered":"2023-10-01T09:03:08.232672+0000","last_clean":"2023-10-01T09:03:08.232672+0000","last_became_active":"2023-10-01T09:03:03.201485+0000","last_became_peered":"2023-10-01T09:03:03.201485+0000","last_unstale":"2023-10-01T09:03:08.232672+0000","last_undegraded":"2023-10-01T09:03:08.232672+0000","last_fullsized":"2023-10-01T09:03:08.232672+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635131+0000","last_change":"2023-10-01T09:03:03.224058+0000","last_active":"2023-10-01T09:03:45.635131+0000","last_peered":"2023-10-01T09:03:45.635131+0000","last_clean":"2023-10-01T09:03:45.635131+0000","last_became_active":"2023-10-01T09:03:03.223598+0000","last_became_peered":"2023-10-01T09:03:03.223598+0000","last_unstale":"2023-10-01T09:03:45.635131+0000","last_undegraded":"2023-10-01T09:03:45.635131+0000","last_fullsized":"2023-10-01T09:03:45.635131+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635683+0000","last_change":"2023-10-01T09:03:03.193542+0000","last_active":"2023-10-01T09:03:45.635683+0000","last_peered":"2023-10-01T09:03:45.635683+0000","last_clean":"2023-10-01T09:03:45.635683+0000","last_became_active":"2023-10-01T09:03:03.193245+0000","last_became_peered":"2023-10-01T09:03:03.193245+0000","last_unstale":"2023-10-01T09:03:45.635683+0000","last_undegraded":"2023-10-01T09:03:45.635683+0000","last_fullsized":"2023-10-01T09:03:45.635683+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634507+0000","last_change":"2023-10-01T09:03:03.218020+0000","last_active":"2023-10-01T09:03:45.634507+0000","last_peered":"2023-10-01T09:03:45.634507+0000","last_clean":"2023-10-01T09:03:45.634507+0000","last_became_active":"2023-10-01T09:03:03.217717+0000","last_became_peered":"2023-10-01T09:03:03.217717+0000","last_unstale":"2023-10-01T09:03:45.634507+0000","last_undegraded":"2023-10-01T09:03:45.634507+0000","last_fullsized":"2023-10-01T09:03:45.634507+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.633112+0000","last_change":"2023-10-01T09:03:03.202063+0000","last_active":"2023-10-01T09:03:45.633112+0000","last_peered":"2023-10-01T09:03:45.633112+0000","last_clean":"2023-10-01T09:03:45.633112+0000","last_became_active":"2023-10-01T09:03:03.201769+0000","last_became_peered":"2023-10-01T09:03:03.201769+0000","last_unstale":"2023-10-01T09:03:45.633112+0000","last_undegraded":"2023-10-01T09:03:45.633112+0000","last_fullsized":"2023-10-01T09:03:45.633112+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635109+0000","last_change":"2023-10-01T09:03:03.221313+0000","last_active":"2023-10-01T09:03:45.635109+0000","last_peered":"2023-10-01T09:03:45.635109+0000","last_clean":"2023-10-01T09:03:45.635109+0000","last_became_active":"2023-10-01T09:03:03.221080+0000","last_became_peered":"2023-10-01T09:03:03.221080+0000","last_unstale":"2023-10-01T09:03:45.635109+0000","last_undegraded":"2023-10-01T09:03:45.635109+0000","last_fullsized":"2023-10-01T09:03:45.635109+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.633222+0000","last_change":"2023-10-01T09:03:03.201659+0000","last_active":"2023-10-01T09:03:45.633222+0000","last_peered":"2023-10-01T09:03:45.633222+0000","last_clean":"2023-10-01T09:03:45.633222+0000","last_became_active":"2023-10-01T09:03:03.201348+0000","last_became_peered":"2023-10-01T09:03:03.201348+0000","last_unstale":"2023-10-01T09:03:45.633222+0000","last_undegraded":"2023-10-01T09:03:45.633222+0000","last_fullsized":"2023-10-01T09:03:45.633222+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.631910+0000","last_change":"2023-10-01T09:03:03.226772+0000","last_active":"2023-10-01T09:03:45.631910+0000","last_peered":"2023-10-01T09:03:45.631910+0000","last_clean":"2023-10-01T09:03:45.631910+0000","last_became_active":"2023-10-01T09:03:03.226446+0000","last_became_peered":"2023-10-01T09:03:03.226446+0000","last_unstale":"2023-10-01T09:03:45.631910+0000","last_undegraded":"2023-10-01T09:03:45.631910+0000","last_fullsized":"2023-10-01T09:03:45.631910+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:08.231820+0000","last_change":"2023-10-01T09:03:03.203002+0000","last_active":"2023-10-01T09:03:08.231820+0000","last_peered":"2023-10-01T09:03:08.231820+0000","last_clean":"2023-10-01T09:03:08.231820+0000","last_became_active":"2023-10-01T09:03:03.202571+0000","last_became_peered":"2023-10-01T09:03:03.202571+0000","last_unstale":"2023-10-01T09:03:08.231820+0000","last_undegraded":"2023-10-01T09:03:08.231820+0000","last_fullsized":"2023-10-01T09:03:08.231820+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634224+0000","last_change":"2023-10-01T09:03:03.754839+0000","last_active":"2023-10-01T09:03:45.634224+0000","last_peered":"2023-10-01T09:03:45.634224+0000","last_clean":"2023-10-01T09:03:45.634224+0000","last_became_active":"2023-10-01T09:03:03.754506+0000","last_became_peered":"2023-10-01T09:03:03.754506+0000","last_unstale":"2023-10-01T09:03:45.634224+0000","last_undegraded":"2023-10-01T09:03:45.634224+0000","last_fullsized":"2023-10-01T09:03:45.634224+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:12.687636+0000","last_change":"2023-10-01T09:03:03.196477+0000","last_active":"2023-10-01T09:03:12.687636+0000","last_peered":"2023-10-01T09:03:12.687636+0000","last_clean":"2023-10-01T09:03:12.687636+0000","last_became_active":"2023-10-01T09:03:03.196266+0000","last_became_peered":"2023-10-01T09:03:03.196266+0000","last_unstale":"2023-10-01T09:03:12.687636+0000","last_undegraded":"2023-10-01T09:03:12.687636+0000","last_fullsized":"2023-10-01T09:03:12.687636+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.636147+0000","last_change":"2023-10-01T09:02:06.997887+0000","last_active":"2023-10-01T09:03:45.636147+0000","last_peered":"2023-10-01T09:03:45.636147+0000","last_clean":"2023-10-01T09:03:45.636147+0000","last_became_active":"2023-10-01T09:02:06.997597+0000","last_became_peered":"2023-10-01T09:02:06.997597+0000","last_unstale":"2023-10-01T09:03:45.636147+0000","last_undegraded":"2023-10-01T09:03:45.636147+0000","last_fullsized":"2023-10-01T09:03:45.636147+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635006+0000","last_change":"2023-10-01T09:03:03.228390+0000","last_active":"2023-10-01T09:03:45.635006+0000","last_peered":"2023-10-01T09:03:45.635006+0000","last_clean":"2023-10-01T09:03:45.635006+0000","last_became_active":"2023-10-01T09:03:03.227254+0000","last_became_peered":"2023-10-01T09:03:03.227254+0000","last_unstale":"2023-10-01T09:03:45.635006+0000","last_undegraded":"2023-10-01T09:03:45.635006+0000","last_fullsized":"2023-10-01T09:03:45.635006+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632276+0000","last_change":"2023-10-01T09:02:07.442083+0000","last_active":"2023-10-01T09:03:45.632276+0000","last_peered":"2023-10-01T09:03:45.632276+0000","last_clean":"2023-10-01T09:03:45.632276+0000","last_became_active":"2023-10-01T09:02:07.441757+0000","last_became_peered":"2023-10-01T09:02:07.441757+0000","last_unstale":"2023-10-01T09:03:45.632276+0000","last_undegraded":"2023-10-01T09:03:45.632276+0000","last_fullsized":"2023-10-01T09:03:45.632276+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635624+0000","last_change":"2023-10-01T09:03:03.198510+0000","last_active":"2023-10-01T09:03:45.635624+0000","last_peered":"2023-10-01T09:03:45.635624+0000","last_clean":"2023-10-01T09:03:45.635624+0000","last_became_active":"2023-10-01T09:03:03.198172+0000","last_became_peered":"2023-10-01T09:03:03.198172+0000","last_unstale":"2023-10-01T09:03:45.635624+0000","last_undegraded":"2023-10-01T09:03:45.635624+0000","last_fullsized":"2023-10-01T09:03:45.635624+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215706+0000","last_change":"2023-10-01T09:03:03.754828+0000","last_active":"2023-10-01T09:03:05.215706+0000","last_peered":"2023-10-01T09:03:05.215706+0000","last_clean":"2023-10-01T09:03:05.215706+0000","last_became_active":"2023-10-01T09:03:03.754492+0000","last_became_peered":"2023-10-01T09:03:03.754492+0000","last_unstale":"2023-10-01T09:03:05.215706+0000","last_undegraded":"2023-10-01T09:03:05.215706+0000","last_fullsized":"2023-10-01T09:03:05.215706+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632764+0000","last_change":"2023-10-01T09:03:03.203345+0000","last_active":"2023-10-01T09:03:45.632764+0000","last_peered":"2023-10-01T09:03:45.632764+0000","last_clean":"2023-10-01T09:03:45.632764+0000","last_became_active":"2023-10-01T09:03:03.203035+0000","last_became_peered":"2023-10-01T09:03:03.203035+0000","last_unstale":"2023-10-01T09:03:45.632764+0000","last_undegraded":"2023-10-01T09:03:45.632764+0000","last_fullsized":"2023-10-01T09:03:45.632764+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932484+0000","last_change":"2023-10-01T09:02:07.442452+0000","last_active":"2023-10-01T09:03:45.932484+0000","last_peered":"2023-10-01T09:03:45.932484+0000","last_clean":"2023-10-01T09:03:45.932484+0000","last_became_active":"2023-10-01T09:02:07.442111+0000","last_became_peered":"2023-10-01T09:02:07.442111+0000","last_unstale":"2023-10-01T09:03:45.932484+0000","last_undegraded":"2023-10-01T09:03:45.932484+0000","last_fullsized":"2023-10-01T09:03:45.932484+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215738+0000","last_change":"2023-10-01T09:03:03.223326+0000","last_active":"2023-10-01T09:03:05.215738+0000","last_peered":"2023-10-01T09:03:05.215738+0000","last_clean":"2023-10-01T09:03:05.215738+0000","last_became_active":"2023-10-01T09:03:03.222974+0000","last_became_peered":"2023-10-01T09:03:03.222974+0000","last_unstale":"2023-10-01T09:03:05.215738+0000","last_undegraded":"2023-10-01T09:03:05.215738+0000","last_fullsized":"2023-10-01T09:03:05.215738+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215539+0000","last_change":"2023-10-01T09:03:03.195712+0000","last_active":"2023-10-01T09:03:05.215539+0000","last_peered":"2023-10-01T09:03:05.215539+0000","last_clean":"2023-10-01T09:03:05.215539+0000","last_became_active":"2023-10-01T09:03:03.195408+0000","last_became_peered":"2023-10-01T09:03:03.195408+0000","last_unstale":"2023-10-01T09:03:05.215539+0000","last_undegraded":"2023-10-01T09:03:05.215539+0000","last_fullsized":"2023-10-01T09:03:05.215539+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932093+0000","last_change":"2023-10-01T09:02:06.999043+0000","last_active":"2023-10-01T09:03:45.932093+0000","last_peered":"2023-10-01T09:03:45.932093+0000","last_clean":"2023-10-01T09:03:45.932093+0000","last_became_active":"2023-10-01T09:02:06.998671+0000","last_became_peered":"2023-10-01T09:02:06.998671+0000","last_unstale":"2023-10-01T09:03:45.932093+0000","last_undegraded":"2023-10-01T09:03:45.932093+0000","last_fullsized":"2023-10-01T09:03:45.932093+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635816+0000","last_change":"2023-10-01T09:03:03.227254+0000","last_active":"2023-10-01T09:03:45.635816+0000","last_peered":"2023-10-01T09:03:45.635816+0000","last_clean":"2023-10-01T09:03:45.635816+0000","last_became_active":"2023-10-01T09:03:03.226941+0000","last_became_peered":"2023-10-01T09:03:03.226941+0000","last_unstale":"2023-10-01T09:03:45.635816+0000","last_undegraded":"2023-10-01T09:03:45.635816+0000","last_fullsized":"2023-10-01T09:03:45.635816+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635127+0000","last_change":"2023-10-01T09:03:03.198835+0000","last_active":"2023-10-01T09:03:45.635127+0000","last_peered":"2023-10-01T09:03:45.635127+0000","last_clean":"2023-10-01T09:03:45.635127+0000","last_became_active":"2023-10-01T09:03:03.198424+0000","last_became_peered":"2023-10-01T09:03:03.198424+0000","last_unstale":"2023-10-01T09:03:45.635127+0000","last_undegraded":"2023-10-01T09:03:45.635127+0000","last_fullsized":"2023-10-01T09:03:45.635127+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932052+0000","last_change":"2023-10-01T09:01:32.510259+0000","last_active":"2023-10-01T09:03:45.932052+0000","last_peered":"2023-10-01T09:03:45.932052+0000","last_clean":"2023-10-01T09:03:45.932052+0000","last_became_active":"2023-10-01T09:01:32.509905+0000","last_became_peered":"2023-10-01T09:01:32.509905+0000","last_unstale":"2023-10-01T09:03:45.932052+0000","last_undegraded":"2023-10-01T09:03:45.932052+0000","last_fullsized":"2023-10-01T09:03:45.932052+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632783+0000","last_change":"2023-10-01T09:02:06.998804+0000","last_active":"2023-10-01T09:03:45.632783+0000","last_peered":"2023-10-01T09:03:45.632783+0000","last_clean":"2023-10-01T09:03:45.632783+0000","last_became_active":"2023-10-01T09:02:06.998516+0000","last_became_peered":"2023-10-01T09:02:06.998516+0000","last_unstale":"2023-10-01T09:03:45.632783+0000","last_undegraded":"2023-10-01T09:03:45.632783+0000","last_fullsized":"2023-10-01T09:03:45.632783+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632512+0000","last_change":"2023-10-01T09:03:03.756021+0000","last_active":"2023-10-01T09:03:45.632512+0000","last_peered":"2023-10-01T09:03:45.632512+0000","last_clean":"2023-10-01T09:03:45.632512+0000","last_became_active":"2023-10-01T09:03:03.755659+0000","last_became_peered":"2023-10-01T09:03:03.755659+0000","last_unstale":"2023-10-01T09:03:45.632512+0000","last_undegraded":"2023-10-01T09:03:45.632512+0000","last_fullsized":"2023-10-01T09:03:45.632512+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632396+0000","last_change":"2023-10-01T09:03:03.201620+0000","last_active":"2023-10-01T09:03:45.632396+0000","last_peered":"2023-10-01T09:03:45.632396+0000","last_clean":"2023-10-01T09:03:45.632396+0000","last_became_active":"2023-10-01T09:03:03.201312+0000","last_became_peered":"2023-10-01T09:03:03.201312+0000","last_unstale":"2023-10-01T09:03:45.632396+0000","last_undegraded":"2023-10-01T09:03:45.632396+0000","last_fullsized":"2023-10-01T09:03:45.632396+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.631546+0000","last_change":"2023-10-01T09:02:07.436492+0000","last_active":"2023-10-01T09:03:45.631546+0000","last_peered":"2023-10-01T09:03:45.631546+0000","last_clean":"2023-10-01T09:03:45.631546+0000","last_became_active":"2023-10-01T09:02:07.436146+0000","last_became_peered":"2023-10-01T09:02:07.436146+0000","last_unstale":"2023-10-01T09:03:45.631546+0000","last_undegraded":"2023-10-01T09:03:45.631546+0000","last_fullsized":"2023-10-01T09:03:45.631546+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632674+0000","last_change":"2023-10-01T09:03:03.227284+0000","last_active":"2023-10-01T09:03:45.632674+0000","last_peered":"2023-10-01T09:03:45.632674+0000","last_clean":"2023-10-01T09:03:45.632674+0000","last_became_active":"2023-10-01T09:03:03.226987+0000","last_became_peered":"2023-10-01T09:03:03.226987+0000","last_unstale":"2023-10-01T09:03:45.632674+0000","last_undegraded":"2023-10-01T09:03:45.632674+0000","last_fullsized":"2023-10-01T09:03:45.632674+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634293+0000","last_change":"2023-10-01T09:03:03.196357+0000","last_active":"2023-10-01T09:03:45.634293+0000","last_peered":"2023-10-01T09:03:45.634293+0000","last_clean":"2023-10-01T09:03:45.634293+0000","last_became_active":"2023-10-01T09:03:03.196017+0000","last_became_peered":"2023-10-01T09:03:03.196017+0000","last_unstale":"2023-10-01T09:03:45.634293+0000","last_undegraded":"2023-10-01T09:03:45.634293+0000","last_fullsized":"2023-10-01T09:03:45.634293+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632885+0000","last_change":"2023-10-01T09:02:07.001644+0000","last_active":"2023-10-01T09:03:45.632885+0000","last_peered":"2023-10-01T09:03:45.632885+0000","last_clean":"2023-10-01T09:03:45.632885+0000","last_became_active":"2023-10-01T09:02:07.001197+0000","last_became_peered":"2023-10-01T09:02:07.001197+0000","last_unstale":"2023-10-01T09:03:45.632885+0000","last_undegraded":"2023-10-01T09:03:45.632885+0000","last_fullsized":"2023-10-01T09:03:45.632885+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632588+0000","last_change":"2023-10-01T09:03:03.755027+0000","last_active":"2023-10-01T09:03:45.632588+0000","last_peered":"2023-10-01T09:03:45.632588+0000","last_clean":"2023-10-01T09:03:45.632588+0000","last_became_active":"2023-10-01T09:03:03.754596+0000","last_became_peered":"2023-10-01T09:03:03.754596+0000","last_unstale":"2023-10-01T09:03:45.632588+0000","last_undegraded":"2023-10-01T09:03:45.632588+0000","last_fullsized":"2023-10-01T09:03:45.632588+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.633017+0000","last_change":"2023-10-01T09:03:03.203012+0000","last_active":"2023-10-01T09:03:45.633017+0000","last_peered":"2023-10-01T09:03:45.633017+0000","last_clean":"2023-10-01T09:03:45.633017+0000","last_became_active":"2023-10-01T09:03:03.202051+0000","last_became_peered":"2023-10-01T09:03:03.202051+0000","last_unstale":"2023-10-01T09:03:45.633017+0000","last_undegraded":"2023-10-01T09:03:45.633017+0000","last_fullsized":"2023-10-01T09:03:45.633017+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632616+0000","last_change":"2023-10-01T09:02:06.998188+0000","last_active":"2023-10-01T09:03:45.632616+0000","last_peered":"2023-10-01T09:03:45.632616+0000","last_clean":"2023-10-01T09:03:45.632616+0000","last_became_active":"2023-10-01T09:02:06.997855+0000","last_became_peered":"2023-10-01T09:02:06.997855+0000","last_unstale":"2023-10-01T09:03:45.632616+0000","last_undegraded":"2023-10-01T09:03:45.632616+0000","last_fullsized":"2023-10-01T09:03:45.632616+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T09:03:46.638938+0000","last_change":"2023-10-01T09:03:46.638938+0000","last_active":"2023-10-01T09:03:46.638938+0000","last_peered":"2023-10-01T09:03:46.638938+0000","last_clean":"2023-10-01T09:03:46.638938+0000","last_became_active":"2023-10-01T09:03:46.638713+0000","last_became_peered":"2023-10-01T09:03:46.638713+0000","last_unstale":"2023-10-01T09:03:46.638938+0000","last_undegraded":"2023-10-01T09:03:46.638938+0000","last_fullsized":"2023-10-01T09:03:46.638938+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635017+0000","last_change":"2023-10-01T09:03:03.200838+0000","last_active":"2023-10-01T09:03:45.635017+0000","last_peered":"2023-10-01T09:03:45.635017+0000","last_clean":"2023-10-01T09:03:45.635017+0000","last_became_active":"2023-10-01T09:03:03.199715+0000","last_became_peered":"2023-10-01T09:03:03.199715+0000","last_unstale":"2023-10-01T09:03:45.635017+0000","last_undegraded":"2023-10-01T09:03:45.635017+0000","last_fullsized":"2023-10-01T09:03:45.635017+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634588+0000","last_change":"2023-10-01T09:03:03.221266+0000","last_active":"2023-10-01T09:03:45.634588+0000","last_peered":"2023-10-01T09:03:45.634588+0000","last_clean":"2023-10-01T09:03:45.634588+0000","last_became_active":"2023-10-01T09:03:03.220690+0000","last_became_peered":"2023-10-01T09:03:03.220690+0000","last_unstale":"2023-10-01T09:03:45.634588+0000","last_undegraded":"2023-10-01T09:03:45.634588+0000","last_fullsized":"2023-10-01T09:03:45.634588+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634373+0000","last_change":"2023-10-01T09:03:03.199237+0000","last_active":"2023-10-01T09:03:45.634373+0000","last_peered":"2023-10-01T09:03:45.634373+0000","last_clean":"2023-10-01T09:03:45.634373+0000","last_became_active":"2023-10-01T09:03:03.198950+0000","last_became_peered":"2023-10-01T09:03:03.198950+0000","last_unstale":"2023-10-01T09:03:45.634373+0000","last_undegraded":"2023-10-01T09:03:45.634373+0000","last_fullsized":"2023-10-01T09:03:45.634373+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634678+0000","last_change":"2023-10-01T09:03:03.223290+0000","last_active":"2023-10-01T09:03:45.634678+0000","last_peered":"2023-10-01T09:03:45.634678+0000","last_clean":"2023-10-01T09:03:45.634678+0000","last_became_active":"2023-10-01T09:03:03.223013+0000","last_became_peered":"2023-10-01T09:03:03.223013+0000","last_unstale":"2023-10-01T09:03:45.634678+0000","last_undegraded":"2023-10-01T09:03:45.634678+0000","last_fullsized":"2023-10-01T09:03:45.634678+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635379+0000","last_change":"2023-10-01T09:03:03.198744+0000","last_active":"2023-10-01T09:03:45.635379+0000","last_peered":"2023-10-01T09:03:45.635379+0000","last_clean":"2023-10-01T09:03:45.635379+0000","last_became_active":"2023-10-01T09:03:03.198452+0000","last_became_peered":"2023-10-01T09:03:03.198452+0000","last_unstale":"2023-10-01T09:03:45.635379+0000","last_undegraded":"2023-10-01T09:03:45.635379+0000","last_fullsized":"2023-10-01T09:03:45.635379+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634570+0000","last_change":"2023-10-01T09:03:03.218071+0000","last_active":"2023-10-01T09:03:45.634570+0000","last_peered":"2023-10-01T09:03:45.634570+0000","last_clean":"2023-10-01T09:03:45.634570+0000","last_became_active":"2023-10-01T09:03:03.217776+0000","last_became_peered":"2023-10-01T09:03:03.217776+0000","last_unstale":"2023-10-01T09:03:45.634570+0000","last_undegraded":"2023-10-01T09:03:45.634570+0000","last_fullsized":"2023-10-01T09:03:45.634570+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635229+0000","last_change":"2023-10-01T09:03:03.200515+0000","last_active":"2023-10-01T09:03:45.635229+0000","last_peered":"2023-10-01T09:03:45.635229+0000","last_clean":"2023-10-01T09:03:45.635229+0000","last_became_active":"2023-10-01T09:03:03.199536+0000","last_became_peered":"2023-10-01T09:03:03.199536+0000","last_unstale":"2023-10-01T09:03:45.635229+0000","last_undegraded":"2023-10-01T09:03:45.635229+0000","last_fullsized":"2023-10-01T09:03:45.635229+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215903+0000","last_change":"2023-10-01T09:03:03.224884+0000","last_active":"2023-10-01T09:03:05.215903+0000","last_peered":"2023-10-01T09:03:05.215903+0000","last_clean":"2023-10-01T09:03:05.215903+0000","last_became_active":"2023-10-01T09:03:03.224529+0000","last_became_peered":"2023-10-01T09:03:03.224529+0000","last_unstale":"2023-10-01T09:03:05.215903+0000","last_undegraded":"2023-10-01T09:03:05.215903+0000","last_fullsized":"2023-10-01T09:03:05.215903+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632780+0000","last_change":"2023-10-01T09:03:03.203721+0000","last_active":"2023-10-01T09:03:45.632780+0000","last_peered":"2023-10-01T09:03:45.632780+0000","last_clean":"2023-10-01T09:03:45.632780+0000","last_became_active":"2023-10-01T09:03:03.203308+0000","last_became_peered":"2023-10-01T09:03:03.203308+0000","last_unstale":"2023-10-01T09:03:45.632780+0000","last_undegraded":"2023-10-01T09:03:45.632780+0000","last_fullsized":"2023-10-01T09:03:45.632780+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635266+0000","last_change":"2023-10-01T09:03:03.227070+0000","last_active":"2023-10-01T09:03:45.635266+0000","last_peered":"2023-10-01T09:03:45.635266+0000","last_clean":"2023-10-01T09:03:45.635266+0000","last_became_active":"2023-10-01T09:03:03.226752+0000","last_became_peered":"2023-10-01T09:03:03.226752+0000","last_unstale":"2023-10-01T09:03:45.635266+0000","last_undegraded":"2023-10-01T09:03:45.635266+0000","last_fullsized":"2023-10-01T09:03:45.635266+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932534+0000","last_change":"2023-10-01T09:03:03.200972+0000","last_active":"2023-10-01T09:03:45.932534+0000","last_peered":"2023-10-01T09:03:45.932534+0000","last_clean":"2023-10-01T09:03:45.932534+0000","last_became_active":"2023-10-01T09:03:03.200695+0000","last_became_peered":"2023-10-01T09:03:03.200695+0000","last_unstale":"2023-10-01T09:03:45.932534+0000","last_undegraded":"2023-10-01T09:03:45.932534+0000","last_fullsized":"2023-10-01T09:03:45.932534+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634095+0000","last_change":"2023-10-01T09:03:03.220741+0000","last_active":"2023-10-01T09:03:45.634095+0000","last_peered":"2023-10-01T09:03:45.634095+0000","last_clean":"2023-10-01T09:03:45.634095+0000","last_became_active":"2023-10-01T09:03:03.220481+0000","last_became_peered":"2023-10-01T09:03:03.220481+0000","last_unstale":"2023-10-01T09:03:45.634095+0000","last_undegraded":"2023-10-01T09:03:45.634095+0000","last_fullsized":"2023-10-01T09:03:45.634095+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932558+0000","last_change":"2023-10-01T09:03:03.200367+0000","last_active":"2023-10-01T09:03:45.932558+0000","last_peered":"2023-10-01T09:03:45.932558+0000","last_clean":"2023-10-01T09:03:45.932558+0000","last_became_active":"2023-10-01T09:03:03.200068+0000","last_became_peered":"2023-10-01T09:03:03.200068+0000","last_unstale":"2023-10-01T09:03:45.932558+0000","last_undegraded":"2023-10-01T09:03:45.932558+0000","last_fullsized":"2023-10-01T09:03:45.932558+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632631+0000","last_change":"2023-10-01T09:03:04.245105+0000","last_active":"2023-10-01T09:03:45.632631+0000","last_peered":"2023-10-01T09:03:45.632631+0000","last_clean":"2023-10-01T09:03:45.632631+0000","last_became_active":"2023-10-01T09:03:04.244807+0000","last_became_peered":"2023-10-01T09:03:04.244807+0000","last_unstale":"2023-10-01T09:03:45.632631+0000","last_undegraded":"2023-10-01T09:03:45.632631+0000","last_fullsized":"2023-10-01T09:03:45.632631+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932682+0000","last_change":"2023-10-01T09:03:03.199109+0000","last_active":"2023-10-01T09:03:45.932682+0000","last_peered":"2023-10-01T09:03:45.932682+0000","last_clean":"2023-10-01T09:03:45.932682+0000","last_became_active":"2023-10-01T09:03:03.198810+0000","last_became_peered":"2023-10-01T09:03:03.198810+0000","last_unstale":"2023-10-01T09:03:45.932682+0000","last_undegraded":"2023-10-01T09:03:45.932682+0000","last_fullsized":"2023-10-01T09:03:45.932682+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635430+0000","last_change":"2023-10-01T09:03:03.227888+0000","last_active":"2023-10-01T09:03:45.635430+0000","last_peered":"2023-10-01T09:03:45.635430+0000","last_clean":"2023-10-01T09:03:45.635430+0000","last_became_active":"2023-10-01T09:03:03.227621+0000","last_became_peered":"2023-10-01T09:03:03.227621+0000","last_unstale":"2023-10-01T09:03:45.635430+0000","last_undegraded":"2023-10-01T09:03:45.635430+0000","last_fullsized":"2023-10-01T09:03:45.635430+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635126+0000","last_change":"2023-10-01T09:03:03.199462+0000","last_active":"2023-10-01T09:03:45.635126+0000","last_peered":"2023-10-01T09:03:45.635126+0000","last_clean":"2023-10-01T09:03:45.635126+0000","last_became_active":"2023-10-01T09:03:03.199188+0000","last_became_peered":"2023-10-01T09:03:03.199188+0000","last_unstale":"2023-10-01T09:03:45.635126+0000","last_undegraded":"2023-10-01T09:03:45.635126+0000","last_fullsized":"2023-10-01T09:03:45.635126+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215653+0000","last_change":"2023-10-01T09:03:03.228462+0000","last_active":"2023-10-01T09:03:05.215653+0000","last_peered":"2023-10-01T09:03:05.215653+0000","last_clean":"2023-10-01T09:03:05.215653+0000","last_became_active":"2023-10-01T09:03:03.228010+0000","last_became_peered":"2023-10-01T09:03:03.228010+0000","last_unstale":"2023-10-01T09:03:05.215653+0000","last_undegraded":"2023-10-01T09:03:05.215653+0000","last_fullsized":"2023-10-01T09:03:05.215653+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932159+0000","last_change":"2023-10-01T09:03:03.194952+0000","last_active":"2023-10-01T09:03:45.932159+0000","last_peered":"2023-10-01T09:03:45.932159+0000","last_clean":"2023-10-01T09:03:45.932159+0000","last_became_active":"2023-10-01T09:03:03.194643+0000","last_became_peered":"2023-10-01T09:03:03.194643+0000","last_unstale":"2023-10-01T09:03:45.932159+0000","last_undegraded":"2023-10-01T09:03:45.932159+0000","last_fullsized":"2023-10-01T09:03:45.932159+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632819+0000","last_change":"2023-10-01T09:03:04.405231+0000","last_active":"2023-10-01T09:03:45.632819+0000","last_peered":"2023-10-01T09:03:45.632819+0000","last_clean":"2023-10-01T09:03:45.632819+0000","last_became_active":"2023-10-01T09:03:04.404876+0000","last_became_peered":"2023-10-01T09:03:04.404876+0000","last_unstale":"2023-10-01T09:03:45.632819+0000","last_undegraded":"2023-10-01T09:03:45.632819+0000","last_fullsized":"2023-10-01T09:03:45.632819+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215761+0000","last_change":"2023-10-01T09:03:03.202905+0000","last_active":"2023-10-01T09:03:05.215761+0000","last_peered":"2023-10-01T09:03:05.215761+0000","last_clean":"2023-10-01T09:03:05.215761+0000","last_became_active":"2023-10-01T09:03:03.202621+0000","last_became_peered":"2023-10-01T09:03:03.202621+0000","last_unstale":"2023-10-01T09:03:05.215761+0000","last_undegraded":"2023-10-01T09:03:05.215761+0000","last_fullsized":"2023-10-01T09:03:05.215761+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635454+0000","last_change":"2023-10-01T09:03:03.228053+0000","last_active":"2023-10-01T09:03:45.635454+0000","last_peered":"2023-10-01T09:03:45.635454+0000","last_clean":"2023-10-01T09:03:45.635454+0000","last_became_active":"2023-10-01T09:03:03.227153+0000","last_became_peered":"2023-10-01T09:03:03.227153+0000","last_unstale":"2023-10-01T09:03:45.635454+0000","last_undegraded":"2023-10-01T09:03:45.635454+0000","last_fullsized":"2023-10-01T09:03:45.635454+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932302+0000","last_change":"2023-10-01T09:03:03.195227+0000","last_active":"2023-10-01T09:03:45.932302+0000","last_peered":"2023-10-01T09:03:45.932302+0000","last_clean":"2023-10-01T09:03:45.932302+0000","last_became_active":"2023-10-01T09:03:03.194964+0000","last_became_peered":"2023-10-01T09:03:03.194964+0000","last_unstale":"2023-10-01T09:03:45.932302+0000","last_undegraded":"2023-10-01T09:03:45.932302+0000","last_fullsized":"2023-10-01T09:03:45.932302+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":21,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T09:03:46.638524+0000","last_change":"2023-10-01T09:03:46.638524+0000","last_active":"2023-10-01T09:03:46.638524+0000","last_peered":"2023-10-01T09:03:46.638524+0000","last_clean":"2023-10-01T09:03:46.638524+0000","last_became_active":"2023-10-01T09:03:46.638209+0000","last_became_peered":"2023-10-01T09:03:46.638209+0000","last_unstale":"2023-10-01T09:03:46.638524+0000","last_undegraded":"2023-10-01T09:03:46.638524+0000","last_fullsized":"2023-10-01T09:03:46.638524+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":19,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634523+0000","last_change":"2023-10-01T09:03:03.200261+0000","last_active":"2023-10-01T09:03:45.634523+0000","last_peered":"2023-10-01T09:03:45.634523+0000","last_clean":"2023-10-01T09:03:45.634523+0000","last_became_active":"2023-10-01T09:03:03.199944+0000","last_became_peered":"2023-10-01T09:03:03.199944+0000","last_unstale":"2023-10-01T09:03:45.634523+0000","last_undegraded":"2023-10-01T09:03:45.634523+0000","last_fullsized":"2023-10-01T09:03:45.634523+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632559+0000","last_change":"2023-10-01T09:03:03.224723+0000","last_active":"2023-10-01T09:03:45.632559+0000","last_peered":"2023-10-01T09:03:45.632559+0000","last_clean":"2023-10-01T09:03:45.632559+0000","last_became_active":"2023-10-01T09:03:03.224379+0000","last_became_peered":"2023-10-01T09:03:03.224379+0000","last_unstale":"2023-10-01T09:03:45.632559+0000","last_undegraded":"2023-10-01T09:03:45.632559+0000","last_fullsized":"2023-10-01T09:03:45.632559+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.932421+0000","last_change":"2023-10-01T09:03:03.199740+0000","last_active":"2023-10-01T09:03:45.932421+0000","last_peered":"2023-10-01T09:03:45.932421+0000","last_clean":"2023-10-01T09:03:45.932421+0000","last_became_active":"2023-10-01T09:03:03.199421+0000","last_became_peered":"2023-10-01T09:03:03.199421+0000","last_unstale":"2023-10-01T09:03:45.932421+0000","last_undegraded":"2023-10-01T09:03:45.932421+0000","last_fullsized":"2023-10-01T09:03:45.932421+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635238+0000","last_change":"2023-10-01T09:03:03.228015+0000","last_active":"2023-10-01T09:03:45.635238+0000","last_peered":"2023-10-01T09:03:45.635238+0000","last_clean":"2023-10-01T09:03:45.635238+0000","last_became_active":"2023-10-01T09:03:03.227754+0000","last_became_peered":"2023-10-01T09:03:03.227754+0000","last_unstale":"2023-10-01T09:03:45.635238+0000","last_undegraded":"2023-10-01T09:03:45.635238+0000","last_fullsized":"2023-10-01T09:03:45.635238+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.632832+0000","last_change":"2023-10-01T09:03:03.201187+0000","last_active":"2023-10-01T09:03:45.632832+0000","last_peered":"2023-10-01T09:03:45.632832+0000","last_clean":"2023-10-01T09:03:45.632832+0000","last_became_active":"2023-10-01T09:03:03.200859+0000","last_became_peered":"2023-10-01T09:03:03.200859+0000","last_unstale":"2023-10-01T09:03:45.632832+0000","last_undegraded":"2023-10-01T09:03:45.632832+0000","last_fullsized":"2023-10-01T09:03:45.632832+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635189+0000","last_change":"2023-10-01T09:03:03.227458+0000","last_active":"2023-10-01T09:03:45.635189+0000","last_peered":"2023-10-01T09:03:45.635189+0000","last_clean":"2023-10-01T09:03:45.635189+0000","last_became_active":"2023-10-01T09:03:03.226857+0000","last_became_peered":"2023-10-01T09:03:03.226857+0000","last_unstale":"2023-10-01T09:03:45.635189+0000","last_undegraded":"2023-10-01T09:03:45.635189+0000","last_fullsized":"2023-10-01T09:03:45.635189+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215921+0000","last_change":"2023-10-01T09:03:03.202503+0000","last_active":"2023-10-01T09:03:05.215921+0000","last_peered":"2023-10-01T09:03:05.215921+0000","last_clean":"2023-10-01T09:03:05.215921+0000","last_became_active":"2023-10-01T09:03:03.202048+0000","last_became_peered":"2023-10-01T09:03:03.202048+0000","last_unstale":"2023-10-01T09:03:05.215921+0000","last_undegraded":"2023-10-01T09:03:05.215921+0000","last_fullsized":"2023-10-01T09:03:05.215921+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T09:03:46.767306+0000","last_change":"2023-10-01T09:03:46.767306+0000","last_active":"2023-10-01T09:03:46.767306+0000","last_peered":"2023-10-01T09:03:46.767306+0000","last_clean":"2023-10-01T09:03:46.767306+0000","last_became_active":"2023-10-01T09:03:46.766871+0000","last_became_peered":"2023-10-01T09:03:46.766871+0000","last_unstale":"2023-10-01T09:03:46.767306+0000","last_undegraded":"2023-10-01T09:03:46.767306+0000","last_fullsized":"2023-10-01T09:03:46.767306+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635118+0000","last_change":"2023-10-01T09:03:03.198727+0000","last_active":"2023-10-01T09:03:45.635118+0000","last_peered":"2023-10-01T09:03:45.635118+0000","last_clean":"2023-10-01T09:03:45.635118+0000","last_became_active":"2023-10-01T09:03:03.198309+0000","last_became_peered":"2023-10-01T09:03:03.198309+0000","last_unstale":"2023-10-01T09:03:45.635118+0000","last_undegraded":"2023-10-01T09:03:45.635118+0000","last_fullsized":"2023-10-01T09:03:45.635118+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T09:03:05.215683+0000","last_change":"2023-10-01T09:03:03.228290+0000","last_active":"2023-10-01T09:03:05.215683+0000","last_peered":"2023-10-01T09:03:05.215683+0000","last_clean":"2023-10-01T09:03:05.215683+0000","last_became_active":"2023-10-01T09:03:03.227612+0000","last_became_peered":"2023-10-01T09:03:03.227612+0000","last_unstale":"2023-10-01T09:03:05.215683+0000","last_undegraded":"2023-10-01T09:03:05.215683+0000","last_fullsized":"2023-10-01T09:03:05.215683+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.631923+0000","last_change":"2023-10-01T09:03:03.199888+0000","last_active":"2023-10-01T09:03:45.631923+0000","last_peered":"2023-10-01T09:03:45.631923+0000","last_clean":"2023-10-01T09:03:45.631923+0000","last_became_active":"2023-10-01T09:03:03.199579+0000","last_became_peered":"2023-10-01T09:03:03.199579+0000","last_unstale":"2023-10-01T09:03:45.631923+0000","last_undegraded":"2023-10-01T09:03:45.631923+0000","last_fullsized":"2023-10-01T09:03:45.631923+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.635485+0000","last_change":"2023-10-01T09:03:03.226961+0000","last_active":"2023-10-01T09:03:45.635485+0000","last_peered":"2023-10-01T09:03:45.635485+0000","last_clean":"2023-10-01T09:03:45.635485+0000","last_became_active":"2023-10-01T09:03:03.226660+0000","last_became_peered":"2023-10-01T09:03:03.226660+0000","last_unstale":"2023-10-01T09:03:45.635485+0000","last_undegraded":"2023-10-01T09:03:45.635485+0000","last_fullsized":"2023-10-01T09:03:45.635485+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T09:03:45.634325+0000","last_change":"2023-10-01T09:03:03.201578+0000","last_active":"2023-10-01T09:03:45.634325+0000","last_peered":"2023-10-01T09:03:45.634325+0000","last_clean":"2023-10-01T09:03:45.634325+0000","last_became_active":"2023-10-01T09:03:03.201137+0000","last_became_peered":"2023-10-01T09:03:03.201137+0000","last_unstale":"2023-10-01T09:03:45.634325+0000","last_undegraded":"2023-10-01T09:03:45.634325+0000","last_fullsized":"2023-10-01T09:03:45.634325+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430241,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297736,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447416,"statfs":{"total":95995035648,"available":95690153984,"internally_reserved":0,"allocated":655360,"data_stored":271700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.54600000000000004,"15min":0.52200000000000002},"min":{"1min":0.34200000000000003,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.9580000000000002,"5min":2.9580000000000002,"15min":2.9580000000000002},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.503,"5min":0.48999999999999999,"15min":0.48799999999999999},"min":{"1min":0.39600000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.63800000000000001,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.56499999999999995}]},{"osd":1,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.48699999999999999,"15min":0.48299999999999998},"min":{"1min":0.38600000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.63400000000000001,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.498,"15min":0.48399999999999999},"min":{"1min":0.36099999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.504}]},{"osd":2,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.71499999999999997,"15min":0.745},"min":{"1min":0.36699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.91000000000000003,"5min":5.5949999999999998,"15min":5.5949999999999998},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.52100000000000002,"15min":0.50700000000000001},"min":{"1min":0.432,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.66000000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.69799999999999995,"15min":0.72399999999999998},"min":{"1min":0.40300000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68700000000000006,"5min":5.6210000000000004,"15min":5.6210000000000004},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.54500000000000004,"15min":0.52300000000000002},"min":{"1min":0.40699999999999997,"5min":0.378,"15min":0.378},"max":{"1min":2.2050000000000001,"5min":2.2050000000000001,"15min":2.2050000000000001},"last":0.60799999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.46500000000000002,"15min":0.45900000000000002},"min":{"1min":0.30299999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.68600000000000005,"15min":0.71399999999999997},"min":{"1min":0.38800000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.72099999999999997,"5min":5.5369999999999999,"15min":5.5369999999999999},"last":0.52400000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.51100000000000001,"15min":0.50700000000000001},"min":{"1min":0.33100000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.70599999999999996,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.53200000000000003,"15min":0.52800000000000002},"min":{"1min":0.35399999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.72799999999999998,"5min":0.746,"15min":0.746},"last":0.626}]},{"osd":6,"last update":"Sun Oct 1 09:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.55400000000000005,"15min":0.55000000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.74099999999999999,"15min":0.76700000000000002},"min":{"1min":0.40000000000000002,"5min":0.37,"15min":0.37},"max":{"1min":0.78100000000000003,"5min":5.5700000000000003,"15min":5.5700000000000003},"last":0.58699999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626470,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.54600000000000004,"15min":0.54800000000000004},"min":{"1min":0.36899999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.65200000000000002,"5min":0.752,"15min":0.752},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.36099999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.64900000000000002,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.60499999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53300000000000003,"15min":0.53500000000000003},"min":{"1min":0.39600000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65300000000000002,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.629,"15min":0.64400000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.70799999999999996,"5min":3.2429999999999999,"15min":3.2429999999999999},"last":0.53300000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.51100000000000001,"15min":0.51300000000000001},"min":{"1min":0.376,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.621,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53400000000000003,"15min":0.53700000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.67400000000000004,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.56299999999999994}]},{"osd":3,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54200000000000004,"15min":0.54300000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.41099999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.86699999999999999,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.47699999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.48999999999999999,"15min":0.48599999999999999},"min":{"1min":0.34599999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.63100000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.504,"15min":0.50700000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.72699999999999998,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52900000000000003,"15min":0.52800000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.76300000000000001,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.55000000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.52000000000000002,"15min":0.51600000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.56899999999999995,"15min":0.56100000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77400000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.59199999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574908,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297704,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447448,"statfs":{"total":95995035648,"available":95690186752,"internally_reserved":0,"allocated":622592,"data_stored":269633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.36399999999999999,"15min":0.36499999999999999},"min":{"1min":0.23899999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.52000000000000002,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.377},{"interface":"front","average":{"1min":0.32500000000000001,"5min":0.40000000000000002,"15min":0.42699999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.46600000000000003,"5min":2.2829999999999999,"15min":2.2829999999999999},"last":0.42999999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.39000000000000001,"15min":0.39600000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.64200000000000002,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.42499999999999999,"15min":0.46800000000000003},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.72099999999999997,"5min":2.3199999999999998,"15min":2.3199999999999998},"last":0.36499999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.40999999999999998,"15min":0.41799999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.46000000000000002,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.45600000000000002,"15min":0.46700000000000003},"min":{"1min":0.33600000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.58399999999999996,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.5}]},{"osd":4,"last update":"Sun Oct 1 09:03:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.55600000000000005,"15min":0.55500000000000005},"min":{"1min":0.40200000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.86399999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58799999999999997,"15min":0.58199999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.86899999999999999,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.57699999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57099999999999995,"15min":0.57699999999999996},"min":{"1min":0.378,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.83699999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60499999999999998,"15min":0.60699999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.84699999999999998,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.55300000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:03:13 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.60999999999999999,"15min":0.621},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.85499999999999998,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.61599999999999999,"15min":0.627},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76300000000000001,"5min":1.03,"15min":1.03},"last":0.52200000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58099999999999996,"15min":0.58299999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77400000000000002,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.61899999999999999,"15min":0.623},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.82999999999999996,"5min":0.999,"15min":0.999},"last":0.70599999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738433,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.34499999999999997,"15min":0.34300000000000003},"min":{"1min":0.245,"5min":0.22,"15min":0.22},"max":{"1min":0.46400000000000002,"5min":0.52900000000000003,"15min":0.52900000000000003},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.245,"5min":0.221,"15min":0.221},"max":{"1min":0.53600000000000003,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.59999999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:03:46 2023","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.39600000000000002,"15min":0.40600000000000003},"min":{"1min":0.26900000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.56299999999999994,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.34000000000000002,"5min":0.376,"15min":0.38100000000000001},"min":{"1min":0.253,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.47899999999999998,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.49399999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.41799999999999998,"15min":0.42999999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.51600000000000001,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.45700000000000002,"15min":0.47299999999999998},"min":{"1min":0.28399999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56999999999999995,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.55400000000000005}]},{"osd":4,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.42599999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.80000000000000004,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.41299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.77100000000000002,"5min":0.87,"15min":0.87},"last":0.84199999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:03:52 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.56699999999999995,"15min":0.57499999999999996},"min":{"1min":0.39100000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.57699999999999996,"15min":0.59199999999999997},"min":{"1min":0.39700000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.68400000000000005,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.88400000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.62,"15min":0.624},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.86599999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.39900000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.749,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.46000000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.91300000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411383,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":0.36099999999999999,"15min":0.371},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.44,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.371},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.23100000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.52500000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.48799999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.375,"15min":0.373},"min":{"1min":0.246,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.56499999999999995,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.37,"5min":0.38900000000000001,"15min":0.39200000000000002},"min":{"1min":0.26900000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.44400000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.435,"15min":0.44600000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.499,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.374,"5min":0.45400000000000001,"15min":0.47499999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.56999999999999995,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.39900000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:03:27 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54000000000000004,"15min":0.54100000000000004},"min":{"1min":0.434,"5min":0.374,"15min":0.374},"max":{"1min":0.65800000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.68200000000000005,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.67700000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.41299999999999998,"5min":0.379,"15min":0.379},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55800000000000005,"15min":0.56200000000000006},"min":{"1min":0.40699999999999997,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.63300000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:03:10 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.59799999999999998,"15min":0.60499999999999998},"min":{"1min":0.44400000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.69899999999999995,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.60599999999999998,"15min":0.61199999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.73699999999999999,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.66000000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:03:33 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.58599999999999997,"15min":0.58899999999999997},"min":{"1min":0.47899999999999998,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.71599999999999997,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.60499999999999998,"15min":0.61099999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.70199999999999996,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.61899999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182451,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.51,"5min":1.51,"15min":1.51},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.41899999999999998,"15min":0.41699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.8599999999999999,"5min":2.8599999999999999,"15min":2.8599999999999999},"last":0.35999999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.42999999999999999,"15min":0.44800000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.503,"5min":0.62,"15min":0.62},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.37,"5min":0.40999999999999998,"15min":0.41899999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.52700000000000002,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.42899999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:03:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.44600000000000001,"15min":0.45800000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.54000000000000004,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.38,"5min":0.437,"15min":0.45600000000000002},"min":{"1min":0.27200000000000002,"5min":0.25,"15min":0.25},"max":{"1min":0.59199999999999997,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.496}]},{"osd":4,"last update":"Sun Oct 1 09:03:31 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59499999999999997,"15min":0.59699999999999998},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.371,"5min":1.371,"15min":1.371},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.59699999999999998,"15min":0.60499999999999998},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.76700000000000002,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.63100000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:03:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.57599999999999996,"15min":0.56699999999999995},"min":{"1min":0.433,"5min":0.377,"15min":0.377},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.60099999999999998,"15min":0.59499999999999997},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.84899999999999998,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.60599999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:03:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.59899999999999998,"15min":0.60699999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.84199999999999997,"5min":1.389,"15min":1.389},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.60199999999999998,"15min":0.60599999999999998},"min":{"1min":0.45300000000000001,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.80000000000000004,"5min":1.278,"15min":1.278},"last":0.58599999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:03:36 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.57299999999999995,"15min":0.56799999999999995},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60399999999999998,"15min":0.60199999999999998},"min":{"1min":0.44,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.82499999999999996,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.67900000000000005}]}]},{"osd":4,"up_from":30,"seq":128849018926,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.55900000000000005,"15min":0.52500000000000002},"min":{"1min":0.34999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.4459999999999997,"5min":5.4459999999999997,"15min":5.4459999999999997},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.57799999999999996,"15min":0.52600000000000002},"min":{"1min":0.35999999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.1890000000000001,"5min":7.1890000000000001,"15min":7.1890000000000001},"last":0.57399999999999995}]},{"osd":1,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51200000000000001,"15min":0.51100000000000001},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.53100000000000003,"15min":0.495},"min":{"1min":0.372,"5min":0.315,"15min":0.315},"max":{"1min":4.2279999999999998,"5min":4.2279999999999998,"15min":4.2279999999999998},"last":0.73999999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.55800000000000005,"15min":0.52100000000000002},"min":{"1min":0.39300000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.3310000000000004,"5min":5.3310000000000004,"15min":5.3310000000000004},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.52000000000000002,"15min":0.51200000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.68000000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:03:30 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.55600000000000005,"15min":0.51700000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.6429999999999998,"5min":5.6429999999999998,"15min":5.6429999999999998},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.52300000000000002,"15min":0.51800000000000002},"min":{"1min":0.38600000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.1279999999999999,"5min":1.1279999999999999,"15min":1.1279999999999999},"last":0.77600000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.41899999999999998,"15min":0.39600000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.41599999999999998,"15min":0.38},"min":{"1min":0.31,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.66000000000000003,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.65700000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:03:14 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.45500000000000002,"15min":0.44500000000000001},"min":{"1min":0.34599999999999997,"5min":0.313,"15min":0.313},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.40200000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.71599999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:03:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.53700000000000003,"15min":0.54100000000000004},"min":{"1min":0.33000000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.55400000000000005,"15min":0.55500000000000005},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.76800000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822699,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.499,"15min":0.498},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.747,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.504,"5min":0.51100000000000001,"15min":0.51500000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.45400000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.49299999999999999,"15min":0.495},"min":{"1min":0.34599999999999997,"5min":0.309,"15min":0.309},"max":{"1min":0.68300000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.504},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.54500000000000004,"15min":0.53300000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.65900000000000003,"5min":4.0490000000000004,"15min":4.0490000000000004},"last":0.495}]},{"osd":2,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.53600000000000003,"15min":0.54400000000000004},"min":{"1min":0.39500000000000002,"5min":0.308,"15min":0.308},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.51300000000000001,"15min":0.50600000000000001},"min":{"1min":0.33900000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.59299999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.53500000000000003,"15min":0.52700000000000002},"min":{"1min":0.36199999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.38800000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.46500000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:03:49 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.42199999999999999,"15min":0.40000000000000002},"min":{"1min":0.28599999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.67500000000000004,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.46100000000000002,"15min":0.42299999999999999},"min":{"1min":0.34699999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.438}]},{"osd":6,"last update":"Sun Oct 1 09:03:11 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.48699999999999999,"15min":0.47899999999999998},"min":{"1min":0.34100000000000003,"5min":0.313,"15min":0.313},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.47699999999999998,"15min":0.46500000000000002},"min":{"1min":0.35999999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.54100000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:03:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54000000000000004,"15min":0.53500000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.53400000000000003,"15min":0.52700000000000002},"min":{"1min":0.36499999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.52000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:04:03.791 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-01T09:04:03.791 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-01T09:04:03.791 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-01T09:04:03.791 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-01T09:04:04.137 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:04:04.713 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:04 smithi138 conmon[100289]: cluster 2023-10-01T09:04:02.855955+0000 mgr.y (mgr.14433) 106 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:04.713 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:04 smithi138 conmon[100289]: audit 2023-10-01T09:04:03.132189+0000 mgr.y (mgr.14433) 107 : audit [DBG] from='client.24707 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:04:04.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:04 smithi138 conmon[109959]: cluster 2023-10-01T09:04:02.855955+0000 mgr.y (mgr.14433) 106 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:04.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:04 smithi138 conmon[109959]: audit 2023-10-01T09:04:03.132189+0000 mgr.y (mgr.14433) 107 : audit [DBG] from='client.24707 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:04:04.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:04 smithi190 conmon[100518]: cluster 2023-10-01T09:04:02.855955+0000 mgr.y (mgr.14433) 106 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:04 smithi190 conmon[100518]: audit 2023-10-01T09:04:03.132189+0000 mgr.y (mgr.14433) 107 : audit [DBG] from='client.24707 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:04:05.441 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:05 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:04:05] "GET /metrics HTTP/1.1" 200 191362 "" "Prometheus/2.33.4" 2023-10-01T09:04:05.788 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:04:06.402 INFO:teuthology.orchestra.run.smithi138.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-01T09:04:06.403 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-01T09:04:06.403 INFO:teuthology.run_tasks:Running task print... 2023-10-01T09:04:06.412 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-10-01T09:04:06.412 INFO:teuthology.run_tasks:Running task parallel... 2023-10-01T09:04:06.428 INFO:teuthology.task.parallel:starting parallel... 2023-10-01T09:04:06.428 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-01T09:04:06.428 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-01T09:04:06.432 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-01T09:04:06.432 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-01T09:04:06.432 DEBUG:teuthology.orchestra.run.smithi138:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-01T09:04:06.433 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-01T09:04:06.433 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-10-01T09:04:06.433 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi138.front.sepia.ceph.com 2023-10-01T09:04:06.433 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-10-01T09:04:06.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T09:04:06.451 INFO:teuthology.orchestra.run.smithi138.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-01T09:04:06.451 DEBUG:teuthology.orchestra.run.smithi138:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-01T09:04:06.507 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-01T09:04:06.507 DEBUG:teuthology.orchestra.run.smithi138:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-01T09:04:06.563 INFO:tasks.workunit:timeout=3h 2023-10-01T09:04:06.564 INFO:tasks.workunit:cleanup=True 2023-10-01T09:04:06.564 DEBUG:teuthology.orchestra.run.smithi138:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-01T09:04:06.621 INFO:tasks.workunit.client.0.smithi138.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-01T09:04:06.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:06 smithi190 conmon[100518]: cluster 2023-10-01T09:04:04.856447+0000 mgr.y (mgr.14433) 108 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:06 smithi190 conmon[100518]: audit 2023-10-01T09:04:05.788344 2023-10-01T09:04:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:06 smithi190 conmon[100518]: +0000 mon.a (mon.0) 680 : audit [DBG] from='client.? 172.21.15.138:0/1116812650' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T09:04:06.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:06 smithi138 conmon[109959]: cluster 2023-10-01T09:04:04.856447+0000 mgr.y (mgr.14433) 108 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:06.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:06 smithi138 conmon[109959]: audit 2023-10-01T09:04:05.788344+0000 mon.a (mon.0) 680 : audit [DBG] from='client.? 172.21.15.138:0/1116812650' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T09:04:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:06 smithi138 conmon[100289]: cluster 2023-10-01T09:04:04.856447+0000 mgr.y (mgr.14433) 108 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:06 smithi138 conmon[100289]: audit 2023-10-01T09:04:05.788344+0000 mon.a (mon.0) 680 : audit [DBG] from='client.? 172.21.15.138:0/1116812650' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T09:04:07.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:04:06 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:04:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:04:08.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[109959]: cluster 2023-10-01T09:04:06.856945 2023-10-01T09:04:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[109959]: +0000 mgr.y (mgr.14433) 109 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[109959]: audit 2023-10-01T09:04:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[109959]: 2023-10-01T09:04:08.309525+0000 mon.a (mon.0) 681 2023-10-01T09:04:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[109959]: : audit [INF] from='client.? 172.21.15.138:0/62246501' entity='client.admin' 2023-10-01T09:04:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:04:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[109959]: 01T09:04:08.346605+0000 mon.c (mon 2023-10-01T09:04:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[109959]: .2) 92 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:04:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100289]: cluster 2023-10-01T09:04:06. 2023-10-01T09:04:08.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100289]: 856945+0000 mgr.y (mgr.14433) 109 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:08.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100289]: audit 2023-10 2023-10-01T09:04:08.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100289]: -01T09:04:08.309525+0000 mon.a (mon.0) 681 : audit [INF] from='client.? 172.21.15.138:0/62246501' entity='client.admin' 2023-10-01T09:04:08.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100289]: 2023-10-01T09:04:08.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100289]: audit 2023-10-01T09:04:08.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100289]: 2023-10-01T09:04 2023-10-01T09:04:08.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100289]: :08.346605+0000 mon.c (mon.2) 92 2023-10-01T09:04:08.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100289]: : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:04:08.571 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100738]: [01/Oct/2023:09:04:08] ENGINE Bus STOPPING 2023-10-01T09:04:08.572 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100738]: [01/Oct/2023:09:04:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T09:04:08.572 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100738]: [01/Oct/2023:09:04:08] ENGINE Bus STOPPED 2023-10-01T09:04:08.572 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100738]: [01/Oct/2023:09:04:08] ENGINE Bus STARTING 2023-10-01T09:04:08.572 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100738]: [01/Oct/2023:09:04:08] ENGINE Serving on http://:::9283 2023-10-01T09:04:08.572 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:08 smithi138 conmon[100738]: [01/Oct/2023:09:04:08] ENGINE Bus STARTED 2023-10-01T09:04:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:08 smithi190 conmon[100518]: cluster 2023-10-01T09:04:06.856945+0000 mgr.y (mgr 2023-10-01T09:04:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:08 smithi190 conmon[100518]: .14433) 109 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:08 smithi190 conmon[100518]: audit 2023-10-01T09:04:08.309525+0000 mon.a (mon.0) 681 : audit [INF] from='client.? 172.21.15.138:0/62246501' entity='client.admin' 2023-10-01T09:04:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:08 smithi190 conmon[100518]: audit 2023-10-01T09:04: 2023-10-01T09:04:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:08 smithi190 conmon[100518]: 08.346605+0000 mon.c (mon.2) 92 : audit 2023-10-01T09:04:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:08 smithi190 conmon[100518]: [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:04:09.087 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph config set global log_to_journald false --force' 2023-10-01T09:04:09.546 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:09 smithi138 conmon[109959]: audit 2023-10-01T09:04:08.937005+0000 2023-10-01T09:04:09.547 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:09 smithi138 conmon[109959]: mon.c (mon.2) 93 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:04:09.547 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:09 smithi138 conmon[109959]: audit 2023-10-01T09:04:08.937693+0000 mon.c (mon.2) 94 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:04:09.547 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:09 smithi138 conmon[109959]: audit 2023-10-01T09:04:08.942544+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:09.547 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:09 smithi138 conmon[100289]: audit 2023-10-01T09:04:08.937005+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:04:09.547 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:09 smithi138 conmon[100289]: audit 2023-10-01T09:04:08.937693+0000 mon.c (mon.2) 94 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:04:09.547 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:09 smithi138 conmon[100289]: audit 2023-10-01T09:04:08.942544+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:09.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:09 smithi190 conmon[100518]: audit 2023-10-01T09:04:08.937005+0000 mon.c (mon.2) 93 : audit 2023-10-01T09:04:09.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:09 smithi190 conmon[100518]: [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:04:09.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:09 smithi190 conmon[100518]: audit 2023-10-01T09:04:08.937693+0000 mon.c (mon.2) 94 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:04:09.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:09 smithi190 conmon[100518]: audit 2023-10-01T09:04:08.942544+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:10.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:10 smithi190 conmon[100518]: cluster 2023-10-01T09:04:08.857302+0000 mgr.y (mgr.14433) 110 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:10.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:10 smithi138 conmon[100289]: cluster 2023-10-01T09:04:08.857302+0000 mgr.y 2023-10-01T09:04:10.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:10 smithi138 conmon[100289]: (mgr.14433) 110 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:10.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:10 smithi138 conmon[109959]: cluster 2023-10-01T09:04:08.857302+0000 mgr.y (mgr.14433 2023-10-01T09:04:10.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:10 smithi138 conmon[109959]: ) 110 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:10.891 DEBUG:teuthology.orchestra.run.smithi138:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-01T09:04:11.157 INFO:tasks.workunit.client.0.smithi138.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-01T09:04:11.159 INFO:tasks.workunit.client.0.smithi138.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-01T09:04:11.159 INFO:tasks.workunit.client.0.smithi138.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-01T09:04:12.435 INFO:tasks.workunit.client.0.smithi138.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-01T09:04:12.489 INFO:tasks.workunit.client.0.smithi138.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-01T09:04:12.526 INFO:tasks.workunit.client.0.smithi138.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-01T09:04:12.528 INFO:tasks.workunit.client.0.smithi138.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-01T09:04:12.528 INFO:tasks.workunit.client.0.smithi138.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-01T09:04:12.564 INFO:tasks.workunit.client.0.smithi138.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-01T09:04:12.567 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T09:04:12.568 DEBUG:teuthology.orchestra.run.smithi138:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-01T09:04:12.587 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-01T09:04:12.588 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-01T09:04:12.588 DEBUG:teuthology.orchestra.run.smithi138:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-01T09:04:12.648 INFO:tasks.workunit.client.0.smithi138.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-01T09:04:12.649 INFO:tasks.workunit.client.0.smithi138.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-10-01T09:04:12.649 INFO:tasks.workunit.client.0.smithi138.stderr:+ TMPDIR=/tmp/rbd_import_export_165459 2023-10-01T09:04:12.649 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm -rf /tmp/rbd_import_export_165459 2023-10-01T09:04:12.649 INFO:tasks.workunit.client.0.smithi138.stderr:+ mkdir /tmp/rbd_import_export_165459 2023-10-01T09:04:12.650 INFO:tasks.workunit.client.0.smithi138.stderr:+ trap 'rm -rf /tmp/rbd_import_export_165459' INT TERM EXIT 2023-10-01T09:04:12.650 INFO:tasks.workunit.client.0.smithi138.stderr:+ mkdir foo.165459 2023-10-01T09:04:12.651 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import foo.165459 foo.dir 2023-10-01T09:04:12.702 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: cannot import a directory 2023-10-01T09:04:12.702 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 0% complete...failed. 2023-10-01T09:04:12.702 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: import failed: (21) Is a directory 2023-10-01T09:04:12.705 INFO:tasks.workunit.client.0.smithi138.stderr:+ true 2023-10-01T09:04:12.706 INFO:tasks.workunit.client.0.smithi138.stderr:+ rmdir foo.165459 2023-10-01T09:04:12.714 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_165459/img bs=1k count=1 seek=10 2023-10-01T09:04:12.714 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records in 2023-10-01T09:04:12.714 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records out 2023-10-01T09:04:12.715 INFO:tasks.workunit.client.0.smithi138.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.7271e-05 s, 21.7 MB/s 2023-10-01T09:04:12.715 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_165459/img bs=1k count=10 seek=100 2023-10-01T09:04:12.715 INFO:tasks.workunit.client.0.smithi138.stderr:10+0 records in 2023-10-01T09:04:12.715 INFO:tasks.workunit.client.0.smithi138.stderr:10+0 records out 2023-10-01T09:04:12.715 INFO:tasks.workunit.client.0.smithi138.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.7345e-05 s, 179 MB/s 2023-10-01T09:04:12.715 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_165459/img bs=1k count=100 seek=1000 2023-10-01T09:04:12.716 INFO:tasks.workunit.client.0.smithi138.stderr:70+1 records in 2023-10-01T09:04:12.716 INFO:tasks.workunit.client.0.smithi138.stderr:70+1 records out 2023-10-01T09:04:12.716 INFO:tasks.workunit.client.0.smithi138.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000169586 s, 425 MB/s 2023-10-01T09:04:12.716 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_165459/img bs=1k seek=10000 2023-10-01T09:04:12.717 INFO:tasks.workunit.client.0.smithi138.stderr:139+1 records in 2023-10-01T09:04:12.717 INFO:tasks.workunit.client.0.smithi138.stderr:139+1 records out 2023-10-01T09:04:12.717 INFO:tasks.workunit.client.0.smithi138.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000303513 s, 472 MB/s 2023-10-01T09:04:12.717 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_165459/img bs=1k seek=100000 2023-10-01T09:04:12.721 INFO:tasks.workunit.client.0.smithi138.stderr:70+1 records in 2023-10-01T09:04:12.722 INFO:tasks.workunit.client.0.smithi138.stderr:70+1 records out 2023-10-01T09:04:12.722 INFO:tasks.workunit.client.0.smithi138.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.00395899 s, 18.2 MB/s 2023-10-01T09:04:12.722 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_165459/img bs=1k seek=1000000 2023-10-01T09:04:12.723 INFO:tasks.workunit.client.0.smithi138.stderr:189+1 records in 2023-10-01T09:04:12.723 INFO:tasks.workunit.client.0.smithi138.stderr:189+1 records out 2023-10-01T09:04:12.723 INFO:tasks.workunit.client.0.smithi138.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000422704 s, 458 MB/s 2023-10-01T09:04:12.723 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm testimg 2023-10-01T09:04:12.758 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 0% complete...failed. 2023-10-01T09:04:12.758 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: delete error: (2) No such file or directory 2023-10-01T09:04:12.761 INFO:tasks.workunit.client.0.smithi138.stderr:+ true 2023-10-01T09:04:12.762 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format /tmp/rbd_import_export_165459/img testimg 2023-10-01T09:04:12.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:12 smithi190 conmon[100518]: cluster 2023-10-01T09:04:10.857934+0000 mgr.y (mgr.14433) 111 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:12.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:12 smithi138 conmon[100289]: cluster 2023-10-01T09:04:10. 2023-10-01T09:04:12.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:12 smithi138 conmon[100289]: 857934+0000 mgr.y (mgr.14433) 111 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:12.820 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:12 smithi138 conmon[100738]: [01/Oct/2023:09:04:12] ENGINE Bus STOPPING 2023-10-01T09:04:12.820 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:12 smithi138 conmon[100738]: [01/Oct/2023:09:04:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T09:04:12.820 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:12 smithi138 conmon[100738]: [01/Oct/2023:09:04:12] ENGINE Bus STOPPED 2023-10-01T09:04:12.820 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:12 smithi138 conmon[100738]: [01/Oct/2023:09:04:12] ENGINE Bus STARTING 2023-10-01T09:04:12.820 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:12 smithi138 conmon[100738]: [01/Oct/2023:09:04:12] ENGINE Serving on http://:::9283 2023-10-01T09:04:12.820 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:12 smithi138 conmon[100738]: [01/Oct/2023:09:04:12] ENGINE Bus STARTED 2023-10-01T09:04:12.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:12 smithi138 conmon[109959]: cluster 2023-10-01T09:04:10.857934+0000 mgr.y 2023-10-01T09:04:12.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:12 smithi138 conmon[109959]: (mgr.14433) 111 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:13 smithi190 conmon[100518]: audit 2023-10-01T09:04:12.654424+0000 mon.a (mon.0) 683 : audit 2023-10-01T09:04:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:13 smithi190 conmon[100518]: [INF] from='client.? ' entity='client.admin' 2023-10-01T09:04:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:13 smithi190 conmon[100518]: audit 2023-10-01T09:04:12.667220+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:04:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:13 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:04:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:13 smithi190 conmon[100518]: :04:12.668925+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:04:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:13 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:04:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:13 smithi190 conmon[100518]: 10-01T09:04:12.669802+0000 mon.c (mon.2) 97 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:04:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:13 smithi190 conmon[100518]: audit 2023-10-01T09:04:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:13 smithi190 conmon[100518]: 2023-10-01T09:04:12.675247+0000 mon.a (mon. 2023-10-01T09:04:14.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:13 smithi190 conmon[100518]: 0) 684 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:14.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:13 smithi190 conmon[100518]: cluster 2023 2023-10-01T09:04:14.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:13 smithi190 conmon[100518]: -10-01T09:04:12.858305+0000 mgr.y (mgr. 2023-10-01T09:04:14.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:13 smithi190 conmon[100518]: 14433) 112 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:14.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[100289]: audit 2023-10-01T09:04:12.654424+0000 mon.a (mon.0) 683 : audit [INF] from='client.? ' entity='client.admin' 2023-10-01T09:04:14.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[100289]: audit 2023-10-01T09:04:12.667220+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:04:14.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[100289]: audit 2023-10-01T09:04:12.668925+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:04:14.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[100289]: audit 2023-10-01T09:04:12.669802+0000 mon.c (mon.2) 97 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:04:14.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[100289]: audit 2023-10-01T09:04:12.675247+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:14.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[100289]: cluster 2023-10-01T09:04:12.858305+0000 mgr.y (mgr.14433) 112 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:14.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[109959]: audit 2023-10-01T09:04:12.654424 2023-10-01T09:04:14.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[109959]: +0000 mon.a (mon.0) 683 : audit [INF] from='client.? ' entity='client.admin' 2023-10-01T09:04:14.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[109959]: audit 2023-10-01T09:04:12.667220+0000 mon.c (mon. 2023-10-01T09:04:14.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[109959]: 2) 95 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:04:14.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:04:14.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[109959]: 01T09:04:12.668925+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:04:14.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[109959]: audit 2023-10-01T09:04:12.669802+0000 2023-10-01T09:04:14.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[109959]: mon.c (mon.2) 97 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:04:14.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[109959]: audit 2023-10-01T09:04:14.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[109959]: 2023-10-01T09:04:12.675247+0000 mon.a ( 2023-10-01T09:04:14.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[109959]: mon.0) 684 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:14.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[109959]: cluster 2023-10-01T09: 2023-10-01T09:04:14.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:13 smithi138 conmon[109959]: 04:12.858305+0000 mgr.y (mgr.14433) 112 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:14.858 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-10-01T09:04:15.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:14 smithi190 conmon[100518]: cluster 2023-10-01T09:04:13.687246+0000 mon.a (mon.0) 685 : 2023-10-01T09:04:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:14 smithi190 conmon[100518]: cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-01T09:04:15.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:14 smithi138 conmon[100289]: cluster 2023-10-01T09:04:13.687246+0000 mon.a (mon. 2023-10-01T09:04:15.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:14 smithi138 conmon[100289]: 0) 685 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-01T09:04:15.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:14 smithi138 conmon[109959]: cluster 2023-10-01T09:04:13.687246+0000 mon.a ( 2023-10-01T09:04:15.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:14 smithi138 conmon[109959]: mon.0) 685 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-01T09:04:15.268 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-01T09:04:15.268 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:04:15.271 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export testimg /tmp/rbd_import_export_165459/img2 2023-10-01T09:04:15.540 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-01T09:04:15.544 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export testimg - 2023-10-01T09:04:15.544 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:15 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:04:15] "GET /metrics HTTP/1.1" 200 191364 "" "Prometheus/2.33.4" 2023-10-01T09:04:15.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:15 smithi138 conmon[109959]: cluster 2023-10-01T09:04:14.689180+0000 mon.a (mon.0) 2023-10-01T09:04:15.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:15 smithi138 conmon[109959]: 686 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-01T09:04:15.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:15 smithi138 conmon[109959]: cluster 2023-10-01T09:04:14.858671+0000 mgr.y (mgr.14433) 113 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:15 smithi138 conmon[100289]: cluster 2023-10-01T09:04:14.689180+0000 mon.a (mon.0) 686 : cluster 2023-10-01T09:04:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:15 smithi138 conmon[100289]: [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-01T09:04:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:15 smithi138 conmon[100289]: cluster 2023-10-01T09:04:14.858671+0000 mgr.y (mgr.14433) 113 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:16.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:15 smithi190 conmon[100518]: cluster 2023-10-01T09:04:14.689180+0000 mon.a (mon.0) 686 2023-10-01T09:04:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:15 smithi190 conmon[100518]: : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-01T09:04:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:15 smithi190 conmon[100518]: cluster 2023-10-01T09:04:14.858671+0000 mgr.y (mgr.14433) 113 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:16.315 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-01T09:04:16.318 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm testimg 2023-10-01T09:04:16.411 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-01T09:04:16.414 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp /tmp/rbd_import_export_165459/img /tmp/rbd_import_export_165459/img2 2023-10-01T09:04:17.036 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp /tmp/rbd_import_export_165459/img /tmp/rbd_import_export_165459/img3 2023-10-01T09:04:17.309 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:04:16 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:04:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:04:17.446 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/img2 /tmp/rbd_import_export_165459/img3 2023-10-01T09:04:18.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:17 smithi190 conmon[100518]: cluster 2023-10-01T09:04:16.859321+0000 mgr.y (mgr.14433) 114 : cluster 2023-10-01T09:04:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:17 smithi190 conmon[100518]: [DBG] pgmap v92: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 63 KiB/s wr, 18 op/s 2023-10-01T09:04:18.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:17 smithi138 conmon[109959]: cluster 2023-10-01T09:04:16.859321+0000 mgr.y (mgr.14433 2023-10-01T09:04:18.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:17 smithi138 conmon[109959]: ) 114 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 63 KiB/s wr, 18 op/s 2023-10-01T09:04:18.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:17 smithi138 conmon[100289]: cluster 2023-10-01T09:04:16.859321+0000 mgr.y (mgr.14433 2023-10-01T09:04:18.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:17 smithi138 conmon[100289]: ) 114 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 63 KiB/s wr, 18 op/s 2023-10-01T09:04:18.326 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format - testimg 2023-10-01T09:04:18.727 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:04:18.728 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:04:18.730 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export testimg /tmp/rbd_import_export_165459/img2 2023-10-01T09:04:18.999 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-01T09:04:19.003 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export testimg - 2023-10-01T09:04:19.639 INFO:teuthology.orchestra.run.smithi138.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T09:04:19.767 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-01T09:04:19.770 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm testimg 2023-10-01T09:04:19.860 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-01T09:04:19.864 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp /tmp/rbd_import_export_165459/img /tmp/rbd_import_export_165459/img2 2023-10-01T09:04:20.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:19 smithi138 conmon[100289]: cluster 2023-10-01T09:04:18.859705+0000 mgr.y (mgr.14433) 115 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 63 KiB/s wr, 18 op/s 2023-10-01T09:04:20.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:19 smithi138 conmon[100289]: audit 2023-10-01T09:04:19.638664+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:20.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:19 smithi138 conmon[100289]: audit 2023-10-01T09:04:19.657247+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:04:20.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:19 smithi138 conmon[109959]: cluster 2023-10-01T09:04:18.859705+0000 mgr.y (mgr.14433) 115 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 63 KiB/s wr, 18 op/s 2023-10-01T09:04:20.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:19 smithi138 conmon[109959]: audit 2023-10-01T09:04:19.638664+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:20.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:19 smithi138 conmon[109959]: audit 2023-10-01T09:04:19.657247+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:04:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:19 smithi190 conmon[100518]: cluster 2023-10-01T09:04:18.859705+0000 mgr.y 2023-10-01T09:04:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:19 smithi190 conmon[100518]: (mgr.14433) 115 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 5.4 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 63 KiB/s wr, 18 op/s 2023-10-01T09:04:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:19 smithi190 conmon[100518]: audit 2023-10-01T09:04:19.638664+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:19 smithi190 conmon[100518]: audit 2023-10-01T09:04:19.657247+0000 mon.c (mon.2) 2023-10-01T09:04:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:19 smithi190 conmon[100518]: 98 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:04:20.447 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp /tmp/rbd_import_export_165459/img /tmp/rbd_import_export_165459/img3 2023-10-01T09:04:20.858 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/img /tmp/rbd_import_export_165459/img2 /tmp/rbd_import_export_165459/img3 2023-10-01T09:04:21.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:21 smithi190 conmon[100518]: audit 2023-10-01T09:04:19.633024+0000 mgr.y (mgr.14433) 116 : audit [DBG] from='client.24785 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:04:21.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:21 smithi190 conmon[100518]: cephadm 2023-10-01T09:04:19.633746+0000 mgr.y (mgr.14433) 117 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T09:04:21.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:21 smithi190 conmon[100518]: cluster 2023-10-01T09:04:20.515627+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-01T09:04:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:21 smithi138 conmon[109959]: audit 2023-10-01T09:04:19.633024+0000 mgr.y (mgr.14433) 116 2023-10-01T09:04:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:21 smithi138 conmon[109959]: : audit [DBG] from='client.24785 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:04:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:21 smithi138 conmon[109959]: cephadm 2023-10-01T09:04:19.633746+0000 mgr.y (mgr.14433) 117 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T09:04:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:21 smithi138 conmon[109959]: cluster 2023-10-01T09:04:20.515627+0000 mon.a (mon.0) 2023-10-01T09:04:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:21 smithi138 conmon[109959]: 688 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-01T09:04:21.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:21 smithi138 conmon[100289]: audit 2023-10-01T09:04:19.633024+0000 mgr.y (mgr.14433) 116 : audit [DBG] 2023-10-01T09:04:21.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:21 smithi138 conmon[100289]: from='client.24785 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:04:21.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:21 smithi138 conmon[100289]: cephadm 2023-10-01T09:04:19.633746+0000 mgr.y (mgr.14433) 117 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T09:04:21.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:21 smithi138 conmon[100289]: cluster 2023-10-01T09:04:20.515627+0000 mon.a 2023-10-01T09:04:21.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:21 smithi138 conmon[100289]: (mon.0) 688 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-01T09:04:22.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:22 smithi190 conmon[100518]: cluster 2023-10-01T09:04:20.860190+0000 mgr.y (mgr. 2023-10-01T09:04:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:22 smithi190 conmon[100518]: 14433) 118 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 352 KiB/s rd, 145 KiB/s wr, 112 op/s 2023-10-01T09:04:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:22 smithi190 conmon[100518]: audit 2023-10-01T09:04:22.282859+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:22 smithi138 conmon[100289]: cluster 2023-10-01T09:04:20.860190+0000 mgr.y (mgr.14433 2023-10-01T09:04:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:22 smithi138 conmon[100289]: ) 118 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 352 KiB/s rd, 145 KiB/s wr, 112 op/s 2023-10-01T09:04:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:22 smithi138 conmon[100289]: audit 2023-10-01T09:04:22.282859+0000 mon.a (mon.0) 689 : audit 2023-10-01T09:04:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:22 smithi138 conmon[100289]: [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:22 smithi138 conmon[109959]: cluster 2023-10-01T09:04:20.860190+0000 mgr.y (mgr.14433) 118 : cluster 2023-10-01T09:04:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:22 smithi138 conmon[109959]: [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 352 KiB/s rd, 145 KiB/s wr, 112 op/s 2023-10-01T09:04:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:22 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:04:22.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:22 smithi138 conmon[109959]: T09:04:22.282859+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:24 smithi190 conmon[100518]: cluster 2023-10-01T09:04:22.860746+0000 mgr.y (mgr.14433) 2023-10-01T09:04:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:24 smithi190 conmon[100518]: 119 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 309 KiB/s rd, 127 KiB/s wr, 98 op/s 2023-10-01T09:04:24.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:24 smithi138 conmon[100289]: cluster 2023-10-01T09:04:22.860746+0000 mgr.y (mgr.14433) 119 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 309 KiB/s rd, 127 KiB/s wr, 98 op/s 2023-10-01T09:04:24.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:24 smithi138 conmon[109959]: cluster 2023-10-01T09:04:22.860746+0000 mgr.y (mgr.14433) 119 : 2023-10-01T09:04:24.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:24 smithi138 conmon[109959]: cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 309 KiB/s rd, 127 KiB/s wr, 98 op/s 2023-10-01T09:04:25.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:25 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:04:25] "GET /metrics HTTP/1.1" 200 191364 "" "Prometheus/2.33.4" 2023-10-01T09:04:26.614 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd help export 2023-10-01T09:04:26.614 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep -q export-format 2023-10-01T09:04:26.633 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_165459/img bs=1k count=10 seek=100 2023-10-01T09:04:26.634 INFO:tasks.workunit.client.0.smithi138.stderr:10+0 records in 2023-10-01T09:04:26.634 INFO:tasks.workunit.client.0.smithi138.stderr:10+0 records out 2023-10-01T09:04:26.634 INFO:tasks.workunit.client.0.smithi138.stderr:10240 bytes (10 kB, 10 KiB) copied, 8.0851e-05 s, 127 MB/s 2023-10-01T09:04:26.635 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format /tmp/rbd_import_export_165459/img testimg 2023-10-01T09:04:26.743 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:04:26.743 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:04:26.745 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap create testimg@snap 2023-10-01T09:04:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:26 smithi190 conmon[100518]: cluster 2023-10-01T09:04:24.861343+0000 mgr.y (mgr.14433) 120 : cluster 2023-10-01T09:04:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:26 smithi190 conmon[100518]: [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 253 KiB/s rd, 104 KiB/s wr, 80 op/s 2023-10-01T09:04:26.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:26 smithi138 conmon[109959]: cluster 2023-10-01T09:04:24.861343 2023-10-01T09:04:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:26 smithi138 conmon[109959]: +0000 mgr.y (mgr.14433) 120 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 253 KiB/s rd, 104 KiB/s wr, 80 op/s 2023-10-01T09:04:26.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:26 smithi138 conmon[100289]: cluster 2023-10-01T09:04:24.861343+0000 mgr.y (mgr.14433) 120 : cluster 2023-10-01T09:04:26.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:26 smithi138 conmon[100289]: [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 253 KiB/s rd, 104 KiB/s wr, 80 op/s 2023-10-01T09:04:27.075 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch ps' 2023-10-01T09:04:27.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:04:26 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:04:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:04:27.550 INFO:tasks.workunit.client.0.smithi138.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-01T09:04:27.557 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd image-meta set testimg key1 value1 2023-10-01T09:04:27.604 INFO:tasks.workunit.client.0.smithi138.stderr:++ rbd image-meta list testimg 2023-10-01T09:04:27.642 INFO:tasks.workunit.client.0.smithi138.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-10-01T09:04:27.642 INFO:tasks.workunit.client.0.smithi138.stderr: 2023-10-01T09:04:27.642 INFO:tasks.workunit.client.0.smithi138.stderr:Key Value 2023-10-01T09:04:27.642 INFO:tasks.workunit.client.0.smithi138.stderr:key1 value1' 2023-10-01T09:04:27.642 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_165459/img_v2 2023-10-01T09:04:27.685 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T09:04:27.687 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_165459/img_v2 testimg_import 2023-10-01T09:04:28.565 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T09:04:28.567 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd info testimg_import 2023-10-01T09:04:28.600 INFO:tasks.workunit.client.0.smithi138.stdout:rbd image 'testimg_import': 2023-10-01T09:04:28.600 INFO:tasks.workunit.client.0.smithi138.stdout: size 110 KiB in 1 objects 2023-10-01T09:04:28.600 INFO:tasks.workunit.client.0.smithi138.stdout: order 22 (4 MiB objects) 2023-10-01T09:04:28.600 INFO:tasks.workunit.client.0.smithi138.stdout: snapshot_count: 1 2023-10-01T09:04:28.600 INFO:tasks.workunit.client.0.smithi138.stdout: id: 3a441a9300ac 2023-10-01T09:04:28.600 INFO:tasks.workunit.client.0.smithi138.stdout: block_name_prefix: rbd_data.3a441a9300ac 2023-10-01T09:04:28.601 INFO:tasks.workunit.client.0.smithi138.stdout: format: 2 2023-10-01T09:04:28.601 INFO:tasks.workunit.client.0.smithi138.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-01T09:04:28.601 INFO:tasks.workunit.client.0.smithi138.stdout: op_features: 2023-10-01T09:04:28.601 INFO:tasks.workunit.client.0.smithi138.stdout: flags: 2023-10-01T09:04:28.601 INFO:tasks.workunit.client.0.smithi138.stdout: create_timestamp: Sun Oct 1 09:04:27 2023 2023-10-01T09:04:28.601 INFO:tasks.workunit.client.0.smithi138.stdout: access_timestamp: Sun Oct 1 09:04:27 2023 2023-10-01T09:04:28.601 INFO:tasks.workunit.client.0.smithi138.stdout: modify_timestamp: Sun Oct 1 09:04:27 2023 2023-10-01T09:04:28.602 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd info testimg_import@snap 2023-10-01T09:04:28.636 INFO:tasks.workunit.client.0.smithi138.stdout:rbd image 'testimg_import': 2023-10-01T09:04:28.636 INFO:tasks.workunit.client.0.smithi138.stdout: size 110 KiB in 1 objects 2023-10-01T09:04:28.636 INFO:tasks.workunit.client.0.smithi138.stdout: order 22 (4 MiB objects) 2023-10-01T09:04:28.636 INFO:tasks.workunit.client.0.smithi138.stdout: snapshot_count: 1 2023-10-01T09:04:28.637 INFO:tasks.workunit.client.0.smithi138.stdout: id: 3a441a9300ac 2023-10-01T09:04:28.637 INFO:tasks.workunit.client.0.smithi138.stdout: block_name_prefix: rbd_data.3a441a9300ac 2023-10-01T09:04:28.637 INFO:tasks.workunit.client.0.smithi138.stdout: format: 2 2023-10-01T09:04:28.637 INFO:tasks.workunit.client.0.smithi138.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-01T09:04:28.637 INFO:tasks.workunit.client.0.smithi138.stdout: op_features: 2023-10-01T09:04:28.637 INFO:tasks.workunit.client.0.smithi138.stdout: flags: 2023-10-01T09:04:28.637 INFO:tasks.workunit.client.0.smithi138.stdout: create_timestamp: Sun Oct 1 09:04:27 2023 2023-10-01T09:04:28.637 INFO:tasks.workunit.client.0.smithi138.stdout: access_timestamp: Sun Oct 1 09:04:27 2023 2023-10-01T09:04:28.637 INFO:tasks.workunit.client.0.smithi138.stdout: modify_timestamp: Sun Oct 1 09:04:27 2023 2023-10-01T09:04:28.637 INFO:tasks.workunit.client.0.smithi138.stdout: protected: False 2023-10-01T09:04:28.639 INFO:tasks.workunit.client.0.smithi138.stderr:++ rbd image-meta list testimg_import 2023-10-01T09:04:28.678 INFO:tasks.workunit.client.0.smithi138.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-10-01T09:04:28.678 INFO:tasks.workunit.client.0.smithi138.stderr: 2023-10-01T09:04:28.678 INFO:tasks.workunit.client.0.smithi138.stderr:Key Value 2023-10-01T09:04:28.678 INFO:tasks.workunit.client.0.smithi138.stderr:key1 value1' 2023-10-01T09:04:28.679 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 'There is 1 metadatum on this image: 2023-10-01T09:04:28.679 INFO:tasks.workunit.client.0.smithi138.stderr: 2023-10-01T09:04:28.679 INFO:tasks.workunit.client.0.smithi138.stderr:Key Value 2023-10-01T09:04:28.679 INFO:tasks.workunit.client.0.smithi138.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-10-01T09:04:28.679 INFO:tasks.workunit.client.0.smithi138.stderr: 2023-10-01T09:04:28.679 INFO:tasks.workunit.client.0.smithi138.stderr:Key Value 2023-10-01T09:04:28.679 INFO:tasks.workunit.client.0.smithi138.stderr:key1 value1' ']' 2023-10-01T09:04:28.679 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export testimg_import /tmp/rbd_import_export_165459/img_import 2023-10-01T09:04:28.712 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete...done. 2023-10-01T09:04:28.715 INFO:tasks.workunit.client.0.smithi138.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_165459/img /tmp/rbd_import_export_165459/img_import 2023-10-01T09:04:28.715 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp -l /tmp/rbd_import_export_165459/img /tmp/rbd_import_export_165459/img_import 2023-10-01T09:04:28.716 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_165459/img --format %b 2023-10-01T09:04:28.717 INFO:tasks.workunit.client.0.smithi138.stderr:+ origsize=24 2023-10-01T09:04:28.717 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_165459/img_import --format %b 2023-10-01T09:04:28.718 INFO:tasks.workunit.client.0.smithi138.stderr:+ exportsize=224 2023-10-01T09:04:28.719 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=200 2023-10-01T09:04:28.719 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=200 2023-10-01T09:04:28.719 INFO:tasks.workunit.client.0.smithi138.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-01T09:04:28.719 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_165459/img_snap 2023-10-01T09:04:28.754 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete...done. 2023-10-01T09:04:28.757 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_165459/img_snap_import 2023-10-01T09:04:28.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:28 smithi138 conmon[109959]: cluster 2023-10-01T09:04:26.861851+0000 mgr.y (mgr 2023-10-01T09:04:28.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:28 smithi138 conmon[109959]: .14433) 121 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 53 KiB/s wr, 65 op/s 2023-10-01T09:04:28.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:28 smithi138 conmon[109959]: cluster 2023-10-01T09:04:27.543744+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-01T09:04:28.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:28 smithi138 conmon[100289]: cluster 2023-10-01T09:04:26.861851+0000 mgr.y 2023-10-01T09:04:28.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:28 smithi138 conmon[100289]: (mgr.14433) 121 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 53 KiB/s wr, 65 op/s 2023-10-01T09:04:28.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:28 smithi138 conmon[100289]: cluster 2023-10-01T09:04:27.543744+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-01T09:04:28.801 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete...done. 2023-10-01T09:04:28.805 INFO:tasks.workunit.client.0.smithi138.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_165459/img_snap /tmp/rbd_import_export_165459/img_snap_import 2023-10-01T09:04:28.805 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp -l /tmp/rbd_import_export_165459/img_snap /tmp/rbd_import_export_165459/img_snap_import 2023-10-01T09:04:28.806 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_165459/img_snap --format %b 2023-10-01T09:04:28.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:28 smithi190 conmon[100518]: cluster 2023-10-01T09:04:26.861851+0000 mgr.y (mgr.14433) 121 : cluster 2023-10-01T09:04:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:28 smithi190 conmon[100518]: [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 53 KiB/s wr, 65 op/s 2023-10-01T09:04:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:28 smithi190 conmon[100518]: cluster 2023-10-01T09:04:27.543744+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-01T09:04:28.807 INFO:tasks.workunit.client.0.smithi138.stderr:+ origsize=224 2023-10-01T09:04:28.808 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_165459/img_snap_import --format %b 2023-10-01T09:04:28.809 INFO:tasks.workunit.client.0.smithi138.stderr:+ exportsize=224 2023-10-01T09:04:28.809 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:04:28.809 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:04:28.809 INFO:tasks.workunit.client.0.smithi138.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T09:04:28.809 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/img_v2 2023-10-01T09:04:28.810 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/img_import 2023-10-01T09:04:28.810 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/img_snap 2023-10-01T09:04:28.810 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/img_snap_import 2023-10-01T09:04:28.811 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap rm testimg_import@snap 2023-10-01T09:04:29.427 INFO:teuthology.orchestra.run.smithi138.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-01T09:04:29.427 INFO:teuthology.orchestra.run.smithi138.stdout:alertmanager.a smithi138 *:9093,9094 running (92s) 75s ago 2m 15.3M - 0.23.0 ba2b418f427c a3bfe0be9ee5 2023-10-01T09:04:29.428 INFO:teuthology.orchestra.run.smithi138.stdout:grafana.a smithi190 *:3000 running (111s) 7s ago 110s 55.4M - 8.3.5 dad864ee21e9 d186140d89cc 2023-10-01T09:04:29.428 INFO:teuthology.orchestra.run.smithi138.stdout:mds.foo.smithi138.brhzod smithi138 running (82s) 75s ago 81s 15.0M - 16.2.14-44-geeb13ac9 94a6c4498547 555d8c2cfe25 2023-10-01T09:04:29.428 INFO:teuthology.orchestra.run.smithi138.stdout:mds.foo.smithi190.weynjc smithi190 running (79s) 7s ago 79s 15.8M - 16.2.14-44-geeb13ac9 94a6c4498547 881fea906eff 2023-10-01T09:04:29.428 INFO:teuthology.orchestra.run.smithi138.stdout:mgr.x smithi190 *:8443 running (5m) 7s ago 5m 395M - 16.2.14-44-geeb13ac9 94a6c4498547 9629bd7ec45f 2023-10-01T09:04:29.428 INFO:teuthology.orchestra.run.smithi138.stdout:mgr.y smithi138 *:9283 running (7m) 75s ago 7m 445M - 16.2.14-44-geeb13ac9 94a6c4498547 5d4bcde0f5d4 2023-10-01T09:04:29.428 INFO:teuthology.orchestra.run.smithi138.stdout:mon.a smithi138 running (7m) 75s ago 7m 104M 2048M 16.2.14-44-geeb13ac9 94a6c4498547 476ba6abfb2c 2023-10-01T09:04:29.428 INFO:teuthology.orchestra.run.smithi138.stdout:mon.b smithi190 running (6m) 7s ago 6m 71.3M 2048M 16.2.14-44-geeb13ac9 94a6c4498547 520f260f067c 2023-10-01T09:04:29.428 INFO:teuthology.orchestra.run.smithi138.stdout:mon.c smithi138 running (6m) 75s ago 6m 101M 2048M 16.2.14-44-geeb13ac9 94a6c4498547 711c0763bf42 2023-10-01T09:04:29.428 INFO:teuthology.orchestra.run.smithi138.stdout:node-exporter.a smithi138 *:9100 running (2m) 75s ago 2m 9701k - 1.3.1 1dbe0e931976 6ec380bfea78 2023-10-01T09:04:29.428 INFO:teuthology.orchestra.run.smithi138.stdout:node-exporter.b smithi190 *:9100 running (2m) 7s ago 2m 24.4M - 1.3.1 1dbe0e931976 9a8d149372ce 2023-10-01T09:04:29.429 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0 smithi138 running (5m) 75s ago 5m 46.9M 4036M 16.2.14-44-geeb13ac9 94a6c4498547 7636a677953c 2023-10-01T09:04:29.429 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1 smithi138 running (5m) 75s ago 5m 49.6M 4036M 16.2.14-44-geeb13ac9 94a6c4498547 a510d44fb2c6 2023-10-01T09:04:29.429 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2 smithi138 running (4m) 75s ago 4m 42.5M 4036M 16.2.14-44-geeb13ac9 94a6c4498547 8a8389da7887 2023-10-01T09:04:29.429 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3 smithi138 running (4m) 75s ago 4m 45.6M 4036M 16.2.14-44-geeb13ac9 94a6c4498547 c3ba3a01539d 2023-10-01T09:04:29.429 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4 smithi190 running (4m) 7s ago 4m 51.4M 4292M 16.2.14-44-geeb13ac9 94a6c4498547 cda54a787614 2023-10-01T09:04:29.429 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5 smithi190 running (3m) 7s ago 3m 52.9M 4292M 16.2.14-44-geeb13ac9 94a6c4498547 4ac584c1fb7d 2023-10-01T09:04:29.429 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6 smithi190 running (3m) 7s ago 3m 48.9M 4292M 16.2.14-44-geeb13ac9 94a6c4498547 fa45a1c37077 2023-10-01T09:04:29.429 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7 smithi190 running (3m) 7s ago 3m 55.8M 4292M 16.2.14-44-geeb13ac9 94a6c4498547 c925b7fc3562 2023-10-01T09:04:29.429 INFO:teuthology.orchestra.run.smithi138.stdout:prometheus.a smithi190 *:9095 running (88s) 7s ago 2m 48.0M - 2.33.4 514e6a882f6e a1782aa730be 2023-10-01T09:04:29.555 INFO:tasks.workunit.client.0.smithi138.stderr: Removing snap: 100% complete...done. 2023-10-01T09:04:29.566 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg_import 2023-10-01T09:04:29.658 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:04:29.661 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap rm testimg@snap 2023-10-01T09:04:29.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:29 smithi190 conmon[100518]: cluster 2023-10-01T09:04:28.545858+0000 mon.a (mon.0) 2023-10-01T09:04:29.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:29 smithi190 conmon[100518]: 691 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-01T09:04:29.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:29 smithi138 conmon[109959]: cluster 2023-10-01T09:04:28.545858+0000 mon.a (mon 2023-10-01T09:04:29.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:29 smithi138 conmon[109959]: .0) 691 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-01T09:04:29.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:29 smithi138 conmon[100289]: cluster 2023-10-01T09:04:28.545858+0000 mon.a (mon.0 2023-10-01T09:04:29.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:29 smithi138 conmon[100289]: ) 691 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-01T09:04:30.200 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'echo wait for minority of mons to upgrade' 2023-10-01T09:04:30.562 INFO:tasks.workunit.client.0.smithi138.stderr: Removing snap: 100% complete...done. 2023-10-01T09:04:30.569 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm testimg 2023-10-01T09:04:30.670 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:04:30.673 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --order 20 /tmp/rbd_import_export_165459/img testimg 2023-10-01T09:04:30.727 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:04:30.728 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T09:04:30.730 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_165459/img_v2 2023-10-01T09:04:30.768 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T09:04:30.771 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_165459/img_v2 testimg_import 2023-10-01T09:04:30.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:30 smithi190 conmon[100518]: cluster 2023-10-01T09:04:28.862192+0000 mgr.y (mgr.14433) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:30.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:30 smithi190 conmon[100518]: audit 2023-10-01T09:04:29.419515+0000 mgr.y (mgr.14433) 123 : audit [DBG] from='client.24875 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:04:30.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:30 smithi190 conmon[100518]: cluster 2023-10-01T09:04:29.550086+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-01T09:04:30.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:30 smithi138 conmon[100289]: cluster 2023-10-01T09:04:28.862192+0000 mgr.y (mgr.14433) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:30.814 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:30 smithi138 conmon[100289]: audit 2023-10-01T09:04:29.419515+0000 mgr.y (mgr.14433) 123 : audit [DBG] from='client.24875 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:04:30.814 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:30 smithi138 conmon[100289]: cluster 2023-10-01T09:04:29.550086+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-01T09:04:30.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:30 smithi138 conmon[109959]: cluster 2023-10-01T09:04:28.862192+0000 mgr.y (mgr.14433) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:04:30.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:30 smithi138 conmon[109959]: audit 2023-10-01T09:04:29.419515+0000 mgr.y (mgr.14433) 123 : audit [DBG] from='client.24875 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:04:30.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:30 smithi138 conmon[109959]: cluster 2023-10-01T09:04:29.550086+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-01T09:04:30.842 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T09:04:30.845 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd info testimg_import 2023-10-01T09:04:30.845 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep order 2023-10-01T09:04:30.845 INFO:tasks.workunit.client.0.smithi138.stderr:+ awk '{print $2}' 2023-10-01T09:04:30.845 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep 20 2023-10-01T09:04:30.880 INFO:tasks.workunit.client.0.smithi138.stdout:20 2023-10-01T09:04:30.880 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/img_v2 2023-10-01T09:04:30.881 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg_import 2023-10-01T09:04:30.968 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:04:30.970 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg 2023-10-01T09:04:31.065 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:04:31.068 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_165459/img testimg 2023-10-01T09:04:31.116 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:04:31.120 INFO:tasks.workunit.client.0.smithi138.stderr:++ rbd info testimg 2023-10-01T09:04:31.121 INFO:tasks.workunit.client.0.smithi138.stderr:++ grep features 2023-10-01T09:04:31.158 INFO:tasks.workunit.client.0.smithi138.stderr:+ FEATURES_BEFORE=' features: layering 2023-10-01T09:04:31.158 INFO:tasks.workunit.client.0.smithi138.stderr: op_features: ' 2023-10-01T09:04:31.158 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_165459/img_v2 2023-10-01T09:04:31.193 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T09:04:31.196 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_165459/img_v2 testimg_import 2023-10-01T09:04:31.246 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T09:04:31.250 INFO:tasks.workunit.client.0.smithi138.stderr:++ rbd info testimg_import 2023-10-01T09:04:31.250 INFO:tasks.workunit.client.0.smithi138.stderr:++ grep features 2023-10-01T09:04:31.288 INFO:tasks.workunit.client.0.smithi138.stderr:+ FEATURES_AFTER=' features: layering 2023-10-01T09:04:31.288 INFO:tasks.workunit.client.0.smithi138.stderr: op_features: ' 2023-10-01T09:04:31.288 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' ' features: layering 2023-10-01T09:04:31.288 INFO:tasks.workunit.client.0.smithi138.stderr: op_features: ' '!=' ' features: layering 2023-10-01T09:04:31.288 INFO:tasks.workunit.client.0.smithi138.stderr: op_features: ' ']' 2023-10-01T09:04:31.288 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/img_v2 2023-10-01T09:04:31.289 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg_import 2023-10-01T09:04:31.356 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:04:31.358 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg 2023-10-01T09:04:31.425 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:04:31.428 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_165459/img testimg 2023-10-01T09:04:31.487 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:04:31.489 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_165459/img_v2 2023-10-01T09:04:31.529 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T09:04:31.531 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_165459/img_v2 testimg_import 2023-10-01T09:04:31.593 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T09:04:31.596 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd info testimg_import 2023-10-01T09:04:31.596 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep 'stripe unit' 2023-10-01T09:04:31.597 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep -Ei '(4 KiB|4096)' 2023-10-01T09:04:31.631 INFO:tasks.workunit.client.0.smithi138.stdout: stripe unit: 4 KiB 2023-10-01T09:04:31.631 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd info testimg_import 2023-10-01T09:04:31.631 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep 'stripe count' 2023-10-01T09:04:31.631 INFO:tasks.workunit.client.0.smithi138.stderr:+ awk '{print $3}' 2023-10-01T09:04:31.631 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep 1000 2023-10-01T09:04:31.665 INFO:tasks.workunit.client.0.smithi138.stdout:1000 2023-10-01T09:04:31.665 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/img_v2 2023-10-01T09:04:31.665 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg_import 2023-10-01T09:04:31.751 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-01T09:04:31.754 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg 2023-10-01T09:04:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:31 smithi138 conmon[109959]: cluster 2023-10-01T09:04:30.557012 2023-10-01T09:04:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:31 smithi138 conmon[109959]: +0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-01T09:04:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:31 smithi138 conmon[109959]: audit 2023-10-01T09:04:30.755164+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:31 smithi138 conmon[109959]: audit 2023-10-01T09:04:30.757853+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:04:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:31 smithi138 conmon[109959]: audit 2023-10-01T09:04:30.758647+0000 mon.c (mon.2) 100 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:04:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:31 smithi138 conmon[109959]: audit 2023-10-01T09:04:30.763923+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:31.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:31 smithi138 conmon[100289]: cluster 2023-10-01T09:04:30.557012+0000 2023-10-01T09:04:31.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:31 smithi138 conmon[100289]: mon.a (mon.0) 693 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-01T09:04:31.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:31 smithi138 conmon[100289]: audit 2023-10-01T09:04:30.755164+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:31.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:31 smithi138 conmon[100289]: audit 2023-10-01T09:04:30.757853+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:04:31.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:31 smithi138 conmon[100289]: audit 2023-10-01T09:04:30.758647+0000 mon.c (mon.2) 100 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:04:31.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:31 smithi138 conmon[100289]: audit 2023-10-01T09:04:30.763923+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:31.844 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-01T09:04:31.847 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_165459/img testimg 2023-10-01T09:04:31.899 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:04:31.902 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap create testimg@snap1 2023-10-01T09:04:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:31 smithi190 conmon[100518]: cluster 2023-10-01T09:04:30.557012+0000 mon.a (mon.0) 693 : 2023-10-01T09:04:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:31 smithi190 conmon[100518]: cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-01T09:04:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:31 smithi190 conmon[100518]: audit 2023-10-01T09:04:30.755164+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:31 smithi190 conmon[100518]: audit 2023-10-01T09:04:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:31 smithi190 conmon[100518]: 2023-10-01T09:04:30.757853+0000 mon.c (mon.2) 99 : audit 2023-10-01T09:04:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:31 smithi190 conmon[100518]: [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:04:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:31 smithi190 conmon[100518]: audit 2023 2023-10-01T09:04:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:31 smithi190 conmon[100518]: -10-01T09:04:30 2023-10-01T09:04:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:31 smithi190 conmon[100518]: .758647+0000 mon.c (mon.2) 2023-10-01T09:04:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:31 smithi190 conmon[100518]: 100 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:04:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:31 smithi190 conmon[100518]: audit 2023 2023-10-01T09:04:32.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:31 smithi190 conmon[100518]: -10-01T09:04: 2023-10-01T09:04:32.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:31 smithi190 conmon[100518]: 30.763923+0000 mon.a (mon.0 2023-10-01T09:04:32.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:31 smithi190 conmon[100518]: ) 695 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:04:32.309 INFO:teuthology.orchestra.run.smithi138.stdout:wait for minority of mons to upgrade 2023-10-01T09:04:32.778 INFO:tasks.workunit.client.0.smithi138.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-01T09:04:32.784 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap create testimg@snap2 2023-10-01T09:04:32.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:32 smithi138 conmon[109959]: cephadm 2023-10-01T09:04:30 2023-10-01T09:04:32.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:32 smithi138 conmon[109959]: .769064+0000 mgr.y (mgr.14433) 124 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T09:04:32.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:32 smithi138 conmon[109959]: cluster 2023-10-01T09:04:30.862682+0000 mgr.y (mgr.14433) 125 : cluster [DBG] pgmap v104: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 233 KiB/s rd, 25 KiB/s wr, 301 op/s 2023-10-01T09:04:32.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:32 smithi138 conmon[100289]: cephadm 2023-10-01T09:04:30.769064 2023-10-01T09:04:32.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:32 smithi138 conmon[100289]: +0000 mgr.y (mgr.14433) 124 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T09:04:32.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:32 smithi138 conmon[100289]: cluster 2023-10-01T09:04:30.862682+0000 mgr.y (mgr.14433) 125 : cluster [DBG] pgmap v104: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 233 KiB/s rd, 25 KiB/s wr, 301 op/s 2023-10-01T09:04:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:32 smithi190 conmon[100518]: cephadm 2023-10-01T09:04:30.769064+0000 mgr.y (mgr.14433) 124 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T09:04:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:32 smithi190 conmon[100518]: cluster 2023-10-01T09:04:30.862682+0000 mgr.y (mgr.14433) 125 : cluster [DBG] pgmap v104: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 233 KiB/s rd, 25 KiB/s wr, 301 op/s 2023-10-01T09:04:33.201 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-10-01T09:04:33.780 INFO:tasks.workunit.client.0.smithi138.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-01T09:04:33.787 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap protect testimg@snap2 2023-10-01T09:04:33.832 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_165459/snap_protect 2023-10-01T09:04:33.880 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T09:04:33.884 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_165459/snap_protect testimg_import 2023-10-01T09:04:34.041 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:33 smithi138 conmon[109959]: cluster 2023-10-01T09:04:32.768942+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-01T09:04:34.042 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:33 smithi138 conmon[109959]: cluster 2023-10-01T09:04:32.862967+0000 mgr.y (mgr.14433) 126 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 216 KiB/s rd, 24 KiB/s wr, 279 op/s 2023-10-01T09:04:34.042 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:33 smithi138 conmon[100289]: cluster 2023-10-01T09:04:32.768942+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-01T09:04:34.042 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:33 smithi138 conmon[100289]: cluster 2023-10-01T09:04:32.862967+0000 mgr.y (mgr.14433) 126 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 216 KiB/s rd, 24 KiB/s wr, 279 op/s 2023-10-01T09:04:34.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:33 smithi190 conmon[100518]: cluster 2023-10-01T09:04:32.768942+0000 mon.a (mon.0) 696 : cluster [DBG] 2023-10-01T09:04:34.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:33 smithi190 conmon[100518]: osdmap e70: 8 total, 8 up, 8 in 2023-10-01T09:04:34.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:33 smithi190 conmon[100518]: cluster 2023-10-01T09:04:32.862967+0000 mgr.y (mgr.14433) 126 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 216 KiB/s rd, 24 KiB/s wr, 279 op/s 2023-10-01T09:04:35.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:34 smithi190 conmon[100518]: cluster 2023-10-01T09:04:33.773268+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-01T09:04:35.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:34 smithi138 conmon[100289]: cluster 2023-10-01T 2023-10-01T09:04:35.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:34 smithi138 conmon[100289]: 09:04:33.773268+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-01T09:04:35.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:34 smithi138 conmon[109959]: cluster 2023-10-01T09:04:33. 2023-10-01T09:04:35.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:34 smithi138 conmon[109959]: 773268+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-01T09:04:35.547 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T09:04:35.550 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd info testimg_import@snap1 2023-10-01T09:04:35.550 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep 'protected: False' 2023-10-01T09:04:35.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:04:35] "GET /metrics HTTP/1.1" 200 191842 "" "Prometheus/2.33.4" 2023-10-01T09:04:35.590 INFO:tasks.workunit.client.0.smithi138.stdout: protected: False 2023-10-01T09:04:35.590 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd info testimg_import@snap2 2023-10-01T09:04:35.590 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep 'protected: True' 2023-10-01T09:04:35.629 INFO:tasks.workunit.client.0.smithi138.stdout: protected: True 2023-10-01T09:04:35.630 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/snap_protect 2023-10-01T09:04:35.630 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap unprotect testimg@snap2 2023-10-01T09:04:35.679 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-01T09:04:35.722 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap purge testimg 2023-10-01T09:04:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:35 smithi190 conmon[100518]: cluster 2023-10-01T09:04:34.778075+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-01T09:04:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:35 smithi190 conmon[100518]: cluster 2023-10-01T09:04:34.863316+0000 mgr.y (mgr.14433) 127 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 216 KiB/s rd, 24 KiB/s wr, 280 op/s 2023-10-01T09:04:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:35 smithi190 conmon[100518]: cluster 2023-10-01T09:04:35.522520+0000 mon.a (mon.0) 699 : cluster 2023-10-01T09:04:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:35 smithi190 conmon[100518]: [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-01T09:04:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:35 smithi190 conmon[100518]: audit 2023-10-01T09:04:35 2023-10-01T09:04:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:35 smithi190 conmon[100518]: .592584+0000 mon.a (mon.0) 700 : audit [DBG] from='client.? 172.21.15.138:0/2565869839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:36.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[109959]: cluster 2023-10-01T09:04:34. 2023-10-01T09:04:36.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[109959]: 778075+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-01T09:04:36.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[109959]: cluster 2023-10-01T09:04:34 2023-10-01T09:04:36.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[109959]: .863316+0000 mgr.y (mgr.14433) 127 : cluster [DBG] 2023-10-01T09:04:36.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[109959]: pgmap v109: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 216 KiB/s rd, 24 KiB/s wr, 280 op/s 2023-10-01T09:04:36.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[109959]: cluster 2023-10-01T09:04:35.522520 2023-10-01T09:04:36.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[109959]: +0000 mon.a (mon.0) 699 : cluster [DBG] 2023-10-01T09:04:36.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[109959]: osdmap e73: 8 total, 8 up, 8 in 2023-10-01T09:04:36.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[109959]: audit 2023-10-01T09:04:36.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[109959]: 2023-10-01T09:04:35.592584+0000 mon.a (mon.0 2023-10-01T09:04:36.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[109959]: ) 700 : audit [DBG] from='client.? 172.21.15.138:0/2565869839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:36.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[100289]: cluster 2023-10-01T09:04: 2023-10-01T09:04:36.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[100289]: 34.778075+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-01T09:04:36.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[100289]: cluster 2023-10-01T09:04:34.863316+0000 2023-10-01T09:04:36.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[100289]: mgr.y (mgr.14433) 127 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 216 KiB/s rd, 24 KiB/s wr, 280 op/s 2023-10-01T09:04:36.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[100289]: cluster 2023-10 2023-10-01T09:04:36.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[100289]: -01T09:04:35.522520+0000 mon.a (mon 2023-10-01T09:04:36.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[100289]: .0) 699 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-01T09:04:36.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[100289]: audit 2023-10-01T09:04:36.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[100289]: 2023-10-01T09:04 2023-10-01T09:04:36.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[100289]: :35.592584+0000 mon.a (mon.0) 700 : 2023-10-01T09:04:36.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:35 smithi138 conmon[100289]: audit [DBG] from='client.? 172.21.15.138:0/2565869839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:37.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:04:36 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:04:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:04:37.539 INFO:tasks.workunit.client.0.smithi138.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-01T09:04:37.546 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap purge testimg_import 2023-10-01T09:04:37.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:37 smithi190 conmon[100518]: cluster 2023-10-01T09:04:36.532930+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-01T09:04:37.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:37 smithi138 conmon[109959]: cluster 2023-10-01T09:04:36.532930 2023-10-01T09:04:37.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:37 smithi138 conmon[109959]: +0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-01T09:04:37.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:37 smithi138 conmon[100289]: cluster 2023-10-01T09:04:36.532930+0000 mon.a (mon 2023-10-01T09:04:37.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:37 smithi138 conmon[100289]: .0) 701 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-01T09:04:38.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:38 smithi190 conmon[100518]: cluster 2023-10-01T09:04:36.863572+0000 mgr.y (mgr.14433) 128 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 389 KiB/s rd, 41 KiB/s wr, 517 op/s 2023-10-01T09:04:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:38 smithi190 conmon[100518]: cluster 2023-10-01T09:04:37.534788+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-01T09:04:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:38 smithi190 conmon[100518]: audit 2023-10-01T09:04:37.992978+0000 mon.c (mon.2) 101 : audit 2023-10-01T09:04:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:38 smithi190 conmon[100518]: [DBG] from='client.? 172.21.15.138:0/1166343450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:38.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:38 smithi138 conmon[100289]: cluster 2023-10-01T09:04:36.863572+0000 mgr.y (mgr 2023-10-01T09:04:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:38 smithi138 conmon[100289]: .14433) 128 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 389 KiB/s rd, 41 KiB/s wr, 517 op/s 2023-10-01T09:04:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:38 smithi138 conmon[100289]: cluster 2023-10-01T09:04:37.534788+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-01T09:04:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:38 smithi138 conmon[100289]: audit 2023-10-01T09:04:37.992978+0000 2023-10-01T09:04:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:38 smithi138 conmon[100289]: mon.c (mon.2) 101 : audit [DBG] from='client.? 172.21.15.138:0/1166343450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:38 smithi138 conmon[109959]: cluster 2023-10-01T09:04:36. 2023-10-01T09:04:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:38 smithi138 conmon[109959]: 863572+0000 mgr.y (mgr.14433) 128 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 389 KiB/s rd, 41 KiB/s wr, 517 op/s 2023-10-01T09:04:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:38 smithi138 conmon[109959]: cluster 2023-10- 2023-10-01T09:04:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:38 smithi138 conmon[109959]: 01T09:04:37.534788+0000 mon.a (mon.0) 702 : cluster [DBG] 2023-10-01T09:04:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:38 smithi138 conmon[109959]: osdmap e75: 8 total, 8 up, 8 in 2023-10-01T09:04:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:38 smithi138 conmon[109959]: audit 2023-10-01T09:04:37 2023-10-01T09:04:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:38 smithi138 conmon[109959]: .992978+0000 mon.c (mon.2) 101 : audit 2023-10-01T09:04:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:38 smithi138 conmon[109959]: [DBG] from='client.? 172.21.15.138:0/1166343450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:39.548 INFO:tasks.workunit.client.0.smithi138.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-01T09:04:39.557 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg 2023-10-01T09:04:39.646 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:04:39.648 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg_import 2023-10-01T09:04:39.744 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:04:39.747 INFO:tasks.workunit.client.0.smithi138.stderr:+ tiered=0 2023-10-01T09:04:39.747 INFO:tasks.workunit.client.0.smithi138.stderr:+ ceph osd dump 2023-10-01T09:04:39.747 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep '^pool' 2023-10-01T09:04:39.748 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep ''\''rbd'\''' 2023-10-01T09:04:39.748 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep tier 2023-10-01T09:04:39.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:39 smithi190 conmon[100518]: cluster 2023-10-01T09:04:38.540961+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-01T09:04:39.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:39 smithi138 conmon[109959]: cluster 2023-10-01T09:04:38.540961+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-01T09:04:39.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:39 smithi138 conmon[100289]: cluster 2023-10-01T09:04:38.540961+0000 mon.a (mon 2023-10-01T09:04:39.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:39 smithi138 conmon[100289]: .0) 703 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-01T09:04:40.194 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_165459/sparse1 2023-10-01T09:04:40.200 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records in 2023-10-01T09:04:40.200 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records out 2023-10-01T09:04:40.200 INFO:tasks.workunit.client.0.smithi138.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.0049747 s, 211 MB/s 2023-10-01T09:04:40.200 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_165459/sparse2 2023-10-01T09:04:40.206 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records in 2023-10-01T09:04:40.206 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records out 2023-10-01T09:04:40.206 INFO:tasks.workunit.client.0.smithi138.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00486759 s, 215 MB/s 2023-10-01T09:04:40.206 INFO:tasks.workunit.client.0.smithi138.stderr:+ truncate /tmp/rbd_import_export_165459/sparse2 -s 2M 2023-10-01T09:04:40.238 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse1 2023-10-01T09:04:40.275 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 0% complete...failed. 2023-10-01T09:04:40.275 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: delete error: (2) No such file or directory 2023-10-01T09:04:40.278 INFO:tasks.workunit.client.0.smithi138.stderr:+ true 2023-10-01T09:04:40.279 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_165459/sparse1 2023-10-01T09:04:40.345 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-01T09:04:40.345 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:04:40.345 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T09:04:40.348 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd ls -l 2023-10-01T09:04:40.348 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep sparse1 2023-10-01T09:04:40.348 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-01T09:04:40.387 INFO:tasks.workunit.client.0.smithi138.stdout:sparse1 2 MiB 2 2023-10-01T09:04:40.388 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects sparse1 2023-10-01T09:04:40.388 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse1 2023-10-01T09:04:40.388 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse1 2023-10-01T09:04:40.388 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep block_name_prefix 2023-10-01T09:04:40.388 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk '{print $NF;}' 2023-10-01T09:04:40.424 INFO:tasks.workunit.client.0.smithi138.stderr:++ prefix=rbd_data.61487b78792 2023-10-01T09:04:40.425 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep rbd_data.61487b78792 2023-10-01T09:04:40.425 INFO:tasks.workunit.client.0.smithi138.stderr:++++ get_image_data_pool sparse1 2023-10-01T09:04:40.425 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sed -e 's/rbd_data.61487b78792\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-01T09:04:40.426 INFO:tasks.workunit.client.0.smithi138.stderr:++++ image=sparse1 2023-10-01T09:04:40.426 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sort -u 2023-10-01T09:04:40.426 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ rbd info sparse1 2023-10-01T09:04:40.426 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ awk -F: '{ print $NF }' 2023-10-01T09:04:40.426 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ grep 'data_pool: ' 2023-10-01T09:04:40.461 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool= 2023-10-01T09:04:40.461 INFO:tasks.workunit.client.0.smithi138.stderr:++++ '[' -z ']' 2023-10-01T09:04:40.461 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool=rbd 2023-10-01T09:04:40.461 INFO:tasks.workunit.client.0.smithi138.stderr:++++ echo rbd 2023-10-01T09:04:40.461 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rados ls -p rbd 2023-10-01T09:04:40.493 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects=1 2023-10-01T09:04:40.493 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo 1 2023-10-01T09:04:40.493 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-01T09:04:40.494 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export sparse1 /tmp/rbd_import_export_165459/sparse1.out 2023-10-01T09:04:40.533 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-01T09:04:40.536 INFO:tasks.workunit.client.0.smithi138.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_165459/sparse1 /tmp/rbd_import_export_165459/sparse1.out 2023-10-01T09:04:40.536 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp -l /tmp/rbd_import_export_165459/sparse1 /tmp/rbd_import_export_165459/sparse1.out 2023-10-01T09:04:40.538 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_165459/sparse1 --format %b 2023-10-01T09:04:40.538 INFO:tasks.workunit.client.0.smithi138.stderr:+ origsize=2048 2023-10-01T09:04:40.539 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_165459/sparse1.out --format %b 2023-10-01T09:04:40.540 INFO:tasks.workunit.client.0.smithi138.stderr:+ exportsize=2048 2023-10-01T09:04:40.540 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:04:40.540 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:04:40.540 INFO:tasks.workunit.client.0.smithi138.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T09:04:40.540 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/sparse1.out 2023-10-01T09:04:40.541 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse1 2023-10-01T09:04:40.635 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-01T09:04:40.638 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse2 2023-10-01T09:04:40.673 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 0% complete...failed. 2023-10-01T09:04:40.674 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: delete error: (2) No such file or directory 2023-10-01T09:04:40.676 INFO:tasks.workunit.client.0.smithi138.stderr:+ true 2023-10-01T09:04:40.676 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_165459/sparse2 2023-10-01T09:04:40.737 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-01T09:04:40.738 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:04:40.738 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T09:04:40.740 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd ls -l 2023-10-01T09:04:40.740 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep sparse2 2023-10-01T09:04:40.740 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-01T09:04:40.776 INFO:tasks.workunit.client.0.smithi138.stdout:sparse2 2 MiB 2 2023-10-01T09:04:40.777 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects sparse2 2023-10-01T09:04:40.777 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse2 2023-10-01T09:04:40.778 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse2 2023-10-01T09:04:40.778 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep block_name_prefix 2023-10-01T09:04:40.778 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk '{print $NF;}' 2023-10-01T09:04:40.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:40 smithi190 conmon[100518]: cluster 2023-10-01T09:04:38.863882+0000 mgr.y (mgr.14433 2023-10-01T09:04:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:40 smithi190 conmon[100518]: ) 129 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 389 KiB/s rd, 41 KiB/s wr, 517 op/s 2023-10-01T09:04:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:40 smithi190 conmon[100518]: cluster 2023-10-01T09:04:39.544362+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-01T09:04:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:40 smithi190 conmon[100518]: audit 2023-10-01T09:04:40.048390 2023-10-01T09:04:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:40 smithi190 conmon[100518]: +0000 mon.a (mon.0) 705 : audit [DBG] from='client.? 172.21.15.138:0/296118254' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:04:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:40 smithi190 conmon[100518]: audit 2023-10-01T09:04:40.182051+0000 mon.a (mon. 2023-10-01T09:04:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:40 smithi190 conmon[100518]: 0) 706 : audit [DBG] from='client.? 172.21.15.138:0/296118254' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:04:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:40 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:04:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:40 smithi190 conmon[100518]: 01T09:04:40.485975+0000 mon.b (mon.1) 27 : audit [DBG] from='client.? 172.21.15.138:0/2173048557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:40.812 INFO:tasks.workunit.client.0.smithi138.stderr:++ prefix=rbd_data.3ba9456101f6 2023-10-01T09:04:40.813 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep rbd_data.3ba9456101f6 2023-10-01T09:04:40.814 INFO:tasks.workunit.client.0.smithi138.stderr:++++ get_image_data_pool sparse2 2023-10-01T09:04:40.814 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sed -e 's/rbd_data.3ba9456101f6\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-01T09:04:40.814 INFO:tasks.workunit.client.0.smithi138.stderr:++++ image=sparse2 2023-10-01T09:04:40.814 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sort -u 2023-10-01T09:04:40.815 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ rbd info sparse2 2023-10-01T09:04:40.815 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ grep 'data_pool: ' 2023-10-01T09:04:40.815 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ awk -F: '{ print $NF }' 2023-10-01T09:04:40.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[100289]: cluster 2023-10-01T09:04:38.863882 2023-10-01T09:04:40.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[100289]: +0000 mgr.y (mgr.14433) 129 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 389 KiB/s rd, 41 KiB/s wr, 517 op/s 2023-10-01T09:04:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[100289]: cluster 2023-10-01T09:04:39.544362+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-01T09:04:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[100289]: audit 2023-10-01T09:04:40.048390+0000 mon.a (mon.0) 705 : audit [DBG] from='client.? 172.21.15.138:0/296118254' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:04:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[100289]: audit 2023-10-01T09:04:40.182051+0000 mon.a (mon. 2023-10-01T09:04:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[100289]: 0) 706 : audit [DBG] from='client.? 172.21.15.138:0/296118254' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:04:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:04:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[100289]: 10-01T09:04:40.485975+0000 mon.b 2023-10-01T09:04:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[100289]: (mon.1) 27 : audit [DBG] from='client.? 172.21.15.138:0/2173048557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[100289]: 2023-10-01T09:04:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: cluster 2023-10-01T09:04:38. 2023-10-01T09:04:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: 863882+0000 mgr.y (mgr.14433) 129 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 389 KiB/s rd, 41 KiB/s wr, 517 op/s 2023-10-01T09:04:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: cluster 2023-10-01T09:04:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: 2023-10-01T09:04:39.544362+0000 2023-10-01T09:04:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: mon.a (mon.0) 704 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-01T09:04:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:04:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: -01T09:04:40.048390+0000 mon.a (mon. 2023-10-01T09:04:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: 0) 705 : audit [DBG] from='client.? 172.21.15.138:0/296118254' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:04:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:04:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: 01T09:04:40.182051+0000 mon.a (mon. 2023-10-01T09:04:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: 0) 706 : audit [DBG] from='client.? 172.21.15.138:0/296118254' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:04:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: audit 2023-10-01T09:04:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: 2023-10-01T09:04:40.485975 2023-10-01T09:04:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: +0000 mon.b (mon.1) 27 : audit [DBG] 2023-10-01T09:04:40.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:40 smithi138 conmon[109959]: from='client.? 172.21.15.138:0/2173048557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:40.849 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool= 2023-10-01T09:04:40.850 INFO:tasks.workunit.client.0.smithi138.stderr:++++ '[' -z ']' 2023-10-01T09:04:40.850 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool=rbd 2023-10-01T09:04:40.850 INFO:tasks.workunit.client.0.smithi138.stderr:++++ echo rbd 2023-10-01T09:04:40.850 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rados ls -p rbd 2023-10-01T09:04:40.881 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects=0 2023-10-01T09:04:40.881 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo 0 2023-10-01T09:04:40.881 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-01T09:04:40.881 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export sparse2 /tmp/rbd_import_export_165459/sparse2.out 2023-10-01T09:04:40.918 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-01T09:04:40.921 INFO:tasks.workunit.client.0.smithi138.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_165459/sparse2 /tmp/rbd_import_export_165459/sparse2.out 2023-10-01T09:04:40.921 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp -l /tmp/rbd_import_export_165459/sparse2 /tmp/rbd_import_export_165459/sparse2.out 2023-10-01T09:04:40.923 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_165459/sparse2 --format %b 2023-10-01T09:04:40.924 INFO:tasks.workunit.client.0.smithi138.stderr:+ origsize=2048 2023-10-01T09:04:40.924 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_165459/sparse2.out --format %b 2023-10-01T09:04:40.925 INFO:tasks.workunit.client.0.smithi138.stderr:+ exportsize=2048 2023-10-01T09:04:40.925 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:04:40.925 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:04:40.925 INFO:tasks.workunit.client.0.smithi138.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T09:04:40.925 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/sparse2.out 2023-10-01T09:04:40.926 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse2 2023-10-01T09:04:41.015 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-01T09:04:41.018 INFO:tasks.workunit.client.0.smithi138.stderr:+ truncate /tmp/rbd_import_export_165459/sparse1 -s 10M 2023-10-01T09:04:41.019 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-01T09:04:41.106 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:04:41.106 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:04:41.106 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T09:04:41.107 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd ls -l 2023-10-01T09:04:41.107 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep sparse1 2023-10-01T09:04:41.107 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-10-01T09:04:41.143 INFO:tasks.workunit.client.0.smithi138.stdout:sparse1 10 MiB 2 2023-10-01T09:04:41.144 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects sparse1 2023-10-01T09:04:41.144 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse1 2023-10-01T09:04:41.144 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse1 2023-10-01T09:04:41.144 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep block_name_prefix 2023-10-01T09:04:41.144 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk '{print $NF;}' 2023-10-01T09:04:41.186 INFO:tasks.workunit.client.0.smithi138.stderr:++ prefix=rbd_data.3bd6d1254fb2 2023-10-01T09:04:41.187 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep rbd_data.3bd6d1254fb2 2023-10-01T09:04:41.187 INFO:tasks.workunit.client.0.smithi138.stderr:++++ get_image_data_pool sparse1 2023-10-01T09:04:41.187 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sed -e 's/rbd_data.3bd6d1254fb2\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-01T09:04:41.187 INFO:tasks.workunit.client.0.smithi138.stderr:++++ image=sparse1 2023-10-01T09:04:41.187 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sort -u 2023-10-01T09:04:41.187 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ rbd info sparse1 2023-10-01T09:04:41.187 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ grep 'data_pool: ' 2023-10-01T09:04:41.187 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ awk -F: '{ print $NF }' 2023-10-01T09:04:41.226 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool= 2023-10-01T09:04:41.226 INFO:tasks.workunit.client.0.smithi138.stderr:++++ '[' -z ']' 2023-10-01T09:04:41.226 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool=rbd 2023-10-01T09:04:41.227 INFO:tasks.workunit.client.0.smithi138.stderr:++++ echo rbd 2023-10-01T09:04:41.227 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rados ls -p rbd 2023-10-01T09:04:41.256 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects=1 2023-10-01T09:04:41.256 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo 1 2023-10-01T09:04:41.256 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-01T09:04:41.256 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export sparse1 /tmp/rbd_import_export_165459/sparse1.out 2023-10-01T09:04:41.303 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-01T09:04:41.308 INFO:tasks.workunit.client.0.smithi138.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_165459/sparse1 /tmp/rbd_import_export_165459/sparse1.out 2023-10-01T09:04:41.308 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp -l /tmp/rbd_import_export_165459/sparse1 /tmp/rbd_import_export_165459/sparse1.out 2023-10-01T09:04:41.316 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_165459/sparse1 --format %b 2023-10-01T09:04:41.317 INFO:tasks.workunit.client.0.smithi138.stderr:+ origsize=2048 2023-10-01T09:04:41.317 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_165459/sparse1.out --format %b 2023-10-01T09:04:41.318 INFO:tasks.workunit.client.0.smithi138.stderr:+ exportsize=2048 2023-10-01T09:04:41.318 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:04:41.318 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:04:41.318 INFO:tasks.workunit.client.0.smithi138.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T09:04:41.318 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/sparse1.out 2023-10-01T09:04:41.319 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse1 2023-10-01T09:04:41.406 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-01T09:04:41.409 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_165459/sparse2 oflag=append conv=notrunc 2023-10-01T09:04:41.420 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records in 2023-10-01T09:04:41.420 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records out 2023-10-01T09:04:41.420 INFO:tasks.workunit.client.0.smithi138.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00993087 s, 211 MB/s 2023-10-01T09:04:41.420 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-01T09:04:41.503 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:04:41.504 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:04:41.504 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T09:04:41.510 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd ls -l 2023-10-01T09:04:41.510 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep sparse2 2023-10-01T09:04:41.510 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-10-01T09:04:41.541 INFO:tasks.workunit.client.0.smithi138.stdout:sparse2 4 MiB 2 2023-10-01T09:04:41.542 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects sparse2 2023-10-01T09:04:41.542 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse2 2023-10-01T09:04:41.543 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse2 2023-10-01T09:04:41.543 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep block_name_prefix 2023-10-01T09:04:41.543 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk '{print $NF;}' 2023-10-01T09:04:41.579 INFO:tasks.workunit.client.0.smithi138.stderr:++ prefix=rbd_data.62b1dacbf0a9 2023-10-01T09:04:41.580 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep rbd_data.62b1dacbf0a9 2023-10-01T09:04:41.580 INFO:tasks.workunit.client.0.smithi138.stderr:++++ get_image_data_pool sparse2 2023-10-01T09:04:41.580 INFO:tasks.workunit.client.0.smithi138.stderr:++++ image=sparse2 2023-10-01T09:04:41.580 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sed -e 's/rbd_data.62b1dacbf0a9\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-01T09:04:41.580 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sort -u 2023-10-01T09:04:41.581 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ rbd info sparse2 2023-10-01T09:04:41.581 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ grep 'data_pool: ' 2023-10-01T09:04:41.581 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ awk -F: '{ print $NF }' 2023-10-01T09:04:41.626 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool= 2023-10-01T09:04:41.626 INFO:tasks.workunit.client.0.smithi138.stderr:++++ '[' -z ']' 2023-10-01T09:04:41.627 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool=rbd 2023-10-01T09:04:41.627 INFO:tasks.workunit.client.0.smithi138.stderr:++++ echo rbd 2023-10-01T09:04:41.627 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rados ls -p rbd 2023-10-01T09:04:41.664 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects='0 2023-10-01T09:04:41.665 INFO:tasks.workunit.client.0.smithi138.stderr:2 2023-10-01T09:04:41.665 INFO:tasks.workunit.client.0.smithi138.stderr:3' 2023-10-01T09:04:41.665 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo 0 2 3 2023-10-01T09:04:41.665 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-10-01T09:04:41.665 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export sparse2 /tmp/rbd_import_export_165459/sparse2.out 2023-10-01T09:04:41.703 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-01T09:04:41.706 INFO:tasks.workunit.client.0.smithi138.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_165459/sparse2 /tmp/rbd_import_export_165459/sparse2.out 2023-10-01T09:04:41.706 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp -l /tmp/rbd_import_export_165459/sparse2 /tmp/rbd_import_export_165459/sparse2.out 2023-10-01T09:04:41.709 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_165459/sparse2 --format %b 2023-10-01T09:04:41.710 INFO:tasks.workunit.client.0.smithi138.stderr:+ origsize=6144 2023-10-01T09:04:41.710 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_165459/sparse2.out --format %b 2023-10-01T09:04:41.711 INFO:tasks.workunit.client.0.smithi138.stderr:+ exportsize=6144 2023-10-01T09:04:41.711 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:04:41.712 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:04:41.712 INFO:tasks.workunit.client.0.smithi138.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T09:04:41.712 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/sparse2.out 2023-10-01T09:04:41.712 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse2 2023-10-01T09:04:41.803 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-01T09:04:41.806 INFO:tasks.workunit.client.0.smithi138.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-10-01T09:04:41.806 INFO:tasks.workunit.client.0.smithi138.stdout:partially-sparse file imports to partially-sparse image 2023-10-01T09:04:41.806 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_165459/sparse1 sparse 2023-10-01T09:04:41.875 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-01T09:04:41.875 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:04:41.876 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T09:04:41.878 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects sparse 2023-10-01T09:04:41.878 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse 2023-10-01T09:04:41.878 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse 2023-10-01T09:04:41.879 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep block_name_prefix 2023-10-01T09:04:41.879 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk '{print $NF;}' 2023-10-01T09:04:41.914 INFO:tasks.workunit.client.0.smithi138.stderr:++ prefix=rbd_data.3c3093b10f59 2023-10-01T09:04:41.914 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep rbd_data.3c3093b10f59 2023-10-01T09:04:41.914 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sed -e 's/rbd_data.3c3093b10f59\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-01T09:04:41.914 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sort -u 2023-10-01T09:04:41.914 INFO:tasks.workunit.client.0.smithi138.stderr:++++ get_image_data_pool sparse 2023-10-01T09:04:41.914 INFO:tasks.workunit.client.0.smithi138.stderr:++++ image=sparse 2023-10-01T09:04:41.915 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ rbd info sparse 2023-10-01T09:04:41.915 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ grep 'data_pool: ' 2023-10-01T09:04:41.915 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ awk -F: '{ print $NF }' 2023-10-01T09:04:41.950 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool= 2023-10-01T09:04:41.950 INFO:tasks.workunit.client.0.smithi138.stderr:++++ '[' -z ']' 2023-10-01T09:04:41.950 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool=rbd 2023-10-01T09:04:41.950 INFO:tasks.workunit.client.0.smithi138.stderr:++++ echo rbd 2023-10-01T09:04:41.950 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rados ls -p rbd 2023-10-01T09:04:41.982 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects=1 2023-10-01T09:04:41.982 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo 1 2023-10-01T09:04:41.983 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-01T09:04:41.983 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse 2023-10-01T09:04:42.072 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-01T09:04:42.075 INFO:tasks.workunit.client.0.smithi138.stdout:zeros import through stdin to sparse image 2023-10-01T09:04:42.076 INFO:tasks.workunit.client.0.smithi138.stderr:+ echo 'zeros import through stdin to sparse image' 2023-10-01T09:04:42.076 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-01T09:04:42.076 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format - sparse 2023-10-01T09:04:42.126 INFO:tasks.workunit.client.0.smithi138.stderr:4+0 records in 2023-10-01T09:04:42.126 INFO:tasks.workunit.client.0.smithi138.stderr:4+0 records out 2023-10-01T09:04:42.126 INFO:tasks.workunit.client.0.smithi138.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0497118 s, 84.4 MB/s 2023-10-01T09:04:42.137 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:04:42.138 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:04:42.140 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects sparse 2023-10-01T09:04:42.140 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse 2023-10-01T09:04:42.141 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse 2023-10-01T09:04:42.141 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep block_name_prefix 2023-10-01T09:04:42.141 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk '{print $NF;}' 2023-10-01T09:04:42.185 INFO:tasks.workunit.client.0.smithi138.stderr:++ prefix=rbd_data.3c51e424782 2023-10-01T09:04:42.186 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep rbd_data.3c51e424782 2023-10-01T09:04:42.186 INFO:tasks.workunit.client.0.smithi138.stderr:++++ get_image_data_pool sparse 2023-10-01T09:04:42.186 INFO:tasks.workunit.client.0.smithi138.stderr:++++ image=sparse 2023-10-01T09:04:42.186 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sed -e 's/rbd_data.3c51e424782\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-01T09:04:42.186 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sort -u 2023-10-01T09:04:42.186 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ rbd info sparse 2023-10-01T09:04:42.186 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ awk -F: '{ print $NF }' 2023-10-01T09:04:42.187 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ grep 'data_pool: ' 2023-10-01T09:04:42.223 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool= 2023-10-01T09:04:42.224 INFO:tasks.workunit.client.0.smithi138.stderr:++++ '[' -z ']' 2023-10-01T09:04:42.224 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool=rbd 2023-10-01T09:04:42.224 INFO:tasks.workunit.client.0.smithi138.stderr:++++ echo rbd 2023-10-01T09:04:42.224 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rados ls -p rbd 2023-10-01T09:04:42.254 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects= 2023-10-01T09:04:42.254 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo 2023-10-01T09:04:42.254 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-10-01T09:04:42.255 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse 2023-10-01T09:04:42.345 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:04:42.348 INFO:tasks.workunit.client.0.smithi138.stdout:zeros export to sparse file 2023-10-01T09:04:42.348 INFO:tasks.workunit.client.0.smithi138.stderr:+ echo 'zeros export to sparse file' 2023-10-01T09:04:42.348 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd create --new-format sparse --size 4 2023-10-01T09:04:42.385 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:04:42.387 INFO:tasks.workunit.client.0.smithi138.stderr:++ rbd info sparse 2023-10-01T09:04:42.387 INFO:tasks.workunit.client.0.smithi138.stderr:++ grep block_name_prefix 2023-10-01T09:04:42.387 INFO:tasks.workunit.client.0.smithi138.stderr:++ awk '{print $NF;}' 2023-10-01T09:04:42.422 INFO:tasks.workunit.client.0.smithi138.stderr:+ prefix=rbd_data.3c72ec45b336 2023-10-01T09:04:42.423 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-01T09:04:42.423 INFO:tasks.workunit.client.0.smithi138.stderr:++ get_image_data_pool sparse 2023-10-01T09:04:42.423 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse 2023-10-01T09:04:42.424 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse 2023-10-01T09:04:42.424 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep 'data_pool: ' 2023-10-01T09:04:42.424 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk -F: '{ print $NF }' 2023-10-01T09:04:42.458 INFO:tasks.workunit.client.0.smithi138.stderr:++ data_pool= 2023-10-01T09:04:42.458 INFO:tasks.workunit.client.0.smithi138.stderr:++ '[' -z ']' 2023-10-01T09:04:42.458 INFO:tasks.workunit.client.0.smithi138.stderr:++ data_pool=rbd 2023-10-01T09:04:42.458 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo rbd 2023-10-01T09:04:42.458 INFO:tasks.workunit.client.0.smithi138.stderr:+ rados -p rbd put rbd_data.3c72ec45b336.000000000000 - 2023-10-01T09:04:42.486 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records in 2023-10-01T09:04:42.487 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records out 2023-10-01T09:04:42.487 INFO:tasks.workunit.client.0.smithi138.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0633807 s, 66.2 MB/s 2023-10-01T09:04:42.526 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects sparse 2023-10-01T09:04:42.527 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse 2023-10-01T09:04:42.527 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse 2023-10-01T09:04:42.527 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep block_name_prefix 2023-10-01T09:04:42.527 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk '{print $NF;}' 2023-10-01T09:04:42.561 INFO:tasks.workunit.client.0.smithi138.stderr:++ prefix=rbd_data.3c72ec45b336 2023-10-01T09:04:42.562 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep rbd_data.3c72ec45b336 2023-10-01T09:04:42.562 INFO:tasks.workunit.client.0.smithi138.stderr:++++ get_image_data_pool sparse 2023-10-01T09:04:42.562 INFO:tasks.workunit.client.0.smithi138.stderr:++++ image=sparse 2023-10-01T09:04:42.562 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sed -e 's/rbd_data.3c72ec45b336\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-01T09:04:42.562 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sort -u 2023-10-01T09:04:42.564 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ rbd info sparse 2023-10-01T09:04:42.564 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ awk -F: '{ print $NF }' 2023-10-01T09:04:42.564 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ grep 'data_pool: ' 2023-10-01T09:04:42.597 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool= 2023-10-01T09:04:42.597 INFO:tasks.workunit.client.0.smithi138.stderr:++++ '[' -z ']' 2023-10-01T09:04:42.597 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool=rbd 2023-10-01T09:04:42.598 INFO:tasks.workunit.client.0.smithi138.stderr:++++ echo rbd 2023-10-01T09:04:42.598 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rados ls -p rbd 2023-10-01T09:04:42.629 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects=0 2023-10-01T09:04:42.630 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo 0 2023-10-01T09:04:42.630 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-01T09:04:42.630 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/sparse 2023-10-01T09:04:42.631 INFO:tasks.workunit.client.0.smithi138.stderr:rm: cannot remove '/tmp/rbd_import_export_165459/sparse': No such file or directory 2023-10-01T09:04:42.631 INFO:tasks.workunit.client.0.smithi138.stderr:+ true 2023-10-01T09:04:42.631 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export sparse /tmp/rbd_import_export_165459/sparse 2023-10-01T09:04:42.670 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete...done. 2023-10-01T09:04:42.673 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_165459/sparse --format=%b 2023-10-01T09:04:42.674 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 = 0 ']' 2023-10-01T09:04:42.674 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse 2023-10-01T09:04:42.759 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:04:42.761 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_165459/sparse /tmp/rbd_import_export_165459/sparse1 /tmp/rbd_import_export_165459/sparse2 /tmp/rbd_import_export_165459/sparse3 2023-10-01T09:04:42.763 INFO:tasks.workunit.client.0.smithi138.stderr:rm: cannot remove '/tmp/rbd_import_export_165459/sparse3': No such file or directory 2023-10-01T09:04:42.764 INFO:tasks.workunit.client.0.smithi138.stderr:+ true 2023-10-01T09:04:42.764 INFO:tasks.workunit.client.0.smithi138.stdout:OK 2023-10-01T09:04:42.765 INFO:tasks.workunit.client.0.smithi138.stderr:+ echo OK 2023-10-01T09:04:42.765 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm -rf /tmp/rbd_import_export_165459 2023-10-01T09:04:42.765 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-01T09:04:42.765 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-01T09:04:42.796 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-01T09:04:42.796 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-01T09:04:42.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:42 smithi190 conmon[100518]: cluster 2023-10-01T09:04:40.864381+0000 mgr.y (mgr.14433) 130 : 2023-10-01T09:04:42.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:42 smithi190 conmon[100518]: cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 484 KiB/s wr, 214 op/s 2023-10-01T09:04:42.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:42 smithi138 conmon[109959]: cluster 2023-10-01T09:04:40.864381+0000 mgr.y (mgr. 2023-10-01T09:04:42.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:42 smithi138 conmon[109959]: 14433) 130 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 484 KiB/s wr, 214 op/s 2023-10-01T09:04:42.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:42 smithi138 conmon[100289]: cluster 2023-10-01T09:04:40.864381+0000 mgr.y ( 2023-10-01T09:04:42.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:42 smithi138 conmon[100289]: mgr.14433) 130 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 484 KiB/s wr, 214 op/s 2023-10-01T09:04:43.062 DEBUG:teuthology.parallel:result is None 2023-10-01T09:04:43.062 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T09:04:43.089 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T09:04:43.089 DEBUG:teuthology.orchestra.run.smithi138:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-01T09:04:43.145 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T09:04:43.146 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-01T09:04:43.146 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-01T09:04:43.146 DEBUG:teuthology.parallel:result is None 2023-10-01T09:04:43.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:43 smithi190 conmon[100518]: audit 2023-10-01T09:04:43.043623+0000 mon.a (mon.0) 707 : audit [DBG] from='client.? 172.21.15.138:0/548247080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:43.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:43 smithi190 conmon[100518]: 2023-10-01T09:04:43.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:43 smithi138 conmon[109959]: audit 2023-10-01T09:04: 2023-10-01T09:04:43.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:43 smithi138 conmon[109959]: 43.043623+0000 mon.a (mon.0) 707 : audit [DBG] from='client.? 172.21.15.138:0/548247080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:43.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:43 smithi138 conmon[100289]: audit 2023-10-01T09:04:43.043623+0000 mon.a (mon 2023-10-01T09:04:43.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:43 smithi138 conmon[100289]: .0) 707 : audit [DBG] from='client.? 172.21.15.138:0/548247080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:44.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:44 smithi190 conmon[100518]: cluster 2023-10-01T09:04:42.864687+0000 mgr.y (mgr.14433) 131 : cluster [DBG] 2023-10-01T09:04:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:44 smithi190 conmon[100518]: pgmap v118: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 284 KiB/s rd, 349 KiB/s wr, 154 op/s 2023-10-01T09:04:44.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:44 smithi138 conmon[100289]: cluster 2023-10-01T09:04:42.864687+0000 2023-10-01T09:04:44.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:44 smithi138 conmon[100289]: mgr.y (mgr.14433) 131 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 284 KiB/s rd, 349 KiB/s wr, 154 op/s 2023-10-01T09:04:44.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:44 smithi138 conmon[109959]: cluster 2023-10-01T09:04:42.864687 2023-10-01T09:04:44.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:44 smithi138 conmon[109959]: +0000 mgr.y (mgr.14433) 131 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 284 KiB/s rd, 349 KiB/s wr, 154 op/s 2023-10-01T09:04:45.559 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:45 smithi138 conmon[100289]: audit 2023-10-01T09:04:45 2023-10-01T09:04:45.559 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:45 smithi138 conmon[100289]: .461276+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.138:0/4130577631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:45.559 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:45 smithi138 conmon[100289]: cluster 2023-10-01T09:04 2023-10-01T09:04:45.559 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:45 smithi138 conmon[100289]: :45.522808+0000 mon.a (mon.0) 708 : 2023-10-01T09:04:45.559 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:45 smithi138 conmon[100289]: cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-01T09:04:45.559 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:45 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:04:45] "GET /metrics HTTP/1.1" 200 192026 "" "Prometheus/2.33.4" 2023-10-01T09:04:45.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:45 smithi138 conmon[109959]: audit 2023-10-01T09:04:45. 2023-10-01T09:04:45.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:45 smithi138 conmon[109959]: 461276+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.138:0/4130577631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:45.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:45 smithi138 conmon[109959]: cluster 2023-10-01T09:04:45.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:45 smithi138 conmon[109959]: 2023-10-01T09:04:45.522808+0000 mon.a (mon.0) 708 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-01T09:04:45.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:45 smithi138 conmon[109959]: 2023-10-01T09:04:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:45 smithi190 conmon[100518]: audit 2023-10-01T09:04:45.461276+0000 mon.c (mon.2) 102 : audit [DBG] from='client.? 172.21.15.138:0/4130577631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:45 smithi190 conmon[100518]: cluster 2023-10- 2023-10-01T09:04:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:45 smithi190 conmon[100518]: 01T09:04:45.522808+0000 mon.a (mon.0) 708 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-10-01T09:04:46.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:46 smithi138 conmon[109959]: cluster 2023-10-01T09:04:44 2023-10-01T09:04:46.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:46 smithi138 conmon[109959]: .864968+0000 mgr.y (mgr.14433) 132 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 233 KiB/s rd, 286 KiB/s wr, 126 op/s 2023-10-01T09:04:46.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:46 smithi138 conmon[100289]: cluster 2023-10-01T09:04:44.864968+0000 mgr.y 2023-10-01T09:04:46.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:46 smithi138 conmon[100289]: (mgr.14433) 132 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 233 KiB/s rd, 286 KiB/s wr, 126 op/s 2023-10-01T09:04:46.843 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:46 smithi190 conmon[100518]: cluster 2023-10-01T09:04:44.864968+0000 mgr.y (mgr.14433) 132 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 233 KiB/s rd, 286 KiB/s wr, 126 op/s 2023-10-01T09:04:47.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:04:46 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:04:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:04:48.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:48 smithi138 conmon[100289]: cluster 2023-10-01T09:04:46.865441+0000 mgr.y (mgr.14433) 133 : 2023-10-01T09:04:48.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:48 smithi138 conmon[100289]: cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 344 op/s 2023-10-01T09:04:48.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:48 smithi138 conmon[100289]: audit 2023-10-01T09:04:47.865116+0000 mon.a (mon.0) 709 : audit [DBG] from='client.? 172.21.15.138:0/2667404394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:48 smithi138 conmon[109959]: cluster 2023-10-01T09:04:46.865441+0000 mgr.y 2023-10-01T09:04:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:48 smithi138 conmon[109959]: (mgr.14433) 133 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 344 op/s 2023-10-01T09:04:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:48 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:04:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:48 smithi138 conmon[109959]: 01T09:04:47.865116+0000 mon.a (mon.0) 709 : audit [DBG] 2023-10-01T09:04:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:48 smithi138 conmon[109959]: from='client.? 172.21.15.138:0/2667404394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:49.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:48 smithi190 conmon[100518]: cluster 2023-10-01T09:04:46.865441+0000 mgr.y (mgr.14433) 133 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 344 op/s 2023-10-01T09:04:49.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:48 smithi190 conmon[100518]: audit 2023-10-01T09:04:47.865116+0000 mon.a (mon.0) 709 : audit [DBG] from='client.? 172.21.15.138:0/2667404394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:50.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:50 smithi138 conmon[100289]: cluster 2023-10-01T09:04:48.865797+0000 2023-10-01T09:04:50.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:50 smithi138 conmon[100289]: mgr.y (mgr.14433) 134 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 881 KiB/s rd, 1.2 MiB/s wr, 296 op/s 2023-10-01T09:04:50.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:50 smithi138 conmon[100289]: audit 2023-10-01T09:04:50.248095+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.138:0/2118250338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:50 smithi138 conmon[109959]: cluster 2023-10-01T09:04: 2023-10-01T09:04:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:50 smithi138 conmon[109959]: 48.865797+0000 mgr.y (mgr.14433) 134 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 881 KiB/s rd, 1.2 MiB/s wr, 296 op/s 2023-10-01T09:04:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:50 smithi138 conmon[109959]: 2023-10-01T09:04:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:50 smithi138 conmon[109959]: audit 2023-10-01T09:04:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:50 smithi138 conmon[109959]: 2023-10-01T09: 2023-10-01T09:04:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:50 smithi138 conmon[109959]: 04:50.248095+0000 mon.c ( 2023-10-01T09:04:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:50 smithi138 conmon[109959]: mon.2) 103 : audit [DBG] 2023-10-01T09:04:50.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:50 smithi138 conmon[109959]: from='client.? 172.21.15.138:0/2118250338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:51.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:50 smithi190 conmon[100518]: cluster 2023-10-01T09:04:48.865797+0000 mgr.y (mgr.14433) 134 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 881 KiB/s rd, 1.2 MiB/s wr, 296 op/s 2023-10-01T09:04:51.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:50 smithi190 conmon[100518]: audit 2023-10-01T09:04:50.248095+0000 mon.c (mon.2) 103 : audit [DBG] from='client.? 172.21.15.138:0/2118250338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:52.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:52 smithi138 conmon[100289]: cluster 2023-10-01T09:04:50.866333+0000 mgr.y (mgr. 2023-10-01T09:04:52.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:52 smithi138 conmon[100289]: 14433) 135 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 649 KiB/s rd, 929 KiB/s wr, 182 op/s 2023-10-01T09:04:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:52 smithi138 conmon[109959]: cluster 2023-10-01T09:04:50. 2023-10-01T09:04:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:52 smithi138 conmon[109959]: 866333+0000 mgr.y (mgr.14433) 135 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 649 KiB/s rd, 929 KiB/s wr, 182 op/s 2023-10-01T09:04:53.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:52 smithi190 conmon[100518]: cluster 2023-10-01T09:04:50.866333+0000 mgr.y (mgr.14433) 135 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 649 KiB/s rd, 929 KiB/s wr, 182 op/s 2023-10-01T09:04:54.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:53 smithi190 conmon[100518]: audit 2023-10-01T09:04:52.672345+0000 mon.a (mon.0) 710 : audit [DBG] from='client.? 172.21.15.138:0/4103975258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:54.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:53 smithi138 conmon[100289]: audit 2023-10-01T09:04:52.672345+0000 2023-10-01T09:04:54.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:53 smithi138 conmon[100289]: mon.a (mon.0) 710 : audit [DBG] from='client.? 172.21.15.138:0/4103975258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:54.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:53 smithi138 conmon[109959]: audit 2023-10-01T09:04:52.672345+0000 mon.a ( 2023-10-01T09:04:54.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:53 smithi138 conmon[109959]: mon.0) 710 : audit [DBG] from='client.? 172.21.15.138:0/4103975258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:55.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:54 smithi190 conmon[100518]: cluster 2023-10-01T09:04:52.866657+0000 mgr.y (mgr.14433) 136 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 649 KiB/s rd, 929 KiB/s wr, 182 op/s 2023-10-01T09:04:55.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:54 smithi138 conmon[109959]: cluster 2023-10-01T09:04:52.866657+0000 mgr.y (mgr. 2023-10-01T09:04:55.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:54 smithi138 conmon[109959]: 14433) 136 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 649 KiB/s rd, 929 KiB/s wr, 182 op/s 2023-10-01T09:04:55.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:54 smithi138 conmon[100289]: cluster 2023-10-01T09:04:52.866657+0000 mgr.y (mgr.14433) 136 : cluster 2023-10-01T09:04:55.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:54 smithi138 conmon[100289]: [DBG] pgmap v124: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 649 KiB/s rd, 929 KiB/s wr, 182 op/s 2023-10-01T09:04:55.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:04:55 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:04:55] "GET /metrics HTTP/1.1" 200 192026 "" "Prometheus/2.33.4" 2023-10-01T09:04:56.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:55 smithi190 conmon[100518]: audit 2023-10-01T09:04:55.080303+0000 mon.a (mon.0) 711 : audit [DBG] from='client.? 172.21.15.138:0/983429421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:56.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:55 smithi138 conmon[109959]: audit 2023-10-01T09:04:55.080303+0000 2023-10-01T09:04:56.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:55 smithi138 conmon[109959]: mon.a (mon.0) 711 : audit [DBG] from='client.? 172.21.15.138:0/983429421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:56.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:55 smithi138 conmon[100289]: audit 2023-10-01T09:04:55.080303+0000 mon.a 2023-10-01T09:04:56.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:55 smithi138 conmon[100289]: (mon.0) 711 : audit [DBG] from='client.? 172.21.15.138:0/983429421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:56.842 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:56 smithi190 conmon[100518]: cluster 2023-10-01T09:04:54.867009+0000 mgr.y (mgr.14433) 137 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 649 KiB/s rd, 929 KiB/s wr, 182 op/s 2023-10-01T09:04:57.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:56 smithi138 conmon[109959]: cluster 2023-10-01T09:04 2023-10-01T09:04:57.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:56 smithi138 conmon[109959]: :54.867009+0000 mgr.y (mgr.14433) 137 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 649 KiB/s rd, 929 KiB/s wr, 182 op/s 2023-10-01T09:04:57.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:56 smithi138 conmon[100289]: cluster 2023-10-01T09:04:54.867009+0000 mgr.y (mgr.14433) 137 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 649 KiB/s rd, 929 KiB/s wr, 182 op/s 2023-10-01T09:04:57.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:04:56 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:04:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:04:58.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:57 smithi190 conmon[100518]: audit 2023-10-01T09:04:57.569280+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.138:0/3984481182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:58.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:57 smithi138 conmon[109959]: audit 2023-10-01T09:04:57.569280+0000 2023-10-01T09:04:58.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:57 smithi138 conmon[109959]: mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.138:0/3984481182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:58.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:57 smithi138 conmon[100289]: audit 2023-10-01T09:04:57.569280+0000 mon.c (mon.2) 104 : audit [DBG] from='client.? 172.21.15.138:0/3984481182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:04:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:04:58 smithi190 conmon[100518]: cluster 2023-10-01T09:04:56.867594+0000 mgr.y (mgr.14433) 138 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 573 KiB/s rd, 819 KiB/s wr, 161 op/s 2023-10-01T09:04:59.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:58 smithi138 conmon[109959]: cluster 2023-10-01T09:04:56.867594+0000 2023-10-01T09:04:59.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:04:58 smithi138 conmon[109959]: mgr.y (mgr.14433) 138 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 573 KiB/s rd, 819 KiB/s wr, 161 op/s 2023-10-01T09:04:59.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:58 smithi138 conmon[100289]: cluster 2023-10-01T09:04:56.867594+0000 mgr.y (mgr 2023-10-01T09:04:59.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:04:58 smithi138 conmon[100289]: .14433) 138 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 573 KiB/s rd, 819 KiB/s wr, 161 op/s 2023-10-01T09:05:01.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:00 smithi190 conmon[100518]: cluster 2023-10-01T09:04:58.867930+0000 mgr.y (mgr.14433) 139 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:01.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:00 smithi190 conmon[100518]: audit 2023-10-01T09:05:00.047499+0000 mon.a (mon.0) 712 : audit [DBG] from='client.? 172.21.15.138:0/2658430499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:01.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:00 smithi138 conmon[109959]: cluster 2023-10-01T09:04:58 2023-10-01T09:05:01.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:00 smithi138 conmon[109959]: .867930+0000 mgr.y (mgr.14433) 139 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:01.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:00 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:05:01.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:00 smithi138 conmon[109959]: -01T09:05:00.047499+0000 mon.a (mon.0) 712 : audit [DBG] from='client.? 172.21.15.138:0/2658430499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:01.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:00 smithi138 conmon[100289]: cluster 2023-10-01T09:04:58. 2023-10-01T09:05:01.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:00 smithi138 conmon[100289]: 867930+0000 mgr.y (mgr.14433) 139 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:01.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:00 smithi138 conmon[100289]: audit 2023-10-01T09:05:00.047499+0000 mon.a (mon.0) 712 : audit [DBG] from='client.? 172.21.15.138:0/2658430499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:02 smithi190 conmon[100518]: cluster 2023-10-01T09:05:00.868370+0000 mgr.y (mgr.14433) 140 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:02 smithi190 conmon[100518]: audit 2023-10-01T09:05:02.423995+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.138:0/922439174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:03.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:02 smithi138 conmon[100289]: cluster 2023-10-01T09:05:00.868370+0000 mgr.y (mgr.14433) 140 : 2023-10-01T09:05:03.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:02 smithi138 conmon[100289]: cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:03.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:02 smithi138 conmon[100289]: audit 2023-10-01T09:05:02.423995+0000 mon.c (mon.2) 105 : audit [DBG] from='client.? 172.21.15.138:0/922439174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:02 smithi138 conmon[109959]: cluster 2023-10-01T09:05:00. 2023-10-01T09:05:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:02 smithi138 conmon[109959]: 868370+0000 mgr.y (mgr.14433) 140 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:02 smithi138 conmon[109959]: audit 2023-10-01T09:05:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:02 smithi138 conmon[109959]: 2023-10-01T09:05:02.423995+0000 mon.c (mon. 2023-10-01T09:05:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:02 smithi138 conmon[109959]: 2) 105 : audit [DBG] from='client.? 172.21.15.138:0/922439174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:05.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:04 smithi190 conmon[100518]: cluster 2023-10-01T09:05:02.868700+0000 mgr.y (mgr.14433) 141 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:05.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:04 smithi138 conmon[100289]: cluster 2023-10-01T09:05:02.868700+0000 mgr.y (mgr 2023-10-01T09:05:05.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:04 smithi138 conmon[100289]: .14433) 141 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:05.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:04 smithi138 conmon[109959]: cluster 2023-10-01T09:05:02. 2023-10-01T09:05:05.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:04 smithi138 conmon[109959]: 868700+0000 mgr.y (mgr.14433) 141 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:05.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:05:05 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:05:05] "GET /metrics HTTP/1.1" 200 192069 "" "Prometheus/2.33.4" 2023-10-01T09:05:06.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:05 smithi190 conmon[100518]: audit 2023-10-01T09:05:04.779304+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.138:0/3242201359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:06.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:05 smithi138 conmon[100289]: audit 2023-10-01T09:05:04.779304+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.138:0/3242201359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:06.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:05 smithi138 conmon[109959]: audit 2023-10-01T09:05:04.779304+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.138:0/3242201359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:06.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:05 smithi138 conmon[109959]: 2023-10-01T09:05:07.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:06 smithi190 conmon[100518]: cluster 2023-10-01T09:05:04.869106+0000 mgr.y (mgr.14433) 142 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:07.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:05:06 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:05:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:05:07.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:06 smithi138 conmon[100289]: cluster 2023-10-01T09:05:04.869106+0000 mgr.y (mgr.14433) 142 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:07.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:06 smithi138 conmon[109959]: cluster 2023-10-01T09:05:04.869106+0000 mgr.y (mgr.14433) 142 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:08.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:07 smithi190 conmon[100518]: audit 2023-10-01T09:05:07.129890+0000 mon.a (mon.0) 713 : audit [DBG] from='client.? 172.21.15.138:0/312641023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:08.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:07 smithi138 conmon[100289]: audit 2023-10-01T09:05:07.129890+0000 mon.a (mon.0) 713 : audit [DBG] from='client.? 172.21.15.138:0/312641023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:08.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:07 smithi138 conmon[109959]: audit 2023-10-01T09:05:07.129890+0000 mon.a (mon.0) 713 : audit [DBG] from='client.? 172.21.15.138:0/312641023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:09.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:08 smithi190 conmon[100518]: cluster 2023-10-01T09:05:06.869626+0000 mgr.y (mgr.14433) 143 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:09.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:08 smithi138 conmon[109959]: cluster 2023-10-01T09:05:06.869626+0000 mgr.y (mgr.14433) 143 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:09.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:08 smithi138 conmon[100289]: cluster 2023-10-01T09:05:06.869626+0000 mgr.y (mgr.14433) 143 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:10.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:09 smithi190 conmon[100518]: audit 2023-10-01T09:05:09.476414+0000 mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.138:0/968804538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:10.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:09 smithi138 conmon[109959]: audit 2023-10-01T09:05:09.476414+0000 2023-10-01T09:05:10.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:09 smithi138 conmon[109959]: mon.c (mon.2) 107 : audit [DBG] from='client.? 172.21.15.138:0/968804538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:10.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:09 smithi138 conmon[100289]: audit 2023-10-01T09:05:09.476414+0000 mon.c (mon. 2023-10-01T09:05:10.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:09 smithi138 conmon[100289]: 2) 107 : audit [DBG] from='client.? 172.21.15.138:0/968804538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:11.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:10 smithi190 conmon[100518]: cluster 2023-10-01T09:05:08.870093+0000 mgr.y (mgr.14433) 144 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:11.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:10 smithi138 conmon[109959]: cluster 2023-10-01T09:05:08.870093+0000 mgr.y ( 2023-10-01T09:05:11.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:10 smithi138 conmon[109959]: mgr.14433) 144 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:11.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:10 smithi138 conmon[100289]: cluster 2023-10-01T09:05:08.870093+0000 mgr.y (mgr.14433) 144 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:12.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:11 smithi190 conmon[100518]: cluster 2023-10-01T09:05:10.870632+0000 mgr.y (mgr.14433) 145 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:12.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:11 smithi138 conmon[109959]: cluster 2023-10-01T09:05:10.870632+0000 mgr.y 2023-10-01T09:05:12.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:11 smithi138 conmon[109959]: (mgr.14433) 145 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:12.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:11 smithi138 conmon[100289]: cluster 2023-10-01T09:05:10.870632 2023-10-01T09:05:12.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:11 smithi138 conmon[100289]: +0000 mgr.y (mgr.14433) 145 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:13.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:12 smithi190 conmon[100518]: audit 2023-10-01T09:05:11.835593+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.138:0/2948849421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:13.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:12 smithi138 conmon[100289]: audit 2023-10-01T09:05:11.835593+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.138:0/2948849421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:13.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:12 smithi138 conmon[109959]: audit 2023-10-01T09:05:11.835593+0000 mon.c (mon.2) 108 2023-10-01T09:05:13.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:12 smithi138 conmon[109959]: : audit [DBG] from='client.? 172.21.15.138:0/2948849421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:13 smithi190 conmon[100518]: cluster 2023-10-01T09:05:12.871070+0000 mgr.y (mgr.14433) 146 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:14.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:13 smithi138 conmon[109959]: cluster 2023-10-01T09:05:12.871070+0000 mgr.y (mgr.14433) 146 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:14.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:13 smithi138 conmon[100289]: cluster 2023-10-01T09:05:12.871070+0000 mgr.y (mgr.14433) 146 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:14.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:13 smithi138 conmon[100289]: 2023-10-01T09:05:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:14 smithi190 conmon[100518]: audit 2023-10-01T09:05:14.187287+0000 mon.a (mon.0) 714 : audit [DBG] from='client.? 172.21.15.138:0/4253317497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:15.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:14 smithi138 conmon[109959]: audit 2023-10-01T09:05:14.187287+0000 mon.a (mon.0) 714 : audit [DBG] from='client.? 172.21.15.138:0/4253317497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:15.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:14 smithi138 conmon[100289]: audit 2023-10-01T09:05:14.187287+0000 mon.a ( 2023-10-01T09:05:15.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:14 smithi138 conmon[100289]: mon.0) 714 : audit [DBG] from='client.? 172.21.15.138:0/4253317497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:15.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:05:15 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:05:15] "GET /metrics HTTP/1.1" 200 192062 "" "Prometheus/2.33.4" 2023-10-01T09:05:16.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:16 smithi190 conmon[100518]: cluster 2023-10-01T09:05:14.871488+0000 mgr.y (mgr.14433) 147 2023-10-01T09:05:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:16 smithi190 conmon[100518]: : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:16 smithi190 conmon[100518]: audit 2023-10-01T09:05:15.757262+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:05:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:16 smithi190 conmon[100518]: audit 2023-10-01T09:05:15.758824+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:05:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:16 smithi190 conmon[100518]: audit 2023-10-01T09:05:15.763432+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:05:16.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[100289]: cluster 2023-10-01T09:05:14.871488+0000 mgr.y 2023-10-01T09:05:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[100289]: (mgr.14433) 147 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[100289]: audit 2023-10-01T09:05:15.757262+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:05:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[100289]: audit 2023-10-01T09:05:15.758824+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:05:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[100289]: audit 2023-10-01T09:05:15.763432+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:05:16.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: cluster 2023-10-01T09:05 2023-10-01T09:05:16.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: :14.871488+0000 mgr.y (mgr.14433) 147 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:16.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: audit 2023-10-01T09:05:16.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: 2023-10-01T09:05: 2023-10-01T09:05:16.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: 15.757262+0000 mon.a (mon 2023-10-01T09:05:16.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: .0) 715 : audit [INF] 2023-10-01T09:05:16.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:05:16.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: audit 2023-10-01T09:05:16.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: 2023-10-01T09 2023-10-01T09:05:16.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: :05:15.758824+0000 2023-10-01T09:05:16.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: mon.c (mon.2) 109 : audit 2023-10-01T09:05:16.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:05:16.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:05:16.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: -01T09:05:15.763432 2023-10-01T09:05:16.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: +0000 mon.a (mon.0) 716 2023-10-01T09:05:16.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:16 smithi138 conmon[109959]: : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:05:17.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:05:16 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:05:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:05:17.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:17 smithi190 conmon[100518]: cephadm 2023-10-01T09:05:15.757884+0000 mgr.y (mgr. 2023-10-01T09:05:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:17 smithi190 conmon[100518]: 14433) 148 : cephadm [INF] Upgrade: Target is version 18.2.0-497-gb5682219 (unknown) 2023-10-01T09:05:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:17 smithi190 conmon[100518]: cephadm 2023-10-01T09:05:15.757956+0000 mgr.y (mgr.14433) 149 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf, digests ['quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf'] 2023-10-01T09:05:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:17 smithi190 conmon[100518]: cephadm 2023-10-01T09:05:15.775585+0000 mgr.y (mgr. 2023-10-01T09:05:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:17 smithi190 conmon[100518]: 14433) 150 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-01T09:05:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:17 smithi190 conmon[100518]: cephadm 2023-10 2023-10-01T09:05:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:17 smithi190 conmon[100518]: -01T09:05:16.063178+0000 mgr.y (mgr 2023-10-01T09:05:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:17 smithi190 conmon[100518]: .14433) 151 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf on smithi190 2023-10-01T09:05:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:17 smithi190 conmon[100518]: audit 2023-10-01T09:05:16.543031+0000 mon.c (mon.2 2023-10-01T09:05:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:17 smithi190 conmon[100518]: ) 110 : audit [DBG] from='client.? 172.21.15.138:0/1822704481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:17.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[100289]: cephadm 2023-10-01T09:05:15.757884+0000 mgr.y (mgr.14433) 148 : cephadm 2023-10-01T09:05:17.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[100289]: [INF] Upgrade: Target is version 18.2.0-497-gb5682219 (unknown) 2023-10-01T09:05:17.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[100289]: cephadm 2023-10-01T09:05:15.757956+0000 mgr.y (mgr.14433) 149 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf, digests ['quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf'] 2023-10-01T09:05:17.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[100289]: cephadm 2023-10-01T09:05:15.775585+0000 2023-10-01T09:05:17.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[100289]: mgr.y (mgr.14433) 150 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-01T09:05:17.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[100289]: cephadm 2023-10-01T09:05:16.063178+0000 2023-10-01T09:05:17.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[100289]: mgr.y (mgr.14433) 151 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf on smithi190 2023-10-01T09:05:17.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[100289]: audit 2023 2023-10-01T09:05:17.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[100289]: -10-01T09:05:16.543031+0000 mon.c (mon.2) 110 : audit 2023-10-01T09:05:17.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[100289]: [DBG] from='client.? 172.21.15.138:0/1822704481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[109959]: cephadm 2023-10-01T09:05:15.757884+0000 mgr.y (mgr. 2023-10-01T09:05:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[109959]: 14433) 148 : cephadm [INF] Upgrade: Target is version 18.2.0-497-gb5682219 (unknown) 2023-10-01T09:05:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[109959]: cephadm 2023-10-01T09:05:15.757956+0000 mgr.y (mgr.14433) 149 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf, digests ['quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf'] 2023-10-01T09:05:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[109959]: 2023-10-01T09:05:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[109959]: cephadm 2023-10-01T09:05:15.775585+0000 mgr.y (mgr.14433) 150 : cephadm [INF] 2023-10-01T09:05:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[109959]: Upgrade: Need to upgrade myself (mgr.y) 2023-10-01T09:05:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[109959]: cephadm 2023-10-01T09:05:16.063178 2023-10-01T09:05:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[109959]: +0000 mgr.y (mgr.14433) 151 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf on smithi190 2023-10-01T09:05:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:05:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[109959]: 05:16.543031+0000 mon.c (mon.2) 110 : audit 2023-10-01T09:05:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:17 smithi138 conmon[109959]: [DBG] from='client.? 172.21.15.138:0/1822704481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:18.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:18 smithi190 conmon[100518]: cluster 2023-10-01T09:05:16.872060+0000 mgr.y (mgr.14433) 2023-10-01T09:05:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:18 smithi190 conmon[100518]: 152 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:18.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:18 smithi138 conmon[100289]: cluster 2023-10-01T09:05:16.872060+0000 mgr.y (mgr.14433) 152 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:18.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:18 smithi138 conmon[109959]: cluster 2023-10-01T09:05:16.872060+0000 mgr.y (mgr.14433) 152 : cluster 2023-10-01T09:05:18.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:18 smithi138 conmon[109959]: [DBG] pgmap v136: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:19.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:19 smithi190 conmon[100518]: audit 2023-10-01T09:05:18.893418+0000 mon.c (mon.2) 111 : audit 2023-10-01T09:05:19.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:19 smithi190 conmon[100518]: [DBG] from='client.? 172.21.15.138:0/797816610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:19.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:19 smithi138 conmon[109959]: audit 2023-10-01T09:05:18.893418+0000 2023-10-01T09:05:19.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:19 smithi138 conmon[109959]: mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.138:0/797816610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:19.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:19 smithi138 conmon[100289]: audit 2023-10-01T09:05:18.893418+0000 mon.c (mon. 2023-10-01T09:05:19.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:19 smithi138 conmon[100289]: 2) 111 : audit [DBG] from='client.? 172.21.15.138:0/797816610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:20.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:20 smithi190 conmon[100518]: cluster 2023-10-01T09:05:18.872445+0000 mgr.y (mgr.14433) 153 : 2023-10-01T09:05:20.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:20 smithi190 conmon[100518]: cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:20.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:20 smithi138 conmon[109959]: cluster 2023-10-01T09:05:18.872445+0000 mgr.y (mgr.14433) 153 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:20.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:20 smithi138 conmon[100289]: cluster 2023-10-01T09:05:18.872445+0000 mgr.y (mgr. 2023-10-01T09:05:20.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:20 smithi138 conmon[100289]: 14433) 153 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:21.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:21 smithi190 conmon[100518]: audit 2023-10-01T09:05:21.241320+0000 mon.c (mon.2) 2023-10-01T09:05:21.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:21 smithi190 conmon[100518]: 112 : audit [DBG] from='client.? 172.21.15.138:0/93037957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:21 smithi138 conmon[109959]: audit 2023-10-01T09:05:21 2023-10-01T09:05:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:21 smithi138 conmon[109959]: .241320+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.138:0/93037957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:21.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:21 smithi138 conmon[100289]: audit 2023-10-01T09:05:21.241320 2023-10-01T09:05:21.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:21 smithi138 conmon[100289]: +0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.138:0/93037957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:22.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:22 smithi190 conmon[100518]: cluster 2023-10-01T09:05:20.872958+0000 mgr.y (mgr.14433) 2023-10-01T09:05:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:22 smithi190 conmon[100518]: 154 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:22.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:22 smithi138 conmon[109959]: cluster 2023-10-01T09:05:20.872958+0000 mgr.y (mgr.14433) 154 : 2023-10-01T09:05:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:22 smithi138 conmon[109959]: cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:22 smithi138 conmon[100289]: cluster 2023-10-01T09:05:20.872958+0000 mgr.y (mgr.14433) 154 : cluster [DBG] 2023-10-01T09:05:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:22 smithi138 conmon[100289]: pgmap v138: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:24 smithi190 conmon[100518]: cluster 2023-10-01T09:05:22.873366+0000 mgr.y (mgr.14433 2023-10-01T09:05:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:24 smithi190 conmon[100518]: ) 155 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:24 smithi190 conmon[100518]: audit 2023-10-01T09:05:23.593815+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.138:0/3306049082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:24.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:24 smithi138 conmon[100289]: cluster 2023-10-01T09:05:22.873366+0000 mgr.y (mgr.14433) 155 : cluster [DBG] 2023-10-01T09:05:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:24 smithi138 conmon[100289]: pgmap v139: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:24 smithi138 conmon[100289]: audit 2023-10-01T09:05:23.593815+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.138:0/3306049082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:24.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:24 smithi138 conmon[109959]: cluster 2023-10-01T09:05:22.873366+0000 mgr.y (mgr.14433 2023-10-01T09:05:24.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:24 smithi138 conmon[109959]: ) 155 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:24.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:24 smithi138 conmon[109959]: audit 2023-10-01T09:05:23.593815+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.138:0/3306049082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:25.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:05:25 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:05:25] "GET /metrics HTTP/1.1" 200 192062 "" "Prometheus/2.33.4" 2023-10-01T09:05:26.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:26 smithi190 conmon[100518]: cluster 2023-10-01T09:05:24.873787+0000 mgr.y (mgr.14433) 156 2023-10-01T09:05:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:26 smithi190 conmon[100518]: : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:26 smithi190 conmon[100518]: audit 2023-10-01T09:05:25.944627+0000 mon.a (mon.0) 717 : audit [DBG] from='client.? 172.21.15.138:0/2300527982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:26.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:26 smithi138 conmon[100289]: cluster 2023-10-01T09:05:24.873787+0000 mgr.y (mgr 2023-10-01T09:05:26.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:26 smithi138 conmon[100289]: .14433) 156 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:26.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:26 smithi138 conmon[100289]: audit 2023-10-01T09:05:25.944627+0000 mon.a (mon.0) 717 : audit [DBG] 2023-10-01T09:05:26.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:26 smithi138 conmon[100289]: from='client.? 172.21.15.138:0/2300527982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:26 smithi138 conmon[109959]: cluster 2023-10-01T09:05:24.873787+0000 2023-10-01T09:05:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:26 smithi138 conmon[109959]: mgr.y (mgr.14433) 156 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:26 smithi138 conmon[109959]: audit 2023-10-01T09:05: 2023-10-01T09:05:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:26 smithi138 conmon[109959]: 25.944627+0000 mon.a (mon.0) 717 : audit [DBG] from='client.? 172.21.15.138:0/2300527982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:27.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:05:26 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:05:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:05:28.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:28 smithi138 conmon[100289]: cluster 2023-10-01T09:05:26.874566+0000 mgr.y (mgr.14433 2023-10-01T09:05:28.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:28 smithi138 conmon[100289]: ) 157 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:28.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:28 smithi138 conmon[100289]: audit 2023-10-01T09:05:28.293785+0000 mon.a (mon.0) 718 : audit [DBG] from='client.? 172.21.15.138:0/2203192348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:28.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:28 smithi138 conmon[109959]: cluster 2023-10-01T09:05:26.874566+0000 mgr.y (mgr.14433) 2023-10-01T09:05:28.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:28 smithi138 conmon[109959]: 157 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:28.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:28 smithi138 conmon[109959]: audit 2023-10-01T09:05:28.293785+0000 mon.a (mon.0) 718 : audit [DBG] from='client.? 172.21.15.138:0/2203192348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:29.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:28 smithi190 conmon[100518]: cluster 2023-10-01T09:05:26.874566+0000 2023-10-01T09:05:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:28 smithi190 conmon[100518]: mgr.y (mgr.14433) 157 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:28 smithi190 conmon[100518]: audit 2023-10-01T09:05:28.293785+0000 mon.a 2023-10-01T09:05:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:28 smithi190 conmon[100518]: (mon.0) 718 : audit [DBG] from='client.? 172.21.15.138:0/2203192348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:30.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:30 smithi138 conmon[109959]: cluster 2023-10-01T09:05: 2023-10-01T09:05:30.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:30 smithi138 conmon[109959]: 28.874932+0000 mgr.y (mgr.14433) 158 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:30.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:30 smithi138 conmon[100289]: cluster 2023-10-01T09:05:28.874932+0000 mgr.y (mgr.14433) 158 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:30.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:30 smithi138 conmon[100289]: 2023-10-01T09:05:31.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:30 smithi190 conmon[100518]: cluster 2023-10-01T09:05:28.874932+0000 mgr.y (mgr.14433) 158 2023-10-01T09:05:31.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:30 smithi190 conmon[100518]: : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:31 smithi138 conmon[109959]: audit 2023-10-01T09:05:30.643845+0000 mon.c (mon.2) 2023-10-01T09:05:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:31 smithi138 conmon[109959]: 114 : audit [DBG] from='client.? 172.21.15.138:0/872954066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:31.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:31 smithi138 conmon[100289]: audit 2023-10-01T09:05:30.643845+0000 mon.c (mon.2) 114 : 2023-10-01T09:05:31.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:31 smithi138 conmon[100289]: audit [DBG] from='client.? 172.21.15.138:0/872954066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:32.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:31 smithi190 conmon[100518]: audit 2023-10-01T09:05:30.643845+0000 mon.c (mon.2 2023-10-01T09:05:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:31 smithi190 conmon[100518]: ) 114 : audit [DBG] from='client.? 172.21.15.138:0/872954066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:32.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:32 smithi138 conmon[109959]: cluster 2023-10-01T09:05:30.875501+0000 mgr.y (mgr.14433 2023-10-01T09:05:32.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:32 smithi138 conmon[109959]: ) 159 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:32.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:32 smithi138 conmon[100289]: cluster 2023-10-01T09:05:30.875501+0000 mgr.y (mgr.14433 2023-10-01T09:05:32.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:32 smithi138 conmon[100289]: ) 159 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:33.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:32 smithi190 conmon[100518]: cluster 2023-10-01T09:05:30.875501+0000 mgr.y (mgr 2023-10-01T09:05:33.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:32 smithi190 conmon[100518]: .14433) 159 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:34.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:33 smithi190 conmon[100518]: audit 2023-10-01T09:05:32.992504+0000 mon.c (mon 2023-10-01T09:05:34.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:33 smithi190 conmon[100518]: .2) 115 : audit [DBG] from='client.? 172.21.15.138:0/2268910626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:34.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:33 smithi138 conmon[100289]: audit 2023-10-01T09:05:32.992504+0000 mon.c (mon.2 2023-10-01T09:05:34.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:33 smithi138 conmon[100289]: ) 115 : audit [DBG] from='client.? 172.21.15.138:0/2268910626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:34.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:33 smithi138 conmon[109959]: audit 2023-10-01T09:05:32.992504+0000 mon.c 2023-10-01T09:05:34.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:33 smithi138 conmon[109959]: (mon.2) 115 : audit [DBG] from='client.? 172.21.15.138:0/2268910626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:35.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:34 smithi190 conmon[100518]: cluster 2023-10-01T09:05:32.875852+0000 mgr.y (mgr.14433) 2023-10-01T09:05:35.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:34 smithi190 conmon[100518]: 160 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:35.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:34 smithi138 conmon[100289]: cluster 2023-10-01T09:05:32.875852+0000 mgr.y (mgr.14433) 160 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:35.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:34 smithi138 conmon[109959]: cluster 2023-10-01T09:05:32.875852+0000 mgr.y (mgr.14433) 160 : cluster 2023-10-01T09:05:35.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:34 smithi138 conmon[109959]: [DBG] pgmap v144: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:35.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:05:35 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:05:35] "GET /metrics HTTP/1.1" 200 192067 "" "Prometheus/2.33.4" 2023-10-01T09:05:36.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:35 smithi190 conmon[100518]: audit 2023-10-01T09:05:35.356086+0000 mon.a ( 2023-10-01T09:05:36.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:35 smithi190 conmon[100518]: mon.0) 719 : audit [DBG] from='client.? 172.21.15.138:0/3836536012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:36.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:35 smithi138 conmon[100289]: audit 2023-10-01T09:05:35.356086+0000 mon.a (mon.0 2023-10-01T09:05:36.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:35 smithi138 conmon[100289]: ) 719 : audit [DBG] from='client.? 172.21.15.138:0/3836536012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:36.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:35 smithi138 conmon[109959]: audit 2023-10-01T09:05:35.356086+0000 2023-10-01T09:05:36.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:35 smithi138 conmon[109959]: mon.a (mon.0) 719 : audit [DBG] from='client.? 172.21.15.138:0/3836536012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:36.842 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:36 smithi190 conmon[100518]: cluster 2023-10-01T09:05:34.876335 2023-10-01T09:05:36.842 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:36 smithi190 conmon[100518]: +0000 mgr.y (mgr.14433) 161 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:37.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:36 smithi138 conmon[100289]: cluster 2023-10-01T09:05:34.876335+0000 mgr.y (mgr.14433) 161 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:37.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:36 smithi138 conmon[109959]: cluster 2023-10-01T09:05:34.876335+0000 mgr.y (mgr.14433 2023-10-01T09:05:37.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:36 smithi138 conmon[109959]: ) 161 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:37.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:05:36 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:05:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:05:39.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:38 smithi190 conmon[100518]: cluster 2023-10-01T09:05:36.876981 2023-10-01T09:05:39.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:38 smithi190 conmon[100518]: +0000 mgr.y (mgr.14433) 162 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:39.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:38 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:05:39.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:38 smithi190 conmon[100518]: :05:37.703997+0000 mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.138:0/630392761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:39.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:38 smithi138 conmon[109959]: cluster 2023-10-01T09:05:36.876981+0000 mgr.y (mgr. 2023-10-01T09:05:39.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:38 smithi138 conmon[109959]: 14433) 162 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:39.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:38 smithi138 conmon[109959]: audit 2023-10-01T09:05:37.703997+0000 2023-10-01T09:05:39.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:38 smithi138 conmon[109959]: mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.138:0/630392761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:38 smithi138 conmon[100289]: cluster 2023-10-01T09:05:36.876981+0000 mgr.y (mgr.14433 2023-10-01T09:05:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:38 smithi138 conmon[100289]: ) 162 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:38 smithi138 conmon[100289]: audit 2023-10-01T09:05:37.703997+0000 mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.138:0/630392761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:41.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:40 smithi190 conmon[100518]: cluster 2023-10-01T09:05:38.877404+0000 mgr.y (mgr.14433) 163 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:41.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:40 smithi190 conmon[100518]: audit 2023-10-01T09:05:40.055731+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.138:0/283973675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:41.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:40 smithi138 conmon[109959]: cluster 2023-10-01T09:05:38.877404+0000 mgr.y (mgr.14433 2023-10-01T09:05:41.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:40 smithi138 conmon[109959]: ) 163 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:41.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:40 smithi138 conmon[109959]: audit 2023-10-01T09:05:40.055731+0000 mon.c (mon.2) 116 : audit [DBG] 2023-10-01T09:05:41.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:40 smithi138 conmon[109959]: from='client.? 172.21.15.138:0/283973675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:41.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:40 smithi138 conmon[100289]: cluster 2023-10-01T09:05:38.877404+0000 mgr.y (mgr.14433) 163 : cluster [DBG] 2023-10-01T09:05:41.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:40 smithi138 conmon[100289]: pgmap v147: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:41.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:40 smithi138 conmon[100289]: audit 2023-10-01T09:05:40.055731+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.138:0/283973675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:43.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:42 smithi190 conmon[100518]: cluster 2023-10-01T09:05:40.878073+0000 mgr.y (mgr.14433) 164 : cluster 2023-10-01T09:05:43.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:42 smithi190 conmon[100518]: [DBG] pgmap v148: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:43.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:42 smithi190 conmon[100518]: audit 2023-10-01T09:05:42.403770+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.138:0/3289029701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:43.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:42 smithi138 conmon[109959]: cluster 2023-10-01T09:05:40.878073+0000 mgr.y (mgr.14433) 164 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:43.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:42 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:05:43.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:42 smithi138 conmon[109959]: T09:05:42.403770+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.138:0/3289029701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:42 smithi138 conmon[100289]: cluster 2023-10-01T09:05:40.878073+0000 mgr.y (mgr.14433) 2023-10-01T09:05:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:42 smithi138 conmon[100289]: 164 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:42 smithi138 conmon[100289]: audit 2023-10-01T09:05:42.403770+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.138:0/3289029701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:45.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:44 smithi190 conmon[100518]: cluster 2023-10-01T09:05:42.878568+0000 mgr.y (mgr.14433) 2023-10-01T09:05:45.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:44 smithi190 conmon[100518]: 165 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:45.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:44 smithi138 conmon[109959]: cluster 2023-10-01T09:05:42.878568+0000 mgr.y ( 2023-10-01T09:05:45.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:44 smithi138 conmon[109959]: mgr.14433) 165 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:45.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:44 smithi138 conmon[100289]: cluster 2023-10-01T09:05:42.878568+0000 mgr.y ( 2023-10-01T09:05:45.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:44 smithi138 conmon[100289]: mgr.14433) 165 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:45.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:05:45 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:05:45] "GET /metrics HTTP/1.1" 200 192064 "" "Prometheus/2.33.4" 2023-10-01T09:05:46.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:45 smithi190 conmon[100518]: audit 2023-10-01T09:05:44.751614+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.138:0/570517018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:46.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:45 smithi138 conmon[109959]: audit 2023-10-01T09:05:44.751614+0000 mon.c (mon.2) 117 : audit 2023-10-01T09:05:46.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:45 smithi138 conmon[109959]: [DBG] from='client.? 172.21.15.138:0/570517018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:46.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:45 smithi138 conmon[100289]: audit 2023-10-01T09:05:44.751614+0000 mon.c ( 2023-10-01T09:05:46.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:45 smithi138 conmon[100289]: mon.2) 117 : audit [DBG] from='client.? 172.21.15.138:0/570517018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:47.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:46 smithi190 conmon[100518]: cluster 2023-10-01T09:05:44.879109+0000 mgr.y (mgr. 2023-10-01T09:05:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:46 smithi190 conmon[100518]: 14433) 166 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:47.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:05:46 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:05:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:05:47.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:46 smithi138 conmon[109959]: cluster 2023-10-01T09:05:44.879109+0000 mgr.y (mgr.14433) 166 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:47.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:46 smithi138 conmon[100289]: cluster 2023-10-01T09:05:44.879109+0000 mgr.y (mgr.14433 2023-10-01T09:05:47.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:46 smithi138 conmon[100289]: ) 166 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:48.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:47 smithi190 conmon[100518]: audit 2023-10-01T09:05:47.103410+0000 mon.c (mon.2) 2023-10-01T09:05:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:47 smithi190 conmon[100518]: 118 : audit [DBG] from='client.? 172.21.15.138:0/1791255988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:48.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:47 smithi138 conmon[109959]: audit 2023-10-01T09:05:47.103410+0000 mon.c 2023-10-01T09:05:48.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:47 smithi138 conmon[109959]: (mon.2) 118 : audit [DBG] from='client.? 172.21.15.138:0/1791255988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:48.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:47 smithi138 conmon[100289]: audit 2023-10-01T09:05:47.103410+0000 mon.c ( 2023-10-01T09:05:48.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:47 smithi138 conmon[100289]: mon.2) 118 : audit [DBG] from='client.? 172.21.15.138:0/1791255988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:49.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:48 smithi190 conmon[100518]: cluster 2023-10-01T09:05:46.879560+0000 mgr.y (mgr.14433) 2023-10-01T09:05:49.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:48 smithi190 conmon[100518]: 167 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:49.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:48 smithi138 conmon[109959]: cluster 2023-10-01T09:05:46.879560+0000 mgr.y (mgr.14433) 167 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:49.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:48 smithi138 conmon[100289]: cluster 2023-10-01T09:05:46.879560+0000 mgr.y (mgr.14433) 167 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:49.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:48 smithi138 conmon[100289]: 2023-10-01T09:05:49.910 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:49 smithi190 conmon[100518]: audit 2023-10-01T09:05:49.451939+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.138:0/2082732337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:50.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:49 smithi138 conmon[100289]: audit 2023-10-01T09:05:49.451939 2023-10-01T09:05:50.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:49 smithi138 conmon[100289]: +0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.138:0/2082732337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:50.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:49 smithi138 conmon[109959]: audit 2023-10-01T09:05:49.451939+0000 mon.a (mon.0) 2023-10-01T09:05:50.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:49 smithi138 conmon[109959]: 722 : audit [DBG] from='client.? 172.21.15.138:0/2082732337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:51.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:50 smithi190 conmon[100518]: cluster 2023-10-01T09:05:48.880042+0000 mgr.y (mgr.14433) 168 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:51.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:50 smithi138 conmon[100289]: cluster 2023-10-01T09:05:48.880042+0000 mgr.y (mgr.14433) 168 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:51.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:50 smithi138 conmon[109959]: cluster 2023-10-01T09:05:48.880042+0000 mgr.y (mgr.14433) 2023-10-01T09:05:51.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:50 smithi138 conmon[109959]: 168 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:52 smithi190 conmon[100518]: cluster 2023-10-01T09:05:50.880630+0000 mgr.y (mgr.14433) 169 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:52 smithi190 conmon[100518]: 2023-10-01T09:05:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:52 smithi190 conmon[100518]: audit 2023-10-01T09:05:51.799501+0000 mon.a (mon.0) 723 : audit [DBG] from='client.? 172.21.15.138:0/269893595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:53.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:52 smithi138 conmon[109959]: cluster 2023-10-01T09:05:50.880630+0000 mgr.y (mgr.14433) 169 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:53.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:52 smithi138 conmon[109959]: audit 2023-10-01T09:05:51.799501+0000 mon.a (mon. 2023-10-01T09:05:53.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:52 smithi138 conmon[109959]: 0) 723 : audit [DBG] from='client.? 172.21.15.138:0/269893595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:53.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:52 smithi138 conmon[100289]: cluster 2023-10-01T09:05:50.880630+0000 mgr.y (mgr.14433) 169 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:53.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:52 smithi138 conmon[100289]: audit 2023-10-01T09:05:51.799501+0000 mon.a (mon.0) 723 : audit [DBG] from='client.? 172.21.15.138:0/269893595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:55.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:54 smithi190 conmon[100518]: cluster 2023-10-01T09:05:52.880995+0000 mgr.y (mgr.14433) 170 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:55.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:54 smithi190 conmon[100518]: audit 2023-10-01T09:05:54.149796+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.138:0/1018441207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:55.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:54 smithi138 conmon[109959]: cluster 2023-10-01T09:05:52.880995+0000 mgr.y (mgr.14433) 170 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:55.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:54 smithi138 conmon[109959]: audit 2023-10-01T09:05:54.149796+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.138:0/1018441207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:55.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:54 smithi138 conmon[100289]: cluster 2023-10-01T09:05:52.880995+0000 mgr.y (mgr.14433) 170 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:55.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:54 smithi138 conmon[100289]: audit 2023-10-01T09:05:54.149796+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.138:0/1018441207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:55.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:05:55 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:05:55] "GET /metrics HTTP/1.1" 200 192064 "" "Prometheus/2.33.4" 2023-10-01T09:05:57.056 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:05:56 smithi190 conmon[102823]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:05:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:05:57.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:56 smithi190 conmon[100518]: cluster 2023-10-01T09:05:54.881456+0000 mgr.y (mgr.14433) 171 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:57.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:56 smithi190 conmon[100518]: audit 2023-10-01T09:05:56.508095+0000 mon.a (mon.0) 724 : audit [DBG] from='client.? 172.21.15.138:0/2310541448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:57.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:56 smithi138 conmon[100289]: cluster 2023-10-01T09:05:54.881456+0000 mgr.y (mgr.14433) 2023-10-01T09:05:57.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:56 smithi138 conmon[100289]: 171 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:57.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:56 smithi138 conmon[100289]: audit 2023-10-01T09:05:56.508095+0000 mon.a (mon.0) 724 : audit [DBG] from='client.? 172.21.15.138:0/2310541448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:57.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:56 smithi138 conmon[109959]: cluster 2023-10-01T09:05:54.881456+0000 mgr.y 2023-10-01T09:05:57.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:56 smithi138 conmon[109959]: (mgr.14433) 171 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:57.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:56 smithi138 conmon[109959]: audit 2023-10-01T09:05:56.508095+0000 mon.a (mon.0 2023-10-01T09:05:57.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:56 smithi138 conmon[109959]: ) 724 : audit [DBG] from='client.? 172.21.15.138:0/2310541448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:58.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:57 smithi190 conmon[100518]: cluster 2023-10-01T09:05:56.882061+0000 2023-10-01T09:05:58.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:57 smithi190 conmon[100518]: mgr.y (mgr.14433) 172 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:58.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:57 smithi138 conmon[109959]: cluster 2023-10-01T09:05:56.882061+0000 mgr.y (mgr.14433) 172 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:58.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:57 smithi138 conmon[109959]: 2023-10-01T09:05:58.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:57 smithi138 conmon[100289]: cluster 2023-10-01T09:05:56.882061+0000 mgr.y (mgr.14433) 172 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:05:58.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:57 smithi138 conmon[100289]: 2023-10-01T09:05:59.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:58 smithi190 conmon[100518]: audit 2023-10-01T09:05:58.853208+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.138:0/322850498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:59.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:58 smithi138 conmon[109959]: audit 2023-10-01T09:05:58.853208+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.138:0/322850498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:05:59.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:58 smithi138 conmon[100289]: audit 2023-10-01T09:05:58.853208+0000 mon.c (mon.2) 120 : audit 2023-10-01T09:05:59.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:58 smithi138 conmon[100289]: [DBG] from='client.? 172.21.15.138:0/322850498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:05:59 smithi190 conmon[100518]: cluster 2023-10-01T09:05:58.882636+0000 mgr.y (mgr.14433) 173 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:00.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:59 smithi138 conmon[109959]: cluster 2023-10-01T09:05:58.882636+0000 mgr.y (mgr.14433) 2023-10-01T09:06:00.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:05:59 smithi138 conmon[109959]: 173 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:00.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:05:59 smithi138 conmon[100289]: cluster 2023-10-01T09:05:58.882636+0000 mgr.y (mgr.14433) 173 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:02.283 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:02 smithi190 conmon[100518]: cluster 2023-10-01T09:06:00.883217+0000 mgr.y (mgr.14433) 174 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:02.283 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:02 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:01 2023-10-01T09:06:02.283 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:02 smithi190 conmon[100518]: .027711+0000 mgr.y (mgr.14433) 175 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-01T09:06:02.283 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:02 smithi190 conmon[100518]: audit 2023-10-01T09:06:01.034358+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:02.283 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:02 smithi190 conmon[100518]: audit 2023-10-01T09:06:01.037311+0000 mon.c (mon.2) 121 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:06:02.283 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:02 smithi190 conmon[100518]: audit 2023-10-01T09:06:01.037655+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:06:02.283 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:02 smithi190 conmon[100518]: audit 2023-10-01T09:06:01.038464+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:06:02.283 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:02 smithi190 conmon[100518]: audit 2023-10-01T09:06:01.039115+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:06:02.284 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:02 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:01.039651+0000 mgr.y (mgr.14433) 176 : cephadm [INF] Deploying daemon mgr.x on smithi190 2023-10-01T09:06:02.284 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:02 smithi190 conmon[100518]: audit 2023-10-01T09:06:01.204442+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.138:0/1792341729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: cluster 2023-10-01T09:06:00.883217+0000 mgr.y (mgr.14433) 174 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:01.027711+0000 mgr.y (mgr.14433) 175 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-01T09:06:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: audit 2023-10-01T09:06:01.034358+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: audit 2023-10-01T09:06:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: 2023-10-01T09:06:01.037311+0000 mon.c (mon.2) 121 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:06:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: audit 2023-10-01T09:06:01.037655+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:06:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: 2023-10-01T09:06:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: audit 2023-10-01T09:06:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: 2023-10-01T09:06:01.038464+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:06:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: audit 2023-10-01T09:06:01.039115+0000 2023-10-01T09:06:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: mon.c (mon.2) 123 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:06:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: cephadm 2023-10- 2023-10-01T09:06:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: 01T09:06:01.039651+0000 mgr.y (mgr.14433) 176 : cephadm [INF] Deploying daemon mgr.x on smithi190 2023-10-01T09:06:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: audit 2023 2023-10-01T09:06:02.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[100289]: -10-01T09:06:01.204442+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.138:0/1792341729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:02.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: cluster 2023-10-01T09:06:00.883217+0000 mgr.y (mgr.14433 2023-10-01T09:06:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: ) 174 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: cephadm 2023-10-01T09: 2023-10-01T09:06:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: 06:01.027711+0000 mgr.y (mgr.14433) 175 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-01T09:06:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:06:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: 06:01.034358+0000 mon.a (mon.0) 725 : audit 2023-10-01T09:06:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:06:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: 06:01.037311+0000 mon.c (mon.2) 121 : audit 2023-10-01T09:06:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:06:02.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: audit 2023-10-01T09:06 2023-10-01T09:06:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: :01.037655+0000 mon.a (mon.0) 726 : audit 2023-10-01T09:06:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:06:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: audit 2023-10-01T09:06:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: 2023-10-01T09:06:01.038464+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:06:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: audit 2023-10-01T09:06:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: 2023-10-01T09:06:01.039115+0000 mon.c ( 2023-10-01T09:06:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: mon.2) 123 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:06:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: cephadm 2023-10-01T09:06 2023-10-01T09:06:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: :01.039651+0000 mgr.y (mgr.14433) 176 : cephadm [INF] 2023-10-01T09:06:02.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: Deploying daemon mgr.x on smithi190 2023-10-01T09:06:02.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:06:02.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: 06:01.204442+0000 mon.a (mon.0) 727 : 2023-10-01T09:06:02.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:02 smithi138 conmon[109959]: audit [DBG] from='client.? 172.21.15.138:0/1792341729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:02.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:02 smithi190 systemd[1]: Stopping Ceph mgr.x for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:06:02.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:02 smithi190 bash[138924]: Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-mgr.x found: no such container 2023-10-01T09:06:03.453 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:03 smithi190 bash[139041]: 9629bd7ec45f54a9ddbd0367d9c28a8bae70be4fdc7109c9fee6fec0174451f3 2023-10-01T09:06:03.453 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:03 smithi190 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-01T09:06:03.453 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:03 smithi190 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-01T09:06:03.453 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:03 smithi190 systemd[1]: Stopped Ceph mgr.x for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:06:03.806 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:03 smithi190 systemd[1]: Starting Ceph mgr.x for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:06:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:04 smithi190 conmon[100518]: cluster 2023-10-01T09:06:02.883606+0000 mgr.y (mgr.14433) 177 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:04 smithi190 conmon[100518]: audit 2023-10-01T09:06:03.555353+0000 mon.c (mon.2) 124 : audit [DBG] from='client.? 172.21.15.138:0/308898179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:04.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:04 smithi190 bash[139396]: d82938b5a62fa91520386313fbda91816cda57b4d392439f2e38f269db574e87 2023-10-01T09:06:04.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:04 smithi190 systemd[1]: Started Ceph mgr.x for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:06:04.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:04 smithi190 conmon[139437]: debug 2023-10-01T09:06:04.162+0000 7f5f6bef4200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T09:06:04.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:04 smithi190 conmon[139437]: debug 2023-10-01T09:06:04.245+0000 7f5f6bef4200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T09:06:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:04 smithi138 conmon[109959]: cluster 2023-10-01T09:06:02.883606+0000 mgr.y ( 2023-10-01T09:06:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:04 smithi138 conmon[109959]: mgr.14433) 177 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:04 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:06:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:04 smithi138 conmon[109959]: T09:06:03.555353+0000 mon.c (mon.2) 124 : audit [DBG] from='client.? 172.21.15.138:0/308898179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:04 smithi138 conmon[100289]: cluster 2023-10-01T09:06:02.883606+0000 mgr.y ( 2023-10-01T09:06:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:04 smithi138 conmon[100289]: mgr.14433) 177 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:04 smithi138 conmon[100289]: audit 2023-10-01T09:06:03.555353 2023-10-01T09:06:04.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:04 smithi138 conmon[100289]: +0000 mon.c (mon.2) 124 : audit [DBG] from='client.? 172.21.15.138:0/308898179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:04.656 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:04 smithi190 conmon[139437]: debug 2023-10-01T09:06:04.345+0000 7f5f6bef4200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T09:06:04.656 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:04 smithi190 conmon[139437]: debug 2023-10-01T09:06:04.458+0000 7f5f6bef4200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T09:06:05.056 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:04 smithi190 conmon[139437]: debug 2023-10-01T09:06:04.655+0000 7f5f6bef4200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T09:06:05.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:05 smithi190 conmon[100518]: audit 2023-10-01T09:06:04.093574+0000 2023-10-01T09:06:05.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:05 smithi190 conmon[100518]: mon.a (mon.0) 728 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:05.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:05 smithi190 conmon[100518]: audit 2023-10-01T09: 2023-10-01T09:06:05.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:05 smithi190 conmon[100518]: 06:04.095016+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:06:05.308 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:05 smithi190 conmon[139437]: debug 2023-10-01T09:06:05.055+0000 7f5f6bef4200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T09:06:05.308 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:05 smithi190 conmon[139437]: debug 2023-10-01T09:06:05.205+0000 7f5f6bef4200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T09:06:05.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:05 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:06:05] "GET /metrics HTTP/1.1" 200 192054 "" "Prometheus/2.33.4" 2023-10-01T09:06:05.650 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:05 smithi138 conmon[100289]: audit 2023-10-01T09:06:04.093574 2023-10-01T09:06:05.651 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:05 smithi138 conmon[100289]: +0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:05.651 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:05 smithi138 conmon[100289]: audit 2023-10-01T09:06:04.095016+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:06:05.651 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:05 smithi138 conmon[109959]: audit 2023-10-01T09:06:04 2023-10-01T09:06:05.651 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:05 smithi138 conmon[109959]: .093574+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:05.651 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:05 smithi138 conmon[109959]: audit 2023-10-01T09:06:04.095016+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:06:05.807 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:05 smithi190 conmon[139437]: debug 2023-10-01T09:06:05.307+0000 7f5f6bef4200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T09:06:05.807 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:05 smithi190 conmon[139437]: debug 2023-10-01T09:06:05.407+0000 7f5f6bef4200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T09:06:06.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:05 smithi190 conmon[139437]: debug 2023-10-01T09:06:05.896+0000 7f5f6bef4200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T09:06:06.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:06 smithi190 conmon[100518]: cluster 2023-10-01T09:06:04.883983+0000 mgr.y ( 2023-10-01T09:06:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:06 smithi190 conmon[100518]: mgr.14433) 178 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:06 smithi190 conmon[100518]: audit 2023-10-01T09:06:05.934074+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.138:0/1412430144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:06 smithi190 conmon[100518]: audit 2023-10-01T09:06:06.289782+0000 mon.a (mon 2023-10-01T09:06:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:06 smithi190 conmon[100518]: .0) 729 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:06.807 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:06 smithi190 conmon[139437]: debug 2023-10-01T09:06:06.621+0000 7f5f6bef4200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T09:06:06.807 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:06 smithi190 conmon[139437]: debug 2023-10-01T09:06:06.707+0000 7f5f6bef4200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T09:06:06.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:06 smithi138 conmon[100289]: cluster 2023-10-01T09:06:04.883983+0000 mgr.y (mgr.14433) 178 : cluster 2023-10-01T09:06:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:06 smithi138 conmon[100289]: [DBG] pgmap v160: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:06 smithi138 conmon[100289]: audit 2023-10-01T09:06:05.934074+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.138:0/1412430144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:06 smithi138 conmon[100289]: audit 2023-10-01T09:06:06.289782+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:06 smithi138 conmon[109959]: cluster 2023-10-01T09:06 2023-10-01T09:06:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:06 smithi138 conmon[109959]: :04.883983+0000 mgr.y (mgr.14433) 178 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:06 smithi138 conmon[109959]: audit 2023-10-01T09:06: 2023-10-01T09:06:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:06 smithi138 conmon[109959]: 05.934074+0000 mon.c (mon.2) 126 : audit [DBG] 2023-10-01T09:06:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:06 smithi138 conmon[109959]: from='client.? 172.21.15.138:0/1412430144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:06 smithi138 conmon[109959]: audit 2023-10-01T09:06: 2023-10-01T09:06:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:06 smithi138 conmon[109959]: 06.289782+0000 mon.a (mon.0) 729 : 2023-10-01T09:06:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:06 smithi138 conmon[109959]: audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:07.842 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:07 smithi190 conmon[139437]: debug 2023-10-01T09:06:07.568+0000 7f5f6bef4200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:06:07.843 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:07 smithi190 conmon[139437]: debug 2023-10-01T09:06:07.665+0000 7f5f6bef4200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T09:06:07.843 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:07 smithi190 conmon[139437]: debug 2023-10-01T09:06:07.841+0000 7f5f6bef4200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:06:08.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:07 smithi138 conmon[100738]: debug 2023-10-01T09:06:07.587+0000 7f2030661700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-01T09:06:08.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:07 smithi138 conmon[100738]: ignoring --setuser ceph since I am not root 2023-10-01T09:06:08.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:07 smithi138 conmon[100738]: ignoring --setgroup ceph since I am not root 2023-10-01T09:06:08.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:07 smithi138 conmon[100738]: debug 2023-10-01T09:06:07.627+0000 7f040b844700 1 -- 172.21.15.138:0/874805283 <== mon.0 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56147757e4e0 con 0x561477557000 2023-10-01T09:06:08.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:07 smithi138 conmon[100738]: debug 2023-10-01T09:06:07.745+0000 7f041ba0b680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T09:06:08.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:07 smithi138 conmon[100738]: debug 2023-10-01T09:06:07.828+0000 7f041ba0b680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T09:06:08.500 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[139437]: debug 2023-10-01T09:06:08.124+0000 7f5f6bef4200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T09:06:08.500 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[139437]: debug 2023-10-01T09:06:08.352+0000 7f5f6bef4200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T09:06:08.500 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: audit 2023-10-01T09:06:07.237634+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:08.500 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: audit 2023-10-01T09:06:07. 2023-10-01T09:06:08.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: 566922+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:06:08.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: audit 2023-10-01T09:06:07.567750+0000 mon.c (mon.2) 128 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:06:08.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: audit 2023-10-01T09:06:07.572280+0000 mon.a 2023-10-01T09:06:08.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: (mon.0) 731 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:08.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:06:08.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: :06:07.577189+0000 mon.c 2023-10-01T09:06:08.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: (mon.2) 129 : audit [DBG] 2023-10-01T09:06:08.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:06:08.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:06:08.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: 10-01T09:06:07. 2023-10-01T09:06:08.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: 581844+0000 mon.c (mon.2) 130 : 2023-10-01T09:06:08.502 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-01T09:06:08.502 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: audit 2023 2023-10-01T09:06:08.502 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: -10-01T09:06:07. 2023-10-01T09:06:08.502 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: 582134+0000 mon.a (mon.0) 732 : 2023-10-01T09:06:08.502 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-01T09:06:08.502 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: cluster 2023- 2023-10-01T09:06:08.502 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: 10-01T09:06:07.588240 2023-10-01T09:06:08.502 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: +0000 mon.a (mon.0) 733 : cluster 2023-10-01T09:06:08.502 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-01T09:06:08.502 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:06:08.502 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: -01T09:06:07.588599+0000 2023-10-01T09:06:08.503 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: mon.a (mon.0) 734 : audit [INF] 2023-10-01T09:06:08.503 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-01T09:06:08.503 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: cluster 2023-10- 2023-10-01T09:06:08.503 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: 01T09:06:07.588661 2023-10-01T09:06:08.503 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: +0000 mon.a (mon.0) 735 : cluster 2023-10-01T09:06:08.503 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[100518]: [DBG] mgrmap e19: x(active, starting, since 0.00609704s) 2023-10-01T09:06:08.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100738]: debug 2023-10-01T09:06:08.200+0000 7f041ba0b680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T09:06:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: audit 2023-10-01T09:06:07.237634 2023-10-01T09:06:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: +0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: audit 2023-10-01T09:06:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: 2023-10-01T09:06:07.566922+0000 mon.c 2023-10-01T09:06:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: (mon.2) 127 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:06:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: audit 2023 2023-10-01T09:06:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: -10-01T09:06:07.567750+0000 mon.c 2023-10-01T09:06:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: (mon.2) 128 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:06:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: audit 2023-10-01T09:06:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: 2023-10-01T09:06:07.572280+0000 2023-10-01T09:06:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: mon.a (mon.0) 731 : audit [INF] 2023-10-01T09:06:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:06:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: :06:07.577189+0000 mon.c (mon.2) 2023-10-01T09:06:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: 129 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:06:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:06:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: 10-01T09:06:07.581844+0000 mon.c ( 2023-10-01T09:06:08.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: mon.2) 130 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-01T09:06:08.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:06:08.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: 10-01T09:06:07.582134+0000 mon.a (mon 2023-10-01T09:06:08.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: .0) 732 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-01T09:06:08.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: cluster 2023-10-01T09:06:08.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: 2023-10-01T09:06:07.588240+0000 2023-10-01T09:06:08.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: mon.a (mon.0) 733 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-01T09:06:08.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: 2023-10-01T09:06:08.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: audit 2023-10-01T09:06 2023-10-01T09:06:08.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: :07.588599+0000 mon.a (mon.0) 734 2023-10-01T09:06:08.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-01T09:06:08.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: cluster 2023-10-01T09:06: 2023-10-01T09:06:08.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: 07.588661+0000 mon.a (mon.0) 735 : cluster 2023-10-01T09:06:08.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[109959]: [DBG] mgrmap e19: x(active, starting, since 0.00609704s) 2023-10-01T09:06:08.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: audit 2023-10-01T09:06:07.237634 2023-10-01T09:06:08.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: +0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:08.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: audit 2023-10-01T09:06:07.566922+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:06:08.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: audit 2023-10-01T09:06:07.567750+0000 mon.c ( 2023-10-01T09:06:08.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: mon.2) 128 : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:06:08.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: audit 2023-10-01T09:06:07.572280+0000 2023-10-01T09:06:08.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: mon.a (mon.0) 731 : audit [INF] from='mgr.14433 ' entity='mgr.y' 2023-10-01T09:06:08.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T09:06:08.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: T09:06:07.577189+0000 mon.c (mon.2 2023-10-01T09:06:08.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: ) 129 : audit [DBG] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:06:08.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: audit 2023-10-01T09:06 2023-10-01T09:06:08.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: :07.581844+0000 mon.c (mon.2) 130 2023-10-01T09:06:08.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: : audit [INF] from='mgr.14433 172.21.15.138:0/2791134413' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-01T09:06:08.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:06:08.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: 06:07.582134+0000 mon.a (mon.0) 732 : audit 2023-10-01T09:06:08.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: [INF] from='mgr.14433 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-01T09:06:08.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: cluster 2023-10-01T09:06:08.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: 2023-10-01T09:06:07.588240+0000 mon.a ( 2023-10-01T09:06:08.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: mon.0) 733 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-10-01T09:06:08.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:06:08.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: 06:07.588599+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14433 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-01T09:06:08.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: 2023-10-01T09:06:08.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: cluster 2023- 2023-10-01T09:06:08.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: 10-01T09:06:07.588661+0000 mon.a (mon.0) 735 2023-10-01T09:06:08.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100289]: : cluster [DBG] mgrmap e19: x(active, starting, since 0.00609704s) 2023-10-01T09:06:09.056 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[139437]: debug 2023-10-01T09:06:08.499+0000 7f5f6bef4200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T09:06:09.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:08 smithi190 conmon[139437]: debug 2023-10-01T09:06:08.766+0000 7f5f6bef4200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T09:06:09.238 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:08 smithi138 conmon[100738]: debug 2023-10-01T09:06:08.943+0000 7f041ba0b680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T09:06:09.238 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:09 smithi138 conmon[100738]: debug 2023-10-01T09:06:09.027+0000 7f041ba0b680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T09:06:09.556 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:09 smithi190 conmon[139437]: debug 2023-10-01T09:06:09.470+0000 7f5f6bef4200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T09:06:09.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:09 smithi190 conmon[100518]: audit 2023-10-01T09:06:08.285058+0000 mon.a (mon.0) 736 : audit 2023-10-01T09:06:09.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:09 smithi190 conmon[100518]: [DBG] from='client.? 172.21.15.138:0/3940734864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:09.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:09 smithi138 conmon[109959]: audit 2023-10-01T09:06:08.285058 2023-10-01T09:06:09.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:09 smithi138 conmon[109959]: +0000 mon.a (mon.0) 736 : audit [DBG] from='client.? 172.21.15.138:0/3940734864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:09 smithi138 conmon[100289]: audit 2023-10-01T09:06:08.285058+0000 mon.a (mon.0) 736 : audit 2023-10-01T09:06:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:09 smithi138 conmon[100289]: [DBG] from='client.? 172.21.15.138:0/3940734864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:09.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:09 smithi138 conmon[100738]: debug 2023-10-01T09:06:09.239+0000 7f041ba0b680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T09:06:09.909 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:09 smithi190 conmon[139437]: debug 2023-10-01T09:06:09.601+0000 7f5f6bef4200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T09:06:09.909 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:09 smithi190 conmon[139437]: debug 2023-10-01T09:06:09.795+0000 7f5f6bef4200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T09:06:10.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:09 smithi190 conmon[139437]: debug 2023-10-01T09:06:09.908+0000 7f5f6bef4200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T09:06:10.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:10 smithi190 conmon[139437]: debug 2023-10-01T09:06:10.138+0000 7f5f6bef4200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T09:06:10.485 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:10 smithi138 conmon[100738]: debug 2023-10-01T09:06:10.176+0000 7f041ba0b680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T09:06:10.486 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:10 smithi138 conmon[100738]: debug 2023-10-01T09:06:10.305+0000 7f041ba0b680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:06:10.486 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:10 smithi138 conmon[100738]: debug 2023-10-01T09:06:10.389+0000 7f041ba0b680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T09:06:10.486 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:10 smithi138 conmon[100738]: debug 2023-10-01T09:06:10.484+0000 7f041ba0b680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T09:06:10.819 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:10 smithi138 conmon[100738]: debug 2023-10-01T09:06:10.687+0000 7f041ba0b680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T09:06:11.287 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:10 smithi138 conmon[100738]: debug 2023-10-01T09:06:10.966+0000 7f041ba0b680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T09:06:11.287 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100738]: debug 2023-10-01T09:06:11.071+0000 7f041ba0b680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T09:06:11.287 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: debug 2023-10-01T09:06:11.012+0000 7f5f6bef4200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T09:06:11.288 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: [01/Oct/2023:09:06:11] ENGINE Bus STARTING 2023-10-01T09:06:11.288 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: CherryPy Checker: 2023-10-01T09:06:11.288 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: The Application mounted at '' has an empty config. 2023-10-01T09:06:11.288 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: 2023-10-01T09:06:11.288 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: [01/Oct/2023:09:06:11] ENGINE Serving on http://:::9283 2023-10-01T09:06:11.288 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: [01/Oct/2023:09:06:11] ENGINE Bus STARTED 2023-10-01T09:06:11.288 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: [01/Oct/2023:09:06:11] ENGINE Bus STOPPING 2023-10-01T09:06:11.288 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: [01/Oct/2023:09:06:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T09:06:11.288 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: [01/Oct/2023:09:06:11] ENGINE Bus STOPPED 2023-10-01T09:06:11.288 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: [01/Oct/2023:09:06:11] ENGINE Bus STARTING 2023-10-01T09:06:11.539 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: debug 2023-10-01T09:06:11.292+0000 7f5eca367700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:06:11.540 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: debug 2023-10-01T09:06:11.292+0000 7f5ecd36d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:06:11.540 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: debug 2023-10-01T09:06:11.292+0000 7f5eca367700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:06:11.540 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: debug 2023-10-01T09:06:11.292+0000 7f5ecd36d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:06:11.540 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: debug 2023-10-01T09:06:11.292+0000 7f5ecd36d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:06:11.540 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: debug 2023-10-01T09:06:11.292+0000 7f5ecd36d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:06:11.540 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: debug 2023-10-01T09:06:11.292+0000 7f5ecd36d700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:06:11.540 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: debug 2023-10-01T09:06:11.292+0000 7f5eca367700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:06:11.540 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: debug 2023-10-01T09:06:11.292+0000 7f5eca367700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:06:11.541 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: debug 2023-10-01T09:06:11.292+0000 7f5eca367700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:06:11.541 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: [01/Oct/2023:09:06:11] ENGINE Serving on http://:::9283 2023-10-01T09:06:11.541 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[139437]: [01/Oct/2023:09:06:11] ENGINE Bus STARTED 2023-10-01T09:06:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:10.632331+0000 mon.c (mon.2) 131 : 2023-10-01T09:06:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit [DBG] from='client.? 172.21.15.138:0/907018577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: cluster 2023-10-01T09:06:11.026526+0000 mon.a (mon.0) 737 : cluster [INF] Active manager daemon x restarted 2023-10-01T09:06:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: cluster 2023-10-01T 2023-10-01T09:06:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 09:06:11.026923+0000 mon.a (mon.0) 738 : cluster [INF] Activating manager daemon x 2023-10-01T09:06:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11 2023-10-01T09:06:11.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: .028272+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:06:11.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: cluster 2023-10- 2023-10-01T09:06:11.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 01T09:06:11.031442+0000 mon.a (mon.0) 2023-10-01T09:06:11.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 739 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-01T09:06:11.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: cluster 2023-10-01T09:06:11.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 2023-10-01T09:06:11.031729+0000 mon.a (mon.0) 740 2023-10-01T09:06:11.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: : cluster [DBG] mgrmap e20: x(active, starting, since 0.00498828s) 2023-10-01T09:06:11.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 2023-10-01T09:06:11.035943+0000 mon.b (mon.1) 2023-10-01T09:06:11.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 29 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:06:11.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.036155+0000 mon.b (mon. 2023-10-01T09:06:11.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 1) 30 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:06:11.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:06:11.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: :06:11.036376+0000 mon.b (mon.1) 31 2023-10-01T09:06:11.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:06:11.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06 2023-10-01T09:06:11.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: :11.036588+0000 mon.b (mon.1) 32 : 2023-10-01T09:06:11.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:06:11.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06: 2023-10-01T09:06:11.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 11.036745+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.brhzod"}]: dispatch 2023-10-01T09:06:11.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:06:11.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 01T09:06:11.036958+0000 mon.b (mon. 2023-10-01T09:06:11.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 1) 34 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi190.weynjc"}]: dispatch 2023-10-01T09:06:11.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11. 2023-10-01T09:06:11.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 037174+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T09:06:11.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 2023-10-01T09:06:11.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:06:11.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 01T09:06:11.037397+0000 mon.b (mon.1 2023-10-01T09:06:11.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: ) 36 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T09:06:11.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 2023-10-01T09:06:11 2023-10-01T09:06:11.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: .037619+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T09:06:11.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11 2023-10-01T09:06:11.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: .037815+0000 mon.b (mon.1) 38 : audit [DBG] 2023-10-01T09:06:11.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T09:06:11.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023 2023-10-01T09:06:11.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: -10-01T09:06:11.038027+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:06:11.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 2023-10-01T09:06:11.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06 2023-10-01T09:06:11.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: :11.038216+0000 mon.b (mon.1) 40 : audit 2023-10-01T09:06:11.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:06:11.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.038430+0000 2023-10-01T09:06:11.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: mon.b (mon.1) 41 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:06:11.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.038626+0000 2023-10-01T09:06:11.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: mon.b (mon.1) 42 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:06:11.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:06:11.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: -01T09:06:11.038820+0000 mon.b (mon.1) 43 : 2023-10-01T09:06:11.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:06:11.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:06:11.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 10-01T09:06:11.039430+0000 mon.b (mon.1) 44 : 2023-10-01T09:06:11.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T09:06:11.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.039840 2023-10-01T09:06:11.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: +0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T09:06:11.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 2023-10-01T09:06:11.040293+0000 mon.b (mon.1) 2023-10-01T09:06:11.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 46 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T09:06:11.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T09:06:11.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: T09:06:11.040506+0000 mon.b (mon.1) 47 2023-10-01T09:06:11.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:06:11.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 2023-10-01T09:06:11.041682+0000 mon.b (mon.1) 48 2023-10-01T09:06:11.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T09:06:11.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.041934+0000 mon.b ( 2023-10-01T09:06:11.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: mon.1) 49 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:06:11.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: cluster 2023-10-01T09:06:11.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 2023-10-01T09:06:11.196800+0000 mon.a (mon.0) 741 : cluster [INF] Manager daemon x is now available 2023-10-01T09:06:11.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09: 2023-10-01T09:06:11.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 06:11.216139+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:11.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11. 2023-10-01T09:06:11.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 220464+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:06:11.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.222844+0000 2023-10-01T09:06:11.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: mon.b (mon.1) 51 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:06:11.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.230243+0000 mon.b ( 2023-10-01T09:06:11.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: mon.1) 52 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:06:11.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 2023-10-01T09:06:11.230603+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:06:11.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11 2023-10-01T09:06:11.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: .242629+0000 mon.b (mon.1) 53 : audit [DBG] 2023-10-01T09:06:11.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:06:11.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.249244+0000 mon.b ( 2023-10-01T09:06:11.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: mon.1) 54 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:06:11.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09: 2023-10-01T09:06:11.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 06:11.249520+0000 mon.a (mon.0) 744 : audit [INF] 2023-10-01T09:06:11.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:06:11.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:06:11.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: :06:11.268517+0000 mon.b (mon.1) 55 : audit [INF] 2023-10-01T09:06:11.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T09:06:11.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023 2023-10-01T09:06:11.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: -10-01T09:06:11.268870+0000 mon.a (mon.0) 745 : audit [INF] 2023-10-01T09:06:11.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T09:06:11.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: 2023-10-01T09:06:11.271993+0000 mon.a (mon.0) 746 : 2023-10-01T09:06:11.817 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:11 smithi190 conmon[100518]: audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:11.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:10.632331+0000 mon.c 2023-10-01T09:06:11.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: (mon.2) 131 : audit [DBG] from='client.? 172.21.15.138:0/907018577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:11.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: cluster 2023-10-01T09:06:11.026526+0000 mon.a (mon.0) 737 : cluster [INF] Active manager daemon x restarted 2023-10-01T09:06:11.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: cluster 2023-10-01T09:06:11.026923+0000 mon.a (mon.0) 738 : cluster [INF] Activating manager daemon x 2023-10-01T09:06:11.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.028272+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:06:11.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: cluster 2023-10-01T09:06:11.031442+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-01T09:06:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: cluster 2023-10-01T09:06:11.031729+0000 mon.a (mon.0) 740 : cluster [DBG] mgrmap e20: x(active, starting, since 0.00498828s) 2023-10-01T09:06:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.035943+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:06:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.036155 2023-10-01T09:06:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: +0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:06:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.036376+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:06:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.036588+0000 2023-10-01T09:06:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: mon.b (mon.1) 32 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:06:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:06:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 06:11.036745+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.brhzod"}]: dispatch 2023-10-01T09:06:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.036958+0000 2023-10-01T09:06:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: mon.b (mon.1) 34 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi190.weynjc"}]: dispatch 2023-10-01T09:06:11.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11 2023-10-01T09:06:11.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: .037174+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T09:06:11.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 2023-10-01T09:06:11.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:06:11.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: :06:11.037397+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T09:06:11.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 2023-10-01T09:06:11.037619+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T09:06:11.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.037815+0000 2023-10-01T09:06:11.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: mon.b (mon.1) 38 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T09:06:11.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.038027+0000 mon.b (mon.1) 2023-10-01T09:06:11.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 39 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:06:11.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:10.632331+0000 mon.c (mon.2) 131 : audit [DBG] from='client.? 172.21.15.138:0/907018577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:11.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: cluster 2023-10-01T09:06:11.026526+0000 mon.a (mon.0) 737 : cluster [INF] Active manager daemon x restarted 2023-10-01T09:06:11.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: cluster 2023-10-01T09:06:11.026923+0000 mon.a (mon.0) 738 : cluster [INF] Activating manager daemon x 2023-10-01T09:06:11.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.028272+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:06:11.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: cluster 2023-10-01T09:06:11.031442+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-10-01T09:06:11.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: cluster 2023-10-01T09:06:11.031729+0000 mon.a (mon.0) 740 : cluster [DBG] mgrmap e20: x(active, starting, since 0.00498828s) 2023-10-01T09:06:11.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.035943+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:06:11.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.036155+0000 mon.b (mon.1) 30 : audit [DBG] 2023-10-01T09:06:11.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:06:11.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.036376+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:06:11.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.036588+0000 mon.b (mon.1) 32 : audit 2023-10-01T09:06:11.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:06:11.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.036745+0000 2023-10-01T09:06:11.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: mon.b (mon.1) 33 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.brhzod"}]: dispatch 2023-10-01T09:06:11.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.036958+0000 2023-10-01T09:06:11.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: mon.b (mon.1) 34 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi190.weynjc"}]: dispatch 2023-10-01T09:06:11.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:06:11.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: T09:06:11.037174+0000 mon.b (mon.1) 2023-10-01T09:06:11.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 35 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T09:06:11.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11. 2023-10-01T09:06:11.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 037397+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T09:06:11.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 2023-10-01T09:06:11.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:06:11.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 01T09:06:11.037619+0000 mon.b (mon.1) 37 : audit [DBG] 2023-10-01T09:06:11.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T09:06:11.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:06:11.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 10-01T09:06:11.037815+0000 mon.b (mon.1) 38 : audit 2023-10-01T09:06:11.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T09:06:11.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 2023-10-01T09:06:11.038027+0000 mon.b (mon.1) 2023-10-01T09:06:11.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 39 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:06:11.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.038216+0000 2023-10-01T09:06:11.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: mon.b (mon.1) 40 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:06:11.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:06:11.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 06:11.038430+0000 mon.b (mon.1) 41 : audit 2023-10-01T09:06:11.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:06:11.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 2023-10-01T09:06:11.038626+0000 mon.b (mon.1 2023-10-01T09:06:11.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: ) 42 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:06:11.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06: 2023-10-01T09:06:11.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 11.038820+0000 mon.b (mon.1) 43 : audit 2023-10-01T09:06:11.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:06:11.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11. 2023-10-01T09:06:11.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 039430+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T09:06:11.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 2023-10-01T09:06:11.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023 2023-10-01T09:06:11.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: -10-01T09:06:11.039840+0000 mon.b (mon.1) 45 2023-10-01T09:06:11.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T09:06:11.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06 2023-10-01T09:06:11.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: :11.040293+0000 2023-10-01T09:06:11.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: mon.b (mon.1) 46 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T09:06:11.830 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:06:11.830 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 01T09:06:11.040506+0000 mon.b (mon. 2023-10-01T09:06:11.830 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 1) 47 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:06:11.830 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06 2023-10-01T09:06:11.830 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: :11.041682+0000 mon.b (mon.1) 48 : 2023-10-01T09:06:11.830 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T09:06:11.830 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.041934 2023-10-01T09:06:11.830 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: +0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:06:11.830 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 2023-10-01T09:06:11.830 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: cluster 2023- 2023-10-01T09:06:11.830 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 10-01T09:06:11.196800+0000 mon.a (mon.0) 741 2023-10-01T09:06:11.831 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: : cluster [INF] Manager daemon x is now available 2023-10-01T09:06:11.831 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11. 2023-10-01T09:06:11.831 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 216139+0000 mon.a (mon.0) 742 : audit [INF] 2023-10-01T09:06:11.831 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:11.831 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.831 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 2023-10-01T09:06:11.220464+0000 mon.b (mon.1 2023-10-01T09:06:11.831 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: ) 50 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:06:11.831 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.222844 2023-10-01T09:06:11.831 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: +0000 mon.b (mon.1) 51 : audit [DBG] 2023-10-01T09:06:11.832 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:06:11.832 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 09:06:11.038216+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:06:11.832 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11 2023-10-01T09:06:11.832 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: .038430+0000 mon.b (mon.1) 41 : audit [DBG] 2023-10-01T09:06:11.832 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:06:11.832 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06: 2023-10-01T09:06:11.832 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 11.038626+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:06:11.832 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 2023-10-01T09:06:11.832 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.833 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 2023-10-01T09:06:11.038820+0000 mon.b (mon.1 2023-10-01T09:06:11.833 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: ) 43 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:06:11.833 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:06:11.833 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 09:06:11.039430+0000 mon.b (mon.1 2023-10-01T09:06:11.833 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: ) 44 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T09:06:11.833 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:06:11.833 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 01T09:06:11.039840+0000 mon.b (mon.1 2023-10-01T09:06:11.833 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: ) 45 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T09:06:11.833 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:06:11.833 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 01T09:06:11.040293+0000 mon.b (mon 2023-10-01T09:06:11.833 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: .1) 46 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T09:06:11.833 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.834 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 2023-10-01 2023-10-01T09:06:11.834 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: T09:06:11.040506+0000 mon.b (mon 2023-10-01T09:06:11.834 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: .1) 47 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:06:11.834 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:06:11.834 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 01T09:06:11.041682+0000 mon.b (mon 2023-10-01T09:06:11.834 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: .1) 48 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T09:06:11.834 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.834 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 2023-10-01T09:06: 2023-10-01T09:06:11.834 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 11.041934+0000 mon.b (mon.1) 49 : 2023-10-01T09:06:11.834 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:06:11.834 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: cluster 2023-10-01T 2023-10-01T09:06:11.835 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 09:06:11.196800+0000 mon.a (mon.0 2023-10-01T09:06:11.835 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: ) 741 : cluster [INF] Manager daemon x is now available 2023-10-01T09:06:11.835 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:06:11.835 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: :06:11.216139+0000 mon.a (mon.0) 742 2023-10-01T09:06:11.835 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:11.835 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:06:11.835 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 06:11.220464+0000 mon.b (mon.1) 50 : 2023-10-01T09:06:11.835 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:06:11.835 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11 2023-10-01T09:06:11.835 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: .222844+0000 mon.b (mon.1) 51 : audit [DBG] 2023-10-01T09:06:11.836 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:06:11.836 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023 2023-10-01T09:06:11.836 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: -10-01T09:06:11.230243+0000 mon.b (mon.1) 2023-10-01T09:06:11.836 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 52 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:06:11.836 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 2023-10-01T09:06:11.836 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.230603+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:06:11.836 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:06:11.836 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 09:06:11.242629+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:06:11.836 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 2023-10-01T09:06:11.837 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:06:11.837 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 01T09:06:11.249244+0000 mon.b (mon.1) 54 : audit [INF] 2023-10-01T09:06:11.837 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:06:11.837 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.837 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 2023-10-01T09:06:11.249520+0000 mon.a (mon.0 2023-10-01T09:06:11.837 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: ) 744 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:06:11.837 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06: 2023-10-01T09:06:11.837 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 11.268517+0000 mon.b (mon.1) 55 : audit 2023-10-01T09:06:11.837 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T09:06:11.837 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06:11.837 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 2023-10-01T09:06:11.268870+0000 mon.a (mon. 2023-10-01T09:06:11.837 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 0) 745 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T09:06:11.838 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: audit 2023-10-01T09:06: 2023-10-01T09:06:11.838 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: 11.271993+0000 mon.a (mon.0) 746 : audit [INF] 2023-10-01T09:06:11.838 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[109959]: from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:11.838 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:06:11.838 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023 2023-10-01T09:06:11.838 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: -10-01T09:06:11.230243+0000 mon.b (mon.1) 52 2023-10-01T09:06:11.838 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:06:11.838 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11 2023-10-01T09:06:11.839 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: .230603+0000 mon.a (mon.0) 743 : audit 2023-10-01T09:06:11.839 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:06:11.839 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11 2023-10-01T09:06:11.839 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: .242629+0000 mon.b (mon.1) 53 : audit [DBG] 2023-10-01T09:06:11.839 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:06:11.839 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023 2023-10-01T09:06:11.839 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: -10-01T09:06:11.249244+0000 mon.b (mon.1) 54 : audit 2023-10-01T09:06:11.839 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:06:11.839 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.249520+0000 2023-10-01T09:06:11.839 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: mon.a (mon.0) 744 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:06:11.839 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:06:11.839 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: :06:11.268517+0000 mon.b (mon.1) 55 : audit 2023-10-01T09:06:11.840 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T09:06:11.840 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.840 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: 2023-10-01T09:06:11.268870+0000 mon.a (mon.0) 745 : 2023-10-01T09:06:11.840 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T09:06:11.840 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: audit 2023-10-01T09:06:11.271993+0000 2023-10-01T09:06:11.840 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100289]: mon.a (mon.0) 746 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:12.205 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100738]: debug 2023-10-01T09:06:11.846+0000 7f041ba0b680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T09:06:12.205 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:11 smithi138 conmon[100738]: debug 2023-10-01T09:06:11.931+0000 7f041ba0b680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T09:06:12.205 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100738]: debug 2023-10-01T09:06:12.023+0000 7f041ba0b680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T09:06:12.744 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100738]: debug 2023-10-01T09:06:12.203+0000 7f041ba0b680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T09:06:12.744 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100738]: debug 2023-10-01T09:06:12.467+0000 7f041ba0b680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T09:06:12.744 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100738]: debug 2023-10-01T09:06:12.600+0000 7f041ba0b680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T09:06:13.018 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100738]: debug 2023-10-01T09:06:12.742+0000 7f041ba0b680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:06:13.018 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100738]: debug 2023-10-01T09:06:12.926+0000 7f041ba0b680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T09:06:13.018 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: cephadm 2023-10-01T09:06:11.619396+0000 2023-10-01T09:06:13.018 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: mgr.x (mgr.25342) 1 : cephadm [INF] Deploying cephadm binary to smithi138 2023-10-01T09:06:13.018 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: cephadm 2023-10-01T09:06 2023-10-01T09:06:13.018 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: :11.620081+0000 mgr.x (mgr.25342) 2 : cephadm [INF] Deploying cephadm binary to smithi190 2023-10-01T09:06:13.018 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: audit 2023-10-01T09:06:13.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: 2023-10-01T09:06:11.869796+0000 2023-10-01T09:06:13.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: mon.a (mon.0) 747 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:13.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: audit 2023-10-01T09:06:13.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: 2023-10-01T09:06:11. 2023-10-01T09:06:13.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: 885226+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:13.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: cluster 2023-10-01T09:06:13.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: 2023-10-01T09:06:12 2023-10-01T09:06:13.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: .035735+0000 mon.a (mon.0) 749 : cluster 2023-10-01T09:06:13.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: [DBG] mgrmap e21: x(active, since 1.00899s) 2023-10-01T09:06:13.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: audit 2023-10-01T09:06:12. 2023-10-01T09:06:13.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: 217193+0000 mon.a (mon.0) 750 : audit [INF] 2023-10-01T09:06:13.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: from='mgr.25342 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:06:13.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: cluster 2023-10-01T09:06:12.217290+0000 2023-10-01T09:06:13.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: mon.a (mon.0) 751 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-01T09:06:13.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: 2023-10-01T09:06:13.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: audit 2023-10-01T09:06:13.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: 2023-10-01T09:06:12.228600+0000 mon.b 2023-10-01T09:06:13.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: (mon.1) 56 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:06:13.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:06:13.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: 01T09:06:12.229118+0000 mon.a (mon. 2023-10-01T09:06:13.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[109959]: 0) 752 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:06:13.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:11.619396+0000 2023-10-01T09:06:13.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: mgr.x (mgr.25342) 1 : cephadm [INF] Deploying cephadm binary to smithi138 2023-10-01T09:06:13.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:11.620081+0000 mgr.x (mgr.25342) 2 : cephadm 2023-10-01T09:06:13.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: [INF] Deploying cephadm binary to smithi190 2023-10-01T09:06:13.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:06:13.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: 10-01T09:06:11.869796+0000 mon.a (mon.0) 747 : audit [INF] 2023-10-01T09:06:13.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:13.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: audit 2023-10-01T09:06:11 2023-10-01T09:06:13.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: .885226+0000 mon.a (mon.0) 748 : audit [INF] 2023-10-01T09:06:13.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:13.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: cluster 2023-10-01T09: 2023-10-01T09:06:13.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: 06:12.035735+0000 mon.a (mon.0) 749 : cluster 2023-10-01T09:06:13.021 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: [DBG] mgrmap e21: x(active, since 1.00899s) 2023-10-01T09:06:13.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:06:13.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: 01T09:06:12.217193+0000 mon.a (mon.0 2023-10-01T09:06:13.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: ) 750 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:06:13.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: cluster 2023-10 2023-10-01T09:06:13.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: -01T09:06:12.217290+0000 mon.a ( 2023-10-01T09:06:13.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: mon.0) 751 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-01T09:06:13.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: audit 2023-10-01T09:06:13.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: 2023-10-01T09:06:12.228600+0000 mon.b 2023-10-01T09:06:13.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: (mon.1) 56 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:06:13.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:06:13.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: 10-01T09:06:12.229118+0000 mon.a (mon 2023-10-01T09:06:13.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:12 smithi138 conmon[100289]: .0) 752 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:06:13.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:12 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:11.619396+0000 mgr.x (mgr.25342) 1 : cephadm 2023-10-01T09:06:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:12 smithi190 conmon[100518]: [INF] Deploying cephadm binary to smithi138 2023-10-01T09:06:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:12 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:11.620081+0000 mgr.x (mgr.25342) 2 : cephadm [INF] Deploying cephadm binary to smithi190 2023-10-01T09:06:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:12 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.869796+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:12 smithi190 conmon[100518]: audit 2023-10-01T09:06:11.885226+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:12 smithi190 conmon[100518]: cluster 2023-10-01T09:06:12.035735+0000 mon.a (mon.0) 749 : cluster [DBG] mgrmap e21: x(active, since 1.00899s) 2023-10-01T09:06:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:12 smithi190 conmon[100518]: audit 2023-10-01T09:06:12.217193+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:06:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:12 smithi190 conmon[100518]: cluster 2023-10-01T09:06:12.217290+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-10-01T09:06:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:12 smithi190 conmon[100518]: audit 2023-10-01T09:06:12.228600+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:06:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:12 smithi190 conmon[100518]: audit 2023-10-01T09:06:12.229118+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:06:13.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100738]: debug 2023-10-01T09:06:13.016+0000 7f041ba0b680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T09:06:13.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100738]: [01/Oct/2023:09:06:13] ENGINE Bus STARTING 2023-10-01T09:06:13.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100738]: CherryPy Checker: 2023-10-01T09:06:13.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100738]: The Application mounted at '' has an empty config. 2023-10-01T09:06:13.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100738]: 2023-10-01T09:06:13.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100738]: [01/Oct/2023:09:06:13] ENGINE Serving on http://:::9283 2023-10-01T09:06:13.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100738]: [01/Oct/2023:09:06:13] ENGINE Bus STARTED 2023-10-01T09:06:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: cluster 2023-10-01T09:06:12.052084+0000 mgr.x (mgr.25342) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: cephadm 2023-10-01T 2023-10-01T09:06:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: 09:06:12.292646+0000 mgr.x (mgr.25342) 4 : cephadm [INF] [01/Oct/2023:09:06:12] ENGINE Bus STARTING 2023-10-01T09:06:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:12.417408+0000 mgr.x (mgr.25342) 5 : cephadm [INF] [01/Oct/2023:09:06:12] ENGINE Serving on http://172.21.15.190:8765 2023-10-01T09:06:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:12.537810+0000 mgr.x (mgr.25342) 6 : cephadm [INF] [01/Oct/2023:09:06:12] ENGINE Serving on https://172.21.15.190:7150 2023-10-01T09:06:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:12.537910+0000 mgr.x (mgr.25342) 7 : cephadm [INF] [01/Oct/2023:09:06:12] ENGINE Bus STARTED 2023-10-01T09:06:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: audit 2023-10-01T09:06:13.023137+0000 mon.c (mon.2) 132 : audit [DBG] from='client.? 172.21.15.138:0/2143206260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: cluster 2023-10-01T09:06:13.031871+0000 mon.a (mon.0) 753 : cluster [DBG] Standby manager daemon y started 2023-10-01T09:06:14.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: audit 2023-10-01T09:06:13.034441+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.? 172.21.15.138:0/2069158267' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T09:06:14.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: audit 2023-10-01T09:06:13.038997+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.? 172.21.15.138:0/2069158267' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:06:14.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: cluster 2023-10-01T09:06:13.039514+0000 mgr.x (mgr.25342) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:14.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: audit 2023-10-01T09:06:13.039998+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.? 172.21.15.138:0/2069158267' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-01T09:06:14.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: audit 2023-10-01T09:06:13.040326+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.? 172.21.15.138:0/2069158267' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:06:14.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: audit 2023-10-01T09:06:13.220652+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:06:14.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: cluster 2023-10-01T09:06:13.220730+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-01T09:06:14.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:13 smithi190 conmon[100518]: audit 2023-10-01T09:06:13.864150+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:14.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: cluster 2023-10-01T09:06:12.052084+0000 2023-10-01T09:06:14.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: mgr.x (mgr.25342) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:14.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:12.292646+0000 mgr.x (mgr.25342) 4 : cephadm [INF] [01/Oct/2023:09:06:12] ENGINE Bus STARTING 2023-10-01T09:06:14.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:12.417408+0000 mgr.x (mgr.25342) 5 : cephadm [INF] [01/Oct/2023:09:06:12] ENGINE Serving on http://172.21.15.190:8765 2023-10-01T09:06:14.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:12.537810+0000 mgr.x (mgr.25342) 6 : cephadm [INF] [01/Oct/2023:09:06:12] ENGINE Serving on https://172.21.15.190:7150 2023-10-01T09:06:14.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:12.537910+0000 mgr.x (mgr.25342) 7 : cephadm [INF] [01/Oct/2023:09:06:12] ENGINE Bus STARTED 2023-10-01T09:06:14.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: audit 2023-10-01T09:06:13.023137+0000 mon.c (mon.2) 132 : audit [DBG] from='client.? 172.21.15.138:0/2143206260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:14.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: cluster 2023-10-01T09:06:13.031871+0000 mon.a (mon.0) 753 : cluster [DBG] Standby manager daemon y started 2023-10-01T09:06:14.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: audit 2023-10-01T09:06:13.034441+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.? 172.21.15.138:0/2069158267' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T09:06:14.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: audit 2023-10-01T09:06:13.038997+0000 mon.c (mon.2) 134 : audit [DBG] from='mgr.? 172.21.15.138:0/2069158267' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:06:14.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: cluster 2023-10-01T09:06:13.039514+0000 mgr.x (mgr.25342) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:14.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: audit 2023-10-01T09:06:13.039998+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.? 172.21.15.138:0/2069158267' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-01T09:06:14.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: audit 2023-10-01T09:06:13.040326+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.? 172.21.15.138:0/2069158267' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:06:14.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: audit 2023-10-01T09:06:13.220652+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:06:14.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: cluster 2023-10-01T09:06:13.220730+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-01T09:06:14.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[100289]: audit 2023-10-01T09:06:13.864150+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: cluster 2023-10-01T09:06 2023-10-01T09:06:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: :12.052084+0000 mgr.x (mgr.25342) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: cephadm 2023-10- 2023-10-01T09:06:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: 01T09:06:12.292646+0000 mgr.x (mgr.25342) 4 : cephadm [INF] [01/Oct/2023:09:06:12] ENGINE Bus STARTING 2023-10-01T09:06:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: cephadm 2023-10-01T 2023-10-01T09:06:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: 09:06:12.417408+0000 mgr.x (mgr.25342) 5 : cephadm [INF] [01/Oct/2023:09:06:12] ENGINE Serving on http://172.21.15.190:8765 2023-10-01T09:06:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: cephadm 2023-10-01T09:06:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: 2023-10-01T09:06:12.537810+0000 mgr.x (mgr.25342) 6 : cephadm [INF] [01/Oct/2023:09:06:12] ENGINE Serving on https://172.21.15.190:7150 2023-10-01T09:06:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: cephadm 2023-10-01T09:06:12.537910+0000 2023-10-01T09:06:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: mgr.x (mgr.25342) 7 : cephadm [INF] [01/Oct/2023:09:06:12] ENGINE Bus STARTED 2023-10-01T09:06:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:06:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: T09:06:13.023137+0000 mon.c (mon.2) 132 : audit [DBG] from='client.? 172.21.15.138:0/2143206260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: cluster 2023-10-01T09:06:13.031871+0000 mon.a (mon 2023-10-01T09:06:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: .0) 753 : cluster [DBG] Standby manager daemon y started 2023-10-01T09:06:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: audit 2023-10-01T09:06:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: 2023-10-01T09:06:13.034441+0000 mon.c (mon.2 2023-10-01T09:06:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: ) 133 : audit [DBG] from='mgr.? 172.21.15.138:0/2069158267' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T09:06:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:06:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: 06:13.038997+0000 mon.c (mon.2) 2023-10-01T09:06:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: 134 : audit [DBG] from='mgr.? 172.21.15.138:0/2069158267' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:06:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: cluster 2023-10-01T09: 2023-10-01T09:06:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: 06:13.039514+0000 mgr.x (mgr.25342) 2023-10-01T09:06:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:06:14.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: 09:06:13.039998+0000 mon.c (mon. 2023-10-01T09:06:14.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: 2) 135 : audit [DBG] from='mgr.? 172.21.15.138:0/2069158267' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-01T09:06:14.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:06:14.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: T09:06:13.040326+0000 mon.c (mon 2023-10-01T09:06:14.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: .2) 136 : audit [DBG] from='mgr.? 172.21.15.138:0/2069158267' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:06:14.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:06:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: 01T09:06:13.220652+0000 mon.a ( 2023-10-01T09:06:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: mon.0) 754 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:06:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: cluster 2023- 2023-10-01T09:06:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: 10-01T09:06:13.220730+0000 2023-10-01T09:06:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: mon.a (mon.0) 755 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-10-01T09:06:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: 2023-10-01T09:06:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:06:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: 10-01T09:06:13.864150+0000 mon.a (mon.0 2023-10-01T09:06:14.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:13 smithi138 conmon[109959]: ) 756 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:15.186 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:14 smithi138 conmon[100289]: cluster 2023-10-01T09:06:13.879136+0000 mon.a (mon.0) 757 : cluster [DBG] mgrmap e22: x(active, since 2s), standbys: y 2023-10-01T09:06:15.186 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:14 smithi138 conmon[100289]: audit 2023-10-01T09:06:13.881450+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T09:06:15.186 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:14 smithi138 conmon[109959]: cluster 2023-10-01T09:06:13. 2023-10-01T09:06:15.186 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:14 smithi138 conmon[109959]: 879136+0000 mon.a (mon.0) 757 : cluster [DBG] mgrmap e22: x(active, since 2s), standbys: y 2023-10-01T09:06:15.186 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:14 smithi138 conmon[109959]: audit 2023-10-01T09:06: 2023-10-01T09:06:15.186 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:14 smithi138 conmon[109959]: 13.881450+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T09:06:15.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:14 smithi190 conmon[100518]: cluster 2023-10-01T09:06:13.879136+0000 mon.a (mon. 2023-10-01T09:06:15.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:14 smithi190 conmon[100518]: 0) 757 : cluster [DBG] mgrmap e22: x(active, since 2s), standbys: y 2023-10-01T09:06:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:14 smithi190 conmon[100518]: audit 2023-10-01T09:06:13.881450+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T09:06:15.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:15 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:06:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:06:16.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:15 smithi190 conmon[100518]: cluster 2023-10-01T09:06:15.039839+0000 mgr.x (mgr.25342) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:15 smithi190 conmon[100518]: audit 2023-10-01T09:06:15.390250+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:15 smithi190 conmon[100518]: audit 2023-10-01T09:06:15.394964+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:15 smithi190 conmon[100518]: audit 2023-10-01T09:06:15.450402+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.138:0/3403644251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:16.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:15 smithi138 conmon[109959]: cluster 2023-10-01T09:06:15.039839+0000 mgr.x (mgr.25342) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:16.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:15 smithi138 conmon[109959]: 2023-10-01T09:06:16.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:15 smithi138 conmon[109959]: audit 2023-10-01T09:06:15.390250+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:16.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:15 smithi138 conmon[109959]: audit 2023-10-01T09:06:15.394964+0000 mon.a ( 2023-10-01T09:06:16.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:15 smithi138 conmon[109959]: mon.0) 759 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:16.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:15 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:06:16.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:15 smithi138 conmon[109959]: -01T09:06:15.450402+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.138:0/3403644251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:16.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:15 smithi138 conmon[100289]: cluster 2023-10-01T09:06:15.039839+0000 mgr.x (mgr.25342) 2023-10-01T09:06:16.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:15 smithi138 conmon[100289]: 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:16.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:15 smithi138 conmon[100289]: audit 2023-10-01T09:06:15.390250+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:16.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:15 smithi138 conmon[100289]: audit 2023-10-01T09:06:15.394964+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:16.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:15 smithi138 conmon[100289]: audit 2023-10-01T09:06:15.450402+0000 mon.c (mon.2) 137 : audit [DBG] from='client.? 172.21.15.138:0/3403644251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:17.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[139437]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:06:16] "GET /metrics HTTP/1.1" 200 30909 "" "Prometheus/2.33.4" 2023-10-01T09:06:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: cluster 2023-10-01T09:06:15.887589+0000 mon.a (mon. 2023-10-01T09:06:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: 0) 760 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-01T09:06:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: audit 2023-10-01T09:06:16.033477+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: audit 2023-10-01T09:06:16.039281+0000 mon.a (mon.0) 762 : audit 2023-10-01T09:06:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:06:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: -01T09:06:16.052855+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: audit 2023-10-01T09:06:16.129958 2023-10-01T09:06:17.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: +0000 mon.a (mon.0) 764 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: audit 2023-10-01T09:06:16.137211+0000 mon.a 2023-10-01T09:06:17.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: (mon.0) 765 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: audit 2023-10-01T09:06:16.824035+0000 2023-10-01T09:06:17.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: mon.a (mon.0) 766 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: audit 2023-10-01T09:06:16.828957+0000 mon.a 2023-10-01T09:06:17.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:16 smithi190 conmon[100518]: (mon.0) 767 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[100289]: cluster 2023-10-01T09:06:15.887589+0000 2023-10-01T09:06:17.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[100289]: mon.a (mon.0) 760 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-01T09:06:17.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[100289]: audit 2023-10-01T09:06:16.033477+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[100289]: audit 2023-10-01T09:06:16.039281+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[100289]: audit 2023-10-01T09:06:16.052855+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[100289]: audit 2023-10-01T09:06:16.129958+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[100289]: audit 2023-10-01T09:06:16.137211+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[100289]: audit 2023-10-01T09:06:16.824035+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[100289]: audit 2023-10-01T09:06:16.828957+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[109959]: cluster 2023-10-01T09:06:15.887589+0000 mon.a (mon.0) 760 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-01T09:06:17.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[109959]: audit 2023-10-01T09:06:16.033477+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[109959]: audit 2023-10-01T09:06:16.039281+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[109959]: audit 2023-10-01T09:06:16.052855+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[109959]: audit 2023-10-01T09:06:16.129958+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[109959]: audit 2023-10-01T09:06:16.137211+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[109959]: audit 2023-10-01T09:06:16.824035+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:17.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:16 smithi138 conmon[109959]: audit 2023-10-01T09:06:16.828957+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:18.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:18 smithi138 conmon[100289]: cluster 2023-10-01T09:06:17.040170+0000 mgr.x (mgr.25342) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:18.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:18 smithi138 conmon[100289]: audit 2023-10-01T09:06:17.635816+0000 mon.a (mon.0) 768 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T09:06:18.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:18 smithi138 conmon[109959]: cluster 2023-10-01T09:06:17.040170+0000 mgr.x (mgr.25342) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:18.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:18 smithi138 conmon[109959]: audit 2023-10-01T09:06:17.635816+0000 mon.a (mon.0) 768 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T09:06:18.575 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:18 smithi190 conmon[100518]: cluster 2023-10-01T09:06:17.040170+0000 mgr.x (mgr.25342) 2023-10-01T09:06:18.575 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:18 smithi190 conmon[100518]: 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:18.575 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:18 smithi190 conmon[100518]: audit 2023-10-01T09:06:17.635816+0000 mon.a (mon.0) 768 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T09:06:19.465 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: audit 2023-10-01T09:06:17.636113+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:06:19.465 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: audit 2023-10-01T09:06:17.636306+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:06:19.465 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: audit 2023-10-01T09:06:17.636520+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:06:19.465 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: audit 2023-10-01T09:06:17.842872+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.138:0/2968159471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:19.465 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: audit 2023-10-01T09:06:18.161530+0000 mon.a (mon.0) 769 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T09:06:19.465 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: audit 2023-10-01T09:06:18.170725+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:06:19.466 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: audit 2023-10-01T09:06:18.170812+0000 mon.b (mon.1) 62 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T09:06:19.466 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: 2023-10-01T09:06:19.466 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: audit 2023-10-01T09:06:18.684439+0000 mon.b (mon.1) 63 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T09:06:19.466 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: audit 2023-10-01T09:06:18.684930+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:06:19.466 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: audit 2023-10-01T09:06:18.685113+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:06:19.466 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: audit 2023-10-01T09:06:18.686628+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:06:19.466 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: audit 2023-10-01T09:06:18.686810+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:06:19.467 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: audit 2023-10-01T09:06:18.686964+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:06:19.467 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:19 smithi190 conmon[100518]: cluster 2023-10-01T09:06:19.040664+0000 mgr.x (mgr.25342) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 308 KiB/s wr, 35 op/s 2023-10-01T09:06:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: audit 2023-10-01T09:06:17.636113+0000 mon.b ( 2023-10-01T09:06:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: mon.1) 58 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:06:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: audit 2023-10-01T09:06:17.636306+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:06:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: audit 2023-10-01T09:06:17.636520+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:06:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: audit 2023-10-01T09:06:17.842872+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.138:0/2968159471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: audit 2023-10-01T09:06:18.161530+0000 mon.a (mon.0) 769 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T09:06:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: audit 2023-10-01T09:06:18.170725+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:06:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: audit 2023-10-01T09:06:18.170812+0000 mon.b (mon.1) 62 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T09:06:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: audit 2023-10-01T09:06:18.684439+0000 mon.b (mon.1) 63 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T09:06:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: audit 2023-10-01T09:06:18.684930+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:06:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: audit 2023-10-01T09:06:18.685113+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:06:19.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: audit 2023-10-01T09:06:18.686628+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:06:19.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: audit 2023-10-01T09:06:18.686810+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:06:19.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: audit 2023-10-01T09:06:18.686964+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:06:19.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[100289]: cluster 2023-10-01T09:06:19.040664+0000 mgr.x (mgr.25342) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 308 KiB/s wr, 35 op/s 2023-10-01T09:06:19.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: audit 2023-10-01T09:06:17.636113+0000 2023-10-01T09:06:19.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: mon.b (mon.1) 58 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:06:19.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: audit 2023-10-01T09:06:17.636306 2023-10-01T09:06:19.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: +0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:06:19.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:06:19.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: 06:17.636520+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:06:19.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:06:19.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: T09:06:17.842872+0000 mon.c (mon 2023-10-01T09:06:19.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: .2) 138 : audit [DBG] from='client.? 172.21.15.138:0/2968159471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:19.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: audit 2023-10-01T09:06 2023-10-01T09:06:19.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: :18.161530+0000 mon.a (mon.0) 769 : audit 2023-10-01T09:06:19.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T09:06:19.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:06:19.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: 09:06:18.170725+0000 mon.b (mon. 2023-10-01T09:06:19.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: 1) 61 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:06:19.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:06:19.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: :06:18.170812+0000 mon.b (mon.1) 62 : audit 2023-10-01T09:06:19.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T09:06:19.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: audit 2023 2023-10-01T09:06:19.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: -10-01T09:06:18.684439+0000 mon.b (mon.1) 63 : 2023-10-01T09:06:19.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T09:06:19.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: audit 2023-10-01T09:06:19.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: 2023-10-01T09:06:18.684930+0000 mon.b (mon.1) 64 : audit 2023-10-01T09:06:19.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:06:19.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:06:19.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: 10-01T09:06:18.685113+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:06:19.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:06:19.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: 06:18.686628+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:06:19.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:06:19.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: 06:18.686810+0000 mon.b (mon.1) 67 : audit 2023-10-01T09:06:19.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:06:19.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: audit 2023-10-01T09:06:19.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: 2023-10-01T09:06:18.686964+0000 mon.b (mon 2023-10-01T09:06:19.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: .1) 68 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:06:19.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: cluster 2023- 2023-10-01T09:06:19.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: 10-01T09:06:19.040664+0000 mgr.x ( 2023-10-01T09:06:19.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: mgr.25342) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 308 KiB/s wr, 35 op/s 2023-10-01T09:06:19.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:19 smithi138 conmon[109959]: 2023-10-01T09:06:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:18 2023-10-01T09:06:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: .686463+0000 mon.c (mon.2) 139 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T09:06:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:19.196213+0000 mon.c (mon.2) 140 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T09:06:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:19.315791+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:19.321694+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:19.323148+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:19.323384+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:19.324174+0000 mon.b (mon.1) 70 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:19.324363+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:19.325047+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:19.325253+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:19.325944+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:19.326193+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:19.326789+0000 mgr.x (mgr.25342) 12 : cephadm [INF] Adjusting osd_memory_target on smithi138 to 2500M 2023-10-01T09:06:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:19.330658+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:20.006924+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:20.013044+0000 mon.a ( 2023-10-01T09:06:20.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: mon.0) 778 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:20.014392+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:20.014645+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:20.015460+0000 mon.b (mon.1) 74 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:20.015649+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:06:20.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: :06:20.016391+0000 mon.b (mon.1) 75 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:06:20.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: :06:20.016582+0000 mon.a (mon.0) 781 : audit [INF] 2023-10-01T09:06:20.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:06:20.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: :06:20.017288+0000 mon.b (mon.1) 76 : 2023-10-01T09:06:20.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:20.017468+0000 mon.a (mon 2023-10-01T09:06:20.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: .0) 782 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:20.018044+0000 2023-10-01T09:06:20.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: mgr.x (mgr.25342) 13 : cephadm [INF] Adjusting osd_memory_target on smithi190 to 2500M 2023-10-01T09:06:20.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:20.022009+0000 mon.a (mon.0 2023-10-01T09:06:20.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: ) 783 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:20.023373+0000 mon.b 2023-10-01T09:06:20.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: (mon.1) 77 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:06:20.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: audit 2023-10-01T09:06:20.023977 2023-10-01T09:06:20.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: +0000 mon.b (mon.1) 78 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:06:20.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: cephadm 2023-10-01T09 2023-10-01T09:06:20.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: :06:20.024893+0000 mgr.x (mgr.25342) 14 : 2023-10-01T09:06:20.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:06:20.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: cephadm 2023 2023-10-01T09:06:20.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: -10-01T09:06:20.025078+0000 2023-10-01T09:06:20.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: mgr.x (mgr.25342) 15 : cephadm [INF] 2023-10-01T09:06:20.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:20 smithi190 conmon[100518]: Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:06:20.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:18.686463+0000 mon.c ( 2023-10-01T09:06:20.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: mon.2) 139 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T09:06:20.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:06:20.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: T09:06:19.196213+0000 mon.c (mon.2) 140 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T09:06:20.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:06:20.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 10-01T09:06:19.315791+0000 mon.a ( 2023-10-01T09:06:20.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: mon.0) 770 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:06:20.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 06:19.321694+0000 mon.a (mon.0) 771 : 2023-10-01T09:06:20.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:20.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 2023-10-01T09:06:19.323148+0000 mon.b (mon.1 2023-10-01T09:06:20.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: ) 69 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:19 2023-10-01T09:06:20.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: .323384+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:06:20.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: T09:06:19.324174+0000 mon.b (mon.1) 2023-10-01T09:06:20.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 70 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:06:20.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: :06:19.324363+0000 mon.a (mon.0) 2023-10-01T09:06:20.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 773 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06: 2023-10-01T09:06:20.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 19.325047+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:06:20.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 01T09:06:19.325253+0000 mon.a (mon.0) 2023-10-01T09:06:20.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 774 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:20.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 2023-10-01T09:06:19.325944+0000 mon.b (mon.1 2023-10-01T09:06:20.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: ) 72 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:19. 2023-10-01T09:06:20.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 326193+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: cephadm 2023-10- 2023-10-01T09:06:20.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 01T09:06:19.326789+0000 mgr.x (mgr.25342) 12 2023-10-01T09:06:20.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: : cephadm [INF] Adjusting osd_memory_target on smithi138 to 2500M 2023-10-01T09:06:20.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:20.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 2023-10-01T09:06:19.330658+0000 mon.a (mon.0) 776 2023-10-01T09:06:20.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:20.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 2023-10-01T09:06:20.006924+0000 mon.a (mon.0) 2023-10-01T09:06:20.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:18.686463+0000 2023-10-01T09:06:20.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: mon.c (mon.2) 139 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T09:06:20.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:19.196213+0000 mon.c (mon.2) 140 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T09:06:20.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:19.315791+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:19.321694+0000 mon.a ( 2023-10-01T09:06:20.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: mon.0) 771 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:19.323148+0000 2023-10-01T09:06:20.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: mon.b (mon.1) 69 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:06:20.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: :06:19.323384+0000 mon.a (mon.0) 772 : 2023-10-01T09:06:20.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:19. 2023-10-01T09:06:20.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 324174+0000 mon.b (mon.1) 70 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 2023-10-01T09:06:20.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:06:20.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 10-01T09:06:19.324363+0000 mon.a (mon.0) 773 : audit 2023-10-01T09:06:20.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:20.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 2023-10-01T09:06:19.325047+0000 mon.b ( 2023-10-01T09:06:20.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: mon.1) 71 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:20.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 2023-10-01T09:06:19.325253+0000 mon.a ( 2023-10-01T09:06:20.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: mon.0) 774 : audit [INF] 2023-10-01T09:06:20.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:20.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 2023-10-01T09:06:19.325944+0000 2023-10-01T09:06:20.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: mon.b (mon.1) 72 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:06:20.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 01T09:06:19.326193+0000 mon.a (mon.0 2023-10-01T09:06:20.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: ) 775 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: cephadm 2023-10-01T09: 2023-10-01T09:06:20.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 06:19.326789+0000 mgr.x (mgr.25342) 12 2023-10-01T09:06:20.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: : cephadm [INF] Adjusting osd_memory_target on smithi138 to 2500M 2023-10-01T09:06:20.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:20.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 2023-10-01T09:06: 2023-10-01T09:06:20.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 19.330658+0000 mon.a (mon.0) 776 : audit 2023-10-01T09:06:20.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.578 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:06:20.578 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 09:06:20.006924+0000 mon.a (mon.0 2023-10-01T09:06:20.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 777 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:20.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 2023-10-01T09:06:20.013044+0000 mon.a (mon.0) 2023-10-01T09:06:20.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 778 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:20.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 2023-10-01T09:06:20.014392+0000 mon.b (mon. 2023-10-01T09:06:20.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 1) 73 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:20 2023-10-01T09:06:20.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: .014645+0000 mon.a (mon.0) 779 : audit [INF] 2023-10-01T09:06:20.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023 2023-10-01T09:06:20.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: -10-01T09:06:20.015460+0000 mon.b (mon.1) 74 : 2023-10-01T09:06:20.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:20.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 2023-10-01T09:06:20.015649+0000 mon.a (mon.0) 780 2023-10-01T09:06:20.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:20.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 2023-10-01T09:06:20.016391+0000 mon.b (mon.1) 2023-10-01T09:06:20.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 75 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:20.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 2023-10-01T09:06:20.016582+0000 mon.a (mon.0) 781 2023-10-01T09:06:20.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:20.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 2023-10-01T09:06:20.017288+0000 mon.b (mon.1) 2023-10-01T09:06:20.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 76 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:20.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 2023-10-01T09:06:20.017468+0000 mon.a (mon.0) 782 2023-10-01T09:06:20.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: cephadm 2023-10-01T09:06:20.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 2023-10-01T09:06:20.018044+0000 mgr.x (mgr.25342) 2023-10-01T09:06:20.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 13 : cephadm [INF] Adjusting osd_memory_target on smithi190 to 2500M 2023-10-01T09:06:20.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06:20 2023-10-01T09:06:20.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: .022009+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:06:20.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 10-01T09:06:20.023373+0000 mon.b (mon 2023-10-01T09:06:20.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: .1) 77 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:06:20.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: audit 2023-10-01T09:06: 2023-10-01T09:06:20.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 20.023977+0000 mon.b (mon.1) 78 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:06:20.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: cephadm 2023-10- 2023-10-01T09:06:20.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 01T09:06:20.024893+0000 mgr.x (mgr.25342) 2023-10-01T09:06:20.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 14 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:06:20.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: cephadm 2023-10-01T09:06:20.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: 2023-10-01T09:06:20.025078+0000 mgr.x (mgr.25342 2023-10-01T09:06:20.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[109959]: ) 15 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:06:20.583 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: ) 777 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.583 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:06:20.583 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 10-01T09:06:20.013044+0000 mon.a ( 2023-10-01T09:06:20.583 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: mon.0) 778 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.583 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:20.583 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 2023-10-01T 2023-10-01T09:06:20.583 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 09:06:20.014392+0000 mon.b (mon. 2023-10-01T09:06:20.583 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 1) 73 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.583 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:20.583 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 2023-10-01T09:06 2023-10-01T09:06:20.584 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: :20.014645+0000 mon.a (mon.0) 779 : audit 2023-10-01T09:06:20.584 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.584 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023 2023-10-01T09:06:20.584 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: -10-01T09:06:20.015460+0000 2023-10-01T09:06:20.584 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: mon.b (mon.1) 74 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.584 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:20.584 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 2023-10-01T09:06: 2023-10-01T09:06:20.584 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 20.015649+0000 mon.a (mon.0) 780 : 2023-10-01T09:06:20.584 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.584 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10 2023-10-01T09:06:20.584 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: -01T09:06:20.016391+0000 mon.b 2023-10-01T09:06:20.585 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: (mon.1) 75 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.585 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:20.585 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 2023-10-01 2023-10-01T09:06:20.585 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: T09:06:20.016582+0000 mon.a (mon. 2023-10-01T09:06:20.585 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 0) 781 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.585 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023 2023-10-01T09:06:20.585 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: -10-01T09:06:20.017288+0000 2023-10-01T09:06:20.585 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: mon.b (mon.1) 76 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.585 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:20.585 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 2023-10-01T09:06:20.017468 2023-10-01T09:06:20.585 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: +0000 mon.a (mon.0) 782 : audit [INF] 2023-10-01T09:06:20.585 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T09:06:20.586 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: cephadm 2023-10-01T09 2023-10-01T09:06:20.586 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: :06:20.018044+0000 mgr.x (mgr.25342 2023-10-01T09:06:20.586 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: ) 13 : cephadm [INF] Adjusting osd_memory_target on smithi190 to 2500M 2023-10-01T09:06:20.586 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10-01T09:06:20.586 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 2023-10-01T09 2023-10-01T09:06:20.586 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: :06:20.022009+0000 mon.a (mon.0 2023-10-01T09:06:20.586 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: ) 783 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:20.586 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023 2023-10-01T09:06:20.586 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: -10-01T09:06:20.023373 2023-10-01T09:06:20.586 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: +0000 mon.b (mon.1) 77 : 2023-10-01T09:06:20.587 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:06:20.587 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: audit 2023-10 2023-10-01T09:06:20.587 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: -01T09:06:20.023977 2023-10-01T09:06:20.587 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: +0000 mon.b (mon.1) 78 2023-10-01T09:06:20.587 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:06:20.587 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: cephadm 2023 2023-10-01T09:06:20.587 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: -10-01T09:06:20. 2023-10-01T09:06:20.587 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 024893+0000 mgr.x (mgr.25342) 2023-10-01T09:06:20.587 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 14 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:06:20.587 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: cephadm 2023 2023-10-01T09:06:20.587 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: -10-01T09:06:20. 2023-10-01T09:06:20.587 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 025078+0000 mgr.x (mgr.25342) 2023-10-01T09:06:20.588 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:20 smithi138 conmon[100289]: 15 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:06:21.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: audit 2023-10-01T09:06:20.231980+0000 mon.c (mon 2023-10-01T09:06:21.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: .2) 141 : audit [DBG] from='client.? 172.21.15.138:0/10717426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:21.472 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: cephadm 2023-10-01T09:06 2023-10-01T09:06:21.472 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: :20.289003+0000 mgr.x (mgr.25342) 16 : cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:06:21.472 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: cephadm 2023-10-01T09:06:21.472 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: 2023-10-01T09:06:20.289159+0000 mgr.x (mgr.25342 2023-10-01T09:06:21.472 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: ) 17 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:06:21.472 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: cephadm 2023-10-01T09:06:21.472 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: 2023-10-01T09:06:20.553697+0000 mgr.x (mgr.25342) 2023-10-01T09:06:21.472 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: 18 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:06:21.472 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: cephadm 2023-10-01T09:06:20.553846 2023-10-01T09:06:21.472 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: +0000 mgr.x (mgr.25342) 19 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:06:21.472 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: cephadm 2023-10-01 2023-10-01T09:06:21.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: T09:06:20.818029+0000 mgr.x (mgr.25342) 20 : cephadm [INF] 2023-10-01T09:06:21.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:06:21.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: cephadm 2023-10-01 2023-10-01T09:06:21.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: T09:06:20.818493+0000 mgr.x (mgr.25342) 21 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:06:21.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: 2023-10-01T09:06:21.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: cluster 2023-10-01T09 2023-10-01T09:06:21.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: :06:21.041034+0000 mgr.x (mgr.25342) 22 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 244 KiB/s wr, 28 op/s 2023-10-01T09:06:21.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: audit 2023-10-01T09:06 2023-10-01T09:06:21.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: :21.086608+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:06:21.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: 09:06:21.089331+0000 mon.a (mon.0) 2023-10-01T09:06:21.474 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: 785 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.474 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: audit 2023-10-01T09:06:21.095496+0000 mon.a ( 2023-10-01T09:06:21.474 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: mon.0) 786 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.474 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: audit 2023-10-01T09:06:21.099978+0000 2023-10-01T09:06:21.474 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: mon.a (mon.0) 787 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.474 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: audit 2023-10-01T09:06: 2023-10-01T09:06:21.474 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: 21.106318+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.474 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: audit 2023-10-01T09:06:21.474 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: 2023-10-01T09: 2023-10-01T09:06:21.474 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: 06:21.179061+0000 mon.a (mon.0) 789 : 2023-10-01T09:06:21.474 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.475 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: audit 2023-10-01T09:06:21.475 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: 2023-10-01T09:06:21.184999+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.475 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: audit 2023-10-01T09:06:21 2023-10-01T09:06:21.475 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: .190519+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.475 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: audit 2023-10-01T09:06:21. 2023-10-01T09:06:21.475 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[109959]: 196153+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.475 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: audit 2023-10-01T09:06:20.231980+0000 mon.c ( 2023-10-01T09:06:21.475 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: mon.2) 141 : audit [DBG] from='client.? 172.21.15.138:0/10717426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:21.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:20.289003+0000 mgr.x 2023-10-01T09:06:21.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: (mgr.25342) 16 : cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:06:21.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:20.289159+0000 2023-10-01T09:06:21.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: mgr.x (mgr.25342) 17 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:06:21.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: 2023-10-01T09:06:21.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:20. 2023-10-01T09:06:21.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: 553697+0000 mgr.x (mgr.25342) 18 : cephadm [INF] 2023-10-01T09:06:21.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:06:21.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: cephadm 2023 2023-10-01T09:06:21.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: -10-01T09:06:20.553846+0000 2023-10-01T09:06:21.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: mgr.x (mgr.25342) 19 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:06:21.477 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: 2023-10-01T09:06:21.477 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: cephadm 2023-10-01T09:06: 2023-10-01T09:06:21.477 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: 20.818029+0000 mgr.x (mgr.25342) 20 : 2023-10-01T09:06:21.477 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:06:21.477 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: cephadm 2023-10- 2023-10-01T09:06:21.477 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: 01T09:06:20.818493+0000 mgr.x ( 2023-10-01T09:06:21.477 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: mgr.25342) 21 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:06:21.477 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: cluster 2023-10-01T09:06:21.477 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: 2023-10-01T09:06 2023-10-01T09:06:21.478 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: :21.041034+0000 mgr.x (mgr.25342) 2023-10-01T09:06:21.478 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: 22 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 244 KiB/s wr, 28 op/s 2023-10-01T09:06:21.478 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:06:21.478 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: 10-01T09:06:21.086608+0000 mon.a 2023-10-01T09:06:21.478 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: (mon.0) 784 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.478 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: audit 2023-10-01T09:06:21.478 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: 2023-10-01T09:06:21.089331 2023-10-01T09:06:21.478 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: +0000 mon.a (mon.0) 785 : audit 2023-10-01T09:06:21.478 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.478 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:06:21.478 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: :06:21.095496+0000 mon.a (mon.0 2023-10-01T09:06:21.478 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: ) 786 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.479 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: audit 2023-10-01T09:06:21.479 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: 2023-10-01T09:06:21.099978 2023-10-01T09:06:21.479 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: +0000 mon.a (mon.0) 787 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.479 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: audit 2023-10-01T09:06:21.479 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: 2023-10-01T09:06: 2023-10-01T09:06:21.479 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: 21.106318+0000 mon.a (mon.0) 788 : 2023-10-01T09:06:21.479 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.479 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:06:21.479 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: :06:21.179061+0000 mon.a (mon.0 2023-10-01T09:06:21.479 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: ) 789 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.479 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: audit 2023-10 2023-10-01T09:06:21.479 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: -01T09:06:21.184999+0000 mon.a ( 2023-10-01T09:06:21.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: mon.0) 790 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: audit 2023-10-01T09:06:21.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: 2023-10-01T09:06:21 2023-10-01T09:06:21.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: .190519+0000 mon.a (mon.0) 791 : audit 2023-10-01T09:06:21.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:06:21.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: 09:06:21.196153+0000 mon.a (mon.0 2023-10-01T09:06:21.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:21 smithi138 conmon[100289]: ) 792 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: audit 2023-10-01T09:06:20.231980+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.138:0/10717426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:20.289003+0000 mgr.x (mgr.25342) 16 : cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:06:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:20.289159+0000 mgr.x (mgr. 2023-10-01T09:06:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: 25342) 17 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:06:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: cephadm 2023-10 2023-10-01T09:06:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: -01T09:06:20.553697+0000 mgr.x (mgr.25342) 18 : 2023-10-01T09:06:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:06:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: cephadm 2023-10-01T09:06 2023-10-01T09:06:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: :20.553846+0000 mgr.x (mgr.25342) 19 : 2023-10-01T09:06:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: cephadm [INF] Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:06:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: cephadm 2023-10-01T09:06 2023-10-01T09:06:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: :20.818029+0000 mgr.x (mgr.25342) 20 2023-10-01T09:06:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: : cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:06:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: cephadm 2023-10-01T09: 2023-10-01T09:06:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: 06:20.818493+0000 mgr.x (mgr.25342) 2023-10-01T09:06:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: 21 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:06:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: cluster 2023-10-01T09: 2023-10-01T09:06:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: 06:21.041034+0000 mgr.x (mgr.25342) 2023-10-01T09:06:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: 22 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 244 KiB/s wr, 28 op/s 2023-10-01T09:06:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:06:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: :06:21.086608+0000 mon.a (mon.0 2023-10-01T09:06:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: ) 784 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T09:06:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: 09:06:21.089331+0000 mon.a (mon. 2023-10-01T09:06:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: 0) 785 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T09:06:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: T09:06:21.095496+0000 mon.a (mon 2023-10-01T09:06:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: .0) 786 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:06:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: -01T09:06:21.099978+0000 mon.a ( 2023-10-01T09:06:21.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: mon.0) 787 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:06:21.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: 01T09:06:21.106318+0000 mon.a (mon 2023-10-01T09:06:21.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: .0) 788 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:06:21.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: 01T09:06:21.179061+0000 mon.a ( 2023-10-01T09:06:21.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: mon.0) 789 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:06:21.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: 10-01T09:06:21.184999+0000 2023-10-01T09:06:21.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: mon.a (mon.0) 790 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: 2023-10-01T09:06:21.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: audit 2023-10-01T09:06:21.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: 2023-10-01T09:06:21.190519+0000 mon.a (mon. 2023-10-01T09:06:21.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: 0) 791 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T09:06:21.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: T09:06:21.196153+0000 mon.a (mon. 2023-10-01T09:06:21.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:21 smithi190 conmon[100518]: 0) 792 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:21.819 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:21 smithi138 systemd[1]: Stopping Ceph node-exporter.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:06:21.819 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:21 smithi138 bash[175654]: Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-node-exporter.a found: no such container 2023-10-01T09:06:22.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:22 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:21.197151+0000 mgr.x (mgr.25342) 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-01T09:06:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:22 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:21.197529+0000 mgr.x (mgr.25342) 24 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi138 2023-10-01T09:06:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:22 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:21.197151+0000 2023-10-01T09:06:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:22 smithi138 conmon[100289]: mgr.x (mgr.25342) 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-01T09:06:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:22 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:21.197529+0000 mgr.x (mgr.25342) 24 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi138 2023-10-01T09:06:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:22 smithi138 conmon[109959]: cephadm 2023-10-01T09:06:21.197151+0000 mgr.x (mgr.25342) 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-01T09:06:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:22 smithi138 conmon[109959]: cephadm 2023 2023-10-01T09:06:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:22 smithi138 conmon[109959]: -10-01T09:06:21.197529+0000 mgr.x (mgr.25342) 24 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi138 2023-10-01T09:06:22.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:22 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-01T09:06:22.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:22 smithi138 bash[175770]: 6ec380bfea78636fe9290a596ad4c5cf358802ef7d56e7cf113f480928e4ad3d 2023-10-01T09:06:22.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:22 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-01T09:06:22.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:22 smithi138 systemd[1]: Stopped Ceph node-exporter.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:06:22.821 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:22 smithi138 systemd[1]: Starting Ceph node-exporter.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:06:23.554 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.140Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-01T09:06:23.554 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.140Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-01T09:06:23.554 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.141Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-01T09:06:23.554 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.141Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-01T09:06:23.554 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.141Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-01T09:06:23.555 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.141Z caller=node_exporter.go:115 level=info collector=arp 2023-10-01T09:06:23.555 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.141Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-01T09:06:23.555 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.141Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-01T09:06:23.555 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.141Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-01T09:06:23.555 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.141Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-01T09:06:23.555 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-01T09:06:23.555 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-01T09:06:23.555 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-01T09:06:23.555 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-01T09:06:23.555 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=edac 2023-10-01T09:06:23.555 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-01T09:06:23.556 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-01T09:06:23.556 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-01T09:06:23.556 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-01T09:06:23.556 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-01T09:06:23.556 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-01T09:06:23.556 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-01T09:06:23.556 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-01T09:06:23.556 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-01T09:06:23.556 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-01T09:06:23.556 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-01T09:06:23.556 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-01T09:06:23.556 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-01T09:06:23.557 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-01T09:06:23.557 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-01T09:06:23.557 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-01T09:06:23.557 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=os 2023-10-01T09:06:23.557 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-01T09:06:23.557 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-01T09:06:23.557 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-01T09:06:23.557 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-01T09:06:23.557 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-01T09:06:23.557 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-01T09:06:23.557 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=stat 2023-10-01T09:06:23.557 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-01T09:06:23.558 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-01T09:06:23.558 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-01T09:06:23.558 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=time 2023-10-01T09:06:23.558 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-01T09:06:23.558 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=uname 2023-10-01T09:06:23.558 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-01T09:06:23.558 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-01T09:06:23.558 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-01T09:06:23.558 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-01T09:06:23.558 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[176162]: ts=2023-10-01T09:06:23.142Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-01T09:06:23.558 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 bash[176121]: bd644aa2145be582d032ba451564f02b6818b10b88656fd439d0a01ec214a925 2023-10-01T09:06:23.558 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:06:23 smithi138 systemd[1]: Started Ceph node-exporter.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:06:23.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:23 smithi190 conmon[100518]: audit 2023-10-01T09:06:22.657708+0000 mon.a (mon.0) 793 : audit [DBG] from='client.? 172.21.15.138:0/2650132769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:23 smithi190 conmon[100518]: audit 2023-10-01T09:06:23.192891+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:23 smithi190 conmon[100518]: audit 2023-10-01T09:06:23.198624+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:23.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[100289]: audit 2023-10-01T09:06:22.657708+0000 mon.a (mon.0) 2023-10-01T09:06:23.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[100289]: 793 : audit [DBG] from='client.? 172.21.15.138:0/2650132769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:23.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[100289]: audit 2023-10-01T09:06:23.192891+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:23.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[100289]: audit 2023-10-01T09:06:23.198624+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:23.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[109959]: audit 2023-10-01T09:06:22.657708 2023-10-01T09:06:23.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[109959]: +0000 mon.a (mon.0) 793 : audit [DBG] from='client.? 172.21.15.138:0/2650132769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:23.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:06:23.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[109959]: 10-01T09:06:23.192891+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:23.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:06:23.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[109959]: 09:06:23.198624+0000 mon.a (mon.0 2023-10-01T09:06:23.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:23 smithi138 conmon[109959]: ) 795 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:24 smithi190 conmon[100518]: cluster 2023-10-01T09:06:23.041789+0000 mgr.x (mgr.25342) 25 : 2023-10-01T09:06:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:24 smithi190 conmon[100518]: cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-10-01T09:06:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:24 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:23.215836+0000 mgr.x (mgr.25342) 26 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-01T09:06:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:24 smithi190 conmon[100518]: cephadm 2023-10 2023-10-01T09:06:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:24 smithi190 conmon[100518]: -01T09:06:23.227649+0000 mgr.x (mgr.25342) 27 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi138 2023-10-01T09:06:24.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:24 smithi138 conmon[100289]: cluster 2023-10-01T09:06:23.041789+0000 2023-10-01T09:06:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:24 smithi138 conmon[100289]: mgr.x (mgr.25342) 25 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-10-01T09:06:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:24 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:23.215836+0000 mgr.x (mgr.25342) 26 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-01T09:06:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:24 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:23.227649 2023-10-01T09:06:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:24 smithi138 conmon[100289]: +0000 mgr.x (mgr.25342) 27 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi138 2023-10-01T09:06:24.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:24 smithi138 conmon[109959]: cluster 2023-10-01T09:06:23.041789+0000 mgr.x ( 2023-10-01T09:06:24.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:24 smithi138 conmon[109959]: mgr.25342) 25 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-10-01T09:06:24.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:24 smithi138 conmon[109959]: cephadm 2023-10-01T09:06:23.215836+0000 mgr.x ( 2023-10-01T09:06:24.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:24 smithi138 conmon[109959]: mgr.25342) 26 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-01T09:06:24.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:24 smithi138 conmon[109959]: cephadm 2023-10-01T09:06:24.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:24 smithi138 conmon[109959]: 2023-10-01T09:06:23.227649+0000 mgr.x (mgr. 2023-10-01T09:06:24.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:24 smithi138 conmon[109959]: 25342) 27 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi138 2023-10-01T09:06:25.562 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:25 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:06:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:06:25.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:25 smithi138 conmon[109959]: audit 2023-10-01T09:06:25.079238+0000 mon.c (mon.2) 142 2023-10-01T09:06:25.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:25 smithi138 conmon[109959]: : audit [DBG] from='client.? 172.21.15.138:0/2186306587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:25.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:25 smithi138 conmon[100289]: audit 2023-10-01T09:06:25.079238+0000 mon.c (mon.2) 2023-10-01T09:06:25.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:25 smithi138 conmon[100289]: 142 : audit [DBG] from='client.? 172.21.15.138:0/2186306587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:25 smithi190 conmon[100518]: audit 2023-10-01T09:06:25.079238+0000 mon.c (mon.2) 142 : audit [DBG] from='client.? 172.21.15.138:0/2186306587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:26.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:26 smithi138 conmon[100289]: cluster 2023-10-01T09:06:25.042239+0000 mgr.x (mgr.25342) 28 : cluster [DBG] 2023-10-01T09:06:26.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:26 smithi138 conmon[100289]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-10-01T09:06:26.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:26 smithi138 conmon[100289]: audit 2023-10-01T09:06:26.243111+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:06:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:26 smithi138 conmon[109959]: cluster 2023-10-01T09:06:25.042239+0000 2023-10-01T09:06:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:26 smithi138 conmon[109959]: mgr.x (mgr.25342) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-10-01T09:06:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:26 smithi138 conmon[109959]: audit 2023-10-01T09:06 2023-10-01T09:06:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:26 smithi138 conmon[109959]: :26.243111+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:06:26.843 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:26 smithi190 conmon[100518]: cluster 2023-10-01T09:06:25.042239+0000 mgr.x (mgr.25342) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-10-01T09:06:26.843 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:26 smithi190 conmon[100518]: 2023-10-01T09:06:26.843 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:26 smithi190 conmon[100518]: audit 2023-10-01T09:06:26.243111+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:06:27.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:26 smithi190 conmon[139437]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:06:26] "GET /metrics HTTP/1.1" 200 33788 "" "Prometheus/2.33.4" 2023-10-01T09:06:27.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:27 smithi138 conmon[109959]: audit 2023-10-01T09:06:27.457775+0000 2023-10-01T09:06:27.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:27 smithi138 conmon[109959]: mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.138:0/3488047168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:27.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:27 smithi138 conmon[100289]: audit 2023-10-01T09:06:27.457775+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.138:0/3488047168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:28.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:27 smithi190 conmon[100518]: audit 2023-10-01T09:06:27.457775+0000 mon.c (mon.2) 143 : audit [DBG] 2023-10-01T09:06:28.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:27 smithi190 conmon[100518]: from='client.? 172.21.15.138:0/3488047168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:29.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:28 smithi190 conmon[100518]: cluster 2023-10-01T09:06:27.042705+0000 mgr.x (mgr.25342) 29 : cluster [DBG] 2023-10-01T09:06:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:28 smithi190 conmon[100518]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-01T09:06:29.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:28 smithi138 conmon[109959]: cluster 2023-10-01T09:06:27.042705+0000 mgr.x (mgr.25342) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-01T09:06:29.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:28 smithi138 conmon[100289]: cluster 2023-10-01T09:06:27.042705+0000 mgr.x (mgr.25342) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-01T09:06:30.654 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:30 smithi138 conmon[100289]: cluster 2023-10-01T09:06:29.043281+0000 mgr.x 2023-10-01T09:06:30.654 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:30 smithi138 conmon[100289]: (mgr.25342) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-01T09:06:30.654 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:30 smithi138 conmon[100289]: audit 2023-10-01T09:06:29.846180+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.138:0/3100791347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:30.654 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:30 smithi138 conmon[109959]: cluster 2023-10-01T09:06 2023-10-01T09:06:30.654 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:30 smithi138 conmon[109959]: :29.043281+0000 mgr.x (mgr.25342) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-01T09:06:30.655 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:30 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:06:30.655 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:30 smithi138 conmon[109959]: 10-01T09:06:29.846180 2023-10-01T09:06:30.655 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:30 smithi138 conmon[109959]: +0000 mon.c (mon.2) 144 : 2023-10-01T09:06:30.655 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:30 smithi138 conmon[109959]: audit [DBG] from='client.? 172.21.15.138:0/3100791347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:30.655 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:30 smithi138 systemd[1]: Stopping Ceph alertmanager.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:06:30.655 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:30 smithi138 bash[176710]: Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-alertmanager.a found: no such container 2023-10-01T09:06:31.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:30 smithi190 conmon[100518]: cluster 2023-10-01T09:06:29.043281+0000 mgr.x (mgr.25342) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-10-01T09:06:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:30 smithi190 conmon[100518]: audit 2023-10-01T09:06:29.846180+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.138:0/3100791347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:31.069 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:30 smithi138 conmon[150478]: level=info ts=2023-10-01T09:06:30.653Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-01T09:06:31.070 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:30 smithi138 bash[176826]: a3bfe0be9ee50f1ea600ee982d9217d5092dce968edc2666fca96289a1a6cabb 2023-10-01T09:06:31.819 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:31 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-01T09:06:31.820 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:31 smithi138 systemd[1]: Stopped Ceph alertmanager.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:06:31.820 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:31 smithi138 systemd[1]: Starting Ceph alertmanager.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:06:32.320 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[177222]: level=info ts=2023-10-01T09:06:32.188Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-01T09:06:32.320 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[177222]: level=info ts=2023-10-01T09:06:32.188Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-01T09:06:32.320 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[177222]: level=info ts=2023-10-01T09:06:32.189Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.138 port=9094 2023-10-01T09:06:32.320 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[177222]: level=info ts=2023-10-01T09:06:32.191Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-01T09:06:32.320 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:32 smithi138 bash[177158]: 8debc681d94ead91fb942c886c8eb421520c07ea3848e0423ef359f52b9ce5fe 2023-10-01T09:06:32.320 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:32 smithi138 systemd[1]: Started Ceph alertmanager.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:06:32.320 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[177222]: level=info ts=2023-10-01T09:06:32.219Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-01T09:06:32.320 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[177222]: level=info ts=2023-10-01T09:06:32.219Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-01T09:06:32.321 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[177222]: level=info ts=2023-10-01T09:06:32.221Z caller=main.go:518 msg=Listening address=:9093 2023-10-01T09:06:32.321 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[177222]: level=info ts=2023-10-01T09:06:32.221Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-01T09:06:32.577 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:32 smithi190 conmon[133744]: ts=2023-10-01T09:06:32.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.138:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:06:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:32 smithi190 conmon[100518]: cluster 2023-10-01T09:06:31.043677+0000 mgr.x 2023-10-01T09:06:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:32 smithi190 conmon[100518]: (mgr.25342) 31 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-01T09:06:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:32 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T09:06:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:32 smithi190 conmon[100518]: 09:06:32.235905+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:32 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:06:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:32 smithi190 conmon[100518]: -01T09:06:32.237816+0000 mon.c (mon. 2023-10-01T09:06:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:32 smithi190 conmon[100518]: 2) 145 : audit [DBG] from='client.? 172.21.15.138:0/2470180152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:32 smithi190 conmon[100518]: audit 2023 2023-10-01T09:06:33.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:32 smithi190 conmon[100518]: -10-01T09:06:32.243134+0000 2023-10-01T09:06:33.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:32 smithi190 conmon[100518]: mon.a (mon.0) 797 : audit [INF] 2023-10-01T09:06:33.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:32 smithi190 conmon[100518]: from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:33.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[100289]: cluster 2023-10-01T09:06:31.043677+0000 2023-10-01T09:06:33.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[100289]: mgr.x (mgr.25342) 31 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-01T09:06:33.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:06:33.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[100289]: 09:06:32.235905+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:33.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[100289]: audit 2023 2023-10-01T09:06:33.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[100289]: -10-01T09:06:32.237816+0000 mon.c (mon. 2023-10-01T09:06:33.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[100289]: 2) 145 : audit [DBG] from='client.? 172.21.15.138:0/2470180152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:33.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:06:33.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[100289]: :06:32.243134+0000 mon.a (mon.0) 2023-10-01T09:06:33.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[100289]: 797 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:33.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[109959]: cluster 2023-10-01T09:06:31.043677+0000 2023-10-01T09:06:33.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[109959]: mgr.x (mgr.25342) 31 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-01T09:06:33.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[109959]: audit 2023-10-01T09:06:33.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[109959]: 2023-10-01T09:06:32.235905+0000 mon.a (mon.0) 796 : audit [INF] 2023-10-01T09:06:33.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[109959]: from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:33.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[109959]: audit 2023-10-01T09:06:32. 2023-10-01T09:06:33.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[109959]: 237816+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.138:0/2470180152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:33.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[109959]: 2023-10-01T09:06:33.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[109959]: audit 2023-10-01T09:06:33.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[109959]: 2023-10-01T09:06:32.243134+0000 mon.a (mon.0 2023-10-01T09:06:33.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:32 smithi138 conmon[109959]: ) 797 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:34.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:33 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:32.264479+0000 mgr.x (mgr.25342) 32 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T09:06:34.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:33 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:32.485893+0000 mgr.x (mgr.25342) 33 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:06:34.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:33 smithi138 conmon[109959]: cephadm 2023-10-01T09:06:32.264479+0000 mgr.x (mgr.25342) 2023-10-01T09:06:34.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:33 smithi138 conmon[109959]: 32 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T09:06:34.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:33 smithi138 conmon[109959]: cephadm 2023-10-01T09:06:32.485893+0000 mgr.x (mgr.25342) 33 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:06:34.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:33 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:32.264479+0000 mgr.x (mgr.25342) 32 : cephadm [INF] 2023-10-01T09:06:34.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:33 smithi138 conmon[100289]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T09:06:34.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:33 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:32.485893+0000 mgr.x (mgr.25342) 33 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:06:34.569 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:34 smithi138 conmon[177222]: level=info ts=2023-10-01T09:06:34.192Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000674972s 2023-10-01T09:06:35.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:34 smithi190 conmon[100518]: cluster 2023-10-01T09:06:33.044148+0000 mgr.x (mgr.25342) 34 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-01T09:06:35.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:34 smithi190 conmon[100518]: 2023-10-01T09:06:35.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:34 smithi138 conmon[109959]: cluster 2023-10-01T09:06:33.044148+0000 2023-10-01T09:06:35.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:34 smithi138 conmon[109959]: mgr.x (mgr.25342) 34 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-01T09:06:35.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:34 smithi138 conmon[100289]: cluster 2023-10-01T09:06:33.044148+0000 mgr.x (mgr.25342 2023-10-01T09:06:35.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:34 smithi138 conmon[100289]: ) 34 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-10-01T09:06:35.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:06:35 smithi138 conmon[100738]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:06:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:06:36.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:35 smithi190 conmon[100518]: audit 2023-10-01T09:06:34.619135+0000 mon.a (mon 2023-10-01T09:06:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:35 smithi190 conmon[100518]: .0) 798 : audit [DBG] from='client.? 172.21.15.138:0/3617016108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:36.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:35 smithi138 conmon[109959]: audit 2023-10-01T09:06:34.619135+0000 2023-10-01T09:06:36.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:35 smithi138 conmon[109959]: mon.a (mon.0) 798 : audit [DBG] from='client.? 172.21.15.138:0/3617016108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:36.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:35 smithi138 conmon[100289]: audit 2023-10-01T09:06:34.619135+0000 mon.a (mon.0) 798 : audit [DBG] from='client.? 172.21.15.138:0/3617016108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:36.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:35 smithi138 conmon[100289]: 2023-10-01T09:06:36.843 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:36 smithi190 conmon[100518]: cluster 2023-10-01T09:06:35.044497+0000 mgr.x 2023-10-01T09:06:36.843 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:36 smithi190 conmon[100518]: (mgr.25342) 35 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:36.844 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:36 smithi190 conmon[139437]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:06:36] "GET /metrics HTTP/1.1" 200 33788 "" "Prometheus/2.33.4" 2023-10-01T09:06:37.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:36 smithi138 conmon[109959]: cluster 2023-10-01T09:06:35.044497 2023-10-01T09:06:37.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:36 smithi138 conmon[109959]: +0000 mgr.x (mgr.25342) 35 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:37.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:36 smithi138 conmon[100289]: cluster 2023-10-01T09:06:35.044497+0000 mgr.x (mgr.25342 2023-10-01T09:06:37.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:36 smithi138 conmon[100289]: ) 35 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:38.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:37 smithi190 conmon[100518]: audit 2023-10-01T09:06:36.998552+0000 mon.c (mon. 2023-10-01T09:06:38.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:37 smithi190 conmon[100518]: 2) 146 : audit [DBG] from='client.? 172.21.15.138:0/1574750683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:38.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:37 smithi138 conmon[109959]: audit 2023-10-01T09:06:36.998552+0000 2023-10-01T09:06:38.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:37 smithi138 conmon[109959]: mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.138:0/1574750683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:37 smithi138 conmon[100289]: audit 2023 2023-10-01T09:06:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:37 smithi138 conmon[100289]: -10-01T09:06:36.998552+0000 mon.c (mon.2) 146 : audit [DBG] from='client.? 172.21.15.138:0/1574750683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:38.914 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:38 smithi190 conmon[100518]: cluster 2023-10-01T09:06:37.044758+0000 mgr.x (mgr 2023-10-01T09:06:38.914 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:38 smithi190 conmon[100518]: .25342) 36 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:39.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:38 smithi138 conmon[109959]: cluster 2023-10-01T09:06:37.044758+0000 mgr.x (mgr.25342) 36 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:38 smithi138 conmon[100289]: cluster 2023-10-01T09:06:37.044758+0000 mgr.x (mgr 2023-10-01T09:06:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:38 smithi138 conmon[100289]: .25342) 36 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:40.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:39 smithi190 conmon[100518]: audit 2023-10-01T09:06:39.383623+0000 mon.a 2023-10-01T09:06:40.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:39 smithi190 conmon[100518]: (mon.0) 799 : audit [DBG] from='client.? 172.21.15.138:0/3778279283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:40.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:39 smithi138 conmon[109959]: audit 2023-10-01T09:06 2023-10-01T09:06:40.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:39 smithi138 conmon[109959]: :39.383623+0000 mon.a (mon.0) 799 : audit [DBG] from='client.? 172.21.15.138:0/3778279283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:40.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:39 smithi138 conmon[100289]: audit 2023-10-01T09:06:39.383623+0000 mon.a (mon.0) 2023-10-01T09:06:40.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:39 smithi138 conmon[100289]: 799 : audit [DBG] from='client.? 172.21.15.138:0/3778279283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:41.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:40 smithi190 conmon[100518]: cluster 2023-10-01T09:06:39.045200+0000 mgr.x (mgr.25342) 37 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:41.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:40 smithi138 conmon[100289]: cluster 2023-10-01T09:06:39.045200+0000 2023-10-01T09:06:41.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:40 smithi138 conmon[100289]: mgr.x (mgr.25342) 37 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:41.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:40 smithi138 conmon[109959]: cluster 2023-10-01T09:06:39.045200+0000 2023-10-01T09:06:41.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:40 smithi138 conmon[109959]: mgr.x (mgr.25342) 37 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:41.711 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:41 smithi190 conmon[100518]: audit 2023-10-01T09:06:41.243281+0000 mon.b (mon.1) 80 : 2023-10-01T09:06:41.711 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:41 smithi190 conmon[100518]: audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:06:42.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:41 smithi138 conmon[100289]: audit 2023-10-01T09:06:41.243281+0000 mon.b (mon. 2023-10-01T09:06:42.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:41 smithi138 conmon[100289]: 1) 80 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:06:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:41 smithi138 conmon[109959]: audit 2023-10-01T09:06:41.243281+0000 2023-10-01T09:06:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:41 smithi138 conmon[109959]: mon.b (mon.1) 80 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:06:42.420 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 systemd[1]: Stopping Ceph prometheus.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:06:42.420 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 bash[144718]: Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-prometheus.a found: no such container 2023-10-01T09:06:42.420 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[133744]: ts=2023-10-01T09:06:42.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.138:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:06:42.420 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[133744]: ts=2023-10-01T09:06:42.419Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-01T09:06:42.420 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[133744]: ts=2023-10-01T09:06:42.419Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-01T09:06:42.420 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[133744]: ts=2023-10-01T09:06:42.419Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-01T09:06:42.569 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:06:42 smithi138 conmon[177222]: level=info ts=2023-10-01T09:06:42.195Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003601829s 2023-10-01T09:06:42.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[100518]: cluster 2023-10-01T09:06:41.045588+0000 mgr.x (mgr. 2023-10-01T09:06:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[100518]: 25342) 38 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[100518]: audit 2023-10-01T09:06:41.768108+0000 2023-10-01T09:06:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[100518]: mon.a (mon.0) 800 : audit [DBG] from='client.? 172.21.15.138:0/3194777503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:42.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[133744]: ts=2023-10-01T09:06:42.419Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-01T09:06:42.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[133744]: ts=2023-10-01T09:06:42.419Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-01T09:06:42.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[133744]: ts=2023-10-01T09:06:42.419Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-01T09:06:42.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[133744]: ts=2023-10-01T09:06:42.419Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-01T09:06:42.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[133744]: ts=2023-10-01T09:06:42.419Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-01T09:06:42.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[133744]: ts=2023-10-01T09:06:42.419Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-01T09:06:42.808 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[133744]: ts=2023-10-01T09:06:42.420Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-01T09:06:42.808 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[133744]: ts=2023-10-01T09:06:42.420Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-01T09:06:42.808 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 conmon[133744]: ts=2023-10-01T09:06:42.420Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-01T09:06:43.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:42 smithi138 conmon[109959]: cluster 2023-10-01T09:06:41.045588+0000 mgr.x (mgr. 2023-10-01T09:06:43.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:42 smithi138 conmon[109959]: 25342) 38 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:43.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:42 smithi138 conmon[109959]: audit 2023 2023-10-01T09:06:43.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:42 smithi138 conmon[109959]: -10-01T09:06:41.768108+0000 mon.a (mon.0) 800 : audit [DBG] from='client.? 172.21.15.138:0/3194777503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:42 smithi138 conmon[100289]: cluster 2023-10-01T09:06:41.045588+0000 mgr.x (mgr.25342) 38 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:42 smithi138 conmon[100289]: audit 2023-10-01T09:06:41.768108+0000 mon.a (mon.0) 800 : audit [DBG] from='client.? 172.21.15.138:0/3194777503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:43.306 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:42 smithi190 bash[144835]: a1782aa730be0f3d514bd719fe4aaf9cc9393a9be9e8700c4cd138a8086a6644 2023-10-01T09:06:43.307 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-01T09:06:43.307 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 systemd[1]: Stopped Ceph prometheus.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:06:43.307 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 systemd[1]: Starting Ceph prometheus.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:06:44.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 conmon[145209]: ts=2023-10-01T09:06:43.725Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-01T09:06:44.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 conmon[145209]: ts=2023-10-01T09:06:43.725Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-01T09:06:44.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 conmon[145209]: ts=2023-10-01T09:06:43.725Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi190 (none))" 2023-10-01T09:06:44.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 conmon[145209]: ts=2023-10-01T09:06:43.725Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-01T09:06:44.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 conmon[145209]: ts=2023-10-01T09:06:43.725Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-01T09:06:44.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 conmon[145209]: ts=2023-10-01T09:06:43.727Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-01T09:06:44.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 conmon[145209]: ts=2023-10-01T09:06:43.728Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-01T09:06:44.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 conmon[145209]: ts=2023-10-01T09:06:43.728Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-01T09:06:44.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 conmon[145209]: ts=2023-10-01T09:06:43.731Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-01T09:06:44.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 conmon[145209]: ts=2023-10-01T09:06:43.731Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.671µs 2023-10-01T09:06:44.058 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 conmon[145209]: ts=2023-10-01T09:06:43.731Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-01T09:06:44.058 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 bash[145168]: f4d0cb1e646645ed4a740fc156a74f00391a732340227e176e328e4542c23af8 2023-10-01T09:06:44.058 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 systemd[1]: Started Ceph prometheus.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:06:44.058 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:43 smithi190 conmon[145209]: ts=2023-10-01T09:06:43.895Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-10-01T09:06:44.536 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:44 smithi190 systemd[1]: Stopping Ceph node-exporter.b for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:06:44.537 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:44 smithi190 bash[145367]: Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-node-exporter.b found: no such container 2023-10-01T09:06:44.537 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[145209]: ts=2023-10-01T09:06:44.118Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-10-01T09:06:44.537 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[145209]: ts=2023-10-01T09:06:44.118Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-10-01T09:06:44.537 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[145209]: ts=2023-10-01T09:06:44.118Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=40.463µs wal_replay_duration=387.430036ms total_replay_duration=387.483983ms 2023-10-01T09:06:44.537 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[145209]: ts=2023-10-01T09:06:44.122Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-01T09:06:44.537 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[145209]: ts=2023-10-01T09:06:44.122Z caller=main.go:947 level=info msg="TSDB started" 2023-10-01T09:06:44.537 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[145209]: ts=2023-10-01T09:06:44.122Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-01T09:06:44.537 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[145209]: ts=2023-10-01T09:06:44.145Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.828164ms db_storage=1.409µs remote_storage=1.802µs web_handler=450ns query_engine=806ns scrape=804.893µs scrape_sd=183.02µs notify=19.437µs notify_sd=56.109µs rules=21.324293ms 2023-10-01T09:06:44.538 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[145209]: ts=2023-10-01T09:06:44.145Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-01T09:06:44.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[100518]: cluster 2023-10-01T09:06:43.046088+0000 mgr.x (mgr. 2023-10-01T09:06:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[100518]: 25342) 39 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[100518]: audit 2023-10-01T09:06:43.767253+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[100518]: audit 2023-10-01T09:06:43.773004+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:43.821363+0000 mgr.x (mgr.25342) 40 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-01T09:06:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:43.821840+0000 mgr.x (mgr.25342) 41 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi190 2023-10-01T09:06:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:44 smithi190 conmon[100518]: audit 2023-10-01T09:06:44.142151+0000 mon.a (mon.0) 803 : audit [DBG] from='client.? 172.21.15.138:0/3361211667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:45.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[100289]: cluster 2023-10-01T09:06:43.046088+0000 mgr.x (mgr.25342) 39 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:45.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[100289]: audit 2023-10-01T09:06:43.767253+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:45.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[100289]: audit 2023-10-01T09:06:43.773004+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:45.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:43.821363+0000 mgr.x (mgr.25342) 40 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-01T09:06:45.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:43.821840+0000 mgr.x (mgr 2023-10-01T09:06:45.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[100289]: .25342) 41 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi190 2023-10-01T09:06:45.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T09:06:45.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[100289]: T09:06:44.142151+0000 mon.a (mon.0 2023-10-01T09:06:45.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[100289]: ) 803 : audit [DBG] from='client.? 172.21.15.138:0/3361211667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:45.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: cluster 2023-10-01T09:06:43.046088+0000 mgr.x 2023-10-01T09:06:45.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: (mgr.25342) 39 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:45.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: audit 2023-10-01T09:06:43.767253 2023-10-01T09:06:45.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: +0000 mon.a (mon.0) 801 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:45.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: audit 2023-10-01T09:06:43. 2023-10-01T09:06:45.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: 773004+0000 mon.a (mon.0) 802 : audit [INF] 2023-10-01T09:06:45.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:45.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: cephadm 2023-10-01T09:06:43 2023-10-01T09:06:45.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: .821363+0000 mgr.x (mgr.25342) 40 : cephadm 2023-10-01T09:06:45.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-01T09:06:45.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: cephadm 2023-10-01T09:06: 2023-10-01T09:06:45.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: 43.821840+0000 mgr.x (mgr.25342) 41 : cephadm 2023-10-01T09:06:45.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: [INF] Reconfiguring daemon node-exporter.b on smithi190 2023-10-01T09:06:45.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: audit 2023-10-01T09:06 2023-10-01T09:06:45.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:44 smithi138 conmon[109959]: :44.142151+0000 mon.a (mon.0) 803 : audit [DBG] from='client.? 172.21.15.138:0/3361211667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:45.307 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:44 smithi190 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-01T09:06:45.307 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:44 smithi190 bash[145483]: 9a8d149372ce669cf3f80ab88d2598eb2bfbb8e74e1d912dde8132f70d4c8c66 2023-10-01T09:06:45.307 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-01T09:06:45.307 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 systemd[1]: Stopped Ceph node-exporter.b for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:06:45.307 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 systemd[1]: Starting Ceph node-exporter.b for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:06:46.057 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.660Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-01T09:06:46.057 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.660Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-01T09:06:46.057 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.660Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-01T09:06:46.057 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.660Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-01T09:06:46.057 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-01T09:06:46.057 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=arp 2023-10-01T09:06:46.057 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-01T09:06:46.058 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-01T09:06:46.058 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-01T09:06:46.058 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-01T09:06:46.058 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-01T09:06:46.058 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-01T09:06:46.058 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-01T09:06:46.058 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-01T09:06:46.058 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=edac 2023-10-01T09:06:46.058 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-01T09:06:46.058 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-01T09:06:46.058 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-01T09:06:46.059 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-01T09:06:46.059 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-01T09:06:46.059 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-01T09:06:46.059 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-01T09:06:46.059 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-01T09:06:46.059 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-01T09:06:46.059 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-01T09:06:46.059 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-01T09:06:46.059 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-01T09:06:46.059 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-01T09:06:46.059 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-01T09:06:46.059 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-01T09:06:46.060 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-01T09:06:46.060 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=os 2023-10-01T09:06:46.060 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-01T09:06:46.060 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-01T09:06:46.060 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-01T09:06:46.060 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-01T09:06:46.060 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-01T09:06:46.060 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-01T09:06:46.060 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=stat 2023-10-01T09:06:46.060 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-01T09:06:46.060 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-01T09:06:46.060 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-01T09:06:46.061 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=time 2023-10-01T09:06:46.061 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-01T09:06:46.061 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=uname 2023-10-01T09:06:46.061 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-01T09:06:46.061 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-01T09:06:46.061 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-01T09:06:46.061 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.661Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-01T09:06:46.061 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 conmon[145855]: ts=2023-10-01T09:06:45.662Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-01T09:06:46.061 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 bash[145814]: c08f7a49a78d5374c7265f1711761c92e078aca8742af1fc2fa72a35d3aeae34 2023-10-01T09:06:46.061 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:06:45 smithi190 systemd[1]: Started Ceph node-exporter.b for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:06:47.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:46 smithi190 conmon[100518]: cluster 2023-10-01T09:06:45.046468+0000 mgr.x (mgr.25342) 42 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:47.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:46 smithi190 conmon[100518]: audit 2023-10-01T09:06:45.716646+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:46 smithi190 conmon[100518]: audit 2023-10-01T09:06:45.721905+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:46 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:45.738934+0000 mgr.x (mgr.25342) 43 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-01T09:06:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:46 smithi190 conmon[100518]: cephadm 2023-10-01T09:06:45.749857+0000 mgr.x (mgr.25342) 44 : cephadm [INF] Reconfiguring daemon grafana.a on smithi190 2023-10-01T09:06:47.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:46 smithi190 conmon[100518]: audit 2023-10-01T09:06:46.520991+0000 mon.b (mon.1) 81 : audit [DBG] from='client.? 172.21.15.138:0/1250403476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:47.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[109959]: cluster 2023-10-01T09:06:45.046468+0000 mgr.x 2023-10-01T09:06:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[109959]: (mgr.25342) 42 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:06:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[109959]: 01T09:06:45.716646+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[109959]: audit 2023-10-01T09:06:45.721905+0000 2023-10-01T09:06:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[109959]: mon.a (mon.0) 805 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[109959]: cephadm 2023-10-01T09:06: 2023-10-01T09:06:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[109959]: 45.738934+0000 mgr.x (mgr.25342) 43 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-01T09:06:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[109959]: cephadm 2023-10 2023-10-01T09:06:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[109959]: -01T09:06:45.749857+0000 mgr.x (mgr.25342) 44 : cephadm 2023-10-01T09:06:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[109959]: [INF] Reconfiguring daemon grafana.a on smithi190 2023-10-01T09:06:47.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[109959]: audit 2023-10-01T09:06:47.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[109959]: 2023-10-01T09:06:46.520991+0000 mon.b (mon.1) 81 : audit [DBG] from='client.? 172.21.15.138:0/1250403476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[100289]: cluster 2023-10-01T09:06:45.046468+0000 2023-10-01T09:06:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[100289]: mgr.x (mgr.25342) 42 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[100289]: audit 2023-10-01T09:06:45.716646+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:06:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[100289]: 09:06:45.721905+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:06:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[100289]: cephadm 2023-10 2023-10-01T09:06:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[100289]: -01T09:06:45.738934+0000 mgr.x (mgr.25342) 43 : cephadm 2023-10-01T09:06:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[100289]: [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-01T09:06:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[100289]: cephadm 2023-10-01T09:06:45. 2023-10-01T09:06:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[100289]: 749857+0000 mgr.x (mgr.25342) 44 : cephadm [INF] 2023-10-01T09:06:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[100289]: Reconfiguring daemon grafana.a on smithi190 2023-10-01T09:06:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[100289]: audit 2023-10-01T09:06:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[100289]: 2023-10-01T09:06:46.520991+0000 mon.b (mon. 2023-10-01T09:06:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:46 smithi138 conmon[100289]: 1) 81 : audit [DBG] from='client.? 172.21.15.138:0/1250403476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:49.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:48 smithi190 conmon[100518]: cluster 2023-10-01T09:06:47.046809+0000 mgr.x (mgr.25342) 45 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:49.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:48 smithi138 conmon[109959]: cluster 2023-10-01T09:06:47. 2023-10-01T09:06:49.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:48 smithi138 conmon[109959]: 046809+0000 mgr.x (mgr.25342) 45 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:49.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:48 smithi138 conmon[100289]: cluster 2023-10-01T09:06:47.046809+0000 mgr.x (mgr.25342) 45 : 2023-10-01T09:06:49.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:48 smithi138 conmon[100289]: cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:50.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:49 smithi190 conmon[100518]: audit 2023-10-01T09:06:48.904185+0000 mon.a (mon.0) 2023-10-01T09:06:50.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:49 smithi190 conmon[100518]: 806 : audit [DBG] from='client.? 172.21.15.138:0/1190882284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:50.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:49 smithi138 conmon[100289]: audit 2023-10-01T09:06:48.904185+0000 mon.a (mon.0) 806 : audit [DBG] from='client.? 172.21.15.138:0/1190882284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:50.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:49 smithi138 conmon[109959]: audit 2023-10-01T09:06:48.904185+0000 mon.a (mon.0) 2023-10-01T09:06:50.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:49 smithi138 conmon[109959]: 806 : audit [DBG] from='client.? 172.21.15.138:0/1190882284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:51.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:50 smithi190 conmon[100518]: cluster 2023-10-01T09 2023-10-01T09:06:51.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:50 smithi190 conmon[100518]: :06:49.047235+0000 mgr.x (mgr.25342) 46 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:51.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:50 smithi138 conmon[109959]: cluster 2023-10-01T09:06:49.047235+0000 mgr.x (mgr. 2023-10-01T09:06:51.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:50 smithi138 conmon[109959]: 25342) 46 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:51.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:50 smithi138 conmon[100289]: cluster 2023-10-01T09:06:49.047235+0000 mgr.x (mgr. 2023-10-01T09:06:51.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:50 smithi138 conmon[100289]: 25342) 46 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:52.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:51 smithi190 conmon[100518]: audit 2023-10-01T09:06:51.285191+0000 2023-10-01T09:06:52.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:51 smithi190 conmon[100518]: mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.138:0/738415351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:52.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:51 smithi138 conmon[100289]: audit 2023-10-01T09:06:51.285191+0000 2023-10-01T09:06:52.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:51 smithi138 conmon[100289]: mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.138:0/738415351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:51 smithi138 conmon[109959]: audit 2023-10-01T09:06:51.285191+0000 2023-10-01T09:06:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:51 smithi138 conmon[109959]: mon.c (mon.2) 147 : audit [DBG] from='client.? 172.21.15.138:0/738415351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:53.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:52 smithi190 conmon[100518]: cluster 2023-10-01T09:06:51.047513+0000 mgr.x 2023-10-01T09:06:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:52 smithi190 conmon[100518]: (mgr.25342) 47 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:53.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:52 smithi138 conmon[100289]: cluster 2023-10-01T09:06:51.047513+0000 mgr.x 2023-10-01T09:06:53.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:52 smithi138 conmon[100289]: (mgr.25342) 47 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:53.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:52 smithi138 conmon[109959]: cluster 2023-10-01T09:06:51.047513 2023-10-01T09:06:53.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:52 smithi138 conmon[109959]: +0000 mgr.x (mgr.25342) 47 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:54.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:53 smithi190 conmon[100518]: audit 2023-10-01T09:06:53.671508+0000 mon.b (mon.1) 2023-10-01T09:06:54.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:53 smithi190 conmon[100518]: 82 : audit [DBG] from='client.? 172.21.15.138:0/2051014539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:54.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:53 smithi138 conmon[109959]: audit 2023-10-01T09:06:53 2023-10-01T09:06:54.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:53 smithi138 conmon[109959]: .671508+0000 mon.b (mon.1) 82 : audit [DBG] from='client.? 172.21.15.138:0/2051014539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:54.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:53 smithi138 conmon[100289]: audit 2023-10-01T09:06:53.671508+0000 mon.b (mon. 2023-10-01T09:06:54.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:53 smithi138 conmon[100289]: 1) 82 : audit [DBG] from='client.? 172.21.15.138:0/2051014539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:55.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:54 smithi190 conmon[100518]: cluster 2023-10-01T09:06:53.048017+0000 mgr.x (mgr.25342) 2023-10-01T09:06:55.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:54 smithi190 conmon[100518]: 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:55.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:54 smithi138 conmon[109959]: cluster 2023-10-01T09:06:53.048017 2023-10-01T09:06:55.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:54 smithi138 conmon[109959]: +0000 mgr.x (mgr.25342) 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:55.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:54 smithi138 conmon[100289]: cluster 2023-10-01T09:06:53.048017+0000 mgr.x ( 2023-10-01T09:06:55.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:54 smithi138 conmon[100289]: mgr.25342) 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:57.017 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:56 smithi190 conmon[100518]: cluster 2023-10-01T09:06:55.048334+0000 mgr.x (mgr.25342) 49 : cluster 2023-10-01T09:06:57.017 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:56 smithi190 conmon[100518]: [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:57.017 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:56 smithi190 conmon[100518]: audit 2023-10-01T09:06:56.056376+0000 mon.c (mon.2) 148 : audit [DBG] from='client.? 172.21.15.138:0/3975634815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:57.017 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:56 smithi190 conmon[100518]: audit 2023-10-01T09:06: 2023-10-01T09:06:57.017 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:56 smithi190 conmon[100518]: 56.243480+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:06:57.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:56 smithi138 conmon[109959]: cluster 2023-10-01T09:06:55.048334+0000 mgr.x (mgr.25342) 49 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:57.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:56 smithi138 conmon[109959]: audit 2023 2023-10-01T09:06:57.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:56 smithi138 conmon[109959]: -10-01T09:06:56.056376+0000 mon.c (mon.2) 148 : audit [DBG] from='client.? 172.21.15.138:0/3975634815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:57.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:56 smithi138 conmon[109959]: audit 2023-10-01T09:06:56.243480+0000 mon.b (mon.1) 83 2023-10-01T09:06:57.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:56 smithi138 conmon[109959]: : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:06:57.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:56 smithi138 conmon[100289]: cluster 2023-10-01T09:06:55.048334+0000 mgr.x (mgr.25342) 49 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:57.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:56 smithi138 conmon[100289]: 2023-10-01T09:06:57.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:56 smithi138 conmon[100289]: audit 2023-10-01T09:06:56.056376+0000 mon.c (mon.2) 148 : audit [DBG] from='client.? 172.21.15.138:0/3975634815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:57.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:56 smithi138 conmon[100289]: audit 2023-10-01T09:06: 2023-10-01T09:06:57.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:56 smithi138 conmon[100289]: 56.243480+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:06:58.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:06:57 smithi190 conmon[139437]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:06:57] "GET /metrics HTTP/1.1" 200 33785 "" "Prometheus/2.33.4" 2023-10-01T09:06:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:58 smithi190 conmon[100518]: cluster 2023-10-01T09:06:57.048667+0000 mgr.x (mgr.25342) 50 2023-10-01T09:06:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:58 smithi190 conmon[100518]: : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:06:58 smithi190 conmon[100518]: audit 2023-10-01T09:06:58.436910+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.138:0/3442395197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:59.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:06:58 smithi190 conmon[145209]: ts=2023-10-01T09:06:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:06:59.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:58 smithi138 conmon[109959]: cluster 2023-10-01T09:06:57.048667+0000 mgr.x ( 2023-10-01T09:06:59.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:58 smithi138 conmon[109959]: mgr.25342) 50 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:59.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:58 smithi138 conmon[109959]: audit 2023-10-01T09:06:58.436910+0000 mon.c 2023-10-01T09:06:59.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:06:58 smithi138 conmon[109959]: (mon.2) 149 : audit [DBG] from='client.? 172.21.15.138:0/3442395197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:06:59.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:58 smithi138 conmon[100289]: cluster 2023-10-01T 2023-10-01T09:06:59.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:58 smithi138 conmon[100289]: 09:06:57.048667+0000 mgr.x (mgr.25342) 50 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:06:59.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:58 smithi138 conmon[100289]: audit 2023-10-01T09:06:59.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:58 smithi138 conmon[100289]: 2023-10-01T09:06:58.436910+0000 mon.c (mon.2) 149 : audit [DBG] 2023-10-01T09:06:59.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:06:58 smithi138 conmon[100289]: from='client.? 172.21.15.138:0/3442395197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:01.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:00 smithi190 conmon[100518]: cluster 2023-10-01T09:06:59.049264+0000 mgr.x (mgr.25342) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:01.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:00 smithi138 conmon[109959]: cluster 2023-10-01T09: 2023-10-01T09:07:01.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:00 smithi138 conmon[109959]: 06:59.049264+0000 mgr.x (mgr.25342) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:01.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:00 smithi138 conmon[100289]: cluster 2023-10-01T09:06:59.049264+0000 mgr.x (mgr 2023-10-01T09:07:01.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:00 smithi138 conmon[100289]: .25342) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:02.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:01 smithi190 conmon[100518]: audit 2023-10-01T09:07:00.819363+0000 mon.a (mon.0) 807 : audit [DBG] from='client.? 172.21.15.138:0/3413042341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:02.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:01 smithi138 conmon[109959]: audit 2023-10-01T09:07:00.819363+0000 mon.a 2023-10-01T09:07:02.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:01 smithi138 conmon[109959]: (mon.0) 807 : audit [DBG] from='client.? 172.21.15.138:0/3413042341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:02.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:01 smithi138 conmon[100289]: audit 2023-10-01T09:07:00.819363+0000 mon.a (mon.0) 2023-10-01T09:07:02.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:01 smithi138 conmon[100289]: 807 : audit [DBG] from='client.? 172.21.15.138:0/3413042341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:03.023 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:02 smithi190 conmon[100518]: cluster 2023-10-01T09:07:01.049746+0000 mgr.x (mgr.25342) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:03.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:02 smithi138 conmon[100289]: cluster 2023-10-01T09:07:01.049746+0000 mgr.x (mgr.25342) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:02 smithi138 conmon[109959]: cluster 2023-10-01T09:07:01.049746+0000 mgr.x (mgr.25342) 52 : cluster 2023-10-01T09:07:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:02 smithi138 conmon[109959]: [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:04.057 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:03 smithi190 systemd[1]: Stopping Ceph grafana.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:07:04.057 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:03 smithi190 bash[146270]: Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-grafana.a found: no such container 2023-10-01T09:07:04.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:03 smithi190 conmon[100518]: audit 2023-10-01T09:07:03.196985+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.138:0/1280570727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:04.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:03 smithi138 conmon[100289]: audit 2023-10-01T09:07:03.196985+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.138:0/1280570727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:04.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:03 smithi138 conmon[109959]: audit 2023-10-01T09:07:03.196985+0000 mon.c (mon 2023-10-01T09:07:04.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:03 smithi138 conmon[109959]: .2) 150 : audit [DBG] from='client.? 172.21.15.138:0/1280570727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:04.376 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:04 smithi190 conmon[131967]: t=2023-10-01T09:07:04+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-01T09:07:04.376 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:04 smithi190 bash[146387]: d186140d89cc59f5a74112ffec828e93ce55ab554620e70c85fff20eee7c6fd0 2023-10-01T09:07:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:04 smithi190 conmon[100518]: cluster 2023-10-01T09:07:03.050200+0000 mgr.x 2023-10-01T09:07:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:04 smithi190 conmon[100518]: (mgr.25342) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:05.057 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:04 smithi190 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-01T09:07:05.057 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:04 smithi190 systemd[1]: Stopped Ceph grafana.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:07:05.057 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:04 smithi190 systemd[1]: Starting Ceph grafana.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:07:05.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:04 smithi138 conmon[100289]: cluster 2023-10-01T09:07:03.050200+0000 mgr.x (mgr.25342) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:05.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:04 smithi138 conmon[109959]: cluster 2023-10-01T09:07:03.050200+0000 mgr.x (mgr.25342) 53 : 2023-10-01T09:07:05.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:04 smithi138 conmon[109959]: cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:05.622 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Bus STOPPING 2023-10-01T09:07:05.622 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T09:07:05.622 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Bus STOPPED 2023-10-01T09:07:05.623 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Bus STARTING 2023-10-01T09:07:05.623 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Serving on http://:::9283 2023-10-01T09:07:05.623 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Bus STARTED 2023-10-01T09:07:05.623 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Bus STOPPING 2023-10-01T09:07:05.623 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T09:07:05.623 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Bus STOPPED 2023-10-01T09:07:05.623 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Bus STARTING 2023-10-01T09:07:05.623 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 bash[146721]: 58ea19a8c949e621b7bb100277b98c89386c960325c4b078c4ab95dcd1119787 2023-10-01T09:07:05.624 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 systemd[1]: Started Ceph grafana.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:07:05.624 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-01T09:07:05.624 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-01T09:07:05.624 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-01T09:07:05.624 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-01T09:07:05.624 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-01T09:07:05.624 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-01T09:07:05.624 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-01T09:07:05.624 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-01T09:07:05.624 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-01T09:07:05.624 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-01T09:07:05.625 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-01T09:07:05.625 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-01T09:07:05.625 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-01T09:07:05.625 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="App mode production" logger=settings 2023-10-01T09:07:05.625 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-01T09:07:05.625 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-01T09:07:05.625 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-01T09:07:05.625 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=558.281µs 2023-10-01T09:07:05.625 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-01T09:07:05.625 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-01T09:07:05.625 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-01T09:07:05.625 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-01T09:07:05.626 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-01T09:07:05.626 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-01T09:07:05.626 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2023-10-01T09:07:05.626 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-01T09:07:05.626 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2023-10-01T09:07:05.626 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-01T09:07:05.626 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-01T09:07:06.056 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[146762]: t=2023-10-01T09:07:05+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-01T09:07:06.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Serving on http://:::9283 2023-10-01T09:07:06.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Bus STARTED 2023-10-01T09:07:06.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Bus STOPPING 2023-10-01T09:07:06.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T09:07:06.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Bus STOPPED 2023-10-01T09:07:06.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Bus STARTING 2023-10-01T09:07:06.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Serving on http://:::9283 2023-10-01T09:07:06.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:05 smithi190 conmon[139437]: [01/Oct/2023:09:07:05] ENGINE Bus STARTED 2023-10-01T09:07:06.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: cluster 2023-10-01T09:07:05.050609+0000 mgr.x (mgr.25342) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:06.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.362873+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.368048+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.414427+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T09:07:06.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.414896+0000 mgr.x (mgr.25342) 55 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T09:07:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.415827+0000 mon.b (mon.1) 85 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi138.front.sepia.ceph.com:9093"}]: dispatch 2023-10-01T09:07:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.416103+0000 mgr.x (mgr.25342) 56 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi138.front.sepia.ceph.com:9093"}]: dispatch 2023-10-01T09:07:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.421479+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.427147+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:07:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.427642+0000 mgr.x (mgr.25342) 57 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:07:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.431952+0000 mon.b (mon.1) 87 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190:9095"}]: dispatch 2023-10-01T09:07:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.432321+0000 mgr.x (mgr.25342) 58 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190:9095"}]: dispatch 2023-10-01T09:07:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.437334+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.443979+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T09:07:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.444425+0000 mgr.x (mgr.25342) 59 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T09:07:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.448867+0000 mon.b (mon.1) 89 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi190:3000"}]: dispatch 2023-10-01T09:07:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.449220+0000 mgr.x (mgr.25342) 60 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi190:3000"}]: dispatch 2023-10-01T09:07:06.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:05.453391+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023-10-01T09:07:06.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: 2023-10-01T09:07:05.466367+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: cluster 2023-10-01T09:07:05.050609+0000 mgr.x (mgr.25342) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.362873+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.368048+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.414427+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T09:07:06.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.414896+0000 mgr.x (mgr.25342) 55 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T09:07:06.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.415827+0000 mon.b (mon.1) 85 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi138.front.sepia.ceph.com:9093"}]: dispatch 2023-10-01T09:07:06.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.416103+0000 mgr.x (mgr.25342 2023-10-01T09:07:06.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: ) 56 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi138.front.sepia.ceph.com:9093"}]: dispatch 2023-10-01T09:07:06.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.421479+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.427147+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:07:06.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.427642+0000 mgr.x (mgr.25342) 57 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:07:06.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.431952+0000 mon.b (mon.1) 87 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190:9095"}]: dispatch 2023-10-01T09:07:06.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.432321+0000 mgr.x (mgr.25342) 58 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190:9095"}]: dispatch 2023-10-01T09:07:06.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.437334+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.443979+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T09:07:06.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.444425+0000 mgr.x (mgr.25342) 59 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T09:07:06.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.448867+0000 mon.b (mon.1) 89 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi190:3000"}]: dispatch 2023-10-01T09:07:06.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.449220+0000 mgr.x (mgr.25342) 60 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi190:3000"}]: dispatch 2023-10-01T09:07:06.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.453391+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.466367+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:06.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:05.468359+0000 mgr.x (mgr.25342) 61 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T09:07:06.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:05.468754+0000 mgr.x (mgr.25342) 62 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T09:07:06.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[100289]: audit 2023-10-01T09:07:05.604318+0000 mon.a (mon.0) 813 : audit [DBG] from='client.? 172.21.15.138:0/3474515219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:05.468359+0000 mgr.x 2023-10-01T09:07:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: (mgr.25342) 61 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T09:07:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: cephadm 2023-10 2023-10-01T09:07:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: -01T09:07:05.468754+0000 2023-10-01T09:07:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: mgr.x (mgr.25342) 62 : cephadm [INF] 2023-10-01T09:07:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T09:07:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: 10-01T09:07:05. 2023-10-01T09:07:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: 604318+0000 mon.a (mon.0) 2023-10-01T09:07:06.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:06 smithi138 conmon[109959]: 813 : audit [DBG] from='client.? 172.21.15.138:0/3474515219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: cluster 2023-10-01T09:07:05.050609+0000 mgr.x (mgr.25342) 2023-10-01T09:07:06.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:06.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023-10-01T09:07:05.362873+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023-10-01T09:07:05. 2023-10-01T09:07:06.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 368048+0000 mon.a (mon.0) 809 : audit 2023-10-01T09:07:06.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T09:07:06.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: T09:07:05.414427+0000 mon.b (mon 2023-10-01T09:07:06.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: .1) 84 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T09:07:06.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 2023-10-01T09:07:06.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: -10-01T09:07:05 2023-10-01T09:07:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: .414896+0000 mgr.x (mgr.25342) 2023-10-01T09:07:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 55 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T09:07:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:07:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 10-01T09:07:05.415827+0000 2023-10-01T09:07:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: mon.b (mon.1) 85 : audit 2023-10-01T09:07:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi138.front.sepia.ceph.com:9093"}]: dispatch 2023-10-01T09:07:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: -10-01T09:07:05.416103 2023-10-01T09:07:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: +0000 mgr.x (mgr.25342) 56 : 2023-10-01T09:07:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi138.front.sepia.ceph.com:9093"}]: dispatch 2023-10-01T09:07:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:06.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: -10-01T09:07:05. 2023-10-01T09:07:06.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 421479+0000 mon.a (mon.0) 810 2023-10-01T09:07:06.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:06.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: -10-01T09:07:05.427147 2023-10-01T09:07:06.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: +0000 mon.b (mon.1) 86 : 2023-10-01T09:07:06.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:07:06.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:06.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: -10-01T09:07:05. 2023-10-01T09:07:06.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 427642+0000 mgr.x (mgr.25342) 2023-10-01T09:07:06.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 57 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:07:06.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:07:06.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 10-01T09:07:05.431952+0000 2023-10-01T09:07:06.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: mon.b (mon.1) 87 : audit [INF] 2023-10-01T09:07:06.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190:9095"}]: dispatch 2023-10-01T09:07:06.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:06.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: -10-01T09:07:05 2023-10-01T09:07:06.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: .432321+0000 mgr.x (mgr.25342 2023-10-01T09:07:06.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: ) 58 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190:9095"}]: dispatch 2023-10-01T09:07:06.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:06.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: -10-01T09:07:05. 2023-10-01T09:07:06.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 437334+0000 mon.a (mon.0) 2023-10-01T09:07:06.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 811 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:06.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: -10-01T09:07:05.443979 2023-10-01T09:07:06.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: +0000 mon.b (mon.1) 88 2023-10-01T09:07:06.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T09:07:06.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:07:06.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 10-01T09:07:05.444425 2023-10-01T09:07:06.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: +0000 mgr.x (mgr.25342) 59 : 2023-10-01T09:07:06.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T09:07:06.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:06.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: -10-01T09:07:05. 2023-10-01T09:07:06.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 448867+0000 mon.b (mon.1) 2023-10-01T09:07:06.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 89 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi190:3000"}]: dispatch 2023-10-01T09:07:06.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:06.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: -10-01T09:07:05.449220 2023-10-01T09:07:06.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: +0000 mgr.x (mgr.25342) 60 : 2023-10-01T09:07:06.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi190:3000"}]: dispatch 2023-10-01T09:07:06.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:06.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: -10-01T09:07:05.453391 2023-10-01T09:07:06.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: +0000 mon.a (mon.0) 812 : 2023-10-01T09:07:06.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:06.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:06.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: -10-01T09:07:05. 2023-10-01T09:07:06.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 466367+0000 mon.b (mon.1) 2023-10-01T09:07:06.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 90 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:06.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: cephadm 2023 2023-10-01T09:07:06.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: -10-01T09:07:05.468359 2023-10-01T09:07:06.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: +0000 mgr.x (mgr.25342) 61 2023-10-01T09:07:06.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T09:07:06.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: cephadm 2023- 2023-10-01T09:07:06.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 10-01T09:07:05.468754 2023-10-01T09:07:06.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: +0000 mgr.x (mgr.25342) 62 : cephadm 2023-10-01T09:07:06.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T09:07:06.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:06.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: -10-01T09:07:05. 2023-10-01T09:07:06.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 604318+0000 mon.a (mon.0) 2023-10-01T09:07:06.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:06 smithi190 conmon[100518]: 813 : audit [DBG] from='client.? 172.21.15.138:0/3474515219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:08.264 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:06 2023-10-01T09:07:08.264 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: .935781+0000 mgr.x (mgr.25342) 63 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-01T09:07:08.264 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: audit 2023-10-01T09:07:06.941736+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:08.264 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: audit 2023-10-01T09:07:06.942911+0000 mon.b (mon.1) 91 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:07:08.264 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: audit 2023-10-01T09:07:06.943237+0000 mon.a (mon.0 2023-10-01T09:07:08.264 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: ) 815 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:07:08.264 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: audit 2023-10-01T09:07:08.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: 2023-10-01T09:07:06. 2023-10-01T09:07:08.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: 944283+0000 mon.b (mon.1) 92 : audit 2023-10-01T09:07:08.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:07:08.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: audit 2023-10-01T09:07:08.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: 2023-10-01T09:07:06. 2023-10-01T09:07:08.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: 945230+0000 mon.b (mon.1) 93 : audit [DBG] 2023-10-01T09:07:08.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:08.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: cephadm 2023-10 2023-10-01T09:07:08.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: -01T09:07:06.946127+0000 mgr.x ( 2023-10-01T09:07:08.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[100289]: mgr.25342) 64 : cephadm [INF] Deploying daemon mgr.y on smithi138 2023-10-01T09:07:08.266 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: cephadm 2023-10-01T09:07 2023-10-01T09:07:08.266 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: :06.935781+0000 mgr.x (mgr.25342) 63 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-01T09:07:08.266 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:07:08.266 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: 07:06.941736+0000 mon.a (mon.0) 814 : audit 2023-10-01T09:07:08.266 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:08.266 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: audit 2023-10-01T09:07:08.266 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: 2023-10-01T09:07:06. 2023-10-01T09:07:08.266 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: 942911+0000 mon.b (mon.1) 91 : audit 2023-10-01T09:07:08.266 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:07:08.266 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:07:08.267 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: -01T09:07:06.943237+0000 mon.a (mon. 2023-10-01T09:07:08.267 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: 0) 815 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:07:08.267 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: audit 2023-10-01T09:07:08.267 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: 2023-10-01T09:07:06.944283+0000 2023-10-01T09:07:08.267 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: mon.b (mon.1) 92 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:07:08.267 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:07:08.267 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: -01T09:07:06.945230+0000 mon.b (mon. 2023-10-01T09:07:08.267 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: 1) 93 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:08.267 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: cephadm 2023 2023-10-01T09:07:08.267 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: -10-01T09:07:06.946127+0000 2023-10-01T09:07:08.267 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:07 smithi138 conmon[109959]: mgr.x (mgr.25342) 64 : cephadm [INF] Deploying daemon mgr.y on smithi138 2023-10-01T09:07:08.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[139437]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:07:07] "GET /metrics HTTP/1.1" 200 33785 "" "Prometheus/2.33.4" 2023-10-01T09:07:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:06.935781+0000 mgr.x (mgr.25342) 63 : 2023-10-01T09:07:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: cephadm [INF] Upgrade: Updating mgr.y 2023-10-01T09:07:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: audit 2023-10-01T09:07:06.941736+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:07:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: -01T09:07:06.942911+0000 mon.b (mon.1) 91 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:07:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:07:08.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: 01T09:07:06.943237+0000 mon.a (mon.0) 815 2023-10-01T09:07:08.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:07:08.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:07:08.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: :07:06.944283+0000 mon.b (mon.1) 92 2023-10-01T09:07:08.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:07:08.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: audit 2023-10-01T09:07 2023-10-01T09:07:08.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: :06.945230+0000 mon.b (mon.1) 93 : 2023-10-01T09:07:08.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:08.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: cephadm 2023-10-01T09: 2023-10-01T09:07:08.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: 07:06.946127+0000 mgr.x (mgr.25342) 64 2023-10-01T09:07:08.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:07 smithi190 conmon[100518]: : cephadm [INF] Deploying daemon mgr.y on smithi138 2023-10-01T09:07:08.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:08 smithi138 systemd[1]: Stopping Ceph mgr.y for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:07:08.945 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:08 smithi190 conmon[145209]: ts=2023-10-01T09:07:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:07:09.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:08 smithi190 conmon[100518]: cluster 2023-10-01T09:07:07.051010+0000 mgr.x (mgr.25342) 65 : cluster 2023-10-01T09:07:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:08 smithi190 conmon[100518]: [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:08 smithi190 conmon[100518]: audit 2023-10-01T09:07:07.989940+0000 mon.c (mon.2) 151 : audit [DBG] from='client.? 172.21.15.138:0/1622584113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:09.318 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:08 smithi138 conmon[109959]: cluster 2023-10-01T09:07:07.051010+0000 mgr.x 2023-10-01T09:07:09.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:08 smithi138 conmon[109959]: (mgr.25342) 65 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:09.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:08 smithi138 conmon[109959]: audit 2023-10-01T09:07:07.989940+0000 mon.c (mon.2) 151 : audit [DBG] from='client.? 172.21.15.138:0/1622584113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:09.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:08 smithi138 conmon[100289]: cluster 2023-10-01T09:07:07.051010+0000 2023-10-01T09:07:09.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:08 smithi138 conmon[100289]: mgr.x (mgr.25342) 65 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:09.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:08 smithi138 conmon[100289]: audit 2023-10-01T09:07:07.989940+0000 mon.c (mon.2) 151 : audit [DBG] from='client.? 172.21.15.138:0/1622584113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:09.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:09 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-10-01T09:07:09.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:09 smithi138 bash[178236]: 5d4bcde0f5d467a176e0f57f0a5fec120abba85bba8bb289ceaeee759499c655 2023-10-01T09:07:09.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:09 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-10-01T09:07:09.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:09 smithi138 systemd[1]: Stopped Ceph mgr.y for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:07:10.054 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:09 smithi138 systemd[1]: Starting Ceph mgr.y for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:07:10.501 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:10 smithi138 ceph-mgr[178676]: -- 172.21.15.138:0/2211476053 <== mon.0 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a7961191e0 con 0x55a7960f1000 2023-10-01T09:07:10.502 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:10 smithi138 bash[178589]: 97da5dac6a8473268d8814cd9ca391d58ba82c086e83e28067e26baa239920fc 2023-10-01T09:07:10.502 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:10 smithi138 systemd[1]: Started Ceph mgr.y for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:07:10.502 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[178630]: 2023-10-01T09:07:10.323+0000 7fbfead04200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T09:07:10.502 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[178630]: 2023-10-01T09:07:10.409+0000 7fbfead04200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T09:07:10.502 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[178630]: 2023-10-01T09:07:10.501+0000 7fbfead04200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T09:07:10.813 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[178630]: 2023-10-01T09:07:10.615+0000 7fbfead04200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T09:07:11.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[178630]: 2023-10-01T09:07:10.813+0000 7fbfead04200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T09:07:11.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[100289]: cluster 2023-10-01T09:07:09.051522+0000 mgr.x (mgr 2023-10-01T09:07:11.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[100289]: .25342) 66 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:11.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[100289]: audit 2023-10-01T09:07:10.268359+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:11.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[100289]: audit 2023-10-01T09:07:10.276823+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:11.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[100289]: audit 2023-10-01T09:07:10.318038+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:11.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[100289]: audit 2023-10-01T09:07:10.392469+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.138:0/1589884024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:11.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: cluster 2023-10-01T09:07:09 2023-10-01T09:07:11.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: .051522+0000 mgr.x (mgr.25342) 66 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:11.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:07:11.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: T09:07:10.268359+0000 mon.a (mon.0) 816 : audit [INF] 2023-10-01T09:07:11.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:11.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: audit 2023 2023-10-01T09:07:11.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: -10-01T09:07:10 2023-10-01T09:07:11.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: .276823+0000 mon.a (mon. 2023-10-01T09:07:11.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: 0) 817 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:11.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: 2023-10-01T09:07:11.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:07:11.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: -01T09:07:10. 2023-10-01T09:07:11.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: 318038+0000 mon.b (mon.1) 2023-10-01T09:07:11.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: 94 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:11.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:07:11.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: -01T09:07:10. 2023-10-01T09:07:11.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: 392469+0000 mon.c (mon.2) 2023-10-01T09:07:11.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:10 smithi138 conmon[109959]: 152 : audit [DBG] from='client.? 172.21.15.138:0/1589884024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:10 smithi190 conmon[100518]: cluster 2023-10-01T09:07:09.051522+0000 mgr.x (mgr.25342) 2023-10-01T09:07:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:10 smithi190 conmon[100518]: 66 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:10 smithi190 conmon[100518]: audit 2023-10-01T09:07:10.268359+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:10 smithi190 conmon[100518]: audit 2023-10-01T09:07:10.276823+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:10 smithi190 conmon[100518]: audit 2023-10-01T09:07:10.318038+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:10 smithi190 conmon[100518]: audit 2023-10-01T09:07:10.392469+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.138:0/1589884024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:11.557 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:11 smithi138 conmon[178630]: 2023-10-01T09:07:11.207+0000 7fbfead04200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T09:07:11.557 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:11 smithi138 conmon[178630]: 2023-10-01T09:07:11.361+0000 7fbfead04200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T09:07:11.558 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:11 smithi138 conmon[178630]: 2023-10-01T09:07:11.455+0000 7fbfead04200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T09:07:11.558 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:11 smithi138 conmon[178630]: 2023-10-01T09:07:11.556+0000 7fbfead04200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T09:07:12.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:11 smithi190 conmon[100518]: audit 2023-10-01T09:07:11.246122+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:07:12.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:12 smithi138 conmon[178630]: 2023-10-01T09:07:12.039+0000 7fbfead04200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T09:07:12.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:11 smithi138 conmon[100289]: audit 2023-10-01T09:07:11.246122+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:07:12.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:11 smithi138 conmon[100289]: 2023-10-01T09:07:12.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:11 smithi138 conmon[109959]: audit 2023-10-01T09:07:11.246122+0000 mon.b (mon.1 2023-10-01T09:07:12.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:11 smithi138 conmon[109959]: ) 95 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:07:13.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:12 smithi138 conmon[100289]: cluster 2023-10-01T09:07:11.051824+0000 mgr.x ( 2023-10-01T09:07:13.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:12 smithi138 conmon[100289]: mgr.25342) 67 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:13.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:12 smithi138 conmon[100289]: audit 2023-10-01T09:07:12.837407+0000 mon.a (mon.0) 818 : audit 2023-10-01T09:07:13.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:12 smithi138 conmon[100289]: [DBG] from='client.? 172.21.15.138:0/868331546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:13.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:12 smithi138 conmon[109959]: cluster 2023-10-01T09:07:11.051824+0000 mgr.x (mgr.25342) 67 : cluster 2023-10-01T09:07:13.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:12 smithi138 conmon[109959]: [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:13.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:12 smithi138 conmon[109959]: audit 2023-10-01T09:07:12.837407+0000 mon.a (mon.0) 818 : audit [DBG] from='client.? 172.21.15.138:0/868331546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:13.071 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:12 smithi138 conmon[178630]: 2023-10-01T09:07:12.780+0000 7fbfead04200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T09:07:13.071 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:12 smithi138 conmon[178630]: 2023-10-01T09:07:12.881+0000 7fbfead04200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T09:07:13.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:12 smithi190 conmon[100518]: cluster 2023-10-01T09:07:11.051824 2023-10-01T09:07:13.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:12 smithi190 conmon[100518]: +0000 mgr.x (mgr.25342) 67 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:12 smithi190 conmon[100518]: audit 2023-10-01T09:07:12.837407+0000 mon.a (mon. 2023-10-01T09:07:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:12 smithi190 conmon[100518]: 0) 818 : audit [DBG] from='client.? 172.21.15.138:0/868331546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:14.024 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:13 smithi138 conmon[178630]: 2023-10-01T09:07:13.724+0000 7fbfead04200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:07:14.024 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:13 smithi138 conmon[178630]: 2023-10-01T09:07:13.837+0000 7fbfead04200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T09:07:14.024 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[178630]: 2023-10-01T09:07:14.024+0000 7fbfead04200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:07:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: cluster 2023-10-01T09:07:13.052327 2023-10-01T09:07:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: +0000 mgr.x (mgr.25342) 68 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: audit 2023-10-01T09:07:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: 2023-10-01T09:07:13.143542+0000 mon.a (mon.0 2023-10-01T09:07:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: ) 819 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: audit 2023-10-01T09:07:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: 2023-10-01T09:07:13.149246+0000 mon.a (mon. 2023-10-01T09:07:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: 0) 820 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:07:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: 07:13.447715+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: 10-01T09:07:13.455796+0000 mon.a (mon.0) 822 : 2023-10-01T09:07:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: audit 2023-10-01T09:07:13.841867 2023-10-01T09:07:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: +0000 mon.a (mon.0) 823 : audit [INF] 2023-10-01T09:07:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: audit 2023-10-01T09:07: 2023-10-01T09:07:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: 13.849973+0000 mon.a (mon.0) 824 : 2023-10-01T09:07:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[109959]: audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[100289]: cluster 2023-10-01T09:07:13.052327+0000 mgr.x (mgr.25342) 68 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:14.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[100289]: audit 2023-10-01T09:07:13.143542+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:07:14.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[100289]: 10-01T09:07:13.149246+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[100289]: audit 2023-10-01T09:07:13.447715+0000 mon.a (mon.0) 2023-10-01T09:07:14.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[100289]: 821 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[100289]: audit 2023-10-01T09:07:13.455796+0000 mon.a (mon.0) 822 : audit 2023-10-01T09:07:14.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[100289]: [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[100289]: audit 2023-10-01T09:07:13.841867 2023-10-01T09:07:14.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[100289]: +0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[100289]: 2023-10-01T09:07:14.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[100289]: audit 2023-10-01T09:07 2023-10-01T09:07:14.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[100289]: :13.849973+0000 mon.a (mon.0) 824 : audit [INF] 2023-10-01T09:07:14.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[100289]: from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:14 smithi190 conmon[100518]: cluster 2023-10-01T09:07:13.052327+0000 mgr.x (mgr.25342) 68 : 2023-10-01T09:07:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:14 smithi190 conmon[100518]: cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:14 smithi190 conmon[100518]: audit 2023-10-01T09:07:13.143542+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:14 smithi190 conmon[100518]: audit 2023-10-01T09:07:13.149246+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:14 smithi190 conmon[100518]: audit 2023-10-01T09:07:13.447715+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:14 smithi190 conmon[100518]: audit 2023-10-01T09:07:13.455796+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:14 smithi190 conmon[100518]: audit 2023-10-01T09: 2023-10-01T09:07:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:14 smithi190 conmon[100518]: 07:13.841867+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:14 smithi190 conmon[100518]: audit 2023-10-01T09:07:13.849973+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:14.634 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[178630]: 2023-10-01T09:07:14.354+0000 7fbfead04200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T09:07:14.727 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[178630]: 2023-10-01T09:07:14.598+0000 7fbfead04200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T09:07:15.052 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:14 smithi138 conmon[178630]: 2023-10-01T09:07:14.761+0000 7fbfead04200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T09:07:15.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:15 smithi138 conmon[178630]: 2023-10-01T09:07:15.052+0000 7fbfead04200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T09:07:15.963 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:15 smithi138 conmon[178630]: 2023-10-01T09:07:15.818+0000 7fbfead04200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T09:07:15.963 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:15 smithi138 conmon[178630]: 2023-10-01T09:07:15.962+0000 7fbfead04200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T09:07:16.300 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:16 smithi138 conmon[109959]: cluster 2023-10-01T09:07:15 2023-10-01T09:07:16.300 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:16 smithi138 conmon[109959]: .052853+0000 mgr.x (mgr.25342) 69 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:16.300 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:16 smithi138 conmon[109959]: audit 2023-10-01T09:07:15.231639+0000 mon.a (mon.0) 825 : audit [DBG] from='client.? 172.21.15.138:0/3833036166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:16.300 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:16 smithi138 conmon[100289]: cluster 2023-10-01T09:07:15.052853+0000 mgr.x (mgr.25342) 69 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:16.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:16 smithi138 conmon[100289]: audit 2023-10-01T09:07:15.231639+0000 mon.a (mon.0) 825 : audit [DBG] from='client.? 172.21.15.138:0/3833036166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:16.301 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:16 smithi138 conmon[178630]: 2023-10-01T09:07:16.173+0000 7fbfead04200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T09:07:16.301 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:16 smithi138 conmon[178630]: 2023-10-01T09:07:16.300+0000 7fbfead04200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T09:07:16.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:16 smithi190 conmon[100518]: cluster 2023-10-01T09:07:15.052853+0000 mgr.x (mgr.25342) 69 : cluster 2023-10-01T09:07:16.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:16 smithi190 conmon[100518]: [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:16.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:16 smithi190 conmon[100518]: audit 2023-10-01T09:07:15.231639+0000 mon.a (mon.0) 825 : audit [DBG] from='client.? 172.21.15.138:0/3833036166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:16.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:16 smithi138 conmon[178630]: 2023-10-01T09:07:16.537+0000 7fbfead04200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T09:07:17.473 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: 2023-10-01T09:07:17.457+0000 7fbfead04200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T09:07:17.473 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: [01/Oct/2023:09:07:17] ENGINE Bus STARTING 2023-10-01T09:07:17.530 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:17 smithi190 conmon[139437]: debug 2023-10-01T09:07:17.193+0000 7f5f5f5a1700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-01T09:07:17.530 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:17 smithi190 conmon[139437]: ignoring --setuser ceph since I am not root 2023-10-01T09:07:17.530 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:17 smithi190 conmon[139437]: ignoring --setgroup ceph since I am not root 2023-10-01T09:07:17.530 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:17 smithi190 conmon[139437]: debug 2023-10-01T09:07:17.244+0000 7f9b1f101700 1 -- 172.21.15.190:0/2322162757 <== mon.1 v2:172.21.15.190:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556fa16371e0 con 0x556fa160e800 2023-10-01T09:07:17.531 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:17 smithi190 conmon[139437]: debug 2023-10-01T09:07:17.358+0000 7f9b28e31200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T09:07:17.531 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:17 smithi190 conmon[139437]: debug 2023-10-01T09:07:17.438+0000 7f9b28e31200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T09:07:17.531 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:17 smithi190 conmon[139437]: debug 2023-10-01T09:07:17.530+0000 7f9b28e31200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T09:07:17.746 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: CherryPy Checker: 2023-10-01T09:07:17.746 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: The Application mounted at '' has an empty config. 2023-10-01T09:07:17.746 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: 2023-10-01T09:07:17.746 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: [01/Oct/2023:09:07:17] ENGINE Serving on http://:::9283 2023-10-01T09:07:17.746 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: [01/Oct/2023:09:07:17] ENGINE Bus STARTED 2023-10-01T09:07:17.746 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: [01/Oct/2023:09:07:17] ENGINE Bus STOPPING 2023-10-01T09:07:17.746 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: [01/Oct/2023:09:07:17] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T09:07:17.746 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: [01/Oct/2023:09:07:17] ENGINE Bus STOPPED 2023-10-01T09:07:17.746 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: [01/Oct/2023:09:07:17] ENGINE Bus STARTING 2023-10-01T09:07:17.804 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:17 smithi190 conmon[139437]: debug 2023-10-01T09:07:17.622+0000 7f9b28e31200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T09:07:18.027 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: 2023-10-01T09:07:17.751+0000 7fbf4496e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:18.028 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: 2023-10-01T09:07:17.751+0000 7fbf4496e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:18.028 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: 2023-10-01T09:07:17.751+0000 7fbf4496e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:18.028 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: 2023-10-01T09:07:17.751+0000 7fbf4496e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:18.028 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: 2023-10-01T09:07:17.751+0000 7fbf4496e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:18.028 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: 2023-10-01T09:07:17.753+0000 7fbf4a179700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:18.028 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: 2023-10-01T09:07:17.753+0000 7fbf4a179700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:18.028 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: 2023-10-01T09:07:17.753+0000 7fbf4a179700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:18.028 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: 2023-10-01T09:07:17.753+0000 7fbf4a179700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:18.028 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: 2023-10-01T09:07:17.753+0000 7fbf4a179700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:18.028 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: [01/Oct/2023:09:07:17] ENGINE Serving on http://:::9283 2023-10-01T09:07:18.029 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:17 smithi138 conmon[178630]: [01/Oct/2023:09:07:17] ENGINE Bus STARTED 2023-10-01T09:07:18.056 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:17 smithi190 conmon[139437]: debug 2023-10-01T09:07:17.804+0000 7f9b28e31200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T09:07:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:17.027036+0000 mon.a (mon.0 2023-10-01T09:07:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: ) 826 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:17.033906+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:17.035243+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:17.036076+0000 mon.b (mon.1) 97 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:07:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:17.041424+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:17.179741+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:17.184957+0000 mon.b (mon.1) 99 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-01T09:07:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:17.185216+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-01T09:07:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: cluster 2023-10-01T09:07:17.192294+0000 mon.a (mon.0) 830 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-01T09:07:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:17.192520+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-01T09:07:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: cluster 2023-10-01T09:07:17.192595+0000 mon.a (mon.0) 832 : cluster [DBG] mgrmap e24: y(active, starting, since 0.00695701s) 2023-10-01T09:07:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: cluster 2023-10-01T09:07:17.470385+0000 mon.a (mon.0) 833 : cluster [INF] Active manager daemon y restarted 2023-10-01T09:07:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: cluster 2023-10-01T09:07:17.470868+0000 mon.a (mon.0) 834 : cluster [INF] Activating manager daemon y 2023-10-01T09:07:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: cluster 2023-10-01T09:07:17.475872+0000 mon.a (mon.0) 835 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-01T09:07:18.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: cluster 2023-10-01T09:07:17.476120+0000 mon.a (mon.0) 836 : cluster [DBG] mgrmap e25: y(active, starting, since 0.00543536s) 2023-10-01T09:07:18.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:17.476469+0000 mon.a (mon.0) 837 : audit [DBG] from='mgr.? 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T09:07:18.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:17.480585+0000 mon.c (mon.2) 153 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T09:07:18.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:17.480799+0000 mon.c (mon.2) 2023-10-01T09:07:18.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 154 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:07:18.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:18.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023-10- 2023-10-01T09:07:18.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 01T09:07:17.481025+0000 2023-10-01T09:07:18.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: mon.c (mon.2) 155 : audit [DBG] 2023-10-01T09:07:18.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:07:18.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:18.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023-10 2023-10-01T09:07:18.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -01T09:07:17.481245 2023-10-01T09:07:18.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: +0000 mon.c (mon.2) 156 : audit [DBG] 2023-10-01T09:07:18.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:07:18.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:18.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023-10 2023-10-01T09:07:18.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -01T09:07:17. 2023-10-01T09:07:18.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 481449+0000 mon.c (mon.2) 2023-10-01T09:07:18.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 157 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.brhzod"}]: dispatch 2023-10-01T09:07:18.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:07:18.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 01T09:07:17.481754+0000 2023-10-01T09:07:18.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: mon.c (mon.2) 158 : audit 2023-10-01T09:07:18.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi190.weynjc"}]: dispatch 2023-10-01T09:07:18.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:18.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023-10-01T09: 2023-10-01T09:07:18.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 07:17.482185+0000 mon.c ( 2023-10-01T09:07:18.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: mon.2) 159 : audit [DBG] 2023-10-01T09:07:18.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T09:07:18.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:18.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023-10-01T09: 2023-10-01T09:07:18.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 07:17.482467+0000 mon.c ( 2023-10-01T09:07:18.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: mon.2) 160 : audit [DBG] 2023-10-01T09:07:18.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T09:07:18.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:18.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023-10- 2023-10-01T09:07:18.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 01T09:07:17.482742 2023-10-01T09:07:18.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: +0000 mon.c (mon.2) 161 2023-10-01T09:07:18.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T09:07:18.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:18.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023-10-01T09: 2023-10-01T09:07:18.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 07:17.482949+0000 mon.c (mon 2023-10-01T09:07:18.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: .2) 162 : audit [DBG] 2023-10-01T09:07:18.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T09:07:18.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:18.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023-10-01T09: 2023-10-01T09:07:18.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 07:17.483213+0000 mon.c ( 2023-10-01T09:07:18.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: mon.2) 163 : audit [DBG] 2023-10-01T09:07:18.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:07:18.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:07:18.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 01T09:07:17.483530+0000 2023-10-01T09:07:18.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: mon.c (mon.2) 164 : audit 2023-10-01T09:07:18.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:07:18.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:07:18.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 10-01T09:07:17. 2023-10-01T09:07:18.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 483822+0000 mon.c (mon.2) 165 2023-10-01T09:07:18.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:07:18.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:18.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -10-01T09:07:17. 2023-10-01T09:07:18.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 484068+0000 mon.c (mon.2) 2023-10-01T09:07:18.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 166 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:07:18.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:18.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -10-01T09:07:17. 2023-10-01T09:07:18.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 484348+0000 mon.c (mon.2) 2023-10-01T09:07:18.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 167 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:07:18.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:18.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -10-01T09:07: 2023-10-01T09:07:18.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 17.484582+0000 mon.c (mon. 2023-10-01T09:07:18.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2) 168 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T09:07:18.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:18.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023-10-01T09:07 2023-10-01T09:07:18.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: :17.484824+0000 mon.c (mon 2023-10-01T09:07:18.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: .2) 169 : audit [DBG] 2023-10-01T09:07:18.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T09:07:18.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:18.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023-10 2023-10-01T09:07:18.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -01T09:07:17. 2023-10-01T09:07:18.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 485618+0000 mon.c (mon.2) 2023-10-01T09:07:18.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 170 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T09:07:18.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:18.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -10-01T09:07:17. 2023-10-01T09:07:18.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 485865+0000 mon.c (mon.2) 2023-10-01T09:07:18.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 171 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:07:18.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:18.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -10-01T09:07:17 2023-10-01T09:07:18.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: .487209+0000 mon.c (mon.2 2023-10-01T09:07:18.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: ) 172 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-01T09:07:18.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:18.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023-10-01T09:07: 2023-10-01T09:07:18.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 17.487542+0000 mon.c (mon. 2023-10-01T09:07:18.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2) 173 : audit [DBG] 2023-10-01T09:07:18.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:07:18.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023-10-01T09:07:18.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023- 2023-10-01T09:07:18.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 10-01T09:07:17 2023-10-01T09:07:18.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: .633164+0000 mon.b (mon.1 2023-10-01T09:07:18.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: ) 100 : audit [DBG] from='client.? 172.21.15.138:0/662490667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:18.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023-10-01T09:07:18.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: cluster 2023-10-01T09:07:18.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023-10-01T09:07 2023-10-01T09:07:18.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: :17.651912+0000 mon.a (mon.0 2023-10-01T09:07:18.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: ) 838 : cluster [INF] Manager daemon y is now available 2023-10-01T09:07:18.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:18.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -10-01T09:07:17. 2023-10-01T09:07:18.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 670413+0000 mon.c (mon.2) 174 2023-10-01T09:07:18.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:18.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:18.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -10-01T09:07:17 2023-10-01T09:07:18.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: .671959+0000 mon.c (mon.2 2023-10-01T09:07:18.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: ) 175 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:18.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:18.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -10-01T09:07:17. 2023-10-01T09:07:18.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 692046+0000 mon.c (mon.2 2023-10-01T09:07:18.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: ) 176 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:07:18.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:18.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -10-01T09:07:17. 2023-10-01T09:07:18.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 698314+0000 mon.c (mon.2) 177 2023-10-01T09:07:18.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:07:18.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:18.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -10-01T09:07 2023-10-01T09:07:18.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: :17.698683+0000 mon.a (mon. 2023-10-01T09:07:18.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 0) 839 : audit [INF] from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:07:18.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:18.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -10-01T09:07:17 2023-10-01T09:07:18.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: .723566+0000 mon.c (mon. 2023-10-01T09:07:18.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2) 178 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T09:07:18.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: 2023-10-01T09:07:18.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:18.070 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: -10-01T09:07:17.723964+0000 mon.a 2023-10-01T09:07:18.070 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[100518]: (mon.0) 840 : audit [INF] from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T09:07:18.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:17.027036+0000 mon.a (mon. 2023-10-01T09:07:18.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 0) 826 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:18.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:07:18.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 01T09:07:17.033906+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:18.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:17 2023-10-01T09:07:18.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: .035243+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:18.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 2023-10-01T09:07:18.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:07:18.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: :07:17.036076+0000 mon.b (mon.1) 97 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:07:18.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:07:18.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: T09:07:17.041424+0000 mon.a (mon. 2023-10-01T09:07:18.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 0) 828 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:18.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:07:18.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 07:17.179741+0000 mon.b (mon.1) 98 : audit 2023-10-01T09:07:18.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:18.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:17.184957 2023-10-01T09:07:18.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: +0000 mon.b (mon.1) 99 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-01T09:07:18.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:17.185216+0000 mon.a 2023-10-01T09:07:18.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: (mon.0) 829 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-01T09:07:18.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: cluster 2023-10-01T09:07:18.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 2023-10-01T09:07:17.192294+0000 mon.a (mon 2023-10-01T09:07:18.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: .0) 830 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-10-01T09:07:18.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:17.192520+0000 mon.a 2023-10-01T09:07:18.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: (mon.0) 831 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-01T09:07:18.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: cluster 2023-10-01T09:07:17.192595 2023-10-01T09:07:18.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: +0000 mon.a (mon.0) 832 : cluster [DBG] 2023-10-01T09:07:18.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: mgrmap e24: y(active, starting, since 0.00695701s) 2023-10-01T09:07:18.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: cluster 2023-10- 2023-10-01T09:07:18.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 01T09:07:17.470385+0000 mon.a (mon.0) 833 : cluster [INF] Active manager daemon y restarted 2023-10-01T09:07:18.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: cluster 2023-10-01T09 2023-10-01T09:07:18.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: :07:17.470868+0000 mon.a (mon.0 2023-10-01T09:07:18.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: ) 834 : cluster [INF] Activating manager daemon y 2023-10-01T09:07:18.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: cluster 2023-10-01T09:07:18.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 2023-10-01T09:07:17.475872+0000 mon.a (mon.0) 835 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-01T09:07:18.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: cluster 2023-10-01T09:07:17 2023-10-01T09:07:18.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: .476120+0000 mon.a (mon.0) 836 : cluster [DBG] mgrmap e25: y(active, starting, since 0.00543536s) 2023-10-01T09:07:18.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07: 2023-10-01T09:07:18.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 17.476469+0000 mon.a (mon.0) 837 : audit 2023-10-01T09:07:18.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: [DBG] from='mgr.? 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T09:07:18.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:18.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17.027036+0000 mon.a (mon.0) 826 : audit [INF] 2023-10-01T09:07:18.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:18.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17.033906+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:18.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17.035243+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:18.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17.036076+0000 mon.b (mon.1) 97 : audit [INF] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:07:18.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17.041424+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.25342 ' entity='mgr.x' 2023-10-01T09:07:18.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17.179741+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:18.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17.184957+0000 mon.b (mon.1) 99 : audit [INF] 2023-10-01T09:07:18.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: from='mgr.25342 172.21.15.190:0/932009593' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-01T09:07:18.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17.185216+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-01T09:07:18.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: cluster 2023-10- 2023-10-01T09:07:18.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 01T09:07:17.192294+0000 mon.a (mon.0) 830 : cluster [DBG] 2023-10-01T09:07:18.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: osdmap e83: 8 total, 8 up, 8 in 2023-10-01T09:07:18.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10 2023-10-01T09:07:18.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: -01T09:07:17.192520+0000 mon.a ( 2023-10-01T09:07:18.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: mon.0) 831 : audit [INF] from='mgr.25342 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-01T09:07:18.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: cluster 2023-10 2023-10-01T09:07:18.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: -01T09:07:17.192595+0000 mon.a (mon 2023-10-01T09:07:18.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: .0) 832 : cluster [DBG] mgrmap e24: y(active, starting, since 0.00695701s) 2023-10-01T09:07:18.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: cluster 2023-10-01 2023-10-01T09:07:18.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: T09:07:17.470385+0000 mon.a (mon.0 2023-10-01T09:07:18.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: ) 833 : cluster [INF] Active manager daemon y restarted 2023-10-01T09:07:18.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: cluster 2023-10-01T09:07:17 2023-10-01T09:07:18.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: .470868+0000 mon.a (mon.0) 834 2023-10-01T09:07:18.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: : cluster [INF] Activating manager daemon y 2023-10-01T09:07:18.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: cluster 2023-10-01T09:07:18.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 2023-10-01T09:07:17.475872+0000 2023-10-01T09:07:18.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: mon.a (mon.0) 835 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-01T09:07:18.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: cluster 2023-10-01T09:07:17.476120+0000 mon.a (mon.0) 836 : cluster [DBG] mgrmap e25: y(active, starting, since 0.00543536s) 2023-10-01T09:07:18.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17.476469+0000 mon.a (mon.0) 837 : audit [DBG] from='mgr.? 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T09:07:18.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17.480585+0000 mon.c (mon.2) 153 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T09:07:18.328 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17.480799+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:07:18.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 10-01T09:07:17.480585+0000 mon.c (mon.2) 153 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T09:07:18.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:07:18.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: :07:17.480799+0000 mon.c (mon.2) 2023-10-01T09:07:18.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 154 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:07:18.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:17.481025+0000 2023-10-01T09:07:18.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: mon.c (mon.2) 155 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:07:18.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:07:18.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 01T09:07:17.481245+0000 mon.c (mon. 2023-10-01T09:07:18.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 2) 156 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:07:18.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:18.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 2023-10-01T09:07:17.481449+0000 mon.c (mon.2) 157 : 2023-10-01T09:07:18.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.brhzod"}]: dispatch 2023-10-01T09:07:18.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:17.481754 2023-10-01T09:07:18.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: +0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi190.weynjc"}]: dispatch 2023-10-01T09:07:18.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:17. 2023-10-01T09:07:18.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 482185+0000 mon.c (mon.2) 159 : audit 2023-10-01T09:07:18.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T09:07:18.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:07:18.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: -01T09:07:17.482467+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T09:07:18.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:17 2023-10-01T09:07:18.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: .482742+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T09:07:18.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:17.482949+0000 mon.c ( 2023-10-01T09:07:18.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: mon.2) 162 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T09:07:18.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:18.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 2023-10-01T09:07:17.483213+0000 mon.c (mon.2) 163 : audit [DBG] 2023-10-01T09:07:18.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:07:18.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:07:18.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 07:17.483530+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:07:18.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:18.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 2023-10-01T09:07:17.483822+0000 mon.c (mon.2) 165 : audit 2023-10-01T09:07:18.331 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:07:18.331 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:07:18.331 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 07:17.484068+0000 mon.c (mon.2) 166 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:07:18.331 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:17.484348+0000 mon.c (mon 2023-10-01T09:07:18.331 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17.481025+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:07:18.331 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17.481245+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:07:18.331 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17.481449+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.brhzod"}]: dispatch 2023-10-01T09:07:18.331 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:07:18.331 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 07:17.481754+0000 mon.c (mon.2) 2023-10-01T09:07:18.331 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 158 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi190.weynjc"}]: dispatch 2023-10-01T09:07:18.332 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:07:18.332 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 07:17.482185+0000 mon.c (mon.2) 159 2023-10-01T09:07:18.332 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T09:07:18.332 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07: 2023-10-01T09:07:18.332 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 17.482467+0000 mon.c (mon.2) 160 : audit 2023-10-01T09:07:18.332 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T09:07:18.332 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07: 2023-10-01T09:07:18.332 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 17.482742+0000 mon.c (mon.2) 161 : audit 2023-10-01T09:07:18.333 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T09:07:18.333 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07: 2023-10-01T09:07:18.333 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 17.482949+0000 mon.c (mon.2) 162 : audit 2023-10-01T09:07:18.333 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T09:07:18.333 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T09:07:18.333 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: T09:07:17.483213+0000 mon.c (mon.2) 2023-10-01T09:07:18.333 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 163 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:07:18.333 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:07:18.333 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: :07:17.483530+0000 mon.c (mon.2) 2023-10-01T09:07:18.333 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 164 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:07:18.333 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:07:18.333 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 07:17.483822+0000 mon.c (mon.2) 165 : 2023-10-01T09:07:18.334 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:07:18.334 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07: 2023-10-01T09:07:18.334 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 17.484068+0000 mon.c (mon.2) 166 : audit 2023-10-01T09:07:18.334 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:07:18.334 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07: 2023-10-01T09:07:18.334 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 17.484348+0000 mon.c (mon.2) 167 : audit 2023-10-01T09:07:18.334 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:07:18.335 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17. 2023-10-01T09:07:18.335 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 484582+0000 mon.c (mon.2) 168 : audit [DBG] 2023-10-01T09:07:18.335 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T09:07:18.335 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: .2) 167 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023 2023-10-01T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: -10-01T09:07:17.484582+0000 mon.c (mon.2) 168 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 2023-10-01T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07: 2023-10-01T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 17.484824+0000 mon.c (mon.2) 169 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: T09:07:17.485618+0000 mon.c (mon.2) 170 : audit 2023-10-01T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T09:07:18.336 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:07:18.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 01T09:07:17.485865+0000 mon.c (mon.2) 171 : audit 2023-10-01T09:07:18.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:07:18.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:18.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 2023-10-01T09:07:17.487209+0000 mon.c (mon.2) 172 : audit [DBG] 2023-10-01T09:07:18.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-01T09:07:18.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:18.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 10-01T09:07:17.487542+0000 mon.c (mon.2) 173 2023-10-01T09:07:18.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:07:18.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:17.633164+0000 mon.b 2023-10-01T09:07:18.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: (mon.1) 100 : audit [DBG] from='client.? 172.21.15.138:0/662490667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:18.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: cluster 2023-10- 2023-10-01T09:07:18.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 01T09:07:17.651912+0000 mon.a (mon 2023-10-01T09:07:18.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: .0) 838 : cluster [INF] Manager daemon y is now available 2023-10-01T09:07:18.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:18.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 2023-10-01T09:07:17.670413+0000 mon.c (mon.2) 2023-10-01T09:07:18.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 174 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:18.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:17.671959 2023-10-01T09:07:18.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: +0000 mon.c (mon.2) 175 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:18.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:07:18.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 07:17.692046+0000 mon.c (mon.2) 176 : 2023-10-01T09:07:18.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:07:18.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:17.698314+0000 mon.c ( 2023-10-01T09:07:18.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: mon.2) 177 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:07:18.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:07:18.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 09:07:17.698683+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:07:18.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 2023-10-01T09:07:18.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:18.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 2023-10-01T09:07:17.723566+0000 mon.c (mon.2) 2023-10-01T09:07:18.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 178 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T09:07:18.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: audit 2023-10-01T09:07:18.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[109959]: 2023-10-01T09:07:17.723964+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T09:07:18.339 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 2023-10-01T09:07:17.484824+0000 mon.c (mon. 2023-10-01T09:07:18.340 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 2) 169 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T09:07:18.340 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:07:18.340 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 09:07:17.485618+0000 mon.c (mon.2 2023-10-01T09:07:18.340 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: ) 170 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T09:07:18.340 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07: 2023-10-01T09:07:18.340 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 17.485865+0000 mon.c (mon.2) 171 : audit 2023-10-01T09:07:18.340 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:07:18.340 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:07:18.340 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 07:17.487209+0000 mon.c (mon.2) 172 : audit 2023-10-01T09:07:18.340 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-01T09:07:18.340 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17. 2023-10-01T09:07:18.341 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 487542+0000 mon.c (mon.2) 173 : audit [DBG] 2023-10-01T09:07:18.341 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:07:18.341 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:18.341 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 2023-10-01T09:07:17.633164+0000 mon.b (mon.1 2023-10-01T09:07:18.341 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: ) 100 : audit [DBG] from='client.? 172.21.15.138:0/662490667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:18.341 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: cluster 2023-10-01T09 2023-10-01T09:07:18.341 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: :07:17.651912+0000 mon.a (mon.0) 2023-10-01T09:07:18.341 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 838 : cluster [INF] Manager daemon y is now available 2023-10-01T09:07:18.341 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:07:18.341 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 07:17.670413+0000 mon.c (mon.2) 174 : audit 2023-10-01T09:07:18.341 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:18.341 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07 2023-10-01T09:07:18.342 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: :17.671959+0000 mon.c (mon.2) 175 2023-10-01T09:07:18.342 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:18.342 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07 2023-10-01T09:07:18.342 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: :17.692046+0000 mon.c (mon.2) 176 2023-10-01T09:07:18.342 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:07:18.342 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17 2023-10-01T09:07:18.342 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: .698314+0000 mon.c (mon.2) 177 : audit 2023-10-01T09:07:18.342 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:07:18.342 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17 2023-10-01T09:07:18.342 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: .698683+0000 mon.a (mon.0) 839 : audit 2023-10-01T09:07:18.342 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: [INF] from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:07:18.342 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:17. 2023-10-01T09:07:18.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 723566+0000 mon.c (mon.2) 178 : audit 2023-10-01T09:07:18.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T09:07:18.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: audit 2023-10-01T09:07:18.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: 2023-10-01T09:07:17.723964+0000 mon.a ( 2023-10-01T09:07:18.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:18 smithi138 conmon[100289]: mon.0) 840 : audit [INF] from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T09:07:18.458 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[139437]: debug 2023-10-01T09:07:18.191+0000 7f9b28e31200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T09:07:18.459 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[139437]: debug 2023-10-01T09:07:18.354+0000 7f9b28e31200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T09:07:18.806 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[139437]: debug 2023-10-01T09:07:18.458+0000 7f9b28e31200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T09:07:18.807 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:18 smithi190 conmon[139437]: debug 2023-10-01T09:07:18.565+0000 7f9b28e31200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T09:07:19.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:19 smithi190 conmon[139437]: debug 2023-10-01T09:07:19.033+0000 7f9b28e31200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T09:07:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:19 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:17.986356+0000 mgr.y (mgr.15957) 1 : cephadm [INF] [01/Oct/2023:09:07:17] ENGINE Bus STARTING 2023-10-01T09:07:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:19 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:18.088049+0000 mgr.y (mgr.15957) 2 : cephadm [INF] [01/Oct/2023:09:07:18] ENGINE Serving on http://172.21.15.138:8765 2023-10-01T09:07:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:19 smithi190 conmon[100518]: cluster 2023-10-01T09:07:18.479260+0000 mon.a (mon.0) 841 : cluster 2023-10-01T09:07:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:19 smithi190 conmon[100518]: [DBG] mgrmap e26: y(active, since 1.00857s) 2023-10-01T09:07:19.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:19 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:17.986356+0000 2023-10-01T09:07:19.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:19 smithi138 conmon[100289]: mgr.y (mgr.15957) 1 : cephadm [INF] [01/Oct/2023:09:07:17] ENGINE Bus STARTING 2023-10-01T09:07:19.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:19 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:18.088049+0000 mgr.y (mgr.15957) 2 : cephadm [INF] [01/Oct/2023:09:07:18] ENGINE Serving on http://172.21.15.138:8765 2023-10-01T09:07:19.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:19 smithi138 conmon[100289]: cluster 2023-10-01T09:07:18.479260+0000 mon.a (mon.0) 841 : cluster [DBG] mgrmap e26: y(active, since 1.00857s) 2023-10-01T09:07:19.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:19 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:17.986356+0000 mgr.y 2023-10-01T09:07:19.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:19 smithi138 conmon[109959]: (mgr.15957) 1 : cephadm [INF] [01/Oct/2023:09:07:17] ENGINE Bus STARTING 2023-10-01T09:07:19.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:19 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:18.088049+0000 mgr.y (mgr.15957) 2 : cephadm [INF] [01/Oct/2023:09:07:18] ENGINE Serving on http://172.21.15.138:8765 2023-10-01T09:07:19.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:19 smithi138 conmon[109959]: cluster 2023-10-01T09:07:18.479260+0000 mon.a (mon.0) 841 : cluster [DBG] mgrmap e26: y(active, since 1.00857s) 2023-10-01T09:07:20.046 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:19 smithi190 conmon[139437]: debug 2023-10-01T09:07:19.764+0000 7f9b28e31200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T09:07:20.046 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:19 smithi190 conmon[139437]: debug 2023-10-01T09:07:19.851+0000 7f9b28e31200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T09:07:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:20 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:18.210670+0000 mgr.y (mgr. 2023-10-01T09:07:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:20 smithi190 conmon[100518]: 15957) 3 : cephadm [INF] [01/Oct/2023:09:07:18] ENGINE Serving on https://172.21.15.138:7150 2023-10-01T09:07:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:20 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:18.210788+0000 mgr.y (mgr.15957) 4 : cephadm [INF] [01/Oct/2023:09:07:18] ENGINE Bus STARTED 2023-10-01T09:07:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:20 smithi190 conmon[100518]: cluster 2023-10-01T09:07:18.483603 2023-10-01T09:07:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:20 smithi190 conmon[100518]: +0000 mgr.y (mgr.15957) 5 : cluster [DBG] 2023-10-01T09:07:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:20 smithi190 conmon[100518]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:20.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:18.210670+0000 mgr.y (mgr 2023-10-01T09:07:20.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[100289]: .15957) 3 : cephadm [INF] [01/Oct/2023:09:07:18] ENGINE Serving on https://172.21.15.138:7150 2023-10-01T09:07:20.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:18.210788+0000 mgr.y (mgr.15957) 4 : cephadm [INF] [01/Oct/2023:09:07:18] ENGINE Bus STARTED 2023-10-01T09:07:20.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[100289]: cluster 2023-10-01T09:07:18.483603+0000 mgr.y (mgr.15957) 5 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:20.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:18 2023-10-01T09:07:20.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[109959]: .210670+0000 mgr.y (mgr.15957) 3 : cephadm [INF] [01/Oct/2023:09:07:18] ENGINE Serving on https://172.21.15.138:7150 2023-10-01T09:07:20.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[109959]: cephadm 2023-10-01T 2023-10-01T09:07:20.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[109959]: 09:07:18.210788+0000 mgr.y ( 2023-10-01T09:07:20.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[109959]: mgr.15957) 4 : cephadm [INF] 2023-10-01T09:07:20.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[109959]: [01/Oct/2023:09:07:18] ENGINE Bus STARTED 2023-10-01T09:07:20.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[109959]: cluster 2023-10-01T09:07:20.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[109959]: 2023-10- 2023-10-01T09:07:20.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[109959]: 01T09:07:18.483603 2023-10-01T09:07:20.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[109959]: +0000 mgr.y (mgr.15957) 2023-10-01T09:07:20.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:20 smithi138 conmon[109959]: 5 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:21.002 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:20 smithi190 conmon[139437]: debug 2023-10-01T09:07:20.718+0000 7f9b28e31200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:07:21.003 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:20 smithi190 conmon[139437]: debug 2023-10-01T09:07:20.818+0000 7f9b28e31200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T09:07:21.003 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:21 smithi190 conmon[139437]: debug 2023-10-01T09:07:21.002+0000 7f9b28e31200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:07:21.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:21 smithi190 conmon[100518]: cluster 2023-10-01T09:07:19.483433+0000 mgr.y (mgr.15957) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:21 smithi190 conmon[100518]: audit 2023-10-01T09:07:20.042803+0000 mon.a (mon.0) 842 : audit [DBG] from='client.? 172.21.15.138:0/3848604763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:21 smithi190 conmon[100518]: cluster 2023-10-01T09:07:20.488090+0000 mon.a (mon.0) 843 : cluster [DBG] mgrmap e27: y(active, since 3s) 2023-10-01T09:07:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:21 smithi190 conmon[100518]: audit 2023-10-01T09:07:20.668320+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:21 smithi190 conmon[100518]: audit 2023-10-01T09:07:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:21 smithi190 conmon[100518]: 2023-10-01T09:07:20.673779+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[100289]: cluster 2023-10-01T09:07:19.483433 2023-10-01T09:07:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[100289]: +0000 mgr.y (mgr.15957) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[100289]: audit 2023-10-01T09:07:20.042803+0000 mon.a (mon.0) 842 : audit [DBG] from='client.? 172.21.15.138:0/3848604763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[100289]: cluster 2023-10-01T09:07:20.488090+0000 mon.a (mon.0) 843 : cluster [DBG] mgrmap e27: y(active, since 3s) 2023-10-01T09:07:21.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[100289]: audit 2023-10-01T09:07:20.668320+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:21.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[100289]: audit 2023-10-01T09:07:20.673779+0000 mon.a 2023-10-01T09:07:21.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[100289]: (mon.0) 845 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:21.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[109959]: cluster 2023-10-01T09:07:19.483433+0000 2023-10-01T09:07:21.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[109959]: mgr.y (mgr.15957) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:21.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[109959]: audit 2023-10-01T09:07:21.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[109959]: 2023-10-01T09:07:20.042803+0000 mon.a (mon.0) 842 : 2023-10-01T09:07:21.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[109959]: audit [DBG] from='client.? 172.21.15.138:0/3848604763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:21.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[109959]: cluster 2023-10-01T09 2023-10-01T09:07:21.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[109959]: :07:20.488090+0000 mon.a (mon. 2023-10-01T09:07:21.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[109959]: 0) 843 : cluster [DBG] mgrmap e27: y(active, since 3s) 2023-10-01T09:07:21.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:07:21.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[109959]: 01T09:07:20.668320+0000 mon.a (mon 2023-10-01T09:07:21.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[109959]: .0) 844 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:21.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:07:21.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[109959]: 01T09:07:20.673779+0000 mon.a (mon 2023-10-01T09:07:21.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:21 smithi138 conmon[109959]: .0) 845 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:21.712 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:21 smithi190 conmon[139437]: debug 2023-10-01T09:07:21.307+0000 7f9b28e31200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T09:07:21.712 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:21 smithi190 conmon[139437]: debug 2023-10-01T09:07:21.553+0000 7f9b28e31200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T09:07:21.712 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:21 smithi190 conmon[139437]: debug 2023-10-01T09:07:21.712+0000 7f9b28e31200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T09:07:22.131 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:22 smithi138 conmon[109959]: audit 2023-10-01T09:07:21.130595+0000 mon.a (mon. 2023-10-01T09:07:22.131 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:22 smithi138 conmon[109959]: 0) 846 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.131 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:22 smithi138 conmon[109959]: audit 2023-10-01T09:07:21.138888+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.132 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:22 smithi138 conmon[109959]: audit 2023-10-01T09:07:21.314067+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.132 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:22 smithi138 conmon[109959]: audit 2023-10-01T09:07:21.321644+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.132 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:22 smithi138 conmon[109959]: audit 2023-10-01T09:07:21.830341+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.132 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:22 smithi138 conmon[109959]: audit 2023-10-01T09:07:21.837525+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.132 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:22 smithi138 conmon[100289]: audit 2023-10-01T09:07:21 2023-10-01T09:07:22.132 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:22 smithi138 conmon[100289]: .130595+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.132 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:22 smithi138 conmon[100289]: audit 2023-10-01T09:07:21.138888+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.132 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:22 smithi138 conmon[100289]: audit 2023-10-01T09:07:21.314067+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.133 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:22 smithi138 conmon[100289]: audit 2023-10-01T09:07:21.321644+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.133 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:22 smithi138 conmon[100289]: audit 2023-10-01T09:07:21.830341+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.133 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:22 smithi138 conmon[100289]: audit 2023-10-01T09:07:21.837525+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:21 smithi190 conmon[139437]: debug 2023-10-01T09:07:21.997+0000 7f9b28e31200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T09:07:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: audit 2023-10-01T09:07:21.130595+0000 mon.a (mon.0) 2023-10-01T09:07:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: 846 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: audit 2023-10-01T09:07:21.138888+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: audit 2023-10-01T09:07:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: 2023-10-01T09:07:21.314067+0000 2023-10-01T09:07:22.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: mon.a (mon.0) 848 : audit 2023-10-01T09:07:22.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:22.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: -10-01T09:07:21. 2023-10-01T09:07:22.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: 321644+0000 mon.a (mon.0) 849 : 2023-10-01T09:07:22.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:22.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: -10-01T09:07:21. 2023-10-01T09:07:22.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: 830341+0000 mon.a (mon.0) 850 : 2023-10-01T09:07:22.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:22.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:22.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: -10-01T09:07:21 2023-10-01T09:07:22.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: .837525+0000 mon.a (mon.0) 2023-10-01T09:07:22.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[100518]: 851 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:23.042 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[139437]: debug 2023-10-01T09:07:22.707+0000 7f9b28e31200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T09:07:23.043 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:22 smithi190 conmon[139437]: debug 2023-10-01T09:07:22.840+0000 7f9b28e31200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T09:07:23.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:23 smithi190 conmon[139437]: debug 2023-10-01T09:07:23.041+0000 7f9b28e31200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T09:07:23.307 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:23 smithi190 conmon[139437]: debug 2023-10-01T09:07:23.162+0000 7f9b28e31200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T09:07:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:23 smithi190 conmon[100518]: cluster 2023-10-01T09:07:21.483736+0000 mgr.y (mgr. 2023-10-01T09:07:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:23 smithi190 conmon[100518]: 15957) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:23 smithi190 conmon[100518]: audit 2023-10-01T09:07:22.436366+0000 mon.a (mon.0) 852 : audit [DBG] from='client.? 172.21.15.138:0/1751543553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:23.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:23 smithi138 conmon[109959]: cluster 2023-10-01T09:07:21.483736+0000 mgr.y 2023-10-01T09:07:23.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:23 smithi138 conmon[109959]: (mgr.15957) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:23.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:23 smithi138 conmon[109959]: audit 2023-10-01T09:07:22.436366+0000 mon.a (mon.0) 852 : 2023-10-01T09:07:23.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:23 smithi138 conmon[109959]: audit [DBG] from='client.? 172.21.15.138:0/1751543553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:23.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:23 smithi138 conmon[100289]: cluster 2023-10-01T09:07:21.483736+0000 mgr.y (mgr.15957) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:23.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:23 smithi138 conmon[100289]: audit 2023-10-01T09:07:22.436366+0000 mon.a (mon.0) 852 : audit [DBG] from='client.? 172.21.15.138:0/1751543553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:23.806 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:23 smithi190 conmon[139437]: debug 2023-10-01T09:07:23.392+0000 7f9b28e31200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T09:07:24.556 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:24 smithi190 conmon[139437]: debug 2023-10-01T09:07:24.267+0000 7f9b28e31200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T09:07:24.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:24 smithi190 conmon[139437]: [01/Oct/2023:09:07:24] ENGINE Bus STARTING 2023-10-01T09:07:24.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:24 smithi190 conmon[139437]: CherryPy Checker: 2023-10-01T09:07:24.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:24 smithi190 conmon[139437]: The Application mounted at '' has an empty config. 2023-10-01T09:07:24.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:24 smithi190 conmon[139437]: 2023-10-01T09:07:24.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:24 smithi190 conmon[139437]: [01/Oct/2023:09:07:24] ENGINE Serving on http://:::9283 2023-10-01T09:07:24.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:24 smithi190 conmon[139437]: [01/Oct/2023:09:07:24] ENGINE Bus STARTED 2023-10-01T09:07:25.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:25 smithi190 conmon[100518]: cluster 2023-10-01T09:07:23.484128+0000 mgr.y (mgr 2023-10-01T09:07:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:25 smithi190 conmon[100518]: .15957) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:25 smithi190 conmon[100518]: audit 2023-10-01T09:07:24.248560+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:25 smithi190 conmon[100518]: audit 2023-10-01T09:07:24.255416+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:25 smithi190 conmon[100518]: audit 2023-10-01T09:07:24.256902+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:25 smithi190 conmon[100518]: audit 2023-10-01T09:07:24.257612+0000 mon.c (mon.2) 180 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:07:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:25 smithi190 conmon[100518]: cluster 2023-10-01T09:07:24.282812+0000 mon.a (mon.0) 855 : cluster [DBG] Standby manager daemon x started 2023-10-01T09:07:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:25 smithi190 conmon[100518]: audit 2023-10-01T09:07:24.284584+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.190:0/1366311623' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:07:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:25 smithi190 conmon[100518]: audit 2023-10-01T09:07:24.285064+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.? 172.21.15.190:0/1366311623' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:07:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:25 smithi190 conmon[100518]: audit 2023-10-01T09:07:24.287018+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.? 172.21.15.190:0/1366311623' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T09:07:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:25 smithi190 conmon[100518]: audit 2023-10-01T09:07:24.288302+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.? 172.21.15.190:0/1366311623' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:07:25.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:25 smithi190 conmon[100518]: audit 2023-10-01T09:07:24.866648+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.138:0/2073396389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:25.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[100289]: cluster 2023-10-01T09:07:23.484128+0000 mgr.y (mgr.15957) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:25.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[100289]: audit 2023-10-01T09:07:24.248560+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:25.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[100289]: audit 2023-10-01T09:07:24.255416+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:25.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[100289]: audit 2023-10-01T09:07:24.256902+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:25.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[100289]: audit 2023-10-01T09:07:24.257612+0000 mon.c (mon.2) 180 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:07:25.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[100289]: cluster 2023-10-01T09:07:24.282812+0000 mon.a (mon.0) 855 : cluster [DBG] Standby manager daemon x started 2023-10-01T09:07:25.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[100289]: audit 2023-10-01T09:07:24.284584+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.190:0/1366311623' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:07:25.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[100289]: audit 2023-10-01T09:07:24.285064+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.? 172.21.15.190:0/1366311623' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:07:25.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[100289]: audit 2023-10-01T09:07:24.287018+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.? 172.21.15.190:0/1366311623' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T09:07:25.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[100289]: audit 2023-10-01T09:07:24.288302+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.? 172.21.15.190:0/1366311623' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:07:25.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[100289]: audit 2023-10-01T09:07:24.866648+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.138:0/2073396389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:25.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: cluster 2023-10-01T09:07 2023-10-01T09:07:25.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: :23.484128+0000 mgr.y (mgr.15957) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:25.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: audit 2023-10-01T09:07:24.248560+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:25.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: audit 2023-10-01T09:07:24.255416+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:25.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: audit 2023-10-01T09:07:25.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: 2023-10-01T09:07:24.256902+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:25.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:25.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: 10-01T09:07:24.257612+0000 2023-10-01T09:07:25.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: mon.c (mon.2) 180 : audit 2023-10-01T09:07:25.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:07:25.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: cluster 2023- 2023-10-01T09:07:25.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: 10-01T09:07:24.282812+0000 mon.a ( 2023-10-01T09:07:25.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: mon.0) 855 : cluster [DBG] Standby manager daemon x started 2023-10-01T09:07:25.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: audit 2023-10-01T09:07:25.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: 2023-10-01T09 2023-10-01T09:07:25.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: :07:24.284584+0000 mon.b (mon.1 2023-10-01T09:07:25.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: ) 101 : audit [DBG] from='mgr.? 172.21.15.190:0/1366311623' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:07:25.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: audit 2023 2023-10-01T09:07:25.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: -10-01T09:07:24.285064+0000 mon.b 2023-10-01T09:07:25.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: (mon.1) 102 2023-10-01T09:07:25.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: : audit [DBG] from='mgr.? 172.21.15.190:0/1366311623' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:07:25.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:25.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: 10-01T09:07:24.287018+0000 mon.b 2023-10-01T09:07:25.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: (mon.1) 103 : audit [DBG] 2023-10-01T09:07:25.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: from='mgr.? 172.21.15.190:0/1366311623' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T09:07:25.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:25.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: 10-01T09:07:24.288302+0000 mon.b ( 2023-10-01T09:07:25.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: mon.1) 104 : audit [DBG] from='mgr.? 172.21.15.190:0/1366311623' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:07:25.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: 2023-10-01T09:07:25.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:25.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: 10-01T09:07:24.866648+0000 2023-10-01T09:07:25.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.138:0/2073396389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:25.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:25 smithi138 conmon[109959]: 2023-10-01T09:07:26.512 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:24.258753+0000 mgr.y (mgr.15957) 9 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:07:26.512 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:24.258912+0000 mgr.y (mgr.15957) 10 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:07:26.512 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:24.523082+0000 mgr.y (mgr.15957) 11 : cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:07:26.512 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:24.523251+0000 mgr.y (mgr.15957) 12 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:07:26.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:24.787609+0000 mgr.y (mgr.15957) 13 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:07:26.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:24.787791+0000 mgr.y (mgr.15957) 14 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:07:26.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:25.052276+0000 mgr.y (mgr.15957) 15 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:07:26.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:25.052414+0000 mgr.y (mgr.15957) 16 : cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:07:26.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: cluster 2023-10-01T09:07:25.262863+0000 mon.a 2023-10-01T09:07:26.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: (mon.0) 857 : cluster [DBG] mgrmap e28: y(active, since 7s), standbys: x 2023-10-01T09:07:26.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: audit 2023-10-01T09:07:25.266141+0000 mon.c (mon.2) 181 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T09:07:26.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: audit 2023-10-01T09:07:25.321611+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:26.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: audit 2023-10-01T09:07:25.324681+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:26.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: audit 2023-10-01T09:07:25.329329+0000 mon.a ( 2023-10-01T09:07:26.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: mon.0) 860 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:26.514 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: audit 2023-10-01T09:07:25 2023-10-01T09:07:26.514 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: .333001+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:26.514 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:26.514 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: -10-01T09:07:25.338427+0000 mon.a (mon 2023-10-01T09:07:26.514 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:26 smithi190 conmon[100518]: .0) 862 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:24.258753+0000 mgr.y (mgr.15957) 2023-10-01T09:07:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: 9 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:07:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:24.258912+0000 mgr.y (mgr.15957) 10 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:07:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:24.523082+0000 mgr.y (mgr.15957) 11 : cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:07:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: cephadm 2023-10-01T09 2023-10-01T09:07:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: :07:24.523251+0000 mgr.y (mgr.15957) 12 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:07:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: 2023-10-01T09:07:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: cephadm 2023-10-01T 2023-10-01T09:07:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: 09:07:24.787609+0000 mgr.y (mgr.15957) 13 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:07:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:24. 2023-10-01T09:07:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: 787791+0000 mgr.y (mgr.15957) 14 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:07:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: cephadm 2023-10-01T09:07 2023-10-01T09:07:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: :25.052276+0000 mgr.y (mgr.15957) 15 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:07:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: cephadm 2023-10-01T09:07 2023-10-01T09:07:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: :25.052414+0000 mgr.y (mgr.15957) 16 : cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:07:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: 2023-10-01T09:07:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: cluster 2023-10- 2023-10-01T09:07:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: 01T09:07:25.262863+0000 mon.a (mon.0) 857 : cluster [DBG] mgrmap e28: y(active, since 7s), standbys: x 2023-10-01T09:07:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:07:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: 07:25.266141+0000 mon.c (mon.2) 181 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T09:07:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: 2023-10-01T09:07:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:07:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: 01T09:07:25.321611+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:07:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: 07:25.324681+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:07:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: 01T09:07:25.329329+0000 mon.a (mon.0) 860 : audit 2023-10-01T09:07:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:26.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:26.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: 10-01T09:07:25.333001+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:26.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: 2023-10-01T09:07:26.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:07:26.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[109959]: 01T09:07:25.338427+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:26.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:24.258753+0000 mgr.y (mgr.15957) 2023-10-01T09:07:26.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 9 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:07:26.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:24.258912+0000 mgr.y (mgr.15957) 10 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:07:26.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: cephadm 2023-10-01T09: 2023-10-01T09:07:26.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 07:24.523082+0000 mgr.y (mgr. 2023-10-01T09:07:26.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 15957) 11 : cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:07:26.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: cephadm 2023- 2023-10-01T09:07:26.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 10-01T09:07:24.523251+0000 mgr.y (mgr. 2023-10-01T09:07:26.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 15957) 12 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:07:26.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: cephadm 2023- 2023-10-01T09:07:26.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 10-01T09:07:24.787609 2023-10-01T09:07:26.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: +0000 mgr.y (mgr.15957) 13 : 2023-10-01T09:07:26.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: cephadm [INF] Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:07:26.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: cephadm 2023 2023-10-01T09:07:26.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: -10-01T09:07:24 2023-10-01T09:07:26.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: .787791+0000 mgr.y (mgr.15957) 2023-10-01T09:07:26.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 14 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:07:26.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:26.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 2023-10-01T09:07:25. 2023-10-01T09:07:26.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 052276+0000 mgr.y (mgr.15957) 15 : 2023-10-01T09:07:26.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:07:26.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: cephadm 2023- 2023-10-01T09:07:26.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 10-01T09:07:25.052414+0000 mgr.y (mgr 2023-10-01T09:07:26.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: .15957) 16 : cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:07:26.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: cluster 2023-10 2023-10-01T09:07:26.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: -01T09:07:25.262863+0000 mon.a ( 2023-10-01T09:07:26.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: mon.0) 857 : cluster [DBG] mgrmap e28: y(active, since 7s), standbys: x 2023-10-01T09:07:26.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:07:26.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 01T09:07:25.266141+0000 mon.c (mon 2023-10-01T09:07:26.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: .2) 181 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T09:07:26.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 2023-10-01T09:07:26.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T09:07:26.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: T09:07:25.321611+0000 mon.a (mon. 2023-10-01T09:07:26.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 0) 858 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:26.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: audit 2023-10-01T09:07:26.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 2023-10-01T09:07:25.324681+0000 2023-10-01T09:07:26.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: mon.a (mon.0) 859 : audit [INF] 2023-10-01T09:07:26.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:26.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T09:07:26.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: T09:07:25.329329+0000 mon.a (mon 2023-10-01T09:07:26.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: .0) 860 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:26.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 2023-10-01T09:07:26.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T09:07:26.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: T09:07:25.333001+0000 mon.a (mon. 2023-10-01T09:07:26.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 0) 861 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:26.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:07:26.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: 10-01T09:07:25.338427+0000 mon.a 2023-10-01T09:07:26.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: (mon.0) 862 : audit [INF] 2023-10-01T09:07:26.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:26 smithi138 conmon[100289]: from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:27.348 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:27 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:25.407086+0000 mgr.y (mgr.15957) 2023-10-01T09:07:27.348 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:27 smithi190 conmon[100518]: 17 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T09:07:27.348 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:27 smithi190 conmon[100518]: cluster 2023-10-01T09:07:25.484426+0000 mgr.y (mgr.15957) 18 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:27.348 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:27 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:25.633185+0000 mgr.y (mgr.15957) 19 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:07:27.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:27 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:25.407086+0000 mgr.y (mgr.15957) 17 2023-10-01T09:07:27.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:27 smithi138 conmon[100289]: : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T09:07:27.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:27 smithi138 conmon[100289]: cluster 2023-10-01T09:07:25.484426+0000 mgr.y (mgr.15957) 18 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:27.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:27 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:25.633185+0000 mgr.y (mgr.15957 2023-10-01T09:07:27.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:27 smithi138 conmon[100289]: ) 19 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:07:27.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:27 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:25.407086+0000 2023-10-01T09:07:27.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:27 smithi138 conmon[109959]: mgr.y (mgr.15957) 17 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T09:07:27.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:27 smithi138 conmon[109959]: cluster 2023-10-01T09:07:27.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:27 smithi138 conmon[109959]: 2023-10-01T09:07:25.484426+0000 mgr.y (mgr.15957 2023-10-01T09:07:27.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:27 smithi138 conmon[109959]: ) 18 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:27.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:27 smithi138 conmon[109959]: cephadm 2023-10-01T09:07 2023-10-01T09:07:27.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:27 smithi138 conmon[109959]: :25.633185+0000 mgr.y (mgr.15957) 19 : 2023-10-01T09:07:27.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:27 smithi138 conmon[109959]: cephadm [INF] Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:07:28.056 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:27 smithi190 conmon[139437]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:07:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T09:07:28.470 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 systemd[1]: Stopping Ceph prometheus.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:07:28.471 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 bash[150675]: Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-prometheus.a found: no such container 2023-10-01T09:07:28.471 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:28 smithi190 conmon[100518]: audit 2023-10-01T09:07:27.254358+0000 mon.c (mon.2 2023-10-01T09:07:28.471 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:28 smithi190 conmon[100518]: ) 182 : audit [DBG] from='client.? 172.21.15.138:0/377930150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:28.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:28 smithi138 conmon[109959]: audit 2023-10-01T09:07:27. 2023-10-01T09:07:28.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:28 smithi138 conmon[109959]: 254358+0000 mon.c (mon.2) 182 : audit [DBG] from='client.? 172.21.15.138:0/377930150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:28.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:28 smithi138 conmon[100289]: audit 2023-10-01T09:07:27.254358+0000 mon.c (mon.2 2023-10-01T09:07:28.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:28 smithi138 conmon[100289]: ) 182 : audit [DBG] from='client.? 172.21.15.138:0/377930150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:28.806 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 conmon[145209]: ts=2023-10-01T09:07:28.470Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-01T09:07:28.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 conmon[145209]: ts=2023-10-01T09:07:28.470Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-01T09:07:28.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 conmon[145209]: ts=2023-10-01T09:07:28.470Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-01T09:07:28.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 conmon[145209]: ts=2023-10-01T09:07:28.470Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-01T09:07:28.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 conmon[145209]: ts=2023-10-01T09:07:28.470Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-01T09:07:28.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 conmon[145209]: ts=2023-10-01T09:07:28.470Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-01T09:07:28.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 conmon[145209]: ts=2023-10-01T09:07:28.470Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-01T09:07:28.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 conmon[145209]: ts=2023-10-01T09:07:28.470Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-01T09:07:28.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 conmon[145209]: ts=2023-10-01T09:07:28.470Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-01T09:07:28.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 conmon[145209]: ts=2023-10-01T09:07:28.471Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-01T09:07:28.807 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 conmon[145209]: ts=2023-10-01T09:07:28.471Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-01T09:07:28.808 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 conmon[145209]: ts=2023-10-01T09:07:28.471Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-01T09:07:29.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:29 smithi190 conmon[100518]: cluster 2023-10-01T09:07:27.484736+0000 mgr.y (mgr.15957) 20 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:29.307 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:28 smithi190 bash[150791]: f4d0cb1e646645ed4a740fc156a74f00391a732340227e176e328e4542c23af8 2023-10-01T09:07:29.307 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-01T09:07:29.307 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 systemd[1]: Stopped Ceph prometheus.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:07:29.308 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 systemd[1]: Starting Ceph prometheus.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:07:29.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:29 smithi138 conmon[100289]: cluster 2023-10-01T09:07:27.484736+0000 mgr.y (mgr.15957) 20 : 2023-10-01T09:07:29.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:29 smithi138 conmon[100289]: cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:29 smithi138 conmon[109959]: cluster 2023-10-01T09:07:27.484736 2023-10-01T09:07:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:29 smithi138 conmon[109959]: +0000 mgr.y (mgr.15957) 20 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:30.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 conmon[151158]: ts=2023-10-01T09:07:29.778Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-01T09:07:30.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 conmon[151158]: ts=2023-10-01T09:07:29.778Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-01T09:07:30.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 conmon[151158]: ts=2023-10-01T09:07:29.778Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi190 (none))" 2023-10-01T09:07:30.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 conmon[151158]: ts=2023-10-01T09:07:29.778Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-01T09:07:30.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 conmon[151158]: ts=2023-10-01T09:07:29.778Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-01T09:07:30.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 conmon[151158]: ts=2023-10-01T09:07:29.780Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-01T09:07:30.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 conmon[151158]: ts=2023-10-01T09:07:29.780Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-01T09:07:30.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 conmon[151158]: ts=2023-10-01T09:07:29.782Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-01T09:07:30.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 conmon[151158]: ts=2023-10-01T09:07:29.783Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-01T09:07:30.058 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 conmon[151158]: ts=2023-10-01T09:07:29.783Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.522µs 2023-10-01T09:07:30.058 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 conmon[151158]: ts=2023-10-01T09:07:29.783Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-01T09:07:30.058 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 bash[151117]: bbdb076a24d4a4a540ff36a46db78e30054c0c30e98483520959a1438fbeb582 2023-10-01T09:07:30.058 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 systemd[1]: Started Ceph prometheus.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:07:30.058 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:29 smithi190 conmon[151158]: ts=2023-10-01T09:07:29.945Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-10-01T09:07:30.149 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:29 smithi138 conmon[178630]: [01/Oct/2023:09:07:29] ENGINE Bus STOPPING 2023-10-01T09:07:30.149 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:29 smithi138 conmon[178630]: [01/Oct/2023:09:07:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T09:07:30.149 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:29 smithi138 conmon[178630]: [01/Oct/2023:09:07:29] ENGINE Bus STOPPED 2023-10-01T09:07:30.150 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:29 smithi138 conmon[178630]: [01/Oct/2023:09:07:29] ENGINE Bus STARTING 2023-10-01T09:07:30.150 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:29 smithi138 conmon[178630]: [01/Oct/2023:09:07:29] ENGINE Serving on http://:::9283 2023-10-01T09:07:30.150 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:29 smithi138 conmon[178630]: [01/Oct/2023:09:07:29] ENGINE Bus STARTED 2023-10-01T09:07:30.152 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: audit 2023-10-01T09:07:29.638342 2023-10-01T09:07:30.152 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: +0000 mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.138:0/1637691920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:30.152 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: audit 2023-10-01T09:07:29. 2023-10-01T09:07:30.152 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: 818787+0000 mon.a (mon.0) 863 : audit 2023-10-01T09:07:30.152 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:30.152 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:07:30.152 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: -01T09:07:29.824428+0000 2023-10-01T09:07:30.152 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: mon.a (mon.0) 864 : 2023-10-01T09:07:30.153 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:30.153 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: audit 2023-10-01T09:07:30.153 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: 2023- 2023-10-01T09:07:30.153 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: 10-01T09:07:29.861373+0000 mon.c (mon.2) 2023-10-01T09:07:30.153 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: 184 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:07:30.153 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: audit 2023-10-01T09:07:30.153 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: 2023-10-01T09: 2023-10-01T09:07:30.153 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: 07:29.862559+0000 mon.c ( 2023-10-01T09:07:30.153 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: mon.2) 185 : audit [INF] 2023-10-01T09:07:30.153 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T09:07:30.154 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: audit 2023-10-01T09:07:30.154 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: 2023-10- 2023-10-01T09:07:30.154 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: 01T09:07:29.868405 2023-10-01T09:07:30.154 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: +0000 mon.a (mon.0) 865 2023-10-01T09:07:30.154 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:30.154 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:30.154 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: 10-01T09:07:29. 2023-10-01T09:07:30.154 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: 880527+0000 mon.c (mon.2) 186 2023-10-01T09:07:30.154 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[109959]: : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:30.410 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: audit 2023-10-01T09:07:29.638342+0000 2023-10-01T09:07:30.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.138:0/1637691920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:30.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: audit 2023-10-01T09:07:29.818787+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:30.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: audit 2023-10-01T09:07:29.824428+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:30.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:07:30.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: 10-01T09:07:29.861373+0000 mon.c (mon.2) 184 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:07:30.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: audit 2023-10-01T09: 2023-10-01T09:07:30.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: 07:29.862559+0000 mon.c (mon.2) 2023-10-01T09:07:30.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: 185 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T09:07:30.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:07:30.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: -01T09:07:29.868405 2023-10-01T09:07:30.412 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: +0000 mon.a (mon.0) 865 : audit 2023-10-01T09:07:30.412 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:30.412 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:07:30.412 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: 01T09:07:29.880527+0000 2023-10-01T09:07:30.412 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[100518]: mon.c (mon.2) 186 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:30.412 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[151158]: ts=2023-10-01T09:07:30.153Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-10-01T09:07:30.412 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[151158]: ts=2023-10-01T09:07:30.191Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-10-01T09:07:30.412 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[151158]: ts=2023-10-01T09:07:30.191Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-10-01T09:07:30.412 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[151158]: ts=2023-10-01T09:07:30.191Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=35.085µs wal_replay_duration=407.408797ms total_replay_duration=407.457257ms 2023-10-01T09:07:30.412 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[151158]: ts=2023-10-01T09:07:30.194Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-01T09:07:30.413 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[151158]: ts=2023-10-01T09:07:30.194Z caller=main.go:947 level=info msg="TSDB started" 2023-10-01T09:07:30.413 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[151158]: ts=2023-10-01T09:07:30.194Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-01T09:07:30.413 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[151158]: ts=2023-10-01T09:07:30.215Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.335054ms db_storage=868ns remote_storage=1.594µs web_handler=486ns query_engine=1.026µs scrape=714.348µs scrape_sd=89.078µs notify=15.698µs notify_sd=17.016µs rules=19.152228ms 2023-10-01T09:07:30.413 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:30 smithi190 conmon[151158]: ts=2023-10-01T09:07:30.215Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-01T09:07:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: audit 2023-10-01T09:07:29.638342+0000 2023-10-01T09:07:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: mon.c (mon.2) 183 : audit [DBG] from='client.? 172.21.15.138:0/1637691920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: audit 2023-10-01T09:07:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: 2023-10-01T09:07:29.818787+0000 mon.a ( 2023-10-01T09:07:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: mon.0) 863 : audit [INF] 2023-10-01T09:07:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:07:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: 07:29.824428+0000 mon.a (mon.0) 864 2023-10-01T09:07:30.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:30.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:07:30.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: 07:29.861373+0000 mon.c (mon.2 2023-10-01T09:07:30.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: ) 184 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:07:30.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: audit 2023- 2023-10-01T09:07:30.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: 10-01T09:07:29.862559+0000 2023-10-01T09:07:30.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: mon.c (mon.2) 185 : audit 2023-10-01T09:07:30.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T09:07:30.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: audit 2023-10-01T09:07:30.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: 2023-10-01T09:07: 2023-10-01T09:07:30.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: 29.868405+0000 mon.a (mon.0 2023-10-01T09:07:30.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: ) 865 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:30.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: audit 2023-10-01T09:07:30.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: 2023-10-01T09:07: 2023-10-01T09:07:30.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: 29.880527+0000 mon.c (mon.2 2023-10-01T09:07:30.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:30 smithi138 conmon[100289]: ) 186 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:31.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:31 smithi190 conmon[100518]: cluster 2023-10-01T09:07:29.485378+0000 mgr.y (mgr.15957) 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:31.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:31 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:07:31.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:31 smithi190 conmon[100518]: :07:29.861807+0000 mgr.y (mgr.15957) 22 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:07:31.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:31 smithi190 conmon[100518]: audit 2023-10-01T09:07:29.862864+0000 mgr.y (mgr.15957) 23 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T09:07:31.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[100289]: cluster 2023-10-01T09:07:29.485378+0000 mgr.y (mgr.15957) 2023-10-01T09:07:31.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[100289]: 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:31.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[100289]: audit 2023-10-01T09:07:29.861807+0000 mgr.y (mgr.15957) 22 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:07:31.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:07:31.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[100289]: 01T09:07:29.862864+0000 mgr.y 2023-10-01T09:07:31.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[100289]: (mgr.15957) 23 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T09:07:31.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[100289]: 2023-10-01T09:07:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[109959]: cluster 2023-10-01T09:07:29. 2023-10-01T09:07:31.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[109959]: 485378+0000 mgr.y (mgr.15957) 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:31.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[109959]: audit 2023-10-01 2023-10-01T09:07:31.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[109959]: T09:07:29.861807+0000 mgr.y 2023-10-01T09:07:31.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[109959]: (mgr.15957) 22 : audit [DBG] 2023-10-01T09:07:31.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[109959]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T09:07:31.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:07:31.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[109959]: 09:07:29.862864+0000 mgr.y (mgr 2023-10-01T09:07:31.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[109959]: .15957) 23 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi190.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T09:07:31.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:31 smithi138 conmon[109959]: 2023-10-01T09:07:32.462 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:32 smithi190 conmon[100518]: audit 2023-10-01T09:07:31.251307+0000 mon.c (mon.2) 187 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:07:32.462 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:32 smithi190 conmon[100518]: audit 2023-10-01T09:07:31.251878+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:07:32.462 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:32 smithi190 conmon[100518]: audit 2023-10-01T09:07:31.253090+0000 mon.c (mon.2) 188 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:07:32.462 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:32 smithi190 conmon[100518]: audit 2023-10-01T09:07:31.254063+0000 mon.c (mon.2) 189 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:32.462 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:32 smithi190 conmon[100518]: audit 2023-10-01T09:07:32.022905+0000 mon.a (mon.0) 867 : audit [DBG] from='client.? 172.21.15.138:0/2994351482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[100289]: audit 2023-10-01T09:07:31.251307+0000 mon.c (mon.2) 2023-10-01T09:07:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[100289]: 187 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:07:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[100289]: audit 2023-10-01T09:07:31.251878+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:07:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[100289]: audit 2023-10-01T09:07:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[100289]: 2023-10-01T09:07:31.253090+0000 mon.c (mon.2) 188 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:07:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:07:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[100289]: 01T09:07:31.254063+0000 mon.c (mon.2) 189 : audit 2023-10-01T09:07:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[100289]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[100289]: audit 2023-10 2023-10-01T09:07:32.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[100289]: -01T09:07:32.022905+0000 mon.a (mon.0) 867 : audit [DBG] from='client.? 172.21.15.138:0/2994351482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[109959]: audit 2023-10-01T09:07:31.251307+0000 mon.c (mon 2023-10-01T09:07:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[109959]: .2) 187 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:07:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[109959]: audit 2023-10-01T09:07:31.251878+0000 mon.a (mon 2023-10-01T09:07:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[109959]: .0) 866 : audit [INF] from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:07:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[109959]: 10-01T09:07:31.253090+0000 mon.c (mon.2) 188 2023-10-01T09:07:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[109959]: : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:07:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[109959]: audit 2023-10-01T09:07: 2023-10-01T09:07:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[109959]: 31.254063+0000 mon.c (mon.2) 189 : audit 2023-10-01T09:07:32.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[109959]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:32.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[109959]: audit 2023-10-01T09:07:32 2023-10-01T09:07:32.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[109959]: .022905+0000 mon.a (mon.0) 867 : audit 2023-10-01T09:07:32.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:32 smithi138 conmon[109959]: [DBG] from='client.? 172.21.15.138:0/2994351482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:32.806 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:32 smithi190 systemd[1]: Stopping Ceph mgr.x for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:07:33.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:33 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:31.250757+0000 mgr.y (mgr.15957) 24 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-01T09:07:33.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:33 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:31.255072+0000 mgr.y (mgr.15957) 25 : cephadm [INF] Deploying daemon mgr.x on smithi190 2023-10-01T09:07:33.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:33 smithi190 conmon[100518]: cluster 2023-10-01T09:07:31.485970+0000 mgr.y (mgr.15957) 26 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:33.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:33 smithi190 conmon[100518]: audit 2023-10-01T09:07:32.692159+0000 mon.c (mon.2) 190 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:07:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:31.250757+0000 mgr.y (mgr.15957) 24 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-01T09:07:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[100289]: 2023-10-01T09:07:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:31.255072+0000 mgr.y (mgr.15957) 25 : cephadm [INF] Deploying daemon mgr.x on smithi190 2023-10-01T09:07:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[100289]: cluster 2023-10-01T09:07:31.485970+0000 mgr.y (mgr.15957) 26 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:07:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[100289]: 09:07:32.692159+0000 mon.c (mon.2) 190 : 2023-10-01T09:07:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[100289]: audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:07:33.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:31.250757+0000 mgr.y (mgr 2023-10-01T09:07:33.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[109959]: .15957) 24 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-01T09:07:33.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:31. 2023-10-01T09:07:33.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[109959]: 255072+0000 mgr.y (mgr.15957) 25 : cephadm [INF] Deploying daemon mgr.x on smithi190 2023-10-01T09:07:33.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[109959]: cluster 2023-10-01T 2023-10-01T09:07:33.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[109959]: 09:07:31.485970+0000 mgr.y (mgr.15957) 26 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:33.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[109959]: audit 2023-10-01T09:07:33.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:33 smithi138 conmon[109959]: 2023-10-01T09:07:32.692159+0000 mon.c (mon.2) 190 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:07:33.782 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:33 smithi190 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-01T09:07:33.782 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:33 smithi190 bash[151794]: d82938b5a62fa91520386313fbda91816cda57b4d392439f2e38f269db574e87 2023-10-01T09:07:33.782 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:33 smithi190 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-01T09:07:33.782 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:33 smithi190 systemd[1]: Stopped Ceph mgr.x for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:07:34.056 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:33 smithi190 systemd[1]: Starting Ceph mgr.x for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:07:34.705 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:34 smithi190 ceph-mgr[152216]: -- 172.21.15.190:0/2861979819 <== mon.1 v2:172.21.15.190:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563e188a51e0 con 0x563e1887d000 2023-10-01T09:07:34.705 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:34 smithi190 bash[152151]: 674e1140402f91286265ec991a3e3488cc36e2d04959e7ac0543b6f0914ad898 2023-10-01T09:07:34.705 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:34 smithi190 systemd[1]: Started Ceph mgr.x for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:07:34.705 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:34 smithi190 conmon[152192]: 2023-10-01T09:07:34.527+0000 7fe22849c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T09:07:34.705 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:34 smithi190 conmon[152192]: 2023-10-01T09:07:34.611+0000 7fe22849c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T09:07:34.705 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:34 smithi190 conmon[152192]: 2023-10-01T09:07:34.703+0000 7fe22849c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T09:07:35.011 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:34 smithi190 conmon[152192]: 2023-10-01T09:07:34.814+0000 7fe22849c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T09:07:35.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[152192]: 2023-10-01T09:07:35.010+0000 7fe22849c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T09:07:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: cluster 2023-10-01T09:07:33.486551+0000 mgr.y (mgr.15957 2023-10-01T09:07:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: ) 27 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: audit 2023-10-01T09:07:34.408129+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.138:0/3563824673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: audit 2023-10-01T09:07:34.482473+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:07:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: 01T09:07:34.487144+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:35.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: audit 2023-10-01T09:07:35.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: 2023-10- 2023-10-01T09:07:35.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: 01T09:07:34.526579+0000 mon.c 2023-10-01T09:07:35.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: (mon.2) 191 : audit [DBG] 2023-10-01T09:07:35.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:35.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: audit 2023-10-01T09:07:35.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: 2023-10-01T09:07:34 2023-10-01T09:07:35.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: .848634+0000 mon.a (mon.0) 871 2023-10-01T09:07:35.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:35.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:07:35.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: 10-01T09:07:34.855699+0000 mon.a 2023-10-01T09:07:35.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: (mon.0) 872 : audit [INF] 2023-10-01T09:07:35.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[100518]: from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:35.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[100289]: cluster 2023-10-01T09:07:33.486551+0000 mgr.y (mgr.15957) 27 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:35.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[100289]: audit 2023-10-01T09:07:34.408129+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.138:0/3563824673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:35.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[100289]: audit 2023-10-01T09:07:34.482473+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:35.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[100289]: audit 2023-10-01T09:07:34.487144+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:35.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[100289]: audit 2023-10-01T09:07:34.526579+0000 mon.c (mon.2) 191 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:35.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[100289]: audit 2023-10-01T09:07:34.848634+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:35.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[100289]: audit 2023-10-01T09:07:34.855699+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:35.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[109959]: cluster 2023-10-01T09:07:33.486551+0000 mgr.y (mgr.15957) 27 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:35.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[109959]: audit 2023-10-01T09:07:34.408129+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.138:0/3563824673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:35.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[109959]: audit 2023-10-01T09:07:34.482473+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:35.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[109959]: audit 2023-10-01T09:07:34.487144+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:35.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[109959]: audit 2023-10-01T09:07:34.526579+0000 mon.c (mon.2) 191 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:35.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[109959]: audit 2023-10-01T09:07:34.848634+0000 mon.a (mon.0 2023-10-01T09:07:35.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[109959]: ) 871 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:35.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:35 smithi138 conmon[109959]: audit 2023-10-01T09:07:34.855699+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:35.757 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[152192]: 2023-10-01T09:07:35.407+0000 7fe22849c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T09:07:35.757 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[152192]: 2023-10-01T09:07:35.558+0000 7fe22849c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T09:07:35.757 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[152192]: 2023-10-01T09:07:35.653+0000 7fe22849c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T09:07:35.757 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:35 smithi190 conmon[152192]: 2023-10-01T09:07:35.755+0000 7fe22849c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T09:07:36.556 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:36 smithi190 conmon[152192]: 2023-10-01T09:07:36.243+0000 7fe22849c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T09:07:37.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:36 smithi190 conmon[152192]: 2023-10-01T09:07:36.995+0000 7fe22849c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T09:07:37.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:37 smithi190 conmon[152192]: 2023-10-01T09:07:37.084+0000 7fe22849c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T09:07:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:37 smithi190 conmon[100518]: cluster 2023-10-01T09:07:35.486937+0000 mgr.y (mgr.15957 2023-10-01T09:07:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:37 smithi190 conmon[100518]: ) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:37 smithi190 conmon[100518]: audit 2023-10-01T09:07:36.793743+0000 mon.c (mon.2) 192 : audit [DBG] from='client.? 172.21.15.138:0/3867201781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:37 smithi138 conmon[109959]: cluster 2023-10-01T09:07:35. 2023-10-01T09:07:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:37 smithi138 conmon[109959]: 486937+0000 mgr.y (mgr.15957) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:37 smithi138 conmon[109959]: audit 2023-10-01T09:07:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:37 smithi138 conmon[109959]: 2023-10- 2023-10-01T09:07:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:37 smithi138 conmon[109959]: 01T09:07:36.793743 2023-10-01T09:07:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:37 smithi138 conmon[109959]: +0000 mon.c (mon.2) 192 2023-10-01T09:07:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:37 smithi138 conmon[109959]: : audit [DBG] from='client.? 172.21.15.138:0/3867201781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:37.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:37 smithi138 conmon[100289]: cluster 2023-10-01T09:07:35.486937+0000 2023-10-01T09:07:37.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:37 smithi138 conmon[100289]: mgr.y (mgr.15957) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:37.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:37 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:07:37.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:37 smithi138 conmon[100289]: 01T09:07:36.793743+0000 mon.c (mon.2) 192 : 2023-10-01T09:07:37.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:37 smithi138 conmon[100289]: audit [DBG] from='client.? 172.21.15.138:0/3867201781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:38.235 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:37 smithi190 conmon[152192]: 2023-10-01T09:07:37.943+0000 7fe22849c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:07:38.235 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[152192]: 2023-10-01T09:07:38.047+0000 7fe22849c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T09:07:38.235 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[152192]: 2023-10-01T09:07:38.233+0000 7fe22849c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:07:38.543 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[152192]: 2023-10-01T09:07:38.541+0000 7fe22849c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T09:07:39.056 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[152192]: 2023-10-01T09:07:38.796+0000 7fe22849c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T09:07:39.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[152192]: 2023-10-01T09:07:38.968+0000 7fe22849c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T09:07:39.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: cluster 2023-10-01T09:07:37. 2023-10-01T09:07:39.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: 487443+0000 mgr.y (mgr.15957) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:39.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: audit 2023-10-01T09:07:37.805110 2023-10-01T09:07:39.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: +0000 mon.a (mon.0) 873 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:39.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:07:39.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: 01T09:07:37.813778+0000 mon.a (mon. 2023-10-01T09:07:39.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: 0) 874 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:39.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: 2023-10-01T09:07:39.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:07:39.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: 10-01T09:07:38.501378+0000 2023-10-01T09:07:39.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: mon.a (mon.0) 875 : audit [INF] 2023-10-01T09:07:39.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:39.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: audit 2023-10-01T09:07:39.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: 2023-10-01 2023-10-01T09:07:39.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: T09:07:38.507340+0000 2023-10-01T09:07:39.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: mon.a (mon.0) 876 2023-10-01T09:07:39.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:38 smithi190 conmon[100518]: : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[100289]: cluster 2023-10-01T09:07:37.487443+0000 mgr.y (mgr.15957) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[100289]: audit 2023-10-01T09:07:37.805110+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[100289]: audit 2023-10-01T09:07:37.813778+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:07:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[100289]: 01T09:07:38.501378+0000 mon.a (mon.0) 875 : audit [INF] 2023-10-01T09:07:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[100289]: from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:07:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[100289]: :07:38.507340+0000 mon.a (mon.0) 876 : audit 2023-10-01T09:07:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[100289]: [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:39.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[109959]: cluster 2023-10-01T09:07:37.487443+0000 mgr.y (mgr.15957) 29 : cluster 2023-10-01T09:07:39.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[109959]: [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:39.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[109959]: audit 2023-10-01T09:07:37.805110+0000 mon.a (mon.0) 873 : 2023-10-01T09:07:39.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[109959]: audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:39.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:07:39.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[109959]: 01T09:07:37.813778+0000 mon.a (mon.0) 874 : audit 2023-10-01T09:07:39.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[109959]: [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:39.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[109959]: audit 2023-10-01T09:07 2023-10-01T09:07:39.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[109959]: :38.501378+0000 mon.a (mon.0) 875 : 2023-10-01T09:07:39.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[109959]: audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:39.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:07:39.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[109959]: 07:38.507340+0000 mon.a (mon.0) 876 2023-10-01T09:07:39.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:38 smithi138 conmon[109959]: : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:39.443 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:39 smithi190 conmon[152192]: 2023-10-01T09:07:39.258+0000 7fe22849c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T09:07:40.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:39 smithi138 conmon[100289]: audit 2023-10-01T09:07:39.171810+0000 mon.a (mon.0 2023-10-01T09:07:40.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:39 smithi138 conmon[100289]: ) 877 : audit [DBG] from='client.? 172.21.15.138:0/2097349585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:40.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:39 smithi138 conmon[109959]: audit 2023-10-01T09:07:39.171810+0000 mon.a ( 2023-10-01T09:07:40.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:39 smithi138 conmon[109959]: mon.0) 877 : audit [DBG] from='client.? 172.21.15.138:0/2097349585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:40.153 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:39 smithi190 conmon[152192]: 2023-10-01T09:07:39.992+0000 7fe22849c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T09:07:40.154 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:40 smithi190 conmon[152192]: 2023-10-01T09:07:40.152+0000 7fe22849c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T09:07:40.154 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:39 smithi190 conmon[100518]: audit 2023-10-01T09:07:39.171810+0000 mon.a (mon.0) 877 2023-10-01T09:07:40.154 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:39 smithi190 conmon[100518]: : audit [DBG] from='client.? 172.21.15.138:0/2097349585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:40.495 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:40 smithi190 conmon[152192]: 2023-10-01T09:07:40.371+0000 7fe22849c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T09:07:40.496 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:40 smithi190 conmon[152192]: 2023-10-01T09:07:40.494+0000 7fe22849c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T09:07:40.806 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:40 smithi190 conmon[152192]: 2023-10-01T09:07:40.733+0000 7fe22849c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T09:07:41.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:40 smithi138 conmon[109959]: cluster 2023-10-01T09:07:39.487928+0000 mgr.y (mgr.15957 2023-10-01T09:07:41.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:40 smithi138 conmon[109959]: ) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:41.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:40 smithi138 conmon[100289]: cluster 2023-10-01T09:07:39.487928+0000 mgr.y (mgr.15957) 30 : cluster [DBG] 2023-10-01T09:07:41.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:40 smithi138 conmon[100289]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:41.110 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:40 smithi190 conmon[100518]: cluster 2023-10-01T09:07:39.487928+0000 mgr.y (mgr.15957) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:41.110 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:40 smithi190 conmon[100518]: 2023-10-01T09:07:41.807 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[152192]: 2023-10-01T09:07:41.650+0000 7fe22849c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T09:07:41.807 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[152192]: [01/Oct/2023:09:07:41] ENGINE Bus STARTING 2023-10-01T09:07:41.807 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[152192]: CherryPy Checker: 2023-10-01T09:07:41.807 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[152192]: The Application mounted at '' has an empty config. 2023-10-01T09:07:41.807 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[152192]: 2023-10-01T09:07:41.807 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[152192]: [01/Oct/2023:09:07:41] ENGINE Serving on http://:::9283 2023-10-01T09:07:41.808 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[152192]: [01/Oct/2023:09:07:41] ENGINE Bus STARTED 2023-10-01T09:07:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023-10-01T09: 2023-10-01T09:07:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 07:41.556021+0000 mon.c (mon.2) 193 : audit [DBG] from='client.? 172.21.15.138:0/1029705174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023 2023-10-01T09:07:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: -10-01T09:07:41.582723+0000 mon.a ( 2023-10-01T09:07:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: mon.0) 878 : audit [INF] 2023-10-01T09:07:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 10-01T09:07:41.598294+0000 2023-10-01T09:07:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: mon.a (mon.0) 879 : audit 2023-10-01T09:07:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023-10-01T09:07:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 2023-10-01T09:07:41.599438 2023-10-01T09:07:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: +0000 mon.c (mon.2) 194 2023-10-01T09:07:42.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:42.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:07:42.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 01T09:07:41.600162+0000 mon.c 2023-10-01T09:07:42.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: (mon.2) 195 : audit 2023-10-01T09:07:42.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:07:42.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023-10-01T09:07:42.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 2023-10-01T09:07:41.605053 2023-10-01T09:07:42.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: +0000 mon.a (mon.0) 2023-10-01T09:07:42.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 880 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:42.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: cluster 2023- 2023-10-01T09:07:42.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 10-01T09:07:41.665090+0000 2023-10-01T09:07:42.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: mon.a (mon.0) 881 : cluster 2023-10-01T09:07:42.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: [DBG] Standby manager daemon x restarted 2023-10-01T09:07:42.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: cluster 2023- 2023-10-01T09:07:42.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 10-01T09:07:41.665193+0000 2023-10-01T09:07:42.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: mon.a (mon.0) 882 : cluster 2023-10-01T09:07:42.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: [DBG] Standby manager daemon x started 2023-10-01T09:07:42.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:42.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 10-01T09:07:41.667331+0000 2023-10-01T09:07:42.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: mon.b (mon.1) 105 : 2023-10-01T09:07:42.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit [DBG] from='mgr.? 172.21.15.190:0/2017602409' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:07:42.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:42.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 10-01T09:07:41.667769+0000 mon.b (mon 2023-10-01T09:07:42.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: .1) 106 : audit [DBG] from='mgr.? 172.21.15.190:0/2017602409' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:07:42.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 2023-10-01T09:07:42.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:07:42.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 01T09:07:41.670614+0000 mon.b 2023-10-01T09:07:42.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: (mon.1) 107 : audit 2023-10-01T09:07:42.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: [DBG] from='mgr.? 172.21.15.190:0/2017602409' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T09:07:42.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023 2023-10-01T09:07:42.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: -10-01T09:07:41.671019 2023-10-01T09:07:42.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: +0000 mon.b (mon.1) 108 : 2023-10-01T09:07:42.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit [DBG] from='mgr.? 172.21.15.190:0/2017602409' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:07:42.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023 2023-10-01T09:07:42.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: -10-01T09:07:41.677584+0000 2023-10-01T09:07:42.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: mon.c (mon.2) 196 : audit 2023-10-01T09:07:42.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09:07:41.556021 2023-10-01T09:07:42.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: +0000 mon.c (mon.2) 193 : audit [DBG] from='client.? 172.21.15.138:0/1029705174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:42.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:07:42.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: :07:41.582723+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:42.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09:07:42.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 2023-10-01T09:07:41.598294+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:42.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09:07:41.599438+0000 2023-10-01T09:07:42.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: mon.c (mon.2) 194 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:42.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01 2023-10-01T09:07:42.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: T09:07:41.600162+0000 mon.c (mon. 2023-10-01T09:07:42.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 2) 195 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:07:42.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:07:42.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 09:07:41.605053+0000 mon.a (mon.0 2023-10-01T09:07:42.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: ) 880 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:42.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: cluster 2023-10-01T09 2023-10-01T09:07:42.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: :07:41.665090+0000 mon.a (mon.0) 881 2023-10-01T09:07:42.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: : cluster [DBG] Standby manager daemon x restarted 2023-10-01T09:07:42.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: cluster 2023-10-01T09:07:41 2023-10-01T09:07:42.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: .665193+0000 mon.a (mon.0) 882 : cluster [DBG] Standby manager daemon x started 2023-10-01T09:07:42.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:07:42.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 01T09:07:41.667331+0000 mon.b ( 2023-10-01T09:07:42.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: mon.1) 105 : audit [DBG] from='mgr.? 172.21.15.190:0/2017602409' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:07:42.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10- 2023-10-01T09:07:42.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 01T09:07:41.667769+0000 mon.b (mon 2023-10-01T09:07:42.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: .1) 106 : audit [DBG] from='mgr.? 172.21.15.190:0/2017602409' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:07:42.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T 2023-10-01T09:07:42.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 09:07:41.670614+0000 mon.b (mon.1) 2023-10-01T09:07:42.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 107 : audit [DBG] from='mgr.? 172.21.15.190:0/2017602409' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T09:07:42.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:07:42.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 07:41.671019+0000 mon.b (mon.1) 2023-10-01T09:07:42.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 108 : audit [DBG] from='mgr.? 172.21.15.190:0/2017602409' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:07:42.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:07:42.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 07:41.677584+0000 mon.c (mon.2) 196 2023-10-01T09:07:42.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:42.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09:07: 2023-10-01T09:07:42.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 41.680099+0000 mon.c (mon.2) 197 : 2023-10-01T09:07:42.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:07:42.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09:07: 2023-10-01T09:07:42.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 41.684632+0000 mon.a (mon.0) 883 : 2023-10-01T09:07:42.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:42.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:42.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023-10-01T09:07:42.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 2023-10-01T 2023-10-01T09:07:42.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 09:07:41.680099+0000 2023-10-01T09:07:42.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: mon.c (mon.2) 197 : audit 2023-10-01T09:07:42.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:07:42.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023-10-01T09:07:42.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 2023-10-01T09:07 2023-10-01T09:07:42.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: :41.684632+0000 mon.a (mon. 2023-10-01T09:07:42.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 0) 883 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:42.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 2023-10-01T09:07:42.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:42.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 10-01T09:07:41.686104+0000 2023-10-01T09:07:42.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: mon.c (mon.2) 198 : audit 2023-10-01T09:07:42.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-01T09:07:42.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023-10-01T09:07:42.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 2023-10-01T09:07 2023-10-01T09:07:42.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: :41.686407+0000 mon.a (mon 2023-10-01T09:07:42.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: .0) 884 : audit [INF] 2023-10-01T09:07:42.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-01T09:07:42.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:42.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 10-01T09:07:41.689400+0000 2023-10-01T09:07:42.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: mon.a (mon.0) 885 : audit 2023-10-01T09:07:42.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: [INF] from='mgr.15957 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-01T09:07:42.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023-10-01T09:07:42.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 2023-10-01T09:07:41.690875+0000 2023-10-01T09:07:42.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: mon.c (mon.2) 199 : audit 2023-10-01T09:07:42.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-01T09:07:42.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023-10-01T09:07:42.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 2023-10-01T09: 2023-10-01T09:07:42.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 07:41.691136+0000 mon.a ( 2023-10-01T09:07:42.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: mon.0) 886 : audit [INF] 2023-10-01T09:07:42.082 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-01T09:07:42.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023-10-01T09:07:42.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 2023-10-01T09:07 2023-10-01T09:07:42.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: :41.694024+0000 mon.a (mon. 2023-10-01T09:07:42.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 0) 887 : audit [INF] from='mgr.15957 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-01T09:07:42.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 2023-10-01T09:07:42.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:42.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: 10-01T09:07:41.696025+0000 2023-10-01T09:07:42.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: mon.c (mon.2) 200 : 2023-10-01T09:07:42.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T09:07:42.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:07:42.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: -01T09:07:41.696685+0000 2023-10-01T09:07:42.083 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: mon.c (mon.2) 201 : audit 2023-10-01T09:07:42.084 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[109959]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-01T09:07:42.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09:07: 2023-10-01T09:07:42.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 41.686104+0000 mon.c (mon.2) 198 : audit 2023-10-01T09:07:42.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-01T09:07:42.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09:07: 2023-10-01T09:07:42.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 41.686407+0000 mon.a (mon.0) 884 : audit 2023-10-01T09:07:42.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: [INF] from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-01T09:07:42.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09:07:41 2023-10-01T09:07:42.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: .689400+0000 mon.a (mon.0) 885 : audit 2023-10-01T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: [INF] from='mgr.15957 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-01T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09:07:41 2023-10-01T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: .690875+0000 mon.c (mon.2) 199 : audit 2023-10-01T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-01T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09:07:41 2023-10-01T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: .691136+0000 mon.a (mon.0) 886 : audit 2023-10-01T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: [INF] from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-01T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 2023-10-01T09:07:41.694024+0000 mon.a (mon 2023-10-01T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: .0) 887 : audit [INF] from='mgr.15957 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-01T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09 2023-10-01T09:07:42.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: :07:41.696025+0000 mon.c (mon.2 2023-10-01T09:07:42.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: ) 200 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T09:07:42.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: audit 2023-10-01T09: 2023-10-01T09:07:42.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: 07:41.696685+0000 mon.c (mon.2) 201 2023-10-01T09:07:42.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:41 smithi138 conmon[100289]: : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-01T09:07:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10-01T09:07:41.556021+0000 mon.c 2023-10-01T09:07:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: (mon.2) 193 : audit [DBG] from='client.? 172.21.15.138:0/1029705174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10-01T09:07:41.582723+0000 2023-10-01T09:07:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: mon.a (mon.0) 878 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:07:42.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: :07:41.598294+0000 mon.a (mon.0 2023-10-01T09:07:42.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: ) 879 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:42.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:07:42.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: -01T09:07:41.599438+0000 2023-10-01T09:07:42.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: mon.c (mon.2) 194 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:42.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:07:42.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: -01T09:07:41.600162+0000 2023-10-01T09:07:42.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: mon.c (mon.2) 195 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:07:42.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 2023-10-01T09:07:42.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10-01T09:07:42.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 2023-10-01T09:07:41.605053+0000 mon.a 2023-10-01T09:07:42.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: (mon.0) 880 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:42.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: cluster 2023 2023-10-01T09:07:42.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: -10-01T09:07:41. 2023-10-01T09:07:42.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 665090+0000 mon.a (mon.0) 881 : cluster [DBG] 2023-10-01T09:07:42.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: Standby manager daemon x restarted 2023-10-01T09:07:42.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: cluster 2023-10-01T09 2023-10-01T09:07:42.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: :07:41.665193+0000 mon.a (mon.0) 2023-10-01T09:07:42.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 882 : cluster [DBG] Standby manager daemon x started 2023-10-01T09:07:42.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:07:42.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: -01T09:07:41.667331+0000 2023-10-01T09:07:42.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: mon.b (mon.1) 105 : audit [DBG] from='mgr.? 172.21.15.190:0/2017602409' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:07:42.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 2023-10-01T09:07:42.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10-01T09:07:42.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 2023-10-01T09:07:41.667769 2023-10-01T09:07:42.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: +0000 mon.b (mon.1) 106 : audit [DBG] 2023-10-01T09:07:42.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: from='mgr.? 172.21.15.190:0/2017602409' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:07:42.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:07:42.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: :07:41.670614+0000 mon.b (mon.1 2023-10-01T09:07:42.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: ) 107 : audit [DBG] from='mgr.? 172.21.15.190:0/2017602409' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T09:07:42.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:07:42.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 01T09:07:41.671019+0000 mon.b 2023-10-01T09:07:42.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: (mon.1) 108 : audit [DBG] from='mgr.? 172.21.15.190:0/2017602409' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:07:42.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10-01T09:07:42.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 2023-10-01T09:07:41 2023-10-01T09:07:42.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: .677584+0000 mon.c (mon.2) 196 : audit 2023-10-01T09:07:42.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:42.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T09:07:42.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 09:07:41.680099+0000 mon.c (mon 2023-10-01T09:07:42.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: .2) 197 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:07:42.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:42.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: -10-01T09:07:41 2023-10-01T09:07:42.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: .684632+0000 mon.a (mon.0) 883 : audit 2023-10-01T09:07:42.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:42.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T09:07:42.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 09:07:41.686104+0000 mon.c (mon. 2023-10-01T09:07:42.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 2) 198 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-01T09:07:42.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:07:42.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: -01T09:07:41. 2023-10-01T09:07:42.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 686407+0000 mon.a (mon.0) 884 : audit 2023-10-01T09:07:42.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: [INF] from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-01T09:07:42.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T09:07:42.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 09:07:41.689400+0000 mon.a (mon. 2023-10-01T09:07:42.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 0) 885 : audit [INF] from='mgr.15957 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-01T09:07:42.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:07:42.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 10-01T09:07:41. 2023-10-01T09:07:42.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 690875+0000 mon.c (mon.2) 199 : audit [INF] 2023-10-01T09:07:42.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-01T09:07:42.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:07:42.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: :07:41.691136+0000 mon.a (mon.0 2023-10-01T09:07:42.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: ) 886 : audit [INF] from='mgr.15957 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-01T09:07:42.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:07:42.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: -01T09:07:41.694024+0000 2023-10-01T09:07:42.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: mon.a (mon.0) 887 : audit [INF] from='mgr.15957 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-01T09:07:42.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 2023-10-01T09:07:42.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10-01T09:07:42.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 2023-10-01T09:07:41.696025+0000 2023-10-01T09:07:42.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: mon.c (mon.2) 200 : audit [DBG] 2023-10-01T09:07:42.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T09:07:42.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: audit 2023-10-01T09:07:42.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 2023-10-01T09:07:41. 2023-10-01T09:07:42.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: 696685+0000 mon.c (mon.2) 201 : audit [DBG] 2023-10-01T09:07:42.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:41 smithi190 conmon[100518]: from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-01T09:07:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[100289]: cluster 2023-10-01T09:07:41.488334+0000 mgr.y (mgr.15957) 31 : cluster 2023-10-01T09:07:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[100289]: [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:41.680680+0000 mgr.y (mgr.15957) 32 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-01T09:07:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:41.697129+0000 mgr.y (mgr.15957) 33 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-01T09:07:43.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[100289]: cluster 2023-10-01T09:07:42.610010+0000 mon.a (mon.0) 888 : cluster [DBG] mgrmap e29: y(active, since 25s), standbys: x 2023-10-01T09:07:43.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[109959]: cluster 2023-10-01T09:07:41 2023-10-01T09:07:43.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[109959]: .488334+0000 mgr.y (mgr.15957) 31 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:43.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:41.680680 2023-10-01T09:07:43.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[109959]: +0000 mgr.y (mgr.15957) 32 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-01T09:07:43.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:41 2023-10-01T09:07:43.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[109959]: .697129+0000 mgr.y (mgr.15957) 33 : cephadm [INF] 2023-10-01T09:07:43.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[109959]: Upgrade: It appears safe to stop mon.a 2023-10-01T09:07:43.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[109959]: cluster 2023-10-01T09:07:43.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[109959]: 2023-10-01T09:07:42.610010+0000 2023-10-01T09:07:43.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[109959]: mon.a (mon.0) 888 : cluster 2023-10-01T09:07:43.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:42 smithi138 conmon[109959]: [DBG] mgrmap e29: y(active, since 25s), standbys: x 2023-10-01T09:07:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:42 smithi190 conmon[100518]: cluster 2023-10-01T09:07:41.488334+0000 mgr.y (mgr.15957) 31 : cluster [DBG] 2023-10-01T09:07:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:42 smithi190 conmon[100518]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:42 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:41.680680+0000 mgr.y (mgr.15957) 32 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-01T09:07:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:42 smithi190 conmon[100518]: cephadm 2023- 2023-10-01T09:07:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:42 smithi190 conmon[100518]: 10-01T09:07:41.697129+0000 mgr.y (mgr.15957) 33 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-01T09:07:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:42 smithi190 conmon[100518]: cluster 2023 2023-10-01T09:07:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:42 smithi190 conmon[100518]: -10-01T09:07:42. 2023-10-01T09:07:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:42 smithi190 conmon[100518]: 610010+0000 mon.a (mon.0) 888 : cluster [DBG] 2023-10-01T09:07:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:42 smithi190 conmon[100518]: mgrmap e29: y(active, since 25s), standbys: x 2023-10-01T09:07:44.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:43.313582+0000 mgr.y (mgr.15957) 34 : cephadm [INF] Upgrade: Updating mon.a 2023-10-01T09:07:44.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[109959]: audit 2023-10-01T09:07:43.318064+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:44.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[109959]: audit 2023-10-01T09:07:43.319557+0000 mon.c (mon.2) 202 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:07:44.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[109959]: audit 2023-10-01T09:07:43.320124+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:07:44.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[109959]: audit 2023-10-01T09:07:43.320681+0000 mon.c (mon.2) 204 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:44.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:43.321388+0000 mgr.y (mgr.15957) 35 : cephadm [INF] Deploying daemon mon.a on smithi138 2023-10-01T09:07:44.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[109959]: cluster 2023-10-01T09:07:43.488896+0000 mgr.y (mgr.15957) 36 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:44.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[109959]: audit 2023-10-01T09:07:43.970544+0000 mon.a (mon.0) 890 : audit [DBG] from='client.? 172.21.15.138:0/851768301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:44.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:43.313582+0000 mgr.y (mgr.15957) 34 : cephadm [INF] Upgrade: Updating mon.a 2023-10-01T09:07:44.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[100289]: audit 2023-10-01T09:07:43.318064+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:44.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[100289]: audit 2023-10-01T09:07:43.319557+0000 mon.c (mon.2) 202 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:07:44.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[100289]: audit 2023-10-01T09:07:43.320124+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:07:44.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[100289]: audit 2023-10-01T09:07:43.320681+0000 mon.c (mon.2) 204 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:44.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[100289]: cephadm 2023-10-01T09:07:43.321388+0000 mgr.y (mgr.15957) 35 : cephadm [INF] Deploying daemon mon.a on smithi138 2023-10-01T09:07:44.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[100289]: cluster 2023-10-01T09:07:43.488896+0000 mgr.y (mgr.15957) 36 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:44.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[100289]: audit 2023-10-01T09:07:44.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[100289]: 2023-10-01T09:07:43.970544+0000 mon.a (mon.0) 890 : audit [DBG] from='client.? 172.21.15.138:0/851768301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:43.313582+0000 mgr.y (mgr.15957) 34 : cephadm [INF] Upgrade: Updating mon.a 2023-10-01T09:07:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: audit 2023-10-01T09:07:43.318064 2023-10-01T09:07:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: +0000 mon.a (mon.0) 889 : audit [INF] from='mgr.15957 ' entity='mgr.y' 2023-10-01T09:07:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: audit 2023-10-01T09:07:43.319557+0000 mon.c (mon.2) 202 : audit [INF] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:07:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: audit 2023-10-01T09:07:43.320124 2023-10-01T09:07:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: +0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:07:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: audit 2023-10-01T09:07:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: 2023-10-01T09:07:43.320681+0000 mon.c (mon 2023-10-01T09:07:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: .2) 204 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:07:44.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: cephadm 2023 2023-10-01T09:07:44.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: -10-01T09:07:43. 2023-10-01T09:07:44.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: 321388+0000 mgr.y (mgr.15957) 35 2023-10-01T09:07:44.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: : cephadm [INF] Deploying daemon mon.a on smithi138 2023-10-01T09:07:44.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: cluster 2023 2023-10-01T09:07:44.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: -10-01T09:07:43 2023-10-01T09:07:44.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: .488896+0000 mgr.y (mgr.15957) 36 : cluster 2023-10-01T09:07:44.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:44.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:44 smithi190 conmon[100518]: audit 2023-10-01T09:07:43.970544+0000 mon.a (mon.0) 890 : audit [DBG] from='client.? 172.21.15.138:0/851768301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:44.935 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:44 smithi138 systemd[1]: Stopping Ceph mon.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:07:45.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:07:44] "GET /metrics HTTP/1.1" 200 33792 "" "Prometheus/2.33.4" 2023-10-01T09:07:45.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:45 smithi138 conmon[100289]: debug 2023-10-01T09:07:45.093+0000 7f41d6aa8700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-01T09:07:45.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:45 smithi138 conmon[100289]: debug 2023-10-01T09:07:45.093+0000 7f41d6aa8700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-01T09:07:45.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:45 smithi138 bash[185433]: 476ba6abfb2c09e4f2dbb2b9301c99ede5fa33e9f07a6663cf138191f39cf5be 2023-10-01T09:07:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-01T09:07:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 systemd[1]: Stopped Ceph mon.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:07:46.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 systemd[1]: Starting Ceph mon.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:07:47.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: set uid:gid to 167:167 (ceph:ceph) 2023-10-01T09:07:47.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable), process ceph-mon, pid 7 2023-10-01T09:07:47.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 bash[185821]: 4f0e28d177103453bbf53e18d3a692c64f6a49d2d70fcb622e1b4a995b85be88 2023-10-01T09:07:47.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: pidfile_write: ignore empty --pid-file 2023-10-01T09:07:47.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 systemd[1]: Started Ceph mon.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:07:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: load: jerasure load: lrc 2023-10-01T09:07:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: RocksDB version: 7.9.2 2023-10-01T09:07:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Git sha 0 2023-10-01T09:07:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Compile date 2023-09-29 15:06:10 2023-10-01T09:07:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: DB SUMMARY 2023-10-01T09:07:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: DB Session ID: 6EDRSY1SN8QQVUIW2BLI 2023-10-01T09:07:47.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: CURRENT file: CURRENT 2023-10-01T09:07:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: IDENTITY file: IDENTITY 2023-10-01T09:07:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: MANIFEST file: MANIFEST-000008 size: 780 Bytes 2023-10-01T09:07:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000020.sst 2023-10-01T09:07:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000018.log size: 5370740 ; 2023-10-01T09:07:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.error_if_exists: 0 2023-10-01T09:07:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.create_if_missing: 0 2023-10-01T09:07:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.paranoid_checks: 1 2023-10-01T09:07:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-01T09:07:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-01T09:07:47.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-01T09:07:47.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.env: 0x55b20d41b720 2023-10-01T09:07:47.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.fs: PosixFileSystem 2023-10-01T09:07:47.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.info_log: 0x55b20f8c7420 2023-10-01T09:07:47.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_file_opening_threads: 16 2023-10-01T09:07:47.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.statistics: (nil) 2023-10-01T09:07:47.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.use_fsync: 0 2023-10-01T09:07:47.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_log_file_size: 0 2023-10-01T09:07:47.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T09:07:47.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T09:07:47.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.keep_log_file_num: 1000 2023-10-01T09:07:47.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.recycle_log_file_num: 0 2023-10-01T09:07:47.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.allow_fallocate: 1 2023-10-01T09:07:47.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.allow_mmap_reads: 0 2023-10-01T09:07:47.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.allow_mmap_writes: 0 2023-10-01T09:07:47.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.use_direct_reads: 0 2023-10-01T09:07:47.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T09:07:47.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.create_missing_column_families: 0 2023-10-01T09:07:47.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.db_log_dir: 2023-10-01T09:07:47.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.wal_dir: 2023-10-01T09:07:47.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T09:07:47.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T09:07:47.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T09:07:47.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T09:07:47.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T09:07:47.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T09:07:47.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.advise_random_on_open: 1 2023-10-01T09:07:47.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.db_write_buffer_size: 0 2023-10-01T09:07:47.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.write_buffer_manager: 0x55b20eb56c80 2023-10-01T09:07:47.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T09:07:47.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T09:07:47.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T09:07:47.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.rate_limiter: (nil) 2023-10-01T09:07:47.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T09:07:47.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.wal_recovery_mode: 2 2023-10-01T09:07:47.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.enable_thread_tracking: 0 2023-10-01T09:07:47.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.enable_pipelined_write: 0 2023-10-01T09:07:47.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.unordered_write: 0 2023-10-01T09:07:47.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T09:07:47.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T09:07:47.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T09:07:47.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T09:07:47.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.row_cache: None 2023-10-01T09:07:47.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.wal_filter: None 2023-10-01T09:07:47.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T09:07:47.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.allow_ingest_behind: 0 2023-10-01T09:07:47.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.two_write_queues: 0 2023-10-01T09:07:47.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.manual_wal_flush: 0 2023-10-01T09:07:47.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.wal_compression: 0 2023-10-01T09:07:47.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.atomic_flush: 0 2023-10-01T09:07:47.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T09:07:47.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T09:07:47.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T09:07:47.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.log_readahead_size: 0 2023-10-01T09:07:47.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-01T09:07:47.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.best_efforts_recovery: 0 2023-10-01T09:07:47.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-01T09:07:47.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-01T09:07:47.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.allow_data_in_errors: 0 2023-10-01T09:07:47.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.db_host_id: __hostname__ 2023-10-01T09:07:47.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-01T09:07:47.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_background_jobs: 2 2023-10-01T09:07:47.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_background_compactions: -1 2023-10-01T09:07:47.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_subcompactions: 1 2023-10-01T09:07:47.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T09:07:47.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T09:07:47.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T09:07:47.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_total_wal_size: 0 2023-10-01T09:07:47.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T09:07:47.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T09:07:47.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T09:07:47.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T09:07:47.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_open_files: -1 2023-10-01T09:07:47.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.bytes_per_sync: 0 2023-10-01T09:07:47.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T09:07:47.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T09:07:47.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compaction_readahead_size: 0 2023-10-01T09:07:47.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_background_flushes: -1 2023-10-01T09:07:47.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Compression algorithms supported: 2023-10-01T09:07:47.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T09:07:47.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: kZSTD supported: 0 2023-10-01T09:07:47.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: kXpressCompression supported: 0 2023-10-01T09:07:47.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: kLZ4HCCompression supported: 1 2023-10-01T09:07:47.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: kZlibCompression supported: 1 2023-10-01T09:07:47.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: kSnappyCompression supported: 1 2023-10-01T09:07:47.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: kLZ4Compression supported: 1 2023-10-01T09:07:47.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: kBZip2Compression supported: 0 2023-10-01T09:07:47.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T09:07:47.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-01T09:07:47.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2023-10-01T09:07:47.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-01T09:07:47.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T09:07:47.087 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.merge_operator: 2023-10-01T09:07:47.087 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compaction_filter: None 2023-10-01T09:07:47.087 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compaction_filter_factory: None 2023-10-01T09:07:47.087 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.sst_partitioner_factory: None 2023-10-01T09:07:47.087 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T09:07:47.087 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T09:07:47.087 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b20f8c7520) 2023-10-01T09:07:47.087 INFO:journalctl@ceph.mon.a.smithi138.stdout: cache_index_and_filter_blocks: 1 2023-10-01T09:07:47.087 INFO:journalctl@ceph.mon.a.smithi138.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T09:07:47.087 INFO:journalctl@ceph.mon.a.smithi138.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T09:07:47.087 INFO:journalctl@ceph.mon.a.smithi138.stdout: pin_top_level_index_and_filter: 1 2023-10-01T09:07:47.088 INFO:journalctl@ceph.mon.a.smithi138.stdout: index_type: 0 2023-10-01T09:07:47.088 INFO:journalctl@ceph.mon.a.smithi138.stdout: data_block_index_type: 0 2023-10-01T09:07:47.088 INFO:journalctl@ceph.mon.a.smithi138.stdout: index_shortening: 1 2023-10-01T09:07:47.088 INFO:journalctl@ceph.mon.a.smithi138.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-01T09:07:47.088 INFO:journalctl@ceph.mon.a.smithi138.stdout: checksum: 4 2023-10-01T09:07:47.088 INFO:journalctl@ceph.mon.a.smithi138.stdout: no_block_cache: 0 2023-10-01T09:07:47.088 INFO:journalctl@ceph.mon.a.smithi138.stdout: block_cache: 0x55b20ebd9350 2023-10-01T09:07:47.088 INFO:journalctl@ceph.mon.a.smithi138.stdout: block_cache_name: BinnedLRUCache 2023-10-01T09:07:47.088 INFO:journalctl@ceph.mon.a.smithi138.stdout: block_cache_options: 2023-10-01T09:07:47.088 INFO:journalctl@ceph.mon.a.smithi138.stdout: capacity : 536870912 2023-10-01T09:07:47.088 INFO:journalctl@ceph.mon.a.smithi138.stdout: num_shard_bits : 4 2023-10-01T09:07:47.089 INFO:journalctl@ceph.mon.a.smithi138.stdout: strict_capacity_limit : 0 2023-10-01T09:07:47.089 INFO:journalctl@ceph.mon.a.smithi138.stdout: high_pri_pool_ratio: 0.000 2023-10-01T09:07:47.089 INFO:journalctl@ceph.mon.a.smithi138.stdout: block_cache_compressed: (nil) 2023-10-01T09:07:47.089 INFO:journalctl@ceph.mon.a.smithi138.stdout: persistent_cache: (nil) 2023-10-01T09:07:47.089 INFO:journalctl@ceph.mon.a.smithi138.stdout: block_size: 4096 2023-10-01T09:07:47.089 INFO:journalctl@ceph.mon.a.smithi138.stdout: block_size_deviation: 10 2023-10-01T09:07:47.089 INFO:journalctl@ceph.mon.a.smithi138.stdout: block_restart_interval: 16 2023-10-01T09:07:47.089 INFO:journalctl@ceph.mon.a.smithi138.stdout: index_block_restart_interval: 1 2023-10-01T09:07:47.089 INFO:journalctl@ceph.mon.a.smithi138.stdout: metadata_block_size: 4096 2023-10-01T09:07:47.089 INFO:journalctl@ceph.mon.a.smithi138.stdout: partition_filters: 0 2023-10-01T09:07:47.089 INFO:journalctl@ceph.mon.a.smithi138.stdout: use_delta_encoding: 1 2023-10-01T09:07:47.089 INFO:journalctl@ceph.mon.a.smithi138.stdout: filter_policy: bloomfilter 2023-10-01T09:07:47.090 INFO:journalctl@ceph.mon.a.smithi138.stdout: whole_key_filtering: 1 2023-10-01T09:07:47.090 INFO:journalctl@ceph.mon.a.smithi138.stdout: verify_compression: 0 2023-10-01T09:07:47.090 INFO:journalctl@ceph.mon.a.smithi138.stdout: read_amp_bytes_per_bit: 0 2023-10-01T09:07:47.090 INFO:journalctl@ceph.mon.a.smithi138.stdout: format_version: 5 2023-10-01T09:07:47.090 INFO:journalctl@ceph.mon.a.smithi138.stdout: enable_index_compression: 1 2023-10-01T09:07:47.090 INFO:journalctl@ceph.mon.a.smithi138.stdout: block_align: 0 2023-10-01T09:07:47.090 INFO:journalctl@ceph.mon.a.smithi138.stdout: max_auto_readahead_size: 262144 2023-10-01T09:07:47.090 INFO:journalctl@ceph.mon.a.smithi138.stdout: prepopulate_block_cache: 0 2023-10-01T09:07:47.090 INFO:journalctl@ceph.mon.a.smithi138.stdout: initial_auto_readahead_size: 8192 2023-10-01T09:07:47.090 INFO:journalctl@ceph.mon.a.smithi138.stdout: num_file_reads_for_auto_readahead: 2 2023-10-01T09:07:47.091 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.write_buffer_size: 33554432 2023-10-01T09:07:47.091 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_write_buffer_number: 2 2023-10-01T09:07:47.091 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compression: NoCompression 2023-10-01T09:07:47.091 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.bottommost_compression: Disabled 2023-10-01T09:07:47.091 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.prefix_extractor: nullptr 2023-10-01T09:07:47.091 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T09:07:47.091 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.num_levels: 7 2023-10-01T09:07:47.091 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T09:07:47.091 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T09:07:47.091 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T09:07:47.091 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T09:07:47.091 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T09:07:47.092 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T09:07:47.092 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T09:07:47.092 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T09:07:47.092 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-01T09:07:47.092 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T09:07:47.092 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-01T09:07:47.092 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-01T09:07:47.092 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T09:07:47.092 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compression_opts.level: 32767 2023-10-01T09:07:47.092 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compression_opts.strategy: 0 2023-10-01T09:07:47.092 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T09:07:47.093 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T09:07:47.093 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-01T09:07:47.093 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-01T09:07:47.093 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compression_opts.enabled: false 2023-10-01T09:07:47.093 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-01T09:07:47.093 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T09:07:47.093 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T09:07:47.093 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T09:07:47.093 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.target_file_size_base: 67108864 2023-10-01T09:07:47.093 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T09:07:47.093 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T09:07:47.093 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T09:07:47.094 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T09:07:47.094 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T09:07:47.094 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T09:07:47.094 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T09:07:47.094 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T09:07:47.094 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T09:07:47.094 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T09:07:47.094 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T09:07:47.094 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T09:07:47.094 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T09:07:47.094 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-01T09:07:47.095 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.arena_block_size: 1048576 2023-10-01T09:07:47.095 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T09:07:47.095 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T09:07:47.095 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.disable_auto_compactions: 0 2023-10-01T09:07:47.095 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T09:07:47.095 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T09:07:47.095 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T09:07:47.095 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T09:07:47.097 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T09:07:47.097 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T09:07:47.097 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T09:07:47.097 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T09:07:47.097 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T09:07:47.097 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T09:07:47.097 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.table_properties_collectors: 2023-10-01T09:07:47.097 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.inplace_update_support: 0 2023-10-01T09:07:47.097 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T09:07:47.098 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T09:07:47.098 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T09:07:47.098 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T09:07:47.098 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.bloom_locality: 0 2023-10-01T09:07:47.098 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.max_successive_merges: 0 2023-10-01T09:07:47.098 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T09:07:47.098 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.paranoid_file_checks: 0 2023-10-01T09:07:47.098 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.force_consistency_checks: 1 2023-10-01T09:07:47.098 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.report_bg_io_stats: 0 2023-10-01T09:07:47.098 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.ttl: 2592000 2023-10-01T09:07:47.099 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T09:07:47.099 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-01T09:07:47.099 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-01T09:07:47.099 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.enable_blob_files: false 2023-10-01T09:07:47.099 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.min_blob_size: 0 2023-10-01T09:07:47.099 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.blob_file_size: 268435456 2023-10-01T09:07:47.099 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-01T09:07:47.099 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-01T09:07:47.099 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-01T09:07:47.099 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-01T09:07:47.100 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-01T09:07:47.100 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.blob_file_starting_level: 0 2023-10-01T09:07:47.100 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-01T09:07:47.100 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 20.sst 2023-10-01T09:07:47.100 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-01T09:07:47.100 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 22, last_sequence is 9924, log_number is 18,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T09:07:47.100 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 18 2023-10-01T09:07:47.100 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 6bf0a807-4b40-40d2-8ad7-48de2d43593f 2023-10-01T09:07:47.100 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696151266815859, "job": 1, "event": "recovery_started", "wal_files": [18]} 2023-10-01T09:07:47.100 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #18 mode 2 2023-10-01T09:07:47.100 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696151266830808, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 23, "file_size": 5135495, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 9913, "largest_seqno": 11146, "table_properties": {"data_size": 5127596, "index_size": 5792, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1093, "raw_key_size": 8813, "raw_average_key_size": 21, "raw_value_size": 5115427, "raw_average_value_size": 12416, "num_data_blocks": 257, "num_entries": 412, "num_filter_entries": 412, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696151266, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "6bf0a807-4b40-40d2-8ad7-48de2d43593f", "db_session_id": "6EDRSY1SN8QQVUIW2BLI", "orig_file_number": 23, "seqno_to_time_mapping": "N/A"}} 2023-10-01T09:07:47.101 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696151266830904, "job": 1, "event": "recovery_finished"} 2023-10-01T09:07:47.101 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: [db/version_set.cc:5047] Creating manifest 25 2023-10-01T09:07:47.101 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-01T09:07:47.101 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000018.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-01T09:07:47.101 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b20ec74000 2023-10-01T09:07:47.101 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: DB pointer 0x55b20ec62000 2023-10-01T09:07:47.101 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-01T09:07:47.101 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-01T09:07:47.101 INFO:journalctl@ceph.mon.a.smithi138.stdout: ** DB Stats ** 2023-10-01T09:07:47.101 INFO:journalctl@ceph.mon.a.smithi138.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T09:07:47.101 INFO:journalctl@ceph.mon.a.smithi138.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T09:07:47.102 INFO:journalctl@ceph.mon.a.smithi138.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T09:07:47.102 INFO:journalctl@ceph.mon.a.smithi138.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T09:07:47.102 INFO:journalctl@ceph.mon.a.smithi138.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T09:07:47.102 INFO:journalctl@ceph.mon.a.smithi138.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T09:07:47.102 INFO:journalctl@ceph.mon.a.smithi138.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T09:07:47.102 INFO:journalctl@ceph.mon.a.smithi138.stdout: 2023-10-01T09:07:47.102 INFO:journalctl@ceph.mon.a.smithi138.stdout: ** Compaction Stats [default] ** 2023-10-01T09:07:47.102 INFO:journalctl@ceph.mon.a.smithi138.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T09:07:47.102 INFO:journalctl@ceph.mon.a.smithi138.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-01T09:07:47.102 INFO:journalctl@ceph.mon.a.smithi138.stdout: L0 1/0 4.90 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 600.3 0.01 0.00 1 0.008 0 0 0.0 0.0 2023-10-01T09:07:47.103 INFO:journalctl@ceph.mon.a.smithi138.stdout: L6 1/0 22.68 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-01T09:07:47.103 INFO:journalctl@ceph.mon.a.smithi138.stdout: Sum 2/0 27.58 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 600.3 0.01 0.00 1 0.008 0 0 0.0 0.0 2023-10-01T09:07:47.103 INFO:journalctl@ceph.mon.a.smithi138.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 600.3 0.01 0.00 1 0.008 0 0 0.0 0.0 2023-10-01T09:07:47.103 INFO:journalctl@ceph.mon.a.smithi138.stdout: 2023-10-01T09:07:47.103 INFO:journalctl@ceph.mon.a.smithi138.stdout: ** Compaction Stats [default] ** 2023-10-01T09:07:47.103 INFO:journalctl@ceph.mon.a.smithi138.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T09:07:47.103 INFO:journalctl@ceph.mon.a.smithi138.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T09:07:47.103 INFO:journalctl@ceph.mon.a.smithi138.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 600.3 0.01 0.00 1 0.008 0 0 0.0 0.0 2023-10-01T09:07:47.103 INFO:journalctl@ceph.mon.a.smithi138.stdout: 2023-10-01T09:07:47.103 INFO:journalctl@ceph.mon.a.smithi138.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-01T09:07:47.103 INFO:journalctl@ceph.mon.a.smithi138.stdout: 2023-10-01T09:07:47.104 INFO:journalctl@ceph.mon.a.smithi138.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T09:07:47.104 INFO:journalctl@ceph.mon.a.smithi138.stdout: Flush(GB): cumulative 0.005, interval 0.005 2023-10-01T09:07:47.104 INFO:journalctl@ceph.mon.a.smithi138.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T09:07:47.104 INFO:journalctl@ceph.mon.a.smithi138.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T09:07:47.104 INFO:journalctl@ceph.mon.a.smithi138.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T09:07:47.104 INFO:journalctl@ceph.mon.a.smithi138.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-01T09:07:47.104 INFO:journalctl@ceph.mon.a.smithi138.stdout: Cumulative compaction: 0.00 GB write, 257.29 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T09:07:47.104 INFO:journalctl@ceph.mon.a.smithi138.stdout: Interval compaction: 0.00 GB write, 257.29 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T09:07:47.104 INFO:journalctl@ceph.mon.a.smithi138.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T09:07:47.104 INFO:journalctl@ceph.mon.a.smithi138.stdout: Block cache BinnedLRUCache@0x55b20ebd9350#7 capacity: 512.00 MB usage: 7.23 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2023-10-01T09:07:47.104 INFO:journalctl@ceph.mon.a.smithi138.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.16 KB,0.000220537%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2023-10-01T09:07:47.105 INFO:journalctl@ceph.mon.a.smithi138.stdout: 2023-10-01T09:07:47.105 INFO:journalctl@ceph.mon.a.smithi138.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-01T09:07:47.105 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: starting mon.a rank 0 at public addrs [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] at bind addrs [v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T09:07:47.105 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: mon.a@-1(???) e3 preinit fsid 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T09:07:47.105 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: mon.a@-1(???).mds e7 new map 2023-10-01T09:07:47.105 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: mon.a@-1(???).mds e7 print_map 2023-10-01T09:07:47.105 INFO:journalctl@ceph.mon.a.smithi138.stdout: e7 2023-10-01T09:07:47.105 INFO:journalctl@ceph.mon.a.smithi138.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-01T09:07:47.105 INFO:journalctl@ceph.mon.a.smithi138.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T09:07:47.106 INFO:journalctl@ceph.mon.a.smithi138.stdout: legacy client fscid: 1 2023-10-01T09:07:47.106 INFO:journalctl@ceph.mon.a.smithi138.stdout: 2023-10-01T09:07:47.106 INFO:journalctl@ceph.mon.a.smithi138.stdout: Filesystem 'foo' (1) 2023-10-01T09:07:47.106 INFO:journalctl@ceph.mon.a.smithi138.stdout: fs_name foo 2023-10-01T09:07:47.106 INFO:journalctl@ceph.mon.a.smithi138.stdout: epoch 5 2023-10-01T09:07:47.106 INFO:journalctl@ceph.mon.a.smithi138.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-01T09:07:47.106 INFO:journalctl@ceph.mon.a.smithi138.stdout: created 2023-10-01T09:03:03.201740+0000 2023-10-01T09:07:47.106 INFO:journalctl@ceph.mon.a.smithi138.stdout: modified 2023-10-01T09:03:09.223385+0000 2023-10-01T09:07:47.106 INFO:journalctl@ceph.mon.a.smithi138.stdout: tableserver 0 2023-10-01T09:07:47.106 INFO:journalctl@ceph.mon.a.smithi138.stdout: root 0 2023-10-01T09:07:47.107 INFO:journalctl@ceph.mon.a.smithi138.stdout: session_timeout 60 2023-10-01T09:07:47.107 INFO:journalctl@ceph.mon.a.smithi138.stdout: session_autoclose 300 2023-10-01T09:07:47.107 INFO:journalctl@ceph.mon.a.smithi138.stdout: max_file_size 1099511627776 2023-10-01T09:07:47.107 INFO:journalctl@ceph.mon.a.smithi138.stdout: required_client_features {} 2023-10-01T09:07:47.107 INFO:journalctl@ceph.mon.a.smithi138.stdout: last_failure 0 2023-10-01T09:07:47.107 INFO:journalctl@ceph.mon.a.smithi138.stdout: last_failure_osd_epoch 0 2023-10-01T09:07:47.107 INFO:journalctl@ceph.mon.a.smithi138.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T09:07:47.107 INFO:journalctl@ceph.mon.a.smithi138.stdout: max_mds 1 2023-10-01T09:07:47.107 INFO:journalctl@ceph.mon.a.smithi138.stdout: in 0 2023-10-01T09:07:47.107 INFO:journalctl@ceph.mon.a.smithi138.stdout: up {0=14637} 2023-10-01T09:07:47.108 INFO:journalctl@ceph.mon.a.smithi138.stdout: failed 2023-10-01T09:07:47.108 INFO:journalctl@ceph.mon.a.smithi138.stdout: damaged 2023-10-01T09:07:47.108 INFO:journalctl@ceph.mon.a.smithi138.stdout: stopped 2023-10-01T09:07:47.108 INFO:journalctl@ceph.mon.a.smithi138.stdout: data_pools [4] 2023-10-01T09:07:47.108 INFO:journalctl@ceph.mon.a.smithi138.stdout: metadata_pool 3 2023-10-01T09:07:47.108 INFO:journalctl@ceph.mon.a.smithi138.stdout: inline_data disabled 2023-10-01T09:07:47.108 INFO:journalctl@ceph.mon.a.smithi138.stdout: balancer 2023-10-01T09:07:47.108 INFO:journalctl@ceph.mon.a.smithi138.stdout: bal_rank_mask -1 2023-10-01T09:07:47.109 INFO:journalctl@ceph.mon.a.smithi138.stdout: standby_count_wanted 1 2023-10-01T09:07:47.109 INFO:journalctl@ceph.mon.a.smithi138.stdout: [mds.foo.smithi138.brhzod{0:14637} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.138:6834/416108704,v1:172.21.15.138:6835/416108704] compat {c=[1],r=[1],i=[7ff]}] 2023-10-01T09:07:47.109 INFO:journalctl@ceph.mon.a.smithi138.stdout: 2023-10-01T09:07:47.109 INFO:journalctl@ceph.mon.a.smithi138.stdout: 2023-10-01T09:07:47.109 INFO:journalctl@ceph.mon.a.smithi138.stdout: Standby daemons: 2023-10-01T09:07:47.109 INFO:journalctl@ceph.mon.a.smithi138.stdout: 2023-10-01T09:07:47.109 INFO:journalctl@ceph.mon.a.smithi138.stdout: [mds.foo.smithi190.weynjc{-1:24505} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.190:6832/4158685175,v1:172.21.15.190:6833/4158685175] compat {c=[1],r=[1],i=[7ff]}] 2023-10-01T09:07:47.109 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: mon.a@-1(???).osd e84 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-01T09:07:47.109 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: mon.a@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T09:07:47.109 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: mon.a@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T09:07:47.109 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: mon.a@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T09:07:47.110 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:07:46 smithi138 ceph-mon[185886]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-10-01T09:07:48.358 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[152192]: ignoring --setuser ceph since I am not root 2023-10-01T09:07:48.359 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[152192]: ignoring --setgroup ceph since I am not root 2023-10-01T09:07:48.359 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: cluster 2023-10-01T09:07:45.489356+0000 mgr.y (mgr.15957) 37 : 2023-10-01T09:07:48.359 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:48.359 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: audit 2023-10-01T09:07:46.381744+0000 mon.c (mon.2) 205 : audit [DBG] from='client.? 172.21.15.138:0/978481902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:48.359 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: cluster 2023-10-01 2023-10-01T09:07:48.359 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: T09:07:47.445185+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-01T09:07:48.359 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: cluster 2023-10-01T09:07:47.489685+0000 2023-10-01T09:07:48.359 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: mgr.y (mgr.15957) 38 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:48.359 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T09:07:48.360 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: T09:07:47.692404+0000 mon.c (mon.2) 206 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:07:48.360 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: cluster 2023-10-01T09:07:48.259594+0000 2023-10-01T09:07:48.360 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T09:07:48.637 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[152192]: 2023-10-01T09:07:48.468+0000 7ff23f678200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T09:07:48.637 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[152192]: 2023-10-01T09:07:48.548+0000 7ff23f678200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T09:07:48.638 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: cluster 2023-10-01T09:07:48.264185+0000 mon.a ( 2023-10-01T09:07:48.638 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:07:48.638 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: cluster 2023-10-01T09:07:48.264807+0000 mon.a (mon. 2023-10-01T09:07:48.638 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: 0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:07:48.638 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: cluster 2023-10-01T09:07:48.638 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: 2023-10-01T09:07:48.264852+0000 mon.a (mon.0) 5 : cluster [DBG] 2023-10-01T09:07:48.638 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: osdmap e84: 8 total, 8 up, 8 in 2023-10-01T09:07:48.638 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: cluster 2023-10-01 2023-10-01T09:07:48.638 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: T09:07:48.265830+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e29: y(active, since 30s), standbys: x 2023-10-01T09:07:48.638 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: cluster 2023-10-01T09:07:48.283523 2023-10-01T09:07:48.638 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: +0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e30: y(active, since 30s), standbys: x 2023-10-01T09:07:48.639 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: cluster 2023-10-01T09:07: 2023-10-01T09:07:48.639 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: 48.283576+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-10-01T09:07:48.639 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: audit 2023-10-01T09:07:48 2023-10-01T09:07:48.639 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[100518]: .283758+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.15957 ' entity='' 2023-10-01T09:07:48.657 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[178630]: ignoring --setuser ceph since I am not root 2023-10-01T09:07:48.657 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[178630]: ignoring --setgroup ceph since I am not root 2023-10-01T09:07:48.657 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[178630]: 2023-10-01T09:07:48.480+0000 7fa04b722200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T09:07:48.657 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[178630]: 2023-10-01T09:07:48.565+0000 7fa04b722200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T09:07:48.657 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[178630]: 2023-10-01T09:07:48.655+0000 7fa04b722200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T09:07:48.658 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: cluster 2023-10-01T09:07:45.489356+0000 mgr.y (mgr.15957 2023-10-01T09:07:48.658 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: ) 37 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:48.658 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: audit 2023-10-01T09:07:46.381744+0000 mon.c (mon.2) 205 : audit [DBG] from='client.? 172.21.15.138:0/978481902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:48.658 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: cluster 2023-10-01T09:07:48.658 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: 2023-10-01T09:07:47.445185+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-01T09:07:48.658 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: cluster 2023-10-01 2023-10-01T09:07:48.658 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: T09:07:47.489685+0000 mgr.y (mgr.15957) 38 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:48.658 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: audit 2023-10-01T09:07:47.692404+0000 2023-10-01T09:07:48.658 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: mon.c (mon.2) 206 : audit [DBG] from='mgr.15957 172.21.15.138:0/2094206445' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:07:48.658 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: cluster 2023-10-01T09:07:48.259594+0000 mon.a ( 2023-10-01T09:07:48.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T09:07:48.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: cluster 2023-10-01T09:07:48.264185+0000 mon.a 2023-10-01T09:07:48.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:07:48.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: cluster 2023-10-01T09:07:48.264807+0000 mon.a 2023-10-01T09:07:48.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:07:48.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: cluster 2023-10-01T09:07 2023-10-01T09:07:48.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: :48.264852+0000 mon.a (mon.0) 5 2023-10-01T09:07:48.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-10-01T09:07:48.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: cluster 2023-10-01 2023-10-01T09:07:48.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: T09:07:48.265830+0000 2023-10-01T09:07:48.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: mon.a (mon.0) 6 : cluster [DBG] 2023-10-01T09:07:48.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: mgrmap e29: y(active, since 30s), standbys: x 2023-10-01T09:07:48.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: cluster 2023- 2023-10-01T09:07:48.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: 10-01T09:07:48.283523+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e30: y(active, since 30s), standbys: x 2023-10-01T09:07:48.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: cluster 2023- 2023-10-01T09:07:48.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: 10-01T09:07:48.283576 2023-10-01T09:07:48.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: +0000 mon.a (mon.0) 8 : cluster [INF] 2023-10-01T09:07:48.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: overall HEALTH_OK 2023-10-01T09:07:48.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:07:48.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: :07:48.283758+0000 mon.a (mon. 2023-10-01T09:07:48.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[109959]: 0) 9 : audit [INF] from='mgr.15957 ' entity='' 2023-10-01T09:07:48.785 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 1 2023-10-01T09:07:48.900 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[152192]: 2023-10-01T09:07:48.637+0000 7ff23f678200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T09:07:48.900 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[152192]: 2023-10-01T09:07:48.731+0000 7ff23f678200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T09:07:48.900 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[151158]: ts=2023-10-01T09:07:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:07:48.936 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[178630]: 2023-10-01T09:07:48.752+0000 7fa04b722200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T09:07:48.937 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:48 smithi138 conmon[178630]: 2023-10-01T09:07:48.935+0000 7fa04b722200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T09:07:49.260 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:48 smithi190 conmon[152192]: 2023-10-01T09:07:48.899+0000 7ff23f678200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T09:07:49.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:49 smithi190 conmon[100518]: audit 2023-10-01T09:07:48.771197+0000 mon.c (mon.2) 2023-10-01T09:07:49.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:49 smithi190 conmon[100518]: 207 : audit [DBG] from='client.? 172.21.15.138:0/1878618349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:49.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:49 smithi190 conmon[152192]: 2023-10-01T09:07:49.259+0000 7ff23f678200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T09:07:49.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:49 smithi190 conmon[152192]: 2023-10-01T09:07:49.391+0000 7ff23f678200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T09:07:49.557 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:49 smithi190 conmon[152192]: 2023-10-01T09:07:49.480+0000 7ff23f678200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T09:07:49.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:49 smithi138 conmon[109959]: audit 2023-10-01T09:07:48.771197+0000 mon.c (mon.2) 207 : audit [DBG] 2023-10-01T09:07:49.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:49 smithi138 conmon[109959]: from='client.? 172.21.15.138:0/1878618349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:07:49.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:49 smithi138 conmon[178630]: 2023-10-01T09:07:49.303+0000 7fa04b722200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T09:07:49.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:49 smithi138 conmon[178630]: 2023-10-01T09:07:49.435+0000 7fa04b722200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T09:07:49.570 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:49 smithi138 conmon[178630]: 2023-10-01T09:07:49.526+0000 7fa04b722200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T09:07:49.615 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch ps' 2023-10-01T09:07:50.000 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:49 smithi190 conmon[152192]: 2023-10-01T09:07:49.576+0000 7ff23f678200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T09:07:50.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:49 smithi138 conmon[178630]: 2023-10-01T09:07:49.628+0000 7fa04b722200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T09:07:50.306 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:50 smithi190 conmon[152192]: 2023-10-01T09:07:49.999+0000 7ff23f678200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T09:07:50.493 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:50 smithi138 conmon[178630]: 2023-10-01T09:07:50.097+0000 7fa04b722200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T09:07:51.056 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:50 smithi190 conmon[152192]: 2023-10-01T09:07:50.657+0000 7ff23f678200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T09:07:51.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:50 smithi190 conmon[152192]: 2023-10-01T09:07:50.741+0000 7ff23f678200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T09:07:51.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:50 smithi138 conmon[178630]: 2023-10-01T09:07:50.778+0000 7fa04b722200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T09:07:51.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:50 smithi138 conmon[178630]: 2023-10-01T09:07:50.862+0000 7fa04b722200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T09:07:51.787 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:51 smithi190 conmon[152192]: 2023-10-01T09:07:51.522+0000 7ff23f678200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:07:51.787 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:51 smithi190 conmon[152192]: 2023-10-01T09:07:51.613+0000 7ff23f678200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T09:07:51.959 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:51 smithi138 conmon[178630]: 2023-10-01T09:07:51.688+0000 7fa04b722200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:07:51.959 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:51 smithi138 conmon[178630]: 2023-10-01T09:07:51.781+0000 7fa04b722200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T09:07:52.056 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:51 smithi190 conmon[152192]: 2023-10-01T09:07:51.787+0000 7ff23f678200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:07:52.241 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:51 smithi138 conmon[178630]: 2023-10-01T09:07:51.957+0000 7fa04b722200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T09:07:52.448 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:52 smithi190 conmon[152192]: 2023-10-01T09:07:52.071+0000 7ff23f678200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T09:07:52.448 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:52 smithi190 conmon[152192]: 2023-10-01T09:07:52.299+0000 7ff23f678200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T09:07:52.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:52 smithi138 conmon[178630]: 2023-10-01T09:07:52.240+0000 7fa04b722200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T09:07:52.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:52 smithi138 conmon[178630]: 2023-10-01T09:07:52.470+0000 7fa04b722200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T09:07:52.716 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:52 smithi190 conmon[152192]: 2023-10-01T09:07:52.448+0000 7ff23f678200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T09:07:52.890 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:52 smithi138 conmon[178630]: 2023-10-01T09:07:52.619+0000 7fa04b722200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T09:07:53.056 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:52 smithi190 conmon[152192]: 2023-10-01T09:07:52.716+0000 7ff23f678200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T09:07:53.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:52 smithi138 conmon[178630]: 2023-10-01T09:07:52.889+0000 7fa04b722200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T09:07:53.770 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:53 smithi190 conmon[152192]: 2023-10-01T09:07:53.434+0000 7ff23f678200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T09:07:53.770 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:53 smithi190 conmon[152192]: 2023-10-01T09:07:53.570+0000 7ff23f678200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T09:07:53.927 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:53 smithi138 conmon[178630]: 2023-10-01T09:07:53.598+0000 7fa04b722200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T09:07:53.927 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:53 smithi138 conmon[178630]: 2023-10-01T09:07:53.729+0000 7fa04b722200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T09:07:54.056 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:53 smithi190 conmon[152192]: 2023-10-01T09:07:53.769+0000 7ff23f678200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T09:07:54.056 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:53 smithi190 conmon[152192]: 2023-10-01T09:07:53.886+0000 7ff23f678200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T09:07:54.264 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:53 smithi138 conmon[178630]: 2023-10-01T09:07:53.926+0000 7fa04b722200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T09:07:54.264 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:54 smithi138 conmon[178630]: 2023-10-01T09:07:54.040+0000 7fa04b722200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T09:07:54.556 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:54 smithi190 conmon[152192]: 2023-10-01T09:07:54.109+0000 7ff23f678200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T09:07:55.300 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:54 smithi190 conmon[152192]: 2023-10-01T09:07:54.982+0000 7ff23f678200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T09:07:55.300 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:54 smithi190 conmon[152192]: [01/Oct/2023:09:07:54] ENGINE Bus STARTING 2023-10-01T09:07:55.300 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[152192]: CherryPy Checker: 2023-10-01T09:07:55.300 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[152192]: The Application mounted at '' has an empty config. 2023-10-01T09:07:55.301 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[152192]: 2023-10-01T09:07:55.301 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[152192]: [01/Oct/2023:09:07:55] ENGINE Serving on http://:::9283 2023-10-01T09:07:55.301 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[152192]: [01/Oct/2023:09:07:55] ENGINE Bus STARTED 2023-10-01T09:07:55.301 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[100518]: cluster 2023-10-01T09:07:54.996092+0000 mon.a (mon.0) 10 : cluster [DBG] Standby manager daemon x restarted 2023-10-01T09:07:55.301 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[100518]: cluster 2023-10-01T09:07:54.996192+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x started 2023-10-01T09:07:55.301 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[100518]: audit 2023-10-01T09:07:54.998388+0000 mon.c (mon.2) 208 : audit [DBG] 2023-10-01T09:07:55.301 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[100518]: from='mgr.? 172.21.15.190:0/4242122519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:07:55.301 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[100518]: audit 2023-10-01T09:07: 2023-10-01T09:07:55.301 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[100518]: 54.999004+0000 mon.c (mon.2) 209 : audit [DBG] from='mgr.? 172.21.15.190:0/4242122519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:07:55.302 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:55.302 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[100518]: -10-01T09:07:55.001229+0000 2023-10-01T09:07:55.302 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[100518]: mon.c (mon.2) 210 : audit [DBG] from='mgr.? 172.21.15.190:0/4242122519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T09:07:55.302 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[100518]: audit 2023-10-01T09:07:55.302 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[100518]: 2023-10-01T09 2023-10-01T09:07:55.302 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[100518]: :07:55.001565+0000 mon.c (mon.2 2023-10-01T09:07:55.302 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[100518]: ) 211 : audit [DBG] from='mgr.? 172.21.15.190:0/4242122519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:07:55.302 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:55 smithi190 conmon[100518]: debug 2023-10-01T09:07:55.160+0000 7faba3e68700 -1 mon.b@1(peon).osd e85 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-01T09:07:55.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:54 smithi138 conmon[178630]: 2023-10-01T09:07:54.262+0000 7fa04b722200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T09:07:55.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: 2023-10-01T09:07:55.142+0000 7fa04b722200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T09:07:55.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: [01/Oct/2023:09:07:55] ENGINE Bus STARTING 2023-10-01T09:07:55.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: 2023-10-01T09:07:55.304+0000 7f9faaa5e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:55.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: 2023-10-01T09:07:55.304+0000 7f9faaa5e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:55.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: 2023-10-01T09:07:55.304+0000 7f9faaa5e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:55.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: 2023-10-01T09:07:55.304+0000 7f9faaa5e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:55.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: 2023-10-01T09:07:55.304+0000 7f9faaa5e700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:55.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: 2023-10-01T09:07:55.304+0000 7f9fa6215700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:55.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: 2023-10-01T09:07:55.304+0000 7f9fa6215700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:55.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: 2023-10-01T09:07:55.304+0000 7f9fa6215700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:55.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: 2023-10-01T09:07:55.304+0000 7f9fa6215700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:55.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: 2023-10-01T09:07:55.304+0000 7f9fa6215700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T09:07:55.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[109959]: cluster 2023-10-01T09:07:54.996092+0000 mon.a (mon.0) 10 : cluster 2023-10-01T09:07:55.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[109959]: [DBG] Standby manager daemon x restarted 2023-10-01T09:07:55.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[109959]: cluster 2023-10-01T09:07:54.996192+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x started 2023-10-01T09:07:55.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[109959]: audit 2023-10-01T09:07:54.998388 2023-10-01T09:07:55.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[109959]: +0000 mon.c (mon.2) 208 : audit [DBG] from='mgr.? 172.21.15.190:0/4242122519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T09:07:55.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[109959]: audit 2023-10-01T09:07:54.999004+0000 2023-10-01T09:07:55.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[109959]: mon.c (mon.2) 209 : audit [DBG] from='mgr.? 172.21.15.190:0/4242122519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T09:07:55.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[109959]: audit 2023-10-01T09:07:55.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[109959]: 2023-10-01T09:07:55.001229+0000 mon.c (mon.2) 210 : audit [DBG] 2023-10-01T09:07:55.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[109959]: from='mgr.? 172.21.15.190:0/4242122519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T09:07:55.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:55.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[109959]: 10-01T09:07:55.001565+0000 mon.c (mon.2) 211 : audit [DBG] from='mgr.? 172.21.15.190:0/4242122519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T09:07:55.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[109959]: debug 2023-10-01T09:07:55.159+0000 7fbda3bc1700 -1 mon.c@2(peon).osd e85 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-01T09:07:55.819 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: CherryPy Checker: 2023-10-01T09:07:55.820 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: The Application mounted at '' has an empty config. 2023-10-01T09:07:55.820 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: 2023-10-01T09:07:55.820 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: [01/Oct/2023:09:07:55] ENGINE Serving on http://:::9283 2023-10-01T09:07:55.820 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:55 smithi138 conmon[178630]: [01/Oct/2023:09:07:55] ENGINE Bus STARTED 2023-10-01T09:07:56.182 INFO:teuthology.orchestra.run.smithi138.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-01T09:07:56.182 INFO:teuthology.orchestra.run.smithi138.stdout:alertmanager.a smithi138 *:9093,9094 running (84s) - 5m 18.3M - 0.23.0 ba2b418f427c 8debc681d94e 2023-10-01T09:07:56.182 INFO:teuthology.orchestra.run.smithi138.stdout:grafana.a smithi190 *:3000 running (51s) - 5m 50.1M - 8.3.5 dad864ee21e9 58ea19a8c949 2023-10-01T09:07:56.182 INFO:teuthology.orchestra.run.smithi138.stdout:mds.foo.smithi138.brhzod smithi138 running (4m) - 4m 17.2M - 16.2.14-44-geeb13ac9 94a6c4498547 555d8c2cfe25 2023-10-01T09:07:56.183 INFO:teuthology.orchestra.run.smithi138.stdout:mds.foo.smithi190.weynjc smithi190 running (4m) - 4m 17.5M - 16.2.14-44-geeb13ac9 94a6c4498547 881fea906eff 2023-10-01T09:07:56.183 INFO:teuthology.orchestra.run.smithi138.stdout:mgr.x smithi190 *:8443,9283,8765 running (22s) - 9m 49.2M - 18.2.0-497-gb5682219 a3fc96fed5ac 674e1140402f 2023-10-01T09:07:56.183 INFO:teuthology.orchestra.run.smithi138.stdout:mgr.y smithi138 *:8443,9283,8765 running (46s) - 10m 477M - 18.2.0-497-gb5682219 a3fc96fed5ac 97da5dac6a84 2023-10-01T09:07:56.183 INFO:teuthology.orchestra.run.smithi138.stdout:mon.a smithi138 running (10m) - 11m 120M 2048M 16.2.14-44-geeb13ac9 94a6c4498547 476ba6abfb2c 2023-10-01T09:07:56.183 INFO:teuthology.orchestra.run.smithi138.stdout:mon.b smithi190 running (9m) - 9m 97.4M 2048M 16.2.14-44-geeb13ac9 94a6c4498547 520f260f067c 2023-10-01T09:07:56.183 INFO:teuthology.orchestra.run.smithi138.stdout:mon.c smithi138 running (9m) - 9m 92.9M 2048M 16.2.14-44-geeb13ac9 94a6c4498547 711c0763bf42 2023-10-01T09:07:56.183 INFO:teuthology.orchestra.run.smithi138.stdout:node-exporter.a smithi138 *:9100 running (93s) - 5m 22.9M - 1.3.1 1dbe0e931976 bd644aa2145b 2023-10-01T09:07:56.183 INFO:teuthology.orchestra.run.smithi138.stdout:node-exporter.b smithi190 *:9100 running (70s) - 5m 22.8M - 1.3.1 1dbe0e931976 c08f7a49a78d 2023-10-01T09:07:56.183 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0 smithi138 running (9m) - 9m 71.4M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 7636a677953c 2023-10-01T09:07:56.184 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1 smithi138 running (8m) - 8m 95.2M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 a510d44fb2c6 2023-10-01T09:07:56.184 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2 smithi138 running (8m) - 8m 63.4M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 8a8389da7887 2023-10-01T09:07:56.184 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3 smithi138 running (7m) - 7m 61.9M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 c3ba3a01539d 2023-10-01T09:07:56.184 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4 smithi190 running (7m) - 7m 67.7M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 cda54a787614 2023-10-01T09:07:56.184 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5 smithi190 running (7m) - 7m 86.7M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 4ac584c1fb7d 2023-10-01T09:07:56.184 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6 smithi190 running (6m) - 6m 68.3M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 fa45a1c37077 2023-10-01T09:07:56.184 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7 smithi190 running (6m) - 6m 88.6M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 c925b7fc3562 2023-10-01T09:07:56.184 INFO:teuthology.orchestra.run.smithi138.stdout:prometheus.a smithi190 *:9095 running (26s) - 5m 44.9M - 2.33.4 514e6a882f6e bbdb076a24d4 2023-10-01T09:07:56.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[178630]: 2023-10-01T09:07:56.169+0000 7fa004e48700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-10-01T09:07:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: cluster 2023-10-01T09:07:55.052791+0000 mon.a ( 2023-10-01T09:07:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: mon.0) 12 : cluster [DBG] mgrmap e31: y(active, since 37s), standbys: x 2023-10-01T09:07:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: cluster 2023-10-01T09:07:55.156635+0000 mon.a (mon.0) 13 : cluster [INF] Active manager daemon y restarted 2023-10-01T09:07:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: cluster 2023-10-01T09:07:55.156964+0000 mon.a (mon.0) 14 : cluster [INF] Activating manager daemon y 2023-10-01T09:07:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: cluster 2023-10-01T09:07:55.162137+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:07:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: cluster 2023-10- 2023-10-01T09:07:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 01T09:07:55.162395+0000 mon.a (mon.0) 16 : cluster [DBG] mgrmap e32: y(active, starting, since 0.00551933s), standbys: x 2023-10-01T09:07:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: cluster 2023-10- 2023-10-01T09:07:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 01T09:07:55.163871+0000 osd.2 (osd.2) 3 : cluster [WRN] 2023-10-01T09:07:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: failed to encode map e85 with expected crc 2023-10-01T09:07:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: cluster 2023-10-01T09:07:55.164622+0000 2023-10-01T09:07:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e85 with expected crc 2023-10-01T09:07:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: cluster 2023-10-01T09 2023-10-01T09:07:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: :07:55.169077+0000 osd.4 (osd.4) 4 : cluster 2023-10-01T09:07:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: [WRN] failed to encode map e85 with expected crc 2023-10-01T09:07:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:07:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: :07:55.170136+0000 mon.a (mon. 2023-10-01T09:07:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 0) 17 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:07:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10-01T09:07:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 2023-10-01T09:07: 2023-10-01T09:07:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 55.170279+0000 mon.a (mon.0) 2023-10-01T09:07:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 18 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:07:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10-01T09:07 2023-10-01T09:07:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: :55.170392+0000 mon.a (mon.0) 19 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:07:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:07:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 09:07:55.170622+0000 mon.a (mon.0) 20 2023-10-01T09:07:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.brhzod"}]: dispatch 2023-10-01T09:07:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10-01T09 2023-10-01T09:07:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: :07:55.170816+0000 mon.a (mon.0) 2023-10-01T09:07:56.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 21 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi190.weynjc"}]: dispatch 2023-10-01T09:07:56.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:56.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 10-01T09:07:55.171541 2023-10-01T09:07:56.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: +0000 mon.a (mon.0) 22 : 2023-10-01T09:07:56.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T09:07:56.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10-01T09:07:56.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 2023-10 2023-10-01T09:07:56.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: -01T09:07:55.171677 2023-10-01T09:07:56.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: +0000 mon.a (mon.0) 23 2023-10-01T09:07:56.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T09:07:56.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10-01T09:07:56.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 2023-10-01T09:07: 2023-10-01T09:07:56.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 55.171906+0000 mon.a (mon.0) 2023-10-01T09:07:56.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 24 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T09:07:56.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:07:56.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 01T09:07:55.172146+0000 mon.a 2023-10-01T09:07:56.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: (mon.0) 25 : audit [DBG] 2023-10-01T09:07:56.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T09:07:56.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:07:56.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: -01T09:07:55.172341+0000 2023-10-01T09:07:56.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: mon.a (mon.0) 26 : audit 2023-10-01T09:07:56.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T09:07:56.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:07:56.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 01T09:07:55.172526 2023-10-01T09:07:56.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: +0000 mon.a (mon.0) 27 : audit 2023-10-01T09:07:56.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:07:56.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023 2023-10-01T09:07:56.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: -10-01T09:07:55. 2023-10-01T09:07:56.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 172701+0000 mon.a (mon.0) 28 : 2023-10-01T09:07:56.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:07:56.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10-01T09:07:56.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 2023-10-01T09:07:55 2023-10-01T09:07:56.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: .172932+0000 mon.a (mon.0) 29 : audit [DBG] 2023-10-01T09:07:56.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:07:56.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10-01T09:07:56.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 2023-10-01T09:07:55.173245+0000 mon.a (mon.0) 30 2023-10-01T09:07:56.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:07:56.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10-01T09:07:55. 2023-10-01T09:07:56.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 173459+0000 mon.a (mon.0) 31 : audit [DBG] 2023-10-01T09:07:56.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:07:56.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10-01T09:07 2023-10-01T09:07:56.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: :55.173671+0000 mon.a (mon.0) 32 : 2023-10-01T09:07:56.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T09:07:56.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:07:56.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 01T09:07:55.173875+0000 mon.a (mon 2023-10-01T09:07:56.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: .0) 33 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T09:07:56.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10-01T09:07:56.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 2023-10-01T09:07:55. 2023-10-01T09:07:56.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 174287+0000 mon.a (mon.0) 34 : audit [DBG] 2023-10-01T09:07:56.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T09:07:56.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: cluster 2023- 2023-10-01T09:07:56.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 10-01T09:07:55 2023-10-01T09:07:56.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: .191980+0000 mon.a (mon.0 2023-10-01T09:07:56.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: ) 35 : cluster [INF] Manager daemon y is now available 2023-10-01T09:07:56.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:07:56.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 10-01T09:07:55.213972 2023-10-01T09:07:56.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: +0000 mon.a (mon.0) 36 2023-10-01T09:07:56.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:56.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023 2023-10-01T09:07:56.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: -10-01T09:07:55.216636 2023-10-01T09:07:56.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: +0000 mon.a (mon.0) 37 2023-10-01T09:07:56.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:56.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023 2023-10-01T09:07:56.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: -10-01T09:07:55 2023-10-01T09:07:56.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: .237520+0000 mon.a (mon.0 2023-10-01T09:07:56.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: ) 38 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:07:56.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023 2023-10-01T09:07:56.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: -10-01T09:07:55 2023-10-01T09:07:56.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: .246755+0000 mon.a (mon.0) 39 2023-10-01T09:07:56.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:07:56.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: cluster 2023 2023-10-01T09:07:56.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: -10-01T09:07:55. 2023-10-01T09:07:56.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 251118+0000 osd.5 (osd.5) 2023-10-01T09:07:56.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 3 : cluster [WRN] failed to encode map e85 with expected crc 2023-10-01T09:07:56.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: cluster 2023 2023-10-01T09:07:56.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: -10-01T09:07:55. 2023-10-01T09:07:56.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 270590+0000 osd.1 (osd.1) 2023-10-01T09:07:56.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 3 : cluster [WRN] failed to encode map e85 with expected crc 2023-10-01T09:07:56.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: audit 2023-10-01T09:07:56.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 2023-10-01T09:07: 2023-10-01T09:07:56.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 55.280425+0000 mon.a (mon. 2023-10-01T09:07:56.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 0) 40 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T09:07:56.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:56 smithi138 conmon[109959]: 2023-10-01T09:07:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: cluster 2023-10-01T09:07:55.052791+0000 mon.a (mon.0) 12 : cluster [DBG] mgrmap e31: y(active, since 37s), standbys: x 2023-10-01T09:07:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: cluster 2023-10-01T09:07:55.156635+0000 mon.a (mon.0) 13 : cluster [INF] Active manager daemon y restarted 2023-10-01T09:07:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: cluster 2023-10-01T09:07:55.156964+0000 mon.a (mon.0) 14 : cluster [INF] Activating manager daemon y 2023-10-01T09:07:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: cluster 2023-10-01T09:07:55.162137+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:07:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: cluster 2023-10-01T09:07:55.162395+0000 mon.a (mon.0) 16 : cluster [DBG] mgrmap e32: y(active, starting, since 0.00551933s), standbys: x 2023-10-01T09:07:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: cluster 2023-10-01T09:07:55.163871+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e85 with expected crc 2023-10-01T09:07:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: cluster 2023-10-01T09:07:55.164622+0000 osd.4 (osd.4) 3 : cluster [WRN] 2023-10-01T09:07:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: failed to encode map e85 with expected crc 2023-10-01T09:07:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: cluster 2023-10-01T09:07:55.169077+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e85 with expected crc 2023-10-01T09:07:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10-01T09:07:55.170136+0000 mon.a (mon.0) 17 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:07:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10-01T09:07:55.170279+0000 mon.a (mon.0) 18 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:07:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10-01T09:07:55.170392+0000 mon.a (mon.0) 19 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:07:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10-01T09:07:55.170622+0000 mon.a (mon.0) 20 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi138.brhzod"}]: dispatch 2023-10-01T09:07:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:07:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: -01T09:07:55.170816+0000 mon.a (mon.0) 21 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi190.weynjc"}]: dispatch 2023-10-01T09:07:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10-01T09:07:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 2023-10-01T09:07:55.171541+0000 2023-10-01T09:07:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: mon.a (mon.0) 22 : audit [DBG] 2023-10-01T09:07:56.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T09:07:56.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T09:07:56.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: T09:07:55.171677+0000 mon.a 2023-10-01T09:07:56.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: (mon.0) 23 : audit [DBG] 2023-10-01T09:07:56.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T09:07:56.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:07:56.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: -01T09:07:55.171906 2023-10-01T09:07:56.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: +0000 mon.a (mon.0) 24 : audit 2023-10-01T09:07:56.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T09:07:56.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:56.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: -10-01T09:07:55. 2023-10-01T09:07:56.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 172146+0000 mon.a (mon.0) 25 : 2023-10-01T09:07:56.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T09:07:56.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:56.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: -10-01T09:07:55. 2023-10-01T09:07:56.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 172341+0000 mon.a (mon.0) 26 : audit 2023-10-01T09:07:56.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T09:07:56.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:07:56.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 01T09:07:55.172526+0000 2023-10-01T09:07:56.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: mon.a (mon.0) 27 : audit [DBG] 2023-10-01T09:07:56.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:07:56.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:07:56.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 01T09:07:55.172701+0000 2023-10-01T09:07:56.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: mon.a (mon.0) 28 : audit [DBG] 2023-10-01T09:07:56.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T09:07:56.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T09:07:56.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: T09:07:55.172932+0000 mon.a 2023-10-01T09:07:56.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: (mon.0) 29 : audit [DBG] 2023-10-01T09:07:56.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:07:56.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T09:07:56.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 09:07:55.173245+0000 mon.a ( 2023-10-01T09:07:56.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: mon.0) 30 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:07:56.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 2023-10-01T09:07:56.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:07:56.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: -01T09:07:55.173459 2023-10-01T09:07:56.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: +0000 mon.a (mon.0) 31 : audit 2023-10-01T09:07:56.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T09:07:56.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:56.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: -10-01T09:07:55. 2023-10-01T09:07:56.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 173671+0000 mon.a (mon.0) 32 : audit 2023-10-01T09:07:56.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T09:07:56.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:07:56.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: -01T09:07:55.173875+0000 2023-10-01T09:07:56.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: mon.a (mon.0) 33 : audit 2023-10-01T09:07:56.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T09:07:56.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:07:56.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: -01T09:07:55.174287 2023-10-01T09:07:56.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: +0000 mon.a (mon.0) 34 : 2023-10-01T09:07:56.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T09:07:56.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: cluster 2023-10-01 2023-10-01T09:07:56.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: T09:07:55.191980+0000 mon.a 2023-10-01T09:07:56.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: (mon.0) 35 : cluster 2023-10-01T09:07:56.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: [INF] Manager daemon y is now available 2023-10-01T09:07:56.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023-10-01T09:07:56.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 2023-10-01T09:07 2023-10-01T09:07:56.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: :55.213972+0000 mon.a (mon. 2023-10-01T09:07:56.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 0) 36 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:56.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 2023-10-01T09:07:56.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:56.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: -10-01T09:07:55.216636+0000 mon.a 2023-10-01T09:07:56.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: (mon.0) 37 : audit [DBG] 2023-10-01T09:07:56.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:07:56.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:56.565 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: -10-01T09:07:55 2023-10-01T09:07:56.565 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: .237520+0000 mon.a (mon.0 2023-10-01T09:07:56.565 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: ) 38 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:07:56.565 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:56.565 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: -10-01T09:07:55. 2023-10-01T09:07:56.565 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 246755+0000 mon.a (mon.0) 39 : 2023-10-01T09:07:56.565 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T09:07:56.565 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: cluster 2023- 2023-10-01T09:07:56.565 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 10-01T09:07:55. 2023-10-01T09:07:56.566 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 251118+0000 osd.5 (osd.5 2023-10-01T09:07:56.566 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: ) 3 : cluster [WRN] failed to encode map e85 with expected crc 2023-10-01T09:07:56.566 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: cluster 2023- 2023-10-01T09:07:56.566 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 10-01T09:07:55. 2023-10-01T09:07:56.566 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: 270590+0000 osd.1 (osd.1) 3 2023-10-01T09:07:56.566 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: : cluster [WRN] failed to encode map e85 with expected crc 2023-10-01T09:07:56.566 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: audit 2023 2023-10-01T09:07:56.566 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: -10-01T09:07:55 2023-10-01T09:07:56.566 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: .280425+0000 mon.a (mon.0 2023-10-01T09:07:56.566 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:56 smithi190 conmon[100518]: ) 40 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T09:07:57.021 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade pause' 2023-10-01T09:07:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:57 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:55.660010+0000 mgr.y (mgr.26000 2023-10-01T09:07:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:57 smithi138 conmon[109959]: ) 1 : cephadm [INF] [01/Oct/2023:09:07:55] ENGINE Bus STARTING 2023-10-01T09:07:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:57 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:55.761825+0000 mgr.y (mgr.26000) 2 : cephadm [INF] [01/Oct/2023:09:07:55] ENGINE Serving on http://172.21.15.138:8765 2023-10-01T09:07:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:57 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:55.877044+0000 mgr.y (mgr.26000) 3 : cephadm [INF] [01/Oct/2023:09:07:55] ENGINE Serving on https://172.21.15.138:7150 2023-10-01T09:07:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:57 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:55.877174+0000 mgr.y (mgr.26000) 4 : cephadm [INF] [01/Oct/2023:09:07:55] ENGINE Bus STARTED 2023-10-01T09:07:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:57 smithi138 conmon[109959]: cluster 2023-10-01T09:07:56.166661+0000 mon.a (mon.0) 41 : cluster [DBG] mgrmap e33: y(active, since 1.00978s), standbys: x 2023-10-01T09:07:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:57 smithi138 conmon[109959]: audit 2023-10-01T09:07:56.174378+0000 mgr.y (mgr.26000) 5 : audit [DBG] from='client.26012 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:07:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:57 smithi138 conmon[109959]: cluster 2023-10-01T09:07:56.174882+0000 mgr.y (mgr.26000) 6 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:57.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:57 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:55.660010+0000 mgr.y ( 2023-10-01T09:07:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:57 smithi190 conmon[100518]: mgr.26000) 1 : cephadm [INF] [01/Oct/2023:09:07:55] ENGINE Bus STARTING 2023-10-01T09:07:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:57 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:55.761825+0000 mgr.y (mgr.26000) 2 : cephadm [INF] [01/Oct/2023:09:07:55] ENGINE Serving on http://172.21.15.138:8765 2023-10-01T09:07:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:57 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:55.877044+0000 mgr.y (mgr.26000) 3 : cephadm [INF] [01/Oct/2023:09:07:55] ENGINE Serving on https://172.21.15.138:7150 2023-10-01T09:07:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:57 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:55.877174+0000 mgr.y (mgr.26000) 4 : cephadm [INF] [01/Oct/2023:09:07:55] ENGINE Bus STARTED 2023-10-01T09:07:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:57 smithi190 conmon[100518]: cluster 2023-10-01T09:07:56.166661+0000 mon.a (mon.0) 41 : cluster [DBG] mgrmap e33: y(active, since 1.00978s), standbys: x 2023-10-01T09:07:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:57 smithi190 conmon[100518]: audit 2023-10-01T09:07:56.174378+0000 mgr.y (mgr.26000) 5 : audit [DBG] from='client.26012 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:07:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:57 smithi190 conmon[100518]: cluster 2023-10-01T09:07:56.174882+0000 mgr.y (mgr.26000) 6 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:58 smithi190 conmon[100518]: cluster 2023-10-01T09:07:57 2023-10-01T09:07:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:58 smithi190 conmon[100518]: .172998+0000 mgr.y (mgr.26000) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:58 smithi190 conmon[100518]: cluster 2023-10-01T09:07:57.848905+0000 mon.a (mon.0) 42 : cluster [DBG] mgrmap e34: y(active, since 2s), standbys: x 2023-10-01T09:07:58.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:58 smithi138 conmon[109959]: cluster 2023-10-01T09:07:57.172998+0000 mgr.y (mgr.26000) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:58.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:58 smithi138 conmon[109959]: cluster 2023-10-01T09:07:57.848905+0000 mon.a (mon.0) 42 : cluster [DBG] mgrmap e34: y(active, since 2s), standbys: x 2023-10-01T09:07:59.559 INFO:teuthology.orchestra.run.smithi138.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:07:59.918 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:59 smithi190 conmon[100518]: audit 2023-10-01T09:07:58. 2023-10-01T09:07:59.919 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:59 smithi190 conmon[100518]: 666245+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:07:59.919 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:59 smithi190 conmon[100518]: audit 2023-10-01T09:07:58.671209+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:07:59.919 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:59 smithi190 conmon[100518]: cluster 2023-10-01T09:07:59.173360+0000 mgr.y (mgr.26000) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:07:59.919 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:59 smithi190 conmon[100518]: audit 2023-10-01T09:07:59.304891+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:07:59.919 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:59 smithi190 conmon[100518]: audit 2023-10-01T09:07:59.312463+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:07:59.919 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:59 smithi190 conmon[100518]: audit 2023-10-01T09:07:59.416422+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:07:59.919 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:59 smithi190 conmon[100518]: audit 2023-10-01T09:07:59.423347+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:07:59.919 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:07:59 smithi190 conmon[100518]: audit 2023-10-01T09:07:59.558972+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:59 smithi138 conmon[109959]: audit 2023-10-01T09:07:58.666245+0000 mon.a (mon.0) 43 2023-10-01T09:08:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:59 smithi138 conmon[109959]: : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:59 smithi138 conmon[109959]: audit 2023-10-01T09:07:58.671209+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:59 smithi138 conmon[109959]: cluster 2023-10-01T09:07:59.173360+0000 mgr.y (mgr.26000) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:59 smithi138 conmon[109959]: audit 2023-10-01T09:07:59.304891+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:59 smithi138 conmon[109959]: audit 2023-10-01T09:07:59.312463+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:59 smithi138 conmon[109959]: audit 2023-10-01T09:07:59.416422+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:59 smithi138 conmon[109959]: audit 2023-10-01T09:07:59.423347+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:07:59 smithi138 conmon[109959]: audit 2023-10-01T09:07:59.558972+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:00.290 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'sleep 60' 2023-10-01T09:08:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:01 smithi190 conmon[100518]: audit 2023-10-01T09:07:59.553041+0000 mgr.y (mgr.26000) 9 : audit [DBG] from='client.26051 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:08:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:01 smithi190 conmon[100518]: cephadm 2023-10-01T09:07:59.553557+0000 mgr.y (mgr.26000) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:08:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:01 smithi190 conmon[100518]: audit 2023-10-01T09:08:00.257031+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:01 smithi190 conmon[100518]: audit 2023-10-01T09:08:00.261282+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:01.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:01 smithi138 conmon[109959]: audit 2023-10-01T09:07:59.553041+0000 mgr.y (mgr.26000) 9 : audit [DBG] from='client.26051 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:08:01.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:01 smithi138 conmon[109959]: cephadm 2023-10-01T09:07:59.553557+0000 mgr.y (mgr.26000) 10 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:08:01.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:01 smithi138 conmon[109959]: audit 2023-10-01T09:08:00.257031+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:01.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:01 smithi138 conmon[109959]: audit 2023-10-01T09:08:00.261282+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:02.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:02 smithi190 conmon[100518]: cluster 2023-10-01T09:08:01.173909+0000 mgr.y (mgr.26000) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:02.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:02 smithi190 conmon[100518]: audit 2023-10-01T09:08:02.043542+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:02 smithi190 conmon[100518]: audit 2023-10-01T09:08:02.050119+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:02 smithi190 conmon[100518]: audit 2023-10-01T09:08:02.051478+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:08:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:02 smithi190 conmon[100518]: audit 2023-10-01T09:08:02.052095+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:08:02.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:02 smithi138 conmon[109959]: cluster 2023-10-01T09:08:01.173909+0000 mgr.y (mgr.26000) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:02.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:02 smithi138 conmon[109959]: audit 2023-10-01T09:08:02.043542+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:02.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:02 smithi138 conmon[109959]: audit 2023-10-01T09:08:02.050119+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:02.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:02 smithi138 conmon[109959]: audit 2023-10-01T09:08:02.051478+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:08:02.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:02 smithi138 conmon[109959]: audit 2023-10-01T09:08:02.052095+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:08:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: cephadm 2023-10-01T09:08:02.052991+0000 mgr.y 2023-10-01T09:08:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: (mgr.26000) 12 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:08:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: cephadm 2023-10-01T09:08:02.053148+0000 mgr.y (mgr.26000) 13 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:08:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: cephadm 2023-10-01T09:08:02.267272+0000 mgr.y (mgr.26000) 14 : cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:08:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: cephadm 2023-10-01T09:08: 2023-10-01T09:08:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: 02.267822+0000 mgr.y (mgr.26000) 15 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:08:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: audit 2023-10-01T09:08:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: 2023-10-01T09:08:02.964424+0000 mon.a (mon.0 2023-10-01T09:08:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: ) 56 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: audit 2023-10-01T09:08:03.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: 2023-10-01T09:08:02 2023-10-01T09:08:03.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: .968577+0000 mon.a (mon.0) 57 : audit 2023-10-01T09:08:03.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:03.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: audit 2023-10-01T09 2023-10-01T09:08:03.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: :08:03.017950+0000 mon.a (mon.0) 2023-10-01T09:08:03.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: 58 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:03.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: audit 2023-10-01T09:08:03.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: 2023-10-01T09:08:03. 2023-10-01T09:08:03.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: 024688+0000 mon.a (mon.0) 59 : audit [INF] 2023-10-01T09:08:03.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:03.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:08:03.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: 01T09:08:03.030806+0000 mon.a (mon. 2023-10-01T09:08:03.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: 0) 60 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:03.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: audit 2023-10-01T09:08:03.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: 2023-10-01T09:08:03.102789+0000 2023-10-01T09:08:03.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: mon.a (mon.0) 61 : audit [DBG] 2023-10-01T09:08:03.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:03 smithi190 conmon[100518]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:08:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: cephadm 2023-10-01T09:08:02.052991+0000 mgr.y (mgr.26000 2023-10-01T09:08:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: ) 12 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:08:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: cephadm 2023-10-01T09:08:02.053148+0000 mgr.y (mgr.26000) 13 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:08:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: cephadm 2023-10-01T09:08:02. 2023-10-01T09:08:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: 267272+0000 mgr.y (mgr.26000) 14 : cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:08:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: cephadm 2023 2023-10-01T09:08:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: -10-01T09:08:02.267822+0000 mgr.y ( 2023-10-01T09:08:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: mgr.26000) 15 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:08:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: audit 2023-10-01T09:08:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: 2023-10-01T09:08:02.964424+0000 2023-10-01T09:08:03.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: mon.a (mon.0) 56 : audit [INF] 2023-10-01T09:08:03.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:03.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: audit 2023-10-01T09:08:02. 2023-10-01T09:08:03.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: 968577+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:03.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: 2023-10-01T09:08:03.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: audit 2023-10 2023-10-01T09:08:03.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: -01T09:08:03.017950+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:03.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: audit 2023- 2023-10-01T09:08:03.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: 10-01T09:08:03.024688+0000 mon.a (mon.0) 2023-10-01T09:08:03.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: 59 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:03.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: audit 2023-10-01T09:08: 2023-10-01T09:08:03.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: 03.030806+0000 mon.a (mon.0) 60 : audit [INF] 2023-10-01T09:08:03.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:03.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: audit 2023-10-01T09:08:03 2023-10-01T09:08:03.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: .102789+0000 mon.a (mon.0) 61 : audit 2023-10-01T09:08:03.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:03 smithi138 conmon[109959]: [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:08:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: cephadm 2023-10-01T09:08:02.531583+0000 mgr.y (mgr.26000) 2023-10-01T09:08:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: 16 : cephadm [INF] Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:08:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: cephadm 2023-10-01T09:08:02.533020+0000 mgr.y (mgr.26000) 17 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:08:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: cephadm 2023-10-01T09:08:02.746451+0000 2023-10-01T09:08:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: mgr.y (mgr.26000) 18 : cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:08:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: cephadm 2023-10-01T09:08:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: 2023-10-01T09:08:02.747416+0000 mgr.y 2023-10-01T09:08:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: (mgr.26000) 19 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:08:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: cluster 2023-10-01T09:08:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: 2023-10-01T09:08:03.174367+0000 mgr.y 2023-10-01T09:08:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: (mgr.26000) 20 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:08:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: 10-01T09:08:03.372410+0000 mon.a (mon.0 2023-10-01T09:08:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: ) 62 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: audit 2023 2023-10-01T09:08:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: -10-01T09:08:03.376764+0000 mon.a (mon 2023-10-01T09:08:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: .0) 63 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: audit 2023- 2023-10-01T09:08:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: 10-01T09:08:03.377740+0000 mon.a (mon 2023-10-01T09:08:04.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: .0) 64 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:08:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: audit 2023 2023-10-01T09:08:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: -10-01T09:08:03.378366+0000 mon.a ( 2023-10-01T09:08:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: mon.0) 65 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:08:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: audit 2023-10-01T09:08:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: 2023-10-01T09:08:03.382484+0000 mon.a 2023-10-01T09:08:04.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:04 smithi190 conmon[100518]: (mon.0) 66 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:04.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: cephadm 2023-10-01T09:08:02.531583+0000 mgr.y (mgr.26000) 16 : cephadm [INF] 2023-10-01T09:08:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:08:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: cephadm 2023-10-01T09:08:02.533020+0000 mgr.y (mgr.26000) 17 : cephadm [INF] Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:08:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: cephadm 2023-10-01T09:08:02.746451+0000 mgr.y (mgr.26000) 18 2023-10-01T09:08:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: : cephadm [INF] Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:08:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: cephadm 2023-10-01T09:08:02.747416+0000 mgr.y (mgr.26000) 2023-10-01T09:08:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: 19 : cephadm [INF] Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:08:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: cluster 2023-10-01T09:08:03. 2023-10-01T09:08:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: 174367+0000 mgr.y (mgr.26000) 20 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: audit 2023-10- 2023-10-01T09:08:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: 01T09:08:03.372410+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: audit 2023-10-01T09:08:03.376764+0000 mon.a (mon.0) 2023-10-01T09:08:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: 63 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: audit 2023-10-01T09:08:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: 2023-10-01T09:08:03 2023-10-01T09:08:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: .377740+0000 mon.a (mon.0) 64 : audit 2023-10-01T09:08:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:08:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: audit 2023-10-01T09:08:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: 2023-10-01T09:08:03.378366+0000 mon.a (mon.0) 65 : audit 2023-10-01T09:08:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:08:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: audit 2023-10-01T 2023-10-01T09:08:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[109959]: 09:08:03.382484+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:08:05.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:08:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:08:04] "GET /metrics HTTP/1.1" 200 31098 "" "Prometheus/2.33.4" 2023-10-01T09:08:06.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:06 smithi190 conmon[100518]: cluster 2023-10-01T09:08:05.174803+0000 mgr.y (mgr.26000) 21 : cluster [DBG] 2023-10-01T09:08:06.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:06 smithi190 conmon[100518]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:06.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:06 smithi138 conmon[109959]: cluster 2023-10-01T09:08:05.174803+0000 mgr.y (mgr.26000) 21 : 2023-10-01T09:08:06.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:06 smithi138 conmon[109959]: cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:08.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:07 smithi190 conmon[100518]: cluster 2023-10-01T09:08:07.175573+0000 mgr.y (mgr.26000) 22 : 2023-10-01T09:08:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:07 smithi190 conmon[100518]: cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:08.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:07 smithi138 conmon[109959]: cluster 2023-10-01T09:08:07.175573+0000 mgr.y (mgr.26000) 22 : cluster 2023-10-01T09:08:08.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:07 smithi138 conmon[109959]: [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:09.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:08:08 smithi190 conmon[151158]: ts=2023-10-01T09:08:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:08:10.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:10 smithi190 conmon[100518]: cluster 2023-10-01T09:08:09.176018+0000 mgr.y (mgr.26000) 23 : cluster [DBG] 2023-10-01T09:08:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:10 smithi190 conmon[100518]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:10 smithi138 conmon[109959]: cluster 2023-10-01T09:08:09.176018+0000 2023-10-01T09:08:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:10 smithi138 conmon[109959]: mgr.y (mgr.26000) 23 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:11.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:11 smithi190 conmon[100518]: audit 2023-10-01T09:08:10.237742+0000 mon.a (mon.0) 67 : audit 2023-10-01T09:08:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:11 smithi190 conmon[100518]: [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:08:11.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:11 smithi138 conmon[109959]: audit 2023-10-01T09:08:10.237742+0000 mon.a ( 2023-10-01T09:08:11.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:11 smithi138 conmon[109959]: mon.0) 67 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:08:12.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:12 smithi190 conmon[100518]: cluster 2023-10-01T09:08:11.176626+0000 mgr.y (mgr.26000) 24 : cluster 2023-10-01T09:08:12.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:12 smithi190 conmon[100518]: [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:12.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:12 smithi138 conmon[109959]: cluster 2023-10-01T09:08:11.176626+0000 mgr.y (mgr.26000) 24 2023-10-01T09:08:12.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:12 smithi138 conmon[109959]: : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:14 smithi190 conmon[100518]: cluster 2023-10-01T09:08:13.177010+0000 mgr.y (mgr.26000) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:14 smithi190 conmon[100518]: 2023-10-01T09:08:14.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:14 smithi138 conmon[109959]: cluster 2023-10-01T09:08:13.177010+0000 mgr.y (mgr.26000) 25 : cluster 2023-10-01T09:08:14.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:14 smithi138 conmon[109959]: [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:15.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:08:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:08:14] "GET /metrics HTTP/1.1" 200 33791 "" "Prometheus/2.33.4" 2023-10-01T09:08:16.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:16 smithi190 conmon[100518]: cluster 2023-10-01T09:08:15.177451+0000 mgr.y (mgr.26000) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:16.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:16 smithi138 conmon[109959]: cluster 2023-10-01T09:08:15.177451+0000 mgr.y (mgr 2023-10-01T09:08:16.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:16 smithi138 conmon[109959]: .26000) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:18.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:17 smithi190 conmon[100518]: cluster 2023-10-01T09:08:17.178043+0000 mgr.y (mgr.26000) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:18.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:17 smithi138 conmon[109959]: cluster 2023-10-01T09:08:17.178043+0000 mgr.y 2023-10-01T09:08:18.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:17 smithi138 conmon[109959]: (mgr.26000) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:19.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:08:18 smithi190 conmon[151158]: ts=2023-10-01T09:08:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:08:20.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:20 smithi190 conmon[100518]: cluster 2023-10-01T09:08:19.178493+0000 mgr.y (mgr.26000) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:20.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:20 smithi138 conmon[109959]: cluster 2023-10-01T09:08:19.178493+0000 mgr.y (mgr.26000) 2023-10-01T09:08:20.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:20 smithi138 conmon[109959]: 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:22.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:22 smithi190 conmon[100518]: cluster 2023-10-01T09:08:21.179264+0000 mgr.y (mgr.26000) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:22.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:22 smithi190 conmon[100518]: 2023-10-01T09:08:22.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:22 smithi138 conmon[109959]: cluster 2023-10-01T09:08:21.179264+0000 mgr.y (mgr.26000) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:24.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:24 smithi190 conmon[100518]: cluster 2023-10-01T09:08:23.179784+0000 mgr.y (mgr.26000) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:24.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:24 smithi138 conmon[109959]: cluster 2023-10-01T09:08:23.179784+0000 mgr.y (mgr.26000) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:24.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:24 smithi138 conmon[109959]: 2023-10-01T09:08:25.250 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:08:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:08:24] "GET /metrics HTTP/1.1" 200 33791 "" "Prometheus/2.33.4" 2023-10-01T09:08:25.250 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:25 smithi138 conmon[109959]: audit 2023-10-01T09:08:25.238083+0000 mon.a (mon.0) 68 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:08:25.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:25 smithi190 conmon[100518]: audit 2023-10-01T09:08:25.238083+0000 mon.a (mon.0) 68 : audit [DBG] 2023-10-01T09:08:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:25 smithi190 conmon[100518]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:08:26.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:26 smithi190 conmon[100518]: cluster 2023-10-01T09:08:25.180325+0000 mgr.y (mgr.26000) 31 : cluster [DBG] 2023-10-01T09:08:26.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:26 smithi190 conmon[100518]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:26.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:26 smithi138 conmon[109959]: cluster 2023-10-01T09:08:25.180325+0000 mgr.y (mgr. 2023-10-01T09:08:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:26 smithi138 conmon[109959]: 26000) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:28.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:27 smithi190 conmon[100518]: cluster 2023-10-01T09:08:27.181134+0000 mgr.y (mgr.26000) 32 : cluster 2023-10-01T09:08:28.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:27 smithi190 conmon[100518]: [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:28.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:27 smithi138 conmon[109959]: cluster 2023-10-01T09:08:27.181134+0000 mgr.y (mgr.26000) 32 : cluster [DBG] 2023-10-01T09:08:28.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:27 smithi138 conmon[109959]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:29.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:08:28 smithi190 conmon[151158]: ts=2023-10-01T09:08:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:08:30.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:30 smithi190 conmon[100518]: cluster 2023-10-01T09:08:29.181535+0000 mgr.y (mgr.26000) 33 : cluster [DBG] 2023-10-01T09:08:30.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:30 smithi190 conmon[100518]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:30.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:30 smithi138 conmon[109959]: cluster 2023-10-01T09:08:29.181535+0000 mgr.y (mgr.26000) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:32.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:32 smithi190 conmon[100518]: cluster 2023-10-01T09:08:31.182130+0000 mgr.y (mgr.26000) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:32.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:32 smithi138 conmon[109959]: cluster 2023-10-01T09:08:31.182130+0000 2023-10-01T09:08:32.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:32 smithi138 conmon[109959]: mgr.y (mgr.26000) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:34.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:34 smithi190 conmon[100518]: cluster 2023-10-01T09:08:33.182576+0000 mgr.y (mgr.26000) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:34.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:34 smithi138 conmon[109959]: cluster 2023-10-01T09:08:33.182576+0000 mgr.y 2023-10-01T09:08:34.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:34 smithi138 conmon[109959]: (mgr.26000) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:35.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:08:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:08:34] "GET /metrics HTTP/1.1" 200 33786 "" "Prometheus/2.33.4" 2023-10-01T09:08:36.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:36 smithi190 conmon[100518]: cluster 2023-10-01T09:08:35.183045+0000 mgr.y (mgr.26000) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:36.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:36 smithi138 conmon[109959]: cluster 2023-10-01T09:08:35.183045+0000 mgr.y 2023-10-01T09:08:36.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:36 smithi138 conmon[109959]: (mgr.26000) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:37 smithi190 conmon[100518]: cluster 2023-10-01T09:08:37.183935+0000 mgr.y (mgr.26000) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:38.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:37 smithi138 conmon[109959]: cluster 2023-10-01T09:08:37.183935+0000 mgr.y (mgr 2023-10-01T09:08:38.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:37 smithi138 conmon[109959]: .26000) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:39.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:08:38 smithi190 conmon[151158]: ts=2023-10-01T09:08:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:08:40.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:40 smithi190 conmon[100518]: cluster 2023-10-01T09:08:39.184544+0000 mgr.y (mgr.26000) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:40 smithi138 conmon[109959]: cluster 2023-10-01T09:08:39.184544+0000 mgr.y (mgr.26000) 38 : cluster 2023-10-01T09:08:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:40 smithi138 conmon[109959]: [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:41.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:41 smithi190 conmon[100518]: audit 2023-10-01T09:08:40.238372+0000 mon.a (mon.0) 69 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:08:41.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:41 smithi138 conmon[109959]: audit 2023-10-01T09:08:40.238372+0000 mon.a (mon.0) 69 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:08:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:41 smithi138 conmon[109959]: 2023-10-01T09:08:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:42 smithi190 conmon[100518]: cluster 2023-10-01T09:08:41.184999+0000 mgr.y (mgr.26000) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:42.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:42 smithi138 conmon[109959]: cluster 2023-10-01T09:08:41.184999+0000 mgr.y (mgr.26000) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:44 smithi190 conmon[100518]: cluster 2023-10-01T09:08:43.185484+0000 mgr.y (mgr.26000) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:44.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:44 smithi138 conmon[109959]: cluster 2023-10-01T09:08:43.185484+0000 mgr.y (mgr.26000) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:44.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:44 smithi138 conmon[109959]: 2023-10-01T09:08:45.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:08:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:08:44] "GET /metrics HTTP/1.1" 200 33786 "" "Prometheus/2.33.4" 2023-10-01T09:08:46.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:46 smithi190 conmon[100518]: cluster 2023-10-01T09:08:45.185904+0000 mgr.y (mgr.26000) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:46.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:46 smithi138 conmon[109959]: cluster 2023-10-01T09:08:45.185904+0000 mgr.y (mgr.26000) 41 : cluster [DBG] 2023-10-01T09:08:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:46 smithi138 conmon[109959]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:48.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:47 smithi190 conmon[100518]: cluster 2023-10-01T09:08:47.186601+0000 mgr.y (mgr.26000) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:48.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:47 smithi138 conmon[109959]: cluster 2023-10-01T09:08:47.186601+0000 mgr.y (mgr.26000) 42 : cluster [DBG] 2023-10-01T09:08:48.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:47 smithi138 conmon[109959]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:49.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:08:48 smithi190 conmon[151158]: ts=2023-10-01T09:08:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:08:50.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:50 smithi190 conmon[100518]: cluster 2023-10-01T09:08:49.187129+0000 mgr.y (mgr.26000) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:50.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:50 smithi138 conmon[109959]: cluster 2023-10-01T09:08:49.187129+0000 mgr.y (mgr.26000) 43 : cluster 2023-10-01T09:08:50.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:50 smithi138 conmon[109959]: [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:52 smithi190 conmon[100518]: cluster 2023-10-01T09:08:51.187726+0000 mgr.y (mgr.26000) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:52.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:52 smithi138 conmon[109959]: cluster 2023-10-01T09:08:51.187726+0000 mgr.y (mgr.26000) 44 : 2023-10-01T09:08:52.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:52 smithi138 conmon[109959]: cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:54.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:54 smithi190 conmon[100518]: cluster 2023-10-01T09:08:53.188100+0000 mgr.y (mgr.26000) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:54.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:54 smithi138 conmon[109959]: cluster 2023-10-01T09:08:53.188100+0000 mgr.y (mgr.26000) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:54.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:54 smithi138 conmon[109959]: 2023-10-01T09:08:55.255 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:08:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:08:54] "GET /metrics HTTP/1.1" 200 33786 "" "Prometheus/2.33.4" 2023-10-01T09:08:55.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:55 smithi190 conmon[100518]: audit 2023-10-01T09:08:55.241358+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:08:55.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:55 smithi138 conmon[109959]: audit 2023-10-01T09:08:55.241358+0000 mon.a (mon.0) 70 2023-10-01T09:08:55.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:55 smithi138 conmon[109959]: : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:08:56.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:56 smithi190 conmon[100518]: cluster 2023-10-01T09:08:55.188495+0000 mgr.y (mgr.26000) 46 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:56.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:56 smithi138 conmon[109959]: cluster 2023-10-01T09:08:55.188495+0000 mgr.y (mgr.26000) 46 : cluster [DBG] 2023-10-01T09:08:56.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:56 smithi138 conmon[109959]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:58.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:08:57 smithi190 conmon[100518]: cluster 2023-10-01T09:08:57.189251+0000 mgr.y (mgr.26000) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:58.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:08:57 smithi138 conmon[109959]: cluster 2023-10-01T09:08:57.189251+0000 mgr.y (mgr.26000) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:08:59.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:08:58 smithi190 conmon[151158]: ts=2023-10-01T09:08:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:09:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:00 smithi190 conmon[100518]: cluster 2023-10-01T09:08:59.189630+0000 mgr.y (mgr.26000) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:00.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:00 smithi138 conmon[109959]: cluster 2023-10-01T09:08:59.189630+0000 mgr.y (mgr.26000) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:02.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:02 smithi138 conmon[109959]: cluster 2023-10-01T09:09:01.190360+0000 mgr.y (mgr.26000 2023-10-01T09:09:02.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:02 smithi138 conmon[109959]: ) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:02.464 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade resume' 2023-10-01T09:09:02.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:02 smithi190 conmon[100518]: cluster 2023-10-01T09:09:01.190360+0000 mgr.y (mgr.26000) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:04 smithi138 conmon[109959]: cluster 2023-10-01T09:09:03.190684+0000 mgr.y (mgr.26000) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:04 smithi138 conmon[109959]: audit 2023-10-01T09:09:03.455020+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:04.440 INFO:teuthology.orchestra.run.smithi138.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:09:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:04 smithi190 conmon[100518]: cluster 2023-10-01T09:09:03.190684+0000 mgr.y (mgr.26000) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:04 smithi190 conmon[100518]: audit 2023-10-01T09:09:03.455020+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:05.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:09:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:09:04] "GET /metrics HTTP/1.1" 200 33784 "" "Prometheus/2.33.4" 2023-10-01T09:09:05.202 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'echo wait for majority of mons to upgrade' 2023-10-01T09:09:05.448 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:05 smithi138 conmon[109959]: audit 2023-10-01T09:09:04.436078+0000 mgr.y (mgr.26000) 51 : audit [DBG] from='client.26057 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:09:05.449 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:05 smithi138 conmon[109959]: cephadm 2023-10-01T09:09:04.436430+0000 mgr.y (mgr.26000) 52 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:09:05.449 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:05 smithi138 conmon[109959]: audit 2023-10-01T09:09:04.439987+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:05.449 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:05 smithi138 conmon[109959]: cluster 2023-10-01T09:09:05.190996+0000 mgr.y (mgr.26000) 53 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:05.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:05 smithi190 conmon[100518]: audit 2023-10-01T09:09:04.436078+0000 mgr.y (mgr.26000) 51 : audit [DBG] from='client.26057 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:09:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:05 smithi190 conmon[100518]: cephadm 2023-10-01T09:09:04.436430+0000 mgr.y (mgr.26000) 52 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:09:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:05 smithi190 conmon[100518]: audit 2023-10-01T09:09:04.439987+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:05 smithi190 conmon[100518]: cluster 2023-10-01T09:09:05.190996+0000 mgr.y (mgr.26000) 53 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:07.172 INFO:teuthology.orchestra.run.smithi138.stdout:wait for majority of mons to upgrade 2023-10-01T09:09:07.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:07 smithi190 conmon[100518]: audit 2023-10-01T09:09:06.429787+0000 mon.a (mon.0 2023-10-01T09:09:07.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:07 smithi190 conmon[100518]: ) 73 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:07.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:07 smithi190 conmon[100518]: audit 2023-10-01T09:09:06.434111+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:07.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:07 smithi190 conmon[100518]: cluster 2023-10-01T09:09:07.191697+0000 mgr.y (mgr.26000) 54 : cluster [DBG] 2023-10-01T09:09:07.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:07 smithi190 conmon[100518]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:07.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:07 smithi138 conmon[109959]: audit 2023-10-01T09:09:06.429787+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:07.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:07 smithi138 conmon[109959]: audit 2023-10-01T09:09:06.434111+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:07 smithi138 conmon[109959]: cluster 2023-10-01T09:09:07.191697+0000 mgr.y (mgr.26000) 54 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:07.896 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-10-01T09:09:09.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:09:08 smithi190 conmon[151158]: ts=2023-10-01T09:09:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:09:09.484 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:09 smithi138 conmon[109959]: audit 2023-10-01T09:09:08 2023-10-01T09:09:09.484 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:09 smithi138 conmon[109959]: .237578+0000 mon.a (mon.0) 75 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:09.484 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:09 smithi138 conmon[109959]: audit 2023-10-01T09:09:08.244274+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:09.485 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:09 smithi138 conmon[109959]: audit 2023-10-01T09:09:08.660948+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:09.485 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:09 smithi138 conmon[109959]: audit 2023-10-01T09:09:08.661698+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:09:09.485 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:09 smithi138 conmon[109959]: audit 2023-10-01T09:09:08.668130+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:09.485 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:09 smithi138 conmon[109959]: audit 2023-10-01T09:09:08.773063+0000 mon.a (mon.0) 80 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:09.485 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:09 smithi138 conmon[109959]: audit 2023-10-01T09:09:08.775252+0000 mon.a (mon.0) 81 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:09:09.485 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:09 smithi138 conmon[109959]: audit 2023-10-01T09:09:08.776177+0000 mon.a (mon.0) 82 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T09:09:09.485 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:09 smithi138 conmon[109959]: audit 2023-10-01T09:09:08.776737+0000 mon.a (mon.0) 83 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-01T09:09:09.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: audit 2023-10-01T09:09:08.237578+0000 mon.a (mon.0) 75 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: audit 2023-10-01T09:09:08.244274+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: audit 2023-10-01T09:09:08.660948+0000 mon.a 2023-10-01T09:09:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: (mon.0) 77 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: audit 2023-10- 2023-10-01T09:09:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: 01T09:09:08.661698+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:09:09.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: audit 2023-10-01T09:09:08 2023-10-01T09:09:09.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: .668130+0000 mon.a (mon.0) 79 : audit [INF] 2023-10-01T09:09:09.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:09.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: audit 2023-10-01T09:09:08. 2023-10-01T09:09:09.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: 773063+0000 mon.a (mon.0) 80 : audit [DBG] 2023-10-01T09:09:09.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:09.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: audit 2023-10-01T09:09:08.775252+0000 2023-10-01T09:09:09.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: mon.a (mon.0) 81 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:09:09.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: audit 2023-10 2023-10-01T09:09:09.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: -01T09:09:08.776177+0000 mon.a (mon. 2023-10-01T09:09:09.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: 0) 82 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T09:09:09.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T09:09:09.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: 09:09:08.776737+0000 mon.a (mon.0) 83 2023-10-01T09:09:09.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:09 smithi190 conmon[100518]: : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-01T09:09:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:10 smithi190 conmon[100518]: cephadm 2023-10-01T09:09:08.777104+0000 mgr.y (mgr.26000) 55 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-01T09:09:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:10 smithi190 conmon[100518]: 2023-10-01T09:09:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:10 smithi190 conmon[100518]: cluster 2023-10-01T09:09:09.192041+0000 mgr.y (mgr.26000) 56 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:10 smithi190 conmon[100518]: audit 2023-10-01T09:09:10.004121 2023-10-01T09:09:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:10 smithi190 conmon[100518]: +0000 mon.c (mon.2) 212 : audit [DBG] from='client.? 172.21.15.138:0/3814562467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:09:10.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:10 smithi138 conmon[109959]: cephadm 2023-10-01T09:09:08.777104+0000 mgr.y (mgr.26000) 55 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-01T09:09:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:10 smithi138 conmon[109959]: cluster 2023-10-01T09:09:09.192041+0000 mgr.y (mgr.26000) 56 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:10 smithi138 conmon[109959]: audit 2023-10-01T09:09:10.004121+0000 mon.c (mon.2) 212 : audit [DBG] from='client.? 172.21.15.138:0/3814562467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:09:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:11 smithi190 conmon[100518]: audit 2023-10-01T09:09:10.241577+0000 mon.a (mon.0) 84 : audit [DBG] 2023-10-01T09:09:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:11 smithi190 conmon[100518]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:11 smithi190 conmon[100518]: audit 2023-10-01T09:09:10.798824+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:11 smithi190 conmon[100518]: audit 2023-10-01T09:09:10.799494+0000 2023-10-01T09:09:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:11 smithi190 conmon[100518]: mon.a (mon.0) 86 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:11 smithi190 conmon[100518]: audit 2023-10-01T09:09: 2023-10-01T09:09:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:11 smithi190 conmon[100518]: 10.800029+0000 mon.a (mon.0) 87 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:11 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T09:09:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:11 smithi190 conmon[100518]: T09:09:10.800562+0000 mon.a (mon.0 2023-10-01T09:09:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:11 smithi190 conmon[100518]: ) 88 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:11.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:11 smithi138 conmon[109959]: audit 2023-10-01T09:09:10.241577+0000 mon.a (mon. 2023-10-01T09:09:11.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:11 smithi138 conmon[109959]: 0) 84 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:11.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:11 smithi138 conmon[109959]: audit 2023-10-01T09:09:10.798824+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:11.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:11 smithi138 conmon[109959]: audit 2023-10-01T09:09:10.799494+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:11.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:11 smithi138 conmon[109959]: audit 2023-10-01T09:09:10.800029 2023-10-01T09:09:11.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:11 smithi138 conmon[109959]: +0000 mon.a (mon.0) 87 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:11.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:11 smithi138 conmon[109959]: audit 2023-10-01T09:09 2023-10-01T09:09:11.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:11 smithi138 conmon[109959]: :10.800562+0000 mon.a (mon.0) 88 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:12.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:12 smithi138 conmon[109959]: cephadm 2023-10-01T09:09:10.794664+0000 mgr.y (mgr.26000) 57 : cephadm [INF] Upgrade: Updating mon.c 2023-10-01T09:09:12.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:12 smithi138 conmon[109959]: cephadm 2023-10-01T09:09:10.801523+0000 mgr.y (mgr.26000) 58 : cephadm [INF] Deploying daemon mon.c on smithi138 2023-10-01T09:09:12.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:12 smithi138 conmon[109959]: cluster 2023-10-01T09:09:11.192491+0000 mgr.y (mgr.26000) 59 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:12.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:12 smithi138 systemd[1]: Stopping Ceph mon.c for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:09:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:12 smithi190 conmon[100518]: cephadm 2023-10-01T09:09:10.794664+0000 mgr.y (mgr.26000) 57 : cephadm [INF] Upgrade: Updating mon.c 2023-10-01T09:09:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:12 smithi190 conmon[100518]: cephadm 2023-10-01T09:09:10.801523+0000 mgr.y (mgr.26000) 58 : cephadm [INF] Deploying daemon mon.c on smithi138 2023-10-01T09:09:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:12 smithi190 conmon[100518]: cluster 2023-10-01T09:09:11.192491+0000 mgr.y (mgr.26000) 59 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:12.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:12 smithi138 conmon[109959]: debug 2023-10-01T09:09:12.579+0000 7fbdaa3ce700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-01T09:09:12.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:12 smithi138 conmon[109959]: debug 2023-10-01T09:09:12.579+0000 7fbdaa3ce700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-10-01T09:09:13.540 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:13 smithi138 bash[192090]: 711c0763bf425596789306b1e0047d574ea94fe22b28af324a0eda0eef996adb 2023-10-01T09:09:13.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:13 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-01T09:09:13.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:13 smithi138 systemd[1]: Stopped Ceph mon.c for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:09:13.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:13 smithi138 systemd[1]: Starting Ceph mon.c for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:09:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: set uid:gid to 167:167 (ceph:ceph) 2023-10-01T09:09:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable), process ceph-mon, pid 7 2023-10-01T09:09:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: pidfile_write: ignore empty --pid-file 2023-10-01T09:09:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 bash[192449]: 9f7c99bda05dfe8573570335368be0f55d4719543769b2c4a3b2e11f401a7c54 2023-10-01T09:09:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: load: jerasure load: lrc 2023-10-01T09:09:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: RocksDB version: 7.9.2 2023-10-01T09:09:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Git sha 0 2023-10-01T09:09:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Compile date 2023-09-29 15:06:10 2023-10-01T09:09:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: DB SUMMARY 2023-10-01T09:09:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: DB Session ID: 7VZXVVYDKQT1SQB3GBTL 2023-10-01T09:09:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: CURRENT file: CURRENT 2023-10-01T09:09:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: IDENTITY file: IDENTITY 2023-10-01T09:09:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: MANIFEST file: MANIFEST-000005 size: 891 Bytes 2023-10-01T09:09:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000020.sst 2023-10-01T09:09:14.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000018.log size: 7465797 ; 2023-10-01T09:09:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.error_if_exists: 0 2023-10-01T09:09:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.create_if_missing: 0 2023-10-01T09:09:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.paranoid_checks: 1 2023-10-01T09:09:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-01T09:09:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-01T09:09:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-01T09:09:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.env: 0x560439dce720 2023-10-01T09:09:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.fs: PosixFileSystem 2023-10-01T09:09:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.info_log: 0x56043c049420 2023-10-01T09:09:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_file_opening_threads: 16 2023-10-01T09:09:14.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.statistics: (nil) 2023-10-01T09:09:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.use_fsync: 0 2023-10-01T09:09:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_log_file_size: 0 2023-10-01T09:09:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T09:09:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T09:09:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.keep_log_file_num: 1000 2023-10-01T09:09:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.recycle_log_file_num: 0 2023-10-01T09:09:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.allow_fallocate: 1 2023-10-01T09:09:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.allow_mmap_reads: 0 2023-10-01T09:09:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.allow_mmap_writes: 0 2023-10-01T09:09:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.use_direct_reads: 0 2023-10-01T09:09:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T09:09:14.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.create_missing_column_families: 0 2023-10-01T09:09:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.db_log_dir: 2023-10-01T09:09:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.wal_dir: 2023-10-01T09:09:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T09:09:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T09:09:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T09:09:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T09:09:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T09:09:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T09:09:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.advise_random_on_open: 1 2023-10-01T09:09:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.db_write_buffer_size: 0 2023-10-01T09:09:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.write_buffer_manager: 0x56043b2d8c80 2023-10-01T09:09:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T09:09:14.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T09:09:14.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T09:09:14.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.rate_limiter: (nil) 2023-10-01T09:09:14.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T09:09:14.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.wal_recovery_mode: 2 2023-10-01T09:09:14.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.enable_thread_tracking: 0 2023-10-01T09:09:14.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.enable_pipelined_write: 0 2023-10-01T09:09:14.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.unordered_write: 0 2023-10-01T09:09:14.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T09:09:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T09:09:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T09:09:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T09:09:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.row_cache: None 2023-10-01T09:09:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.wal_filter: None 2023-10-01T09:09:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T09:09:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.allow_ingest_behind: 0 2023-10-01T09:09:14.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.two_write_queues: 0 2023-10-01T09:09:14.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.manual_wal_flush: 0 2023-10-01T09:09:14.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.wal_compression: 0 2023-10-01T09:09:14.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.atomic_flush: 0 2023-10-01T09:09:14.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T09:09:14.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T09:09:14.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T09:09:14.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.log_readahead_size: 0 2023-10-01T09:09:14.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-01T09:09:14.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.best_efforts_recovery: 0 2023-10-01T09:09:14.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-01T09:09:14.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-01T09:09:14.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.allow_data_in_errors: 0 2023-10-01T09:09:14.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.db_host_id: __hostname__ 2023-10-01T09:09:14.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-01T09:09:14.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_background_jobs: 2 2023-10-01T09:09:14.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_background_compactions: -1 2023-10-01T09:09:14.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_subcompactions: 1 2023-10-01T09:09:14.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T09:09:14.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T09:09:14.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T09:09:14.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_total_wal_size: 0 2023-10-01T09:09:14.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T09:09:14.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T09:09:14.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T09:09:14.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T09:09:14.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_open_files: -1 2023-10-01T09:09:14.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.bytes_per_sync: 0 2023-10-01T09:09:14.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T09:09:14.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T09:09:14.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compaction_readahead_size: 0 2023-10-01T09:09:14.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_background_flushes: -1 2023-10-01T09:09:14.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Compression algorithms supported: 2023-10-01T09:09:14.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T09:09:14.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: kZSTD supported: 0 2023-10-01T09:09:14.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: kXpressCompression supported: 0 2023-10-01T09:09:14.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: kLZ4HCCompression supported: 1 2023-10-01T09:09:14.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: kZlibCompression supported: 1 2023-10-01T09:09:14.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: kSnappyCompression supported: 1 2023-10-01T09:09:14.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: kLZ4Compression supported: 1 2023-10-01T09:09:14.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: kBZip2Compression supported: 0 2023-10-01T09:09:14.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T09:09:14.331 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-01T09:09:14.331 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-10-01T09:09:14.331 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-01T09:09:14.331 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T09:09:14.331 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.merge_operator: 2023-10-01T09:09:14.331 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compaction_filter: None 2023-10-01T09:09:14.331 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compaction_filter_factory: None 2023-10-01T09:09:14.331 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.sst_partitioner_factory: None 2023-10-01T09:09:14.331 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T09:09:14.331 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T09:09:14.332 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56043c049520) 2023-10-01T09:09:14.332 INFO:journalctl@ceph.mon.c.smithi138.stdout: cache_index_and_filter_blocks: 1 2023-10-01T09:09:14.333 INFO:journalctl@ceph.mon.c.smithi138.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T09:09:14.333 INFO:journalctl@ceph.mon.c.smithi138.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T09:09:14.333 INFO:journalctl@ceph.mon.c.smithi138.stdout: pin_top_level_index_and_filter: 1 2023-10-01T09:09:14.333 INFO:journalctl@ceph.mon.c.smithi138.stdout: index_type: 0 2023-10-01T09:09:14.333 INFO:journalctl@ceph.mon.c.smithi138.stdout: data_block_index_type: 0 2023-10-01T09:09:14.333 INFO:journalctl@ceph.mon.c.smithi138.stdout: index_shortening: 1 2023-10-01T09:09:14.333 INFO:journalctl@ceph.mon.c.smithi138.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-01T09:09:14.333 INFO:journalctl@ceph.mon.c.smithi138.stdout: checksum: 4 2023-10-01T09:09:14.334 INFO:journalctl@ceph.mon.c.smithi138.stdout: no_block_cache: 0 2023-10-01T09:09:14.334 INFO:journalctl@ceph.mon.c.smithi138.stdout: block_cache: 0x56043b35b1f0 2023-10-01T09:09:14.334 INFO:journalctl@ceph.mon.c.smithi138.stdout: block_cache_name: BinnedLRUCache 2023-10-01T09:09:14.334 INFO:journalctl@ceph.mon.c.smithi138.stdout: block_cache_options: 2023-10-01T09:09:14.334 INFO:journalctl@ceph.mon.c.smithi138.stdout: capacity : 536870912 2023-10-01T09:09:14.334 INFO:journalctl@ceph.mon.c.smithi138.stdout: num_shard_bits : 4 2023-10-01T09:09:14.334 INFO:journalctl@ceph.mon.c.smithi138.stdout: strict_capacity_limit : 0 2023-10-01T09:09:14.334 INFO:journalctl@ceph.mon.c.smithi138.stdout: high_pri_pool_ratio: 0.000 2023-10-01T09:09:14.334 INFO:journalctl@ceph.mon.c.smithi138.stdout: block_cache_compressed: (nil) 2023-10-01T09:09:14.334 INFO:journalctl@ceph.mon.c.smithi138.stdout: persistent_cache: (nil) 2023-10-01T09:09:14.335 INFO:journalctl@ceph.mon.c.smithi138.stdout: block_size: 4096 2023-10-01T09:09:14.335 INFO:journalctl@ceph.mon.c.smithi138.stdout: block_size_deviation: 10 2023-10-01T09:09:14.335 INFO:journalctl@ceph.mon.c.smithi138.stdout: block_restart_interval: 16 2023-10-01T09:09:14.335 INFO:journalctl@ceph.mon.c.smithi138.stdout: index_block_restart_interval: 1 2023-10-01T09:09:14.335 INFO:journalctl@ceph.mon.c.smithi138.stdout: metadata_block_size: 4096 2023-10-01T09:09:14.335 INFO:journalctl@ceph.mon.c.smithi138.stdout: partition_filters: 0 2023-10-01T09:09:14.335 INFO:journalctl@ceph.mon.c.smithi138.stdout: use_delta_encoding: 1 2023-10-01T09:09:14.335 INFO:journalctl@ceph.mon.c.smithi138.stdout: filter_policy: bloomfilter 2023-10-01T09:09:14.335 INFO:journalctl@ceph.mon.c.smithi138.stdout: whole_key_filtering: 1 2023-10-01T09:09:14.336 INFO:journalctl@ceph.mon.c.smithi138.stdout: verify_compression: 0 2023-10-01T09:09:14.336 INFO:journalctl@ceph.mon.c.smithi138.stdout: read_amp_bytes_per_bit: 0 2023-10-01T09:09:14.336 INFO:journalctl@ceph.mon.c.smithi138.stdout: format_version: 5 2023-10-01T09:09:14.336 INFO:journalctl@ceph.mon.c.smithi138.stdout: enable_index_compression: 1 2023-10-01T09:09:14.336 INFO:journalctl@ceph.mon.c.smithi138.stdout: block_align: 0 2023-10-01T09:09:14.336 INFO:journalctl@ceph.mon.c.smithi138.stdout: max_auto_readahead_size: 262144 2023-10-01T09:09:14.336 INFO:journalctl@ceph.mon.c.smithi138.stdout: prepopulate_block_cache: 0 2023-10-01T09:09:14.336 INFO:journalctl@ceph.mon.c.smithi138.stdout: initial_auto_readahead_size: 8192 2023-10-01T09:09:14.336 INFO:journalctl@ceph.mon.c.smithi138.stdout: num_file_reads_for_auto_readahead: 2 2023-10-01T09:09:14.336 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.write_buffer_size: 33554432 2023-10-01T09:09:14.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_write_buffer_number: 2 2023-10-01T09:09:14.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compression: NoCompression 2023-10-01T09:09:14.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.bottommost_compression: Disabled 2023-10-01T09:09:14.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.prefix_extractor: nullptr 2023-10-01T09:09:14.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T09:09:14.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.num_levels: 7 2023-10-01T09:09:14.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T09:09:14.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T09:09:14.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T09:09:14.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T09:09:14.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T09:09:14.337 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T09:09:14.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T09:09:14.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T09:09:14.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-01T09:09:14.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T09:09:14.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-01T09:09:14.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-01T09:09:14.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T09:09:14.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compression_opts.level: 32767 2023-10-01T09:09:14.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compression_opts.strategy: 0 2023-10-01T09:09:14.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T09:09:14.338 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T09:09:14.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-01T09:09:14.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-01T09:09:14.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compression_opts.enabled: false 2023-10-01T09:09:14.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-01T09:09:14.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T09:09:14.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T09:09:14.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T09:09:14.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.target_file_size_base: 67108864 2023-10-01T09:09:14.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T09:09:14.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T09:09:14.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T09:09:14.339 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T09:09:14.340 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T09:09:14.340 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T09:09:14.340 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T09:09:14.340 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T09:09:14.340 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T09:09:14.340 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T09:09:14.340 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T09:09:14.340 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T09:09:14.340 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T09:09:14.340 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-01T09:09:14.340 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.arena_block_size: 1048576 2023-10-01T09:09:14.340 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T09:09:14.341 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T09:09:14.341 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.disable_auto_compactions: 0 2023-10-01T09:09:14.341 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T09:09:14.341 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T09:09:14.341 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T09:09:14.341 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T09:09:14.341 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T09:09:14.341 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T09:09:14.341 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T09:09:14.341 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T09:09:14.341 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T09:09:14.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T09:09:14.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.table_properties_collectors: 2023-10-01T09:09:14.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.inplace_update_support: 0 2023-10-01T09:09:14.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T09:09:14.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T09:09:14.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T09:09:14.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T09:09:14.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.bloom_locality: 0 2023-10-01T09:09:14.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.max_successive_merges: 0 2023-10-01T09:09:14.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T09:09:14.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.paranoid_file_checks: 0 2023-10-01T09:09:14.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.force_consistency_checks: 1 2023-10-01T09:09:14.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.report_bg_io_stats: 0 2023-10-01T09:09:14.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.ttl: 2592000 2023-10-01T09:09:14.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T09:09:14.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-01T09:09:14.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-01T09:09:14.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.enable_blob_files: false 2023-10-01T09:09:14.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.min_blob_size: 0 2023-10-01T09:09:14.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.blob_file_size: 268435456 2023-10-01T09:09:14.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-01T09:09:14.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-01T09:09:14.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-01T09:09:14.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-01T09:09:14.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-01T09:09:14.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.blob_file_starting_level: 0 2023-10-01T09:09:14.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-01T09:09:14.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 20.sst 2023-10-01T09:09:14.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-01T09:09:14.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 22, last_sequence is 12068, log_number is 18,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T09:09:14.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 18 2023-10-01T09:09:14.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: cb7d02c5-eb21-420e-a8a1-dc748259a6c8 2023-10-01T09:09:14.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696151354114922, "job": 1, "event": "recovery_started", "wal_files": [18]} 2023-10-01T09:09:14.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #18 mode 2 2023-10-01T09:09:14.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 systemd[1]: Started Ceph mon.c for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:09:14.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696151354130372, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 23, "file_size": 4866325, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12070, "largest_seqno": 13295, "table_properties": {"data_size": 4859099, "index_size": 5311, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 901, "raw_key_size": 6798, "raw_average_key_size": 19, "raw_value_size": 4848674, "raw_average_value_size": 13813, "num_data_blocks": 232, "num_entries": 351, "num_filter_entries": 351, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696151354, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "cb7d02c5-eb21-420e-a8a1-dc748259a6c8", "db_session_id": "7VZXVVYDKQT1SQB3GBTL", "orig_file_number": 23, "seqno_to_time_mapping": "N/A"}} 2023-10-01T09:09:14.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696151354130471, "job": 1, "event": "recovery_finished"} 2023-10-01T09:09:14.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: [db/version_set.cc:5047] Creating manifest 25 2023-10-01T09:09:14.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-01T09:09:14.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000018.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-01T09:09:14.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56043b3f6000 2023-10-01T09:09:14.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: DB pointer 0x56043b3e4000 2023-10-01T09:09:14.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-01T09:09:14.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-01T09:09:14.345 INFO:journalctl@ceph.mon.c.smithi138.stdout: ** DB Stats ** 2023-10-01T09:09:14.346 INFO:journalctl@ceph.mon.c.smithi138.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T09:09:14.346 INFO:journalctl@ceph.mon.c.smithi138.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T09:09:14.346 INFO:journalctl@ceph.mon.c.smithi138.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T09:09:14.346 INFO:journalctl@ceph.mon.c.smithi138.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T09:09:14.346 INFO:journalctl@ceph.mon.c.smithi138.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T09:09:14.346 INFO:journalctl@ceph.mon.c.smithi138.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T09:09:14.346 INFO:journalctl@ceph.mon.c.smithi138.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T09:09:14.346 INFO:journalctl@ceph.mon.c.smithi138.stdout: 2023-10-01T09:09:14.346 INFO:journalctl@ceph.mon.c.smithi138.stdout: ** Compaction Stats [default] ** 2023-10-01T09:09:14.346 INFO:journalctl@ceph.mon.c.smithi138.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T09:09:14.346 INFO:journalctl@ceph.mon.c.smithi138.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-01T09:09:14.347 INFO:journalctl@ceph.mon.c.smithi138.stdout: L0 1/0 4.64 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 741.1 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-10-01T09:09:14.347 INFO:journalctl@ceph.mon.c.smithi138.stdout: L6 1/0 23.87 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-01T09:09:14.347 INFO:journalctl@ceph.mon.c.smithi138.stdout: Sum 2/0 28.51 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 741.1 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-10-01T09:09:14.347 INFO:journalctl@ceph.mon.c.smithi138.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 741.1 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-10-01T09:09:14.347 INFO:journalctl@ceph.mon.c.smithi138.stdout: 2023-10-01T09:09:14.347 INFO:journalctl@ceph.mon.c.smithi138.stdout: ** Compaction Stats [default] ** 2023-10-01T09:09:14.347 INFO:journalctl@ceph.mon.c.smithi138.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T09:09:14.347 INFO:journalctl@ceph.mon.c.smithi138.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T09:09:14.347 INFO:journalctl@ceph.mon.c.smithi138.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 741.1 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-10-01T09:09:14.347 INFO:journalctl@ceph.mon.c.smithi138.stdout: 2023-10-01T09:09:14.347 INFO:journalctl@ceph.mon.c.smithi138.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-01T09:09:14.347 INFO:journalctl@ceph.mon.c.smithi138.stdout: 2023-10-01T09:09:14.348 INFO:journalctl@ceph.mon.c.smithi138.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T09:09:14.348 INFO:journalctl@ceph.mon.c.smithi138.stdout: Flush(GB): cumulative 0.005, interval 0.005 2023-10-01T09:09:14.348 INFO:journalctl@ceph.mon.c.smithi138.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T09:09:14.348 INFO:journalctl@ceph.mon.c.smithi138.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T09:09:14.348 INFO:journalctl@ceph.mon.c.smithi138.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T09:09:14.348 INFO:journalctl@ceph.mon.c.smithi138.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-01T09:09:14.348 INFO:journalctl@ceph.mon.c.smithi138.stdout: Cumulative compaction: 0.00 GB write, 251.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T09:09:14.348 INFO:journalctl@ceph.mon.c.smithi138.stdout: Interval compaction: 0.00 GB write, 251.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T09:09:14.348 INFO:journalctl@ceph.mon.c.smithi138.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T09:09:14.348 INFO:journalctl@ceph.mon.c.smithi138.stdout: Block cache BinnedLRUCache@0x56043b35b1f0#7 capacity: 512.00 MB usage: 7.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2023-10-01T09:09:14.348 INFO:journalctl@ceph.mon.c.smithi138.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.03 KB,0.000196695%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2023-10-01T09:09:14.348 INFO:journalctl@ceph.mon.c.smithi138.stdout: 2023-10-01T09:09:14.349 INFO:journalctl@ceph.mon.c.smithi138.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-01T09:09:14.349 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: starting mon.c rank 2 at public addrs [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] at bind addrs [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T09:09:14.349 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: mon.c@-1(???) e3 preinit fsid 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T09:09:14.349 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: mon.c@-1(???).mds e7 new map 2023-10-01T09:09:14.349 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: mon.c@-1(???).mds e7 print_map 2023-10-01T09:09:14.349 INFO:journalctl@ceph.mon.c.smithi138.stdout: e7 2023-10-01T09:09:14.349 INFO:journalctl@ceph.mon.c.smithi138.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-01T09:09:14.349 INFO:journalctl@ceph.mon.c.smithi138.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T09:09:14.349 INFO:journalctl@ceph.mon.c.smithi138.stdout: legacy client fscid: 1 2023-10-01T09:09:14.349 INFO:journalctl@ceph.mon.c.smithi138.stdout: 2023-10-01T09:09:14.349 INFO:journalctl@ceph.mon.c.smithi138.stdout: Filesystem 'foo' (1) 2023-10-01T09:09:14.349 INFO:journalctl@ceph.mon.c.smithi138.stdout: fs_name foo 2023-10-01T09:09:14.350 INFO:journalctl@ceph.mon.c.smithi138.stdout: epoch 5 2023-10-01T09:09:14.350 INFO:journalctl@ceph.mon.c.smithi138.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-01T09:09:14.350 INFO:journalctl@ceph.mon.c.smithi138.stdout: created 2023-10-01T09:03:03.201740+0000 2023-10-01T09:09:14.350 INFO:journalctl@ceph.mon.c.smithi138.stdout: modified 2023-10-01T09:03:09.223385+0000 2023-10-01T09:09:14.350 INFO:journalctl@ceph.mon.c.smithi138.stdout: tableserver 0 2023-10-01T09:09:14.350 INFO:journalctl@ceph.mon.c.smithi138.stdout: root 0 2023-10-01T09:09:14.350 INFO:journalctl@ceph.mon.c.smithi138.stdout: session_timeout 60 2023-10-01T09:09:14.350 INFO:journalctl@ceph.mon.c.smithi138.stdout: session_autoclose 300 2023-10-01T09:09:14.350 INFO:journalctl@ceph.mon.c.smithi138.stdout: max_file_size 1099511627776 2023-10-01T09:09:14.350 INFO:journalctl@ceph.mon.c.smithi138.stdout: required_client_features {} 2023-10-01T09:09:14.350 INFO:journalctl@ceph.mon.c.smithi138.stdout: last_failure 0 2023-10-01T09:09:14.350 INFO:journalctl@ceph.mon.c.smithi138.stdout: last_failure_osd_epoch 0 2023-10-01T09:09:14.351 INFO:journalctl@ceph.mon.c.smithi138.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T09:09:14.351 INFO:journalctl@ceph.mon.c.smithi138.stdout: max_mds 1 2023-10-01T09:09:14.351 INFO:journalctl@ceph.mon.c.smithi138.stdout: in 0 2023-10-01T09:09:14.351 INFO:journalctl@ceph.mon.c.smithi138.stdout: up {0=14637} 2023-10-01T09:09:14.351 INFO:journalctl@ceph.mon.c.smithi138.stdout: failed 2023-10-01T09:09:14.351 INFO:journalctl@ceph.mon.c.smithi138.stdout: damaged 2023-10-01T09:09:14.351 INFO:journalctl@ceph.mon.c.smithi138.stdout: stopped 2023-10-01T09:09:14.351 INFO:journalctl@ceph.mon.c.smithi138.stdout: data_pools [4] 2023-10-01T09:09:14.351 INFO:journalctl@ceph.mon.c.smithi138.stdout: metadata_pool 3 2023-10-01T09:09:14.351 INFO:journalctl@ceph.mon.c.smithi138.stdout: inline_data disabled 2023-10-01T09:09:14.351 INFO:journalctl@ceph.mon.c.smithi138.stdout: balancer 2023-10-01T09:09:14.351 INFO:journalctl@ceph.mon.c.smithi138.stdout: bal_rank_mask -1 2023-10-01T09:09:14.352 INFO:journalctl@ceph.mon.c.smithi138.stdout: standby_count_wanted 1 2023-10-01T09:09:14.352 INFO:journalctl@ceph.mon.c.smithi138.stdout: [mds.foo.smithi138.brhzod{0:14637} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.138:6834/416108704,v1:172.21.15.138:6835/416108704] compat {c=[1],r=[1],i=[7ff]}] 2023-10-01T09:09:14.352 INFO:journalctl@ceph.mon.c.smithi138.stdout: 2023-10-01T09:09:14.352 INFO:journalctl@ceph.mon.c.smithi138.stdout: 2023-10-01T09:09:14.352 INFO:journalctl@ceph.mon.c.smithi138.stdout: Standby daemons: 2023-10-01T09:09:14.352 INFO:journalctl@ceph.mon.c.smithi138.stdout: 2023-10-01T09:09:14.352 INFO:journalctl@ceph.mon.c.smithi138.stdout: [mds.foo.smithi190.weynjc{-1:24505} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.190:6832/4158685175,v1:172.21.15.190:6833/4158685175] compat {c=[1],r=[1],i=[7ff]}] 2023-10-01T09:09:14.352 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: mon.c@-1(???).osd e85 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-01T09:09:14.352 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: mon.c@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T09:09:14.352 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: mon.c@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T09:09:14.352 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: mon.c@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T09:09:14.353 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-10-01T09:09:14.353 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: expand_channel_meta expand map: {default=false} 2023-10-01T09:09:14.353 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: expand_channel_meta from 'false' to 'false' 2023-10-01T09:09:14.353 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: expand_channel_meta expanded map: {default=false} 2023-10-01T09:09:14.353 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: expand_channel_meta expand map: {default=info} 2023-10-01T09:09:14.353 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: expand_channel_meta from 'info' to 'info' 2023-10-01T09:09:14.353 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: expand_channel_meta expanded map: {default=info} 2023-10-01T09:09:14.353 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: expand_channel_meta expand map: {default=daemon} 2023-10-01T09:09:14.353 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-01T09:09:14.353 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: expand_channel_meta expanded map: {default=daemon} 2023-10-01T09:09:14.353 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: expand_channel_meta expand map: {default=debug} 2023-10-01T09:09:14.353 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: expand_channel_meta from 'debug' to 'debug' 2023-10-01T09:09:14.354 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:14 smithi138 ceph-mon[192513]: expand_channel_meta expanded map: {default=debug} 2023-10-01T09:09:15.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:09:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:09:14] "GET /metrics HTTP/1.1" 200 33781 "" "Prometheus/2.33.4" 2023-10-01T09:09:19.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:09:18 smithi190 conmon[151158]: ts=2023-10-01T09:09:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:09:20.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01T09:09:13.192899+0000 mgr.y (mgr.26000) 60 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:20.648 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: 2023-10-01T09:09:20.648 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01T09:09:14.242206+0000 mon.b (mon.1) 109 : cluster [INF] mon.b calling monitor election 2023-10-01T09:09:20.648 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01T09:09:14.245779+0000 mon.a (mon.0) 89 : cluster 2023-10-01T09:09:20.648 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: [INF] mon.a calling monitor election 2023-10-01T09:09:20.648 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01T09:09:15.193359+0000 mgr.y ( 2023-10-01T09:09:20.648 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: mgr.26000) 61 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:20.648 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01T09:09:17 2023-10-01T09:09:20.648 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: .193827+0000 mgr.y (mgr.26000) 62 : cluster [DBG] 2023-10-01T09:09:20.648 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:20.648 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01T09:09:20.649 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: 2023-10-01T09:09:19.194388+0000 mgr.y (mgr.26000 2023-10-01T09:09:20.649 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: ) 63 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:20.649 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01T09 2023-10-01T09:09:20.649 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: :09:19.246997+0000 mon.a (mon.0) 90 : 2023-10-01T09:09:20.649 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-01T09:09:20.649 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01T09:09 2023-10-01T09:09:20.649 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: :19.254089+0000 mon.a (mon.0) 91 : cluster 2023-10-01T09:09:20.649 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:09:20.649 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01T09:09: 2023-10-01T09:09:20.649 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: 19.267136+0000 mon.a (mon.0) 92 : cluster 2023-10-01T09:09:20.650 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:09:20.650 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01T09:09:19. 2023-10-01T09:09:20.650 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: 267190+0000 mon.a (mon.0) 93 : cluster [DBG] 2023-10-01T09:09:20.650 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:09:20.650 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01T09:09:20.650 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: 2023-10-01T09:09:19.267546+0000 mon.a (mon. 2023-10-01T09:09:20.650 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: 0) 94 : cluster [DBG] mgrmap e34: y(active, since 84s), standbys: x 2023-10-01T09:09:20.650 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01 2023-10-01T09:09:20.650 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: T09:09:19.267805+0000 mon.a (mon.0 2023-10-01T09:09:20.650 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: ) 95 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-01T09:09:20.650 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01T09:09 2023-10-01T09:09:20.650 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: :19.272841+0000 mon.a (mon.0) 96 : cluster 2023-10-01T09:09:20.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-01T09:09:20.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01T09:09:19. 2023-10-01T09:09:20.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: 272864+0000 mon.a (mon.0) 97 : cluster [WRN] 2023-10-01T09:09:20.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-01T09:09:20.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: cluster 2023-10-01T09:09:19.272873 2023-10-01T09:09:20.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: +0000 mon.a (mon.0) 98 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] is down (out of quorum) 2023-10-01T09:09:20.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: 2023-10-01T09:09:20.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: audit 2023-10-01T09:09:20.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: 2023-10-01T09:09:19.277387+0000 mon.a (mon.0 2023-10-01T09:09:20.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: ) 99 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:20.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: audit 2023-10-01T09:09:20.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: 2023-10-01T09:09:19.282286+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:20.652 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: audit 2023-10-01T 2023-10-01T09:09:20.652 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: 09:09:19.283708+0000 mon.a (mon.0 2023-10-01T09:09:20.653 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:20 smithi190 conmon[100518]: ) 101 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:20.819 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 2 2023-10-01T09:09:21.524 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch ps' 2023-10-01T09:09:21.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: cluster 2023-10-01T09:09:14.241524+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-01T09:09:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: 2023-10-01T09:09:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: cluster 2023-10-01T09:09:20.259544+0000 mon.b (mon.1) 110 : cluster [INF] mon.b calling monitor election 2023-10-01T09:09:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: cluster 2023-10-01T09:09:20.261328+0000 mon.a (mon 2023-10-01T09:09:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: .0) 102 : cluster [INF] mon.a calling monitor election 2023-10-01T09:09:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: cluster 2023-10-01T09:09: 2023-10-01T09:09:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: 20.272919+0000 mon.a (mon.0) 103 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T09:09:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: cluster 2023-10-01 2023-10-01T09:09:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: T09:09:20.280033+0000 mon.a (mon.0 2023-10-01T09:09:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: ) 104 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:09:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: cluster 2023-10-01T09 2023-10-01T09:09:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: :09:20.280685+0000 mon.a (mon.0) 105 2023-10-01T09:09:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: : cluster [DBG] fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:09:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: cluster 2023-10-01T09:09 2023-10-01T09:09:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: :20.280714+0000 mon.a (mon.0) 106 : cluster 2023-10-01T09:09:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: [DBG] osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:09:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: cluster 2023-10-01T09:09:20 2023-10-01T09:09:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: .281045+0000 mon.a (mon.0) 107 : cluster [DBG] 2023-10-01T09:09:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: mgrmap e34: y(active, since 85s), standbys: x 2023-10-01T09:09:21.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: cluster 2023-10-01T09:09:20. 2023-10-01T09:09:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: 281176+0000 mon.a (mon.0) 108 : cluster [INF] 2023-10-01T09:09:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-01T09:09:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: cluster 2023-10-01T09:09:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: 2023-10-01T09:09:20.281192+0000 mon.a (mon. 2023-10-01T09:09:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: 0) 109 : cluster [INF] Cluster is now healthy 2023-10-01T09:09:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: cluster 2023-10-01T09: 2023-10-01T09:09:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: 09:20.293488+0000 mon.a (mon.0) 110 : cluster 2023-10-01T09:09:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: [INF] overall HEALTH_OK 2023-10-01T09:09:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: audit 2023-10-01T09:09:20.804489 2023-10-01T09:09:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: +0000 mon.a (mon.0) 111 : audit [DBG] from='client.? 172.21.15.138:0/898293204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:09:21.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:21 smithi190 conmon[100518]: 2023-10-01T09:09:22.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:22 smithi190 conmon[100518]: cluster 2023-10-01T09:09:21.194838+0000 mgr.y (mgr.26000) 64 : cluster [DBG] 2023-10-01T09:09:22.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:22 smithi190 conmon[100518]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:24.009 INFO:teuthology.orchestra.run.smithi138.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-01T09:09:24.009 INFO:teuthology.orchestra.run.smithi138.stdout:alertmanager.a smithi138 *:9093,9094 running (2m) - 7m 27.1M - 0.23.0 ba2b418f427c 8debc681d94e 2023-10-01T09:09:24.009 INFO:teuthology.orchestra.run.smithi138.stdout:grafana.a smithi190 *:3000 running (2m) - 6m 55.8M - 8.3.5 dad864ee21e9 58ea19a8c949 2023-10-01T09:09:24.009 INFO:teuthology.orchestra.run.smithi138.stdout:mds.foo.smithi138.brhzod smithi138 running (6m) - 6m 18.6M - 16.2.14-44-geeb13ac9 94a6c4498547 555d8c2cfe25 2023-10-01T09:09:24.010 INFO:teuthology.orchestra.run.smithi138.stdout:mds.foo.smithi190.weynjc smithi190 running (6m) - 6m 18.0M - 16.2.14-44-geeb13ac9 94a6c4498547 881fea906eff 2023-10-01T09:09:24.010 INFO:teuthology.orchestra.run.smithi138.stdout:mgr.x smithi190 *:8443,9283,8765 running (109s) - 10m 447M - 18.2.0-497-gb5682219 a3fc96fed5ac 674e1140402f 2023-10-01T09:09:24.010 INFO:teuthology.orchestra.run.smithi138.stdout:mgr.y smithi138 *:8443,9283,8765 running (2m) - 12m 499M - 18.2.0-497-gb5682219 a3fc96fed5ac 97da5dac6a84 2023-10-01T09:09:24.010 INFO:teuthology.orchestra.run.smithi138.stdout:mon.a smithi138 running (97s) - 12m 80.9M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 4f0e28d17710 2023-10-01T09:09:24.010 INFO:teuthology.orchestra.run.smithi138.stdout:mon.b smithi190 running (11m) - 11m 103M 2048M 16.2.14-44-geeb13ac9 94a6c4498547 520f260f067c 2023-10-01T09:09:24.010 INFO:teuthology.orchestra.run.smithi138.stdout:mon.c smithi138 running (10s) - 11m 29.6M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 9f7c99bda05d 2023-10-01T09:09:24.010 INFO:teuthology.orchestra.run.smithi138.stdout:node-exporter.a smithi138 *:9100 running (3m) - 7m 24.8M - 1.3.1 1dbe0e931976 bd644aa2145b 2023-10-01T09:09:24.010 INFO:teuthology.orchestra.run.smithi138.stdout:node-exporter.b smithi190 *:9100 running (2m) - 7m 25.7M - 1.3.1 1dbe0e931976 c08f7a49a78d 2023-10-01T09:09:24.010 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0 smithi138 running (10m) - 10m 76.5M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 7636a677953c 2023-10-01T09:09:24.010 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1 smithi138 running (10m) - 10m 100M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 a510d44fb2c6 2023-10-01T09:09:24.011 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2 smithi138 running (9m) - 9m 66.5M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 8a8389da7887 2023-10-01T09:09:24.011 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3 smithi138 running (9m) - 9m 67.0M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 c3ba3a01539d 2023-10-01T09:09:24.011 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4 smithi190 running (9m) - 9m 72.1M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 cda54a787614 2023-10-01T09:09:24.011 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5 smithi190 running (8m) - 8m 89.4M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 4ac584c1fb7d 2023-10-01T09:09:24.011 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6 smithi190 running (8m) - 8m 71.7M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 fa45a1c37077 2023-10-01T09:09:24.011 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7 smithi190 running (7m) - 7m 93.2M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 c925b7fc3562 2023-10-01T09:09:24.011 INFO:teuthology.orchestra.run.smithi138.stdout:prometheus.a smithi190 *:9095 running (114s) - 7m 59.7M - 2.33.4 514e6a882f6e bbdb076a24d4 2023-10-01T09:09:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:23 smithi190 conmon[100518]: audit 2023-10-01T09:09:22.659458+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:23 smithi190 conmon[100518]: audit 2023-10-01T09:09:22.664301+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:23 smithi190 conmon[100518]: cluster 2023-10-01T09:09:23.195191+0000 mgr.y (mgr.26000) 65 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:23 smithi190 conmon[100518]: audit 2023-10-01T09:09:23.553758+0000 mon.a (mon.0) 2023-10-01T09:09:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:23 smithi190 conmon[100518]: 114 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:23 smithi190 conmon[100518]: audit 2023-10-01T09:09 2023-10-01T09:09:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:23 smithi190 conmon[100518]: :23.559631+0000 mon.a (mon.0) 115 : audit 2023-10-01T09:09:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:23 smithi190 conmon[100518]: [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:24.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:09:24 smithi138 conmon[178630]: 2023-10-01T09:09:24.139+0000 7fa004e48700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-10-01T09:09:25.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:09:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:09:24] "GET /metrics HTTP/1.1" 200 33781 "" "Prometheus/2.33.4" 2023-10-01T09:09:25.419 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade pause' 2023-10-01T09:09:25.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:25 smithi190 conmon[100518]: audit 2023-10-01T09:09:24.001447+0000 mgr.y (mgr.26000) 66 : audit [DBG] from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:09:26.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:26 smithi190 conmon[100518]: cluster 2023-10-01T09:09:25.195670+0000 mgr.y (mgr.26000) 67 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:26.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:26 smithi190 conmon[100518]: audit 2023-10-01T09:09:25.241814+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:27.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:27 smithi190 conmon[100518]: cluster 2023-10-01T09:09:26.153975+0000 mon.a (mon.0) 117 : cluster [DBG] mgrmap e35: y(active, since 90s), standbys: x 2023-10-01T09:09:27.773 INFO:teuthology.orchestra.run.smithi138.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:09:28.485 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'sleep 60' 2023-10-01T09:09:28.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:28 smithi190 conmon[100518]: cluster 2023-10-01T09:09:27.196166+0000 mgr.y (mgr.26000) 68 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:28.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:28 smithi190 conmon[100518]: audit 2023-10-01T09:09:27.235917+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:28.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:28 smithi190 conmon[100518]: audit 2023-10-01T09:09:27.241367+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:28.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:28 smithi190 conmon[100518]: audit 2023-10-01T09:09:27.242235+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:28.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:28 smithi190 conmon[100518]: audit 2023-10-01T09:09:27.242945+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:09:28.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:28 smithi190 conmon[100518]: audit 2023-10-01T09:09:27.247268+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:28.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:28 smithi190 conmon[100518]: audit 2023-10-01T09:09:27.388282+0000 mon.a (mon.0) 123 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:28.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:28 smithi190 conmon[100518]: audit 2023-10-01T09:09:27.390563+0000 mon.a (mon.0) 124 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:09:28.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:28 smithi190 conmon[100518]: audit 2023-10-01T09:09:27.391540+0000 mon.a (mon.0) 125 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T09:09:28.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:28 smithi190 conmon[100518]: audit 2023-10-01T09:09:27.392097+0000 mon.a (mon.0) 126 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-01T09:09:28.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:28 smithi190 conmon[100518]: audit 2023-10-01T09:09:27.772283+0000 mon.a 2023-10-01T09:09:28.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:28 smithi190 conmon[100518]: (mon.0) 127 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:29.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:09:28 smithi190 conmon[151158]: ts=2023-10-01T09:09:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:09:29.551 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: cephadm 2023-10-01T09:09:27.392541+0000 mgr.y (mgr 2023-10-01T09:09:29.551 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: .26000) 69 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-10-01T09:09:29.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: audit 2023-10-01T09:09:27.767843+0000 mgr.y (mgr.26000) 70 : audit [DBG] from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:09:29.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: cephadm 2023-10-01T09:09:27.768236+0000 mgr.y (mgr.26000) 71 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:09:29.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: audit 2023-10-01T09:09:28 2023-10-01T09:09:29.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: .763816+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:29.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: audit 2023-10-01T09: 2023-10-01T09:09:29.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: 09:28.764701+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:29.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: 2023-10-01T09:09:29.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: audit 2023-10-01T09:09:29.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: 2023-10-01T09:09:28.765326+0000 mon.a (mon 2023-10-01T09:09:29.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: .0) 130 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:29.553 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: audit 2023-10-01 2023-10-01T09:09:29.553 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: T09:09:28.765870+0000 mon.a (mon 2023-10-01T09:09:29.553 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:29 smithi190 conmon[100518]: .0) 131 : audit [DBG] from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:30.240 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:30 smithi190 systemd[1]: Stopping Ceph mon.b for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:09:30.240 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:30 smithi190 conmon[100518]: cephadm 2023-10-01T09:09:28.758925+0000 mgr.y (mgr.26000 2023-10-01T09:09:30.240 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:30 smithi190 conmon[100518]: ) 72 : cephadm [INF] Upgrade: Updating mon.b 2023-10-01T09:09:30.240 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:30 smithi190 conmon[100518]: cephadm 2023-10-01T09:09:28.766590+0000 mgr.y (mgr.26000) 73 : cephadm 2023-10-01T09:09:30.240 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:30 smithi190 conmon[100518]: [INF] Deploying daemon mon.b on smithi190 2023-10-01T09:09:30.240 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:30 smithi190 conmon[100518]: cluster 2023-10-01T 2023-10-01T09:09:30.241 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:30 smithi190 conmon[100518]: 09:09:29.196517+0000 mgr.y (mgr 2023-10-01T09:09:30.241 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:30 smithi190 conmon[100518]: .26000) 74 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:30.241 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:30 smithi190 conmon[100518]: 2023-10-01T09:09:30.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:30 smithi190 conmon[100518]: debug 2023-10-01T09:09:30.375+0000 7fabaa675700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-01T09:09:30.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:30 smithi190 conmon[100518]: debug 2023-10-01T09:09:30.375+0000 7fabaa675700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-10-01T09:09:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:30 smithi190 bash[160005]: 520f260f067cbf49006b6a61421850c9196ae7274f1d7372925efbde02bc38b9 2023-10-01T09:09:31.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.b.service: Succeeded. 2023-10-01T09:09:31.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 systemd[1]: Stopped Ceph mon.b for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:09:31.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 systemd[1]: Starting Ceph mon.b for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:09:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: set uid:gid to 167:167 (ceph:ceph) 2023-10-01T09:09:32.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable), process ceph-mon, pid 7 2023-10-01T09:09:32.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: pidfile_write: ignore empty --pid-file 2023-10-01T09:09:32.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 bash[160370]: c0ac51d9b5e0a27314b8d74e1f40200b80d08c93cf95152b2610076c03645540 2023-10-01T09:09:32.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: load: jerasure load: lrc 2023-10-01T09:09:32.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: RocksDB version: 7.9.2 2023-10-01T09:09:32.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Git sha 0 2023-10-01T09:09:32.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Compile date 2023-09-29 15:06:10 2023-10-01T09:09:32.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: DB SUMMARY 2023-10-01T09:09:32.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: DB Session ID: W5E60NOAIB5UH5S9HMAJ 2023-10-01T09:09:32.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: CURRENT file: CURRENT 2023-10-01T09:09:32.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: IDENTITY file: IDENTITY 2023-10-01T09:09:32.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: MANIFEST file: MANIFEST-000005 size: 891 Bytes 2023-10-01T09:09:32.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000020.sst 2023-10-01T09:09:32.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000018.log size: 9102769 ; 2023-10-01T09:09:32.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.error_if_exists: 0 2023-10-01T09:09:32.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.create_if_missing: 0 2023-10-01T09:09:32.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.paranoid_checks: 1 2023-10-01T09:09:32.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.flush_verify_memtable_count: 1 2023-10-01T09:09:32.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-01T09:09:32.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-01T09:09:32.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.env: 0x55c75d4ce720 2023-10-01T09:09:32.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.fs: PosixFileSystem 2023-10-01T09:09:32.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.info_log: 0x55c760435420 2023-10-01T09:09:32.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_file_opening_threads: 16 2023-10-01T09:09:32.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.statistics: (nil) 2023-10-01T09:09:32.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.use_fsync: 0 2023-10-01T09:09:32.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_log_file_size: 0 2023-10-01T09:09:32.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T09:09:32.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T09:09:32.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.keep_log_file_num: 1000 2023-10-01T09:09:32.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.recycle_log_file_num: 0 2023-10-01T09:09:32.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.allow_fallocate: 1 2023-10-01T09:09:32.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.allow_mmap_reads: 0 2023-10-01T09:09:32.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.allow_mmap_writes: 0 2023-10-01T09:09:32.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.use_direct_reads: 0 2023-10-01T09:09:32.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T09:09:32.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.create_missing_column_families: 0 2023-10-01T09:09:32.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.db_log_dir: 2023-10-01T09:09:32.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.wal_dir: 2023-10-01T09:09:32.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T09:09:32.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T09:09:32.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T09:09:32.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T09:09:32.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T09:09:32.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T09:09:32.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.advise_random_on_open: 1 2023-10-01T09:09:32.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.db_write_buffer_size: 0 2023-10-01T09:09:32.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.write_buffer_manager: 0x55c75f6c4c80 2023-10-01T09:09:32.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T09:09:32.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T09:09:32.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T09:09:32.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.rate_limiter: (nil) 2023-10-01T09:09:32.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T09:09:32.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.wal_recovery_mode: 2 2023-10-01T09:09:32.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.enable_thread_tracking: 0 2023-10-01T09:09:32.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.enable_pipelined_write: 0 2023-10-01T09:09:32.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.unordered_write: 0 2023-10-01T09:09:32.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T09:09:32.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T09:09:32.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T09:09:32.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T09:09:32.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.row_cache: None 2023-10-01T09:09:32.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.wal_filter: None 2023-10-01T09:09:32.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T09:09:32.064 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.allow_ingest_behind: 0 2023-10-01T09:09:32.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.two_write_queues: 0 2023-10-01T09:09:32.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.manual_wal_flush: 0 2023-10-01T09:09:32.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.wal_compression: 0 2023-10-01T09:09:32.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.atomic_flush: 0 2023-10-01T09:09:32.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T09:09:32.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T09:09:32.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T09:09:32.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.log_readahead_size: 0 2023-10-01T09:09:32.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-01T09:09:32.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.best_efforts_recovery: 0 2023-10-01T09:09:32.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-01T09:09:32.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-01T09:09:32.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.allow_data_in_errors: 0 2023-10-01T09:09:32.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.db_host_id: __hostname__ 2023-10-01T09:09:32.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.enforce_single_del_contracts: true 2023-10-01T09:09:32.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_background_jobs: 2 2023-10-01T09:09:32.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_background_compactions: -1 2023-10-01T09:09:32.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_subcompactions: 1 2023-10-01T09:09:32.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T09:09:32.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T09:09:32.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T09:09:32.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_total_wal_size: 0 2023-10-01T09:09:32.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T09:09:32.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T09:09:32.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T09:09:32.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T09:09:32.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_open_files: -1 2023-10-01T09:09:32.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.bytes_per_sync: 0 2023-10-01T09:09:32.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T09:09:32.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T09:09:32.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compaction_readahead_size: 0 2023-10-01T09:09:32.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_background_flushes: -1 2023-10-01T09:09:32.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Compression algorithms supported: 2023-10-01T09:09:32.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T09:09:32.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: kZSTD supported: 0 2023-10-01T09:09:32.067 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: kXpressCompression supported: 0 2023-10-01T09:09:32.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: kLZ4HCCompression supported: 1 2023-10-01T09:09:32.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: kZlibCompression supported: 1 2023-10-01T09:09:32.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: kSnappyCompression supported: 1 2023-10-01T09:09:32.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: kLZ4Compression supported: 1 2023-10-01T09:09:32.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: kBZip2Compression supported: 0 2023-10-01T09:09:32.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T09:09:32.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: DMutex implementation: pthread_mutex_t 2023-10-01T09:09:32.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-10-01T09:09:32.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-01T09:09:32.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T09:09:32.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.merge_operator: 2023-10-01T09:09:32.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compaction_filter: None 2023-10-01T09:09:32.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compaction_filter_factory: None 2023-10-01T09:09:32.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.sst_partitioner_factory: None 2023-10-01T09:09:32.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T09:09:32.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T09:09:32.069 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c760435520) 2023-10-01T09:09:32.069 INFO:journalctl@ceph.mon.b.smithi190.stdout: cache_index_and_filter_blocks: 1 2023-10-01T09:09:32.069 INFO:journalctl@ceph.mon.b.smithi190.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T09:09:32.069 INFO:journalctl@ceph.mon.b.smithi190.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T09:09:32.069 INFO:journalctl@ceph.mon.b.smithi190.stdout: pin_top_level_index_and_filter: 1 2023-10-01T09:09:32.069 INFO:journalctl@ceph.mon.b.smithi190.stdout: index_type: 0 2023-10-01T09:09:32.069 INFO:journalctl@ceph.mon.b.smithi190.stdout: data_block_index_type: 0 2023-10-01T09:09:32.069 INFO:journalctl@ceph.mon.b.smithi190.stdout: index_shortening: 1 2023-10-01T09:09:32.070 INFO:journalctl@ceph.mon.b.smithi190.stdout: data_block_hash_table_util_ratio: 0.750000 2023-10-01T09:09:32.070 INFO:journalctl@ceph.mon.b.smithi190.stdout: checksum: 4 2023-10-01T09:09:32.070 INFO:journalctl@ceph.mon.b.smithi190.stdout: no_block_cache: 0 2023-10-01T09:09:32.070 INFO:journalctl@ceph.mon.b.smithi190.stdout: block_cache: 0x55c75f747350 2023-10-01T09:09:32.070 INFO:journalctl@ceph.mon.b.smithi190.stdout: block_cache_name: BinnedLRUCache 2023-10-01T09:09:32.070 INFO:journalctl@ceph.mon.b.smithi190.stdout: block_cache_options: 2023-10-01T09:09:32.070 INFO:journalctl@ceph.mon.b.smithi190.stdout: capacity : 536870912 2023-10-01T09:09:32.070 INFO:journalctl@ceph.mon.b.smithi190.stdout: num_shard_bits : 4 2023-10-01T09:09:32.070 INFO:journalctl@ceph.mon.b.smithi190.stdout: strict_capacity_limit : 0 2023-10-01T09:09:32.070 INFO:journalctl@ceph.mon.b.smithi190.stdout: high_pri_pool_ratio: 0.000 2023-10-01T09:09:32.070 INFO:journalctl@ceph.mon.b.smithi190.stdout: block_cache_compressed: (nil) 2023-10-01T09:09:32.070 INFO:journalctl@ceph.mon.b.smithi190.stdout: persistent_cache: (nil) 2023-10-01T09:09:32.071 INFO:journalctl@ceph.mon.b.smithi190.stdout: block_size: 4096 2023-10-01T09:09:32.071 INFO:journalctl@ceph.mon.b.smithi190.stdout: block_size_deviation: 10 2023-10-01T09:09:32.071 INFO:journalctl@ceph.mon.b.smithi190.stdout: block_restart_interval: 16 2023-10-01T09:09:32.071 INFO:journalctl@ceph.mon.b.smithi190.stdout: index_block_restart_interval: 1 2023-10-01T09:09:32.071 INFO:journalctl@ceph.mon.b.smithi190.stdout: metadata_block_size: 4096 2023-10-01T09:09:32.071 INFO:journalctl@ceph.mon.b.smithi190.stdout: partition_filters: 0 2023-10-01T09:09:32.071 INFO:journalctl@ceph.mon.b.smithi190.stdout: use_delta_encoding: 1 2023-10-01T09:09:32.071 INFO:journalctl@ceph.mon.b.smithi190.stdout: filter_policy: bloomfilter 2023-10-01T09:09:32.071 INFO:journalctl@ceph.mon.b.smithi190.stdout: whole_key_filtering: 1 2023-10-01T09:09:32.071 INFO:journalctl@ceph.mon.b.smithi190.stdout: verify_compression: 0 2023-10-01T09:09:32.071 INFO:journalctl@ceph.mon.b.smithi190.stdout: read_amp_bytes_per_bit: 0 2023-10-01T09:09:32.071 INFO:journalctl@ceph.mon.b.smithi190.stdout: format_version: 5 2023-10-01T09:09:32.072 INFO:journalctl@ceph.mon.b.smithi190.stdout: enable_index_compression: 1 2023-10-01T09:09:32.072 INFO:journalctl@ceph.mon.b.smithi190.stdout: block_align: 0 2023-10-01T09:09:32.072 INFO:journalctl@ceph.mon.b.smithi190.stdout: max_auto_readahead_size: 262144 2023-10-01T09:09:32.072 INFO:journalctl@ceph.mon.b.smithi190.stdout: prepopulate_block_cache: 0 2023-10-01T09:09:32.072 INFO:journalctl@ceph.mon.b.smithi190.stdout: initial_auto_readahead_size: 8192 2023-10-01T09:09:32.072 INFO:journalctl@ceph.mon.b.smithi190.stdout: num_file_reads_for_auto_readahead: 2 2023-10-01T09:09:32.072 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.write_buffer_size: 33554432 2023-10-01T09:09:32.072 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_write_buffer_number: 2 2023-10-01T09:09:32.072 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compression: NoCompression 2023-10-01T09:09:32.072 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.bottommost_compression: Disabled 2023-10-01T09:09:32.072 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.prefix_extractor: nullptr 2023-10-01T09:09:32.072 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T09:09:32.072 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.num_levels: 7 2023-10-01T09:09:32.073 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T09:09:32.073 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T09:09:32.073 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T09:09:32.073 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T09:09:32.073 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T09:09:32.073 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T09:09:32.074 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T09:09:32.074 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T09:09:32.074 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-01T09:09:32.074 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T09:09:32.074 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-01T09:09:32.075 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-01T09:09:32.075 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T09:09:32.075 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compression_opts.level: 32767 2023-10-01T09:09:32.075 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compression_opts.strategy: 0 2023-10-01T09:09:32.075 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T09:09:32.075 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T09:09:32.075 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-01T09:09:32.075 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-01T09:09:32.075 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compression_opts.enabled: false 2023-10-01T09:09:32.075 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-01T09:09:32.075 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T09:09:32.076 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T09:09:32.076 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T09:09:32.076 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.target_file_size_base: 67108864 2023-10-01T09:09:32.076 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T09:09:32.076 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T09:09:32.076 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T09:09:32.076 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T09:09:32.076 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T09:09:32.076 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T09:09:32.076 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T09:09:32.076 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T09:09:32.076 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T09:09:32.077 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T09:09:32.077 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T09:09:32.077 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T09:09:32.077 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T09:09:32.077 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-01T09:09:32.077 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.arena_block_size: 1048576 2023-10-01T09:09:32.077 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T09:09:32.077 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T09:09:32.077 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.disable_auto_compactions: 0 2023-10-01T09:09:32.077 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T09:09:32.077 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T09:09:32.077 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T09:09:32.078 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T09:09:32.078 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T09:09:32.078 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T09:09:32.078 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T09:09:32.078 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T09:09:32.078 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T09:09:32.078 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T09:09:32.078 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.table_properties_collectors: 2023-10-01T09:09:32.078 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.inplace_update_support: 0 2023-10-01T09:09:32.078 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T09:09:32.078 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T09:09:32.078 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T09:09:32.079 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T09:09:32.079 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.bloom_locality: 0 2023-10-01T09:09:32.079 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.max_successive_merges: 0 2023-10-01T09:09:32.079 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T09:09:32.079 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.paranoid_file_checks: 0 2023-10-01T09:09:32.079 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.force_consistency_checks: 1 2023-10-01T09:09:32.079 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.report_bg_io_stats: 0 2023-10-01T09:09:32.079 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.ttl: 2592000 2023-10-01T09:09:32.079 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T09:09:32.079 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-01T09:09:32.079 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-01T09:09:32.079 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.enable_blob_files: false 2023-10-01T09:09:32.080 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.min_blob_size: 0 2023-10-01T09:09:32.080 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.blob_file_size: 268435456 2023-10-01T09:09:32.080 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.blob_compression_type: NoCompression 2023-10-01T09:09:32.080 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.enable_blob_garbage_collection: false 2023-10-01T09:09:32.080 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-01T09:09:32.080 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-01T09:09:32.080 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-01T09:09:32.080 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.blob_file_starting_level: 0 2023-10-01T09:09:32.080 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-01T09:09:32.080 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 20.sst 2023-10-01T09:09:32.080 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-01T09:09:32.081 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 22, last_sequence is 12167, log_number is 18,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T09:09:32.081 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 18 2023-10-01T09:09:32.081 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 9b741d74-321c-40bd-952b-62d5c4aec7c8 2023-10-01T09:09:32.081 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696151371771940, "job": 1, "event": "recovery_started", "wal_files": [18]} 2023-10-01T09:09:32.081 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #18 mode 2 2023-10-01T09:09:32.081 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 systemd[1]: Started Ceph mon.b for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:09:32.081 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696151371791721, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 23, "file_size": 5897414, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12169, "largest_seqno": 13709, "table_properties": {"data_size": 5888868, "index_size": 6439, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1093, "raw_key_size": 8323, "raw_average_key_size": 19, "raw_value_size": 5876192, "raw_average_value_size": 13665, "num_data_blocks": 283, "num_entries": 430, "num_filter_entries": 430, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696151371, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "9b741d74-321c-40bd-952b-62d5c4aec7c8", "db_session_id": "W5E60NOAIB5UH5S9HMAJ", "orig_file_number": 23, "seqno_to_time_mapping": "N/A"}} 2023-10-01T09:09:32.081 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: EVENT_LOG_v1 {"time_micros": 1696151371791829, "job": 1, "event": "recovery_finished"} 2023-10-01T09:09:32.081 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: [db/version_set.cc:5047] Creating manifest 25 2023-10-01T09:09:32.081 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-01T09:09:32.081 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000018.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-01T09:09:32.082 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c75f7e2000 2023-10-01T09:09:32.082 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: DB pointer 0x55c75f7d0000 2023-10-01T09:09:32.082 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-01T09:09:32.082 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-01T09:09:32.082 INFO:journalctl@ceph.mon.b.smithi190.stdout: ** DB Stats ** 2023-10-01T09:09:32.082 INFO:journalctl@ceph.mon.b.smithi190.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T09:09:32.082 INFO:journalctl@ceph.mon.b.smithi190.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T09:09:32.082 INFO:journalctl@ceph.mon.b.smithi190.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T09:09:32.082 INFO:journalctl@ceph.mon.b.smithi190.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T09:09:32.082 INFO:journalctl@ceph.mon.b.smithi190.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T09:09:32.082 INFO:journalctl@ceph.mon.b.smithi190.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T09:09:32.083 INFO:journalctl@ceph.mon.b.smithi190.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T09:09:32.083 INFO:journalctl@ceph.mon.b.smithi190.stdout: 2023-10-01T09:09:32.083 INFO:journalctl@ceph.mon.b.smithi190.stdout: ** Compaction Stats [default] ** 2023-10-01T09:09:32.083 INFO:journalctl@ceph.mon.b.smithi190.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T09:09:32.083 INFO:journalctl@ceph.mon.b.smithi190.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-01T09:09:32.083 INFO:journalctl@ceph.mon.b.smithi190.stdout: L0 1/0 5.62 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 627.1 0.01 0.00 1 0.009 0 0 0.0 0.0 2023-10-01T09:09:32.083 INFO:journalctl@ceph.mon.b.smithi190.stdout: L6 1/0 23.87 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-01T09:09:32.083 INFO:journalctl@ceph.mon.b.smithi190.stdout: Sum 2/0 29.49 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 627.1 0.01 0.00 1 0.009 0 0 0.0 0.0 2023-10-01T09:09:32.083 INFO:journalctl@ceph.mon.b.smithi190.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 627.1 0.01 0.00 1 0.009 0 0 0.0 0.0 2023-10-01T09:09:32.083 INFO:journalctl@ceph.mon.b.smithi190.stdout: 2023-10-01T09:09:32.083 INFO:journalctl@ceph.mon.b.smithi190.stdout: ** Compaction Stats [default] ** 2023-10-01T09:09:32.083 INFO:journalctl@ceph.mon.b.smithi190.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T09:09:32.084 INFO:journalctl@ceph.mon.b.smithi190.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T09:09:32.084 INFO:journalctl@ceph.mon.b.smithi190.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 627.1 0.01 0.00 1 0.009 0 0 0.0 0.0 2023-10-01T09:09:32.084 INFO:journalctl@ceph.mon.b.smithi190.stdout: 2023-10-01T09:09:32.084 INFO:journalctl@ceph.mon.b.smithi190.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-01T09:09:32.084 INFO:journalctl@ceph.mon.b.smithi190.stdout: 2023-10-01T09:09:32.084 INFO:journalctl@ceph.mon.b.smithi190.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T09:09:32.084 INFO:journalctl@ceph.mon.b.smithi190.stdout: Flush(GB): cumulative 0.005, interval 0.005 2023-10-01T09:09:32.084 INFO:journalctl@ceph.mon.b.smithi190.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T09:09:32.084 INFO:journalctl@ceph.mon.b.smithi190.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T09:09:32.084 INFO:journalctl@ceph.mon.b.smithi190.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T09:09:32.084 INFO:journalctl@ceph.mon.b.smithi190.stdout: AddFile(Keys): cumulative 0, interval 0 2023-10-01T09:09:32.084 INFO:journalctl@ceph.mon.b.smithi190.stdout: Cumulative compaction: 0.01 GB write, 228.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T09:09:32.085 INFO:journalctl@ceph.mon.b.smithi190.stdout: Interval compaction: 0.01 GB write, 228.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T09:09:32.085 INFO:journalctl@ceph.mon.b.smithi190.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T09:09:32.085 INFO:journalctl@ceph.mon.b.smithi190.stdout: Block cache BinnedLRUCache@0x55c75f747350#7 capacity: 512.00 MB usage: 7.73 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2023-10-01T09:09:32.085 INFO:journalctl@ceph.mon.b.smithi190.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.16 KB,0.000220537%) IndexBlock(1,6.58 KB,0.00125468%) Misc(1,0.00 KB,0%) 2023-10-01T09:09:32.085 INFO:journalctl@ceph.mon.b.smithi190.stdout: 2023-10-01T09:09:32.085 INFO:journalctl@ceph.mon.b.smithi190.stdout: ** File Read Latency Histogram By Level [default] ** 2023-10-01T09:09:32.085 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: starting mon.b rank 1 at public addrs [v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0] at bind addrs [v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T09:09:32.085 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: mon.b@-1(???) e3 preinit fsid 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T09:09:32.085 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: mon.b@-1(???).mds e7 new map 2023-10-01T09:09:32.085 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: mon.b@-1(???).mds e7 print_map 2023-10-01T09:09:32.085 INFO:journalctl@ceph.mon.b.smithi190.stdout: e7 2023-10-01T09:09:32.086 INFO:journalctl@ceph.mon.b.smithi190.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-10-01T09:09:32.086 INFO:journalctl@ceph.mon.b.smithi190.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T09:09:32.086 INFO:journalctl@ceph.mon.b.smithi190.stdout: legacy client fscid: 1 2023-10-01T09:09:32.086 INFO:journalctl@ceph.mon.b.smithi190.stdout: 2023-10-01T09:09:32.086 INFO:journalctl@ceph.mon.b.smithi190.stdout: Filesystem 'foo' (1) 2023-10-01T09:09:32.086 INFO:journalctl@ceph.mon.b.smithi190.stdout: fs_name foo 2023-10-01T09:09:32.086 INFO:journalctl@ceph.mon.b.smithi190.stdout: epoch 5 2023-10-01T09:09:32.086 INFO:journalctl@ceph.mon.b.smithi190.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-01T09:09:32.086 INFO:journalctl@ceph.mon.b.smithi190.stdout: created 2023-10-01T09:03:03.201740+0000 2023-10-01T09:09:32.086 INFO:journalctl@ceph.mon.b.smithi190.stdout: modified 2023-10-01T09:03:09.223385+0000 2023-10-01T09:09:32.086 INFO:journalctl@ceph.mon.b.smithi190.stdout: tableserver 0 2023-10-01T09:09:32.087 INFO:journalctl@ceph.mon.b.smithi190.stdout: root 0 2023-10-01T09:09:32.087 INFO:journalctl@ceph.mon.b.smithi190.stdout: session_timeout 60 2023-10-01T09:09:32.087 INFO:journalctl@ceph.mon.b.smithi190.stdout: session_autoclose 300 2023-10-01T09:09:32.087 INFO:journalctl@ceph.mon.b.smithi190.stdout: max_file_size 1099511627776 2023-10-01T09:09:32.087 INFO:journalctl@ceph.mon.b.smithi190.stdout: required_client_features {} 2023-10-01T09:09:32.087 INFO:journalctl@ceph.mon.b.smithi190.stdout: last_failure 0 2023-10-01T09:09:32.087 INFO:journalctl@ceph.mon.b.smithi190.stdout: last_failure_osd_epoch 0 2023-10-01T09:09:32.087 INFO:journalctl@ceph.mon.b.smithi190.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T09:09:32.088 INFO:journalctl@ceph.mon.b.smithi190.stdout: max_mds 1 2023-10-01T09:09:32.088 INFO:journalctl@ceph.mon.b.smithi190.stdout: in 0 2023-10-01T09:09:32.088 INFO:journalctl@ceph.mon.b.smithi190.stdout: up {0=14637} 2023-10-01T09:09:32.088 INFO:journalctl@ceph.mon.b.smithi190.stdout: failed 2023-10-01T09:09:32.088 INFO:journalctl@ceph.mon.b.smithi190.stdout: damaged 2023-10-01T09:09:32.088 INFO:journalctl@ceph.mon.b.smithi190.stdout: stopped 2023-10-01T09:09:32.088 INFO:journalctl@ceph.mon.b.smithi190.stdout: data_pools [4] 2023-10-01T09:09:32.088 INFO:journalctl@ceph.mon.b.smithi190.stdout: metadata_pool 3 2023-10-01T09:09:32.088 INFO:journalctl@ceph.mon.b.smithi190.stdout: inline_data disabled 2023-10-01T09:09:32.088 INFO:journalctl@ceph.mon.b.smithi190.stdout: balancer 2023-10-01T09:09:32.088 INFO:journalctl@ceph.mon.b.smithi190.stdout: bal_rank_mask -1 2023-10-01T09:09:32.088 INFO:journalctl@ceph.mon.b.smithi190.stdout: standby_count_wanted 1 2023-10-01T09:09:32.089 INFO:journalctl@ceph.mon.b.smithi190.stdout: [mds.foo.smithi138.brhzod{0:14637} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.138:6834/416108704,v1:172.21.15.138:6835/416108704] compat {c=[1],r=[1],i=[7ff]}] 2023-10-01T09:09:32.089 INFO:journalctl@ceph.mon.b.smithi190.stdout: 2023-10-01T09:09:32.089 INFO:journalctl@ceph.mon.b.smithi190.stdout: 2023-10-01T09:09:32.089 INFO:journalctl@ceph.mon.b.smithi190.stdout: Standby daemons: 2023-10-01T09:09:32.089 INFO:journalctl@ceph.mon.b.smithi190.stdout: 2023-10-01T09:09:32.089 INFO:journalctl@ceph.mon.b.smithi190.stdout: [mds.foo.smithi190.weynjc{-1:24505} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.190:6832/4158685175,v1:172.21.15.190:6833/4158685175] compat {c=[1],r=[1],i=[7ff]}] 2023-10-01T09:09:32.089 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: mon.b@-1(???).osd e85 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-01T09:09:32.089 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: mon.b@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T09:09:32.089 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: mon.b@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T09:09:32.089 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: mon.b@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T09:09:32.089 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-10-01T09:09:32.090 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: expand_channel_meta expand map: {default=false} 2023-10-01T09:09:32.090 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: expand_channel_meta from 'false' to 'false' 2023-10-01T09:09:32.090 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: expand_channel_meta expanded map: {default=false} 2023-10-01T09:09:32.090 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: expand_channel_meta expand map: {default=info} 2023-10-01T09:09:32.090 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: expand_channel_meta from 'info' to 'info' 2023-10-01T09:09:32.090 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: expand_channel_meta expanded map: {default=info} 2023-10-01T09:09:32.090 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: expand_channel_meta expand map: {default=daemon} 2023-10-01T09:09:32.090 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: expand_channel_meta from 'daemon' to 'daemon' 2023-10-01T09:09:32.090 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: expand_channel_meta expanded map: {default=daemon} 2023-10-01T09:09:32.090 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: expand_channel_meta expand map: {default=debug} 2023-10-01T09:09:32.090 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: expand_channel_meta from 'debug' to 'debug' 2023-10-01T09:09:32.090 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:31 smithi190 ceph-mon[160435]: expand_channel_meta expanded map: {default=debug} 2023-10-01T09:09:35.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:09:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:09:34] "GET /metrics HTTP/1.1" 200 33781 "" "Prometheus/2.33.4" 2023-10-01T09:09:37.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:09:36 smithi138 conmon[178630]: 2023-10-01T09:09:36.805+0000 7fa004e48700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-01T09:09:37.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mkfs 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T09:09:37.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:09:37.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-01T09:09:37.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Reconfiguring daemon node-exporter.b on smithi190 2023-10-01T09:09:37.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Reconfiguring grafana.a (dependencies changed)... 2023-10-01T09:09:37.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Reconfiguring daemon grafana.a on smithi190 2023-10-01T09:09:37.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T09:09:37.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T09:09:37.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Upgrade: Updating mgr.y 2023-10-01T09:09:37.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Deploying daemon mgr.y on smithi138 2023-10-01T09:09:37.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: [01/Oct/2023:09:07:17] ENGINE Bus STARTING 2023-10-01T09:09:37.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: [01/Oct/2023:09:07:18] ENGINE Serving on http://172.21.15.138:8765 2023-10-01T09:09:37.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: [01/Oct/2023:09:07:18] ENGINE Serving on https://172.21.15.138:7150 2023-10-01T09:09:37.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: [01/Oct/2023:09:07:18] ENGINE Bus STARTED 2023-10-01T09:09:37.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:09:37.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:09:37.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:09:37.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:09:37.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:09:37.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:09:37.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:09:37.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:09:37.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T09:09:37.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:09:37.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Upgrade: Updating mgr.x 2023-10-01T09:09:37.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Deploying daemon mgr.x on smithi190 2023-10-01T09:09:37.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Upgrade: Setting container_image for all mgr 2023-10-01T09:09:37.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Upgrade: It appears safe to stop mon.a 2023-10-01T09:09:37.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Upgrade: Updating mon.a 2023-10-01T09:09:37.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Deploying daemon mon.a on smithi138 2023-10-01T09:09:37.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: [01/Oct/2023:09:07:55] ENGINE Bus STARTING 2023-10-01T09:09:37.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: [01/Oct/2023:09:07:55] ENGINE Serving on http://172.21.15.138:8765 2023-10-01T09:09:37.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: [01/Oct/2023:09:07:55] ENGINE Serving on https://172.21.15.138:7150 2023-10-01T09:09:37.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: [01/Oct/2023:09:07:55] ENGINE Bus STARTED 2023-10-01T09:09:37.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:09:37.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:09:37.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:09:37.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:09:37.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:09:37.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:09:37.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:09:37.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:09:37.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:09:37.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:37.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:37.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:37.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:37.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:37.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='client.26057 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:09:37.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:09:37.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:37.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:09:37.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:37.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:09:37.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T09:09:37.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-01T09:09:37.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Upgrade: It appears safe to stop mon.c 2023-10-01T09:09:37.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3814562467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:09:37.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:37.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Upgrade: Updating mon.c 2023-10-01T09:09:37.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:37.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:37.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:37.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Deploying daemon mon.c on smithi138 2023-10-01T09:09:37.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mon.c calling monitor election 2023-10-01T09:09:37.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mon.b calling monitor election 2023-10-01T09:09:37.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mon.a calling monitor election 2023-10-01T09:09:37.079 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.080 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.080 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.080 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-01T09:09:37.080 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:09:37.080 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:09:37.080 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:09:37.080 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mgrmap e34: y(active, since 84s), standbys: x 2023-10-01T09:09:37.080 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-01T09:09:37.080 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-01T09:09:37.080 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-01T09:09:37.080 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mon.c (rank 2) addr [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] is down (out of quorum) 2023-10-01T09:09:37.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:37.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mon.b calling monitor election 2023-10-01T09:09:37.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mon.a calling monitor election 2023-10-01T09:09:37.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T09:09:37.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:09:37.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:09:37.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:09:37.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mgrmap e34: y(active, since 85s), standbys: x 2023-10-01T09:09:37.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-01T09:09:37.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:09:37.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: overall HEALTH_OK 2023-10-01T09:09:37.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/898293204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:09:37.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:09:37.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:37.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mgrmap e35: y(active, since 90s), standbys: x 2023-10-01T09:09:37.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:37.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:37.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:09:37.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:37.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:09:37.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T09:09:37.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-01T09:09:37.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Upgrade: It appears safe to stop mon.b 2023-10-01T09:09:37.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:09:37.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:09:37.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Upgrade: Updating mon.b 2023-10-01T09:09:37.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:37.084 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:37.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:37.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Deploying daemon mon.b on smithi190 2023-10-01T09:09:37.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:37.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: overall HEALTH_OK 2023-10-01T09:09:37.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:09:37.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mon.a calling monitor election 2023-10-01T09:09:37.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mon.b calling monitor election 2023-10-01T09:09:37.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:09:37.085 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:09:37.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:37.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:37.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-01T09:09:37.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: monmap e4: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:09:37.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:09:37.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:09:37.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mgrmap e35: y(active, since 101s), standbys: x 2023-10-01T09:09:37.086 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-01T09:09:37.087 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-01T09:09:37.087 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-01T09:09:37.087 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: mon.c (rank 2) addr [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] is down (out of quorum) 2023-10-01T09:09:37.087 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.087 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mkfs 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T09:09:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:09:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-01T09:09:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Reconfiguring daemon node-exporter.b on smithi190 2023-10-01T09:09:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Reconfiguring grafana.a (dependencies changed)... 2023-10-01T09:09:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Reconfiguring daemon grafana.a on smithi190 2023-10-01T09:09:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T09:09:37.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T09:09:37.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Upgrade: Updating mgr.y 2023-10-01T09:09:37.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Deploying daemon mgr.y on smithi138 2023-10-01T09:09:37.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: [01/Oct/2023:09:07:17] ENGINE Bus STARTING 2023-10-01T09:09:37.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: [01/Oct/2023:09:07:18] ENGINE Serving on http://172.21.15.138:8765 2023-10-01T09:09:37.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: [01/Oct/2023:09:07:18] ENGINE Serving on https://172.21.15.138:7150 2023-10-01T09:09:37.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: [01/Oct/2023:09:07:18] ENGINE Bus STARTED 2023-10-01T09:09:37.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:09:37.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:09:37.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:09:37.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:09:37.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:09:37.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:09:37.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:09:37.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:09:37.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T09:09:37.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:09:37.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Upgrade: Updating mgr.x 2023-10-01T09:09:37.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Deploying daemon mgr.x on smithi190 2023-10-01T09:09:37.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Upgrade: Setting container_image for all mgr 2023-10-01T09:09:37.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Upgrade: It appears safe to stop mon.a 2023-10-01T09:09:37.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Upgrade: Updating mon.a 2023-10-01T09:09:37.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Deploying daemon mon.a on smithi138 2023-10-01T09:09:37.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: [01/Oct/2023:09:07:55] ENGINE Bus STARTING 2023-10-01T09:09:37.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: [01/Oct/2023:09:07:55] ENGINE Serving on http://172.21.15.138:8765 2023-10-01T09:09:37.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: [01/Oct/2023:09:07:55] ENGINE Serving on https://172.21.15.138:7150 2023-10-01T09:09:37.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: [01/Oct/2023:09:07:55] ENGINE Bus STARTED 2023-10-01T09:09:37.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:09:37.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:09:37.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:09:37.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:09:37.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:09:37.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:09:37.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:09:37.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:09:37.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:09:37.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:37.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:37.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:37.311 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:37.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.312 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:37.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='client.26057 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:09:37.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:09:37.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:37.313 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:09:37.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:37.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:09:37.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T09:09:37.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-01T09:09:37.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Upgrade: It appears safe to stop mon.c 2023-10-01T09:09:37.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3814562467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:09:37.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:37.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Upgrade: Updating mon.c 2023-10-01T09:09:37.314 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:37.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:37.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:37.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Deploying daemon mon.c on smithi138 2023-10-01T09:09:37.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mon.c calling monitor election 2023-10-01T09:09:37.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mon.b calling monitor election 2023-10-01T09:09:37.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mon.a calling monitor election 2023-10-01T09:09:37.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-01T09:09:37.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:09:37.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:09:37.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:09:37.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mgrmap e34: y(active, since 84s), standbys: x 2023-10-01T09:09:37.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-01T09:09:37.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-01T09:09:37.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-01T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mon.c (rank 2) addr [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] is down (out of quorum) 2023-10-01T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mon.b calling monitor election 2023-10-01T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mon.a calling monitor election 2023-10-01T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:09:37.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:09:37.318 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mgrmap e34: y(active, since 85s), standbys: x 2023-10-01T09:09:37.318 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-01T09:09:37.318 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:09:37.318 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: overall HEALTH_OK 2023-10-01T09:09:37.318 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/898293204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:09:37.318 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.318 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.318 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.318 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.318 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.319 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.319 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:09:37.319 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:37.319 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:37.319 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mgrmap e35: y(active, since 90s), standbys: x 2023-10-01T09:09:37.319 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:37.319 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.319 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.319 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:37.319 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:09:37.319 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.319 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:37.320 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:09:37.320 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T09:09:37.320 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-01T09:09:37.320 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Upgrade: It appears safe to stop mon.b 2023-10-01T09:09:37.320 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:09:37.320 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:09:37.320 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.320 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Upgrade: Updating mon.b 2023-10-01T09:09:37.320 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.320 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:37.320 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:37.320 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:37.321 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Deploying daemon mon.b on smithi190 2023-10-01T09:09:37.321 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:37.321 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: overall HEALTH_OK 2023-10-01T09:09:37.321 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:09:37.321 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mon.a calling monitor election 2023-10-01T09:09:37.321 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mon.b calling monitor election 2023-10-01T09:09:37.321 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:09:37.321 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:09:37.321 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:37.321 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:37.321 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-01T09:09:37.322 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: monmap e4: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:09:37.322 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:09:37.322 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:09:37.322 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mgrmap e35: y(active, since 101s), standbys: x 2023-10-01T09:09:37.322 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-01T09:09:37.322 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-01T09:09:37.322 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-01T09:09:37.322 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: mon.c (rank 2) addr [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] is down (out of quorum) 2023-10-01T09:09:37.322 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:37.322 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:38.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:38 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:38 smithi190 ceph-mon[160435]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:38.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:38 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:38.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:38 smithi138 ceph-mon[185886]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:39.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:09:38 smithi190 conmon[151158]: ts=2023-10-01T09:09:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:09:39.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[185886]: mon.c calling monitor election 2023-10-01T09:09:39.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[185886]: mon.b calling monitor election 2023-10-01T09:09:39.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[185886]: mon.a calling monitor election 2023-10-01T09:09:39.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[185886]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T09:09:39.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[185886]: monmap e4: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:09:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[185886]: fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:09:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[185886]: osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:09:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[185886]: mgrmap e35: y(active, since 104s), standbys: x 2023-10-01T09:09:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[185886]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-01T09:09:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:09:39.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[185886]: overall HEALTH_OK 2023-10-01T09:09:39.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mkfs 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T09:09:39.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:09:39.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Reconfiguring node-exporter.b (dependencies changed)... 2023-10-01T09:09:39.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Reconfiguring daemon node-exporter.b on smithi190 2023-10-01T09:09:39.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Reconfiguring grafana.a (dependencies changed)... 2023-10-01T09:09:39.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Reconfiguring daemon grafana.a on smithi190 2023-10-01T09:09:39.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T09:09:39.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T09:09:39.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Upgrade: Updating mgr.y 2023-10-01T09:09:39.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Deploying daemon mgr.y on smithi138 2023-10-01T09:09:39.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: [01/Oct/2023:09:07:17] ENGINE Bus STARTING 2023-10-01T09:09:39.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: [01/Oct/2023:09:07:18] ENGINE Serving on http://172.21.15.138:8765 2023-10-01T09:09:39.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: [01/Oct/2023:09:07:18] ENGINE Serving on https://172.21.15.138:7150 2023-10-01T09:09:39.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: [01/Oct/2023:09:07:18] ENGINE Bus STARTED 2023-10-01T09:09:39.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:09:39.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:09:39.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:09:39.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:09:39.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:09:39.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:09:39.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:09:39.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:09:39.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T09:09:39.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Reconfiguring daemon prometheus.a on smithi190 2023-10-01T09:09:39.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Upgrade: Updating mgr.x 2023-10-01T09:09:39.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Deploying daemon mgr.x on smithi190 2023-10-01T09:09:39.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Upgrade: Setting container_image for all mgr 2023-10-01T09:09:39.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Upgrade: It appears safe to stop mon.a 2023-10-01T09:09:39.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Upgrade: Updating mon.a 2023-10-01T09:09:39.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Deploying daemon mon.a on smithi138 2023-10-01T09:09:39.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: [01/Oct/2023:09:07:55] ENGINE Bus STARTING 2023-10-01T09:09:39.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: [01/Oct/2023:09:07:55] ENGINE Serving on http://172.21.15.138:8765 2023-10-01T09:09:39.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: [01/Oct/2023:09:07:55] ENGINE Serving on https://172.21.15.138:7150 2023-10-01T09:09:39.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: [01/Oct/2023:09:07:55] ENGINE Bus STARTED 2023-10-01T09:09:39.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:09:39.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi138:/etc/ceph/ceph.conf 2023-10-01T09:09:39.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi190:/etc/ceph/ceph.conf 2023-10-01T09:09:39.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:09:39.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T09:09:39.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi138:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:09:39.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi190:/etc/ceph/ceph.client.admin.keyring 2023-10-01T09:09:39.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:09:39.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Updating smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T09:09:39.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:39.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:39.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:39.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:39.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:39.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='client.26057 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:09:39.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:09:39.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:39.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:09:39.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:39.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:09:39.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T09:09:39.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-01T09:09:39.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Upgrade: It appears safe to stop mon.c 2023-10-01T09:09:39.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3814562467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:09:39.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:39.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Upgrade: Updating mon.c 2023-10-01T09:09:39.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:39.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:39.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:39.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Deploying daemon mon.c on smithi138 2023-10-01T09:09:39.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.c calling monitor election 2023-10-01T09:09:39.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.b calling monitor election 2023-10-01T09:09:39.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.a calling monitor election 2023-10-01T09:09:39.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-01T09:09:39.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:09:39.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:09:39.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:09:39.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mgrmap e34: y(active, since 84s), standbys: x 2023-10-01T09:09:39.582 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-01T09:09:39.583 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-01T09:09:39.583 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-01T09:09:39.583 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.c (rank 2) addr [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] is down (out of quorum) 2023-10-01T09:09:39.583 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.583 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.583 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:39.583 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.b calling monitor election 2023-10-01T09:09:39.583 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.a calling monitor election 2023-10-01T09:09:39.583 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T09:09:39.584 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: monmap e3: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:09:39.584 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:09:39.584 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:09:39.584 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mgrmap e34: y(active, since 85s), standbys: x 2023-10-01T09:09:39.584 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-01T09:09:39.584 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:09:39.584 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: overall HEALTH_OK 2023-10-01T09:09:39.584 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/898293204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:09:39.584 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.584 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.585 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.585 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.585 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.585 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.585 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:09:39.585 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:39.585 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:39.585 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mgrmap e35: y(active, since 90s), standbys: x 2023-10-01T09:09:39.585 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:39.585 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.586 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.586 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:39.586 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:09:39.586 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.586 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:39.586 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:09:39.586 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T09:09:39.586 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-01T09:09:39.586 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Upgrade: It appears safe to stop mon.b 2023-10-01T09:09:39.587 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:09:39.587 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:09:39.587 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.587 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Upgrade: Updating mon.b 2023-10-01T09:09:39.587 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.587 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:39.587 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:39.587 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:39.587 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Deploying daemon mon.b on smithi190 2023-10-01T09:09:39.587 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:39.588 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: overall HEALTH_OK 2023-10-01T09:09:39.588 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T09:09:39.588 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.a calling monitor election 2023-10-01T09:09:39.588 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.b calling monitor election 2023-10-01T09:09:39.588 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T09:09:39.588 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T09:09:39.588 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:39.588 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:39.588 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-01T09:09:39.588 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: monmap e4: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:09:39.588 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:09:39.589 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:09:39.589 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mgrmap e35: y(active, since 101s), standbys: x 2023-10-01T09:09:39.589 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-01T09:09:39.589 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-01T09:09:39.589 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-01T09:09:39.589 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.c (rank 2) addr [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0] is down (out of quorum) 2023-10-01T09:09:39.589 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.589 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:39.589 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:09:39.589 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-10-01T09:09:39.590 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.c calling monitor election 2023-10-01T09:09:39.590 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.b calling monitor election 2023-10-01T09:09:39.590 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.a calling monitor election 2023-10-01T09:09:39.590 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T09:09:39.590 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: monmap e4: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:09:39.590 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:09:39.590 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:09:39.590 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: mgrmap e35: y(active, since 104s), standbys: x 2023-10-01T09:09:39.590 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-01T09:09:39.590 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:09:39.591 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:39 smithi138 ceph-mon[192513]: overall HEALTH_OK 2023-10-01T09:09:39.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:39 smithi190 ceph-mon[160435]: mon.c calling monitor election 2023-10-01T09:09:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:39 smithi190 ceph-mon[160435]: mon.b calling monitor election 2023-10-01T09:09:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:39 smithi190 ceph-mon[160435]: mon.a calling monitor election 2023-10-01T09:09:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:39 smithi190 ceph-mon[160435]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T09:09:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:39 smithi190 ceph-mon[160435]: monmap e4: 3 mons at {a=[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0],b=[v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0],c=[v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]} removed_ranks: {} 2023-10-01T09:09:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:39 smithi190 ceph-mon[160435]: fsmap foo:1 {0=foo.smithi138.brhzod=up:active} 1 up:standby 2023-10-01T09:09:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:39 smithi190 ceph-mon[160435]: osdmap e85: 8 total, 8 up, 8 in 2023-10-01T09:09:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:39 smithi190 ceph-mon[160435]: mgrmap e35: y(active, since 104s), standbys: x 2023-10-01T09:09:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:39 smithi190 ceph-mon[160435]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-01T09:09:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:39 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:09:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:39 smithi190 ceph-mon[160435]: overall HEALTH_OK 2023-10-01T09:09:40.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:40 smithi138 ceph-mon[185886]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:40.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:40.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:40.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:40 smithi138 ceph-mon[192513]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:40.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:40 smithi190 ceph-mon[160435]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:42.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:42.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:42.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:41 smithi190 ceph-mon[160435]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:42.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:41 smithi138 ceph-mon[192513]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:42.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:42.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:42.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:41 smithi138 ceph-mon[185886]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:44 smithi190 ceph-mon[160435]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:44 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:44 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:44 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:44 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:09:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:44 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:44 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:44 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:44 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:44.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[185886]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:44.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:44.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:44.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:44.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:09:44.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:44.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:44.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:44.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:44.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[192513]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:44.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:44.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:44.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:44.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:09:44.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:44.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:44.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:44.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:44 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:45.259 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:45 smithi138 ceph-mon[192513]: Reconfiguring mon.a (monmap changed)... 2023-10-01T09:09:45.259 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:45 smithi138 ceph-mon[192513]: Reconfiguring daemon mon.a on smithi138 2023-10-01T09:09:45.259 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:09:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:09:44] "GET /metrics HTTP/1.1" 200 33775 "" "Prometheus/2.33.4" 2023-10-01T09:09:45.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:45 smithi190 ceph-mon[160435]: Reconfiguring mon.a (monmap changed)... 2023-10-01T09:09:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:45 smithi190 ceph-mon[160435]: Reconfiguring daemon mon.a on smithi138 2023-10-01T09:09:45.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:45 smithi138 ceph-mon[185886]: Reconfiguring mon.a (monmap changed)... 2023-10-01T09:09:45.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:45 smithi138 ceph-mon[185886]: Reconfiguring daemon mon.a on smithi138 2023-10-01T09:09:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[192513]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:46.603 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:46.603 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:46.603 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[192513]: Reconfiguring mgr.y (monmap changed)... 2023-10-01T09:09:46.603 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:09:46.603 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:09:46.603 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:46.603 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[192513]: Reconfiguring daemon mgr.y on smithi138 2023-10-01T09:09:46.604 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[185886]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:46.604 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:46.604 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:46.604 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[185886]: Reconfiguring mgr.y (monmap changed)... 2023-10-01T09:09:46.604 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:09:46.604 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:09:46.604 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:46.604 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:46 smithi138 ceph-mon[185886]: Reconfiguring daemon mgr.y on smithi138 2023-10-01T09:09:46.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:46 smithi190 ceph-mon[160435]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:46 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:46 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:46 smithi190 ceph-mon[160435]: Reconfiguring mgr.y (monmap changed)... 2023-10-01T09:09:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:46 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:09:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:46 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:09:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:46 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:46 smithi190 ceph-mon[160435]: Reconfiguring daemon mgr.y on smithi138 2023-10-01T09:09:48.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:48.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:48.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[185886]: Reconfiguring mon.c (monmap changed)... 2023-10-01T09:09:48.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:48.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:48.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:48.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[185886]: Reconfiguring daemon mon.c on smithi138 2023-10-01T09:09:48.023 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[185886]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:48.023 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:48.023 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:48.023 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[192513]: Reconfiguring mon.c (monmap changed)... 2023-10-01T09:09:48.023 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:48.023 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:48.023 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:48.023 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[192513]: Reconfiguring daemon mon.c on smithi138 2023-10-01T09:09:48.023 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:47 smithi138 ceph-mon[192513]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:48.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:47 smithi190 ceph-mon[160435]: Reconfiguring mon.c (monmap changed)... 2023-10-01T09:09:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:47 smithi190 ceph-mon[160435]: Reconfiguring daemon mon.c on smithi138 2023-10-01T09:09:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:47 smithi190 ceph-mon[160435]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:49.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:09:48 smithi190 conmon[151158]: ts=2023-10-01T09:09:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:09:49.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:49.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:49.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:49 smithi138 ceph-mon[192513]: Reconfiguring osd.0 (monmap changed)... 2023-10-01T09:09:49.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T09:09:49.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:49.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:49 smithi138 ceph-mon[192513]: Reconfiguring daemon osd.0 on smithi138 2023-10-01T09:09:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:49 smithi138 ceph-mon[185886]: Reconfiguring osd.0 (monmap changed)... 2023-10-01T09:09:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T09:09:49.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:49.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:49 smithi138 ceph-mon[185886]: Reconfiguring daemon osd.0 on smithi138 2023-10-01T09:09:49.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:49.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:49.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:49 smithi190 ceph-mon[160435]: Reconfiguring osd.0 (monmap changed)... 2023-10-01T09:09:49.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T09:09:49.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:49.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:49 smithi190 ceph-mon[160435]: Reconfiguring daemon osd.0 on smithi138 2023-10-01T09:09:50.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:50 smithi138 ceph-mon[192513]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:50.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:50 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:50.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:50 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:50.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:50 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T09:09:50.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:50 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:50.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:50 smithi138 ceph-mon[185886]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:50.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:50 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:50.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:50 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:50.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:50 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T09:09:50.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:50 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:50.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:50 smithi190 ceph-mon[160435]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:50 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:50 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:50 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T09:09:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:50 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:51.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:51 smithi138 ceph-mon[192513]: Reconfiguring osd.1 (monmap changed)... 2023-10-01T09:09:51.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:51 smithi138 ceph-mon[192513]: Reconfiguring daemon osd.1 on smithi138 2023-10-01T09:09:51.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:51 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:51.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:51 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:51.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:51 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T09:09:51.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:51 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:51.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:51 smithi138 ceph-mon[185886]: Reconfiguring osd.1 (monmap changed)... 2023-10-01T09:09:51.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:51 smithi138 ceph-mon[185886]: Reconfiguring daemon osd.1 on smithi138 2023-10-01T09:09:51.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:51 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:51.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:51 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:51.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:51 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T09:09:51.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:51 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:51.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:51 smithi190 ceph-mon[160435]: Reconfiguring osd.1 (monmap changed)... 2023-10-01T09:09:51.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:51 smithi190 ceph-mon[160435]: Reconfiguring daemon osd.1 on smithi138 2023-10-01T09:09:51.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:51 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:51.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:51 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:51.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:51 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T09:09:51.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:51 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:52.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:52 smithi190 ceph-mon[160435]: Reconfiguring osd.2 (monmap changed)... 2023-10-01T09:09:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:52 smithi190 ceph-mon[160435]: Reconfiguring daemon osd.2 on smithi138 2023-10-01T09:09:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:52 smithi190 ceph-mon[160435]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:52.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:52 smithi138 ceph-mon[192513]: Reconfiguring osd.2 (monmap changed)... 2023-10-01T09:09:52.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:52 smithi138 ceph-mon[192513]: Reconfiguring daemon osd.2 on smithi138 2023-10-01T09:09:52.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:52 smithi138 ceph-mon[192513]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:52.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:52 smithi138 ceph-mon[185886]: Reconfiguring osd.2 (monmap changed)... 2023-10-01T09:09:52.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:52 smithi138 ceph-mon[185886]: Reconfiguring daemon osd.2 on smithi138 2023-10-01T09:09:52.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:52 smithi138 ceph-mon[185886]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:53.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:53 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:53.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:53 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:53.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:53 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T09:09:53.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:53 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:53.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:53 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:53.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:53 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:53.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:53 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T09:09:53.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:53 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:53 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:53 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:53 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T09:09:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:53 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:54.690 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:54 smithi138 ceph-mon[192513]: Reconfiguring osd.3 (monmap changed)... 2023-10-01T09:09:54.691 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:54 smithi138 ceph-mon[192513]: Reconfiguring daemon osd.3 on smithi138 2023-10-01T09:09:54.691 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:54 smithi138 ceph-mon[192513]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:54.691 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:54 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:54.691 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:54 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:54.691 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:54 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.brhzod", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:09:54.691 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:54 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:54.692 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:54 smithi138 ceph-mon[185886]: Reconfiguring osd.3 (monmap changed)... 2023-10-01T09:09:54.692 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:54 smithi138 ceph-mon[185886]: Reconfiguring daemon osd.3 on smithi138 2023-10-01T09:09:54.692 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:54 smithi138 ceph-mon[185886]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:54.692 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:54 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:54.692 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:54 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:54.693 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:54 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.brhzod", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:09:54.693 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:54 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:54.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:54 smithi190 ceph-mon[160435]: Reconfiguring osd.3 (monmap changed)... 2023-10-01T09:09:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:54 smithi190 ceph-mon[160435]: Reconfiguring daemon osd.3 on smithi138 2023-10-01T09:09:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:54 smithi190 ceph-mon[160435]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:54 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:54 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:54 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi138.brhzod", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:09:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:54 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:55.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:09:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:09:54] "GET /metrics HTTP/1.1" 200 33775 "" "Prometheus/2.33.4" 2023-10-01T09:09:55.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[192513]: Reconfiguring mds.foo.smithi138.brhzod (monmap changed)... 2023-10-01T09:09:55.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[192513]: Reconfiguring daemon mds.foo.smithi138.brhzod on smithi138 2023-10-01T09:09:55.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:55.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:55.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:55.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:55.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:55.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:55.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[185886]: Reconfiguring mds.foo.smithi138.brhzod (monmap changed)... 2023-10-01T09:09:55.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[185886]: Reconfiguring daemon mds.foo.smithi138.brhzod on smithi138 2023-10-01T09:09:55.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:55.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:55.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:55.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:55.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:55.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:55.781 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:55 smithi190 ceph-mon[160435]: Reconfiguring mds.foo.smithi138.brhzod (monmap changed)... 2023-10-01T09:09:55.781 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:55 smithi190 ceph-mon[160435]: Reconfiguring daemon mds.foo.smithi138.brhzod on smithi138 2023-10-01T09:09:55.782 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:09:55.782 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:55.782 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:55.782 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T09:09:55.782 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T09:09:55.782 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:56.554 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:56 smithi190 ceph-mon[160435]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:56.554 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:56 smithi190 ceph-mon[160435]: Reconfiguring mon.b (monmap changed)... 2023-10-01T09:09:56.554 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:56 smithi190 ceph-mon[160435]: Reconfiguring daemon mon.b on smithi190 2023-10-01T09:09:56.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:56 smithi138 ceph-mon[185886]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:56.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:56 smithi138 ceph-mon[185886]: Reconfiguring mon.b (monmap changed)... 2023-10-01T09:09:56.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:56 smithi138 ceph-mon[185886]: Reconfiguring daemon mon.b on smithi190 2023-10-01T09:09:56.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:56 smithi138 ceph-mon[192513]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:56.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:56 smithi138 ceph-mon[192513]: Reconfiguring mon.b (monmap changed)... 2023-10-01T09:09:56.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:56 smithi138 ceph-mon[192513]: Reconfiguring daemon mon.b on smithi190 2023-10-01T09:09:57.789 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:57 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:57.789 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:57 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:57.789 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:57 smithi190 ceph-mon[160435]: Reconfiguring mgr.x (monmap changed)... 2023-10-01T09:09:57.789 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:57 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:09:57.789 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:57 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:09:57.789 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:57 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:57.790 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:57 smithi190 ceph-mon[160435]: Reconfiguring daemon mgr.x on smithi190 2023-10-01T09:09:57.790 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:57 smithi190 ceph-mon[160435]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:58.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:58.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:58.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[192513]: Reconfiguring mgr.x (monmap changed)... 2023-10-01T09:09:58.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:09:58.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:09:58.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:58.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[192513]: Reconfiguring daemon mgr.x on smithi190 2023-10-01T09:09:58.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[192513]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:58.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:58.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:58.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[185886]: Reconfiguring mgr.x (monmap changed)... 2023-10-01T09:09:58.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T09:09:58.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T09:09:58.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:58.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[185886]: Reconfiguring daemon mgr.x on smithi190 2023-10-01T09:09:58.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:57 smithi138 ceph-mon[185886]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:09:59.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:09:58 smithi190 conmon[151158]: ts=2023-10-01T09:09:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:09:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:58 smithi190 ceph-mon[160435]: Reconfiguring osd.4 (monmap changed)... 2023-10-01T09:09:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T09:09:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:09:58 smithi190 ceph-mon[160435]: Reconfiguring daemon osd.4 on smithi190 2023-10-01T09:09:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:58 smithi138 ceph-mon[192513]: Reconfiguring osd.4 (monmap changed)... 2023-10-01T09:09:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T09:09:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:09:58 smithi138 ceph-mon[192513]: Reconfiguring daemon osd.4 on smithi190 2023-10-01T09:09:59.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:59.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:09:59.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:58 smithi138 ceph-mon[185886]: Reconfiguring osd.4 (monmap changed)... 2023-10-01T09:09:59.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T09:09:59.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:09:59.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:09:58 smithi138 ceph-mon[185886]: Reconfiguring daemon osd.4 on smithi190 2023-10-01T09:10:00.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:00 smithi190 ceph-mon[160435]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:00 smithi190 ceph-mon[160435]: Reconfiguring osd.5 (monmap changed)... 2023-10-01T09:10:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T09:10:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:00 smithi190 ceph-mon[160435]: Reconfiguring daemon osd.5 on smithi190 2023-10-01T09:10:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:00 smithi190 ceph-mon[160435]: overall HEALTH_OK 2023-10-01T09:10:00.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:00.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:00.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[185886]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:00.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[185886]: Reconfiguring osd.5 (monmap changed)... 2023-10-01T09:10:00.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T09:10:00.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:00.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[185886]: Reconfiguring daemon osd.5 on smithi190 2023-10-01T09:10:00.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[185886]: overall HEALTH_OK 2023-10-01T09:10:00.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:00.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:00.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[192513]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:00.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[192513]: Reconfiguring osd.5 (monmap changed)... 2023-10-01T09:10:00.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T09:10:00.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:00.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[192513]: Reconfiguring daemon osd.5 on smithi190 2023-10-01T09:10:00.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:00 smithi138 ceph-mon[192513]: overall HEALTH_OK 2023-10-01T09:10:01.795 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:01 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:01.795 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:01 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:01.795 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:01 smithi190 ceph-mon[160435]: Reconfiguring osd.6 (monmap changed)... 2023-10-01T09:10:01.795 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:01 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T09:10:01.795 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:01 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:01.795 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:01 smithi190 ceph-mon[160435]: Reconfiguring daemon osd.6 on smithi190 2023-10-01T09:10:01.796 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:01 smithi190 ceph-mon[160435]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:01.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:01 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:01.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:01 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:01.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:01 smithi138 ceph-mon[192513]: Reconfiguring osd.6 (monmap changed)... 2023-10-01T09:10:01.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:01 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T09:10:01.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:01 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:01.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:01 smithi138 ceph-mon[192513]: Reconfiguring daemon osd.6 on smithi190 2023-10-01T09:10:01.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:01 smithi138 ceph-mon[192513]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:01.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:01 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:01.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:01 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:01.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:01 smithi138 ceph-mon[185886]: Reconfiguring osd.6 (monmap changed)... 2023-10-01T09:10:01.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:01 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T09:10:01.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:01 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:01.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:01 smithi138 ceph-mon[185886]: Reconfiguring daemon osd.6 on smithi190 2023-10-01T09:10:01.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:01 smithi138 ceph-mon[185886]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:03.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:02 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:02 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:02 smithi190 ceph-mon[160435]: Reconfiguring osd.7 (monmap changed)... 2023-10-01T09:10:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:02 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T09:10:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:02 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:02 smithi190 ceph-mon[160435]: Reconfiguring daemon osd.7 on smithi190 2023-10-01T09:10:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:02 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:02 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:02 smithi138 ceph-mon[185886]: Reconfiguring osd.7 (monmap changed)... 2023-10-01T09:10:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:02 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T09:10:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:02 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:02 smithi138 ceph-mon[185886]: Reconfiguring daemon osd.7 on smithi190 2023-10-01T09:10:03.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:02 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:03.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:02 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:03.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:02 smithi138 ceph-mon[192513]: Reconfiguring osd.7 (monmap changed)... 2023-10-01T09:10:03.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:02 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T09:10:03.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:02 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:03.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:02 smithi138 ceph-mon[192513]: Reconfiguring daemon osd.7 on smithi190 2023-10-01T09:10:04.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:04 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:04 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:04 smithi190 ceph-mon[160435]: Reconfiguring mds.foo.smithi190.weynjc (monmap changed)... 2023-10-01T09:10:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:04 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi190.weynjc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:10:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:04 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:04 smithi190 ceph-mon[160435]: Reconfiguring daemon mds.foo.smithi190.weynjc on smithi190 2023-10-01T09:10:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:04 smithi190 ceph-mon[160435]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:04.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:04 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:04.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:04 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:04.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:04 smithi138 ceph-mon[185886]: Reconfiguring mds.foo.smithi190.weynjc (monmap changed)... 2023-10-01T09:10:04.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:04 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi190.weynjc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:10:04.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:04 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:04.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:04 smithi138 ceph-mon[185886]: Reconfiguring daemon mds.foo.smithi190.weynjc on smithi190 2023-10-01T09:10:04.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:04 smithi138 ceph-mon[185886]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:04 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:04 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:04 smithi138 ceph-mon[192513]: Reconfiguring mds.foo.smithi190.weynjc (monmap changed)... 2023-10-01T09:10:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:04 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi190.weynjc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T09:10:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:04 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:04 smithi138 ceph-mon[192513]: Reconfiguring daemon mds.foo.smithi190.weynjc on smithi190 2023-10-01T09:10:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:04 smithi138 ceph-mon[192513]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:05.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:10:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:10:04] "GET /metrics HTTP/1.1" 200 33778 "" "Prometheus/2.33.4" 2023-10-01T09:10:05.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:05 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:05 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:05 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:10:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:05 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:05 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:10:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:05 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:05 smithi190 ceph-mon[160435]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:05.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:05 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:05 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:05 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:10:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:05 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:05 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:10:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:05 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:05 smithi138 ceph-mon[192513]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:05 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:05 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:05 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:10:05.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:05 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:05.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:05 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:10:05.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:05 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:05.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:05 smithi138 ceph-mon[185886]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:08.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:07 smithi190 ceph-mon[160435]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:08.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:07 smithi138 ceph-mon[185886]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:07 smithi138 ceph-mon[192513]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:09.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:10:08 smithi190 conmon[151158]: ts=2023-10-01T09:10:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:10:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:10 smithi190 ceph-mon[160435]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:10.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:10:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:10 smithi138 ceph-mon[192513]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:10:10.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:10 smithi138 ceph-mon[185886]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:10.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:10:12.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:12 smithi190 ceph-mon[160435]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:12.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:12 smithi138 ceph-mon[185886]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:12.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:12 smithi138 ceph-mon[192513]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:14 smithi190 ceph-mon[160435]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:14.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:14 smithi138 ceph-mon[192513]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:14 smithi138 ceph-mon[185886]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:15.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:10:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:10:14] "GET /metrics HTTP/1.1" 200 33790 "" "Prometheus/2.33.4" 2023-10-01T09:10:16.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:16 smithi190 ceph-mon[160435]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:16.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:16 smithi138 ceph-mon[192513]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:16.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:16 smithi138 ceph-mon[185886]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:18.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:17 smithi190 ceph-mon[160435]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:18.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:17 smithi138 ceph-mon[185886]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:18.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:17 smithi138 ceph-mon[192513]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:19.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:10:18 smithi190 conmon[151158]: ts=2023-10-01T09:10:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:10:20.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:20 smithi190 ceph-mon[160435]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:20.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:20 smithi138 ceph-mon[192513]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:20.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:20 smithi138 ceph-mon[185886]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:22.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:22 smithi190 ceph-mon[160435]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:22.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:22 smithi138 ceph-mon[192513]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:22.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:22 smithi138 ceph-mon[185886]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:24.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:24 smithi190 ceph-mon[160435]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:24.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:24 smithi138 ceph-mon[192513]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:24.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:24 smithi138 ceph-mon[185886]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:25.275 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:10:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:10:24] "GET /metrics HTTP/1.1" 200 33790 "" "Prometheus/2.33.4" 2023-10-01T09:10:25.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:10:25.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:10:25.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:10:26.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:26 smithi190 ceph-mon[160435]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:26.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:26 smithi138 ceph-mon[185886]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:26 smithi138 ceph-mon[192513]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:28.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:27 smithi190 ceph-mon[160435]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:28.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:27 smithi138 ceph-mon[192513]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:28.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:27 smithi138 ceph-mon[185886]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:29.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:10:28 smithi190 conmon[151158]: ts=2023-10-01T09:10:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:10:30.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:30 smithi190 ceph-mon[160435]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:30.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:30 smithi138 ceph-mon[192513]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:30 smithi138 ceph-mon[185886]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:30.692 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade resume' 2023-10-01T09:10:32.270 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:32 smithi138 ceph-mon[185886]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:32.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:32 smithi190 ceph-mon[160435]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:32.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:32 smithi138 ceph-mon[192513]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:32.707 INFO:teuthology.orchestra.run.smithi138.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:10:33.382 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'echo wait for all mons to upgrade' 2023-10-01T09:10:34.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:33 smithi190 ceph-mon[160435]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:10:34.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:33 smithi190 ceph-mon[160435]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:10:34.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:33 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:34.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:33 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:10:34.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:33 smithi190 ceph-mon[160435]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:34.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:33 smithi138 ceph-mon[192513]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:10:34.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:33 smithi138 ceph-mon[192513]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:10:34.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:33 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:34.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:33 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:10:34.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:33 smithi138 ceph-mon[192513]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:34.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:33 smithi138 ceph-mon[185886]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:10:34.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:33 smithi138 ceph-mon[185886]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:10:34.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:33 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:34.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:33 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:10:34.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:33 smithi138 ceph-mon[185886]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:35.230 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:10:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:10:34] "GET /metrics HTTP/1.1" 200 33718 "" "Prometheus/2.33.4" 2023-10-01T09:10:35.415 INFO:teuthology.orchestra.run.smithi138.stdout:wait for all mons to upgrade 2023-10-01T09:10:36.188 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-10-01T09:10:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:36 smithi190 ceph-mon[160435]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:36.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:36 smithi138 ceph-mon[192513]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:36.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:36 smithi138 ceph-mon[185886]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:38.212 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:37 smithi138 ceph-mon[192513]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:38.212 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:37 smithi138 ceph-mon[185886]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:38.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:37 smithi190 ceph-mon[160435]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:38.712 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 3 2023-10-01T09:10:39.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:10:38 smithi190 conmon[151158]: ts=2023-10-01T09:10:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:10:39.363 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.363 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.363 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:39.363 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:10:39.363 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.363 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:10:39.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:10:39.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:10:39.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-01T09:10:39.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-01T09:10:39.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-01T09:10:39.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-01T09:10:39.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-01T09:10:39.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-01T09:10:39.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:10:39.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-01T09:10:39.365 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/686086948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:10:39.366 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch ps' 2023-10-01T09:10:39.366 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.366 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.366 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:39.366 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:10:39.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:10:39.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:10:39.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:10:39.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-01T09:10:39.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-01T09:10:39.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-01T09:10:39.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-01T09:10:39.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-01T09:10:39.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-01T09:10:39.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:10:39.368 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.368 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-01T09:10:39.368 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:39 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/686086948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:10:39.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:39.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:10:39.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:10:39.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:10:39.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:10:39.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-01T09:10:39.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-01T09:10:39.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-01T09:10:39.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-01T09:10:39.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-01T09:10:39.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-01T09:10:39.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:10:39.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:39.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-01T09:10:39.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:39 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/686086948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T09:10:40.250 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:40 smithi138 ceph-mon[185886]: Upgrade: Setting container_image for all mon 2023-10-01T09:10:40.250 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:40 smithi138 ceph-mon[185886]: Upgrade: Setting container_image for all crash 2023-10-01T09:10:40.251 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:40 smithi138 ceph-mon[185886]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-01T09:10:40.251 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:40 smithi138 ceph-mon[185886]: Upgrade: osd.0 is safe to restart 2023-10-01T09:10:40.251 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:40 smithi138 ceph-mon[185886]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:40.251 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:40 smithi138 ceph-mon[192513]: Upgrade: Setting container_image for all mon 2023-10-01T09:10:40.251 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:40 smithi138 ceph-mon[192513]: Upgrade: Setting container_image for all crash 2023-10-01T09:10:40.251 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:40 smithi138 ceph-mon[192513]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-01T09:10:40.251 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:40 smithi138 ceph-mon[192513]: Upgrade: osd.0 is safe to restart 2023-10-01T09:10:40.251 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:40 smithi138 ceph-mon[192513]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:40.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:40 smithi190 ceph-mon[160435]: Upgrade: Setting container_image for all mon 2023-10-01T09:10:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:40 smithi190 ceph-mon[160435]: Upgrade: Setting container_image for all crash 2023-10-01T09:10:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:40 smithi190 ceph-mon[160435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-01T09:10:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:40 smithi190 ceph-mon[160435]: Upgrade: osd.0 is safe to restart 2023-10-01T09:10:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:40 smithi190 ceph-mon[160435]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:41.546 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:10:41.547 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:10:41.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:10:41.997 INFO:teuthology.orchestra.run.smithi138.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-01T09:10:41.997 INFO:teuthology.orchestra.run.smithi138.stdout:alertmanager.a smithi138 *:9093,9094 running (4m) - 8m 27.1M - 0.23.0 ba2b418f427c 8debc681d94e 2023-10-01T09:10:41.998 INFO:teuthology.orchestra.run.smithi138.stdout:grafana.a smithi190 *:3000 running (3m) - 8m 57.9M - 8.3.5 dad864ee21e9 58ea19a8c949 2023-10-01T09:10:41.998 INFO:teuthology.orchestra.run.smithi138.stdout:mds.foo.smithi138.brhzod smithi138 running (7m) - 7m 19.0M - 16.2.14-44-geeb13ac9 94a6c4498547 555d8c2cfe25 2023-10-01T09:10:41.998 INFO:teuthology.orchestra.run.smithi138.stdout:mds.foo.smithi190.weynjc smithi190 running (7m) - 7m 18.2M - 16.2.14-44-geeb13ac9 94a6c4498547 881fea906eff 2023-10-01T09:10:41.998 INFO:teuthology.orchestra.run.smithi138.stdout:mgr.x smithi190 *:8443,9283,8765 running (3m) - 12m 449M - 18.2.0-497-gb5682219 a3fc96fed5ac 674e1140402f 2023-10-01T09:10:41.998 INFO:teuthology.orchestra.run.smithi138.stdout:mgr.y smithi138 *:8443,9283,8765 running (3m) - 13m 504M - 18.2.0-497-gb5682219 a3fc96fed5ac 97da5dac6a84 2023-10-01T09:10:41.999 INFO:teuthology.orchestra.run.smithi138.stdout:mon.a smithi138 running (2m) - 13m 89.3M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 4f0e28d17710 2023-10-01T09:10:41.999 INFO:teuthology.orchestra.run.smithi138.stdout:mon.b smithi190 running (70s) - 12m 34.0M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac c0ac51d9b5e0 2023-10-01T09:10:41.999 INFO:teuthology.orchestra.run.smithi138.stdout:mon.c smithi138 running (88s) - 12m 72.3M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 9f7c99bda05d 2023-10-01T09:10:41.999 INFO:teuthology.orchestra.run.smithi138.stdout:node-exporter.a smithi138 *:9100 running (4m) - 8m 24.4M - 1.3.1 1dbe0e931976 bd644aa2145b 2023-10-01T09:10:41.999 INFO:teuthology.orchestra.run.smithi138.stdout:node-exporter.b smithi190 *:9100 running (3m) - 8m 26.1M - 1.3.1 1dbe0e931976 c08f7a49a78d 2023-10-01T09:10:41.999 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0 smithi138 running (11m) - 11m 79.0M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 7636a677953c 2023-10-01T09:10:42.000 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1 smithi138 running (11m) - 11m 102M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 a510d44fb2c6 2023-10-01T09:10:42.000 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2 smithi138 running (11m) - 11m 68.4M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 8a8389da7887 2023-10-01T09:10:42.000 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3 smithi138 running (10m) - 10m 69.7M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 c3ba3a01539d 2023-10-01T09:10:42.000 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4 smithi190 running (10m) - 10m 73.3M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 cda54a787614 2023-10-01T09:10:42.000 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5 smithi190 running (10m) - 10m 90.6M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 4ac584c1fb7d 2023-10-01T09:10:42.000 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6 smithi190 running (9m) - 9m 73.0M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 fa45a1c37077 2023-10-01T09:10:42.001 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7 smithi190 running (9m) - 9m 94.5M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 c925b7fc3562 2023-10-01T09:10:42.001 INFO:teuthology.orchestra.run.smithi138.stdout:prometheus.a smithi190 *:9095 running (3m) - 8m 60.1M - 2.33.4 514e6a882f6e bbdb076a24d4 2023-10-01T09:10:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:42 smithi190 ceph-mon[160435]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:42 smithi190 ceph-mon[160435]: Upgrade: Updating osd.0 2023-10-01T09:10:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T09:10:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:42 smithi190 ceph-mon[160435]: Deploying daemon osd.0 on smithi138 2023-10-01T09:10:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:42 smithi138 ceph-mon[192513]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:42 smithi138 ceph-mon[192513]: Upgrade: Updating osd.0 2023-10-01T09:10:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T09:10:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:42 smithi138 ceph-mon[192513]: Deploying daemon osd.0 on smithi138 2023-10-01T09:10:42.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:42 smithi138 ceph-mon[185886]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:42.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:42 smithi138 ceph-mon[185886]: Upgrade: Updating osd.0 2023-10-01T09:10:42.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:42.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T09:10:42.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:10:42.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:42 smithi138 ceph-mon[185886]: Deploying daemon osd.0 on smithi138 2023-10-01T09:10:43.133 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade pause' 2023-10-01T09:10:43.459 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:43 smithi138 ceph-mon[185886]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:10:43.460 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:43 smithi138 ceph-mon[192513]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:10:43.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:43 smithi190 ceph-mon[160435]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:10:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:44 smithi138 ceph-mon[185886]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:44.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:44 smithi138 ceph-mon[192513]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:44 smithi190 ceph-mon[160435]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:44.998 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:10:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:10:44] "GET /metrics HTTP/1.1" 200 33790 "" "Prometheus/2.33.4" 2023-10-01T09:10:45.789 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:45 smithi138 systemd[1]: Stopping Ceph osd.0 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:10:45.963 INFO:teuthology.orchestra.run.smithi138.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:10:46.069 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:45 smithi138 conmon[115078]: debug 2023-10-01T09:10:45.786+0000 7fbf810ce700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T09:10:46.070 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:45 smithi138 conmon[115078]: debug 2023-10-01T09:10:45.786+0000 7fbf810ce700 -1 osd.0 85 *** Got signal Terminated *** 2023-10-01T09:10:46.070 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:45 smithi138 conmon[115078]: debug 2023-10-01T09:10:45.786+0000 7fbf810ce700 -1 osd.0 85 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:10:46.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:46 smithi190 ceph-mon[160435]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:46.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:46 smithi190 ceph-mon[160435]: osd.0 marked itself down and dead 2023-10-01T09:10:46.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:46 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:46 smithi138 ceph-mon[192513]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:46 smithi138 ceph-mon[192513]: osd.0 marked itself down and dead 2023-10-01T09:10:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:46 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:46 smithi138 ceph-mon[185886]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:46 smithi138 ceph-mon[185886]: osd.0 marked itself down and dead 2023-10-01T09:10:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:46 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:47.094 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'sleep 60' 2023-10-01T09:10:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:47 smithi190 ceph-mon[160435]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:10:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:47 smithi190 ceph-mon[160435]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:10:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:47 smithi190 ceph-mon[160435]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:10:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:47 smithi190 ceph-mon[160435]: failed to encode map e86 with expected crc 2023-10-01T09:10:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:47 smithi190 ceph-mon[160435]: osdmap e86: 8 total, 7 up, 8 in 2023-10-01T09:10:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:47 smithi190 ceph-mon[160435]: failed to encode map e86 with expected crc 2023-10-01T09:10:47.569 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:47 smithi138 bash[201259]: 7636a677953ca34e09ca9086ce7f1b7d58b5b5a2e47e373e05fc29a055daf8e8 2023-10-01T09:10:47.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:47 smithi138 ceph-mon[185886]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:10:47.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:47 smithi138 ceph-mon[185886]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:10:47.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:47 smithi138 ceph-mon[185886]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:10:47.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:47 smithi138 ceph-mon[185886]: failed to encode map e86 with expected crc 2023-10-01T09:10:47.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:47 smithi138 ceph-mon[185886]: osdmap e86: 8 total, 7 up, 8 in 2023-10-01T09:10:47.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:47 smithi138 ceph-mon[185886]: failed to encode map e86 with expected crc 2023-10-01T09:10:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:47 smithi138 ceph-mon[192513]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:10:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:47 smithi138 ceph-mon[192513]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:10:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:47 smithi138 ceph-mon[192513]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:10:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:47 smithi138 ceph-mon[192513]: failed to encode map e86 with expected crc 2023-10-01T09:10:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:47 smithi138 ceph-mon[192513]: osdmap e86: 8 total, 7 up, 8 in 2023-10-01T09:10:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:47 smithi138 ceph-mon[192513]: failed to encode map e86 with expected crc 2023-10-01T09:10:48.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:48 smithi190 ceph-mon[160435]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:48 smithi190 ceph-mon[160435]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:48 smithi190 ceph-mon[160435]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:48 smithi190 ceph-mon[160435]: osdmap e87: 8 total, 7 up, 8 in 2023-10-01T09:10:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:48 smithi190 ceph-mon[160435]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:48 smithi190 ceph-mon[160435]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:48 smithi190 ceph-mon[160435]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:48 smithi190 ceph-mon[160435]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:48 smithi190 ceph-mon[160435]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:48 smithi190 ceph-mon[160435]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:48 smithi190 ceph-mon[160435]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[185886]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:48.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[185886]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[185886]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[185886]: osdmap e87: 8 total, 7 up, 8 in 2023-10-01T09:10:48.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[185886]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[185886]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[185886]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[185886]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[185886]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[185886]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[185886]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[192513]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:48.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[192513]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[192513]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[192513]: osdmap e87: 8 total, 7 up, 8 in 2023-10-01T09:10:48.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[192513]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[192513]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[192513]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[192513]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[192513]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[192513]: failed to encode map e87 with expected crc 2023-10-01T09:10:48.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:48 smithi138 ceph-mon[192513]: failed to encode map e87 with expected crc 2023-10-01T09:10:49.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:10:48 smithi190 conmon[151158]: ts=2023-10-01T09:10:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:10:50.070 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:49 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-01T09:10:50.070 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:49 smithi138 systemd[1]: Stopped Ceph osd.0 for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:10:50.070 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:49 smithi138 systemd[1]: Starting Ceph osd.0 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:10:50.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:50 smithi190 ceph-mon[160435]: pgmap v92: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:50.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:50 smithi138 ceph-mon[185886]: pgmap v92: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:50.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:50 smithi138 ceph-mon[192513]: pgmap v92: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:10:51.156 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:50 smithi138 bash[202219]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-01T09:10:51.157 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:50 smithi138 bash[202219]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-01T09:10:51.157 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:50 smithi138 bash[202219]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-01T09:10:51.157 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:50 smithi138 bash[202219]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-01T09:10:51.157 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:50 smithi138 bash[202219]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-01T09:10:51.157 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:50 smithi138 bash[202219]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-01T09:10:51.157 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:50 smithi138 bash[202219]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-01T09:10:51.157 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:50 smithi138 bash[202219]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-01T09:10:51.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:51 smithi138 ceph-mon[185886]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T09:10:51.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:51 smithi138 ceph-mon[192513]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T09:10:51.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:51 smithi190 ceph-mon[160435]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T09:10:52.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:52 smithi138 ceph-mon[185886]: pgmap v93: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:10:52.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:52 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:52.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:52 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:52.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:52 smithi138 ceph-mon[192513]: pgmap v93: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:10:52.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:52 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:52.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:52 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:52.572 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:52 smithi138 ceph-osd[202707]: -- 172.21.15.138:0/2097286667 <== mon.0 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55aa931e44e0 con 0x55aa931b6c00 2023-10-01T09:10:52.572 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:52 smithi138 bash[202644]: cb94dc6c4a672007131dbce497dc9249e0fdbc74b6e180b986a2d1473de1b895 2023-10-01T09:10:52.572 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:52 smithi138 systemd[1]: Started Ceph osd.0 for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:10:52.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:52 smithi190 ceph-mon[160435]: pgmap v93: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:10:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:52 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:52 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:53.070 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:52 smithi138 conmon[202683]: 2023-10-01T09:10:52.786+0000 7fdd309ce640 -1 Falling back to public interface 2023-10-01T09:10:53.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:53 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:10:53.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:53 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:10:53.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:53 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:10:54.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:54 smithi190 ceph-mon[160435]: pgmap v94: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:10:54.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:54 smithi138 ceph-mon[192513]: pgmap v94: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:10:54.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:54 smithi138 ceph-mon[185886]: pgmap v94: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:10:55.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:10:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:10:54] "GET /metrics HTTP/1.1" 200 33790 "" "Prometheus/2.33.4" 2023-10-01T09:10:56.320 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:56 smithi138 conmon[202683]: 2023-10-01T09:10:56.045+0000 7fdd309ce640 -1 osd.0 85 log_to_monitors true 2023-10-01T09:10:56.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[185886]: pgmap v95: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:10:56.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:10:56.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[185886]: from='osd.0 [v2:172.21.15.138:6802/2668617192,v1:172.21.15.138:6803/2668617192]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T09:10:56.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[185886]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T09:10:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[192513]: pgmap v95: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:10:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:10:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[192513]: from='osd.0 [v2:172.21.15.138:6802/2668617192,v1:172.21.15.138:6803/2668617192]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T09:10:56.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:56 smithi138 ceph-mon[192513]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T09:10:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:56 smithi190 ceph-mon[160435]: pgmap v95: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:10:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:10:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:10:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:56 smithi190 ceph-mon[160435]: from='osd.0 [v2:172.21.15.138:6802/2668617192,v1:172.21.15.138:6803/2668617192]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T09:10:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:56 smithi190 ceph-mon[160435]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T09:10:57.069 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:10:56 smithi138 conmon[202683]: 2023-10-01T09:10:56.802+0000 7fdd1de32700 -1 osd.0 85 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T09:10:58.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:57 smithi190 ceph-mon[160435]: failed to encode map e88 with expected crc 2023-10-01T09:10:58.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:57 smithi190 ceph-mon[160435]: failed to encode map e88 with expected crc 2023-10-01T09:10:58.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:57 smithi190 ceph-mon[160435]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T09:10:58.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:57 smithi190 ceph-mon[160435]: osdmap e88: 8 total, 7 up, 8 in 2023-10-01T09:10:58.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:57 smithi190 ceph-mon[160435]: from='osd.0 [v2:172.21.15.138:6802/2668617192,v1:172.21.15.138:6803/2668617192]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:10:58.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:57 smithi190 ceph-mon[160435]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:10:58.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:57 smithi190 ceph-mon[160435]: pgmap v97: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:10:58.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:57 smithi138 ceph-mon[185886]: failed to encode map e88 with expected crc 2023-10-01T09:10:58.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:57 smithi138 ceph-mon[185886]: failed to encode map e88 with expected crc 2023-10-01T09:10:58.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:57 smithi138 ceph-mon[185886]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T09:10:58.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:57 smithi138 ceph-mon[185886]: osdmap e88: 8 total, 7 up, 8 in 2023-10-01T09:10:58.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:57 smithi138 ceph-mon[185886]: from='osd.0 [v2:172.21.15.138:6802/2668617192,v1:172.21.15.138:6803/2668617192]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:10:58.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:57 smithi138 ceph-mon[185886]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:10:58.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:57 smithi138 ceph-mon[185886]: pgmap v97: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:10:58.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:57 smithi138 ceph-mon[192513]: failed to encode map e88 with expected crc 2023-10-01T09:10:58.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:57 smithi138 ceph-mon[192513]: failed to encode map e88 with expected crc 2023-10-01T09:10:58.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:57 smithi138 ceph-mon[192513]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T09:10:58.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:57 smithi138 ceph-mon[192513]: osdmap e88: 8 total, 7 up, 8 in 2023-10-01T09:10:58.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:57 smithi138 ceph-mon[192513]: from='osd.0 [v2:172.21.15.138:6802/2668617192,v1:172.21.15.138:6803/2668617192]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:10:58.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:57 smithi138 ceph-mon[192513]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:10:58.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:57 smithi138 ceph-mon[192513]: pgmap v97: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:10:59.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:10:58 smithi190 conmon[151158]: ts=2023-10-01T09:10:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:10:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:58 smithi190 ceph-mon[160435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:10:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:58 smithi190 ceph-mon[160435]: failed to encode map e89 with expected crc 2023-10-01T09:10:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:58 smithi190 ceph-mon[160435]: failed to encode map e89 with expected crc 2023-10-01T09:10:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:58 smithi190 ceph-mon[160435]: osd.0 [v2:172.21.15.138:6802/2668617192,v1:172.21.15.138:6803/2668617192] boot 2023-10-01T09:10:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:58 smithi190 ceph-mon[160435]: osdmap e89: 8 total, 8 up, 8 in 2023-10-01T09:10:59.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T09:10:59.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:58 smithi138 ceph-mon[192513]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:10:59.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:58 smithi138 ceph-mon[192513]: failed to encode map e89 with expected crc 2023-10-01T09:10:59.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:58 smithi138 ceph-mon[192513]: failed to encode map e89 with expected crc 2023-10-01T09:10:59.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:58 smithi138 ceph-mon[192513]: osd.0 [v2:172.21.15.138:6802/2668617192,v1:172.21.15.138:6803/2668617192] boot 2023-10-01T09:10:59.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:58 smithi138 ceph-mon[192513]: osdmap e89: 8 total, 8 up, 8 in 2023-10-01T09:10:59.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T09:10:59.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:58 smithi138 ceph-mon[185886]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:10:59.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:58 smithi138 ceph-mon[185886]: failed to encode map e89 with expected crc 2023-10-01T09:10:59.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:58 smithi138 ceph-mon[185886]: failed to encode map e89 with expected crc 2023-10-01T09:10:59.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:58 smithi138 ceph-mon[185886]: osd.0 [v2:172.21.15.138:6802/2668617192,v1:172.21.15.138:6803/2668617192] boot 2023-10-01T09:10:59.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:58 smithi138 ceph-mon[185886]: osdmap e89: 8 total, 8 up, 8 in 2023-10-01T09:10:59.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T09:11:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: osdmap e90: 8 total, 8 up, 8 in 2023-10-01T09:11:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:11:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:11:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:11:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:11:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:11:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:10:59 smithi190 ceph-mon[160435]: pgmap v100: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:11:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: osdmap e90: 8 total, 8 up, 8 in 2023-10-01T09:11:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:11:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:11:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:11:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:11:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:11:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:00.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[192513]: pgmap v100: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:11:00.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: osdmap e90: 8 total, 8 up, 8 in 2023-10-01T09:11:00.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: failed to encode map e90 with expected crc 2023-10-01T09:11:00.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:00.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:00.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:11:00.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:11:00.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:00.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:11:00.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:11:00.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:11:00.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:00.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:10:59 smithi138 ceph-mon[185886]: pgmap v100: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-10-01T09:11:01.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:01 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:11:01.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:01 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:11:01.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:01 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:11:02.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:02 smithi190 ceph-mon[160435]: pgmap v101: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-01T09:11:02.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:02 smithi138 ceph-mon[192513]: pgmap v101: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-01T09:11:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:02 smithi138 ceph-mon[185886]: pgmap v101: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-10-01T09:11:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:03 smithi190 ceph-mon[160435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-10-01T09:11:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:03 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:11:03.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:03 smithi138 ceph-mon[185886]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-10-01T09:11:03.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:03 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:11:03.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:03 smithi138 ceph-mon[192513]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-10-01T09:11:03.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:03 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:11:04.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:04 smithi190 ceph-mon[160435]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:04.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:04 smithi138 ceph-mon[192513]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:04.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:04 smithi138 ceph-mon[185886]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:05.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:11:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:11:04] "GET /metrics HTTP/1.1" 200 33917 "" "Prometheus/2.33.4" 2023-10-01T09:11:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:06 smithi190 ceph-mon[160435]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:06 smithi138 ceph-mon[185886]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:06 smithi138 ceph-mon[192513]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:08.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:07 smithi190 ceph-mon[160435]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:08.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:07 smithi138 ceph-mon[192513]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:08.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:07 smithi138 ceph-mon[185886]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:09.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:11:08 smithi190 conmon[151158]: ts=2023-10-01T09:11:08.659Z caller=alerting.go:353 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2023-10-01T09:11:09.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:11:08 smithi190 conmon[151158]: ts=2023-10-01T09:11:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:11:10.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:10 smithi190 ceph-mon[160435]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:11:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:10 smithi138 ceph-mon[192513]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:11:10.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:10 smithi138 ceph-mon[185886]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:10.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:10.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:11:11.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:11.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:12.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:12 smithi190 ceph-mon[160435]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:12.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:12 smithi138 ceph-mon[192513]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:12 smithi138 ceph-mon[185886]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:14 smithi190 ceph-mon[160435]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:14.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:14 smithi138 ceph-mon[192513]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:14 smithi138 ceph-mon[185886]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:15.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:11:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:11:14] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.33.4" 2023-10-01T09:11:16.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:16 smithi190 ceph-mon[160435]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:16.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:16 smithi138 ceph-mon[192513]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:16.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:16 smithi138 ceph-mon[185886]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:18.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:17 smithi190 ceph-mon[160435]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:18.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:17 smithi138 ceph-mon[192513]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:18.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:17 smithi138 ceph-mon[185886]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:19.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:11:18 smithi190 conmon[151158]: ts=2023-10-01T09:11:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:11:20.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:20 smithi190 ceph-mon[160435]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:20.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:20 smithi138 ceph-mon[192513]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:20.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:20 smithi138 ceph-mon[185886]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:22.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:22 smithi190 ceph-mon[160435]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:22.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:22 smithi138 ceph-mon[192513]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:22.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:22 smithi138 ceph-mon[185886]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:24.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:24 smithi190 ceph-mon[160435]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:24.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:24 smithi138 ceph-mon[192513]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:24.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:24 smithi138 ceph-mon[185886]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:25.293 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:11:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:11:24] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.33.4" 2023-10-01T09:11:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:11:25.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:11:25.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:11:26.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:26 smithi190 ceph-mon[160435]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:26.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:26 smithi138 ceph-mon[192513]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:26 smithi138 ceph-mon[185886]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:28.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:27 smithi190 ceph-mon[160435]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:28.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:27 smithi138 ceph-mon[192513]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:28.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:27 smithi138 ceph-mon[185886]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:29.056 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:11:28 smithi190 conmon[151158]: ts=2023-10-01T09:11:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi138:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.138\", device_class=\"ssd\", hostname=\"smithi138\", instance=\"smithi190.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.138\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T09:11:30.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:30 smithi190 ceph-mon[160435]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:30.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:30 smithi138 ceph-mon[185886]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:30.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:30 smithi138 ceph-mon[192513]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:32.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:32 smithi190 ceph-mon[160435]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:32.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:32 smithi138 ceph-mon[192513]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:32 smithi138 ceph-mon[185886]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:34 smithi190 ceph-mon[160435]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:34.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:34 smithi138 ceph-mon[192513]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:34.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:34 smithi138 ceph-mon[185886]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:35.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:11:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:11:34] "GET /metrics HTTP/1.1" 200 33911 "" "Prometheus/2.33.4" 2023-10-01T09:11:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:36 smithi190 ceph-mon[160435]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:36.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:36 smithi138 ceph-mon[185886]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:36.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:36 smithi138 ceph-mon[192513]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:38.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:37 smithi190 ceph-mon[160435]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:38.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:37 smithi138 ceph-mon[192513]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:38.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:37 smithi138 ceph-mon[185886]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:40.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:40 smithi190 ceph-mon[160435]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:11:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:40 smithi138 ceph-mon[192513]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:11:40.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:40 smithi138 ceph-mon[185886]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:40.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:11:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:42 smithi190 ceph-mon[160435]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:42.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:42 smithi138 ceph-mon[185886]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:42 smithi138 ceph-mon[192513]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:44 smithi190 ceph-mon[160435]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:44.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:44 smithi138 ceph-mon[185886]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:44.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:44 smithi138 ceph-mon[192513]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:45.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:11:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:11:44] "GET /metrics HTTP/1.1" 200 33914 "" "Prometheus/2.33.4" 2023-10-01T09:11:46.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:46 smithi190 ceph-mon[160435]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:46.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:46 smithi138 ceph-mon[192513]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:46 smithi138 ceph-mon[185886]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:48.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:47 smithi190 ceph-mon[160435]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:48.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:47 smithi138 ceph-mon[192513]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:48.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:47 smithi138 ceph-mon[185886]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:49.614 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade resume' 2023-10-01T09:11:50.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:50 smithi138 ceph-mon[192513]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:50.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:50 smithi138 ceph-mon[185886]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:50 smithi190 ceph-mon[160435]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:51.637 INFO:teuthology.orchestra.run.smithi138.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:11:52.342 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'echo wait for half of osds to upgrade' 2023-10-01T09:11:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:52 smithi190 ceph-mon[160435]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:52 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:52 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:11:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:52 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:11:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:52 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:11:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:52 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:52 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:11:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:52 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:11:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:52 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:11:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:52 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:11:52.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:52 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T09:11:52.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[185886]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:52.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:52.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:11:52.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:11:52.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:11:52.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:52.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:11:52.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:11:52.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:11:52.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:11:52.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T09:11:52.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[192513]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:52.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:52.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:11:52.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:11:52.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:11:52.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:52.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:11:52.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:11:52.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:11:52.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:11:52.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:52 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T09:11:53.546 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:53 smithi138 ceph-mon[192513]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:11:53.546 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:53 smithi138 ceph-mon[192513]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:11:53.546 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:53 smithi138 ceph-mon[192513]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T09:11:53.546 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:53 smithi138 ceph-mon[192513]: Upgrade: osd.1 is safe to restart 2023-10-01T09:11:53.546 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:53 smithi138 ceph-mon[185886]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:11:53.547 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:53 smithi138 ceph-mon[185886]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:11:53.547 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:53 smithi138 ceph-mon[185886]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T09:11:53.547 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:53 smithi138 ceph-mon[185886]: Upgrade: osd.1 is safe to restart 2023-10-01T09:11:53.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:53 smithi190 ceph-mon[160435]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:11:53.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:53 smithi190 ceph-mon[160435]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:11:53.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:53 smithi190 ceph-mon[160435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T09:11:53.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:53 smithi190 ceph-mon[160435]: Upgrade: osd.1 is safe to restart 2023-10-01T09:11:54.296 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:54 smithi138 ceph-mon[192513]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:54.296 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:54 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:54.296 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:54 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T09:11:54.297 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:54 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:11:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:54 smithi190 ceph-mon[160435]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:54 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:54 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T09:11:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:54 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:11:54.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:54 smithi138 ceph-mon[185886]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:54.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:54 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:11:54.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:54 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T09:11:54.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:54 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:11:55.301 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:11:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:11:54] "GET /metrics HTTP/1.1" 200 33914 "" "Prometheus/2.33.4" 2023-10-01T09:11:55.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:55 smithi138 ceph-mon[185886]: Upgrade: Updating osd.1 2023-10-01T09:11:55.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:55 smithi138 ceph-mon[185886]: Deploying daemon osd.1 on smithi138 2023-10-01T09:11:55.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:11:55.503 INFO:teuthology.orchestra.run.smithi138.stdout:wait for half of osds to upgrade 2023-10-01T09:11:55.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:55 smithi190 ceph-mon[160435]: Upgrade: Updating osd.1 2023-10-01T09:11:55.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:55 smithi190 ceph-mon[160435]: Deploying daemon osd.1 on smithi138 2023-10-01T09:11:55.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:11:55.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:55 smithi138 ceph-mon[192513]: Upgrade: Updating osd.1 2023-10-01T09:11:55.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:55 smithi138 ceph-mon[192513]: Deploying daemon osd.1 on smithi138 2023-10-01T09:11:55.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:11:56.369 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-10-01T09:11:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:56 smithi190 ceph-mon[160435]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:56.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:56 smithi138 ceph-mon[185886]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:56 smithi138 ceph-mon[192513]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:58.227 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:57 smithi138 ceph-mon[192513]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:58.227 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:57 smithi138 ceph-mon[185886]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:58.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:57 smithi190 ceph-mon[160435]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:11:58.570 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:11:58 smithi138 systemd[1]: Stopping Ceph osd.1 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:11:58.900 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:11:58 smithi138 conmon[120861]: debug 2023-10-01T09:11:58.605+0000 7fdb67bc7700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T09:11:58.901 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:11:58 smithi138 conmon[120861]: debug 2023-10-01T09:11:58.605+0000 7fdb67bc7700 -1 osd.1 90 *** Got signal Terminated *** 2023-10-01T09:11:58.901 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:11:58 smithi138 conmon[120861]: debug 2023-10-01T09:11:58.605+0000 7fdb67bc7700 -1 osd.1 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:11:59.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:58 smithi190 ceph-mon[160435]: osd.1 marked itself down and dead 2023-10-01T09:11:59.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:58 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2603257929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:11:59.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:58 smithi138 ceph-mon[185886]: osd.1 marked itself down and dead 2023-10-01T09:11:59.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:58 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2603257929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:11:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:58 smithi138 ceph-mon[192513]: osd.1 marked itself down and dead 2023-10-01T09:11:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:58 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2603257929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:11:59.901 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:11:59 smithi138 bash[207237]: a510d44fb2c68734e2902e467a4db58c331c9b6fd4683da982c3b16f716d172d 2023-10-01T09:11:59.902 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:59 smithi138 ceph-mon[185886]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:11:59.902 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:59 smithi138 ceph-mon[185886]: failed to encode map e91 with expected crc 2023-10-01T09:11:59.902 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:59 smithi138 ceph-mon[185886]: osdmap e91: 8 total, 7 up, 8 in 2023-10-01T09:11:59.903 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:59 smithi138 ceph-mon[185886]: failed to encode map e91 with expected crc 2023-10-01T09:11:59.903 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:11:59 smithi138 ceph-mon[185886]: pgmap v131: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:00.211 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:59 smithi138 ceph-mon[192513]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:12:00.211 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:59 smithi138 ceph-mon[192513]: failed to encode map e91 with expected crc 2023-10-01T09:12:00.211 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:59 smithi138 ceph-mon[192513]: osdmap e91: 8 total, 7 up, 8 in 2023-10-01T09:12:00.211 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:59 smithi138 ceph-mon[192513]: failed to encode map e91 with expected crc 2023-10-01T09:12:00.211 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:11:59 smithi138 ceph-mon[192513]: pgmap v131: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:00.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:59 smithi190 ceph-mon[160435]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:12:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:59 smithi190 ceph-mon[160435]: failed to encode map e91 with expected crc 2023-10-01T09:12:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:59 smithi190 ceph-mon[160435]: osdmap e91: 8 total, 7 up, 8 in 2023-10-01T09:12:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:59 smithi190 ceph-mon[160435]: failed to encode map e91 with expected crc 2023-10-01T09:12:00.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:11:59 smithi190 ceph-mon[160435]: pgmap v131: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:00.914 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[185886]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:00 smithi190 ceph-mon[160435]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:00 smithi190 ceph-mon[160435]: osdmap e92: 8 total, 7 up, 8 in 2023-10-01T09:12:01.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:00 smithi190 ceph-mon[160435]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:00 smithi190 ceph-mon[160435]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:00 smithi190 ceph-mon[160435]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:00 smithi190 ceph-mon[160435]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:00 smithi190 ceph-mon[160435]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:00 smithi190 ceph-mon[160435]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:00 smithi190 ceph-mon[160435]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:00 smithi190 ceph-mon[160435]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[192513]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[192513]: osdmap e92: 8 total, 7 up, 8 in 2023-10-01T09:12:01.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[192513]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[192513]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[192513]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[192513]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[192513]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[192513]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[192513]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[192513]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.314 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[185886]: osdmap e92: 8 total, 7 up, 8 in 2023-10-01T09:12:01.314 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[185886]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.314 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[185886]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.314 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[185886]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.314 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[185886]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.314 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[185886]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.314 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[185886]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.314 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[185886]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.314 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:00 smithi138 ceph-mon[185886]: failed to encode map e92 with expected crc 2023-10-01T09:12:01.570 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:01 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-01T09:12:01.570 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:01 smithi138 systemd[1]: Stopped Ceph osd.1 for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:12:01.570 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:01 smithi138 systemd[1]: Starting Ceph osd.1 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:12:02.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1196369763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:02.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:01 smithi190 ceph-mon[160435]: pgmap v133: 73 pgs: 4 active+undersized+degraded, 1 stale+active+clean, 24 active+undersized, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:12:02.317 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1196369763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:02.317 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:01 smithi138 ceph-mon[192513]: pgmap v133: 73 pgs: 4 active+undersized+degraded, 1 stale+active+clean, 24 active+undersized, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:12:02.317 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1196369763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:02.317 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:01 smithi138 ceph-mon[185886]: pgmap v133: 73 pgs: 4 active+undersized+degraded, 1 stale+active+clean, 24 active+undersized, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:12:02.570 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:02 smithi138 bash[207927]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-01T09:12:02.570 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:02 smithi138 bash[207927]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-01T09:12:02.570 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:02 smithi138 bash[207927]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-01T09:12:02.570 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:02 smithi138 bash[207927]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-01T09:12:02.570 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:02 smithi138 bash[207927]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-01T09:12:02.571 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:02 smithi138 bash[207927]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-01T09:12:02.571 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:02 smithi138 bash[207927]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-01T09:12:02.571 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:02 smithi138 bash[207927]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-01T09:12:03.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:02 smithi190 ceph-mon[160435]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T09:12:03.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:02 smithi138 ceph-mon[192513]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T09:12:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:02 smithi138 ceph-mon[185886]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T09:12:03.819 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:03 smithi138 ceph-osd[208426]: -- 172.21.15.138:0/2437817095 <== mon.0 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5606aa3944e0 con 0x5606aa366c00 2023-10-01T09:12:03.820 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:03 smithi138 bash[208338]: 6abadff62084753562edc3f5a2adfc3846033b7df91c7009afa0bbee644709e3 2023-10-01T09:12:03.820 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:03 smithi138 systemd[1]: Started Ceph osd.1 for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:12:04.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:03 smithi190 ceph-mon[160435]: pgmap v134: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-01T09:12:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:03 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3686506360' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:03 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:03 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:03 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:12:04.319 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:04 smithi138 conmon[208402]: 2023-10-01T09:12:04.095+0000 7fb2d3ab0640 -1 Falling back to public interface 2023-10-01T09:12:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:03 smithi138 ceph-mon[185886]: pgmap v134: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-01T09:12:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:03 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3686506360' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:03 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:03 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:03 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:12:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:03 smithi138 ceph-mon[192513]: pgmap v134: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-01T09:12:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:03 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3686506360' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:03 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:03 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:03 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:12:05.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:12:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:12:04] "GET /metrics HTTP/1.1" 200 33842 "" "Prometheus/2.33.4" 2023-10-01T09:12:06.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:06 smithi190 ceph-mon[160435]: pgmap v135: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-01T09:12:06.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:06 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3664800100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:06.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:06 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:06.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:06 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:06.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:06 smithi138 ceph-mon[185886]: pgmap v135: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-01T09:12:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:06 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3664800100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:06 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:06 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:06.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:06 smithi138 ceph-mon[192513]: pgmap v135: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-01T09:12:06.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:06 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3664800100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:06.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:06 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:06.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:06 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:07.510 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:07 smithi138 conmon[208402]: 2023-10-01T09:12:07.093+0000 7fb2d3ab0640 -1 osd.1 90 log_to_monitors true 2023-10-01T09:12:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:07 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:07.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:07 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:07.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:07 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:07.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:07 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:07.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:07 smithi138 ceph-mon[192513]: from='osd.1 [v2:172.21.15.138:6810/2021985596,v1:172.21.15.138:6811/2021985596]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T09:12:07.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:07 smithi138 ceph-mon[192513]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T09:12:07.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:07 smithi138 ceph-mon[192513]: pgmap v136: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-01T09:12:07.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:07 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:07.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:07 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:07.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:07 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:07.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:07 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:07.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:07 smithi138 ceph-mon[185886]: from='osd.1 [v2:172.21.15.138:6810/2021985596,v1:172.21.15.138:6811/2021985596]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T09:12:07.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:07 smithi138 ceph-mon[185886]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T09:12:07.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:07 smithi138 ceph-mon[185886]: pgmap v136: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-01T09:12:08.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:07 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:08.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:07 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:08.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:07 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:08.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:07 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:08.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:07 smithi190 ceph-mon[160435]: from='osd.1 [v2:172.21.15.138:6810/2021985596,v1:172.21.15.138:6811/2021985596]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T09:12:08.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:07 smithi190 ceph-mon[160435]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T09:12:08.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:07 smithi190 ceph-mon[160435]: pgmap v136: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-01T09:12:08.319 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:12:08 smithi138 conmon[208402]: 2023-10-01T09:12:08.006+0000 7fb2c0f14700 -1 osd.1 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T09:12:09.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[192513]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[192513]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T09:12:09.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[192513]: osdmap e93: 8 total, 7 up, 8 in 2023-10-01T09:12:09.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[192513]: from='osd.1 [v2:172.21.15.138:6810/2021985596,v1:172.21.15.138:6811/2021985596]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:12:09.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[192513]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:12:09.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[192513]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[192513]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[192513]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[192513]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[192513]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/311581219' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:09.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[185886]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[185886]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T09:12:09.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[185886]: osdmap e93: 8 total, 7 up, 8 in 2023-10-01T09:12:09.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[185886]: from='osd.1 [v2:172.21.15.138:6810/2021985596,v1:172.21.15.138:6811/2021985596]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:12:09.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[185886]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:12:09.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[185886]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[185886]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[185886]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[185886]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[185886]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:08 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/311581219' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:08 smithi190 ceph-mon[160435]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:08 smithi190 ceph-mon[160435]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T09:12:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:08 smithi190 ceph-mon[160435]: osdmap e93: 8 total, 7 up, 8 in 2023-10-01T09:12:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:08 smithi190 ceph-mon[160435]: from='osd.1 [v2:172.21.15.138:6810/2021985596,v1:172.21.15.138:6811/2021985596]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:12:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:08 smithi190 ceph-mon[160435]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:12:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:08 smithi190 ceph-mon[160435]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:08 smithi190 ceph-mon[160435]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:08 smithi190 ceph-mon[160435]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:08 smithi190 ceph-mon[160435]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:08 smithi190 ceph-mon[160435]: failed to encode map e93 with expected crc 2023-10-01T09:12:09.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:08 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/311581219' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:10.285 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:10 smithi138 ceph-mon[192513]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:12:10.286 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:10 smithi138 ceph-mon[192513]: failed to encode map e94 with expected crc 2023-10-01T09:12:10.286 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:10 smithi138 ceph-mon[192513]: osd.1 [v2:172.21.15.138:6810/2021985596,v1:172.21.15.138:6811/2021985596] boot 2023-10-01T09:12:10.286 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:10 smithi138 ceph-mon[192513]: osdmap e94: 8 total, 8 up, 8 in 2023-10-01T09:12:10.286 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T09:12:10.286 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:10 smithi138 ceph-mon[192513]: pgmap v139: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-01T09:12:10.286 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:10 smithi138 ceph-mon[192513]: osdmap e95: 8 total, 8 up, 8 in 2023-10-01T09:12:10.287 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:12:10 smithi138 conmon[178630]: 2023-10-01T09:12:10.283+0000 7fa004e48700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-01T09:12:10.287 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:10 smithi138 ceph-mon[185886]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:12:10.287 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:10 smithi138 ceph-mon[185886]: failed to encode map e94 with expected crc 2023-10-01T09:12:10.287 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:10 smithi138 ceph-mon[185886]: osd.1 [v2:172.21.15.138:6810/2021985596,v1:172.21.15.138:6811/2021985596] boot 2023-10-01T09:12:10.287 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:10 smithi138 ceph-mon[185886]: osdmap e94: 8 total, 8 up, 8 in 2023-10-01T09:12:10.287 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T09:12:10.287 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:10 smithi138 ceph-mon[185886]: pgmap v139: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-01T09:12:10.288 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:10 smithi138 ceph-mon[185886]: osdmap e95: 8 total, 8 up, 8 in 2023-10-01T09:12:10.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:10 smithi190 ceph-mon[160435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:12:10.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:10 smithi190 ceph-mon[160435]: failed to encode map e94 with expected crc 2023-10-01T09:12:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:10 smithi190 ceph-mon[160435]: osd.1 [v2:172.21.15.138:6810/2021985596,v1:172.21.15.138:6811/2021985596] boot 2023-10-01T09:12:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:10 smithi190 ceph-mon[160435]: osdmap e94: 8 total, 8 up, 8 in 2023-10-01T09:12:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T09:12:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:10 smithi190 ceph-mon[160435]: pgmap v139: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-10-01T09:12:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:10 smithi190 ceph-mon[160435]: osdmap e95: 8 total, 8 up, 8 in 2023-10-01T09:12:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:11.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:11.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:12:11.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:12:11.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:11.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:11.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:12:11.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:12:11.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:11.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:11.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:11.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T09:12:11.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T09:12:11.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-01T09:12:11.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1072799996' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:11.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:11.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:12:11.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:12:11.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:11.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:11.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:12:11.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:12:11.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:11.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:11.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:11.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T09:12:11.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T09:12:11.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-01T09:12:11.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1072799996' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:11.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: failed to encode map e95 with expected crc 2023-10-01T09:12:11.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:11.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:11.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:12:11.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:12:11.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:11.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:11.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:12:11.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:12:11.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:11.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:11.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:11.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T09:12:11.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T09:12:11.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-01T09:12:11.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1072799996' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:12 smithi190 ceph-mon[160435]: pgmap v141: 73 pgs: 2 active+undersized+degraded, 10 active+undersized, 61 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-01T09:12:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:12 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:12:12.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:12 smithi138 ceph-mon[185886]: pgmap v141: 73 pgs: 2 active+undersized+degraded, 10 active+undersized, 61 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-01T09:12:12.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:12 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:12:12.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:12 smithi138 ceph-mon[192513]: pgmap v141: 73 pgs: 2 active+undersized+degraded, 10 active+undersized, 61 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-01T09:12:12.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:12 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:12:13.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:13 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2528365840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:13.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:13 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2528365840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:13.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:13 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2528365840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:14.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:14 smithi190 ceph-mon[160435]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:14.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:14 smithi138 ceph-mon[192513]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:14 smithi138 ceph-mon[185886]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:15.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:12:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:12:14] "GET /metrics HTTP/1.1" 200 33761 "" "Prometheus/2.33.4" 2023-10-01T09:12:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:15 smithi138 ceph-mon[185886]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-10-01T09:12:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:15 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:12:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:15 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4066085228' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:15.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:15 smithi138 ceph-mon[192513]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-10-01T09:12:15.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:15 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:12:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:15 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4066085228' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:15.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:15 smithi190 ceph-mon[160435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-10-01T09:12:15.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:15 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:12:15.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:15 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4066085228' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:16.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:16 smithi190 ceph-mon[160435]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:16.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:16 smithi138 ceph-mon[185886]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:16.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:16 smithi138 ceph-mon[192513]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:18.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:17 smithi190 ceph-mon[160435]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:17 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3020257530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:18.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:17 smithi138 ceph-mon[192513]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:18.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:17 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3020257530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:18.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:17 smithi138 ceph-mon[185886]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:18.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:17 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3020257530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:20.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:20 smithi190 ceph-mon[160435]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:20.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:20 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2399263914' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:20.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:20 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:20.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:20 smithi138 ceph-mon[192513]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:20.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:20 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2399263914' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:20.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:20 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:20.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:20 smithi138 ceph-mon[185886]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:20.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:20 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2399263914' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:20.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:20 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:22.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:22 smithi190 ceph-mon[160435]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:22.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:22 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3189791272' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:22.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:22 smithi138 ceph-mon[192513]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:22.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:22 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3189791272' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:22.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:22 smithi138 ceph-mon[185886]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:22.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:22 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3189791272' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:24.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:24 smithi138 ceph-mon[185886]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:24.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1954255568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:24.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:24 smithi138 ceph-mon[192513]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:24.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1954255568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:24 smithi190 ceph-mon[160435]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1954255568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:25.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:12:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:12:24] "GET /metrics HTTP/1.1" 200 33761 "" "Prometheus/2.33.4" 2023-10-01T09:12:26.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:26 smithi190 ceph-mon[160435]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:26.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:26.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:12:26.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T09:12:26.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:26 smithi190 ceph-mon[160435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T09:12:26.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:26 smithi190 ceph-mon[160435]: Upgrade: osd.2 is safe to restart 2023-10-01T09:12:26.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:26 smithi138 ceph-mon[192513]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:26.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:26.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:12:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T09:12:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:26 smithi138 ceph-mon[192513]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T09:12:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:26 smithi138 ceph-mon[192513]: Upgrade: osd.2 is safe to restart 2023-10-01T09:12:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:26 smithi138 ceph-mon[185886]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:12:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T09:12:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:26 smithi138 ceph-mon[185886]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T09:12:26.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:26 smithi138 ceph-mon[185886]: Upgrade: osd.2 is safe to restart 2023-10-01T09:12:27.536 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:27 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/724024641' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:27.537 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:27 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:27.537 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:27 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T09:12:27.537 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:27 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:12:27.537 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:27 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/724024641' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:27.538 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:27 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:27.538 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:27 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T09:12:27.538 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:27 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:12:27.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:27 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/724024641' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:27.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:27 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:27.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:27 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T09:12:27.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:27 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:12:28.493 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:28 smithi138 ceph-mon[185886]: Upgrade: Updating osd.2 2023-10-01T09:12:28.493 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:28 smithi138 ceph-mon[185886]: Deploying daemon osd.2 on smithi138 2023-10-01T09:12:28.493 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:28 smithi138 ceph-mon[185886]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:28.494 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:28 smithi138 ceph-mon[192513]: Upgrade: Updating osd.2 2023-10-01T09:12:28.494 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:28 smithi138 ceph-mon[192513]: Deploying daemon osd.2 on smithi138 2023-10-01T09:12:28.494 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:28 smithi138 ceph-mon[192513]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:28.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:28 smithi190 ceph-mon[160435]: Upgrade: Updating osd.2 2023-10-01T09:12:28.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:28 smithi190 ceph-mon[160435]: Deploying daemon osd.2 on smithi138 2023-10-01T09:12:28.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:28 smithi190 ceph-mon[160435]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:29.546 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:29 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1272444060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:29.547 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:29 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1272444060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:29.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:29 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1272444060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:30.260 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:29 smithi138 systemd[1]: Stopping Ceph osd.2 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:12:30.261 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:30 smithi138 conmon[126645]: debug 2023-10-01T09:12:30.064+0000 7f3713021700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T09:12:30.261 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:30 smithi138 conmon[126645]: debug 2023-10-01T09:12:30.064+0000 7f3713021700 -1 osd.2 95 *** Got signal Terminated *** 2023-10-01T09:12:30.261 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:30 smithi138 conmon[126645]: debug 2023-10-01T09:12:30.064+0000 7f3713021700 -1 osd.2 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:12:30.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:30 smithi190 ceph-mon[160435]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:30.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:30 smithi190 ceph-mon[160435]: osd.2 marked itself down and dead 2023-10-01T09:12:30.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:30 smithi138 ceph-mon[192513]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:30.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:30 smithi138 ceph-mon[192513]: osd.2 marked itself down and dead 2023-10-01T09:12:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:30 smithi138 ceph-mon[185886]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:30 smithi138 ceph-mon[185886]: osd.2 marked itself down and dead 2023-10-01T09:12:31.261 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:30 smithi138 bash[212245]: 8a8389da788775583f2d68feb481a99466f781ab5588a8d374709aef0cf6c0f1 2023-10-01T09:12:31.533 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:31 smithi138 ceph-mon[192513]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:12:31.534 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:31 smithi138 ceph-mon[192513]: failed to encode map e96 with expected crc 2023-10-01T09:12:31.534 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:31 smithi138 ceph-mon[192513]: failed to encode map e96 with expected crc 2023-10-01T09:12:31.534 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:31 smithi138 ceph-mon[192513]: osdmap e96: 8 total, 7 up, 8 in 2023-10-01T09:12:31.534 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:31 smithi138 ceph-mon[185886]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:12:31.534 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:31 smithi138 ceph-mon[185886]: failed to encode map e96 with expected crc 2023-10-01T09:12:31.534 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:31 smithi138 ceph-mon[185886]: failed to encode map e96 with expected crc 2023-10-01T09:12:31.535 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:31 smithi138 ceph-mon[185886]: osdmap e96: 8 total, 7 up, 8 in 2023-10-01T09:12:31.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:31 smithi190 ceph-mon[160435]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:12:31.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:31 smithi190 ceph-mon[160435]: failed to encode map e96 with expected crc 2023-10-01T09:12:31.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:31 smithi190 ceph-mon[160435]: failed to encode map e96 with expected crc 2023-10-01T09:12:31.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:31 smithi190 ceph-mon[160435]: osdmap e96: 8 total, 7 up, 8 in 2023-10-01T09:12:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:32 smithi190 ceph-mon[160435]: pgmap v152: 73 pgs: 13 peering, 1 stale+active+clean, 59 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:32 smithi190 ceph-mon[160435]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T09:12:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:32 smithi190 ceph-mon[160435]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:32 smithi190 ceph-mon[160435]: osdmap e97: 8 total, 7 up, 8 in 2023-10-01T09:12:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:32 smithi190 ceph-mon[160435]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:32 smithi190 ceph-mon[160435]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:32 smithi190 ceph-mon[160435]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:32 smithi190 ceph-mon[160435]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:32 smithi190 ceph-mon[160435]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:32 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3070662448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[185886]: pgmap v152: 73 pgs: 13 peering, 1 stale+active+clean, 59 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[185886]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T09:12:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[185886]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[185886]: osdmap e97: 8 total, 7 up, 8 in 2023-10-01T09:12:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[185886]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[185886]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[185886]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[185886]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[185886]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3070662448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[192513]: pgmap v152: 73 pgs: 13 peering, 1 stale+active+clean, 59 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:12:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[192513]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T09:12:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[192513]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[192513]: osdmap e97: 8 total, 7 up, 8 in 2023-10-01T09:12:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[192513]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[192513]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[192513]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[192513]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[192513]: failed to encode map e97 with expected crc 2023-10-01T09:12:32.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:32 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3070662448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:33.070 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:32 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.2.service: Succeeded. 2023-10-01T09:12:33.070 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:32 smithi138 systemd[1]: Stopped Ceph osd.2 for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:12:33.070 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:32 smithi138 systemd[1]: Starting Ceph osd.2 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:12:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:33 smithi190 ceph-mon[160435]: failed to encode map e97 with expected crc 2023-10-01T09:12:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:33 smithi190 ceph-mon[160435]: failed to encode map e97 with expected crc 2023-10-01T09:12:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:33 smithi190 ceph-mon[160435]: failed to encode map e97 with expected crc 2023-10-01T09:12:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:33 smithi138 ceph-mon[185886]: failed to encode map e97 with expected crc 2023-10-01T09:12:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:33 smithi138 ceph-mon[185886]: failed to encode map e97 with expected crc 2023-10-01T09:12:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:33 smithi138 ceph-mon[185886]: failed to encode map e97 with expected crc 2023-10-01T09:12:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:33 smithi138 ceph-mon[192513]: failed to encode map e97 with expected crc 2023-10-01T09:12:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:33 smithi138 ceph-mon[192513]: failed to encode map e97 with expected crc 2023-10-01T09:12:33.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:33 smithi138 ceph-mon[192513]: failed to encode map e97 with expected crc 2023-10-01T09:12:34.069 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:33 smithi138 bash[212906]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-01T09:12:34.070 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:33 smithi138 bash[212906]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-01T09:12:34.070 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:33 smithi138 bash[212906]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-01T09:12:34.070 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:33 smithi138 bash[212906]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-01T09:12:34.070 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:33 smithi138 bash[212906]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-01T09:12:34.070 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:33 smithi138 bash[212906]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-01T09:12:34.070 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:33 smithi138 bash[212906]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-01T09:12:34.070 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:33 smithi138 bash[212906]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-01T09:12:34.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:34 smithi190 ceph-mon[160435]: pgmap v154: 73 pgs: 1 active+undersized+degraded, 13 peering, 6 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-01T09:12:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:34 smithi190 ceph-mon[160435]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:12:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:34 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4109611188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:34.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:34 smithi138 ceph-mon[192513]: pgmap v154: 73 pgs: 1 active+undersized+degraded, 13 peering, 6 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-01T09:12:34.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:34 smithi138 ceph-mon[192513]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:12:34.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:34 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4109611188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:34.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:34 smithi138 ceph-mon[185886]: pgmap v154: 73 pgs: 1 active+undersized+degraded, 13 peering, 6 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-01T09:12:34.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:34 smithi138 ceph-mon[185886]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:12:34.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:34 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4109611188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:35.320 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:34 smithi138 ceph-osd[213406]: -- 172.21.15.138:0/1618566603 <== mon.0 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558eea5664e0 con 0x558eea539400 2023-10-01T09:12:35.320 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:34 smithi138 bash[213340]: ef89ad6ba2b692978388fffbcd2eb9c1a9e39efbeb3d77ae54562be88112b3b0 2023-10-01T09:12:35.320 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:34 smithi138 systemd[1]: Started Ceph osd.2 for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:12:35.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:12:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:12:34] "GET /metrics HTTP/1.1" 200 33910 "" "Prometheus/2.33.4" 2023-10-01T09:12:35.819 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:35 smithi138 conmon[213382]: 2023-10-01T09:12:35.417+0000 7f5599eca640 -1 Falling back to public interface 2023-10-01T09:12:36.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:35 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:36.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:35 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:36.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:35 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:12:36.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:35 smithi190 ceph-mon[160435]: pgmap v155: 73 pgs: 1 active+undersized+degraded, 13 peering, 6 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-01T09:12:36.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3313300511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:36.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:35 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:36.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:35 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:36.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:35 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:12:36.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:35 smithi138 ceph-mon[185886]: pgmap v155: 73 pgs: 1 active+undersized+degraded, 13 peering, 6 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-01T09:12:36.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3313300511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:36.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:35 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:36.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:35 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:36.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:35 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:12:36.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:35 smithi138 ceph-mon[192513]: pgmap v155: 73 pgs: 1 active+undersized+degraded, 13 peering, 6 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-01T09:12:36.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3313300511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:38.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:37 smithi190 ceph-mon[160435]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-01T09:12:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:37 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:37 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:38.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:37 smithi138 ceph-mon[192513]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-01T09:12:38.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:37 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:38.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:37 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:38.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:37 smithi138 ceph-mon[185886]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-01T09:12:38.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:37 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:38.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:37 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:38.819 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:38 smithi138 conmon[213382]: 2023-10-01T09:12:38.360+0000 7f5599eca640 -1 osd.2 95 log_to_monitors true 2023-10-01T09:12:39.287 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:38 smithi138 ceph-mon[192513]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-01T09:12:39.287 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3581203577' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:39.288 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:38 smithi138 ceph-mon[192513]: from='osd.2 [v2:172.21.15.138:6818/4290419390,v1:172.21.15.138:6819/4290419390]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T09:12:39.288 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:38 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:39.288 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:38 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:39.288 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:12:38 smithi138 conmon[213382]: 2023-10-01T09:12:38.938+0000 7f558ab35700 -1 osd.2 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T09:12:39.288 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:38 smithi138 ceph-mon[185886]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-01T09:12:39.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3581203577' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:39.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:38 smithi138 ceph-mon[185886]: from='osd.2 [v2:172.21.15.138:6818/4290419390,v1:172.21.15.138:6819/4290419390]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T09:12:39.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:38 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:39.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:38 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:39.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:38 smithi190 ceph-mon[160435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-10-01T09:12:39.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3581203577' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:39.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:38 smithi190 ceph-mon[160435]: from='osd.2 [v2:172.21.15.138:6818/4290419390,v1:172.21.15.138:6819/4290419390]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T09:12:39.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:38 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:39.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:38 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:40.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:39 smithi190 ceph-mon[160435]: failed to encode map e98 with expected crc 2023-10-01T09:12:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:39 smithi190 ceph-mon[160435]: failed to encode map e98 with expected crc 2023-10-01T09:12:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:39 smithi190 ceph-mon[160435]: from='osd.2 [v2:172.21.15.138:6818/4290419390,v1:172.21.15.138:6819/4290419390]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T09:12:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:39 smithi190 ceph-mon[160435]: osdmap e98: 8 total, 7 up, 8 in 2023-10-01T09:12:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:39 smithi190 ceph-mon[160435]: from='osd.2 [v2:172.21.15.138:6818/4290419390,v1:172.21.15.138:6819/4290419390]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:12:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:39 smithi190 ceph-mon[160435]: failed to encode map e98 with expected crc 2023-10-01T09:12:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:39 smithi190 ceph-mon[160435]: pgmap v158: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-01T09:12:40.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:39 smithi138 ceph-mon[185886]: failed to encode map e98 with expected crc 2023-10-01T09:12:40.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:39 smithi138 ceph-mon[185886]: failed to encode map e98 with expected crc 2023-10-01T09:12:40.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:39 smithi138 ceph-mon[185886]: from='osd.2 [v2:172.21.15.138:6818/4290419390,v1:172.21.15.138:6819/4290419390]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T09:12:40.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:39 smithi138 ceph-mon[185886]: osdmap e98: 8 total, 7 up, 8 in 2023-10-01T09:12:40.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:39 smithi138 ceph-mon[185886]: from='osd.2 [v2:172.21.15.138:6818/4290419390,v1:172.21.15.138:6819/4290419390]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:12:40.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:39 smithi138 ceph-mon[185886]: failed to encode map e98 with expected crc 2023-10-01T09:12:40.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:39 smithi138 ceph-mon[185886]: pgmap v158: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-01T09:12:40.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:39 smithi138 ceph-mon[192513]: failed to encode map e98 with expected crc 2023-10-01T09:12:40.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:39 smithi138 ceph-mon[192513]: failed to encode map e98 with expected crc 2023-10-01T09:12:40.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:39 smithi138 ceph-mon[192513]: from='osd.2 [v2:172.21.15.138:6818/4290419390,v1:172.21.15.138:6819/4290419390]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T09:12:40.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:39 smithi138 ceph-mon[192513]: osdmap e98: 8 total, 7 up, 8 in 2023-10-01T09:12:40.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:39 smithi138 ceph-mon[192513]: from='osd.2 [v2:172.21.15.138:6818/4290419390,v1:172.21.15.138:6819/4290419390]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:12:40.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:39 smithi138 ceph-mon[192513]: failed to encode map e98 with expected crc 2023-10-01T09:12:40.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:39 smithi138 ceph-mon[192513]: pgmap v158: 73 pgs: 3 active+undersized+degraded, 17 active+undersized, 53 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-10-01T09:12:41.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[185886]: failed to encode map e98 with expected crc 2023-10-01T09:12:41.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[185886]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:12:41.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[185886]: failed to encode map e99 with expected crc 2023-10-01T09:12:41.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[185886]: failed to encode map e99 with expected crc 2023-10-01T09:12:41.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[185886]: osd.2 [v2:172.21.15.138:6818/4290419390,v1:172.21.15.138:6819/4290419390] boot 2023-10-01T09:12:41.223 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[185886]: osdmap e99: 8 total, 8 up, 8 in 2023-10-01T09:12:41.223 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T09:12:41.223 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:41.223 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:12:41.223 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2527448312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:41.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[192513]: failed to encode map e98 with expected crc 2023-10-01T09:12:41.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[192513]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:12:41.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[192513]: failed to encode map e99 with expected crc 2023-10-01T09:12:41.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[192513]: failed to encode map e99 with expected crc 2023-10-01T09:12:41.224 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[192513]: osd.2 [v2:172.21.15.138:6818/4290419390,v1:172.21.15.138:6819/4290419390] boot 2023-10-01T09:12:41.224 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[192513]: osdmap e99: 8 total, 8 up, 8 in 2023-10-01T09:12:41.224 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T09:12:41.224 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:41.224 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:12:41.224 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:40 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2527448312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:41.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:40 smithi190 ceph-mon[160435]: failed to encode map e98 with expected crc 2023-10-01T09:12:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:40 smithi190 ceph-mon[160435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:12:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:40 smithi190 ceph-mon[160435]: failed to encode map e99 with expected crc 2023-10-01T09:12:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:40 smithi190 ceph-mon[160435]: failed to encode map e99 with expected crc 2023-10-01T09:12:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:40 smithi190 ceph-mon[160435]: osd.2 [v2:172.21.15.138:6818/4290419390,v1:172.21.15.138:6819/4290419390] boot 2023-10-01T09:12:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:40 smithi190 ceph-mon[160435]: osdmap e99: 8 total, 8 up, 8 in 2023-10-01T09:12:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T09:12:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:12:41.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:40 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2527448312' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:42.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:12:42 smithi138 conmon[178630]: 2023-10-01T09:12:42.013+0000 7fa004e48700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-10-01T09:12:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[192513]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[192513]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[192513]: osdmap e100: 8 total, 8 up, 8 in 2023-10-01T09:12:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[192513]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[192513]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[192513]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[192513]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[192513]: pgmap v161: 73 pgs: 1 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-01T09:12:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:12:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:12:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:12:42.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[185886]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[185886]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[185886]: osdmap e100: 8 total, 8 up, 8 in 2023-10-01T09:12:42.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[185886]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[185886]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[185886]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[185886]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[185886]: pgmap v161: 73 pgs: 1 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-01T09:12:42.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:42.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:42.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:12:42.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:12:42.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:42.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:41 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:12:42.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:41 smithi190 ceph-mon[160435]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:41 smithi190 ceph-mon[160435]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:41 smithi190 ceph-mon[160435]: osdmap e100: 8 total, 8 up, 8 in 2023-10-01T09:12:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:41 smithi190 ceph-mon[160435]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:41 smithi190 ceph-mon[160435]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:41 smithi190 ceph-mon[160435]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:41 smithi190 ceph-mon[160435]: failed to encode map e100 with expected crc 2023-10-01T09:12:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:41 smithi190 ceph-mon[160435]: pgmap v161: 73 pgs: 1 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-10-01T09:12:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:42.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:42.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:12:42.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:12:42.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:42.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:41 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:12:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:12:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T09:12:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:42 smithi190 ceph-mon[160435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T09:12:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:42 smithi190 ceph-mon[160435]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-10-01T09:12:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:42 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3795344785' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:43.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:12:43.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:43.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:43.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:43.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T09:12:43.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[185886]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T09:12:43.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[185886]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-10-01T09:12:43.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3795344785' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:43.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:12:43.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:43.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:43.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:12:43.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T09:12:43.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[192513]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T09:12:43.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[192513]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-10-01T09:12:43.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:42 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3795344785' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:44.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:43 smithi190 ceph-mon[160435]: pgmap v162: 73 pgs: 3 active+undersized, 70 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:44.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:43 smithi138 ceph-mon[185886]: pgmap v162: 73 pgs: 3 active+undersized, 70 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:44.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:43 smithi138 ceph-mon[192513]: pgmap v162: 73 pgs: 3 active+undersized, 70 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:45.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:44 smithi190 ceph-mon[160435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-10-01T09:12:45.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:44 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:12:45.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:12:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:12:44] "GET /metrics HTTP/1.1" 200 33755 "" "Prometheus/2.33.4" 2023-10-01T09:12:45.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:44 smithi138 ceph-mon[185886]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-10-01T09:12:45.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:44 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:12:45.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:44 smithi138 ceph-mon[192513]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-10-01T09:12:45.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:44 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:12:46.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:45 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3605981197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:45 smithi190 ceph-mon[160435]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:46.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:45 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3605981197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:46.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:45 smithi138 ceph-mon[185886]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:46.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:45 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3605981197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:46.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:45 smithi138 ceph-mon[192513]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:48.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:47 smithi190 ceph-mon[160435]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:47 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2106065459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:48.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:47 smithi138 ceph-mon[192513]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:48.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:47 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2106065459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:48.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:47 smithi138 ceph-mon[185886]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:48.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:47 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2106065459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:50.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:50 smithi190 ceph-mon[160435]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:50.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:50 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3698693313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:50 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:50.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:50 smithi138 ceph-mon[192513]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:50.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:50 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3698693313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:50.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:50 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:50.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:50 smithi138 ceph-mon[185886]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:50.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:50 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3698693313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:50.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:50 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:52.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:52 smithi190 ceph-mon[160435]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:52 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2041901143' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:52.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:52 smithi138 ceph-mon[185886]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:52.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:52 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2041901143' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:52.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:52 smithi138 ceph-mon[192513]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:52.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:52 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2041901143' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:54.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:54 smithi138 ceph-mon[192513]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:54.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:54 smithi138 ceph-mon[185886]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:54.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:54 smithi190 ceph-mon[160435]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:55.310 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:12:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:12:54] "GET /metrics HTTP/1.1" 200 33755 "" "Prometheus/2.33.4" 2023-10-01T09:12:55.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:55 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/452003321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:55.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:55.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:12:55.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:55 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/452003321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:55.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:55.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:12:55.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:55 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/452003321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:12:56.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:56 smithi138 ceph-mon[185886]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:56 smithi138 ceph-mon[192513]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:56 smithi190 ceph-mon[160435]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:57.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/144157961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:57.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:57 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T09:12:57.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/144157961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:57.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:57 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T09:12:57.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/144157961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:57 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T09:12:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:58 smithi138 ceph-mon[185886]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T09:12:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:58 smithi138 ceph-mon[185886]: Upgrade: osd.3 is safe to restart 2023-10-01T09:12:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:58 smithi138 ceph-mon[185886]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:58 smithi138 ceph-mon[192513]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T09:12:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:58 smithi138 ceph-mon[192513]: Upgrade: osd.3 is safe to restart 2023-10-01T09:12:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:58 smithi138 ceph-mon[192513]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:58.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:58 smithi190 ceph-mon[160435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T09:12:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:58 smithi190 ceph-mon[160435]: Upgrade: osd.3 is safe to restart 2023-10-01T09:12:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:58 smithi190 ceph-mon[160435]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:59 smithi138 ceph-mon[192513]: Upgrade: Updating osd.3 2023-10-01T09:12:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T09:12:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:12:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:59 smithi138 ceph-mon[192513]: Deploying daemon osd.3 on smithi138 2023-10-01T09:12:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/214045623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:12:59 smithi138 ceph-mon[192513]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:12:59.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:59 smithi138 ceph-mon[185886]: Upgrade: Updating osd.3 2023-10-01T09:12:59.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:12:59.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T09:12:59.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:12:59.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:59 smithi138 ceph-mon[185886]: Deploying daemon osd.3 on smithi138 2023-10-01T09:12:59.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/214045623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:12:59.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:12:59 smithi138 ceph-mon[185886]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:00.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:59 smithi190 ceph-mon[160435]: Upgrade: Updating osd.3 2023-10-01T09:13:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T09:13:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:13:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:59 smithi190 ceph-mon[160435]: Deploying daemon osd.3 on smithi138 2023-10-01T09:13:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/214045623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:12:59 smithi190 ceph-mon[160435]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:02.070 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:01 smithi138 systemd[1]: Stopping Ceph osd.3 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:13:02.070 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:01 smithi138 conmon[132596]: debug 2023-10-01T09:13:01.843+0000 7f9233aa2700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T09:13:02.070 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:01 smithi138 conmon[132596]: debug 2023-10-01T09:13:01.843+0000 7f9233aa2700 -1 osd.3 100 *** Got signal Terminated *** 2023-10-01T09:13:02.070 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:01 smithi138 conmon[132596]: debug 2023-10-01T09:13:01.843+0000 7f9233aa2700 -1 osd.3 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:13:02.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:02 smithi138 ceph-mon[185886]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:02.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:02 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2237970320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:02.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:02 smithi138 ceph-mon[185886]: osd.3 marked itself down and dead 2023-10-01T09:13:02.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:02 smithi138 ceph-mon[192513]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:02 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2237970320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:02 smithi138 ceph-mon[192513]: osd.3 marked itself down and dead 2023-10-01T09:13:02.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:02 smithi190 ceph-mon[160435]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:02 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2237970320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:02 smithi190 ceph-mon[160435]: osd.3 marked itself down and dead 2023-10-01T09:13:03.320 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:02 smithi138 bash[217242]: c3ba3a01539d7d393526110c01d8e39f23f73600d2d396b2e7a5d859c49ebd9c 2023-10-01T09:13:03.601 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[192513]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:13:03.601 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[192513]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.601 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[192513]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.601 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[192513]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.601 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[192513]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.601 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[192513]: osdmap e101: 8 total, 7 up, 8 in 2023-10-01T09:13:03.601 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[192513]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.601 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[192513]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.601 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[192513]: osdmap e102: 8 total, 7 up, 8 in 2023-10-01T09:13:03.602 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[185886]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:13:03.602 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[185886]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.602 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[185886]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.602 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[185886]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.602 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[185886]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.602 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[185886]: osdmap e101: 8 total, 7 up, 8 in 2023-10-01T09:13:03.602 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[185886]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.602 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[185886]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.602 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:03 smithi138 ceph-mon[185886]: osdmap e102: 8 total, 7 up, 8 in 2023-10-01T09:13:03.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:03 smithi190 ceph-mon[160435]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:13:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:03 smithi190 ceph-mon[160435]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:03 smithi190 ceph-mon[160435]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:03 smithi190 ceph-mon[160435]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:03 smithi190 ceph-mon[160435]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:03 smithi190 ceph-mon[160435]: osdmap e101: 8 total, 7 up, 8 in 2023-10-01T09:13:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:03 smithi190 ceph-mon[160435]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:03 smithi190 ceph-mon[160435]: failed to encode map e101 with expected crc 2023-10-01T09:13:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:03 smithi190 ceph-mon[160435]: osdmap e102: 8 total, 7 up, 8 in 2023-10-01T09:13:04.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[185886]: pgmap v173: 73 pgs: 6 peering, 9 stale+active+clean, 58 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:04.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[185886]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[185886]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[185886]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[185886]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[185886]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[185886]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[185886]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2447715498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:04.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[192513]: pgmap v173: 73 pgs: 6 peering, 9 stale+active+clean, 58 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:04.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[192513]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[192513]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[192513]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[192513]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[192513]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[192513]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[192513]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2447715498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:04 smithi190 ceph-mon[160435]: pgmap v173: 73 pgs: 6 peering, 9 stale+active+clean, 58 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:04 smithi190 ceph-mon[160435]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:04 smithi190 ceph-mon[160435]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:04 smithi190 ceph-mon[160435]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:04 smithi190 ceph-mon[160435]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:04 smithi190 ceph-mon[160435]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:04 smithi190 ceph-mon[160435]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:04 smithi190 ceph-mon[160435]: failed to encode map e102 with expected crc 2023-10-01T09:13:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2447715498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:04.935 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:04 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-01T09:13:04.935 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:04 smithi138 systemd[1]: Stopped Ceph osd.3 for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:13:04.936 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:04 smithi138 systemd[1]: Starting Ceph osd.3 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:13:04.936 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:13:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:13:04] "GET /metrics HTTP/1.1" 200 33903 "" "Prometheus/2.33.4" 2023-10-01T09:13:06.069 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:05 smithi138 bash[217899]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-01T09:13:06.070 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:05 smithi138 bash[217899]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-01T09:13:06.070 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:05 smithi138 bash[217899]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-01T09:13:06.070 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:05 smithi138 bash[217899]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-01T09:13:06.070 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:05 smithi138 bash[217899]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-01T09:13:06.070 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:05 smithi138 bash[217899]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-01T09:13:06.070 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:05 smithi138 bash[217899]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-01T09:13:06.070 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:05 smithi138 bash[217899]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-01T09:13:06.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:06 smithi190 ceph-mon[160435]: pgmap v175: 73 pgs: 6 peering, 8 stale+active+clean, 4 active+undersized, 55 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:06 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/845604347' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:06 smithi138 ceph-mon[185886]: pgmap v175: 73 pgs: 6 peering, 8 stale+active+clean, 4 active+undersized, 55 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:06 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/845604347' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:06 smithi138 ceph-mon[192513]: pgmap v175: 73 pgs: 6 peering, 8 stale+active+clean, 4 active+undersized, 55 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:06 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/845604347' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:07.570 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:07 smithi138 ceph-osd[218411]: -- 172.21.15.138:0/3303191123 <== mon.0 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5638901e84e0 con 0x5638901bb400 2023-10-01T09:13:07.570 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:07 smithi138 bash[218346]: 4b9563996612ff07dac8428c2fe7524decf3879521dfaeb75d98446e703f8ba9 2023-10-01T09:13:07.570 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:07 smithi138 systemd[1]: Started Ceph osd.3 for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:13:08.069 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:07 smithi138 conmon[218387]: 2023-10-01T09:13:07.681+0000 7ff654e35640 -1 Falling back to public interface 2023-10-01T09:13:08.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:08 smithi190 ceph-mon[160435]: pgmap v176: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:08.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:08 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:08 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:08 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:13:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:08 smithi190 ceph-mon[160435]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:13:08.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:08 smithi138 ceph-mon[185886]: pgmap v176: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:08 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:08 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:08 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:13:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:08 smithi138 ceph-mon[185886]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:13:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:08 smithi138 ceph-mon[192513]: pgmap v176: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:08 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:08 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:08 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:13:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:08 smithi138 ceph-mon[192513]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:13:09.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:09 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1794730574' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:09.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:09 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1794730574' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:09 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1794730574' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:10 smithi190 ceph-mon[160435]: pgmap v177: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:10.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:10.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:13:10.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:10.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:10 smithi138 ceph-mon[192513]: pgmap v177: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:13:10.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:10.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:10 smithi138 ceph-mon[185886]: pgmap v177: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:10.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:10.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:13:10.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.069 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:10 smithi138 conmon[218387]: 2023-10-01T09:13:10.577+0000 7ff654e35640 -1 osd.3 100 log_to_monitors true 2023-10-01T09:13:11.547 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.548 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.548 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.548 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[192513]: from='osd.3 [v2:172.21.15.138:6826/244903146,v1:172.21.15.138:6827/244903146]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T09:13:11.548 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[192513]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T09:13:11.548 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/882518227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:11.548 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.548 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[185886]: from='osd.3 [v2:172.21.15.138:6826/244903146,v1:172.21.15.138:6827/244903146]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T09:13:11.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[185886]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T09:13:11.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/882518227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:11.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:11 smithi190 ceph-mon[160435]: from='osd.3 [v2:172.21.15.138:6826/244903146,v1:172.21.15.138:6827/244903146]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T09:13:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:11 smithi190 ceph-mon[160435]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T09:13:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/882518227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:11.820 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:13:11 smithi138 conmon[218387]: 2023-10-01T09:13:11.559+0000 7ff643a9c700 -1 osd.3 100 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T09:13:12.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:12 smithi190 ceph-mon[160435]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:12 smithi190 ceph-mon[160435]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T09:13:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:12 smithi190 ceph-mon[160435]: osdmap e103: 8 total, 7 up, 8 in 2023-10-01T09:13:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:12 smithi190 ceph-mon[160435]: from='osd.3 [v2:172.21.15.138:6826/244903146,v1:172.21.15.138:6827/244903146]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:13:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:12 smithi190 ceph-mon[160435]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:13:12.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:12 smithi138 ceph-mon[192513]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:12.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:12 smithi138 ceph-mon[192513]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T09:13:12.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:12 smithi138 ceph-mon[192513]: osdmap e103: 8 total, 7 up, 8 in 2023-10-01T09:13:12.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:12 smithi138 ceph-mon[192513]: from='osd.3 [v2:172.21.15.138:6826/244903146,v1:172.21.15.138:6827/244903146]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:13:12.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:12 smithi138 ceph-mon[192513]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:13:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:12 smithi138 ceph-mon[185886]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:12 smithi138 ceph-mon[185886]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T09:13:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:12 smithi138 ceph-mon[185886]: osdmap e103: 8 total, 7 up, 8 in 2023-10-01T09:13:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:12 smithi138 ceph-mon[185886]: from='osd.3 [v2:172.21.15.138:6826/244903146,v1:172.21.15.138:6827/244903146]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:13:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:12 smithi138 ceph-mon[185886]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi138", "root=default"]}]: dispatch 2023-10-01T09:13:13.026 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 4 2023-10-01T09:13:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[192513]: failed to encode map e103 with expected crc 2023-10-01T09:13:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[192513]: failed to encode map e103 with expected crc 2023-10-01T09:13:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[192513]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:13:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[192513]: failed to encode map e103 with expected crc 2023-10-01T09:13:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[192513]: osd.3 [v2:172.21.15.138:6826/244903146,v1:172.21.15.138:6827/244903146] boot 2023-10-01T09:13:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[192513]: osdmap e104: 8 total, 8 up, 8 in 2023-10-01T09:13:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:13:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[192513]: failed to encode map e104 with expected crc 2023-10-01T09:13:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3573625630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[185886]: failed to encode map e103 with expected crc 2023-10-01T09:13:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[185886]: failed to encode map e103 with expected crc 2023-10-01T09:13:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[185886]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:13:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[185886]: failed to encode map e103 with expected crc 2023-10-01T09:13:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[185886]: osd.3 [v2:172.21.15.138:6826/244903146,v1:172.21.15.138:6827/244903146] boot 2023-10-01T09:13:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[185886]: osdmap e104: 8 total, 8 up, 8 in 2023-10-01T09:13:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:13:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[185886]: failed to encode map e104 with expected crc 2023-10-01T09:13:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:13 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3573625630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:13.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:13 smithi190 ceph-mon[160435]: failed to encode map e103 with expected crc 2023-10-01T09:13:13.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:13 smithi190 ceph-mon[160435]: failed to encode map e103 with expected crc 2023-10-01T09:13:13.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:13 smithi190 ceph-mon[160435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:13:13.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:13 smithi190 ceph-mon[160435]: failed to encode map e103 with expected crc 2023-10-01T09:13:13.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:13 smithi190 ceph-mon[160435]: osd.3 [v2:172.21.15.138:6826/244903146,v1:172.21.15.138:6827/244903146] boot 2023-10-01T09:13:13.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:13 smithi190 ceph-mon[160435]: osdmap e104: 8 total, 8 up, 8 in 2023-10-01T09:13:13.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:13 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T09:13:13.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:13 smithi190 ceph-mon[160435]: failed to encode map e104 with expected crc 2023-10-01T09:13:13.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:13 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3573625630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T09:13:14.206 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade pause' 2023-10-01T09:13:14.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:14 smithi190 ceph-mon[160435]: pgmap v181: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:14 smithi190 ceph-mon[160435]: osdmap e105: 8 total, 8 up, 8 in 2023-10-01T09:13:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:14 smithi190 ceph-mon[160435]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:14 smithi190 ceph-mon[160435]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:14 smithi190 ceph-mon[160435]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:14 smithi190 ceph-mon[160435]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:14 smithi190 ceph-mon[160435]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:14 smithi190 ceph-mon[160435]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:14 smithi190 ceph-mon[160435]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:14 smithi190 ceph-mon[160435]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[185886]: pgmap v181: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[185886]: osdmap e105: 8 total, 8 up, 8 in 2023-10-01T09:13:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[185886]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[185886]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[185886]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[185886]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[185886]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[185886]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[185886]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[185886]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[192513]: pgmap v181: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:14.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[192513]: osdmap e105: 8 total, 8 up, 8 in 2023-10-01T09:13:14.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[192513]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[192513]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[192513]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[192513]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[192513]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[192513]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[192513]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:14 smithi138 ceph-mon[192513]: failed to encode map e105 with expected crc 2023-10-01T09:13:14.886 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:13:14 smithi138 conmon[178630]: 2023-10-01T09:13:14.603+0000 7fa004e48700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-01T09:13:15.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:13:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:13:14] "GET /metrics HTTP/1.1" 200 33907 "" "Prometheus/2.33.4" 2023-10-01T09:13:15.699 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:15.699 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:15.699 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:13:15.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:13:15.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:15.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:13:15.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:13:15.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:13:15.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:13:15.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-01T09:13:15.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[192513]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-01T09:13:15.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[192513]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-01T09:13:15.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[192513]: pgmap v183: 73 pgs: 2 peering, 3 active+undersized+degraded, 23 active+undersized, 45 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:15.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:15.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:15.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:15.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:13:15.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:13:15.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:15.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:13:15.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:13:15.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:13:15.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:13:15.702 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-01T09:13:15.702 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[185886]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-01T09:13:15.702 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[185886]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-01T09:13:15.702 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[185886]: pgmap v183: 73 pgs: 2 peering, 3 active+undersized+degraded, 23 active+undersized, 45 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:15.702 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:15 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:15.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:15 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:15 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:15 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:13:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:15 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:13:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:15 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:15 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:13:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:15 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:13:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:15 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:13:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:15 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:13:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:15 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-01T09:13:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:15 smithi190 ceph-mon[160435]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-01T09:13:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:15 smithi190 ceph-mon[160435]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-10-01T09:13:15.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:15 smithi190 ceph-mon[160435]: pgmap v183: 73 pgs: 2 peering, 3 active+undersized+degraded, 23 active+undersized, 45 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:13:15.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:15 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:16.339 INFO:teuthology.orchestra.run.smithi138.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:13:17.132 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch ps' 2023-10-01T09:13:17.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:17 smithi138 ceph-mon[192513]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:13:17.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:17 smithi138 ceph-mon[192513]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:13:17.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:17 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:17.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:17 smithi138 ceph-mon[185886]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:13:17.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:17 smithi138 ceph-mon[185886]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:13:17.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:17 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:17.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:17 smithi190 ceph-mon[160435]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:13:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:17 smithi190 ceph-mon[160435]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T09:13:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:17 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:18.692 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:18 smithi138 ceph-mon[192513]: pgmap v184: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:18.692 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:18 smithi138 ceph-mon[192513]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2023-10-01T09:13:18.692 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:18 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:13:18.692 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:18 smithi138 ceph-mon[185886]: pgmap v184: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:18.692 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:18 smithi138 ceph-mon[185886]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2023-10-01T09:13:18.693 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:18 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:13:18.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:18 smithi190 ceph-mon[160435]: pgmap v184: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:18 smithi190 ceph-mon[160435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2023-10-01T09:13:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:18 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:13:19.083 INFO:teuthology.orchestra.run.smithi138.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-01T09:13:19.084 INFO:teuthology.orchestra.run.smithi138.stdout:alertmanager.a smithi138 *:9093,9094 running (6m) - 10m 27.1M - 0.23.0 ba2b418f427c 8debc681d94e 2023-10-01T09:13:19.084 INFO:teuthology.orchestra.run.smithi138.stdout:grafana.a smithi190 *:3000 running (6m) - 10m 58.6M - 8.3.5 dad864ee21e9 58ea19a8c949 2023-10-01T09:13:19.084 INFO:teuthology.orchestra.run.smithi138.stdout:mds.foo.smithi138.brhzod smithi138 running (10m) - 10m 19.7M - 16.2.14-44-geeb13ac9 94a6c4498547 555d8c2cfe25 2023-10-01T09:13:19.084 INFO:teuthology.orchestra.run.smithi138.stdout:mds.foo.smithi190.weynjc smithi190 running (10m) - 10m 18.8M - 16.2.14-44-geeb13ac9 94a6c4498547 881fea906eff 2023-10-01T09:13:19.084 INFO:teuthology.orchestra.run.smithi138.stdout:mgr.x smithi190 *:8443,9283,8765 running (5m) - 14m 450M - 18.2.0-497-gb5682219 a3fc96fed5ac 674e1140402f 2023-10-01T09:13:19.084 INFO:teuthology.orchestra.run.smithi138.stdout:mgr.y smithi138 *:8443,9283,8765 running (6m) - 16m 515M - 18.2.0-497-gb5682219 a3fc96fed5ac 97da5dac6a84 2023-10-01T09:13:19.084 INFO:teuthology.orchestra.run.smithi138.stdout:mon.a smithi138 running (5m) - 16m 101M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 4f0e28d17710 2023-10-01T09:13:19.085 INFO:teuthology.orchestra.run.smithi138.stdout:mon.b smithi190 running (3m) - 15m 71.6M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac c0ac51d9b5e0 2023-10-01T09:13:19.085 INFO:teuthology.orchestra.run.smithi138.stdout:mon.c smithi138 running (4m) - 15m 75.3M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 9f7c99bda05d 2023-10-01T09:13:19.085 INFO:teuthology.orchestra.run.smithi138.stdout:node-exporter.a smithi138 *:9100 running (6m) - 11m 27.4M - 1.3.1 1dbe0e931976 bd644aa2145b 2023-10-01T09:13:19.085 INFO:teuthology.orchestra.run.smithi138.stdout:node-exporter.b smithi190 *:9100 running (6m) - 11m 27.1M - 1.3.1 1dbe0e931976 c08f7a49a78d 2023-10-01T09:13:19.085 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0 smithi138 running (2m) - 14m 62.3M 2500M 18.2.0-497-gb5682219 a3fc96fed5ac cb94dc6c4a67 2023-10-01T09:13:19.085 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1 smithi138 running (75s) - 14m 57.1M 2500M 18.2.0-497-gb5682219 a3fc96fed5ac 6abadff62084 2023-10-01T09:13:19.085 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2 smithi138 running (44s) - 13m 48.3M 2500M 18.2.0-497-gb5682219 a3fc96fed5ac ef89ad6ba2b6 2023-10-01T09:13:19.085 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3 smithi138 running (12s) - 13m 14.8M 2500M 18.2.0-497-gb5682219 a3fc96fed5ac 4b9563996612 2023-10-01T09:13:19.085 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4 smithi190 running (12m) - 12m 92.9M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 cda54a787614 2023-10-01T09:13:19.085 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5 smithi190 running (12m) - 12m 103M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 4ac584c1fb7d 2023-10-01T09:13:19.085 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6 smithi190 running (12m) - 12m 86.6M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 fa45a1c37077 2023-10-01T09:13:19.086 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7 smithi190 running (11m) - 11m 109M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 c925b7fc3562 2023-10-01T09:13:19.086 INFO:teuthology.orchestra.run.smithi138.stdout:prometheus.a smithi190 *:9095 running (5m) - 11m 71.1M - 2.33.4 514e6a882f6e bbdb076a24d4 2023-10-01T09:13:19.850 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch ps' 2023-10-01T09:13:20.749 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:20 smithi138 ceph-mon[185886]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:13:20.749 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:20 smithi138 ceph-mon[185886]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:20.750 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:20 smithi138 ceph-mon[192513]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:13:20.750 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:20 smithi138 ceph-mon[192513]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:20.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:20 smithi190 ceph-mon[160435]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:13:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:20 smithi190 ceph-mon[160435]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:21.795 INFO:teuthology.orchestra.run.smithi138.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-01T09:13:21.796 INFO:teuthology.orchestra.run.smithi138.stdout:alertmanager.a smithi138 *:9093,9094 running (6m) - 11m 27.1M - 0.23.0 ba2b418f427c 8debc681d94e 2023-10-01T09:13:21.796 INFO:teuthology.orchestra.run.smithi138.stdout:grafana.a smithi190 *:3000 running (6m) - 10m 58.6M - 8.3.5 dad864ee21e9 58ea19a8c949 2023-10-01T09:13:21.796 INFO:teuthology.orchestra.run.smithi138.stdout:mds.foo.smithi138.brhzod smithi138 running (10m) - 10m 19.7M - 16.2.14-44-geeb13ac9 94a6c4498547 555d8c2cfe25 2023-10-01T09:13:21.796 INFO:teuthology.orchestra.run.smithi138.stdout:mds.foo.smithi190.weynjc smithi190 running (10m) - 10m 18.8M - 16.2.14-44-geeb13ac9 94a6c4498547 881fea906eff 2023-10-01T09:13:21.796 INFO:teuthology.orchestra.run.smithi138.stdout:mgr.x smithi190 *:8443,9283,8765 running (5m) - 14m 450M - 18.2.0-497-gb5682219 a3fc96fed5ac 674e1140402f 2023-10-01T09:13:21.796 INFO:teuthology.orchestra.run.smithi138.stdout:mgr.y smithi138 *:8443,9283,8765 running (6m) - 16m 515M - 18.2.0-497-gb5682219 a3fc96fed5ac 97da5dac6a84 2023-10-01T09:13:21.796 INFO:teuthology.orchestra.run.smithi138.stdout:mon.a smithi138 running (5m) - 16m 101M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 4f0e28d17710 2023-10-01T09:13:21.796 INFO:teuthology.orchestra.run.smithi138.stdout:mon.b smithi190 running (3m) - 15m 71.6M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac c0ac51d9b5e0 2023-10-01T09:13:21.796 INFO:teuthology.orchestra.run.smithi138.stdout:mon.c smithi138 running (4m) - 15m 75.3M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 9f7c99bda05d 2023-10-01T09:13:21.797 INFO:teuthology.orchestra.run.smithi138.stdout:node-exporter.a smithi138 *:9100 running (6m) - 11m 27.4M - 1.3.1 1dbe0e931976 bd644aa2145b 2023-10-01T09:13:21.797 INFO:teuthology.orchestra.run.smithi138.stdout:node-exporter.b smithi190 *:9100 running (6m) - 11m 27.1M - 1.3.1 1dbe0e931976 c08f7a49a78d 2023-10-01T09:13:21.797 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0 smithi138 running (2m) - 14m 62.3M 2500M 18.2.0-497-gb5682219 a3fc96fed5ac cb94dc6c4a67 2023-10-01T09:13:21.797 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1 smithi138 running (78s) - 14m 57.1M 2500M 18.2.0-497-gb5682219 a3fc96fed5ac 6abadff62084 2023-10-01T09:13:21.797 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2 smithi138 running (47s) - 13m 48.3M 2500M 18.2.0-497-gb5682219 a3fc96fed5ac ef89ad6ba2b6 2023-10-01T09:13:21.797 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3 smithi138 running (14s) - 13m 14.8M 2500M 18.2.0-497-gb5682219 a3fc96fed5ac 4b9563996612 2023-10-01T09:13:21.797 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4 smithi190 running (13m) - 13m 92.9M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 cda54a787614 2023-10-01T09:13:21.797 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5 smithi190 running (12m) - 12m 103M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 4ac584c1fb7d 2023-10-01T09:13:21.797 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6 smithi190 running (12m) - 12m 86.6M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 fa45a1c37077 2023-10-01T09:13:21.797 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7 smithi190 running (11m) - 11m 109M 2500M 16.2.14-44-geeb13ac9 94a6c4498547 c925b7fc3562 2023-10-01T09:13:21.797 INFO:teuthology.orchestra.run.smithi138.stdout:prometheus.a smithi190 *:9095 running (5m) - 11m 71.1M - 2.33.4 514e6a882f6e bbdb076a24d4 2023-10-01T09:13:22.550 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph versions' 2023-10-01T09:13:22.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:22 smithi190 ceph-mon[160435]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:22.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:22 smithi138 ceph-mon[192513]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:22.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:22 smithi138 ceph-mon[185886]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:23.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:23 smithi190 ceph-mon[160435]: from='client.44369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:13:23.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:23 smithi138 ceph-mon[192513]: from='client.44369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:13:23.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:23 smithi138 ceph-mon[185886]: from='client.44369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:13:24.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:24 smithi138 ceph-mon[192513]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:24.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:24 smithi138 ceph-mon[185886]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:24.656 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:13:24.656 INFO:teuthology.orchestra.run.smithi138.stdout: "mon": { 2023-10-01T09:13:24.656 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 3 2023-10-01T09:13:24.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:13:24.656 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr": { 2023-10-01T09:13:24.656 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 2 2023-10-01T09:13:24.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:13:24.657 INFO:teuthology.orchestra.run.smithi138.stdout: "osd": { 2023-10-01T09:13:24.657 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)": 4, 2023-10-01T09:13:24.657 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 4 2023-10-01T09:13:24.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:13:24.657 INFO:teuthology.orchestra.run.smithi138.stdout: "mds": { 2023-10-01T09:13:24.657 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)": 2 2023-10-01T09:13:24.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:13:24.657 INFO:teuthology.orchestra.run.smithi138.stdout: "overall": { 2023-10-01T09:13:24.657 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)": 6, 2023-10-01T09:13:24.657 INFO:teuthology.orchestra.run.smithi138.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 9 2023-10-01T09:13:24.657 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:13:24.658 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:13:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:24 smithi190 ceph-mon[160435]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:25.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:13:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:13:24] "GET /metrics HTTP/1.1" 200 33907 "" "Prometheus/2.33.4" 2023-10-01T09:13:25.462 DEBUG:teuthology.parallel:result is None 2023-10-01T09:13:25.462 INFO:teuthology.run_tasks:Running task print... 2023-10-01T09:13:25.472 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-10-01T09:13:25.472 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-01T09:13:25.499 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-01T09:13:25.499 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-01T09:13:25.500 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-01T09:13:25.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:25 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3152964489' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:13:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:13:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:25.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:25 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3152964489' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:13:25.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:25.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:13:25.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:25.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:25 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3152964489' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T09:13:25.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:25.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:13:25.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:25.858 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:26.539 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:26 smithi138 ceph-mon[185886]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:26.540 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:26 smithi138 ceph-mon[192513]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:26.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:26 smithi190 ceph-mon[160435]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:27.697 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:13:28.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:27 smithi138 ceph-mon[185886]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:28.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:27 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3505450318' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T09:13:28.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:27 smithi138 ceph-mon[192513]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:28.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:27 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3505450318' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T09:13:28.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:27 smithi190 ceph-mon[160435]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:28.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:27 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3505450318' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T09:13:28.497 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":35,"active_gid":26000,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6800","nonce":961393236},{"type":"v1","addr":"172.21.15.138:6801","nonce":961393236}]},"active_addr":"172.21.15.138:6801/961393236","active_change":"2023-10-01T09:07:55.156867+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25994,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":85,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.138:0","nonce":3126736175}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.138:0","nonce":3197055667}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.138:0","nonce":3940745485}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.138:0","nonce":1671289123}]}]} 2023-10-01T09:13:28.501 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-01T09:13:28.501 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-01T09:13:28.502 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T09:13:28.850 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:30.310 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[185886]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:30.311 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:13:30.311 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:13:30.311 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:13:30.311 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:13:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:13:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:13:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:30.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[192513]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:30.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:13:30.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:13:30.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:13:30.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:30.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:13:30.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:13:30.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:13:30.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:30 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:30.637 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:13:30.638 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":105,"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","created":"2023-10-01T08:56:55.039846+0000","modified":"2023-10-01T09:13:13.525285+0000","last_up_change":"2023-10-01T09:13:12.523301+0000","last_in_change":"2023-10-01T09:01:14.134268+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-01T08:59:41.882085+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T09:02:05.272330+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"77","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":77,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-01T09:03:01.537913+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-01T09:03:02.452550+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"12f71e77-6099-48ee-a89e-e931e000e9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":85,"up_from":89,"up_thru":104,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6803","nonce":2668617192}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6805","nonce":2668617192}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6809","nonce":2668617192}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6807","nonce":2668617192}]},"public_addr":"172.21.15.138:6803/2668617192","cluster_addr":"172.21.15.138:6805/2668617192","heartbeat_back_addr":"172.21.15.138:6809/2668617192","heartbeat_front_addr":"172.21.15.138:6807/2668617192","state":["exists","up"]},{"osd":1,"uuid":"525b5e81-4fa5-4b10-9147-83fcb549ca30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":90,"up_from":94,"up_thru":104,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6811","nonce":2021985596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6813","nonce":2021985596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6817","nonce":2021985596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6815","nonce":2021985596}]},"public_addr":"172.21.15.138:6811/2021985596","cluster_addr":"172.21.15.138:6813/2021985596","heartbeat_back_addr":"172.21.15.138:6817/2021985596","heartbeat_front_addr":"172.21.15.138:6815/2021985596","state":["exists","up"]},{"osd":2,"uuid":"97443e05-2513-4832-bb54-f65493df5967","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":95,"up_from":99,"up_thru":104,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6819","nonce":4290419390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6821","nonce":4290419390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6825","nonce":4290419390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6823","nonce":4290419390}]},"public_addr":"172.21.15.138:6819/4290419390","cluster_addr":"172.21.15.138:6821/4290419390","heartbeat_back_addr":"172.21.15.138:6825/4290419390","heartbeat_front_addr":"172.21.15.138:6823/4290419390","state":["exists","up"]},{"osd":3,"uuid":"ac51c165-e007-4390-b95b-26eef984d637","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":100,"up_from":104,"up_thru":104,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6827","nonce":244903146}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6829","nonce":244903146}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6832","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6833","nonce":244903146}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6831","nonce":244903146}]},"public_addr":"172.21.15.138:6827/244903146","cluster_addr":"172.21.15.138:6829/244903146","heartbeat_back_addr":"172.21.15.138:6833/244903146","heartbeat_front_addr":"172.21.15.138:6831/244903146","state":["exists","up"]},{"osd":4,"uuid":"4bef630c-bff4-41a3-a932-de0501541f78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":104,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6801","nonce":743095145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6803","nonce":743095145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6807","nonce":743095145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6805","nonce":743095145}]},"public_addr":"172.21.15.190:6801/743095145","cluster_addr":"172.21.15.190:6803/743095145","heartbeat_back_addr":"172.21.15.190:6807/743095145","heartbeat_front_addr":"172.21.15.190:6805/743095145","state":["exists","up"]},{"osd":5,"uuid":"ac6e16ed-681d-4e9d-87a3-d20d50e01437","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":104,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6808","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6809","nonce":4080051847}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6810","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6811","nonce":4080051847}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6814","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6815","nonce":4080051847}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6812","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6813","nonce":4080051847}]},"public_addr":"172.21.15.190:6809/4080051847","cluster_addr":"172.21.15.190:6811/4080051847","heartbeat_back_addr":"172.21.15.190:6815/4080051847","heartbeat_front_addr":"172.21.15.190:6813/4080051847","state":["exists","up"]},{"osd":6,"uuid":"7a6f65a5-16dc-45de-854d-751d9151e85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6816","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6817","nonce":797186189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6818","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6819","nonce":797186189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6822","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6823","nonce":797186189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6820","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6821","nonce":797186189}]},"public_addr":"172.21.15.190:6817/797186189","cluster_addr":"172.21.15.190:6819/797186189","heartbeat_back_addr":"172.21.15.190:6823/797186189","heartbeat_front_addr":"172.21.15.190:6821/797186189","state":["exists","up"]},{"osd":7,"uuid":"09a5ad5d-d58f-4fcf-8b09-10f395bd01f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":104,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6824","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6825","nonce":185238034}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6826","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6827","nonce":185238034}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6830","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6831","nonce":185238034}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6828","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6829","nonce":185238034}]},"public_addr":"172.21.15.190:6825/185238034","cluster_addr":"172.21.15.190:6827/185238034","heartbeat_back_addr":"172.21.15.190:6831/185238034","heartbeat_front_addr":"172.21.15.190:6829/185238034","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:58:54.313619+0000","dead_epoch":85},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:16.837452+0000","dead_epoch":90},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:39.610865+0000","dead_epoch":95},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:03.127916+0000","dead_epoch":100},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:23.838460+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:45.582295+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:07.078976+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:29.080850+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.138:6801/3849457598":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/319567031":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/2091838076":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/1464782684":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/25071367":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/3940341494":"2023-10-02T08:57:39.299924+0000","172.21.15.190:0/3151728630":"2023-10-02T09:07:17.185267+0000","172.21.15.138:6801/3165627846":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6800/3165627846":"2023-10-02T08:57:18.125987+0000","172.21.15.138:0/3256141027":"2023-10-02T09:07:55.156656+0000","172.21.15.190:6835/2242576319":"2023-10-02T09:07:17.185267+0000","172.21.15.190:0/2286153897":"2023-10-02T09:07:17.185267+0000","172.21.15.138:0/2575047401":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6801/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.190:0/2209516562":"2023-10-02T09:07:17.185267+0000","172.21.15.190:0/3337553974":"2023-10-02T09:07:17.185267+0000","172.21.15.138:0/2018177063":"2023-10-02T08:57:18.125987+0000","172.21.15.138:0/523071753":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/2054656381":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/1836785572":"2023-10-02T09:06:07.582189+0000","172.21.15.138:6800/3849457598":"2023-10-02T09:07:55.156656+0000","172.21.15.138:6800/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.190:6834/2242576319":"2023-10-02T09:07:17.185267+0000","172.21.15.190:0/299211527":"2023-10-02T09:07:17.185267+0000","172.21.15.138:6801/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:0/3533950111":"2023-10-02T09:06:07.582189+0000","172.21.15.190:0/710760207":"2023-10-02T09:07:17.185267+0000","172.21.15.138:6800/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:0/3675584653":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6800/974662983":"2023-10-02T09:06:07.582189+0000","172.21.15.138:6801/974662983":"2023-10-02T09:06:07.582189+0000","172.21.15.138:0/2334114390":"2023-10-02T09:01:44.817357+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T09:13:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:30 smithi190 ceph-mon[160435]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:30 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:13:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:30 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:13:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:30 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:13:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:30 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:30 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:13:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:30 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:13:30.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:30 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:13:30.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:30 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:13:31.407 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-01T09:13:31.407 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T09:13:31.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2491361158' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:13:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2491361158' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:13:31.754 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:31.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2491361158' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:13:32.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:32 smithi138 ceph-mon[185886]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:32.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:32 smithi138 ceph-mon[192513]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:32 smithi190 ceph-mon[160435]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:33.501 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:13:33.502 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":105,"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","created":"2023-10-01T08:56:55.039846+0000","modified":"2023-10-01T09:13:13.525285+0000","last_up_change":"2023-10-01T09:13:12.523301+0000","last_in_change":"2023-10-01T09:01:14.134268+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-01T08:59:41.882085+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T09:02:05.272330+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"77","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":77,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-01T09:03:01.537913+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-01T09:03:02.452550+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"12f71e77-6099-48ee-a89e-e931e000e9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":85,"up_from":89,"up_thru":104,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6803","nonce":2668617192}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6805","nonce":2668617192}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6809","nonce":2668617192}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6807","nonce":2668617192}]},"public_addr":"172.21.15.138:6803/2668617192","cluster_addr":"172.21.15.138:6805/2668617192","heartbeat_back_addr":"172.21.15.138:6809/2668617192","heartbeat_front_addr":"172.21.15.138:6807/2668617192","state":["exists","up"]},{"osd":1,"uuid":"525b5e81-4fa5-4b10-9147-83fcb549ca30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":90,"up_from":94,"up_thru":104,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6811","nonce":2021985596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6813","nonce":2021985596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6817","nonce":2021985596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6815","nonce":2021985596}]},"public_addr":"172.21.15.138:6811/2021985596","cluster_addr":"172.21.15.138:6813/2021985596","heartbeat_back_addr":"172.21.15.138:6817/2021985596","heartbeat_front_addr":"172.21.15.138:6815/2021985596","state":["exists","up"]},{"osd":2,"uuid":"97443e05-2513-4832-bb54-f65493df5967","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":95,"up_from":99,"up_thru":104,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6819","nonce":4290419390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6821","nonce":4290419390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6825","nonce":4290419390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6823","nonce":4290419390}]},"public_addr":"172.21.15.138:6819/4290419390","cluster_addr":"172.21.15.138:6821/4290419390","heartbeat_back_addr":"172.21.15.138:6825/4290419390","heartbeat_front_addr":"172.21.15.138:6823/4290419390","state":["exists","up"]},{"osd":3,"uuid":"ac51c165-e007-4390-b95b-26eef984d637","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":100,"up_from":104,"up_thru":104,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6827","nonce":244903146}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6829","nonce":244903146}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6832","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6833","nonce":244903146}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6831","nonce":244903146}]},"public_addr":"172.21.15.138:6827/244903146","cluster_addr":"172.21.15.138:6829/244903146","heartbeat_back_addr":"172.21.15.138:6833/244903146","heartbeat_front_addr":"172.21.15.138:6831/244903146","state":["exists","up"]},{"osd":4,"uuid":"4bef630c-bff4-41a3-a932-de0501541f78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":104,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6801","nonce":743095145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6803","nonce":743095145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6807","nonce":743095145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6805","nonce":743095145}]},"public_addr":"172.21.15.190:6801/743095145","cluster_addr":"172.21.15.190:6803/743095145","heartbeat_back_addr":"172.21.15.190:6807/743095145","heartbeat_front_addr":"172.21.15.190:6805/743095145","state":["exists","up"]},{"osd":5,"uuid":"ac6e16ed-681d-4e9d-87a3-d20d50e01437","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":104,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6808","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6809","nonce":4080051847}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6810","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6811","nonce":4080051847}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6814","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6815","nonce":4080051847}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6812","nonce":4080051847},{"type":"v1","addr":"172.21.15.190:6813","nonce":4080051847}]},"public_addr":"172.21.15.190:6809/4080051847","cluster_addr":"172.21.15.190:6811/4080051847","heartbeat_back_addr":"172.21.15.190:6815/4080051847","heartbeat_front_addr":"172.21.15.190:6813/4080051847","state":["exists","up"]},{"osd":6,"uuid":"7a6f65a5-16dc-45de-854d-751d9151e85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6816","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6817","nonce":797186189}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6818","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6819","nonce":797186189}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6822","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6823","nonce":797186189}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6820","nonce":797186189},{"type":"v1","addr":"172.21.15.190:6821","nonce":797186189}]},"public_addr":"172.21.15.190:6817/797186189","cluster_addr":"172.21.15.190:6819/797186189","heartbeat_back_addr":"172.21.15.190:6823/797186189","heartbeat_front_addr":"172.21.15.190:6821/797186189","state":["exists","up"]},{"osd":7,"uuid":"09a5ad5d-d58f-4fcf-8b09-10f395bd01f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":104,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6824","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6825","nonce":185238034}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6826","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6827","nonce":185238034}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6830","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6831","nonce":185238034}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6828","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6829","nonce":185238034}]},"public_addr":"172.21.15.190:6825/185238034","cluster_addr":"172.21.15.190:6827/185238034","heartbeat_back_addr":"172.21.15.190:6831/185238034","heartbeat_front_addr":"172.21.15.190:6829/185238034","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:58:54.313619+0000","dead_epoch":85},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:16.837452+0000","dead_epoch":90},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:39.610865+0000","dead_epoch":95},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:03.127916+0000","dead_epoch":100},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:23.838460+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:45.582295+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:07.078976+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:29.080850+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.13","mappings":[{"from":1,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.138:6801/3849457598":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/319567031":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/2091838076":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/1464782684":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/25071367":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/3940341494":"2023-10-02T08:57:39.299924+0000","172.21.15.190:0/3151728630":"2023-10-02T09:07:17.185267+0000","172.21.15.138:6801/3165627846":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6800/3165627846":"2023-10-02T08:57:18.125987+0000","172.21.15.138:0/3256141027":"2023-10-02T09:07:55.156656+0000","172.21.15.190:6835/2242576319":"2023-10-02T09:07:17.185267+0000","172.21.15.190:0/2286153897":"2023-10-02T09:07:17.185267+0000","172.21.15.138:0/2575047401":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6801/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.190:0/2209516562":"2023-10-02T09:07:17.185267+0000","172.21.15.190:0/3337553974":"2023-10-02T09:07:17.185267+0000","172.21.15.138:0/2018177063":"2023-10-02T08:57:18.125987+0000","172.21.15.138:0/523071753":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/2054656381":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/1836785572":"2023-10-02T09:06:07.582189+0000","172.21.15.138:6800/3849457598":"2023-10-02T09:07:55.156656+0000","172.21.15.138:6800/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.190:6834/2242576319":"2023-10-02T09:07:17.185267+0000","172.21.15.190:0/299211527":"2023-10-02T09:07:17.185267+0000","172.21.15.138:6801/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:0/3533950111":"2023-10-02T09:06:07.582189+0000","172.21.15.190:0/710760207":"2023-10-02T09:07:17.185267+0000","172.21.15.138:6800/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:0/3675584653":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6800/974662983":"2023-10-02T09:06:07.582189+0000","172.21.15.138:6801/974662983":"2023-10-02T09:06:07.582189+0000","172.21.15.138:0/2334114390":"2023-10-02T09:01:44.817357+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T09:13:34.255 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-01T09:13:34.256 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-01T09:13:34.256 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-01T09:13:34.257 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-01T09:13:34.257 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-01T09:13:34.258 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-01T09:13:34.258 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-01T09:13:34.258 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-01T09:13:34.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:34 smithi138 ceph-mon[185886]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:34.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:34 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4036178889' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:13:34.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:34 smithi138 ceph-mon[192513]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:34.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:34 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4036178889' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:13:34.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:34 smithi190 ceph-mon[160435]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:34 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4036178889' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:13:35.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:13:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:13:34] "GET /metrics HTTP/1.1" 200 33899 "" "Prometheus/2.33.4" 2023-10-01T09:13:35.588 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:35.732 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:35.847 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:35.901 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:36.143 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:36.279 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:36.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:36 smithi138 ceph-mon[185886]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:36.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:36 smithi138 ceph-mon[192513]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:36.759 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:36.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:36 smithi190 ceph-mon[160435]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:37.384 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:37 smithi190 ceph-mon[160435]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:38.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:37 smithi138 ceph-mon[185886]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:38.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:37 smithi138 ceph-mon[192513]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:40.311 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:40 smithi138 ceph-mon[192513]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:40.311 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:40 smithi138 ceph-mon[185886]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:40.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:13:40.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:13:40.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:40 smithi190 ceph-mon[160435]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:13:42.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:42 smithi138 ceph-mon[185886]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:42 smithi138 ceph-mon[192513]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:42.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:42 smithi190 ceph-mon[160435]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:43.460 INFO:teuthology.orchestra.run.smithi138.stdout:446676598792 2023-10-01T09:13:43.461 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-01T09:13:43.512 INFO:teuthology.orchestra.run.smithi138.stderr:time="2023-10-01T09:13:43Z" level=error msg="unable to close conn: \"close unixpacket @->/proc/self/fd/14/attach: use of closed network connection\"" 2023-10-01T09:13:44.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:44 smithi138 ceph-mon[185886]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:44.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:44 smithi138 ceph-mon[192513]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:44.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:44 smithi190 ceph-mon[160435]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:45.042 INFO:teuthology.orchestra.run.smithi138.stdout:403726925845 2023-10-01T09:13:45.042 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-01T09:13:45.258 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:13:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:13:44] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.33.4" 2023-10-01T09:13:45.278 INFO:teuthology.orchestra.run.smithi138.stdout:180388626586 2023-10-01T09:13:45.279 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-01T09:13:45.513 INFO:teuthology.orchestra.run.smithi138.stdout:382252089379 2023-10-01T09:13:45.513 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-01T09:13:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:46 smithi138 ceph-mon[192513]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:46 smithi138 ceph-mon[185886]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:46.744 INFO:teuthology.orchestra.run.smithi138.stdout:425201762318 2023-10-01T09:13:46.744 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-01T09:13:46.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:46 smithi190 ceph-mon[160435]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:48.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:47 smithi190 ceph-mon[160435]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:48.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:47 smithi138 ceph-mon[192513]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:48.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:47 smithi138 ceph-mon[185886]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:48.859 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:48.882 INFO:teuthology.orchestra.run.smithi138.stdout:206158430357 2023-10-01T09:13:48.883 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-01T09:13:48.944 INFO:teuthology.orchestra.run.smithi138.stdout:154618822816 2023-10-01T09:13:48.944 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-01T09:13:48.981 INFO:teuthology.orchestra.run.smithi138.stdout:128849019043 2023-10-01T09:13:48.981 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-01T09:13:49.264 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:49.317 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:49.483 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:49.553 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:50.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:50 smithi138 ceph-mon[192513]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:50.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:50 smithi138 ceph-mon[185886]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:50.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:50 smithi190 ceph-mon[160435]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:50.908 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:50.908 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:52 smithi190 ceph-mon[160435]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:52.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:52 smithi138 ceph-mon[185886]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:52 smithi138 ceph-mon[192513]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:53.302 INFO:teuthology.orchestra.run.smithi138.stdout:382252089380 2023-10-01T09:13:53.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/533219034' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T09:13:53.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/533219034' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T09:13:53.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/533219034' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T09:13:54.509 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:54 smithi138 ceph-mon[185886]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:54.510 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:54 smithi138 ceph-mon[192513]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:54.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:54 smithi190 ceph-mon[160435]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:55.042 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:13:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:13:54] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.33.4" 2023-10-01T09:13:55.059 INFO:tasks.cephadm.ceph_manager.ceph:need seq 382252089379 got 382252089380 for osd.0 2023-10-01T09:13:55.060 DEBUG:teuthology.parallel:result is None 2023-10-01T09:13:55.140 INFO:teuthology.orchestra.run.smithi138.stdout:180388626588 2023-10-01T09:13:55.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:55 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1701670068' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T09:13:55.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:13:55.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:55 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1701670068' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T09:13:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:13:55.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:55 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1701670068' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T09:13:55.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:13:56.750 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:56 smithi138 ceph-mon[185886]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:56.751 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:56 smithi138 ceph-mon[192513]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:56.751 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626586 got 180388626588 for osd.6 2023-10-01T09:13:56.752 DEBUG:teuthology.parallel:result is None 2023-10-01T09:13:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:56 smithi190 ceph-mon[160435]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:56.888 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:13:56.985 INFO:teuthology.orchestra.run.smithi138.stdout:425201762321 2023-10-01T09:13:57.202 INFO:teuthology.orchestra.run.smithi138.stdout:206158430359 2023-10-01T09:13:57.650 INFO:teuthology.orchestra.run.smithi138.stdout:403726925847 2023-10-01T09:13:58.095 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762318 got 425201762321 for osd.2 2023-10-01T09:13:58.095 DEBUG:teuthology.parallel:result is None 2023-10-01T09:13:58.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1827824291' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T09:13:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4128054714' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T09:13:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:57 smithi190 ceph-mon[160435]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2456649997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T09:13:58.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1827824291' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T09:13:58.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4128054714' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T09:13:58.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:57 smithi138 ceph-mon[185886]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:58.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2456649997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T09:13:58.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1827824291' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T09:13:58.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4128054714' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T09:13:58.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:57 smithi138 ceph-mon[192513]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:58.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2456649997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T09:13:58.857 INFO:teuthology.orchestra.run.smithi138.stdout:128849019045 2023-10-01T09:13:59.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:58 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/106773469' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T09:13:59.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:58 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/106773469' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T09:13:59.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:58 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/106773469' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T09:13:59.632 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925845 got 403726925847 for osd.1 2023-10-01T09:13:59.632 DEBUG:teuthology.parallel:result is None 2023-10-01T09:13:59.721 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430357 got 206158430359 for osd.7 2023-10-01T09:13:59.721 DEBUG:teuthology.parallel:result is None 2023-10-01T09:13:59.957 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:13:59 smithi138 ceph-mon[192513]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:59.957 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:13:59 smithi138 ceph-mon[185886]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:13:59.979 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019043 got 128849019045 for osd.4 2023-10-01T09:13:59.979 DEBUG:teuthology.parallel:result is None 2023-10-01T09:14:00.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:13:59 smithi190 ceph-mon[160435]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:00.683 INFO:teuthology.orchestra.run.smithi138.stdout:446676598795 2023-10-01T09:14:00.977 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:00 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3786031892' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T09:14:00.977 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:00 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3786031892' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T09:14:01.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:00 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3786031892' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T09:14:01.598 INFO:tasks.cephadm.ceph_manager.ceph:need seq 446676598792 got 446676598795 for osd.3 2023-10-01T09:14:01.599 DEBUG:teuthology.parallel:result is None 2023-10-01T09:14:01.948 INFO:teuthology.orchestra.run.smithi138.stdout:154618822819 2023-10-01T09:14:02.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:01 smithi190 ceph-mon[160435]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:02.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:01 smithi138 ceph-mon[192513]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:01 smithi138 ceph-mon[185886]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:02.692 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822816 got 154618822819 for osd.5 2023-10-01T09:14:02.693 DEBUG:teuthology.parallel:result is None 2023-10-01T09:14:02.693 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-01T09:14:02.693 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:14:02.943 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:02 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1381581715' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T09:14:02.944 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:02 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1381581715' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T09:14:03.044 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:03.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:02 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1381581715' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T09:14:04.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:03 smithi190 ceph-mon[160435]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:03 smithi138 ceph-mon[192513]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:03 smithi138 ceph-mon[185886]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:04.808 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:14:04.810 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:14:05.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:14:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:14:04] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.33.4" 2023-10-01T09:14:05.524 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":207,"stamp":"2023-10-01T09:14:03.262740+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6302,"num_read_kb":12700,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341100,"kb_used_data":22636,"kb_used_omap":30,"kb_used_meta":1318369,"kb_avail":748620116,"statfs":{"total":767960285184,"available":766586998784,"internally_reserved":0,"allocated":23179264,"data_stored":19087576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30777,"internal_metadata":1350010823},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002355"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.529858+0000","last_change":"2023-10-01T09:12:10.003833+0000","last_active":"2023-10-01T09:13:12.529858+0000","last_peered":"2023-10-01T09:13:12.529858+0000","last_clean":"2023-10-01T09:13:12.529858+0000","last_became_active":"2023-10-01T09:12:10.003482+0000","last_became_peered":"2023-10-01T09:12:10.003482+0000","last_unstale":"2023-10-01T09:13:12.529858+0000","last_undegraded":"2023-10-01T09:13:12.529858+0000","last_fullsized":"2023-10-01T09:13:12.529858+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530638+0000","last_change":"2023-10-01T09:12:40.944944+0000","last_active":"2023-10-01T09:13:12.530638+0000","last_peered":"2023-10-01T09:13:12.530638+0000","last_clean":"2023-10-01T09:13:12.530638+0000","last_became_active":"2023-10-01T09:12:40.944613+0000","last_became_peered":"2023-10-01T09:12:40.944613+0000","last_unstale":"2023-10-01T09:13:12.530638+0000","last_undegraded":"2023-10-01T09:13:12.530638+0000","last_fullsized":"2023-10-01T09:13:12.530638+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:48:20.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.549800+0000","last_change":"2023-10-01T09:13:13.549800+0000","last_active":"2023-10-01T09:13:13.549800+0000","last_peered":"2023-10-01T09:13:13.549800+0000","last_clean":"2023-10-01T09:13:13.549800+0000","last_became_active":"2023-10-01T09:13:13.549480+0000","last_became_peered":"2023-10-01T09:13:13.549480+0000","last_unstale":"2023-10-01T09:13:13.549800+0000","last_undegraded":"2023-10-01T09:13:13.549800+0000","last_fullsized":"2023-10-01T09:13:13.549800+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:08.673916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":79,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556295+0000","last_change":"2023-10-01T09:13:13.556295+0000","last_active":"2023-10-01T09:13:13.556295+0000","last_peered":"2023-10-01T09:13:13.556295+0000","last_clean":"2023-10-01T09:13:13.556295+0000","last_became_active":"2023-10-01T09:13:13.555953+0000","last_became_peered":"2023-10-01T09:13:13.555953+0000","last_unstale":"2023-10-01T09:13:13.556295+0000","last_undegraded":"2023-10-01T09:13:13.556295+0000","last_fullsized":"2023-10-01T09:13:13.556295+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:04:06.892156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554959+0000","last_change":"2023-10-01T09:13:13.554959+0000","last_active":"2023-10-01T09:13:13.554959+0000","last_peered":"2023-10-01T09:13:13.554959+0000","last_clean":"2023-10-01T09:13:13.554959+0000","last_became_active":"2023-10-01T09:13:13.554589+0000","last_became_peered":"2023-10-01T09:13:13.554589+0000","last_unstale":"2023-10-01T09:13:13.554959+0000","last_undegraded":"2023-10-01T09:13:13.554959+0000","last_fullsized":"2023-10-01T09:13:13.554959+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":58,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.541271+0000","last_change":"2023-10-01T09:03:03.202063+0000","last_active":"2023-10-01T09:13:12.541271+0000","last_peered":"2023-10-01T09:13:12.541271+0000","last_clean":"2023-10-01T09:13:12.541271+0000","last_became_active":"2023-10-01T09:03:03.201769+0000","last_became_peered":"2023-10-01T09:03:03.201769+0000","last_unstale":"2023-10-01T09:13:12.541271+0000","last_undegraded":"2023-10-01T09:13:12.541271+0000","last_fullsized":"2023-10-01T09:13:12.541271+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555744+0000","last_change":"2023-10-01T09:13:13.555744+0000","last_active":"2023-10-01T09:13:13.555744+0000","last_peered":"2023-10-01T09:13:13.555744+0000","last_clean":"2023-10-01T09:13:13.555744+0000","last_became_active":"2023-10-01T09:13:13.555163+0000","last_became_peered":"2023-10-01T09:13:13.555163+0000","last_unstale":"2023-10-01T09:13:13.555744+0000","last_undegraded":"2023-10-01T09:13:13.555744+0000","last_fullsized":"2023-10-01T09:13:13.555744+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.540456+0000","last_change":"2023-10-01T09:12:10.000449+0000","last_active":"2023-10-01T09:13:12.540456+0000","last_peered":"2023-10-01T09:13:12.540456+0000","last_clean":"2023-10-01T09:13:12.540456+0000","last_became_active":"2023-10-01T09:12:10.000000+0000","last_became_peered":"2023-10-01T09:12:10.000000+0000","last_unstale":"2023-10-01T09:13:12.540456+0000","last_undegraded":"2023-10-01T09:13:12.540456+0000","last_fullsized":"2023-10-01T09:13:12.540456+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":93,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.548647+0000","last_change":"2023-10-01T09:13:13.548647+0000","last_active":"2023-10-01T09:13:13.548647+0000","last_peered":"2023-10-01T09:13:13.548647+0000","last_clean":"2023-10-01T09:13:13.548647+0000","last_became_active":"2023-10-01T09:13:13.548382+0000","last_became_peered":"2023-10-01T09:13:13.548382+0000","last_unstale":"2023-10-01T09:13:13.548647+0000","last_undegraded":"2023-10-01T09:13:13.548647+0000","last_fullsized":"2023-10-01T09:13:13.548647+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:17:31.791258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.540696+0000","last_change":"2023-10-01T09:10:58.801161+0000","last_active":"2023-10-01T09:13:13.540696+0000","last_peered":"2023-10-01T09:13:13.540696+0000","last_clean":"2023-10-01T09:13:13.540696+0000","last_became_active":"2023-10-01T09:10:58.800794+0000","last_became_peered":"2023-10-01T09:10:58.800794+0000","last_unstale":"2023-10-01T09:13:13.540696+0000","last_undegraded":"2023-10-01T09:13:13.540696+0000","last_fullsized":"2023-10-01T09:13:13.540696+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:35.019445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.531915+0000","last_change":"2023-10-01T09:03:03.754839+0000","last_active":"2023-10-01T09:13:12.531915+0000","last_peered":"2023-10-01T09:13:12.531915+0000","last_clean":"2023-10-01T09:13:12.531915+0000","last_became_active":"2023-10-01T09:03:03.754506+0000","last_became_peered":"2023-10-01T09:03:03.754506+0000","last_unstale":"2023-10-01T09:13:12.531915+0000","last_undegraded":"2023-10-01T09:13:12.531915+0000","last_fullsized":"2023-10-01T09:13:12.531915+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.532609+0000","last_change":"2023-10-01T09:12:10.003174+0000","last_active":"2023-10-01T09:13:12.532609+0000","last_peered":"2023-10-01T09:13:12.532609+0000","last_clean":"2023-10-01T09:13:12.532609+0000","last_became_active":"2023-10-01T09:12:10.002808+0000","last_became_peered":"2023-10-01T09:12:10.002808+0000","last_unstale":"2023-10-01T09:13:12.532609+0000","last_undegraded":"2023-10-01T09:13:12.532609+0000","last_fullsized":"2023-10-01T09:13:12.532609+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:25.037091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":143,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535113+0000","last_change":"2023-10-01T09:12:40.949544+0000","last_active":"2023-10-01T09:13:12.535113+0000","last_peered":"2023-10-01T09:13:12.535113+0000","last_clean":"2023-10-01T09:13:12.535113+0000","last_became_active":"2023-10-01T09:12:40.949054+0000","last_became_peered":"2023-10-01T09:12:40.949054+0000","last_unstale":"2023-10-01T09:13:12.535113+0000","last_undegraded":"2023-10-01T09:13:12.535113+0000","last_fullsized":"2023-10-01T09:13:12.535113+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.552192+0000","last_change":"2023-10-01T09:13:13.552192+0000","last_active":"2023-10-01T09:13:13.552192+0000","last_peered":"2023-10-01T09:13:13.552192+0000","last_clean":"2023-10-01T09:13:13.552192+0000","last_became_active":"2023-10-01T09:13:13.551887+0000","last_became_peered":"2023-10-01T09:13:13.551887+0000","last_unstale":"2023-10-01T09:13:13.552192+0000","last_undegraded":"2023-10-01T09:13:13.552192+0000","last_fullsized":"2023-10-01T09:13:13.552192+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:23.633557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":491,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534872+0000","last_change":"2023-10-01T09:12:10.002334+0000","last_active":"2023-10-01T09:13:12.534872+0000","last_peered":"2023-10-01T09:13:12.534872+0000","last_clean":"2023-10-01T09:13:12.534872+0000","last_became_active":"2023-10-01T09:12:10.001791+0000","last_became_peered":"2023-10-01T09:12:10.001791+0000","last_unstale":"2023-10-01T09:13:12.534872+0000","last_undegraded":"2023-10-01T09:13:12.534872+0000","last_fullsized":"2023-10-01T09:13:12.534872+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:24:27.632780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":71,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.553580+0000","last_change":"2023-10-01T09:13:13.553580+0000","last_active":"2023-10-01T09:13:13.553580+0000","last_peered":"2023-10-01T09:13:13.553580+0000","last_clean":"2023-10-01T09:13:13.553580+0000","last_became_active":"2023-10-01T09:13:13.553011+0000","last_became_peered":"2023-10-01T09:13:13.553011+0000","last_unstale":"2023-10-01T09:13:13.553580+0000","last_undegraded":"2023-10-01T09:13:13.553580+0000","last_fullsized":"2023-10-01T09:13:13.553580+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:54:30.369308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530101+0000","last_change":"2023-10-01T09:10:58.797034+0000","last_active":"2023-10-01T09:13:12.530101+0000","last_peered":"2023-10-01T09:13:12.530101+0000","last_clean":"2023-10-01T09:13:12.530101+0000","last_became_active":"2023-10-01T09:10:58.796730+0000","last_became_peered":"2023-10-01T09:10:58.796730+0000","last_unstale":"2023-10-01T09:13:12.530101+0000","last_undegraded":"2023-10-01T09:13:12.530101+0000","last_fullsized":"2023-10-01T09:13:12.530101+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:32:15.982890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.536708+0000","last_change":"2023-10-01T09:12:40.948216+0000","last_active":"2023-10-01T09:13:12.536708+0000","last_peered":"2023-10-01T09:13:12.536708+0000","last_clean":"2023-10-01T09:13:12.536708+0000","last_became_active":"2023-10-01T09:12:40.947879+0000","last_became_peered":"2023-10-01T09:12:40.947879+0000","last_unstale":"2023-10-01T09:13:12.536708+0000","last_undegraded":"2023-10-01T09:13:12.536708+0000","last_fullsized":"2023-10-01T09:13:12.536708+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:46:40.612337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":465,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534902+0000","last_change":"2023-10-01T09:10:58.797964+0000","last_active":"2023-10-01T09:13:12.534902+0000","last_peered":"2023-10-01T09:13:12.534902+0000","last_clean":"2023-10-01T09:13:12.534902+0000","last_became_active":"2023-10-01T09:10:58.797447+0000","last_became_peered":"2023-10-01T09:10:58.797447+0000","last_unstale":"2023-10-01T09:13:12.534902+0000","last_undegraded":"2023-10-01T09:13:12.534902+0000","last_fullsized":"2023-10-01T09:13:12.534902+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.532709+0000","last_change":"2023-10-01T09:12:40.944991+0000","last_active":"2023-10-01T09:13:12.532709+0000","last_peered":"2023-10-01T09:13:12.532709+0000","last_clean":"2023-10-01T09:13:12.532709+0000","last_became_active":"2023-10-01T09:12:40.944688+0000","last_became_peered":"2023-10-01T09:12:40.944688+0000","last_unstale":"2023-10-01T09:13:12.532709+0000","last_undegraded":"2023-10-01T09:13:12.532709+0000","last_fullsized":"2023-10-01T09:13:12.532709+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:30.015801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.552391+0000","last_change":"2023-10-01T09:13:13.552391+0000","last_active":"2023-10-01T09:13:13.552391+0000","last_peered":"2023-10-01T09:13:13.552391+0000","last_clean":"2023-10-01T09:13:13.552391+0000","last_became_active":"2023-10-01T09:13:13.552082+0000","last_became_peered":"2023-10-01T09:13:13.552082+0000","last_unstale":"2023-10-01T09:13:13.552391+0000","last_undegraded":"2023-10-01T09:13:13.552391+0000","last_fullsized":"2023-10-01T09:13:13.552391+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:07:19.929732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":201,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535494+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:13:12.535494+0000","last_peered":"2023-10-01T09:13:12.535494+0000","last_clean":"2023-10-01T09:13:12.535494+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:13:12.535494+0000","last_undegraded":"2023-10-01T09:13:12.535494+0000","last_fullsized":"2023-10-01T09:13:12.535494+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":55,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534655+0000","last_change":"2023-10-01T09:03:03.227254+0000","last_active":"2023-10-01T09:13:12.534655+0000","last_peered":"2023-10-01T09:13:12.534655+0000","last_clean":"2023-10-01T09:13:12.534655+0000","last_became_active":"2023-10-01T09:03:03.226941+0000","last_became_peered":"2023-10-01T09:03:03.226941+0000","last_unstale":"2023-10-01T09:13:12.534655+0000","last_undegraded":"2023-10-01T09:13:12.534655+0000","last_fullsized":"2023-10-01T09:13:12.534655+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":63,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556049+0000","last_change":"2023-10-01T09:13:13.556049+0000","last_active":"2023-10-01T09:13:13.556049+0000","last_peered":"2023-10-01T09:13:13.556049+0000","last_clean":"2023-10-01T09:13:13.556049+0000","last_became_active":"2023-10-01T09:13:13.555739+0000","last_became_peered":"2023-10-01T09:13:13.555739+0000","last_unstale":"2023-10-01T09:13:13.556049+0000","last_undegraded":"2023-10-01T09:13:13.556049+0000","last_fullsized":"2023-10-01T09:13:13.556049+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:50:34.475690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":307,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534586+0000","last_change":"2023-10-01T09:10:58.798697+0000","last_active":"2023-10-01T09:13:12.534586+0000","last_peered":"2023-10-01T09:13:12.534586+0000","last_clean":"2023-10-01T09:13:12.534586+0000","last_became_active":"2023-10-01T09:10:58.798203+0000","last_became_peered":"2023-10-01T09:10:58.798203+0000","last_unstale":"2023-10-01T09:13:12.534586+0000","last_undegraded":"2023-10-01T09:13:12.534586+0000","last_fullsized":"2023-10-01T09:13:12.534586+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":921,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539380+0000","last_change":"2023-10-01T09:12:40.946131+0000","last_active":"2023-10-01T09:13:12.539380+0000","last_peered":"2023-10-01T09:13:12.539380+0000","last_clean":"2023-10-01T09:13:12.539380+0000","last_became_active":"2023-10-01T09:12:40.945524+0000","last_became_peered":"2023-10-01T09:12:40.945524+0000","last_unstale":"2023-10-01T09:13:12.539380+0000","last_undegraded":"2023-10-01T09:13:12.539380+0000","last_fullsized":"2023-10-01T09:13:12.539380+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535348+0000","last_change":"2023-10-01T09:12:09.994071+0000","last_active":"2023-10-01T09:13:12.535348+0000","last_peered":"2023-10-01T09:13:12.535348+0000","last_clean":"2023-10-01T09:13:12.535348+0000","last_became_active":"2023-10-01T09:12:09.993782+0000","last_became_peered":"2023-10-01T09:12:09.993782+0000","last_unstale":"2023-10-01T09:13:12.535348+0000","last_undegraded":"2023-10-01T09:13:12.535348+0000","last_fullsized":"2023-10-01T09:13:12.535348+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:12:49.880685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535495+0000","last_change":"2023-10-01T09:12:40.946267+0000","last_active":"2023-10-01T09:13:12.535495+0000","last_peered":"2023-10-01T09:13:12.535495+0000","last_clean":"2023-10-01T09:13:12.535495+0000","last_became_active":"2023-10-01T09:12:40.945970+0000","last_became_peered":"2023-10-01T09:12:40.945970+0000","last_unstale":"2023-10-01T09:13:12.535495+0000","last_undegraded":"2023-10-01T09:13:12.535495+0000","last_fullsized":"2023-10-01T09:13:12.535495+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:42.904509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":375,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551669+0000","last_change":"2023-10-01T09:13:13.551669+0000","last_active":"2023-10-01T09:13:13.551669+0000","last_peered":"2023-10-01T09:13:13.551669+0000","last_clean":"2023-10-01T09:13:13.551669+0000","last_became_active":"2023-10-01T09:13:13.550982+0000","last_became_peered":"2023-10-01T09:13:13.550982+0000","last_unstale":"2023-10-01T09:13:13.551669+0000","last_undegraded":"2023-10-01T09:13:13.551669+0000","last_fullsized":"2023-10-01T09:13:13.551669+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:21:09.879713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.549385+0000","last_change":"2023-10-01T09:13:13.549385+0000","last_active":"2023-10-01T09:13:13.549385+0000","last_peered":"2023-10-01T09:13:13.549385+0000","last_clean":"2023-10-01T09:13:13.549385+0000","last_became_active":"2023-10-01T09:13:13.549035+0000","last_became_peered":"2023-10-01T09:13:13.549035+0000","last_unstale":"2023-10-01T09:13:13.549385+0000","last_undegraded":"2023-10-01T09:13:13.549385+0000","last_fullsized":"2023-10-01T09:13:13.549385+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:52:41.408995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530244+0000","last_change":"2023-10-01T09:10:58.799722+0000","last_active":"2023-10-01T09:13:12.530244+0000","last_peered":"2023-10-01T09:13:12.530244+0000","last_clean":"2023-10-01T09:13:12.530244+0000","last_became_active":"2023-10-01T09:10:58.799313+0000","last_became_peered":"2023-10-01T09:10:58.799313+0000","last_unstale":"2023-10-01T09:13:12.530244+0000","last_undegraded":"2023-10-01T09:13:12.530244+0000","last_fullsized":"2023-10-01T09:13:12.530244+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":805,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539880+0000","last_change":"2023-10-01T09:12:09.997936+0000","last_active":"2023-10-01T09:13:12.539880+0000","last_peered":"2023-10-01T09:13:12.539880+0000","last_clean":"2023-10-01T09:13:12.539880+0000","last_became_active":"2023-10-01T09:12:09.997427+0000","last_became_peered":"2023-10-01T09:12:09.997427+0000","last_unstale":"2023-10-01T09:13:12.539880+0000","last_undegraded":"2023-10-01T09:13:12.539880+0000","last_fullsized":"2023-10-01T09:13:12.539880+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1219,"num_read_kb":1114,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.533191+0000","last_change":"2023-10-01T09:12:10.001493+0000","last_active":"2023-10-01T09:13:12.533191+0000","last_peered":"2023-10-01T09:13:12.533191+0000","last_clean":"2023-10-01T09:13:12.533191+0000","last_became_active":"2023-10-01T09:12:10.000899+0000","last_became_peered":"2023-10-01T09:12:10.000899+0000","last_unstale":"2023-10-01T09:13:12.533191+0000","last_undegraded":"2023-10-01T09:13:12.533191+0000","last_fullsized":"2023-10-01T09:13:12.533191+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:47:54.011059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551681+0000","last_change":"2023-10-01T09:13:13.551681+0000","last_active":"2023-10-01T09:13:13.551681+0000","last_peered":"2023-10-01T09:13:13.551681+0000","last_clean":"2023-10-01T09:13:13.551681+0000","last_became_active":"2023-10-01T09:13:13.551268+0000","last_became_peered":"2023-10-01T09:13:13.551268+0000","last_unstale":"2023-10-01T09:13:13.551681+0000","last_undegraded":"2023-10-01T09:13:13.551681+0000","last_fullsized":"2023-10-01T09:13:13.551681+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":917,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.536626+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:13:12.536626+0000","last_peered":"2023-10-01T09:13:12.536626+0000","last_clean":"2023-10-01T09:13:12.536626+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:13:12.536626+0000","last_undegraded":"2023-10-01T09:13:12.536626+0000","last_fullsized":"2023-10-01T09:13:12.536626+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551757+0000","last_change":"2023-10-01T09:13:13.551757+0000","last_active":"2023-10-01T09:13:13.551757+0000","last_peered":"2023-10-01T09:13:13.551757+0000","last_clean":"2023-10-01T09:13:13.551757+0000","last_became_active":"2023-10-01T09:13:13.551483+0000","last_became_peered":"2023-10-01T09:13:13.551483+0000","last_unstale":"2023-10-01T09:13:13.551757+0000","last_undegraded":"2023-10-01T09:13:13.551757+0000","last_fullsized":"2023-10-01T09:13:13.551757+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:33:46.184050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555563+0000","last_change":"2023-10-01T09:13:13.555563+0000","last_active":"2023-10-01T09:13:13.555563+0000","last_peered":"2023-10-01T09:13:13.555563+0000","last_clean":"2023-10-01T09:13:13.555563+0000","last_became_active":"2023-10-01T09:13:13.555070+0000","last_became_peered":"2023-10-01T09:13:13.555070+0000","last_unstale":"2023-10-01T09:13:13.555563+0000","last_undegraded":"2023-10-01T09:13:13.555563+0000","last_fullsized":"2023-10-01T09:13:13.555563+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:44.557168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.531259+0000","last_change":"2023-10-01T09:12:10.003847+0000","last_active":"2023-10-01T09:13:12.531259+0000","last_peered":"2023-10-01T09:13:12.531259+0000","last_clean":"2023-10-01T09:13:12.531259+0000","last_became_active":"2023-10-01T09:12:10.003525+0000","last_became_peered":"2023-10-01T09:12:10.003525+0000","last_unstale":"2023-10-01T09:13:12.531259+0000","last_undegraded":"2023-10-01T09:13:12.531259+0000","last_fullsized":"2023-10-01T09:13:12.531259+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.532071+0000","last_change":"2023-10-01T09:12:40.945099+0000","last_active":"2023-10-01T09:13:12.532071+0000","last_peered":"2023-10-01T09:13:12.532071+0000","last_clean":"2023-10-01T09:13:12.532071+0000","last_became_active":"2023-10-01T09:12:40.944712+0000","last_became_peered":"2023-10-01T09:12:40.944712+0000","last_unstale":"2023-10-01T09:13:12.532071+0000","last_undegraded":"2023-10-01T09:13:12.532071+0000","last_fullsized":"2023-10-01T09:13:12.532071+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.531487+0000","last_change":"2023-10-01T09:12:40.946791+0000","last_active":"2023-10-01T09:13:12.531487+0000","last_peered":"2023-10-01T09:13:12.531487+0000","last_clean":"2023-10-01T09:13:12.531487+0000","last_became_active":"2023-10-01T09:12:40.946418+0000","last_became_peered":"2023-10-01T09:12:40.946418+0000","last_unstale":"2023-10-01T09:13:12.531487+0000","last_undegraded":"2023-10-01T09:13:12.531487+0000","last_fullsized":"2023-10-01T09:13:12.531487+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.536203+0000","last_change":"2023-10-01T09:12:10.003934+0000","last_active":"2023-10-01T09:13:12.536203+0000","last_peered":"2023-10-01T09:13:12.536203+0000","last_clean":"2023-10-01T09:13:12.536203+0000","last_became_active":"2023-10-01T09:12:10.003597+0000","last_became_peered":"2023-10-01T09:12:10.003597+0000","last_unstale":"2023-10-01T09:13:12.536203+0000","last_undegraded":"2023-10-01T09:13:12.536203+0000","last_fullsized":"2023-10-01T09:13:12.536203+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556491+0000","last_change":"2023-10-01T09:13:13.556491+0000","last_active":"2023-10-01T09:13:13.556491+0000","last_peered":"2023-10-01T09:13:13.556491+0000","last_clean":"2023-10-01T09:13:13.556491+0000","last_became_active":"2023-10-01T09:13:13.556118+0000","last_became_peered":"2023-10-01T09:13:13.556118+0000","last_unstale":"2023-10-01T09:13:13.556491+0000","last_undegraded":"2023-10-01T09:13:13.556491+0000","last_fullsized":"2023-10-01T09:13:13.556491+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554321+0000","last_change":"2023-10-01T09:13:13.554321+0000","last_active":"2023-10-01T09:13:13.554321+0000","last_peered":"2023-10-01T09:13:13.554321+0000","last_clean":"2023-10-01T09:13:13.554321+0000","last_became_active":"2023-10-01T09:13:13.553986+0000","last_became_peered":"2023-10-01T09:13:13.553986+0000","last_unstale":"2023-10-01T09:13:13.554321+0000","last_undegraded":"2023-10-01T09:13:13.554321+0000","last_fullsized":"2023-10-01T09:13:13.554321+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:09:12.848269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530790+0000","last_change":"2023-10-01T09:12:10.001565+0000","last_active":"2023-10-01T09:13:12.530790+0000","last_peered":"2023-10-01T09:13:12.530790+0000","last_clean":"2023-10-01T09:13:12.530790+0000","last_became_active":"2023-10-01T09:12:10.001275+0000","last_became_peered":"2023-10-01T09:12:10.001275+0000","last_unstale":"2023-10-01T09:13:12.530790+0000","last_undegraded":"2023-10-01T09:13:12.530790+0000","last_fullsized":"2023-10-01T09:13:12.530790+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:39.067046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555458+0000","last_change":"2023-10-01T09:13:13.555458+0000","last_active":"2023-10-01T09:13:13.555458+0000","last_peered":"2023-10-01T09:13:13.555458+0000","last_clean":"2023-10-01T09:13:13.555458+0000","last_became_active":"2023-10-01T09:13:13.555058+0000","last_became_peered":"2023-10-01T09:13:13.555058+0000","last_unstale":"2023-10-01T09:13:13.555458+0000","last_undegraded":"2023-10-01T09:13:13.555458+0000","last_fullsized":"2023-10-01T09:13:13.555458+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535389+0000","last_change":"2023-10-01T09:12:09.999571+0000","last_active":"2023-10-01T09:13:12.535389+0000","last_peered":"2023-10-01T09:13:12.535389+0000","last_clean":"2023-10-01T09:13:12.535389+0000","last_became_active":"2023-10-01T09:12:09.999272+0000","last_became_peered":"2023-10-01T09:12:09.999272+0000","last_unstale":"2023-10-01T09:13:12.535389+0000","last_undegraded":"2023-10-01T09:13:12.535389+0000","last_fullsized":"2023-10-01T09:13:12.535389+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":57,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535047+0000","last_change":"2023-10-01T09:03:03.200972+0000","last_active":"2023-10-01T09:13:12.535047+0000","last_peered":"2023-10-01T09:13:12.535047+0000","last_clean":"2023-10-01T09:13:12.535047+0000","last_became_active":"2023-10-01T09:03:03.200695+0000","last_became_peered":"2023-10-01T09:03:03.200695+0000","last_unstale":"2023-10-01T09:13:12.535047+0000","last_undegraded":"2023-10-01T09:13:12.535047+0000","last_fullsized":"2023-10-01T09:13:12.535047+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556674+0000","last_change":"2023-10-01T09:13:13.556674+0000","last_active":"2023-10-01T09:13:13.556674+0000","last_peered":"2023-10-01T09:13:13.556674+0000","last_clean":"2023-10-01T09:13:13.556674+0000","last_became_active":"2023-10-01T09:13:13.556271+0000","last_became_peered":"2023-10-01T09:13:13.556271+0000","last_unstale":"2023-10-01T09:13:13.556674+0000","last_undegraded":"2023-10-01T09:13:13.556674+0000","last_fullsized":"2023-10-01T09:13:13.556674+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535994+0000","last_change":"2023-10-01T09:12:10.003552+0000","last_active":"2023-10-01T09:13:12.535994+0000","last_peered":"2023-10-01T09:13:12.535994+0000","last_clean":"2023-10-01T09:13:12.535994+0000","last_became_active":"2023-10-01T09:12:10.003131+0000","last_became_peered":"2023-10-01T09:12:10.003131+0000","last_unstale":"2023-10-01T09:13:12.535994+0000","last_undegraded":"2023-10-01T09:13:12.535994+0000","last_fullsized":"2023-10-01T09:13:12.535994+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539254+0000","last_change":"2023-10-01T09:03:04.245105+0000","last_active":"2023-10-01T09:13:12.539254+0000","last_peered":"2023-10-01T09:13:12.539254+0000","last_clean":"2023-10-01T09:13:12.539254+0000","last_became_active":"2023-10-01T09:03:04.244807+0000","last_became_peered":"2023-10-01T09:03:04.244807+0000","last_unstale":"2023-10-01T09:13:12.539254+0000","last_undegraded":"2023-10-01T09:13:12.539254+0000","last_fullsized":"2023-10-01T09:13:12.539254+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.537360+0000","last_change":"2023-10-01T09:10:58.797850+0000","last_active":"2023-10-01T09:13:12.537360+0000","last_peered":"2023-10-01T09:13:12.537360+0000","last_clean":"2023-10-01T09:13:12.537360+0000","last_became_active":"2023-10-01T09:10:58.797403+0000","last_became_peered":"2023-10-01T09:10:58.797403+0000","last_unstale":"2023-10-01T09:13:12.537360+0000","last_undegraded":"2023-10-01T09:13:12.537360+0000","last_fullsized":"2023-10-01T09:13:12.537360+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550588+0000","last_change":"2023-10-01T09:13:13.550588+0000","last_active":"2023-10-01T09:13:13.550588+0000","last_peered":"2023-10-01T09:13:13.550588+0000","last_clean":"2023-10-01T09:13:13.550588+0000","last_became_active":"2023-10-01T09:13:13.550271+0000","last_became_peered":"2023-10-01T09:13:13.550271+0000","last_unstale":"2023-10-01T09:13:13.550588+0000","last_undegraded":"2023-10-01T09:13:13.550588+0000","last_fullsized":"2023-10-01T09:13:13.550588+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:19:47.301376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535409+0000","last_change":"2023-10-01T09:10:58.798168+0000","last_active":"2023-10-01T09:13:12.535409+0000","last_peered":"2023-10-01T09:13:12.535409+0000","last_clean":"2023-10-01T09:13:12.535409+0000","last_became_active":"2023-10-01T09:10:58.797781+0000","last_became_peered":"2023-10-01T09:10:58.797781+0000","last_unstale":"2023-10-01T09:13:12.535409+0000","last_undegraded":"2023-10-01T09:13:12.535409+0000","last_fullsized":"2023-10-01T09:13:12.535409+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550434+0000","last_change":"2023-10-01T09:13:13.548531+0000","last_active":"2023-10-01T09:13:13.550434+0000","last_peered":"2023-10-01T09:13:13.550434+0000","last_clean":"2023-10-01T09:13:13.550434+0000","last_became_active":"2023-10-01T09:13:13.547881+0000","last_became_peered":"2023-10-01T09:13:13.547881+0000","last_unstale":"2023-10-01T09:13:13.550434+0000","last_undegraded":"2023-10-01T09:13:13.550434+0000","last_fullsized":"2023-10-01T09:13:13.550434+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:13:24.068359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534146+0000","last_change":"2023-10-01T09:03:03.194952+0000","last_active":"2023-10-01T09:13:12.534146+0000","last_peered":"2023-10-01T09:13:12.534146+0000","last_clean":"2023-10-01T09:13:12.534146+0000","last_became_active":"2023-10-01T09:03:03.194643+0000","last_became_peered":"2023-10-01T09:03:03.194643+0000","last_unstale":"2023-10-01T09:13:12.534146+0000","last_undegraded":"2023-10-01T09:13:12.534146+0000","last_fullsized":"2023-10-01T09:13:12.534146+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.558367+0000","last_change":"2023-10-01T09:13:13.558367+0000","last_active":"2023-10-01T09:13:13.558367+0000","last_peered":"2023-10-01T09:13:13.558367+0000","last_clean":"2023-10-01T09:13:13.558367+0000","last_became_active":"2023-10-01T09:13:13.557952+0000","last_became_peered":"2023-10-01T09:13:13.557952+0000","last_unstale":"2023-10-01T09:13:13.558367+0000","last_undegraded":"2023-10-01T09:13:13.558367+0000","last_fullsized":"2023-10-01T09:13:13.558367+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550521+0000","last_change":"2023-10-01T09:13:13.548094+0000","last_active":"2023-10-01T09:13:13.550521+0000","last_peered":"2023-10-01T09:13:13.550521+0000","last_clean":"2023-10-01T09:13:13.550521+0000","last_became_active":"2023-10-01T09:13:13.547782+0000","last_became_peered":"2023-10-01T09:13:13.547782+0000","last_unstale":"2023-10-01T09:13:13.550521+0000","last_undegraded":"2023-10-01T09:13:13.550521+0000","last_fullsized":"2023-10-01T09:13:13.550521+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:15.220827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550633+0000","last_change":"2023-10-01T09:13:13.550633+0000","last_active":"2023-10-01T09:13:13.550633+0000","last_peered":"2023-10-01T09:13:13.550633+0000","last_clean":"2023-10-01T09:13:13.550633+0000","last_became_active":"2023-10-01T09:13:13.550305+0000","last_became_peered":"2023-10-01T09:13:13.550305+0000","last_unstale":"2023-10-01T09:13:13.550633+0000","last_undegraded":"2023-10-01T09:13:13.550633+0000","last_fullsized":"2023-10-01T09:13:13.550633+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:51:46.559383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534816+0000","last_change":"2023-10-01T09:12:40.945277+0000","last_active":"2023-10-01T09:13:12.534816+0000","last_peered":"2023-10-01T09:13:12.534816+0000","last_clean":"2023-10-01T09:13:12.534816+0000","last_became_active":"2023-10-01T09:12:40.944843+0000","last_became_peered":"2023-10-01T09:12:40.944843+0000","last_unstale":"2023-10-01T09:13:12.534816+0000","last_undegraded":"2023-10-01T09:13:12.534816+0000","last_fullsized":"2023-10-01T09:13:12.534816+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530812+0000","last_change":"2023-10-01T09:12:40.946973+0000","last_active":"2023-10-01T09:13:12.530812+0000","last_peered":"2023-10-01T09:13:12.530812+0000","last_clean":"2023-10-01T09:13:12.530812+0000","last_became_active":"2023-10-01T09:12:40.946427+0000","last_became_peered":"2023-10-01T09:12:40.946427+0000","last_unstale":"2023-10-01T09:13:12.530812+0000","last_undegraded":"2023-10-01T09:13:12.530812+0000","last_fullsized":"2023-10-01T09:13:12.530812+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":77,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530204+0000","last_change":"2023-10-01T09:03:03.200261+0000","last_active":"2023-10-01T09:13:12.530204+0000","last_peered":"2023-10-01T09:13:12.530204+0000","last_clean":"2023-10-01T09:13:12.530204+0000","last_became_active":"2023-10-01T09:03:03.199944+0000","last_became_peered":"2023-10-01T09:03:03.199944+0000","last_unstale":"2023-10-01T09:13:12.530204+0000","last_undegraded":"2023-10-01T09:13:12.530204+0000","last_fullsized":"2023-10-01T09:13:12.530204+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.533264+0000","last_change":"2023-10-01T09:12:40.946520+0000","last_active":"2023-10-01T09:13:12.533264+0000","last_peered":"2023-10-01T09:13:12.533264+0000","last_clean":"2023-10-01T09:13:12.533264+0000","last_became_active":"2023-10-01T09:12:40.946227+0000","last_became_peered":"2023-10-01T09:12:40.946227+0000","last_unstale":"2023-10-01T09:13:12.533264+0000","last_undegraded":"2023-10-01T09:13:12.533264+0000","last_fullsized":"2023-10-01T09:13:12.533264+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:06:06.451923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556801+0000","last_change":"2023-10-01T09:13:13.556801+0000","last_active":"2023-10-01T09:13:13.556801+0000","last_peered":"2023-10-01T09:13:13.556801+0000","last_clean":"2023-10-01T09:13:13.556801+0000","last_became_active":"2023-10-01T09:13:13.556397+0000","last_became_peered":"2023-10-01T09:13:13.556397+0000","last_unstale":"2023-10-01T09:13:13.556801+0000","last_undegraded":"2023-10-01T09:13:13.556801+0000","last_fullsized":"2023-10-01T09:13:13.556801+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554942+0000","last_change":"2023-10-01T09:13:13.554942+0000","last_active":"2023-10-01T09:13:13.554942+0000","last_peered":"2023-10-01T09:13:13.554942+0000","last_clean":"2023-10-01T09:13:13.554942+0000","last_became_active":"2023-10-01T09:13:13.554664+0000","last_became_peered":"2023-10-01T09:13:13.554664+0000","last_unstale":"2023-10-01T09:13:13.554942+0000","last_undegraded":"2023-10-01T09:13:13.554942+0000","last_fullsized":"2023-10-01T09:13:13.554942+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:43:14.532709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539899+0000","last_change":"2023-10-01T09:12:10.003091+0000","last_active":"2023-10-01T09:13:12.539899+0000","last_peered":"2023-10-01T09:13:12.539899+0000","last_clean":"2023-10-01T09:13:12.539899+0000","last_became_active":"2023-10-01T09:12:10.002773+0000","last_became_peered":"2023-10-01T09:12:10.002773+0000","last_unstale":"2023-10-01T09:13:12.539899+0000","last_undegraded":"2023-10-01T09:13:12.539899+0000","last_fullsized":"2023-10-01T09:13:12.539899+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555368+0000","last_change":"2023-10-01T09:13:13.555368+0000","last_active":"2023-10-01T09:13:13.555368+0000","last_peered":"2023-10-01T09:13:13.555368+0000","last_clean":"2023-10-01T09:13:13.555368+0000","last_became_active":"2023-10-01T09:13:13.554806+0000","last_became_peered":"2023-10-01T09:13:13.554806+0000","last_unstale":"2023-10-01T09:13:13.555368+0000","last_undegraded":"2023-10-01T09:13:13.555368+0000","last_fullsized":"2023-10-01T09:13:13.555368+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:24:32.687833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550551+0000","last_change":"2023-10-01T09:13:13.550193+0000","last_active":"2023-10-01T09:13:13.550551+0000","last_peered":"2023-10-01T09:13:13.550551+0000","last_clean":"2023-10-01T09:13:13.550551+0000","last_became_active":"2023-10-01T09:13:13.549900+0000","last_became_peered":"2023-10-01T09:13:13.549900+0000","last_unstale":"2023-10-01T09:13:13.550551+0000","last_undegraded":"2023-10-01T09:13:13.550551+0000","last_fullsized":"2023-10-01T09:13:13.550551+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:22.643161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534962+0000","last_change":"2023-10-01T09:12:40.947591+0000","last_active":"2023-10-01T09:13:12.534962+0000","last_peered":"2023-10-01T09:13:12.534962+0000","last_clean":"2023-10-01T09:13:12.534962+0000","last_became_active":"2023-10-01T09:12:40.947242+0000","last_became_peered":"2023-10-01T09:12:40.947242+0000","last_unstale":"2023-10-01T09:13:12.534962+0000","last_undegraded":"2023-10-01T09:13:12.534962+0000","last_fullsized":"2023-10-01T09:13:12.534962+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555490+0000","last_change":"2023-10-01T09:13:13.555490+0000","last_active":"2023-10-01T09:13:13.555490+0000","last_peered":"2023-10-01T09:13:13.555490+0000","last_clean":"2023-10-01T09:13:13.555490+0000","last_became_active":"2023-10-01T09:13:13.554506+0000","last_became_peered":"2023-10-01T09:13:13.554506+0000","last_unstale":"2023-10-01T09:13:13.555490+0000","last_undegraded":"2023-10-01T09:13:13.555490+0000","last_fullsized":"2023-10-01T09:13:13.555490+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:18.834474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.548695+0000","last_change":"2023-10-01T09:13:13.548695+0000","last_active":"2023-10-01T09:13:13.548695+0000","last_peered":"2023-10-01T09:13:13.548695+0000","last_clean":"2023-10-01T09:13:13.548695+0000","last_became_active":"2023-10-01T09:13:13.548256+0000","last_became_peered":"2023-10-01T09:13:13.548256+0000","last_unstale":"2023-10-01T09:13:13.548695+0000","last_undegraded":"2023-10-01T09:13:13.548695+0000","last_fullsized":"2023-10-01T09:13:13.548695+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:35:28.655733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551596+0000","last_change":"2023-10-01T09:13:13.551596+0000","last_active":"2023-10-01T09:13:13.551596+0000","last_peered":"2023-10-01T09:13:13.551596+0000","last_clean":"2023-10-01T09:13:13.551596+0000","last_became_active":"2023-10-01T09:13:13.551300+0000","last_became_peered":"2023-10-01T09:13:13.551300+0000","last_unstale":"2023-10-01T09:13:13.551596+0000","last_undegraded":"2023-10-01T09:13:13.551596+0000","last_fullsized":"2023-10-01T09:13:13.551596+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:57:45.081780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554780+0000","last_change":"2023-10-01T09:13:13.554780+0000","last_active":"2023-10-01T09:13:13.554780+0000","last_peered":"2023-10-01T09:13:13.554780+0000","last_clean":"2023-10-01T09:13:13.554780+0000","last_became_active":"2023-10-01T09:13:13.554394+0000","last_became_peered":"2023-10-01T09:13:13.554394+0000","last_unstale":"2023-10-01T09:13:13.554780+0000","last_undegraded":"2023-10-01T09:13:13.554780+0000","last_fullsized":"2023-10-01T09:13:13.554780+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:09:02.869321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.529938+0000","last_change":"2023-10-01T09:12:40.947346+0000","last_active":"2023-10-01T09:13:12.529938+0000","last_peered":"2023-10-01T09:13:12.529938+0000","last_clean":"2023-10-01T09:13:12.529938+0000","last_became_active":"2023-10-01T09:12:40.946939+0000","last_became_peered":"2023-10-01T09:12:40.946939+0000","last_unstale":"2023-10-01T09:13:12.529938+0000","last_undegraded":"2023-10-01T09:13:12.529938+0000","last_fullsized":"2023-10-01T09:13:12.529938+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27349,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12188,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430361,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306940,"kb_used_data":5748,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438212,"statfs":{"total":95995035648,"available":95680729088,"internally_reserved":0,"allocated":5885952,"data_stored":5375322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:13:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56899999999999995,"15min":0.56999999999999995},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.55100000000000005,"15min":0.54900000000000004},"min":{"1min":0.43099999999999999,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.38500000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:13:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.60199999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.54700000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:12:56 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.57399999999999995,"15min":0.60599999999999998},"min":{"1min":0.441,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":5.6210000000000004},"last":0.503},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.58399999999999996,"15min":0.55400000000000005},"min":{"1min":0.41599999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81299999999999994,"5min":0.89200000000000002,"15min":2.2050000000000001},"last":0.48799999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.47699999999999998,"15min":0.47599999999999998},"min":{"1min":0.34399999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.71799999999999997,"5min":0.77700000000000002,"15min":0.79100000000000004},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.51700000000000002,"15min":0.56599999999999995},"min":{"1min":0.29199999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.78700000000000003,"5min":0.82799999999999996,"15min":5.5369999999999999},"last":0.63300000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.498,"15min":0.50900000000000001},"min":{"1min":0.33000000000000002,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":0.69099999999999995,"5min":0.76500000000000001,"15min":0.84899999999999998},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.54700000000000004,"15min":0.54300000000000004},"min":{"1min":0.41999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.52700000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.39900000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.84699999999999998,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.624},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.58099999999999996,"15min":0.628},"min":{"1min":0.34399999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.82399999999999995,"5min":0.88500000000000001,"15min":5.5700000000000003},"last":0.66600000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626590,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302828,"kb_used_data":1636,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442324,"statfs":{"total":95995035648,"available":95684939776,"internally_reserved":0,"allocated":1675264,"data_stored":1180261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:13:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.58299999999999996,"15min":0.61199999999999999},"min":{"1min":0.35899999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.71399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.57899999999999996,"15min":0.61299999999999999},"min":{"1min":0.38900000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.748,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.53700000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:13:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.60799999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.61899999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:12:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.57599999999999996,"15min":0.55900000000000005},"min":{"1min":0.38500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.86499999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.57199999999999995,"15min":0.56999999999999995},"min":{"1min":0.40699999999999997,"5min":0.36399999999999999,"15min":0.28399999999999997},"max":{"1min":0.86299999999999999,"5min":1.0449999999999999,"15min":1.4490000000000001},"last":0.56999999999999995}]},{"osd":4,"last update":"Sun Oct 1 09:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.48099999999999998,"15min":0.47199999999999998},"min":{"1min":0.32600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.70799999999999996,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51800000000000002,"15min":0.51000000000000001},"min":{"1min":0.372,"5min":0.30099999999999999,"15min":0.29399999999999998},"max":{"1min":0.74199999999999999,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.73899999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51600000000000001,"15min":0.51100000000000001},"min":{"1min":0.36599999999999999,"5min":0.315,"15min":0.251},"max":{"1min":0.69299999999999995,"5min":0.79200000000000004,"15min":0.80200000000000005},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56000000000000005,"15min":0.54800000000000004},"min":{"1min":0.35899999999999999,"5min":0.30499999999999999,"15min":0.28299999999999997},"max":{"1min":0.76700000000000002,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.77500000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56100000000000005,"15min":0.54600000000000004},"min":{"1min":0.371,"5min":0.32600000000000001,"15min":0.29899999999999999},"max":{"1min":0.76300000000000001,"5min":0.79600000000000004,"15min":0.80500000000000005},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.60799999999999998,"15min":0.58599999999999997},"min":{"1min":0.38900000000000001,"5min":0.34999999999999998,"15min":0.32400000000000001},"max":{"1min":0.78900000000000003,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.80500000000000005}]}]},{"osd":1,"up_from":94,"seq":403726925849,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36744,"kb_used_data":5184,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93708408,"statfs":{"total":95995035648,"available":95957409792,"internally_reserved":0,"allocated":5308416,"data_stored":4788066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":32301479},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.313}]},{"osd":2,"last update":"Sun Oct 1 09:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.26900000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":4,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.68000000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.53700000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.65000000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.624},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.78200000000000003}]}]},{"osd":0,"up_from":89,"seq":382252089383,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37352,"kb_used_data":5792,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93707800,"statfs":{"total":95995035648,"available":95956787200,"internally_reserved":0,"allocated":5931008,"data_stored":5383726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":32300994},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:13:12 2023","interfaces":[{"interface":"back","average":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.25900000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.33100000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.32100000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:13:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.64000000000000001,"15min":0.64100000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.90700000000000003,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.67000000000000004,"15min":0.67500000000000004},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.88,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.56699999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:13:03 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.67500000000000004,"15min":0.68500000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.875,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.67800000000000005,"15min":0.68400000000000005},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.86399999999999999,"5min":0.999,"15min":0.999},"last":0.73999999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:13:03 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.68600000000000005,"15min":0.69399999999999995},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.92400000000000004,"5min":1.0329999999999999,"15min":1.0329999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.67800000000000005,"15min":0.68300000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.88900000000000001,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.70199999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:13:03 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.69599999999999995,"15min":0.70299999999999996},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.91000000000000003,"5min":1.081,"15min":1.081},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.69099999999999995,"15min":0.69899999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.86199999999999999,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.67800000000000005}]}]},{"osd":2,"up_from":99,"seq":425201762322,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32584,"kb_used_data":1088,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93712568,"statfs":{"total":95995035648,"available":95961669632,"internally_reserved":0,"allocated":1114112,"data_stored":593360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":32232311},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.499},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.44700000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.372,"5min":0.372,"15min":0.372},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.48899999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50800000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.71499999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.878}]},{"osd":6,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.91100000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.92800000000000005},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.755}]}]},{"osd":3,"up_from":104,"seq":446676598796,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28340,"kb_used_data":1068,"kb_used_omap":3,"kb_used_meta":27260,"kb_avail":93716812,"statfs":{"total":95995035648,"available":95966015488,"internally_reserved":0,"allocated":1093632,"data_stored":590233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":27914983},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41099999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33900000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51500000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52400000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48699999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019046,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298164,"kb_used_data":1068,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446988,"statfs":{"total":95995035648,"available":95689715712,"internally_reserved":0,"allocated":1093632,"data_stored":590580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.55200000000000005,"15min":0.57499999999999996},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.73399999999999999,"5min":0.746,"15min":0.746},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.56799999999999995,"15min":0.57399999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.70399999999999996,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.39400000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.53100000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.56000000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:12:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56000000000000005,"15min":0.56699999999999995},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.71599999999999997,"5min":0.81699999999999995,"15min":5.6429999999999998},"last":0.499},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56499999999999995,"15min":0.53700000000000003},"min":{"1min":0.41199999999999998,"5min":0.36099999999999999,"15min":0.34000000000000002},"max":{"1min":0.68799999999999994,"5min":0.86799999999999999,"15min":1.1279999999999999},"last":0.45700000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.47399999999999998,"15min":0.443},"min":{"1min":0.35699999999999998,"5min":0.317,"15min":0.26300000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.375},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.496,"15min":0.45800000000000002},"min":{"1min":0.39100000000000001,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.74299999999999999},"last":0.42599999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:13:37 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.52400000000000002,"15min":0.497},"min":{"1min":0.40699999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.77800000000000002,"5min":0.79500000000000004,"15min":0.84799999999999998},"last":0.627},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.57799999999999996,"15min":0.55000000000000004},"min":{"1min":0.48399999999999999,"5min":0.40000000000000002,"15min":0.33700000000000002},"max":{"1min":0.75600000000000001,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.58499999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:13:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.55000000000000004,"15min":0.54200000000000004},"min":{"1min":0.34100000000000003,"5min":0.33000000000000002,"15min":0.317},"max":{"1min":0.74099999999999999,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.626,"5min":0.58299999999999996,"15min":0.56499999999999995},"min":{"1min":0.41999999999999998,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.873},"last":0.59799999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822819,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298148,"kb_used_data":1052,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447004,"statfs":{"total":95995035648,"available":95689732096,"internally_reserved":0,"allocated":1077248,"data_stored":586028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:13:00 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.61799999999999999,"15min":0.63200000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.79200000000000004,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.58999999999999997,"15min":0.60199999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.71899999999999997,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.52200000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.58599999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:13:41 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":2.5430000000000001,"5min":2.5430000000000001,"15min":2.5430000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.5190000000000001,"5min":2.5190000000000001,"15min":2.5190000000000001},"last":0.56100000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:12:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.58999999999999997,"15min":0.55100000000000005},"min":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.27000000000000002},"max":{"1min":0.78100000000000003,"5min":5.5700000000000003,"15min":5.5700000000000003},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.57099999999999995,"15min":0.54600000000000004},"min":{"1min":0.44600000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.61899999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:13:06 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.49199999999999999,"15min":0.45500000000000002},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.25800000000000001},"max":{"1min":0.65800000000000003,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.53000000000000003,"15min":0.497},"min":{"1min":0.36099999999999999,"5min":0.35799999999999998,"15min":0.28100000000000003},"max":{"1min":0.69999999999999996,"5min":0.85799999999999998,"15min":0.86399999999999999},"last":0.502}]},{"osd":6,"last update":"Sun Oct 1 09:13:29 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.54400000000000004,"15min":0.52300000000000002},"min":{"1min":0.38100000000000001,"5min":0.32300000000000001,"15min":0.28599999999999998},"max":{"1min":0.65800000000000003,"5min":0.81799999999999995,"15min":0.88},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.58599999999999997,"15min":0.53400000000000003},"min":{"1min":0.35199999999999998,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":0.71999999999999997,"5min":0.83399999999999996,"15min":0.85799999999999998},"last":0.63100000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:13:52 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60199999999999998,"15min":0.57099999999999995},"min":{"1min":0.40799999999999997,"5min":0.34300000000000003,"15min":0.32300000000000001},"max":{"1min":0.79400000000000004,"5min":0.873,"15min":0.89300000000000002},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.59499999999999997,"15min":0.56599999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.32000000000000001},"max":{"1min":0.75900000000000001,"5min":0.86599999999999999,"15min":0.88500000000000001},"last":0.64500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:14:05.529 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:14:05.883 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:06.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:06 smithi138 ceph-mon[185886]: from='client.44495 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:14:06.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:06 smithi138 ceph-mon[185886]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:06.451 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:06 smithi138 ceph-mon[192513]: from='client.44495 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:14:06.451 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:06 smithi138 ceph-mon[192513]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:06 smithi190 ceph-mon[160435]: from='client.44495 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:14:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:06 smithi190 ceph-mon[160435]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:07.522 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:14:07.523 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:14:08.290 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:07 smithi138 ceph-mon[185886]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:08.290 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:07 smithi138 ceph-mon[192513]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:08.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:07 smithi190 ceph-mon[160435]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:08.308 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":209,"stamp":"2023-10-01T09:14:07.263498+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6302,"num_read_kb":12700,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341100,"kb_used_data":22636,"kb_used_omap":30,"kb_used_meta":1318369,"kb_avail":748620116,"statfs":{"total":767960285184,"available":766586998784,"internally_reserved":0,"allocated":23179264,"data_stored":19087576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30777,"internal_metadata":1350010823},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002253"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.529858+0000","last_change":"2023-10-01T09:12:10.003833+0000","last_active":"2023-10-01T09:13:12.529858+0000","last_peered":"2023-10-01T09:13:12.529858+0000","last_clean":"2023-10-01T09:13:12.529858+0000","last_became_active":"2023-10-01T09:12:10.003482+0000","last_became_peered":"2023-10-01T09:12:10.003482+0000","last_unstale":"2023-10-01T09:13:12.529858+0000","last_undegraded":"2023-10-01T09:13:12.529858+0000","last_fullsized":"2023-10-01T09:13:12.529858+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530638+0000","last_change":"2023-10-01T09:12:40.944944+0000","last_active":"2023-10-01T09:13:12.530638+0000","last_peered":"2023-10-01T09:13:12.530638+0000","last_clean":"2023-10-01T09:13:12.530638+0000","last_became_active":"2023-10-01T09:12:40.944613+0000","last_became_peered":"2023-10-01T09:12:40.944613+0000","last_unstale":"2023-10-01T09:13:12.530638+0000","last_undegraded":"2023-10-01T09:13:12.530638+0000","last_fullsized":"2023-10-01T09:13:12.530638+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:48:20.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.549800+0000","last_change":"2023-10-01T09:13:13.549800+0000","last_active":"2023-10-01T09:13:13.549800+0000","last_peered":"2023-10-01T09:13:13.549800+0000","last_clean":"2023-10-01T09:13:13.549800+0000","last_became_active":"2023-10-01T09:13:13.549480+0000","last_became_peered":"2023-10-01T09:13:13.549480+0000","last_unstale":"2023-10-01T09:13:13.549800+0000","last_undegraded":"2023-10-01T09:13:13.549800+0000","last_fullsized":"2023-10-01T09:13:13.549800+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:08.673916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":79,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556295+0000","last_change":"2023-10-01T09:13:13.556295+0000","last_active":"2023-10-01T09:13:13.556295+0000","last_peered":"2023-10-01T09:13:13.556295+0000","last_clean":"2023-10-01T09:13:13.556295+0000","last_became_active":"2023-10-01T09:13:13.555953+0000","last_became_peered":"2023-10-01T09:13:13.555953+0000","last_unstale":"2023-10-01T09:13:13.556295+0000","last_undegraded":"2023-10-01T09:13:13.556295+0000","last_fullsized":"2023-10-01T09:13:13.556295+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:04:06.892156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554959+0000","last_change":"2023-10-01T09:13:13.554959+0000","last_active":"2023-10-01T09:13:13.554959+0000","last_peered":"2023-10-01T09:13:13.554959+0000","last_clean":"2023-10-01T09:13:13.554959+0000","last_became_active":"2023-10-01T09:13:13.554589+0000","last_became_peered":"2023-10-01T09:13:13.554589+0000","last_unstale":"2023-10-01T09:13:13.554959+0000","last_undegraded":"2023-10-01T09:13:13.554959+0000","last_fullsized":"2023-10-01T09:13:13.554959+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":58,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.541271+0000","last_change":"2023-10-01T09:03:03.202063+0000","last_active":"2023-10-01T09:13:12.541271+0000","last_peered":"2023-10-01T09:13:12.541271+0000","last_clean":"2023-10-01T09:13:12.541271+0000","last_became_active":"2023-10-01T09:03:03.201769+0000","last_became_peered":"2023-10-01T09:03:03.201769+0000","last_unstale":"2023-10-01T09:13:12.541271+0000","last_undegraded":"2023-10-01T09:13:12.541271+0000","last_fullsized":"2023-10-01T09:13:12.541271+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555744+0000","last_change":"2023-10-01T09:13:13.555744+0000","last_active":"2023-10-01T09:13:13.555744+0000","last_peered":"2023-10-01T09:13:13.555744+0000","last_clean":"2023-10-01T09:13:13.555744+0000","last_became_active":"2023-10-01T09:13:13.555163+0000","last_became_peered":"2023-10-01T09:13:13.555163+0000","last_unstale":"2023-10-01T09:13:13.555744+0000","last_undegraded":"2023-10-01T09:13:13.555744+0000","last_fullsized":"2023-10-01T09:13:13.555744+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.540456+0000","last_change":"2023-10-01T09:12:10.000449+0000","last_active":"2023-10-01T09:13:12.540456+0000","last_peered":"2023-10-01T09:13:12.540456+0000","last_clean":"2023-10-01T09:13:12.540456+0000","last_became_active":"2023-10-01T09:12:10.000000+0000","last_became_peered":"2023-10-01T09:12:10.000000+0000","last_unstale":"2023-10-01T09:13:12.540456+0000","last_undegraded":"2023-10-01T09:13:12.540456+0000","last_fullsized":"2023-10-01T09:13:12.540456+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":93,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.548647+0000","last_change":"2023-10-01T09:13:13.548647+0000","last_active":"2023-10-01T09:13:13.548647+0000","last_peered":"2023-10-01T09:13:13.548647+0000","last_clean":"2023-10-01T09:13:13.548647+0000","last_became_active":"2023-10-01T09:13:13.548382+0000","last_became_peered":"2023-10-01T09:13:13.548382+0000","last_unstale":"2023-10-01T09:13:13.548647+0000","last_undegraded":"2023-10-01T09:13:13.548647+0000","last_fullsized":"2023-10-01T09:13:13.548647+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:17:31.791258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.540696+0000","last_change":"2023-10-01T09:10:58.801161+0000","last_active":"2023-10-01T09:13:13.540696+0000","last_peered":"2023-10-01T09:13:13.540696+0000","last_clean":"2023-10-01T09:13:13.540696+0000","last_became_active":"2023-10-01T09:10:58.800794+0000","last_became_peered":"2023-10-01T09:10:58.800794+0000","last_unstale":"2023-10-01T09:13:13.540696+0000","last_undegraded":"2023-10-01T09:13:13.540696+0000","last_fullsized":"2023-10-01T09:13:13.540696+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:35.019445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.531915+0000","last_change":"2023-10-01T09:03:03.754839+0000","last_active":"2023-10-01T09:13:12.531915+0000","last_peered":"2023-10-01T09:13:12.531915+0000","last_clean":"2023-10-01T09:13:12.531915+0000","last_became_active":"2023-10-01T09:03:03.754506+0000","last_became_peered":"2023-10-01T09:03:03.754506+0000","last_unstale":"2023-10-01T09:13:12.531915+0000","last_undegraded":"2023-10-01T09:13:12.531915+0000","last_fullsized":"2023-10-01T09:13:12.531915+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.532609+0000","last_change":"2023-10-01T09:12:10.003174+0000","last_active":"2023-10-01T09:13:12.532609+0000","last_peered":"2023-10-01T09:13:12.532609+0000","last_clean":"2023-10-01T09:13:12.532609+0000","last_became_active":"2023-10-01T09:12:10.002808+0000","last_became_peered":"2023-10-01T09:12:10.002808+0000","last_unstale":"2023-10-01T09:13:12.532609+0000","last_undegraded":"2023-10-01T09:13:12.532609+0000","last_fullsized":"2023-10-01T09:13:12.532609+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:25.037091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":143,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535113+0000","last_change":"2023-10-01T09:12:40.949544+0000","last_active":"2023-10-01T09:13:12.535113+0000","last_peered":"2023-10-01T09:13:12.535113+0000","last_clean":"2023-10-01T09:13:12.535113+0000","last_became_active":"2023-10-01T09:12:40.949054+0000","last_became_peered":"2023-10-01T09:12:40.949054+0000","last_unstale":"2023-10-01T09:13:12.535113+0000","last_undegraded":"2023-10-01T09:13:12.535113+0000","last_fullsized":"2023-10-01T09:13:12.535113+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.552192+0000","last_change":"2023-10-01T09:13:13.552192+0000","last_active":"2023-10-01T09:13:13.552192+0000","last_peered":"2023-10-01T09:13:13.552192+0000","last_clean":"2023-10-01T09:13:13.552192+0000","last_became_active":"2023-10-01T09:13:13.551887+0000","last_became_peered":"2023-10-01T09:13:13.551887+0000","last_unstale":"2023-10-01T09:13:13.552192+0000","last_undegraded":"2023-10-01T09:13:13.552192+0000","last_fullsized":"2023-10-01T09:13:13.552192+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:23.633557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":491,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534872+0000","last_change":"2023-10-01T09:12:10.002334+0000","last_active":"2023-10-01T09:13:12.534872+0000","last_peered":"2023-10-01T09:13:12.534872+0000","last_clean":"2023-10-01T09:13:12.534872+0000","last_became_active":"2023-10-01T09:12:10.001791+0000","last_became_peered":"2023-10-01T09:12:10.001791+0000","last_unstale":"2023-10-01T09:13:12.534872+0000","last_undegraded":"2023-10-01T09:13:12.534872+0000","last_fullsized":"2023-10-01T09:13:12.534872+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:24:27.632780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":71,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.553580+0000","last_change":"2023-10-01T09:13:13.553580+0000","last_active":"2023-10-01T09:13:13.553580+0000","last_peered":"2023-10-01T09:13:13.553580+0000","last_clean":"2023-10-01T09:13:13.553580+0000","last_became_active":"2023-10-01T09:13:13.553011+0000","last_became_peered":"2023-10-01T09:13:13.553011+0000","last_unstale":"2023-10-01T09:13:13.553580+0000","last_undegraded":"2023-10-01T09:13:13.553580+0000","last_fullsized":"2023-10-01T09:13:13.553580+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:54:30.369308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530101+0000","last_change":"2023-10-01T09:10:58.797034+0000","last_active":"2023-10-01T09:13:12.530101+0000","last_peered":"2023-10-01T09:13:12.530101+0000","last_clean":"2023-10-01T09:13:12.530101+0000","last_became_active":"2023-10-01T09:10:58.796730+0000","last_became_peered":"2023-10-01T09:10:58.796730+0000","last_unstale":"2023-10-01T09:13:12.530101+0000","last_undegraded":"2023-10-01T09:13:12.530101+0000","last_fullsized":"2023-10-01T09:13:12.530101+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:32:15.982890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.536708+0000","last_change":"2023-10-01T09:12:40.948216+0000","last_active":"2023-10-01T09:13:12.536708+0000","last_peered":"2023-10-01T09:13:12.536708+0000","last_clean":"2023-10-01T09:13:12.536708+0000","last_became_active":"2023-10-01T09:12:40.947879+0000","last_became_peered":"2023-10-01T09:12:40.947879+0000","last_unstale":"2023-10-01T09:13:12.536708+0000","last_undegraded":"2023-10-01T09:13:12.536708+0000","last_fullsized":"2023-10-01T09:13:12.536708+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:46:40.612337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":465,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534902+0000","last_change":"2023-10-01T09:10:58.797964+0000","last_active":"2023-10-01T09:13:12.534902+0000","last_peered":"2023-10-01T09:13:12.534902+0000","last_clean":"2023-10-01T09:13:12.534902+0000","last_became_active":"2023-10-01T09:10:58.797447+0000","last_became_peered":"2023-10-01T09:10:58.797447+0000","last_unstale":"2023-10-01T09:13:12.534902+0000","last_undegraded":"2023-10-01T09:13:12.534902+0000","last_fullsized":"2023-10-01T09:13:12.534902+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.532709+0000","last_change":"2023-10-01T09:12:40.944991+0000","last_active":"2023-10-01T09:13:12.532709+0000","last_peered":"2023-10-01T09:13:12.532709+0000","last_clean":"2023-10-01T09:13:12.532709+0000","last_became_active":"2023-10-01T09:12:40.944688+0000","last_became_peered":"2023-10-01T09:12:40.944688+0000","last_unstale":"2023-10-01T09:13:12.532709+0000","last_undegraded":"2023-10-01T09:13:12.532709+0000","last_fullsized":"2023-10-01T09:13:12.532709+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:30.015801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.552391+0000","last_change":"2023-10-01T09:13:13.552391+0000","last_active":"2023-10-01T09:13:13.552391+0000","last_peered":"2023-10-01T09:13:13.552391+0000","last_clean":"2023-10-01T09:13:13.552391+0000","last_became_active":"2023-10-01T09:13:13.552082+0000","last_became_peered":"2023-10-01T09:13:13.552082+0000","last_unstale":"2023-10-01T09:13:13.552391+0000","last_undegraded":"2023-10-01T09:13:13.552391+0000","last_fullsized":"2023-10-01T09:13:13.552391+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:07:19.929732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":201,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535494+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:13:12.535494+0000","last_peered":"2023-10-01T09:13:12.535494+0000","last_clean":"2023-10-01T09:13:12.535494+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:13:12.535494+0000","last_undegraded":"2023-10-01T09:13:12.535494+0000","last_fullsized":"2023-10-01T09:13:12.535494+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":55,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534655+0000","last_change":"2023-10-01T09:03:03.227254+0000","last_active":"2023-10-01T09:13:12.534655+0000","last_peered":"2023-10-01T09:13:12.534655+0000","last_clean":"2023-10-01T09:13:12.534655+0000","last_became_active":"2023-10-01T09:03:03.226941+0000","last_became_peered":"2023-10-01T09:03:03.226941+0000","last_unstale":"2023-10-01T09:13:12.534655+0000","last_undegraded":"2023-10-01T09:13:12.534655+0000","last_fullsized":"2023-10-01T09:13:12.534655+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":63,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556049+0000","last_change":"2023-10-01T09:13:13.556049+0000","last_active":"2023-10-01T09:13:13.556049+0000","last_peered":"2023-10-01T09:13:13.556049+0000","last_clean":"2023-10-01T09:13:13.556049+0000","last_became_active":"2023-10-01T09:13:13.555739+0000","last_became_peered":"2023-10-01T09:13:13.555739+0000","last_unstale":"2023-10-01T09:13:13.556049+0000","last_undegraded":"2023-10-01T09:13:13.556049+0000","last_fullsized":"2023-10-01T09:13:13.556049+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:50:34.475690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":307,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534586+0000","last_change":"2023-10-01T09:10:58.798697+0000","last_active":"2023-10-01T09:13:12.534586+0000","last_peered":"2023-10-01T09:13:12.534586+0000","last_clean":"2023-10-01T09:13:12.534586+0000","last_became_active":"2023-10-01T09:10:58.798203+0000","last_became_peered":"2023-10-01T09:10:58.798203+0000","last_unstale":"2023-10-01T09:13:12.534586+0000","last_undegraded":"2023-10-01T09:13:12.534586+0000","last_fullsized":"2023-10-01T09:13:12.534586+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":921,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539380+0000","last_change":"2023-10-01T09:12:40.946131+0000","last_active":"2023-10-01T09:13:12.539380+0000","last_peered":"2023-10-01T09:13:12.539380+0000","last_clean":"2023-10-01T09:13:12.539380+0000","last_became_active":"2023-10-01T09:12:40.945524+0000","last_became_peered":"2023-10-01T09:12:40.945524+0000","last_unstale":"2023-10-01T09:13:12.539380+0000","last_undegraded":"2023-10-01T09:13:12.539380+0000","last_fullsized":"2023-10-01T09:13:12.539380+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535348+0000","last_change":"2023-10-01T09:12:09.994071+0000","last_active":"2023-10-01T09:13:12.535348+0000","last_peered":"2023-10-01T09:13:12.535348+0000","last_clean":"2023-10-01T09:13:12.535348+0000","last_became_active":"2023-10-01T09:12:09.993782+0000","last_became_peered":"2023-10-01T09:12:09.993782+0000","last_unstale":"2023-10-01T09:13:12.535348+0000","last_undegraded":"2023-10-01T09:13:12.535348+0000","last_fullsized":"2023-10-01T09:13:12.535348+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:12:49.880685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535495+0000","last_change":"2023-10-01T09:12:40.946267+0000","last_active":"2023-10-01T09:13:12.535495+0000","last_peered":"2023-10-01T09:13:12.535495+0000","last_clean":"2023-10-01T09:13:12.535495+0000","last_became_active":"2023-10-01T09:12:40.945970+0000","last_became_peered":"2023-10-01T09:12:40.945970+0000","last_unstale":"2023-10-01T09:13:12.535495+0000","last_undegraded":"2023-10-01T09:13:12.535495+0000","last_fullsized":"2023-10-01T09:13:12.535495+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:42.904509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":375,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551669+0000","last_change":"2023-10-01T09:13:13.551669+0000","last_active":"2023-10-01T09:13:13.551669+0000","last_peered":"2023-10-01T09:13:13.551669+0000","last_clean":"2023-10-01T09:13:13.551669+0000","last_became_active":"2023-10-01T09:13:13.550982+0000","last_became_peered":"2023-10-01T09:13:13.550982+0000","last_unstale":"2023-10-01T09:13:13.551669+0000","last_undegraded":"2023-10-01T09:13:13.551669+0000","last_fullsized":"2023-10-01T09:13:13.551669+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:21:09.879713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.549385+0000","last_change":"2023-10-01T09:13:13.549385+0000","last_active":"2023-10-01T09:13:13.549385+0000","last_peered":"2023-10-01T09:13:13.549385+0000","last_clean":"2023-10-01T09:13:13.549385+0000","last_became_active":"2023-10-01T09:13:13.549035+0000","last_became_peered":"2023-10-01T09:13:13.549035+0000","last_unstale":"2023-10-01T09:13:13.549385+0000","last_undegraded":"2023-10-01T09:13:13.549385+0000","last_fullsized":"2023-10-01T09:13:13.549385+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:52:41.408995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530244+0000","last_change":"2023-10-01T09:10:58.799722+0000","last_active":"2023-10-01T09:13:12.530244+0000","last_peered":"2023-10-01T09:13:12.530244+0000","last_clean":"2023-10-01T09:13:12.530244+0000","last_became_active":"2023-10-01T09:10:58.799313+0000","last_became_peered":"2023-10-01T09:10:58.799313+0000","last_unstale":"2023-10-01T09:13:12.530244+0000","last_undegraded":"2023-10-01T09:13:12.530244+0000","last_fullsized":"2023-10-01T09:13:12.530244+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":805,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539880+0000","last_change":"2023-10-01T09:12:09.997936+0000","last_active":"2023-10-01T09:13:12.539880+0000","last_peered":"2023-10-01T09:13:12.539880+0000","last_clean":"2023-10-01T09:13:12.539880+0000","last_became_active":"2023-10-01T09:12:09.997427+0000","last_became_peered":"2023-10-01T09:12:09.997427+0000","last_unstale":"2023-10-01T09:13:12.539880+0000","last_undegraded":"2023-10-01T09:13:12.539880+0000","last_fullsized":"2023-10-01T09:13:12.539880+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1219,"num_read_kb":1114,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.533191+0000","last_change":"2023-10-01T09:12:10.001493+0000","last_active":"2023-10-01T09:13:12.533191+0000","last_peered":"2023-10-01T09:13:12.533191+0000","last_clean":"2023-10-01T09:13:12.533191+0000","last_became_active":"2023-10-01T09:12:10.000899+0000","last_became_peered":"2023-10-01T09:12:10.000899+0000","last_unstale":"2023-10-01T09:13:12.533191+0000","last_undegraded":"2023-10-01T09:13:12.533191+0000","last_fullsized":"2023-10-01T09:13:12.533191+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:47:54.011059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551681+0000","last_change":"2023-10-01T09:13:13.551681+0000","last_active":"2023-10-01T09:13:13.551681+0000","last_peered":"2023-10-01T09:13:13.551681+0000","last_clean":"2023-10-01T09:13:13.551681+0000","last_became_active":"2023-10-01T09:13:13.551268+0000","last_became_peered":"2023-10-01T09:13:13.551268+0000","last_unstale":"2023-10-01T09:13:13.551681+0000","last_undegraded":"2023-10-01T09:13:13.551681+0000","last_fullsized":"2023-10-01T09:13:13.551681+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":917,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.536626+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:13:12.536626+0000","last_peered":"2023-10-01T09:13:12.536626+0000","last_clean":"2023-10-01T09:13:12.536626+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:13:12.536626+0000","last_undegraded":"2023-10-01T09:13:12.536626+0000","last_fullsized":"2023-10-01T09:13:12.536626+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551757+0000","last_change":"2023-10-01T09:13:13.551757+0000","last_active":"2023-10-01T09:13:13.551757+0000","last_peered":"2023-10-01T09:13:13.551757+0000","last_clean":"2023-10-01T09:13:13.551757+0000","last_became_active":"2023-10-01T09:13:13.551483+0000","last_became_peered":"2023-10-01T09:13:13.551483+0000","last_unstale":"2023-10-01T09:13:13.551757+0000","last_undegraded":"2023-10-01T09:13:13.551757+0000","last_fullsized":"2023-10-01T09:13:13.551757+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:33:46.184050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555563+0000","last_change":"2023-10-01T09:13:13.555563+0000","last_active":"2023-10-01T09:13:13.555563+0000","last_peered":"2023-10-01T09:13:13.555563+0000","last_clean":"2023-10-01T09:13:13.555563+0000","last_became_active":"2023-10-01T09:13:13.555070+0000","last_became_peered":"2023-10-01T09:13:13.555070+0000","last_unstale":"2023-10-01T09:13:13.555563+0000","last_undegraded":"2023-10-01T09:13:13.555563+0000","last_fullsized":"2023-10-01T09:13:13.555563+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:44.557168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.531259+0000","last_change":"2023-10-01T09:12:10.003847+0000","last_active":"2023-10-01T09:13:12.531259+0000","last_peered":"2023-10-01T09:13:12.531259+0000","last_clean":"2023-10-01T09:13:12.531259+0000","last_became_active":"2023-10-01T09:12:10.003525+0000","last_became_peered":"2023-10-01T09:12:10.003525+0000","last_unstale":"2023-10-01T09:13:12.531259+0000","last_undegraded":"2023-10-01T09:13:12.531259+0000","last_fullsized":"2023-10-01T09:13:12.531259+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.532071+0000","last_change":"2023-10-01T09:12:40.945099+0000","last_active":"2023-10-01T09:13:12.532071+0000","last_peered":"2023-10-01T09:13:12.532071+0000","last_clean":"2023-10-01T09:13:12.532071+0000","last_became_active":"2023-10-01T09:12:40.944712+0000","last_became_peered":"2023-10-01T09:12:40.944712+0000","last_unstale":"2023-10-01T09:13:12.532071+0000","last_undegraded":"2023-10-01T09:13:12.532071+0000","last_fullsized":"2023-10-01T09:13:12.532071+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.531487+0000","last_change":"2023-10-01T09:12:40.946791+0000","last_active":"2023-10-01T09:13:12.531487+0000","last_peered":"2023-10-01T09:13:12.531487+0000","last_clean":"2023-10-01T09:13:12.531487+0000","last_became_active":"2023-10-01T09:12:40.946418+0000","last_became_peered":"2023-10-01T09:12:40.946418+0000","last_unstale":"2023-10-01T09:13:12.531487+0000","last_undegraded":"2023-10-01T09:13:12.531487+0000","last_fullsized":"2023-10-01T09:13:12.531487+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.536203+0000","last_change":"2023-10-01T09:12:10.003934+0000","last_active":"2023-10-01T09:13:12.536203+0000","last_peered":"2023-10-01T09:13:12.536203+0000","last_clean":"2023-10-01T09:13:12.536203+0000","last_became_active":"2023-10-01T09:12:10.003597+0000","last_became_peered":"2023-10-01T09:12:10.003597+0000","last_unstale":"2023-10-01T09:13:12.536203+0000","last_undegraded":"2023-10-01T09:13:12.536203+0000","last_fullsized":"2023-10-01T09:13:12.536203+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556491+0000","last_change":"2023-10-01T09:13:13.556491+0000","last_active":"2023-10-01T09:13:13.556491+0000","last_peered":"2023-10-01T09:13:13.556491+0000","last_clean":"2023-10-01T09:13:13.556491+0000","last_became_active":"2023-10-01T09:13:13.556118+0000","last_became_peered":"2023-10-01T09:13:13.556118+0000","last_unstale":"2023-10-01T09:13:13.556491+0000","last_undegraded":"2023-10-01T09:13:13.556491+0000","last_fullsized":"2023-10-01T09:13:13.556491+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554321+0000","last_change":"2023-10-01T09:13:13.554321+0000","last_active":"2023-10-01T09:13:13.554321+0000","last_peered":"2023-10-01T09:13:13.554321+0000","last_clean":"2023-10-01T09:13:13.554321+0000","last_became_active":"2023-10-01T09:13:13.553986+0000","last_became_peered":"2023-10-01T09:13:13.553986+0000","last_unstale":"2023-10-01T09:13:13.554321+0000","last_undegraded":"2023-10-01T09:13:13.554321+0000","last_fullsized":"2023-10-01T09:13:13.554321+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:09:12.848269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530790+0000","last_change":"2023-10-01T09:12:10.001565+0000","last_active":"2023-10-01T09:13:12.530790+0000","last_peered":"2023-10-01T09:13:12.530790+0000","last_clean":"2023-10-01T09:13:12.530790+0000","last_became_active":"2023-10-01T09:12:10.001275+0000","last_became_peered":"2023-10-01T09:12:10.001275+0000","last_unstale":"2023-10-01T09:13:12.530790+0000","last_undegraded":"2023-10-01T09:13:12.530790+0000","last_fullsized":"2023-10-01T09:13:12.530790+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:39.067046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555458+0000","last_change":"2023-10-01T09:13:13.555458+0000","last_active":"2023-10-01T09:13:13.555458+0000","last_peered":"2023-10-01T09:13:13.555458+0000","last_clean":"2023-10-01T09:13:13.555458+0000","last_became_active":"2023-10-01T09:13:13.555058+0000","last_became_peered":"2023-10-01T09:13:13.555058+0000","last_unstale":"2023-10-01T09:13:13.555458+0000","last_undegraded":"2023-10-01T09:13:13.555458+0000","last_fullsized":"2023-10-01T09:13:13.555458+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535389+0000","last_change":"2023-10-01T09:12:09.999571+0000","last_active":"2023-10-01T09:13:12.535389+0000","last_peered":"2023-10-01T09:13:12.535389+0000","last_clean":"2023-10-01T09:13:12.535389+0000","last_became_active":"2023-10-01T09:12:09.999272+0000","last_became_peered":"2023-10-01T09:12:09.999272+0000","last_unstale":"2023-10-01T09:13:12.535389+0000","last_undegraded":"2023-10-01T09:13:12.535389+0000","last_fullsized":"2023-10-01T09:13:12.535389+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":57,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535047+0000","last_change":"2023-10-01T09:03:03.200972+0000","last_active":"2023-10-01T09:13:12.535047+0000","last_peered":"2023-10-01T09:13:12.535047+0000","last_clean":"2023-10-01T09:13:12.535047+0000","last_became_active":"2023-10-01T09:03:03.200695+0000","last_became_peered":"2023-10-01T09:03:03.200695+0000","last_unstale":"2023-10-01T09:13:12.535047+0000","last_undegraded":"2023-10-01T09:13:12.535047+0000","last_fullsized":"2023-10-01T09:13:12.535047+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556674+0000","last_change":"2023-10-01T09:13:13.556674+0000","last_active":"2023-10-01T09:13:13.556674+0000","last_peered":"2023-10-01T09:13:13.556674+0000","last_clean":"2023-10-01T09:13:13.556674+0000","last_became_active":"2023-10-01T09:13:13.556271+0000","last_became_peered":"2023-10-01T09:13:13.556271+0000","last_unstale":"2023-10-01T09:13:13.556674+0000","last_undegraded":"2023-10-01T09:13:13.556674+0000","last_fullsized":"2023-10-01T09:13:13.556674+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535994+0000","last_change":"2023-10-01T09:12:10.003552+0000","last_active":"2023-10-01T09:13:12.535994+0000","last_peered":"2023-10-01T09:13:12.535994+0000","last_clean":"2023-10-01T09:13:12.535994+0000","last_became_active":"2023-10-01T09:12:10.003131+0000","last_became_peered":"2023-10-01T09:12:10.003131+0000","last_unstale":"2023-10-01T09:13:12.535994+0000","last_undegraded":"2023-10-01T09:13:12.535994+0000","last_fullsized":"2023-10-01T09:13:12.535994+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539254+0000","last_change":"2023-10-01T09:03:04.245105+0000","last_active":"2023-10-01T09:13:12.539254+0000","last_peered":"2023-10-01T09:13:12.539254+0000","last_clean":"2023-10-01T09:13:12.539254+0000","last_became_active":"2023-10-01T09:03:04.244807+0000","last_became_peered":"2023-10-01T09:03:04.244807+0000","last_unstale":"2023-10-01T09:13:12.539254+0000","last_undegraded":"2023-10-01T09:13:12.539254+0000","last_fullsized":"2023-10-01T09:13:12.539254+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.537360+0000","last_change":"2023-10-01T09:10:58.797850+0000","last_active":"2023-10-01T09:13:12.537360+0000","last_peered":"2023-10-01T09:13:12.537360+0000","last_clean":"2023-10-01T09:13:12.537360+0000","last_became_active":"2023-10-01T09:10:58.797403+0000","last_became_peered":"2023-10-01T09:10:58.797403+0000","last_unstale":"2023-10-01T09:13:12.537360+0000","last_undegraded":"2023-10-01T09:13:12.537360+0000","last_fullsized":"2023-10-01T09:13:12.537360+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550588+0000","last_change":"2023-10-01T09:13:13.550588+0000","last_active":"2023-10-01T09:13:13.550588+0000","last_peered":"2023-10-01T09:13:13.550588+0000","last_clean":"2023-10-01T09:13:13.550588+0000","last_became_active":"2023-10-01T09:13:13.550271+0000","last_became_peered":"2023-10-01T09:13:13.550271+0000","last_unstale":"2023-10-01T09:13:13.550588+0000","last_undegraded":"2023-10-01T09:13:13.550588+0000","last_fullsized":"2023-10-01T09:13:13.550588+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:19:47.301376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535409+0000","last_change":"2023-10-01T09:10:58.798168+0000","last_active":"2023-10-01T09:13:12.535409+0000","last_peered":"2023-10-01T09:13:12.535409+0000","last_clean":"2023-10-01T09:13:12.535409+0000","last_became_active":"2023-10-01T09:10:58.797781+0000","last_became_peered":"2023-10-01T09:10:58.797781+0000","last_unstale":"2023-10-01T09:13:12.535409+0000","last_undegraded":"2023-10-01T09:13:12.535409+0000","last_fullsized":"2023-10-01T09:13:12.535409+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550434+0000","last_change":"2023-10-01T09:13:13.548531+0000","last_active":"2023-10-01T09:13:13.550434+0000","last_peered":"2023-10-01T09:13:13.550434+0000","last_clean":"2023-10-01T09:13:13.550434+0000","last_became_active":"2023-10-01T09:13:13.547881+0000","last_became_peered":"2023-10-01T09:13:13.547881+0000","last_unstale":"2023-10-01T09:13:13.550434+0000","last_undegraded":"2023-10-01T09:13:13.550434+0000","last_fullsized":"2023-10-01T09:13:13.550434+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:13:24.068359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534146+0000","last_change":"2023-10-01T09:03:03.194952+0000","last_active":"2023-10-01T09:13:12.534146+0000","last_peered":"2023-10-01T09:13:12.534146+0000","last_clean":"2023-10-01T09:13:12.534146+0000","last_became_active":"2023-10-01T09:03:03.194643+0000","last_became_peered":"2023-10-01T09:03:03.194643+0000","last_unstale":"2023-10-01T09:13:12.534146+0000","last_undegraded":"2023-10-01T09:13:12.534146+0000","last_fullsized":"2023-10-01T09:13:12.534146+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.558367+0000","last_change":"2023-10-01T09:13:13.558367+0000","last_active":"2023-10-01T09:13:13.558367+0000","last_peered":"2023-10-01T09:13:13.558367+0000","last_clean":"2023-10-01T09:13:13.558367+0000","last_became_active":"2023-10-01T09:13:13.557952+0000","last_became_peered":"2023-10-01T09:13:13.557952+0000","last_unstale":"2023-10-01T09:13:13.558367+0000","last_undegraded":"2023-10-01T09:13:13.558367+0000","last_fullsized":"2023-10-01T09:13:13.558367+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550521+0000","last_change":"2023-10-01T09:13:13.548094+0000","last_active":"2023-10-01T09:13:13.550521+0000","last_peered":"2023-10-01T09:13:13.550521+0000","last_clean":"2023-10-01T09:13:13.550521+0000","last_became_active":"2023-10-01T09:13:13.547782+0000","last_became_peered":"2023-10-01T09:13:13.547782+0000","last_unstale":"2023-10-01T09:13:13.550521+0000","last_undegraded":"2023-10-01T09:13:13.550521+0000","last_fullsized":"2023-10-01T09:13:13.550521+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:15.220827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550633+0000","last_change":"2023-10-01T09:13:13.550633+0000","last_active":"2023-10-01T09:13:13.550633+0000","last_peered":"2023-10-01T09:13:13.550633+0000","last_clean":"2023-10-01T09:13:13.550633+0000","last_became_active":"2023-10-01T09:13:13.550305+0000","last_became_peered":"2023-10-01T09:13:13.550305+0000","last_unstale":"2023-10-01T09:13:13.550633+0000","last_undegraded":"2023-10-01T09:13:13.550633+0000","last_fullsized":"2023-10-01T09:13:13.550633+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:51:46.559383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534816+0000","last_change":"2023-10-01T09:12:40.945277+0000","last_active":"2023-10-01T09:13:12.534816+0000","last_peered":"2023-10-01T09:13:12.534816+0000","last_clean":"2023-10-01T09:13:12.534816+0000","last_became_active":"2023-10-01T09:12:40.944843+0000","last_became_peered":"2023-10-01T09:12:40.944843+0000","last_unstale":"2023-10-01T09:13:12.534816+0000","last_undegraded":"2023-10-01T09:13:12.534816+0000","last_fullsized":"2023-10-01T09:13:12.534816+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530812+0000","last_change":"2023-10-01T09:12:40.946973+0000","last_active":"2023-10-01T09:13:12.530812+0000","last_peered":"2023-10-01T09:13:12.530812+0000","last_clean":"2023-10-01T09:13:12.530812+0000","last_became_active":"2023-10-01T09:12:40.946427+0000","last_became_peered":"2023-10-01T09:12:40.946427+0000","last_unstale":"2023-10-01T09:13:12.530812+0000","last_undegraded":"2023-10-01T09:13:12.530812+0000","last_fullsized":"2023-10-01T09:13:12.530812+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":77,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530204+0000","last_change":"2023-10-01T09:03:03.200261+0000","last_active":"2023-10-01T09:13:12.530204+0000","last_peered":"2023-10-01T09:13:12.530204+0000","last_clean":"2023-10-01T09:13:12.530204+0000","last_became_active":"2023-10-01T09:03:03.199944+0000","last_became_peered":"2023-10-01T09:03:03.199944+0000","last_unstale":"2023-10-01T09:13:12.530204+0000","last_undegraded":"2023-10-01T09:13:12.530204+0000","last_fullsized":"2023-10-01T09:13:12.530204+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.533264+0000","last_change":"2023-10-01T09:12:40.946520+0000","last_active":"2023-10-01T09:13:12.533264+0000","last_peered":"2023-10-01T09:13:12.533264+0000","last_clean":"2023-10-01T09:13:12.533264+0000","last_became_active":"2023-10-01T09:12:40.946227+0000","last_became_peered":"2023-10-01T09:12:40.946227+0000","last_unstale":"2023-10-01T09:13:12.533264+0000","last_undegraded":"2023-10-01T09:13:12.533264+0000","last_fullsized":"2023-10-01T09:13:12.533264+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:06:06.451923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556801+0000","last_change":"2023-10-01T09:13:13.556801+0000","last_active":"2023-10-01T09:13:13.556801+0000","last_peered":"2023-10-01T09:13:13.556801+0000","last_clean":"2023-10-01T09:13:13.556801+0000","last_became_active":"2023-10-01T09:13:13.556397+0000","last_became_peered":"2023-10-01T09:13:13.556397+0000","last_unstale":"2023-10-01T09:13:13.556801+0000","last_undegraded":"2023-10-01T09:13:13.556801+0000","last_fullsized":"2023-10-01T09:13:13.556801+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554942+0000","last_change":"2023-10-01T09:13:13.554942+0000","last_active":"2023-10-01T09:13:13.554942+0000","last_peered":"2023-10-01T09:13:13.554942+0000","last_clean":"2023-10-01T09:13:13.554942+0000","last_became_active":"2023-10-01T09:13:13.554664+0000","last_became_peered":"2023-10-01T09:13:13.554664+0000","last_unstale":"2023-10-01T09:13:13.554942+0000","last_undegraded":"2023-10-01T09:13:13.554942+0000","last_fullsized":"2023-10-01T09:13:13.554942+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:43:14.532709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539899+0000","last_change":"2023-10-01T09:12:10.003091+0000","last_active":"2023-10-01T09:13:12.539899+0000","last_peered":"2023-10-01T09:13:12.539899+0000","last_clean":"2023-10-01T09:13:12.539899+0000","last_became_active":"2023-10-01T09:12:10.002773+0000","last_became_peered":"2023-10-01T09:12:10.002773+0000","last_unstale":"2023-10-01T09:13:12.539899+0000","last_undegraded":"2023-10-01T09:13:12.539899+0000","last_fullsized":"2023-10-01T09:13:12.539899+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555368+0000","last_change":"2023-10-01T09:13:13.555368+0000","last_active":"2023-10-01T09:13:13.555368+0000","last_peered":"2023-10-01T09:13:13.555368+0000","last_clean":"2023-10-01T09:13:13.555368+0000","last_became_active":"2023-10-01T09:13:13.554806+0000","last_became_peered":"2023-10-01T09:13:13.554806+0000","last_unstale":"2023-10-01T09:13:13.555368+0000","last_undegraded":"2023-10-01T09:13:13.555368+0000","last_fullsized":"2023-10-01T09:13:13.555368+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:24:32.687833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550551+0000","last_change":"2023-10-01T09:13:13.550193+0000","last_active":"2023-10-01T09:13:13.550551+0000","last_peered":"2023-10-01T09:13:13.550551+0000","last_clean":"2023-10-01T09:13:13.550551+0000","last_became_active":"2023-10-01T09:13:13.549900+0000","last_became_peered":"2023-10-01T09:13:13.549900+0000","last_unstale":"2023-10-01T09:13:13.550551+0000","last_undegraded":"2023-10-01T09:13:13.550551+0000","last_fullsized":"2023-10-01T09:13:13.550551+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:22.643161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534962+0000","last_change":"2023-10-01T09:12:40.947591+0000","last_active":"2023-10-01T09:13:12.534962+0000","last_peered":"2023-10-01T09:13:12.534962+0000","last_clean":"2023-10-01T09:13:12.534962+0000","last_became_active":"2023-10-01T09:12:40.947242+0000","last_became_peered":"2023-10-01T09:12:40.947242+0000","last_unstale":"2023-10-01T09:13:12.534962+0000","last_undegraded":"2023-10-01T09:13:12.534962+0000","last_fullsized":"2023-10-01T09:13:12.534962+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555490+0000","last_change":"2023-10-01T09:13:13.555490+0000","last_active":"2023-10-01T09:13:13.555490+0000","last_peered":"2023-10-01T09:13:13.555490+0000","last_clean":"2023-10-01T09:13:13.555490+0000","last_became_active":"2023-10-01T09:13:13.554506+0000","last_became_peered":"2023-10-01T09:13:13.554506+0000","last_unstale":"2023-10-01T09:13:13.555490+0000","last_undegraded":"2023-10-01T09:13:13.555490+0000","last_fullsized":"2023-10-01T09:13:13.555490+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:18.834474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.548695+0000","last_change":"2023-10-01T09:13:13.548695+0000","last_active":"2023-10-01T09:13:13.548695+0000","last_peered":"2023-10-01T09:13:13.548695+0000","last_clean":"2023-10-01T09:13:13.548695+0000","last_became_active":"2023-10-01T09:13:13.548256+0000","last_became_peered":"2023-10-01T09:13:13.548256+0000","last_unstale":"2023-10-01T09:13:13.548695+0000","last_undegraded":"2023-10-01T09:13:13.548695+0000","last_fullsized":"2023-10-01T09:13:13.548695+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:35:28.655733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551596+0000","last_change":"2023-10-01T09:13:13.551596+0000","last_active":"2023-10-01T09:13:13.551596+0000","last_peered":"2023-10-01T09:13:13.551596+0000","last_clean":"2023-10-01T09:13:13.551596+0000","last_became_active":"2023-10-01T09:13:13.551300+0000","last_became_peered":"2023-10-01T09:13:13.551300+0000","last_unstale":"2023-10-01T09:13:13.551596+0000","last_undegraded":"2023-10-01T09:13:13.551596+0000","last_fullsized":"2023-10-01T09:13:13.551596+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:57:45.081780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554780+0000","last_change":"2023-10-01T09:13:13.554780+0000","last_active":"2023-10-01T09:13:13.554780+0000","last_peered":"2023-10-01T09:13:13.554780+0000","last_clean":"2023-10-01T09:13:13.554780+0000","last_became_active":"2023-10-01T09:13:13.554394+0000","last_became_peered":"2023-10-01T09:13:13.554394+0000","last_unstale":"2023-10-01T09:13:13.554780+0000","last_undegraded":"2023-10-01T09:13:13.554780+0000","last_fullsized":"2023-10-01T09:13:13.554780+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:09:02.869321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.529938+0000","last_change":"2023-10-01T09:12:40.947346+0000","last_active":"2023-10-01T09:13:12.529938+0000","last_peered":"2023-10-01T09:13:12.529938+0000","last_clean":"2023-10-01T09:13:12.529938+0000","last_became_active":"2023-10-01T09:12:40.946939+0000","last_became_peered":"2023-10-01T09:12:40.946939+0000","last_unstale":"2023-10-01T09:13:12.529938+0000","last_undegraded":"2023-10-01T09:13:12.529938+0000","last_fullsized":"2023-10-01T09:13:12.529938+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27349,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12188,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430362,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306940,"kb_used_data":5748,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438212,"statfs":{"total":95995035648,"available":95680729088,"internally_reserved":0,"allocated":5885952,"data_stored":5375322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:14:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.80300000000000005,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.54600000000000004,"15min":0.54700000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.81899999999999995,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.54600000000000004}]},{"osd":1,"last update":"Sun Oct 1 09:13:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.47499999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.57499999999999996}]},{"osd":3,"last update":"Sun Oct 1 09:12:56 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.57399999999999995,"15min":0.60599999999999998},"min":{"1min":0.441,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":5.6210000000000004},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.58399999999999996,"15min":0.55400000000000005},"min":{"1min":0.41599999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81299999999999994,"5min":0.89200000000000002,"15min":2.2050000000000001},"last":0.51000000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.47699999999999998,"15min":0.47599999999999998},"min":{"1min":0.34399999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.71799999999999997,"5min":0.77700000000000002,"15min":0.79100000000000004},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.51700000000000002,"15min":0.56599999999999995},"min":{"1min":0.29199999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.78700000000000003,"5min":0.82799999999999996,"15min":5.5369999999999999},"last":0.60199999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.498,"15min":0.50900000000000001},"min":{"1min":0.33000000000000002,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":0.69099999999999995,"5min":0.76500000000000001,"15min":0.84899999999999998},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.54700000000000004,"15min":0.54300000000000004},"min":{"1min":0.41999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.58699999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.39900000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.84699999999999998,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.58099999999999996,"15min":0.628},"min":{"1min":0.34399999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.82399999999999995,"5min":0.88500000000000001,"15min":5.5700000000000003},"last":0.625}]}]},{"osd":6,"up_from":42,"seq":180388626591,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302828,"kb_used_data":1636,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442324,"statfs":{"total":95995035648,"available":95684939776,"internally_reserved":0,"allocated":1675264,"data_stored":1180261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:13:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.58299999999999996,"15min":0.61199999999999999},"min":{"1min":0.35899999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.71399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.57899999999999996,"15min":0.61299999999999999},"min":{"1min":0.38900000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.748,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.61899999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:13:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.66500000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.64300000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:12:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.57599999999999996,"15min":0.55900000000000005},"min":{"1min":0.38500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.86499999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.57199999999999995,"15min":0.56999999999999995},"min":{"1min":0.40699999999999997,"5min":0.36399999999999999,"15min":0.28399999999999997},"max":{"1min":0.86299999999999999,"5min":1.0449999999999999,"15min":1.4490000000000001},"last":0.70199999999999996}]},{"osd":4,"last update":"Sun Oct 1 09:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.48099999999999998,"15min":0.47199999999999998},"min":{"1min":0.32600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.70799999999999996,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51800000000000002,"15min":0.51000000000000001},"min":{"1min":0.372,"5min":0.30099999999999999,"15min":0.29399999999999998},"max":{"1min":0.74199999999999999,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.46500000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51600000000000001,"15min":0.51100000000000001},"min":{"1min":0.36599999999999999,"5min":0.315,"15min":0.251},"max":{"1min":0.69299999999999995,"5min":0.79200000000000004,"15min":0.80200000000000005},"last":0.501},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56000000000000005,"15min":0.54800000000000004},"min":{"1min":0.35899999999999999,"5min":0.30499999999999999,"15min":0.28299999999999997},"max":{"1min":0.76700000000000002,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.53100000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56100000000000005,"15min":0.54600000000000004},"min":{"1min":0.371,"5min":0.32600000000000001,"15min":0.29899999999999999},"max":{"1min":0.76300000000000001,"5min":0.79600000000000004,"15min":0.80500000000000005},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.60799999999999998,"15min":0.58599999999999997},"min":{"1min":0.38900000000000001,"5min":0.34999999999999998,"15min":0.32400000000000001},"max":{"1min":0.78900000000000003,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.63100000000000001}]}]},{"osd":1,"up_from":94,"seq":403726925849,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36744,"kb_used_data":5184,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93708408,"statfs":{"total":95995035648,"available":95957409792,"internally_reserved":0,"allocated":5308416,"data_stored":4788066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":32301479},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.313}]},{"osd":2,"last update":"Sun Oct 1 09:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.26900000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":4,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.68000000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.53700000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.65000000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.624},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.78200000000000003}]}]},{"osd":0,"up_from":89,"seq":382252089384,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37352,"kb_used_data":5792,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93707800,"statfs":{"total":95995035648,"available":95956787200,"internally_reserved":0,"allocated":5931008,"data_stored":5383726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":32300994},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:13:12 2023","interfaces":[{"interface":"back","average":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.39100000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.35799999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.29799999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:14:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.63,"15min":0.63800000000000001},"min":{"1min":0.39900000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.76600000000000001,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.65000000000000002,"15min":0.66800000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.88300000000000001,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.59299999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:14:04 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.65500000000000003,"15min":0.67800000000000005},"min":{"1min":0.42899999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.82499999999999996,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.65700000000000003,"15min":0.67700000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.73999999999999999,"5min":0.999,"15min":0.999},"last":0.58399999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:14:04 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.66200000000000003,"15min":0.68600000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.78100000000000003,"5min":1.0329999999999999,"15min":1.0329999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.66200000000000003,"15min":0.67800000000000005},"min":{"1min":0.40999999999999998,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78900000000000003,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.752}]},{"osd":7,"last update":"Sun Oct 1 09:14:04 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.67500000000000004,"15min":0.69599999999999995},"min":{"1min":0.38900000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.77400000000000002,"5min":1.081,"15min":1.081},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.67100000000000004,"15min":0.69199999999999995},"min":{"1min":0.42299999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80600000000000005,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.74099999999999999}]}]},{"osd":2,"up_from":99,"seq":425201762323,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32584,"kb_used_data":1088,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93712568,"statfs":{"total":95995035648,"available":95961669632,"internally_reserved":0,"allocated":1114112,"data_stored":593360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":32232311},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.503},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.40999999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.372,"5min":0.372,"15min":0.372},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.44600000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42799999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.58999999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.63700000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.67300000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.58099999999999996}]}]},{"osd":3,"up_from":104,"seq":446676598797,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28340,"kb_used_data":1068,"kb_used_omap":3,"kb_used_meta":27260,"kb_avail":93716812,"statfs":{"total":95995035648,"available":95966015488,"internally_reserved":0,"allocated":1093632,"data_stored":590233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":27914983},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26100000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38900000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32000000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58099999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64400000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72499999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78200000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019047,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298164,"kb_used_data":1068,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446988,"statfs":{"total":95995035648,"available":95689715712,"internally_reserved":0,"allocated":1093632,"data_stored":590580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.55200000000000005,"15min":0.57499999999999996},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.73399999999999999,"5min":0.746,"15min":0.746},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.56799999999999995,"15min":0.57399999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.70399999999999996,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.64400000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.39300000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.51700000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:12:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56000000000000005,"15min":0.56699999999999995},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.71599999999999997,"5min":0.81699999999999995,"15min":5.6429999999999998},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56499999999999995,"15min":0.53700000000000003},"min":{"1min":0.41199999999999998,"5min":0.36099999999999999,"15min":0.34000000000000002},"max":{"1min":0.68799999999999994,"5min":0.86799999999999999,"15min":1.1279999999999999},"last":0.45200000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.47399999999999998,"15min":0.443},"min":{"1min":0.35699999999999998,"5min":0.317,"15min":0.26300000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.496,"15min":0.45800000000000002},"min":{"1min":0.39100000000000001,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.74299999999999999},"last":0.59699999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:13:37 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.52400000000000002,"15min":0.497},"min":{"1min":0.40699999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.77800000000000002,"5min":0.79500000000000004,"15min":0.84799999999999998},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.57799999999999996,"15min":0.55000000000000004},"min":{"1min":0.48399999999999999,"5min":0.40000000000000002,"15min":0.33700000000000002},"max":{"1min":0.75600000000000001,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.63500000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:13:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.55000000000000004,"15min":0.54200000000000004},"min":{"1min":0.34100000000000003,"5min":0.33000000000000002,"15min":0.317},"max":{"1min":0.74099999999999999,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.626,"5min":0.58299999999999996,"15min":0.56499999999999995},"min":{"1min":0.41999999999999998,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.873},"last":0.72899999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822820,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298148,"kb_used_data":1052,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447004,"statfs":{"total":95995035648,"available":95689732096,"internally_reserved":0,"allocated":1077248,"data_stored":586028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:14:01 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.61599999999999999,"15min":0.63100000000000001},"min":{"1min":0.39800000000000002,"5min":0.309,"15min":0.309},"max":{"1min":2.6179999999999999,"5min":2.6179999999999999,"15min":2.6179999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.85099999999999998,"5min":0.63900000000000001,"15min":0.61799999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.7789999999999999,"5min":7.7789999999999999,"15min":7.7789999999999999},"last":0.52200000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.441},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.59199999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:13:41 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":2.5430000000000001,"5min":2.5430000000000001,"15min":2.5430000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.5190000000000001,"5min":2.5190000000000001,"15min":2.5190000000000001},"last":0.56200000000000006}]},{"osd":3,"last update":"Sun Oct 1 09:12:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.58999999999999997,"15min":0.55100000000000005},"min":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.27000000000000002},"max":{"1min":0.78100000000000003,"5min":5.5700000000000003,"15min":5.5700000000000003},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.57099999999999995,"15min":0.54600000000000004},"min":{"1min":0.44600000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.621}]},{"osd":4,"last update":"Sun Oct 1 09:13:06 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.49199999999999999,"15min":0.45500000000000002},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.25800000000000001},"max":{"1min":0.65800000000000003,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.53000000000000003,"15min":0.497},"min":{"1min":0.36099999999999999,"5min":0.35799999999999998,"15min":0.28100000000000003},"max":{"1min":0.69999999999999996,"5min":0.85799999999999998,"15min":0.86399999999999999},"last":0.57199999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:13:29 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.54400000000000004,"15min":0.52300000000000002},"min":{"1min":0.38100000000000001,"5min":0.32300000000000001,"15min":0.28599999999999998},"max":{"1min":0.65800000000000003,"5min":0.81799999999999995,"15min":0.88},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.58599999999999997,"15min":0.53400000000000003},"min":{"1min":0.35199999999999998,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":0.71999999999999997,"5min":0.83399999999999996,"15min":0.85799999999999998},"last":0.57699999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:13:52 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60199999999999998,"15min":0.57099999999999995},"min":{"1min":0.40799999999999997,"5min":0.34300000000000003,"15min":0.32300000000000001},"max":{"1min":0.79400000000000004,"5min":0.873,"15min":0.89300000000000002},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.59499999999999997,"15min":0.56599999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.32000000000000001},"max":{"1min":0.75900000000000001,"5min":0.86599999999999999,"15min":0.88500000000000001},"last":0.67300000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:14:08.315 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-01T09:14:08.315 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-01T09:14:08.315 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-01T09:14:08.315 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-01T09:14:08.664 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:09.233 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:08 smithi138 ceph-mon[185886]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:14:09.233 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:08 smithi138 ceph-mon[192513]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:14:09.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:08 smithi190 ceph-mon[160435]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:14:10.066 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:09 smithi138 ceph-mon[185886]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:10.067 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:09 smithi138 ceph-mon[192513]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:10.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:09 smithi190 ceph-mon[160435]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:10.465 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:14:10.465 INFO:teuthology.orchestra.run.smithi138.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-01T09:14:11.198 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:14:11.198 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:10 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/983628645' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T09:14:11.198 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:14:11.198 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:10 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/983628645' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T09:14:11.217 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-01T09:14:11.217 INFO:teuthology.run_tasks:Running task print... 2023-10-01T09:14:11.227 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-10-01T09:14:11.227 INFO:teuthology.run_tasks:Running task parallel... 2023-10-01T09:14:11.249 INFO:teuthology.task.parallel:starting parallel... 2023-10-01T09:14:11.249 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-01T09:14:11.249 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-10-01T09:14:11.250 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-01T09:14:11.251 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b56822191ad0416d0c05ed59720df03ad9c2a44e'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'b56822191ad0416d0c05ed59720df03ad9c2a44e'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'fc1e90a395a6a5631142eb70fd9bf3d22740a9da'}} 2023-10-01T09:14:11.251 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-01T09:14:11.251 INFO:tasks.thrashosds:Beginning thrashosds... 2023-10-01T09:14:11.251 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-01T09:14:11.251 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:14:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:14:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:10 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/983628645' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T09:14:11.598 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:12.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:11 smithi138 ceph-mon[192513]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:12.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:11 smithi138 ceph-mon[185886]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:12.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:11 smithi190 ceph-mon[160435]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:13.424 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:14:13.426 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:14:14.139 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":212,"stamp":"2023-10-01T09:14:13.264778+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6302,"num_read_kb":12700,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341100,"kb_used_data":22636,"kb_used_omap":30,"kb_used_meta":1318369,"kb_avail":748620116,"statfs":{"total":767960285184,"available":766586998784,"internally_reserved":0,"allocated":23179264,"data_stored":19087576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30777,"internal_metadata":1350010823},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002428"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.529858+0000","last_change":"2023-10-01T09:12:10.003833+0000","last_active":"2023-10-01T09:13:12.529858+0000","last_peered":"2023-10-01T09:13:12.529858+0000","last_clean":"2023-10-01T09:13:12.529858+0000","last_became_active":"2023-10-01T09:12:10.003482+0000","last_became_peered":"2023-10-01T09:12:10.003482+0000","last_unstale":"2023-10-01T09:13:12.529858+0000","last_undegraded":"2023-10-01T09:13:12.529858+0000","last_fullsized":"2023-10-01T09:13:12.529858+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530638+0000","last_change":"2023-10-01T09:12:40.944944+0000","last_active":"2023-10-01T09:13:12.530638+0000","last_peered":"2023-10-01T09:13:12.530638+0000","last_clean":"2023-10-01T09:13:12.530638+0000","last_became_active":"2023-10-01T09:12:40.944613+0000","last_became_peered":"2023-10-01T09:12:40.944613+0000","last_unstale":"2023-10-01T09:13:12.530638+0000","last_undegraded":"2023-10-01T09:13:12.530638+0000","last_fullsized":"2023-10-01T09:13:12.530638+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:48:20.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.549800+0000","last_change":"2023-10-01T09:13:13.549800+0000","last_active":"2023-10-01T09:13:13.549800+0000","last_peered":"2023-10-01T09:13:13.549800+0000","last_clean":"2023-10-01T09:13:13.549800+0000","last_became_active":"2023-10-01T09:13:13.549480+0000","last_became_peered":"2023-10-01T09:13:13.549480+0000","last_unstale":"2023-10-01T09:13:13.549800+0000","last_undegraded":"2023-10-01T09:13:13.549800+0000","last_fullsized":"2023-10-01T09:13:13.549800+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:08.673916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":79,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556295+0000","last_change":"2023-10-01T09:13:13.556295+0000","last_active":"2023-10-01T09:13:13.556295+0000","last_peered":"2023-10-01T09:13:13.556295+0000","last_clean":"2023-10-01T09:13:13.556295+0000","last_became_active":"2023-10-01T09:13:13.555953+0000","last_became_peered":"2023-10-01T09:13:13.555953+0000","last_unstale":"2023-10-01T09:13:13.556295+0000","last_undegraded":"2023-10-01T09:13:13.556295+0000","last_fullsized":"2023-10-01T09:13:13.556295+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:04:06.892156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554959+0000","last_change":"2023-10-01T09:13:13.554959+0000","last_active":"2023-10-01T09:13:13.554959+0000","last_peered":"2023-10-01T09:13:13.554959+0000","last_clean":"2023-10-01T09:13:13.554959+0000","last_became_active":"2023-10-01T09:13:13.554589+0000","last_became_peered":"2023-10-01T09:13:13.554589+0000","last_unstale":"2023-10-01T09:13:13.554959+0000","last_undegraded":"2023-10-01T09:13:13.554959+0000","last_fullsized":"2023-10-01T09:13:13.554959+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":58,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.541271+0000","last_change":"2023-10-01T09:03:03.202063+0000","last_active":"2023-10-01T09:13:12.541271+0000","last_peered":"2023-10-01T09:13:12.541271+0000","last_clean":"2023-10-01T09:13:12.541271+0000","last_became_active":"2023-10-01T09:03:03.201769+0000","last_became_peered":"2023-10-01T09:03:03.201769+0000","last_unstale":"2023-10-01T09:13:12.541271+0000","last_undegraded":"2023-10-01T09:13:12.541271+0000","last_fullsized":"2023-10-01T09:13:12.541271+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555744+0000","last_change":"2023-10-01T09:13:13.555744+0000","last_active":"2023-10-01T09:13:13.555744+0000","last_peered":"2023-10-01T09:13:13.555744+0000","last_clean":"2023-10-01T09:13:13.555744+0000","last_became_active":"2023-10-01T09:13:13.555163+0000","last_became_peered":"2023-10-01T09:13:13.555163+0000","last_unstale":"2023-10-01T09:13:13.555744+0000","last_undegraded":"2023-10-01T09:13:13.555744+0000","last_fullsized":"2023-10-01T09:13:13.555744+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.540456+0000","last_change":"2023-10-01T09:12:10.000449+0000","last_active":"2023-10-01T09:13:12.540456+0000","last_peered":"2023-10-01T09:13:12.540456+0000","last_clean":"2023-10-01T09:13:12.540456+0000","last_became_active":"2023-10-01T09:12:10.000000+0000","last_became_peered":"2023-10-01T09:12:10.000000+0000","last_unstale":"2023-10-01T09:13:12.540456+0000","last_undegraded":"2023-10-01T09:13:12.540456+0000","last_fullsized":"2023-10-01T09:13:12.540456+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":93,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.548647+0000","last_change":"2023-10-01T09:13:13.548647+0000","last_active":"2023-10-01T09:13:13.548647+0000","last_peered":"2023-10-01T09:13:13.548647+0000","last_clean":"2023-10-01T09:13:13.548647+0000","last_became_active":"2023-10-01T09:13:13.548382+0000","last_became_peered":"2023-10-01T09:13:13.548382+0000","last_unstale":"2023-10-01T09:13:13.548647+0000","last_undegraded":"2023-10-01T09:13:13.548647+0000","last_fullsized":"2023-10-01T09:13:13.548647+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:17:31.791258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.540696+0000","last_change":"2023-10-01T09:10:58.801161+0000","last_active":"2023-10-01T09:13:13.540696+0000","last_peered":"2023-10-01T09:13:13.540696+0000","last_clean":"2023-10-01T09:13:13.540696+0000","last_became_active":"2023-10-01T09:10:58.800794+0000","last_became_peered":"2023-10-01T09:10:58.800794+0000","last_unstale":"2023-10-01T09:13:13.540696+0000","last_undegraded":"2023-10-01T09:13:13.540696+0000","last_fullsized":"2023-10-01T09:13:13.540696+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:35.019445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.531915+0000","last_change":"2023-10-01T09:03:03.754839+0000","last_active":"2023-10-01T09:13:12.531915+0000","last_peered":"2023-10-01T09:13:12.531915+0000","last_clean":"2023-10-01T09:13:12.531915+0000","last_became_active":"2023-10-01T09:03:03.754506+0000","last_became_peered":"2023-10-01T09:03:03.754506+0000","last_unstale":"2023-10-01T09:13:12.531915+0000","last_undegraded":"2023-10-01T09:13:12.531915+0000","last_fullsized":"2023-10-01T09:13:12.531915+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.532609+0000","last_change":"2023-10-01T09:12:10.003174+0000","last_active":"2023-10-01T09:13:12.532609+0000","last_peered":"2023-10-01T09:13:12.532609+0000","last_clean":"2023-10-01T09:13:12.532609+0000","last_became_active":"2023-10-01T09:12:10.002808+0000","last_became_peered":"2023-10-01T09:12:10.002808+0000","last_unstale":"2023-10-01T09:13:12.532609+0000","last_undegraded":"2023-10-01T09:13:12.532609+0000","last_fullsized":"2023-10-01T09:13:12.532609+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:25.037091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":143,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535113+0000","last_change":"2023-10-01T09:12:40.949544+0000","last_active":"2023-10-01T09:13:12.535113+0000","last_peered":"2023-10-01T09:13:12.535113+0000","last_clean":"2023-10-01T09:13:12.535113+0000","last_became_active":"2023-10-01T09:12:40.949054+0000","last_became_peered":"2023-10-01T09:12:40.949054+0000","last_unstale":"2023-10-01T09:13:12.535113+0000","last_undegraded":"2023-10-01T09:13:12.535113+0000","last_fullsized":"2023-10-01T09:13:12.535113+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.552192+0000","last_change":"2023-10-01T09:13:13.552192+0000","last_active":"2023-10-01T09:13:13.552192+0000","last_peered":"2023-10-01T09:13:13.552192+0000","last_clean":"2023-10-01T09:13:13.552192+0000","last_became_active":"2023-10-01T09:13:13.551887+0000","last_became_peered":"2023-10-01T09:13:13.551887+0000","last_unstale":"2023-10-01T09:13:13.552192+0000","last_undegraded":"2023-10-01T09:13:13.552192+0000","last_fullsized":"2023-10-01T09:13:13.552192+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:23.633557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":491,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534872+0000","last_change":"2023-10-01T09:12:10.002334+0000","last_active":"2023-10-01T09:13:12.534872+0000","last_peered":"2023-10-01T09:13:12.534872+0000","last_clean":"2023-10-01T09:13:12.534872+0000","last_became_active":"2023-10-01T09:12:10.001791+0000","last_became_peered":"2023-10-01T09:12:10.001791+0000","last_unstale":"2023-10-01T09:13:12.534872+0000","last_undegraded":"2023-10-01T09:13:12.534872+0000","last_fullsized":"2023-10-01T09:13:12.534872+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:24:27.632780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":71,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.553580+0000","last_change":"2023-10-01T09:13:13.553580+0000","last_active":"2023-10-01T09:13:13.553580+0000","last_peered":"2023-10-01T09:13:13.553580+0000","last_clean":"2023-10-01T09:13:13.553580+0000","last_became_active":"2023-10-01T09:13:13.553011+0000","last_became_peered":"2023-10-01T09:13:13.553011+0000","last_unstale":"2023-10-01T09:13:13.553580+0000","last_undegraded":"2023-10-01T09:13:13.553580+0000","last_fullsized":"2023-10-01T09:13:13.553580+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:54:30.369308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530101+0000","last_change":"2023-10-01T09:10:58.797034+0000","last_active":"2023-10-01T09:13:12.530101+0000","last_peered":"2023-10-01T09:13:12.530101+0000","last_clean":"2023-10-01T09:13:12.530101+0000","last_became_active":"2023-10-01T09:10:58.796730+0000","last_became_peered":"2023-10-01T09:10:58.796730+0000","last_unstale":"2023-10-01T09:13:12.530101+0000","last_undegraded":"2023-10-01T09:13:12.530101+0000","last_fullsized":"2023-10-01T09:13:12.530101+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:32:15.982890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.543135+0000","last_change":"2023-10-01T09:12:40.948216+0000","last_active":"2023-10-01T09:13:13.543135+0000","last_peered":"2023-10-01T09:13:13.543135+0000","last_clean":"2023-10-01T09:13:13.543135+0000","last_became_active":"2023-10-01T09:12:40.947879+0000","last_became_peered":"2023-10-01T09:12:40.947879+0000","last_unstale":"2023-10-01T09:13:13.543135+0000","last_undegraded":"2023-10-01T09:13:13.543135+0000","last_fullsized":"2023-10-01T09:13:13.543135+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:46:40.612337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":465,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534902+0000","last_change":"2023-10-01T09:10:58.797964+0000","last_active":"2023-10-01T09:13:12.534902+0000","last_peered":"2023-10-01T09:13:12.534902+0000","last_clean":"2023-10-01T09:13:12.534902+0000","last_became_active":"2023-10-01T09:10:58.797447+0000","last_became_peered":"2023-10-01T09:10:58.797447+0000","last_unstale":"2023-10-01T09:13:12.534902+0000","last_undegraded":"2023-10-01T09:13:12.534902+0000","last_fullsized":"2023-10-01T09:13:12.534902+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.532709+0000","last_change":"2023-10-01T09:12:40.944991+0000","last_active":"2023-10-01T09:13:12.532709+0000","last_peered":"2023-10-01T09:13:12.532709+0000","last_clean":"2023-10-01T09:13:12.532709+0000","last_became_active":"2023-10-01T09:12:40.944688+0000","last_became_peered":"2023-10-01T09:12:40.944688+0000","last_unstale":"2023-10-01T09:13:12.532709+0000","last_undegraded":"2023-10-01T09:13:12.532709+0000","last_fullsized":"2023-10-01T09:13:12.532709+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:30.015801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.552391+0000","last_change":"2023-10-01T09:13:13.552391+0000","last_active":"2023-10-01T09:13:13.552391+0000","last_peered":"2023-10-01T09:13:13.552391+0000","last_clean":"2023-10-01T09:13:13.552391+0000","last_became_active":"2023-10-01T09:13:13.552082+0000","last_became_peered":"2023-10-01T09:13:13.552082+0000","last_unstale":"2023-10-01T09:13:13.552391+0000","last_undegraded":"2023-10-01T09:13:13.552391+0000","last_fullsized":"2023-10-01T09:13:13.552391+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:07:19.929732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":201,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535494+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:13:12.535494+0000","last_peered":"2023-10-01T09:13:12.535494+0000","last_clean":"2023-10-01T09:13:12.535494+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:13:12.535494+0000","last_undegraded":"2023-10-01T09:13:12.535494+0000","last_fullsized":"2023-10-01T09:13:12.535494+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":55,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534655+0000","last_change":"2023-10-01T09:03:03.227254+0000","last_active":"2023-10-01T09:13:12.534655+0000","last_peered":"2023-10-01T09:13:12.534655+0000","last_clean":"2023-10-01T09:13:12.534655+0000","last_became_active":"2023-10-01T09:03:03.226941+0000","last_became_peered":"2023-10-01T09:03:03.226941+0000","last_unstale":"2023-10-01T09:13:12.534655+0000","last_undegraded":"2023-10-01T09:13:12.534655+0000","last_fullsized":"2023-10-01T09:13:12.534655+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":63,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556049+0000","last_change":"2023-10-01T09:13:13.556049+0000","last_active":"2023-10-01T09:13:13.556049+0000","last_peered":"2023-10-01T09:13:13.556049+0000","last_clean":"2023-10-01T09:13:13.556049+0000","last_became_active":"2023-10-01T09:13:13.555739+0000","last_became_peered":"2023-10-01T09:13:13.555739+0000","last_unstale":"2023-10-01T09:13:13.556049+0000","last_undegraded":"2023-10-01T09:13:13.556049+0000","last_fullsized":"2023-10-01T09:13:13.556049+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:50:34.475690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":307,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534586+0000","last_change":"2023-10-01T09:10:58.798697+0000","last_active":"2023-10-01T09:13:12.534586+0000","last_peered":"2023-10-01T09:13:12.534586+0000","last_clean":"2023-10-01T09:13:12.534586+0000","last_became_active":"2023-10-01T09:10:58.798203+0000","last_became_peered":"2023-10-01T09:10:58.798203+0000","last_unstale":"2023-10-01T09:13:12.534586+0000","last_undegraded":"2023-10-01T09:13:12.534586+0000","last_fullsized":"2023-10-01T09:13:12.534586+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":921,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539380+0000","last_change":"2023-10-01T09:12:40.946131+0000","last_active":"2023-10-01T09:13:12.539380+0000","last_peered":"2023-10-01T09:13:12.539380+0000","last_clean":"2023-10-01T09:13:12.539380+0000","last_became_active":"2023-10-01T09:12:40.945524+0000","last_became_peered":"2023-10-01T09:12:40.945524+0000","last_unstale":"2023-10-01T09:13:12.539380+0000","last_undegraded":"2023-10-01T09:13:12.539380+0000","last_fullsized":"2023-10-01T09:13:12.539380+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535348+0000","last_change":"2023-10-01T09:12:09.994071+0000","last_active":"2023-10-01T09:13:12.535348+0000","last_peered":"2023-10-01T09:13:12.535348+0000","last_clean":"2023-10-01T09:13:12.535348+0000","last_became_active":"2023-10-01T09:12:09.993782+0000","last_became_peered":"2023-10-01T09:12:09.993782+0000","last_unstale":"2023-10-01T09:13:12.535348+0000","last_undegraded":"2023-10-01T09:13:12.535348+0000","last_fullsized":"2023-10-01T09:13:12.535348+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:12:49.880685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.543126+0000","last_change":"2023-10-01T09:12:40.946267+0000","last_active":"2023-10-01T09:13:13.543126+0000","last_peered":"2023-10-01T09:13:13.543126+0000","last_clean":"2023-10-01T09:13:13.543126+0000","last_became_active":"2023-10-01T09:12:40.945970+0000","last_became_peered":"2023-10-01T09:12:40.945970+0000","last_unstale":"2023-10-01T09:13:13.543126+0000","last_undegraded":"2023-10-01T09:13:13.543126+0000","last_fullsized":"2023-10-01T09:13:13.543126+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:42.904509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":375,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551669+0000","last_change":"2023-10-01T09:13:13.551669+0000","last_active":"2023-10-01T09:13:13.551669+0000","last_peered":"2023-10-01T09:13:13.551669+0000","last_clean":"2023-10-01T09:13:13.551669+0000","last_became_active":"2023-10-01T09:13:13.550982+0000","last_became_peered":"2023-10-01T09:13:13.550982+0000","last_unstale":"2023-10-01T09:13:13.551669+0000","last_undegraded":"2023-10-01T09:13:13.551669+0000","last_fullsized":"2023-10-01T09:13:13.551669+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:21:09.879713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554041+0000","last_change":"2023-10-01T09:13:13.549385+0000","last_active":"2023-10-01T09:13:13.554041+0000","last_peered":"2023-10-01T09:13:13.554041+0000","last_clean":"2023-10-01T09:13:13.554041+0000","last_became_active":"2023-10-01T09:13:13.549035+0000","last_became_peered":"2023-10-01T09:13:13.549035+0000","last_unstale":"2023-10-01T09:13:13.554041+0000","last_undegraded":"2023-10-01T09:13:13.554041+0000","last_fullsized":"2023-10-01T09:13:13.554041+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:52:41.408995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530244+0000","last_change":"2023-10-01T09:10:58.799722+0000","last_active":"2023-10-01T09:13:12.530244+0000","last_peered":"2023-10-01T09:13:12.530244+0000","last_clean":"2023-10-01T09:13:12.530244+0000","last_became_active":"2023-10-01T09:10:58.799313+0000","last_became_peered":"2023-10-01T09:10:58.799313+0000","last_unstale":"2023-10-01T09:13:12.530244+0000","last_undegraded":"2023-10-01T09:13:12.530244+0000","last_fullsized":"2023-10-01T09:13:12.530244+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":805,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539880+0000","last_change":"2023-10-01T09:12:09.997936+0000","last_active":"2023-10-01T09:13:12.539880+0000","last_peered":"2023-10-01T09:13:12.539880+0000","last_clean":"2023-10-01T09:13:12.539880+0000","last_became_active":"2023-10-01T09:12:09.997427+0000","last_became_peered":"2023-10-01T09:12:09.997427+0000","last_unstale":"2023-10-01T09:13:12.539880+0000","last_undegraded":"2023-10-01T09:13:12.539880+0000","last_fullsized":"2023-10-01T09:13:12.539880+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1219,"num_read_kb":1114,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.544355+0000","last_change":"2023-10-01T09:12:10.001493+0000","last_active":"2023-10-01T09:13:13.544355+0000","last_peered":"2023-10-01T09:13:13.544355+0000","last_clean":"2023-10-01T09:13:13.544355+0000","last_became_active":"2023-10-01T09:12:10.000899+0000","last_became_peered":"2023-10-01T09:12:10.000899+0000","last_unstale":"2023-10-01T09:13:13.544355+0000","last_undegraded":"2023-10-01T09:13:13.544355+0000","last_fullsized":"2023-10-01T09:13:13.544355+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:47:54.011059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551681+0000","last_change":"2023-10-01T09:13:13.551681+0000","last_active":"2023-10-01T09:13:13.551681+0000","last_peered":"2023-10-01T09:13:13.551681+0000","last_clean":"2023-10-01T09:13:13.551681+0000","last_became_active":"2023-10-01T09:13:13.551268+0000","last_became_peered":"2023-10-01T09:13:13.551268+0000","last_unstale":"2023-10-01T09:13:13.551681+0000","last_undegraded":"2023-10-01T09:13:13.551681+0000","last_fullsized":"2023-10-01T09:13:13.551681+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":918,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.543112+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:13:13.543112+0000","last_peered":"2023-10-01T09:13:13.543112+0000","last_clean":"2023-10-01T09:13:13.543112+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:13:13.543112+0000","last_undegraded":"2023-10-01T09:13:13.543112+0000","last_fullsized":"2023-10-01T09:13:13.543112+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551757+0000","last_change":"2023-10-01T09:13:13.551757+0000","last_active":"2023-10-01T09:13:13.551757+0000","last_peered":"2023-10-01T09:13:13.551757+0000","last_clean":"2023-10-01T09:13:13.551757+0000","last_became_active":"2023-10-01T09:13:13.551483+0000","last_became_peered":"2023-10-01T09:13:13.551483+0000","last_unstale":"2023-10-01T09:13:13.551757+0000","last_undegraded":"2023-10-01T09:13:13.551757+0000","last_fullsized":"2023-10-01T09:13:13.551757+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:33:46.184050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555563+0000","last_change":"2023-10-01T09:13:13.555563+0000","last_active":"2023-10-01T09:13:13.555563+0000","last_peered":"2023-10-01T09:13:13.555563+0000","last_clean":"2023-10-01T09:13:13.555563+0000","last_became_active":"2023-10-01T09:13:13.555070+0000","last_became_peered":"2023-10-01T09:13:13.555070+0000","last_unstale":"2023-10-01T09:13:13.555563+0000","last_undegraded":"2023-10-01T09:13:13.555563+0000","last_fullsized":"2023-10-01T09:13:13.555563+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:44.557168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.531259+0000","last_change":"2023-10-01T09:12:10.003847+0000","last_active":"2023-10-01T09:13:12.531259+0000","last_peered":"2023-10-01T09:13:12.531259+0000","last_clean":"2023-10-01T09:13:12.531259+0000","last_became_active":"2023-10-01T09:12:10.003525+0000","last_became_peered":"2023-10-01T09:12:10.003525+0000","last_unstale":"2023-10-01T09:13:12.531259+0000","last_undegraded":"2023-10-01T09:13:12.531259+0000","last_fullsized":"2023-10-01T09:13:12.531259+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.532071+0000","last_change":"2023-10-01T09:12:40.945099+0000","last_active":"2023-10-01T09:13:12.532071+0000","last_peered":"2023-10-01T09:13:12.532071+0000","last_clean":"2023-10-01T09:13:12.532071+0000","last_became_active":"2023-10-01T09:12:40.944712+0000","last_became_peered":"2023-10-01T09:12:40.944712+0000","last_unstale":"2023-10-01T09:13:12.532071+0000","last_undegraded":"2023-10-01T09:13:12.532071+0000","last_fullsized":"2023-10-01T09:13:12.532071+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.531487+0000","last_change":"2023-10-01T09:12:40.946791+0000","last_active":"2023-10-01T09:13:12.531487+0000","last_peered":"2023-10-01T09:13:12.531487+0000","last_clean":"2023-10-01T09:13:12.531487+0000","last_became_active":"2023-10-01T09:12:40.946418+0000","last_became_peered":"2023-10-01T09:12:40.946418+0000","last_unstale":"2023-10-01T09:13:12.531487+0000","last_undegraded":"2023-10-01T09:13:12.531487+0000","last_fullsized":"2023-10-01T09:13:12.531487+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.536203+0000","last_change":"2023-10-01T09:12:10.003934+0000","last_active":"2023-10-01T09:13:12.536203+0000","last_peered":"2023-10-01T09:13:12.536203+0000","last_clean":"2023-10-01T09:13:12.536203+0000","last_became_active":"2023-10-01T09:12:10.003597+0000","last_became_peered":"2023-10-01T09:12:10.003597+0000","last_unstale":"2023-10-01T09:13:12.536203+0000","last_undegraded":"2023-10-01T09:13:12.536203+0000","last_fullsized":"2023-10-01T09:13:12.536203+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556491+0000","last_change":"2023-10-01T09:13:13.556491+0000","last_active":"2023-10-01T09:13:13.556491+0000","last_peered":"2023-10-01T09:13:13.556491+0000","last_clean":"2023-10-01T09:13:13.556491+0000","last_became_active":"2023-10-01T09:13:13.556118+0000","last_became_peered":"2023-10-01T09:13:13.556118+0000","last_unstale":"2023-10-01T09:13:13.556491+0000","last_undegraded":"2023-10-01T09:13:13.556491+0000","last_fullsized":"2023-10-01T09:13:13.556491+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554321+0000","last_change":"2023-10-01T09:13:13.554321+0000","last_active":"2023-10-01T09:13:13.554321+0000","last_peered":"2023-10-01T09:13:13.554321+0000","last_clean":"2023-10-01T09:13:13.554321+0000","last_became_active":"2023-10-01T09:13:13.553986+0000","last_became_peered":"2023-10-01T09:13:13.553986+0000","last_unstale":"2023-10-01T09:13:13.554321+0000","last_undegraded":"2023-10-01T09:13:13.554321+0000","last_fullsized":"2023-10-01T09:13:13.554321+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:09:12.848269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530790+0000","last_change":"2023-10-01T09:12:10.001565+0000","last_active":"2023-10-01T09:13:12.530790+0000","last_peered":"2023-10-01T09:13:12.530790+0000","last_clean":"2023-10-01T09:13:12.530790+0000","last_became_active":"2023-10-01T09:12:10.001275+0000","last_became_peered":"2023-10-01T09:12:10.001275+0000","last_unstale":"2023-10-01T09:13:12.530790+0000","last_undegraded":"2023-10-01T09:13:12.530790+0000","last_fullsized":"2023-10-01T09:13:12.530790+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:39.067046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555458+0000","last_change":"2023-10-01T09:13:13.555458+0000","last_active":"2023-10-01T09:13:13.555458+0000","last_peered":"2023-10-01T09:13:13.555458+0000","last_clean":"2023-10-01T09:13:13.555458+0000","last_became_active":"2023-10-01T09:13:13.555058+0000","last_became_peered":"2023-10-01T09:13:13.555058+0000","last_unstale":"2023-10-01T09:13:13.555458+0000","last_undegraded":"2023-10-01T09:13:13.555458+0000","last_fullsized":"2023-10-01T09:13:13.555458+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535389+0000","last_change":"2023-10-01T09:12:09.999571+0000","last_active":"2023-10-01T09:13:12.535389+0000","last_peered":"2023-10-01T09:13:12.535389+0000","last_clean":"2023-10-01T09:13:12.535389+0000","last_became_active":"2023-10-01T09:12:09.999272+0000","last_became_peered":"2023-10-01T09:12:09.999272+0000","last_unstale":"2023-10-01T09:13:12.535389+0000","last_undegraded":"2023-10-01T09:13:12.535389+0000","last_fullsized":"2023-10-01T09:13:12.535389+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":57,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535047+0000","last_change":"2023-10-01T09:03:03.200972+0000","last_active":"2023-10-01T09:13:12.535047+0000","last_peered":"2023-10-01T09:13:12.535047+0000","last_clean":"2023-10-01T09:13:12.535047+0000","last_became_active":"2023-10-01T09:03:03.200695+0000","last_became_peered":"2023-10-01T09:03:03.200695+0000","last_unstale":"2023-10-01T09:13:12.535047+0000","last_undegraded":"2023-10-01T09:13:12.535047+0000","last_fullsized":"2023-10-01T09:13:12.535047+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556674+0000","last_change":"2023-10-01T09:13:13.556674+0000","last_active":"2023-10-01T09:13:13.556674+0000","last_peered":"2023-10-01T09:13:13.556674+0000","last_clean":"2023-10-01T09:13:13.556674+0000","last_became_active":"2023-10-01T09:13:13.556271+0000","last_became_peered":"2023-10-01T09:13:13.556271+0000","last_unstale":"2023-10-01T09:13:13.556674+0000","last_undegraded":"2023-10-01T09:13:13.556674+0000","last_fullsized":"2023-10-01T09:13:13.556674+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535994+0000","last_change":"2023-10-01T09:12:10.003552+0000","last_active":"2023-10-01T09:13:12.535994+0000","last_peered":"2023-10-01T09:13:12.535994+0000","last_clean":"2023-10-01T09:13:12.535994+0000","last_became_active":"2023-10-01T09:12:10.003131+0000","last_became_peered":"2023-10-01T09:12:10.003131+0000","last_unstale":"2023-10-01T09:13:12.535994+0000","last_undegraded":"2023-10-01T09:13:12.535994+0000","last_fullsized":"2023-10-01T09:13:12.535994+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539254+0000","last_change":"2023-10-01T09:03:04.245105+0000","last_active":"2023-10-01T09:13:12.539254+0000","last_peered":"2023-10-01T09:13:12.539254+0000","last_clean":"2023-10-01T09:13:12.539254+0000","last_became_active":"2023-10-01T09:03:04.244807+0000","last_became_peered":"2023-10-01T09:03:04.244807+0000","last_unstale":"2023-10-01T09:13:12.539254+0000","last_undegraded":"2023-10-01T09:13:12.539254+0000","last_fullsized":"2023-10-01T09:13:12.539254+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.537360+0000","last_change":"2023-10-01T09:10:58.797850+0000","last_active":"2023-10-01T09:13:12.537360+0000","last_peered":"2023-10-01T09:13:12.537360+0000","last_clean":"2023-10-01T09:13:12.537360+0000","last_became_active":"2023-10-01T09:10:58.797403+0000","last_became_peered":"2023-10-01T09:10:58.797403+0000","last_unstale":"2023-10-01T09:13:12.537360+0000","last_undegraded":"2023-10-01T09:13:12.537360+0000","last_fullsized":"2023-10-01T09:13:12.537360+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550588+0000","last_change":"2023-10-01T09:13:13.550588+0000","last_active":"2023-10-01T09:13:13.550588+0000","last_peered":"2023-10-01T09:13:13.550588+0000","last_clean":"2023-10-01T09:13:13.550588+0000","last_became_active":"2023-10-01T09:13:13.550271+0000","last_became_peered":"2023-10-01T09:13:13.550271+0000","last_unstale":"2023-10-01T09:13:13.550588+0000","last_undegraded":"2023-10-01T09:13:13.550588+0000","last_fullsized":"2023-10-01T09:13:13.550588+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:19:47.301376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535409+0000","last_change":"2023-10-01T09:10:58.798168+0000","last_active":"2023-10-01T09:13:12.535409+0000","last_peered":"2023-10-01T09:13:12.535409+0000","last_clean":"2023-10-01T09:13:12.535409+0000","last_became_active":"2023-10-01T09:10:58.797781+0000","last_became_peered":"2023-10-01T09:10:58.797781+0000","last_unstale":"2023-10-01T09:13:12.535409+0000","last_undegraded":"2023-10-01T09:13:12.535409+0000","last_fullsized":"2023-10-01T09:13:12.535409+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550434+0000","last_change":"2023-10-01T09:13:13.548531+0000","last_active":"2023-10-01T09:13:13.550434+0000","last_peered":"2023-10-01T09:13:13.550434+0000","last_clean":"2023-10-01T09:13:13.550434+0000","last_became_active":"2023-10-01T09:13:13.547881+0000","last_became_peered":"2023-10-01T09:13:13.547881+0000","last_unstale":"2023-10-01T09:13:13.550434+0000","last_undegraded":"2023-10-01T09:13:13.550434+0000","last_fullsized":"2023-10-01T09:13:13.550434+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:13:24.068359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534146+0000","last_change":"2023-10-01T09:03:03.194952+0000","last_active":"2023-10-01T09:13:12.534146+0000","last_peered":"2023-10-01T09:13:12.534146+0000","last_clean":"2023-10-01T09:13:12.534146+0000","last_became_active":"2023-10-01T09:03:03.194643+0000","last_became_peered":"2023-10-01T09:03:03.194643+0000","last_unstale":"2023-10-01T09:13:12.534146+0000","last_undegraded":"2023-10-01T09:13:12.534146+0000","last_fullsized":"2023-10-01T09:13:12.534146+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.558367+0000","last_change":"2023-10-01T09:13:13.558367+0000","last_active":"2023-10-01T09:13:13.558367+0000","last_peered":"2023-10-01T09:13:13.558367+0000","last_clean":"2023-10-01T09:13:13.558367+0000","last_became_active":"2023-10-01T09:13:13.557952+0000","last_became_peered":"2023-10-01T09:13:13.557952+0000","last_unstale":"2023-10-01T09:13:13.558367+0000","last_undegraded":"2023-10-01T09:13:13.558367+0000","last_fullsized":"2023-10-01T09:13:13.558367+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550521+0000","last_change":"2023-10-01T09:13:13.548094+0000","last_active":"2023-10-01T09:13:13.550521+0000","last_peered":"2023-10-01T09:13:13.550521+0000","last_clean":"2023-10-01T09:13:13.550521+0000","last_became_active":"2023-10-01T09:13:13.547782+0000","last_became_peered":"2023-10-01T09:13:13.547782+0000","last_unstale":"2023-10-01T09:13:13.550521+0000","last_undegraded":"2023-10-01T09:13:13.550521+0000","last_fullsized":"2023-10-01T09:13:13.550521+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:15.220827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550633+0000","last_change":"2023-10-01T09:13:13.550633+0000","last_active":"2023-10-01T09:13:13.550633+0000","last_peered":"2023-10-01T09:13:13.550633+0000","last_clean":"2023-10-01T09:13:13.550633+0000","last_became_active":"2023-10-01T09:13:13.550305+0000","last_became_peered":"2023-10-01T09:13:13.550305+0000","last_unstale":"2023-10-01T09:13:13.550633+0000","last_undegraded":"2023-10-01T09:13:13.550633+0000","last_fullsized":"2023-10-01T09:13:13.550633+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:51:46.559383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534816+0000","last_change":"2023-10-01T09:12:40.945277+0000","last_active":"2023-10-01T09:13:12.534816+0000","last_peered":"2023-10-01T09:13:12.534816+0000","last_clean":"2023-10-01T09:13:12.534816+0000","last_became_active":"2023-10-01T09:12:40.944843+0000","last_became_peered":"2023-10-01T09:12:40.944843+0000","last_unstale":"2023-10-01T09:13:12.534816+0000","last_undegraded":"2023-10-01T09:13:12.534816+0000","last_fullsized":"2023-10-01T09:13:12.534816+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530812+0000","last_change":"2023-10-01T09:12:40.946973+0000","last_active":"2023-10-01T09:13:12.530812+0000","last_peered":"2023-10-01T09:13:12.530812+0000","last_clean":"2023-10-01T09:13:12.530812+0000","last_became_active":"2023-10-01T09:12:40.946427+0000","last_became_peered":"2023-10-01T09:12:40.946427+0000","last_unstale":"2023-10-01T09:13:12.530812+0000","last_undegraded":"2023-10-01T09:13:12.530812+0000","last_fullsized":"2023-10-01T09:13:12.530812+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":77,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530204+0000","last_change":"2023-10-01T09:03:03.200261+0000","last_active":"2023-10-01T09:13:12.530204+0000","last_peered":"2023-10-01T09:13:12.530204+0000","last_clean":"2023-10-01T09:13:12.530204+0000","last_became_active":"2023-10-01T09:03:03.199944+0000","last_became_peered":"2023-10-01T09:03:03.199944+0000","last_unstale":"2023-10-01T09:13:12.530204+0000","last_undegraded":"2023-10-01T09:13:12.530204+0000","last_fullsized":"2023-10-01T09:13:12.530204+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.533264+0000","last_change":"2023-10-01T09:12:40.946520+0000","last_active":"2023-10-01T09:13:12.533264+0000","last_peered":"2023-10-01T09:13:12.533264+0000","last_clean":"2023-10-01T09:13:12.533264+0000","last_became_active":"2023-10-01T09:12:40.946227+0000","last_became_peered":"2023-10-01T09:12:40.946227+0000","last_unstale":"2023-10-01T09:13:12.533264+0000","last_undegraded":"2023-10-01T09:13:12.533264+0000","last_fullsized":"2023-10-01T09:13:12.533264+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:06:06.451923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556801+0000","last_change":"2023-10-01T09:13:13.556801+0000","last_active":"2023-10-01T09:13:13.556801+0000","last_peered":"2023-10-01T09:13:13.556801+0000","last_clean":"2023-10-01T09:13:13.556801+0000","last_became_active":"2023-10-01T09:13:13.556397+0000","last_became_peered":"2023-10-01T09:13:13.556397+0000","last_unstale":"2023-10-01T09:13:13.556801+0000","last_undegraded":"2023-10-01T09:13:13.556801+0000","last_fullsized":"2023-10-01T09:13:13.556801+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554942+0000","last_change":"2023-10-01T09:13:13.554942+0000","last_active":"2023-10-01T09:13:13.554942+0000","last_peered":"2023-10-01T09:13:13.554942+0000","last_clean":"2023-10-01T09:13:13.554942+0000","last_became_active":"2023-10-01T09:13:13.554664+0000","last_became_peered":"2023-10-01T09:13:13.554664+0000","last_unstale":"2023-10-01T09:13:13.554942+0000","last_undegraded":"2023-10-01T09:13:13.554942+0000","last_fullsized":"2023-10-01T09:13:13.554942+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:43:14.532709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539899+0000","last_change":"2023-10-01T09:12:10.003091+0000","last_active":"2023-10-01T09:13:12.539899+0000","last_peered":"2023-10-01T09:13:12.539899+0000","last_clean":"2023-10-01T09:13:12.539899+0000","last_became_active":"2023-10-01T09:12:10.002773+0000","last_became_peered":"2023-10-01T09:12:10.002773+0000","last_unstale":"2023-10-01T09:13:12.539899+0000","last_undegraded":"2023-10-01T09:13:12.539899+0000","last_fullsized":"2023-10-01T09:13:12.539899+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555368+0000","last_change":"2023-10-01T09:13:13.555368+0000","last_active":"2023-10-01T09:13:13.555368+0000","last_peered":"2023-10-01T09:13:13.555368+0000","last_clean":"2023-10-01T09:13:13.555368+0000","last_became_active":"2023-10-01T09:13:13.554806+0000","last_became_peered":"2023-10-01T09:13:13.554806+0000","last_unstale":"2023-10-01T09:13:13.555368+0000","last_undegraded":"2023-10-01T09:13:13.555368+0000","last_fullsized":"2023-10-01T09:13:13.555368+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:24:32.687833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550551+0000","last_change":"2023-10-01T09:13:13.550193+0000","last_active":"2023-10-01T09:13:13.550551+0000","last_peered":"2023-10-01T09:13:13.550551+0000","last_clean":"2023-10-01T09:13:13.550551+0000","last_became_active":"2023-10-01T09:13:13.549900+0000","last_became_peered":"2023-10-01T09:13:13.549900+0000","last_unstale":"2023-10-01T09:13:13.550551+0000","last_undegraded":"2023-10-01T09:13:13.550551+0000","last_fullsized":"2023-10-01T09:13:13.550551+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:22.643161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534962+0000","last_change":"2023-10-01T09:12:40.947591+0000","last_active":"2023-10-01T09:13:12.534962+0000","last_peered":"2023-10-01T09:13:12.534962+0000","last_clean":"2023-10-01T09:13:12.534962+0000","last_became_active":"2023-10-01T09:12:40.947242+0000","last_became_peered":"2023-10-01T09:12:40.947242+0000","last_unstale":"2023-10-01T09:13:12.534962+0000","last_undegraded":"2023-10-01T09:13:12.534962+0000","last_fullsized":"2023-10-01T09:13:12.534962+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555490+0000","last_change":"2023-10-01T09:13:13.555490+0000","last_active":"2023-10-01T09:13:13.555490+0000","last_peered":"2023-10-01T09:13:13.555490+0000","last_clean":"2023-10-01T09:13:13.555490+0000","last_became_active":"2023-10-01T09:13:13.554506+0000","last_became_peered":"2023-10-01T09:13:13.554506+0000","last_unstale":"2023-10-01T09:13:13.555490+0000","last_undegraded":"2023-10-01T09:13:13.555490+0000","last_fullsized":"2023-10-01T09:13:13.555490+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:18.834474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.548695+0000","last_change":"2023-10-01T09:13:13.548695+0000","last_active":"2023-10-01T09:13:13.548695+0000","last_peered":"2023-10-01T09:13:13.548695+0000","last_clean":"2023-10-01T09:13:13.548695+0000","last_became_active":"2023-10-01T09:13:13.548256+0000","last_became_peered":"2023-10-01T09:13:13.548256+0000","last_unstale":"2023-10-01T09:13:13.548695+0000","last_undegraded":"2023-10-01T09:13:13.548695+0000","last_fullsized":"2023-10-01T09:13:13.548695+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:35:28.655733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551596+0000","last_change":"2023-10-01T09:13:13.551596+0000","last_active":"2023-10-01T09:13:13.551596+0000","last_peered":"2023-10-01T09:13:13.551596+0000","last_clean":"2023-10-01T09:13:13.551596+0000","last_became_active":"2023-10-01T09:13:13.551300+0000","last_became_peered":"2023-10-01T09:13:13.551300+0000","last_unstale":"2023-10-01T09:13:13.551596+0000","last_undegraded":"2023-10-01T09:13:13.551596+0000","last_fullsized":"2023-10-01T09:13:13.551596+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:57:45.081780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554780+0000","last_change":"2023-10-01T09:13:13.554780+0000","last_active":"2023-10-01T09:13:13.554780+0000","last_peered":"2023-10-01T09:13:13.554780+0000","last_clean":"2023-10-01T09:13:13.554780+0000","last_became_active":"2023-10-01T09:13:13.554394+0000","last_became_peered":"2023-10-01T09:13:13.554394+0000","last_unstale":"2023-10-01T09:13:13.554780+0000","last_undegraded":"2023-10-01T09:13:13.554780+0000","last_fullsized":"2023-10-01T09:13:13.554780+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:09:02.869321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.529938+0000","last_change":"2023-10-01T09:12:40.947346+0000","last_active":"2023-10-01T09:13:12.529938+0000","last_peered":"2023-10-01T09:13:12.529938+0000","last_clean":"2023-10-01T09:13:12.529938+0000","last_became_active":"2023-10-01T09:12:40.946939+0000","last_became_peered":"2023-10-01T09:12:40.946939+0000","last_unstale":"2023-10-01T09:13:12.529938+0000","last_undegraded":"2023-10-01T09:13:12.529938+0000","last_fullsized":"2023-10-01T09:13:12.529938+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27349,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12188,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430363,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306940,"kb_used_data":5748,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438212,"statfs":{"total":95995035648,"available":95680729088,"internally_reserved":0,"allocated":5885952,"data_stored":5375322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:14:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.80300000000000005,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.503},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.54600000000000004,"15min":0.54700000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.81899999999999995,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.46000000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:13:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.434},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.52800000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.49199999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:12:56 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.57399999999999995,"15min":0.60599999999999998},"min":{"1min":0.441,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":5.6210000000000004},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.58399999999999996,"15min":0.55400000000000005},"min":{"1min":0.41599999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81299999999999994,"5min":0.89200000000000002,"15min":2.2050000000000001},"last":0.47299999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.47699999999999998,"15min":0.47599999999999998},"min":{"1min":0.34399999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.71799999999999997,"5min":0.77700000000000002,"15min":0.79100000000000004},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.51700000000000002,"15min":0.56599999999999995},"min":{"1min":0.29199999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.78700000000000003,"5min":0.82799999999999996,"15min":5.5369999999999999},"last":0.34799999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.498,"15min":0.50900000000000001},"min":{"1min":0.33000000000000002,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":0.69099999999999995,"5min":0.76500000000000001,"15min":0.84899999999999998},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.54700000000000004,"15min":0.54300000000000004},"min":{"1min":0.41999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.39000000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.39900000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.84699999999999998,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.58099999999999996,"15min":0.628},"min":{"1min":0.34399999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.82399999999999995,"5min":0.88500000000000001,"15min":5.5700000000000003},"last":0.56899999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626592,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302828,"kb_used_data":1636,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442324,"statfs":{"total":95995035648,"available":95684939776,"internally_reserved":0,"allocated":1675264,"data_stored":1180261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:13:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.58299999999999996,"15min":0.61199999999999999},"min":{"1min":0.35899999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.71399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.57899999999999996,"15min":0.61299999999999999},"min":{"1min":0.38900000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.748,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.59899999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:13:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.57799999999999996}]},{"osd":2,"last update":"Sun Oct 1 09:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.503}]},{"osd":3,"last update":"Sun Oct 1 09:12:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.57599999999999996,"15min":0.55900000000000005},"min":{"1min":0.38500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.86499999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.57199999999999995,"15min":0.56999999999999995},"min":{"1min":0.40699999999999997,"5min":0.36399999999999999,"15min":0.28399999999999997},"max":{"1min":0.86299999999999999,"5min":1.0449999999999999,"15min":1.4490000000000001},"last":0.53700000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.48099999999999998,"15min":0.47199999999999998},"min":{"1min":0.32600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.70799999999999996,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.379},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51800000000000002,"15min":0.51000000000000001},"min":{"1min":0.372,"5min":0.30099999999999999,"15min":0.29399999999999998},"max":{"1min":0.74199999999999999,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.61499999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51600000000000001,"15min":0.51100000000000001},"min":{"1min":0.36599999999999999,"5min":0.315,"15min":0.251},"max":{"1min":0.69299999999999995,"5min":0.79200000000000004,"15min":0.80200000000000005},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56000000000000005,"15min":0.54800000000000004},"min":{"1min":0.35899999999999999,"5min":0.30499999999999999,"15min":0.28299999999999997},"max":{"1min":0.76700000000000002,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.58499999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56100000000000005,"15min":0.54600000000000004},"min":{"1min":0.371,"5min":0.32600000000000001,"15min":0.29899999999999999},"max":{"1min":0.76300000000000001,"5min":0.79600000000000004,"15min":0.80500000000000005},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.60799999999999998,"15min":0.58599999999999997},"min":{"1min":0.38900000000000001,"5min":0.34999999999999998,"15min":0.32400000000000001},"max":{"1min":0.78900000000000003,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.63600000000000001}]}]},{"osd":1,"up_from":94,"seq":403726925851,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36744,"kb_used_data":5184,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93708408,"statfs":{"total":95995035648,"available":95957409792,"internally_reserved":0,"allocated":5308416,"data_stored":4788066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":32301479},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.311,"5min":0.33900000000000002,"15min":0.34300000000000003},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.51300000000000001,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.33900000000000002,"5min":0.36099999999999999,"15min":0.36499999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.50900000000000001,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.311}]},{"osd":2,"last update":"Sun Oct 1 09:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.377}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34799999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54400000000000004,"15min":0.54300000000000004},"min":{"1min":0.41299999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.504},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.56599999999999995,"15min":0.56200000000000006},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.85699999999999998,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.51700000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.39900000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.59099999999999997,"15min":0.59499999999999997},"min":{"1min":0.41199999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.82099999999999995,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.438}]},{"osd":6,"last update":"Sun Oct 1 09:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.59699999999999998,"15min":0.59899999999999998},"min":{"1min":0.42499999999999999,"5min":0.374,"15min":0.374},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.627,"15min":0.63100000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.85299999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.56899999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59699999999999998,"15min":0.59999999999999998},"min":{"1min":0.38700000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.88,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60699999999999998,"15min":0.60599999999999998},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.55000000000000004}]}]},{"osd":0,"up_from":89,"seq":382252089385,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37352,"kb_used_data":5792,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93707800,"statfs":{"total":95995035648,"available":95956787200,"internally_reserved":0,"allocated":5931008,"data_stored":5383726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":32300994},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:13:12 2023","interfaces":[{"interface":"back","average":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.28399999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.36199999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.33000000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:14:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.63,"15min":0.63800000000000001},"min":{"1min":0.39900000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.76600000000000001,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.65000000000000002,"15min":0.66800000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.88300000000000001,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.48399999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:14:04 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.65500000000000003,"15min":0.67800000000000005},"min":{"1min":0.42899999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.82499999999999996,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.65700000000000003,"15min":0.67700000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.73999999999999999,"5min":0.999,"15min":0.999},"last":0.497}]},{"osd":6,"last update":"Sun Oct 1 09:14:04 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.66200000000000003,"15min":0.68600000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.78100000000000003,"5min":1.0329999999999999,"15min":1.0329999999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.66200000000000003,"15min":0.67800000000000005},"min":{"1min":0.40999999999999998,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78900000000000003,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.629}]},{"osd":7,"last update":"Sun Oct 1 09:14:04 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.67500000000000004,"15min":0.69599999999999995},"min":{"1min":0.38900000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.77400000000000002,"5min":1.081,"15min":1.081},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.67100000000000004,"15min":0.69199999999999995},"min":{"1min":0.42299999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80600000000000005,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.60599999999999998}]}]},{"osd":2,"up_from":99,"seq":425201762324,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32584,"kb_used_data":1088,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93712568,"statfs":{"total":95995035648,"available":95961669632,"internally_reserved":0,"allocated":1114112,"data_stored":593360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":32232311},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.34200000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.372,"5min":0.372,"15min":0.372},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.372}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38}]},{"osd":4,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.60499999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.62},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.56399999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.59499999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.55500000000000005}]}]},{"osd":3,"up_from":104,"seq":446676598798,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28340,"kb_used_data":1068,"kb_used_omap":3,"kb_used_meta":27260,"kb_avail":93716812,"statfs":{"total":95995035648,"available":95966015488,"internally_reserved":0,"allocated":1093632,"data_stored":590233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":27914983},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26100000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38900000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32000000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58099999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64400000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72499999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78200000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019048,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298164,"kb_used_data":1068,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446988,"statfs":{"total":95995035648,"available":95689715712,"internally_reserved":0,"allocated":1093632,"data_stored":590580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.55200000000000005,"15min":0.57499999999999996},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.73399999999999999,"5min":0.746,"15min":0.746},"last":0.625},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.56799999999999995,"15min":0.57399999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.70399999999999996,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.52900000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.63900000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.66100000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:12:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56000000000000005,"15min":0.56699999999999995},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.71599999999999997,"5min":0.81699999999999995,"15min":5.6429999999999998},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56499999999999995,"15min":0.53700000000000003},"min":{"1min":0.41199999999999998,"5min":0.36099999999999999,"15min":0.34000000000000002},"max":{"1min":0.68799999999999994,"5min":0.86799999999999999,"15min":1.1279999999999999},"last":0.70299999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.47399999999999998,"15min":0.443},"min":{"1min":0.35699999999999998,"5min":0.317,"15min":0.26300000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.496,"15min":0.45800000000000002},"min":{"1min":0.39100000000000001,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.74299999999999999},"last":0.47799999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:13:37 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.52400000000000002,"15min":0.497},"min":{"1min":0.40699999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.77800000000000002,"5min":0.79500000000000004,"15min":0.84799999999999998},"last":0.435},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.57799999999999996,"15min":0.55000000000000004},"min":{"1min":0.48399999999999999,"5min":0.40000000000000002,"15min":0.33700000000000002},"max":{"1min":0.75600000000000001,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.57299999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:13:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.55000000000000004,"15min":0.54200000000000004},"min":{"1min":0.34100000000000003,"5min":0.33000000000000002,"15min":0.317},"max":{"1min":0.74099999999999999,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.626,"5min":0.58299999999999996,"15min":0.56499999999999995},"min":{"1min":0.41999999999999998,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.873},"last":0.71499999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822821,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298148,"kb_used_data":1052,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447004,"statfs":{"total":95995035648,"available":95689732096,"internally_reserved":0,"allocated":1077248,"data_stored":586028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:14:01 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.61599999999999999,"15min":0.63100000000000001},"min":{"1min":0.39800000000000002,"5min":0.309,"15min":0.309},"max":{"1min":2.6179999999999999,"5min":2.6179999999999999,"15min":2.6179999999999999},"last":0.503},{"interface":"front","average":{"1min":0.85099999999999998,"5min":0.63900000000000001,"15min":0.61799999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.7789999999999999,"5min":7.7789999999999999,"15min":7.7789999999999999},"last":0.34300000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:14:10 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.57799999999999996,"15min":0.54000000000000004},"min":{"1min":0.441,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.5209999999999999,"5min":3.5209999999999999,"15min":3.5209999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.56999999999999995,"15min":0.55400000000000005},"min":{"1min":0.38200000000000001,"5min":0.376,"15min":0.376},"max":{"1min":3.4260000000000002,"5min":3.4260000000000002,"15min":3.4260000000000002},"last":0.42599999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:13:41 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":2.5430000000000001,"5min":2.5430000000000001,"15min":2.5430000000000001},"last":0.378},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.5190000000000001,"5min":2.5190000000000001,"15min":2.5190000000000001},"last":0.47999999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:12:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.58999999999999997,"15min":0.55100000000000005},"min":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.27000000000000002},"max":{"1min":0.78100000000000003,"5min":5.5700000000000003,"15min":5.5700000000000003},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.57099999999999995,"15min":0.54600000000000004},"min":{"1min":0.44600000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.54900000000000004}]},{"osd":4,"last update":"Sun Oct 1 09:14:07 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.496,"15min":0.46300000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.25800000000000001},"max":{"1min":0.64500000000000002,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.52900000000000003,"15min":0.503},"min":{"1min":0.316,"5min":0.316,"15min":0.28100000000000003},"max":{"1min":0.61699999999999999,"5min":0.85799999999999998,"15min":0.86399999999999999},"last":0.41599999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:13:29 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.54400000000000004,"15min":0.52300000000000002},"min":{"1min":0.38100000000000001,"5min":0.32300000000000001,"15min":0.28599999999999998},"max":{"1min":0.65800000000000003,"5min":0.81799999999999995,"15min":0.88},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.58599999999999997,"15min":0.53400000000000003},"min":{"1min":0.35199999999999998,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":0.71999999999999997,"5min":0.83399999999999996,"15min":0.85799999999999998},"last":0.61899999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:13:52 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60199999999999998,"15min":0.57099999999999995},"min":{"1min":0.40799999999999997,"5min":0.34300000000000003,"15min":0.32300000000000001},"max":{"1min":0.79400000000000004,"5min":0.873,"15min":0.89300000000000002},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.59499999999999997,"15min":0.56599999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.32000000000000001},"max":{"1min":0.75900000000000001,"5min":0.86599999999999999,"15min":0.88500000000000001},"last":0.56899999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:14:14.147 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:14:14.497 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:14.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:14 smithi138 ceph-mon[192513]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:14.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:14 smithi138 ceph-mon[192513]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:14:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:14 smithi138 ceph-mon[185886]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:14 smithi138 ceph-mon[185886]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:14:14.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:14 smithi190 ceph-mon[160435]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:14 smithi190 ceph-mon[160435]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:14:15.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:14:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:14:14] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.33.4" 2023-10-01T09:14:16.211 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:14:16.212 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:14:16.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:16 smithi138 ceph-mon[192513]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:16.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:16 smithi138 ceph-mon[185886]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:16 smithi190 ceph-mon[160435]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:16.922 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":213,"stamp":"2023-10-01T09:14:15.265176+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6302,"num_read_kb":12700,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341100,"kb_used_data":22636,"kb_used_omap":30,"kb_used_meta":1318369,"kb_avail":748620116,"statfs":{"total":767960285184,"available":766586998784,"internally_reserved":0,"allocated":23179264,"data_stored":19087576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30777,"internal_metadata":1350010823},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002436"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.529858+0000","last_change":"2023-10-01T09:12:10.003833+0000","last_active":"2023-10-01T09:13:12.529858+0000","last_peered":"2023-10-01T09:13:12.529858+0000","last_clean":"2023-10-01T09:13:12.529858+0000","last_became_active":"2023-10-01T09:12:10.003482+0000","last_became_peered":"2023-10-01T09:12:10.003482+0000","last_unstale":"2023-10-01T09:13:12.529858+0000","last_undegraded":"2023-10-01T09:13:12.529858+0000","last_fullsized":"2023-10-01T09:13:12.529858+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530638+0000","last_change":"2023-10-01T09:12:40.944944+0000","last_active":"2023-10-01T09:13:12.530638+0000","last_peered":"2023-10-01T09:13:12.530638+0000","last_clean":"2023-10-01T09:13:12.530638+0000","last_became_active":"2023-10-01T09:12:40.944613+0000","last_became_peered":"2023-10-01T09:12:40.944613+0000","last_unstale":"2023-10-01T09:13:12.530638+0000","last_undegraded":"2023-10-01T09:13:12.530638+0000","last_fullsized":"2023-10-01T09:13:12.530638+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:48:20.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.549800+0000","last_change":"2023-10-01T09:13:13.549800+0000","last_active":"2023-10-01T09:13:13.549800+0000","last_peered":"2023-10-01T09:13:13.549800+0000","last_clean":"2023-10-01T09:13:13.549800+0000","last_became_active":"2023-10-01T09:13:13.549480+0000","last_became_peered":"2023-10-01T09:13:13.549480+0000","last_unstale":"2023-10-01T09:13:13.549800+0000","last_undegraded":"2023-10-01T09:13:13.549800+0000","last_fullsized":"2023-10-01T09:13:13.549800+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:08.673916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":79,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556295+0000","last_change":"2023-10-01T09:13:13.556295+0000","last_active":"2023-10-01T09:13:13.556295+0000","last_peered":"2023-10-01T09:13:13.556295+0000","last_clean":"2023-10-01T09:13:13.556295+0000","last_became_active":"2023-10-01T09:13:13.555953+0000","last_became_peered":"2023-10-01T09:13:13.555953+0000","last_unstale":"2023-10-01T09:13:13.556295+0000","last_undegraded":"2023-10-01T09:13:13.556295+0000","last_fullsized":"2023-10-01T09:13:13.556295+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:04:06.892156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554959+0000","last_change":"2023-10-01T09:13:13.554959+0000","last_active":"2023-10-01T09:13:13.554959+0000","last_peered":"2023-10-01T09:13:13.554959+0000","last_clean":"2023-10-01T09:13:13.554959+0000","last_became_active":"2023-10-01T09:13:13.554589+0000","last_became_peered":"2023-10-01T09:13:13.554589+0000","last_unstale":"2023-10-01T09:13:13.554959+0000","last_undegraded":"2023-10-01T09:13:13.554959+0000","last_fullsized":"2023-10-01T09:13:13.554959+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":58,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.541271+0000","last_change":"2023-10-01T09:03:03.202063+0000","last_active":"2023-10-01T09:13:12.541271+0000","last_peered":"2023-10-01T09:13:12.541271+0000","last_clean":"2023-10-01T09:13:12.541271+0000","last_became_active":"2023-10-01T09:03:03.201769+0000","last_became_peered":"2023-10-01T09:03:03.201769+0000","last_unstale":"2023-10-01T09:13:12.541271+0000","last_undegraded":"2023-10-01T09:13:12.541271+0000","last_fullsized":"2023-10-01T09:13:12.541271+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555744+0000","last_change":"2023-10-01T09:13:13.555744+0000","last_active":"2023-10-01T09:13:13.555744+0000","last_peered":"2023-10-01T09:13:13.555744+0000","last_clean":"2023-10-01T09:13:13.555744+0000","last_became_active":"2023-10-01T09:13:13.555163+0000","last_became_peered":"2023-10-01T09:13:13.555163+0000","last_unstale":"2023-10-01T09:13:13.555744+0000","last_undegraded":"2023-10-01T09:13:13.555744+0000","last_fullsized":"2023-10-01T09:13:13.555744+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.540456+0000","last_change":"2023-10-01T09:12:10.000449+0000","last_active":"2023-10-01T09:13:12.540456+0000","last_peered":"2023-10-01T09:13:12.540456+0000","last_clean":"2023-10-01T09:13:12.540456+0000","last_became_active":"2023-10-01T09:12:10.000000+0000","last_became_peered":"2023-10-01T09:12:10.000000+0000","last_unstale":"2023-10-01T09:13:12.540456+0000","last_undegraded":"2023-10-01T09:13:12.540456+0000","last_fullsized":"2023-10-01T09:13:12.540456+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":93,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.548647+0000","last_change":"2023-10-01T09:13:13.548647+0000","last_active":"2023-10-01T09:13:13.548647+0000","last_peered":"2023-10-01T09:13:13.548647+0000","last_clean":"2023-10-01T09:13:13.548647+0000","last_became_active":"2023-10-01T09:13:13.548382+0000","last_became_peered":"2023-10-01T09:13:13.548382+0000","last_unstale":"2023-10-01T09:13:13.548647+0000","last_undegraded":"2023-10-01T09:13:13.548647+0000","last_fullsized":"2023-10-01T09:13:13.548647+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:17:31.791258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":41,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.540696+0000","last_change":"2023-10-01T09:10:58.801161+0000","last_active":"2023-10-01T09:13:13.540696+0000","last_peered":"2023-10-01T09:13:13.540696+0000","last_clean":"2023-10-01T09:13:13.540696+0000","last_became_active":"2023-10-01T09:10:58.800794+0000","last_became_peered":"2023-10-01T09:10:58.800794+0000","last_unstale":"2023-10-01T09:13:13.540696+0000","last_undegraded":"2023-10-01T09:13:13.540696+0000","last_fullsized":"2023-10-01T09:13:13.540696+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:35.019445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.531915+0000","last_change":"2023-10-01T09:03:03.754839+0000","last_active":"2023-10-01T09:13:12.531915+0000","last_peered":"2023-10-01T09:13:12.531915+0000","last_clean":"2023-10-01T09:13:12.531915+0000","last_became_active":"2023-10-01T09:03:03.754506+0000","last_became_peered":"2023-10-01T09:03:03.754506+0000","last_unstale":"2023-10-01T09:13:12.531915+0000","last_undegraded":"2023-10-01T09:13:12.531915+0000","last_fullsized":"2023-10-01T09:13:12.531915+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.532609+0000","last_change":"2023-10-01T09:12:10.003174+0000","last_active":"2023-10-01T09:13:12.532609+0000","last_peered":"2023-10-01T09:13:12.532609+0000","last_clean":"2023-10-01T09:13:12.532609+0000","last_became_active":"2023-10-01T09:12:10.002808+0000","last_became_peered":"2023-10-01T09:12:10.002808+0000","last_unstale":"2023-10-01T09:13:12.532609+0000","last_undegraded":"2023-10-01T09:13:12.532609+0000","last_fullsized":"2023-10-01T09:13:12.532609+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:25.037091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":143,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535113+0000","last_change":"2023-10-01T09:12:40.949544+0000","last_active":"2023-10-01T09:13:12.535113+0000","last_peered":"2023-10-01T09:13:12.535113+0000","last_clean":"2023-10-01T09:13:12.535113+0000","last_became_active":"2023-10-01T09:12:40.949054+0000","last_became_peered":"2023-10-01T09:12:40.949054+0000","last_unstale":"2023-10-01T09:13:12.535113+0000","last_undegraded":"2023-10-01T09:13:12.535113+0000","last_fullsized":"2023-10-01T09:13:12.535113+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.552192+0000","last_change":"2023-10-01T09:13:13.552192+0000","last_active":"2023-10-01T09:13:13.552192+0000","last_peered":"2023-10-01T09:13:13.552192+0000","last_clean":"2023-10-01T09:13:13.552192+0000","last_became_active":"2023-10-01T09:13:13.551887+0000","last_became_peered":"2023-10-01T09:13:13.551887+0000","last_unstale":"2023-10-01T09:13:13.552192+0000","last_undegraded":"2023-10-01T09:13:13.552192+0000","last_fullsized":"2023-10-01T09:13:13.552192+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:23.633557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":491,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534872+0000","last_change":"2023-10-01T09:12:10.002334+0000","last_active":"2023-10-01T09:13:12.534872+0000","last_peered":"2023-10-01T09:13:12.534872+0000","last_clean":"2023-10-01T09:13:12.534872+0000","last_became_active":"2023-10-01T09:12:10.001791+0000","last_became_peered":"2023-10-01T09:12:10.001791+0000","last_unstale":"2023-10-01T09:13:12.534872+0000","last_undegraded":"2023-10-01T09:13:12.534872+0000","last_fullsized":"2023-10-01T09:13:12.534872+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:24:27.632780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":71,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.553580+0000","last_change":"2023-10-01T09:13:13.553580+0000","last_active":"2023-10-01T09:13:13.553580+0000","last_peered":"2023-10-01T09:13:13.553580+0000","last_clean":"2023-10-01T09:13:13.553580+0000","last_became_active":"2023-10-01T09:13:13.553011+0000","last_became_peered":"2023-10-01T09:13:13.553011+0000","last_unstale":"2023-10-01T09:13:13.553580+0000","last_undegraded":"2023-10-01T09:13:13.553580+0000","last_fullsized":"2023-10-01T09:13:13.553580+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:54:30.369308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530101+0000","last_change":"2023-10-01T09:10:58.797034+0000","last_active":"2023-10-01T09:13:12.530101+0000","last_peered":"2023-10-01T09:13:12.530101+0000","last_clean":"2023-10-01T09:13:12.530101+0000","last_became_active":"2023-10-01T09:10:58.796730+0000","last_became_peered":"2023-10-01T09:10:58.796730+0000","last_unstale":"2023-10-01T09:13:12.530101+0000","last_undegraded":"2023-10-01T09:13:12.530101+0000","last_fullsized":"2023-10-01T09:13:12.530101+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:32:15.982890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.543135+0000","last_change":"2023-10-01T09:12:40.948216+0000","last_active":"2023-10-01T09:13:13.543135+0000","last_peered":"2023-10-01T09:13:13.543135+0000","last_clean":"2023-10-01T09:13:13.543135+0000","last_became_active":"2023-10-01T09:12:40.947879+0000","last_became_peered":"2023-10-01T09:12:40.947879+0000","last_unstale":"2023-10-01T09:13:13.543135+0000","last_undegraded":"2023-10-01T09:13:13.543135+0000","last_fullsized":"2023-10-01T09:13:13.543135+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:46:40.612337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":465,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534902+0000","last_change":"2023-10-01T09:10:58.797964+0000","last_active":"2023-10-01T09:13:12.534902+0000","last_peered":"2023-10-01T09:13:12.534902+0000","last_clean":"2023-10-01T09:13:12.534902+0000","last_became_active":"2023-10-01T09:10:58.797447+0000","last_became_peered":"2023-10-01T09:10:58.797447+0000","last_unstale":"2023-10-01T09:13:12.534902+0000","last_undegraded":"2023-10-01T09:13:12.534902+0000","last_fullsized":"2023-10-01T09:13:12.534902+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.532709+0000","last_change":"2023-10-01T09:12:40.944991+0000","last_active":"2023-10-01T09:13:12.532709+0000","last_peered":"2023-10-01T09:13:12.532709+0000","last_clean":"2023-10-01T09:13:12.532709+0000","last_became_active":"2023-10-01T09:12:40.944688+0000","last_became_peered":"2023-10-01T09:12:40.944688+0000","last_unstale":"2023-10-01T09:13:12.532709+0000","last_undegraded":"2023-10-01T09:13:12.532709+0000","last_fullsized":"2023-10-01T09:13:12.532709+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:30.015801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.552391+0000","last_change":"2023-10-01T09:13:13.552391+0000","last_active":"2023-10-01T09:13:13.552391+0000","last_peered":"2023-10-01T09:13:13.552391+0000","last_clean":"2023-10-01T09:13:13.552391+0000","last_became_active":"2023-10-01T09:13:13.552082+0000","last_became_peered":"2023-10-01T09:13:13.552082+0000","last_unstale":"2023-10-01T09:13:13.552391+0000","last_undegraded":"2023-10-01T09:13:13.552391+0000","last_fullsized":"2023-10-01T09:13:13.552391+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:07:19.929732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":201,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535494+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:13:12.535494+0000","last_peered":"2023-10-01T09:13:12.535494+0000","last_clean":"2023-10-01T09:13:12.535494+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:13:12.535494+0000","last_undegraded":"2023-10-01T09:13:12.535494+0000","last_fullsized":"2023-10-01T09:13:12.535494+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":55,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534655+0000","last_change":"2023-10-01T09:03:03.227254+0000","last_active":"2023-10-01T09:13:12.534655+0000","last_peered":"2023-10-01T09:13:12.534655+0000","last_clean":"2023-10-01T09:13:12.534655+0000","last_became_active":"2023-10-01T09:03:03.226941+0000","last_became_peered":"2023-10-01T09:03:03.226941+0000","last_unstale":"2023-10-01T09:13:12.534655+0000","last_undegraded":"2023-10-01T09:13:12.534655+0000","last_fullsized":"2023-10-01T09:13:12.534655+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":63,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556049+0000","last_change":"2023-10-01T09:13:13.556049+0000","last_active":"2023-10-01T09:13:13.556049+0000","last_peered":"2023-10-01T09:13:13.556049+0000","last_clean":"2023-10-01T09:13:13.556049+0000","last_became_active":"2023-10-01T09:13:13.555739+0000","last_became_peered":"2023-10-01T09:13:13.555739+0000","last_unstale":"2023-10-01T09:13:13.556049+0000","last_undegraded":"2023-10-01T09:13:13.556049+0000","last_fullsized":"2023-10-01T09:13:13.556049+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:50:34.475690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":307,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534586+0000","last_change":"2023-10-01T09:10:58.798697+0000","last_active":"2023-10-01T09:13:12.534586+0000","last_peered":"2023-10-01T09:13:12.534586+0000","last_clean":"2023-10-01T09:13:12.534586+0000","last_became_active":"2023-10-01T09:10:58.798203+0000","last_became_peered":"2023-10-01T09:10:58.798203+0000","last_unstale":"2023-10-01T09:13:12.534586+0000","last_undegraded":"2023-10-01T09:13:12.534586+0000","last_fullsized":"2023-10-01T09:13:12.534586+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":921,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539380+0000","last_change":"2023-10-01T09:12:40.946131+0000","last_active":"2023-10-01T09:13:12.539380+0000","last_peered":"2023-10-01T09:13:12.539380+0000","last_clean":"2023-10-01T09:13:12.539380+0000","last_became_active":"2023-10-01T09:12:40.945524+0000","last_became_peered":"2023-10-01T09:12:40.945524+0000","last_unstale":"2023-10-01T09:13:12.539380+0000","last_undegraded":"2023-10-01T09:13:12.539380+0000","last_fullsized":"2023-10-01T09:13:12.539380+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535348+0000","last_change":"2023-10-01T09:12:09.994071+0000","last_active":"2023-10-01T09:13:12.535348+0000","last_peered":"2023-10-01T09:13:12.535348+0000","last_clean":"2023-10-01T09:13:12.535348+0000","last_became_active":"2023-10-01T09:12:09.993782+0000","last_became_peered":"2023-10-01T09:12:09.993782+0000","last_unstale":"2023-10-01T09:13:12.535348+0000","last_undegraded":"2023-10-01T09:13:12.535348+0000","last_fullsized":"2023-10-01T09:13:12.535348+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:12:49.880685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.543126+0000","last_change":"2023-10-01T09:12:40.946267+0000","last_active":"2023-10-01T09:13:13.543126+0000","last_peered":"2023-10-01T09:13:13.543126+0000","last_clean":"2023-10-01T09:13:13.543126+0000","last_became_active":"2023-10-01T09:12:40.945970+0000","last_became_peered":"2023-10-01T09:12:40.945970+0000","last_unstale":"2023-10-01T09:13:13.543126+0000","last_undegraded":"2023-10-01T09:13:13.543126+0000","last_fullsized":"2023-10-01T09:13:13.543126+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:42.904509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":375,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551669+0000","last_change":"2023-10-01T09:13:13.551669+0000","last_active":"2023-10-01T09:13:13.551669+0000","last_peered":"2023-10-01T09:13:13.551669+0000","last_clean":"2023-10-01T09:13:13.551669+0000","last_became_active":"2023-10-01T09:13:13.550982+0000","last_became_peered":"2023-10-01T09:13:13.550982+0000","last_unstale":"2023-10-01T09:13:13.551669+0000","last_undegraded":"2023-10-01T09:13:13.551669+0000","last_fullsized":"2023-10-01T09:13:13.551669+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:21:09.879713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554041+0000","last_change":"2023-10-01T09:13:13.549385+0000","last_active":"2023-10-01T09:13:13.554041+0000","last_peered":"2023-10-01T09:13:13.554041+0000","last_clean":"2023-10-01T09:13:13.554041+0000","last_became_active":"2023-10-01T09:13:13.549035+0000","last_became_peered":"2023-10-01T09:13:13.549035+0000","last_unstale":"2023-10-01T09:13:13.554041+0000","last_undegraded":"2023-10-01T09:13:13.554041+0000","last_fullsized":"2023-10-01T09:13:13.554041+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:52:41.408995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530244+0000","last_change":"2023-10-01T09:10:58.799722+0000","last_active":"2023-10-01T09:13:12.530244+0000","last_peered":"2023-10-01T09:13:12.530244+0000","last_clean":"2023-10-01T09:13:12.530244+0000","last_became_active":"2023-10-01T09:10:58.799313+0000","last_became_peered":"2023-10-01T09:10:58.799313+0000","last_unstale":"2023-10-01T09:13:12.530244+0000","last_undegraded":"2023-10-01T09:13:12.530244+0000","last_fullsized":"2023-10-01T09:13:12.530244+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":805,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539880+0000","last_change":"2023-10-01T09:12:09.997936+0000","last_active":"2023-10-01T09:13:12.539880+0000","last_peered":"2023-10-01T09:13:12.539880+0000","last_clean":"2023-10-01T09:13:12.539880+0000","last_became_active":"2023-10-01T09:12:09.997427+0000","last_became_peered":"2023-10-01T09:12:09.997427+0000","last_unstale":"2023-10-01T09:13:12.539880+0000","last_undegraded":"2023-10-01T09:13:12.539880+0000","last_fullsized":"2023-10-01T09:13:12.539880+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1219,"num_read_kb":1114,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.544355+0000","last_change":"2023-10-01T09:12:10.001493+0000","last_active":"2023-10-01T09:13:13.544355+0000","last_peered":"2023-10-01T09:13:13.544355+0000","last_clean":"2023-10-01T09:13:13.544355+0000","last_became_active":"2023-10-01T09:12:10.000899+0000","last_became_peered":"2023-10-01T09:12:10.000899+0000","last_unstale":"2023-10-01T09:13:13.544355+0000","last_undegraded":"2023-10-01T09:13:13.544355+0000","last_fullsized":"2023-10-01T09:13:13.544355+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:47:54.011059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551681+0000","last_change":"2023-10-01T09:13:13.551681+0000","last_active":"2023-10-01T09:13:13.551681+0000","last_peered":"2023-10-01T09:13:13.551681+0000","last_clean":"2023-10-01T09:13:13.551681+0000","last_became_active":"2023-10-01T09:13:13.551268+0000","last_became_peered":"2023-10-01T09:13:13.551268+0000","last_unstale":"2023-10-01T09:13:13.551681+0000","last_undegraded":"2023-10-01T09:13:13.551681+0000","last_fullsized":"2023-10-01T09:13:13.551681+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":918,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.543112+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:13:13.543112+0000","last_peered":"2023-10-01T09:13:13.543112+0000","last_clean":"2023-10-01T09:13:13.543112+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:13:13.543112+0000","last_undegraded":"2023-10-01T09:13:13.543112+0000","last_fullsized":"2023-10-01T09:13:13.543112+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551757+0000","last_change":"2023-10-01T09:13:13.551757+0000","last_active":"2023-10-01T09:13:13.551757+0000","last_peered":"2023-10-01T09:13:13.551757+0000","last_clean":"2023-10-01T09:13:13.551757+0000","last_became_active":"2023-10-01T09:13:13.551483+0000","last_became_peered":"2023-10-01T09:13:13.551483+0000","last_unstale":"2023-10-01T09:13:13.551757+0000","last_undegraded":"2023-10-01T09:13:13.551757+0000","last_fullsized":"2023-10-01T09:13:13.551757+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:33:46.184050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555563+0000","last_change":"2023-10-01T09:13:13.555563+0000","last_active":"2023-10-01T09:13:13.555563+0000","last_peered":"2023-10-01T09:13:13.555563+0000","last_clean":"2023-10-01T09:13:13.555563+0000","last_became_active":"2023-10-01T09:13:13.555070+0000","last_became_peered":"2023-10-01T09:13:13.555070+0000","last_unstale":"2023-10-01T09:13:13.555563+0000","last_undegraded":"2023-10-01T09:13:13.555563+0000","last_fullsized":"2023-10-01T09:13:13.555563+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:44.557168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.531259+0000","last_change":"2023-10-01T09:12:10.003847+0000","last_active":"2023-10-01T09:13:12.531259+0000","last_peered":"2023-10-01T09:13:12.531259+0000","last_clean":"2023-10-01T09:13:12.531259+0000","last_became_active":"2023-10-01T09:12:10.003525+0000","last_became_peered":"2023-10-01T09:12:10.003525+0000","last_unstale":"2023-10-01T09:13:12.531259+0000","last_undegraded":"2023-10-01T09:13:12.531259+0000","last_fullsized":"2023-10-01T09:13:12.531259+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.532071+0000","last_change":"2023-10-01T09:12:40.945099+0000","last_active":"2023-10-01T09:13:12.532071+0000","last_peered":"2023-10-01T09:13:12.532071+0000","last_clean":"2023-10-01T09:13:12.532071+0000","last_became_active":"2023-10-01T09:12:40.944712+0000","last_became_peered":"2023-10-01T09:12:40.944712+0000","last_unstale":"2023-10-01T09:13:12.532071+0000","last_undegraded":"2023-10-01T09:13:12.532071+0000","last_fullsized":"2023-10-01T09:13:12.532071+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.531487+0000","last_change":"2023-10-01T09:12:40.946791+0000","last_active":"2023-10-01T09:13:12.531487+0000","last_peered":"2023-10-01T09:13:12.531487+0000","last_clean":"2023-10-01T09:13:12.531487+0000","last_became_active":"2023-10-01T09:12:40.946418+0000","last_became_peered":"2023-10-01T09:12:40.946418+0000","last_unstale":"2023-10-01T09:13:12.531487+0000","last_undegraded":"2023-10-01T09:13:12.531487+0000","last_fullsized":"2023-10-01T09:13:12.531487+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.536203+0000","last_change":"2023-10-01T09:12:10.003934+0000","last_active":"2023-10-01T09:13:12.536203+0000","last_peered":"2023-10-01T09:13:12.536203+0000","last_clean":"2023-10-01T09:13:12.536203+0000","last_became_active":"2023-10-01T09:12:10.003597+0000","last_became_peered":"2023-10-01T09:12:10.003597+0000","last_unstale":"2023-10-01T09:13:12.536203+0000","last_undegraded":"2023-10-01T09:13:12.536203+0000","last_fullsized":"2023-10-01T09:13:12.536203+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556491+0000","last_change":"2023-10-01T09:13:13.556491+0000","last_active":"2023-10-01T09:13:13.556491+0000","last_peered":"2023-10-01T09:13:13.556491+0000","last_clean":"2023-10-01T09:13:13.556491+0000","last_became_active":"2023-10-01T09:13:13.556118+0000","last_became_peered":"2023-10-01T09:13:13.556118+0000","last_unstale":"2023-10-01T09:13:13.556491+0000","last_undegraded":"2023-10-01T09:13:13.556491+0000","last_fullsized":"2023-10-01T09:13:13.556491+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554321+0000","last_change":"2023-10-01T09:13:13.554321+0000","last_active":"2023-10-01T09:13:13.554321+0000","last_peered":"2023-10-01T09:13:13.554321+0000","last_clean":"2023-10-01T09:13:13.554321+0000","last_became_active":"2023-10-01T09:13:13.553986+0000","last_became_peered":"2023-10-01T09:13:13.553986+0000","last_unstale":"2023-10-01T09:13:13.554321+0000","last_undegraded":"2023-10-01T09:13:13.554321+0000","last_fullsized":"2023-10-01T09:13:13.554321+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:09:12.848269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530790+0000","last_change":"2023-10-01T09:12:10.001565+0000","last_active":"2023-10-01T09:13:12.530790+0000","last_peered":"2023-10-01T09:13:12.530790+0000","last_clean":"2023-10-01T09:13:12.530790+0000","last_became_active":"2023-10-01T09:12:10.001275+0000","last_became_peered":"2023-10-01T09:12:10.001275+0000","last_unstale":"2023-10-01T09:13:12.530790+0000","last_undegraded":"2023-10-01T09:13:12.530790+0000","last_fullsized":"2023-10-01T09:13:12.530790+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:39.067046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":71,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555458+0000","last_change":"2023-10-01T09:13:13.555458+0000","last_active":"2023-10-01T09:13:13.555458+0000","last_peered":"2023-10-01T09:13:13.555458+0000","last_clean":"2023-10-01T09:13:13.555458+0000","last_became_active":"2023-10-01T09:13:13.555058+0000","last_became_peered":"2023-10-01T09:13:13.555058+0000","last_unstale":"2023-10-01T09:13:13.555458+0000","last_undegraded":"2023-10-01T09:13:13.555458+0000","last_fullsized":"2023-10-01T09:13:13.555458+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":70,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535389+0000","last_change":"2023-10-01T09:12:09.999571+0000","last_active":"2023-10-01T09:13:12.535389+0000","last_peered":"2023-10-01T09:13:12.535389+0000","last_clean":"2023-10-01T09:13:12.535389+0000","last_became_active":"2023-10-01T09:12:09.999272+0000","last_became_peered":"2023-10-01T09:12:09.999272+0000","last_unstale":"2023-10-01T09:13:12.535389+0000","last_undegraded":"2023-10-01T09:13:12.535389+0000","last_fullsized":"2023-10-01T09:13:12.535389+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":57,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535047+0000","last_change":"2023-10-01T09:03:03.200972+0000","last_active":"2023-10-01T09:13:12.535047+0000","last_peered":"2023-10-01T09:13:12.535047+0000","last_clean":"2023-10-01T09:13:12.535047+0000","last_became_active":"2023-10-01T09:03:03.200695+0000","last_became_peered":"2023-10-01T09:03:03.200695+0000","last_unstale":"2023-10-01T09:13:12.535047+0000","last_undegraded":"2023-10-01T09:13:12.535047+0000","last_fullsized":"2023-10-01T09:13:12.535047+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556674+0000","last_change":"2023-10-01T09:13:13.556674+0000","last_active":"2023-10-01T09:13:13.556674+0000","last_peered":"2023-10-01T09:13:13.556674+0000","last_clean":"2023-10-01T09:13:13.556674+0000","last_became_active":"2023-10-01T09:13:13.556271+0000","last_became_peered":"2023-10-01T09:13:13.556271+0000","last_unstale":"2023-10-01T09:13:13.556674+0000","last_undegraded":"2023-10-01T09:13:13.556674+0000","last_fullsized":"2023-10-01T09:13:13.556674+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535994+0000","last_change":"2023-10-01T09:12:10.003552+0000","last_active":"2023-10-01T09:13:12.535994+0000","last_peered":"2023-10-01T09:13:12.535994+0000","last_clean":"2023-10-01T09:13:12.535994+0000","last_became_active":"2023-10-01T09:12:10.003131+0000","last_became_peered":"2023-10-01T09:12:10.003131+0000","last_unstale":"2023-10-01T09:13:12.535994+0000","last_undegraded":"2023-10-01T09:13:12.535994+0000","last_fullsized":"2023-10-01T09:13:12.535994+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":55,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539254+0000","last_change":"2023-10-01T09:03:04.245105+0000","last_active":"2023-10-01T09:13:12.539254+0000","last_peered":"2023-10-01T09:13:12.539254+0000","last_clean":"2023-10-01T09:13:12.539254+0000","last_became_active":"2023-10-01T09:03:04.244807+0000","last_became_peered":"2023-10-01T09:03:04.244807+0000","last_unstale":"2023-10-01T09:13:12.539254+0000","last_undegraded":"2023-10-01T09:13:12.539254+0000","last_fullsized":"2023-10-01T09:13:12.539254+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":72,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.537360+0000","last_change":"2023-10-01T09:10:58.797850+0000","last_active":"2023-10-01T09:13:12.537360+0000","last_peered":"2023-10-01T09:13:12.537360+0000","last_clean":"2023-10-01T09:13:12.537360+0000","last_became_active":"2023-10-01T09:10:58.797403+0000","last_became_peered":"2023-10-01T09:10:58.797403+0000","last_unstale":"2023-10-01T09:13:12.537360+0000","last_undegraded":"2023-10-01T09:13:12.537360+0000","last_fullsized":"2023-10-01T09:13:12.537360+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550588+0000","last_change":"2023-10-01T09:13:13.550588+0000","last_active":"2023-10-01T09:13:13.550588+0000","last_peered":"2023-10-01T09:13:13.550588+0000","last_clean":"2023-10-01T09:13:13.550588+0000","last_became_active":"2023-10-01T09:13:13.550271+0000","last_became_peered":"2023-10-01T09:13:13.550271+0000","last_unstale":"2023-10-01T09:13:13.550588+0000","last_undegraded":"2023-10-01T09:13:13.550588+0000","last_fullsized":"2023-10-01T09:13:13.550588+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:19:47.301376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.535409+0000","last_change":"2023-10-01T09:10:58.798168+0000","last_active":"2023-10-01T09:13:12.535409+0000","last_peered":"2023-10-01T09:13:12.535409+0000","last_clean":"2023-10-01T09:13:12.535409+0000","last_became_active":"2023-10-01T09:10:58.797781+0000","last_became_peered":"2023-10-01T09:10:58.797781+0000","last_unstale":"2023-10-01T09:13:12.535409+0000","last_undegraded":"2023-10-01T09:13:12.535409+0000","last_fullsized":"2023-10-01T09:13:12.535409+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550434+0000","last_change":"2023-10-01T09:13:13.548531+0000","last_active":"2023-10-01T09:13:13.550434+0000","last_peered":"2023-10-01T09:13:13.550434+0000","last_clean":"2023-10-01T09:13:13.550434+0000","last_became_active":"2023-10-01T09:13:13.547881+0000","last_became_peered":"2023-10-01T09:13:13.547881+0000","last_unstale":"2023-10-01T09:13:13.550434+0000","last_undegraded":"2023-10-01T09:13:13.550434+0000","last_fullsized":"2023-10-01T09:13:13.550434+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:13:24.068359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":56,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534146+0000","last_change":"2023-10-01T09:03:03.194952+0000","last_active":"2023-10-01T09:13:12.534146+0000","last_peered":"2023-10-01T09:13:12.534146+0000","last_clean":"2023-10-01T09:13:12.534146+0000","last_became_active":"2023-10-01T09:03:03.194643+0000","last_became_peered":"2023-10-01T09:03:03.194643+0000","last_unstale":"2023-10-01T09:13:12.534146+0000","last_undegraded":"2023-10-01T09:13:12.534146+0000","last_fullsized":"2023-10-01T09:13:12.534146+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":70,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.558367+0000","last_change":"2023-10-01T09:13:13.558367+0000","last_active":"2023-10-01T09:13:13.558367+0000","last_peered":"2023-10-01T09:13:13.558367+0000","last_clean":"2023-10-01T09:13:13.558367+0000","last_became_active":"2023-10-01T09:13:13.557952+0000","last_became_peered":"2023-10-01T09:13:13.557952+0000","last_unstale":"2023-10-01T09:13:13.558367+0000","last_undegraded":"2023-10-01T09:13:13.558367+0000","last_fullsized":"2023-10-01T09:13:13.558367+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550521+0000","last_change":"2023-10-01T09:13:13.548094+0000","last_active":"2023-10-01T09:13:13.550521+0000","last_peered":"2023-10-01T09:13:13.550521+0000","last_clean":"2023-10-01T09:13:13.550521+0000","last_became_active":"2023-10-01T09:13:13.547782+0000","last_became_peered":"2023-10-01T09:13:13.547782+0000","last_unstale":"2023-10-01T09:13:13.550521+0000","last_undegraded":"2023-10-01T09:13:13.550521+0000","last_fullsized":"2023-10-01T09:13:13.550521+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:15.220827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550633+0000","last_change":"2023-10-01T09:13:13.550633+0000","last_active":"2023-10-01T09:13:13.550633+0000","last_peered":"2023-10-01T09:13:13.550633+0000","last_clean":"2023-10-01T09:13:13.550633+0000","last_became_active":"2023-10-01T09:13:13.550305+0000","last_became_peered":"2023-10-01T09:13:13.550305+0000","last_unstale":"2023-10-01T09:13:13.550633+0000","last_undegraded":"2023-10-01T09:13:13.550633+0000","last_fullsized":"2023-10-01T09:13:13.550633+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:51:46.559383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":73,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534816+0000","last_change":"2023-10-01T09:12:40.945277+0000","last_active":"2023-10-01T09:13:12.534816+0000","last_peered":"2023-10-01T09:13:12.534816+0000","last_clean":"2023-10-01T09:13:12.534816+0000","last_became_active":"2023-10-01T09:12:40.944843+0000","last_became_peered":"2023-10-01T09:12:40.944843+0000","last_unstale":"2023-10-01T09:13:12.534816+0000","last_undegraded":"2023-10-01T09:13:12.534816+0000","last_fullsized":"2023-10-01T09:13:12.534816+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":78,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530812+0000","last_change":"2023-10-01T09:12:40.946973+0000","last_active":"2023-10-01T09:13:12.530812+0000","last_peered":"2023-10-01T09:13:12.530812+0000","last_clean":"2023-10-01T09:13:12.530812+0000","last_became_active":"2023-10-01T09:12:40.946427+0000","last_became_peered":"2023-10-01T09:12:40.946427+0000","last_unstale":"2023-10-01T09:13:12.530812+0000","last_undegraded":"2023-10-01T09:13:12.530812+0000","last_fullsized":"2023-10-01T09:13:12.530812+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":77,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.530204+0000","last_change":"2023-10-01T09:03:03.200261+0000","last_active":"2023-10-01T09:13:12.530204+0000","last_peered":"2023-10-01T09:13:12.530204+0000","last_clean":"2023-10-01T09:13:12.530204+0000","last_became_active":"2023-10-01T09:03:03.199944+0000","last_became_peered":"2023-10-01T09:03:03.199944+0000","last_unstale":"2023-10-01T09:13:12.530204+0000","last_undegraded":"2023-10-01T09:13:12.530204+0000","last_fullsized":"2023-10-01T09:13:12.530204+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.533264+0000","last_change":"2023-10-01T09:12:40.946520+0000","last_active":"2023-10-01T09:13:12.533264+0000","last_peered":"2023-10-01T09:13:12.533264+0000","last_clean":"2023-10-01T09:13:12.533264+0000","last_became_active":"2023-10-01T09:12:40.946227+0000","last_became_peered":"2023-10-01T09:12:40.946227+0000","last_unstale":"2023-10-01T09:13:12.533264+0000","last_undegraded":"2023-10-01T09:13:12.533264+0000","last_fullsized":"2023-10-01T09:13:12.533264+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:06:06.451923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":74,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.556801+0000","last_change":"2023-10-01T09:13:13.556801+0000","last_active":"2023-10-01T09:13:13.556801+0000","last_peered":"2023-10-01T09:13:13.556801+0000","last_clean":"2023-10-01T09:13:13.556801+0000","last_became_active":"2023-10-01T09:13:13.556397+0000","last_became_peered":"2023-10-01T09:13:13.556397+0000","last_unstale":"2023-10-01T09:13:13.556801+0000","last_undegraded":"2023-10-01T09:13:13.556801+0000","last_fullsized":"2023-10-01T09:13:13.556801+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554942+0000","last_change":"2023-10-01T09:13:13.554942+0000","last_active":"2023-10-01T09:13:13.554942+0000","last_peered":"2023-10-01T09:13:13.554942+0000","last_clean":"2023-10-01T09:13:13.554942+0000","last_became_active":"2023-10-01T09:13:13.554664+0000","last_became_peered":"2023-10-01T09:13:13.554664+0000","last_unstale":"2023-10-01T09:13:13.554942+0000","last_undegraded":"2023-10-01T09:13:13.554942+0000","last_fullsized":"2023-10-01T09:13:13.554942+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:43:14.532709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.539899+0000","last_change":"2023-10-01T09:12:10.003091+0000","last_active":"2023-10-01T09:13:12.539899+0000","last_peered":"2023-10-01T09:13:12.539899+0000","last_clean":"2023-10-01T09:13:12.539899+0000","last_became_active":"2023-10-01T09:12:10.002773+0000","last_became_peered":"2023-10-01T09:12:10.002773+0000","last_unstale":"2023-10-01T09:13:12.539899+0000","last_undegraded":"2023-10-01T09:13:12.539899+0000","last_fullsized":"2023-10-01T09:13:12.539899+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555368+0000","last_change":"2023-10-01T09:13:13.555368+0000","last_active":"2023-10-01T09:13:13.555368+0000","last_peered":"2023-10-01T09:13:13.555368+0000","last_clean":"2023-10-01T09:13:13.555368+0000","last_became_active":"2023-10-01T09:13:13.554806+0000","last_became_peered":"2023-10-01T09:13:13.554806+0000","last_unstale":"2023-10-01T09:13:13.555368+0000","last_undegraded":"2023-10-01T09:13:13.555368+0000","last_fullsized":"2023-10-01T09:13:13.555368+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:24:32.687833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.550551+0000","last_change":"2023-10-01T09:13:13.550193+0000","last_active":"2023-10-01T09:13:13.550551+0000","last_peered":"2023-10-01T09:13:13.550551+0000","last_clean":"2023-10-01T09:13:13.550551+0000","last_became_active":"2023-10-01T09:13:13.549900+0000","last_became_peered":"2023-10-01T09:13:13.549900+0000","last_unstale":"2023-10-01T09:13:13.550551+0000","last_undegraded":"2023-10-01T09:13:13.550551+0000","last_fullsized":"2023-10-01T09:13:13.550551+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:22.643161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":79,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.534962+0000","last_change":"2023-10-01T09:12:40.947591+0000","last_active":"2023-10-01T09:13:12.534962+0000","last_peered":"2023-10-01T09:13:12.534962+0000","last_clean":"2023-10-01T09:13:12.534962+0000","last_became_active":"2023-10-01T09:12:40.947242+0000","last_became_peered":"2023-10-01T09:12:40.947242+0000","last_unstale":"2023-10-01T09:13:12.534962+0000","last_undegraded":"2023-10-01T09:13:12.534962+0000","last_fullsized":"2023-10-01T09:13:12.534962+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.555490+0000","last_change":"2023-10-01T09:13:13.555490+0000","last_active":"2023-10-01T09:13:13.555490+0000","last_peered":"2023-10-01T09:13:13.555490+0000","last_clean":"2023-10-01T09:13:13.555490+0000","last_became_active":"2023-10-01T09:13:13.554506+0000","last_became_peered":"2023-10-01T09:13:13.554506+0000","last_unstale":"2023-10-01T09:13:13.555490+0000","last_undegraded":"2023-10-01T09:13:13.555490+0000","last_fullsized":"2023-10-01T09:13:13.555490+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:18.834474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.548695+0000","last_change":"2023-10-01T09:13:13.548695+0000","last_active":"2023-10-01T09:13:13.548695+0000","last_peered":"2023-10-01T09:13:13.548695+0000","last_clean":"2023-10-01T09:13:13.548695+0000","last_became_active":"2023-10-01T09:13:13.548256+0000","last_became_peered":"2023-10-01T09:13:13.548256+0000","last_unstale":"2023-10-01T09:13:13.548695+0000","last_undegraded":"2023-10-01T09:13:13.548695+0000","last_fullsized":"2023-10-01T09:13:13.548695+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:35:28.655733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.551596+0000","last_change":"2023-10-01T09:13:13.551596+0000","last_active":"2023-10-01T09:13:13.551596+0000","last_peered":"2023-10-01T09:13:13.551596+0000","last_clean":"2023-10-01T09:13:13.551596+0000","last_became_active":"2023-10-01T09:13:13.551300+0000","last_became_peered":"2023-10-01T09:13:13.551300+0000","last_unstale":"2023-10-01T09:13:13.551596+0000","last_undegraded":"2023-10-01T09:13:13.551596+0000","last_fullsized":"2023-10-01T09:13:13.551596+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:57:45.081780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":73,"reported_epoch":105,"state":"active+clean","last_fresh":"2023-10-01T09:13:13.554780+0000","last_change":"2023-10-01T09:13:13.554780+0000","last_active":"2023-10-01T09:13:13.554780+0000","last_peered":"2023-10-01T09:13:13.554780+0000","last_clean":"2023-10-01T09:13:13.554780+0000","last_became_active":"2023-10-01T09:13:13.554394+0000","last_became_peered":"2023-10-01T09:13:13.554394+0000","last_unstale":"2023-10-01T09:13:13.554780+0000","last_undegraded":"2023-10-01T09:13:13.554780+0000","last_fullsized":"2023-10-01T09:13:13.554780+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:09:02.869321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":86,"reported_epoch":104,"state":"active+clean","last_fresh":"2023-10-01T09:13:12.529938+0000","last_change":"2023-10-01T09:12:40.947346+0000","last_active":"2023-10-01T09:13:12.529938+0000","last_peered":"2023-10-01T09:13:12.529938+0000","last_clean":"2023-10-01T09:13:12.529938+0000","last_became_active":"2023-10-01T09:12:40.946939+0000","last_became_peered":"2023-10-01T09:12:40.946939+0000","last_unstale":"2023-10-01T09:13:12.529938+0000","last_undegraded":"2023-10-01T09:13:12.529938+0000","last_fullsized":"2023-10-01T09:13:12.529938+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27349,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12188,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430363,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":306940,"kb_used_data":5748,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438212,"statfs":{"total":95995035648,"available":95680729088,"internally_reserved":0,"allocated":5885952,"data_stored":5375322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:14:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.80300000000000005,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.503},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.54600000000000004,"15min":0.54700000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.81899999999999995,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.46000000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:13:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.434},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.52800000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.49199999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:12:56 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.57399999999999995,"15min":0.60599999999999998},"min":{"1min":0.441,"5min":0.32500000000000001,"15min":0.312},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":5.6210000000000004},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.58399999999999996,"15min":0.55400000000000005},"min":{"1min":0.41599999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81299999999999994,"5min":0.89200000000000002,"15min":2.2050000000000001},"last":0.47299999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.47699999999999998,"15min":0.47599999999999998},"min":{"1min":0.34399999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.71799999999999997,"5min":0.77700000000000002,"15min":0.79100000000000004},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.51700000000000002,"15min":0.56599999999999995},"min":{"1min":0.29199999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.78700000000000003,"5min":0.82799999999999996,"15min":5.5369999999999999},"last":0.34799999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.498,"15min":0.50900000000000001},"min":{"1min":0.33000000000000002,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":0.69099999999999995,"5min":0.76500000000000001,"15min":0.84899999999999998},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.54700000000000004,"15min":0.54300000000000004},"min":{"1min":0.41999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.39000000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.39900000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.84699999999999998,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.58099999999999996,"15min":0.628},"min":{"1min":0.34399999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.82399999999999995,"5min":0.88500000000000001,"15min":5.5700000000000003},"last":0.56899999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626592,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302828,"kb_used_data":1636,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442324,"statfs":{"total":95995035648,"available":95684939776,"internally_reserved":0,"allocated":1675264,"data_stored":1180261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:13:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.58299999999999996,"15min":0.61199999999999999},"min":{"1min":0.35899999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.71399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.57899999999999996,"15min":0.61299999999999999},"min":{"1min":0.38900000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.748,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.59899999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:13:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.57799999999999996}]},{"osd":2,"last update":"Sun Oct 1 09:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.503}]},{"osd":3,"last update":"Sun Oct 1 09:12:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.57599999999999996,"15min":0.55900000000000005},"min":{"1min":0.38500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.86499999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.57199999999999995,"15min":0.56999999999999995},"min":{"1min":0.40699999999999997,"5min":0.36399999999999999,"15min":0.28399999999999997},"max":{"1min":0.86299999999999999,"5min":1.0449999999999999,"15min":1.4490000000000001},"last":0.53700000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.48099999999999998,"15min":0.47199999999999998},"min":{"1min":0.32600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.70799999999999996,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.379},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51800000000000002,"15min":0.51000000000000001},"min":{"1min":0.372,"5min":0.30099999999999999,"15min":0.29399999999999998},"max":{"1min":0.74199999999999999,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.61499999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51600000000000001,"15min":0.51100000000000001},"min":{"1min":0.36599999999999999,"5min":0.315,"15min":0.251},"max":{"1min":0.69299999999999995,"5min":0.79200000000000004,"15min":0.80200000000000005},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56000000000000005,"15min":0.54800000000000004},"min":{"1min":0.35899999999999999,"5min":0.30499999999999999,"15min":0.28299999999999997},"max":{"1min":0.76700000000000002,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.58499999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56100000000000005,"15min":0.54600000000000004},"min":{"1min":0.371,"5min":0.32600000000000001,"15min":0.29899999999999999},"max":{"1min":0.76300000000000001,"5min":0.79600000000000004,"15min":0.80500000000000005},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.60799999999999998,"15min":0.58599999999999997},"min":{"1min":0.38900000000000001,"5min":0.34999999999999998,"15min":0.32400000000000001},"max":{"1min":0.78900000000000003,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.63600000000000001}]}]},{"osd":1,"up_from":94,"seq":403726925851,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36744,"kb_used_data":5184,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93708408,"statfs":{"total":95995035648,"available":95957409792,"internally_reserved":0,"allocated":5308416,"data_stored":4788066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":32301479},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.311,"5min":0.33900000000000002,"15min":0.34300000000000003},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.51300000000000001,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.33900000000000002,"5min":0.36099999999999999,"15min":0.36499999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.50900000000000001,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.311}]},{"osd":2,"last update":"Sun Oct 1 09:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.377}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34799999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54400000000000004,"15min":0.54300000000000004},"min":{"1min":0.41299999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.504},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.56599999999999995,"15min":0.56200000000000006},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.85699999999999998,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.51700000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.39900000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.59099999999999997,"15min":0.59499999999999997},"min":{"1min":0.41199999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.82099999999999995,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.438}]},{"osd":6,"last update":"Sun Oct 1 09:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.59699999999999998,"15min":0.59899999999999998},"min":{"1min":0.42499999999999999,"5min":0.374,"15min":0.374},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.627,"15min":0.63100000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.85299999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.56899999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:14:11 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59699999999999998,"15min":0.59999999999999998},"min":{"1min":0.38700000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.88,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60699999999999998,"15min":0.60599999999999998},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.55000000000000004}]}]},{"osd":0,"up_from":89,"seq":382252089385,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37352,"kb_used_data":5792,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93707800,"statfs":{"total":95995035648,"available":95956787200,"internally_reserved":0,"allocated":5931008,"data_stored":5383726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":32300994},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:13:12 2023","interfaces":[{"interface":"back","average":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.28399999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.36199999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:12:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.33000000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:14:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.63,"15min":0.63800000000000001},"min":{"1min":0.39900000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.76600000000000001,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.65000000000000002,"15min":0.66800000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.88300000000000001,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.48399999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:14:04 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.65500000000000003,"15min":0.67800000000000005},"min":{"1min":0.42899999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.82499999999999996,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.65700000000000003,"15min":0.67700000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.73999999999999999,"5min":0.999,"15min":0.999},"last":0.497}]},{"osd":6,"last update":"Sun Oct 1 09:14:04 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.66200000000000003,"15min":0.68600000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.78100000000000003,"5min":1.0329999999999999,"15min":1.0329999999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.66200000000000003,"15min":0.67800000000000005},"min":{"1min":0.40999999999999998,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78900000000000003,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.629}]},{"osd":7,"last update":"Sun Oct 1 09:14:04 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.67500000000000004,"15min":0.69599999999999995},"min":{"1min":0.38900000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.77400000000000002,"5min":1.081,"15min":1.081},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.67100000000000004,"15min":0.69199999999999995},"min":{"1min":0.42299999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80600000000000005,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.60599999999999998}]}]},{"osd":2,"up_from":99,"seq":425201762325,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32584,"kb_used_data":1088,"kb_used_omap":11,"kb_used_meta":31476,"kb_avail":93712568,"statfs":{"total":95995035648,"available":95961669632,"internally_reserved":0,"allocated":1114112,"data_stored":593360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":32232311},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.28199999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.372,"5min":0.372,"15min":0.372},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.316}]},{"osd":3,"last update":"Sun Oct 1 09:14:13 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.37,"15min":0.37},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.27000000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.40799999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.45100000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.503},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.47999999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.57699999999999996}]}]},{"osd":3,"up_from":104,"seq":446676598798,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28340,"kb_used_data":1068,"kb_used_omap":3,"kb_used_meta":27260,"kb_avail":93716812,"statfs":{"total":95995035648,"available":95966015488,"internally_reserved":0,"allocated":1093632,"data_stored":590233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":27914983},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26100000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38900000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32000000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58099999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64400000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72499999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78200000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019048,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298164,"kb_used_data":1068,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446988,"statfs":{"total":95995035648,"available":95689715712,"internally_reserved":0,"allocated":1093632,"data_stored":590580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:14:00 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.55200000000000005,"15min":0.57499999999999996},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.73399999999999999,"5min":0.746,"15min":0.746},"last":0.625},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.56799999999999995,"15min":0.57399999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.70399999999999996,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.52900000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.63900000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:13:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.66100000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:12:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56000000000000005,"15min":0.56699999999999995},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.71599999999999997,"5min":0.81699999999999995,"15min":5.6429999999999998},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56499999999999995,"15min":0.53700000000000003},"min":{"1min":0.41199999999999998,"5min":0.36099999999999999,"15min":0.34000000000000002},"max":{"1min":0.68799999999999994,"5min":0.86799999999999999,"15min":1.1279999999999999},"last":0.70299999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.47399999999999998,"15min":0.443},"min":{"1min":0.35699999999999998,"5min":0.317,"15min":0.26300000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.496,"15min":0.45800000000000002},"min":{"1min":0.39100000000000001,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.74299999999999999},"last":0.47799999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:13:37 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.52400000000000002,"15min":0.497},"min":{"1min":0.40699999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.77800000000000002,"5min":0.79500000000000004,"15min":0.84799999999999998},"last":0.435},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.57799999999999996,"15min":0.55000000000000004},"min":{"1min":0.48399999999999999,"5min":0.40000000000000002,"15min":0.33700000000000002},"max":{"1min":0.75600000000000001,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.57299999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:13:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.55000000000000004,"15min":0.54200000000000004},"min":{"1min":0.34100000000000003,"5min":0.33000000000000002,"15min":0.317},"max":{"1min":0.74099999999999999,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.626,"5min":0.58299999999999996,"15min":0.56499999999999995},"min":{"1min":0.41999999999999998,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.873},"last":0.71499999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822821,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298148,"kb_used_data":1052,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447004,"statfs":{"total":95995035648,"available":95689732096,"internally_reserved":0,"allocated":1077248,"data_stored":586028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:14:01 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.61599999999999999,"15min":0.63100000000000001},"min":{"1min":0.39800000000000002,"5min":0.309,"15min":0.309},"max":{"1min":2.6179999999999999,"5min":2.6179999999999999,"15min":2.6179999999999999},"last":0.503},{"interface":"front","average":{"1min":0.85099999999999998,"5min":0.63900000000000001,"15min":0.61799999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.7789999999999999,"5min":7.7789999999999999,"15min":7.7789999999999999},"last":0.34300000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:14:10 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.57799999999999996,"15min":0.54000000000000004},"min":{"1min":0.441,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.5209999999999999,"5min":3.5209999999999999,"15min":3.5209999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.56999999999999995,"15min":0.55400000000000005},"min":{"1min":0.38200000000000001,"5min":0.376,"15min":0.376},"max":{"1min":3.4260000000000002,"5min":3.4260000000000002,"15min":3.4260000000000002},"last":0.42599999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:13:41 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":2.5430000000000001,"5min":2.5430000000000001,"15min":2.5430000000000001},"last":0.378},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.5190000000000001,"5min":2.5190000000000001,"15min":2.5190000000000001},"last":0.47999999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:12:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.58999999999999997,"15min":0.55100000000000005},"min":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.27000000000000002},"max":{"1min":0.78100000000000003,"5min":5.5700000000000003,"15min":5.5700000000000003},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.57099999999999995,"15min":0.54600000000000004},"min":{"1min":0.44600000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.54900000000000004}]},{"osd":4,"last update":"Sun Oct 1 09:14:07 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.496,"15min":0.46300000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.25800000000000001},"max":{"1min":0.64500000000000002,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.52900000000000003,"15min":0.503},"min":{"1min":0.316,"5min":0.316,"15min":0.28100000000000003},"max":{"1min":0.61699999999999999,"5min":0.85799999999999998,"15min":0.86399999999999999},"last":0.41599999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:13:29 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.54400000000000004,"15min":0.52300000000000002},"min":{"1min":0.38100000000000001,"5min":0.32300000000000001,"15min":0.28599999999999998},"max":{"1min":0.65800000000000003,"5min":0.81799999999999995,"15min":0.88},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.58599999999999997,"15min":0.53400000000000003},"min":{"1min":0.35199999999999998,"5min":0.33800000000000002,"15min":0.29399999999999998},"max":{"1min":0.71999999999999997,"5min":0.83399999999999996,"15min":0.85799999999999998},"last":0.61899999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:13:52 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60199999999999998,"15min":0.57099999999999995},"min":{"1min":0.40799999999999997,"5min":0.34300000000000003,"15min":0.32300000000000001},"max":{"1min":0.79400000000000004,"5min":0.873,"15min":0.89300000000000002},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.59499999999999997,"15min":0.56599999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.32000000000000001},"max":{"1min":0.75900000000000001,"5min":0.86599999999999999,"15min":0.88500000000000001},"last":0.56899999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:14:16.927 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-01T09:14:16.928 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd dump 2023-10-01T09:14:17.275 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:17.733 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:17 smithi138 ceph-mon[185886]: from='client.34614 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:14:17.734 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:17 smithi138 ceph-mon[192513]: from='client.34614 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:14:17.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:17 smithi190 ceph-mon[160435]: from='client.34614 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:14:18.733 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:18 smithi138 ceph-mon[185886]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:18.733 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:18 smithi138 ceph-mon[192513]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:18.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:18 smithi190 ceph-mon[160435]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:19.121 INFO:teuthology.orchestra.run.smithi138.stdout:epoch 105 2023-10-01T09:14:19.121 INFO:teuthology.orchestra.run.smithi138.stdout:fsid 465870a4-6038-11ee-8db5-212e2dc638e7 2023-10-01T09:14:19.121 INFO:teuthology.orchestra.run.smithi138.stdout:created 2023-10-01T08:56:55.039846+0000 2023-10-01T09:14:19.122 INFO:teuthology.orchestra.run.smithi138.stdout:modified 2023-10-01T09:13:13.525285+0000 2023-10-01T09:14:19.122 INFO:teuthology.orchestra.run.smithi138.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-10-01T09:14:19.122 INFO:teuthology.orchestra.run.smithi138.stdout:crush_version 22 2023-10-01T09:14:19.122 INFO:teuthology.orchestra.run.smithi138.stdout:full_ratio 0.95 2023-10-01T09:14:19.122 INFO:teuthology.orchestra.run.smithi138.stdout:backfillfull_ratio 0.9 2023-10-01T09:14:19.122 INFO:teuthology.orchestra.run.smithi138.stdout:nearfull_ratio 0.85 2023-10-01T09:14:19.122 INFO:teuthology.orchestra.run.smithi138.stdout:require_min_compat_client luminous 2023-10-01T09:14:19.122 INFO:teuthology.orchestra.run.smithi138.stdout:min_compat_client luminous 2023-10-01T09:14:19.122 INFO:teuthology.orchestra.run.smithi138.stdout:require_osd_release pacific 2023-10-01T09:14:19.122 INFO:teuthology.orchestra.run.smithi138.stdout:stretch_mode_enabled false 2023-10-01T09:14:19.123 INFO:teuthology.orchestra.run.smithi138.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 82 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr,mgr_devicehealth read_balance_score 7.89 2023-10-01T09:14:19.123 INFO:teuthology.orchestra.run.smithi138.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 77 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2023-10-01T09:14:19.123 INFO:teuthology.orchestra.run.smithi138.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-10-01T09:14:19.123 INFO:teuthology.orchestra.run.smithi138.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 60 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-10-01T09:14:19.123 INFO:teuthology.orchestra.run.smithi138.stdout:max_osd 8 2023-10-01T09:14:19.123 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0 up in weight 1 up_from 89 up_thru 104 down_at 86 last_clean_interval [8,85) [v2:172.21.15.138:6802/2668617192,v1:172.21.15.138:6803/2668617192] [v2:172.21.15.138:6804/2668617192,v1:172.21.15.138:6805/2668617192] exists,up 12f71e77-6099-48ee-a89e-e931e000e9fa 2023-10-01T09:14:19.123 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1 up in weight 1 up_from 94 up_thru 104 down_at 91 last_clean_interval [13,90) [v2:172.21.15.138:6810/2021985596,v1:172.21.15.138:6811/2021985596] [v2:172.21.15.138:6812/2021985596,v1:172.21.15.138:6813/2021985596] exists,up 525b5e81-4fa5-4b10-9147-83fcb549ca30 2023-10-01T09:14:19.123 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2 up in weight 1 up_from 99 up_thru 104 down_at 96 last_clean_interval [18,95) [v2:172.21.15.138:6818/4290419390,v1:172.21.15.138:6819/4290419390] [v2:172.21.15.138:6820/4290419390,v1:172.21.15.138:6821/4290419390] exists,up 97443e05-2513-4832-bb54-f65493df5967 2023-10-01T09:14:19.123 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3 up in weight 1 up_from 104 up_thru 104 down_at 101 last_clean_interval [25,100) [v2:172.21.15.138:6826/244903146,v1:172.21.15.138:6827/244903146] [v2:172.21.15.138:6828/244903146,v1:172.21.15.138:6829/244903146] exists,up ac51c165-e007-4390-b95b-26eef984d637 2023-10-01T09:14:19.124 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4 up in weight 1 up_from 30 up_thru 104 down_at 0 last_clean_interval [0,0) [v2:172.21.15.190:6800/743095145,v1:172.21.15.190:6801/743095145] [v2:172.21.15.190:6802/743095145,v1:172.21.15.190:6803/743095145] exists,up 4bef630c-bff4-41a3-a932-de0501541f78 2023-10-01T09:14:19.124 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5 up in weight 1 up_from 36 up_thru 104 down_at 0 last_clean_interval [0,0) [v2:172.21.15.190:6808/4080051847,v1:172.21.15.190:6809/4080051847] [v2:172.21.15.190:6810/4080051847,v1:172.21.15.190:6811/4080051847] exists,up ac6e16ed-681d-4e9d-87a3-d20d50e01437 2023-10-01T09:14:19.124 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6 up in weight 1 up_from 42 up_thru 101 down_at 0 last_clean_interval [0,0) [v2:172.21.15.190:6816/797186189,v1:172.21.15.190:6817/797186189] [v2:172.21.15.190:6818/797186189,v1:172.21.15.190:6819/797186189] exists,up 7a6f65a5-16dc-45de-854d-751d9151e85b 2023-10-01T09:14:19.124 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7 up in weight 1 up_from 48 up_thru 104 down_at 0 last_clean_interval [0,0) [v2:172.21.15.190:6824/185238034,v1:172.21.15.190:6825/185238034] [v2:172.21.15.190:6826/185238034,v1:172.21.15.190:6827/185238034] exists,up 09a5ad5d-d58f-4fcf-8b09-10f395bd01f0 2023-10-01T09:14:19.124 INFO:teuthology.orchestra.run.smithi138.stdout:pg_upmap_items 4.f [1,2] 2023-10-01T09:14:19.124 INFO:teuthology.orchestra.run.smithi138.stdout:pg_upmap_items 4.13 [1,2] 2023-10-01T09:14:19.124 INFO:teuthology.orchestra.run.smithi138.stdout:pg_upmap_items 4.1f [1,2] 2023-10-01T09:14:19.124 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:6801/3849457598 expires 2023-10-02T09:07:55.156656+0000 2023-10-01T09:14:19.124 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:0/319567031 expires 2023-10-02T09:07:55.156656+0000 2023-10-01T09:14:19.124 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:0/2091838076 expires 2023-10-02T09:07:55.156656+0000 2023-10-01T09:14:19.125 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:0/1464782684 expires 2023-10-02T09:07:55.156656+0000 2023-10-01T09:14:19.125 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:0/25071367 expires 2023-10-02T09:07:55.156656+0000 2023-10-01T09:14:19.125 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:0/3940341494 expires 2023-10-02T08:57:39.299924+0000 2023-10-01T09:14:19.125 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.190:0/3151728630 expires 2023-10-02T09:07:17.185267+0000 2023-10-01T09:14:19.125 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:6801/3165627846 expires 2023-10-02T08:57:18.125987+0000 2023-10-01T09:14:19.125 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:6800/3165627846 expires 2023-10-02T08:57:18.125987+0000 2023-10-01T09:14:19.125 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:0/3256141027 expires 2023-10-02T09:07:55.156656+0000 2023-10-01T09:14:19.125 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.190:6835/2242576319 expires 2023-10-02T09:07:17.185267+0000 2023-10-01T09:14:19.125 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.190:0/2286153897 expires 2023-10-02T09:07:17.185267+0000 2023-10-01T09:14:19.125 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:0/2575047401 expires 2023-10-02T09:01:44.817357+0000 2023-10-01T09:14:19.126 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:6801/62009114 expires 2023-10-02T08:57:39.299924+0000 2023-10-01T09:14:19.126 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.190:0/2209516562 expires 2023-10-02T09:07:17.185267+0000 2023-10-01T09:14:19.126 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.190:0/3337553974 expires 2023-10-02T09:07:17.185267+0000 2023-10-01T09:14:19.126 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:0/2018177063 expires 2023-10-02T08:57:18.125987+0000 2023-10-01T09:14:19.126 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:0/523071753 expires 2023-10-02T09:07:55.156656+0000 2023-10-01T09:14:19.126 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:0/2054656381 expires 2023-10-02T08:57:39.299924+0000 2023-10-01T09:14:19.126 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:0/1836785572 expires 2023-10-02T09:06:07.582189+0000 2023-10-01T09:14:19.126 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:6800/3849457598 expires 2023-10-02T09:07:55.156656+0000 2023-10-01T09:14:19.126 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:6800/62009114 expires 2023-10-02T08:57:39.299924+0000 2023-10-01T09:14:19.127 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.190:6834/2242576319 expires 2023-10-02T09:07:17.185267+0000 2023-10-01T09:14:19.127 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.190:0/299211527 expires 2023-10-02T09:07:17.185267+0000 2023-10-01T09:14:19.127 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:6801/4033793848 expires 2023-10-02T09:01:44.817357+0000 2023-10-01T09:14:19.127 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:0/3533950111 expires 2023-10-02T09:06:07.582189+0000 2023-10-01T09:14:19.127 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.190:0/710760207 expires 2023-10-02T09:07:17.185267+0000 2023-10-01T09:14:19.127 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:6800/4033793848 expires 2023-10-02T09:01:44.817357+0000 2023-10-01T09:14:19.127 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:0/3675584653 expires 2023-10-02T08:57:18.125987+0000 2023-10-01T09:14:19.127 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:6800/974662983 expires 2023-10-02T09:06:07.582189+0000 2023-10-01T09:14:19.127 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:6801/974662983 expires 2023-10-02T09:06:07.582189+0000 2023-10-01T09:14:19.127 INFO:teuthology.orchestra.run.smithi138.stdout:blocklist 172.21.15.138:0/2334114390 expires 2023-10-02T09:01:44.817357+0000 2023-10-01T09:14:19.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:19 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1417029413' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:14:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1417029413' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:14:19.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:19 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1417029413' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:14:19.838 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 89 up_thru 104 down_at 86 last_clean_interval [8,85) [v2:172.21.15.138:6802/2668617192,v1:172.21.15.138:6803/2668617192] [v2:172.21.15.138:6804/2668617192,v1:172.21.15.138:6805/2668617192] exists,up 12f71e77-6099-48ee-a89e-e931e000e9fa', 'osd.1 up in weight 1 up_from 94 up_thru 104 down_at 91 last_clean_interval [13,90) [v2:172.21.15.138:6810/2021985596,v1:172.21.15.138:6811/2021985596] [v2:172.21.15.138:6812/2021985596,v1:172.21.15.138:6813/2021985596] exists,up 525b5e81-4fa5-4b10-9147-83fcb549ca30', 'osd.2 up in weight 1 up_from 99 up_thru 104 down_at 96 last_clean_interval [18,95) [v2:172.21.15.138:6818/4290419390,v1:172.21.15.138:6819/4290419390] [v2:172.21.15.138:6820/4290419390,v1:172.21.15.138:6821/4290419390] exists,up 97443e05-2513-4832-bb54-f65493df5967', 'osd.3 up in weight 1 up_from 104 up_thru 104 down_at 101 last_clean_interval [25,100) [v2:172.21.15.138:6826/244903146,v1:172.21.15.138:6827/244903146] [v2:172.21.15.138:6828/244903146,v1:172.21.15.138:6829/244903146] exists,up ac51c165-e007-4390-b95b-26eef984d637', 'osd.4 up in weight 1 up_from 30 up_thru 104 down_at 0 last_clean_interval [0,0) [v2:172.21.15.190:6800/743095145,v1:172.21.15.190:6801/743095145] [v2:172.21.15.190:6802/743095145,v1:172.21.15.190:6803/743095145] exists,up 4bef630c-bff4-41a3-a932-de0501541f78', 'osd.5 up in weight 1 up_from 36 up_thru 104 down_at 0 last_clean_interval [0,0) [v2:172.21.15.190:6808/4080051847,v1:172.21.15.190:6809/4080051847] [v2:172.21.15.190:6810/4080051847,v1:172.21.15.190:6811/4080051847] exists,up ac6e16ed-681d-4e9d-87a3-d20d50e01437', 'osd.6 up in weight 1 up_from 42 up_thru 101 down_at 0 last_clean_interval [0,0) [v2:172.21.15.190:6816/797186189,v1:172.21.15.190:6817/797186189] [v2:172.21.15.190:6818/797186189,v1:172.21.15.190:6819/797186189] exists,up 7a6f65a5-16dc-45de-854d-751d9151e85b', 'osd.7 up in weight 1 up_from 48 up_thru 104 down_at 0 last_clean_interval [0,0) [v2:172.21.15.190:6824/185238034,v1:172.21.15.190:6825/185238034] [v2:172.21.15.190:6826/185238034,v1:172.21.15.190:6827/185238034] exists,up 09a5ad5d-d58f-4fcf-8b09-10f395bd01f0'] 2023-10-01T09:14:19.839 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon mon.a config show 2023-10-01T09:14:20.187 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:20.650 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:20 smithi138 ceph-mon[185886]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:20.651 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:20 smithi138 ceph-mon[192513]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:20.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:20 smithi190 ceph-mon[160435]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:21.822 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:14:21.822 INFO:teuthology.orchestra.run.smithi138.stdout: "name": "mon.a", 2023-10-01T09:14:21.822 INFO:teuthology.orchestra.run.smithi138.stdout: "cluster": "ceph", 2023-10-01T09:14:21.823 INFO:teuthology.orchestra.run.smithi138.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-10-01T09:14:21.823 INFO:teuthology.orchestra.run.smithi138.stdout: "admin_socket_mode": "", 2023-10-01T09:14:21.823 INFO:teuthology.orchestra.run.smithi138.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-10-01T09:14:21.823 INFO:teuthology.orchestra.run.smithi138.stdout: "auth_client_required": "cephx, none", 2023-10-01T09:14:21.823 INFO:teuthology.orchestra.run.smithi138.stdout: "auth_cluster_required": "cephx", 2023-10-01T09:14:21.823 INFO:teuthology.orchestra.run.smithi138.stdout: "auth_debug": "true", 2023-10-01T09:14:21.823 INFO:teuthology.orchestra.run.smithi138.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-10-01T09:14:21.823 INFO:teuthology.orchestra.run.smithi138.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-10-01T09:14:21.823 INFO:teuthology.orchestra.run.smithi138.stdout: "auth_service_required": "cephx", 2023-10-01T09:14:21.823 INFO:teuthology.orchestra.run.smithi138.stdout: "auth_service_ticket_ttl": "240.000000", 2023-10-01T09:14:21.823 INFO:teuthology.orchestra.run.smithi138.stdout: "auth_supported": "", 2023-10-01T09:14:21.824 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_aio": "true", 2023-10-01T09:14:21.824 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_aio_max_queue_depth": "1024", 2023-10-01T09:14:21.824 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_aio_poll_ms": "250", 2023-10-01T09:14:21.824 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_aio_reap_max": "16", 2023-10-01T09:14:21.824 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_async_discard": "false", 2023-10-01T09:14:21.824 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_block_size": "4096", 2023-10-01T09:14:21.824 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_debug_aio": "false", 2023-10-01T09:14:21.824 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-10-01T09:14:21.824 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-10-01T09:14:21.824 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_debug_inflight_ios": "false", 2023-10-01T09:14:21.824 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_enable_discard": "false", 2023-10-01T09:14:21.825 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_flock_retry": "3", 2023-10-01T09:14:21.825 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_flock_retry_interval": "0.100000", 2023-10-01T09:14:21.825 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_inject_crash": "0", 2023-10-01T09:14:21.825 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_inject_crash_flush_delay": "2", 2023-10-01T09:14:21.825 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_ioring": "false", 2023-10-01T09:14:21.825 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_ioring_hipri": "false", 2023-10-01T09:14:21.825 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_ioring_sqthread_poll": "false", 2023-10-01T09:14:21.825 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-10-01T09:14:21.825 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_read_buffer_alignment": "4096", 2023-10-01T09:14:21.825 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-10-01T09:14:21.825 INFO:teuthology.orchestra.run.smithi138.stdout: "bdev_type": "", 2023-10-01T09:14:21.825 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_alloc_size": "1048576", 2023-10-01T09:14:21.826 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_allocator": "hybrid", 2023-10-01T09:14:21.826 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_buffered_io": "true", 2023-10-01T09:14:21.826 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_check_for_zeros": "false", 2023-10-01T09:14:21.826 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_check_volume_selector_often": "false", 2023-10-01T09:14:21.826 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-10-01T09:14:21.826 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_compact_log_sync": "false", 2023-10-01T09:14:21.826 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-10-01T09:14:21.826 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-10-01T09:14:21.826 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_log_compact_min_size": "16777216", 2023-10-01T09:14:21.826 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_log_replay_check_allocations": "true", 2023-10-01T09:14:21.826 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_max_log_runway": "4194304", 2023-10-01T09:14:21.827 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_max_prefetch": "1048576", 2023-10-01T09:14:21.827 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_min_flush_size": "524288", 2023-10-01T09:14:21.827 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_min_log_runway": "1048576", 2023-10-01T09:14:21.827 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_replay_recovery": "false", 2023-10-01T09:14:21.827 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-10-01T09:14:21.827 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_shared_alloc_size": "65536", 2023-10-01T09:14:21.827 INFO:teuthology.orchestra.run.smithi138.stdout: "bluefs_sync_write": "false", 2023-10-01T09:14:21.827 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-10-01T09:14:21.827 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-10-01T09:14:21.827 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-10-01T09:14:21.827 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_allocation_from_file": "true", 2023-10-01T09:14:21.828 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_allocator": "hybrid", 2023-10-01T09:14:21.828 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-10-01T09:14:21.828 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-10-01T09:14:21.828 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-10-01T09:14:21.828 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-10-01T09:14:21.828 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-10-01T09:14:21.828 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-10-01T09:14:21.828 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_blobid_prealloc": "10240", 2023-10-01T09:14:21.828 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_block_create": "true", 2023-10-01T09:14:21.828 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_block_db_create": "false", 2023-10-01T09:14:21.828 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_block_db_path": "", 2023-10-01T09:14:21.828 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_block_db_size": "0", 2023-10-01T09:14:21.829 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_block_path": "", 2023-10-01T09:14:21.829 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_block_preallocate_file": "false", 2023-10-01T09:14:21.829 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_block_size": "107374182400", 2023-10-01T09:14:21.829 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_block_wal_create": "false", 2023-10-01T09:14:21.829 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_block_wal_path": "", 2023-10-01T09:14:21.829 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_block_wal_size": "100663296", 2023-10-01T09:14:21.829 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_bluefs": "true", 2023-10-01T09:14:21.829 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-10-01T09:14:21.829 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_bluefs_env_mirror": "false", 2023-10-01T09:14:21.829 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-10-01T09:14:21.829 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-10-01T09:14:21.829 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-10-01T09:14:21.830 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-10-01T09:14:21.830 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-10-01T09:14:21.830 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-10-01T09:14:21.830 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_autotune": "true", 2023-10-01T09:14:21.830 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-10-01T09:14:21.830 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-10-01T09:14:21.830 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-10-01T09:14:21.830 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-10-01T09:14:21.830 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_size": "0", 2023-10-01T09:14:21.830 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-10-01T09:14:21.831 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-10-01T09:14:21.831 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-10-01T09:14:21.831 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-10-01T09:14:21.831 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cache_type": "2q", 2023-10-01T09:14:21.831 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-10-01T09:14:21.831 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_clone_cow": "true", 2023-10-01T09:14:21.831 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_compression_algorithm": "snappy", 2023-10-01T09:14:21.831 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_compression_max_blob_size": "0", 2023-10-01T09:14:21.831 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-10-01T09:14:21.831 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-10-01T09:14:21.831 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_compression_min_blob_size": "0", 2023-10-01T09:14:21.832 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-10-01T09:14:21.832 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-10-01T09:14:21.832 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_compression_mode": "none", 2023-10-01T09:14:21.832 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-10-01T09:14:21.832 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_csum_type": "crc32c", 2023-10-01T09:14:21.832 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_enforce_settings": "default", 2023-10-01T09:14:21.832 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_freelist": "false", 2023-10-01T09:14:21.832 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_fsck_abort": "false", 2023-10-01T09:14:21.832 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-10-01T09:14:21.832 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_inject_bug21040": "false", 2023-10-01T09:14:21.833 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-10-01T09:14:21.833 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_inject_read_err": "false", 2023-10-01T09:14:21.833 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_legacy_omap": "false", 2023-10-01T09:14:21.833 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-10-01T09:14:21.833 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-10-01T09:14:21.833 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-10-01T09:14:21.833 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-10-01T09:14:21.833 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_prefill": "0.000000", 2023-10-01T09:14:21.833 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-10-01T09:14:21.833 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-10-01T09:14:21.834 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-10-01T09:14:21.834 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_small_allocations": "0", 2023-10-01T09:14:21.834 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-10-01T09:14:21.834 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_default_buffered_read": "true", 2023-10-01T09:14:21.834 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_default_buffered_write": "false", 2023-10-01T09:14:21.834 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_deferred_batch_ops": "0", 2023-10-01T09:14:21.834 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-10-01T09:14:21.834 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-10-01T09:14:21.834 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-10-01T09:14:21.835 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-10-01T09:14:21.835 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-10-01T09:14:21.835 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-10-01T09:14:21.835 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-10-01T09:14:21.835 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_fail_eio": "false", 2023-10-01T09:14:21.835 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-10-01T09:14:21.835 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-10-01T09:14:21.835 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-10-01T09:14:21.835 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-10-01T09:14:21.835 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_fsck_on_mkfs": "true", 2023-10-01T09:14:21.836 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-10-01T09:14:21.836 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_fsck_on_mount": "false", 2023-10-01T09:14:21.836 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-10-01T09:14:21.836 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_fsck_on_umount": "false", 2023-10-01T09:14:21.836 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-10-01T09:14:21.836 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-10-01T09:14:21.836 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-10-01T09:14:21.836 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-10-01T09:14:21.836 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-10-01T09:14:21.836 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-10-01T09:14:21.836 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-10-01T09:14:21.837 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-10-01T09:14:21.837 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_ignore_data_csum": "false", 2023-10-01T09:14:21.837 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-10-01T09:14:21.837 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_kvbackend": "rocksdb", 2023-10-01T09:14:21.837 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-10-01T09:14:21.837 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-10-01T09:14:21.837 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_log_op_age": "5.000000", 2023-10-01T09:14:21.837 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_max_alloc_size": "0", 2023-10-01T09:14:21.837 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_max_blob_size": "0", 2023-10-01T09:14:21.837 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-10-01T09:14:21.838 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-10-01T09:14:21.838 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_max_defer_interval": "3.000000", 2023-10-01T09:14:21.838 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_max_deferred_txc": "32", 2023-10-01T09:14:21.838 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_min_alloc_size": "0", 2023-10-01T09:14:21.838 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-10-01T09:14:21.838 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-10-01T09:14:21.838 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_nid_prealloc": "1024", 2023-10-01T09:14:21.838 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_prefer_deferred_size": "0", 2023-10-01T09:14:21.838 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-10-01T09:14:21.838 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-10-01T09:14:21.838 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_qfsck_on_mount": "true", 2023-10-01T09:14:21.838 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_retry_disk_reads": "3", 2023-10-01T09:14:21.839 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_rocksdb_cf": "true", 2023-10-01T09:14:21.839 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-10-01T09:14:21.839 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-10-01T09:14:21.839 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_rocksdb_options_annex": "", 2023-10-01T09:14:21.839 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_spdk_coremask": "0x1", 2023-10-01T09:14:21.839 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_spdk_io_sleep": "5", 2023-10-01T09:14:21.839 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_spdk_max_io_completion": "0", 2023-10-01T09:14:21.839 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_spdk_mem": "512", 2023-10-01T09:14:21.839 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_sync_submit_transaction": "false", 2023-10-01T09:14:21.839 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_throttle_bytes": "67108864", 2023-10-01T09:14:21.839 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_throttle_cost_per_io": "0", 2023-10-01T09:14:21.840 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-10-01T09:14:21.840 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-10-01T09:14:21.840 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-10-01T09:14:21.840 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-10-01T09:14:21.840 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_tracing": "false", 2023-10-01T09:14:21.840 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-10-01T09:14:21.840 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-10-01T09:14:21.840 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_volume_selection_reserved": "0", 2023-10-01T09:14:21.840 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-10-01T09:14:21.840 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-10-01T09:14:21.840 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-10-01T09:14:21.841 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-10-01T09:14:21.841 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-10-01T09:14:21.841 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-10-01T09:14:21.841 INFO:teuthology.orchestra.run.smithi138.stdout: "bluestore_zero_block_detection": "false", 2023-10-01T09:14:21.841 INFO:teuthology.orchestra.run.smithi138.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-10-01T09:14:21.841 INFO:teuthology.orchestra.run.smithi138.stdout: "cephfs_mirror_action_update_interval": "2", 2023-10-01T09:14:21.841 INFO:teuthology.orchestra.run.smithi138.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-10-01T09:14:21.841 INFO:teuthology.orchestra.run.smithi138.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-10-01T09:14:21.841 INFO:teuthology.orchestra.run.smithi138.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-10-01T09:14:21.841 INFO:teuthology.orchestra.run.smithi138.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-10-01T09:14:21.842 INFO:teuthology.orchestra.run.smithi138.stdout: "cephfs_mirror_mount_timeout": "10", 2023-10-01T09:14:21.842 INFO:teuthology.orchestra.run.smithi138.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-10-01T09:14:21.842 INFO:teuthology.orchestra.run.smithi138.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-10-01T09:14:21.842 INFO:teuthology.orchestra.run.smithi138.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-10-01T09:14:21.842 INFO:teuthology.orchestra.run.smithi138.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-10-01T09:14:21.842 INFO:teuthology.orchestra.run.smithi138.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-10-01T09:14:21.842 INFO:teuthology.orchestra.run.smithi138.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-10-01T09:14:21.842 INFO:teuthology.orchestra.run.smithi138.stdout: "cephx_cluster_require_signatures": "false", 2023-10-01T09:14:21.842 INFO:teuthology.orchestra.run.smithi138.stdout: "cephx_cluster_require_version": "2", 2023-10-01T09:14:21.842 INFO:teuthology.orchestra.run.smithi138.stdout: "cephx_require_signatures": "false", 2023-10-01T09:14:21.842 INFO:teuthology.orchestra.run.smithi138.stdout: "cephx_require_version": "2", 2023-10-01T09:14:21.843 INFO:teuthology.orchestra.run.smithi138.stdout: "cephx_service_require_signatures": "false", 2023-10-01T09:14:21.843 INFO:teuthology.orchestra.run.smithi138.stdout: "cephx_service_require_version": "2", 2023-10-01T09:14:21.843 INFO:teuthology.orchestra.run.smithi138.stdout: "cephx_sign_messages": "true", 2023-10-01T09:14:21.843 INFO:teuthology.orchestra.run.smithi138.stdout: "chdir": "", 2023-10-01T09:14:21.843 INFO:teuthology.orchestra.run.smithi138.stdout: "client_acl_type": "", 2023-10-01T09:14:21.843 INFO:teuthology.orchestra.run.smithi138.stdout: "client_asio_thread_count": "2", 2023-10-01T09:14:21.843 INFO:teuthology.orchestra.run.smithi138.stdout: "client_cache_mid": "0.750000", 2023-10-01T09:14:21.843 INFO:teuthology.orchestra.run.smithi138.stdout: "client_cache_size": "16384", 2023-10-01T09:14:21.843 INFO:teuthology.orchestra.run.smithi138.stdout: "client_caps_release_delay": "5", 2023-10-01T09:14:21.843 INFO:teuthology.orchestra.run.smithi138.stdout: "client_check_pool_perm": "true", 2023-10-01T09:14:21.843 INFO:teuthology.orchestra.run.smithi138.stdout: "client_collect_and_send_global_metrics": "false", 2023-10-01T09:14:21.844 INFO:teuthology.orchestra.run.smithi138.stdout: "client_debug_force_sync_read": "false", 2023-10-01T09:14:21.844 INFO:teuthology.orchestra.run.smithi138.stdout: "client_debug_getattr_caps": "false", 2023-10-01T09:14:21.844 INFO:teuthology.orchestra.run.smithi138.stdout: "client_debug_inject_tick_delay": "0", 2023-10-01T09:14:21.844 INFO:teuthology.orchestra.run.smithi138.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-10-01T09:14:21.844 INFO:teuthology.orchestra.run.smithi138.stdout: "client_die_on_failed_remount": "false", 2023-10-01T09:14:21.844 INFO:teuthology.orchestra.run.smithi138.stdout: "client_dirsize_rbytes": "true", 2023-10-01T09:14:21.844 INFO:teuthology.orchestra.run.smithi138.stdout: "client_force_lazyio": "false", 2023-10-01T09:14:21.844 INFO:teuthology.orchestra.run.smithi138.stdout: "client_fs": "", 2023-10-01T09:14:21.844 INFO:teuthology.orchestra.run.smithi138.stdout: "client_inject_fixed_oldest_tid": "false", 2023-10-01T09:14:21.844 INFO:teuthology.orchestra.run.smithi138.stdout: "client_inject_release_failure": "false", 2023-10-01T09:14:21.844 INFO:teuthology.orchestra.run.smithi138.stdout: "client_max_inline_size": "4096", 2023-10-01T09:14:21.844 INFO:teuthology.orchestra.run.smithi138.stdout: "client_max_retries_on_remount_failure": "5", 2023-10-01T09:14:21.845 INFO:teuthology.orchestra.run.smithi138.stdout: "client_mds_namespace": "", 2023-10-01T09:14:21.845 INFO:teuthology.orchestra.run.smithi138.stdout: "client_metadata": "", 2023-10-01T09:14:21.845 INFO:teuthology.orchestra.run.smithi138.stdout: "client_mount_gid": "-1", 2023-10-01T09:14:21.845 INFO:teuthology.orchestra.run.smithi138.stdout: "client_mount_timeout": "300", 2023-10-01T09:14:21.845 INFO:teuthology.orchestra.run.smithi138.stdout: "client_mount_uid": "-1", 2023-10-01T09:14:21.845 INFO:teuthology.orchestra.run.smithi138.stdout: "client_mountpoint": "/", 2023-10-01T09:14:21.845 INFO:teuthology.orchestra.run.smithi138.stdout: "client_notify_timeout": "10", 2023-10-01T09:14:21.845 INFO:teuthology.orchestra.run.smithi138.stdout: "client_oc": "true", 2023-10-01T09:14:21.845 INFO:teuthology.orchestra.run.smithi138.stdout: "client_oc_max_dirty": "104857600", 2023-10-01T09:14:21.845 INFO:teuthology.orchestra.run.smithi138.stdout: "client_oc_max_dirty_age": "5.000000", 2023-10-01T09:14:21.845 INFO:teuthology.orchestra.run.smithi138.stdout: "client_oc_max_objects": "1000", 2023-10-01T09:14:21.846 INFO:teuthology.orchestra.run.smithi138.stdout: "client_oc_size": "209715200", 2023-10-01T09:14:21.846 INFO:teuthology.orchestra.run.smithi138.stdout: "client_oc_target_dirty": "8388608", 2023-10-01T09:14:21.846 INFO:teuthology.orchestra.run.smithi138.stdout: "client_permissions": "true", 2023-10-01T09:14:21.846 INFO:teuthology.orchestra.run.smithi138.stdout: "client_quota": "true", 2023-10-01T09:14:21.846 INFO:teuthology.orchestra.run.smithi138.stdout: "client_quota_df": "true", 2023-10-01T09:14:21.846 INFO:teuthology.orchestra.run.smithi138.stdout: "client_readahead_max_bytes": "0", 2023-10-01T09:14:21.846 INFO:teuthology.orchestra.run.smithi138.stdout: "client_readahead_max_periods": "4", 2023-10-01T09:14:21.846 INFO:teuthology.orchestra.run.smithi138.stdout: "client_readahead_min": "131072", 2023-10-01T09:14:21.846 INFO:teuthology.orchestra.run.smithi138.stdout: "client_reconnect_stale": "false", 2023-10-01T09:14:21.846 INFO:teuthology.orchestra.run.smithi138.stdout: "client_shutdown_timeout": "30", 2023-10-01T09:14:21.846 INFO:teuthology.orchestra.run.smithi138.stdout: "client_snapdir": ".snap", 2023-10-01T09:14:21.847 INFO:teuthology.orchestra.run.smithi138.stdout: "client_tick_interval": "1", 2023-10-01T09:14:21.847 INFO:teuthology.orchestra.run.smithi138.stdout: "client_trace": "", 2023-10-01T09:14:21.847 INFO:teuthology.orchestra.run.smithi138.stdout: "client_try_dentry_invalidate": "false", 2023-10-01T09:14:21.847 INFO:teuthology.orchestra.run.smithi138.stdout: "client_use_faked_inos": "false", 2023-10-01T09:14:21.847 INFO:teuthology.orchestra.run.smithi138.stdout: "client_use_random_mds": "false", 2023-10-01T09:14:21.847 INFO:teuthology.orchestra.run.smithi138.stdout: "clog_to_graylog": "false", 2023-10-01T09:14:21.847 INFO:teuthology.orchestra.run.smithi138.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-10-01T09:14:21.847 INFO:teuthology.orchestra.run.smithi138.stdout: "clog_to_graylog_port": "12201", 2023-10-01T09:14:21.847 INFO:teuthology.orchestra.run.smithi138.stdout: "clog_to_monitors": "default=true", 2023-10-01T09:14:21.847 INFO:teuthology.orchestra.run.smithi138.stdout: "clog_to_syslog": "false", 2023-10-01T09:14:21.847 INFO:teuthology.orchestra.run.smithi138.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-10-01T09:14:21.848 INFO:teuthology.orchestra.run.smithi138.stdout: "clog_to_syslog_level": "info", 2023-10-01T09:14:21.848 INFO:teuthology.orchestra.run.smithi138.stdout: "cluster_addr": "-", 2023-10-01T09:14:21.848 INFO:teuthology.orchestra.run.smithi138.stdout: "cluster_network": "", 2023-10-01T09:14:21.848 INFO:teuthology.orchestra.run.smithi138.stdout: "cluster_network_interface": "", 2023-10-01T09:14:21.848 INFO:teuthology.orchestra.run.smithi138.stdout: "compressor_zlib_isal": "false", 2023-10-01T09:14:21.848 INFO:teuthology.orchestra.run.smithi138.stdout: "compressor_zlib_level": "5", 2023-10-01T09:14:21.848 INFO:teuthology.orchestra.run.smithi138.stdout: "compressor_zlib_winsize": "-15", 2023-10-01T09:14:21.848 INFO:teuthology.orchestra.run.smithi138.stdout: "compressor_zstd_level": "1", 2023-10-01T09:14:21.848 INFO:teuthology.orchestra.run.smithi138.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf", 2023-10-01T09:14:21.848 INFO:teuthology.orchestra.run.smithi138.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-10-01T09:14:21.848 INFO:teuthology.orchestra.run.smithi138.stdout: "crimson_alien_op_num_threads": "6", 2023-10-01T09:14:21.848 INFO:teuthology.orchestra.run.smithi138.stdout: "crimson_alien_thread_cpu_cores": "", 2023-10-01T09:14:21.849 INFO:teuthology.orchestra.run.smithi138.stdout: "crimson_osd_obc_lru_size": "10", 2023-10-01T09:14:21.849 INFO:teuthology.orchestra.run.smithi138.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-10-01T09:14:21.849 INFO:teuthology.orchestra.run.smithi138.stdout: "crimson_seastar_smp": "1", 2023-10-01T09:14:21.849 INFO:teuthology.orchestra.run.smithi138.stdout: "crush_location": "", 2023-10-01T09:14:21.849 INFO:teuthology.orchestra.run.smithi138.stdout: "crush_location_hook": "", 2023-10-01T09:14:21.849 INFO:teuthology.orchestra.run.smithi138.stdout: "crush_location_hook_timeout": "10", 2023-10-01T09:14:21.849 INFO:teuthology.orchestra.run.smithi138.stdout: "daemonize": "false", 2023-10-01T09:14:21.849 INFO:teuthology.orchestra.run.smithi138.stdout: "daos_pool": "tank", 2023-10-01T09:14:21.849 INFO:teuthology.orchestra.run.smithi138.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-10-01T09:14:21.849 INFO:teuthology.orchestra.run.smithi138.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-10-01T09:14:21.849 INFO:teuthology.orchestra.run.smithi138.stdout: "dbstore_db_name_prefix": "dbstore", 2023-10-01T09:14:21.850 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_alienstore": "0/5", 2023-10-01T09:14:21.850 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_allow_any_pool_priority": "false", 2023-10-01T09:14:21.850 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_asok": "1/5", 2023-10-01T09:14:21.850 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_asok_assert_abort": "false", 2023-10-01T09:14:21.850 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_asserts_on_shutdown": "true", 2023-10-01T09:14:21.850 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_auth": "1/5", 2023-10-01T09:14:21.850 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_bdev": "1/3", 2023-10-01T09:14:21.850 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_bluefs": "1/5", 2023-10-01T09:14:21.850 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_bluestore": "1/5", 2023-10-01T09:14:21.850 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_buffer": "0/1", 2023-10-01T09:14:21.850 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_ceph_exporter": "1/5", 2023-10-01T09:14:21.851 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_cephfs_mirror": "0/5", 2023-10-01T09:14:21.851 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_cephsqlite": "0/5", 2023-10-01T09:14:21.851 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_client": "0/5", 2023-10-01T09:14:21.851 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_compressor": "1/5", 2023-10-01T09:14:21.851 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_context": "0/1", 2023-10-01T09:14:21.851 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_crush": "1/1", 2023-10-01T09:14:21.851 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_crypto": "1/5", 2023-10-01T09:14:21.851 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_cyanstore": "0/5", 2023-10-01T09:14:21.851 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_deliberately_leak_memory": "false", 2023-10-01T09:14:21.851 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_disable_randomized_ping": "false", 2023-10-01T09:14:21.852 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_dpdk": "1/5", 2023-10-01T09:14:21.852 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_eventtrace": "1/5", 2023-10-01T09:14:21.852 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_filer": "0/1", 2023-10-01T09:14:21.852 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_filestore": "1/3", 2023-10-01T09:14:21.852 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_finisher": "1/1", 2023-10-01T09:14:21.852 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_fuse": "1/5", 2023-10-01T09:14:21.852 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_heartbeat_testing_span": "0", 2023-10-01T09:14:21.852 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_heartbeatmap": "1/5", 2023-10-01T09:14:21.852 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_immutable_obj_cache": "0/5", 2023-10-01T09:14:21.852 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_javaclient": "1/5", 2023-10-01T09:14:21.852 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_journal": "1/3", 2023-10-01T09:14:21.852 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_journaler": "0/5", 2023-10-01T09:14:21.852 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_kstore": "1/5", 2023-10-01T09:14:21.853 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_leveldb": "4/5", 2023-10-01T09:14:21.853 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_lockdep": "0/1", 2023-10-01T09:14:21.853 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_mclock": "1/5", 2023-10-01T09:14:21.853 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_mds": "1/5", 2023-10-01T09:14:21.853 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_mds_balancer": "1/5", 2023-10-01T09:14:21.853 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_mds_locker": "1/5", 2023-10-01T09:14:21.853 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_mds_log": "1/5", 2023-10-01T09:14:21.853 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_mds_log_expire": "1/5", 2023-10-01T09:14:21.853 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_mds_migrator": "1/5", 2023-10-01T09:14:21.853 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_memstore": "1/5", 2023-10-01T09:14:21.853 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_mgr": "2/5", 2023-10-01T09:14:21.853 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_mgrc": "1/5", 2023-10-01T09:14:21.854 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_mon": "20/20", 2023-10-01T09:14:21.854 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_monc": "0/10", 2023-10-01T09:14:21.854 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_ms": "1/1", 2023-10-01T09:14:21.854 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_none": "0/5", 2023-10-01T09:14:21.854 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_objclass": "0/5", 2023-10-01T09:14:21.854 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_objectcacher": "0/5", 2023-10-01T09:14:21.854 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_objecter": "0/1", 2023-10-01T09:14:21.854 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_optracker": "0/5", 2023-10-01T09:14:21.854 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_osd": "1/5", 2023-10-01T09:14:21.854 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_paxos": "20/20", 2023-10-01T09:14:21.854 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_perfcounter": "1/5", 2023-10-01T09:14:21.855 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_prioritycache": "1/5", 2023-10-01T09:14:21.855 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_rados": "0/5", 2023-10-01T09:14:21.855 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_rbd": "0/5", 2023-10-01T09:14:21.855 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_rbd_mirror": "0/5", 2023-10-01T09:14:21.855 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_rbd_pwl": "0/5", 2023-10-01T09:14:21.855 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_rbd_replay": "0/5", 2023-10-01T09:14:21.855 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_refs": "0/0", 2023-10-01T09:14:21.855 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_reserver": "1/1", 2023-10-01T09:14:21.855 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_rgw": "1/5", 2023-10-01T09:14:21.855 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_rgw_access": "1/5", 2023-10-01T09:14:21.855 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_rgw_datacache": "1/5", 2023-10-01T09:14:21.856 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_rgw_dbstore": "1/5", 2023-10-01T09:14:21.856 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_rgw_flight": "1/5", 2023-10-01T09:14:21.856 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_rgw_sync": "1/5", 2023-10-01T09:14:21.856 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_rocksdb": "4/5", 2023-10-01T09:14:21.856 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_seastore": "0/5", 2023-10-01T09:14:21.856 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_seastore_backref": "0/5", 2023-10-01T09:14:21.856 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_seastore_cache": "0/5", 2023-10-01T09:14:21.856 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_seastore_cleaner": "0/5", 2023-10-01T09:14:21.856 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_seastore_device": "0/5", 2023-10-01T09:14:21.856 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_seastore_epm": "0/5", 2023-10-01T09:14:21.856 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-10-01T09:14:21.856 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_seastore_journal": "0/5", 2023-10-01T09:14:21.857 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_seastore_lba": "0/5", 2023-10-01T09:14:21.857 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_seastore_odata": "0/5", 2023-10-01T09:14:21.857 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_seastore_omap": "0/5", 2023-10-01T09:14:21.857 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_seastore_onode": "0/5", 2023-10-01T09:14:21.857 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_seastore_t": "0/5", 2023-10-01T09:14:21.857 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_seastore_tm": "0/5", 2023-10-01T09:14:21.857 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_striper": "0/1", 2023-10-01T09:14:21.857 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_test": "0/5", 2023-10-01T09:14:21.857 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_throttle": "1/1", 2023-10-01T09:14:21.857 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_timer": "0/1", 2023-10-01T09:14:21.857 INFO:teuthology.orchestra.run.smithi138.stdout: "debug_tp": "0/5", 2023-10-01T09:14:21.857 INFO:teuthology.orchestra.run.smithi138.stdout: "device_failure_prediction_mode": "none", 2023-10-01T09:14:21.858 INFO:teuthology.orchestra.run.smithi138.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-10-01T09:14:21.858 INFO:teuthology.orchestra.run.smithi138.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-10-01T09:14:21.858 INFO:teuthology.orchestra.run.smithi138.stdout: "err_to_graylog": "false", 2023-10-01T09:14:21.858 INFO:teuthology.orchestra.run.smithi138.stdout: "err_to_journald": "false", 2023-10-01T09:14:21.858 INFO:teuthology.orchestra.run.smithi138.stdout: "err_to_stderr": "true", 2023-10-01T09:14:21.858 INFO:teuthology.orchestra.run.smithi138.stdout: "err_to_syslog": "false", 2023-10-01T09:14:21.858 INFO:teuthology.orchestra.run.smithi138.stdout: "event_tracing": "false", 2023-10-01T09:14:21.858 INFO:teuthology.orchestra.run.smithi138.stdout: "exporter_addr": "0.0.0.0", 2023-10-01T09:14:21.858 INFO:teuthology.orchestra.run.smithi138.stdout: "exporter_http_port": "9926", 2023-10-01T09:14:21.858 INFO:teuthology.orchestra.run.smithi138.stdout: "exporter_prio_limit": "5", 2023-10-01T09:14:21.859 INFO:teuthology.orchestra.run.smithi138.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-10-01T09:14:21.859 INFO:teuthology.orchestra.run.smithi138.stdout: "exporter_sort_metrics": "true", 2023-10-01T09:14:21.859 INFO:teuthology.orchestra.run.smithi138.stdout: "exporter_stats_period": "5", 2023-10-01T09:14:21.859 INFO:teuthology.orchestra.run.smithi138.stdout: "fake_statfs_for_testing": "0", 2023-10-01T09:14:21.859 INFO:teuthology.orchestra.run.smithi138.stdout: "fatal_signal_handlers": "true", 2023-10-01T09:14:21.859 INFO:teuthology.orchestra.run.smithi138.stdout: "filer_max_purge_ops": "10", 2023-10-01T09:14:21.859 INFO:teuthology.orchestra.run.smithi138.stdout: "filer_max_truncate_ops": "128", 2023-10-01T09:14:21.859 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_apply_finisher_threads": "1", 2023-10-01T09:14:21.859 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_blackhole": "false", 2023-10-01T09:14:21.859 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_btrfs_clone_range": "true", 2023-10-01T09:14:21.859 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_btrfs_snap": "true", 2023-10-01T09:14:21.859 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_caller_concurrency": "10", 2023-10-01T09:14:21.860 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_collect_device_partition_information": "true", 2023-10-01T09:14:21.860 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_commit_timeout": "600.000000", 2023-10-01T09:14:21.860 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_debug_inject_read_err": "false", 2023-10-01T09:14:21.860 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_debug_omap_check": "false", 2023-10-01T09:14:21.860 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_debug_random_read_err": "0.000000", 2023-10-01T09:14:21.860 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_debug_verify_split": "false", 2023-10-01T09:14:21.860 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_dump_file": "", 2023-10-01T09:14:21.860 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-10-01T09:14:21.860 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-10-01T09:14:21.860 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_fadvise": "true", 2023-10-01T09:14:21.861 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_fail_eio": "true", 2023-10-01T09:14:21.861 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_fd_cache_shards": "16", 2023-10-01T09:14:21.861 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_fd_cache_size": "128", 2023-10-01T09:14:21.861 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_fiemap": "false", 2023-10-01T09:14:21.861 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_fiemap_threshold": "4096", 2023-10-01T09:14:21.861 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-10-01T09:14:21.861 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_index_retry_probability": "0.000000", 2023-10-01T09:14:21.861 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_inject_stall": "0", 2023-10-01T09:14:21.861 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_journal_parallel": "false", 2023-10-01T09:14:21.861 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_journal_trailing": "false", 2023-10-01T09:14:21.861 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_journal_writeahead": "false", 2023-10-01T09:14:21.861 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_kill_at": "0", 2023-10-01T09:14:21.862 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-10-01T09:14:21.862 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_max_inline_xattr_size": "0", 2023-10-01T09:14:21.862 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-10-01T09:14:21.862 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-10-01T09:14:21.862 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-10-01T09:14:21.862 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_max_inline_xattrs": "0", 2023-10-01T09:14:21.862 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-10-01T09:14:21.862 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_max_inline_xattrs_other": "2", 2023-10-01T09:14:21.862 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-10-01T09:14:21.862 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_max_sync_interval": "5.000000", 2023-10-01T09:14:21.862 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_max_xattr_value_size": "0", 2023-10-01T09:14:21.863 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-10-01T09:14:21.863 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-10-01T09:14:21.863 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-10-01T09:14:21.863 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_merge_threshold": "-10", 2023-10-01T09:14:21.863 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_min_sync_interval": "0.010000", 2023-10-01T09:14:21.863 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_odsync_write": "false", 2023-10-01T09:14:21.863 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_omap_backend": "rocksdb", 2023-10-01T09:14:21.863 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_omap_backend_path": "", 2023-10-01T09:14:21.863 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_omap_header_cache_size": "1024", 2023-10-01T09:14:21.863 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_ondisk_finisher_threads": "1", 2023-10-01T09:14:21.863 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-10-01T09:14:21.863 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_op_thread_timeout": "60", 2023-10-01T09:14:21.864 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_op_threads": "2", 2023-10-01T09:14:21.864 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_punch_hole": "false", 2023-10-01T09:14:21.864 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-10-01T09:14:21.864 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-10-01T09:14:21.864 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-10-01T09:14:21.864 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-10-01T09:14:21.864 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-10-01T09:14:21.864 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_queue_max_bytes": "104857600", 2023-10-01T09:14:21.864 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-10-01T09:14:21.864 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-10-01T09:14:21.864 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-10-01T09:14:21.864 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_queue_max_ops": "50", 2023-10-01T09:14:21.865 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-10-01T09:14:21.865 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_seek_data_hole": "false", 2023-10-01T09:14:21.865 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_sloppy_crc": "false", 2023-10-01T09:14:21.865 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-10-01T09:14:21.865 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_splice": "false", 2023-10-01T09:14:21.865 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_split_multiple": "2", 2023-10-01T09:14:21.865 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_split_rand_factor": "20", 2023-10-01T09:14:21.865 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_update_to": "1000", 2023-10-01T09:14:21.865 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-10-01T09:14:21.865 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-10-01T09:14:21.865 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-10-01T09:14:21.866 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-10-01T09:14:21.866 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-10-01T09:14:21.866 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-10-01T09:14:21.866 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_wbthrottle_enable": "true", 2023-10-01T09:14:21.866 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-10-01T09:14:21.866 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-10-01T09:14:21.866 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-10-01T09:14:21.866 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-10-01T09:14:21.866 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-10-01T09:14:21.866 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-10-01T09:14:21.867 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_xfs_extsize": "false", 2023-10-01T09:14:21.867 INFO:teuthology.orchestra.run.smithi138.stdout: "filestore_zfs_snap": "false", 2023-10-01T09:14:21.867 INFO:teuthology.orchestra.run.smithi138.stdout: "fio_dir": "/tmp/fio", 2023-10-01T09:14:21.867 INFO:teuthology.orchestra.run.smithi138.stdout: "fsid": "465870a4-6038-11ee-8db5-212e2dc638e7", 2023-10-01T09:14:21.867 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_allow_other": "true", 2023-10-01T09:14:21.867 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_atomic_o_trunc": "true", 2023-10-01T09:14:21.867 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_big_writes": "true", 2023-10-01T09:14:21.867 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_debug": "false", 2023-10-01T09:14:21.867 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_default_permissions": "false", 2023-10-01T09:14:21.867 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_disable_pagecache": "false", 2023-10-01T09:14:21.867 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_max_write": "0", 2023-10-01T09:14:21.868 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_multithreaded": "true", 2023-10-01T09:14:21.868 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_require_active_mds": "true", 2023-10-01T09:14:21.868 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_set_user_groups": "true", 2023-10-01T09:14:21.868 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_splice_move": "true", 2023-10-01T09:14:21.868 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_splice_read": "true", 2023-10-01T09:14:21.868 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_splice_write": "true", 2023-10-01T09:14:21.868 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_syncfs_on_mksnap": "true", 2023-10-01T09:14:21.868 INFO:teuthology.orchestra.run.smithi138.stdout: "fuse_use_invalidate_cb": "true", 2023-10-01T09:14:21.868 INFO:teuthology.orchestra.run.smithi138.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-10-01T09:14:21.868 INFO:teuthology.orchestra.run.smithi138.stdout: "gss_target_name": "ceph", 2023-10-01T09:14:21.869 INFO:teuthology.orchestra.run.smithi138.stdout: "heartbeat_file": "", 2023-10-01T09:14:21.869 INFO:teuthology.orchestra.run.smithi138.stdout: "heartbeat_inject_failure": "0", 2023-10-01T09:14:21.869 INFO:teuthology.orchestra.run.smithi138.stdout: "heartbeat_interval": "5", 2023-10-01T09:14:21.869 INFO:teuthology.orchestra.run.smithi138.stdout: "host": "smithi138", 2023-10-01T09:14:21.869 INFO:teuthology.orchestra.run.smithi138.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-10-01T09:14:21.869 INFO:teuthology.orchestra.run.smithi138.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-10-01T09:14:21.869 INFO:teuthology.orchestra.run.smithi138.stdout: "immutable_object_cache_max_size": "1073741824", 2023-10-01T09:14:21.869 INFO:teuthology.orchestra.run.smithi138.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-10-01T09:14:21.869 INFO:teuthology.orchestra.run.smithi138.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-10-01T09:14:21.869 INFO:teuthology.orchestra.run.smithi138.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-10-01T09:14:21.870 INFO:teuthology.orchestra.run.smithi138.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-10-01T09:14:21.870 INFO:teuthology.orchestra.run.smithi138.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-10-01T09:14:21.870 INFO:teuthology.orchestra.run.smithi138.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-10-01T09:14:21.870 INFO:teuthology.orchestra.run.smithi138.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-10-01T09:14:21.870 INFO:teuthology.orchestra.run.smithi138.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-10-01T09:14:21.870 INFO:teuthology.orchestra.run.smithi138.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-10-01T09:14:21.870 INFO:teuthology.orchestra.run.smithi138.stdout: "immutable_object_cache_watermark": "0.900000", 2023-10-01T09:14:21.870 INFO:teuthology.orchestra.run.smithi138.stdout: "inject_early_sigterm": "false", 2023-10-01T09:14:21.870 INFO:teuthology.orchestra.run.smithi138.stdout: "jaeger_agent_port": "6799", 2023-10-01T09:14:21.870 INFO:teuthology.orchestra.run.smithi138.stdout: "jaeger_tracing_enable": "false", 2023-10-01T09:14:21.871 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_aio": "true", 2023-10-01T09:14:21.871 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_align_min_size": "65536", 2023-10-01T09:14:21.871 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_block_align": "true", 2023-10-01T09:14:21.871 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_block_size": "4096", 2023-10-01T09:14:21.871 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_dio": "true", 2023-10-01T09:14:21.871 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_discard": "false", 2023-10-01T09:14:21.871 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_force_aio": "false", 2023-10-01T09:14:21.871 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_ignore_corruption": "false", 2023-10-01T09:14:21.871 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_max_write_bytes": "10485760", 2023-10-01T09:14:21.871 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_max_write_entries": "100", 2023-10-01T09:14:21.872 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_replay_from": "0", 2023-10-01T09:14:21.872 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_throttle_high_multiple": "0.000000", 2023-10-01T09:14:21.872 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-10-01T09:14:21.872 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-10-01T09:14:21.872 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_throttle_max_multiple": "0.000000", 2023-10-01T09:14:21.872 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_write_header_frequency": "0", 2023-10-01T09:14:21.872 INFO:teuthology.orchestra.run.smithi138.stdout: "journal_zero_on_create": "false", 2023-10-01T09:14:21.872 INFO:teuthology.orchestra.run.smithi138.stdout: "journaler_prefetch_periods": "10", 2023-10-01T09:14:21.872 INFO:teuthology.orchestra.run.smithi138.stdout: "journaler_prezero_periods": "5", 2023-10-01T09:14:21.872 INFO:teuthology.orchestra.run.smithi138.stdout: "journaler_write_head_interval": "15", 2023-10-01T09:14:21.873 INFO:teuthology.orchestra.run.smithi138.stdout: "key": "", 2023-10-01T09:14:21.873 INFO:teuthology.orchestra.run.smithi138.stdout: "keyfile": "", 2023-10-01T09:14:21.873 INFO:teuthology.orchestra.run.smithi138.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-10-01T09:14:21.873 INFO:teuthology.orchestra.run.smithi138.stdout: "kstore_backend": "rocksdb", 2023-10-01T09:14:21.873 INFO:teuthology.orchestra.run.smithi138.stdout: "kstore_default_stripe_size": "65536", 2023-10-01T09:14:21.873 INFO:teuthology.orchestra.run.smithi138.stdout: "kstore_fsck_on_mount": "false", 2023-10-01T09:14:21.873 INFO:teuthology.orchestra.run.smithi138.stdout: "kstore_fsck_on_mount_deep": "true", 2023-10-01T09:14:21.873 INFO:teuthology.orchestra.run.smithi138.stdout: "kstore_max_bytes": "67108864", 2023-10-01T09:14:21.873 INFO:teuthology.orchestra.run.smithi138.stdout: "kstore_max_ops": "512", 2023-10-01T09:14:21.873 INFO:teuthology.orchestra.run.smithi138.stdout: "kstore_nid_prealloc": "1024", 2023-10-01T09:14:21.873 INFO:teuthology.orchestra.run.smithi138.stdout: "kstore_onode_map_size": "1024", 2023-10-01T09:14:21.873 INFO:teuthology.orchestra.run.smithi138.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-10-01T09:14:21.874 INFO:teuthology.orchestra.run.smithi138.stdout: "kstore_sync_submit_transaction": "false", 2023-10-01T09:14:21.874 INFO:teuthology.orchestra.run.smithi138.stdout: "kstore_sync_transaction": "false", 2023-10-01T09:14:21.874 INFO:teuthology.orchestra.run.smithi138.stdout: "leveldb_block_size": "65536", 2023-10-01T09:14:21.874 INFO:teuthology.orchestra.run.smithi138.stdout: "leveldb_bloom_size": "0", 2023-10-01T09:14:21.874 INFO:teuthology.orchestra.run.smithi138.stdout: "leveldb_cache_size": "536870912", 2023-10-01T09:14:21.874 INFO:teuthology.orchestra.run.smithi138.stdout: "leveldb_compact_on_mount": "false", 2023-10-01T09:14:21.874 INFO:teuthology.orchestra.run.smithi138.stdout: "leveldb_compression": "false", 2023-10-01T09:14:21.874 INFO:teuthology.orchestra.run.smithi138.stdout: "leveldb_log": "", 2023-10-01T09:14:21.874 INFO:teuthology.orchestra.run.smithi138.stdout: "leveldb_log_to_ceph_log": "true", 2023-10-01T09:14:21.874 INFO:teuthology.orchestra.run.smithi138.stdout: "leveldb_max_open_files": "0", 2023-10-01T09:14:21.874 INFO:teuthology.orchestra.run.smithi138.stdout: "leveldb_paranoid": "false", 2023-10-01T09:14:21.875 INFO:teuthology.orchestra.run.smithi138.stdout: "leveldb_write_buffer_size": "33554432", 2023-10-01T09:14:21.875 INFO:teuthology.orchestra.run.smithi138.stdout: "librados_thread_count": "2", 2023-10-01T09:14:21.875 INFO:teuthology.orchestra.run.smithi138.stdout: "lockdep": "false", 2023-10-01T09:14:21.875 INFO:teuthology.orchestra.run.smithi138.stdout: "lockdep_force_backtrace": "false", 2023-10-01T09:14:21.875 INFO:teuthology.orchestra.run.smithi138.stdout: "log_coarse_timestamps": "true", 2023-10-01T09:14:21.875 INFO:teuthology.orchestra.run.smithi138.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-10-01T09:14:21.875 INFO:teuthology.orchestra.run.smithi138.stdout: "log_flush_on_exit": "false", 2023-10-01T09:14:21.875 INFO:teuthology.orchestra.run.smithi138.stdout: "log_graylog_host": "127.0.0.1", 2023-10-01T09:14:21.875 INFO:teuthology.orchestra.run.smithi138.stdout: "log_graylog_port": "12201", 2023-10-01T09:14:21.875 INFO:teuthology.orchestra.run.smithi138.stdout: "log_max_new": "1000", 2023-10-01T09:14:21.876 INFO:teuthology.orchestra.run.smithi138.stdout: "log_max_recent": "10000", 2023-10-01T09:14:21.876 INFO:teuthology.orchestra.run.smithi138.stdout: "log_stderr_prefix": "", 2023-10-01T09:14:21.876 INFO:teuthology.orchestra.run.smithi138.stdout: "log_stop_at_utilization": "0.970000", 2023-10-01T09:14:21.876 INFO:teuthology.orchestra.run.smithi138.stdout: "log_to_file": "true", 2023-10-01T09:14:21.876 INFO:teuthology.orchestra.run.smithi138.stdout: "log_to_graylog": "false", 2023-10-01T09:14:21.876 INFO:teuthology.orchestra.run.smithi138.stdout: "log_to_journald": "false", 2023-10-01T09:14:21.876 INFO:teuthology.orchestra.run.smithi138.stdout: "log_to_stderr": "false", 2023-10-01T09:14:21.876 INFO:teuthology.orchestra.run.smithi138.stdout: "log_to_syslog": "false", 2023-10-01T09:14:21.876 INFO:teuthology.orchestra.run.smithi138.stdout: "max_rotating_auth_attempts": "10", 2023-10-01T09:14:21.876 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-10-01T09:14:21.876 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_action_on_write_error": "1", 2023-10-01T09:14:21.877 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_alternate_name_max": "8192", 2023-10-01T09:14:21.877 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_asio_thread_count": "2", 2023-10-01T09:14:21.877 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_export_pin": "true", 2023-10-01T09:14:21.877 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_fragment_dirs": "true", 2023-10-01T09:14:21.877 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-10-01T09:14:21.877 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_fragment_interval": "5", 2023-10-01T09:14:21.877 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_fragment_size_max": "100000", 2023-10-01T09:14:21.877 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_idle_threshold": "0.000000", 2023-10-01T09:14:21.877 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_interval": "10", 2023-10-01T09:14:21.877 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_max": "-1", 2023-10-01T09:14:21.877 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_max_until": "-1", 2023-10-01T09:14:21.877 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_merge_size": "50", 2023-10-01T09:14:21.878 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_midchunk": "0.300000", 2023-10-01T09:14:21.878 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_min_rebalance": "0.100000", 2023-10-01T09:14:21.878 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_min_start": "0.200000", 2023-10-01T09:14:21.878 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_minchunk": "0.001000", 2023-10-01T09:14:21.878 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_mode": "0", 2023-10-01T09:14:21.878 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_need_max": "1.200000", 2023-10-01T09:14:21.878 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_need_min": "0.800000", 2023-10-01T09:14:21.878 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-10-01T09:14:21.878 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_sample_interval": "3.000000", 2023-10-01T09:14:21.878 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_split_bits": "3", 2023-10-01T09:14:21.878 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_split_rd": "25000.000000", 2023-10-01T09:14:21.879 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_split_size": "10000", 2023-10-01T09:14:21.879 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_split_wr": "10000.000000", 2023-10-01T09:14:21.879 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_target_decay": "10.000000", 2023-10-01T09:14:21.879 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-10-01T09:14:21.879 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_beacon_grace": "15.000000", 2023-10-01T09:14:21.879 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_beacon_interval": "4.000000", 2023-10-01T09:14:21.879 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_beacon_mon_down_grace": "60", 2023-10-01T09:14:21.879 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_cache_memory_limit": "4294967296", 2023-10-01T09:14:21.879 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_cache_mid": "0.700000", 2023-10-01T09:14:21.879 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_cache_release_free_interval": "10", 2023-10-01T09:14:21.879 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_cache_reservation": "0.050000", 2023-10-01T09:14:21.879 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-10-01T09:14:21.880 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_cache_trim_interval": "1", 2023-10-01T09:14:21.880 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_cache_trim_threshold": "262144", 2023-10-01T09:14:21.880 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-10-01T09:14:21.880 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-10-01T09:14:21.880 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_client_delegate_inos_pct": "50", 2023-10-01T09:14:21.880 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_client_prealloc_inos": "1000", 2023-10-01T09:14:21.880 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-10-01T09:14:21.880 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_connect_bootstrapping": "false", 2023-10-01T09:14:21.880 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_damage_table_max_entries": "10000", 2023-10-01T09:14:21.880 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-10-01T09:14:21.880 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_debug_auth_pins": "false", 2023-10-01T09:14:21.881 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_debug_frag": "false", 2023-10-01T09:14:21.881 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_debug_scatterstat": "false", 2023-10-01T09:14:21.881 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_debug_subtrees": "false", 2023-10-01T09:14:21.881 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_decay_halflife": "5.000000", 2023-10-01T09:14:21.881 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_default_dir_hash": "2", 2023-10-01T09:14:21.881 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_defer_session_stale": "true", 2023-10-01T09:14:21.881 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_deny_all_reconnect": "false", 2023-10-01T09:14:21.881 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_dir_keys_per_op": "16384", 2023-10-01T09:14:21.881 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_dir_max_commit_size": "10", 2023-10-01T09:14:21.881 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_dir_max_entries": "0", 2023-10-01T09:14:21.881 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_dir_prefetch": "true", 2023-10-01T09:14:21.882 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_dirstat_min_interval": "1.000000", 2023-10-01T09:14:21.882 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_dump_cache_after_rejoin": "false", 2023-10-01T09:14:21.882 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_dump_cache_on_map": "false", 2023-10-01T09:14:21.882 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_dump_cache_threshold_file": "0", 2023-10-01T09:14:21.882 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-10-01T09:14:21.882 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_early_reply": "true", 2023-10-01T09:14:21.882 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_enable_op_tracker": "true", 2023-10-01T09:14:21.882 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_enforce_unique_name": "true", 2023-10-01T09:14:21.882 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_export_ephemeral_distributed": "true", 2023-10-01T09:14:21.882 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-10-01T09:14:21.882 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_export_ephemeral_random": "true", 2023-10-01T09:14:21.883 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-10-01T09:14:21.883 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-10-01T09:14:21.883 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_forward_all_requests_to_auth": "false", 2023-10-01T09:14:21.883 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-10-01T09:14:21.883 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-10-01T09:14:21.883 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-10-01T09:14:21.883 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-10-01T09:14:21.883 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_health_cache_threshold": "1.500000", 2023-10-01T09:14:21.883 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_health_summarize_threshold": "10", 2023-10-01T09:14:21.883 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_heartbeat_grace": "15.000000", 2023-10-01T09:14:21.884 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_heartbeat_reset_grace": "1000", 2023-10-01T09:14:21.884 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_inject_health_dummy": "false", 2023-10-01T09:14:21.884 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-10-01T09:14:21.884 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_inject_migrator_session_race": "false", 2023-10-01T09:14:21.884 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-10-01T09:14:21.884 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-10-01T09:14:21.884 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-10-01T09:14:21.884 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_join_fs": "", 2023-10-01T09:14:21.884 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_journal_format": "1", 2023-10-01T09:14:21.884 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_kill_create_at": "0", 2023-10-01T09:14:21.884 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_kill_export_at": "0", 2023-10-01T09:14:21.885 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_kill_import_at": "0", 2023-10-01T09:14:21.885 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_kill_journal_at": "0", 2023-10-01T09:14:21.885 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_kill_journal_expire_at": "0", 2023-10-01T09:14:21.885 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_kill_journal_replay_at": "0", 2023-10-01T09:14:21.885 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_kill_link_at": "0", 2023-10-01T09:14:21.885 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_kill_mdstable_at": "0", 2023-10-01T09:14:21.885 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_kill_openc_at": "0", 2023-10-01T09:14:21.885 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_kill_rename_at": "0", 2023-10-01T09:14:21.885 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-10-01T09:14:21.885 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_log_events_per_segment": "1024", 2023-10-01T09:14:21.885 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_log_max_events": "-1", 2023-10-01T09:14:21.886 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_log_max_segments": "128", 2023-10-01T09:14:21.886 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_log_pause": "false", 2023-10-01T09:14:21.886 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_log_segment_size": "0", 2023-10-01T09:14:21.886 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_log_skip_corrupt_events": "false", 2023-10-01T09:14:21.886 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_log_warn_factor": "2.000000", 2023-10-01T09:14:21.886 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_max_caps_per_client": "1048576", 2023-10-01T09:14:21.886 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_max_completed_flushes": "100000", 2023-10-01T09:14:21.886 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_max_completed_requests": "100000", 2023-10-01T09:14:21.886 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_max_export_size": "20971520", 2023-10-01T09:14:21.886 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_max_file_recover": "32", 2023-10-01T09:14:21.886 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_max_purge_files": "64", 2023-10-01T09:14:21.887 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_max_purge_ops": "8192", 2023-10-01T09:14:21.887 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-10-01T09:14:21.887 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-10-01T09:14:21.887 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_max_snaps_per_dir": "100", 2023-10-01T09:14:21.887 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_max_xattr_pairs_size": "65536", 2023-10-01T09:14:21.887 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_metrics_update_interval": "2", 2023-10-01T09:14:21.887 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_min_caps_per_client": "100", 2023-10-01T09:14:21.887 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_min_caps_working_set": "10000", 2023-10-01T09:14:21.887 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-10-01T09:14:21.887 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_numa_node": "-1", 2023-10-01T09:14:21.887 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-10-01T09:14:21.887 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_op_complaint_time": "30.000000", 2023-10-01T09:14:21.888 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_op_history_duration": "600", 2023-10-01T09:14:21.888 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_op_history_size": "20", 2023-10-01T09:14:21.888 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_op_log_threshold": "5", 2023-10-01T09:14:21.888 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_ping_grace": "15", 2023-10-01T09:14:21.888 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_ping_interval": "5", 2023-10-01T09:14:21.888 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-10-01T09:14:21.888 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-10-01T09:14:21.888 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_recall_max_caps": "30000", 2023-10-01T09:14:21.888 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-10-01T09:14:21.888 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_recall_max_decay_threshold": "131072", 2023-10-01T09:14:21.888 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-10-01T09:14:21.888 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_recall_warning_threshold": "262144", 2023-10-01T09:14:21.889 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_reconnect_timeout": "45.000000", 2023-10-01T09:14:21.889 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_replay_interval": "1.000000", 2023-10-01T09:14:21.889 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-10-01T09:14:21.889 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-10-01T09:14:21.889 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_root_ino_gid": "0", 2023-10-01T09:14:21.889 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_root_ino_uid": "0", 2023-10-01T09:14:21.889 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-10-01T09:14:21.889 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_session_blocklist_on_evict": "true", 2023-10-01T09:14:21.889 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_session_blocklist_on_timeout": "true", 2023-10-01T09:14:21.889 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-10-01T09:14:21.889 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-10-01T09:14:21.889 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-10-01T09:14:21.890 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-10-01T09:14:21.890 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-10-01T09:14:21.890 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-10-01T09:14:21.890 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_shutdown_check": "0", 2023-10-01T09:14:21.890 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_skip_ino": "0", 2023-10-01T09:14:21.890 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_sleep_rank_change": "0.000000", 2023-10-01T09:14:21.890 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_snap_max_uid": "4294967294", 2023-10-01T09:14:21.890 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_snap_min_uid": "0", 2023-10-01T09:14:21.890 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_snap_rstat": "false", 2023-10-01T09:14:21.890 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_standby_replay_damaged": "false", 2023-10-01T09:14:21.890 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_symlink_recovery": "true", 2023-10-01T09:14:21.891 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_task_status_update_interval": "2.000000", 2023-10-01T09:14:21.891 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_thrash_exports": "0", 2023-10-01T09:14:21.891 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_thrash_fragments": "0", 2023-10-01T09:14:21.891 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_tick_interval": "5.000000", 2023-10-01T09:14:21.891 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_valgrind_exit": "false", 2023-10-01T09:14:21.891 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_verify_backtrace": "1", 2023-10-01T09:14:21.891 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_verify_scatter": "false", 2023-10-01T09:14:21.891 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_wipe_ino_prealloc": "false", 2023-10-01T09:14:21.891 INFO:teuthology.orchestra.run.smithi138.stdout: "mds_wipe_sessions": "false", 2023-10-01T09:14:21.891 INFO:teuthology.orchestra.run.smithi138.stdout: "mempool_debug": "false", 2023-10-01T09:14:21.891 INFO:teuthology.orchestra.run.smithi138.stdout: "memstore_debug_omit_block_device_write": "false", 2023-10-01T09:14:21.891 INFO:teuthology.orchestra.run.smithi138.stdout: "memstore_device_bytes": "1073741824", 2023-10-01T09:14:21.892 INFO:teuthology.orchestra.run.smithi138.stdout: "memstore_page_set": "false", 2023-10-01T09:14:21.892 INFO:teuthology.orchestra.run.smithi138.stdout: "memstore_page_size": "65536", 2023-10-01T09:14:21.892 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_client_bytes": "134217728", 2023-10-01T09:14:21.892 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_client_messages": "512", 2023-10-01T09:14:21.892 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-10-01T09:14:21.892 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_connect_retry_interval": "1.000000", 2023-10-01T09:14:21.892 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-10-01T09:14:21.892 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-10-01T09:14:21.892 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_disabled_modules": "", 2023-10-01T09:14:21.892 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-10-01T09:14:21.892 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_max_pg_num_change": "128", 2023-10-01T09:14:21.893 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_mds_bytes": "134217728", 2023-10-01T09:14:21.893 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_mds_messages": "128", 2023-10-01T09:14:21.893 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-10-01T09:14:21.893 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_mon_bytes": "134217728", 2023-10-01T09:14:21.893 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_mon_messages": "128", 2023-10-01T09:14:21.893 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_osd_bytes": "536870912", 2023-10-01T09:14:21.893 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_osd_messages": "8192", 2023-10-01T09:14:21.893 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_pool": "true", 2023-10-01T09:14:21.893 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_service_beacon_grace": "60.000000", 2023-10-01T09:14:21.894 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_standby_modules": "true", 2023-10-01T09:14:21.894 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_stats_period": "5", 2023-10-01T09:14:21.894 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_stats_threshold": "5", 2023-10-01T09:14:21.894 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_tick_period": "2", 2023-10-01T09:14:21.894 INFO:teuthology.orchestra.run.smithi138.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-10-01T09:14:21.894 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_accept_timeout_factor": "2.000000", 2023-10-01T09:14:21.894 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_allow_pool_delete": "true", 2023-10-01T09:14:21.894 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_allow_pool_size_one": "false", 2023-10-01T09:14:21.894 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_auth_validate_all_caps": "true", 2023-10-01T09:14:21.894 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-10-01T09:14:21.894 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-10-01T09:14:21.895 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_client_bytes": "104857600", 2023-10-01T09:14:21.895 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_client_directed_command_retry": "2", 2023-10-01T09:14:21.895 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_client_hunt_interval": "3.000000", 2023-10-01T09:14:21.895 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-10-01T09:14:21.895 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-10-01T09:14:21.895 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-10-01T09:14:21.895 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_client_hunt_parallel": "3", 2023-10-01T09:14:21.895 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_client_log_interval": "1.000000", 2023-10-01T09:14:21.895 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-10-01T09:14:21.895 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_client_ping_interval": "10.000000", 2023-10-01T09:14:21.895 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_client_ping_timeout": "30.000000", 2023-10-01T09:14:21.895 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_clock_drift_allowed": "1.000000", 2023-10-01T09:14:21.896 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-10-01T09:14:21.896 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-10-01T09:14:21.896 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_cluster_log_file_level": "debug", 2023-10-01T09:14:21.896 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_cluster_log_to_file": "false", 2023-10-01T09:14:21.896 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_cluster_log_to_graylog": "false", 2023-10-01T09:14:21.896 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-10-01T09:14:21.896 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-10-01T09:14:21.896 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_cluster_log_to_journald": "true", 2023-10-01T09:14:21.896 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_cluster_log_to_stderr": "false", 2023-10-01T09:14:21.896 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-10-01T09:14:21.896 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-10-01T09:14:21.897 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-10-01T09:14:21.897 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_compact_on_bootstrap": "false", 2023-10-01T09:14:21.897 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_compact_on_start": "false", 2023-10-01T09:14:21.897 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_compact_on_trim": "true", 2023-10-01T09:14:21.897 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_con_tracker_persist_interval": "10", 2023-10-01T09:14:21.897 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_con_tracker_score_halflife": "43200", 2023-10-01T09:14:21.897 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_config_key_max_entry_size": "65536", 2023-10-01T09:14:21.897 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_cpu_threads": "4", 2023-10-01T09:14:21.897 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_crush_min_required_version": "hammer", 2023-10-01T09:14:21.897 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_daemon_bytes": "419430400", 2023-10-01T09:14:21.897 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-10-01T09:14:21.898 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_data_avail_crit": "5", 2023-10-01T09:14:21.898 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_data_avail_warn": "5", 2023-10-01T09:14:21.898 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_data_size_warn": "16106127360", 2023-10-01T09:14:21.898 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_debug_block_osdmap_trim": "false", 2023-10-01T09:14:21.898 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-10-01T09:14:21.898 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_debug_dump_json": "false", 2023-10-01T09:14:21.898 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-10-01T09:14:21.898 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_debug_dump_transactions": "false", 2023-10-01T09:14:21.898 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_debug_extra_checks": "false", 2023-10-01T09:14:21.898 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-10-01T09:14:21.898 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-10-01T09:14:21.899 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_debug_no_require_quincy": "false", 2023-10-01T09:14:21.899 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_debug_no_require_reef": "false", 2023-10-01T09:14:21.899 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-10-01T09:14:21.899 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_delta_reset_interval": "10.000000", 2023-10-01T09:14:21.899 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_dns_srv_name": "ceph-mon", 2023-10-01T09:14:21.899 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_down_mkfs_grace": "60", 2023-10-01T09:14:21.899 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_election_default_strategy": "3", 2023-10-01T09:14:21.899 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_election_timeout": "5.000000", 2023-10-01T09:14:21.899 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-10-01T09:14:21.899 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_elector_ping_divisor": "2", 2023-10-01T09:14:21.899 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_elector_ping_timeout": "2.000000", 2023-10-01T09:14:21.899 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_enable_op_tracker": "true", 2023-10-01T09:14:21.900 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_fake_pool_delete": "false", 2023-10-01T09:14:21.900 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_force_quorum_join": "false", 2023-10-01T09:14:21.900 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_globalid_prealloc": "10000", 2023-10-01T09:14:21.900 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_health_detail_to_clog": "true", 2023-10-01T09:14:21.900 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_health_log_update_period": "5", 2023-10-01T09:14:21.900 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_health_max_detail": "50", 2023-10-01T09:14:21.900 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_health_to_clog": "true", 2023-10-01T09:14:21.900 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_health_to_clog_interval": "600", 2023-10-01T09:14:21.900 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-10-01T09:14:21.900 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_host": "[v2:172.21.15.138:3300/0,v1:172.21.15.138:6789/0] [v2:172.21.15.190:3300/0,v1:172.21.15.190:6789/0] [v2:172.21.15.138:3301/0,v1:172.21.15.138:6790/0]", 2023-10-01T09:14:21.900 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_host_override": "", 2023-10-01T09:14:21.901 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_initial_members": "", 2023-10-01T09:14:21.901 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-10-01T09:14:21.901 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-10-01T09:14:21.901 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-10-01T09:14:21.901 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-10-01T09:14:21.901 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_keyvaluedb": "rocksdb", 2023-10-01T09:14:21.901 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_lease": "5.000000", 2023-10-01T09:14:21.901 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-10-01T09:14:21.901 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-10-01T09:14:21.901 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_log_full_interval": "50", 2023-10-01T09:14:21.901 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_log_max": "10000", 2023-10-01T09:14:21.901 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_log_max_summary": "50", 2023-10-01T09:14:21.902 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_max_log_entries_per_event": "4096", 2023-10-01T09:14:21.902 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_max_log_epochs": "500", 2023-10-01T09:14:21.902 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_max_mdsmap_epochs": "500", 2023-10-01T09:14:21.902 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_max_mgrmap_epochs": "500", 2023-10-01T09:14:21.902 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_max_osd": "10000", 2023-10-01T09:14:21.902 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_max_pg_per_osd": "10000", 2023-10-01T09:14:21.902 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_max_pool_pg_num": "65536", 2023-10-01T09:14:21.902 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-10-01T09:14:21.902 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-10-01T09:14:21.902 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_mds_force_trim_to": "0", 2023-10-01T09:14:21.903 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_mds_skip_sanity": "true", 2023-10-01T09:14:21.903 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_memory_autotune": "true", 2023-10-01T09:14:21.903 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_memory_target": "2147483648", 2023-10-01T09:14:21.903 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_mgr_beacon_grace": "30", 2023-10-01T09:14:21.903 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-10-01T09:14:21.903 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_mgr_digest_period": "5", 2023-10-01T09:14:21.903 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_mgr_inactive_grace": "60", 2023-10-01T09:14:21.903 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_mgr_mkfs_grace": "240", 2023-10-01T09:14:21.903 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-10-01T09:14:21.903 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_min_osdmap_epochs": "500", 2023-10-01T09:14:21.903 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_op_complaint_time": "30", 2023-10-01T09:14:21.904 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_op_history_duration": "600", 2023-10-01T09:14:21.904 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_op_history_size": "20", 2023-10-01T09:14:21.904 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_op_history_slow_op_size": "20", 2023-10-01T09:14:21.904 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_op_history_slow_op_threshold": "10", 2023-10-01T09:14:21.904 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_op_log_threshold": "5", 2023-10-01T09:14:21.904 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-10-01T09:14:21.904 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-10-01T09:14:21.904 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-10-01T09:14:21.904 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_auto_mark_in": "false", 2023-10-01T09:14:21.904 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_auto_mark_new_in": "true", 2023-10-01T09:14:21.904 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-10-01T09:14:21.904 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-10-01T09:14:21.905 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_cache_size": "500", 2023-10-01T09:14:21.905 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_cache_size_min": "134217728", 2023-10-01T09:14:21.905 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_crush_smoke_test": "true", 2023-10-01T09:14:21.905 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_destroyed_out_interval": "600", 2023-10-01T09:14:21.905 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_down_out_interval": "600", 2023-10-01T09:14:21.905 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-10-01T09:14:21.905 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-10-01T09:14:21.905 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_force_trim_to": "0", 2023-10-01T09:14:21.905 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_full_ratio": "0.950000", 2023-10-01T09:14:21.905 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-10-01T09:14:21.905 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_laggy_halflife": "3600", 2023-10-01T09:14:21.906 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_laggy_max_interval": "300", 2023-10-01T09:14:21.906 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_laggy_weight": "0.300000", 2023-10-01T09:14:21.906 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-10-01T09:14:21.906 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_max_creating_pgs": "1024", 2023-10-01T09:14:21.906 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_max_initial_pgs": "1024", 2023-10-01T09:14:21.906 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_min_down_reporters": "2", 2023-10-01T09:14:21.906 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-10-01T09:14:21.906 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-10-01T09:14:21.906 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-10-01T09:14:21.906 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_prime_pg_temp": "true", 2023-10-01T09:14:21.906 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-10-01T09:14:21.906 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-10-01T09:14:21.907 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_report_timeout": "900", 2023-10-01T09:14:21.907 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-10-01T09:14:21.907 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-10-01T09:14:21.907 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_warn_num_repaired": "10", 2023-10-01T09:14:21.907 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osd_warn_op_age": "32.000000", 2023-10-01T09:14:21.907 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-10-01T09:14:21.907 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osdmap_full_prune_interval": "10", 2023-10-01T09:14:21.907 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osdmap_full_prune_min": "10000", 2023-10-01T09:14:21.907 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-10-01T09:14:21.907 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-10-01T09:14:21.907 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_pg_stuck_threshold": "60", 2023-10-01T09:14:21.907 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-10-01T09:14:21.908 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_pg_warn_min_objects": "10000", 2023-10-01T09:14:21.908 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_pg_warn_min_per_osd": "0", 2023-10-01T09:14:21.908 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-10-01T09:14:21.908 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_pool_quota_crit_threshold": "0", 2023-10-01T09:14:21.908 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_pool_quota_warn_threshold": "0", 2023-10-01T09:14:21.908 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_probe_timeout": "2.000000", 2023-10-01T09:14:21.908 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_reweight_max_change": "0.050000", 2023-10-01T09:14:21.908 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_reweight_max_osds": "4", 2023-10-01T09:14:21.908 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-10-01T09:14:21.908 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-10-01T09:14:21.908 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-10-01T09:14:21.909 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-10-01T09:14:21.909 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-10-01T09:14:21.909 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_scrub_interval": "86400", 2023-10-01T09:14:21.909 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_scrub_max_keys": "100", 2023-10-01T09:14:21.909 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_scrub_timeout": "300", 2023-10-01T09:14:21.909 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_session_timeout": "300", 2023-10-01T09:14:21.909 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_smart_report_timeout": "5", 2023-10-01T09:14:21.909 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_stat_smooth_intervals": "6", 2023-10-01T09:14:21.909 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-10-01T09:14:21.909 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_stretch_pool_min_size": "2", 2023-10-01T09:14:21.909 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_stretch_pool_size": "4", 2023-10-01T09:14:21.909 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-10-01T09:14:21.910 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_subscribe_interval": "86400.000000", 2023-10-01T09:14:21.910 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_sync_debug": "false", 2023-10-01T09:14:21.910 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_sync_max_payload_keys": "2000", 2023-10-01T09:14:21.910 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_sync_max_payload_size": "1048576", 2023-10-01T09:14:21.910 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_sync_provider_kill_at": "0", 2023-10-01T09:14:21.910 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_sync_requester_kill_at": "0", 2023-10-01T09:14:21.910 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_sync_timeout": "60.000000", 2023-10-01T09:14:21.910 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_target_pg_per_osd": "100", 2023-10-01T09:14:21.910 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_tick_interval": "5", 2023-10-01T09:14:21.910 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_timecheck_interval": "300.000000", 2023-10-01T09:14:21.910 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-10-01T09:14:21.911 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_use_min_delay_socket": "false", 2023-10-01T09:14:21.911 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_older_version_delay": "604800", 2023-10-01T09:14:21.911 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-10-01T09:14:21.911 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-10-01T09:14:21.911 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-10-01T09:14:21.911 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_filestore_osds": "true", 2023-10-01T09:14:21.911 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-10-01T09:14:21.911 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-10-01T09:14:21.911 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-10-01T09:14:21.911 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_misplaced": "false", 2023-10-01T09:14:21.911 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-10-01T09:14:21.911 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_older_version": "true", 2023-10-01T09:14:21.912 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-10-01T09:14:21.912 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_pool_no_app": "true", 2023-10-01T09:14:21.912 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-10-01T09:14:21.912 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-10-01T09:14:21.912 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-10-01T09:14:21.912 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-10-01T09:14:21.912 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_on_too_few_osds": "false", 2023-10-01T09:14:21.912 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-10-01T09:14:21.912 INFO:teuthology.orchestra.run.smithi138.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-10-01T09:14:21.912 INFO:teuthology.orchestra.run.smithi138.stdout: "monmap": "", 2023-10-01T09:14:21.912 INFO:teuthology.orchestra.run.smithi138.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-01T09:14:21.913 INFO:teuthology.orchestra.run.smithi138.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-10-01T09:14:21.913 INFO:teuthology.orchestra.run.smithi138.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-10-01T09:14:21.913 INFO:teuthology.orchestra.run.smithi138.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-01T09:14:21.913 INFO:teuthology.orchestra.run.smithi138.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-10-01T09:14:21.913 INFO:teuthology.orchestra.run.smithi138.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-10-01T09:14:21.913 INFO:teuthology.orchestra.run.smithi138.stdout: "motr_tracing_enabled": "false", 2023-10-01T09:14:21.913 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_op_threads": "3", 2023-10-01T09:14:21.913 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_buffer_size": "131072", 2023-10-01T09:14:21.913 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_cm": "false", 2023-10-01T09:14:21.913 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_device_name": "", 2023-10-01T09:14:21.913 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_dscp": "96", 2023-10-01T09:14:21.913 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-10-01T09:14:21.914 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_gid_idx": "0", 2023-10-01T09:14:21.914 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_local_gid": "", 2023-10-01T09:14:21.914 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_polling_us": "1000", 2023-10-01T09:14:21.914 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_port_num": "1", 2023-10-01T09:14:21.914 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-10-01T09:14:21.914 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-10-01T09:14:21.914 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_roce_ver": "1", 2023-10-01T09:14:21.914 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_send_buffers": "1024", 2023-10-01T09:14:21.914 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_sl": "3", 2023-10-01T09:14:21.914 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_support_srq": "true", 2023-10-01T09:14:21.914 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_rdma_type": "ib", 2023-10-01T09:14:21.915 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_async_reap_threshold": "5", 2023-10-01T09:14:21.915 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_bind_before_connect": "false", 2023-10-01T09:14:21.915 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_bind_ipv4": "true", 2023-10-01T09:14:21.915 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_bind_ipv6": "false", 2023-10-01T09:14:21.915 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_bind_msgr1": "true", 2023-10-01T09:14:21.915 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_bind_msgr2": "true", 2023-10-01T09:14:21.915 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_bind_port_max": "7568", 2023-10-01T09:14:21.915 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_bind_port_min": "6800", 2023-10-01T09:14:21.915 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_bind_prefer_ipv4": "false", 2023-10-01T09:14:21.915 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_bind_retry_count": "3", 2023-10-01T09:14:21.915 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_bind_retry_delay": "5", 2023-10-01T09:14:21.916 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_blackhole_client": "false", 2023-10-01T09:14:21.916 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_blackhole_mds": "false", 2023-10-01T09:14:21.916 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_blackhole_mgr": "false", 2023-10-01T09:14:21.916 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_blackhole_mon": "false", 2023-10-01T09:14:21.916 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_blackhole_osd": "false", 2023-10-01T09:14:21.916 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_client_mode": "crc secure", 2023-10-01T09:14:21.916 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_cluster_mode": "crc secure", 2023-10-01T09:14:21.916 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_cluster_type": "", 2023-10-01T09:14:21.916 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_compress_secure": "false", 2023-10-01T09:14:21.916 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_connection_idle_timeout": "900", 2023-10-01T09:14:21.916 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_connection_ready_timeout": "10", 2023-10-01T09:14:21.916 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_crc_data": "true", 2023-10-01T09:14:21.917 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_crc_header": "true", 2023-10-01T09:14:21.917 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_die_on_bad_msg": "false", 2023-10-01T09:14:21.917 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_die_on_bug": "true", 2023-10-01T09:14:21.917 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_die_on_old_message": "true", 2023-10-01T09:14:21.917 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_die_on_skipped_message": "false", 2023-10-01T09:14:21.917 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_die_on_unhandled_msg": "false", 2023-10-01T09:14:21.917 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-10-01T09:14:21.917 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_coremask": "0xF", 2023-10-01T09:14:21.917 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-10-01T09:14:21.917 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_devs_allowlist": "", 2023-10-01T09:14:21.917 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_enable_tso": "true", 2023-10-01T09:14:21.917 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-10-01T09:14:21.918 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-10-01T09:14:21.918 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_hugepages": "", 2023-10-01T09:14:21.918 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_hw_flow_control": "true", 2023-10-01T09:14:21.918 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-10-01T09:14:21.918 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_lro": "true", 2023-10-01T09:14:21.918 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_memory_channel": "4", 2023-10-01T09:14:21.918 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-10-01T09:14:21.918 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_pmd": "", 2023-10-01T09:14:21.918 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_port_id": "0", 2023-10-01T09:14:21.918 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-10-01T09:14:21.918 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dump_corrupt_message_level": "1", 2023-10-01T09:14:21.919 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_dump_on_send": "false", 2023-10-01T09:14:21.919 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_initial_backoff": "0.200000", 2023-10-01T09:14:21.919 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_inject_delay_max": "1.000000", 2023-10-01T09:14:21.919 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_inject_delay_probability": "0.000000", 2023-10-01T09:14:21.919 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_inject_delay_type": "", 2023-10-01T09:14:21.919 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_inject_internal_delays": "0.000000", 2023-10-01T09:14:21.919 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_inject_network_congestion": "0", 2023-10-01T09:14:21.919 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_inject_socket_failures": "0", 2023-10-01T09:14:21.919 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_learn_addr_from_peer": "true", 2023-10-01T09:14:21.919 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_max_accept_failures": "4", 2023-10-01T09:14:21.919 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_max_backoff": "15.000000", 2023-10-01T09:14:21.919 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_mon_client_mode": "secure crc", 2023-10-01T09:14:21.920 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_mon_cluster_mode": "secure crc", 2023-10-01T09:14:21.920 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_mon_service_mode": "secure crc", 2023-10-01T09:14:21.920 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_osd_compress_min_size": "1024", 2023-10-01T09:14:21.920 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_osd_compress_mode": "none", 2023-10-01T09:14:21.920 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_osd_compression_algorithm": "snappy", 2023-10-01T09:14:21.920 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-10-01T09:14:21.920 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_pq_min_cost": "65536", 2023-10-01T09:14:21.920 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_public_type": "", 2023-10-01T09:14:21.920 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_service_mode": "crc secure", 2023-10-01T09:14:21.920 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_tcp_listen_backlog": "512", 2023-10-01T09:14:21.920 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_tcp_nodelay": "true", 2023-10-01T09:14:21.921 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-10-01T09:14:21.921 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_tcp_rcvbuf": "0", 2023-10-01T09:14:21.921 INFO:teuthology.orchestra.run.smithi138.stdout: "ms_type": "async+posix", 2023-10-01T09:14:21.921 INFO:teuthology.orchestra.run.smithi138.stdout: "no_config_file": "false", 2023-10-01T09:14:21.921 INFO:teuthology.orchestra.run.smithi138.stdout: "objecter_completion_locks_per_session": "32", 2023-10-01T09:14:21.921 INFO:teuthology.orchestra.run.smithi138.stdout: "objecter_debug_inject_relock_delay": "false", 2023-10-01T09:14:21.921 INFO:teuthology.orchestra.run.smithi138.stdout: "objecter_inflight_op_bytes": "104857600", 2023-10-01T09:14:21.921 INFO:teuthology.orchestra.run.smithi138.stdout: "objecter_inflight_ops": "1024", 2023-10-01T09:14:21.921 INFO:teuthology.orchestra.run.smithi138.stdout: "objecter_inject_no_watch_ping": "false", 2023-10-01T09:14:21.921 INFO:teuthology.orchestra.run.smithi138.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-10-01T09:14:21.921 INFO:teuthology.orchestra.run.smithi138.stdout: "objecter_tick_interval": "5.000000", 2023-10-01T09:14:21.922 INFO:teuthology.orchestra.run.smithi138.stdout: "objecter_timeout": "10.000000", 2023-10-01T09:14:21.922 INFO:teuthology.orchestra.run.smithi138.stdout: "objectstore_blackhole": "false", 2023-10-01T09:14:21.922 INFO:teuthology.orchestra.run.smithi138.stdout: "openssl_engine_opts": "", 2023-10-01T09:14:21.922 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_agent_delay_time": "5.000000", 2023-10-01T09:14:21.922 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_agent_hist_halflife": "1000", 2023-10-01T09:14:21.922 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_agent_max_low_ops": "2", 2023-10-01T09:14:21.922 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_agent_max_ops": "4", 2023-10-01T09:14:21.922 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-10-01T09:14:21.922 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_agent_quantize_effort": "0.100000", 2023-10-01T09:14:21.922 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_agent_slop": "0.020000", 2023-10-01T09:14:21.923 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-10-01T09:14:21.923 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_allow_recovery_below_min_size": "true", 2023-10-01T09:14:21.923 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_asio_thread_count": "2", 2023-10-01T09:14:21.923 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_async_recovery_min_cost": "100", 2023-10-01T09:14:21.923 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_auto_mark_unfound_lost": "false", 2023-10-01T09:14:21.923 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_backfill_retry_interval": "30.000000", 2023-10-01T09:14:21.923 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_backfill_scan_max": "512", 2023-10-01T09:14:21.923 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_backfill_scan_min": "64", 2023-10-01T09:14:21.923 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_backoff_on_degraded": "false", 2023-10-01T09:14:21.923 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_backoff_on_peering": "false", 2023-10-01T09:14:21.923 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_backoff_on_unfound": "true", 2023-10-01T09:14:21.924 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_beacon_report_interval": "300", 2023-10-01T09:14:21.924 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_bench_duration": "30", 2023-10-01T09:14:21.924 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-10-01T09:14:21.924 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_bench_max_block_size": "67108864", 2023-10-01T09:14:21.924 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_bench_small_size_max_iops": "100", 2023-10-01T09:14:21.924 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_blkin_trace_all": "false", 2023-10-01T09:14:21.924 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_blocked_scrub_grace_period": "120", 2023-10-01T09:14:21.924 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-10-01T09:14:21.924 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-10-01T09:14:21.924 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-10-01T09:14:21.924 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_check_for_log_corruption": "false", 2023-10-01T09:14:21.924 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-10-01T09:14:21.925 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-01T09:14:21.925 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-10-01T09:14:21.925 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-01T09:14:21.925 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_class_update_on_start": "true", 2023-10-01T09:14:21.925 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_client_message_cap": "256", 2023-10-01T09:14:21.925 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_client_message_size_cap": "524288000", 2023-10-01T09:14:21.925 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_client_op_priority": "63", 2023-10-01T09:14:21.925 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_client_watch_timeout": "30", 2023-10-01T09:14:21.925 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_command_max_records": "256", 2023-10-01T09:14:21.925 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_compact_on_start": "false", 2023-10-01T09:14:21.925 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-10-01T09:14:21.926 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_crush_chooseleaf_type": "1", 2023-10-01T09:14:21.926 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_crush_initial_weight": "-1.000000", 2023-10-01T09:14:21.926 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_crush_update_on_start": "true", 2023-10-01T09:14:21.926 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_crush_update_weight_set": "true", 2023-10-01T09:14:21.926 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-10-01T09:14:21.926 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-10-01T09:14:21.926 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-10-01T09:14:21.926 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_drop_ping_duration": "0", 2023-10-01T09:14:21.926 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-10-01T09:14:21.926 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_feed_pullee": "-1", 2023-10-01T09:14:21.926 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-10-01T09:14:21.927 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-10-01T09:14:21.927 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-10-01T09:14:21.927 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_misdirected_ops": "false", 2023-10-01T09:14:21.927 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_no_acting_change": "false", 2023-10-01T09:14:21.927 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_no_purge_strays": "false", 2023-10-01T09:14:21.927 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_op_order": "false", 2023-10-01T09:14:21.927 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_pg_log_writeout": "false", 2023-10-01T09:14:21.927 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_pretend_recovery_active": "false", 2023-10-01T09:14:21.927 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-10-01T09:14:21.927 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-10-01T09:14:21.927 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_shutdown": "false", 2023-10-01T09:14:21.927 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-10-01T09:14:21.928 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-10-01T09:14:21.928 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_verify_cached_snaps": "false", 2023-10-01T09:14:21.928 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_verify_missing_on_start": "false", 2023-10-01T09:14:21.928 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_verify_snaps": "false", 2023-10-01T09:14:21.928 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-10-01T09:14:21.928 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-10-01T09:14:21.928 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_deep_scrub_keys": "1024", 2023-10-01T09:14:21.928 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-10-01T09:14:21.928 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-10-01T09:14:21.928 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-10-01T09:14:21.928 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_deep_scrub_stride": "524288", 2023-10-01T09:14:21.929 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-10-01T09:14:21.929 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_default_data_pool_replay_window": "45", 2023-10-01T09:14:21.929 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_default_notify_timeout": "30", 2023-10-01T09:14:21.929 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_delete_sleep": "0.000000", 2023-10-01T09:14:21.929 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-10-01T09:14:21.929 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-10-01T09:14:21.929 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-10-01T09:14:21.929 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_discard_disconnected_ops": "true", 2023-10-01T09:14:21.929 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_enable_op_tracker": "true", 2023-10-01T09:14:21.929 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-10-01T09:14:21.929 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_extblkdev_plugins": "vdo", 2023-10-01T09:14:21.929 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-10-01T09:14:21.930 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-10-01T09:14:21.930 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_fast_info": "true", 2023-10-01T09:14:21.930 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_fast_shutdown": "true", 2023-10-01T09:14:21.930 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-10-01T09:14:21.930 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_fast_shutdown_timeout": "15", 2023-10-01T09:14:21.930 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-10-01T09:14:21.930 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-10-01T09:14:21.930 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-10-01T09:14:21.930 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_function_tracing": "false", 2023-10-01T09:14:21.930 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_heartbeat_grace": "20", 2023-10-01T09:14:21.930 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_heartbeat_interval": "6", 2023-10-01T09:14:21.931 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-10-01T09:14:21.931 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_heartbeat_min_peers": "10", 2023-10-01T09:14:21.931 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_heartbeat_min_size": "2000", 2023-10-01T09:14:21.931 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_heartbeat_stale": "600", 2023-10-01T09:14:21.931 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-10-01T09:14:21.931 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_hit_set_max_size": "100000", 2023-10-01T09:14:21.931 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_hit_set_min_size": "1000", 2023-10-01T09:14:21.931 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-10-01T09:14:21.931 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-10-01T09:14:21.931 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-10-01T09:14:21.933 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-10-01T09:14:21.933 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-10-01T09:14:21.933 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_journal_flush_on_shutdown": "true", 2023-10-01T09:14:21.933 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_journal_size": "5120", 2023-10-01T09:14:21.933 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_kill_backfill_at": "0", 2023-10-01T09:14:21.933 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_loop_before_reset_tphandle": "64", 2023-10-01T09:14:21.933 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_map_cache_size": "50", 2023-10-01T09:14:21.933 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_map_dedup": "true", 2023-10-01T09:14:21.934 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_map_message_max": "40", 2023-10-01T09:14:21.934 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_map_message_max_bytes": "10485760", 2023-10-01T09:14:21.934 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_map_share_max_epochs": "40", 2023-10-01T09:14:21.934 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_attr_name_len": "100", 2023-10-01T09:14:21.934 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_attr_size": "0", 2023-10-01T09:14:21.934 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_backfills": "1", 2023-10-01T09:14:21.934 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_markdown_count": "5", 2023-10-01T09:14:21.934 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_markdown_period": "600", 2023-10-01T09:14:21.934 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_object_name_len": "2048", 2023-10-01T09:14:21.934 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_object_namespace_len": "256", 2023-10-01T09:14:21.934 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_object_size": "134217728", 2023-10-01T09:14:21.935 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-10-01T09:14:21.935 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_omap_entries_per_request": "1024", 2023-10-01T09:14:21.935 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_pg_blocked_by": "16", 2023-10-01T09:14:21.935 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_pg_log_entries": "10000", 2023-10-01T09:14:21.935 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-10-01T09:14:21.935 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_pgls": "1024", 2023-10-01T09:14:21.935 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_push_cost": "8388608", 2023-10-01T09:14:21.935 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_push_objects": "10", 2023-10-01T09:14:21.935 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_scrubs": "1", 2023-10-01T09:14:21.935 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-10-01T09:14:21.936 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_trimming_pgs": "2", 2023-10-01T09:14:21.936 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_write_op_reply_len": "64", 2023-10-01T09:14:21.936 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_max_write_size": "90", 2023-10-01T09:14:21.936 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-10-01T09:14:21.936 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-10-01T09:14:21.936 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-10-01T09:14:21.936 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-10-01T09:14:21.936 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-10-01T09:14:21.936 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-10-01T09:14:21.936 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-10-01T09:14:21.936 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_override_recovery_settings": "false", 2023-10-01T09:14:21.937 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_profile": "balanced", 2023-10-01T09:14:21.937 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-10-01T09:14:21.937 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-10-01T09:14:21.937 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-10-01T09:14:21.937 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-10-01T09:14:21.937 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-10-01T09:14:21.937 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-10-01T09:14:21.937 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-10-01T09:14:21.937 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-10-01T09:14:21.937 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-10-01T09:14:21.937 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-10-01T09:14:21.938 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mclock_skip_benchmark": "false", 2023-10-01T09:14:21.938 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_memory_base": "805306368", 2023-10-01T09:14:21.938 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_memory_cache_min": "134217728", 2023-10-01T09:14:21.938 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-10-01T09:14:21.938 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-10-01T09:14:21.938 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_memory_target": "4294967296", 2023-10-01T09:14:21.938 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_memory_target_autotune": "false", 2023-10-01T09:14:21.938 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-10-01T09:14:21.938 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_min_pg_log_entries": "250", 2023-10-01T09:14:21.938 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_min_recovery_priority": "0", 2023-10-01T09:14:21.938 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mon_heartbeat_interval": "30", 2023-10-01T09:14:21.939 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-10-01T09:14:21.939 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mon_report_interval": "5", 2023-10-01T09:14:21.939 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mon_report_max_in_flight": "2", 2023-10-01T09:14:21.939 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-10-01T09:14:21.939 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_num_cache_shards": "32", 2023-10-01T09:14:21.939 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_num_op_tracker_shard": "32", 2023-10-01T09:14:21.939 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_numa_auto_affinity": "true", 2023-10-01T09:14:21.939 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_numa_node": "-1", 2023-10-01T09:14:21.939 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_numa_prefer_iface": "true", 2023-10-01T09:14:21.939 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-10-01T09:14:21.939 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_objecter_finishers": "1", 2023-10-01T09:14:21.940 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_objectstore": "bluestore", 2023-10-01T09:14:21.940 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_objectstore_fuse": "false", 2023-10-01T09:14:21.940 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_objectstore_tracing": "false", 2023-10-01T09:14:21.940 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_complaint_time": "30.000000", 2023-10-01T09:14:21.940 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_history_duration": "600", 2023-10-01T09:14:21.940 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_history_size": "20", 2023-10-01T09:14:21.940 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_history_slow_op_size": "20", 2023-10-01T09:14:21.940 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-10-01T09:14:21.940 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_log_threshold": "5", 2023-10-01T09:14:21.940 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_num_shards": "0", 2023-10-01T09:14:21.940 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_num_shards_hdd": "5", 2023-10-01T09:14:21.940 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_num_shards_ssd": "8", 2023-10-01T09:14:21.940 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_num_threads_per_shard": "0", 2023-10-01T09:14:21.941 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-10-01T09:14:21.941 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-10-01T09:14:21.941 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-10-01T09:14:21.941 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_pq_min_cost": "65536", 2023-10-01T09:14:21.941 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_queue": "mclock_scheduler", 2023-10-01T09:14:21.941 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_queue_cut_off": "high", 2023-10-01T09:14:21.941 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_thread_suicide_timeout": "150", 2023-10-01T09:14:21.941 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_op_thread_timeout": "15", 2023-10-01T09:14:21.941 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_open_classes_on_start": "true", 2023-10-01T09:14:21.941 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_os_flags": "0", 2023-10-01T09:14:21.941 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_peering_op_priority": "255", 2023-10-01T09:14:21.942 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pg_delete_cost": "1048576", 2023-10-01T09:14:21.942 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pg_delete_priority": "5", 2023-10-01T09:14:21.942 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-10-01T09:14:21.942 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-10-01T09:14:21.942 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pg_log_dups_tracked": "3000", 2023-10-01T09:14:21.942 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pg_log_trim_max": "10000", 2023-10-01T09:14:21.942 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pg_log_trim_min": "100", 2023-10-01T09:14:21.942 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-10-01T09:14:21.942 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pg_object_context_cache_count": "64", 2023-10-01T09:14:21.942 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pg_stat_report_interval_max": "500", 2023-10-01T09:14:21.942 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-10-01T09:14:21.943 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-10-01T09:14:21.943 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-10-01T09:14:21.943 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-10-01T09:14:21.943 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-10-01T09:14:21.943 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-10-01T09:14:21.943 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_crimson": "false", 2023-10-01T09:14:21.943 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_crush_rule": "-1", 2023-10-01T09:14:21.943 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_ec_fast_read": "false", 2023-10-01T09:14:21.943 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-10-01T09:14:21.943 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_flag_bulk": "false", 2023-10-01T09:14:21.943 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-10-01T09:14:21.944 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_flag_nodelete": "false", 2023-10-01T09:14:21.944 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-10-01T09:14:21.944 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-10-01T09:14:21.944 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_flags": "0", 2023-10-01T09:14:21.944 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-10-01T09:14:21.944 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_min_size": "0", 2023-10-01T09:14:21.944 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-10-01T09:14:21.944 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_pg_num": "32", 2023-10-01T09:14:21.944 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_pgp_num": "0", 2023-10-01T09:14:21.944 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-10-01T09:14:21.944 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_size": "3", 2023-10-01T09:14:21.945 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_default_type": "replicated", 2023-10-01T09:14:21.945 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-10-01T09:14:21.945 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_pool_use_gmt_hitset": "true", 2023-10-01T09:14:21.945 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_push_per_object_cost": "1000", 2023-10-01T09:14:21.945 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_read_ec_check_for_errors": "false", 2023-10-01T09:14:21.945 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recover_clone_overlap": "true", 2023-10-01T09:14:21.945 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recover_clone_overlap_limit": "10", 2023-10-01T09:14:21.945 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_cost": "20971520", 2023-10-01T09:14:21.945 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_delay_start": "0.000000", 2023-10-01T09:14:21.945 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_max_active": "0", 2023-10-01T09:14:21.945 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_max_active_hdd": "3", 2023-10-01T09:14:21.945 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_max_active_ssd": "10", 2023-10-01T09:14:21.946 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_max_chunk": "8388608", 2023-10-01T09:14:21.946 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-10-01T09:14:21.946 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_max_single_start": "1", 2023-10-01T09:14:21.946 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_op_priority": "3", 2023-10-01T09:14:21.946 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_op_warn_multiple": "16", 2023-10-01T09:14:21.946 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_priority": "5", 2023-10-01T09:14:21.946 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_retry_interval": "30.000000", 2023-10-01T09:14:21.946 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_sleep": "0.000000", 2023-10-01T09:14:21.946 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-10-01T09:14:21.946 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-10-01T09:14:21.947 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-10-01T09:14:21.947 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_repair_during_recovery": "false", 2023-10-01T09:14:21.947 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_requested_scrub_priority": "120", 2023-10-01T09:14:21.947 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-10-01T09:14:21.947 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_rollback_to_cluster_snap": "", 2023-10-01T09:14:21.947 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_auto_repair": "false", 2023-10-01T09:14:21.947 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-10-01T09:14:21.947 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-10-01T09:14:21.947 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_begin_hour": "0", 2023-10-01T09:14:21.947 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_begin_week_day": "0", 2023-10-01T09:14:21.947 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_chunk_max": "25", 2023-10-01T09:14:21.948 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_chunk_min": "5", 2023-10-01T09:14:21.948 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_cost": "52428800", 2023-10-01T09:14:21.948 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_during_recovery": "false", 2023-10-01T09:14:21.948 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_end_hour": "0", 2023-10-01T09:14:21.948 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_end_week_day": "0", 2023-10-01T09:14:21.948 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_event_cost": "4096", 2023-10-01T09:14:21.948 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-10-01T09:14:21.948 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-10-01T09:14:21.948 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_invalid_stats": "true", 2023-10-01T09:14:21.948 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_load_threshold": "0.500000", 2023-10-01T09:14:21.948 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_max_interval": "604800.000000", 2023-10-01T09:14:21.949 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_max_preemptions": "5", 2023-10-01T09:14:21.949 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_min_interval": "86400.000000", 2023-10-01T09:14:21.949 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_priority": "5", 2023-10-01T09:14:21.949 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_reservation_timeout": "5000", 2023-10-01T09:14:21.949 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_sleep": "0.000000", 2023-10-01T09:14:21.949 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-10-01T09:14:21.949 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-10-01T09:14:21.949 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-10-01T09:14:21.949 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_shutdown_pgref_assert": "false", 2023-10-01T09:14:21.949 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_skip_data_digest": "false", 2023-10-01T09:14:21.949 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_smart_report_timeout": "5", 2023-10-01T09:14:21.949 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_snap_trim_cost": "1048576", 2023-10-01T09:14:21.950 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_snap_trim_priority": "5", 2023-10-01T09:14:21.950 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_snap_trim_sleep": "0.000000", 2023-10-01T09:14:21.950 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-10-01T09:14:21.950 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-10-01T09:14:21.950 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-10-01T09:14:21.950 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-10-01T09:14:21.950 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_stats_update_period_scrubbing": "15", 2023-10-01T09:14:21.950 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-10-01T09:14:21.950 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_target_transaction_size": "30", 2023-10-01T09:14:21.950 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-10-01T09:14:21.950 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-10-01T09:14:21.951 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-10-01T09:14:21.951 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-10-01T09:14:21.951 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-10-01T09:14:21.951 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-10-01T09:14:21.951 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-10-01T09:14:21.951 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_tier_default_cache_mode": "writeback", 2023-10-01T09:14:21.951 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-10-01T09:14:21.951 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-10-01T09:14:21.951 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_tracing": "false", 2023-10-01T09:14:21.951 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_use_stale_snap": "false", 2023-10-01T09:14:21.951 INFO:teuthology.orchestra.run.smithi138.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-10-01T09:14:21.951 INFO:teuthology.orchestra.run.smithi138.stdout: "osdc_blkin_trace_all": "false", 2023-10-01T09:14:21.952 INFO:teuthology.orchestra.run.smithi138.stdout: "paxos_kill_at": "0", 2023-10-01T09:14:21.952 INFO:teuthology.orchestra.run.smithi138.stdout: "paxos_max_join_drift": "10", 2023-10-01T09:14:21.952 INFO:teuthology.orchestra.run.smithi138.stdout: "paxos_min": "500", 2023-10-01T09:14:21.952 INFO:teuthology.orchestra.run.smithi138.stdout: "paxos_min_wait": "0.050000", 2023-10-01T09:14:21.952 INFO:teuthology.orchestra.run.smithi138.stdout: "paxos_propose_interval": "1.000000", 2023-10-01T09:14:21.952 INFO:teuthology.orchestra.run.smithi138.stdout: "paxos_service_trim_max": "500", 2023-10-01T09:14:21.952 INFO:teuthology.orchestra.run.smithi138.stdout: "paxos_service_trim_max_multiplier": "20", 2023-10-01T09:14:21.952 INFO:teuthology.orchestra.run.smithi138.stdout: "paxos_service_trim_min": "250", 2023-10-01T09:14:21.952 INFO:teuthology.orchestra.run.smithi138.stdout: "paxos_stash_full_interval": "25", 2023-10-01T09:14:21.952 INFO:teuthology.orchestra.run.smithi138.stdout: "paxos_trim_max": "500", 2023-10-01T09:14:21.952 INFO:teuthology.orchestra.run.smithi138.stdout: "paxos_trim_min": "250", 2023-10-01T09:14:21.953 INFO:teuthology.orchestra.run.smithi138.stdout: "perf": "true", 2023-10-01T09:14:21.953 INFO:teuthology.orchestra.run.smithi138.stdout: "pid_file": "", 2023-10-01T09:14:21.953 INFO:teuthology.orchestra.run.smithi138.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-10-01T09:14:21.953 INFO:teuthology.orchestra.run.smithi138.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-10-01T09:14:21.953 INFO:teuthology.orchestra.run.smithi138.stdout: "public_addr": "-", 2023-10-01T09:14:21.953 INFO:teuthology.orchestra.run.smithi138.stdout: "public_addrv": "", 2023-10-01T09:14:21.953 INFO:teuthology.orchestra.run.smithi138.stdout: "public_bind_addr": "-", 2023-10-01T09:14:21.953 INFO:teuthology.orchestra.run.smithi138.stdout: "public_network": "172.21.0.0/20", 2023-10-01T09:14:21.953 INFO:teuthology.orchestra.run.smithi138.stdout: "public_network_interface": "", 2023-10-01T09:14:21.953 INFO:teuthology.orchestra.run.smithi138.stdout: "qat_compressor_enabled": "false", 2023-10-01T09:14:21.953 INFO:teuthology.orchestra.run.smithi138.stdout: "qat_compressor_session_max_number": "256", 2023-10-01T09:14:21.953 INFO:teuthology.orchestra.run.smithi138.stdout: "rados_mon_op_timeout": "0", 2023-10-01T09:14:21.954 INFO:teuthology.orchestra.run.smithi138.stdout: "rados_osd_op_timeout": "0", 2023-10-01T09:14:21.954 INFO:teuthology.orchestra.run.smithi138.stdout: "rados_tracing": "false", 2023-10-01T09:14:21.954 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_atime_update_interval": "60", 2023-10-01T09:14:21.954 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-10-01T09:14:21.954 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_balance_parent_reads": "false", 2023-10-01T09:14:21.954 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_balance_snap_reads": "false", 2023-10-01T09:14:21.954 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_blkin_trace_all": "false", 2023-10-01T09:14:21.954 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_blocklist_expire_seconds": "0", 2023-10-01T09:14:21.954 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_blocklist_on_break_lock": "true", 2023-10-01T09:14:21.954 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_cache": "true", 2023-10-01T09:14:21.954 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_cache_block_writes_upfront": "false", 2023-10-01T09:14:21.955 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_cache_max_dirty": "25165824", 2023-10-01T09:14:21.955 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-10-01T09:14:21.955 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_cache_max_dirty_object": "0", 2023-10-01T09:14:21.955 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_cache_policy": "writearound", 2023-10-01T09:14:21.955 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_cache_size": "33554432", 2023-10-01T09:14:21.955 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_cache_target_dirty": "16777216", 2023-10-01T09:14:21.955 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-10-01T09:14:21.955 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_clone_copy_on_read": "false", 2023-10-01T09:14:21.955 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_compression_hint": "none", 2023-10-01T09:14:21.955 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_concurrent_management_ops": "10", 2023-10-01T09:14:21.955 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-10-01T09:14:21.956 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_default_clone_format": "auto", 2023-10-01T09:14:21.956 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_default_data_pool": "", 2023-10-01T09:14:21.956 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_default_features": "61", 2023-10-01T09:14:21.956 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_default_format": "2", 2023-10-01T09:14:21.956 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_default_map_options": "", 2023-10-01T09:14:21.956 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_default_order": "22", 2023-10-01T09:14:21.956 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_default_pool": "rbd", 2023-10-01T09:14:21.956 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-10-01T09:14:21.956 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_default_stripe_count": "0", 2023-10-01T09:14:21.956 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_default_stripe_unit": "0", 2023-10-01T09:14:21.956 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_disable_zero_copy_writes": "true", 2023-10-01T09:14:21.956 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_discard_granularity_bytes": "65536", 2023-10-01T09:14:21.957 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-10-01T09:14:21.957 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_enable_alloc_hint": "true", 2023-10-01T09:14:21.957 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-10-01T09:14:21.957 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_io_scheduler": "simple", 2023-10-01T09:14:21.957 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-10-01T09:14:21.957 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_journal_commit_age": "5.000000", 2023-10-01T09:14:21.957 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-10-01T09:14:21.957 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-10-01T09:14:21.957 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-10-01T09:14:21.957 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-10-01T09:14:21.957 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_journal_object_flush_interval": "0", 2023-10-01T09:14:21.957 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-10-01T09:14:21.958 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-10-01T09:14:21.958 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_journal_order": "24", 2023-10-01T09:14:21.958 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_journal_pool": "", 2023-10-01T09:14:21.958 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_journal_splay_width": "4", 2023-10-01T09:14:21.958 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_localize_parent_reads": "false", 2023-10-01T09:14:21.958 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_localize_snap_reads": "false", 2023-10-01T09:14:21.958 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-10-01T09:14:21.958 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-10-01T09:14:21.958 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-10-01T09:14:21.958 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-10-01T09:14:21.958 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-10-01T09:14:21.959 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-10-01T09:14:21.959 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_image_policy_type": "simple", 2023-10-01T09:14:21.959 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-10-01T09:14:21.959 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-10-01T09:14:21.959 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-10-01T09:14:21.959 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-10-01T09:14:21.959 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-10-01T09:14:21.959 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-10-01T09:14:21.959 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-10-01T09:14:21.959 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_memory_autotune": "true", 2023-10-01T09:14:21.959 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_memory_base": "805306368", 2023-10-01T09:14:21.960 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-10-01T09:14:21.960 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-10-01T09:14:21.960 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-10-01T09:14:21.960 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-10-01T09:14:21.960 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_memory_target": "4294967296", 2023-10-01T09:14:21.960 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-10-01T09:14:21.960 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-10-01T09:14:21.960 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-10-01T09:14:21.960 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirroring_delete_delay": "0", 2023-10-01T09:14:21.960 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-10-01T09:14:21.960 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirroring_replay_delay": "0", 2023-10-01T09:14:21.960 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-10-01T09:14:21.961 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-10-01T09:14:21.961 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_move_to_trash_on_remove": "false", 2023-10-01T09:14:21.961 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-10-01T09:14:21.961 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_mtime_update_interval": "60", 2023-10-01T09:14:21.961 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_non_blocking_aio": "true", 2023-10-01T09:14:21.961 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_op_thread_timeout": "60", 2023-10-01T09:14:21.961 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_op_threads": "1", 2023-10-01T09:14:21.961 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_parent_cache_enabled": "false", 2023-10-01T09:14:21.961 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_persistent_cache_mode": "disabled", 2023-10-01T09:14:21.961 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_persistent_cache_path": "/tmp", 2023-10-01T09:14:21.961 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_persistent_cache_size": "1073741824", 2023-10-01T09:14:21.962 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_plugins": "", 2023-10-01T09:14:21.962 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_bps_burst": "0", 2023-10-01T09:14:21.962 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-10-01T09:14:21.962 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_bps_limit": "0", 2023-10-01T09:14:21.962 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_exclude_ops": "0", 2023-10-01T09:14:21.962 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_iops_burst": "0", 2023-10-01T09:14:21.962 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-10-01T09:14:21.962 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_iops_limit": "0", 2023-10-01T09:14:21.962 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_read_bps_burst": "0", 2023-10-01T09:14:21.962 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-10-01T09:14:21.963 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_read_bps_limit": "0", 2023-10-01T09:14:21.963 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_read_iops_burst": "0", 2023-10-01T09:14:21.963 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-10-01T09:14:21.963 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_read_iops_limit": "0", 2023-10-01T09:14:21.963 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_schedule_tick_min": "50", 2023-10-01T09:14:21.963 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_write_bps_burst": "0", 2023-10-01T09:14:21.963 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-10-01T09:14:21.963 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_write_bps_limit": "0", 2023-10-01T09:14:21.963 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_write_iops_burst": "0", 2023-10-01T09:14:21.963 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-10-01T09:14:21.963 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_qos_write_iops_limit": "0", 2023-10-01T09:14:21.964 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_quiesce_notification_attempts": "10", 2023-10-01T09:14:21.964 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_read_from_replica_policy": "default", 2023-10-01T09:14:21.964 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-10-01T09:14:21.964 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_readahead_max_bytes": "524288", 2023-10-01T09:14:21.964 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_readahead_trigger_requests": "10", 2023-10-01T09:14:21.964 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_request_timed_out_seconds": "30", 2023-10-01T09:14:21.964 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_skip_partial_discard": "true", 2023-10-01T09:14:21.964 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-10-01T09:14:21.964 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_tracing": "false", 2023-10-01T09:14:21.964 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_validate_names": "true", 2023-10-01T09:14:21.964 INFO:teuthology.orchestra.run.smithi138.stdout: "rbd_validate_pool": "true", 2023-10-01T09:14:21.965 INFO:teuthology.orchestra.run.smithi138.stdout: "restapi_base_url": "", 2023-10-01T09:14:21.965 INFO:teuthology.orchestra.run.smithi138.stdout: "restapi_log_level": "", 2023-10-01T09:14:21.965 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_acl_grants_max_num": "100", 2023-10-01T09:14:21.965 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_admin_entry": "admin", 2023-10-01T09:14:21.965 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-10-01T09:14:21.965 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_backend_store": "rados", 2023-10-01T09:14:21.965 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_barbican_url": "", 2023-10-01T09:14:21.965 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_beast_enable_async": "true", 2023-10-01T09:14:21.965 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-10-01T09:14:21.965 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-10-01T09:14:21.966 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_bucket_index_max_aio": "128", 2023-10-01T09:14:21.966 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-10-01T09:14:21.966 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-10-01T09:14:21.966 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_bucket_quota_ttl": "600", 2023-10-01T09:14:21.966 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-10-01T09:14:21.966 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_cache_enabled": "true", 2023-10-01T09:14:21.966 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_cache_expiry_interval": "900", 2023-10-01T09:14:21.966 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_cache_lru_size": "10000", 2023-10-01T09:14:21.966 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_config_store": "rados", 2023-10-01T09:14:21.966 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_content_length_compat": "false", 2023-10-01T09:14:21.966 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_copy_obj_progress": "true", 2023-10-01T09:14:21.967 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-10-01T09:14:21.967 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_cors_rules_max_num": "100", 2023-10-01T09:14:21.967 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_cross_domain_policy": "", 2023-10-01T09:14:21.967 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_default_encryption_key": "", 2023-10-01T09:14:21.967 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_kmip_addr": "", 2023-10-01T09:14:21.967 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_kmip_ca_path": "", 2023-10-01T09:14:21.967 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_kmip_client_cert": "", 2023-10-01T09:14:21.967 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_kmip_client_key": "", 2023-10-01T09:14:21.967 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-10-01T09:14:21.967 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_kmip_password": "", 2023-10-01T09:14:21.967 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-10-01T09:14:21.967 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_kmip_username": "", 2023-10-01T09:14:21.967 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_require_ssl": "true", 2023-10-01T09:14:21.968 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-10-01T09:14:21.968 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-10-01T09:14:21.968 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-10-01T09:14:21.968 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-10-01T09:14:21.968 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-10-01T09:14:21.968 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-10-01T09:14:21.968 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-10-01T09:14:21.968 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-10-01T09:14:21.968 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-10-01T09:14:21.968 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-10-01T09:14:21.968 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-10-01T09:14:21.968 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-10-01T09:14:21.969 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-10-01T09:14:21.969 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-10-01T09:14:21.969 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_suppress_logs": "true", 2023-10-01T09:14:21.969 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_vault_addr": "", 2023-10-01T09:14:21.969 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_vault_auth": "token", 2023-10-01T09:14:21.969 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_vault_namespace": "", 2023-10-01T09:14:21.969 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_vault_prefix": "", 2023-10-01T09:14:21.969 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-10-01T09:14:21.969 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-10-01T09:14:21.969 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-10-01T09:14:21.969 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-10-01T09:14:21.970 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_vault_token_file": "", 2023-10-01T09:14:21.970 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-10-01T09:14:21.970 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_curl_buffersize": "524288", 2023-10-01T09:14:21.970 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_curl_low_speed_limit": "1024", 2023-10-01T09:14:21.970 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_curl_low_speed_time": "300", 2023-10-01T09:14:21.970 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_curl_tcp_keepalive": "0", 2023-10-01T09:14:21.970 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-10-01T09:14:21.970 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-10-01T09:14:21.970 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-10-01T09:14:21.970 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-10-01T09:14:21.971 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-10-01T09:14:21.971 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_d3n_l1_fadvise": "4", 2023-10-01T09:14:21.971 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-10-01T09:14:21.971 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-10-01T09:14:21.971 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-10-01T09:14:21.971 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-10-01T09:14:21.971 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_data_log_changes_size": "1000", 2023-10-01T09:14:21.971 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_data_log_num_shards": "128", 2023-10-01T09:14:21.971 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-10-01T09:14:21.971 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_data_log_window": "30", 2023-10-01T09:14:21.971 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_data_notify_interval_msec": "0", 2023-10-01T09:14:21.971 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_data_sync_poll_interval": "20", 2023-10-01T09:14:21.972 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_data_sync_spawn_window": "20", 2023-10-01T09:14:21.972 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-10-01T09:14:21.972 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-10-01T09:14:21.972 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_default_data_log_backing": "fifo", 2023-10-01T09:14:21.972 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-10-01T09:14:21.972 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_default_region_info_oid": "default.region", 2023-10-01T09:14:21.972 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-10-01T09:14:21.972 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-10-01T09:14:21.972 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_defer_to_bucket_acls": "", 2023-10-01T09:14:21.972 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-10-01T09:14:21.972 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-10-01T09:14:21.973 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-10-01T09:14:21.973 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-10-01T09:14:21.973 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-10-01T09:14:21.973 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-10-01T09:14:21.973 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-10-01T09:14:21.973 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-10-01T09:14:21.973 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dmclock_data_res": "500.000000", 2023-10-01T09:14:21.973 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-10-01T09:14:21.973 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-10-01T09:14:21.973 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-10-01T09:14:21.974 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-10-01T09:14:21.974 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dns_name": "", 2023-10-01T09:14:21.974 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dns_s3website_name": "", 2023-10-01T09:14:21.974 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_dynamic_resharding": "true", 2023-10-01T09:14:21.974 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-10-01T09:14:21.974 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_enable_gc_threads": "true", 2023-10-01T09:14:21.974 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_enable_lc_threads": "true", 2023-10-01T09:14:21.974 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_enable_ops_log": "false", 2023-10-01T09:14:21.974 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_enable_quota_threads": "true", 2023-10-01T09:14:21.974 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_enable_static_website": "false", 2023-10-01T09:14:21.974 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_enable_usage_log": "false", 2023-10-01T09:14:21.975 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_enforce_swift_acls": "true", 2023-10-01T09:14:21.975 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_exit_timeout_secs": "120", 2023-10-01T09:14:21.975 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_expose_bucket": "false", 2023-10-01T09:14:21.975 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_extended_http_attrs": "", 2023-10-01T09:14:21.975 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_filter": "none", 2023-10-01T09:14:21.975 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-10-01T09:14:21.975 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_frontends": "beast port=7480", 2023-10-01T09:14:21.975 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_gc_max_concurrent_io": "10", 2023-10-01T09:14:21.975 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_gc_max_deferred": "50", 2023-10-01T09:14:21.975 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-10-01T09:14:21.975 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_gc_max_objs": "32", 2023-10-01T09:14:21.975 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_gc_max_queue_size": "134213632", 2023-10-01T09:14:21.976 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_gc_max_trim_chunk": "16", 2023-10-01T09:14:21.976 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_gc_obj_min_wait": "7200", 2023-10-01T09:14:21.976 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_gc_processor_max_time": "3600", 2023-10-01T09:14:21.976 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_gc_processor_period": "3600", 2023-10-01T09:14:21.976 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-10-01T09:14:21.976 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_get_obj_window_size": "16777216", 2023-10-01T09:14:21.976 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_healthcheck_disabling_path": "", 2023-10-01T09:14:21.976 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_ignore_get_invalid_range": "false", 2023-10-01T09:14:21.976 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_init_timeout": "300", 2023-10-01T09:14:21.976 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-10-01T09:14:21.976 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-10-01T09:14:21.976 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-10-01T09:14:21.977 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-10-01T09:14:21.977 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_admin_domain": "", 2023-10-01T09:14:21.977 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_admin_password": "", 2023-10-01T09:14:21.977 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_admin_password_path": "", 2023-10-01T09:14:21.977 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_admin_project": "", 2023-10-01T09:14:21.977 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_admin_tenant": "", 2023-10-01T09:14:21.977 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_admin_token": "", 2023-10-01T09:14:21.977 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_admin_token_path": "", 2023-10-01T09:14:21.977 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_admin_user": "", 2023-10-01T09:14:21.977 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_api_version": "2", 2023-10-01T09:14:21.977 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_barbican_domain": "", 2023-10-01T09:14:21.978 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_barbican_password": "", 2023-10-01T09:14:21.978 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_barbican_project": "", 2023-10-01T09:14:21.978 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_barbican_tenant": "", 2023-10-01T09:14:21.978 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_barbican_user": "", 2023-10-01T09:14:21.978 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-10-01T09:14:21.978 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_implicit_tenants": "false", 2023-10-01T09:14:21.978 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-10-01T09:14:21.978 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_service_token_enabled": "false", 2023-10-01T09:14:21.978 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_token_cache_size": "10000", 2023-10-01T09:14:21.978 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_url": "", 2023-10-01T09:14:21.978 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_keystone_verify_ssl": "true", 2023-10-01T09:14:21.979 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_lc_debug_interval": "-1", 2023-10-01T09:14:21.979 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_lc_lock_max_time": "90", 2023-10-01T09:14:21.979 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_lc_max_objs": "32", 2023-10-01T09:14:21.979 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_lc_max_rules": "1000", 2023-10-01T09:14:21.979 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_lc_max_worker": "3", 2023-10-01T09:14:21.979 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_lc_max_wp_worker": "3", 2023-10-01T09:14:21.979 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_lc_thread_delay": "0", 2023-10-01T09:14:21.979 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-10-01T09:14:21.979 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_ldap_dnattr": "uid", 2023-10-01T09:14:21.979 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-10-01T09:14:21.979 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_ldap_searchfilter": "", 2023-10-01T09:14:21.979 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-10-01T09:14:21.980 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_ldap_uri": "ldaps://", 2023-10-01T09:14:21.980 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-10-01T09:14:21.980 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-10-01T09:14:21.980 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-10-01T09:14:21.980 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_log_http_headers": "", 2023-10-01T09:14:21.980 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_log_nonexistent_bucket": "false", 2023-10-01T09:14:21.980 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-10-01T09:14:21.980 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_log_object_name_utc": "false", 2023-10-01T09:14:21.980 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-10-01T09:14:21.980 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_max_attr_name_len": "0", 2023-10-01T09:14:21.980 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_max_attr_size": "0", 2023-10-01T09:14:21.981 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_max_attrs_num_in_req": "0", 2023-10-01T09:14:21.981 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_max_chunk_size": "4194304", 2023-10-01T09:14:21.981 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_max_concurrent_requests": "1024", 2023-10-01T09:14:21.981 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-10-01T09:14:21.981 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_max_dynamic_shards": "1999", 2023-10-01T09:14:21.981 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_max_listing_results": "1000", 2023-10-01T09:14:21.981 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_max_notify_retries": "10", 2023-10-01T09:14:21.981 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_max_objs_per_shard": "100000", 2023-10-01T09:14:21.981 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_max_put_param_size": "1048576", 2023-10-01T09:14:21.981 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_max_put_size": "5368709120", 2023-10-01T09:14:21.981 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_max_slo_entries": "1000", 2023-10-01T09:14:21.982 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_md_log_max_shards": "64", 2023-10-01T09:14:21.982 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_md_notify_interval_msec": "200", 2023-10-01T09:14:21.982 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_meta_sync_poll_interval": "20", 2023-10-01T09:14:21.982 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_meta_sync_spawn_window": "20", 2023-10-01T09:14:21.982 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-10-01T09:14:21.982 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_mp_lock_max_time": "600", 2023-10-01T09:14:21.982 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-10-01T09:14:21.982 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_multipart_min_part_size": "5242880", 2023-10-01T09:14:21.982 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-10-01T09:14:21.982 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-10-01T09:14:21.983 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_nfs_fhcache_size": "2017", 2023-10-01T09:14:21.983 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-10-01T09:14:21.983 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-10-01T09:14:21.983 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_nfs_lru_lanes": "5", 2023-10-01T09:14:21.983 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_nfs_max_gc": "300", 2023-10-01T09:14:21.983 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-10-01T09:14:21.983 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_nfs_run_gc_threads": "false", 2023-10-01T09:14:21.983 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_nfs_run_lc_threads": "false", 2023-10-01T09:14:21.983 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_nfs_run_quota_threads": "false", 2023-10-01T09:14:21.983 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_nfs_run_sync_thread": "false", 2023-10-01T09:14:21.983 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-10-01T09:14:21.983 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-10-01T09:14:21.984 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_num_async_rados_threads": "32", 2023-10-01T09:14:21.984 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_num_control_oids": "8", 2023-10-01T09:14:21.984 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_numa_node": "-1", 2023-10-01T09:14:21.984 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_obj_stripe_size": "4194304", 2023-10-01T09:14:21.984 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-10-01T09:14:21.984 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_objexp_chunk_size": "100", 2023-10-01T09:14:21.984 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_objexp_gc_interval": "600", 2023-10-01T09:14:21.984 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_objexp_hints_num_shards": "127", 2023-10-01T09:14:21.984 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-10-01T09:14:21.984 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-10-01T09:14:21.984 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_op_thread_timeout": "600", 2023-10-01T09:14:21.985 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_op_tracing": "false", 2023-10-01T09:14:21.985 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_opa_token": "", 2023-10-01T09:14:21.985 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_opa_url": "", 2023-10-01T09:14:21.985 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_opa_verify_ssl": "true", 2023-10-01T09:14:21.985 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-10-01T09:14:21.985 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-10-01T09:14:21.985 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_ops_log_rados": "false", 2023-10-01T09:14:21.985 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_ops_log_socket_path": "", 2023-10-01T09:14:21.985 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-10-01T09:14:21.985 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-10-01T09:14:21.985 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-10-01T09:14:21.986 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_period_push_interval": "2.000000", 2023-10-01T09:14:21.986 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_period_push_interval_max": "30.000000", 2023-10-01T09:14:21.986 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_period_root_pool": ".rgw.root", 2023-10-01T09:14:21.986 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-10-01T09:14:21.986 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_print_continue": "true", 2023-10-01T09:14:21.986 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_print_prohibited_content_length": "false", 2023-10-01T09:14:21.986 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-10-01T09:14:21.986 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-10-01T09:14:21.986 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-10-01T09:14:21.986 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-10-01T09:14:21.986 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_rados_tracing": "false", 2023-10-01T09:14:21.987 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_realm": "", 2023-10-01T09:14:21.987 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_realm_id": "", 2023-10-01T09:14:21.987 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-10-01T09:14:21.987 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_region": "", 2023-10-01T09:14:21.987 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_region_root_pool": ".rgw.root", 2023-10-01T09:14:21.987 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_relaxed_region_enforcement": "false", 2023-10-01T09:14:21.987 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-10-01T09:14:21.987 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-10-01T09:14:21.987 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_request_uri": "", 2023-10-01T09:14:21.987 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_reshard_batch_size": "64", 2023-10-01T09:14:21.988 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-10-01T09:14:21.988 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_reshard_max_aio": "128", 2023-10-01T09:14:21.988 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_reshard_num_logs": "16", 2023-10-01T09:14:21.988 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_reshard_thread_interval": "600", 2023-10-01T09:14:21.988 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_resolve_cname": "false", 2023-10-01T09:14:21.988 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_rest_getusage_op_compat": "false", 2023-10-01T09:14:21.988 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_run_sync_thread": "true", 2023-10-01T09:14:21.988 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-10-01T09:14:21.988 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_s3_auth_use_keystone": "false", 2023-10-01T09:14:21.988 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_s3_auth_use_ldap": "false", 2023-10-01T09:14:21.988 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_s3_auth_use_rados": "true", 2023-10-01T09:14:21.989 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_s3_auth_use_sts": "false", 2023-10-01T09:14:21.989 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-10-01T09:14:21.989 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_s3_success_create_obj_status": "0", 2023-10-01T09:14:21.989 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-10-01T09:14:21.989 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_scheduler_type": "throttler", 2023-10-01T09:14:21.989 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_script_uri": "", 2023-10-01T09:14:21.989 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_service_provider_name": "", 2023-10-01T09:14:21.989 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-10-01T09:14:21.989 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sts_client_id": "", 2023-10-01T09:14:21.989 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sts_client_secret": "", 2023-10-01T09:14:21.990 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sts_entry": "sts", 2023-10-01T09:14:21.990 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sts_key": "sts", 2023-10-01T09:14:21.990 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sts_max_session_duration": "43200", 2023-10-01T09:14:21.990 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sts_min_session_duration": "900", 2023-10-01T09:14:21.990 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sts_token_introspection_url": "", 2023-10-01T09:14:21.990 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_swift_account_in_url": "false", 2023-10-01T09:14:21.990 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_swift_auth_entry": "auth", 2023-10-01T09:14:21.990 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_swift_auth_url": "", 2023-10-01T09:14:21.990 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_swift_custom_header": "", 2023-10-01T09:14:21.991 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_swift_enforce_content_length": "false", 2023-10-01T09:14:21.991 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_swift_need_stats": "true", 2023-10-01T09:14:21.991 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_swift_tenant_name": "", 2023-10-01T09:14:21.991 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_swift_token_expiration": "86400", 2023-10-01T09:14:21.991 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_swift_url": "", 2023-10-01T09:14:21.991 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_swift_url_prefix": "swift", 2023-10-01T09:14:21.991 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_swift_versioning_enabled": "false", 2023-10-01T09:14:21.991 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-10-01T09:14:21.991 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-10-01T09:14:21.992 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sync_lease_period": "120", 2023-10-01T09:14:21.992 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-10-01T09:14:21.992 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sync_log_trim_interval": "1200", 2023-10-01T09:14:21.992 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-10-01T09:14:21.992 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-10-01T09:14:21.992 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-10-01T09:14:21.992 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sync_obj_etag_verify": "false", 2023-10-01T09:14:21.992 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sync_trace_history_size": "4096", 2023-10-01T09:14:21.992 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-10-01T09:14:21.992 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-10-01T09:14:21.992 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_thread_pool_size": "512", 2023-10-01T09:14:21.992 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_torrent_comment": "", 2023-10-01T09:14:21.993 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_torrent_createby": "", 2023-10-01T09:14:21.993 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_torrent_encoding": "", 2023-10-01T09:14:21.993 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_torrent_flag": "false", 2023-10-01T09:14:21.993 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_torrent_origin": "", 2023-10-01T09:14:21.993 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_torrent_sha_unit": "524288", 2023-10-01T09:14:21.993 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_torrent_tracker": "", 2023-10-01T09:14:21.993 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_trust_forwarded_https": "false", 2023-10-01T09:14:21.993 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-10-01T09:14:21.993 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_usage_log_tick_interval": "30", 2023-10-01T09:14:21.993 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_usage_max_shards": "32", 2023-10-01T09:14:21.993 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_usage_max_user_shards": "1", 2023-10-01T09:14:21.994 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_use_opa_authz": "false", 2023-10-01T09:14:21.994 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-10-01T09:14:21.994 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_user_default_quota_max_size": "-1", 2023-10-01T09:14:21.994 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_user_max_buckets": "1000", 2023-10-01T09:14:21.994 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_user_policies_max_num": "100", 2023-10-01T09:14:21.994 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-10-01T09:14:21.994 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-10-01T09:14:21.994 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_user_quota_sync_interval": "86400", 2023-10-01T09:14:21.994 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-10-01T09:14:21.994 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_user_unique_email": "true", 2023-10-01T09:14:21.994 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_verify_ssl": "true", 2023-10-01T09:14:21.995 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_website_routing_rules_max_num": "50", 2023-10-01T09:14:21.995 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_zone": "", 2023-10-01T09:14:21.995 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_zone_id": "", 2023-10-01T09:14:21.995 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-10-01T09:14:21.995 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_zonegroup": "", 2023-10-01T09:14:21.995 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_zonegroup_id": "", 2023-10-01T09:14:21.995 INFO:teuthology.orchestra.run.smithi138.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-10-01T09:14:21.995 INFO:teuthology.orchestra.run.smithi138.stdout: "rgwlc_auto_session_clear": "true", 2023-10-01T09:14:21.995 INFO:teuthology.orchestra.run.smithi138.stdout: "rgwlc_skip_bucket_step": "false", 2023-10-01T09:14:21.995 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_block_size": "4096", 2023-10-01T09:14:21.995 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-10-01T09:14:21.996 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-10-01T09:14:21.996 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-10-01T09:14:21.996 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-10-01T09:14:21.996 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_cache_shard_bits": "4", 2023-10-01T09:14:21.996 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_cache_size": "536870912", 2023-10-01T09:14:21.996 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_cache_type": "binned_lru", 2023-10-01T09:14:21.996 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-10-01T09:14:21.996 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-10-01T09:14:21.996 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-10-01T09:14:21.996 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_collect_compaction_stats": "false", 2023-10-01T09:14:21.996 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_collect_extended_stats": "false", 2023-10-01T09:14:21.997 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_collect_memory_stats": "false", 2023-10-01T09:14:21.997 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-10-01T09:14:21.997 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_index_type": "binary_search", 2023-10-01T09:14:21.997 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_log_to_ceph_log": "true", 2023-10-01T09:14:21.997 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_metadata_block_size": "4096", 2023-10-01T09:14:21.997 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_partition_filters": "false", 2023-10-01T09:14:21.997 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_perf": "false", 2023-10-01T09:14:21.997 INFO:teuthology.orchestra.run.smithi138.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-10-01T09:14:21.997 INFO:teuthology.orchestra.run.smithi138.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-10-01T09:14:21.997 INFO:teuthology.orchestra.run.smithi138.stdout: "rotating_keys_renewal_timeout": "10", 2023-10-01T09:14:21.997 INFO:teuthology.orchestra.run.smithi138.stdout: "run_dir": "/var/run/ceph", 2023-10-01T09:14:21.997 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_block_create": "true", 2023-10-01T09:14:21.998 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_cache_lru_size": "67108864", 2023-10-01T09:14:21.998 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_cbjournal_size": "5368709120", 2023-10-01T09:14:21.998 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_default_max_object_size": "16777216", 2023-10-01T09:14:21.998 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-10-01T09:14:21.998 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_device_size": "53687091200", 2023-10-01T09:14:21.998 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_journal_batch_capacity": "16", 2023-10-01T09:14:21.998 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-10-01T09:14:21.998 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-10-01T09:14:21.998 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_journal_iodepth_limit": "5", 2023-10-01T09:14:21.998 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_main_device_type": "SSD", 2023-10-01T09:14:21.998 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_max_concurrent_transactions": "8", 2023-10-01T09:14:21.998 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-10-01T09:14:21.999 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-10-01T09:14:21.999 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-10-01T09:14:21.999 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-10-01T09:14:21.999 INFO:teuthology.orchestra.run.smithi138.stdout: "seastore_segment_size": "67108864", 2023-10-01T09:14:21.999 INFO:teuthology.orchestra.run.smithi138.stdout: "set_keepcaps": "false", 2023-10-01T09:14:21.999 INFO:teuthology.orchestra.run.smithi138.stdout: "setgroup": "ceph", 2023-10-01T09:14:21.999 INFO:teuthology.orchestra.run.smithi138.stdout: "setuser": "ceph", 2023-10-01T09:14:21.999 INFO:teuthology.orchestra.run.smithi138.stdout: "setuser_match_path": "", 2023-10-01T09:14:21.999 INFO:teuthology.orchestra.run.smithi138.stdout: "target_max_misplaced_ratio": "0.050000", 2023-10-01T09:14:21.999 INFO:teuthology.orchestra.run.smithi138.stdout: "thp": "false", 2023-10-01T09:14:21.999 INFO:teuthology.orchestra.run.smithi138.stdout: "threadpool_default_timeout": "60", 2023-10-01T09:14:22.000 INFO:teuthology.orchestra.run.smithi138.stdout: "threadpool_empty_queue_max_wait": "2", 2023-10-01T09:14:22.000 INFO:teuthology.orchestra.run.smithi138.stdout: "throttler_perf_counter": "true" 2023-10-01T09:14:22.000 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:14:22.589 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:22 smithi138 ceph-mon[185886]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:22.589 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:22 smithi138 ceph-mon[192513]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:22.610 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-10-01T09:14:22.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:22 smithi190 ceph-mon[160435]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:22.961 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:24.393 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:24 smithi138 ceph-mon[185886]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:24.394 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:24 smithi138 ceph-mon[192513]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:24.599 INFO:teuthology.orchestra.run.smithi138.stdout:mon.a: {} 2023-10-01T09:14:24.610 INFO:teuthology.orchestra.run.smithi138.stdout:mon.b: {} 2023-10-01T09:14:24.619 INFO:teuthology.orchestra.run.smithi138.stdout:mon.c: {} 2023-10-01T09:14:24.620 INFO:teuthology.orchestra.run.smithi138.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-01T09:14:24.620 INFO:teuthology.orchestra.run.smithi138.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-01T09:14:24.621 INFO:teuthology.orchestra.run.smithi138.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-01T09:14:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:24 smithi190 ceph-mon[160435]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:25.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:14:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:14:24] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.33.4" 2023-10-01T09:14:25.387 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-10-01T09:14:25.387 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-01T09:14:25.388 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T09:14:25.390 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T09:14:25.390 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-10-01T09:14:25.390 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-10-01T09:14:25.390 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-10-01T09:14:25.391 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:14:25.391 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-01T09:14:25.391 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-10-01T09:14:25.391 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:14:25.392 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-10-01T09:14:25.392 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:14:25.392 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-10-01T09:14:25.392 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:14:25.393 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-10-01T09:14:25.393 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:14:25.434 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:25.532 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:14:25.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:25.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:14:25.641 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:14:25.703 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:25 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3004221407' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T09:14:25.703 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:14:25.704 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:25 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3004221407' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T09:14:25.704 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:14:25.708 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:25.779 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:14:25.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:25.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:14:25.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:25 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3004221407' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T09:14:25.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:14:25.840 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:25.883 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:14:25.899 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:25.915 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:25.984 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:14:25.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:25.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:14:26.088 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:14:26.120 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:26.382 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:26.382 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:26 smithi138 ceph-mon[192513]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:26 smithi138 ceph-mon[185886]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:26.728 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:14:26.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:26.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:14:26.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:26 smithi190 ceph-mon[160435]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:26.832 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:14:26.883 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:26.933 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:14:26.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:26.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:14:27.042 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:14:27.070 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:27.113 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:14:27.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:27.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:14:27.220 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:14:27.246 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:27.286 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:14:27.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:27.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:14:27.394 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:14:27.425 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:27.725 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:14:27.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:27.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:14:27.829 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:14:27.857 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:27 smithi138 ceph-mon[192513]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:28.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:27 smithi138 ceph-mon[185886]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:28.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:27 smithi190 ceph-mon[160435]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:28.353 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:14:28.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:28.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:14:28.457 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:14:28.488 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:28.536 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:14:28.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:28.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:14:28.644 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:14:28.672 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:29.570 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:14:29.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:29.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:14:29.674 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:14:29.701 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:30.043 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:14:30.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:30.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:14:30.122 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:30.147 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:14:30.175 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:30.220 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:14:30.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:30.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:14:30.326 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:14:30.350 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.0 to 1 (10000) 2023-10-01T09:14:30.351 INFO:teuthology.orchestra.run.smithi138.stderr:pool 'unique_pool_0' created 2023-10-01T09:14:30.365 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:30.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:30 smithi190 ceph-mon[160435]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:30.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:30 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/520467328' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-01T09:14:30.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:30 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-01T09:14:30.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:30 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/351608030' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:14:30.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:30 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:14:30.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:30 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:14:30.598 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:30 smithi138 ceph-mon[185886]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:30.599 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:30 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/520467328' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-01T09:14:30.599 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:30 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-01T09:14:30.599 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:30 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/351608030' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:14:30.599 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:30 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:14:30.599 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:30 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:14:30.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:30 smithi138 ceph-mon[192513]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:30.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:30 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/520467328' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-01T09:14:30.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:30 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-01T09:14:30.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:30 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/351608030' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:14:30.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:30 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:14:30.600 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:30 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:14:30.656 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:14:30.656 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:14:30.656 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-10-01T09:14:30.656 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:14:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:31 smithi138 ceph-mon[192513]: failed to encode map e106 with expected crc 2023-10-01T09:14:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:31 smithi138 ceph-mon[192513]: failed to encode map e106 with expected crc 2023-10-01T09:14:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:31 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-01T09:14:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:31 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:14:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:31 smithi138 ceph-mon[192513]: osdmap e106: 8 total, 8 up, 8 in 2023-10-01T09:14:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/344793197' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:14:31.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:31 smithi138 ceph-mon[185886]: failed to encode map e106 with expected crc 2023-10-01T09:14:31.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:31 smithi138 ceph-mon[185886]: failed to encode map e106 with expected crc 2023-10-01T09:14:31.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:31 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-01T09:14:31.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:31 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:14:31.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:31 smithi138 ceph-mon[185886]: osdmap e106: 8 total, 8 up, 8 in 2023-10-01T09:14:31.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/344793197' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:14:31.620 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:14:31.620 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:14:31.620 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:14:31.620 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:14:31.620 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:14:31.620 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:14:31.620 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:14:31.620 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:14:31.620 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:14:31.621 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:14:31.621 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:14:31.621 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:14:31.621 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:14:31.621 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:14:31.621 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:14:31.621 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:14:31.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:31 smithi190 ceph-mon[160435]: failed to encode map e106 with expected crc 2023-10-01T09:14:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:31 smithi190 ceph-mon[160435]: failed to encode map e106 with expected crc 2023-10-01T09:14:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:31 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-01T09:14:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:31 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:14:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:31 smithi190 ceph-mon[160435]: osdmap e106: 8 total, 8 up, 8 in 2023-10-01T09:14:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/344793197' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:14:32.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:32 smithi138 ceph-mon[192513]: pgmap v222: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:32.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:32 smithi138 ceph-mon[192513]: osdmap e107: 8 total, 8 up, 8 in 2023-10-01T09:14:32.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:32 smithi138 ceph-mon[192513]: failed to encode map e107 with expected crc 2023-10-01T09:14:32.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:32 smithi138 ceph-mon[192513]: failed to encode map e107 with expected crc 2023-10-01T09:14:32.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:32 smithi138 ceph-mon[192513]: failed to encode map e107 with expected crc 2023-10-01T09:14:32.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:32 smithi138 ceph-mon[192513]: failed to encode map e107 with expected crc 2023-10-01T09:14:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:32 smithi138 ceph-mon[185886]: pgmap v222: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:32.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:32 smithi138 ceph-mon[185886]: osdmap e107: 8 total, 8 up, 8 in 2023-10-01T09:14:32.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:32 smithi138 ceph-mon[185886]: failed to encode map e107 with expected crc 2023-10-01T09:14:32.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:32 smithi138 ceph-mon[185886]: failed to encode map e107 with expected crc 2023-10-01T09:14:32.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:32 smithi138 ceph-mon[185886]: failed to encode map e107 with expected crc 2023-10-01T09:14:32.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:32 smithi138 ceph-mon[185886]: failed to encode map e107 with expected crc 2023-10-01T09:14:32.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:32 smithi190 ceph-mon[160435]: pgmap v222: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:32 smithi190 ceph-mon[160435]: osdmap e107: 8 total, 8 up, 8 in 2023-10-01T09:14:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:32 smithi190 ceph-mon[160435]: failed to encode map e107 with expected crc 2023-10-01T09:14:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:32 smithi190 ceph-mon[160435]: failed to encode map e107 with expected crc 2023-10-01T09:14:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:32 smithi190 ceph-mon[160435]: failed to encode map e107 with expected crc 2023-10-01T09:14:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:32 smithi190 ceph-mon[160435]: failed to encode map e107 with expected crc 2023-10-01T09:14:33.058 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:14:33.070 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:14:33.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:33.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:14:33.174 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:14:33.203 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:33.247 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:14:33.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:33.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:14:33.353 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:14:33.379 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:33.495 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-01T09:14:33.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:33 smithi190 ceph-mon[160435]: failed to encode map e108 with expected crc 2023-10-01T09:14:33.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:33 smithi190 ceph-mon[160435]: osdmap e108: 8 total, 8 up, 8 in 2023-10-01T09:14:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:33 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:33 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:33 smithi190 ceph-mon[160435]: failed to encode map e108 with expected crc 2023-10-01T09:14:33.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:33 smithi138 ceph-mon[192513]: failed to encode map e108 with expected crc 2023-10-01T09:14:33.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:33 smithi138 ceph-mon[192513]: osdmap e108: 8 total, 8 up, 8 in 2023-10-01T09:14:33.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:33 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:33.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:33 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:33.778 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:33 smithi138 ceph-mon[192513]: failed to encode map e108 with expected crc 2023-10-01T09:14:33.779 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:33 smithi138 ceph-mon[185886]: failed to encode map e108 with expected crc 2023-10-01T09:14:33.779 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:33 smithi138 ceph-mon[185886]: osdmap e108: 8 total, 8 up, 8 in 2023-10-01T09:14:33.780 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:33 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:33.780 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:33 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:33.780 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:33 smithi138 ceph-mon[185886]: failed to encode map e108 with expected crc 2023-10-01T09:14:33.798 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-10-01T09:14:33.886 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:14:33.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:33.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:14:33.997 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:14:34.037 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:34.140 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:14:34.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:34.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:14:34.244 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:14:34.277 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:34.326 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:14:34.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:34.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:14:34.384 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:34.431 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:14:34.459 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:34.497 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:14:34.503 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:14:34.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:34.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:14:34.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:34 smithi190 ceph-mon[160435]: failed to encode map e108 with expected crc 2023-10-01T09:14:34.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:34 smithi190 ceph-mon[160435]: pgmap v225: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:34 smithi190 ceph-mon[160435]: failed to encode map e108 with expected crc 2023-10-01T09:14:34.610 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:14:34.635 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:34 smithi138 ceph-mon[185886]: failed to encode map e108 with expected crc 2023-10-01T09:14:34.635 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:34 smithi138 ceph-mon[185886]: pgmap v225: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:34.635 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:34 smithi138 ceph-mon[185886]: failed to encode map e108 with expected crc 2023-10-01T09:14:34.636 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:34 smithi138 ceph-mon[192513]: failed to encode map e108 with expected crc 2023-10-01T09:14:34.636 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:34 smithi138 ceph-mon[192513]: pgmap v225: 89 pgs: 7 creating+peering, 5 unknown, 77 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:34.636 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:34 smithi138 ceph-mon[192513]: failed to encode map e108 with expected crc 2023-10-01T09:14:34.638 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:34.808 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:34.844 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:14:34.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:34.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:14:34.948 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:14:34.978 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:35.125 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:14:35.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:35.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:14:35.229 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:14:35.261 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:35.310 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:14:35.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:35.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:14:35.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:14:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:14:34] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.33.4" 2023-10-01T09:14:35.357 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:14:35.415 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:14:35.443 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:35.487 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:14:35.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:35.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:14:35.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/621638357' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:14:35.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:35 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:14:35.592 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:14:35.617 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/621638357' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:14:35.617 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:35 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:14:35.617 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/621638357' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:14:35.617 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:35 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:14:35.620 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:35.915 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:14:35.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:35.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:14:36.019 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:14:36.046 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:36.089 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:14:36.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:36.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:14:36.194 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:14:36.220 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:36.261 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:14:36.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:36.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:14:36.368 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:14:36.397 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:36.460 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:36.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:36 smithi138 ceph-mon[192513]: pgmap v226: 89 pgs: 7 creating+peering, 82 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:36.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:36 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:14:36.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:36 smithi138 ceph-mon[192513]: failed to encode map e109 with expected crc 2023-10-01T09:14:36.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:36 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:14:36.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:36 smithi138 ceph-mon[192513]: osdmap e109: 8 total, 8 up, 8 in 2023-10-01T09:14:36.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:36 smithi138 ceph-mon[185886]: pgmap v226: 89 pgs: 7 creating+peering, 82 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:36.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:36 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:14:36.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:36 smithi138 ceph-mon[185886]: failed to encode map e109 with expected crc 2023-10-01T09:14:36.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:36 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:14:36.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:36 smithi138 ceph-mon[185886]: osdmap e109: 8 total, 8 up, 8 in 2023-10-01T09:14:36.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:36 smithi190 ceph-mon[160435]: pgmap v226: 89 pgs: 7 creating+peering, 82 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:36 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:14:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:36 smithi190 ceph-mon[160435]: failed to encode map e109 with expected crc 2023-10-01T09:14:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:36 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:14:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:36 smithi190 ceph-mon[160435]: osdmap e109: 8 total, 8 up, 8 in 2023-10-01T09:14:37.324 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:37.351 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:14:37.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:37.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:14:37.455 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:14:37.488 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:37.535 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:14:37.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:37.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:14:37.640 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:14:37.666 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:37.941 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.1 to 1 (10000) 2023-10-01T09:14:38.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:37 smithi138 ceph-mon[192513]: failed to encode map e109 with expected crc 2023-10-01T09:14:38.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:37 smithi138 ceph-mon[192513]: failed to encode map e109 with expected crc 2023-10-01T09:14:38.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:37 smithi138 ceph-mon[192513]: pgmap v228: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:38.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:37 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1189975505' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:14:38.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:37 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:14:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:37 smithi138 ceph-mon[185886]: failed to encode map e109 with expected crc 2023-10-01T09:14:38.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:37 smithi138 ceph-mon[185886]: failed to encode map e109 with expected crc 2023-10-01T09:14:38.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:37 smithi138 ceph-mon[185886]: pgmap v228: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:38.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:37 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1189975505' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:14:38.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:37 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:14:38.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:37 smithi190 ceph-mon[160435]: failed to encode map e109 with expected crc 2023-10-01T09:14:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:37 smithi190 ceph-mon[160435]: failed to encode map e109 with expected crc 2023-10-01T09:14:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:37 smithi190 ceph-mon[160435]: pgmap v228: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:37 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1189975505' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:14:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:37 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:14:38.921 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:14:39.069 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:14:39.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:39.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:14:39.172 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:14:39.197 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:38 smithi138 ceph-mon[192513]: failed to encode map e110 with expected crc 2023-10-01T09:14:39.197 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:38 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:14:39.198 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:38 smithi138 ceph-mon[192513]: osdmap e110: 8 total, 8 up, 8 in 2023-10-01T09:14:39.198 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:38 smithi138 ceph-mon[185886]: failed to encode map e110 with expected crc 2023-10-01T09:14:39.198 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:38 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:14:39.198 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:38 smithi138 ceph-mon[185886]: osdmap e110: 8 total, 8 up, 8 in 2023-10-01T09:14:39.200 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:39.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:38 smithi190 ceph-mon[160435]: failed to encode map e110 with expected crc 2023-10-01T09:14:39.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:38 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:14:39.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:38 smithi190 ceph-mon[160435]: osdmap e110: 8 total, 8 up, 8 in 2023-10-01T09:14:39.452 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:14:39.452 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:14:40.253 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:39 smithi138 ceph-mon[192513]: failed to encode map e110 with expected crc 2023-10-01T09:14:40.254 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:39 smithi138 ceph-mon[192513]: pgmap v230: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:40.254 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:39 smithi138 ceph-mon[185886]: failed to encode map e110 with expected crc 2023-10-01T09:14:40.254 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:39 smithi138 ceph-mon[185886]: pgmap v230: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:40.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:39 smithi190 ceph-mon[160435]: failed to encode map e110 with expected crc 2023-10-01T09:14:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:39 smithi190 ceph-mon[160435]: pgmap v230: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:40.350 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-01T09:14:40.554 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:14:40.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:40.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:14:40.658 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:14:40.668 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:14:40.691 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:40.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:40.900 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:14:40.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:14:41.001 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:14:41.033 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:41.080 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:14:41.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:41.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:14:41.185 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:14:41.214 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:41.256 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:14:41.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:41.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:14:41.270 INFO:teuthology.orchestra.run.smithi138.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-10-01T09:14:41.292 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:41.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:14:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:41 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3353678930' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:14:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:41 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:14:41.364 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:14:41.395 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:41.479 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:14:41.493 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:14:41.495 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:41.495 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:14:41.496 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:41 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3353678930' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:14:41.496 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:41 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:14:41.496 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:41.496 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:14:41.496 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:41 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3353678930' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:14:41.496 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:41 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:14:41.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:41.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:14:41.503 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:14:41.525 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:14:41.550 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:14:41.577 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:14:41.597 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:14:41.598 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:14:41.617 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:14:41.626 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:41.637 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:14:41.639 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:14:41.639 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:14:41.639 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:14:41.639 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:14:41.639 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:14:41.639 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:14:41.640 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:14:41.640 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:14:41.728 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:14:41.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:41.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:14:41.739 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:41.833 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:14:41.876 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:42.322 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:14:42.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:42.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:14:42.425 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:14:42.450 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:14:42.450 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[185886]: osdmap e111: 8 total, 8 up, 8 in 2023-10-01T09:14:42.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[185886]: pgmap v232: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:42.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1411547289' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:14:42.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[185886]: failed to encode map e111 with expected crc 2023-10-01T09:14:42.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[185886]: failed to encode map e111 with expected crc 2023-10-01T09:14:42.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[185886]: failed to encode map e111 with expected crc 2023-10-01T09:14:42.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[185886]: failed to encode map e111 with expected crc 2023-10-01T09:14:42.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[185886]: failed to encode map e111 with expected crc 2023-10-01T09:14:42.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:14:42.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[192513]: osdmap e111: 8 total, 8 up, 8 in 2023-10-01T09:14:42.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[192513]: pgmap v232: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:42.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1411547289' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:14:42.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[192513]: failed to encode map e111 with expected crc 2023-10-01T09:14:42.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[192513]: failed to encode map e111 with expected crc 2023-10-01T09:14:42.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[192513]: failed to encode map e111 with expected crc 2023-10-01T09:14:42.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[192513]: failed to encode map e111 with expected crc 2023-10-01T09:14:42.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:42 smithi138 ceph-mon[192513]: failed to encode map e111 with expected crc 2023-10-01T09:14:42.453 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:42.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:42 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:14:42.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:42 smithi190 ceph-mon[160435]: osdmap e111: 8 total, 8 up, 8 in 2023-10-01T09:14:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:42 smithi190 ceph-mon[160435]: pgmap v232: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:42 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1411547289' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:14:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:42 smithi190 ceph-mon[160435]: failed to encode map e111 with expected crc 2023-10-01T09:14:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:42 smithi190 ceph-mon[160435]: failed to encode map e111 with expected crc 2023-10-01T09:14:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:42 smithi190 ceph-mon[160435]: failed to encode map e111 with expected crc 2023-10-01T09:14:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:42 smithi190 ceph-mon[160435]: failed to encode map e111 with expected crc 2023-10-01T09:14:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:42 smithi190 ceph-mon[160435]: failed to encode map e111 with expected crc 2023-10-01T09:14:43.431 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:43.451 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:14:43.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:43.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:14:43.555 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:14:43.582 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:44.460 DEBUG:teuthology.orchestra.run.smithi138:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-10-01T09:14:44.463 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T09:14:44.463 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T09:14:44.508 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:hints = 1 2023-10-01T09:14:44.508 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T09:14:44.508 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Object prefix: benchmark_data_smithi138_240338 2023-10-01T09:14:44.509 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:14:44.509 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 0 0 0 0 0 0 - 0 2023-10-01T09:14:44.583 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:14:44.589 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:44 smithi138 ceph-mon[192513]: pgmap v233: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:44.590 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:44 smithi138 ceph-mon[185886]: pgmap v233: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:44.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:44.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:14:44.694 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:14:44.762 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:44.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:44 smithi190 ceph-mon[160435]: pgmap v233: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:44.915 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:14:44.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:44.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:14:45.023 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:14:45.081 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:45.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:14:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:14:44] "GET /metrics HTTP/1.1" 200 36258 "" "Prometheus/2.33.4" 2023-10-01T09:14:45.508 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 1 16 3633 3617 226.049 226.062 0.00519152 0.00440749 2023-10-01T09:14:45.539 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:14:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:45 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:45 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:45 smithi190 ceph-mon[160435]: pgmap v234: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:45 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:46.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:45 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:46.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:45 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:46.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:45 smithi138 ceph-mon[192513]: pgmap v234: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:46.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:45 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:46.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:45 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:46.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:45 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:46.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:45 smithi138 ceph-mon[185886]: pgmap v234: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:14:46.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:45 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:46.508 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 2 16 6551 6535 204.204 182.375 0.00472663 0.00488822 2023-10-01T09:14:46.792 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:14:46.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:46.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:14:46.899 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:14:46.953 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:47.389 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:14:47.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:47.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:14:47.494 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:14:47.508 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 3 16 9447 9431 196.464 181 0.00595911 0.00508123 2023-10-01T09:14:47.531 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:47.619 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:14:47.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:47.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:14:47.725 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:14:47.767 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:47.833 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:14:47.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:47.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:14:47.940 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:14:47.975 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:48.015 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:47 smithi138 ceph-mon[185886]: pgmap v235: 89 pgs: 89 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s wr, 387 op/s 2023-10-01T09:14:48.015 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:14:48.015 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:14:48.015 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:48.017 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:47 smithi138 ceph-mon[192513]: pgmap v235: 89 pgs: 89 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s wr, 387 op/s 2023-10-01T09:14:48.017 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:14:48.017 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:14:48.017 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:48.041 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:14:48.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:48.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:14:48.146 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:14:48.149 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:48.184 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:14:48.184 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:14:48.184 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:14:48.184 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:14:48.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.184 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:3621 5.5 5:a241a76b:::benchmark_data_smithi138_240338_object3620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:14:48.185 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.505821+0000", 2023-10-01T09:14:48.185 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.6733268629999998, 2023-10-01T09:14:48.185 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17830392, 2023-10-01T09:14:48.185 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.185 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.185 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:14:48.185 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:14:48.185 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:14:48.185 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 3621 2023-10-01T09:14:48.185 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.185 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.186 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.505821+0000", 2023-10-01T09:14:48.186 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.186 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.186 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.505821+0000", 2023-10-01T09:14:48.186 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.186 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.505820+0000", 2023-10-01T09:14:48.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:14:48.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.505840+0000", 2023-10-01T09:14:48.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0401e-05 2023-10-01T09:14:48.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.505856+0000", 2023-10-01T09:14:48.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6099e-05 2023-10-01T09:14:48.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.189 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.189 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.505864+0000", 2023-10-01T09:14:48.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4349999999999997e-06 2023-10-01T09:14:48.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.189 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.189 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.505911+0000", 2023-10-01T09:14:48.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6281e-05 2023-10-01T09:14:48.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.189 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.190 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506027+0000", 2023-10-01T09:14:48.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011587299999999999 2023-10-01T09:14:48.190 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.190 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-10-01T09:14:48.190 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506158+0000", 2023-10-01T09:14:48.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013091200000000001 2023-10-01T09:14:48.190 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.190 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:14:48.190 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507217+0000", 2023-10-01T09:14:48.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001059171 2023-10-01T09:14:48.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507247+0000", 2023-10-01T09:14:48.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0329e-05 2023-10-01T09:14:48.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.684053+0000", 2023-10-01T09:14:48.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17680601900000001 2023-10-01T09:14:48.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.684094+0000", 2023-10-01T09:14:48.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0868999999999998e-05 2023-10-01T09:14:48.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.684125+0000", 2023-10-01T09:14:48.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0966999999999997e-05 2023-10-01T09:14:48.192 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.192 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.192 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.193 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:3622 5.5 5:a4d596e2:::benchmark_data_smithi138_240338_object3621:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:14:48.193 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.506458+0000", 2023-10-01T09:14:48.193 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.6726902250000002, 2023-10-01T09:14:48.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17818736299999999, 2023-10-01T09:14:48.193 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.193 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.193 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:14:48.193 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:14:48.193 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:14:48.193 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 3622 2023-10-01T09:14:48.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.193 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.194 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.194 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.194 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506458+0000", 2023-10-01T09:14:48.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.194 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.194 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.194 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.194 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506458+0000", 2023-10-01T09:14:48.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.194 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.194 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506457+0000", 2023-10-01T09:14:48.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:14:48.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506479+0000", 2023-10-01T09:14:48.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1991999999999999e-05 2023-10-01T09:14:48.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506491+0000", 2023-10-01T09:14:48.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2140000000000001e-05 2023-10-01T09:14:48.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506498+0000", 2023-10-01T09:14:48.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2960000000000001e-06 2023-10-01T09:14:48.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506597+0000", 2023-10-01T09:14:48.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8276000000000006e-05 2023-10-01T09:14:48.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506709+0000", 2023-10-01T09:14:48.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011270200000000001 2023-10-01T09:14:48.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-10-01T09:14:48.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506832+0000", 2023-10-01T09:14:48.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000122266 2023-10-01T09:14:48.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.198 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:14:48.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507557+0000", 2023-10-01T09:14:48.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00072518699999999997 2023-10-01T09:14:48.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.198 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.508305+0000", 2023-10-01T09:14:48.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00074822500000000004 2023-10-01T09:14:48.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.199 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.684567+0000", 2023-10-01T09:14:48.199 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.176262319 2023-10-01T09:14:48.199 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.199 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.684611+0000", 2023-10-01T09:14:48.199 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3559000000000001e-05 2023-10-01T09:14:48.199 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.199 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.684645+0000", 2023-10-01T09:14:48.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4167e-05 2023-10-01T09:14:48.200 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.200 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.200 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.200 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.200 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.200 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:3635 5.5 5:a08fdf0c:::benchmark_data_smithi138_240338_object3634:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:14:48.200 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.509657+0000", 2023-10-01T09:14:48.200 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.6694910439999999, 2023-10-01T09:14:48.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.175086928, 2023-10-01T09:14:48.200 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.201 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.201 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:14:48.201 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:14:48.201 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:14:48.201 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 3635 2023-10-01T09:14:48.201 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.201 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.201 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.201 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.201 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.509657+0000", 2023-10-01T09:14:48.201 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.201 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.202 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.202 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.202 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.509657+0000", 2023-10-01T09:14:48.202 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.202 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.202 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.202 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.202 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.509656+0000", 2023-10-01T09:14:48.202 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:14:48.202 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.202 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.202 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.203 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.509677+0000", 2023-10-01T09:14:48.203 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1557999999999999e-05 2023-10-01T09:14:48.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.203 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.203 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.203 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.509689+0000", 2023-10-01T09:14:48.203 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1420000000000001e-05 2023-10-01T09:14:48.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.203 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.203 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.203 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.509694+0000", 2023-10-01T09:14:48.203 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2800000000000003e-06 2023-10-01T09:14:48.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.204 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.509743+0000", 2023-10-01T09:14:48.204 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9002000000000003e-05 2023-10-01T09:14:48.204 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.204 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.509856+0000", 2023-10-01T09:14:48.204 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113355 2023-10-01T09:14:48.204 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-10-01T09:14:48.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.509987+0000", 2023-10-01T09:14:48.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013045900000000001 2023-10-01T09:14:48.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.205 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:14:48.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.510454+0000", 2023-10-01T09:14:48.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00046707299999999998 2023-10-01T09:14:48.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.205 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.510986+0000", 2023-10-01T09:14:48.206 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00053182800000000003 2023-10-01T09:14:48.206 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.206 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.206 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.206 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.684677+0000", 2023-10-01T09:14:48.206 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.173691597 2023-10-01T09:14:48.206 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.206 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.206 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.206 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.684713+0000", 2023-10-01T09:14:48.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5658000000000003e-05 2023-10-01T09:14:48.207 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.207 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.207 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.207 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.684744+0000", 2023-10-01T09:14:48.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0896999999999999e-05 2023-10-01T09:14:48.207 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.207 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.207 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.207 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.208 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.208 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:4611 5.2 e111/106 5:4c2a16a7:::benchmark_data_smithi138_240338_object4610:head v 111'277, mlcod=111'274)", 2023-10-01T09:14:48.208 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.939065+0000", 2023-10-01T09:14:48.208 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.2400829629999999, 2023-10-01T09:14:48.208 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.082913684000000001, 2023-10-01T09:14:48.208 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.208 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.208 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.208 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.208 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.208 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939065+0000", 2023-10-01T09:14:48.208 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.209 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.209 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939065+0000", 2023-10-01T09:14:48.209 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.209 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.209 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939064+0000", 2023-10-01T09:14:48.209 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:14:48.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.209 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.210 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939170+0000", 2023-10-01T09:14:48.210 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105586 2023-10-01T09:14:48.210 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.210 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.210 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.210 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939179+0000", 2023-10-01T09:14:48.210 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.251e-06 2023-10-01T09:14:48.210 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.210 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.210 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.210 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939185+0000", 2023-10-01T09:14:48.210 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3419999999999996e-06 2023-10-01T09:14:48.211 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.211 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.211 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.211 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939225+0000", 2023-10-01T09:14:48.211 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9892999999999999e-05 2023-10-01T09:14:48.211 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.211 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.211 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.211 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939244+0000", 2023-10-01T09:14:48.211 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8870000000000001e-05 2023-10-01T09:14:48.211 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.211 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.212 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.212 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.021950+0000", 2023-10-01T09:14:48.212 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.082706273999999996 2023-10-01T09:14:48.212 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.212 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.212 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.212 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.021979+0000", 2023-10-01T09:14:48.212 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8068e-05 2023-10-01T09:14:48.212 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.212 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.212 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.212 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.213 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.213 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:4609 5.e e111/106 5:710239bd:::benchmark_data_smithi138_240338_object4608:head v 111'309, mlcod=111'303)", 2023-10-01T09:14:48.213 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.939110+0000", 2023-10-01T09:14:48.213 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.2400380599999998, 2023-10-01T09:14:48.213 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.082748664, 2023-10-01T09:14:48.213 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.213 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.213 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.213 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.213 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.213 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939110+0000", 2023-10-01T09:14:48.213 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.214 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.214 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.214 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939110+0000", 2023-10-01T09:14:48.214 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.214 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.214 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.214 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939110+0000", 2023-10-01T09:14:48.214 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967296 2023-10-01T09:14:48.214 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.214 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.215 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939245+0000", 2023-10-01T09:14:48.215 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013579400000000001 2023-10-01T09:14:48.215 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.215 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.215 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.215 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939252+0000", 2023-10-01T09:14:48.215 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0199999999999997e-06 2023-10-01T09:14:48.215 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.215 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.215 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.215 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939258+0000", 2023-10-01T09:14:48.215 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2390000000000002e-06 2023-10-01T09:14:48.216 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.216 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.216 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.216 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939299+0000", 2023-10-01T09:14:48.216 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1010000000000002e-05 2023-10-01T09:14:48.216 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.216 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.216 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.216 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.020648+0000", 2023-10-01T09:14:48.216 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.081349793000000004 2023-10-01T09:14:48.216 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.217 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.217 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.217 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.021814+0000", 2023-10-01T09:14:48.217 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001165428 2023-10-01T09:14:48.217 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.217 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.217 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.217 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.021858+0000", 2023-10-01T09:14:48.217 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4595999999999998e-05 2023-10-01T09:14:48.217 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.217 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.217 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.218 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.218 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.218 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.34683.0:4595 5.3 e111/106 ondisk, result = 0)", 2023-10-01T09:14:48.218 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.939151+0000", 2023-10-01T09:14:48.218 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.2399970300000001, 2023-10-01T09:14:48.218 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.081968316999999999, 2023-10-01T09:14:48.218 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.218 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:14:48.218 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.218 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.218 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.218 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939151+0000", 2023-10-01T09:14:48.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.219 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.219 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.219 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.219 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939151+0000", 2023-10-01T09:14:48.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.219 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.219 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.219 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.219 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939150+0000", 2023-10-01T09:14:48.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:14:48.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.220 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.220 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939157+0000", 2023-10-01T09:14:48.220 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5599999999999999e-06 2023-10-01T09:14:48.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.220 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.220 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939167+0000", 2023-10-01T09:14:48.220 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7680000000000003e-06 2023-10-01T09:14:48.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.221 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939173+0000", 2023-10-01T09:14:48.221 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5339999999999996e-06 2023-10-01T09:14:48.221 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.221 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.221 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939213+0000", 2023-10-01T09:14:48.221 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9857999999999997e-05 2023-10-01T09:14:48.221 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.221 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.221 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939221+0000", 2023-10-01T09:14:48.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0809999999999994e-06 2023-10-01T09:14:48.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.222 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.021119+0000", 2023-10-01T09:14:48.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.081897974999999998 2023-10-01T09:14:48.222 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.222 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.222 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.223 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.223 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:4614 5.3 5:c5d13591:::benchmark_data_smithi138_240338_object4613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:14:48.223 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.939949+0000", 2023-10-01T09:14:48.223 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.2391993029999999, 2023-10-01T09:14:48.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.083377051999999993, 2023-10-01T09:14:48.223 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.223 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.223 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:14:48.223 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:14:48.223 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:14:48.223 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 4614 2023-10-01T09:14:48.223 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.224 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.224 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.224 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939949+0000", 2023-10-01T09:14:48.224 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.224 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.224 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939949+0000", 2023-10-01T09:14:48.224 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.225 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.225 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939947+0000", 2023-10-01T09:14:48.225 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:14:48.225 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.225 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.225 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.225 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.939968+0000", 2023-10-01T09:14:48.225 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0489999999999999e-05 2023-10-01T09:14:48.225 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.225 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.225 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.225 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.020659+0000", 2023-10-01T09:14:48.226 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.080691015000000005 2023-10-01T09:14:48.226 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.226 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.226 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.226 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.020667+0000", 2023-10-01T09:14:48.226 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2179999999999999e-06 2023-10-01T09:14:48.226 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.226 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.226 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.226 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.021323+0000", 2023-10-01T09:14:48.226 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00065617500000000005 2023-10-01T09:14:48.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.227 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.227 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.021488+0000", 2023-10-01T09:14:48.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016472500000000001 2023-10-01T09:14:48.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.227 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,6", 2023-10-01T09:14:48.227 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.021628+0000", 2023-10-01T09:14:48.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000139782 2023-10-01T09:14:48.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.228 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:14:48.228 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.022229+0000", 2023-10-01T09:14:48.228 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00060172600000000002 2023-10-01T09:14:48.228 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.228 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.228 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.228 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.023195+0000", 2023-10-01T09:14:48.228 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00096520600000000003 2023-10-01T09:14:48.228 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.228 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.228 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.023252+0000", 2023-10-01T09:14:48.229 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7868999999999998e-05 2023-10-01T09:14:48.229 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.229 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.023294+0000", 2023-10-01T09:14:48.229 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1498999999999997e-05 2023-10-01T09:14:48.229 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.229 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.023326+0000", 2023-10-01T09:14:48.229 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1618000000000001e-05 2023-10-01T09:14:48.230 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.230 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.230 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.230 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.230 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.230 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:4612 5.7 e111/106 5:ec0db43a:::benchmark_data_smithi138_240338_object4611:head v 111'300, mlcod=111'298)", 2023-10-01T09:14:48.230 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.940199+0000", 2023-10-01T09:14:48.230 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.2389485840000001, 2023-10-01T09:14:48.230 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.081671058000000005, 2023-10-01T09:14:48.230 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.230 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.230 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.231 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.231 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.231 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.940199+0000", 2023-10-01T09:14:48.231 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.231 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.231 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.231 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.231 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.940199+0000", 2023-10-01T09:14:48.231 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.231 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.231 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.231 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.940199+0000", 2023-10-01T09:14:48.232 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:14:48.232 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.232 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.940303+0000", 2023-10-01T09:14:48.232 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000104276 2023-10-01T09:14:48.232 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.232 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.020296+0000", 2023-10-01T09:14:48.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.079992726 2023-10-01T09:14:48.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.233 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.020307+0000", 2023-10-01T09:14:48.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1792e-05 2023-10-01T09:14:48.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.233 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.020552+0000", 2023-10-01T09:14:48.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00024451799999999999 2023-10-01T09:14:48.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.234 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.234 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.234 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.020575+0000", 2023-10-01T09:14:48.234 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2643999999999999e-05 2023-10-01T09:14:48.234 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.234 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.234 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.234 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.021820+0000", 2023-10-01T09:14:48.234 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012458599999999999 2023-10-01T09:14:48.234 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.235 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.235 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.235 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.021870+0000", 2023-10-01T09:14:48.235 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9805999999999997e-05 2023-10-01T09:14:48.235 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.235 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.235 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.235 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.235 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.235 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:4618 5.c e111/106 5:338bc560:::benchmark_data_smithi138_240338_object4617:head v 111'282, mlcod=111'280)", 2023-10-01T09:14:48.235 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.941010+0000", 2023-10-01T09:14:48.235 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.2381383010000002, 2023-10-01T09:14:48.236 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.080853622999999999, 2023-10-01T09:14:48.236 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.236 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.236 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.236 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.236 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.236 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.941010+0000", 2023-10-01T09:14:48.236 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.236 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.236 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.236 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.236 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.941010+0000", 2023-10-01T09:14:48.237 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.237 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.237 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.237 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.237 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.941009+0000", 2023-10-01T09:14:48.237 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:14:48.237 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.237 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.237 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.237 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.941030+0000", 2023-10-01T09:14:48.237 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0769999999999999e-05 2023-10-01T09:14:48.237 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.238 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.238 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.238 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.020337+0000", 2023-10-01T09:14:48.238 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.079307218999999998 2023-10-01T09:14:48.238 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.238 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.238 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.238 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.020348+0000", 2023-10-01T09:14:48.238 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0859999999999999e-05 2023-10-01T09:14:48.238 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.238 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.239 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.239 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.020708+0000", 2023-10-01T09:14:48.239 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00036030200000000001 2023-10-01T09:14:48.239 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.239 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.239 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.239 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.020730+0000", 2023-10-01T09:14:48.239 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1729000000000002e-05 2023-10-01T09:14:48.239 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.239 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.239 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.239 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.021822+0000", 2023-10-01T09:14:48.240 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010927090000000001 2023-10-01T09:14:48.240 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.240 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.240 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.021863+0000", 2023-10-01T09:14:48.240 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0726000000000003e-05 2023-10-01T09:14:48.240 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.240 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.240 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.240 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.240 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:6699 5.3 5:ce479e8a:::benchmark_data_smithi138_240338_object6698:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:14:48.241 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:46.612797+0000", 2023-10-01T09:14:48.241 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.5663508240000001, 2023-10-01T09:14:48.241 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019815692999999999, 2023-10-01T09:14:48.241 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.241 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.241 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:14:48.241 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:14:48.241 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:14:48.241 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 6699 2023-10-01T09:14:48.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.241 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.241 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.612797+0000", 2023-10-01T09:14:48.242 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.242 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.242 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.612797+0000", 2023-10-01T09:14:48.242 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.242 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.242 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.612796+0000", 2023-10-01T09:14:48.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:14:48.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.243 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.243 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.612817+0000", 2023-10-01T09:14:48.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0496000000000002e-05 2023-10-01T09:14:48.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.243 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.243 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.612828+0000", 2023-10-01T09:14:48.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1508000000000001e-05 2023-10-01T09:14:48.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.612836+0000", 2023-10-01T09:14:48.244 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9140000000000008e-06 2023-10-01T09:14:48.244 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.613475+0000", 2023-10-01T09:14:48.244 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00063925199999999998 2023-10-01T09:14:48.244 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.613586+0000", 2023-10-01T09:14:48.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110921 2023-10-01T09:14:48.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.245 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.245 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,6", 2023-10-01T09:14:48.245 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.613713+0000", 2023-10-01T09:14:48.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012734700000000001 2023-10-01T09:14:48.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.245 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.245 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:14:48.245 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.614443+0000", 2023-10-01T09:14:48.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00072985300000000003 2023-10-01T09:14:48.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.246 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.246 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.246 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.632478+0000", 2023-10-01T09:14:48.246 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018034284000000001 2023-10-01T09:14:48.246 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.246 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.246 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.246 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.632529+0000", 2023-10-01T09:14:48.246 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1819e-05 2023-10-01T09:14:48.246 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.246 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.246 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.247 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.632577+0000", 2023-10-01T09:14:48.247 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7307e-05 2023-10-01T09:14:48.247 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.247 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.247 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.247 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.632613+0000", 2023-10-01T09:14:48.247 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5958999999999999e-05 2023-10-01T09:14:48.247 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.247 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.247 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.247 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.247 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.247 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:6693 5.8 e111/106 5:1595d0ef:::benchmark_data_smithi138_240338_object6692:head v 111'444, mlcod=111'439)", 2023-10-01T09:14:48.248 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:46.614573+0000", 2023-10-01T09:14:48.248 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.564574551, 2023-10-01T09:14:48.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018917367000000001, 2023-10-01T09:14:48.248 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.248 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.248 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.248 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.248 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.248 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.614573+0000", 2023-10-01T09:14:48.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.248 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.249 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.614573+0000", 2023-10-01T09:14:48.249 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.249 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.249 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.614573+0000", 2023-10-01T09:14:48.249 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:14:48.249 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.249 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.614594+0000", 2023-10-01T09:14:48.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1511000000000001e-05 2023-10-01T09:14:48.250 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.250 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.614603+0000", 2023-10-01T09:14:48.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3519999999999996e-06 2023-10-01T09:14:48.250 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.250 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.614610+0000", 2023-10-01T09:14:48.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3720000000000001e-06 2023-10-01T09:14:48.250 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.251 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.251 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.251 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.614681+0000", 2023-10-01T09:14:48.251 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1470999999999994e-05 2023-10-01T09:14:48.251 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.251 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.251 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.251 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.632399+0000", 2023-10-01T09:14:48.251 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017717600999999999 2023-10-01T09:14:48.251 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.251 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.251 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.251 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.633457+0000", 2023-10-01T09:14:48.252 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010579739999999999 2023-10-01T09:14:48.252 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.252 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.252 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.252 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.633491+0000", 2023-10-01T09:14:48.252 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3806000000000001e-05 2023-10-01T09:14:48.252 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.252 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.252 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.252 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.252 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.252 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:6702 5.c e111/106 5:3d1a8163:::benchmark_data_smithi138_240338_object6701:head v 111'437, mlcod=111'434)", 2023-10-01T09:14:48.253 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:46.615359+0000", 2023-10-01T09:14:48.253 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.563788524, 2023-10-01T09:14:48.253 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018115016000000001, 2023-10-01T09:14:48.253 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.253 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.253 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.253 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.253 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.253 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.615359+0000", 2023-10-01T09:14:48.253 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.253 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.253 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.253 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.254 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.615359+0000", 2023-10-01T09:14:48.254 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.254 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.254 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.254 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.254 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.615359+0000", 2023-10-01T09:14:48.254 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:14:48.254 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.254 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.254 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.254 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.615380+0000", 2023-10-01T09:14:48.255 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1730999999999999e-05 2023-10-01T09:14:48.255 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.255 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.255 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.255 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.632380+0000", 2023-10-01T09:14:48.255 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016999938999999999 2023-10-01T09:14:48.255 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.255 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.255 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.255 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.632396+0000", 2023-10-01T09:14:48.255 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5204e-05 2023-10-01T09:14:48.255 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.256 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.632436+0000", 2023-10-01T09:14:48.256 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0166999999999997e-05 2023-10-01T09:14:48.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.256 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.632459+0000", 2023-10-01T09:14:48.256 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3068999999999999e-05 2023-10-01T09:14:48.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.633439+0000", 2023-10-01T09:14:48.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00097999900000000006 2023-10-01T09:14:48.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.257 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.257 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.633474+0000", 2023-10-01T09:14:48.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5487000000000001e-05 2023-10-01T09:14:48.257 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.257 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.257 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.257 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.258 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:6709 5.5 5:acc1098e:::benchmark_data_smithi138_240338_object6708:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:14:48.258 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:46.615938+0000", 2023-10-01T09:14:48.258 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.563209474, 2023-10-01T09:14:48.258 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01789112, 2023-10-01T09:14:48.258 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.258 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.258 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:14:48.258 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:14:48.258 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:14:48.258 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 6709 2023-10-01T09:14:48.258 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.259 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.259 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.615938+0000", 2023-10-01T09:14:48.259 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.259 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.259 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.615938+0000", 2023-10-01T09:14:48.259 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.259 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.615937+0000", 2023-10-01T09:14:48.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:14:48.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.615957+0000", 2023-10-01T09:14:48.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9394000000000001e-05 2023-10-01T09:14:48.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.632403+0000", 2023-10-01T09:14:48.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016446361 2023-10-01T09:14:48.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.632416+0000", 2023-10-01T09:14:48.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3193000000000001e-05 2023-10-01T09:14:48.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.632459+0000", 2023-10-01T09:14:48.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2867000000000001e-05 2023-10-01T09:14:48.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.262 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.262 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.632594+0000", 2023-10-01T09:14:48.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000134358 2023-10-01T09:14:48.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.262 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-10-01T09:14:48.262 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.632722+0000", 2023-10-01T09:14:48.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000128508 2023-10-01T09:14:48.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:14:48.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.633484+0000", 2023-10-01T09:14:48.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00076144299999999999 2023-10-01T09:14:48.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.633580+0000", 2023-10-01T09:14:48.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6124000000000003e-05 2023-10-01T09:14:48.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.633748+0000", 2023-10-01T09:14:48.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000167894 2023-10-01T09:14:48.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.633792+0000", 2023-10-01T09:14:48.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4749000000000001e-05 2023-10-01T09:14:48.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.633829+0000", 2023-10-01T09:14:48.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7172999999999998e-05 2023-10-01T09:14:48.264 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.265 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.265 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.265 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:7481 5.5 5:a6d0921e:::benchmark_data_smithi138_240338_object7480:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:14:48.265 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:46.828854+0000", 2023-10-01T09:14:48.265 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.3502942769999999, 2023-10-01T09:14:48.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.028678008000000001, 2023-10-01T09:14:48.265 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.265 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.266 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:14:48.266 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:14:48.266 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:14:48.266 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 7481 2023-10-01T09:14:48.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.266 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.828854+0000", 2023-10-01T09:14:48.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.828854+0000", 2023-10-01T09:14:48.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.828853+0000", 2023-10-01T09:14:48.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:14:48.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.828875+0000", 2023-10-01T09:14:48.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1996000000000001e-05 2023-10-01T09:14:48.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.828897+0000", 2023-10-01T09:14:48.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2242000000000002e-05 2023-10-01T09:14:48.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.828904+0000", 2023-10-01T09:14:48.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1559999999999998e-06 2023-10-01T09:14:48.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.828953+0000", 2023-10-01T09:14:48.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8860999999999999e-05 2023-10-01T09:14:48.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.829078+0000", 2023-10-01T09:14:48.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012481699999999999 2023-10-01T09:14:48.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-10-01T09:14:48.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.829251+0000", 2023-10-01T09:14:48.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017376699999999999 2023-10-01T09:14:48.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.830992+0000", 2023-10-01T09:14:48.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001740449 2023-10-01T09:14:48.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:14:48.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.831503+0000", 2023-10-01T09:14:48.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00051140199999999999 2023-10-01T09:14:48.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.857444+0000", 2023-10-01T09:14:48.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.025940995000000001 2023-10-01T09:14:48.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.272 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.272 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.857494+0000", 2023-10-01T09:14:48.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9932999999999998e-05 2023-10-01T09:14:48.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.272 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.272 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.857532+0000", 2023-10-01T09:14:48.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7407999999999998e-05 2023-10-01T09:14:48.272 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.272 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.272 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.273 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:7482 5.5 5:a9409db7:::benchmark_data_smithi138_240338_object7481:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:14:48.273 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:46.828919+0000", 2023-10-01T09:14:48.273 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.3502291319999999, 2023-10-01T09:14:48.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.028745262000000001, 2023-10-01T09:14:48.273 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.273 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.273 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:14:48.273 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:14:48.273 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:14:48.274 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 7482 2023-10-01T09:14:48.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.274 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.828919+0000", 2023-10-01T09:14:48.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.828919+0000", 2023-10-01T09:14:48.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.828918+0000", 2023-10-01T09:14:48.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:14:48.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.828936+0000", 2023-10-01T09:14:48.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8105999999999999e-05 2023-10-01T09:14:48.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.828943+0000", 2023-10-01T09:14:48.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7139999999999996e-06 2023-10-01T09:14:48.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.828948+0000", 2023-10-01T09:14:48.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3240000000000002e-06 2023-10-01T09:14:48.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.829825+0000", 2023-10-01T09:14:48.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00087679900000000005 2023-10-01T09:14:48.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.829955+0000", 2023-10-01T09:14:48.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012966200000000001 2023-10-01T09:14:48.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-10-01T09:14:48.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.830086+0000", 2023-10-01T09:14:48.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131302 2023-10-01T09:14:48.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.831012+0000", 2023-10-01T09:14:48.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00092553199999999996 2023-10-01T09:14:48.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:14:48.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.831648+0000", 2023-10-01T09:14:48.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00063633799999999996 2023-10-01T09:14:48.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.857579+0000", 2023-10-01T09:14:48.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.025931111 2023-10-01T09:14:48.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.857624+0000", 2023-10-01T09:14:48.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4869e-05 2023-10-01T09:14:48.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.857664+0000", 2023-10-01T09:14:48.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0024000000000002e-05 2023-10-01T09:14:48.280 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.280 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.280 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.280 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:7484 5.5 5:a42b7c1b:::benchmark_data_smithi138_240338_object7483:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:14:48.280 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:46.829472+0000", 2023-10-01T09:14:48.280 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.349675542, 2023-10-01T09:14:48.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.028342859000000001, 2023-10-01T09:14:48.280 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.280 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.280 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:14:48.281 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:14:48.281 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:14:48.281 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 7484 2023-10-01T09:14:48.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.281 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.829472+0000", 2023-10-01T09:14:48.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.829472+0000", 2023-10-01T09:14:48.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.282 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.829471+0000", 2023-10-01T09:14:48.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:14:48.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.282 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.829492+0000", 2023-10-01T09:14:48.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0784999999999998e-05 2023-10-01T09:14:48.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.829512+0000", 2023-10-01T09:14:48.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9653e-05 2023-10-01T09:14:48.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.829520+0000", 2023-10-01T09:14:48.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.49e-06 2023-10-01T09:14:48.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.830492+0000", 2023-10-01T09:14:48.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000971842 2023-10-01T09:14:48.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.830610+0000", 2023-10-01T09:14:48.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118298 2023-10-01T09:14:48.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-10-01T09:14:48.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.830753+0000", 2023-10-01T09:14:48.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000142223 2023-10-01T09:14:48.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:14:48.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.831654+0000", 2023-10-01T09:14:48.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00090124200000000002 2023-10-01T09:14:48.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.831933+0000", 2023-10-01T09:14:48.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00027938799999999999 2023-10-01T09:14:48.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.857727+0000", 2023-10-01T09:14:48.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.025793244 2023-10-01T09:14:48.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.857777+0000", 2023-10-01T09:14:48.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0436000000000003e-05 2023-10-01T09:14:48.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.857815+0000", 2023-10-01T09:14:48.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8132e-05 2023-10-01T09:14:48.287 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.287 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.287 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.287 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:7487 5.5 5:a3400f9e:::benchmark_data_smithi138_240338_object7486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:14:48.287 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:46.830688+0000", 2023-10-01T09:14:48.287 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.3484596019999999, 2023-10-01T09:14:48.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.027247084000000001, 2023-10-01T09:14:48.288 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.288 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.288 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:14:48.288 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:14:48.288 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:14:48.288 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 7487 2023-10-01T09:14:48.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.288 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.830688+0000", 2023-10-01T09:14:48.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.289 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.830688+0000", 2023-10-01T09:14:48.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.289 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.830687+0000", 2023-10-01T09:14:48.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:14:48.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.830708+0000", 2023-10-01T09:14:48.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1143999999999999e-05 2023-10-01T09:14:48.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.830725+0000", 2023-10-01T09:14:48.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6277999999999999e-05 2023-10-01T09:14:48.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.830745+0000", 2023-10-01T09:14:48.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.018e-05 2023-10-01T09:14:48.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.831026+0000", 2023-10-01T09:14:48.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00028115099999999999 2023-10-01T09:14:48.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.831120+0000", 2023-10-01T09:14:48.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3620000000000002e-05 2023-10-01T09:14:48.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-10-01T09:14:48.292 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.831233+0000", 2023-10-01T09:14:48.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113744 2023-10-01T09:14:48.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:14:48.292 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.831734+0000", 2023-10-01T09:14:48.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00050009399999999997 2023-10-01T09:14:48.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.832035+0000", 2023-10-01T09:14:48.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00030112699999999999 2023-10-01T09:14:48.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.857860+0000", 2023-10-01T09:14:48.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.02582487 2023-10-01T09:14:48.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.857904+0000", 2023-10-01T09:14:48.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3980000000000002e-05 2023-10-01T09:14:48.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.857935+0000", 2023-10-01T09:14:48.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1786999999999999e-05 2023-10-01T09:14:48.294 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.294 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.294 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.295 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:7489 5.5 5:abcee4c5:::benchmark_data_smithi138_240338_object7488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:14:48.295 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:46.831976+0000", 2023-10-01T09:14:48.295 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.347172126, 2023-10-01T09:14:48.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.026074165, 2023-10-01T09:14:48.295 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.295 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.295 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:14:48.295 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:14:48.295 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:14:48.295 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 7489 2023-10-01T09:14:48.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.295 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.831976+0000", 2023-10-01T09:14:48.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.831976+0000", 2023-10-01T09:14:48.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.831975+0000", 2023-10-01T09:14:48.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:14:48.297 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.297 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.297 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.297 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.831998+0000", 2023-10-01T09:14:48.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2855000000000001e-05 2023-10-01T09:14:48.297 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.297 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.297 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.297 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.832010+0000", 2023-10-01T09:14:48.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1953000000000001e-05 2023-10-01T09:14:48.297 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.297 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.298 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.298 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.832017+0000", 2023-10-01T09:14:48.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.678e-06 2023-10-01T09:14:48.298 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.298 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.298 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.298 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.832052+0000", 2023-10-01T09:14:48.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5179999999999999e-05 2023-10-01T09:14:48.298 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.298 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.298 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.832130+0000", 2023-10-01T09:14:48.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7681000000000004e-05 2023-10-01T09:14:48.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-10-01T09:14:48.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.832229+0000", 2023-10-01T09:14:48.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9073999999999994e-05 2023-10-01T09:14:48.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:14:48.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.832845+0000", 2023-10-01T09:14:48.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00061569399999999996 2023-10-01T09:14:48.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.833511+0000", 2023-10-01T09:14:48.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00066631199999999996 2023-10-01T09:14:48.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.857971+0000", 2023-10-01T09:14:48.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024459569 2023-10-01T09:14:48.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.301 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.858013+0000", 2023-10-01T09:14:48.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2267e-05 2023-10-01T09:14:48.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.301 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.301 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.301 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.858050+0000", 2023-10-01T09:14:48.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6801999999999998e-05 2023-10-01T09:14:48.301 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.301 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.301 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.302 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:7498 5.5 5:ad94af94:::benchmark_data_smithi138_240338_object7497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:14:48.302 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:46.836887+0000", 2023-10-01T09:14:48.302 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.342260639, 2023-10-01T09:14:48.302 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.021272756, 2023-10-01T09:14:48.302 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.302 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.302 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:14:48.302 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:14:48.302 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:14:48.302 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 7498 2023-10-01T09:14:48.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.303 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.303 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.303 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.836887+0000", 2023-10-01T09:14:48.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.303 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.303 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.836887+0000", 2023-10-01T09:14:48.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.304 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.304 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.304 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.836886+0000", 2023-10-01T09:14:48.304 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:14:48.304 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.304 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.304 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.304 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.836904+0000", 2023-10-01T09:14:48.304 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7988000000000001e-05 2023-10-01T09:14:48.304 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.304 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.304 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.305 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.836917+0000", 2023-10-01T09:14:48.305 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2353e-05 2023-10-01T09:14:48.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.305 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.305 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.836924+0000", 2023-10-01T09:14:48.305 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4789999999999999e-06 2023-10-01T09:14:48.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.305 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.305 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.836976+0000", 2023-10-01T09:14:48.305 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1248000000000001e-05 2023-10-01T09:14:48.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.306 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.306 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.306 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.837062+0000", 2023-10-01T09:14:48.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6100000000000006e-05 2023-10-01T09:14:48.306 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.306 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.306 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-10-01T09:14:48.306 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.837145+0000", 2023-10-01T09:14:48.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3832999999999995e-05 2023-10-01T09:14:48.306 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.306 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:14:48.307 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.837646+0000", 2023-10-01T09:14:48.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000500185 2023-10-01T09:14:48.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.307 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.837929+0000", 2023-10-01T09:14:48.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000283431 2023-10-01T09:14:48.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.307 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.858086+0000", 2023-10-01T09:14:48.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020157066000000001 2023-10-01T09:14:48.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.858127+0000", 2023-10-01T09:14:48.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0444000000000001e-05 2023-10-01T09:14:48.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.858160+0000", 2023-10-01T09:14:48.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3380999999999998e-05 2023-10-01T09:14:48.308 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.309 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.309 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.309 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:9389 5.5 5:adecfbf3:::benchmark_data_smithi138_240338_object9388:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:14:48.309 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:47.481268+0000", 2023-10-01T09:14:48.309 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.697879476, 2023-10-01T09:14:48.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019682822999999999, 2023-10-01T09:14:48.309 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:14:48.309 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:14:48.309 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:14:48.309 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:14:48.309 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:14:48.310 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 9389 2023-10-01T09:14:48.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.310 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:14:48.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:14:48.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:47.481268+0000", 2023-10-01T09:14:48.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:14:48.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:47.481268+0000", 2023-10-01T09:14:48.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:14:48.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:14:48.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:47.481268+0000", 2023-10-01T09:14:48.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:14:48.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:14:48.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:47.481292+0000", 2023-10-01T09:14:48.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3918999999999999e-05 2023-10-01T09:14:48.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:14:48.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:47.481301+0000", 2023-10-01T09:14:48.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6819999999999998e-06 2023-10-01T09:14:48.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:14:48.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:47.481307+0000", 2023-10-01T09:14:48.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0449999999999997e-06 2023-10-01T09:14:48.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:14:48.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:47.482237+0000", 2023-10-01T09:14:48.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00092949299999999999 2023-10-01T09:14:48.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:14:48.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:47.482361+0000", 2023-10-01T09:14:48.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012375 2023-10-01T09:14:48.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-10-01T09:14:48.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:47.482489+0000", 2023-10-01T09:14:48.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012835200000000001 2023-10-01T09:14:48.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:47.491601+0000", 2023-10-01T09:14:48.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0091119549999999997 2023-10-01T09:14:48.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:14:48.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:47.491810+0000", 2023-10-01T09:14:48.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000209278 2023-10-01T09:14:48.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:14:48.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:47.500856+0000", 2023-10-01T09:14:48.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0090459459999999992 2023-10-01T09:14:48.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:14:48.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:47.500910+0000", 2023-10-01T09:14:48.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3599e-05 2023-10-01T09:14:48.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:14:48.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:14:48.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:14:48.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:47.500951+0000", 2023-10-01T09:14:48.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1461000000000002e-05 2023-10-01T09:14:48.315 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.316 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.316 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.316 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:14:48.316 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:14:48.316 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:14:48.316 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:47 smithi190 ceph-mon[160435]: pgmap v235: 89 pgs: 89 active+clean; 231 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s wr, 387 op/s 2023-10-01T09:14:48.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:14:48.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:14:48.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:14:48.317 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:48.400 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:14:48.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:48.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:14:48.505 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:14:48.511 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 4 16 11898 11882 185.525 153.188 0.00500409 0.00538257 2023-10-01T09:14:48.553 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:48.636 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:14:48.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:48.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:14:48.746 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:14:48.801 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:48.898 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:14:48.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:48.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:14:48.967 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:14:48.970 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.2 to 1 (10000) 2023-10-01T09:14:49.003 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:14:49.033 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:49.096 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:14:49.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:49.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:14:49.203 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:14:49.249 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:48 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1503587666' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:14:49.249 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:48 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2017794986' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:14:49.253 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:48 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1503587666' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:14:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:48 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2017794986' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:14:49.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:48 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1503587666' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:14:49.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:48 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2017794986' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:14:49.354 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:14:49.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:49.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:14:49.461 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:14:49.508 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:49.511 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 5 16 14841 14825 185.204 183.938 0.0057652 0.00539335 2023-10-01T09:14:49.596 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:14:49.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:49.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:14:49.704 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:14:49.757 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:50.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:14:50.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1503587666' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:14:50.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2017794986' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:14:50.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[185886]: osdmap e112: 8 total, 8 up, 8 in 2023-10-01T09:14:50.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[185886]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[185886]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[185886]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[185886]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[185886]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[185886]: pgmap v237: 89 pgs: 89 active+clean; 536 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T09:14:50.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:14:50.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1503587666' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:14:50.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2017794986' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:14:50.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[192513]: osdmap e112: 8 total, 8 up, 8 in 2023-10-01T09:14:50.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[192513]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[192513]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[192513]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[192513]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[192513]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:49 smithi138 ceph-mon[192513]: pgmap v237: 89 pgs: 89 active+clean; 536 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T09:14:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:49 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:14:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1503587666' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:14:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2017794986' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:14:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:49 smithi190 ceph-mon[160435]: osdmap e112: 8 total, 8 up, 8 in 2023-10-01T09:14:50.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:49 smithi190 ceph-mon[160435]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:49 smithi190 ceph-mon[160435]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:49 smithi190 ceph-mon[160435]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:49 smithi190 ceph-mon[160435]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:49 smithi190 ceph-mon[160435]: failed to encode map e112 with expected crc 2023-10-01T09:14:50.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:49 smithi190 ceph-mon[160435]: pgmap v237: 89 pgs: 89 active+clean; 536 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T09:14:50.511 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 6 16 17854 17838 185.719 188.312 0.00700324 0.00537871 2023-10-01T09:14:51.512 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 7 16 20469 20453 182.535 163.438 0.0053214 0.00544521 2023-10-01T09:14:52.512 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 8 16 22152 22136 172.868 105.188 0.00651081 0.00577923 2023-10-01T09:14:52.592 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:52 smithi138 ceph-mon[192513]: pgmap v238: 89 pgs: 89 active+clean; 931 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-10-01T09:14:52.593 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:52 smithi138 ceph-mon[185886]: pgmap v238: 89 pgs: 89 active+clean; 931 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-10-01T09:14:52.775 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:14:52.796 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-01T09:14:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:52 smithi190 ceph-mon[160435]: pgmap v238: 89 pgs: 89 active+clean; 931 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-10-01T09:14:52.914 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:14:52.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:52.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:14:53.022 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:14:53.067 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:53.143 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:14:53.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:53.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:14:53.248 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:14:53.289 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:53.382 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:14:53.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:53.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:14:53.489 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:14:53.512 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 9 16 24286 24270 168.48 133.375 0.00176334 0.00588577 2023-10-01T09:14:53.515 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:53.557 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:14:53.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:53.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:14:53.662 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:14:53.687 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:53.730 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:14:53.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:53.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:14:53.835 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:14:53.891 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:54.159 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:14:54.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:54.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:14:54.271 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:14:54.315 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:54.364 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:54.406 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:14:54.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:54.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:14:54.428 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:54.512 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:14:54.513 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 10 16 25985 25969 162.238 106.188 0.00554978 0.00615795 2023-10-01T09:14:54.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:54 smithi138 ceph-mon[185886]: pgmap v239: 89 pgs: 89 active+clean; 1.1 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T09:14:54.520 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:54 smithi138 ceph-mon[192513]: pgmap v239: 89 pgs: 89 active+clean; 1.1 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T09:14:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:54 smithi190 ceph-mon[160435]: pgmap v239: 89 pgs: 89 active+clean; 1.1 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T09:14:54.577 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:54.816 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:14:54.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:54.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:14:54.855 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:14:54.890 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:14:54.894 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:14:54.925 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:14:54.926 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:14:54.967 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:54.971 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:14:55.013 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:14:55.032 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:14:55.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:55.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:14:55.050 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:14:55.075 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:14:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:14:54] "GET /metrics HTTP/1.1" 200 36258 "" "Prometheus/2.33.4" 2023-10-01T09:14:55.085 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:14:55.130 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:14:55.137 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:14:55.137 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:14:55.138 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:14:55.138 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:14:55.138 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:14:55.138 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:14:55.138 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:14:55.138 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:14:55.139 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:14:55.226 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:55.513 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 11 16 27924 27908 158.506 121.188 0.00853621 0.0063038 2023-10-01T09:14:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:55 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2301697157' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:14:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:14:55.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:55 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2301697157' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:14:55.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:14:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:55 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2301697157' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:14:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:14:56.513 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 12 16 30157 30141 156.927 139.562 0.00584871 0.00636745 2023-10-01T09:14:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:56 smithi190 ceph-mon[160435]: pgmap v240: 89 pgs: 89 active+clean; 1.3 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-10-01T09:14:56.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:56 smithi138 ceph-mon[192513]: pgmap v240: 89 pgs: 89 active+clean; 1.3 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-10-01T09:14:56.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:56 smithi138 ceph-mon[185886]: pgmap v240: 89 pgs: 89 active+clean; 1.3 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-10-01T09:14:57.513 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 13 16 31823 31807 152.866 104.125 0.00504361 0.00653651 2023-10-01T09:14:57.687 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:14:57.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:57.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:14:57.803 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:14:57.849 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:57.894 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:14:57.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:57.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:14:58.003 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:14:58.056 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:58.144 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:14:58.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:58.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:14:58.156 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:14:57 smithi190 ceph-mon[160435]: pgmap v241: 89 pgs: 89 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 152 MiB/s wr, 2.42k op/s 2023-10-01T09:14:58.257 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:14:58.282 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:14:57 smithi138 ceph-mon[185886]: pgmap v241: 89 pgs: 89 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 152 MiB/s wr, 2.42k op/s 2023-10-01T09:14:58.283 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:14:57 smithi138 ceph-mon[192513]: pgmap v241: 89 pgs: 89 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 152 MiB/s wr, 2.42k op/s 2023-10-01T09:14:58.285 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:58.513 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 14 16 33503 33487 149.447 105 0.0095399 0.00668677 2023-10-01T09:14:58.651 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:14:58.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:58.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:14:58.757 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:14:58.817 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:58.903 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:14:58.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:58.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:14:59.011 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:14:59.072 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:59.146 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:14:59.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:59.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:14:59.253 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:14:59.284 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:14:59.333 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:59.421 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:14:59.436 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:14:59.436 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:14:59.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:59.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:14:59.513 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 15 16 35351 35335 147.182 115.5 0.00575324 0.00678953 2023-10-01T09:14:59.540 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:14:59.590 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:14:59.670 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:14:59.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:14:59.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:14:59.722 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:14:59.780 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:14:59.829 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:00.340 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.3 to 1 (10000) 2023-10-01T09:15:00.513 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 16 16 36777 36761 143.553 89.125 0.0127151 0.00696041 2023-10-01T09:15:00.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:00 smithi190 ceph-mon[160435]: pgmap v242: 89 pgs: 89 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-10-01T09:15:00.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:00 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/356599830' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:15:00.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:00 smithi138 ceph-mon[192513]: pgmap v242: 89 pgs: 89 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-10-01T09:15:00.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:00 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/356599830' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:15:00.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:00 smithi138 ceph-mon[185886]: pgmap v242: 89 pgs: 89 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-10-01T09:15:00.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:00 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/356599830' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:15:00.869 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:15:00.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:00.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:15:00.973 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:15:01.025 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:15:01.046 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:01.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:01 smithi138 ceph-mon[185886]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/356599830' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:15:01.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:01 smithi138 ceph-mon[185886]: osdmap e113: 8 total, 8 up, 8 in 2023-10-01T09:15:01.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:01 smithi138 ceph-mon[185886]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:01 smithi138 ceph-mon[185886]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.451 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:01 smithi138 ceph-mon[185886]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.452 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:01 smithi138 ceph-mon[185886]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:01 smithi138 ceph-mon[192513]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/356599830' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:15:01.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:01 smithi138 ceph-mon[192513]: osdmap e113: 8 total, 8 up, 8 in 2023-10-01T09:15:01.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:01 smithi138 ceph-mon[192513]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:01 smithi138 ceph-mon[192513]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:01 smithi138 ceph-mon[192513]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:01 smithi138 ceph-mon[192513]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.513 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 17 16 38489 38473 141.402 107 0.00403096 0.00705421 2023-10-01T09:15:01.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:01 smithi190 ceph-mon[160435]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/356599830' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:15:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:01 smithi190 ceph-mon[160435]: osdmap e113: 8 total, 8 up, 8 in 2023-10-01T09:15:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:01 smithi190 ceph-mon[160435]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:01 smithi190 ceph-mon[160435]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:01 smithi190 ceph-mon[160435]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:01 smithi190 ceph-mon[160435]: failed to encode map e113 with expected crc 2023-10-01T09:15:01.935 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:15:01.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:01.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:15:02.040 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:15:02.092 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:02.273 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-01T09:15:02.336 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:15:02.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:02.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:15:02.447 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:15:02.502 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:02.514 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 18 16 40241 40225 139.628 109.5 0.00465241 0.0071571 2023-10-01T09:15:02.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:02 smithi138 ceph-mon[192513]: pgmap v244: 89 pgs: 89 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-01T09:15:02.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:02 smithi138 ceph-mon[185886]: pgmap v244: 89 pgs: 89 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-01T09:15:02.593 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:15:02.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:02.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:15:02.699 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:15:02.753 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:02.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:02 smithi190 ceph-mon[160435]: pgmap v244: 89 pgs: 89 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-01T09:15:03.052 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:15:03.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:03.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:15:03.158 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:15:03.191 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:03.243 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:03.273 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:15:03.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:03.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:15:03.333 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:03.336 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:03.379 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:15:03.453 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:03.515 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 19 16 41940 41924 137.856 106.188 0.0193926 0.00724965 2023-10-01T09:15:03.601 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:15:03.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:03.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:15:03.709 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:03.770 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:04.363 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:04 smithi138 ceph-mon[192513]: pgmap v245: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-01T09:15:04.363 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2104496186' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:15:04.363 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:04 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:15:04.363 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:04 smithi138 ceph-mon[185886]: pgmap v245: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-01T09:15:04.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2104496186' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:15:04.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:04 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:15:04.392 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:15:04.517 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:15:04.517559+0000 min lat: 0.00109137 max lat: 0.222614 avg lat: 0.00734679 2023-10-01T09:15:04.517 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:15:04.517 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 20 16 43558 43542 136.012 101.125 0.00899861 0.00734679 2023-10-01T09:15:04.642 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:04.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:04.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:04.750 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:15:04.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:04 smithi190 ceph-mon[160435]: pgmap v245: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-01T09:15:04.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2104496186' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:15:04.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:04 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:15:04.812 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:05.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:15:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:15:04] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.33.4" 2023-10-01T09:15:05.517 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 21 16 45323 45307 134.788 110.312 0.00202704 0.00739774 2023-10-01T09:15:05.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:05 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:15:05.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:05 smithi190 ceph-mon[160435]: failed to encode map e114 with expected crc 2023-10-01T09:15:05.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:05 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:15:05.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:05 smithi190 ceph-mon[160435]: osdmap e114: 8 total, 8 up, 8 in 2023-10-01T09:15:05.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:05 smithi190 ceph-mon[160435]: failed to encode map e114 with expected crc 2023-10-01T09:15:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:05 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:15:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:05 smithi138 ceph-mon[185886]: failed to encode map e114 with expected crc 2023-10-01T09:15:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:05 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:15:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:05 smithi138 ceph-mon[185886]: osdmap e114: 8 total, 8 up, 8 in 2023-10-01T09:15:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:05 smithi138 ceph-mon[185886]: failed to encode map e114 with expected crc 2023-10-01T09:15:05.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:05 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:15:05.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:05 smithi138 ceph-mon[192513]: failed to encode map e114 with expected crc 2023-10-01T09:15:05.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:05 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:15:05.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:05 smithi138 ceph-mon[192513]: osdmap e114: 8 total, 8 up, 8 in 2023-10-01T09:15:05.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:05 smithi138 ceph-mon[192513]: failed to encode map e114 with expected crc 2023-10-01T09:15:06.517 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 22 16 46764 46748 132.755 90.0625 0.00383442 0.0075191 2023-10-01T09:15:06.772 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:15:06.772 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:06 smithi138 ceph-mon[185886]: pgmap v247: 89 pgs: 89 active+clean; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-01T09:15:06.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:06 smithi138 ceph-mon[192513]: pgmap v247: 89 pgs: 89 active+clean; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-01T09:15:06.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:06.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:15:06.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:06 smithi190 ceph-mon[160435]: pgmap v247: 89 pgs: 89 active+clean; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-10-01T09:15:06.883 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:15:06.911 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:06.953 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:15:06.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:06.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:15:07.058 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:15:07.085 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:07.518 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 23 16 47893 47877 130.045 70.5625 0.00610915 0.00768454 2023-10-01T09:15:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:07 smithi190 ceph-mon[160435]: pgmap v248: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-01T09:15:08.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:07 smithi138 ceph-mon[185886]: pgmap v248: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-01T09:15:08.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:07 smithi138 ceph-mon[192513]: pgmap v248: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-10-01T09:15:08.518 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:15:08.520 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 24 16 49168 49152 127.942 79.6875 0.0107017 0.00781085 2023-10-01T09:15:08.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:08.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:15:08.629 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:15:08.696 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:08.871 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:15:08.908 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:15:08.946 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:15:08.986 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:15:08.990 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:15:08.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:08.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:15:09.023 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:15:09.075 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:15:09.097 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:15:09.108 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:15:09.139 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:15:09.144 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:15:09.145 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:15:09.145 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:15:09.145 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:15:09.145 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:15:09.145 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:15:09.145 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:15:09.145 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:15:09.178 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:09.221 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:15:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:15:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:15:09.221 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:15:09.221 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:15:09.242 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:08 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1681716819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:09.242 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:08 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1681716819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:09.300 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:15:09.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:08 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1681716819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:09.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:09.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:15:09.314 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:15:09.412 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:09.509 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:09.519 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 25 16 50555 50539 126.292 86.6875 0.00269331 0.00791005 2023-10-01T09:15:09.974 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.4 to 1 (10000) 2023-10-01T09:15:10.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:09 smithi138 ceph-mon[192513]: pgmap v249: 89 pgs: 89 active+clean; 2.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-01T09:15:10.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:09 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2896182056' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:15:10.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:09 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:15:10.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:09 smithi138 ceph-mon[185886]: pgmap v249: 89 pgs: 89 active+clean; 2.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-01T09:15:10.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:09 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2896182056' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:15:10.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:09 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:15:10.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:09 smithi190 ceph-mon[160435]: pgmap v249: 89 pgs: 89 active+clean; 2.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-10-01T09:15:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:09 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2896182056' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:15:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:09 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:15:10.519 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 26 15 51727 51712 124.255 73.3125 0.00360394 0.00804318 2023-10-01T09:15:11.090 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:15:11.254 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:11.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:11.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:15:11.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[185886]: osdmap e115: 8 total, 8 up, 8 in 2023-10-01T09:15:11.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[185886]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[185886]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[185886]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[185886]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[185886]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:15:11.261 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:15:11.262 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[192513]: osdmap e115: 8 total, 8 up, 8 in 2023-10-01T09:15:11.262 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[192513]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.262 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[192513]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.262 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[192513]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.262 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[192513]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.262 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[192513]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.262 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:15:11.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:11.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:10 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:15:11.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:10 smithi190 ceph-mon[160435]: osdmap e115: 8 total, 8 up, 8 in 2023-10-01T09:15:11.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:10 smithi190 ceph-mon[160435]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:10 smithi190 ceph-mon[160435]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:10 smithi190 ceph-mon[160435]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:10 smithi190 ceph-mon[160435]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:10 smithi190 ceph-mon[160435]: failed to encode map e115 with expected crc 2023-10-01T09:15:11.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:15:11.364 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:15:11.410 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:11.450 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:15:11.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:11.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:15:11.521 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 27 16 52974 52958 122.537 77.875 0.00274356 0.00811947 2023-10-01T09:15:11.556 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:15:11.587 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:11.685 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:15:11.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:11.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:15:11.793 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:15:11.839 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:12.008 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-01T09:15:12.012 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:15:12.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:12.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:15:12.115 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:12.190 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:12.221 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:11 smithi138 ceph-mon[185886]: pgmap v251: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-01T09:15:12.221 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:11 smithi138 ceph-mon[192513]: pgmap v251: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-01T09:15:12.221 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:12.270 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:12.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:11 smithi190 ceph-mon[160435]: pgmap v251: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-01T09:15:12.370 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:12.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:12.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:12.476 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:15:12.520 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 28 16 54295 54279 121.11 82.5625 0.0150301 0.00825226 2023-10-01T09:15:12.520 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:12.599 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:15:12.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:12.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:15:12.709 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:15:12.759 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:12.857 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:15:12.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:12.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:15:12.962 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:15:13.014 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:15:13.016 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:13.105 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:15:13.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:13.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:15:13.211 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:13.291 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:13.520 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 29 16 55748 55732 120.065 90.8125 0.012714 0.00832362 2023-10-01T09:15:13.587 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:14.223 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:14.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:14.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:14.328 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:15:14.386 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:14.478 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:15:14.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:14.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:15:14.520 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 30 16 56980 56964 118.63 77 0.0101019 0.00842421 2023-10-01T09:15:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:14 smithi138 ceph-mon[185886]: pgmap v252: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T09:15:14.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:14 smithi138 ceph-mon[192513]: pgmap v252: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T09:15:14.593 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:14.620 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:14.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:14 smithi190 ceph-mon[160435]: pgmap v252: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T09:15:15.200 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:15.202 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:15:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:15:14] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.33.4" 2023-10-01T09:15:15.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:15.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:15.303 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:15:15.336 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:15.409 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:15:15.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:15.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:15:15.516 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:15:15.520 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 31 16 57952 57936 116.763 60.75 0.0111339 0.00856065 2023-10-01T09:15:15.614 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:16.238 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:15:16.238 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:15:16.239 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:15:16.239 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:15:16.239 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.239 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:3619 5.c 5:39d3dfa2:::benchmark_data_smithi138_240338_object3618:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:15:16.239 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.504610+0000", 2023-10-01T09:15:16.239 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 30.540246450000001, 2023-10-01T09:15:16.239 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.178131441, 2023-10-01T09:15:16.239 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.239 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.239 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:16.239 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:16.240 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:16.240 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 3619 2023-10-01T09:15:16.240 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.240 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.240 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.240 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.504610+0000", 2023-10-01T09:15:16.240 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.240 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.240 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.240 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.504610+0000", 2023-10-01T09:15:16.240 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.241 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.241 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.241 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.504609+0000", 2023-10-01T09:15:16.241 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:15:16.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.241 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.241 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.241 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.504636+0000", 2023-10-01T09:15:16.241 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7234999999999999e-05 2023-10-01T09:15:16.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.241 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.504647+0000", 2023-10-01T09:15:16.242 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0689999999999999e-05 2023-10-01T09:15:16.242 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.242 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.504654+0000", 2023-10-01T09:15:16.242 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8090000000000004e-06 2023-10-01T09:15:16.242 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.242 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.243 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.504695+0000", 2023-10-01T09:15:16.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1003e-05 2023-10-01T09:15:16.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.243 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.243 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.504788+0000", 2023-10-01T09:15:16.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3265999999999998e-05 2023-10-01T09:15:16.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.243 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:15:16.243 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.504917+0000", 2023-10-01T09:15:16.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000129047 2023-10-01T09:15:16.244 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.505773+0000", 2023-10-01T09:15:16.244 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00085646300000000002 2023-10-01T09:15:16.244 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:16.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.505852+0000", 2023-10-01T09:15:16.244 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8288000000000007e-05 2023-10-01T09:15:16.244 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.245 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682648+0000", 2023-10-01T09:15:16.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17679640999999999 2023-10-01T09:15:16.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.245 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.245 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.245 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682698+0000", 2023-10-01T09:15:16.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0191000000000001e-05 2023-10-01T09:15:16.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.245 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.245 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.246 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682741+0000", 2023-10-01T09:15:16.246 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3136999999999997e-05 2023-10-01T09:15:16.246 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.246 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.246 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.246 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.246 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.246 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:3625 5.c 5:37f1eea7:::benchmark_data_smithi138_240338_object3624:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:15:16.246 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.506275+0000", 2023-10-01T09:15:16.247 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 30.538581009000001, 2023-10-01T09:15:16.247 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17719827499999999, 2023-10-01T09:15:16.247 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.247 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.247 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:16.247 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:16.247 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:16.247 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 3625 2023-10-01T09:15:16.247 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.247 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.247 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.247 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.248 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506275+0000", 2023-10-01T09:15:16.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.248 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.248 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.248 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506275+0000", 2023-10-01T09:15:16.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.248 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.248 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.248 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506274+0000", 2023-10-01T09:15:16.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:15:16.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.249 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506299+0000", 2023-10-01T09:15:16.249 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4935000000000001e-05 2023-10-01T09:15:16.249 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.249 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506310+0000", 2023-10-01T09:15:16.249 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.065e-05 2023-10-01T09:15:16.249 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506317+0000", 2023-10-01T09:15:16.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4080000000000001e-06 2023-10-01T09:15:16.250 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.250 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506365+0000", 2023-10-01T09:15:16.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7781000000000002e-05 2023-10-01T09:15:16.250 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.250 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506481+0000", 2023-10-01T09:15:16.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115721 2023-10-01T09:15:16.251 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.251 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.251 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:15:16.251 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506613+0000", 2023-10-01T09:15:16.251 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000132447 2023-10-01T09:15:16.251 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.251 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.251 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.251 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682786+0000", 2023-10-01T09:15:16.251 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.176173514 2023-10-01T09:15:16.251 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.251 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.252 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.252 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.683370+0000", 2023-10-01T09:15:16.252 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00058372800000000005 2023-10-01T09:15:16.252 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.252 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.252 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:16.252 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.683379+0000", 2023-10-01T09:15:16.252 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8489999999999995e-06 2023-10-01T09:15:16.252 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.252 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.252 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.252 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.683434+0000", 2023-10-01T09:15:16.252 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5176000000000002e-05 2023-10-01T09:15:16.253 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.253 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.253 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.253 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.683474+0000", 2023-10-01T09:15:16.253 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9446000000000001e-05 2023-10-01T09:15:16.253 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.253 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.253 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.253 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.253 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.253 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.34683.0:3619 5.c e111/106 ondisk, result = 0)", 2023-10-01T09:15:16.253 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.506358+0000", 2023-10-01T09:15:16.253 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 30.538498012000002, 2023-10-01T09:15:16.254 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17639429400000001, 2023-10-01T09:15:16.254 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.254 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:15:16.254 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.254 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.254 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.254 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506358+0000", 2023-10-01T09:15:16.254 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.254 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.254 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.254 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.255 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506358+0000", 2023-10-01T09:15:16.255 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.255 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.255 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.255 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.255 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506358+0000", 2023-10-01T09:15:16.255 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:16.255 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.255 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.255 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.255 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506363+0000", 2023-10-01T09:15:16.255 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0549999999999999e-06 2023-10-01T09:15:16.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.256 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506372+0000", 2023-10-01T09:15:16.256 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2289999999999992e-06 2023-10-01T09:15:16.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.256 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506378+0000", 2023-10-01T09:15:16.256 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4300000000000003e-06 2023-10-01T09:15:16.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682630+0000", 2023-10-01T09:15:16.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17625115399999999 2023-10-01T09:15:16.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.257 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.257 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682641+0000", 2023-10-01T09:15:16.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1479e-05 2023-10-01T09:15:16.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.257 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.257 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682753+0000", 2023-10-01T09:15:16.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011157700000000001 2023-10-01T09:15:16.257 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.258 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.258 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.258 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.258 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.258 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:3621 5.5 e111/106 5:a241a76b:::benchmark_data_smithi138_240338_object3620:head v 111'204, mlcod=111'201)", 2023-10-01T09:15:16.258 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.506390+0000", 2023-10-01T09:15:16.258 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 30.538465981000002, 2023-10-01T09:15:16.258 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17707672399999999, 2023-10-01T09:15:16.258 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.258 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.258 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.259 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506390+0000", 2023-10-01T09:15:16.259 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.259 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.259 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506390+0000", 2023-10-01T09:15:16.259 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.259 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.259 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506390+0000", 2023-10-01T09:15:16.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:16.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506412+0000", 2023-10-01T09:15:16.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.19e-05 2023-10-01T09:15:16.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506427+0000", 2023-10-01T09:15:16.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5227e-05 2023-10-01T09:15:16.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506434+0000", 2023-10-01T09:15:16.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8909999999999998e-06 2023-10-01T09:15:16.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506465+0000", 2023-10-01T09:15:16.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0660999999999997e-05 2023-10-01T09:15:16.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506483+0000", 2023-10-01T09:15:16.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7737999999999999e-05 2023-10-01T09:15:16.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.262 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.262 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.683425+0000", 2023-10-01T09:15:16.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17694288899999999 2023-10-01T09:15:16.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.262 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.262 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.683467+0000", 2023-10-01T09:15:16.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1749000000000003e-05 2023-10-01T09:15:16.262 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.263 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.263 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.263 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:3622 5.5 e111/106 5:a4d596e2:::benchmark_data_smithi138_240338_object3621:head v 111'205, mlcod=111'201)", 2023-10-01T09:15:16.263 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.506968+0000", 2023-10-01T09:15:16.263 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 30.537887914999999, 2023-10-01T09:15:16.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17711558299999999, 2023-10-01T09:15:16.263 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.263 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.263 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506968+0000", 2023-10-01T09:15:16.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506968+0000", 2023-10-01T09:15:16.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506968+0000", 2023-10-01T09:15:16.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:15:16.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.506991+0000", 2023-10-01T09:15:16.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3302000000000002e-05 2023-10-01T09:15:16.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507002+0000", 2023-10-01T09:15:16.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0689999999999999e-05 2023-10-01T09:15:16.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507008+0000", 2023-10-01T09:15:16.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5760000000000002e-06 2023-10-01T09:15:16.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507065+0000", 2023-10-01T09:15:16.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6711e-05 2023-10-01T09:15:16.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507084+0000", 2023-10-01T09:15:16.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9108000000000001e-05 2023-10-01T09:15:16.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.684057+0000", 2023-10-01T09:15:16.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.176972934 2023-10-01T09:15:16.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.684084+0000", 2023-10-01T09:15:16.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7093e-05 2023-10-01T09:15:16.267 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.268 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.268 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.268 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:3626 5.8 e111/106 5:1337bbfc:::benchmark_data_smithi138_240338_object3625:head v 111'223, mlcod=111'221)", 2023-10-01T09:15:16.268 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.507388+0000", 2023-10-01T09:15:16.268 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 30.537468623999999, 2023-10-01T09:15:16.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.176783527, 2023-10-01T09:15:16.268 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.268 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.268 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507388+0000", 2023-10-01T09:15:16.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507388+0000", 2023-10-01T09:15:16.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507387+0000", 2023-10-01T09:15:16.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:15:16.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507412+0000", 2023-10-01T09:15:16.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4895000000000001e-05 2023-10-01T09:15:16.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682596+0000", 2023-10-01T09:15:16.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17518404100000001 2023-10-01T09:15:16.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682615+0000", 2023-10-01T09:15:16.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8838000000000002e-05 2023-10-01T09:15:16.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682667+0000", 2023-10-01T09:15:16.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2420999999999998e-05 2023-10-01T09:15:16.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682687+0000", 2023-10-01T09:15:16.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9276999999999999e-05 2023-10-01T09:15:16.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.272 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.272 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.684132+0000", 2023-10-01T09:15:16.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014456530000000001 2023-10-01T09:15:16.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.272 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.272 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.684171+0000", 2023-10-01T09:15:16.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9121e-05 2023-10-01T09:15:16.272 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.273 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.273 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.273 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.34683.0:3625 5.c e111/106 ondisk, result = 0)", 2023-10-01T09:15:16.273 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.507563+0000", 2023-10-01T09:15:16.273 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 30.537293117000001, 2023-10-01T09:15:16.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17591327600000001, 2023-10-01T09:15:16.273 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.273 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:15:16.273 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507563+0000", 2023-10-01T09:15:16.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507563+0000", 2023-10-01T09:15:16.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507563+0000", 2023-10-01T09:15:16.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:16.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507568+0000", 2023-10-01T09:15:16.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8659999999999997e-06 2023-10-01T09:15:16.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682906+0000", 2023-10-01T09:15:16.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17533833900000001 2023-10-01T09:15:16.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682913+0000", 2023-10-01T09:15:16.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2049999999999996e-06 2023-10-01T09:15:16.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.683355+0000", 2023-10-01T09:15:16.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00044230199999999999 2023-10-01T09:15:16.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.683364+0000", 2023-10-01T09:15:16.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.524e-06 2023-10-01T09:15:16.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.683477+0000", 2023-10-01T09:15:16.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112643 2023-10-01T09:15:16.277 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.277 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.277 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.278 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.34683.0:3625 5.c e111/106 ondisk, result = 0)", 2023-10-01T09:15:16.278 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.507787+0000", 2023-10-01T09:15:16.278 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 30.537069850999998, 2023-10-01T09:15:16.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.175003567, 2023-10-01T09:15:16.278 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.278 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:15:16.278 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507787+0000", 2023-10-01T09:15:16.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507787+0000", 2023-10-01T09:15:16.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507786+0000", 2023-10-01T09:15:16.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:16.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.280 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.507789+0000", 2023-10-01T09:15:16.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1889999999999999e-06 2023-10-01T09:15:16.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.280 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.280 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682618+0000", 2023-10-01T09:15:16.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17482821200000001 2023-10-01T09:15:16.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.280 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.280 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682630+0000", 2023-10-01T09:15:16.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2170999999999999e-05 2023-10-01T09:15:16.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682774+0000", 2023-10-01T09:15:16.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014432900000000001 2023-10-01T09:15:16.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682781+0000", 2023-10-01T09:15:16.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9330000000000004e-06 2023-10-01T09:15:16.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.682790+0000", 2023-10-01T09:15:16.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1430000000000004e-06 2023-10-01T09:15:16.282 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.282 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.282 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.282 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:4618 5.c 5:338bc560:::benchmark_data_smithi138_240338_object4617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:15:16.282 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:45.940679+0000", 2023-10-01T09:15:16.282 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 30.104176998, 2023-10-01T09:15:16.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.081626252999999996, 2023-10-01T09:15:16.282 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.283 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.283 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:16.283 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:16.283 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:16.283 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 4618 2023-10-01T09:15:16.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.283 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.940679+0000", 2023-10-01T09:15:16.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.940679+0000", 2023-10-01T09:15:16.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.940678+0000", 2023-10-01T09:15:16.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:15:16.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.940702+0000", 2023-10-01T09:15:16.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3654000000000001e-05 2023-10-01T09:15:16.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.940715+0000", 2023-10-01T09:15:16.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2797e-05 2023-10-01T09:15:16.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.940722+0000", 2023-10-01T09:15:16.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6040000000000004e-06 2023-10-01T09:15:16.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.940764+0000", 2023-10-01T09:15:16.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2009999999999999e-05 2023-10-01T09:15:16.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.940858+0000", 2023-10-01T09:15:16.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3431999999999996e-05 2023-10-01T09:15:16.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:15:16.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.940937+0000", 2023-10-01T09:15:16.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9278999999999998e-05 2023-10-01T09:15:16.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:16.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.941349+0000", 2023-10-01T09:15:16.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00041177199999999997 2023-10-01T09:15:16.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:45.942075+0000", 2023-10-01T09:15:16.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00072592000000000002 2023-10-01T09:15:16.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.022194+0000", 2023-10-01T09:15:16.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.080119613000000006 2023-10-01T09:15:16.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.022259+0000", 2023-10-01T09:15:16.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5007999999999998e-05 2023-10-01T09:15:16.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.289 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:46.022306+0000", 2023-10-01T09:15:16.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6390000000000001e-05 2023-10-01T09:15:16.289 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.289 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.289 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.289 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:20459 5.c 5:31a572e0:::benchmark_data_smithi138_240338_object20458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-01T09:15:16.290 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:51.474264+0000", 2023-10-01T09:15:16.290 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 24.570592662999999, 2023-10-01T09:15:16.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16401922099999999, 2023-10-01T09:15:16.290 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.290 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.290 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:16.290 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:16.290 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:16.290 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 20459 2023-10-01T09:15:16.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.290 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.474264+0000", 2023-10-01T09:15:16.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.474264+0000", 2023-10-01T09:15:16.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.474262+0000", 2023-10-01T09:15:16.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:15:16.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.292 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.474288+0000", 2023-10-01T09:15:16.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5695e-05 2023-10-01T09:15:16.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.292 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.474303+0000", 2023-10-01T09:15:16.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5102999999999999e-05 2023-10-01T09:15:16.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.474311+0000", 2023-10-01T09:15:16.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0099999999999995e-06 2023-10-01T09:15:16.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.474387+0000", 2023-10-01T09:15:16.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5605000000000005e-05 2023-10-01T09:15:16.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.474529+0000", 2023-10-01T09:15:16.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014221500000000001 2023-10-01T09:15:16.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:15:16.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.474666+0000", 2023-10-01T09:15:16.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013704 2023-10-01T09:15:16.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.476970+0000", 2023-10-01T09:15:16.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002304216 2023-10-01T09:15:16.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.295 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:16.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.479291+0000", 2023-10-01T09:15:16.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002321198 2023-10-01T09:15:16.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.295 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.638174+0000", 2023-10-01T09:15:16.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.15888259299999999 2023-10-01T09:15:16.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.638242+0000", 2023-10-01T09:15:16.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7910999999999994e-05 2023-10-01T09:15:16.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.638283+0000", 2023-10-01T09:15:16.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1137999999999998e-05 2023-10-01T09:15:16.296 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.296 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.296 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.297 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.297 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:21188 5.c 5:32fcffe3:::benchmark_data_smithi138_240338_object21187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-01T09:15:16.297 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:51.938292+0000", 2023-10-01T09:15:16.297 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 24.106564025000001, 2023-10-01T09:15:16.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.092296390000000006, 2023-10-01T09:15:16.297 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.297 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.297 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:16.297 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:16.297 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:16.297 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21188 2023-10-01T09:15:16.297 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.298 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.298 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.298 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.298 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.938292+0000", 2023-10-01T09:15:16.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.298 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.298 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.298 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.298 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.938292+0000", 2023-10-01T09:15:16.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.298 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.298 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.938290+0000", 2023-10-01T09:15:16.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:15:16.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.938315+0000", 2023-10-01T09:15:16.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5058e-05 2023-10-01T09:15:16.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.938331+0000", 2023-10-01T09:15:16.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5781999999999999e-05 2023-10-01T09:15:16.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.938339+0000", 2023-10-01T09:15:16.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0499999999999992e-06 2023-10-01T09:15:16.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.938398+0000", 2023-10-01T09:15:16.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9126000000000003e-05 2023-10-01T09:15:16.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.301 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.301 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.301 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.938516+0000", 2023-10-01T09:15:16.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118306 2023-10-01T09:15:16.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.301 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.301 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:15:16.301 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.938619+0000", 2023-10-01T09:15:16.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000102828 2023-10-01T09:15:16.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.301 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.301 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:16.302 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.939540+0000", 2023-10-01T09:15:16.302 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00092038099999999996 2023-10-01T09:15:16.302 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.302 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.302 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.940947+0000", 2023-10-01T09:15:16.302 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014069340000000001 2023-10-01T09:15:16.302 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.302 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.302 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:52.030483+0000", 2023-10-01T09:15:16.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.089536004000000002 2023-10-01T09:15:16.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.303 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.303 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:52.030539+0000", 2023-10-01T09:15:16.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6666000000000003e-05 2023-10-01T09:15:16.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.303 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.303 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:52.030589+0000", 2023-10-01T09:15:16.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9512999999999998e-05 2023-10-01T09:15:16.303 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.303 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.304 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.304 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.304 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.304 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:21200 5.c 5:34692c59:::benchmark_data_smithi138_240338_object21199:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-01T09:15:16.304 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:51.942951+0000", 2023-10-01T09:15:16.304 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 24.101905524999999, 2023-10-01T09:15:16.304 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.087749202999999998, 2023-10-01T09:15:16.304 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.304 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.304 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:16.304 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:16.304 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:16.305 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 21200 2023-10-01T09:15:16.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.305 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.305 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.305 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.942951+0000", 2023-10-01T09:15:16.305 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.305 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.305 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.942951+0000", 2023-10-01T09:15:16.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.306 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.306 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.306 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.306 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.942949+0000", 2023-10-01T09:15:16.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:15:16.306 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.306 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.306 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.306 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.942971+0000", 2023-10-01T09:15:16.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1633000000000001e-05 2023-10-01T09:15:16.306 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.307 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.942984+0000", 2023-10-01T09:15:16.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2979999999999999e-05 2023-10-01T09:15:16.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.307 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.942990+0000", 2023-10-01T09:15:16.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7690000000000002e-06 2023-10-01T09:15:16.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.307 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.943043+0000", 2023-10-01T09:15:16.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.376e-05 2023-10-01T09:15:16.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.943150+0000", 2023-10-01T09:15:16.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106828 2023-10-01T09:15:16.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:15:16.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.943259+0000", 2023-10-01T09:15:16.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0001083 2023-10-01T09:15:16.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:16.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.943979+0000", 2023-10-01T09:15:16.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00072069400000000002 2023-10-01T09:15:16.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:51.944250+0000", 2023-10-01T09:15:16.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00027116199999999998 2023-10-01T09:15:16.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:52.030631+0000", 2023-10-01T09:15:16.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.086380495000000002 2023-10-01T09:15:16.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:52.030672+0000", 2023-10-01T09:15:16.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0765e-05 2023-10-01T09:15:16.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:52.030700+0000", 2023-10-01T09:15:16.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8368999999999999e-05 2023-10-01T09:15:16.311 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.311 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.311 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.311 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:24337 5.c 5:312de6e4:::benchmark_data_smithi138_240338_object24336:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-01T09:15:16.311 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:53.639477+0000", 2023-10-01T09:15:16.311 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 22.405379084, 2023-10-01T09:15:16.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.150263812, 2023-10-01T09:15:16.311 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.311 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.311 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:16.312 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:16.312 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:16.312 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 24337 2023-10-01T09:15:16.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.312 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.639477+0000", 2023-10-01T09:15:16.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.639477+0000", 2023-10-01T09:15:16.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.639476+0000", 2023-10-01T09:15:16.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:15:16.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.639498+0000", 2023-10-01T09:15:16.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1814000000000002e-05 2023-10-01T09:15:16.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.639509+0000", 2023-10-01T09:15:16.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0956e-05 2023-10-01T09:15:16.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.639515+0000", 2023-10-01T09:15:16.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3609999999999997e-06 2023-10-01T09:15:16.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.639561+0000", 2023-10-01T09:15:16.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6187999999999999e-05 2023-10-01T09:15:16.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.639656+0000", 2023-10-01T09:15:16.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4141000000000005e-05 2023-10-01T09:15:16.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:15:16.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.639786+0000", 2023-10-01T09:15:16.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013037300000000001 2023-10-01T09:15:16.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.316 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:16.316 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.640846+0000", 2023-10-01T09:15:16.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010601950000000001 2023-10-01T09:15:16.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.316 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.316 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.641931+0000", 2023-10-01T09:15:16.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001084539 2023-10-01T09:15:16.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.789656+0000", 2023-10-01T09:15:16.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.14772533400000001 2023-10-01T09:15:16.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.789703+0000", 2023-10-01T09:15:16.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7151000000000003e-05 2023-10-01T09:15:16.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.789741+0000", 2023-10-01T09:15:16.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8012000000000001e-05 2023-10-01T09:15:16.318 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.318 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.318 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.318 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:24346 5.c 5:3884d9d0:::benchmark_data_smithi138_240338_object24345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-01T09:15:16.318 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:53.642278+0000", 2023-10-01T09:15:16.318 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 22.402578504000001, 2023-10-01T09:15:16.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.14758669299999999, 2023-10-01T09:15:16.318 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.319 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.319 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:16.319 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:16.319 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:16.319 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 24346 2023-10-01T09:15:16.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.319 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.642278+0000", 2023-10-01T09:15:16.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.642278+0000", 2023-10-01T09:15:16.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.642276+0000", 2023-10-01T09:15:16.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:15:16.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.642298+0000", 2023-10-01T09:15:16.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2044000000000001e-05 2023-10-01T09:15:16.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.642308+0000", 2023-10-01T09:15:16.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8789999999999998e-06 2023-10-01T09:15:16.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.642314+0000", 2023-10-01T09:15:16.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5400000000000003e-06 2023-10-01T09:15:16.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.642349+0000", 2023-10-01T09:15:16.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4659999999999997e-05 2023-10-01T09:15:16.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.642427+0000", 2023-10-01T09:15:16.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8744000000000004e-05 2023-10-01T09:15:16.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:15:16.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.642532+0000", 2023-10-01T09:15:16.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010438 2023-10-01T09:15:16.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:16.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.643326+0000", 2023-10-01T09:15:16.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00079384900000000001 2023-10-01T09:15:16.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.324 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.324 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.644498+0000", 2023-10-01T09:15:16.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011729030000000001 2023-10-01T09:15:16.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.324 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.324 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.789788+0000", 2023-10-01T09:15:16.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.145289947 2023-10-01T09:15:16.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.789832+0000", 2023-10-01T09:15:16.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3226000000000003e-05 2023-10-01T09:15:16.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.789865+0000", 2023-10-01T09:15:16.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3008e-05 2023-10-01T09:15:16.325 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.325 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.325 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.326 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:24356 5.8 e112/106 5:1ff45074:::benchmark_data_smithi138_240338_object24355:head v 112'1526, mlcod=112'1524)", 2023-10-01T09:15:16.326 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:53.648512+0000", 2023-10-01T09:15:16.326 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 22.396344409000001, 2023-10-01T09:15:16.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.14077716400000001, 2023-10-01T09:15:16.326 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.326 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.326 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.648512+0000", 2023-10-01T09:15:16.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.648512+0000", 2023-10-01T09:15:16.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.648511+0000", 2023-10-01T09:15:16.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:15:16.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.648538+0000", 2023-10-01T09:15:16.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7250000000000002e-05 2023-10-01T09:15:16.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.648552+0000", 2023-10-01T09:15:16.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3787e-05 2023-10-01T09:15:16.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.648560+0000", 2023-10-01T09:15:16.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0409999999999998e-06 2023-10-01T09:15:16.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.329 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.648608+0000", 2023-10-01T09:15:16.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8310000000000003e-05 2023-10-01T09:15:16.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.329 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.648628+0000", 2023-10-01T09:15:16.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9799000000000001e-05 2023-10-01T09:15:16.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.789255+0000", 2023-10-01T09:15:16.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.14062655199999999 2023-10-01T09:15:16.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.789289+0000", 2023-10-01T09:15:16.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4374e-05 2023-10-01T09:15:16.330 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.330 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.331 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.331 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:24359 5.c 5:308ddf8b:::benchmark_data_smithi138_240338_object24358:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-01T09:15:16.331 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:14:53.651192+0000", 2023-10-01T09:15:16.331 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 22.393663979999999, 2023-10-01T09:15:16.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.144155164, 2023-10-01T09:15:16.331 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.331 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.331 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:16.331 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:16.331 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:16.332 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 24359 2023-10-01T09:15:16.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.332 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.651192+0000", 2023-10-01T09:15:16.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.651192+0000", 2023-10-01T09:15:16.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.651191+0000", 2023-10-01T09:15:16.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:15:16.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.651225+0000", 2023-10-01T09:15:16.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4248999999999997e-05 2023-10-01T09:15:16.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.787079+0000", 2023-10-01T09:15:16.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.13585420100000001 2023-10-01T09:15:16.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.787094+0000", 2023-10-01T09:15:16.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4333e-05 2023-10-01T09:15:16.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.787149+0000", 2023-10-01T09:15:16.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5448000000000002e-05 2023-10-01T09:15:16.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.787300+0000", 2023-10-01T09:15:16.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015101399999999999 2023-10-01T09:15:16.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:15:16.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.787448+0000", 2023-10-01T09:15:16.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014753300000000001 2023-10-01T09:15:16.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:16.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.789753+0000", 2023-10-01T09:15:16.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002305156 2023-10-01T09:15:16.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.789901+0000", 2023-10-01T09:15:16.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014842800000000001 2023-10-01T09:15:16.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.795266+0000", 2023-10-01T09:15:16.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0053652680000000003 2023-10-01T09:15:16.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.795306+0000", 2023-10-01T09:15:16.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9928000000000001e-05 2023-10-01T09:15:16.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:14:53.795348+0000", 2023-10-01T09:15:16.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1196000000000003e-05 2023-10-01T09:15:16.337 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.338 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.338 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.338 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:50510 5.8 e114/106 5:1bf8a332:::benchmark_data_smithi138_240338_object50509:head v 114'3220, mlcod=114'3216)", 2023-10-01T09:15:16.338 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:09.493571+0000", 2023-10-01T09:15:16.338 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 6.5512852449999999, 2023-10-01T09:15:16.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.072926052000000005, 2023-10-01T09:15:16.338 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.338 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.338 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:09.493571+0000", 2023-10-01T09:15:16.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:09.493571+0000", 2023-10-01T09:15:16.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:09.493571+0000", 2023-10-01T09:15:16.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:16.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:09.493589+0000", 2023-10-01T09:15:16.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8535000000000001e-05 2023-10-01T09:15:16.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:09.493596+0000", 2023-10-01T09:15:16.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1049999999999997e-06 2023-10-01T09:15:16.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:09.493601+0000", 2023-10-01T09:15:16.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7690000000000004e-06 2023-10-01T09:15:16.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:09.493638+0000", 2023-10-01T09:15:16.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7199000000000001e-05 2023-10-01T09:15:16.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:09.493660+0000", 2023-10-01T09:15:16.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1367000000000001e-05 2023-10-01T09:15:16.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:09.566456+0000", 2023-10-01T09:15:16.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.072796760000000002 2023-10-01T09:15:16.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:09.566497+0000", 2023-10-01T09:15:16.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0836e-05 2023-10-01T09:15:16.343 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.343 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.343 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.343 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:55961 5.c 5:300f7b57:::benchmark_data_smithi138_240338_object55960:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:16.343 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:13.664754+0000", 2023-10-01T09:15:16.343 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.3801028870000001, 2023-10-01T09:15:16.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.177662978, 2023-10-01T09:15:16.343 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.343 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.343 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:16.344 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:16.344 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:16.344 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 55961 2023-10-01T09:15:16.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.344 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664754+0000", 2023-10-01T09:15:16.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664754+0000", 2023-10-01T09:15:16.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664752+0000", 2023-10-01T09:15:16.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:15:16.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664778+0000", 2023-10-01T09:15:16.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5780000000000001e-05 2023-10-01T09:15:16.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664789+0000", 2023-10-01T09:15:16.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1657999999999999e-05 2023-10-01T09:15:16.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664799+0000", 2023-10-01T09:15:16.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0559999999999994e-06 2023-10-01T09:15:16.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664844+0000", 2023-10-01T09:15:16.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5562999999999997e-05 2023-10-01T09:15:16.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.665008+0000", 2023-10-01T09:15:16.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000164179 2023-10-01T09:15:16.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:15:16.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.665147+0000", 2023-10-01T09:15:16.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000138417 2023-10-01T09:15:16.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.348 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:16.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.666951+0000", 2023-10-01T09:15:16.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0018045310000000001 2023-10-01T09:15:16.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.348 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.670613+0000", 2023-10-01T09:15:16.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036620519999999998 2023-10-01T09:15:16.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.842339+0000", 2023-10-01T09:15:16.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.171726182 2023-10-01T09:15:16.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.842385+0000", 2023-10-01T09:15:16.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5179000000000002e-05 2023-10-01T09:15:16.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.842416+0000", 2023-10-01T09:15:16.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1841000000000002e-05 2023-10-01T09:15:16.350 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.350 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.350 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.350 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:55979 5.c 5:3811f30d:::benchmark_data_smithi138_240338_object55978:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:16.350 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:13.673785+0000", 2023-10-01T09:15:16.350 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.371071105, 2023-10-01T09:15:16.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.168727664, 2023-10-01T09:15:16.350 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.351 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.351 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:16.351 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:16.351 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:16.351 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 55979 2023-10-01T09:15:16.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.351 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.673785+0000", 2023-10-01T09:15:16.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.673785+0000", 2023-10-01T09:15:16.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.673784+0000", 2023-10-01T09:15:16.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:15:16.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.673802+0000", 2023-10-01T09:15:16.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7884999999999999e-05 2023-10-01T09:15:16.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.673811+0000", 2023-10-01T09:15:16.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.166e-06 2023-10-01T09:15:16.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.673817+0000", 2023-10-01T09:15:16.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3539999999999999e-06 2023-10-01T09:15:16.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.673859+0000", 2023-10-01T09:15:16.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2209000000000002e-05 2023-10-01T09:15:16.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.673955+0000", 2023-10-01T09:15:16.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6137000000000005e-05 2023-10-01T09:15:16.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:15:16.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.674037+0000", 2023-10-01T09:15:16.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2338999999999999e-05 2023-10-01T09:15:16.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:16.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.675586+0000", 2023-10-01T09:15:16.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001549101 2023-10-01T09:15:16.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.675690+0000", 2023-10-01T09:15:16.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103958 2023-10-01T09:15:16.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.842458+0000", 2023-10-01T09:15:16.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16676796499999999 2023-10-01T09:15:16.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.842488+0000", 2023-10-01T09:15:16.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9892000000000001e-05 2023-10-01T09:15:16.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.842513+0000", 2023-10-01T09:15:16.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4839000000000001e-05 2023-10-01T09:15:16.357 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.357 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.357 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.357 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:57127 5.c 5:39b1f8d7:::benchmark_data_smithi138_240338_object57126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:16.357 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:14.705478+0000", 2023-10-01T09:15:16.357 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.3393779750000001, 2023-10-01T09:15:16.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16845568899999999, 2023-10-01T09:15:16.358 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:16.358 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:16.358 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:16.358 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:16.358 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:16.358 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 57127 2023-10-01T09:15:16.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.358 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:16.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:16.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:14.705478+0000", 2023-10-01T09:15:16.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.359 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:16.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:14.705478+0000", 2023-10-01T09:15:16.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:16.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.359 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:16.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:14.705477+0000", 2023-10-01T09:15:16.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:15:16.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:16.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:14.705497+0000", 2023-10-01T09:15:16.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9565e-05 2023-10-01T09:15:16.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:16.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:14.705509+0000", 2023-10-01T09:15:16.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1980999999999999e-05 2023-10-01T09:15:16.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:16.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:14.705517+0000", 2023-10-01T09:15:16.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7910000000000005e-06 2023-10-01T09:15:16.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:16.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:14.705560+0000", 2023-10-01T09:15:16.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.316e-05 2023-10-01T09:15:16.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:16.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:14.705657+0000", 2023-10-01T09:15:16.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7100000000000002e-05 2023-10-01T09:15:16.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:15:16.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:14.705754+0000", 2023-10-01T09:15:16.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7266e-05 2023-10-01T09:15:16.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:16.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:14.707229+0000", 2023-10-01T09:15:16.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014751790000000001 2023-10-01T09:15:16.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:14.707547+0000", 2023-10-01T09:15:16.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031775 2023-10-01T09:15:16.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:16.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:14.873859+0000", 2023-10-01T09:15:16.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16631175200000001 2023-10-01T09:15:16.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:16.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:14.873898+0000", 2023-10-01T09:15:16.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8921000000000002e-05 2023-10-01T09:15:16.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:16.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:16.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:16.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:14.873934+0000", 2023-10-01T09:15:16.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6402999999999997e-05 2023-10-01T09:15:16.364 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.364 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.364 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.364 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:16.364 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:16.364 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:15:16.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:16.366 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:15:16.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:15:16.467 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:15:16.499 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:16.520 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 32 16 59091 59075 115.339 71.1875 0.0119774 0.00866539 2023-10-01T09:15:16.599 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:15:16.605 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:16 smithi190 ceph-mon[160435]: pgmap v253: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-01T09:15:16.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:16.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:15:16.707 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:15:16.754 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:16 smithi138 ceph-mon[192513]: pgmap v253: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-01T09:15:16.754 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:16 smithi138 ceph-mon[185886]: pgmap v253: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-01T09:15:16.764 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:17.520 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 33 16 60218 60202 113.978 70.4375 0.0381114 0.00876304 2023-10-01T09:15:17.774 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:15:17.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:17.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:15:17.850 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:17.878 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:15:17.886 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:15:17.923 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:17.990 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:17 smithi138 ceph-mon[192513]: pgmap v254: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T09:15:17.994 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:17 smithi138 ceph-mon[185886]: pgmap v254: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T09:15:18.016 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:15:18.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:18.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:15:18.126 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:15:18.169 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:17 smithi190 ceph-mon[160435]: pgmap v254: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T09:15:18.172 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:18.233 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:15:18.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:18.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:15:18.340 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:18.403 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:18.521 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 34 16 61426 61410 112.846 75.5 0.0181606 0.00885552 2023-10-01T09:15:18.566 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:18.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:18.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:18.677 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:18.748 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:18.946 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:18.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:18.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:18.993 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:18 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1420028333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:15:18.993 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:18 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/683760803' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:15:19.021 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:15:19.021 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.5 to 1 (10000) 2023-10-01T09:15:19.060 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:19.115 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:19.208 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:19.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:19.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:18 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1420028333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:15:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:18 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/683760803' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:15:19.314 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:15:19.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:18 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1420028333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:15:19.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:18 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/683760803' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:15:19.361 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:19.520 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 35 16 62738 62722 111.965 82 0.0103447 0.00892635 2023-10-01T09:15:20.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:19 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:15:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:19 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:15:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:19 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1420028333' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:15:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:19 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/683760803' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:15:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:19 smithi190 ceph-mon[160435]: osdmap e116: 8 total, 8 up, 8 in 2023-10-01T09:15:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:19 smithi190 ceph-mon[160435]: failed to encode map e116 with expected crc 2023-10-01T09:15:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:19 smithi190 ceph-mon[160435]: failed to encode map e116 with expected crc 2023-10-01T09:15:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:19 smithi190 ceph-mon[160435]: failed to encode map e116 with expected crc 2023-10-01T09:15:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:19 smithi190 ceph-mon[160435]: pgmap v256: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T09:15:20.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:15:20.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:15:20.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1420028333' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:15:20.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/683760803' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:15:20.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[192513]: osdmap e116: 8 total, 8 up, 8 in 2023-10-01T09:15:20.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[192513]: failed to encode map e116 with expected crc 2023-10-01T09:15:20.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[192513]: failed to encode map e116 with expected crc 2023-10-01T09:15:20.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[192513]: failed to encode map e116 with expected crc 2023-10-01T09:15:20.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[192513]: pgmap v256: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T09:15:20.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:15:20.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:15:20.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1420028333' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:15:20.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/683760803' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:15:20.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[185886]: osdmap e116: 8 total, 8 up, 8 in 2023-10-01T09:15:20.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[185886]: failed to encode map e116 with expected crc 2023-10-01T09:15:20.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[185886]: failed to encode map e116 with expected crc 2023-10-01T09:15:20.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[185886]: failed to encode map e116 with expected crc 2023-10-01T09:15:20.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:19 smithi138 ceph-mon[185886]: pgmap v256: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T09:15:20.521 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 36 16 64029 64013 111.096 80.6875 0.0106367 0.00899692 2023-10-01T09:15:20.726 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:15:20.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:20.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:15:20.833 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:20.897 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:20.918 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-01T09:15:21.109 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:21.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:21.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:21.213 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:21.277 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:21.521 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 37 16 65103 65087 109.908 67.125 0.010419 0.00909362 2023-10-01T09:15:21.795 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:21.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:21.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:21.899 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:15:21.966 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:22.231 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:15:22.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:22.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:15:22.257 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:22.326 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:15:22.335 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:22.363 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:22.367 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:15:22.395 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:22.413 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:15:22.457 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:15:22.492 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:15:22.521 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 38 16 66418 66402 109.177 82.1875 0.0122828 0.0091552 2023-10-01T09:15:22.525 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:15:22.543 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:22.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:22.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:22.558 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:15:22.602 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:15:22.602 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:15:22.602 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:15:22.602 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:15:22.602 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:15:22.603 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:15:22.603 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:15:22.603 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:15:22.603 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:15:22.636 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:22 smithi138 ceph-mon[192513]: pgmap v257: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-01T09:15:22.636 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:22 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1404818915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:22.637 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:22 smithi138 ceph-mon[185886]: pgmap v257: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-01T09:15:22.637 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:22 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1404818915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:22.655 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:15:22.729 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:22.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:22 smithi190 ceph-mon[160435]: pgmap v257: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-01T09:15:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:22 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1404818915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:23.166 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:15:23.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:23.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:15:23.271 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:15:23.326 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:23.446 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:15:23.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:23.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:15:23.521 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 39 16 67767 67751 108.539 84.3125 0.0110225 0.00920904 2023-10-01T09:15:23.553 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:23.602 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:24.084 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:15:24.137 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:24.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:24.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:24.250 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:15:24.316 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:24.404 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:15:24.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:24.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:15:24.514 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:24.521 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:15:24.522307+0000 min lat: 0.00109137 max lat: 0.284703 avg lat: 0.00928777 2023-10-01T09:15:24.522 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:15:24.522 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 40 16 68917 68901 107.622 71.875 0.0138931 0.00928777 2023-10-01T09:15:24.549 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:24 smithi138 ceph-mon[192513]: pgmap v258: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-01T09:15:24.550 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:24 smithi138 ceph-mon[185886]: pgmap v258: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-01T09:15:24.553 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:24.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:24 smithi190 ceph-mon[160435]: pgmap v258: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-01T09:15:25.199 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:15:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:15:24] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.33.4" 2023-10-01T09:15:25.201 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:25.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:25.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:25.304 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:25.375 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:25.523 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 41 16 70157 70141 106.883 77.5 0.00417545 0.00935141 2023-10-01T09:15:26.158 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:26.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:26.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:26.263 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:26.335 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:26.392 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:26.426 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:15:26.426 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:15:26.426 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:26 smithi138 ceph-mon[192513]: pgmap v259: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-01T09:15:26.427 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:15:26.427 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:15:26.427 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:26 smithi138 ceph-mon[185886]: pgmap v259: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-01T09:15:26.510 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:26.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:26.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:26.523 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 42 16 71254 71238 105.971 68.5625 0.0168286 0.00943125 2023-10-01T09:15:26.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:15:26.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:15:26.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:26 smithi190 ceph-mon[160435]: pgmap v259: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-01T09:15:26.613 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:15:26.702 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:26.706 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:15:26.706 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:15:26.997 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:15:27.524 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 43 16 72341 72325 105.086 67.9375 0.00358054 0.00949014 2023-10-01T09:15:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:27 smithi138 ceph-mon[192513]: pgmap v260: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-01T09:15:28.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:27 smithi138 ceph-mon[185886]: pgmap v260: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-01T09:15:28.089 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:15:28.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:28.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:15:28.201 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:28.265 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:28.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:27 smithi190 ceph-mon[160435]: pgmap v260: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-01T09:15:28.524 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 44 16 73455 73439 104.279 69.625 0.0144615 0.00958493 2023-10-01T09:15:28.864 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:28.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:28.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:28.970 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:15:29.029 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:15:29.039 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:29.099 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.6 to 1 (10000) 2023-10-01T09:15:29.106 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:15:29.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:29.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:15:29.129 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:29 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3301174214' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:15:29.129 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:29 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:15:29.131 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:29 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3301174214' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:15:29.132 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:29 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:15:29.213 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:15:29.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:29 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3301174214' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:15:29.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:29 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:15:29.339 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:29.525 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 45 16 74618 74602 103.577 72.6875 0.0168099 0.00964979 2023-10-01T09:15:30.102 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:15:30.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:30.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:15:30.210 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:15:30.267 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:30.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:30 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:15:30.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:30 smithi138 ceph-mon[185886]: osdmap e117: 8 total, 8 up, 8 in 2023-10-01T09:15:30.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:30 smithi138 ceph-mon[185886]: failed to encode map e117 with expected crc 2023-10-01T09:15:30.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:30 smithi138 ceph-mon[185886]: failed to encode map e117 with expected crc 2023-10-01T09:15:30.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:30 smithi138 ceph-mon[185886]: pgmap v262: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-01T09:15:30.524 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 46 16 75759 75743 102.876 71.3125 0.0082999 0.00971663 2023-10-01T09:15:30.528 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:30.529 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:30 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:15:30.530 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:30 smithi138 ceph-mon[192513]: osdmap e117: 8 total, 8 up, 8 in 2023-10-01T09:15:30.530 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:30 smithi138 ceph-mon[192513]: failed to encode map e117 with expected crc 2023-10-01T09:15:30.530 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:30 smithi138 ceph-mon[192513]: failed to encode map e117 with expected crc 2023-10-01T09:15:30.530 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:30 smithi138 ceph-mon[192513]: pgmap v262: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-01T09:15:30.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:30 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:15:30.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:30 smithi190 ceph-mon[160435]: osdmap e117: 8 total, 8 up, 8 in 2023-10-01T09:15:30.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:30 smithi190 ceph-mon[160435]: failed to encode map e117 with expected crc 2023-10-01T09:15:30.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:30 smithi190 ceph-mon[160435]: failed to encode map e117 with expected crc 2023-10-01T09:15:30.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:30 smithi190 ceph-mon[160435]: pgmap v262: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-01T09:15:30.558 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-01T09:15:30.608 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:15:30.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:30.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:15:30.715 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:15:30.725 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:30.772 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:30.861 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:15:30.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:30.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:15:30.971 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:15:31.024 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:31.109 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:15:31.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:31.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:15:31.139 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:15:31.170 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/658008430' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:15:31.170 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:31 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:15:31.170 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/658008430' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:15:31.170 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:31 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:15:31.220 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:15:31.270 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/658008430' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:15:31.270 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:31 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:15:31.273 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:31.364 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:15:31.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:31.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:15:31.474 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:15:31.525 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 47 16 76885 76869 102.184 70.375 0.00572529 0.00978042 2023-10-01T09:15:31.532 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:31.620 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:15:31.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:31.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:15:31.728 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:15:31.801 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:31.907 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:15:31.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:31.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:15:32.013 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:15:32.045 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:32.090 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:15:32.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:32.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:15:32.197 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:32.255 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:32.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:32 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:15:32.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:32 smithi190 ceph-mon[160435]: failed to encode map e118 with expected crc 2023-10-01T09:15:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:32 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:15:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:32 smithi190 ceph-mon[160435]: osdmap e118: 8 total, 8 up, 8 in 2023-10-01T09:15:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:32 smithi190 ceph-mon[160435]: pgmap v264: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T09:15:32.457 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:32 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:15:32.457 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:32 smithi138 ceph-mon[185886]: failed to encode map e118 with expected crc 2023-10-01T09:15:32.457 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:32 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:15:32.457 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:32 smithi138 ceph-mon[185886]: osdmap e118: 8 total, 8 up, 8 in 2023-10-01T09:15:32.457 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:32 smithi138 ceph-mon[185886]: pgmap v264: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T09:15:32.458 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:32 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:15:32.458 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:32 smithi138 ceph-mon[192513]: failed to encode map e118 with expected crc 2023-10-01T09:15:32.458 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:32 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:15:32.458 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:32 smithi138 ceph-mon[192513]: osdmap e118: 8 total, 8 up, 8 in 2023-10-01T09:15:32.458 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:32 smithi138 ceph-mon[192513]: pgmap v264: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T09:15:32.525 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 48 16 77699 77683 101.115 50.875 0.0150638 0.0098848 2023-10-01T09:15:33.525 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 49 16 78717 78701 100.35 63.625 0.00718524 0.00996056 2023-10-01T09:15:33.654 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:33.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:33.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:33.759 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:15:33.805 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:33.877 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:15:33.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:33.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:15:33.968 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:33.982 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:34.057 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:34.238 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:34.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:34.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:34.342 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:15:34.391 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:34.492 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:15:34.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:34.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:15:34.525 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 50 16 79846 79830 99.7546 70.5625 0.0157707 0.0100194 2023-10-01T09:15:34.598 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:34.607 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:15:34.635 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:34 smithi138 ceph-mon[185886]: pgmap v265: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-01T09:15:34.640 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:34 smithi138 ceph-mon[192513]: pgmap v265: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-01T09:15:34.652 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:15:34.681 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:34.692 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:15:34.706 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:15:34.706 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:15:34.706 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:15:34.706 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:15:34.706 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:15:34.737 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:15:34.767 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:34.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:34.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:34.772 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:15:34.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:34 smithi190 ceph-mon[160435]: pgmap v265: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-01T09:15:34.819 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:15:34.848 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:15:34.872 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:15:34.890 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:15:34.892 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:15:34.892 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:15:34.892 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:15:34.892 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:15:34.892 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:15:34.892 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:15:34.893 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:15:34.893 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:15:34.929 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:35.015 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:15:35.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:35.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:15:35.124 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:15:35.181 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:35.302 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:15:35.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:35.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:15:35.317 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:15:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:15:34] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.33.4" 2023-10-01T09:15:35.416 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:35.466 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:15:35.481 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:35.528 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 51 16 80956 80940 99.1588 69.375 0.018307 0.0100807 2023-10-01T09:15:35.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1709088227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:35.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1709088227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:35.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1709088227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:36.369 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:36 smithi138 ceph-mon[192513]: pgmap v266: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-10-01T09:15:36.371 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:36 smithi138 ceph-mon[185886]: pgmap v266: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-10-01T09:15:36.498 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:36.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:36.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:36.525 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 52 16 81918 81902 98.4082 60.125 0.0146197 0.0101571 2023-10-01T09:15:36.603 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:15:36.659 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:36 smithi190 ceph-mon[160435]: pgmap v266: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-10-01T09:15:36.672 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:36.736 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:15:36.777 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:15:36.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:36.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:15:36.884 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:15:36.946 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:37.036 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:15:37.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:37.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:15:37.141 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:15:37.170 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:37.264 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:15:37.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:37.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:15:37.378 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:15:37.430 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:37.525 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 53 16 82900 82884 97.7096 61.375 0.0143428 0.0102298 2023-10-01T09:15:37.535 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:15:37.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:37.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:15:37.644 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:15:37.692 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:37.737 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:37.800 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:15:37.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:37.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:15:37.826 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:37.910 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:37.979 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:38.246 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:38.252 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:37 smithi138 ceph-mon[192513]: pgmap v267: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T09:15:38.253 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:37 smithi138 ceph-mon[185886]: pgmap v267: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T09:15:38.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:38.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:37 smithi190 ceph-mon[160435]: pgmap v267: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T09:15:38.356 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:15:38.394 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:38.462 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:15:38.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:38.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:15:38.525 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 54 16 83931 83915 97.0934 64.4375 0.0139264 0.0102943 2023-10-01T09:15:38.574 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:38.641 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:38.692 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:15:39.031 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.7 to 1 (10000) 2023-10-01T09:15:39.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:39 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/397994703' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:15:39.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:39 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:15:39.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:39 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/397994703' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:15:39.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:39 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:15:39.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:39 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/397994703' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:15:39.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:39 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:15:39.526 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 55 16 85028 85012 96.5747 68.5625 0.00325085 0.0103461 2023-10-01T09:15:40.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:40 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:15:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:40 smithi190 ceph-mon[160435]: osdmap e119: 8 total, 8 up, 8 in 2023-10-01T09:15:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:40 smithi190 ceph-mon[160435]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:40 smithi190 ceph-mon[160435]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:40 smithi190 ceph-mon[160435]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:40 smithi190 ceph-mon[160435]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:40 smithi190 ceph-mon[160435]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:40 smithi190 ceph-mon[160435]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:40 smithi190 ceph-mon[160435]: pgmap v269: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T09:15:40.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:15:40.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[192513]: osdmap e119: 8 total, 8 up, 8 in 2023-10-01T09:15:40.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[192513]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[192513]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[192513]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[192513]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[192513]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[192513]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[192513]: pgmap v269: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T09:15:40.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:15:40.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[185886]: osdmap e119: 8 total, 8 up, 8 in 2023-10-01T09:15:40.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[185886]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[185886]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[185886]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[185886]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[185886]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[185886]: failed to encode map e119 with expected crc 2023-10-01T09:15:40.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:40 smithi138 ceph-mon[185886]: pgmap v269: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T09:15:40.527 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 56 16 86004 85988 95.9361 61 0.0169709 0.0104188 2023-10-01T09:15:40.650 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:40.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:40.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:40.758 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:15:40.812 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:40.951 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:15:40.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:40.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:15:41.065 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:15:41.129 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:41.239 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:15:41.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:41.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:15:41.263 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:15:41.263 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:15:41.263 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:15:41.264 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:15:41.346 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:15:41.415 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:15:41.415 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:15:41.415 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:41.478 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:15:41.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:41.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:15:41.528 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 57 16 86958 86942 95.299 59.625 0.00366176 0.0104737 2023-10-01T09:15:41.582 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:15:41.608 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:42.086 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:15:42.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:42.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:15:42.191 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:15:42.237 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:42.272 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:42 smithi138 ceph-mon[185886]: pgmap v270: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-01T09:15:42.320 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:15:42.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:42.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:15:42.426 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:42.451 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:42.497 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-01T09:15:42.497 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:15:42.524 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:42 smithi138 ceph-mon[192513]: pgmap v270: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-01T09:15:42.529 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 58 16 87886 87870 94.6532 58 0.0253998 0.0105588 2023-10-01T09:15:42.539 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:15:42.539 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:15:42.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:15:42.539 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:15:42.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.539 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:51586 5.6 5:64f89edb:::benchmark_data_smithi138_240338_object51585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:42.539 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:10.187685+0000", 2023-10-01T09:15:42.540 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 32.340377029999999, 2023-10-01T09:15:42.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.211679215, 2023-10-01T09:15:42.540 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.540 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.540 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 51586 2023-10-01T09:15:42.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.540 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187685+0000", 2023-10-01T09:15:42.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187685+0000", 2023-10-01T09:15:42.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187684+0000", 2023-10-01T09:15:42.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:42.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187710+0000", 2023-10-01T09:15:42.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6098999999999999e-05 2023-10-01T09:15:42.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187718+0000", 2023-10-01T09:15:42.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1410000000000005e-06 2023-10-01T09:15:42.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187724+0000", 2023-10-01T09:15:42.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.857e-06 2023-10-01T09:15:42.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187770+0000", 2023-10-01T09:15:42.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5670999999999997e-05 2023-10-01T09:15:42.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187886+0000", 2023-10-01T09:15:42.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000116358 2023-10-01T09:15:42.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:15:42.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.188020+0000", 2023-10-01T09:15:42.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013424199999999999 2023-10-01T09:15:42.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.190097+0000", 2023-10-01T09:15:42.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020765409999999999 2023-10-01T09:15:42.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.194751+0000", 2023-10-01T09:15:42.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0046543130000000002 2023-10-01T09:15:42.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.399283+0000", 2023-10-01T09:15:42.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20453162799999999 2023-10-01T09:15:42.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.399328+0000", 2023-10-01T09:15:42.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5281999999999998e-05 2023-10-01T09:15:42.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.399364+0000", 2023-10-01T09:15:42.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5512000000000002e-05 2023-10-01T09:15:42.546 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.546 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.547 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.547 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:51587 5.d 5:b6ed4186:::benchmark_data_smithi138_240338_object51586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:42.547 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:10.187736+0000", 2023-10-01T09:15:42.547 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 32.340325976000003, 2023-10-01T09:15:42.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.209777993, 2023-10-01T09:15:42.547 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.547 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.547 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.547 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.547 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.548 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 51587 2023-10-01T09:15:42.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.548 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187736+0000", 2023-10-01T09:15:42.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187736+0000", 2023-10-01T09:15:42.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187735+0000", 2023-10-01T09:15:42.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:42.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187755+0000", 2023-10-01T09:15:42.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9463999999999999e-05 2023-10-01T09:15:42.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187762+0000", 2023-10-01T09:15:42.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2309999999999999e-06 2023-10-01T09:15:42.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187767+0000", 2023-10-01T09:15:42.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9239999999999998e-06 2023-10-01T09:15:42.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187811+0000", 2023-10-01T09:15:42.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3828000000000001e-05 2023-10-01T09:15:42.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187891+0000", 2023-10-01T09:15:42.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0913000000000003e-05 2023-10-01T09:15:42.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:15:42.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.187985+0000", 2023-10-01T09:15:42.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3692999999999999e-05 2023-10-01T09:15:42.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.192020+0000", 2023-10-01T09:15:42.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040353300000000002 2023-10-01T09:15:42.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.393119+0000", 2023-10-01T09:15:42.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.201098528 2023-10-01T09:15:42.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.397425+0000", 2023-10-01T09:15:42.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004305957 2023-10-01T09:15:42.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.397473+0000", 2023-10-01T09:15:42.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8086e-05 2023-10-01T09:15:42.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.397514+0000", 2023-10-01T09:15:42.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0552000000000001e-05 2023-10-01T09:15:42.554 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.554 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.554 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.555 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:51591 5.d 5:bc4f6af6:::benchmark_data_smithi138_240338_object51590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:42.555 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:10.189937+0000", 2023-10-01T09:15:42.555 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 32.338124094000001, 2023-10-01T09:15:42.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.207661397, 2023-10-01T09:15:42.555 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.555 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.555 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.555 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.555 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.556 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 51591 2023-10-01T09:15:42.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.556 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.189937+0000", 2023-10-01T09:15:42.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.189937+0000", 2023-10-01T09:15:42.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.189936+0000", 2023-10-01T09:15:42.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:15:42.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.558 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.189964+0000", 2023-10-01T09:15:42.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8109000000000001e-05 2023-10-01T09:15:42.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.558 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.558 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.189980+0000", 2023-10-01T09:15:42.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5707999999999999e-05 2023-10-01T09:15:42.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.189990+0000", 2023-10-01T09:15:42.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0359e-05 2023-10-01T09:15:42.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.559 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.190039+0000", 2023-10-01T09:15:42.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8692000000000001e-05 2023-10-01T09:15:42.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.190153+0000", 2023-10-01T09:15:42.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000114047 2023-10-01T09:15:42.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:15:42.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.190281+0000", 2023-10-01T09:15:42.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000128288 2023-10-01T09:15:42.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.193352+0000", 2023-10-01T09:15:42.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030703739999999998 2023-10-01T09:15:42.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.393151+0000", 2023-10-01T09:15:42.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19979897899999999 2023-10-01T09:15:42.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.397541+0000", 2023-10-01T09:15:42.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043907119999999997 2023-10-01T09:15:42.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.397568+0000", 2023-10-01T09:15:42.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6580999999999998e-05 2023-10-01T09:15:42.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.397599+0000", 2023-10-01T09:15:42.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0858000000000002e-05 2023-10-01T09:15:42.563 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.563 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.563 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.563 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:51597 5.8 5:144d22f2:::benchmark_data_smithi138_240338_object51596:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:42.564 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:10.193691+0000", 2023-10-01T09:15:42.564 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 32.334370659999998, 2023-10-01T09:15:42.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19938302999999999, 2023-10-01T09:15:42.564 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.564 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.564 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.564 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.564 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.564 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 51597 2023-10-01T09:15:42.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.564 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.193691+0000", 2023-10-01T09:15:42.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.193691+0000", 2023-10-01T09:15:42.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.193690+0000", 2023-10-01T09:15:42.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:15:42.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.193709+0000", 2023-10-01T09:15:42.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9618000000000001e-05 2023-10-01T09:15:42.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.193719+0000", 2023-10-01T09:15:42.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3899999999999999e-06 2023-10-01T09:15:42.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.193724+0000", 2023-10-01T09:15:42.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.536e-06 2023-10-01T09:15:42.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.193766+0000", 2023-10-01T09:15:42.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2403e-05 2023-10-01T09:15:42.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.193889+0000", 2023-10-01T09:15:42.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012255999999999999 2023-10-01T09:15:42.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:15:42.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.194002+0000", 2023-10-01T09:15:42.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113076 2023-10-01T09:15:42.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.195911+0000", 2023-10-01T09:15:42.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0019090190000000001 2023-10-01T09:15:42.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.195952+0000", 2023-10-01T09:15:42.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0518000000000001e-05 2023-10-01T09:15:42.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.196186+0000", 2023-10-01T09:15:42.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00023405600000000001 2023-10-01T09:15:42.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.196234+0000", 2023-10-01T09:15:42.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8761999999999998e-05 2023-10-01T09:15:42.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.393074+0000", 2023-10-01T09:15:42.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.196839446 2023-10-01T09:15:42.571 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.571 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.571 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.571 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:51598 5.a 5:5c8636c7:::benchmark_data_smithi138_240338_object51597:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:42.571 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:10.194035+0000", 2023-10-01T09:15:42.571 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 32.334026840999996, 2023-10-01T09:15:42.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19942737999999999, 2023-10-01T09:15:42.572 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.572 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.572 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.572 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.572 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.572 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 51598 2023-10-01T09:15:42.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.572 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.194035+0000", 2023-10-01T09:15:42.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.194035+0000", 2023-10-01T09:15:42.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.194034+0000", 2023-10-01T09:15:42.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:42.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.194051+0000", 2023-10-01T09:15:42.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7336000000000001e-05 2023-10-01T09:15:42.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.194057+0000", 2023-10-01T09:15:42.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9429999999999997e-06 2023-10-01T09:15:42.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.194061+0000", 2023-10-01T09:15:42.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9899999999999999e-06 2023-10-01T09:15:42.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.194101+0000", 2023-10-01T09:15:42.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9912e-05 2023-10-01T09:15:42.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.194218+0000", 2023-10-01T09:15:42.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011673800000000001 2023-10-01T09:15:42.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:15:42.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.194340+0000", 2023-10-01T09:15:42.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012222599999999999 2023-10-01T09:15:42.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.196117+0000", 2023-10-01T09:15:42.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017770850000000001 2023-10-01T09:15:42.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.393122+0000", 2023-10-01T09:15:42.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19700441199999999 2023-10-01T09:15:42.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.393376+0000", 2023-10-01T09:15:42.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025392500000000001 2023-10-01T09:15:42.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.393420+0000", 2023-10-01T09:15:42.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4255999999999998e-05 2023-10-01T09:15:42.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:10.393462+0000", 2023-10-01T09:15:42.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2188999999999999e-05 2023-10-01T09:15:42.579 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.579 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.579 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.579 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:55958 5.8 5:1fc69509:::benchmark_data_smithi138_240338_object55957:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:42.579 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:13.661437+0000", 2023-10-01T09:15:42.579 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 28.866624428000002, 2023-10-01T09:15:42.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20815783700000001, 2023-10-01T09:15:42.579 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.580 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.580 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.580 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.580 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.580 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 55958 2023-10-01T09:15:42.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.580 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.661437+0000", 2023-10-01T09:15:42.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.661437+0000", 2023-10-01T09:15:42.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.661435+0000", 2023-10-01T09:15:42.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:15:42.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.661468+0000", 2023-10-01T09:15:42.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2586000000000003e-05 2023-10-01T09:15:42.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.661477+0000", 2023-10-01T09:15:42.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1099999999999992e-06 2023-10-01T09:15:42.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.661482+0000", 2023-10-01T09:15:42.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2639999999999999e-06 2023-10-01T09:15:42.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.661539+0000", 2023-10-01T09:15:42.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6347000000000001e-05 2023-10-01T09:15:42.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.661674+0000", 2023-10-01T09:15:42.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013502800000000001 2023-10-01T09:15:42.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:15:42.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.661805+0000", 2023-10-01T09:15:42.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131285 2023-10-01T09:15:42.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865127+0000", 2023-10-01T09:15:42.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.203321637 2023-10-01T09:15:42.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865158+0000", 2023-10-01T09:15:42.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0756000000000002e-05 2023-10-01T09:15:42.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.869499+0000", 2023-10-01T09:15:42.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043413699999999998 2023-10-01T09:15:42.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.869555+0000", 2023-10-01T09:15:42.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6236000000000002e-05 2023-10-01T09:15:42.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.869595+0000", 2023-10-01T09:15:42.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9838e-05 2023-10-01T09:15:42.586 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.586 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.587 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.587 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:55960 5.8 5:1864da53:::benchmark_data_smithi138_240338_object55959:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:42.587 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:13.663273+0000", 2023-10-01T09:15:42.587 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 28.864788996000001, 2023-10-01T09:15:42.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.206395157, 2023-10-01T09:15:42.587 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.587 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.587 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.587 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.587 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.588 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 55960 2023-10-01T09:15:42.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.588 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.663273+0000", 2023-10-01T09:15:42.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.663273+0000", 2023-10-01T09:15:42.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.663271+0000", 2023-10-01T09:15:42.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:15:42.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.663292+0000", 2023-10-01T09:15:42.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0741999999999999e-05 2023-10-01T09:15:42.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.663305+0000", 2023-10-01T09:15:42.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2816999999999999e-05 2023-10-01T09:15:42.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.663313+0000", 2023-10-01T09:15:42.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4509999999999992e-06 2023-10-01T09:15:42.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.663791+0000", 2023-10-01T09:15:42.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00047804999999999999 2023-10-01T09:15:42.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.663922+0000", 2023-10-01T09:15:42.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013078300000000001 2023-10-01T09:15:42.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:15:42.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664013+0000", 2023-10-01T09:15:42.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0618999999999994e-05 2023-10-01T09:15:42.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865197+0000", 2023-10-01T09:15:42.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20118375099999999 2023-10-01T09:15:42.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865245+0000", 2023-10-01T09:15:42.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.799e-05 2023-10-01T09:15:42.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.869603+0000", 2023-10-01T09:15:42.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043580859999999997 2023-10-01T09:15:42.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.869638+0000", 2023-10-01T09:15:42.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5543000000000002e-05 2023-10-01T09:15:42.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.869668+0000", 2023-10-01T09:15:42.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9522e-05 2023-10-01T09:15:42.594 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.594 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.594 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.594 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.34683.0:55958 5.8 e115/106 ondisk, result = 0)", 2023-10-01T09:15:42.595 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:13.663825+0000", 2023-10-01T09:15:42.595 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 28.864236984000001, 2023-10-01T09:15:42.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20130467899999999, 2023-10-01T09:15:42.595 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.595 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:15:42.595 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.663825+0000", 2023-10-01T09:15:42.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.663825+0000", 2023-10-01T09:15:42.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.663824+0000", 2023-10-01T09:15:42.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:42.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.663828+0000", 2023-10-01T09:15:42.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1999999999999996e-06 2023-10-01T09:15:42.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.663836+0000", 2023-10-01T09:15:42.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7819999999999994e-06 2023-10-01T09:15:42.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.663841+0000", 2023-10-01T09:15:42.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0860000000000001e-06 2023-10-01T09:15:42.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865114+0000", 2023-10-01T09:15:42.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.201273123 2023-10-01T09:15:42.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865121+0000", 2023-10-01T09:15:42.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7599999999999997e-06 2023-10-01T09:15:42.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865129+0000", 2023-10-01T09:15:42.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3080000000000008e-06 2023-10-01T09:15:42.599 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.599 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.599 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.600 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.34683.0:55958 5.8 e115/106 ondisk, result = 0)", 2023-10-01T09:15:42.600 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:13.664440+0000", 2023-10-01T09:15:42.600 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 28.863622009, 2023-10-01T09:15:42.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20072452399999999, 2023-10-01T09:15:42.600 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.600 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:15:42.600 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664440+0000", 2023-10-01T09:15:42.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664440+0000", 2023-10-01T09:15:42.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664439+0000", 2023-10-01T09:15:42.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:42.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664444+0000", 2023-10-01T09:15:42.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7589999999999997e-06 2023-10-01T09:15:42.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664454+0000", 2023-10-01T09:15:42.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9480000000000003e-06 2023-10-01T09:15:42.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664460+0000", 2023-10-01T09:15:42.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.353e-06 2023-10-01T09:15:42.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865148+0000", 2023-10-01T09:15:42.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20068762300000001 2023-10-01T09:15:42.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865153+0000", 2023-10-01T09:15:42.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5400000000000003e-06 2023-10-01T09:15:42.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865164+0000", 2023-10-01T09:15:42.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0969e-05 2023-10-01T09:15:42.604 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.604 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.604 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.605 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:55964 5.b 5:d2c29d90:::benchmark_data_smithi138_240338_object55963:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:42.605 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:13.664768+0000", 2023-10-01T09:15:42.605 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 28.863293283000001, 2023-10-01T09:15:42.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20731902499999999, 2023-10-01T09:15:42.605 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.605 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.605 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.605 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.605 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.606 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 55964 2023-10-01T09:15:42.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.606 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664768+0000", 2023-10-01T09:15:42.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664768+0000", 2023-10-01T09:15:42.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664767+0000", 2023-10-01T09:15:42.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:15:42.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664790+0000", 2023-10-01T09:15:42.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3014e-05 2023-10-01T09:15:42.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664802+0000", 2023-10-01T09:15:42.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1744e-05 2023-10-01T09:15:42.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664809+0000", 2023-10-01T09:15:42.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.464e-06 2023-10-01T09:15:42.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865063+0000", 2023-10-01T09:15:42.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20025393 2023-10-01T09:15:42.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865201+0000", 2023-10-01T09:15:42.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013750700000000001 2023-10-01T09:15:42.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:15:42.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865333+0000", 2023-10-01T09:15:42.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013191200000000001 2023-10-01T09:15:42.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.871522+0000", 2023-10-01T09:15:42.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0061897519999999998 2023-10-01T09:15:42.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.871590+0000", 2023-10-01T09:15:42.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7451000000000002e-05 2023-10-01T09:15:42.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.872006+0000", 2023-10-01T09:15:42.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000416432 2023-10-01T09:15:42.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.872051+0000", 2023-10-01T09:15:42.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4249999999999998e-05 2023-10-01T09:15:42.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.872087+0000", 2023-10-01T09:15:42.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6674999999999997e-05 2023-10-01T09:15:42.612 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.612 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.612 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.612 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:55966 5.a 5:506ad48e:::benchmark_data_smithi138_240338_object55965:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:42.612 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:13.664838+0000", 2023-10-01T09:15:42.612 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 28.863223971, 2023-10-01T09:15:42.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20979150699999999, 2023-10-01T09:15:42.613 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.613 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.613 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.613 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.613 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.613 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 55966 2023-10-01T09:15:42.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.613 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664838+0000", 2023-10-01T09:15:42.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664838+0000", 2023-10-01T09:15:42.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664837+0000", 2023-10-01T09:15:42.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:42.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.664864+0000", 2023-10-01T09:15:42.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6798e-05 2023-10-01T09:15:42.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865040+0000", 2023-10-01T09:15:42.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20017658399999999 2023-10-01T09:15:42.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865057+0000", 2023-10-01T09:15:42.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6265000000000001e-05 2023-10-01T09:15:42.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865105+0000", 2023-10-01T09:15:42.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7929999999999997e-05 2023-10-01T09:15:42.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865233+0000", 2023-10-01T09:15:42.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000128505 2023-10-01T09:15:42.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:15:42.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.865345+0000", 2023-10-01T09:15:42.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112248 2023-10-01T09:15:42.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.871344+0000", 2023-10-01T09:15:42.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059984610000000001 2023-10-01T09:15:42.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.871375+0000", 2023-10-01T09:15:42.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1621000000000001e-05 2023-10-01T09:15:42.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.874547+0000", 2023-10-01T09:15:42.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031711159999999999 2023-10-01T09:15:42.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.874594+0000", 2023-10-01T09:15:42.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7649999999999999e-05 2023-10-01T09:15:42.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:13.874629+0000", 2023-10-01T09:15:42.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4892999999999999e-05 2023-10-01T09:15:42.619 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.619 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.619 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.620 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:59027 5.6 5:691491df:::benchmark_data_smithi138_240338_object59026:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:42.620 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:16.271551+0000", 2023-10-01T09:15:42.620 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 26.256510950999999, 2023-10-01T09:15:42.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21041401700000001, 2023-10-01T09:15:42.620 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.620 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.620 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.620 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.620 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.621 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 59027 2023-10-01T09:15:42.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.621 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.271551+0000", 2023-10-01T09:15:42.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.271551+0000", 2023-10-01T09:15:42.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.271549+0000", 2023-10-01T09:15:42.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:15:42.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.271572+0000", 2023-10-01T09:15:42.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3172000000000001e-05 2023-10-01T09:15:42.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.271584+0000", 2023-10-01T09:15:42.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2019e-05 2023-10-01T09:15:42.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.271592+0000", 2023-10-01T09:15:42.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5970000000000003e-06 2023-10-01T09:15:42.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.271633+0000", 2023-10-01T09:15:42.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0918999999999999e-05 2023-10-01T09:15:42.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.271749+0000", 2023-10-01T09:15:42.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011651599999999999 2023-10-01T09:15:42.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:15:42.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.271876+0000", 2023-10-01T09:15:42.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000126907 2023-10-01T09:15:42.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.281308+0000", 2023-10-01T09:15:42.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094315200000000005 2023-10-01T09:15:42.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.281337+0000", 2023-10-01T09:15:42.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9601e-05 2023-10-01T09:15:42.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.481878+0000", 2023-10-01T09:15:42.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20054074899999999 2023-10-01T09:15:42.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.481927+0000", 2023-10-01T09:15:42.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8983000000000002e-05 2023-10-01T09:15:42.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.481965+0000", 2023-10-01T09:15:42.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7441000000000003e-05 2023-10-01T09:15:42.627 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.627 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.627 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.627 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:59028 5.a 5:59daa292:::benchmark_data_smithi138_240338_object59027:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:42.627 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:16.273936+0000", 2023-10-01T09:15:42.628 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 26.254125613999999, 2023-10-01T09:15:42.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20800682100000001, 2023-10-01T09:15:42.628 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.628 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.628 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.628 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.628 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.628 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 59028 2023-10-01T09:15:42.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.628 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.273936+0000", 2023-10-01T09:15:42.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.273936+0000", 2023-10-01T09:15:42.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.273934+0000", 2023-10-01T09:15:42.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:15:42.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.273964+0000", 2023-10-01T09:15:42.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0318999999999999e-05 2023-10-01T09:15:42.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.273978+0000", 2023-10-01T09:15:42.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3521e-05 2023-10-01T09:15:42.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.273987+0000", 2023-10-01T09:15:42.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1439999999999992e-06 2023-10-01T09:15:42.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.274115+0000", 2023-10-01T09:15:42.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012767200000000001 2023-10-01T09:15:42.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.274284+0000", 2023-10-01T09:15:42.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000169319 2023-10-01T09:15:42.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:15:42.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.274444+0000", 2023-10-01T09:15:42.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000159393 2023-10-01T09:15:42.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.281434+0000", 2023-10-01T09:15:42.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069907770000000001 2023-10-01T09:15:42.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.281449+0000", 2023-10-01T09:15:42.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4739999999999999e-05 2023-10-01T09:15:42.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.481856+0000", 2023-10-01T09:15:42.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20040664 2023-10-01T09:15:42.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.481913+0000", 2023-10-01T09:15:42.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6805999999999998e-05 2023-10-01T09:15:42.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.481943+0000", 2023-10-01T09:15:42.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0239e-05 2023-10-01T09:15:42.635 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.635 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.635 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.635 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:59029 5.a 5:5b69c3fe:::benchmark_data_smithi138_240338_object59028:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:42.635 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:16.274003+0000", 2023-10-01T09:15:42.635 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 26.254058625999999, 2023-10-01T09:15:42.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.208126171, 2023-10-01T09:15:42.635 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.636 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.636 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.636 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.636 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.636 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 59029 2023-10-01T09:15:42.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.636 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.274003+0000", 2023-10-01T09:15:42.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.274003+0000", 2023-10-01T09:15:42.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.274002+0000", 2023-10-01T09:15:42.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:42.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.274039+0000", 2023-10-01T09:15:42.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6276000000000003e-05 2023-10-01T09:15:42.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.274047+0000", 2023-10-01T09:15:42.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.841e-06 2023-10-01T09:15:42.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.274052+0000", 2023-10-01T09:15:42.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9560000000000002e-06 2023-10-01T09:15:42.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.276616+0000", 2023-10-01T09:15:42.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025636360000000002 2023-10-01T09:15:42.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.276774+0000", 2023-10-01T09:15:42.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000158252 2023-10-01T09:15:42.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:15:42.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.276943+0000", 2023-10-01T09:15:42.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000168901 2023-10-01T09:15:42.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.281454+0000", 2023-10-01T09:15:42.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045106199999999999 2023-10-01T09:15:42.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.481993+0000", 2023-10-01T09:15:42.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20053939500000001 2023-10-01T09:15:42.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.482071+0000", 2023-10-01T09:15:42.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7704e-05 2023-10-01T09:15:42.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.482096+0000", 2023-10-01T09:15:42.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5225e-05 2023-10-01T09:15:42.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.482129+0000", 2023-10-01T09:15:42.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2872e-05 2023-10-01T09:15:42.642 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.643 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.643 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.643 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:59036 5.6 5:670dbdc8:::benchmark_data_smithi138_240338_object59035:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:42.643 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:16.277072+0000", 2023-10-01T09:15:42.643 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 26.250989209, 2023-10-01T09:15:42.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20504208700000001, 2023-10-01T09:15:42.643 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.643 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.643 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.644 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.644 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.644 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 59036 2023-10-01T09:15:42.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.644 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.277072+0000", 2023-10-01T09:15:42.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.277072+0000", 2023-10-01T09:15:42.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.277071+0000", 2023-10-01T09:15:42.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:15:42.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.277097+0000", 2023-10-01T09:15:42.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6191999999999999e-05 2023-10-01T09:15:42.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.277109+0000", 2023-10-01T09:15:42.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1735e-05 2023-10-01T09:15:42.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.277116+0000", 2023-10-01T09:15:42.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3069999999999997e-06 2023-10-01T09:15:42.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.278744+0000", 2023-10-01T09:15:42.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001628242 2023-10-01T09:15:42.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.278868+0000", 2023-10-01T09:15:42.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012348900000000001 2023-10-01T09:15:42.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:15:42.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.279046+0000", 2023-10-01T09:15:42.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000178309 2023-10-01T09:15:42.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.481995+0000", 2023-10-01T09:15:42.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.202949139 2023-10-01T09:15:42.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.482011+0000", 2023-10-01T09:15:42.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6399999999999999e-05 2023-10-01T09:15:42.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.482056+0000", 2023-10-01T09:15:42.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4508000000000001e-05 2023-10-01T09:15:42.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.482095+0000", 2023-10-01T09:15:42.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8921000000000002e-05 2023-10-01T09:15:42.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.482114+0000", 2023-10-01T09:15:42.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9573e-05 2023-10-01T09:15:42.650 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.650 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.650 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.651 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:59041 5.a 5:58d3526c:::benchmark_data_smithi138_240338_object59040:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:15:42.651 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:16.278853+0000", 2023-10-01T09:15:42.651 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 26.249208842000002, 2023-10-01T09:15:42.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20340439199999999, 2023-10-01T09:15:42.651 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.651 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.651 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.651 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.651 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.651 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 59041 2023-10-01T09:15:42.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.652 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.278853+0000", 2023-10-01T09:15:42.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.278853+0000", 2023-10-01T09:15:42.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.278852+0000", 2023-10-01T09:15:42.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:15:42.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.278876+0000", 2023-10-01T09:15:42.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4012999999999999e-05 2023-10-01T09:15:42.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.278887+0000", 2023-10-01T09:15:42.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1790999999999999e-05 2023-10-01T09:15:42.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.278895+0000", 2023-10-01T09:15:42.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9710000000000004e-06 2023-10-01T09:15:42.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.279092+0000", 2023-10-01T09:15:42.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000196725 2023-10-01T09:15:42.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.279198+0000", 2023-10-01T09:15:42.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010624399999999999 2023-10-01T09:15:42.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:15:42.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.279322+0000", 2023-10-01T09:15:42.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012386100000000001 2023-10-01T09:15:42.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.482038+0000", 2023-10-01T09:15:42.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.202715964 2023-10-01T09:15:42.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.482049+0000", 2023-10-01T09:15:42.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0268000000000001e-05 2023-10-01T09:15:42.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.482167+0000", 2023-10-01T09:15:42.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118243 2023-10-01T09:15:42.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.482221+0000", 2023-10-01T09:15:42.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4249000000000002e-05 2023-10-01T09:15:42.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.482257+0000", 2023-10-01T09:15:42.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6084000000000002e-05 2023-10-01T09:15:42.658 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.658 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.658 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.658 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.34683.0:59028 5.a e115/106 ondisk, result = 0)", 2023-10-01T09:15:42.658 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:16.281428+0000", 2023-10-01T09:15:42.658 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 26.246633898999999, 2023-10-01T09:15:42.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20052588499999999, 2023-10-01T09:15:42.659 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.659 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:15:42.659 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.281428+0000", 2023-10-01T09:15:42.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.281428+0000", 2023-10-01T09:15:42.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.281427+0000", 2023-10-01T09:15:42.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:42.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.281432+0000", 2023-10-01T09:15:42.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1440000000000002e-06 2023-10-01T09:15:42.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.281442+0000", 2023-10-01T09:15:42.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9250000000000007e-06 2023-10-01T09:15:42.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.281449+0000", 2023-10-01T09:15:42.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6660000000000002e-06 2023-10-01T09:15:42.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.281471+0000", 2023-10-01T09:15:42.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2007000000000001e-05 2023-10-01T09:15:42.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.281477+0000", 2023-10-01T09:15:42.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2310000000000001e-06 2023-10-01T09:15:42.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:16.481954+0000", 2023-10-01T09:15:42.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20047647499999999 2023-10-01T09:15:42.663 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.663 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.663 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.663 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:86932 5.b 5:dcd1f6cb:::benchmark_data_smithi138_240338_object86931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-01T09:15:42.663 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:41.430980+0000", 2023-10-01T09:15:42.663 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.0970819439999999, 2023-10-01T09:15:42.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19973049200000001, 2023-10-01T09:15:42.664 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.664 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.664 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.664 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.664 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.664 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 86932 2023-10-01T09:15:42.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.664 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.430980+0000", 2023-10-01T09:15:42.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.430980+0000", 2023-10-01T09:15:42.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.430978+0000", 2023-10-01T09:15:42.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:15:42.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431014+0000", 2023-10-01T09:15:42.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5865000000000003e-05 2023-10-01T09:15:42.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431026+0000", 2023-10-01T09:15:42.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1807999999999999e-05 2023-10-01T09:15:42.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431034+0000", 2023-10-01T09:15:42.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5909999999999996e-06 2023-10-01T09:15:42.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431084+0000", 2023-10-01T09:15:42.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9688000000000003e-05 2023-10-01T09:15:42.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431193+0000", 2023-10-01T09:15:42.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010916099999999999 2023-10-01T09:15:42.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:15:42.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.620832+0000", 2023-10-01T09:15:42.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18963891799999999 2023-10-01T09:15:42.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.629388+0000", 2023-10-01T09:15:42.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085555330000000006 2023-10-01T09:15:42.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.630609+0000", 2023-10-01T09:15:42.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001221123 2023-10-01T09:15:42.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.630637+0000", 2023-10-01T09:15:42.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8563000000000001e-05 2023-10-01T09:15:42.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.630677+0000", 2023-10-01T09:15:42.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9798000000000001e-05 2023-10-01T09:15:42.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.630710+0000", 2023-10-01T09:15:42.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3079e-05 2023-10-01T09:15:42.670 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.670 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.670 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.671 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:86933 5.b 5:d1c21cc0:::benchmark_data_smithi138_240338_object86932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-01T09:15:42.671 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:41.431049+0000", 2023-10-01T09:15:42.671 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.097012589, 2023-10-01T09:15:42.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20266077900000001, 2023-10-01T09:15:42.671 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.671 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.671 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.671 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.671 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.672 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 86933 2023-10-01T09:15:42.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.672 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431049+0000", 2023-10-01T09:15:42.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431049+0000", 2023-10-01T09:15:42.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431048+0000", 2023-10-01T09:15:42.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:42.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431082+0000", 2023-10-01T09:15:42.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3269000000000003e-05 2023-10-01T09:15:42.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431090+0000", 2023-10-01T09:15:42.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0739999999999993e-06 2023-10-01T09:15:42.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431101+0000", 2023-10-01T09:15:42.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1596e-05 2023-10-01T09:15:42.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.623468+0000", 2023-10-01T09:15:42.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19236693599999999 2023-10-01T09:15:42.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.623585+0000", 2023-10-01T09:15:42.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000116542 2023-10-01T09:15:42.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:15:42.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.623705+0000", 2023-10-01T09:15:42.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120351 2023-10-01T09:15:42.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.629391+0000", 2023-10-01T09:15:42.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056857310000000003 2023-10-01T09:15:42.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.630741+0000", 2023-10-01T09:15:42.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001350026 2023-10-01T09:15:42.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.633613+0000", 2023-10-01T09:15:42.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002872351 2023-10-01T09:15:42.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.633666+0000", 2023-10-01T09:15:42.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2879e-05 2023-10-01T09:15:42.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.633710+0000", 2023-10-01T09:15:42.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3633000000000001e-05 2023-10-01T09:15:42.678 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.678 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.678 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.678 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:86934 5.8 5:11c1581a:::benchmark_data_smithi138_240338_object86933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e119)", 2023-10-01T09:15:42.678 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:15:41.431111+0000", 2023-10-01T09:15:42.678 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.0969504750000001, 2023-10-01T09:15:42.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19949729799999999, 2023-10-01T09:15:42.679 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:15:42.679 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:15:42.679 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:15:42.679 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:15:42.679 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:15:42.679 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 86934 2023-10-01T09:15:42.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.679 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:15:42.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:15:42.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431111+0000", 2023-10-01T09:15:42.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:15:42.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431111+0000", 2023-10-01T09:15:42.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:15:42.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:15:42.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431110+0000", 2023-10-01T09:15:42.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:15:42.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:15:42.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431130+0000", 2023-10-01T09:15:42.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9122000000000001e-05 2023-10-01T09:15:42.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:15:42.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431136+0000", 2023-10-01T09:15:42.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6279999999999999e-06 2023-10-01T09:15:42.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:15:42.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431141+0000", 2023-10-01T09:15:42.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9300000000000002e-06 2023-10-01T09:15:42.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:15:42.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.431193+0000", 2023-10-01T09:15:42.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1820999999999997e-05 2023-10-01T09:15:42.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:15:42.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.618935+0000", 2023-10-01T09:15:42.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18774175100000001 2023-10-01T09:15:42.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:15:42.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.619071+0000", 2023-10-01T09:15:42.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013601999999999999 2023-10-01T09:15:42.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:15:42.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.622220+0000", 2023-10-01T09:15:42.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031496110000000001 2023-10-01T09:15:42.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.625405+0000", 2023-10-01T09:15:42.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003184667 2023-10-01T09:15:42.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.687 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:15:42.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.630516+0000", 2023-10-01T09:15:42.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051108890000000004 2023-10-01T09:15:42.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.687 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:15:42.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.630573+0000", 2023-10-01T09:15:42.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6870999999999998e-05 2023-10-01T09:15:42.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:15:42.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:15:42.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:15:42.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:15:41.630608+0000", 2023-10-01T09:15:42.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5571999999999998e-05 2023-10-01T09:15:42.688 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.688 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.688 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.688 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:15:42.688 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:15:42.688 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:15:42.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:42 smithi190 ceph-mon[160435]: pgmap v270: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-01T09:15:42.689 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:42.744 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:42.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:42.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:42.853 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:42.931 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:43.286 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:15:43.530 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 59 16 88732 88716 93.9452 52.875 0.0091672 0.0106397 2023-10-01T09:15:43.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:43 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1602343212' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:15:43.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:43 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:15:43.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:43 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1602343212' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:15:43.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:43 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:15:43.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:43 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1602343212' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:15:43.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:43 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:15:44.070 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:44.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:44.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:44.176 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:44.234 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:44.334 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:44 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:15:44.334 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:44 smithi138 ceph-mon[192513]: failed to encode map e120 with expected crc 2023-10-01T09:15:44.334 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:44 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:15:44.334 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:44 smithi138 ceph-mon[192513]: osdmap e120: 8 total, 8 up, 8 in 2023-10-01T09:15:44.334 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:44 smithi138 ceph-mon[192513]: pgmap v272: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-01T09:15:44.335 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:44 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:15:44.335 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:44 smithi138 ceph-mon[185886]: failed to encode map e120 with expected crc 2023-10-01T09:15:44.335 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:44 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:15:44.335 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:44 smithi138 ceph-mon[185886]: osdmap e120: 8 total, 8 up, 8 in 2023-10-01T09:15:44.335 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:44 smithi138 ceph-mon[185886]: pgmap v272: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-01T09:15:44.503 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:15:44.530 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:15:44.530370+0000 min lat: 0.00109137 max lat: 0.284703 avg lat: 0.0106967 2023-10-01T09:15:44.530 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:15:44.530 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 60 16 89764 89748 93.4545 64.5 0.0174554 0.0106967 2023-10-01T09:15:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:44 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:15:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:44 smithi190 ceph-mon[160435]: failed to encode map e120 with expected crc 2023-10-01T09:15:44.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:44 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:15:44.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:44 smithi190 ceph-mon[160435]: osdmap e120: 8 total, 8 up, 8 in 2023-10-01T09:15:44.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:44 smithi190 ceph-mon[160435]: pgmap v272: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-01T09:15:45.052 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:45.060 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:15:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:15:44] "GET /metrics HTTP/1.1" 200 36324 "" "Prometheus/2.33.4" 2023-10-01T09:15:45.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:45.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:45.163 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:15:45.188 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:45.229 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:15:45.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:45.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:15:45.336 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:15:45.392 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:45.502 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:15:45.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:45.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:15:45.530 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 61 16 90573 90557 92.7514 50.5625 0.00664296 0.0107768 2023-10-01T09:15:45.612 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:15:45.656 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:45.686 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:45.844 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:15:45.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:45.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:15:45.948 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:15:45.992 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:46.102 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:15:46.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:46.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:15:46.210 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:15:46.267 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:46.338 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:46.370 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:15:46.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:46.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:15:46.485 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:15:46.530 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 62 16 91596 91580 92.2867 63.9375 0.0145588 0.0108309 2023-10-01T09:15:46.531 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:46 smithi190 ceph-mon[160435]: pgmap v273: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-01T09:15:46.536 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:46 smithi138 ceph-mon[192513]: pgmap v273: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-01T09:15:46.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:46 smithi138 ceph-mon[185886]: pgmap v273: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-01T09:15:46.580 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:15:46.619 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:15:46.645 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:15:46.650 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:15:46.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:46.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:15:46.673 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:15:46.715 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:15:46.751 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:15:46.770 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:15:46.787 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:15:46.826 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:15:46.829 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:15:46.830 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:15:46.830 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:15:46.830 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:15:46.830 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:15:46.830 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:15:46.830 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:15:46.830 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:15:46.831 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:47.302 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:47 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2565653208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:47.395 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:15:47.530 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 63 16 92676 92660 91.8932 67.5 0.0201255 0.0108772 2023-10-01T09:15:47.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:47 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2565653208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:47.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:47 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2565653208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:47.767 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:15:47.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:47.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:15:47.874 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:15:47.954 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:48.036 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:15:48.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:48.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:15:48.142 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:15:48.185 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:48.530 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 64 16 93532 93516 91.2934 53.5 0.0117585 0.0109485 2023-10-01T09:15:48.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:48 smithi190 ceph-mon[160435]: pgmap v274: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-01T09:15:48.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:48 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:15:48.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:48 smithi138 ceph-mon[192513]: pgmap v274: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-01T09:15:48.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:48 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:15:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:48 smithi138 ceph-mon[185886]: pgmap v274: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-10-01T09:15:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:48 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:15:48.925 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:15:48.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:48.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:15:49.029 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:15:49.125 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:49.193 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:15:49.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:49.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:15:49.299 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:15:49.328 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:49.530 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 65 16 94326 94310 90.6523 49.625 0.0145946 0.011026 2023-10-01T09:15:49.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:49.533 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:15:49.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:15:49.634 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:49.690 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:49.950 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:49.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:49.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:50.057 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:15:50.146 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:50.199 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:50.247 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:15:50.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:50.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:15:50.317 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:50 smithi138 ceph-mon[185886]: pgmap v275: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-01T09:15:50.356 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:15:50.405 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:50 smithi190 ceph-mon[160435]: pgmap v275: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-01T09:15:50.409 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:50.504 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:15:50.509 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:15:50.522 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:15:50.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:50.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:15:50.531 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 66 16 95245 95229 90.1491 57.4375 0.0237426 0.0110886 2023-10-01T09:15:50.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:50 smithi138 ceph-mon[192513]: pgmap v275: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-01T09:15:50.632 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:50.732 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:50.793 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:15:50.793 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:15:50.936 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:15:51.530 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 67 16 96119 96103 89.6189 54.625 0.0125311 0.0111536 2023-10-01T09:15:52.530 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 68 16 97119 97103 89.2201 62.5 0.0202327 0.0112028 2023-10-01T09:15:52.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:52 smithi138 ceph-mon[185886]: from='client.34743 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:15:52.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:52 smithi138 ceph-mon[185886]: pgmap v276: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-10-01T09:15:52.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:52 smithi138 ceph-mon[192513]: from='client.34743 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:15:52.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:52 smithi138 ceph-mon[192513]: pgmap v276: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-10-01T09:15:52.800 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:52.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:52 smithi190 ceph-mon[160435]: from='client.34743 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:15:52.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:52 smithi190 ceph-mon[160435]: pgmap v276: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-10-01T09:15:52.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:52.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:52.913 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:15:52.988 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:53.059 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:15:53.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:53.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:15:53.167 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:53.235 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:53.519 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:53 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:15:53.520 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:53 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:15:53.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:53 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:15:53.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:53 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:15:53.531 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 69 16 98080 98064 88.7976 60.0625 0.0206131 0.0112571 2023-10-01T09:15:53.546 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":275,"stamp":"2023-10-01T09:15:49.295590+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6060119987,"num_objects":92425,"num_object_clones":0,"num_object_copies":277275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":92425,"num_whiteouts":0,"num_read":6302,"num_read_kb":12700,"num_write":93503,"num_write_kb":5928549,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93567,"ondisk_log_size":93567,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19058224,"kb_used_data":17623920,"kb_used_omap":30,"kb_used_meta":1434209,"kb_avail":730902992,"statfs":{"total":767960285184,"available":748444663808,"internally_reserved":0,"allocated":18046894080,"data_stored":18042578688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30777,"internal_metadata":1468630983},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,2,4,5,0,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":635371520,"num_objects":9695,"num_object_clones":0,"num_object_copies":29085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9695,"num_write_kb":620480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002275"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":101,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.029257+0000","last_change":"2023-10-01T09:12:40.947346+0000","last_active":"2023-10-01T09:15:39.029257+0000","last_peered":"2023-10-01T09:15:39.029257+0000","last_clean":"2023-10-01T09:15:39.029257+0000","last_became_active":"2023-10-01T09:12:40.946939+0000","last_became_peered":"2023-10-01T09:12:40.946939+0000","last_unstale":"2023-10-01T09:15:39.029257+0000","last_undegraded":"2023-10-01T09:15:39.029257+0000","last_fullsized":"2023-10-01T09:15:39.029257+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":89,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:41.625662+0000","last_change":"2023-10-01T09:13:13.554780+0000","last_active":"2023-10-01T09:15:41.625662+0000","last_peered":"2023-10-01T09:15:41.625662+0000","last_clean":"2023-10-01T09:15:41.625662+0000","last_became_active":"2023-10-01T09:13:13.554394+0000","last_became_peered":"2023-10-01T09:13:13.554394+0000","last_unstale":"2023-10-01T09:15:41.625662+0000","last_undegraded":"2023-10-01T09:15:41.625662+0000","last_fullsized":"2023-10-01T09:15:41.625662+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:09:02.869321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":63,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.035104+0000","last_change":"2023-10-01T09:13:13.551596+0000","last_active":"2023-10-01T09:15:39.035104+0000","last_peered":"2023-10-01T09:15:39.035104+0000","last_clean":"2023-10-01T09:15:39.035104+0000","last_became_active":"2023-10-01T09:13:13.551300+0000","last_became_peered":"2023-10-01T09:13:13.551300+0000","last_unstale":"2023-10-01T09:15:39.035104+0000","last_undegraded":"2023-10-01T09:15:39.035104+0000","last_fullsized":"2023-10-01T09:15:39.035104+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:57:45.081780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":68,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.014349+0000","last_change":"2023-10-01T09:13:13.548695+0000","last_active":"2023-10-01T09:15:39.014349+0000","last_peered":"2023-10-01T09:15:39.014349+0000","last_clean":"2023-10-01T09:15:39.014349+0000","last_became_active":"2023-10-01T09:13:13.548256+0000","last_became_peered":"2023-10-01T09:13:13.548256+0000","last_unstale":"2023-10-01T09:15:39.014349+0000","last_undegraded":"2023-10-01T09:15:39.014349+0000","last_fullsized":"2023-10-01T09:15:39.014349+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:35:28.655733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":105,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.038998+0000","last_change":"2023-10-01T09:13:13.555490+0000","last_active":"2023-10-01T09:15:39.038998+0000","last_peered":"2023-10-01T09:15:39.038998+0000","last_clean":"2023-10-01T09:15:39.038998+0000","last_became_active":"2023-10-01T09:13:13.554506+0000","last_became_peered":"2023-10-01T09:13:13.554506+0000","last_unstale":"2023-10-01T09:15:39.038998+0000","last_undegraded":"2023-10-01T09:15:39.038998+0000","last_fullsized":"2023-10-01T09:15:39.038998+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:18.834474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":94,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.029728+0000","last_change":"2023-10-01T09:12:40.947591+0000","last_active":"2023-10-01T09:15:39.029728+0000","last_peered":"2023-10-01T09:15:39.029728+0000","last_clean":"2023-10-01T09:15:39.029728+0000","last_became_active":"2023-10-01T09:12:40.947242+0000","last_became_peered":"2023-10-01T09:12:40.947242+0000","last_unstale":"2023-10-01T09:15:39.029728+0000","last_undegraded":"2023-10-01T09:15:39.029728+0000","last_fullsized":"2023-10-01T09:15:39.029728+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":68,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.333234+0000","last_change":"2023-10-01T09:13:13.550193+0000","last_active":"2023-10-01T09:15:43.333234+0000","last_peered":"2023-10-01T09:15:43.333234+0000","last_clean":"2023-10-01T09:15:43.333234+0000","last_became_active":"2023-10-01T09:13:13.549900+0000","last_became_peered":"2023-10-01T09:13:13.549900+0000","last_unstale":"2023-10-01T09:15:43.333234+0000","last_undegraded":"2023-10-01T09:15:43.333234+0000","last_fullsized":"2023-10-01T09:15:43.333234+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:22.643161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":89,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.038941+0000","last_change":"2023-10-01T09:13:13.555368+0000","last_active":"2023-10-01T09:15:39.038941+0000","last_peered":"2023-10-01T09:15:39.038941+0000","last_clean":"2023-10-01T09:15:39.038941+0000","last_became_active":"2023-10-01T09:13:13.554806+0000","last_became_peered":"2023-10-01T09:13:13.554806+0000","last_unstale":"2023-10-01T09:15:39.038941+0000","last_undegraded":"2023-10-01T09:15:39.038941+0000","last_fullsized":"2023-10-01T09:15:39.038941+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:24:32.687833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":86,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.029855+0000","last_change":"2023-10-01T09:12:10.003091+0000","last_active":"2023-10-01T09:15:39.029855+0000","last_peered":"2023-10-01T09:15:39.029855+0000","last_clean":"2023-10-01T09:15:39.029855+0000","last_became_active":"2023-10-01T09:12:10.002773+0000","last_became_peered":"2023-10-01T09:12:10.002773+0000","last_unstale":"2023-10-01T09:15:39.029855+0000","last_undegraded":"2023-10-01T09:15:39.029855+0000","last_fullsized":"2023-10-01T09:15:39.029855+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.039549+0000","last_change":"2023-10-01T09:13:13.554942+0000","last_active":"2023-10-01T09:15:39.039549+0000","last_peered":"2023-10-01T09:15:39.039549+0000","last_clean":"2023-10-01T09:15:39.039549+0000","last_became_active":"2023-10-01T09:13:13.554664+0000","last_became_peered":"2023-10-01T09:13:13.554664+0000","last_unstale":"2023-10-01T09:15:39.039549+0000","last_undegraded":"2023-10-01T09:15:39.039549+0000","last_fullsized":"2023-10-01T09:15:39.039549+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:43:14.532709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":89,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.028247+0000","last_change":"2023-10-01T09:13:13.556801+0000","last_active":"2023-10-01T09:15:39.028247+0000","last_peered":"2023-10-01T09:15:39.028247+0000","last_clean":"2023-10-01T09:15:39.028247+0000","last_became_active":"2023-10-01T09:13:13.556397+0000","last_became_peered":"2023-10-01T09:13:13.556397+0000","last_unstale":"2023-10-01T09:15:39.028247+0000","last_undegraded":"2023-10-01T09:15:39.028247+0000","last_fullsized":"2023-10-01T09:15:39.028247+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":63,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.035662+0000","last_change":"2023-10-01T09:12:40.946520+0000","last_active":"2023-10-01T09:15:39.035662+0000","last_peered":"2023-10-01T09:15:39.035662+0000","last_clean":"2023-10-01T09:15:39.035662+0000","last_became_active":"2023-10-01T09:12:40.946227+0000","last_became_peered":"2023-10-01T09:12:40.946227+0000","last_unstale":"2023-10-01T09:15:39.035662+0000","last_undegraded":"2023-10-01T09:15:39.035662+0000","last_fullsized":"2023-10-01T09:15:39.035662+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:06:06.451923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":92,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.028893+0000","last_change":"2023-10-01T09:03:03.200261+0000","last_active":"2023-10-01T09:15:39.028893+0000","last_peered":"2023-10-01T09:15:39.028893+0000","last_clean":"2023-10-01T09:15:39.028893+0000","last_became_active":"2023-10-01T09:03:03.199944+0000","last_became_peered":"2023-10-01T09:03:03.199944+0000","last_unstale":"2023-10-01T09:15:39.028893+0000","last_undegraded":"2023-10-01T09:15:39.028893+0000","last_fullsized":"2023-10-01T09:15:39.028893+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":93,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.028795+0000","last_change":"2023-10-01T09:12:40.946973+0000","last_active":"2023-10-01T09:15:39.028795+0000","last_peered":"2023-10-01T09:15:39.028795+0000","last_clean":"2023-10-01T09:15:39.028795+0000","last_became_active":"2023-10-01T09:12:40.946427+0000","last_became_peered":"2023-10-01T09:12:40.946427+0000","last_unstale":"2023-10-01T09:15:39.028795+0000","last_undegraded":"2023-10-01T09:15:39.028795+0000","last_fullsized":"2023-10-01T09:15:39.028795+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":88,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.027984+0000","last_change":"2023-10-01T09:12:40.945277+0000","last_active":"2023-10-01T09:15:39.027984+0000","last_peered":"2023-10-01T09:15:39.027984+0000","last_clean":"2023-10-01T09:15:39.027984+0000","last_became_active":"2023-10-01T09:12:40.944843+0000","last_became_peered":"2023-10-01T09:12:40.944843+0000","last_unstale":"2023-10-01T09:15:39.027984+0000","last_undegraded":"2023-10-01T09:15:39.027984+0000","last_fullsized":"2023-10-01T09:15:39.027984+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":89,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.040699+0000","last_change":"2023-10-01T09:13:13.550633+0000","last_active":"2023-10-01T09:15:39.040699+0000","last_peered":"2023-10-01T09:15:39.040699+0000","last_clean":"2023-10-01T09:15:39.040699+0000","last_became_active":"2023-10-01T09:13:13.550305+0000","last_became_peered":"2023-10-01T09:13:13.550305+0000","last_unstale":"2023-10-01T09:15:39.040699+0000","last_undegraded":"2023-10-01T09:15:39.040699+0000","last_fullsized":"2023-10-01T09:15:39.040699+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:51:46.559383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":68,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.014292+0000","last_change":"2023-10-01T09:13:13.548094+0000","last_active":"2023-10-01T09:15:39.014292+0000","last_peered":"2023-10-01T09:15:39.014292+0000","last_clean":"2023-10-01T09:15:39.014292+0000","last_became_active":"2023-10-01T09:13:13.547782+0000","last_became_peered":"2023-10-01T09:13:13.547782+0000","last_unstale":"2023-10-01T09:15:39.014292+0000","last_undegraded":"2023-10-01T09:15:39.014292+0000","last_fullsized":"2023-10-01T09:15:39.014292+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:15.220827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":85,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.030080+0000","last_change":"2023-10-01T09:13:13.558367+0000","last_active":"2023-10-01T09:15:39.030080+0000","last_peered":"2023-10-01T09:15:39.030080+0000","last_clean":"2023-10-01T09:15:39.030080+0000","last_became_active":"2023-10-01T09:13:13.557952+0000","last_became_peered":"2023-10-01T09:13:13.557952+0000","last_unstale":"2023-10-01T09:15:39.030080+0000","last_undegraded":"2023-10-01T09:15:39.030080+0000","last_fullsized":"2023-10-01T09:15:39.030080+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":71,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.027713+0000","last_change":"2023-10-01T09:03:03.194952+0000","last_active":"2023-10-01T09:15:39.027713+0000","last_peered":"2023-10-01T09:15:39.027713+0000","last_clean":"2023-10-01T09:15:39.027713+0000","last_became_active":"2023-10-01T09:03:03.194643+0000","last_became_peered":"2023-10-01T09:03:03.194643+0000","last_unstale":"2023-10-01T09:15:39.027713+0000","last_undegraded":"2023-10-01T09:15:39.027713+0000","last_fullsized":"2023-10-01T09:15:39.027713+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":68,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.331018+0000","last_change":"2023-10-01T09:13:13.548531+0000","last_active":"2023-10-01T09:15:43.331018+0000","last_peered":"2023-10-01T09:15:43.331018+0000","last_clean":"2023-10-01T09:15:43.331018+0000","last_became_active":"2023-10-01T09:13:13.547881+0000","last_became_peered":"2023-10-01T09:13:13.547881+0000","last_unstale":"2023-10-01T09:15:43.331018+0000","last_undegraded":"2023-10-01T09:15:43.331018+0000","last_fullsized":"2023-10-01T09:15:43.331018+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:13:24.068359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":94,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.028097+0000","last_change":"2023-10-01T09:10:58.798168+0000","last_active":"2023-10-01T09:15:39.028097+0000","last_peered":"2023-10-01T09:15:39.028097+0000","last_clean":"2023-10-01T09:15:39.028097+0000","last_became_active":"2023-10-01T09:10:58.797781+0000","last_became_peered":"2023-10-01T09:10:58.797781+0000","last_unstale":"2023-10-01T09:15:39.028097+0000","last_undegraded":"2023-10-01T09:15:39.028097+0000","last_fullsized":"2023-10-01T09:15:39.028097+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":89,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.039305+0000","last_change":"2023-10-01T09:13:13.550588+0000","last_active":"2023-10-01T09:15:39.039305+0000","last_peered":"2023-10-01T09:15:39.039305+0000","last_clean":"2023-10-01T09:15:39.039305+0000","last_became_active":"2023-10-01T09:13:13.550271+0000","last_became_peered":"2023-10-01T09:13:13.550271+0000","last_unstale":"2023-10-01T09:15:39.039305+0000","last_undegraded":"2023-10-01T09:15:39.039305+0000","last_fullsized":"2023-10-01T09:15:39.039305+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:19:47.301376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"120'5817","reported_seq":5839,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:46.224466+0000","last_change":"2023-10-01T09:14:31.351571+0000","last_active":"2023-10-01T09:15:46.224466+0000","last_peered":"2023-10-01T09:15:46.224466+0000","last_clean":"2023-10-01T09:15:46.224466+0000","last_became_active":"2023-10-01T09:14:31.351242+0000","last_became_peered":"2023-10-01T09:14:31.351242+0000","last_unstale":"2023-10-01T09:15:46.224466+0000","last_undegraded":"2023-10-01T09:15:46.224466+0000","last_fullsized":"2023-10-01T09:15:46.224466+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5817,"log_dups_size":0,"ondisk_log_size":5817,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381222912,"num_objects":5817,"num_object_clones":0,"num_object_copies":17451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5817,"num_write_kb":372288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":87,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.027807+0000","last_change":"2023-10-01T09:10:58.797850+0000","last_active":"2023-10-01T09:15:39.027807+0000","last_peered":"2023-10-01T09:15:39.027807+0000","last_clean":"2023-10-01T09:15:39.027807+0000","last_became_active":"2023-10-01T09:10:58.797403+0000","last_became_peered":"2023-10-01T09:10:58.797403+0000","last_unstale":"2023-10-01T09:15:39.027807+0000","last_undegraded":"2023-10-01T09:15:39.027807+0000","last_fullsized":"2023-10-01T09:15:39.027807+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":70,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.029715+0000","last_change":"2023-10-01T09:03:04.245105+0000","last_active":"2023-10-01T09:15:39.029715+0000","last_peered":"2023-10-01T09:15:39.029715+0000","last_clean":"2023-10-01T09:15:39.029715+0000","last_became_active":"2023-10-01T09:03:04.244807+0000","last_became_peered":"2023-10-01T09:03:04.244807+0000","last_unstale":"2023-10-01T09:15:39.029715+0000","last_undegraded":"2023-10-01T09:15:39.029715+0000","last_fullsized":"2023-10-01T09:15:39.029715+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"120'6029","reported_seq":6052,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:48.890935+0000","last_change":"2023-10-01T09:14:31.344601+0000","last_active":"2023-10-01T09:15:48.890935+0000","last_peered":"2023-10-01T09:15:48.890935+0000","last_clean":"2023-10-01T09:15:48.890935+0000","last_became_active":"2023-10-01T09:14:31.344266+0000","last_became_peered":"2023-10-01T09:14:31.344266+0000","last_unstale":"2023-10-01T09:15:48.890935+0000","last_undegraded":"2023-10-01T09:15:48.890935+0000","last_fullsized":"2023-10-01T09:15:48.890935+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6029,"log_dups_size":0,"ondisk_log_size":6029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:31:20.164212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395116544,"num_objects":6029,"num_object_clones":0,"num_object_copies":18087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6029,"num_write_kb":385856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":88,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.028694+0000","last_change":"2023-10-01T09:12:10.003552+0000","last_active":"2023-10-01T09:15:39.028694+0000","last_peered":"2023-10-01T09:15:39.028694+0000","last_clean":"2023-10-01T09:15:39.028694+0000","last_became_active":"2023-10-01T09:12:10.003131+0000","last_became_peered":"2023-10-01T09:12:10.003131+0000","last_unstale":"2023-10-01T09:15:39.028694+0000","last_undegraded":"2023-10-01T09:15:39.028694+0000","last_fullsized":"2023-10-01T09:15:39.028694+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":99,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.029393+0000","last_change":"2023-10-01T09:13:13.556674+0000","last_active":"2023-10-01T09:15:39.029393+0000","last_peered":"2023-10-01T09:15:39.029393+0000","last_clean":"2023-10-01T09:15:39.029393+0000","last_became_active":"2023-10-01T09:13:13.556271+0000","last_became_peered":"2023-10-01T09:13:13.556271+0000","last_unstale":"2023-10-01T09:15:39.029393+0000","last_undegraded":"2023-10-01T09:15:39.029393+0000","last_fullsized":"2023-10-01T09:15:39.029393+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"120'5881","reported_seq":5904,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:48.971136+0000","last_change":"2023-10-01T09:14:31.357184+0000","last_active":"2023-10-01T09:15:48.971136+0000","last_peered":"2023-10-01T09:15:48.971136+0000","last_clean":"2023-10-01T09:15:48.971136+0000","last_became_active":"2023-10-01T09:14:31.356912+0000","last_became_peered":"2023-10-01T09:14:31.356912+0000","last_unstale":"2023-10-01T09:15:48.971136+0000","last_undegraded":"2023-10-01T09:15:48.971136+0000","last_fullsized":"2023-10-01T09:15:48.971136+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5881,"log_dups_size":0,"ondisk_log_size":5881,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:21:18.004826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385417216,"num_objects":5881,"num_object_clones":0,"num_object_copies":17643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5881,"num_write_kb":376384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":72,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.028424+0000","last_change":"2023-10-01T09:03:03.200972+0000","last_active":"2023-10-01T09:15:39.028424+0000","last_peered":"2023-10-01T09:15:39.028424+0000","last_clean":"2023-10-01T09:15:39.028424+0000","last_became_active":"2023-10-01T09:03:03.200695+0000","last_became_peered":"2023-10-01T09:03:03.200695+0000","last_unstale":"2023-10-01T09:15:39.028424+0000","last_undegraded":"2023-10-01T09:15:39.028424+0000","last_fullsized":"2023-10-01T09:15:39.028424+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":85,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.029969+0000","last_change":"2023-10-01T09:12:09.999571+0000","last_active":"2023-10-01T09:15:39.029969+0000","last_peered":"2023-10-01T09:15:39.029969+0000","last_clean":"2023-10-01T09:15:39.029969+0000","last_became_active":"2023-10-01T09:12:09.999272+0000","last_became_peered":"2023-10-01T09:12:09.999272+0000","last_unstale":"2023-10-01T09:15:39.029969+0000","last_undegraded":"2023-10-01T09:15:39.029969+0000","last_fullsized":"2023-10-01T09:15:39.029969+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"120'5787","reported_seq":5810,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:48.970799+0000","last_change":"2023-10-01T09:14:31.349127+0000","last_active":"2023-10-01T09:15:48.970799+0000","last_peered":"2023-10-01T09:15:48.970799+0000","last_clean":"2023-10-01T09:15:48.970799+0000","last_became_active":"2023-10-01T09:14:31.348832+0000","last_became_peered":"2023-10-01T09:14:31.348832+0000","last_unstale":"2023-10-01T09:15:48.970799+0000","last_undegraded":"2023-10-01T09:15:48.970799+0000","last_fullsized":"2023-10-01T09:15:48.970799+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5787,"log_dups_size":0,"ondisk_log_size":5787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:08:29.276824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379256832,"num_objects":5787,"num_object_clones":0,"num_object_copies":17361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5787,"num_write_kb":370368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":86,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.029956+0000","last_change":"2023-10-01T09:13:13.555458+0000","last_active":"2023-10-01T09:15:39.029956+0000","last_peered":"2023-10-01T09:15:39.029956+0000","last_clean":"2023-10-01T09:15:39.029956+0000","last_became_active":"2023-10-01T09:13:13.555058+0000","last_became_peered":"2023-10-01T09:13:13.555058+0000","last_unstale":"2023-10-01T09:15:39.029956+0000","last_undegraded":"2023-10-01T09:15:39.029956+0000","last_fullsized":"2023-10-01T09:15:39.029956+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":68,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.014609+0000","last_change":"2023-10-01T09:12:10.001565+0000","last_active":"2023-10-01T09:15:39.014609+0000","last_peered":"2023-10-01T09:15:39.014609+0000","last_clean":"2023-10-01T09:15:39.014609+0000","last_became_active":"2023-10-01T09:12:10.001275+0000","last_became_peered":"2023-10-01T09:12:10.001275+0000","last_unstale":"2023-10-01T09:15:39.014609+0000","last_undegraded":"2023-10-01T09:15:39.014609+0000","last_fullsized":"2023-10-01T09:15:39.014609+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:39.067046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"120'5846","reported_seq":5868,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:48.967484+0000","last_change":"2023-10-01T09:14:31.356715+0000","last_active":"2023-10-01T09:15:48.967484+0000","last_peered":"2023-10-01T09:15:48.967484+0000","last_clean":"2023-10-01T09:15:48.967484+0000","last_became_active":"2023-10-01T09:14:31.356417+0000","last_became_peered":"2023-10-01T09:14:31.356417+0000","last_unstale":"2023-10-01T09:15:48.967484+0000","last_undegraded":"2023-10-01T09:15:48.967484+0000","last_fullsized":"2023-10-01T09:15:48.967484+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5846,"log_dups_size":0,"ondisk_log_size":5846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:50:26.342593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383123456,"num_objects":5846,"num_object_clones":0,"num_object_copies":17538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5846,"num_write_kb":374144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":87,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.039215+0000","last_change":"2023-10-01T09:13:13.554321+0000","last_active":"2023-10-01T09:15:39.039215+0000","last_peered":"2023-10-01T09:15:39.039215+0000","last_clean":"2023-10-01T09:15:39.039215+0000","last_became_active":"2023-10-01T09:13:13.553986+0000","last_became_peered":"2023-10-01T09:13:13.553986+0000","last_unstale":"2023-10-01T09:15:39.039215+0000","last_undegraded":"2023-10-01T09:15:39.039215+0000","last_fullsized":"2023-10-01T09:15:39.039215+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:09:12.848269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":84,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.028968+0000","last_change":"2023-10-01T09:13:13.556491+0000","last_active":"2023-10-01T09:15:39.028968+0000","last_peered":"2023-10-01T09:15:39.028968+0000","last_clean":"2023-10-01T09:15:39.028968+0000","last_became_active":"2023-10-01T09:13:13.556118+0000","last_became_peered":"2023-10-01T09:13:13.556118+0000","last_unstale":"2023-10-01T09:15:39.028968+0000","last_undegraded":"2023-10-01T09:15:39.028968+0000","last_fullsized":"2023-10-01T09:15:39.028968+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"120'5821","reported_seq":5840,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:48.005130+0000","last_change":"2023-10-01T09:14:31.349314+0000","last_active":"2023-10-01T09:15:48.005130+0000","last_peered":"2023-10-01T09:15:48.005130+0000","last_clean":"2023-10-01T09:15:48.005130+0000","last_became_active":"2023-10-01T09:14:31.349041+0000","last_became_peered":"2023-10-01T09:14:31.349041+0000","last_unstale":"2023-10-01T09:15:48.005130+0000","last_undegraded":"2023-10-01T09:15:48.005130+0000","last_fullsized":"2023-10-01T09:15:48.005130+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5821,"log_dups_size":0,"ondisk_log_size":5821,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:20:18.137610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381485056,"num_objects":5821,"num_object_clones":0,"num_object_copies":17463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5821,"num_write_kb":372544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":86,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.030117+0000","last_change":"2023-10-01T09:12:10.003934+0000","last_active":"2023-10-01T09:15:39.030117+0000","last_peered":"2023-10-01T09:15:39.030117+0000","last_clean":"2023-10-01T09:15:39.030117+0000","last_became_active":"2023-10-01T09:12:10.003597+0000","last_became_peered":"2023-10-01T09:12:10.003597+0000","last_unstale":"2023-10-01T09:15:39.030117+0000","last_undegraded":"2023-10-01T09:15:39.030117+0000","last_fullsized":"2023-10-01T09:15:39.030117+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":99,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.029031+0000","last_change":"2023-10-01T09:12:40.946791+0000","last_active":"2023-10-01T09:15:39.029031+0000","last_peered":"2023-10-01T09:15:39.029031+0000","last_clean":"2023-10-01T09:15:39.029031+0000","last_became_active":"2023-10-01T09:12:40.946418+0000","last_became_peered":"2023-10-01T09:12:40.946418+0000","last_unstale":"2023-10-01T09:15:39.029031+0000","last_undegraded":"2023-10-01T09:15:39.029031+0000","last_fullsized":"2023-10-01T09:15:39.029031+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"120'5753","reported_seq":5775,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:46.224503+0000","last_change":"2023-10-01T09:14:31.355388+0000","last_active":"2023-10-01T09:15:46.224503+0000","last_peered":"2023-10-01T09:15:46.224503+0000","last_clean":"2023-10-01T09:15:46.224503+0000","last_became_active":"2023-10-01T09:14:31.355065+0000","last_became_peered":"2023-10-01T09:14:31.355065+0000","last_unstale":"2023-10-01T09:15:46.224503+0000","last_undegraded":"2023-10-01T09:15:46.224503+0000","last_fullsized":"2023-10-01T09:15:46.224503+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5753,"log_dups_size":0,"ondisk_log_size":5753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377028608,"num_objects":5753,"num_object_clones":0,"num_object_copies":17259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5753,"num_write_kb":368192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":85,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.029243+0000","last_change":"2023-10-01T09:12:40.945099+0000","last_active":"2023-10-01T09:15:39.029243+0000","last_peered":"2023-10-01T09:15:39.029243+0000","last_clean":"2023-10-01T09:15:39.029243+0000","last_became_active":"2023-10-01T09:12:40.944712+0000","last_became_peered":"2023-10-01T09:12:40.944712+0000","last_unstale":"2023-10-01T09:15:39.029243+0000","last_undegraded":"2023-10-01T09:15:39.029243+0000","last_fullsized":"2023-10-01T09:15:39.029243+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":84,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.028624+0000","last_change":"2023-10-01T09:12:10.003847+0000","last_active":"2023-10-01T09:15:39.028624+0000","last_peered":"2023-10-01T09:15:39.028624+0000","last_clean":"2023-10-01T09:15:39.028624+0000","last_became_active":"2023-10-01T09:12:10.003525+0000","last_became_peered":"2023-10-01T09:12:10.003525+0000","last_unstale":"2023-10-01T09:15:39.028624+0000","last_undegraded":"2023-10-01T09:15:39.028624+0000","last_fullsized":"2023-10-01T09:15:39.028624+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"120'5685","reported_seq":5707,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:46.196494+0000","last_change":"2023-10-01T09:14:31.356833+0000","last_active":"2023-10-01T09:15:46.196494+0000","last_peered":"2023-10-01T09:15:46.196494+0000","last_clean":"2023-10-01T09:15:46.196494+0000","last_became_active":"2023-10-01T09:14:31.356520+0000","last_became_peered":"2023-10-01T09:14:31.356520+0000","last_unstale":"2023-10-01T09:15:46.196494+0000","last_undegraded":"2023-10-01T09:15:46.196494+0000","last_fullsized":"2023-10-01T09:15:46.196494+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5685,"log_dups_size":0,"ondisk_log_size":5685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372572160,"num_objects":5685,"num_object_clones":0,"num_object_copies":17055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5685,"num_write_kb":363840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":90,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.039079+0000","last_change":"2023-10-01T09:13:13.555563+0000","last_active":"2023-10-01T09:15:39.039079+0000","last_peered":"2023-10-01T09:15:39.039079+0000","last_clean":"2023-10-01T09:15:39.039079+0000","last_became_active":"2023-10-01T09:13:13.555070+0000","last_became_peered":"2023-10-01T09:13:13.555070+0000","last_unstale":"2023-10-01T09:15:39.039079+0000","last_undegraded":"2023-10-01T09:15:39.039079+0000","last_fullsized":"2023-10-01T09:15:39.039079+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:44.557168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":59,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.010148+0000","last_change":"2023-10-01T09:13:13.551757+0000","last_active":"2023-10-01T09:15:39.010148+0000","last_peered":"2023-10-01T09:15:39.010148+0000","last_clean":"2023-10-01T09:15:39.010148+0000","last_became_active":"2023-10-01T09:13:13.551483+0000","last_became_peered":"2023-10-01T09:13:13.551483+0000","last_unstale":"2023-10-01T09:15:39.010148+0000","last_undegraded":"2023-10-01T09:15:39.010148+0000","last_fullsized":"2023-10-01T09:15:39.010148+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:33:46.184050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":933,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.036712+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:15:39.036712+0000","last_peered":"2023-10-01T09:15:39.036712+0000","last_clean":"2023-10-01T09:15:39.036712+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:15:39.036712+0000","last_undegraded":"2023-10-01T09:15:39.036712+0000","last_fullsized":"2023-10-01T09:15:39.036712+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":101,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.030158+0000","last_change":"2023-10-01T09:13:13.551681+0000","last_active":"2023-10-01T09:15:39.030158+0000","last_peered":"2023-10-01T09:15:39.030158+0000","last_clean":"2023-10-01T09:15:39.030158+0000","last_became_active":"2023-10-01T09:13:13.551268+0000","last_became_peered":"2023-10-01T09:13:13.551268+0000","last_unstale":"2023-10-01T09:15:39.030158+0000","last_undegraded":"2023-10-01T09:15:39.030158+0000","last_fullsized":"2023-10-01T09:15:39.030158+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":48,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.035600+0000","last_change":"2023-10-01T09:12:10.001493+0000","last_active":"2023-10-01T09:15:39.035600+0000","last_peered":"2023-10-01T09:15:39.035600+0000","last_clean":"2023-10-01T09:15:39.035600+0000","last_became_active":"2023-10-01T09:12:10.000899+0000","last_became_peered":"2023-10-01T09:12:10.000899+0000","last_unstale":"2023-10-01T09:15:39.035600+0000","last_undegraded":"2023-10-01T09:15:39.035600+0000","last_fullsized":"2023-10-01T09:15:39.035600+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:47:54.011059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"120'5782","reported_seq":5805,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:46.908511+0000","last_change":"2023-10-01T09:14:31.357395+0000","last_active":"2023-10-01T09:15:46.908511+0000","last_peered":"2023-10-01T09:15:46.908511+0000","last_clean":"2023-10-01T09:15:46.908511+0000","last_became_active":"2023-10-01T09:14:31.357014+0000","last_became_peered":"2023-10-01T09:14:31.357014+0000","last_unstale":"2023-10-01T09:15:46.908511+0000","last_undegraded":"2023-10-01T09:15:46.908511+0000","last_fullsized":"2023-10-01T09:15:46.908511+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5782,"log_dups_size":0,"ondisk_log_size":5782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:29.461431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378929152,"num_objects":5782,"num_object_clones":0,"num_object_copies":17346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5782,"num_write_kb":370048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"120'5633","reported_seq":5656,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:46.862180+0000","last_change":"2023-10-01T09:14:31.349232+0000","last_active":"2023-10-01T09:15:46.862180+0000","last_peered":"2023-10-01T09:15:46.862180+0000","last_clean":"2023-10-01T09:15:46.862180+0000","last_became_active":"2023-10-01T09:14:31.348940+0000","last_became_peered":"2023-10-01T09:14:31.348940+0000","last_unstale":"2023-10-01T09:15:46.862180+0000","last_undegraded":"2023-10-01T09:15:46.862180+0000","last_fullsized":"2023-10-01T09:15:46.862180+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5633,"log_dups_size":0,"ondisk_log_size":5633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:17.358604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369164288,"num_objects":5633,"num_object_clones":0,"num_object_copies":16899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5633,"num_write_kb":360512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":820,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.030271+0000","last_change":"2023-10-01T09:12:09.997936+0000","last_active":"2023-10-01T09:15:39.030271+0000","last_peered":"2023-10-01T09:15:39.030271+0000","last_clean":"2023-10-01T09:15:39.030271+0000","last_became_active":"2023-10-01T09:12:09.997427+0000","last_became_peered":"2023-10-01T09:12:09.997427+0000","last_unstale":"2023-10-01T09:15:39.030271+0000","last_undegraded":"2023-10-01T09:15:39.030271+0000","last_fullsized":"2023-10-01T09:15:39.030271+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1219,"num_read_kb":1114,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":85,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.028657+0000","last_change":"2023-10-01T09:10:58.799722+0000","last_active":"2023-10-01T09:15:39.028657+0000","last_peered":"2023-10-01T09:15:39.028657+0000","last_clean":"2023-10-01T09:15:39.028657+0000","last_became_active":"2023-10-01T09:10:58.799313+0000","last_became_peered":"2023-10-01T09:10:58.799313+0000","last_unstale":"2023-10-01T09:15:39.028657+0000","last_undegraded":"2023-10-01T09:15:39.028657+0000","last_fullsized":"2023-10-01T09:15:39.028657+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":78,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.036667+0000","last_change":"2023-10-01T09:13:13.549385+0000","last_active":"2023-10-01T09:15:39.036667+0000","last_peered":"2023-10-01T09:15:39.036667+0000","last_clean":"2023-10-01T09:15:39.036667+0000","last_became_active":"2023-10-01T09:13:13.549035+0000","last_became_peered":"2023-10-01T09:13:13.549035+0000","last_unstale":"2023-10-01T09:15:39.036667+0000","last_undegraded":"2023-10-01T09:15:39.036667+0000","last_fullsized":"2023-10-01T09:15:39.036667+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:52:41.408995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"120'5898","reported_seq":5921,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:48.936224+0000","last_change":"2023-10-01T09:14:31.357130+0000","last_active":"2023-10-01T09:15:48.936224+0000","last_peered":"2023-10-01T09:15:48.936224+0000","last_clean":"2023-10-01T09:15:48.936224+0000","last_became_active":"2023-10-01T09:14:31.356877+0000","last_became_peered":"2023-10-01T09:14:31.356877+0000","last_unstale":"2023-10-01T09:15:48.936224+0000","last_undegraded":"2023-10-01T09:15:48.936224+0000","last_fullsized":"2023-10-01T09:15:48.936224+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5898,"log_dups_size":0,"ondisk_log_size":5898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:32:02.594887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386531328,"num_objects":5898,"num_object_clones":0,"num_object_copies":17694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5898,"num_write_kb":377472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":391,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.009906+0000","last_change":"2023-10-01T09:13:13.551669+0000","last_active":"2023-10-01T09:15:39.009906+0000","last_peered":"2023-10-01T09:15:39.009906+0000","last_clean":"2023-10-01T09:15:39.009906+0000","last_became_active":"2023-10-01T09:13:13.550982+0000","last_became_peered":"2023-10-01T09:13:13.550982+0000","last_unstale":"2023-10-01T09:15:39.009906+0000","last_undegraded":"2023-10-01T09:15:39.009906+0000","last_fullsized":"2023-10-01T09:15:39.009906+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:21:09.879713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":63,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.035369+0000","last_change":"2023-10-01T09:12:40.946267+0000","last_active":"2023-10-01T09:15:39.035369+0000","last_peered":"2023-10-01T09:15:39.035369+0000","last_clean":"2023-10-01T09:15:39.035369+0000","last_became_active":"2023-10-01T09:12:40.945970+0000","last_became_peered":"2023-10-01T09:12:40.945970+0000","last_unstale":"2023-10-01T09:15:39.035369+0000","last_undegraded":"2023-10-01T09:15:39.035369+0000","last_fullsized":"2023-10-01T09:15:39.035369+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:42.904509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":93,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.035948+0000","last_change":"2023-10-01T09:12:09.994071+0000","last_active":"2023-10-01T09:15:39.035948+0000","last_peered":"2023-10-01T09:15:39.035948+0000","last_clean":"2023-10-01T09:15:39.035948+0000","last_became_active":"2023-10-01T09:12:09.993782+0000","last_became_peered":"2023-10-01T09:12:09.993782+0000","last_unstale":"2023-10-01T09:15:39.035948+0000","last_undegraded":"2023-10-01T09:15:39.035948+0000","last_fullsized":"2023-10-01T09:15:39.035948+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:12:49.880685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":936,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.032172+0000","last_change":"2023-10-01T09:12:40.946131+0000","last_active":"2023-10-01T09:15:39.032172+0000","last_peered":"2023-10-01T09:15:39.032172+0000","last_clean":"2023-10-01T09:15:39.032172+0000","last_became_active":"2023-10-01T09:12:40.945524+0000","last_became_peered":"2023-10-01T09:12:40.945524+0000","last_unstale":"2023-10-01T09:15:39.032172+0000","last_undegraded":"2023-10-01T09:15:39.032172+0000","last_fullsized":"2023-10-01T09:15:39.032172+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":322,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.027447+0000","last_change":"2023-10-01T09:10:58.798697+0000","last_active":"2023-10-01T09:15:39.027447+0000","last_peered":"2023-10-01T09:15:39.027447+0000","last_clean":"2023-10-01T09:15:39.027447+0000","last_became_active":"2023-10-01T09:10:58.798203+0000","last_became_peered":"2023-10-01T09:10:58.798203+0000","last_unstale":"2023-10-01T09:15:39.027447+0000","last_undegraded":"2023-10-01T09:15:39.027447+0000","last_fullsized":"2023-10-01T09:15:39.027447+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":79,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.039837+0000","last_change":"2023-10-01T09:13:13.556049+0000","last_active":"2023-10-01T09:15:39.039837+0000","last_peered":"2023-10-01T09:15:39.039837+0000","last_clean":"2023-10-01T09:15:39.039837+0000","last_became_active":"2023-10-01T09:13:13.555739+0000","last_became_peered":"2023-10-01T09:13:13.555739+0000","last_unstale":"2023-10-01T09:15:39.039837+0000","last_undegraded":"2023-10-01T09:15:39.039837+0000","last_fullsized":"2023-10-01T09:15:39.039837+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:50:34.475690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":70,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.029297+0000","last_change":"2023-10-01T09:03:03.227254+0000","last_active":"2023-10-01T09:15:39.029297+0000","last_peered":"2023-10-01T09:15:39.029297+0000","last_clean":"2023-10-01T09:15:39.029297+0000","last_became_active":"2023-10-01T09:03:03.226941+0000","last_became_peered":"2023-10-01T09:03:03.226941+0000","last_unstale":"2023-10-01T09:15:39.029297+0000","last_undegraded":"2023-10-01T09:15:39.029297+0000","last_fullsized":"2023-10-01T09:15:39.029297+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"120'5622","reported_seq":5644,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:46.192273+0000","last_change":"2023-10-01T09:14:31.353858+0000","last_active":"2023-10-01T09:15:46.192273+0000","last_peered":"2023-10-01T09:15:46.192273+0000","last_clean":"2023-10-01T09:15:46.192273+0000","last_became_active":"2023-10-01T09:14:31.353572+0000","last_became_peered":"2023-10-01T09:14:31.353572+0000","last_unstale":"2023-10-01T09:15:46.192273+0000","last_undegraded":"2023-10-01T09:15:46.192273+0000","last_fullsized":"2023-10-01T09:15:46.192273+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5622,"log_dups_size":0,"ondisk_log_size":5622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368443392,"num_objects":5622,"num_object_clones":0,"num_object_copies":16866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5622,"num_write_kb":359808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"120'5734","reported_seq":5755,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:46.230532+0000","last_change":"2023-10-01T09:14:31.354366+0000","last_active":"2023-10-01T09:15:46.230532+0000","last_peered":"2023-10-01T09:15:46.230532+0000","last_clean":"2023-10-01T09:15:46.230532+0000","last_became_active":"2023-10-01T09:14:31.354059+0000","last_became_peered":"2023-10-01T09:14:31.354059+0000","last_unstale":"2023-10-01T09:15:46.230532+0000","last_undegraded":"2023-10-01T09:15:46.230532+0000","last_fullsized":"2023-10-01T09:15:46.230532+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5734,"log_dups_size":0,"ondisk_log_size":5734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375783424,"num_objects":5734,"num_object_clones":0,"num_object_copies":17202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5734,"num_write_kb":366976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":216,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.027536+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:15:39.027536+0000","last_peered":"2023-10-01T09:15:39.027536+0000","last_clean":"2023-10-01T09:15:39.027536+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:15:39.027536+0000","last_undegraded":"2023-10-01T09:15:39.027536+0000","last_fullsized":"2023-10-01T09:15:39.027536+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":68,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.014050+0000","last_change":"2023-10-01T09:13:13.552391+0000","last_active":"2023-10-01T09:15:39.014050+0000","last_peered":"2023-10-01T09:15:39.014050+0000","last_clean":"2023-10-01T09:15:39.014050+0000","last_became_active":"2023-10-01T09:13:13.552082+0000","last_became_peered":"2023-10-01T09:13:13.552082+0000","last_unstale":"2023-10-01T09:15:39.014050+0000","last_undegraded":"2023-10-01T09:15:39.014050+0000","last_fullsized":"2023-10-01T09:15:39.014050+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:07:19.929732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":83,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.014454+0000","last_change":"2023-10-01T09:12:40.944991+0000","last_active":"2023-10-01T09:15:39.014454+0000","last_peered":"2023-10-01T09:15:39.014454+0000","last_clean":"2023-10-01T09:15:39.014454+0000","last_became_active":"2023-10-01T09:12:40.944688+0000","last_became_peered":"2023-10-01T09:12:40.944688+0000","last_unstale":"2023-10-01T09:15:39.014454+0000","last_undegraded":"2023-10-01T09:15:39.014454+0000","last_fullsized":"2023-10-01T09:15:39.014454+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:30.015801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":480,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.028331+0000","last_change":"2023-10-01T09:10:58.797964+0000","last_active":"2023-10-01T09:15:39.028331+0000","last_peered":"2023-10-01T09:15:39.028331+0000","last_clean":"2023-10-01T09:15:39.028331+0000","last_became_active":"2023-10-01T09:10:58.797447+0000","last_became_peered":"2023-10-01T09:10:58.797447+0000","last_unstale":"2023-10-01T09:15:39.028331+0000","last_undegraded":"2023-10-01T09:15:39.028331+0000","last_fullsized":"2023-10-01T09:15:39.028331+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":63,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.036757+0000","last_change":"2023-10-01T09:12:40.948216+0000","last_active":"2023-10-01T09:15:39.036757+0000","last_peered":"2023-10-01T09:15:39.036757+0000","last_clean":"2023-10-01T09:15:39.036757+0000","last_became_active":"2023-10-01T09:12:40.947879+0000","last_became_peered":"2023-10-01T09:12:40.947879+0000","last_unstale":"2023-10-01T09:15:39.036757+0000","last_undegraded":"2023-10-01T09:15:39.036757+0000","last_fullsized":"2023-10-01T09:15:39.036757+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:46:40.612337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":53,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.014773+0000","last_change":"2023-10-01T09:10:58.797034+0000","last_active":"2023-10-01T09:15:39.014773+0000","last_peered":"2023-10-01T09:15:39.014773+0000","last_clean":"2023-10-01T09:15:39.014773+0000","last_became_active":"2023-10-01T09:10:58.796730+0000","last_became_peered":"2023-10-01T09:10:58.796730+0000","last_unstale":"2023-10-01T09:15:39.014773+0000","last_undegraded":"2023-10-01T09:15:39.014773+0000","last_fullsized":"2023-10-01T09:15:39.014773+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:32:15.982890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"120'5693","reported_seq":5713,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:46.175376+0000","last_change":"2023-10-01T09:14:31.353560+0000","last_active":"2023-10-01T09:15:46.175376+0000","last_peered":"2023-10-01T09:15:46.175376+0000","last_clean":"2023-10-01T09:15:46.175376+0000","last_became_active":"2023-10-01T09:14:31.353242+0000","last_became_peered":"2023-10-01T09:14:31.353242+0000","last_unstale":"2023-10-01T09:15:46.175376+0000","last_undegraded":"2023-10-01T09:15:46.175376+0000","last_fullsized":"2023-10-01T09:15:46.175376+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5693,"log_dups_size":0,"ondisk_log_size":5693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373096448,"num_objects":5693,"num_object_clones":0,"num_object_copies":17079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5693,"num_write_kb":364352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":87,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.039403+0000","last_change":"2023-10-01T09:13:13.553580+0000","last_active":"2023-10-01T09:15:39.039403+0000","last_peered":"2023-10-01T09:15:39.039403+0000","last_clean":"2023-10-01T09:15:39.039403+0000","last_became_active":"2023-10-01T09:13:13.553011+0000","last_became_peered":"2023-10-01T09:13:13.553011+0000","last_unstale":"2023-10-01T09:15:39.039403+0000","last_undegraded":"2023-10-01T09:15:39.039403+0000","last_fullsized":"2023-10-01T09:15:39.039403+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:54:30.369308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":507,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.040938+0000","last_change":"2023-10-01T09:12:10.002334+0000","last_active":"2023-10-01T09:15:39.040938+0000","last_peered":"2023-10-01T09:15:39.040938+0000","last_clean":"2023-10-01T09:15:39.040938+0000","last_became_active":"2023-10-01T09:12:10.001791+0000","last_became_peered":"2023-10-01T09:12:10.001791+0000","last_unstale":"2023-10-01T09:15:39.040938+0000","last_undegraded":"2023-10-01T09:15:39.040938+0000","last_fullsized":"2023-10-01T09:15:39.040938+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:24:27.632780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":84,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.039014+0000","last_change":"2023-10-01T09:13:13.552192+0000","last_active":"2023-10-01T09:15:39.039014+0000","last_peered":"2023-10-01T09:15:39.039014+0000","last_clean":"2023-10-01T09:15:39.039014+0000","last_became_active":"2023-10-01T09:13:13.551887+0000","last_became_peered":"2023-10-01T09:13:13.551887+0000","last_unstale":"2023-10-01T09:15:39.039014+0000","last_undegraded":"2023-10-01T09:15:39.039014+0000","last_fullsized":"2023-10-01T09:15:39.039014+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:23.633557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"120'5782","reported_seq":5803,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:46.259317+0000","last_change":"2023-10-01T09:14:31.350884+0000","last_active":"2023-10-01T09:15:46.259317+0000","last_peered":"2023-10-01T09:15:46.259317+0000","last_clean":"2023-10-01T09:15:46.259317+0000","last_became_active":"2023-10-01T09:14:31.350533+0000","last_became_peered":"2023-10-01T09:14:31.350533+0000","last_unstale":"2023-10-01T09:15:46.259317+0000","last_undegraded":"2023-10-01T09:15:46.259317+0000","last_fullsized":"2023-10-01T09:15:46.259317+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5782,"log_dups_size":0,"ondisk_log_size":5782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378929152,"num_objects":5782,"num_object_clones":0,"num_object_copies":17346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5782,"num_write_kb":370048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":158,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.029650+0000","last_change":"2023-10-01T09:12:40.949544+0000","last_active":"2023-10-01T09:15:39.029650+0000","last_peered":"2023-10-01T09:15:39.029650+0000","last_clean":"2023-10-01T09:15:39.029650+0000","last_became_active":"2023-10-01T09:12:40.949054+0000","last_became_peered":"2023-10-01T09:12:40.949054+0000","last_unstale":"2023-10-01T09:15:39.029650+0000","last_undegraded":"2023-10-01T09:15:39.029650+0000","last_fullsized":"2023-10-01T09:15:39.029650+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":72,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.014446+0000","last_change":"2023-10-01T09:12:10.003174+0000","last_active":"2023-10-01T09:15:39.014446+0000","last_peered":"2023-10-01T09:15:39.014446+0000","last_clean":"2023-10-01T09:15:39.014446+0000","last_became_active":"2023-10-01T09:12:10.002808+0000","last_became_peered":"2023-10-01T09:12:10.002808+0000","last_unstale":"2023-10-01T09:15:39.014446+0000","last_undegraded":"2023-10-01T09:15:39.014446+0000","last_fullsized":"2023-10-01T09:15:39.014446+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:25.037091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":69,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.029288+0000","last_change":"2023-10-01T09:03:03.754839+0000","last_active":"2023-10-01T09:15:39.029288+0000","last_peered":"2023-10-01T09:15:39.029288+0000","last_clean":"2023-10-01T09:15:39.029288+0000","last_became_active":"2023-10-01T09:03:03.754506+0000","last_became_peered":"2023-10-01T09:03:03.754506+0000","last_unstale":"2023-10-01T09:15:39.029288+0000","last_undegraded":"2023-10-01T09:15:39.029288+0000","last_fullsized":"2023-10-01T09:15:39.029288+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"120'5634","reported_seq":5657,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:46.615012+0000","last_change":"2023-10-01T09:14:31.352926+0000","last_active":"2023-10-01T09:15:46.615012+0000","last_peered":"2023-10-01T09:15:46.615012+0000","last_clean":"2023-10-01T09:15:46.615012+0000","last_became_active":"2023-10-01T09:14:31.352602+0000","last_became_peered":"2023-10-01T09:14:31.352602+0000","last_unstale":"2023-10-01T09:15:46.615012+0000","last_undegraded":"2023-10-01T09:15:46.615012+0000","last_fullsized":"2023-10-01T09:15:46.615012+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":5634,"log_dups_size":0,"ondisk_log_size":5634,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:40:47.614933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369229824,"num_objects":5634,"num_object_clones":0,"num_object_copies":16902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5634,"num_write_kb":360576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":56,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.014732+0000","last_change":"2023-10-01T09:10:58.801161+0000","last_active":"2023-10-01T09:15:39.014732+0000","last_peered":"2023-10-01T09:15:39.014732+0000","last_clean":"2023-10-01T09:15:39.014732+0000","last_became_active":"2023-10-01T09:10:58.800794+0000","last_became_peered":"2023-10-01T09:10:58.800794+0000","last_unstale":"2023-10-01T09:15:39.014732+0000","last_undegraded":"2023-10-01T09:15:39.014732+0000","last_fullsized":"2023-10-01T09:15:39.014732+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:35.019445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.012058+0000","last_change":"2023-10-01T09:13:13.548647+0000","last_active":"2023-10-01T09:15:39.012058+0000","last_peered":"2023-10-01T09:15:39.012058+0000","last_clean":"2023-10-01T09:15:39.012058+0000","last_became_active":"2023-10-01T09:13:13.548382+0000","last_became_peered":"2023-10-01T09:13:13.548382+0000","last_unstale":"2023-10-01T09:15:39.012058+0000","last_undegraded":"2023-10-01T09:15:39.012058+0000","last_fullsized":"2023-10-01T09:15:39.012058+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:17:31.791258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":88,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.030000+0000","last_change":"2023-10-01T09:12:10.000449+0000","last_active":"2023-10-01T09:15:39.030000+0000","last_peered":"2023-10-01T09:15:39.030000+0000","last_clean":"2023-10-01T09:15:39.030000+0000","last_became_active":"2023-10-01T09:12:10.000000+0000","last_became_peered":"2023-10-01T09:12:10.000000+0000","last_unstale":"2023-10-01T09:15:39.030000+0000","last_undegraded":"2023-10-01T09:15:39.030000+0000","last_fullsized":"2023-10-01T09:15:39.030000+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":99,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.028946+0000","last_change":"2023-10-01T09:13:13.555744+0000","last_active":"2023-10-01T09:15:39.028946+0000","last_peered":"2023-10-01T09:15:39.028946+0000","last_clean":"2023-10-01T09:15:39.028946+0000","last_became_active":"2023-10-01T09:13:13.555163+0000","last_became_peered":"2023-10-01T09:13:13.555163+0000","last_unstale":"2023-10-01T09:15:39.028946+0000","last_undegraded":"2023-10-01T09:15:39.028946+0000","last_fullsized":"2023-10-01T09:15:39.028946+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":73,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.030214+0000","last_change":"2023-10-01T09:03:03.202063+0000","last_active":"2023-10-01T09:15:39.030214+0000","last_peered":"2023-10-01T09:15:39.030214+0000","last_clean":"2023-10-01T09:15:39.030214+0000","last_became_active":"2023-10-01T09:03:03.201769+0000","last_became_peered":"2023-10-01T09:03:03.201769+0000","last_unstale":"2023-10-01T09:15:39.030214+0000","last_undegraded":"2023-10-01T09:15:39.030214+0000","last_fullsized":"2023-10-01T09:15:39.030214+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":99,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.029333+0000","last_change":"2023-10-01T09:13:13.554959+0000","last_active":"2023-10-01T09:15:39.029333+0000","last_peered":"2023-10-01T09:15:39.029333+0000","last_clean":"2023-10-01T09:15:39.029333+0000","last_became_active":"2023-10-01T09:13:13.554589+0000","last_became_peered":"2023-10-01T09:13:13.554589+0000","last_unstale":"2023-10-01T09:15:39.029333+0000","last_undegraded":"2023-10-01T09:15:39.029333+0000","last_fullsized":"2023-10-01T09:15:39.029333+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":95,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.039221+0000","last_change":"2023-10-01T09:13:13.556295+0000","last_active":"2023-10-01T09:15:39.039221+0000","last_peered":"2023-10-01T09:15:39.039221+0000","last_clean":"2023-10-01T09:15:39.039221+0000","last_became_active":"2023-10-01T09:13:13.555953+0000","last_became_peered":"2023-10-01T09:13:13.555953+0000","last_unstale":"2023-10-01T09:15:39.039221+0000","last_undegraded":"2023-10-01T09:15:39.039221+0000","last_fullsized":"2023-10-01T09:15:39.039221+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:04:06.892156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":108,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.039424+0000","last_change":"2023-10-01T09:13:13.549800+0000","last_active":"2023-10-01T09:15:39.039424+0000","last_peered":"2023-10-01T09:15:39.039424+0000","last_clean":"2023-10-01T09:15:39.039424+0000","last_became_active":"2023-10-01T09:13:13.549480+0000","last_became_peered":"2023-10-01T09:13:13.549480+0000","last_unstale":"2023-10-01T09:15:39.039424+0000","last_undegraded":"2023-10-01T09:15:39.039424+0000","last_fullsized":"2023-10-01T09:15:39.039424+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:08.673916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":72,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.014381+0000","last_change":"2023-10-01T09:12:40.944944+0000","last_active":"2023-10-01T09:15:39.014381+0000","last_peered":"2023-10-01T09:15:39.014381+0000","last_clean":"2023-10-01T09:15:39.014381+0000","last_became_active":"2023-10-01T09:12:40.944613+0000","last_became_peered":"2023-10-01T09:12:40.944613+0000","last_unstale":"2023-10-01T09:15:39.014381+0000","last_undegraded":"2023-10-01T09:15:39.014381+0000","last_fullsized":"2023-10-01T09:15:39.014381+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:48:20.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":84,"reported_epoch":119,"state":"active+clean","last_fresh":"2023-10-01T09:15:39.028505+0000","last_change":"2023-10-01T09:12:10.003833+0000","last_active":"2023-10-01T09:15:39.028505+0000","last_peered":"2023-10-01T09:15:39.028505+0000","last_clean":"2023-10-01T09:15:39.028505+0000","last_became_active":"2023-10-01T09:12:10.003482+0000","last_became_peered":"2023-10-01T09:12:10.003482+0000","last_unstale":"2023-10-01T09:15:39.028505+0000","last_undegraded":"2023-10-01T09:15:39.028505+0000","last_fullsized":"2023-10-01T09:15:39.028505+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":6055329792,"num_objects":92397,"num_object_clones":0,"num_object_copies":277191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":92397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":92397,"num_write_kb":5913408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18081579008,"data_stored":18081579008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92397,"ondisk_log_size":92397,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27349,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12188,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430382,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2120244,"kb_used_data":1823020,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91624908,"statfs":{"total":95995035648,"available":93823905792,"internally_reserved":0,"allocated":1866772480,"data_stored":1866256465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:03 2023","interfaces":[{"interface":"back","average":{"1min":5.5339999999999998,"5min":1.556,"15min":0.89900000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":94.162999999999997,"5min":94.162999999999997,"15min":94.162999999999997},"last":0.51500000000000001},{"interface":"front","average":{"1min":5.5300000000000002,"5min":1.542,"15min":0.879},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":94.135000000000005,"5min":94.135000000000005,"15min":94.135000000000005},"last":0.56799999999999995}]},{"osd":1,"last update":"Sun Oct 1 09:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3080000000000001,"5min":0.71399999999999997,"15min":0.61399999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.868,"5min":12.868,"15min":12.868},"last":0.5},{"interface":"front","average":{"1min":1.4059999999999999,"5min":0.71999999999999997,"15min":0.60899999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":13.106,"5min":13.106,"15min":13.106},"last":0.59799999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:15:45 2023","interfaces":[{"interface":"back","average":{"1min":4.8049999999999997,"5min":1.395,"15min":0.83099999999999996},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":82.379000000000005,"5min":82.379000000000005,"15min":82.379000000000005},"last":4.1580000000000004},{"interface":"front","average":{"1min":5.2199999999999998,"5min":1.48,"15min":0.85199999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":85.596000000000004,"5min":85.596000000000004,"15min":85.596000000000004},"last":4.7789999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:15:15 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.59799999999999998,"15min":0.57799999999999996},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.2069999999999999,"5min":2.2069999999999999,"15min":2.2069999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.56000000000000005,"15min":0.55100000000000005},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.6140000000000001,"5min":1.6140000000000001,"15min":1.6140000000000001},"last":0.59399999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:15:00 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47799999999999998,"15min":0.47699999999999998},"min":{"1min":0.29699999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.71099999999999997,"5min":0.77700000000000002,"15min":0.79100000000000004},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52300000000000002,"15min":0.55200000000000005},"min":{"1min":0.34100000000000003,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":5.5369999999999999},"last":0.43099999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:15:00 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":1.5600000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.58699999999999997,"15min":0.55700000000000005},"min":{"1min":0.39000000000000001,"5min":0.35099999999999998,"15min":0.28599999999999998},"max":{"1min":3.2839999999999998,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":0.41499999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:15:00 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.56999999999999995,"15min":0.56100000000000005},"min":{"1min":0.35699999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.85499999999999998,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.60099999999999998,"15min":0.62},"min":{"1min":0.35199999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.655,"5min":1.655,"15min":5.5700000000000003},"last":0.46899999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626611,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1763812,"kb_used_data":1466588,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91981340,"statfs":{"total":95995035648,"available":94188892160,"internally_reserved":0,"allocated":1501786112,"data_stored":1501285724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.61899999999999999,"15min":0.624},"min":{"1min":0.41499999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.7749999999999999,"5min":3.7749999999999999,"15min":3.7749999999999999},"last":1.998},{"interface":"front","average":{"1min":0.78800000000000003,"5min":0.61099999999999999,"15min":0.623},"min":{"1min":0.40200000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.1259999999999999,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.247}]},{"osd":1,"last update":"Sun Oct 1 09:15:20 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.624,"15min":0.59199999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.2730000000000001,"5min":3.2730000000000001,"15min":3.2730000000000001},"last":1.879},{"interface":"front","average":{"1min":1.2470000000000001,"5min":0.72699999999999998,"15min":0.64300000000000002},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":3.9830000000000001,"5min":3.9830000000000001,"15min":3.9830000000000001},"last":0.32600000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:15:44 2023","interfaces":[{"interface":"back","average":{"1min":11.718,"5min":2.7749999999999999,"15min":1.2769999999999999},"min":{"1min":0.41599999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":82.978999999999999,"5min":82.978999999999999,"15min":82.978999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":11.433999999999999,"5min":2.7210000000000001,"15min":1.2549999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":82.747,"5min":82.747,"15min":82.747},"last":2.4620000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:15:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.64400000000000002,"15min":0.59099999999999997},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":3.3109999999999999,"5min":3.3109999999999999,"15min":3.3109999999999999},"last":1.948},{"interface":"front","average":{"1min":1.4019999999999999,"5min":0.72999999999999998,"15min":0.61799999999999999},"min":{"1min":0.41899999999999998,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":5.0359999999999996,"5min":5.0359999999999996,"15min":5.0359999999999996},"last":0.30499999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:15:42 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.57499999999999996,"15min":0.502},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.25900000000000001},"max":{"1min":4.0099999999999998,"5min":4.0099999999999998,"15min":4.0099999999999998},"last":9.2780000000000005},{"interface":"front","average":{"1min":1.0669999999999999,"5min":0.64100000000000001,"15min":0.55400000000000005},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.1920000000000002,"5min":5.1920000000000002,"15min":5.1920000000000002},"last":1.9199999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:15:42 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":0.63,"15min":0.54800000000000004},"min":{"1min":0.34599999999999997,"5min":0.32200000000000001,"15min":0.251},"max":{"1min":4.0990000000000002,"5min":4.0990000000000002,"15min":4.0990000000000002},"last":5.226},{"interface":"front","average":{"1min":1.1519999999999999,"5min":0.69799999999999995,"15min":0.59499999999999997},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":5.1559999999999997,"5min":5.1559999999999997,"15min":5.1559999999999997},"last":1.974}]},{"osd":7,"last update":"Sun Oct 1 09:14:52 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.57099999999999995,"15min":0.55200000000000005},"min":{"1min":0.41399999999999998,"5min":0.32600000000000001,"15min":0.29899999999999999},"max":{"1min":0.77300000000000002,"5min":0.79600000000000004,"15min":0.80500000000000005},"last":5.2640000000000002},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.627,"15min":0.59499999999999997},"min":{"1min":0.41399999999999998,"5min":0.34999999999999998,"15min":0.32400000000000001},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":6.4710000000000001}]}]},{"osd":1,"up_from":94,"seq":403726925870,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1904180,"kb_used_data":1854444,"kb_used_omap":7,"kb_used_meta":49720,"kb_avail":91840972,"statfs":{"total":95995035648,"available":94045155328,"internally_reserved":0,"allocated":1898950656,"data_stored":1898376788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":50913703},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:11 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.38500000000000001,"15min":0.35899999999999999},"min":{"1min":0.20399999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.8490000000000002,"5min":2.8490000000000002,"15min":2.8490000000000002},"last":2.2269999999999999},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.44400000000000001,"15min":0.39200000000000002},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":5.2549999999999999,"5min":5.2549999999999999,"15min":5.2549999999999999},"last":5.7489999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":0.69099999999999995,"15min":0.45200000000000001},"min":{"1min":0.24299999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":13.9,"5min":13.9,"15min":13.9},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.3839999999999999,"5min":0.75600000000000001,"15min":0.47599999999999998},"min":{"1min":0.216,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":8.2089999999999996,"5min":8.2089999999999996,"15min":8.2089999999999996},"last":5.6420000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:15:18 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.49199999999999999,"15min":0.41199999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":5.3029999999999999,"5min":5.3029999999999999,"15min":5.3029999999999999},"last":5.7759999999999998},{"interface":"front","average":{"1min":1.75,"5min":0.65600000000000003,"15min":0.47399999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":13.853,"5min":13.853,"15min":13.853},"last":0.44800000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":0.65800000000000003,"15min":0.58099999999999996},"min":{"1min":0.40000000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":3.698,"5min":3.698,"15min":3.698},"last":1.5469999999999999},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.63800000000000001,"15min":0.58599999999999997},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.3550000000000004,"5min":6.3550000000000004,"15min":6.3550000000000004},"last":2.0089999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.329,"5min":0.71299999999999997,"15min":0.60699999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.3490000000000002,"5min":5.3490000000000002,"15min":5.3490000000000002},"last":5.6890000000000001},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.67100000000000004,"15min":0.622},"min":{"1min":0.46600000000000003,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":3.673,"5min":3.673,"15min":3.673},"last":1.577}]},{"osd":6,"last update":"Sun Oct 1 09:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":0.69299999999999995,"15min":0.63100000000000001},"min":{"1min":0.45100000000000001,"5min":0.374,"15min":0.374},"max":{"1min":5.3609999999999998,"5min":5.3609999999999998,"15min":5.3609999999999998},"last":5.7199999999999998},{"interface":"front","average":{"1min":1.018,"5min":0.70399999999999996,"15min":0.65600000000000003},"min":{"1min":0.44,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":3.6309999999999998,"5min":3.6309999999999998,"15min":3.6309999999999998},"last":1.6020000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:15:11 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.67200000000000004,"15min":0.625},"min":{"1min":0.43099999999999999,"5min":0.376,"15min":0.376},"max":{"1min":3.7170000000000001,"5min":3.7170000000000001,"15min":3.7170000000000001},"last":2.149},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.64300000000000002,"15min":0.61799999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.976,"5min":2.976,"15min":2.976},"last":2.0449999999999999}]}]},{"osd":0,"up_from":89,"seq":382252089404,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3009544,"kb_used_data":2945856,"kb_used_omap":8,"kb_used_meta":63671,"kb_avail":90735608,"statfs":{"total":95995035648,"available":92913262592,"internally_reserved":0,"allocated":3016556544,"data_stored":3015967358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":65200066},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1499999999999999,"5min":0.48999999999999999,"15min":0.378},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":8.641,"5min":8.641,"15min":8.641},"last":0.187},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.42499999999999999,"15min":0.39000000000000001},"min":{"1min":0.23499999999999999,"5min":0.219,"15min":0.219},"max":{"1min":2.4089999999999998,"5min":2.4089999999999998,"15min":2.4089999999999998},"last":0.29699999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:14:43 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.38300000000000001,"15min":0.38800000000000001},"min":{"1min":0.25700000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.39900000000000002,"15min":0.40400000000000003},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.61099999999999999,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.47199999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":0.53200000000000003,"15min":0.42799999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.5660000000000007,"5min":8.5660000000000007,"15min":8.5660000000000007},"last":0.251},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.41099999999999998,"15min":0.38100000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":2.056,"5min":2.056,"15min":2.056},"last":0.35999999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.64000000000000001,"15min":0.64100000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.768,"5min":1.768,"15min":1.768},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64300000000000002,"15min":0.66600000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.8320000000000001,"5min":1.8320000000000001,"15min":1.8320000000000001},"last":1.73}]},{"osd":5,"last update":"Sun Oct 1 09:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.66500000000000004,"15min":0.68100000000000005},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.786,"5min":1.786,"15min":1.786},"last":2.9289999999999998},{"interface":"front","average":{"1min":1.3480000000000001,"5min":0.78900000000000003,"15min":0.72099999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.7230000000000008,"5min":8.7230000000000008,"15min":8.7230000000000008},"last":7.758}]},{"osd":6,"last update":"Sun Oct 1 09:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.69499999999999995,"15min":0.69699999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.8889999999999998,"5min":3.8889999999999998,"15min":3.8889999999999998},"last":2.9020000000000001},{"interface":"front","average":{"1min":1.226,"5min":0.77000000000000002,"15min":0.71399999999999997},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":8.6910000000000007,"5min":8.6910000000000007,"15min":8.6910000000000007},"last":3.5750000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:15:08 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":0.79700000000000004,"15min":0.73699999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":8.7850000000000001,"5min":8.7850000000000001,"15min":8.7850000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.71399999999999997,"15min":0.70599999999999996},"min":{"1min":0.47799999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":3.0830000000000002,"5min":3.0830000000000002,"15min":3.0830000000000002},"last":0.80000000000000004}]}]},{"osd":2,"up_from":99,"seq":425201762344,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2332716,"kb_used_data":2252708,"kb_used_omap":11,"kb_used_meta":79988,"kb_avail":91412436,"statfs":{"total":95995035648,"available":93606334464,"internally_reserved":0,"allocated":2306772992,"data_stored":2306206211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":81908599},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":5.266,"5min":1.3260000000000001,"15min":0.66000000000000003},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":85.838999999999999,"5min":85.838999999999999,"15min":85.838999999999999},"last":2.9049999999999998},{"interface":"front","average":{"1min":6.601,"5min":1.605,"15min":0.76300000000000001},"min":{"1min":0.35999999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":85.938000000000002,"5min":85.938000000000002,"15min":85.938000000000002},"last":3.2850000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":6.6360000000000001,"5min":1.6240000000000001,"15min":0.78500000000000003},"min":{"1min":0.21299999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":105.3,"5min":105.3,"15min":105.3},"last":5.859},{"interface":"front","average":{"1min":7.7619999999999996,"5min":1.859,"15min":0.86699999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":107.729,"5min":107.729,"15min":107.729},"last":4.1130000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:15:15 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.42599999999999999,"15min":0.38800000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":2.4660000000000002,"5min":2.4660000000000002,"15min":2.4660000000000002},"last":2.9329999999999998},{"interface":"front","average":{"1min":0.79800000000000004,"5min":0.48899999999999999,"15min":0.437},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.3209999999999997,"5min":4.3209999999999997,"15min":4.3209999999999997},"last":5.9169999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":0.747,"15min":0.65100000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.5170000000000003,"5min":6.5170000000000003,"15min":6.5170000000000003},"last":3.0779999999999998},{"interface":"front","average":{"1min":1.7290000000000001,"5min":0.79800000000000004,"15min":0.63900000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.1689999999999996,"5min":6.1689999999999996,"15min":6.1689999999999996},"last":2.984}]},{"osd":5,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":0.89600000000000002,"15min":0.68200000000000005},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.2999999999999998,"5min":7.2999999999999998,"15min":7.2999999999999998},"last":2.7370000000000001},{"interface":"front","average":{"1min":1.798,"5min":0.82699999999999996,"15min":0.65800000000000003},"min":{"1min":0.377,"5min":0.317,"15min":0.317},"max":{"1min":5.1959999999999997,"5min":5.1959999999999997,"15min":5.1959999999999997},"last":3.2559999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":0.78500000000000003,"15min":0.66200000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.9699999999999998,"5min":5.9699999999999998,"15min":5.9699999999999998},"last":3.016},{"interface":"front","average":{"1min":1.026,"5min":0.71099999999999997,"15min":0.65200000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.9580000000000002,"5min":5.9580000000000002,"15min":5.9580000000000002},"last":5.9470000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":0.78100000000000003,"15min":0.66100000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.1970000000000001,"5min":6.1970000000000001,"15min":6.1970000000000001},"last":5.9729999999999999},{"interface":"front","average":{"1min":2.1110000000000002,"5min":0.92200000000000004,"15min":0.71599999999999997},"min":{"1min":0.40100000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.5510000000000002,"5min":7.5510000000000002,"15min":7.5510000000000002},"last":2.4689999999999999}]}]},{"osd":3,"up_from":104,"seq":446676598817,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1123048,"kb_used_data":1083360,"kb_used_omap":3,"kb_used_meta":39676,"kb_avail":92622104,"statfs":{"total":95995035648,"available":94845034496,"internally_reserved":0,"allocated":1109360640,"data_stored":1108796528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":40628967},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":17.649000000000001,"5min":3.8220000000000001,"15min":1.518},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":199.422,"5min":199.422,"15min":199.422},"last":0.995},{"interface":"front","average":{"1min":17.515999999999998,"5min":3.7839999999999998,"15min":1.4950000000000001},"min":{"1min":0.23799999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":199.91200000000001,"5min":199.91200000000001,"15min":199.91200000000001},"last":1.0820000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":8.2769999999999992,"5min":1.958,"15min":0.90500000000000003},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":154.32599999999999,"5min":154.32599999999999,"15min":154.32599999999999},"last":0.95699999999999996},{"interface":"front","average":{"1min":8.3840000000000003,"5min":1.976,"15min":0.90800000000000003},"min":{"1min":0.25700000000000001,"5min":0.25,"15min":0.25},"max":{"1min":154.249,"5min":154.249,"15min":154.249},"last":1.044}]},{"osd":2,"last update":"Sun Oct 1 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":0.48299999999999998,"15min":0.39600000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":7.6260000000000003,"5min":7.6260000000000003,"15min":7.6260000000000003},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.48799999999999999,"15min":0.441},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.407,"5min":2.407,"15min":2.407},"last":5.96}]},{"osd":4,"last update":"Sun Oct 1 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.64400000000000002,"15min":0.59199999999999997},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":4.093,"5min":4.093,"15min":4.093},"last":1.099},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.65300000000000002,"15min":0.627},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":2.129,"5min":2.129,"15min":2.129},"last":1.1220000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.65900000000000003,"15min":0.623},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":2.7269999999999999,"5min":2.7269999999999999,"15min":2.7269999999999999},"last":1.595},{"interface":"front","average":{"1min":1.0609999999999999,"5min":0.69699999999999995,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.601,"5min":7.601,"15min":7.601},"last":0.48999999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":0.70999999999999996,"15min":0.65600000000000003},"min":{"1min":0.40100000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":6.6040000000000001,"5min":6.6040000000000001,"15min":6.6040000000000001},"last":1.153},{"interface":"front","average":{"1min":1.4530000000000001,"5min":0.80100000000000005,"15min":0.69199999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":7.556,"5min":7.556,"15min":7.556},"last":0.53000000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":0.75800000000000001,"15min":0.67800000000000005},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.4950000000000001,"5min":7.4950000000000001,"15min":7.4950000000000001},"last":1.7549999999999999},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.70999999999999996,"15min":0.66500000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":2.4279999999999999,"5min":2.4279999999999999,"15min":2.4279999999999999},"last":1.768}]}]},{"osd":4,"up_from":30,"seq":128849019067,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3209964,"kb_used_data":2912740,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":90535188,"statfs":{"total":95995035648,"available":92708032512,"internally_reserved":0,"allocated":2982645760,"data_stored":2982137323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":0.90300000000000002,"15min":0.69199999999999995},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":29.553000000000001,"5min":29.553000000000001,"15min":29.553000000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":2.2850000000000001,"5min":0.90900000000000003,"15min":0.68799999999999994},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":28.408000000000001,"5min":28.408000000000001,"15min":28.408000000000001},"last":1.9179999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:15:16 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.63100000000000001,"15min":0.59999999999999998},"min":{"1min":0.34499999999999997,"5min":0.313,"15min":0.313},"max":{"1min":3.8140000000000001,"5min":3.8140000000000001,"15min":3.8140000000000001},"last":6.3319999999999999},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.60099999999999998,"15min":0.56399999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.1560000000000001,"5min":3.1560000000000001,"15min":3.1560000000000001},"last":3.536}]},{"osd":2,"last update":"Sun Oct 1 09:15:44 2023","interfaces":[{"interface":"back","average":{"1min":13.41,"5min":3.1139999999999999,"15min":1.3919999999999999},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":85.441000000000003,"5min":85.441000000000003,"15min":85.441000000000003},"last":6.3659999999999997},{"interface":"front","average":{"1min":12.993,"5min":3.048,"15min":1.385},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":84.736000000000004,"5min":84.736000000000004,"15min":84.736000000000004},"last":2.891}]},{"osd":3,"last update":"Sun Oct 1 09:15:18 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.60899999999999999,"15min":0.56200000000000006},"min":{"1min":0.40999999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":3.1040000000000001,"5min":3.1040000000000001,"15min":3.1040000000000001},"last":6.165},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.63600000000000001,"15min":0.57999999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":4.5570000000000004,"5min":4.5570000000000004,"15min":4.5570000000000004},"last":1.7470000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:15:16 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.54200000000000004,"15min":0.47899999999999998},"min":{"1min":0.35199999999999998,"5min":0.317,"15min":0.26300000000000001},"max":{"1min":2.8570000000000002,"5min":2.8570000000000002,"15min":2.8570000000000002},"last":4.577},{"interface":"front","average":{"1min":1.0349999999999999,"5min":0.60899999999999999,"15min":0.51400000000000001},"min":{"1min":0.33400000000000002,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":3.798,"5min":3.798,"15min":3.798},"last":0.33200000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:15:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":0.67800000000000005,"15min":0.55900000000000005},"min":{"1min":0.30499999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.016,"5min":5.016,"15min":5.016},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.1950000000000001,"5min":0.70599999999999996,"15min":0.60199999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.33700000000000002},"max":{"1min":3.1190000000000002,"5min":3.1190000000000002,"15min":3.1190000000000002},"last":1.714}]},{"osd":7,"last update":"Sun Oct 1 09:14:53 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.55900000000000005,"15min":0.54500000000000004},"min":{"1min":0.371,"5min":0.33000000000000002,"15min":0.317},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":1.268},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.63600000000000001,"15min":0.58399999999999996},"min":{"1min":0.51300000000000001,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":3.3410000000000002,"5min":3.3410000000000002,"15min":3.3410000000000002},"last":0.40300000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822840,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3594716,"kb_used_data":3285204,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":90150436,"statfs":{"total":95995035648,"available":92314046464,"internally_reserved":0,"allocated":3364048896,"data_stored":3363552291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:05 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":0.82999999999999996,"15min":0.70199999999999996},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":14.186,"5min":14.186,"15min":14.186},"last":1.169},{"interface":"front","average":{"1min":1.758,"5min":0.86899999999999999,"15min":0.69499999999999995},"min":{"1min":0.34300000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":14.308,"5min":14.308,"15min":14.308},"last":1.236}]},{"osd":1,"last update":"Sun Oct 1 09:15:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":0.90700000000000003,"15min":0.65000000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":32.420000000000002,"5min":32.420000000000002,"15min":32.420000000000002},"last":1.48},{"interface":"front","average":{"1min":2.2320000000000002,"5min":0.90700000000000003,"15min":0.66600000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":32.481999999999999,"5min":32.481999999999999,"15min":32.481999999999999},"last":2.3599999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.65200000000000002,"15min":0.66600000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.78600000000000003,"5min":2.5430000000000001,"15min":2.5430000000000001},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.61199999999999999,"15min":0.62},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.75,"5min":2.5190000000000001,"15min":2.5190000000000001},"last":0.36099999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:15:18 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.85799999999999998,"15min":0.86599999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":2.8029999999999999,"5min":4.5990000000000002,"15min":4.5990000000000002},"last":1.8580000000000001},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.71099999999999997,"15min":0.68700000000000006},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":2.9159999999999999,"5min":2.9159999999999999,"15min":2.9159999999999999},"last":0.34200000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:15:09 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.502,"15min":0.47199999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.6899999999999999,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":0.315},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.56799999999999995,"15min":0.52500000000000002},"min":{"1min":0.34000000000000002,"5min":0.316,"15min":0.28100000000000003},"max":{"1min":2.5699999999999998,"5min":2.5699999999999998,"15min":2.5699999999999998},"last":0.33000000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:15:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1930000000000001,"5min":0.66400000000000003,"15min":0.57399999999999995},"min":{"1min":0.312,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":5.2430000000000003,"5min":5.2430000000000003,"15min":5.2430000000000003},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.65300000000000002,"15min":0.56999999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.29399999999999998},"max":{"1min":4.7439999999999998,"5min":4.7439999999999998,"15min":4.7439999999999998},"last":0.42499999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:14:53 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61199999999999999,"15min":0.57599999999999996},"min":{"1min":0.40899999999999997,"5min":0.34300000000000003,"15min":0.32300000000000001},"max":{"1min":0.83999999999999997,"5min":0.873,"15min":0.89300000000000002},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.61899999999999999,"15min":0.57899999999999996},"min":{"1min":0.436,"5min":0.38700000000000001,"15min":0.32000000000000001},"max":{"1min":2.5800000000000001,"5min":2.5800000000000001,"15min":2.5800000000000001},"last":0.46100000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3016097792,"data_stored":3016097792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1903886336,"data_stored":1903886336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2309554176,"data_stored":2309554176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1120010240,"data_stored":1120010240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2986934272,"data_stored":2986934272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3363110912,"data_stored":3363110912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1506672640,"data_stored":1506672640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1875312640,"data_stored":1875312640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:15:53.554 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:15:53.681 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:15:53.708 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:53.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:53.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:53 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:15:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:53 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:15:53.819 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:53.946 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:53.988 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:54.203 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:54.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:54.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:54.307 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:15:54.358 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:54.440 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:15:54.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:54.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:15:54.531 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 70 16 98895 98879 88.2563 50.9375 0.0149247 0.0113253 2023-10-01T09:15:54.550 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:15:54.608 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:54.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:54 smithi138 ceph-mon[185886]: pgmap v277: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2023-10-01T09:15:54.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:54 smithi138 ceph-mon[192513]: pgmap v277: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2023-10-01T09:15:54.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:54 smithi190 ceph-mon[160435]: pgmap v277: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2023-10-01T09:15:54.842 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:15:54.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:54.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:15:54.952 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:15:55.005 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:55.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:15:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:15:54] "GET /metrics HTTP/1.1" 200 36324 "" "Prometheus/2.33.4" 2023-10-01T09:15:55.093 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:15:55.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:55.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:15:55.202 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:15:55.275 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:55.381 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:15:55.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:55.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:15:55.435 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:55 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4085022234' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:15:55.436 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:55 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:15:55.436 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:15:55.436 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:55 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4085022234' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:15:55.436 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:55 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:15:55.436 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:15:55.449 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:15:55.497 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:15:55.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:55 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4085022234' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:15:55.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:55 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:15:55.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:15:55.523 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:55.531 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 71 16 99729 99713 87.7474 52.125 0.0059631 0.0113796 2023-10-01T09:15:55.564 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:15:55.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:55.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:15:55.672 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:15:55.713 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:55.814 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:15:55.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:55.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:15:55.919 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:55.977 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:56.532 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 72 16 100443 100427 87.1481 44.625 0.0149833 0.0114703 2023-10-01T09:15:56.750 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:56.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:56 smithi138 ceph-mon[185886]: pgmap v278: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-10-01T09:15:56.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:56 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:15:56.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:56 smithi138 ceph-mon[185886]: failed to encode map e121 with expected crc 2023-10-01T09:15:56.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:56 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:15:56.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:56 smithi138 ceph-mon[185886]: osdmap e121: 8 total, 8 up, 8 in 2023-10-01T09:15:56.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:56 smithi138 ceph-mon[185886]: failed to encode map e121 with expected crc 2023-10-01T09:15:56.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:56 smithi138 ceph-mon[185886]: failed to encode map e121 with expected crc 2023-10-01T09:15:56.756 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:56 smithi138 ceph-mon[192513]: pgmap v278: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-10-01T09:15:56.756 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:56 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:15:56.756 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:56 smithi138 ceph-mon[192513]: failed to encode map e121 with expected crc 2023-10-01T09:15:56.756 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:56 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:15:56.756 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:56 smithi138 ceph-mon[192513]: osdmap e121: 8 total, 8 up, 8 in 2023-10-01T09:15:56.756 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:56 smithi138 ceph-mon[192513]: failed to encode map e121 with expected crc 2023-10-01T09:15:56.756 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:56 smithi138 ceph-mon[192513]: failed to encode map e121 with expected crc 2023-10-01T09:15:56.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:56.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:56 smithi190 ceph-mon[160435]: pgmap v278: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 887 op/s 2023-10-01T09:15:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:56 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:15:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:56 smithi190 ceph-mon[160435]: failed to encode map e121 with expected crc 2023-10-01T09:15:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:56 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:15:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:56 smithi190 ceph-mon[160435]: osdmap e121: 8 total, 8 up, 8 in 2023-10-01T09:15:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:56 smithi190 ceph-mon[160435]: failed to encode map e121 with expected crc 2023-10-01T09:15:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:56 smithi190 ceph-mon[160435]: failed to encode map e121 with expected crc 2023-10-01T09:15:56.857 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:56.885 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:56.895 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:15:56.973 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:57.125 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:15:57.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:57.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:15:57.239 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:15:57.317 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:57.532 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 73 16 101396 101380 86.7702 59.5625 0.0189174 0.0115202 2023-10-01T09:15:57.610 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:15:57.640 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:15:57.681 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:15:57.728 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:15:57.764 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:15:57.804 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:15:57.841 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:15:57.880 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:15:57.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:15:57.890 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:15:57.890 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:15:57.890 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:15:57.890 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:15:57.890 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:15:57.890 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:15:57.890 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:15:58.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:57 smithi190 ceph-mon[160435]: pgmap v280: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2023-10-01T09:15:58.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:15:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1498827882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:58.317 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:57 smithi138 ceph-mon[185886]: pgmap v280: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2023-10-01T09:15:58.317 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:15:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1498827882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:58.318 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:57 smithi138 ceph-mon[192513]: pgmap v280: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2023-10-01T09:15:58.318 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:15:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1498827882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:15:58.341 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:15:58.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:58.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:15:58.452 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:15:58.519 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:15:58.532 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 74 16 102352 102336 86.4051 59.75 0.017687 0.011568 2023-10-01T09:15:58.935 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:15:59.532 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 75 16 103136 103120 85.9064 49 0.0191462 0.0116365 2023-10-01T09:15:59.606 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:15:59.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:15:59.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:15:59.719 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:15:59.799 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:00.268 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:16:00.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:00.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:16:00.374 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:16:00.476 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:00.532 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 76 16 103956 103940 85.4504 51.25 0.00850746 0.0116987 2023-10-01T09:16:00.718 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:16:00.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:00.727 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:00 smithi138 ceph-mon[185886]: pgmap v281: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2023-10-01T09:16:00.727 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:00 smithi138 ceph-mon[192513]: pgmap v281: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2023-10-01T09:16:00.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:16:00.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:00 smithi190 ceph-mon[160435]: pgmap v281: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2023-10-01T09:16:00.829 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:00.899 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:00.992 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:00.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:00.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:01.099 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:01.147 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:01.153 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:16:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:16:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:16:01.153 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:16:01.153 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:16:01.220 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:01.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:01.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:01.328 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:01.398 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:01.509 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:01.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:01.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:01.532 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 77 16 104862 104846 85.0761 56.625 0.0126485 0.0117491 2023-10-01T09:16:01.618 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:01.661 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:02.090 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:02.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:02.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:02.196 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:02.243 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:16:02.255 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:16:02.274 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:02.363 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:02 smithi138 ceph-mon[185886]: pgmap v282: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2023-10-01T09:16:02.402 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:16:02.403 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:02.443 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:16:02.504 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:02.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:02.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:02.533 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 78 16 105496 105480 84.4923 39.625 0.00903816 0.0118306 2023-10-01T09:16:02.611 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:02.647 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:02.648 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:02 smithi190 ceph-mon[160435]: pgmap v282: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2023-10-01T09:16:02.720 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:02.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:02.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:02.804 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:02 smithi138 ceph-mon[192513]: pgmap v282: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2023-10-01T09:16:02.825 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:02.891 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:03.533 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 79 16 106272 106256 84.0368 48.5 0.0163993 0.0118945 2023-10-01T09:16:03.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:03 smithi190 ceph-mon[160435]: from='client.34758 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:16:03.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:03 smithi138 ceph-mon[192513]: from='client.34758 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:16:03.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:03 smithi138 ceph-mon[185886]: from='client.34758 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:16:03.941 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":282,"stamp":"2023-10-01T09:16:01.299231+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6759716787,"num_objects":103100,"num_object_clones":0,"num_object_copies":309300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":103100,"num_whiteouts":0,"num_read":6302,"num_read_kb":12700,"num_write":104178,"num_write_kb":6611749,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104242,"ondisk_log_size":104242,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21263420,"kb_used_data":19766780,"kb_used_omap":30,"kb_used_meta":1496545,"kb_avail":728697796,"statfs":{"total":767960285184,"available":746186543104,"internally_reserved":0,"allocated":20241182720,"data_stored":20236853185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30777,"internal_metadata":1532463047},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2,0,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":569573376,"num_objects":8691,"num_object_clones":0,"num_object_copies":26073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8691,"num_write_kb":556224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003247"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":102,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288676+0000","last_change":"2023-10-01T09:12:40.947346+0000","last_active":"2023-10-01T09:15:43.288676+0000","last_peered":"2023-10-01T09:15:43.288676+0000","last_clean":"2023-10-01T09:15:43.288676+0000","last_became_active":"2023-10-01T09:12:40.946939+0000","last_became_peered":"2023-10-01T09:12:40.946939+0000","last_unstale":"2023-10-01T09:15:43.288676+0000","last_undegraded":"2023-10-01T09:15:43.288676+0000","last_fullsized":"2023-10-01T09:15:43.288676+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":90,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288691+0000","last_change":"2023-10-01T09:13:13.554780+0000","last_active":"2023-10-01T09:15:43.288691+0000","last_peered":"2023-10-01T09:15:43.288691+0000","last_clean":"2023-10-01T09:15:43.288691+0000","last_became_active":"2023-10-01T09:13:13.554394+0000","last_became_peered":"2023-10-01T09:13:13.554394+0000","last_unstale":"2023-10-01T09:15:43.288691+0000","last_undegraded":"2023-10-01T09:15:43.288691+0000","last_fullsized":"2023-10-01T09:15:43.288691+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:09:02.869321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":64,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.283384+0000","last_change":"2023-10-01T09:13:13.551596+0000","last_active":"2023-10-01T09:15:43.283384+0000","last_peered":"2023-10-01T09:15:43.283384+0000","last_clean":"2023-10-01T09:15:43.283384+0000","last_became_active":"2023-10-01T09:13:13.551300+0000","last_became_peered":"2023-10-01T09:13:13.551300+0000","last_unstale":"2023-10-01T09:15:43.283384+0000","last_undegraded":"2023-10-01T09:15:43.283384+0000","last_fullsized":"2023-10-01T09:15:43.283384+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:57:45.081780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":69,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.331050+0000","last_change":"2023-10-01T09:13:13.548695+0000","last_active":"2023-10-01T09:15:43.331050+0000","last_peered":"2023-10-01T09:15:43.331050+0000","last_clean":"2023-10-01T09:15:43.331050+0000","last_became_active":"2023-10-01T09:13:13.548256+0000","last_became_peered":"2023-10-01T09:13:13.548256+0000","last_unstale":"2023-10-01T09:15:43.331050+0000","last_undegraded":"2023-10-01T09:15:43.331050+0000","last_fullsized":"2023-10-01T09:15:43.331050+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:35:28.655733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":106,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.289163+0000","last_change":"2023-10-01T09:13:13.555490+0000","last_active":"2023-10-01T09:15:43.289163+0000","last_peered":"2023-10-01T09:15:43.289163+0000","last_clean":"2023-10-01T09:15:43.289163+0000","last_became_active":"2023-10-01T09:13:13.554506+0000","last_became_peered":"2023-10-01T09:13:13.554506+0000","last_unstale":"2023-10-01T09:15:43.289163+0000","last_undegraded":"2023-10-01T09:15:43.289163+0000","last_fullsized":"2023-10-01T09:15:43.289163+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:18.834474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":95,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.287743+0000","last_change":"2023-10-01T09:12:40.947591+0000","last_active":"2023-10-01T09:15:43.287743+0000","last_peered":"2023-10-01T09:15:43.287743+0000","last_clean":"2023-10-01T09:15:43.287743+0000","last_became_active":"2023-10-01T09:12:40.947242+0000","last_became_peered":"2023-10-01T09:12:40.947242+0000","last_unstale":"2023-10-01T09:15:43.287743+0000","last_undegraded":"2023-10-01T09:15:43.287743+0000","last_fullsized":"2023-10-01T09:15:43.287743+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":69,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:46.909686+0000","last_change":"2023-10-01T09:13:13.550193+0000","last_active":"2023-10-01T09:15:46.909686+0000","last_peered":"2023-10-01T09:15:46.909686+0000","last_clean":"2023-10-01T09:15:46.909686+0000","last_became_active":"2023-10-01T09:13:13.549900+0000","last_became_peered":"2023-10-01T09:13:13.549900+0000","last_unstale":"2023-10-01T09:15:46.909686+0000","last_undegraded":"2023-10-01T09:15:46.909686+0000","last_fullsized":"2023-10-01T09:15:46.909686+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:22.643161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":90,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.289066+0000","last_change":"2023-10-01T09:13:13.555368+0000","last_active":"2023-10-01T09:15:43.289066+0000","last_peered":"2023-10-01T09:15:43.289066+0000","last_clean":"2023-10-01T09:15:43.289066+0000","last_became_active":"2023-10-01T09:13:13.554806+0000","last_became_peered":"2023-10-01T09:13:13.554806+0000","last_unstale":"2023-10-01T09:15:43.289066+0000","last_undegraded":"2023-10-01T09:15:43.289066+0000","last_fullsized":"2023-10-01T09:15:43.289066+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:24:32.687833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":87,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.280698+0000","last_change":"2023-10-01T09:12:10.003091+0000","last_active":"2023-10-01T09:15:43.280698+0000","last_peered":"2023-10-01T09:15:43.280698+0000","last_clean":"2023-10-01T09:15:43.280698+0000","last_became_active":"2023-10-01T09:12:10.002773+0000","last_became_peered":"2023-10-01T09:12:10.002773+0000","last_unstale":"2023-10-01T09:15:43.280698+0000","last_undegraded":"2023-10-01T09:15:43.280698+0000","last_fullsized":"2023-10-01T09:15:43.280698+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":80,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288291+0000","last_change":"2023-10-01T09:13:13.554942+0000","last_active":"2023-10-01T09:15:43.288291+0000","last_peered":"2023-10-01T09:15:43.288291+0000","last_clean":"2023-10-01T09:15:43.288291+0000","last_became_active":"2023-10-01T09:13:13.554664+0000","last_became_peered":"2023-10-01T09:13:13.554664+0000","last_unstale":"2023-10-01T09:15:43.288291+0000","last_undegraded":"2023-10-01T09:15:43.288291+0000","last_fullsized":"2023-10-01T09:15:43.288291+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:43:14.532709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":90,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.292684+0000","last_change":"2023-10-01T09:13:13.556801+0000","last_active":"2023-10-01T09:15:43.292684+0000","last_peered":"2023-10-01T09:15:43.292684+0000","last_clean":"2023-10-01T09:15:43.292684+0000","last_became_active":"2023-10-01T09:13:13.556397+0000","last_became_peered":"2023-10-01T09:13:13.556397+0000","last_unstale":"2023-10-01T09:15:43.292684+0000","last_undegraded":"2023-10-01T09:15:43.292684+0000","last_fullsized":"2023-10-01T09:15:43.292684+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":64,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.284625+0000","last_change":"2023-10-01T09:12:40.946520+0000","last_active":"2023-10-01T09:15:43.284625+0000","last_peered":"2023-10-01T09:15:43.284625+0000","last_clean":"2023-10-01T09:15:43.284625+0000","last_became_active":"2023-10-01T09:12:40.946227+0000","last_became_peered":"2023-10-01T09:12:40.946227+0000","last_unstale":"2023-10-01T09:15:43.284625+0000","last_undegraded":"2023-10-01T09:15:43.284625+0000","last_fullsized":"2023-10-01T09:15:43.284625+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:06:06.451923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":93,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.289054+0000","last_change":"2023-10-01T09:03:03.200261+0000","last_active":"2023-10-01T09:15:43.289054+0000","last_peered":"2023-10-01T09:15:43.289054+0000","last_clean":"2023-10-01T09:15:43.289054+0000","last_became_active":"2023-10-01T09:03:03.199944+0000","last_became_peered":"2023-10-01T09:03:03.199944+0000","last_unstale":"2023-10-01T09:15:43.289054+0000","last_undegraded":"2023-10-01T09:15:43.289054+0000","last_fullsized":"2023-10-01T09:15:43.289054+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":94,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288870+0000","last_change":"2023-10-01T09:12:40.946973+0000","last_active":"2023-10-01T09:15:43.288870+0000","last_peered":"2023-10-01T09:15:43.288870+0000","last_clean":"2023-10-01T09:15:43.288870+0000","last_became_active":"2023-10-01T09:12:40.946427+0000","last_became_peered":"2023-10-01T09:12:40.946427+0000","last_unstale":"2023-10-01T09:15:43.288870+0000","last_undegraded":"2023-10-01T09:15:43.288870+0000","last_fullsized":"2023-10-01T09:15:43.288870+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":89,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.292672+0000","last_change":"2023-10-01T09:12:40.945277+0000","last_active":"2023-10-01T09:15:43.292672+0000","last_peered":"2023-10-01T09:15:43.292672+0000","last_clean":"2023-10-01T09:15:43.292672+0000","last_became_active":"2023-10-01T09:12:40.944843+0000","last_became_peered":"2023-10-01T09:12:40.944843+0000","last_unstale":"2023-10-01T09:15:43.292672+0000","last_undegraded":"2023-10-01T09:15:43.292672+0000","last_fullsized":"2023-10-01T09:15:43.292672+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":90,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288569+0000","last_change":"2023-10-01T09:13:13.550633+0000","last_active":"2023-10-01T09:15:43.288569+0000","last_peered":"2023-10-01T09:15:43.288569+0000","last_clean":"2023-10-01T09:15:43.288569+0000","last_became_active":"2023-10-01T09:13:13.550305+0000","last_became_peered":"2023-10-01T09:13:13.550305+0000","last_unstale":"2023-10-01T09:15:43.288569+0000","last_undegraded":"2023-10-01T09:15:43.288569+0000","last_fullsized":"2023-10-01T09:15:43.288569+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:51:46.559383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":69,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.295056+0000","last_change":"2023-10-01T09:13:13.548094+0000","last_active":"2023-10-01T09:15:43.295056+0000","last_peered":"2023-10-01T09:15:43.295056+0000","last_clean":"2023-10-01T09:15:43.295056+0000","last_became_active":"2023-10-01T09:13:13.547782+0000","last_became_peered":"2023-10-01T09:13:13.547782+0000","last_unstale":"2023-10-01T09:15:43.295056+0000","last_undegraded":"2023-10-01T09:15:43.295056+0000","last_fullsized":"2023-10-01T09:15:43.295056+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:15.220827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":86,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.281438+0000","last_change":"2023-10-01T09:13:13.558367+0000","last_active":"2023-10-01T09:15:43.281438+0000","last_peered":"2023-10-01T09:15:43.281438+0000","last_clean":"2023-10-01T09:15:43.281438+0000","last_became_active":"2023-10-01T09:13:13.557952+0000","last_became_peered":"2023-10-01T09:13:13.557952+0000","last_unstale":"2023-10-01T09:15:43.281438+0000","last_undegraded":"2023-10-01T09:15:43.281438+0000","last_fullsized":"2023-10-01T09:15:43.281438+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":72,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.294563+0000","last_change":"2023-10-01T09:03:03.194952+0000","last_active":"2023-10-01T09:15:43.294563+0000","last_peered":"2023-10-01T09:15:43.294563+0000","last_clean":"2023-10-01T09:15:43.294563+0000","last_became_active":"2023-10-01T09:03:03.194643+0000","last_became_peered":"2023-10-01T09:03:03.194643+0000","last_unstale":"2023-10-01T09:15:43.294563+0000","last_undegraded":"2023-10-01T09:15:43.294563+0000","last_fullsized":"2023-10-01T09:15:43.294563+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":69,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:46.909604+0000","last_change":"2023-10-01T09:13:13.548531+0000","last_active":"2023-10-01T09:15:46.909604+0000","last_peered":"2023-10-01T09:15:46.909604+0000","last_clean":"2023-10-01T09:15:46.909604+0000","last_became_active":"2023-10-01T09:13:13.547881+0000","last_became_peered":"2023-10-01T09:13:13.547881+0000","last_unstale":"2023-10-01T09:15:46.909604+0000","last_undegraded":"2023-10-01T09:15:46.909604+0000","last_fullsized":"2023-10-01T09:15:46.909604+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:13:24.068359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":95,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.285814+0000","last_change":"2023-10-01T09:10:58.798168+0000","last_active":"2023-10-01T09:15:43.285814+0000","last_peered":"2023-10-01T09:15:43.285814+0000","last_clean":"2023-10-01T09:15:43.285814+0000","last_became_active":"2023-10-01T09:10:58.797781+0000","last_became_peered":"2023-10-01T09:10:58.797781+0000","last_unstale":"2023-10-01T09:15:43.285814+0000","last_undegraded":"2023-10-01T09:15:43.285814+0000","last_fullsized":"2023-10-01T09:15:43.285814+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288925+0000","last_change":"2023-10-01T09:13:13.550588+0000","last_active":"2023-10-01T09:15:43.288925+0000","last_peered":"2023-10-01T09:15:43.288925+0000","last_clean":"2023-10-01T09:15:43.288925+0000","last_became_active":"2023-10-01T09:13:13.550271+0000","last_became_peered":"2023-10-01T09:13:13.550271+0000","last_unstale":"2023-10-01T09:15:43.288925+0000","last_undegraded":"2023-10-01T09:15:43.288925+0000","last_fullsized":"2023-10-01T09:15:43.288925+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:19:47.301376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"121'6649","reported_seq":6672,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:16:01.196745+0000","last_change":"2023-10-01T09:14:31.351571+0000","last_active":"2023-10-01T09:16:01.196745+0000","last_peered":"2023-10-01T09:16:01.196745+0000","last_clean":"2023-10-01T09:16:01.196745+0000","last_became_active":"2023-10-01T09:14:31.351242+0000","last_became_peered":"2023-10-01T09:14:31.351242+0000","last_unstale":"2023-10-01T09:16:01.196745+0000","last_undegraded":"2023-10-01T09:16:01.196745+0000","last_fullsized":"2023-10-01T09:16:01.196745+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6649,"log_dups_size":0,"ondisk_log_size":6649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435748864,"num_objects":6649,"num_object_clones":0,"num_object_copies":19947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6649,"num_write_kb":425536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":88,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.292918+0000","last_change":"2023-10-01T09:10:58.797850+0000","last_active":"2023-10-01T09:15:43.292918+0000","last_peered":"2023-10-01T09:15:43.292918+0000","last_clean":"2023-10-01T09:15:43.292918+0000","last_became_active":"2023-10-01T09:10:58.797403+0000","last_became_peered":"2023-10-01T09:10:58.797403+0000","last_unstale":"2023-10-01T09:15:43.292918+0000","last_undegraded":"2023-10-01T09:15:43.292918+0000","last_fullsized":"2023-10-01T09:15:43.292918+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":71,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.280508+0000","last_change":"2023-10-01T09:03:04.245105+0000","last_active":"2023-10-01T09:15:43.280508+0000","last_peered":"2023-10-01T09:15:43.280508+0000","last_clean":"2023-10-01T09:15:43.280508+0000","last_became_active":"2023-10-01T09:03:04.244807+0000","last_became_peered":"2023-10-01T09:03:04.244807+0000","last_unstale":"2023-10-01T09:15:43.280508+0000","last_undegraded":"2023-10-01T09:15:43.280508+0000","last_fullsized":"2023-10-01T09:15:43.280508+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"121'6580","reported_seq":6604,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:15:58.944578+0000","last_change":"2023-10-01T09:14:31.344601+0000","last_active":"2023-10-01T09:15:58.944578+0000","last_peered":"2023-10-01T09:15:58.944578+0000","last_clean":"2023-10-01T09:15:58.944578+0000","last_became_active":"2023-10-01T09:14:31.344266+0000","last_became_peered":"2023-10-01T09:14:31.344266+0000","last_unstale":"2023-10-01T09:15:58.944578+0000","last_undegraded":"2023-10-01T09:15:58.944578+0000","last_fullsized":"2023-10-01T09:15:58.944578+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6580,"log_dups_size":0,"ondisk_log_size":6580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:31:20.164212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431226880,"num_objects":6580,"num_object_clones":0,"num_object_copies":19740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6580,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":89,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.293459+0000","last_change":"2023-10-01T09:12:10.003552+0000","last_active":"2023-10-01T09:15:43.293459+0000","last_peered":"2023-10-01T09:15:43.293459+0000","last_clean":"2023-10-01T09:15:43.293459+0000","last_became_active":"2023-10-01T09:12:10.003131+0000","last_became_peered":"2023-10-01T09:12:10.003131+0000","last_unstale":"2023-10-01T09:15:43.293459+0000","last_undegraded":"2023-10-01T09:15:43.293459+0000","last_fullsized":"2023-10-01T09:15:43.293459+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":100,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288714+0000","last_change":"2023-10-01T09:13:13.556674+0000","last_active":"2023-10-01T09:15:43.288714+0000","last_peered":"2023-10-01T09:15:43.288714+0000","last_clean":"2023-10-01T09:15:43.288714+0000","last_became_active":"2023-10-01T09:13:13.556271+0000","last_became_peered":"2023-10-01T09:13:13.556271+0000","last_unstale":"2023-10-01T09:15:43.288714+0000","last_undegraded":"2023-10-01T09:15:43.288714+0000","last_fullsized":"2023-10-01T09:15:43.288714+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"121'6418","reported_seq":6442,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:15:58.934965+0000","last_change":"2023-10-01T09:14:31.357184+0000","last_active":"2023-10-01T09:15:58.934965+0000","last_peered":"2023-10-01T09:15:58.934965+0000","last_clean":"2023-10-01T09:15:58.934965+0000","last_became_active":"2023-10-01T09:14:31.356912+0000","last_became_peered":"2023-10-01T09:14:31.356912+0000","last_unstale":"2023-10-01T09:15:58.934965+0000","last_undegraded":"2023-10-01T09:15:58.934965+0000","last_fullsized":"2023-10-01T09:15:58.934965+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6418,"log_dups_size":0,"ondisk_log_size":6418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:21:18.004826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420610048,"num_objects":6418,"num_object_clones":0,"num_object_copies":19254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6418,"num_write_kb":410752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":73,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.292864+0000","last_change":"2023-10-01T09:03:03.200972+0000","last_active":"2023-10-01T09:15:43.292864+0000","last_peered":"2023-10-01T09:15:43.292864+0000","last_clean":"2023-10-01T09:15:43.292864+0000","last_became_active":"2023-10-01T09:03:03.200695+0000","last_became_peered":"2023-10-01T09:03:03.200695+0000","last_unstale":"2023-10-01T09:15:43.292864+0000","last_undegraded":"2023-10-01T09:15:43.292864+0000","last_fullsized":"2023-10-01T09:15:43.292864+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":86,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.286213+0000","last_change":"2023-10-01T09:12:09.999571+0000","last_active":"2023-10-01T09:15:43.286213+0000","last_peered":"2023-10-01T09:15:43.286213+0000","last_clean":"2023-10-01T09:15:43.286213+0000","last_became_active":"2023-10-01T09:12:09.999272+0000","last_became_peered":"2023-10-01T09:12:09.999272+0000","last_unstale":"2023-10-01T09:15:43.286213+0000","last_undegraded":"2023-10-01T09:15:43.286213+0000","last_fullsized":"2023-10-01T09:15:43.286213+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"121'6278","reported_seq":6302,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:15:58.968385+0000","last_change":"2023-10-01T09:14:31.349127+0000","last_active":"2023-10-01T09:15:58.968385+0000","last_peered":"2023-10-01T09:15:58.968385+0000","last_clean":"2023-10-01T09:15:58.968385+0000","last_became_active":"2023-10-01T09:14:31.348832+0000","last_became_peered":"2023-10-01T09:14:31.348832+0000","last_unstale":"2023-10-01T09:15:58.968385+0000","last_undegraded":"2023-10-01T09:15:58.968385+0000","last_fullsized":"2023-10-01T09:15:58.968385+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6278,"log_dups_size":0,"ondisk_log_size":6278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:08:29.276824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411435008,"num_objects":6278,"num_object_clones":0,"num_object_copies":18834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6278,"num_write_kb":401792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":87,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.280787+0000","last_change":"2023-10-01T09:13:13.555458+0000","last_active":"2023-10-01T09:15:43.280787+0000","last_peered":"2023-10-01T09:15:43.280787+0000","last_clean":"2023-10-01T09:15:43.280787+0000","last_became_active":"2023-10-01T09:13:13.555058+0000","last_became_peered":"2023-10-01T09:13:13.555058+0000","last_unstale":"2023-10-01T09:15:43.280787+0000","last_undegraded":"2023-10-01T09:15:43.280787+0000","last_fullsized":"2023-10-01T09:15:43.280787+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":69,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.295042+0000","last_change":"2023-10-01T09:12:10.001565+0000","last_active":"2023-10-01T09:15:43.295042+0000","last_peered":"2023-10-01T09:15:43.295042+0000","last_clean":"2023-10-01T09:15:43.295042+0000","last_became_active":"2023-10-01T09:12:10.001275+0000","last_became_peered":"2023-10-01T09:12:10.001275+0000","last_unstale":"2023-10-01T09:15:43.295042+0000","last_undegraded":"2023-10-01T09:15:43.295042+0000","last_fullsized":"2023-10-01T09:15:43.295042+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:39.067046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"121'6431","reported_seq":6455,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:15:58.973175+0000","last_change":"2023-10-01T09:14:31.356715+0000","last_active":"2023-10-01T09:15:58.973175+0000","last_peered":"2023-10-01T09:15:58.973175+0000","last_clean":"2023-10-01T09:15:58.973175+0000","last_became_active":"2023-10-01T09:14:31.356417+0000","last_became_peered":"2023-10-01T09:14:31.356417+0000","last_unstale":"2023-10-01T09:15:58.973175+0000","last_undegraded":"2023-10-01T09:15:58.973175+0000","last_fullsized":"2023-10-01T09:15:58.973175+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6431,"log_dups_size":0,"ondisk_log_size":6431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:50:26.342593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421462016,"num_objects":6431,"num_object_clones":0,"num_object_copies":19293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6431,"num_write_kb":411584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":88,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288331+0000","last_change":"2023-10-01T09:13:13.554321+0000","last_active":"2023-10-01T09:15:43.288331+0000","last_peered":"2023-10-01T09:15:43.288331+0000","last_clean":"2023-10-01T09:15:43.288331+0000","last_became_active":"2023-10-01T09:13:13.553986+0000","last_became_peered":"2023-10-01T09:13:13.553986+0000","last_unstale":"2023-10-01T09:15:43.288331+0000","last_undegraded":"2023-10-01T09:15:43.288331+0000","last_fullsized":"2023-10-01T09:15:43.288331+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:09:12.848269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":85,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.289094+0000","last_change":"2023-10-01T09:13:13.556491+0000","last_active":"2023-10-01T09:15:43.289094+0000","last_peered":"2023-10-01T09:15:43.289094+0000","last_clean":"2023-10-01T09:15:43.289094+0000","last_became_active":"2023-10-01T09:13:13.556118+0000","last_became_peered":"2023-10-01T09:13:13.556118+0000","last_unstale":"2023-10-01T09:15:43.289094+0000","last_undegraded":"2023-10-01T09:15:43.289094+0000","last_fullsized":"2023-10-01T09:15:43.289094+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"121'6398","reported_seq":6422,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:15:58.027199+0000","last_change":"2023-10-01T09:14:31.349314+0000","last_active":"2023-10-01T09:15:58.027199+0000","last_peered":"2023-10-01T09:15:58.027199+0000","last_clean":"2023-10-01T09:15:58.027199+0000","last_became_active":"2023-10-01T09:14:31.349041+0000","last_became_peered":"2023-10-01T09:14:31.349041+0000","last_unstale":"2023-10-01T09:15:58.027199+0000","last_undegraded":"2023-10-01T09:15:58.027199+0000","last_fullsized":"2023-10-01T09:15:58.027199+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6398,"log_dups_size":0,"ondisk_log_size":6398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:20:18.137610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419299328,"num_objects":6398,"num_object_clones":0,"num_object_copies":19194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6398,"num_write_kb":409472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":87,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.286346+0000","last_change":"2023-10-01T09:12:10.003934+0000","last_active":"2023-10-01T09:15:43.286346+0000","last_peered":"2023-10-01T09:15:43.286346+0000","last_clean":"2023-10-01T09:15:43.286346+0000","last_became_active":"2023-10-01T09:12:10.003597+0000","last_became_peered":"2023-10-01T09:12:10.003597+0000","last_unstale":"2023-10-01T09:15:43.286346+0000","last_undegraded":"2023-10-01T09:15:43.286346+0000","last_fullsized":"2023-10-01T09:15:43.286346+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":100,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.289161+0000","last_change":"2023-10-01T09:12:40.946791+0000","last_active":"2023-10-01T09:15:43.289161+0000","last_peered":"2023-10-01T09:15:43.289161+0000","last_clean":"2023-10-01T09:15:43.289161+0000","last_became_active":"2023-10-01T09:12:40.946418+0000","last_became_peered":"2023-10-01T09:12:40.946418+0000","last_unstale":"2023-10-01T09:15:43.289161+0000","last_undegraded":"2023-10-01T09:15:43.289161+0000","last_fullsized":"2023-10-01T09:15:43.289161+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"121'6595","reported_seq":6618,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:16:01.200741+0000","last_change":"2023-10-01T09:14:31.355388+0000","last_active":"2023-10-01T09:16:01.200741+0000","last_peered":"2023-10-01T09:16:01.200741+0000","last_clean":"2023-10-01T09:16:01.200741+0000","last_became_active":"2023-10-01T09:14:31.355065+0000","last_became_peered":"2023-10-01T09:14:31.355065+0000","last_unstale":"2023-10-01T09:16:01.200741+0000","last_undegraded":"2023-10-01T09:16:01.200741+0000","last_fullsized":"2023-10-01T09:16:01.200741+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6595,"log_dups_size":0,"ondisk_log_size":6595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432209920,"num_objects":6595,"num_object_clones":0,"num_object_copies":19785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6595,"num_write_kb":422080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":86,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288551+0000","last_change":"2023-10-01T09:12:40.945099+0000","last_active":"2023-10-01T09:15:43.288551+0000","last_peered":"2023-10-01T09:15:43.288551+0000","last_clean":"2023-10-01T09:15:43.288551+0000","last_became_active":"2023-10-01T09:12:40.944712+0000","last_became_peered":"2023-10-01T09:12:40.944712+0000","last_unstale":"2023-10-01T09:15:43.288551+0000","last_undegraded":"2023-10-01T09:15:43.288551+0000","last_fullsized":"2023-10-01T09:15:43.288551+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":85,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.289236+0000","last_change":"2023-10-01T09:12:10.003847+0000","last_active":"2023-10-01T09:15:43.289236+0000","last_peered":"2023-10-01T09:15:43.289236+0000","last_clean":"2023-10-01T09:15:43.289236+0000","last_became_active":"2023-10-01T09:12:10.003525+0000","last_became_peered":"2023-10-01T09:12:10.003525+0000","last_unstale":"2023-10-01T09:15:43.289236+0000","last_undegraded":"2023-10-01T09:15:43.289236+0000","last_fullsized":"2023-10-01T09:15:43.289236+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"121'6478","reported_seq":6501,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:16:01.217241+0000","last_change":"2023-10-01T09:14:31.356833+0000","last_active":"2023-10-01T09:16:01.217241+0000","last_peered":"2023-10-01T09:16:01.217241+0000","last_clean":"2023-10-01T09:16:01.217241+0000","last_became_active":"2023-10-01T09:14:31.356520+0000","last_became_peered":"2023-10-01T09:14:31.356520+0000","last_unstale":"2023-10-01T09:16:01.217241+0000","last_undegraded":"2023-10-01T09:16:01.217241+0000","last_fullsized":"2023-10-01T09:16:01.217241+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6478,"log_dups_size":0,"ondisk_log_size":6478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424542208,"num_objects":6478,"num_object_clones":0,"num_object_copies":19434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6478,"num_write_kb":414592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":91,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.289267+0000","last_change":"2023-10-01T09:13:13.555563+0000","last_active":"2023-10-01T09:15:43.289267+0000","last_peered":"2023-10-01T09:15:43.289267+0000","last_clean":"2023-10-01T09:15:43.289267+0000","last_became_active":"2023-10-01T09:13:13.555070+0000","last_became_peered":"2023-10-01T09:13:13.555070+0000","last_unstale":"2023-10-01T09:15:43.289267+0000","last_undegraded":"2023-10-01T09:15:43.289267+0000","last_fullsized":"2023-10-01T09:15:43.289267+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:44.557168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":60,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.351349+0000","last_change":"2023-10-01T09:13:13.551757+0000","last_active":"2023-10-01T09:15:43.351349+0000","last_peered":"2023-10-01T09:15:43.351349+0000","last_clean":"2023-10-01T09:15:43.351349+0000","last_became_active":"2023-10-01T09:13:13.551483+0000","last_became_peered":"2023-10-01T09:13:13.551483+0000","last_unstale":"2023-10-01T09:15:43.351349+0000","last_undegraded":"2023-10-01T09:15:43.351349+0000","last_fullsized":"2023-10-01T09:15:43.351349+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:33:46.184050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":934,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.283626+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:15:43.283626+0000","last_peered":"2023-10-01T09:15:43.283626+0000","last_clean":"2023-10-01T09:15:43.283626+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:15:43.283626+0000","last_undegraded":"2023-10-01T09:15:43.283626+0000","last_fullsized":"2023-10-01T09:15:43.283626+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":102,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.281525+0000","last_change":"2023-10-01T09:13:13.551681+0000","last_active":"2023-10-01T09:15:43.281525+0000","last_peered":"2023-10-01T09:15:43.281525+0000","last_clean":"2023-10-01T09:15:43.281525+0000","last_became_active":"2023-10-01T09:13:13.551268+0000","last_became_peered":"2023-10-01T09:13:13.551268+0000","last_unstale":"2023-10-01T09:15:43.281525+0000","last_undegraded":"2023-10-01T09:15:43.281525+0000","last_fullsized":"2023-10-01T09:15:43.281525+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":49,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.284642+0000","last_change":"2023-10-01T09:12:10.001493+0000","last_active":"2023-10-01T09:15:43.284642+0000","last_peered":"2023-10-01T09:15:43.284642+0000","last_clean":"2023-10-01T09:15:43.284642+0000","last_became_active":"2023-10-01T09:12:10.000899+0000","last_became_peered":"2023-10-01T09:12:10.000899+0000","last_unstale":"2023-10-01T09:15:43.284642+0000","last_undegraded":"2023-10-01T09:15:43.284642+0000","last_fullsized":"2023-10-01T09:15:43.284642+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:47:54.011059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"121'6347","reported_seq":6371,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:15:56.870670+0000","last_change":"2023-10-01T09:14:31.357395+0000","last_active":"2023-10-01T09:15:56.870670+0000","last_peered":"2023-10-01T09:15:56.870670+0000","last_clean":"2023-10-01T09:15:56.870670+0000","last_became_active":"2023-10-01T09:14:31.357014+0000","last_became_peered":"2023-10-01T09:14:31.357014+0000","last_unstale":"2023-10-01T09:15:56.870670+0000","last_undegraded":"2023-10-01T09:15:56.870670+0000","last_fullsized":"2023-10-01T09:15:56.870670+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6347,"log_dups_size":0,"ondisk_log_size":6347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:29.461431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415956992,"num_objects":6347,"num_object_clones":0,"num_object_copies":19041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6347,"num_write_kb":406208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"121'6157","reported_seq":6181,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:15:56.909870+0000","last_change":"2023-10-01T09:14:31.349232+0000","last_active":"2023-10-01T09:15:56.909870+0000","last_peered":"2023-10-01T09:15:56.909870+0000","last_clean":"2023-10-01T09:15:56.909870+0000","last_became_active":"2023-10-01T09:14:31.348940+0000","last_became_peered":"2023-10-01T09:14:31.348940+0000","last_unstale":"2023-10-01T09:15:56.909870+0000","last_undegraded":"2023-10-01T09:15:56.909870+0000","last_fullsized":"2023-10-01T09:15:56.909870+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6157,"log_dups_size":0,"ondisk_log_size":6157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:17.358604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403505152,"num_objects":6157,"num_object_clones":0,"num_object_copies":18471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6157,"num_write_kb":394048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":821,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.280770+0000","last_change":"2023-10-01T09:12:09.997936+0000","last_active":"2023-10-01T09:15:43.280770+0000","last_peered":"2023-10-01T09:15:43.280770+0000","last_clean":"2023-10-01T09:15:43.280770+0000","last_became_active":"2023-10-01T09:12:09.997427+0000","last_became_peered":"2023-10-01T09:12:09.997427+0000","last_unstale":"2023-10-01T09:15:43.280770+0000","last_undegraded":"2023-10-01T09:15:43.280770+0000","last_fullsized":"2023-10-01T09:15:43.280770+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1219,"num_read_kb":1114,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":86,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288789+0000","last_change":"2023-10-01T09:10:58.799722+0000","last_active":"2023-10-01T09:15:43.288789+0000","last_peered":"2023-10-01T09:15:43.288789+0000","last_clean":"2023-10-01T09:15:43.288789+0000","last_became_active":"2023-10-01T09:10:58.799313+0000","last_became_peered":"2023-10-01T09:10:58.799313+0000","last_unstale":"2023-10-01T09:15:43.288789+0000","last_undegraded":"2023-10-01T09:15:43.288789+0000","last_fullsized":"2023-10-01T09:15:43.288789+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":79,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.283607+0000","last_change":"2023-10-01T09:13:13.549385+0000","last_active":"2023-10-01T09:15:43.283607+0000","last_peered":"2023-10-01T09:15:43.283607+0000","last_clean":"2023-10-01T09:15:43.283607+0000","last_became_active":"2023-10-01T09:13:13.549035+0000","last_became_peered":"2023-10-01T09:13:13.549035+0000","last_unstale":"2023-10-01T09:15:43.283607+0000","last_undegraded":"2023-10-01T09:15:43.283607+0000","last_fullsized":"2023-10-01T09:15:43.283607+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:52:41.408995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"121'6415","reported_seq":6439,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:15:58.961805+0000","last_change":"2023-10-01T09:14:31.357130+0000","last_active":"2023-10-01T09:15:58.961805+0000","last_peered":"2023-10-01T09:15:58.961805+0000","last_clean":"2023-10-01T09:15:58.961805+0000","last_became_active":"2023-10-01T09:14:31.356877+0000","last_became_peered":"2023-10-01T09:14:31.356877+0000","last_unstale":"2023-10-01T09:15:58.961805+0000","last_undegraded":"2023-10-01T09:15:58.961805+0000","last_fullsized":"2023-10-01T09:15:58.961805+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6415,"log_dups_size":0,"ondisk_log_size":6415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:32:02.594887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420413440,"num_objects":6415,"num_object_clones":0,"num_object_copies":19245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6415,"num_write_kb":410560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":392,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.283920+0000","last_change":"2023-10-01T09:13:13.551669+0000","last_active":"2023-10-01T09:15:43.283920+0000","last_peered":"2023-10-01T09:15:43.283920+0000","last_clean":"2023-10-01T09:15:43.283920+0000","last_became_active":"2023-10-01T09:13:13.550982+0000","last_became_peered":"2023-10-01T09:13:13.550982+0000","last_unstale":"2023-10-01T09:15:43.283920+0000","last_undegraded":"2023-10-01T09:15:43.283920+0000","last_fullsized":"2023-10-01T09:15:43.283920+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:21:09.879713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":64,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.283383+0000","last_change":"2023-10-01T09:12:40.946267+0000","last_active":"2023-10-01T09:15:43.283383+0000","last_peered":"2023-10-01T09:15:43.283383+0000","last_clean":"2023-10-01T09:15:43.283383+0000","last_became_active":"2023-10-01T09:12:40.945970+0000","last_became_peered":"2023-10-01T09:12:40.945970+0000","last_unstale":"2023-10-01T09:15:43.283383+0000","last_undegraded":"2023-10-01T09:15:43.283383+0000","last_fullsized":"2023-10-01T09:15:43.283383+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:42.904509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":94,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.283745+0000","last_change":"2023-10-01T09:12:09.994071+0000","last_active":"2023-10-01T09:15:43.283745+0000","last_peered":"2023-10-01T09:15:43.283745+0000","last_clean":"2023-10-01T09:15:43.283745+0000","last_became_active":"2023-10-01T09:12:09.993782+0000","last_became_peered":"2023-10-01T09:12:09.993782+0000","last_unstale":"2023-10-01T09:15:43.283745+0000","last_undegraded":"2023-10-01T09:15:43.283745+0000","last_fullsized":"2023-10-01T09:15:43.283745+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:12:49.880685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":937,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.281336+0000","last_change":"2023-10-01T09:12:40.946131+0000","last_active":"2023-10-01T09:15:43.281336+0000","last_peered":"2023-10-01T09:15:43.281336+0000","last_clean":"2023-10-01T09:15:43.281336+0000","last_became_active":"2023-10-01T09:12:40.945524+0000","last_became_peered":"2023-10-01T09:12:40.945524+0000","last_unstale":"2023-10-01T09:15:43.281336+0000","last_undegraded":"2023-10-01T09:15:43.281336+0000","last_fullsized":"2023-10-01T09:15:43.281336+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":323,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.292250+0000","last_change":"2023-10-01T09:10:58.798697+0000","last_active":"2023-10-01T09:15:43.292250+0000","last_peered":"2023-10-01T09:15:43.292250+0000","last_clean":"2023-10-01T09:15:43.292250+0000","last_became_active":"2023-10-01T09:10:58.798203+0000","last_became_peered":"2023-10-01T09:10:58.798203+0000","last_unstale":"2023-10-01T09:15:43.292250+0000","last_undegraded":"2023-10-01T09:15:43.292250+0000","last_fullsized":"2023-10-01T09:15:43.292250+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":80,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288264+0000","last_change":"2023-10-01T09:13:13.556049+0000","last_active":"2023-10-01T09:15:43.288264+0000","last_peered":"2023-10-01T09:15:43.288264+0000","last_clean":"2023-10-01T09:15:43.288264+0000","last_became_active":"2023-10-01T09:13:13.555739+0000","last_became_peered":"2023-10-01T09:13:13.555739+0000","last_unstale":"2023-10-01T09:15:43.288264+0000","last_undegraded":"2023-10-01T09:15:43.288264+0000","last_fullsized":"2023-10-01T09:15:43.288264+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:50:34.475690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":71,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.286191+0000","last_change":"2023-10-01T09:03:03.227254+0000","last_active":"2023-10-01T09:15:43.286191+0000","last_peered":"2023-10-01T09:15:43.286191+0000","last_clean":"2023-10-01T09:15:43.286191+0000","last_became_active":"2023-10-01T09:03:03.226941+0000","last_became_peered":"2023-10-01T09:03:03.226941+0000","last_unstale":"2023-10-01T09:15:43.286191+0000","last_undegraded":"2023-10-01T09:15:43.286191+0000","last_fullsized":"2023-10-01T09:15:43.286191+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"121'6408","reported_seq":6431,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:16:01.182781+0000","last_change":"2023-10-01T09:14:31.353858+0000","last_active":"2023-10-01T09:16:01.182781+0000","last_peered":"2023-10-01T09:16:01.182781+0000","last_clean":"2023-10-01T09:16:01.182781+0000","last_became_active":"2023-10-01T09:14:31.353572+0000","last_became_peered":"2023-10-01T09:14:31.353572+0000","last_unstale":"2023-10-01T09:16:01.182781+0000","last_undegraded":"2023-10-01T09:16:01.182781+0000","last_fullsized":"2023-10-01T09:16:01.182781+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419954688,"num_objects":6408,"num_object_clones":0,"num_object_copies":19224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6408,"num_write_kb":410112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"121'6524","reported_seq":6547,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:16:01.227635+0000","last_change":"2023-10-01T09:14:31.354366+0000","last_active":"2023-10-01T09:16:01.227635+0000","last_peered":"2023-10-01T09:16:01.227635+0000","last_clean":"2023-10-01T09:16:01.227635+0000","last_became_active":"2023-10-01T09:14:31.354059+0000","last_became_peered":"2023-10-01T09:14:31.354059+0000","last_unstale":"2023-10-01T09:16:01.227635+0000","last_undegraded":"2023-10-01T09:16:01.227635+0000","last_fullsized":"2023-10-01T09:16:01.227635+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6524,"log_dups_size":0,"ondisk_log_size":6524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427556864,"num_objects":6524,"num_object_clones":0,"num_object_copies":19572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6524,"num_write_kb":417536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":217,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.292378+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:15:43.292378+0000","last_peered":"2023-10-01T09:15:43.292378+0000","last_clean":"2023-10-01T09:15:43.292378+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:15:43.292378+0000","last_undegraded":"2023-10-01T09:15:43.292378+0000","last_fullsized":"2023-10-01T09:15:43.292378+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":69,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.294907+0000","last_change":"2023-10-01T09:13:13.552391+0000","last_active":"2023-10-01T09:15:43.294907+0000","last_peered":"2023-10-01T09:15:43.294907+0000","last_clean":"2023-10-01T09:15:43.294907+0000","last_became_active":"2023-10-01T09:13:13.552082+0000","last_became_peered":"2023-10-01T09:13:13.552082+0000","last_unstale":"2023-10-01T09:15:43.294907+0000","last_undegraded":"2023-10-01T09:15:43.294907+0000","last_fullsized":"2023-10-01T09:15:43.294907+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:07:19.929732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":84,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.331099+0000","last_change":"2023-10-01T09:12:40.944991+0000","last_active":"2023-10-01T09:15:43.331099+0000","last_peered":"2023-10-01T09:15:43.331099+0000","last_clean":"2023-10-01T09:15:43.331099+0000","last_became_active":"2023-10-01T09:12:40.944688+0000","last_became_peered":"2023-10-01T09:12:40.944688+0000","last_unstale":"2023-10-01T09:15:43.331099+0000","last_undegraded":"2023-10-01T09:15:43.331099+0000","last_fullsized":"2023-10-01T09:15:43.331099+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:30.015801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":481,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.292773+0000","last_change":"2023-10-01T09:10:58.797964+0000","last_active":"2023-10-01T09:15:43.292773+0000","last_peered":"2023-10-01T09:15:43.292773+0000","last_clean":"2023-10-01T09:15:43.292773+0000","last_became_active":"2023-10-01T09:10:58.797447+0000","last_became_peered":"2023-10-01T09:10:58.797447+0000","last_unstale":"2023-10-01T09:15:43.292773+0000","last_undegraded":"2023-10-01T09:15:43.292773+0000","last_fullsized":"2023-10-01T09:15:43.292773+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":64,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.283682+0000","last_change":"2023-10-01T09:12:40.948216+0000","last_active":"2023-10-01T09:15:43.283682+0000","last_peered":"2023-10-01T09:15:43.283682+0000","last_clean":"2023-10-01T09:15:43.283682+0000","last_became_active":"2023-10-01T09:12:40.947879+0000","last_became_peered":"2023-10-01T09:12:40.947879+0000","last_unstale":"2023-10-01T09:15:43.283682+0000","last_undegraded":"2023-10-01T09:15:43.283682+0000","last_fullsized":"2023-10-01T09:15:43.283682+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:46:40.612337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.330790+0000","last_change":"2023-10-01T09:10:58.797034+0000","last_active":"2023-10-01T09:15:43.330790+0000","last_peered":"2023-10-01T09:15:43.330790+0000","last_clean":"2023-10-01T09:15:43.330790+0000","last_became_active":"2023-10-01T09:10:58.796730+0000","last_became_peered":"2023-10-01T09:10:58.796730+0000","last_unstale":"2023-10-01T09:15:43.330790+0000","last_undegraded":"2023-10-01T09:15:43.330790+0000","last_fullsized":"2023-10-01T09:15:43.330790+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:32:15.982890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"121'6515","reported_seq":6538,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:16:01.187824+0000","last_change":"2023-10-01T09:14:31.353560+0000","last_active":"2023-10-01T09:16:01.187824+0000","last_peered":"2023-10-01T09:16:01.187824+0000","last_clean":"2023-10-01T09:16:01.187824+0000","last_became_active":"2023-10-01T09:14:31.353242+0000","last_became_peered":"2023-10-01T09:14:31.353242+0000","last_unstale":"2023-10-01T09:16:01.187824+0000","last_undegraded":"2023-10-01T09:16:01.187824+0000","last_fullsized":"2023-10-01T09:16:01.187824+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6515,"log_dups_size":0,"ondisk_log_size":6515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426967040,"num_objects":6515,"num_object_clones":0,"num_object_copies":19545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6515,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":88,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288957+0000","last_change":"2023-10-01T09:13:13.553580+0000","last_active":"2023-10-01T09:15:43.288957+0000","last_peered":"2023-10-01T09:15:43.288957+0000","last_clean":"2023-10-01T09:15:43.288957+0000","last_became_active":"2023-10-01T09:13:13.553011+0000","last_became_peered":"2023-10-01T09:13:13.553011+0000","last_unstale":"2023-10-01T09:15:43.288957+0000","last_undegraded":"2023-10-01T09:15:43.288957+0000","last_fullsized":"2023-10-01T09:15:43.288957+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:54:30.369308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":508,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.283899+0000","last_change":"2023-10-01T09:12:10.002334+0000","last_active":"2023-10-01T09:15:43.283899+0000","last_peered":"2023-10-01T09:15:43.283899+0000","last_clean":"2023-10-01T09:15:43.283899+0000","last_became_active":"2023-10-01T09:12:10.001791+0000","last_became_peered":"2023-10-01T09:12:10.001791+0000","last_unstale":"2023-10-01T09:15:43.283899+0000","last_undegraded":"2023-10-01T09:15:43.283899+0000","last_fullsized":"2023-10-01T09:15:43.283899+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:24:27.632780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":85,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.289173+0000","last_change":"2023-10-01T09:13:13.552192+0000","last_active":"2023-10-01T09:15:43.289173+0000","last_peered":"2023-10-01T09:15:43.289173+0000","last_clean":"2023-10-01T09:15:43.289173+0000","last_became_active":"2023-10-01T09:13:13.551887+0000","last_became_peered":"2023-10-01T09:13:13.551887+0000","last_unstale":"2023-10-01T09:15:43.289173+0000","last_undegraded":"2023-10-01T09:15:43.289173+0000","last_fullsized":"2023-10-01T09:15:43.289173+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:23.633557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"121'6687","reported_seq":6710,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:16:01.272159+0000","last_change":"2023-10-01T09:14:31.350884+0000","last_active":"2023-10-01T09:16:01.272159+0000","last_peered":"2023-10-01T09:16:01.272159+0000","last_clean":"2023-10-01T09:16:01.272159+0000","last_became_active":"2023-10-01T09:14:31.350533+0000","last_became_peered":"2023-10-01T09:14:31.350533+0000","last_unstale":"2023-10-01T09:16:01.272159+0000","last_undegraded":"2023-10-01T09:16:01.272159+0000","last_fullsized":"2023-10-01T09:16:01.272159+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6687,"log_dups_size":0,"ondisk_log_size":6687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438239232,"num_objects":6687,"num_object_clones":0,"num_object_copies":20061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6687,"num_write_kb":427968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":159,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.287695+0000","last_change":"2023-10-01T09:12:40.949544+0000","last_active":"2023-10-01T09:15:43.287695+0000","last_peered":"2023-10-01T09:15:43.287695+0000","last_clean":"2023-10-01T09:15:43.287695+0000","last_became_active":"2023-10-01T09:12:40.949054+0000","last_became_peered":"2023-10-01T09:12:40.949054+0000","last_unstale":"2023-10-01T09:15:43.287695+0000","last_undegraded":"2023-10-01T09:15:43.287695+0000","last_fullsized":"2023-10-01T09:15:43.287695+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":73,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.331112+0000","last_change":"2023-10-01T09:12:10.003174+0000","last_active":"2023-10-01T09:15:43.331112+0000","last_peered":"2023-10-01T09:15:43.331112+0000","last_clean":"2023-10-01T09:15:43.331112+0000","last_became_active":"2023-10-01T09:12:10.002808+0000","last_became_peered":"2023-10-01T09:12:10.002808+0000","last_unstale":"2023-10-01T09:15:43.331112+0000","last_undegraded":"2023-10-01T09:15:43.331112+0000","last_fullsized":"2023-10-01T09:15:43.331112+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:25.037091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":70,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288631+0000","last_change":"2023-10-01T09:03:03.754839+0000","last_active":"2023-10-01T09:15:43.288631+0000","last_peered":"2023-10-01T09:15:43.288631+0000","last_clean":"2023-10-01T09:15:43.288631+0000","last_became_active":"2023-10-01T09:03:03.754506+0000","last_became_peered":"2023-10-01T09:03:03.754506+0000","last_unstale":"2023-10-01T09:15:43.288631+0000","last_undegraded":"2023-10-01T09:15:43.288631+0000","last_fullsized":"2023-10-01T09:15:43.288631+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"121'6192","reported_seq":6216,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:15:56.588935+0000","last_change":"2023-10-01T09:14:31.352926+0000","last_active":"2023-10-01T09:15:56.588935+0000","last_peered":"2023-10-01T09:15:56.588935+0000","last_clean":"2023-10-01T09:15:56.588935+0000","last_became_active":"2023-10-01T09:14:31.352602+0000","last_became_peered":"2023-10-01T09:14:31.352602+0000","last_unstale":"2023-10-01T09:15:56.588935+0000","last_undegraded":"2023-10-01T09:15:56.588935+0000","last_fullsized":"2023-10-01T09:15:56.588935+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6192,"log_dups_size":0,"ondisk_log_size":6192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:40:47.614933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405798912,"num_objects":6192,"num_object_clones":0,"num_object_copies":18576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6192,"num_write_kb":396288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":58,"reported_epoch":121,"state":"active+clean","last_fresh":"2023-10-01T09:15:55.448393+0000","last_change":"2023-10-01T09:10:58.801161+0000","last_active":"2023-10-01T09:15:55.448393+0000","last_peered":"2023-10-01T09:15:55.448393+0000","last_clean":"2023-10-01T09:15:55.448393+0000","last_became_active":"2023-10-01T09:10:58.800794+0000","last_became_peered":"2023-10-01T09:10:58.800794+0000","last_unstale":"2023-10-01T09:15:55.448393+0000","last_undegraded":"2023-10-01T09:15:55.448393+0000","last_fullsized":"2023-10-01T09:15:55.448393+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:35.019445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":110,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.351007+0000","last_change":"2023-10-01T09:13:13.548647+0000","last_active":"2023-10-01T09:15:43.351007+0000","last_peered":"2023-10-01T09:15:43.351007+0000","last_clean":"2023-10-01T09:15:43.351007+0000","last_became_active":"2023-10-01T09:13:13.548382+0000","last_became_peered":"2023-10-01T09:13:13.548382+0000","last_unstale":"2023-10-01T09:15:43.351007+0000","last_undegraded":"2023-10-01T09:15:43.351007+0000","last_fullsized":"2023-10-01T09:15:43.351007+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:17:31.791258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":89,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.280865+0000","last_change":"2023-10-01T09:12:10.000449+0000","last_active":"2023-10-01T09:15:43.280865+0000","last_peered":"2023-10-01T09:15:43.280865+0000","last_clean":"2023-10-01T09:15:43.280865+0000","last_became_active":"2023-10-01T09:12:10.000000+0000","last_became_peered":"2023-10-01T09:12:10.000000+0000","last_unstale":"2023-10-01T09:15:43.280865+0000","last_undegraded":"2023-10-01T09:15:43.280865+0000","last_fullsized":"2023-10-01T09:15:43.280865+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":100,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288943+0000","last_change":"2023-10-01T09:13:13.555744+0000","last_active":"2023-10-01T09:15:43.288943+0000","last_peered":"2023-10-01T09:15:43.288943+0000","last_clean":"2023-10-01T09:15:43.288943+0000","last_became_active":"2023-10-01T09:13:13.555163+0000","last_became_peered":"2023-10-01T09:13:13.555163+0000","last_unstale":"2023-10-01T09:15:43.288943+0000","last_undegraded":"2023-10-01T09:15:43.288943+0000","last_fullsized":"2023-10-01T09:15:43.288943+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":74,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.281618+0000","last_change":"2023-10-01T09:03:03.202063+0000","last_active":"2023-10-01T09:15:43.281618+0000","last_peered":"2023-10-01T09:15:43.281618+0000","last_clean":"2023-10-01T09:15:43.281618+0000","last_became_active":"2023-10-01T09:03:03.201769+0000","last_became_peered":"2023-10-01T09:03:03.201769+0000","last_unstale":"2023-10-01T09:15:43.281618+0000","last_undegraded":"2023-10-01T09:15:43.281618+0000","last_fullsized":"2023-10-01T09:15:43.281618+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":100,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288726+0000","last_change":"2023-10-01T09:13:13.554959+0000","last_active":"2023-10-01T09:15:43.288726+0000","last_peered":"2023-10-01T09:15:43.288726+0000","last_clean":"2023-10-01T09:15:43.288726+0000","last_became_active":"2023-10-01T09:13:13.554589+0000","last_became_peered":"2023-10-01T09:13:13.554589+0000","last_unstale":"2023-10-01T09:15:43.288726+0000","last_undegraded":"2023-10-01T09:15:43.288726+0000","last_fullsized":"2023-10-01T09:15:43.288726+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":96,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288776+0000","last_change":"2023-10-01T09:13:13.556295+0000","last_active":"2023-10-01T09:15:43.288776+0000","last_peered":"2023-10-01T09:15:43.288776+0000","last_clean":"2023-10-01T09:15:43.288776+0000","last_became_active":"2023-10-01T09:13:13.555953+0000","last_became_peered":"2023-10-01T09:13:13.555953+0000","last_unstale":"2023-10-01T09:15:43.288776+0000","last_undegraded":"2023-10-01T09:15:43.288776+0000","last_fullsized":"2023-10-01T09:15:43.288776+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:04:06.892156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":109,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.288146+0000","last_change":"2023-10-01T09:13:13.549800+0000","last_active":"2023-10-01T09:15:43.288146+0000","last_peered":"2023-10-01T09:15:43.288146+0000","last_clean":"2023-10-01T09:15:43.288146+0000","last_became_active":"2023-10-01T09:13:13.549480+0000","last_became_peered":"2023-10-01T09:13:13.549480+0000","last_unstale":"2023-10-01T09:15:43.288146+0000","last_undegraded":"2023-10-01T09:15:43.288146+0000","last_fullsized":"2023-10-01T09:15:43.288146+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:08.673916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":73,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.333187+0000","last_change":"2023-10-01T09:12:40.944944+0000","last_active":"2023-10-01T09:15:43.333187+0000","last_peered":"2023-10-01T09:15:43.333187+0000","last_clean":"2023-10-01T09:15:43.333187+0000","last_became_active":"2023-10-01T09:12:40.944613+0000","last_became_peered":"2023-10-01T09:12:40.944613+0000","last_unstale":"2023-10-01T09:15:43.333187+0000","last_undegraded":"2023-10-01T09:15:43.333187+0000","last_fullsized":"2023-10-01T09:15:43.333187+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:48:20.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":85,"reported_epoch":120,"state":"active+clean","last_fresh":"2023-10-01T09:15:43.292608+0000","last_change":"2023-10-01T09:12:10.003833+0000","last_active":"2023-10-01T09:15:43.292608+0000","last_peered":"2023-10-01T09:15:43.292608+0000","last_clean":"2023-10-01T09:15:43.292608+0000","last_became_active":"2023-10-01T09:12:10.003482+0000","last_became_peered":"2023-10-01T09:12:10.003482+0000","last_unstale":"2023-10-01T09:15:43.292608+0000","last_undegraded":"2023-10-01T09:15:43.292608+0000","last_fullsized":"2023-10-01T09:15:43.292608+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":6754926592,"num_objects":103072,"num_object_clones":0,"num_object_copies":309216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":103072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":103072,"num_write_kb":6596608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20288372736,"data_stored":20288372736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103072,"ondisk_log_size":103072,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27349,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12188,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430385,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2390844,"kb_used_data":2093620,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91354308,"statfs":{"total":95995035648,"available":93546811392,"internally_reserved":0,"allocated":2143866880,"data_stored":2143350509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:03 2023","interfaces":[{"interface":"back","average":{"1min":5.5339999999999998,"5min":1.556,"15min":0.89900000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":94.162999999999997,"5min":94.162999999999997,"15min":94.162999999999997},"last":2.4220000000000002},{"interface":"front","average":{"1min":5.5300000000000002,"5min":1.542,"15min":0.879},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":94.135000000000005,"5min":94.135000000000005,"15min":94.135000000000005},"last":1.006}]},{"osd":1,"last update":"Sun Oct 1 09:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3080000000000001,"5min":0.71399999999999997,"15min":0.61399999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.868,"5min":12.868,"15min":12.868},"last":1.1699999999999999},{"interface":"front","average":{"1min":1.4059999999999999,"5min":0.71999999999999997,"15min":0.60899999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":13.106,"5min":13.106,"15min":13.106},"last":1.03}]},{"osd":2,"last update":"Sun Oct 1 09:15:45 2023","interfaces":[{"interface":"back","average":{"1min":4.8049999999999997,"5min":1.395,"15min":0.83099999999999996},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":82.379000000000005,"5min":82.379000000000005,"15min":82.379000000000005},"last":1.2929999999999999},{"interface":"front","average":{"1min":5.2199999999999998,"5min":1.48,"15min":0.85199999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":85.596000000000004,"5min":85.596000000000004,"15min":85.596000000000004},"last":0.98399999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:15:15 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.59799999999999998,"15min":0.57799999999999996},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.2069999999999999,"5min":2.2069999999999999,"15min":2.2069999999999999},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.56000000000000005,"15min":0.55100000000000005},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.6140000000000001,"5min":1.6140000000000001,"15min":1.6140000000000001},"last":0.44600000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:15:00 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47799999999999998,"15min":0.47699999999999998},"min":{"1min":0.29699999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.71099999999999997,"5min":0.77700000000000002,"15min":0.79100000000000004},"last":2.4580000000000002},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52300000000000002,"15min":0.55200000000000005},"min":{"1min":0.34100000000000003,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":5.5369999999999999},"last":1.647}]},{"osd":5,"last update":"Sun Oct 1 09:15:00 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.30499999999999999,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":1.5600000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":2.0800000000000001},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.58699999999999997,"15min":0.55700000000000005},"min":{"1min":0.39000000000000001,"5min":0.35099999999999998,"15min":0.28599999999999998},"max":{"1min":3.2839999999999998,"5min":3.2839999999999998,"15min":3.2839999999999998},"last":1.206}]},{"osd":6,"last update":"Sun Oct 1 09:15:00 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.56999999999999995,"15min":0.56100000000000005},"min":{"1min":0.35699999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.85499999999999998,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":1.2310000000000001},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.60099999999999998,"15min":0.62},"min":{"1min":0.35199999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.655,"5min":1.655,"15min":5.5700000000000003},"last":1.6799999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626614,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1989484,"kb_used_data":1679972,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91755668,"statfs":{"total":95995035648,"available":93957804032,"internally_reserved":0,"allocated":1720291328,"data_stored":1719790584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.61899999999999999,"15min":0.624},"min":{"1min":0.41499999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.7749999999999999,"5min":3.7749999999999999,"15min":3.7749999999999999},"last":3.4990000000000001},{"interface":"front","average":{"1min":0.78800000000000003,"5min":0.61099999999999999,"15min":0.623},"min":{"1min":0.40200000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.1259999999999999,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.75600000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:15:20 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.624,"15min":0.59199999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.2730000000000001,"5min":3.2730000000000001,"15min":3.2730000000000001},"last":0.23999999999999999},{"interface":"front","average":{"1min":1.2470000000000001,"5min":0.72699999999999998,"15min":0.64300000000000002},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":3.9830000000000001,"5min":3.9830000000000001,"15min":3.9830000000000001},"last":0.72399999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:15:44 2023","interfaces":[{"interface":"back","average":{"1min":11.718,"5min":2.7749999999999999,"15min":1.2769999999999999},"min":{"1min":0.41599999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":82.978999999999999,"5min":82.978999999999999,"15min":82.978999999999999},"last":20.329999999999998},{"interface":"front","average":{"1min":11.433999999999999,"5min":2.7210000000000001,"15min":1.2549999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":82.747,"5min":82.747,"15min":82.747},"last":19.449999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:15:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.64400000000000002,"15min":0.59099999999999997},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":3.3109999999999999,"5min":3.3109999999999999,"15min":3.3109999999999999},"last":0.27000000000000002},{"interface":"front","average":{"1min":1.4019999999999999,"5min":0.72999999999999998,"15min":0.61799999999999999},"min":{"1min":0.41899999999999998,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":5.0359999999999996,"5min":5.0359999999999996,"15min":5.0359999999999996},"last":0.77500000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:15:42 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.57499999999999996,"15min":0.502},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.25900000000000001},"max":{"1min":4.0099999999999998,"5min":4.0099999999999998,"15min":4.0099999999999998},"last":0.95799999999999996},{"interface":"front","average":{"1min":1.0669999999999999,"5min":0.64100000000000001,"15min":0.55400000000000005},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.1920000000000002,"5min":5.1920000000000002,"15min":5.1920000000000002},"last":0.38100000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:15:42 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":0.63,"15min":0.54800000000000004},"min":{"1min":0.34599999999999997,"5min":0.32200000000000001,"15min":0.251},"max":{"1min":4.0990000000000002,"5min":4.0990000000000002,"15min":4.0990000000000002},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.1519999999999999,"5min":0.69799999999999995,"15min":0.59499999999999997},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":5.1559999999999997,"5min":5.1559999999999997,"15min":5.1559999999999997},"last":0.33100000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:15:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":0.69699999999999995,"15min":0.59699999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.29899999999999999},"max":{"1min":5.2640000000000002,"5min":5.2640000000000002,"15min":5.2640000000000002},"last":0.98299999999999998},{"interface":"front","average":{"1min":1.1299999999999999,"5min":0.73399999999999999,"15min":0.63300000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.4710000000000001,"5min":6.4710000000000001,"15min":6.4710000000000001},"last":0.35899999999999999}]}]},{"osd":1,"up_from":94,"seq":403726925872,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2106880,"kb_used_data":2030904,"kb_used_omap":7,"kb_used_meta":75960,"kb_avail":91638272,"statfs":{"total":95995035648,"available":93837590528,"internally_reserved":0,"allocated":2079645696,"data_stored":2079067707,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":77783463},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:11 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.38500000000000001,"15min":0.35899999999999999},"min":{"1min":0.20399999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.8490000000000002,"5min":2.8490000000000002,"15min":2.8490000000000002},"last":1.7849999999999999},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.44400000000000001,"15min":0.39200000000000002},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":5.2549999999999999,"5min":5.2549999999999999,"15min":5.2549999999999999},"last":2.3690000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":0.69099999999999995,"15min":0.45200000000000001},"min":{"1min":0.24299999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":13.9,"5min":13.9,"15min":13.9},"last":1.8220000000000001},{"interface":"front","average":{"1min":2.3839999999999999,"5min":0.75600000000000001,"15min":0.47599999999999998},"min":{"1min":0.216,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":8.2089999999999996,"5min":8.2089999999999996,"15min":8.2089999999999996},"last":8.8770000000000007}]},{"osd":3,"last update":"Sun Oct 1 09:15:18 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.49199999999999999,"15min":0.41199999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":5.3029999999999999,"5min":5.3029999999999999,"15min":5.3029999999999999},"last":1.9059999999999999},{"interface":"front","average":{"1min":1.75,"5min":0.65600000000000003,"15min":0.47399999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":13.853,"5min":13.853,"15min":13.853},"last":1.9490000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":0.65800000000000003,"15min":0.58099999999999996},"min":{"1min":0.40000000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":3.698,"5min":3.698,"15min":3.698},"last":3.9649999999999999},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.63800000000000001,"15min":0.58599999999999997},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.3550000000000004,"5min":6.3550000000000004,"15min":6.3550000000000004},"last":3.5339999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.329,"5min":0.71299999999999997,"15min":0.60699999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.3490000000000002,"5min":5.3490000000000002,"15min":5.3490000000000002},"last":1.839},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.67100000000000004,"15min":0.622},"min":{"1min":0.46600000000000003,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":3.673,"5min":3.673,"15min":3.673},"last":7.2309999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:15:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":0.69299999999999995,"15min":0.63100000000000001},"min":{"1min":0.45100000000000001,"5min":0.374,"15min":0.374},"max":{"1min":5.3609999999999998,"5min":5.3609999999999998,"15min":5.3609999999999998},"last":1.8819999999999999},{"interface":"front","average":{"1min":1.018,"5min":0.70399999999999996,"15min":0.65600000000000003},"min":{"1min":0.44,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":3.6309999999999998,"5min":3.6309999999999998,"15min":3.6309999999999998},"last":3.9340000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:15:11 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":0.67200000000000004,"15min":0.625},"min":{"1min":0.43099999999999999,"5min":0.376,"15min":0.376},"max":{"1min":3.7170000000000001,"5min":3.7170000000000001,"15min":3.7170000000000001},"last":7.2770000000000001},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.64300000000000002,"15min":0.61799999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.976,"5min":2.976,"15min":2.976},"last":10.210000000000001}]}]},{"osd":0,"up_from":89,"seq":382252089406,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3305424,"kb_used_data":3211400,"kb_used_omap":8,"kb_used_meta":94007,"kb_avail":90439728,"statfs":{"total":95995035648,"available":92610281472,"internally_reserved":0,"allocated":3288473600,"data_stored":3287884058,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":96264130},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1499999999999999,"5min":0.48999999999999999,"15min":0.378},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":8.641,"5min":8.641,"15min":8.641},"last":9.6669999999999998},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.42499999999999999,"15min":0.39000000000000001},"min":{"1min":0.23499999999999999,"5min":0.219,"15min":0.219},"max":{"1min":2.4089999999999998,"5min":2.4089999999999998,"15min":2.4089999999999998},"last":2.173}]},{"osd":2,"last update":"Sun Oct 1 09:15:49 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":0.72899999999999998,"15min":0.503},"min":{"1min":0.248,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.7539999999999996,"5min":8.7539999999999996,"15min":8.7539999999999996},"last":12.944000000000001},{"interface":"front","average":{"1min":1.389,"5min":0.59499999999999997,"15min":0.46899999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.9159999999999999,"5min":3.9159999999999999,"15min":3.9159999999999999},"last":2.0920000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:15:15 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":0.53200000000000003,"15min":0.42799999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.5660000000000007,"5min":8.5660000000000007,"15min":8.5660000000000007},"last":12.906000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.41099999999999998,"15min":0.38100000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":2.056,"5min":2.056,"15min":2.056},"last":5.7430000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.64000000000000001,"15min":0.64100000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.768,"5min":1.768,"15min":1.768},"last":2.02},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64300000000000002,"15min":0.66600000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.8320000000000001,"5min":1.8320000000000001,"15min":1.8320000000000001},"last":5.6669999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.66500000000000004,"15min":0.68100000000000005},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.786,"5min":1.786,"15min":1.786},"last":5.7069999999999999},{"interface":"front","average":{"1min":1.3480000000000001,"5min":0.78900000000000003,"15min":0.72099999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.7230000000000008,"5min":8.7230000000000008,"15min":8.7230000000000008},"last":12.779}]},{"osd":6,"last update":"Sun Oct 1 09:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.69499999999999995,"15min":0.69699999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.8889999999999998,"5min":3.8889999999999998,"15min":3.8889999999999998},"last":5.7229999999999999},{"interface":"front","average":{"1min":1.226,"5min":0.77000000000000002,"15min":0.71399999999999997},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":8.6910000000000007,"5min":8.6910000000000007,"15min":8.6910000000000007},"last":12.875}]},{"osd":7,"last update":"Sun Oct 1 09:15:08 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":0.79700000000000004,"15min":0.73699999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":8.7850000000000001,"5min":8.7850000000000001,"15min":8.7850000000000001},"last":12.840999999999999},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.71399999999999997,"15min":0.70599999999999996},"min":{"1min":0.47799999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":3.0830000000000002,"5min":3.0830000000000002,"15min":3.0830000000000002},"last":2.0600000000000001}]}]},{"osd":2,"up_from":99,"seq":425201762346,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2530680,"kb_used_data":2458864,"kb_used_omap":11,"kb_used_meta":71796,"kb_avail":91214472,"statfs":{"total":95995035648,"available":93403619328,"internally_reserved":0,"allocated":2517876736,"data_stored":2517305834,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":73519991},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":5.266,"5min":1.3260000000000001,"15min":0.66000000000000003},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":85.838999999999999,"5min":85.838999999999999,"15min":85.838999999999999},"last":0.224},{"interface":"front","average":{"1min":6.601,"5min":1.605,"15min":0.76300000000000001},"min":{"1min":0.35999999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":85.938000000000002,"5min":85.938000000000002,"15min":85.938000000000002},"last":0.33700000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":6.6360000000000001,"5min":1.6240000000000001,"15min":0.78500000000000003},"min":{"1min":0.21299999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":105.3,"5min":105.3,"15min":105.3},"last":0.25600000000000001},{"interface":"front","average":{"1min":7.7619999999999996,"5min":1.859,"15min":0.86699999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":107.729,"5min":107.729,"15min":107.729},"last":0.439}]},{"osd":3,"last update":"Sun Oct 1 09:15:15 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.42599999999999999,"15min":0.38800000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":2.4660000000000002,"5min":2.4660000000000002,"15min":2.4660000000000002},"last":0.21299999999999999},{"interface":"front","average":{"1min":0.79800000000000004,"5min":0.48899999999999999,"15min":0.437},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.3209999999999997,"5min":4.3209999999999997,"15min":4.3209999999999997},"last":0.32500000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":0.747,"15min":0.65100000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.5170000000000003,"5min":6.5170000000000003,"15min":6.5170000000000003},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.7290000000000001,"5min":0.79800000000000004,"15min":0.63900000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.1689999999999996,"5min":6.1689999999999996,"15min":6.1689999999999996},"last":0.30299999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":0.89600000000000002,"15min":0.68200000000000005},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.2999999999999998,"5min":7.2999999999999998,"15min":7.2999999999999998},"last":1.4330000000000001},{"interface":"front","average":{"1min":1.798,"5min":0.82699999999999996,"15min":0.65800000000000003},"min":{"1min":0.377,"5min":0.317,"15min":0.317},"max":{"1min":5.1959999999999997,"5min":5.1959999999999997,"15min":5.1959999999999997},"last":1.6020000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":0.78500000000000003,"15min":0.66200000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.9699999999999998,"5min":5.9699999999999998,"15min":5.9699999999999998},"last":0.79800000000000004},{"interface":"front","average":{"1min":1.026,"5min":0.71099999999999997,"15min":0.65200000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.9580000000000002,"5min":5.9580000000000002,"15min":5.9580000000000002},"last":0.79000000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":0.78100000000000003,"15min":0.66100000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.1970000000000001,"5min":6.1970000000000001,"15min":6.1970000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.1110000000000002,"5min":0.92200000000000004,"15min":0.71599999999999997},"min":{"1min":0.40100000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.5510000000000002,"5min":7.5510000000000002,"15min":7.5510000000000002},"last":0.61099999999999999}]}]},{"osd":3,"up_from":104,"seq":446676598819,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1242484,"kb_used_data":1192940,"kb_used_omap":3,"kb_used_meta":49532,"kb_avail":92502668,"statfs":{"total":95995035648,"available":94722732032,"internally_reserved":0,"allocated":1221570560,"data_stored":1221002327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":50721511},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":17.649000000000001,"5min":3.8220000000000001,"15min":1.518},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":199.422,"5min":199.422,"15min":199.422},"last":0.879},{"interface":"front","average":{"1min":17.515999999999998,"5min":3.7839999999999998,"15min":1.4950000000000001},"min":{"1min":0.23799999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":199.91200000000001,"5min":199.91200000000001,"15min":199.91200000000001},"last":0.68600000000000005}]},{"osd":1,"last update":"Sun Oct 1 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":8.2769999999999992,"5min":1.958,"15min":0.90500000000000003},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":154.32599999999999,"5min":154.32599999999999,"15min":154.32599999999999},"last":0.63},{"interface":"front","average":{"1min":8.3840000000000003,"5min":1.976,"15min":0.90800000000000003},"min":{"1min":0.25700000000000001,"5min":0.25,"15min":0.25},"max":{"1min":154.249,"5min":154.249,"15min":154.249},"last":0.48699999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":0.48299999999999998,"15min":0.39600000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":7.6260000000000003,"5min":7.6260000000000003,"15min":7.6260000000000003},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.48799999999999999,"15min":0.441},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.407,"5min":2.407,"15min":2.407},"last":1.0820000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.64400000000000002,"15min":0.59199999999999997},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":4.093,"5min":4.093,"15min":4.093},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.65300000000000002,"15min":0.627},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":2.129,"5min":2.129,"15min":2.129},"last":0.56699999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.65900000000000003,"15min":0.623},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":2.7269999999999999,"5min":2.7269999999999999,"15min":2.7269999999999999},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.0609999999999999,"5min":0.69699999999999995,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.601,"5min":7.601,"15min":7.601},"last":0.377}]},{"osd":6,"last update":"Sun Oct 1 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":0.70999999999999996,"15min":0.65600000000000003},"min":{"1min":0.40100000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":6.6040000000000001,"5min":6.6040000000000001,"15min":6.6040000000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.4530000000000001,"5min":0.80100000000000005,"15min":0.69199999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":7.556,"5min":7.556,"15min":7.556},"last":0.46500000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:15:14 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":0.75800000000000001,"15min":0.67800000000000005},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.4950000000000001,"5min":7.4950000000000001,"15min":7.4950000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.70999999999999996,"15min":0.66500000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":2.4279999999999999,"5min":2.4279999999999999,"15min":2.4279999999999999},"last":0.53200000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019070,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3640884,"kb_used_data":3339564,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":90104268,"statfs":{"total":95995035648,"available":92266770432,"internally_reserved":0,"allocated":3419713536,"data_stored":3419204743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3399999999999999,"5min":0.90300000000000002,"15min":0.69199999999999995},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":29.553000000000001,"5min":29.553000000000001,"15min":29.553000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.2850000000000001,"5min":0.90900000000000003,"15min":0.68799999999999994},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":28.408000000000001,"5min":28.408000000000001,"15min":28.408000000000001},"last":0.30599999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:15:16 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.63100000000000001,"15min":0.59999999999999998},"min":{"1min":0.34499999999999997,"5min":0.313,"15min":0.313},"max":{"1min":3.8140000000000001,"5min":3.8140000000000001,"15min":3.8140000000000001},"last":3.948},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.60099999999999998,"15min":0.56399999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.1560000000000001,"5min":3.1560000000000001,"15min":3.1560000000000001},"last":2.02}]},{"osd":2,"last update":"Sun Oct 1 09:15:44 2023","interfaces":[{"interface":"back","average":{"1min":13.41,"5min":3.1139999999999999,"15min":1.3919999999999999},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":85.441000000000003,"5min":85.441000000000003,"15min":85.441000000000003},"last":3.9830000000000001},{"interface":"front","average":{"1min":12.993,"5min":3.048,"15min":1.385},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":84.736000000000004,"5min":84.736000000000004,"15min":84.736000000000004},"last":0.38300000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:15:18 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.60899999999999999,"15min":0.56200000000000006},"min":{"1min":0.40999999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":3.1040000000000001,"5min":3.1040000000000001,"15min":3.1040000000000001},"last":2.5419999999999998},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.63600000000000001,"15min":0.57999999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":4.5570000000000004,"5min":4.5570000000000004,"15min":4.5570000000000004},"last":2.0449999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:15:16 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.54200000000000004,"15min":0.47899999999999998},"min":{"1min":0.35199999999999998,"5min":0.317,"15min":0.26300000000000001},"max":{"1min":2.8570000000000002,"5min":2.8570000000000002,"15min":2.8570000000000002},"last":6.6260000000000003},{"interface":"front","average":{"1min":1.0349999999999999,"5min":0.60899999999999999,"15min":0.51400000000000001},"min":{"1min":0.33400000000000002,"5min":0.30099999999999999,"15min":0.28899999999999998},"max":{"1min":3.798,"5min":3.798,"15min":3.798},"last":2.1299999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:15:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":0.67800000000000005,"15min":0.55900000000000005},"min":{"1min":0.30499999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.016,"5min":5.016,"15min":5.016},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.1950000000000001,"5min":0.70599999999999996,"15min":0.60199999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.33700000000000002},"max":{"1min":3.1190000000000002,"5min":3.1190000000000002,"15min":3.1190000000000002},"last":1.98}]},{"osd":7,"last update":"Sun Oct 1 09:15:55 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":0.747,"15min":0.60699999999999998},"min":{"1min":0.38300000000000001,"5min":0.34100000000000003,"15min":0.317},"max":{"1min":4.819,"5min":4.819,"15min":4.819},"last":4.5460000000000003},{"interface":"front","average":{"1min":1.2549999999999999,"5min":0.77800000000000002,"15min":0.63100000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.31},"max":{"1min":5.0529999999999999,"5min":5.0529999999999999,"15min":5.0529999999999999},"last":0.56699999999999995}]}]},{"osd":5,"up_from":36,"seq":154618822843,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4056740,"kb_used_data":3759516,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":89688412,"statfs":{"total":95995035648,"available":91840933888,"internally_reserved":0,"allocated":3849744384,"data_stored":3849247423,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:05 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":0.82999999999999996,"15min":0.70199999999999996},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":14.186,"5min":14.186,"15min":14.186},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.758,"5min":0.86899999999999999,"15min":0.69499999999999995},"min":{"1min":0.34300000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":14.308,"5min":14.308,"15min":14.308},"last":14.151}]},{"osd":1,"last update":"Sun Oct 1 09:15:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":0.90700000000000003,"15min":0.65000000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":32.420000000000002,"5min":32.420000000000002,"15min":32.420000000000002},"last":3.3660000000000001},{"interface":"front","average":{"1min":2.2320000000000002,"5min":0.90700000000000003,"15min":0.66600000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":32.481999999999999,"5min":32.481999999999999,"15min":32.481999999999999},"last":2.1339999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":0.88900000000000001,"15min":0.745},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":11.359,"5min":11.359,"15min":11.359},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.921,"5min":0.871,"15min":0.70699999999999996},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.449,"5min":10.449,"15min":10.449},"last":3.4409999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:15:18 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.85799999999999998,"15min":0.86599999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":2.8029999999999999,"5min":4.5990000000000002,"15min":4.5990000000000002},"last":2.3149999999999999},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.71099999999999997,"15min":0.68700000000000006},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":2.9159999999999999,"5min":2.9159999999999999,"15min":2.9159999999999999},"last":3.8460000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:15:09 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.502,"15min":0.47199999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.6899999999999999,"5min":1.6899999999999999,"15min":1.6899999999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.56799999999999995,"15min":0.52500000000000002},"min":{"1min":0.34000000000000002,"5min":0.316,"15min":0.28100000000000003},"max":{"1min":2.5699999999999998,"5min":2.5699999999999998,"15min":2.5699999999999998},"last":0.38}]},{"osd":6,"last update":"Sun Oct 1 09:15:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1930000000000001,"5min":0.66400000000000003,"15min":0.57399999999999995},"min":{"1min":0.312,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":5.2430000000000003,"5min":5.2430000000000003,"15min":5.2430000000000003},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.65300000000000002,"15min":0.56999999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.29399999999999998},"max":{"1min":4.7439999999999998,"5min":4.7439999999999998,"15min":4.7439999999999998},"last":1.4950000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:15:57 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.67900000000000005,"15min":0.60399999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":3.7839999999999998,"5min":3.7839999999999998,"15min":3.7839999999999998},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.1699999999999999,"5min":0.73399999999999999,"15min":0.622},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.2800000000000002,"5min":5.2800000000000002,"15min":5.2800000000000002},"last":0.44600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3305963520,"data_stored":3305963520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2083127296,"data_stored":2083127296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2517434368,"data_stored":2517434368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1230372864,"data_stored":1230372864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3423469568,"data_stored":3423469568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3849846784,"data_stored":3849846784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1726611456,"data_stored":1726611456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2151546880,"data_stored":2151546880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:16:03.947 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-01T09:16:03.947 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:16:03.948 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T09:16:03.948 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 1.000000 2023-10-01T09:16:03.948 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-01T09:16:03.948 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:03.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:03.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:04.077 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:04.156 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:04.391 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:04.398 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:04 smithi138 ceph-mon[185886]: pgmap v283: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-01T09:16:04.398 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:04 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:04.398 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:04 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:04.398 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:04 smithi138 ceph-mon[192513]: pgmap v283: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-01T09:16:04.399 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:04 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:04.399 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:04 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:04.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:04.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:04.500 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:04.533 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:16:04.534187+0000 min lat: 0.00109137 max lat: 0.285716 avg lat: 0.0119514 2023-10-01T09:16:04.534 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:16:04.534 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 80 16 107109 107093 83.6402 52.3125 0.0123665 0.0119514 2023-10-01T09:16:04.544 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:04.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:04 smithi190 ceph-mon[160435]: pgmap v283: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2023-10-01T09:16:04.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:04 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:04.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:04 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:05.067 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:05.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:05.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:05.159 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:05.170 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:05.209 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:05.219 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:16:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:16:04] "GET /metrics HTTP/1.1" 200 36324 "" "Prometheus/2.33.4" 2023-10-01T09:16:05.278 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:05.410 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:05.432 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:05.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:05.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:05.537 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 81 16 107819 107803 83.1525 44.375 0.0138119 0.0120218 2023-10-01T09:16:05.545 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:05.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:05 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:16:05.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:05 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:16:05.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:05 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:05.565 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:05 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:16:05.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:05 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:16:05.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:05 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:05.604 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:05.726 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:05.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:05.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:05.739 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:05 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:16:05.739 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:05 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:16:05.739 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:05 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:05.840 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:05.919 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:06.537 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 82 16 108751 108735 82.8488 58.25 0.0125409 0.0120653 2023-10-01T09:16:06.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:06 smithi138 ceph-mon[192513]: pgmap v284: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-01T09:16:06.714 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:06 smithi138 ceph-mon[185886]: pgmap v284: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-01T09:16:06.721 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:06.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:06.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:06.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:06 smithi190 ceph-mon[160435]: pgmap v284: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-10-01T09:16:06.829 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:06.889 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:06.995 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:07.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:07.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:07.107 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:16:07.163 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:07.302 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:16:07.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:07.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:16:07.406 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:07.440 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:16:07.467 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:07 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/393552258' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:07.468 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:07 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:07.468 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:07 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/393552258' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:07.468 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:07 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:07.481 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:07.537 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 83 16 109573 109557 82.4697 51.375 0.0220514 0.0121208 2023-10-01T09:16:07.613 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:07.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:07.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:07.729 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:07.781 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:07 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/393552258' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:07.781 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:07 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:07.790 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:07.901 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:07.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:07.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:08.013 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:08.068 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:08.537 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 84 16 110342 110326 82.0601 48.0625 0.013987 0.0121816 2023-10-01T09:16:08.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[185886]: pgmap v285: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-10-01T09:16:08.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:16:08.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:16:08.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:16:08.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[185886]: osdmap e122: 8 total, 8 up, 8 in 2023-10-01T09:16:08.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[185886]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[185886]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[185886]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[185886]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[185886]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[185886]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[192513]: pgmap v285: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-10-01T09:16:08.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:16:08.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:16:08.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:16:08.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[192513]: osdmap e122: 8 total, 8 up, 8 in 2023-10-01T09:16:08.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[192513]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[192513]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[192513]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[192513]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[192513]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:08 smithi138 ceph-mon[192513]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:08 smithi190 ceph-mon[160435]: pgmap v285: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-10-01T09:16:08.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:08 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:16:08.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:08 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:16:08.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:08 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:16:08.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:08 smithi190 ceph-mon[160435]: osdmap e122: 8 total, 8 up, 8 in 2023-10-01T09:16:08.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:08 smithi190 ceph-mon[160435]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:08 smithi190 ceph-mon[160435]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:08 smithi190 ceph-mon[160435]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:08 smithi190 ceph-mon[160435]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:08 smithi190 ceph-mon[160435]: failed to encode map e122 with expected crc 2023-10-01T09:16:08.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:08 smithi190 ceph-mon[160435]: failed to encode map e122 with expected crc 2023-10-01T09:16:09.537 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 85 16 111169 111153 81.7028 51.6875 0.00765665 0.0122342 2023-10-01T09:16:10.012 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:10.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:10.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:10.123 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:10.181 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:10.245 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:16:10.250 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:10.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:10.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:10.287 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:16:10.328 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:16:10.363 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:10.373 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:16:10.407 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:10.410 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:16:10.451 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:16:10.486 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:16:10.497 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:10.507 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:16:10.508 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:16:10.508 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:16:10.508 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:16:10.508 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:16:10.509 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:16:10.509 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:16:10.509 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:16:10.509 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:16:10.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:10.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:10.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:10 smithi190 ceph-mon[160435]: pgmap v287: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-01T09:16:10.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:10 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2696832037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:10.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:10.513 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:16:10.537 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 86 16 111922 111906 81.3001 47.0625 0.011691 0.0122956 2023-10-01T09:16:10.613 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:10.683 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:10 smithi138 ceph-mon[192513]: pgmap v287: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-01T09:16:10.684 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:10 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2696832037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:10.684 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:10.684 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:16:10.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:10 smithi138 ceph-mon[185886]: pgmap v287: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-01T09:16:10.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:10 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2696832037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:10.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:10.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:16:10.696 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:11.442 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-01T09:16:11.537 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 87 16 112695 112679 80.921 48.3125 0.0189843 0.0123538 2023-10-01T09:16:11.732 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1794824891' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:16:11.732 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:11 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:16:11.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1794824891' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:16:11.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:11 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:16:11.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1794824891' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:16:11.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:11 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:16:12.077 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:12.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:12.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:12.182 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:12.207 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:16:12.247 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:12.537 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 88 16 113445 113429 80.5341 46.875 0.021467 0.0124122 2023-10-01T09:16:12.615 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:12.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:12.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:12.733 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:16:12.734 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:16:12.736 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:12.760 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[185886]: pgmap v288: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s 2023-10-01T09:16:12.760 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:16:12.760 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[185886]: osdmap e123: 8 total, 8 up, 8 in 2023-10-01T09:16:12.760 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[185886]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.760 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[185886]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.760 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[185886]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.760 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[185886]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.761 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[185886]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.761 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[185886]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.761 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[192513]: pgmap v288: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s 2023-10-01T09:16:12.761 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:16:12.761 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[192513]: osdmap e123: 8 total, 8 up, 8 in 2023-10-01T09:16:12.761 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[192513]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.761 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[192513]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.761 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[192513]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.761 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[192513]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.761 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[192513]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.762 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:12 smithi138 ceph-mon[192513]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:12 smithi190 ceph-mon[160435]: pgmap v288: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s 2023-10-01T09:16:12.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:12 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:16:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:12 smithi190 ceph-mon[160435]: osdmap e123: 8 total, 8 up, 8 in 2023-10-01T09:16:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:12 smithi190 ceph-mon[160435]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:12 smithi190 ceph-mon[160435]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:12 smithi190 ceph-mon[160435]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:12 smithi190 ceph-mon[160435]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:12 smithi190 ceph-mon[160435]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:12 smithi190 ceph-mon[160435]: failed to encode map e123 with expected crc 2023-10-01T09:16:12.810 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:13.069 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:13.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:13.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:13.174 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:13.247 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:13.392 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:13.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:13.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:13.500 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:13.537 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 89 16 114336 114320 80.2547 55.6875 0.024996 0.0124553 2023-10-01T09:16:13.555 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:13.640 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:13.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:13.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:13.751 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:13.777 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:13.822 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:16:13.830 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:13.970 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:13.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:13.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:14.074 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:14.148 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:14.210 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:16:14.250 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:14.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:14.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:14.376 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:14.448 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:14.506 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:16:14.536 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:14.537 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Total time run: 90.0142 2023-10-01T09:16:14.538 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Total writes made: 115134 2023-10-01T09:16:14.538 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Write size: 65536 2023-10-01T09:16:14.538 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Object size: 65536 2023-10-01T09:16:14.538 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Bandwidth (MB/sec): 79.9415 2023-10-01T09:16:14.538 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev Bandwidth: 37.133 2023-10-01T09:16:14.538 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max bandwidth (MB/sec): 226.062 2023-10-01T09:16:14.538 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min bandwidth (MB/sec): 39.625 2023-10-01T09:16:14.538 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Average IOPS: 1279 2023-10-01T09:16:14.538 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev IOPS: 594.129 2023-10-01T09:16:14.538 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max IOPS: 3617 2023-10-01T09:16:14.538 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min IOPS: 634 2023-10-01T09:16:14.539 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Average Latency(s): 0.0125052 2023-10-01T09:16:14.539 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev Latency(s): 0.0152369 2023-10-01T09:16:14.539 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max latency(s): 0.285716 2023-10-01T09:16:14.539 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min latency(s): 0.00109137 2023-10-01T09:16:14.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:14.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:14.543 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T09:16:14.644 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:14.685 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:14.731 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:14 smithi190 ceph-mon[160435]: pgmap v290: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-10-01T09:16:14.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:14 smithi138 ceph-mon[192513]: pgmap v290: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-10-01T09:16:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:14 smithi138 ceph-mon[185886]: pgmap v290: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-10-01T09:16:15.260 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:15.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:15.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:15.275 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:16:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:16:14] "GET /metrics HTTP/1.1" 200 36327 "" "Prometheus/2.33.4" 2023-10-01T09:16:15.369 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:16.193 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:16.245 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:16.461 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:16:16.502 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:16 smithi190 ceph-mon[160435]: pgmap v291: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s 2023-10-01T09:16:16.502 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:16 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2662510865' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:16:16.502 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:16 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:16:16.521 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:16.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:16.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:16.633 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:16.704 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:16 smithi138 ceph-mon[185886]: pgmap v291: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s 2023-10-01T09:16:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:16 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2662510865' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:16:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:16 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:16:16.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:16 smithi138 ceph-mon[192513]: pgmap v291: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s 2023-10-01T09:16:16.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:16 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2662510865' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:16:16.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:16 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:16:16.858 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:16.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:16.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:16.965 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:17.048 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:17.159 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:17.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:17.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:17.182 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:16:17.183 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:16:17.266 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:17.368 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:17.458 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:17.464 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:17 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:17.464 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:17 smithi190 ceph-mon[160435]: failed to encode map e124 with expected crc 2023-10-01T09:16:17.464 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:17 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:16:17.464 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:17 smithi190 ceph-mon[160435]: osdmap e124: 8 total, 8 up, 8 in 2023-10-01T09:16:17.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:17.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:17.532 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:16:17.532 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:16:17.532 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T09:16:17.532 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.000000 2023-10-01T09:16:17.532 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0 2023-10-01T09:16:17.569 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:17.600 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:17.668 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:17.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:17.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:17.772 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:17 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:17 smithi138 ceph-mon[192513]: failed to encode map e124 with expected crc 2023-10-01T09:16:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:17 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:16:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:17 smithi138 ceph-mon[192513]: osdmap e124: 8 total, 8 up, 8 in 2023-10-01T09:16:17.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:17 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:17.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:17 smithi138 ceph-mon[185886]: failed to encode map e124 with expected crc 2023-10-01T09:16:17.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:17 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:16:17.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:17 smithi138 ceph-mon[185886]: osdmap e124: 8 total, 8 up, 8 in 2023-10-01T09:16:17.826 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:18.117 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:16:18.124 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:18.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:18.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:18.235 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:18.284 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:18.397 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:18.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:18.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:18.466 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:18 smithi190 ceph-mon[160435]: pgmap v293: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 808 op/s 2023-10-01T09:16:18.505 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:18.522 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:18.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:18 smithi138 ceph-mon[185886]: pgmap v293: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 808 op/s 2023-10-01T09:16:18.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:18 smithi138 ceph-mon[192513]: pgmap v293: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 808 op/s 2023-10-01T09:16:18.595 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:18.649 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:16:18.696 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:16:18.738 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:16:18.764 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:18.765 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:16:18.775 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:16:18.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:18.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:18.804 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:16:18.839 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:16:18.878 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:18.879 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:16:18.908 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:16:18.922 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:16:18.923 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:16:18.923 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:16:18.923 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:16:18.923 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:16:18.923 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:16:18.923 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:16:18.923 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:16:18.949 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:19.048 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:19.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:19.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:19.162 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:19.199 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:19.291 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:19.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:19.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:19.395 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:19.467 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:19.527 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3845575467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:19.527 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:19 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3845575467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:19.628 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:19.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:19.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:19.735 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:19.787 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:19 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3845575467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:20.117 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:16:20.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:20 smithi190 ceph-mon[160435]: pgmap v294: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 35 MiB/s wr, 844 op/s 2023-10-01T09:16:20.775 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:20.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:20 smithi138 ceph-mon[192513]: pgmap v294: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 35 MiB/s wr, 844 op/s 2023-10-01T09:16:20.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:20.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:20.780 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:20 smithi138 ceph-mon[185886]: pgmap v294: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 35 MiB/s wr, 844 op/s 2023-10-01T09:16:20.848 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:16:20.848 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:16:20.848 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:16:20.848 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:16:20.848 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:16:20.879 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:20.961 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:21.067 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:21.086 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:21.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:21.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:21.196 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:21.249 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:21.474 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:21.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:21.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:21.586 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:21.667 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:21.698 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:16:21.754 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:21 smithi190 ceph-mon[160435]: pgmap v295: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 103 B/s rd, 25 MiB/s wr, 700 op/s 2023-10-01T09:16:21.769 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:21.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:21.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:21.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:21 smithi138 ceph-mon[185886]: pgmap v295: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 103 B/s rd, 25 MiB/s wr, 700 op/s 2023-10-01T09:16:21.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:21 smithi138 ceph-mon[192513]: pgmap v295: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 103 B/s rd, 25 MiB/s wr, 700 op/s 2023-10-01T09:16:21.880 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:21.932 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:22.024 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:22.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:22.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:22.130 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:22.213 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:22.317 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:22.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:22.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:22.413 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:16:22.423 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:22.488 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:22.529 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.2 primary-affinity to 0 (802) 2023-10-01T09:16:22.610 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:22.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:22.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:22.648 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:16:22.713 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:22.740 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:22.746 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:22 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1814648498' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-01T09:16:22.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:22 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1814648498' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-01T09:16:22.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:22 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1814648498' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-10-01T09:16:23.245 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:23.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:23.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:23.348 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:23.378 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:23.643 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:23.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:23.665 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:23 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1814648498' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-01T09:16:23.665 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:23 smithi190 ceph-mon[160435]: osdmap e125: 8 total, 8 up, 8 in 2023-10-01T09:16:23.666 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:23 smithi190 ceph-mon[160435]: failed to encode map e125 with expected crc 2023-10-01T09:16:23.666 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:23 smithi190 ceph-mon[160435]: failed to encode map e125 with expected crc 2023-10-01T09:16:23.666 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:23 smithi190 ceph-mon[160435]: failed to encode map e125 with expected crc 2023-10-01T09:16:23.666 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:23 smithi190 ceph-mon[160435]: failed to encode map e125 with expected crc 2023-10-01T09:16:23.666 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:23 smithi190 ceph-mon[160435]: pgmap v297: 89 pgs: 1 peering, 88 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 14 MiB/s wr, 831 op/s 2023-10-01T09:16:23.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:23.767 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:23.780 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:23.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:23 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1814648498' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-01T09:16:23.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:23 smithi138 ceph-mon[192513]: osdmap e125: 8 total, 8 up, 8 in 2023-10-01T09:16:23.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:23 smithi138 ceph-mon[192513]: failed to encode map e125 with expected crc 2023-10-01T09:16:23.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:23 smithi138 ceph-mon[192513]: failed to encode map e125 with expected crc 2023-10-01T09:16:23.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:23 smithi138 ceph-mon[192513]: failed to encode map e125 with expected crc 2023-10-01T09:16:23.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:23 smithi138 ceph-mon[192513]: failed to encode map e125 with expected crc 2023-10-01T09:16:23.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:23 smithi138 ceph-mon[192513]: pgmap v297: 89 pgs: 1 peering, 88 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 14 MiB/s wr, 831 op/s 2023-10-01T09:16:23.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:23 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1814648498' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-10-01T09:16:23.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:23 smithi138 ceph-mon[185886]: osdmap e125: 8 total, 8 up, 8 in 2023-10-01T09:16:23.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:23 smithi138 ceph-mon[185886]: failed to encode map e125 with expected crc 2023-10-01T09:16:23.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:23 smithi138 ceph-mon[185886]: failed to encode map e125 with expected crc 2023-10-01T09:16:23.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:23 smithi138 ceph-mon[185886]: failed to encode map e125 with expected crc 2023-10-01T09:16:23.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:23 smithi138 ceph-mon[185886]: failed to encode map e125 with expected crc 2023-10-01T09:16:23.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:23 smithi138 ceph-mon[185886]: pgmap v297: 89 pgs: 1 peering, 88 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 14 MiB/s wr, 831 op/s 2023-10-01T09:16:23.833 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:24.032 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:24.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:24.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:24.140 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:24.198 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:24.301 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:16:24.301 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:16:24.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:16:24.302 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:16:24.302 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.302 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:20463 5.e 5:74e5671f:::benchmark_data_smithi138_240338_object20462:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-01T09:16:24.302 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:14:51.474705+0000", 2023-10-01T09:16:24.302 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 92.821996107999993, 2023-10-01T09:16:24.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16362856000000001, 2023-10-01T09:16:24.302 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.302 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.302 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.302 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.303 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.303 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 20463 2023-10-01T09:16:24.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.303 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.303 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:51.474705+0000", 2023-10-01T09:16:24.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.303 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:51.474705+0000", 2023-10-01T09:16:24.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:24.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:51.474704+0000", 2023-10-01T09:16:24.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2429999999999998e-05 2023-10-01T09:16:24.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:51.474756+0000", 2023-10-01T09:16:24.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5996000000000001e-05 2023-10-01T09:16:24.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:51.474772+0000", 2023-10-01T09:16:24.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3289999999999996e-06 2023-10-01T09:16:24.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:51.474779+0000", 2023-10-01T09:16:24.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6783999999999997e-05 2023-10-01T09:16:24.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:51.474836+0000", 2023-10-01T09:16:24.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012535199999999999 2023-10-01T09:16:24.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:51.474962+0000", 2023-10-01T09:16:24.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000158481 2023-10-01T09:16:24.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:16:24.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:51.475120+0000", 2023-10-01T09:16:24.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0010411070000000001 2023-10-01T09:16:24.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:51.476161+0000", 2023-10-01T09:16:24.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00088659600000000002 2023-10-01T09:16:24.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:51.477048+0000", 2023-10-01T09:16:24.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.161206087 2023-10-01T09:16:24.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:51.638254+0000", 2023-10-01T09:16:24.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7920999999999998e-05 2023-10-01T09:16:24.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:51.638292+0000", 2023-10-01T09:16:24.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2129999999999998e-05 2023-10-01T09:16:24.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:51.638334+0000", 2023-10-01T09:16:24.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16362856000000001 2023-10-01T09:16:24.309 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.309 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.309 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.309 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:24326 5.e 5:78e4825d:::benchmark_data_smithi138_240338_object24325:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-01T09:16:24.309 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:14:53.636330+0000", 2023-10-01T09:16:24.309 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 90.660371401000006, 2023-10-01T09:16:24.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13526139400000001, 2023-10-01T09:16:24.310 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.310 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.310 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.310 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.310 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.310 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 24326 2023-10-01T09:16:24.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.310 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.636330+0000", 2023-10-01T09:16:24.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.636330+0000", 2023-10-01T09:16:24.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:24.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.636329+0000", 2023-10-01T09:16:24.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2527e-05 2023-10-01T09:16:24.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.636381+0000", 2023-10-01T09:16:24.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9490000000000008e-06 2023-10-01T09:16:24.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.636391+0000", 2023-10-01T09:16:24.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8660000000000002e-06 2023-10-01T09:16:24.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.636397+0000", 2023-10-01T09:16:24.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4178999999999997e-05 2023-10-01T09:16:24.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.636451+0000", 2023-10-01T09:16:24.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000121113 2023-10-01T09:16:24.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.636572+0000", 2023-10-01T09:16:24.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000100245 2023-10-01T09:16:24.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:16:24.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.636672+0000", 2023-10-01T09:16:24.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0011462849999999999 2023-10-01T09:16:24.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.637819+0000", 2023-10-01T09:16:24.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0028968940000000001 2023-10-01T09:16:24.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.640716+0000", 2023-10-01T09:16:24.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13077461500000001 2023-10-01T09:16:24.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.771490+0000", 2023-10-01T09:16:24.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1074000000000005e-05 2023-10-01T09:16:24.318 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.318 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.318 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.318 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.771551+0000", 2023-10-01T09:16:24.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0213000000000003e-05 2023-10-01T09:16:24.318 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.319 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.319 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.771591+0000", 2023-10-01T09:16:24.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13526139400000001 2023-10-01T09:16:24.319 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.319 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.319 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.320 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:24340 5.e 5:7841b73c:::benchmark_data_smithi138_240338_object24339:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e112)", 2023-10-01T09:16:24.320 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:14:53.640794+0000", 2023-10-01T09:16:24.320 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 90.655907188, 2023-10-01T09:16:24.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13093136, 2023-10-01T09:16:24.320 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.320 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.320 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.320 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.320 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.321 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 24340 2023-10-01T09:16:24.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.321 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.640794+0000", 2023-10-01T09:16:24.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.640794+0000", 2023-10-01T09:16:24.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:16:24.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.640793+0000", 2023-10-01T09:16:24.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.8611999999999998e-05 2023-10-01T09:16:24.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.640882+0000", 2023-10-01T09:16:24.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3735e-05 2023-10-01T09:16:24.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.640895+0000", 2023-10-01T09:16:24.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5349999999999998e-06 2023-10-01T09:16:24.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.640900+0000", 2023-10-01T09:16:24.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7930999999999995e-05 2023-10-01T09:16:24.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.640988+0000", 2023-10-01T09:16:24.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013409899999999999 2023-10-01T09:16:24.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.641122+0000", 2023-10-01T09:16:24.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016231899999999999 2023-10-01T09:16:24.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.325 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:16:24.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.641284+0000", 2023-10-01T09:16:24.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00099234900000000001 2023-10-01T09:16:24.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.325 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.642277+0000", 2023-10-01T09:16:24.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0076974779999999998 2023-10-01T09:16:24.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.326 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.326 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.649974+0000", 2023-10-01T09:16:24.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.12167589600000001 2023-10-01T09:16:24.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.326 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.326 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.771650+0000", 2023-10-01T09:16:24.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8028000000000002e-05 2023-10-01T09:16:24.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.327 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.327 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.771698+0000", 2023-10-01T09:16:24.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7532e-05 2023-10-01T09:16:24.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.327 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.327 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:53.771726+0000", 2023-10-01T09:16:24.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13093136 2023-10-01T09:16:24.327 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.327 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.329 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.329 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:59038 5.e 5:74842654:::benchmark_data_smithi138_240338_object59037:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:16:24.329 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:15:16.277291+0000", 2023-10-01T09:16:24.329 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 68.019410884999999, 2023-10-01T09:16:24.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.226115134, 2023-10-01T09:16:24.329 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.329 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.330 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.330 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.330 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.330 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 59038 2023-10-01T09:16:24.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.330 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277291+0000", 2023-10-01T09:16:24.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277291+0000", 2023-10-01T09:16:24.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:16:24.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277289+0000", 2023-10-01T09:16:24.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0015555149999999999 2023-10-01T09:16:24.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.278844+0000", 2023-10-01T09:16:24.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3913e-05 2023-10-01T09:16:24.332 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.278858+0000", 2023-10-01T09:16:24.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5660000000000003e-06 2023-10-01T09:16:24.332 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.278865+0000", 2023-10-01T09:16:24.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0026173260000000001 2023-10-01T09:16:24.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.333 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.281482+0000", 2023-10-01T09:16:24.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013407600000000001 2023-10-01T09:16:24.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.333 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.281616+0000", 2023-10-01T09:16:24.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015696500000000001 2023-10-01T09:16:24.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.334 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:16:24.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.281773+0000", 2023-10-01T09:16:24.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001914568 2023-10-01T09:16:24.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.335 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.335 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.283688+0000", 2023-10-01T09:16:24.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00142968 2023-10-01T09:16:24.335 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.335 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.335 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.285117+0000", 2023-10-01T09:16:24.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.218189357 2023-10-01T09:16:24.335 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.336 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.336 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.503307+0000", 2023-10-01T09:16:24.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0662999999999998e-05 2023-10-01T09:16:24.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.336 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.336 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.503368+0000", 2023-10-01T09:16:24.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8223999999999999e-05 2023-10-01T09:16:24.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.336 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.503406+0000", 2023-10-01T09:16:24.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.226115134 2023-10-01T09:16:24.337 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.337 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.337 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.337 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.337 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:59809 5.e 5:7706c479:::benchmark_data_smithi138_240338_object59808:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:16:24.337 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:15:17.102414+0000", 2023-10-01T09:16:24.337 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 67.194287633000002, 2023-10-01T09:16:24.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13925678999999999, 2023-10-01T09:16:24.337 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.338 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.338 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.338 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.338 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.338 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 59809 2023-10-01T09:16:24.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.338 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102414+0000", 2023-10-01T09:16:24.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102414+0000", 2023-10-01T09:16:24.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:24.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102413+0000", 2023-10-01T09:16:24.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0605e-05 2023-10-01T09:16:24.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102473+0000", 2023-10-01T09:16:24.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8082999999999999e-05 2023-10-01T09:16:24.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102491+0000", 2023-10-01T09:16:24.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9247999999999999e-05 2023-10-01T09:16:24.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102510+0000", 2023-10-01T09:16:24.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2046999999999998e-05 2023-10-01T09:16:24.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102562+0000", 2023-10-01T09:16:24.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000116857 2023-10-01T09:16:24.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102679+0000", 2023-10-01T09:16:24.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012727800000000001 2023-10-01T09:16:24.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.342 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:16:24.342 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102807+0000", 2023-10-01T09:16:24.342 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038879359999999998 2023-10-01T09:16:24.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.342 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.342 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.106695+0000", 2023-10-01T09:16:24.342 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00081432200000000003 2023-10-01T09:16:24.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.107509+0000", 2023-10-01T09:16:24.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.134038928 2023-10-01T09:16:24.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.241548+0000", 2023-10-01T09:16:24.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5835999999999998e-05 2023-10-01T09:16:24.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.241614+0000", 2023-10-01T09:16:24.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6996000000000001e-05 2023-10-01T09:16:24.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.241671+0000", 2023-10-01T09:16:24.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13925678999999999 2023-10-01T09:16:24.344 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.344 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.344 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.345 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:59811 5.e 5:71e8ae2e:::benchmark_data_smithi138_240338_object59810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:16:24.345 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:15:17.102615+0000", 2023-10-01T09:16:24.345 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 67.194086944000006, 2023-10-01T09:16:24.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13920237099999999, 2023-10-01T09:16:24.345 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.345 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.345 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.345 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.345 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.346 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 59811 2023-10-01T09:16:24.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.346 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102615+0000", 2023-10-01T09:16:24.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102615+0000", 2023-10-01T09:16:24.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:16:24.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102614+0000", 2023-10-01T09:16:24.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5213999999999999e-05 2023-10-01T09:16:24.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102639+0000", 2023-10-01T09:16:24.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8790000000000003e-06 2023-10-01T09:16:24.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102647+0000", 2023-10-01T09:16:24.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4730000000000002e-06 2023-10-01T09:16:24.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.102652+0000", 2023-10-01T09:16:24.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0021392910000000002 2023-10-01T09:16:24.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.104791+0000", 2023-10-01T09:16:24.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012781399999999999 2023-10-01T09:16:24.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.104919+0000", 2023-10-01T09:16:24.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000192566 2023-10-01T09:16:24.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:16:24.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.105111+0000", 2023-10-01T09:16:24.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0017348730000000001 2023-10-01T09:16:24.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.106846+0000", 2023-10-01T09:16:24.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0015630520000000001 2023-10-01T09:16:24.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.108409+0000", 2023-10-01T09:16:24.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13331168900000001 2023-10-01T09:16:24.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.241721+0000", 2023-10-01T09:16:24.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2797000000000003e-05 2023-10-01T09:16:24.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.241774+0000", 2023-10-01T09:16:24.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3291000000000003e-05 2023-10-01T09:16:24.352 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.352 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.352 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.241817+0000", 2023-10-01T09:16:24.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13920237099999999 2023-10-01T09:16:24.352 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.352 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.352 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.352 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.352 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:59825 5.e 5:7905937f:::benchmark_data_smithi138_240338_object59824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:16:24.353 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:15:17.110557+0000", 2023-10-01T09:16:24.353 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 67.186144377000005, 2023-10-01T09:16:24.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13399065700000001, 2023-10-01T09:16:24.353 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.353 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.353 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.353 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.353 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.353 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 59825 2023-10-01T09:16:24.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.353 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.110557+0000", 2023-10-01T09:16:24.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.110557+0000", 2023-10-01T09:16:24.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:24.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.110556+0000", 2023-10-01T09:16:24.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2602000000000002e-05 2023-10-01T09:16:24.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.110608+0000", 2023-10-01T09:16:24.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4918e-05 2023-10-01T09:16:24.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.110623+0000", 2023-10-01T09:16:24.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2860000000000002e-06 2023-10-01T09:16:24.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.110630+0000", 2023-10-01T09:16:24.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0518e-05 2023-10-01T09:16:24.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.110681+0000", 2023-10-01T09:16:24.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014697700000000001 2023-10-01T09:16:24.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.110828+0000", 2023-10-01T09:16:24.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00019275999999999999 2023-10-01T09:16:24.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:16:24.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.111021+0000", 2023-10-01T09:16:24.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0020723790000000001 2023-10-01T09:16:24.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.113093+0000", 2023-10-01T09:16:24.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0022701660000000001 2023-10-01T09:16:24.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.115363+0000", 2023-10-01T09:16:24.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.12909240599999999 2023-10-01T09:16:24.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.359 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.244456+0000", 2023-10-01T09:16:24.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3771999999999999e-05 2023-10-01T09:16:24.359 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.359 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.359 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.359 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.244509+0000", 2023-10-01T09:16:24.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8306000000000002e-05 2023-10-01T09:16:24.359 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.359 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.359 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.359 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:17.244548+0000", 2023-10-01T09:16:24.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13399065700000001 2023-10-01T09:16:24.360 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.360 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.360 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.360 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:65033 5.e 5:7bb6a1f3:::benchmark_data_smithi138_240338_object65032:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e116)", 2023-10-01T09:16:24.360 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:15:21.318979+0000", 2023-10-01T09:16:24.360 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 62.977722372000002, 2023-10-01T09:16:24.360 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.148423786, 2023-10-01T09:16:24.360 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.360 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.361 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.361 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.361 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.361 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 65033 2023-10-01T09:16:24.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.361 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:21.318979+0000", 2023-10-01T09:16:24.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:21.318979+0000", 2023-10-01T09:16:24.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:24.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:21.318977+0000", 2023-10-01T09:16:24.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000110643 2023-10-01T09:16:24.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:21.319088+0000", 2023-10-01T09:16:24.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3378e-05 2023-10-01T09:16:24.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:21.319101+0000", 2023-10-01T09:16:24.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0639999999999994e-06 2023-10-01T09:16:24.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:21.319110+0000", 2023-10-01T09:16:24.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8293e-05 2023-10-01T09:16:24.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:21.319158+0000", 2023-10-01T09:16:24.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015338 2023-10-01T09:16:24.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:21.319311+0000", 2023-10-01T09:16:24.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000170471 2023-10-01T09:16:24.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:16:24.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:21.319482+0000", 2023-10-01T09:16:24.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0023809999999999999 2023-10-01T09:16:24.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:21.321863+0000", 2023-10-01T09:16:24.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002259727 2023-10-01T09:16:24.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:21.324122+0000", 2023-10-01T09:16:24.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14316305800000001 2023-10-01T09:16:24.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:21.467285+0000", 2023-10-01T09:16:24.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2218000000000005e-05 2023-10-01T09:16:24.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.367 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:21.467358+0000", 2023-10-01T09:16:24.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5198999999999999e-05 2023-10-01T09:16:24.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.367 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.367 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:21.467403+0000", 2023-10-01T09:16:24.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.148423786 2023-10-01T09:16:24.367 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.367 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.367 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.368 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:104879 5.e 5:767300dc:::benchmark_data_smithi138_240338_object104878:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-01T09:16:24.368 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:01.649079+0000", 2023-10-01T09:16:24.368 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 22.647622344999998, 2023-10-01T09:16:24.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14444090800000001, 2023-10-01T09:16:24.368 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.368 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.368 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.368 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.368 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.368 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 104879 2023-10-01T09:16:24.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.369 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.649079+0000", 2023-10-01T09:16:24.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.649079+0000", 2023-10-01T09:16:24.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:16:24.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.649077+0000", 2023-10-01T09:16:24.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3246999999999999e-05 2023-10-01T09:16:24.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.649150+0000", 2023-10-01T09:16:24.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5179e-05 2023-10-01T09:16:24.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.649165+0000", 2023-10-01T09:16:24.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1029999999999998e-06 2023-10-01T09:16:24.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.649172+0000", 2023-10-01T09:16:24.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8872000000000002e-05 2023-10-01T09:16:24.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.649220+0000", 2023-10-01T09:16:24.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014991000000000001 2023-10-01T09:16:24.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.649370+0000", 2023-10-01T09:16:24.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017669499999999999 2023-10-01T09:16:24.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:16:24.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.649547+0000", 2023-10-01T09:16:24.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033545720000000001 2023-10-01T09:16:24.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.652902+0000", 2023-10-01T09:16:24.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0026719619999999999 2023-10-01T09:16:24.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.655574+0000", 2023-10-01T09:16:24.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.137868564 2023-10-01T09:16:24.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.374 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.793442+0000", 2023-10-01T09:16:24.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1805000000000003e-05 2023-10-01T09:16:24.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.374 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.793494+0000", 2023-10-01T09:16:24.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6086999999999999e-05 2023-10-01T09:16:24.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.374 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.793520+0000", 2023-10-01T09:16:24.375 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14444090800000001 2023-10-01T09:16:24.375 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.375 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.375 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.375 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.375 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.375 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:105922 5.1 5:8bb25be3:::benchmark_data_smithi138_240338_object105921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-01T09:16:24.375 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:03.091556+0000", 2023-10-01T09:16:24.375 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 21.205145403, 2023-10-01T09:16:24.375 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.134433304, 2023-10-01T09:16:24.375 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.375 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.376 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.376 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.376 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.376 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 105922 2023-10-01T09:16:24.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.376 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.376 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:03.091556+0000", 2023-10-01T09:16:24.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:03.091556+0000", 2023-10-01T09:16:24.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:24.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:03.091555+0000", 2023-10-01T09:16:24.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8186999999999997e-05 2023-10-01T09:16:24.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:03.091613+0000", 2023-10-01T09:16:24.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7272999999999999e-05 2023-10-01T09:16:24.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:03.091630+0000", 2023-10-01T09:16:24.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6279999999999999e-06 2023-10-01T09:16:24.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:03.091637+0000", 2023-10-01T09:16:24.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8087999999999997e-05 2023-10-01T09:16:24.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:03.091695+0000", 2023-10-01T09:16:24.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000146599 2023-10-01T09:16:24.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:03.091841+0000", 2023-10-01T09:16:24.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016781500000000001 2023-10-01T09:16:24.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.380 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:16:24.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:03.092009+0000", 2023-10-01T09:16:24.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038575789999999999 2023-10-01T09:16:24.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.380 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:03.095867+0000", 2023-10-01T09:16:24.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0047175159999999997 2023-10-01T09:16:24.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:03.100584+0000", 2023-10-01T09:16:24.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.12530617799999999 2023-10-01T09:16:24.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:03.225890+0000", 2023-10-01T09:16:24.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2596000000000003e-05 2023-10-01T09:16:24.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:03.225943+0000", 2023-10-01T09:16:24.382 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6415000000000002e-05 2023-10-01T09:16:24.382 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.382 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.382 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.382 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:03.225989+0000", 2023-10-01T09:16:24.382 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.134433304 2023-10-01T09:16:24.382 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.382 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.382 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.382 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.382 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.383 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:110057 5.e 5:7faf45b8:::benchmark_data_smithi138_240338_object110056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-01T09:16:24.383 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:08.050553+0000", 2023-10-01T09:16:24.383 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 16.246148883, 2023-10-01T09:16:24.383 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22493601599999999, 2023-10-01T09:16:24.383 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.383 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.383 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 110057 2023-10-01T09:16:24.383 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.383 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050553+0000", 2023-10-01T09:16:24.384 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050553+0000", 2023-10-01T09:16:24.384 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:16:24.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050551+0000", 2023-10-01T09:16:24.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000123542 2023-10-01T09:16:24.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050674+0000", 2023-10-01T09:16:24.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7989e-05 2023-10-01T09:16:24.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050692+0000", 2023-10-01T09:16:24.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4090000000000003e-06 2023-10-01T09:16:24.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.386 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050701+0000", 2023-10-01T09:16:24.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0016255810000000001 2023-10-01T09:16:24.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.386 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.052326+0000", 2023-10-01T09:16:24.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000140653 2023-10-01T09:16:24.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.052467+0000", 2023-10-01T09:16:24.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014696699999999999 2023-10-01T09:16:24.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:16:24.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.052614+0000", 2023-10-01T09:16:24.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003798516 2023-10-01T09:16:24.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.388 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.056412+0000", 2023-10-01T09:16:24.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0023911510000000002 2023-10-01T09:16:24.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.388 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.058804+0000", 2023-10-01T09:16:24.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21657992000000001 2023-10-01T09:16:24.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.388 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275383+0000", 2023-10-01T09:16:24.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4166999999999998e-05 2023-10-01T09:16:24.389 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.389 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.389 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.389 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275438+0000", 2023-10-01T09:16:24.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0992000000000002e-05 2023-10-01T09:16:24.389 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.389 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.389 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.389 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275489+0000", 2023-10-01T09:16:24.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22493601599999999 2023-10-01T09:16:24.390 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.390 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.390 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.390 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.390 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.390 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:110065 5.e 5:7ba4cc44:::benchmark_data_smithi138_240338_object110064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-01T09:16:24.390 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:08.132412+0000", 2023-10-01T09:16:24.390 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 16.164289644, 2023-10-01T09:16:24.390 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14333959600000001, 2023-10-01T09:16:24.390 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.390 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.391 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.391 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.391 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.391 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 110065 2023-10-01T09:16:24.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.391 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.391 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.391 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.132412+0000", 2023-10-01T09:16:24.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.132412+0000", 2023-10-01T09:16:24.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:24.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.132410+0000", 2023-10-01T09:16:24.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4248000000000001e-05 2023-10-01T09:16:24.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.132454+0000", 2023-10-01T09:16:24.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4691e-05 2023-10-01T09:16:24.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.132469+0000", 2023-10-01T09:16:24.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7289999999999997e-06 2023-10-01T09:16:24.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.132475+0000", 2023-10-01T09:16:24.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2834999999999998e-05 2023-10-01T09:16:24.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.394 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.394 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.132528+0000", 2023-10-01T09:16:24.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014906899999999999 2023-10-01T09:16:24.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.394 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.394 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.132677+0000", 2023-10-01T09:16:24.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000176625 2023-10-01T09:16:24.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:16:24.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.132853+0000", 2023-10-01T09:16:24.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035590909999999999 2023-10-01T09:16:24.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.136413+0000", 2023-10-01T09:16:24.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00087328099999999995 2023-10-01T09:16:24.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.137286+0000", 2023-10-01T09:16:24.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13838877499999999 2023-10-01T09:16:24.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275675+0000", 2023-10-01T09:16:24.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4472999999999999e-05 2023-10-01T09:16:24.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275719+0000", 2023-10-01T09:16:24.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2369999999999997e-05 2023-10-01T09:16:24.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275751+0000", 2023-10-01T09:16:24.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14333959600000001 2023-10-01T09:16:24.397 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.397 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.397 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.397 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:119398 5.1 5:897844bd:::benchmark_data_smithi138_240338_object4261:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-01T09:16:24.398 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:19.811869+0000", 2023-10-01T09:16:24.398 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 4.4848320829999997, 2023-10-01T09:16:24.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.138406534, 2023-10-01T09:16:24.398 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.398 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.398 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.398 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.398 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.398 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 119398 2023-10-01T09:16:24.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.398 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.811869+0000", 2023-10-01T09:16:24.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.811869+0000", 2023-10-01T09:16:24.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:16:24.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.811867+0000", 2023-10-01T09:16:24.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2316e-05 2023-10-01T09:16:24.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.400 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.811879+0000", 2023-10-01T09:16:24.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2886000000000001e-05 2023-10-01T09:16:24.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.400 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.811902+0000", 2023-10-01T09:16:24.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2320000000000001e-05 2023-10-01T09:16:24.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.811915+0000", 2023-10-01T09:16:24.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6371000000000003e-05 2023-10-01T09:16:24.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.812011+0000", 2023-10-01T09:16:24.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00018028799999999999 2023-10-01T09:16:24.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.812191+0000", 2023-10-01T09:16:24.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016322400000000001 2023-10-01T09:16:24.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.402 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:16:24.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.812355+0000", 2023-10-01T09:16:24.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0029706910000000001 2023-10-01T09:16:24.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.402 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.815325+0000", 2023-10-01T09:16:24.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13484092 2023-10-01T09:16:24.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950166+0000", 2023-10-01T09:16:24.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1673e-05 2023-10-01T09:16:24.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950198+0000", 2023-10-01T09:16:24.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9468999999999998e-05 2023-10-01T09:16:24.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950237+0000", 2023-10-01T09:16:24.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8665000000000003e-05 2023-10-01T09:16:24.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950276+0000", 2023-10-01T09:16:24.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.138406534 2023-10-01T09:16:24.404 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.404 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.404 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.405 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop_reply(client.34683.0:119398 5.1 e124/106 ondisk, result = 0)", 2023-10-01T09:16:24.405 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:19.816312+0000", 2023-10-01T09:16:24.405 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 4.4803893840000004, 2023-10-01T09:16:24.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.133856369, 2023-10-01T09:16:24.405 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.405 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:16:24.405 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.816312+0000", 2023-10-01T09:16:24.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.816312+0000", 2023-10-01T09:16:24.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:16:24.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.816311+0000", 2023-10-01T09:16:24.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8890000000000004e-06 2023-10-01T09:16:24.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.407 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.407 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.816315+0000", 2023-10-01T09:16:24.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0389999999999999e-06 2023-10-01T09:16:24.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.407 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.407 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.816322+0000", 2023-10-01T09:16:24.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5460000000000002e-06 2023-10-01T09:16:24.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.816327+0000", 2023-10-01T09:16:24.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13381601300000001 2023-10-01T09:16:24.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950143+0000", 2023-10-01T09:16:24.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1061e-05 2023-10-01T09:16:24.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950154+0000", 2023-10-01T09:16:24.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4508999999999999e-05 2023-10-01T09:16:24.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950168+0000", 2023-10-01T09:16:24.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.133856369 2023-10-01T09:16:24.409 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.409 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.409 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.410 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.34683.0:119404 5.c e124/106 5:3a7c8829:::benchmark_data_smithi138_240338_object4267:head v 124'7443, mlcod=124'7441)", 2023-10-01T09:16:24.410 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:19.816407+0000", 2023-10-01T09:16:24.410 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 4.480294078, 2023-10-01T09:16:24.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.146037692, 2023-10-01T09:16:24.410 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.410 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.410 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.410 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.410 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.410 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.816407+0000", 2023-10-01T09:16:24.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.410 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.411 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.411 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.411 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.816407+0000", 2023-10-01T09:16:24.411 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:16:24.411 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.411 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.411 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.411 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.816407+0000", 2023-10-01T09:16:24.411 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9959999999999996e-06 2023-10-01T09:16:24.411 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.411 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.412 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.412 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.816414+0000", 2023-10-01T09:16:24.412 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13374792299999999 2023-10-01T09:16:24.412 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.412 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.412 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.412 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950162+0000", 2023-10-01T09:16:24.412 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0790000000000001e-06 2023-10-01T09:16:24.412 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.412 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.412 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.412 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950170+0000", 2023-10-01T09:16:24.413 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8859000000000002e-05 2023-10-01T09:16:24.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.413 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.413 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950209+0000", 2023-10-01T09:16:24.413 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4076999999999999e-05 2023-10-01T09:16:24.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.413 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.413 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950223+0000", 2023-10-01T09:16:24.413 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012182267 2023-10-01T09:16:24.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.414 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.414 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.414 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.962405+0000", 2023-10-01T09:16:24.414 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0188999999999998e-05 2023-10-01T09:16:24.414 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.414 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.414 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.414 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.962445+0000", 2023-10-01T09:16:24.414 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.146037692 2023-10-01T09:16:24.414 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.414 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.415 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.415 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.415 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.415 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop_reply(client.34683.0:119398 5.1 e124/106 ondisk, result = 0)", 2023-10-01T09:16:24.415 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:19.818371+0000", 2023-10-01T09:16:24.415 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 4.4783301130000002, 2023-10-01T09:16:24.415 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.131908319, 2023-10-01T09:16:24.415 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.415 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:16:24.415 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.415 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.415 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.416 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.818371+0000", 2023-10-01T09:16:24.416 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.416 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.416 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.416 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.416 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.818371+0000", 2023-10-01T09:16:24.416 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:16:24.416 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.416 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.416 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.416 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.818371+0000", 2023-10-01T09:16:24.416 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9590000000000003e-06 2023-10-01T09:16:24.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.417 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.417 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.818376+0000", 2023-10-01T09:16:24.417 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.131761407 2023-10-01T09:16:24.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.417 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.417 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950137+0000", 2023-10-01T09:16:24.417 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.8169999999999994e-06 2023-10-01T09:16:24.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.418 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.418 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950147+0000", 2023-10-01T09:16:24.418 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5512000000000002e-05 2023-10-01T09:16:24.418 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.418 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.418 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.418 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950182+0000", 2023-10-01T09:16:24.418 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4489999999999999e-06 2023-10-01T09:16:24.418 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.418 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.418 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.419 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950191+0000", 2023-10-01T09:16:24.419 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.8949999999999994e-05 2023-10-01T09:16:24.419 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.419 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.419 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.419 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.950280+0000", 2023-10-01T09:16:24.419 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.131908319 2023-10-01T09:16:24.419 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.419 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.419 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.419 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.419 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.420 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:119705 5.e 5:779a2057:::benchmark_data_smithi138_240338_object4568:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-01T09:16:24.420 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:20.269601+0000", 2023-10-01T09:16:24.420 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 4.0271006829999996, 2023-10-01T09:16:24.420 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24117385499999999, 2023-10-01T09:16:24.420 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.420 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.420 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.420 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.420 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.420 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 119705 2023-10-01T09:16:24.420 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.421 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.421 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.421 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.421 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269601+0000", 2023-10-01T09:16:24.421 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.421 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.421 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.421 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.421 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269601+0000", 2023-10-01T09:16:24.421 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:24.421 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.422 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.422 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.422 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269599+0000", 2023-10-01T09:16:24.422 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9090000000000003e-06 2023-10-01T09:16:24.422 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.422 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.422 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.422 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269606+0000", 2023-10-01T09:16:24.422 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.446e-05 2023-10-01T09:16:24.422 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.422 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.423 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.423 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269621+0000", 2023-10-01T09:16:24.423 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0389999999999999e-06 2023-10-01T09:16:24.423 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.423 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.423 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.423 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269628+0000", 2023-10-01T09:16:24.423 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0097709400000000005 2023-10-01T09:16:24.423 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.423 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.423 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.423 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.279399+0000", 2023-10-01T09:16:24.424 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014551399999999999 2023-10-01T09:16:24.424 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.424 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.424 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.424 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.279544+0000", 2023-10-01T09:16:24.424 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000171409 2023-10-01T09:16:24.424 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.424 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.424 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:16:24.424 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.279715+0000", 2023-10-01T09:16:24.424 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0039263379999999997 2023-10-01T09:16:24.425 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.425 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.425 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.425 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.283642+0000", 2023-10-01T09:16:24.425 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012843927 2023-10-01T09:16:24.425 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.425 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.425 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.425 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.296486+0000", 2023-10-01T09:16:24.425 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.214182922 2023-10-01T09:16:24.425 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.425 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.426 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.426 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.510669+0000", 2023-10-01T09:16:24.426 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4887000000000004e-05 2023-10-01T09:16:24.426 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.426 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.426 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.426 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.510734+0000", 2023-10-01T09:16:24.426 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1088999999999999e-05 2023-10-01T09:16:24.426 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.426 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.426 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.427 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.510775+0000", 2023-10-01T09:16:24.427 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24117385499999999 2023-10-01T09:16:24.427 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.427 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.427 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.427 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.427 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.427 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:119730 5.e 5:7b341087:::benchmark_data_smithi138_240338_object4593:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-01T09:16:24.427 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:20.311290+0000", 2023-10-01T09:16:24.427 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 3.9854111539999999, 2023-10-01T09:16:24.427 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20994359400000001, 2023-10-01T09:16:24.428 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.428 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.428 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.428 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.428 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.428 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 119730 2023-10-01T09:16:24.428 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.428 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.428 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.428 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.428 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.311290+0000", 2023-10-01T09:16:24.428 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.429 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.429 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.429 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.429 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.311290+0000", 2023-10-01T09:16:24.429 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:24.429 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.429 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.429 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.429 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.311289+0000", 2023-10-01T09:16:24.429 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6480000000000003e-06 2023-10-01T09:16:24.429 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.430 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.430 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.430 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.311297+0000", 2023-10-01T09:16:24.430 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4042e-05 2023-10-01T09:16:24.430 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.430 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.430 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.430 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.311311+0000", 2023-10-01T09:16:24.430 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0779999999999998e-06 2023-10-01T09:16:24.430 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.430 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.430 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.431 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.311318+0000", 2023-10-01T09:16:24.431 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5347999999999999e-05 2023-10-01T09:16:24.431 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.431 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.431 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.431 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.311373+0000", 2023-10-01T09:16:24.431 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013484599999999999 2023-10-01T09:16:24.431 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.431 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.431 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.431 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.311508+0000", 2023-10-01T09:16:24.432 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016659500000000001 2023-10-01T09:16:24.432 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.432 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.432 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:16:24.432 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.311674+0000", 2023-10-01T09:16:24.432 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035924469999999999 2023-10-01T09:16:24.432 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.432 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.432 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.432 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.315267+0000", 2023-10-01T09:16:24.432 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00044691199999999999 2023-10-01T09:16:24.433 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.433 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.433 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.433 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.315714+0000", 2023-10-01T09:16:24.433 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20542138500000001 2023-10-01T09:16:24.433 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.433 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.433 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.433 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.521135+0000", 2023-10-01T09:16:24.433 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0967000000000002e-05 2023-10-01T09:16:24.433 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.433 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.434 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.434 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.521186+0000", 2023-10-01T09:16:24.434 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7756000000000002e-05 2023-10-01T09:16:24.434 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.434 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.434 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.434 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.521234+0000", 2023-10-01T09:16:24.434 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20994359400000001 2023-10-01T09:16:24.434 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.434 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.434 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.434 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.435 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.435 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:121380 5.a 5:55f57801:::benchmark_data_smithi138_240338_object6243:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:16:24.435 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:22.569549+0000", 2023-10-01T09:16:24.435 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.7271520570000001, 2023-10-01T09:16:24.435 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98289109100000005, 2023-10-01T09:16:24.435 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.435 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.435 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.435 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.435 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.435 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121380 2023-10-01T09:16:24.436 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.436 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.436 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.436 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.436 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569549+0000", 2023-10-01T09:16:24.436 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.436 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.436 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.436 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.436 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569549+0000", 2023-10-01T09:16:24.436 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:24.436 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.437 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.437 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.437 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569548+0000", 2023-10-01T09:16:24.437 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7849999999999992e-06 2023-10-01T09:16:24.437 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.437 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.437 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.437 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569556+0000", 2023-10-01T09:16:24.437 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5118e-05 2023-10-01T09:16:24.437 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.437 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.438 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.438 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569571+0000", 2023-10-01T09:16:24.438 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0104e-05 2023-10-01T09:16:24.438 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.438 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.438 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.438 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569581+0000", 2023-10-01T09:16:24.438 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1700999999999997e-05 2023-10-01T09:16:24.438 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.438 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.438 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.439 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569653+0000", 2023-10-01T09:16:24.439 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4584e-05 2023-10-01T09:16:24.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:24.440 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.440 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.440 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:16:24.440 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569667+0000", 2023-10-01T09:16:24.440 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.96702125500000002 2023-10-01T09:16:24.440 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.440 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.440 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.440 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.536688+0000", 2023-10-01T09:16:24.440 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0026996390000000002 2023-10-01T09:16:24.440 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.441 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.441 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.441 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.539388+0000", 2023-10-01T09:16:24.441 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014500400000000001 2023-10-01T09:16:24.441 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.441 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.441 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:16:24.441 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.539533+0000", 2023-10-01T09:16:24.441 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0075687469999999998 2023-10-01T09:16:24.441 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.441 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.441 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.442 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.547102+0000", 2023-10-01T09:16:24.442 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.005176333 2023-10-01T09:16:24.442 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.442 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.442 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.442 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.552278+0000", 2023-10-01T09:16:24.442 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9648999999999998e-05 2023-10-01T09:16:24.442 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.442 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.442 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.442 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.552328+0000", 2023-10-01T09:16:24.443 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8442000000000006e-05 2023-10-01T09:16:24.443 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.443 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.443 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.443 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.552396+0000", 2023-10-01T09:16:24.443 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4354999999999998e-05 2023-10-01T09:16:24.443 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.443 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.443 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.443 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.552441+0000", 2023-10-01T09:16:24.443 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98289109100000005 2023-10-01T09:16:24.443 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.444 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.444 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.444 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.444 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.444 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:121383 5.a 5:5a1d8215:::benchmark_data_smithi138_240338_object6246:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:16:24.444 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:22.574503+0000", 2023-10-01T09:16:24.444 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.722198181, 2023-10-01T09:16:24.444 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98544083199999999, 2023-10-01T09:16:24.444 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:24.444 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:24.444 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:24.444 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:24.445 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:24.445 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121383 2023-10-01T09:16:24.445 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.445 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:24.445 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.445 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:24.445 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574503+0000", 2023-10-01T09:16:24.445 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:24.445 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.445 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.445 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:24.445 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574503+0000", 2023-10-01T09:16:24.446 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999924 2023-10-01T09:16:24.446 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.446 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.446 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:24.446 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574496+0000", 2023-10-01T09:16:24.446 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8362000000000001e-05 2023-10-01T09:16:24.446 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.446 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.446 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:24.446 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574514+0000", 2023-10-01T09:16:24.446 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7633000000000001e-05 2023-10-01T09:16:24.446 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.447 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.447 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:24.447 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574542+0000", 2023-10-01T09:16:24.447 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.331e-05 2023-10-01T09:16:24.447 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.447 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.447 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:24.447 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574555+0000", 2023-10-01T09:16:24.447 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6639999999999996e-05 2023-10-01T09:16:24.447 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.447 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.447 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.448 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574651+0000", 2023-10-01T09:16:24.448 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7617e-05 2023-10-01T09:16:24.448 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.448 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.448 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:16:24.448 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574679+0000", 2023-10-01T09:16:24.448 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.968492256 2023-10-01T09:16:24.448 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.448 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.448 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:24.448 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.543171+0000", 2023-10-01T09:16:24.448 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012240600000000001 2023-10-01T09:16:24.449 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.449 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.449 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:24.449 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.543294+0000", 2023-10-01T09:16:24.449 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016234499999999999 2023-10-01T09:16:24.449 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.449 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.449 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:16:24.449 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.543456+0000", 2023-10-01T09:16:24.449 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003924332 2023-10-01T09:16:24.449 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.449 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.450 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:24.450 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.547380+0000", 2023-10-01T09:16:24.450 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012435430000000001 2023-10-01T09:16:24.450 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.450 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.450 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.450 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.559816+0000", 2023-10-01T09:16:24.450 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1671000000000001e-05 2023-10-01T09:16:24.450 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.450 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.450 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:24.450 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.559858+0000", 2023-10-01T09:16:24.451 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9632000000000003e-05 2023-10-01T09:16:24.451 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.451 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.451 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:24.451 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.559897+0000", 2023-10-01T09:16:24.451 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6953999999999998e-05 2023-10-01T09:16:24.451 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:24.451 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:24.451 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:24.451 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.559944+0000", 2023-10-01T09:16:24.451 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98544083199999999 2023-10-01T09:16:24.451 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.452 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.452 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.452 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:24.452 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:24.452 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:16:24.452 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:24.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:24.540 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:16:24.554 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:24.611 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:24.716 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:24.720 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[192513]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:16:24.720 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[192513]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.720 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[192513]: osdmap e126: 8 total, 8 up, 8 in 2023-10-01T09:16:24.720 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[192513]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.721 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[192513]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.721 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[192513]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.721 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[192513]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.721 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[192513]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.721 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[192513]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.721 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/629687022' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:24.721 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[185886]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:16:24.721 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[185886]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.721 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[185886]: osdmap e126: 8 total, 8 up, 8 in 2023-10-01T09:16:24.722 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[185886]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.722 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[185886]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.722 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[185886]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.722 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[185886]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.722 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[185886]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.722 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[185886]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.722 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/629687022' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:24.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:24.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:24 smithi190 ceph-mon[160435]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:16:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:24 smithi190 ceph-mon[160435]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:24 smithi190 ceph-mon[160435]: osdmap e126: 8 total, 8 up, 8 in 2023-10-01T09:16:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:24 smithi190 ceph-mon[160435]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:24 smithi190 ceph-mon[160435]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:24 smithi190 ceph-mon[160435]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:24 smithi190 ceph-mon[160435]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:24 smithi190 ceph-mon[160435]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:24 smithi190 ceph-mon[160435]: failed to encode map e126 with expected crc 2023-10-01T09:16:24.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/629687022' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:24.824 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:24.864 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:25.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:16:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:16:24] "GET /metrics HTTP/1.1" 200 36327 "" "Prometheus/2.33.4" 2023-10-01T09:16:25.219 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:16:25.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:25 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:25.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:25 smithi190 ceph-mon[160435]: failed to encode map e127 with expected crc 2023-10-01T09:16:25.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:25 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/629687022' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:16:25.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:25 smithi190 ceph-mon[160435]: osdmap e127: 8 total, 8 up, 8 in 2023-10-01T09:16:25.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:25 smithi190 ceph-mon[160435]: failed to encode map e127 with expected crc 2023-10-01T09:16:25.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:25 smithi190 ceph-mon[160435]: failed to encode map e127 with expected crc 2023-10-01T09:16:25.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:25 smithi190 ceph-mon[160435]: failed to encode map e127 with expected crc 2023-10-01T09:16:25.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:25.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:16:25.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:25 smithi190 ceph-mon[160435]: pgmap v300: 89 pgs: 1 peering, 88 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 436 op/s 2023-10-01T09:16:25.656 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:25.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:25.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:25.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:25.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[192513]: failed to encode map e127 with expected crc 2023-10-01T09:16:25.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/629687022' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:16:25.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[192513]: osdmap e127: 8 total, 8 up, 8 in 2023-10-01T09:16:25.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[192513]: failed to encode map e127 with expected crc 2023-10-01T09:16:25.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[192513]: failed to encode map e127 with expected crc 2023-10-01T09:16:25.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[192513]: failed to encode map e127 with expected crc 2023-10-01T09:16:25.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:25.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:16:25.660 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[192513]: pgmap v300: 89 pgs: 1 peering, 88 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 436 op/s 2023-10-01T09:16:25.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:25.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[185886]: failed to encode map e127 with expected crc 2023-10-01T09:16:25.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/629687022' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:16:25.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[185886]: osdmap e127: 8 total, 8 up, 8 in 2023-10-01T09:16:25.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[185886]: failed to encode map e127 with expected crc 2023-10-01T09:16:25.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[185886]: failed to encode map e127 with expected crc 2023-10-01T09:16:25.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[185886]: failed to encode map e127 with expected crc 2023-10-01T09:16:25.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:25.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:16:25.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:25 smithi138 ceph-mon[185886]: pgmap v300: 89 pgs: 1 peering, 88 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 436 op/s 2023-10-01T09:16:25.760 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:25.783 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:16:25.823 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:25.956 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:25.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:25.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:26.062 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:26.138 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:26.245 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:26.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:26.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:26.354 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:26.415 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:26.524 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:26.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:26.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:26.549 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:16:26.574 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:16:26.599 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:16:26.628 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:26.633 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:16:26.664 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:16:26.704 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:26.706 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:16:26.737 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:16:26.775 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:26.777 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:16:26.778 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:16:26.778 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:16:26.778 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:16:26.778 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:16:26.779 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:16:26.779 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:16:26.779 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:16:26.779 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:16:26.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:26.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:26.883 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:26.912 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:27.136 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:27.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:27.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:27.245 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:27.281 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:27.354 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:27.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:27.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:27.462 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:27.489 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:27 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3370655981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:27.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:27 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3370655981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:27.492 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:27.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:27 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3370655981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:27.721 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:27.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:27.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:27.830 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:16:27.862 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:27.952 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:16:27.952 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:16:27.958 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:16:27.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:27.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:16:27.989 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:16:28.069 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:28.118 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:28.220 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:28.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:28.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:28.274 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:28 smithi190 ceph-mon[160435]: pgmap v301: 89 pgs: 1 peering, 88 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-10-01T09:16:28.328 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:28.385 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:28.428 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:28.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:28.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:28.459 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:16:28.459 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:16:28.460 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-01T09:16:28.460 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:16:28.460 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:16:28.460 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-01T09:16:28.511 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:28 smithi138 ceph-mon[192513]: pgmap v301: 89 pgs: 1 peering, 88 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-10-01T09:16:28.512 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:28 smithi138 ceph-mon[185886]: pgmap v301: 89 pgs: 1 peering, 88 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-10-01T09:16:28.531 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:28.545 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:28.608 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:28.743 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:28.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:28.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:28.848 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:28.883 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T09:16:28.927 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:29.058 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:29.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:29.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:29.170 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:29.238 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:29.391 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:29.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:29.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:29.496 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:29.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:29 smithi138 ceph-mon[185886]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:16:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:29 smithi138 ceph-mon[192513]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:16:29.581 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:29.635 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:16:29.657 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:29.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:29.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:29.728 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:16:29.763 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:16:29.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:29 smithi190 ceph-mon[160435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:16:29.821 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:30.104 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:16:30.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:30.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:16:30.200 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:30.208 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:30.260 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:30.354 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:30.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:30.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:30.421 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:30 smithi138 ceph-mon[185886]: pgmap v302: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-01T09:16:30.428 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:30 smithi138 ceph-mon[192513]: pgmap v302: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-01T09:16:30.464 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:16:30.492 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:30.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:30 smithi190 ceph-mon[160435]: pgmap v302: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-01T09:16:30.794 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:16:30.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:30.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:16:30.926 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:30.979 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:31.011 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:31.270 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:31.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:31.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:31.374 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:31.410 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:31.437 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:16:31.506 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:31.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:31.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:31.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/733226384' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:16:31.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/733226384' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:16:31.612 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:31.613 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:16:31.613 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:16:31.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/733226384' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:16:31.694 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:31.908 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:31.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:31.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:32.015 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:16:32.065 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:32.588 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T09:16:32.732 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:16:32.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:32.744 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:32 smithi138 ceph-mon[185886]: pgmap v303: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-10-01T09:16:32.744 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:32 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:32.745 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:32 smithi138 ceph-mon[185886]: failed to encode map e128 with expected crc 2023-10-01T09:16:32.745 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:32 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/733226384' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:16:32.745 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:32 smithi138 ceph-mon[185886]: osdmap e128: 8 total, 8 up, 8 in 2023-10-01T09:16:32.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:32 smithi138 ceph-mon[192513]: pgmap v303: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-10-01T09:16:32.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:32 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:32.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:32 smithi138 ceph-mon[192513]: failed to encode map e128 with expected crc 2023-10-01T09:16:32.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:32 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/733226384' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:16:32.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:32 smithi138 ceph-mon[192513]: osdmap e128: 8 total, 8 up, 8 in 2023-10-01T09:16:32.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:16:32.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:32 smithi190 ceph-mon[160435]: pgmap v303: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-10-01T09:16:32.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:32 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:32.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:32 smithi190 ceph-mon[160435]: failed to encode map e128 with expected crc 2023-10-01T09:16:32.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:32 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/733226384' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:16:32.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:32 smithi190 ceph-mon[160435]: osdmap e128: 8 total, 8 up, 8 in 2023-10-01T09:16:32.847 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:16:32.896 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:33.263 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:16:33.263 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T09:16:33.263 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:16:33.291 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:16:33.296 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:16:33.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:33.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:16:33.400 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:33.492 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:33.586 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:33.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:33.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:33.690 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:33.736 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:34.579 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:34.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:34.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:34.685 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:34.762 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:34 smithi190 ceph-mon[160435]: pgmap v305: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-10-01T09:16:34.784 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:34.796 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:34 smithi138 ceph-mon[192513]: pgmap v305: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-10-01T09:16:34.796 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:34 smithi138 ceph-mon[185886]: pgmap v305: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-10-01T09:16:34.861 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:16:34.939 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:34.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:34.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:35.048 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:16:35.072 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:16:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:16:34] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.33.4" 2023-10-01T09:16:35.109 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:35.116 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:16:35.155 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:16:35.194 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:16:35.203 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:16:35.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:35.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:16:35.226 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:16:35.263 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:16:35.312 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:35.316 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:16:35.353 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:16:35.353 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:16:35.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:16:35.353 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:16:35.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.354 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:3616 5.7 5:ee7d02b5:::benchmark_data_smithi138_240338_object3615:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:16:35.354 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:14:45.505618+0000", 2023-10-01T09:16:35.354 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 109.843792583, 2023-10-01T09:16:35.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17920050500000001, 2023-10-01T09:16:35.354 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.354 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.354 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.354 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.354 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.354 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 3616 2023-10-01T09:16:35.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.354 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.505618+0000", 2023-10-01T09:16:35.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.505618+0000", 2023-10-01T09:16:35.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:16:35.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.505618+0000", 2023-10-01T09:16:35.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9100000000000002e-05 2023-10-01T09:16:35.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.505657+0000", 2023-10-01T09:16:35.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0088999999999999e-05 2023-10-01T09:16:35.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.505667+0000", 2023-10-01T09:16:35.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0299999999999999e-06 2023-10-01T09:16:35.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.505673+0000", 2023-10-01T09:16:35.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00060276299999999998 2023-10-01T09:16:35.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.506276+0000", 2023-10-01T09:16:35.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012399199999999999 2023-10-01T09:16:35.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.506400+0000", 2023-10-01T09:16:35.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015341500000000001 2023-10-01T09:16:35.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.506553+0000", 2023-10-01T09:16:35.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001506294 2023-10-01T09:16:35.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.508059+0000", 2023-10-01T09:16:35.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00091542499999999998 2023-10-01T09:16:35.359 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.359 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.359 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.359 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.508975+0000", 2023-10-01T09:16:35.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17569453300000001 2023-10-01T09:16:35.359 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.359 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.359 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.359 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.684669+0000", 2023-10-01T09:16:35.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000104607 2023-10-01T09:16:35.359 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.360 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.360 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.684774+0000", 2023-10-01T09:16:35.360 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5102000000000003e-05 2023-10-01T09:16:35.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.360 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.360 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.684819+0000", 2023-10-01T09:16:35.360 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17920050500000001 2023-10-01T09:16:35.360 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.360 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.360 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.361 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:3624 5.7 5:ec9dd2e5:::benchmark_data_smithi138_240338_object3623:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:16:35.361 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:14:45.506358+0000", 2023-10-01T09:16:35.361 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 109.84305339399999, 2023-10-01T09:16:35.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17860648300000001, 2023-10-01T09:16:35.361 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.361 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.361 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.361 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.362 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.363 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 3624 2023-10-01T09:16:35.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.363 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.506358+0000", 2023-10-01T09:16:35.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.506358+0000", 2023-10-01T09:16:35.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:16:35.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.506357+0000", 2023-10-01T09:16:35.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3362999999999999e-05 2023-10-01T09:16:35.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.506390+0000", 2023-10-01T09:16:35.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1717e-05 2023-10-01T09:16:35.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.506402+0000", 2023-10-01T09:16:35.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8689999999999998e-06 2023-10-01T09:16:35.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.506409+0000", 2023-10-01T09:16:35.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0016714259999999999 2023-10-01T09:16:35.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.508080+0000", 2023-10-01T09:16:35.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000103541 2023-10-01T09:16:35.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.508184+0000", 2023-10-01T09:16:35.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014856099999999999 2023-10-01T09:16:35.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.508332+0000", 2023-10-01T09:16:35.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00065533300000000002 2023-10-01T09:16:35.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.367 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.367 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.508988+0000", 2023-10-01T09:16:35.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00032273799999999999 2023-10-01T09:16:35.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.367 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.367 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.509310+0000", 2023-10-01T09:16:35.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17555533100000001 2023-10-01T09:16:35.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.368 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.368 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.684866+0000", 2023-10-01T09:16:35.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7295e-05 2023-10-01T09:16:35.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.368 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.368 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.684913+0000", 2023-10-01T09:16:35.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1103000000000002e-05 2023-10-01T09:16:35.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.684964+0000", 2023-10-01T09:16:35.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17860648300000001 2023-10-01T09:16:35.369 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.369 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.369 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.369 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:3639 5.7 5:e24aa857:::benchmark_data_smithi138_240338_object3638:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e111)", 2023-10-01T09:16:35.369 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:14:45.510269+0000", 2023-10-01T09:16:35.369 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 109.839142383, 2023-10-01T09:16:35.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17529436200000001, 2023-10-01T09:16:35.370 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.370 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.370 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.370 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.370 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.370 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 3639 2023-10-01T09:16:35.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.370 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.510269+0000", 2023-10-01T09:16:35.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.510269+0000", 2023-10-01T09:16:35.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:35.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.510267+0000", 2023-10-01T09:16:35.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016795300000000001 2023-10-01T09:16:35.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.510435+0000", 2023-10-01T09:16:35.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2082000000000001e-05 2023-10-01T09:16:35.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.510447+0000", 2023-10-01T09:16:35.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7190000000000004e-06 2023-10-01T09:16:35.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.510454+0000", 2023-10-01T09:16:35.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6026999999999998e-05 2023-10-01T09:16:35.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.510500+0000", 2023-10-01T09:16:35.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010877799999999999 2023-10-01T09:16:35.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.510609+0000", 2023-10-01T09:16:35.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000164375 2023-10-01T09:16:35.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.374 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.510773+0000", 2023-10-01T09:16:35.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00049949800000000002 2023-10-01T09:16:35.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.374 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.511273+0000", 2023-10-01T09:16:35.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00033447799999999998 2023-10-01T09:16:35.375 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.375 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.375 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.375 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.511607+0000", 2023-10-01T09:16:35.375 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.173863183 2023-10-01T09:16:35.375 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.375 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.375 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.375 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.685470+0000", 2023-10-01T09:16:35.375 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4962e-05 2023-10-01T09:16:35.375 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.375 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.376 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.685525+0000", 2023-10-01T09:16:35.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7758e-05 2023-10-01T09:16:35.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.376 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:14:45.685563+0000", 2023-10-01T09:16:35.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17529436200000001 2023-10-01T09:16:35.376 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.376 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.376 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.377 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:59034 5.7 5:e096144b:::benchmark_data_smithi138_240338_object59033:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:16:35.377 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:15:16.277083+0000", 2023-10-01T09:16:35.377 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 79.072327584999996, 2023-10-01T09:16:35.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22331911600000001, 2023-10-01T09:16:35.377 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.377 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.377 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.377 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.378 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.378 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 59034 2023-10-01T09:16:35.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.378 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277083+0000", 2023-10-01T09:16:35.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277083+0000", 2023-10-01T09:16:35.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:16:35.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277082+0000", 2023-10-01T09:16:35.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5989999999999998e-05 2023-10-01T09:16:35.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.380 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277138+0000", 2023-10-01T09:16:35.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6120000000000005e-06 2023-10-01T09:16:35.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277148+0000", 2023-10-01T09:16:35.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3259999999999997e-06 2023-10-01T09:16:35.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277152+0000", 2023-10-01T09:16:35.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3316999999999998e-05 2023-10-01T09:16:35.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277206+0000", 2023-10-01T09:16:35.382 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014156800000000001 2023-10-01T09:16:35.382 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.382 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.382 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.382 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277347+0000", 2023-10-01T09:16:35.382 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000183528 2023-10-01T09:16:35.382 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.382 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.382 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.382 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277531+0000", 2023-10-01T09:16:35.382 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0046728500000000001 2023-10-01T09:16:35.383 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.383 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.383 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.383 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.282203+0000", 2023-10-01T09:16:35.383 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017041100000000001 2023-10-01T09:16:35.383 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.383 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.383 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.383 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.282374+0000", 2023-10-01T09:16:35.383 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.217918533 2023-10-01T09:16:35.383 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.383 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.500292+0000", 2023-10-01T09:16:35.384 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7150999999999995e-05 2023-10-01T09:16:35.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.500360+0000", 2023-10-01T09:16:35.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2923999999999998e-05 2023-10-01T09:16:35.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.500403+0000", 2023-10-01T09:16:35.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22331911600000001 2023-10-01T09:16:35.385 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.385 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.385 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.386 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:59039 5.7 5:e088c7a4:::benchmark_data_smithi138_240338_object59038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:16:35.386 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:15:16.277173+0000", 2023-10-01T09:16:35.386 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 79.072237670999996, 2023-10-01T09:16:35.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22522009000000001, 2023-10-01T09:16:35.386 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.386 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.386 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.386 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.387 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.387 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 59039 2023-10-01T09:16:35.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.387 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277173+0000", 2023-10-01T09:16:35.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277173+0000", 2023-10-01T09:16:35.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:16:35.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.388 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277173+0000", 2023-10-01T09:16:35.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5589000000000001e-05 2023-10-01T09:16:35.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.388 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277208+0000", 2023-10-01T09:16:35.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9999999999999996e-06 2023-10-01T09:16:35.389 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.389 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.389 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.389 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277216+0000", 2023-10-01T09:16:35.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6299999999999997e-06 2023-10-01T09:16:35.389 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.389 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.389 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.389 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277221+0000", 2023-10-01T09:16:35.390 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0023609479999999999 2023-10-01T09:16:35.390 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.390 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.390 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.390 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.279582+0000", 2023-10-01T09:16:35.390 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012953199999999999 2023-10-01T09:16:35.390 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.390 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.390 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.390 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.279712+0000", 2023-10-01T09:16:35.390 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000162285 2023-10-01T09:16:35.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.391 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.391 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.279874+0000", 2023-10-01T09:16:35.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0036032429999999999 2023-10-01T09:16:35.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.391 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.391 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.283477+0000", 2023-10-01T09:16:35.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3853000000000003e-05 2023-10-01T09:16:35.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.283511+0000", 2023-10-01T09:16:35.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.218782168 2023-10-01T09:16:35.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.502293+0000", 2023-10-01T09:16:35.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9048000000000001e-05 2023-10-01T09:16:35.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.502352+0000", 2023-10-01T09:16:35.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1251999999999997e-05 2023-10-01T09:16:35.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.502393+0000", 2023-10-01T09:16:35.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22522009000000001 2023-10-01T09:16:35.393 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.393 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.394 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.394 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:104858 5.7 5:e2ba0b8c:::benchmark_data_smithi138_240338_object104857:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-01T09:16:35.394 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:01.525198+0000", 2023-10-01T09:16:35.394 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 33.824213008000001, 2023-10-01T09:16:35.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.268526457, 2023-10-01T09:16:35.394 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.394 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.394 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.394 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.395 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.395 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 104858 2023-10-01T09:16:35.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.395 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.525198+0000", 2023-10-01T09:16:35.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.525198+0000", 2023-10-01T09:16:35.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:35.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.525196+0000", 2023-10-01T09:16:35.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7395000000000002e-05 2023-10-01T09:16:35.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.525254+0000", 2023-10-01T09:16:35.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0685e-05 2023-10-01T09:16:35.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.525265+0000", 2023-10-01T09:16:35.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7049999999999996e-06 2023-10-01T09:16:35.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.525270+0000", 2023-10-01T09:16:35.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0051863339999999999 2023-10-01T09:16:35.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.530457+0000", 2023-10-01T09:16:35.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000136816 2023-10-01T09:16:35.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.530593+0000", 2023-10-01T09:16:35.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014115199999999999 2023-10-01T09:16:35.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.530735+0000", 2023-10-01T09:16:35.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003424552 2023-10-01T09:16:35.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.534159+0000", 2023-10-01T09:16:35.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.048096319999999998 2023-10-01T09:16:35.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.582255+0000", 2023-10-01T09:16:35.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.211366785 2023-10-01T09:16:35.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.793622+0000", 2023-10-01T09:16:35.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8789000000000002e-05 2023-10-01T09:16:35.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.402 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.793681+0000", 2023-10-01T09:16:35.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3476000000000002e-05 2023-10-01T09:16:35.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.402 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.793724+0000", 2023-10-01T09:16:35.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.268526457 2023-10-01T09:16:35.402 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.402 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.402 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.403 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:107106 5.7 5:ea0f488f:::benchmark_data_smithi138_240338_object107105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-01T09:16:35.403 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:04.528562+0000", 2023-10-01T09:16:35.403 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 30.820848869999999, 2023-10-01T09:16:35.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21855391800000001, 2023-10-01T09:16:35.403 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.403 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.403 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.403 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.403 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.404 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 107106 2023-10-01T09:16:35.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.404 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528562+0000", 2023-10-01T09:16:35.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528562+0000", 2023-10-01T09:16:35.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:35.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528561+0000", 2023-10-01T09:16:35.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.32e-05 2023-10-01T09:16:35.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528604+0000", 2023-10-01T09:16:35.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4414e-05 2023-10-01T09:16:35.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528618+0000", 2023-10-01T09:16:35.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.353e-06 2023-10-01T09:16:35.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.407 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528625+0000", 2023-10-01T09:16:35.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7601e-05 2023-10-01T09:16:35.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.407 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.407 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528672+0000", 2023-10-01T09:16:35.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5357000000000002e-05 2023-10-01T09:16:35.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528768+0000", 2023-10-01T09:16:35.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010036500000000001 2023-10-01T09:16:35.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528868+0000", 2023-10-01T09:16:35.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038284339999999999 2023-10-01T09:16:35.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.532696+0000", 2023-10-01T09:16:35.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0018256360000000001 2023-10-01T09:16:35.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.534522+0000", 2023-10-01T09:16:35.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.212494879 2023-10-01T09:16:35.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.410 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.747017+0000", 2023-10-01T09:16:35.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5136999999999997e-05 2023-10-01T09:16:35.410 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.410 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.410 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.410 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.747072+0000", 2023-10-01T09:16:35.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4032999999999997e-05 2023-10-01T09:16:35.410 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.410 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.410 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.410 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.747116+0000", 2023-10-01T09:16:35.411 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21855391800000001 2023-10-01T09:16:35.411 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.411 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.411 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.411 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.411 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.411 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:110043 5.7 5:e22f4a5a:::benchmark_data_smithi138_240338_object110042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-01T09:16:35.411 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:08.040537+0000", 2023-10-01T09:16:35.411 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 27.308873968, 2023-10-01T09:16:35.411 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22794239899999999, 2023-10-01T09:16:35.411 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.412 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.412 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.412 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.412 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.412 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 110043 2023-10-01T09:16:35.412 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.412 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.412 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.412 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.412 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.040537+0000", 2023-10-01T09:16:35.412 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.413 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.413 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.040537+0000", 2023-10-01T09:16:35.413 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:16:35.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.413 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.413 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.040536+0000", 2023-10-01T09:16:35.413 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2589000000000003e-05 2023-10-01T09:16:35.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.414 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.414 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.040569+0000", 2023-10-01T09:16:35.414 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.481e-06 2023-10-01T09:16:35.414 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.414 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.414 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.414 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.040576+0000", 2023-10-01T09:16:35.414 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4029999999999999e-06 2023-10-01T09:16:35.414 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.414 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.415 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.415 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.040580+0000", 2023-10-01T09:16:35.415 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0018192690000000001 2023-10-01T09:16:35.415 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.415 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.415 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.415 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.042399+0000", 2023-10-01T09:16:35.415 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012977400000000001 2023-10-01T09:16:35.415 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.415 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.415 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.415 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.042529+0000", 2023-10-01T09:16:35.416 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010893800000000001 2023-10-01T09:16:35.416 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.416 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.416 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.416 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.042638+0000", 2023-10-01T09:16:35.416 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0036797980000000002 2023-10-01T09:16:35.416 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.416 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.416 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.416 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.046318+0000", 2023-10-01T09:16:35.416 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.008435929 2023-10-01T09:16:35.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.417 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.417 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.054753+0000", 2023-10-01T09:16:35.417 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21356623899999999 2023-10-01T09:16:35.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.417 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.417 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.268320+0000", 2023-10-01T09:16:35.417 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000129014 2023-10-01T09:16:35.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.418 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.418 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.268449+0000", 2023-10-01T09:16:35.418 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0658999999999999e-05 2023-10-01T09:16:35.418 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.418 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.418 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.418 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.268479+0000", 2023-10-01T09:16:35.418 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22794239899999999 2023-10-01T09:16:35.418 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.418 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.419 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.419 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.419 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.419 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:110059 5.7 5:e180c101:::benchmark_data_smithi138_240338_object110058:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-01T09:16:35.419 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:08.050617+0000", 2023-10-01T09:16:35.419 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 27.298793840999998, 2023-10-01T09:16:35.419 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22482184799999999, 2023-10-01T09:16:35.419 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.419 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.419 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.419 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.420 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.420 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 110059 2023-10-01T09:16:35.420 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.420 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.420 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.420 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.420 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050617+0000", 2023-10-01T09:16:35.420 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.420 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.420 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.420 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.420 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050617+0000", 2023-10-01T09:16:35.421 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:35.421 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.421 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.421 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.421 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050616+0000", 2023-10-01T09:16:35.421 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2482999999999999e-05 2023-10-01T09:16:35.421 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.421 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.421 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.421 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050668+0000", 2023-10-01T09:16:35.421 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5464e-05 2023-10-01T09:16:35.422 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.422 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.422 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.422 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050683+0000", 2023-10-01T09:16:35.422 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0600000000000002e-06 2023-10-01T09:16:35.422 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.422 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.422 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.422 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050691+0000", 2023-10-01T09:16:35.422 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9706999999999997e-05 2023-10-01T09:16:35.422 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.423 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.423 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.423 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050740+0000", 2023-10-01T09:16:35.423 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000140378 2023-10-01T09:16:35.423 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.423 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.423 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.423 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050881+0000", 2023-10-01T09:16:35.423 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00021324199999999999 2023-10-01T09:16:35.423 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.423 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.423 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.424 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.051094+0000", 2023-10-01T09:16:35.424 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0072607069999999999 2023-10-01T09:16:35.424 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.424 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.424 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.424 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.058355+0000", 2023-10-01T09:16:35.424 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7947e-05 2023-10-01T09:16:35.424 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.424 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.424 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.424 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.058402+0000", 2023-10-01T09:16:35.424 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21694487800000001 2023-10-01T09:16:35.425 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.425 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.425 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.425 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275347+0000", 2023-10-01T09:16:35.425 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1925999999999997e-05 2023-10-01T09:16:35.425 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.425 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.425 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.425 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275399+0000", 2023-10-01T09:16:35.425 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9684000000000002e-05 2023-10-01T09:16:35.425 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.426 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.426 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.426 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275439+0000", 2023-10-01T09:16:35.426 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22482184799999999 2023-10-01T09:16:35.426 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.426 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.426 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.426 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.426 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.426 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:119373 5.7 5:ea985ceb:::benchmark_data_smithi138_240338_object4236:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-01T09:16:35.426 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:19.781138+0000", 2023-10-01T09:16:35.426 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 15.568273338999999, 2023-10-01T09:16:35.427 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17033890900000001, 2023-10-01T09:16:35.427 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.427 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.427 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.427 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.427 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.427 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 119373 2023-10-01T09:16:35.427 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.427 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.427 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.427 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.427 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.781138+0000", 2023-10-01T09:16:35.428 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.428 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.428 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.428 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.428 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.781138+0000", 2023-10-01T09:16:35.428 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:35.428 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.428 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.428 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.428 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.781136+0000", 2023-10-01T09:16:35.428 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8359999999999997e-06 2023-10-01T09:16:35.428 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.429 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.429 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.429 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.781142+0000", 2023-10-01T09:16:35.429 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6209999999999999e-06 2023-10-01T09:16:35.429 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.429 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.429 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.429 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.781151+0000", 2023-10-01T09:16:35.429 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3229999999999997e-06 2023-10-01T09:16:35.429 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.429 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.429 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.430 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.781157+0000", 2023-10-01T09:16:35.430 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6675000000000003e-05 2023-10-01T09:16:35.430 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.430 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.430 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.430 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.781203+0000", 2023-10-01T09:16:35.430 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000120766 2023-10-01T09:16:35.430 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.430 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.430 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.430 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.781324+0000", 2023-10-01T09:16:35.430 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013834900000000001 2023-10-01T09:16:35.431 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.431 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.431 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.431 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.781463+0000", 2023-10-01T09:16:35.431 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003456724 2023-10-01T09:16:35.431 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.431 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.431 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.431 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.784919+0000", 2023-10-01T09:16:35.431 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010812197000000001 2023-10-01T09:16:35.431 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.431 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.432 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.432 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.795732+0000", 2023-10-01T09:16:35.432 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.155651292 2023-10-01T09:16:35.432 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.432 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.432 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.432 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.951383+0000", 2023-10-01T09:16:35.432 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9608999999999999e-05 2023-10-01T09:16:35.432 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.432 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.432 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.432 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.951432+0000", 2023-10-01T09:16:35.433 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4124e-05 2023-10-01T09:16:35.433 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.433 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.433 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.433 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.951477+0000", 2023-10-01T09:16:35.433 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17033890900000001 2023-10-01T09:16:35.433 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.433 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.433 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.433 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.433 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.433 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:119374 5.7 5:ec064b72:::benchmark_data_smithi138_240338_object4237:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-01T09:16:35.434 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:19.785643+0000", 2023-10-01T09:16:35.434 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 15.563767899, 2023-10-01T09:16:35.434 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.165913485, 2023-10-01T09:16:35.434 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.434 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.434 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.434 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.434 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.434 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 119374 2023-10-01T09:16:35.434 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.434 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.434 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.435 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.435 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.785643+0000", 2023-10-01T09:16:35.435 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.435 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.435 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.435 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.435 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.785643+0000", 2023-10-01T09:16:35.435 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:16:35.435 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.435 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.435 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.435 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.785641+0000", 2023-10-01T09:16:35.436 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1720000000000007e-06 2023-10-01T09:16:35.436 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.436 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.436 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.436 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.785649+0000", 2023-10-01T09:16:35.436 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1921e-05 2023-10-01T09:16:35.436 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.436 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.436 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.436 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.785661+0000", 2023-10-01T09:16:35.436 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1580000000000002e-06 2023-10-01T09:16:35.437 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.437 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.437 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.437 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.785667+0000", 2023-10-01T09:16:35.437 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7482999999999999e-05 2023-10-01T09:16:35.437 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.437 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.437 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.437 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.785715+0000", 2023-10-01T09:16:35.437 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000121583 2023-10-01T09:16:35.437 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.437 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.438 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.438 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.785836+0000", 2023-10-01T09:16:35.438 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000127384 2023-10-01T09:16:35.438 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.438 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.438 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.438 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.785964+0000", 2023-10-01T09:16:35.438 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0097720199999999993 2023-10-01T09:16:35.438 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.438 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.438 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.438 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.795736+0000", 2023-10-01T09:16:35.439 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6509999999999999e-05 2023-10-01T09:16:35.439 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.439 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.439 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.439 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.795752+0000", 2023-10-01T09:16:35.439 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.155755693 2023-10-01T09:16:35.439 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.439 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.439 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.439 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.951508+0000", 2023-10-01T09:16:35.439 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8005e-05 2023-10-01T09:16:35.440 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.440 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.440 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.440 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.951536+0000", 2023-10-01T09:16:35.440 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0588999999999999e-05 2023-10-01T09:16:35.440 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.440 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.440 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.440 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.951557+0000", 2023-10-01T09:16:35.440 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.165913485 2023-10-01T09:16:35.440 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.440 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.441 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.441 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.441 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.441 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:119381 5.f 5:f15dd7d7:::benchmark_data_smithi138_240338_object4244:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-01T09:16:35.441 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:19.788493+0000", 2023-10-01T09:16:35.441 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 15.560917687, 2023-10-01T09:16:35.441 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.167028658, 2023-10-01T09:16:35.441 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.441 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.441 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.441 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.441 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.442 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 119381 2023-10-01T09:16:35.442 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.442 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.442 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.442 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.442 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.788493+0000", 2023-10-01T09:16:35.442 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.442 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.442 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.442 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.442 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.788493+0000", 2023-10-01T09:16:35.443 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:16:35.443 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.443 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.443 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.443 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.788493+0000", 2023-10-01T09:16:35.443 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8080000000000001e-06 2023-10-01T09:16:35.443 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.443 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.443 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.443 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.788496+0000", 2023-10-01T09:16:35.443 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8720000000000001e-06 2023-10-01T09:16:35.443 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.444 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.444 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.444 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.788501+0000", 2023-10-01T09:16:35.444 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5820000000000001e-06 2023-10-01T09:16:35.444 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.444 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.444 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.444 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.788506+0000", 2023-10-01T09:16:35.444 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0055560790000000002 2023-10-01T09:16:35.444 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.444 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.444 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.445 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.794062+0000", 2023-10-01T09:16:35.445 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000109609 2023-10-01T09:16:35.445 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.445 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.445 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.445 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.794172+0000", 2023-10-01T09:16:35.445 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000115565 2023-10-01T09:16:35.445 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.445 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.445 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:16:35.445 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.794287+0000", 2023-10-01T09:16:35.445 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038606360000000002 2023-10-01T09:16:35.446 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.446 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.446 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.446 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.798148+0000", 2023-10-01T09:16:35.446 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002145199 2023-10-01T09:16:35.446 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.446 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.446 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.446 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.800293+0000", 2023-10-01T09:16:35.446 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15514096799999999 2023-10-01T09:16:35.446 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.446 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.447 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.447 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.955434+0000", 2023-10-01T09:16:35.447 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6764000000000002e-05 2023-10-01T09:16:35.447 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.447 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.447 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.447 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.955481+0000", 2023-10-01T09:16:35.447 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0985000000000001e-05 2023-10-01T09:16:35.447 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.447 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.447 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.447 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.955522+0000", 2023-10-01T09:16:35.448 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.167028658 2023-10-01T09:16:35.448 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.448 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.448 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.448 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.448 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.448 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:119383 5.7 5:e247526b:::benchmark_data_smithi138_240338_object4246:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-01T09:16:35.448 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:19.788512+0000", 2023-10-01T09:16:35.448 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 15.560899346999999, 2023-10-01T09:16:35.448 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16314374000000001, 2023-10-01T09:16:35.448 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.448 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.449 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.449 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.449 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.449 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 119383 2023-10-01T09:16:35.449 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.449 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.449 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.449 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.449 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.788512+0000", 2023-10-01T09:16:35.449 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.449 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.450 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.450 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.450 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.788512+0000", 2023-10-01T09:16:35.450 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:16:35.450 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.450 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.450 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.450 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.788511+0000", 2023-10-01T09:16:35.450 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.34e-06 2023-10-01T09:16:35.450 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.450 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.450 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.451 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.788513+0000", 2023-10-01T09:16:35.451 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.687e-06 2023-10-01T09:16:35.451 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.451 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.451 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.451 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.788517+0000", 2023-10-01T09:16:35.451 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6909999999999999e-06 2023-10-01T09:16:35.451 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.451 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.451 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.451 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.788521+0000", 2023-10-01T09:16:35.451 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003896521 2023-10-01T09:16:35.452 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.452 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.452 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.452 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.792417+0000", 2023-10-01T09:16:35.452 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000119271 2023-10-01T09:16:35.452 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.452 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.452 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.452 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.792537+0000", 2023-10-01T09:16:35.452 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000124268 2023-10-01T09:16:35.452 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.452 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.453 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.453 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.792661+0000", 2023-10-01T09:16:35.453 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053603909999999999 2023-10-01T09:16:35.453 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.453 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.453 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.453 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.798021+0000", 2023-10-01T09:16:35.453 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15329638900000001 2023-10-01T09:16:35.453 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.453 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.453 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.453 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.951318+0000", 2023-10-01T09:16:35.454 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00026651600000000001 2023-10-01T09:16:35.454 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.454 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.454 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.454 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.951584+0000", 2023-10-01T09:16:35.454 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7595999999999997e-05 2023-10-01T09:16:35.454 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.454 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.454 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.454 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.951622+0000", 2023-10-01T09:16:35.454 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3637000000000003e-05 2023-10-01T09:16:35.454 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.455 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.455 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.455 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.951655+0000", 2023-10-01T09:16:35.455 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16314374000000001 2023-10-01T09:16:35.455 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.455 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.455 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.455 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.455 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.455 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:119393 5.f 5:faf6d93f:::benchmark_data_smithi138_240338_object4256:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-01T09:16:35.455 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:19.802871+0000", 2023-10-01T09:16:35.455 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 15.546539900999999, 2023-10-01T09:16:35.456 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17156501699999999, 2023-10-01T09:16:35.456 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.456 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.456 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.456 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.456 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.456 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 119393 2023-10-01T09:16:35.456 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.456 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.456 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.456 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.456 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.802871+0000", 2023-10-01T09:16:35.457 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.457 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.457 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.457 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.457 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.802871+0000", 2023-10-01T09:16:35.457 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:35.457 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.457 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.457 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.457 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.802870+0000", 2023-10-01T09:16:35.457 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1389999999999998e-06 2023-10-01T09:16:35.457 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.458 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.458 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.458 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.802877+0000", 2023-10-01T09:16:35.458 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9769999999999995e-06 2023-10-01T09:16:35.458 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.458 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.458 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.458 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.802887+0000", 2023-10-01T09:16:35.458 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4949999999999999e-06 2023-10-01T09:16:35.458 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.458 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.459 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.459 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.802892+0000", 2023-10-01T09:16:35.459 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14841201000000001 2023-10-01T09:16:35.459 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.459 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.459 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.459 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.951304+0000", 2023-10-01T09:16:35.459 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013452700000000001 2023-10-01T09:16:35.459 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.459 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.459 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.459 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.951439+0000", 2023-10-01T09:16:35.459 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013138399999999999 2023-10-01T09:16:35.460 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.460 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.460 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:16:35.460 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.951570+0000", 2023-10-01T09:16:35.460 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.007997845 2023-10-01T09:16:35.460 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.460 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.460 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.460 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.959568+0000", 2023-10-01T09:16:35.460 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014631373 2023-10-01T09:16:35.460 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.461 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.461 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.461 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.974199+0000", 2023-10-01T09:16:35.461 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014564299999999999 2023-10-01T09:16:35.461 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.461 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.461 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.461 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.974345+0000", 2023-10-01T09:16:35.461 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0970000000000002e-05 2023-10-01T09:16:35.461 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.461 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.461 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.462 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.974396+0000", 2023-10-01T09:16:35.462 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0219000000000002e-05 2023-10-01T09:16:35.462 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.462 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.462 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.462 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:19.974436+0000", 2023-10-01T09:16:35.462 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17156501699999999 2023-10-01T09:16:35.462 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.462 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.462 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.462 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.462 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.463 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:119726 5.7 5:e6554c9b:::benchmark_data_smithi138_240338_object4589:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-01T09:16:35.463 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:20.301881+0000", 2023-10-01T09:16:35.463 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 15.047530392000001, 2023-10-01T09:16:35.463 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20550888, 2023-10-01T09:16:35.463 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.463 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.463 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.463 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.463 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.463 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 119726 2023-10-01T09:16:35.463 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.464 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.464 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.464 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.464 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301881+0000", 2023-10-01T09:16:35.464 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.464 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.464 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.464 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.464 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301881+0000", 2023-10-01T09:16:35.464 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:16:35.464 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.464 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.464 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.465 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301879+0000", 2023-10-01T09:16:35.465 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0860000000000002e-06 2023-10-01T09:16:35.465 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.465 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.465 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.465 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301887+0000", 2023-10-01T09:16:35.465 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3305000000000001e-05 2023-10-01T09:16:35.465 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.465 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.465 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.465 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301900+0000", 2023-10-01T09:16:35.465 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9320000000000004e-06 2023-10-01T09:16:35.466 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.466 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.466 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.466 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301905+0000", 2023-10-01T09:16:35.466 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3999000000000003e-05 2023-10-01T09:16:35.466 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.466 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.466 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.466 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301959+0000", 2023-10-01T09:16:35.466 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000136933 2023-10-01T09:16:35.466 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.467 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.467 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.467 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.302096+0000", 2023-10-01T09:16:35.467 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000130295 2023-10-01T09:16:35.467 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.467 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.467 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.467 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.302226+0000", 2023-10-01T09:16:35.467 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0040403949999999996 2023-10-01T09:16:35.467 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.467 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.467 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.467 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.306267+0000", 2023-10-01T09:16:35.468 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001816149 2023-10-01T09:16:35.468 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.468 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.468 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.468 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.308083+0000", 2023-10-01T09:16:35.468 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.199197229 2023-10-01T09:16:35.468 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.468 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.468 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.468 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.507280+0000", 2023-10-01T09:16:35.468 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7046999999999999e-05 2023-10-01T09:16:35.469 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.469 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.469 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.469 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.507327+0000", 2023-10-01T09:16:35.469 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.232e-05 2023-10-01T09:16:35.469 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.469 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.469 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.469 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.507389+0000", 2023-10-01T09:16:35.469 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20550888 2023-10-01T09:16:35.469 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.469 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.470 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.470 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.470 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.470 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:121350 5.6 5:60c69cc8:::benchmark_data_smithi138_240338_object6213:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e125)", 2023-10-01T09:16:35.470 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:22.527232+0000", 2023-10-01T09:16:35.470 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 12.822179434000001, 2023-10-01T09:16:35.470 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.013085148, 2023-10-01T09:16:35.470 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.470 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:16:35.470 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.470 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.470 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.471 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121350 2023-10-01T09:16:35.471 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.471 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.471 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.471 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.471 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.527232+0000", 2023-10-01T09:16:35.471 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.471 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.471 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.471 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.471 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.527232+0000", 2023-10-01T09:16:35.472 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:16:35.472 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.472 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.472 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.472 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.527230+0000", 2023-10-01T09:16:35.472 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8749999999999997e-06 2023-10-01T09:16:35.472 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.472 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.472 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.472 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.527236+0000", 2023-10-01T09:16:35.472 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2639e-05 2023-10-01T09:16:35.472 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.473 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.473 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.473 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.527249+0000", 2023-10-01T09:16:35.473 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9750000000000001e-06 2023-10-01T09:16:35.473 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.473 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.473 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.473 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.527256+0000", 2023-10-01T09:16:35.473 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038199219999999999 2023-10-01T09:16:35.473 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.473 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.474 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.474 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.531076+0000", 2023-10-01T09:16:35.474 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3699999999999997e-06 2023-10-01T09:16:35.474 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.474 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.474 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op must wait for map", 2023-10-01T09:16:35.474 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.531083+0000", 2023-10-01T09:16:35.474 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0026150959999999999 2023-10-01T09:16:35.474 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.474 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.474 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.474 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.533698+0000", 2023-10-01T09:16:35.475 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5810000000000003e-06 2023-10-01T09:16:35.475 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.475 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.475 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:16:35.475 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.533708+0000", 2023-10-01T09:16:35.475 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.004413848 2023-10-01T09:16:35.475 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.475 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.475 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.475 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.538122+0000", 2023-10-01T09:16:35.475 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0021949059999999999 2023-10-01T09:16:35.475 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.476 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.476 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.476 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.540317+0000", 2023-10-01T09:16:35.476 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.013085148 2023-10-01T09:16:35.476 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.476 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.476 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.476 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.476 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.476 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:121381 5.6 5:6c2d7d69:::benchmark_data_smithi138_240338_object6244:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:16:35.476 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:22.571622+0000", 2023-10-01T09:16:35.476 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 12.77778874, 2023-10-01T09:16:35.477 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97724141899999994, 2023-10-01T09:16:35.477 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.477 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.477 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.477 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.477 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.477 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121381 2023-10-01T09:16:35.477 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.477 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.477 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.477 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.477 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571622+0000", 2023-10-01T09:16:35.478 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.478 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.478 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.478 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.478 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571622+0000", 2023-10-01T09:16:35.478 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:16:35.478 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.478 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.478 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.478 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571621+0000", 2023-10-01T09:16:35.478 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5870000000000003e-06 2023-10-01T09:16:35.479 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.479 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.479 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.479 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571626+0000", 2023-10-01T09:16:35.479 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0426e-05 2023-10-01T09:16:35.479 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.479 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.479 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.479 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571636+0000", 2023-10-01T09:16:35.479 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4710000000000001e-06 2023-10-01T09:16:35.479 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.479 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.480 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.480 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571641+0000", 2023-10-01T09:16:35.480 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8782000000000002e-05 2023-10-01T09:16:35.480 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.480 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.480 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.480 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571689+0000", 2023-10-01T09:16:35.480 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7560000000000005e-06 2023-10-01T09:16:35.480 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.480 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.480 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:16:35.480 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571698+0000", 2023-10-01T09:16:35.481 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.96865936200000002 2023-10-01T09:16:35.481 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.481 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.481 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.481 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.540358+0000", 2023-10-01T09:16:35.481 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00073541700000000004 2023-10-01T09:16:35.481 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.481 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.481 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.481 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.541093+0000", 2023-10-01T09:16:35.481 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8483000000000001e-05 2023-10-01T09:16:35.481 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.482 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.482 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,7", 2023-10-01T09:16:35.482 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.541171+0000", 2023-10-01T09:16:35.482 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.007198734 2023-10-01T09:16:35.482 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.482 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.482 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.482 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.548370+0000", 2023-10-01T09:16:35.482 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000376363 2023-10-01T09:16:35.482 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.482 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.483 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.483 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.548747+0000", 2023-10-01T09:16:35.483 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8316000000000001e-05 2023-10-01T09:16:35.483 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.483 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.483 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.483 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.548775+0000", 2023-10-01T09:16:35.483 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4267000000000001e-05 2023-10-01T09:16:35.483 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.483 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.483 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.483 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.548829+0000", 2023-10-01T09:16:35.484 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4561000000000003e-05 2023-10-01T09:16:35.484 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.484 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.484 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.484 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.548864+0000", 2023-10-01T09:16:35.484 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97724141899999994 2023-10-01T09:16:35.484 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.484 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.484 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.484 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.484 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.484 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:121382 5.6 5:676fecac:::benchmark_data_smithi138_240338_object6245:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:16:35.485 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:22.574219+0000", 2023-10-01T09:16:35.485 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 12.775192292, 2023-10-01T09:16:35.485 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98247247800000004, 2023-10-01T09:16:35.485 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.485 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.485 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.485 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.485 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.485 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121382 2023-10-01T09:16:35.485 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.485 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.485 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.486 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.486 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574219+0000", 2023-10-01T09:16:35.486 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.486 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.486 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.486 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.486 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574219+0000", 2023-10-01T09:16:35.486 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:16:35.486 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.486 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.486 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.487 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574218+0000", 2023-10-01T09:16:35.487 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1109999999999999e-06 2023-10-01T09:16:35.487 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.487 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.487 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.487 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574223+0000", 2023-10-01T09:16:35.487 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0179e-05 2023-10-01T09:16:35.487 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.487 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.487 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.487 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574233+0000", 2023-10-01T09:16:35.487 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1960000000000001e-06 2023-10-01T09:16:35.488 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.488 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.488 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.488 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574237+0000", 2023-10-01T09:16:35.488 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0365000000000002e-05 2023-10-01T09:16:35.488 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.488 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.488 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.488 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574298+0000", 2023-10-01T09:16:35.488 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0120000000000002e-06 2023-10-01T09:16:35.488 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.488 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.489 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:16:35.490 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:16:35.490 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574304+0000", 2023-10-01T09:16:35.490 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97017618500000002 2023-10-01T09:16:35.490 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.490 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.490 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.490 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.544480+0000", 2023-10-01T09:16:35.490 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000106017 2023-10-01T09:16:35.490 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.490 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.490 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.491 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.544586+0000", 2023-10-01T09:16:35.491 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013026699999999999 2023-10-01T09:16:35.491 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.491 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.491 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,7", 2023-10-01T09:16:35.491 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.544716+0000", 2023-10-01T09:16:35.491 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0060876430000000002 2023-10-01T09:16:35.491 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.491 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.491 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.491 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.550804+0000", 2023-10-01T09:16:35.492 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00076178200000000002 2023-10-01T09:16:35.492 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.492 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.492 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.492 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.551566+0000", 2023-10-01T09:16:35.492 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050625649999999998 2023-10-01T09:16:35.492 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.492 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.492 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.492 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.556628+0000", 2023-10-01T09:16:35.492 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7085000000000002e-05 2023-10-01T09:16:35.492 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.493 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.493 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.493 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.556665+0000", 2023-10-01T09:16:35.493 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5840999999999999e-05 2023-10-01T09:16:35.493 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.493 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.493 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.493 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.556691+0000", 2023-10-01T09:16:35.493 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98247247800000004 2023-10-01T09:16:35.493 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.493 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.493 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.494 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.494 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.494 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:123038 5.7 5:ee2dadff:::benchmark_data_smithi138_240338_object7901:head [delete] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-10-01T09:16:35.494 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:25.462228+0000", 2023-10-01T09:16:35.494 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 9.8871825340000008, 2023-10-01T09:16:35.494 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16486260799999999, 2023-10-01T09:16:35.494 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.494 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.494 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.494 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.494 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.495 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 123038 2023-10-01T09:16:35.495 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.495 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.495 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.495 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.495 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.462228+0000", 2023-10-01T09:16:35.495 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.495 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.495 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.495 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.495 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.462228+0000", 2023-10-01T09:16:35.495 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:35.496 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.496 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.496 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.496 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.462227+0000", 2023-10-01T09:16:35.496 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8119999999999998e-06 2023-10-01T09:16:35.496 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.496 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.496 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.496 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.462232+0000", 2023-10-01T09:16:35.496 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2136999999999999e-05 2023-10-01T09:16:35.496 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.496 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.497 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.497 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.462244+0000", 2023-10-01T09:16:35.497 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3770000000000004e-06 2023-10-01T09:16:35.497 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.497 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.497 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.497 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.462250+0000", 2023-10-01T09:16:35.497 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000225028 2023-10-01T09:16:35.497 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.497 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.497 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.497 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.462475+0000", 2023-10-01T09:16:35.498 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011102700000000001 2023-10-01T09:16:35.498 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.498 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.498 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.498 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.462586+0000", 2023-10-01T09:16:35.498 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011320900000000001 2023-10-01T09:16:35.498 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.498 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.498 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.498 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.462699+0000", 2023-10-01T09:16:35.498 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0056304049999999998 2023-10-01T09:16:35.498 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.499 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.499 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.499 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.468329+0000", 2023-10-01T09:16:35.499 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0021707940000000002 2023-10-01T09:16:35.499 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.499 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.499 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.499 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.470500+0000", 2023-10-01T09:16:35.499 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.156511961 2023-10-01T09:16:35.499 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.499 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.499 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.500 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.627012+0000", 2023-10-01T09:16:35.500 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3066999999999999e-05 2023-10-01T09:16:35.500 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.500 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.500 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.500 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.627055+0000", 2023-10-01T09:16:35.500 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6018e-05 2023-10-01T09:16:35.500 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.500 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.500 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.500 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.627091+0000", 2023-10-01T09:16:35.500 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16486260799999999 2023-10-01T09:16:35.501 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.501 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.501 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.501 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.501 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.501 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:123049 5.7 5:e3c3cf1d:::benchmark_data_smithi138_240338_object7912:head [delete] snapc 0=[] ondisk+write+known_if_redirected e127)", 2023-10-01T09:16:35.501 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:25.474424+0000", 2023-10-01T09:16:35.501 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 9.8749868579999998, 2023-10-01T09:16:35.501 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.163263087, 2023-10-01T09:16:35.501 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:35.501 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:16:35.501 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:35.502 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:35.502 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:35.502 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 123049 2023-10-01T09:16:35.502 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.502 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:35.502 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.502 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:35.502 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.474424+0000", 2023-10-01T09:16:35.502 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:35.502 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.502 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.503 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:35.503 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.474424+0000", 2023-10-01T09:16:35.503 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:35.503 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.503 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.503 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:35.503 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.474423+0000", 2023-10-01T09:16:35.503 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8010000000000003e-06 2023-10-01T09:16:35.503 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.503 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.503 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:35.503 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.474428+0000", 2023-10-01T09:16:35.504 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2209999999999997e-06 2023-10-01T09:16:35.504 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.504 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.504 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:35.504 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.474436+0000", 2023-10-01T09:16:35.504 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.857e-06 2023-10-01T09:16:35.504 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.504 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.504 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:35.504 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.474442+0000", 2023-10-01T09:16:35.504 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1786999999999998e-05 2023-10-01T09:16:35.504 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.505 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.505 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:35.505 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.474483+0000", 2023-10-01T09:16:35.505 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000132531 2023-10-01T09:16:35.505 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.505 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.505 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:35.505 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.474616+0000", 2023-10-01T09:16:35.505 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000125943 2023-10-01T09:16:35.505 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.505 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.505 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:16:35.506 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.474742+0000", 2023-10-01T09:16:35.506 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078036010000000003 2023-10-01T09:16:35.506 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.506 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.506 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.506 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.482546+0000", 2023-10-01T09:16:35.506 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7440000000000004e-06 2023-10-01T09:16:35.506 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.506 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.506 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:16:35.506 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.482551+0000", 2023-10-01T09:16:35.506 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.155054101 2023-10-01T09:16:35.507 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.507 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.507 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:16:35.507 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.637605+0000", 2023-10-01T09:16:35.507 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3161999999999998e-05 2023-10-01T09:16:35.507 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.507 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.507 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:16:35.507 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.637649+0000", 2023-10-01T09:16:35.507 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8670000000000001e-05 2023-10-01T09:16:35.507 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:35.508 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:35.508 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:16:35.508 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:25.637687+0000", 2023-10-01T09:16:35.508 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.163263087 2023-10-01T09:16:35.508 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.508 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.508 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.508 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:35.508 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:35.508 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:16:35.508 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:35.509 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:16:35.509 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:16:35.509 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:16:35.509 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:16:35.509 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:16:35.509 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:16:35.510 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:16:35.510 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:16:35.510 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:16:35.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:35.511 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:35.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:35.511 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:35.612 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:35.675 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/634617364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:35.676 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:35 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:35.677 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/634617364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:35.677 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:35 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:35.678 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/634617364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:35.678 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:35 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:35.679 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:35.755 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:35.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:35.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:35.858 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:35.892 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:36.283 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:36.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:36.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:36.312 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:16:36.389 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:36.485 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:36.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:36 smithi138 ceph-mon[192513]: pgmap v306: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2023-10-01T09:16:36.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:36 smithi138 ceph-mon[185886]: pgmap v306: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2023-10-01T09:16:36.566 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:36 smithi190 ceph-mon[160435]: pgmap v306: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2023-10-01T09:16:36.652 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:36.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:36.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:36.771 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:36.855 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:36.975 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:36.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:36.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:37.024 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:16:37.098 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:16:37.141 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:37.212 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:16:37.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:37.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:16:37.315 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:37.402 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:37.724 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:37.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:37.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:37.831 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:16:37.886 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:37.996 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:16:37.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:37.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:16:38.099 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:38.147 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:38.315 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:37 smithi190 ceph-mon[160435]: pgmap v307: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-01T09:16:38.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:37 smithi138 ceph-mon[185886]: pgmap v307: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-01T09:16:38.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:37 smithi138 ceph-mon[192513]: pgmap v307: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-01T09:16:38.469 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:38.471 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:16:38.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:38.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:38.575 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:38.662 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:38.749 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:38.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:38.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:38.854 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:38.901 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:39.042 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:16:39.056 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:39.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3012354441' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:39.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:39.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:39.093 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:39.100 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:16:39.100 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:16:39.101 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.101 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.34683.0:132802 5.3 e129/106 5:c258214d:::benchmark_data_smithi138_240338_object17665:head v 129'8341, mlcod=129'8339)", 2023-10-01T09:16:39.101 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:39.095237+0000", 2023-10-01T09:16:39.101 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0027155679999999998, 2023-10-01T09:16:39.101 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0027447180000000002, 2023-10-01T09:16:39.101 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:39.101 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:16:39.101 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:39.101 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.101 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:39.101 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095237+0000", 2023-10-01T09:16:39.101 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:39.102 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.102 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.102 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:39.102 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095237+0000", 2023-10-01T09:16:39.102 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:16:39.102 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.102 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.102 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:39.102 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095235+0000", 2023-10-01T09:16:39.102 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2738999999999999e-05 2023-10-01T09:16:39.102 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.102 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.102 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:39.103 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095248+0000", 2023-10-01T09:16:39.103 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1659999999999999e-05 2023-10-01T09:16:39.103 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.103 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.103 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:39.103 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095270+0000", 2023-10-01T09:16:39.103 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9610000000000005e-06 2023-10-01T09:16:39.103 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.103 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.103 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:39.103 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095278+0000", 2023-10-01T09:16:39.103 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015073699999999999 2023-10-01T09:16:39.103 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.104 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.104 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:39.104 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095428+0000", 2023-10-01T09:16:39.104 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1095e-05 2023-10-01T09:16:39.104 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.104 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.104 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:39.104 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095450+0000", 2023-10-01T09:16:39.104 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00021239100000000001 2023-10-01T09:16:39.104 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:39.104 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:39.104 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:39.104 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.105 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.105 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.34683.0:132803 5.3 e129/106)", 2023-10-01T09:16:39.105 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:39.095293+0000", 2023-10-01T09:16:39.105 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0026598860000000002, 2023-10-01T09:16:39.105 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002758, 2023-10-01T09:16:39.105 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:39.105 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:16:39.105 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:39.105 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.105 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:39.105 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095293+0000", 2023-10-01T09:16:39.106 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:39.106 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.106 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.106 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:39.106 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095293+0000", 2023-10-01T09:16:39.106 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:16:39.106 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.106 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.106 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:39.106 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095293+0000", 2023-10-01T09:16:39.106 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8409999999999999e-06 2023-10-01T09:16:39.107 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.107 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.107 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:39.107 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095297+0000", 2023-10-01T09:16:39.107 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5990000000000001e-06 2023-10-01T09:16:39.107 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.107 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.107 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:39.107 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095303+0000", 2023-10-01T09:16:39.107 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8619999999999996e-06 2023-10-01T09:16:39.107 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.107 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.107 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:39.108 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095307+0000", 2023-10-01T09:16:39.108 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3995999999999999e-05 2023-10-01T09:16:39.108 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:39.108 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:39.108 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:39.108 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.108 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.108 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.34683.0:132809 5.3 e129/106)", 2023-10-01T09:16:39.108 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:39.095311+0000", 2023-10-01T09:16:39.108 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0026422279999999999, 2023-10-01T09:16:39.108 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0027686220000000001, 2023-10-01T09:16:39.108 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:39.109 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:16:39.109 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:39.109 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.109 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:39.109 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095311+0000", 2023-10-01T09:16:39.109 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:39.109 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.109 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.109 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:39.109 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095311+0000", 2023-10-01T09:16:39.109 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967296 2023-10-01T09:16:39.109 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.110 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.110 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:39.110 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095310+0000", 2023-10-01T09:16:39.110 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3589999999999996e-06 2023-10-01T09:16:39.110 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.110 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.110 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:39.111 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095317+0000", 2023-10-01T09:16:39.111 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7600000000000002e-06 2023-10-01T09:16:39.111 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.111 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.111 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:39.111 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095321+0000", 2023-10-01T09:16:39.111 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1540000000000004e-06 2023-10-01T09:16:39.111 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:39.111 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:39.111 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:39.111 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:39.095326+0000", 2023-10-01T09:16:39.111 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5121e-05 2023-10-01T09:16:39.112 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:39.112 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:39.112 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:39.112 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:39.112 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:16:39.112 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 3 2023-10-01T09:16:39.112 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:16:39.169 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:39.242 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:39.268 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3012354441' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:39.268 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3012354441' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:39.412 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:39.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:39.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:39.517 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:39.553 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:39.582 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:16:39.582 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:16:39.583 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2023-10-01T09:16:39.583 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:16:39.637 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:39.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:39.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:39.754 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:39.822 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:40.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:40 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:16:40.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:40 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:16:40.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:40 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3012354441' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:16:40.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:40 smithi138 ceph-mon[185886]: osdmap e129: 8 total, 8 up, 8 in 2023-10-01T09:16:40.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:40 smithi138 ceph-mon[185886]: failed to encode map e129 with expected crc 2023-10-01T09:16:40.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:40 smithi138 ceph-mon[185886]: pgmap v309: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-01T09:16:40.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:40 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:16:40.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:40 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:16:40.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:40 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3012354441' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:16:40.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:40 smithi138 ceph-mon[192513]: osdmap e129: 8 total, 8 up, 8 in 2023-10-01T09:16:40.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:40 smithi138 ceph-mon[192513]: failed to encode map e129 with expected crc 2023-10-01T09:16:40.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:40 smithi138 ceph-mon[192513]: pgmap v309: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-01T09:16:40.074 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:16:40.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:40 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:16:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:40 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:16:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:40 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3012354441' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:16:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:40 smithi190 ceph-mon[160435]: osdmap e129: 8 total, 8 up, 8 in 2023-10-01T09:16:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:40 smithi190 ceph-mon[160435]: failed to encode map e129 with expected crc 2023-10-01T09:16:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:40 smithi190 ceph-mon[160435]: pgmap v309: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-01T09:16:40.322 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:40.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:40.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:40.429 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:40.480 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:40.610 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:40.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:40.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:40.715 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:40.793 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:40.811 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:16:40.886 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:40.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:40.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:41.009 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:41.059 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:41.129 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:41.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:41.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:41.232 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:16:41.279 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:41.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:41.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:16:41.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:41.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:16:41.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:41.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:16:41.614 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:16:41.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:41.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:16:41.696 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:41.723 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:41.792 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:41.877 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:41.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:41.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:41.981 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:42.015 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:42.275 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:16:42.281 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:42 smithi190 ceph-mon[160435]: pgmap v310: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-01T09:16:42.282 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:42 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3805404497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:42.285 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:42 smithi138 ceph-mon[192513]: pgmap v310: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-01T09:16:42.285 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:42 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3805404497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:42.285 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:42 smithi138 ceph-mon[185886]: pgmap v310: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-01T09:16:42.285 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:42 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3805404497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:42.316 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:16:42.317 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:16:42.355 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:42.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:42.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:42.366 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:16:42.392 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:16:42.429 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:16:42.459 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:16:42.461 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:42.487 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:16:42.536 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:16:42.554 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:42.559 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:16:42.559 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:16:42.559 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:16:42.559 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:16:42.559 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:16:42.559 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:16:42.559 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:16:42.560 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:16:43.015 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:16:43.016 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:16:43.060 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:43.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:43.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:43.165 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:43.240 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:43.365 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:43.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:43.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:43.483 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:43.510 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:43.927 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:43.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:43.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:43.946 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:16:44.030 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:44.075 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:44.207 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:44.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:44.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:44.312 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:44.391 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:44.447 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:44.479 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:44.488 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:44 smithi138 ceph-mon[185886]: pgmap v311: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T09:16:44.488 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:44 smithi138 ceph-mon[192513]: pgmap v311: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T09:16:44.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:44.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:44.595 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:16:44.649 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:44.659 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:16:44.766 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:16:44.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:44.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:16:44.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:44 smithi190 ceph-mon[160435]: pgmap v311: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T09:16:44.872 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:44.936 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:45.029 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:16:45.036 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:16:45.048 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:45.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:45.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:45.150 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:45.192 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:16:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:16:44] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-01T09:16:45.209 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:45.357 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:45.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:45.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:45.469 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:45.530 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:45.622 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:45.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:45.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:45.725 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:45.783 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:45.854 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:45.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:45.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:45.963 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:46.027 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:46.215 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:46.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:46.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:46.228 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:16:46.328 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:16:46.395 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:46.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:46 smithi190 ceph-mon[160435]: from='client.44726 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:16:46.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:46 smithi190 ceph-mon[160435]: pgmap v312: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T09:16:46.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:46 smithi138 ceph-mon[185886]: from='client.44726 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:16:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:46 smithi138 ceph-mon[185886]: pgmap v312: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T09:16:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:46 smithi138 ceph-mon[192513]: from='client.44726 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:16:46.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:46 smithi138 ceph-mon[192513]: pgmap v312: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T09:16:46.742 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:16:46.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:46.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:16:46.824 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":311,"stamp":"2023-10-01T09:16:43.312052+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6263150539,"num_objects":95524,"num_object_clones":0,"num_object_copies":286572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":95524,"num_whiteouts":0,"num_read":6303,"num_read_kb":12701,"num_write":135880,"num_write_kb":7383718,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132544,"ondisk_log_size":132544,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20018140,"kb_used_data":18503196,"kb_used_omap":30,"kb_used_meta":1514593,"kb_avail":729943076,"statfs":{"total":767960285184,"available":747461709824,"internally_reserved":0,"allocated":18947272704,"data_stored":18942823946,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30777,"internal_metadata":1550944199},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,6,8,5,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-475529216,"num_objects":-7256,"num_object_clones":0,"num_object_copies":-21768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7256,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002877"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":110,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.514045+0000","last_change":"2023-10-01T09:12:40.947346+0000","last_active":"2023-10-01T09:16:31.514045+0000","last_peered":"2023-10-01T09:16:31.514045+0000","last_clean":"2023-10-01T09:16:31.514045+0000","last_became_active":"2023-10-01T09:12:40.946939+0000","last_became_peered":"2023-10-01T09:12:40.946939+0000","last_unstale":"2023-10-01T09:16:31.514045+0000","last_undegraded":"2023-10-01T09:16:31.514045+0000","last_fullsized":"2023-10-01T09:16:31.514045+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":98,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.522122+0000","last_change":"2023-10-01T09:13:13.554780+0000","last_active":"2023-10-01T09:16:31.522122+0000","last_peered":"2023-10-01T09:16:31.522122+0000","last_clean":"2023-10-01T09:16:31.522122+0000","last_became_active":"2023-10-01T09:13:13.554394+0000","last_became_peered":"2023-10-01T09:13:13.554394+0000","last_unstale":"2023-10-01T09:16:31.522122+0000","last_undegraded":"2023-10-01T09:16:31.522122+0000","last_fullsized":"2023-10-01T09:16:31.522122+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:09:02.869321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":72,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.432163+0000","last_change":"2023-10-01T09:13:13.551596+0000","last_active":"2023-10-01T09:16:31.432163+0000","last_peered":"2023-10-01T09:16:31.432163+0000","last_clean":"2023-10-01T09:16:31.432163+0000","last_became_active":"2023-10-01T09:13:13.551300+0000","last_became_peered":"2023-10-01T09:13:13.551300+0000","last_unstale":"2023-10-01T09:16:31.432163+0000","last_undegraded":"2023-10-01T09:16:31.432163+0000","last_fullsized":"2023-10-01T09:16:31.432163+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:57:45.081780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":77,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.515299+0000","last_change":"2023-10-01T09:13:13.548695+0000","last_active":"2023-10-01T09:16:31.515299+0000","last_peered":"2023-10-01T09:16:31.515299+0000","last_clean":"2023-10-01T09:16:31.515299+0000","last_became_active":"2023-10-01T09:13:13.548256+0000","last_became_peered":"2023-10-01T09:13:13.548256+0000","last_unstale":"2023-10-01T09:16:31.515299+0000","last_undegraded":"2023-10-01T09:16:31.515299+0000","last_fullsized":"2023-10-01T09:16:31.515299+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:35:28.655733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":114,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521570+0000","last_change":"2023-10-01T09:13:13.555490+0000","last_active":"2023-10-01T09:16:31.521570+0000","last_peered":"2023-10-01T09:16:31.521570+0000","last_clean":"2023-10-01T09:16:31.521570+0000","last_became_active":"2023-10-01T09:13:13.554506+0000","last_became_peered":"2023-10-01T09:13:13.554506+0000","last_unstale":"2023-10-01T09:16:31.521570+0000","last_undegraded":"2023-10-01T09:16:31.521570+0000","last_fullsized":"2023-10-01T09:16:31.521570+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:18.834474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":103,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521076+0000","last_change":"2023-10-01T09:12:40.947591+0000","last_active":"2023-10-01T09:16:31.521076+0000","last_peered":"2023-10-01T09:16:31.521076+0000","last_clean":"2023-10-01T09:16:31.521076+0000","last_became_active":"2023-10-01T09:12:40.947242+0000","last_became_peered":"2023-10-01T09:12:40.947242+0000","last_unstale":"2023-10-01T09:16:31.521076+0000","last_undegraded":"2023-10-01T09:16:31.521076+0000","last_fullsized":"2023-10-01T09:16:31.521076+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":77,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.517038+0000","last_change":"2023-10-01T09:13:13.550193+0000","last_active":"2023-10-01T09:16:31.517038+0000","last_peered":"2023-10-01T09:16:31.517038+0000","last_clean":"2023-10-01T09:16:31.517038+0000","last_became_active":"2023-10-01T09:13:13.549900+0000","last_became_peered":"2023-10-01T09:13:13.549900+0000","last_unstale":"2023-10-01T09:16:31.517038+0000","last_undegraded":"2023-10-01T09:16:31.517038+0000","last_fullsized":"2023-10-01T09:16:31.517038+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:22.643161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":98,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521505+0000","last_change":"2023-10-01T09:13:13.555368+0000","last_active":"2023-10-01T09:16:31.521505+0000","last_peered":"2023-10-01T09:16:31.521505+0000","last_clean":"2023-10-01T09:16:31.521505+0000","last_became_active":"2023-10-01T09:13:13.554806+0000","last_became_peered":"2023-10-01T09:13:13.554806+0000","last_unstale":"2023-10-01T09:16:31.521505+0000","last_undegraded":"2023-10-01T09:16:31.521505+0000","last_fullsized":"2023-10-01T09:16:31.521505+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:24:32.687833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":95,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.436215+0000","last_change":"2023-10-01T09:12:10.003091+0000","last_active":"2023-10-01T09:16:31.436215+0000","last_peered":"2023-10-01T09:16:31.436215+0000","last_clean":"2023-10-01T09:16:31.436215+0000","last_became_active":"2023-10-01T09:12:10.002773+0000","last_became_peered":"2023-10-01T09:12:10.002773+0000","last_unstale":"2023-10-01T09:16:31.436215+0000","last_undegraded":"2023-10-01T09:16:31.436215+0000","last_fullsized":"2023-10-01T09:16:31.436215+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":88,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521824+0000","last_change":"2023-10-01T09:13:13.554942+0000","last_active":"2023-10-01T09:16:31.521824+0000","last_peered":"2023-10-01T09:16:31.521824+0000","last_clean":"2023-10-01T09:16:31.521824+0000","last_became_active":"2023-10-01T09:13:13.554664+0000","last_became_peered":"2023-10-01T09:13:13.554664+0000","last_unstale":"2023-10-01T09:16:31.521824+0000","last_undegraded":"2023-10-01T09:16:31.521824+0000","last_fullsized":"2023-10-01T09:16:31.521824+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:43:14.532709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":98,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521514+0000","last_change":"2023-10-01T09:13:13.556801+0000","last_active":"2023-10-01T09:16:31.521514+0000","last_peered":"2023-10-01T09:16:31.521514+0000","last_clean":"2023-10-01T09:16:31.521514+0000","last_became_active":"2023-10-01T09:13:13.556397+0000","last_became_peered":"2023-10-01T09:13:13.556397+0000","last_unstale":"2023-10-01T09:16:31.521514+0000","last_undegraded":"2023-10-01T09:16:31.521514+0000","last_fullsized":"2023-10-01T09:16:31.521514+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":72,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.513626+0000","last_change":"2023-10-01T09:12:40.946520+0000","last_active":"2023-10-01T09:16:31.513626+0000","last_peered":"2023-10-01T09:16:31.513626+0000","last_clean":"2023-10-01T09:16:31.513626+0000","last_became_active":"2023-10-01T09:12:40.946227+0000","last_became_peered":"2023-10-01T09:12:40.946227+0000","last_unstale":"2023-10-01T09:16:31.513626+0000","last_undegraded":"2023-10-01T09:16:31.513626+0000","last_fullsized":"2023-10-01T09:16:31.513626+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:06:06.451923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":101,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.514317+0000","last_change":"2023-10-01T09:03:03.200261+0000","last_active":"2023-10-01T09:16:31.514317+0000","last_peered":"2023-10-01T09:16:31.514317+0000","last_clean":"2023-10-01T09:16:31.514317+0000","last_became_active":"2023-10-01T09:03:03.199944+0000","last_became_peered":"2023-10-01T09:03:03.199944+0000","last_unstale":"2023-10-01T09:16:31.514317+0000","last_undegraded":"2023-10-01T09:16:31.514317+0000","last_fullsized":"2023-10-01T09:16:31.514317+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":102,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.514061+0000","last_change":"2023-10-01T09:12:40.946973+0000","last_active":"2023-10-01T09:16:31.514061+0000","last_peered":"2023-10-01T09:16:31.514061+0000","last_clean":"2023-10-01T09:16:31.514061+0000","last_became_active":"2023-10-01T09:12:40.946427+0000","last_became_peered":"2023-10-01T09:12:40.946427+0000","last_unstale":"2023-10-01T09:16:31.514061+0000","last_undegraded":"2023-10-01T09:16:31.514061+0000","last_fullsized":"2023-10-01T09:16:31.514061+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":97,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521077+0000","last_change":"2023-10-01T09:12:40.945277+0000","last_active":"2023-10-01T09:16:31.521077+0000","last_peered":"2023-10-01T09:16:31.521077+0000","last_clean":"2023-10-01T09:16:31.521077+0000","last_became_active":"2023-10-01T09:12:40.944843+0000","last_became_peered":"2023-10-01T09:12:40.944843+0000","last_unstale":"2023-10-01T09:16:31.521077+0000","last_undegraded":"2023-10-01T09:16:31.521077+0000","last_fullsized":"2023-10-01T09:16:31.521077+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":98,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521758+0000","last_change":"2023-10-01T09:13:13.550633+0000","last_active":"2023-10-01T09:16:31.521758+0000","last_peered":"2023-10-01T09:16:31.521758+0000","last_clean":"2023-10-01T09:16:31.521758+0000","last_became_active":"2023-10-01T09:13:13.550305+0000","last_became_peered":"2023-10-01T09:13:13.550305+0000","last_unstale":"2023-10-01T09:16:31.521758+0000","last_undegraded":"2023-10-01T09:16:31.521758+0000","last_fullsized":"2023-10-01T09:16:31.521758+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:51:46.559383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":77,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.517624+0000","last_change":"2023-10-01T09:13:13.548094+0000","last_active":"2023-10-01T09:16:31.517624+0000","last_peered":"2023-10-01T09:16:31.517624+0000","last_clean":"2023-10-01T09:16:31.517624+0000","last_became_active":"2023-10-01T09:13:13.547782+0000","last_became_peered":"2023-10-01T09:13:13.547782+0000","last_unstale":"2023-10-01T09:16:31.517624+0000","last_undegraded":"2023-10-01T09:16:31.517624+0000","last_fullsized":"2023-10-01T09:16:31.517624+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:15.220827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":94,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.435924+0000","last_change":"2023-10-01T09:13:13.558367+0000","last_active":"2023-10-01T09:16:31.435924+0000","last_peered":"2023-10-01T09:16:31.435924+0000","last_clean":"2023-10-01T09:16:31.435924+0000","last_became_active":"2023-10-01T09:13:13.557952+0000","last_became_peered":"2023-10-01T09:13:13.557952+0000","last_unstale":"2023-10-01T09:16:31.435924+0000","last_undegraded":"2023-10-01T09:16:31.435924+0000","last_fullsized":"2023-10-01T09:16:31.435924+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":80,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521638+0000","last_change":"2023-10-01T09:03:03.194952+0000","last_active":"2023-10-01T09:16:31.521638+0000","last_peered":"2023-10-01T09:16:31.521638+0000","last_clean":"2023-10-01T09:16:31.521638+0000","last_became_active":"2023-10-01T09:03:03.194643+0000","last_became_peered":"2023-10-01T09:03:03.194643+0000","last_unstale":"2023-10-01T09:16:31.521638+0000","last_undegraded":"2023-10-01T09:16:31.521638+0000","last_fullsized":"2023-10-01T09:16:31.521638+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":77,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.515230+0000","last_change":"2023-10-01T09:13:13.548531+0000","last_active":"2023-10-01T09:16:31.515230+0000","last_peered":"2023-10-01T09:16:31.515230+0000","last_clean":"2023-10-01T09:16:31.515230+0000","last_became_active":"2023-10-01T09:13:13.547881+0000","last_became_peered":"2023-10-01T09:13:13.547881+0000","last_unstale":"2023-10-01T09:16:31.515230+0000","last_undegraded":"2023-10-01T09:16:31.515230+0000","last_fullsized":"2023-10-01T09:16:31.515230+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:13:24.068359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":103,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.517089+0000","last_change":"2023-10-01T09:10:58.798168+0000","last_active":"2023-10-01T09:16:31.517089+0000","last_peered":"2023-10-01T09:16:31.517089+0000","last_clean":"2023-10-01T09:16:31.517089+0000","last_became_active":"2023-10-01T09:10:58.797781+0000","last_became_peered":"2023-10-01T09:10:58.797781+0000","last_unstale":"2023-10-01T09:16:31.517089+0000","last_undegraded":"2023-10-01T09:16:31.517089+0000","last_fullsized":"2023-10-01T09:16:31.517089+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":98,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521854+0000","last_change":"2023-10-01T09:13:13.550588+0000","last_active":"2023-10-01T09:16:31.521854+0000","last_peered":"2023-10-01T09:16:31.521854+0000","last_clean":"2023-10-01T09:16:31.521854+0000","last_became_active":"2023-10-01T09:13:13.550271+0000","last_became_peered":"2023-10-01T09:13:13.550271+0000","last_unstale":"2023-10-01T09:16:31.521854+0000","last_undegraded":"2023-10-01T09:16:31.521854+0000","last_fullsized":"2023-10-01T09:16:31.521854+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:19:47.301376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"129'8482","reported_seq":8513,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.272193+0000","last_change":"2023-10-01T09:14:31.351571+0000","last_active":"2023-10-01T09:16:41.272193+0000","last_peered":"2023-10-01T09:16:41.272193+0000","last_clean":"2023-10-01T09:16:41.272193+0000","last_became_active":"2023-10-01T09:14:31.351242+0000","last_became_peered":"2023-10-01T09:14:31.351242+0000","last_unstale":"2023-10-01T09:16:41.272193+0000","last_undegraded":"2023-10-01T09:16:41.272193+0000","last_fullsized":"2023-10-01T09:16:41.272193+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8482,"log_dups_size":0,"ondisk_log_size":8482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402259968,"num_objects":6138,"num_object_clones":0,"num_object_copies":18414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8482,"num_write_kb":467840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":96,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521298+0000","last_change":"2023-10-01T09:10:58.797850+0000","last_active":"2023-10-01T09:16:31.521298+0000","last_peered":"2023-10-01T09:16:31.521298+0000","last_clean":"2023-10-01T09:16:31.521298+0000","last_became_active":"2023-10-01T09:10:58.797403+0000","last_became_peered":"2023-10-01T09:10:58.797403+0000","last_unstale":"2023-10-01T09:16:31.521298+0000","last_undegraded":"2023-10-01T09:16:31.521298+0000","last_fullsized":"2023-10-01T09:16:31.521298+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":79,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.435756+0000","last_change":"2023-10-01T09:03:04.245105+0000","last_active":"2023-10-01T09:16:31.435756+0000","last_peered":"2023-10-01T09:16:31.435756+0000","last_clean":"2023-10-01T09:16:31.435756+0000","last_became_active":"2023-10-01T09:03:04.244807+0000","last_became_peered":"2023-10-01T09:03:04.244807+0000","last_unstale":"2023-10-01T09:16:31.435756+0000","last_undegraded":"2023-10-01T09:16:31.435756+0000","last_fullsized":"2023-10-01T09:16:31.435756+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"129'8626","reported_seq":8664,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.841438+0000","last_change":"2023-10-01T09:16:23.535190+0000","last_active":"2023-10-01T09:16:41.841438+0000","last_peered":"2023-10-01T09:16:41.841438+0000","last_clean":"2023-10-01T09:16:41.841438+0000","last_became_active":"2023-10-01T09:16:23.525193+0000","last_became_peered":"2023-10-01T09:16:23.525193+0000","last_unstale":"2023-10-01T09:16:41.841438+0000","last_undegraded":"2023-10-01T09:16:41.841438+0000","last_fullsized":"2023-10-01T09:16:41.841438+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8626,"log_dups_size":0,"ondisk_log_size":8626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:55:19.695436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401735680,"num_objects":6130,"num_object_clones":0,"num_object_copies":18390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8626,"num_write_kb":472192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":97,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521806+0000","last_change":"2023-10-01T09:12:10.003552+0000","last_active":"2023-10-01T09:16:31.521806+0000","last_peered":"2023-10-01T09:16:31.521806+0000","last_clean":"2023-10-01T09:16:31.521806+0000","last_became_active":"2023-10-01T09:12:10.003131+0000","last_became_peered":"2023-10-01T09:12:10.003131+0000","last_unstale":"2023-10-01T09:16:31.521806+0000","last_undegraded":"2023-10-01T09:16:31.521806+0000","last_fullsized":"2023-10-01T09:16:31.521806+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":108,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.515651+0000","last_change":"2023-10-01T09:13:13.556674+0000","last_active":"2023-10-01T09:16:31.515651+0000","last_peered":"2023-10-01T09:16:31.515651+0000","last_clean":"2023-10-01T09:16:31.515651+0000","last_became_active":"2023-10-01T09:13:13.556271+0000","last_became_peered":"2023-10-01T09:13:13.556271+0000","last_unstale":"2023-10-01T09:16:31.515651+0000","last_undegraded":"2023-10-01T09:16:31.515651+0000","last_fullsized":"2023-10-01T09:16:31.515651+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"129'8468","reported_seq":8508,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.730783+0000","last_change":"2023-10-01T09:16:23.536677+0000","last_active":"2023-10-01T09:16:41.730783+0000","last_peered":"2023-10-01T09:16:41.730783+0000","last_clean":"2023-10-01T09:16:41.730783+0000","last_became_active":"2023-10-01T09:16:23.527778+0000","last_became_peered":"2023-10-01T09:16:23.527778+0000","last_unstale":"2023-10-01T09:16:41.730783+0000","last_undegraded":"2023-10-01T09:16:41.730783+0000","last_fullsized":"2023-10-01T09:16:41.730783+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8468,"log_dups_size":0,"ondisk_log_size":8468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:12:23.765297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394264576,"num_objects":6016,"num_object_clones":0,"num_object_copies":18048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8468,"num_write_kb":463488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":81,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521686+0000","last_change":"2023-10-01T09:03:03.200972+0000","last_active":"2023-10-01T09:16:31.521686+0000","last_peered":"2023-10-01T09:16:31.521686+0000","last_clean":"2023-10-01T09:16:31.521686+0000","last_became_active":"2023-10-01T09:03:03.200695+0000","last_became_peered":"2023-10-01T09:03:03.200695+0000","last_unstale":"2023-10-01T09:16:31.521686+0000","last_undegraded":"2023-10-01T09:16:31.521686+0000","last_fullsized":"2023-10-01T09:16:31.521686+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":94,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.523193+0000","last_change":"2023-10-01T09:12:09.999571+0000","last_active":"2023-10-01T09:16:31.523193+0000","last_peered":"2023-10-01T09:16:31.523193+0000","last_clean":"2023-10-01T09:16:31.523193+0000","last_became_active":"2023-10-01T09:12:09.999272+0000","last_became_peered":"2023-10-01T09:12:09.999272+0000","last_unstale":"2023-10-01T09:16:31.523193+0000","last_undegraded":"2023-10-01T09:16:31.523193+0000","last_fullsized":"2023-10-01T09:16:31.523193+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"129'8284","reported_seq":8322,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.345572+0000","last_change":"2023-10-01T09:16:23.547265+0000","last_active":"2023-10-01T09:16:41.345572+0000","last_peered":"2023-10-01T09:16:41.345572+0000","last_clean":"2023-10-01T09:16:41.345572+0000","last_became_active":"2023-10-01T09:16:23.528916+0000","last_became_peered":"2023-10-01T09:16:23.528916+0000","last_unstale":"2023-10-01T09:16:41.345572+0000","last_undegraded":"2023-10-01T09:16:41.345572+0000","last_fullsized":"2023-10-01T09:16:41.345572+0000","mapping_epoch":125,"log_start":"112'900","ondisk_log_start":"112'900","created":106,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":7384,"log_dups_size":0,"ondisk_log_size":7384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379584512,"num_objects":5792,"num_object_clones":0,"num_object_copies":17376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8284,"num_write_kb":450432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":95,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.435991+0000","last_change":"2023-10-01T09:13:13.555458+0000","last_active":"2023-10-01T09:16:31.435991+0000","last_peered":"2023-10-01T09:16:31.435991+0000","last_clean":"2023-10-01T09:16:31.435991+0000","last_became_active":"2023-10-01T09:13:13.555058+0000","last_became_peered":"2023-10-01T09:13:13.555058+0000","last_unstale":"2023-10-01T09:16:31.435991+0000","last_undegraded":"2023-10-01T09:16:31.435991+0000","last_fullsized":"2023-10-01T09:16:31.435991+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":77,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.516824+0000","last_change":"2023-10-01T09:12:10.001565+0000","last_active":"2023-10-01T09:16:31.516824+0000","last_peered":"2023-10-01T09:16:31.516824+0000","last_clean":"2023-10-01T09:16:31.516824+0000","last_became_active":"2023-10-01T09:12:10.001275+0000","last_became_peered":"2023-10-01T09:12:10.001275+0000","last_unstale":"2023-10-01T09:16:31.516824+0000","last_undegraded":"2023-10-01T09:16:31.516824+0000","last_fullsized":"2023-10-01T09:16:31.516824+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:39.067046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"129'8443","reported_seq":8484,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.261272+0000","last_change":"2023-10-01T09:16:23.542801+0000","last_active":"2023-10-01T09:16:41.261272+0000","last_peered":"2023-10-01T09:16:41.261272+0000","last_clean":"2023-10-01T09:16:41.261272+0000","last_became_active":"2023-10-01T09:16:23.529252+0000","last_became_peered":"2023-10-01T09:16:23.529252+0000","last_unstale":"2023-10-01T09:16:41.261272+0000","last_undegraded":"2023-10-01T09:16:41.261272+0000","last_fullsized":"2023-10-01T09:16:41.261272+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8443,"log_dups_size":0,"ondisk_log_size":8443,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391118872,"num_objects":5969,"num_object_clones":0,"num_object_copies":17907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5969,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8443,"num_write_kb":461121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":96,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521971+0000","last_change":"2023-10-01T09:13:13.554321+0000","last_active":"2023-10-01T09:16:31.521971+0000","last_peered":"2023-10-01T09:16:31.521971+0000","last_clean":"2023-10-01T09:16:31.521971+0000","last_became_active":"2023-10-01T09:13:13.553986+0000","last_became_peered":"2023-10-01T09:13:13.553986+0000","last_unstale":"2023-10-01T09:16:31.521971+0000","last_undegraded":"2023-10-01T09:16:31.521971+0000","last_fullsized":"2023-10-01T09:16:31.521971+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:09:12.848269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":93,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.514406+0000","last_change":"2023-10-01T09:13:13.556491+0000","last_active":"2023-10-01T09:16:31.514406+0000","last_peered":"2023-10-01T09:16:31.514406+0000","last_clean":"2023-10-01T09:16:31.514406+0000","last_became_active":"2023-10-01T09:13:13.556118+0000","last_became_peered":"2023-10-01T09:13:13.556118+0000","last_unstale":"2023-10-01T09:16:31.514406+0000","last_undegraded":"2023-10-01T09:16:31.514406+0000","last_fullsized":"2023-10-01T09:16:31.514406+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"129'8491","reported_seq":8523,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:43.007041+0000","last_change":"2023-10-01T09:14:31.349314+0000","last_active":"2023-10-01T09:16:43.007041+0000","last_peered":"2023-10-01T09:16:43.007041+0000","last_clean":"2023-10-01T09:16:43.007041+0000","last_became_active":"2023-10-01T09:14:31.349041+0000","last_became_peered":"2023-10-01T09:14:31.349041+0000","last_unstale":"2023-10-01T09:16:43.007041+0000","last_undegraded":"2023-10-01T09:16:43.007041+0000","last_fullsized":"2023-10-01T09:16:43.007041+0000","mapping_epoch":106,"log_start":"111'300","ondisk_log_start":"111'300","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8191,"log_dups_size":0,"ondisk_log_size":8191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:20:18.137610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384761856,"num_objects":5871,"num_object_clones":0,"num_object_copies":17613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8491,"num_write_kb":459584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":95,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.523365+0000","last_change":"2023-10-01T09:12:10.003934+0000","last_active":"2023-10-01T09:16:31.523365+0000","last_peered":"2023-10-01T09:16:31.523365+0000","last_clean":"2023-10-01T09:16:31.523365+0000","last_became_active":"2023-10-01T09:12:10.003597+0000","last_became_peered":"2023-10-01T09:12:10.003597+0000","last_unstale":"2023-10-01T09:16:31.523365+0000","last_undegraded":"2023-10-01T09:16:31.523365+0000","last_fullsized":"2023-10-01T09:16:31.523365+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":108,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.514431+0000","last_change":"2023-10-01T09:12:40.946791+0000","last_active":"2023-10-01T09:16:31.514431+0000","last_peered":"2023-10-01T09:16:31.514431+0000","last_clean":"2023-10-01T09:16:31.514431+0000","last_became_active":"2023-10-01T09:12:40.946418+0000","last_became_peered":"2023-10-01T09:12:40.946418+0000","last_unstale":"2023-10-01T09:16:31.514431+0000","last_undegraded":"2023-10-01T09:16:31.514431+0000","last_fullsized":"2023-10-01T09:16:31.514431+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"129'8453","reported_seq":8484,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.262541+0000","last_change":"2023-10-01T09:14:31.355388+0000","last_active":"2023-10-01T09:16:41.262541+0000","last_peered":"2023-10-01T09:16:41.262541+0000","last_clean":"2023-10-01T09:16:41.262541+0000","last_became_active":"2023-10-01T09:14:31.355065+0000","last_became_peered":"2023-10-01T09:14:31.355065+0000","last_unstale":"2023-10-01T09:16:41.262541+0000","last_undegraded":"2023-10-01T09:16:41.262541+0000","last_fullsized":"2023-10-01T09:16:41.262541+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8453,"log_dups_size":0,"ondisk_log_size":8453,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392101888,"num_objects":5983,"num_object_clones":0,"num_object_copies":17949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8453,"num_write_kb":461952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":94,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.515445+0000","last_change":"2023-10-01T09:12:40.945099+0000","last_active":"2023-10-01T09:16:31.515445+0000","last_peered":"2023-10-01T09:16:31.515445+0000","last_clean":"2023-10-01T09:16:31.515445+0000","last_became_active":"2023-10-01T09:12:40.944712+0000","last_became_peered":"2023-10-01T09:12:40.944712+0000","last_unstale":"2023-10-01T09:16:31.515445+0000","last_undegraded":"2023-10-01T09:16:31.515445+0000","last_fullsized":"2023-10-01T09:16:31.515445+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":93,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.515963+0000","last_change":"2023-10-01T09:12:10.003847+0000","last_active":"2023-10-01T09:16:31.515963+0000","last_peered":"2023-10-01T09:16:31.515963+0000","last_clean":"2023-10-01T09:16:31.515963+0000","last_became_active":"2023-10-01T09:12:10.003525+0000","last_became_peered":"2023-10-01T09:12:10.003525+0000","last_unstale":"2023-10-01T09:16:31.515963+0000","last_undegraded":"2023-10-01T09:16:31.515963+0000","last_fullsized":"2023-10-01T09:16:31.515963+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"129'8362","reported_seq":8393,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.328520+0000","last_change":"2023-10-01T09:14:31.356833+0000","last_active":"2023-10-01T09:16:41.328520+0000","last_peered":"2023-10-01T09:16:41.328520+0000","last_clean":"2023-10-01T09:16:41.328520+0000","last_became_active":"2023-10-01T09:14:31.356520+0000","last_became_peered":"2023-10-01T09:14:31.356520+0000","last_unstale":"2023-10-01T09:16:41.328520+0000","last_undegraded":"2023-10-01T09:16:41.328520+0000","last_fullsized":"2023-10-01T09:16:41.328520+0000","mapping_epoch":106,"log_start":"112'1000","ondisk_log_start":"112'1000","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":7362,"log_dups_size":0,"ondisk_log_size":7362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387186688,"num_objects":5908,"num_object_clones":0,"num_object_copies":17724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8362,"num_write_kb":456640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":99,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521723+0000","last_change":"2023-10-01T09:13:13.555563+0000","last_active":"2023-10-01T09:16:31.521723+0000","last_peered":"2023-10-01T09:16:31.521723+0000","last_clean":"2023-10-01T09:16:31.521723+0000","last_became_active":"2023-10-01T09:13:13.555070+0000","last_became_peered":"2023-10-01T09:13:13.555070+0000","last_unstale":"2023-10-01T09:16:31.521723+0000","last_undegraded":"2023-10-01T09:16:31.521723+0000","last_fullsized":"2023-10-01T09:16:31.521723+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:44.557168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":49,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.522023+0000","last_change":"2023-10-01T09:16:23.527487+0000","last_active":"2023-10-01T09:16:31.522023+0000","last_peered":"2023-10-01T09:16:31.522023+0000","last_clean":"2023-10-01T09:16:31.522023+0000","last_became_active":"2023-10-01T09:16:23.527214+0000","last_became_peered":"2023-10-01T09:16:23.527214+0000","last_unstale":"2023-10-01T09:16:31.522023+0000","last_undegraded":"2023-10-01T09:16:31.522023+0000","last_fullsized":"2023-10-01T09:16:31.522023+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:18:54.916504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":943,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.511245+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:16:31.511245+0000","last_peered":"2023-10-01T09:16:31.511245+0000","last_clean":"2023-10-01T09:16:31.511245+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:16:31.511245+0000","last_undegraded":"2023-10-01T09:16:31.511245+0000","last_fullsized":"2023-10-01T09:16:31.511245+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":110,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.436009+0000","last_change":"2023-10-01T09:13:13.551681+0000","last_active":"2023-10-01T09:16:31.436009+0000","last_peered":"2023-10-01T09:16:31.436009+0000","last_clean":"2023-10-01T09:16:31.436009+0000","last_became_active":"2023-10-01T09:13:13.551268+0000","last_became_peered":"2023-10-01T09:13:13.551268+0000","last_unstale":"2023-10-01T09:16:31.436009+0000","last_undegraded":"2023-10-01T09:16:31.436009+0000","last_fullsized":"2023-10-01T09:16:31.436009+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":58,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.510836+0000","last_change":"2023-10-01T09:12:10.001493+0000","last_active":"2023-10-01T09:16:31.510836+0000","last_peered":"2023-10-01T09:16:31.510836+0000","last_clean":"2023-10-01T09:16:31.510836+0000","last_became_active":"2023-10-01T09:12:10.000899+0000","last_became_peered":"2023-10-01T09:12:10.000899+0000","last_unstale":"2023-10-01T09:16:31.510836+0000","last_undegraded":"2023-10-01T09:16:31.510836+0000","last_fullsized":"2023-10-01T09:16:31.510836+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:47:54.011059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"129'8476","reported_seq":8509,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.858657+0000","last_change":"2023-10-01T09:14:31.357395+0000","last_active":"2023-10-01T09:16:41.858657+0000","last_peered":"2023-10-01T09:16:41.858657+0000","last_clean":"2023-10-01T09:16:41.858657+0000","last_became_active":"2023-10-01T09:14:31.357014+0000","last_became_peered":"2023-10-01T09:14:31.357014+0000","last_unstale":"2023-10-01T09:16:41.858657+0000","last_undegraded":"2023-10-01T09:16:41.858657+0000","last_fullsized":"2023-10-01T09:16:41.858657+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8476,"log_dups_size":0,"ondisk_log_size":8476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:29.461431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393478144,"num_objects":6004,"num_object_clones":0,"num_object_copies":18012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8476,"num_write_kb":463360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"129'8288","reported_seq":8320,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.820397+0000","last_change":"2023-10-01T09:14:31.349232+0000","last_active":"2023-10-01T09:16:41.820397+0000","last_peered":"2023-10-01T09:16:41.820397+0000","last_clean":"2023-10-01T09:16:41.820397+0000","last_became_active":"2023-10-01T09:14:31.348940+0000","last_became_peered":"2023-10-01T09:14:31.348940+0000","last_unstale":"2023-10-01T09:16:41.820397+0000","last_undegraded":"2023-10-01T09:16:41.820397+0000","last_fullsized":"2023-10-01T09:16:41.820397+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8288,"log_dups_size":0,"ondisk_log_size":8288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:17.358604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382992384,"num_objects":5844,"num_object_clones":0,"num_object_copies":17532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8288,"num_write_kb":452224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":829,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.435827+0000","last_change":"2023-10-01T09:12:09.997936+0000","last_active":"2023-10-01T09:16:31.435827+0000","last_peered":"2023-10-01T09:16:31.435827+0000","last_clean":"2023-10-01T09:16:31.435827+0000","last_became_active":"2023-10-01T09:12:09.997427+0000","last_became_peered":"2023-10-01T09:12:09.997427+0000","last_unstale":"2023-10-01T09:16:31.435827+0000","last_undegraded":"2023-10-01T09:16:31.435827+0000","last_fullsized":"2023-10-01T09:16:31.435827+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1219,"num_read_kb":1114,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":94,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.513979+0000","last_change":"2023-10-01T09:10:58.799722+0000","last_active":"2023-10-01T09:16:31.513979+0000","last_peered":"2023-10-01T09:16:31.513979+0000","last_clean":"2023-10-01T09:16:31.513979+0000","last_became_active":"2023-10-01T09:10:58.799313+0000","last_became_peered":"2023-10-01T09:10:58.799313+0000","last_unstale":"2023-10-01T09:16:31.513979+0000","last_undegraded":"2023-10-01T09:16:31.513979+0000","last_fullsized":"2023-10-01T09:16:31.513979+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":88,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.511162+0000","last_change":"2023-10-01T09:13:13.549385+0000","last_active":"2023-10-01T09:16:31.511162+0000","last_peered":"2023-10-01T09:16:31.511162+0000","last_clean":"2023-10-01T09:16:31.511162+0000","last_became_active":"2023-10-01T09:13:13.549035+0000","last_became_peered":"2023-10-01T09:13:13.549035+0000","last_unstale":"2023-10-01T09:16:31.511162+0000","last_undegraded":"2023-10-01T09:16:31.511162+0000","last_fullsized":"2023-10-01T09:16:31.511162+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:52:41.408995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"129'8449","reported_seq":8489,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.260759+0000","last_change":"2023-10-01T09:16:23.548576+0000","last_active":"2023-10-01T09:16:41.260759+0000","last_peered":"2023-10-01T09:16:41.260759+0000","last_clean":"2023-10-01T09:16:41.260759+0000","last_became_active":"2023-10-01T09:16:23.528916+0000","last_became_peered":"2023-10-01T09:16:23.528916+0000","last_unstale":"2023-10-01T09:16:41.260759+0000","last_undegraded":"2023-10-01T09:16:41.260759+0000","last_fullsized":"2023-10-01T09:16:41.260759+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8449,"log_dups_size":0,"ondisk_log_size":8449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395640832,"num_objects":6037,"num_object_clones":0,"num_object_copies":18111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8449,"num_write_kb":463552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":335,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521880+0000","last_change":"2023-10-01T09:16:23.526129+0000","last_active":"2023-10-01T09:16:31.521880+0000","last_peered":"2023-10-01T09:16:31.521880+0000","last_clean":"2023-10-01T09:16:31.521880+0000","last_became_active":"2023-10-01T09:16:23.525493+0000","last_became_peered":"2023-10-01T09:16:23.525493+0000","last_unstale":"2023-10-01T09:16:31.521880+0000","last_undegraded":"2023-10-01T09:16:31.521880+0000","last_fullsized":"2023-10-01T09:16:31.521880+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:10.097349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":73,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.432137+0000","last_change":"2023-10-01T09:12:40.946267+0000","last_active":"2023-10-01T09:16:31.432137+0000","last_peered":"2023-10-01T09:16:31.432137+0000","last_clean":"2023-10-01T09:16:31.432137+0000","last_became_active":"2023-10-01T09:12:40.945970+0000","last_became_peered":"2023-10-01T09:12:40.945970+0000","last_unstale":"2023-10-01T09:16:31.432137+0000","last_undegraded":"2023-10-01T09:16:31.432137+0000","last_fullsized":"2023-10-01T09:16:31.432137+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:42.904509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":103,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.510998+0000","last_change":"2023-10-01T09:12:09.994071+0000","last_active":"2023-10-01T09:16:31.510998+0000","last_peered":"2023-10-01T09:16:31.510998+0000","last_clean":"2023-10-01T09:16:31.510998+0000","last_became_active":"2023-10-01T09:12:09.993782+0000","last_became_peered":"2023-10-01T09:12:09.993782+0000","last_unstale":"2023-10-01T09:16:31.510998+0000","last_undegraded":"2023-10-01T09:16:31.510998+0000","last_fullsized":"2023-10-01T09:16:31.510998+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:12:49.880685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":945,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.435899+0000","last_change":"2023-10-01T09:12:40.946131+0000","last_active":"2023-10-01T09:16:31.435899+0000","last_peered":"2023-10-01T09:16:31.435899+0000","last_clean":"2023-10-01T09:16:31.435899+0000","last_became_active":"2023-10-01T09:12:40.945524+0000","last_became_peered":"2023-10-01T09:12:40.945524+0000","last_unstale":"2023-10-01T09:16:31.435899+0000","last_undegraded":"2023-10-01T09:16:31.435899+0000","last_fullsized":"2023-10-01T09:16:31.435899+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":331,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.520643+0000","last_change":"2023-10-01T09:10:58.798697+0000","last_active":"2023-10-01T09:16:31.520643+0000","last_peered":"2023-10-01T09:16:31.520643+0000","last_clean":"2023-10-01T09:16:31.520643+0000","last_became_active":"2023-10-01T09:10:58.798203+0000","last_became_peered":"2023-10-01T09:10:58.798203+0000","last_unstale":"2023-10-01T09:16:31.520643+0000","last_undegraded":"2023-10-01T09:16:31.520643+0000","last_fullsized":"2023-10-01T09:16:31.520643+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":88,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521687+0000","last_change":"2023-10-01T09:13:13.556049+0000","last_active":"2023-10-01T09:16:31.521687+0000","last_peered":"2023-10-01T09:16:31.521687+0000","last_clean":"2023-10-01T09:16:31.521687+0000","last_became_active":"2023-10-01T09:13:13.555739+0000","last_became_peered":"2023-10-01T09:13:13.555739+0000","last_unstale":"2023-10-01T09:16:31.521687+0000","last_undegraded":"2023-10-01T09:16:31.521687+0000","last_fullsized":"2023-10-01T09:16:31.521687+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:50:34.475690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":79,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.517294+0000","last_change":"2023-10-01T09:03:03.227254+0000","last_active":"2023-10-01T09:16:31.517294+0000","last_peered":"2023-10-01T09:16:31.517294+0000","last_clean":"2023-10-01T09:16:31.517294+0000","last_became_active":"2023-10-01T09:03:03.226941+0000","last_became_peered":"2023-10-01T09:03:03.226941+0000","last_unstale":"2023-10-01T09:16:31.517294+0000","last_undegraded":"2023-10-01T09:16:31.517294+0000","last_fullsized":"2023-10-01T09:16:31.517294+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"129'8199","reported_seq":8230,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.233768+0000","last_change":"2023-10-01T09:14:31.353858+0000","last_active":"2023-10-01T09:16:41.233768+0000","last_peered":"2023-10-01T09:16:41.233768+0000","last_clean":"2023-10-01T09:16:41.233768+0000","last_became_active":"2023-10-01T09:14:31.353572+0000","last_became_peered":"2023-10-01T09:14:31.353572+0000","last_unstale":"2023-10-01T09:16:41.233768+0000","last_undegraded":"2023-10-01T09:16:41.233768+0000","last_fullsized":"2023-10-01T09:16:41.233768+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8199,"log_dups_size":0,"ondisk_log_size":8199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383451136,"num_objects":5851,"num_object_clones":0,"num_object_copies":17553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8199,"num_write_kb":449600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"129'8454","reported_seq":8485,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.299665+0000","last_change":"2023-10-01T09:14:31.354366+0000","last_active":"2023-10-01T09:16:41.299665+0000","last_peered":"2023-10-01T09:16:41.299665+0000","last_clean":"2023-10-01T09:16:41.299665+0000","last_became_active":"2023-10-01T09:14:31.354059+0000","last_became_peered":"2023-10-01T09:14:31.354059+0000","last_unstale":"2023-10-01T09:16:41.299665+0000","last_undegraded":"2023-10-01T09:16:41.299665+0000","last_fullsized":"2023-10-01T09:16:41.299665+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8454,"log_dups_size":0,"ondisk_log_size":8454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394657792,"num_objects":6022,"num_object_clones":0,"num_object_copies":18066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8454,"num_write_kb":463232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":225,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.520719+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:16:31.520719+0000","last_peered":"2023-10-01T09:16:31.520719+0000","last_clean":"2023-10-01T09:16:31.520719+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:16:31.520719+0000","last_undegraded":"2023-10-01T09:16:31.520719+0000","last_fullsized":"2023-10-01T09:16:31.520719+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":77,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.515749+0000","last_change":"2023-10-01T09:13:13.552391+0000","last_active":"2023-10-01T09:16:31.515749+0000","last_peered":"2023-10-01T09:16:31.515749+0000","last_clean":"2023-10-01T09:16:31.515749+0000","last_became_active":"2023-10-01T09:13:13.552082+0000","last_became_peered":"2023-10-01T09:13:13.552082+0000","last_unstale":"2023-10-01T09:16:31.515749+0000","last_undegraded":"2023-10-01T09:16:31.515749+0000","last_fullsized":"2023-10-01T09:16:31.515749+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:07:19.929732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":92,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.515452+0000","last_change":"2023-10-01T09:12:40.944991+0000","last_active":"2023-10-01T09:16:31.515452+0000","last_peered":"2023-10-01T09:16:31.515452+0000","last_clean":"2023-10-01T09:16:31.515452+0000","last_became_active":"2023-10-01T09:12:40.944688+0000","last_became_peered":"2023-10-01T09:12:40.944688+0000","last_unstale":"2023-10-01T09:16:31.515452+0000","last_undegraded":"2023-10-01T09:16:31.515452+0000","last_fullsized":"2023-10-01T09:16:31.515452+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:30.015801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":489,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521594+0000","last_change":"2023-10-01T09:10:58.797964+0000","last_active":"2023-10-01T09:16:31.521594+0000","last_peered":"2023-10-01T09:16:31.521594+0000","last_clean":"2023-10-01T09:16:31.521594+0000","last_became_active":"2023-10-01T09:10:58.797447+0000","last_became_peered":"2023-10-01T09:10:58.797447+0000","last_unstale":"2023-10-01T09:16:31.521594+0000","last_undegraded":"2023-10-01T09:16:31.521594+0000","last_fullsized":"2023-10-01T09:16:31.521594+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":73,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.511324+0000","last_change":"2023-10-01T09:12:40.948216+0000","last_active":"2023-10-01T09:16:31.511324+0000","last_peered":"2023-10-01T09:16:31.511324+0000","last_clean":"2023-10-01T09:16:31.511324+0000","last_became_active":"2023-10-01T09:12:40.947879+0000","last_became_peered":"2023-10-01T09:12:40.947879+0000","last_unstale":"2023-10-01T09:16:31.511324+0000","last_undegraded":"2023-10-01T09:16:31.511324+0000","last_fullsized":"2023-10-01T09:16:31.511324+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:46:40.612337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":62,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.517030+0000","last_change":"2023-10-01T09:10:58.797034+0000","last_active":"2023-10-01T09:16:31.517030+0000","last_peered":"2023-10-01T09:16:31.517030+0000","last_clean":"2023-10-01T09:16:31.517030+0000","last_became_active":"2023-10-01T09:10:58.796730+0000","last_became_peered":"2023-10-01T09:10:58.796730+0000","last_unstale":"2023-10-01T09:16:31.517030+0000","last_undegraded":"2023-10-01T09:16:31.517030+0000","last_fullsized":"2023-10-01T09:16:31.517030+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:32:15.982890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"129'8373","reported_seq":8403,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.186530+0000","last_change":"2023-10-01T09:14:31.353560+0000","last_active":"2023-10-01T09:16:41.186530+0000","last_peered":"2023-10-01T09:16:41.186530+0000","last_clean":"2023-10-01T09:16:41.186530+0000","last_became_active":"2023-10-01T09:14:31.353242+0000","last_became_peered":"2023-10-01T09:14:31.353242+0000","last_unstale":"2023-10-01T09:16:41.186530+0000","last_undegraded":"2023-10-01T09:16:41.186530+0000","last_fullsized":"2023-10-01T09:16:41.186530+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8373,"log_dups_size":0,"ondisk_log_size":8373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390660096,"num_objects":5961,"num_object_clones":0,"num_object_copies":17883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8373,"num_write_kb":458688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":96,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521945+0000","last_change":"2023-10-01T09:13:13.553580+0000","last_active":"2023-10-01T09:16:31.521945+0000","last_peered":"2023-10-01T09:16:31.521945+0000","last_clean":"2023-10-01T09:16:31.521945+0000","last_became_active":"2023-10-01T09:13:13.553011+0000","last_became_peered":"2023-10-01T09:13:13.553011+0000","last_unstale":"2023-10-01T09:16:31.521945+0000","last_undegraded":"2023-10-01T09:16:31.521945+0000","last_fullsized":"2023-10-01T09:16:31.521945+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:54:30.369308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":517,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.511993+0000","last_change":"2023-10-01T09:12:10.002334+0000","last_active":"2023-10-01T09:16:31.511993+0000","last_peered":"2023-10-01T09:16:31.511993+0000","last_clean":"2023-10-01T09:16:31.511993+0000","last_became_active":"2023-10-01T09:12:10.001791+0000","last_became_peered":"2023-10-01T09:12:10.001791+0000","last_unstale":"2023-10-01T09:16:31.511993+0000","last_undegraded":"2023-10-01T09:16:31.511993+0000","last_fullsized":"2023-10-01T09:16:31.511993+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:24:27.632780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":93,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521633+0000","last_change":"2023-10-01T09:13:13.552192+0000","last_active":"2023-10-01T09:16:31.521633+0000","last_peered":"2023-10-01T09:16:31.521633+0000","last_clean":"2023-10-01T09:16:31.521633+0000","last_became_active":"2023-10-01T09:13:13.551887+0000","last_became_peered":"2023-10-01T09:13:13.551887+0000","last_unstale":"2023-10-01T09:16:31.521633+0000","last_undegraded":"2023-10-01T09:16:31.521633+0000","last_fullsized":"2023-10-01T09:16:31.521633+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:23.633557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"129'8589","reported_seq":8620,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.289522+0000","last_change":"2023-10-01T09:14:31.350884+0000","last_active":"2023-10-01T09:16:41.289522+0000","last_peered":"2023-10-01T09:16:41.289522+0000","last_clean":"2023-10-01T09:16:41.289522+0000","last_became_active":"2023-10-01T09:14:31.350533+0000","last_became_peered":"2023-10-01T09:14:31.350533+0000","last_unstale":"2023-10-01T09:16:41.289522+0000","last_undegraded":"2023-10-01T09:16:41.289522+0000","last_fullsized":"2023-10-01T09:16:41.289522+0000","mapping_epoch":106,"log_start":"112'1200","ondisk_log_start":"112'1200","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":7389,"log_dups_size":0,"ondisk_log_size":7389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400883712,"num_objects":6117,"num_object_clones":0,"num_object_copies":18351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8589,"num_write_kb":470592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":167,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521000+0000","last_change":"2023-10-01T09:12:40.949544+0000","last_active":"2023-10-01T09:16:31.521000+0000","last_peered":"2023-10-01T09:16:31.521000+0000","last_clean":"2023-10-01T09:16:31.521000+0000","last_became_active":"2023-10-01T09:12:40.949054+0000","last_became_peered":"2023-10-01T09:12:40.949054+0000","last_unstale":"2023-10-01T09:16:31.521000+0000","last_undegraded":"2023-10-01T09:16:31.521000+0000","last_fullsized":"2023-10-01T09:16:31.521000+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":81,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.515380+0000","last_change":"2023-10-01T09:12:10.003174+0000","last_active":"2023-10-01T09:16:31.515380+0000","last_peered":"2023-10-01T09:16:31.515380+0000","last_clean":"2023-10-01T09:16:31.515380+0000","last_became_active":"2023-10-01T09:12:10.002808+0000","last_became_peered":"2023-10-01T09:12:10.002808+0000","last_unstale":"2023-10-01T09:16:31.515380+0000","last_undegraded":"2023-10-01T09:16:31.515380+0000","last_fullsized":"2023-10-01T09:16:31.515380+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:25.037091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":78,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.515525+0000","last_change":"2023-10-01T09:03:03.754839+0000","last_active":"2023-10-01T09:16:31.515525+0000","last_peered":"2023-10-01T09:16:31.515525+0000","last_clean":"2023-10-01T09:16:31.515525+0000","last_became_active":"2023-10-01T09:03:03.754506+0000","last_became_peered":"2023-10-01T09:03:03.754506+0000","last_unstale":"2023-10-01T09:16:31.515525+0000","last_undegraded":"2023-10-01T09:16:31.515525+0000","last_fullsized":"2023-10-01T09:16:31.515525+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"129'8337","reported_seq":8370,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-10-01T09:16:41.620113+0000","last_change":"2023-10-01T09:14:31.352926+0000","last_active":"2023-10-01T09:16:41.620113+0000","last_peered":"2023-10-01T09:16:41.620113+0000","last_clean":"2023-10-01T09:16:41.620113+0000","last_became_active":"2023-10-01T09:14:31.352602+0000","last_became_peered":"2023-10-01T09:14:31.352602+0000","last_unstale":"2023-10-01T09:16:41.620113+0000","last_undegraded":"2023-10-01T09:16:41.620113+0000","last_fullsized":"2023-10-01T09:16:41.620113+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8337,"log_dups_size":0,"ondisk_log_size":8337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:40:47.614933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383582208,"num_objects":5853,"num_object_clones":0,"num_object_copies":17559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8337,"num_write_kb":454080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":66,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.516961+0000","last_change":"2023-10-01T09:10:58.801161+0000","last_active":"2023-10-01T09:16:31.516961+0000","last_peered":"2023-10-01T09:16:31.516961+0000","last_clean":"2023-10-01T09:16:31.516961+0000","last_became_active":"2023-10-01T09:10:58.800794+0000","last_became_peered":"2023-10-01T09:10:58.800794+0000","last_unstale":"2023-10-01T09:16:31.516961+0000","last_undegraded":"2023-10-01T09:16:31.516961+0000","last_fullsized":"2023-10-01T09:16:31.516961+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:35.019445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":99,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.511388+0000","last_change":"2023-10-01T09:16:23.527340+0000","last_active":"2023-10-01T09:16:31.511388+0000","last_peered":"2023-10-01T09:16:31.511388+0000","last_clean":"2023-10-01T09:16:31.511388+0000","last_became_active":"2023-10-01T09:16:23.527019+0000","last_became_peered":"2023-10-01T09:16:23.527019+0000","last_unstale":"2023-10-01T09:16:31.511388+0000","last_undegraded":"2023-10-01T09:16:31.511388+0000","last_fullsized":"2023-10-01T09:16:31.511388+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:32:15.521151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":97,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.436082+0000","last_change":"2023-10-01T09:12:10.000449+0000","last_active":"2023-10-01T09:16:31.436082+0000","last_peered":"2023-10-01T09:16:31.436082+0000","last_clean":"2023-10-01T09:16:31.436082+0000","last_became_active":"2023-10-01T09:12:10.000000+0000","last_became_peered":"2023-10-01T09:12:10.000000+0000","last_unstale":"2023-10-01T09:16:31.436082+0000","last_undegraded":"2023-10-01T09:16:31.436082+0000","last_fullsized":"2023-10-01T09:16:31.436082+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":108,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.514142+0000","last_change":"2023-10-01T09:13:13.555744+0000","last_active":"2023-10-01T09:16:31.514142+0000","last_peered":"2023-10-01T09:16:31.514142+0000","last_clean":"2023-10-01T09:16:31.514142+0000","last_became_active":"2023-10-01T09:13:13.555163+0000","last_became_peered":"2023-10-01T09:13:13.555163+0000","last_unstale":"2023-10-01T09:16:31.514142+0000","last_undegraded":"2023-10-01T09:16:31.514142+0000","last_fullsized":"2023-10-01T09:16:31.514142+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":82,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.436020+0000","last_change":"2023-10-01T09:03:03.202063+0000","last_active":"2023-10-01T09:16:31.436020+0000","last_peered":"2023-10-01T09:16:31.436020+0000","last_clean":"2023-10-01T09:16:31.436020+0000","last_became_active":"2023-10-01T09:03:03.201769+0000","last_became_peered":"2023-10-01T09:03:03.201769+0000","last_unstale":"2023-10-01T09:16:31.436020+0000","last_undegraded":"2023-10-01T09:16:31.436020+0000","last_fullsized":"2023-10-01T09:16:31.436020+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":108,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.514176+0000","last_change":"2023-10-01T09:13:13.554959+0000","last_active":"2023-10-01T09:16:31.514176+0000","last_peered":"2023-10-01T09:16:31.514176+0000","last_clean":"2023-10-01T09:16:31.514176+0000","last_became_active":"2023-10-01T09:13:13.554589+0000","last_became_peered":"2023-10-01T09:13:13.554589+0000","last_unstale":"2023-10-01T09:16:31.514176+0000","last_undegraded":"2023-10-01T09:16:31.514176+0000","last_fullsized":"2023-10-01T09:16:31.514176+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":104,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521520+0000","last_change":"2023-10-01T09:13:13.556295+0000","last_active":"2023-10-01T09:16:31.521520+0000","last_peered":"2023-10-01T09:16:31.521520+0000","last_clean":"2023-10-01T09:16:31.521520+0000","last_became_active":"2023-10-01T09:13:13.555953+0000","last_became_peered":"2023-10-01T09:13:13.555953+0000","last_unstale":"2023-10-01T09:16:31.521520+0000","last_undegraded":"2023-10-01T09:16:31.521520+0000","last_fullsized":"2023-10-01T09:16:31.521520+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:04:06.892156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":117,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.521718+0000","last_change":"2023-10-01T09:13:13.549800+0000","last_active":"2023-10-01T09:16:31.521718+0000","last_peered":"2023-10-01T09:16:31.521718+0000","last_clean":"2023-10-01T09:16:31.521718+0000","last_became_active":"2023-10-01T09:13:13.549480+0000","last_became_peered":"2023-10-01T09:13:13.549480+0000","last_unstale":"2023-10-01T09:16:31.521718+0000","last_undegraded":"2023-10-01T09:16:31.521718+0000","last_fullsized":"2023-10-01T09:16:31.521718+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:08.673916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":81,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.516967+0000","last_change":"2023-10-01T09:12:40.944944+0000","last_active":"2023-10-01T09:16:31.516967+0000","last_peered":"2023-10-01T09:16:31.516967+0000","last_clean":"2023-10-01T09:16:31.516967+0000","last_became_active":"2023-10-01T09:12:40.944613+0000","last_became_peered":"2023-10-01T09:12:40.944613+0000","last_unstale":"2023-10-01T09:16:31.516967+0000","last_undegraded":"2023-10-01T09:16:31.516967+0000","last_fullsized":"2023-10-01T09:16:31.516967+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:48:20.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":93,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-10-01T09:16:31.513805+0000","last_change":"2023-10-01T09:12:10.003833+0000","last_active":"2023-10-01T09:16:31.513805+0000","last_peered":"2023-10-01T09:16:31.513805+0000","last_clean":"2023-10-01T09:16:31.513805+0000","last_became_active":"2023-10-01T09:12:10.003482+0000","last_became_peered":"2023-10-01T09:12:10.003482+0000","last_unstale":"2023-10-01T09:16:31.513805+0000","last_undegraded":"2023-10-01T09:16:31.513805+0000","last_fullsized":"2023-10-01T09:16:31.513805+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":6258360344,"num_objects":95496,"num_object_clones":0,"num_object_copies":286488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":95496,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":134774,"num_write_kb":7368577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18811334656,"data_stored":18811322440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131374,"ondisk_log_size":131374,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27349,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12188,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430393,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2242176,"kb_used_data":1944888,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91502976,"statfs":{"total":95995035648,"available":93699047424,"internally_reserved":0,"allocated":1991565312,"data_stored":1991042245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:16:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":1.8460000000000001,"15min":0.996},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":7.359,"5min":94.162999999999997,"15min":94.162999999999997},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.444,"5min":1.722,"15min":0.93899999999999995},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":9.1270000000000007,"5min":94.135000000000005,"15min":94.135000000000005},"last":2.3220000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:16:18 2023","interfaces":[{"interface":"back","average":{"1min":4.718,"5min":1.5449999999999999,"15min":0.89100000000000001},"min":{"1min":0.312,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":43.801000000000002,"5min":43.801000000000002,"15min":43.801000000000002},"last":0.23999999999999999},{"interface":"front","average":{"1min":3.8210000000000002,"5min":1.373,"15min":0.82699999999999996},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":43.328000000000003,"5min":43.328000000000003,"15min":43.328000000000003},"last":2.2709999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:15:45 2023","interfaces":[{"interface":"back","average":{"1min":4.8049999999999997,"5min":1.395,"15min":0.83099999999999996},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":82.379000000000005,"5min":82.379000000000005,"15min":82.379000000000005},"last":1.075},{"interface":"front","average":{"1min":5.2199999999999998,"5min":1.48,"15min":0.85199999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":85.596000000000004,"5min":85.596000000000004,"15min":85.596000000000004},"last":2.298}]},{"osd":3,"last update":"Sun Oct 1 09:16:18 2023","interfaces":[{"interface":"back","average":{"1min":1.9830000000000001,"5min":0.88100000000000001,"15min":0.67300000000000004},"min":{"1min":0.36699999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.1970000000000001,"5min":6.1970000000000001,"15min":6.1970000000000001},"last":2.1850000000000001},{"interface":"front","average":{"1min":1.8109999999999999,"5min":0.81299999999999994,"15min":0.63500000000000001},"min":{"1min":0.312,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.4080000000000004,"5min":6.4080000000000004,"15min":6.4080000000000004},"last":0.32100000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:16:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5269999999999999,"5min":0.69099999999999995,"15min":0.54900000000000004},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.883,"5min":4.883,"15min":4.883},"last":2.1720000000000002},{"interface":"front","average":{"1min":1.504,"5min":0.72399999999999998,"15min":0.60399999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":5.4950000000000001,"5min":5.4950000000000001,"15min":5.5369999999999999},"last":2.238}]},{"osd":5,"last update":"Sun Oct 1 09:16:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":0.65300000000000002,"15min":0.55900000000000005},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28799999999999998},"max":{"1min":6.3719999999999999,"5min":6.3719999999999999,"15min":6.3719999999999999},"last":3.6920000000000002},{"interface":"front","average":{"1min":1.72,"5min":0.82299999999999995,"15min":0.63700000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28599999999999998},"max":{"1min":7.8310000000000004,"5min":7.8310000000000004,"15min":7.8310000000000004},"last":2.8719999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:16:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":0.748,"15min":0.61899999999999999},"min":{"1min":0.34699999999999998,"5min":0.311,"15min":0.29999999999999999},"max":{"1min":4.3399999999999999,"5min":4.3399999999999999,"15min":4.3399999999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.3720000000000001,"5min":0.76600000000000001,"15min":0.66000000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.33400000000000002},"max":{"1min":5.5250000000000004,"5min":5.5250000000000004,"15min":5.5700000000000003},"last":2.7599999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626622,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1855468,"kb_used_data":1550052,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91889684,"statfs":{"total":95995035648,"available":94095036416,"internally_reserved":0,"allocated":1587253248,"data_stored":1586749880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:16:02 2023","interfaces":[{"interface":"back","average":{"1min":11.459,"5min":2.786,"15min":1.3460000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":139.69900000000001,"5min":139.69900000000001,"15min":139.69900000000001},"last":0.94499999999999995},{"interface":"front","average":{"1min":11.013999999999999,"5min":2.6869999999999998,"15min":1.3149999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":139.614,"5min":139.614,"15min":139.614},"last":9.6270000000000007}]},{"osd":1,"last update":"Sun Oct 1 09:16:20 2023","interfaces":[{"interface":"back","average":{"1min":3.1320000000000001,"5min":1.135,"15min":0.76200000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":24.488,"5min":24.488,"15min":24.488},"last":0.97299999999999998},{"interface":"front","average":{"1min":3.3679999999999999,"5min":1.28,"15min":0.82799999999999996},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":24.643999999999998,"5min":24.643999999999998,"15min":24.643999999999998},"last":9.5950000000000006}]},{"osd":2,"last update":"Sun Oct 1 09:15:44 2023","interfaces":[{"interface":"back","average":{"1min":11.718,"5min":2.7749999999999999,"15min":1.2769999999999999},"min":{"1min":0.41599999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":82.978999999999999,"5min":82.978999999999999,"15min":82.978999999999999},"last":9.5640000000000001},{"interface":"front","average":{"1min":11.433999999999999,"5min":2.7210000000000001,"15min":1.2549999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":82.747,"5min":82.747,"15min":82.747},"last":10.861000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:16:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":0.81799999999999995,"15min":0.64900000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.0119999999999996,"5min":7.0119999999999996,"15min":7.0119999999999996},"last":0.995},{"interface":"front","average":{"1min":2.3130000000000002,"5min":1.0800000000000001,"15min":0.73399999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.5119999999999996,"5min":7.5119999999999996,"15min":7.5119999999999996},"last":9.6519999999999992}]},{"osd":4,"last update":"Sun Oct 1 09:15:42 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.57499999999999996,"15min":0.502},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.25900000000000001},"max":{"1min":4.0099999999999998,"5min":4.0099999999999998,"15min":4.0099999999999998},"last":7.3109999999999999},{"interface":"front","average":{"1min":1.0669999999999999,"5min":0.64100000000000001,"15min":0.55400000000000005},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.1920000000000002,"5min":5.1920000000000002,"15min":5.1920000000000002},"last":2.5150000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:15:42 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":0.63,"15min":0.54800000000000004},"min":{"1min":0.34599999999999997,"5min":0.32200000000000001,"15min":0.251},"max":{"1min":4.0990000000000002,"5min":4.0990000000000002,"15min":4.0990000000000002},"last":9.1180000000000003},{"interface":"front","average":{"1min":1.1519999999999999,"5min":0.69799999999999995,"15min":0.59499999999999997},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":5.1559999999999997,"5min":5.1559999999999997,"15min":5.1559999999999997},"last":1.0189999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:15:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":0.69699999999999995,"15min":0.59699999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.29899999999999999},"max":{"1min":5.2640000000000002,"5min":5.2640000000000002,"15min":5.2640000000000002},"last":12.866},{"interface":"front","average":{"1min":1.1299999999999999,"5min":0.73399999999999999,"15min":0.63300000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.4710000000000001,"5min":6.4710000000000001,"15min":6.4710000000000001},"last":1.04}]}]},{"osd":1,"up_from":94,"seq":403726925881,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1966104,"kb_used_data":1898320,"kb_used_omap":7,"kb_used_meta":67768,"kb_avail":91779048,"statfs":{"total":95995035648,"available":93981745152,"internally_reserved":0,"allocated":1943879680,"data_stored":1943276541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":69394855},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":4.226,"5min":1.161,"15min":0.61699999999999999},"min":{"1min":0.23200000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":28.449999999999999,"5min":28.449999999999999,"15min":28.449999999999999},"last":0.28199999999999997},{"interface":"front","average":{"1min":3.5369999999999999,"5min":1.0780000000000001,"15min":0.60399999999999998},"min":{"1min":0.19400000000000001,"5min":0.188,"15min":0.188},"max":{"1min":25.806999999999999,"5min":25.806999999999999,"15min":25.806999999999999},"last":0.39700000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:15:46 2023","interfaces":[{"interface":"back","average":{"1min":2.0950000000000002,"5min":0.69099999999999995,"15min":0.45200000000000001},"min":{"1min":0.24299999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":13.9,"5min":13.9,"15min":13.9},"last":0.251},{"interface":"front","average":{"1min":2.3839999999999999,"5min":0.75600000000000001,"15min":0.47599999999999998},"min":{"1min":0.216,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":8.2089999999999996,"5min":8.2089999999999996,"15min":8.2089999999999996},"last":0.22600000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:16:21 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":0.77900000000000003,"15min":0.50800000000000001},"min":{"1min":0.25900000000000001,"5min":0.218,"15min":0.218},"max":{"1min":6.2640000000000002,"5min":6.2640000000000002,"15min":6.2640000000000002},"last":3.3700000000000001},{"interface":"front","average":{"1min":1.1319999999999999,"5min":0.80600000000000005,"15min":0.52400000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.9009999999999998,"5min":13.853,"15min":13.853},"last":0.46899999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":0.82799999999999996,"15min":0.63800000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.9649999999999999,"5min":3.9649999999999999,"15min":3.9649999999999999},"last":1.327},{"interface":"front","average":{"1min":2.5169999999999999,"5min":1.0289999999999999,"15min":0.71699999999999997},"min":{"1min":0.33400000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":13.112,"5min":13.112,"15min":13.112},"last":0.49199999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":0.98399999999999999,"15min":0.69799999999999995},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":6.2919999999999998,"5min":6.2919999999999998,"15min":6.2919999999999998},"last":2.4660000000000002},{"interface":"front","average":{"1min":1.9299999999999999,"5min":0.93799999999999994,"15min":0.71099999999999997},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.2309999999999999,"5min":7.2309999999999999,"15min":7.2309999999999999},"last":1.649}]},{"osd":6,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":0.92100000000000004,"15min":0.70699999999999996},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.2300000000000004,"5min":6.2300000000000004,"15min":6.2300000000000004},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.421,"5min":0.86099999999999999,"15min":0.70899999999999996},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":3.9340000000000002,"5min":3.9340000000000002,"15min":3.9340000000000002},"last":1.292}]},{"osd":7,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8280000000000001,"5min":0.91700000000000004,"15min":0.70699999999999996},"min":{"1min":0.39600000000000002,"5min":0.376,"15min":0.376},"max":{"1min":7.2770000000000001,"5min":7.2770000000000001,"15min":7.2770000000000001},"last":2.5880000000000001},{"interface":"front","average":{"1min":2.714,"5min":1.0640000000000001,"15min":0.75800000000000001},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":13.071999999999999,"5min":13.071999999999999,"15min":13.071999999999999},"last":2.569}]}]},{"osd":0,"up_from":89,"seq":382252089415,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3182500,"kb_used_data":3078620,"kb_used_omap":8,"kb_used_meta":103863,"kb_avail":90562652,"statfs":{"total":95995035648,"available":92736155648,"internally_reserved":0,"allocated":3152506880,"data_stored":3151895953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":106356674},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":0.78300000000000003,"15min":0.47599999999999998},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":9.6669999999999998,"5min":9.6669999999999998,"15min":9.6669999999999998},"last":0.26300000000000001},{"interface":"front","average":{"1min":2.1970000000000001,"5min":0.79000000000000004,"15min":0.51200000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":9.3079999999999998,"5min":9.3079999999999998,"15min":9.3079999999999998},"last":0.36299999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:15:49 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":0.72899999999999998,"15min":0.503},"min":{"1min":0.248,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.7539999999999996,"5min":8.7539999999999996,"15min":8.7539999999999996},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.389,"5min":0.59499999999999997,"15min":0.46899999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.9159999999999999,"5min":3.9159999999999999,"15min":3.9159999999999999},"last":0.45800000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":0.80700000000000005,"15min":0.52000000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.906000000000001,"5min":12.906000000000001,"15min":12.906000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":2.3149999999999999,"5min":0.80100000000000005,"15min":0.51100000000000001},"min":{"1min":0.29399999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":12.145,"5min":12.145,"15min":12.145},"last":5.2519999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:16:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5209999999999999,"5min":1.0149999999999999,"15min":0.76600000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":9.9540000000000006,"5min":9.9540000000000006,"15min":9.9540000000000006},"last":0.438},{"interface":"front","average":{"1min":2.9990000000000001,"5min":1.107,"15min":0.82099999999999995},"min":{"1min":0.29299999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":12.212999999999999,"5min":12.212999999999999,"15min":12.212999999999999},"last":13.459}]},{"osd":5,"last update":"Sun Oct 1 09:16:10 2023","interfaces":[{"interface":"back","average":{"1min":3.032,"5min":1.133,"15min":0.83699999999999997},"min":{"1min":0.36199999999999999,"5min":0.311,"15min":0.311},"max":{"1min":12.263999999999999,"5min":12.263999999999999,"15min":12.263999999999999},"last":5.2990000000000004},{"interface":"front","average":{"1min":2.6669999999999998,"5min":1.1850000000000001,"15min":0.85299999999999998},"min":{"1min":0.45700000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":12.779,"5min":12.779,"15min":12.779},"last":1.0900000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:16:10 2023","interfaces":[{"interface":"back","average":{"1min":3.6640000000000001,"5min":1.2889999999999999,"15min":0.89500000000000002},"min":{"1min":0.42399999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":12.239000000000001,"5min":12.239000000000001,"15min":12.239000000000001},"last":5.327},{"interface":"front","average":{"1min":2.6160000000000001,"5min":1.1559999999999999,"15min":0.84199999999999997},"min":{"1min":0.44800000000000001,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":12.875,"5min":12.875,"15min":12.875},"last":0.46800000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:16:10 2023","interfaces":[{"interface":"back","average":{"1min":2.907,"5min":1.2370000000000001,"15min":0.88300000000000001},"min":{"1min":0.49299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":12.840999999999999,"5min":12.840999999999999,"15min":12.840999999999999},"last":16.48},{"interface":"front","average":{"1min":2.6880000000000002,"5min":1.111,"15min":0.83899999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":5.8899999999999997,"5min":5.8899999999999997,"15min":5.8899999999999997},"last":2.0670000000000002}]}]},{"osd":2,"up_from":99,"seq":425201762354,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2430344,"kb_used_data":2350336,"kb_used_omap":11,"kb_used_meta":79988,"kb_avail":91314808,"statfs":{"total":95995035648,"available":93506363392,"internally_reserved":0,"allocated":2406744064,"data_stored":2406148029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":81908599},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":5.266,"5min":1.3260000000000001,"15min":0.66000000000000003},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":85.838999999999999,"5min":85.838999999999999,"15min":85.838999999999999},"last":3.52},{"interface":"front","average":{"1min":6.601,"5min":1.605,"15min":0.76300000000000001},"min":{"1min":0.35999999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":85.938000000000002,"5min":85.938000000000002,"15min":85.938000000000002},"last":4.6820000000000004}]},{"osd":1,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":6.6360000000000001,"5min":1.6240000000000001,"15min":0.78500000000000003},"min":{"1min":0.21299999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":105.3,"5min":105.3,"15min":105.3},"last":3.339},{"interface":"front","average":{"1min":7.7619999999999996,"5min":1.859,"15min":0.86699999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":107.729,"5min":107.729,"15min":107.729},"last":4.899}]},{"osd":3,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":0.623,"15min":0.45400000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.758,"5min":4.758,"15min":4.758},"last":3.5419999999999998},{"interface":"front","average":{"1min":1.4730000000000001,"5min":0.70099999999999996,"15min":0.50800000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":5.9169999999999998,"5min":5.9169999999999998,"15min":5.9169999999999998},"last":3.3599999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":0.747,"15min":0.65100000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.5170000000000003,"5min":6.5170000000000003,"15min":6.5170000000000003},"last":2.3879999999999999},{"interface":"front","average":{"1min":1.7290000000000001,"5min":0.79800000000000004,"15min":0.63900000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.1689999999999996,"5min":6.1689999999999996,"15min":6.1689999999999996},"last":3.4889999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":0.89600000000000002,"15min":0.68200000000000005},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.2999999999999998,"5min":7.2999999999999998,"15min":7.2999999999999998},"last":4.3460000000000001},{"interface":"front","average":{"1min":1.798,"5min":0.82699999999999996,"15min":0.65800000000000003},"min":{"1min":0.377,"5min":0.317,"15min":0.317},"max":{"1min":5.1959999999999997,"5min":5.1959999999999997,"15min":5.1959999999999997},"last":8.2609999999999992}]},{"osd":6,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":0.78500000000000003,"15min":0.66200000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.9699999999999998,"5min":5.9699999999999998,"15min":5.9699999999999998},"last":4.3630000000000004},{"interface":"front","average":{"1min":1.026,"5min":0.71099999999999997,"15min":0.65200000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.9580000000000002,"5min":5.9580000000000002,"15min":5.9580000000000002},"last":3.3100000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":0.78100000000000003,"15min":0.66100000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.1970000000000001,"5min":6.1970000000000001,"15min":6.1970000000000001},"last":8.7690000000000001},{"interface":"front","average":{"1min":2.1110000000000002,"5min":0.92200000000000004,"15min":0.71599999999999997},"min":{"1min":0.40100000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.5510000000000002,"5min":7.5510000000000002,"15min":7.5510000000000002},"last":10.885999999999999}]}]},{"osd":3,"up_from":104,"seq":446676598828,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1202384,"kb_used_data":1144648,"kb_used_omap":3,"kb_used_meta":57724,"kb_avail":92542768,"statfs":{"total":95995035648,"available":94763794432,"internally_reserved":0,"allocated":1172119552,"data_stored":1171522404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":59110119},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6160000000000001,"5min":4.4720000000000004,"15min":1.734},"min":{"1min":0.214,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":41.917000000000002,"5min":199.422,"15min":199.422},"last":2.343},{"interface":"front","average":{"1min":4.6020000000000003,"5min":4.6340000000000003,"15min":1.778},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":41.567,"5min":199.91200000000001,"15min":199.91200000000001},"last":3.8919999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.76,"5min":2.234,"15min":0.997},"min":{"1min":0.27000000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":9.9399999999999995,"5min":154.32599999999999,"15min":154.32599999999999},"last":4.2329999999999997},{"interface":"front","average":{"1min":1.742,"5min":2.25,"15min":1},"min":{"1min":0.255,"5min":0.25,"15min":0.25},"max":{"1min":8.6649999999999991,"5min":154.249,"15min":154.249},"last":2.1589999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":0.94399999999999995,"15min":0.55000000000000004},"min":{"1min":0.23000000000000001,"5min":0.224,"15min":0.224},"max":{"1min":30.771999999999998,"5min":30.771999999999998,"15min":30.771999999999998},"last":2.222},{"interface":"front","average":{"1min":3.0569999999999999,"5min":1.016,"15min":0.61699999999999999},"min":{"1min":0.215,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":31.878,"5min":31.878,"15min":31.878},"last":2.1899999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":0.73999999999999999,"15min":0.624},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.1120000000000001,"5min":4.093,"15min":4.093},"last":5.3300000000000001},{"interface":"front","average":{"1min":1.575,"5min":0.84499999999999997,"15min":0.69099999999999995},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":8.2300000000000004,"5min":8.2300000000000004,"15min":8.2300000000000004},"last":4.0880000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2269999999999999,"5min":0.98299999999999998,"15min":0.73099999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.2769999999999992,"5min":8.2769999999999992,"15min":8.2769999999999992},"last":3.996},{"interface":"front","average":{"1min":1.5840000000000001,"5min":0.89200000000000002,"15min":0.70099999999999996},"min":{"1min":0.377,"5min":0.317,"15min":0.317},"max":{"1min":4.5540000000000003,"5min":7.601,"15min":7.601},"last":11.028}]},{"osd":6,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":1.0429999999999999,"15min":0.76700000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":8.3140000000000001,"5min":8.3140000000000001,"15min":8.3140000000000001},"last":6.6760000000000002},{"interface":"front","average":{"1min":1.583,"5min":0.98999999999999999,"15min":0.755},"min":{"1min":0.41499999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":6.5590000000000002,"5min":7.556,"15min":7.556},"last":6.718}]},{"osd":7,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":0.94999999999999996,"15min":0.74199999999999999},"min":{"1min":0.45600000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.1280000000000001,"5min":7.4950000000000001,"15min":7.4950000000000001},"last":5.0910000000000002},{"interface":"front","average":{"1min":1.9419999999999999,"5min":0.96899999999999997,"15min":0.751},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.5920000000000005,"5min":8.5920000000000005,"15min":8.5920000000000005},"last":5.0700000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019078,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3371700,"kb_used_data":3070252,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":90373452,"statfs":{"total":95995035648,"available":92542414848,"internally_reserved":0,"allocated":3143938048,"data_stored":3143426631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:16:05 2023","interfaces":[{"interface":"back","average":{"1min":8.9570000000000007,"5min":2.577,"15min":1.25},"min":{"1min":0.246,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":132.607,"5min":132.607,"15min":132.607},"last":0.91400000000000003},{"interface":"front","average":{"1min":8.2680000000000007,"5min":2.4470000000000001,"15min":1.2010000000000001},"min":{"1min":0.29399999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":132.49799999999999,"5min":132.49799999999999,"15min":132.49799999999999},"last":2.9009999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.089,"15min":0.753},"min":{"1min":0.315,"5min":0.313,"15min":0.313},"max":{"1min":17.559000000000001,"5min":17.559000000000001,"15min":17.559000000000001},"last":2.8799999999999999},{"interface":"front","average":{"1min":2.9369999999999998,"5min":1.079,"15min":0.72299999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":17.571000000000002,"5min":17.571000000000002,"15min":17.571000000000002},"last":6.3789999999999996}]},{"osd":2,"last update":"Sun Oct 1 09:15:44 2023","interfaces":[{"interface":"back","average":{"1min":13.41,"5min":3.1139999999999999,"15min":1.3919999999999999},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":85.441000000000003,"5min":85.441000000000003,"15min":85.441000000000003},"last":2.8159999999999998},{"interface":"front","average":{"1min":12.993,"5min":3.048,"15min":1.385},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":84.736000000000004,"5min":84.736000000000004,"15min":84.736000000000004},"last":2.7959999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:16:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0059999999999998,"5min":1.1020000000000001,"15min":0.72599999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":13.569000000000001,"5min":13.569000000000001,"15min":13.569000000000001},"last":2.9649999999999999},{"interface":"front","average":{"1min":2.4140000000000001,"5min":1.0089999999999999,"15min":0.70399999999999996},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":6.3799999999999999,"5min":6.3799999999999999,"15min":6.3799999999999999},"last":6.4649999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:16:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5739999999999998,"5min":0.96299999999999997,"15min":0.625},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26300000000000001},"max":{"1min":6.6260000000000003,"5min":6.6260000000000003,"15min":6.6260000000000003},"last":8.7089999999999996},{"interface":"front","average":{"1min":2.4500000000000002,"5min":1.0049999999999999,"15min":0.65400000000000003},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":13.446999999999999,"5min":13.446999999999999,"15min":13.446999999999999},"last":2.9790000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:16:40 2023","interfaces":[{"interface":"back","average":{"1min":7.0880000000000001,"5min":1.994,"15min":1.002},"min":{"1min":0.33500000000000002,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":133.90799999999999,"5min":133.90799999999999,"15min":133.90799999999999},"last":2.8370000000000002},{"interface":"front","average":{"1min":9.3849999999999998,"5min":2.4670000000000001,"15min":1.1930000000000001},"min":{"1min":0.34599999999999997,"5min":0.34200000000000003,"15min":0.33700000000000002},"max":{"1min":141.03,"5min":141.03,"15min":141.03},"last":6.4269999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:15:55 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":0.747,"15min":0.60699999999999998},"min":{"1min":0.38300000000000001,"5min":0.34100000000000003,"15min":0.317},"max":{"1min":4.819,"5min":4.819,"15min":4.819},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.2549999999999999,"5min":0.77800000000000002,"15min":0.63100000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.31},"max":{"1min":5.0529999999999999,"5min":5.0529999999999999,"15min":5.0529999999999999},"last":2.855}]}]},{"osd":5,"up_from":36,"seq":154618822851,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3767464,"kb_used_data":3466080,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":89977688,"statfs":{"total":95995035648,"available":92137152512,"internally_reserved":0,"allocated":3549265920,"data_stored":3548762263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5,2,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:16:07 2023","interfaces":[{"interface":"back","average":{"1min":1.7330000000000001,"5min":1.0489999999999999,"15min":0.77500000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":5.5039999999999996,"5min":14.186,"15min":14.186},"last":4.2430000000000003},{"interface":"front","average":{"1min":1.911,"5min":1.1299999999999999,"15min":0.78200000000000003},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":14.151,"5min":14.308,"15min":14.308},"last":2.7989999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:16:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":1.1799999999999999,"15min":0.74099999999999999},"min":{"1min":0.33300000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.6040000000000001,"5min":32.420000000000002,"15min":32.420000000000002},"last":4.21},{"interface":"front","average":{"1min":1.968,"5min":1.1919999999999999,"15min":0.76100000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.0700000000000003,"5min":32.481999999999999,"15min":32.481999999999999},"last":2.8759999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":0.88900000000000001,"15min":0.745},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":11.359,"5min":11.359,"15min":11.359},"last":4.2670000000000003},{"interface":"front","average":{"1min":1.921,"5min":0.871,"15min":0.70699999999999996},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.449,"5min":10.449,"15min":10.449},"last":2.8530000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:16:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3690000000000002,"5min":1.1579999999999999,"15min":0.96599999999999997},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.274,"5min":7.274,"15min":7.274},"last":4.2880000000000003},{"interface":"front","average":{"1min":1.9279999999999999,"5min":0.96199999999999997,"15min":0.77100000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":7.1340000000000003,"5min":7.1340000000000003,"15min":7.1340000000000003},"last":2.8279999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:16:10 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.57999999999999996,"15min":0.504},"min":{"1min":0.29499999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.8220000000000001,"5min":3.8220000000000001,"15min":3.8220000000000001},"last":2.758},{"interface":"front","average":{"1min":1.863,"5min":0.83299999999999996,"15min":0.622},"min":{"1min":0.32800000000000001,"5min":0.316,"15min":0.28100000000000003},"max":{"1min":8.4199999999999999,"5min":8.4199999999999999,"15min":8.4199999999999999},"last":0.747}]},{"osd":6,"last update":"Sun Oct 1 09:16:36 2023","interfaces":[{"interface":"back","average":{"1min":2.3170000000000002,"5min":1.02,"15min":0.69699999999999995},"min":{"1min":0.32000000000000001,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":9.6210000000000004,"5min":9.6210000000000004,"15min":9.6210000000000004},"last":0.77400000000000002},{"interface":"front","average":{"1min":2.7909999999999999,"5min":1.091,"15min":0.72599999999999998},"min":{"1min":0.35499999999999998,"5min":0.32600000000000001,"15min":0.29399999999999998},"max":{"1min":8.1880000000000006,"5min":8.1880000000000006,"15min":8.1880000000000006},"last":4.3639999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:15:57 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.67900000000000005,"15min":0.60399999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":3.7839999999999998,"5min":3.7839999999999998,"15min":3.7839999999999998},"last":8.3219999999999992},{"interface":"front","average":{"1min":1.1699999999999999,"5min":0.73399999999999999,"15min":0.622},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.2800000000000002,"5min":5.2800000000000002,"15min":5.2800000000000002},"last":4.0090000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3128426496,"data_stored":3128426496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1931804672,"data_stored":1931804672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2389446656,"data_stored":2389442584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1161428992,"data_stored":1161428992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3121348608,"data_stored":3121348608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3524923392,"data_stored":3524919320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1580531712,"data_stored":1580531712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1973424128,"data_stored":1973420056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:16:46.831 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_0 pgp_num 16 2023-10-01T09:16:46.835 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:16:46.835 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:16:46.853 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:46.963 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:47.059 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:47.192 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:47.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:47.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:47.297 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:47.325 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:47.368 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:47.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:47.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:47.472 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:47.516 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:47.797 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:47.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:47.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:47.814 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:16:47.915 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:47.996 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:48.031 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:16:48.044 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:48 smithi190 ceph-mon[160435]: pgmap v313: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T09:16:48.045 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:48 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1790161240' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:16:48.045 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:48 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:16:48.132 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:48.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:48.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:48.246 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:48.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:48 smithi138 ceph-mon[185886]: pgmap v313: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T09:16:48.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:48 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1790161240' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:16:48.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:48 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:16:48.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:48 smithi138 ceph-mon[192513]: pgmap v313: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T09:16:48.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:48 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1790161240' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:16:48.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:48 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:16:48.343 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:48.487 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:48.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:48.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:48.563 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:16:48.601 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:48.673 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:48.897 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:48.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:48.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:49.009 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:49.057 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:49.255 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:49 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:49.255 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:49 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:16:49.255 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:49 smithi190 ceph-mon[160435]: osdmap e130: 8 total, 8 up, 8 in 2023-10-01T09:16:49.255 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:49 smithi190 ceph-mon[160435]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.255 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:49 smithi190 ceph-mon[160435]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.255 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:49 smithi190 ceph-mon[160435]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.255 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:49 smithi190 ceph-mon[160435]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.256 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:49 smithi190 ceph-mon[160435]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:16:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[185886]: osdmap e130: 8 total, 8 up, 8 in 2023-10-01T09:16:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[185886]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[185886]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[185886]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[185886]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[185886]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:49.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:16:49.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[192513]: osdmap e130: 8 total, 8 up, 8 in 2023-10-01T09:16:49.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[192513]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[192513]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[192513]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[192513]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:49 smithi138 ceph-mon[192513]: failed to encode map e130 with expected crc 2023-10-01T09:16:49.593 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:49.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:49.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:49.613 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:49.697 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:49.776 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:49.881 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:49.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:49.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:49.987 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:50.055 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:50.129 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:50 smithi138 ceph-mon[185886]: pgmap v315: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-01T09:16:50.130 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:50 smithi138 ceph-mon[192513]: pgmap v315: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-01T09:16:50.171 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:16:50.174 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:50.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:50.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:50.204 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:16:50.243 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:16:50.278 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:16:50.280 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:50.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:50 smithi190 ceph-mon[160435]: pgmap v315: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-01T09:16:50.312 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:16:50.340 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:50.389 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:16:50.416 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:16:50.440 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:50.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:50.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:50.456 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:16:50.459 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:16:50.459 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:16:50.459 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:16:50.459 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:16:50.459 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:16:50.460 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:16:50.460 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:16:50.460 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:16:50.550 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:50.647 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:50.665 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:16:50.665 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:16:50.666 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.666 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.34683.0:141190 5.6 e130/125 5:61d8a79a:::benchmark_data_smithi138_240338_object26053:head v 130'8870, mlcod=130'8862)", 2023-10-01T09:16:50.666 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:50.637713+0000", 2023-10-01T09:16:50.666 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.017779587999999999, 2023-10-01T09:16:50.666 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017810796, 2023-10-01T09:16:50.666 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:50.667 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:16:50.667 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:50.667 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.667 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:50.667 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.637713+0000", 2023-10-01T09:16:50.667 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:50.667 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.667 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.667 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:50.667 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.637713+0000", 2023-10-01T09:16:50.668 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:16:50.668 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.668 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.668 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:50.668 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.637712+0000", 2023-10-01T09:16:50.668 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5569999999999998e-06 2023-10-01T09:16:50.668 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.668 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.668 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:50.668 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.637720+0000", 2023-10-01T09:16:50.668 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0713999999999999e-05 2023-10-01T09:16:50.668 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.669 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.669 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:50.669 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.637730+0000", 2023-10-01T09:16:50.669 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5609999999999998e-06 2023-10-01T09:16:50.669 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.669 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.669 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:50.669 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.637736+0000", 2023-10-01T09:16:50.669 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0076791619999999998 2023-10-01T09:16:50.669 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.669 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.669 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:50.670 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.645415+0000", 2023-10-01T09:16:50.670 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4826000000000002e-05 2023-10-01T09:16:50.670 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.670 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.670 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:50.670 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.645450+0000", 2023-10-01T09:16:50.670 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0077369939999999996 2023-10-01T09:16:50.670 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:50.670 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:50.670 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:50.670 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.671 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.671 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:141205 5.d 5:bf26e211:::benchmark_data_smithi138_240338_object26068:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-10-01T09:16:50.671 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:50.650208+0000", 2023-10-01T09:16:50.671 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0052846189999999999, 2023-10-01T09:16:50.671 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0054127070000000001, 2023-10-01T09:16:50.671 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:50.671 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:16:50.671 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:50.671 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:50.671 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:50.671 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 141205 2023-10-01T09:16:50.671 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.672 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:50.672 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.672 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:50.672 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650208+0000", 2023-10-01T09:16:50.672 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:50.672 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.672 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.672 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:50.672 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650208+0000", 2023-10-01T09:16:50.672 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:50.672 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.672 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.673 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:50.673 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650206+0000", 2023-10-01T09:16:50.673 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2119999999999992e-06 2023-10-01T09:16:50.673 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.673 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.673 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:50.673 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650215+0000", 2023-10-01T09:16:50.673 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3159999999999999e-05 2023-10-01T09:16:50.673 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.673 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.673 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:50.673 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650229+0000", 2023-10-01T09:16:50.673 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2649999999999999e-06 2023-10-01T09:16:50.674 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.674 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.674 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:50.674 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650236+0000", 2023-10-01T09:16:50.674 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000107816 2023-10-01T09:16:50.674 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.674 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.674 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:50.674 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650344+0000", 2023-10-01T09:16:50.674 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014740900000000001 2023-10-01T09:16:50.675 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.675 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.675 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:50.675 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650491+0000", 2023-10-01T09:16:50.675 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015850900000000001 2023-10-01T09:16:50.675 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.675 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.675 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:16:50.675 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650650+0000", 2023-10-01T09:16:50.675 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00044183999999999998 2023-10-01T09:16:50.675 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:50.675 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:50.675 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:50.676 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.676 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.676 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:141208 5.9 5:9e54ffac:::benchmark_data_smithi138_240338_object26071:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-10-01T09:16:50.676 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:50.650251+0000", 2023-10-01T09:16:50.676 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.00524158, 2023-10-01T09:16:50.676 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.005449873, 2023-10-01T09:16:50.676 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:50.676 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:16:50.676 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:50.676 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:50.676 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:50.676 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 141208 2023-10-01T09:16:50.677 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.677 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:50.677 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.677 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:50.677 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650251+0000", 2023-10-01T09:16:50.677 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:50.677 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.677 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.677 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:50.677 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650251+0000", 2023-10-01T09:16:50.677 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:16:50.677 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.678 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.678 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:50.678 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650250+0000", 2023-10-01T09:16:50.678 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4020000000000002e-06 2023-10-01T09:16:50.678 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.678 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.678 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:50.678 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650254+0000", 2023-10-01T09:16:50.678 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1739999999999997e-06 2023-10-01T09:16:50.678 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.678 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.678 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:50.679 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650260+0000", 2023-10-01T09:16:50.679 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1559999999999997e-06 2023-10-01T09:16:50.679 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.679 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.679 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:50.679 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650264+0000", 2023-10-01T09:16:50.679 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000102901 2023-10-01T09:16:50.679 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.679 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.679 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:50.679 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650367+0000", 2023-10-01T09:16:50.679 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000127952 2023-10-01T09:16:50.680 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.680 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.680 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:50.680 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.650495+0000", 2023-10-01T09:16:50.680 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.005041121 2023-10-01T09:16:50.680 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.680 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.680 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:16:50.680 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.655536+0000", 2023-10-01T09:16:50.680 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0052851850000000004 2023-10-01T09:16:50.680 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:50.681 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:50.681 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:50.681 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.681 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.681 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.34683.0:141203 5.1 e130/106 5:8f22b543:::benchmark_data_smithi138_240338_object26066:head v 130'9024, mlcod=130'9022)", 2023-10-01T09:16:50.681 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:50.654967+0000", 2023-10-01T09:16:50.681 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.00052537600000000003, 2023-10-01T09:16:50.681 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00080725600000000001, 2023-10-01T09:16:50.681 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:50.681 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:16:50.681 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:50.681 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.682 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:50.682 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.654967+0000", 2023-10-01T09:16:50.682 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:50.682 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.682 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.682 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:50.682 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.654967+0000", 2023-10-01T09:16:50.682 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:16:50.682 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.682 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.682 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:50.682 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.654966+0000", 2023-10-01T09:16:50.683 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0791999999999999e-05 2023-10-01T09:16:50.683 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.683 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.683 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:50.683 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.654977+0000", 2023-10-01T09:16:50.683 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1182000000000001e-05 2023-10-01T09:16:50.683 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.683 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.683 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:50.683 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.654998+0000", 2023-10-01T09:16:50.683 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8259999999999993e-06 2023-10-01T09:16:50.683 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.684 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.684 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:50.684 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.655006+0000", 2023-10-01T09:16:50.684 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012688599999999999 2023-10-01T09:16:50.684 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.684 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.684 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:50.684 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.655133+0000", 2023-10-01T09:16:50.684 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4773000000000002e-05 2023-10-01T09:16:50.684 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.684 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.684 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:50.685 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.655157+0000", 2023-10-01T09:16:50.685 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00019039299999999999 2023-10-01T09:16:50.685 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:50.685 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:50.685 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:50.685 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.685 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.685 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:141212 5.d 5:b7a63c38:::benchmark_data_smithi138_240338_object26075:head [delete] snapc 0=[] ondisk+write+known_if_redirected e130)", 2023-10-01T09:16:50.685 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:50.655099+0000", 2023-10-01T09:16:50.685 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.00039311199999999999, 2023-10-01T09:16:50.685 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000738052, 2023-10-01T09:16:50.686 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:16:50.686 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:16:50.686 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:16:50.686 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:16:50.686 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:16:50.686 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 141212 2023-10-01T09:16:50.686 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.686 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:16:50.686 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.687 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:16:50.687 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.655099+0000", 2023-10-01T09:16:50.687 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:16:50.687 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.687 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.687 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:16:50.687 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.655099+0000", 2023-10-01T09:16:50.687 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:16:50.687 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.687 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.687 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:16:50.688 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.655098+0000", 2023-10-01T09:16:50.688 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6100000000000006e-06 2023-10-01T09:16:50.688 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.688 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.688 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:16:50.688 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.655106+0000", 2023-10-01T09:16:50.688 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.199e-05 2023-10-01T09:16:50.688 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.688 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.689 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:16:50.689 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.655118+0000", 2023-10-01T09:16:50.689 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9789999999999996e-06 2023-10-01T09:16:50.689 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.689 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.689 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:16:50.689 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.655125+0000", 2023-10-01T09:16:50.689 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00051293400000000005 2023-10-01T09:16:50.689 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.689 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.690 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:16:50.690 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.655638+0000", 2023-10-01T09:16:50.690 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000106507 2023-10-01T09:16:50.690 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:16:50.690 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:16:50.690 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:16:50.690 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:50.655745+0000", 2023-10-01T09:16:50.690 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00064552299999999995 2023-10-01T09:16:50.691 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:50.691 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:16:50.691 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:50.691 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:16:50.691 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:16:50.691 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 5 2023-10-01T09:16:50.691 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:16:50.901 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:50.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:50.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:51.007 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:51.069 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:51.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:51 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/705731639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:51.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:51 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/705731639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:51.112 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:51.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:51.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:51.215 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:51.252 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:51.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:51 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/705731639' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:51.517 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:51.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:51.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:51.594 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:16:51.603 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:16:51.623 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:51.696 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:52.051 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:52.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:52 smithi190 ceph-mon[160435]: pgmap v316: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-10-01T09:16:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:52.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:52.167 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:52.241 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:52.302 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:16:52.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:52 smithi138 ceph-mon[192513]: pgmap v316: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-10-01T09:16:52.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:52 smithi138 ceph-mon[185886]: pgmap v316: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2023-10-01T09:16:52.449 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:52.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:52.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:52.560 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:52.628 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:52.656 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:52.736 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:52.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:52.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:52.845 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:52.916 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:53.014 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:53.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:53.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:53.127 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:53.159 INFO:teuthology.orchestra.run.smithi138.stderr:set pool 5 pgp_num to 16 2023-10-01T09:16:53.181 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:53.243 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:53.244 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3491111760' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T09:16:53.244 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:53 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T09:16:53.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:53.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:53.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3491111760' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T09:16:53.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:53 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T09:16:53.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3491111760' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T09:16:53.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:53 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T09:16:53.345 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:53.373 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:54.175 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:16:54.407 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:16:54.407 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:16:54.407 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:16:54.408 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:16:54.408 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:16:54.425 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:54 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T09:16:54.425 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:54 smithi190 ceph-mon[160435]: osdmap e131: 8 total, 8 up, 8 in 2023-10-01T09:16:54.425 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:54 smithi190 ceph-mon[160435]: failed to encode map e131 with expected crc 2023-10-01T09:16:54.425 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:54 smithi190 ceph-mon[160435]: failed to encode map e131 with expected crc 2023-10-01T09:16:54.425 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:54 smithi190 ceph-mon[160435]: pgmap v318: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-10-01T09:16:54.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:54 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T09:16:54.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:54 smithi138 ceph-mon[192513]: osdmap e131: 8 total, 8 up, 8 in 2023-10-01T09:16:54.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:54 smithi138 ceph-mon[192513]: failed to encode map e131 with expected crc 2023-10-01T09:16:54.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:54 smithi138 ceph-mon[192513]: failed to encode map e131 with expected crc 2023-10-01T09:16:54.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:54 smithi138 ceph-mon[192513]: pgmap v318: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-10-01T09:16:54.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:54 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T09:16:54.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:54 smithi138 ceph-mon[185886]: osdmap e131: 8 total, 8 up, 8 in 2023-10-01T09:16:54.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:54 smithi138 ceph-mon[185886]: failed to encode map e131 with expected crc 2023-10-01T09:16:54.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:54 smithi138 ceph-mon[185886]: failed to encode map e131 with expected crc 2023-10-01T09:16:54.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:54 smithi138 ceph-mon[185886]: pgmap v318: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-10-01T09:16:54.857 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:54.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:54.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:54.973 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:55.018 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:55.148 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:55.229 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:16:55.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:55.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:16:55.234 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:16:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:16:54] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-01T09:16:55.321 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:16:55.334 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:16:55.401 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:55.612 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:16:55.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:55.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:16:55.717 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:16:55.829 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:55.894 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:16:55.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:55.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:16:56.009 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:56.020 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:16:56.090 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:56.217 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:56.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:56.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:56.271 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:16:56.320 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:56.343 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:56.343 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:16:56.343 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:56 smithi190 ceph-mon[160435]: pgmap v319: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 452 op/s 2023-10-01T09:16:56.343 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:56 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4177363068' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:56.350 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:56.532 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:56.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:56.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:16:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:56 smithi138 ceph-mon[192513]: pgmap v319: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 452 op/s 2023-10-01T09:16:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:56 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4177363068' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:56.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:16:56.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:16:56.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:56 smithi138 ceph-mon[185886]: pgmap v319: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 452 op/s 2023-10-01T09:16:56.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:56 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4177363068' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:16:56.641 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:16:56.733 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:56.818 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:16:56.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:56.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:16:56.921 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:56.995 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:57.055 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:57.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:57.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:57.158 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:57.218 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:57.277 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:57 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:57.277 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:57 smithi190 ceph-mon[160435]: failed to encode map e132 with expected crc 2023-10-01T09:16:57.277 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4177363068' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:16:57.277 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:57 smithi190 ceph-mon[160435]: osdmap e132: 8 total, 8 up, 8 in 2023-10-01T09:16:57.546 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:57.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:57.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:57.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:57 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:57.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:57 smithi138 ceph-mon[185886]: failed to encode map e132 with expected crc 2023-10-01T09:16:57.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4177363068' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:16:57.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:57 smithi138 ceph-mon[185886]: osdmap e132: 8 total, 8 up, 8 in 2023-10-01T09:16:57.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:57 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:16:57.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:57 smithi138 ceph-mon[192513]: failed to encode map e132 with expected crc 2023-10-01T09:16:57.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4177363068' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:16:57.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:57 smithi138 ceph-mon[192513]: osdmap e132: 8 total, 8 up, 8 in 2023-10-01T09:16:57.667 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:57.739 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:57.899 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:16:57.925 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:57.932 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:16:57.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:57.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:57.965 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:16:58.001 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:16:58.035 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:58.039 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:16:58.076 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:16:58.087 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:58.102 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:16:58.133 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:16:58.134 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:16:58.135 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:16:58.135 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:16:58.135 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:16:58.135 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:16:58.135 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:16:58.135 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:16:58.135 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:16:58.215 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:16:58.216 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:16:58.220 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:58.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:58.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:58.326 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:58.399 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:58.516 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:58.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:58.530 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:58 smithi190 ceph-mon[160435]: pgmap v321: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2023-10-01T09:16:58.530 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:16:58 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/735945920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:58.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:58.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:58 smithi138 ceph-mon[192513]: pgmap v321: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2023-10-01T09:16:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:16:58 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/735945920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:58 smithi138 ceph-mon[185886]: pgmap v321: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2023-10-01T09:16:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:16:58 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/735945920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:16:58.632 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:58.657 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:58.699 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:58.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:58.803 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:16:58.850 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:59.128 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:16:59.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:59.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:16:59.140 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:16:59.239 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:16:59.292 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:59.386 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:16:59.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:59.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:16:59.430 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:16:59.501 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:16:59.596 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:16:59.770 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:16:59.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:16:59.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:16:59.864 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:16:59.873 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:16:59.873 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:16:59.873 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-01T09:16:59.873 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:16:59.873 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:16:59.873 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-01T09:16:59.879 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:16:59.984 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:00.134 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:00.211 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:00.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:00.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:00.317 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:00.371 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:00.463 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:00.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:00.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:00.483 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:00 smithi138 ceph-mon[185886]: pgmap v322: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2023-10-01T09:17:00.483 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:00 smithi138 ceph-mon[192513]: pgmap v322: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2023-10-01T09:17:00.576 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:00.655 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:00 smithi190 ceph-mon[160435]: pgmap v322: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2023-10-01T09:17:00.658 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:00.769 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:00.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:00.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:00.881 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:00.944 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:01.048 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:01.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:01.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:01.154 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:17:01.236 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:01.246 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:17:01.316 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:17:01.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:01.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:17:01.425 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:17:01.493 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:01.574 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:17:01.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:01.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:17:01.687 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:17:01.761 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:01.826 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:17:01.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:01.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:17:01.931 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:17:02.001 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:02.564 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:02.564 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:17:02.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:02.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:17:02.670 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:02.733 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:02.738 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:02 smithi138 ceph-mon[192513]: pgmap v323: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 534 op/s 2023-10-01T09:17:02.739 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:02 smithi138 ceph-mon[185886]: pgmap v323: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 534 op/s 2023-10-01T09:17:02.744 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:02.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:02 smithi190 ceph-mon[160435]: pgmap v323: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 534 op/s 2023-10-01T09:17:02.823 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:02.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:02.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:02.931 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:03.017 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:03.347 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:03.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:03.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:03.454 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:17:03.505 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:03.543 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:17:03.544 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:17:03.619 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:17:03.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:03.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:17:03.727 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:17:03.817 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:04.407 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:17:04.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:04 smithi138 ceph-mon[192513]: pgmap v324: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-10-01T09:17:04.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3709724584' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:17:04.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:04 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:17:04.782 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:04 smithi138 ceph-mon[185886]: pgmap v324: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-10-01T09:17:04.782 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3709724584' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:17:04.782 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:04 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:17:04.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:04 smithi190 ceph-mon[160435]: pgmap v324: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-10-01T09:17:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3709724584' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:17:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:04 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:17:05.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:17:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:17:04] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.33.4" 2023-10-01T09:17:05.725 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:05 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:17:05.725 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:05 smithi138 ceph-mon[185886]: failed to encode map e133 with expected crc 2023-10-01T09:17:05.725 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:05 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:17:05.725 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:05 smithi138 ceph-mon[185886]: osdmap e133: 8 total, 8 up, 8 in 2023-10-01T09:17:05.725 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:05 smithi138 ceph-mon[185886]: failed to encode map e133 with expected crc 2023-10-01T09:17:05.725 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:05 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:17:05.725 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:05 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:17:05.726 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:05 smithi138 ceph-mon[192513]: failed to encode map e133 with expected crc 2023-10-01T09:17:05.726 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:05 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:17:05.726 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:05 smithi138 ceph-mon[192513]: osdmap e133: 8 total, 8 up, 8 in 2023-10-01T09:17:05.726 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:05 smithi138 ceph-mon[192513]: failed to encode map e133 with expected crc 2023-10-01T09:17:05.726 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:05 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:17:05.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:05 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:17:05.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:05 smithi190 ceph-mon[160435]: failed to encode map e133 with expected crc 2023-10-01T09:17:05.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:05 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:17:05.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:05 smithi190 ceph-mon[160435]: osdmap e133: 8 total, 8 up, 8 in 2023-10-01T09:17:05.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:05 smithi190 ceph-mon[160435]: failed to encode map e133 with expected crc 2023-10-01T09:17:05.814 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:05 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:17:06.057 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:17:06.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:06.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:17:06.163 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:17:06.259 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:06.359 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:17:06.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:06.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:17:06.471 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:17:06.500 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:06.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:06 smithi190 ceph-mon[160435]: pgmap v326: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2023-10-01T09:17:06.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:06 smithi138 ceph-mon[192513]: pgmap v326: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2023-10-01T09:17:06.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:06 smithi138 ceph-mon[185886]: pgmap v326: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2023-10-01T09:17:06.895 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:17:06.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:06.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:17:07.002 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:17:07.003 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:17:07.046 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:07.156 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:17:07.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:07.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:17:07.246 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:17:07.246 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "" 2023-10-01T09:17:07.246 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:17:07.267 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:07.370 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:07.972 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:17:08.182 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:08.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:08.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:08.270 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:08 smithi138 ceph-mon[192513]: pgmap v327: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-01T09:17:08.270 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:08 smithi138 ceph-mon[185886]: pgmap v327: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-01T09:17:08.290 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:08.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:08 smithi190 ceph-mon[160435]: pgmap v327: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-01T09:17:08.375 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:08.486 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:08.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:08.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:08.607 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:17:08.621 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:17:08.669 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:17:08.674 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:08.675 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:08.714 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:08.715 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:17:08.759 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:17:08.806 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:17:08.833 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:17:08.840 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:17:08.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:08.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:17:08.884 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:17:08.909 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:17:08.922 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:17:08.922 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:17:08.922 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:17:08.923 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:17:08.923 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:17:08.923 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:17:08.923 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:17:08.923 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:17:08.943 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:08.997 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:09.305 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:09.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:09 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1773316348' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:09.316 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:09 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1773316348' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:09.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:09.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:09.318 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:09 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1773316348' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:09.419 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:09.497 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:09.623 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:09.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:09.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:09.744 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:09.789 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:09.908 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:09.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:09.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:10.034 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:10.039 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:10 smithi190 ceph-mon[160435]: pgmap v328: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-10-01T09:17:10.039 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:10.040 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:10.063 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:10.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:10 smithi138 ceph-mon[192513]: pgmap v328: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-10-01T09:17:10.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:10.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:10.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:10 smithi138 ceph-mon[185886]: pgmap v328: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-10-01T09:17:10.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:10.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:10.798 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:10.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:10.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:10.903 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:17:10.965 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:11.052 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:17:11.085 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:17:11.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:11.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:17:11.194 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:11.223 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:17:11.226 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:11.267 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:11.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:11.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:11.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:17:11.372 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:17:11.436 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:11.494 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:17:11.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:11.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:17:11.601 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:17:11.654 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:11.944 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:17:11.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:11.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:17:12.065 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:17:12.142 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:12.288 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:12 smithi138 ceph-mon[192513]: pgmap v329: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-01T09:17:12.295 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:12 smithi138 ceph-mon[185886]: pgmap v329: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-01T09:17:12.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:12 smithi190 ceph-mon[160435]: pgmap v329: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-01T09:17:12.605 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:17:12.605 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:17:12.605 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:17:12.605 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:17:12.605 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:17:12.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:12.661 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:17:12.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:17:12.704 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:17:12.762 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:12.852 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:13.376 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:17:13.377 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:17:13.377 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T09:17:13.377 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.000000 2023-10-01T09:17:13.377 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 0 2023-10-01T09:17:13.639 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:13.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:13.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:13.757 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:17:13.790 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:17:13.811 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:13.929 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:17:13.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:13.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:17:13.966 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:14.045 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:17:14.096 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:14.222 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:17:14.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:14.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:17:14.349 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:14.403 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:14 smithi190 ceph-mon[160435]: pgmap v330: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-10-01T09:17:14.403 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1757309043' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:14.412 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:14.420 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:17:14.423 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:14.788 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:14 smithi138 ceph-mon[185886]: pgmap v330: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-10-01T09:17:14.788 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1757309043' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:14.788 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:14 smithi138 ceph-mon[192513]: pgmap v330: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-10-01T09:17:14.788 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1757309043' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:14.838 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:14.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:14.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:14.954 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:15.005 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:15.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:17:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:17:14] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-01T09:17:15.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:15 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:17:15.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:15 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:17:15.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:15 smithi190 ceph-mon[160435]: failed to encode map e134 with expected crc 2023-10-01T09:17:15.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:15 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1757309043' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:17:15.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:15 smithi190 ceph-mon[160435]: osdmap e134: 8 total, 8 up, 8 in 2023-10-01T09:17:15.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:15 smithi190 ceph-mon[160435]: failed to encode map e134 with expected crc 2023-10-01T09:17:15.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:15 smithi190 ceph-mon[160435]: failed to encode map e134 with expected crc 2023-10-01T09:17:15.817 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:15 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:17:15.817 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:15 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:17:15.817 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:15 smithi138 ceph-mon[192513]: failed to encode map e134 with expected crc 2023-10-01T09:17:15.817 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:15 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1757309043' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:17:15.818 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:15 smithi138 ceph-mon[192513]: osdmap e134: 8 total, 8 up, 8 in 2023-10-01T09:17:15.818 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:15 smithi138 ceph-mon[192513]: failed to encode map e134 with expected crc 2023-10-01T09:17:15.818 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:15 smithi138 ceph-mon[192513]: failed to encode map e134 with expected crc 2023-10-01T09:17:15.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:15 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:17:15.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:15 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:17:15.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:15 smithi138 ceph-mon[185886]: failed to encode map e134 with expected crc 2023-10-01T09:17:15.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:15 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1757309043' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:17:15.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:15 smithi138 ceph-mon[185886]: osdmap e134: 8 total, 8 up, 8 in 2023-10-01T09:17:15.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:15 smithi138 ceph-mon[185886]: failed to encode map e134 with expected crc 2023-10-01T09:17:15.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:15 smithi138 ceph-mon[185886]: failed to encode map e134 with expected crc 2023-10-01T09:17:16.625 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:16.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:16.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:16.736 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:16.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:16 smithi138 ceph-mon[185886]: pgmap v332: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-10-01T09:17:16.776 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:16 smithi138 ceph-mon[192513]: pgmap v332: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-10-01T09:17:16.783 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:16.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:16 smithi190 ceph-mon[160435]: pgmap v332: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-10-01T09:17:17.705 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:17.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:17.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:17.725 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:17.813 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:17:17.880 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:18.182 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:17:18.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:18.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:17:18.270 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:17:18.291 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:18.302 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:17:18.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:18 smithi190 ceph-mon[160435]: pgmap v333: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T09:17:18.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:18 smithi138 ceph-mon[192513]: pgmap v333: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T09:17:18.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:18 smithi138 ceph-mon[185886]: pgmap v333: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T09:17:18.325 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:18.347 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:17:18.380 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:17:18.399 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:17:18.408 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:17:18.416 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:18.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:18.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:18.438 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:17:18.486 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:17:18.516 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:17:18.525 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:17:18.560 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:17:18.560 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:17:18.560 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:17:18.560 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:17:18.560 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:17:18.561 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:17:18.561 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:17:18.561 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:17:18.563 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:18.669 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:17:18.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:18.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:17:18.794 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:18.873 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:19.016 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:19 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2042184419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:19.026 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2042184419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:19.027 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:19 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2042184419' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:19.088 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:19.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:19.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:19.196 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:17:19.242 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:20.068 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.6 primary-affinity to 0 (802) 2023-10-01T09:17:20.099 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:20 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2612030905' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-01T09:17:20.100 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:20 smithi138 ceph-mon[185886]: pgmap v334: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T09:17:20.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:20 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2612030905' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-01T09:17:20.100 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:20 smithi138 ceph-mon[192513]: pgmap v334: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T09:17:20.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:20 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2612030905' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-01T09:17:20.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:20 smithi190 ceph-mon[160435]: pgmap v334: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T09:17:20.817 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:17:20.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:20.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:17:20.922 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:17:20.949 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:21.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:21 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2612030905' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-01T09:17:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:21 smithi138 ceph-mon[185886]: osdmap e135: 8 total, 8 up, 8 in 2023-10-01T09:17:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:21 smithi138 ceph-mon[185886]: failed to encode map e135 with expected crc 2023-10-01T09:17:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:21 smithi138 ceph-mon[185886]: failed to encode map e135 with expected crc 2023-10-01T09:17:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:21 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:21 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:21.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:21 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2612030905' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-01T09:17:21.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:21 smithi138 ceph-mon[192513]: osdmap e135: 8 total, 8 up, 8 in 2023-10-01T09:17:21.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:21 smithi138 ceph-mon[192513]: failed to encode map e135 with expected crc 2023-10-01T09:17:21.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:21 smithi138 ceph-mon[192513]: failed to encode map e135 with expected crc 2023-10-01T09:17:21.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:21 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:21.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:21 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:21.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:21 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2612030905' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-01T09:17:21.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:21 smithi190 ceph-mon[160435]: osdmap e135: 8 total, 8 up, 8 in 2023-10-01T09:17:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:21 smithi190 ceph-mon[160435]: failed to encode map e135 with expected crc 2023-10-01T09:17:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:21 smithi190 ceph-mon[160435]: failed to encode map e135 with expected crc 2023-10-01T09:17:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:21 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:21 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:21.621 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:17:21.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:21.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:17:21.661 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:21.731 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:21.780 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:22.061 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[192513]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.061 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[192513]: osdmap e136: 8 total, 8 up, 8 in 2023-10-01T09:17:22.061 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[192513]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.061 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[192513]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.062 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[192513]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.062 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[192513]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.062 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:17:22.062 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:17:22.062 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:22.062 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[192513]: pgmap v337: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-01T09:17:22.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[185886]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[185886]: osdmap e136: 8 total, 8 up, 8 in 2023-10-01T09:17:22.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[185886]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[185886]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[185886]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[185886]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.063 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:17:22.064 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:17:22.064 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:22.064 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:22 smithi138 ceph-mon[185886]: pgmap v337: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-01T09:17:22.243 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:17:22.243 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:17:22.244 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:22.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:22.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:22.363 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:22.436 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:22.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:22 smithi190 ceph-mon[160435]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:22 smithi190 ceph-mon[160435]: osdmap e136: 8 total, 8 up, 8 in 2023-10-01T09:17:22.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:22 smithi190 ceph-mon[160435]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:22 smithi190 ceph-mon[160435]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:22 smithi190 ceph-mon[160435]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:22 smithi190 ceph-mon[160435]: failed to encode map e136 with expected crc 2023-10-01T09:17:22.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:17:22.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:17:22.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:22.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:22 smithi190 ceph-mon[160435]: pgmap v337: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-01T09:17:23.153 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:17:23.478 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:17:23.534 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:23.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:23.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:23.640 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:17:23.717 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:23.971 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:17:23.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:23.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:17:24.075 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:17:24.130 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:24.249 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:17:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:24.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:17:24.361 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:17:24.442 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:24.515 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:24.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:24 smithi138 ceph-mon[192513]: pgmap v338: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-10-01T09:17:24.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:24 smithi138 ceph-mon[185886]: pgmap v338: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-10-01T09:17:24.658 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:24.666 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:17:24.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:24.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:17:24.772 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:17:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:24 smithi190 ceph-mon[160435]: pgmap v338: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-10-01T09:17:24.818 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:24.925 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:17:24.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:24.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:17:25.034 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:17:25.086 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:25.171 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:17:25.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:25.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:17:25.197 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:17:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:17:24] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-01T09:17:25.284 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:17:25.364 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:26.069 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:17:26.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:26.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:17:26.172 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:17:26.229 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:26.293 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:17:26.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:26.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:17:26.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:26 smithi138 ceph-mon[192513]: pgmap v339: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 422 op/s 2023-10-01T09:17:26.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:26 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/572043155' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:17:26.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:26 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:17:26.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:26.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:17:26.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:26 smithi138 ceph-mon[185886]: pgmap v339: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 422 op/s 2023-10-01T09:17:26.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:26 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/572043155' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:17:26.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:26 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:17:26.418 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:17:26.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:26.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:17:26.496 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:17:26.496 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:17:26.496 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-01T09:17:26.496 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:17:26.496 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:17:26.496 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-01T09:17:26.529 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:17:26.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:26.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:17:26.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:26 smithi190 ceph-mon[160435]: pgmap v339: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 422 op/s 2023-10-01T09:17:26.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:26 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/572043155' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:17:26.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:26 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:17:26.620 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:27.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:27 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:17:27.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:27 smithi190 ceph-mon[160435]: failed to encode map e137 with expected crc 2023-10-01T09:17:27.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:27 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:17:27.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:27 smithi190 ceph-mon[160435]: osdmap e137: 8 total, 8 up, 8 in 2023-10-01T09:17:27.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:27 smithi190 ceph-mon[160435]: failed to encode map e137 with expected crc 2023-10-01T09:17:27.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:27 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:17:27.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:27 smithi138 ceph-mon[192513]: failed to encode map e137 with expected crc 2023-10-01T09:17:27.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:27 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:17:27.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:27 smithi138 ceph-mon[192513]: osdmap e137: 8 total, 8 up, 8 in 2023-10-01T09:17:27.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:27 smithi138 ceph-mon[192513]: failed to encode map e137 with expected crc 2023-10-01T09:17:27.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:27 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:17:27.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:27 smithi138 ceph-mon[185886]: failed to encode map e137 with expected crc 2023-10-01T09:17:27.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:27 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:17:27.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:27 smithi138 ceph-mon[185886]: osdmap e137: 8 total, 8 up, 8 in 2023-10-01T09:17:27.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:27 smithi138 ceph-mon[185886]: failed to encode map e137 with expected crc 2023-10-01T09:17:27.755 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:17:27.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:27.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:17:27.867 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:27.932 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:28.181 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:28.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:28.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:28.273 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:28.289 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:28.295 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:28 smithi138 ceph-mon[185886]: pgmap v341: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-10-01T09:17:28.390 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:28.411 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:17:28.458 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:17:28.460 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:28.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:28.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:28.501 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:17:28.539 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:17:28.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:28 smithi190 ceph-mon[160435]: pgmap v341: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-10-01T09:17:28.564 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:17:28.567 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:17:28.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:28 smithi138 ceph-mon[192513]: pgmap v341: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-10-01T09:17:28.600 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:17:28.629 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:17:28.630 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:28.680 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:17:28.693 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:17:28.693 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:17:28.693 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:17:28.693 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:17:28.693 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:17:28.693 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:17:28.693 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:17:28.693 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:17:28.726 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:17:28.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:28.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:17:28.778 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:17:28.778 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:17:28.778 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.778 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:168159 5.0 e137/106 5:04a9f348:::benchmark_data_smithi138_240338_object53022:head v 137'10339, mlcod=137'10336)", 2023-10-01T09:17:28.778 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:17:28.761260+0000", 2023-10-01T09:17:28.778 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0063875959999999997, 2023-10-01T09:17:28.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064132069999999998, 2023-10-01T09:17:28.779 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:17:28.779 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:17:28.779 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:17:28.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.779 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:17:28.779 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.761260+0000", 2023-10-01T09:17:28.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:17:28.779 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.779 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:17:28.779 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.761260+0000", 2023-10-01T09:17:28.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:17:28.780 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.780 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:17:28.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.761259+0000", 2023-10-01T09:17:28.780 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:17:28.780 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.780 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:17:28.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.761265+0000", 2023-10-01T09:17:28.780 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.336e-06 2023-10-01T09:17:28.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.781 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:17:28.781 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.761282+0000", 2023-10-01T09:17:28.781 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6518999999999999e-05 2023-10-01T09:17:28.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.781 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:17:28.781 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.761290+0000", 2023-10-01T09:17:28.781 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3999999999999992e-06 2023-10-01T09:17:28.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.782 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:17:28.782 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.761337+0000", 2023-10-01T09:17:28.782 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6313999999999997e-05 2023-10-01T09:17:28.782 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.782 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.782 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:17:28.782 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.761357+0000", 2023-10-01T09:17:28.782 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0818e-05 2023-10-01T09:17:28.782 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:17:28.783 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:17:28.783 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:17:28.783 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.783 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:168164 5.8 e137/125 5:1f9f91aa:::benchmark_data_smithi138_240338_object53027:head v 137'10771, mlcod=137'10769)", 2023-10-01T09:17:28.783 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:17:28.765988+0000", 2023-10-01T09:17:28.783 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.001659521, 2023-10-01T09:17:28.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017259230000000001, 2023-10-01T09:17:28.783 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:17:28.783 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:17:28.783 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:17:28.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:17:28.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.765988+0000", 2023-10-01T09:17:28.784 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:17:28.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:17:28.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.765988+0000", 2023-10-01T09:17:28.784 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:17:28.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:17:28.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.765987+0000", 2023-10-01T09:17:28.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:17:28.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.785 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:17:28.785 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.765994+0000", 2023-10-01T09:17:28.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1749999999999999e-06 2023-10-01T09:17:28.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.785 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:17:28.785 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.766008+0000", 2023-10-01T09:17:28.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3755e-05 2023-10-01T09:17:28.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.786 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:17:28.786 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.766016+0000", 2023-10-01T09:17:28.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1650000000000006e-06 2023-10-01T09:17:28.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.786 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:17:28.786 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.766976+0000", 2023-10-01T09:17:28.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00095953599999999998 2023-10-01T09:17:28.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.787 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:17:28.787 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.766998+0000", 2023-10-01T09:17:28.787 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2724000000000001e-05 2023-10-01T09:17:28.787 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:17:28.787 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:17:28.787 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:17:28.787 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.787 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.787 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:168165 5.c 5:326f1a92:::benchmark_data_smithi138_240338_object53028:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-01T09:17:28.787 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:17:28.767045+0000", 2023-10-01T09:17:28.787 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.00060215699999999997, 2023-10-01T09:17:28.787 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00071186999999999997, 2023-10-01T09:17:28.787 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:17:28.788 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:17:28.788 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:17:28.788 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:17:28.788 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:17:28.788 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 168165 2023-10-01T09:17:28.788 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.788 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:17:28.788 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.788 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:17:28.788 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.767045+0000", 2023-10-01T09:17:28.788 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:17:28.788 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.789 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.789 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:17:28.789 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.767045+0000", 2023-10-01T09:17:28.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:17:28.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.789 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.789 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:17:28.789 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.767044+0000", 2023-10-01T09:17:28.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:17:28.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.789 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.789 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:17:28.789 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.767052+0000", 2023-10-01T09:17:28.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1219999999999995e-06 2023-10-01T09:17:28.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:17:28.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.767064+0000", 2023-10-01T09:17:28.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2686e-05 2023-10-01T09:17:28.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:17:28.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.767073+0000", 2023-10-01T09:17:28.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9590000000000001e-06 2023-10-01T09:17:28.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:17:28.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.767125+0000", 2023-10-01T09:17:28.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1727999999999997e-05 2023-10-01T09:17:28.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:17:28.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.767276+0000", 2023-10-01T09:17:28.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015056999999999999 2023-10-01T09:17:28.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:28.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:28.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:17:28.792 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:28.767404+0000", 2023-10-01T09:17:28.792 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000128357 2023-10-01T09:17:28.792 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:17:28.792 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:17:28.792 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:17:28.792 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:17:28.792 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:17:28.792 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 3 2023-10-01T09:17:28.792 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:17:28.840 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:17:28.905 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:28.999 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:17:29.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:29.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:17:29.110 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:29.159 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:29.296 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:29 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2573279070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:29.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:29 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2573279070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:29 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2573279070' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:29.794 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:17:30.280 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:17:30.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:30 smithi138 ceph-mon[185886]: pgmap v342: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2023-10-01T09:17:30.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:30 smithi138 ceph-mon[192513]: pgmap v342: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2023-10-01T09:17:30.596 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:30.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:30.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:30.705 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:30.773 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:30.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:30 smithi190 ceph-mon[160435]: pgmap v342: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2023-10-01T09:17:31.014 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:31.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:31.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:31.120 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:31.198 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:31.245 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:31.317 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:31.331 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:31.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:31.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:31.440 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:31.494 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:31.885 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:17:31.885 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T09:17:31.885 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:17:31.924 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:31.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:31.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:32.030 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:32.081 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:32.372 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:17:32.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:32 smithi190 ceph-mon[160435]: pgmap v343: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-01T09:17:32.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:32 smithi138 ceph-mon[192513]: pgmap v343: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-01T09:17:32.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:32 smithi138 ceph-mon[185886]: pgmap v343: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-01T09:17:33.732 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:33.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:33.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:33.838 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:17:33.889 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:34.105 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:17:34.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:34.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:17:34.215 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:34.242 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:34.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:34 smithi190 ceph-mon[160435]: pgmap v344: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2023-10-01T09:17:34.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:34 smithi138 ceph-mon[192513]: pgmap v344: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2023-10-01T09:17:34.807 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:34 smithi138 ceph-mon[185886]: pgmap v344: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2023-10-01T09:17:34.824 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:34.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:34.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:34.935 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:34.991 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:35.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:17:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:17:34] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-01T09:17:35.208 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:35.448 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:35.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:35.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:35.473 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:17:35.473 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:17:35.473 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:17:35.473 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:17:35.473 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:17:35.559 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:17:35.660 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:35.934 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:17:35.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:35.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:17:36.040 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:17:36.096 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:36.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:36 smithi138 ceph-mon[185886]: pgmap v345: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2023-10-01T09:17:36.456 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:36 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/55094210' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:36.456 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:36 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:36.558 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:17:36.598 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:17:36.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:36.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:17:36.701 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:36.735 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:36 smithi138 ceph-mon[192513]: pgmap v345: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2023-10-01T09:17:36.735 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:36 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/55094210' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:36.735 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:36 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:36.738 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:36.810 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:36.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:36 smithi190 ceph-mon[160435]: pgmap v345: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2023-10-01T09:17:36.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:36 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/55094210' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:36.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:36 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:36.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:36.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:36.920 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:17:36.958 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:37.000 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:17:37.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:37.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:17:37.105 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:37.138 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:37.262 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:37.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:37.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:37.378 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:37.446 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:37.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:37 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:17:37.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:37 smithi190 ceph-mon[160435]: failed to encode map e138 with expected crc 2023-10-01T09:17:37.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:37 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:17:37.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:37 smithi190 ceph-mon[160435]: osdmap e138: 8 total, 8 up, 8 in 2023-10-01T09:17:37.613 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:17:37.613 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:37.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:37.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:37.723 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:37.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:37 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:17:37.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:37 smithi138 ceph-mon[185886]: failed to encode map e138 with expected crc 2023-10-01T09:17:37.744 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:37 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:17:37.744 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:37 smithi138 ceph-mon[185886]: osdmap e138: 8 total, 8 up, 8 in 2023-10-01T09:17:37.744 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:37 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:17:37.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:37 smithi138 ceph-mon[192513]: failed to encode map e138 with expected crc 2023-10-01T09:17:37.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:37 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:17:37.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:37 smithi138 ceph-mon[192513]: osdmap e138: 8 total, 8 up, 8 in 2023-10-01T09:17:37.783 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:37.851 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:37.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:37.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:37.955 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:17:38.032 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:38.550 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:17:38.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:38.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:17:38.656 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:17:38.720 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:38.752 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:38.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:38 smithi190 ceph-mon[160435]: pgmap v347: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-10-01T09:17:38.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:38 smithi138 ceph-mon[192513]: pgmap v347: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-10-01T09:17:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:38 smithi138 ceph-mon[185886]: pgmap v347: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-10-01T09:17:38.820 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:17:38.826 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:17:38.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:38.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:17:38.851 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:17:38.891 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:17:38.920 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:17:38.943 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:17:38.949 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:17:38.989 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:17:39.023 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:39.025 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:17:39.062 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:17:39.081 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:17:39.082 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:17:39.082 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:17:39.082 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:17:39.082 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:17:39.082 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:17:39.082 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:17:39.082 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:17:39.105 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:17:39.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:39.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:17:39.214 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:39.293 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:39.401 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:39.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:39.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:39.488 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:39 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1735114991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:39.515 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:39.554 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:39.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:39 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1735114991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:39.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:39 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1735114991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:39.843 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:17:39.843 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:17:39.843 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-01T09:17:39.843 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:17:39.844 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:17:39.844 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-01T09:17:40.501 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:17:40.562 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:17:40.573 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:40.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:40.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:40.677 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:17:40.715 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:40.749 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:40 smithi190 ceph-mon[160435]: pgmap v348: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-01T09:17:40.749 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:40.749 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:17:40.758 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:40 smithi138 ceph-mon[192513]: pgmap v348: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-01T09:17:40.758 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:40.758 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:17:40.759 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:40 smithi138 ceph-mon[185886]: pgmap v348: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-01T09:17:40.759 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:40.759 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:17:40.844 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:17:40.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:40.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:17:40.949 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:41.023 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:41.121 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:41.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:41.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:41.239 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:41.293 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:41.304 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:41.474 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:41.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:41.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:41.581 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:17:41.659 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:41.755 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:17:41.755 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:17:41.806 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:17:41.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:41.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:17:41.914 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:17:41.975 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:42.634 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:17:42.635 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:42 smithi138 ceph-mon[192513]: pgmap v349: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-10-01T09:17:42.635 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:42 smithi138 ceph-mon[185886]: pgmap v349: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-10-01T09:17:42.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:42.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:17:42.737 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:42.780 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:17:42.783 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:42.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:42 smithi190 ceph-mon[160435]: pgmap v349: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2023-10-01T09:17:43.513 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:17:43.559 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:43.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:43.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:43.674 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:17:43.720 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:43.725 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:43.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:43 smithi190 ceph-mon[160435]: pgmap v350: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-01T09:17:43.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:43 smithi138 ceph-mon[192513]: pgmap v350: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-01T09:17:43.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:43 smithi138 ceph-mon[185886]: pgmap v350: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-01T09:17:43.897 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:17:43.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:43.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:17:44.011 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:17:44.069 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:44.238 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:17:44.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:44.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:17:44.347 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:17:44.432 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:44.517 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:44 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2716495513' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:17:44.517 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:44 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:17:44.518 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:44 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2716495513' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:17:44.518 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:44 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:17:44.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:44 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2716495513' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:17:44.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:44 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:17:44.530 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:17:44.530 INFO:teuthology.orchestra.run.smithi190.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T09:17:44.530 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:17:44.545 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:17:44.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:44.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:17:44.566 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:17:44.658 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:17:44.760 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:44.868 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:17:44.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:44.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:17:44.980 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:45.015 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:45.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:17:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:17:44] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.33.4" 2023-10-01T09:17:45.482 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:45.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:45.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:45.592 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:17:45.625 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:45.632 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:45 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:17:45.632 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:45 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:17:45.632 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:45 smithi190 ceph-mon[160435]: osdmap e139: 8 total, 8 up, 8 in 2023-10-01T09:17:45.632 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:45 smithi190 ceph-mon[160435]: failed to encode map e139 with expected crc 2023-10-01T09:17:45.632 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:45 smithi190 ceph-mon[160435]: failed to encode map e139 with expected crc 2023-10-01T09:17:45.632 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:45 smithi190 ceph-mon[160435]: failed to encode map e139 with expected crc 2023-10-01T09:17:45.632 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:45 smithi190 ceph-mon[160435]: failed to encode map e139 with expected crc 2023-10-01T09:17:45.632 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:45 smithi190 ceph-mon[160435]: pgmap v352: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2023-10-01T09:17:45.767 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:17:45.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:45.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:17:45.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:17:45.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:17:45.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[192513]: osdmap e139: 8 total, 8 up, 8 in 2023-10-01T09:17:45.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[192513]: failed to encode map e139 with expected crc 2023-10-01T09:17:45.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[192513]: failed to encode map e139 with expected crc 2023-10-01T09:17:45.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[192513]: failed to encode map e139 with expected crc 2023-10-01T09:17:45.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[192513]: failed to encode map e139 with expected crc 2023-10-01T09:17:45.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[192513]: pgmap v352: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2023-10-01T09:17:45.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:17:45.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:17:45.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[185886]: osdmap e139: 8 total, 8 up, 8 in 2023-10-01T09:17:45.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[185886]: failed to encode map e139 with expected crc 2023-10-01T09:17:45.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[185886]: failed to encode map e139 with expected crc 2023-10-01T09:17:45.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[185886]: failed to encode map e139 with expected crc 2023-10-01T09:17:45.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[185886]: failed to encode map e139 with expected crc 2023-10-01T09:17:45.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:45 smithi138 ceph-mon[185886]: pgmap v352: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 556 op/s 2023-10-01T09:17:45.885 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:17:45.961 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:46.054 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:17:46.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:46.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:17:46.089 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:46.161 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:46.214 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:46.363 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:46.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:46.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:46.474 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:17:46.547 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:46.644 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:17:46.644 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:17:46.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:46.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:17:46.676 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:17:46.717 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:17:46.751 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:17:46.762 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:17:46.787 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:17:46.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:46 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/403906099' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:46.823 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:17:46.827 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:46.864 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:17:46.906 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:17:46.912 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:17:46.913 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:17:46.913 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:17:46.913 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:17:46.913 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:17:46.913 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:17:46.913 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:17:46.913 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:17:46.934 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:46 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/403906099' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:46.935 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:46 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/403906099' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:46.936 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:17:46.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:46.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:17:47.051 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:17:47.137 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:47.255 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:17:47.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:47.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:17:47.367 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:47.401 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:47.442 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:47.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:47.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:47.549 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:17:47.574 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:47.615 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:17:47.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:47.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:17:47.722 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:17:47.774 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:48.019 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:48 smithi138 ceph-mon[185886]: pgmap v353: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-01T09:17:48.026 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:48 smithi138 ceph-mon[192513]: pgmap v353: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-01T09:17:48.124 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:17:48.253 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:17:48.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:48.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:17:48.317 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:48 smithi190 ceph-mon[160435]: pgmap v353: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-01T09:17:48.358 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:17:48.421 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:48.524 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:17:48.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:48.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:17:48.633 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:17:48.686 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:48.770 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:17:48.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:48.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:17:48.881 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:48.946 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:49.040 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:49.064 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:49.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:49.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:49.180 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:49.187 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:17:49.187 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:17:49.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.187 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:182311 5.8 e139/125)", 2023-10-01T09:17:49.187 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:17:49.180311+0000", 2023-10-01T09:17:49.187 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0024304000000000001, 2023-10-01T09:17:49.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024573450000000001, 2023-10-01T09:17:49.187 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:17:49.187 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:17:49.187 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:17:49.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:17:49.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:49.180311+0000", 2023-10-01T09:17:49.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:17:49.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:49.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:17:49.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:49.180311+0000", 2023-10-01T09:17:49.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:17:49.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:49.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:17:49.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:49.180310+0000", 2023-10-01T09:17:49.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:17:49.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:49.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.189 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:17:49.189 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:49.180318+0000", 2023-10-01T09:17:49.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1799999999999996e-06 2023-10-01T09:17:49.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:49.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.189 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:17:49.189 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:49.180336+0000", 2023-10-01T09:17:49.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7668000000000001e-05 2023-10-01T09:17:49.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:49.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.190 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:17:49.190 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:49.180349+0000", 2023-10-01T09:17:49.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2690999999999999e-05 2023-10-01T09:17:49.190 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:17:49.190 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:17:49.190 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:17:49.190 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:49.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.190 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.34683.0:182321 5.b e139/125 5:d893d0df:::benchmark_data_smithi138_240338_object67184:head v 139'11448, mlcod=139'11446)", 2023-10-01T09:17:49.190 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:17:49.180362+0000", 2023-10-01T09:17:49.190 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0023792349999999999, 2023-10-01T09:17:49.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002460524, 2023-10-01T09:17:49.191 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:17:49.191 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:17:49.191 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:17:49.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:17:49.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:49.180362+0000", 2023-10-01T09:17:49.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:17:49.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:49.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:17:49.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:49.180362+0000", 2023-10-01T09:17:49.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:17:49.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:49.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:17:49.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:49.180362+0000", 2023-10-01T09:17:49.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:17:49.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:49.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:17:49.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:49.180366+0000", 2023-10-01T09:17:49.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6820000000000001e-06 2023-10-01T09:17:49.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:49.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:17:49.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:49.180371+0000", 2023-10-01T09:17:49.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4439999999999999e-06 2023-10-01T09:17:49.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:49.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:17:49.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:49.180375+0000", 2023-10-01T09:17:49.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2259999999999999e-06 2023-10-01T09:17:49.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:49.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:17:49.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:49.182261+0000", 2023-10-01T09:17:49.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00188557 2023-10-01T09:17:49.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:17:49.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:17:49.194 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:17:49.194 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:17:49.182286+0000", 2023-10-01T09:17:49.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4647999999999998e-05 2023-10-01T09:17:49.194 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:17:49.194 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:17:49.194 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:17:49.194 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:17:49.194 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:17:49.194 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 2 2023-10-01T09:17:49.194 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:17:49.244 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:49.792 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:49.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:49.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:49.899 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:17:49.958 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:50.048 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:17:50.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:50.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:17:50.159 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:50.224 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:50.326 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:50.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:50.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:50.385 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:17:50.385 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:17:50.385 INFO:tasks.thrashosds.thrasher:Reweighting osd 3 to 0.5093433429873988 2023-10-01T09:17:50.385 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 0.5093433429873988 2023-10-01T09:17:50.448 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:50.461 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:17:50.560 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:50.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:50 smithi190 ceph-mon[160435]: pgmap v354: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-01T09:17:50.562 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:17:50.615 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:50 smithi138 ceph-mon[185886]: pgmap v354: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-01T09:17:50.615 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:50 smithi138 ceph-mon[192513]: pgmap v354: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-10-01T09:17:50.914 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:50.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:50.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:51.021 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:51.049 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:51.529 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:51.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:51.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:51.637 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:51.746 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:51.833 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:51.878 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:51.967 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:51.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:51.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:51.984 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:52.072 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:17:52.122 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:52.477 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:17:52.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:52.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:17:52.583 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:17:52.631 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:52.652 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:52 smithi138 ceph-mon[192513]: pgmap v355: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T09:17:52.652 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:52 smithi138 ceph-mon[185886]: pgmap v355: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T09:17:52.715 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:17:52.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:52.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:17:52.726 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:52 smithi190 ceph-mon[160435]: pgmap v355: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T09:17:52.827 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:17:52.909 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:53.002 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:17:53.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:53.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:17:53.114 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:53.177 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:53.316 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:53.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:53.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:53.427 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:17:53.460 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:17:53.495 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:53.598 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:17:53.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:53.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:17:53.617 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1615519769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:53.618 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:53 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:53.712 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:53.777 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1615519769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:53.777 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:53 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:53.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1615519769' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:53.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:53 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:17:53.791 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:54.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:54 smithi190 ceph-mon[160435]: pgmap v356: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-01T09:17:54.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:54 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:17:54.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:54 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:17:54.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:54 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:17:54.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:54 smithi190 ceph-mon[160435]: osdmap e140: 8 total, 8 up, 8 in 2023-10-01T09:17:54.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:54 smithi190 ceph-mon[160435]: failed to encode map e140 with expected crc 2023-10-01T09:17:54.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:54 smithi190 ceph-mon[160435]: failed to encode map e140 with expected crc 2023-10-01T09:17:54.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:54 smithi190 ceph-mon[160435]: failed to encode map e140 with expected crc 2023-10-01T09:17:54.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:54 smithi190 ceph-mon[160435]: failed to encode map e140 with expected crc 2023-10-01T09:17:54.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:54 smithi190 ceph-mon[160435]: failed to encode map e140 with expected crc 2023-10-01T09:17:54.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[185886]: pgmap v356: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-01T09:17:54.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:17:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:17:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:17:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[185886]: osdmap e140: 8 total, 8 up, 8 in 2023-10-01T09:17:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[185886]: failed to encode map e140 with expected crc 2023-10-01T09:17:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[185886]: failed to encode map e140 with expected crc 2023-10-01T09:17:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[185886]: failed to encode map e140 with expected crc 2023-10-01T09:17:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[185886]: failed to encode map e140 with expected crc 2023-10-01T09:17:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[185886]: failed to encode map e140 with expected crc 2023-10-01T09:17:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[192513]: pgmap v356: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-01T09:17:54.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:17:54.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:17:54.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:17:54.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[192513]: osdmap e140: 8 total, 8 up, 8 in 2023-10-01T09:17:54.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[192513]: failed to encode map e140 with expected crc 2023-10-01T09:17:54.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[192513]: failed to encode map e140 with expected crc 2023-10-01T09:17:54.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[192513]: failed to encode map e140 with expected crc 2023-10-01T09:17:54.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[192513]: failed to encode map e140 with expected crc 2023-10-01T09:17:54.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:54 smithi138 ceph-mon[192513]: failed to encode map e140 with expected crc 2023-10-01T09:17:55.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:17:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:17:54] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.33.4" 2023-10-01T09:17:55.888 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:55.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:55.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:55.999 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:56.067 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:56.225 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:56.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:56.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:56.334 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:17:56.377 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:56.464 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:17:56.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:56.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:17:56.483 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:56.483 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:17:56.483 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:56 smithi138 ceph-mon[185886]: pgmap v358: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-01T09:17:56.484 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:56.484 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:17:56.484 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:56 smithi138 ceph-mon[192513]: pgmap v358: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-01T09:17:56.530 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:17:56.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:17:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:17:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:56 smithi190 ceph-mon[160435]: pgmap v358: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-10-01T09:17:56.574 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:17:56.577 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:56.627 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:17:56.656 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:17:56.661 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:56.698 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:17:56.739 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:17:56.779 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:17:56.825 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:17:56.848 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:17:56.849 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:17:56.850 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:17:56.851 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:17:56.851 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:17:56.851 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:17:56.851 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:17:56.851 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:17:56.851 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:17:56.851 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:17:56.902 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:56.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:56.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:57.012 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:57.085 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:57.278 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3947824025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:57.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3947824025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:57.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3947824025' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:17:57.788 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:17:58.321 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.3 to 0.509343 (8264) 2023-10-01T09:17:58.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:58 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1502664444' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.5093433429873988}]: dispatch 2023-10-01T09:17:58.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:58 smithi190 ceph-mon[160435]: pgmap v359: 89 pgs: 89 active+clean; 2.6 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-10-01T09:17:58.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:58 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1502664444' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.5093433429873988}]: dispatch 2023-10-01T09:17:58.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:58 smithi138 ceph-mon[192513]: pgmap v359: 89 pgs: 89 active+clean; 2.6 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-10-01T09:17:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:58 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1502664444' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.5093433429873988}]: dispatch 2023-10-01T09:17:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:58 smithi138 ceph-mon[185886]: pgmap v359: 89 pgs: 89 active+clean; 2.6 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-10-01T09:17:58.594 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:58.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:58.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:17:58.698 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:58.724 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:58.993 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:17:59.068 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:59.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:59.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:59.171 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:17:59.197 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:59.240 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:17:59.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:59.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:17:59.344 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:17:59.371 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:59.522 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:17:59.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1502664444' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.5093433429873988}]': finished 2023-10-01T09:17:59.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:59 smithi138 ceph-mon[192513]: osdmap e141: 8 total, 8 up, 8 in 2023-10-01T09:17:59.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:59 smithi138 ceph-mon[192513]: failed to encode map e141 with expected crc 2023-10-01T09:17:59.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:17:59 smithi138 ceph-mon[192513]: failed to encode map e141 with expected crc 2023-10-01T09:17:59.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:59.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:17:59.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1502664444' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.5093433429873988}]': finished 2023-10-01T09:17:59.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:59 smithi138 ceph-mon[185886]: osdmap e141: 8 total, 8 up, 8 in 2023-10-01T09:17:59.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:59 smithi138 ceph-mon[185886]: failed to encode map e141 with expected crc 2023-10-01T09:17:59.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:17:59 smithi138 ceph-mon[185886]: failed to encode map e141 with expected crc 2023-10-01T09:17:59.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1502664444' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.5093433429873988}]': finished 2023-10-01T09:17:59.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:59 smithi190 ceph-mon[160435]: osdmap e141: 8 total, 8 up, 8 in 2023-10-01T09:17:59.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:59 smithi190 ceph-mon[160435]: failed to encode map e141 with expected crc 2023-10-01T09:17:59.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:17:59 smithi190 ceph-mon[160435]: failed to encode map e141 with expected crc 2023-10-01T09:17:59.626 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:17:59.651 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:59.704 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:17:59.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:59.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:17:59.722 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:59.783 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:17:59.813 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:17:59.839 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:17:59.946 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:17:59.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:17:59.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:00.050 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:00.081 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:00.129 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:00.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:00.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:00.235 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:00.263 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:00.307 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:00.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:00.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:00.413 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:00.439 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[192513]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.439 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[192513]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.439 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[192513]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.439 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[192513]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.439 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[192513]: osdmap e142: 8 total, 8 up, 8 in 2023-10-01T09:18:00.439 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[192513]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.440 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[192513]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.440 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[192513]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.440 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[192513]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.440 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[192513]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.440 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[192513]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.440 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[192513]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.440 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[192513]: pgmap v362: 89 pgs: 89 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-01T09:18:00.440 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[185886]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.440 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[185886]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.440 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[185886]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.440 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[185886]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[185886]: osdmap e142: 8 total, 8 up, 8 in 2023-10-01T09:18:00.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[185886]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[185886]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[185886]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[185886]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[185886]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[185886]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[185886]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:00 smithi138 ceph-mon[185886]: pgmap v362: 89 pgs: 89 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-01T09:18:00.442 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:00.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:00 smithi190 ceph-mon[160435]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:00 smithi190 ceph-mon[160435]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:00 smithi190 ceph-mon[160435]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:00 smithi190 ceph-mon[160435]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:00 smithi190 ceph-mon[160435]: osdmap e142: 8 total, 8 up, 8 in 2023-10-01T09:18:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:00 smithi190 ceph-mon[160435]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:00 smithi190 ceph-mon[160435]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:00 smithi190 ceph-mon[160435]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:00 smithi190 ceph-mon[160435]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:00 smithi190 ceph-mon[160435]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:00 smithi190 ceph-mon[160435]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:00 smithi190 ceph-mon[160435]: failed to encode map e142 with expected crc 2023-10-01T09:18:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:00 smithi190 ceph-mon[160435]: pgmap v362: 89 pgs: 89 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-01T09:18:00.643 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:18:01.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:01 smithi138 ceph-mon[185886]: failed to encode map e143 with expected crc 2023-10-01T09:18:01.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:01 smithi138 ceph-mon[185886]: 5.0 starting backfill to osd.0 from (0'0,0'0] MAX to 140'11639 2023-10-01T09:18:01.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:01 smithi138 ceph-mon[185886]: failed to encode map e143 with expected crc 2023-10-01T09:18:01.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:01 smithi138 ceph-mon[185886]: osdmap e143: 8 total, 8 up, 8 in 2023-10-01T09:18:01.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:01 smithi138 ceph-mon[185886]: failed to encode map e143 with expected crc 2023-10-01T09:18:01.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:01 smithi138 ceph-mon[185886]: 5.a starting backfill to osd.0 from (0'0,0'0] MAX to 140'11644 2023-10-01T09:18:01.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:01 smithi138 ceph-mon[185886]: 5.1 starting backfill to osd.0 from (0'0,0'0] MAX to 140'11995 2023-10-01T09:18:01.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:01 smithi138 ceph-mon[192513]: failed to encode map e143 with expected crc 2023-10-01T09:18:01.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:01 smithi138 ceph-mon[192513]: 5.0 starting backfill to osd.0 from (0'0,0'0] MAX to 140'11639 2023-10-01T09:18:01.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:01 smithi138 ceph-mon[192513]: failed to encode map e143 with expected crc 2023-10-01T09:18:01.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:01 smithi138 ceph-mon[192513]: osdmap e143: 8 total, 8 up, 8 in 2023-10-01T09:18:01.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:01 smithi138 ceph-mon[192513]: failed to encode map e143 with expected crc 2023-10-01T09:18:01.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:01 smithi138 ceph-mon[192513]: 5.a starting backfill to osd.0 from (0'0,0'0] MAX to 140'11644 2023-10-01T09:18:01.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:01 smithi138 ceph-mon[192513]: 5.1 starting backfill to osd.0 from (0'0,0'0] MAX to 140'11995 2023-10-01T09:18:01.629 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:01.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:01.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:01.741 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:01.789 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:01 smithi190 ceph-mon[160435]: failed to encode map e143 with expected crc 2023-10-01T09:18:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:01 smithi190 ceph-mon[160435]: 5.0 starting backfill to osd.0 from (0'0,0'0] MAX to 140'11639 2023-10-01T09:18:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:01 smithi190 ceph-mon[160435]: failed to encode map e143 with expected crc 2023-10-01T09:18:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:01 smithi190 ceph-mon[160435]: osdmap e143: 8 total, 8 up, 8 in 2023-10-01T09:18:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:01 smithi190 ceph-mon[160435]: failed to encode map e143 with expected crc 2023-10-01T09:18:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:01 smithi190 ceph-mon[160435]: 5.a starting backfill to osd.0 from (0'0,0'0] MAX to 140'11644 2023-10-01T09:18:01.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:01 smithi190 ceph-mon[160435]: 5.1 starting backfill to osd.0 from (0'0,0'0] MAX to 140'11995 2023-10-01T09:18:02.006 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:02.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:02.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:02.111 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:02.154 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:02.291 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:02.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:02.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:02.401 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:02.453 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:02 smithi190 ceph-mon[160435]: pgmap v364: 89 pgs: 89 active+clean; 2.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T09:18:02.460 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:02.508 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:02.546 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:18:02.546 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:18:02.575 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:02.580 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:02 smithi138 ceph-mon[185886]: pgmap v364: 89 pgs: 89 active+clean; 2.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T09:18:02.580 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:02 smithi138 ceph-mon[192513]: pgmap v364: 89 pgs: 89 active+clean; 2.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T09:18:02.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:02.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:02.683 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:02.757 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:02.889 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:02.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:02.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:03.000 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:03.042 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:03.118 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:03.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:03.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:03.228 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:03.307 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:03.370 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:18:03.438 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:03.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:03.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:03.554 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:03 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3738917802' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:18:03.610 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:03 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3738917802' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:18:03.611 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:03 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3738917802' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:18:03.623 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:03.912 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:18:03.945 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:03.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:03.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:04.050 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:04.083 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:04.504 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:04.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:04.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:04.613 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:04.627 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:18:04.627 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:18:04.627 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-01T09:18:04.627 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:18:04.678 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:04 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:04.678 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:04 smithi190 ceph-mon[160435]: failed to encode map e144 with expected crc 2023-10-01T09:18:04.678 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3738917802' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:18:04.678 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:04 smithi190 ceph-mon[160435]: osdmap e144: 8 total, 8 up, 8 in 2023-10-01T09:18:04.678 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:04 smithi190 ceph-mon[160435]: failed to encode map e144 with expected crc 2023-10-01T09:18:04.678 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:04 smithi190 ceph-mon[160435]: pgmap v366: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 7474/120792 objects misplaced (6.187%); 177 KiB/s, 5 objects/s recovering 2023-10-01T09:18:04.678 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:04 smithi190 ceph-mon[160435]: failed to encode map e144 with expected crc 2023-10-01T09:18:04.684 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:04 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:04.684 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:04 smithi138 ceph-mon[192513]: failed to encode map e144 with expected crc 2023-10-01T09:18:04.684 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3738917802' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:18:04.684 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:04 smithi138 ceph-mon[192513]: osdmap e144: 8 total, 8 up, 8 in 2023-10-01T09:18:04.684 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:04 smithi138 ceph-mon[192513]: failed to encode map e144 with expected crc 2023-10-01T09:18:04.685 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:04 smithi138 ceph-mon[192513]: pgmap v366: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 7474/120792 objects misplaced (6.187%); 177 KiB/s, 5 objects/s recovering 2023-10-01T09:18:04.685 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:04 smithi138 ceph-mon[192513]: failed to encode map e144 with expected crc 2023-10-01T09:18:04.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:04 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:04.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:04 smithi138 ceph-mon[185886]: failed to encode map e144 with expected crc 2023-10-01T09:18:04.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3738917802' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:18:04.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:04 smithi138 ceph-mon[185886]: osdmap e144: 8 total, 8 up, 8 in 2023-10-01T09:18:04.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:04 smithi138 ceph-mon[185886]: failed to encode map e144 with expected crc 2023-10-01T09:18:04.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:04 smithi138 ceph-mon[185886]: pgmap v366: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 7474/120792 objects misplaced (6.187%); 177 KiB/s, 5 objects/s recovering 2023-10-01T09:18:04.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:04 smithi138 ceph-mon[185886]: failed to encode map e144 with expected crc 2023-10-01T09:18:04.690 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:04.790 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:04.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:04.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:04.904 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:04.930 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:05.269 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:18:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:18:04] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.33.4" 2023-10-01T09:18:05.489 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:05.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:05.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:05.600 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:05.606 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:05.635 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:05.695 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:05.973 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:05.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:05.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:06.079 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:06.170 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:18:06.176 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:06.210 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:18:06.243 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:18:06.278 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:18:06.285 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:06.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:06.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:06.318 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:18:06.366 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:18:06.396 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:18:06.400 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:06.445 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:18:06.473 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:18:06.473 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:18:06.473 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:18:06.473 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:18:06.473 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:18:06.473 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:18:06.473 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:18:06.473 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:18:06.473 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:06.499 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:06 smithi138 ceph-mon[185886]: pgmap v367: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 360 op/s; 7474/120792 objects misplaced (6.187%); 148 KiB/s, 4 objects/s recovering 2023-10-01T09:18:06.499 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:06 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3451595795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:06.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:06 smithi138 ceph-mon[192513]: pgmap v367: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 360 op/s; 7474/120792 objects misplaced (6.187%); 148 KiB/s, 4 objects/s recovering 2023-10-01T09:18:06.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:06 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3451595795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:06.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:06 smithi190 ceph-mon[160435]: pgmap v367: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.5 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 360 op/s; 7474/120792 objects misplaced (6.187%); 148 KiB/s, 4 objects/s recovering 2023-10-01T09:18:06.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:06 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3451595795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:07.288 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:18:07.831 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:07.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:07.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:07.935 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:08.000 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:08.043 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:08.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:08.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:08.149 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:08.220 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:08.235 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:08 smithi138 ceph-mon[185886]: pgmap v368: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 6958/111630 objects misplaced (6.233%); 824 KiB/s, 14 objects/s recovering 2023-10-01T09:18:08.238 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:08 smithi138 ceph-mon[192513]: pgmap v368: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 6958/111630 objects misplaced (6.233%); 824 KiB/s, 14 objects/s recovering 2023-10-01T09:18:08.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:08 smithi190 ceph-mon[160435]: pgmap v368: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 6958/111630 objects misplaced (6.233%); 824 KiB/s, 14 objects/s recovering 2023-10-01T09:18:08.308 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:08.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:08.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:08.415 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:08.487 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:08.616 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:08.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:08.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:08.734 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:08.808 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:08.957 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:08.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:08.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:09.069 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:09.132 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:09.396 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:09.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:09.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:09.411 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:09.490 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:18:09.490 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:18:09.491 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-10-01T09:18:09.491 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:18:09.502 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:09.559 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:09.942 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:09.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:09.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:10.049 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:10.056 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:18:10.077 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:18:10.132 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:10.210 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:10.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:10.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:10.325 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:10.386 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:10.452 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:10 smithi190 ceph-mon[160435]: pgmap v369: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 6759/110409 objects misplaced (6.122%); 728 KiB/s, 12 objects/s recovering 2023-10-01T09:18:10.452 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:10.452 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:18:10.728 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:10.730 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:10 smithi138 ceph-mon[185886]: pgmap v369: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 6759/110409 objects misplaced (6.122%); 728 KiB/s, 12 objects/s recovering 2023-10-01T09:18:10.730 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:10.730 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:18:10.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:10.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:10.732 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:10 smithi138 ceph-mon[192513]: pgmap v369: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 6759/110409 objects misplaced (6.122%); 728 KiB/s, 12 objects/s recovering 2023-10-01T09:18:10.732 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:10.732 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:18:10.833 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:10.854 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:18:10.881 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:10.900 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:18:10.960 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:10.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:10.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:11.073 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:11.100 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:11.501 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:11.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:11.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:11.586 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":369,"stamp":"2023-10-01T09:18:09.365662+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2414811083,"num_objects":36803,"num_object_clones":0,"num_object_copies":110409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6759,"num_objects_unfound":0,"num_objects_dirty":36803,"num_whiteouts":0,"num_read":6303,"num_read_kb":12701,"num_write":194601,"num_write_kb":7383718,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":115,"num_bytes_recovered":6750208,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131168,"ondisk_log_size":131168,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8860576,"kb_used_data":7237216,"kb_used_omap":30,"kb_used_meta":1622881,"kb_avail":741100640,"statfs":{"total":767960285184,"available":758887055360,"internally_reserved":0,"allocated":7410909184,"data_stored":7406250531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30777,"internal_metadata":1661831111},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-370212864,"num_objects":-5649,"num_object_clones":0,"num_object_copies":-16947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6759,"num_objects_unfound":0,"num_objects_dirty":-5649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5649,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":115,"num_bytes_recovered":6750208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.058496"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":125,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.336980+0000","last_change":"2023-10-01T09:12:40.947346+0000","last_active":"2023-10-01T09:18:00.336980+0000","last_peered":"2023-10-01T09:18:00.336980+0000","last_clean":"2023-10-01T09:18:00.336980+0000","last_became_active":"2023-10-01T09:12:40.946939+0000","last_became_peered":"2023-10-01T09:12:40.946939+0000","last_unstale":"2023-10-01T09:18:00.336980+0000","last_undegraded":"2023-10-01T09:18:00.336980+0000","last_fullsized":"2023-10-01T09:18:00.336980+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":78,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:03.196267+0000","last_change":"2023-10-01T09:17:59.307189+0000","last_active":"2023-10-01T09:18:03.196267+0000","last_peered":"2023-10-01T09:18:03.196267+0000","last_clean":"2023-10-01T09:18:03.196267+0000","last_became_active":"2023-10-01T09:17:59.306888+0000","last_became_peered":"2023-10-01T09:17:59.306888+0000","last_unstale":"2023-10-01T09:18:03.196267+0000","last_undegraded":"2023-10-01T09:18:03.196267+0000","last_fullsized":"2023-10-01T09:18:03.196267+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:15:46.903546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":88,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.311073+0000","last_change":"2023-10-01T09:13:13.551596+0000","last_active":"2023-10-01T09:18:00.311073+0000","last_peered":"2023-10-01T09:18:00.311073+0000","last_clean":"2023-10-01T09:18:00.311073+0000","last_became_active":"2023-10-01T09:13:13.551300+0000","last_became_peered":"2023-10-01T09:13:13.551300+0000","last_unstale":"2023-10-01T09:18:00.311073+0000","last_undegraded":"2023-10-01T09:18:00.311073+0000","last_fullsized":"2023-10-01T09:18:00.311073+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:57:45.081780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":102,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.513033+0000","last_change":"2023-10-01T09:17:59.311912+0000","last_active":"2023-10-01T09:18:00.513033+0000","last_peered":"2023-10-01T09:18:00.513033+0000","last_clean":"2023-10-01T09:18:00.513033+0000","last_became_active":"2023-10-01T09:17:59.311581+0000","last_became_peered":"2023-10-01T09:17:59.311581+0000","last_unstale":"2023-10-01T09:18:00.513033+0000","last_undegraded":"2023-10-01T09:18:00.513033+0000","last_fullsized":"2023-10-01T09:18:00.513033+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:49:49.652135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":94,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:02.182965+0000","last_change":"2023-10-01T09:17:59.306474+0000","last_active":"2023-10-01T09:18:02.182965+0000","last_peered":"2023-10-01T09:18:02.182965+0000","last_clean":"2023-10-01T09:18:02.182965+0000","last_became_active":"2023-10-01T09:17:59.306122+0000","last_became_peered":"2023-10-01T09:17:59.306122+0000","last_unstale":"2023-10-01T09:18:02.182965+0000","last_undegraded":"2023-10-01T09:18:02.182965+0000","last_fullsized":"2023-10-01T09:18:02.182965+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:00:31.403192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":86,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.548385+0000","last_change":"2023-10-01T09:17:21.062397+0000","last_active":"2023-10-01T09:18:00.548385+0000","last_peered":"2023-10-01T09:18:00.548385+0000","last_clean":"2023-10-01T09:18:00.548385+0000","last_became_active":"2023-10-01T09:17:21.062025+0000","last_became_peered":"2023-10-01T09:17:21.062025+0000","last_unstale":"2023-10-01T09:18:00.548385+0000","last_undegraded":"2023-10-01T09:18:00.548385+0000","last_fullsized":"2023-10-01T09:18:00.548385+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":101,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.513374+0000","last_change":"2023-10-01T09:17:59.313644+0000","last_active":"2023-10-01T09:18:00.513374+0000","last_peered":"2023-10-01T09:18:00.513374+0000","last_clean":"2023-10-01T09:18:00.513374+0000","last_became_active":"2023-10-01T09:17:59.313395+0000","last_became_peered":"2023-10-01T09:17:59.313395+0000","last_unstale":"2023-10-01T09:18:00.513374+0000","last_undegraded":"2023-10-01T09:18:00.513374+0000","last_fullsized":"2023-10-01T09:18:00.513374+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:27:46.514515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":114,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.309288+0000","last_change":"2023-10-01T09:13:13.555368+0000","last_active":"2023-10-01T09:18:00.309288+0000","last_peered":"2023-10-01T09:18:00.309288+0000","last_clean":"2023-10-01T09:18:00.309288+0000","last_became_active":"2023-10-01T09:13:13.554806+0000","last_became_peered":"2023-10-01T09:13:13.554806+0000","last_unstale":"2023-10-01T09:18:00.309288+0000","last_undegraded":"2023-10-01T09:18:00.309288+0000","last_fullsized":"2023-10-01T09:18:00.309288+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:24:32.687833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.548305+0000","last_change":"2023-10-01T09:12:10.003091+0000","last_active":"2023-10-01T09:18:00.548305+0000","last_peered":"2023-10-01T09:18:00.548305+0000","last_clean":"2023-10-01T09:18:00.548305+0000","last_became_active":"2023-10-01T09:12:10.002773+0000","last_became_peered":"2023-10-01T09:12:10.002773+0000","last_unstale":"2023-10-01T09:18:00.548305+0000","last_undegraded":"2023-10-01T09:18:00.548305+0000","last_fullsized":"2023-10-01T09:18:00.548305+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":104,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.309922+0000","last_change":"2023-10-01T09:13:13.554942+0000","last_active":"2023-10-01T09:18:00.309922+0000","last_peered":"2023-10-01T09:18:00.309922+0000","last_clean":"2023-10-01T09:18:00.309922+0000","last_became_active":"2023-10-01T09:13:13.554664+0000","last_became_peered":"2023-10-01T09:13:13.554664+0000","last_unstale":"2023-10-01T09:18:00.309922+0000","last_undegraded":"2023-10-01T09:18:00.309922+0000","last_fullsized":"2023-10-01T09:18:00.309922+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:43:14.532709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":113,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.335299+0000","last_change":"2023-10-01T09:13:13.556801+0000","last_active":"2023-10-01T09:18:00.335299+0000","last_peered":"2023-10-01T09:18:00.335299+0000","last_clean":"2023-10-01T09:18:00.335299+0000","last_became_active":"2023-10-01T09:13:13.556397+0000","last_became_peered":"2023-10-01T09:13:13.556397+0000","last_unstale":"2023-10-01T09:18:00.335299+0000","last_undegraded":"2023-10-01T09:18:00.335299+0000","last_fullsized":"2023-10-01T09:18:00.335299+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":88,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.314312+0000","last_change":"2023-10-01T09:12:40.946520+0000","last_active":"2023-10-01T09:18:00.314312+0000","last_peered":"2023-10-01T09:18:00.314312+0000","last_clean":"2023-10-01T09:18:00.314312+0000","last_became_active":"2023-10-01T09:12:40.946227+0000","last_became_peered":"2023-10-01T09:12:40.946227+0000","last_unstale":"2023-10-01T09:18:00.314312+0000","last_undegraded":"2023-10-01T09:18:00.314312+0000","last_fullsized":"2023-10-01T09:18:00.314312+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:06:06.451923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":116,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.315046+0000","last_change":"2023-10-01T09:03:03.200261+0000","last_active":"2023-10-01T09:18:00.315046+0000","last_peered":"2023-10-01T09:18:00.315046+0000","last_clean":"2023-10-01T09:18:00.315046+0000","last_became_active":"2023-10-01T09:03:03.199944+0000","last_became_peered":"2023-10-01T09:03:03.199944+0000","last_unstale":"2023-10-01T09:18:00.315046+0000","last_undegraded":"2023-10-01T09:18:00.315046+0000","last_fullsized":"2023-10-01T09:18:00.315046+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":117,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.315085+0000","last_change":"2023-10-01T09:12:40.946973+0000","last_active":"2023-10-01T09:18:00.315085+0000","last_peered":"2023-10-01T09:18:00.315085+0000","last_clean":"2023-10-01T09:18:00.315085+0000","last_became_active":"2023-10-01T09:12:40.946427+0000","last_became_peered":"2023-10-01T09:12:40.946427+0000","last_unstale":"2023-10-01T09:18:00.315085+0000","last_undegraded":"2023-10-01T09:18:00.315085+0000","last_fullsized":"2023-10-01T09:18:00.315085+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":112,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.335039+0000","last_change":"2023-10-01T09:12:40.945277+0000","last_active":"2023-10-01T09:18:00.335039+0000","last_peered":"2023-10-01T09:18:00.335039+0000","last_clean":"2023-10-01T09:18:00.335039+0000","last_became_active":"2023-10-01T09:12:40.944843+0000","last_became_peered":"2023-10-01T09:12:40.944843+0000","last_unstale":"2023-10-01T09:18:00.335039+0000","last_undegraded":"2023-10-01T09:18:00.335039+0000","last_fullsized":"2023-10-01T09:18:00.335039+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":114,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.309597+0000","last_change":"2023-10-01T09:13:13.550633+0000","last_active":"2023-10-01T09:18:00.309597+0000","last_peered":"2023-10-01T09:18:00.309597+0000","last_clean":"2023-10-01T09:18:00.309597+0000","last_became_active":"2023-10-01T09:13:13.550305+0000","last_became_peered":"2023-10-01T09:13:13.550305+0000","last_unstale":"2023-10-01T09:18:00.309597+0000","last_undegraded":"2023-10-01T09:18:00.309597+0000","last_fullsized":"2023-10-01T09:18:00.309597+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:51:46.559383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":102,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.596257+0000","last_change":"2023-10-01T09:17:59.312113+0000","last_active":"2023-10-01T09:18:00.596257+0000","last_peered":"2023-10-01T09:18:00.596257+0000","last_clean":"2023-10-01T09:18:00.596257+0000","last_became_active":"2023-10-01T09:17:59.311811+0000","last_became_peered":"2023-10-01T09:17:59.311811+0000","last_unstale":"2023-10-01T09:18:00.596257+0000","last_undegraded":"2023-10-01T09:18:00.596257+0000","last_fullsized":"2023-10-01T09:18:00.596257+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:58:44.377802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":109,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.548184+0000","last_change":"2023-10-01T09:13:13.558367+0000","last_active":"2023-10-01T09:18:00.548184+0000","last_peered":"2023-10-01T09:18:00.548184+0000","last_clean":"2023-10-01T09:18:00.548184+0000","last_became_active":"2023-10-01T09:13:13.557952+0000","last_became_peered":"2023-10-01T09:13:13.557952+0000","last_unstale":"2023-10-01T09:18:00.548184+0000","last_undegraded":"2023-10-01T09:18:00.548184+0000","last_fullsized":"2023-10-01T09:18:00.548184+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":95,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.334929+0000","last_change":"2023-10-01T09:03:03.194952+0000","last_active":"2023-10-01T09:18:00.334929+0000","last_peered":"2023-10-01T09:18:00.334929+0000","last_clean":"2023-10-01T09:18:00.334929+0000","last_became_active":"2023-10-01T09:03:03.194643+0000","last_became_peered":"2023-10-01T09:03:03.194643+0000","last_unstale":"2023-10-01T09:18:00.334929+0000","last_undegraded":"2023-10-01T09:18:00.334929+0000","last_fullsized":"2023-10-01T09:18:00.334929+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.512974+0000","last_change":"2023-10-01T09:13:13.548531+0000","last_active":"2023-10-01T09:18:00.512974+0000","last_peered":"2023-10-01T09:18:00.512974+0000","last_clean":"2023-10-01T09:18:00.512974+0000","last_became_active":"2023-10-01T09:13:13.547881+0000","last_became_peered":"2023-10-01T09:13:13.547881+0000","last_unstale":"2023-10-01T09:18:00.512974+0000","last_undegraded":"2023-10-01T09:18:00.512974+0000","last_fullsized":"2023-10-01T09:18:00.512974+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:13:24.068359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":31,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.607251+0000","last_change":"2023-10-01T09:17:21.059877+0000","last_active":"2023-10-01T09:18:00.607251+0000","last_peered":"2023-10-01T09:18:00.607251+0000","last_clean":"2023-10-01T09:18:00.607251+0000","last_became_active":"2023-10-01T09:17:21.059577+0000","last_became_peered":"2023-10-01T09:17:21.059577+0000","last_unstale":"2023-10-01T09:18:00.607251+0000","last_undegraded":"2023-10-01T09:18:00.607251+0000","last_fullsized":"2023-10-01T09:18:00.607251+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:21:42.428953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":78,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.335554+0000","last_change":"2023-10-01T09:17:59.310379+0000","last_active":"2023-10-01T09:18:00.335554+0000","last_peered":"2023-10-01T09:18:00.335554+0000","last_clean":"2023-10-01T09:18:00.335554+0000","last_became_active":"2023-10-01T09:17:59.309999+0000","last_became_peered":"2023-10-01T09:17:59.309999+0000","last_unstale":"2023-10-01T09:18:00.335554+0000","last_undegraded":"2023-10-01T09:18:00.335554+0000","last_fullsized":"2023-10-01T09:18:00.335554+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.9","version":"144'12260","reported_seq":12306,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-01T09:18:06.345656+0000","last_change":"2023-10-01T09:14:31.351571+0000","last_active":"2023-10-01T09:18:06.345656+0000","last_peered":"2023-10-01T09:18:06.345656+0000","last_clean":"2023-10-01T09:18:06.345656+0000","last_became_active":"2023-10-01T09:14:31.351242+0000","last_became_peered":"2023-10-01T09:14:31.351242+0000","last_unstale":"2023-10-01T09:18:06.345656+0000","last_undegraded":"2023-10-01T09:18:06.345656+0000","last_fullsized":"2023-10-01T09:18:06.345656+0000","mapping_epoch":106,"log_start":"115'3600","ondisk_log_start":"115'3600","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8660,"log_dups_size":0,"ondisk_log_size":8660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154664960,"num_objects":2360,"num_object_clones":0,"num_object_copies":7080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12260,"num_write_kb":467840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":111,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.335644+0000","last_change":"2023-10-01T09:10:58.797850+0000","last_active":"2023-10-01T09:18:00.335644+0000","last_peered":"2023-10-01T09:18:00.335644+0000","last_clean":"2023-10-01T09:18:00.335644+0000","last_became_active":"2023-10-01T09:10:58.797403+0000","last_became_peered":"2023-10-01T09:10:58.797403+0000","last_unstale":"2023-10-01T09:18:00.335644+0000","last_undegraded":"2023-10-01T09:18:00.335644+0000","last_fullsized":"2023-10-01T09:18:00.335644+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":94,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.547870+0000","last_change":"2023-10-01T09:03:04.245105+0000","last_active":"2023-10-01T09:18:00.547870+0000","last_peered":"2023-10-01T09:18:00.547870+0000","last_clean":"2023-10-01T09:18:00.547870+0000","last_became_active":"2023-10-01T09:03:04.244807+0000","last_became_peered":"2023-10-01T09:03:04.244807+0000","last_unstale":"2023-10-01T09:18:00.547870+0000","last_undegraded":"2023-10-01T09:18:00.547870+0000","last_fullsized":"2023-10-01T09:18:00.547870+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"144'12397","reported_seq":12450,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-01T09:18:07.144382+0000","last_change":"2023-10-01T09:16:23.535190+0000","last_active":"2023-10-01T09:18:07.144382+0000","last_peered":"2023-10-01T09:18:07.144382+0000","last_clean":"2023-10-01T09:18:07.144382+0000","last_became_active":"2023-10-01T09:16:23.525193+0000","last_became_peered":"2023-10-01T09:16:23.525193+0000","last_unstale":"2023-10-01T09:18:07.144382+0000","last_undegraded":"2023-10-01T09:18:07.144382+0000","last_fullsized":"2023-10-01T09:18:07.144382+0000","mapping_epoch":125,"log_start":"116'4427","ondisk_log_start":"116'4427","created":106,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":7970,"log_dups_size":0,"ondisk_log_size":7970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:55:19.695436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154599424,"num_objects":2359,"num_object_clones":0,"num_object_copies":7077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12397,"num_write_kb":472192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":112,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.335000+0000","last_change":"2023-10-01T09:12:10.003552+0000","last_active":"2023-10-01T09:18:00.335000+0000","last_peered":"2023-10-01T09:18:00.335000+0000","last_clean":"2023-10-01T09:18:00.335000+0000","last_became_active":"2023-10-01T09:12:10.003131+0000","last_became_peered":"2023-10-01T09:12:10.003131+0000","last_unstale":"2023-10-01T09:18:00.335000+0000","last_undegraded":"2023-10-01T09:18:00.335000+0000","last_fullsized":"2023-10-01T09:18:00.335000+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":132,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.333903+0000","last_change":"2023-10-01T09:17:59.312696+0000","last_active":"2023-10-01T09:18:00.333903+0000","last_peered":"2023-10-01T09:18:00.333903+0000","last_clean":"2023-10-01T09:18:00.333903+0000","last_became_active":"2023-10-01T09:17:59.312344+0000","last_became_peered":"2023-10-01T09:17:59.312344+0000","last_unstale":"2023-10-01T09:18:00.333903+0000","last_undegraded":"2023-10-01T09:18:00.333903+0000","last_fullsized":"2023-10-01T09:18:00.333903+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"144'12161","reported_seq":12216,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-01T09:18:06.974406+0000","last_change":"2023-10-01T09:16:23.536677+0000","last_active":"2023-10-01T09:18:06.974406+0000","last_peered":"2023-10-01T09:18:06.974406+0000","last_clean":"2023-10-01T09:18:06.974406+0000","last_became_active":"2023-10-01T09:16:23.527778+0000","last_became_peered":"2023-10-01T09:16:23.527778+0000","last_unstale":"2023-10-01T09:18:06.974406+0000","last_undegraded":"2023-10-01T09:18:06.974406+0000","last_fullsized":"2023-10-01T09:18:06.974406+0000","mapping_epoch":125,"log_start":"116'4176","ondisk_log_start":"116'4176","created":106,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":7985,"log_dups_size":0,"ondisk_log_size":7985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:12:23.765297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152240128,"num_objects":2323,"num_object_clones":0,"num_object_copies":6969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12161,"num_write_kb":463488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":96,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.335474+0000","last_change":"2023-10-01T09:03:03.200972+0000","last_active":"2023-10-01T09:18:00.335474+0000","last_peered":"2023-10-01T09:18:00.335474+0000","last_clean":"2023-10-01T09:18:00.335474+0000","last_became_active":"2023-10-01T09:03:03.200695+0000","last_became_peered":"2023-10-01T09:03:03.200695+0000","last_unstale":"2023-10-01T09:18:00.335474+0000","last_undegraded":"2023-10-01T09:18:00.335474+0000","last_fullsized":"2023-10-01T09:18:00.335474+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":72,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.314615+0000","last_change":"2023-10-01T09:17:21.060912+0000","last_active":"2023-10-01T09:18:00.314615+0000","last_peered":"2023-10-01T09:18:00.314615+0000","last_clean":"2023-10-01T09:18:00.314615+0000","last_became_active":"2023-10-01T09:17:21.060646+0000","last_became_peered":"2023-10-01T09:17:21.060646+0000","last_unstale":"2023-10-01T09:18:00.314615+0000","last_undegraded":"2023-10-01T09:18:00.314615+0000","last_fullsized":"2023-10-01T09:18:00.314615+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:48:15.112481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.a","version":"144'11894","reported_seq":11964,"reported_epoch":144,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:18:06.338708+0000","last_change":"2023-10-01T09:18:00.697356+0000","last_active":"2023-10-01T09:18:06.338708+0000","last_peered":"2023-10-01T09:18:06.338708+0000","last_clean":"2023-10-01T09:17:58.286133+0000","last_became_active":"2023-10-01T09:18:00.601313+0000","last_became_peered":"2023-10-01T09:18:00.601313+0000","last_unstale":"2023-10-01T09:18:06.338708+0000","last_undegraded":"2023-10-01T09:18:06.338708+0000","last_fullsized":"2023-10-01T09:18:06.338708+0000","mapping_epoch":142,"log_start":"118'4868","ondisk_log_start":"118'4868","created":106,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":7026,"log_dups_size":0,"ondisk_log_size":7026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142999552,"num_objects":2182,"num_object_clones":0,"num_object_copies":6546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2182,"num_objects_unfound":0,"num_objects_dirty":2182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11894,"num_write_kb":450432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,3],"avail_no_missing":["4","2","3"],"object_location_counts":[{"shards":"2,3,4","objects":2182}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":119,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.548175+0000","last_change":"2023-10-01T09:17:59.311161+0000","last_active":"2023-10-01T09:18:00.548175+0000","last_peered":"2023-10-01T09:18:00.548175+0000","last_clean":"2023-10-01T09:18:00.548175+0000","last_became_active":"2023-10-01T09:17:59.310840+0000","last_became_peered":"2023-10-01T09:17:59.310840+0000","last_unstale":"2023-10-01T09:18:00.548175+0000","last_undegraded":"2023-10-01T09:18:00.548175+0000","last_fullsized":"2023-10-01T09:18:00.548175+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":93,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.512771+0000","last_change":"2023-10-01T09:12:10.001565+0000","last_active":"2023-10-01T09:18:00.512771+0000","last_peered":"2023-10-01T09:18:00.512771+0000","last_clean":"2023-10-01T09:18:00.512771+0000","last_became_active":"2023-10-01T09:12:10.001275+0000","last_became_peered":"2023-10-01T09:12:10.001275+0000","last_unstale":"2023-10-01T09:18:00.512771+0000","last_undegraded":"2023-10-01T09:18:00.512771+0000","last_fullsized":"2023-10-01T09:18:00.512771+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:39.067046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"144'12122","reported_seq":12178,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-01T09:18:06.338696+0000","last_change":"2023-10-01T09:16:23.542801+0000","last_active":"2023-10-01T09:18:06.338696+0000","last_peered":"2023-10-01T09:18:06.338696+0000","last_clean":"2023-10-01T09:18:06.338696+0000","last_became_active":"2023-10-01T09:16:23.529252+0000","last_became_peered":"2023-10-01T09:16:23.529252+0000","last_unstale":"2023-10-01T09:18:06.338696+0000","last_undegraded":"2023-10-01T09:18:06.338696+0000","last_fullsized":"2023-10-01T09:18:06.338696+0000","mapping_epoch":125,"log_start":"115'3500","ondisk_log_start":"115'3500","created":106,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8622,"log_dups_size":0,"ondisk_log_size":8622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150011928,"num_objects":2290,"num_object_clones":0,"num_object_copies":6870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2290,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12122,"num_write_kb":461121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":112,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.309433+0000","last_change":"2023-10-01T09:13:13.554321+0000","last_active":"2023-10-01T09:18:00.309433+0000","last_peered":"2023-10-01T09:18:00.309433+0000","last_clean":"2023-10-01T09:18:00.309433+0000","last_became_active":"2023-10-01T09:13:13.553986+0000","last_became_peered":"2023-10-01T09:13:13.553986+0000","last_unstale":"2023-10-01T09:18:00.309433+0000","last_undegraded":"2023-10-01T09:18:00.309433+0000","last_fullsized":"2023-10-01T09:18:00.309433+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:09:12.848269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":108,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.315117+0000","last_change":"2023-10-01T09:13:13.556491+0000","last_active":"2023-10-01T09:18:00.315117+0000","last_peered":"2023-10-01T09:18:00.315117+0000","last_clean":"2023-10-01T09:18:00.315117+0000","last_became_active":"2023-10-01T09:13:13.556118+0000","last_became_peered":"2023-10-01T09:13:13.556118+0000","last_unstale":"2023-10-01T09:18:00.315117+0000","last_undegraded":"2023-10-01T09:18:00.315117+0000","last_fullsized":"2023-10-01T09:18:00.315117+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"144'12123","reported_seq":12171,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-01T09:18:08.197680+0000","last_change":"2023-10-01T09:14:31.349314+0000","last_active":"2023-10-01T09:18:08.197680+0000","last_peered":"2023-10-01T09:18:08.197680+0000","last_clean":"2023-10-01T09:18:08.197680+0000","last_became_active":"2023-10-01T09:14:31.349041+0000","last_became_peered":"2023-10-01T09:14:31.349041+0000","last_unstale":"2023-10-01T09:18:08.197680+0000","last_undegraded":"2023-10-01T09:18:08.197680+0000","last_fullsized":"2023-10-01T09:18:08.197680+0000","mapping_epoch":106,"log_start":"118'5123","ondisk_log_start":"118'5123","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":7000,"log_dups_size":0,"ondisk_log_size":7000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:20:18.137610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146735104,"num_objects":2239,"num_object_clones":0,"num_object_copies":6717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12123,"num_write_kb":459584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":73,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.336992+0000","last_change":"2023-10-01T09:17:21.062618+0000","last_active":"2023-10-01T09:18:00.336992+0000","last_peered":"2023-10-01T09:18:00.336992+0000","last_clean":"2023-10-01T09:18:00.336992+0000","last_became_active":"2023-10-01T09:17:21.062271+0000","last_became_peered":"2023-10-01T09:17:21.062271+0000","last_unstale":"2023-10-01T09:18:00.336992+0000","last_undegraded":"2023-10-01T09:18:00.336992+0000","last_fullsized":"2023-10-01T09:18:00.336992+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":123,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.315224+0000","last_change":"2023-10-01T09:12:40.946791+0000","last_active":"2023-10-01T09:18:00.315224+0000","last_peered":"2023-10-01T09:18:00.315224+0000","last_clean":"2023-10-01T09:18:00.315224+0000","last_became_active":"2023-10-01T09:12:40.946418+0000","last_became_peered":"2023-10-01T09:12:40.946418+0000","last_unstale":"2023-10-01T09:18:00.315224+0000","last_undegraded":"2023-10-01T09:18:00.315224+0000","last_fullsized":"2023-10-01T09:18:00.315224+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"144'12078","reported_seq":12124,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-01T09:18:06.425689+0000","last_change":"2023-10-01T09:14:31.355388+0000","last_active":"2023-10-01T09:18:06.425689+0000","last_peered":"2023-10-01T09:18:06.425689+0000","last_clean":"2023-10-01T09:18:06.425689+0000","last_became_active":"2023-10-01T09:14:31.355065+0000","last_became_peered":"2023-10-01T09:14:31.355065+0000","last_unstale":"2023-10-01T09:18:06.425689+0000","last_undegraded":"2023-10-01T09:18:06.425689+0000","last_fullsized":"2023-10-01T09:18:06.425689+0000","mapping_epoch":106,"log_start":"113'2347","ondisk_log_start":"113'2347","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":9731,"log_dups_size":0,"ondisk_log_size":9731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154533888,"num_objects":2358,"num_object_clones":0,"num_object_copies":7074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12078,"num_write_kb":461952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":109,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.333804+0000","last_change":"2023-10-01T09:12:40.945099+0000","last_active":"2023-10-01T09:18:00.333804+0000","last_peered":"2023-10-01T09:18:00.333804+0000","last_clean":"2023-10-01T09:18:00.333804+0000","last_became_active":"2023-10-01T09:12:40.944712+0000","last_became_peered":"2023-10-01T09:12:40.944712+0000","last_unstale":"2023-10-01T09:18:00.333804+0000","last_undegraded":"2023-10-01T09:18:00.333804+0000","last_fullsized":"2023-10-01T09:18:00.333804+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":108,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.315371+0000","last_change":"2023-10-01T09:12:10.003847+0000","last_active":"2023-10-01T09:18:00.315371+0000","last_peered":"2023-10-01T09:18:00.315371+0000","last_clean":"2023-10-01T09:18:00.315371+0000","last_became_active":"2023-10-01T09:12:10.003525+0000","last_became_peered":"2023-10-01T09:12:10.003525+0000","last_unstale":"2023-10-01T09:18:00.315371+0000","last_undegraded":"2023-10-01T09:18:00.315371+0000","last_fullsized":"2023-10-01T09:18:00.315371+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"144'11976","reported_seq":12022,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-01T09:18:06.440899+0000","last_change":"2023-10-01T09:14:31.356833+0000","last_active":"2023-10-01T09:18:06.440899+0000","last_peered":"2023-10-01T09:18:06.440899+0000","last_clean":"2023-10-01T09:18:06.440899+0000","last_became_active":"2023-10-01T09:14:31.356520+0000","last_became_peered":"2023-10-01T09:14:31.356520+0000","last_unstale":"2023-10-01T09:18:06.440899+0000","last_undegraded":"2023-10-01T09:18:06.440899+0000","last_fullsized":"2023-10-01T09:18:06.440899+0000","mapping_epoch":106,"log_start":"118'4958","ondisk_log_start":"118'4958","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150339584,"num_objects":2294,"num_object_clones":0,"num_object_copies":6882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11976,"num_write_kb":456640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:03.194459+0000","last_change":"2023-10-01T09:17:59.310176+0000","last_active":"2023-10-01T09:18:03.194459+0000","last_peered":"2023-10-01T09:18:03.194459+0000","last_clean":"2023-10-01T09:18:03.194459+0000","last_became_active":"2023-10-01T09:17:59.309857+0000","last_became_peered":"2023-10-01T09:17:59.309857+0000","last_unstale":"2023-10-01T09:18:03.194459+0000","last_undegraded":"2023-10-01T09:18:03.194459+0000","last_fullsized":"2023-10-01T09:18:03.194459+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:17:24.906057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":64,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.309700+0000","last_change":"2023-10-01T09:16:23.527487+0000","last_active":"2023-10-01T09:18:00.309700+0000","last_peered":"2023-10-01T09:18:00.309700+0000","last_clean":"2023-10-01T09:18:00.309700+0000","last_became_active":"2023-10-01T09:16:23.527214+0000","last_became_peered":"2023-10-01T09:16:23.527214+0000","last_unstale":"2023-10-01T09:18:00.309700+0000","last_undegraded":"2023-10-01T09:18:00.309700+0000","last_fullsized":"2023-10-01T09:18:00.309700+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:18:54.916504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":958,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.310955+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:18:00.310955+0000","last_peered":"2023-10-01T09:18:00.310955+0000","last_clean":"2023-10-01T09:18:00.310955+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:18:00.310955+0000","last_undegraded":"2023-10-01T09:18:00.310955+0000","last_fullsized":"2023-10-01T09:18:00.310955+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":134,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.548284+0000","last_change":"2023-10-01T09:17:59.313692+0000","last_active":"2023-10-01T09:18:00.548284+0000","last_peered":"2023-10-01T09:18:00.548284+0000","last_clean":"2023-10-01T09:18:00.548284+0000","last_became_active":"2023-10-01T09:17:59.313371+0000","last_became_peered":"2023-10-01T09:17:59.313371+0000","last_unstale":"2023-10-01T09:18:00.548284+0000","last_undegraded":"2023-10-01T09:18:00.548284+0000","last_fullsized":"2023-10-01T09:18:00.548284+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.314376+0000","last_change":"2023-10-01T09:12:10.001493+0000","last_active":"2023-10-01T09:18:00.314376+0000","last_peered":"2023-10-01T09:18:00.314376+0000","last_clean":"2023-10-01T09:18:00.314376+0000","last_became_active":"2023-10-01T09:12:10.000899+0000","last_became_peered":"2023-10-01T09:12:10.000899+0000","last_unstale":"2023-10-01T09:18:00.314376+0000","last_undegraded":"2023-10-01T09:18:00.314376+0000","last_fullsized":"2023-10-01T09:18:00.314376+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:47:54.011059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"144'12168","reported_seq":12216,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-01T09:18:07.155794+0000","last_change":"2023-10-01T09:14:31.357395+0000","last_active":"2023-10-01T09:18:07.155794+0000","last_peered":"2023-10-01T09:18:07.155794+0000","last_clean":"2023-10-01T09:18:07.155794+0000","last_became_active":"2023-10-01T09:14:31.357014+0000","last_became_peered":"2023-10-01T09:14:31.357014+0000","last_unstale":"2023-10-01T09:18:07.155794+0000","last_undegraded":"2023-10-01T09:18:07.155794+0000","last_fullsized":"2023-10-01T09:18:07.155794+0000","mapping_epoch":106,"log_start":"116'4193","ondisk_log_start":"116'4193","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":7975,"log_dups_size":0,"ondisk_log_size":7975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:29.461431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151519232,"num_objects":2312,"num_object_clones":0,"num_object_copies":6936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12168,"num_write_kb":463360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"144'11882","reported_seq":11930,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-01T09:18:07.156853+0000","last_change":"2023-10-01T09:14:31.349232+0000","last_active":"2023-10-01T09:18:07.156853+0000","last_peered":"2023-10-01T09:18:07.156853+0000","last_clean":"2023-10-01T09:18:07.156853+0000","last_became_active":"2023-10-01T09:14:31.348940+0000","last_became_peered":"2023-10-01T09:14:31.348940+0000","last_unstale":"2023-10-01T09:18:07.156853+0000","last_undegraded":"2023-10-01T09:18:07.156853+0000","last_fullsized":"2023-10-01T09:18:07.156853+0000","mapping_epoch":106,"log_start":"116'3899","ondisk_log_start":"116'3899","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":7983,"log_dups_size":0,"ondisk_log_size":7983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:17.358604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147456000,"num_objects":2250,"num_object_clones":0,"num_object_copies":6750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11882,"num_write_kb":452224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":844,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.548044+0000","last_change":"2023-10-01T09:12:09.997936+0000","last_active":"2023-10-01T09:18:00.548044+0000","last_peered":"2023-10-01T09:18:00.548044+0000","last_clean":"2023-10-01T09:18:00.548044+0000","last_became_active":"2023-10-01T09:12:09.997427+0000","last_became_peered":"2023-10-01T09:12:09.997427+0000","last_unstale":"2023-10-01T09:18:00.548044+0000","last_undegraded":"2023-10-01T09:18:00.548044+0000","last_fullsized":"2023-10-01T09:18:00.548044+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1219,"num_read_kb":1114,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":109,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.314958+0000","last_change":"2023-10-01T09:10:58.799722+0000","last_active":"2023-10-01T09:18:00.314958+0000","last_peered":"2023-10-01T09:18:00.314958+0000","last_clean":"2023-10-01T09:18:00.314958+0000","last_became_active":"2023-10-01T09:10:58.799313+0000","last_became_peered":"2023-10-01T09:10:58.799313+0000","last_unstale":"2023-10-01T09:18:00.314958+0000","last_undegraded":"2023-10-01T09:18:00.314958+0000","last_fullsized":"2023-10-01T09:18:00.314958+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":112,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.310874+0000","last_change":"2023-10-01T09:17:59.307092+0000","last_active":"2023-10-01T09:18:00.310874+0000","last_peered":"2023-10-01T09:18:00.310874+0000","last_clean":"2023-10-01T09:18:00.310874+0000","last_became_active":"2023-10-01T09:17:59.306799+0000","last_became_peered":"2023-10-01T09:17:59.306799+0000","last_unstale":"2023-10-01T09:18:00.310874+0000","last_undegraded":"2023-10-01T09:18:00.310874+0000","last_fullsized":"2023-10-01T09:18:00.310874+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:47:37.914090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"144'12145","reported_seq":12200,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-01T09:18:06.398589+0000","last_change":"2023-10-01T09:16:23.548576+0000","last_active":"2023-10-01T09:18:06.398589+0000","last_peered":"2023-10-01T09:18:06.398589+0000","last_clean":"2023-10-01T09:18:06.398589+0000","last_became_active":"2023-10-01T09:16:23.528916+0000","last_became_peered":"2023-10-01T09:16:23.528916+0000","last_unstale":"2023-10-01T09:18:06.398589+0000","last_undegraded":"2023-10-01T09:18:06.398589+0000","last_fullsized":"2023-10-01T09:18:06.398589+0000","mapping_epoch":125,"log_start":"113'2391","ondisk_log_start":"113'2391","created":106,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":9754,"log_dups_size":0,"ondisk_log_size":9754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153419776,"num_objects":2341,"num_object_clones":0,"num_object_copies":7023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12145,"num_write_kb":463552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":347,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:02.183018+0000","last_change":"2023-10-01T09:17:59.322630+0000","last_active":"2023-10-01T09:18:02.183018+0000","last_peered":"2023-10-01T09:18:02.183018+0000","last_clean":"2023-10-01T09:18:02.183018+0000","last_became_active":"2023-10-01T09:17:59.306615+0000","last_became_peered":"2023-10-01T09:17:59.306615+0000","last_unstale":"2023-10-01T09:18:02.183018+0000","last_undegraded":"2023-10-01T09:18:02.183018+0000","last_fullsized":"2023-10-01T09:18:02.183018+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:47:35.120113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":89,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-01T09:18:03.363995+0000","last_change":"2023-10-01T09:12:40.946267+0000","last_active":"2023-10-01T09:18:03.363995+0000","last_peered":"2023-10-01T09:18:03.363995+0000","last_clean":"2023-10-01T09:18:03.363995+0000","last_became_active":"2023-10-01T09:12:40.945970+0000","last_became_peered":"2023-10-01T09:12:40.945970+0000","last_unstale":"2023-10-01T09:18:03.363995+0000","last_undegraded":"2023-10-01T09:18:03.363995+0000","last_fullsized":"2023-10-01T09:18:03.363995+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:42.904509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":118,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.311379+0000","last_change":"2023-10-01T09:12:09.994071+0000","last_active":"2023-10-01T09:18:00.311379+0000","last_peered":"2023-10-01T09:18:00.311379+0000","last_clean":"2023-10-01T09:18:00.311379+0000","last_became_active":"2023-10-01T09:12:09.993782+0000","last_became_peered":"2023-10-01T09:12:09.993782+0000","last_unstale":"2023-10-01T09:18:00.311379+0000","last_undegraded":"2023-10-01T09:18:00.311379+0000","last_fullsized":"2023-10-01T09:18:00.311379+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:12:49.880685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":960,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.548033+0000","last_change":"2023-10-01T09:12:40.946131+0000","last_active":"2023-10-01T09:18:00.548033+0000","last_peered":"2023-10-01T09:18:00.548033+0000","last_clean":"2023-10-01T09:18:00.548033+0000","last_became_active":"2023-10-01T09:12:40.945524+0000","last_became_peered":"2023-10-01T09:12:40.945524+0000","last_unstale":"2023-10-01T09:18:00.548033+0000","last_undegraded":"2023-10-01T09:18:00.548033+0000","last_fullsized":"2023-10-01T09:18:00.548033+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":346,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.334743+0000","last_change":"2023-10-01T09:10:58.798697+0000","last_active":"2023-10-01T09:18:00.334743+0000","last_peered":"2023-10-01T09:18:00.334743+0000","last_clean":"2023-10-01T09:18:00.334743+0000","last_became_active":"2023-10-01T09:10:58.798203+0000","last_became_peered":"2023-10-01T09:10:58.798203+0000","last_unstale":"2023-10-01T09:18:00.334743+0000","last_undegraded":"2023-10-01T09:18:00.334743+0000","last_fullsized":"2023-10-01T09:18:00.334743+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":68,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.548096+0000","last_change":"2023-10-01T09:17:59.314128+0000","last_active":"2023-10-01T09:18:00.548096+0000","last_peered":"2023-10-01T09:18:00.548096+0000","last_clean":"2023-10-01T09:18:00.548096+0000","last_became_active":"2023-10-01T09:17:59.313794+0000","last_became_peered":"2023-10-01T09:17:59.313794+0000","last_unstale":"2023-10-01T09:18:00.548096+0000","last_undegraded":"2023-10-01T09:18:00.548096+0000","last_fullsized":"2023-10-01T09:18:00.548096+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":20,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.315423+0000","last_change":"2023-10-01T09:17:21.062704+0000","last_active":"2023-10-01T09:18:00.315423+0000","last_peered":"2023-10-01T09:18:00.315423+0000","last_clean":"2023-10-01T09:18:00.315423+0000","last_became_active":"2023-10-01T09:17:21.062393+0000","last_became_peered":"2023-10-01T09:17:21.062393+0000","last_unstale":"2023-10-01T09:18:00.315423+0000","last_undegraded":"2023-10-01T09:18:00.315423+0000","last_fullsized":"2023-10-01T09:18:00.315423+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.4","version":"144'11900","reported_seq":11946,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-01T09:18:06.302129+0000","last_change":"2023-10-01T09:14:31.353858+0000","last_active":"2023-10-01T09:18:06.302129+0000","last_peered":"2023-10-01T09:18:06.302129+0000","last_clean":"2023-10-01T09:18:06.302129+0000","last_became_active":"2023-10-01T09:14:31.353572+0000","last_became_peered":"2023-10-01T09:14:31.353572+0000","last_unstale":"2023-10-01T09:18:06.302129+0000","last_undegraded":"2023-10-01T09:18:06.302129+0000","last_fullsized":"2023-10-01T09:18:06.302129+0000","mapping_epoch":106,"log_start":"115'3300","ondisk_log_start":"115'3300","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":8600,"log_dups_size":0,"ondisk_log_size":8600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140902400,"num_objects":2150,"num_object_clones":0,"num_object_copies":6450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11900,"num_write_kb":449600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"144'12095","reported_seq":12141,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-01T09:18:06.416580+0000","last_change":"2023-10-01T09:14:31.354366+0000","last_active":"2023-10-01T09:18:06.416580+0000","last_peered":"2023-10-01T09:18:06.416580+0000","last_clean":"2023-10-01T09:18:06.416580+0000","last_became_active":"2023-10-01T09:14:31.354059+0000","last_became_peered":"2023-10-01T09:14:31.354059+0000","last_unstale":"2023-10-01T09:18:06.416580+0000","last_undegraded":"2023-10-01T09:18:06.416580+0000","last_fullsized":"2023-10-01T09:18:06.416580+0000","mapping_epoch":106,"log_start":"113'2364","ondisk_log_start":"113'2364","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":9731,"log_dups_size":0,"ondisk_log_size":9731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156041216,"num_objects":2381,"num_object_clones":0,"num_object_copies":7143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12095,"num_write_kb":463232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":240,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.335103+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:18:00.335103+0000","last_peered":"2023-10-01T09:18:00.335103+0000","last_clean":"2023-10-01T09:18:00.335103+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:18:00.335103+0000","last_undegraded":"2023-10-01T09:18:00.335103+0000","last_fullsized":"2023-10-01T09:18:00.335103+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":102,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.597429+0000","last_change":"2023-10-01T09:17:59.309095+0000","last_active":"2023-10-01T09:18:00.597429+0000","last_peered":"2023-10-01T09:18:00.597429+0000","last_clean":"2023-10-01T09:18:00.597429+0000","last_became_active":"2023-10-01T09:17:59.308810+0000","last_became_peered":"2023-10-01T09:17:59.308810+0000","last_unstale":"2023-10-01T09:18:00.597429+0000","last_undegraded":"2023-10-01T09:18:00.597429+0000","last_fullsized":"2023-10-01T09:18:00.597429+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:04:41.671740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":108,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.513100+0000","last_change":"2023-10-01T09:12:40.944991+0000","last_active":"2023-10-01T09:18:00.513100+0000","last_peered":"2023-10-01T09:18:00.513100+0000","last_clean":"2023-10-01T09:18:00.513100+0000","last_became_active":"2023-10-01T09:12:40.944688+0000","last_became_peered":"2023-10-01T09:12:40.944688+0000","last_unstale":"2023-10-01T09:18:00.513100+0000","last_undegraded":"2023-10-01T09:18:00.513100+0000","last_fullsized":"2023-10-01T09:18:00.513100+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:30.015801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":504,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.335388+0000","last_change":"2023-10-01T09:10:58.797964+0000","last_active":"2023-10-01T09:18:00.335388+0000","last_peered":"2023-10-01T09:18:00.335388+0000","last_clean":"2023-10-01T09:18:00.335388+0000","last_became_active":"2023-10-01T09:10:58.797447+0000","last_became_peered":"2023-10-01T09:10:58.797447+0000","last_unstale":"2023-10-01T09:18:00.335388+0000","last_undegraded":"2023-10-01T09:18:00.335388+0000","last_fullsized":"2023-10-01T09:18:00.335388+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":88,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.311043+0000","last_change":"2023-10-01T09:12:40.948216+0000","last_active":"2023-10-01T09:18:00.311043+0000","last_peered":"2023-10-01T09:18:00.311043+0000","last_clean":"2023-10-01T09:18:00.311043+0000","last_became_active":"2023-10-01T09:12:40.947879+0000","last_became_peered":"2023-10-01T09:12:40.947879+0000","last_unstale":"2023-10-01T09:18:00.311043+0000","last_undegraded":"2023-10-01T09:18:00.311043+0000","last_fullsized":"2023-10-01T09:18:00.311043+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:46:40.612337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":78,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.512872+0000","last_change":"2023-10-01T09:10:58.797034+0000","last_active":"2023-10-01T09:18:00.512872+0000","last_peered":"2023-10-01T09:18:00.512872+0000","last_clean":"2023-10-01T09:18:00.512872+0000","last_became_active":"2023-10-01T09:10:58.796730+0000","last_became_peered":"2023-10-01T09:10:58.796730+0000","last_unstale":"2023-10-01T09:18:00.512872+0000","last_undegraded":"2023-10-01T09:18:00.512872+0000","last_fullsized":"2023-10-01T09:18:00.512872+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:32:15.982890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"144'12070","reported_seq":12122,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-10-01T09:18:07.146542+0000","last_change":"2023-10-01T09:17:21.092632+0000","last_active":"2023-10-01T09:18:07.146542+0000","last_peered":"2023-10-01T09:18:07.146542+0000","last_clean":"2023-10-01T09:18:07.146542+0000","last_became_active":"2023-10-01T09:17:21.060817+0000","last_became_peered":"2023-10-01T09:17:21.060817+0000","last_unstale":"2023-10-01T09:18:07.146542+0000","last_undegraded":"2023-10-01T09:18:07.146542+0000","last_fullsized":"2023-10-01T09:18:07.146542+0000","mapping_epoch":135,"log_start":"116'4169","ondisk_log_start":"116'4169","created":106,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":7901,"log_dups_size":0,"ondisk_log_size":7901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:54:06.150528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148373504,"num_objects":2264,"num_object_clones":0,"num_object_copies":6792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12070,"num_write_kb":458688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":112,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.309627+0000","last_change":"2023-10-01T09:13:13.553580+0000","last_active":"2023-10-01T09:18:00.309627+0000","last_peered":"2023-10-01T09:18:00.309627+0000","last_clean":"2023-10-01T09:18:00.309627+0000","last_became_active":"2023-10-01T09:13:13.553011+0000","last_became_peered":"2023-10-01T09:13:13.553011+0000","last_unstale":"2023-10-01T09:18:00.309627+0000","last_undegraded":"2023-10-01T09:18:00.309627+0000","last_fullsized":"2023-10-01T09:18:00.309627+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:54:30.369308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":532,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.311136+0000","last_change":"2023-10-01T09:12:10.002334+0000","last_active":"2023-10-01T09:18:00.311136+0000","last_peered":"2023-10-01T09:18:00.311136+0000","last_clean":"2023-10-01T09:18:00.311136+0000","last_became_active":"2023-10-01T09:12:10.001791+0000","last_became_peered":"2023-10-01T09:12:10.001791+0000","last_unstale":"2023-10-01T09:18:00.311136+0000","last_undegraded":"2023-10-01T09:18:00.311136+0000","last_fullsized":"2023-10-01T09:18:00.311136+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:24:27.632780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":73,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.334654+0000","last_change":"2023-10-01T09:17:59.312147+0000","last_active":"2023-10-01T09:18:00.334654+0000","last_peered":"2023-10-01T09:18:00.334654+0000","last_clean":"2023-10-01T09:18:00.334654+0000","last_became_active":"2023-10-01T09:17:59.311801+0000","last_became_peered":"2023-10-01T09:17:59.311801+0000","last_unstale":"2023-10-01T09:18:00.334654+0000","last_undegraded":"2023-10-01T09:18:00.334654+0000","last_fullsized":"2023-10-01T09:18:00.334654+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.1","version":"144'12273","reported_seq":12536,"reported_epoch":144,"state":"active+remapped+backfilling","last_fresh":"2023-10-01T09:18:06.440917+0000","last_change":"2023-10-01T09:18:00.701850+0000","last_active":"2023-10-01T09:18:06.440917+0000","last_peered":"2023-10-01T09:18:06.440917+0000","last_clean":"2023-10-01T09:17:58.302059+0000","last_became_active":"2023-10-01T09:18:00.601296+0000","last_became_peered":"2023-10-01T09:18:00.601296+0000","last_unstale":"2023-10-01T09:18:06.440917+0000","last_undegraded":"2023-10-01T09:18:06.440917+0000","last_fullsized":"2023-10-01T09:18:06.440917+0000","mapping_epoch":142,"log_start":"118'5219","ondisk_log_start":"118'5219","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":7054,"log_dups_size":0,"ondisk_log_size":7054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159449088,"num_objects":2433,"num_object_clones":0,"num_object_copies":7299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2338,"num_objects_unfound":0,"num_objects_dirty":2433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12273,"num_write_kb":470592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":103,"num_bytes_recovered":6750208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,3],"avail_no_missing":["4","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":2433}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":118,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.335569+0000","last_change":"2023-10-01T09:17:21.061272+0000","last_active":"2023-10-01T09:18:00.335569+0000","last_peered":"2023-10-01T09:18:00.335569+0000","last_clean":"2023-10-01T09:18:00.335569+0000","last_became_active":"2023-10-01T09:17:21.060884+0000","last_became_peered":"2023-10-01T09:17:21.060884+0000","last_unstale":"2023-10-01T09:18:00.335569+0000","last_undegraded":"2023-10-01T09:18:00.335569+0000","last_fullsized":"2023-10-01T09:18:00.335569+0000","mapping_epoch":135,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":136,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":97,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.513069+0000","last_change":"2023-10-01T09:12:10.003174+0000","last_active":"2023-10-01T09:18:00.513069+0000","last_peered":"2023-10-01T09:18:00.513069+0000","last_clean":"2023-10-01T09:18:00.513069+0000","last_became_active":"2023-10-01T09:12:10.002808+0000","last_became_peered":"2023-10-01T09:12:10.002808+0000","last_unstale":"2023-10-01T09:18:00.513069+0000","last_undegraded":"2023-10-01T09:18:00.513069+0000","last_fullsized":"2023-10-01T09:18:00.513069+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:25.037091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":93,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.333802+0000","last_change":"2023-10-01T09:03:03.754839+0000","last_active":"2023-10-01T09:18:00.333802+0000","last_peered":"2023-10-01T09:18:00.333802+0000","last_clean":"2023-10-01T09:18:00.333802+0000","last_became_active":"2023-10-01T09:03:03.754506+0000","last_became_peered":"2023-10-01T09:03:03.754506+0000","last_unstale":"2023-10-01T09:18:00.333802+0000","last_undegraded":"2023-10-01T09:18:00.333802+0000","last_fullsized":"2023-10-01T09:18:00.333802+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"144'11951","reported_seq":12008,"reported_epoch":144,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:18:08.166270+0000","last_change":"2023-10-01T09:18:00.733784+0000","last_active":"2023-10-01T09:18:08.166270+0000","last_peered":"2023-10-01T09:18:08.166270+0000","last_clean":"2023-10-01T09:17:58.285447+0000","last_became_active":"2023-10-01T09:18:00.609036+0000","last_became_peered":"2023-10-01T09:18:00.609036+0000","last_unstale":"2023-10-01T09:18:08.166270+0000","last_undegraded":"2023-10-01T09:18:08.166270+0000","last_fullsized":"2023-10-01T09:18:08.166270+0000","mapping_epoch":142,"log_start":"118'4963","ondisk_log_start":"118'4963","created":106,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:14:30.321026+0000","last_clean_scrub_stamp":"2023-10-01T09:14:30.321026+0000","objects_scrubbed":0,"log_size":6988,"log_dups_size":0,"ondisk_log_size":6988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:12:03.177067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146735104,"num_objects":2239,"num_object_clones":0,"num_object_copies":6717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2239,"num_objects_unfound":0,"num_objects_dirty":2239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11951,"num_write_kb":454080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":2239}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":81,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.512896+0000","last_change":"2023-10-01T09:10:58.801161+0000","last_active":"2023-10-01T09:18:00.512896+0000","last_peered":"2023-10-01T09:18:00.512896+0000","last_clean":"2023-10-01T09:18:00.512896+0000","last_became_active":"2023-10-01T09:10:58.800794+0000","last_became_peered":"2023-10-01T09:10:58.800794+0000","last_unstale":"2023-10-01T09:18:00.512896+0000","last_undegraded":"2023-10-01T09:18:00.512896+0000","last_fullsized":"2023-10-01T09:18:00.512896+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:35.019445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":123,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.311119+0000","last_change":"2023-10-01T09:17:59.309354+0000","last_active":"2023-10-01T09:18:00.311119+0000","last_peered":"2023-10-01T09:18:00.311119+0000","last_clean":"2023-10-01T09:18:00.311119+0000","last_became_active":"2023-10-01T09:17:59.308978+0000","last_became_peered":"2023-10-01T09:17:59.308978+0000","last_unstale":"2023-10-01T09:18:00.311119+0000","last_undegraded":"2023-10-01T09:18:00.311119+0000","last_fullsized":"2023-10-01T09:18:00.311119+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:31:00.623173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":112,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.548177+0000","last_change":"2023-10-01T09:12:10.000449+0000","last_active":"2023-10-01T09:18:00.548177+0000","last_peered":"2023-10-01T09:18:00.548177+0000","last_clean":"2023-10-01T09:18:00.548177+0000","last_became_active":"2023-10-01T09:12:10.000000+0000","last_became_peered":"2023-10-01T09:12:10.000000+0000","last_unstale":"2023-10-01T09:18:00.548177+0000","last_undegraded":"2023-10-01T09:18:00.548177+0000","last_fullsized":"2023-10-01T09:18:00.548177+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":123,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.315177+0000","last_change":"2023-10-01T09:13:13.555744+0000","last_active":"2023-10-01T09:18:00.315177+0000","last_peered":"2023-10-01T09:18:00.315177+0000","last_clean":"2023-10-01T09:18:00.315177+0000","last_became_active":"2023-10-01T09:13:13.555163+0000","last_became_peered":"2023-10-01T09:13:13.555163+0000","last_unstale":"2023-10-01T09:18:00.315177+0000","last_undegraded":"2023-10-01T09:18:00.315177+0000","last_fullsized":"2023-10-01T09:18:00.315177+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":97,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.548350+0000","last_change":"2023-10-01T09:03:03.202063+0000","last_active":"2023-10-01T09:18:00.548350+0000","last_peered":"2023-10-01T09:18:00.548350+0000","last_clean":"2023-10-01T09:18:00.548350+0000","last_became_active":"2023-10-01T09:03:03.201769+0000","last_became_peered":"2023-10-01T09:03:03.201769+0000","last_unstale":"2023-10-01T09:18:00.548350+0000","last_undegraded":"2023-10-01T09:18:00.548350+0000","last_fullsized":"2023-10-01T09:18:00.548350+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":123,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.337029+0000","last_change":"2023-10-01T09:13:13.554959+0000","last_active":"2023-10-01T09:18:00.337029+0000","last_peered":"2023-10-01T09:18:00.337029+0000","last_clean":"2023-10-01T09:18:00.337029+0000","last_became_active":"2023-10-01T09:13:13.554589+0000","last_became_peered":"2023-10-01T09:13:13.554589+0000","last_unstale":"2023-10-01T09:18:00.337029+0000","last_undegraded":"2023-10-01T09:18:00.337029+0000","last_fullsized":"2023-10-01T09:18:00.337029+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":13,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:03.196282+0000","last_change":"2023-10-01T09:17:59.307314+0000","last_active":"2023-10-01T09:18:03.196282+0000","last_peered":"2023-10-01T09:18:03.196282+0000","last_clean":"2023-10-01T09:18:03.196282+0000","last_became_active":"2023-10-01T09:17:59.306846+0000","last_became_peered":"2023-10-01T09:17:59.306846+0000","last_unstale":"2023-10-01T09:18:03.196282+0000","last_undegraded":"2023-10-01T09:18:03.196282+0000","last_fullsized":"2023-10-01T09:18:03.196282+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:49:04.919973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":97,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:02.183000+0000","last_change":"2023-10-01T09:17:59.308146+0000","last_active":"2023-10-01T09:18:02.183000+0000","last_peered":"2023-10-01T09:18:02.183000+0000","last_clean":"2023-10-01T09:18:02.183000+0000","last_became_active":"2023-10-01T09:17:59.307679+0000","last_became_peered":"2023-10-01T09:17:59.307679+0000","last_unstale":"2023-10-01T09:18:02.183000+0000","last_undegraded":"2023-10-01T09:18:02.183000+0000","last_fullsized":"2023-10-01T09:18:02.183000+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:13:55.096451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":97,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.513299+0000","last_change":"2023-10-01T09:12:40.944944+0000","last_active":"2023-10-01T09:18:00.513299+0000","last_peered":"2023-10-01T09:18:00.513299+0000","last_clean":"2023-10-01T09:18:00.513299+0000","last_became_active":"2023-10-01T09:12:40.944613+0000","last_became_peered":"2023-10-01T09:12:40.944613+0000","last_unstale":"2023-10-01T09:18:00.513299+0000","last_undegraded":"2023-10-01T09:18:00.513299+0000","last_fullsized":"2023-10-01T09:18:00.513299+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:48:20.808947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":108,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-10-01T09:18:00.314757+0000","last_change":"2023-10-01T09:12:10.003833+0000","last_active":"2023-10-01T09:18:00.314757+0000","last_peered":"2023-10-01T09:18:00.314757+0000","last_clean":"2023-10-01T09:18:00.314757+0000","last_became_active":"2023-10-01T09:12:10.003482+0000","last_became_peered":"2023-10-01T09:12:10.003482+0000","last_unstale":"2023-10-01T09:18:00.314757+0000","last_undegraded":"2023-10-01T09:18:00.314757+0000","last_fullsized":"2023-10-01T09:18:00.314757+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":2410020888,"num_objects":36775,"num_object_clones":0,"num_object_copies":110325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6759,"num_objects_unfound":0,"num_objects_dirty":36775,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":193495,"num_write_kb":7368577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":105,"num_bytes_recovered":6750208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7285321728,"data_stored":7285309512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129998,"ondisk_log_size":129998,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27349,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12188,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430410,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1057912,"kb_used_data":760560,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92687240,"statfs":{"total":95995035648,"available":94911733760,"internally_reserved":0,"allocated":778813440,"data_stored":778285810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:17:05 2023","interfaces":[{"interface":"back","average":{"1min":6.6369999999999996,"5min":3.0600000000000001,"15min":1.3999999999999999},"min":{"1min":0.22500000000000001,"5min":0.219,"15min":0.219},"max":{"1min":34.405999999999999,"5min":94.162999999999997,"15min":94.162999999999997},"last":4.2190000000000003},{"interface":"front","average":{"1min":4.9850000000000003,"5min":2.609,"15min":1.2350000000000001},"min":{"1min":0.26000000000000001,"5min":0.254,"15min":0.254},"max":{"1min":34.305999999999997,"5min":94.135000000000005,"15min":94.135000000000005},"last":0.40300000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:17:20 2023","interfaces":[{"interface":"back","average":{"1min":3.0600000000000001,"5min":2.044,"15min":1.0580000000000001},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.9640000000000004,"5min":43.801000000000002,"15min":43.801000000000002},"last":3.9670000000000001},{"interface":"front","average":{"1min":2.1909999999999998,"5min":1.7010000000000001,"15min":0.93600000000000005},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.2690000000000001,"5min":43.328000000000003,"15min":43.328000000000003},"last":6.0579999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:17:48 2023","interfaces":[{"interface":"back","average":{"1min":2.5,"5min":2.5339999999999998,"15min":1.21},"min":{"1min":0.317,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.7380000000000004,"5min":82.379000000000005,"15min":82.379000000000005},"last":3.7069999999999999},{"interface":"front","average":{"1min":2.5230000000000001,"5min":2.4950000000000001,"15min":1.1899999999999999},"min":{"1min":0.31,"5min":0.309,"15min":0.309},"max":{"1min":4.7610000000000001,"5min":85.596000000000004,"15min":85.596000000000004},"last":1.8}]},{"osd":3,"last update":"Sun Oct 1 09:17:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3220000000000001,"5min":1.232,"15min":0.79000000000000004},"min":{"1min":0.31900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.9960000000000004,"5min":6.1970000000000001,"15min":6.1970000000000001},"last":6.0250000000000004},{"interface":"front","average":{"1min":2.8399999999999999,"5min":1.272,"15min":0.78800000000000003},"min":{"1min":0.29599999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":11.262,"5min":11.262,"15min":11.262},"last":3.9049999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:17:05 2023","interfaces":[{"interface":"back","average":{"1min":7.3520000000000003,"5min":2.0640000000000001,"15min":1.008},"min":{"1min":0.28599999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":82.992000000000004,"5min":82.992000000000004,"15min":82.992000000000004},"last":3.0950000000000002},{"interface":"front","average":{"1min":7.2759999999999998,"5min":2.073,"15min":1.0409999999999999},"min":{"1min":0.35699999999999998,"5min":0.252,"15min":0.252},"max":{"1min":83.058999999999997,"5min":83.058999999999997,"15min":83.058999999999997},"last":0.64100000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:17:05 2023","interfaces":[{"interface":"back","average":{"1min":6.6139999999999999,"5min":1.8700000000000001,"15min":0.96699999999999997},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":90.656999999999996,"5min":90.656999999999996,"15min":90.656999999999996},"last":2.964},{"interface":"front","average":{"1min":7.851,"5min":2.2810000000000001,"15min":1.125},"min":{"1min":0.35599999999999998,"5min":0.32500000000000001,"15min":0.28599999999999998},"max":{"1min":90.747,"5min":90.747,"15min":90.747},"last":2.786}]},{"osd":6,"last update":"Sun Oct 1 09:17:05 2023","interfaces":[{"interface":"back","average":{"1min":2.6360000000000001,"5min":1.1579999999999999,"15min":0.75900000000000001},"min":{"1min":0.38400000000000001,"5min":0.311,"15min":0.29999999999999999},"max":{"1min":10.468999999999999,"5min":10.468999999999999,"15min":10.468999999999999},"last":2.819},{"interface":"front","average":{"1min":2.3170000000000002,"5min":1.1060000000000001,"15min":0.76200000000000001},"min":{"1min":0.377,"5min":0.34300000000000003,"15min":0.33400000000000002},"max":{"1min":10.641999999999999,"5min":10.641999999999999,"15min":10.641999999999999},"last":0.433}]}]},{"osd":6,"up_from":42,"seq":180388626639,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":915812,"kb_used_data":606300,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92829340,"statfs":{"total":95995035648,"available":95057244160,"internally_reserved":0,"allocated":620851200,"data_stored":620343269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:18:05 2023","interfaces":[{"interface":"back","average":{"1min":6.3869999999999996,"5min":7.6399999999999997,"15min":2.956},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":70.013999999999996,"5min":178.738,"15min":178.738},"last":6.9219999999999997},{"interface":"front","average":{"1min":6.6900000000000004,"5min":8.1660000000000004,"15min":3.133},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":69.981999999999999,"5min":178.684,"15min":178.684},"last":7.1760000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:17:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1459999999999999,"5min":1.4490000000000001,"15min":0.86699999999999999},"min":{"1min":0.25,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":11.755000000000001,"5min":24.488,"15min":24.488},"last":1.2589999999999999},{"interface":"front","average":{"1min":4.125,"5min":1.984,"15min":1.0620000000000001},"min":{"1min":0.32100000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":17.792000000000002,"5min":24.643999999999998,"15min":24.643999999999998},"last":7.1269999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:17:47 2023","interfaces":[{"interface":"back","average":{"1min":6.4279999999999999,"5min":4.6799999999999997,"15min":1.9119999999999999},"min":{"1min":0.27200000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":57.753,"5min":82.978999999999999,"15min":82.978999999999999},"last":7.2050000000000001},{"interface":"front","average":{"1min":4.7889999999999997,"5min":4.0839999999999996,"15min":1.71},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":57.695999999999998,"5min":82.747,"15min":82.747},"last":2.9289999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:17:20 2023","interfaces":[{"interface":"back","average":{"1min":3.8900000000000001,"5min":1.4830000000000001,"15min":0.871},"min":{"1min":0.33300000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":35.718000000000004,"5min":35.718000000000004,"15min":35.718000000000004},"last":6.8879999999999999},{"interface":"front","average":{"1min":5.1879999999999997,"5min":2.0049999999999999,"15min":1.0429999999999999},"min":{"1min":0.35599999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":31.568999999999999,"5min":31.568999999999999,"15min":31.568999999999999},"last":7.2240000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:17:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":1.4330000000000001,"15min":0.79100000000000004},"min":{"1min":0.36599999999999999,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":6.7709999999999999,"5min":28.271999999999998,"15min":28.271999999999998},"last":2.4580000000000002},{"interface":"front","average":{"1min":2.633,"5min":1.5569999999999999,"15min":0.86599999999999999},"min":{"1min":0.32700000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.619999999999999,"5min":28.413,"15min":28.413},"last":2.746}]},{"osd":5,"last update":"Sun Oct 1 09:17:47 2023","interfaces":[{"interface":"back","average":{"1min":3.1269999999999998,"5min":1.948,"15min":0.98999999999999999},"min":{"1min":0.33400000000000002,"5min":0.32600000000000001,"15min":0.251},"max":{"1min":9.2159999999999993,"5min":36.063000000000002,"15min":36.063000000000002},"last":2.4849999999999999},{"interface":"front","average":{"1min":2.8210000000000002,"5min":1.657,"15min":0.92100000000000004},"min":{"1min":0.35999999999999999,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":13.648999999999999,"5min":35.932000000000002,"15min":35.932000000000002},"last":0.39100000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:18:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2370000000000001,"5min":1.401,"15min":0.83899999999999997},"min":{"1min":0.33800000000000002,"5min":0.307,"15min":0.29899999999999999},"max":{"1min":12.836,"5min":12.866,"15min":12.866},"last":2.512},{"interface":"front","average":{"1min":2.798,"5min":1.3560000000000001,"15min":0.84899999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":13.574999999999999,"5min":13.574999999999999,"15min":13.574999999999999},"last":0.31}]}]},{"osd":1,"up_from":94,"seq":403726925898,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":841352,"kb_used_data":737472,"kb_used_omap":7,"kb_used_meta":103864,"kb_avail":92903800,"statfs":{"total":95995035648,"available":95133491200,"internally_reserved":0,"allocated":755171328,"data_stored":754511908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":106357159},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:17:16 2023","interfaces":[{"interface":"back","average":{"1min":3.444,"5min":1.78,"15min":0.82399999999999995},"min":{"1min":0.23200000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":26.030000000000001,"5min":28.449999999999999,"15min":28.449999999999999},"last":8.9179999999999993},{"interface":"front","average":{"1min":4.1719999999999997,"5min":1.839,"15min":0.85699999999999998},"min":{"1min":0.24199999999999999,"5min":0.188,"15min":0.188},"max":{"1min":30.539000000000001,"5min":30.539000000000001,"15min":30.539000000000001},"last":9.2370000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.946,"5min":1.6879999999999999,"15min":0.78400000000000003},"min":{"1min":0.21099999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":9.8379999999999992,"5min":25.632000000000001,"15min":25.632000000000001},"last":8.9440000000000008},{"interface":"front","average":{"1min":2.1429999999999998,"5min":1.919,"15min":0.86299999999999999},"min":{"1min":0.252,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":11.552,"5min":29.425999999999998,"15min":29.425999999999998},"last":9.2690000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:17:21 2023","interfaces":[{"interface":"back","average":{"1min":2.8079999999999998,"5min":1.266,"15min":0.67000000000000004},"min":{"1min":0.25,"5min":0.218,"15min":0.218},"max":{"1min":9.5589999999999993,"5min":9.5589999999999993,"15min":9.5589999999999993},"last":9.4060000000000006},{"interface":"front","average":{"1min":1.478,"5min":1.0249999999999999,"15min":0.59699999999999998},"min":{"1min":0.30099999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.6319999999999997,"5min":13.853,"15min":13.853},"last":8.8930000000000007}]},{"osd":4,"last update":"Sun Oct 1 09:17:16 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.24,"15min":0.77500000000000002},"min":{"1min":0.32500000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.3360000000000003,"5min":8.3360000000000003,"15min":8.3360000000000003},"last":5.4080000000000004},{"interface":"front","average":{"1min":1.7410000000000001,"5min":1.2649999999999999,"15min":0.79500000000000004},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":8.5980000000000008,"5min":13.112,"15min":13.112},"last":8.8040000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:17:16 2023","interfaces":[{"interface":"back","average":{"1min":2.5510000000000002,"5min":1.383,"15min":0.83099999999999996},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":5.5439999999999996,"5min":6.2919999999999998,"15min":6.2919999999999998},"last":9.375},{"interface":"front","average":{"1min":3.036,"5min":1.425,"15min":0.873},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":9.8010000000000002,"5min":9.8010000000000002,"15min":9.8010000000000002},"last":5.3289999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:17:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3279999999999998,"5min":1.266,"15min":0.82199999999999995},"min":{"1min":0.34999999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.1050000000000004,"5min":6.2300000000000004,"15min":6.2300000000000004},"last":9.3379999999999992},{"interface":"front","average":{"1min":1.829,"5min":1.101,"15min":0.78900000000000003},"min":{"1min":0.38100000000000001,"5min":0.372,"15min":0.372},"max":{"1min":6.2480000000000002,"5min":6.2480000000000002,"15min":6.2480000000000002},"last":5.3609999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:17:16 2023","interfaces":[{"interface":"back","average":{"1min":3.1299999999999999,"5min":1.423,"15min":0.875},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.8439999999999994,"5min":9.8439999999999994,"15min":9.8439999999999994},"last":5.3840000000000003},{"interface":"front","average":{"1min":2.3479999999999999,"5min":1.413,"15min":0.875},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":9.9440000000000008,"5min":13.071999999999999,"15min":13.071999999999999},"last":8.859}]}]},{"osd":0,"up_from":89,"seq":382252089432,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1334732,"kb_used_data":1204548,"kb_used_omap":8,"kb_used_meta":130103,"kb_avail":92410420,"statfs":{"total":95995035648,"available":94628270080,"internally_reserved":0,"allocated":1233457152,"data_stored":1232797434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":133226434},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:17:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0899999999999999,"5min":1.1359999999999999,"15min":0.59399999999999997},"min":{"1min":0.20000000000000001,"5min":0.187,"15min":0.187},"max":{"1min":11.321,"5min":11.321,"15min":11.321},"last":3.9900000000000002},{"interface":"front","average":{"1min":2.1909999999999998,"5min":1.1539999999999999,"15min":0.63300000000000001},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":8.1989999999999998,"5min":9.3079999999999998,"15min":9.3079999999999998},"last":0.437}]},{"osd":2,"last update":"Sun Oct 1 09:17:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5910000000000002,"5min":1.5429999999999999,"15min":0.77500000000000002},"min":{"1min":0.26100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":11.353999999999999,"5min":12.944000000000001,"15min":12.944000000000001},"last":3.9540000000000002},{"interface":"front","average":{"1min":3.3210000000000002,"5min":1.595,"15min":0.80300000000000005},"min":{"1min":0.27700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":11.045999999999999,"5min":12.304,"15min":12.304},"last":3.8220000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:17:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":1.143,"15min":0.63200000000000001},"min":{"1min":0.247,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":11.375,"5min":12.906000000000001,"15min":12.906000000000001},"last":4.0110000000000001},{"interface":"front","average":{"1min":2.48,"5min":1.224,"15min":0.65200000000000002},"min":{"1min":0.30399999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":17.318000000000001,"5min":17.318000000000001,"15min":17.318000000000001},"last":3.6600000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:17:12 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.3799999999999999,"15min":0.88800000000000001},"min":{"1min":0.38900000000000001,"5min":0.317,"15min":0.317},"max":{"1min":8.3100000000000005,"5min":9.9540000000000006,"15min":9.9540000000000006},"last":3.7930000000000001},{"interface":"front","average":{"1min":3.645,"5min":1.7,"15min":1.018},"min":{"1min":0.36499999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":13.459,"5min":13.459,"15min":13.459},"last":3.5819999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:17:12 2023","interfaces":[{"interface":"back","average":{"1min":2.911,"5min":1.577,"15min":0.98499999999999999},"min":{"1min":0.39500000000000002,"5min":0.311,"15min":0.311},"max":{"1min":10.911,"5min":12.263999999999999,"15min":12.263999999999999},"last":3.6389999999999998},{"interface":"front","average":{"1min":1.859,"5min":1.4199999999999999,"15min":0.93100000000000005},"min":{"1min":0.44500000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":9.0229999999999997,"5min":12.779,"15min":12.779},"last":3.7269999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:17:12 2023","interfaces":[{"interface":"back","average":{"1min":2.7410000000000001,"5min":1.6970000000000001,"15min":1.0309999999999999},"min":{"1min":0.53400000000000003,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.885,"5min":12.239000000000001,"15min":12.239000000000001},"last":3.6120000000000001},{"interface":"front","average":{"1min":1.506,"5min":1.3200000000000001,"15min":0.89700000000000002},"min":{"1min":0.41699999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":8.9969999999999999,"5min":12.875,"15min":12.875},"last":3.9260000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:17:12 2023","interfaces":[{"interface":"back","average":{"1min":2.7709999999999999,"5min":1.6499999999999999,"15min":1.0209999999999999},"min":{"1min":0.45900000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":16.48,"5min":16.48,"15min":16.48},"last":3.8809999999999998},{"interface":"front","average":{"1min":2.403,"5min":1.4510000000000001,"15min":0.95199999999999996},"min":{"1min":0.44,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":8.2140000000000004,"5min":8.2140000000000004,"15min":8.2140000000000004},"last":3.7570000000000001}]}]},{"osd":2,"up_from":99,"seq":425201762371,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1026544,"kb_used_data":926760,"kb_used_omap":11,"kb_used_meta":99700,"kb_avail":92718608,"statfs":{"total":95995035648,"available":94943854592,"internally_reserved":0,"allocated":949002240,"data_stored":948364620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":102093687},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:17:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9889999999999999,"5min":2.9199999999999999,"15min":1.1919999999999999},"min":{"1min":0.22700000000000001,"5min":0.188,"15min":0.188},"max":{"1min":24.018000000000001,"5min":93.634,"15min":93.634},"last":0.247},{"interface":"front","average":{"1min":4.2279999999999998,"5min":3.6379999999999999,"15min":1.4399999999999999},"min":{"1min":0.20899999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":24.882999999999999,"5min":93.906999999999996,"15min":93.906999999999996},"last":0.34200000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:17:54 2023","interfaces":[{"interface":"back","average":{"1min":2.569,"5min":2.698,"15min":1.143},"min":{"1min":0.27600000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":17.338000000000001,"5min":105.3,"15min":105.3},"last":1.087},{"interface":"front","average":{"1min":2.6429999999999998,"5min":3.218,"15min":1.3200000000000001},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":17.826000000000001,"5min":107.729,"15min":107.729},"last":0.27100000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:17:20 2023","interfaces":[{"interface":"back","average":{"1min":3.387,"5min":1.226,"15min":0.65500000000000003},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":17.498999999999999,"5min":17.498999999999999,"15min":17.498999999999999},"last":1.0600000000000001},{"interface":"front","average":{"1min":3.5430000000000001,"5min":1.327,"15min":0.71699999999999997},"min":{"1min":0.316,"5min":0.25,"15min":0.25},"max":{"1min":17.446999999999999,"5min":17.446999999999999,"15min":17.446999999999999},"last":0.309}]},{"osd":4,"last update":"Sun Oct 1 09:17:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3980000000000001,"5min":1.3839999999999999,"15min":0.86399999999999999},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":17.751999999999999,"5min":17.751999999999999,"15min":17.751999999999999},"last":16.059000000000001},{"interface":"front","average":{"1min":2.847,"5min":1.5660000000000001,"15min":0.89500000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":17.568000000000001,"5min":17.568000000000001,"15min":17.568000000000001},"last":7.0789999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:17:54 2023","interfaces":[{"interface":"back","average":{"1min":4.7489999999999997,"5min":2.1190000000000002,"15min":1.089},"min":{"1min":0.33600000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":34.084000000000003,"5min":34.084000000000003,"15min":34.084000000000003},"last":2.8969999999999998},{"interface":"front","average":{"1min":4.5339999999999998,"5min":2.0920000000000001,"15min":1.0800000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":34.341999999999999,"5min":34.341999999999999,"15min":34.341999999999999},"last":4.1509999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:17:54 2023","interfaces":[{"interface":"back","average":{"1min":3.121,"5min":1.673,"15min":0.95799999999999996},"min":{"1min":0.34000000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":17.591000000000001,"5min":17.591000000000001,"15min":17.591000000000001},"last":4.6120000000000001},{"interface":"front","average":{"1min":1.978,"5min":1.218,"15min":0.82099999999999995},"min":{"1min":0.371,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":17.736000000000001,"5min":17.736000000000001,"15min":17.736000000000001},"last":0.45400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:17:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5859999999999999,"5min":2.7330000000000001,"15min":1.3120000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":17.713000000000001,"5min":109.506,"15min":109.506},"last":2.609},{"interface":"front","average":{"1min":3.6150000000000002,"5min":3.0470000000000002,"15min":1.425},"min":{"1min":0.38600000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":22.353999999999999,"5min":109.58799999999999,"15min":109.58799999999999},"last":2.6480000000000001}]}]},{"osd":3,"up_from":104,"seq":446676598845,"num_pgs":16,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":516404,"kb_used_data":448748,"kb_used_omap":3,"kb_used_meta":67580,"kb_avail":93228748,"statfs":{"total":95995035648,"available":95466237952,"internally_reserved":0,"allocated":459517952,"data_stored":458875818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":69202663},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:17:17 2023","interfaces":[{"interface":"back","average":{"1min":1.778,"5min":4.7549999999999999,"15min":1.829},"min":{"1min":0.246,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":6.5579999999999998,"5min":199.422,"15min":199.422},"last":0.24199999999999999},{"interface":"front","average":{"1min":2.7959999999999998,"5min":5.1230000000000002,"15min":1.9410000000000001},"min":{"1min":0.25900000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":7.3520000000000003,"5min":199.91200000000001,"15min":199.91200000000001},"last":0.55900000000000005}]},{"osd":1,"last update":"Sun Oct 1 09:17:17 2023","interfaces":[{"interface":"back","average":{"1min":8.1899999999999995,"5min":3.7970000000000002,"15min":1.518},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":135.006,"5min":154.32599999999999,"15min":154.32599999999999},"last":0.94099999999999995},{"interface":"front","average":{"1min":7.4770000000000003,"5min":3.6699999999999999,"15min":1.4730000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":134.358,"5min":154.249,"15min":154.249},"last":0.29199999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:17:17 2023","interfaces":[{"interface":"back","average":{"1min":1.9419999999999999,"5min":1.2609999999999999,"15min":0.65500000000000003},"min":{"1min":0.311,"5min":0.224,"15min":0.224},"max":{"1min":5.016,"5min":30.771999999999998,"15min":30.771999999999998},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.954,"5min":1.323,"15min":0.71899999999999997},"min":{"1min":0.22700000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":11.869,"5min":31.878,"15min":31.878},"last":0.73399999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:17:17 2023","interfaces":[{"interface":"back","average":{"1min":4.6550000000000002,"5min":1.5580000000000001,"15min":0.89600000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":73.396000000000001,"5min":73.396000000000001,"15min":73.396000000000001},"last":0.70599999999999996},{"interface":"front","average":{"1min":5.7930000000000001,"5min":1.8799999999999999,"15min":1.036},"min":{"1min":0.32800000000000001,"5min":0.248,"15min":0.248},"max":{"1min":73.471000000000004,"5min":73.471000000000004,"15min":73.471000000000004},"last":0.53100000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:17:17 2023","interfaces":[{"interface":"back","average":{"1min":6.6609999999999996,"5min":2.194,"15min":1.135},"min":{"1min":0.373,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":76.893000000000001,"5min":76.893000000000001,"15min":76.893000000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":6.9340000000000002,"5min":2.1579999999999999,"15min":1.123},"min":{"1min":0.32100000000000001,"5min":0.317,"15min":0.317},"max":{"1min":76.959999999999994,"5min":76.959999999999994,"15min":76.959999999999994},"last":3.4039999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:17:17 2023","interfaces":[{"interface":"back","average":{"1min":3.3170000000000002,"5min":1.5800000000000001,"15min":0.94599999999999995},"min":{"1min":0.41799999999999998,"5min":0.318,"15min":0.318},"max":{"1min":10.27,"5min":10.27,"15min":10.27},"last":0.58799999999999997},{"interface":"front","average":{"1min":2.9060000000000001,"5min":1.4430000000000001,"15min":0.90600000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":9.3200000000000003,"5min":9.3200000000000003,"15min":9.3200000000000003},"last":0.47599999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:17:17 2023","interfaces":[{"interface":"back","average":{"1min":6.1550000000000002,"5min":2.0529999999999999,"15min":1.109},"min":{"1min":0.42399999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":79.412999999999997,"5min":79.412999999999997,"15min":79.412999999999997},"last":0.64300000000000002},{"interface":"front","average":{"1min":5.0380000000000003,"5min":1.8480000000000001,"15min":1.044},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":79.527000000000001,"5min":79.527000000000001,"15min":79.527000000000001},"last":0.75700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019095,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1504492,"kb_used_data":1203108,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92240660,"statfs":{"total":95995035648,"available":94454435840,"internally_reserved":0,"allocated":1231982592,"data_stored":1231466612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:17:07 2023","interfaces":[{"interface":"back","average":{"1min":9.4570000000000007,"5min":4.351,"15min":1.841},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":133.91499999999999,"5min":133.91499999999999,"15min":133.91499999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":8.7720000000000002,"5min":4.0860000000000003,"15min":1.7470000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":133.83199999999999,"5min":133.83199999999999,"15min":133.83199999999999},"last":0.48199999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:17:17 2023","interfaces":[{"interface":"back","average":{"1min":8.2479999999999993,"5min":2.6219999999999999,"15min":1.264},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":133.93600000000001,"5min":133.93600000000001,"15min":133.93600000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":10.343,"5min":3.0379999999999998,"15min":1.3759999999999999},"min":{"1min":0.39900000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":141.06,"5min":141.06,"15min":141.06},"last":0.44800000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:17:53 2023","interfaces":[{"interface":"back","average":{"1min":3.3719999999999999,"5min":5.1790000000000003,"15min":2.0800000000000001},"min":{"1min":0.307,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.112,"5min":133.96100000000001,"15min":133.96100000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.4910000000000001,"5min":4.8579999999999997,"15min":1.988},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":8.8520000000000003,"5min":134.018,"15min":134.018},"last":0.54700000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:17:22 2023","interfaces":[{"interface":"back","average":{"1min":8.0399999999999991,"5min":2.6030000000000002,"15min":1.2270000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":134.297,"5min":134.297,"15min":134.297},"last":1.6639999999999999},{"interface":"front","average":{"1min":10.569000000000001,"5min":3.012,"15min":1.3720000000000001},"min":{"1min":0.36199999999999999,"5min":0.316,"15min":0.316},"max":{"1min":141.089,"5min":141.089,"15min":141.089},"last":0.60899999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:17:17 2023","interfaces":[{"interface":"back","average":{"1min":10.84,"5min":3.04,"15min":1.3220000000000001},"min":{"1min":0.439,"5min":0.27000000000000002,"15min":0.26300000000000001},"max":{"1min":140.99100000000001,"5min":140.99100000000001,"15min":140.99100000000001},"last":8.6829999999999998},{"interface":"front","average":{"1min":8.3810000000000002,"5min":2.5859999999999999,"15min":1.1879999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":133.863,"5min":133.863,"15min":133.863},"last":1.698}]},{"osd":6,"last update":"Sun Oct 1 09:17:42 2023","interfaces":[{"interface":"back","average":{"1min":1.323,"5min":2.157,"15min":1.0609999999999999},"min":{"1min":0.35499999999999998,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":4.6349999999999998,"5min":133.90799999999999,"15min":133.90799999999999},"last":6.649},{"interface":"front","average":{"1min":2.6389999999999998,"5min":2.8809999999999998,"15min":1.335},"min":{"1min":0.38300000000000001,"5min":0.34200000000000003,"15min":0.33700000000000002},"max":{"1min":16.018999999999998,"5min":141.03,"15min":141.03},"last":3.1699999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:18:00 2023","interfaces":[{"interface":"back","average":{"1min":1.8600000000000001,"5min":2.8759999999999999,"15min":1.3180000000000001},"min":{"1min":0.49199999999999999,"5min":0.31,"15min":0.31},"max":{"1min":6.3330000000000002,"5min":133.946,"15min":133.946},"last":6.8129999999999997},{"interface":"front","average":{"1min":2.6890000000000001,"5min":2.774,"15min":1.3},"min":{"1min":0.40200000000000002,"5min":0.33700000000000002,"15min":0.31},"max":{"1min":9.0510000000000002,"5min":133.98400000000001,"15min":133.98400000000001},"last":0.45500000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822868,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1663328,"kb_used_data":1349720,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":92081824,"statfs":{"total":95995035648,"available":94291787776,"internally_reserved":0,"allocated":1382113280,"data_stored":1381605060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:17:10 2023","interfaces":[{"interface":"back","average":{"1min":5.0860000000000003,"5min":1.9379999999999999,"15min":1.0720000000000001},"min":{"1min":0.27100000000000002,"5min":0.217,"15min":0.217},"max":{"1min":60.792999999999999,"5min":60.792999999999999,"15min":60.792999999999999},"last":1.2210000000000001},{"interface":"front","average":{"1min":5.4829999999999997,"5min":2.105,"15min":1.107},"min":{"1min":0.27800000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":60.634,"5min":60.634,"15min":60.634},"last":0.57399999999999995}]},{"osd":1,"last update":"Sun Oct 1 09:17:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6589999999999998,"5min":1.8069999999999999,"15min":0.94999999999999996},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":23.745000000000001,"5min":32.420000000000002,"15min":32.420000000000002},"last":0.309},{"interface":"front","average":{"1min":3.3519999999999999,"5min":1.7529999999999999,"15min":0.94799999999999995},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":23.815999999999999,"5min":32.481999999999999,"15min":32.481999999999999},"last":0.59899999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:17:51 2023","interfaces":[{"interface":"back","average":{"1min":3.621,"5min":2.2200000000000002,"15min":1.1890000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":14.579000000000001,"5min":26.427,"15min":26.427},"last":0.25},{"interface":"front","average":{"1min":2.677,"5min":1.7549999999999999,"15min":1.0009999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.212,"5min":26.553000000000001,"15min":26.553000000000001},"last":0.68100000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:17:23 2023","interfaces":[{"interface":"back","average":{"1min":4.3559999999999999,"5min":1.855,"15min":1.1990000000000001},"min":{"1min":0.36399999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":24.309000000000001,"5min":24.309000000000001,"15min":24.309000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":3.5870000000000002,"5min":1.544,"15min":0.96499999999999997},"min":{"1min":0.36899999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":31.317,"5min":31.317,"15min":31.317},"last":0.628}]},{"osd":4,"last update":"Sun Oct 1 09:17:11 2023","interfaces":[{"interface":"back","average":{"1min":12.866,"5min":3.048,"15min":1.335},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":193.26300000000001,"5min":193.26300000000001,"15min":193.26300000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":2.8919999999999999,"5min":1.298,"15min":0.78800000000000003},"min":{"1min":0.33200000000000002,"5min":0.316,"15min":0.30299999999999999},"max":{"1min":9.6669999999999998,"5min":9.6669999999999998,"15min":9.6669999999999998},"last":0.33800000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:17:37 2023","interfaces":[{"interface":"back","average":{"1min":4.2709999999999999,"5min":1.764,"15min":0.94999999999999996},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28599999999999998},"max":{"1min":14.897,"5min":14.897,"15min":14.897},"last":0.71199999999999997},{"interface":"front","average":{"1min":4.0730000000000004,"5min":1.78,"15min":0.96699999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":14.547000000000001,"5min":14.547000000000001,"15min":14.547000000000001},"last":0.55700000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:18:00 2023","interfaces":[{"interface":"back","average":{"1min":3.8399999999999999,"5min":3.7759999999999998,"15min":1.647},"min":{"1min":0.34699999999999998,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":13.696999999999999,"5min":193.13800000000001,"15min":193.13800000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.2789999999999999,"5min":3.218,"15min":1.4610000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":10.074999999999999,"5min":193.203,"15min":193.203},"last":0.75800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1211891712,"data_stored":1211891712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":740425728,"data_stored":740425728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":931401728,"data_stored":931397656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":451936256,"data_stored":451936256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1213202432,"data_stored":1213202432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1362235392,"data_stored":1362231320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":615251968,"data_stored":615251968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":758976512,"data_stored":758972440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:18:11.592 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 1 2023-10-01T09:18:11.609 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:11.663 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:11.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:11 smithi138 ceph-mon[185886]: from='client.44825 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:18:11.702 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:11 smithi138 ceph-mon[192513]: from='client.44825 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:18:11.759 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:11.769 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:11 smithi190 ceph-mon[160435]: from='client.44825 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:18:11.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:11.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:11.871 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:11.938 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:12.035 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:12.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:12.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:12.154 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:12.179 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:12.249 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:12.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:12.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:12.356 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:12.417 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:12.638 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:12.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:12.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:12.672 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:12.680 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:12.749 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:12.796 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:12 smithi190 ceph-mon[160435]: pgmap v370: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 6759/108513 objects misplaced (6.229%); 659 KiB/s, 11 objects/s recovering 2023-10-01T09:18:12.798 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:12.806 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:12.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:12 smithi138 ceph-mon[185886]: pgmap v370: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 6759/108513 objects misplaced (6.229%); 659 KiB/s, 11 objects/s recovering 2023-10-01T09:18:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:12 smithi138 ceph-mon[192513]: pgmap v370: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 6759/108513 objects misplaced (6.229%); 659 KiB/s, 11 objects/s recovering 2023-10-01T09:18:12.945 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:12.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:12.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:13.064 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:13.114 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:13.473 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:13.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:13.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:13.581 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:13.672 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:14.000 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:14.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:14.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:14.104 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:14.158 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:14.518 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:18:14.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:14 smithi190 ceph-mon[160435]: pgmap v371: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.0 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 6065/100251 objects misplaced (6.050%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:14.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1146593699' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:14.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:14 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:14.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:14 smithi138 ceph-mon[185886]: pgmap v371: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.0 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 6065/100251 objects misplaced (6.050%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:14.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1146593699' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:14.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:14 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:14 smithi138 ceph-mon[192513]: pgmap v371: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.0 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 6065/100251 objects misplaced (6.050%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1146593699' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:14 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:15.230 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:18:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:18:14] "GET /metrics HTTP/1.1" 200 36419 "" "Prometheus/2.33.4" 2023-10-01T09:18:15.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:15 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:15.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:15 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:18:15.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:15 smithi138 ceph-mon[185886]: osdmap e145: 8 total, 8 up, 8 in 2023-10-01T09:18:15.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:15 smithi138 ceph-mon[185886]: failed to encode map e145 with expected crc 2023-10-01T09:18:15.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:15 smithi138 ceph-mon[185886]: failed to encode map e145 with expected crc 2023-10-01T09:18:15.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:15 smithi138 ceph-mon[185886]: failed to encode map e145 with expected crc 2023-10-01T09:18:15.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:15 smithi138 ceph-mon[185886]: failed to encode map e145 with expected crc 2023-10-01T09:18:15.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:15 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:15.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:15 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:18:15.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:15 smithi138 ceph-mon[192513]: osdmap e145: 8 total, 8 up, 8 in 2023-10-01T09:18:15.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:15 smithi138 ceph-mon[192513]: failed to encode map e145 with expected crc 2023-10-01T09:18:15.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:15 smithi138 ceph-mon[192513]: failed to encode map e145 with expected crc 2023-10-01T09:18:15.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:15 smithi138 ceph-mon[192513]: failed to encode map e145 with expected crc 2023-10-01T09:18:15.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:15 smithi138 ceph-mon[192513]: failed to encode map e145 with expected crc 2023-10-01T09:18:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:15 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:15 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:18:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:15 smithi190 ceph-mon[160435]: osdmap e145: 8 total, 8 up, 8 in 2023-10-01T09:18:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:15 smithi190 ceph-mon[160435]: failed to encode map e145 with expected crc 2023-10-01T09:18:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:15 smithi190 ceph-mon[160435]: failed to encode map e145 with expected crc 2023-10-01T09:18:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:15 smithi190 ceph-mon[160435]: failed to encode map e145 with expected crc 2023-10-01T09:18:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:15 smithi190 ceph-mon[160435]: failed to encode map e145 with expected crc 2023-10-01T09:18:16.746 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:16 smithi138 ceph-mon[192513]: failed to encode map e145 with expected crc 2023-10-01T09:18:16.747 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:16 smithi138 ceph-mon[192513]: pgmap v373: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.0 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 6065/100251 objects misplaced (6.050%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:16.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:16 smithi138 ceph-mon[185886]: failed to encode map e145 with expected crc 2023-10-01T09:18:16.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:16 smithi138 ceph-mon[185886]: pgmap v373: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.0 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 6065/100251 objects misplaced (6.050%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:16.773 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:16.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:16.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:16.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:16 smithi190 ceph-mon[160435]: failed to encode map e145 with expected crc 2023-10-01T09:18:16.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:16 smithi190 ceph-mon[160435]: pgmap v373: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 2.0 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 6065/100251 objects misplaced (6.050%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:16.879 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:16.973 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:17.264 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:17.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:17.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:17.374 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:17.457 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:17.464 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:18:17.465 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:18:17.465 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:18:17.465 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:18:17.465 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:18:17.875 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:17.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:17.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:17.984 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:18.061 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:18.292 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:18.294 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:18 smithi138 ceph-mon[192513]: pgmap v374: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.9 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 5566/90873 objects misplaced (6.125%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:18.294 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:18 smithi138 ceph-mon[185886]: pgmap v374: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.9 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 5566/90873 objects misplaced (6.125%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:18.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:18.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:18.315 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T09:18:18.396 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:18.458 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:18.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:18 smithi190 ceph-mon[160435]: pgmap v374: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.9 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 5566/90873 objects misplaced (6.125%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:18.721 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:18:19.067 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:19 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/525436253' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:18:19.067 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:19 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:18:19.067 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:19 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/699547266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:19.251 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:18:19.251 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:18:19.251 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:18:19.251 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:18:19.251 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:18:19.252 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:18:19.252 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:18:19.252 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:18:19.252 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:18:19.253 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:18:19.253 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:18:19.253 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:18:19.253 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:18:19.253 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:18:19.253 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:18:19.253 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:18:19.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/525436253' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:18:19.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:19 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:18:19.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/699547266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:19.413 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:18:19.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:19 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/525436253' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:18:19.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:19 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:18:19.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:19 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/699547266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:19.567 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:19.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:19.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:19.679 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:19.705 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:19.999 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T09:18:20.234 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:20 smithi138 ceph-mon[192513]: pgmap v375: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 5329/89520 objects misplaced (5.953%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:20.236 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:20 smithi138 ceph-mon[185886]: pgmap v375: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 5329/89520 objects misplaced (5.953%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:20.294 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:20.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:20.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:20.385 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:20.443 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:20.477 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:20 smithi190 ceph-mon[160435]: pgmap v375: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 5329/89520 objects misplaced (5.953%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:20.484 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:20.499 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:20.611 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:20.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:20.722 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:20.795 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:20.929 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:20.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:21.037 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:21.094 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:21.199 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:21.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:21.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:21.312 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:21.367 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:21.396 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:21 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:18:21.702 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:21 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:18:21.702 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:21 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:18:22.236 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:18:22.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:22 smithi138 ceph-mon[185886]: pgmap v376: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 5329/89520 objects misplaced (5.953%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:22.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:22.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:22.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:22 smithi138 ceph-mon[192513]: pgmap v376: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 5329/89520 objects misplaced (5.953%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:22.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:22.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:22.740 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:22.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:22.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:22.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:22 smithi190 ceph-mon[160435]: pgmap v376: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 5329/89520 objects misplaced (5.953%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:22.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:22.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:22.848 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:22.972 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:23.420 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:23.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:23.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:23.494 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:23.536 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:23.619 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:23.742 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:18:23.742 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:18:23.784 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:23 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:23.784 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:23 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:23.784 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:23 smithi138 ceph-mon[185886]: pgmap v377: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 4615/79104 objects misplaced (5.834%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:23.785 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:23 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:23.785 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:23 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:23.785 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:23 smithi138 ceph-mon[192513]: pgmap v377: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 4615/79104 objects misplaced (5.834%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:24.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:23 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:23 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:23 smithi190 ceph-mon[160435]: pgmap v377: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 4615/79104 objects misplaced (5.834%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:24.705 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:18:24.741 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2810503973' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:18:24.741 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2810503973' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:18:24.752 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:24.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:24.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:24.860 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:24.934 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2810503973' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:18:24.936 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:24.993 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:18:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:18:24] "GET /metrics HTTP/1.1" 200 36419 "" "Prometheus/2.33.4" 2023-10-01T09:18:25.001 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:18:25.001 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:18:25.001 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:18:25.002 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-01T09:18:25.073 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:25.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:25.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:25.204 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:25.284 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:25.518 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:25.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:25.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:25.634 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:25.738 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:25.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:25 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:25 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2810503973' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:18:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:25 smithi190 ceph-mon[160435]: osdmap e146: 8 total, 8 up, 8 in 2023-10-01T09:18:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:25 smithi190 ceph-mon[160435]: failed to encode map e146 with expected crc 2023-10-01T09:18:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:25 smithi190 ceph-mon[160435]: failed to encode map e146 with expected crc 2023-10-01T09:18:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:25 smithi190 ceph-mon[160435]: failed to encode map e146 with expected crc 2023-10-01T09:18:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:25 smithi190 ceph-mon[160435]: failed to encode map e146 with expected crc 2023-10-01T09:18:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:18:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:25 smithi190 ceph-mon[160435]: pgmap v379: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 4615/79104 objects misplaced (5.834%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2810503973' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:18:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[185886]: osdmap e146: 8 total, 8 up, 8 in 2023-10-01T09:18:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[185886]: failed to encode map e146 with expected crc 2023-10-01T09:18:26.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[185886]: failed to encode map e146 with expected crc 2023-10-01T09:18:26.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[185886]: failed to encode map e146 with expected crc 2023-10-01T09:18:26.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[185886]: failed to encode map e146 with expected crc 2023-10-01T09:18:26.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:18:26.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[185886]: pgmap v379: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 4615/79104 objects misplaced (5.834%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:26.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:26.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2810503973' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:18:26.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[192513]: osdmap e146: 8 total, 8 up, 8 in 2023-10-01T09:18:26.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[192513]: failed to encode map e146 with expected crc 2023-10-01T09:18:26.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[192513]: failed to encode map e146 with expected crc 2023-10-01T09:18:26.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[192513]: failed to encode map e146 with expected crc 2023-10-01T09:18:26.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[192513]: failed to encode map e146 with expected crc 2023-10-01T09:18:26.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:18:26.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:25 smithi138 ceph-mon[192513]: pgmap v379: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 4615/79104 objects misplaced (5.834%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:26.145 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:18:26.377 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:26.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:26.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:26.490 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:26.580 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:26.677 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:26.860 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:26.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:26.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:26.969 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:27.041 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:27.154 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:18:27.164 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:27.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:27.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:27.188 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:18:27.219 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:18:27.275 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:18:27.277 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:27.322 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:18:27.358 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:18:27.376 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:27.428 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:18:27.461 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:27.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:27.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:27.473 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:18:27.510 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:18:27.523 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:18:27.523 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:18:27.523 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:18:27.523 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:18:27.523 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:18:27.523 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:18:27.524 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:18:27.524 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:18:27.573 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:27.622 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:28.046 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:28.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:28.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:28.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2300490345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:28.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:28 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:28.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:28 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:28.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:28 smithi190 ceph-mon[160435]: pgmap v380: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 4164/70509 objects misplaced (5.906%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:28.152 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:28.182 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:28.224 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:28.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:28.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:28.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2300490345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:28.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:28 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:28.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:28 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:28.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:28 smithi138 ceph-mon[192513]: pgmap v380: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 4164/70509 objects misplaced (5.906%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:28.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2300490345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:28.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:28 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:28.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:28 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:28.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:28 smithi138 ceph-mon[185886]: pgmap v380: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 4164/70509 objects misplaced (5.906%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:28.327 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:28.392 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:18:28.429 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:28.498 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:28.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:28.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:28.602 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:28.666 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:29.183 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:29.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:29.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:29.290 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:29.351 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:29.402 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:29.458 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:29.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:29.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:29.566 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:29.610 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:29.712 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:29.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:29.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:29.816 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:29.870 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:30.079 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:30.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:30.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:30.189 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:30.268 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:30.380 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:30.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:30.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:30.410 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:30 smithi138 conmon[185862]: 2023-10-01T09:18:30.155+0000 7f70d64ec700 -1 mon.a@0(leader).osd e146 definitely_dead 0 2023-10-01T09:18:30.486 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:30.510 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:30 smithi190 ceph-mon[160435]: pgmap v381: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 3974/69309 objects misplaced (5.734%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:30.510 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:30 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3402600680' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:18:30.510 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:30 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:18:30.510 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:30 smithi190 ceph-mon[160435]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-01T09:18:30.512 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:30.548 INFO:teuthology.orchestra.run.smithi138.stderr:marked out osd.2. 2023-10-01T09:18:30.610 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:30.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:30.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:30.716 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:30.743 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:30 smithi138 ceph-mon[192513]: pgmap v381: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 3974/69309 objects misplaced (5.734%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:30.743 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:30 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3402600680' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:18:30.743 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:30 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:18:30.743 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:30 smithi138 ceph-mon[192513]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-01T09:18:30.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:30 smithi138 ceph-mon[185886]: pgmap v381: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 3974/69309 objects misplaced (5.734%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:18:30.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:30 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3402600680' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:18:30.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:30 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:18:30.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:30 smithi138 ceph-mon[185886]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-01T09:18:30.746 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:30.777 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:18:30.778 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:18:30.778 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:30.778 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:209442 5.a 5:5b9dfdca:::benchmark_data_smithi138_240338_object94305:head [delete] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-01T09:18:30.778 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:30.410390+0000", 2023-10-01T09:18:30.778 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.367271601, 2023-10-01T09:18:30.778 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.36730159099999998, 2023-10-01T09:18:30.778 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:30.778 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:18:30.778 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:30.778 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:30.778 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:30.779 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 209442 2023-10-01T09:18:30.779 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:30.779 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:30.779 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:30.779 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:30.779 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410390+0000", 2023-10-01T09:18:30.779 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:30.779 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:30.779 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:30.779 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:30.779 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410390+0000", 2023-10-01T09:18:30.780 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:18:30.780 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:30.780 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:30.780 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:30.780 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410388+0000", 2023-10-01T09:18:30.780 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3309999999999998e-06 2023-10-01T09:18:30.780 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:30.780 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:30.781 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:30.781 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410396+0000", 2023-10-01T09:18:30.781 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8128e-05 2023-10-01T09:18:30.781 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:30.781 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:30.781 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:30.781 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410414+0000", 2023-10-01T09:18:30.782 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5809999999999997e-06 2023-10-01T09:18:30.782 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:30.782 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:30.782 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:30.782 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410422+0000", 2023-10-01T09:18:30.782 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013988600000000001 2023-10-01T09:18:30.782 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:30.782 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:30.782 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:30.782 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410562+0000", 2023-10-01T09:18:30.782 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000141182 2023-10-01T09:18:30.783 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:30.783 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:30.783 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:30.783 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410703+0000", 2023-10-01T09:18:30.783 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016722300000000001 2023-10-01T09:18:30.783 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:30.783 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:30.783 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2,3", 2023-10-01T09:18:30.783 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410871+0000", 2023-10-01T09:18:30.783 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0069463420000000003 2023-10-01T09:18:30.783 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:30.783 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:30.783 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:30.784 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.417817+0000", 2023-10-01T09:18:30.784 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0036694050000000001 2023-10-01T09:18:30.784 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:30.784 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:30.784 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:30.784 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.421486+0000", 2023-10-01T09:18:30.784 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002669224 2023-10-01T09:18:30.784 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:30.784 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:30.784 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:30.784 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.424156+0000", 2023-10-01T09:18:30.784 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1552000000000001e-05 2023-10-01T09:18:30.784 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:30.785 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:30.785 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:18:30.785 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.424167+0000", 2023-10-01T09:18:30.785 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013777098 2023-10-01T09:18:30.785 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:30.785 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:30.785 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:30.785 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:30.785 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:18:30.785 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 1 2023-10-01T09:18:30.785 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:18:31.059 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:18:31.245 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:31.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:31.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:31.349 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:31.375 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:31.534 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:18:31.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:31 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-01T09:18:31.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:31 smithi138 ceph-mon[192513]: osdmap e147: 8 total, 8 up, 7 in 2023-10-01T09:18:31.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:31 smithi138 ceph-mon[192513]: failed to encode map e147 with expected crc 2023-10-01T09:18:31.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:31 smithi138 ceph-mon[192513]: failed to encode map e147 with expected crc 2023-10-01T09:18:31.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:31 smithi138 ceph-mon[192513]: failed to encode map e147 with expected crc 2023-10-01T09:18:31.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:31 smithi138 ceph-mon[192513]: failed to encode map e147 with expected crc 2023-10-01T09:18:31.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:31 smithi138 ceph-mon[192513]: osdmap e148: 8 total, 8 up, 7 in 2023-10-01T09:18:31.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:31 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-01T09:18:31.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:31 smithi138 ceph-mon[185886]: osdmap e147: 8 total, 8 up, 7 in 2023-10-01T09:18:31.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:31 smithi138 ceph-mon[185886]: failed to encode map e147 with expected crc 2023-10-01T09:18:31.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:31 smithi138 ceph-mon[185886]: failed to encode map e147 with expected crc 2023-10-01T09:18:31.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:31 smithi138 ceph-mon[185886]: failed to encode map e147 with expected crc 2023-10-01T09:18:31.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:31 smithi138 ceph-mon[185886]: failed to encode map e147 with expected crc 2023-10-01T09:18:31.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:31 smithi138 ceph-mon[185886]: osdmap e148: 8 total, 8 up, 7 in 2023-10-01T09:18:31.679 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:31.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:31.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:31.782 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:31.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:31 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-01T09:18:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:31 smithi190 ceph-mon[160435]: osdmap e147: 8 total, 8 up, 7 in 2023-10-01T09:18:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:31 smithi190 ceph-mon[160435]: failed to encode map e147 with expected crc 2023-10-01T09:18:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:31 smithi190 ceph-mon[160435]: failed to encode map e147 with expected crc 2023-10-01T09:18:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:31 smithi190 ceph-mon[160435]: failed to encode map e147 with expected crc 2023-10-01T09:18:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:31 smithi190 ceph-mon[160435]: failed to encode map e147 with expected crc 2023-10-01T09:18:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:31 smithi190 ceph-mon[160435]: osdmap e148: 8 total, 8 up, 7 in 2023-10-01T09:18:31.809 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:31.856 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:31.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:31.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:31.892 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:18:31.962 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:31.992 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:32.138 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:32.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:32.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:32.244 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:32.275 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:32.312 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:32.321 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:32.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:32.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:32.427 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:32 smithi190 ceph-mon[160435]: pgmap v383: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 408 op/s; 3974/69309 objects misplaced (5.734%); 712 KiB/s, 11 objects/s recovering 2023-10-01T09:18:32.427 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:32 smithi190 ceph-mon[160435]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.427 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:32 smithi190 ceph-mon[160435]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.427 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:32 smithi190 ceph-mon[160435]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.427 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:32 smithi190 ceph-mon[160435]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.427 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:32 smithi190 ceph-mon[160435]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.428 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:32 smithi190 ceph-mon[160435]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.428 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:32 smithi190 ceph-mon[160435]: osdmap e149: 8 total, 8 up, 7 in 2023-10-01T09:18:32.428 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:32.493 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[185886]: pgmap v383: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 408 op/s; 3974/69309 objects misplaced (5.734%); 712 KiB/s, 11 objects/s recovering 2023-10-01T09:18:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[185886]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[185886]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[185886]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[185886]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[185886]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[185886]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[185886]: osdmap e149: 8 total, 8 up, 7 in 2023-10-01T09:18:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[192513]: pgmap v383: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 408 op/s; 3974/69309 objects misplaced (5.734%); 712 KiB/s, 11 objects/s recovering 2023-10-01T09:18:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[192513]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[192513]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[192513]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[192513]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[192513]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[192513]: failed to encode map e148 with expected crc 2023-10-01T09:18:32.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:32 smithi138 ceph-mon[192513]: osdmap e149: 8 total, 8 up, 7 in 2023-10-01T09:18:32.578 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:32.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:32.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:32.682 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:32.710 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:32.909 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:32.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:32.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:33.014 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:33.072 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:33.198 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:33.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:33.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:33.307 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:33.361 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:33.482 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:18:33.496 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:33.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:33.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:33.603 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:33.659 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.659 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.660 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.660 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.660 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.660 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: 5.b starting backfill to osd.7 from (0'0,0'0] MAX to 146'13144 2023-10-01T09:18:33.660 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: 5.8 starting backfill to osd.5 from (0'0,0'0] MAX to 146'13428 2023-10-01T09:18:33.660 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.660 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 146'12833 2023-10-01T09:18:33.660 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: 5.d starting backfill to osd.0 from (0'0,0'0] MAX to 146'13072 2023-10-01T09:18:33.660 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: 5.6 starting backfill to osd.4 from (0'0,0'0] MAX to 146'13161 2023-10-01T09:18:33.660 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: 5.a starting backfill to osd.0 from (120'5868,146'12846] MIN to 146'12847 2023-10-01T09:18:33.660 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: 5.a starting backfill to osd.1 from (0'0,0'0] MAX to 146'12847 2023-10-01T09:18:33.661 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/862114484' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:33.661 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:33 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:33.663 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:33.731 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:18:33.731 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:18:33.731 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:18:33.731 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:18:33.732 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:18:33.780 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:33.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:33.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:33.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: 5.b starting backfill to osd.7 from (0'0,0'0] MAX to 146'13144 2023-10-01T09:18:33.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: 5.8 starting backfill to osd.5 from (0'0,0'0] MAX to 146'13428 2023-10-01T09:18:33.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 146'12833 2023-10-01T09:18:33.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: 5.d starting backfill to osd.0 from (0'0,0'0] MAX to 146'13072 2023-10-01T09:18:33.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: 5.6 starting backfill to osd.4 from (0'0,0'0] MAX to 146'13161 2023-10-01T09:18:33.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: 5.a starting backfill to osd.0 from (120'5868,146'12846] MIN to 146'12847 2023-10-01T09:18:33.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: 5.a starting backfill to osd.1 from (0'0,0'0] MAX to 146'12847 2023-10-01T09:18:33.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/862114484' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:33.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:33.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: 5.b starting backfill to osd.7 from (0'0,0'0] MAX to 146'13144 2023-10-01T09:18:33.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: 5.8 starting backfill to osd.5 from (0'0,0'0] MAX to 146'13428 2023-10-01T09:18:33.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: failed to encode map e149 with expected crc 2023-10-01T09:18:33.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 146'12833 2023-10-01T09:18:33.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: 5.d starting backfill to osd.0 from (0'0,0'0] MAX to 146'13072 2023-10-01T09:18:33.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: 5.6 starting backfill to osd.4 from (0'0,0'0] MAX to 146'13161 2023-10-01T09:18:33.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: 5.a starting backfill to osd.0 from (120'5868,146'12846] MIN to 146'12847 2023-10-01T09:18:33.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: 5.a starting backfill to osd.1 from (0'0,0'0] MAX to 146'12847 2023-10-01T09:18:33.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/862114484' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:33.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:33 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:33.888 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:33.942 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:34.746 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:34 smithi190 ceph-mon[160435]: pgmap v386: 89 pgs: 2 active+remapped+backfilling, 5 remapped+peering, 5 peering, 77 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 450 op/s; 2253/62403 objects misplaced (3.610%); 1.2 MiB/s, 20 objects/s recovering 2023-10-01T09:18:34.746 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:34 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:18:34.746 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:34 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:18:34.746 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:34 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:18:34.746 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:34 smithi190 ceph-mon[160435]: osdmap e150: 8 total, 8 up, 7 in 2023-10-01T09:18:34.746 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:34 smithi190 ceph-mon[160435]: failed to encode map e150 with expected crc 2023-10-01T09:18:34.746 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:34 smithi190 ceph-mon[160435]: failed to encode map e150 with expected crc 2023-10-01T09:18:34.746 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:34 smithi190 ceph-mon[160435]: failed to encode map e150 with expected crc 2023-10-01T09:18:34.746 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:34 smithi190 ceph-mon[160435]: failed to encode map e150 with expected crc 2023-10-01T09:18:34.746 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:34 smithi190 ceph-mon[160435]: failed to encode map e150 with expected crc 2023-10-01T09:18:34.781 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:18:34.785 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[192513]: pgmap v386: 89 pgs: 2 active+remapped+backfilling, 5 remapped+peering, 5 peering, 77 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 450 op/s; 2253/62403 objects misplaced (3.610%); 1.2 MiB/s, 20 objects/s recovering 2023-10-01T09:18:34.785 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:18:34.785 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:18:34.785 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:18:34.785 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[192513]: osdmap e150: 8 total, 8 up, 7 in 2023-10-01T09:18:34.785 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[192513]: failed to encode map e150 with expected crc 2023-10-01T09:18:34.786 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[192513]: failed to encode map e150 with expected crc 2023-10-01T09:18:34.786 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[192513]: failed to encode map e150 with expected crc 2023-10-01T09:18:34.786 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[192513]: failed to encode map e150 with expected crc 2023-10-01T09:18:34.786 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[192513]: failed to encode map e150 with expected crc 2023-10-01T09:18:34.787 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[185886]: pgmap v386: 89 pgs: 2 active+remapped+backfilling, 5 remapped+peering, 5 peering, 77 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 450 op/s; 2253/62403 objects misplaced (3.610%); 1.2 MiB/s, 20 objects/s recovering 2023-10-01T09:18:34.787 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:18:34.787 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:18:34.787 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:18:34.787 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[185886]: osdmap e150: 8 total, 8 up, 7 in 2023-10-01T09:18:34.787 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[185886]: failed to encode map e150 with expected crc 2023-10-01T09:18:34.787 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[185886]: failed to encode map e150 with expected crc 2023-10-01T09:18:34.787 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[185886]: failed to encode map e150 with expected crc 2023-10-01T09:18:34.787 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[185886]: failed to encode map e150 with expected crc 2023-10-01T09:18:34.787 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:34 smithi138 ceph-mon[185886]: failed to encode map e150 with expected crc 2023-10-01T09:18:35.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:18:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:18:34] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.33.4" 2023-10-01T09:18:35.164 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:35.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:35.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:35.276 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:35.348 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:35.405 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:18:35.461 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:35.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:35.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:35.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:35 smithi138 ceph-mon[185886]: osdmap e151: 8 total, 8 up, 7 in 2023-10-01T09:18:35.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:35 smithi138 ceph-mon[185886]: failed to encode map e151 with expected crc 2023-10-01T09:18:35.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:35 smithi138 ceph-mon[185886]: failed to encode map e151 with expected crc 2023-10-01T09:18:35.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:35 smithi138 ceph-mon[185886]: failed to encode map e151 with expected crc 2023-10-01T09:18:35.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:35 smithi138 ceph-mon[185886]: failed to encode map e151 with expected crc 2023-10-01T09:18:35.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:35 smithi138 ceph-mon[192513]: osdmap e151: 8 total, 8 up, 7 in 2023-10-01T09:18:35.481 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:35 smithi138 ceph-mon[192513]: failed to encode map e151 with expected crc 2023-10-01T09:18:35.481 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:35 smithi138 ceph-mon[192513]: failed to encode map e151 with expected crc 2023-10-01T09:18:35.481 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:35 smithi138 ceph-mon[192513]: failed to encode map e151 with expected crc 2023-10-01T09:18:35.481 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:35 smithi138 ceph-mon[192513]: failed to encode map e151 with expected crc 2023-10-01T09:18:35.580 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:35.638 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:35 smithi190 ceph-mon[160435]: osdmap e151: 8 total, 8 up, 7 in 2023-10-01T09:18:35.638 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:35 smithi190 ceph-mon[160435]: failed to encode map e151 with expected crc 2023-10-01T09:18:35.638 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:35 smithi190 ceph-mon[160435]: failed to encode map e151 with expected crc 2023-10-01T09:18:35.639 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:35 smithi190 ceph-mon[160435]: failed to encode map e151 with expected crc 2023-10-01T09:18:35.639 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:35 smithi190 ceph-mon[160435]: failed to encode map e151 with expected crc 2023-10-01T09:18:35.641 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:35.749 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:35.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:35.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:35.868 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:35.952 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:36.096 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:36.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:36.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:36.203 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:36.281 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:36.285 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:18:36.321 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:18:36.353 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:18:36.390 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:18:36.391 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:36.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:36.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:36.432 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:18:36.466 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:18:36.493 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:36 smithi138 ceph-mon[185886]: pgmap v389: 89 pgs: 2 active+remapped+backfilling, 5 remapped+peering, 5 peering, 77 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2253/62403 objects misplaced (3.610%); 1.8 MiB/s, 31 objects/s recovering 2023-10-01T09:18:36.493 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:36.493 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:36.493 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:36 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1850525236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:36.493 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:18:36.493 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:18:36.493 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:36.500 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:36.501 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:18:36.550 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:18:36.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:36 smithi190 ceph-mon[160435]: pgmap v389: 89 pgs: 2 active+remapped+backfilling, 5 remapped+peering, 5 peering, 77 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2253/62403 objects misplaced (3.610%); 1.8 MiB/s, 31 objects/s recovering 2023-10-01T09:18:36.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:36.562 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:36.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:36 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1850525236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:36.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:18:36.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:18:36.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:36.568 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:36.569 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:18:36.569 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:18:36.569 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:18:36.569 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:18:36.569 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:18:36.569 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:18:36.569 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:18:36.569 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:18:36.728 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:36.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:36.738 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:36 smithi138 ceph-mon[192513]: pgmap v389: 89 pgs: 2 active+remapped+backfilling, 5 remapped+peering, 5 peering, 77 active+clean; 1.3 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 2253/62403 objects misplaced (3.610%); 1.8 MiB/s, 31 objects/s recovering 2023-10-01T09:18:36.738 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:36 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:36.738 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:36 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:36.738 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:36 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1850525236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:36.739 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:36 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:18:36.739 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:36 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:18:36.739 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:36 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:36.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:36.840 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:36.909 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:36.984 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:36.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:36.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:37.083 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:18:37.083 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:18:37.083 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-01T09:18:37.083 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-01T09:18:37.091 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:37.193 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:37.311 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:37.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:37.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:37.421 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:37.475 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:37.606 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:18:37.606 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:18:37.766 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:18:37.940 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:37.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:37.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:38.052 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:38.087 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:38.138 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:38.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:38.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:38.171 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:38.242 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:38.290 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:38.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:38 smithi138 ceph-mon[185886]: pgmap v390: 89 pgs: 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 81 active+clean; 1.1 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 9698/55899 objects misplaced (17.349%); 5.7 MiB/s, 1 keys/s, 96 objects/s recovering 2023-10-01T09:18:38.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:38 smithi138 ceph-mon[192513]: pgmap v390: 89 pgs: 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 81 active+clean; 1.1 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 9698/55899 objects misplaced (17.349%); 5.7 MiB/s, 1 keys/s, 96 objects/s recovering 2023-10-01T09:18:38.443 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:38.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:38.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:38.549 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:38 smithi190 ceph-mon[160435]: pgmap v390: 89 pgs: 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 81 active+clean; 1.1 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 663 op/s; 9698/55899 objects misplaced (17.349%); 5.7 MiB/s, 1 keys/s, 96 objects/s recovering 2023-10-01T09:18:38.554 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:38.588 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:18:38.628 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:38.688 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:38.751 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:38.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:38.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:38.866 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:38.920 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:39.382 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:18:39.824 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:39.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:39.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:39.927 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:39.955 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:40.002 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:40.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:40.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:40.067 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:18:40.106 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:40.136 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:40.191 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:40.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:40.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:40.294 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:40.323 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:40.372 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:40.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:40.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:40.476 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:40.510 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:40 smithi190 ceph-mon[160435]: osdmap e152: 8 total, 8 up, 7 in 2023-10-01T09:18:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:40 smithi190 ceph-mon[160435]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:40 smithi190 ceph-mon[160435]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:40 smithi190 ceph-mon[160435]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:40 smithi190 ceph-mon[160435]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:40 smithi190 ceph-mon[160435]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:40 smithi190 ceph-mon[160435]: pgmap v392: 89 pgs: 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 81 active+clean; 1.1 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 8653/54633 objects misplaced (15.838%); 14 MiB/s, 1 keys/s, 227 objects/s recovering 2023-10-01T09:18:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:18:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[192513]: osdmap e152: 8 total, 8 up, 7 in 2023-10-01T09:18:40.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[192513]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[192513]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[192513]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[192513]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[192513]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[192513]: pgmap v392: 89 pgs: 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 81 active+clean; 1.1 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 8653/54633 objects misplaced (15.838%); 14 MiB/s, 1 keys/s, 227 objects/s recovering 2023-10-01T09:18:40.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:40.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:18:40.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[185886]: osdmap e152: 8 total, 8 up, 7 in 2023-10-01T09:18:40.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[185886]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[185886]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[185886]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[185886]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[185886]: failed to encode map e152 with expected crc 2023-10-01T09:18:40.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[185886]: pgmap v392: 89 pgs: 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 81 active+clean; 1.1 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 8653/54633 objects misplaced (15.838%); 14 MiB/s, 1 keys/s, 227 objects/s recovering 2023-10-01T09:18:40.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:40.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:18:41.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:41 smithi138 conmon[185862]: 2023-10-01T09:18:41.118+0000 7f70d64ec700 -1 mon.a@0(leader).osd e153 definitely_dead 0 2023-10-01T09:18:41.393 INFO:teuthology.orchestra.run.smithi138.stderr:marked in osd.2. 2023-10-01T09:18:41.471 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:18:41.471 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:18:41.491 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:41 smithi190 ceph-mon[160435]: failed to encode map e152 with expected crc 2023-10-01T09:18:41.492 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:41 smithi190 ceph-mon[160435]: failed to encode map e153 with expected crc 2023-10-01T09:18:41.492 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:41 smithi190 ceph-mon[160435]: osdmap e153: 8 total, 8 up, 7 in 2023-10-01T09:18:41.492 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:41 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2784838045' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:18:41.492 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:41 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:18:41.507 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:41.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:41.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:41.612 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:41.651 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:41.700 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:41.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:41.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:41.804 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:41.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:41 smithi138 ceph-mon[192513]: failed to encode map e152 with expected crc 2023-10-01T09:18:41.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:41 smithi138 ceph-mon[192513]: failed to encode map e153 with expected crc 2023-10-01T09:18:41.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:41 smithi138 ceph-mon[192513]: osdmap e153: 8 total, 8 up, 7 in 2023-10-01T09:18:41.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:41 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2784838045' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:18:41.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:41 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:18:41.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:41 smithi138 ceph-mon[185886]: failed to encode map e152 with expected crc 2023-10-01T09:18:41.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:41 smithi138 ceph-mon[185886]: failed to encode map e153 with expected crc 2023-10-01T09:18:41.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:41 smithi138 ceph-mon[185886]: osdmap e153: 8 total, 8 up, 7 in 2023-10-01T09:18:41.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:41 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2784838045' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:18:41.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:41 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:18:41.828 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:41.870 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:41.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:41.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:41.974 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:41.999 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:42.049 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:42.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:42.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:42.156 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:42.188 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:42.291 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T09:18:42.454 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:42.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e153 with expected crc 2023-10-01T09:18:42.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e153 with expected crc 2023-10-01T09:18:42.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: pgmap v395: 89 pgs: 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 81 active+clean; 1.1 GiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 8653/54633 objects misplaced (15.838%); 14 MiB/s, 1 keys/s, 227 objects/s recovering 2023-10-01T09:18:42.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-01T09:18:42.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: osdmap e154: 8 total, 8 up, 8 in 2023-10-01T09:18:42.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.457 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.457 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.457 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.457 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.457 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.457 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.457 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.457 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.457 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.457 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: osdmap e155: 8 total, 8 up, 8 in 2023-10-01T09:18:42.458 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.458 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.458 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.458 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[192513]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.458 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e153 with expected crc 2023-10-01T09:18:42.458 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e153 with expected crc 2023-10-01T09:18:42.461 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: pgmap v395: 89 pgs: 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 81 active+clean; 1.1 GiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 8653/54633 objects misplaced (15.838%); 14 MiB/s, 1 keys/s, 227 objects/s recovering 2023-10-01T09:18:42.461 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-01T09:18:42.461 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: osdmap e154: 8 total, 8 up, 8 in 2023-10-01T09:18:42.461 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.462 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.462 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.462 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.462 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.462 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.462 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.462 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.462 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.462 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.462 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.463 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: osdmap e155: 8 total, 8 up, 8 in 2023-10-01T09:18:42.463 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.463 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.463 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.463 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:42 smithi138 ceph-mon[185886]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:42.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:42.553 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-01T09:18:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e153 with expected crc 2023-10-01T09:18:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e153 with expected crc 2023-10-01T09:18:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: pgmap v395: 89 pgs: 5 active+remapped+backfill_wait, 3 active+remapped+backfilling, 81 active+clean; 1.1 GiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 431 op/s; 8653/54633 objects misplaced (15.838%); 14 MiB/s, 1 keys/s, 227 objects/s recovering 2023-10-01T09:18:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-01T09:18:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: osdmap e154: 8 total, 8 up, 8 in 2023-10-01T09:18:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e154 with expected crc 2023-10-01T09:18:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: osdmap e155: 8 total, 8 up, 8 in 2023-10-01T09:18:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:42 smithi190 ceph-mon[160435]: failed to encode map e155 with expected crc 2023-10-01T09:18:42.565 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:42.585 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:42.591 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:42.636 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:42.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:42.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:42.700 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:18:42.743 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:42.773 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:42.874 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:42.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:42.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:42.978 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:43.005 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:43.060 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:18:43.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:43 smithi190 ceph-mon[160435]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:43 smithi190 ceph-mon[160435]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:43 smithi190 ceph-mon[160435]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:43 smithi190 ceph-mon[160435]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:43 smithi190 ceph-mon[160435]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:43 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1965297232' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:18:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:43 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:43 smithi190 ceph-mon[160435]: failed to encode map e156 with expected crc 2023-10-01T09:18:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:43 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1965297232' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:18:43.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:43 smithi190 ceph-mon[160435]: osdmap e156: 8 total, 8 up, 8 in 2023-10-01T09:18:43.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:43 smithi190 ceph-mon[160435]: failed to encode map e156 with expected crc 2023-10-01T09:18:43.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:43 smithi190 ceph-mon[160435]: 5.a continuing backfill to osd.0 from (121'6381,153'13131] MIN to 153'13131 2023-10-01T09:18:43.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[185886]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[185886]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[185886]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[185886]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[185886]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1965297232' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:18:43.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:43.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[185886]: failed to encode map e156 with expected crc 2023-10-01T09:18:43.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1965297232' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:18:43.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[185886]: osdmap e156: 8 total, 8 up, 8 in 2023-10-01T09:18:43.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[185886]: failed to encode map e156 with expected crc 2023-10-01T09:18:43.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[185886]: 5.a continuing backfill to osd.0 from (121'6381,153'13131] MIN to 153'13131 2023-10-01T09:18:43.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[192513]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[192513]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[192513]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[192513]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[192513]: failed to encode map e155 with expected crc 2023-10-01T09:18:43.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1965297232' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:18:43.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:43.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[192513]: failed to encode map e156 with expected crc 2023-10-01T09:18:43.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1965297232' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:18:43.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[192513]: osdmap e156: 8 total, 8 up, 8 in 2023-10-01T09:18:43.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[192513]: failed to encode map e156 with expected crc 2023-10-01T09:18:43.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:43 smithi138 ceph-mon[192513]: 5.a continuing backfill to osd.0 from (121'6381,153'13131] MIN to 153'13131 2023-10-01T09:18:44.531 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:18:44.532 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:18:44.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:44 smithi190 ceph-mon[160435]: failed to encode map e156 with expected crc 2023-10-01T09:18:44.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:44 smithi190 ceph-mon[160435]: failed to encode map e156 with expected crc 2023-10-01T09:18:44.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:44 smithi190 ceph-mon[160435]: pgmap v398: 89 pgs: 2 remapped+peering, 7 peering, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 77 active+clean; 987 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 2613/47244 objects misplaced (5.531%); 15 MiB/s, 246 objects/s recovering 2023-10-01T09:18:44.634 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:44.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:44.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:44.741 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:44.778 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:44 smithi138 ceph-mon[192513]: failed to encode map e156 with expected crc 2023-10-01T09:18:44.778 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:44 smithi138 ceph-mon[192513]: failed to encode map e156 with expected crc 2023-10-01T09:18:44.778 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:44 smithi138 ceph-mon[192513]: pgmap v398: 89 pgs: 2 remapped+peering, 7 peering, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 77 active+clean; 987 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 2613/47244 objects misplaced (5.531%); 15 MiB/s, 246 objects/s recovering 2023-10-01T09:18:44.779 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:44 smithi138 ceph-mon[185886]: failed to encode map e156 with expected crc 2023-10-01T09:18:44.779 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:44 smithi138 ceph-mon[185886]: failed to encode map e156 with expected crc 2023-10-01T09:18:44.779 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:44 smithi138 ceph-mon[185886]: pgmap v398: 89 pgs: 2 remapped+peering, 7 peering, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 77 active+clean; 987 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 2613/47244 objects misplaced (5.531%); 15 MiB/s, 246 objects/s recovering 2023-10-01T09:18:44.797 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:44.844 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:44.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:44.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:44.947 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:44.976 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:45.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:18:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:18:44] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.33.4" 2023-10-01T09:18:45.070 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:45.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:45.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:45.174 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:18:45.182 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:18:45.219 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:18:45.230 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:45.243 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:18:45.293 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:18:45.325 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:18:45.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:45.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:18:45.337 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:18:45.377 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:18:45.383 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T09:18:45.406 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:18:45.434 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:45.453 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:18:45.462 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:18:45.462 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:18:45.462 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:18:45.462 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:18:45.463 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:18:45.463 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:18:45.463 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:18:45.463 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:18:45.541 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:45.640 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:45 smithi138 ceph-mon[185886]: osdmap e157: 8 total, 8 up, 8 in 2023-10-01T09:18:45.640 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:45 smithi138 ceph-mon[185886]: failed to encode map e157 with expected crc 2023-10-01T09:18:45.640 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:45 smithi138 ceph-mon[185886]: failed to encode map e157 with expected crc 2023-10-01T09:18:45.640 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:45 smithi138 ceph-mon[185886]: failed to encode map e157 with expected crc 2023-10-01T09:18:45.640 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:45 smithi138 ceph-mon[185886]: failed to encode map e157 with expected crc 2023-10-01T09:18:45.640 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:45 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1574937849' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:45.641 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:45 smithi138 ceph-mon[192513]: osdmap e157: 8 total, 8 up, 8 in 2023-10-01T09:18:45.642 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:45 smithi138 ceph-mon[192513]: failed to encode map e157 with expected crc 2023-10-01T09:18:45.642 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:45 smithi138 ceph-mon[192513]: failed to encode map e157 with expected crc 2023-10-01T09:18:45.642 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:45 smithi138 ceph-mon[192513]: failed to encode map e157 with expected crc 2023-10-01T09:18:45.642 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:45 smithi138 ceph-mon[192513]: failed to encode map e157 with expected crc 2023-10-01T09:18:45.642 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:45 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1574937849' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:45.694 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:45.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:45.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:45.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:45 smithi190 ceph-mon[160435]: osdmap e157: 8 total, 8 up, 8 in 2023-10-01T09:18:45.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:45 smithi190 ceph-mon[160435]: failed to encode map e157 with expected crc 2023-10-01T09:18:45.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:45 smithi190 ceph-mon[160435]: failed to encode map e157 with expected crc 2023-10-01T09:18:45.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:45 smithi190 ceph-mon[160435]: failed to encode map e157 with expected crc 2023-10-01T09:18:45.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:45 smithi190 ceph-mon[160435]: failed to encode map e157 with expected crc 2023-10-01T09:18:45.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:45 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1574937849' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:45.812 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:45.889 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:45.972 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:45.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:45.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:46.078 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:46.122 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:18:46.171 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:46.277 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:46.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:46.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:46.385 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:46.423 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:46.526 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:46.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:46.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:46.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:46 smithi190 ceph-mon[160435]: pgmap v400: 89 pgs: 2 remapped+peering, 7 peering, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 77 active+clean; 987 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 2613/47244 objects misplaced (5.531%); 15 MiB/s, 246 objects/s recovering 2023-10-01T09:18:46.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:46 smithi190 ceph-mon[160435]: osdmap e158: 8 total, 8 up, 8 in 2023-10-01T09:18:46.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:46 smithi190 ceph-mon[160435]: failed to encode map e158 with expected crc 2023-10-01T09:18:46.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:46 smithi190 ceph-mon[160435]: failed to encode map e158 with expected crc 2023-10-01T09:18:46.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:46 smithi190 ceph-mon[160435]: failed to encode map e158 with expected crc 2023-10-01T09:18:46.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:46 smithi190 ceph-mon[160435]: failed to encode map e158 with expected crc 2023-10-01T09:18:46.626 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:18:46.633 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:46.670 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:46 smithi138 ceph-mon[192513]: pgmap v400: 89 pgs: 2 remapped+peering, 7 peering, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 77 active+clean; 987 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 2613/47244 objects misplaced (5.531%); 15 MiB/s, 246 objects/s recovering 2023-10-01T09:18:46.671 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:46 smithi138 ceph-mon[192513]: osdmap e158: 8 total, 8 up, 8 in 2023-10-01T09:18:46.671 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:46 smithi138 ceph-mon[192513]: failed to encode map e158 with expected crc 2023-10-01T09:18:46.671 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:46 smithi138 ceph-mon[192513]: failed to encode map e158 with expected crc 2023-10-01T09:18:46.671 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:46 smithi138 ceph-mon[192513]: failed to encode map e158 with expected crc 2023-10-01T09:18:46.671 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:46 smithi138 ceph-mon[192513]: failed to encode map e158 with expected crc 2023-10-01T09:18:46.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:46 smithi138 ceph-mon[185886]: pgmap v400: 89 pgs: 2 remapped+peering, 7 peering, 2 active+remapped+backfill_wait, 1 active+remapped+backfilling, 77 active+clean; 987 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 2613/47244 objects misplaced (5.531%); 15 MiB/s, 246 objects/s recovering 2023-10-01T09:18:46.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:46 smithi138 ceph-mon[185886]: osdmap e158: 8 total, 8 up, 8 in 2023-10-01T09:18:46.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:46 smithi138 ceph-mon[185886]: failed to encode map e158 with expected crc 2023-10-01T09:18:46.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:46 smithi138 ceph-mon[185886]: failed to encode map e158 with expected crc 2023-10-01T09:18:46.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:46 smithi138 ceph-mon[185886]: failed to encode map e158 with expected crc 2023-10-01T09:18:46.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:46 smithi138 ceph-mon[185886]: failed to encode map e158 with expected crc 2023-10-01T09:18:46.717 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:46.958 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:46.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:46.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:47.077 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:47.134 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:47.178 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:47.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:47.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:47.182 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:47.282 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:47.325 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:47.493 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:47.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:47.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:47.554 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:18:47.554 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:18:47.554 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T09:18:47.554 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2023-10-01T09:18:47.554 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 0 2023-10-01T09:18:47.609 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:47.688 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:47.794 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:47.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:47.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:47.905 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:47.994 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:48.074 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:48.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:48.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:48.178 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:48.206 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:48.252 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:48.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:48.257 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:48 smithi190 ceph-mon[160435]: pgmap v402: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 87 active+clean; 856 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 1308/40950 objects misplaced (3.194%); 11 MiB/s, 1 keys/s, 178 objects/s recovering 2023-10-01T09:18:48.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:48.267 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:18:48.267 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:18:48.267 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:18:48.267 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:18:48.267 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.267 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:59034 5.7 5:e096144b:::benchmark_data_smithi138_240338_object59033:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:18:48.267 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:15:16.277083+0000", 2023-10-01T09:18:48.267 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 211.98060377600001, 2023-10-01T09:18:48.267 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22331911600000001, 2023-10-01T09:18:48.268 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.268 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.268 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.268 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.268 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.268 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 59034 2023-10-01T09:18:48.268 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.268 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.268 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.268 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.268 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277083+0000", 2023-10-01T09:18:48.268 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.269 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.269 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277083+0000", 2023-10-01T09:18:48.269 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:18:48.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.269 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.269 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277082+0000", 2023-10-01T09:18:48.269 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5989999999999998e-05 2023-10-01T09:18:48.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.270 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.270 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277138+0000", 2023-10-01T09:18:48.270 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6120000000000005e-06 2023-10-01T09:18:48.270 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.270 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.270 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277148+0000", 2023-10-01T09:18:48.270 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3259999999999997e-06 2023-10-01T09:18:48.270 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.270 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277152+0000", 2023-10-01T09:18:48.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3316999999999998e-05 2023-10-01T09:18:48.271 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.271 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277206+0000", 2023-10-01T09:18:48.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014156800000000001 2023-10-01T09:18:48.271 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.271 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277347+0000", 2023-10-01T09:18:48.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000183528 2023-10-01T09:18:48.272 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.272 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.272 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:18:48.272 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277531+0000", 2023-10-01T09:18:48.272 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0046728500000000001 2023-10-01T09:18:48.272 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.272 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.272 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.272 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.282203+0000", 2023-10-01T09:18:48.272 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017041100000000001 2023-10-01T09:18:48.272 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.273 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.273 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.282374+0000", 2023-10-01T09:18:48.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.217918533 2023-10-01T09:18:48.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.273 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.273 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.500292+0000", 2023-10-01T09:18:48.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7150999999999995e-05 2023-10-01T09:18:48.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.273 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.500360+0000", 2023-10-01T09:18:48.274 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2923999999999998e-05 2023-10-01T09:18:48.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.500403+0000", 2023-10-01T09:18:48.274 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22331911600000001 2023-10-01T09:18:48.274 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.274 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.274 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.275 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.275 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:59039 5.7 5:e088c7a4:::benchmark_data_smithi138_240338_object59038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:18:48.275 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:15:16.277173+0000", 2023-10-01T09:18:48.275 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 211.98051386200001, 2023-10-01T09:18:48.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22522009000000001, 2023-10-01T09:18:48.275 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.275 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.275 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.275 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.275 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.275 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 59039 2023-10-01T09:18:48.276 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.276 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.276 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.276 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.276 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277173+0000", 2023-10-01T09:18:48.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.276 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.276 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.276 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.276 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277173+0000", 2023-10-01T09:18:48.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:18:48.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.277 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.277 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.277 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277173+0000", 2023-10-01T09:18:48.277 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5589000000000001e-05 2023-10-01T09:18:48.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.277 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.277 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.277 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277208+0000", 2023-10-01T09:18:48.277 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9999999999999996e-06 2023-10-01T09:18:48.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.278 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.278 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.278 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277216+0000", 2023-10-01T09:18:48.278 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6299999999999997e-06 2023-10-01T09:18:48.278 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.278 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.278 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.278 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277221+0000", 2023-10-01T09:18:48.278 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0023609479999999999 2023-10-01T09:18:48.278 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.278 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.279 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.279582+0000", 2023-10-01T09:18:48.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012953199999999999 2023-10-01T09:18:48.279 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.279 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.279 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.279712+0000", 2023-10-01T09:18:48.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000162285 2023-10-01T09:18:48.279 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.279 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:18:48.279 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.279874+0000", 2023-10-01T09:18:48.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0036032429999999999 2023-10-01T09:18:48.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.283477+0000", 2023-10-01T09:18:48.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3853000000000003e-05 2023-10-01T09:18:48.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.283511+0000", 2023-10-01T09:18:48.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.218782168 2023-10-01T09:18:48.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.502293+0000", 2023-10-01T09:18:48.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9048000000000001e-05 2023-10-01T09:18:48.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.502352+0000", 2023-10-01T09:18:48.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1251999999999997e-05 2023-10-01T09:18:48.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.502393+0000", 2023-10-01T09:18:48.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22522009000000001 2023-10-01T09:18:48.282 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.282 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.282 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.282 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.282 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:104858 5.7 5:e2ba0b8c:::benchmark_data_smithi138_240338_object104857:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-01T09:18:48.282 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:01.525198+0000", 2023-10-01T09:18:48.282 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 166.73248919900001, 2023-10-01T09:18:48.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.268526457, 2023-10-01T09:18:48.282 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.282 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.283 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.283 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.283 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.283 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 104858 2023-10-01T09:18:48.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.283 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.283 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.283 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.525198+0000", 2023-10-01T09:18:48.283 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.525198+0000", 2023-10-01T09:18:48.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:18:48.284 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.525196+0000", 2023-10-01T09:18:48.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7395000000000002e-05 2023-10-01T09:18:48.284 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.285 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.285 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.525254+0000", 2023-10-01T09:18:48.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0685e-05 2023-10-01T09:18:48.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.286 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.286 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.286 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.525265+0000", 2023-10-01T09:18:48.286 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7049999999999996e-06 2023-10-01T09:18:48.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.287 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.287 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.525270+0000", 2023-10-01T09:18:48.287 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0051863339999999999 2023-10-01T09:18:48.287 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.287 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.287 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.530457+0000", 2023-10-01T09:18:48.287 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000136816 2023-10-01T09:18:48.287 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.288 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.288 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.530593+0000", 2023-10-01T09:18:48.288 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014115199999999999 2023-10-01T09:18:48.288 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.288 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:18:48.288 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.530735+0000", 2023-10-01T09:18:48.288 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003424552 2023-10-01T09:18:48.288 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.288 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.289 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.534159+0000", 2023-10-01T09:18:48.289 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.048096319999999998 2023-10-01T09:18:48.289 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.289 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.289 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.289 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.582255+0000", 2023-10-01T09:18:48.289 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.211366785 2023-10-01T09:18:48.289 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.289 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.289 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.793622+0000", 2023-10-01T09:18:48.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8789000000000002e-05 2023-10-01T09:18:48.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.793681+0000", 2023-10-01T09:18:48.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3476000000000002e-05 2023-10-01T09:18:48.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:01.793724+0000", 2023-10-01T09:18:48.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.268526457 2023-10-01T09:18:48.291 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.291 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.291 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.291 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.291 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.291 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:107106 5.7 5:ea0f488f:::benchmark_data_smithi138_240338_object107105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e121)", 2023-10-01T09:18:48.291 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:04.528562+0000", 2023-10-01T09:18:48.291 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 163.72912506099999, 2023-10-01T09:18:48.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21855391800000001, 2023-10-01T09:18:48.292 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.292 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.292 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.292 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.292 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.292 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 107106 2023-10-01T09:18:48.292 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.292 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.292 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.292 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.292 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528562+0000", 2023-10-01T09:18:48.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.293 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.293 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.293 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528562+0000", 2023-10-01T09:18:48.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:18:48.293 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.293 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.293 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528561+0000", 2023-10-01T09:18:48.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.32e-05 2023-10-01T09:18:48.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.294 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.294 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528604+0000", 2023-10-01T09:18:48.294 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4414e-05 2023-10-01T09:18:48.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.294 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.294 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528618+0000", 2023-10-01T09:18:48.294 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.353e-06 2023-10-01T09:18:48.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.295 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528625+0000", 2023-10-01T09:18:48.295 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7601e-05 2023-10-01T09:18:48.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.295 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.295 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528672+0000", 2023-10-01T09:18:48.295 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5357000000000002e-05 2023-10-01T09:18:48.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.295 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528768+0000", 2023-10-01T09:18:48.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010036500000000001 2023-10-01T09:18:48.296 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:18:48.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.528868+0000", 2023-10-01T09:18:48.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038284339999999999 2023-10-01T09:18:48.296 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.532696+0000", 2023-10-01T09:18:48.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0018256360000000001 2023-10-01T09:18:48.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.297 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.297 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.534522+0000", 2023-10-01T09:18:48.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.212494879 2023-10-01T09:18:48.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.297 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.297 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.747017+0000", 2023-10-01T09:18:48.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5136999999999997e-05 2023-10-01T09:18:48.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.298 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.298 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.298 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.747072+0000", 2023-10-01T09:18:48.298 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4032999999999997e-05 2023-10-01T09:18:48.298 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.298 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.298 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.298 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:04.747116+0000", 2023-10-01T09:18:48.298 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21855391800000001 2023-10-01T09:18:48.298 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.298 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.298 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.299 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.299 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.299 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:110043 5.7 5:e22f4a5a:::benchmark_data_smithi138_240338_object110042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-01T09:18:48.299 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:08.040537+0000", 2023-10-01T09:18:48.299 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 160.217150159, 2023-10-01T09:18:48.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22794239899999999, 2023-10-01T09:18:48.299 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.299 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.299 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.299 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.299 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.300 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 110043 2023-10-01T09:18:48.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.300 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.300 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.300 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.040537+0000", 2023-10-01T09:18:48.300 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.300 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.300 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.040537+0000", 2023-10-01T09:18:48.300 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:18:48.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.301 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.040536+0000", 2023-10-01T09:18:48.301 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2589000000000003e-05 2023-10-01T09:18:48.301 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.301 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.040569+0000", 2023-10-01T09:18:48.301 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.481e-06 2023-10-01T09:18:48.301 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.302 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.040576+0000", 2023-10-01T09:18:48.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4029999999999999e-06 2023-10-01T09:18:48.302 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.302 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.302 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.302 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.040580+0000", 2023-10-01T09:18:48.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0018192690000000001 2023-10-01T09:18:48.302 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.302 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.302 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.302 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.042399+0000", 2023-10-01T09:18:48.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012977400000000001 2023-10-01T09:18:48.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.303 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.042529+0000", 2023-10-01T09:18:48.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010893800000000001 2023-10-01T09:18:48.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.303 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:18:48.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.042638+0000", 2023-10-01T09:18:48.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0036797980000000002 2023-10-01T09:18:48.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.046318+0000", 2023-10-01T09:18:48.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.008435929 2023-10-01T09:18:48.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.054753+0000", 2023-10-01T09:18:48.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21356623899999999 2023-10-01T09:18:48.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.268320+0000", 2023-10-01T09:18:48.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000129014 2023-10-01T09:18:48.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.268449+0000", 2023-10-01T09:18:48.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0658999999999999e-05 2023-10-01T09:18:48.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.268479+0000", 2023-10-01T09:18:48.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22794239899999999 2023-10-01T09:18:48.306 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.306 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.306 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.306 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:110059 5.7 5:e180c101:::benchmark_data_smithi138_240338_object110058:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-01T09:18:48.306 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:08.050617+0000", 2023-10-01T09:18:48.306 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 160.20707003199999, 2023-10-01T09:18:48.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22482184799999999, 2023-10-01T09:18:48.306 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.306 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.307 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.307 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.307 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.307 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 110059 2023-10-01T09:18:48.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.307 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050617+0000", 2023-10-01T09:18:48.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050617+0000", 2023-10-01T09:18:48.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:18:48.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050616+0000", 2023-10-01T09:18:48.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2482999999999999e-05 2023-10-01T09:18:48.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050668+0000", 2023-10-01T09:18:48.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5464e-05 2023-10-01T09:18:48.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050683+0000", 2023-10-01T09:18:48.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0600000000000002e-06 2023-10-01T09:18:48.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050691+0000", 2023-10-01T09:18:48.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9706999999999997e-05 2023-10-01T09:18:48.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.310 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.310 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050740+0000", 2023-10-01T09:18:48.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000140378 2023-10-01T09:18:48.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.310 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.310 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050881+0000", 2023-10-01T09:18:48.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00021324199999999999 2023-10-01T09:18:48.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:18:48.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.051094+0000", 2023-10-01T09:18:48.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0072607069999999999 2023-10-01T09:18:48.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.058355+0000", 2023-10-01T09:18:48.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7947e-05 2023-10-01T09:18:48.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.058402+0000", 2023-10-01T09:18:48.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21694487800000001 2023-10-01T09:18:48.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275347+0000", 2023-10-01T09:18:48.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1925999999999997e-05 2023-10-01T09:18:48.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275399+0000", 2023-10-01T09:18:48.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9684000000000002e-05 2023-10-01T09:18:48.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275439+0000", 2023-10-01T09:18:48.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22482184799999999 2023-10-01T09:18:48.313 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.313 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.313 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.313 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:119726 5.7 5:e6554c9b:::benchmark_data_smithi138_240338_object4589:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-01T09:18:48.313 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:20.301881+0000", 2023-10-01T09:18:48.314 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 147.955806583, 2023-10-01T09:18:48.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20550888, 2023-10-01T09:18:48.314 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.314 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.314 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.314 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.314 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.314 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 119726 2023-10-01T09:18:48.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.314 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301881+0000", 2023-10-01T09:18:48.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301881+0000", 2023-10-01T09:18:48.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:18:48.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301879+0000", 2023-10-01T09:18:48.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0860000000000002e-06 2023-10-01T09:18:48.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301887+0000", 2023-10-01T09:18:48.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3305000000000001e-05 2023-10-01T09:18:48.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301900+0000", 2023-10-01T09:18:48.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9320000000000004e-06 2023-10-01T09:18:48.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301905+0000", 2023-10-01T09:18:48.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3999000000000003e-05 2023-10-01T09:18:48.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301959+0000", 2023-10-01T09:18:48.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000136933 2023-10-01T09:18:48.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.318 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.302096+0000", 2023-10-01T09:18:48.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000130295 2023-10-01T09:18:48.318 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.318 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.318 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:18:48.318 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.302226+0000", 2023-10-01T09:18:48.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0040403949999999996 2023-10-01T09:18:48.318 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.318 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.318 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.318 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.306267+0000", 2023-10-01T09:18:48.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001816149 2023-10-01T09:18:48.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.319 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.319 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.308083+0000", 2023-10-01T09:18:48.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.199197229 2023-10-01T09:18:48.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.319 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.319 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.507280+0000", 2023-10-01T09:18:48.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7046999999999999e-05 2023-10-01T09:18:48.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.320 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.320 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.507327+0000", 2023-10-01T09:18:48.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.232e-05 2023-10-01T09:18:48.320 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.320 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.320 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.507389+0000", 2023-10-01T09:18:48.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20550888 2023-10-01T09:18:48.320 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.320 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.320 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.321 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:121350 5.6 5:60c69cc8:::benchmark_data_smithi138_240338_object6213:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e125)", 2023-10-01T09:18:48.321 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:22.527232+0000", 2023-10-01T09:18:48.321 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 145.73045562499999, 2023-10-01T09:18:48.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.013085148, 2023-10-01T09:18:48.321 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.321 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:18:48.321 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.321 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.321 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.321 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121350 2023-10-01T09:18:48.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.322 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.527232+0000", 2023-10-01T09:18:48.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.527232+0000", 2023-10-01T09:18:48.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:18:48.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.527230+0000", 2023-10-01T09:18:48.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8749999999999997e-06 2023-10-01T09:18:48.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.527236+0000", 2023-10-01T09:18:48.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2639e-05 2023-10-01T09:18:48.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.527249+0000", 2023-10-01T09:18:48.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9750000000000001e-06 2023-10-01T09:18:48.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.527256+0000", 2023-10-01T09:18:48.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038199219999999999 2023-10-01T09:18:48.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.531076+0000", 2023-10-01T09:18:48.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3699999999999997e-06 2023-10-01T09:18:48.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.325 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op must wait for map", 2023-10-01T09:18:48.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.531083+0000", 2023-10-01T09:18:48.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0026150959999999999 2023-10-01T09:18:48.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.325 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.533698+0000", 2023-10-01T09:18:48.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5810000000000003e-06 2023-10-01T09:18:48.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.325 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:18:48.326 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.533708+0000", 2023-10-01T09:18:48.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.004413848 2023-10-01T09:18:48.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.326 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.326 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.538122+0000", 2023-10-01T09:18:48.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0021949059999999999 2023-10-01T09:18:48.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.326 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.326 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.540317+0000", 2023-10-01T09:18:48.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.013085148 2023-10-01T09:18:48.327 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.327 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.327 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.327 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:121381 5.6 5:6c2d7d69:::benchmark_data_smithi138_240338_object6244:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:18:48.327 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:22.571622+0000", 2023-10-01T09:18:48.327 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 145.686064931, 2023-10-01T09:18:48.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97724141899999994, 2023-10-01T09:18:48.327 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.327 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.327 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.328 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.328 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.328 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121381 2023-10-01T09:18:48.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.328 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.328 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571622+0000", 2023-10-01T09:18:48.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571622+0000", 2023-10-01T09:18:48.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:18:48.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571621+0000", 2023-10-01T09:18:48.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5870000000000003e-06 2023-10-01T09:18:48.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571626+0000", 2023-10-01T09:18:48.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0426e-05 2023-10-01T09:18:48.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571636+0000", 2023-10-01T09:18:48.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4710000000000001e-06 2023-10-01T09:18:48.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571641+0000", 2023-10-01T09:18:48.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8782000000000002e-05 2023-10-01T09:18:48.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571689+0000", 2023-10-01T09:18:48.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7560000000000005e-06 2023-10-01T09:18:48.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:18:48.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.571698+0000", 2023-10-01T09:18:48.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.96865936200000002 2023-10-01T09:18:48.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.540358+0000", 2023-10-01T09:18:48.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00073541700000000004 2023-10-01T09:18:48.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.334 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.541093+0000", 2023-10-01T09:18:48.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8483000000000001e-05 2023-10-01T09:18:48.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.334 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,7", 2023-10-01T09:18:48.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.541171+0000", 2023-10-01T09:18:48.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.007198734 2023-10-01T09:18:48.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.334 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.548370+0000", 2023-10-01T09:18:48.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000376363 2023-10-01T09:18:48.335 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.335 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.335 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.548747+0000", 2023-10-01T09:18:48.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8316000000000001e-05 2023-10-01T09:18:48.335 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.335 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.335 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.548775+0000", 2023-10-01T09:18:48.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4267000000000001e-05 2023-10-01T09:18:48.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.336 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.336 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.548829+0000", 2023-10-01T09:18:48.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4561000000000003e-05 2023-10-01T09:18:48.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.336 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.336 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.548864+0000", 2023-10-01T09:18:48.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97724141899999994 2023-10-01T09:18:48.336 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.337 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.337 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.337 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.337 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:121382 5.6 5:676fecac:::benchmark_data_smithi138_240338_object6245:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:18:48.337 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:22.574219+0000", 2023-10-01T09:18:48.337 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 145.68346848300001, 2023-10-01T09:18:48.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98247247800000004, 2023-10-01T09:18:48.337 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.337 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.337 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.337 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.338 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.338 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121382 2023-10-01T09:18:48.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.338 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574219+0000", 2023-10-01T09:18:48.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574219+0000", 2023-10-01T09:18:48.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:18:48.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574218+0000", 2023-10-01T09:18:48.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1109999999999999e-06 2023-10-01T09:18:48.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574223+0000", 2023-10-01T09:18:48.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0179e-05 2023-10-01T09:18:48.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574233+0000", 2023-10-01T09:18:48.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1960000000000001e-06 2023-10-01T09:18:48.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574237+0000", 2023-10-01T09:18:48.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0365000000000002e-05 2023-10-01T09:18:48.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574298+0000", 2023-10-01T09:18:48.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0120000000000002e-06 2023-10-01T09:18:48.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:18:48.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574304+0000", 2023-10-01T09:18:48.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97017618500000002 2023-10-01T09:18:48.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.544480+0000", 2023-10-01T09:18:48.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000106017 2023-10-01T09:18:48.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.342 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.342 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.544586+0000", 2023-10-01T09:18:48.342 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013026699999999999 2023-10-01T09:18:48.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.342 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,7", 2023-10-01T09:18:48.342 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.544716+0000", 2023-10-01T09:18:48.342 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0060876430000000002 2023-10-01T09:18:48.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.550804+0000", 2023-10-01T09:18:48.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00076178200000000002 2023-10-01T09:18:48.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.551566+0000", 2023-10-01T09:18:48.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050625649999999998 2023-10-01T09:18:48.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.556628+0000", 2023-10-01T09:18:48.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7085000000000002e-05 2023-10-01T09:18:48.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.556665+0000", 2023-10-01T09:18:48.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5840999999999999e-05 2023-10-01T09:18:48.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.556691+0000", 2023-10-01T09:18:48.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98247247800000004 2023-10-01T09:18:48.344 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.345 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.345 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.345 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:132006 5.f 5:f3b89b9a:::benchmark_data_smithi138_240338_object16869:head [delete] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-10-01T09:18:48.345 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:37.912588+0000", 2023-10-01T09:18:48.345 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 130.34509958500001, 2023-10-01T09:18:48.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20444188699999999, 2023-10-01T09:18:48.345 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.345 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.345 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.346 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.346 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.346 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 132006 2023-10-01T09:18:48.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.346 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:37.912588+0000", 2023-10-01T09:18:48.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:37.912588+0000", 2023-10-01T09:18:48.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:18:48.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:37.912586+0000", 2023-10-01T09:18:48.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7359999999999998e-06 2023-10-01T09:18:48.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:37.912592+0000", 2023-10-01T09:18:48.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6590000000000002e-06 2023-10-01T09:18:48.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:37.912602+0000", 2023-10-01T09:18:48.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4280000000000004e-06 2023-10-01T09:18:48.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:37.912607+0000", 2023-10-01T09:18:48.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00058776499999999997 2023-10-01T09:18:48.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:37.913195+0000", 2023-10-01T09:18:48.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000105425 2023-10-01T09:18:48.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:37.913300+0000", 2023-10-01T09:18:48.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014488399999999999 2023-10-01T09:18:48.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:18:48.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:37.913445+0000", 2023-10-01T09:18:48.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17746637200000001 2023-10-01T09:18:48.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:38.090912+0000", 2023-10-01T09:18:48.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000100741 2023-10-01T09:18:48.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:38.091012+0000", 2023-10-01T09:18:48.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.025921091 2023-10-01T09:18:48.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:38.116933+0000", 2023-10-01T09:18:48.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2284000000000003e-05 2023-10-01T09:18:48.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:38.116986+0000", 2023-10-01T09:18:48.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3812e-05 2023-10-01T09:18:48.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.352 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.352 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:38.117029+0000", 2023-10-01T09:18:48.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20444188699999999 2023-10-01T09:18:48.352 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.352 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.352 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.352 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.352 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:135011 5.f 5:f224852f:::benchmark_data_smithi138_240338_object19874:head [delete] snapc 0=[] ondisk+write+known_if_redirected e129)", 2023-10-01T09:18:48.352 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:41.838934+0000", 2023-10-01T09:18:48.352 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 126.418753287, 2023-10-01T09:18:48.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21124153800000001, 2023-10-01T09:18:48.352 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.353 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.353 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.353 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.353 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.353 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 135011 2023-10-01T09:18:48.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.353 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.353 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.353 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:41.838934+0000", 2023-10-01T09:18:48.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:41.838934+0000", 2023-10-01T09:18:48.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:18:48.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:41.838932+0000", 2023-10-01T09:18:48.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7479999999999996e-06 2023-10-01T09:18:48.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:41.838939+0000", 2023-10-01T09:18:48.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3940000000000002e-06 2023-10-01T09:18:48.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:41.838948+0000", 2023-10-01T09:18:48.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.95e-06 2023-10-01T09:18:48.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:41.838953+0000", 2023-10-01T09:18:48.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5674999999999999e-05 2023-10-01T09:18:48.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:41.838999+0000", 2023-10-01T09:18:48.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010450000000000001 2023-10-01T09:18:48.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:41.839104+0000", 2023-10-01T09:18:48.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000113276 2023-10-01T09:18:48.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:18:48.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:41.839217+0000", 2023-10-01T09:18:48.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0097769559999999998 2023-10-01T09:18:48.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:41.848994+0000", 2023-10-01T09:18:48.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001161282 2023-10-01T09:18:48.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:41.850155+0000", 2023-10-01T09:18:48.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.19992015099999999 2023-10-01T09:18:48.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:42.050075+0000", 2023-10-01T09:18:48.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2278999999999999e-05 2023-10-01T09:18:48.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.359 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.359 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.359 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:42.050128+0000", 2023-10-01T09:18:48.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7871000000000003e-05 2023-10-01T09:18:48.359 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.359 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.359 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.359 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:42.050175+0000", 2023-10-01T09:18:48.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21124153800000001 2023-10-01T09:18:48.359 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.359 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.360 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.360 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:143008 5.6 5:6f48508f:::benchmark_data_smithi138_240338_object27871:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-10-01T09:18:48.360 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:53.203595+0000", 2023-10-01T09:18:48.360 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 115.05409213900001, 2023-10-01T09:18:48.360 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21089801899999999, 2023-10-01T09:18:48.360 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.360 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.360 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.360 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.361 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.361 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 143008 2023-10-01T09:18:48.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.361 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.203595+0000", 2023-10-01T09:18:48.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.203595+0000", 2023-10-01T09:18:48.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:18:48.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.203594+0000", 2023-10-01T09:18:48.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8900000000000001e-06 2023-10-01T09:18:48.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.203600+0000", 2023-10-01T09:18:48.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1029999999999999e-05 2023-10-01T09:18:48.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.203611+0000", 2023-10-01T09:18:48.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8069999999999996e-06 2023-10-01T09:18:48.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.203617+0000", 2023-10-01T09:18:48.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8633000000000001e-05 2023-10-01T09:18:48.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.203656+0000", 2023-10-01T09:18:48.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.8653000000000006e-05 2023-10-01T09:18:48.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.203755+0000", 2023-10-01T09:18:48.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011296800000000001 2023-10-01T09:18:48.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,7", 2023-10-01T09:18:48.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.203867+0000", 2023-10-01T09:18:48.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.007449359 2023-10-01T09:18:48.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.211317+0000", 2023-10-01T09:18:48.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2941000000000001e-05 2023-10-01T09:18:48.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.211360+0000", 2023-10-01T09:18:48.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20296324399999999 2023-10-01T09:18:48.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.414323+0000", 2023-10-01T09:18:48.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012609700000000001 2023-10-01T09:18:48.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.414449+0000", 2023-10-01T09:18:48.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3917e-05 2023-10-01T09:18:48.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.367 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.367 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.414493+0000", 2023-10-01T09:18:48.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21089801899999999 2023-10-01T09:18:48.367 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.367 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.367 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.367 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:143020 5.6 5:6a96d8ae:::benchmark_data_smithi138_240338_object27883:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-10-01T09:18:48.368 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:53.219747+0000", 2023-10-01T09:18:48.368 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 115.03794062599999, 2023-10-01T09:18:48.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20372995799999999, 2023-10-01T09:18:48.368 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.368 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.368 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.368 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.368 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.368 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 143020 2023-10-01T09:18:48.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.368 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.219747+0000", 2023-10-01T09:18:48.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.219747+0000", 2023-10-01T09:18:48.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:18:48.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.219746+0000", 2023-10-01T09:18:48.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.878e-06 2023-10-01T09:18:48.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.219750+0000", 2023-10-01T09:18:48.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6809999999999998e-06 2023-10-01T09:18:48.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.219758+0000", 2023-10-01T09:18:48.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8389999999999997e-06 2023-10-01T09:18:48.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.219762+0000", 2023-10-01T09:18:48.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18982892800000001 2023-10-01T09:18:48.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.409591+0000", 2023-10-01T09:18:48.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000114257 2023-10-01T09:18:48.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.409706+0000", 2023-10-01T09:18:48.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000130331 2023-10-01T09:18:48.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,7", 2023-10-01T09:18:48.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.409836+0000", 2023-10-01T09:18:48.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013387447 2023-10-01T09:18:48.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.423223+0000", 2023-10-01T09:18:48.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3343000000000002e-05 2023-10-01T09:18:48.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.423257+0000", 2023-10-01T09:18:48.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015485999999999999 2023-10-01T09:18:48.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.423412+0000", 2023-10-01T09:18:48.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3803999999999997e-05 2023-10-01T09:18:48.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.374 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.423445+0000", 2023-10-01T09:18:48.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1155999999999998e-05 2023-10-01T09:18:48.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.374 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:53.423477+0000", 2023-10-01T09:18:48.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20372995799999999 2023-10-01T09:18:48.374 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.374 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.374 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.375 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.375 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.375 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:145125 5.7 5:e77a601a:::benchmark_data_smithi138_240338_object29988:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-10-01T09:18:48.375 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:56.185770+0000", 2023-10-01T09:18:48.375 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 112.071916745, 2023-10-01T09:18:48.375 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20522225899999999, 2023-10-01T09:18:48.375 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.375 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.375 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.375 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.375 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.375 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 145125 2023-10-01T09:18:48.375 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.376 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.376 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.185770+0000", 2023-10-01T09:18:48.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.376 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.185770+0000", 2023-10-01T09:18:48.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:18:48.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.185769+0000", 2023-10-01T09:18:48.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.511e-06 2023-10-01T09:18:48.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.185776+0000", 2023-10-01T09:18:48.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1636e-05 2023-10-01T09:18:48.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.185788+0000", 2023-10-01T09:18:48.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5409999999999997e-06 2023-10-01T09:18:48.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.185794+0000", 2023-10-01T09:18:48.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0047038959999999999 2023-10-01T09:18:48.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.190498+0000", 2023-10-01T09:18:48.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000114838 2023-10-01T09:18:48.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.190613+0000", 2023-10-01T09:18:48.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.1049000000000002e-05 2023-10-01T09:18:48.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:18:48.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.190704+0000", 2023-10-01T09:18:48.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0036134019999999999 2023-10-01T09:18:48.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.380 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.194317+0000", 2023-10-01T09:18:48.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0789000000000001e-05 2023-10-01T09:18:48.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.380 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.194408+0000", 2023-10-01T09:18:48.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.196483186 2023-10-01T09:18:48.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.390891+0000", 2023-10-01T09:18:48.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3782000000000001e-05 2023-10-01T09:18:48.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.390945+0000", 2023-10-01T09:18:48.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7567e-05 2023-10-01T09:18:48.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.382 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.382 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.390993+0000", 2023-10-01T09:18:48.382 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20522225899999999 2023-10-01T09:18:48.382 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.382 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.382 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.382 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.382 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.382 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:183704 5.7 5:e313672e:::benchmark_data_smithi138_240338_object68567:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T09:18:48.382 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:17:51.014363+0000", 2023-10-01T09:18:48.382 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 57.243324200000004, 2023-10-01T09:18:48.383 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22882143799999999, 2023-10-01T09:18:48.383 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.383 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.383 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183704 2023-10-01T09:18:48.383 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.383 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.383 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.383 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.383 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.014363+0000", 2023-10-01T09:18:48.383 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.014363+0000", 2023-10-01T09:18:48.384 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:18:48.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.014363+0000", 2023-10-01T09:18:48.384 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7829999999999999e-06 2023-10-01T09:18:48.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.014365+0000", 2023-10-01T09:18:48.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9209999999999997e-06 2023-10-01T09:18:48.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.014371+0000", 2023-10-01T09:18:48.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7100000000000001e-06 2023-10-01T09:18:48.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.014375+0000", 2023-10-01T09:18:48.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045538319999999998 2023-10-01T09:18:48.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.386 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.018929+0000", 2023-10-01T09:18:48.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000126857 2023-10-01T09:18:48.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.386 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.019056+0000", 2023-10-01T09:18:48.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0001563 2023-10-01T09:18:48.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:18:48.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.019212+0000", 2023-10-01T09:18:48.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010334147 2023-10-01T09:18:48.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.029546+0000", 2023-10-01T09:18:48.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00083725 2023-10-01T09:18:48.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.388 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.030383+0000", 2023-10-01T09:18:48.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21273308799999999 2023-10-01T09:18:48.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.388 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.243116+0000", 2023-10-01T09:18:48.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9093e-05 2023-10-01T09:18:48.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.389 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.389 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.243155+0000", 2023-10-01T09:18:48.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8918e-05 2023-10-01T09:18:48.389 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.389 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.389 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.389 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.243184+0000", 2023-10-01T09:18:48.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22882143799999999 2023-10-01T09:18:48.389 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.389 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.389 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.389 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.390 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.390 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:196409 5.7 5:e32a3eb1:::benchmark_data_smithi138_240338_object81272:head [delete] snapc 0=[] ondisk+write+known_if_redirected e144)", 2023-10-01T09:18:48.390 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:10.967170+0000", 2023-10-01T09:18:48.390 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 37.290516762999999, 2023-10-01T09:18:48.390 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.23119947699999999, 2023-10-01T09:18:48.390 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.390 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.390 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.390 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.390 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.390 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 196409 2023-10-01T09:18:48.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.391 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.391 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.391 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.967170+0000", 2023-10-01T09:18:48.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.391 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.391 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.967170+0000", 2023-10-01T09:18:48.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:18:48.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.967169+0000", 2023-10-01T09:18:48.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4069999999999998e-06 2023-10-01T09:18:48.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.967176+0000", 2023-10-01T09:18:48.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3762e-05 2023-10-01T09:18:48.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.967190+0000", 2023-10-01T09:18:48.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.498e-06 2023-10-01T09:18:48.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.967197+0000", 2023-10-01T09:18:48.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00018680699999999999 2023-10-01T09:18:48.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.394 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.967384+0000", 2023-10-01T09:18:48.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010816500000000001 2023-10-01T09:18:48.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.394 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.394 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.967492+0000", 2023-10-01T09:18:48.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012716800000000001 2023-10-01T09:18:48.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.394 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:18:48.394 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.967619+0000", 2023-10-01T09:18:48.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0086468680000000003 2023-10-01T09:18:48.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.976266+0000", 2023-10-01T09:18:48.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0039189949999999998 2023-10-01T09:18:48.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.980185+0000", 2023-10-01T09:18:48.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.218093914 2023-10-01T09:18:48.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:11.198279+0000", 2023-10-01T09:18:48.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4567000000000002e-05 2023-10-01T09:18:48.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:11.198324+0000", 2023-10-01T09:18:48.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6344000000000002e-05 2023-10-01T09:18:48.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:11.198370+0000", 2023-10-01T09:18:48.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.23119947699999999 2023-10-01T09:18:48.397 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.397 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.397 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.397 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:196415 5.7 5:e8967b08:::benchmark_data_smithi138_240338_object81278:head [delete] snapc 0=[] ondisk+write+known_if_redirected e144)", 2023-10-01T09:18:48.397 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:10.980576+0000", 2023-10-01T09:18:48.397 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 37.277111038999998, 2023-10-01T09:18:48.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.218065011, 2023-10-01T09:18:48.397 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.397 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.398 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.398 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.398 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.398 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 196415 2023-10-01T09:18:48.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.398 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.980576+0000", 2023-10-01T09:18:48.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.980576+0000", 2023-10-01T09:18:48.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:18:48.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.980575+0000", 2023-10-01T09:18:48.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4339999999999997e-06 2023-10-01T09:18:48.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.980581+0000", 2023-10-01T09:18:48.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.222e-05 2023-10-01T09:18:48.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.400 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.980594+0000", 2023-10-01T09:18:48.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4479999999999997e-06 2023-10-01T09:18:48.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.400 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.980601+0000", 2023-10-01T09:18:48.401 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:48.402 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:48 smithi138 ceph-mon[185886]: pgmap v402: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 87 active+clean; 856 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 1308/40950 objects misplaced (3.194%); 11 MiB/s, 1 keys/s, 178 objects/s recovering 2023-10-01T09:18:48.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0830000000000001e-05 2023-10-01T09:18:48.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.402 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.980632+0000", 2023-10-01T09:18:48.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7261000000000004e-05 2023-10-01T09:18:48.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.980719+0000", 2023-10-01T09:18:48.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0412000000000001e-05 2023-10-01T09:18:48.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1", 2023-10-01T09:18:48.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.980810+0000", 2023-10-01T09:18:48.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00383491 2023-10-01T09:18:48.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.984644+0000", 2023-10-01T09:18:48.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0728999999999997e-05 2023-10-01T09:18:48.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.984675+0000", 2023-10-01T09:18:48.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21387584700000001 2023-10-01T09:18:48.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:11.198551+0000", 2023-10-01T09:18:48.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7205e-05 2023-10-01T09:18:48.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:11.198598+0000", 2023-10-01T09:18:48.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2899999999999999e-05 2023-10-01T09:18:48.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:11.198641+0000", 2023-10-01T09:18:48.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.218065011 2023-10-01T09:18:48.405 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.406 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.406 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.406 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:209450 5.6 5:6f57ba85:::benchmark_data_smithi138_240338_object94313:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-01T09:18:48.406 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:30.564292+0000", 2023-10-01T09:18:48.406 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 17.693394720000001, 2023-10-01T09:18:48.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3225590170000001, 2023-10-01T09:18:48.406 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.406 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.406 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.407 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.407 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.407 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 209450 2023-10-01T09:18:48.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.407 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.407 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.407 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.564292+0000", 2023-10-01T09:18:48.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.407 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.564292+0000", 2023-10-01T09:18:48.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:18:48.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.564291+0000", 2023-10-01T09:18:48.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9669999999999998e-06 2023-10-01T09:18:48.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.564297+0000", 2023-10-01T09:18:48.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6113999999999999e-05 2023-10-01T09:18:48.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.564313+0000", 2023-10-01T09:18:48.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4230000000000002e-06 2023-10-01T09:18:48.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.564319+0000", 2023-10-01T09:18:48.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6684e-05 2023-10-01T09:18:48.410 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.410 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.410 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.410 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.564396+0000", 2023-10-01T09:18:48.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1790999999999999e-05 2023-10-01T09:18:48.410 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.410 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.410 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:18:48.410 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.564408+0000", 2023-10-01T09:18:48.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.84421806099999996 2023-10-01T09:18:48.410 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.411 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.411 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.411 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.408626+0000", 2023-10-01T09:18:48.411 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2663000000000001e-05 2023-10-01T09:18:48.411 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.411 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.411 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:18:48.411 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.408638+0000", 2023-10-01T09:18:48.411 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.448535546 2023-10-01T09:18:48.411 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.411 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.411 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.412 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.857174+0000", 2023-10-01T09:18:48.412 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012804900000000001 2023-10-01T09:18:48.412 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.412 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.412 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.412 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.857302+0000", 2023-10-01T09:18:48.412 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00075739799999999995 2023-10-01T09:18:48.412 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.412 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.412 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,4,7", 2023-10-01T09:18:48.412 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.858059+0000", 2023-10-01T09:18:48.412 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021170910000000001 2023-10-01T09:18:48.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.413 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.413 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.879230+0000", 2023-10-01T09:18:48.413 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.005175242 2023-10-01T09:18:48.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.413 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.413 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.884406+0000", 2023-10-01T09:18:48.413 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1769999999999998e-06 2023-10-01T09:18:48.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.414 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.414 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.884414+0000", 2023-10-01T09:18:48.414 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0022555539999999999 2023-10-01T09:18:48.414 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.414 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.414 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.414 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.886669+0000", 2023-10-01T09:18:48.414 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0668999999999998e-05 2023-10-01T09:18:48.414 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.414 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.414 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.415 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.886730+0000", 2023-10-01T09:18:48.415 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000121426 2023-10-01T09:18:48.415 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.415 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.415 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.415 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.886851+0000", 2023-10-01T09:18:48.415 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3225590170000001 2023-10-01T09:18:48.415 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.415 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.415 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.415 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.415 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.416 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:209470 5.6 5:6ec449d3:::benchmark_data_smithi138_240338_object94333:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-01T09:18:48.416 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:30.613716+0000", 2023-10-01T09:18:48.416 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 17.643971435000001, 2023-10-01T09:18:48.416 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.293182673, 2023-10-01T09:18:48.416 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:48.416 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:18:48.416 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:18:48.416 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:18:48.416 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:18:48.416 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 209470 2023-10-01T09:18:48.416 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.416 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:48.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.417 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:48.417 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.613716+0000", 2023-10-01T09:18:48.417 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:48.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.417 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:48.417 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.613716+0000", 2023-10-01T09:18:48.417 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:18:48.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.417 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:48.418 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.613714+0000", 2023-10-01T09:18:48.418 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1550000000000007e-06 2023-10-01T09:18:48.418 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.418 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.418 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:48.418 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.613722+0000", 2023-10-01T09:18:48.418 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5145e-05 2023-10-01T09:18:48.418 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.418 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.418 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:48.418 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.613737+0000", 2023-10-01T09:18:48.419 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.17e-06 2023-10-01T09:18:48.419 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.419 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.419 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:48.419 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.613745+0000", 2023-10-01T09:18:48.419 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2527999999999995e-05 2023-10-01T09:18:48.419 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.419 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.419 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.419 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.613807+0000", 2023-10-01T09:18:48.419 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.8100000000000004e-06 2023-10-01T09:18:48.419 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.420 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.420 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:18:48.420 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.613816+0000", 2023-10-01T09:18:48.420 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.794845625 2023-10-01T09:18:48.420 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.420 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.420 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.420 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.408662+0000", 2023-10-01T09:18:48.420 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1030000000000002e-06 2023-10-01T09:18:48.420 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.420 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.420 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:18:48.421 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.408670+0000", 2023-10-01T09:18:48.421 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4572917940000001 2023-10-01T09:18:48.421 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.421 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.421 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:48.421 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.865961+0000", 2023-10-01T09:18:48.421 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012610099999999999 2023-10-01T09:18:48.421 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.421 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.421 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:48.421 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.866088+0000", 2023-10-01T09:18:48.421 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000157357 2023-10-01T09:18:48.422 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.422 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.422 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,4,7", 2023-10-01T09:18:48.422 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.866245+0000", 2023-10-01T09:18:48.422 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013027581999999999 2023-10-01T09:18:48.422 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.422 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.422 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.422 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.879272+0000", 2023-10-01T09:18:48.422 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0051474900000000002 2023-10-01T09:18:48.422 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.423 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.423 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:18:48.423 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.884420+0000", 2023-10-01T09:18:48.423 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8763000000000001e-05 2023-10-01T09:18:48.423 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.423 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.423 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.423 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.884459+0000", 2023-10-01T09:18:48.423 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.022279150000000001 2023-10-01T09:18:48.423 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.423 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.423 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:18:48.424 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.906738+0000", 2023-10-01T09:18:48.424 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0257999999999998e-05 2023-10-01T09:18:48.424 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.424 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.424 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:18:48.424 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.906798+0000", 2023-10-01T09:18:48.424 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000100289 2023-10-01T09:18:48.424 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:48.424 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:48.424 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:18:48.424 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.906898+0000", 2023-10-01T09:18:48.425 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.293182673 2023-10-01T09:18:48.425 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.425 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.425 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.425 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:48.425 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:48.425 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:18:48.425 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:48 smithi138 ceph-mon[192513]: pgmap v402: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 87 active+clean; 856 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 1308/40950 objects misplaced (3.194%); 11 MiB/s, 1 keys/s, 178 objects/s recovering 2023-10-01T09:18:48.465 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:18:48.489 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:48.653 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:48.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:48.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:48.746 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:48.768 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:48.815 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:49.025 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:49.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:49.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:49.126 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:18:49.140 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:49.233 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:49.342 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:49.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:49.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:49.455 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:49.472 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:49.518 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:49.706 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:49.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:49.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:49.811 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:49.921 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:49.996 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:18:50.016 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:50.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:50.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:50.136 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:50.198 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:50.414 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:50.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:50.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:50.460 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:50 smithi138 ceph-mon[192513]: pgmap v403: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 87 active+clean; 829 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 1308/39654 objects misplaced (3.299%); 2.9 MiB/s, 1 keys/s, 50 objects/s recovering 2023-10-01T09:18:50.460 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:50 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2968156358' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:50.498 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:18:50.522 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:50.603 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:50 smithi190 ceph-mon[160435]: pgmap v403: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 87 active+clean; 829 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 1308/39654 objects misplaced (3.299%); 2.9 MiB/s, 1 keys/s, 50 objects/s recovering 2023-10-01T09:18:50.604 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:50 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2968156358' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:50.609 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:50.699 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:50.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:50.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:50.804 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:50.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:50 smithi138 ceph-mon[185886]: pgmap v403: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 87 active+clean; 829 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 1308/39654 objects misplaced (3.299%); 2.9 MiB/s, 1 keys/s, 50 objects/s recovering 2023-10-01T09:18:50.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:50 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2968156358' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:18:50.886 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:50.959 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:50.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:50.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:51.063 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:18:51.133 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:51.174 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:18:51.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:51.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:18:51.277 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:51.343 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:51.466 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:51 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:51.467 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:51 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2968156358' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:18:51.467 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:51 smithi190 ceph-mon[160435]: osdmap e159: 8 total, 8 up, 8 in 2023-10-01T09:18:51.467 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:51 smithi190 ceph-mon[160435]: failed to encode map e159 with expected crc 2023-10-01T09:18:51.467 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:51 smithi190 ceph-mon[160435]: failed to encode map e159 with expected crc 2023-10-01T09:18:51.467 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:51 smithi190 ceph-mon[160435]: failed to encode map e159 with expected crc 2023-10-01T09:18:51.467 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:51 smithi190 ceph-mon[160435]: failed to encode map e159 with expected crc 2023-10-01T09:18:51.618 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:51.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:51.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:51.740 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:51.810 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:51.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:51 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:51.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:51 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2968156358' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:18:51.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:51 smithi138 ceph-mon[192513]: osdmap e159: 8 total, 8 up, 8 in 2023-10-01T09:18:51.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:51 smithi138 ceph-mon[192513]: failed to encode map e159 with expected crc 2023-10-01T09:18:51.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:51 smithi138 ceph-mon[192513]: failed to encode map e159 with expected crc 2023-10-01T09:18:51.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:51 smithi138 ceph-mon[192513]: failed to encode map e159 with expected crc 2023-10-01T09:18:51.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:51 smithi138 ceph-mon[192513]: failed to encode map e159 with expected crc 2023-10-01T09:18:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:51 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:51 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2968156358' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:18:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:51 smithi138 ceph-mon[185886]: osdmap e159: 8 total, 8 up, 8 in 2023-10-01T09:18:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:51 smithi138 ceph-mon[185886]: failed to encode map e159 with expected crc 2023-10-01T09:18:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:51 smithi138 ceph-mon[185886]: failed to encode map e159 with expected crc 2023-10-01T09:18:51.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:51 smithi138 ceph-mon[185886]: failed to encode map e159 with expected crc 2023-10-01T09:18:51.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:51 smithi138 ceph-mon[185886]: failed to encode map e159 with expected crc 2023-10-01T09:18:51.853 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:51.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:51.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:51.957 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:51.984 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:52.088 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:52.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:52.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:52.194 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:52.195 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:18:52.196 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:18:52.196 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:52.196 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.34683.0:220524 5.9 e159/106 5:972a5261:::benchmark_data_smithi138_240338_object105387:head v 159'14015, mlcod=159'14011)", 2023-10-01T09:18:52.196 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:52.190615+0000", 2023-10-01T09:18:52.196 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0044905860000000004, 2023-10-01T09:18:52.196 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045119060000000004, 2023-10-01T09:18:52.196 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:18:52.196 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:18:52.196 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:18:52.196 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:52.197 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:18:52.197 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.190615+0000", 2023-10-01T09:18:52.197 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:18:52.197 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:52.197 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:52.197 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:18:52.197 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.190615+0000", 2023-10-01T09:18:52.197 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:18:52.197 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:52.197 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:52.197 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:18:52.198 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.190614+0000", 2023-10-01T09:18:52.198 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3940000000000002e-06 2023-10-01T09:18:52.198 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:52.198 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:52.198 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:18:52.198 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.190624+0000", 2023-10-01T09:18:52.198 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9108999999999999e-05 2023-10-01T09:18:52.198 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:52.198 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:52.198 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:18:52.198 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.190643+0000", 2023-10-01T09:18:52.199 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0600000000000008e-06 2023-10-01T09:18:52.199 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:52.199 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:52.199 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:18:52.199 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.190651+0000", 2023-10-01T09:18:52.199 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00030844200000000003 2023-10-01T09:18:52.199 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:52.199 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:52.199 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:18:52.199 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.190959+0000", 2023-10-01T09:18:52.199 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2135000000000001e-05 2023-10-01T09:18:52.200 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:18:52.200 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:18:52.200 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:18:52.200 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.190981+0000", 2023-10-01T09:18:52.200 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000366158 2023-10-01T09:18:52.200 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:52.200 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:18:52.200 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:52.200 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:18:52.200 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:18:52.200 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 1 2023-10-01T09:18:52.200 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:18:52.297 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:52.509 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:52.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:52.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:52.612 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:52.638 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:52.680 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:52.681 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:52 smithi190 ceph-mon[160435]: pgmap v405: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 87 active+clean; 829 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 1308/39654 objects misplaced (3.299%); 843 KiB/s, 1 keys/s, 16 objects/s recovering 2023-10-01T09:18:52.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:52.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:52.783 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:52.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:52 smithi138 ceph-mon[192513]: pgmap v405: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 87 active+clean; 829 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 1308/39654 objects misplaced (3.299%); 843 KiB/s, 1 keys/s, 16 objects/s recovering 2023-10-01T09:18:52.817 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:52 smithi138 ceph-mon[185886]: pgmap v405: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 87 active+clean; 829 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 1308/39654 objects misplaced (3.299%); 843 KiB/s, 1 keys/s, 16 objects/s recovering 2023-10-01T09:18:52.839 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:53.061 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:18:53.064 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:53.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:53.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:53.160 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:18:53.178 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:53.192 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:18:53.240 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:18:53.254 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:53.277 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:18:53.310 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:18:53.349 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:53.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:53.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:53.359 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:18:53.388 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:18:53.430 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:18:53.431 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:18:53.431 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:18:53.432 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:18:53.432 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:18:53.432 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:18:53.432 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:18:53.432 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:18:53.432 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:18:53.456 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:53.503 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3029352389' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-01T09:18:53.503 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1345541655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:53.518 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.4 primary-affinity to 0 (802) 2023-10-01T09:18:53.528 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:53.612 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:53.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:53.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:53.656 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:18:53.700 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:18:53.720 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:18:53.748 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:53.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3029352389' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-01T09:18:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1345541655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3029352389' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-01T09:18:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1345541655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:18:53.824 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:18:53.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:53.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:18:53.928 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:53.957 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:54.000 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:54.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:54.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:54.104 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:54.133 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:54.227 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:54.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:54.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:54.331 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:54.358 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:54.514 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:54 smithi190 ceph-mon[160435]: pgmap v406: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 621 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 913/29667 objects misplaced (3.077%); 1.4 MiB/s, 1 keys/s, 24 objects/s recovering 2023-10-01T09:18:54.514 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:54 smithi190 ceph-mon[160435]: failed to encode map e160 with expected crc 2023-10-01T09:18:54.514 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:54 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3029352389' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-01T09:18:54.514 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:54 smithi190 ceph-mon[160435]: osdmap e160: 8 total, 8 up, 8 in 2023-10-01T09:18:54.515 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:54 smithi190 ceph-mon[160435]: failed to encode map e160 with expected crc 2023-10-01T09:18:54.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:54 smithi138 ceph-mon[192513]: pgmap v406: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 621 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 913/29667 objects misplaced (3.077%); 1.4 MiB/s, 1 keys/s, 24 objects/s recovering 2023-10-01T09:18:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:54 smithi138 ceph-mon[192513]: failed to encode map e160 with expected crc 2023-10-01T09:18:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:54 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3029352389' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-01T09:18:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:54 smithi138 ceph-mon[192513]: osdmap e160: 8 total, 8 up, 8 in 2023-10-01T09:18:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:54 smithi138 ceph-mon[192513]: failed to encode map e160 with expected crc 2023-10-01T09:18:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:54 smithi138 ceph-mon[185886]: pgmap v406: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 621 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 913/29667 objects misplaced (3.077%); 1.4 MiB/s, 1 keys/s, 24 objects/s recovering 2023-10-01T09:18:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:54 smithi138 ceph-mon[185886]: failed to encode map e160 with expected crc 2023-10-01T09:18:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:54 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3029352389' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-01T09:18:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:54 smithi138 ceph-mon[185886]: osdmap e160: 8 total, 8 up, 8 in 2023-10-01T09:18:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:54 smithi138 ceph-mon[185886]: failed to encode map e160 with expected crc 2023-10-01T09:18:55.011 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:55.054 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:55.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:55.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:55.157 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:18:55.180 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:18:55.180 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:18:55.181 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:18:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:18:54] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.33.4" 2023-10-01T09:18:55.184 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:55.328 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:18:55.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:55.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:18:55.431 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:55.461 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:55.506 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:55.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:55.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:55.609 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:55.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.635 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.635 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.635 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: osdmap e161: 8 total, 8 up, 8 in 2023-10-01T09:18:55.635 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.635 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.635 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.635 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: 5.a starting backfill to osd.0 from (121'6481,159'13548] MIN to 159'13550 2023-10-01T09:18:55.635 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]: dispatch 2023-10-01T09:18:55.635 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 3]}]: dispatch 2023-10-01T09:18:55.636 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-01T09:18:55.636 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-01T09:18:55.636 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2023-10-01T09:18:55.636 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-01T09:18:55.636 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-01T09:18:55.636 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:55.636 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:18:55.638 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: osdmap e161: 8 total, 8 up, 8 in 2023-10-01T09:18:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: 5.a starting backfill to osd.0 from (121'6481,159'13548] MIN to 159'13550 2023-10-01T09:18:55.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]: dispatch 2023-10-01T09:18:55.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 3]}]: dispatch 2023-10-01T09:18:55.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-01T09:18:55.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-01T09:18:55.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2023-10-01T09:18:55.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-01T09:18:55.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-01T09:18:55.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:55.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:18:55.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: osdmap e161: 8 total, 8 up, 8 in 2023-10-01T09:18:55.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: failed to encode map e161 with expected crc 2023-10-01T09:18:55.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: 5.a starting backfill to osd.0 from (121'6481,159'13548] MIN to 159'13550 2023-10-01T09:18:55.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]: dispatch 2023-10-01T09:18:55.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 3]}]: dispatch 2023-10-01T09:18:55.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-01T09:18:55.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2023-10-01T09:18:55.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2023-10-01T09:18:55.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-10-01T09:18:55.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-01T09:18:55.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:18:55.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:18:55.974 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:18:55.974 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:55.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:55.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:56.083 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:56.138 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:56.393 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:56.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:56.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:56.501 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:56.560 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:56.709 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[185886]: pgmap v409: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 621 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 913/29667 objects misplaced (3.077%); 917 KiB/s, 14 objects/s recovering 2023-10-01T09:18:56.710 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[185886]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.710 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[185886]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.710 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2023-10-01T09:18:56.710 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 3]}]': finished 2023-10-01T09:18:56.710 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-01T09:18:56.710 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-01T09:18:56.710 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2023-10-01T09:18:56.710 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-10-01T09:18:56.711 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-01T09:18:56.711 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[185886]: osdmap e162: 8 total, 8 up, 8 in 2023-10-01T09:18:56.711 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[185886]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.711 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[185886]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.711 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[185886]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[192513]: pgmap v409: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 621 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 913/29667 objects misplaced (3.077%); 917 KiB/s, 14 objects/s recovering 2023-10-01T09:18:56.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[192513]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[192513]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2023-10-01T09:18:56.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 3]}]': finished 2023-10-01T09:18:56.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-01T09:18:56.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-01T09:18:56.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2023-10-01T09:18:56.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-10-01T09:18:56.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-01T09:18:56.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[192513]: osdmap e162: 8 total, 8 up, 8 in 2023-10-01T09:18:56.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[192513]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[192513]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:56 smithi138 ceph-mon[192513]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.751 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:56.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:56.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:56.769 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:18:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:56 smithi190 ceph-mon[160435]: pgmap v409: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 621 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 913/29667 objects misplaced (3.077%); 917 KiB/s, 14 objects/s recovering 2023-10-01T09:18:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:56 smithi190 ceph-mon[160435]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:56 smithi190 ceph-mon[160435]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2023-10-01T09:18:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 3]}]': finished 2023-10-01T09:18:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-01T09:18:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2023-10-01T09:18:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2023-10-01T09:18:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-10-01T09:18:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-01T09:18:56.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:56 smithi190 ceph-mon[160435]: osdmap e162: 8 total, 8 up, 8 in 2023-10-01T09:18:56.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:56 smithi190 ceph-mon[160435]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:56 smithi190 ceph-mon[160435]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:56 smithi190 ceph-mon[160435]: failed to encode map e162 with expected crc 2023-10-01T09:18:56.858 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:56.905 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:18:56.915 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:57.044 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:57.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:57.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:57.167 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:57.213 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:57.332 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:57.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:57.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:57.436 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:57.522 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:57.552 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:18:57.634 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:57.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:57.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:57.749 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:18:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:57 smithi190 ceph-mon[160435]: 5.1 continuing backfill to osd.0 from (122'7065,159'14139] 5:891037df:::benchmark_data_smithi138_240338_object109236:head to 159'14139 2023-10-01T09:18:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:57 smithi190 ceph-mon[160435]: osdmap e163: 8 total, 8 up, 8 in 2023-10-01T09:18:57.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:57 smithi190 ceph-mon[160435]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:57 smithi190 ceph-mon[160435]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:57 smithi190 ceph-mon[160435]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:57 smithi190 ceph-mon[160435]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:57 smithi190 ceph-mon[160435]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:57 smithi190 ceph-mon[160435]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3837068439' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:18:57.812 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[185886]: 5.1 continuing backfill to osd.0 from (122'7065,159'14139] 5:891037df:::benchmark_data_smithi138_240338_object109236:head to 159'14139 2023-10-01T09:18:57.812 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[185886]: osdmap e163: 8 total, 8 up, 8 in 2023-10-01T09:18:57.812 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[185886]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.812 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[185886]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.812 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[185886]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.812 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[185886]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.812 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[185886]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[185886]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3837068439' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:18:57.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[192513]: 5.1 continuing backfill to osd.0 from (122'7065,159'14139] 5:891037df:::benchmark_data_smithi138_240338_object109236:head to 159'14139 2023-10-01T09:18:57.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[192513]: osdmap e163: 8 total, 8 up, 8 in 2023-10-01T09:18:57.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[192513]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[192513]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[192513]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[192513]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[192513]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[192513]: failed to encode map e163 with expected crc 2023-10-01T09:18:57.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3837068439' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:18:57.820 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:57.900 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:18:57.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:57.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:18:58.003 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:18:58.059 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:58.081 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:18:58.342 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:18:58.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:58.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:18:58.456 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:18:58.534 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:58.544 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[192513]: failed to encode map e163 with expected crc 2023-10-01T09:18:58.544 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[192513]: failed to encode map e163 with expected crc 2023-10-01T09:18:58.545 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[192513]: pgmap v412: 89 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+remapped+backfill_wait, 84 active+clean; 505 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 653/24093 objects misplaced (2.710%); 1.0 MiB/s, 17 objects/s recovering 2023-10-01T09:18:58.545 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[192513]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:18:58.545 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:58.545 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[192513]: failed to encode map e164 with expected crc 2023-10-01T09:18:58.545 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3837068439' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:18:58.545 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[192513]: osdmap e164: 8 total, 8 up, 8 in 2023-10-01T09:18:58.545 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[192513]: failed to encode map e164 with expected crc 2023-10-01T09:18:58.545 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[192513]: failed to encode map e164 with expected crc 2023-10-01T09:18:58.546 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[185886]: failed to encode map e163 with expected crc 2023-10-01T09:18:58.546 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[185886]: failed to encode map e163 with expected crc 2023-10-01T09:18:58.546 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[185886]: pgmap v412: 89 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+remapped+backfill_wait, 84 active+clean; 505 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 653/24093 objects misplaced (2.710%); 1.0 MiB/s, 17 objects/s recovering 2023-10-01T09:18:58.546 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[185886]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:18:58.546 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:58.546 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[185886]: failed to encode map e164 with expected crc 2023-10-01T09:18:58.546 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3837068439' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:18:58.546 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[185886]: osdmap e164: 8 total, 8 up, 8 in 2023-10-01T09:18:58.546 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[185886]: failed to encode map e164 with expected crc 2023-10-01T09:18:58.547 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:58 smithi138 ceph-mon[185886]: failed to encode map e164 with expected crc 2023-10-01T09:18:58.705 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:58 smithi190 ceph-mon[160435]: failed to encode map e163 with expected crc 2023-10-01T09:18:58.705 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:58 smithi190 ceph-mon[160435]: failed to encode map e163 with expected crc 2023-10-01T09:18:58.705 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:58 smithi190 ceph-mon[160435]: pgmap v412: 89 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+remapped+backfill_wait, 84 active+clean; 505 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 653/24093 objects misplaced (2.710%); 1.0 MiB/s, 17 objects/s recovering 2023-10-01T09:18:58.705 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:58 smithi190 ceph-mon[160435]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:18:58.705 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:58 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:18:58.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:58 smithi190 ceph-mon[160435]: failed to encode map e164 with expected crc 2023-10-01T09:18:58.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:58 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3837068439' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:18:58.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:58 smithi190 ceph-mon[160435]: osdmap e164: 8 total, 8 up, 8 in 2023-10-01T09:18:58.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:58 smithi190 ceph-mon[160435]: failed to encode map e164 with expected crc 2023-10-01T09:18:58.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:58 smithi190 ceph-mon[160435]: failed to encode map e164 with expected crc 2023-10-01T09:18:58.728 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:18:58.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:58.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:18:58.834 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:18:58.861 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:18:58.945 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:18:58.946 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:18:59.813 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:18:59 smithi190 ceph-mon[160435]: pgmap v414: 89 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+remapped+backfill_wait, 84 active+clean; 488 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 653/23286 objects misplaced (2.804%); 878 KiB/s, 14 objects/s recovering 2023-10-01T09:18:59.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:18:59 smithi138 ceph-mon[185886]: pgmap v414: 89 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+remapped+backfill_wait, 84 active+clean; 488 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 653/23286 objects misplaced (2.804%); 878 KiB/s, 14 objects/s recovering 2023-10-01T09:18:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:18:59 smithi138 ceph-mon[192513]: pgmap v414: 89 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+remapped+backfill_wait, 84 active+clean; 488 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 653/23286 objects misplaced (2.804%); 878 KiB/s, 14 objects/s recovering 2023-10-01T09:18:59.843 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:18:59.945 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:18:59.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:18:59.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:18:59.949 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:18:59.949 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:18:59.949 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-01T09:18:59.949 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-01T09:19:00.049 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:00.096 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:00.241 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:00.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:00.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:00.351 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:00.433 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:00.478 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:19:00.479 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:19:00.511 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:19:00.553 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:00.553 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:19:00.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:00.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:00.584 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:19:00.626 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:00.645 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:19:00.660 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:00.685 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:19:00.704 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:00.717 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:19:00.759 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:19:00.763 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:19:00.763 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:19:00.763 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:19:00.763 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:19:00.763 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:19:00.763 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:19:00.763 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:19:00.764 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:19:00.789 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:00 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/157002903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:00.789 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:00 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/157002903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:00.822 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:00 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/157002903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:00.825 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:00.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:00.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:00.941 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:01.000 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:01.719 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:01.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:01.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:01.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:01 smithi138 ceph-mon[192513]: pgmap v415: 89 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+remapped+backfill_wait, 84 active+clean; 488 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 354 op/s; 653/23286 objects misplaced (2.804%); 714 KiB/s, 11 objects/s recovering 2023-10-01T09:19:01.827 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:01 smithi138 ceph-mon[185886]: pgmap v415: 89 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+remapped+backfill_wait, 84 active+clean; 488 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 354 op/s; 653/23286 objects misplaced (2.804%); 714 KiB/s, 11 objects/s recovering 2023-10-01T09:19:01.837 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:01.899 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:01.911 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:01 smithi190 ceph-mon[160435]: pgmap v415: 89 pgs: 1 active+remapped+backfilling, 3 peering, 1 active+remapped+backfill_wait, 84 active+clean; 488 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 354 op/s; 653/23286 objects misplaced (2.804%); 714 KiB/s, 11 objects/s recovering 2023-10-01T09:19:01.938 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:19:02.201 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:02.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:02.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:02.308 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:02.335 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:02.542 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:19:02.543 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:19:02.543 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.543 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:226195 5.d 5:b147ee46:::benchmark_data_smithi138_240338_object111058:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T09:19:02.543 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:19:02.530550+0000", 2023-10-01T09:19:02.543 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0086997359999999996, 2023-10-01T09:19:02.543 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0087330649999999999, 2023-10-01T09:19:02.543 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:19:02.543 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:19:02.543 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:19:02.543 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:19:02.543 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:02.543 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 226195 2023-10-01T09:19:02.544 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.544 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:19:02.544 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.544 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:19:02.544 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530550+0000", 2023-10-01T09:19:02.544 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:19:02.544 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.544 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.544 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:19:02.544 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530550+0000", 2023-10-01T09:19:02.544 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:19:02.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.545 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:19:02.545 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530549+0000", 2023-10-01T09:19:02.545 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5849999999999996e-06 2023-10-01T09:19:02.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.545 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:19:02.545 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530555+0000", 2023-10-01T09:19:02.545 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3869999999999999e-05 2023-10-01T09:19:02.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:19:02.546 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530569+0000", 2023-10-01T09:19:02.546 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9360000000000002e-06 2023-10-01T09:19:02.546 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.546 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:19:02.546 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530576+0000", 2023-10-01T09:19:02.546 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5915999999999999e-05 2023-10-01T09:19:02.546 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.546 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:19:02.547 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530622+0000", 2023-10-01T09:19:02.547 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011925399999999999 2023-10-01T09:19:02.547 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.547 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.547 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:19:02.547 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530741+0000", 2023-10-01T09:19:02.547 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000121432 2023-10-01T09:19:02.547 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.547 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.547 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:19:02.547 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530863+0000", 2023-10-01T09:19:02.547 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045083170000000004 2023-10-01T09:19:02.548 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.548 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.548 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:19:02.548 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.535371+0000", 2023-10-01T09:19:02.548 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0048208640000000002 2023-10-01T09:19:02.548 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:19:02.548 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:19:02.548 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:19:02.548 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.548 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.548 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:226199 5.9 5:9e0a839c:::benchmark_data_smithi138_240338_object111062:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T09:19:02.549 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:19:02.530588+0000", 2023-10-01T09:19:02.549 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0086614789999999997, 2023-10-01T09:19:02.549 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0088072800000000007, 2023-10-01T09:19:02.549 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:19:02.549 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:19:02.549 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:19:02.549 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:19:02.549 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:02.549 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 226199 2023-10-01T09:19:02.549 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.549 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:19:02.549 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.549 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:19:02.550 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530588+0000", 2023-10-01T09:19:02.550 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:19:02.550 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.550 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.550 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:19:02.550 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530588+0000", 2023-10-01T09:19:02.550 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:19:02.550 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.550 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.550 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:19:02.550 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530588+0000", 2023-10-01T09:19:02.551 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.659e-06 2023-10-01T09:19:02.551 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.551 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.551 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:19:02.551 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530592+0000", 2023-10-01T09:19:02.551 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9630000000000003e-06 2023-10-01T09:19:02.551 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.551 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.551 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:19:02.551 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530597+0000", 2023-10-01T09:19:02.551 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9620000000000004e-06 2023-10-01T09:19:02.551 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:19:02.552 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530601+0000", 2023-10-01T09:19:02.552 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00037171900000000001 2023-10-01T09:19:02.552 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:19:02.552 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.530973+0000", 2023-10-01T09:19:02.552 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000100024 2023-10-01T09:19:02.552 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:19:02.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.531073+0000", 2023-10-01T09:19:02.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000105682 2023-10-01T09:19:02.553 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.553 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.553 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:19:02.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.531179+0000", 2023-10-01T09:19:02.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0042582640000000003 2023-10-01T09:19:02.553 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.553 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.553 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:19:02.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.535437+0000", 2023-10-01T09:19:02.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002955142 2023-10-01T09:19:02.554 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.554 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.554 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:02.554 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.538392+0000", 2023-10-01T09:19:02.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078038459999999997 2023-10-01T09:19:02.554 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:19:02.554 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:19:02.554 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:19:02.554 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.554 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.554 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:226202 5.9 5:9750eccb:::benchmark_data_smithi138_240338_object111065:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T09:19:02.555 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:19:02.539261+0000", 2023-10-01T09:19:02.555 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 4294967295.999989, 2023-10-01T09:19:02.555 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000230516, 2023-10-01T09:19:02.555 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:19:02.555 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:19:02.555 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:19:02.555 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:19:02.555 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:02.555 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 226202 2023-10-01T09:19:02.555 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.555 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:19:02.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.556 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:19:02.556 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.539261+0000", 2023-10-01T09:19:02.556 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:19:02.556 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.556 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:19:02.556 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.539261+0000", 2023-10-01T09:19:02.556 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:19:02.556 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.556 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:19:02.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.539259+0000", 2023-10-01T09:19:02.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7480000000000005e-06 2023-10-01T09:19:02.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:19:02.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.539268+0000", 2023-10-01T09:19:02.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2533e-05 2023-10-01T09:19:02.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:19:02.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.539281+0000", 2023-10-01T09:19:02.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6359999999999997e-06 2023-10-01T09:19:02.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:19:02.558 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.539288+0000", 2023-10-01T09:19:02.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6932999999999999e-05 2023-10-01T09:19:02.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:19:02.558 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.539335+0000", 2023-10-01T09:19:02.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014699400000000001 2023-10-01T09:19:02.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:19:02.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:19:02.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:19:02.559 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:02.539481+0000", 2023-10-01T09:19:02.559 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00022066800000000001 2023-10-01T09:19:02.559 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:19:02.559 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:19:02.559 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:19:02.559 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:19:02.559 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:19:02.559 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 3 2023-10-01T09:19:02.559 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:19:02.676 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:02.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:02.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:02.782 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:02.852 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:02.937 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:02.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:02.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:02.963 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:03.041 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:19:03.138 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:03.309 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:19:03.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:03.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:19:03.343 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:19:03.415 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:19:03.505 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:03.671 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:19:03.682 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:03 smithi190 ceph-mon[160435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:19:03.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:03.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:19:03.719 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:03 smithi138 ceph-mon[185886]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:19:03.719 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:03 smithi138 conmon[185862]: 2023-10-01T09:19:03.716+0000 7f70d64ec700 -1 mon.a@0(leader).osd e164 definitely_dead 0 2023-10-01T09:19:03.719 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:03 smithi138 ceph-mon[192513]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:19:03.783 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:03.794 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:19:03.858 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:03.979 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:03.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:03.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:04.060 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:19:04.091 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:04.119 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:04.320 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:04.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:04.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:04.428 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:04.502 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:04.576 INFO:teuthology.orchestra.run.smithi138.stderr:marked out osd.2. 2023-10-01T09:19:04.611 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:04 smithi190 ceph-mon[160435]: pgmap v416: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 281 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 334/13371 objects misplaced (2.498%); 1.2 MiB/s, 19 objects/s recovering 2023-10-01T09:19:04.611 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4057099532' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:19:04.611 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:04 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:19:04.611 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:04 smithi190 ceph-mon[160435]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-01T09:19:04.800 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:04.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:04.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:04.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:04 smithi138 ceph-mon[192513]: pgmap v416: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 281 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 334/13371 objects misplaced (2.498%); 1.2 MiB/s, 19 objects/s recovering 2023-10-01T09:19:04.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4057099532' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:19:04.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:04 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:19:04.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:04 smithi138 ceph-mon[192513]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-01T09:19:04.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:04 smithi138 ceph-mon[185886]: pgmap v416: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 281 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 334/13371 objects misplaced (2.498%); 1.2 MiB/s, 19 objects/s recovering 2023-10-01T09:19:04.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4057099532' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:19:04.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:04 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:19:04.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:04 smithi138 ceph-mon[185886]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-01T09:19:04.903 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:04.932 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:04.975 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:04.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:04.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:05.079 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:05.106 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:05.148 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:05.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:05.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:05.251 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:05.275 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:19:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:19:04] "GET /metrics HTTP/1.1" 200 36415 "" "Prometheus/2.33.4" 2023-10-01T09:19:05.278 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:05.422 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:05.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:05.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:05.480 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:05.526 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:05.567 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:05.612 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:05.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:05.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:05.716 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:05.732 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:05 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-01T09:19:05.732 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:05 smithi190 ceph-mon[160435]: osdmap e165: 8 total, 8 up, 7 in 2023-10-01T09:19:05.732 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:05 smithi190 ceph-mon[160435]: failed to encode map e165 with expected crc 2023-10-01T09:19:05.732 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:05 smithi190 ceph-mon[160435]: failed to encode map e165 with expected crc 2023-10-01T09:19:05.732 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:05 smithi190 ceph-mon[160435]: failed to encode map e165 with expected crc 2023-10-01T09:19:05.732 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:05 smithi190 ceph-mon[160435]: pgmap v418: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 281 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 334/13371 objects misplaced (2.498%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:19:05.750 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:05.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:05 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-01T09:19:05.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:05 smithi138 ceph-mon[192513]: osdmap e165: 8 total, 8 up, 7 in 2023-10-01T09:19:05.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:05 smithi138 ceph-mon[192513]: failed to encode map e165 with expected crc 2023-10-01T09:19:05.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:05 smithi138 ceph-mon[192513]: failed to encode map e165 with expected crc 2023-10-01T09:19:05.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:05 smithi138 ceph-mon[192513]: failed to encode map e165 with expected crc 2023-10-01T09:19:05.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:05 smithi138 ceph-mon[192513]: pgmap v418: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 281 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 334/13371 objects misplaced (2.498%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:19:05.814 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:05 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-01T09:19:05.814 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:05 smithi138 ceph-mon[185886]: osdmap e165: 8 total, 8 up, 7 in 2023-10-01T09:19:05.814 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:05 smithi138 ceph-mon[185886]: failed to encode map e165 with expected crc 2023-10-01T09:19:05.814 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:05 smithi138 ceph-mon[185886]: failed to encode map e165 with expected crc 2023-10-01T09:19:05.814 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:05 smithi138 ceph-mon[185886]: failed to encode map e165 with expected crc 2023-10-01T09:19:05.814 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:05 smithi138 ceph-mon[185886]: pgmap v418: 89 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 87 active+clean; 281 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 334/13371 objects misplaced (2.498%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:19:05.850 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:05.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:05.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:05.945 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:19:05.945 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:19:05.945 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:19:05.945 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:19:05.946 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:19:05.954 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:05.981 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:06.026 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:06.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:06.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:06.132 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:06.160 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:06.204 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:06.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:06.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:06.307 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:06.333 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:06.375 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:06.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:06.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:06.479 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:06.506 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:06.559 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:19:06.707 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:19:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:06 smithi190 ceph-mon[160435]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:06 smithi190 ceph-mon[160435]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:06 smithi190 ceph-mon[160435]: osdmap e166: 8 total, 8 up, 7 in 2023-10-01T09:19:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:06 smithi190 ceph-mon[160435]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:06 smithi190 ceph-mon[160435]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:06 smithi190 ceph-mon[160435]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:06 smithi190 ceph-mon[160435]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:06 smithi190 ceph-mon[160435]: 5.a continuing backfill to osd.0 from (121'6581,164'13920] 5:50eac726:::benchmark_data_smithi138_240338_object110690:head to 164'13920 2023-10-01T09:19:06.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:06 smithi190 ceph-mon[160435]: 5.a starting backfill to osd.1 from (121'6381,153'13131] MIN to 164'13920 2023-10-01T09:19:06.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:06 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2887920820' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:19:06.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:06 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:19:06.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[185886]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[185886]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[185886]: osdmap e166: 8 total, 8 up, 7 in 2023-10-01T09:19:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[185886]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[185886]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[185886]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[185886]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[185886]: 5.a continuing backfill to osd.0 from (121'6581,164'13920] 5:50eac726:::benchmark_data_smithi138_240338_object110690:head to 164'13920 2023-10-01T09:19:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[185886]: 5.a starting backfill to osd.1 from (121'6381,153'13131] MIN to 164'13920 2023-10-01T09:19:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2887920820' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:19:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:19:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[192513]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[192513]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[192513]: osdmap e166: 8 total, 8 up, 7 in 2023-10-01T09:19:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[192513]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[192513]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[192513]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[192513]: failed to encode map e166 with expected crc 2023-10-01T09:19:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[192513]: 5.a continuing backfill to osd.0 from (121'6581,164'13920] 5:50eac726:::benchmark_data_smithi138_240338_object110690:head to 164'13920 2023-10-01T09:19:06.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[192513]: 5.a starting backfill to osd.1 from (121'6381,153'13131] MIN to 164'13920 2023-10-01T09:19:06.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2887920820' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:19:06.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:06 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:19:07.221 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:19:07.480 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:07.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:07.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:07.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:19:07.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:19:07.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: 5.8 starting backfill to osd.5 from (0'0,0'0] MAX to 164'14589 2023-10-01T09:19:07.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:19:07.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: osdmap e167: 8 total, 8 up, 7 in 2023-10-01T09:19:07.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: 5.6 starting backfill to osd.4 from (0'0,0'0] MAX to 164'14298 2023-10-01T09:19:07.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: 5.b starting backfill to osd.7 from (121'6626,153'13464] MIN to 164'14311 2023-10-01T09:19:07.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[185886]: pgmap v421: 89 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 81 active+clean; 178 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 592/8391 objects misplaced (7.055%); 1.6 MiB/s, 1 keys/s, 28 objects/s recovering 2023-10-01T09:19:07.588 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:07.633 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:07.805 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:19:07.805 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:19:07.805 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.805 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: 5.8 starting backfill to osd.5 from (0'0,0'0] MAX to 164'14589 2023-10-01T09:19:07.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:19:07.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: osdmap e167: 8 total, 8 up, 7 in 2023-10-01T09:19:07.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: 5.6 starting backfill to osd.4 from (0'0,0'0] MAX to 164'14298 2023-10-01T09:19:07.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: 5.b starting backfill to osd.7 from (121'6626,153'13464] MIN to 164'14311 2023-10-01T09:19:07.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:07 smithi138 ceph-mon[192513]: pgmap v421: 89 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 81 active+clean; 178 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 592/8391 objects misplaced (7.055%); 1.6 MiB/s, 1 keys/s, 28 objects/s recovering 2023-10-01T09:19:07.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:19:07.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:19:07.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: 5.8 starting backfill to osd.5 from (0'0,0'0] MAX to 164'14589 2023-10-01T09:19:07.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:19:07.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: osdmap e167: 8 total, 8 up, 7 in 2023-10-01T09:19:07.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: 5.6 starting backfill to osd.4 from (0'0,0'0] MAX to 164'14298 2023-10-01T09:19:07.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: 5.b starting backfill to osd.7 from (121'6626,153'13464] MIN to 164'14311 2023-10-01T09:19:07.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: failed to encode map e167 with expected crc 2023-10-01T09:19:07.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:07 smithi190 ceph-mon[160435]: pgmap v421: 89 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 81 active+clean; 178 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 592/8391 objects misplaced (7.055%); 1.6 MiB/s, 1 keys/s, 28 objects/s recovering 2023-10-01T09:19:07.821 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:07.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:07.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:07.930 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:08.031 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:08.066 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:19:08.096 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:19:08.128 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:19:08.129 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:08.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:08.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:08.152 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:19:08.195 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:19:08.231 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:19:08.239 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:08.267 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:19:08.286 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:08.334 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:08.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:08.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:08.438 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:08.466 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:08.492 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:19:08.497 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:19:08.497 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:19:08.497 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:19:08.497 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:19:08.497 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:19:08.497 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:19:08.497 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:19:08.497 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:19:08.539 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:08.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:08.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:08.646 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:19:08.706 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:08.781 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:19:08.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:08.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:19:08.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:08 smithi190 ceph-mon[160435]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 164'13915 2023-10-01T09:19:08.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:08 smithi190 ceph-mon[160435]: 5.d starting backfill to osd.0 from (0'0,0'0] MAX to 164'14240 2023-10-01T09:19:08.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:08 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2017202879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:08.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:08 smithi190 ceph-mon[160435]: failed to encode map e168 with expected crc 2023-10-01T09:19:08.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:08 smithi190 ceph-mon[160435]: osdmap e168: 8 total, 8 up, 7 in 2023-10-01T09:19:08.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:08 smithi190 ceph-mon[160435]: failed to encode map e168 with expected crc 2023-10-01T09:19:08.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:08 smithi138 ceph-mon[192513]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 164'13915 2023-10-01T09:19:08.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:08 smithi138 ceph-mon[192513]: 5.d starting backfill to osd.0 from (0'0,0'0] MAX to 164'14240 2023-10-01T09:19:08.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:08 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2017202879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:08.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:08 smithi138 ceph-mon[192513]: failed to encode map e168 with expected crc 2023-10-01T09:19:08.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:08 smithi138 ceph-mon[192513]: osdmap e168: 8 total, 8 up, 7 in 2023-10-01T09:19:08.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:08 smithi138 ceph-mon[192513]: failed to encode map e168 with expected crc 2023-10-01T09:19:08.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:08 smithi138 ceph-mon[185886]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 164'13915 2023-10-01T09:19:08.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:08 smithi138 ceph-mon[185886]: 5.d starting backfill to osd.0 from (0'0,0'0] MAX to 164'14240 2023-10-01T09:19:08.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:08 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2017202879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:08.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:08 smithi138 ceph-mon[185886]: failed to encode map e168 with expected crc 2023-10-01T09:19:08.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:08 smithi138 ceph-mon[185886]: osdmap e168: 8 total, 8 up, 7 in 2023-10-01T09:19:08.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:08 smithi138 ceph-mon[185886]: failed to encode map e168 with expected crc 2023-10-01T09:19:08.889 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:08.938 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:09.031 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:09.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:09.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:09.134 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:09.167 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:09.267 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:09.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:09.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:09.371 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:09.405 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:09.407 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:19:09.407 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:19:09.584 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:19:09.692 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:09 smithi138 ceph-mon[192513]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.693 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:09 smithi138 ceph-mon[192513]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.693 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:09 smithi138 ceph-mon[192513]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.693 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:09 smithi138 ceph-mon[192513]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.693 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:09 smithi138 ceph-mon[192513]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.693 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:09 smithi138 ceph-mon[192513]: osdmap e169: 8 total, 8 up, 7 in 2023-10-01T09:19:09.693 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:09 smithi138 ceph-mon[192513]: pgmap v424: 89 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 81 active+clean; 163 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 592/7704 objects misplaced (7.684%); 1.1 MiB/s, 2 keys/s, 23 objects/s recovering 2023-10-01T09:19:09.693 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:09 smithi138 ceph-mon[185886]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.693 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:09 smithi138 ceph-mon[185886]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.693 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:09 smithi138 ceph-mon[185886]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.694 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:09 smithi138 ceph-mon[185886]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.694 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:09 smithi138 ceph-mon[185886]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.694 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:09 smithi138 ceph-mon[185886]: osdmap e169: 8 total, 8 up, 7 in 2023-10-01T09:19:09.694 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:09 smithi138 ceph-mon[185886]: pgmap v424: 89 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 81 active+clean; 163 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 592/7704 objects misplaced (7.684%); 1.1 MiB/s, 2 keys/s, 23 objects/s recovering 2023-10-01T09:19:09.705 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:09.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:09.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:09.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:09 smithi190 ceph-mon[160435]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:09 smithi190 ceph-mon[160435]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:09 smithi190 ceph-mon[160435]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:09 smithi190 ceph-mon[160435]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:09 smithi190 ceph-mon[160435]: failed to encode map e168 with expected crc 2023-10-01T09:19:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:09 smithi190 ceph-mon[160435]: osdmap e169: 8 total, 8 up, 7 in 2023-10-01T09:19:09.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:09 smithi190 ceph-mon[160435]: pgmap v424: 89 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 81 active+clean; 163 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 592/7704 objects misplaced (7.684%); 1.1 MiB/s, 2 keys/s, 23 objects/s recovering 2023-10-01T09:19:09.811 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:19:09.839 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:09.885 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:19:09.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:09.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:19:09.986 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:09.990 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:10.022 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:10.178 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:10.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:10.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:10.284 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:10.320 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:10.350 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:19:10.387 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:10.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:10.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:10.493 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:10.508 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:19:10.508 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:19:10.508 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-10-01T09:19:10.508 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:19:10.508 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:19:10.509 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-10-01T09:19:10.537 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:10.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:10 smithi190 ceph-mon[160435]: failed to encode map e169 with expected crc 2023-10-01T09:19:10.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:10 smithi190 ceph-mon[160435]: failed to encode map e169 with expected crc 2023-10-01T09:19:10.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:10 smithi190 ceph-mon[160435]: failed to encode map e169 with expected crc 2023-10-01T09:19:10.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:10 smithi190 ceph-mon[160435]: failed to encode map e169 with expected crc 2023-10-01T09:19:10.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:10 smithi190 ceph-mon[160435]: failed to encode map e170 with expected crc 2023-10-01T09:19:10.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:10 smithi190 ceph-mon[160435]: osdmap e170: 8 total, 8 up, 7 in 2023-10-01T09:19:10.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:10.564 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:19:10.595 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:10.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:10.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:10.705 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:10.731 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:10.777 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:10.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:10.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:10.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[192513]: failed to encode map e169 with expected crc 2023-10-01T09:19:10.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[192513]: failed to encode map e169 with expected crc 2023-10-01T09:19:10.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[192513]: failed to encode map e169 with expected crc 2023-10-01T09:19:10.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[192513]: failed to encode map e169 with expected crc 2023-10-01T09:19:10.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[192513]: failed to encode map e170 with expected crc 2023-10-01T09:19:10.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[192513]: osdmap e170: 8 total, 8 up, 7 in 2023-10-01T09:19:10.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:10.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:19:10.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[185886]: failed to encode map e169 with expected crc 2023-10-01T09:19:10.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[185886]: failed to encode map e169 with expected crc 2023-10-01T09:19:10.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[185886]: failed to encode map e169 with expected crc 2023-10-01T09:19:10.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[185886]: failed to encode map e169 with expected crc 2023-10-01T09:19:10.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[185886]: failed to encode map e170 with expected crc 2023-10-01T09:19:10.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[185886]: osdmap e170: 8 total, 8 up, 7 in 2023-10-01T09:19:10.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:10.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:19:10.886 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:10.911 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:11.366 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:11.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:11.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:11.469 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:11.494 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:11.525 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:11.535 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:11.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:11.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:11.617 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:11.640 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:11.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:11 smithi138 ceph-mon[185886]: failed to encode map e170 with expected crc 2023-10-01T09:19:11.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:11 smithi138 ceph-mon[185886]: osdmap e171: 8 total, 8 up, 7 in 2023-10-01T09:19:11.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:11 smithi138 ceph-mon[185886]: failed to encode map e171 with expected crc 2023-10-01T09:19:11.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:11 smithi138 ceph-mon[185886]: failed to encode map e171 with expected crc 2023-10-01T09:19:11.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:11 smithi138 ceph-mon[185886]: pgmap v427: 89 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 81 active+clean; 163 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 57 op/s; 592/7704 objects misplaced (7.684%) 2023-10-01T09:19:11.667 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:11 smithi138 ceph-mon[192513]: failed to encode map e170 with expected crc 2023-10-01T09:19:11.667 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:11 smithi138 ceph-mon[192513]: osdmap e171: 8 total, 8 up, 7 in 2023-10-01T09:19:11.667 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:11 smithi138 ceph-mon[192513]: failed to encode map e171 with expected crc 2023-10-01T09:19:11.667 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:11 smithi138 ceph-mon[192513]: failed to encode map e171 with expected crc 2023-10-01T09:19:11.667 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:11 smithi138 ceph-mon[192513]: pgmap v427: 89 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 81 active+clean; 163 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 57 op/s; 592/7704 objects misplaced (7.684%) 2023-10-01T09:19:11.669 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:11.712 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:19:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:11 smithi190 ceph-mon[160435]: failed to encode map e170 with expected crc 2023-10-01T09:19:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:11 smithi190 ceph-mon[160435]: osdmap e171: 8 total, 8 up, 7 in 2023-10-01T09:19:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:11 smithi190 ceph-mon[160435]: failed to encode map e171 with expected crc 2023-10-01T09:19:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:11 smithi190 ceph-mon[160435]: failed to encode map e171 with expected crc 2023-10-01T09:19:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:11 smithi190 ceph-mon[160435]: pgmap v427: 89 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 4 remapped+peering, 1 peering, 81 active+clean; 163 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 57 op/s; 592/7704 objects misplaced (7.684%) 2023-10-01T09:19:11.885 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:11.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:11.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:11.989 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:19:12.016 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:12.059 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:19:12.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:12.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:19:12.167 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:12.210 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:12.467 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:12.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:12.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:12.570 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:12.596 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:12.637 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:12.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:12.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:12.744 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:12.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[192513]: failed to encode map e171 with expected crc 2023-10-01T09:19:12.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[192513]: failed to encode map e171 with expected crc 2023-10-01T09:19:12.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[192513]: failed to encode map e171 with expected crc 2023-10-01T09:19:12.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[192513]: failed to encode map e172 with expected crc 2023-10-01T09:19:12.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[192513]: osdmap e172: 8 total, 8 up, 7 in 2023-10-01T09:19:12.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[192513]: failed to encode map e172 with expected crc 2023-10-01T09:19:12.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[192513]: failed to encode map e172 with expected crc 2023-10-01T09:19:12.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[192513]: failed to encode map e172 with expected crc 2023-10-01T09:19:12.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/452117459' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:19:12.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:19:12.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[185886]: failed to encode map e171 with expected crc 2023-10-01T09:19:12.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[185886]: failed to encode map e171 with expected crc 2023-10-01T09:19:12.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[185886]: failed to encode map e171 with expected crc 2023-10-01T09:19:12.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[185886]: failed to encode map e172 with expected crc 2023-10-01T09:19:12.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[185886]: osdmap e172: 8 total, 8 up, 7 in 2023-10-01T09:19:12.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[185886]: failed to encode map e172 with expected crc 2023-10-01T09:19:12.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[185886]: failed to encode map e172 with expected crc 2023-10-01T09:19:12.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[185886]: failed to encode map e172 with expected crc 2023-10-01T09:19:12.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/452117459' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:19:12.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:12 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:19:12.775 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:13.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:12 smithi190 ceph-mon[160435]: failed to encode map e171 with expected crc 2023-10-01T09:19:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:12 smithi190 ceph-mon[160435]: failed to encode map e171 with expected crc 2023-10-01T09:19:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:12 smithi190 ceph-mon[160435]: failed to encode map e171 with expected crc 2023-10-01T09:19:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:12 smithi190 ceph-mon[160435]: failed to encode map e172 with expected crc 2023-10-01T09:19:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:12 smithi190 ceph-mon[160435]: osdmap e172: 8 total, 8 up, 7 in 2023-10-01T09:19:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:12 smithi190 ceph-mon[160435]: failed to encode map e172 with expected crc 2023-10-01T09:19:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:12 smithi190 ceph-mon[160435]: failed to encode map e172 with expected crc 2023-10-01T09:19:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:12 smithi190 ceph-mon[160435]: failed to encode map e172 with expected crc 2023-10-01T09:19:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:12 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/452117459' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:19:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:12 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:19:13.075 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:19:13.751 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Removed 115134 objects 2023-10-01T09:19:13.751 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Clean up completed and total clean up time :179.208 2023-10-01T09:19:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:13 smithi138 ceph-mon[185886]: failed to encode map e172 with expected crc 2023-10-01T09:19:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:13 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:19:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:13 smithi138 ceph-mon[185886]: failed to encode map e173 with expected crc 2023-10-01T09:19:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:13 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:19:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:13 smithi138 ceph-mon[185886]: osdmap e173: 8 total, 8 up, 7 in 2023-10-01T09:19:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:13 smithi138 ceph-mon[185886]: failed to encode map e173 with expected crc 2023-10-01T09:19:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:13 smithi138 ceph-mon[185886]: pgmap v430: 89 pgs: 1 peering, 86 active+clean, 2 active+clean+remapped; 42 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 6.8 MiB/s, 111 objects/s recovering 2023-10-01T09:19:13.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:13 smithi138 ceph-mon[192513]: failed to encode map e172 with expected crc 2023-10-01T09:19:13.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:13 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:19:13.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:13 smithi138 ceph-mon[192513]: failed to encode map e173 with expected crc 2023-10-01T09:19:13.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:13 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:19:13.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:13 smithi138 ceph-mon[192513]: osdmap e173: 8 total, 8 up, 7 in 2023-10-01T09:19:13.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:13 smithi138 ceph-mon[192513]: failed to encode map e173 with expected crc 2023-10-01T09:19:13.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:13 smithi138 ceph-mon[192513]: pgmap v430: 89 pgs: 1 peering, 86 active+clean, 2 active+clean+remapped; 42 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 6.8 MiB/s, 111 objects/s recovering 2023-10-01T09:19:13.977 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:13.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:13.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:14.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:13 smithi190 ceph-mon[160435]: failed to encode map e172 with expected crc 2023-10-01T09:19:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:13 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:19:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:13 smithi190 ceph-mon[160435]: failed to encode map e173 with expected crc 2023-10-01T09:19:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:13 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:19:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:13 smithi190 ceph-mon[160435]: osdmap e173: 8 total, 8 up, 7 in 2023-10-01T09:19:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:13 smithi190 ceph-mon[160435]: failed to encode map e173 with expected crc 2023-10-01T09:19:14.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:13 smithi190 ceph-mon[160435]: pgmap v430: 89 pgs: 1 peering, 86 active+clean, 2 active+clean+remapped; 42 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 6.8 MiB/s, 111 objects/s recovering 2023-10-01T09:19:14.081 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:14.108 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:14.203 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:14.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:14.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:14.307 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:14.333 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:14.376 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:14.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:14.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:14.484 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:14.511 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:14.856 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:14.958 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:14.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:14.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:15.029 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:19:15.029 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:19:15.063 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:15.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:19:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:19:14] "GET /metrics HTTP/1.1" 200 36415 "" "Prometheus/2.33.4" 2023-10-01T09:19:15.090 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:15.864 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:15.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:15.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:15.967 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:15.985 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:19:16.000 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:16.049 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:16.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:16.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:16.153 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:16.181 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:16.225 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:16.229 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:19:16.229 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T09:19:16.229 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:19:16.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:16.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:16.259 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:19:16.333 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:16.359 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:16.455 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:16.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:16.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:16.562 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:19:16.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:16 smithi138 ceph-mon[192513]: pgmap v431: 89 pgs: 1 peering, 86 active+clean, 2 active+clean+remapped; 42 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 5.1 MiB/s, 84 objects/s recovering 2023-10-01T09:19:16.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:16 smithi138 ceph-mon[185886]: pgmap v431: 89 pgs: 1 peering, 86 active+clean, 2 active+clean+remapped; 42 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 5.1 MiB/s, 84 objects/s recovering 2023-10-01T09:19:16.588 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:16.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:16 smithi190 ceph-mon[160435]: pgmap v431: 89 pgs: 1 peering, 86 active+clean, 2 active+clean+remapped; 42 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 5.1 MiB/s, 84 objects/s recovering 2023-10-01T09:19:17.183 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:19:17.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:17.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:19:17.287 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:19:17.319 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:17.367 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:19:17.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:17.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:19:17.472 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:17.511 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:17.546 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:17.564 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:17.669 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:17.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:17.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:17.773 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:17.802 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:17.946 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:17.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:17.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:17.997 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:19:18.015 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:19:18.033 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:19:18.051 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:18.055 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:19:18.058 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:18 smithi138 ceph-mon[185886]: pgmap v432: 89 pgs: 89 active+clean; 8.0 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 4.3 MiB/s, 70 objects/s recovering 2023-10-01T09:19:18.058 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:18 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/859118166' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:18.077 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:19:18.082 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:18.102 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:19:18.122 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:19:18.141 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:19:18.142 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:19:18.142 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:19:18.142 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:19:18.142 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:19:18.142 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:19:18.142 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:19:18.143 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:19:18.143 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:19:18.185 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:18.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:18.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:18.290 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:19:18.318 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:18 smithi138 ceph-mon[192513]: pgmap v432: 89 pgs: 89 active+clean; 8.0 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 4.3 MiB/s, 70 objects/s recovering 2023-10-01T09:19:18.318 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:18 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/859118166' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:18.321 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:18.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:18 smithi190 ceph-mon[160435]: pgmap v432: 89 pgs: 89 active+clean; 8.0 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 4.3 MiB/s, 70 objects/s recovering 2023-10-01T09:19:18.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:18 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/859118166' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:19.230 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-10-01T09:19:19.231 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-10-01T09:19:19.569 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:19:19.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:19.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:19:19.673 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:19.704 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:19.753 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:19.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:19.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:19.860 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:19.888 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:19.931 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:19.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:19.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:20.038 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:20.065 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:20.105 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:20.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:20.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:20.212 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:20.237 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:20.279 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:20.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:20.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:20.384 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:19:20.411 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:20.453 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:19:20.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:20.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:19:20.561 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:20.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:20 smithi138 ceph-mon[192513]: pgmap v433: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 317 op/s; 3.4 MiB/s, 55 objects/s recovering 2023-10-01T09:19:20.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:20 smithi138 ceph-mon[185886]: pgmap v433: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 317 op/s; 3.4 MiB/s, 55 objects/s recovering 2023-10-01T09:19:20.589 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:20.629 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:20.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:20.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:20.645 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:20.737 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:20.761 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:20 smithi190 ceph-mon[160435]: pgmap v433: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 317 op/s; 3.4 MiB/s, 55 objects/s recovering 2023-10-01T09:19:20.763 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:20.802 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:19:20.802 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:19:20.802 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:19:20.802 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:19:20.802 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:19:20.804 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:20.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:20.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:20.909 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:20.935 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:20.977 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:20.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:20.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:21.084 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:19:21.114 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:21.158 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:19:21.464 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:19:21.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:21 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4093598951' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:19:21.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:21 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4093598951' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:19:21.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:21 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4093598951' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:19:21.877 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:19:21.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:21.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:19:21.981 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:22.014 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:22.019 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:19:22.443 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:19:22.444 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:19:22.444 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-01T09:19:22.444 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-01T09:19:22.676 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:22 smithi138 ceph-mon[192513]: pgmap v434: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 160 op/s; 1.8 MiB/s, 28 objects/s recovering 2023-10-01T09:19:22.676 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:22 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:19:22.676 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:22 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4093598951' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:19:22.676 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:22 smithi138 ceph-mon[192513]: osdmap e174: 8 total, 8 up, 7 in 2023-10-01T09:19:22.676 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:22 smithi138 ceph-mon[192513]: failed to encode map e174 with expected crc 2023-10-01T09:19:22.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:22 smithi138 ceph-mon[185886]: pgmap v434: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 160 op/s; 1.8 MiB/s, 28 objects/s recovering 2023-10-01T09:19:22.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:22 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:19:22.677 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:22 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4093598951' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:19:22.677 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:22 smithi138 ceph-mon[185886]: osdmap e174: 8 total, 8 up, 7 in 2023-10-01T09:19:22.677 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:22 smithi138 ceph-mon[185886]: failed to encode map e174 with expected crc 2023-10-01T09:19:22.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:22 smithi190 ceph-mon[160435]: pgmap v434: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 160 op/s; 1.8 MiB/s, 28 objects/s recovering 2023-10-01T09:19:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:22 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:19:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:22 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4093598951' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:19:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:22 smithi190 ceph-mon[160435]: osdmap e174: 8 total, 8 up, 7 in 2023-10-01T09:19:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:22 smithi190 ceph-mon[160435]: failed to encode map e174 with expected crc 2023-10-01T09:19:22.814 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:22.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:22.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:22.919 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:22.947 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:23.023 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:23.115 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:23.140 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:23.272 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:23.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:23.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:23.379 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:19:23.411 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:23.460 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:19:23.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:23.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:19:23.568 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:19:23.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:23 smithi138 ceph-mon[192513]: failed to encode map e174 with expected crc 2023-10-01T09:19:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:23 smithi138 ceph-mon[192513]: failed to encode map e174 with expected crc 2023-10-01T09:19:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:23 smithi138 ceph-mon[192513]: failed to encode map e174 with expected crc 2023-10-01T09:19:23.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:23 smithi138 ceph-mon[185886]: failed to encode map e174 with expected crc 2023-10-01T09:19:23.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:23 smithi138 ceph-mon[185886]: failed to encode map e174 with expected crc 2023-10-01T09:19:23.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:23 smithi138 ceph-mon[185886]: failed to encode map e174 with expected crc 2023-10-01T09:19:23.598 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:23.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:23 smithi190 ceph-mon[160435]: failed to encode map e174 with expected crc 2023-10-01T09:19:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:23 smithi190 ceph-mon[160435]: failed to encode map e174 with expected crc 2023-10-01T09:19:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:23 smithi190 ceph-mon[160435]: failed to encode map e174 with expected crc 2023-10-01T09:19:23.961 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:19:23.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:23.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:19:24.065 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:24.096 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:24.478 INFO:teuthology.orchestra.run.smithi138.stderr:pool 'unique_pool_0' removed 2023-10-01T09:19:24.679 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:19:24.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:24 smithi138 ceph-mon[192513]: pgmap v436: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 59 op/s 2023-10-01T09:19:24.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3868190961' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:19:24.717 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:24 smithi138 ceph-mon[185886]: pgmap v436: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 59 op/s 2023-10-01T09:19:24.717 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3868190961' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:19:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:24 smithi190 ceph-mon[160435]: pgmap v436: 89 pgs: 89 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 59 op/s 2023-10-01T09:19:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3868190961' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:19:25.010 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:19:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:19:24] "GET /metrics HTTP/1.1" 200 36415 "" "Prometheus/2.33.4" 2023-10-01T09:19:25.610 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:25.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:25.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:25.714 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:25.738 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[185886]: failed to encode map e175 with expected crc 2023-10-01T09:19:25.738 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3868190961' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:19:25.739 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[185886]: osdmap e175: 8 total, 8 up, 7 in 2023-10-01T09:19:25.739 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[185886]: failed to encode map e175 with expected crc 2023-10-01T09:19:25.739 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[185886]: failed to encode map e175 with expected crc 2023-10-01T09:19:25.739 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[185886]: failed to encode map e175 with expected crc 2023-10-01T09:19:25.739 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:25.739 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:19:25.739 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:25.739 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[192513]: failed to encode map e175 with expected crc 2023-10-01T09:19:25.739 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3868190961' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:19:25.739 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[192513]: osdmap e175: 8 total, 8 up, 7 in 2023-10-01T09:19:25.740 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[192513]: failed to encode map e175 with expected crc 2023-10-01T09:19:25.740 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[192513]: failed to encode map e175 with expected crc 2023-10-01T09:19:25.740 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[192513]: failed to encode map e175 with expected crc 2023-10-01T09:19:25.740 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:25.740 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:19:25.740 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:25.741 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:25 smithi190 ceph-mon[160435]: failed to encode map e175 with expected crc 2023-10-01T09:19:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:25 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3868190961' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:19:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:25 smithi190 ceph-mon[160435]: osdmap e175: 8 total, 8 up, 7 in 2023-10-01T09:19:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:25 smithi190 ceph-mon[160435]: failed to encode map e175 with expected crc 2023-10-01T09:19:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:25 smithi190 ceph-mon[160435]: failed to encode map e175 with expected crc 2023-10-01T09:19:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:25 smithi190 ceph-mon[160435]: failed to encode map e175 with expected crc 2023-10-01T09:19:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:19:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:25.902 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T09:19:25.902 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T09:19:25.903 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-10-01T09:19:25.903 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-10-01T09:19:25.989 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:25.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:25.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:26.094 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:26.120 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:26 smithi138 ceph-mon[192513]: pgmap v438: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:19:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:26 smithi138 ceph-mon[185886]: pgmap v438: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:19:26.616 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:26.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:26.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:26.721 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:26.752 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:26 smithi190 ceph-mon[160435]: pgmap v438: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:19:26.755 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:26.798 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:19:26.798 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:19:26.798 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:19:26.798 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:19:26.799 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.799 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:121341 5.b 5:db80cfa5:::benchmark_data_smithi138_240338_object6204:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e125)", 2023-10-01T09:19:26.799 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:16:22.525767+0000", 2023-10-01T09:19:26.799 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 184.26921387600001, 2023-10-01T09:19:26.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.010709842, 2023-10-01T09:19:26.799 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.799 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:19:26.799 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.799 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.799 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.800 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121341 2023-10-01T09:19:26.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.800 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525767+0000", 2023-10-01T09:19:26.800 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525767+0000", 2023-10-01T09:19:26.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.801 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.801 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.801 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.801 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525766+0000", 2023-10-01T09:19:26.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:19:26.801 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.801 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.801 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.802 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525769+0000", 2023-10-01T09:19:26.802 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.993e-06 2023-10-01T09:19:26.802 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.802 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.802 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.802 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525776+0000", 2023-10-01T09:19:26.802 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3829999999999997e-06 2023-10-01T09:19:26.802 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.802 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.803 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.803 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525782+0000", 2023-10-01T09:19:26.803 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7440000000000001e-06 2023-10-01T09:19:26.803 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.803 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.803 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.803 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525829+0000", 2023-10-01T09:19:26.803 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7169999999999997e-05 2023-10-01T09:19:26.803 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.804 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.804 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525840+0000", 2023-10-01T09:19:26.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0494e-05 2023-10-01T09:19:26.804 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.804 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.804 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.534631+0000", 2023-10-01T09:19:26.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0087906170000001 2023-10-01T09:19:26.804 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.536476+0000", 2023-10-01T09:19:26.805 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001845867 2023-10-01T09:19:26.805 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.805 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.805 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.805 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.806 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:121349 5.b 5:dee15a14:::benchmark_data_smithi138_240338_object6212:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e125)", 2023-10-01T09:19:26.806 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:16:22.525794+0000", 2023-10-01T09:19:26.806 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 184.26918698899999, 2023-10-01T09:19:26.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0346446920000001, 2023-10-01T09:19:26.806 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.806 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.806 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.806 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.806 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.807 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121349 2023-10-01T09:19:26.807 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.808 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.808 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.808 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.808 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525794+0000", 2023-10-01T09:19:26.808 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.808 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.808 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.809 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.809 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525794+0000", 2023-10-01T09:19:26.809 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.809 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.809 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.809 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.809 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525793+0000", 2023-10-01T09:19:26.809 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:19:26.809 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.809 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.809 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.810 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525796+0000", 2023-10-01T09:19:26.810 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4490000000000002e-06 2023-10-01T09:19:26.810 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.810 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.810 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.810 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525801+0000", 2023-10-01T09:19:26.810 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.536e-06 2023-10-01T09:19:26.810 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.810 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.810 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.811 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525806+0000", 2023-10-01T09:19:26.811 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7330000000000003e-06 2023-10-01T09:19:26.811 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.812 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.812 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.812 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525866+0000", 2023-10-01T09:19:26.812 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0538999999999998e-05 2023-10-01T09:19:26.812 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.812 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.812 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.812 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525875+0000", 2023-10-01T09:19:26.812 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.772e-06 2023-10-01T09:19:26.812 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.812 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.813 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.536750+0000", 2023-10-01T09:19:26.813 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0108747680000001 2023-10-01T09:19:26.813 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.813 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.813 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.537437+0000", 2023-10-01T09:19:26.813 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00068692000000000004 2023-10-01T09:19:26.813 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.813 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.813 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:19:26.813 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.537550+0000", 2023-10-01T09:19:26.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011352 2023-10-01T09:19:26.814 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.814 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.814 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.547975+0000", 2023-10-01T09:19:26.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010424638999999999 2023-10-01T09:19:26.814 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.814 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.814 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.814 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.548036+0000", 2023-10-01T09:19:26.814 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0909000000000002e-05 2023-10-01T09:19:26.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.815 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.815 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560353+0000", 2023-10-01T09:19:26.815 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012316720999999999 2023-10-01T09:19:26.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.815 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.815 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.815 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560397+0000", 2023-10-01T09:19:26.815 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4697999999999998e-05 2023-10-01T09:19:26.815 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.816 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.816 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.816 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560438+0000", 2023-10-01T09:19:26.816 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0871000000000002e-05 2023-10-01T09:19:26.816 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.816 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.816 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.816 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.816 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.816 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:121358 5.8 5:166d68c4:::benchmark_data_smithi138_240338_object6221:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:19:26.816 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:16:22.539405+0000", 2023-10-01T09:19:26.817 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 184.255575462, 2023-10-01T09:19:26.817 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0166601280000001, 2023-10-01T09:19:26.817 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.817 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.817 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.817 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.817 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.817 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121358 2023-10-01T09:19:26.817 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.817 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.817 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.817 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.817 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539405+0000", 2023-10-01T09:19:26.818 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.818 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.818 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.818 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539405+0000", 2023-10-01T09:19:26.818 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.818 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.818 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.818 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.818 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539403+0000", 2023-10-01T09:19:26.818 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:19:26.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.819 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.819 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.819 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539413+0000", 2023-10-01T09:19:26.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8060000000000006e-06 2023-10-01T09:19:26.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.819 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.819 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.819 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539434+0000", 2023-10-01T09:19:26.819 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0557e-05 2023-10-01T09:19:26.819 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.819 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.820 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539444+0000", 2023-10-01T09:19:26.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0197000000000001e-05 2023-10-01T09:19:26.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.820 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539526+0000", 2023-10-01T09:19:26.820 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2010999999999998e-05 2023-10-01T09:19:26.820 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.820 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.820 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.820 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539539+0000", 2023-10-01T09:19:26.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3654e-05 2023-10-01T09:19:26.821 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.535296+0000", 2023-10-01T09:19:26.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99575626900000003 2023-10-01T09:19:26.821 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.821 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.821 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.821 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.535893+0000", 2023-10-01T09:19:26.821 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00059725200000000005 2023-10-01T09:19:26.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.822 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-10-01T09:19:26.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.535973+0000", 2023-10-01T09:19:26.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0136000000000007e-05 2023-10-01T09:19:26.822 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.822 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.822 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.822 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.546652+0000", 2023-10-01T09:19:26.822 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01067912 2023-10-01T09:19:26.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.823 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.546866+0000", 2023-10-01T09:19:26.823 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021356899999999999 2023-10-01T09:19:26.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.823 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.823 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.555975+0000", 2023-10-01T09:19:26.823 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0091089359999999998 2023-10-01T09:19:26.823 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.823 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.824 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.556025+0000", 2023-10-01T09:19:26.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0182999999999997e-05 2023-10-01T09:19:26.824 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.824 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.824 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.824 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.556065+0000", 2023-10-01T09:19:26.824 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0333000000000002e-05 2023-10-01T09:19:26.824 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.824 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.824 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.824 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.825 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.825 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:121363 5.b 5:d08c45c8:::benchmark_data_smithi138_240338_object6226:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:19:26.825 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:16:22.544661+0000", 2023-10-01T09:19:26.825 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 184.250319112, 2023-10-01T09:19:26.825 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.015913072, 2023-10-01T09:19:26.825 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.825 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.825 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.825 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.825 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.825 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121363 2023-10-01T09:19:26.826 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.826 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.826 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.826 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.826 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544661+0000", 2023-10-01T09:19:26.826 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.826 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.826 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.826 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.826 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544661+0000", 2023-10-01T09:19:26.826 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.826 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.827 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.827 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544660+0000", 2023-10-01T09:19:26.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:19:26.827 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.827 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.827 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544666+0000", 2023-10-01T09:19:26.827 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.366e-06 2023-10-01T09:19:26.827 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.827 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.827 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.828 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544677+0000", 2023-10-01T09:19:26.828 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1012000000000001e-05 2023-10-01T09:19:26.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.828 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.828 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.828 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544683+0000", 2023-10-01T09:19:26.828 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.178e-06 2023-10-01T09:19:26.828 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.828 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.828 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544729+0000", 2023-10-01T09:19:26.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6149999999999997e-05 2023-10-01T09:19:26.829 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544739+0000", 2023-10-01T09:19:26.829 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0042e-05 2023-10-01T09:19:26.829 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.829 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.829 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.829 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.541089+0000", 2023-10-01T09:19:26.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99635048000000004 2023-10-01T09:19:26.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.830 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.541181+0000", 2023-10-01T09:19:26.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1774000000000006e-05 2023-10-01T09:19:26.830 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.830 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.830 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:19:26.830 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.541279+0000", 2023-10-01T09:19:26.830 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7918999999999995e-05 2023-10-01T09:19:26.831 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.548094+0000", 2023-10-01T09:19:26.831 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0068148089999999998 2023-10-01T09:19:26.831 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.831 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.831 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.831 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.548102+0000", 2023-10-01T09:19:26.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.875e-06 2023-10-01T09:19:26.832 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.832 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.832 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.832 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560506+0000", 2023-10-01T09:19:26.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012403765000000001 2023-10-01T09:19:26.832 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.832 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.832 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.832 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560543+0000", 2023-10-01T09:19:26.832 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7039000000000002e-05 2023-10-01T09:19:26.832 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.833 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.833 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.833 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560574+0000", 2023-10-01T09:19:26.833 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1794000000000001e-05 2023-10-01T09:19:26.833 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.833 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.833 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.833 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.833 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.833 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:121366 5.b 5:dacdb783:::benchmark_data_smithi138_240338_object6229:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:19:26.833 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:16:22.553337+0000", 2023-10-01T09:19:26.834 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 184.241643549, 2023-10-01T09:19:26.834 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.012110101, 2023-10-01T09:19:26.834 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.834 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.834 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.834 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.834 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.834 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121366 2023-10-01T09:19:26.834 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.834 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.834 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.834 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.835 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553337+0000", 2023-10-01T09:19:26.835 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.835 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.835 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.835 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.835 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553337+0000", 2023-10-01T09:19:26.835 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.835 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.835 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.835 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.835 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553335+0000", 2023-10-01T09:19:26.836 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:19:26.836 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.836 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.836 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.836 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553342+0000", 2023-10-01T09:19:26.836 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0210000000000002e-06 2023-10-01T09:19:26.836 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.836 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.836 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.836 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553356+0000", 2023-10-01T09:19:26.836 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3891e-05 2023-10-01T09:19:26.836 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.837 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.837 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.837 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553363+0000", 2023-10-01T09:19:26.837 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8549999999999996e-06 2023-10-01T09:19:26.837 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.837 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.837 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.837 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553411+0000", 2023-10-01T09:19:26.837 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8300999999999997e-05 2023-10-01T09:19:26.837 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.837 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.837 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.838 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553420+0000", 2023-10-01T09:19:26.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8859999999999993e-06 2023-10-01T09:19:26.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.838 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.838 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.838 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.544670+0000", 2023-10-01T09:19:26.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99124969399999996 2023-10-01T09:19:26.838 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.838 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.838 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.838 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.544794+0000", 2023-10-01T09:19:26.838 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000124003 2023-10-01T09:19:26.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:19:26.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.544915+0000", 2023-10-01T09:19:26.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120844 2023-10-01T09:19:26.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.839 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.839 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560476+0000", 2023-10-01T09:19:26.839 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015561339 2023-10-01T09:19:26.839 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.839 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560612+0000", 2023-10-01T09:19:26.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013593200000000001 2023-10-01T09:19:26.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.565305+0000", 2023-10-01T09:19:26.840 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0046935630000000004 2023-10-01T09:19:26.840 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.840 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.840 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.840 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.565403+0000", 2023-10-01T09:19:26.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7567999999999998e-05 2023-10-01T09:19:26.841 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.841 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.841 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.841 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.565447+0000", 2023-10-01T09:19:26.841 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4085999999999998e-05 2023-10-01T09:19:26.841 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.841 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.841 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.841 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.841 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.841 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:121371 5.b 5:db764039:::benchmark_data_smithi138_240338_object6234:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:19:26.842 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:16:22.554473+0000", 2023-10-01T09:19:26.842 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 184.24050752700001, 2023-10-01T09:19:26.842 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0121766640000001, 2023-10-01T09:19:26.842 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.842 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.842 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.842 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.842 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.842 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121371 2023-10-01T09:19:26.842 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.842 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554473+0000", 2023-10-01T09:19:26.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554473+0000", 2023-10-01T09:19:26.843 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.843 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.843 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.843 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.843 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554472+0000", 2023-10-01T09:19:26.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:19:26.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.844 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.844 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554476+0000", 2023-10-01T09:19:26.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.726e-06 2023-10-01T09:19:26.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.844 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.844 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.844 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554487+0000", 2023-10-01T09:19:26.844 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0890000000000001e-05 2023-10-01T09:19:26.844 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554493+0000", 2023-10-01T09:19:26.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6350000000000003e-06 2023-10-01T09:19:26.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.845 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554527+0000", 2023-10-01T09:19:26.845 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.392e-05 2023-10-01T09:19:26.845 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.845 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.845 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554534+0000", 2023-10-01T09:19:26.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8010000000000004e-06 2023-10-01T09:19:26.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.548141+0000", 2023-10-01T09:19:26.846 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99360651499999997 2023-10-01T09:19:26.846 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.846 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.846 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.846 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.548274+0000", 2023-10-01T09:19:26.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013291900000000001 2023-10-01T09:19:26.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:19:26.847 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.548391+0000", 2023-10-01T09:19:26.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117315 2023-10-01T09:19:26.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.847 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.847 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.847 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.565456+0000", 2023-10-01T09:19:26.847 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017065104000000001 2023-10-01T09:19:26.847 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.848 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.848 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.848 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.566498+0000", 2023-10-01T09:19:26.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001041464 2023-10-01T09:19:26.848 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.848 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.848 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.848 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.566556+0000", 2023-10-01T09:19:26.848 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8624e-05 2023-10-01T09:19:26.848 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.848 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.566601+0000", 2023-10-01T09:19:26.849 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5129000000000001e-05 2023-10-01T09:19:26.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.849 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.849 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.849 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.566650+0000", 2023-10-01T09:19:26.849 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8253e-05 2023-10-01T09:19:26.849 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.849 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.849 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.849 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.850 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.850 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:121373 5.8 5:1129836a:::benchmark_data_smithi138_240338_object6236:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:19:26.850 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:16:22.558483+0000", 2023-10-01T09:19:26.850 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 184.236497813, 2023-10-01T09:19:26.850 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.009962169, 2023-10-01T09:19:26.850 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.850 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.850 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.850 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.850 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.850 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121373 2023-10-01T09:19:26.850 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.851 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.558483+0000", 2023-10-01T09:19:26.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.851 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.851 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.851 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.558483+0000", 2023-10-01T09:19:26.851 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.851 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.558481+0000", 2023-10-01T09:19:26.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:19:26.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.852 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.558486+0000", 2023-10-01T09:19:26.852 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9989999999999999e-06 2023-10-01T09:19:26.852 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.852 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.852 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.558496+0000", 2023-10-01T09:19:26.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1030000000000008e-06 2023-10-01T09:19:26.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.558500+0000", 2023-10-01T09:19:26.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8300000000000003e-06 2023-10-01T09:19:26.853 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.853 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.853 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.853 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.558542+0000", 2023-10-01T09:19:26.853 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1916000000000003e-05 2023-10-01T09:19:26.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.558550+0000", 2023-10-01T09:19:26.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0290000000000005e-06 2023-10-01T09:19:26.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.854 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.854 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.542853+0000", 2023-10-01T09:19:26.854 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98430290399999998 2023-10-01T09:19:26.854 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.854 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.542974+0000", 2023-10-01T09:19:26.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120417 2023-10-01T09:19:26.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-10-01T09:19:26.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.543073+0000", 2023-10-01T09:19:26.855 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9407999999999994e-05 2023-10-01T09:19:26.855 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.855 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.855 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.855 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.546804+0000", 2023-10-01T09:19:26.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037310529999999998 2023-10-01T09:19:26.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.856 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.856 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.856 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.556668+0000", 2023-10-01T09:19:26.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0098638760000000006 2023-10-01T09:19:26.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.856 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.856 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.856 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.568361+0000", 2023-10-01T09:19:26.856 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011693365000000001 2023-10-01T09:19:26.856 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.568409+0000", 2023-10-01T09:19:26.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7258000000000001e-05 2023-10-01T09:19:26.857 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.857 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.857 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.857 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.568445+0000", 2023-10-01T09:19:26.857 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6291000000000002e-05 2023-10-01T09:19:26.857 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.857 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.857 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.858 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.858 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.858 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:209439 5.b 5:dfe595a2:::benchmark_data_smithi138_240338_object94302:head [delete] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-01T09:19:26.858 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:18:30.402237+0000", 2023-10-01T09:19:26.858 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.392743830999997, 2023-10-01T09:19:26.858 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1907945639999999, 2023-10-01T09:19:26.858 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.858 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:19:26.858 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.858 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.858 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.858 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 209439 2023-10-01T09:19:26.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.859 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402237+0000", 2023-10-01T09:19:26.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.859 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.859 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.859 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402237+0000", 2023-10-01T09:19:26.859 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.859 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402235+0000", 2023-10-01T09:19:26.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:19:26.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.860 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402243+0000", 2023-10-01T09:19:26.860 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5229999999999998e-06 2023-10-01T09:19:26.860 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.860 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.860 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402254+0000", 2023-10-01T09:19:26.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1741e-05 2023-10-01T09:19:26.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402262+0000", 2023-10-01T09:19:26.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8019999999999992e-06 2023-10-01T09:19:26.861 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.861 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.861 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.861 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402309+0000", 2023-10-01T09:19:26.861 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6397000000000003e-05 2023-10-01T09:19:26.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.862 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.862 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.862 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402441+0000", 2023-10-01T09:19:26.862 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013244 2023-10-01T09:19:26.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.862 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.862 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:19:26.862 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402560+0000", 2023-10-01T09:19:26.862 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011885800000000001 2023-10-01T09:19:26.862 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.862 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.560276+0000", 2023-10-01T09:19:26.863 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.15771628900000001 2023-10-01T09:19:26.863 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.560286+0000", 2023-10-01T09:19:26.863 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5850000000000006e-06 2023-10-01T09:19:26.863 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.863 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.863 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.863 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404172+0000", 2023-10-01T09:19:26.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.843886358 2023-10-01T09:19:26.864 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.864 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.864 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.864 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404187+0000", 2023-10-01T09:19:26.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4751e-05 2023-10-01T09:19:26.864 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.864 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.864 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.864 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.592961+0000", 2023-10-01T09:19:26.864 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1887738130000001 2023-10-01T09:19:26.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.865 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.865 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.593031+0000", 2023-10-01T09:19:26.865 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0541000000000001e-05 2023-10-01T09:19:26.865 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.865 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.865 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.865 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.865 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.865 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:209448 5.8 5:1e06e43d:::benchmark_data_smithi138_240338_object94311:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-01T09:19:26.865 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:18:30.550675+0000", 2023-10-01T09:19:26.866 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.24430504, 2023-10-01T09:19:26.866 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2198924120000001, 2023-10-01T09:19:26.866 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.866 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.866 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.866 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.866 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.866 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 209448 2023-10-01T09:19:26.866 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.867 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.550675+0000", 2023-10-01T09:19:26.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.867 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.867 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.867 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.550675+0000", 2023-10-01T09:19:26.867 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.867 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.867 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.868 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.550675+0000", 2023-10-01T09:19:26.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:19:26.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.868 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.550678+0000", 2023-10-01T09:19:26.868 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0529999999999998e-06 2023-10-01T09:19:26.868 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.868 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.868 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.868 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.550684+0000", 2023-10-01T09:19:26.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6749999999999999e-06 2023-10-01T09:19:26.869 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.869 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.869 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.550689+0000", 2023-10-01T09:19:26.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9599999999999999e-06 2023-10-01T09:19:26.869 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.869 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.869 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.869 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.559523+0000", 2023-10-01T09:19:26.869 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088342590000000006 2023-10-01T09:19:26.869 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.870 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.870 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.870 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.559534+0000", 2023-10-01T09:19:26.870 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1218e-05 2023-10-01T09:19:26.870 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.870 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.870 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.870 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404403+0000", 2023-10-01T09:19:26.870 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84486879400000003 2023-10-01T09:19:26.870 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.871 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404413+0000", 2023-10-01T09:19:26.871 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9769999999999995e-06 2023-10-01T09:19:26.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.871 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.632637+0000", 2023-10-01T09:19:26.871 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.228223751 2023-10-01T09:19:26.871 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.871 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.632784+0000", 2023-10-01T09:19:26.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000147377 2023-10-01T09:19:26.872 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.872 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.872 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:19:26.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.633399+0000", 2023-10-01T09:19:26.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00061477500000000002 2023-10-01T09:19:26.872 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.872 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.872 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.872 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.648265+0000", 2023-10-01T09:19:26.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014866084999999999 2023-10-01T09:19:26.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.648340+0000", 2023-10-01T09:19:26.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4616999999999994e-05 2023-10-01T09:19:26.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.652364+0000", 2023-10-01T09:19:26.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040248239999999998 2023-10-01T09:19:26.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770452+0000", 2023-10-01T09:19:26.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.118087964 2023-10-01T09:19:26.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770486+0000", 2023-10-01T09:19:26.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3207000000000002e-05 2023-10-01T09:19:26.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.875 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.875 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770568+0000", 2023-10-01T09:19:26.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2259e-05 2023-10-01T09:19:26.875 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.875 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.875 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.875 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:209453 5.b 5:d1ec2937:::benchmark_data_smithi138_240338_object94316:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-01T09:19:26.875 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:18:30.567968+0000", 2023-10-01T09:19:26.875 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.227012217000002, 2023-10-01T09:19:26.876 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2009798319999998, 2023-10-01T09:19:26.876 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.876 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.876 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.876 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.876 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.876 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 209453 2023-10-01T09:19:26.876 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.876 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.567968+0000", 2023-10-01T09:19:26.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.877 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.877 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.877 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.567968+0000", 2023-10-01T09:19:26.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.877 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.877 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.877 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.567967+0000", 2023-10-01T09:19:26.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:19:26.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.567973+0000", 2023-10-01T09:19:26.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7300000000000002e-06 2023-10-01T09:19:26.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.567985+0000", 2023-10-01T09:19:26.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2026e-05 2023-10-01T09:19:26.879 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.567993+0000", 2023-10-01T09:19:26.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9540000000000004e-06 2023-10-01T09:19:26.879 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.569152+0000", 2023-10-01T09:19:26.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011594540000000001 2023-10-01T09:19:26.879 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.569159+0000", 2023-10-01T09:19:26.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5860000000000001e-06 2023-10-01T09:19:26.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404618+0000", 2023-10-01T09:19:26.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.83545915000000004 2023-10-01T09:19:26.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404626+0000", 2023-10-01T09:19:26.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3089999999999996e-06 2023-10-01T09:19:26.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.593313+0000", 2023-10-01T09:19:26.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.188686565 2023-10-01T09:19:26.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.593397+0000", 2023-10-01T09:19:26.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4592999999999995e-05 2023-10-01T09:19:26.882 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.882 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5,7", 2023-10-01T09:19:26.882 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.593957+0000", 2023-10-01T09:19:26.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00055995199999999995 2023-10-01T09:19:26.882 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.882 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.882 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.603958+0000", 2023-10-01T09:19:26.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010000754000000001 2023-10-01T09:19:26.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.604212+0000", 2023-10-01T09:19:26.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025370699999999999 2023-10-01T09:19:26.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.765406+0000", 2023-10-01T09:19:26.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16119386699999999 2023-10-01T09:19:26.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.884 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.884 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.768761+0000", 2023-10-01T09:19:26.884 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003355369 2023-10-01T09:19:26.884 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.884 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.884 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.884 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.768822+0000", 2023-10-01T09:19:26.884 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1327000000000004e-05 2023-10-01T09:19:26.884 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.884 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.884 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.884 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.768948+0000", 2023-10-01T09:19:26.885 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000125759 2023-10-01T09:19:26.885 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.885 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.885 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.885 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.885 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:209456 5.8 5:180e28dd:::benchmark_data_smithi138_240338_object94319:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-01T09:19:26.885 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:18:30.571789+0000", 2023-10-01T09:19:26.885 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.223191684, 2023-10-01T09:19:26.885 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1990186619999998, 2023-10-01T09:19:26.886 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.886 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.886 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.886 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.886 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.886 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 209456 2023-10-01T09:19:26.886 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.886 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.886 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.886 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.886 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571789+0000", 2023-10-01T09:19:26.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571789+0000", 2023-10-01T09:19:26.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571788+0000", 2023-10-01T09:19:26.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:19:26.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.888 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.888 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.888 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571792+0000", 2023-10-01T09:19:26.888 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7000000000000002e-06 2023-10-01T09:19:26.888 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.888 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.888 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.888 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571800+0000", 2023-10-01T09:19:26.888 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9719999999999992e-06 2023-10-01T09:19:26.888 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.888 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.888 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571805+0000", 2023-10-01T09:19:26.889 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9099999999999996e-06 2023-10-01T09:19:26.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.889 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.889 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571839+0000", 2023-10-01T09:19:26.889 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4468999999999998e-05 2023-10-01T09:19:26.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.889 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.889 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571845+0000", 2023-10-01T09:19:26.889 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1120000000000001e-06 2023-10-01T09:19:26.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.890 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.890 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404441+0000", 2023-10-01T09:19:26.890 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.83259549099999997 2023-10-01T09:19:26.890 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.890 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.890 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404449+0000", 2023-10-01T09:19:26.890 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7400000000000004e-06 2023-10-01T09:19:26.890 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.891 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.891 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.642769+0000", 2023-10-01T09:19:26.891 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.238320885 2023-10-01T09:19:26.891 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.891 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.891 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.891 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.642923+0000", 2023-10-01T09:19:26.891 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015377899999999999 2023-10-01T09:19:26.891 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.891 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.891 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:19:26.891 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.643084+0000", 2023-10-01T09:19:26.892 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016034900000000001 2023-10-01T09:19:26.892 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.892 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.892 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.892 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.648400+0000", 2023-10-01T09:19:26.892 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0053166079999999996 2023-10-01T09:19:26.892 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.892 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.892 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.892 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.648409+0000", 2023-10-01T09:19:26.892 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.789e-06 2023-10-01T09:19:26.892 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.892 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.652411+0000", 2023-10-01T09:19:26.893 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040017539999999997 2023-10-01T09:19:26.893 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.893 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770628+0000", 2023-10-01T09:19:26.893 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11821683700000001 2023-10-01T09:19:26.893 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.893 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770682+0000", 2023-10-01T09:19:26.894 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4422999999999997e-05 2023-10-01T09:19:26.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.894 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.894 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770807+0000", 2023-10-01T09:19:26.894 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012551500000000001 2023-10-01T09:19:26.894 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.894 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.894 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.895 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:209458 5.8 5:156e792b:::benchmark_data_smithi138_240338_object94321:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-01T09:19:26.895 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:18:30.576504+0000", 2023-10-01T09:19:26.895 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.218476314, 2023-10-01T09:19:26.895 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1944605589999999, 2023-10-01T09:19:26.895 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.895 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.895 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.895 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.895 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.895 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 209458 2023-10-01T09:19:26.895 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.895 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.896 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.896 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.896 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576504+0000", 2023-10-01T09:19:26.896 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.896 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.896 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.896 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.896 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576504+0000", 2023-10-01T09:19:26.896 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.896 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.896 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.896 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576503+0000", 2023-10-01T09:19:26.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:19:26.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.897 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576509+0000", 2023-10-01T09:19:26.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3899999999999998e-06 2023-10-01T09:19:26.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.897 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576520+0000", 2023-10-01T09:19:26.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0701e-05 2023-10-01T09:19:26.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.898 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576525+0000", 2023-10-01T09:19:26.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4659999999999998e-06 2023-10-01T09:19:26.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.898 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576571+0000", 2023-10-01T09:19:26.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6239999999999998e-05 2023-10-01T09:19:26.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.899 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.899 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576580+0000", 2023-10-01T09:19:26.899 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0629999999999995e-06 2023-10-01T09:19:26.899 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.899 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.899 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.899 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404546+0000", 2023-10-01T09:19:26.899 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.82796542900000003 2023-10-01T09:19:26.899 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.899 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.899 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.899 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404556+0000", 2023-10-01T09:19:26.899 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8879999999999992e-06 2023-10-01T09:19:26.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.900 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.900 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.645642+0000", 2023-10-01T09:19:26.900 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.241086329 2023-10-01T09:19:26.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.900 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.900 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.645749+0000", 2023-10-01T09:19:26.900 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106863 2023-10-01T09:19:26.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:19:26.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.645871+0000", 2023-10-01T09:19:26.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121784 2023-10-01T09:19:26.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.648417+0000", 2023-10-01T09:19:26.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002546169 2023-10-01T09:19:26.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.902 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.648632+0000", 2023-10-01T09:19:26.902 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021547800000000001 2023-10-01T09:19:26.902 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.902 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.902 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.652454+0000", 2023-10-01T09:19:26.902 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0038215530000000001 2023-10-01T09:19:26.902 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.902 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.902 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770853+0000", 2023-10-01T09:19:26.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.118399108 2023-10-01T09:19:26.903 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.903 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.903 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.903 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770890+0000", 2023-10-01T09:19:26.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7047e-05 2023-10-01T09:19:26.903 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.903 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.903 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.903 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770965+0000", 2023-10-01T09:19:26.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4610999999999994e-05 2023-10-01T09:19:26.903 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.904 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.904 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.904 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.904 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227518 5.b 5:dd303cc1:::benchmark_data_smithi138_240338_object112381:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T09:19:26.904 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.540699+0000", 2023-10-01T09:19:26.904 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 22.254281695, 2023-10-01T09:19:26.904 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2259289409999998, 2023-10-01T09:19:26.904 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.904 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:19:26.905 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.905 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.905 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.905 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227518 2023-10-01T09:19:26.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.905 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540699+0000", 2023-10-01T09:19:26.905 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.906 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.906 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540699+0000", 2023-10-01T09:19:26.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.906 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.906 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.906 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540698+0000", 2023-10-01T09:19:26.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:19:26.906 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.906 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.907 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540702+0000", 2023-10-01T09:19:26.907 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5999999999999998e-06 2023-10-01T09:19:26.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.907 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540708+0000", 2023-10-01T09:19:26.908 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7790000000000001e-06 2023-10-01T09:19:26.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.908 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540713+0000", 2023-10-01T09:19:26.908 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0540000000000002e-06 2023-10-01T09:19:26.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.908 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540759+0000", 2023-10-01T09:19:26.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6001000000000002e-05 2023-10-01T09:19:26.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.909 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540886+0000", 2023-10-01T09:19:26.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012746099999999999 2023-10-01T09:19:26.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.909 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:19:26.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540997+0000", 2023-10-01T09:19:26.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110389 2023-10-01T09:19:26.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.910 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.910 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.553848+0000", 2023-10-01T09:19:26.910 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012850996 2023-10-01T09:19:26.910 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.910 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.910 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.559796+0000", 2023-10-01T09:19:26.910 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059481589999999997 2023-10-01T09:19:26.910 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.910 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.559807+0000", 2023-10-01T09:19:26.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0993e-05 2023-10-01T09:19:26.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547860+0000", 2023-10-01T09:19:26.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98805305899999996 2023-10-01T09:19:26.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547873+0000", 2023-10-01T09:19:26.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.322e-05 2023-10-01T09:19:26.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.912 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.750500+0000", 2023-10-01T09:19:26.912 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.202626795 2023-10-01T09:19:26.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:19:26.912 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.750659+0000", 2023-10-01T09:19:26.912 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015917500000000001 2023-10-01T09:19:26.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.913 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.913 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.766393+0000", 2023-10-01T09:19:26.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015734056999999999 2023-10-01T09:19:26.913 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.913 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.913 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.913 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.766628+0000", 2023-10-01T09:19:26.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00023459500000000001 2023-10-01T09:19:26.913 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.913 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.913 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.913 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.914 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.914 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227520 5.b 5:d8712104:::benchmark_data_smithi138_240338_object112383:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T09:19:26.914 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.547004+0000", 2023-10-01T09:19:26.914 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 22.247976505, 2023-10-01T09:19:26.914 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.227945434, 2023-10-01T09:19:26.914 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.914 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:19:26.914 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.914 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.914 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.914 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227520 2023-10-01T09:19:26.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.915 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.915 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.915 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.915 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547004+0000", 2023-10-01T09:19:26.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.915 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.915 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.915 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547004+0000", 2023-10-01T09:19:26.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.916 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.916 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.916 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547002+0000", 2023-10-01T09:19:26.916 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:19:26.916 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.916 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.916 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.916 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547011+0000", 2023-10-01T09:19:26.916 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3419999999999997e-06 2023-10-01T09:19:26.916 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.916 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.916 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547026+0000", 2023-10-01T09:19:26.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4453e-05 2023-10-01T09:19:26.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547042+0000", 2023-10-01T09:19:26.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5999999999999999e-05 2023-10-01T09:19:26.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.553877+0000", 2023-10-01T09:19:26.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006835163 2023-10-01T09:19:26.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.554019+0000", 2023-10-01T09:19:26.918 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014198699999999999 2023-10-01T09:19:26.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:19:26.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.554141+0000", 2023-10-01T09:19:26.918 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000122032 2023-10-01T09:19:26.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.919 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.559831+0000", 2023-10-01T09:19:26.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005690339 2023-10-01T09:19:26.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.919 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.559838+0000", 2023-10-01T09:19:26.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1690000000000003e-06 2023-10-01T09:19:26.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.919 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547900+0000", 2023-10-01T09:19:26.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98806163599999997 2023-10-01T09:19:26.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547905+0000", 2023-10-01T09:19:26.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2580000000000003e-06 2023-10-01T09:19:26.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.750939+0000", 2023-10-01T09:19:26.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2030342000000001 2023-10-01T09:19:26.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.921 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.921 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:19:26.921 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751046+0000", 2023-10-01T09:19:26.921 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106037 2023-10-01T09:19:26.921 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.921 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.921 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.921 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.774856+0000", 2023-10-01T09:19:26.921 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.023810120000000001 2023-10-01T09:19:26.921 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.921 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.921 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.922 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.774949+0000", 2023-10-01T09:19:26.922 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3781000000000002e-05 2023-10-01T09:19:26.922 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.922 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.922 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.922 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.922 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.922 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227542 5.8 5:163a5025:::benchmark_data_smithi138_240338_object112405:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:19:26.922 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.593796+0000", 2023-10-01T09:19:26.922 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 22.201184051999999, 2023-10-01T09:19:26.922 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.158330061, 2023-10-01T09:19:26.923 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.923 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.923 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.923 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.923 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.923 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227542 2023-10-01T09:19:26.923 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.923 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.923 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.923 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.923 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593796+0000", 2023-10-01T09:19:26.923 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.924 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.924 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.924 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593796+0000", 2023-10-01T09:19:26.924 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.924 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.924 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.924 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593795+0000", 2023-10-01T09:19:26.924 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:19:26.924 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.925 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.925 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593802+0000", 2023-10-01T09:19:26.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2359999999999998e-06 2023-10-01T09:19:26.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.925 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.925 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.925 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593815+0000", 2023-10-01T09:19:26.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2765000000000001e-05 2023-10-01T09:19:26.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.925 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.925 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.926 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593823+0000", 2023-10-01T09:19:26.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0220000000000004e-06 2023-10-01T09:19:26.926 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.926 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.926 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.596113+0000", 2023-10-01T09:19:26.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002289999 2023-10-01T09:19:26.926 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.926 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.926 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.596128+0000", 2023-10-01T09:19:26.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4827000000000001e-05 2023-10-01T09:19:26.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.549844+0000", 2023-10-01T09:19:26.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95371578099999998 2023-10-01T09:19:26.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.549855+0000", 2023-10-01T09:19:26.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.163e-05 2023-10-01T09:19:26.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.734943+0000", 2023-10-01T09:19:26.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1850879670000001 2023-10-01T09:19:26.928 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.928 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.735123+0000", 2023-10-01T09:19:26.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000179347 2023-10-01T09:19:26.928 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.928 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:19:26.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.735771+0000", 2023-10-01T09:19:26.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00064860999999999999 2023-10-01T09:19:26.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.929 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.929 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.929 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751876+0000", 2023-10-01T09:19:26.929 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016104579000000001 2023-10-01T09:19:26.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.929 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.929 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751917+0000", 2023-10-01T09:19:26.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1201000000000001e-05 2023-10-01T09:19:26.930 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.930 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.930 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751925+0000", 2023-10-01T09:19:26.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8830000000000006e-06 2023-10-01T09:19:26.930 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.930 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.930 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.930 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751962+0000", 2023-10-01T09:19:26.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7521999999999998e-05 2023-10-01T09:19:26.931 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.931 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.931 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.752015+0000", 2023-10-01T09:19:26.931 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2407999999999997e-05 2023-10-01T09:19:26.931 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.931 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.931 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.752127+0000", 2023-10-01T09:19:26.931 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111671 2023-10-01T09:19:26.931 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.931 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.932 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.932 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.932 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.932 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227543 5.b 5:d7088ca7:::benchmark_data_smithi138_240338_object112406:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:19:26.932 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.593966+0000", 2023-10-01T09:19:26.932 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 22.201014619999999, 2023-10-01T09:19:26.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2140065830000002, 2023-10-01T09:19:26.932 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.932 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.932 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.932 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.932 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.932 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227543 2023-10-01T09:19:26.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.933 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593966+0000", 2023-10-01T09:19:26.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593966+0000", 2023-10-01T09:19:26.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.934 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593965+0000", 2023-10-01T09:19:26.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:19:26.934 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.934 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593971+0000", 2023-10-01T09:19:26.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5830000000000003e-06 2023-10-01T09:19:26.934 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593981+0000", 2023-10-01T09:19:26.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4229999999999994e-06 2023-10-01T09:19:26.935 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.935 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593989+0000", 2023-10-01T09:19:26.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2039999999999997e-06 2023-10-01T09:19:26.935 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.935 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.594069+0000", 2023-10-01T09:19:26.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0267999999999998e-05 2023-10-01T09:19:26.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.594082+0000", 2023-10-01T09:19:26.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2901000000000001e-05 2023-10-01T09:19:26.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547917+0000", 2023-10-01T09:19:26.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95383516800000001 2023-10-01T09:19:26.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547921+0000", 2023-10-01T09:19:26.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9430000000000002e-06 2023-10-01T09:19:26.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751073+0000", 2023-10-01T09:19:26.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2031520040000001 2023-10-01T09:19:26.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751182+0000", 2023-10-01T09:19:26.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108553 2023-10-01T09:19:26.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5,7", 2023-10-01T09:19:26.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751816+0000", 2023-10-01T09:19:26.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00063413800000000002 2023-10-01T09:19:26.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.771765+0000", 2023-10-01T09:19:26.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019948585000000001 2023-10-01T09:19:26.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.771773+0000", 2023-10-01T09:19:26.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8589999999999994e-06 2023-10-01T09:19:26.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.789897+0000", 2023-10-01T09:19:26.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018123378999999998 2023-10-01T09:19:26.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.807826+0000", 2023-10-01T09:19:26.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017929269000000001 2023-10-01T09:19:26.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.807867+0000", 2023-10-01T09:19:26.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1131000000000003e-05 2023-10-01T09:19:26.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.807972+0000", 2023-10-01T09:19:26.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105274 2023-10-01T09:19:26.941 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.941 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.941 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.941 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227549 5.b 5:dca5cded:::benchmark_data_smithi138_240338_object112412:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:19:26.941 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.607851+0000", 2023-10-01T09:19:26.941 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 22.187129635000002, 2023-10-01T09:19:26.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2003441270000001, 2023-10-01T09:19:26.941 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.942 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.942 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.942 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.942 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.942 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227549 2023-10-01T09:19:26.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.942 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.942 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.942 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607851+0000", 2023-10-01T09:19:26.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607851+0000", 2023-10-01T09:19:26.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607849+0000", 2023-10-01T09:19:26.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:19:26.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.944 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607857+0000", 2023-10-01T09:19:26.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0649999999999999e-06 2023-10-01T09:19:26.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.944 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.944 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607872+0000", 2023-10-01T09:19:26.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5038000000000001e-05 2023-10-01T09:19:26.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.944 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.944 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607880+0000", 2023-10-01T09:19:26.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8690000000000004e-06 2023-10-01T09:19:26.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607947+0000", 2023-10-01T09:19:26.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6847000000000006e-05 2023-10-01T09:19:26.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607959+0000", 2023-10-01T09:19:26.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2575000000000001e-05 2023-10-01T09:19:26.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547932+0000", 2023-10-01T09:19:26.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93997244999999996 2023-10-01T09:19:26.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547936+0000", 2023-10-01T09:19:26.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.878e-06 2023-10-01T09:19:26.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.762267+0000", 2023-10-01T09:19:26.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.21433116 2023-10-01T09:19:26.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.762399+0000", 2023-10-01T09:19:26.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013192999999999999 2023-10-01T09:19:26.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5,7", 2023-10-01T09:19:26.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.762536+0000", 2023-10-01T09:19:26.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000137439 2023-10-01T09:19:26.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.771780+0000", 2023-10-01T09:19:26.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0092435450000000006 2023-10-01T09:19:26.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.776336+0000", 2023-10-01T09:19:26.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004556264 2023-10-01T09:19:26.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.789967+0000", 2023-10-01T09:19:26.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013631228 2023-10-01T09:19:26.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808023+0000", 2023-10-01T09:19:26.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018056274000000001 2023-10-01T09:19:26.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808075+0000", 2023-10-01T09:19:26.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1910000000000003e-05 2023-10-01T09:19:26.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808195+0000", 2023-10-01T09:19:26.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011964 2023-10-01T09:19:26.950 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.950 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.950 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.950 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.950 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227553 5.8 5:1337bdd0:::benchmark_data_smithi138_240338_object112416:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:19:26.950 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.622592+0000", 2023-10-01T09:19:26.950 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 22.172388259000002, 2023-10-01T09:19:26.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.142155657, 2023-10-01T09:19:26.950 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.951 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.951 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.951 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.951 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.951 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227553 2023-10-01T09:19:26.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.951 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622592+0000", 2023-10-01T09:19:26.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622592+0000", 2023-10-01T09:19:26.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622590+0000", 2023-10-01T09:19:26.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:19:26.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622599+0000", 2023-10-01T09:19:26.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9509999999999995e-06 2023-10-01T09:19:26.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622616+0000", 2023-10-01T09:19:26.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6458000000000001e-05 2023-10-01T09:19:26.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622626+0000", 2023-10-01T09:19:26.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0059e-05 2023-10-01T09:19:26.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622691+0000", 2023-10-01T09:19:26.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4848e-05 2023-10-01T09:19:26.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622705+0000", 2023-10-01T09:19:26.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4041e-05 2023-10-01T09:19:26.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.550082+0000", 2023-10-01T09:19:26.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92737758299999995 2023-10-01T09:19:26.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.550091+0000", 2023-10-01T09:19:26.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0710000000000001e-06 2023-10-01T09:19:26.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.744482+0000", 2023-10-01T09:19:26.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1943907620000001 2023-10-01T09:19:26.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.744576+0000", 2023-10-01T09:19:26.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4320999999999994e-05 2023-10-01T09:19:26.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:19:26.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.744661+0000", 2023-10-01T09:19:26.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4855e-05 2023-10-01T09:19:26.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751931+0000", 2023-10-01T09:19:26.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.007269574 2023-10-01T09:19:26.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.759742+0000", 2023-10-01T09:19:26.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0078116840000000002 2023-10-01T09:19:26.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764514+0000", 2023-10-01T09:19:26.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047718409999999998 2023-10-01T09:19:26.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764594+0000", 2023-10-01T09:19:26.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9671000000000003e-05 2023-10-01T09:19:26.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764645+0000", 2023-10-01T09:19:26.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0713e-05 2023-10-01T09:19:26.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.959 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764748+0000", 2023-10-01T09:19:26.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103174 2023-10-01T09:19:26.959 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.959 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.959 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.959 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227561 5.8 5:1c2dcf03:::benchmark_data_smithi138_240338_object112424:head [delete] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-01T09:19:26.959 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:05.744421+0000", 2023-10-01T09:19:26.959 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 21.050559709000002, 2023-10-01T09:19:26.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.020494499, 2023-10-01T09:19:26.959 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.959 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.960 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.960 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.960 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.960 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227561 2023-10-01T09:19:26.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.960 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744421+0000", 2023-10-01T09:19:26.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744421+0000", 2023-10-01T09:19:26.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744419+0000", 2023-10-01T09:19:26.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:19:26.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744427+0000", 2023-10-01T09:19:26.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6160000000000002e-06 2023-10-01T09:19:26.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744445+0000", 2023-10-01T09:19:26.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7246999999999999e-05 2023-10-01T09:19:26.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744453+0000", 2023-10-01T09:19:26.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6740000000000003e-06 2023-10-01T09:19:26.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744541+0000", 2023-10-01T09:19:26.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7819999999999996e-05 2023-10-01T09:19:26.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744554+0000", 2023-10-01T09:19:26.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.237e-05 2023-10-01T09:19:26.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.748068+0000", 2023-10-01T09:19:26.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0035148250000001 2023-10-01T09:19:26.964 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.964 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.964 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.748233+0000", 2023-10-01T09:19:26.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016479900000000001 2023-10-01T09:19:26.964 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.964 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:19:26.964 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.748378+0000", 2023-10-01T09:19:26.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014517999999999999 2023-10-01T09:19:26.964 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.965 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.752176+0000", 2023-10-01T09:19:26.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037973389999999998 2023-10-01T09:19:26.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.965 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.965 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.759790+0000", 2023-10-01T09:19:26.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0076139179999999999 2023-10-01T09:19:26.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.965 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764554+0000", 2023-10-01T09:19:26.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047645869999999998 2023-10-01T09:19:26.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.966 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764783+0000", 2023-10-01T09:19:26.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00022926399999999999 2023-10-01T09:19:26.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.966 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764818+0000", 2023-10-01T09:19:26.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4400999999999998e-05 2023-10-01T09:19:26.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764915+0000", 2023-10-01T09:19:26.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7405000000000007e-05 2023-10-01T09:19:26.967 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.967 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.967 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.967 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227562 5.b 5:d7e9814e:::benchmark_data_smithi138_240338_object112425:head [delete] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-01T09:19:26.967 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:05.744494+0000", 2023-10-01T09:19:26.968 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 21.050486377999999, 2023-10-01T09:19:26.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.064026162, 2023-10-01T09:19:26.968 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:19:26.968 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:19:26.968 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:19:26.968 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:19:26.968 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:19:26.968 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227562 2023-10-01T09:19:26.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.968 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:19:26.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:19:26.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744494+0000", 2023-10-01T09:19:26.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:19:26.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744494+0000", 2023-10-01T09:19:26.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:19:26.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:19:26.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744494+0000", 2023-10-01T09:19:26.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:19:26.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:19:26.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744497+0000", 2023-10-01T09:19:26.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3390000000000001e-06 2023-10-01T09:19:26.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:19:26.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744507+0000", 2023-10-01T09:19:26.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0276e-05 2023-10-01T09:19:26.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:19:26.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744511+0000", 2023-10-01T09:19:26.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9160000000000003e-06 2023-10-01T09:19:26.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744548+0000", 2023-10-01T09:19:26.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7197999999999999e-05 2023-10-01T09:19:26.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:19:26.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744558+0000", 2023-10-01T09:19:26.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.55e-06 2023-10-01T09:19:26.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:19:26.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.767699+0000", 2023-10-01T09:19:26.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0231407809999999 2023-10-01T09:19:26.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:19:26.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.767807+0000", 2023-10-01T09:19:26.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010877199999999999 2023-10-01T09:19:26.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5,7", 2023-10-01T09:19:26.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.767928+0000", 2023-10-01T09:19:26.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120122 2023-10-01T09:19:26.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.777766+0000", 2023-10-01T09:19:26.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0098384030000000008 2023-10-01T09:19:26.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:19:26.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.782314+0000", 2023-10-01T09:19:26.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045480240000000003 2023-10-01T09:19:26.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.789999+0000", 2023-10-01T09:19:26.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0076853340000000003 2023-10-01T09:19:26.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:19:26.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808355+0000", 2023-10-01T09:19:26.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018356006000000001 2023-10-01T09:19:26.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.975 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:19:26.975 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808402+0000", 2023-10-01T09:19:26.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6980000000000001e-05 2023-10-01T09:19:26.975 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:19:26.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:19:26.975 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:19:26.975 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808520+0000", 2023-10-01T09:19:26.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118026 2023-10-01T09:19:26.975 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.975 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.975 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.976 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:19:26.976 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:19:26.976 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:19:26.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:26.977 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:26.977 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:26.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:27.078 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:27.108 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:27.222 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:27.252 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:19:27.271 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:19:27.291 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:19:27.312 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:19:27.331 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:19:27.350 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:19:27.371 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:19:27.390 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:19:27.391 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:19:27.391 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:19:27.391 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:19:27.392 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:19:27.392 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:19:27.392 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:19:27.392 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:19:27.392 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:19:28.299 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/307276289' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:28.299 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:28 smithi138 ceph-mon[185886]: pgmap v439: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:19:28.300 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:28 smithi138 conmon[185862]: 2023-10-01T09:19:28.170+0000 7f70d64ec700 -1 mon.a@0(leader).osd e175 definitely_dead 0 2023-10-01T09:19:28.300 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/307276289' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:28.300 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:28 smithi138 ceph-mon[192513]: pgmap v439: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:19:28.309 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:28.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:28.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:28.413 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:28.442 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:28.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/307276289' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:28.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:28 smithi190 ceph-mon[160435]: pgmap v439: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:19:29.081 INFO:teuthology.orchestra.run.smithi138.stderr:marked in osd.2. 2023-10-01T09:19:29.185 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:19:29.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:29 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3757620828' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:19:29.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:29 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3757620828' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:19:29.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:29 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3757620828' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:19:30.175 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:30.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:30.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:30.205 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-01T09:19:30.282 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:30.321 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:30.361 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:30.366 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:30 smithi138 ceph-mon[185886]: failed to encode map e176 with expected crc 2023-10-01T09:19:30.366 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:30 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3757620828' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-01T09:19:30.366 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:30 smithi138 ceph-mon[185886]: osdmap e176: 8 total, 8 up, 8 in 2023-10-01T09:19:30.366 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:30 smithi138 ceph-mon[185886]: failed to encode map e176 with expected crc 2023-10-01T09:19:30.366 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:30 smithi138 ceph-mon[185886]: failed to encode map e176 with expected crc 2023-10-01T09:19:30.366 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:30 smithi138 ceph-mon[185886]: pgmap v441: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:19:30.366 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:30 smithi138 ceph-mon[192513]: failed to encode map e176 with expected crc 2023-10-01T09:19:30.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:30 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3757620828' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-01T09:19:30.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:30 smithi138 ceph-mon[192513]: osdmap e176: 8 total, 8 up, 8 in 2023-10-01T09:19:30.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:30 smithi138 ceph-mon[192513]: failed to encode map e176 with expected crc 2023-10-01T09:19:30.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:30 smithi138 ceph-mon[192513]: failed to encode map e176 with expected crc 2023-10-01T09:19:30.367 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:30 smithi138 ceph-mon[192513]: pgmap v441: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:19:30.367 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:30 smithi190 ceph-mon[160435]: failed to encode map e176 with expected crc 2023-10-01T09:19:30.367 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:30 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3757620828' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-01T09:19:30.367 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:30 smithi190 ceph-mon[160435]: osdmap e176: 8 total, 8 up, 8 in 2023-10-01T09:19:30.368 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:30 smithi190 ceph-mon[160435]: failed to encode map e176 with expected crc 2023-10-01T09:19:30.368 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:30 smithi190 ceph-mon[160435]: failed to encode map e176 with expected crc 2023-10-01T09:19:30.368 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:30 smithi190 ceph-mon[160435]: pgmap v441: 73 pgs: 73 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail 2023-10-01T09:19:30.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:30.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:30.469 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:30.501 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:30.899 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:30.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:30.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:31.003 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:31.030 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:31.072 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:31.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:31.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:31.085 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:19:31.176 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:31.205 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:31.252 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:31.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:31.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:31.357 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:31.381 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:31 smithi190 ceph-mon[160435]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.382 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:31 smithi190 ceph-mon[160435]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.382 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:31 smithi190 ceph-mon[160435]: osdmap e177: 8 total, 8 up, 8 in 2023-10-01T09:19:31.382 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:31 smithi190 ceph-mon[160435]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.382 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:31 smithi190 ceph-mon[160435]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.382 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:31 smithi190 ceph-mon[160435]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.382 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:31 smithi190 ceph-mon[160435]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.382 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:31 smithi190 ceph-mon[160435]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.382 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:31 smithi190 ceph-mon[160435]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.382 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1576257653' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:19:31.383 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:31 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:19:31.384 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:31.427 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:31.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:31.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:31.534 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:31.557 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[192513]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.557 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[192513]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.557 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[192513]: osdmap e177: 8 total, 8 up, 8 in 2023-10-01T09:19:31.557 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[192513]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.557 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[192513]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.557 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[192513]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.557 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[192513]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.557 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[192513]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.557 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[192513]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.557 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1576257653' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:19:31.558 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:19:31.558 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[185886]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.558 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[185886]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.558 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[185886]: osdmap e177: 8 total, 8 up, 8 in 2023-10-01T09:19:31.558 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[185886]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.558 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[185886]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.558 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[185886]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.558 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[185886]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.558 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[185886]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.559 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[185886]: failed to encode map e177 with expected crc 2023-10-01T09:19:31.559 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1576257653' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:19:31.559 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:31 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:19:31.560 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:31.603 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:31.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:31.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:31.708 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:31.733 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:31.775 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:31.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:31.800 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:31.883 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:31.908 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:31.950 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:31.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:31.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:32.057 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:32.083 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:32.126 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:32.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:32.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:32.195 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:19:32.231 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:19:32.258 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:32.299 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:19:32.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:32.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:19:32.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:32 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:19:32.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:32 smithi138 ceph-mon[185886]: failed to encode map e178 with expected crc 2023-10-01T09:19:32.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:32 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:19:32.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:32 smithi138 ceph-mon[185886]: osdmap e178: 8 total, 8 up, 8 in 2023-10-01T09:19:32.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:32 smithi138 ceph-mon[185886]: failed to encode map e178 with expected crc 2023-10-01T09:19:32.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:32 smithi138 ceph-mon[185886]: pgmap v444: 73 pgs: 73 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:19:32.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:32 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:19:32.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:32 smithi138 ceph-mon[192513]: failed to encode map e178 with expected crc 2023-10-01T09:19:32.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:32 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:19:32.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:32 smithi138 ceph-mon[192513]: osdmap e178: 8 total, 8 up, 8 in 2023-10-01T09:19:32.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:32 smithi138 ceph-mon[192513]: failed to encode map e178 with expected crc 2023-10-01T09:19:32.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:32 smithi138 ceph-mon[192513]: pgmap v444: 73 pgs: 73 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:19:32.408 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:19:32.440 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:32.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:32 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:19:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:32 smithi190 ceph-mon[160435]: failed to encode map e178 with expected crc 2023-10-01T09:19:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:32 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:19:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:32 smithi190 ceph-mon[160435]: osdmap e178: 8 total, 8 up, 8 in 2023-10-01T09:19:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:32 smithi190 ceph-mon[160435]: failed to encode map e178 with expected crc 2023-10-01T09:19:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:32 smithi190 ceph-mon[160435]: pgmap v444: 73 pgs: 73 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-10-01T09:19:32.586 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:19:32.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:32.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:19:32.690 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:32.722 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:32.769 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:32.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:32.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:32.873 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:19:32.899 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:32.995 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:19:32.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:32.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:19:33.099 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:33.128 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:33.173 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:33.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:33.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:33.254 INFO:teuthology.orchestra.run.smithi138.stderr:pool 'unique_pool_1' created 2023-10-01T09:19:33.281 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:33.308 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:33.391 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:33.499 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:33.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:33.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:33.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:33 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/628139686' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-01T09:19:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:33 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-01T09:19:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:33 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-01T09:19:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:33 smithi190 ceph-mon[160435]: osdmap e179: 8 total, 8 up, 8 in 2023-10-01T09:19:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:33 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/628139686' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-01T09:19:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:33 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-01T09:19:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:33 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-01T09:19:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:33 smithi138 ceph-mon[192513]: osdmap e179: 8 total, 8 up, 8 in 2023-10-01T09:19:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:33 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/628139686' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-01T09:19:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:33 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-01T09:19:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:33 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-01T09:19:33.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:33 smithi138 ceph-mon[185886]: osdmap e179: 8 total, 8 up, 8 in 2023-10-01T09:19:33.603 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:33.630 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:33.723 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:33.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:33.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:33.826 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:33.855 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:34.064 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:19:34.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:34 smithi138 ceph-mon[192513]: failed to encode map e179 with expected crc 2023-10-01T09:19:34.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:34 smithi138 ceph-mon[192513]: failed to encode map e179 with expected crc 2023-10-01T09:19:34.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:34 smithi138 ceph-mon[192513]: failed to encode map e179 with expected crc 2023-10-01T09:19:34.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:34 smithi138 ceph-mon[192513]: failed to encode map e179 with expected crc 2023-10-01T09:19:34.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:34 smithi138 ceph-mon[192513]: pgmap v446: 89 pgs: 3 creating+peering, 13 unknown, 1 activating+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13/84 objects degraded (15.476%); 135 B/s, 2 keys/s, 4 objects/s recovering 2023-10-01T09:19:34.346 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:34 smithi138 ceph-mon[192513]: osdmap e180: 8 total, 8 up, 8 in 2023-10-01T09:19:34.346 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:34 smithi138 ceph-mon[185886]: failed to encode map e179 with expected crc 2023-10-01T09:19:34.346 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:34 smithi138 ceph-mon[185886]: failed to encode map e179 with expected crc 2023-10-01T09:19:34.346 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:34 smithi138 ceph-mon[185886]: failed to encode map e179 with expected crc 2023-10-01T09:19:34.346 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:34 smithi138 ceph-mon[185886]: failed to encode map e179 with expected crc 2023-10-01T09:19:34.346 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:34 smithi138 ceph-mon[185886]: pgmap v446: 89 pgs: 3 creating+peering, 13 unknown, 1 activating+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13/84 objects degraded (15.476%); 135 B/s, 2 keys/s, 4 objects/s recovering 2023-10-01T09:19:34.346 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:34 smithi138 ceph-mon[185886]: osdmap e180: 8 total, 8 up, 8 in 2023-10-01T09:19:34.347 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-10-01T09:19:34.362 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:34.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:34.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:34.468 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:34.500 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:34.555 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:34.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:34 smithi190 ceph-mon[160435]: failed to encode map e179 with expected crc 2023-10-01T09:19:34.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:34 smithi190 ceph-mon[160435]: failed to encode map e179 with expected crc 2023-10-01T09:19:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:34 smithi190 ceph-mon[160435]: failed to encode map e179 with expected crc 2023-10-01T09:19:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:34 smithi190 ceph-mon[160435]: failed to encode map e179 with expected crc 2023-10-01T09:19:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:34 smithi190 ceph-mon[160435]: pgmap v446: 89 pgs: 3 creating+peering, 13 unknown, 1 activating+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13/84 objects degraded (15.476%); 135 B/s, 2 keys/s, 4 objects/s recovering 2023-10-01T09:19:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:34 smithi190 ceph-mon[160435]: osdmap e180: 8 total, 8 up, 8 in 2023-10-01T09:19:34.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:34.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:34.660 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:34.690 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:34.823 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:19:34.824 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:19:34.839 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:34.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:34.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:34.943 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:34.974 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:35.023 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:35.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:35.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:35.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:19:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:19:34] "GET /metrics HTTP/1.1" 200 34045 "" "Prometheus/2.33.4" 2023-10-01T09:19:35.130 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:35.159 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:35.203 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:35.205 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:19:35.205 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:19:35.206 INFO:tasks.thrashosds.thrasher:Reweighting osd 7 to 0.6369718089357644 2023-10-01T09:19:35.206 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 0.6369718089357644 2023-10-01T09:19:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:35.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:35.310 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:35.337 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:35.379 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:35.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:35.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:35.484 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:19:35.509 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:35.540 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[185886]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.540 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[185886]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.540 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[185886]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.540 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[185886]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.540 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[185886]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.540 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[185886]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.540 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[185886]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:19:35.540 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[185886]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[185886]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[185886]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[185886]: osdmap e181: 8 total, 8 up, 8 in 2023-10-01T09:19:35.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[192513]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[192513]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[192513]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[192513]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[192513]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[192513]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[192513]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:19:35.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[192513]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[192513]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[192513]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:35 smithi138 ceph-mon[192513]: osdmap e181: 8 total, 8 up, 8 in 2023-10-01T09:19:35.551 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:19:35.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:35 smithi190 ceph-mon[160435]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:35 smithi190 ceph-mon[160435]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:35 smithi190 ceph-mon[160435]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:35 smithi190 ceph-mon[160435]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:35 smithi190 ceph-mon[160435]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:35 smithi190 ceph-mon[160435]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:35 smithi190 ceph-mon[160435]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:19:35.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:35 smithi190 ceph-mon[160435]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:35 smithi190 ceph-mon[160435]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:35 smithi190 ceph-mon[160435]: failed to encode map e180 with expected crc 2023-10-01T09:19:35.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:35 smithi190 ceph-mon[160435]: osdmap e181: 8 total, 8 up, 8 in 2023-10-01T09:19:35.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:35.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:19:35.660 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:35.686 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:35.727 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:35.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:35.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:35.834 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:35.859 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:35.901 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:35.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:35.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:36.006 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:19:36.031 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:36.050 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:19:36.072 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:36.073 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:19:36.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:36.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:19:36.084 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:36.180 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:36.207 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:36.253 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:36.301 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:36 smithi138 ceph-mon[192513]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.301 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:36 smithi138 ceph-mon[192513]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.302 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:36 smithi138 ceph-mon[192513]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.302 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:36 smithi138 ceph-mon[192513]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.302 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:36 smithi138 ceph-mon[192513]: pgmap v449: 89 pgs: 3 creating+peering, 12 unknown, 1 activating+undersized+degraded+remapped, 73 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13/84 objects degraded (15.476%); 136 B/s, 2 keys/s, 4 objects/s recovering 2023-10-01T09:19:36.302 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:36 smithi138 ceph-mon[192513]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.318 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:36.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:36.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:36.423 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:36.449 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:36.491 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:36.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:36.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:36 smithi190 ceph-mon[160435]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:36 smithi190 ceph-mon[160435]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:36 smithi190 ceph-mon[160435]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:36 smithi190 ceph-mon[160435]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:36 smithi190 ceph-mon[160435]: pgmap v449: 89 pgs: 3 creating+peering, 12 unknown, 1 activating+undersized+degraded+remapped, 73 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13/84 objects degraded (15.476%); 136 B/s, 2 keys/s, 4 objects/s recovering 2023-10-01T09:19:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:36 smithi190 ceph-mon[160435]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:36 smithi138 ceph-mon[185886]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:36 smithi138 ceph-mon[185886]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:36 smithi138 ceph-mon[185886]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:36 smithi138 ceph-mon[185886]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:36 smithi138 ceph-mon[185886]: pgmap v449: 89 pgs: 3 creating+peering, 12 unknown, 1 activating+undersized+degraded+remapped, 73 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13/84 objects degraded (15.476%); 136 B/s, 2 keys/s, 4 objects/s recovering 2023-10-01T09:19:36.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:36 smithi138 ceph-mon[185886]: failed to encode map e181 with expected crc 2023-10-01T09:19:36.599 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:36.627 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:37.524 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:37.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:37.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:37.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:37 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:19:37.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:37 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:19:37.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:37 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:19:37.629 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:19:37.656 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:19:37.658 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:37.675 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:19:37.693 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:19:37.704 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:19:37.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:37.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:19:37.723 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:19:37.742 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:19:37.759 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:19:37.777 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:19:37.798 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:19:37.799 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:19:37.799 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:19:37.799 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:19:37.799 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:19:37.799 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:19:37.799 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:19:37.801 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:19:37.801 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:19:37.809 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:37.841 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:38.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:38 smithi138 ceph-mon[192513]: pgmap v450: 89 pgs: 3 creating+peering, 86 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 97 B/s, 1 keys/s, 5 objects/s recovering 2023-10-01T09:19:38.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1951749728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:38.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:38 smithi138 ceph-mon[185886]: pgmap v450: 89 pgs: 3 creating+peering, 86 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 97 B/s, 1 keys/s, 5 objects/s recovering 2023-10-01T09:19:38.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1951749728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:38.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:38 smithi190 ceph-mon[160435]: pgmap v450: 89 pgs: 3 creating+peering, 86 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 97 B/s, 1 keys/s, 5 objects/s recovering 2023-10-01T09:19:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1951749728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:39.315 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:39.343 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:39.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:39.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:39.448 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:39.475 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:39.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:39 smithi138 ceph-mon[192513]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/84 objects degraded (15.476%), 1 pg degraded) 2023-10-01T09:19:39.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:39 smithi138 ceph-mon[185886]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/84 objects degraded (15.476%), 1 pg degraded) 2023-10-01T09:19:39.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:39 smithi190 ceph-mon[160435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/84 objects degraded (15.476%), 1 pg degraded) 2023-10-01T09:19:39.840 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:39.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:39.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:39.944 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:39.973 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:40.019 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:40.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:40.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:40.123 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:40.150 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:40.192 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:40.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:40.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:40.297 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:40.335 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:40.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:40 smithi138 ceph-mon[192513]: pgmap v451: 89 pgs: 89 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-01T09:19:40.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:40.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:40.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:19:40.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:40 smithi138 ceph-mon[185886]: pgmap v451: 89 pgs: 89 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-01T09:19:40.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:40.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:40.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:19:40.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:40 smithi190 ceph-mon[160435]: pgmap v451: 89 pgs: 89 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-01T09:19:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:19:41.004 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:19:41.131 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:41.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:41.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:41.235 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:41.267 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:41.318 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:41.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:41.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:41.329 INFO:teuthology.orchestra.run.smithi138.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-10-01T09:19:41.329 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:19:41.423 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:41.451 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:41.497 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:41.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:41.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:41 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/210445053' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:19:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:41 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:19:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:41 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/670252078' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:19:41.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:41 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/210445053' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:19:41.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:41 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:19:41.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:41 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/670252078' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:19:41.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:41 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/210445053' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:19:41.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:41 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:19:41.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:41 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/670252078' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:19:41.603 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:41.631 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:42.502 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:42 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:19:42.502 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:42 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:19:42.502 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:42 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:19:42.502 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:42 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/670252078' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:19:42.502 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:42 smithi138 ceph-mon[192513]: osdmap e182: 8 total, 8 up, 8 in 2023-10-01T09:19:42.502 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:42 smithi138 ceph-mon[192513]: pgmap v453: 89 pgs: 89 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:19:42.502 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:42 smithi138 ceph-mon[192513]: failed to encode map e182 with expected crc 2023-10-01T09:19:42.503 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:42 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:19:42.503 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:42 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:19:42.503 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:42 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:19:42.504 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:42 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/670252078' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:19:42.504 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:42 smithi138 ceph-mon[185886]: osdmap e182: 8 total, 8 up, 8 in 2023-10-01T09:19:42.504 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:42 smithi138 ceph-mon[185886]: pgmap v453: 89 pgs: 89 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:19:42.504 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:42 smithi138 ceph-mon[185886]: failed to encode map e182 with expected crc 2023-10-01T09:19:42.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:42 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:19:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:42 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:19:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:42 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:19:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:42 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/670252078' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:19:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:42 smithi190 ceph-mon[160435]: osdmap e182: 8 total, 8 up, 8 in 2023-10-01T09:19:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:42 smithi190 ceph-mon[160435]: pgmap v453: 89 pgs: 89 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:19:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:42 smithi190 ceph-mon[160435]: failed to encode map e182 with expected crc 2023-10-01T09:19:42.831 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:42.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:42.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:42.935 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:19:42.968 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:43.018 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:19:43.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:43.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:19:43.125 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:43.151 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:43.153 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:43.197 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:43.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:43.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:43.247 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:19:43.248 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:19:43.248 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:19:43.248 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:19:43.248 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:19:43.303 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:43.329 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:43.346 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.7 to 0.636972 (a310) 2023-10-01T09:19:43.374 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:43.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:43.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:43.480 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:43.513 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:43.547 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:43 smithi138 ceph-mon[192513]: failed to encode map e182 with expected crc 2023-10-01T09:19:43.547 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:43 smithi138 ceph-mon[192513]: failed to encode map e182 with expected crc 2023-10-01T09:19:43.548 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:43 smithi138 ceph-mon[192513]: failed to encode map e182 with expected crc 2023-10-01T09:19:43.548 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:43 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3362505978' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.6369718089357644}]: dispatch 2023-10-01T09:19:43.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:43 smithi138 ceph-mon[185886]: failed to encode map e182 with expected crc 2023-10-01T09:19:43.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:43 smithi138 ceph-mon[185886]: failed to encode map e182 with expected crc 2023-10-01T09:19:43.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:43 smithi138 ceph-mon[185886]: failed to encode map e182 with expected crc 2023-10-01T09:19:43.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:43 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3362505978' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.6369718089357644}]: dispatch 2023-10-01T09:19:43.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:43 smithi190 ceph-mon[160435]: failed to encode map e182 with expected crc 2023-10-01T09:19:43.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:43 smithi190 ceph-mon[160435]: failed to encode map e182 with expected crc 2023-10-01T09:19:43.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:43 smithi190 ceph-mon[160435]: failed to encode map e182 with expected crc 2023-10-01T09:19:43.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:43 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3362505978' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.6369718089357644}]: dispatch 2023-10-01T09:19:43.904 DEBUG:teuthology.orchestra.run.smithi138:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-10-01T09:19:43.906 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T09:19:43.906 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T09:19:43.954 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:hints = 1 2023-10-01T09:19:43.954 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T09:19:43.954 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Object prefix: benchmark_data_smithi138_307751 2023-10-01T09:19:43.955 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:19:43.955 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 0 0 0 0 0 0 - 0 2023-10-01T09:19:44.377 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:19:44.425 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:44.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:44.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:44.504 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:19:44.538 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:44.577 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:44.617 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3362505978' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.6369718089357644}]': finished 2023-10-01T09:19:44.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[185886]: osdmap e183: 8 total, 8 up, 8 in 2023-10-01T09:19:44.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[185886]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[185886]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[185886]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[185886]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[185886]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[185886]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[185886]: pgmap v455: 89 pgs: 89 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:19:44.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[185886]: osdmap e184: 8 total, 8 up, 8 in 2023-10-01T09:19:44.623 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3362505978' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.6369718089357644}]': finished 2023-10-01T09:19:44.623 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[192513]: osdmap e183: 8 total, 8 up, 8 in 2023-10-01T09:19:44.623 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[192513]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.623 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[192513]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.623 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[192513]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.623 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[192513]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.623 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[192513]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.623 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[192513]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.623 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[192513]: pgmap v455: 89 pgs: 89 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:19:44.623 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:44 smithi138 ceph-mon[192513]: osdmap e184: 8 total, 8 up, 8 in 2023-10-01T09:19:44.651 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:44.656 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:44 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3362505978' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.6369718089357644}]': finished 2023-10-01T09:19:44.656 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:44 smithi190 ceph-mon[160435]: osdmap e183: 8 total, 8 up, 8 in 2023-10-01T09:19:44.656 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:44 smithi190 ceph-mon[160435]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.656 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:44 smithi190 ceph-mon[160435]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.656 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:44 smithi190 ceph-mon[160435]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.657 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:44 smithi190 ceph-mon[160435]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.657 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:44 smithi190 ceph-mon[160435]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.657 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:44 smithi190 ceph-mon[160435]: failed to encode map e183 with expected crc 2023-10-01T09:19:44.657 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:44 smithi190 ceph-mon[160435]: pgmap v455: 89 pgs: 89 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:19:44.657 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:44 smithi190 ceph-mon[160435]: osdmap e184: 8 total, 8 up, 8 in 2023-10-01T09:19:44.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:44.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:44.759 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:44.788 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:44.954 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 1 16 1249 1233 77.0539 77.0625 0.00126125 0.0116565 2023-10-01T09:19:45.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:19:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:19:44] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.33.4" 2023-10-01T09:19:45.352 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:45.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:45.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:45.463 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:45.545 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:45.638 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:45.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:45.645 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:45 smithi138 ceph-mon[185886]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.646 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:45 smithi138 ceph-mon[185886]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.646 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:45 smithi138 ceph-mon[185886]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.646 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:45 smithi138 ceph-mon[185886]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.646 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:45 smithi138 ceph-mon[185886]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.646 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:45 smithi138 ceph-mon[192513]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.646 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:45 smithi138 ceph-mon[192513]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.646 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:45 smithi138 ceph-mon[192513]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.646 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:45 smithi138 ceph-mon[192513]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.647 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:45 smithi138 ceph-mon[192513]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:45.748 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:45.797 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:45.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:45 smithi190 ceph-mon[160435]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:45 smithi190 ceph-mon[160435]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:45 smithi190 ceph-mon[160435]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:45 smithi190 ceph-mon[160435]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:45 smithi190 ceph-mon[160435]: failed to encode map e184 with expected crc 2023-10-01T09:19:45.954 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 2 16 4485 4469 139.642 202.25 0.00348576 0.00715016 2023-10-01T09:19:46.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:46 smithi190 ceph-mon[160435]: failed to encode map e185 with expected crc 2023-10-01T09:19:46.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:46 smithi190 ceph-mon[160435]: osdmap e185: 8 total, 8 up, 8 in 2023-10-01T09:19:46.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:46 smithi190 ceph-mon[160435]: failed to encode map e185 with expected crc 2023-10-01T09:19:46.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:46 smithi190 ceph-mon[160435]: pgmap v458: 89 pgs: 89 active+clean; 5.8 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 314 KiB/s wr, 4 op/s 2023-10-01T09:19:46.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:46 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:46.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:46 smithi138 ceph-mon[185886]: failed to encode map e185 with expected crc 2023-10-01T09:19:46.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:46 smithi138 ceph-mon[185886]: osdmap e185: 8 total, 8 up, 8 in 2023-10-01T09:19:46.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:46 smithi138 ceph-mon[185886]: failed to encode map e185 with expected crc 2023-10-01T09:19:46.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:46 smithi138 ceph-mon[185886]: pgmap v458: 89 pgs: 89 active+clean; 5.8 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 314 KiB/s wr, 4 op/s 2023-10-01T09:19:46.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:46 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:46.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:46 smithi138 ceph-mon[192513]: failed to encode map e185 with expected crc 2023-10-01T09:19:46.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:46 smithi138 ceph-mon[192513]: osdmap e185: 8 total, 8 up, 8 in 2023-10-01T09:19:46.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:46 smithi138 ceph-mon[192513]: failed to encode map e185 with expected crc 2023-10-01T09:19:46.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:46 smithi138 ceph-mon[192513]: pgmap v458: 89 pgs: 89 active+clean; 5.8 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 314 KiB/s wr, 4 op/s 2023-10-01T09:19:46.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:46 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:46.954 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 3 16 6922 6906 143.861 152.312 0.00428375 0.00694237 2023-10-01T09:19:47.185 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:47.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:47.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:47.291 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:47.311 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:47.353 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:47.354 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:47.463 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:47.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:47.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:47.470 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:47 smithi138 ceph-mon[185886]: failed to encode map e186 with expected crc 2023-10-01T09:19:47.470 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:47 smithi138 ceph-mon[185886]: osdmap e186: 8 total, 8 up, 8 in 2023-10-01T09:19:47.470 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:47 smithi138 ceph-mon[185886]: failed to encode map e186 with expected crc 2023-10-01T09:19:47.470 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:47 smithi138 ceph-mon[185886]: failed to encode map e186 with expected crc 2023-10-01T09:19:47.470 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:47 smithi138 ceph-mon[185886]: failed to encode map e186 with expected crc 2023-10-01T09:19:47.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:47 smithi138 ceph-mon[192513]: failed to encode map e186 with expected crc 2023-10-01T09:19:47.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:47 smithi138 ceph-mon[192513]: osdmap e186: 8 total, 8 up, 8 in 2023-10-01T09:19:47.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:47 smithi138 ceph-mon[192513]: failed to encode map e186 with expected crc 2023-10-01T09:19:47.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:47 smithi138 ceph-mon[192513]: failed to encode map e186 with expected crc 2023-10-01T09:19:47.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:47 smithi138 ceph-mon[192513]: failed to encode map e186 with expected crc 2023-10-01T09:19:47.571 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:19:47.626 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:47.700 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:19:47.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:47.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:19:47.708 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:47 smithi190 ceph-mon[160435]: failed to encode map e186 with expected crc 2023-10-01T09:19:47.708 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:47 smithi190 ceph-mon[160435]: osdmap e186: 8 total, 8 up, 8 in 2023-10-01T09:19:47.708 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:47 smithi190 ceph-mon[160435]: failed to encode map e186 with expected crc 2023-10-01T09:19:47.708 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:47 smithi190 ceph-mon[160435]: failed to encode map e186 with expected crc 2023-10-01T09:19:47.709 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:47 smithi190 ceph-mon[160435]: failed to encode map e186 with expected crc 2023-10-01T09:19:47.808 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:47.860 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:47.936 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:47.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:47.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:47.955 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 4 16 9890 9874 154.244 185.5 0.0055139 0.00647638 2023-10-01T09:19:48.046 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:48.085 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:48.173 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:48.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:48.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:48.280 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:48.317 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:48.325 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:19:48.362 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:19:48.396 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:48.398 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:48 smithi190 ceph-mon[160435]: osdmap e187: 8 total, 8 up, 8 in 2023-10-01T09:19:48.398 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:48 smithi190 ceph-mon[160435]: failed to encode map e187 with expected crc 2023-10-01T09:19:48.398 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:48 smithi190 ceph-mon[160435]: pgmap v461: 89 pgs: 2 peering, 87 active+clean; 313 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 144 KiB/s, 4 objects/s recovering 2023-10-01T09:19:48.398 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:48 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2054727887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:48.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:48.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:48.405 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:19:48.442 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:19:48.478 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:19:48.504 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:19:48.514 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:19:48.549 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:48.552 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:19:48.588 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:19:48.590 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:19:48.590 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:19:48.590 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:19:48.590 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:19:48.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:19:48.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:19:48.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:19:48.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:19:48.615 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:48 smithi138 ceph-mon[185886]: osdmap e187: 8 total, 8 up, 8 in 2023-10-01T09:19:48.615 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:48 smithi138 ceph-mon[185886]: failed to encode map e187 with expected crc 2023-10-01T09:19:48.615 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:48 smithi138 ceph-mon[185886]: pgmap v461: 89 pgs: 2 peering, 87 active+clean; 313 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 144 KiB/s, 4 objects/s recovering 2023-10-01T09:19:48.615 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:48 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2054727887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:48.616 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:48 smithi138 ceph-mon[192513]: osdmap e187: 8 total, 8 up, 8 in 2023-10-01T09:19:48.616 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:48 smithi138 ceph-mon[192513]: failed to encode map e187 with expected crc 2023-10-01T09:19:48.616 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:48 smithi138 ceph-mon[192513]: pgmap v461: 89 pgs: 2 peering, 87 active+clean; 313 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s; 144 KiB/s, 4 objects/s recovering 2023-10-01T09:19:48.616 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:48 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2054727887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:19:48.633 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:19:48.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:48.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:19:48.744 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:48.789 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:48.843 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:48.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:48.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:48.954 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:19:48.955 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 5 16 12697 12681 158.479 175.438 0.00408773 0.00630442 2023-10-01T09:19:49.014 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:49.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:49 smithi190 ceph-mon[160435]: failed to encode map e187 with expected crc 2023-10-01T09:19:49.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:49 smithi138 ceph-mon[185886]: failed to encode map e187 with expected crc 2023-10-01T09:19:49.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:49 smithi138 ceph-mon[192513]: failed to encode map e187 with expected crc 2023-10-01T09:19:49.955 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 6 16 15834 15818 164.74 196.062 0.00586693 0.00606422 2023-10-01T09:19:50.509 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:19:50.509 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:19:50.509 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T09:19:50.509 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.462768 2023-10-01T09:19:50.509 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 0.46276781783510945 2023-10-01T09:19:50.729 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:50 smithi138 ceph-mon[185886]: pgmap v462: 89 pgs: 2 peering, 87 active+clean; 441 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 114 KiB/s, 3 objects/s recovering 2023-10-01T09:19:50.729 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:50 smithi138 ceph-mon[192513]: pgmap v462: 89 pgs: 2 peering, 87 active+clean; 441 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 114 KiB/s, 3 objects/s recovering 2023-10-01T09:19:50.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:50 smithi190 ceph-mon[160435]: pgmap v462: 89 pgs: 2 peering, 87 active+clean; 441 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 114 KiB/s, 3 objects/s recovering 2023-10-01T09:19:50.956 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 7 16 18572 18556 165.647 171.125 0.00589704 0.00603064 2023-10-01T09:19:51.597 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:19:51.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:51.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:19:51.700 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:51.750 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:51.956 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 8 16 20773 20757 162.133 137.562 0.00408531 0.00616081 2023-10-01T09:19:52.230 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:52.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:52.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:52.335 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:52.364 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:52.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:52 smithi138 ceph-mon[185886]: pgmap v463: 89 pgs: 2 peering, 87 active+clean; 489 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 95 KiB/s, 3 objects/s recovering 2023-10-01T09:19:52.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:52 smithi138 ceph-mon[192513]: pgmap v463: 89 pgs: 2 peering, 87 active+clean; 489 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 95 KiB/s, 3 objects/s recovering 2023-10-01T09:19:52.725 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:52.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:52.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:52 smithi190 ceph-mon[160435]: pgmap v463: 89 pgs: 2 peering, 87 active+clean; 489 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 95 KiB/s, 3 objects/s recovering 2023-10-01T09:19:52.831 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:19:52.882 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:52.962 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 9 16 22608 22592 156.753 114.688 0.0046166 0.00637307 2023-10-01T09:19:52.964 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:19:52.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:52.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:19:53.070 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:53.117 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:53.217 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:53.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:53.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:53.325 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:19:53.393 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:53.482 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:53.505 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:19:53.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:53.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:19:53.621 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:53.662 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:53.732 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:53.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:53.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:53.840 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:53.887 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:53.959 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:53.962 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 10 16 24623 24607 153.67 125.938 0.00840604 0.00650224 2023-10-01T09:19:53.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:53.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:54.067 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:54.081 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:19:54.081 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:19:54.156 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:54.441 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:19:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:54 smithi190 ceph-mon[160435]: pgmap v464: 89 pgs: 89 active+clean; 1.2 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 159 MiB/s wr, 2.54k op/s; 72 KiB/s, 2 objects/s recovering 2023-10-01T09:19:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:54 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/695130887' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:19:54.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:54 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:19:54.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:54 smithi138 ceph-mon[185886]: pgmap v464: 89 pgs: 89 active+clean; 1.2 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 159 MiB/s wr, 2.54k op/s; 72 KiB/s, 2 objects/s recovering 2023-10-01T09:19:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:54 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/695130887' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:19:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:54 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:19:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:54 smithi138 ceph-mon[192513]: pgmap v464: 89 pgs: 89 active+clean; 1.2 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 159 MiB/s wr, 2.54k op/s; 72 KiB/s, 2 objects/s recovering 2023-10-01T09:19:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:54 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/695130887' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:19:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:54 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:19:54.962 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 11 16 26869 26853 152.461 140.375 0.0120198 0.00655357 2023-10-01T09:19:54.979 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:19:55.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:19:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:19:54] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.33.4" 2023-10-01T09:19:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:55 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:19:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:55 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:19:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:55 smithi190 ceph-mon[160435]: osdmap e188: 8 total, 8 up, 8 in 2023-10-01T09:19:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:55 smithi190 ceph-mon[160435]: failed to encode map e188 with expected crc 2023-10-01T09:19:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:55 smithi190 ceph-mon[160435]: failed to encode map e188 with expected crc 2023-10-01T09:19:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:55 smithi190 ceph-mon[160435]: failed to encode map e188 with expected crc 2023-10-01T09:19:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:55 smithi190 ceph-mon[160435]: failed to encode map e188 with expected crc 2023-10-01T09:19:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:19:55.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:19:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:19:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[185886]: osdmap e188: 8 total, 8 up, 8 in 2023-10-01T09:19:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[185886]: failed to encode map e188 with expected crc 2023-10-01T09:19:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[185886]: failed to encode map e188 with expected crc 2023-10-01T09:19:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[185886]: failed to encode map e188 with expected crc 2023-10-01T09:19:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[185886]: failed to encode map e188 with expected crc 2023-10-01T09:19:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:19:55.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:19:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:19:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[192513]: osdmap e188: 8 total, 8 up, 8 in 2023-10-01T09:19:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[192513]: failed to encode map e188 with expected crc 2023-10-01T09:19:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[192513]: failed to encode map e188 with expected crc 2023-10-01T09:19:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[192513]: failed to encode map e188 with expected crc 2023-10-01T09:19:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[192513]: failed to encode map e188 with expected crc 2023-10-01T09:19:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:19:55.963 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 12 16 28966 28950 150.677 131.062 0.0025809 0.00658196 2023-10-01T09:19:56.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:56 smithi138 ceph-mon[185886]: pgmap v466: 89 pgs: 89 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 125 MiB/s wr, 2.01k op/s 2023-10-01T09:19:56.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-01T09:19:56.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-01T09:19:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:56 smithi138 ceph-mon[192513]: pgmap v466: 89 pgs: 89 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 125 MiB/s wr, 2.01k op/s 2023-10-01T09:19:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-01T09:19:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-01T09:19:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:56 smithi190 ceph-mon[160435]: pgmap v466: 89 pgs: 89 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 125 MiB/s wr, 2.01k op/s 2023-10-01T09:19:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-01T09:19:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-10-01T09:19:56.831 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:56.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:56.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:56.941 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:56.962 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 13 16 30202 30186 145.032 77.25 0.00174934 0.00685875 2023-10-01T09:19:56.968 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:57.517 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:19:57.517 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:19:57.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:57.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:19:57.630 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:19:57.696 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:57.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:57 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-01T09:19:57.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:57 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-01T09:19:57.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:57 smithi138 ceph-mon[192513]: osdmap e189: 8 total, 8 up, 8 in 2023-10-01T09:19:57.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:57 smithi138 ceph-mon[192513]: failed to encode map e189 with expected crc 2023-10-01T09:19:57.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:57 smithi138 ceph-mon[192513]: failed to encode map e189 with expected crc 2023-10-01T09:19:57.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:57 smithi138 ceph-mon[192513]: failed to encode map e189 with expected crc 2023-10-01T09:19:57.749 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:57 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-01T09:19:57.749 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:57 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-01T09:19:57.749 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:57 smithi138 ceph-mon[185886]: osdmap e189: 8 total, 8 up, 8 in 2023-10-01T09:19:57.749 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:57 smithi138 ceph-mon[185886]: failed to encode map e189 with expected crc 2023-10-01T09:19:57.749 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:57 smithi138 ceph-mon[185886]: failed to encode map e189 with expected crc 2023-10-01T09:19:57.750 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:57 smithi138 ceph-mon[185886]: failed to encode map e189 with expected crc 2023-10-01T09:19:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:57 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-01T09:19:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:57 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-10-01T09:19:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:57 smithi190 ceph-mon[160435]: osdmap e189: 8 total, 8 up, 8 in 2023-10-01T09:19:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:57 smithi190 ceph-mon[160435]: failed to encode map e189 with expected crc 2023-10-01T09:19:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:57 smithi190 ceph-mon[160435]: failed to encode map e189 with expected crc 2023-10-01T09:19:57.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:57 smithi190 ceph-mon[160435]: failed to encode map e189 with expected crc 2023-10-01T09:19:57.901 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:19:57.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:57.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:19:57.963 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 14 16 31928 31912 142.378 107.875 0.014772 0.00701738 2023-10-01T09:19:58.009 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:19:58.059 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:58.151 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:19:58.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:58.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:19:58.175 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:58.260 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:19:58.313 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:58.403 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:19:58.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:58.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:19:58.512 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:19:58.611 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:58.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:58 smithi138 ceph-mon[185886]: pgmap v468: 89 pgs: 1 peering, 88 active+clean; 1.7 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2023-10-01T09:19:58.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:58 smithi138 ceph-mon[185886]: failed to encode map e190 with expected crc 2023-10-01T09:19:58.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:58 smithi138 ceph-mon[185886]: osdmap e190: 8 total, 8 up, 8 in 2023-10-01T09:19:58.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:58.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:58.755 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:58 smithi138 ceph-mon[192513]: pgmap v468: 89 pgs: 1 peering, 88 active+clean; 1.7 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2023-10-01T09:19:58.755 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:58 smithi138 ceph-mon[192513]: failed to encode map e190 with expected crc 2023-10-01T09:19:58.755 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:58 smithi138 ceph-mon[192513]: osdmap e190: 8 total, 8 up, 8 in 2023-10-01T09:19:58.756 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:58.756 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:58.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:58 smithi190 ceph-mon[160435]: pgmap v468: 89 pgs: 1 peering, 88 active+clean; 1.7 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2023-10-01T09:19:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:58 smithi190 ceph-mon[160435]: failed to encode map e190 with expected crc 2023-10-01T09:19:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:58 smithi190 ceph-mon[160435]: osdmap e190: 8 total, 8 up, 8 in 2023-10-01T09:19:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:58.842 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:19:58.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:58.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:19:58.959 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:19:58.964 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 15 16 33600 33584 139.846 104.5 0.0137162 0.0071419 2023-10-01T09:19:58.998 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:59.023 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:19:59.076 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:19:59.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:59.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:19:59.181 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:19:59.232 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:59.317 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:19:59.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:19:59.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:19:59.422 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:19:59.475 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:19:59.494 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:19:59.494 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:19:59.494 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:59.494 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3371901719' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.46276781783510945}]: dispatch 2023-10-01T09:19:59.495 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:19:59 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.46276781783510945}]: dispatch 2023-10-01T09:19:59.558 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:19:59.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:19:59.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:19:59.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:59.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3371901719' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.46276781783510945}]: dispatch 2023-10-01T09:19:59.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:19:59 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.46276781783510945}]: dispatch 2023-10-01T09:19:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:19:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:19:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:19:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3371901719' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.46276781783510945}]: dispatch 2023-10-01T09:19:59.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:19:59 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.46276781783510945}]: dispatch 2023-10-01T09:19:59.964 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 16 16 35337 35321 137.891 108.562 0.00157575 0.00721123 2023-10-01T09:20:00.074 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.6 primary-affinity to 0.462768 (8303272) 2023-10-01T09:20:00.658 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:00.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:00.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:00.762 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:20:00.787 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[192513]: pgmap v470: 89 pgs: 1 peering, 88 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-01T09:20:00.787 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[192513]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-01T09:20:00.787 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[192513]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-01T09:20:00.787 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.46276781783510945}]': finished 2023-10-01T09:20:00.787 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[192513]: failed to encode map e191 with expected crc 2023-10-01T09:20:00.787 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[192513]: osdmap e191: 8 total, 8 up, 8 in 2023-10-01T09:20:00.787 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[192513]: failed to encode map e191 with expected crc 2023-10-01T09:20:00.788 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:00.788 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[185886]: pgmap v470: 89 pgs: 1 peering, 88 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-01T09:20:00.788 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[185886]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-01T09:20:00.788 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[185886]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-01T09:20:00.788 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.46276781783510945}]': finished 2023-10-01T09:20:00.788 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[185886]: failed to encode map e191 with expected crc 2023-10-01T09:20:00.788 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[185886]: osdmap e191: 8 total, 8 up, 8 in 2023-10-01T09:20:00.788 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[185886]: failed to encode map e191 with expected crc 2023-10-01T09:20:00.788 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:00.789 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:00.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:00 smithi190 ceph-mon[160435]: pgmap v470: 89 pgs: 1 peering, 88 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-01T09:20:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:00 smithi190 ceph-mon[160435]: Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-01T09:20:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:00 smithi190 ceph-mon[160435]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-01T09:20:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:00 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.46276781783510945}]': finished 2023-10-01T09:20:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:00 smithi190 ceph-mon[160435]: failed to encode map e191 with expected crc 2023-10-01T09:20:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:00 smithi190 ceph-mon[160435]: osdmap e191: 8 total, 8 up, 8 in 2023-10-01T09:20:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:00 smithi190 ceph-mon[160435]: failed to encode map e191 with expected crc 2023-10-01T09:20:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:00.831 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:20:00.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:00.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:20:00.934 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:20:00.961 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:00.964 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 17 16 35832 35816 131.603 30.9375 0.00169321 0.00733507 2023-10-01T09:20:01.678 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:20:01.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:01.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:20:01.785 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:20:01.842 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:01.911 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:01.946 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:20:01.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:01.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:20:01.965 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 18 16 37450 37434 129.897 101.125 0.007579 0.00769244 2023-10-01T09:20:02.056 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:02.124 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:02 smithi138 ceph-mon[192513]: osdmap e192: 8 total, 8 up, 8 in 2023-10-01T09:20:02.124 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:02 smithi138 ceph-mon[192513]: failed to encode map e192 with expected crc 2023-10-01T09:20:02.124 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:02 smithi138 ceph-mon[192513]: failed to encode map e192 with expected crc 2023-10-01T09:20:02.124 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:02 smithi138 ceph-mon[192513]: failed to encode map e192 with expected crc 2023-10-01T09:20:02.124 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:02 smithi138 ceph-mon[192513]: pgmap v473: 89 pgs: 1 peering, 88 active+clean; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-10-01T09:20:02.125 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:02 smithi138 ceph-mon[185886]: osdmap e192: 8 total, 8 up, 8 in 2023-10-01T09:20:02.125 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:02 smithi138 ceph-mon[185886]: failed to encode map e192 with expected crc 2023-10-01T09:20:02.125 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:02 smithi138 ceph-mon[185886]: failed to encode map e192 with expected crc 2023-10-01T09:20:02.125 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:02 smithi138 ceph-mon[185886]: failed to encode map e192 with expected crc 2023-10-01T09:20:02.125 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:02 smithi138 ceph-mon[185886]: pgmap v473: 89 pgs: 1 peering, 88 active+clean; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-10-01T09:20:02.132 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:02.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:02 smithi190 ceph-mon[160435]: osdmap e192: 8 total, 8 up, 8 in 2023-10-01T09:20:02.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:02 smithi190 ceph-mon[160435]: failed to encode map e192 with expected crc 2023-10-01T09:20:02.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:02 smithi190 ceph-mon[160435]: failed to encode map e192 with expected crc 2023-10-01T09:20:02.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:02 smithi190 ceph-mon[160435]: failed to encode map e192 with expected crc 2023-10-01T09:20:02.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:02 smithi190 ceph-mon[160435]: pgmap v473: 89 pgs: 1 peering, 88 active+clean; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 102 MiB/s wr, 1.64k op/s 2023-10-01T09:20:02.409 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:20:02.448 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:20:02.481 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:20:02.528 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:20:02.564 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:20:02.601 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:20:02.634 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:20:02.668 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:20:02.673 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:20:02.674 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:20:02.674 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:20:02.674 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:20:02.674 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:20:02.674 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:20:02.674 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:20:02.674 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:20:02.956 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:02.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:02.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:02.966 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 19 16 39132 39116 128.587 105.125 0.00774615 0.00777107 2023-10-01T09:20:03.061 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:20:03.140 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:03 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3151416032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:03.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:03 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3151416032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:03.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:03 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3151416032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:03.967 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:20:03.967420+0000 min lat: 0.00099931 max lat: 0.862148 avg lat: 0.00792406 2023-10-01T09:20:03.967 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:20:03.967 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 20 16 40368 40352 126.021 77.25 0.00283227 0.00792406 2023-10-01T09:20:04.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:04 smithi138 ceph-mon[185886]: pgmap v474: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-01T09:20:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:04 smithi138 ceph-mon[192513]: pgmap v474: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-01T09:20:04.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:04 smithi190 ceph-mon[160435]: pgmap v474: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-01T09:20:04.809 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:20:04.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:04.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:20:04.914 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:04.969 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 21 16 41595 41579 123.661 76.6875 0.00876957 0.00808179 2023-10-01T09:20:04.977 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:05.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:20:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:20:04] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.33.4" 2023-10-01T09:20:05.315 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:05.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:05.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:05.427 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:20:05.484 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:05.943 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:20:05.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:05.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:20:05.969 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 22 16 43285 43269 122.842 105.625 0.0161208 0.00813531 2023-10-01T09:20:06.014 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:20:06.014 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:20:06.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.014 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35070.0:43313 6.9 e192/179 6:98d89b6c:::benchmark_data_smithi138_307751_object43312:head v 192'2785, mlcod=192'2783)", 2023-10-01T09:20:06.014 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:20:05.996964+0000", 2023-10-01T09:20:06.014 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0096748909999999997, 2023-10-01T09:20:06.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0097056250000000007, 2023-10-01T09:20:06.015 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:06.015 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:06.015 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:06.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:06.015 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.996964+0000", 2023-10-01T09:20:06.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:06.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:06.015 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.996964+0000", 2023-10-01T09:20:06.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:06.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:06.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.996963+0000", 2023-10-01T09:20:06.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:20:06.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:06.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.996986+0000", 2023-10-01T09:20:06.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3286999999999999e-05 2023-10-01T09:20:06.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:06.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.996997+0000", 2023-10-01T09:20:06.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1019000000000001e-05 2023-10-01T09:20:06.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:06.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.997004+0000", 2023-10-01T09:20:06.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.199e-06 2023-10-01T09:20:06.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:06.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.998582+0000", 2023-10-01T09:20:06.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015782319999999999 2023-10-01T09:20:06.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:06.018 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.998603+0000", 2023-10-01T09:20:06.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0627000000000001e-05 2023-10-01T09:20:06.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:06.018 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.005440+0000", 2023-10-01T09:20:06.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0068364280000000003 2023-10-01T09:20:06.019 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:06.019 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:06.019 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:06.019 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.019 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35070.0:43322 6.1 e192/179 6:85adda5e:::benchmark_data_smithi138_307751_object43321:head v 192'2658, mlcod=192'2656)", 2023-10-01T09:20:06.019 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:20:05.997918+0000", 2023-10-01T09:20:06.019 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0087202080000000001, 2023-10-01T09:20:06.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.008833601, 2023-10-01T09:20:06.019 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:06.019 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:20:06.019 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:06.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:06.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.997918+0000", 2023-10-01T09:20:06.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:06.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:06.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.997918+0000", 2023-10-01T09:20:06.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:06.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:06.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.997917+0000", 2023-10-01T09:20:06.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:20:06.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:06.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.997952+0000", 2023-10-01T09:20:06.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4709000000000003e-05 2023-10-01T09:20:06.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:06.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.997969+0000", 2023-10-01T09:20:06.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6546999999999999e-05 2023-10-01T09:20:06.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:06.022 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.997977+0000", 2023-10-01T09:20:06.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5129999999999996e-06 2023-10-01T09:20:06.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:06.022 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.999879+0000", 2023-10-01T09:20:06.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001901921 2023-10-01T09:20:06.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:06.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:05.999897+0000", 2023-10-01T09:20:06.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.821e-05 2023-10-01T09:20:06.023 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:06.023 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:06.023 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:06.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.023 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35070.0:43330 6.1 e192/179 6:84948d2b:::benchmark_data_smithi138_307751_object43329:head v 192'2659, mlcod=192'2656)", 2023-10-01T09:20:06.024 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:20:06.002075+0000", 2023-10-01T09:20:06.024 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0045632800000000003, 2023-10-01T09:20:06.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047372860000000003, 2023-10-01T09:20:06.024 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:06.024 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:20:06.024 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:06.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:06.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.002075+0000", 2023-10-01T09:20:06.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:06.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.025 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:06.025 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.002075+0000", 2023-10-01T09:20:06.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:06.025 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.025 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.025 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:06.025 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.002074+0000", 2023-10-01T09:20:06.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:20:06.025 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.025 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.025 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:06.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.002099+0000", 2023-10-01T09:20:06.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.491e-05 2023-10-01T09:20:06.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:06.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.002112+0000", 2023-10-01T09:20:06.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2465e-05 2023-10-01T09:20:06.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:06.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.002119+0000", 2023-10-01T09:20:06.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7840000000000004e-06 2023-10-01T09:20:06.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:06.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.003223+0000", 2023-10-01T09:20:06.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011033950000000001 2023-10-01T09:20:06.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:06.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.003245+0000", 2023-10-01T09:20:06.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1710999999999999e-05 2023-10-01T09:20:06.027 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:06.028 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:06.028 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:06.028 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.028 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.028 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35070.0:43335 6.1 e192/179 6:8a47f523:::benchmark_data_smithi138_307751_object43334:head v 192'2660, mlcod=192'2656)", 2023-10-01T09:20:06.028 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:20:06.005057+0000", 2023-10-01T09:20:06.028 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0015812459999999999, 2023-10-01T09:20:06.028 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001812917, 2023-10-01T09:20:06.028 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:06.028 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:20:06.028 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:06.028 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.029 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:06.029 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.005057+0000", 2023-10-01T09:20:06.029 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:06.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.029 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:06.029 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.005057+0000", 2023-10-01T09:20:06.029 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:06.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.029 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:06.029 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.005056+0000", 2023-10-01T09:20:06.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:20:06.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:06.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.005082+0000", 2023-10-01T09:20:06.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5831999999999999e-05 2023-10-01T09:20:06.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:06.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.005100+0000", 2023-10-01T09:20:06.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8394999999999999e-05 2023-10-01T09:20:06.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:06.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.005110+0000", 2023-10-01T09:20:06.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3749999999999992e-06 2023-10-01T09:20:06.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:06.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.005483+0000", 2023-10-01T09:20:06.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00037347699999999999 2023-10-01T09:20:06.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:06.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:06.032 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:06.032 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:06.005506+0000", 2023-10-01T09:20:06.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2532e-05 2023-10-01T09:20:06.032 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:06.032 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:06.032 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:06.032 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:06.032 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:20:06.032 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 4 2023-10-01T09:20:06.032 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:20:06.050 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:06.106 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:06.190 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:06.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:06.301 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:06.358 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:06.462 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:06.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:06.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:06.566 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:06.599 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:20:06.599 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:20:06.599 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T09:20:06.599 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.478434 2023-10-01T09:20:06.599 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0.478433625547684 2023-10-01T09:20:06.620 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:06.730 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:06.733 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:06 smithi138 ceph-mon[185886]: pgmap v475: 89 pgs: 89 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-01T09:20:06.733 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:06 smithi138 ceph-mon[192513]: pgmap v475: 89 pgs: 89 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-01T09:20:06.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:06.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:06 smithi190 ceph-mon[160435]: pgmap v475: 89 pgs: 89 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-01T09:20:06.835 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:06.865 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:06.912 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:06.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:06.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:06.956 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:20:06.969 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 23 16 44767 44751 121.528 92.625 0.002333 0.00819635 2023-10-01T09:20:07.016 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:07.074 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:07.169 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:07.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:07.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:07.276 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:07.336 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:07.431 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:07.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:07.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:07.506 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:20:07.523 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:20:07.541 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:20:07.589 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:07 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/507065924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:20:07.593 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:07.691 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:20:07.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:07.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:20:07.797 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:20:07.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:07 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/507065924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:20:07.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:07 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/507065924' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:20:07.871 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:07.966 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:20:07.969 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 24 16 46264 46248 120.363 93.5625 0.0059222 0.00830096 2023-10-01T09:20:07.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:07.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:20:08.071 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:08.114 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:08.155 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:08.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:08.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:08.262 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:08.290 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:08 smithi138 ceph-mon[192513]: pgmap v476: 89 pgs: 89 active+clean; 2.6 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-01T09:20:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:08 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:08 smithi138 ceph-mon[192513]: failed to encode map e193 with expected crc 2023-10-01T09:20:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:08 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/507065924' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:20:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:08 smithi138 ceph-mon[192513]: osdmap e193: 8 total, 8 up, 8 in 2023-10-01T09:20:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:08 smithi138 ceph-mon[192513]: failed to encode map e193 with expected crc 2023-10-01T09:20:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:08 smithi138 ceph-mon[185886]: pgmap v476: 89 pgs: 89 active+clean; 2.6 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-01T09:20:08.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:08 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:08.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:08 smithi138 ceph-mon[185886]: failed to encode map e193 with expected crc 2023-10-01T09:20:08.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:08 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/507065924' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:20:08.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:08 smithi138 ceph-mon[185886]: osdmap e193: 8 total, 8 up, 8 in 2023-10-01T09:20:08.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:08 smithi138 ceph-mon[185886]: failed to encode map e193 with expected crc 2023-10-01T09:20:08.650 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:08.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:08.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:08.756 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:08 smithi190 ceph-mon[160435]: pgmap v476: 89 pgs: 89 active+clean; 2.6 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-01T09:20:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:08 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:08 smithi190 ceph-mon[160435]: failed to encode map e193 with expected crc 2023-10-01T09:20:08.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:08 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/507065924' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:20:08.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:08 smithi190 ceph-mon[160435]: osdmap e193: 8 total, 8 up, 8 in 2023-10-01T09:20:08.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:08 smithi190 ceph-mon[160435]: failed to encode map e193 with expected crc 2023-10-01T09:20:08.811 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:08.830 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:08.885 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:08.887 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:08.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:08.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:08.969 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 25 16 47617 47601 118.931 84.5625 0.0224089 0.00840419 2023-10-01T09:20:08.991 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:20:09.053 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:09.074 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:09.240 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:20:09.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:09.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:20:09.351 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:09.416 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:09.969 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 26 16 49174 49158 118.1 97.3125 0.00960355 0.0084628 2023-10-01T09:20:10.486 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:10 smithi138 ceph-mon[185886]: pgmap v478: 89 pgs: 89 active+clean; 2.7 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-01T09:20:10.486 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:20:10.486 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:10 smithi190 ceph-mon[160435]: pgmap v478: 89 pgs: 89 active+clean; 2.7 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-01T09:20:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:20:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:10.812 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:20:10.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:10 smithi138 ceph-mon[192513]: pgmap v478: 89 pgs: 89 active+clean; 2.7 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-01T09:20:10.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:20:10.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:10.971 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 27 16 50722 50706 117.303 96.75 0.0104769 0.00851918 2023-10-01T09:20:10.977 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:10.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:10.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:11.086 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:20:11.112 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:11.156 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:20:11.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:11.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:20:11.261 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:11.309 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:11.971 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 28 16 51686 51670 115.266 60.25 0.00285029 0.00865204 2023-10-01T09:20:12.471 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:12.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:12.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:12.577 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:12.671 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:12.774 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:12.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:12.785 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:12 smithi138 ceph-mon[185886]: pgmap v479: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T09:20:12.785 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:12 smithi138 ceph-mon[192513]: pgmap v479: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T09:20:12.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:12.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:12 smithi190 ceph-mon[160435]: pgmap v479: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T09:20:12.809 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:12.887 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:12.942 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:12.971 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 29 16 52986 52970 114.093 81.25 0.00576017 0.00875925 2023-10-01T09:20:13.044 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:13.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:13.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:13.152 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:13.242 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:13.316 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:20:13.316 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:20:13.316 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:20:13.316 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:20:13.316 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:20:13.592 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:13.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:13.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:13.707 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:20:13.793 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:13.971 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 30 16 54337 54321 113.105 84.4375 0.0164679 0.00883651 2023-10-01T09:20:14.007 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:20:14.007 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:20:14.007 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:20:14.009 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:20:14.049 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:20:14.080 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:20:14.116 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:20:14.149 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:20:14.156 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:20:14.156 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:20:14.156 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:20:14.156 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:20:14.157 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:20:14.157 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:20:14.157 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:20:14.157 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:20:14.265 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:20:14.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:14.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:20:14.370 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:14.398 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:14.465 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:14.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:14.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:14.514 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:14 smithi138 ceph-mon[185886]: pgmap v480: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-01T09:20:14.514 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/896254290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:14.514 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3447631961' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.478433625547684}]: dispatch 2023-10-01T09:20:14.528 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.2 primary-affinity to 0.478434 (8313542) 2023-10-01T09:20:14.575 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:20:14.624 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:14.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:14 smithi190 ceph-mon[160435]: pgmap v480: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-01T09:20:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/896254290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3447631961' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.478433625547684}]: dispatch 2023-10-01T09:20:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:14 smithi138 ceph-mon[192513]: pgmap v480: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-01T09:20:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/896254290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3447631961' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.478433625547684}]: dispatch 2023-10-01T09:20:14.972 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 31 16 55200 55184 111.197 53.9375 0.00280847 0.00888678 2023-10-01T09:20:15.022 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:20:15.029 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:20:15.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:15.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:20:15.134 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:15.158 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:20:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:20:14] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.33.4" 2023-10-01T09:20:15.160 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:15.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:15 smithi138 ceph-mon[185886]: failed to encode map e194 with expected crc 2023-10-01T09:20:15.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:15 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3447631961' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.478433625547684}]': finished 2023-10-01T09:20:15.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:15 smithi138 ceph-mon[185886]: osdmap e194: 8 total, 8 up, 8 in 2023-10-01T09:20:15.519 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:15 smithi138 ceph-mon[185886]: osdmap e195: 8 total, 8 up, 8 in 2023-10-01T09:20:15.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:15 smithi138 ceph-mon[192513]: failed to encode map e194 with expected crc 2023-10-01T09:20:15.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:15 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3447631961' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.478433625547684}]': finished 2023-10-01T09:20:15.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:15 smithi138 ceph-mon[192513]: osdmap e194: 8 total, 8 up, 8 in 2023-10-01T09:20:15.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:15 smithi138 ceph-mon[192513]: osdmap e195: 8 total, 8 up, 8 in 2023-10-01T09:20:15.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:15 smithi190 ceph-mon[160435]: failed to encode map e194 with expected crc 2023-10-01T09:20:15.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:15 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3447631961' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.478433625547684}]': finished 2023-10-01T09:20:15.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:15 smithi190 ceph-mon[160435]: osdmap e194: 8 total, 8 up, 8 in 2023-10-01T09:20:15.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:15 smithi190 ceph-mon[160435]: osdmap e195: 8 total, 8 up, 8 in 2023-10-01T09:20:15.831 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:15.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:15.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:15.935 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:15.976 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 32 16 55840 55824 108.955 40 0.00603728 0.00917273 2023-10-01T09:20:15.993 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:16.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:16 smithi138 ceph-mon[192513]: pgmap v482: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T09:20:16.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:16 smithi138 ceph-mon[192513]: failed to encode map e195 with expected crc 2023-10-01T09:20:16.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:16 smithi138 ceph-mon[192513]: failed to encode map e195 with expected crc 2023-10-01T09:20:16.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:16 smithi138 ceph-mon[192513]: failed to encode map e195 with expected crc 2023-10-01T09:20:16.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:16 smithi138 ceph-mon[192513]: failed to encode map e195 with expected crc 2023-10-01T09:20:16.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:16 smithi138 ceph-mon[185886]: pgmap v482: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T09:20:16.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:16 smithi138 ceph-mon[185886]: failed to encode map e195 with expected crc 2023-10-01T09:20:16.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:16 smithi138 ceph-mon[185886]: failed to encode map e195 with expected crc 2023-10-01T09:20:16.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:16 smithi138 ceph-mon[185886]: failed to encode map e195 with expected crc 2023-10-01T09:20:16.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:16 smithi138 ceph-mon[185886]: failed to encode map e195 with expected crc 2023-10-01T09:20:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:16 smithi190 ceph-mon[160435]: pgmap v482: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T09:20:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:16 smithi190 ceph-mon[160435]: failed to encode map e195 with expected crc 2023-10-01T09:20:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:16 smithi190 ceph-mon[160435]: failed to encode map e195 with expected crc 2023-10-01T09:20:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:16 smithi190 ceph-mon[160435]: failed to encode map e195 with expected crc 2023-10-01T09:20:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:16 smithi190 ceph-mon[160435]: failed to encode map e195 with expected crc 2023-10-01T09:20:16.976 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 33 16 57278 57262 108.377 89.875 0.0182018 0.00922249 2023-10-01T09:20:17.331 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:17.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:17.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:17.384 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:17.450 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:17.511 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:17.665 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:17.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:17.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:17.774 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:20:17.825 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:17.918 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:20:17.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:17.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:20:17.977 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 34 16 58654 58638 107.719 86 0.0133058 0.00927894 2023-10-01T09:20:18.031 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:18.081 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:18.114 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:18 smithi138 ceph-mon[192513]: pgmap v484: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-01T09:20:18.172 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:18.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:18.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:18.285 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:18.344 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:18 smithi190 ceph-mon[160435]: pgmap v484: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-01T09:20:18.351 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:18.355 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:20:18.422 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:18.424 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:18 smithi138 ceph-mon[185886]: pgmap v484: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-01T09:20:18.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:18.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:18.531 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:20:18.577 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:18.631 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:20:18.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:18.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:20:18.737 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:20:18.763 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:18.804 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:20:18.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:18.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:20:18.909 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:20:18.960 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:18.982 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 35 16 59606 59590 106.336 59.5 0.010675 0.0093991 2023-10-01T09:20:19.146 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:20:19.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:19 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1341059161' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:20:19.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:19 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:20:19.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1341059161' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:20:19.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:19 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:20:19.449 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:20:19.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:19.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:20:19.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:19 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1341059161' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:20:19.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:19 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:20:19.559 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:20:19.601 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:19.979 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 36 16 60849 60833 105.54 77.6875 0.0198958 0.00947036 2023-10-01T09:20:20.049 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:20:20.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:20.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:20:20.156 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:20.180 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:20 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:20.180 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:20 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:20:20.180 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:20 smithi138 ceph-mon[185886]: osdmap e196: 8 total, 8 up, 8 in 2023-10-01T09:20:20.180 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:20 smithi138 ceph-mon[185886]: failed to encode map e196 with expected crc 2023-10-01T09:20:20.180 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:20 smithi138 ceph-mon[185886]: failed to encode map e196 with expected crc 2023-10-01T09:20:20.181 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:20 smithi138 ceph-mon[185886]: failed to encode map e196 with expected crc 2023-10-01T09:20:20.181 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:20 smithi138 ceph-mon[185886]: pgmap v486: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s 2023-10-01T09:20:20.181 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:20 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:20.181 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:20 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:20:20.181 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:20 smithi138 ceph-mon[192513]: osdmap e196: 8 total, 8 up, 8 in 2023-10-01T09:20:20.181 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:20 smithi138 ceph-mon[192513]: failed to encode map e196 with expected crc 2023-10-01T09:20:20.181 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:20 smithi138 ceph-mon[192513]: failed to encode map e196 with expected crc 2023-10-01T09:20:20.181 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:20 smithi138 ceph-mon[192513]: failed to encode map e196 with expected crc 2023-10-01T09:20:20.182 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:20 smithi138 ceph-mon[192513]: pgmap v486: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s 2023-10-01T09:20:20.211 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:20.305 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:20.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:20.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:20.412 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:20.468 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:20.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:20 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:20.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:20 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:20:20.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:20 smithi190 ceph-mon[160435]: osdmap e196: 8 total, 8 up, 8 in 2023-10-01T09:20:20.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:20 smithi190 ceph-mon[160435]: failed to encode map e196 with expected crc 2023-10-01T09:20:20.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:20 smithi190 ceph-mon[160435]: failed to encode map e196 with expected crc 2023-10-01T09:20:20.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:20 smithi190 ceph-mon[160435]: failed to encode map e196 with expected crc 2023-10-01T09:20:20.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:20 smithi190 ceph-mon[160435]: pgmap v486: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 55 MiB/s wr, 877 op/s 2023-10-01T09:20:20.576 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:20.614 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:20.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:20.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:20.712 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:20:20.712 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:20:20.724 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:20:20.777 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:20.866 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:20:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:20.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:20:20.973 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:20.979 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 37 16 62168 62152 104.916 82.4375 0.012033 0.00952544 2023-10-01T09:20:21.027 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:21.979 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 38 16 63278 63262 103.981 69.375 0.00843132 0.00961183 2023-10-01T09:20:22.189 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:22.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:22.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:22.204 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:20:22.211 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:20:22.211 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:20:22.212 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-01T09:20:22.212 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:20:22.212 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:20:22.212 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-01T09:20:22.296 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:22.364 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:22.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:22 smithi190 ceph-mon[160435]: pgmap v487: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-10-01T09:20:22.718 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:20:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:22 smithi138 ceph-mon[185886]: pgmap v487: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-10-01T09:20:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:22 smithi138 ceph-mon[192513]: pgmap v487: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-10-01T09:20:22.935 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:20:22.980 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 39 16 64375 64359 103.072 68.5625 0.014757 0.00969665 2023-10-01T09:20:23.281 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:23.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:23.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:23.390 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:20:23.453 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:23.462 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:23.666 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:20:23.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:23.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:20:23.768 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:23.845 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:23.927 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:23.943 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:20:23.948 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:23.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:23.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:23.980 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:20:23.980491+0000 min lat: 0.00099931 max lat: 0.996251 avg lat: 0.00976465 2023-10-01T09:20:23.980 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:20:23.980 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 40 16 65565 65549 102.355 74.375 0.00897007 0.00976465 2023-10-01T09:20:23.982 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:20:24.032 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:20:24.055 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:20:24.076 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:20:24.129 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:24.170 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:20:24.175 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:20:24.175 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:20:24.197 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:20:24.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:20:24.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:20:24.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:20:24.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:20:24.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:20:24.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:20:24.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:20:24.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:20:24.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:24 smithi190 ceph-mon[160435]: pgmap v488: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-01T09:20:24.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1376853330' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:24.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:24 smithi138 ceph-mon[192513]: pgmap v488: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-01T09:20:24.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1376853330' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:24 smithi138 ceph-mon[185886]: pgmap v488: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-01T09:20:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1376853330' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:24.980 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 41 16 66823 66807 101.776 78.625 0.00760074 0.00981998 2023-10-01T09:20:25.322 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:20:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:20:24] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.33.4" 2023-10-01T09:20:25.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:20:25.872 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:20:25.873 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:20:25.980 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 42 16 67649 67633 100.582 51.625 0.0185152 0.00993767 2023-10-01T09:20:26.225 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:20:26.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:26.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:20:26.348 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:26.407 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:26.487 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:26 smithi138 ceph-mon[185886]: pgmap v489: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-01T09:20:26.492 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:26.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:26.599 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:26.650 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:26.697 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:26.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:26.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:26.700 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:26 smithi190 ceph-mon[160435]: pgmap v489: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-01T09:20:26.801 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:26 smithi138 ceph-mon[192513]: pgmap v489: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-01T09:20:26.828 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:26.869 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:26.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:26.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:26.972 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:26.980 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 43 16 68549 68533 99.5512 56.25 0.00318538 0.00998231 2023-10-01T09:20:27.012 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:27.104 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:27.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:27.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:27.215 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:27.289 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:27.369 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:27.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:27.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:27.474 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:20:27.519 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:27.622 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:20:27.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:27.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:20:27.726 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:27.801 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:27.881 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:27.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:27.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:27.981 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 44 16 69737 69721 98.9763 74.25 0.0253134 0.0100974 2023-10-01T09:20:27.987 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:28.057 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:28.142 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:20:28.161 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:28.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:28.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:28.246 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:20:28.259 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:20:28.259 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:20:28.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.259 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:70012 6.8 6:1a58e7a8:::benchmark_data_smithi138_307751_object70011:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-10-01T09:20:28.259 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:20:28.233623+0000", 2023-10-01T09:20:28.260 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.021292846000000001, 2023-10-01T09:20:28.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.021330040000000001, 2023-10-01T09:20:28.260 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:28.260 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:28.260 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:28.260 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:28.260 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:28.260 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70012 2023-10-01T09:20:28.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.260 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:28.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:28.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.233623+0000", 2023-10-01T09:20:28.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:28.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:28.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.233623+0000", 2023-10-01T09:20:28.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:28.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.262 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:28.262 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.233623+0000", 2023-10-01T09:20:28.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:28.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:28.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.233643+0000", 2023-10-01T09:20:28.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0237e-05 2023-10-01T09:20:28.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:28.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.233650+0000", 2023-10-01T09:20:28.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2490000000000004e-06 2023-10-01T09:20:28.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:28.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.233656+0000", 2023-10-01T09:20:28.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3879999999999999e-06 2023-10-01T09:20:28.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:28.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239075+0000", 2023-10-01T09:20:28.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054188910000000003 2023-10-01T09:20:28.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:28.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239214+0000", 2023-10-01T09:20:28.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000139825 2023-10-01T09:20:28.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:28.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239363+0000", 2023-10-01T09:20:28.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000148796 2023-10-01T09:20:28.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:28.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.250008+0000", 2023-10-01T09:20:28.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010644409000000001 2023-10-01T09:20:28.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:28.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.252029+0000", 2023-10-01T09:20:28.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020213919999999999 2023-10-01T09:20:28.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:28.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.254905+0000", 2023-10-01T09:20:28.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028758130000000001 2023-10-01T09:20:28.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:28.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.254943+0000", 2023-10-01T09:20:28.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8565000000000001e-05 2023-10-01T09:20:28.269 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.269 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:28.269 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.270 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:70017 6.8 6:191a6244:::benchmark_data_smithi138_307751_object70016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-10-01T09:20:28.270 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:20:28.233660+0000", 2023-10-01T09:20:28.270 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.021255684, 2023-10-01T09:20:28.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.021429633, 2023-10-01T09:20:28.270 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:28.270 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:20:28.270 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:28.270 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:28.271 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:28.271 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70017 2023-10-01T09:20:28.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.271 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:28.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:28.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.233660+0000", 2023-10-01T09:20:28.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:28.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.272 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:28.272 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.233660+0000", 2023-10-01T09:20:28.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:28.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.272 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:28.272 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.233660+0000", 2023-10-01T09:20:28.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:28.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:28.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.233681+0000", 2023-10-01T09:20:28.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0803000000000001e-05 2023-10-01T09:20:28.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:28.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.233690+0000", 2023-10-01T09:20:28.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8759999999999994e-06 2023-10-01T09:20:28.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:28.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.233695+0000", 2023-10-01T09:20:28.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6779999999999997e-06 2023-10-01T09:20:28.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:28.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.247247+0000", 2023-10-01T09:20:28.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013551914 2023-10-01T09:20:28.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:28.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.247418+0000", 2023-10-01T09:20:28.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017118799999999999 2023-10-01T09:20:28.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:28.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.247566+0000", 2023-10-01T09:20:28.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014781700000000001 2023-10-01T09:20:28.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:28.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.252061+0000", 2023-10-01T09:20:28.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044948979999999998 2023-10-01T09:20:28.276 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.276 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:28.276 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.277 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35070.0:70006 6.0 e197/179 6:06e23e37:::benchmark_data_smithi138_307751_object70005:head v 197'4322, mlcod=197'4319)", 2023-10-01T09:20:28.277 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:20:28.239609+0000", 2023-10-01T09:20:28.277 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.015307094, 2023-10-01T09:20:28.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015566155, 2023-10-01T09:20:28.277 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:28.277 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:20:28.277 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:28.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:28.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239609+0000", 2023-10-01T09:20:28.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:28.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:28.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239609+0000", 2023-10-01T09:20:28.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:28.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:28.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239608+0000", 2023-10-01T09:20:28.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:28.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.280 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:28.280 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239630+0000", 2023-10-01T09:20:28.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1877000000000001e-05 2023-10-01T09:20:28.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.280 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:28.280 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239641+0000", 2023-10-01T09:20:28.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0309000000000001e-05 2023-10-01T09:20:28.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:28.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239647+0000", 2023-10-01T09:20:28.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3890000000000002e-06 2023-10-01T09:20:28.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:28.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.241407+0000", 2023-10-01T09:20:28.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017599029999999999 2023-10-01T09:20:28.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:28.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.241432+0000", 2023-10-01T09:20:28.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4878000000000001e-05 2023-10-01T09:20:28.282 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.282 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:28.282 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.282 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35070.0:70013 6.0 e197/179 6:02d6f262:::benchmark_data_smithi138_307751_object70012:head v 197'4323, mlcod=197'4319)", 2023-10-01T09:20:28.282 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:20:28.239653+0000", 2023-10-01T09:20:28.282 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.015263501, 2023-10-01T09:20:28.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01560424, 2023-10-01T09:20:28.283 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:28.283 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:20:28.283 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:28.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:28.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239653+0000", 2023-10-01T09:20:28.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:28.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:28.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239653+0000", 2023-10-01T09:20:28.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:28.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:28.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239652+0000", 2023-10-01T09:20:28.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:28.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:28.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239671+0000", 2023-10-01T09:20:28.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8851e-05 2023-10-01T09:20:28.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:28.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239680+0000", 2023-10-01T09:20:28.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8349999999999993e-06 2023-10-01T09:20:28.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:28.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.239684+0000", 2023-10-01T09:20:28.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3460000000000004e-06 2023-10-01T09:20:28.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:28.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.243625+0000", 2023-10-01T09:20:28.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039403529999999997 2023-10-01T09:20:28.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:28.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.243646+0000", 2023-10-01T09:20:28.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1471000000000002e-05 2023-10-01T09:20:28.286 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.286 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:28.286 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.287 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35070.0:70018 6.0 e197/179 6:0a582a53:::benchmark_data_smithi138_307751_object70017:head v 197'4324, mlcod=197'4319)", 2023-10-01T09:20:28.287 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:20:28.245247+0000", 2023-10-01T09:20:28.287 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0096688940000000008, 2023-10-01T09:20:28.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010082216, 2023-10-01T09:20:28.287 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:28.287 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:20:28.287 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:28.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.245247+0000", 2023-10-01T09:20:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.245247+0000", 2023-10-01T09:20:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.245246+0000", 2023-10-01T09:20:28.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:20:28.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:28.289 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.245279+0000", 2023-10-01T09:20:28.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2532e-05 2023-10-01T09:20:28.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:28.289 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.245295+0000", 2023-10-01T09:20:28.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6192e-05 2023-10-01T09:20:28.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:28.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.245303+0000", 2023-10-01T09:20:28.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9270000000000005e-06 2023-10-01T09:20:28.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:28.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.245609+0000", 2023-10-01T09:20:28.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000306595 2023-10-01T09:20:28.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:28.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.245631+0000", 2023-10-01T09:20:28.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1301000000000002e-05 2023-10-01T09:20:28.291 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.291 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:28.291 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.291 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35070.0:70019 6.0 e197/179 6:0278357d:::benchmark_data_smithi138_307751_object70018:head v 197'4325, mlcod=197'4319)", 2023-10-01T09:20:28.291 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:20:28.245310+0000", 2023-10-01T09:20:28.291 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0096062040000000001, 2023-10-01T09:20:28.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010080988000000001, 2023-10-01T09:20:28.291 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:28.292 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:20:28.292 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:28.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:28.292 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.245310+0000", 2023-10-01T09:20:28.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:28.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:28.292 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.245310+0000", 2023-10-01T09:20:28.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:28.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:28.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.245310+0000", 2023-10-01T09:20:28.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:28.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:28.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.245335+0000", 2023-10-01T09:20:28.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5863e-05 2023-10-01T09:20:28.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:28.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.245345+0000", 2023-10-01T09:20:28.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6870000000000006e-06 2023-10-01T09:20:28.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:28.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.245350+0000", 2023-10-01T09:20:28.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2220000000000001e-06 2023-10-01T09:20:28.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:28.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.250028+0000", 2023-10-01T09:20:28.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0046772029999999996 2023-10-01T09:20:28.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.295 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:28.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.250045+0000", 2023-10-01T09:20:28.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7527e-05 2023-10-01T09:20:28.295 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.295 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:28.295 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.295 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:70023 6.8 6:1ee0bf58:::benchmark_data_smithi138_307751_object70022:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-10-01T09:20:28.296 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:20:28.247348+0000", 2023-10-01T09:20:28.296 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0075680820000000003, 2023-10-01T09:20:28.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0081108650000000001, 2023-10-01T09:20:28.296 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:28.296 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:20:28.296 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:28.296 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:28.296 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:28.296 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 70023 2023-10-01T09:20:28.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.296 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:28.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:28.297 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.247348+0000", 2023-10-01T09:20:28.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:28.297 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.297 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.297 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:28.297 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.247348+0000", 2023-10-01T09:20:28.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:28.297 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.297 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.297 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:28.297 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.247347+0000", 2023-10-01T09:20:28.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:20:28.298 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.298 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.298 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:28.298 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.247367+0000", 2023-10-01T09:20:28.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9814e-05 2023-10-01T09:20:28.298 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.298 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.298 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:28.298 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.247378+0000", 2023-10-01T09:20:28.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.093e-05 2023-10-01T09:20:28.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:28.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.247385+0000", 2023-10-01T09:20:28.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2710000000000003e-06 2023-10-01T09:20:28.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:28.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.252392+0000", 2023-10-01T09:20:28.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050069540000000001 2023-10-01T09:20:28.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:28.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.252518+0000", 2023-10-01T09:20:28.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012580299999999999 2023-10-01T09:20:28.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:28.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:28.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:28.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:28.252609+0000", 2023-10-01T09:20:28.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0897999999999996e-05 2023-10-01T09:20:28.300 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.300 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:28.300 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.300 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:28.301 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:20:28.301 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 7 2023-10-01T09:20:28.301 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:20:28.301 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:28.302 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:28 smithi138 ceph-mon[185886]: pgmap v490: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T09:20:28.302 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/931108166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:20:28.302 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:28 smithi138 ceph-mon[192513]: pgmap v490: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T09:20:28.302 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/931108166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:20:28.386 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:28.532 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:28 smithi190 ceph-mon[160435]: pgmap v490: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T09:20:28.533 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/931108166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:20:28.853 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:20:28.853 INFO:teuthology.orchestra.run.smithi190.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T09:20:28.853 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:20:28.981 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 45 16 70835 70819 98.3019 68.625 0.00320611 0.0101582 2023-10-01T09:20:29.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:29 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:20:29.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:29 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:20:29.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:29 smithi190 ceph-mon[160435]: failed to encode map e197 with expected crc 2023-10-01T09:20:29.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:29 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/931108166' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:20:29.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:29 smithi190 ceph-mon[160435]: osdmap e197: 8 total, 8 up, 8 in 2023-10-01T09:20:29.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:29 smithi190 ceph-mon[160435]: failed to encode map e197 with expected crc 2023-10-01T09:20:29.539 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:29.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:29 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:20:29.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:29 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:20:29.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:29 smithi138 ceph-mon[192513]: failed to encode map e197 with expected crc 2023-10-01T09:20:29.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:29 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/931108166' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:20:29.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:29 smithi138 ceph-mon[192513]: osdmap e197: 8 total, 8 up, 8 in 2023-10-01T09:20:29.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:29 smithi138 ceph-mon[192513]: failed to encode map e197 with expected crc 2023-10-01T09:20:29.543 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:29 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:20:29.543 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:29 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:20:29.543 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:29 smithi138 ceph-mon[185886]: failed to encode map e197 with expected crc 2023-10-01T09:20:29.543 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:29 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/931108166' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:20:29.543 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:29 smithi138 ceph-mon[185886]: osdmap e197: 8 total, 8 up, 8 in 2023-10-01T09:20:29.543 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:29 smithi138 ceph-mon[185886]: failed to encode map e197 with expected crc 2023-10-01T09:20:29.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:29.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:29.647 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:20:29.690 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:29.981 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 46 16 71609 71593 97.217 48.375 0.00600554 0.0102813 2023-10-01T09:20:30.164 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:20:30.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:30.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:20:30.182 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:20:30.275 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:30.285 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:30 smithi138 ceph-mon[185886]: pgmap v492: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-01T09:20:30.286 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:30 smithi138 ceph-mon[192513]: pgmap v492: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-01T09:20:30.339 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:30.354 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:30.517 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:30.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:30.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:30.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:30 smithi190 ceph-mon[160435]: pgmap v492: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-10-01T09:20:30.634 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:30.692 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:30.766 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:30.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:30.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:30.879 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:20:30.949 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:30.983 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 47 16 72774 72758 96.6933 72.8125 0.0168343 0.0103359 2023-10-01T09:20:31.052 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:20:31.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:31.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:20:31.163 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:31.232 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:31.331 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:31.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:31.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:31.445 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:20:31.471 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:31.514 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:20:31.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:31.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:20:31.620 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:31.680 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:31.894 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:31.913 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:31.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:31.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:31.971 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:20:31.984 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 48 16 73752 73736 95.9497 61.125 0.0131583 0.0104169 2023-10-01T09:20:32.029 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:32.093 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:32.188 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:32.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:32.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:32.299 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:32.395 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:32.468 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:32 smithi138 ceph-mon[185886]: pgmap v493: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T09:20:32.469 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:32 smithi190 ceph-mon[160435]: pgmap v493: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T09:20:32.552 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:32.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:32.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:32.657 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:32.711 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:32.734 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:20:32.768 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:20:32.791 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:32.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:32.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:32.800 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:20:32.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:32 smithi138 ceph-mon[192513]: pgmap v493: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T09:20:32.832 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:20:32.865 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:20:32.897 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:20:32.903 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:20:32.936 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:20:32.967 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:20:32.968 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:32.975 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:20:32.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:20:32.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:20:32.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:20:32.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:20:32.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:20:32.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:20:32.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:20:32.985 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 49 16 74848 74832 95.3897 68.5 0.00683324 0.0104781 2023-10-01T09:20:33.069 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:20:33.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:33.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:20:33.173 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:20:33.254 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:33.373 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:20:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:33.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:20:33.489 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:33.512 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:33 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/889686345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:33.514 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:33.609 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:33.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:33.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:33.725 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:33.839 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:33 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/889686345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:33.839 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:33 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/889686345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:33.878 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:33.985 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 50 16 75938 75922 94.8446 68.125 0.014212 0.0105385 2023-10-01T09:20:34.348 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:34.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:34.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:34.376 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:34.461 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:34.511 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:34.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:34 smithi138 ceph-mon[185886]: pgmap v494: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-01T09:20:34.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:34 smithi138 ceph-mon[192513]: pgmap v494: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-01T09:20:34.590 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:34.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:34.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:34.696 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:34.742 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:34 smithi190 ceph-mon[160435]: pgmap v494: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-01T09:20:34.751 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:34.791 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:20:34.792 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:20:34.792 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T09:20:34.792 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-01T09:20:34.792 DEBUG:teuthology.orchestra.run.smithi190:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.5 2023-10-01T09:20:34.857 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:34.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:34.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:34.973 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:34.985 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 51 16 77108 77092 94.4188 73.125 0.0112465 0.0105855 2023-10-01T09:20:35.000 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:20:35.000 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:20:35.014 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:20:34 smithi190 systemd[1]: Stopping Ceph osd.5 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:20:35.026 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:35.126 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:35.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:35.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:35.236 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:35.290 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:20:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:20:34] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.33.4" 2023-10-01T09:20:35.306 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:35.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:35 smithi138 ceph-mon[192513]: osd.5 marked itself down and dead 2023-10-01T09:20:35.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:35 smithi138 ceph-mon[185886]: osd.5 marked itself down and dead 2023-10-01T09:20:35.674 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:35 smithi190 ceph-mon[160435]: osd.5 marked itself down and dead 2023-10-01T09:20:35.674 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:20:35 smithi190 conmon[112665]: debug 2023-10-01T09:20:35.352+0000 7f8dba8b6700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T09:20:35.674 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:20:35 smithi190 conmon[112665]: debug 2023-10-01T09:20:35.352+0000 7f8dba8b6700 -1 osd.5 197 *** Got signal Terminated *** 2023-10-01T09:20:35.674 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:20:35 smithi190 conmon[112665]: debug 2023-10-01T09:20:35.352+0000 7f8dba8b6700 -1 osd.5 197 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:20:35.677 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:35.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:35.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:35.780 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:35.806 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:35.985 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 52 16 77627 77611 93.2274 32.4375 0.00364084 0.010626 2023-10-01T09:20:36.361 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:20:36.390 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:36.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:36.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:36.409 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:20:36.497 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:36.535 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:36.586 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:36.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:36.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:36.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:36 smithi138 ceph-mon[185886]: pgmap v495: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-01T09:20:36.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:36 smithi138 ceph-mon[185886]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:20:36.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:36 smithi138 ceph-mon[185886]: failed to encode map e198 with expected crc 2023-10-01T09:20:36.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:36 smithi138 ceph-mon[185886]: osdmap e198: 8 total, 7 up, 8 in 2023-10-01T09:20:36.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:36 smithi138 ceph-mon[185886]: failed to encode map e198 with expected crc 2023-10-01T09:20:36.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:36 smithi138 ceph-mon[185886]: failed to encode map e198 with expected crc 2023-10-01T09:20:36.666 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:36 smithi138 ceph-mon[192513]: pgmap v495: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-01T09:20:36.666 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:36 smithi138 ceph-mon[192513]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:20:36.666 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:36 smithi138 ceph-mon[192513]: failed to encode map e198 with expected crc 2023-10-01T09:20:36.666 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:36 smithi138 ceph-mon[192513]: osdmap e198: 8 total, 7 up, 8 in 2023-10-01T09:20:36.666 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:36 smithi138 ceph-mon[192513]: failed to encode map e198 with expected crc 2023-10-01T09:20:36.666 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:36 smithi138 ceph-mon[192513]: failed to encode map e198 with expected crc 2023-10-01T09:20:36.690 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:36.725 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:36 smithi190 ceph-mon[160435]: pgmap v495: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-01T09:20:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:36 smithi190 ceph-mon[160435]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:20:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:36 smithi190 ceph-mon[160435]: failed to encode map e198 with expected crc 2023-10-01T09:20:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:36 smithi190 ceph-mon[160435]: osdmap e198: 8 total, 7 up, 8 in 2023-10-01T09:20:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:36 smithi190 ceph-mon[160435]: failed to encode map e198 with expected crc 2023-10-01T09:20:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:36 smithi190 ceph-mon[160435]: failed to encode map e198 with expected crc 2023-10-01T09:20:36.938 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:36.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:36.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:36.985 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 53 16 77978 77962 91.8829 21.9375 0.0274229 0.0108772 2023-10-01T09:20:37.017 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:37.045 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:20:37.076 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:37.147 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:37.296 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:20:37.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:37.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:20:37.308 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:20:36 smithi190 bash[206237]: 4ac584c1fb7d9883df72bcebe5ca30a50264434b640b236d1e22ed75486e1e58 2023-10-01T09:20:37.404 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:37.466 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:37.552 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:37.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:37.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:37.671 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:20:37.736 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:37 smithi190 ceph-mon[160435]: osdmap e199: 8 total, 7 up, 8 in 2023-10-01T09:20:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:37 smithi190 ceph-mon[160435]: failed to encode map e199 with expected crc 2023-10-01T09:20:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:37 smithi190 ceph-mon[160435]: failed to encode map e199 with expected crc 2023-10-01T09:20:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:37 smithi190 ceph-mon[160435]: failed to encode map e199 with expected crc 2023-10-01T09:20:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:37 smithi190 ceph-mon[160435]: failed to encode map e199 with expected crc 2023-10-01T09:20:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:37 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1080543278' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:20:37.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:37 smithi138 ceph-mon[185886]: osdmap e199: 8 total, 7 up, 8 in 2023-10-01T09:20:37.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:37 smithi138 ceph-mon[185886]: failed to encode map e199 with expected crc 2023-10-01T09:20:37.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:37 smithi138 ceph-mon[185886]: failed to encode map e199 with expected crc 2023-10-01T09:20:37.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:37 smithi138 ceph-mon[185886]: failed to encode map e199 with expected crc 2023-10-01T09:20:37.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:37 smithi138 ceph-mon[185886]: failed to encode map e199 with expected crc 2023-10-01T09:20:37.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:37 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1080543278' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:20:37.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:37 smithi138 ceph-mon[192513]: osdmap e199: 8 total, 7 up, 8 in 2023-10-01T09:20:37.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:37 smithi138 ceph-mon[192513]: failed to encode map e199 with expected crc 2023-10-01T09:20:37.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:37 smithi138 ceph-mon[192513]: failed to encode map e199 with expected crc 2023-10-01T09:20:37.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:37 smithi138 ceph-mon[192513]: failed to encode map e199 with expected crc 2023-10-01T09:20:37.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:37 smithi138 ceph-mon[192513]: failed to encode map e199 with expected crc 2023-10-01T09:20:37.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:37 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1080543278' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:20:37.985 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 54 16 79103 79087 91.4835 70.3125 0.00423928 0.0109259 2023-10-01T09:20:38.110 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:20:38.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:38 smithi190 ceph-mon[160435]: pgmap v498: 89 pgs: 1 activating+undersized, 3 active+undersized+degraded, 7 active+undersized, 11 peering, 8 stale+active+clean, 59 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 4896/230766 objects degraded (2.122%) 2023-10-01T09:20:38.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:38 smithi190 ceph-mon[160435]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-01T09:20:38.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:38 smithi190 ceph-mon[160435]: Health check failed: Degraded data redundancy: 4896/230766 objects degraded (2.122%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:20:38.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:38 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:38.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:38 smithi190 ceph-mon[160435]: failed to encode map e200 with expected crc 2023-10-01T09:20:38.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1080543278' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:20:38.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:38 smithi190 ceph-mon[160435]: osdmap e200: 8 total, 7 up, 8 in 2023-10-01T09:20:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:38 smithi138 ceph-mon[185886]: pgmap v498: 89 pgs: 1 activating+undersized, 3 active+undersized+degraded, 7 active+undersized, 11 peering, 8 stale+active+clean, 59 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 4896/230766 objects degraded (2.122%) 2023-10-01T09:20:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:38 smithi138 ceph-mon[185886]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-01T09:20:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:38 smithi138 ceph-mon[185886]: Health check failed: Degraded data redundancy: 4896/230766 objects degraded (2.122%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:20:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:38 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:38 smithi138 ceph-mon[185886]: failed to encode map e200 with expected crc 2023-10-01T09:20:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1080543278' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:20:38.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:38 smithi138 ceph-mon[185886]: osdmap e200: 8 total, 7 up, 8 in 2023-10-01T09:20:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:38 smithi138 ceph-mon[192513]: pgmap v498: 89 pgs: 1 activating+undersized, 3 active+undersized+degraded, 7 active+undersized, 11 peering, 8 stale+active+clean, 59 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 4896/230766 objects degraded (2.122%) 2023-10-01T09:20:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:38 smithi138 ceph-mon[192513]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-01T09:20:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:38 smithi138 ceph-mon[192513]: Health check failed: Degraded data redundancy: 4896/230766 objects degraded (2.122%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:20:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:38 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:38 smithi138 ceph-mon[192513]: failed to encode map e200 with expected crc 2023-10-01T09:20:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1080543278' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:20:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:38 smithi138 ceph-mon[192513]: osdmap e200: 8 total, 7 up, 8 in 2023-10-01T09:20:38.985 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 55 16 80058 80042 90.9056 59.6875 0.00245244 0.0109902 2023-10-01T09:20:39.275 DEBUG:teuthology.orchestra.run.smithi190:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-01T09:20:39.343 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:20:39.343 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-01T09:20:39.371 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:20:39.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:39.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:20:39.478 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:39.510 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:39.615 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:39.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:39.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:39.724 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:39.774 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:39.872 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:39.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:39.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:39.980 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:20:39.989 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 56 16 81104 81088 90.4498 65.375 0.0187674 0.01105 2023-10-01T09:20:40.065 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:40.360 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:20:40.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:40.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:20:40.460 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:20:40.467 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:40.514 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:20:40.531 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:40.558 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:20:40.595 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:20:40.630 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:20:40.644 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:20:40.644 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:20:40.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:20:40.644 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:20:40.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.644 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:1 6.8 6:16564324:::benchmark_data_smithi138_307751_object0:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.644 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.955433+0000", 2023-10-01T09:20:40.644 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.685226790000002, 2023-10-01T09:20:40.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39902836400000002, 2023-10-01T09:20:40.645 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.645 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.645 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.645 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.645 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.645 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 1 2023-10-01T09:20:40.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.645 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.955433+0000", 2023-10-01T09:20:40.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.955433+0000", 2023-10-01T09:20:40.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.955431+0000", 2023-10-01T09:20:40.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:20:40.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.955458+0000", 2023-10-01T09:20:40.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7098999999999999e-05 2023-10-01T09:20:40.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.955472+0000", 2023-10-01T09:20:40.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3360000000000001e-05 2023-10-01T09:20:40.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.955481+0000", 2023-10-01T09:20:40.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4569999999999994e-06 2023-10-01T09:20:40.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.955583+0000", 2023-10-01T09:20:40.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010173700000000001 2023-10-01T09:20:40.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.955601+0000", 2023-10-01T09:20:40.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8009e-05 2023-10-01T09:20:40.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.346397+0000", 2023-10-01T09:20:40.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39079611199999997 2023-10-01T09:20:40.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.346535+0000", 2023-10-01T09:20:40.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013753100000000001 2023-10-01T09:20:40.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:40.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.346652+0000", 2023-10-01T09:20:40.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117358 2023-10-01T09:20:40.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.349945+0000", 2023-10-01T09:20:40.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032931269999999999 2023-10-01T09:20:40.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.354304+0000", 2023-10-01T09:20:40.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043594740000000003 2023-10-01T09:20:40.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.354334+0000", 2023-10-01T09:20:40.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9241e-05 2023-10-01T09:20:40.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.354412+0000", 2023-10-01T09:20:40.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7849999999999995e-05 2023-10-01T09:20:40.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.354462+0000", 2023-10-01T09:20:40.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0039e-05 2023-10-01T09:20:40.652 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.652 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.652 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.653 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:25 6.8 6:16fe62e4:::benchmark_data_smithi138_307751_object24:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.653 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.959668+0000", 2023-10-01T09:20:40.653 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.680991767999998, 2023-10-01T09:20:40.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39844260300000001, 2023-10-01T09:20:40.653 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.653 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.653 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.653 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.653 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.653 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 25 2023-10-01T09:20:40.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.653 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.959668+0000", 2023-10-01T09:20:40.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.959668+0000", 2023-10-01T09:20:40.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.959667+0000", 2023-10-01T09:20:40.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:20:40.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.959691+0000", 2023-10-01T09:20:40.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4735e-05 2023-10-01T09:20:40.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.959708+0000", 2023-10-01T09:20:40.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6254999999999999e-05 2023-10-01T09:20:40.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.959716+0000", 2023-10-01T09:20:40.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7649999999999999e-06 2023-10-01T09:20:40.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.959770+0000", 2023-10-01T09:20:40.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3281e-05 2023-10-01T09:20:40.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.959779+0000", 2023-10-01T09:20:40.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4259999999999992e-06 2023-10-01T09:20:40.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.348819+0000", 2023-10-01T09:20:40.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38903942000000002 2023-10-01T09:20:40.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.348973+0000", 2023-10-01T09:20:40.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000153998 2023-10-01T09:20:40.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:40.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.349136+0000", 2023-10-01T09:20:40.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016375600000000001 2023-10-01T09:20:40.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.350444+0000", 2023-10-01T09:20:40.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001307784 2023-10-01T09:20:40.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.355614+0000", 2023-10-01T09:20:40.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005170076 2023-10-01T09:20:40.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358026+0000", 2023-10-01T09:20:40.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002411369 2023-10-01T09:20:40.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358075+0000", 2023-10-01T09:20:40.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9880999999999999e-05 2023-10-01T09:20:40.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358111+0000", 2023-10-01T09:20:40.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5410999999999997e-05 2023-10-01T09:20:40.660 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.660 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.660 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.661 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:35 6.b 6:d5bc797b:::benchmark_data_smithi138_307751_object34:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.661 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.962449+0000", 2023-10-01T09:20:40.661 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.678210733999997, 2023-10-01T09:20:40.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38924400199999998, 2023-10-01T09:20:40.661 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.661 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.661 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.661 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.661 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.661 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 35 2023-10-01T09:20:40.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.662 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.962449+0000", 2023-10-01T09:20:40.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.962449+0000", 2023-10-01T09:20:40.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.962448+0000", 2023-10-01T09:20:40.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:20:40.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.962473+0000", 2023-10-01T09:20:40.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4870000000000001e-05 2023-10-01T09:20:40.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.962488+0000", 2023-10-01T09:20:40.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5455e-05 2023-10-01T09:20:40.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.962496+0000", 2023-10-01T09:20:40.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5220000000000001e-06 2023-10-01T09:20:40.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.962561+0000", 2023-10-01T09:20:40.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5369000000000004e-05 2023-10-01T09:20:40.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.962572+0000", 2023-10-01T09:20:40.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0511e-05 2023-10-01T09:20:40.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.341146+0000", 2023-10-01T09:20:40.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.37857471799999998 2023-10-01T09:20:40.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.341289+0000", 2023-10-01T09:20:40.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000142433 2023-10-01T09:20:40.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:20:40.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.341492+0000", 2023-10-01T09:20:40.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000202857 2023-10-01T09:20:40.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.343531+0000", 2023-10-01T09:20:40.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020389509999999998 2023-10-01T09:20:40.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.345855+0000", 2023-10-01T09:20:40.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023239240000000002 2023-10-01T09:20:40.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.351606+0000", 2023-10-01T09:20:40.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057516620000000003 2023-10-01T09:20:40.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.351651+0000", 2023-10-01T09:20:40.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4833000000000003e-05 2023-10-01T09:20:40.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.351693+0000", 2023-10-01T09:20:40.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2199e-05 2023-10-01T09:20:40.668 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.668 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.668 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.669 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:40 6.8 6:1cea41b3:::benchmark_data_smithi138_307751_object39:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.669 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.963341+0000", 2023-10-01T09:20:40.669 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.677318995999997, 2023-10-01T09:20:40.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39489699700000003, 2023-10-01T09:20:40.669 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.669 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.669 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.669 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.669 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.669 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 40 2023-10-01T09:20:40.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.670 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.963341+0000", 2023-10-01T09:20:40.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.963341+0000", 2023-10-01T09:20:40.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.963340+0000", 2023-10-01T09:20:40.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:20:40.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.963391+0000", 2023-10-01T09:20:40.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1443999999999997e-05 2023-10-01T09:20:40.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.963407+0000", 2023-10-01T09:20:40.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.541e-05 2023-10-01T09:20:40.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.963414+0000", 2023-10-01T09:20:40.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5179999999999998e-06 2023-10-01T09:20:40.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.964572+0000", 2023-10-01T09:20:40.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011578809999999999 2023-10-01T09:20:40.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.964581+0000", 2023-10-01T09:20:40.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.268e-06 2023-10-01T09:20:40.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.349465+0000", 2023-10-01T09:20:40.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38488357000000001 2023-10-01T09:20:40.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.349579+0000", 2023-10-01T09:20:40.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113818 2023-10-01T09:20:40.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:40.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.349713+0000", 2023-10-01T09:20:40.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000133795 2023-10-01T09:20:40.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.350778+0000", 2023-10-01T09:20:40.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001065919 2023-10-01T09:20:40.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.356162+0000", 2023-10-01T09:20:40.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005383461 2023-10-01T09:20:40.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358151+0000", 2023-10-01T09:20:40.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001988863 2023-10-01T09:20:40.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358191+0000", 2023-10-01T09:20:40.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0259000000000002e-05 2023-10-01T09:20:40.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358238+0000", 2023-10-01T09:20:40.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7006999999999999e-05 2023-10-01T09:20:40.677 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.677 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.677 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.677 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:42 6.8 6:1922c13c:::benchmark_data_smithi138_307751_object41:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.677 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.964893+0000", 2023-10-01T09:20:40.677 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.675767507000003, 2023-10-01T09:20:40.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39343032700000002, 2023-10-01T09:20:40.677 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.677 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.678 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.678 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.678 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.678 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 42 2023-10-01T09:20:40.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.678 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.964893+0000", 2023-10-01T09:20:40.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.964893+0000", 2023-10-01T09:20:40.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.964891+0000", 2023-10-01T09:20:40.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:20:40.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.964913+0000", 2023-10-01T09:20:40.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1486999999999999e-05 2023-10-01T09:20:40.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.964925+0000", 2023-10-01T09:20:40.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.173e-05 2023-10-01T09:20:40.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.964933+0000", 2023-10-01T09:20:40.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8809999999999996e-06 2023-10-01T09:20:40.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.964983+0000", 2023-10-01T09:20:40.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0300999999999998e-05 2023-10-01T09:20:40.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.964993+0000", 2023-10-01T09:20:40.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0139000000000001e-05 2023-10-01T09:20:40.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.349969+0000", 2023-10-01T09:20:40.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38497641500000002 2023-10-01T09:20:40.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.350069+0000", 2023-10-01T09:20:40.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000100116 2023-10-01T09:20:40.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:40.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.350208+0000", 2023-10-01T09:20:40.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000138813 2023-10-01T09:20:40.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.352581+0000", 2023-10-01T09:20:40.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023722050000000001 2023-10-01T09:20:40.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.356342+0000", 2023-10-01T09:20:40.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037613519999999999 2023-10-01T09:20:40.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358275+0000", 2023-10-01T09:20:40.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0019332380000000001 2023-10-01T09:20:40.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358303+0000", 2023-10-01T09:20:40.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8008999999999999e-05 2023-10-01T09:20:40.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358323+0000", 2023-10-01T09:20:40.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9758999999999998e-05 2023-10-01T09:20:40.686 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.686 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.686 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.686 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:51 6.8 6:13c6f5e4:::benchmark_data_smithi138_307751_object50:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.687 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.965741+0000", 2023-10-01T09:20:40.687 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.674918673000001, 2023-10-01T09:20:40.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39274039100000002, 2023-10-01T09:20:40.687 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.687 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.687 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.687 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.687 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.687 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 51 2023-10-01T09:20:40.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.687 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.965741+0000", 2023-10-01T09:20:40.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.965741+0000", 2023-10-01T09:20:40.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.965740+0000", 2023-10-01T09:20:40.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:20:40.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.965792+0000", 2023-10-01T09:20:40.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2021000000000002e-05 2023-10-01T09:20:40.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.965809+0000", 2023-10-01T09:20:40.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6198999999999999e-05 2023-10-01T09:20:40.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.965817+0000", 2023-10-01T09:20:40.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0139999999999998e-06 2023-10-01T09:20:40.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.965871+0000", 2023-10-01T09:20:40.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4177e-05 2023-10-01T09:20:40.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.965897+0000", 2023-10-01T09:20:40.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5712e-05 2023-10-01T09:20:40.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.350469+0000", 2023-10-01T09:20:40.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38457218700000001 2023-10-01T09:20:40.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.350544+0000", 2023-10-01T09:20:40.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5804999999999997e-05 2023-10-01T09:20:40.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:40.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.350635+0000", 2023-10-01T09:20:40.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0357000000000002e-05 2023-10-01T09:20:40.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.352586+0000", 2023-10-01T09:20:40.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0019509530000000001 2023-10-01T09:20:40.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.356368+0000", 2023-10-01T09:20:40.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037817710000000002 2023-10-01T09:20:40.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358382+0000", 2023-10-01T09:20:40.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002014521 2023-10-01T09:20:40.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358442+0000", 2023-10-01T09:20:40.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9985000000000003e-05 2023-10-01T09:20:40.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358482+0000", 2023-10-01T09:20:40.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.968e-05 2023-10-01T09:20:40.694 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.694 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.694 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.694 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:55 6.8 6:1422a6e9:::benchmark_data_smithi138_307751_object54:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.695 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.966838+0000", 2023-10-01T09:20:40.695 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.673821824999997, 2023-10-01T09:20:40.695 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.39202392200000002, 2023-10-01T09:20:40.695 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.695 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.695 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.695 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.695 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.695 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 55 2023-10-01T09:20:40.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.695 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.966838+0000", 2023-10-01T09:20:40.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.966838+0000", 2023-10-01T09:20:40.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.966838+0000", 2023-10-01T09:20:40.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:40.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.966887+0000", 2023-10-01T09:20:40.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9669000000000002e-05 2023-10-01T09:20:40.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.966902+0000", 2023-10-01T09:20:40.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4398e-05 2023-10-01T09:20:40.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.966910+0000", 2023-10-01T09:20:40.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8339999999999999e-06 2023-10-01T09:20:40.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.966959+0000", 2023-10-01T09:20:40.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9410000000000003e-05 2023-10-01T09:20:40.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.966969+0000", 2023-10-01T09:20:40.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9040000000000004e-06 2023-10-01T09:20:40.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.350806+0000", 2023-10-01T09:20:40.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38383678700000001 2023-10-01T09:20:40.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.350916+0000", 2023-10-01T09:20:40.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110213 2023-10-01T09:20:40.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:40.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.351049+0000", 2023-10-01T09:20:40.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000132874 2023-10-01T09:20:40.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.352810+0000", 2023-10-01T09:20:40.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001761507 2023-10-01T09:20:40.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.356395+0000", 2023-10-01T09:20:40.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035850320000000001 2023-10-01T09:20:40.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358766+0000", 2023-10-01T09:20:40.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023707929999999999 2023-10-01T09:20:40.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358815+0000", 2023-10-01T09:20:40.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9251e-05 2023-10-01T09:20:40.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358862+0000", 2023-10-01T09:20:40.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6817999999999998e-05 2023-10-01T09:20:40.702 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.702 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.702 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.702 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:78 6.8 6:1ec1dd86:::benchmark_data_smithi138_307751_object77:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.702 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.972516+0000", 2023-10-01T09:20:40.702 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.668143923000002, 2023-10-01T09:20:40.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38644836900000001, 2023-10-01T09:20:40.703 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.703 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.703 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.703 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.703 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.703 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 78 2023-10-01T09:20:40.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.703 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.972516+0000", 2023-10-01T09:20:40.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.972516+0000", 2023-10-01T09:20:40.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.972514+0000", 2023-10-01T09:20:40.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:20:40.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.972542+0000", 2023-10-01T09:20:40.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7696999999999999e-05 2023-10-01T09:20:40.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.972560+0000", 2023-10-01T09:20:40.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7740999999999998e-05 2023-10-01T09:20:40.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.972569+0000", 2023-10-01T09:20:40.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1970000000000002e-06 2023-10-01T09:20:40.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.972627+0000", 2023-10-01T09:20:40.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8012e-05 2023-10-01T09:20:40.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.972638+0000", 2023-10-01T09:20:40.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1534e-05 2023-10-01T09:20:40.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.351314+0000", 2023-10-01T09:20:40.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.37867604300000002 2023-10-01T09:20:40.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.351421+0000", 2023-10-01T09:20:40.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010634800000000001 2023-10-01T09:20:40.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:40.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.351549+0000", 2023-10-01T09:20:40.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012852299999999999 2023-10-01T09:20:40.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.352816+0000", 2023-10-01T09:20:40.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012667209999999999 2023-10-01T09:20:40.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.356414+0000", 2023-10-01T09:20:40.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035976279999999999 2023-10-01T09:20:40.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358897+0000", 2023-10-01T09:20:40.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024837140000000001 2023-10-01T09:20:40.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358936+0000", 2023-10-01T09:20:40.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8383000000000002e-05 2023-10-01T09:20:40.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358964+0000", 2023-10-01T09:20:40.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8813e-05 2023-10-01T09:20:40.710 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.710 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.710 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.710 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:80 6.b 6:d13ccf8f:::benchmark_data_smithi138_307751_object79:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.710 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.973776+0000", 2023-10-01T09:20:40.710 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.666883935999998, 2023-10-01T09:20:40.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38318386900000001, 2023-10-01T09:20:40.711 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.711 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.711 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.711 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.711 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.711 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 80 2023-10-01T09:20:40.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.711 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.973776+0000", 2023-10-01T09:20:40.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.973776+0000", 2023-10-01T09:20:40.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.973775+0000", 2023-10-01T09:20:40.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:20:40.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.973798+0000", 2023-10-01T09:20:40.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2957999999999999e-05 2023-10-01T09:20:40.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.973814+0000", 2023-10-01T09:20:40.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5925000000000001e-05 2023-10-01T09:20:40.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.973820+0000", 2023-10-01T09:20:40.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3899999999999998e-06 2023-10-01T09:20:40.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.973876+0000", 2023-10-01T09:20:40.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6019e-05 2023-10-01T09:20:40.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.973887+0000", 2023-10-01T09:20:40.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.11e-05 2023-10-01T09:20:40.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.342185+0000", 2023-10-01T09:20:40.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.36829716299999998 2023-10-01T09:20:40.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.342306+0000", 2023-10-01T09:20:40.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012184399999999999 2023-10-01T09:20:40.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:20:40.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.342443+0000", 2023-10-01T09:20:40.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013633100000000001 2023-10-01T09:20:40.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.343998+0000", 2023-10-01T09:20:40.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015555 2023-10-01T09:20:40.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.353267+0000", 2023-10-01T09:20:40.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0092691909999999995 2023-10-01T09:20:40.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.356879+0000", 2023-10-01T09:20:40.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003611398 2023-10-01T09:20:40.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.356917+0000", 2023-10-01T09:20:40.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7679000000000003e-05 2023-10-01T09:20:40.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.356960+0000", 2023-10-01T09:20:40.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3470999999999998e-05 2023-10-01T09:20:40.718 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.718 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.718 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.718 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:81 6.8 6:14eb823a:::benchmark_data_smithi138_307751_object80:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.718 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.973979+0000", 2023-10-01T09:20:40.718 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.666681216999997, 2023-10-01T09:20:40.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38507853199999997, 2023-10-01T09:20:40.719 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.719 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.719 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.719 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.719 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.719 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 81 2023-10-01T09:20:40.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.719 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.973979+0000", 2023-10-01T09:20:40.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.720 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.720 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.720 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.973979+0000", 2023-10-01T09:20:40.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.720 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.720 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.720 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.973978+0000", 2023-10-01T09:20:40.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:40.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.973996+0000", 2023-10-01T09:20:40.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7960000000000001e-05 2023-10-01T09:20:40.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974008+0000", 2023-10-01T09:20:40.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1426e-05 2023-10-01T09:20:40.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974014+0000", 2023-10-01T09:20:40.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5729999999999996e-06 2023-10-01T09:20:40.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974067+0000", 2023-10-01T09:20:40.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2913e-05 2023-10-01T09:20:40.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974076+0000", 2023-10-01T09:20:40.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8850000000000005e-06 2023-10-01T09:20:40.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.351788+0000", 2023-10-01T09:20:40.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.37771203599999997 2023-10-01T09:20:40.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.351903+0000", 2023-10-01T09:20:40.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115247 2023-10-01T09:20:40.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:40.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.351998+0000", 2023-10-01T09:20:40.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4989999999999997e-05 2023-10-01T09:20:40.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.352820+0000", 2023-10-01T09:20:40.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00082195100000000002 2023-10-01T09:20:40.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.356600+0000", 2023-10-01T09:20:40.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037797569999999999 2023-10-01T09:20:40.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358996+0000", 2023-10-01T09:20:40.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002395683 2023-10-01T09:20:40.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.359029+0000", 2023-10-01T09:20:40.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2830000000000002e-05 2023-10-01T09:20:40.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.359057+0000", 2023-10-01T09:20:40.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8841000000000001e-05 2023-10-01T09:20:40.726 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.726 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.726 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.726 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:82 6.b 6:db81643f:::benchmark_data_smithi138_307751_object81:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.726 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.974028+0000", 2023-10-01T09:20:40.726 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.666631629999998, 2023-10-01T09:20:40.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38305284000000001, 2023-10-01T09:20:40.727 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.727 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.727 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.727 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.727 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.727 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 82 2023-10-01T09:20:40.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.727 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974028+0000", 2023-10-01T09:20:40.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974028+0000", 2023-10-01T09:20:40.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974028+0000", 2023-10-01T09:20:40.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:40.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974079+0000", 2023-10-01T09:20:40.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1218999999999999e-05 2023-10-01T09:20:40.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974087+0000", 2023-10-01T09:20:40.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8199999999999997e-06 2023-10-01T09:20:40.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974094+0000", 2023-10-01T09:20:40.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2409999999999998e-06 2023-10-01T09:20:40.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974135+0000", 2023-10-01T09:20:40.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1208000000000003e-05 2023-10-01T09:20:40.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974143+0000", 2023-10-01T09:20:40.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0330000000000008e-06 2023-10-01T09:20:40.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.342754+0000", 2023-10-01T09:20:40.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.36861094100000003 2023-10-01T09:20:40.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.342857+0000", 2023-10-01T09:20:40.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000102476 2023-10-01T09:20:40.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:20:40.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.342970+0000", 2023-10-01T09:20:40.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112938 2023-10-01T09:20:40.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.344004+0000", 2023-10-01T09:20:40.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010340200000000001 2023-10-01T09:20:40.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.353312+0000", 2023-10-01T09:20:40.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0093086539999999995 2023-10-01T09:20:40.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.357000+0000", 2023-10-01T09:20:40.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036875969999999999 2023-10-01T09:20:40.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.357048+0000", 2023-10-01T09:20:40.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8257999999999998e-05 2023-10-01T09:20:40.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.357081+0000", 2023-10-01T09:20:40.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2907999999999997e-05 2023-10-01T09:20:40.734 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.734 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.734 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.734 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:83 6.b 6:d600dac4:::benchmark_data_smithi138_307751_object82:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.734 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.974265+0000", 2023-10-01T09:20:40.734 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.666395317999999, 2023-10-01T09:20:40.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38310116700000002, 2023-10-01T09:20:40.734 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.735 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.735 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.735 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.735 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.735 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 83 2023-10-01T09:20:40.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.735 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974265+0000", 2023-10-01T09:20:40.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974265+0000", 2023-10-01T09:20:40.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974264+0000", 2023-10-01T09:20:40.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:20:40.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974288+0000", 2023-10-01T09:20:40.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3748e-05 2023-10-01T09:20:40.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974304+0000", 2023-10-01T09:20:40.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6549999999999999e-05 2023-10-01T09:20:40.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974313+0000", 2023-10-01T09:20:40.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8470000000000002e-06 2023-10-01T09:20:40.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974378+0000", 2023-10-01T09:20:40.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5013999999999998e-05 2023-10-01T09:20:40.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.974390+0000", 2023-10-01T09:20:40.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2296999999999999e-05 2023-10-01T09:20:40.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.343157+0000", 2023-10-01T09:20:40.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.36876618999999999 2023-10-01T09:20:40.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.343260+0000", 2023-10-01T09:20:40.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103448 2023-10-01T09:20:40.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:20:40.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.343370+0000", 2023-10-01T09:20:40.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109905 2023-10-01T09:20:40.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.344516+0000", 2023-10-01T09:20:40.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011460159999999999 2023-10-01T09:20:40.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.353344+0000", 2023-10-01T09:20:40.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088285389999999998 2023-10-01T09:20:40.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.357283+0000", 2023-10-01T09:20:40.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039386179999999996 2023-10-01T09:20:40.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.357328+0000", 2023-10-01T09:20:40.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4960999999999999e-05 2023-10-01T09:20:40.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.357366+0000", 2023-10-01T09:20:40.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7868999999999999e-05 2023-10-01T09:20:40.742 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.742 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.742 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.742 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:87 6.8 6:147d7424:::benchmark_data_smithi138_307751_object86:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.742 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.975180+0000", 2023-10-01T09:20:40.743 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.665479611999999, 2023-10-01T09:20:40.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.38400252000000001, 2023-10-01T09:20:40.743 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.743 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.743 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.743 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.743 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.743 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 87 2023-10-01T09:20:40.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.743 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.975180+0000", 2023-10-01T09:20:40.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.975180+0000", 2023-10-01T09:20:40.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.975179+0000", 2023-10-01T09:20:40.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:20:40.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.975237+0000", 2023-10-01T09:20:40.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7225000000000002e-05 2023-10-01T09:20:40.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.975247+0000", 2023-10-01T09:20:40.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.075e-05 2023-10-01T09:20:40.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.975255+0000", 2023-10-01T09:20:40.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6990000000000003e-06 2023-10-01T09:20:40.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.746 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.975429+0000", 2023-10-01T09:20:40.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017436 2023-10-01T09:20:40.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.746 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.975443+0000", 2023-10-01T09:20:40.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3304e-05 2023-10-01T09:20:40.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.747 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.747 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.352159+0000", 2023-10-01T09:20:40.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.37671630900000003 2023-10-01T09:20:40.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.747 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.747 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.352252+0000", 2023-10-01T09:20:40.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3239999999999995e-05 2023-10-01T09:20:40.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:40.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.352369+0000", 2023-10-01T09:20:40.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011625600000000001 2023-10-01T09:20:40.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.352824+0000", 2023-10-01T09:20:40.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00045562000000000001 2023-10-01T09:20:40.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.357518+0000", 2023-10-01T09:20:40.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0046938889999999997 2023-10-01T09:20:40.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.359091+0000", 2023-10-01T09:20:40.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015725369999999999 2023-10-01T09:20:40.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.359134+0000", 2023-10-01T09:20:40.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3912000000000002e-05 2023-10-01T09:20:40.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.359183+0000", 2023-10-01T09:20:40.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8486000000000003e-05 2023-10-01T09:20:40.750 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.750 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.750 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.750 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:94 6.b 6:db4326ab:::benchmark_data_smithi138_307751_object93:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.750 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.978367+0000", 2023-10-01T09:20:40.750 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.662293085000002, 2023-10-01T09:20:40.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.37960089200000002, 2023-10-01T09:20:40.751 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.751 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.751 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.751 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.751 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.751 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 94 2023-10-01T09:20:40.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.751 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978367+0000", 2023-10-01T09:20:40.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978367+0000", 2023-10-01T09:20:40.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978366+0000", 2023-10-01T09:20:40.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:20:40.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978407+0000", 2023-10-01T09:20:40.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1350999999999998e-05 2023-10-01T09:20:40.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978419+0000", 2023-10-01T09:20:40.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1786e-05 2023-10-01T09:20:40.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978425+0000", 2023-10-01T09:20:40.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0650000000000004e-06 2023-10-01T09:20:40.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978477+0000", 2023-10-01T09:20:40.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2194000000000002e-05 2023-10-01T09:20:40.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978488+0000", 2023-10-01T09:20:40.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0294e-05 2023-10-01T09:20:40.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.343560+0000", 2023-10-01T09:20:40.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.36507196800000002 2023-10-01T09:20:40.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.343629+0000", 2023-10-01T09:20:40.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9523000000000005e-05 2023-10-01T09:20:40.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:20:40.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.343744+0000", 2023-10-01T09:20:40.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000114418 2023-10-01T09:20:40.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.756 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.756 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.344522+0000", 2023-10-01T09:20:40.756 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:40.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00077840099999999996 2023-10-01T09:20:40.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.757 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.353577+0000", 2023-10-01T09:20:40.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0090548429999999999 2023-10-01T09:20:40.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.357854+0000", 2023-10-01T09:20:40.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042769879999999998 2023-10-01T09:20:40.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.357916+0000", 2023-10-01T09:20:40.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1828999999999994e-05 2023-10-01T09:20:40.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.759 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.357968+0000", 2023-10-01T09:20:40.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2120999999999998e-05 2023-10-01T09:20:40.759 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.759 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.759 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.759 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96 6.b 6:db85f06f:::benchmark_data_smithi138_307751_object95:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.759 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.978712+0000", 2023-10-01T09:20:40.759 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.661947654000002, 2023-10-01T09:20:40.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.37938266399999998, 2023-10-01T09:20:40.760 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.760 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.760 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.760 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.760 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.760 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96 2023-10-01T09:20:40.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.760 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978712+0000", 2023-10-01T09:20:40.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978712+0000", 2023-10-01T09:20:40.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978712+0000", 2023-10-01T09:20:40.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:40.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978749+0000", 2023-10-01T09:20:40.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6820000000000003e-05 2023-10-01T09:20:40.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978757+0000", 2023-10-01T09:20:40.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1140000000000006e-06 2023-10-01T09:20:40.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978761+0000", 2023-10-01T09:20:40.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6519999999999997e-06 2023-10-01T09:20:40.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978798+0000", 2023-10-01T09:20:40.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6782999999999997e-05 2023-10-01T09:20:40.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.978806+0000", 2023-10-01T09:20:40.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9130000000000003e-06 2023-10-01T09:20:40.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.344035+0000", 2023-10-01T09:20:40.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.36522893099999998 2023-10-01T09:20:40.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.344148+0000", 2023-10-01T09:20:40.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113476 2023-10-01T09:20:40.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:20:40.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.344321+0000", 2023-10-01T09:20:40.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000172576 2023-10-01T09:20:40.765 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.345027+0000", 2023-10-01T09:20:40.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00070553599999999999 2023-10-01T09:20:40.765 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.353610+0000", 2023-10-01T09:20:40.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085830579999999993 2023-10-01T09:20:40.765 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.766 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.766 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358030+0000", 2023-10-01T09:20:40.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044204689999999998 2023-10-01T09:20:40.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.766 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.766 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358064+0000", 2023-10-01T09:20:40.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4214999999999997e-05 2023-10-01T09:20:40.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.766 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.767 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358095+0000", 2023-10-01T09:20:40.767 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0735000000000003e-05 2023-10-01T09:20:40.767 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.767 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.767 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.767 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.767 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:113 6.b 6:d3ff2ca7:::benchmark_data_smithi138_307751_object112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T09:20:40.767 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:43.998724+0000", 2023-10-01T09:20:40.767 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 56.641936248999997, 2023-10-01T09:20:40.767 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.35951585899999999, 2023-10-01T09:20:40.767 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.768 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.768 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.768 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.768 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.768 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 113 2023-10-01T09:20:40.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.768 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.768 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.998724+0000", 2023-10-01T09:20:40.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.998724+0000", 2023-10-01T09:20:40.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.998723+0000", 2023-10-01T09:20:40.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:20:40.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.998740+0000", 2023-10-01T09:20:40.770 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7059999999999999e-05 2023-10-01T09:20:40.770 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.770 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.770 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.998749+0000", 2023-10-01T09:20:40.770 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6039999999999999e-06 2023-10-01T09:20:40.770 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.770 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.770 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.998755+0000", 2023-10-01T09:20:40.770 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0159999999999997e-06 2023-10-01T09:20:40.770 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.771 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.771 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.771 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.998811+0000", 2023-10-01T09:20:40.771 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5952999999999998e-05 2023-10-01T09:20:40.771 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.771 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.771 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:20:40.771 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:43.998823+0000", 2023-10-01T09:20:40.771 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1693e-05 2023-10-01T09:20:40.771 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.771 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.771 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.772 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.344547+0000", 2023-10-01T09:20:40.772 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.34572415200000001 2023-10-01T09:20:40.772 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.772 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.772 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.772 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.344650+0000", 2023-10-01T09:20:40.772 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010276399999999999 2023-10-01T09:20:40.772 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.772 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.772 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:20:40.772 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.344777+0000", 2023-10-01T09:20:40.772 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012661899999999999 2023-10-01T09:20:40.773 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.773 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.773 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.773 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.345140+0000", 2023-10-01T09:20:40.773 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00036348499999999998 2023-10-01T09:20:40.773 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.773 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.773 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.773 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.353640+0000", 2023-10-01T09:20:40.773 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085004060000000003 2023-10-01T09:20:40.773 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.773 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358146+0000", 2023-10-01T09:20:40.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045058190000000003 2023-10-01T09:20:40.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358186+0000", 2023-10-01T09:20:40.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9839000000000003e-05 2023-10-01T09:20:40.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:44.358240+0000", 2023-10-01T09:20:40.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3563000000000002e-05 2023-10-01T09:20:40.775 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.775 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.775 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.775 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:6257 6.8 6:155c40d1:::benchmark_data_smithi138_307751_object6256:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-10-01T09:20:40.775 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:46.461870+0000", 2023-10-01T09:20:40.775 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 54.178789576, 2023-10-01T09:20:40.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.32360313899999998, 2023-10-01T09:20:40.775 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.775 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.776 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.776 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.776 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.776 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 6257 2023-10-01T09:20:40.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.776 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.776 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:46.461870+0000", 2023-10-01T09:20:40.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.777 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:46.461870+0000", 2023-10-01T09:20:40.777 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.777 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:46.461869+0000", 2023-10-01T09:20:40.777 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:20:40.777 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.777 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.777 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.777 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:46.461890+0000", 2023-10-01T09:20:40.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0818e-05 2023-10-01T09:20:40.778 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.778 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.778 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:46.461903+0000", 2023-10-01T09:20:40.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2500999999999999e-05 2023-10-01T09:20:40.778 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.778 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.778 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.778 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:46.461909+0000", 2023-10-01T09:20:40.778 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0209999999999996e-06 2023-10-01T09:20:40.778 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.779 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.779 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:46.461948+0000", 2023-10-01T09:20:40.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9079000000000003e-05 2023-10-01T09:20:40.779 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.779 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.779 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:46.462052+0000", 2023-10-01T09:20:40.779 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000104066 2023-10-01T09:20:40.779 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.779 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.779 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:40.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:46.462185+0000", 2023-10-01T09:20:40.780 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013270900000000001 2023-10-01T09:20:40.780 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.780 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:46.462689+0000", 2023-10-01T09:20:40.780 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00050484100000000001 2023-10-01T09:20:40.780 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.780 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.780 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.780 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:46.463346+0000", 2023-10-01T09:20:40.780 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00065698799999999995 2023-10-01T09:20:40.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.781 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.781 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:46.785378+0000", 2023-10-01T09:20:40.781 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.32203167700000002 2023-10-01T09:20:40.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.781 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.781 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:46.785435+0000", 2023-10-01T09:20:40.781 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6564999999999998e-05 2023-10-01T09:20:40.781 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.781 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.782 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.782 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:46.785474+0000", 2023-10-01T09:20:40.782 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8962999999999999e-05 2023-10-01T09:20:40.782 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.782 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.782 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.782 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.782 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.782 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35070.0:41240 6.0 e192/179 6:0e2d0076:::benchmark_data_smithi138_307751_object41239:head v 192'2515, mlcod=192'2513)", 2023-10-01T09:20:40.782 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:20:04.539961+0000", 2023-10-01T09:20:40.782 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 36.100699288999998, 2023-10-01T09:20:40.782 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26431374600000002, 2023-10-01T09:20:40.783 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.783 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.783 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.539961+0000", 2023-10-01T09:20:40.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.783 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.783 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.783 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.783 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.539961+0000", 2023-10-01T09:20:40.783 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.539960+0000", 2023-10-01T09:20:40.784 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:20:40.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.784 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.784 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.539987+0000", 2023-10-01T09:20:40.784 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6944000000000002e-05 2023-10-01T09:20:40.784 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.784 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.785 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.785 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.539997+0000", 2023-10-01T09:20:40.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0063e-05 2023-10-01T09:20:40.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.785 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.785 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.540003+0000", 2023-10-01T09:20:40.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3860000000000004e-06 2023-10-01T09:20:40.785 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.785 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.785 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.785 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.540065+0000", 2023-10-01T09:20:40.785 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2334000000000003e-05 2023-10-01T09:20:40.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.786 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.786 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.540085+0000", 2023-10-01T09:20:40.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9789999999999999e-05 2023-10-01T09:20:40.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.786 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.786 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.804241+0000", 2023-10-01T09:20:40.786 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26415540599999998 2023-10-01T09:20:40.786 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.786 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.787 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.787 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.804275+0000", 2023-10-01T09:20:40.787 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3900999999999999e-05 2023-10-01T09:20:40.787 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.787 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.787 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.787 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.787 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.787 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:41242 6.8 6:13086b6d:::benchmark_data_smithi138_307751_object41241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e192)", 2023-10-01T09:20:40.787 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:20:04.541746+0000", 2023-10-01T09:20:40.787 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 36.098913873999997, 2023-10-01T09:20:40.787 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.262688266, 2023-10-01T09:20:40.788 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.788 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.788 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.788 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.788 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.788 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 41242 2023-10-01T09:20:40.788 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.788 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.788 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.788 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.788 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.541746+0000", 2023-10-01T09:20:40.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.789 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.789 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.789 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.541746+0000", 2023-10-01T09:20:40.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.789 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.789 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.789 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.541745+0000", 2023-10-01T09:20:40.789 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:20:40.789 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.541781+0000", 2023-10-01T09:20:40.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6254999999999998e-05 2023-10-01T09:20:40.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.790 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.541793+0000", 2023-10-01T09:20:40.790 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2006e-05 2023-10-01T09:20:40.790 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.790 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.790 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.541801+0000", 2023-10-01T09:20:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8960000000000003e-06 2023-10-01T09:20:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.541845+0000", 2023-10-01T09:20:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3849e-05 2023-10-01T09:20:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.541965+0000", 2023-10-01T09:20:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120345 2023-10-01T09:20:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.792 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:40.792 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.542096+0000", 2023-10-01T09:20:40.792 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013095899999999999 2023-10-01T09:20:40.792 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.792 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.792 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.804284+0000", 2023-10-01T09:20:40.792 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26218835600000001 2023-10-01T09:20:40.792 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.792 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.792 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.793 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.804338+0000", 2023-10-01T09:20:40.793 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3371000000000001e-05 2023-10-01T09:20:40.793 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.793 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.793 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.793 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.804345+0000", 2023-10-01T09:20:40.793 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2400000000000001e-06 2023-10-01T09:20:40.793 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.793 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.793 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.793 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.804392+0000", 2023-10-01T09:20:40.793 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6901999999999999e-05 2023-10-01T09:20:40.794 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.794 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.794 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.794 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.804434+0000", 2023-10-01T09:20:40.794 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2565999999999999e-05 2023-10-01T09:20:40.794 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.794 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.794 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.794 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.794 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.794 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:41243 6.8 6:1c3b6fab:::benchmark_data_smithi138_307751_object41242:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e192)", 2023-10-01T09:20:40.794 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:20:04.541815+0000", 2023-10-01T09:20:40.795 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 36.098844956999997, 2023-10-01T09:20:40.795 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26713870000000001, 2023-10-01T09:20:40.795 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:20:40.795 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:40.795 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:20:40.795 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:20:40.795 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:40.795 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 41243 2023-10-01T09:20:40.795 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.795 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:20:40.795 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.795 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:20:40.796 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.541815+0000", 2023-10-01T09:20:40.796 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.796 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.796 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.796 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:20:40.796 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.541815+0000", 2023-10-01T09:20:40.796 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:20:40.796 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.796 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.796 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:20:40.796 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.541814+0000", 2023-10-01T09:20:40.796 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:40.796 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.797 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:20:40.797 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.541837+0000", 2023-10-01T09:20:40.797 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2719000000000001e-05 2023-10-01T09:20:40.797 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.797 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:20:40.797 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.541845+0000", 2023-10-01T09:20:40.797 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7710000000000006e-06 2023-10-01T09:20:40.797 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.797 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.797 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:20:40.798 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.541851+0000", 2023-10-01T09:20:40.798 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1179999999999997e-06 2023-10-01T09:20:40.798 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.798 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.798 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:20:40.798 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.543731+0000", 2023-10-01T09:20:40.798 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001879765 2023-10-01T09:20:40.798 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.798 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.798 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:20:40.798 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.543851+0000", 2023-10-01T09:20:40.798 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000119936 2023-10-01T09:20:40.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.799 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.799 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:20:40.799 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.543984+0000", 2023-10-01T09:20:40.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013304199999999999 2023-10-01T09:20:40.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.799 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.799 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.799 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.805709+0000", 2023-10-01T09:20:40.799 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26172543300000001 2023-10-01T09:20:40.799 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.799 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:40.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.805748+0000", 2023-10-01T09:20:40.800 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8503e-05 2023-10-01T09:20:40.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:20:40.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.808856+0000", 2023-10-01T09:20:40.800 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003108029 2023-10-01T09:20:40.800 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.800 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.800 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:20:40.800 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.808901+0000", 2023-10-01T09:20:40.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5311e-05 2023-10-01T09:20:40.801 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:20:40.801 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:20:40.801 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:20:40.801 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:20:04.808954+0000", 2023-10-01T09:20:40.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2692999999999998e-05 2023-10-01T09:20:40.801 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.801 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.801 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.801 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:20:40.801 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:20:40.801 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:20:40.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:40.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:40.803 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:40 smithi138 ceph-mon[185886]: pgmap v500: 89 pgs: 1 activating+undersized, 7 active+undersized+degraded, 12 active+undersized, 11 peering, 3 stale+active+clean, 55 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 14812/234261 objects degraded (6.323%) 2023-10-01T09:20:40.803 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:40.803 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:20:40.803 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:40 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1296837810' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:40.803 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:40 smithi138 ceph-mon[192513]: pgmap v500: 89 pgs: 1 activating+undersized, 7 active+undersized+degraded, 12 active+undersized, 11 peering, 3 stale+active+clean, 55 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 14812/234261 objects degraded (6.323%) 2023-10-01T09:20:40.803 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:40.804 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:20:40.804 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:40 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1296837810' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:40.804 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:20:40.804 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:20:40.804 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:20:40.804 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:20:40.804 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:20:40.805 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:20:40.805 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:20:40.805 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-01T09:20:40.805 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:20:40.805 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:20:40.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:40 smithi190 ceph-mon[160435]: pgmap v500: 89 pgs: 1 activating+undersized, 7 active+undersized+degraded, 12 active+undersized, 11 peering, 3 stale+active+clean, 55 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 14812/234261 objects degraded (6.323%) 2023-10-01T09:20:40.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:20:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:40 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1296837810' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:40.903 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:20:40.944 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:40.985 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 57 16 82189 82173 90.0527 67.8125 0.00753973 0.0111007 2023-10-01T09:20:41.922 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:20:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:41.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:20:41.980 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:20:41.985 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 58 16 83317 83301 89.7156 70.5 0.00353649 0.0111414 2023-10-01T09:20:42.035 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:42.082 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:42.140 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:42.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:42.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:42.229 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:20:42.249 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:42.347 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:42.419 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:20:42.419 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:20:42.436 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:42.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:42.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:42.530 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:42 smithi138 ceph-mon[185886]: pgmap v501: 89 pgs: 1 activating+undersized, 12 active+undersized+degraded, 19 active+undersized, 2 peering, 55 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 25194/237765 objects degraded (10.596%) 2023-10-01T09:20:42.547 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:42.607 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:42.730 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:42.733 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:42.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:42.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:42 smithi190 ceph-mon[160435]: pgmap v501: 89 pgs: 1 activating+undersized, 12 active+undersized+degraded, 19 active+undersized, 2 peering, 55 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 25194/237765 objects degraded (10.596%) 2023-10-01T09:20:42.810 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:20:42.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:42 smithi138 ceph-mon[192513]: pgmap v501: 89 pgs: 1 activating+undersized, 12 active+undersized+degraded, 19 active+undersized, 2 peering, 55 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 25194/237765 objects degraded (10.596%) 2023-10-01T09:20:42.845 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:20:42.924 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:42.987 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 59 16 84419 84403 89.3624 68.875 0.0126935 0.0111858 2023-10-01T09:20:43.111 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:20:43.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:43.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:20:43.216 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:43.283 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:43.574 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:43.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:43.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:43.680 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:43.721 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:43 smithi190 ceph-mon[160435]: pgmap v502: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 26193/249453 objects degraded (10.500%) 2023-10-01T09:20:43.721 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:43 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 26193/249453 objects degraded (10.500%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:20:43.721 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:43 smithi190 ceph-mon[160435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-10-01T09:20:43.725 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:43.798 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:43.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:43.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:43.901 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:43.928 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:43.969 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:43.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:43.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:43.986 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:20:43.986401+0000 min lat: 0.00099931 max lat: 1.15245 avg lat: 0.0112115 2023-10-01T09:20:43.986 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:20:43.986 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 60 16 85372 85356 88.8658 59.5625 0.00361121 0.0112115 2023-10-01T09:20:44.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:43 smithi138 ceph-mon[192513]: pgmap v502: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 26193/249453 objects degraded (10.500%) 2023-10-01T09:20:44.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:43 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 26193/249453 objects degraded (10.500%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:20:44.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:43 smithi138 ceph-mon[192513]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-10-01T09:20:44.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:43 smithi138 ceph-mon[185886]: pgmap v502: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 26193/249453 objects degraded (10.500%) 2023-10-01T09:20:44.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:43 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 26193/249453 objects degraded (10.500%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:20:44.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:43 smithi138 ceph-mon[185886]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-10-01T09:20:44.073 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:44.118 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:44.344 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 6, 7, 2] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2023-10-01T09:20:44.344 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:20:44.345 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 3, 4, 5, 6, 7, 2] 2023-10-01T09:20:44.345 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-01T09:20:44.421 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:20:44.459 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:44.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:44.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:44.579 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:44.643 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:44.724 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:44.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:44.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:44.827 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:44.868 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:44.955 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:44.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:44.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:44.986 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 61 16 86294 86278 88.3537 57.625 0.0119174 0.0113134 2023-10-01T09:20:45.062 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:45.110 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:45.178 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:45.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:45.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:45.184 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:20:45.185 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:20:45.185 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:45.185 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:86480 6.e e200/179 6:7ec87178:::benchmark_data_smithi138_307751_object86479:head v 200'5457, mlcod=200'5455)", 2023-10-01T09:20:45.185 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:20:45.183943+0000", 2023-10-01T09:20:45.185 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.00069487399999999995, 2023-10-01T09:20:45.185 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00072128000000000003, 2023-10-01T09:20:45.185 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:45.185 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:20:45.185 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:45.185 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:45.185 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:45.186 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.183943+0000", 2023-10-01T09:20:45.186 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:45.186 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:45.186 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:45.186 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:45.186 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.183943+0000", 2023-10-01T09:20:45.186 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:20:45.186 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:45.186 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:45.186 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:45.186 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.183941+0000", 2023-10-01T09:20:45.187 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4529e-05 2023-10-01T09:20:45.187 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:45.187 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:45.187 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:45.187 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.183986+0000", 2023-10-01T09:20:45.187 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2082e-05 2023-10-01T09:20:45.187 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:45.187 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:45.187 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:45.187 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.184008+0000", 2023-10-01T09:20:45.187 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9950000000000005e-06 2023-10-01T09:20:45.187 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:45.188 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:45.188 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:45.188 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.184016+0000", 2023-10-01T09:20:45.188 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8270000000000003e-05 2023-10-01T09:20:45.188 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:45.188 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:45.188 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:45.188 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.184074+0000", 2023-10-01T09:20:45.188 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7350000000000001e-05 2023-10-01T09:20:45.188 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:45.188 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:45.188 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:45.189 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.184102+0000", 2023-10-01T09:20:45.189 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015891599999999999 2023-10-01T09:20:45.189 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:45.189 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:45.189 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:45.189 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:45.189 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:20:45.189 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 1 2023-10-01T09:20:45.189 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:20:45.281 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:45.283 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:45.284 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:45.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:20:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:20:44] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.33.4" 2023-10-01T09:20:45.378 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:45.502 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:45.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:45.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:45.605 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:45.639 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:45.960 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:20:45.986 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 62 16 87095 87079 87.7363 50.0625 0.00895335 0.0113924 2023-10-01T09:20:46.477 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:46 smithi138 ceph-mon[185886]: pgmap v503: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s; 26193/251406 objects degraded (10.419%) 2023-10-01T09:20:46.478 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:46 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3890780328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:20:46.478 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:46 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:20:46.518 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:20:46.580 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:20:46.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:46 smithi190 ceph-mon[160435]: pgmap v503: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s; 26193/251406 objects degraded (10.419%) 2023-10-01T09:20:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:46 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3890780328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:20:46.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:46 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:20:46.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:46 smithi138 ceph-mon[192513]: pgmap v503: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s; 26193/251406 objects degraded (10.419%) 2023-10-01T09:20:46.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:46 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3890780328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:20:46.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:46 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:20:46.988 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 63 16 88037 88021 87.2756 58.875 0.0985407 0.0114463 2023-10-01T09:20:47.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:47 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:47.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:47 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:20:47.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:47 smithi190 ceph-mon[160435]: osdmap e201: 8 total, 7 up, 8 in 2023-10-01T09:20:47.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:47 smithi190 ceph-mon[160435]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:47 smithi190 ceph-mon[160435]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:47 smithi190 ceph-mon[160435]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:47 smithi190 ceph-mon[160435]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:47 smithi190 ceph-mon[160435]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:47.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:20:47.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[185886]: osdmap e201: 8 total, 7 up, 8 in 2023-10-01T09:20:47.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[185886]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[185886]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[185886]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[185886]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[185886]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:47.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:20:47.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[192513]: osdmap e201: 8 total, 7 up, 8 in 2023-10-01T09:20:47.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[192513]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[192513]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[192513]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[192513]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:47 smithi138 ceph-mon[192513]: failed to encode map e201 with expected crc 2023-10-01T09:20:47.988 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 64 16 88889 88873 86.7441 53.25 0.0174721 0.0115234 2023-10-01T09:20:48.067 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:48.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:48.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:48.177 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:48.237 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:48.314 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:20:48.314 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:20:48.315 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:20:48.315 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:20:48.315 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:20:48.338 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:48.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:48.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:48.445 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:48.499 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:48 smithi138 ceph-mon[185886]: pgmap v505: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s; 27124/260517 objects degraded (10.412%) 2023-10-01T09:20:48.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:48 smithi138 ceph-mon[192513]: pgmap v505: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s; 27124/260517 objects degraded (10.412%) 2023-10-01T09:20:48.712 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:48.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:48.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:48.769 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:20:48.803 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:20:48.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:48 smithi190 ceph-mon[160435]: pgmap v505: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s; 27124/260517 objects degraded (10.412%) 2023-10-01T09:20:48.824 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:48.834 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:20:48.866 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:20:48.866 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:48.898 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:20:48.936 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:20:48.978 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:20:48.978 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:20:48.978 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:20:48.978 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:20:48.978 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:20:48.979 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:20:48.979 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-01T09:20:48.979 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:20:48.979 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:20:48.988 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 65 16 89907 89891 86.3885 63.625 0.00676982 0.011571 2023-10-01T09:20:49.144 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:49.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:49.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:49.153 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:20:49.253 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:49.270 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:49 smithi138 conmon[185862]: 2023-10-01T09:20:49.183+0000 7f70d64ec700 -1 mon.a@0(leader).osd e201 definitely_dead 0 2023-10-01T09:20:49.313 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:49.411 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:49.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:49.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:49.524 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:20:49.525 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4242165634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:49.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/161546029' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T09:20:49.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:49 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T09:20:49.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:49 smithi138 ceph-mon[185886]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-01T09:20:49.546 INFO:teuthology.orchestra.run.smithi138.stderr:marked out osd.6. 2023-10-01T09:20:49.581 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:49.624 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:20:49.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:49.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:20:49.682 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:20:49.728 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:49.752 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4242165634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:49.752 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/161546029' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T09:20:49.752 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:49 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T09:20:49.752 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:49 smithi190 ceph-mon[160435]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-01T09:20:49.754 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:49.808 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:49.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:49.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:49.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4242165634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:49.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/161546029' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T09:20:49.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:49 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T09:20:49.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:49 smithi138 ceph-mon[192513]: Client client.admin marked osd.6 out, while it was still marked up 2023-10-01T09:20:49.911 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:49.940 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:49.980 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:49.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:49.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:49.982 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:20:49.989 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 66 16 90501 90485 85.6423 37.125 0.00365386 0.0115941 2023-10-01T09:20:50.083 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:20:50.114 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:50.495 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:20:50.495 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:20:50.524 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:20:50.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:50.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:20:50.600 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:50.629 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:50.660 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:50.721 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:50.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:50.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:50.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:50 smithi190 ceph-mon[160435]: pgmap v506: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s; 27730/264132 objects degraded (10.499%) 2023-10-01T09:20:50.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:50 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 27730/264132 objects degraded (10.499%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:20:50.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:50 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T09:20:50.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:50 smithi190 ceph-mon[160435]: osdmap e202: 8 total, 7 up, 7 in 2023-10-01T09:20:50.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:50 smithi190 ceph-mon[160435]: failed to encode map e202 with expected crc 2023-10-01T09:20:50.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:50 smithi190 ceph-mon[160435]: failed to encode map e202 with expected crc 2023-10-01T09:20:50.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:50 smithi190 ceph-mon[160435]: failed to encode map e202 with expected crc 2023-10-01T09:20:50.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:50 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:50.810 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[185886]: pgmap v506: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s; 27730/264132 objects degraded (10.499%) 2023-10-01T09:20:50.810 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 27730/264132 objects degraded (10.499%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:20:50.810 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T09:20:50.810 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[185886]: osdmap e202: 8 total, 7 up, 7 in 2023-10-01T09:20:50.811 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[185886]: failed to encode map e202 with expected crc 2023-10-01T09:20:50.811 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[185886]: failed to encode map e202 with expected crc 2023-10-01T09:20:50.811 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[185886]: failed to encode map e202 with expected crc 2023-10-01T09:20:50.811 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:50.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[192513]: pgmap v506: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s; 27730/264132 objects degraded (10.499%) 2023-10-01T09:20:50.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 27730/264132 objects degraded (10.499%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:20:50.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T09:20:50.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[192513]: osdmap e202: 8 total, 7 up, 7 in 2023-10-01T09:20:50.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[192513]: failed to encode map e202 with expected crc 2023-10-01T09:20:50.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[192513]: failed to encode map e202 with expected crc 2023-10-01T09:20:50.812 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[192513]: failed to encode map e202 with expected crc 2023-10-01T09:20:50.812 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:50 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:50.825 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:50.855 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:50.902 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:50.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:50.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:50.989 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 67 16 90501 90485 84.3646 0 - 0.0115941 2023-10-01T09:20:51.008 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:20:51.037 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:51.181 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:20:51.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:51.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:20:51.279 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:20:51.280 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:20:51.285 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:20:51.311 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:51.540 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:51 smithi138 ceph-mon[185886]: failed to encode map e203 with expected crc 2023-10-01T09:20:51.540 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:51 smithi138 ceph-mon[185886]: osdmap e203: 8 total, 7 up, 7 in 2023-10-01T09:20:51.540 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:51 smithi138 ceph-mon[185886]: failed to encode map e203 with expected crc 2023-10-01T09:20:51.540 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:51 smithi138 ceph-mon[185886]: failed to encode map e203 with expected crc 2023-10-01T09:20:51.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:51 smithi138 ceph-mon[192513]: failed to encode map e203 with expected crc 2023-10-01T09:20:51.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:51 smithi138 ceph-mon[192513]: osdmap e203: 8 total, 7 up, 7 in 2023-10-01T09:20:51.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:51 smithi138 ceph-mon[192513]: failed to encode map e203 with expected crc 2023-10-01T09:20:51.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:51 smithi138 ceph-mon[192513]: failed to encode map e203 with expected crc 2023-10-01T09:20:51.548 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:20:51.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:51 smithi190 ceph-mon[160435]: failed to encode map e203 with expected crc 2023-10-01T09:20:51.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:51 smithi190 ceph-mon[160435]: osdmap e203: 8 total, 7 up, 7 in 2023-10-01T09:20:51.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:51 smithi190 ceph-mon[160435]: failed to encode map e203 with expected crc 2023-10-01T09:20:51.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:51 smithi190 ceph-mon[160435]: failed to encode map e203 with expected crc 2023-10-01T09:20:51.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:51.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:20:51.662 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:51.751 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:51.964 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:20:51.982 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:51.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:51.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:51.996 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 68 16 90514 90498 83.1313 0.40625 2.42489 0.0118332 2023-10-01T09:20:52.092 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:52.192 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:52.294 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:52.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:52.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:52.402 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:20:52.406 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:52.473 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:52.496 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:20:52.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[185886]: pgmap v509: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s; 27730/265656 objects degraded (10.438%) 2023-10-01T09:20:52.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[185886]: failed to encode map e204 with expected crc 2023-10-01T09:20:52.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[185886]: osdmap e204: 8 total, 7 up, 7 in 2023-10-01T09:20:52.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[185886]: failed to encode map e204 with expected crc 2023-10-01T09:20:52.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[185886]: failed to encode map e204 with expected crc 2023-10-01T09:20:52.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[185886]: osdmap e205: 8 total, 7 up, 7 in 2023-10-01T09:20:52.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[185886]: failed to encode map e205 with expected crc 2023-10-01T09:20:52.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[185886]: failed to encode map e205 with expected crc 2023-10-01T09:20:52.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[192513]: pgmap v509: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s; 27730/265656 objects degraded (10.438%) 2023-10-01T09:20:52.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[192513]: failed to encode map e204 with expected crc 2023-10-01T09:20:52.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[192513]: osdmap e204: 8 total, 7 up, 7 in 2023-10-01T09:20:52.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[192513]: failed to encode map e204 with expected crc 2023-10-01T09:20:52.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[192513]: failed to encode map e204 with expected crc 2023-10-01T09:20:52.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[192513]: osdmap e205: 8 total, 7 up, 7 in 2023-10-01T09:20:52.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[192513]: failed to encode map e205 with expected crc 2023-10-01T09:20:52.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:52 smithi138 ceph-mon[192513]: failed to encode map e205 with expected crc 2023-10-01T09:20:52.698 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:52.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:52.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:52.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:52 smithi190 ceph-mon[160435]: pgmap v509: 89 pgs: 13 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s; 27730/265656 objects degraded (10.438%) 2023-10-01T09:20:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:52 smithi190 ceph-mon[160435]: failed to encode map e204 with expected crc 2023-10-01T09:20:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:52 smithi190 ceph-mon[160435]: osdmap e204: 8 total, 7 up, 7 in 2023-10-01T09:20:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:52 smithi190 ceph-mon[160435]: failed to encode map e204 with expected crc 2023-10-01T09:20:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:52 smithi190 ceph-mon[160435]: failed to encode map e204 with expected crc 2023-10-01T09:20:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:52 smithi190 ceph-mon[160435]: osdmap e205: 8 total, 7 up, 7 in 2023-10-01T09:20:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:52 smithi190 ceph-mon[160435]: failed to encode map e205 with expected crc 2023-10-01T09:20:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:52 smithi190 ceph-mon[160435]: failed to encode map e205 with expected crc 2023-10-01T09:20:52.807 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:52.843 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:52.993 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 69 16 91022 91006 82.3869 31.75 0.033168 0.0121317 2023-10-01T09:20:53.056 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:53.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:53.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:53.159 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:53.166 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:20:53.262 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:53.516 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:53.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:53.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:53.637 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:53.688 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:53.800 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:53 smithi190 ceph-mon[160435]: failed to encode map e205 with expected crc 2023-10-01T09:20:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3203490633' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:20:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:53 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3203490633' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:20:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:53 smithi190 ceph-mon[160435]: osdmap e206: 8 total, 7 up, 7 in 2023-10-01T09:20:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:53 smithi190 ceph-mon[160435]: failed to encode map e206 with expected crc 2023-10-01T09:20:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:53 smithi190 ceph-mon[160435]: failed to encode map e206 with expected crc 2023-10-01T09:20:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:53 smithi190 ceph-mon[160435]: failed to encode map e206 with expected crc 2023-10-01T09:20:53.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[192513]: failed to encode map e205 with expected crc 2023-10-01T09:20:53.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3203490633' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:20:53.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:53.812 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3203490633' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:20:53.812 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[192513]: osdmap e206: 8 total, 7 up, 7 in 2023-10-01T09:20:53.812 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[192513]: failed to encode map e206 with expected crc 2023-10-01T09:20:53.812 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[192513]: failed to encode map e206 with expected crc 2023-10-01T09:20:53.812 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[192513]: failed to encode map e206 with expected crc 2023-10-01T09:20:53.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:53.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:53.817 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[185886]: failed to encode map e205 with expected crc 2023-10-01T09:20:53.817 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3203490633' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:20:53.817 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:20:53.817 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3203490633' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:20:53.817 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[185886]: osdmap e206: 8 total, 7 up, 7 in 2023-10-01T09:20:53.817 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[185886]: failed to encode map e206 with expected crc 2023-10-01T09:20:53.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[185886]: failed to encode map e206 with expected crc 2023-10-01T09:20:53.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:53 smithi138 ceph-mon[185886]: failed to encode map e206 with expected crc 2023-10-01T09:20:53.914 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:20:53.952 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:53.993 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 70 16 91575 91559 81.7039 34.5625 0.016088 0.0122336 2023-10-01T09:20:54.063 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:20:54.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:54.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:20:54.139 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:20:54.139 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:20:54.168 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:20:54.199 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:54.348 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:20:54.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:54.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:20:54.452 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:20:54.478 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:54.713 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:20:54.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:54.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:20:54.806 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:20:54.819 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:20:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:54 smithi138 ceph-mon[185886]: failed to encode map e206 with expected crc 2023-10-01T09:20:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:54 smithi138 ceph-mon[185886]: pgmap v513: 89 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 15 active+clean+remapped, 5 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 47 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 35 MiB/s wr, 555 op/s; 39699/272142 objects degraded (14.588%); 11391/272142 objects misplaced (4.186%); 149 KiB/s, 8 objects/s recovering 2023-10-01T09:20:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:54 smithi138 ceph-mon[192513]: failed to encode map e206 with expected crc 2023-10-01T09:20:54.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:54 smithi138 ceph-mon[192513]: pgmap v513: 89 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 15 active+clean+remapped, 5 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 47 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 35 MiB/s wr, 555 op/s; 39699/272142 objects degraded (14.588%); 11391/272142 objects misplaced (4.186%); 149 KiB/s, 8 objects/s recovering 2023-10-01T09:20:54.821 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:54.836 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:54 smithi190 ceph-mon[160435]: failed to encode map e206 with expected crc 2023-10-01T09:20:54.836 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:54 smithi190 ceph-mon[160435]: pgmap v513: 89 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 15 active+clean+remapped, 5 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 47 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 35 MiB/s wr, 555 op/s; 39699/272142 objects degraded (14.588%); 11391/272142 objects misplaced (4.186%); 149 KiB/s, 8 objects/s recovering 2023-10-01T09:20:54.910 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:54.993 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 71 16 91991 91975 80.9197 26 0.0301771 0.0123494 2023-10-01T09:20:55.180 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:20:55.190 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:20:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:20:54] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.33.4" 2023-10-01T09:20:55.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:55.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:20:55.237 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:20:55.292 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:55.416 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:55.560 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 7, 2] out_osds: [6] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2023-10-01T09:20:55.560 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:20:55.560 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-01T09:20:55.560 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-01T09:20:55.690 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:55.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:55.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:55.802 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:55.886 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:55.977 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:55.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:55.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:55.993 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 72 16 92520 92504 80.2553 33.0625 0.0793108 0.0124519 2023-10-01T09:20:56.081 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:56.114 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:56.327 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:56.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:56.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:56.434 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:20:56.521 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:56.546 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:56.546 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:20:56.547 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:56 smithi190 ceph-mon[160435]: pgmap v514: 89 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 17 active+clean+remapped, 5 remapped+peering, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 44 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 30 MiB/s wr, 474 op/s; 45197/272613 objects degraded (16.579%); 17243/272613 objects misplaced (6.325%); 145 KiB/s, 10 objects/s recovering 2023-10-01T09:20:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:20:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:56 smithi138 ceph-mon[192513]: pgmap v514: 89 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 17 active+clean+remapped, 5 remapped+peering, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 44 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 30 MiB/s wr, 474 op/s; 45197/272613 objects degraded (16.579%); 17243/272613 objects misplaced (6.325%); 145 KiB/s, 10 objects/s recovering 2023-10-01T09:20:56.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:20:56.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:20:56.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:56 smithi138 ceph-mon[185886]: pgmap v514: 89 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 17 active+clean+remapped, 5 remapped+peering, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 44 active+clean; 5.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 30 MiB/s wr, 474 op/s; 45197/272613 objects degraded (16.579%); 17243/272613 objects misplaced (6.325%); 145 KiB/s, 10 objects/s recovering 2023-10-01T09:20:56.649 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:20:56.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:56.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:20:56.758 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:20:56.782 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:56.784 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:20:56.784 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:20:56.796 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:56.882 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:20:56.993 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 73 16 93076 93060 79.6321 34.75 0.0382151 0.0125527 2023-10-01T09:20:57.005 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:20:57.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:57.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:20:57.109 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:20:57.243 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:57.360 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:20:57.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:57.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:20:57.380 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:20:57.421 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:20:57.453 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:20:57.473 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:57.492 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:20:57.511 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:20:57.555 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:20:57.567 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:57 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 45197/272613 objects degraded (16.579%), 12 pgs degraded (PG_DEGRADED) 2023-10-01T09:20:57.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:57 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 45197/272613 objects degraded (16.579%), 12 pgs degraded (PG_DEGRADED) 2023-10-01T09:20:57.580 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:57.618 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:20:57.650 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:20:57.651 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:20:57.651 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:20:57.651 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:20:57.651 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:20:57.651 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:20:57.651 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-01T09:20:57.651 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:20:57.652 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:20:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:57 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 45197/272613 objects degraded (16.579%), 12 pgs degraded (PG_DEGRADED) 2023-10-01T09:20:57.922 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:20:57.995 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 74 15 93524 93509 78.9337 28.0625 0.0164712 0.0126612 2023-10-01T09:20:58.460 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:58 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/598114646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:58.460 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:58 smithi190 ceph-mon[160435]: pgmap v515: 89 pgs: 19 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 45 active+clean; 5.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 40 MiB/s wr, 640 op/s; 56629/277185 objects degraded (20.430%); 34556/277185 objects misplaced (12.467%); 1.2 MiB/s, 26 objects/s recovering 2023-10-01T09:20:58.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:58 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/598114646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:58.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:58 smithi138 ceph-mon[192513]: pgmap v515: 89 pgs: 19 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 45 active+clean; 5.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 40 MiB/s wr, 640 op/s; 56629/277185 objects degraded (20.430%); 34556/277185 objects misplaced (12.467%); 1.2 MiB/s, 26 objects/s recovering 2023-10-01T09:20:58.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:58 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/598114646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:20:58.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:58 smithi138 ceph-mon[185886]: pgmap v515: 89 pgs: 19 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 45 active+clean; 5.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 40 MiB/s wr, 640 op/s; 56629/277185 objects degraded (20.430%); 34556/277185 objects misplaced (12.467%); 1.2 MiB/s, 26 objects/s recovering 2023-10-01T09:20:58.920 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:58.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:58.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:58.995 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 75 16 94056 94040 78.324 33.1875 0.0318907 0.0127599 2023-10-01T09:20:59.028 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:20:59.100 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:59.484 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:20:59.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:20:59.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:20:59.513 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:20:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:20:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:20:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:20:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:59037 5.2 5:47874101:::benchmark_data_smithi138_240338_object59036:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:20:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:15:16.277259+0000", 2023-10-01T09:20:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 343.23245236899999, 2023-10-01T09:20:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22143442799999999, 2023-10-01T09:20:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:20:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:20:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 59037 2023-10-01T09:20:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277259+0000", 2023-10-01T09:20:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277259+0000", 2023-10-01T09:20:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:20:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277258+0000", 2023-10-01T09:20:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012673599999999999 2023-10-01T09:20:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277384+0000", 2023-10-01T09:20:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5079000000000002e-05 2023-10-01T09:20:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277409+0000", 2023-10-01T09:20:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3341e-05 2023-10-01T09:20:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277423+0000", 2023-10-01T09:20:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5586999999999994e-05 2023-10-01T09:20:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277488+0000", 2023-10-01T09:20:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015682599999999999 2023-10-01T09:20:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277645+0000", 2023-10-01T09:20:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014917700000000001 2023-10-01T09:20:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:20:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277794+0000", 2023-10-01T09:20:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0064553759999999996 2023-10-01T09:20:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.284250+0000", 2023-10-01T09:20:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002231631 2023-10-01T09:20:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.286481+0000", 2023-10-01T09:20:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21211115999999999 2023-10-01T09:20:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.498592+0000", 2023-10-01T09:20:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7336999999999997e-05 2023-10-01T09:20:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.498650+0000", 2023-10-01T09:20:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3631000000000003e-05 2023-10-01T09:20:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.498693+0000", 2023-10-01T09:20:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22143442799999999 2023-10-01T09:20:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:59040 5.2 5:4f019d16:::benchmark_data_smithi138_240338_object59039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:20:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:15:16.277463+0000", 2023-10-01T09:20:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 343.23224847699998, 2023-10-01T09:20:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22673643700000001, 2023-10-01T09:20:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:20:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:20:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 59040 2023-10-01T09:20:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277463+0000", 2023-10-01T09:20:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277463+0000", 2023-10-01T09:20:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999967 2023-10-01T09:20:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277460+0000", 2023-10-01T09:20:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6584000000000002e-05 2023-10-01T09:20:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277486+0000", 2023-10-01T09:20:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5369999999999997e-06 2023-10-01T09:20:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277495+0000", 2023-10-01T09:20:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6690000000000003e-06 2023-10-01T09:20:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.277500+0000", 2023-10-01T09:20:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0047659249999999998 2023-10-01T09:20:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.282266+0000", 2023-10-01T09:20:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015794600000000001 2023-10-01T09:20:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.282424+0000", 2023-10-01T09:20:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012711499999999999 2023-10-01T09:20:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:20:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.282551+0000", 2023-10-01T09:20:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001851281 2023-10-01T09:20:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.284403+0000", 2023-10-01T09:20:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0021141440000000001 2023-10-01T09:20:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.286517+0000", 2023-10-01T09:20:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21758307199999999 2023-10-01T09:20:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.504100+0000", 2023-10-01T09:20:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6016e-05 2023-10-01T09:20:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.504156+0000", 2023-10-01T09:20:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3319000000000003e-05 2023-10-01T09:20:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.504199+0000", 2023-10-01T09:20:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22673643700000001 2023-10-01T09:20:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:59042 5.2 5:47d501e1:::benchmark_data_smithi138_240338_object59041:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e115)", 2023-10-01T09:20:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:15:16.284761+0000", 2023-10-01T09:20:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 343.22495014100002, 2023-10-01T09:20:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22018362699999999, 2023-10-01T09:20:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:20:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:20:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 59042 2023-10-01T09:20:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.284761+0000", 2023-10-01T09:20:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.284761+0000", 2023-10-01T09:20:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999886 2023-10-01T09:20:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.284750+0000", 2023-10-01T09:20:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1539000000000002e-05 2023-10-01T09:20:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.284801+0000", 2023-10-01T09:20:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.435e-05 2023-10-01T09:20:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.284815+0000", 2023-10-01T09:20:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1080000000000004e-06 2023-10-01T09:20:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.284823+0000", 2023-10-01T09:20:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3553e-05 2023-10-01T09:20:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.284876+0000", 2023-10-01T09:20:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9931000000000003e-05 2023-10-01T09:20:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.284976+0000", 2023-10-01T09:20:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000151686 2023-10-01T09:20:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:20:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.285128+0000", 2023-10-01T09:20:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0014280580000000001 2023-10-01T09:20:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.286556+0000", 2023-10-01T09:20:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6716000000000003e-05 2023-10-01T09:20:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.286623+0000", 2023-10-01T09:20:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.218192681 2023-10-01T09:20:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.504815+0000", 2023-10-01T09:20:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8267999999999998e-05 2023-10-01T09:20:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.504873+0000", 2023-10-01T09:20:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1359000000000006e-05 2023-10-01T09:20:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:15:16.504945+0000", 2023-10-01T09:20:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22018362699999999 2023-10-01T09:20:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:110053 5.2 5:47f72d69:::benchmark_data_smithi138_240338_object110052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-01T09:20:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:08.050293+0000", 2023-10-01T09:20:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 291.45941820299998, 2023-10-01T09:20:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22519323799999999, 2023-10-01T09:20:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:20:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:20:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 110053 2023-10-01T09:20:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050293+0000", 2023-10-01T09:20:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050293+0000", 2023-10-01T09:20:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:20:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050292+0000", 2023-10-01T09:20:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.537e-05 2023-10-01T09:20:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050347+0000", 2023-10-01T09:20:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5852999999999999e-05 2023-10-01T09:20:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050363+0000", 2023-10-01T09:20:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5590000000000002e-06 2023-10-01T09:20:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050370+0000", 2023-10-01T09:20:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4067999999999998e-05 2023-10-01T09:20:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050424+0000", 2023-10-01T09:20:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014595999999999999 2023-10-01T09:20:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050570+0000", 2023-10-01T09:20:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00019942700000000001 2023-10-01T09:20:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:20:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050769+0000", 2023-10-01T09:20:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0066986199999999997 2023-10-01T09:20:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.057468+0000", 2023-10-01T09:20:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3139999999999997e-06 2023-10-01T09:20:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.057472+0000", 2023-10-01T09:20:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21792381899999999 2023-10-01T09:20:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275396+0000", 2023-10-01T09:20:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2527e-05 2023-10-01T09:20:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275448+0000", 2023-10-01T09:20:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8033e-05 2023-10-01T09:20:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275486+0000", 2023-10-01T09:20:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22519323799999999 2023-10-01T09:20:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:110061 5.2 5:488c8e9e:::benchmark_data_smithi138_240338_object110060:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-01T09:20:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:08.054004+0000", 2023-10-01T09:20:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 291.45570730399999, 2023-10-01T09:20:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22790993500000001, 2023-10-01T09:20:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:20:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:20:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 110061 2023-10-01T09:20:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.054004+0000", 2023-10-01T09:20:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.054004+0000", 2023-10-01T09:20:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999971 2023-10-01T09:20:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.054001+0000", 2023-10-01T09:20:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7870000000000001e-05 2023-10-01T09:20:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.054049+0000", 2023-10-01T09:20:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4868000000000001e-05 2023-10-01T09:20:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.054064+0000", 2023-10-01T09:20:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9039999999999992e-06 2023-10-01T09:20:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.054072+0000", 2023-10-01T09:20:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.1599000000000002e-05 2023-10-01T09:20:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.054164+0000", 2023-10-01T09:20:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000124824 2023-10-01T09:20:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.054288+0000", 2023-10-01T09:20:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015939599999999999 2023-10-01T09:20:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:20:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.054448+0000", 2023-10-01T09:20:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0040427329999999997 2023-10-01T09:20:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.058491+0000", 2023-10-01T09:20:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000234348 2023-10-01T09:20:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.058725+0000", 2023-10-01T09:20:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.223098828 2023-10-01T09:20:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.281824+0000", 2023-10-01T09:20:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1585000000000002e-05 2023-10-01T09:20:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.281875+0000", 2023-10-01T09:20:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8689000000000002e-05 2023-10-01T09:20:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.281914+0000", 2023-10-01T09:20:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22790993500000001 2023-10-01T09:20:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:119707 5.2 5:40c34076:::benchmark_data_smithi138_240338_object4570:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-01T09:20:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:20.269607+0000", 2023-10-01T09:20:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 279.24010411099999, 2023-10-01T09:20:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24083607100000001, 2023-10-01T09:20:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:20:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:20:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 119707 2023-10-01T09:20:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269607+0000", 2023-10-01T09:20:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269607+0000", 2023-10-01T09:20:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:20:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269605+0000", 2023-10-01T09:20:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5280000000000003e-06 2023-10-01T09:20:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269613+0000", 2023-10-01T09:20:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8167000000000001e-05 2023-10-01T09:20:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269632+0000", 2023-10-01T09:20:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2099999999999993e-06 2023-10-01T09:20:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269640+0000", 2023-10-01T09:20:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0532999999999999e-05 2023-10-01T09:20:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269690+0000", 2023-10-01T09:20:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015282699999999999 2023-10-01T09:20:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269843+0000", 2023-10-01T09:20:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014871 2023-10-01T09:20:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:20:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269992+0000", 2023-10-01T09:20:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0048685279999999996 2023-10-01T09:20:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.274860+0000", 2023-10-01T09:20:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0039101919999999998 2023-10-01T09:20:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.278771+0000", 2023-10-01T09:20:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.23156614 2023-10-01T09:20:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.510337+0000", 2023-10-01T09:20:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0591999999999999e-05 2023-10-01T09:20:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.510397+0000", 2023-10-01T09:20:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6020000000000003e-05 2023-10-01T09:20:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.510443+0000", 2023-10-01T09:20:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24083607100000001 2023-10-01T09:20:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:119717 5.2 5:4e9905f5:::benchmark_data_smithi138_240338_object4580:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-01T09:20:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:20.287301+0000", 2023-10-01T09:20:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 279.22240988200002, 2023-10-01T09:20:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.226888276, 2023-10-01T09:20:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:20:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:20:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 119717 2023-10-01T09:20:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.287301+0000", 2023-10-01T09:20:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.287301+0000", 2023-10-01T09:20:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:20:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.287300+0000", 2023-10-01T09:20:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6100000000000002e-06 2023-10-01T09:20:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.287307+0000", 2023-10-01T09:20:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0393e-05 2023-10-01T09:20:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.287317+0000", 2023-10-01T09:20:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.857e-06 2023-10-01T09:20:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.287323+0000", 2023-10-01T09:20:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6471999999999997e-05 2023-10-01T09:20:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.287379+0000", 2023-10-01T09:20:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013086700000000001 2023-10-01T09:20:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.287510+0000", 2023-10-01T09:20:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014142700000000001 2023-10-01T09:20:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:20:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.287652+0000", 2023-10-01T09:20:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038524150000000001 2023-10-01T09:20:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.291504+0000", 2023-10-01T09:20:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002770706 2023-10-01T09:20:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.294275+0000", 2023-10-01T09:20:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21981107799999999 2023-10-01T09:20:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.514086+0000", 2023-10-01T09:20:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4700000000000001e-05 2023-10-01T09:20:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.514141+0000", 2023-10-01T09:20:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9156000000000002e-05 2023-10-01T09:20:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.514190+0000", 2023-10-01T09:20:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.226888276 2023-10-01T09:20:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:119722 5.2 5:487f1f0b:::benchmark_data_smithi138_240338_object4585:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-01T09:20:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:20.297210+0000", 2023-10-01T09:20:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 279.21250176199999, 2023-10-01T09:20:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22126501400000001, 2023-10-01T09:20:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:20:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:20:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 119722 2023-10-01T09:20:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.297210+0000", 2023-10-01T09:20:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.297210+0000", 2023-10-01T09:20:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:20:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.297208+0000", 2023-10-01T09:20:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7549999999999994e-06 2023-10-01T09:20:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.297216+0000", 2023-10-01T09:20:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0883e-05 2023-10-01T09:20:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.297236+0000", 2023-10-01T09:20:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8050000000000007e-06 2023-10-01T09:20:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.297244+0000", 2023-10-01T09:20:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1872999999999994e-05 2023-10-01T09:20:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.297316+0000", 2023-10-01T09:20:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014248700000000001 2023-10-01T09:20:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.297459+0000", 2023-10-01T09:20:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013415000000000001 2023-10-01T09:20:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:20:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.297593+0000", 2023-10-01T09:20:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0037878809999999999 2023-10-01T09:20:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.301381+0000", 2023-10-01T09:20:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0042085450000000002 2023-10-01T09:20:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.305589+0000", 2023-10-01T09:20:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21280226199999999 2023-10-01T09:20:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.518391+0000", 2023-10-01T09:20:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6310000000000002e-05 2023-10-01T09:20:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.518438+0000", 2023-10-01T09:20:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6869000000000002e-05 2023-10-01T09:20:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.518475+0000", 2023-10-01T09:20:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22126501400000001 2023-10-01T09:20:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:145104 5.2 5:4ce35393:::benchmark_data_smithi138_240338_object29967:head [delete] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-10-01T09:20:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:56.154193+0000", 2023-10-01T09:20:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 243.35551804100001, 2023-10-01T09:20:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22674106899999999, 2023-10-01T09:20:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:20:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:20:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 145104 2023-10-01T09:20:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.154193+0000", 2023-10-01T09:20:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.154193+0000", 2023-10-01T09:20:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:20:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.154192+0000", 2023-10-01T09:20:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8780000000000003e-06 2023-10-01T09:20:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.154198+0000", 2023-10-01T09:20:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1089e-05 2023-10-01T09:20:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.154209+0000", 2023-10-01T09:20:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9580000000000004e-06 2023-10-01T09:20:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.154215+0000", 2023-10-01T09:20:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.858e-05 2023-10-01T09:20:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.154253+0000", 2023-10-01T09:20:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000125277 2023-10-01T09:20:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.154379+0000", 2023-10-01T09:20:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000129765 2023-10-01T09:20:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:20:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.154508+0000", 2023-10-01T09:20:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0059085550000000002 2023-10-01T09:20:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.160417+0000", 2023-10-01T09:20:59.580 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035164020000000001 2023-10-01T09:20:59.580 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.580 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.580 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.580 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.163933+0000", 2023-10-01T09:20:59.580 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21691553599999999 2023-10-01T09:20:59.580 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.580 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.580 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.580 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.380849+0000", 2023-10-01T09:20:59.580 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5568999999999997e-05 2023-10-01T09:20:59.580 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.380894+0000", 2023-10-01T09:20:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0049000000000002e-05 2023-10-01T09:20:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:56.380934+0000", 2023-10-01T09:20:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22674106899999999 2023-10-01T09:20:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:71130 6.c 6:3bc24bb8:::benchmark_data_smithi138_307751_object71129:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-10-01T09:20:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:20:29.290240+0000", 2023-10-01T09:20:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 30.219471084999999, 2023-10-01T09:20:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.198034872, 2023-10-01T09:20:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:20:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 71130 2023-10-01T09:20:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:29.290240+0000", 2023-10-01T09:20:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:29.290240+0000", 2023-10-01T09:20:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:29.290240+0000", 2023-10-01T09:20:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2951999999999999e-05 2023-10-01T09:20:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:29.290273+0000", 2023-10-01T09:20:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8969999999999999e-06 2023-10-01T09:20:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:29.290278+0000", 2023-10-01T09:20:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.422e-06 2023-10-01T09:20:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:29.290281+0000", 2023-10-01T09:20:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019241690000000001 2023-10-01T09:20:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:29.292205+0000", 2023-10-01T09:20:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011449499999999999 2023-10-01T09:20:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:29.292320+0000", 2023-10-01T09:20:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000117942 2023-10-01T09:20:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:20:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:29.292438+0000", 2023-10-01T09:20:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003097533 2023-10-01T09:20:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:29.295535+0000", 2023-10-01T09:20:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00096687699999999999 2023-10-01T09:20:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:29.296502+0000", 2023-10-01T09:20:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.191692046 2023-10-01T09:20:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:29.488194+0000", 2023-10-01T09:20:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6474000000000003e-05 2023-10-01T09:20:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:29.488241+0000", 2023-10-01T09:20:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4517999999999997e-05 2023-10-01T09:20:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:29.488275+0000", 2023-10-01T09:20:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.198034872 2023-10-01T09:20:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:86678 6.a 6:57528fa7:::benchmark_data_smithi138_307751_object86677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-01T09:20:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:20:45.408918+0000", 2023-10-01T09:20:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 14.100793213999999, 2023-10-01T09:20:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22536179000000001, 2023-10-01T09:20:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:20:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 86678 2023-10-01T09:20:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.408918+0000", 2023-10-01T09:20:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.408918+0000", 2023-10-01T09:20:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:20:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.408917+0000", 2023-10-01T09:20:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6956e-05 2023-10-01T09:20:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.408994+0000", 2023-10-01T09:20:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.102e-05 2023-10-01T09:20:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.409005+0000", 2023-10-01T09:20:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.229e-06 2023-10-01T09:20:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.409011+0000", 2023-10-01T09:20:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7166999999999998e-05 2023-10-01T09:20:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.409058+0000", 2023-10-01T09:20:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014359900000000001 2023-10-01T09:20:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.409202+0000", 2023-10-01T09:20:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016588400000000001 2023-10-01T09:20:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0", 2023-10-01T09:20:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.409368+0000", 2023-10-01T09:20:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0077234089999999997 2023-10-01T09:20:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.417091+0000", 2023-10-01T09:20:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21704465100000001 2023-10-01T09:20:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.634136+0000", 2023-10-01T09:20:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0538999999999998e-05 2023-10-01T09:20:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.634186+0000", 2023-10-01T09:20:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3461999999999994e-05 2023-10-01T09:20:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.634280+0000", 2023-10-01T09:20:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22536179000000001 2023-10-01T09:20:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:86685 6.a 6:5985074a:::benchmark_data_smithi138_307751_object86684:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-01T09:20:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:20:45.409130+0000", 2023-10-01T09:20:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 14.10058175, 2023-10-01T09:20:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22532654899999999, 2023-10-01T09:20:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:20:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 86685 2023-10-01T09:20:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.409130+0000", 2023-10-01T09:20:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.409130+0000", 2023-10-01T09:20:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.409129+0000", 2023-10-01T09:20:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2400000000000001e-05 2023-10-01T09:20:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.409161+0000", 2023-10-01T09:20:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3329999999999999e-06 2023-10-01T09:20:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.409169+0000", 2023-10-01T09:20:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6589999999999998e-06 2023-10-01T09:20:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.409173+0000", 2023-10-01T09:20:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0025636330000000001 2023-10-01T09:20:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.411737+0000", 2023-10-01T09:20:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000116113 2023-10-01T09:20:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.411853+0000", 2023-10-01T09:20:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000103484 2023-10-01T09:20:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0", 2023-10-01T09:20:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.411957+0000", 2023-10-01T09:20:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0051422309999999997 2023-10-01T09:20:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.417099+0000", 2023-10-01T09:20:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21722192300000001 2023-10-01T09:20:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.634321+0000", 2023-10-01T09:20:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5461999999999999e-05 2023-10-01T09:20:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.634376+0000", 2023-10-01T09:20:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0005000000000004e-05 2023-10-01T09:20:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.634456+0000", 2023-10-01T09:20:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22532654899999999 2023-10-01T09:20:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:86686 6.a 6:59b3304e:::benchmark_data_smithi138_307751_object86685:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-01T09:20:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:20:45.413054+0000", 2023-10-01T09:20:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 14.096656914, 2023-10-01T09:20:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22168384399999999, 2023-10-01T09:20:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:20:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 86686 2023-10-01T09:20:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.413054+0000", 2023-10-01T09:20:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.413054+0000", 2023-10-01T09:20:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:20:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.413053+0000", 2023-10-01T09:20:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7117999999999998e-05 2023-10-01T09:20:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.413110+0000", 2023-10-01T09:20:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1085e-05 2023-10-01T09:20:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.413121+0000", 2023-10-01T09:20:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1029999999999998e-06 2023-10-01T09:20:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.413127+0000", 2023-10-01T09:20:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001303532 2023-10-01T09:20:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.414431+0000", 2023-10-01T09:20:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012985000000000001 2023-10-01T09:20:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.414561+0000", 2023-10-01T09:20:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016429300000000001 2023-10-01T09:20:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0", 2023-10-01T09:20:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.414725+0000", 2023-10-01T09:20:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0052630619999999998 2023-10-01T09:20:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.419988+0000", 2023-10-01T09:20:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21453828799999999 2023-10-01T09:20:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.634526+0000", 2023-10-01T09:20:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7435999999999997e-05 2023-10-01T09:20:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.634584+0000", 2023-10-01T09:20:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015463299999999999 2023-10-01T09:20:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.634738+0000", 2023-10-01T09:20:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22168384399999999 2023-10-01T09:20:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:86688 6.a 6:5395164c:::benchmark_data_smithi138_307751_object86687:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-01T09:20:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:20:45.416368+0000", 2023-10-01T09:20:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 14.093343244, 2023-10-01T09:20:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21856147000000001, 2023-10-01T09:20:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:20:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 86688 2023-10-01T09:20:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.416368+0000", 2023-10-01T09:20:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.416368+0000", 2023-10-01T09:20:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:20:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.416367+0000", 2023-10-01T09:20:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3864999999999999e-05 2023-10-01T09:20:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.416391+0000", 2023-10-01T09:20:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0519e-05 2023-10-01T09:20:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.416401+0000", 2023-10-01T09:20:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0309999999999998e-06 2023-10-01T09:20:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.416406+0000", 2023-10-01T09:20:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00077395100000000004 2023-10-01T09:20:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.417180+0000", 2023-10-01T09:20:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012720599999999999 2023-10-01T09:20:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.417307+0000", 2023-10-01T09:20:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015305200000000001 2023-10-01T09:20:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0", 2023-10-01T09:20:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.417460+0000", 2023-10-01T09:20:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002635802 2023-10-01T09:20:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.420096+0000", 2023-10-01T09:20:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21469068299999999 2023-10-01T09:20:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.634787+0000", 2023-10-01T09:20:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1700000000000003e-05 2023-10-01T09:20:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.634839+0000", 2023-10-01T09:20:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0944000000000002e-05 2023-10-01T09:20:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.634930+0000", 2023-10-01T09:20:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21856147000000001 2023-10-01T09:20:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:86696 6.a 6:5d9e2b09:::benchmark_data_smithi138_307751_object86695:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-10-01T09:20:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:20:45.422486+0000", 2023-10-01T09:20:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 14.08722566, 2023-10-01T09:20:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21262656099999999, 2023-10-01T09:20:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:20:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:20:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:20:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 86696 2023-10-01T09:20:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.422486+0000", 2023-10-01T09:20:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.422486+0000", 2023-10-01T09:20:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:20:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.422484+0000", 2023-10-01T09:20:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7549000000000003e-05 2023-10-01T09:20:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.422522+0000", 2023-10-01T09:20:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1046999999999999e-05 2023-10-01T09:20:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.422533+0000", 2023-10-01T09:20:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.767e-06 2023-10-01T09:20:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.422538+0000", 2023-10-01T09:20:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7115999999999998e-05 2023-10-01T09:20:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.422615+0000", 2023-10-01T09:20:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000139555 2023-10-01T09:20:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.422755+0000", 2023-10-01T09:20:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000152545 2023-10-01T09:20:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0", 2023-10-01T09:20:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.422908+0000", 2023-10-01T09:20:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21088058200000001 2023-10-01T09:20:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:20:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.633788+0000", 2023-10-01T09:20:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0011814 2023-10-01T09:20:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:20:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.634970+0000", 2023-10-01T09:20:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6804000000000001e-05 2023-10-01T09:20:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.635016+0000", 2023-10-01T09:20:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5890000000000005e-05 2023-10-01T09:20:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.635112+0000", 2023-10-01T09:20:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21262656099999999 2023-10-01T09:20:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:86690 6.2 e200/194 6:415dac7f:::benchmark_data_smithi138_307751_object86689:head v 200'5552, mlcod=200'5550)", 2023-10-01T09:20:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:20:45.425076+0000", 2023-10-01T09:20:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 14.084635551, 2023-10-01T09:20:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.208749088, 2023-10-01T09:20:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425076+0000", 2023-10-01T09:20:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425076+0000", 2023-10-01T09:20:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:20:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425075+0000", 2023-10-01T09:20:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7610000000000002e-05 2023-10-01T09:20:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425103+0000", 2023-10-01T09:20:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1009000000000001e-05 2023-10-01T09:20:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425114+0000", 2023-10-01T09:20:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6789999999999996e-06 2023-10-01T09:20:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425118+0000", 2023-10-01T09:20:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9377999999999999e-05 2023-10-01T09:20:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425148+0000", 2023-10-01T09:20:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3859999999999999e-05 2023-10-01T09:20:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425161+0000", 2023-10-01T09:20:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20862914499999999 2023-10-01T09:20:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.633791+0000", 2023-10-01T09:20:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4258999999999998e-05 2023-10-01T09:20:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.633825+0000", 2023-10-01T09:20:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.208749088 2023-10-01T09:20:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:86693 6.1 e200/179 6:84c65574:::benchmark_data_smithi138_307751_object86692:head v 200'5355, mlcod=200'5353)", 2023-10-01T09:20:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:20:45.425370+0000", 2023-10-01T09:20:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 14.084341831, 2023-10-01T09:20:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21331361700000001, 2023-10-01T09:20:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425370+0000", 2023-10-01T09:20:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425370+0000", 2023-10-01T09:20:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:20:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425369+0000", 2023-10-01T09:20:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7930000000000003e-05 2023-10-01T09:20:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425426+0000", 2023-10-01T09:20:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1338999999999999e-05 2023-10-01T09:20:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425438+0000", 2023-10-01T09:20:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6650000000000002e-06 2023-10-01T09:20:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425442+0000", 2023-10-01T09:20:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8197e-05 2023-10-01T09:20:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425481+0000", 2023-10-01T09:20:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3039e-05 2023-10-01T09:20:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425494+0000", 2023-10-01T09:20:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21314983300000001 2023-10-01T09:20:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.638644+0000", 2023-10-01T09:20:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9586999999999999e-05 2023-10-01T09:20:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.638683+0000", 2023-10-01T09:20:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21331361700000001 2023-10-01T09:20:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:86691 6.4 e200/198 6:207cdbc1:::benchmark_data_smithi138_307751_object86690:head v 200'5367, mlcod=200'5365)", 2023-10-01T09:20:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:20:45.425455+0000", 2023-10-01T09:20:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 14.084255945000001, 2023-10-01T09:20:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21908409200000001, 2023-10-01T09:20:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425455+0000", 2023-10-01T09:20:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425455+0000", 2023-10-01T09:20:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425455+0000", 2023-10-01T09:20:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8121999999999999e-05 2023-10-01T09:20:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425483+0000", 2023-10-01T09:20:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5719999999999996e-06 2023-10-01T09:20:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425488+0000", 2023-10-01T09:20:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8550000000000002e-06 2023-10-01T09:20:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425491+0000", 2023-10-01T09:20:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.208274922 2023-10-01T09:20:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.633766+0000", 2023-10-01T09:20:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8894999999999999e-05 2023-10-01T09:20:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.633804+0000", 2023-10-01T09:20:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010691865 2023-10-01T09:20:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.644496+0000", 2023-10-01T09:20:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3182000000000001e-05 2023-10-01T09:20:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.644539+0000", 2023-10-01T09:20:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21908409200000001 2023-10-01T09:20:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:86695 6.1 e200/179 6:8fce8392:::benchmark_data_smithi138_307751_object86694:head v 200'5356, mlcod=200'5353)", 2023-10-01T09:20:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:20:45.425757+0000", 2023-10-01T09:20:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 14.083954021, 2023-10-01T09:20:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.213098702, 2023-10-01T09:20:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425757+0000", 2023-10-01T09:20:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425757+0000", 2023-10-01T09:20:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425757+0000", 2023-10-01T09:20:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3737999999999999e-05 2023-10-01T09:20:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.425811+0000", 2023-10-01T09:20:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20793977999999999 2023-10-01T09:20:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.633750+0000", 2023-10-01T09:20:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0471e-05 2023-10-01T09:20:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.633761+0000", 2023-10-01T09:20:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024589540000000002 2023-10-01T09:20:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.636220+0000", 2023-10-01T09:20:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1455999999999999e-05 2023-10-01T09:20:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.636241+0000", 2023-10-01T09:20:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0025846810000000001 2023-10-01T09:20:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.638826+0000", 2023-10-01T09:20:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0119000000000001e-05 2023-10-01T09:20:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.638856+0000", 2023-10-01T09:20:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.213098702 2023-10-01T09:20:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:86701 6.b e200/183 6:dd42a402:::benchmark_data_smithi138_307751_object86700:head v 200'5436, mlcod=200'5434)", 2023-10-01T09:20:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:20:45.429647+0000", 2023-10-01T09:20:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 14.080064713000001, 2023-10-01T09:20:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20828039000000001, 2023-10-01T09:20:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:20:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:20:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:20:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:20:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.429647+0000", 2023-10-01T09:20:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:20:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:20:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.429647+0000", 2023-10-01T09:20:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:20:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:20:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.429646+0000", 2023-10-01T09:20:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2747999999999997e-05 2023-10-01T09:20:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:20:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.429699+0000", 2023-10-01T09:20:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20405680100000001 2023-10-01T09:20:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:20:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.633755+0000", 2023-10-01T09:20:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0373999999999999e-05 2023-10-01T09:20:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:20:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.633766+0000", 2023-10-01T09:20:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9289999999999997e-05 2023-10-01T09:20:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:20:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.633815+0000", 2023-10-01T09:20:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0103999999999999e-05 2023-10-01T09:20:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:20:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.633835+0000", 2023-10-01T09:20:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.004049609 2023-10-01T09:20:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:20:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.637885+0000", 2023-10-01T09:20:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2175000000000002e-05 2023-10-01T09:20:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:20:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:20:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:20:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:20:45.637927+0000", 2023-10-01T09:20:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20828039000000001 2023-10-01T09:20:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:20:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:20:59.650 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:20:59.650 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:20:59.651 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:20:59.651 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:20:59.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:20:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:20:59.652 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:20:59.780 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:20:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:20:59.781 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:20:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:20:59.793 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:20:59.996 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 76 16 94525 94509 77.6789 29.3125 0.0300191 0.0128667 2023-10-01T09:21:00.124 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:00.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:00.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:00.155 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:21:00.244 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:00.297 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:00.370 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:00.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:00.380 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:00 smithi190 ceph-mon[160435]: pgmap v516: 89 pgs: 19 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 45 active+clean; 5.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 30 MiB/s wr, 474 op/s; 56786/279012 objects degraded (20.353%); 34708/279012 objects misplaced (12.440%); 958 KiB/s, 19 objects/s recovering 2023-10-01T09:21:00.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:00.482 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:00.573 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:00.676 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:00.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:00.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:00.699 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:00 smithi138 ceph-mon[192513]: pgmap v516: 89 pgs: 19 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 45 active+clean; 5.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 30 MiB/s wr, 474 op/s; 56786/279012 objects degraded (20.353%); 34708/279012 objects misplaced (12.440%); 958 KiB/s, 19 objects/s recovering 2023-10-01T09:21:00.703 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:00 smithi138 ceph-mon[185886]: pgmap v516: 89 pgs: 19 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 45 active+clean; 5.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 30 MiB/s wr, 474 op/s; 56786/279012 objects degraded (20.353%); 34708/279012 objects misplaced (12.440%); 958 KiB/s, 19 objects/s recovering 2023-10-01T09:21:00.703 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:00 smithi138 conmon[185862]: 2023-10-01T09:21:00.606+0000 7f70d64ec700 -1 mon.a@0(leader).osd e206 definitely_dead 0 2023-10-01T09:21:00.799 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:00.845 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:21:00.880 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:00.996 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 77 16 95037 95021 77.0856 32 0.0203936 0.0129666 2023-10-01T09:21:01.004 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:01.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:01.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:01.110 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:01.150 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:01.236 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:01.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:01.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:01.340 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:01.414 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3181519540' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T09:21:01.414 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:01 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T09:21:01.446 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:01.450 INFO:teuthology.orchestra.run.smithi138.stderr:marked in osd.6. 2023-10-01T09:21:01.510 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:01.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:01.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:01.614 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:01.643 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:01.653 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:21:01.679 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3181519540' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T09:21:01.679 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:01 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T09:21:01.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3181519540' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T09:21:01.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:01 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T09:21:01.996 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 78 16 95283 95267 76.2948 15.375 0.0155868 0.0130051 2023-10-01T09:21:02.278 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:02.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:02.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:02.350 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-01T09:21:02.390 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:02.427 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:02.540 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T09:21:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[192513]: osdmap e207: 8 total, 7 up, 8 in 2023-10-01T09:21:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[192513]: failed to encode map e207 with expected crc 2023-10-01T09:21:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[192513]: failed to encode map e207 with expected crc 2023-10-01T09:21:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[192513]: pgmap v518: 89 pgs: 19 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 45 active+clean; 5.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 23 MiB/s wr, 373 op/s; 56786/280002 objects degraded (20.281%); 34872/280002 objects misplaced (12.454%); 841 KiB/s, 17 objects/s recovering 2023-10-01T09:21:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 56786/279012 objects degraded (20.353%), 14 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[192513]: osdmap e208: 8 total, 7 up, 8 in 2023-10-01T09:21:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[192513]: failed to encode map e208 with expected crc 2023-10-01T09:21:02.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[192513]: failed to encode map e208 with expected crc 2023-10-01T09:21:02.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T09:21:02.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[185886]: osdmap e207: 8 total, 7 up, 8 in 2023-10-01T09:21:02.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[185886]: failed to encode map e207 with expected crc 2023-10-01T09:21:02.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[185886]: failed to encode map e207 with expected crc 2023-10-01T09:21:02.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[185886]: pgmap v518: 89 pgs: 19 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 45 active+clean; 5.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 23 MiB/s wr, 373 op/s; 56786/280002 objects degraded (20.281%); 34872/280002 objects misplaced (12.454%); 841 KiB/s, 17 objects/s recovering 2023-10-01T09:21:02.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 56786/279012 objects degraded (20.353%), 14 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:02.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[185886]: osdmap e208: 8 total, 7 up, 8 in 2023-10-01T09:21:02.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[185886]: failed to encode map e208 with expected crc 2023-10-01T09:21:02.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:02 smithi138 ceph-mon[185886]: failed to encode map e208 with expected crc 2023-10-01T09:21:02.585 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:02.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:02.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:02.615 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:02 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T09:21:02.615 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:02 smithi190 ceph-mon[160435]: osdmap e207: 8 total, 7 up, 8 in 2023-10-01T09:21:02.615 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:02 smithi190 ceph-mon[160435]: failed to encode map e207 with expected crc 2023-10-01T09:21:02.615 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:02 smithi190 ceph-mon[160435]: failed to encode map e207 with expected crc 2023-10-01T09:21:02.615 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:02 smithi190 ceph-mon[160435]: pgmap v518: 89 pgs: 19 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 45 active+clean; 5.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 23 MiB/s wr, 373 op/s; 56786/280002 objects degraded (20.281%); 34872/280002 objects misplaced (12.454%); 841 KiB/s, 17 objects/s recovering 2023-10-01T09:21:02.615 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:02 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 56786/279012 objects degraded (20.353%), 14 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:02.615 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:02 smithi190 ceph-mon[160435]: osdmap e208: 8 total, 7 up, 8 in 2023-10-01T09:21:02.615 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:02 smithi190 ceph-mon[160435]: failed to encode map e208 with expected crc 2023-10-01T09:21:02.616 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:02 smithi190 ceph-mon[160435]: failed to encode map e208 with expected crc 2023-10-01T09:21:02.689 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:02.721 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:02.815 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:02.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:02.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:02.825 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:21:02.826 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:21:02.826 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 0 2023-10-01T09:21:02.826 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:21:02.920 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:02.946 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:02.996 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 79 16 95300 95284 75.3429 1.0625 0.00443281 0.0130648 2023-10-01T09:21:03.132 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:21:03.593 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:03.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:03.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:03.697 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:03.723 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[192513]: failed to encode map e208 with expected crc 2023-10-01T09:21:03.723 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[192513]: failed to encode map e208 with expected crc 2023-10-01T09:21:03.723 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[192513]: failed to encode map e208 with expected crc 2023-10-01T09:21:03.723 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3927463339' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:03.723 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:21:03.723 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3927463339' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:21:03.723 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[192513]: osdmap e209: 8 total, 7 up, 8 in 2023-10-01T09:21:03.723 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[192513]: failed to encode map e209 with expected crc 2023-10-01T09:21:03.724 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[185886]: failed to encode map e208 with expected crc 2023-10-01T09:21:03.724 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[185886]: failed to encode map e208 with expected crc 2023-10-01T09:21:03.724 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[185886]: failed to encode map e208 with expected crc 2023-10-01T09:21:03.724 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3927463339' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:03.724 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:21:03.724 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3927463339' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:21:03.724 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[185886]: osdmap e209: 8 total, 7 up, 8 in 2023-10-01T09:21:03.724 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:03 smithi138 ceph-mon[185886]: failed to encode map e209 with expected crc 2023-10-01T09:21:03.726 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:03.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:03 smithi190 ceph-mon[160435]: failed to encode map e208 with expected crc 2023-10-01T09:21:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:03 smithi190 ceph-mon[160435]: failed to encode map e208 with expected crc 2023-10-01T09:21:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:03 smithi190 ceph-mon[160435]: failed to encode map e208 with expected crc 2023-10-01T09:21:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:03 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3927463339' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:03 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:21:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:03 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3927463339' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:21:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:03 smithi190 ceph-mon[160435]: osdmap e209: 8 total, 7 up, 8 in 2023-10-01T09:21:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:03 smithi190 ceph-mon[160435]: failed to encode map e209 with expected crc 2023-10-01T09:21:03.869 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:21:03.996 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:21:03.996873+0000 min lat: 0.00099931 max lat: 2.48093 avg lat: 0.0132219 2023-10-01T09:21:03.997 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:21:03.997 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 80 16 95366 95350 74.4531 4.125 0.00314142 0.0132219 2023-10-01T09:21:04.473 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:04.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:04.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:04.489 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:21:04.577 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:04.607 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:04.660 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:04.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:04.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:04.680 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:04 smithi138 ceph-mon[192513]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.680 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:04 smithi138 ceph-mon[192513]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.680 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:04 smithi138 ceph-mon[192513]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.680 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:04 smithi138 ceph-mon[192513]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.680 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:04 smithi138 ceph-mon[192513]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.680 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:04 smithi138 ceph-mon[192513]: pgmap v521: 89 pgs: 1 activating+remapped, 6 peering, 10 remapped+peering, 13 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 38 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 28 MiB/s wr, 442 op/s; 29119/285153 objects degraded (10.212%); 12165/285153 objects misplaced (4.266%); 1024 KiB/s, 15 objects/s recovering 2023-10-01T09:21:04.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:04 smithi138 ceph-mon[185886]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:04 smithi138 ceph-mon[185886]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:04 smithi138 ceph-mon[185886]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:04 smithi138 ceph-mon[185886]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:04 smithi138 ceph-mon[185886]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.681 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:04 smithi138 ceph-mon[185886]: pgmap v521: 89 pgs: 1 activating+remapped, 6 peering, 10 remapped+peering, 13 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 38 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 28 MiB/s wr, 442 op/s; 29119/285153 objects degraded (10.212%); 12165/285153 objects misplaced (4.266%); 1024 KiB/s, 15 objects/s recovering 2023-10-01T09:21:04.765 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:04.790 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:04 smithi190 ceph-mon[160435]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.790 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:04 smithi190 ceph-mon[160435]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.791 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:04 smithi190 ceph-mon[160435]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.791 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:04 smithi190 ceph-mon[160435]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.791 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:04 smithi190 ceph-mon[160435]: failed to encode map e209 with expected crc 2023-10-01T09:21:04.791 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:04 smithi190 ceph-mon[160435]: pgmap v521: 89 pgs: 1 activating+remapped, 6 peering, 10 remapped+peering, 13 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 38 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 28 MiB/s wr, 442 op/s; 29119/285153 objects degraded (10.212%); 12165/285153 objects misplaced (4.266%); 1024 KiB/s, 15 objects/s recovering 2023-10-01T09:21:04.793 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:04.858 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:21:04.878 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:21:04.901 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:21:04.921 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:21:05.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:05.329 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:21:05.330 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:21:05.330 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:21:05.330 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:21:05.330 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:21:05.330 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:21:05.330 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:21:05.330 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:21:05.330 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-01T09:21:05.331 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:21:05.331 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:21:05.331 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 81 16 95366 95350 73.5343 0 - 0.0132219 2023-10-01T09:21:05.331 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:05.331 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:21:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:21:04] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.33.4" 2023-10-01T09:21:05.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:05.432 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:05.461 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:05.747 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:05.749 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1650787429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:05.749 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:05 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:05.749 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:05 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:05.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1650787429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:05.750 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:05 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:05.750 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:05 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:05.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:05.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:05.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1650787429' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:05 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:05 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:05.851 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:05.859 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:21:05.859 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:21:05.884 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:05.965 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:05.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:05.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:05.997 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 82 16 95366 95350 72.6379 0 - 0.0132219 2023-10-01T09:21:06.069 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:06.097 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:06.191 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:06.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:06.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:06.274 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:21:06.274 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:21:06.300 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:06.330 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:06.470 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:21:06.535 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:06.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:06.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:06.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:06 smithi190 ceph-mon[160435]: pgmap v522: 89 pgs: 1 activating+remapped, 6 peering, 10 remapped+peering, 11 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 40 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 23 MiB/s wr, 361 op/s; 23915/285516 objects degraded (8.376%); 6149/285516 objects misplaced (2.154%); 1024 KiB/s, 15 objects/s recovering 2023-10-01T09:21:06.639 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:06.666 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:06.717 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:06.723 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:06 smithi138 ceph-mon[192513]: pgmap v522: 89 pgs: 1 activating+remapped, 6 peering, 10 remapped+peering, 11 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 40 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 23 MiB/s wr, 361 op/s; 23915/285516 objects degraded (8.376%); 6149/285516 objects misplaced (2.154%); 1024 KiB/s, 15 objects/s recovering 2023-10-01T09:21:06.723 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:06 smithi138 ceph-mon[185886]: pgmap v522: 89 pgs: 1 activating+remapped, 6 peering, 10 remapped+peering, 11 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 40 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 23 MiB/s wr, 361 op/s; 23915/285516 objects degraded (8.376%); 6149/285516 objects misplaced (2.154%); 1024 KiB/s, 15 objects/s recovering 2023-10-01T09:21:06.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:06.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:06.825 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:06.837 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:06.854 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:06.953 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:06.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:06.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:06.997 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 83 16 95366 95350 71.7631 0 - 0.0132219 2023-10-01T09:21:07.011 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:21:07.057 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:07.086 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:07.136 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:07.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:07.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:07.242 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:07.270 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:07.326 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:07.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:07.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:07.350 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 7, 2, 6] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2023-10-01T09:21:07.351 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:21:07.351 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-01T09:21:07.351 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:21:07.430 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:07.457 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:21:07.458 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:07.503 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:07.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:07.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:07.607 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:07.633 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:07.854 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:07.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:07.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:07.860 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:21:07.915 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:07 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:07.915 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:07 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:07.915 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:07 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 23915/285516 objects degraded (8.376%), 11 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:07.915 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:07 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:21:07.915 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:07 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:21:07.915 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:07 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:07.915 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:07 smithi138 ceph-mon[192513]: pgmap v523: 89 pgs: 1 activating+remapped, 1 remapped+peering, 15 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 46 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 19 MiB/s wr, 311 op/s; 24882/285651 objects degraded (8.711%); 6337/285651 objects misplaced (2.218%); 2.0 MiB/s, 32 objects/s recovering 2023-10-01T09:21:07.916 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:07 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:07.916 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:07 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:07.916 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:07 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 23915/285516 objects degraded (8.376%), 11 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:07.916 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:07 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:21:07.916 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:07 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:21:07.916 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:07 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:07.916 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:07 smithi138 ceph-mon[185886]: pgmap v523: 89 pgs: 1 activating+remapped, 1 remapped+peering, 15 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 46 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 19 MiB/s wr, 311 op/s; 24882/285651 objects degraded (8.711%); 6337/285651 objects misplaced (2.218%); 2.0 MiB/s, 32 objects/s recovering 2023-10-01T09:21:07.957 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:07.987 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:07 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:07.987 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:07 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:07.987 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:07 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 23915/285516 objects degraded (8.376%), 11 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:07.987 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:07 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:21:07.987 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:07 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:21:07.987 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:07 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:07.987 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:07 smithi190 ceph-mon[160435]: pgmap v523: 89 pgs: 1 activating+remapped, 1 remapped+peering, 15 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 46 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 19 MiB/s wr, 311 op/s; 24882/285651 objects degraded (8.711%); 6337/285651 objects misplaced (2.218%); 2.0 MiB/s, 32 objects/s recovering 2023-10-01T09:21:07.995 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:07.996 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 84 16 95366 95350 70.9091 0 - 0.0132219 2023-10-01T09:21:08.045 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:08.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:08.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:08.060 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:08.148 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:08.187 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:08.334 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:08.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:08.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:08.439 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:08.473 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:08.517 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:08.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:08.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:08.621 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:08.639 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:08.651 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:08.801 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:08.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:08.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:08.904 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:08.927 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:21:08.927 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:21:08.938 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:08.989 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:08.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:08.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:08.997 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 85 16 95366 95350 70.0752 0 - 0.0132219 2023-10-01T09:21:09.094 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:09.121 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:09.184 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:09 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2952717082' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:21:09.198 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:21:09.223 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:09.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:09.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:09.327 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:09.356 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:09.425 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:09 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2952717082' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:21:09.425 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:09 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2952717082' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:21:09.452 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:09.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:09.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:09.556 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:09.583 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:09.654 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:21:09.905 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:09.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:09.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:09.997 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 86 16 95366 95350 69.2608 0 - 0.0132219 2023-10-01T09:21:10.011 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:10.038 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:10.086 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:10.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:10.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:10.192 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:10.217 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:10 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:10.218 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:10 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2952717082' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:21:10.218 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:10 smithi138 ceph-mon[185886]: osdmap e210: 8 total, 7 up, 8 in 2023-10-01T09:21:10.218 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:10 smithi138 ceph-mon[185886]: failed to encode map e210 with expected crc 2023-10-01T09:21:10.218 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:10 smithi138 ceph-mon[185886]: failed to encode map e210 with expected crc 2023-10-01T09:21:10.218 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:10 smithi138 ceph-mon[185886]: pgmap v525: 89 pgs: 1 activating+remapped, 1 remapped+peering, 15 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 46 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 6.9 MiB/s wr, 110 op/s; 24886/285699 objects degraded (8.711%); 6345/285699 objects misplaced (2.221%); 840 KiB/s, 13 objects/s recovering 2023-10-01T09:21:10.218 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:10 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:10.218 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:10 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2952717082' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:21:10.218 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:10 smithi138 ceph-mon[192513]: osdmap e210: 8 total, 7 up, 8 in 2023-10-01T09:21:10.218 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:10 smithi138 ceph-mon[192513]: failed to encode map e210 with expected crc 2023-10-01T09:21:10.219 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:10 smithi138 ceph-mon[192513]: failed to encode map e210 with expected crc 2023-10-01T09:21:10.219 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:10 smithi138 ceph-mon[192513]: pgmap v525: 89 pgs: 1 activating+remapped, 1 remapped+peering, 15 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 46 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 6.9 MiB/s wr, 110 op/s; 24886/285699 objects degraded (8.711%); 6345/285699 objects misplaced (2.221%); 840 KiB/s, 13 objects/s recovering 2023-10-01T09:21:10.219 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:10.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:10 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:10 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2952717082' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:21:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:10 smithi190 ceph-mon[160435]: osdmap e210: 8 total, 7 up, 8 in 2023-10-01T09:21:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:10 smithi190 ceph-mon[160435]: failed to encode map e210 with expected crc 2023-10-01T09:21:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:10 smithi190 ceph-mon[160435]: failed to encode map e210 with expected crc 2023-10-01T09:21:10.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:10 smithi190 ceph-mon[160435]: pgmap v525: 89 pgs: 1 activating+remapped, 1 remapped+peering, 15 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 46 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 6.9 MiB/s wr, 110 op/s; 24886/285699 objects degraded (8.711%); 6345/285699 objects misplaced (2.221%); 840 KiB/s, 13 objects/s recovering 2023-10-01T09:21:10.733 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:10.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:10.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:10.837 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:10.864 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:10.909 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:10.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:10.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:10.997 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 87 16 95366 95350 68.465 0 - 0.0132219 2023-10-01T09:21:11.010 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:11.015 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:11.048 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:11.100 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:11.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:11.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:11.204 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:11.209 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:21:11.228 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:21:11.234 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:11.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:21:11.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:21:11.332 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:21:11.351 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:21:11.372 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:21:11.391 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:21:11.410 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:21:11.480 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:11.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:11.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:11.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:21:11.584 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:11.612 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:11.964 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:21:11.997 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 88 16 95366 95350 67.6873 0 - 0.0132219 2023-10-01T09:21:12.010 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:12.012 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":525,"stamp":"2023-10-01T09:21:09.425804+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6244145075,"num_objects":95233,"num_object_clones":0,"num_object_copies":285699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24886,"num_objects_misplaced":6345,"num_objects_unfound":0,"num_objects_dirty":95233,"num_whiteouts":0,"num_read":6259,"num_read_kb":12659,"num_write":96311,"num_write_kb":6108261,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":457,"num_bytes_recovered":20919093,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94309,"ondisk_log_size":94309,"up":232,"acting":241,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":251,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15625812,"kb_used_data":13879644,"kb_used_omap":30,"kb_used_meta":1746081,"kb_avail":640590252,"statfs":{"total":671965249536,"available":655964418048,"internally_reserved":0,"allocated":14212755456,"data_stored":14207575779,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30777,"internal_metadata":1787987911},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":53018624,"num_objects":809,"num_object_clones":0,"num_object_copies":2427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-9440,"num_objects_misplaced":-11371,"num_objects_unfound":0,"num_objects_dirty":809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":809,"num_write_kb":51776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":6296190,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.322561"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":78,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.126057+0000","last_change":"2023-10-01T09:19:30.091981+0000","last_active":"2023-10-01T09:21:02.126057+0000","last_peered":"2023-10-01T09:21:02.126057+0000","last_clean":"2023-10-01T09:21:02.126057+0000","last_became_active":"2023-10-01T09:19:30.091531+0000","last_became_peered":"2023-10-01T09:19:30.091531+0000","last_unstale":"2023-10-01T09:21:02.126057+0000","last_undegraded":"2023-10-01T09:21:02.126057+0000","last_fullsized":"2023-10-01T09:21:02.126057+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:28:13.442498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":49,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.139145+0000","last_change":"2023-10-01T09:21:02.131395+0000","last_active":"2023-10-01T09:21:02.139145+0000","last_peered":"2023-10-01T09:21:02.139145+0000","last_clean":"2023-10-01T09:21:02.139145+0000","last_became_active":"2023-10-01T09:21:02.131066+0000","last_became_peered":"2023-10-01T09:21:02.131066+0000","last_unstale":"2023-10-01T09:21:02.139145+0000","last_undegraded":"2023-10-01T09:21:02.139145+0000","last_fullsized":"2023-10-01T09:21:02.139145+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:40:12.021772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":154,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.122822+0000","last_change":"2023-10-01T09:13:13.551596+0000","last_active":"2023-10-01T09:21:02.122822+0000","last_peered":"2023-10-01T09:21:02.122822+0000","last_clean":"2023-10-01T09:21:02.122822+0000","last_became_active":"2023-10-01T09:13:13.551300+0000","last_became_peered":"2023-10-01T09:13:13.551300+0000","last_unstale":"2023-10-01T09:21:02.122822+0000","last_undegraded":"2023-10-01T09:21:02.122822+0000","last_fullsized":"2023-10-01T09:21:02.122822+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:57:45.081780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":184,"reported_epoch":208,"state":"active+undersized","last_fresh":"2023-10-01T09:21:02.111837+0000","last_change":"2023-10-01T09:20:36.515604+0000","last_active":"2023-10-01T09:21:02.111837+0000","last_peered":"2023-10-01T09:21:02.111837+0000","last_clean":"2023-10-01T09:20:28.116381+0000","last_became_active":"2023-10-01T09:20:36.515604+0000","last_became_peered":"2023-10-01T09:20:36.515604+0000","last_unstale":"2023-10-01T09:21:02.111837+0000","last_undegraded":"2023-10-01T09:21:02.111837+0000","last_fullsized":"2023-10-01T09:20:36.507014+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:19:39.031820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":196,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.120466+0000","last_change":"2023-10-01T09:19:30.091032+0000","last_active":"2023-10-01T09:21:02.120466+0000","last_peered":"2023-10-01T09:21:02.120466+0000","last_clean":"2023-10-01T09:21:02.120466+0000","last_became_active":"2023-10-01T09:19:30.090644+0000","last_became_peered":"2023-10-01T09:19:30.090644+0000","last_unstale":"2023-10-01T09:21:02.120466+0000","last_undegraded":"2023-10-01T09:21:02.120466+0000","last_fullsized":"2023-10-01T09:21:02.120466+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:19.617722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":142,"reported_epoch":206,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:20:53.159688+0000","last_change":"2023-10-01T09:20:51.625153+0000","last_active":"2023-10-01T09:20:53.159688+0000","last_peered":"2023-10-01T09:20:53.159688+0000","last_clean":"2023-10-01T09:20:53.159688+0000","last_became_active":"2023-10-01T09:20:51.624687+0000","last_became_peered":"2023-10-01T09:20:51.624687+0000","last_unstale":"2023-10-01T09:20:53.159688+0000","last_undegraded":"2023-10-01T09:20:53.159688+0000","last_fullsized":"2023-10-01T09:20:53.159688+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:16.652123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,6],"avail_no_missing":["1","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":194,"reported_epoch":209,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:03.355091+0000","last_change":"2023-10-01T09:21:03.355091+0000","last_active":"2023-10-01T09:21:03.355091+0000","last_peered":"2023-10-01T09:21:03.355091+0000","last_clean":"2023-10-01T09:21:03.355091+0000","last_became_active":"2023-10-01T09:21:03.354777+0000","last_became_peered":"2023-10-01T09:21:03.354777+0000","last_unstale":"2023-10-01T09:21:03.355091+0000","last_undegraded":"2023-10-01T09:21:03.355091+0000","last_fullsized":"2023-10-01T09:21:03.355091+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:22:37.229201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":198,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.140507+0000","last_change":"2023-10-01T09:21:02.139865+0000","last_active":"2023-10-01T09:21:02.140507+0000","last_peered":"2023-10-01T09:21:02.140507+0000","last_clean":"2023-10-01T09:21:02.140507+0000","last_became_active":"2023-10-01T09:21:02.139507+0000","last_became_peered":"2023-10-01T09:21:02.139507+0000","last_unstale":"2023-10-01T09:21:02.140507+0000","last_undegraded":"2023-10-01T09:21:02.140507+0000","last_fullsized":"2023-10-01T09:21:02.140507+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:46:09.214220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":169,"reported_epoch":208,"state":"active+undersized","last_fresh":"2023-10-01T09:21:02.110563+0000","last_change":"2023-10-01T09:20:36.511477+0000","last_active":"2023-10-01T09:21:02.110563+0000","last_peered":"2023-10-01T09:21:02.110563+0000","last_clean":"2023-10-01T09:19:41.977469+0000","last_became_active":"2023-10-01T09:20:36.511477+0000","last_became_peered":"2023-10-01T09:20:36.511477+0000","last_unstale":"2023-10-01T09:21:02.110563+0000","last_undegraded":"2023-10-01T09:21:02.110563+0000","last_fullsized":"2023-10-01T09:20:36.508783+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:16:03.885738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":205,"reported_epoch":209,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:03.236933+0000","last_change":"2023-10-01T09:21:03.236933+0000","last_active":"2023-10-01T09:21:03.236933+0000","last_peered":"2023-10-01T09:21:03.236933+0000","last_clean":"2023-10-01T09:21:03.236933+0000","last_became_active":"2023-10-01T09:21:03.236508+0000","last_became_peered":"2023-10-01T09:21:03.236508+0000","last_unstale":"2023-10-01T09:21:03.236933+0000","last_undegraded":"2023-10-01T09:21:03.236933+0000","last_fullsized":"2023-10-01T09:21:03.236933+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:58:15.332216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,0],"avail_no_missing":["3","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":54,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.107637+0000","last_change":"2023-10-01T09:19:44.369992+0000","last_active":"2023-10-01T09:21:02.107637+0000","last_peered":"2023-10-01T09:21:02.107637+0000","last_clean":"2023-10-01T09:21:02.107637+0000","last_became_active":"2023-10-01T09:19:44.346816+0000","last_became_peered":"2023-10-01T09:19:44.346816+0000","last_unstale":"2023-10-01T09:21:02.107637+0000","last_undegraded":"2023-10-01T09:21:02.107637+0000","last_fullsized":"2023-10-01T09:21:02.107637+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:44:40.345845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":208,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.187907+0000","last_change":"2023-10-01T09:21:02.158958+0000","last_active":"2023-10-01T09:21:02.187907+0000","last_peered":"2023-10-01T09:21:02.187907+0000","last_clean":"2023-10-01T09:21:02.187907+0000","last_became_active":"2023-10-01T09:21:02.158491+0000","last_became_peered":"2023-10-01T09:21:02.158491+0000","last_unstale":"2023-10-01T09:21:02.187907+0000","last_undegraded":"2023-10-01T09:21:02.187907+0000","last_fullsized":"2023-10-01T09:21:02.187907+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:21:48.885982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":154,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.164950+0000","last_change":"2023-10-01T09:21:02.163403+0000","last_active":"2023-10-01T09:21:02.164950+0000","last_peered":"2023-10-01T09:21:02.164950+0000","last_clean":"2023-10-01T09:21:02.164950+0000","last_became_active":"2023-10-01T09:21:02.121680+0000","last_became_peered":"2023-10-01T09:21:02.121680+0000","last_unstale":"2023-10-01T09:21:02.164950+0000","last_undegraded":"2023-10-01T09:21:02.164950+0000","last_fullsized":"2023-10-01T09:21:02.164950+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":176,"reported_epoch":205,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:20:52.116870+0000","last_change":"2023-10-01T09:20:51.570020+0000","last_active":"2023-10-01T09:20:52.116870+0000","last_peered":"2023-10-01T09:20:52.116870+0000","last_clean":"2023-10-01T09:20:52.116870+0000","last_became_active":"2023-10-01T09:20:51.569619+0000","last_became_peered":"2023-10-01T09:20:51.569619+0000","last_unstale":"2023-10-01T09:20:52.116870+0000","last_undegraded":"2023-10-01T09:20:52.116870+0000","last_fullsized":"2023-10-01T09:20:52.116870+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:17:49.361613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,6],"avail_no_missing":["1","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":237,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.107616+0000","last_change":"2023-10-01T09:19:30.182046+0000","last_active":"2023-10-01T09:21:02.107616+0000","last_peered":"2023-10-01T09:21:02.107616+0000","last_clean":"2023-10-01T09:21:02.107616+0000","last_became_active":"2023-10-01T09:19:30.094579+0000","last_became_peered":"2023-10-01T09:19:30.094579+0000","last_unstale":"2023-10-01T09:21:02.107616+0000","last_undegraded":"2023-10-01T09:21:02.107616+0000","last_fullsized":"2023-10-01T09:21:02.107616+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":189,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.108635+0000","last_change":"2023-10-01T09:19:57.493670+0000","last_active":"2023-10-01T09:21:02.108635+0000","last_peered":"2023-10-01T09:21:02.108635+0000","last_clean":"2023-10-01T09:21:02.108635+0000","last_became_active":"2023-10-01T09:19:57.493349+0000","last_became_peered":"2023-10-01T09:19:57.493349+0000","last_unstale":"2023-10-01T09:21:02.108635+0000","last_undegraded":"2023-10-01T09:21:02.108635+0000","last_fullsized":"2023-10-01T09:21:02.108635+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:26:56.468779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":177,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.116645+0000","last_change":"2023-10-01T09:19:44.354776+0000","last_active":"2023-10-01T09:21:02.116645+0000","last_peered":"2023-10-01T09:21:02.116645+0000","last_clean":"2023-10-01T09:21:02.116645+0000","last_became_active":"2023-10-01T09:19:44.354447+0000","last_became_peered":"2023-10-01T09:19:44.354447+0000","last_unstale":"2023-10-01T09:21:02.116645+0000","last_undegraded":"2023-10-01T09:21:02.116645+0000","last_fullsized":"2023-10-01T09:21:02.116645+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:53.120763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":80,"reported_epoch":208,"state":"active+undersized","last_fresh":"2023-10-01T09:21:02.117140+0000","last_change":"2023-10-01T09:20:36.516058+0000","last_active":"2023-10-01T09:21:02.117140+0000","last_peered":"2023-10-01T09:21:02.117140+0000","last_clean":"2023-10-01T09:12:40.941890+0000","last_became_active":"2023-10-01T09:20:36.516058+0000","last_became_peered":"2023-10-01T09:20:36.516058+0000","last_unstale":"2023-10-01T09:21:02.117140+0000","last_undegraded":"2023-10-01T09:21:02.117140+0000","last_fullsized":"2023-10-01T09:20:36.510370+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":178,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.123538+0000","last_change":"2023-10-01T09:21:02.121558+0000","last_active":"2023-10-01T09:21:02.123538+0000","last_peered":"2023-10-01T09:21:02.123538+0000","last_clean":"2023-10-01T09:21:02.123538+0000","last_became_active":"2023-10-01T09:21:02.121153+0000","last_became_peered":"2023-10-01T09:21:02.121153+0000","last_unstale":"2023-10-01T09:21:02.123538+0000","last_undegraded":"2023-10-01T09:21:02.123538+0000","last_fullsized":"2023-10-01T09:21:02.123538+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":168,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.111703+0000","last_change":"2023-10-01T09:19:44.345829+0000","last_active":"2023-10-01T09:21:02.111703+0000","last_peered":"2023-10-01T09:21:02.111703+0000","last_clean":"2023-10-01T09:21:02.111703+0000","last_became_active":"2023-10-01T09:19:44.345499+0000","last_became_peered":"2023-10-01T09:19:44.345499+0000","last_unstale":"2023-10-01T09:21:02.111703+0000","last_undegraded":"2023-10-01T09:21:02.111703+0000","last_fullsized":"2023-10-01T09:21:02.111703+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:23:30.028532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":102,"reported_epoch":205,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:20:52.114485+0000","last_change":"2023-10-01T09:20:51.659192+0000","last_active":"2023-10-01T09:20:52.114485+0000","last_peered":"2023-10-01T09:20:52.114485+0000","last_clean":"2023-10-01T09:20:52.114485+0000","last_became_active":"2023-10-01T09:20:51.561674+0000","last_became_peered":"2023-10-01T09:20:51.561674+0000","last_unstale":"2023-10-01T09:20:52.114485+0000","last_undegraded":"2023-10-01T09:20:52.114485+0000","last_fullsized":"2023-10-01T09:20:52.114485+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:34:06.618403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":193,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.154850+0000","last_change":"2023-10-01T09:21:02.137372+0000","last_active":"2023-10-01T09:21:02.154850+0000","last_peered":"2023-10-01T09:21:02.154850+0000","last_clean":"2023-10-01T09:21:02.154850+0000","last_became_active":"2023-10-01T09:21:02.137080+0000","last_became_peered":"2023-10-01T09:21:02.137080+0000","last_unstale":"2023-10-01T09:21:02.154850+0000","last_undegraded":"2023-10-01T09:21:02.154850+0000","last_fullsized":"2023-10-01T09:21:02.154850+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:38:03.902930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.a","version":"206'5874","reported_seq":5920,"reported_epoch":206,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:20:57.164391+0000","last_change":"2023-10-01T09:20:51.962003+0000","last_active":"2023-10-01T09:20:57.164391+0000","last_peered":"2023-10-01T09:20:57.164391+0000","last_clean":"2023-10-01T09:20:35.491448+0000","last_became_active":"2023-10-01T09:20:51.943877+0000","last_became_peered":"2023-10-01T09:20:51.943877+0000","last_unstale":"2023-10-01T09:20:57.164391+0000","last_undegraded":"2023-10-01T09:20:51.580253+0000","last_fullsized":"2023-10-01T09:20:51.540484+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5874,"log_dups_size":0,"ondisk_log_size":5874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:19:04.863277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384958464,"num_objects":5874,"num_object_clones":0,"num_object_copies":17622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5704,"num_objects_misplaced":5874,"num_objects_unfound":0,"num_objects_dirty":5874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5874,"num_write_kb":375936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":170}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":52,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.113589+0000","last_change":"2023-10-01T09:19:44.373526+0000","last_active":"2023-10-01T09:21:02.113589+0000","last_peered":"2023-10-01T09:21:02.113589+0000","last_clean":"2023-10-01T09:21:02.113589+0000","last_became_active":"2023-10-01T09:19:44.354026+0000","last_became_peered":"2023-10-01T09:19:44.354026+0000","last_unstale":"2023-10-01T09:21:02.113589+0000","last_undegraded":"2023-10-01T09:21:02.113589+0000","last_fullsized":"2023-10-01T09:21:02.113589+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:01:49.696317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":15,"reported_epoch":206,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:20:53.146669+0000","last_change":"2023-10-01T09:20:51.570951+0000","last_active":"2023-10-01T09:20:53.146669+0000","last_peered":"2023-10-01T09:20:53.146669+0000","last_clean":"2023-10-01T09:20:53.146669+0000","last_became_active":"2023-10-01T09:20:51.570604+0000","last_became_peered":"2023-10-01T09:20:51.570604+0000","last_unstale":"2023-10-01T09:20:53.146669+0000","last_undegraded":"2023-10-01T09:20:53.146669+0000","last_fullsized":"2023-10-01T09:20:53.146669+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:27.753267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,6],"avail_no_missing":["1","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.b","version":"209'5990","reported_seq":6056,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:21:03.125133+0000","last_change":"2023-10-01T09:21:02.181394+0000","last_active":"2023-10-01T09:21:03.125133+0000","last_peered":"2023-10-01T09:21:03.125133+0000","last_clean":"2023-10-01T09:20:49.475349+0000","last_became_active":"2023-10-01T09:21:02.175087+0000","last_became_peered":"2023-10-01T09:21:02.175087+0000","last_unstale":"2023-10-01T09:21:03.125133+0000","last_undegraded":"2023-10-01T09:21:02.134632+0000","last_fullsized":"2023-10-01T09:21:02.106953+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5990,"log_dups_size":0,"ondisk_log_size":5990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:21:33.658961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392560640,"num_objects":5990,"num_object_clones":0,"num_object_copies":17970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":318,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5990,"num_write_kb":383360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"1,3","objects":5672},{"shards":"1,3,4","objects":318}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":177,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.108873+0000","last_change":"2023-10-01T09:12:10.003552+0000","last_active":"2023-10-01T09:21:02.108873+0000","last_peered":"2023-10-01T09:21:02.108873+0000","last_clean":"2023-10-01T09:21:02.108873+0000","last_became_active":"2023-10-01T09:12:10.003131+0000","last_became_peered":"2023-10-01T09:12:10.003131+0000","last_unstale":"2023-10-01T09:21:02.108873+0000","last_undegraded":"2023-10-01T09:21:02.108873+0000","last_fullsized":"2023-10-01T09:21:02.108873+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":200,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-01T09:21:03.119675+0000","last_change":"2023-10-01T09:21:02.159564+0000","last_active":"2023-10-01T09:21:03.119675+0000","last_peered":"2023-10-01T09:21:03.119675+0000","last_clean":"2023-10-01T09:21:03.119675+0000","last_became_active":"2023-10-01T09:21:02.159342+0000","last_became_peered":"2023-10-01T09:21:02.159342+0000","last_unstale":"2023-10-01T09:21:03.119675+0000","last_undegraded":"2023-10-01T09:21:03.119675+0000","last_fullsized":"2023-10-01T09:21:03.119675+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:45:04.227823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.8","version":"209'5972","reported_seq":6431,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:21:03.539270+0000","last_change":"2023-10-01T09:21:02.182885+0000","last_active":"2023-10-01T09:21:03.539270+0000","last_peered":"2023-10-01T09:21:03.539270+0000","last_clean":"2023-10-01T09:20:49.525699+0000","last_became_active":"2023-10-01T09:21:02.175091+0000","last_became_peered":"2023-10-01T09:21:02.175091+0000","last_unstale":"2023-10-01T09:21:03.539270+0000","last_undegraded":"2023-10-01T09:21:02.136996+0000","last_fullsized":"2023-10-01T09:21:02.106788+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5972,"log_dups_size":0,"ondisk_log_size":5972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:16:25.782704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391380992,"num_objects":5972,"num_object_clones":0,"num_object_copies":17916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":324,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5972,"num_write_kb":382208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":203,"num_bytes_recovered":13303808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"0,2,3","objects":324},{"shards":"2,3","objects":5648}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":191,"reported_epoch":209,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:03.353156+0000","last_change":"2023-10-01T09:21:03.353156+0000","last_active":"2023-10-01T09:21:03.353156+0000","last_peered":"2023-10-01T09:21:03.353156+0000","last_clean":"2023-10-01T09:21:03.353156+0000","last_became_active":"2023-10-01T09:21:03.352649+0000","last_became_peered":"2023-10-01T09:21:03.352649+0000","last_unstale":"2023-10-01T09:21:03.353156+0000","last_undegraded":"2023-10-01T09:21:03.353156+0000","last_fullsized":"2023-10-01T09:21:03.353156+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6,3],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":147,"reported_epoch":205,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:20:52.116341+0000","last_change":"2023-10-01T09:20:51.626919+0000","last_active":"2023-10-01T09:20:52.116341+0000","last_peered":"2023-10-01T09:20:52.116341+0000","last_clean":"2023-10-01T09:20:52.116341+0000","last_became_active":"2023-10-01T09:20:51.625798+0000","last_became_peered":"2023-10-01T09:20:51.625798+0000","last_unstale":"2023-10-01T09:20:52.116341+0000","last_undegraded":"2023-10-01T09:20:52.116341+0000","last_fullsized":"2023-10-01T09:20:52.116341+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:15:25.823439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,6],"avail_no_missing":["1","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"209'5941","reported_seq":5980,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-01T09:21:03.515093+0000","last_change":"2023-10-01T09:19:34.259103+0000","last_active":"2023-10-01T09:21:03.515093+0000","last_peered":"2023-10-01T09:21:03.515093+0000","last_clean":"2023-10-01T09:21:03.515093+0000","last_became_active":"2023-10-01T09:19:34.258143+0000","last_became_peered":"2023-10-01T09:19:34.258143+0000","last_unstale":"2023-10-01T09:21:03.515093+0000","last_undegraded":"2023-10-01T09:21:03.515093+0000","last_fullsized":"2023-10-01T09:21:03.515093+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5941,"log_dups_size":0,"ondisk_log_size":5941,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:55:08.621837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389349376,"num_objects":5941,"num_object_clones":0,"num_object_copies":17823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5941,"num_write_kb":380224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":147,"reported_epoch":209,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:03.357675+0000","last_change":"2023-10-01T09:21:03.357675+0000","last_active":"2023-10-01T09:21:03.357675+0000","last_peered":"2023-10-01T09:21:03.357675+0000","last_clean":"2023-10-01T09:21:03.357675+0000","last_became_active":"2023-10-01T09:21:03.357333+0000","last_became_peered":"2023-10-01T09:21:03.357333+0000","last_unstale":"2023-10-01T09:21:03.357675+0000","last_undegraded":"2023-10-01T09:21:03.357675+0000","last_fullsized":"2023-10-01T09:21:03.357675+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:32:45.291673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,0],"avail_no_missing":["1","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":159,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.114358+0000","last_change":"2023-10-01T09:12:10.001565+0000","last_active":"2023-10-01T09:21:02.114358+0000","last_peered":"2023-10-01T09:21:02.114358+0000","last_clean":"2023-10-01T09:21:02.114358+0000","last_became_active":"2023-10-01T09:12:10.001275+0000","last_became_peered":"2023-10-01T09:12:10.001275+0000","last_unstale":"2023-10-01T09:21:02.114358+0000","last_undegraded":"2023-10-01T09:21:02.114358+0000","last_fullsized":"2023-10-01T09:21:02.114358+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:39.067046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"209'6028","reported_seq":6067,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-01T09:21:03.495248+0000","last_change":"2023-10-01T09:19:34.254772+0000","last_active":"2023-10-01T09:21:03.495248+0000","last_peered":"2023-10-01T09:21:03.495248+0000","last_clean":"2023-10-01T09:21:03.495248+0000","last_became_active":"2023-10-01T09:19:34.254457+0000","last_became_peered":"2023-10-01T09:19:34.254457+0000","last_unstale":"2023-10-01T09:21:03.495248+0000","last_undegraded":"2023-10-01T09:21:03.495248+0000","last_fullsized":"2023-10-01T09:21:03.495248+0000","mapping_epoch":179,"log_start":"187'587","ondisk_log_start":"187'587","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5441,"log_dups_size":0,"ondisk_log_size":5441,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:19:20.462736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395051008,"num_objects":6028,"num_object_clones":0,"num_object_copies":18084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6028,"num_write_kb":385792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":178,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.107409+0000","last_change":"2023-10-01T09:13:13.554321+0000","last_active":"2023-10-01T09:21:02.107409+0000","last_peered":"2023-10-01T09:21:02.107409+0000","last_clean":"2023-10-01T09:21:02.107409+0000","last_became_active":"2023-10-01T09:13:13.553986+0000","last_became_peered":"2023-10-01T09:13:13.553986+0000","last_unstale":"2023-10-01T09:21:02.107409+0000","last_undegraded":"2023-10-01T09:21:02.107409+0000","last_fullsized":"2023-10-01T09:21:02.107409+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:09:12.848269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":138,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.141383+0000","last_change":"2023-10-01T09:21:02.139718+0000","last_active":"2023-10-01T09:21:02.141383+0000","last_peered":"2023-10-01T09:21:02.141383+0000","last_clean":"2023-10-01T09:21:02.141383+0000","last_became_active":"2023-10-01T09:21:02.139406+0000","last_became_peered":"2023-10-01T09:21:02.139406+0000","last_unstale":"2023-10-01T09:21:02.141383+0000","last_undegraded":"2023-10-01T09:21:02.141383+0000","last_fullsized":"2023-10-01T09:21:02.141383+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:10.823093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.f","version":"209'5801","reported_seq":5858,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:21:03.506103+0000","last_change":"2023-10-01T09:21:02.259500+0000","last_active":"2023-10-01T09:21:03.506103+0000","last_peered":"2023-10-01T09:21:03.506103+0000","last_clean":"2023-10-01T09:20:49.538689+0000","last_became_active":"2023-10-01T09:21:02.164035+0000","last_became_peered":"2023-10-01T09:21:02.164035+0000","last_unstale":"2023-10-01T09:21:03.506103+0000","last_undegraded":"2023-10-01T09:21:02.121200+0000","last_fullsized":"2023-10-01T09:21:02.111274+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5801,"log_dups_size":0,"ondisk_log_size":5801,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:44:20.685678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380174336,"num_objects":5801,"num_object_clones":0,"num_object_copies":17403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":294,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5801,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5801,"num_write_kb":371264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"1,2,4","objects":294},{"shards":"2,4","objects":5507}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":135,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-01T09:21:03.119423+0000","last_change":"2023-10-01T09:21:02.136776+0000","last_active":"2023-10-01T09:21:03.119423+0000","last_peered":"2023-10-01T09:21:03.119423+0000","last_clean":"2023-10-01T09:21:03.119423+0000","last_became_active":"2023-10-01T09:21:02.136476+0000","last_became_peered":"2023-10-01T09:21:02.136476+0000","last_unstale":"2023-10-01T09:21:03.119423+0000","last_undegraded":"2023-10-01T09:21:03.119423+0000","last_fullsized":"2023-10-01T09:21:03.119423+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:44:41.289825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":206,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.118366+0000","last_change":"2023-10-01T09:20:15.505658+0000","last_active":"2023-10-01T09:21:02.118366+0000","last_peered":"2023-10-01T09:21:02.118366+0000","last_clean":"2023-10-01T09:21:02.118366+0000","last_became_active":"2023-10-01T09:20:15.505361+0000","last_became_peered":"2023-10-01T09:20:15.505361+0000","last_unstale":"2023-10-01T09:21:02.118366+0000","last_undegraded":"2023-10-01T09:21:02.118366+0000","last_fullsized":"2023-10-01T09:21:02.118366+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:22:16.176432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.c","version":"206'5956","reported_seq":6009,"reported_epoch":207,"state":"remapped+peering","last_fresh":"2023-10-01T09:21:01.410005+0000","last_change":"2023-10-01T09:21:01.410955+0000","last_active":"2023-10-01T09:21:01.410005+0000","last_peered":"2023-10-01T09:21:01.337269+0000","last_clean":"2023-10-01T09:20:35.505999+0000","last_became_active":"2023-10-01T09:20:51.992989+0000","last_became_peered":"2023-10-01T09:20:51.992989+0000","last_unstale":"2023-10-01T09:21:01.410005+0000","last_undegraded":"2023-10-01T09:21:01.410005+0000","last_fullsized":"2023-10-01T09:21:01.410005+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5956,"log_dups_size":0,"ondisk_log_size":5956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390332416,"num_objects":5956,"num_object_clones":0,"num_object_copies":17868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5956,"num_write_kb":381184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,6],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":76,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-01T09:21:03.122896+0000","last_change":"2023-10-01T09:19:30.092681+0000","last_active":"2023-10-01T09:21:03.122896+0000","last_peered":"2023-10-01T09:21:03.122896+0000","last_clean":"2023-10-01T09:21:03.122896+0000","last_became_active":"2023-10-01T09:19:30.092394+0000","last_became_peered":"2023-10-01T09:19:30.092394+0000","last_unstale":"2023-10-01T09:21:03.122896+0000","last_undegraded":"2023-10-01T09:21:03.122896+0000","last_fullsized":"2023-10-01T09:21:03.122896+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:41:50.127253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":144,"reported_epoch":209,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:03.352540+0000","last_change":"2023-10-01T09:21:03.352540+0000","last_active":"2023-10-01T09:21:03.352540+0000","last_peered":"2023-10-01T09:21:03.352540+0000","last_clean":"2023-10-01T09:21:03.352540+0000","last_became_active":"2023-10-01T09:21:03.351992+0000","last_became_peered":"2023-10-01T09:21:03.351992+0000","last_unstale":"2023-10-01T09:21:03.352540+0000","last_undegraded":"2023-10-01T09:21:03.352540+0000","last_fullsized":"2023-10-01T09:21:03.352540+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,3],"avail_no_missing":["4","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"209'6015","reported_seq":6057,"reported_epoch":209,"state":"active+undersized+degraded","last_fresh":"2023-10-01T09:21:03.518341+0000","last_change":"2023-10-01T09:20:36.517952+0000","last_active":"2023-10-01T09:21:03.518341+0000","last_peered":"2023-10-01T09:21:03.518341+0000","last_clean":"2023-10-01T09:20:35.465131+0000","last_became_active":"2023-10-01T09:20:36.517952+0000","last_became_peered":"2023-10-01T09:20:36.517952+0000","last_unstale":"2023-10-01T09:21:03.518341+0000","last_undegraded":"2023-10-01T09:20:36.511581+0000","last_fullsized":"2023-10-01T09:20:36.511516+0000","mapping_epoch":198,"log_start":"187'561","ondisk_log_start":"187'561","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5454,"log_dups_size":0,"ondisk_log_size":5454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:40:19.389753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394199040,"num_objects":6015,"num_object_clones":0,"num_object_copies":18045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6015,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6015,"num_write_kb":384960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":6015}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":145,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.110332+0000","last_change":"2023-10-01T09:17:59.310176+0000","last_active":"2023-10-01T09:21:02.110332+0000","last_peered":"2023-10-01T09:21:02.110332+0000","last_clean":"2023-10-01T09:21:02.110332+0000","last_became_active":"2023-10-01T09:17:59.309857+0000","last_became_peered":"2023-10-01T09:17:59.309857+0000","last_unstale":"2023-10-01T09:21:02.110332+0000","last_undegraded":"2023-10-01T09:21:02.110332+0000","last_fullsized":"2023-10-01T09:21:02.110332+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:17:24.906057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":73,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.109820+0000","last_change":"2023-10-01T09:18:31.419430+0000","last_active":"2023-10-01T09:21:02.109820+0000","last_peered":"2023-10-01T09:21:02.109820+0000","last_clean":"2023-10-01T09:21:02.109820+0000","last_became_active":"2023-10-01T09:18:31.419189+0000","last_became_peered":"2023-10-01T09:18:31.419189+0000","last_unstale":"2023-10-01T09:21:02.109820+0000","last_undegraded":"2023-10-01T09:21:02.109820+0000","last_fullsized":"2023-10-01T09:21:02.109820+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:08:48.247926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"209'5918","reported_seq":5957,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-01T09:21:03.452481+0000","last_change":"2023-10-01T09:19:34.258841+0000","last_active":"2023-10-01T09:21:03.452481+0000","last_peered":"2023-10-01T09:21:03.452481+0000","last_clean":"2023-10-01T09:21:03.452481+0000","last_became_active":"2023-10-01T09:19:34.258571+0000","last_became_peered":"2023-10-01T09:19:34.258571+0000","last_unstale":"2023-10-01T09:21:03.452481+0000","last_undegraded":"2023-10-01T09:21:03.452481+0000","last_fullsized":"2023-10-01T09:21:03.452481+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:12:27.336017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387842048,"num_objects":5918,"num_object_clones":0,"num_object_copies":17754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5918,"num_write_kb":378752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1025,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.127352+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:21:02.127352+0000","last_peered":"2023-10-01T09:21:02.127352+0000","last_clean":"2023-10-01T09:21:02.127352+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:21:02.127352+0000","last_undegraded":"2023-10-01T09:21:02.127352+0000","last_fullsized":"2023-10-01T09:21:02.127352+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":215,"reported_epoch":209,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:03.236916+0000","last_change":"2023-10-01T09:21:03.236916+0000","last_active":"2023-10-01T09:21:03.236916+0000","last_peered":"2023-10-01T09:21:03.236916+0000","last_clean":"2023-10-01T09:21:03.236916+0000","last_became_active":"2023-10-01T09:21:03.236521+0000","last_became_peered":"2023-10-01T09:21:03.236521+0000","last_unstale":"2023-10-01T09:21:03.236916+0000","last_undegraded":"2023-10-01T09:21:03.236916+0000","last_fullsized":"2023-10-01T09:21:03.236916+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:27:49.593898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,1],"avail_no_missing":["2","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":146,"reported_epoch":208,"state":"active+undersized","last_fresh":"2023-10-01T09:21:02.122751+0000","last_change":"2023-10-01T09:20:36.512182+0000","last_active":"2023-10-01T09:21:02.122751+0000","last_peered":"2023-10-01T09:21:02.122751+0000","last_clean":"2023-10-01T09:20:28.303986+0000","last_became_active":"2023-10-01T09:20:36.512182+0000","last_became_peered":"2023-10-01T09:20:36.512182+0000","last_unstale":"2023-10-01T09:21:02.122751+0000","last_undegraded":"2023-10-01T09:21:02.122751+0000","last_fullsized":"2023-10-01T09:20:36.509527+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:11:17.293636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"209'5975","reported_seq":6032,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-10-01T09:21:03.540865+0000","last_change":"2023-10-01T09:21:02.135520+0000","last_active":"2023-10-01T09:21:03.540865+0000","last_peered":"2023-10-01T09:21:03.540865+0000","last_clean":"2023-10-01T09:21:03.540865+0000","last_became_active":"2023-10-01T09:21:02.125689+0000","last_became_peered":"2023-10-01T09:21:02.125689+0000","last_unstale":"2023-10-01T09:21:03.540865+0000","last_undegraded":"2023-10-01T09:21:03.540865+0000","last_fullsized":"2023-10-01T09:21:03.540865+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5975,"log_dups_size":0,"ondisk_log_size":5975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391577600,"num_objects":5975,"num_object_clones":0,"num_object_copies":17925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5975,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":785,"reported_epoch":209,"state":"activating+remapped","last_fresh":"2023-10-01T09:21:03.119685+0000","last_change":"2023-10-01T09:21:03.119685+0000","last_active":"2023-10-01T09:21:01.400736+0000","last_peered":"2023-10-01T09:20:53.149802+0000","last_clean":"2023-10-01T09:20:53.149802+0000","last_became_active":"2023-10-01T09:20:51.627196+0000","last_became_peered":"2023-10-01T09:20:51.627196+0000","last_unstale":"2023-10-01T09:21:03.119685+0000","last_undegraded":"2023-10-01T09:21:03.119685+0000","last_fullsized":"2023-10-01T09:21:03.119685+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:42:09.042479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,0],"avail_no_missing":["1","0","6"],"object_location_counts":[{"shards":"0,1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":126,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.275730+0000","last_change":"2023-10-01T09:21:02.121225+0000","last_active":"2023-10-01T09:21:02.275730+0000","last_peered":"2023-10-01T09:21:02.275730+0000","last_clean":"2023-10-01T09:21:02.275730+0000","last_became_active":"2023-10-01T09:21:02.120911+0000","last_became_peered":"2023-10-01T09:21:02.120911+0000","last_unstale":"2023-10-01T09:21:02.275730+0000","last_undegraded":"2023-10-01T09:21:02.275730+0000","last_fullsized":"2023-10-01T09:21:02.275730+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:34:44.385101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":54,"reported_epoch":209,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:03.359828+0000","last_change":"2023-10-01T09:21:03.359828+0000","last_active":"2023-10-01T09:21:03.359828+0000","last_peered":"2023-10-01T09:21:03.359828+0000","last_clean":"2023-10-01T09:21:03.359828+0000","last_became_active":"2023-10-01T09:21:03.359385+0000","last_became_peered":"2023-10-01T09:21:03.359385+0000","last_unstale":"2023-10-01T09:21:03.359828+0000","last_undegraded":"2023-10-01T09:21:03.359828+0000","last_fullsized":"2023-10-01T09:21:03.359828+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:27:31.856759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,1],"avail_no_missing":["2","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"209'5902","reported_seq":5956,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:21:03.393200+0000","last_change":"2023-10-01T09:21:02.198833+0000","last_active":"2023-10-01T09:21:03.393200+0000","last_peered":"2023-10-01T09:21:03.393200+0000","last_clean":"2023-10-01T09:20:49.508087+0000","last_became_active":"2023-10-01T09:21:02.187934+0000","last_became_peered":"2023-10-01T09:21:02.187934+0000","last_unstale":"2023-10-01T09:21:03.393200+0000","last_undegraded":"2023-10-01T09:21:02.120904+0000","last_fullsized":"2023-10-01T09:21:02.107758+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5902,"log_dups_size":0,"ondisk_log_size":5902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386793472,"num_objects":5902,"num_object_clones":0,"num_object_copies":17706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":316,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5902,"num_write_kb":377728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"2,3,7","objects":316},{"shards":"3,7","objects":5586}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":502,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.113396+0000","last_change":"2023-10-01T09:20:15.506193+0000","last_active":"2023-10-01T09:21:02.113396+0000","last_peered":"2023-10-01T09:21:02.113396+0000","last_clean":"2023-10-01T09:21:02.113396+0000","last_became_active":"2023-10-01T09:20:15.505599+0000","last_became_peered":"2023-10-01T09:20:15.505599+0000","last_unstale":"2023-10-01T09:21:02.113396+0000","last_undegraded":"2023-10-01T09:21:02.113396+0000","last_fullsized":"2023-10-01T09:21:02.113396+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:09:20.837289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":209,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.141486+0000","last_change":"2023-10-01T09:21:02.140363+0000","last_active":"2023-10-01T09:21:02.141486+0000","last_peered":"2023-10-01T09:21:02.141486+0000","last_clean":"2023-10-01T09:21:02.141486+0000","last_became_active":"2023-10-01T09:21:02.137198+0000","last_became_peered":"2023-10-01T09:21:02.137198+0000","last_unstale":"2023-10-01T09:21:02.141486+0000","last_undegraded":"2023-10-01T09:21:02.141486+0000","last_fullsized":"2023-10-01T09:21:02.141486+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:38:38.043146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":60,"reported_epoch":208,"state":"active+undersized","last_fresh":"2023-10-01T09:21:02.110415+0000","last_change":"2023-10-01T09:20:36.516518+0000","last_active":"2023-10-01T09:21:02.110415+0000","last_peered":"2023-10-01T09:21:02.110415+0000","last_clean":"2023-10-01T09:20:28.124400+0000","last_became_active":"2023-10-01T09:20:36.516518+0000","last_became_peered":"2023-10-01T09:20:36.516518+0000","last_unstale":"2023-10-01T09:21:02.110415+0000","last_undegraded":"2023-10-01T09:21:02.110415+0000","last_fullsized":"2023-10-01T09:20:36.511376+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:54:42.493772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"209'5977","reported_seq":6019,"reported_epoch":209,"state":"active+undersized+degraded","last_fresh":"2023-10-01T09:21:03.461697+0000","last_change":"2023-10-01T09:20:36.626688+0000","last_active":"2023-10-01T09:21:03.461697+0000","last_peered":"2023-10-01T09:21:03.461697+0000","last_clean":"2023-10-01T09:20:35.374995+0000","last_became_active":"2023-10-01T09:20:36.626688+0000","last_became_peered":"2023-10-01T09:20:36.626688+0000","last_unstale":"2023-10-01T09:21:03.461697+0000","last_undegraded":"2023-10-01T09:20:36.515631+0000","last_fullsized":"2023-10-01T09:20:36.515526+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5977,"log_dups_size":0,"ondisk_log_size":5977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:41:51.119576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391708672,"num_objects":5977,"num_object_clones":0,"num_object_copies":17931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5977,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5977,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":5977}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1149,"reported_epoch":208,"state":"active+undersized+degraded","last_fresh":"2023-10-01T09:21:02.115331+0000","last_change":"2023-10-01T09:20:36.518526+0000","last_active":"2023-10-01T09:21:02.115331+0000","last_peered":"2023-10-01T09:21:02.115331+0000","last_clean":"2023-10-01T09:19:24.623365+0000","last_became_active":"2023-10-01T09:20:36.518526+0000","last_became_peered":"2023-10-01T09:20:36.518526+0000","last_unstale":"2023-10-01T09:21:02.115331+0000","last_undegraded":"2023-10-01T09:20:36.503982+0000","last_fullsized":"2023-10-01T09:20:36.503855+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:56:30.953630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":353,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:21:03.360224+0000","last_change":"2023-10-01T09:21:03.360339+0000","last_active":"2023-10-01T09:21:03.360224+0000","last_peered":"2023-10-01T09:21:03.360224+0000","last_clean":"2023-10-01T09:20:53.169665+0000","last_became_active":"2023-10-01T09:21:03.360129+0000","last_became_peered":"2023-10-01T09:21:03.360129+0000","last_unstale":"2023-10-01T09:21:03.360224+0000","last_undegraded":"2023-10-01T09:21:03.122861+0000","last_fullsized":"2023-10-01T09:21:03.122312+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:34:18.558226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,1],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":206,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:20:53.149374+0000","last_change":"2023-10-01T09:20:51.570218+0000","last_active":"2023-10-01T09:20:53.149374+0000","last_peered":"2023-10-01T09:20:53.149374+0000","last_clean":"2023-10-01T09:20:53.149374+0000","last_became_active":"2023-10-01T09:20:51.569532+0000","last_became_peered":"2023-10-01T09:20:51.569532+0000","last_unstale":"2023-10-01T09:20:53.149374+0000","last_undegraded":"2023-10-01T09:20:53.149374+0000","last_fullsized":"2023-10-01T09:20:53.149374+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:26:55.956171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,6],"avail_no_missing":["1","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":81,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.132244+0000","last_change":"2023-10-01T09:21:02.125167+0000","last_active":"2023-10-01T09:21:02.132244+0000","last_peered":"2023-10-01T09:21:02.132244+0000","last_clean":"2023-10-01T09:21:02.132244+0000","last_became_active":"2023-10-01T09:21:02.124791+0000","last_became_peered":"2023-10-01T09:21:02.124791+0000","last_unstale":"2023-10-01T09:21:02.132244+0000","last_undegraded":"2023-10-01T09:21:02.132244+0000","last_fullsized":"2023-10-01T09:21:02.132244+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.4","version":"209'5908","reported_seq":5966,"reported_epoch":209,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-01T09:21:03.501064+0000","last_change":"2023-10-01T09:21:03.373345+0000","last_active":"2023-10-01T09:21:03.501064+0000","last_peered":"2023-10-01T09:21:03.501064+0000","last_clean":"2023-10-01T09:20:35.496239+0000","last_became_active":"2023-10-01T09:21:03.362558+0000","last_became_peered":"2023-10-01T09:21:03.362558+0000","last_unstale":"2023-10-01T09:21:03.501064+0000","last_undegraded":"2023-10-01T09:21:03.134122+0000","last_fullsized":"2023-10-01T09:21:03.501064+0000","mapping_epoch":208,"log_start":"186'474","ondisk_log_start":"186'474","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5434,"log_dups_size":0,"ondisk_log_size":5434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:03:07.180667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387186688,"num_objects":5908,"num_object_clones":0,"num_object_copies":17724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5631,"num_objects_misplaced":277,"num_objects_unfound":0,"num_objects_dirty":5908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5908,"num_write_kb":378112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,4],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,4,6","objects":277},{"shards":"1,6","objects":5631}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":305,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.107273+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:21:02.107273+0000","last_peered":"2023-10-01T09:21:02.107273+0000","last_clean":"2023-10-01T09:21:02.107273+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:21:02.107273+0000","last_undegraded":"2023-10-01T09:21:02.107273+0000","last_fullsized":"2023-10-01T09:21:02.107273+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":174,"reported_epoch":208,"state":"active+undersized","last_fresh":"2023-10-01T09:21:02.116168+0000","last_change":"2023-10-01T09:20:36.511145+0000","last_active":"2023-10-01T09:21:02.116168+0000","last_peered":"2023-10-01T09:21:02.116168+0000","last_clean":"2023-10-01T09:20:28.116617+0000","last_became_active":"2023-10-01T09:20:36.511145+0000","last_became_peered":"2023-10-01T09:20:36.511145+0000","last_unstale":"2023-10-01T09:21:02.116168+0000","last_undegraded":"2023-10-01T09:21:02.116168+0000","last_fullsized":"2023-10-01T09:20:36.507550+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:12:49.301931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":210,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.112044+0000","last_change":"2023-10-01T09:19:30.091481+0000","last_active":"2023-10-01T09:21:02.112044+0000","last_peered":"2023-10-01T09:21:02.112044+0000","last_clean":"2023-10-01T09:21:02.112044+0000","last_became_active":"2023-10-01T09:19:30.091151+0000","last_became_peered":"2023-10-01T09:19:30.091151+0000","last_unstale":"2023-10-01T09:21:02.112044+0000","last_undegraded":"2023-10-01T09:21:02.112044+0000","last_fullsized":"2023-10-01T09:21:02.112044+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:09.127408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"209'5892","reported_seq":5945,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:21:03.508823+0000","last_change":"2023-10-01T09:21:02.250464+0000","last_active":"2023-10-01T09:21:03.508823+0000","last_peered":"2023-10-01T09:21:03.508823+0000","last_clean":"2023-10-01T09:20:49.504348+0000","last_became_active":"2023-10-01T09:21:02.245328+0000","last_became_peered":"2023-10-01T09:21:02.245328+0000","last_unstale":"2023-10-01T09:21:03.508823+0000","last_undegraded":"2023-10-01T09:21:02.128036+0000","last_fullsized":"2023-10-01T09:21:02.110597+0000","mapping_epoch":207,"log_start":"186'444","ondisk_log_start":"186'444","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5448,"log_dups_size":0,"ondisk_log_size":5448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:13:02.021176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386138112,"num_objects":5892,"num_object_clones":0,"num_object_copies":17676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":289,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5892,"num_write_kb":377088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,2","objects":289},{"shards":"1,2","objects":5603}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":569,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.114521+0000","last_change":"2023-10-01T09:10:58.797964+0000","last_active":"2023-10-01T09:21:02.114521+0000","last_peered":"2023-10-01T09:21:02.114521+0000","last_clean":"2023-10-01T09:21:02.114521+0000","last_became_active":"2023-10-01T09:10:58.797447+0000","last_became_peered":"2023-10-01T09:10:58.797447+0000","last_unstale":"2023-10-01T09:21:02.114521+0000","last_undegraded":"2023-10-01T09:21:02.114521+0000","last_fullsized":"2023-10-01T09:21:02.114521+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":197,"reported_epoch":208,"state":"active+undersized","last_fresh":"2023-10-01T09:21:02.127449+0000","last_change":"2023-10-01T09:20:36.513234+0000","last_active":"2023-10-01T09:21:02.127449+0000","last_peered":"2023-10-01T09:21:02.127449+0000","last_clean":"2023-10-01T09:20:28.304048+0000","last_became_active":"2023-10-01T09:20:36.513234+0000","last_became_peered":"2023-10-01T09:20:36.513234+0000","last_unstale":"2023-10-01T09:21:02.127449+0000","last_undegraded":"2023-10-01T09:21:02.127449+0000","last_fullsized":"2023-10-01T09:20:36.508669+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:53:27.896564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":150,"reported_epoch":208,"state":"active+undersized","last_fresh":"2023-10-01T09:21:02.115285+0000","last_change":"2023-10-01T09:20:36.511317+0000","last_active":"2023-10-01T09:21:02.115285+0000","last_peered":"2023-10-01T09:21:02.115285+0000","last_clean":"2023-10-01T09:20:28.115996+0000","last_became_active":"2023-10-01T09:20:36.511317+0000","last_became_peered":"2023-10-01T09:20:36.511317+0000","last_unstale":"2023-10-01T09:21:02.115285+0000","last_undegraded":"2023-10-01T09:21:02.115285+0000","last_fullsized":"2023-10-01T09:20:36.507026+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:25.829214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"208'6077","reported_seq":6142,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.243597+0000","last_change":"2023-10-01T09:21:02.130556+0000","last_active":"2023-10-01T09:21:02.243597+0000","last_peered":"2023-10-01T09:21:02.243597+0000","last_clean":"2023-10-01T09:21:02.243597+0000","last_became_active":"2023-10-01T09:21:02.124268+0000","last_became_peered":"2023-10-01T09:21:02.124268+0000","last_unstale":"2023-10-01T09:21:02.243597+0000","last_undegraded":"2023-10-01T09:21:02.243597+0000","last_fullsized":"2023-10-01T09:21:02.243597+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":6077,"log_dups_size":0,"ondisk_log_size":6077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:16:11.483474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398262272,"num_objects":6077,"num_object_clones":0,"num_object_copies":18231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6077,"num_write_kb":388928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":197,"reported_epoch":208,"state":"active+undersized+degraded","last_fresh":"2023-10-01T09:21:02.108084+0000","last_change":"2023-10-01T09:20:36.518571+0000","last_active":"2023-10-01T09:21:02.108084+0000","last_peered":"2023-10-01T09:21:02.108084+0000","last_clean":"2023-10-01T09:20:28.127360+0000","last_became_active":"2023-10-01T09:20:36.518571+0000","last_became_peered":"2023-10-01T09:20:36.518571+0000","last_unstale":"2023-10-01T09:21:02.108084+0000","last_undegraded":"2023-10-01T09:20:36.515187+0000","last_fullsized":"2023-10-01T09:20:36.515081+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:04:31.470403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":648,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:21:03.357556+0000","last_change":"2023-10-01T09:21:03.357697+0000","last_active":"2023-10-01T09:21:03.357556+0000","last_peered":"2023-10-01T09:21:03.357556+0000","last_clean":"2023-10-01T09:20:53.174296+0000","last_became_active":"2023-10-01T09:21:03.357290+0000","last_became_peered":"2023-10-01T09:21:03.357290+0000","last_unstale":"2023-10-01T09:21:03.357556+0000","last_undegraded":"2023-10-01T09:21:03.125276+0000","last_fullsized":"2023-10-01T09:21:03.124738+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:57:21.247636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,4],"avail_no_missing":["1","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":105,"reported_epoch":208,"state":"active+undersized","last_fresh":"2023-10-01T09:21:02.119580+0000","last_change":"2023-10-01T09:20:36.515693+0000","last_active":"2023-10-01T09:21:02.119580+0000","last_peered":"2023-10-01T09:21:02.119580+0000","last_clean":"2023-10-01T09:20:28.115718+0000","last_became_active":"2023-10-01T09:20:36.515693+0000","last_became_peered":"2023-10-01T09:20:36.515693+0000","last_unstale":"2023-10-01T09:21:02.119580+0000","last_undegraded":"2023-10-01T09:21:02.119580+0000","last_fullsized":"2023-10-01T09:20:36.506652+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:06:45.896132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.3","version":"209'5979","reported_seq":6223,"reported_epoch":209,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-01T09:21:06.411256+0000","last_change":"2023-10-01T09:21:02.265006+0000","last_active":"2023-10-01T09:21:06.411256+0000","last_peered":"2023-10-01T09:21:06.411256+0000","last_clean":"2023-10-01T09:20:49.504903+0000","last_became_active":"2023-10-01T09:21:02.141128+0000","last_became_peered":"2023-10-01T09:21:02.141128+0000","last_unstale":"2023-10-01T09:21:06.411256+0000","last_undegraded":"2023-10-01T09:21:06.411256+0000","last_fullsized":"2023-10-01T09:21:02.107418+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5979,"log_dups_size":0,"ondisk_log_size":5979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391839744,"num_objects":5979,"num_object_clones":0,"num_object_copies":17937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":187,"num_objects_unfound":0,"num_objects_dirty":5979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5979,"num_write_kb":382656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":95,"num_bytes_recovered":6225920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"1,2,7","objects":187},{"shards":"2,7","objects":5792}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":233,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T09:20:53.162333+0000","last_change":"2023-10-01T09:20:50.569651+0000","last_active":"2023-10-01T09:20:53.162333+0000","last_peered":"2023-10-01T09:20:53.162333+0000","last_clean":"2023-10-01T09:20:53.162333+0000","last_became_active":"2023-10-01T09:20:50.555642+0000","last_became_peered":"2023-10-01T09:20:50.555642+0000","last_unstale":"2023-10-01T09:20:53.162333+0000","last_undegraded":"2023-10-01T09:20:53.162333+0000","last_fullsized":"2023-10-01T09:20:53.162333+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:52:06.393935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":189,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.111967+0000","last_change":"2023-10-01T09:19:05.565472+0000","last_active":"2023-10-01T09:21:02.111967+0000","last_peered":"2023-10-01T09:21:02.111967+0000","last_clean":"2023-10-01T09:21:02.111967+0000","last_became_active":"2023-10-01T09:19:05.561161+0000","last_became_peered":"2023-10-01T09:19:05.561161+0000","last_unstale":"2023-10-01T09:21:02.111967+0000","last_undegraded":"2023-10-01T09:21:02.111967+0000","last_fullsized":"2023-10-01T09:21:02.111967+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:50:29.273388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":15,"reported_epoch":205,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:20:52.261683+0000","last_change":"2023-10-01T09:20:51.551363+0000","last_active":"2023-10-01T09:20:52.261683+0000","last_peered":"2023-10-01T09:20:52.261683+0000","last_clean":"2023-10-01T09:20:52.261683+0000","last_became_active":"2023-10-01T09:20:51.549631+0000","last_became_peered":"2023-10-01T09:20:51.549631+0000","last_unstale":"2023-10-01T09:20:52.261683+0000","last_undegraded":"2023-10-01T09:20:52.261683+0000","last_fullsized":"2023-10-01T09:20:52.261683+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:38:00.045385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,6],"avail_no_missing":["0","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":148,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.115583+0000","last_change":"2023-10-01T09:10:58.801161+0000","last_active":"2023-10-01T09:21:02.115583+0000","last_peered":"2023-10-01T09:21:02.115583+0000","last_clean":"2023-10-01T09:21:02.115583+0000","last_became_active":"2023-10-01T09:10:58.800794+0000","last_became_peered":"2023-10-01T09:10:58.800794+0000","last_unstale":"2023-10-01T09:21:02.115583+0000","last_undegraded":"2023-10-01T09:21:02.115583+0000","last_fullsized":"2023-10-01T09:21:02.115583+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:35.019445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":225,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.127552+0000","last_change":"2023-10-01T09:19:30.092736+0000","last_active":"2023-10-01T09:21:02.127552+0000","last_peered":"2023-10-01T09:21:02.127552+0000","last_clean":"2023-10-01T09:21:02.127552+0000","last_became_active":"2023-10-01T09:19:30.092334+0000","last_became_peered":"2023-10-01T09:19:30.092334+0000","last_unstale":"2023-10-01T09:21:02.127552+0000","last_undegraded":"2023-10-01T09:21:02.127552+0000","last_fullsized":"2023-10-01T09:21:02.127552+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:13:21.787586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":30,"reported_epoch":208,"state":"active+undersized+degraded","last_fresh":"2023-10-01T09:21:02.127634+0000","last_change":"2023-10-01T09:20:36.626453+0000","last_active":"2023-10-01T09:21:02.127634+0000","last_peered":"2023-10-01T09:21:02.127634+0000","last_clean":"2023-10-01T09:03:05.237925+0000","last_became_active":"2023-10-01T09:20:36.626453+0000","last_became_peered":"2023-10-01T09:20:36.626453+0000","last_unstale":"2023-10-01T09:21:02.127634+0000","last_undegraded":"2023-10-01T09:20:36.510883+0000","last_fullsized":"2023-10-01T09:20:36.510781+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:30.976322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":153,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.107230+0000","last_change":"2023-10-01T09:19:05.553200+0000","last_active":"2023-10-01T09:21:02.107230+0000","last_peered":"2023-10-01T09:21:02.107230+0000","last_clean":"2023-10-01T09:21:02.107230+0000","last_became_active":"2023-10-01T09:19:05.552874+0000","last_became_peered":"2023-10-01T09:19:05.552874+0000","last_unstale":"2023-10-01T09:21:02.107230+0000","last_undegraded":"2023-10-01T09:21:02.107230+0000","last_fullsized":"2023-10-01T09:21:02.107230+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:14:31.040887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":19,"reported_epoch":206,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:20:53.153216+0000","last_change":"2023-10-01T09:20:51.629824+0000","last_active":"2023-10-01T09:20:53.153216+0000","last_peered":"2023-10-01T09:20:53.153216+0000","last_clean":"2023-10-01T09:20:53.153216+0000","last_became_active":"2023-10-01T09:20:51.569125+0000","last_became_peered":"2023-10-01T09:20:51.569125+0000","last_unstale":"2023-10-01T09:20:53.153216+0000","last_undegraded":"2023-10-01T09:20:53.153216+0000","last_fullsized":"2023-10-01T09:20:53.153216+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:51:41.214083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,6],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":135,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.106936+0000","last_change":"2023-10-01T09:18:54.533869+0000","last_active":"2023-10-01T09:21:02.106936+0000","last_peered":"2023-10-01T09:21:02.106936+0000","last_clean":"2023-10-01T09:21:02.106936+0000","last_became_active":"2023-10-01T09:18:54.533542+0000","last_became_peered":"2023-10-01T09:18:54.533542+0000","last_unstale":"2023-10-01T09:21:02.106936+0000","last_undegraded":"2023-10-01T09:21:02.106936+0000","last_fullsized":"2023-10-01T09:21:02.106936+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:18:19.126440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":117,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.145473+0000","last_change":"2023-10-01T09:21:02.138726+0000","last_active":"2023-10-01T09:21:02.145473+0000","last_peered":"2023-10-01T09:21:02.145473+0000","last_clean":"2023-10-01T09:21:02.145473+0000","last_became_active":"2023-10-01T09:21:02.138365+0000","last_became_peered":"2023-10-01T09:21:02.138365+0000","last_unstale":"2023-10-01T09:21:02.145473+0000","last_undegraded":"2023-10-01T09:21:02.145473+0000","last_fullsized":"2023-10-01T09:21:02.145473+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:57:42.151867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":205,"reported_epoch":208,"state":"active+undersized","last_fresh":"2023-10-01T09:21:02.121239+0000","last_change":"2023-10-01T09:20:36.508489+0000","last_active":"2023-10-01T09:21:02.121239+0000","last_peered":"2023-10-01T09:21:02.121239+0000","last_clean":"2023-10-01T09:20:28.116723+0000","last_became_active":"2023-10-01T09:20:36.508489+0000","last_became_peered":"2023-10-01T09:20:36.508489+0000","last_unstale":"2023-10-01T09:21:02.121239+0000","last_undegraded":"2023-10-01T09:21:02.121239+0000","last_fullsized":"2023-10-01T09:20:36.506135+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:57:45.525939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":229,"reported_epoch":208,"state":"active+undersized+degraded","last_fresh":"2023-10-01T09:21:02.113747+0000","last_change":"2023-10-01T09:20:36.508033+0000","last_active":"2023-10-01T09:21:02.113747+0000","last_peered":"2023-10-01T09:21:02.113747+0000","last_clean":"2023-10-01T09:20:28.116139+0000","last_became_active":"2023-10-01T09:20:36.508033+0000","last_became_peered":"2023-10-01T09:20:36.508033+0000","last_unstale":"2023-10-01T09:21:02.113747+0000","last_undegraded":"2023-10-01T09:20:36.506233+0000","last_fullsized":"2023-10-01T09:20:36.506111+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:55:00.830503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":129,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T09:21:02.142442+0000","last_change":"2023-10-01T09:21:02.141180+0000","last_active":"2023-10-01T09:21:02.142442+0000","last_peered":"2023-10-01T09:21:02.142442+0000","last_clean":"2023-10-01T09:21:02.142442+0000","last_became_active":"2023-10-01T09:21:02.136796+0000","last_became_peered":"2023-10-01T09:21:02.136796+0000","last_unstale":"2023-10-01T09:21:02.142442+0000","last_undegraded":"2023-10-01T09:21:02.142442+0000","last_fullsized":"2023-10-01T09:21:02.142442+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:03:46.040197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6239354880,"num_objects":95205,"num_object_clones":0,"num_object_copies":285615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24868,"num_objects_misplaced":6338,"num_objects_unfound":0,"num_objects_dirty":95205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":95205,"num_write_kb":6093120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":301,"num_bytes_recovered":19726336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18355585024,"data_stored":18355585024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93139,"ondisk_log_size":93139,"up":43,"acting":38,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":81,"acting":89,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":6,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":12002,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":90112,"data_stored":11848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27349,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":83,"acting":90,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":22,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430446,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2211856,"kb_used_data":1910536,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91533296,"statfs":{"total":95995035648,"available":93730095104,"internally_reserved":0,"allocated":1956388864,"data_stored":1955832414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:20:13 2023","interfaces":[{"interface":"back","average":{"1min":5.1200000000000001,"5min":6.9109999999999996,"15min":3.0129999999999999},"min":{"1min":0.23400000000000001,"5min":0.219,"15min":0.219},"max":{"1min":44.353000000000002,"5min":117.855,"15min":117.855},"last":0.34799999999999998},{"interface":"front","average":{"1min":5.0010000000000003,"5min":6.3129999999999997,"15min":2.8010000000000002},"min":{"1min":0.26700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":43.240000000000002,"5min":120.348,"15min":120.348},"last":0.27600000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:20:24 2023","interfaces":[{"interface":"back","average":{"1min":9.7330000000000005,"5min":4.5430000000000001,"15min":1.9399999999999999},"min":{"1min":0.25600000000000001,"5min":0.23400000000000001,"15min":0.22700000000000001},"max":{"1min":105.045,"5min":105.045,"15min":105.045},"last":0.251},{"interface":"front","average":{"1min":8.8390000000000004,"5min":4.0300000000000002,"15min":1.768},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":104.71599999999999,"5min":104.71599999999999,"15min":104.71599999999999},"last":0.36499999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:20:51 2023","interfaces":[{"interface":"back","average":{"1min":3.6549999999999998,"5min":4.2729999999999997,"15min":2.0720000000000001},"min":{"1min":0.32600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":40.921999999999997,"5min":104.837,"15min":104.837},"last":0.30599999999999999},{"interface":"front","average":{"1min":3.6139999999999999,"5min":3.964,"15min":1.994},"min":{"1min":0.29199999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":40.912999999999997,"5min":106.379,"15min":106.379},"last":0.317}]},{"osd":3,"last update":"Sun Oct 1 09:20:24 2023","interfaces":[{"interface":"back","average":{"1min":6.6189999999999998,"5min":3.2189999999999999,"15min":1.462},"min":{"1min":0.34399999999999997,"5min":0.31900000000000001,"15min":0.26400000000000001},"max":{"1min":104.797,"5min":104.797,"15min":104.797},"last":0.40699999999999997},{"interface":"front","average":{"1min":6.8449999999999998,"5min":3.2090000000000001,"15min":1.4390000000000001},"min":{"1min":0.33400000000000002,"5min":0.29599999999999999,"15min":0.27200000000000002},"max":{"1min":104.774,"5min":104.774,"15min":104.774},"last":0.379}]},{"osd":4,"last update":"Sun Oct 1 09:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":3.02,"15min":1.3260000000000001},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.2349999999999994,"5min":82.992000000000004,"15min":82.992000000000004},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.68799999999999994,"5min":3.101,"15min":1.3720000000000001},"min":{"1min":0.30099999999999999,"5min":0.252,"15min":0.252},"max":{"1min":3.3039999999999998,"5min":83.058999999999997,"15min":83.058999999999997},"last":0.44700000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:20:13 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":2.7320000000000002,"15min":1.2509999999999999},"min":{"1min":0.29799999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.9379999999999999,"5min":90.656999999999996,"15min":90.656999999999996},"last":3.113},{"interface":"front","average":{"1min":0.70299999999999996,"5min":3.5179999999999998,"15min":1.548},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.655,"5min":90.747,"15min":90.747},"last":2.7349999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:20:13 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":1.9790000000000001,"15min":1.0329999999999999},"min":{"1min":0.371,"5min":0.34699999999999998,"15min":0.29999999999999999},"max":{"1min":2.5880000000000001,"5min":11.685,"15min":11.685},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.86499999999999999,"5min":1.734,"15min":0.96399999999999997},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.718,"5min":11.871,"15min":11.871},"last":0.43099999999999999}]}]},{"osd":6,"up_from":0,"seq":180388626674,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":94,"seq":403726925934,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2134408,"kb_used_data":1972288,"kb_used_omap":7,"kb_used_meta":162104,"kb_avail":91610744,"statfs":{"total":95995035648,"available":93809401856,"internally_reserved":0,"allocated":2019622912,"data_stored":2018724576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":165994919},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:20:23 2023","interfaces":[{"interface":"back","average":{"1min":16.550000000000001,"5min":6.8970000000000002,"15min":2.5419999999999998},"min":{"1min":0.26400000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":134.84999999999999,"5min":174.40199999999999,"15min":174.40199999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":16.459,"5min":7.0549999999999997,"15min":2.6219999999999999},"min":{"1min":0.252,"5min":0.19400000000000001,"15min":0.188},"max":{"1min":134.77099999999999,"5min":173.77000000000001,"15min":173.77000000000001},"last":0.29499999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:20:59 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.897,"15min":0.97299999999999998},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":5.1509999999999998,"5min":25.632000000000001,"15min":25.632000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.708,"5min":2.0350000000000001,"15min":1.042},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":6.0529999999999999,"5min":29.425999999999998,"15min":29.425999999999998},"last":0.371}]},{"osd":3,"last update":"Sun Oct 1 09:20:34 2023","interfaces":[{"interface":"back","average":{"1min":9.1259999999999994,"5min":3.2090000000000001,"15min":1.3580000000000001},"min":{"1min":0.224,"5min":0.222,"15min":0.218},"max":{"1min":99.421000000000006,"5min":99.421000000000006,"15min":99.421000000000006},"last":0.33300000000000002},{"interface":"front","average":{"1min":8.6199999999999992,"5min":2.8079999999999998,"15min":1.282},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":99.364999999999995,"5min":99.364999999999995,"15min":99.364999999999995},"last":0.47799999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:20:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":2.0059999999999998,"15min":1.069},"min":{"1min":0.30299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.4130000000000003,"5min":40.673999999999999,"15min":40.673999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.3260000000000001,"5min":2.3740000000000001,"15min":1.1910000000000001},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.5549999999999997,"5min":40.810000000000002,"15min":40.810000000000002},"last":0.437}]},{"osd":5,"last update":"Sun Oct 1 09:20:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":5.1340000000000003,"15min":2.1339999999999999},"min":{"1min":0.34499999999999997,"5min":0.313,"15min":0.313},"max":{"1min":6.351,"5min":282.04700000000003,"15min":282.04700000000003},"last":5.6500000000000004},{"interface":"front","average":{"1min":0.97499999999999998,"5min":4.9740000000000002,"15min":2.081},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.097,"5min":281.89499999999998,"15min":281.89499999999998},"last":2.613}]},{"osd":6,"last update":"Sun Oct 1 09:20:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.8280000000000001,"15min":1.0409999999999999},"min":{"1min":0.438,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.4669999999999996,"5min":9.7449999999999992,"15min":9.7449999999999992},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1.476,"15min":0.93700000000000006},"min":{"1min":0.45800000000000002,"5min":0.372,"15min":0.372},"max":{"1min":2.032,"5min":10.787000000000001,"15min":10.787000000000001},"last":0.45400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:20:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":2.0680000000000001,"15min":1.1140000000000001},"min":{"1min":0.39800000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":4.3769999999999998,"5min":23.710999999999999,"15min":23.710999999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.1639999999999999,"5min":2.1440000000000001,"15min":1.131},"min":{"1min":0.438,"5min":0.372,"15min":0.372},"max":{"1min":6.0780000000000003,"5min":23.736999999999998,"15min":23.736999999999998},"last":0.54500000000000004}]}]},{"osd":0,"up_from":89,"seq":382252089468,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1881948,"kb_used_data":1588180,"kb_used_omap":8,"kb_used_meta":293751,"kb_avail":91863204,"statfs":{"total":95995035648,"available":94067920896,"internally_reserved":0,"allocated":1626296320,"data_stored":1625382642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":300801986},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":3.2050000000000001,"5min":2.2050000000000001,"15min":1.006},"min":{"1min":0.23400000000000001,"5min":0.187,"15min":0.187},"max":{"1min":50.643999999999998,"5min":50.643999999999998,"15min":50.643999999999998},"last":0.28599999999999998},{"interface":"front","average":{"1min":3.347,"5min":2.5089999999999999,"15min":1.1020000000000001},"min":{"1min":0.21099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":50.923999999999999,"5min":50.923999999999999,"15min":50.923999999999999},"last":0.40300000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:20:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7509999999999999,"5min":2.1739999999999999,"15min":1.0980000000000001},"min":{"1min":0.25900000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":31.024999999999999,"5min":31.024999999999999,"15min":31.024999999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":3.7799999999999998,"5min":2.823,"15min":1.2749999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":31.157,"5min":31.157,"15min":31.157},"last":0.45500000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.756,"15min":0.88800000000000001},"min":{"1min":0.317,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":5.6609999999999996,"5min":12.906000000000001,"15min":12.906000000000001},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.4079999999999999,"5min":2.1869999999999998,"15min":0.98799999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.23400000000000001},"max":{"1min":5.3239999999999998,"5min":17.318000000000001,"15min":17.318000000000001},"last":0.443}]},{"osd":4,"last update":"Sun Oct 1 09:20:20 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":3.6699999999999999,"15min":1.6499999999999999},"min":{"1min":0.32000000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.2290000000000001,"5min":112.52200000000001,"15min":112.52200000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.79400000000000004,"5min":3.976,"15min":1.7649999999999999},"min":{"1min":0.36899999999999999,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":1.7789999999999999,"5min":113.124,"15min":113.124},"last":0.38200000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:20:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":3.6819999999999999,"15min":1.679},"min":{"1min":0.33200000000000002,"5min":0.32700000000000001,"15min":0.311},"max":{"1min":3.645,"5min":98.730999999999995,"15min":98.730999999999995},"last":4.5720000000000001},{"interface":"front","average":{"1min":0.752,"5min":3.343,"15min":1.6060000000000001},"min":{"1min":0.40600000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.3439999999999999,"5min":98.870999999999995,"15min":98.870999999999995},"last":3.1309999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:20:20 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":2.5830000000000002,"15min":1.325},"min":{"1min":0.39400000000000002,"5min":0.35899999999999999,"15min":0.33200000000000002},"max":{"1min":2.1179999999999999,"5min":34.375999999999998,"15min":34.375999999999998},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.81799999999999995,"5min":2.0209999999999999,"15min":1.159},"min":{"1min":0.42799999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.464,"5min":31.745000000000001,"15min":31.745000000000001},"last":0.42499999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:20:20 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":3.536,"15min":1.6799999999999999},"min":{"1min":0.45000000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":2.0859999999999999,"5min":98.283000000000001,"15min":98.283000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.81200000000000006,"5min":3.5979999999999999,"15min":1.6719999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.1419999999999999,"5min":98.311999999999998,"15min":98.311999999999998},"last":0.51700000000000002}]}]},{"osd":2,"up_from":99,"seq":425201762407,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3342564,"kb_used_data":3090268,"kb_used_omap":11,"kb_used_meta":252276,"kb_avail":90402588,"statfs":{"total":95995035648,"available":92572250112,"internally_reserved":0,"allocated":3164434432,"data_stored":3163570020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":258331511},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:20:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":2.9980000000000002,"15min":1.5509999999999999},"min":{"1min":0.23200000000000001,"5min":0.20100000000000001,"15min":0.188},"max":{"1min":8.1639999999999997,"5min":93.634,"15min":93.634},"last":0.437},{"interface":"front","average":{"1min":1.4450000000000001,"5min":3.5640000000000001,"15min":1.837},"min":{"1min":0.247,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":7.1589999999999998,"5min":93.906999999999996,"15min":93.906999999999996},"last":0.34599999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:20:58 2023","interfaces":[{"interface":"back","average":{"1min":9.077,"5min":3.5430000000000001,"15min":1.8440000000000001},"min":{"1min":0.27000000000000002,"5min":0.23699999999999999,"15min":0.20999999999999999},"max":{"1min":105.798,"5min":105.798,"15min":105.798},"last":0.36099999999999999},{"interface":"front","average":{"1min":9.0310000000000006,"5min":3.7490000000000001,"15min":1.9930000000000001},"min":{"1min":0.29299999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":105.56399999999999,"5min":105.56399999999999,"15min":107.729},"last":0.39400000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":15.981999999999999,"5min":4.8579999999999997,"15min":1.8839999999999999},"min":{"1min":0.23999999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":253.238,"5min":253.238,"15min":253.238},"last":0.29399999999999998},{"interface":"front","average":{"1min":15.813000000000001,"5min":4.6970000000000001,"15min":1.8660000000000001},"min":{"1min":0.27600000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":252.54900000000001,"5min":252.54900000000001,"15min":252.54900000000001},"last":0.41299999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:20:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":2.0499999999999998,"15min":1.1339999999999999},"min":{"1min":0.32700000000000001,"5min":0.30599999999999999,"15min":0.27000000000000002},"max":{"1min":7.1260000000000003,"5min":17.751999999999999,"15min":17.751999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.0019999999999998,"5min":2.2229999999999999,"15min":1.194},"min":{"1min":0.32500000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.9649999999999999,"5min":17.568000000000001,"15min":17.568000000000001},"last":0.38600000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:19:57 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":2.7549999999999999,"15min":1.3069999999999999},"min":{"1min":0.38,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.002000000000001,"5min":34.084000000000003,"15min":34.084000000000003},"last":1.7030000000000001},{"interface":"front","average":{"1min":1.6459999999999999,"5min":2.7989999999999999,"15min":1.3180000000000001},"min":{"1min":0.436,"5min":0.316,"15min":0.316},"max":{"1min":7.5620000000000003,"5min":34.341999999999999,"15min":34.341999999999999},"last":1.2789999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:20:58 2023","interfaces":[{"interface":"back","average":{"1min":1.8999999999999999,"5min":2.335,"15min":1.24},"min":{"1min":0.35399999999999998,"5min":0.33900000000000002,"15min":0.33200000000000002},"max":{"1min":5.4509999999999996,"5min":17.591000000000001,"15min":17.591000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":2.1339999999999999,"5min":1.637,"15min":0.98999999999999999},"min":{"1min":0.375,"5min":0.36599999999999999,"15min":0.34200000000000003},"max":{"1min":8.0389999999999997,"5min":17.736000000000001,"15min":17.736000000000001},"last":0.46999999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:20:58 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":3.1789999999999998,"15min":1.5209999999999999},"min":{"1min":0.40699999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.0820000000000007,"5min":109.506,"15min":109.506},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.776,"5min":3.4220000000000002,"15min":1.653},"min":{"1min":0.375,"5min":0.375,"15min":0.36399999999999999},"max":{"1min":7.1769999999999996,"5min":109.58799999999999,"15min":109.58799999999999},"last":0.44600000000000001}]}]},{"osd":3,"up_from":104,"seq":446676598881,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2024480,"kb_used_data":1906840,"kb_used_omap":3,"kb_used_meta":117628,"kb_avail":91720672,"statfs":{"total":95995035648,"available":93921968128,"internally_reserved":0,"allocated":1952604160,"data_stored":1951740751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":120451815},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":3.7999999999999998,"15min":2.6629999999999998},"min":{"1min":0.189,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":38.314,"5min":99.034000000000006,"15min":199.422},"last":0.34000000000000002},{"interface":"front","average":{"1min":2.5609999999999999,"5min":4.2469999999999999,"15min":2.794},"min":{"1min":0.23400000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":38.118000000000002,"5min":98.602999999999994,"15min":199.91200000000001},"last":0.26300000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.9140000000000001,"5min":3.2559999999999998,"15min":1.8640000000000001},"min":{"1min":0.23300000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":41.369,"5min":135.006,"15min":154.32599999999999},"last":0.374},{"interface":"front","average":{"1min":2.677,"5min":3.0489999999999999,"15min":1.8},"min":{"1min":0.23899999999999999,"5min":0.215,"15min":0.215},"max":{"1min":39.979999999999997,"5min":134.358,"15min":154.249},"last":0.27800000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":4.8099999999999996,"5min":2.3570000000000002,"15min":1.0640000000000001},"min":{"1min":0.28399999999999997,"5min":0.23000000000000001,"15min":0.224},"max":{"1min":55.238999999999997,"5min":55.238999999999997,"15min":55.238999999999997},"last":0.313},{"interface":"front","average":{"1min":5.391,"5min":2.798,"15min":1.234},"min":{"1min":0.30499999999999999,"5min":0.215,"15min":0.20399999999999999},"max":{"1min":55.692999999999998,"5min":55.692999999999998,"15min":55.692999999999998},"last":0.54000000000000004}]},{"osd":4,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.7130000000000001,"5min":3.9980000000000002,"15min":1.736},"min":{"1min":0.29499999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":40.078000000000003,"5min":154.00399999999999,"15min":154.00399999999999},"last":0.505},{"interface":"front","average":{"1min":2.7810000000000001,"5min":4.3860000000000001,"15min":1.8839999999999999},"min":{"1min":0.30099999999999999,"5min":0.248,"15min":0.248},"max":{"1min":40.177,"5min":156.87700000000001,"15min":156.87700000000001},"last":0.35199999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.8980000000000001,"5min":3.3239999999999998,"15min":1.5289999999999999},"min":{"1min":0.32000000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":40.215000000000003,"5min":76.893000000000001,"15min":76.893000000000001},"last":3.298},{"interface":"front","average":{"1min":2.7029999999999998,"5min":3.1760000000000002,"15min":1.4930000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":38.43,"5min":76.959999999999994,"15min":76.959999999999994},"last":5.1379999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.9510000000000001,"5min":2.7130000000000001,"15min":1.351},"min":{"1min":0.38100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":40.124000000000002,"5min":40.124000000000002,"15min":40.124000000000002},"last":0.29999999999999999},{"interface":"front","average":{"1min":2.754,"5min":2.21,"15min":1.216},"min":{"1min":0.32600000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":38.459000000000003,"5min":38.459000000000003,"15min":38.459000000000003},"last":0.42499999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.677,"5min":2.6829999999999998,"15min":1.3600000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":38.478999999999999,"5min":79.412999999999997,"15min":79.412999999999997},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.7210000000000001,"5min":2.7839999999999998,"15min":1.379},"min":{"1min":0.36399999999999999,"5min":0.308,"15min":0.308},"max":{"1min":40.039999999999999,"5min":79.527000000000001,"15min":79.527000000000001},"last":0.56200000000000006}]}]},{"osd":4,"up_from":30,"seq":128849019131,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2260820,"kb_used_data":1951308,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91484332,"statfs":{"total":95995035648,"available":93679955968,"internally_reserved":0,"allocated":1998139392,"data_stored":1997578888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:20:14 2023","interfaces":[{"interface":"back","average":{"1min":9.5329999999999995,"5min":8.2140000000000004,"15min":3.2349999999999999},"min":{"1min":0.36299999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":111.27800000000001,"5min":133.91499999999999,"15min":133.91499999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":9.6349999999999998,"5min":7.8209999999999997,"15min":3.1019999999999999},"min":{"1min":0.28899999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":111.289,"5min":133.83199999999999,"15min":133.83199999999999},"last":0.34999999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:20:26 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":3.3420000000000001,"15min":1.5189999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.2380000000000004,"5min":133.93600000000001,"15min":133.93600000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.82199999999999995,"5min":3.8290000000000002,"15min":1.6579999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.2290000000000001,"5min":141.06,"15min":141.06},"last":0.48699999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:20:58 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":3.1099999999999999,"15min":2.2519999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.3900000000000001,"5min":133.96100000000001,"15min":133.96100000000001},"last":0.438},{"interface":"front","average":{"1min":1.375,"5min":2.9780000000000002,"15min":2.1930000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.4870000000000001,"5min":134.018,"15min":134.018},"last":0.39500000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:20:26 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":3.1459999999999999,"15min":1.431},"min":{"1min":0.32100000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.9909999999999997,"5min":134.297,"15min":134.297},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.879,"5min":3.5510000000000002,"15min":1.579},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.1320000000000001,"5min":141.089,"15min":141.089},"last":0.44900000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:20:26 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":4.1459999999999999,"15min":1.714},"min":{"1min":0.26500000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.0219999999999998,"5min":140.99100000000001,"15min":140.99100000000001},"last":4.298},{"interface":"front","average":{"1min":0.59499999999999997,"5min":3.0710000000000002,"15min":1.3879999999999999},"min":{"1min":0.318,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.5219999999999998,"5min":133.863,"15min":133.863},"last":4.3490000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:20:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":2.6309999999999998,"15min":1.2689999999999999},"min":{"1min":0.36899999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.806,"5min":133.90799999999999,"15min":133.90799999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.5669999999999999,"5min":3.3980000000000001,"15min":1.55},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.3250000000000002,"5min":141.03,"15min":141.03},"last":0.52700000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:21:06 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":3.3119999999999998,"15min":1.528},"min":{"1min":0.32400000000000001,"5min":0.31,"15min":0.31},"max":{"1min":4.399,"5min":133.946,"15min":133.946},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.054,"5min":3.048,"15min":1.4550000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.31},"max":{"1min":3.9569999999999999,"5min":133.98400000000001,"15min":133.98400000000001},"last":0.499}]}]},{"osd":5,"up_from":36,"seq":154618822897,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1769736,"kb_used_data":1460224,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91975416,"statfs":{"total":95995035648,"available":94182825984,"internally_reserved":0,"allocated":1495269376,"data_stored":1494746488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:20:15 2023","interfaces":[{"interface":"back","average":{"1min":4.7629999999999999,"5min":4.0609999999999999,"15min":1.843},"min":{"1min":0.253,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":82.219999999999999,"5min":82.219999999999999,"15min":82.219999999999999},"last":0.379},{"interface":"front","average":{"1min":4.79,"5min":4.0599999999999996,"15min":1.845},"min":{"1min":0.254,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":82.227999999999994,"5min":82.227999999999994,"15min":82.227999999999994},"last":0.27400000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:20:17 2023","interfaces":[{"interface":"back","average":{"1min":5.0110000000000001,"5min":4.3789999999999996,"15min":1.9359999999999999},"min":{"1min":0.26200000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":82.585999999999999,"5min":82.873000000000005,"15min":82.873000000000005},"last":1.276},{"interface":"front","average":{"1min":4.6970000000000001,"5min":3.9769999999999999,"15min":1.8100000000000001},"min":{"1min":0.35299999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":82.344999999999999,"5min":82.853999999999999,"15min":82.853999999999999},"last":0.42399999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:19:56 2023","interfaces":[{"interface":"back","average":{"1min":11.343,"5min":4.9340000000000002,"15min":2.0859999999999999},"min":{"1min":0.33300000000000002,"5min":0.25,"15min":0.25},"max":{"1min":139.56399999999999,"5min":139.56399999999999,"15min":139.56399999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":11.606999999999999,"5min":4.3529999999999998,"15min":1.863},"min":{"1min":0.30499999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":139.785,"5min":139.785,"15min":139.785},"last":0.39100000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:20:26 2023","interfaces":[{"interface":"back","average":{"1min":19.617000000000001,"5min":6.5869999999999997,"15min":2.7719999999999998},"min":{"1min":0.377,"5min":0.31,"15min":0.31},"max":{"1min":225.46199999999999,"5min":225.46199999999999,"15min":225.46199999999999},"last":1.605},{"interface":"front","average":{"1min":19.053000000000001,"5min":5.9530000000000003,"15min":2.4470000000000001},"min":{"1min":0.34100000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":225.40100000000001,"5min":225.40100000000001,"15min":225.40100000000001},"last":0.48599999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:20:15 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":4.056,"15min":1.6759999999999999},"min":{"1min":0.30499999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.25,"5min":193.26300000000001,"15min":193.26300000000001},"last":0.31},{"interface":"front","average":{"1min":0.72899999999999998,"5min":2.48,"15min":1.1879999999999999},"min":{"1min":0.34000000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.4820000000000002,"5min":15.432,"15min":15.432},"last":0.26100000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:19:40 2023","interfaces":[{"interface":"back","average":{"1min":2.302,"5min":2.669,"15min":1.246},"min":{"1min":0.317,"5min":0.30299999999999999,"15min":0.28599999999999998},"max":{"1min":13.015000000000001,"5min":15.551,"15min":15.551},"last":14.878},{"interface":"front","average":{"1min":3.5190000000000001,"5min":2.8559999999999999,"15min":1.3280000000000001},"min":{"1min":0.35199999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":16.548999999999999,"5min":16.548999999999999,"15min":16.548999999999999},"last":16.759}]},{"osd":7,"last update":"Sun Oct 1 09:20:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":4.4779999999999998,"15min":1.8839999999999999},"min":{"1min":0.37,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":9.9060000000000006,"5min":193.13800000000001,"15min":193.13800000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.74399999999999999,"5min":3.911,"15min":1.6990000000000001},"min":{"1min":0.36499999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.1779999999999999,"5min":193.203,"15min":193.203},"last":0.35299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1619132416,"data_stored":1619132416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2012479488,"data_stored":2012479488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3161522176,"data_stored":3161522176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1950285824,"data_stored":1950285824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1996161024,"data_stored":1996161024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1498480640,"data_stored":1498480640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4167303168,"data_stored":4167303168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1950220288,"data_stored":1950220288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:21:12.019 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 1 2023-10-01T09:21:12.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:12.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:12.122 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:12.148 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:12.195 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:12.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:12.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:12.250 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:12 smithi138 ceph-mon[185886]: from='client.35172 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:21:12.250 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:12 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2104833897' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:12.250 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:12 smithi138 ceph-mon[185886]: pgmap v526: 89 pgs: 1 activating+remapped, 1 remapped+peering, 15 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 46 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 4.1 MiB/s wr, 65 op/s; 24886/285717 objects degraded (8.710%); 6345/285717 objects misplaced (2.221%); 740 KiB/s, 12 objects/s recovering 2023-10-01T09:21:12.251 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:12 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 24886/285699 objects degraded (8.711%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:12.251 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:12 smithi138 ceph-mon[192513]: from='client.35172 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:21:12.251 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:12 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2104833897' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:12.251 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:12 smithi138 ceph-mon[192513]: pgmap v526: 89 pgs: 1 activating+remapped, 1 remapped+peering, 15 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 46 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 4.1 MiB/s wr, 65 op/s; 24886/285717 objects degraded (8.710%); 6345/285717 objects misplaced (2.221%); 740 KiB/s, 12 objects/s recovering 2023-10-01T09:21:12.251 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:12 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 24886/285699 objects degraded (8.711%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:12.301 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:12.327 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:12.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:12 smithi190 ceph-mon[160435]: from='client.35172 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:21:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:12 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2104833897' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:12 smithi190 ceph-mon[160435]: pgmap v526: 89 pgs: 1 activating+remapped, 1 remapped+peering, 15 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 46 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 4.1 MiB/s wr, 65 op/s; 24886/285717 objects degraded (8.710%); 6345/285717 objects misplaced (2.221%); 740 KiB/s, 12 objects/s recovering 2023-10-01T09:21:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:12 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 24886/285699 objects degraded (8.711%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:12.575 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:12.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:12.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:12.678 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:12.712 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:12.750 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:12.817 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:12.825 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:12.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:12.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:12.929 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:12.960 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:12.997 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 89 16 95366 95350 66.927 0 - 0.0132219 2023-10-01T09:21:13.106 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:21:13.107 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:21:13.107 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-10-01T09:21:13.107 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:21:13.428 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:13.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:13.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:13.535 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:13.563 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:13.613 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:13.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:13.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:13.719 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:13.744 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:13.791 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:13.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:13.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:13.898 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:13.923 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:13.966 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:13.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:13.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:13.997 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 90 16 95366 95350 66.1836 0 - 0.0132219 2023-10-01T09:21:14.071 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:14.101 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:14.616 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:21:14.634 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:21:14.635 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:21:14.635 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:21:14.635 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:21:14.635 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:21:14.635 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:21:14.635 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-01T09:21:14.635 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:21:14.635 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:21:14.778 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:14 smithi138 ceph-mon[192513]: pgmap v527: 89 pgs: 1 activating+remapped, 1 remapped+peering, 15 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 46 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 1.2 MiB/s wr, 18 op/s; 24886/285717 objects degraded (8.710%); 6228/285717 objects misplaced (2.180%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:21:14.778 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:14 smithi138 ceph-mon[185886]: pgmap v527: 89 pgs: 1 activating+remapped, 1 remapped+peering, 15 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 46 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 1.2 MiB/s wr, 18 op/s; 24886/285717 objects degraded (8.710%); 6228/285717 objects misplaced (2.180%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:21:14.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:14 smithi190 ceph-mon[160435]: pgmap v527: 89 pgs: 1 activating+remapped, 1 remapped+peering, 15 active+clean+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 1 active+recovery_wait+degraded+remapped, 46 active+clean; 5.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 1.2 MiB/s wr, 18 op/s; 24886/285717 objects degraded (8.710%); 6228/285717 objects misplaced (2.180%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:21:14.906 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:14.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:14.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:14.998 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Total time run: 90.8606 2023-10-01T09:21:14.998 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Total writes made: 95366 2023-10-01T09:21:14.998 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Write size: 65536 2023-10-01T09:21:14.998 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Object size: 65536 2023-10-01T09:21:14.998 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Bandwidth (MB/sec): 65.5991 2023-10-01T09:21:14.998 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev Bandwidth: 47.1765 2023-10-01T09:21:14.998 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max bandwidth (MB/sec): 202.25 2023-10-01T09:21:14.998 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min bandwidth (MB/sec): 0 2023-10-01T09:21:14.998 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Average IOPS: 1049 2023-10-01T09:21:14.999 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev IOPS: 754.832 2023-10-01T09:21:14.999 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max IOPS: 3236 2023-10-01T09:21:14.999 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min IOPS: 0 2023-10-01T09:21:14.999 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Average Latency(s): 0.0152303 2023-10-01T09:21:14.999 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev Latency(s): 0.161895 2023-10-01T09:21:14.999 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max latency(s): 13.4227 2023-10-01T09:21:14.999 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min latency(s): 0.00099931 2023-10-01T09:21:15.005 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T09:21:15.009 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:15.035 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:21:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:21:14] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.33.4" 2023-10-01T09:21:15.039 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:15.316 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:15.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:15.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:15.419 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:21:15.431 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:15.558 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:15.880 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:15.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:15.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:15.900 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:21:15.901 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:21:16.000 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:16.072 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:16.158 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:16.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:16.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:16.212 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:16.263 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:16.297 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:16.360 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:16.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:16.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:16.398 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T09:21:16.463 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:16.489 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:16.521 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:21:16.541 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:16.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:16.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:16.646 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:16.687 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[192513]: pgmap v528: 89 pgs: 1 activating+degraded+remapped, 2 activating+remapped, 12 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 2 active+recovering+undersized+remapped, 10 active+undersized, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 47 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 1.3 MiB/s wr, 21 op/s; 30257/286158 objects degraded (10.574%); 1242/286158 objects misplaced (0.434%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:21:16.687 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[192513]: osdmap e211: 8 total, 7 up, 8 in 2023-10-01T09:21:16.687 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[192513]: failed to encode map e211 with expected crc 2023-10-01T09:21:16.688 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[192513]: failed to encode map e211 with expected crc 2023-10-01T09:21:16.688 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2278912373' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:16.688 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:16.688 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3166875788' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:21:16.688 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:21:16.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[185886]: pgmap v528: 89 pgs: 1 activating+degraded+remapped, 2 activating+remapped, 12 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 2 active+recovering+undersized+remapped, 10 active+undersized, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 47 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 1.3 MiB/s wr, 21 op/s; 30257/286158 objects degraded (10.574%); 1242/286158 objects misplaced (0.434%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:21:16.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[185886]: osdmap e211: 8 total, 7 up, 8 in 2023-10-01T09:21:16.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[185886]: failed to encode map e211 with expected crc 2023-10-01T09:21:16.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[185886]: failed to encode map e211 with expected crc 2023-10-01T09:21:16.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2278912373' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:16.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:16.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3166875788' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:21:16.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:16 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:21:16.690 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:16 smithi190 ceph-mon[160435]: pgmap v528: 89 pgs: 1 activating+degraded+remapped, 2 activating+remapped, 12 active+clean+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 2 active+recovering+undersized+remapped, 10 active+undersized, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 47 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 1.3 MiB/s wr, 21 op/s; 30257/286158 objects degraded (10.574%); 1242/286158 objects misplaced (0.434%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:21:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:16 smithi190 ceph-mon[160435]: osdmap e211: 8 total, 7 up, 8 in 2023-10-01T09:21:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:16 smithi190 ceph-mon[160435]: failed to encode map e211 with expected crc 2023-10-01T09:21:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:16 smithi190 ceph-mon[160435]: failed to encode map e211 with expected crc 2023-10-01T09:21:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:16 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2278912373' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:16 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:16 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3166875788' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:21:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:16 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:21:17.662 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:17.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:17.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:17.767 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:17.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:17 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:17.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:17 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:21:17.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:17 smithi138 ceph-mon[192513]: osdmap e212: 8 total, 7 up, 8 in 2023-10-01T09:21:17.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:17 smithi138 ceph-mon[192513]: failed to encode map e212 with expected crc 2023-10-01T09:21:17.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:17 smithi138 ceph-mon[192513]: failed to encode map e212 with expected crc 2023-10-01T09:21:17.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:17 smithi138 ceph-mon[192513]: failed to encode map e212 with expected crc 2023-10-01T09:21:17.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:17 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 30257/286158 objects degraded (10.574%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:17.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:17 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:17.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:17 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:21:17.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:17 smithi138 ceph-mon[185886]: osdmap e212: 8 total, 7 up, 8 in 2023-10-01T09:21:17.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:17 smithi138 ceph-mon[185886]: failed to encode map e212 with expected crc 2023-10-01T09:21:17.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:17 smithi138 ceph-mon[185886]: failed to encode map e212 with expected crc 2023-10-01T09:21:17.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:17 smithi138 ceph-mon[185886]: failed to encode map e212 with expected crc 2023-10-01T09:21:17.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:17 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 30257/286158 objects degraded (10.574%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:17.794 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:17 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:17 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:21:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:17 smithi190 ceph-mon[160435]: osdmap e212: 8 total, 7 up, 8 in 2023-10-01T09:21:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:17 smithi190 ceph-mon[160435]: failed to encode map e212 with expected crc 2023-10-01T09:21:17.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:17 smithi190 ceph-mon[160435]: failed to encode map e212 with expected crc 2023-10-01T09:21:17.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:17 smithi190 ceph-mon[160435]: failed to encode map e212 with expected crc 2023-10-01T09:21:17.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:17 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 30257/286158 objects degraded (10.574%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:17.901 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:21:17.946 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T09:21:18.080 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:18.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:18.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:18.184 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:18.209 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:18.256 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:18.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:18.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:18.364 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:18.392 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:18.436 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:18.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:18.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:18.540 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:18.593 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:18.684 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:18.686 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:18 smithi138 ceph-mon[192513]: pgmap v531: 89 pgs: 1 peering, 1 activating+degraded+remapped, 2 activating+remapped, 12 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 2 active+recovering+undersized+remapped, 10 active+undersized, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 47 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 1.2 MiB/s wr, 68 op/s; 29888/284982 objects degraded (10.488%); 1305/284982 objects misplaced (0.458%); 3.0 MiB/s, 47 objects/s recovering 2023-10-01T09:21:18.686 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:18 smithi138 ceph-mon[192513]: failed to encode map e213 with expected crc 2023-10-01T09:21:18.686 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:18 smithi138 ceph-mon[192513]: osdmap e213: 8 total, 7 up, 8 in 2023-10-01T09:21:18.687 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:18 smithi138 ceph-mon[192513]: failed to encode map e213 with expected crc 2023-10-01T09:21:18.687 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:18 smithi138 ceph-mon[192513]: failed to encode map e213 with expected crc 2023-10-01T09:21:18.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:18.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:18.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:18 smithi138 ceph-mon[185886]: pgmap v531: 89 pgs: 1 peering, 1 activating+degraded+remapped, 2 activating+remapped, 12 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 2 active+recovering+undersized+remapped, 10 active+undersized, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 47 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 1.2 MiB/s wr, 68 op/s; 29888/284982 objects degraded (10.488%); 1305/284982 objects misplaced (0.458%); 3.0 MiB/s, 47 objects/s recovering 2023-10-01T09:21:18.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:18 smithi138 ceph-mon[185886]: failed to encode map e213 with expected crc 2023-10-01T09:21:18.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:18 smithi138 ceph-mon[185886]: osdmap e213: 8 total, 7 up, 8 in 2023-10-01T09:21:18.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:18 smithi138 ceph-mon[185886]: failed to encode map e213 with expected crc 2023-10-01T09:21:18.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:18 smithi138 ceph-mon[185886]: failed to encode map e213 with expected crc 2023-10-01T09:21:18.788 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:18.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:18 smithi190 ceph-mon[160435]: pgmap v531: 89 pgs: 1 peering, 1 activating+degraded+remapped, 2 activating+remapped, 12 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 2 active+recovering+undersized+remapped, 10 active+undersized, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 47 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 1.2 MiB/s wr, 68 op/s; 29888/284982 objects degraded (10.488%); 1305/284982 objects misplaced (0.458%); 3.0 MiB/s, 47 objects/s recovering 2023-10-01T09:21:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:18 smithi190 ceph-mon[160435]: failed to encode map e213 with expected crc 2023-10-01T09:21:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:18 smithi190 ceph-mon[160435]: osdmap e213: 8 total, 7 up, 8 in 2023-10-01T09:21:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:18 smithi190 ceph-mon[160435]: failed to encode map e213 with expected crc 2023-10-01T09:21:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:18 smithi190 ceph-mon[160435]: failed to encode map e213 with expected crc 2023-10-01T09:21:18.826 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:19.043 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:19.069 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:19.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:19.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:19.173 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:19.202 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:19.313 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:19.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:19.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:19.418 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:19.450 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:19.619 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:19.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:19.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:19.718 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:21:19.718 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:21:19.718 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:21:19.718 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:21:19.718 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:21:19.725 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:19.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:19 smithi138 ceph-mon[192513]: failed to encode map e214 with expected crc 2023-10-01T09:21:19.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:19 smithi138 ceph-mon[192513]: osdmap e214: 8 total, 7 up, 8 in 2023-10-01T09:21:19.746 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:19 smithi138 ceph-mon[192513]: failed to encode map e214 with expected crc 2023-10-01T09:21:19.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:19 smithi138 ceph-mon[185886]: failed to encode map e214 with expected crc 2023-10-01T09:21:19.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:19 smithi138 ceph-mon[185886]: osdmap e214: 8 total, 7 up, 8 in 2023-10-01T09:21:19.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:19 smithi138 ceph-mon[185886]: failed to encode map e214 with expected crc 2023-10-01T09:21:19.765 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:19 smithi190 ceph-mon[160435]: failed to encode map e214 with expected crc 2023-10-01T09:21:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:19 smithi190 ceph-mon[160435]: osdmap e214: 8 total, 7 up, 8 in 2023-10-01T09:21:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:19 smithi190 ceph-mon[160435]: failed to encode map e214 with expected crc 2023-10-01T09:21:20.184 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:21:20.533 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:20 smithi138 ceph-mon[185886]: pgmap v534: 89 pgs: 1 peering, 1 activating+degraded+remapped, 1 activating+remapped, 13 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 3 active+recovering+undersized+remapped, 10 active+undersized, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 47 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 165 op/s; 29459/284178 objects degraded (10.366%); 1554/284178 objects misplaced (0.547%); 5.6 MiB/s, 89 objects/s recovering 2023-10-01T09:21:20.533 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:20 smithi138 ceph-mon[185886]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.534 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:20 smithi138 ceph-mon[185886]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.534 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:20 smithi138 ceph-mon[185886]: osdmap e215: 8 total, 7 up, 8 in 2023-10-01T09:21:20.534 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:20 smithi138 ceph-mon[185886]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.534 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:20 smithi138 ceph-mon[185886]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.534 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:20 smithi138 ceph-mon[185886]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.541 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:20.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:20.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:20.648 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:20.715 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:20.804 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:20 smithi190 ceph-mon[160435]: pgmap v534: 89 pgs: 1 peering, 1 activating+degraded+remapped, 1 activating+remapped, 13 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 3 active+recovering+undersized+remapped, 10 active+undersized, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 47 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 165 op/s; 29459/284178 objects degraded (10.366%); 1554/284178 objects misplaced (0.547%); 5.6 MiB/s, 89 objects/s recovering 2023-10-01T09:21:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:20 smithi190 ceph-mon[160435]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:20 smithi190 ceph-mon[160435]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:20 smithi190 ceph-mon[160435]: osdmap e215: 8 total, 7 up, 8 in 2023-10-01T09:21:20.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:20 smithi190 ceph-mon[160435]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:20 smithi190 ceph-mon[160435]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:20 smithi190 ceph-mon[160435]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:20.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:20.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:20 smithi138 ceph-mon[192513]: pgmap v534: 89 pgs: 1 peering, 1 activating+degraded+remapped, 1 activating+remapped, 13 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 3 active+recovering+undersized+remapped, 10 active+undersized, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 47 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 165 op/s; 29459/284178 objects degraded (10.366%); 1554/284178 objects misplaced (0.547%); 5.6 MiB/s, 89 objects/s recovering 2023-10-01T09:21:20.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:20 smithi138 ceph-mon[192513]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:20 smithi138 ceph-mon[192513]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:20 smithi138 ceph-mon[192513]: osdmap e215: 8 total, 7 up, 8 in 2023-10-01T09:21:20.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:20 smithi138 ceph-mon[192513]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:20 smithi138 ceph-mon[192513]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:20 smithi138 ceph-mon[192513]: failed to encode map e215 with expected crc 2023-10-01T09:21:20.916 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:20.964 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:21.046 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:21.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:21.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:21.156 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:21.218 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:21.356 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:21:21.373 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:21.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:21.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:21.488 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:21.634 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:21.654 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:21.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:21 smithi138 ceph-mon[185886]: failed to encode map e216 with expected crc 2023-10-01T09:21:21.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:21 smithi138 ceph-mon[185886]: osdmap e216: 8 total, 7 up, 8 in 2023-10-01T09:21:21.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:21 smithi138 ceph-mon[185886]: failed to encode map e216 with expected crc 2023-10-01T09:21:21.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:21 smithi138 ceph-mon[185886]: failed to encode map e216 with expected crc 2023-10-01T09:21:21.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:21 smithi138 ceph-mon[185886]: failed to encode map e216 with expected crc 2023-10-01T09:21:21.784 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:21 smithi138 ceph-mon[192513]: failed to encode map e216 with expected crc 2023-10-01T09:21:21.784 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:21 smithi138 ceph-mon[192513]: osdmap e216: 8 total, 7 up, 8 in 2023-10-01T09:21:21.784 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:21 smithi138 ceph-mon[192513]: failed to encode map e216 with expected crc 2023-10-01T09:21:21.784 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:21 smithi138 ceph-mon[192513]: failed to encode map e216 with expected crc 2023-10-01T09:21:21.784 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:21 smithi138 ceph-mon[192513]: failed to encode map e216 with expected crc 2023-10-01T09:21:21.831 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:21.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:21.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:21.941 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:22.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:21 smithi190 ceph-mon[160435]: failed to encode map e216 with expected crc 2023-10-01T09:21:22.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:21 smithi190 ceph-mon[160435]: osdmap e216: 8 total, 7 up, 8 in 2023-10-01T09:21:22.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:21 smithi190 ceph-mon[160435]: failed to encode map e216 with expected crc 2023-10-01T09:21:22.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:21 smithi190 ceph-mon[160435]: failed to encode map e216 with expected crc 2023-10-01T09:21:22.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:21 smithi190 ceph-mon[160435]: failed to encode map e216 with expected crc 2023-10-01T09:21:22.059 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:22.167 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:22.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:22.182 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:21:22.232 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:21:22.255 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:21:22.280 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:22.293 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:21:22.329 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:22.337 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:21:22.391 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:21:22.415 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:22.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:22.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:22.428 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:21:22.429 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:21:22.429 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:21:22.429 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:21:22.429 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:21:22.429 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:21:22.429 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-01T09:21:22.429 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:21:22.429 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:21:22.522 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:22.557 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:22.639 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:22.646 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:22 smithi190 ceph-mon[160435]: pgmap v537: 89 pgs: 1 peering, 15 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 2 active+recovering+undersized+remapped, 10 active+undersized, 2 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 48 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 255 B/s rd, 128 KiB/s wr, 147 op/s; 29928/283272 objects degraded (10.565%); 666/283272 objects misplaced (0.235%); 8.9 MiB/s, 142 objects/s recovering 2023-10-01T09:21:22.646 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:22 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 29928/283272 objects degraded (10.565%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:22.646 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:22 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3518710251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:22.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:22.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:22.693 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:22 smithi138 ceph-mon[185886]: pgmap v537: 89 pgs: 1 peering, 15 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 2 active+recovering+undersized+remapped, 10 active+undersized, 2 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 48 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 255 B/s rd, 128 KiB/s wr, 147 op/s; 29928/283272 objects degraded (10.565%); 666/283272 objects misplaced (0.235%); 8.9 MiB/s, 142 objects/s recovering 2023-10-01T09:21:22.693 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:22 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 29928/283272 objects degraded (10.565%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:22.693 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:22 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3518710251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:22.695 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:22 smithi138 ceph-mon[192513]: pgmap v537: 89 pgs: 1 peering, 15 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 2 active+recovering+undersized+remapped, 10 active+undersized, 2 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 48 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 255 B/s rd, 128 KiB/s wr, 147 op/s; 29928/283272 objects degraded (10.565%); 666/283272 objects misplaced (0.235%); 8.9 MiB/s, 142 objects/s recovering 2023-10-01T09:21:22.695 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:22 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 29928/283272 objects degraded (10.565%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:22.695 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:22 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3518710251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:22.748 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:22.822 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:22.948 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 7, 2, 6] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2023-10-01T09:21:22.948 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:21:22.948 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2023-10-01T09:21:22.948 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:21:24.169 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:24.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:24.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:24.243 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:24.282 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:24.292 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:21:24.293 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:21:24.374 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:24.537 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:24.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:24.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:24.631 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:24 smithi138 ceph-mon[185886]: pgmap v538: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 172 B/s rd, 87 KiB/s wr, 345 op/s; 29480/278910 objects degraded (10.570%); 7/278910 objects misplaced (0.003%); 15 MiB/s, 241 objects/s recovering 2023-10-01T09:21:24.635 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:24 smithi138 ceph-mon[192513]: pgmap v538: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 172 B/s rd, 87 KiB/s wr, 345 op/s; 29480/278910 objects degraded (10.570%); 7/278910 objects misplaced (0.003%); 15 MiB/s, 241 objects/s recovering 2023-10-01T09:21:24.644 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:24.702 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:24.777 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:24.786 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:24.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:24.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:24.889 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:24.955 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:21:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:21:24] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.33.4" 2023-10-01T09:21:24.974 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:25.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:24 smithi190 ceph-mon[160435]: pgmap v538: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 172 B/s rd, 87 KiB/s wr, 345 op/s; 29480/278910 objects degraded (10.570%); 7/278910 objects misplaced (0.003%); 15 MiB/s, 241 objects/s recovering 2023-10-01T09:21:25.531 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:25.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:25.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:25.578 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:21:25.636 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:25.674 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:25.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:25 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2381553288' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:21:25.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:25 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:21:25.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:21:25.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:25 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2381553288' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:21:25.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:25 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:21:25.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:21:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:25 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2381553288' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:21:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:25 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:21:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:21:26.295 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:21:26.558 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:26.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:26.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:26.664 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:26.698 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:26 smithi138 ceph-mon[192513]: pgmap v539: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 74 KiB/s wr, 365 op/s; 29103/276639 objects degraded (10.520%); 7/276639 objects misplaced (0.003%); 15 MiB/s, 245 objects/s recovering 2023-10-01T09:21:26.698 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:26 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:26.698 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:26 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:21:26.698 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:26 smithi138 ceph-mon[192513]: osdmap e217: 8 total, 7 up, 8 in 2023-10-01T09:21:26.698 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:26 smithi138 ceph-mon[192513]: failed to encode map e217 with expected crc 2023-10-01T09:21:26.698 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:26 smithi138 ceph-mon[192513]: failed to encode map e217 with expected crc 2023-10-01T09:21:26.700 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:26 smithi138 ceph-mon[185886]: pgmap v539: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 74 KiB/s wr, 365 op/s; 29103/276639 objects degraded (10.520%); 7/276639 objects misplaced (0.003%); 15 MiB/s, 245 objects/s recovering 2023-10-01T09:21:26.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:26 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:26.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:26 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:21:26.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:26 smithi138 ceph-mon[185886]: osdmap e217: 8 total, 7 up, 8 in 2023-10-01T09:21:26.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:26 smithi138 ceph-mon[185886]: failed to encode map e217 with expected crc 2023-10-01T09:21:26.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:26 smithi138 ceph-mon[185886]: failed to encode map e217 with expected crc 2023-10-01T09:21:26.703 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:27.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:26 smithi190 ceph-mon[160435]: pgmap v539: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 74 KiB/s wr, 365 op/s; 29103/276639 objects degraded (10.520%); 7/276639 objects misplaced (0.003%); 15 MiB/s, 245 objects/s recovering 2023-10-01T09:21:27.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:26 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:27.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:26 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:21:27.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:26 smithi190 ceph-mon[160435]: osdmap e217: 8 total, 7 up, 8 in 2023-10-01T09:21:27.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:26 smithi190 ceph-mon[160435]: failed to encode map e217 with expected crc 2023-10-01T09:21:27.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:26 smithi190 ceph-mon[160435]: failed to encode map e217 with expected crc 2023-10-01T09:21:27.532 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:27.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:27.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:27.624 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:27 smithi138 ceph-mon[185886]: failed to encode map e217 with expected crc 2023-10-01T09:21:27.624 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:27 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 29103/276639 objects degraded (10.520%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:27.624 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:27 smithi138 ceph-mon[192513]: failed to encode map e217 with expected crc 2023-10-01T09:21:27.625 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:27 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 29103/276639 objects degraded (10.520%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:27.654 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:27.698 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:27.789 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:27.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:27.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:27.894 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:27.986 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:28.047 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:21:28.047 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:21:28.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:27 smithi190 ceph-mon[160435]: failed to encode map e217 with expected crc 2023-10-01T09:21:28.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:27 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 29103/276639 objects degraded (10.520%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:28.089 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:28.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:28.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:28.194 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:28.233 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:28.306 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:28.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:28.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:28.416 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:28.490 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:28.589 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:28.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:28.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:28.664 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:21:28.670 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:21:28.696 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:28.697 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:28 smithi138 ceph-mon[185886]: pgmap v541: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 KiB/s wr, 529 op/s; 28894/271653 objects degraded (10.636%); 7/271653 objects misplaced (0.003%); 13 MiB/s, 214 objects/s recovering 2023-10-01T09:21:28.698 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:28 smithi138 ceph-mon[192513]: pgmap v541: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 KiB/s wr, 529 op/s; 28894/271653 objects degraded (10.636%); 7/271653 objects misplaced (0.003%); 13 MiB/s, 214 objects/s recovering 2023-10-01T09:21:28.725 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:28.798 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:28.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:28.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:28.905 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:28.936 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:28 smithi190 ceph-mon[160435]: pgmap v541: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 KiB/s wr, 529 op/s; 28894/271653 objects degraded (10.636%); 7/271653 objects misplaced (0.003%); 13 MiB/s, 214 objects/s recovering 2023-10-01T09:21:28.945 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:29.022 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:29.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:29.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:29.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:29.175 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:29.375 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:21:29.616 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:29.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:29.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:29.645 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:29.682 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:21:29.723 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:29.768 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:29.841 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:29.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:29.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:29.916 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":541,"stamp":"2023-10-01T09:21:27.437896+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5937240011,"num_objects":90551,"num_object_clones":0,"num_object_copies":271653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":28894,"num_objects_misplaced":7,"num_objects_unfound":0,"num_objects_dirty":90551,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":101317,"num_write_kb":6118566,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2642,"num_bytes_recovered":164181333,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99315,"ondisk_log_size":99315,"up":233,"acting":250,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":292,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19337784,"kb_used_data":17263928,"kb_used_omap":30,"kb_used_meta":2073633,"kb_avail":730623432,"statfs":{"total":767960285184,"available":748158394368,"internally_reserved":0,"allocated":17678262272,"data_stored":17672244331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30777,"internal_metadata":2123401159},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-273612800,"num_objects":-4175,"num_object_clones":0,"num_object_copies":-12525,"num_objects_missing_on_primary":-1,"num_objects_missing":-1,"num_objects_degraded":-565,"num_objects_misplaced":-1547,"num_objects_unfound":0,"num_objects_dirty":-4175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4191,"num_write_kb":512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1699,"num_bytes_recovered":111084064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.915042"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":85,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.525561+0000","last_change":"2023-10-01T09:19:30.091981+0000","last_active":"2023-10-01T09:21:19.525561+0000","last_peered":"2023-10-01T09:21:19.525561+0000","last_clean":"2023-10-01T09:21:19.525561+0000","last_became_active":"2023-10-01T09:19:30.091531+0000","last_became_peered":"2023-10-01T09:19:30.091531+0000","last_unstale":"2023-10-01T09:21:19.525561+0000","last_undegraded":"2023-10-01T09:21:19.525561+0000","last_fullsized":"2023-10-01T09:21:19.525561+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:28:13.442498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":56,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.529974+0000","last_change":"2023-10-01T09:21:02.131395+0000","last_active":"2023-10-01T09:21:19.529974+0000","last_peered":"2023-10-01T09:21:19.529974+0000","last_clean":"2023-10-01T09:21:19.529974+0000","last_became_active":"2023-10-01T09:21:02.131066+0000","last_became_peered":"2023-10-01T09:21:02.131066+0000","last_unstale":"2023-10-01T09:21:19.529974+0000","last_undegraded":"2023-10-01T09:21:19.529974+0000","last_fullsized":"2023-10-01T09:21:19.529974+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:40:12.021772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":161,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.527493+0000","last_change":"2023-10-01T09:13:13.551596+0000","last_active":"2023-10-01T09:21:19.527493+0000","last_peered":"2023-10-01T09:21:19.527493+0000","last_clean":"2023-10-01T09:21:19.527493+0000","last_became_active":"2023-10-01T09:13:13.551300+0000","last_became_peered":"2023-10-01T09:13:13.551300+0000","last_unstale":"2023-10-01T09:21:19.527493+0000","last_undegraded":"2023-10-01T09:21:19.527493+0000","last_fullsized":"2023-10-01T09:21:19.527493+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:57:45.081780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":192,"reported_epoch":216,"state":"active+undersized","last_fresh":"2023-10-01T09:21:20.533761+0000","last_change":"2023-10-01T09:20:36.515604+0000","last_active":"2023-10-01T09:21:20.533761+0000","last_peered":"2023-10-01T09:21:20.533761+0000","last_clean":"2023-10-01T09:20:28.116381+0000","last_became_active":"2023-10-01T09:20:36.515604+0000","last_became_peered":"2023-10-01T09:20:36.515604+0000","last_unstale":"2023-10-01T09:21:20.533761+0000","last_undegraded":"2023-10-01T09:21:20.533761+0000","last_fullsized":"2023-10-01T09:20:36.507014+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:19:39.031820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":204,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.534498+0000","last_change":"2023-10-01T09:19:30.091032+0000","last_active":"2023-10-01T09:21:20.534498+0000","last_peered":"2023-10-01T09:21:20.534498+0000","last_clean":"2023-10-01T09:21:20.534498+0000","last_became_active":"2023-10-01T09:19:30.090644+0000","last_became_peered":"2023-10-01T09:19:30.090644+0000","last_unstale":"2023-10-01T09:21:20.534498+0000","last_undegraded":"2023-10-01T09:21:20.534498+0000","last_fullsized":"2023-10-01T09:21:20.534498+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:19.617722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":152,"reported_epoch":215,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:19.535004+0000","last_change":"2023-10-01T09:21:03.353296+0000","last_active":"2023-10-01T09:21:19.535004+0000","last_peered":"2023-10-01T09:21:19.535004+0000","last_clean":"2023-10-01T09:21:19.535004+0000","last_became_active":"2023-10-01T09:21:03.352862+0000","last_became_peered":"2023-10-01T09:21:03.352862+0000","last_unstale":"2023-10-01T09:21:19.535004+0000","last_undegraded":"2023-10-01T09:21:19.535004+0000","last_fullsized":"2023-10-01T09:21:19.535004+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,0],"avail_no_missing":["6","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":202,"reported_epoch":216,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:20.533566+0000","last_change":"2023-10-01T09:21:03.355091+0000","last_active":"2023-10-01T09:21:20.533566+0000","last_peered":"2023-10-01T09:21:20.533566+0000","last_clean":"2023-10-01T09:21:20.533566+0000","last_became_active":"2023-10-01T09:21:03.354777+0000","last_became_peered":"2023-10-01T09:21:03.354777+0000","last_unstale":"2023-10-01T09:21:20.533566+0000","last_undegraded":"2023-10-01T09:21:20.533566+0000","last_fullsized":"2023-10-01T09:21:20.533566+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:22:37.229201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":206,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.529474+0000","last_change":"2023-10-01T09:21:02.139865+0000","last_active":"2023-10-01T09:21:20.529474+0000","last_peered":"2023-10-01T09:21:20.529474+0000","last_clean":"2023-10-01T09:21:20.529474+0000","last_became_active":"2023-10-01T09:21:02.139507+0000","last_became_peered":"2023-10-01T09:21:02.139507+0000","last_unstale":"2023-10-01T09:21:20.529474+0000","last_undegraded":"2023-10-01T09:21:20.529474+0000","last_fullsized":"2023-10-01T09:21:20.529474+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:46:09.214220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":176,"reported_epoch":215,"state":"active+undersized","last_fresh":"2023-10-01T09:21:19.526937+0000","last_change":"2023-10-01T09:20:36.511477+0000","last_active":"2023-10-01T09:21:19.526937+0000","last_peered":"2023-10-01T09:21:19.526937+0000","last_clean":"2023-10-01T09:19:41.977469+0000","last_became_active":"2023-10-01T09:20:36.511477+0000","last_became_peered":"2023-10-01T09:20:36.511477+0000","last_unstale":"2023-10-01T09:21:19.526937+0000","last_undegraded":"2023-10-01T09:21:19.526937+0000","last_fullsized":"2023-10-01T09:20:36.508783+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:16:03.885738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":213,"reported_epoch":216,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:20.529108+0000","last_change":"2023-10-01T09:21:03.236933+0000","last_active":"2023-10-01T09:21:20.529108+0000","last_peered":"2023-10-01T09:21:20.529108+0000","last_clean":"2023-10-01T09:21:20.529108+0000","last_became_active":"2023-10-01T09:21:03.236508+0000","last_became_peered":"2023-10-01T09:21:03.236508+0000","last_unstale":"2023-10-01T09:21:20.529108+0000","last_undegraded":"2023-10-01T09:21:20.529108+0000","last_fullsized":"2023-10-01T09:21:20.529108+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:58:15.332216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,0],"avail_no_missing":["3","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":62,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.530683+0000","last_change":"2023-10-01T09:19:44.369992+0000","last_active":"2023-10-01T09:21:20.530683+0000","last_peered":"2023-10-01T09:21:20.530683+0000","last_clean":"2023-10-01T09:21:20.530683+0000","last_became_active":"2023-10-01T09:19:44.346816+0000","last_became_peered":"2023-10-01T09:19:44.346816+0000","last_unstale":"2023-10-01T09:21:20.530683+0000","last_undegraded":"2023-10-01T09:21:20.530683+0000","last_fullsized":"2023-10-01T09:21:20.530683+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:44:40.345845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":215,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.525018+0000","last_change":"2023-10-01T09:21:02.158958+0000","last_active":"2023-10-01T09:21:19.525018+0000","last_peered":"2023-10-01T09:21:19.525018+0000","last_clean":"2023-10-01T09:21:19.525018+0000","last_became_active":"2023-10-01T09:21:02.158491+0000","last_became_peered":"2023-10-01T09:21:02.158491+0000","last_unstale":"2023-10-01T09:21:19.525018+0000","last_undegraded":"2023-10-01T09:21:19.525018+0000","last_fullsized":"2023-10-01T09:21:19.525018+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:21:48.885982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":162,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.532442+0000","last_change":"2023-10-01T09:21:02.163403+0000","last_active":"2023-10-01T09:21:20.532442+0000","last_peered":"2023-10-01T09:21:20.532442+0000","last_clean":"2023-10-01T09:21:20.532442+0000","last_became_active":"2023-10-01T09:21:02.121680+0000","last_became_peered":"2023-10-01T09:21:02.121680+0000","last_unstale":"2023-10-01T09:21:20.532442+0000","last_undegraded":"2023-10-01T09:21:20.532442+0000","last_fullsized":"2023-10-01T09:21:20.532442+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":184,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.534892+0000","last_change":"2023-10-01T09:21:03.349475+0000","last_active":"2023-10-01T09:21:19.534892+0000","last_peered":"2023-10-01T09:21:19.534892+0000","last_clean":"2023-10-01T09:21:19.534892+0000","last_became_active":"2023-10-01T09:21:03.348251+0000","last_became_peered":"2023-10-01T09:21:03.348251+0000","last_unstale":"2023-10-01T09:21:19.534892+0000","last_undegraded":"2023-10-01T09:21:19.534892+0000","last_fullsized":"2023-10-01T09:21:19.534892+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":245,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.532549+0000","last_change":"2023-10-01T09:19:30.182046+0000","last_active":"2023-10-01T09:21:20.532549+0000","last_peered":"2023-10-01T09:21:20.532549+0000","last_clean":"2023-10-01T09:21:20.532549+0000","last_became_active":"2023-10-01T09:19:30.094579+0000","last_became_peered":"2023-10-01T09:19:30.094579+0000","last_unstale":"2023-10-01T09:21:20.532549+0000","last_undegraded":"2023-10-01T09:21:20.532549+0000","last_fullsized":"2023-10-01T09:21:20.532549+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":197,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.529475+0000","last_change":"2023-10-01T09:19:57.493670+0000","last_active":"2023-10-01T09:21:20.529475+0000","last_peered":"2023-10-01T09:21:20.529475+0000","last_clean":"2023-10-01T09:21:20.529475+0000","last_became_active":"2023-10-01T09:19:57.493349+0000","last_became_peered":"2023-10-01T09:19:57.493349+0000","last_unstale":"2023-10-01T09:21:20.529475+0000","last_undegraded":"2023-10-01T09:21:20.529475+0000","last_fullsized":"2023-10-01T09:21:20.529475+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:26:56.468779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":185,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.536004+0000","last_change":"2023-10-01T09:19:44.354776+0000","last_active":"2023-10-01T09:21:20.536004+0000","last_peered":"2023-10-01T09:21:20.536004+0000","last_clean":"2023-10-01T09:21:20.536004+0000","last_became_active":"2023-10-01T09:19:44.354447+0000","last_became_peered":"2023-10-01T09:19:44.354447+0000","last_unstale":"2023-10-01T09:21:20.536004+0000","last_undegraded":"2023-10-01T09:21:20.536004+0000","last_fullsized":"2023-10-01T09:21:20.536004+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:53.120763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":88,"reported_epoch":216,"state":"active+undersized","last_fresh":"2023-10-01T09:21:20.532839+0000","last_change":"2023-10-01T09:20:36.516058+0000","last_active":"2023-10-01T09:21:20.532839+0000","last_peered":"2023-10-01T09:21:20.532839+0000","last_clean":"2023-10-01T09:12:40.941890+0000","last_became_active":"2023-10-01T09:20:36.516058+0000","last_became_peered":"2023-10-01T09:20:36.516058+0000","last_unstale":"2023-10-01T09:21:20.532839+0000","last_undegraded":"2023-10-01T09:21:20.532839+0000","last_fullsized":"2023-10-01T09:20:36.510370+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":186,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.532247+0000","last_change":"2023-10-01T09:21:02.121558+0000","last_active":"2023-10-01T09:21:20.532247+0000","last_peered":"2023-10-01T09:21:20.532247+0000","last_clean":"2023-10-01T09:21:20.532247+0000","last_became_active":"2023-10-01T09:21:02.121153+0000","last_became_peered":"2023-10-01T09:21:02.121153+0000","last_unstale":"2023-10-01T09:21:20.532247+0000","last_undegraded":"2023-10-01T09:21:20.532247+0000","last_fullsized":"2023-10-01T09:21:20.532247+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":176,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.533668+0000","last_change":"2023-10-01T09:19:44.345829+0000","last_active":"2023-10-01T09:21:20.533668+0000","last_peered":"2023-10-01T09:21:20.533668+0000","last_clean":"2023-10-01T09:21:20.533668+0000","last_became_active":"2023-10-01T09:19:44.345499+0000","last_became_peered":"2023-10-01T09:19:44.345499+0000","last_unstale":"2023-10-01T09:21:20.533668+0000","last_undegraded":"2023-10-01T09:21:20.533668+0000","last_fullsized":"2023-10-01T09:21:20.533668+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:23:30.028532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":152,"reported_epoch":215,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:19.535460+0000","last_change":"2023-10-01T09:21:03.359509+0000","last_active":"2023-10-01T09:21:19.535460+0000","last_peered":"2023-10-01T09:21:19.535460+0000","last_clean":"2023-10-01T09:21:19.535460+0000","last_became_active":"2023-10-01T09:21:03.352862+0000","last_became_peered":"2023-10-01T09:21:03.352862+0000","last_unstale":"2023-10-01T09:21:19.535460+0000","last_undegraded":"2023-10-01T09:21:19.535460+0000","last_fullsized":"2023-10-01T09:21:19.535460+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,2],"avail_no_missing":["6","0","2"],"object_location_counts":[{"shards":"0,2,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":200,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.525829+0000","last_change":"2023-10-01T09:21:02.137372+0000","last_active":"2023-10-01T09:21:19.525829+0000","last_peered":"2023-10-01T09:21:19.525829+0000","last_clean":"2023-10-01T09:21:19.525829+0000","last_became_active":"2023-10-01T09:21:02.137080+0000","last_became_peered":"2023-10-01T09:21:02.137080+0000","last_unstale":"2023-10-01T09:21:19.525829+0000","last_undegraded":"2023-10-01T09:21:19.525829+0000","last_fullsized":"2023-10-01T09:21:19.525829+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:38:03.902930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.a","version":"216'6302","reported_seq":6996,"reported_epoch":216,"state":"active+recovering+degraded+remapped","last_fresh":"2023-10-01T09:21:24.616727+0000","last_change":"2023-10-01T09:21:19.279472+0000","last_active":"2023-10-01T09:21:24.616727+0000","last_peered":"2023-10-01T09:21:24.616727+0000","last_clean":"2023-10-01T09:20:35.491448+0000","last_became_active":"2023-10-01T09:21:14.646464+0000","last_became_peered":"2023-10-01T09:21:14.646464+0000","last_unstale":"2023-10-01T09:21:24.616727+0000","last_undegraded":"2023-10-01T09:21:14.563013+0000","last_fullsized":"2023-10-01T09:21:24.616727+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":6302,"log_dups_size":0,"ondisk_log_size":6302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376569856,"num_objects":5746,"num_object_clones":0,"num_object_copies":17238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5746,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6302,"num_write_kb":385536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":315,"num_bytes_recovered":20643840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,2],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,2,6","objects":356},{"shards":"0,6","objects":5390}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":60,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.533425+0000","last_change":"2023-10-01T09:19:44.373526+0000","last_active":"2023-10-01T09:21:20.533425+0000","last_peered":"2023-10-01T09:21:20.533425+0000","last_clean":"2023-10-01T09:21:20.533425+0000","last_became_active":"2023-10-01T09:19:44.354026+0000","last_became_peered":"2023-10-01T09:19:44.354026+0000","last_unstale":"2023-10-01T09:21:20.533425+0000","last_undegraded":"2023-10-01T09:21:20.533425+0000","last_fullsized":"2023-10-01T09:21:20.533425+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:01:49.696317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":163,"reported_epoch":215,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:19.535545+0000","last_change":"2023-10-01T09:21:03.359488+0000","last_active":"2023-10-01T09:21:19.535545+0000","last_peered":"2023-10-01T09:21:19.535545+0000","last_clean":"2023-10-01T09:21:19.535545+0000","last_became_active":"2023-10-01T09:21:03.354675+0000","last_became_peered":"2023-10-01T09:21:03.354675+0000","last_unstale":"2023-10-01T09:21:19.535545+0000","last_undegraded":"2023-10-01T09:21:19.535545+0000","last_fullsized":"2023-10-01T09:21:19.535545+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,1],"avail_no_missing":["6","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.b","version":"217'6353","reported_seq":7074,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-01T09:21:26.841975+0000","last_change":"2023-10-01T09:21:18.531925+0000","last_active":"2023-10-01T09:21:26.841975+0000","last_peered":"2023-10-01T09:21:26.841975+0000","last_clean":"2023-10-01T09:21:26.841975+0000","last_became_active":"2023-10-01T09:21:18.522953+0000","last_became_peered":"2023-10-01T09:21:18.522953+0000","last_unstale":"2023-10-01T09:21:26.841975+0000","last_undegraded":"2023-10-01T09:21:26.841975+0000","last_fullsized":"2023-10-01T09:21:26.841975+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":6353,"log_dups_size":0,"ondisk_log_size":6353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:30:39.062864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368771072,"num_objects":5627,"num_object_clones":0,"num_object_copies":16881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6353,"num_write_kb":383360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":319,"num_bytes_recovered":20905984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":185,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.532601+0000","last_change":"2023-10-01T09:12:10.003552+0000","last_active":"2023-10-01T09:21:20.532601+0000","last_peered":"2023-10-01T09:21:20.532601+0000","last_clean":"2023-10-01T09:21:20.532601+0000","last_became_active":"2023-10-01T09:12:10.003131+0000","last_became_peered":"2023-10-01T09:12:10.003131+0000","last_unstale":"2023-10-01T09:21:20.532601+0000","last_undegraded":"2023-10-01T09:21:20.532601+0000","last_fullsized":"2023-10-01T09:21:20.532601+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":207,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.527422+0000","last_change":"2023-10-01T09:21:02.159564+0000","last_active":"2023-10-01T09:21:19.527422+0000","last_peered":"2023-10-01T09:21:19.527422+0000","last_clean":"2023-10-01T09:21:19.527422+0000","last_became_active":"2023-10-01T09:21:02.159342+0000","last_became_peered":"2023-10-01T09:21:02.159342+0000","last_unstale":"2023-10-01T09:21:19.527422+0000","last_undegraded":"2023-10-01T09:21:19.527422+0000","last_fullsized":"2023-10-01T09:21:19.527422+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:45:04.227823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.8","version":"217'6347","reported_seq":6813,"reported_epoch":217,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:21:26.903018+0000","last_change":"2023-10-01T09:21:02.182885+0000","last_active":"2023-10-01T09:21:26.903018+0000","last_peered":"2023-10-01T09:21:26.903018+0000","last_clean":"2023-10-01T09:20:49.525699+0000","last_became_active":"2023-10-01T09:21:02.175091+0000","last_became_peered":"2023-10-01T09:21:02.175091+0000","last_unstale":"2023-10-01T09:21:26.903018+0000","last_undegraded":"2023-10-01T09:21:02.136996+0000","last_fullsized":"2023-10-01T09:21:02.106788+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":6347,"log_dups_size":0,"ondisk_log_size":6347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:16:25.782704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366804992,"num_objects":5597,"num_object_clones":0,"num_object_copies":16791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":324,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6347,"num_write_kb":382208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":203,"num_bytes_recovered":13303808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"0,2,3","objects":324},{"shards":"2,3","objects":5273}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":199,"reported_epoch":216,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:20.532586+0000","last_change":"2023-10-01T09:21:03.353156+0000","last_active":"2023-10-01T09:21:20.532586+0000","last_peered":"2023-10-01T09:21:20.532586+0000","last_clean":"2023-10-01T09:21:20.532586+0000","last_became_active":"2023-10-01T09:21:03.352649+0000","last_became_peered":"2023-10-01T09:21:03.352649+0000","last_unstale":"2023-10-01T09:21:20.532586+0000","last_undegraded":"2023-10-01T09:21:20.532586+0000","last_fullsized":"2023-10-01T09:21:20.532586+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6,3],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":158,"reported_epoch":215,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:19.534896+0000","last_change":"2023-10-01T09:21:14.629915+0000","last_active":"2023-10-01T09:21:19.534896+0000","last_peered":"2023-10-01T09:21:19.534896+0000","last_clean":"2023-10-01T09:21:19.534896+0000","last_became_active":"2023-10-01T09:21:14.598397+0000","last_became_peered":"2023-10-01T09:21:14.598397+0000","last_unstale":"2023-10-01T09:21:19.534896+0000","last_undegraded":"2023-10-01T09:21:19.534896+0000","last_fullsized":"2023-10-01T09:21:19.534896+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,0],"avail_no_missing":["6","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"217'6336","reported_seq":6383,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-01T09:21:27.252188+0000","last_change":"2023-10-01T09:19:34.259103+0000","last_active":"2023-10-01T09:21:27.252188+0000","last_peered":"2023-10-01T09:21:27.252188+0000","last_clean":"2023-10-01T09:21:27.252188+0000","last_became_active":"2023-10-01T09:19:34.258143+0000","last_became_peered":"2023-10-01T09:19:34.258143+0000","last_unstale":"2023-10-01T09:21:27.252188+0000","last_undegraded":"2023-10-01T09:21:27.252188+0000","last_fullsized":"2023-10-01T09:21:27.252188+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":6336,"log_dups_size":0,"ondisk_log_size":6336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:55:08.621837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363462656,"num_objects":5546,"num_object_clones":0,"num_object_copies":16638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6336,"num_write_kb":380224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":154,"reported_epoch":215,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:19.531808+0000","last_change":"2023-10-01T09:21:03.357675+0000","last_active":"2023-10-01T09:21:19.531808+0000","last_peered":"2023-10-01T09:21:19.531808+0000","last_clean":"2023-10-01T09:21:19.531808+0000","last_became_active":"2023-10-01T09:21:03.357333+0000","last_became_peered":"2023-10-01T09:21:03.357333+0000","last_unstale":"2023-10-01T09:21:19.531808+0000","last_undegraded":"2023-10-01T09:21:19.531808+0000","last_fullsized":"2023-10-01T09:21:19.531808+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:32:45.291673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,0],"avail_no_missing":["1","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":167,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.532819+0000","last_change":"2023-10-01T09:12:10.001565+0000","last_active":"2023-10-01T09:21:20.532819+0000","last_peered":"2023-10-01T09:21:20.532819+0000","last_clean":"2023-10-01T09:21:20.532819+0000","last_became_active":"2023-10-01T09:12:10.001275+0000","last_became_peered":"2023-10-01T09:12:10.001275+0000","last_unstale":"2023-10-01T09:21:20.532819+0000","last_undegraded":"2023-10-01T09:21:20.532819+0000","last_fullsized":"2023-10-01T09:21:20.532819+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:39.067046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"216'6263","reported_seq":6309,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:23.244461+0000","last_change":"2023-10-01T09:19:34.254772+0000","last_active":"2023-10-01T09:21:23.244461+0000","last_peered":"2023-10-01T09:21:23.244461+0000","last_clean":"2023-10-01T09:21:23.244461+0000","last_became_active":"2023-10-01T09:19:34.254457+0000","last_became_peered":"2023-10-01T09:19:34.254457+0000","last_unstale":"2023-10-01T09:21:23.244461+0000","last_undegraded":"2023-10-01T09:21:23.244461+0000","last_fullsized":"2023-10-01T09:21:23.244461+0000","mapping_epoch":179,"log_start":"187'587","ondisk_log_start":"187'587","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5676,"log_dups_size":0,"ondisk_log_size":5676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:19:20.462736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379650048,"num_objects":5793,"num_object_clones":0,"num_object_copies":17379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6263,"num_write_kb":385792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":186,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.530069+0000","last_change":"2023-10-01T09:13:13.554321+0000","last_active":"2023-10-01T09:21:20.530069+0000","last_peered":"2023-10-01T09:21:20.530069+0000","last_clean":"2023-10-01T09:21:20.530069+0000","last_became_active":"2023-10-01T09:13:13.553986+0000","last_became_peered":"2023-10-01T09:13:13.553986+0000","last_unstale":"2023-10-01T09:21:20.530069+0000","last_undegraded":"2023-10-01T09:21:20.530069+0000","last_fullsized":"2023-10-01T09:21:20.530069+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:09:12.848269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":147,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.529404+0000","last_change":"2023-10-01T09:21:02.139718+0000","last_active":"2023-10-01T09:21:20.529404+0000","last_peered":"2023-10-01T09:21:20.529404+0000","last_clean":"2023-10-01T09:21:20.529404+0000","last_became_active":"2023-10-01T09:21:02.139406+0000","last_became_peered":"2023-10-01T09:21:02.139406+0000","last_unstale":"2023-10-01T09:21:20.529404+0000","last_undegraded":"2023-10-01T09:21:20.529404+0000","last_fullsized":"2023-10-01T09:21:20.529404+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:10.823093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.f","version":"216'6059","reported_seq":6722,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:24.456833+0000","last_change":"2023-10-01T09:21:17.533001+0000","last_active":"2023-10-01T09:21:24.456833+0000","last_peered":"2023-10-01T09:21:24.456833+0000","last_clean":"2023-10-01T09:21:24.456833+0000","last_became_active":"2023-10-01T09:21:17.525307+0000","last_became_peered":"2023-10-01T09:21:17.525307+0000","last_unstale":"2023-10-01T09:21:24.456833+0000","last_undegraded":"2023-10-01T09:21:24.456833+0000","last_fullsized":"2023-10-01T09:21:24.456833+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":6059,"log_dups_size":0,"ondisk_log_size":6059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:31:13.895798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363266048,"num_objects":5543,"num_object_clones":0,"num_object_copies":16629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6059,"num_write_kb":371264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":296,"num_bytes_recovered":19398656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":142,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.525197+0000","last_change":"2023-10-01T09:21:02.136776+0000","last_active":"2023-10-01T09:21:19.525197+0000","last_peered":"2023-10-01T09:21:19.525197+0000","last_clean":"2023-10-01T09:21:19.525197+0000","last_became_active":"2023-10-01T09:21:02.136476+0000","last_became_peered":"2023-10-01T09:21:02.136476+0000","last_unstale":"2023-10-01T09:21:19.525197+0000","last_undegraded":"2023-10-01T09:21:19.525197+0000","last_fullsized":"2023-10-01T09:21:19.525197+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:44:41.289825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":213,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.530182+0000","last_change":"2023-10-01T09:20:15.505658+0000","last_active":"2023-10-01T09:21:19.530182+0000","last_peered":"2023-10-01T09:21:19.530182+0000","last_clean":"2023-10-01T09:21:19.530182+0000","last_became_active":"2023-10-01T09:20:15.505361+0000","last_became_peered":"2023-10-01T09:20:15.505361+0000","last_unstale":"2023-10-01T09:21:19.530182+0000","last_undegraded":"2023-10-01T09:21:19.530182+0000","last_fullsized":"2023-10-01T09:21:19.530182+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:22:16.176432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.c","version":"216'6207","reported_seq":6768,"reported_epoch":216,"state":"active+recovering+degraded+remapped","last_fresh":"2023-10-01T09:21:24.611867+0000","last_change":"2023-10-01T09:21:19.283373+0000","last_active":"2023-10-01T09:21:24.611867+0000","last_peered":"2023-10-01T09:21:24.611867+0000","last_clean":"2023-10-01T09:20:35.505999+0000","last_became_active":"2023-10-01T09:21:03.363274+0000","last_became_peered":"2023-10-01T09:21:03.363274+0000","last_unstale":"2023-10-01T09:21:24.611867+0000","last_undegraded":"2023-10-01T09:21:03.349509+0000","last_fullsized":"2023-10-01T09:21:24.611867+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":6207,"log_dups_size":0,"ondisk_log_size":6207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374538240,"num_objects":5715,"num_object_clones":0,"num_object_copies":17145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5715,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6207,"num_write_kb":381504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":247,"num_bytes_recovered":16187392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7,1],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"1,6,7","objects":299},{"shards":"6,7","objects":5416}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":84,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.529018+0000","last_change":"2023-10-01T09:19:30.092681+0000","last_active":"2023-10-01T09:21:20.529018+0000","last_peered":"2023-10-01T09:21:20.529018+0000","last_clean":"2023-10-01T09:21:20.529018+0000","last_became_active":"2023-10-01T09:19:30.092394+0000","last_became_peered":"2023-10-01T09:19:30.092394+0000","last_unstale":"2023-10-01T09:21:20.529018+0000","last_undegraded":"2023-10-01T09:21:20.529018+0000","last_fullsized":"2023-10-01T09:21:20.529018+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:41:50.127253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":152,"reported_epoch":216,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:20.541299+0000","last_change":"2023-10-01T09:21:03.352540+0000","last_active":"2023-10-01T09:21:20.541299+0000","last_peered":"2023-10-01T09:21:20.541299+0000","last_clean":"2023-10-01T09:21:20.541299+0000","last_became_active":"2023-10-01T09:21:03.351992+0000","last_became_peered":"2023-10-01T09:21:03.351992+0000","last_unstale":"2023-10-01T09:21:20.541299+0000","last_undegraded":"2023-10-01T09:21:20.541299+0000","last_fullsized":"2023-10-01T09:21:20.541299+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,3],"avail_no_missing":["4","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"216'6250","reported_seq":6300,"reported_epoch":216,"state":"active+undersized+degraded","last_fresh":"2023-10-01T09:21:23.404516+0000","last_change":"2023-10-01T09:20:36.517952+0000","last_active":"2023-10-01T09:21:23.404516+0000","last_peered":"2023-10-01T09:21:23.404516+0000","last_clean":"2023-10-01T09:20:35.465131+0000","last_became_active":"2023-10-01T09:20:36.517952+0000","last_became_peered":"2023-10-01T09:20:36.517952+0000","last_unstale":"2023-10-01T09:21:23.404516+0000","last_undegraded":"2023-10-01T09:20:36.511581+0000","last_fullsized":"2023-10-01T09:20:36.511516+0000","mapping_epoch":198,"log_start":"187'561","ondisk_log_start":"187'561","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:40:19.389753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378863640,"num_objects":5782,"num_object_clones":0,"num_object_copies":17346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5782,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5782,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6250,"num_write_kb":384961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":5782}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":152,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.526358+0000","last_change":"2023-10-01T09:17:59.310176+0000","last_active":"2023-10-01T09:21:19.526358+0000","last_peered":"2023-10-01T09:21:19.526358+0000","last_clean":"2023-10-01T09:21:19.526358+0000","last_became_active":"2023-10-01T09:17:59.309857+0000","last_became_peered":"2023-10-01T09:17:59.309857+0000","last_unstale":"2023-10-01T09:21:19.526358+0000","last_undegraded":"2023-10-01T09:21:19.526358+0000","last_fullsized":"2023-10-01T09:21:19.526358+0000","mapping_epoch":141,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:17:24.906057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":80,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.525970+0000","last_change":"2023-10-01T09:18:31.419430+0000","last_active":"2023-10-01T09:21:19.525970+0000","last_peered":"2023-10-01T09:21:19.525970+0000","last_clean":"2023-10-01T09:21:19.525970+0000","last_became_active":"2023-10-01T09:18:31.419189+0000","last_became_peered":"2023-10-01T09:18:31.419189+0000","last_unstale":"2023-10-01T09:21:19.525970+0000","last_undegraded":"2023-10-01T09:21:19.525970+0000","last_fullsized":"2023-10-01T09:21:19.525970+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:08:48.247926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"217'6273","reported_seq":6320,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-01T09:21:27.224849+0000","last_change":"2023-10-01T09:19:34.258841+0000","last_active":"2023-10-01T09:21:27.224849+0000","last_peered":"2023-10-01T09:21:27.224849+0000","last_clean":"2023-10-01T09:21:27.224849+0000","last_became_active":"2023-10-01T09:19:34.258571+0000","last_became_peered":"2023-10-01T09:19:34.258571+0000","last_unstale":"2023-10-01T09:21:27.224849+0000","last_undegraded":"2023-10-01T09:21:27.224849+0000","last_fullsized":"2023-10-01T09:21:27.224849+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":6273,"log_dups_size":0,"ondisk_log_size":6273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:12:27.336017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364576768,"num_objects":5563,"num_object_clones":0,"num_object_copies":16689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6273,"num_write_kb":378752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1032,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.531995+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:21:19.531995+0000","last_peered":"2023-10-01T09:21:19.531995+0000","last_clean":"2023-10-01T09:21:19.531995+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:21:19.531995+0000","last_undegraded":"2023-10-01T09:21:19.531995+0000","last_fullsized":"2023-10-01T09:21:19.531995+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":222,"reported_epoch":215,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:19.530058+0000","last_change":"2023-10-01T09:21:03.236916+0000","last_active":"2023-10-01T09:21:19.530058+0000","last_peered":"2023-10-01T09:21:19.530058+0000","last_clean":"2023-10-01T09:21:19.530058+0000","last_became_active":"2023-10-01T09:21:03.236521+0000","last_became_peered":"2023-10-01T09:21:03.236521+0000","last_unstale":"2023-10-01T09:21:19.530058+0000","last_undegraded":"2023-10-01T09:21:19.530058+0000","last_fullsized":"2023-10-01T09:21:19.530058+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:27:49.593898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,1],"avail_no_missing":["2","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":153,"reported_epoch":215,"state":"active+undersized","last_fresh":"2023-10-01T09:21:19.525099+0000","last_change":"2023-10-01T09:20:36.512182+0000","last_active":"2023-10-01T09:21:19.525099+0000","last_peered":"2023-10-01T09:21:19.525099+0000","last_clean":"2023-10-01T09:20:28.303986+0000","last_became_active":"2023-10-01T09:20:36.512182+0000","last_became_peered":"2023-10-01T09:20:36.512182+0000","last_unstale":"2023-10-01T09:21:19.525099+0000","last_undegraded":"2023-10-01T09:21:19.525099+0000","last_fullsized":"2023-10-01T09:20:36.509527+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:11:17.293636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"217'6321","reported_seq":6386,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-01T09:21:26.388384+0000","last_change":"2023-10-01T09:21:02.135520+0000","last_active":"2023-10-01T09:21:26.388384+0000","last_peered":"2023-10-01T09:21:26.388384+0000","last_clean":"2023-10-01T09:21:26.388384+0000","last_became_active":"2023-10-01T09:21:02.125689+0000","last_became_peered":"2023-10-01T09:21:02.125689+0000","last_unstale":"2023-10-01T09:21:26.388384+0000","last_undegraded":"2023-10-01T09:21:26.388384+0000","last_fullsized":"2023-10-01T09:21:26.388384+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":6321,"log_dups_size":0,"ondisk_log_size":6321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368902144,"num_objects":5629,"num_object_clones":0,"num_object_copies":16887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6321,"num_write_kb":382400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":798,"reported_epoch":215,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:19.525338+0000","last_change":"2023-10-01T09:21:14.616000+0000","last_active":"2023-10-01T09:21:19.525338+0000","last_peered":"2023-10-01T09:21:19.525338+0000","last_clean":"2023-10-01T09:21:19.525338+0000","last_became_active":"2023-10-01T09:21:14.615356+0000","last_became_peered":"2023-10-01T09:21:14.615356+0000","last_unstale":"2023-10-01T09:21:19.525338+0000","last_undegraded":"2023-10-01T09:21:19.525338+0000","last_fullsized":"2023-10-01T09:21:19.525338+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:42:09.042479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,0],"avail_no_missing":["1","0","6"],"object_location_counts":[{"shards":"0,1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":134,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.533193+0000","last_change":"2023-10-01T09:21:02.121225+0000","last_active":"2023-10-01T09:21:20.533193+0000","last_peered":"2023-10-01T09:21:20.533193+0000","last_clean":"2023-10-01T09:21:20.533193+0000","last_became_active":"2023-10-01T09:21:02.120911+0000","last_became_peered":"2023-10-01T09:21:02.120911+0000","last_unstale":"2023-10-01T09:21:20.533193+0000","last_undegraded":"2023-10-01T09:21:20.533193+0000","last_fullsized":"2023-10-01T09:21:20.533193+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:34:44.385101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":61,"reported_epoch":215,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:19.530615+0000","last_change":"2023-10-01T09:21:03.359828+0000","last_active":"2023-10-01T09:21:19.530615+0000","last_peered":"2023-10-01T09:21:19.530615+0000","last_clean":"2023-10-01T09:21:19.530615+0000","last_became_active":"2023-10-01T09:21:03.359385+0000","last_became_peered":"2023-10-01T09:21:03.359385+0000","last_unstale":"2023-10-01T09:21:19.530615+0000","last_undegraded":"2023-10-01T09:21:19.530615+0000","last_fullsized":"2023-10-01T09:21:19.530615+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:27:31.856759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,1],"avail_no_missing":["2","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"217'6239","reported_seq":6944,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-01T09:21:26.379020+0000","last_change":"2023-10-01T09:21:19.546095+0000","last_active":"2023-10-01T09:21:26.379020+0000","last_peered":"2023-10-01T09:21:26.379020+0000","last_clean":"2023-10-01T09:21:26.379020+0000","last_became_active":"2023-10-01T09:21:19.537089+0000","last_became_peered":"2023-10-01T09:21:19.537089+0000","last_unstale":"2023-10-01T09:21:26.379020+0000","last_undegraded":"2023-10-01T09:21:26.379020+0000","last_fullsized":"2023-10-01T09:21:26.379020+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":6239,"log_dups_size":0,"ondisk_log_size":6239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364707840,"num_objects":5565,"num_object_clones":0,"num_object_copies":16695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6239,"num_write_kb":377728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":316,"num_bytes_recovered":20709376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":509,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.530102+0000","last_change":"2023-10-01T09:20:15.506193+0000","last_active":"2023-10-01T09:21:19.530102+0000","last_peered":"2023-10-01T09:21:19.530102+0000","last_clean":"2023-10-01T09:21:19.530102+0000","last_became_active":"2023-10-01T09:20:15.505599+0000","last_became_peered":"2023-10-01T09:20:15.505599+0000","last_unstale":"2023-10-01T09:21:19.530102+0000","last_undegraded":"2023-10-01T09:21:19.530102+0000","last_fullsized":"2023-10-01T09:21:19.530102+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:09:20.837289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":216,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.526517+0000","last_change":"2023-10-01T09:21:02.140363+0000","last_active":"2023-10-01T09:21:19.526517+0000","last_peered":"2023-10-01T09:21:19.526517+0000","last_clean":"2023-10-01T09:21:19.526517+0000","last_became_active":"2023-10-01T09:21:02.137198+0000","last_became_peered":"2023-10-01T09:21:02.137198+0000","last_unstale":"2023-10-01T09:21:19.526517+0000","last_undegraded":"2023-10-01T09:21:19.526517+0000","last_fullsized":"2023-10-01T09:21:19.526517+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:38:38.043146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":215,"state":"active+undersized","last_fresh":"2023-10-01T09:21:19.526174+0000","last_change":"2023-10-01T09:20:36.516518+0000","last_active":"2023-10-01T09:21:19.526174+0000","last_peered":"2023-10-01T09:21:19.526174+0000","last_clean":"2023-10-01T09:20:28.124400+0000","last_became_active":"2023-10-01T09:20:36.516518+0000","last_became_peered":"2023-10-01T09:20:36.516518+0000","last_unstale":"2023-10-01T09:21:19.526174+0000","last_undegraded":"2023-10-01T09:21:19.526174+0000","last_fullsized":"2023-10-01T09:20:36.511376+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:54:42.493772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"217'6329","reported_seq":6379,"reported_epoch":217,"state":"active+undersized+degraded","last_fresh":"2023-10-01T09:21:26.890970+0000","last_change":"2023-10-01T09:20:36.626688+0000","last_active":"2023-10-01T09:21:26.890970+0000","last_peered":"2023-10-01T09:21:26.890970+0000","last_clean":"2023-10-01T09:20:35.374995+0000","last_became_active":"2023-10-01T09:20:36.626688+0000","last_became_peered":"2023-10-01T09:20:36.626688+0000","last_unstale":"2023-10-01T09:21:26.890970+0000","last_undegraded":"2023-10-01T09:20:36.515631+0000","last_fullsized":"2023-10-01T09:20:36.515526+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":6329,"log_dups_size":0,"ondisk_log_size":6329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:41:51.119576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368640000,"num_objects":5625,"num_object_clones":0,"num_object_copies":16875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5625,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6329,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":5625}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1156,"reported_epoch":215,"state":"active+undersized+degraded","last_fresh":"2023-10-01T09:21:19.530191+0000","last_change":"2023-10-01T09:20:36.518526+0000","last_active":"2023-10-01T09:21:19.530191+0000","last_peered":"2023-10-01T09:21:19.530191+0000","last_clean":"2023-10-01T09:19:24.623365+0000","last_became_active":"2023-10-01T09:20:36.518526+0000","last_became_peered":"2023-10-01T09:20:36.518526+0000","last_unstale":"2023-10-01T09:21:19.530191+0000","last_undegraded":"2023-10-01T09:20:36.503982+0000","last_fullsized":"2023-10-01T09:20:36.503855+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:56:30.953630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":376,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.540722+0000","last_change":"2023-10-01T09:21:20.540416+0000","last_active":"2023-10-01T09:21:20.540722+0000","last_peered":"2023-10-01T09:21:20.540722+0000","last_clean":"2023-10-01T09:21:20.540722+0000","last_became_active":"2023-10-01T09:21:20.539921+0000","last_became_peered":"2023-10-01T09:21:20.539921+0000","last_unstale":"2023-10-01T09:21:20.540722+0000","last_undegraded":"2023-10-01T09:21:20.540722+0000","last_fullsized":"2023-10-01T09:21:20.540722+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:56:57.788006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":86,"reported_epoch":215,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:19.535043+0000","last_change":"2023-10-01T09:21:14.615478+0000","last_active":"2023-10-01T09:21:19.535043+0000","last_peered":"2023-10-01T09:21:19.535043+0000","last_clean":"2023-10-01T09:21:19.535043+0000","last_became_active":"2023-10-01T09:21:14.575818+0000","last_became_peered":"2023-10-01T09:21:14.575818+0000","last_unstale":"2023-10-01T09:21:19.535043+0000","last_undegraded":"2023-10-01T09:21:19.535043+0000","last_fullsized":"2023-10-01T09:21:19.535043+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,1],"avail_no_missing":["6","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":89,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.532774+0000","last_change":"2023-10-01T09:21:02.125167+0000","last_active":"2023-10-01T09:21:20.532774+0000","last_peered":"2023-10-01T09:21:20.532774+0000","last_clean":"2023-10-01T09:21:20.532774+0000","last_became_active":"2023-10-01T09:21:02.124791+0000","last_became_peered":"2023-10-01T09:21:02.124791+0000","last_unstale":"2023-10-01T09:21:20.532774+0000","last_undegraded":"2023-10-01T09:21:20.532774+0000","last_fullsized":"2023-10-01T09:21:20.532774+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.4","version":"216'6119","reported_seq":6604,"reported_epoch":216,"state":"active+recovering+degraded+remapped","last_fresh":"2023-10-01T09:21:23.418907+0000","last_change":"2023-10-01T09:21:19.294270+0000","last_active":"2023-10-01T09:21:23.418907+0000","last_peered":"2023-10-01T09:21:23.418907+0000","last_clean":"2023-10-01T09:20:35.496239+0000","last_became_active":"2023-10-01T09:21:03.362558+0000","last_became_peered":"2023-10-01T09:21:03.362558+0000","last_unstale":"2023-10-01T09:21:23.418907+0000","last_undegraded":"2023-10-01T09:21:03.134122+0000","last_fullsized":"2023-10-01T09:21:23.418907+0000","mapping_epoch":208,"log_start":"186'474","ondisk_log_start":"186'474","created":179,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5645,"log_dups_size":0,"ondisk_log_size":5645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:03:07.180667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373358592,"num_objects":5697,"num_object_clones":0,"num_object_copies":17091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5697,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6119,"num_write_kb":378112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":211,"num_bytes_recovered":13828096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,4],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,4,6","objects":277},{"shards":"1,6","objects":5420}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":313,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.532256+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:21:20.532256+0000","last_peered":"2023-10-01T09:21:20.532256+0000","last_clean":"2023-10-01T09:21:20.532256+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:21:20.532256+0000","last_undegraded":"2023-10-01T09:21:20.532256+0000","last_fullsized":"2023-10-01T09:21:20.532256+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":183,"reported_epoch":216,"state":"active+undersized","last_fresh":"2023-10-01T09:21:22.283985+0000","last_change":"2023-10-01T09:20:36.511145+0000","last_active":"2023-10-01T09:21:22.283985+0000","last_peered":"2023-10-01T09:21:22.283985+0000","last_clean":"2023-10-01T09:20:28.116617+0000","last_became_active":"2023-10-01T09:20:36.511145+0000","last_became_peered":"2023-10-01T09:20:36.511145+0000","last_unstale":"2023-10-01T09:21:22.283985+0000","last_undegraded":"2023-10-01T09:21:22.283985+0000","last_fullsized":"2023-10-01T09:20:36.507550+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:12:49.301931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":218,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.533991+0000","last_change":"2023-10-01T09:19:30.091481+0000","last_active":"2023-10-01T09:21:20.533991+0000","last_peered":"2023-10-01T09:21:20.533991+0000","last_clean":"2023-10-01T09:21:20.533991+0000","last_became_active":"2023-10-01T09:19:30.091151+0000","last_became_peered":"2023-10-01T09:19:30.091151+0000","last_unstale":"2023-10-01T09:21:20.533991+0000","last_undegraded":"2023-10-01T09:21:20.533991+0000","last_fullsized":"2023-10-01T09:21:20.533991+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:09.127408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"216'6112","reported_seq":6761,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:23.424401+0000","last_change":"2023-10-01T09:21:20.548452+0000","last_active":"2023-10-01T09:21:23.424401+0000","last_peered":"2023-10-01T09:21:23.424401+0000","last_clean":"2023-10-01T09:21:23.424401+0000","last_became_active":"2023-10-01T09:21:20.540044+0000","last_became_peered":"2023-10-01T09:21:20.540044+0000","last_unstale":"2023-10-01T09:21:23.424401+0000","last_undegraded":"2023-10-01T09:21:23.424401+0000","last_fullsized":"2023-10-01T09:21:23.424401+0000","mapping_epoch":215,"log_start":"186'444","ondisk_log_start":"186'444","created":179,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":5668,"log_dups_size":0,"ondisk_log_size":5668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:39:51.673957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371720192,"num_objects":5672,"num_object_clones":0,"num_object_copies":17016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6112,"num_write_kb":377088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":289,"num_bytes_recovered":18939904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":577,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.532744+0000","last_change":"2023-10-01T09:10:58.797964+0000","last_active":"2023-10-01T09:21:20.532744+0000","last_peered":"2023-10-01T09:21:20.532744+0000","last_clean":"2023-10-01T09:21:20.532744+0000","last_became_active":"2023-10-01T09:10:58.797447+0000","last_became_peered":"2023-10-01T09:10:58.797447+0000","last_unstale":"2023-10-01T09:21:20.532744+0000","last_undegraded":"2023-10-01T09:21:20.532744+0000","last_fullsized":"2023-10-01T09:21:20.532744+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":204,"reported_epoch":215,"state":"active+undersized","last_fresh":"2023-10-01T09:21:19.532103+0000","last_change":"2023-10-01T09:20:36.513234+0000","last_active":"2023-10-01T09:21:19.532103+0000","last_peered":"2023-10-01T09:21:19.532103+0000","last_clean":"2023-10-01T09:20:28.304048+0000","last_became_active":"2023-10-01T09:20:36.513234+0000","last_became_peered":"2023-10-01T09:20:36.513234+0000","last_unstale":"2023-10-01T09:21:19.532103+0000","last_undegraded":"2023-10-01T09:21:19.532103+0000","last_fullsized":"2023-10-01T09:20:36.508669+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:53:27.896564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":159,"reported_epoch":216,"state":"active+undersized","last_fresh":"2023-10-01T09:21:22.284069+0000","last_change":"2023-10-01T09:20:36.511317+0000","last_active":"2023-10-01T09:21:22.284069+0000","last_peered":"2023-10-01T09:21:22.284069+0000","last_clean":"2023-10-01T09:20:28.115996+0000","last_became_active":"2023-10-01T09:20:36.511317+0000","last_became_peered":"2023-10-01T09:20:36.511317+0000","last_unstale":"2023-10-01T09:21:22.284069+0000","last_undegraded":"2023-10-01T09:21:22.284069+0000","last_fullsized":"2023-10-01T09:20:36.507026+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:25.829214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"216'6368","reported_seq":6440,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:24.513117+0000","last_change":"2023-10-01T09:21:02.130556+0000","last_active":"2023-10-01T09:21:24.513117+0000","last_peered":"2023-10-01T09:21:24.513117+0000","last_clean":"2023-10-01T09:21:24.513117+0000","last_became_active":"2023-10-01T09:21:02.124268+0000","last_became_peered":"2023-10-01T09:21:02.124268+0000","last_unstale":"2023-10-01T09:21:24.513117+0000","last_undegraded":"2023-10-01T09:21:24.513117+0000","last_fullsized":"2023-10-01T09:21:24.513117+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":6368,"log_dups_size":0,"ondisk_log_size":6368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:16:11.483474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379977728,"num_objects":5798,"num_object_clones":0,"num_object_copies":17394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6368,"num_write_kb":389312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":205,"reported_epoch":216,"state":"active+undersized+degraded","last_fresh":"2023-10-01T09:21:20.530407+0000","last_change":"2023-10-01T09:20:36.518571+0000","last_active":"2023-10-01T09:21:20.530407+0000","last_peered":"2023-10-01T09:21:20.530407+0000","last_clean":"2023-10-01T09:20:28.127360+0000","last_became_active":"2023-10-01T09:20:36.518571+0000","last_became_peered":"2023-10-01T09:20:36.518571+0000","last_unstale":"2023-10-01T09:21:20.530407+0000","last_undegraded":"2023-10-01T09:20:36.515187+0000","last_fullsized":"2023-10-01T09:20:36.515081+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:04:31.470403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":688,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.545328+0000","last_change":"2023-10-01T09:21:20.545328+0000","last_active":"2023-10-01T09:21:20.545328+0000","last_peered":"2023-10-01T09:21:20.545328+0000","last_clean":"2023-10-01T09:21:20.545328+0000","last_became_active":"2023-10-01T09:21:20.544828+0000","last_became_peered":"2023-10-01T09:21:20.544828+0000","last_unstale":"2023-10-01T09:21:20.545328+0000","last_undegraded":"2023-10-01T09:21:20.545328+0000","last_fullsized":"2023-10-01T09:21:20.545328+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:23:52.633471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":113,"reported_epoch":216,"state":"active+undersized","last_fresh":"2023-10-01T09:21:20.533972+0000","last_change":"2023-10-01T09:20:36.515693+0000","last_active":"2023-10-01T09:21:20.533972+0000","last_peered":"2023-10-01T09:21:20.533972+0000","last_clean":"2023-10-01T09:20:28.115718+0000","last_became_active":"2023-10-01T09:20:36.515693+0000","last_became_peered":"2023-10-01T09:20:36.515693+0000","last_unstale":"2023-10-01T09:21:20.533972+0000","last_undegraded":"2023-10-01T09:21:20.533972+0000","last_fullsized":"2023-10-01T09:20:36.506652+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:06:45.896132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.3","version":"217'6333","reported_seq":6969,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-01T09:21:26.379444+0000","last_change":"2023-10-01T09:21:16.567633+0000","last_active":"2023-10-01T09:21:26.379444+0000","last_peered":"2023-10-01T09:21:26.379444+0000","last_clean":"2023-10-01T09:21:26.379444+0000","last_became_active":"2023-10-01T09:21:16.523642+0000","last_became_peered":"2023-10-01T09:21:16.523642+0000","last_unstale":"2023-10-01T09:21:26.379444+0000","last_undegraded":"2023-10-01T09:21:26.379444+0000","last_fullsized":"2023-10-01T09:21:26.379444+0000","mapping_epoch":211,"log_start":"0'0","ondisk_log_start":"0'0","created":179,"last_epoch_clean":212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:19:33.238000+0000","last_clean_scrub_stamp":"2023-10-01T09:19:33.238000+0000","objects_scrubbed":0,"log_size":6333,"log_dups_size":0,"ondisk_log_size":6333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368640000,"num_objects":5625,"num_object_clones":0,"num_object_copies":16875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6333,"num_write_kb":382656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":282,"num_bytes_recovered":18481152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":28,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.534787+0000","last_change":"2023-10-01T09:21:02.195366+0000","last_active":"2023-10-01T09:21:19.534787+0000","last_peered":"2023-10-01T09:21:19.534787+0000","last_clean":"2023-10-01T09:21:19.534787+0000","last_became_active":"2023-10-01T09:21:02.135537+0000","last_became_peered":"2023-10-01T09:21:02.135537+0000","last_unstale":"2023-10-01T09:21:19.534787+0000","last_undegraded":"2023-10-01T09:21:19.534787+0000","last_fullsized":"2023-10-01T09:21:19.534787+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":197,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.533799+0000","last_change":"2023-10-01T09:19:05.565472+0000","last_active":"2023-10-01T09:21:20.533799+0000","last_peered":"2023-10-01T09:21:20.533799+0000","last_clean":"2023-10-01T09:21:20.533799+0000","last_became_active":"2023-10-01T09:19:05.561161+0000","last_became_peered":"2023-10-01T09:19:05.561161+0000","last_unstale":"2023-10-01T09:21:20.533799+0000","last_undegraded":"2023-10-01T09:21:20.533799+0000","last_fullsized":"2023-10-01T09:21:20.533799+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:50:29.273388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":32,"reported_epoch":215,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:19.535073+0000","last_change":"2023-10-01T09:21:03.353032+0000","last_active":"2023-10-01T09:21:19.535073+0000","last_peered":"2023-10-01T09:21:19.535073+0000","last_clean":"2023-10-01T09:21:19.535073+0000","last_became_active":"2023-10-01T09:21:03.351414+0000","last_became_peered":"2023-10-01T09:21:03.351414+0000","last_unstale":"2023-10-01T09:21:19.535073+0000","last_undegraded":"2023-10-01T09:21:19.535073+0000","last_fullsized":"2023-10-01T09:21:19.535073+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,0],"avail_no_missing":["6","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":156,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.532980+0000","last_change":"2023-10-01T09:10:58.801161+0000","last_active":"2023-10-01T09:21:20.532980+0000","last_peered":"2023-10-01T09:21:20.532980+0000","last_clean":"2023-10-01T09:21:20.532980+0000","last_became_active":"2023-10-01T09:10:58.800794+0000","last_became_peered":"2023-10-01T09:10:58.800794+0000","last_unstale":"2023-10-01T09:21:20.532980+0000","last_undegraded":"2023-10-01T09:21:20.532980+0000","last_fullsized":"2023-10-01T09:21:20.532980+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:35.019445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":233,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.535510+0000","last_change":"2023-10-01T09:19:30.092736+0000","last_active":"2023-10-01T09:21:20.535510+0000","last_peered":"2023-10-01T09:21:20.535510+0000","last_clean":"2023-10-01T09:21:20.535510+0000","last_became_active":"2023-10-01T09:19:30.092334+0000","last_became_peered":"2023-10-01T09:19:30.092334+0000","last_unstale":"2023-10-01T09:21:20.535510+0000","last_undegraded":"2023-10-01T09:21:20.535510+0000","last_fullsized":"2023-10-01T09:21:20.535510+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:13:21.787586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":37,"reported_epoch":215,"state":"active+undersized+degraded","last_fresh":"2023-10-01T09:21:19.532344+0000","last_change":"2023-10-01T09:20:36.626453+0000","last_active":"2023-10-01T09:21:19.532344+0000","last_peered":"2023-10-01T09:21:19.532344+0000","last_clean":"2023-10-01T09:03:05.237925+0000","last_became_active":"2023-10-01T09:20:36.626453+0000","last_became_peered":"2023-10-01T09:20:36.626453+0000","last_unstale":"2023-10-01T09:21:19.532344+0000","last_undegraded":"2023-10-01T09:20:36.510883+0000","last_fullsized":"2023-10-01T09:20:36.510781+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:30.976322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":162,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-01T09:21:25.647796+0000","last_change":"2023-10-01T09:19:05.553200+0000","last_active":"2023-10-01T09:21:25.647796+0000","last_peered":"2023-10-01T09:21:25.647796+0000","last_clean":"2023-10-01T09:21:25.647796+0000","last_became_active":"2023-10-01T09:19:05.552874+0000","last_became_peered":"2023-10-01T09:19:05.552874+0000","last_unstale":"2023-10-01T09:21:25.647796+0000","last_undegraded":"2023-10-01T09:21:25.647796+0000","last_fullsized":"2023-10-01T09:21:25.647796+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:14:31.040887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":40,"reported_epoch":215,"state":"active+clean+remapped","last_fresh":"2023-10-01T09:21:19.536445+0000","last_change":"2023-10-01T09:21:03.354264+0000","last_active":"2023-10-01T09:21:19.536445+0000","last_peered":"2023-10-01T09:21:19.536445+0000","last_clean":"2023-10-01T09:21:19.536445+0000","last_became_active":"2023-10-01T09:21:03.352883+0000","last_became_peered":"2023-10-01T09:21:03.352883+0000","last_unstale":"2023-10-01T09:21:19.536445+0000","last_undegraded":"2023-10-01T09:21:19.536445+0000","last_fullsized":"2023-10-01T09:21:19.536445+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":144,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-10-01T09:21:25.647770+0000","last_change":"2023-10-01T09:18:54.533869+0000","last_active":"2023-10-01T09:21:25.647770+0000","last_peered":"2023-10-01T09:21:25.647770+0000","last_clean":"2023-10-01T09:21:25.647770+0000","last_became_active":"2023-10-01T09:18:54.533542+0000","last_became_peered":"2023-10-01T09:18:54.533542+0000","last_unstale":"2023-10-01T09:21:25.647770+0000","last_undegraded":"2023-10-01T09:21:25.647770+0000","last_fullsized":"2023-10-01T09:21:25.647770+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:18:19.126440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":124,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-10-01T09:21:19.530489+0000","last_change":"2023-10-01T09:21:02.138726+0000","last_active":"2023-10-01T09:21:19.530489+0000","last_peered":"2023-10-01T09:21:19.530489+0000","last_clean":"2023-10-01T09:21:19.530489+0000","last_became_active":"2023-10-01T09:21:02.138365+0000","last_became_peered":"2023-10-01T09:21:02.138365+0000","last_unstale":"2023-10-01T09:21:19.530489+0000","last_undegraded":"2023-10-01T09:21:19.530489+0000","last_fullsized":"2023-10-01T09:21:19.530489+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:57:42.151867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":213,"reported_epoch":216,"state":"active+undersized","last_fresh":"2023-10-01T09:21:20.534136+0000","last_change":"2023-10-01T09:20:36.508489+0000","last_active":"2023-10-01T09:21:20.534136+0000","last_peered":"2023-10-01T09:21:20.534136+0000","last_clean":"2023-10-01T09:20:28.116723+0000","last_became_active":"2023-10-01T09:20:36.508489+0000","last_became_peered":"2023-10-01T09:20:36.508489+0000","last_unstale":"2023-10-01T09:21:20.534136+0000","last_undegraded":"2023-10-01T09:21:20.534136+0000","last_fullsized":"2023-10-01T09:20:36.506135+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:57:45.525939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":238,"reported_epoch":216,"state":"active+undersized+degraded","last_fresh":"2023-10-01T09:21:22.284192+0000","last_change":"2023-10-01T09:20:36.508033+0000","last_active":"2023-10-01T09:21:22.284192+0000","last_peered":"2023-10-01T09:21:22.284192+0000","last_clean":"2023-10-01T09:20:28.116139+0000","last_became_active":"2023-10-01T09:20:36.508033+0000","last_became_peered":"2023-10-01T09:20:36.508033+0000","last_unstale":"2023-10-01T09:21:22.284192+0000","last_undegraded":"2023-10-01T09:20:36.506233+0000","last_fullsized":"2023-10-01T09:20:36.506111+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:55:00.830503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":137,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-10-01T09:21:20.532144+0000","last_change":"2023-10-01T09:21:02.141180+0000","last_active":"2023-10-01T09:21:20.532144+0000","last_peered":"2023-10-01T09:21:20.532144+0000","last_clean":"2023-10-01T09:21:20.532144+0000","last_became_active":"2023-10-01T09:21:02.136796+0000","last_became_peered":"2023-10-01T09:21:02.136796+0000","last_unstale":"2023-10-01T09:21:20.532144+0000","last_undegraded":"2023-10-01T09:21:20.532144+0000","last_fullsized":"2023-10-01T09:21:20.532144+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:03:46.040197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5932449816,"num_objects":90523,"num_object_clones":0,"num_object_copies":271569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":28889,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90523,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":100211,"num_write_kb":6103425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2478,"num_bytes_recovered":162398208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17568047104,"data_stored":17568038960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98145,"ondisk_log_size":98145,"up":43,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":82,"acting":89,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":6,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":12002,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27349,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":83,"acting":90,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":22,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430450,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2117284,"kb_used_data":1811868,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91627868,"statfs":{"total":95995035648,"available":93826936832,"internally_reserved":0,"allocated":1855352832,"data_stored":1854770934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:21:13 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":6.7320000000000002,"15min":3.0489999999999999},"min":{"1min":0.26600000000000001,"5min":0.22500000000000001,"15min":0.219},"max":{"1min":3.3450000000000002,"5min":117.855,"15min":117.855},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.129,"5min":6.25,"15min":2.8399999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":3.6899999999999999,"5min":120.348,"15min":120.348},"last":1.901}]},{"osd":1,"last update":"Sun Oct 1 09:20:24 2023","interfaces":[{"interface":"back","average":{"1min":9.7330000000000005,"5min":4.5430000000000001,"15min":1.9399999999999999},"min":{"1min":0.25600000000000001,"5min":0.23400000000000001,"15min":0.22700000000000001},"max":{"1min":105.045,"5min":105.045,"15min":105.045},"last":2.7850000000000001},{"interface":"front","average":{"1min":8.8390000000000004,"5min":4.0300000000000002,"15min":1.768},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":104.71599999999999,"5min":104.71599999999999,"15min":104.71599999999999},"last":2.6520000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:20:51 2023","interfaces":[{"interface":"back","average":{"1min":3.6549999999999998,"5min":4.2729999999999997,"15min":2.0720000000000001},"min":{"1min":0.32600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":40.921999999999997,"5min":104.837,"15min":104.837},"last":0.90500000000000003},{"interface":"front","average":{"1min":3.6139999999999999,"5min":3.964,"15min":1.994},"min":{"1min":0.29199999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":40.912999999999997,"5min":106.379,"15min":106.379},"last":0.46300000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:20:24 2023","interfaces":[{"interface":"back","average":{"1min":6.6189999999999998,"5min":3.2189999999999999,"15min":1.462},"min":{"1min":0.34399999999999997,"5min":0.31900000000000001,"15min":0.26400000000000001},"max":{"1min":104.797,"5min":104.797,"15min":104.797},"last":0.432},{"interface":"front","average":{"1min":6.8449999999999998,"5min":3.2090000000000001,"15min":1.4390000000000001},"min":{"1min":0.33400000000000002,"5min":0.29599999999999999,"15min":0.27200000000000002},"max":{"1min":104.774,"5min":104.774,"15min":104.774},"last":0.44400000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:21:13 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":2.8900000000000001,"15min":1.3520000000000001},"min":{"1min":0.28499999999999998,"5min":0.26500000000000001,"15min":0.23699999999999999},"max":{"1min":3.7229999999999999,"5min":82.992000000000004,"15min":82.992000000000004},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.80300000000000005,"5min":2.9609999999999999,"15min":1.393},"min":{"1min":0.34300000000000003,"5min":0.26700000000000002,"15min":0.252},"max":{"1min":3.5720000000000001,"5min":83.058999999999997,"15min":83.058999999999997},"last":0.34200000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:20:13 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":2.7320000000000002,"15min":1.2509999999999999},"min":{"1min":0.29799999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.9379999999999999,"5min":90.656999999999996,"15min":90.656999999999996},"last":3.113},{"interface":"front","average":{"1min":0.70299999999999996,"5min":3.5179999999999998,"15min":1.548},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.655,"5min":90.747,"15min":90.747},"last":2.7349999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:21:13 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.8759999999999999,"15min":1.056},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.3140000000000001,"5min":11.685,"15min":11.685},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.121,"5min":1.6839999999999999,"15min":1.0009999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.6269999999999998,"5min":11.871,"15min":11.871},"last":0.36899999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626677,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4317984,"kb_used_data":4012568,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":89427168,"statfs":{"total":95995035648,"available":91573420032,"internally_reserved":0,"allocated":4108869632,"data_stored":4108291729,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:21:10 2023","interfaces":[{"interface":"back","average":{"1min":3.7679999999999998,"5min":7.9429999999999996,"15min":3.7850000000000001},"min":{"1min":0.253,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":52.347000000000001,"5min":178.738,"15min":178.738},"last":0.23899999999999999},{"interface":"front","average":{"1min":4.1929999999999996,"5min":8.7110000000000003,"15min":4.0090000000000003},"min":{"1min":0.30099999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":52.375999999999998,"5min":178.684,"15min":178.684},"last":0.41099999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:20:26 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":1.859,"15min":1.0189999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.6859999999999999,"5min":24.488,"15min":24.488},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.1339999999999999,"5min":2.6920000000000002,"15min":1.3400000000000001},"min":{"1min":0.35299999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.8660000000000001,"5min":24.643999999999998,"15min":24.643999999999998},"last":0.48899999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:20:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6989999999999998,"5min":3.4380000000000002,"15min":2.2469999999999999},"min":{"1min":0.26500000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":16.513999999999999,"5min":57.753,"15min":82.978999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.9690000000000001,"5min":2.6749999999999998,"15min":1.9730000000000001},"min":{"1min":0.36699999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":17.036999999999999,"5min":57.695999999999998,"15min":82.747},"last":0.36599999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:20:26 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":2.0960000000000001,"15min":1.1020000000000001},"min":{"1min":0.33400000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.8890000000000002,"5min":35.718000000000004,"15min":35.718000000000004},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.1779999999999999,"5min":2.7309999999999999,"15min":1.341},"min":{"1min":0.34399999999999997,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":3.8149999999999999,"5min":31.568999999999999,"15min":31.568999999999999},"last":0.52900000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:20:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.8080000000000001,"15min":0.94199999999999995},"min":{"1min":0.33900000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.5110000000000001,"5min":28.271999999999998,"15min":28.271999999999998},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.81299999999999994,"5min":1.841,"15min":0.99099999999999999},"min":{"1min":0.33000000000000002,"5min":0.30099999999999999,"15min":0.26000000000000001},"max":{"1min":4.0949999999999998,"5min":28.413,"15min":28.413},"last":0.56799999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":2.601,"15min":1.2090000000000001},"min":{"1min":0.315,"5min":0.31,"15min":0.28499999999999998},"max":{"1min":14.007,"5min":36.063000000000002,"15min":36.063000000000002},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.077,"5min":1.8999999999999999,"15min":1.0049999999999999},"min":{"1min":0.35799999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":5.9649999999999999,"5min":35.932000000000002,"15min":35.932000000000002},"last":0.44}]},{"osd":7,"last update":"Sun Oct 1 09:21:10 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":3.2559999999999998,"15min":1.498},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.8799999999999999,"5min":145.37299999999999,"15min":145.37299999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.754,"5min":3.0590000000000002,"15min":1.4550000000000001},"min":{"1min":0.34699999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.0449999999999999,"5min":145.28200000000001,"15min":145.28200000000001},"last":0.42299999999999999}]}]},{"osd":1,"up_from":94,"seq":403726925937,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2067896,"kb_used_data":1901616,"kb_used_omap":7,"kb_used_meta":166200,"kb_avail":91677256,"statfs":{"total":95995035648,"available":93877510144,"internally_reserved":0,"allocated":1947254784,"data_stored":1946304900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":170189223},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:20:23 2023","interfaces":[{"interface":"back","average":{"1min":16.550000000000001,"5min":6.8970000000000002,"15min":2.5419999999999998},"min":{"1min":0.26400000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":134.84999999999999,"5min":174.40199999999999,"15min":174.40199999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":16.459,"5min":7.0549999999999997,"15min":2.6219999999999999},"min":{"1min":0.252,"5min":0.19400000000000001,"15min":0.188},"max":{"1min":134.77099999999999,"5min":173.77000000000001,"15min":173.77000000000001},"last":0.28899999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:20:59 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.897,"15min":0.97299999999999998},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":5.1509999999999998,"5min":25.632000000000001,"15min":25.632000000000001},"last":0.26800000000000002},{"interface":"front","average":{"1min":1.708,"5min":2.0350000000000001,"15min":1.042},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":6.0529999999999999,"5min":29.425999999999998,"15min":29.425999999999998},"last":0.39700000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:20:34 2023","interfaces":[{"interface":"back","average":{"1min":9.1259999999999994,"5min":3.2090000000000001,"15min":1.3580000000000001},"min":{"1min":0.224,"5min":0.222,"15min":0.218},"max":{"1min":99.421000000000006,"5min":99.421000000000006,"15min":99.421000000000006},"last":0.35299999999999998},{"interface":"front","average":{"1min":8.6199999999999992,"5min":2.8079999999999998,"15min":1.282},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":99.364999999999995,"5min":99.364999999999995,"15min":99.364999999999995},"last":0.317}]},{"osd":4,"last update":"Sun Oct 1 09:20:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":2.0059999999999998,"15min":1.069},"min":{"1min":0.30299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.4130000000000003,"5min":40.673999999999999,"15min":40.673999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.3260000000000001,"5min":2.3740000000000001,"15min":1.1910000000000001},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.5549999999999997,"5min":40.810000000000002,"15min":40.810000000000002},"last":0.42299999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:20:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":5.1340000000000003,"15min":2.1339999999999999},"min":{"1min":0.34499999999999997,"5min":0.313,"15min":0.313},"max":{"1min":6.351,"5min":282.04700000000003,"15min":282.04700000000003},"last":5.6500000000000004},{"interface":"front","average":{"1min":0.97499999999999998,"5min":4.9740000000000002,"15min":2.081},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.097,"5min":281.89499999999998,"15min":281.89499999999998},"last":2.613}]},{"osd":6,"last update":"Sun Oct 1 09:20:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.8280000000000001,"15min":1.0409999999999999},"min":{"1min":0.438,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.4669999999999996,"5min":9.7449999999999992,"15min":9.7449999999999992},"last":0.435},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1.476,"15min":0.93700000000000006},"min":{"1min":0.45800000000000002,"5min":0.372,"15min":0.372},"max":{"1min":2.032,"5min":10.787000000000001,"15min":10.787000000000001},"last":0.34200000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:20:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":2.0680000000000001,"15min":1.1140000000000001},"min":{"1min":0.39800000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":4.3769999999999998,"5min":23.710999999999999,"15min":23.710999999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.1639999999999999,"5min":2.1440000000000001,"15min":1.131},"min":{"1min":0.438,"5min":0.372,"15min":0.372},"max":{"1min":6.0780000000000003,"5min":23.736999999999998,"15min":23.736999999999998},"last":0.46200000000000002}]}]},{"osd":0,"up_from":89,"seq":382252089472,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1802332,"kb_used_data":1486420,"kb_used_omap":8,"kb_used_meta":315895,"kb_avail":91942820,"statfs":{"total":95995035648,"available":94149447680,"internally_reserved":0,"allocated":1522094080,"data_stored":1521121408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":323477442},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":3.2050000000000001,"5min":2.2050000000000001,"15min":1.006},"min":{"1min":0.23400000000000001,"5min":0.187,"15min":0.187},"max":{"1min":50.643999999999998,"5min":50.643999999999998,"15min":50.643999999999998},"last":0.61199999999999999},{"interface":"front","average":{"1min":3.347,"5min":2.5089999999999999,"15min":1.1020000000000001},"min":{"1min":0.21099999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":50.923999999999999,"5min":50.923999999999999,"15min":50.923999999999999},"last":0.193}]},{"osd":2,"last update":"Sun Oct 1 09:20:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7509999999999999,"5min":2.1739999999999999,"15min":1.0980000000000001},"min":{"1min":0.25900000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":31.024999999999999,"5min":31.024999999999999,"15min":31.024999999999999},"last":1.4830000000000001},{"interface":"front","average":{"1min":3.7799999999999998,"5min":2.823,"15min":1.2749999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":31.157,"5min":31.157,"15min":31.157},"last":2.6890000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.756,"15min":0.88800000000000001},"min":{"1min":0.317,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":5.6609999999999996,"5min":12.906000000000001,"15min":12.906000000000001},"last":12.305},{"interface":"front","average":{"1min":1.4079999999999999,"5min":2.1869999999999998,"15min":0.98799999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.23400000000000001},"max":{"1min":5.3239999999999998,"5min":17.318000000000001,"15min":17.318000000000001},"last":2.1800000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:21:22 2023","interfaces":[{"interface":"back","average":{"1min":1.8799999999999999,"5min":3.5419999999999998,"15min":1.7330000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":15.138999999999999,"5min":112.52200000000001,"15min":112.52200000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.83099999999999996,"5min":3.5419999999999998,"15min":1.7749999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28599999999999998},"max":{"1min":2.3090000000000002,"5min":113.124,"15min":113.124},"last":2.1549999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:20:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":3.6819999999999999,"15min":1.679},"min":{"1min":0.33200000000000002,"5min":0.32700000000000001,"15min":0.311},"max":{"1min":3.645,"5min":98.730999999999995,"15min":98.730999999999995},"last":4.5720000000000001},{"interface":"front","average":{"1min":0.752,"5min":3.343,"15min":1.6060000000000001},"min":{"1min":0.40600000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.3439999999999999,"5min":98.870999999999995,"15min":98.870999999999995},"last":3.1309999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:21:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2949999999999999,"5min":2.109,"15min":1.365},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.585,"5min":34.375999999999998,"15min":34.375999999999998},"last":2.101},{"interface":"front","average":{"1min":0.82999999999999996,"5min":1.6639999999999999,"15min":1.169},"min":{"1min":0.34799999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.5600000000000001,"5min":31.745000000000001,"15min":31.745000000000001},"last":0.72799999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:21:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":3.1789999999999998,"15min":1.7070000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.9129999999999998,"5min":98.283000000000001,"15min":98.283000000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":1.4359999999999999,"5min":3.3479999999999999,"15min":1.7210000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":5.4320000000000004,"5min":98.311999999999998,"15min":98.311999999999998},"last":0.312}]}]},{"osd":2,"up_from":99,"seq":425201762411,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3207240,"kb_used_data":2963136,"kb_used_omap":11,"kb_used_meta":244084,"kb_avail":90537912,"statfs":{"total":95995035648,"available":92710821888,"internally_reserved":0,"allocated":3034251264,"data_stored":3033346598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":249942903},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:20:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":2.9980000000000002,"15min":1.5509999999999999},"min":{"1min":0.23200000000000001,"5min":0.20100000000000001,"15min":0.188},"max":{"1min":8.1639999999999997,"5min":93.634,"15min":93.634},"last":1.355},{"interface":"front","average":{"1min":1.4450000000000001,"5min":3.5640000000000001,"15min":1.837},"min":{"1min":0.247,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":7.1589999999999998,"5min":93.906999999999996,"15min":93.906999999999996},"last":3.7130000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:20:58 2023","interfaces":[{"interface":"back","average":{"1min":9.077,"5min":3.5430000000000001,"15min":1.8440000000000001},"min":{"1min":0.27000000000000002,"5min":0.23699999999999999,"15min":0.20999999999999999},"max":{"1min":105.798,"5min":105.798,"15min":105.798},"last":1.4139999999999999},{"interface":"front","average":{"1min":9.0310000000000006,"5min":3.7490000000000001,"15min":1.9930000000000001},"min":{"1min":0.29299999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":105.56399999999999,"5min":105.56399999999999,"15min":107.729},"last":3.8559999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":15.981999999999999,"5min":4.8579999999999997,"15min":1.8839999999999999},"min":{"1min":0.23999999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":253.238,"5min":253.238,"15min":253.238},"last":3.4550000000000001},{"interface":"front","average":{"1min":15.813000000000001,"5min":4.6970000000000001,"15min":1.8660000000000001},"min":{"1min":0.27600000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":252.54900000000001,"5min":252.54900000000001,"15min":252.54900000000001},"last":1.3939999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:20:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":2.0499999999999998,"15min":1.1339999999999999},"min":{"1min":0.32700000000000001,"5min":0.30599999999999999,"15min":0.27000000000000002},"max":{"1min":7.1260000000000003,"5min":17.751999999999999,"15min":17.751999999999999},"last":3.6890000000000001},{"interface":"front","average":{"1min":2.0019999999999998,"5min":2.2229999999999999,"15min":1.194},"min":{"1min":0.32500000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.9649999999999999,"5min":17.568000000000001,"15min":17.568000000000001},"last":3.427}]},{"osd":5,"last update":"Sun Oct 1 09:19:57 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":2.7549999999999999,"15min":1.3069999999999999},"min":{"1min":0.38,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.002000000000001,"5min":34.084000000000003,"15min":34.084000000000003},"last":1.7030000000000001},{"interface":"front","average":{"1min":1.6459999999999999,"5min":2.7989999999999999,"15min":1.3180000000000001},"min":{"1min":0.436,"5min":0.316,"15min":0.316},"max":{"1min":7.5620000000000003,"5min":34.341999999999999,"15min":34.341999999999999},"last":1.2789999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:20:58 2023","interfaces":[{"interface":"back","average":{"1min":1.8999999999999999,"5min":2.335,"15min":1.24},"min":{"1min":0.35399999999999998,"5min":0.33900000000000002,"15min":0.33200000000000002},"max":{"1min":5.4509999999999996,"5min":17.591000000000001,"15min":17.591000000000001},"last":3.3820000000000001},{"interface":"front","average":{"1min":2.1339999999999999,"5min":1.637,"15min":0.98999999999999999},"min":{"1min":0.375,"5min":0.36599999999999999,"15min":0.34200000000000003},"max":{"1min":8.0389999999999997,"5min":17.736000000000001,"15min":17.736000000000001},"last":1.325}]},{"osd":7,"last update":"Sun Oct 1 09:20:58 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":3.1789999999999998,"15min":1.5209999999999999},"min":{"1min":0.40699999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.0820000000000007,"5min":109.506,"15min":109.506},"last":1.375},{"interface":"front","average":{"1min":1.776,"5min":3.4220000000000002,"15min":1.653},"min":{"1min":0.375,"5min":0.375,"15min":0.36399999999999999},"max":{"1min":7.1769999999999996,"5min":109.58799999999999,"15min":109.58799999999999},"last":3.6549999999999998}]}]},{"osd":3,"up_from":104,"seq":446676598885,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1927388,"kb_used_data":1805652,"kb_used_omap":3,"kb_used_meta":121724,"kb_avail":91817764,"statfs":{"total":95995035648,"available":94021390336,"internally_reserved":0,"allocated":1848987648,"data_stored":1848069309,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":124646119},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":3.7999999999999998,"15min":2.6629999999999998},"min":{"1min":0.189,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":38.314,"5min":99.034000000000006,"15min":199.422},"last":1.609},{"interface":"front","average":{"1min":2.5609999999999999,"5min":4.2469999999999999,"15min":2.794},"min":{"1min":0.23400000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":38.118000000000002,"5min":98.602999999999994,"15min":199.91200000000001},"last":1.206}]},{"osd":1,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.9140000000000001,"5min":3.2559999999999998,"15min":1.8640000000000001},"min":{"1min":0.23300000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":41.369,"5min":135.006,"15min":154.32599999999999},"last":1.236},{"interface":"front","average":{"1min":2.677,"5min":3.0489999999999999,"15min":1.8},"min":{"1min":0.23899999999999999,"5min":0.215,"15min":0.215},"max":{"1min":39.979999999999997,"5min":134.358,"15min":154.249},"last":1.329}]},{"osd":2,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":4.8099999999999996,"5min":2.3570000000000002,"15min":1.0640000000000001},"min":{"1min":0.28399999999999997,"5min":0.23000000000000001,"15min":0.224},"max":{"1min":55.238999999999997,"5min":55.238999999999997,"15min":55.238999999999997},"last":0.26400000000000001},{"interface":"front","average":{"1min":5.391,"5min":2.798,"15min":1.234},"min":{"1min":0.30499999999999999,"5min":0.215,"15min":0.20399999999999999},"max":{"1min":55.692999999999998,"5min":55.692999999999998,"15min":55.692999999999998},"last":1.3979999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.7130000000000001,"5min":3.9980000000000002,"15min":1.736},"min":{"1min":0.29499999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":40.078000000000003,"5min":154.00399999999999,"15min":154.00399999999999},"last":1.3680000000000001},{"interface":"front","average":{"1min":2.7810000000000001,"5min":4.3860000000000001,"15min":1.8839999999999999},"min":{"1min":0.30099999999999999,"5min":0.248,"15min":0.248},"max":{"1min":40.177,"5min":156.87700000000001,"15min":156.87700000000001},"last":1.2989999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.8980000000000001,"5min":3.3239999999999998,"15min":1.5289999999999999},"min":{"1min":0.32000000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":40.215000000000003,"5min":76.893000000000001,"15min":76.893000000000001},"last":3.298},{"interface":"front","average":{"1min":2.7029999999999998,"5min":3.1760000000000002,"15min":1.4930000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":38.43,"5min":76.959999999999994,"15min":76.959999999999994},"last":5.1379999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.9510000000000001,"5min":2.7130000000000001,"15min":1.351},"min":{"1min":0.38100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":40.124000000000002,"5min":40.124000000000002,"15min":40.124000000000002},"last":1.2629999999999999},{"interface":"front","average":{"1min":2.754,"5min":2.21,"15min":1.216},"min":{"1min":0.32600000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":38.459000000000003,"5min":38.459000000000003,"15min":38.459000000000003},"last":0.46400000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:20:27 2023","interfaces":[{"interface":"back","average":{"1min":2.677,"5min":2.6829999999999998,"15min":1.3600000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":38.478999999999999,"5min":79.412999999999997,"15min":79.412999999999997},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.7210000000000001,"5min":2.7839999999999998,"15min":1.379},"min":{"1min":0.36399999999999999,"5min":0.308,"15min":0.308},"max":{"1min":40.039999999999999,"5min":79.527000000000001,"15min":79.527000000000001},"last":1.431}]}]},{"osd":4,"up_from":30,"seq":128849019135,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2127924,"kb_used_data":1822444,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91617228,"statfs":{"total":95995035648,"available":93816041472,"internally_reserved":0,"allocated":1866182656,"data_stored":1865592965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:21:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":6.7160000000000002,"15min":3.2930000000000001},"min":{"1min":0.245,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.3040000000000003,"5min":133.91499999999999,"15min":133.91499999999999},"last":0.252},{"interface":"front","average":{"1min":0.96899999999999997,"5min":6.3609999999999998,"15min":3.1280000000000001},"min":{"1min":0.23599999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":3.2949999999999999,"5min":133.83199999999999,"15min":133.83199999999999},"last":0.33700000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:20:26 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":3.3420000000000001,"15min":1.5189999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.2380000000000004,"5min":133.93600000000001,"15min":133.93600000000001},"last":7.694},{"interface":"front","average":{"1min":0.82199999999999995,"5min":3.8290000000000002,"15min":1.6579999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.2290000000000001,"5min":141.06,"15min":141.06},"last":7.7060000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:20:58 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":3.1099999999999999,"15min":2.2519999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.3900000000000001,"5min":133.96100000000001,"15min":133.96100000000001},"last":0.29699999999999999},{"interface":"front","average":{"1min":1.375,"5min":2.9780000000000002,"15min":2.1930000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.4870000000000001,"5min":134.018,"15min":134.018},"last":0.379}]},{"osd":3,"last update":"Sun Oct 1 09:20:26 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":3.1459999999999999,"15min":1.431},"min":{"1min":0.32100000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.9909999999999997,"5min":134.297,"15min":134.297},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.879,"5min":3.5510000000000002,"15min":1.579},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.1320000000000001,"5min":141.089,"15min":141.089},"last":0.33000000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:20:26 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":4.1459999999999999,"15min":1.714},"min":{"1min":0.26500000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.0219999999999998,"5min":140.99100000000001,"15min":140.99100000000001},"last":4.298},{"interface":"front","average":{"1min":0.59499999999999997,"5min":3.0710000000000002,"15min":1.3879999999999999},"min":{"1min":0.318,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.5219999999999998,"5min":133.863,"15min":133.863},"last":4.3490000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:20:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":2.6309999999999998,"15min":1.2689999999999999},"min":{"1min":0.36899999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.806,"5min":133.90799999999999,"15min":133.90799999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.5669999999999999,"5min":3.3980000000000001,"15min":1.55},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.3250000000000002,"5min":141.03,"15min":141.03},"last":0.55900000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:21:06 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":3.3119999999999998,"15min":1.528},"min":{"1min":0.32400000000000001,"5min":0.31,"15min":0.31},"max":{"1min":4.399,"5min":133.946,"15min":133.946},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.054,"5min":3.048,"15min":1.4550000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.31},"max":{"1min":3.9569999999999999,"5min":133.98400000000001,"15min":133.98400000000001},"last":0.41199999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822897,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1769736,"kb_used_data":1460224,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91975416,"statfs":{"total":95995035648,"available":94182825984,"internally_reserved":0,"allocated":1495269376,"data_stored":1494746488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:20:15 2023","interfaces":[{"interface":"back","average":{"1min":4.7629999999999999,"5min":4.0609999999999999,"15min":1.843},"min":{"1min":0.253,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":82.219999999999999,"5min":82.219999999999999,"15min":82.219999999999999},"last":0.379},{"interface":"front","average":{"1min":4.79,"5min":4.0599999999999996,"15min":1.845},"min":{"1min":0.254,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":82.227999999999994,"5min":82.227999999999994,"15min":82.227999999999994},"last":0.27400000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:20:17 2023","interfaces":[{"interface":"back","average":{"1min":5.0110000000000001,"5min":4.3789999999999996,"15min":1.9359999999999999},"min":{"1min":0.26200000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":82.585999999999999,"5min":82.873000000000005,"15min":82.873000000000005},"last":1.276},{"interface":"front","average":{"1min":4.6970000000000001,"5min":3.9769999999999999,"15min":1.8100000000000001},"min":{"1min":0.35299999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":82.344999999999999,"5min":82.853999999999999,"15min":82.853999999999999},"last":0.42399999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:19:56 2023","interfaces":[{"interface":"back","average":{"1min":11.343,"5min":4.9340000000000002,"15min":2.0859999999999999},"min":{"1min":0.33300000000000002,"5min":0.25,"15min":0.25},"max":{"1min":139.56399999999999,"5min":139.56399999999999,"15min":139.56399999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":11.606999999999999,"5min":4.3529999999999998,"15min":1.863},"min":{"1min":0.30499999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":139.785,"5min":139.785,"15min":139.785},"last":0.39100000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:20:26 2023","interfaces":[{"interface":"back","average":{"1min":19.617000000000001,"5min":6.5869999999999997,"15min":2.7719999999999998},"min":{"1min":0.377,"5min":0.31,"15min":0.31},"max":{"1min":225.46199999999999,"5min":225.46199999999999,"15min":225.46199999999999},"last":1.605},{"interface":"front","average":{"1min":19.053000000000001,"5min":5.9530000000000003,"15min":2.4470000000000001},"min":{"1min":0.34100000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":225.40100000000001,"5min":225.40100000000001,"15min":225.40100000000001},"last":0.48599999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:20:15 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":4.056,"15min":1.6759999999999999},"min":{"1min":0.30499999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.25,"5min":193.26300000000001,"15min":193.26300000000001},"last":0.31},{"interface":"front","average":{"1min":0.72899999999999998,"5min":2.48,"15min":1.1879999999999999},"min":{"1min":0.34000000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.4820000000000002,"5min":15.432,"15min":15.432},"last":0.26100000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:19:40 2023","interfaces":[{"interface":"back","average":{"1min":2.302,"5min":2.669,"15min":1.246},"min":{"1min":0.317,"5min":0.30299999999999999,"15min":0.28599999999999998},"max":{"1min":13.015000000000001,"5min":15.551,"15min":15.551},"last":14.878},{"interface":"front","average":{"1min":3.5190000000000001,"5min":2.8559999999999999,"15min":1.3280000000000001},"min":{"1min":0.35199999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":16.548999999999999,"5min":16.548999999999999,"15min":16.548999999999999},"last":16.759}]},{"osd":7,"last update":"Sun Oct 1 09:20:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":4.4779999999999998,"15min":1.8839999999999999},"min":{"1min":0.37,"5min":0.34000000000000002,"15min":0.32300000000000001},"max":{"1min":9.9060000000000006,"5min":193.13800000000001,"15min":193.13800000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.74399999999999999,"5min":3.911,"15min":1.6990000000000001},"min":{"1min":0.36499999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.1779999999999999,"5min":193.203,"15min":193.203},"last":0.35299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1507069952,"data_stored":1507065880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1938427904,"data_stored":1938423832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3012493312,"data_stored":3012493312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1833500672,"data_stored":1833500672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1863385088,"data_stored":1863385088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1498480640,"data_stored":1498480640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4076994560,"data_stored":4076994560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1837694976,"data_stored":1837694976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:21:29.922 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-01T09:21:29.922 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_1 pg_num 26 2023-10-01T09:21:29.953 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:30.098 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:30.259 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:30.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:30.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:30.374 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:30.416 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:30.479 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:30.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:30.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:30.586 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:30.627 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:30.706 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:30.715 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:30 smithi190 ceph-mon[160435]: from='client.54955 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:21:30.715 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:30 smithi190 ceph-mon[160435]: pgmap v542: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 28471/269184 objects degraded (10.577%); 7/269184 objects misplaced (0.003%); 10 MiB/s, 161 objects/s recovering 2023-10-01T09:21:30.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:30.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:30.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:30 smithi138 ceph-mon[185886]: from='client.54955 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:21:30.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:30 smithi138 ceph-mon[185886]: pgmap v542: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 28471/269184 objects degraded (10.577%); 7/269184 objects misplaced (0.003%); 10 MiB/s, 161 objects/s recovering 2023-10-01T09:21:30.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:30 smithi138 ceph-mon[192513]: from='client.54955 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:21:30.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:30 smithi138 ceph-mon[192513]: pgmap v542: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 28471/269184 objects degraded (10.577%); 7/269184 objects misplaced (0.003%); 10 MiB/s, 161 objects/s recovering 2023-10-01T09:21:30.817 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:30.868 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:30.964 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:30.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:30.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:31.071 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:31.106 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:31.152 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:31.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:31.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:31.261 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:31.313 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:31.564 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:31.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:31.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:31.604 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:31.662 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:31.672 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:31.702 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:31.775 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:31.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:31.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:31.809 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:31.884 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:31.925 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:32.010 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:32.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:32.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:32.118 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:32.155 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:32.237 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:32.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:32.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:32.347 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:32.413 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:32.472 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:21:32.506 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:21:32.546 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:21:32.593 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:21:32.629 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:21:32.643 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:32.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:32.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:32.672 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:21:32.714 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:21:32.714 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:21:32.714 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:21:32.715 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:21:32.715 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:21:32.715 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:21:32.715 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-01T09:21:32.715 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:21:32.715 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:21:32.756 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:32.795 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:32 smithi138 ceph-mon[185886]: pgmap v543: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 28095/266946 objects degraded (10.525%); 7/266946 objects misplaced (0.003%); 11 MiB/s, 183 objects/s recovering 2023-10-01T09:21:32.795 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:32 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 28095/266946 objects degraded (10.525%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:32.795 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:32 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4125266781' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:32.795 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:32 smithi138 ceph-mon[192513]: pgmap v543: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 28095/266946 objects degraded (10.525%); 7/266946 objects misplaced (0.003%); 11 MiB/s, 183 objects/s recovering 2023-10-01T09:21:32.795 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:32 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 28095/266946 objects degraded (10.525%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:32.796 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:32 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4125266781' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:32.799 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:33.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:32 smithi190 ceph-mon[160435]: pgmap v543: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 28095/266946 objects degraded (10.525%); 7/266946 objects misplaced (0.003%); 11 MiB/s, 183 objects/s recovering 2023-10-01T09:21:33.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:32 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 28095/266946 objects degraded (10.525%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:33.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:32 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4125266781' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:33.902 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:33.907 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:33 smithi138 ceph-mon[192513]: pgmap v544: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 27504/259866 objects degraded (10.584%); 7/259866 objects misplaced (0.003%); 7.3 MiB/s, 117 objects/s recovering 2023-10-01T09:21:33.908 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:33 smithi138 ceph-mon[185886]: pgmap v544: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 27504/259866 objects degraded (10.584%); 7/259866 objects misplaced (0.003%); 7.3 MiB/s, 117 objects/s recovering 2023-10-01T09:21:33.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:33.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:34.009 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:34.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:33 smithi190 ceph-mon[160435]: pgmap v544: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 27504/259866 objects degraded (10.584%); 7/259866 objects misplaced (0.003%); 7.3 MiB/s, 117 objects/s recovering 2023-10-01T09:21:34.087 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:34.566 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:34.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:34.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:34.672 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:34.718 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:34.802 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:34.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:34.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:34.908 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:34.945 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:35.056 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:35.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:35.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:35.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:21:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:21:34] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.33.4" 2023-10-01T09:21:35.164 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:35.227 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:35.972 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:35.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:35.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:36.077 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:36.119 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:36.131 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:21:36.131 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:21:36.191 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:36.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:36.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:36.301 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:36.378 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:36.596 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:36.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:36.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:36.629 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:36 smithi138 ceph-mon[185886]: pgmap v545: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 27084/257352 objects degraded (10.524%); 7/257352 objects misplaced (0.003%); 7.6 MiB/s, 121 objects/s recovering 2023-10-01T09:21:36.629 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:36 smithi138 ceph-mon[192513]: pgmap v545: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 27084/257352 objects degraded (10.524%); 7/257352 objects misplaced (0.003%); 7.6 MiB/s, 121 objects/s recovering 2023-10-01T09:21:36.711 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:36.798 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:36.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:36 smithi190 ceph-mon[160435]: pgmap v545: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 27084/257352 objects degraded (10.524%); 7/257352 objects misplaced (0.003%); 7.6 MiB/s, 121 objects/s recovering 2023-10-01T09:21:36.978 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:21:36.978 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:21:36.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.978 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35070.0:107664 6.c e217/208 6:3a38f1e9:::benchmark_data_smithi138_307751_object12295:head v 217'6718, mlcod=187'756)", 2023-10-01T09:21:36.979 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:36.950800+0000", 2023-10-01T09:21:36.979 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.023945587000000001, 2023-10-01T09:21:36.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.023977845000000001, 2023-10-01T09:21:36.979 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:36.979 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:21:36.979 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:36.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:36.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.950800+0000", 2023-10-01T09:21:36.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:36.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:36.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.950800+0000", 2023-10-01T09:21:36.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:36.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:36.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.950799+0000", 2023-10-01T09:21:36.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:21:36.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:36.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.950814+0000", 2023-10-01T09:21:36.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4865000000000001e-05 2023-10-01T09:21:36.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:36.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.950826+0000", 2023-10-01T09:21:36.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2666e-05 2023-10-01T09:21:36.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:36.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.950834+0000", 2023-10-01T09:21:36.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5490000000000001e-06 2023-10-01T09:21:36.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.982 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:21:36.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.961687+0000", 2023-10-01T09:21:36.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010852911 2023-10-01T09:21:36.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.982 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:21:36.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.961714+0000", 2023-10-01T09:21:36.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7402e-05 2023-10-01T09:21:36.983 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:36.983 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:36.983 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:36.983 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.983 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.983 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(6.c 217/208 [PushOp(6:3ec9cef8:::benchmark_data_smithi138_307751_object12331:head, version: 187'759, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:3ec9cef8:::benchmark_data_smithi138_307751_object12331:head@187'759, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-01T09:21:36.983 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:36.961062+0000", 2023-10-01T09:21:36.983 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.013683305999999999, 2023-10-01T09:21:36.983 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013811574, 2023-10-01T09:21:36.983 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:36.983 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:21:36.983 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:36.983 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:36.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.961062+0000", 2023-10-01T09:21:36.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:36.984 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:36.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.961062+0000", 2023-10-01T09:21:36.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:36.984 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:36.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.961061+0000", 2023-10-01T09:21:36.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:21:36.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:36.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.961117+0000", 2023-10-01T09:21:36.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6898999999999999e-05 2023-10-01T09:21:36.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:36.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.961148+0000", 2023-10-01T09:21:36.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0205999999999998e-05 2023-10-01T09:21:36.986 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.986 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:36.986 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.961164+0000", 2023-10-01T09:21:36.986 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6738000000000001e-05 2023-10-01T09:21:36.986 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:36.986 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:36.986 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:36.986 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.986 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.35070.0:107667 6.4 e217/208 ondisk, result = 0)", 2023-10-01T09:21:36.986 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:36.961180+0000", 2023-10-01T09:21:36.987 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.013565569, 2023-10-01T09:21:36.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013739800999999999, 2023-10-01T09:21:36.987 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:36.987 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:21:36.987 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:36.987 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:36.987 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.961180+0000", 2023-10-01T09:21:36.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:36.987 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.987 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:36.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.961180+0000", 2023-10-01T09:21:36.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:36.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:36.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.961179+0000", 2023-10-01T09:21:36.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:21:36.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:36.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.961183+0000", 2023-10-01T09:21:36.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5760000000000002e-06 2023-10-01T09:21:36.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:36.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.961189+0000", 2023-10-01T09:21:36.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8579999999999996e-06 2023-10-01T09:21:36.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:36.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.961193+0000", 2023-10-01T09:21:36.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3930000000000001e-06 2023-10-01T09:21:36.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:21:36.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.971780+0000", 2023-10-01T09:21:36.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010587116000000001 2023-10-01T09:21:36.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:21:36.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.971791+0000", 2023-10-01T09:21:36.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1097999999999999e-05 2023-10-01T09:21:36.990 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:36.990 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:36.991 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:36.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.991 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:107696 6.4 6.78e57934 (undecoded) ondisk+write+known_if_redirected e217)", 2023-10-01T09:21:36.991 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:36.962816+0000", 2023-10-01T09:21:36.991 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.011928909, 2023-10-01T09:21:36.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012160516, 2023-10-01T09:21:36.991 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:36.991 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:21:36.991 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:21:36.991 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:21:36.991 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:21:36.992 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 107696 2023-10-01T09:21:36.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.992 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:36.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:36.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.962816+0000", 2023-10-01T09:21:36.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:36.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:36.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.962816+0000", 2023-10-01T09:21:36.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:36.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:36.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.962815+0000", 2023-10-01T09:21:36.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:21:36.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:36.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.962823+0000", 2023-10-01T09:21:36.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8949999999999998e-06 2023-10-01T09:21:36.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:36.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.962838+0000", 2023-10-01T09:21:36.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5063e-05 2023-10-01T09:21:36.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:36.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.962847+0000", 2023-10-01T09:21:36.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1800000000000002e-06 2023-10-01T09:21:36.994 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:36.995 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:36.995 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:36.995 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.995 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.35070.0:107677 6.4 e217/208)", 2023-10-01T09:21:36.995 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:36.966409+0000", 2023-10-01T09:21:36.995 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0083363640000000006, 2023-10-01T09:21:36.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086213599999999998, 2023-10-01T09:21:36.995 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:36.995 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:21:36.995 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:36.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:36.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.966409+0000", 2023-10-01T09:21:36.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:36.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:36.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.966409+0000", 2023-10-01T09:21:36.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:36.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:36.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.966408+0000", 2023-10-01T09:21:36.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:21:36.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:36.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.966421+0000", 2023-10-01T09:21:36.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.358e-05 2023-10-01T09:21:36.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:36.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.966438+0000", 2023-10-01T09:21:36.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6697999999999998e-05 2023-10-01T09:21:36.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:36.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.966447+0000", 2023-10-01T09:21:36.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0880000000000001e-06 2023-10-01T09:21:36.998 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:36.998 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:36.998 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:36.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.998 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.35070.0:107677 6.4 e217/208)", 2023-10-01T09:21:36.998 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:36.968259+0000", 2023-10-01T09:21:36.998 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0064858670000000002, 2023-10-01T09:21:36.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0068139170000000001, 2023-10-01T09:21:36.998 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:36.999 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:21:36.999 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:36.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.999 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:36.999 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.968259+0000", 2023-10-01T09:21:36.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:36.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:36.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:36.999 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:36.999 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.968259+0000", 2023-10-01T09:21:36.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.000 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:37.000 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.968258+0000", 2023-10-01T09:21:37.000 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:21:37.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.000 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:37.000 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.968265+0000", 2023-10-01T09:21:37.000 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.759e-06 2023-10-01T09:21:37.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:37.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.968280+0000", 2023-10-01T09:21:37.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4691999999999999e-05 2023-10-01T09:21:37.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:37.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.968287+0000", 2023-10-01T09:21:37.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3869999999999997e-06 2023-10-01T09:21:37.001 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.001 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:37.001 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.002 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.002 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:107667 6.4 6:20870af7:::benchmark_data_smithi138_307751_object12298:head [delete] snapc 0=[] ondisk+write+known_if_redirected e217)", 2023-10-01T09:21:37.002 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:36.911386+0000", 2023-10-01T09:21:37.002 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.063358838000000001, 2023-10-01T09:21:37.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.063727779999999998, 2023-10-01T09:21:37.002 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:37.002 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:21:37.002 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:21:37.002 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:21:37.002 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:21:37.002 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 107667 2023-10-01T09:21:37.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.003 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:37.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.003 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:37.003 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.911386+0000", 2023-10-01T09:21:37.003 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.003 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:37.003 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.911386+0000", 2023-10-01T09:21:37.003 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.004 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.004 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:37.004 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.911385+0000", 2023-10-01T09:21:37.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:21:37.004 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.004 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.004 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:37.004 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.911391+0000", 2023-10-01T09:21:37.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4049999999999997e-06 2023-10-01T09:21:37.004 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:37.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.911406+0000", 2023-10-01T09:21:37.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4286e-05 2023-10-01T09:21:37.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:37.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.911415+0000", 2023-10-01T09:21:37.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6029999999999994e-06 2023-10-01T09:21:37.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:21:37.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.929469+0000", 2023-10-01T09:21:37.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018053479000000001 2023-10-01T09:21:37.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:21:37.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.929563+0000", 2023-10-01T09:21:37.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4574999999999995e-05 2023-10-01T09:21:37.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:21:37.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.929694+0000", 2023-10-01T09:21:37.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000130748 2023-10-01T09:21:37.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.007 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:21:37.007 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.942246+0000", 2023-10-01T09:21:37.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012552153999999999 2023-10-01T09:21:37.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.007 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:21:37.007 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.943621+0000", 2023-10-01T09:21:37.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013748530000000001 2023-10-01T09:21:37.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:21:37.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.971800+0000", 2023-10-01T09:21:37.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.028178695 2023-10-01T09:21:37.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:21:37.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.971862+0000", 2023-10-01T09:21:37.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2169999999999996e-05 2023-10-01T09:21:37.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:21:37.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.975114+0000", 2023-10-01T09:21:37.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032523090000000001 2023-10-01T09:21:37.009 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.009 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:37.009 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.009 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.35070.0:107690 6.4 e217/208)", 2023-10-01T09:21:37.009 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:36.974692+0000", 2023-10-01T09:21:37.009 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 5.3678999999999999e-05, 2023-10-01T09:21:37.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00053715999999999996, 2023-10-01T09:21:37.009 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:37.010 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:21:37.010 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:37.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:37.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.974692+0000", 2023-10-01T09:21:37.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:37.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.974692+0000", 2023-10-01T09:21:37.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:37.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.974691+0000", 2023-10-01T09:21:37.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:21:37.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:37.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.974696+0000", 2023-10-01T09:21:37.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3140000000000003e-06 2023-10-01T09:21:37.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:37.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.974711+0000", 2023-10-01T09:21:37.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4707e-05 2023-10-01T09:21:37.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.012 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:37.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.974719+0000", 2023-10-01T09:21:37.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7390000000000005e-06 2023-10-01T09:21:37.012 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.012 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:37.012 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.013 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:107701 6.4 6.65d6b674 (undecoded) ondisk+write+known_if_redirected e217)", 2023-10-01T09:21:37.013 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:36.974738+0000", 2023-10-01T09:21:37.013 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 7.2030000000000003e-06, 2023-10-01T09:21:37.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00054119200000000002, 2023-10-01T09:21:37.013 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:37.013 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:21:37.013 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:21:37.013 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:21:37.013 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:21:37.013 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 107701 2023-10-01T09:21:37.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.014 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:37.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:37.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.974738+0000", 2023-10-01T09:21:37.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:37.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.974738+0000", 2023-10-01T09:21:37.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:37.015 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.974737+0000", 2023-10-01T09:21:37.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:21:37.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:37.015 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.974742+0000", 2023-10-01T09:21:37.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8919999999999999e-06 2023-10-01T09:21:37.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:37.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.974750+0000", 2023-10-01T09:21:37.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8049999999999996e-06 2023-10-01T09:21:37.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:37.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.974756+0000", 2023-10-01T09:21:37.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6509999999999998e-06 2023-10-01T09:21:37.016 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.016 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:37.016 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.017 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.35070.0:107690 6.4 e217/208)", 2023-10-01T09:21:37.017 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:36.975247+0000", 2023-10-01T09:21:37.017 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 4294967295.9994984, 2023-10-01T09:21:37.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3078000000000007e-05, 2023-10-01T09:21:37.017 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:37.017 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:21:37.017 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:37.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:37.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.975247+0000", 2023-10-01T09:21:37.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:37.018 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.975247+0000", 2023-10-01T09:21:37.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:37.018 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.975246+0000", 2023-10-01T09:21:37.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:21:37.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:37.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.975252+0000", 2023-10-01T09:21:37.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3800000000000002e-06 2023-10-01T09:21:37.019 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:37.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.975261+0000", 2023-10-01T09:21:37.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.628e-06 2023-10-01T09:21:37.019 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:37.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.975267+0000", 2023-10-01T09:21:37.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.891e-06 2023-10-01T09:21:37.020 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.020 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:37.020 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.020 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:107672 6.4 6:2500dfe4:::benchmark_data_smithi138_307751_object12303:head [delete] snapc 0=[] ondisk+write+known_if_redirected e217)", 2023-10-01T09:21:37.020 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:36.917321+0000", 2023-10-01T09:21:37.020 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.057424317000000002, 2023-10-01T09:21:37.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.058058312000000001, 2023-10-01T09:21:37.021 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:37.021 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "delayed", 2023-10-01T09:21:37.021 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:21:37.021 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:21:37.021 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:21:37.021 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 107672 2023-10-01T09:21:37.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.021 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:37.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:37.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.917321+0000", 2023-10-01T09:21:37.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:37.022 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.917321+0000", 2023-10-01T09:21:37.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:37.022 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.917319+0000", 2023-10-01T09:21:37.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:21:37.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:37.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.917326+0000", 2023-10-01T09:21:37.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3849999999999996e-06 2023-10-01T09:21:37.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:37.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.917342+0000", 2023-10-01T09:21:37.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.588e-05 2023-10-01T09:21:37.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:37.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.917352+0000", 2023-10-01T09:21:37.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2019999999999993e-06 2023-10-01T09:21:37.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:21:37.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.932687+0000", 2023-10-01T09:21:37.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015335720000000001 2023-10-01T09:21:37.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:21:37.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.932756+0000", 2023-10-01T09:21:37.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9215999999999996e-05 2023-10-01T09:21:37.025 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.025 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:37.025 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.025 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.025 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.025 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPushReply(6.4 217/208 [PushReplyOp(6:2500dfe4:::benchmark_data_smithi138_307751_object12303:head)])", 2023-10-01T09:21:37.025 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:36.918270+0000", 2023-10-01T09:21:37.025 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.056475466000000002, 2023-10-01T09:21:37.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.057173493999999998, 2023-10-01T09:21:37.025 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:37.025 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:21:37.026 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:37.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:37.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.918270+0000", 2023-10-01T09:21:37.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:37.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.918270+0000", 2023-10-01T09:21:37.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:37.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.918269+0000", 2023-10-01T09:21:37.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:21:37.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:37.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.918280+0000", 2023-10-01T09:21:37.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1279e-05 2023-10-01T09:21:37.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:37.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.918292+0000", 2023-10-01T09:21:37.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.189e-05 2023-10-01T09:21:37.028 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.028 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.028 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:37.028 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.918300+0000", 2023-10-01T09:21:37.028 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1210000000000007e-06 2023-10-01T09:21:37.028 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.028 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:37.028 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.028 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.028 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.028 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:107677 6.4 6:249c0148:::benchmark_data_smithi138_307751_object12308:head [delete] snapc 0=[] ondisk+write+known_if_redirected e217)", 2023-10-01T09:21:37.028 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:36.923559+0000", 2023-10-01T09:21:37.028 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.051186422000000002, 2023-10-01T09:21:37.029 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.051932886999999997, 2023-10-01T09:21:37.029 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:37.029 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:21:37.029 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:21:37.029 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:21:37.029 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:21:37.029 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 107677 2023-10-01T09:21:37.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.029 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:37.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.029 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:37.029 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.923559+0000", 2023-10-01T09:21:37.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:37.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.923559+0000", 2023-10-01T09:21:37.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:37.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.923558+0000", 2023-10-01T09:21:37.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:21:37.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:37.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.923561+0000", 2023-10-01T09:21:37.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8370000000000001e-06 2023-10-01T09:21:37.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:37.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.923569+0000", 2023-10-01T09:21:37.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0099999999999995e-06 2023-10-01T09:21:37.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:37.032 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.923574+0000", 2023-10-01T09:21:37.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9840000000000001e-06 2023-10-01T09:21:37.032 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.032 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:21:37.032 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.940550+0000", 2023-10-01T09:21:37.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016975635999999999 2023-10-01T09:21:37.032 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.032 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:21:37.032 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.942184+0000", 2023-10-01T09:21:37.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001633808 2023-10-01T09:21:37.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:21:37.033 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.958004+0000", 2023-10-01T09:21:37.033 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015819863999999999 2023-10-01T09:21:37.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:21:37.033 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.958094+0000", 2023-10-01T09:21:37.033 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9969999999999994e-05 2023-10-01T09:21:37.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.034 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.034 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:21:37.034 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.958345+0000", 2023-10-01T09:21:37.034 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025137599999999999 2023-10-01T09:21:37.034 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.034 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.034 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:21:37.034 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.975323+0000", 2023-10-01T09:21:37.034 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016978147999999998 2023-10-01T09:21:37.034 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.034 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:21:37.035 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.975361+0000", 2023-10-01T09:21:37.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.807e-05 2023-10-01T09:21:37.035 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.035 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:37.035 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.035 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:107690 6.4 6:2f6c95ea:::benchmark_data_smithi138_307751_object12321:head [delete] snapc 0=[] ondisk+write+known_if_redirected e217)", 2023-10-01T09:21:37.035 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:36.942813+0000", 2023-10-01T09:21:37.035 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.031931901999999998, 2023-10-01T09:21:37.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.032771203999999998, 2023-10-01T09:21:37.036 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:37.036 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:21:37.036 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:21:37.036 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:21:37.036 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:21:37.036 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 107690 2023-10-01T09:21:37.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.036 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:37.036 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:37.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.942813+0000", 2023-10-01T09:21:37.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.037 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:37.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.942813+0000", 2023-10-01T09:21:37.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:37.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.037 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:37.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.942812+0000", 2023-10-01T09:21:37.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:21:37.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.038 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:37.038 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.942819+0000", 2023-10-01T09:21:37.038 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5340000000000002e-06 2023-10-01T09:21:37.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.038 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.038 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:37.038 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.942831+0000", 2023-10-01T09:21:37.038 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1547e-05 2023-10-01T09:21:37.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.038 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.038 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:37.038 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.942838+0000", 2023-10-01T09:21:37.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6070000000000002e-06 2023-10-01T09:21:37.039 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.039 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.039 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:21:37.039 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.943645+0000", 2023-10-01T09:21:37.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00080613699999999996 2023-10-01T09:21:37.039 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.039 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.039 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:21:37.039 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.944296+0000", 2023-10-01T09:21:37.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00065121299999999999 2023-10-01T09:21:37.039 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.039 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.040 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:21:37.040 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.968573+0000", 2023-10-01T09:21:37.040 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024277396 2023-10-01T09:21:37.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.040 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:21:37.040 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.968668+0000", 2023-10-01T09:21:37.040 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4990999999999999e-05 2023-10-01T09:21:37.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.040 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:21:37.040 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.968800+0000", 2023-10-01T09:21:37.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000132378 2023-10-01T09:21:37.041 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.041 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.041 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:21:37.041 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.975395+0000", 2023-10-01T09:21:37.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0065946379999999999 2023-10-01T09:21:37.041 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:37.041 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:37.041 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:21:37.041 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:36.975431+0000", 2023-10-01T09:21:37.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5743e-05 2023-10-01T09:21:37.041 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.041 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:37.042 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.042 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:37.042 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:21:37.042 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 14 2023-10-01T09:21:37.042 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:21:37.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:37.043 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:37.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:37.144 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:37.176 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:37.277 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:37.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:37.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:37.390 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:37.433 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:37.518 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:37.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:37.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:37.545 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:21:37.629 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:37.686 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:37.790 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:37.800 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:37 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2443891862' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:37.800 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:37 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:37.800 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:37 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 27084/257352 objects degraded (10.524%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:37.801 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:37 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2443891862' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:37.801 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:37 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:37.801 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:37 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 27084/257352 objects degraded (10.524%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:37.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:37.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:37 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2443891862' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:37 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:37 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 27084/257352 objects degraded (10.524%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:21:37.902 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:37.958 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:38.034 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:38.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:38.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:38.132 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:21:38.145 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:38.278 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:38.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[185886]: pgmap v546: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 26897/252315 objects degraded (10.660%); 7/252315 objects misplaced (0.003%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T09:21:38.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:21:38.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:21:38.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[185886]: osdmap e218: 8 total, 7 up, 8 in 2023-10-01T09:21:38.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[185886]: failed to encode map e218 with expected crc 2023-10-01T09:21:38.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[185886]: failed to encode map e218 with expected crc 2023-10-01T09:21:38.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[185886]: failed to encode map e218 with expected crc 2023-10-01T09:21:38.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2225379715' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T09:21:38.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T09:21:38.521 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[192513]: pgmap v546: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 26897/252315 objects degraded (10.660%); 7/252315 objects misplaced (0.003%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T09:21:38.521 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:21:38.521 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:21:38.521 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[192513]: osdmap e218: 8 total, 7 up, 8 in 2023-10-01T09:21:38.521 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[192513]: failed to encode map e218 with expected crc 2023-10-01T09:21:38.546 INFO:teuthology.orchestra.run.smithi138.stderr:set pool 6 pg_num to 26 2023-10-01T09:21:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:38 smithi190 ceph-mon[160435]: pgmap v546: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 26897/252315 objects degraded (10.660%); 7/252315 objects misplaced (0.003%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T09:21:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:38 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:21:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:38 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:21:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:38 smithi190 ceph-mon[160435]: osdmap e218: 8 total, 7 up, 8 in 2023-10-01T09:21:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:38 smithi190 ceph-mon[160435]: failed to encode map e218 with expected crc 2023-10-01T09:21:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:38 smithi190 ceph-mon[160435]: failed to encode map e218 with expected crc 2023-10-01T09:21:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:38 smithi190 ceph-mon[160435]: failed to encode map e218 with expected crc 2023-10-01T09:21:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2225379715' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T09:21:38.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:38 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T09:21:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[192513]: failed to encode map e218 with expected crc 2023-10-01T09:21:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[192513]: failed to encode map e218 with expected crc 2023-10-01T09:21:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2225379715' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T09:21:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:38 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T09:21:39.274 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:21:39.313 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:39.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:39.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:39.418 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:39.447 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:39.516 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:39.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:39.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:39.625 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:39.723 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:39 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-10-01T09:21:39.723 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:39 smithi190 ceph-mon[160435]: osdmap e219: 8 total, 7 up, 8 in 2023-10-01T09:21:39.723 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:39 smithi190 ceph-mon[160435]: failed to encode map e219 with expected crc 2023-10-01T09:21:39.723 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:39 smithi190 ceph-mon[160435]: failed to encode map e219 with expected crc 2023-10-01T09:21:39.723 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:39 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-01T09:21:39.723 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:39.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:39 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-10-01T09:21:39.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:39 smithi138 ceph-mon[185886]: osdmap e219: 8 total, 7 up, 8 in 2023-10-01T09:21:39.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:39 smithi138 ceph-mon[185886]: failed to encode map e219 with expected crc 2023-10-01T09:21:39.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:39 smithi138 ceph-mon[185886]: failed to encode map e219 with expected crc 2023-10-01T09:21:39.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:39 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-01T09:21:39.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:39 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-10-01T09:21:39.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:39 smithi138 ceph-mon[192513]: osdmap e219: 8 total, 7 up, 8 in 2023-10-01T09:21:39.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:39 smithi138 ceph-mon[192513]: failed to encode map e219 with expected crc 2023-10-01T09:21:39.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:39 smithi138 ceph-mon[192513]: failed to encode map e219 with expected crc 2023-10-01T09:21:39.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:39 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-01T09:21:39.839 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:39.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:39.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:39.946 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:39.963 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:39.974 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:39.998 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:40.022 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:40.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:40.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:40.128 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:40.155 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:40.291 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:40.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:40.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:40.394 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:40.423 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:40.467 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:40.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:40.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:40.573 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:40.615 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:40.673 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:40.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:40.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:40.683 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:40 smithi190 ceph-mon[160435]: pgmap v549: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 26460/249747 objects degraded (10.595%); 7/249747 objects misplaced (0.003%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T09:21:40.683 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-01T09:21:40.683 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:40 smithi190 ceph-mon[160435]: osdmap e220: 8 total, 7 up, 8 in 2023-10-01T09:21:40.683 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:40 smithi190 ceph-mon[160435]: failed to encode map e220 with expected crc 2023-10-01T09:21:40.683 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:40.683 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:21:40.783 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:40.803 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:40 smithi138 ceph-mon[185886]: pgmap v549: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 26460/249747 objects degraded (10.595%); 7/249747 objects misplaced (0.003%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T09:21:40.803 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-01T09:21:40.803 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:40 smithi138 ceph-mon[185886]: osdmap e220: 8 total, 7 up, 8 in 2023-10-01T09:21:40.803 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:40 smithi138 ceph-mon[185886]: failed to encode map e220 with expected crc 2023-10-01T09:21:40.803 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:40.803 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:21:40.804 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:40 smithi138 ceph-mon[192513]: pgmap v549: 89 pgs: 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 54 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 26460/249747 objects degraded (10.595%); 7/249747 objects misplaced (0.003%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T09:21:40.804 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-01T09:21:40.804 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:40 smithi138 ceph-mon[192513]: osdmap e220: 8 total, 7 up, 8 in 2023-10-01T09:21:40.804 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:40 smithi138 ceph-mon[192513]: failed to encode map e220 with expected crc 2023-10-01T09:21:40.804 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:40.804 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:21:40.821 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:41.219 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:41.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:41.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:41.323 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:41.349 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:41.728 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:21:41.789 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:41 smithi138 ceph-mon[192513]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:41 smithi138 ceph-mon[192513]: osdmap e221: 8 total, 7 up, 8 in 2023-10-01T09:21:41.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:41 smithi138 ceph-mon[192513]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:41 smithi138 ceph-mon[192513]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:41 smithi138 ceph-mon[192513]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:41 smithi138 ceph-mon[192513]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:41 smithi138 ceph-mon[192513]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:41 smithi138 ceph-mon[185886]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:41 smithi138 ceph-mon[185886]: osdmap e221: 8 total, 7 up, 8 in 2023-10-01T09:21:41.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:41 smithi138 ceph-mon[185886]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:41 smithi138 ceph-mon[185886]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:41 smithi138 ceph-mon[185886]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:41 smithi138 ceph-mon[185886]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:41 smithi138 ceph-mon[185886]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:41 smithi190 ceph-mon[160435]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:41 smithi190 ceph-mon[160435]: osdmap e221: 8 total, 7 up, 8 in 2023-10-01T09:21:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:41 smithi190 ceph-mon[160435]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:41 smithi190 ceph-mon[160435]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:41 smithi190 ceph-mon[160435]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:41 smithi190 ceph-mon[160435]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:41 smithi190 ceph-mon[160435]: failed to encode map e221 with expected crc 2023-10-01T09:21:41.852 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:41.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:41.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:41.960 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:41.989 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:42.038 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:42.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:42.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:42.141 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:42.168 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:42.218 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:42.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:42.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:42.263 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:21:42.263 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:21:42.263 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:21:42.264 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:21:42.264 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:21:42.326 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:42.362 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:42.411 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:42.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:42.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:42.481 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:21:42.501 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:42.502 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:21:42.517 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:42.526 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:21:42.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:42 smithi138 ceph-mon[185886]: pgmap v552: 99 pgs: 1 peering, 10 unknown, 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 53 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 26054/247359 objects degraded (10.533%); 7/247359 objects misplaced (0.003%); 9.2 MiB/s, 147 objects/s recovering 2023-10-01T09:21:42.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:42 smithi138 ceph-mon[185886]: osdmap e222: 8 total, 7 up, 8 in 2023-10-01T09:21:42.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:42 smithi138 ceph-mon[185886]: failed to encode map e222 with expected crc 2023-10-01T09:21:42.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:42 smithi138 ceph-mon[185886]: failed to encode map e222 with expected crc 2023-10-01T09:21:42.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:42 smithi138 ceph-mon[185886]: failed to encode map e222 with expected crc 2023-10-01T09:21:42.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:42 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 26054/247359 objects degraded (10.533%), 10 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-01T09:21:42.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:42 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2982380972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:42.551 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:42.551 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:21:42.578 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:21:42.608 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:21:42.635 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:21:42.635 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:21:42.636 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:21:42.636 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:21:42.636 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:21:42.636 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:21:42.636 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-01T09:21:42.636 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:21:42.636 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:21:42.728 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:42.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:42.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:42.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:42 smithi190 ceph-mon[160435]: pgmap v552: 99 pgs: 1 peering, 10 unknown, 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 53 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 26054/247359 objects degraded (10.533%); 7/247359 objects misplaced (0.003%); 9.2 MiB/s, 147 objects/s recovering 2023-10-01T09:21:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:42 smithi190 ceph-mon[160435]: osdmap e222: 8 total, 7 up, 8 in 2023-10-01T09:21:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:42 smithi190 ceph-mon[160435]: failed to encode map e222 with expected crc 2023-10-01T09:21:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:42 smithi190 ceph-mon[160435]: failed to encode map e222 with expected crc 2023-10-01T09:21:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:42 smithi190 ceph-mon[160435]: failed to encode map e222 with expected crc 2023-10-01T09:21:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:42 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 26054/247359 objects degraded (10.533%), 10 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-01T09:21:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:42 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2982380972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:42.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:42 smithi138 ceph-mon[192513]: pgmap v552: 99 pgs: 1 peering, 10 unknown, 15 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 53 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 26054/247359 objects degraded (10.533%); 7/247359 objects misplaced (0.003%); 9.2 MiB/s, 147 objects/s recovering 2023-10-01T09:21:42.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:42 smithi138 ceph-mon[192513]: osdmap e222: 8 total, 7 up, 8 in 2023-10-01T09:21:42.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:42 smithi138 ceph-mon[192513]: failed to encode map e222 with expected crc 2023-10-01T09:21:42.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:42 smithi138 ceph-mon[192513]: failed to encode map e222 with expected crc 2023-10-01T09:21:42.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:42 smithi138 ceph-mon[192513]: failed to encode map e222 with expected crc 2023-10-01T09:21:42.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:42 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 26054/247359 objects degraded (10.533%), 10 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-10-01T09:21:42.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:42 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2982380972' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:42.834 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:42.860 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:43.413 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:43.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:43.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:43.523 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:43.570 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:43.636 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:43.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:43.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:43.683 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:43 smithi138 ceph-mon[192513]: osdmap e223: 8 total, 7 up, 8 in 2023-10-01T09:21:43.683 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:43 smithi138 ceph-mon[192513]: failed to encode map e223 with expected crc 2023-10-01T09:21:43.684 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:43 smithi138 ceph-mon[192513]: failed to encode map e223 with expected crc 2023-10-01T09:21:43.685 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:21:43.687 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:43 smithi138 ceph-mon[185886]: osdmap e223: 8 total, 7 up, 8 in 2023-10-01T09:21:43.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:43 smithi138 ceph-mon[185886]: failed to encode map e223 with expected crc 2023-10-01T09:21:43.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:43 smithi138 ceph-mon[185886]: failed to encode map e223 with expected crc 2023-10-01T09:21:43.743 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:43.796 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:44.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:43 smithi190 ceph-mon[160435]: osdmap e223: 8 total, 7 up, 8 in 2023-10-01T09:21:44.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:43 smithi190 ceph-mon[160435]: failed to encode map e223 with expected crc 2023-10-01T09:21:44.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:43 smithi190 ceph-mon[160435]: failed to encode map e223 with expected crc 2023-10-01T09:21:44.390 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:21:44.390 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:21:44.490 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:44.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:44.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:44.597 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:44.646 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:44 smithi138 ceph-mon[192513]: pgmap v555: 99 pgs: 2 active+recovery_wait+degraded, 1 peering, 3 unknown, 15 active+clean+remapped, 1 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 57 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 25941/244350 objects degraded (10.616%); 7/244350 objects misplaced (0.003%); 6.0 MiB/s, 96 objects/s recovering 2023-10-01T09:21:44.647 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:44 smithi138 ceph-mon[185886]: pgmap v555: 99 pgs: 2 active+recovery_wait+degraded, 1 peering, 3 unknown, 15 active+clean+remapped, 1 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 57 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 25941/244350 objects degraded (10.616%); 7/244350 objects misplaced (0.003%); 6.0 MiB/s, 96 objects/s recovering 2023-10-01T09:21:44.655 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:44.739 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:44.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:44.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:44.849 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:44.897 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:44 smithi190 ceph-mon[160435]: pgmap v555: 99 pgs: 2 active+recovery_wait+degraded, 1 peering, 3 unknown, 15 active+clean+remapped, 1 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 3 active+recovering+degraded+remapped, 57 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 25941/244350 objects degraded (10.616%); 7/244350 objects misplaced (0.003%); 6.0 MiB/s, 96 objects/s recovering 2023-10-01T09:21:44.904 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:44.922 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 7, 2, 6] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 6, 7] 2023-10-01T09:21:44.922 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:21:44.923 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2023-10-01T09:21:44.923 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2023-10-01T09:21:44.923 DEBUG:teuthology.orchestra.run.smithi190:osd.5> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-01T09:21:44.934 DEBUG:teuthology.orchestra.run.smithi190:> sudo systemctl start ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.5 2023-10-01T09:21:44.937 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:21:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:21:44] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.33.4" 2023-10-01T09:21:44.991 INFO:journalctl@ceph.osd.5.smithi190.stdout:-- Logs begin at Sun 2023-10-01 08:38:18 UTC. -- 2023-10-01T09:21:45.004 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:45.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:45.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:45.111 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:45.155 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:21:45 smithi190 systemd[1]: Starting Ceph osd.5 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:21:45.161 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:45.261 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:45.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:45.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:45.348 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:45.371 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:45.456 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:45.744 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:45.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:45.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:45.853 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:45.911 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:46.022 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:46.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:46.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:46.125 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:46.184 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:46.269 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:46.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:46.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:46.373 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:46.391 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:21:46.427 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:46.546 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:46.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:46.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:46.607 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:21:46.655 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:46.694 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:46 smithi138 ceph-mon[192513]: pgmap v556: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 25134/241206 objects degraded (10.420%); 284/241206 objects misplaced (0.118%); 11 MiB/s, 182 objects/s recovering 2023-10-01T09:21:46.694 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:46 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/852692111' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:21:46.694 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:46 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:21:46.694 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:46 smithi138 ceph-mon[185886]: pgmap v556: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 25134/241206 objects degraded (10.420%); 284/241206 objects misplaced (0.118%); 11 MiB/s, 182 objects/s recovering 2023-10-01T09:21:46.695 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:46 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/852692111' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:21:46.695 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:46 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:21:46.699 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:46.753 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:46.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:46.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:46 smithi190 ceph-mon[160435]: pgmap v556: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 25134/241206 objects degraded (10.420%); 284/241206 objects misplaced (0.118%); 11 MiB/s, 182 objects/s recovering 2023-10-01T09:21:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:46 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/852692111' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:21:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:46 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:21:46.807 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:21:46 smithi190 bash[215740]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-10-01T09:21:46.807 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:21:46 smithi190 bash[215740]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2023-10-01T09:21:46.807 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:21:46 smithi190 bash[215740]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2023-10-01T09:21:46.808 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:21:46 smithi190 bash[215740]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-01T09:21:46.808 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:21:46 smithi190 bash[215740]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2023-10-01T09:21:46.808 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:21:46 smithi190 bash[215740]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-10-01T09:21:46.808 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:21:46 smithi190 bash[215740]: --> ceph-volume raw activate successful for osd ID: 5 2023-10-01T09:21:46.856 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:46.939 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:47.203 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:47.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:47.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:47.314 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:47.371 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:47.803 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:47.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:47.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:47.886 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:47 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:47.886 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:47 smithi138 ceph-mon[185886]: failed to encode map e224 with expected crc 2023-10-01T09:21:47.886 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:47 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:21:47.886 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:47 smithi138 ceph-mon[185886]: osdmap e224: 8 total, 7 up, 8 in 2023-10-01T09:21:47.886 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:47 smithi138 ceph-mon[185886]: failed to encode map e224 with expected crc 2023-10-01T09:21:47.887 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:47 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 25134/241206 objects degraded (10.420%), 13 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-01T09:21:47.887 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:47 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:47.887 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:47 smithi138 ceph-mon[192513]: failed to encode map e224 with expected crc 2023-10-01T09:21:47.887 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:47 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:21:47.887 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:47 smithi138 ceph-mon[192513]: osdmap e224: 8 total, 7 up, 8 in 2023-10-01T09:21:47.887 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:47 smithi138 ceph-mon[192513]: failed to encode map e224 with expected crc 2023-10-01T09:21:47.887 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:47 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 25134/241206 objects degraded (10.420%), 13 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-01T09:21:47.909 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:47.976 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:47 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:47 smithi190 ceph-mon[160435]: failed to encode map e224 with expected crc 2023-10-01T09:21:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:47 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:21:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:47 smithi190 ceph-mon[160435]: osdmap e224: 8 total, 7 up, 8 in 2023-10-01T09:21:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:47 smithi190 ceph-mon[160435]: failed to encode map e224 with expected crc 2023-10-01T09:21:48.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:47 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 25134/241206 objects degraded (10.420%), 13 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-01T09:21:48.133 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:48.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:48.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:48.243 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:48.259 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:48.320 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:48.405 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:21:48.405 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:21:48.420 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:48.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:48.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:48.530 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:48.594 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:48.683 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:48 smithi138 ceph-mon[192513]: failed to encode map e224 with expected crc 2023-10-01T09:21:48.683 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:48 smithi138 ceph-mon[192513]: pgmap v558: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 24856/235356 objects degraded (10.561%); 284/235356 objects misplaced (0.121%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:21:48.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:48 smithi138 ceph-mon[185886]: failed to encode map e224 with expected crc 2023-10-01T09:21:48.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:48 smithi138 ceph-mon[185886]: pgmap v558: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 24856/235356 objects degraded (10.561%); 284/235356 objects misplaced (0.121%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:21:48.920 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:48.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:48.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:49.027 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:49.054 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:49.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:48 smithi190 ceph-mon[160435]: failed to encode map e224 with expected crc 2023-10-01T09:21:49.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:48 smithi190 ceph-mon[160435]: pgmap v558: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 24856/235356 objects degraded (10.561%); 284/235356 objects misplaced (0.121%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:21:49.095 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:49.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:49.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:49.199 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:49.231 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:49.279 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:49.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:49.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:49.382 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:49.427 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:49.500 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:49.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:49.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:49.604 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:49.657 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:49.691 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:21:49.738 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:49.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:49.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:49.843 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:49.921 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:21:49.924 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:50.003 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:50.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:50.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:50.107 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:50.152 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:50.489 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:50.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:50.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:50.599 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:50.666 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:50.801 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:50.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:50.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:50.815 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:50 smithi138 ceph-mon[185886]: pgmap v559: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 24264/231804 objects degraded (10.467%); 284/231804 objects misplaced (0.123%); 7.7 MiB/s, 124 objects/s recovering 2023-10-01T09:21:50.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:50 smithi138 ceph-mon[192513]: pgmap v559: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 24264/231804 objects degraded (10.467%); 284/231804 objects misplaced (0.123%); 7.7 MiB/s, 124 objects/s recovering 2023-10-01T09:21:50.904 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:50.908 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:50.940 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:50 smithi190 ceph-mon[160435]: pgmap v559: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 24264/231804 objects degraded (10.467%); 284/231804 objects misplaced (0.123%); 7.7 MiB/s, 124 objects/s recovering 2023-10-01T09:21:50.944 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:50.985 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:51.008 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:51.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:51.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:51.111 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:51.138 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:21:51.150 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:51.160 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:21:51.188 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:21:51.209 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:21:51.237 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:51.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:51.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:51.332 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:21:51.341 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:51.366 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:21:51.406 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:21:51.407 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:21:51.407 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:21:51.407 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:21:51.407 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:21:51.407 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:21:51.407 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-01T09:21:51.407 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:21:51.407 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:21:51.436 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:51 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1642210420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:51 smithi138 ceph-mon[185886]: pgmap v560: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 23700/228483 objects degraded (10.373%); 284/228483 objects misplaced (0.124%); 9.8 MiB/s, 156 objects/s recovering 2023-10-01T09:21:51.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:51 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1642210420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:51.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:51 smithi138 ceph-mon[192513]: pgmap v560: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 23700/228483 objects degraded (10.373%); 284/228483 objects misplaced (0.124%); 9.8 MiB/s, 156 objects/s recovering 2023-10-01T09:21:51.995 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:52.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:52.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:52.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:51 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1642210420' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:52.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:51 smithi190 ceph-mon[160435]: pgmap v560: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 23700/228483 objects degraded (10.373%); 284/228483 objects misplaced (0.124%); 9.8 MiB/s, 156 objects/s recovering 2023-10-01T09:21:52.106 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:52.145 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:52.218 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:52.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:52.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:52.323 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:52.391 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:52.472 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:52.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:52.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:52.578 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:52.618 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:52.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:52 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 23700/228483 objects degraded (10.373%), 13 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-01T09:21:52.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:52 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 23700/228483 objects degraded (10.373%), 13 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-01T09:21:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:52 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 23700/228483 objects degraded (10.373%), 13 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-01T09:21:53.673 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:21:53.814 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:21:53.814 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:21:53.837 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:53.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:53.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:53.933 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:21:53 smithi190 bash[216544]: 48f5e8c26c664763985f42da4753ecf61a6cd038e8f27e260ce0b3b34c04b1f8 2023-10-01T09:21:53.933 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:21:53 smithi190 conmon[216583]: debug 2023-10-01T09:21:53.659+0000 7f4b79c49700 1 -- 172.21.15.190:0/1912327084 <== mon.0 v2:172.21.15.138:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563ec619c680 con 0x563ec6163000 2023-10-01T09:21:53.933 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:21:53 smithi190 systemd[1]: Started Ceph osd.5 for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:21:53.933 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:21:53 smithi190 conmon[216583]: debug 2023-10-01T09:21:53.926+0000 7f4b816c3200 -1 Falling back to public interface 2023-10-01T09:21:53.933 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:53 smithi190 ceph-mon[160435]: pgmap v561: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 23416/222387 objects degraded (10.529%); 284/222387 objects misplaced (0.128%); 8.7 MiB/s, 138 objects/s recovering 2023-10-01T09:21:53.952 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:54.014 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:54.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:53 smithi138 ceph-mon[192513]: pgmap v561: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 23416/222387 objects degraded (10.529%); 284/222387 objects misplaced (0.128%); 8.7 MiB/s, 138 objects/s recovering 2023-10-01T09:21:54.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:53 smithi138 ceph-mon[185886]: pgmap v561: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 23416/222387 objects degraded (10.529%); 284/222387 objects misplaced (0.128%); 8.7 MiB/s, 138 objects/s recovering 2023-10-01T09:21:54.088 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:54.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:54.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:54.201 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:54.284 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:21:54.287 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:54.458 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:54.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:54.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:54.564 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:21:54.565 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:21:54.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.565 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:120060 6.f 6:f552dc87:::benchmark_data_smithi138_307751_object24691:head [delete] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-10-01T09:21:54.565 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:54.555319+0000", 2023-10-01T09:21:54.565 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0085520849999999992, 2023-10-01T09:21:54.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085831009999999992, 2023-10-01T09:21:54.565 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:54.565 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:21:54.565 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:21:54.566 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:21:54.566 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:21:54.566 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 120060 2023-10-01T09:21:54.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.566 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:54.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:54.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.555319+0000", 2023-10-01T09:21:54.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:54.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:54.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.555319+0000", 2023-10-01T09:21:54.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:54.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:54.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.555317+0000", 2023-10-01T09:21:54.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:21:54.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:54.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.555331+0000", 2023-10-01T09:21:54.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3515e-05 2023-10-01T09:21:54.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:54.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.555346+0000", 2023-10-01T09:21:54.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4863e-05 2023-10-01T09:21:54.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:54.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.555353+0000", 2023-10-01T09:21:54.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1280000000000002e-06 2023-10-01T09:21:54.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:21:54.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.555436+0000", 2023-10-01T09:21:54.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3042999999999998e-05 2023-10-01T09:21:54.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:21:54.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.555552+0000", 2023-10-01T09:21:54.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115727 2023-10-01T09:21:54.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:21:54.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.555657+0000", 2023-10-01T09:21:54.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105813 2023-10-01T09:21:54.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:21:54.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.563509+0000", 2023-10-01T09:21:54.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0078514039999999993 2023-10-01T09:21:54.571 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:54.571 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:54.571 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:54.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.571 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:120064 6.2 6:422f4ecf:::benchmark_data_smithi138_307751_object24695:head [delete] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-10-01T09:21:54.571 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:54.557379+0000", 2023-10-01T09:21:54.571 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0064915800000000003, 2023-10-01T09:21:54.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006611759, 2023-10-01T09:21:54.572 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:54.572 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:21:54.572 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:21:54.572 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:21:54.572 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:21:54.572 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 120064 2023-10-01T09:21:54.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.572 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:54.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:54.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.557379+0000", 2023-10-01T09:21:54.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:54.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:54.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.557379+0000", 2023-10-01T09:21:54.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:54.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:54.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.557378+0000", 2023-10-01T09:21:54.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:21:54.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:54.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.557384+0000", 2023-10-01T09:21:54.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.75e-06 2023-10-01T09:21:54.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:54.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.557394+0000", 2023-10-01T09:21:54.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0434999999999999e-05 2023-10-01T09:21:54.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:54.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.557401+0000", 2023-10-01T09:21:54.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1389999999999998e-06 2023-10-01T09:21:54.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:21:54.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.557444+0000", 2023-10-01T09:21:54.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2317999999999997e-05 2023-10-01T09:21:54.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:21:54.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.557554+0000", 2023-10-01T09:21:54.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110154 2023-10-01T09:21:54.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:21:54.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.557654+0000", 2023-10-01T09:21:54.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000100503 2023-10-01T09:21:54.576 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:54.576 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:54.577 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:54.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.577 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35070.0:120065 6.a e224/208 6:59801986:::benchmark_data_smithi138_307751_object24696:head v 224'7571, mlcod=187'1563)", 2023-10-01T09:21:54.577 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:54.559210+0000", 2023-10-01T09:21:54.577 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0046608630000000003, 2023-10-01T09:21:54.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048550620000000003, 2023-10-01T09:21:54.577 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:54.577 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:21:54.578 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:54.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:54.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.559210+0000", 2023-10-01T09:21:54.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:54.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:54.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.559210+0000", 2023-10-01T09:21:54.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:54.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:54.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.559209+0000", 2023-10-01T09:21:54.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:21:54.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:54.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.559220+0000", 2023-10-01T09:21:54.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0706999999999999e-05 2023-10-01T09:21:54.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:54.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.559229+0000", 2023-10-01T09:21:54.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3209999999999994e-06 2023-10-01T09:21:54.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:54.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.559236+0000", 2023-10-01T09:21:54.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6479999999999997e-06 2023-10-01T09:21:54.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:21:54.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.559463+0000", 2023-10-01T09:21:54.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000227374 2023-10-01T09:21:54.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:21:54.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.559486+0000", 2023-10-01T09:21:54.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2385e-05 2023-10-01T09:21:54.581 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:54.581 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:54.581 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:54.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.582 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35070.0:120061 6.9 e224/220 6:90b0a7c5:::benchmark_data_smithi138_307751_object24692:head v 224'7240, mlcod=224'7238)", 2023-10-01T09:21:54.582 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:54.559404+0000", 2023-10-01T09:21:54.582 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.004466869, 2023-10-01T09:21:54.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047220259999999998, 2023-10-01T09:21:54.582 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:54.582 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:21:54.582 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:54.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:54.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.559404+0000", 2023-10-01T09:21:54.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:54.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:54.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.559404+0000", 2023-10-01T09:21:54.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:54.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:54.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.559403+0000", 2023-10-01T09:21:54.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:21:54.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:54.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.559410+0000", 2023-10-01T09:21:54.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3199999999999996e-06 2023-10-01T09:21:54.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:54.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.559419+0000", 2023-10-01T09:21:54.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2699999999999993e-06 2023-10-01T09:21:54.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:54.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.559425+0000", 2023-10-01T09:21:54.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.502e-06 2023-10-01T09:21:54.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:21:54.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.560729+0000", 2023-10-01T09:21:54.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001304747 2023-10-01T09:21:54.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:21:54.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.560749+0000", 2023-10-01T09:21:54.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0169e-05 2023-10-01T09:21:54.586 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:54.586 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:54.586 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:54.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.586 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:120068 6.2 6:442ed45c:::benchmark_data_smithi138_307751_object24699:head [delete] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-10-01T09:21:54.586 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:54.562963+0000", 2023-10-01T09:21:54.586 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.00090769199999999998, 2023-10-01T09:21:54.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012255040000000001, 2023-10-01T09:21:54.587 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:21:54.587 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:21:54.587 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:21:54.587 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:21:54.587 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:21:54.587 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 120068 2023-10-01T09:21:54.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.587 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:21:54.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:21:54.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.562963+0000", 2023-10-01T09:21:54.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:54.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:21:54.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.562963+0000", 2023-10-01T09:21:54.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:21:54.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:21:54.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.562962+0000", 2023-10-01T09:21:54.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:21:54.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:21:54.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.562968+0000", 2023-10-01T09:21:54.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3409999999999999e-06 2023-10-01T09:21:54.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:21:54.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.562982+0000", 2023-10-01T09:21:54.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3952e-05 2023-10-01T09:21:54.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:21:54.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.562991+0000", 2023-10-01T09:21:54.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7860000000000002e-06 2023-10-01T09:21:54.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:21:54.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.563046+0000", 2023-10-01T09:21:54.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5151000000000001e-05 2023-10-01T09:21:54.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:21:54.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.563155+0000", 2023-10-01T09:21:54.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109268 2023-10-01T09:21:54.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:21:54.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:21:54.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:21:54.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:54.563241+0000", 2023-10-01T09:21:54.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5469000000000004e-05 2023-10-01T09:21:54.591 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:54.591 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:21:54.591 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:54.591 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:21:54.592 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:21:54.592 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 5 2023-10-01T09:21:54.592 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:21:54.592 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:54.676 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:54.816 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:54.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:54.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:54.898 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:21:54.925 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:54.977 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:55.059 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:55.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:55.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:55.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:21:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:21:54] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.33.4" 2023-10-01T09:21:55.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:54 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1131205052' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:55.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:54 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:55.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:54 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1131205052' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:55.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:54 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:55.165 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:55.234 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:55.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:54 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1131205052' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:55.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:54 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:21:55.816 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:21:55.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:55.827 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:55.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:55.927 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:21:55.953 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:56.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:56.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:21:56.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[192513]: osdmap e225: 8 total, 7 up, 8 in 2023-10-01T09:21:56.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[192513]: failed to encode map e225 with expected crc 2023-10-01T09:21:56.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[192513]: failed to encode map e225 with expected crc 2023-10-01T09:21:56.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[192513]: failed to encode map e225 with expected crc 2023-10-01T09:21:56.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:56.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:21:56.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[192513]: pgmap v563: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 22430/216141 objects degraded (10.377%); 285/216141 objects misplaced (0.132%); 9.5 MiB/s, 152 objects/s recovering 2023-10-01T09:21:56.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:56.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:21:56.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[185886]: osdmap e225: 8 total, 7 up, 8 in 2023-10-01T09:21:56.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[185886]: failed to encode map e225 with expected crc 2023-10-01T09:21:56.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[185886]: failed to encode map e225 with expected crc 2023-10-01T09:21:56.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[185886]: failed to encode map e225 with expected crc 2023-10-01T09:21:56.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:56.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:21:56.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:55 smithi138 ceph-mon[185886]: pgmap v563: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 22430/216141 objects degraded (10.377%); 285/216141 objects misplaced (0.132%); 9.5 MiB/s, 152 objects/s recovering 2023-10-01T09:21:56.208 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:21:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:55 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:21:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:55 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:21:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:55 smithi190 ceph-mon[160435]: osdmap e225: 8 total, 7 up, 8 in 2023-10-01T09:21:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:55 smithi190 ceph-mon[160435]: failed to encode map e225 with expected crc 2023-10-01T09:21:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:55 smithi190 ceph-mon[160435]: failed to encode map e225 with expected crc 2023-10-01T09:21:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:55 smithi190 ceph-mon[160435]: failed to encode map e225 with expected crc 2023-10-01T09:21:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:21:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:21:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:55 smithi190 ceph-mon[160435]: pgmap v563: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 22430/216141 objects degraded (10.377%); 285/216141 objects misplaced (0.132%); 9.5 MiB/s, 152 objects/s recovering 2023-10-01T09:21:56.333 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:21:56.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:56.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:21:56.436 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:56.492 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:56.567 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:56.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:56.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:56.578 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:56.673 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:56.769 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:56.937 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:56.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:56.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:57.043 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:21:57.142 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:57.180 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:21:57.282 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:57 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 22430/216141 objects degraded (10.377%), 13 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-01T09:21:57.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:57 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 22430/216141 objects degraded (10.377%), 13 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-01T09:21:57.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:57 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 22430/216141 objects degraded (10.377%), 13 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-01T09:21:57.377 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:21:57.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:57.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:21:57.484 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:57.576 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:57.654 INFO:teuthology.orchestra.run.smithi190.stderr:no valid command found; 10 closest matches: 2023-10-01T09:21:57.654 INFO:teuthology.orchestra.run.smithi190.stderr:0 2023-10-01T09:21:57.655 INFO:teuthology.orchestra.run.smithi190.stderr:1 2023-10-01T09:21:57.655 INFO:teuthology.orchestra.run.smithi190.stderr:2 2023-10-01T09:21:57.655 INFO:teuthology.orchestra.run.smithi190.stderr:abort 2023-10-01T09:21:57.655 INFO:teuthology.orchestra.run.smithi190.stderr:assert 2023-10-01T09:21:57.655 INFO:teuthology.orchestra.run.smithi190.stderr:bluefs debug_inject_read_zeros 2023-10-01T09:21:57.655 INFO:teuthology.orchestra.run.smithi190.stderr:bluefs files list 2023-10-01T09:21:57.655 INFO:teuthology.orchestra.run.smithi190.stderr:bluefs stats 2023-10-01T09:21:57.656 INFO:teuthology.orchestra.run.smithi190.stderr:bluestore allocator dump block 2023-10-01T09:21:57.656 INFO:teuthology.orchestra.run.smithi190.stderr:bluestore allocator fragmentation block 2023-10-01T09:21:57.656 INFO:teuthology.orchestra.run.smithi190.stderr:admin_socket: invalid command 2023-10-01T09:21:57.691 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:57.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:57.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:57.802 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:57.864 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:58.169 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:58.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:58.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:58.280 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:21:58.336 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:58.438 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:21:58.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:58 smithi138 ceph-mon[192513]: pgmap v564: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 22166/209985 objects degraded (10.556%); 285/209985 objects misplaced (0.136%); 8.4 MiB/s, 134 objects/s recovering 2023-10-01T09:21:58.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:58 smithi138 ceph-mon[185886]: pgmap v564: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 22166/209985 objects degraded (10.556%); 285/209985 objects misplaced (0.136%); 8.4 MiB/s, 134 objects/s recovering 2023-10-01T09:21:58.523 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:21:58.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:58.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:21:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:58 smithi190 ceph-mon[160435]: pgmap v564: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 22166/209985 objects degraded (10.556%); 285/209985 objects misplaced (0.136%); 8.4 MiB/s, 134 objects/s recovering 2023-10-01T09:21:58.627 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:58.658 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:58.724 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:58.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:58.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:58.784 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:21:58.817 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:21:58.830 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:58.860 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:21:58.903 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:21:58.905 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:58.935 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:21:58.978 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:21:58.993 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:21:58.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:58.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:21:59.018 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:21:59.018 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:21:59.018 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:21:59.019 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:21:59.019 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:21:59.019 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:21:59.019 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-01T09:21:59.019 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:21:59.019 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:21:59.098 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:21:59.127 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:59.149 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-01T09:21:59.150 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2023-10-01T09:21:59.177 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:21:59.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:59.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:21:59.280 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:21:59.307 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:21:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/155060586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:59.307 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:21:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/155060586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:59.342 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:59.418 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:21:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/155060586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:21:59.619 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:21:59.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:59.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:21:59.726 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:21:59.759 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:21:59 smithi190 conmon[216583]: debug 2023-10-01T09:21:59.403+0000 7f4b816c3200 -1 osd.5 197 log_to_monitors {default=true} 2023-10-01T09:21:59.763 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:21:59.823 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:21:59.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:21:59.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:21:59.941 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:21:59.968 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:00.299 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:00.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:00.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:00.405 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:22:00.414 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:22:00.414 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:22:00.457 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:00.477 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:00.562 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:00 smithi138 ceph-mon[192513]: from='osd.5 [v2:172.21.15.190:6808/80913179,v1:172.21.15.190:6809/80913179]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:22:00.562 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:00 smithi138 ceph-mon[192513]: pgmap v565: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 21641/206946 objects degraded (10.457%); 284/206946 objects misplaced (0.137%); 8.3 MiB/s, 132 objects/s recovering 2023-10-01T09:22:00.563 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:00 smithi138 ceph-mon[185886]: from='osd.5 [v2:172.21.15.190:6808/80913179,v1:172.21.15.190:6809/80913179]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:22:00.563 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:00 smithi138 ceph-mon[185886]: pgmap v565: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 21641/206946 objects degraded (10.457%); 284/206946 objects misplaced (0.137%); 8.3 MiB/s, 132 objects/s recovering 2023-10-01T09:22:00.608 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:22:00.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:00.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:22:00.714 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:00.789 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:00.806 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:22:00 smithi190 conmon[216583]: debug 2023-10-01T09:22:00.357+0000 7f4b743c1700 -1 osd.5 197 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T09:22:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:00 smithi190 ceph-mon[160435]: from='osd.5 [v2:172.21.15.190:6808/80913179,v1:172.21.15.190:6809/80913179]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:22:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:00 smithi190 ceph-mon[160435]: pgmap v565: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 21641/206946 objects degraded (10.457%); 284/206946 objects misplaced (0.137%); 8.3 MiB/s, 132 objects/s recovering 2023-10-01T09:22:00.814 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:22:00.814 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:22:01.013 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:01.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:01.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:01.126 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:22:01.181 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:01.277 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:22:01.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:01.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:22:01.394 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:01.425 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:01.468 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:01.472 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:01 smithi190 ceph-mon[160435]: from='osd.5 [v2:172.21.15.190:6808/80913179,v1:172.21.15.190:6809/80913179]' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T09:22:01.472 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:01 smithi190 ceph-mon[160435]: osdmap e226: 8 total, 7 up, 8 in 2023-10-01T09:22:01.473 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:01 smithi190 ceph-mon[160435]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.473 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:01 smithi190 ceph-mon[160435]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.473 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:01 smithi190 ceph-mon[160435]: from='osd.5 [v2:172.21.15.190:6808/80913179,v1:172.21.15.190:6809/80913179]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:22:01.473 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:01 smithi190 ceph-mon[160435]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.473 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:01 smithi190 ceph-mon[160435]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.474 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:01 smithi190 ceph-mon[160435]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:01.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:01.576 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:01.602 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:01.643 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:01.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:01.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:01.749 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:22:01.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[185886]: from='osd.5 [v2:172.21.15.190:6808/80913179,v1:172.21.15.190:6809/80913179]' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T09:22:01.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[185886]: osdmap e226: 8 total, 7 up, 8 in 2023-10-01T09:22:01.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[185886]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[185886]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[185886]: from='osd.5 [v2:172.21.15.190:6808/80913179,v1:172.21.15.190:6809/80913179]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:22:01.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[185886]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[185886]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[185886]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.776 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[192513]: from='osd.5 [v2:172.21.15.190:6808/80913179,v1:172.21.15.190:6809/80913179]' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T09:22:01.776 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[192513]: osdmap e226: 8 total, 7 up, 8 in 2023-10-01T09:22:01.776 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[192513]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.776 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[192513]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.776 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[192513]: from='osd.5 [v2:172.21.15.190:6808/80913179,v1:172.21.15.190:6809/80913179]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:22:01.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[192513]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[192513]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:01 smithi138 ceph-mon[192513]: failed to encode map e226 with expected crc 2023-10-01T09:22:01.778 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:02.415 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:22:02.477 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:22:02.526 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:02.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:02.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:02.633 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:22:02.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[192513]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:22:02.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[192513]: osd.5 [v2:172.21.15.190:6808/80913179,v1:172.21.15.190:6809/80913179] boot 2023-10-01T09:22:02.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[192513]: osdmap e227: 8 total, 8 up, 8 in 2023-10-01T09:22:02.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[192513]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:22:02.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[192513]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[192513]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[192513]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[192513]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[192513]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[192513]: pgmap v568: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 21152/204108 objects degraded (10.363%); 284/204108 objects misplaced (0.139%); 6.8 MiB/s, 108 objects/s recovering 2023-10-01T09:22:02.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 21641/206946 objects degraded (10.457%), 13 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-01T09:22:02.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[192513]: osdmap e228: 8 total, 8 up, 8 in 2023-10-01T09:22:02.663 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[185886]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:22:02.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[185886]: osd.5 [v2:172.21.15.190:6808/80913179,v1:172.21.15.190:6809/80913179] boot 2023-10-01T09:22:02.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[185886]: osdmap e227: 8 total, 8 up, 8 in 2023-10-01T09:22:02.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[185886]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:22:02.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[185886]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[185886]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[185886]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[185886]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[185886]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.664 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[185886]: pgmap v568: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 21152/204108 objects degraded (10.363%); 284/204108 objects misplaced (0.139%); 6.8 MiB/s, 108 objects/s recovering 2023-10-01T09:22:02.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 21641/206946 objects degraded (10.457%), 13 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-01T09:22:02.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:02 smithi138 ceph-mon[185886]: osdmap e228: 8 total, 8 up, 8 in 2023-10-01T09:22:02.665 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:02.760 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:22:02.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:02.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:22:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:02 smithi190 ceph-mon[160435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:22:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:02 smithi190 ceph-mon[160435]: osd.5 [v2:172.21.15.190:6808/80913179,v1:172.21.15.190:6809/80913179] boot 2023-10-01T09:22:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:02 smithi190 ceph-mon[160435]: osdmap e227: 8 total, 8 up, 8 in 2023-10-01T09:22:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:02 smithi190 ceph-mon[160435]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:02 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:22:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:02 smithi190 ceph-mon[160435]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:02 smithi190 ceph-mon[160435]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:02 smithi190 ceph-mon[160435]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:02 smithi190 ceph-mon[160435]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:02 smithi190 ceph-mon[160435]: failed to encode map e227 with expected crc 2023-10-01T09:22:02.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:02 smithi190 ceph-mon[160435]: pgmap v568: 99 pgs: 2 active+recovery_wait+degraded, 15 active+clean+remapped, 2 active+recovery_wait, 7 active+undersized+degraded, 10 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 59 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 21152/204108 objects degraded (10.363%); 284/204108 objects misplaced (0.139%); 6.8 MiB/s, 108 objects/s recovering 2023-10-01T09:22:02.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:02 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 21641/206946 objects degraded (10.457%), 13 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-10-01T09:22:02.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:02 smithi190 ceph-mon[160435]: osdmap e228: 8 total, 8 up, 8 in 2023-10-01T09:22:02.866 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:22:02.893 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:02.934 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:22:02.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:02.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:22:03.011 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:03.021 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:03.040 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:03.073 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:03.267 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:03.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:03.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:03.370 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:03.411 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:03.660 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:03.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:03.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:03.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[185886]: failed to encode map e198 with expected crc 2023-10-01T09:22:03.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[185886]: failed to encode map e227 with expected crc 2023-10-01T09:22:03.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[185886]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[185886]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[185886]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[185886]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[185886]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[185886]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[185886]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[185886]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[185886]: osdmap e229: 8 total, 8 up, 8 in 2023-10-01T09:22:03.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/325868753' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:22:03.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:22:03.667 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[192513]: failed to encode map e198 with expected crc 2023-10-01T09:22:03.667 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[192513]: failed to encode map e227 with expected crc 2023-10-01T09:22:03.667 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[192513]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.667 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[192513]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.667 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[192513]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.667 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[192513]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.668 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[192513]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.668 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[192513]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.668 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[192513]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.668 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[192513]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.668 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[192513]: osdmap e229: 8 total, 8 up, 8 in 2023-10-01T09:22:03.668 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/325868753' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:22:03.668 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:03 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:22:03.766 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:03.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:03 smithi190 ceph-mon[160435]: failed to encode map e198 with expected crc 2023-10-01T09:22:03.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:03 smithi190 ceph-mon[160435]: failed to encode map e227 with expected crc 2023-10-01T09:22:03.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:03 smithi190 ceph-mon[160435]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:03 smithi190 ceph-mon[160435]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:03 smithi190 ceph-mon[160435]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:03 smithi190 ceph-mon[160435]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:03 smithi190 ceph-mon[160435]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:03 smithi190 ceph-mon[160435]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:03 smithi190 ceph-mon[160435]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:03 smithi190 ceph-mon[160435]: failed to encode map e228 with expected crc 2023-10-01T09:22:03.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:03 smithi190 ceph-mon[160435]: osdmap e229: 8 total, 8 up, 8 in 2023-10-01T09:22:03.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:03 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/325868753' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:22:03.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:03 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:22:03.831 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:03.930 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:03.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:03.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:04.042 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:22:04.102 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:04.148 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:22:04.150 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:22:04.195 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:22:04.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:04.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:22:04.312 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:22:04.343 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:04.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[185886]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[185886]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[185886]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[185886]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[185886]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[185886]: pgmap v571: 99 pgs: 4 remapped+peering, 2 peering, 1 active+recovery_wait+degraded, 11 active+clean+remapped, 1 active+recovering, 1 active+recovery_wait, 4 active+undersized+degraded, 5 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 66 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 16777/198912 objects degraded (8.434%); 294/198912 objects misplaced (0.148%); 9.4 MiB/s, 150 objects/s recovering 2023-10-01T09:22:04.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:22:04.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:22:04.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[185886]: osdmap e230: 8 total, 8 up, 8 in 2023-10-01T09:22:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[192513]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[192513]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[192513]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[192513]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[192513]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[192513]: pgmap v571: 99 pgs: 4 remapped+peering, 2 peering, 1 active+recovery_wait+degraded, 11 active+clean+remapped, 1 active+recovering, 1 active+recovery_wait, 4 active+undersized+degraded, 5 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 66 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 16777/198912 objects degraded (8.434%); 294/198912 objects misplaced (0.148%); 9.4 MiB/s, 150 objects/s recovering 2023-10-01T09:22:04.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:22:04.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:22:04.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:04 smithi138 ceph-mon[192513]: osdmap e230: 8 total, 8 up, 8 in 2023-10-01T09:22:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:04 smithi190 ceph-mon[160435]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:04 smithi190 ceph-mon[160435]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:04 smithi190 ceph-mon[160435]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:04 smithi190 ceph-mon[160435]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:04 smithi190 ceph-mon[160435]: failed to encode map e229 with expected crc 2023-10-01T09:22:04.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:04 smithi190 ceph-mon[160435]: pgmap v571: 99 pgs: 4 remapped+peering, 2 peering, 1 active+recovery_wait+degraded, 11 active+clean+remapped, 1 active+recovering, 1 active+recovery_wait, 4 active+undersized+degraded, 5 active+undersized, 2 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 66 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 16777/198912 objects degraded (8.434%); 294/198912 objects misplaced (0.148%); 9.4 MiB/s, 150 objects/s recovering 2023-10-01T09:22:04.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:04 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:22:04.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:04 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:22:04.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:04 smithi190 ceph-mon[160435]: osdmap e230: 8 total, 8 up, 8 in 2023-10-01T09:22:04.846 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:22:05.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:22:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:22:04] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.33.4" 2023-10-01T09:22:05.315 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:05.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:05.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:05.422 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:22:05.477 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:05.545 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:05 smithi138 ceph-mon[185886]: failed to encode map e230 with expected crc 2023-10-01T09:22:05.545 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:05 smithi138 ceph-mon[185886]: failed to encode map e230 with expected crc 2023-10-01T09:22:05.545 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:05 smithi138 ceph-mon[192513]: failed to encode map e230 with expected crc 2023-10-01T09:22:05.545 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:05 smithi138 ceph-mon[192513]: failed to encode map e230 with expected crc 2023-10-01T09:22:05.553 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:05 smithi190 ceph-mon[160435]: failed to encode map e230 with expected crc 2023-10-01T09:22:05.554 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:05 smithi190 ceph-mon[160435]: failed to encode map e230 with expected crc 2023-10-01T09:22:05.896 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:22:05.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:05.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:22:06.001 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:22:06.071 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:06.222 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:22:06.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:06.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:22:06.332 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:06.379 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:06.402 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:06 smithi190 ceph-mon[160435]: 6.2 scrub starts 2023-10-01T09:22:06.403 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:06 smithi190 ceph-mon[160435]: pgmap v573: 99 pgs: 3 remapped+peering, 2 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 2 active+undersized+degraded, 86 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 11206/195594 objects degraded (5.729%); 331/195594 objects misplaced (0.169%); 7.6 MiB/s, 121 objects/s recovering 2023-10-01T09:22:06.466 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:06 smithi138 ceph-mon[185886]: 6.2 scrub starts 2023-10-01T09:22:06.466 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:06 smithi138 ceph-mon[185886]: pgmap v573: 99 pgs: 3 remapped+peering, 2 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 2 active+undersized+degraded, 86 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 11206/195594 objects degraded (5.729%); 331/195594 objects misplaced (0.169%); 7.6 MiB/s, 121 objects/s recovering 2023-10-01T09:22:06.468 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:06 smithi138 ceph-mon[192513]: 6.2 scrub starts 2023-10-01T09:22:06.468 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:06 smithi138 ceph-mon[192513]: pgmap v573: 99 pgs: 3 remapped+peering, 2 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 2 active+undersized+degraded, 86 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 11206/195594 objects degraded (5.729%); 331/195594 objects misplaced (0.169%); 7.6 MiB/s, 121 objects/s recovering 2023-10-01T09:22:06.588 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:06.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:06.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:06.615 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:22:06.615 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:22:06.696 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:22:06.717 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:22:06.763 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:22:06.764 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:06.796 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:22:06.841 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:22:06.864 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:22:06.871 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:22:06.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:06.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:22:06.903 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:22:06.940 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:22:06.974 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:22:06.975 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:22:06.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:22:06.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:22:06.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:22:06.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:22:06.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:22:06.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:22:06.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:22:06.976 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:22:07.028 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:07.074 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:22:07.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:07.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:22:07.177 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:07.203 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:07.275 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:07.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:07.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:07.380 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:22:07.433 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:07.550 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:07 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3511774930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:07.550 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:07 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:22:07.550 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:07 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 11206/195594 objects degraded (5.729%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:07.550 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:07 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:22:07.550 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:07 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3511774930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:07.551 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:07 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:22:07.551 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:07 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 11206/195594 objects degraded (5.729%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:07.551 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:07 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:22:07.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:07 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3511774930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:07.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:07 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:22:07.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:07 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 11206/195594 objects degraded (5.729%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:07.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:07 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:22:08.130 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:22:08.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:08.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:22:08.140 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:22:08.238 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:22:08.395 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:22:08.481 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:08.613 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:08.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:08.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:08.721 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:08.771 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:08 smithi190 ceph-mon[160435]: pgmap v574: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 12771/189339 objects degraded (6.745%); 320/189339 objects misplaced (0.169%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:22:08.779 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:08.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:08 smithi138 ceph-mon[192513]: pgmap v574: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 12771/189339 objects degraded (6.745%); 320/189339 objects misplaced (0.169%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:22:08.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:08 smithi138 ceph-mon[185886]: pgmap v574: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 12771/189339 objects degraded (6.745%); 320/189339 objects misplaced (0.169%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:22:08.933 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:08.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:08.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:09.024 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:09.040 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:09.112 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:09.193 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:22:09.193 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:22:09.193 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.193 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:130650 6.d e230/228 6:baae39fc:::benchmark_data_smithi138_307751_object35281:head v 230'8299, mlcod=197'4889)", 2023-10-01T09:22:09.194 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:09.183385+0000", 2023-10-01T09:22:09.194 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.003256475, 2023-10-01T09:22:09.194 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0032890200000000001, 2023-10-01T09:22:09.194 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:09.194 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:22:09.194 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:09.194 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.194 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:09.194 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183385+0000", 2023-10-01T09:22:09.194 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:09.195 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.195 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.195 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:09.195 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183385+0000", 2023-10-01T09:22:09.195 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:22:09.195 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.195 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.195 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:09.195 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183384+0000", 2023-10-01T09:22:09.195 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0440000000000001e-06 2023-10-01T09:22:09.195 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.195 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.196 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:09.196 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183393+0000", 2023-10-01T09:22:09.196 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3708e-05 2023-10-01T09:22:09.196 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.196 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.196 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:09.196 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183407+0000", 2023-10-01T09:22:09.196 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2009999999999999e-06 2023-10-01T09:22:09.196 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.196 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.196 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:09.196 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183415+0000", 2023-10-01T09:22:09.197 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8417000000000001e-05 2023-10-01T09:22:09.197 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.197 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.197 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:09.197 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183463+0000", 2023-10-01T09:22:09.197 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3988000000000001e-05 2023-10-01T09:22:09.197 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.197 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.197 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:09.197 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183487+0000", 2023-10-01T09:22:09.197 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000102446 2023-10-01T09:22:09.197 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:09.198 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:09.198 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:09.198 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.198 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.198 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:130652 6.14 e230/228 6:298422a8:::benchmark_data_smithi138_307751_object35283:head v 230'7413, mlcod=197'4765)", 2023-10-01T09:22:09.198 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:09.183788+0000", 2023-10-01T09:22:09.198 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.002853637, 2023-10-01T09:22:09.198 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002960215, 2023-10-01T09:22:09.198 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:09.198 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:22:09.198 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:09.198 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.199 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:09.199 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183788+0000", 2023-10-01T09:22:09.199 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:09.199 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.199 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.199 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:09.199 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183788+0000", 2023-10-01T09:22:09.199 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:22:09.199 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.199 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.199 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:09.199 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183787+0000", 2023-10-01T09:22:09.200 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9910000000000002e-06 2023-10-01T09:22:09.200 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.200 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.200 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:09.200 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183795+0000", 2023-10-01T09:22:09.200 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4056000000000001e-05 2023-10-01T09:22:09.200 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.200 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.200 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:09.200 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183809+0000", 2023-10-01T09:22:09.200 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4700000000000001e-06 2023-10-01T09:22:09.200 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.200 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.201 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:09.201 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183815+0000", 2023-10-01T09:22:09.201 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5104000000000005e-05 2023-10-01T09:22:09.201 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.201 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.201 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:09.201 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183890+0000", 2023-10-01T09:22:09.201 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0707e-05 2023-10-01T09:22:09.201 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.201 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.201 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:09.201 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.183910+0000", 2023-10-01T09:22:09.202 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000122648 2023-10-01T09:22:09.202 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:09.202 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:09.202 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:09.202 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.202 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.202 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:130647 6.17 e230/228)", 2023-10-01T09:22:09.202 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:09.184412+0000", 2023-10-01T09:22:09.202 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0022289929999999999, 2023-10-01T09:22:09.202 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0023965739999999998, 2023-10-01T09:22:09.202 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:09.202 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:22:09.203 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:09.203 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.203 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:09.203 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.184412+0000", 2023-10-01T09:22:09.203 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:09.203 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.203 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.203 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:09.203 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.184412+0000", 2023-10-01T09:22:09.203 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:22:09.204 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.204 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.204 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:09.204 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.184412+0000", 2023-10-01T09:22:09.204 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3243e-05 2023-10-01T09:22:09.204 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.204 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.204 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:09.204 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.184425+0000", 2023-10-01T09:22:09.204 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1134e-05 2023-10-01T09:22:09.204 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.204 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.205 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:09.205 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.184436+0000", 2023-10-01T09:22:09.205 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0610000000000001e-06 2023-10-01T09:22:09.205 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.205 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.205 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:09.205 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.184442+0000", 2023-10-01T09:22:09.205 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9621e-05 2023-10-01T09:22:09.205 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:09.205 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:09.205 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:09.206 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.206 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.207 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:130651 6.7 e230/228)", 2023-10-01T09:22:09.207 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:09.184455+0000", 2023-10-01T09:22:09.207 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0021863730000000001, 2023-10-01T09:22:09.207 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024004339999999999, 2023-10-01T09:22:09.207 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:09.207 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:22:09.207 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:09.207 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.207 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:09.207 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.184455+0000", 2023-10-01T09:22:09.207 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:09.207 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.208 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.208 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:09.208 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.184455+0000", 2023-10-01T09:22:09.208 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:22:09.208 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.208 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.208 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:09.208 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.184455+0000", 2023-10-01T09:22:09.208 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7350000000000004e-06 2023-10-01T09:22:09.208 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.208 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.208 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:09.209 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.184459+0000", 2023-10-01T09:22:09.209 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3460000000000001e-06 2023-10-01T09:22:09.209 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.209 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.209 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:09.209 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.184465+0000", 2023-10-01T09:22:09.209 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.788e-06 2023-10-01T09:22:09.209 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:09.209 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:09.209 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:09.209 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:09.184469+0000", 2023-10-01T09:22:09.209 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3546e-05 2023-10-01T09:22:09.210 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:09.210 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:09.210 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:09.210 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:09.210 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:22:09.210 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 4 2023-10-01T09:22:09.210 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:22:09.286 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:09.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:09.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:09.392 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:09.440 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:09.604 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:09.624 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:09.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:09.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:09.732 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:09.823 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:10.072 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:22:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:10 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:22:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:10 smithi190 ceph-mon[160435]: pgmap v575: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 12771/186936 objects degraded (6.832%); 320/186936 objects misplaced (0.171%); 4.7 MiB/s, 74 objects/s recovering 2023-10-01T09:22:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:22:10.811 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:10 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:22:10.811 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:10 smithi138 ceph-mon[185886]: pgmap v575: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 12771/186936 objects degraded (6.832%); 320/186936 objects misplaced (0.171%); 4.7 MiB/s, 74 objects/s recovering 2023-10-01T09:22:10.811 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:10.811 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:22:10.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:10 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:22:10.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:10 smithi138 ceph-mon[192513]: pgmap v575: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 12771/186936 objects degraded (6.832%); 320/186936 objects misplaced (0.171%); 4.7 MiB/s, 74 objects/s recovering 2023-10-01T09:22:10.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:10.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:22:10.947 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:10.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:10.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:11.050 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:11.104 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:11.157 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:11.170 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:11.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:11.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:11.278 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:22:11.341 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:11.671 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:22:11.671 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:22:11.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:22:11.671 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:132838 6.17 6:ea5ea5d5:::benchmark_data_smithi138_307751_object37469:head [delete] snapc 0=[] ondisk+write+known_if_redirected e230)", 2023-10-01T09:22:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:22:11.655102+0000", 2023-10-01T09:22:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.011980131, 2023-10-01T09:22:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012004427999999999, 2023-10-01T09:22:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:22:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:22:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:22:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:22:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 132838 2023-10-01T09:22:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:22:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:22:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:22:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:22:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:22:11.655102+0000", 2023-10-01T09:22:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:22:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:22:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:22:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:22:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:22:11.655102+0000", 2023-10-01T09:22:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:22:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:22:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:22:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:22:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:22:11.655101+0000", 2023-10-01T09:22:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:22:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:22:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:22:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:22:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:22:11.655107+0000", 2023-10-01T09:22:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6799999999999998e-06 2023-10-01T09:22:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:22:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:22:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:22:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:22:11.655120+0000", 2023-10-01T09:22:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3371e-05 2023-10-01T09:22:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:22:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:22:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:22:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:22:11.655128+0000", 2023-10-01T09:22:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4889999999999998e-06 2023-10-01T09:22:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:22:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:22:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:22:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:22:11.655176+0000", 2023-10-01T09:22:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.812e-05 2023-10-01T09:22:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:22:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:22:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:22:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:22:11.655351+0000", 2023-10-01T09:22:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017550899999999999 2023-10-01T09:22:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:22:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:22:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5", 2023-10-01T09:22:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:22:11.655500+0000", 2023-10-01T09:22:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014890699999999999 2023-10-01T09:22:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:22:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:22:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:22:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:22:11.657279+0000", 2023-10-01T09:22:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001778487 2023-10-01T09:22:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:22:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:22:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:22:11.666921+0000", 2023-10-01T09:22:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0096425130000000001 2023-10-01T09:22:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:22:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:22:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:22:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:22:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:22:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 1 2023-10-01T09:22:11.679 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:22:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:11 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:22:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:11 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:22:11.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:11 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:22:11.907 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:11.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:11.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:12.011 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:22:12.061 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:12 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:22:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:12 smithi190 ceph-mon[160435]: pgmap v576: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 12509/183450 objects degraded (6.819%); 320/183450 objects misplaced (0.174%); 6.1 MiB/s, 97 objects/s recovering 2023-10-01T09:22:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:12 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 12509/183450 objects degraded (6.819%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:12.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:12 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:22:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:12 smithi138 ceph-mon[192513]: pgmap v576: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 12509/183450 objects degraded (6.819%); 320/183450 objects misplaced (0.174%); 6.1 MiB/s, 97 objects/s recovering 2023-10-01T09:22:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:12 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 12509/183450 objects degraded (6.819%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:12.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:12 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:22:12.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:12 smithi138 ceph-mon[185886]: pgmap v576: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 12509/183450 objects degraded (6.819%); 320/183450 objects misplaced (0.174%); 6.1 MiB/s, 97 objects/s recovering 2023-10-01T09:22:12.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:12 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 12509/183450 objects degraded (6.819%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:12.937 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:22:12.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:12.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:22:13.042 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:13.085 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:13.174 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:22:13.333 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:13.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:13.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:13.442 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:22:13.474 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:13 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:22:13.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:13 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:22:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:13 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:22:14.209 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:22:14.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:14.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:22:14.318 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:14.395 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:14.533 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:14.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:14.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:14.541 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:14.641 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:14.704 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:14.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:14 smithi190 ceph-mon[160435]: pgmap v577: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 12509/176217 objects degraded (7.099%); 320/176217 objects misplaced (0.182%); 5.1 MiB/s, 80 objects/s recovering 2023-10-01T09:22:14.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:14 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:22:14.812 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:14.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:14.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:14.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:14 smithi138 ceph-mon[192513]: pgmap v577: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 12509/176217 objects degraded (7.099%); 320/176217 objects misplaced (0.182%); 5.1 MiB/s, 80 objects/s recovering 2023-10-01T09:22:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:14 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:22:14.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:14 smithi138 ceph-mon[185886]: pgmap v577: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 12509/176217 objects degraded (7.099%); 320/176217 objects misplaced (0.182%); 5.1 MiB/s, 80 objects/s recovering 2023-10-01T09:22:14.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:14 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:22:14.918 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:14.985 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:22:14.994 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:15.018 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:22:15.064 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:22:15.096 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:22:15.134 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:22:15.169 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:22:15.215 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:22:15.219 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:15.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:15.229 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:22:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:22:14] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.33.4" 2023-10-01T09:22:15.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:15.256 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:22:15.258 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:22:15.258 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:22:15.258 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:22:15.259 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:22:15.259 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:22:15.259 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:22:15.259 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:22:15.259 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:22:15.330 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:15.403 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:15.618 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:22:15.797 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:15.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:15 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:15 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:15 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/354722485' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:15 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:15 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/598497395' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:15.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:15 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:15.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:15 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:15.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:15 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/354722485' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:15.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:15 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:15.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:15 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/598497395' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:15.901 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:15.971 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:15 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:15.972 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:15 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:15.972 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:15 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/354722485' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:15.972 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:15 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:15.972 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:15 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/598497395' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:15.973 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:16.050 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 7, 2, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2023-10-01T09:22:16.050 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-01T09:22:16.078 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:16.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:16.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:16.196 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:16.263 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:16.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[192513]: 6.12 scrub starts 2023-10-01T09:22:16.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[192513]: pgmap v578: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 12224/170328 objects degraded (7.177%); 320/170328 objects misplaced (0.188%); 5.3 MiB/s, 83 objects/s recovering 2023-10-01T09:22:16.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:22:16.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[192513]: failed to encode map e231 with expected crc 2023-10-01T09:22:16.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:22:16.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[192513]: osdmap e231: 8 total, 8 up, 8 in 2023-10-01T09:22:16.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[192513]: failed to encode map e231 with expected crc 2023-10-01T09:22:16.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[192513]: failed to encode map e231 with expected crc 2023-10-01T09:22:16.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[192513]: failed to encode map e231 with expected crc 2023-10-01T09:22:16.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[192513]: failed to encode map e231 with expected crc 2023-10-01T09:22:16.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[192513]: failed to encode map e231 with expected crc 2023-10-01T09:22:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[185886]: 6.12 scrub starts 2023-10-01T09:22:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[185886]: pgmap v578: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 12224/170328 objects degraded (7.177%); 320/170328 objects misplaced (0.188%); 5.3 MiB/s, 83 objects/s recovering 2023-10-01T09:22:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:22:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[185886]: failed to encode map e231 with expected crc 2023-10-01T09:22:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:22:16.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[185886]: osdmap e231: 8 total, 8 up, 8 in 2023-10-01T09:22:16.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[185886]: failed to encode map e231 with expected crc 2023-10-01T09:22:16.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[185886]: failed to encode map e231 with expected crc 2023-10-01T09:22:16.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[185886]: failed to encode map e231 with expected crc 2023-10-01T09:22:16.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[185886]: failed to encode map e231 with expected crc 2023-10-01T09:22:16.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:16 smithi138 ceph-mon[185886]: failed to encode map e231 with expected crc 2023-10-01T09:22:17.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:16 smithi190 ceph-mon[160435]: 6.12 scrub starts 2023-10-01T09:22:17.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:16 smithi190 ceph-mon[160435]: pgmap v578: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 12224/170328 objects degraded (7.177%); 320/170328 objects misplaced (0.188%); 5.3 MiB/s, 83 objects/s recovering 2023-10-01T09:22:17.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:16 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:22:17.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:16 smithi190 ceph-mon[160435]: failed to encode map e231 with expected crc 2023-10-01T09:22:17.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:16 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:22:17.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:16 smithi190 ceph-mon[160435]: osdmap e231: 8 total, 8 up, 8 in 2023-10-01T09:22:17.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:16 smithi190 ceph-mon[160435]: failed to encode map e231 with expected crc 2023-10-01T09:22:17.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:16 smithi190 ceph-mon[160435]: failed to encode map e231 with expected crc 2023-10-01T09:22:17.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:16 smithi190 ceph-mon[160435]: failed to encode map e231 with expected crc 2023-10-01T09:22:17.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:16 smithi190 ceph-mon[160435]: failed to encode map e231 with expected crc 2023-10-01T09:22:17.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:16 smithi190 ceph-mon[160435]: failed to encode map e231 with expected crc 2023-10-01T09:22:17.656 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:17.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:17.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:17.769 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:17.831 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:17 smithi138 ceph-mon[185886]: 6.2 scrub starts 2023-10-01T09:22:17.831 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:17 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:22:17.831 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:17 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 12224/170328 objects degraded (7.177%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:17.832 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:17 smithi138 ceph-mon[192513]: 6.2 scrub starts 2023-10-01T09:22:17.832 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:17 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:22:17.832 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:17 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 12224/170328 objects degraded (7.177%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:17.835 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:17.929 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:17.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:17.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:18.038 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:18.041 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:22:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:17 smithi190 ceph-mon[160435]: 6.2 scrub starts 2023-10-01T09:22:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:17 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:22:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:17 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 12224/170328 objects degraded (7.177%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:18.093 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:18.188 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:22:18.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:18.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:22:18.297 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:22:18.368 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:18.443 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:22:18.443 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:22:18.721 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:22:18.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:18.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:22:18.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:18 smithi138 ceph-mon[192513]: pgmap v580: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 12224/162654 objects degraded (7.515%); 320/162654 objects misplaced (0.197%); 3.4 MiB/s, 54 objects/s recovering 2023-10-01T09:22:18.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:18 smithi138 ceph-mon[185886]: pgmap v580: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 12224/162654 objects degraded (7.515%); 320/162654 objects misplaced (0.197%); 3.4 MiB/s, 54 objects/s recovering 2023-10-01T09:22:18.827 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:18.887 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:19.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:18 smithi190 ceph-mon[160435]: pgmap v580: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 12224/162654 objects degraded (7.515%); 320/162654 objects misplaced (0.197%); 3.4 MiB/s, 54 objects/s recovering 2023-10-01T09:22:19.677 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:22:19.687 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:22:19.744 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:19.747 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:22:19.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:19.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:19.850 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:19.883 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:19 smithi138 ceph-mon[185886]: 6.12 scrub starts 2023-10-01T09:22:19.883 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:19 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:19.883 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:19 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:19.883 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:19 smithi138 ceph-mon[185886]: pgmap v581: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 12224/160044 objects degraded (7.638%); 320/160044 objects misplaced (0.200%); 3.4 MiB/s, 54 objects/s recovering 2023-10-01T09:22:19.883 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:19 smithi138 ceph-mon[192513]: 6.12 scrub starts 2023-10-01T09:22:19.883 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:19 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:19.883 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:19 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:19.883 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:19 smithi138 ceph-mon[192513]: pgmap v581: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 12224/160044 objects degraded (7.638%); 320/160044 objects misplaced (0.200%); 3.4 MiB/s, 54 objects/s recovering 2023-10-01T09:22:20.046 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:20.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:19 smithi190 ceph-mon[160435]: 6.12 scrub starts 2023-10-01T09:22:20.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:19 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:20.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:19 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:20.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:19 smithi190 ceph-mon[160435]: pgmap v581: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 12224/160044 objects degraded (7.638%); 320/160044 objects misplaced (0.200%); 3.4 MiB/s, 54 objects/s recovering 2023-10-01T09:22:20.185 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:20.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:20.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:20.289 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:22:20.350 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:20.486 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:22:20.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:20.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:22:20.595 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:20.678 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:20.780 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:20.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:20.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:20.900 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:20.956 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:20 smithi190 ceph-mon[160435]: 6.2 scrub starts 2023-10-01T09:22:20.956 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:20 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:22:20.957 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:20 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:22:20.957 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:20 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:20.967 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:21.003 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:21.032 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:21.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:20 smithi138 ceph-mon[192513]: 6.2 scrub starts 2023-10-01T09:22:21.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:20 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:22:21.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:20 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:22:21.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:20 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:20 smithi138 ceph-mon[185886]: 6.2 scrub starts 2023-10-01T09:22:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:20 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:22:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:20 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:22:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:20 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:21.095 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:21.100 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:21.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:21.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:21.210 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:21.275 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:21.385 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:21.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:21.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:21.499 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:21.605 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:21.810 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:21 smithi138 ceph-mon[185886]: 6.12 scrub starts 2023-10-01T09:22:21.811 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:21 smithi138 ceph-mon[185886]: pgmap v582: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 992 op/s; 12005/153663 objects degraded (7.813%); 206/153663 objects misplaced (0.134%); 3.9 MiB/s, 61 objects/s recovering 2023-10-01T09:22:21.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:21 smithi138 ceph-mon[192513]: 6.12 scrub starts 2023-10-01T09:22:21.814 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:21 smithi138 ceph-mon[192513]: pgmap v582: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 992 op/s; 12005/153663 objects degraded (7.813%); 206/153663 objects misplaced (0.134%); 3.9 MiB/s, 61 objects/s recovering 2023-10-01T09:22:22.048 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:22.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:22.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:22.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:21 smithi190 ceph-mon[160435]: 6.12 scrub starts 2023-10-01T09:22:22.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:21 smithi190 ceph-mon[160435]: pgmap v582: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 992 op/s; 12005/153663 objects degraded (7.813%); 206/153663 objects misplaced (0.134%); 3.9 MiB/s, 61 objects/s recovering 2023-10-01T09:22:22.154 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:22:22.213 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:22.684 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.0 to 1 (10000) 2023-10-01T09:22:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:22 smithi190 ceph-mon[160435]: 6.2 scrub starts 2023-10-01T09:22:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:22 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2611868703' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:22:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:22 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 12005/153663 objects degraded (7.813%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:23.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:22 smithi138 ceph-mon[185886]: 6.2 scrub starts 2023-10-01T09:22:23.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:22 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2611868703' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:22:23.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:22 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 12005/153663 objects degraded (7.813%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:23.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:22 smithi138 ceph-mon[192513]: 6.2 scrub starts 2023-10-01T09:22:23.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:22 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2611868703' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:22:23.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:22 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 12005/153663 objects degraded (7.813%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:23.944 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:23.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:23.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:23.953 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:23 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:22:23.953 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:23 smithi138 ceph-mon[192513]: failed to encode map e232 with expected crc 2023-10-01T09:22:23.953 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:23 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2611868703' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:22:23.953 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:23 smithi138 ceph-mon[192513]: osdmap e232: 8 total, 8 up, 8 in 2023-10-01T09:22:23.953 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:23 smithi138 ceph-mon[192513]: failed to encode map e232 with expected crc 2023-10-01T09:22:23.954 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:23 smithi138 ceph-mon[192513]: pgmap v584: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 12005/148323 objects degraded (8.094%); 206/148323 objects misplaced (0.139%); 2.6 MiB/s, 41 objects/s recovering 2023-10-01T09:22:23.954 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:23 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:22:23.954 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:23 smithi138 ceph-mon[185886]: failed to encode map e232 with expected crc 2023-10-01T09:22:23.954 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:23 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2611868703' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:22:23.954 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:23 smithi138 ceph-mon[185886]: osdmap e232: 8 total, 8 up, 8 in 2023-10-01T09:22:23.954 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:23 smithi138 ceph-mon[185886]: failed to encode map e232 with expected crc 2023-10-01T09:22:23.954 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:23 smithi138 ceph-mon[185886]: pgmap v584: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 12005/148323 objects degraded (8.094%); 206/148323 objects misplaced (0.139%); 2.6 MiB/s, 41 objects/s recovering 2023-10-01T09:22:24.053 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:24.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:23 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:22:24.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:23 smithi190 ceph-mon[160435]: failed to encode map e232 with expected crc 2023-10-01T09:22:24.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:23 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2611868703' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:22:24.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:23 smithi190 ceph-mon[160435]: osdmap e232: 8 total, 8 up, 8 in 2023-10-01T09:22:24.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:23 smithi190 ceph-mon[160435]: failed to encode map e232 with expected crc 2023-10-01T09:22:24.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:23 smithi190 ceph-mon[160435]: pgmap v584: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 12005/148323 objects degraded (8.094%); 206/148323 objects misplaced (0.139%); 2.6 MiB/s, 41 objects/s recovering 2023-10-01T09:22:24.094 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:24.171 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:24.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:24.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:24.278 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:24.340 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:24.919 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:24.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:24.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:24.976 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-01T09:22:25.025 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:22:25.099 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:25.286 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:22:25.298 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:22:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:22:24] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.33.4" 2023-10-01T09:22:25.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:25.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:22:25.400 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:25.464 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:25.767 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:22:25.767 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:22:25.903 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:22:25.955 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:22:26.002 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:22:26.050 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:22:26.080 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:22:26.113 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:22:26.146 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:22:26.185 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:22:26.196 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:22:26.196 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:22:26.196 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:22:26.196 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:22:26.196 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:22:26.196 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:22:26.196 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:22:26.196 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:22:26.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:26.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:22:26.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:26 smithi190 ceph-mon[160435]: pgmap v585: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 12005/142290 objects degraded (8.437%); 3.4 MiB/s, 77 objects/s recovering 2023-10-01T09:22:26.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:26 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3546989136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:22:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:26 smithi138 ceph-mon[192513]: pgmap v585: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 12005/142290 objects degraded (8.437%); 3.4 MiB/s, 77 objects/s recovering 2023-10-01T09:22:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:26 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3546989136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:22:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:26 smithi138 ceph-mon[185886]: pgmap v585: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 12005/142290 objects degraded (8.437%); 3.4 MiB/s, 77 objects/s recovering 2023-10-01T09:22:26.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:26 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3546989136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:26.696 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:26.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:26.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:26.802 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:26.846 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:27.322 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:22:27.334 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:27.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:27.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:27.438 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:27.509 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:27.541 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:22:27.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:27 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3685544298' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:22:27.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:27 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:22:27.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:27 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 12005/142290 objects degraded (8.437%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:27.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:27 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3685544298' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:22:27.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:27 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:22:27.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:27 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 12005/142290 objects degraded (8.437%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:27.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:27 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3685544298' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:22:27.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:27 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:22:27.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:27 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 12005/142290 objects degraded (8.437%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:27.622 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:27.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:27.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:27.737 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:27.803 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:27.897 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:27.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:27.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:28.019 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:22:28.055 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:28.104 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:22:28.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:28.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:22:28.130 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:22:28.210 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:28.270 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:28.505 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:28.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:28.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:28.540 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:28 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:22:28.540 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:28 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:22:28.540 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:28 smithi190 ceph-mon[160435]: osdmap e233: 8 total, 8 up, 8 in 2023-10-01T09:22:28.540 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:28 smithi190 ceph-mon[160435]: failed to encode map e233 with expected crc 2023-10-01T09:22:28.540 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:28 smithi190 ceph-mon[160435]: failed to encode map e233 with expected crc 2023-10-01T09:22:28.540 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:28 smithi190 ceph-mon[160435]: failed to encode map e233 with expected crc 2023-10-01T09:22:28.540 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:28 smithi190 ceph-mon[160435]: failed to encode map e233 with expected crc 2023-10-01T09:22:28.540 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:28 smithi190 ceph-mon[160435]: pgmap v587: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 12005/134412 objects degraded (8.931%); 4.2 MiB/s, 95 objects/s recovering 2023-10-01T09:22:28.568 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:28.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:22:28.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:22:28.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[185886]: osdmap e233: 8 total, 8 up, 8 in 2023-10-01T09:22:28.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[185886]: failed to encode map e233 with expected crc 2023-10-01T09:22:28.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[185886]: failed to encode map e233 with expected crc 2023-10-01T09:22:28.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[185886]: failed to encode map e233 with expected crc 2023-10-01T09:22:28.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[185886]: failed to encode map e233 with expected crc 2023-10-01T09:22:28.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[185886]: pgmap v587: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 12005/134412 objects degraded (8.931%); 4.2 MiB/s, 95 objects/s recovering 2023-10-01T09:22:28.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:22:28.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:22:28.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[192513]: osdmap e233: 8 total, 8 up, 8 in 2023-10-01T09:22:28.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[192513]: failed to encode map e233 with expected crc 2023-10-01T09:22:28.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[192513]: failed to encode map e233 with expected crc 2023-10-01T09:22:28.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[192513]: failed to encode map e233 with expected crc 2023-10-01T09:22:28.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[192513]: failed to encode map e233 with expected crc 2023-10-01T09:22:28.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:28 smithi138 ceph-mon[192513]: pgmap v587: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 12005/134412 objects degraded (8.931%); 4.2 MiB/s, 95 objects/s recovering 2023-10-01T09:22:28.617 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:22:28.701 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:28.902 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:22:28.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:28.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:22:29.015 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:29.117 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:29.256 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:29.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:29.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:29.363 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:29.425 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:29.506 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:29.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:29.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:29.615 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:22:29.647 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:29.732 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:22:29.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:29.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:22:29.836 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:29.879 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:30.036 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:30.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:30.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:30.155 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:30.228 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:30.292 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:22:30.318 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:30.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:30.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:30.428 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:30.506 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:22:30.526 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:30.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:30 smithi138 ceph-mon[192513]: pgmap v588: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 12005/131859 objects degraded (9.104%); 1.6 MiB/s, 53 objects/s recovering 2023-10-01T09:22:30.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:30 smithi138 ceph-mon[185886]: pgmap v588: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 12005/131859 objects degraded (9.104%); 1.6 MiB/s, 53 objects/s recovering 2023-10-01T09:22:30.631 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:30.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:30.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:30.741 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:22:30.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:30 smithi190 ceph-mon[160435]: pgmap v588: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 12005/131859 objects degraded (9.104%); 1.6 MiB/s, 53 objects/s recovering 2023-10-01T09:22:30.891 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:30.996 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:31.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:31.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:31.099 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:31.101 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:22:31.158 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:31.263 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:22:31.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:31.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:22:31.279 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:31.375 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:22:31.430 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:31.555 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:22:31.558 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.1 to 1 (10000) 2023-10-01T09:22:31.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:31.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:22:31.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4231002817' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:22:31.665 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4231002817' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:22:31.670 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:31.732 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4231002817' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:22:32.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:32 smithi190 ceph-mon[160435]: pgmap v589: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 12005/125607 objects degraded (9.558%); 1.5 MiB/s, 92 objects/s recovering 2023-10-01T09:22:32.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:32 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4231002817' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:22:32.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:32 smithi190 ceph-mon[160435]: osdmap e234: 8 total, 8 up, 8 in 2023-10-01T09:22:32.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:32 smithi190 ceph-mon[160435]: failed to encode map e234 with expected crc 2023-10-01T09:22:32.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:32 smithi190 ceph-mon[160435]: failed to encode map e234 with expected crc 2023-10-01T09:22:32.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:32 smithi190 ceph-mon[160435]: failed to encode map e234 with expected crc 2023-10-01T09:22:32.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:32 smithi190 ceph-mon[160435]: failed to encode map e234 with expected crc 2023-10-01T09:22:32.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:32 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 12005/125607 objects degraded (9.558%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:32.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[192513]: pgmap v589: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 12005/125607 objects degraded (9.558%); 1.5 MiB/s, 92 objects/s recovering 2023-10-01T09:22:32.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4231002817' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:22:32.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[192513]: osdmap e234: 8 total, 8 up, 8 in 2023-10-01T09:22:32.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[192513]: failed to encode map e234 with expected crc 2023-10-01T09:22:32.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[192513]: failed to encode map e234 with expected crc 2023-10-01T09:22:32.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[192513]: failed to encode map e234 with expected crc 2023-10-01T09:22:32.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[192513]: failed to encode map e234 with expected crc 2023-10-01T09:22:32.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 12005/125607 objects degraded (9.558%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:32.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[185886]: pgmap v589: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 12005/125607 objects degraded (9.558%); 1.5 MiB/s, 92 objects/s recovering 2023-10-01T09:22:32.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4231002817' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:22:32.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[185886]: osdmap e234: 8 total, 8 up, 8 in 2023-10-01T09:22:32.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[185886]: failed to encode map e234 with expected crc 2023-10-01T09:22:32.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[185886]: failed to encode map e234 with expected crc 2023-10-01T09:22:32.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[185886]: failed to encode map e234 with expected crc 2023-10-01T09:22:32.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[185886]: failed to encode map e234 with expected crc 2023-10-01T09:22:32.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:32 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 12005/125607 objects degraded (9.558%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:33.967 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:33.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:33.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:33.985 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-01T09:22:34.081 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:22:34.115 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:34.201 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:22:34.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:34.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:22:34.307 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:22:34.394 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:34 smithi190 ceph-mon[160435]: pgmap v591: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 12005/120669 objects degraded (9.949%); 0 B/s, 47 objects/s recovering 2023-10-01T09:22:34.716 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:22:34.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:34.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:22:34.730 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:34 smithi138 ceph-mon[185886]: pgmap v591: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 12005/120669 objects degraded (9.949%); 0 B/s, 47 objects/s recovering 2023-10-01T09:22:34.730 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:34 smithi138 ceph-mon[192513]: pgmap v591: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 12005/120669 objects degraded (9.949%); 0 B/s, 47 objects/s recovering 2023-10-01T09:22:34.824 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:34.885 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:22:34.885 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:34.886 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:34.913 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:22:34.950 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:22:34.999 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:34.999 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:22:35.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:35.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:35.048 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:22:35.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:22:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:22:34] "GET /metrics HTTP/1.1" 200 36436 "" "Prometheus/2.33.4" 2023-10-01T09:22:35.089 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:22:35.111 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:35.150 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:22:35.158 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:35.184 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:22:35.198 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:22:35.198 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:22:35.198 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:22:35.198 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:22:35.198 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:22:35.198 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:22:35.198 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:22:35.199 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:22:35.252 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:35.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:35.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:35.371 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:35.434 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:35.547 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:35.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:35.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:35.572 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2927811830' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:35.572 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/751701116' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:35.606 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:22:35.619 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:22:35.620 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:22:35.656 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:22:35.747 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:35.786 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2927811830' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:35.786 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/751701116' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:35.786 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2927811830' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:35.787 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/751701116' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:35.900 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:22:35.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:35.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:22:36.010 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:22:36.066 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:36.752 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:22:36.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:36.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:22:36.758 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:36 smithi138 ceph-mon[192513]: pgmap v592: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 12005/114738 objects degraded (10.463%); 0 B/s, 117 objects/s recovering 2023-10-01T09:22:36.758 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:36 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:22:36.758 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:36 smithi138 ceph-mon[192513]: failed to encode map e235 with expected crc 2023-10-01T09:22:36.758 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:36 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/751701116' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:22:36.758 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:36 smithi138 ceph-mon[192513]: osdmap e235: 8 total, 8 up, 8 in 2023-10-01T09:22:36.758 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:36 smithi138 ceph-mon[192513]: failed to encode map e235 with expected crc 2023-10-01T09:22:36.758 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:36 smithi138 ceph-mon[192513]: failed to encode map e235 with expected crc 2023-10-01T09:22:36.758 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:36 smithi138 ceph-mon[185886]: pgmap v592: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 12005/114738 objects degraded (10.463%); 0 B/s, 117 objects/s recovering 2023-10-01T09:22:36.759 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:36 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:22:36.759 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:36 smithi138 ceph-mon[185886]: failed to encode map e235 with expected crc 2023-10-01T09:22:36.759 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:36 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/751701116' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:22:36.759 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:36 smithi138 ceph-mon[185886]: osdmap e235: 8 total, 8 up, 8 in 2023-10-01T09:22:36.759 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:36 smithi138 ceph-mon[185886]: failed to encode map e235 with expected crc 2023-10-01T09:22:36.759 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:36 smithi138 ceph-mon[185886]: failed to encode map e235 with expected crc 2023-10-01T09:22:36.826 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:22:36.826 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:36 smithi190 ceph-mon[160435]: pgmap v592: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 908 op/s; 12005/114738 objects degraded (10.463%); 0 B/s, 117 objects/s recovering 2023-10-01T09:22:36.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:36 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:22:36.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:36 smithi190 ceph-mon[160435]: failed to encode map e235 with expected crc 2023-10-01T09:22:36.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:36 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/751701116' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:22:36.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:36 smithi190 ceph-mon[160435]: osdmap e235: 8 total, 8 up, 8 in 2023-10-01T09:22:36.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:36 smithi190 ceph-mon[160435]: failed to encode map e235 with expected crc 2023-10-01T09:22:36.827 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:36 smithi190 ceph-mon[160435]: failed to encode map e235 with expected crc 2023-10-01T09:22:36.857 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:22:36.904 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:37.515 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:37.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:37.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:37.548 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:22:37.620 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:22:37.712 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:37.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:37 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 12005/114738 objects degraded (10.463%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:37.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:37 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 12005/114738 objects degraded (10.463%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:37.905 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:37 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 12005/114738 objects degraded (10.463%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:37.916 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:22:37.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:37.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:22:38.033 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:38.113 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:38.337 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:38.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:38.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:38.444 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:38.517 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:38.602 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:38.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:38.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:38.708 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:22:38.734 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:38.775 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:22:38.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:38.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:22:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:38 smithi138 ceph-mon[192513]: pgmap v594: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 12005/107331 objects degraded (11.185%); 0 B/s, 119 objects/s recovering 2023-10-01T09:22:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/26542858' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:22:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:38 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:22:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:38 smithi138 ceph-mon[185886]: pgmap v594: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 12005/107331 objects degraded (11.185%); 0 B/s, 119 objects/s recovering 2023-10-01T09:22:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/26542858' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:22:38.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:38 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:22:38.878 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:38.902 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:38 smithi190 ceph-mon[160435]: pgmap v594: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 12005/107331 objects degraded (11.185%); 0 B/s, 119 objects/s recovering 2023-10-01T09:22:38.903 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/26542858' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:22:38.903 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:38 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:22:38.905 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:38.996 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:39.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:39.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:39.106 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:39.177 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.3 to 1 (10000) 2023-10-01T09:22:39.184 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:39.300 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:39.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:39.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:39.404 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:39.443 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:39.476 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:22:39.486 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:39.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:39.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:39.517 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:22:39.590 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:39.622 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:39.669 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:39.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:39.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:39.755 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:22:39.755 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:22:39.773 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:39.800 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:40.268 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:40.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:40.371 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-01T09:22:40.371 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:40.398 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:40.443 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:40.499 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:40.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:40 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:22:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:40 smithi190 ceph-mon[160435]: osdmap e236: 8 total, 8 up, 8 in 2023-10-01T09:22:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:40 smithi190 ceph-mon[160435]: failed to encode map e236 with expected crc 2023-10-01T09:22:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:40 smithi190 ceph-mon[160435]: failed to encode map e236 with expected crc 2023-10-01T09:22:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:40 smithi190 ceph-mon[160435]: failed to encode map e236 with expected crc 2023-10-01T09:22:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:40 smithi190 ceph-mon[160435]: failed to encode map e236 with expected crc 2023-10-01T09:22:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:40 smithi190 ceph-mon[160435]: pgmap v596: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 12005/104538 objects degraded (11.484%); 0 B/s, 73 objects/s recovering 2023-10-01T09:22:40.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:40 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:22:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:40 smithi138 ceph-mon[192513]: osdmap e236: 8 total, 8 up, 8 in 2023-10-01T09:22:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:40 smithi138 ceph-mon[192513]: failed to encode map e236 with expected crc 2023-10-01T09:22:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:40 smithi138 ceph-mon[192513]: failed to encode map e236 with expected crc 2023-10-01T09:22:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:40 smithi138 ceph-mon[192513]: failed to encode map e236 with expected crc 2023-10-01T09:22:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:40 smithi138 ceph-mon[192513]: failed to encode map e236 with expected crc 2023-10-01T09:22:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:40 smithi138 ceph-mon[192513]: pgmap v596: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 12005/104538 objects degraded (11.484%); 0 B/s, 73 objects/s recovering 2023-10-01T09:22:40.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:40 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:22:40.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:40 smithi138 ceph-mon[185886]: osdmap e236: 8 total, 8 up, 8 in 2023-10-01T09:22:40.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:40 smithi138 ceph-mon[185886]: failed to encode map e236 with expected crc 2023-10-01T09:22:40.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:40 smithi138 ceph-mon[185886]: failed to encode map e236 with expected crc 2023-10-01T09:22:40.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:40 smithi138 ceph-mon[185886]: failed to encode map e236 with expected crc 2023-10-01T09:22:40.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:40 smithi138 ceph-mon[185886]: failed to encode map e236 with expected crc 2023-10-01T09:22:40.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:40 smithi138 ceph-mon[185886]: pgmap v596: 99 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 12005/104538 objects degraded (11.484%); 0 B/s, 73 objects/s recovering 2023-10-01T09:22:40.620 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:40.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:40.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:40.677 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:22:40.724 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:40.750 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:40.794 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:40.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:40.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:40.900 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:40.928 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:41.025 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:41.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:41.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:41.048 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:22:41.130 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:22:41.160 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:41.247 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:41 smithi190 ceph-mon[160435]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.248 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:41 smithi190 ceph-mon[160435]: osdmap e237: 8 total, 8 up, 8 in 2023-10-01T09:22:41.248 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:41 smithi190 ceph-mon[160435]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.248 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:41 smithi190 ceph-mon[160435]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.248 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:41 smithi190 ceph-mon[160435]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.248 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:41 smithi190 ceph-mon[160435]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.248 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:41 smithi190 ceph-mon[160435]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.248 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:41 smithi190 ceph-mon[160435]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.248 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:41 smithi190 ceph-mon[160435]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.248 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:41 smithi190 ceph-mon[160435]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.248 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:41.249 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:22:41.383 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:41.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:41.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:41.394 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[192513]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.394 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[192513]: osdmap e237: 8 total, 8 up, 8 in 2023-10-01T09:22:41.395 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[192513]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.395 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[192513]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.395 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[192513]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.395 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[192513]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.395 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[192513]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.395 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[192513]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.395 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[192513]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.395 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[192513]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.395 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:41.395 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:22:41.397 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[185886]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.397 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[185886]: osdmap e237: 8 total, 8 up, 8 in 2023-10-01T09:22:41.397 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[185886]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.397 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[185886]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.397 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[185886]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.397 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[185886]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.397 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[185886]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.397 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[185886]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.397 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[185886]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.397 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[185886]: failed to encode map e237 with expected crc 2023-10-01T09:22:41.398 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:22:41.398 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:22:41.495 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:41.546 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:41.672 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:41.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:41.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:41.787 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:22:41.852 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:42.181 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[185886]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.181 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[185886]: osdmap e238: 8 total, 8 up, 8 in 2023-10-01T09:22:42.181 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[185886]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.181 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[185886]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.181 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[185886]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.181 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[185886]: 6.f starting backfill to osd.3 from (0'0,0'0] MAX to 235'9642 2023-10-01T09:22:42.182 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[185886]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.182 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[185886]: 6.4 starting backfill to osd.3 from (0'0,0'0] MAX to 235'8191 2023-10-01T09:22:42.182 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[185886]: 6.14 starting backfill to osd.3 from (0'0,0'0] MAX to 235'8330 2023-10-01T09:22:42.182 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[185886]: 6.c starting backfill to osd.3 from (0'0,0'0] MAX to 235'9877 2023-10-01T09:22:42.182 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[185886]: pgmap v599: 99 pgs: 1 remapped+peering, 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 8999/103158 objects degraded (8.724%); 0 B/s, 67 objects/s recovering 2023-10-01T09:22:42.182 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 12005/104538 objects degraded (11.484%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:42.182 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[192513]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.183 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[192513]: osdmap e238: 8 total, 8 up, 8 in 2023-10-01T09:22:42.183 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[192513]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.183 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[192513]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.183 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[192513]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.183 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[192513]: 6.f starting backfill to osd.3 from (0'0,0'0] MAX to 235'9642 2023-10-01T09:22:42.183 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[192513]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.183 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[192513]: 6.4 starting backfill to osd.3 from (0'0,0'0] MAX to 235'8191 2023-10-01T09:22:42.183 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[192513]: 6.14 starting backfill to osd.3 from (0'0,0'0] MAX to 235'8330 2023-10-01T09:22:42.183 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[192513]: 6.c starting backfill to osd.3 from (0'0,0'0] MAX to 235'9877 2023-10-01T09:22:42.183 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[192513]: pgmap v599: 99 pgs: 1 remapped+peering, 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 8999/103158 objects degraded (8.724%); 0 B/s, 67 objects/s recovering 2023-10-01T09:22:42.183 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:42 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 12005/104538 objects degraded (11.484%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:42.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:42 smithi190 ceph-mon[160435]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:42 smithi190 ceph-mon[160435]: osdmap e238: 8 total, 8 up, 8 in 2023-10-01T09:22:42.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:42 smithi190 ceph-mon[160435]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:42 smithi190 ceph-mon[160435]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:42 smithi190 ceph-mon[160435]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:42 smithi190 ceph-mon[160435]: 6.f starting backfill to osd.3 from (0'0,0'0] MAX to 235'9642 2023-10-01T09:22:42.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:42 smithi190 ceph-mon[160435]: failed to encode map e238 with expected crc 2023-10-01T09:22:42.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:42 smithi190 ceph-mon[160435]: 6.4 starting backfill to osd.3 from (0'0,0'0] MAX to 235'8191 2023-10-01T09:22:42.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:42 smithi190 ceph-mon[160435]: 6.14 starting backfill to osd.3 from (0'0,0'0] MAX to 235'8330 2023-10-01T09:22:42.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:42 smithi190 ceph-mon[160435]: 6.c starting backfill to osd.3 from (0'0,0'0] MAX to 235'9877 2023-10-01T09:22:42.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:42 smithi190 ceph-mon[160435]: pgmap v599: 99 pgs: 1 remapped+peering, 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 8999/103158 objects degraded (8.724%); 0 B/s, 67 objects/s recovering 2023-10-01T09:22:42.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:42 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 12005/104538 objects degraded (11.484%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:42.877 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:22:42.878 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:22:43.472 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:43.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:43.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:43.534 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:43.580 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:22:43.671 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:43.672 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:22:43.833 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:43.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:43.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:43.944 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:44.025 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:44.247 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:22:44.262 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:44.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:44.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:44.374 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:22:44.416 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:44.548 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:22:44.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:44.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:22:44.656 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:22:44.690 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:22:44.704 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:44 smithi190 ceph-mon[160435]: pgmap v600: 99 pgs: 1 remapped+peering, 2 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 13681/99117 objects degraded (13.803%); 3.3 MiB/s, 119 objects/s recovering 2023-10-01T09:22:44.710 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:44.726 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:22:44.763 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:22:44.806 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:22:44.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:44 smithi138 ceph-mon[192513]: pgmap v600: 99 pgs: 1 remapped+peering, 2 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 13681/99117 objects degraded (13.803%); 3.3 MiB/s, 119 objects/s recovering 2023-10-01T09:22:44.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:44 smithi138 ceph-mon[185886]: pgmap v600: 99 pgs: 1 remapped+peering, 2 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 13681/99117 objects degraded (13.803%); 3.3 MiB/s, 119 objects/s recovering 2023-10-01T09:22:44.853 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:22:44.890 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:22:44.934 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:22:44.963 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:22:44.965 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:22:44.965 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:22:44.965 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:22:44.965 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:22:44.965 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:22:44.965 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:22:44.966 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:22:44.966 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:22:44.969 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:22:44.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:44.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:22:45.076 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:45.145 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:22:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:22:44] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.33.4" 2023-10-01T09:22:45.159 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:45.291 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:45.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:45.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:45.400 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:22:45.469 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:45.540 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:22:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:45 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/162112832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:45 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3666242247' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:22:45.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:45 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:22:45.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:45 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/162112832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:45.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:45 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3666242247' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:22:45.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:45 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:22:45.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:45 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/162112832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:45.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:45 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3666242247' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:22:45.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:45 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:22:45.899 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:45.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:45.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:46.010 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:46.087 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:46.124 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:22:46.125 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:22:46.125 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.125 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:161664 6.c 6:3368ea55:::benchmark_data_smithi138_307751_object66295:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-01T09:22:46.125 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:46.099905+0000", 2023-10-01T09:22:46.125 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.022887877000000001, 2023-10-01T09:22:46.125 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.022922676999999999, 2023-10-01T09:22:46.125 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:46.126 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:22:46.126 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:46.126 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:46.126 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:46.126 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 161664 2023-10-01T09:22:46.126 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.126 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:46.126 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.126 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:46.127 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.099905+0000", 2023-10-01T09:22:46.127 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:46.127 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.127 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.127 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:46.127 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.099905+0000", 2023-10-01T09:22:46.127 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:22:46.127 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.127 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.128 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:46.128 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.099902+0000", 2023-10-01T09:22:46.128 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0176e-05 2023-10-01T09:22:46.128 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.128 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.128 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:46.128 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.099913+0000", 2023-10-01T09:22:46.128 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0205999999999999e-05 2023-10-01T09:22:46.128 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.128 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.128 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:46.129 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.099933+0000", 2023-10-01T09:22:46.129 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3780000000000001e-06 2023-10-01T09:22:46.129 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.129 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.129 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:46.129 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.099941+0000", 2023-10-01T09:22:46.129 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6118000000000006e-05 2023-10-01T09:22:46.129 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.130 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.130 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:46.130 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100007+0000", 2023-10-01T09:22:46.130 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00019761099999999999 2023-10-01T09:22:46.130 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.130 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.130 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:46.130 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100205+0000", 2023-10-01T09:22:46.130 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000199514 2023-10-01T09:22:46.131 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.131 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.131 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,5,7", 2023-10-01T09:22:46.131 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100404+0000", 2023-10-01T09:22:46.131 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016533031 2023-10-01T09:22:46.131 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.131 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.131 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:46.131 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.116937+0000", 2023-10-01T09:22:46.131 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7800999999999999e-05 2023-10-01T09:22:46.131 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.131 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.132 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:46.132 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.116985+0000", 2023-10-01T09:22:46.132 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017080352 2023-10-01T09:22:46.132 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.132 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:46.132 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.132 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.132 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.132 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:161665 6.c 6:385f2ce4:::benchmark_data_smithi138_307751_object66296:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-01T09:22:46.132 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:46.099956+0000", 2023-10-01T09:22:46.132 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.022836545999999999, 2023-10-01T09:22:46.133 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.022955427, 2023-10-01T09:22:46.133 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:46.133 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:22:46.133 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:46.133 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:46.133 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:46.133 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 161665 2023-10-01T09:22:46.133 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.133 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:46.133 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.133 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:46.133 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.099956+0000", 2023-10-01T09:22:46.134 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:46.134 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.134 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.134 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:46.134 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.099956+0000", 2023-10-01T09:22:46.134 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:22:46.134 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.134 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.134 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:46.134 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.099956+0000", 2023-10-01T09:22:46.135 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1309999999999999e-06 2023-10-01T09:22:46.135 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.135 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.135 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:46.135 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.099960+0000", 2023-10-01T09:22:46.135 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0019e-05 2023-10-01T09:22:46.135 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.135 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.135 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:46.135 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.099970+0000", 2023-10-01T09:22:46.135 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3920000000000002e-06 2023-10-01T09:22:46.135 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.135 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.136 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:46.136 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.099975+0000", 2023-10-01T09:22:46.136 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003942058 2023-10-01T09:22:46.136 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.136 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.136 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:46.136 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.103917+0000", 2023-10-01T09:22:46.136 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014757299999999999 2023-10-01T09:22:46.136 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.136 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.136 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:46.137 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.104065+0000", 2023-10-01T09:22:46.137 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000158942 2023-10-01T09:22:46.137 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.137 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.137 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,5,7", 2023-10-01T09:22:46.137 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.104224+0000", 2023-10-01T09:22:46.137 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012804719000000001 2023-10-01T09:22:46.137 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.137 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.138 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:46.138 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.117028+0000", 2023-10-01T09:22:46.138 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017072232 2023-10-01T09:22:46.138 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.138 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:46.138 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.138 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.138 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.138 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:161666 6.c 6:3cc5800c:::benchmark_data_smithi138_307751_object66297:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-01T09:22:46.138 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:46.100007+0000", 2023-10-01T09:22:46.139 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.022786042999999999, 2023-10-01T09:22:46.139 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.022996834000000001, 2023-10-01T09:22:46.139 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:46.139 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:22:46.139 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:46.139 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:46.139 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:46.139 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 161666 2023-10-01T09:22:46.139 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.139 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:46.139 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.139 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:46.140 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100007+0000", 2023-10-01T09:22:46.140 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:46.140 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.140 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.140 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:46.140 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100007+0000", 2023-10-01T09:22:46.140 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:22:46.140 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.140 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.140 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:46.140 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100006+0000", 2023-10-01T09:22:46.140 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6500000000000002e-06 2023-10-01T09:22:46.141 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.141 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.141 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:46.141 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100010+0000", 2023-10-01T09:22:46.141 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0829000000000001e-05 2023-10-01T09:22:46.141 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.141 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.141 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:46.141 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100021+0000", 2023-10-01T09:22:46.141 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9150000000000004e-06 2023-10-01T09:22:46.141 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.142 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.142 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:46.142 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100026+0000", 2023-10-01T09:22:46.142 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0076082709999999998 2023-10-01T09:22:46.142 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.142 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.142 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:46.142 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.107634+0000", 2023-10-01T09:22:46.142 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013856400000000001 2023-10-01T09:22:46.142 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.142 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.143 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:46.143 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.107772+0000", 2023-10-01T09:22:46.143 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015625599999999999 2023-10-01T09:22:46.143 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.143 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.143 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,5,7", 2023-10-01T09:22:46.143 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.107929+0000", 2023-10-01T09:22:46.143 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011611567 2023-10-01T09:22:46.143 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.143 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.143 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:46.143 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.119540+0000", 2023-10-01T09:22:46.145 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019533564 2023-10-01T09:22:46.145 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.145 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:46.145 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.146 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.146 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.146 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:161669 6.c 6:3f386a3f:::benchmark_data_smithi138_307751_object66300:head [delete] snapc 0=[] ondisk+write+known_if_redirected e239)", 2023-10-01T09:22:46.146 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:46.100031+0000", 2023-10-01T09:22:46.146 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.022761409999999999, 2023-10-01T09:22:46.146 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.023061694000000001, 2023-10-01T09:22:46.146 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:46.146 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:22:46.146 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:46.146 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:46.147 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:46.147 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 161669 2023-10-01T09:22:46.147 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.147 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:46.147 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.147 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:46.147 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100031+0000", 2023-10-01T09:22:46.147 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:46.147 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.147 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.147 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:46.148 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100031+0000", 2023-10-01T09:22:46.148 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:22:46.148 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.148 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.148 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:46.148 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100031+0000", 2023-10-01T09:22:46.148 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2839999999999998e-06 2023-10-01T09:22:46.148 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.148 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.148 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:46.148 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100034+0000", 2023-10-01T09:22:46.149 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0703999999999999e-05 2023-10-01T09:22:46.149 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.149 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.149 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:46.149 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100045+0000", 2023-10-01T09:22:46.149 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7090000000000001e-06 2023-10-01T09:22:46.149 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.149 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.149 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:46.149 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100049+0000", 2023-10-01T09:22:46.149 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019521459000000001 2023-10-01T09:22:46.149 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.150 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.150 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:46.150 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.119571+0000", 2023-10-01T09:22:46.150 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015175599999999999 2023-10-01T09:22:46.150 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.150 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.150 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:46.150 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.119723+0000", 2023-10-01T09:22:46.150 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017024400000000001 2023-10-01T09:22:46.150 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.150 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.150 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,5,7", 2023-10-01T09:22:46.151 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.119893+0000", 2023-10-01T09:22:46.151 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019861568 2023-10-01T09:22:46.151 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.151 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:46.151 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.151 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.151 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.151 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:161671 6.c 6.88aad62c (undecoded) ondisk+write+known_if_redirected e239)", 2023-10-01T09:22:46.151 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:46.100054+0000", 2023-10-01T09:22:46.151 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.022738576999999999, 2023-10-01T09:22:46.151 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.02311237, 2023-10-01T09:22:46.151 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:46.152 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:22:46.152 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:46.152 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:46.152 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:46.152 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 161671 2023-10-01T09:22:46.152 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.152 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:46.152 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.152 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:46.152 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100054+0000", 2023-10-01T09:22:46.152 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:46.152 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.153 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.153 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:46.153 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100054+0000", 2023-10-01T09:22:46.153 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:22:46.153 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.153 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.153 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:46.153 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100054+0000", 2023-10-01T09:22:46.153 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.954e-06 2023-10-01T09:22:46.153 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.153 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.153 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:46.154 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100057+0000", 2023-10-01T09:22:46.154 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2867999999999999e-05 2023-10-01T09:22:46.154 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.154 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.154 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:46.154 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100070+0000", 2023-10-01T09:22:46.154 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9089999999999999e-06 2023-10-01T09:22:46.154 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.154 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.154 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:46.154 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.100074+0000", 2023-10-01T09:22:46.154 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0287000000000001e-05 2023-10-01T09:22:46.155 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.155 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:46.155 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.155 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.155 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.155 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:161676 6.4 e239/237 6:22bc7007:::benchmark_data_smithi138_307751_object66307:head v 239'8306, mlcod=197'4763)", 2023-10-01T09:22:46.155 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:46.109207+0000", 2023-10-01T09:22:46.155 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.013585463000000001, 2023-10-01T09:22:46.155 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014016585999999999, 2023-10-01T09:22:46.155 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:46.155 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:22:46.156 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:46.156 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.156 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:46.156 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.109207+0000", 2023-10-01T09:22:46.156 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:46.156 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.156 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.156 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:46.156 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.109207+0000", 2023-10-01T09:22:46.156 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:22:46.156 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.156 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.157 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:46.157 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.109207+0000", 2023-10-01T09:22:46.157 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.365e-06 2023-10-01T09:22:46.157 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.157 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.157 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:46.157 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.109213+0000", 2023-10-01T09:22:46.157 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1231e-05 2023-10-01T09:22:46.157 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.157 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.157 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:46.157 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.109224+0000", 2023-10-01T09:22:46.158 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4909999999999996e-06 2023-10-01T09:22:46.158 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.158 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.158 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:46.158 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.109230+0000", 2023-10-01T09:22:46.158 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078221000000000002 2023-10-01T09:22:46.158 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.158 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.158 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:46.158 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.117052+0000", 2023-10-01T09:22:46.158 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3569000000000001e-05 2023-10-01T09:22:46.159 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.159 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.159 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:46.159 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.117075+0000", 2023-10-01T09:22:46.159 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078680999999999994 2023-10-01T09:22:46.159 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.159 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:46.159 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.159 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.159 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.159 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:161679 6.5 e239/220 6:a6e53f26:::benchmark_data_smithi138_307751_object66310:head v 239'8427, mlcod=239'8425)", 2023-10-01T09:22:46.159 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:46.122774+0000", 2023-10-01T09:22:46.160 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.8467000000000001e-05, 2023-10-01T09:22:46.160 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00051140099999999998, 2023-10-01T09:22:46.160 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:46.160 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:22:46.160 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:46.160 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.160 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:46.160 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122774+0000", 2023-10-01T09:22:46.160 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:46.160 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.160 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.160 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:46.161 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122774+0000", 2023-10-01T09:22:46.161 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:22:46.161 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.161 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.161 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:46.161 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122773+0000", 2023-10-01T09:22:46.161 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0319999999999999e-05 2023-10-01T09:22:46.161 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.161 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.161 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:46.161 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122783+0000", 2023-10-01T09:22:46.161 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4241e-05 2023-10-01T09:22:46.162 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.162 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.162 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:46.162 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122798+0000", 2023-10-01T09:22:46.162 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0339999999999999e-06 2023-10-01T09:22:46.162 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.162 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.162 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:46.162 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122805+0000", 2023-10-01T09:22:46.162 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00023089399999999999 2023-10-01T09:22:46.162 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.163 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.163 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:46.163 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.123035+0000", 2023-10-01T09:22:46.163 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8719000000000002e-05 2023-10-01T09:22:46.163 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.163 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.163 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:46.163 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.123054+0000", 2023-10-01T09:22:46.163 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00027994100000000002 2023-10-01T09:22:46.163 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.163 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:46.163 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.164 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.164 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.164 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop_reply(client.35070.0:161665 6.c e239/237)", 2023-10-01T09:22:46.164 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:46.122822+0000", 2023-10-01T09:22:46.164 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 4294967295.9999709, 2023-10-01T09:22:46.164 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00053302099999999995, 2023-10-01T09:22:46.164 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:46.164 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:22:46.164 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:46.164 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.164 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:46.164 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122822+0000", 2023-10-01T09:22:46.165 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:46.165 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.165 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.165 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:46.165 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122822+0000", 2023-10-01T09:22:46.165 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:22:46.165 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.165 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.165 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:46.165 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122821+0000", 2023-10-01T09:22:46.165 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.455e-06 2023-10-01T09:22:46.165 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.166 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.166 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:46.166 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122825+0000", 2023-10-01T09:22:46.166 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8590000000000001e-06 2023-10-01T09:22:46.166 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.166 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.166 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:46.166 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122831+0000", 2023-10-01T09:22:46.166 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5700000000000001e-06 2023-10-01T09:22:46.166 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.166 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.167 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:46.167 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122834+0000", 2023-10-01T09:22:46.167 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2584e-05 2023-10-01T09:22:46.167 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.167 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:46.167 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.167 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.167 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.167 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop_reply(client.35070.0:161666 6.c e239/237)", 2023-10-01T09:22:46.167 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:46.122839+0000", 2023-10-01T09:22:46.167 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 4294967295.9999542, 2023-10-01T09:22:46.168 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00056327899999999997, 2023-10-01T09:22:46.168 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:46.168 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:22:46.168 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:46.168 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.168 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:46.168 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122839+0000", 2023-10-01T09:22:46.168 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:46.168 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.168 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.168 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:46.168 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122839+0000", 2023-10-01T09:22:46.169 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967296 2023-10-01T09:22:46.169 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.169 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.169 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:46.169 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122838+0000", 2023-10-01T09:22:46.169 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9740000000000001e-06 2023-10-01T09:22:46.169 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.169 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.169 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:46.169 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122840+0000", 2023-10-01T09:22:46.169 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4259999999999996e-06 2023-10-01T09:22:46.169 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.169 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.170 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:46.170 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122845+0000", 2023-10-01T09:22:46.170 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.847e-06 2023-10-01T09:22:46.170 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.170 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.170 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:46.170 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122848+0000", 2023-10-01T09:22:46.170 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0699999999999996e-06 2023-10-01T09:22:46.170 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.170 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:46.170 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.171 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.171 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.171 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:161684 6.15 e239/220)", 2023-10-01T09:22:46.171 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:46.122851+0000", 2023-10-01T09:22:46.171 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 4294967295.9999418, 2023-10-01T09:22:46.171 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00059835699999999999, 2023-10-01T09:22:46.171 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:46.171 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:22:46.171 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:46.171 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.171 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:46.171 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122851+0000", 2023-10-01T09:22:46.172 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:46.172 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.172 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.172 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:46.172 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122851+0000", 2023-10-01T09:22:46.172 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967296 2023-10-01T09:22:46.172 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.172 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.172 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:46.172 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122851+0000", 2023-10-01T09:22:46.172 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8879999999999999e-06 2023-10-01T09:22:46.173 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.173 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.173 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:46.173 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122855+0000", 2023-10-01T09:22:46.173 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6709999999999998e-06 2023-10-01T09:22:46.173 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.173 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.173 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:46.173 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122859+0000", 2023-10-01T09:22:46.173 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0939999999999999e-06 2023-10-01T09:22:46.173 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.173 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.173 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:46.174 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122863+0000", 2023-10-01T09:22:46.174 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1464000000000001e-05 2023-10-01T09:22:46.174 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.174 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:46.174 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.174 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.174 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.174 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:161680 6.1 e239/220 6:8567c76c:::benchmark_data_smithi138_307751_object66311:head v 239'8377, mlcod=239'8375)", 2023-10-01T09:22:46.174 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:46.122879+0000", 2023-10-01T09:22:46.174 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 4294967295.9999142, 2023-10-01T09:22:46.175 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00062195200000000005, 2023-10-01T09:22:46.175 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:46.175 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:22:46.175 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:46.175 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.175 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:46.175 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122879+0000", 2023-10-01T09:22:46.175 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:46.175 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.175 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.175 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:46.175 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122879+0000", 2023-10-01T09:22:46.176 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:22:46.176 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.176 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.176 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:46.176 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122878+0000", 2023-10-01T09:22:46.176 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6539999999999998e-06 2023-10-01T09:22:46.176 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.176 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.176 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:46.176 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122883+0000", 2023-10-01T09:22:46.176 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7039999999999999e-06 2023-10-01T09:22:46.176 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.177 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.177 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:46.177 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122889+0000", 2023-10-01T09:22:46.177 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4520000000000002e-06 2023-10-01T09:22:46.177 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.177 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.177 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:46.177 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122892+0000", 2023-10-01T09:22:46.177 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3365000000000002e-05 2023-10-01T09:22:46.177 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.177 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.177 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:46.178 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122935+0000", 2023-10-01T09:22:46.178 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2540000000000001e-05 2023-10-01T09:22:46.178 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:46.178 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:46.178 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:46.178 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:46.122958+0000", 2023-10-01T09:22:46.178 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9375999999999994e-05 2023-10-01T09:22:46.178 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.178 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:46.178 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.178 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:46.179 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:22:46.179 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 11 2023-10-01T09:22:46.179 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:22:46.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:46.180 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:46.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:46.281 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:46.332 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:46.403 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:46.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:46.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:46.510 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:22:46.567 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:46 smithi190 ceph-mon[160435]: pgmap v601: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 20626/94599 objects degraded (21.804%); 7755/94599 objects misplaced (8.198%); 3.1 MiB/s, 114 objects/s recovering 2023-10-01T09:22:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:46 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:22:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:46 smithi190 ceph-mon[160435]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:46 smithi190 ceph-mon[160435]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:46 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:22:46.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:46 smithi190 ceph-mon[160435]: osdmap e239: 8 total, 8 up, 8 in 2023-10-01T09:22:46.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:46 smithi190 ceph-mon[160435]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:46 smithi190 ceph-mon[160435]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:46 smithi190 ceph-mon[160435]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:46 smithi190 ceph-mon[160435]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[185886]: pgmap v601: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 20626/94599 objects degraded (21.804%); 7755/94599 objects misplaced (8.198%); 3.1 MiB/s, 114 objects/s recovering 2023-10-01T09:22:46.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:22:46.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[185886]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[185886]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:22:46.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[185886]: osdmap e239: 8 total, 8 up, 8 in 2023-10-01T09:22:46.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[185886]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[185886]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[185886]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[185886]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[192513]: pgmap v601: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 20626/94599 objects degraded (21.804%); 7755/94599 objects misplaced (8.198%); 3.1 MiB/s, 114 objects/s recovering 2023-10-01T09:22:46.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:22:46.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[192513]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[192513]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:22:46.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[192513]: osdmap e239: 8 total, 8 up, 8 in 2023-10-01T09:22:46.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[192513]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[192513]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[192513]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:46 smithi138 ceph-mon[192513]: failed to encode map e239 with expected crc 2023-10-01T09:22:46.965 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:22:47.387 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:47.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:47.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:47.493 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:47.505 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:22:47.557 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:47.776 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:47.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:47.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:47.794 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:47 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:22:47.794 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:47 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 20626/94599 objects degraded (21.804%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:47.805 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:47 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:22:47.805 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:47 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 20626/94599 objects degraded (21.804%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:47.806 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:47 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:22:47.806 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:47 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 20626/94599 objects degraded (21.804%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:47.884 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:22:47.934 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:48.117 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:22:48.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:48.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:22:48.221 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:48.262 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:48.363 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:48.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:48.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:48.471 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:22:48.537 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:48.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:48 smithi190 ceph-mon[160435]: pgmap v603: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 20419/88122 objects degraded (23.171%); 7755/88122 objects misplaced (8.800%); 4.5 MiB/s, 90 objects/s recovering 2023-10-01T09:22:48.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:48 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1444881374' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:22:48.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:48 smithi138 ceph-mon[185886]: pgmap v603: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 20419/88122 objects degraded (23.171%); 7755/88122 objects misplaced (8.800%); 4.5 MiB/s, 90 objects/s recovering 2023-10-01T09:22:48.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:48 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1444881374' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:22:48.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:48 smithi138 ceph-mon[192513]: pgmap v603: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 20419/88122 objects degraded (23.171%); 7755/88122 objects misplaced (8.800%); 4.5 MiB/s, 90 objects/s recovering 2023-10-01T09:22:48.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:48 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1444881374' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:22:48.874 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:22:48.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:48.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:22:48.984 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:22:49.061 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:49.184 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.4 to 1 (10000) 2023-10-01T09:22:49.218 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:22:49.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:49.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:22:49.276 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:22:49.332 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:49.391 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:49.495 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:49.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:49.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:49.560 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:22:49.561 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:22:49.561 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:22:49.561 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:22:49.561 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:22:49.613 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:49.632 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:22:49.703 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:50.390 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:22:50.451 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:50 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:22:50.451 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:50 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1444881374' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:22:50.451 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:50 smithi190 ceph-mon[160435]: osdmap e240: 8 total, 8 up, 8 in 2023-10-01T09:22:50.451 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:50 smithi190 ceph-mon[160435]: failed to encode map e240 with expected crc 2023-10-01T09:22:50.451 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:50 smithi190 ceph-mon[160435]: failed to encode map e240 with expected crc 2023-10-01T09:22:50.452 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:50 smithi190 ceph-mon[160435]: failed to encode map e240 with expected crc 2023-10-01T09:22:50.452 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:50 smithi190 ceph-mon[160435]: pgmap v605: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.7 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 20157/85779 objects degraded (23.499%); 7755/85779 objects misplaced (9.041%); 4.1 MiB/s, 117 objects/s recovering 2023-10-01T09:22:50.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:50 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:22:50.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:50 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1444881374' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:22:50.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:50 smithi138 ceph-mon[185886]: osdmap e240: 8 total, 8 up, 8 in 2023-10-01T09:22:50.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:50 smithi138 ceph-mon[185886]: failed to encode map e240 with expected crc 2023-10-01T09:22:50.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:50 smithi138 ceph-mon[185886]: failed to encode map e240 with expected crc 2023-10-01T09:22:50.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:50 smithi138 ceph-mon[185886]: failed to encode map e240 with expected crc 2023-10-01T09:22:50.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:50 smithi138 ceph-mon[185886]: pgmap v605: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.7 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 20157/85779 objects degraded (23.499%); 7755/85779 objects misplaced (9.041%); 4.1 MiB/s, 117 objects/s recovering 2023-10-01T09:22:50.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:50 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:22:50.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:50 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1444881374' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:22:50.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:50 smithi138 ceph-mon[192513]: osdmap e240: 8 total, 8 up, 8 in 2023-10-01T09:22:50.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:50 smithi138 ceph-mon[192513]: failed to encode map e240 with expected crc 2023-10-01T09:22:50.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:50 smithi138 ceph-mon[192513]: failed to encode map e240 with expected crc 2023-10-01T09:22:50.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:50 smithi138 ceph-mon[192513]: failed to encode map e240 with expected crc 2023-10-01T09:22:50.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:50 smithi138 ceph-mon[192513]: pgmap v605: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.7 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 20157/85779 objects degraded (23.499%); 7755/85779 objects misplaced (9.041%); 4.1 MiB/s, 117 objects/s recovering 2023-10-01T09:22:50.669 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:50.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:50.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:50.777 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:50.790 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-01T09:22:50.866 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:50.943 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:22:50.968 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:51.011 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:51.018 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:51.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:51.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:51.136 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:51.182 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:51.292 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:51.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:51.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:51.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:51 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:22:51.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:51 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:22:51.395 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:51.437 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:51 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:22:51.479 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:51.862 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:51.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:51.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:51.966 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:52.023 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:52.184 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:52 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:22:52.184 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:52 smithi138 ceph-mon[185886]: pgmap v606: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 19916/82821 objects degraded (24.047%); 7515/82821 objects misplaced (9.074%); 1.6 MiB/s, 77 objects/s recovering 2023-10-01T09:22:52.184 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:52 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 20157/85779 objects degraded (23.499%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:52.185 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:52 smithi138 ceph-mon[185886]: osdmap e241: 8 total, 8 up, 8 in 2023-10-01T09:22:52.253 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:52.265 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:52 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:22:52.265 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:52 smithi190 ceph-mon[160435]: pgmap v606: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 19916/82821 objects degraded (24.047%); 7515/82821 objects misplaced (9.074%); 1.6 MiB/s, 77 objects/s recovering 2023-10-01T09:22:52.265 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:52 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 20157/85779 objects degraded (23.499%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:52.265 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:52 smithi190 ceph-mon[160435]: osdmap e241: 8 total, 8 up, 8 in 2023-10-01T09:22:52.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:52.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:52.367 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:52.394 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:52.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:52 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:22:52.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:52 smithi138 ceph-mon[192513]: pgmap v606: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 19916/82821 objects degraded (24.047%); 7515/82821 objects misplaced (9.074%); 1.6 MiB/s, 77 objects/s recovering 2023-10-01T09:22:52.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:52 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 20157/85779 objects degraded (23.499%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:52.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:52 smithi138 ceph-mon[192513]: osdmap e241: 8 total, 8 up, 8 in 2023-10-01T09:22:52.694 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:52.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:52.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:52.732 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:22:52.732 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:22:52.732 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:22:52.732 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:22:52.733 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.733 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:95248 6.a 6:52b16094:::benchmark_data_smithi138_307751_object95247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e208)", 2023-10-01T09:22:52.733 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:02.136736+0000", 2023-10-01T09:22:52.733 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 110.586951848, 2023-10-01T09:22:52.733 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.549160117, 2023-10-01T09:22:52.733 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.733 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:22:52.733 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:52.733 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:52.733 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:52.733 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 95248 2023-10-01T09:22:52.734 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.734 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.734 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.734 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.734 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.136736+0000", 2023-10-01T09:22:52.734 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.734 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.734 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.734 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.734 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.136736+0000", 2023-10-01T09:22:52.734 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999924 2023-10-01T09:22:52.734 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.735 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.735 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.735 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.136729+0000", 2023-10-01T09:22:52.735 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0017559050000000001 2023-10-01T09:22:52.735 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.735 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.735 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.735 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138484+0000", 2023-10-01T09:22:52.735 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7159999999999992e-06 2023-10-01T09:22:52.735 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.735 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.735 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.736 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138493+0000", 2023-10-01T09:22:52.736 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1099999999999999e-06 2023-10-01T09:22:52.736 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.736 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.736 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.736 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138499+0000", 2023-10-01T09:22:52.736 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019390360000000001 2023-10-01T09:22:52.736 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.736 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.736 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.736 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.140438+0000", 2023-10-01T09:22:52.737 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7399999999999999e-06 2023-10-01T09:22:52.737 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.737 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.737 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:22:52.737 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.140448+0000", 2023-10-01T09:22:52.737 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.540506481 2023-10-01T09:22:52.737 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.737 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.737 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.737 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.680955+0000", 2023-10-01T09:22:52.737 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049419169999999997 2023-10-01T09:22:52.737 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.738 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.738 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.738 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.685896+0000", 2023-10-01T09:22:52.738 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.549160117 2023-10-01T09:22:52.738 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.738 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.738 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.738 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.738 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.738 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:95258 6.c 6:3d2ffb96:::benchmark_data_smithi138_307751_object95257:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e208)", 2023-10-01T09:22:52.738 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:02.138681+0000", 2023-10-01T09:22:52.738 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 110.585006847, 2023-10-01T09:22:52.739 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2332431580000001, 2023-10-01T09:22:52.739 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.739 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:22:52.739 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:52.739 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:52.739 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:52.739 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 95258 2023-10-01T09:22:52.739 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.739 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.739 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.739 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.739 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138681+0000", 2023-10-01T09:22:52.740 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.740 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.740 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.740 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.740 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138681+0000", 2023-10-01T09:22:52.740 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:22:52.740 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.740 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.740 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.740 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138681+0000", 2023-10-01T09:22:52.740 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5492000000000002e-05 2023-10-01T09:22:52.740 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.741 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.741 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.741 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138756+0000", 2023-10-01T09:22:52.741 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0139999999999999e-05 2023-10-01T09:22:52.741 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.741 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.741 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.741 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138766+0000", 2023-10-01T09:22:52.741 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4380000000000003e-06 2023-10-01T09:22:52.741 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.741 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.741 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.742 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138772+0000", 2023-10-01T09:22:52.742 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001883888 2023-10-01T09:22:52.742 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.742 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.742 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.742 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.140656+0000", 2023-10-01T09:22:52.742 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9410000000000001e-06 2023-10-01T09:22:52.742 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.742 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.742 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:22:52.742 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.140662+0000", 2023-10-01T09:22:52.742 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.231159575 2023-10-01T09:22:52.743 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.743 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.743 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.743 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.371822+0000", 2023-10-01T09:22:52.743 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010246 2023-10-01T09:22:52.743 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.743 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.743 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.743 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.371925+0000", 2023-10-01T09:22:52.743 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2332431580000001 2023-10-01T09:22:52.743 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.743 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.744 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.744 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.744 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.744 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:95261 6.c 6:3528b9c1:::benchmark_data_smithi138_307751_object95260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e208)", 2023-10-01T09:22:52.744 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:02.138778+0000", 2023-10-01T09:22:52.744 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 110.584910564, 2023-10-01T09:22:52.744 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.233266902, 2023-10-01T09:22:52.744 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.744 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:22:52.744 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:52.744 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:52.744 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:52.745 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 95261 2023-10-01T09:22:52.745 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.745 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.745 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.745 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.745 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138778+0000", 2023-10-01T09:22:52.745 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.745 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.745 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.745 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.745 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138778+0000", 2023-10-01T09:22:52.745 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:22:52.745 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.746 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.746 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.746 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138777+0000", 2023-10-01T09:22:52.746 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7418e-05 2023-10-01T09:22:52.746 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.746 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.746 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.746 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138814+0000", 2023-10-01T09:22:52.746 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5450000000000003e-06 2023-10-01T09:22:52.746 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.746 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.747 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.747 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138823+0000", 2023-10-01T09:22:52.747 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9559999999999996e-06 2023-10-01T09:22:52.747 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.747 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.747 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.747 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138828+0000", 2023-10-01T09:22:52.747 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0018540620000000001 2023-10-01T09:22:52.747 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.747 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.747 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.748 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.140682+0000", 2023-10-01T09:22:52.748 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9789999999999998e-06 2023-10-01T09:22:52.748 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.748 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.748 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:22:52.748 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.140688+0000", 2023-10-01T09:22:52.748 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2312748250000001 2023-10-01T09:22:52.748 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.748 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.748 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.748 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.371963+0000", 2023-10-01T09:22:52.748 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1797000000000003e-05 2023-10-01T09:22:52.749 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.749 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.749 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.749 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.372045+0000", 2023-10-01T09:22:52.749 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.233266902 2023-10-01T09:22:52.749 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.749 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.749 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.749 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.749 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.749 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:95264 6.a 6:58e4548c:::benchmark_data_smithi138_307751_object95263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e208)", 2023-10-01T09:22:52.750 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:02.138834+0000", 2023-10-01T09:22:52.750 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 110.58485404300001, 2023-10-01T09:22:52.750 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.664207217, 2023-10-01T09:22:52.750 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.750 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:22:52.750 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:52.750 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:52.750 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:52.750 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 95264 2023-10-01T09:22:52.750 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.750 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.751 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.751 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.751 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138834+0000", 2023-10-01T09:22:52.751 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.751 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.751 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.751 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.751 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138834+0000", 2023-10-01T09:22:52.751 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:22:52.751 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.751 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.751 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.752 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138833+0000", 2023-10-01T09:22:52.752 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8503000000000001e-05 2023-10-01T09:22:52.752 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.752 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.752 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.752 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138862+0000", 2023-10-01T09:22:52.752 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9970000000000004e-06 2023-10-01T09:22:52.752 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.752 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.752 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.752 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138868+0000", 2023-10-01T09:22:52.752 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3100000000000002e-06 2023-10-01T09:22:52.753 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.753 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.753 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.753 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.138872+0000", 2023-10-01T09:22:52.753 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0015968029999999999 2023-10-01T09:22:52.753 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.753 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.753 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.753 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.140469+0000", 2023-10-01T09:22:52.753 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3230000000000003e-06 2023-10-01T09:22:52.753 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.753 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.754 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:22:52.754 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.140475+0000", 2023-10-01T09:22:52.754 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.568567269999999 2023-10-01T09:22:52.754 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.754 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.754 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.754 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.709043+0000", 2023-10-01T09:22:52.754 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0067509939999999997 2023-10-01T09:22:52.754 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.754 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.754 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:22:52.754 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.715794+0000", 2023-10-01T09:22:52.755 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.046927971999999998 2023-10-01T09:22:52.755 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.755 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.755 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.755 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.762722+0000", 2023-10-01T09:22:52.755 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8133000000000006e-05 2023-10-01T09:22:52.755 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.755 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.755 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for degraded object", 2023-10-01T09:22:52.755 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.762800+0000", 2023-10-01T09:22:52.755 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.040121855999999997 2023-10-01T09:22:52.755 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.755 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.756 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.756 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.802922+0000", 2023-10-01T09:22:52.756 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000119737 2023-10-01T09:22:52.756 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.756 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.756 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.756 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.803041+0000", 2023-10-01T09:22:52.756 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.664207217 2023-10-01T09:22:52.756 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.756 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.756 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.757 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.757 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.757 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:95279 6.c 6:3be5087a:::benchmark_data_smithi138_307751_object95278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e208)", 2023-10-01T09:22:52.757 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:02.139506+0000", 2023-10-01T09:22:52.757 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 110.584182594, 2023-10-01T09:22:52.757 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.266229405, 2023-10-01T09:22:52.757 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.757 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.757 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:52.757 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:52.757 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:52.758 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 95279 2023-10-01T09:22:52.758 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.758 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.758 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.758 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.758 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.139506+0000", 2023-10-01T09:22:52.758 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.758 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.758 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.758 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.758 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.139506+0000", 2023-10-01T09:22:52.759 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:22:52.759 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.759 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.759 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.759 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.139505+0000", 2023-10-01T09:22:52.759 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0069141410000000004 2023-10-01T09:22:52.759 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.759 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.759 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.759 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.146419+0000", 2023-10-01T09:22:52.759 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2450999999999999e-05 2023-10-01T09:22:52.759 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.760 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.760 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.760 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.146441+0000", 2023-10-01T09:22:52.760 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1783000000000001e-05 2023-10-01T09:22:52.760 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.760 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.760 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.760 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.146453+0000", 2023-10-01T09:22:52.760 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00043400399999999999 2023-10-01T09:22:52.760 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.760 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.761 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.761 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.146887+0000", 2023-10-01T09:22:52.761 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1567e-05 2023-10-01T09:22:52.761 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.761 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.761 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:22:52.761 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.146899+0000", 2023-10-01T09:22:52.761 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.225238542 2023-10-01T09:22:52.761 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.761 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.761 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.761 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.372137+0000", 2023-10-01T09:22:52.762 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015195599999999999 2023-10-01T09:22:52.762 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.762 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.762 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.762 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.372289+0000", 2023-10-01T09:22:52.762 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00021577599999999999 2023-10-01T09:22:52.762 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.762 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.762 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,7", 2023-10-01T09:22:52.762 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.372505+0000", 2023-10-01T09:22:52.762 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010041529 2023-10-01T09:22:52.763 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.763 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.763 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:22:52.763 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.382546+0000", 2023-10-01T09:22:52.763 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5704999999999998e-05 2023-10-01T09:22:52.763 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.763 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.763 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.763 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.382582+0000", 2023-10-01T09:22:52.763 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020561255000000001 2023-10-01T09:22:52.763 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.763 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.763 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.764 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.403143+0000", 2023-10-01T09:22:52.764 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0899999999999998e-05 2023-10-01T09:22:52.764 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.764 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.764 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.764 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.403184+0000", 2023-10-01T09:22:52.764 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0025507770000000002 2023-10-01T09:22:52.764 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.764 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.764 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.765 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.405735+0000", 2023-10-01T09:22:52.765 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.266229405 2023-10-01T09:22:52.765 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.765 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.765 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.765 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.765 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.765 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:95282 6.c 6:3ec7c21b:::benchmark_data_smithi138_307751_object95281:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e208)", 2023-10-01T09:22:52.765 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:02.146471+0000", 2023-10-01T09:22:52.766 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 110.577216974, 2023-10-01T09:22:52.766 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.26468062, 2023-10-01T09:22:52.766 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.766 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.766 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:52.766 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:52.766 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:52.766 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 95282 2023-10-01T09:22:52.766 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.766 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.767 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.767 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.767 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.146471+0000", 2023-10-01T09:22:52.767 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.767 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.767 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.767 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.767 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.146471+0000", 2023-10-01T09:22:52.767 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:22:52.767 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.768 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.768 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.768 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.146469+0000", 2023-10-01T09:22:52.768 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9401000000000003e-05 2023-10-01T09:22:52.768 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.768 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.768 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.768 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.146528+0000", 2023-10-01T09:22:52.768 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9689999999999994e-06 2023-10-01T09:22:52.768 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.768 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.769 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.769 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.146536+0000", 2023-10-01T09:22:52.769 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3739999999999997e-06 2023-10-01T09:22:52.769 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.769 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.769 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.769 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.146541+0000", 2023-10-01T09:22:52.769 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00037655700000000002 2023-10-01T09:22:52.769 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.769 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.769 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.769 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.146918+0000", 2023-10-01T09:22:52.770 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2400000000000004e-06 2023-10-01T09:22:52.770 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.770 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.770 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:22:52.770 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.146924+0000", 2023-10-01T09:22:52.770 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.228783806 2023-10-01T09:22:52.770 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.770 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.770 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.770 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.375708+0000", 2023-10-01T09:22:52.770 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000154592 2023-10-01T09:22:52.770 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.771 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.771 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.771 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.375863+0000", 2023-10-01T09:22:52.771 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00019618500000000001 2023-10-01T09:22:52.771 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.771 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.771 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,7", 2023-10-01T09:22:52.771 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.376059+0000", 2023-10-01T09:22:52.771 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0065765750000000003 2023-10-01T09:22:52.771 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.771 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.772 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.772 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.382635+0000", 2023-10-01T09:22:52.772 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2398000000000001e-05 2023-10-01T09:22:52.772 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.772 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.772 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:22:52.772 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.382648+0000", 2023-10-01T09:22:52.772 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.025744870999999999 2023-10-01T09:22:52.772 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.772 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.772 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.772 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.408393+0000", 2023-10-01T09:22:52.773 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8556e-05 2023-10-01T09:22:52.773 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.773 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.773 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.773 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.408451+0000", 2023-10-01T09:22:52.773 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0027005929999999998 2023-10-01T09:22:52.773 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.773 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.773 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.773 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.411152+0000", 2023-10-01T09:22:52.773 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.26468062 2023-10-01T09:22:52.774 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.774 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.774 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.774 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.774 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.774 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:95291 6.a 6:58d05ced:::benchmark_data_smithi138_307751_object95290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-10-01T09:22:52.774 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:02.266478+0000", 2023-10-01T09:22:52.774 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 110.45721027899999, 2023-10-01T09:22:52.774 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.49768955, 2023-10-01T09:22:52.774 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.774 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.774 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:52.775 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:52.775 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:52.775 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 95291 2023-10-01T09:22:52.775 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.775 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.775 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.775 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.775 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.266478+0000", 2023-10-01T09:22:52.775 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.775 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.776 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.776 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.776 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.266478+0000", 2023-10-01T09:22:52.776 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:22:52.776 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.776 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.776 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.776 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.266476+0000", 2023-10-01T09:22:52.776 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3526000000000002e-05 2023-10-01T09:22:52.776 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.776 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.776 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.777 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.266530+0000", 2023-10-01T09:22:52.777 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1838000000000001e-05 2023-10-01T09:22:52.777 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.777 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.777 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.777 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.266542+0000", 2023-10-01T09:22:52.777 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2809999999999999e-06 2023-10-01T09:22:52.777 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.777 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.777 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.777 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.266547+0000", 2023-10-01T09:22:52.777 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8146999999999996e-05 2023-10-01T09:22:52.777 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.778 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.778 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.778 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.266625+0000", 2023-10-01T09:22:52.778 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7317999999999999e-05 2023-10-01T09:22:52.778 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.778 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.778 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:22:52.778 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.266642+0000", 2023-10-01T09:22:52.778 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.450032616 2023-10-01T09:22:52.778 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.778 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.779 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.779 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.716675+0000", 2023-10-01T09:22:52.779 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0082306589999999995 2023-10-01T09:22:52.779 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.779 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.779 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.779 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.724906+0000", 2023-10-01T09:22:52.779 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013628569 2023-10-01T09:22:52.779 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.779 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.779 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:22:52.779 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.738534+0000", 2023-10-01T09:22:52.780 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.024280149000000001 2023-10-01T09:22:52.780 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.780 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.780 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:22:52.780 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.762814+0000", 2023-10-01T09:22:52.780 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000121118 2023-10-01T09:22:52.780 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.780 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.780 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.780 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.762936+0000", 2023-10-01T09:22:52.780 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5009000000000001e-05 2023-10-01T09:22:52.780 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.781 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.781 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.781 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.762991+0000", 2023-10-01T09:22:52.781 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4137000000000001e-05 2023-10-01T09:22:52.781 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.781 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.781 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.781 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.763035+0000", 2023-10-01T09:22:52.781 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0011327710000000001 2023-10-01T09:22:52.781 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.782 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.782 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.782 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.764167+0000", 2023-10-01T09:22:52.782 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.49768955 2023-10-01T09:22:52.782 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.782 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.782 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.782 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.782 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.782 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:95294 6.a 6:50247c40:::benchmark_data_smithi138_307751_object95293:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-10-01T09:22:52.782 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:02.270890+0000", 2023-10-01T09:22:52.783 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 110.45279789, 2023-10-01T09:22:52.783 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.517976516000001, 2023-10-01T09:22:52.783 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.783 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.783 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:52.783 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:52.783 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:52.783 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 95294 2023-10-01T09:22:52.783 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.783 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.783 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.783 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.784 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.270890+0000", 2023-10-01T09:22:52.784 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.784 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.784 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.784 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.784 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.270890+0000", 2023-10-01T09:22:52.784 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:22:52.784 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.784 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.784 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.784 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.270889+0000", 2023-10-01T09:22:52.785 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6186000000000006e-05 2023-10-01T09:22:52.785 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.785 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.785 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.785 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.270955+0000", 2023-10-01T09:22:52.785 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4779e-05 2023-10-01T09:22:52.785 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.785 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.785 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.785 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.270970+0000", 2023-10-01T09:22:52.785 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9099999999999999e-06 2023-10-01T09:22:52.786 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.786 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.786 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.786 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.270977+0000", 2023-10-01T09:22:52.786 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4438999999999999e-05 2023-10-01T09:22:52.786 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.786 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.786 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.786 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.271021+0000", 2023-10-01T09:22:52.786 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0020000000000006e-06 2023-10-01T09:22:52.786 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.787 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.787 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:22:52.787 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.271029+0000", 2023-10-01T09:22:52.787 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.493180921 2023-10-01T09:22:52.787 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.787 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.787 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.787 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.764210+0000", 2023-10-01T09:22:52.787 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000111569 2023-10-01T09:22:52.787 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.787 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.787 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.788 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.764322+0000", 2023-10-01T09:22:52.788 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014088100000000001 2023-10-01T09:22:52.788 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.788 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.788 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:22:52.788 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.764463+0000", 2023-10-01T09:22:52.788 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0092049300000000001 2023-10-01T09:22:52.788 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.788 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.788 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.788 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.773667+0000", 2023-10-01T09:22:52.789 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2944e-05 2023-10-01T09:22:52.789 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.789 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.789 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:22:52.789 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.773680+0000", 2023-10-01T09:22:52.789 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013441972 2023-10-01T09:22:52.789 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.789 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.789 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.789 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.787122+0000", 2023-10-01T09:22:52.789 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8400999999999998e-05 2023-10-01T09:22:52.789 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.790 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.790 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.790 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.787191+0000", 2023-10-01T09:22:52.790 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0016760569999999999 2023-10-01T09:22:52.790 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.790 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.790 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.790 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.788867+0000", 2023-10-01T09:22:52.790 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.517976516000001 2023-10-01T09:22:52.790 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.790 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.791 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.791 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.791 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.791 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:95295 6.a 6:5107aeaa:::benchmark_data_smithi138_307751_object95294:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-10-01T09:22:52.791 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:02.277992+0000", 2023-10-01T09:22:52.791 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 110.44569659299999, 2023-10-01T09:22:52.791 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.517674496, 2023-10-01T09:22:52.791 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.791 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.791 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:52.791 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:52.792 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:52.792 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 95295 2023-10-01T09:22:52.792 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.792 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.792 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.792 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.792 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.277992+0000", 2023-10-01T09:22:52.792 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.792 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.792 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.792 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.792 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.277992+0000", 2023-10-01T09:22:52.793 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:22:52.793 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.793 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.793 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.793 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.277991+0000", 2023-10-01T09:22:52.793 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9100999999999997e-05 2023-10-01T09:22:52.793 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.793 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.793 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.793 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.278030+0000", 2023-10-01T09:22:52.793 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2153999999999999e-05 2023-10-01T09:22:52.794 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.794 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.794 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.794 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.278042+0000", 2023-10-01T09:22:52.794 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5049999999999998e-06 2023-10-01T09:22:52.794 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.794 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.794 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.794 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.278048+0000", 2023-10-01T09:22:52.794 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2410000000000001e-05 2023-10-01T09:22:52.794 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.795 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.795 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.795 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.278100+0000", 2023-10-01T09:22:52.795 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.64e-06 2023-10-01T09:22:52.795 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.795 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.795 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:22:52.795 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.278107+0000", 2023-10-01T09:22:52.795 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.488614151 2023-10-01T09:22:52.795 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.795 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.796 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.796 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.766721+0000", 2023-10-01T09:22:52.796 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000146773 2023-10-01T09:22:52.796 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.796 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.796 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.796 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.766868+0000", 2023-10-01T09:22:52.796 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012710999999999999 2023-10-01T09:22:52.796 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.796 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.796 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:22:52.796 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.766995+0000", 2023-10-01T09:22:52.797 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0066964210000000001 2023-10-01T09:22:52.797 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.797 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.797 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:22:52.797 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.773691+0000", 2023-10-01T09:22:52.797 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0097753809999999997 2023-10-01T09:22:52.797 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.797 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.797 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.797 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.783466+0000", 2023-10-01T09:22:52.797 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010897627 2023-10-01T09:22:52.797 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.798 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.798 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.798 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.794364+0000", 2023-10-01T09:22:52.798 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8835999999999999e-05 2023-10-01T09:22:52.798 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.798 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.798 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.798 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.794403+0000", 2023-10-01T09:22:52.798 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0012631840000000001 2023-10-01T09:22:52.798 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.798 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.799 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.799 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.795666+0000", 2023-10-01T09:22:52.799 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.517674496 2023-10-01T09:22:52.799 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.799 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.799 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.799 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.799 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.799 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:95298 6.a 6:53750a22:::benchmark_data_smithi138_307751_object95297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-10-01T09:22:52.799 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:02.289457+0000", 2023-10-01T09:22:52.799 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 110.434230773, 2023-10-01T09:22:52.799 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.507370508999999, 2023-10-01T09:22:52.800 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.800 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.800 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:52.800 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:52.800 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:52.800 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 95298 2023-10-01T09:22:52.800 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.800 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.800 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.800 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.800 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.289457+0000", 2023-10-01T09:22:52.800 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.801 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.801 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.801 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.801 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.289457+0000", 2023-10-01T09:22:52.801 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:22:52.801 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.801 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.801 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.801 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.289456+0000", 2023-10-01T09:22:52.801 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6541e-05 2023-10-01T09:22:52.801 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.801 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.802 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.802 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.289503+0000", 2023-10-01T09:22:52.802 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2439999999999999e-05 2023-10-01T09:22:52.802 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.802 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.802 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.802 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.289515+0000", 2023-10-01T09:22:52.802 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.409e-06 2023-10-01T09:22:52.802 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.802 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.802 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.803 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.289521+0000", 2023-10-01T09:22:52.803 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7312000000000003e-05 2023-10-01T09:22:52.803 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.803 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.803 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.803 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.289579+0000", 2023-10-01T09:22:52.803 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0726e-05 2023-10-01T09:22:52.803 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.803 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.803 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:22:52.803 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.289589+0000", 2023-10-01T09:22:52.803 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.479183151000001 2023-10-01T09:22:52.804 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.804 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.804 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.804 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.768773+0000", 2023-10-01T09:22:52.804 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014836100000000001 2023-10-01T09:22:52.804 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.804 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.804 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.804 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.768921+0000", 2023-10-01T09:22:52.804 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012400700000000001 2023-10-01T09:22:52.804 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.805 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.805 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:22:52.805 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.769045+0000", 2023-10-01T09:22:52.805 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0046559349999999999 2023-10-01T09:22:52.805 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.805 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.805 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:22:52.805 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.773701+0000", 2023-10-01T09:22:52.805 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0098169680000000006 2023-10-01T09:22:52.805 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.805 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.806 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.806 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.783518+0000", 2023-10-01T09:22:52.806 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012193776999999999 2023-10-01T09:22:52.806 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.806 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.806 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.806 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.795712+0000", 2023-10-01T09:22:52.806 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2184999999999997e-05 2023-10-01T09:22:52.806 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.806 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.806 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.807 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.795754+0000", 2023-10-01T09:22:52.807 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00107408 2023-10-01T09:22:52.807 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.807 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.807 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.807 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.796828+0000", 2023-10-01T09:22:52.807 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.507370508999999 2023-10-01T09:22:52.807 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.807 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.807 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.807 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.807 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.807 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:95299 6.a 6:53e38441:::benchmark_data_smithi138_307751_object95298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-10-01T09:22:52.808 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:02.291646+0000", 2023-10-01T09:22:52.808 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 110.43204186, 2023-10-01T09:22:52.808 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.516792899, 2023-10-01T09:22:52.808 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.808 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.808 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:52.808 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:52.808 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:52.808 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 95299 2023-10-01T09:22:52.808 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.808 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.809 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.809 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.809 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.291646+0000", 2023-10-01T09:22:52.809 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.809 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.809 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.809 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.809 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.291646+0000", 2023-10-01T09:22:52.809 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:22:52.809 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.809 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.809 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.810 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.291645+0000", 2023-10-01T09:22:52.810 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7425000000000001e-05 2023-10-01T09:22:52.810 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.810 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.810 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.810 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.291693+0000", 2023-10-01T09:22:52.810 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2e-05 2023-10-01T09:22:52.810 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.810 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.810 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.810 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.291705+0000", 2023-10-01T09:22:52.810 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2949999999999996e-06 2023-10-01T09:22:52.811 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.811 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.811 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.811 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.291712+0000", 2023-10-01T09:22:52.811 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6505999999999997e-05 2023-10-01T09:22:52.811 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.811 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.811 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.811 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.291768+0000", 2023-10-01T09:22:52.811 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.075e-05 2023-10-01T09:22:52.811 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.811 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.812 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:22:52.812 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.291779+0000", 2023-10-01T09:22:52.812 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.478524009999999 2023-10-01T09:22:52.812 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.812 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.812 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.812 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.770303+0000", 2023-10-01T09:22:52.812 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016076 2023-10-01T09:22:52.812 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.812 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.812 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.813 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.770464+0000", 2023-10-01T09:22:52.813 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000140548 2023-10-01T09:22:52.813 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.813 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.813 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:22:52.813 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.770604+0000", 2023-10-01T09:22:52.813 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0063257570000000004 2023-10-01T09:22:52.813 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.813 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.813 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:22:52.813 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.776930+0000", 2023-10-01T09:22:52.814 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0095766900000000005 2023-10-01T09:22:52.814 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.814 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.814 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.814 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.786507+0000", 2023-10-01T09:22:52.814 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020617687999999999 2023-10-01T09:22:52.814 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.814 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.814 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.814 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.807125+0000", 2023-10-01T09:22:52.814 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3269999999999997e-05 2023-10-01T09:22:52.815 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.815 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.815 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.815 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.807168+0000", 2023-10-01T09:22:52.815 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0012713710000000001 2023-10-01T09:22:52.815 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.815 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.815 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.815 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.808439+0000", 2023-10-01T09:22:52.815 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.516792899 2023-10-01T09:22:52.815 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.816 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.816 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.816 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.816 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.816 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:95300 6.a 6:58859a58:::benchmark_data_smithi138_307751_object95299:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-10-01T09:22:52.816 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:02.293599+0000", 2023-10-01T09:22:52.816 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 110.43008961, 2023-10-01T09:22:52.816 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.52044403, 2023-10-01T09:22:52.816 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.816 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.816 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:52.817 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:52.817 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:52.817 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 95300 2023-10-01T09:22:52.817 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.817 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.817 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.817 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.817 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.293599+0000", 2023-10-01T09:22:52.817 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.817 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.817 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.818 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.818 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.293599+0000", 2023-10-01T09:22:52.818 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:22:52.818 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.818 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.818 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.818 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.293598+0000", 2023-10-01T09:22:52.818 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0979000000000001e-05 2023-10-01T09:22:52.818 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.818 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.818 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.819 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.293649+0000", 2023-10-01T09:22:52.819 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2121e-05 2023-10-01T09:22:52.819 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.819 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.819 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.819 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.293661+0000", 2023-10-01T09:22:52.819 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2469999999999999e-06 2023-10-01T09:22:52.819 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.819 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.819 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.819 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.293666+0000", 2023-10-01T09:22:52.820 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8194000000000001e-05 2023-10-01T09:22:52.820 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.820 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.820 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.820 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.293704+0000", 2023-10-01T09:22:52.820 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0515e-05 2023-10-01T09:22:52.820 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.820 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.820 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:22:52.820 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:02.293715+0000", 2023-10-01T09:22:52.820 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.478368041 2023-10-01T09:22:52.820 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.821 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.821 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.821 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.772083+0000", 2023-10-01T09:22:52.821 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010409800000000001 2023-10-01T09:22:52.821 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.821 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.821 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.821 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.772187+0000", 2023-10-01T09:22:52.821 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000123573 2023-10-01T09:22:52.821 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.821 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.822 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:22:52.822 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.772311+0000", 2023-10-01T09:22:52.822 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0046272800000000001 2023-10-01T09:22:52.822 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.822 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.822 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:22:52.822 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.776938+0000", 2023-10-01T09:22:52.822 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0096017429999999994 2023-10-01T09:22:52.822 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.822 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.822 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.823 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.786540+0000", 2023-10-01T09:22:52.823 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.026141339999999999 2023-10-01T09:22:52.823 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.823 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.823 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.823 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.812681+0000", 2023-10-01T09:22:52.823 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0680000000000006e-05 2023-10-01T09:22:52.823 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.823 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.823 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.823 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.812772+0000", 2023-10-01T09:22:52.823 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001271065 2023-10-01T09:22:52.824 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.824 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.824 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.824 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.814043+0000", 2023-10-01T09:22:52.824 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 12.52044403 2023-10-01T09:22:52.824 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.824 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.824 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.824 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.824 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.824 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:95302 6.2 e209/207 6:4510ce89:::benchmark_data_smithi138_307751_object95301:head v 209'6078, mlcod=206'6076)", 2023-10-01T09:22:52.825 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:03.372564+0000", 2023-10-01T09:22:52.825 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 109.351124075, 2023-10-01T09:22:52.825 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.263961712, 2023-10-01T09:22:52.825 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.825 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.825 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.825 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.825 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.825 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.372564+0000", 2023-10-01T09:22:52.825 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.825 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.826 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.826 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.826 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.372564+0000", 2023-10-01T09:22:52.826 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:22:52.826 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.826 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.826 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.826 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.372563+0000", 2023-10-01T09:22:52.826 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0044649069999999997 2023-10-01T09:22:52.826 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.826 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.826 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.827 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.377028+0000", 2023-10-01T09:22:52.827 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4793e-05 2023-10-01T09:22:52.827 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.827 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.827 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.827 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.377043+0000", 2023-10-01T09:22:52.827 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6919999999999999e-06 2023-10-01T09:22:52.827 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.827 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.828 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.828 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.377049+0000", 2023-10-01T09:22:52.828 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.253229949 2023-10-01T09:22:52.828 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.828 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.828 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.828 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.630279+0000", 2023-10-01T09:22:52.828 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5710999999999999e-05 2023-10-01T09:22:52.828 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.828 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.828 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.828 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.630294+0000", 2023-10-01T09:22:52.829 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0061968509999999997 2023-10-01T09:22:52.829 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.829 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.829 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.829 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.636491+0000", 2023-10-01T09:22:52.829 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4748e-05 2023-10-01T09:22:52.829 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.829 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.829 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.829 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.636526+0000", 2023-10-01T09:22:52.829 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.263961712 2023-10-01T09:22:52.829 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.830 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.830 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.830 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.830 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.830 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:95322 6.2 e209/207 6:456699d3:::benchmark_data_smithi138_307751_object95321:head v 209'6079, mlcod=206'6076)", 2023-10-01T09:22:52.830 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:03.431360+0000", 2023-10-01T09:22:52.830 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 109.292327703, 2023-10-01T09:22:52.830 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.254533152, 2023-10-01T09:22:52.830 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.830 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.830 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.830 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.831 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.831 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.431360+0000", 2023-10-01T09:22:52.831 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.831 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.831 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.831 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.831 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.431360+0000", 2023-10-01T09:22:52.831 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:22:52.831 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.831 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.831 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.832 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.431360+0000", 2023-10-01T09:22:52.832 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2704000000000002e-05 2023-10-01T09:22:52.832 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.832 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.832 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.832 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.431412+0000", 2023-10-01T09:22:52.832 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2928999999999999e-05 2023-10-01T09:22:52.832 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.832 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.832 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.832 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.431425+0000", 2023-10-01T09:22:52.832 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5690000000000004e-06 2023-10-01T09:22:52.833 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.833 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.833 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.833 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.431431+0000", 2023-10-01T09:22:52.833 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.2011228 2023-10-01T09:22:52.833 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.833 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.833 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.833 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.632554+0000", 2023-10-01T09:22:52.833 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2331e-05 2023-10-01T09:22:52.833 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.834 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.834 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.834 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.632576+0000", 2023-10-01T09:22:52.834 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.053296285999999998 2023-10-01T09:22:52.834 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.834 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.834 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.834 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.685872+0000", 2023-10-01T09:22:52.834 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1497000000000001e-05 2023-10-01T09:22:52.834 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.834 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.834 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.835 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.685894+0000", 2023-10-01T09:22:52.835 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.254533152 2023-10-01T09:22:52.835 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.835 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.835 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.835 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.835 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.835 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:95334 6.2 e209/207 6:42c96aa7:::benchmark_data_smithi138_307751_object95333:head v 209'6080, mlcod=206'6076)", 2023-10-01T09:22:52.835 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:03.456061+0000", 2023-10-01T09:22:52.835 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 109.267627491, 2023-10-01T09:22:52.835 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.237491667, 2023-10-01T09:22:52.836 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.836 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.836 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.836 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.836 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.836 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.456061+0000", 2023-10-01T09:22:52.836 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.836 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.836 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.836 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.836 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.456061+0000", 2023-10-01T09:22:52.836 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:22:52.836 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.837 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.837 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.837 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.456060+0000", 2023-10-01T09:22:52.837 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4931999999999997e-05 2023-10-01T09:22:52.837 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.837 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.837 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.837 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.456095+0000", 2023-10-01T09:22:52.837 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0686999999999999e-05 2023-10-01T09:22:52.837 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.837 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.837 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.838 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.456105+0000", 2023-10-01T09:22:52.838 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8309999999999997e-06 2023-10-01T09:22:52.838 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.838 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.838 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.838 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.456111+0000", 2023-10-01T09:22:52.838 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.178453596000001 2023-10-01T09:22:52.838 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.838 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.838 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.838 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.634565+0000", 2023-10-01T09:22:52.839 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0438e-05 2023-10-01T09:22:52.839 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.839 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.839 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.839 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.634585+0000", 2023-10-01T09:22:52.839 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.058942095999999999 2023-10-01T09:22:52.839 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.839 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.839 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.839 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.693527+0000", 2023-10-01T09:22:52.839 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.497e-05 2023-10-01T09:22:52.839 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.840 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.840 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.840 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.693552+0000", 2023-10-01T09:22:52.840 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.237491667 2023-10-01T09:22:52.840 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.840 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.840 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.840 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.840 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.840 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:95340 6.a 6:552dc0fd:::benchmark_data_smithi138_307751_object95339:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-01T09:22:52.840 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:03.492668+0000", 2023-10-01T09:22:52.840 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 109.231020379, 2023-10-01T09:22:52.841 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.322745347, 2023-10-01T09:22:52.841 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.841 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.841 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:52.841 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:52.841 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:52.841 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 95340 2023-10-01T09:22:52.841 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.841 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.841 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.841 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.841 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.492668+0000", 2023-10-01T09:22:52.842 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.842 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.842 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.842 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.842 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.492668+0000", 2023-10-01T09:22:52.842 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:22:52.842 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.842 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.842 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.842 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.492666+0000", 2023-10-01T09:22:52.842 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6789999999999997e-05 2023-10-01T09:22:52.843 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.843 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.843 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.843 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.492723+0000", 2023-10-01T09:22:52.843 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2825e-05 2023-10-01T09:22:52.843 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.843 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.843 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.843 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.492736+0000", 2023-10-01T09:22:52.843 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7599999999999997e-06 2023-10-01T09:22:52.843 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.844 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.844 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.844 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.492743+0000", 2023-10-01T09:22:52.844 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.280979812 2023-10-01T09:22:52.844 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.844 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.844 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.844 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.773723+0000", 2023-10-01T09:22:52.844 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0321000000000005e-05 2023-10-01T09:22:52.844 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.844 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.845 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.845 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.773813+0000", 2023-10-01T09:22:52.845 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011702200000000001 2023-10-01T09:22:52.845 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.845 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.845 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:22:52.845 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.773930+0000", 2023-10-01T09:22:52.845 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0030161749999999998 2023-10-01T09:22:52.845 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.845 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.845 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:22:52.846 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.776946+0000", 2023-10-01T09:22:52.846 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0096454469999999997 2023-10-01T09:22:52.846 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.846 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.846 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.846 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.786592+0000", 2023-10-01T09:22:52.846 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.027517651000000001 2023-10-01T09:22:52.846 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.846 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.846 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.846 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.814109+0000", 2023-10-01T09:22:52.846 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5826000000000003e-05 2023-10-01T09:22:52.847 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.847 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.847 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.847 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.814175+0000", 2023-10-01T09:22:52.847 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001238119 2023-10-01T09:22:52.847 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.847 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.847 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.847 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.815413+0000", 2023-10-01T09:22:52.847 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.322745347 2023-10-01T09:22:52.847 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.848 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.848 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.848 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.848 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:95348 6.2 e209/207 6:46ec8a63:::benchmark_data_smithi138_307751_object95347:head v 209'6081, mlcod=206'6076)", 2023-10-01T09:22:52.848 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:03.514204+0000", 2023-10-01T09:22:52.848 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 109.20948428299999, 2023-10-01T09:22:52.848 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.185555584999999, 2023-10-01T09:22:52.848 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.848 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.848 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.849 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.849 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.514204+0000", 2023-10-01T09:22:52.849 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.849 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.849 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.849 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.849 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.514204+0000", 2023-10-01T09:22:52.849 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:22:52.849 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.849 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.849 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.850 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.514203+0000", 2023-10-01T09:22:52.850 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4428999999999997e-05 2023-10-01T09:22:52.850 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.850 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.850 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.850 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.514257+0000", 2023-10-01T09:22:52.850 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3669000000000001e-05 2023-10-01T09:22:52.850 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.850 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.850 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.850 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.514271+0000", 2023-10-01T09:22:52.851 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6260000000000003e-06 2023-10-01T09:22:52.851 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.851 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.851 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.851 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.514278+0000", 2023-10-01T09:22:52.851 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.179293345 2023-10-01T09:22:52.851 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.851 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.851 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.851 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.693572+0000", 2023-10-01T09:22:52.851 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4878000000000001e-05 2023-10-01T09:22:52.851 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.852 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.852 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.852 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.693587+0000", 2023-10-01T09:22:52.852 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0061377979999999999 2023-10-01T09:22:52.852 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.852 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.852 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.852 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.699724+0000", 2023-10-01T09:22:52.852 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5052000000000003e-05 2023-10-01T09:22:52.852 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.852 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.852 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.853 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.699759+0000", 2023-10-01T09:22:52.853 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.185555584999999 2023-10-01T09:22:52.853 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.853 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.853 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.853 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.853 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.853 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:95350 6.a 6:533274a8:::benchmark_data_smithi138_307751_object95349:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-01T09:22:52.853 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:03.517602+0000", 2023-10-01T09:22:52.853 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 109.20608586, 2023-10-01T09:22:52.853 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.29910707, 2023-10-01T09:22:52.854 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.854 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.854 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:22:52.854 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:22:52.854 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:22:52.854 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 95350 2023-10-01T09:22:52.854 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.854 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.854 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.854 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.854 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.517602+0000", 2023-10-01T09:22:52.855 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.855 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.855 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.855 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.855 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.517602+0000", 2023-10-01T09:22:52.855 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:22:52.855 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.855 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.855 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.855 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.517600+0000", 2023-10-01T09:22:52.855 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1515999999999999e-05 2023-10-01T09:22:52.856 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.856 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.856 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.856 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.517652+0000", 2023-10-01T09:22:52.856 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.823e-05 2023-10-01T09:22:52.856 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.856 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.856 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.856 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.517670+0000", 2023-10-01T09:22:52.856 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1251e-05 2023-10-01T09:22:52.856 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.857 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.857 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.857 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.517681+0000", 2023-10-01T09:22:52.857 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.257534757 2023-10-01T09:22:52.857 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.857 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.857 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.857 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.775216+0000", 2023-10-01T09:22:52.857 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015269400000000001 2023-10-01T09:22:52.857 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.857 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.858 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.858 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.775369+0000", 2023-10-01T09:22:52.858 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00020445199999999999 2023-10-01T09:22:52.858 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.858 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.858 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:22:52.858 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.775573+0000", 2023-10-01T09:22:52.858 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019186920000000001 2023-10-01T09:22:52.858 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.858 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.858 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:22:52.859 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.777492+0000", 2023-10-01T09:22:52.859 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0091234820000000005 2023-10-01T09:22:52.859 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.859 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.859 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.859 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.786615+0000", 2023-10-01T09:22:52.859 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.028866505000000001 2023-10-01T09:22:52.859 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.859 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.859 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:22:52.859 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.815482+0000", 2023-10-01T09:22:52.860 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9375999999999997e-05 2023-10-01T09:22:52.860 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.860 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.860 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.860 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.815521+0000", 2023-10-01T09:22:52.860 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0011883880000000001 2023-10-01T09:22:52.860 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.860 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.860 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.860 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.816709+0000", 2023-10-01T09:22:52.860 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.29910707 2023-10-01T09:22:52.861 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.861 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.861 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.861 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.861 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.861 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:95362 6.2 e209/207 6:4eb5c0df:::benchmark_data_smithi138_307751_object95361:head v 209'6082, mlcod=206'6076)", 2023-10-01T09:22:52.861 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:03.545050+0000", 2023-10-01T09:22:52.861 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 109.178637702, 2023-10-01T09:22:52.861 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.170789941000001, 2023-10-01T09:22:52.861 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.861 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.861 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.862 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.862 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.862 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.545050+0000", 2023-10-01T09:22:52.862 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.862 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.862 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.862 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.862 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.545050+0000", 2023-10-01T09:22:52.862 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:22:52.862 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.862 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.863 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.863 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.545050+0000", 2023-10-01T09:22:52.863 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4658000000000002e-05 2023-10-01T09:22:52.863 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.863 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.863 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.863 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.545124+0000", 2023-10-01T09:22:52.863 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4637000000000001e-05 2023-10-01T09:22:52.863 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.863 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.863 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.863 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.545139+0000", 2023-10-01T09:22:52.864 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8440000000000003e-06 2023-10-01T09:22:52.864 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.864 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.864 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.864 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.545145+0000", 2023-10-01T09:22:52.864 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.150749341999999 2023-10-01T09:22:52.864 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.864 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.864 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.864 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.695894+0000", 2023-10-01T09:22:52.864 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7725999999999999e-05 2023-10-01T09:22:52.864 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.865 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.865 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.865 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.695912+0000", 2023-10-01T09:22:52.865 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019908239000000001 2023-10-01T09:22:52.865 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.865 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.865 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.865 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.715820+0000", 2023-10-01T09:22:52.865 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0210000000000001e-05 2023-10-01T09:22:52.865 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.865 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.866 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.866 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.715840+0000", 2023-10-01T09:22:52.866 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.170789941000001 2023-10-01T09:22:52.866 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.866 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.866 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.866 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.866 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.866 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35070.0:95366 6.2 e209/207 6:4a76bb19:::benchmark_data_smithi138_307751_object95365:head v 209'6083, mlcod=206'6076)", 2023-10-01T09:22:52.866 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:21:03.558617+0000", 2023-10-01T09:22:52.866 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 109.16507149900001, 2023-10-01T09:22:52.867 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.191152008, 2023-10-01T09:22:52.867 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:22:52.867 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:22:52.867 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:22:52.867 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.867 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:22:52.867 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.558617+0000", 2023-10-01T09:22:52.867 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:22:52.867 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.867 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.867 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:22:52.867 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.558617+0000", 2023-10-01T09:22:52.868 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:22:52.868 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.868 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.868 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:22:52.868 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.558615+0000", 2023-10-01T09:22:52.868 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1624999999999994e-05 2023-10-01T09:22:52.868 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.868 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.868 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:22:52.868 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.558677+0000", 2023-10-01T09:22:52.868 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9791000000000001e-05 2023-10-01T09:22:52.868 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.869 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.869 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:22:52.869 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.558697+0000", 2023-10-01T09:22:52.869 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9079999999999995e-06 2023-10-01T09:22:52.869 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.869 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.869 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:22:52.869 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:03.558705+0000", 2023-10-01T09:22:52.869 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.139161896999999 2023-10-01T09:22:52.869 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.869 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.869 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:22:52.870 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.697867+0000", 2023-10-01T09:22:52.870 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3332e-05 2023-10-01T09:22:52.870 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.870 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.870 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:22:52.870 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.697880+0000", 2023-10-01T09:22:52.870 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.051858397000000001 2023-10-01T09:22:52.870 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.870 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.870 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:22:52.870 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.749738+0000", 2023-10-01T09:22:52.870 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0437999999999999e-05 2023-10-01T09:22:52.871 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:22:52.871 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:22:52.871 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:22:52.871 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:21:14.749769+0000", 2023-10-01T09:22:52.871 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 11.191152008 2023-10-01T09:22:52.871 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.871 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.871 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.871 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:22:52.871 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:22:52.871 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:22:52.872 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:22:52.902 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:52.948 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:22:52.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:52.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:22:53.051 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:22:53.078 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:53.123 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:22:53.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:53.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:22:53.225 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:53.250 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:53 smithi138 ceph-mon[192513]: failed to encode map e241 with expected crc 2023-10-01T09:22:53.250 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:53 smithi138 ceph-mon[192513]: failed to encode map e241 with expected crc 2023-10-01T09:22:53.251 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:53 smithi138 ceph-mon[192513]: failed to encode map e241 with expected crc 2023-10-01T09:22:53.251 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:53 smithi138 ceph-mon[192513]: failed to encode map e241 with expected crc 2023-10-01T09:22:53.251 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:53 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:22:53.251 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:53 smithi138 ceph-mon[192513]: osdmap e242: 8 total, 8 up, 8 in 2023-10-01T09:22:53.251 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:53 smithi138 ceph-mon[185886]: failed to encode map e241 with expected crc 2023-10-01T09:22:53.251 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:53 smithi138 ceph-mon[185886]: failed to encode map e241 with expected crc 2023-10-01T09:22:53.251 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:53 smithi138 ceph-mon[185886]: failed to encode map e241 with expected crc 2023-10-01T09:22:53.251 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:53 smithi138 ceph-mon[185886]: failed to encode map e241 with expected crc 2023-10-01T09:22:53.251 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:53 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:22:53.252 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:53 smithi138 ceph-mon[185886]: osdmap e242: 8 total, 8 up, 8 in 2023-10-01T09:22:53.253 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:53.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:53 smithi190 ceph-mon[160435]: failed to encode map e241 with expected crc 2023-10-01T09:22:53.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:53 smithi190 ceph-mon[160435]: failed to encode map e241 with expected crc 2023-10-01T09:22:53.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:53 smithi190 ceph-mon[160435]: failed to encode map e241 with expected crc 2023-10-01T09:22:53.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:53 smithi190 ceph-mon[160435]: failed to encode map e241 with expected crc 2023-10-01T09:22:53.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:53 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:22:53.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:53 smithi190 ceph-mon[160435]: osdmap e242: 8 total, 8 up, 8 in 2023-10-01T09:22:53.590 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:22:53.639 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:53.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:53.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:53.748 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:22:53.794 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:53.868 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:22:53.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:53.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:22:53.945 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:22:53.982 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:22:54.063 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:54.180 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:22:54.204 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:22:54.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:54.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:22:54.313 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:54.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:54 smithi138 ceph-mon[192513]: failed to encode map e242 with expected crc 2023-10-01T09:22:54.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:54 smithi138 ceph-mon[192513]: failed to encode map e242 with expected crc 2023-10-01T09:22:54.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:54 smithi138 ceph-mon[192513]: failed to encode map e242 with expected crc 2023-10-01T09:22:54.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:54 smithi138 ceph-mon[192513]: pgmap v609: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 17946/75984 objects degraded (23.618%); 7515/75984 objects misplaced (9.890%); 320 KiB/s, 100 objects/s recovering 2023-10-01T09:22:54.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:54 smithi138 ceph-mon[185886]: failed to encode map e242 with expected crc 2023-10-01T09:22:54.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:54 smithi138 ceph-mon[185886]: failed to encode map e242 with expected crc 2023-10-01T09:22:54.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:54 smithi138 ceph-mon[185886]: failed to encode map e242 with expected crc 2023-10-01T09:22:54.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:54 smithi138 ceph-mon[185886]: pgmap v609: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 17946/75984 objects degraded (23.618%); 7515/75984 objects misplaced (9.890%); 320 KiB/s, 100 objects/s recovering 2023-10-01T09:22:54.387 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:54.488 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:54 smithi190 ceph-mon[160435]: failed to encode map e242 with expected crc 2023-10-01T09:22:54.488 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:54 smithi190 ceph-mon[160435]: failed to encode map e242 with expected crc 2023-10-01T09:22:54.488 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:54 smithi190 ceph-mon[160435]: failed to encode map e242 with expected crc 2023-10-01T09:22:54.488 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:54 smithi190 ceph-mon[160435]: pgmap v609: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 88 active+clean; 1.5 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 17946/75984 objects degraded (23.618%); 7515/75984 objects misplaced (9.890%); 320 KiB/s, 100 objects/s recovering 2023-10-01T09:22:54.536 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:54.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:54.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:54.634 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:22:54.652 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:22:54.682 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:22:54.704 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:54.721 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:22:54.760 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:22:54.807 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:22:54.840 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:22:54.848 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:22:54.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:54.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:22:54.892 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:22:54.923 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:22:54.938 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:22:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:22:54] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.33.4" 2023-10-01T09:22:54.959 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:22:54.962 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:22:54.962 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:22:54.962 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:22:54.962 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:22:54.963 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:22:54.963 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:22:54.963 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:22:54.963 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:22:55.048 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:55.235 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:55 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3496949048' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:55.237 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:55 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3496949048' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:55.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:55 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3496949048' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:22:55.668 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:55.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:55.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:55.780 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:55.862 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:56.080 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:56.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:56.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:56.170 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:22:56.170 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:22:56.194 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:56.265 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:56.286 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:22:56.355 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:56.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:56.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:56.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:22:56.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:56 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2528322849' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:56.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:56 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:56.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:56 smithi190 ceph-mon[160435]: pgmap v610: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 17518/71376 objects degraded (24.543%); 7343/71376 objects misplaced (10.288%); 304 KiB/s, 95 objects/s recovering 2023-10-01T09:22:56.477 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:56.516 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:56.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:22:56.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:56 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2528322849' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:56.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:56 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:56.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:56 smithi138 ceph-mon[185886]: pgmap v610: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 17518/71376 objects degraded (24.543%); 7343/71376 objects misplaced (10.288%); 304 KiB/s, 95 objects/s recovering 2023-10-01T09:22:56.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:22:56.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:56 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2528322849' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:56.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:56 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:22:56.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:56 smithi138 ceph-mon[192513]: pgmap v610: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 17518/71376 objects degraded (24.543%); 7343/71376 objects misplaced (10.288%); 304 KiB/s, 95 objects/s recovering 2023-10-01T09:22:56.583 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:56.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:56.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:56.690 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:56.716 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:56.757 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:56.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:56.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:56.860 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:22:56.893 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:57.157 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:22:57.208 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:22:57.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:57.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:22:57.319 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:57.346 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:57.416 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:57.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:57.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:57.428 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:57 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:22:57.428 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:57 smithi190 ceph-mon[160435]: failed to encode map e243 with expected crc 2023-10-01T09:22:57.428 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:57 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:22:57.428 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:57 smithi190 ceph-mon[160435]: osdmap e243: 8 total, 8 up, 8 in 2023-10-01T09:22:57.428 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:57 smithi190 ceph-mon[160435]: failed to encode map e243 with expected crc 2023-10-01T09:22:57.429 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:57 smithi190 ceph-mon[160435]: failed to encode map e243 with expected crc 2023-10-01T09:22:57.429 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:57 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 17518/71376 objects degraded (24.543%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:57.528 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:57.552 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:57 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:22:57.552 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:57 smithi138 ceph-mon[192513]: failed to encode map e243 with expected crc 2023-10-01T09:22:57.552 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:57 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:22:57.552 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:57 smithi138 ceph-mon[192513]: osdmap e243: 8 total, 8 up, 8 in 2023-10-01T09:22:57.553 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:57 smithi138 ceph-mon[192513]: failed to encode map e243 with expected crc 2023-10-01T09:22:57.553 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:57 smithi138 ceph-mon[192513]: failed to encode map e243 with expected crc 2023-10-01T09:22:57.553 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:57 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 17518/71376 objects degraded (24.543%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:57.553 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:57 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:22:57.553 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:57 smithi138 ceph-mon[185886]: failed to encode map e243 with expected crc 2023-10-01T09:22:57.553 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:57 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:22:57.553 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:57 smithi138 ceph-mon[185886]: osdmap e243: 8 total, 8 up, 8 in 2023-10-01T09:22:57.553 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:57 smithi138 ceph-mon[185886]: failed to encode map e243 with expected crc 2023-10-01T09:22:57.553 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:57 smithi138 ceph-mon[185886]: failed to encode map e243 with expected crc 2023-10-01T09:22:57.554 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:57 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 17518/71376 objects degraded (24.543%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T09:22:57.554 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:57.648 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:22:58.126 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:58.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:58.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:58.232 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:58.302 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:58.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:58 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:22:58.364 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:58 smithi138 ceph-mon[185886]: pgmap v612: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 17215/65604 objects degraded (26.241%); 7343/65604 objects misplaced (11.193%); 2.7 MiB/s, 137 objects/s recovering 2023-10-01T09:22:58.364 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:58 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:22:58.364 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:58 smithi138 ceph-mon[192513]: pgmap v612: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 17215/65604 objects degraded (26.241%); 7343/65604 objects misplaced (11.193%); 2.7 MiB/s, 137 objects/s recovering 2023-10-01T09:22:58.395 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:58.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:58.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:58.501 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:22:58.540 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:58.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:58 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:22:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:58 smithi190 ceph-mon[160435]: pgmap v612: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 17215/65604 objects degraded (26.241%); 7343/65604 objects misplaced (11.193%); 2.7 MiB/s, 137 objects/s recovering 2023-10-01T09:22:58.585 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:22:58.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:58.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:22:58.688 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:22:58.719 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:59.044 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:22:59.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:59.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:22:59.157 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:22:59.212 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:59.287 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:22:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1056244118' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:22:59.334 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.5 to 1 (10000) 2023-10-01T09:22:59.353 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:22:59.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:59.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:22:59.461 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:22:59.532 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:59.536 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:22:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1056244118' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:22:59.539 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:22:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1056244118' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:22:59.615 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:22:59.615 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:22:59.643 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:22:59.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:59.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:22:59.747 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:22:59.819 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:22:59.946 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:22:59.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:22:59.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:23:00.050 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:00.075 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:23:00.077 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:00.123 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:00.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:00.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:00.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:23:00.227 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:00.270 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:00.496 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:00.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:00.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:00.550 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:00 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1056244118' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:23:00.550 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:00 smithi190 ceph-mon[160435]: osdmap e244: 8 total, 8 up, 8 in 2023-10-01T09:23:00.550 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:00 smithi190 ceph-mon[160435]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.550 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:00 smithi190 ceph-mon[160435]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.550 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:00 smithi190 ceph-mon[160435]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.551 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:00 smithi190 ceph-mon[160435]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.551 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:00 smithi190 ceph-mon[160435]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.551 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:00 smithi190 ceph-mon[160435]: pgmap v614: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 16993/63504 objects degraded (26.759%); 7343/63504 objects misplaced (11.563%); 2.2 MiB/s, 35 objects/s recovering 2023-10-01T09:23:00.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1056244118' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:23:00.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[185886]: osdmap e244: 8 total, 8 up, 8 in 2023-10-01T09:23:00.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[185886]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[185886]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[185886]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[185886]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[185886]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[185886]: pgmap v614: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 16993/63504 objects degraded (26.759%); 7343/63504 objects misplaced (11.563%); 2.2 MiB/s, 35 objects/s recovering 2023-10-01T09:23:00.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1056244118' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:23:00.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[192513]: osdmap e244: 8 total, 8 up, 8 in 2023-10-01T09:23:00.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[192513]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[192513]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[192513]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[192513]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[192513]: failed to encode map e244 with expected crc 2023-10-01T09:23:00.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:00 smithi138 ceph-mon[192513]: pgmap v614: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.3 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 16993/63504 objects degraded (26.759%); 7343/63504 objects misplaced (11.563%); 2.2 MiB/s, 35 objects/s recovering 2023-10-01T09:23:00.574 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:23:00.607 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:23:00.702 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:00.837 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:23:00.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:00.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:23:00.947 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:23:00.997 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-01T09:23:00.997 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:01.129 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:23:01.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:01.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:23:01.166 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:23:01.216 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:01.235 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:23:01.311 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:01.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:01 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:23:01.387 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:01.391 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:01 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:23:01.494 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:23:01.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:01.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:23:01.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:01 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:23:01.600 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:01.666 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:01.973 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:01.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:01.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:02.077 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:02.146 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:02.303 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:02.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:02.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:02.413 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:02.466 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:02 smithi190 ceph-mon[160435]: pgmap v615: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 16768/60210 objects degraded (27.849%); 7095/60210 objects misplaced (11.784%); 1.8 MiB/s, 28 objects/s recovering 2023-10-01T09:23:02.466 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:02 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 16993/63504 objects degraded (26.759%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T09:23:02.469 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:02.550 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:02.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:02.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:02.658 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:02.734 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:02 smithi138 ceph-mon[185886]: pgmap v615: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 16768/60210 objects degraded (27.849%); 7095/60210 objects misplaced (11.784%); 1.8 MiB/s, 28 objects/s recovering 2023-10-01T09:23:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:02 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 16993/63504 objects degraded (26.759%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T09:23:02.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:02 smithi138 ceph-mon[192513]: pgmap v615: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.2 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 16768/60210 objects degraded (27.849%); 7095/60210 objects misplaced (11.784%); 1.8 MiB/s, 28 objects/s recovering 2023-10-01T09:23:02.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:02 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 16993/63504 objects degraded (26.759%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T09:23:03.085 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:03.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:03.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:03.199 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:03.228 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:03.639 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:03.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:03.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:03.719 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:23:03.719 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:23:03.746 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:23:03.822 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:03.950 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:23:03.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:03.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:23:04.057 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:23:04.117 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:04.217 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:23:04.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:04.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:23:04.323 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:04.393 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:04.562 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:04.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:04.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:04.667 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:23:04.672 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:04 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:23:04.672 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:04 smithi190 ceph-mon[160435]: pgmap v616: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 16541/54198 objects degraded (30.520%); 7095/54198 objects misplaced (13.091%); 3.5 MiB/s, 56 objects/s recovering 2023-10-01T09:23:04.690 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:04.701 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:23:04.732 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:04.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:04 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:23:04.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:04 smithi138 ceph-mon[192513]: pgmap v616: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 16541/54198 objects degraded (30.520%); 7095/54198 objects misplaced (13.091%); 3.5 MiB/s, 56 objects/s recovering 2023-10-01T09:23:04.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:04 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:23:04.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:04 smithi138 ceph-mon[185886]: pgmap v616: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 16541/54198 objects degraded (30.520%); 7095/54198 objects misplaced (13.091%); 3.5 MiB/s, 56 objects/s recovering 2023-10-01T09:23:04.831 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:23:04.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:04.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:23:04.945 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:23:05.039 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:05.209 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:23:05.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:05.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:23:05.216 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:23:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:23:04] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.33.4" 2023-10-01T09:23:05.269 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:23:05.311 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:23:05.317 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:05.346 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:23:05.387 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:23:05.394 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:05.427 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:23:05.445 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:23:05.474 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:23:05.512 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:23:05.552 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:23:05.576 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:23:05.576 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:23:05.576 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:23:05.576 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:23:05.576 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:23:05.576 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:23:05.576 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:23:05.576 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:23:05.596 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:23:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2489930328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:23:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:05 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:23:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/441895203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:05.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2489930328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:23:05.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:05 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:23:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/441895203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2489930328' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:23:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:05 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:23:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/441895203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:06.530 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:06.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:06.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:06.639 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:23:06.714 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:06 smithi190 ceph-mon[160435]: pgmap v617: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.0 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 16307/49899 objects degraded (32.680%); 6756/49899 objects misplaced (13.539%); 3.9 MiB/s, 62 objects/s recovering 2023-10-01T09:23:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:06 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:23:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:06 smithi190 ceph-mon[160435]: failed to encode map e245 with expected crc 2023-10-01T09:23:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:06 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:23:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:06 smithi190 ceph-mon[160435]: osdmap e245: 8 total, 8 up, 8 in 2023-10-01T09:23:06.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:06 smithi190 ceph-mon[160435]: failed to encode map e245 with expected crc 2023-10-01T09:23:06.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:06 smithi138 ceph-mon[185886]: pgmap v617: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.0 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 16307/49899 objects degraded (32.680%); 6756/49899 objects misplaced (13.539%); 3.9 MiB/s, 62 objects/s recovering 2023-10-01T09:23:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:06 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:23:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:06 smithi138 ceph-mon[185886]: failed to encode map e245 with expected crc 2023-10-01T09:23:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:06 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:23:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:06 smithi138 ceph-mon[185886]: osdmap e245: 8 total, 8 up, 8 in 2023-10-01T09:23:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:06 smithi138 ceph-mon[185886]: failed to encode map e245 with expected crc 2023-10-01T09:23:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:06 smithi138 ceph-mon[192513]: pgmap v617: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 1.0 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 16307/49899 objects degraded (32.680%); 6756/49899 objects misplaced (13.539%); 3.9 MiB/s, 62 objects/s recovering 2023-10-01T09:23:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:06 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:23:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:06 smithi138 ceph-mon[192513]: failed to encode map e245 with expected crc 2023-10-01T09:23:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:06 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:23:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:06 smithi138 ceph-mon[192513]: osdmap e245: 8 total, 8 up, 8 in 2023-10-01T09:23:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:06 smithi138 ceph-mon[192513]: failed to encode map e245 with expected crc 2023-10-01T09:23:06.847 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:23:06.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:06.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:23:06.952 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:07.023 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:07.345 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:23:07.345 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:23:07.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.345 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:177107 6.a 6:5ff07b70:::benchmark_data_smithi138_307751_object81738:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-10-01T09:23:07.345 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:23:07.304789+0000", 2023-10-01T09:23:07.345 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.039295675000000002, 2023-10-01T09:23:07.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.039317906, 2023-10-01T09:23:07.345 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:07.346 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:23:07.346 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:07.346 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:07.346 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:07.346 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 177107 2023-10-01T09:23:07.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.346 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:07.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:07.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.304789+0000", 2023-10-01T09:23:07.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:07.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:07.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.304789+0000", 2023-10-01T09:23:07.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:23:07.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:07.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.304787+0000", 2023-10-01T09:23:07.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9390000000000002e-06 2023-10-01T09:23:07.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:07.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.304793+0000", 2023-10-01T09:23:07.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.025e-05 2023-10-01T09:23:07.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:07.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.304804+0000", 2023-10-01T09:23:07.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3530000000000002e-06 2023-10-01T09:23:07.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:07.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.304809+0000", 2023-10-01T09:23:07.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0046798070000000002 2023-10-01T09:23:07.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:07.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.309489+0000", 2023-10-01T09:23:07.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3739999999999994e-05 2023-10-01T09:23:07.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:07.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.309582+0000", 2023-10-01T09:23:07.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8926999999999998e-05 2023-10-01T09:23:07.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:23:07.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.309661+0000", 2023-10-01T09:23:07.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0027433150000000001 2023-10-01T09:23:07.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:07.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.312405+0000", 2023-10-01T09:23:07.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0039087899999999997 2023-10-01T09:23:07.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:07.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.316313+0000", 2023-10-01T09:23:07.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011524987 2023-10-01T09:23:07.351 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:07.351 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:07.351 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:07.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.351 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:177072 6.a 6:5c04b441:::benchmark_data_smithi138_307751_object81703:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-10-01T09:23:07.351 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:23:07.259065+0000", 2023-10-01T09:23:07.352 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.085019301000000005, 2023-10-01T09:23:07.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.085100993, 2023-10-01T09:23:07.352 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:07.352 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:23:07.352 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:07.352 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:07.352 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:07.352 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 177072 2023-10-01T09:23:07.352 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.352 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:07.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.352 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:07.353 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.259065+0000", 2023-10-01T09:23:07.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:07.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.353 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:07.353 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.259065+0000", 2023-10-01T09:23:07.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:23:07.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.353 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:07.353 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.259063+0000", 2023-10-01T09:23:07.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2790000000000001e-06 2023-10-01T09:23:07.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:07.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.259071+0000", 2023-10-01T09:23:07.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5926999999999998e-05 2023-10-01T09:23:07.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:07.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.259086+0000", 2023-10-01T09:23:07.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4930000000000003e-06 2023-10-01T09:23:07.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:07.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.259093+0000", 2023-10-01T09:23:07.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0618000000000002e-05 2023-10-01T09:23:07.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:07.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.259144+0000", 2023-10-01T09:23:07.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000113397 2023-10-01T09:23:07.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:07.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.259257+0000", 2023-10-01T09:23:07.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013349799999999999 2023-10-01T09:23:07.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:23:07.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.259390+0000", 2023-10-01T09:23:07.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0064130350000000001 2023-10-01T09:23:07.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:07.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.265803+0000", 2023-10-01T09:23:07.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00035381799999999999 2023-10-01T09:23:07.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:07.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.266157+0000", 2023-10-01T09:23:07.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0070924289999999999 2023-10-01T09:23:07.358 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:07.358 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:07.358 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:07.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.358 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:177090 6.a 6:5c06eff5:::benchmark_data_smithi138_307751_object81721:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-10-01T09:23:07.359 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:23:07.286175+0000", 2023-10-01T09:23:07.359 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.057909367000000003, 2023-10-01T09:23:07.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.058042679999999999, 2023-10-01T09:23:07.359 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:07.359 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:23:07.359 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:07.359 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:07.359 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:07.359 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 177090 2023-10-01T09:23:07.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.360 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:07.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.360 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:07.360 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.286175+0000", 2023-10-01T09:23:07.360 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:07.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.360 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:07.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.286175+0000", 2023-10-01T09:23:07.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:23:07.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:07.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.286173+0000", 2023-10-01T09:23:07.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7649999999999993e-06 2023-10-01T09:23:07.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:07.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.286181+0000", 2023-10-01T09:23:07.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2258e-05 2023-10-01T09:23:07.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:07.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.286193+0000", 2023-10-01T09:23:07.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2460000000000003e-06 2023-10-01T09:23:07.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:07.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.286201+0000", 2023-10-01T09:23:07.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7064999999999997e-05 2023-10-01T09:23:07.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:07.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.286248+0000", 2023-10-01T09:23:07.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015130499999999999 2023-10-01T09:23:07.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:07.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.286400+0000", 2023-10-01T09:23:07.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000128725 2023-10-01T09:23:07.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:23:07.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.286529+0000", 2023-10-01T09:23:07.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035537429999999998 2023-10-01T09:23:07.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:07.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.290082+0000", 2023-10-01T09:23:07.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0029095840000000002 2023-10-01T09:23:07.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:07.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.292992+0000", 2023-10-01T09:23:07.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.006817025 2023-10-01T09:23:07.366 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:07.366 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:07.366 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:07.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.366 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:177104 6.a 6:505234df:::benchmark_data_smithi138_307751_object81735:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-10-01T09:23:07.366 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:23:07.301464+0000", 2023-10-01T09:23:07.366 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.042620539999999998, 2023-10-01T09:23:07.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.042810806, 2023-10-01T09:23:07.367 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:07.367 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:23:07.367 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:07.367 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:07.367 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:07.367 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 177104 2023-10-01T09:23:07.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.368 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:07.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.368 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:07.368 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301464+0000", 2023-10-01T09:23:07.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:07.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:07.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301464+0000", 2023-10-01T09:23:07.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:23:07.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:07.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301462+0000", 2023-10-01T09:23:07.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6300000000000003e-06 2023-10-01T09:23:07.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:07.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301468+0000", 2023-10-01T09:23:07.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0424999999999999e-05 2023-10-01T09:23:07.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:07.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301478+0000", 2023-10-01T09:23:07.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7550000000000002e-06 2023-10-01T09:23:07.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:07.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301483+0000", 2023-10-01T09:23:07.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7075000000000003e-05 2023-10-01T09:23:07.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:07.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301560+0000", 2023-10-01T09:23:07.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012642999999999999 2023-10-01T09:23:07.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:07.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301686+0000", 2023-10-01T09:23:07.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015231000000000001 2023-10-01T09:23:07.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:23:07.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301839+0000", 2023-10-01T09:23:07.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010351377 2023-10-01T09:23:07.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:07.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.312190+0000", 2023-10-01T09:23:07.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2794e-05 2023-10-01T09:23:07.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:07.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.312203+0000", 2023-10-01T09:23:07.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010739322000000001 2023-10-01T09:23:07.373 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:07.373 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:07.374 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:07.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.374 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:177105 6.a 6:52483100:::benchmark_data_smithi138_307751_object81736:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-10-01T09:23:07.374 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:23:07.301494+0000", 2023-10-01T09:23:07.374 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.042590344000000002, 2023-10-01T09:23:07.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.042830299000000002, 2023-10-01T09:23:07.374 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:07.374 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:23:07.374 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:07.374 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:07.375 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:07.375 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 177105 2023-10-01T09:23:07.375 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.375 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:07.375 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.375 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:07.375 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301494+0000", 2023-10-01T09:23:07.375 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:07.375 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.375 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.375 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:07.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301494+0000", 2023-10-01T09:23:07.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:23:07.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.376 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:07.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301493+0000", 2023-10-01T09:23:07.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4939999999999997e-06 2023-10-01T09:23:07.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.376 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:07.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301498+0000", 2023-10-01T09:23:07.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6909999999999997e-06 2023-10-01T09:23:07.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:07.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301506+0000", 2023-10-01T09:23:07.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7979999999999999e-06 2023-10-01T09:23:07.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:07.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.301509+0000", 2023-10-01T09:23:07.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0034198869999999999 2023-10-01T09:23:07.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:07.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.304929+0000", 2023-10-01T09:23:07.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000121335 2023-10-01T09:23:07.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:07.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.305051+0000", 2023-10-01T09:23:07.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000144497 2023-10-01T09:23:07.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:23:07.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.305195+0000", 2023-10-01T09:23:07.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0070133569999999996 2023-10-01T09:23:07.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:07.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.312208+0000", 2023-10-01T09:23:07.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0039087419999999998 2023-10-01T09:23:07.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:07.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:07.380 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:07.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:23:07.316117+0000", 2023-10-01T09:23:07.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014623387 2023-10-01T09:23:07.380 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:07.380 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:07.380 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:07.380 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:07.380 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:23:07.380 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 5 2023-10-01T09:23:07.380 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:23:07.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:07 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 16307/49899 objects degraded (32.680%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:07.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:07 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 16307/49899 objects degraded (32.680%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:07 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 16307/49899 objects degraded (32.680%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:08.275 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T09:23:08.279 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:08.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:08.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:08.392 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:08.479 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:08.697 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:08.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:08.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:08.728 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:08 smithi138 ceph-mon[185886]: pgmap v619: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 917 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 15857/43872 objects degraded (36.144%); 6545/43872 objects misplaced (14.918%); 6.1 MiB/s, 97 objects/s recovering 2023-10-01T09:23:08.736 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:08 smithi138 ceph-mon[192513]: pgmap v619: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 917 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 15857/43872 objects degraded (36.144%); 6545/43872 objects misplaced (14.918%); 6.1 MiB/s, 97 objects/s recovering 2023-10-01T09:23:08.801 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:08.840 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:23:08.893 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:09.022 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:09.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:09.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:09.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:08 smithi190 ceph-mon[160435]: pgmap v619: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 917 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 15857/43872 objects degraded (36.144%); 6545/43872 objects misplaced (14.918%); 6.1 MiB/s, 97 objects/s recovering 2023-10-01T09:23:09.141 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:09.197 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:09.306 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:09.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:09.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:09.412 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:23:09.472 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:09.545 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:23:09.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:09.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:23:09.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:09 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4157365706' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:23:09.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:09 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:23:09.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:09 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4157365706' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:23:09.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:09 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:23:09.580 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.7 to 1 (10000) 2023-10-01T09:23:09.649 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:09.686 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:09.730 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:09.732 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:09 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4157365706' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:23:09.732 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:09 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:23:09.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:09.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:09.834 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:09.865 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:09.921 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:09.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:09.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:10.025 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:10.051 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:10.068 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:23:10.097 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:10.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:10.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:10.276 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:23:10.277 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:23:10.305 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:10.349 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:23:10.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:10.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:23:10.452 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:10.479 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:10.651 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:23:10.651 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:23:10.651 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:23:10.651 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:23:10.651 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:23:10.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:10 smithi190 ceph-mon[160435]: pgmap v620: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 878 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 15653/41997 objects degraded (37.272%); 6342/41997 objects misplaced (15.101%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:23:10.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:10 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:23:10.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:10 smithi190 ceph-mon[160435]: osdmap e246: 8 total, 8 up, 8 in 2023-10-01T09:23:10.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:10 smithi190 ceph-mon[160435]: failed to encode map e246 with expected crc 2023-10-01T09:23:10.834 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:10 smithi190 ceph-mon[160435]: failed to encode map e246 with expected crc 2023-10-01T09:23:10.835 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:10 smithi190 ceph-mon[160435]: failed to encode map e246 with expected crc 2023-10-01T09:23:10.835 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:10 smithi190 ceph-mon[160435]: failed to encode map e246 with expected crc 2023-10-01T09:23:10.835 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:23:11.001 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:11.003 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[185886]: pgmap v620: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 878 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 15653/41997 objects degraded (37.272%); 6342/41997 objects misplaced (15.101%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:23:11.003 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:23:11.003 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[185886]: osdmap e246: 8 total, 8 up, 8 in 2023-10-01T09:23:11.003 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[185886]: failed to encode map e246 with expected crc 2023-10-01T09:23:11.003 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[185886]: failed to encode map e246 with expected crc 2023-10-01T09:23:11.003 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[185886]: failed to encode map e246 with expected crc 2023-10-01T09:23:11.003 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[185886]: failed to encode map e246 with expected crc 2023-10-01T09:23:11.003 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:23:11.004 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[192513]: pgmap v620: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 89 active+clean; 878 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 15653/41997 objects degraded (37.272%); 6342/41997 objects misplaced (15.101%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:23:11.004 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:23:11.004 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[192513]: osdmap e246: 8 total, 8 up, 8 in 2023-10-01T09:23:11.004 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[192513]: failed to encode map e246 with expected crc 2023-10-01T09:23:11.004 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[192513]: failed to encode map e246 with expected crc 2023-10-01T09:23:11.004 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[192513]: failed to encode map e246 with expected crc 2023-10-01T09:23:11.004 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[192513]: failed to encode map e246 with expected crc 2023-10-01T09:23:11.004 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:23:11.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:11.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:11.082 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-01T09:23:11.106 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:11.149 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:11.160 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:11.203 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:11.226 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:11.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:11.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:11.331 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T09:23:11.333 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:11.361 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:11.413 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:11.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:11.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:11.519 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:11.547 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:11.589 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:11 smithi190 ceph-mon[160435]: osdmap e247: 8 total, 8 up, 8 in 2023-10-01T09:23:11.589 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:11 smithi190 ceph-mon[160435]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.589 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:11 smithi190 ceph-mon[160435]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.589 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:11 smithi190 ceph-mon[160435]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.589 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:11 smithi190 ceph-mon[160435]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.589 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:11 smithi190 ceph-mon[160435]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.589 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:11 smithi190 ceph-mon[160435]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.589 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:11 smithi190 ceph-mon[160435]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.589 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:11 smithi190 ceph-mon[160435]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.589 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:11 smithi190 ceph-mon[160435]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.590 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:11 smithi190 ceph-mon[160435]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[185886]: osdmap e247: 8 total, 8 up, 8 in 2023-10-01T09:23:11.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[185886]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[185886]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[185886]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[185886]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[185886]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.663 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[185886]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.663 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[185886]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.663 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[185886]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.663 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[185886]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.663 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[185886]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[192513]: osdmap e247: 8 total, 8 up, 8 in 2023-10-01T09:23:11.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[192513]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[192513]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[192513]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[192513]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[192513]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[192513]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[192513]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[192513]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[192513]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:11 smithi138 ceph-mon[192513]: failed to encode map e247 with expected crc 2023-10-01T09:23:11.720 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:23:12.472 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:12.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:12.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:12.580 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:12.608 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[185886]: pgmap v623: 99 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 820 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 15479/39219 objects degraded (39.468%); 5929/39219 objects misplaced (15.118%); 8.6 MiB/s, 137 objects/s recovering 2023-10-01T09:23:12.608 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[185886]: failed to encode map e248 with expected crc 2023-10-01T09:23:12.608 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[185886]: osdmap e248: 8 total, 8 up, 8 in 2023-10-01T09:23:12.608 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[185886]: failed to encode map e248 with expected crc 2023-10-01T09:23:12.609 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[185886]: failed to encode map e248 with expected crc 2023-10-01T09:23:12.609 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[185886]: 6.b starting backfill to osd.7 from (0'0,0'0] MAX to 245'11215 2023-10-01T09:23:12.609 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 15479/39219 objects degraded (39.468%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:12.609 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[185886]: failed to encode map e249 with expected crc 2023-10-01T09:23:12.609 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[185886]: osdmap e249: 8 total, 8 up, 8 in 2023-10-01T09:23:12.609 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[185886]: failed to encode map e249 with expected crc 2023-10-01T09:23:12.609 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[185886]: failed to encode map e249 with expected crc 2023-10-01T09:23:12.609 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[185886]: failed to encode map e249 with expected crc 2023-10-01T09:23:12.609 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[192513]: pgmap v623: 99 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 820 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 15479/39219 objects degraded (39.468%); 5929/39219 objects misplaced (15.118%); 8.6 MiB/s, 137 objects/s recovering 2023-10-01T09:23:12.610 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[192513]: failed to encode map e248 with expected crc 2023-10-01T09:23:12.610 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[192513]: osdmap e248: 8 total, 8 up, 8 in 2023-10-01T09:23:12.610 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[192513]: failed to encode map e248 with expected crc 2023-10-01T09:23:12.610 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[192513]: failed to encode map e248 with expected crc 2023-10-01T09:23:12.610 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[192513]: 6.b starting backfill to osd.7 from (0'0,0'0] MAX to 245'11215 2023-10-01T09:23:12.610 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 15479/39219 objects degraded (39.468%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:12.610 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[192513]: failed to encode map e249 with expected crc 2023-10-01T09:23:12.610 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[192513]: osdmap e249: 8 total, 8 up, 8 in 2023-10-01T09:23:12.610 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[192513]: failed to encode map e249 with expected crc 2023-10-01T09:23:12.610 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[192513]: failed to encode map e249 with expected crc 2023-10-01T09:23:12.610 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:12 smithi138 ceph-mon[192513]: failed to encode map e249 with expected crc 2023-10-01T09:23:12.611 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:12.654 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:12.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:12.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:12.758 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:12 smithi190 ceph-mon[160435]: pgmap v623: 99 pgs: 1 active+recovery_wait+degraded, 3 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 820 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 15479/39219 objects degraded (39.468%); 5929/39219 objects misplaced (15.118%); 8.6 MiB/s, 137 objects/s recovering 2023-10-01T09:23:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:12 smithi190 ceph-mon[160435]: failed to encode map e248 with expected crc 2023-10-01T09:23:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:12 smithi190 ceph-mon[160435]: osdmap e248: 8 total, 8 up, 8 in 2023-10-01T09:23:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:12 smithi190 ceph-mon[160435]: failed to encode map e248 with expected crc 2023-10-01T09:23:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:12 smithi190 ceph-mon[160435]: failed to encode map e248 with expected crc 2023-10-01T09:23:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:12 smithi190 ceph-mon[160435]: 6.b starting backfill to osd.7 from (0'0,0'0] MAX to 245'11215 2023-10-01T09:23:12.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:12 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 15479/39219 objects degraded (39.468%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:12.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:12 smithi190 ceph-mon[160435]: failed to encode map e249 with expected crc 2023-10-01T09:23:12.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:12 smithi190 ceph-mon[160435]: osdmap e249: 8 total, 8 up, 8 in 2023-10-01T09:23:12.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:12 smithi190 ceph-mon[160435]: failed to encode map e249 with expected crc 2023-10-01T09:23:12.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:12 smithi190 ceph-mon[160435]: failed to encode map e249 with expected crc 2023-10-01T09:23:12.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:12 smithi190 ceph-mon[160435]: failed to encode map e249 with expected crc 2023-10-01T09:23:12.830 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:12.986 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:12.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:13.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:13.100 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:13.158 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:13.249 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:13.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:13.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:13.354 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:13.410 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:13.488 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:13.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:13.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:13.554 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:23:13.554 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:23:13.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:23:13.554 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:23:13.554 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.554 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:125418 6.d 6.2bc0704d (undecoded) ondisk+retry+write+known_if_redirected e227)", 2023-10-01T09:23:13.554 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:01.365555+0000", 2023-10-01T09:23:13.554 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 72.183781451000002, 2023-10-01T09:23:13.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81812904099999995, 2023-10-01T09:23:13.554 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.555 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:23:13.555 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.555 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.555 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.555 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 125418 2023-10-01T09:23:13.555 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.555 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.555 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.555 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.555 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365555+0000", 2023-10-01T09:23:13.555 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.555 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.556 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.556 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365555+0000", 2023-10-01T09:23:13.556 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:23:13.556 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.556 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.556 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365553+0000", 2023-10-01T09:23:13.556 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8529999999999992e-06 2023-10-01T09:23:13.556 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.556 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365561+0000", 2023-10-01T09:23:13.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7089999999999997e-06 2023-10-01T09:23:13.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365571+0000", 2023-10-01T09:23:13.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.075e-06 2023-10-01T09:23:13.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365578+0000", 2023-10-01T09:23:13.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000104657 2023-10-01T09:23:13.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.558 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365683+0000", 2023-10-01T09:23:13.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5913000000000002e-05 2023-10-01T09:23:13.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.558 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365699+0000", 2023-10-01T09:23:13.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.817972484 2023-10-01T09:23:13.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.559 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.559 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183671+0000", 2023-10-01T09:23:13.559 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3343000000000001e-05 2023-10-01T09:23:13.559 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.559 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.559 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.559 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183684+0000", 2023-10-01T09:23:13.559 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81812904099999995 2023-10-01T09:23:13.559 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.559 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.559 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.560 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.560 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.560 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:125430 6.d 6.d1d166cd (undecoded) ondisk+write+known_if_redirected e227)", 2023-10-01T09:23:13.560 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:01.397994+0000", 2023-10-01T09:23:13.560 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 72.151342753999998, 2023-10-01T09:23:13.560 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78591710299999995, 2023-10-01T09:23:13.560 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.560 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:23:13.560 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.560 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.560 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.560 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 125430 2023-10-01T09:23:13.561 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.561 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.561 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.561 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.561 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.397994+0000", 2023-10-01T09:23:13.561 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.561 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.561 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.561 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.561 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.397994+0000", 2023-10-01T09:23:13.561 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:23:13.561 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.561 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.562 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.397992+0000", 2023-10-01T09:23:13.562 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0169999999999999e-05 2023-10-01T09:23:13.562 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.562 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.562 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398002+0000", 2023-10-01T09:23:13.562 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6186999999999999e-05 2023-10-01T09:23:13.562 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.562 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.562 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398019+0000", 2023-10-01T09:23:13.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.17e-06 2023-10-01T09:23:13.563 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.563 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.563 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398026+0000", 2023-10-01T09:23:13.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2902000000000002e-05 2023-10-01T09:23:13.563 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.563 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.563 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398089+0000", 2023-10-01T09:23:13.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1385e-05 2023-10-01T09:23:13.564 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.564 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.564 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.564 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398100+0000", 2023-10-01T09:23:13.564 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78580161999999998 2023-10-01T09:23:13.564 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.564 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.564 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.564 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183902+0000", 2023-10-01T09:23:13.564 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3659999999999998e-06 2023-10-01T09:23:13.564 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.564 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.565 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.565 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183911+0000", 2023-10-01T09:23:13.565 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78591710299999995 2023-10-01T09:23:13.565 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.565 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.565 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.565 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.565 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.565 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:125431 6.d 6.45b3c99d (undecoded) ondisk+write+known_if_redirected e227)", 2023-10-01T09:23:13.565 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:01.399470+0000", 2023-10-01T09:23:13.565 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 72.149867037999996, 2023-10-01T09:23:13.565 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78447003999999998, 2023-10-01T09:23:13.566 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.566 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:23:13.566 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.566 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.566 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.566 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 125431 2023-10-01T09:23:13.566 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.566 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.566 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.566 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.566 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399470+0000", 2023-10-01T09:23:13.566 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.567 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.567 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.567 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.567 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399470+0000", 2023-10-01T09:23:13.567 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:23:13.567 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.567 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.567 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.567 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399468+0000", 2023-10-01T09:23:13.567 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8000000000000001e-06 2023-10-01T09:23:13.567 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.568 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.568 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.568 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399475+0000", 2023-10-01T09:23:13.568 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6650000000000002e-05 2023-10-01T09:23:13.568 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.568 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.568 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.568 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399492+0000", 2023-10-01T09:23:13.568 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5230000000000001e-06 2023-10-01T09:23:13.568 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.568 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.569 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.569 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399498+0000", 2023-10-01T09:23:13.569 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1703999999999998e-05 2023-10-01T09:23:13.569 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.569 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.569 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.569 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399550+0000", 2023-10-01T09:23:13.569 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8299999999999998e-06 2023-10-01T09:23:13.569 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.569 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.569 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.569 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399557+0000", 2023-10-01T09:23:13.569 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78437442400000001 2023-10-01T09:23:13.570 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.570 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.570 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.570 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183931+0000", 2023-10-01T09:23:13.570 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4589999999999998e-06 2023-10-01T09:23:13.570 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.570 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.570 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.570 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183940+0000", 2023-10-01T09:23:13.570 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78447003999999998 2023-10-01T09:23:13.571 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.571 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.571 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.571 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.571 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.571 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:125447 6.7 6.c1000b47 (undecoded) ondisk+write+known_if_redirected e227)", 2023-10-01T09:23:13.571 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:01.419192+0000", 2023-10-01T09:23:13.571 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 72.130144393999998, 2023-10-01T09:23:13.571 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.73714969699999999, 2023-10-01T09:23:13.571 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.571 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:23:13.572 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.572 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.572 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.572 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 125447 2023-10-01T09:23:13.572 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.572 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.572 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.572 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.572 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419192+0000", 2023-10-01T09:23:13.572 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.572 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.572 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.573 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.573 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419192+0000", 2023-10-01T09:23:13.573 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:23:13.573 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.573 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.573 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.573 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419191+0000", 2023-10-01T09:23:13.573 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1951e-05 2023-10-01T09:23:13.573 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.573 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.573 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.573 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419203+0000", 2023-10-01T09:23:13.574 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8039000000000001e-05 2023-10-01T09:23:13.574 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.574 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.574 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.574 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419221+0000", 2023-10-01T09:23:13.574 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6539999999999999e-06 2023-10-01T09:23:13.574 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.574 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.574 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.574 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419229+0000", 2023-10-01T09:23:13.575 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2092999999999997e-05 2023-10-01T09:23:13.575 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.575 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.575 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.575 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419281+0000", 2023-10-01T09:23:13.575 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.994e-06 2023-10-01T09:23:13.575 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.575 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.575 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.575 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419289+0000", 2023-10-01T09:23:13.575 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.73703525000000003 2023-10-01T09:23:13.575 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.576 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.576 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.576 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.156324+0000", 2023-10-01T09:23:13.576 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8236e-05 2023-10-01T09:23:13.576 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.576 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.576 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.576 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.156342+0000", 2023-10-01T09:23:13.576 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.73714969699999999 2023-10-01T09:23:13.576 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.576 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.577 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.577 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.577 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.577 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:125450 6.7 6.376de267 (undecoded) ondisk+write+known_if_redirected e227)", 2023-10-01T09:23:13.577 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:01.421544+0000", 2023-10-01T09:23:13.577 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 72.127792784999997, 2023-10-01T09:23:13.577 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.73483475099999995, 2023-10-01T09:23:13.577 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.577 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:23:13.577 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.577 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.577 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.577 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 125450 2023-10-01T09:23:13.578 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.578 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.578 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.578 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.578 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421544+0000", 2023-10-01T09:23:13.578 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.578 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.578 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.578 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.578 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421544+0000", 2023-10-01T09:23:13.578 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:23:13.578 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.579 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.579 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.579 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421543+0000", 2023-10-01T09:23:13.579 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4030000000000007e-06 2023-10-01T09:23:13.579 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.579 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.579 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.579 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421551+0000", 2023-10-01T09:23:13.579 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7561999999999999e-05 2023-10-01T09:23:13.579 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.579 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.580 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.580 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421569+0000", 2023-10-01T09:23:13.580 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2339999999999997e-06 2023-10-01T09:23:13.580 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.580 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.580 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.580 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421576+0000", 2023-10-01T09:23:13.580 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7411999999999999e-05 2023-10-01T09:23:13.580 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.580 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.580 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.580 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421623+0000", 2023-10-01T09:23:13.581 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3320000000000003e-06 2023-10-01T09:23:13.581 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.581 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.581 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.581 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421631+0000", 2023-10-01T09:23:13.581 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.734737784 2023-10-01T09:23:13.581 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.581 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.581 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.581 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.156369+0000", 2023-10-01T09:23:13.581 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0196e-05 2023-10-01T09:23:13.581 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.582 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.582 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.582 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.156379+0000", 2023-10-01T09:23:13.582 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.73483475099999995 2023-10-01T09:23:13.582 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.582 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.582 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.582 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.582 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.582 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166458 6.7 6:e2130551:::benchmark_data_smithi138_307751_object71089:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.582 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.189221+0000", 2023-10-01T09:23:13.582 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 21.360116024, 2023-10-01T09:23:13.583 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.07166782, 2023-10-01T09:23:13.583 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.583 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.583 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.583 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.583 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.583 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166458 2023-10-01T09:23:13.583 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.583 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.583 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.583 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.583 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189221+0000", 2023-10-01T09:23:13.584 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.584 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.584 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.584 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.584 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189221+0000", 2023-10-01T09:23:13.584 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:23:13.584 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.584 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.584 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.584 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189219+0000", 2023-10-01T09:23:13.584 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5189999999999992e-06 2023-10-01T09:23:13.584 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.585 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.585 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.585 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189228+0000", 2023-10-01T09:23:13.585 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6232e-05 2023-10-01T09:23:13.585 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.585 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.585 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.585 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189244+0000", 2023-10-01T09:23:13.585 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3189999999999997e-06 2023-10-01T09:23:13.585 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.585 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.585 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.586 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189251+0000", 2023-10-01T09:23:13.586 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0047771130000000004 2023-10-01T09:23:13.586 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.586 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.586 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.586 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.194029+0000", 2023-10-01T09:23:13.586 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1163e-05 2023-10-01T09:23:13.586 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.586 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.586 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.586 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.194040+0000", 2023-10-01T09:23:13.586 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0557915819999999 2023-10-01T09:23:13.587 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.587 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.587 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.587 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.249831+0000", 2023-10-01T09:23:13.587 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010584799999999999 2023-10-01T09:23:13.587 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.587 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.587 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.587 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.249937+0000", 2023-10-01T09:23:13.587 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6229000000000003e-05 2023-10-01T09:23:13.587 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.587 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.588 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.588 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.250033+0000", 2023-10-01T09:23:13.588 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049686640000000002 2023-10-01T09:23:13.588 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.588 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.588 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.588 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.255002+0000", 2023-10-01T09:23:13.588 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001218654 2023-10-01T09:23:13.588 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.588 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.588 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.588 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.256221+0000", 2023-10-01T09:23:13.589 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045970969999999996 2023-10-01T09:23:13.589 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.589 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.589 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.589 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260818+0000", 2023-10-01T09:23:13.589 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9246999999999999e-05 2023-10-01T09:23:13.589 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.589 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.589 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.589 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260857+0000", 2023-10-01T09:23:13.589 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1477999999999999e-05 2023-10-01T09:23:13.589 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.590 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.590 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.590 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260889+0000", 2023-10-01T09:23:13.590 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.07166782 2023-10-01T09:23:13.590 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.590 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.590 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.590 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.590 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.590 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166487 6.7 6:e01c315f:::benchmark_data_smithi138_307751_object71118:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.590 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.213314+0000", 2023-10-01T09:23:13.590 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 21.336022741000001, 2023-10-01T09:23:13.591 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0525543909999999, 2023-10-01T09:23:13.591 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.591 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.591 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.591 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.591 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.591 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166487 2023-10-01T09:23:13.591 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.591 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.591 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.591 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.592 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213314+0000", 2023-10-01T09:23:13.592 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.592 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.592 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.592 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.592 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213314+0000", 2023-10-01T09:23:13.592 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:23:13.592 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.592 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.592 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.592 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213312+0000", 2023-10-01T09:23:13.593 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5779999999999999e-06 2023-10-01T09:23:13.593 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.593 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.593 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.593 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213321+0000", 2023-10-01T09:23:13.593 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5784e-05 2023-10-01T09:23:13.593 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.593 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.593 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.593 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213337+0000", 2023-10-01T09:23:13.593 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2225000000000002e-05 2023-10-01T09:23:13.593 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.594 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.594 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.594 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213359+0000", 2023-10-01T09:23:13.594 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000137766 2023-10-01T09:23:13.594 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.594 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.594 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.594 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213497+0000", 2023-10-01T09:23:13.594 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2595999999999999e-05 2023-10-01T09:23:13.594 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.595 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.595 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.595 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213509+0000", 2023-10-01T09:23:13.595 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0382323870000001 2023-10-01T09:23:13.595 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.595 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.595 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.595 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.251742+0000", 2023-10-01T09:23:13.595 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0978999999999998e-05 2023-10-01T09:23:13.595 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.595 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.595 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.596 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.251823+0000", 2023-10-01T09:23:13.596 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7830999999999995e-05 2023-10-01T09:23:13.596 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.596 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.596 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.596 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.251890+0000", 2023-10-01T09:23:13.596 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0056844479999999999 2023-10-01T09:23:13.596 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.596 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.596 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.597 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257575+0000", 2023-10-01T09:23:13.597 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003355055 2023-10-01T09:23:13.597 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.597 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.597 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.597 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260930+0000", 2023-10-01T09:23:13.597 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0048781969999999999 2023-10-01T09:23:13.597 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.598 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.598 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.598 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265808+0000", 2023-10-01T09:23:13.598 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2873000000000002e-05 2023-10-01T09:23:13.598 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.598 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.598 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.598 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265841+0000", 2023-10-01T09:23:13.598 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7410999999999999e-05 2023-10-01T09:23:13.598 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.598 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.599 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.599 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265868+0000", 2023-10-01T09:23:13.599 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0525543909999999 2023-10-01T09:23:13.599 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.599 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.599 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.599 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.599 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.599 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166491 6.7 6:e6d8849d:::benchmark_data_smithi138_307751_object71122:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.599 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.219132+0000", 2023-10-01T09:23:13.599 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 21.330204334000001, 2023-10-01T09:23:13.599 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0468224770000001, 2023-10-01T09:23:13.600 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.600 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.600 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.600 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.600 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.600 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166491 2023-10-01T09:23:13.600 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.600 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.600 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.600 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.600 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219132+0000", 2023-10-01T09:23:13.601 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.601 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.601 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.601 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.601 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219132+0000", 2023-10-01T09:23:13.601 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:23:13.601 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.601 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.601 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.601 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219131+0000", 2023-10-01T09:23:13.601 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1768e-05 2023-10-01T09:23:13.602 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.602 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.602 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.602 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219143+0000", 2023-10-01T09:23:13.602 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2629000000000001e-05 2023-10-01T09:23:13.602 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.602 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.602 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.602 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219155+0000", 2023-10-01T09:23:13.602 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.942e-06 2023-10-01T09:23:13.602 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.602 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.603 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.603 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219161+0000", 2023-10-01T09:23:13.603 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5195000000000003e-05 2023-10-01T09:23:13.603 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.603 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.603 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.603 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219216+0000", 2023-10-01T09:23:13.603 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0199e-05 2023-10-01T09:23:13.603 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.603 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.603 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.603 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219226+0000", 2023-10-01T09:23:13.604 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.033648492 2023-10-01T09:23:13.604 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.604 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.604 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.604 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.252875+0000", 2023-10-01T09:23:13.604 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9799999999999997e-05 2023-10-01T09:23:13.604 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.604 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.604 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.604 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.252935+0000", 2023-10-01T09:23:13.605 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5607999999999999e-05 2023-10-01T09:23:13.605 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.605 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.605 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.605 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.253000+0000", 2023-10-01T09:23:13.605 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00458422 2023-10-01T09:23:13.605 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.605 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.605 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.605 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257585+0000", 2023-10-01T09:23:13.605 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033770720000000001 2023-10-01T09:23:13.606 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.606 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.606 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.606 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260962+0000", 2023-10-01T09:23:13.606 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049325590000000004 2023-10-01T09:23:13.606 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.606 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.606 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.606 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265894+0000", 2023-10-01T09:23:13.606 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6339e-05 2023-10-01T09:23:13.606 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.607 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.607 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.607 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265921+0000", 2023-10-01T09:23:13.607 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4357999999999999e-05 2023-10-01T09:23:13.607 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.607 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.607 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.607 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265955+0000", 2023-10-01T09:23:13.607 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0468224770000001 2023-10-01T09:23:13.607 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.607 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.607 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.608 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.608 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.608 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166496 6.7 6:e4e4b310:::benchmark_data_smithi138_307751_object71127:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.608 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.230525+0000", 2023-10-01T09:23:13.608 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 21.318811652000001, 2023-10-01T09:23:13.608 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0393795619999999, 2023-10-01T09:23:13.608 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.608 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.608 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.608 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.608 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.608 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166496 2023-10-01T09:23:13.609 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.609 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.609 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.609 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.609 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230525+0000", 2023-10-01T09:23:13.609 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.609 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.609 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.609 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.609 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230525+0000", 2023-10-01T09:23:13.609 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:23:13.609 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.609 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.610 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.610 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230524+0000", 2023-10-01T09:23:13.610 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8950000000000003e-06 2023-10-01T09:23:13.610 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.610 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.610 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.610 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230530+0000", 2023-10-01T09:23:13.610 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2658e-05 2023-10-01T09:23:13.610 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.610 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.610 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.611 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230543+0000", 2023-10-01T09:23:13.611 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2159999999999997e-06 2023-10-01T09:23:13.611 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.611 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.611 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.611 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230548+0000", 2023-10-01T09:23:13.611 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9517e-05 2023-10-01T09:23:13.611 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.611 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.611 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.612 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230597+0000", 2023-10-01T09:23:13.612 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1009000000000001e-05 2023-10-01T09:23:13.612 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.612 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.612 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.612 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230608+0000", 2023-10-01T09:23:13.612 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.023332213 2023-10-01T09:23:13.612 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.612 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.612 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.612 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.253940+0000", 2023-10-01T09:23:13.613 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0046000000000001e-05 2023-10-01T09:23:13.613 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.613 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.613 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.613 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.254001+0000", 2023-10-01T09:23:13.613 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6650999999999996e-05 2023-10-01T09:23:13.613 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.613 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.613 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.613 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.254067+0000", 2023-10-01T09:23:13.613 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035276270000000002 2023-10-01T09:23:13.614 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.614 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.614 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.614 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257595+0000", 2023-10-01T09:23:13.614 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0084019709999999994 2023-10-01T09:23:13.614 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.614 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.614 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.614 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265997+0000", 2023-10-01T09:23:13.614 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038161779999999999 2023-10-01T09:23:13.614 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.615 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.615 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.615 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.269813+0000", 2023-10-01T09:23:13.615 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0507999999999998e-05 2023-10-01T09:23:13.615 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.615 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.615 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.615 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.269863+0000", 2023-10-01T09:23:13.615 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1207e-05 2023-10-01T09:23:13.615 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.615 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.615 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.616 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.269905+0000", 2023-10-01T09:23:13.616 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0393795619999999 2023-10-01T09:23:13.616 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.616 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.616 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.616 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.616 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.616 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166506 6.7 6:e0267cc2:::benchmark_data_smithi138_307751_object71137:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.616 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.230586+0000", 2023-10-01T09:23:13.616 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 21.318750689000002, 2023-10-01T09:23:13.616 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.039509367, 2023-10-01T09:23:13.616 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.617 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.617 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.617 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.617 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.617 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166506 2023-10-01T09:23:13.617 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.617 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.617 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.617 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.617 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230586+0000", 2023-10-01T09:23:13.617 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.617 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.617 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.618 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.618 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230586+0000", 2023-10-01T09:23:13.618 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:23:13.618 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.618 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.618 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.618 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230586+0000", 2023-10-01T09:23:13.618 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7310000000000002e-06 2023-10-01T09:23:13.618 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.618 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.618 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.618 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230588+0000", 2023-10-01T09:23:13.619 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6399999999999997e-06 2023-10-01T09:23:13.619 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.619 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.619 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.619 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230596+0000", 2023-10-01T09:23:13.619 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9950000000000001e-06 2023-10-01T09:23:13.619 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.619 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.619 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.619 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230599+0000", 2023-10-01T09:23:13.619 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4839000000000003e-05 2023-10-01T09:23:13.620 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.620 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.620 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.620 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230634+0000", 2023-10-01T09:23:13.620 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6569999999999997e-06 2023-10-01T09:23:13.620 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.620 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.620 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.620 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230641+0000", 2023-10-01T09:23:13.620 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0243796119999999 2023-10-01T09:23:13.620 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.621 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.621 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.621 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.255021+0000", 2023-10-01T09:23:13.621 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5718999999999999e-05 2023-10-01T09:23:13.621 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.621 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.621 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.621 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.255087+0000", 2023-10-01T09:23:13.621 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1915999999999994e-05 2023-10-01T09:23:13.621 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.621 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.622 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.622 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.255159+0000", 2023-10-01T09:23:13.622 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0056119830000000001 2023-10-01T09:23:13.622 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.622 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.622 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.622 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260771+0000", 2023-10-01T09:23:13.622 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0052694889999999996 2023-10-01T09:23:13.622 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.622 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.623 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.623 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.266040+0000", 2023-10-01T09:23:13.623 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003962826 2023-10-01T09:23:13.623 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.623 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.623 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.623 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270003+0000", 2023-10-01T09:23:13.623 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2723999999999999e-05 2023-10-01T09:23:13.623 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.623 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.623 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.624 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270056+0000", 2023-10-01T09:23:13.624 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9711e-05 2023-10-01T09:23:13.624 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.624 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.624 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.624 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270095+0000", 2023-10-01T09:23:13.624 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.039509367 2023-10-01T09:23:13.624 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.624 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.624 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.624 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.625 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.625 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166513 6.7 6:e69cb40c:::benchmark_data_smithi138_307751_object71144:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.625 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.247029+0000", 2023-10-01T09:23:13.625 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 21.302308064000002, 2023-10-01T09:23:13.625 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0288884, 2023-10-01T09:23:13.625 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.625 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.625 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.625 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.625 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.625 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166513 2023-10-01T09:23:13.625 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.626 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.626 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.626 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.626 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247029+0000", 2023-10-01T09:23:13.626 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.626 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.626 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.626 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.626 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247029+0000", 2023-10-01T09:23:13.626 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:23:13.626 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.627 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.627 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.627 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247027+0000", 2023-10-01T09:23:13.627 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0600000000000002e-06 2023-10-01T09:23:13.627 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.627 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.627 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.627 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247034+0000", 2023-10-01T09:23:13.627 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2286000000000001e-05 2023-10-01T09:23:13.627 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.627 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.628 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.628 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247046+0000", 2023-10-01T09:23:13.628 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7830000000000001e-06 2023-10-01T09:23:13.628 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.628 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.628 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.628 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247053+0000", 2023-10-01T09:23:13.628 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7210000000000003e-05 2023-10-01T09:23:13.628 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.628 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.628 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.628 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247110+0000", 2023-10-01T09:23:13.629 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1158999999999999e-05 2023-10-01T09:23:13.629 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.629 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.629 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.629 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247122+0000", 2023-10-01T09:23:13.629 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.009139901 2023-10-01T09:23:13.629 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.629 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.629 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.629 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.256262+0000", 2023-10-01T09:23:13.629 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6122e-05 2023-10-01T09:23:13.629 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.630 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.630 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.630 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.256348+0000", 2023-10-01T09:23:13.630 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6933000000000004e-05 2023-10-01T09:23:13.630 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.630 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.630 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.630 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.256425+0000", 2023-10-01T09:23:13.630 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045112299999999998 2023-10-01T09:23:13.630 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.630 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.630 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.631 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260936+0000", 2023-10-01T09:23:13.631 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0090185890000000005 2023-10-01T09:23:13.631 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.631 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.631 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.631 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.269954+0000", 2023-10-01T09:23:13.631 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0058642859999999998 2023-10-01T09:23:13.631 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.631 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.631 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.631 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.275819+0000", 2023-10-01T09:23:13.631 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7589999999999997e-05 2023-10-01T09:23:13.632 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.632 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.632 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.632 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.275866+0000", 2023-10-01T09:23:13.632 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0816999999999998e-05 2023-10-01T09:23:13.632 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.632 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.632 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.632 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.275917+0000", 2023-10-01T09:23:13.632 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0288884 2023-10-01T09:23:13.632 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.632 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.633 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.633 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.633 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.633 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166528 6.7 6:e73602e0:::benchmark_data_smithi138_307751_object71159:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.633 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.261428+0000", 2023-10-01T09:23:13.633 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 21.28790833, 2023-10-01T09:23:13.633 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.014679273, 2023-10-01T09:23:13.634 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.634 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.634 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.634 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.635 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.635 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166528 2023-10-01T09:23:13.635 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.635 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.635 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.635 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.635 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261428+0000", 2023-10-01T09:23:13.635 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.635 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.635 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.636 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.636 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261428+0000", 2023-10-01T09:23:13.636 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:23:13.636 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.636 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.636 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.636 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261426+0000", 2023-10-01T09:23:13.636 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3349999999999996e-06 2023-10-01T09:23:13.636 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.636 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.637 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.637 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261436+0000", 2023-10-01T09:23:13.637 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4049000000000001e-05 2023-10-01T09:23:13.637 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.637 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.637 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.637 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261450+0000", 2023-10-01T09:23:13.637 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5750000000000003e-06 2023-10-01T09:23:13.637 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.637 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.637 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.638 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261457+0000", 2023-10-01T09:23:13.638 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8458999999999997e-05 2023-10-01T09:23:13.638 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.638 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.638 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.638 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261516+0000", 2023-10-01T09:23:13.638 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1588e-05 2023-10-01T09:23:13.638 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.638 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.638 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.638 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261527+0000", 2023-10-01T09:23:13.639 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99610873700000002 2023-10-01T09:23:13.639 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.639 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.639 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.639 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257636+0000", 2023-10-01T09:23:13.639 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000114474 2023-10-01T09:23:13.639 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.639 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.639 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.639 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257751+0000", 2023-10-01T09:23:13.639 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014375799999999999 2023-10-01T09:23:13.639 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.640 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.640 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.640 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257894+0000", 2023-10-01T09:23:13.640 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0030463740000000001 2023-10-01T09:23:13.640 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.640 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.640 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.640 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260941+0000", 2023-10-01T09:23:13.640 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.009213377 2023-10-01T09:23:13.640 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.640 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.641 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.641 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270154+0000", 2023-10-01T09:23:13.641 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0058627649999999998 2023-10-01T09:23:13.641 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.641 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.641 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.641 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.276017+0000", 2023-10-01T09:23:13.641 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0591999999999999e-05 2023-10-01T09:23:13.641 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.641 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.641 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.641 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.276067+0000", 2023-10-01T09:23:13.642 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0309999999999999e-05 2023-10-01T09:23:13.642 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.642 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.642 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.642 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.276108+0000", 2023-10-01T09:23:13.642 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.014679273 2023-10-01T09:23:13.642 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.642 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.642 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.642 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.642 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.643 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166545 6.7 6:e60e3404:::benchmark_data_smithi138_307751_object71176:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.643 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.284403+0000", 2023-10-01T09:23:13.643 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 21.264933797000001, 2023-10-01T09:23:13.643 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99383394000000003, 2023-10-01T09:23:13.643 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.643 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.643 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.643 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.643 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.643 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166545 2023-10-01T09:23:13.643 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.643 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.643 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.644 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.644 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284403+0000", 2023-10-01T09:23:13.644 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.644 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.644 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.644 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.644 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284403+0000", 2023-10-01T09:23:13.644 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:23:13.644 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.644 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.644 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.645 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284401+0000", 2023-10-01T09:23:13.645 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0525e-05 2023-10-01T09:23:13.645 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.645 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.645 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.645 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284411+0000", 2023-10-01T09:23:13.645 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8612999999999999e-05 2023-10-01T09:23:13.645 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.645 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.645 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.645 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284430+0000", 2023-10-01T09:23:13.645 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7730000000000002e-06 2023-10-01T09:23:13.645 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.646 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.646 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.646 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284436+0000", 2023-10-01T09:23:13.646 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4496000000000001e-05 2023-10-01T09:23:13.646 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.646 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.646 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.646 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284491+0000", 2023-10-01T09:23:13.646 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1708999999999999e-05 2023-10-01T09:23:13.646 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.646 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.647 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.647 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284503+0000", 2023-10-01T09:23:13.647 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97494659800000005 2023-10-01T09:23:13.647 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.647 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.647 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.647 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.259449+0000", 2023-10-01T09:23:13.647 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000104481 2023-10-01T09:23:13.647 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.647 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.647 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.648 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.259554+0000", 2023-10-01T09:23:13.648 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2804000000000002e-05 2023-10-01T09:23:13.648 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.648 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.648 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.648 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.259637+0000", 2023-10-01T09:23:13.648 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002740433 2023-10-01T09:23:13.648 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.648 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.648 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.648 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.262377+0000", 2023-10-01T09:23:13.648 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078248339999999993 2023-10-01T09:23:13.649 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.649 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.649 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.649 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270202+0000", 2023-10-01T09:23:13.649 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0079470840000000001 2023-10-01T09:23:13.649 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.649 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.649 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.649 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278149+0000", 2023-10-01T09:23:13.649 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8479999999999997e-05 2023-10-01T09:23:13.649 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.649 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.650 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.650 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278197+0000", 2023-10-01T09:23:13.650 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9505000000000002e-05 2023-10-01T09:23:13.650 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.650 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.650 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.650 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278237+0000", 2023-10-01T09:23:13.650 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99383394000000003 2023-10-01T09:23:13.650 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.650 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.650 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.651 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.651 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.651 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166600 6.7 6:e23e27f5:::benchmark_data_smithi138_307751_object71231:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.651 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.346015+0000", 2023-10-01T09:23:13.651 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 21.203321505000002, 2023-10-01T09:23:13.651 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.93235052399999996, 2023-10-01T09:23:13.651 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.651 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.651 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.651 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.651 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.651 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166600 2023-10-01T09:23:13.652 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.652 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.652 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.652 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.652 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346015+0000", 2023-10-01T09:23:13.652 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.652 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.652 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.652 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.652 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346015+0000", 2023-10-01T09:23:13.652 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:23:13.652 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.653 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.653 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.653 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346014+0000", 2023-10-01T09:23:13.653 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1130000000000001e-06 2023-10-01T09:23:13.653 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.653 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.653 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.653 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346022+0000", 2023-10-01T09:23:13.653 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4234e-05 2023-10-01T09:23:13.653 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.653 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.654 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.654 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346036+0000", 2023-10-01T09:23:13.654 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9259999999999997e-06 2023-10-01T09:23:13.654 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.654 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.654 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.654 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346042+0000", 2023-10-01T09:23:13.654 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9398999999999998e-05 2023-10-01T09:23:13.654 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.654 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.655 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.655 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346102+0000", 2023-10-01T09:23:13.655 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.202e-05 2023-10-01T09:23:13.655 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.655 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.655 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.655 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346114+0000", 2023-10-01T09:23:13.655 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.91486791099999998 2023-10-01T09:23:13.655 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.655 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.656 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.656 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260982+0000", 2023-10-01T09:23:13.656 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3937999999999997e-05 2023-10-01T09:23:13.656 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.656 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.656 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.656 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.261045+0000", 2023-10-01T09:23:13.656 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9540000000000001e-05 2023-10-01T09:23:13.656 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.656 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.656 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.656 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.261125+0000", 2023-10-01T09:23:13.657 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0046514310000000001 2023-10-01T09:23:13.657 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.657 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.657 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.657 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265776+0000", 2023-10-01T09:23:13.657 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0077202750000000004 2023-10-01T09:23:13.657 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.657 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.657 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.658 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.273497+0000", 2023-10-01T09:23:13.658 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0047860460000000004 2023-10-01T09:23:13.658 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.658 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.658 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.658 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278283+0000", 2023-10-01T09:23:13.658 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6359000000000002e-05 2023-10-01T09:23:13.658 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.658 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.658 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.659 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278319+0000", 2023-10-01T09:23:13.659 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6647000000000002e-05 2023-10-01T09:23:13.659 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.659 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.659 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.660 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278366+0000", 2023-10-01T09:23:13.660 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.93235052399999996 2023-10-01T09:23:13.661 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.661 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.661 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.661 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.661 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.661 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166608 6.7 6:e3e1df92:::benchmark_data_smithi138_307751_object71239:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.661 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.493260+0000", 2023-10-01T09:23:13.661 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 21.056076342000001, 2023-10-01T09:23:13.661 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78524043099999996, 2023-10-01T09:23:13.662 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.662 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.662 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.662 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.662 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.662 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166608 2023-10-01T09:23:13.662 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.662 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.662 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.662 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.663 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493260+0000", 2023-10-01T09:23:13.663 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.663 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.663 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.663 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.663 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493260+0000", 2023-10-01T09:23:13.663 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:23:13.663 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.663 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.663 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.663 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493260+0000", 2023-10-01T09:23:13.663 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6130000000000003e-06 2023-10-01T09:23:13.664 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.664 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.664 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.664 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493265+0000", 2023-10-01T09:23:13.664 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1015e-05 2023-10-01T09:23:13.664 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.664 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.664 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.664 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493276+0000", 2023-10-01T09:23:13.664 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5259999999999999e-06 2023-10-01T09:23:13.664 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.664 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.665 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.665 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493283+0000", 2023-10-01T09:23:13.665 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9543999999999997e-05 2023-10-01T09:23:13.665 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.665 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.665 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.665 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493352+0000", 2023-10-01T09:23:13.665 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.118e-05 2023-10-01T09:23:13.665 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.665 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.665 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.665 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493364+0000", 2023-10-01T09:23:13.666 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.76903849099999999 2023-10-01T09:23:13.666 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.666 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.666 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.666 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.262402+0000", 2023-10-01T09:23:13.666 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7131000000000002e-05 2023-10-01T09:23:13.666 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.666 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.666 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.666 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.262499+0000", 2023-10-01T09:23:13.666 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000120406 2023-10-01T09:23:13.667 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.667 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.667 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.667 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.262620+0000", 2023-10-01T09:23:13.667 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0032544620000000001 2023-10-01T09:23:13.667 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.667 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.667 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.667 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265874+0000", 2023-10-01T09:23:13.667 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010088079 2023-10-01T09:23:13.667 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.667 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.668 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.668 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.275962+0000", 2023-10-01T09:23:13.668 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024570550000000001 2023-10-01T09:23:13.668 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.668 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.668 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.668 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278419+0000", 2023-10-01T09:23:13.668 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5637999999999999e-05 2023-10-01T09:23:13.668 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.668 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.668 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.668 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278465+0000", 2023-10-01T09:23:13.669 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6021e-05 2023-10-01T09:23:13.669 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.669 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.669 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.669 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278501+0000", 2023-10-01T09:23:13.669 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78524043099999996 2023-10-01T09:23:13.669 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.669 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.669 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.670 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.670 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.670 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166718 6.7 6:e4a2c62d:::benchmark_data_smithi138_307751_object71349:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.670 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.665722+0000", 2023-10-01T09:23:13.670 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 20.883615224, 2023-10-01T09:23:13.670 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.61779461199999997, 2023-10-01T09:23:13.670 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.670 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.670 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.670 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.670 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.671 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166718 2023-10-01T09:23:13.671 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.671 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.671 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.671 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.671 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665722+0000", 2023-10-01T09:23:13.671 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.671 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.671 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.671 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.671 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665722+0000", 2023-10-01T09:23:13.671 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:23:13.672 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.672 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.672 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.672 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665720+0000", 2023-10-01T09:23:13.672 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5480000000000002e-06 2023-10-01T09:23:13.672 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.672 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.672 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.672 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665728+0000", 2023-10-01T09:23:13.672 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6147999999999999e-05 2023-10-01T09:23:13.672 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.672 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.673 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.673 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665745+0000", 2023-10-01T09:23:13.673 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2089999999999999e-06 2023-10-01T09:23:13.673 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.673 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.673 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.673 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665752+0000", 2023-10-01T09:23:13.673 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4418999999999994e-05 2023-10-01T09:23:13.673 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.673 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.673 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.673 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665816+0000", 2023-10-01T09:23:13.674 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0353999999999999e-05 2023-10-01T09:23:13.674 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.674 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.674 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:23:13.675 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.675 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665827+0000", 2023-10-01T09:23:13.675 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.59826122199999998 2023-10-01T09:23:13.675 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.676 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.676 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.676 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.264088+0000", 2023-10-01T09:23:13.676 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0003e-05 2023-10-01T09:23:13.676 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.676 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.676 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.676 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.264158+0000", 2023-10-01T09:23:13.676 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7170999999999996e-05 2023-10-01T09:23:13.676 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.676 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.677 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.677 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.264245+0000", 2023-10-01T09:23:13.677 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001758414 2023-10-01T09:23:13.677 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.677 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.677 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.677 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.266003+0000", 2023-10-01T09:23:13.677 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016782042 2023-10-01T09:23:13.677 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.677 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.677 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.678 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.282785+0000", 2023-10-01T09:23:13.678 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00063452500000000004 2023-10-01T09:23:13.678 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.678 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.678 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.678 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.283420+0000", 2023-10-01T09:23:13.678 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0348e-05 2023-10-01T09:23:13.678 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.678 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.678 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.678 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.283470+0000", 2023-10-01T09:23:13.678 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5849000000000001e-05 2023-10-01T09:23:13.679 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.679 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.679 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.679 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.283516+0000", 2023-10-01T09:23:13.679 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.61779461199999997 2023-10-01T09:23:13.679 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.679 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.679 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.679 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.679 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.679 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166770 6.7 6:e5b59087:::benchmark_data_smithi138_307751_object71401:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.680 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.753490+0000", 2023-10-01T09:23:13.680 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 20.795846597000001, 2023-10-01T09:23:13.680 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.53017308900000004, 2023-10-01T09:23:13.680 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.680 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.680 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.680 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.680 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.680 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166770 2023-10-01T09:23:13.680 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.680 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.680 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.681 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.681 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753490+0000", 2023-10-01T09:23:13.681 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.681 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.681 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.681 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.681 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753490+0000", 2023-10-01T09:23:13.681 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:23:13.681 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.681 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.681 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.681 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753489+0000", 2023-10-01T09:23:13.682 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9090000000000003e-06 2023-10-01T09:23:13.682 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.682 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.682 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.682 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753496+0000", 2023-10-01T09:23:13.682 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3482000000000001e-05 2023-10-01T09:23:13.682 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.682 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.682 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.682 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753509+0000", 2023-10-01T09:23:13.682 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4609999999999999e-06 2023-10-01T09:23:13.682 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.683 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.683 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.683 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753515+0000", 2023-10-01T09:23:13.683 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8496999999999999e-05 2023-10-01T09:23:13.683 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.683 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.683 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.683 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753573+0000", 2023-10-01T09:23:13.683 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0601999999999999e-05 2023-10-01T09:23:13.683 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.683 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.684 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.684 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753584+0000", 2023-10-01T09:23:13.684 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.51248056099999995 2023-10-01T09:23:13.684 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.684 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.684 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.684 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.266064+0000", 2023-10-01T09:23:13.684 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000108893 2023-10-01T09:23:13.684 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.684 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.684 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.684 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.266173+0000", 2023-10-01T09:23:13.685 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000125909 2023-10-01T09:23:13.685 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.685 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.685 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.685 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.266299+0000", 2023-10-01T09:23:13.685 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003810857 2023-10-01T09:23:13.685 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.685 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.685 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.685 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270110+0000", 2023-10-01T09:23:13.685 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012703811000000001 2023-10-01T09:23:13.686 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.686 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.686 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.686 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.282814+0000", 2023-10-01T09:23:13.686 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00075106700000000001 2023-10-01T09:23:13.686 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.686 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.686 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.686 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.283565+0000", 2023-10-01T09:23:13.686 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6557000000000001e-05 2023-10-01T09:23:13.686 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.687 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.687 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.687 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.283621+0000", 2023-10-01T09:23:13.687 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1903000000000002e-05 2023-10-01T09:23:13.687 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.687 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.687 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.687 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.283663+0000", 2023-10-01T09:23:13.687 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.53017308900000004 2023-10-01T09:23:13.687 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.687 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.687 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.687 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.688 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.688 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166817 6.7 6:e32b8a7a:::benchmark_data_smithi138_307751_object71448:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.688 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.827383+0000", 2023-10-01T09:23:13.688 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 20.721953861999999, 2023-10-01T09:23:13.688 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.45803607899999998, 2023-10-01T09:23:13.688 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.688 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.688 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.688 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.688 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.688 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166817 2023-10-01T09:23:13.688 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.689 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.689 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.689 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.689 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827383+0000", 2023-10-01T09:23:13.689 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.689 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.689 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.689 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.689 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827383+0000", 2023-10-01T09:23:13.689 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:23:13.689 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.689 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.690 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.690 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827381+0000", 2023-10-01T09:23:13.690 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0366999999999999e-05 2023-10-01T09:23:13.690 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.690 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.690 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.690 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827391+0000", 2023-10-01T09:23:13.690 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5305999999999999e-05 2023-10-01T09:23:13.690 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.690 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.690 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.691 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827406+0000", 2023-10-01T09:23:13.691 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6420000000000007e-06 2023-10-01T09:23:13.691 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.691 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.691 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.691 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827414+0000", 2023-10-01T09:23:13.691 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1822999999999994e-05 2023-10-01T09:23:13.691 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.691 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.691 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.691 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827476+0000", 2023-10-01T09:23:13.691 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0706e-05 2023-10-01T09:23:13.692 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.692 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.692 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.692 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827487+0000", 2023-10-01T09:23:13.692 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.44055778499999998 2023-10-01T09:23:13.692 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.692 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.692 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.692 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.268044+0000", 2023-10-01T09:23:13.692 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4426999999999997e-05 2023-10-01T09:23:13.692 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.692 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.693 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.693 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.268129+0000", 2023-10-01T09:23:13.693 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7678999999999999e-05 2023-10-01T09:23:13.693 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.693 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.693 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.693 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.268217+0000", 2023-10-01T09:23:13.693 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019031390000000001 2023-10-01T09:23:13.693 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.693 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.693 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.694 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270120+0000", 2023-10-01T09:23:13.694 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014529134000000001 2023-10-01T09:23:13.694 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.694 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.694 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.694 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.284649+0000", 2023-10-01T09:23:13.694 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00067139599999999999 2023-10-01T09:23:13.694 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.694 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.694 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.694 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285320+0000", 2023-10-01T09:23:13.694 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0686000000000002e-05 2023-10-01T09:23:13.695 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.695 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.695 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.695 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285371+0000", 2023-10-01T09:23:13.695 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8123e-05 2023-10-01T09:23:13.695 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.695 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.695 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.695 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285419+0000", 2023-10-01T09:23:13.695 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.45803607899999998 2023-10-01T09:23:13.695 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.695 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.696 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.696 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.696 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.696 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166820 6.7 6:e587feea:::benchmark_data_smithi138_307751_object71451:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.696 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.832585+0000", 2023-10-01T09:23:13.696 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 20.716751691999999, 2023-10-01T09:23:13.696 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.45297539100000001, 2023-10-01T09:23:13.696 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.696 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.696 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.696 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.697 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.697 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166820 2023-10-01T09:23:13.697 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.697 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.697 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.697 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.697 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832585+0000", 2023-10-01T09:23:13.697 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.697 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.697 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.697 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.697 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832585+0000", 2023-10-01T09:23:13.698 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:23:13.698 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.698 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.698 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.698 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832583+0000", 2023-10-01T09:23:13.698 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1138000000000001e-05 2023-10-01T09:23:13.698 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.698 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.698 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.698 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832594+0000", 2023-10-01T09:23:13.698 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3779e-05 2023-10-01T09:23:13.699 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.699 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.699 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.699 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832608+0000", 2023-10-01T09:23:13.699 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7260000000000002e-06 2023-10-01T09:23:13.699 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.699 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.699 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.699 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832616+0000", 2023-10-01T09:23:13.699 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8177000000000002e-05 2023-10-01T09:23:13.699 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.699 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.700 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.700 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832674+0000", 2023-10-01T09:23:13.700 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2645e-05 2023-10-01T09:23:13.700 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.700 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.700 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.700 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832687+0000", 2023-10-01T09:23:13.700 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.43753456800000001 2023-10-01T09:23:13.700 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.700 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.700 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.700 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270221+0000", 2023-10-01T09:23:13.701 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4250000000000002e-05 2023-10-01T09:23:13.701 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.701 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.701 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.701 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270295+0000", 2023-10-01T09:23:13.701 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4099999999999998e-05 2023-10-01T09:23:13.701 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.701 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.701 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.701 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270380+0000", 2023-10-01T09:23:13.701 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0031290200000000002 2023-10-01T09:23:13.702 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.702 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.702 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.702 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.273509+0000", 2023-10-01T09:23:13.702 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011158936 2023-10-01T09:23:13.702 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.702 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.702 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.702 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.284667+0000", 2023-10-01T09:23:13.702 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00081128500000000002 2023-10-01T09:23:13.702 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.703 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.703 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.703 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285479+0000", 2023-10-01T09:23:13.703 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6978999999999999e-05 2023-10-01T09:23:13.703 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.703 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.703 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.703 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285526+0000", 2023-10-01T09:23:13.703 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4718000000000002e-05 2023-10-01T09:23:13.703 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.703 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.704 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.704 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285560+0000", 2023-10-01T09:23:13.704 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.45297539100000001 2023-10-01T09:23:13.704 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.704 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.704 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.704 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.704 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.704 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166823 6.7 6:e67d4cd1:::benchmark_data_smithi138_307751_object71454:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:23:13.704 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.838568+0000", 2023-10-01T09:23:13.704 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 20.710768993999999, 2023-10-01T09:23:13.705 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.44710524600000001, 2023-10-01T09:23:13.705 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:23:13.705 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:23:13.705 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:23:13.705 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:23:13.705 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:23:13.705 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166823 2023-10-01T09:23:13.705 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.705 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:23:13.705 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.705 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:23:13.705 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838568+0000", 2023-10-01T09:23:13.706 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:23:13.706 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.706 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.706 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:23:13.706 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838568+0000", 2023-10-01T09:23:13.706 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:23:13.706 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.706 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.706 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:23:13.706 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838566+0000", 2023-10-01T09:23:13.706 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1300000000000001e-06 2023-10-01T09:23:13.706 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.707 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.707 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:23:13.707 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838574+0000", 2023-10-01T09:23:13.707 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6716000000000001e-05 2023-10-01T09:23:13.707 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.707 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.707 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:23:13.707 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838591+0000", 2023-10-01T09:23:13.707 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8700000000000003e-06 2023-10-01T09:23:13.707 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.707 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.708 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:23:13.708 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838597+0000", 2023-10-01T09:23:13.708 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9017000000000002e-05 2023-10-01T09:23:13.708 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.708 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.708 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.708 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838647+0000", 2023-10-01T09:23:13.708 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.986e-06 2023-10-01T09:23:13.708 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.708 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.708 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:23:13.708 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838655+0000", 2023-10-01T09:23:13.709 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.43301372500000002 2023-10-01T09:23:13.709 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.709 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.709 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:23:13.709 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.271669+0000", 2023-10-01T09:23:13.709 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8678999999999998e-05 2023-10-01T09:23:13.709 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.709 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.709 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:23:13.709 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.271738+0000", 2023-10-01T09:23:13.709 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9672999999999995e-05 2023-10-01T09:23:13.709 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.710 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.710 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:23:13.710 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.271808+0000", 2023-10-01T09:23:13.710 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003960547 2023-10-01T09:23:13.710 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.710 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.710 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:23:13.710 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.275768+0000", 2023-10-01T09:23:13.710 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0089158359999999999 2023-10-01T09:23:13.710 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.710 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.710 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.711 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.284684+0000", 2023-10-01T09:23:13.711 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000918366 2023-10-01T09:23:13.711 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.711 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.711 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:23:13.711 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285602+0000", 2023-10-01T09:23:13.711 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0027999999999997e-05 2023-10-01T09:23:13.711 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.711 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.711 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:23:13.711 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285642+0000", 2023-10-01T09:23:13.712 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0675e-05 2023-10-01T09:23:13.712 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:23:13.712 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:23:13.712 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:23:13.712 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285673+0000", 2023-10-01T09:23:13.712 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.44710524600000001 2023-10-01T09:23:13.712 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.712 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.712 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.712 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:23:13.712 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:23:13.712 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:23:13.799 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:13.884 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:23:13.891 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:13 smithi190 ceph-mon[160435]: failed to encode map e249 with expected crc 2023-10-01T09:23:13.891 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:13 smithi190 ceph-mon[160435]: osdmap e250: 8 total, 8 up, 8 in 2023-10-01T09:23:13.891 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:13 smithi190 ceph-mon[160435]: failed to encode map e250 with expected crc 2023-10-01T09:23:13.892 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:13 smithi190 ceph-mon[160435]: failed to encode map e250 with expected crc 2023-10-01T09:23:13.892 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:13 smithi190 ceph-mon[160435]: failed to encode map e250 with expected crc 2023-10-01T09:23:13.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:13.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:23:13.995 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:14.040 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:13 smithi138 ceph-mon[192513]: failed to encode map e249 with expected crc 2023-10-01T09:23:14.040 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:13 smithi138 ceph-mon[192513]: osdmap e250: 8 total, 8 up, 8 in 2023-10-01T09:23:14.040 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:13 smithi138 ceph-mon[192513]: failed to encode map e250 with expected crc 2023-10-01T09:23:14.040 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:13 smithi138 ceph-mon[192513]: failed to encode map e250 with expected crc 2023-10-01T09:23:14.040 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:13 smithi138 ceph-mon[192513]: failed to encode map e250 with expected crc 2023-10-01T09:23:14.040 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:13 smithi138 ceph-mon[185886]: failed to encode map e249 with expected crc 2023-10-01T09:23:14.041 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:13 smithi138 ceph-mon[185886]: osdmap e250: 8 total, 8 up, 8 in 2023-10-01T09:23:14.041 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:13 smithi138 ceph-mon[185886]: failed to encode map e250 with expected crc 2023-10-01T09:23:14.041 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:13 smithi138 ceph-mon[185886]: failed to encode map e250 with expected crc 2023-10-01T09:23:14.041 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:13 smithi138 ceph-mon[185886]: failed to encode map e250 with expected crc 2023-10-01T09:23:14.042 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:14.436 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:14.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:14.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:14.448 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:23:14.542 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:14.579 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:14.635 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:14.651 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:14.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:14.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:14.759 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:14.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:14 smithi190 ceph-mon[160435]: failed to encode map e250 with expected crc 2023-10-01T09:23:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:14 smithi190 ceph-mon[160435]: pgmap v627: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 3 peering, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 83 active+clean; 758 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 15073/36234 objects degraded (41.599%); 6657/36234 objects misplaced (18.372%); 9.6 MiB/s, 174 objects/s recovering 2023-10-01T09:23:14.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:14 smithi138 ceph-mon[185886]: failed to encode map e250 with expected crc 2023-10-01T09:23:14.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:14 smithi138 ceph-mon[185886]: pgmap v627: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 3 peering, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 83 active+clean; 758 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 15073/36234 objects degraded (41.599%); 6657/36234 objects misplaced (18.372%); 9.6 MiB/s, 174 objects/s recovering 2023-10-01T09:23:14.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:14 smithi138 ceph-mon[192513]: failed to encode map e250 with expected crc 2023-10-01T09:23:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:14 smithi138 ceph-mon[192513]: pgmap v627: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 3 peering, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 83 active+clean; 758 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 15073/36234 objects degraded (41.599%); 6657/36234 objects misplaced (18.372%); 9.6 MiB/s, 174 objects/s recovering 2023-10-01T09:23:14.824 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:14.983 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:15.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:15.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:15.074 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:23:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:23:14] "GET /metrics HTTP/1.1" 200 36415 "" "Prometheus/2.33.4" 2023-10-01T09:23:15.101 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:23:15.153 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:15.380 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:23:15.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:15.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:23:15.496 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:15.534 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:23:15.581 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:23:15.611 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:15.653 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:23:15.712 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:23:15.748 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:23:15.771 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:15.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:15.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:15.781 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:23:15.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:15 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1889171160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:15 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1889171160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:15.828 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:23:15.862 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:23:15.880 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:15.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:23:15.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:23:15.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:23:15.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:23:15.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:23:15.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:23:15.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:23:15.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:23:15.940 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:16.022 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:16.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:16.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:15 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1889171160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:16.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:16.190 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:16.665 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:23:17.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:16 smithi190 ceph-mon[160435]: pgmap v628: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 688 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 20912/32910 objects degraded (63.543%); 6406/32910 objects misplaced (19.465%); 9.4 MiB/s, 169 objects/s recovering 2023-10-01T09:23:17.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:16 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3943388817' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:23:17.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:16 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:23:17.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:16 smithi138 ceph-mon[185886]: pgmap v628: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 688 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 20912/32910 objects degraded (63.543%); 6406/32910 objects misplaced (19.465%); 9.4 MiB/s, 169 objects/s recovering 2023-10-01T09:23:17.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:16 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3943388817' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:23:17.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:16 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:23:17.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:16 smithi138 ceph-mon[192513]: pgmap v628: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 688 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 20912/32910 objects degraded (63.543%); 6406/32910 objects misplaced (19.465%); 9.4 MiB/s, 169 objects/s recovering 2023-10-01T09:23:17.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:16 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3943388817' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:23:17.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:16 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:23:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:23:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[192513]: failed to encode map e251 with expected crc 2023-10-01T09:23:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:23:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[192513]: osdmap e251: 8 total, 8 up, 8 in 2023-10-01T09:23:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[192513]: failed to encode map e251 with expected crc 2023-10-01T09:23:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[192513]: failed to encode map e251 with expected crc 2023-10-01T09:23:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[192513]: failed to encode map e251 with expected crc 2023-10-01T09:23:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 20912/32910 objects degraded (63.543%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[192513]: pgmap v630: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 580 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 20719/27714 objects degraded (74.760%); 5485/27714 objects misplaced (19.791%); 15 MiB/s, 329 objects/s recovering 2023-10-01T09:23:17.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:23:17.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[185886]: failed to encode map e251 with expected crc 2023-10-01T09:23:17.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:23:17.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[185886]: osdmap e251: 8 total, 8 up, 8 in 2023-10-01T09:23:17.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[185886]: failed to encode map e251 with expected crc 2023-10-01T09:23:17.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[185886]: failed to encode map e251 with expected crc 2023-10-01T09:23:17.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[185886]: failed to encode map e251 with expected crc 2023-10-01T09:23:17.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 20912/32910 objects degraded (63.543%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:17.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:17 smithi138 ceph-mon[185886]: pgmap v630: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 580 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 20719/27714 objects degraded (74.760%); 5485/27714 objects misplaced (19.791%); 15 MiB/s, 329 objects/s recovering 2023-10-01T09:23:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:17 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:23:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:17 smithi190 ceph-mon[160435]: failed to encode map e251 with expected crc 2023-10-01T09:23:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:17 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:23:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:17 smithi190 ceph-mon[160435]: osdmap e251: 8 total, 8 up, 8 in 2023-10-01T09:23:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:17 smithi190 ceph-mon[160435]: failed to encode map e251 with expected crc 2023-10-01T09:23:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:17 smithi190 ceph-mon[160435]: failed to encode map e251 with expected crc 2023-10-01T09:23:18.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:17 smithi190 ceph-mon[160435]: failed to encode map e251 with expected crc 2023-10-01T09:23:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:17 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 20912/32910 objects degraded (63.543%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:18.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:17 smithi190 ceph-mon[160435]: pgmap v630: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 580 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 20719/27714 objects degraded (74.760%); 5485/27714 objects misplaced (19.791%); 15 MiB/s, 329 objects/s recovering 2023-10-01T09:23:18.977 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:18.982 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:18 smithi138 ceph-mon[185886]: osdmap e252: 8 total, 8 up, 8 in 2023-10-01T09:23:18.982 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:18 smithi138 ceph-mon[185886]: failed to encode map e252 with expected crc 2023-10-01T09:23:18.982 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:18 smithi138 ceph-mon[185886]: failed to encode map e252 with expected crc 2023-10-01T09:23:18.982 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:18 smithi138 ceph-mon[185886]: failed to encode map e252 with expected crc 2023-10-01T09:23:18.982 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:18 smithi138 ceph-mon[185886]: failed to encode map e252 with expected crc 2023-10-01T09:23:18.983 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:18 smithi138 ceph-mon[192513]: osdmap e252: 8 total, 8 up, 8 in 2023-10-01T09:23:18.983 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:18 smithi138 ceph-mon[192513]: failed to encode map e252 with expected crc 2023-10-01T09:23:18.983 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:18 smithi138 ceph-mon[192513]: failed to encode map e252 with expected crc 2023-10-01T09:23:18.983 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:18 smithi138 ceph-mon[192513]: failed to encode map e252 with expected crc 2023-10-01T09:23:18.983 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:18 smithi138 ceph-mon[192513]: failed to encode map e252 with expected crc 2023-10-01T09:23:18.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:18.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:19.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:18 smithi190 ceph-mon[160435]: osdmap e252: 8 total, 8 up, 8 in 2023-10-01T09:23:19.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:18 smithi190 ceph-mon[160435]: failed to encode map e252 with expected crc 2023-10-01T09:23:19.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:18 smithi190 ceph-mon[160435]: failed to encode map e252 with expected crc 2023-10-01T09:23:19.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:18 smithi190 ceph-mon[160435]: failed to encode map e252 with expected crc 2023-10-01T09:23:19.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:18 smithi190 ceph-mon[160435]: failed to encode map e252 with expected crc 2023-10-01T09:23:19.085 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:23:19.159 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:19.271 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:23:19.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:19.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:23:19.308 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:19.379 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:23:19.491 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:19.740 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:23:19.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:19.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:23:19.847 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:19.909 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:19.952 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:19 smithi138 ceph-mon[192513]: pgmap v632: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 539 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 20494/25746 objects degraded (79.601%); 5258/25746 objects misplaced (20.423%); 12 MiB/s, 264 objects/s recovering 2023-10-01T09:23:19.953 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:19 smithi138 ceph-mon[185886]: pgmap v632: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 539 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 20494/25746 objects degraded (79.601%); 5258/25746 objects misplaced (20.423%); 12 MiB/s, 264 objects/s recovering 2023-10-01T09:23:20.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:19 smithi190 ceph-mon[160435]: pgmap v632: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 539 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 20494/25746 objects degraded (79.601%); 5258/25746 objects misplaced (20.423%); 12 MiB/s, 264 objects/s recovering 2023-10-01T09:23:20.151 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:20.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:20.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:20.262 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:20.351 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:20.475 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:20.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:20.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:20.588 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:23:20.671 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:20.714 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.2 to 1 (10000) 2023-10-01T09:23:20.802 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:23:20.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:20.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:23:20.914 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:20.938 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:20 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:23:20.938 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:20 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2994486540' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:23:20.938 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:20 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:23:20.940 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:20.991 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:20.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:20.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:21.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:20 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:23:21.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:20 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2994486540' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:23:21.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:20 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:23:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:20 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:23:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:20 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2994486540' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:23:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:20 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:23:21.095 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:21.139 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:21.434 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:23:21.522 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:23:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:21 smithi138 ceph-mon[192513]: failed to encode map e253 with expected crc 2023-10-01T09:23:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:21 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:23:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:21 smithi138 ceph-mon[192513]: osdmap e253: 8 total, 8 up, 8 in 2023-10-01T09:23:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:21 smithi138 ceph-mon[192513]: failed to encode map e253 with expected crc 2023-10-01T09:23:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:21 smithi138 ceph-mon[192513]: failed to encode map e253 with expected crc 2023-10-01T09:23:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:21 smithi138 ceph-mon[192513]: pgmap v634: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 477 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 20313/22779 objects degraded (89.174%); 4916/22779 objects misplaced (21.581%); 13 MiB/s, 283 objects/s recovering 2023-10-01T09:23:21.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:21 smithi138 ceph-mon[185886]: failed to encode map e253 with expected crc 2023-10-01T09:23:21.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:21 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:23:21.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:21 smithi138 ceph-mon[185886]: osdmap e253: 8 total, 8 up, 8 in 2023-10-01T09:23:21.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:21 smithi138 ceph-mon[185886]: failed to encode map e253 with expected crc 2023-10-01T09:23:21.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:21 smithi138 ceph-mon[185886]: failed to encode map e253 with expected crc 2023-10-01T09:23:21.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:21 smithi138 ceph-mon[185886]: pgmap v634: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 477 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 20313/22779 objects degraded (89.174%); 4916/22779 objects misplaced (21.581%); 13 MiB/s, 283 objects/s recovering 2023-10-01T09:23:21.926 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:21.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:21.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:22.031 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:23:22.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:21 smithi190 ceph-mon[160435]: failed to encode map e253 with expected crc 2023-10-01T09:23:22.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:21 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:23:22.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:21 smithi190 ceph-mon[160435]: osdmap e253: 8 total, 8 up, 8 in 2023-10-01T09:23:22.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:21 smithi190 ceph-mon[160435]: failed to encode map e253 with expected crc 2023-10-01T09:23:22.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:21 smithi190 ceph-mon[160435]: failed to encode map e253 with expected crc 2023-10-01T09:23:22.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:21 smithi190 ceph-mon[160435]: pgmap v634: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 peering, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 84 active+clean; 477 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 20313/22779 objects degraded (89.174%); 4916/22779 objects misplaced (21.581%); 13 MiB/s, 283 objects/s recovering 2023-10-01T09:23:22.112 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:22.205 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:23:22.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:22.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:23:22.240 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-01T09:23:22.317 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:23:22.366 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:22.443 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:23:22.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:22.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:23:22.551 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:23:22.653 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:22.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:22 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 20313/22779 objects degraded (89.174%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:22.770 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:23:22.773 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:22.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:22.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:23:22.871 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:23:22.871 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:23:22.883 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:22.922 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:22.955 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:23.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:22 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 20313/22779 objects degraded (89.174%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:23.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:22 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 20313/22779 objects degraded (89.174%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:23.188 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:23.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:23.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:23.301 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:23:23.394 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:23.487 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:23:23.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:23.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:23:23.602 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:23.659 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:23 smithi190 ceph-mon[160435]: pgmap v635: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 384 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 20313/18303 objects degraded (110.982%); 4916/18303 objects misplaced (26.859%); 9.5 MiB/s, 295 objects/s recovering 2023-10-01T09:23:24.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:23 smithi138 ceph-mon[192513]: pgmap v635: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 384 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 20313/18303 objects degraded (110.982%); 4916/18303 objects misplaced (26.859%); 9.5 MiB/s, 295 objects/s recovering 2023-10-01T09:23:24.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:23 smithi138 ceph-mon[185886]: pgmap v635: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 384 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 20313/18303 objects degraded (110.982%); 4916/18303 objects misplaced (26.859%); 9.5 MiB/s, 295 objects/s recovering 2023-10-01T09:23:24.089 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:24.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:24.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:24.200 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:23:24.224 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:23:24.291 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:24.545 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:23:24.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:24.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:23:24.663 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:24.717 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:24.791 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:24.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:24.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:24.899 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:24.953 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:25.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:23:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:23:24] "GET /metrics HTTP/1.1" 200 36415 "" "Prometheus/2.33.4" 2023-10-01T09:23:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:23:26.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:25 smithi190 ceph-mon[160435]: pgmap v636: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 279 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 20080/13239 objects degraded (151.673%); 4480/13239 objects misplaced (33.839%); 5.0 MiB/s, 143 objects/s recovering 2023-10-01T09:23:26.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:26.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:26.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:23:26.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:25 smithi138 ceph-mon[192513]: pgmap v636: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 279 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 20080/13239 objects degraded (151.673%); 4480/13239 objects misplaced (33.839%); 5.0 MiB/s, 143 objects/s recovering 2023-10-01T09:23:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:23:26.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:25 smithi138 ceph-mon[185886]: pgmap v636: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 279 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 20080/13239 objects degraded (151.673%); 4480/13239 objects misplaced (33.839%); 5.0 MiB/s, 143 objects/s recovering 2023-10-01T09:23:26.405 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:26.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:26.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:26.514 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:26.530 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:26.608 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:26.945 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:26.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:26.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:27.067 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:23:27.070 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:27.116 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:23:27.162 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:23:27.173 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:27.202 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:23:27.247 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:23:27.282 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:23:27.304 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:27 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3457489086' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:27.306 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:27 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3457489086' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:27.320 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:23:27.365 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:23:27.376 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:23:27.376 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:23:27.376 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:23:27.376 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:23:27.376 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:23:27.376 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:23:27.377 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:23:27.377 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:23:27.441 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:27.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:27.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:27.551 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:23:27.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:27 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3457489086' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:27.638 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:28.266 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:23:28.361 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:28 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 20080/13239 objects degraded (151.673%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:28.361 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:28 smithi138 ceph-mon[192513]: pgmap v637: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 156 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 19861/7359 objects degraded (269.887%); 4260/7359 objects misplaced (57.888%); 5.5 MiB/s, 178 objects/s recovering 2023-10-01T09:23:28.361 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2674801479' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:23:28.361 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:28 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 20080/13239 objects degraded (151.673%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:28.361 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:28 smithi138 ceph-mon[185886]: pgmap v637: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 156 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 19861/7359 objects degraded (269.887%); 4260/7359 objects misplaced (57.888%); 5.5 MiB/s, 178 objects/s recovering 2023-10-01T09:23:28.361 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2674801479' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:23:28.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:28 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 20080/13239 objects degraded (151.673%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:28.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:28 smithi190 ceph-mon[160435]: pgmap v637: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 85 active+clean; 156 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 19861/7359 objects degraded (269.887%); 4260/7359 objects misplaced (57.888%); 5.5 MiB/s, 178 objects/s recovering 2023-10-01T09:23:28.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2674801479' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:23:29.056 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:23:29.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:29.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:23:29.164 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:29.180 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Removed 95366 objects 2023-10-01T09:23:29.181 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Clean up completed and total clean up time :134.176 2023-10-01T09:23:29.194 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:29.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:23:29.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2674801479' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:23:29.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[192513]: osdmap e254: 8 total, 8 up, 8 in 2023-10-01T09:23:29.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[192513]: failed to encode map e254 with expected crc 2023-10-01T09:23:29.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[192513]: failed to encode map e254 with expected crc 2023-10-01T09:23:29.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[192513]: failed to encode map e254 with expected crc 2023-10-01T09:23:29.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[192513]: failed to encode map e254 with expected crc 2023-10-01T09:23:29.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:23:29.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:23:29.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2674801479' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:23:29.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[185886]: osdmap e254: 8 total, 8 up, 8 in 2023-10-01T09:23:29.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[185886]: failed to encode map e254 with expected crc 2023-10-01T09:23:29.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[185886]: failed to encode map e254 with expected crc 2023-10-01T09:23:29.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[185886]: failed to encode map e254 with expected crc 2023-10-01T09:23:29.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[185886]: failed to encode map e254 with expected crc 2023-10-01T09:23:29.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:29 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:23:29.420 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:29.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:29 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:23:29.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:29 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2674801479' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:23:29.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:29 smithi190 ceph-mon[160435]: osdmap e254: 8 total, 8 up, 8 in 2023-10-01T09:23:29.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:29 smithi190 ceph-mon[160435]: failed to encode map e254 with expected crc 2023-10-01T09:23:29.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:29 smithi190 ceph-mon[160435]: failed to encode map e254 with expected crc 2023-10-01T09:23:29.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:29 smithi190 ceph-mon[160435]: failed to encode map e254 with expected crc 2023-10-01T09:23:29.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:29 smithi190 ceph-mon[160435]: failed to encode map e254 with expected crc 2023-10-01T09:23:29.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:29 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:23:29.653 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-10-01T09:23:29.653 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-10-01T09:23:30.045 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:30.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:30.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:30.148 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:30.176 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:30.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:30 smithi138 ceph-mon[192513]: pgmap v639: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean; 121 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 19728/5679 objects degraded (347.385%); 4057/5679 objects misplaced (71.439%); 5.9 MiB/s, 197 objects/s recovering 2023-10-01T09:23:30.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:30 smithi138 ceph-mon[185886]: pgmap v639: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean; 121 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 19728/5679 objects degraded (347.385%); 4057/5679 objects misplaced (71.439%); 5.9 MiB/s, 197 objects/s recovering 2023-10-01T09:23:30.375 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:30.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:30.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:30.481 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:23:30.508 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:30 smithi190 ceph-mon[160435]: pgmap v639: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean; 121 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 19728/5679 objects degraded (347.385%); 4057/5679 objects misplaced (71.439%); 5.9 MiB/s, 197 objects/s recovering 2023-10-01T09:23:30.511 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:30.556 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:23:30.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:30.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:23:30.661 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:30.693 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:30.715 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:30.799 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:30.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:30.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:30.902 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:30.928 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:30.973 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:30.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:30.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:31.078 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:31.110 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:31.213 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:31.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:31.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:31.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:31 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:23:31.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:31 smithi190 ceph-mon[160435]: 6.9 scrub starts 2023-10-01T09:23:31.317 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:31.343 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:31.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:31 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:23:31.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:31 smithi138 ceph-mon[192513]: 6.9 scrub starts 2023-10-01T09:23:31.345 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:31 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:23:31.345 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:31 smithi138 ceph-mon[185886]: 6.9 scrub starts 2023-10-01T09:23:31.387 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:31.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:31.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:31.449 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:23:31.492 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:31.518 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:31.560 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:31.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:31.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:31.669 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:23:31.695 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:31.739 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:23:31.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:31.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:23:31.844 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:23:31.870 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:31.912 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:23:31.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:31.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:23:32.017 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:32.044 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:32.093 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:23:32.138 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.6 to 1 (10000) 2023-10-01T09:23:32.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[192513]: osdmap e255: 8 total, 8 up, 8 in 2023-10-01T09:23:32.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[192513]: failed to encode map e255 with expected crc 2023-10-01T09:23:32.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[192513]: failed to encode map e255 with expected crc 2023-10-01T09:23:32.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[192513]: failed to encode map e255 with expected crc 2023-10-01T09:23:32.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2031282556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:23:32.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:23:32.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[192513]: pgmap v641: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 67 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 19596/3102 objects degraded (631.721%); 3735/3102 objects misplaced (120.406%); 6.1 MiB/s, 146 objects/s recovering 2023-10-01T09:23:32.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[192513]: 6.0 scrub starts 2023-10-01T09:23:32.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[185886]: osdmap e255: 8 total, 8 up, 8 in 2023-10-01T09:23:32.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[185886]: failed to encode map e255 with expected crc 2023-10-01T09:23:32.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[185886]: failed to encode map e255 with expected crc 2023-10-01T09:23:32.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[185886]: failed to encode map e255 with expected crc 2023-10-01T09:23:32.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2031282556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:23:32.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:23:32.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[185886]: pgmap v641: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 67 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 19596/3102 objects degraded (631.721%); 3735/3102 objects misplaced (120.406%); 6.1 MiB/s, 146 objects/s recovering 2023-10-01T09:23:32.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:32 smithi138 ceph-mon[185886]: 6.0 scrub starts 2023-10-01T09:23:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:32 smithi190 ceph-mon[160435]: osdmap e255: 8 total, 8 up, 8 in 2023-10-01T09:23:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:32 smithi190 ceph-mon[160435]: failed to encode map e255 with expected crc 2023-10-01T09:23:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:32 smithi190 ceph-mon[160435]: failed to encode map e255 with expected crc 2023-10-01T09:23:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:32 smithi190 ceph-mon[160435]: failed to encode map e255 with expected crc 2023-10-01T09:23:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:32 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2031282556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:23:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:32 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:23:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:32 smithi190 ceph-mon[160435]: pgmap v641: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 67 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 19596/3102 objects degraded (631.721%); 3735/3102 objects misplaced (120.406%); 6.1 MiB/s, 146 objects/s recovering 2023-10-01T09:23:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:32 smithi190 ceph-mon[160435]: 6.0 scrub starts 2023-10-01T09:23:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:33 smithi190 ceph-mon[160435]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:33 smithi190 ceph-mon[160435]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:33 smithi190 ceph-mon[160435]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:33 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:23:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:33 smithi190 ceph-mon[160435]: osdmap e256: 8 total, 8 up, 8 in 2023-10-01T09:23:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:33 smithi190 ceph-mon[160435]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:33 smithi190 ceph-mon[160435]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:33 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 19596/3102 objects degraded (631.721%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[192513]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[192513]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[192513]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:23:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[192513]: osdmap e256: 8 total, 8 up, 8 in 2023-10-01T09:23:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[192513]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[192513]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 19596/3102 objects degraded (631.721%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:33.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[185886]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[185886]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[185886]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:23:33.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[185886]: osdmap e256: 8 total, 8 up, 8 in 2023-10-01T09:23:33.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[185886]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[185886]: failed to encode map e256 with expected crc 2023-10-01T09:23:33.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:33 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 19596/3102 objects degraded (631.721%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:23:33.849 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:33.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:33.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:33.880 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-01T09:23:33.881 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:23:33.955 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:33.981 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:34.007 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:34.023 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:34.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:34.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:34.130 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:34.163 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:34.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:34 smithi190 ceph-mon[160435]: pgmap v643: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 35 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 379 op/s; 19594/1545 objects degraded (1268.220%); 3735/1545 objects misplaced (241.748%); 3.5 MiB/s, 55 objects/s recovering 2023-10-01T09:23:34.367 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:34.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:34.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:34.470 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:34.508 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:34 smithi138 ceph-mon[185886]: pgmap v643: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 35 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 379 op/s; 19594/1545 objects degraded (1268.220%); 3735/1545 objects misplaced (241.748%); 3.5 MiB/s, 55 objects/s recovering 2023-10-01T09:23:34.509 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:34 smithi138 ceph-mon[192513]: pgmap v643: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 35 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 379 op/s; 19594/1545 objects degraded (1268.220%); 3735/1545 objects misplaced (241.748%); 3.5 MiB/s, 55 objects/s recovering 2023-10-01T09:23:34.512 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:34.574 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:23:34.575 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:23:34.575 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:23:34.575 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:23:34.575 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:23:34.588 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:34.815 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:34.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:34.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:34.919 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:34.945 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:34.988 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:34.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:34.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:35.036 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:35.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:23:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:23:34] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.33.4" 2023-10-01T09:23:35.096 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:23:35.125 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:35.201 INFO:teuthology.orchestra.run.smithi138.stderr:pool 'unique_pool_1' removed 2023-10-01T09:23:35.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:35 smithi190 ceph-mon[160435]: 6.5 scrub starts 2023-10-01T09:23:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:35 smithi190 ceph-mon[160435]: 6.19 scrub starts 2023-10-01T09:23:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2757326986' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:23:35.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:35 smithi138 ceph-mon[192513]: 6.5 scrub starts 2023-10-01T09:23:35.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:35 smithi138 ceph-mon[192513]: 6.19 scrub starts 2023-10-01T09:23:35.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2757326986' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:23:35.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:35 smithi138 ceph-mon[185886]: 6.5 scrub starts 2023-10-01T09:23:35.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:35 smithi138 ceph-mon[185886]: 6.19 scrub starts 2023-10-01T09:23:35.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2757326986' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:23:36.482 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:36 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2757326986' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:23:36.482 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:36 smithi138 ceph-mon[192513]: osdmap e257: 8 total, 8 up, 8 in 2023-10-01T09:23:36.482 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:36 smithi138 ceph-mon[192513]: failed to encode map e257 with expected crc 2023-10-01T09:23:36.482 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:36 smithi138 ceph-mon[192513]: failed to encode map e257 with expected crc 2023-10-01T09:23:36.483 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:36 smithi138 ceph-mon[192513]: failed to encode map e257 with expected crc 2023-10-01T09:23:36.483 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:36 smithi138 ceph-mon[192513]: pgmap v645: 73 pgs: 1 peering, 1 active+recovery_wait+degraded, 71 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:23:36.483 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:36 smithi138 ceph-mon[192513]: 6.10 scrub starts 2023-10-01T09:23:36.483 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:36 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2757326986' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:23:36.483 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:36 smithi138 ceph-mon[185886]: osdmap e257: 8 total, 8 up, 8 in 2023-10-01T09:23:36.483 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:36 smithi138 ceph-mon[185886]: failed to encode map e257 with expected crc 2023-10-01T09:23:36.483 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:36 smithi138 ceph-mon[185886]: failed to encode map e257 with expected crc 2023-10-01T09:23:36.483 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:36 smithi138 ceph-mon[185886]: failed to encode map e257 with expected crc 2023-10-01T09:23:36.483 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:36 smithi138 ceph-mon[185886]: pgmap v645: 73 pgs: 1 peering, 1 active+recovery_wait+degraded, 71 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:23:36.483 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:36 smithi138 ceph-mon[185886]: 6.10 scrub starts 2023-10-01T09:23:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:36 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2757326986' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:23:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:36 smithi190 ceph-mon[160435]: osdmap e257: 8 total, 8 up, 8 in 2023-10-01T09:23:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:36 smithi190 ceph-mon[160435]: failed to encode map e257 with expected crc 2023-10-01T09:23:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:36 smithi190 ceph-mon[160435]: failed to encode map e257 with expected crc 2023-10-01T09:23:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:36 smithi190 ceph-mon[160435]: failed to encode map e257 with expected crc 2023-10-01T09:23:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:36 smithi190 ceph-mon[160435]: pgmap v645: 73 pgs: 1 peering, 1 active+recovery_wait+degraded, 71 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-10-01T09:23:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:36 smithi190 ceph-mon[160435]: 6.10 scrub starts 2023-10-01T09:23:37.290 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:37 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:23:37.290 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:37 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:23:37.291 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T09:23:37.292 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T09:23:37.292 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-10-01T09:23:37.292 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-10-01T09:23:37.530 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:23:37.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:37 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:23:38.214 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:38 smithi138 ceph-mon[185886]: pgmap v646: 73 pgs: 73 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-01T09:23:38.482 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:38 smithi138 ceph-mon[192513]: pgmap v646: 73 pgs: 73 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-01T09:23:38.511 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:23:38.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:38.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:23:38.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:38 smithi190 ceph-mon[160435]: pgmap v646: 73 pgs: 73 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-01T09:23:38.616 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:38.649 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:38.697 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:38.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:38.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:38.707 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:23:38.707 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:23:38.707 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:23:38.707 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:23:38.708 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:23:38.708 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:23:38.708 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:23:38.708 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:23:38.708 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:23:38.708 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:23:38.708 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:23:38.708 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:23:38.708 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:23:38.708 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:23:38.709 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:23:38.709 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:23:38.804 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:38.832 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:38.877 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:38.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:38.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:38.984 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:39.014 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:39.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:39 smithi190 ceph-mon[160435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 1 pg degraded) 2023-10-01T09:23:39.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:39 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/153778524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:39.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:39 smithi138 ceph-mon[192513]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 1 pg degraded) 2023-10-01T09:23:39.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:39 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/153778524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:39.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:39 smithi138 ceph-mon[185886]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 1 pg degraded) 2023-10-01T09:23:39.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:39 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/153778524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:39.675 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:39.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:39.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:39.779 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:39.809 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:40.008 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:40.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:40.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:40.016 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:40.112 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:40.144 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:40.192 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:40.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:40.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:40.287 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:40 smithi138 ceph-mon[185886]: pgmap v647: 73 pgs: 73 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-01T09:23:40.287 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:40.287 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:40.287 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:40 smithi138 ceph-mon[192513]: pgmap v647: 73 pgs: 73 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-01T09:23:40.288 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:40.288 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:40.301 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:40.330 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:40.363 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:40.377 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:40.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:40.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:40.483 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:40.515 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:40 smithi190 ceph-mon[160435]: pgmap v647: 73 pgs: 73 active+clean; 4.6 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2023-10-01T09:23:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:40.621 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:40.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:40.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:40.673 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:23:40.674 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:23:40.726 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:40.754 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:40.800 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:40.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:40.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:40.909 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:40.938 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:40.983 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:40.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:40.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:41.087 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:41.114 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:23:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:23:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:23:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:41 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1120766159' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:23:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:23:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:23:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:23:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:41 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1120766159' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:23:41.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:41.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:23:41.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:23:41.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:23:41.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:41.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:41 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1120766159' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:23:41.607 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:23:41.630 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:23:41.702 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:41.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:41.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:41.811 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:41.844 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:41.892 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:41.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:41.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:41.997 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:23:42.026 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:42.071 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:23:42.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:42.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:23:42.178 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:42.205 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:42.513 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:42 smithi138 ceph-mon[185886]: from='client.35373 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:23:42.514 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:42 smithi138 ceph-mon[185886]: pgmap v648: 73 pgs: 73 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-01T09:23:42.514 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:42 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:23:42.514 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:42 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:23:42.514 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:42 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1120766159' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:23:42.514 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:42 smithi138 ceph-mon[185886]: osdmap e258: 8 total, 8 up, 8 in 2023-10-01T09:23:42.514 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:42 smithi138 ceph-mon[192513]: from='client.35373 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:23:42.514 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:42 smithi138 ceph-mon[192513]: pgmap v648: 73 pgs: 73 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-01T09:23:42.514 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:42 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:23:42.514 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:42 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:23:42.515 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:42 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1120766159' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:23:42.515 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:42 smithi138 ceph-mon[192513]: osdmap e258: 8 total, 8 up, 8 in 2023-10-01T09:23:42.534 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":647,"stamp":"2023-10-01T09:23:39.491254+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6259,"num_read_kb":12659,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":186,"num_bytes_recovered":2373496,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":308,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3014620,"kb_used_data":605212,"kb_used_omap":38,"kb_used_meta":2409305,"kb_avail":746946596,"statfs":{"total":767960285184,"available":764873314304,"internally_reserved":0,"allocated":619737088,"data_stored":612209242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39221,"internal_metadata":2467129035},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":45,"apply_latency_ms":45,"commit_latency_ns":45000000,"apply_latency_ns":45000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-65929240,"num_objects":-1006,"num_object_clones":0,"num_object_copies":-3018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-19596,"num_objects_misplaced":-3735,"num_objects_unfound":0,"num_objects_dirty":-1006,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-189942,"num_write_kb":-6103425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-17362,"num_bytes_recovered":-875101688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001984"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":127,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136609+0000","last_change":"2023-10-01T09:19:30.091981+0000","last_active":"2023-10-01T09:23:32.136609+0000","last_peered":"2023-10-01T09:23:32.136609+0000","last_clean":"2023-10-01T09:23:32.136609+0000","last_became_active":"2023-10-01T09:19:30.091531+0000","last_became_peered":"2023-10-01T09:19:30.091531+0000","last_unstale":"2023-10-01T09:23:32.136609+0000","last_undegraded":"2023-10-01T09:23:32.136609+0000","last_fullsized":"2023-10-01T09:23:32.136609+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:28:13.442498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":31,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.176618+0000","last_change":"2023-10-01T09:22:40.311894+0000","last_active":"2023-10-01T09:23:32.176618+0000","last_peered":"2023-10-01T09:23:32.176618+0000","last_clean":"2023-10-01T09:23:32.176618+0000","last_became_active":"2023-10-01T09:22:40.311581+0000","last_became_peered":"2023-10-01T09:22:40.311581+0000","last_unstale":"2023-10-01T09:23:32.176618+0000","last_undegraded":"2023-10-01T09:23:32.176618+0000","last_fullsized":"2023-10-01T09:23:32.176618+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:52:27.649535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":203,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136461+0000","last_change":"2023-10-01T09:13:13.551596+0000","last_active":"2023-10-01T09:23:32.136461+0000","last_peered":"2023-10-01T09:23:32.136461+0000","last_clean":"2023-10-01T09:23:32.136461+0000","last_became_active":"2023-10-01T09:13:13.551300+0000","last_became_peered":"2023-10-01T09:13:13.551300+0000","last_unstale":"2023-10-01T09:23:32.136461+0000","last_undegraded":"2023-10-01T09:23:32.136461+0000","last_fullsized":"2023-10-01T09:23:32.136461+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:57:45.081780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":260,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130951+0000","last_change":"2023-10-01T09:23:10.611220+0000","last_active":"2023-10-01T09:23:32.130951+0000","last_peered":"2023-10-01T09:23:32.130951+0000","last_clean":"2023-10-01T09:23:32.130951+0000","last_became_active":"2023-10-01T09:23:10.610573+0000","last_became_peered":"2023-10-01T09:23:10.610573+0000","last_unstale":"2023-10-01T09:23:32.130951+0000","last_undegraded":"2023-10-01T09:23:32.130951+0000","last_fullsized":"2023-10-01T09:23:32.130951+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:58:32.020825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":31,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.176014+0000","last_change":"2023-10-01T09:22:40.172183+0000","last_active":"2023-10-01T09:23:32.176014+0000","last_peered":"2023-10-01T09:23:32.176014+0000","last_clean":"2023-10-01T09:23:32.176014+0000","last_became_active":"2023-10-01T09:22:40.170164+0000","last_became_peered":"2023-10-01T09:22:40.170164+0000","last_unstale":"2023-10-01T09:23:32.176014+0000","last_undegraded":"2023-10-01T09:23:32.176014+0000","last_fullsized":"2023-10-01T09:23:32.176014+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:33:10.407999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":203,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.138753+0000","last_change":"2023-10-01T09:22:03.153196+0000","last_active":"2023-10-01T09:23:32.138753+0000","last_peered":"2023-10-01T09:23:32.138753+0000","last_clean":"2023-10-01T09:23:32.138753+0000","last_became_active":"2023-10-01T09:22:03.151768+0000","last_became_peered":"2023-10-01T09:22:03.151768+0000","last_unstale":"2023-10-01T09:23:32.138753+0000","last_undegraded":"2023-10-01T09:23:32.138753+0000","last_fullsized":"2023-10-01T09:23:32.138753+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":262,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.131980+0000","last_change":"2023-10-01T09:22:40.175704+0000","last_active":"2023-10-01T09:23:32.131980+0000","last_peered":"2023-10-01T09:23:32.131980+0000","last_clean":"2023-10-01T09:23:32.131980+0000","last_became_active":"2023-10-01T09:22:40.175369+0000","last_became_peered":"2023-10-01T09:22:40.175369+0000","last_unstale":"2023-10-01T09:23:32.131980+0000","last_undegraded":"2023-10-01T09:23:32.131980+0000","last_fullsized":"2023-10-01T09:23:32.131980+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:07.773407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":247,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.176103+0000","last_change":"2023-10-01T09:21:02.139865+0000","last_active":"2023-10-01T09:23:32.176103+0000","last_peered":"2023-10-01T09:23:32.176103+0000","last_clean":"2023-10-01T09:23:32.176103+0000","last_became_active":"2023-10-01T09:21:02.139507+0000","last_became_peered":"2023-10-01T09:21:02.139507+0000","last_unstale":"2023-10-01T09:23:32.176103+0000","last_undegraded":"2023-10-01T09:23:32.176103+0000","last_fullsized":"2023-10-01T09:23:32.176103+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:46:09.214220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":219,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.134579+0000","last_change":"2023-10-01T09:23:10.610494+0000","last_active":"2023-10-01T09:23:32.134579+0000","last_peered":"2023-10-01T09:23:32.134579+0000","last_clean":"2023-10-01T09:23:32.134579+0000","last_became_active":"2023-10-01T09:23:10.610037+0000","last_became_peered":"2023-10-01T09:23:10.610037+0000","last_unstale":"2023-10-01T09:23:32.134579+0000","last_undegraded":"2023-10-01T09:23:32.134579+0000","last_fullsized":"2023-10-01T09:23:32.134579+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":273,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.176382+0000","last_change":"2023-10-01T09:23:10.613042+0000","last_active":"2023-10-01T09:23:32.176382+0000","last_peered":"2023-10-01T09:23:32.176382+0000","last_clean":"2023-10-01T09:23:32.176382+0000","last_became_active":"2023-10-01T09:23:10.612686+0000","last_became_peered":"2023-10-01T09:23:10.612686+0000","last_unstale":"2023-10-01T09:23:32.176382+0000","last_undegraded":"2023-10-01T09:23:32.176382+0000","last_fullsized":"2023-10-01T09:23:32.176382+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:12:00.239369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":26,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136170+0000","last_change":"2023-10-01T09:23:10.616550+0000","last_active":"2023-10-01T09:23:32.136170+0000","last_peered":"2023-10-01T09:23:32.136170+0000","last_clean":"2023-10-01T09:23:32.136170+0000","last_became_active":"2023-10-01T09:23:10.611509+0000","last_became_peered":"2023-10-01T09:23:10.611509+0000","last_unstale":"2023-10-01T09:23:32.136170+0000","last_undegraded":"2023-10-01T09:23:32.136170+0000","last_fullsized":"2023-10-01T09:23:32.136170+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":257,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136274+0000","last_change":"2023-10-01T09:21:02.158958+0000","last_active":"2023-10-01T09:23:32.136274+0000","last_peered":"2023-10-01T09:23:32.136274+0000","last_clean":"2023-10-01T09:23:32.136274+0000","last_became_active":"2023-10-01T09:21:02.158491+0000","last_became_peered":"2023-10-01T09:21:02.158491+0000","last_unstale":"2023-10-01T09:23:32.136274+0000","last_undegraded":"2023-10-01T09:23:32.136274+0000","last_fullsized":"2023-10-01T09:23:32.136274+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:21:48.885982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":202,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136212+0000","last_change":"2023-10-01T09:21:02.163403+0000","last_active":"2023-10-01T09:23:32.136212+0000","last_peered":"2023-10-01T09:23:32.136212+0000","last_clean":"2023-10-01T09:23:32.136212+0000","last_became_active":"2023-10-01T09:21:02.121680+0000","last_became_peered":"2023-10-01T09:21:02.121680+0000","last_unstale":"2023-10-01T09:23:32.136212+0000","last_undegraded":"2023-10-01T09:23:32.136212+0000","last_fullsized":"2023-10-01T09:23:32.136212+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":225,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.138395+0000","last_change":"2023-10-01T09:21:03.349475+0000","last_active":"2023-10-01T09:23:32.138395+0000","last_peered":"2023-10-01T09:23:32.138395+0000","last_clean":"2023-10-01T09:23:32.138395+0000","last_became_active":"2023-10-01T09:21:03.348251+0000","last_became_peered":"2023-10-01T09:21:03.348251+0000","last_unstale":"2023-10-01T09:23:32.138395+0000","last_undegraded":"2023-10-01T09:23:32.138395+0000","last_fullsized":"2023-10-01T09:23:32.138395+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":285,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136022+0000","last_change":"2023-10-01T09:19:30.182046+0000","last_active":"2023-10-01T09:23:32.136022+0000","last_peered":"2023-10-01T09:23:32.136022+0000","last_clean":"2023-10-01T09:23:32.136022+0000","last_became_active":"2023-10-01T09:19:30.094579+0000","last_became_peered":"2023-10-01T09:19:30.094579+0000","last_unstale":"2023-10-01T09:23:32.136022+0000","last_undegraded":"2023-10-01T09:23:32.136022+0000","last_fullsized":"2023-10-01T09:23:32.136022+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":238,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.176932+0000","last_change":"2023-10-01T09:19:57.493670+0000","last_active":"2023-10-01T09:23:32.176932+0000","last_peered":"2023-10-01T09:23:32.176932+0000","last_clean":"2023-10-01T09:23:32.176932+0000","last_became_active":"2023-10-01T09:19:57.493349+0000","last_became_peered":"2023-10-01T09:19:57.493349+0000","last_unstale":"2023-10-01T09:23:32.176932+0000","last_undegraded":"2023-10-01T09:23:32.176932+0000","last_fullsized":"2023-10-01T09:23:32.176932+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:26:56.468779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":244,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130735+0000","last_change":"2023-10-01T09:23:10.607388+0000","last_active":"2023-10-01T09:23:32.130735+0000","last_peered":"2023-10-01T09:23:32.130735+0000","last_clean":"2023-10-01T09:23:32.130735+0000","last_became_active":"2023-10-01T09:23:10.607084+0000","last_became_peered":"2023-10-01T09:23:10.607084+0000","last_unstale":"2023-10-01T09:23:32.130735+0000","last_undegraded":"2023-10-01T09:23:32.130735+0000","last_fullsized":"2023-10-01T09:23:32.130735+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:19.669393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":195,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.135006+0000","last_change":"2023-10-01T09:22:02.185127+0000","last_active":"2023-10-01T09:23:32.135006+0000","last_peered":"2023-10-01T09:23:32.135006+0000","last_clean":"2023-10-01T09:23:32.135006+0000","last_became_active":"2023-10-01T09:22:02.184059+0000","last_became_peered":"2023-10-01T09:22:02.184059+0000","last_unstale":"2023-10-01T09:23:32.135006+0000","last_undegraded":"2023-10-01T09:23:32.135006+0000","last_fullsized":"2023-10-01T09:23:32.135006+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":226,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.135928+0000","last_change":"2023-10-01T09:21:02.121558+0000","last_active":"2023-10-01T09:23:32.135928+0000","last_peered":"2023-10-01T09:23:32.135928+0000","last_clean":"2023-10-01T09:23:32.135928+0000","last_became_active":"2023-10-01T09:21:02.121153+0000","last_became_peered":"2023-10-01T09:21:02.121153+0000","last_unstale":"2023-10-01T09:23:32.135928+0000","last_undegraded":"2023-10-01T09:23:32.135928+0000","last_fullsized":"2023-10-01T09:23:32.135928+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":226,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130884+0000","last_change":"2023-10-01T09:23:10.611400+0000","last_active":"2023-10-01T09:23:32.130884+0000","last_peered":"2023-10-01T09:23:32.130884+0000","last_clean":"2023-10-01T09:23:32.130884+0000","last_became_active":"2023-10-01T09:23:10.610928+0000","last_became_peered":"2023-10-01T09:23:10.610928+0000","last_unstale":"2023-10-01T09:23:32.130884+0000","last_undegraded":"2023-10-01T09:23:32.130884+0000","last_fullsized":"2023-10-01T09:23:32.130884+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:17.141540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":203,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.137816+0000","last_change":"2023-10-01T09:22:03.163438+0000","last_active":"2023-10-01T09:23:32.137816+0000","last_peered":"2023-10-01T09:23:32.137816+0000","last_clean":"2023-10-01T09:23:32.137816+0000","last_became_active":"2023-10-01T09:22:03.162821+0000","last_became_peered":"2023-10-01T09:22:03.162821+0000","last_unstale":"2023-10-01T09:23:32.137816+0000","last_undegraded":"2023-10-01T09:23:32.137816+0000","last_fullsized":"2023-10-01T09:23:32.137816+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":40,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.177677+0000","last_change":"2023-10-01T09:23:10.613386+0000","last_active":"2023-10-01T09:23:32.177677+0000","last_peered":"2023-10-01T09:23:32.177677+0000","last_clean":"2023-10-01T09:23:32.177677+0000","last_became_active":"2023-10-01T09:23:10.613046+0000","last_became_peered":"2023-10-01T09:23:10.613046+0000","last_unstale":"2023-10-01T09:23:32.177677+0000","last_undegraded":"2023-10-01T09:23:32.177677+0000","last_fullsized":"2023-10-01T09:23:32.177677+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:23:01.448841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":24,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136305+0000","last_change":"2023-10-01T09:23:10.601126+0000","last_active":"2023-10-01T09:23:32.136305+0000","last_peered":"2023-10-01T09:23:32.136305+0000","last_clean":"2023-10-01T09:23:32.136305+0000","last_became_active":"2023-10-01T09:23:10.598657+0000","last_became_peered":"2023-10-01T09:23:10.598657+0000","last_unstale":"2023-10-01T09:23:32.136305+0000","last_undegraded":"2023-10-01T09:23:32.136305+0000","last_fullsized":"2023-10-01T09:23:32.136305+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":192,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.134503+0000","last_change":"2023-10-01T09:23:10.596661+0000","last_active":"2023-10-01T09:23:32.134503+0000","last_peered":"2023-10-01T09:23:32.134503+0000","last_clean":"2023-10-01T09:23:32.134503+0000","last_became_active":"2023-10-01T09:23:10.596299+0000","last_became_peered":"2023-10-01T09:23:10.596299+0000","last_unstale":"2023-10-01T09:23:32.134503+0000","last_undegraded":"2023-10-01T09:23:32.134503+0000","last_fullsized":"2023-10-01T09:23:32.134503+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":225,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.137399+0000","last_change":"2023-10-01T09:12:10.003552+0000","last_active":"2023-10-01T09:23:32.137399+0000","last_peered":"2023-10-01T09:23:32.137399+0000","last_clean":"2023-10-01T09:23:32.137399+0000","last_became_active":"2023-10-01T09:12:10.003131+0000","last_became_peered":"2023-10-01T09:12:10.003131+0000","last_unstale":"2023-10-01T09:23:32.137399+0000","last_undegraded":"2023-10-01T09:23:32.137399+0000","last_fullsized":"2023-10-01T09:23:32.137399+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":258,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136375+0000","last_change":"2023-10-01T09:22:40.167822+0000","last_active":"2023-10-01T09:23:32.136375+0000","last_peered":"2023-10-01T09:23:32.136375+0000","last_clean":"2023-10-01T09:23:32.136375+0000","last_became_active":"2023-10-01T09:22:40.167502+0000","last_became_peered":"2023-10-01T09:22:40.167502+0000","last_unstale":"2023-10-01T09:23:32.136375+0000","last_undegraded":"2023-10-01T09:23:32.136375+0000","last_fullsized":"2023-10-01T09:23:32.136375+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:40:27.950717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":249,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136459+0000","last_change":"2023-10-01T09:22:03.151056+0000","last_active":"2023-10-01T09:23:32.136459+0000","last_peered":"2023-10-01T09:23:32.136459+0000","last_clean":"2023-10-01T09:23:32.136459+0000","last_became_active":"2023-10-01T09:22:03.150643+0000","last_became_peered":"2023-10-01T09:22:03.150643+0000","last_unstale":"2023-10-01T09:23:32.136459+0000","last_undegraded":"2023-10-01T09:23:32.136459+0000","last_fullsized":"2023-10-01T09:23:32.136459+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":218,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.138031+0000","last_change":"2023-10-01T09:23:10.610311+0000","last_active":"2023-10-01T09:23:32.138031+0000","last_peered":"2023-10-01T09:23:32.138031+0000","last_clean":"2023-10-01T09:23:32.138031+0000","last_became_active":"2023-10-01T09:23:10.609935+0000","last_became_peered":"2023-10-01T09:23:10.609935+0000","last_unstale":"2023-10-01T09:23:32.138031+0000","last_undegraded":"2023-10-01T09:23:32.138031+0000","last_fullsized":"2023-10-01T09:23:32.138031+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":208,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.134991+0000","last_change":"2023-10-01T09:22:40.313521+0000","last_active":"2023-10-01T09:23:32.134991+0000","last_peered":"2023-10-01T09:23:32.134991+0000","last_clean":"2023-10-01T09:23:32.134991+0000","last_became_active":"2023-10-01T09:22:40.312941+0000","last_became_peered":"2023-10-01T09:22:40.312941+0000","last_unstale":"2023-10-01T09:23:32.134991+0000","last_undegraded":"2023-10-01T09:23:32.134991+0000","last_fullsized":"2023-10-01T09:23:32.134991+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":208,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130647+0000","last_change":"2023-10-01T09:12:10.001565+0000","last_active":"2023-10-01T09:23:32.130647+0000","last_peered":"2023-10-01T09:23:32.130647+0000","last_clean":"2023-10-01T09:23:32.130647+0000","last_became_active":"2023-10-01T09:12:10.001275+0000","last_became_peered":"2023-10-01T09:12:10.001275+0000","last_unstale":"2023-10-01T09:23:32.130647+0000","last_undegraded":"2023-10-01T09:23:32.130647+0000","last_fullsized":"2023-10-01T09:23:32.130647+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:39.067046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":227,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.176343+0000","last_change":"2023-10-01T09:13:13.554321+0000","last_active":"2023-10-01T09:23:32.176343+0000","last_peered":"2023-10-01T09:23:32.176343+0000","last_clean":"2023-10-01T09:23:32.176343+0000","last_became_active":"2023-10-01T09:13:13.553986+0000","last_became_peered":"2023-10-01T09:13:13.553986+0000","last_unstale":"2023-10-01T09:23:32.176343+0000","last_undegraded":"2023-10-01T09:23:32.176343+0000","last_fullsized":"2023-10-01T09:23:32.176343+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:09:12.848269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":188,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.177077+0000","last_change":"2023-10-01T09:21:02.139718+0000","last_active":"2023-10-01T09:23:32.177077+0000","last_peered":"2023-10-01T09:23:32.177077+0000","last_clean":"2023-10-01T09:23:32.177077+0000","last_became_active":"2023-10-01T09:21:02.139406+0000","last_became_peered":"2023-10-01T09:21:02.139406+0000","last_unstale":"2023-10-01T09:23:32.177077+0000","last_undegraded":"2023-10-01T09:23:32.177077+0000","last_fullsized":"2023-10-01T09:23:32.177077+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:10.823093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":184,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136420+0000","last_change":"2023-10-01T09:21:02.136776+0000","last_active":"2023-10-01T09:23:32.136420+0000","last_peered":"2023-10-01T09:23:32.136420+0000","last_clean":"2023-10-01T09:23:32.136420+0000","last_became_active":"2023-10-01T09:21:02.136476+0000","last_became_peered":"2023-10-01T09:21:02.136476+0000","last_unstale":"2023-10-01T09:23:32.136420+0000","last_undegraded":"2023-10-01T09:23:32.136420+0000","last_fullsized":"2023-10-01T09:23:32.136420+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:44:41.289825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":254,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-01T09:23:31.413811+0000","last_change":"2023-10-01T09:20:15.505658+0000","last_active":"2023-10-01T09:23:31.413811+0000","last_peered":"2023-10-01T09:23:31.413811+0000","last_clean":"2023-10-01T09:23:31.413811+0000","last_became_active":"2023-10-01T09:20:15.505361+0000","last_became_peered":"2023-10-01T09:20:15.505361+0000","last_unstale":"2023-10-01T09:23:31.413811+0000","last_undegraded":"2023-10-01T09:23:31.413811+0000","last_fullsized":"2023-10-01T09:23:31.413811+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:22:16.176432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":125,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.176304+0000","last_change":"2023-10-01T09:19:30.092681+0000","last_active":"2023-10-01T09:23:32.176304+0000","last_peered":"2023-10-01T09:23:32.176304+0000","last_clean":"2023-10-01T09:23:32.176304+0000","last_became_active":"2023-10-01T09:19:30.092394+0000","last_became_peered":"2023-10-01T09:19:30.092394+0000","last_unstale":"2023-10-01T09:23:32.176304+0000","last_undegraded":"2023-10-01T09:23:32.176304+0000","last_fullsized":"2023-10-01T09:23:32.176304+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:41:50.127253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":50,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.134663+0000","last_change":"2023-10-01T09:22:03.144199+0000","last_active":"2023-10-01T09:23:32.134663+0000","last_peered":"2023-10-01T09:23:32.134663+0000","last_clean":"2023-10-01T09:23:32.134663+0000","last_became_active":"2023-10-01T09:22:03.143883+0000","last_became_peered":"2023-10-01T09:22:03.143883+0000","last_unstale":"2023-10-01T09:23:32.134663+0000","last_undegraded":"2023-10-01T09:23:32.134663+0000","last_fullsized":"2023-10-01T09:23:32.134663+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":31,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.176023+0000","last_change":"2023-10-01T09:22:40.316008+0000","last_active":"2023-10-01T09:23:32.176023+0000","last_peered":"2023-10-01T09:23:32.176023+0000","last_clean":"2023-10-01T09:23:32.176023+0000","last_became_active":"2023-10-01T09:22:40.315602+0000","last_became_peered":"2023-10-01T09:22:40.315602+0000","last_unstale":"2023-10-01T09:23:32.176023+0000","last_undegraded":"2023-10-01T09:23:32.176023+0000","last_fullsized":"2023-10-01T09:23:32.176023+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:39:37.485241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":122,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.137283+0000","last_change":"2023-10-01T09:18:31.419430+0000","last_active":"2023-10-01T09:23:32.137283+0000","last_peered":"2023-10-01T09:23:32.137283+0000","last_clean":"2023-10-01T09:23:32.137283+0000","last_became_active":"2023-10-01T09:18:31.419189+0000","last_became_peered":"2023-10-01T09:18:31.419189+0000","last_unstale":"2023-10-01T09:23:32.137283+0000","last_undegraded":"2023-10-01T09:23:32.137283+0000","last_fullsized":"2023-10-01T09:23:32.137283+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:08:48.247926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1074,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.137012+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:23:32.137012+0000","last_peered":"2023-10-01T09:23:32.137012+0000","last_clean":"2023-10-01T09:23:32.137012+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:23:32.137012+0000","last_undegraded":"2023-10-01T09:23:32.137012+0000","last_fullsized":"2023-10-01T09:23:32.137012+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":252,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.135061+0000","last_change":"2023-10-01T09:22:40.170877+0000","last_active":"2023-10-01T09:23:32.135061+0000","last_peered":"2023-10-01T09:23:32.135061+0000","last_clean":"2023-10-01T09:23:32.135061+0000","last_became_active":"2023-10-01T09:22:40.170578+0000","last_became_peered":"2023-10-01T09:22:40.170578+0000","last_unstale":"2023-10-01T09:23:32.135061+0000","last_undegraded":"2023-10-01T09:23:32.135061+0000","last_fullsized":"2023-10-01T09:23:32.135061+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":204,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136343+0000","last_change":"2023-10-01T09:22:02.151482+0000","last_active":"2023-10-01T09:23:32.136343+0000","last_peered":"2023-10-01T09:23:32.136343+0000","last_clean":"2023-10-01T09:23:32.136343+0000","last_became_active":"2023-10-01T09:22:02.151180+0000","last_became_peered":"2023-10-01T09:22:02.151180+0000","last_unstale":"2023-10-01T09:23:32.136343+0000","last_undegraded":"2023-10-01T09:23:32.136343+0000","last_fullsized":"2023-10-01T09:23:32.136343+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:11.745198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":918,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.134758+0000","last_change":"2023-10-01T09:22:03.258449+0000","last_active":"2023-10-01T09:23:32.134758+0000","last_peered":"2023-10-01T09:23:32.134758+0000","last_clean":"2023-10-01T09:23:32.134758+0000","last_became_active":"2023-10-01T09:22:03.257766+0000","last_became_peered":"2023-10-01T09:22:03.257766+0000","last_unstale":"2023-10-01T09:23:32.134758+0000","last_undegraded":"2023-10-01T09:23:32.134758+0000","last_fullsized":"2023-10-01T09:23:32.134758+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":175,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130917+0000","last_change":"2023-10-01T09:21:02.121225+0000","last_active":"2023-10-01T09:23:32.130917+0000","last_peered":"2023-10-01T09:23:32.130917+0000","last_clean":"2023-10-01T09:23:32.130917+0000","last_became_active":"2023-10-01T09:21:02.120911+0000","last_became_peered":"2023-10-01T09:21:02.120911+0000","last_unstale":"2023-10-01T09:23:32.130917+0000","last_undegraded":"2023-10-01T09:23:32.130917+0000","last_fullsized":"2023-10-01T09:23:32.130917+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:34:44.385101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":91,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.134668+0000","last_change":"2023-10-01T09:22:40.172426+0000","last_active":"2023-10-01T09:23:32.134668+0000","last_peered":"2023-10-01T09:23:32.134668+0000","last_clean":"2023-10-01T09:23:32.134668+0000","last_became_active":"2023-10-01T09:22:40.172160+0000","last_became_peered":"2023-10-01T09:22:40.172160+0000","last_unstale":"2023-10-01T09:23:32.134668+0000","last_undegraded":"2023-10-01T09:23:32.134668+0000","last_fullsized":"2023-10-01T09:23:32.134668+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":581,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-10-01T09:23:31.413118+0000","last_change":"2023-10-01T09:22:40.183431+0000","last_active":"2023-10-01T09:23:31.413118+0000","last_peered":"2023-10-01T09:23:31.413118+0000","last_clean":"2023-10-01T09:23:31.413118+0000","last_became_active":"2023-10-01T09:22:40.171414+0000","last_became_peered":"2023-10-01T09:22:40.171414+0000","last_unstale":"2023-10-01T09:23:31.413118+0000","last_undegraded":"2023-10-01T09:23:31.413118+0000","last_fullsized":"2023-10-01T09:23:31.413118+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:56:42.904846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":258,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136375+0000","last_change":"2023-10-01T09:21:02.140363+0000","last_active":"2023-10-01T09:23:32.136375+0000","last_peered":"2023-10-01T09:23:32.136375+0000","last_clean":"2023-10-01T09:23:32.136375+0000","last_became_active":"2023-10-01T09:21:02.137198+0000","last_became_peered":"2023-10-01T09:21:02.137198+0000","last_unstale":"2023-10-01T09:23:32.136375+0000","last_undegraded":"2023-10-01T09:23:32.136375+0000","last_fullsized":"2023-10-01T09:23:32.136375+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:38:38.043146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":119,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.137351+0000","last_change":"2023-10-01T09:22:02.156724+0000","last_active":"2023-10-01T09:23:32.137351+0000","last_peered":"2023-10-01T09:23:32.137351+0000","last_clean":"2023-10-01T09:23:32.137351+0000","last_became_active":"2023-10-01T09:22:02.156451+0000","last_became_peered":"2023-10-01T09:22:02.156451+0000","last_unstale":"2023-10-01T09:23:32.137351+0000","last_undegraded":"2023-10-01T09:23:32.137351+0000","last_fullsized":"2023-10-01T09:23:32.137351+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:29:05.232127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1315,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.135010+0000","last_change":"2023-10-01T09:23:13.197266+0000","last_active":"2023-10-01T09:23:32.135010+0000","last_peered":"2023-10-01T09:23:32.135010+0000","last_clean":"2023-10-01T09:23:32.135010+0000","last_became_active":"2023-10-01T09:23:13.196795+0000","last_became_peered":"2023-10-01T09:23:13.196795+0000","last_unstale":"2023-10-01T09:23:32.135010+0000","last_undegraded":"2023-10-01T09:23:32.135010+0000","last_fullsized":"2023-10-01T09:23:32.135010+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":420,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.139546+0000","last_change":"2023-10-01T09:23:32.139254+0000","last_active":"2023-10-01T09:23:32.139546+0000","last_peered":"2023-10-01T09:23:32.139546+0000","last_clean":"2023-10-01T09:23:32.139546+0000","last_became_active":"2023-10-01T09:23:32.138752+0000","last_became_peered":"2023-10-01T09:23:32.138752+0000","last_unstale":"2023-10-01T09:23:32.139546+0000","last_undegraded":"2023-10-01T09:23:32.139546+0000","last_fullsized":"2023-10-01T09:23:32.139546+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":31,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.178041+0000","last_change":"2023-10-01T09:22:40.317678+0000","last_active":"2023-10-01T09:23:32.178041+0000","last_peered":"2023-10-01T09:23:32.178041+0000","last_clean":"2023-10-01T09:23:32.178041+0000","last_became_active":"2023-10-01T09:22:40.317460+0000","last_became_peered":"2023-10-01T09:22:40.317460+0000","last_unstale":"2023-10-01T09:23:32.178041+0000","last_undegraded":"2023-10-01T09:23:32.178041+0000","last_fullsized":"2023-10-01T09:23:32.178041+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:07:41.681679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":129,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136543+0000","last_change":"2023-10-01T09:21:02.125167+0000","last_active":"2023-10-01T09:23:32.136543+0000","last_peered":"2023-10-01T09:23:32.136543+0000","last_clean":"2023-10-01T09:23:32.136543+0000","last_became_active":"2023-10-01T09:21:02.124791+0000","last_became_peered":"2023-10-01T09:21:02.124791+0000","last_unstale":"2023-10-01T09:23:32.136543+0000","last_undegraded":"2023-10-01T09:23:32.136543+0000","last_fullsized":"2023-10-01T09:23:32.136543+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":353,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136183+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:23:32.136183+0000","last_peered":"2023-10-01T09:23:32.136183+0000","last_clean":"2023-10-01T09:23:32.136183+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:23:32.136183+0000","last_undegraded":"2023-10-01T09:23:32.136183+0000","last_fullsized":"2023-10-01T09:23:32.136183+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":242,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130554+0000","last_change":"2023-10-01T09:22:40.171122+0000","last_active":"2023-10-01T09:23:32.130554+0000","last_peered":"2023-10-01T09:23:32.130554+0000","last_clean":"2023-10-01T09:23:32.130554+0000","last_became_active":"2023-10-01T09:22:40.170773+0000","last_became_peered":"2023-10-01T09:22:40.170773+0000","last_unstale":"2023-10-01T09:23:32.130554+0000","last_undegraded":"2023-10-01T09:23:32.130554+0000","last_fullsized":"2023-10-01T09:23:32.130554+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:50:30.349100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":259,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.131111+0000","last_change":"2023-10-01T09:19:30.091481+0000","last_active":"2023-10-01T09:23:32.131111+0000","last_peered":"2023-10-01T09:23:32.131111+0000","last_clean":"2023-10-01T09:23:32.131111+0000","last_became_active":"2023-10-01T09:19:30.091151+0000","last_became_peered":"2023-10-01T09:19:30.091151+0000","last_unstale":"2023-10-01T09:23:32.131111+0000","last_undegraded":"2023-10-01T09:23:32.131111+0000","last_fullsized":"2023-10-01T09:23:32.131111+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:09.127408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":617,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136544+0000","last_change":"2023-10-01T09:10:58.797964+0000","last_active":"2023-10-01T09:23:32.136544+0000","last_peered":"2023-10-01T09:23:32.136544+0000","last_clean":"2023-10-01T09:23:32.136544+0000","last_became_active":"2023-10-01T09:10:58.797447+0000","last_became_peered":"2023-10-01T09:10:58.797447+0000","last_unstale":"2023-10-01T09:23:32.136544+0000","last_undegraded":"2023-10-01T09:23:32.136544+0000","last_fullsized":"2023-10-01T09:23:32.136544+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":255,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.137089+0000","last_change":"2023-10-01T09:22:02.160793+0000","last_active":"2023-10-01T09:23:32.137089+0000","last_peered":"2023-10-01T09:23:32.137089+0000","last_clean":"2023-10-01T09:23:32.137089+0000","last_became_active":"2023-10-01T09:22:02.160579+0000","last_became_peered":"2023-10-01T09:22:02.160579+0000","last_unstale":"2023-10-01T09:23:32.137089+0000","last_undegraded":"2023-10-01T09:23:32.137089+0000","last_fullsized":"2023-10-01T09:23:32.137089+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:34:29.335908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":209,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130846+0000","last_change":"2023-10-01T09:22:02.151402+0000","last_active":"2023-10-01T09:23:32.130846+0000","last_peered":"2023-10-01T09:23:32.130846+0000","last_clean":"2023-10-01T09:23:32.130846+0000","last_became_active":"2023-10-01T09:22:02.151028+0000","last_became_peered":"2023-10-01T09:22:02.151028+0000","last_unstale":"2023-10-01T09:23:32.130846+0000","last_undegraded":"2023-10-01T09:23:32.130846+0000","last_fullsized":"2023-10-01T09:23:32.130846+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:19.623044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":268,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.176538+0000","last_change":"2023-10-01T09:23:10.615956+0000","last_active":"2023-10-01T09:23:32.176538+0000","last_peered":"2023-10-01T09:23:32.176538+0000","last_clean":"2023-10-01T09:23:32.176538+0000","last_became_active":"2023-10-01T09:23:10.610919+0000","last_became_peered":"2023-10-01T09:23:10.610919+0000","last_unstale":"2023-10-01T09:23:32.176538+0000","last_undegraded":"2023-10-01T09:23:32.176538+0000","last_fullsized":"2023-10-01T09:23:32.176538+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:03:44.294373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":730,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136994+0000","last_change":"2023-10-01T09:21:20.545328+0000","last_active":"2023-10-01T09:23:32.136994+0000","last_peered":"2023-10-01T09:23:32.136994+0000","last_clean":"2023-10-01T09:23:32.136994+0000","last_became_active":"2023-10-01T09:21:20.544828+0000","last_became_peered":"2023-10-01T09:21:20.544828+0000","last_unstale":"2023-10-01T09:23:32.136994+0000","last_undegraded":"2023-10-01T09:23:32.136994+0000","last_fullsized":"2023-10-01T09:23:32.136994+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:23:52.633471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":40,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.176464+0000","last_change":"2023-10-01T09:23:10.612309+0000","last_active":"2023-10-01T09:23:32.176464+0000","last_peered":"2023-10-01T09:23:32.176464+0000","last_clean":"2023-10-01T09:23:32.176464+0000","last_became_active":"2023-10-01T09:23:10.612034+0000","last_became_peered":"2023-10-01T09:23:10.612034+0000","last_unstale":"2023-10-01T09:23:32.176464+0000","last_undegraded":"2023-10-01T09:23:32.176464+0000","last_fullsized":"2023-10-01T09:23:32.176464+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:09:47.963186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":88,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.503601+0000","last_change":"2023-10-01T09:23:35.503601+0000","last_active":"2023-10-01T09:23:35.503601+0000","last_peered":"2023-10-01T09:23:35.503601+0000","last_clean":"2023-10-01T09:23:35.503601+0000","last_became_active":"2023-10-01T09:23:10.613270+0000","last_became_peered":"2023-10-01T09:23:10.613270+0000","last_unstale":"2023-10-01T09:23:35.503601+0000","last_undegraded":"2023-10-01T09:23:35.503601+0000","last_fullsized":"2023-10-01T09:23:35.503601+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":238,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.131032+0000","last_change":"2023-10-01T09:19:05.565472+0000","last_active":"2023-10-01T09:23:32.131032+0000","last_peered":"2023-10-01T09:23:32.131032+0000","last_clean":"2023-10-01T09:23:32.131032+0000","last_became_active":"2023-10-01T09:19:05.561161+0000","last_became_peered":"2023-10-01T09:19:05.561161+0000","last_unstale":"2023-10-01T09:23:32.131032+0000","last_undegraded":"2023-10-01T09:23:32.131032+0000","last_fullsized":"2023-10-01T09:23:32.131032+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:50:29.273388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.134427+0000","last_change":"2023-10-01T09:22:03.144097+0000","last_active":"2023-10-01T09:23:32.134427+0000","last_peered":"2023-10-01T09:23:32.134427+0000","last_clean":"2023-10-01T09:23:32.134427+0000","last_became_active":"2023-10-01T09:22:03.143800+0000","last_became_peered":"2023-10-01T09:22:03.143800+0000","last_unstale":"2023-10-01T09:23:32.134427+0000","last_undegraded":"2023-10-01T09:23:32.134427+0000","last_fullsized":"2023-10-01T09:23:32.134427+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":197,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130779+0000","last_change":"2023-10-01T09:10:58.801161+0000","last_active":"2023-10-01T09:23:32.130779+0000","last_peered":"2023-10-01T09:23:32.130779+0000","last_clean":"2023-10-01T09:23:32.130779+0000","last_became_active":"2023-10-01T09:10:58.800794+0000","last_became_peered":"2023-10-01T09:10:58.800794+0000","last_unstale":"2023-10-01T09:23:32.130779+0000","last_undegraded":"2023-10-01T09:23:32.130779+0000","last_fullsized":"2023-10-01T09:23:32.130779+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:35.019445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":283,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.137159+0000","last_change":"2023-10-01T09:22:40.166927+0000","last_active":"2023-10-01T09:23:32.137159+0000","last_peered":"2023-10-01T09:23:32.137159+0000","last_clean":"2023-10-01T09:23:32.137159+0000","last_became_active":"2023-10-01T09:22:40.166606+0000","last_became_peered":"2023-10-01T09:22:40.166606+0000","last_unstale":"2023-10-01T09:23:32.137159+0000","last_undegraded":"2023-10-01T09:23:32.137159+0000","last_fullsized":"2023-10-01T09:23:32.137159+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:54:18.044207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":188,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.134904+0000","last_change":"2023-10-01T09:22:02.180112+0000","last_active":"2023-10-01T09:23:32.134904+0000","last_peered":"2023-10-01T09:23:32.134904+0000","last_clean":"2023-10-01T09:23:32.134904+0000","last_became_active":"2023-10-01T09:22:02.161271+0000","last_became_peered":"2023-10-01T09:22:02.161271+0000","last_unstale":"2023-10-01T09:23:32.134904+0000","last_undegraded":"2023-10-01T09:23:32.134904+0000","last_fullsized":"2023-10-01T09:23:32.134904+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":202,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.176246+0000","last_change":"2023-10-01T09:19:05.553200+0000","last_active":"2023-10-01T09:23:32.176246+0000","last_peered":"2023-10-01T09:23:32.176246+0000","last_clean":"2023-10-01T09:23:32.176246+0000","last_became_active":"2023-10-01T09:19:05.552874+0000","last_became_peered":"2023-10-01T09:19:05.552874+0000","last_unstale":"2023-10-01T09:23:32.176246+0000","last_undegraded":"2023-10-01T09:23:32.176246+0000","last_fullsized":"2023-10-01T09:23:32.176246+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:14:31.040887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":177,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.134935+0000","last_change":"2023-10-01T09:22:03.149673+0000","last_active":"2023-10-01T09:23:32.134935+0000","last_peered":"2023-10-01T09:23:32.134935+0000","last_clean":"2023-10-01T09:23:32.134935+0000","last_became_active":"2023-10-01T09:22:03.149236+0000","last_became_peered":"2023-10-01T09:22:03.149236+0000","last_unstale":"2023-10-01T09:23:32.134935+0000","last_undegraded":"2023-10-01T09:23:32.134935+0000","last_fullsized":"2023-10-01T09:23:32.134935+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":184,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.178198+0000","last_change":"2023-10-01T09:18:54.533869+0000","last_active":"2023-10-01T09:23:32.178198+0000","last_peered":"2023-10-01T09:23:32.178198+0000","last_clean":"2023-10-01T09:23:32.178198+0000","last_became_active":"2023-10-01T09:18:54.533542+0000","last_became_peered":"2023-10-01T09:18:54.533542+0000","last_unstale":"2023-10-01T09:23:32.178198+0000","last_undegraded":"2023-10-01T09:23:32.178198+0000","last_fullsized":"2023-10-01T09:23:32.178198+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:18:19.126440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":31,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.176658+0000","last_change":"2023-10-01T09:22:40.313999+0000","last_active":"2023-10-01T09:23:32.176658+0000","last_peered":"2023-10-01T09:23:32.176658+0000","last_clean":"2023-10-01T09:23:32.176658+0000","last_became_active":"2023-10-01T09:22:40.313605+0000","last_became_peered":"2023-10-01T09:22:40.313605+0000","last_unstale":"2023-10-01T09:23:32.176658+0000","last_undegraded":"2023-10-01T09:23:32.176658+0000","last_fullsized":"2023-10-01T09:23:32.176658+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:52:32.154690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":31,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.176221+0000","last_change":"2023-10-01T09:22:40.170676+0000","last_active":"2023-10-01T09:23:32.176221+0000","last_peered":"2023-10-01T09:23:32.176221+0000","last_clean":"2023-10-01T09:23:32.176221+0000","last_became_active":"2023-10-01T09:22:40.170260+0000","last_became_peered":"2023-10-01T09:22:40.170260+0000","last_unstale":"2023-10-01T09:23:32.176221+0000","last_undegraded":"2023-10-01T09:23:32.176221+0000","last_fullsized":"2023-10-01T09:23:32.176221+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:14.693287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":288,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.132021+0000","last_change":"2023-10-01T09:22:02.157406+0000","last_active":"2023-10-01T09:23:32.132021+0000","last_peered":"2023-10-01T09:23:32.132021+0000","last_clean":"2023-10-01T09:23:32.132021+0000","last_became_active":"2023-10-01T09:22:02.157041+0000","last_became_peered":"2023-10-01T09:22:02.157041+0000","last_unstale":"2023-10-01T09:23:32.132021+0000","last_undegraded":"2023-10-01T09:23:32.132021+0000","last_fullsized":"2023-10-01T09:23:32.132021+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:24:52.008578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":178,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136245+0000","last_change":"2023-10-01T09:21:02.141180+0000","last_active":"2023-10-01T09:23:32.136245+0000","last_peered":"2023-10-01T09:23:32.136245+0000","last_clean":"2023-10-01T09:23:32.136245+0000","last_became_active":"2023-10-01T09:21:02.136796+0000","last_became_peered":"2023-10-01T09:21:02.136796+0000","last_unstale":"2023-10-01T09:23:32.136245+0000","last_undegraded":"2023-10-01T09:23:32.136245+0000","last_fullsized":"2023-10-01T09:23:32.136245+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:03:46.040197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":12024,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35793,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430476,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312568,"kb_used_data":7152,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93432584,"statfs":{"total":95995035648,"available":95674966016,"internally_reserved":0,"allocated":7323648,"data_stored":6604003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:15 2023","interfaces":[{"interface":"back","average":{"1min":2.573,"5min":5.2359999999999998,"15min":3.234},"min":{"1min":0.28100000000000003,"5min":0.23400000000000001,"15min":0.219},"max":{"1min":7.5190000000000001,"5min":117.855,"15min":117.855},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.891,"5min":5.2889999999999997,"15min":3.0249999999999999},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":6.1159999999999997,"5min":120.348,"15min":120.348},"last":0.34000000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.8600000000000001,"5min":5.3099999999999996,"15min":2.794},"min":{"1min":0.251,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":7.4279999999999999,"5min":135.023,"15min":135.023},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.7719999999999998,"5min":5.2800000000000002,"15min":2.6040000000000001},"min":{"1min":0.29499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":10.034000000000001,"5min":134.51499999999999,"15min":134.51499999999999},"last":0.43099999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:22:54 2023","interfaces":[{"interface":"back","average":{"1min":4.9550000000000001,"5min":4.3860000000000001,"15min":2.4900000000000002},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":59.878,"5min":104.837,"15min":104.837},"last":0.36499999999999999},{"interface":"front","average":{"1min":6.2590000000000003,"5min":4.2640000000000002,"15min":2.4319999999999999},"min":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":60.186,"5min":106.379,"15min":106.379},"last":0.41299999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1739999999999999,"5min":3.4870000000000001,"15min":1.8660000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26400000000000001},"max":{"1min":10.1,"5min":104.797,"15min":104.797},"last":0.315},{"interface":"front","average":{"1min":2.8839999999999999,"5min":3.0760000000000001,"15min":1.8260000000000001},"min":{"1min":0.34100000000000003,"5min":0.29399999999999998,"15min":0.27200000000000002},"max":{"1min":7.2039999999999997,"5min":104.774,"15min":104.774},"last":0.34799999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:23:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5870000000000002,"5min":1.726,"15min":1.546},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.23699999999999999},"max":{"1min":6.5510000000000002,"5min":16.062999999999999,"15min":82.992000000000004},"last":0.45200000000000001},{"interface":"front","average":{"1min":2.1859999999999999,"5min":1.7629999999999999,"15min":1.579},"min":{"1min":0.39700000000000002,"5min":0.29499999999999998,"15min":0.252},"max":{"1min":6.2089999999999996,"5min":24.161999999999999,"15min":83.058999999999997},"last":0.375}]},{"osd":5,"last update":"Sun Oct 1 09:23:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":3.1339999999999999,"15min":3.1339999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":13.265000000000001,"5min":13.265000000000001,"15min":13.265000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.8940000000000001,"5min":2.8940000000000001,"15min":2.8940000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":13.292999999999999,"5min":13.292999999999999,"15min":13.292999999999999},"last":0.44}]},{"osd":6,"last update":"Sun Oct 1 09:23:15 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.4510000000000001,"15min":1.2070000000000001},"min":{"1min":0.35899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.5330000000000004,"5min":11.685,"15min":11.685},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.0750000000000002,"5min":1.6619999999999999,"15min":1.1910000000000001},"min":{"1min":0.38,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.4220000000000006,"5min":11.871,"15min":11.871},"last":0.39400000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626704,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":646888,"kb_used_data":345568,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93098264,"statfs":{"total":95995035648,"available":95332622336,"internally_reserved":0,"allocated":353861632,"data_stored":353157614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8919999999999999,"5min":3.4470000000000001,"15min":3.8959999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20000000000000001},"max":{"1min":6.8200000000000003,"5min":137.941,"15min":178.738},"last":0.49199999999999999},{"interface":"front","average":{"1min":2.3599999999999999,"5min":3.7109999999999999,"15min":4.1600000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":10.992000000000001,"5min":138.20599999999999,"15min":178.684},"last":0.27700000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:23:30 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":4.6360000000000001,"15min":2.2879999999999998},"min":{"1min":0.253,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.1550000000000002,"5min":115.523,"15min":115.523},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.1699999999999999,"5min":5.0949999999999998,"15min":2.7240000000000002},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":11.042999999999999,"5min":115.681,"15min":115.681},"last":0.312}]},{"osd":2,"last update":"Sun Oct 1 09:22:58 2023","interfaces":[{"interface":"back","average":{"1min":3.3330000000000002,"5min":2.3220000000000001,"15min":2.5099999999999998},"min":{"1min":0.34499999999999997,"5min":0.26000000000000001,"15min":0.25900000000000001},"max":{"1min":14.367000000000001,"5min":16.513999999999999,"15min":82.978999999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.532,"5min":1.7749999999999999,"15min":2.1269999999999998},"min":{"1min":0.34300000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.2859999999999996,"5min":17.036999999999999,"15min":82.747},"last":0.84399999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:23:30 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":4.8799999999999999,"15min":2.403},"min":{"1min":0.307,"5min":0.307,"15min":0.27000000000000002},"max":{"1min":7.2050000000000001,"5min":219.32400000000001,"15min":219.32400000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.7769999999999999,"5min":5.1139999999999999,"15min":2.7069999999999999},"min":{"1min":0.37,"5min":0.33600000000000002,"15min":0.30499999999999999},"max":{"1min":8.6989999999999998,"5min":219.77099999999999,"15min":219.77099999999999},"last":0.38700000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:22:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":1.4570000000000001,"15min":1.1220000000000001},"min":{"1min":0.30199999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":14.089,"5min":14.089,"15min":28.271999999999998},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.4039999999999999,"5min":1.3640000000000001,"15min":1.1459999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.26000000000000001},"max":{"1min":9.0440000000000005,"5min":9.907,"15min":28.413},"last":0.55000000000000004}]},{"osd":5,"last update":"Sun Oct 1 09:23:05 2023","interfaces":[{"interface":"back","average":{"1min":2.956,"5min":2.956,"15min":2.956},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.0760000000000005,"5min":9.0760000000000005,"15min":9.0760000000000005},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.1389999999999998,"5min":2.1389999999999998,"15min":2.1389999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":13.999000000000001,"5min":13.999000000000001,"15min":13.999000000000001},"last":0.39900000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:23:12 2023","interfaces":[{"interface":"back","average":{"1min":2.355,"5min":3.1869999999999998,"15min":1.718},"min":{"1min":0.39000000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":11.492000000000001,"5min":145.37299999999999,"15min":145.37299999999999},"last":0.434},{"interface":"front","average":{"1min":2.657,"5min":3.012,"15min":1.6579999999999999},"min":{"1min":0.35699999999999998,"5min":0.31,"15min":0.30499999999999999},"max":{"1min":9.1010000000000009,"5min":145.28200000000001,"15min":145.28200000000001},"last":0.57799999999999996}]}]},{"osd":1,"up_from":94,"seq":403726925964,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":246200,"kb_used_data":22640,"kb_used_omap":7,"kb_used_meta":223544,"kb_avail":93498952,"statfs":{"total":95995035648,"available":95742926848,"internally_reserved":0,"allocated":23183360,"data_stored":22001307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":228909479},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":6.3200000000000003,"15min":2.891},"min":{"1min":0.19700000000000001,"5min":0.19400000000000001,"15min":0.17499999999999999},"max":{"1min":7.2770000000000001,"5min":174.40199999999999,"15min":174.40199999999999},"last":0.23599999999999999},{"interface":"front","average":{"1min":2.952,"5min":6.4820000000000002,"15min":3.0430000000000001},"min":{"1min":0.219,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":7.702,"5min":173.77000000000001,"15min":173.77000000000001},"last":0.32200000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:23:03 2023","interfaces":[{"interface":"back","average":{"1min":8.3699999999999992,"5min":3,"15min":1.673},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":77.325000000000003,"5min":77.325000000000003,"15min":77.325000000000003},"last":0.26700000000000002},{"interface":"front","average":{"1min":9.0679999999999996,"5min":3.222,"15min":1.825},"min":{"1min":0.26900000000000002,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":80.361000000000004,"5min":80.361000000000004,"15min":80.361000000000004},"last":0.34999999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:23:38 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":4.0899999999999999,"15min":1.992},"min":{"1min":0.224,"5min":0.224,"15min":0.218},"max":{"1min":6.593,"5min":99.421000000000006,"15min":99.421000000000006},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.526,"5min":3.8860000000000001,"15min":1.853},"min":{"1min":0.30599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.4409999999999998,"5min":99.364999999999995,"15min":99.364999999999995},"last":0.376}]},{"osd":4,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":1.4079999999999999,"15min":1.2849999999999999},"min":{"1min":0.28799999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.8259999999999996,"5min":9.9169999999999998,"15min":40.673999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.7150000000000001,"5min":1.514,"15min":1.417},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.25800000000000001},"max":{"1min":7.2249999999999996,"5min":9.4610000000000003,"15min":40.810000000000002},"last":0.434}]},{"osd":5,"last update":"Sun Oct 1 09:23:03 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.403,"15min":1.403},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.734,"5min":4.734,"15min":4.734},"last":0.55900000000000005},{"interface":"front","average":{"1min":2.4369999999999998,"5min":2.4369999999999998,"15min":2.4369999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":5.5090000000000003,"5min":5.5090000000000003,"15min":5.5090000000000003},"last":0.41399999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8260000000000001,"5min":1.5980000000000001,"15min":1.3300000000000001},"min":{"1min":0.35099999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.9860000000000007,"5min":14.452,"15min":14.452},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.29,"5min":1.605,"15min":1.226},"min":{"1min":0.35799999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.9740000000000002,"5min":9.9740000000000002,"15min":10.787000000000001},"last":0.48699999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":1.96,"15min":1.431},"min":{"1min":0.42099999999999999,"5min":0.317,"15min":0.317},"max":{"1min":7.8129999999999997,"5min":23.710999999999999,"15min":23.710999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.853,"5min":2.0049999999999999,"15min":1.4450000000000001},"min":{"1min":0.39500000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.3579999999999997,"5min":23.736999999999998,"15min":23.736999999999998},"last":0.54900000000000004}]}]},{"osd":0,"up_from":89,"seq":382252089498,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":363796,"kb_used_data":7692,"kb_used_omap":8,"kb_used_meta":356087,"kb_avail":93381356,"statfs":{"total":95995035648,"available":95622508544,"internally_reserved":0,"allocated":7876608,"data_stored":6672362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":364634050},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:23:28 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":4.202,"15min":1.99},"min":{"1min":0.221,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":13.054,"5min":187.41399999999999,"15min":187.41399999999999},"last":2.6019999999999999},{"interface":"front","average":{"1min":3.9529999999999998,"5min":4.6779999999999999,"15min":2.1880000000000002},"min":{"1min":0.23400000000000001,"5min":0.193,"15min":0.193},"max":{"1min":17.088999999999999,"5min":187.78700000000001,"15min":187.78700000000001},"last":1.645}]},{"osd":2,"last update":"Sun Oct 1 09:22:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":2.2490000000000001,"15min":1.3939999999999999},"min":{"1min":0.24399999999999999,"5min":0.23300000000000001,"15min":0.23000000000000001},"max":{"1min":4.7699999999999996,"5min":39.048999999999999,"15min":39.048999999999999},"last":2.6509999999999998},{"interface":"front","average":{"1min":1.514,"5min":2.7719999999999998,"15min":1.591},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.673,"5min":38.945999999999998,"15min":38.945999999999998},"last":0.47599999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:23:28 2023","interfaces":[{"interface":"back","average":{"1min":2.335,"5min":3.2770000000000001,"15min":1.655},"min":{"1min":0.26600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":17.914999999999999,"5min":85.840000000000003,"15min":85.840000000000003},"last":2.681},{"interface":"front","average":{"1min":3.5259999999999998,"5min":3.6829999999999998,"15min":1.8500000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23400000000000001},"max":{"1min":11.141999999999999,"5min":86.046000000000006,"15min":86.046000000000006},"last":0.41599999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":3.3420000000000001,"15min":1.8799999999999999},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.6479999999999997,"5min":112.52200000000001,"15min":112.52200000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.3439999999999999,"5min":3.2999999999999998,"15min":1.9830000000000001},"min":{"1min":0.33700000000000002,"5min":0.32500000000000001,"15min":0.28599999999999998},"max":{"1min":8.9890000000000008,"5min":113.124,"15min":113.124},"last":0.49299999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:23:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8480000000000001,"5min":1.8480000000000001,"15min":1.8480000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":5.8710000000000004,"5min":5.8710000000000004,"15min":5.8710000000000004},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.958,"5min":1.958,"15min":1.958},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":5.8879999999999999,"5min":5.8879999999999999,"15min":5.8879999999999999},"last":2.7709999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":2.0910000000000002,"15min":1.577},"min":{"1min":0.375,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.0239999999999991,"5min":34.375999999999998,"15min":34.375999999999998},"last":0.52700000000000002},{"interface":"front","average":{"1min":2.6709999999999998,"5min":1.8899999999999999,"15min":1.379},"min":{"1min":0.39200000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":9.0820000000000007,"5min":31.745000000000001,"15min":31.745000000000001},"last":2.7440000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":3.0150000000000001,"15min":1.867},"min":{"1min":0.44,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":13.096,"5min":98.283000000000001,"15min":98.283000000000001},"last":2.7149999999999999},{"interface":"front","average":{"1min":2.3119999999999998,"5min":2.762,"15min":1.8700000000000001},"min":{"1min":0.36399999999999999,"5min":0.312,"15min":0.312},"max":{"1min":5.7560000000000002,"5min":98.311999999999998,"15min":98.311999999999998},"last":0.44400000000000001}]}]},{"osd":2,"up_from":99,"seq":425201762437,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":346952,"kb_used_data":2944,"kb_used_omap":11,"kb_used_meta":343988,"kb_avail":93398200,"statfs":{"total":95995035648,"available":95639756800,"internally_reserved":0,"allocated":3014656,"data_stored":1870208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":352244599},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":7.5149999999999997,"5min":3.0230000000000001,"15min":2.0899999999999999},"min":{"1min":0.19800000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":119.621,"5min":119.621,"15min":119.621},"last":0.23400000000000001},{"interface":"front","average":{"1min":7.0289999999999999,"5min":3.2010000000000001,"15min":2.3940000000000001},"min":{"1min":0.20999999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":119.411,"5min":119.411,"15min":119.411},"last":0.254}]},{"osd":1,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.032,"5min":3.3170000000000002,"15min":2.1269999999999998},"min":{"1min":0.19600000000000001,"5min":0.193,"15min":0.193},"max":{"1min":8.4399999999999995,"5min":105.798,"15min":105.798},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.452,"5min":3.2120000000000002,"15min":2.2669999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.4039999999999999,"5min":105.56399999999999,"15min":107.729},"last":0.22}]},{"osd":3,"last update":"Sun Oct 1 09:23:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6059999999999999,"5min":6.2910000000000004,"15min":2.7429999999999999},"min":{"1min":0.217,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":25.975000000000001,"5min":253.238,"15min":253.238},"last":0.21099999999999999},{"interface":"front","average":{"1min":2.8929999999999998,"5min":6.1580000000000004,"15min":2.6970000000000001},"min":{"1min":0.30599999999999999,"5min":0.253,"15min":0.22600000000000001},"max":{"1min":22.498999999999999,"5min":252.54900000000001,"15min":252.54900000000001},"last":0.35699999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1850000000000001,"5min":1.974,"15min":1.321},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27000000000000002},"max":{"1min":8.2530000000000001,"5min":16.059000000000001,"15min":17.751999999999999},"last":0.32200000000000001},{"interface":"front","average":{"1min":2.5579999999999998,"5min":2.1030000000000002,"15min":1.409},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":14.359,"5min":14.359,"15min":17.568000000000001},"last":0.30099999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:23:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5289999999999999,"5min":2.5289999999999999,"15min":2.5289999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.2840000000000007,"5min":8.2840000000000007,"15min":8.2840000000000007},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.3580000000000001,"5min":2.3580000000000001,"15min":2.3580000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":9.3160000000000007,"5min":9.3160000000000007,"15min":9.3160000000000007},"last":0.46000000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":3.2320000000000002,"5min":2.1259999999999999,"15min":1.466},"min":{"1min":0.35199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":14.411,"5min":14.411,"15min":17.591000000000001},"last":0.38},{"interface":"front","average":{"1min":2.8919999999999999,"5min":1.7629999999999999,"15min":1.2010000000000001},"min":{"1min":0.40400000000000003,"5min":0.36699999999999999,"15min":0.34200000000000003},"max":{"1min":8.6620000000000008,"5min":8.6620000000000008,"15min":17.736000000000001},"last":0.44800000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":3.6739999999999999,"5min":2.0129999999999999,"15min":1.782},"min":{"1min":0.38200000000000001,"5min":0.34699999999999998,"15min":0.32800000000000001},"max":{"1min":16.169,"5min":16.169,"15min":109.506},"last":0.49099999999999999},{"interface":"front","average":{"1min":3.1680000000000001,"5min":2.0329999999999999,"15min":1.8979999999999999},"min":{"1min":0.375,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":16.152000000000001,"5min":16.152000000000001,"15min":109.58799999999999},"last":0.438}]}]},{"osd":3,"up_from":104,"seq":446676598911,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":249124,"kb_used_data":2972,"kb_used_omap":3,"kb_used_meta":246140,"kb_avail":93496028,"statfs":{"total":95995035648,"available":95739932672,"internally_reserved":0,"allocated":3043328,"data_stored":1877832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":252048103},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":2.9199999999999999,"5min":1.9630000000000001,"15min":2.9689999999999999},"min":{"1min":0.20899999999999999,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":18.635000000000002,"5min":38.314,"15min":199.422},"last":0.23699999999999999},{"interface":"front","average":{"1min":2.976,"5min":2.1680000000000001,"15min":3.1509999999999998},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":18.792000000000002,"5min":38.118000000000002,"15min":199.91200000000001},"last":0.25}]},{"osd":1,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":5.431,"5min":5.702,"15min":3.3940000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":37.011000000000003,"5min":228.60499999999999,"15min":228.60499999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":4.3399999999999999,"5min":5.3099999999999996,"15min":3.2269999999999999},"min":{"1min":0.26500000000000001,"5min":0.23899999999999999,"15min":0.215},"max":{"1min":36.866,"5min":228.28399999999999,"15min":228.28399999999999},"last":0.28899999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":2.6930000000000001,"15min":1.51},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.224},"max":{"1min":7.524,"5min":55.238999999999997,"15min":55.238999999999997},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.99,"5min":2.9409999999999998,"15min":1.6779999999999999},"min":{"1min":0.23999999999999999,"5min":0.219,"15min":0.20399999999999999},"max":{"1min":6.7919999999999998,"5min":55.692999999999998,"15min":55.692999999999998},"last":0.39900000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5,"5min":4.2729999999999997,"15min":2.274},"min":{"1min":0.32600000000000001,"5min":0.27300000000000002,"15min":0.25900000000000001},"max":{"1min":9.0250000000000004,"5min":154.00399999999999,"15min":154.00399999999999},"last":0.438},{"interface":"front","average":{"1min":3.8420000000000001,"5min":4.8140000000000001,"15min":2.601},"min":{"1min":0.316,"5min":0.30099999999999999,"15min":0.248},"max":{"1min":36.947000000000003,"5min":156.87700000000001,"15min":156.87700000000001},"last":0.375}]},{"osd":5,"last update":"Sun Oct 1 09:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":1.7769999999999999,"15min":1.7769999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.5670000000000002,"5min":7.5670000000000002,"15min":7.5670000000000002},"last":0.34899999999999998},{"interface":"front","average":{"1min":3.8439999999999999,"5min":3.8439999999999999,"15min":3.8439999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":36.984000000000002,"5min":36.984000000000002,"15min":36.984000000000002},"last":0.41699999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":4.3070000000000004,"5min":3.5739999999999998,"15min":2.0369999999999999},"min":{"1min":0.36499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":36.901000000000003,"5min":72.433999999999997,"15min":72.433999999999997},"last":0.45000000000000001},{"interface":"front","average":{"1min":4.1470000000000002,"5min":3.1899999999999999,"15min":1.8100000000000001},"min":{"1min":0.33600000000000002,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":37.274999999999999,"5min":72.450999999999993,"15min":72.450999999999993},"last":0.42899999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":4.7910000000000004,"5min":2.2290000000000001,"15min":1.712},"min":{"1min":0.35499999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":37.024000000000001,"5min":38.478999999999999,"15min":79.412999999999997},"last":0.38600000000000001},{"interface":"front","average":{"1min":4.867,"5min":2.415,"15min":1.7490000000000001},"min":{"1min":0.40799999999999997,"5min":0.36399999999999999,"15min":0.308},"max":{"1min":36.762,"5min":40.039999999999999,"15min":79.527000000000001},"last":0.46800000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019161,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":317808,"kb_used_data":16488,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93427344,"statfs":{"total":95995035648,"available":95669600256,"internally_reserved":0,"allocated":16883712,"data_stored":16171645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":3.468,"15min":3.3849999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.20999999999999999},"max":{"1min":4.4690000000000003,"5min":111.27800000000001,"15min":133.91499999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":2.581,"5min":3.7410000000000001,"15min":3.3540000000000001},"min":{"1min":0.26600000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.5099999999999998,"5min":111.289,"15min":133.83199999999999},"last":0.58299999999999996}]},{"osd":1,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":2.6110000000000002,"5min":2.4399999999999999,"15min":2.02},"min":{"1min":0.30599999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.5589999999999993,"5min":37.326999999999998,"15min":133.93600000000001},"last":0.61799999999999999},{"interface":"front","average":{"1min":2.0600000000000001,"5min":2.234,"15min":2.105},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.0009999999999994,"5min":37.335000000000001,"15min":141.06},"last":0.59399999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:23:07 2023","interfaces":[{"interface":"back","average":{"1min":4.367,"5min":3.2690000000000001,"15min":2.9929999999999999},"min":{"1min":0.35299999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":26.545000000000002,"5min":91.474999999999994,"15min":133.96100000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":4.375,"5min":3.4969999999999999,"15min":2.9689999999999999},"min":{"1min":0.33000000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":28.324000000000002,"5min":91.510999999999996,"15min":134.018},"last":0.55200000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.5029999999999999,"15min":1.6439999999999999},"min":{"1min":0.32000000000000001,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":6.0679999999999996,"5min":6.9909999999999997,"15min":134.297},"last":0.64000000000000001},{"interface":"front","average":{"1min":2.9540000000000002,"5min":1.982,"15min":1.9650000000000001},"min":{"1min":0.30399999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":18.370000000000001,"5min":18.370000000000001,"15min":141.089},"last":0.56399999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:23:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.6899999999999999,"15min":1.6899999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.7839999999999998,"5min":5.7839999999999998,"15min":5.7839999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.681,"5min":2.681,"15min":2.681},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.6539999999999999,"5min":8.6539999999999999,"15min":8.6539999999999999},"last":0.41999999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:22:50 2023","interfaces":[{"interface":"back","average":{"1min":3.097,"5min":1.728,"15min":1.464},"min":{"1min":0.36799999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.6839999999999993,"5min":9.1370000000000005,"15min":133.90799999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.3719999999999999,"5min":1.595,"15min":1.6819999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":20.622,"5min":20.622,"15min":141.03},"last":0.51900000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:23:08 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":1.401,"15min":1.6100000000000001},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.0579999999999998,"5min":8.8369999999999997,"15min":133.946},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.5859999999999999,"5min":1.5449999999999999,"15min":1.631},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":8.7140000000000004,"5min":11.215,"15min":133.98400000000001},"last":0.49299999999999999}]}]},{"osd":5,"up_from":227,"seq":974957576212,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":531284,"kb_used_data":199756,"kb_used_omap":8,"kb_used_meta":331511,"kb_avail":93213868,"statfs":{"total":95995035648,"available":95451000832,"internally_reserved":0,"allocated":204550144,"data_stored":203854271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":339468036},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":2.0499999999999998,"15min":2.0499999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":7.7699999999999996,"5min":7.7699999999999996,"15min":7.7699999999999996},"last":0.32500000000000001},{"interface":"front","average":{"1min":2.7130000000000001,"5min":2.7130000000000001,"15min":2.7130000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":10.867000000000001,"5min":10.867000000000001,"15min":10.867000000000001},"last":0.435}]},{"osd":1,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":2.2690000000000001,"15min":2.2690000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.2110000000000003,"5min":6.2110000000000003,"15min":6.2110000000000003},"last":0.38500000000000001},{"interface":"front","average":{"1min":3.3039999999999998,"5min":3.3039999999999998,"15min":3.3039999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.4280000000000008,"5min":9.4280000000000008,"15min":9.4280000000000008},"last":0.48299999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8879999999999999,"5min":2.8879999999999999,"15min":2.8879999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.3049999999999997,"5min":9.3049999999999997,"15min":9.3049999999999997},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.7589999999999999,"15min":1.7589999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.8419999999999996,"5min":4.8419999999999996,"15min":4.8419999999999996},"last":0.47299999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":2.2250000000000001,"15min":2.2250000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.5640000000000001,"5min":6.5640000000000001,"15min":6.5640000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.1739999999999999,"5min":2.1739999999999999,"15min":2.1739999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.7199999999999998,"5min":7.7199999999999998,"15min":7.7199999999999998},"last":0.49199999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":1.891,"15min":1.891},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":7.532,"5min":7.532,"15min":7.532},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":4.117,"5min":4.117,"15min":4.117},"last":0.54700000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":1.948,"5min":1.948,"15min":1.948},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.7520000000000007,"5min":8.7520000000000007,"15min":8.7520000000000007},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.401,"5min":1.401,"15min":1.401},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.7460000000000004,"5min":7.7460000000000004,"15min":7.7460000000000004},"last":0.52700000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":1.6599999999999999,"15min":1.6599999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.6440000000000001,"5min":4.6440000000000001,"15min":4.6440000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.2330000000000001,"5min":2.2330000000000001,"15min":2.2330000000000001},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":10.836,"5min":10.836,"15min":10.836},"last":0.53900000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:23:42.540 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:23:42.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:42 smithi190 ceph-mon[160435]: from='client.35373 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:23:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:42 smithi190 ceph-mon[160435]: pgmap v648: 73 pgs: 73 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2023-10-01T09:23:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:42 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:23:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:42 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:23:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:42 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1120766159' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:23:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:42 smithi190 ceph-mon[160435]: osdmap e258: 8 total, 8 up, 8 in 2023-10-01T09:23:42.698 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:42.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:42.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:42.803 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:23:42.831 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:42.876 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:23:42.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:42.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:23:42.984 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:43.011 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:43.056 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:43.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:43.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:43.165 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:43.191 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:43.232 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:43.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:43.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:43.338 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:43.363 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:43.405 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:43.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:43.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:43.494 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:43.513 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:43.536 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:43 smithi190 ceph-mon[160435]: failed to encode map e258 with expected crc 2023-10-01T09:23:43.536 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:43 smithi190 ceph-mon[160435]: failed to encode map e258 with expected crc 2023-10-01T09:23:43.536 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:43 smithi190 ceph-mon[160435]: failed to encode map e258 with expected crc 2023-10-01T09:23:43.539 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:43.559 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:43 smithi138 ceph-mon[192513]: failed to encode map e258 with expected crc 2023-10-01T09:23:43.559 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:43 smithi138 ceph-mon[192513]: failed to encode map e258 with expected crc 2023-10-01T09:23:43.559 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:43 smithi138 ceph-mon[192513]: failed to encode map e258 with expected crc 2023-10-01T09:23:43.560 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:43 smithi138 ceph-mon[185886]: failed to encode map e258 with expected crc 2023-10-01T09:23:43.560 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:43 smithi138 ceph-mon[185886]: failed to encode map e258 with expected crc 2023-10-01T09:23:43.560 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:43 smithi138 ceph-mon[185886]: failed to encode map e258 with expected crc 2023-10-01T09:23:43.580 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:43.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:43.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:43.685 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:43.716 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:43.733 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:43.768 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:23:43.769 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:23:43.829 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:43.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:43.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:43.932 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:43.960 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:44.285 INFO:teuthology.orchestra.run.smithi138.stderr:pool 'unique_pool_2' created 2023-10-01T09:23:44.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:44 smithi138 ceph-mon[185886]: pgmap v650: 73 pgs: 73 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 69 KiB/s, 0 objects/s recovering 2023-10-01T09:23:44.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:44 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1932270242' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-01T09:23:44.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:44 smithi138 ceph-mon[192513]: pgmap v650: 73 pgs: 73 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 69 KiB/s, 0 objects/s recovering 2023-10-01T09:23:44.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:44 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1932270242' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-01T09:23:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:44 smithi190 ceph-mon[160435]: pgmap v650: 73 pgs: 73 active+clean; 4.6 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 69 KiB/s, 0 objects/s recovering 2023-10-01T09:23:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:44 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1932270242' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-01T09:23:44.762 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:23:45.160 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:23:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:23:44] "GET /metrics HTTP/1.1" 200 34041 "" "Prometheus/2.33.4" 2023-10-01T09:23:45.405 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-10-01T09:23:45.408 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:23:45.431 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[185886]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.431 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[185886]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.431 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[185886]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.431 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[185886]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.431 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[185886]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.432 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1932270242' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-01T09:23:45.432 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[185886]: osdmap e259: 8 total, 8 up, 8 in 2023-10-01T09:23:45.432 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[185886]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.432 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[185886]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.432 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[185886]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.432 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[185886]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.433 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[192513]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.433 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[192513]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.433 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[192513]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.433 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[192513]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.433 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[192513]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.433 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1932270242' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-01T09:23:45.433 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[192513]: osdmap e259: 8 total, 8 up, 8 in 2023-10-01T09:23:45.433 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[192513]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.433 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[192513]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.433 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[192513]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.433 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:45 smithi138 ceph-mon[192513]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:45 smithi190 ceph-mon[160435]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:45 smithi190 ceph-mon[160435]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:45 smithi190 ceph-mon[160435]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:45 smithi190 ceph-mon[160435]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:45 smithi190 ceph-mon[160435]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:45 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1932270242' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-01T09:23:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:45 smithi190 ceph-mon[160435]: osdmap e259: 8 total, 8 up, 8 in 2023-10-01T09:23:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:45 smithi190 ceph-mon[160435]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:45 smithi190 ceph-mon[160435]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:45 smithi190 ceph-mon[160435]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:45 smithi190 ceph-mon[160435]: failed to encode map e259 with expected crc 2023-10-01T09:23:45.669 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:45.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:45.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:45.773 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:45.806 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:45.954 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:45.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:45.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:46.058 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:46.090 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:46.135 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:46.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:46.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:46.241 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:46.269 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:46.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:46 smithi138 ceph-mon[192513]: failed to encode map e260 with expected crc 2023-10-01T09:23:46.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:46 smithi138 ceph-mon[192513]: osdmap e260: 8 total, 8 up, 8 in 2023-10-01T09:23:46.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:46 smithi138 ceph-mon[192513]: failed to encode map e260 with expected crc 2023-10-01T09:23:46.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:46 smithi138 ceph-mon[192513]: failed to encode map e260 with expected crc 2023-10-01T09:23:46.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:46 smithi138 ceph-mon[192513]: failed to encode map e260 with expected crc 2023-10-01T09:23:46.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:46 smithi138 ceph-mon[192513]: pgmap v653: 89 pgs: 2 creating+peering, 13 unknown, 74 active+clean; 4.6 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T09:23:46.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:46 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:46.314 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:46.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:46.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:46.422 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:46.446 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:46 smithi190 ceph-mon[160435]: failed to encode map e260 with expected crc 2023-10-01T09:23:46.446 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:46 smithi190 ceph-mon[160435]: osdmap e260: 8 total, 8 up, 8 in 2023-10-01T09:23:46.446 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:46 smithi190 ceph-mon[160435]: failed to encode map e260 with expected crc 2023-10-01T09:23:46.446 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:46 smithi190 ceph-mon[160435]: failed to encode map e260 with expected crc 2023-10-01T09:23:46.446 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:46 smithi190 ceph-mon[160435]: failed to encode map e260 with expected crc 2023-10-01T09:23:46.446 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:46 smithi190 ceph-mon[160435]: pgmap v653: 89 pgs: 2 creating+peering, 13 unknown, 74 active+clean; 4.6 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T09:23:46.446 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:46 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:46.448 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:46.490 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:46.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:46.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:46.556 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:46 smithi138 ceph-mon[185886]: failed to encode map e260 with expected crc 2023-10-01T09:23:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:46 smithi138 ceph-mon[185886]: osdmap e260: 8 total, 8 up, 8 in 2023-10-01T09:23:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:46 smithi138 ceph-mon[185886]: failed to encode map e260 with expected crc 2023-10-01T09:23:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:46 smithi138 ceph-mon[185886]: failed to encode map e260 with expected crc 2023-10-01T09:23:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:46 smithi138 ceph-mon[185886]: failed to encode map e260 with expected crc 2023-10-01T09:23:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:46 smithi138 ceph-mon[185886]: pgmap v653: 89 pgs: 2 creating+peering, 13 unknown, 74 active+clean; 4.6 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T09:23:46.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:46 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:46.596 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:46.629 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:46.635 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:46.721 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:23:46.722 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:46.745 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:23:46.768 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:23:46.788 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:23:46.804 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:46.808 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:23:46.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:46.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:46.826 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:23:46.860 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:23:46.863 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:23:46.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:23:46.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:23:46.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:23:46.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:23:46.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:23:46.866 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:23:46.866 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:23:46.866 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:23:46.910 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:46.941 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:47.138 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:47.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:47.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:47.240 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:23:47.241 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:23:47.244 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:47.273 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:47.417 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:47 smithi138 ceph-mon[185886]: failed to encode map e260 with expected crc 2023-10-01T09:23:47.417 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:47 smithi138 ceph-mon[185886]: failed to encode map e261 with expected crc 2023-10-01T09:23:47.418 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:47 smithi138 ceph-mon[185886]: osdmap e261: 8 total, 8 up, 8 in 2023-10-01T09:23:47.418 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:47 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3463670135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:47.418 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:47 smithi138 ceph-mon[185886]: failed to encode map e261 with expected crc 2023-10-01T09:23:47.418 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:47 smithi138 ceph-mon[192513]: failed to encode map e260 with expected crc 2023-10-01T09:23:47.418 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:47 smithi138 ceph-mon[192513]: failed to encode map e261 with expected crc 2023-10-01T09:23:47.418 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:47 smithi138 ceph-mon[192513]: osdmap e261: 8 total, 8 up, 8 in 2023-10-01T09:23:47.418 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:47 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3463670135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:47.419 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:47 smithi138 ceph-mon[192513]: failed to encode map e261 with expected crc 2023-10-01T09:23:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:47 smithi190 ceph-mon[160435]: failed to encode map e260 with expected crc 2023-10-01T09:23:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:47 smithi190 ceph-mon[160435]: failed to encode map e261 with expected crc 2023-10-01T09:23:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:47 smithi190 ceph-mon[160435]: osdmap e261: 8 total, 8 up, 8 in 2023-10-01T09:23:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:47 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3463670135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:47 smithi190 ceph-mon[160435]: failed to encode map e261 with expected crc 2023-10-01T09:23:48.263 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":654,"stamp":"2023-10-01T09:23:46.289379+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6259,"num_read_kb":12659,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":186,"num_bytes_recovered":2373496,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":231,"acting":231,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":234,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2799024,"kb_used_data":389616,"kb_used_omap":38,"kb_used_meta":2409305,"kb_avail":747162192,"statfs":{"total":767960285184,"available":765094084608,"internally_reserved":0,"allocated":398966784,"data_stored":391423425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39221,"internal_metadata":2467129035},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.797487"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":128,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.829831+0000","last_change":"2023-10-01T09:19:30.091981+0000","last_active":"2023-10-01T09:23:35.829831+0000","last_peered":"2023-10-01T09:23:35.829831+0000","last_clean":"2023-10-01T09:23:35.829831+0000","last_became_active":"2023-10-01T09:19:30.091531+0000","last_became_peered":"2023-10-01T09:19:30.091531+0000","last_unstale":"2023-10-01T09:23:35.829831+0000","last_undegraded":"2023-10-01T09:23:35.829831+0000","last_fullsized":"2023-10-01T09:23:35.829831+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:28:13.442498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":32,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.158099+0000","last_change":"2023-10-01T09:22:40.311894+0000","last_active":"2023-10-01T09:23:35.158099+0000","last_peered":"2023-10-01T09:23:35.158099+0000","last_clean":"2023-10-01T09:23:35.158099+0000","last_became_active":"2023-10-01T09:22:40.311581+0000","last_became_peered":"2023-10-01T09:22:40.311581+0000","last_unstale":"2023-10-01T09:23:35.158099+0000","last_undegraded":"2023-10-01T09:23:35.158099+0000","last_fullsized":"2023-10-01T09:23:35.158099+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:52:27.649535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":204,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.831657+0000","last_change":"2023-10-01T09:13:13.551596+0000","last_active":"2023-10-01T09:23:35.831657+0000","last_peered":"2023-10-01T09:23:35.831657+0000","last_clean":"2023-10-01T09:23:35.831657+0000","last_became_active":"2023-10-01T09:13:13.551300+0000","last_became_peered":"2023-10-01T09:13:13.551300+0000","last_unstale":"2023-10-01T09:23:35.831657+0000","last_undegraded":"2023-10-01T09:23:35.831657+0000","last_fullsized":"2023-10-01T09:23:35.831657+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:57:45.081780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":260,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130951+0000","last_change":"2023-10-01T09:23:10.611220+0000","last_active":"2023-10-01T09:23:32.130951+0000","last_peered":"2023-10-01T09:23:32.130951+0000","last_clean":"2023-10-01T09:23:32.130951+0000","last_became_active":"2023-10-01T09:23:10.610573+0000","last_became_peered":"2023-10-01T09:23:10.610573+0000","last_unstale":"2023-10-01T09:23:32.130951+0000","last_undegraded":"2023-10-01T09:23:32.130951+0000","last_fullsized":"2023-10-01T09:23:32.130951+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:58:32.020825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":32,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.155722+0000","last_change":"2023-10-01T09:22:40.172183+0000","last_active":"2023-10-01T09:23:35.155722+0000","last_peered":"2023-10-01T09:23:35.155722+0000","last_clean":"2023-10-01T09:23:35.155722+0000","last_became_active":"2023-10-01T09:22:40.170164+0000","last_became_peered":"2023-10-01T09:22:40.170164+0000","last_unstale":"2023-10-01T09:23:35.155722+0000","last_undegraded":"2023-10-01T09:23:35.155722+0000","last_fullsized":"2023-10-01T09:23:35.155722+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:33:10.407999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":206,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.284780+0000","last_change":"2023-10-01T09:22:03.153196+0000","last_active":"2023-10-01T09:23:44.284780+0000","last_peered":"2023-10-01T09:23:44.284780+0000","last_clean":"2023-10-01T09:23:44.284780+0000","last_became_active":"2023-10-01T09:22:03.151768+0000","last_became_peered":"2023-10-01T09:22:03.151768+0000","last_unstale":"2023-10-01T09:23:44.284780+0000","last_undegraded":"2023-10-01T09:23:44.284780+0000","last_fullsized":"2023-10-01T09:23:44.284780+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":262,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.131980+0000","last_change":"2023-10-01T09:22:40.175704+0000","last_active":"2023-10-01T09:23:32.131980+0000","last_peered":"2023-10-01T09:23:32.131980+0000","last_clean":"2023-10-01T09:23:32.131980+0000","last_became_active":"2023-10-01T09:22:40.175369+0000","last_became_peered":"2023-10-01T09:22:40.175369+0000","last_unstale":"2023-10-01T09:23:32.131980+0000","last_undegraded":"2023-10-01T09:23:32.131980+0000","last_fullsized":"2023-10-01T09:23:32.131980+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:07.773407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":248,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.155819+0000","last_change":"2023-10-01T09:21:02.139865+0000","last_active":"2023-10-01T09:23:35.155819+0000","last_peered":"2023-10-01T09:23:35.155819+0000","last_clean":"2023-10-01T09:23:35.155819+0000","last_became_active":"2023-10-01T09:21:02.139507+0000","last_became_peered":"2023-10-01T09:21:02.139507+0000","last_unstale":"2023-10-01T09:23:35.155819+0000","last_undegraded":"2023-10-01T09:23:35.155819+0000","last_fullsized":"2023-10-01T09:23:35.155819+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:46:09.214220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":222,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.285929+0000","last_change":"2023-10-01T09:23:10.610494+0000","last_active":"2023-10-01T09:23:44.285929+0000","last_peered":"2023-10-01T09:23:44.285929+0000","last_clean":"2023-10-01T09:23:44.285929+0000","last_became_active":"2023-10-01T09:23:10.610037+0000","last_became_peered":"2023-10-01T09:23:10.610037+0000","last_unstale":"2023-10-01T09:23:44.285929+0000","last_undegraded":"2023-10-01T09:23:44.285929+0000","last_fullsized":"2023-10-01T09:23:44.285929+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":274,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.158286+0000","last_change":"2023-10-01T09:23:10.613042+0000","last_active":"2023-10-01T09:23:35.158286+0000","last_peered":"2023-10-01T09:23:35.158286+0000","last_clean":"2023-10-01T09:23:35.158286+0000","last_became_active":"2023-10-01T09:23:10.612686+0000","last_became_peered":"2023-10-01T09:23:10.612686+0000","last_unstale":"2023-10-01T09:23:35.158286+0000","last_undegraded":"2023-10-01T09:23:35.158286+0000","last_fullsized":"2023-10-01T09:23:35.158286+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:12:00.239369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":26,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136170+0000","last_change":"2023-10-01T09:23:10.616550+0000","last_active":"2023-10-01T09:23:32.136170+0000","last_peered":"2023-10-01T09:23:32.136170+0000","last_clean":"2023-10-01T09:23:32.136170+0000","last_became_active":"2023-10-01T09:23:10.611509+0000","last_became_peered":"2023-10-01T09:23:10.611509+0000","last_unstale":"2023-10-01T09:23:32.136170+0000","last_undegraded":"2023-10-01T09:23:32.136170+0000","last_fullsized":"2023-10-01T09:23:32.136170+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":258,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.829535+0000","last_change":"2023-10-01T09:21:02.158958+0000","last_active":"2023-10-01T09:23:35.829535+0000","last_peered":"2023-10-01T09:23:35.829535+0000","last_clean":"2023-10-01T09:23:35.829535+0000","last_became_active":"2023-10-01T09:21:02.158491+0000","last_became_peered":"2023-10-01T09:21:02.158491+0000","last_unstale":"2023-10-01T09:23:35.829535+0000","last_undegraded":"2023-10-01T09:23:35.829535+0000","last_fullsized":"2023-10-01T09:23:35.829535+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:21:48.885982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":202,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136212+0000","last_change":"2023-10-01T09:21:02.163403+0000","last_active":"2023-10-01T09:23:32.136212+0000","last_peered":"2023-10-01T09:23:32.136212+0000","last_clean":"2023-10-01T09:23:32.136212+0000","last_became_active":"2023-10-01T09:21:02.121680+0000","last_became_peered":"2023-10-01T09:21:02.121680+0000","last_unstale":"2023-10-01T09:23:32.136212+0000","last_undegraded":"2023-10-01T09:23:32.136212+0000","last_fullsized":"2023-10-01T09:23:32.136212+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":228,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.283958+0000","last_change":"2023-10-01T09:21:03.349475+0000","last_active":"2023-10-01T09:23:44.283958+0000","last_peered":"2023-10-01T09:23:44.283958+0000","last_clean":"2023-10-01T09:23:44.283958+0000","last_became_active":"2023-10-01T09:21:03.348251+0000","last_became_peered":"2023-10-01T09:21:03.348251+0000","last_unstale":"2023-10-01T09:23:44.283958+0000","last_undegraded":"2023-10-01T09:23:44.283958+0000","last_fullsized":"2023-10-01T09:23:44.283958+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":285,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136022+0000","last_change":"2023-10-01T09:19:30.182046+0000","last_active":"2023-10-01T09:23:32.136022+0000","last_peered":"2023-10-01T09:23:32.136022+0000","last_clean":"2023-10-01T09:23:32.136022+0000","last_became_active":"2023-10-01T09:19:30.094579+0000","last_became_peered":"2023-10-01T09:19:30.094579+0000","last_unstale":"2023-10-01T09:23:32.136022+0000","last_undegraded":"2023-10-01T09:23:32.136022+0000","last_fullsized":"2023-10-01T09:23:32.136022+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":239,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.157532+0000","last_change":"2023-10-01T09:19:57.493670+0000","last_active":"2023-10-01T09:23:35.157532+0000","last_peered":"2023-10-01T09:23:35.157532+0000","last_clean":"2023-10-01T09:23:35.157532+0000","last_became_active":"2023-10-01T09:19:57.493349+0000","last_became_peered":"2023-10-01T09:19:57.493349+0000","last_unstale":"2023-10-01T09:23:35.157532+0000","last_undegraded":"2023-10-01T09:23:35.157532+0000","last_fullsized":"2023-10-01T09:23:35.157532+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:26:56.468779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":244,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130735+0000","last_change":"2023-10-01T09:23:10.607388+0000","last_active":"2023-10-01T09:23:32.130735+0000","last_peered":"2023-10-01T09:23:32.130735+0000","last_clean":"2023-10-01T09:23:32.130735+0000","last_became_active":"2023-10-01T09:23:10.607084+0000","last_became_peered":"2023-10-01T09:23:10.607084+0000","last_unstale":"2023-10-01T09:23:32.130735+0000","last_undegraded":"2023-10-01T09:23:32.130735+0000","last_fullsized":"2023-10-01T09:23:32.130735+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:19.669393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":198,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.285744+0000","last_change":"2023-10-01T09:22:02.185127+0000","last_active":"2023-10-01T09:23:44.285744+0000","last_peered":"2023-10-01T09:23:44.285744+0000","last_clean":"2023-10-01T09:23:44.285744+0000","last_became_active":"2023-10-01T09:22:02.184059+0000","last_became_peered":"2023-10-01T09:22:02.184059+0000","last_unstale":"2023-10-01T09:23:44.285744+0000","last_undegraded":"2023-10-01T09:23:44.285744+0000","last_fullsized":"2023-10-01T09:23:44.285744+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":226,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.135928+0000","last_change":"2023-10-01T09:21:02.121558+0000","last_active":"2023-10-01T09:23:32.135928+0000","last_peered":"2023-10-01T09:23:32.135928+0000","last_clean":"2023-10-01T09:23:32.135928+0000","last_became_active":"2023-10-01T09:21:02.121153+0000","last_became_peered":"2023-10-01T09:21:02.121153+0000","last_unstale":"2023-10-01T09:23:32.135928+0000","last_undegraded":"2023-10-01T09:23:32.135928+0000","last_fullsized":"2023-10-01T09:23:32.135928+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":226,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130884+0000","last_change":"2023-10-01T09:23:10.611400+0000","last_active":"2023-10-01T09:23:32.130884+0000","last_peered":"2023-10-01T09:23:32.130884+0000","last_clean":"2023-10-01T09:23:32.130884+0000","last_became_active":"2023-10-01T09:23:10.610928+0000","last_became_peered":"2023-10-01T09:23:10.610928+0000","last_unstale":"2023-10-01T09:23:32.130884+0000","last_undegraded":"2023-10-01T09:23:32.130884+0000","last_fullsized":"2023-10-01T09:23:32.130884+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:17.141540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":206,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.283835+0000","last_change":"2023-10-01T09:22:03.163438+0000","last_active":"2023-10-01T09:23:44.283835+0000","last_peered":"2023-10-01T09:23:44.283835+0000","last_clean":"2023-10-01T09:23:44.283835+0000","last_became_active":"2023-10-01T09:22:03.162821+0000","last_became_peered":"2023-10-01T09:22:03.162821+0000","last_unstale":"2023-10-01T09:23:44.283835+0000","last_undegraded":"2023-10-01T09:23:44.283835+0000","last_fullsized":"2023-10-01T09:23:44.283835+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":41,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.158005+0000","last_change":"2023-10-01T09:23:10.613386+0000","last_active":"2023-10-01T09:23:35.158005+0000","last_peered":"2023-10-01T09:23:35.158005+0000","last_clean":"2023-10-01T09:23:35.158005+0000","last_became_active":"2023-10-01T09:23:10.613046+0000","last_became_peered":"2023-10-01T09:23:10.613046+0000","last_unstale":"2023-10-01T09:23:35.158005+0000","last_undegraded":"2023-10-01T09:23:35.158005+0000","last_fullsized":"2023-10-01T09:23:35.158005+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:23:01.448841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T09:23:44.270133+0000","last_change":"2023-10-01T09:23:44.270133+0000","last_active":"2023-10-01T09:23:44.270133+0000","last_peered":"2023-10-01T09:23:44.270133+0000","last_clean":"2023-10-01T09:23:44.270133+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:23:44.270133+0000","last_undegraded":"2023-10-01T09:23:44.270133+0000","last_fullsized":"2023-10-01T09:23:44.270133+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":24,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136305+0000","last_change":"2023-10-01T09:23:10.601126+0000","last_active":"2023-10-01T09:23:32.136305+0000","last_peered":"2023-10-01T09:23:32.136305+0000","last_clean":"2023-10-01T09:23:32.136305+0000","last_became_active":"2023-10-01T09:23:10.598657+0000","last_became_peered":"2023-10-01T09:23:10.598657+0000","last_unstale":"2023-10-01T09:23:32.136305+0000","last_undegraded":"2023-10-01T09:23:32.136305+0000","last_fullsized":"2023-10-01T09:23:32.136305+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":195,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.285155+0000","last_change":"2023-10-01T09:23:10.596661+0000","last_active":"2023-10-01T09:23:44.285155+0000","last_peered":"2023-10-01T09:23:44.285155+0000","last_clean":"2023-10-01T09:23:44.285155+0000","last_became_active":"2023-10-01T09:23:10.596299+0000","last_became_peered":"2023-10-01T09:23:10.596299+0000","last_unstale":"2023-10-01T09:23:44.285155+0000","last_undegraded":"2023-10-01T09:23:44.285155+0000","last_fullsized":"2023-10-01T09:23:44.285155+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T09:23:44.270133+0000","last_change":"2023-10-01T09:23:44.270133+0000","last_active":"2023-10-01T09:23:44.270133+0000","last_peered":"2023-10-01T09:23:44.270133+0000","last_clean":"2023-10-01T09:23:44.270133+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:23:44.270133+0000","last_undegraded":"2023-10-01T09:23:44.270133+0000","last_fullsized":"2023-10-01T09:23:44.270133+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":225,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.137399+0000","last_change":"2023-10-01T09:12:10.003552+0000","last_active":"2023-10-01T09:23:32.137399+0000","last_peered":"2023-10-01T09:23:32.137399+0000","last_clean":"2023-10-01T09:23:32.137399+0000","last_became_active":"2023-10-01T09:12:10.003131+0000","last_became_peered":"2023-10-01T09:12:10.003131+0000","last_unstale":"2023-10-01T09:23:32.137399+0000","last_undegraded":"2023-10-01T09:23:32.137399+0000","last_fullsized":"2023-10-01T09:23:32.137399+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":259,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.831581+0000","last_change":"2023-10-01T09:22:40.167822+0000","last_active":"2023-10-01T09:23:35.831581+0000","last_peered":"2023-10-01T09:23:35.831581+0000","last_clean":"2023-10-01T09:23:35.831581+0000","last_became_active":"2023-10-01T09:22:40.167502+0000","last_became_peered":"2023-10-01T09:22:40.167502+0000","last_unstale":"2023-10-01T09:23:35.831581+0000","last_undegraded":"2023-10-01T09:23:35.831581+0000","last_fullsized":"2023-10-01T09:23:35.831581+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:40:27.950717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T09:23:44.270133+0000","last_change":"2023-10-01T09:23:44.270133+0000","last_active":"2023-10-01T09:23:44.270133+0000","last_peered":"2023-10-01T09:23:44.270133+0000","last_clean":"2023-10-01T09:23:44.270133+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:23:44.270133+0000","last_undegraded":"2023-10-01T09:23:44.270133+0000","last_fullsized":"2023-10-01T09:23:44.270133+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":249,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136459+0000","last_change":"2023-10-01T09:22:03.151056+0000","last_active":"2023-10-01T09:23:32.136459+0000","last_peered":"2023-10-01T09:23:32.136459+0000","last_clean":"2023-10-01T09:23:32.136459+0000","last_became_active":"2023-10-01T09:22:03.150643+0000","last_became_peered":"2023-10-01T09:22:03.150643+0000","last_unstale":"2023-10-01T09:23:32.136459+0000","last_undegraded":"2023-10-01T09:23:32.136459+0000","last_fullsized":"2023-10-01T09:23:32.136459+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":221,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.284386+0000","last_change":"2023-10-01T09:23:10.610311+0000","last_active":"2023-10-01T09:23:44.284386+0000","last_peered":"2023-10-01T09:23:44.284386+0000","last_clean":"2023-10-01T09:23:44.284386+0000","last_became_active":"2023-10-01T09:23:10.609935+0000","last_became_peered":"2023-10-01T09:23:10.609935+0000","last_unstale":"2023-10-01T09:23:44.284386+0000","last_undegraded":"2023-10-01T09:23:44.284386+0000","last_fullsized":"2023-10-01T09:23:44.284386+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":1,"reported_epoch":259,"state":"creating+peering","last_fresh":"2023-10-01T09:23:44.281858+0000","last_change":"2023-10-01T09:23:44.281858+0000","last_active":"2023-10-01T09:23:44.270133+0000","last_peered":"2023-10-01T09:23:44.270133+0000","last_clean":"2023-10-01T09:23:44.270133+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:23:44.281858+0000","last_undegraded":"2023-10-01T09:23:44.281858+0000","last_fullsized":"2023-10-01T09:23:44.281858+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:33:08.487287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":211,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.285862+0000","last_change":"2023-10-01T09:22:40.313521+0000","last_active":"2023-10-01T09:23:44.285862+0000","last_peered":"2023-10-01T09:23:44.285862+0000","last_clean":"2023-10-01T09:23:44.285862+0000","last_became_active":"2023-10-01T09:22:40.312941+0000","last_became_peered":"2023-10-01T09:22:40.312941+0000","last_unstale":"2023-10-01T09:23:44.285862+0000","last_undegraded":"2023-10-01T09:23:44.285862+0000","last_fullsized":"2023-10-01T09:23:44.285862+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":208,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130647+0000","last_change":"2023-10-01T09:12:10.001565+0000","last_active":"2023-10-01T09:23:32.130647+0000","last_peered":"2023-10-01T09:23:32.130647+0000","last_clean":"2023-10-01T09:23:32.130647+0000","last_became_active":"2023-10-01T09:12:10.001275+0000","last_became_peered":"2023-10-01T09:12:10.001275+0000","last_unstale":"2023-10-01T09:23:32.130647+0000","last_undegraded":"2023-10-01T09:23:32.130647+0000","last_fullsized":"2023-10-01T09:23:32.130647+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:39.067046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T09:23:44.270133+0000","last_change":"2023-10-01T09:23:44.270133+0000","last_active":"2023-10-01T09:23:44.270133+0000","last_peered":"2023-10-01T09:23:44.270133+0000","last_clean":"2023-10-01T09:23:44.270133+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:23:44.270133+0000","last_undegraded":"2023-10-01T09:23:44.270133+0000","last_fullsized":"2023-10-01T09:23:44.270133+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":228,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.158156+0000","last_change":"2023-10-01T09:13:13.554321+0000","last_active":"2023-10-01T09:23:35.158156+0000","last_peered":"2023-10-01T09:23:35.158156+0000","last_clean":"2023-10-01T09:23:35.158156+0000","last_became_active":"2023-10-01T09:13:13.553986+0000","last_became_peered":"2023-10-01T09:13:13.553986+0000","last_unstale":"2023-10-01T09:23:35.158156+0000","last_undegraded":"2023-10-01T09:23:35.158156+0000","last_fullsized":"2023-10-01T09:23:35.158156+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:09:12.848269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":189,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.157815+0000","last_change":"2023-10-01T09:21:02.139718+0000","last_active":"2023-10-01T09:23:35.157815+0000","last_peered":"2023-10-01T09:23:35.157815+0000","last_clean":"2023-10-01T09:23:35.157815+0000","last_became_active":"2023-10-01T09:21:02.139406+0000","last_became_peered":"2023-10-01T09:21:02.139406+0000","last_unstale":"2023-10-01T09:23:35.157815+0000","last_undegraded":"2023-10-01T09:23:35.157815+0000","last_fullsized":"2023-10-01T09:23:35.157815+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:10.823093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":1,"reported_epoch":259,"state":"creating+peering","last_fresh":"2023-10-01T09:23:44.281417+0000","last_change":"2023-10-01T09:23:44.281417+0000","last_active":"2023-10-01T09:23:44.270133+0000","last_peered":"2023-10-01T09:23:44.270133+0000","last_clean":"2023-10-01T09:23:44.270133+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:23:44.281417+0000","last_undegraded":"2023-10-01T09:23:44.281417+0000","last_fullsized":"2023-10-01T09:23:44.281417+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:27:28.625644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":185,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.829682+0000","last_change":"2023-10-01T09:21:02.136776+0000","last_active":"2023-10-01T09:23:35.829682+0000","last_peered":"2023-10-01T09:23:35.829682+0000","last_clean":"2023-10-01T09:23:35.829682+0000","last_became_active":"2023-10-01T09:21:02.136476+0000","last_became_peered":"2023-10-01T09:21:02.136476+0000","last_unstale":"2023-10-01T09:23:35.829682+0000","last_undegraded":"2023-10-01T09:23:35.829682+0000","last_fullsized":"2023-10-01T09:23:35.829682+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:44:41.289825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":257,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-10-01T09:23:42.743070+0000","last_change":"2023-10-01T09:20:15.505658+0000","last_active":"2023-10-01T09:23:42.743070+0000","last_peered":"2023-10-01T09:23:42.743070+0000","last_clean":"2023-10-01T09:23:42.743070+0000","last_became_active":"2023-10-01T09:20:15.505361+0000","last_became_peered":"2023-10-01T09:20:15.505361+0000","last_unstale":"2023-10-01T09:23:42.743070+0000","last_undegraded":"2023-10-01T09:23:42.743070+0000","last_fullsized":"2023-10-01T09:23:42.743070+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:22:16.176432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T09:23:44.270133+0000","last_change":"2023-10-01T09:23:44.270133+0000","last_active":"2023-10-01T09:23:44.270133+0000","last_peered":"2023-10-01T09:23:44.270133+0000","last_clean":"2023-10-01T09:23:44.270133+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:23:44.270133+0000","last_undegraded":"2023-10-01T09:23:44.270133+0000","last_fullsized":"2023-10-01T09:23:44.270133+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":126,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.158181+0000","last_change":"2023-10-01T09:19:30.092681+0000","last_active":"2023-10-01T09:23:35.158181+0000","last_peered":"2023-10-01T09:23:35.158181+0000","last_clean":"2023-10-01T09:23:35.158181+0000","last_became_active":"2023-10-01T09:19:30.092394+0000","last_became_peered":"2023-10-01T09:19:30.092394+0000","last_unstale":"2023-10-01T09:23:35.158181+0000","last_undegraded":"2023-10-01T09:23:35.158181+0000","last_fullsized":"2023-10-01T09:23:35.158181+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:41:50.127253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":53,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.286032+0000","last_change":"2023-10-01T09:22:03.144199+0000","last_active":"2023-10-01T09:23:44.286032+0000","last_peered":"2023-10-01T09:23:44.286032+0000","last_clean":"2023-10-01T09:23:44.286032+0000","last_became_active":"2023-10-01T09:22:03.143883+0000","last_became_peered":"2023-10-01T09:22:03.143883+0000","last_unstale":"2023-10-01T09:23:44.286032+0000","last_undegraded":"2023-10-01T09:23:44.286032+0000","last_fullsized":"2023-10-01T09:23:44.286032+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T09:23:44.270133+0000","last_change":"2023-10-01T09:23:44.270133+0000","last_active":"2023-10-01T09:23:44.270133+0000","last_peered":"2023-10-01T09:23:44.270133+0000","last_clean":"2023-10-01T09:23:44.270133+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:23:44.270133+0000","last_undegraded":"2023-10-01T09:23:44.270133+0000","last_fullsized":"2023-10-01T09:23:44.270133+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":32,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.155723+0000","last_change":"2023-10-01T09:22:40.316008+0000","last_active":"2023-10-01T09:23:35.155723+0000","last_peered":"2023-10-01T09:23:35.155723+0000","last_clean":"2023-10-01T09:23:35.155723+0000","last_became_active":"2023-10-01T09:22:40.315602+0000","last_became_peered":"2023-10-01T09:22:40.315602+0000","last_unstale":"2023-10-01T09:23:35.155723+0000","last_undegraded":"2023-10-01T09:23:35.155723+0000","last_fullsized":"2023-10-01T09:23:35.155723+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:39:37.485241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":123,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.829307+0000","last_change":"2023-10-01T09:18:31.419430+0000","last_active":"2023-10-01T09:23:35.829307+0000","last_peered":"2023-10-01T09:23:35.829307+0000","last_clean":"2023-10-01T09:23:35.829307+0000","last_became_active":"2023-10-01T09:18:31.419189+0000","last_became_peered":"2023-10-01T09:18:31.419189+0000","last_unstale":"2023-10-01T09:23:35.829307+0000","last_undegraded":"2023-10-01T09:23:35.829307+0000","last_fullsized":"2023-10-01T09:23:35.829307+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:08:48.247926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T09:23:44.270133+0000","last_change":"2023-10-01T09:23:44.270133+0000","last_active":"2023-10-01T09:23:44.270133+0000","last_peered":"2023-10-01T09:23:44.270133+0000","last_clean":"2023-10-01T09:23:44.270133+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:23:44.270133+0000","last_undegraded":"2023-10-01T09:23:44.270133+0000","last_fullsized":"2023-10-01T09:23:44.270133+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1075,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.832077+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:23:35.832077+0000","last_peered":"2023-10-01T09:23:35.832077+0000","last_clean":"2023-10-01T09:23:35.832077+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:23:35.832077+0000","last_undegraded":"2023-10-01T09:23:35.832077+0000","last_fullsized":"2023-10-01T09:23:35.832077+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":255,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.285692+0000","last_change":"2023-10-01T09:22:40.170877+0000","last_active":"2023-10-01T09:23:44.285692+0000","last_peered":"2023-10-01T09:23:44.285692+0000","last_clean":"2023-10-01T09:23:44.285692+0000","last_became_active":"2023-10-01T09:22:40.170578+0000","last_became_peered":"2023-10-01T09:22:40.170578+0000","last_unstale":"2023-10-01T09:23:44.285692+0000","last_undegraded":"2023-10-01T09:23:44.285692+0000","last_fullsized":"2023-10-01T09:23:44.285692+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":205,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.829652+0000","last_change":"2023-10-01T09:22:02.151482+0000","last_active":"2023-10-01T09:23:35.829652+0000","last_peered":"2023-10-01T09:23:35.829652+0000","last_clean":"2023-10-01T09:23:35.829652+0000","last_became_active":"2023-10-01T09:22:02.151180+0000","last_became_peered":"2023-10-01T09:22:02.151180+0000","last_unstale":"2023-10-01T09:23:35.829652+0000","last_undegraded":"2023-10-01T09:23:35.829652+0000","last_fullsized":"2023-10-01T09:23:35.829652+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:11.745198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":9,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-01T09:23:45.303757+0000","last_change":"2023-10-01T09:23:45.303757+0000","last_active":"2023-10-01T09:23:45.303757+0000","last_peered":"2023-10-01T09:23:45.303757+0000","last_clean":"2023-10-01T09:23:45.303757+0000","last_became_active":"2023-10-01T09:23:45.303439+0000","last_became_peered":"2023-10-01T09:23:45.303439+0000","last_unstale":"2023-10-01T09:23:45.303757+0000","last_undegraded":"2023-10-01T09:23:45.303757+0000","last_fullsized":"2023-10-01T09:23:45.303757+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":921,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.285915+0000","last_change":"2023-10-01T09:22:03.258449+0000","last_active":"2023-10-01T09:23:44.285915+0000","last_peered":"2023-10-01T09:23:44.285915+0000","last_clean":"2023-10-01T09:23:44.285915+0000","last_became_active":"2023-10-01T09:22:03.257766+0000","last_became_peered":"2023-10-01T09:22:03.257766+0000","last_unstale":"2023-10-01T09:23:44.285915+0000","last_undegraded":"2023-10-01T09:23:44.285915+0000","last_fullsized":"2023-10-01T09:23:44.285915+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":175,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130917+0000","last_change":"2023-10-01T09:21:02.121225+0000","last_active":"2023-10-01T09:23:32.130917+0000","last_peered":"2023-10-01T09:23:32.130917+0000","last_clean":"2023-10-01T09:23:32.130917+0000","last_became_active":"2023-10-01T09:21:02.120911+0000","last_became_peered":"2023-10-01T09:21:02.120911+0000","last_unstale":"2023-10-01T09:23:32.130917+0000","last_undegraded":"2023-10-01T09:23:32.130917+0000","last_fullsized":"2023-10-01T09:23:32.130917+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:34:44.385101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":94,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.285570+0000","last_change":"2023-10-01T09:22:40.172426+0000","last_active":"2023-10-01T09:23:44.285570+0000","last_peered":"2023-10-01T09:23:44.285570+0000","last_clean":"2023-10-01T09:23:44.285570+0000","last_became_active":"2023-10-01T09:22:40.172160+0000","last_became_peered":"2023-10-01T09:22:40.172160+0000","last_unstale":"2023-10-01T09:23:44.285570+0000","last_undegraded":"2023-10-01T09:23:44.285570+0000","last_fullsized":"2023-10-01T09:23:44.285570+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T09:23:44.270133+0000","last_change":"2023-10-01T09:23:44.270133+0000","last_active":"2023-10-01T09:23:44.270133+0000","last_peered":"2023-10-01T09:23:44.270133+0000","last_clean":"2023-10-01T09:23:44.270133+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:23:44.270133+0000","last_undegraded":"2023-10-01T09:23:44.270133+0000","last_fullsized":"2023-10-01T09:23:44.270133+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":584,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-10-01T09:23:42.743464+0000","last_change":"2023-10-01T09:22:40.183431+0000","last_active":"2023-10-01T09:23:42.743464+0000","last_peered":"2023-10-01T09:23:42.743464+0000","last_clean":"2023-10-01T09:23:42.743464+0000","last_became_active":"2023-10-01T09:22:40.171414+0000","last_became_peered":"2023-10-01T09:22:40.171414+0000","last_unstale":"2023-10-01T09:23:42.743464+0000","last_undegraded":"2023-10-01T09:23:42.743464+0000","last_fullsized":"2023-10-01T09:23:42.743464+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:56:42.904846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":259,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.829551+0000","last_change":"2023-10-01T09:21:02.140363+0000","last_active":"2023-10-01T09:23:35.829551+0000","last_peered":"2023-10-01T09:23:35.829551+0000","last_clean":"2023-10-01T09:23:35.829551+0000","last_became_active":"2023-10-01T09:21:02.137198+0000","last_became_peered":"2023-10-01T09:21:02.137198+0000","last_unstale":"2023-10-01T09:23:35.829551+0000","last_undegraded":"2023-10-01T09:23:35.829551+0000","last_fullsized":"2023-10-01T09:23:35.829551+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:38:38.043146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":120,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.831553+0000","last_change":"2023-10-01T09:22:02.156724+0000","last_active":"2023-10-01T09:23:35.831553+0000","last_peered":"2023-10-01T09:23:35.831553+0000","last_clean":"2023-10-01T09:23:35.831553+0000","last_became_active":"2023-10-01T09:22:02.156451+0000","last_became_peered":"2023-10-01T09:22:02.156451+0000","last_unstale":"2023-10-01T09:23:35.831553+0000","last_undegraded":"2023-10-01T09:23:35.831553+0000","last_fullsized":"2023-10-01T09:23:35.831553+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:29:05.232127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T09:23:44.270133+0000","last_change":"2023-10-01T09:23:44.270133+0000","last_active":"2023-10-01T09:23:44.270133+0000","last_peered":"2023-10-01T09:23:44.270133+0000","last_clean":"2023-10-01T09:23:44.270133+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:23:44.270133+0000","last_undegraded":"2023-10-01T09:23:44.270133+0000","last_fullsized":"2023-10-01T09:23:44.270133+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1318,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.285623+0000","last_change":"2023-10-01T09:23:13.197266+0000","last_active":"2023-10-01T09:23:44.285623+0000","last_peered":"2023-10-01T09:23:44.285623+0000","last_clean":"2023-10-01T09:23:44.285623+0000","last_became_active":"2023-10-01T09:23:13.196795+0000","last_became_peered":"2023-10-01T09:23:13.196795+0000","last_unstale":"2023-10-01T09:23:44.285623+0000","last_undegraded":"2023-10-01T09:23:44.285623+0000","last_fullsized":"2023-10-01T09:23:44.285623+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":420,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.139546+0000","last_change":"2023-10-01T09:23:32.139254+0000","last_active":"2023-10-01T09:23:32.139546+0000","last_peered":"2023-10-01T09:23:32.139546+0000","last_clean":"2023-10-01T09:23:32.139546+0000","last_became_active":"2023-10-01T09:23:32.138752+0000","last_became_peered":"2023-10-01T09:23:32.138752+0000","last_unstale":"2023-10-01T09:23:32.139546+0000","last_undegraded":"2023-10-01T09:23:32.139546+0000","last_fullsized":"2023-10-01T09:23:32.139546+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":32,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.155918+0000","last_change":"2023-10-01T09:22:40.317678+0000","last_active":"2023-10-01T09:23:35.155918+0000","last_peered":"2023-10-01T09:23:35.155918+0000","last_clean":"2023-10-01T09:23:35.155918+0000","last_became_active":"2023-10-01T09:22:40.317460+0000","last_became_peered":"2023-10-01T09:22:40.317460+0000","last_unstale":"2023-10-01T09:23:35.155918+0000","last_undegraded":"2023-10-01T09:23:35.155918+0000","last_fullsized":"2023-10-01T09:23:35.155918+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:07:41.681679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":129,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136543+0000","last_change":"2023-10-01T09:21:02.125167+0000","last_active":"2023-10-01T09:23:32.136543+0000","last_peered":"2023-10-01T09:23:32.136543+0000","last_clean":"2023-10-01T09:23:32.136543+0000","last_became_active":"2023-10-01T09:21:02.124791+0000","last_became_peered":"2023-10-01T09:21:02.124791+0000","last_unstale":"2023-10-01T09:23:32.136543+0000","last_undegraded":"2023-10-01T09:23:32.136543+0000","last_fullsized":"2023-10-01T09:23:32.136543+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T09:23:44.270133+0000","last_change":"2023-10-01T09:23:44.270133+0000","last_active":"2023-10-01T09:23:44.270133+0000","last_peered":"2023-10-01T09:23:44.270133+0000","last_clean":"2023-10-01T09:23:44.270133+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:23:44.270133+0000","last_undegraded":"2023-10-01T09:23:44.270133+0000","last_fullsized":"2023-10-01T09:23:44.270133+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":353,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136183+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:23:32.136183+0000","last_peered":"2023-10-01T09:23:32.136183+0000","last_clean":"2023-10-01T09:23:32.136183+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:23:32.136183+0000","last_undegraded":"2023-10-01T09:23:32.136183+0000","last_fullsized":"2023-10-01T09:23:32.136183+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":242,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130554+0000","last_change":"2023-10-01T09:22:40.171122+0000","last_active":"2023-10-01T09:23:32.130554+0000","last_peered":"2023-10-01T09:23:32.130554+0000","last_clean":"2023-10-01T09:23:32.130554+0000","last_became_active":"2023-10-01T09:22:40.170773+0000","last_became_peered":"2023-10-01T09:22:40.170773+0000","last_unstale":"2023-10-01T09:23:32.130554+0000","last_undegraded":"2023-10-01T09:23:32.130554+0000","last_fullsized":"2023-10-01T09:23:32.130554+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:50:30.349100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":259,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.131111+0000","last_change":"2023-10-01T09:19:30.091481+0000","last_active":"2023-10-01T09:23:32.131111+0000","last_peered":"2023-10-01T09:23:32.131111+0000","last_clean":"2023-10-01T09:23:32.131111+0000","last_became_active":"2023-10-01T09:19:30.091151+0000","last_became_peered":"2023-10-01T09:19:30.091151+0000","last_unstale":"2023-10-01T09:23:32.131111+0000","last_undegraded":"2023-10-01T09:23:32.131111+0000","last_fullsized":"2023-10-01T09:23:32.131111+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:09.127408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":9,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-10-01T09:23:45.287796+0000","last_change":"2023-10-01T09:23:45.287796+0000","last_active":"2023-10-01T09:23:45.287796+0000","last_peered":"2023-10-01T09:23:45.287796+0000","last_clean":"2023-10-01T09:23:45.287796+0000","last_became_active":"2023-10-01T09:23:45.287413+0000","last_became_peered":"2023-10-01T09:23:45.287413+0000","last_unstale":"2023-10-01T09:23:45.287796+0000","last_undegraded":"2023-10-01T09:23:45.287796+0000","last_fullsized":"2023-10-01T09:23:45.287796+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":617,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.136544+0000","last_change":"2023-10-01T09:10:58.797964+0000","last_active":"2023-10-01T09:23:32.136544+0000","last_peered":"2023-10-01T09:23:32.136544+0000","last_clean":"2023-10-01T09:23:32.136544+0000","last_became_active":"2023-10-01T09:10:58.797447+0000","last_became_peered":"2023-10-01T09:10:58.797447+0000","last_unstale":"2023-10-01T09:23:32.136544+0000","last_undegraded":"2023-10-01T09:23:32.136544+0000","last_fullsized":"2023-10-01T09:23:32.136544+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":256,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.832164+0000","last_change":"2023-10-01T09:22:02.160793+0000","last_active":"2023-10-01T09:23:35.832164+0000","last_peered":"2023-10-01T09:23:35.832164+0000","last_clean":"2023-10-01T09:23:35.832164+0000","last_became_active":"2023-10-01T09:22:02.160579+0000","last_became_peered":"2023-10-01T09:22:02.160579+0000","last_unstale":"2023-10-01T09:23:35.832164+0000","last_undegraded":"2023-10-01T09:23:35.832164+0000","last_fullsized":"2023-10-01T09:23:35.832164+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:34:29.335908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":209,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130846+0000","last_change":"2023-10-01T09:22:02.151402+0000","last_active":"2023-10-01T09:23:32.130846+0000","last_peered":"2023-10-01T09:23:32.130846+0000","last_clean":"2023-10-01T09:23:32.130846+0000","last_became_active":"2023-10-01T09:22:02.151028+0000","last_became_peered":"2023-10-01T09:22:02.151028+0000","last_unstale":"2023-10-01T09:23:32.130846+0000","last_undegraded":"2023-10-01T09:23:32.130846+0000","last_fullsized":"2023-10-01T09:23:32.130846+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:19.623044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T09:23:44.270133+0000","last_change":"2023-10-01T09:23:44.270133+0000","last_active":"2023-10-01T09:23:44.270133+0000","last_peered":"2023-10-01T09:23:44.270133+0000","last_clean":"2023-10-01T09:23:44.270133+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:23:44.270133+0000","last_undegraded":"2023-10-01T09:23:44.270133+0000","last_fullsized":"2023-10-01T09:23:44.270133+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":269,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.158127+0000","last_change":"2023-10-01T09:23:10.615956+0000","last_active":"2023-10-01T09:23:35.158127+0000","last_peered":"2023-10-01T09:23:35.158127+0000","last_clean":"2023-10-01T09:23:35.158127+0000","last_became_active":"2023-10-01T09:23:10.610919+0000","last_became_peered":"2023-10-01T09:23:10.610919+0000","last_unstale":"2023-10-01T09:23:35.158127+0000","last_undegraded":"2023-10-01T09:23:35.158127+0000","last_fullsized":"2023-10-01T09:23:35.158127+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:03:44.294373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":731,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.830494+0000","last_change":"2023-10-01T09:21:20.545328+0000","last_active":"2023-10-01T09:23:35.830494+0000","last_peered":"2023-10-01T09:23:35.830494+0000","last_clean":"2023-10-01T09:23:35.830494+0000","last_became_active":"2023-10-01T09:21:20.544828+0000","last_became_peered":"2023-10-01T09:21:20.544828+0000","last_unstale":"2023-10-01T09:23:35.830494+0000","last_undegraded":"2023-10-01T09:23:35.830494+0000","last_fullsized":"2023-10-01T09:23:35.830494+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:23:52.633471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":41,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.156974+0000","last_change":"2023-10-01T09:23:10.612309+0000","last_active":"2023-10-01T09:23:35.156974+0000","last_peered":"2023-10-01T09:23:35.156974+0000","last_clean":"2023-10-01T09:23:35.156974+0000","last_became_active":"2023-10-01T09:23:10.612034+0000","last_became_peered":"2023-10-01T09:23:10.612034+0000","last_unstale":"2023-10-01T09:23:35.156974+0000","last_undegraded":"2023-10-01T09:23:35.156974+0000","last_fullsized":"2023-10-01T09:23:35.156974+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:09:47.963186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T09:23:44.270133+0000","last_change":"2023-10-01T09:23:44.270133+0000","last_active":"2023-10-01T09:23:44.270133+0000","last_peered":"2023-10-01T09:23:44.270133+0000","last_clean":"2023-10-01T09:23:44.270133+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T09:23:44.270133+0000","last_undegraded":"2023-10-01T09:23:44.270133+0000","last_fullsized":"2023-10-01T09:23:44.270133+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":91,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.284696+0000","last_change":"2023-10-01T09:23:35.503601+0000","last_active":"2023-10-01T09:23:44.284696+0000","last_peered":"2023-10-01T09:23:44.284696+0000","last_clean":"2023-10-01T09:23:44.284696+0000","last_became_active":"2023-10-01T09:23:10.613270+0000","last_became_peered":"2023-10-01T09:23:10.613270+0000","last_unstale":"2023-10-01T09:23:44.284696+0000","last_undegraded":"2023-10-01T09:23:44.284696+0000","last_fullsized":"2023-10-01T09:23:44.284696+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":238,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.131032+0000","last_change":"2023-10-01T09:19:05.565472+0000","last_active":"2023-10-01T09:23:32.131032+0000","last_peered":"2023-10-01T09:23:32.131032+0000","last_clean":"2023-10-01T09:23:32.131032+0000","last_became_active":"2023-10-01T09:19:05.561161+0000","last_became_peered":"2023-10-01T09:19:05.561161+0000","last_unstale":"2023-10-01T09:23:32.131032+0000","last_undegraded":"2023-10-01T09:23:32.131032+0000","last_fullsized":"2023-10-01T09:23:32.131032+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:50:29.273388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":55,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.285273+0000","last_change":"2023-10-01T09:22:03.144097+0000","last_active":"2023-10-01T09:23:44.285273+0000","last_peered":"2023-10-01T09:23:44.285273+0000","last_clean":"2023-10-01T09:23:44.285273+0000","last_became_active":"2023-10-01T09:22:03.143800+0000","last_became_peered":"2023-10-01T09:22:03.143800+0000","last_unstale":"2023-10-01T09:23:44.285273+0000","last_undegraded":"2023-10-01T09:23:44.285273+0000","last_fullsized":"2023-10-01T09:23:44.285273+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":197,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.130779+0000","last_change":"2023-10-01T09:10:58.801161+0000","last_active":"2023-10-01T09:23:32.130779+0000","last_peered":"2023-10-01T09:23:32.130779+0000","last_clean":"2023-10-01T09:23:32.130779+0000","last_became_active":"2023-10-01T09:10:58.800794+0000","last_became_peered":"2023-10-01T09:10:58.800794+0000","last_unstale":"2023-10-01T09:23:32.130779+0000","last_undegraded":"2023-10-01T09:23:32.130779+0000","last_fullsized":"2023-10-01T09:23:32.130779+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:35.019445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":284,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.832255+0000","last_change":"2023-10-01T09:22:40.166927+0000","last_active":"2023-10-01T09:23:35.832255+0000","last_peered":"2023-10-01T09:23:35.832255+0000","last_clean":"2023-10-01T09:23:35.832255+0000","last_became_active":"2023-10-01T09:22:40.166606+0000","last_became_peered":"2023-10-01T09:22:40.166606+0000","last_unstale":"2023-10-01T09:23:35.832255+0000","last_undegraded":"2023-10-01T09:23:35.832255+0000","last_fullsized":"2023-10-01T09:23:35.832255+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:54:18.044207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":191,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.285782+0000","last_change":"2023-10-01T09:22:02.180112+0000","last_active":"2023-10-01T09:23:44.285782+0000","last_peered":"2023-10-01T09:23:44.285782+0000","last_clean":"2023-10-01T09:23:44.285782+0000","last_became_active":"2023-10-01T09:22:02.161271+0000","last_became_peered":"2023-10-01T09:22:02.161271+0000","last_unstale":"2023-10-01T09:23:44.285782+0000","last_undegraded":"2023-10-01T09:23:44.285782+0000","last_fullsized":"2023-10-01T09:23:44.285782+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":203,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.158057+0000","last_change":"2023-10-01T09:19:05.553200+0000","last_active":"2023-10-01T09:23:35.158057+0000","last_peered":"2023-10-01T09:23:35.158057+0000","last_clean":"2023-10-01T09:23:35.158057+0000","last_became_active":"2023-10-01T09:19:05.552874+0000","last_became_peered":"2023-10-01T09:19:05.552874+0000","last_unstale":"2023-10-01T09:23:35.158057+0000","last_undegraded":"2023-10-01T09:23:35.158057+0000","last_fullsized":"2023-10-01T09:23:35.158057+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:14:31.040887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":180,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-10-01T09:23:44.285657+0000","last_change":"2023-10-01T09:22:03.149673+0000","last_active":"2023-10-01T09:23:44.285657+0000","last_peered":"2023-10-01T09:23:44.285657+0000","last_clean":"2023-10-01T09:23:44.285657+0000","last_became_active":"2023-10-01T09:22:03.149236+0000","last_became_peered":"2023-10-01T09:22:03.149236+0000","last_unstale":"2023-10-01T09:23:44.285657+0000","last_undegraded":"2023-10-01T09:23:44.285657+0000","last_fullsized":"2023-10-01T09:23:44.285657+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":185,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.156957+0000","last_change":"2023-10-01T09:18:54.533869+0000","last_active":"2023-10-01T09:23:35.156957+0000","last_peered":"2023-10-01T09:23:35.156957+0000","last_clean":"2023-10-01T09:23:35.156957+0000","last_became_active":"2023-10-01T09:18:54.533542+0000","last_became_peered":"2023-10-01T09:18:54.533542+0000","last_unstale":"2023-10-01T09:23:35.156957+0000","last_undegraded":"2023-10-01T09:23:35.156957+0000","last_fullsized":"2023-10-01T09:23:35.156957+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:18:19.126440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":32,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.160005+0000","last_change":"2023-10-01T09:22:40.313999+0000","last_active":"2023-10-01T09:23:35.160005+0000","last_peered":"2023-10-01T09:23:35.160005+0000","last_clean":"2023-10-01T09:23:35.160005+0000","last_became_active":"2023-10-01T09:22:40.313605+0000","last_became_peered":"2023-10-01T09:22:40.313605+0000","last_unstale":"2023-10-01T09:23:35.160005+0000","last_undegraded":"2023-10-01T09:23:35.160005+0000","last_fullsized":"2023-10-01T09:23:35.160005+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:52:32.154690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":32,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.158094+0000","last_change":"2023-10-01T09:22:40.170676+0000","last_active":"2023-10-01T09:23:35.158094+0000","last_peered":"2023-10-01T09:23:35.158094+0000","last_clean":"2023-10-01T09:23:35.158094+0000","last_became_active":"2023-10-01T09:22:40.170260+0000","last_became_peered":"2023-10-01T09:22:40.170260+0000","last_unstale":"2023-10-01T09:23:35.158094+0000","last_undegraded":"2023-10-01T09:23:35.158094+0000","last_fullsized":"2023-10-01T09:23:35.158094+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:14.693287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":288,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-10-01T09:23:32.132021+0000","last_change":"2023-10-01T09:22:02.157406+0000","last_active":"2023-10-01T09:23:32.132021+0000","last_peered":"2023-10-01T09:23:32.132021+0000","last_clean":"2023-10-01T09:23:32.132021+0000","last_became_active":"2023-10-01T09:22:02.157041+0000","last_became_peered":"2023-10-01T09:22:02.157041+0000","last_unstale":"2023-10-01T09:23:32.132021+0000","last_undegraded":"2023-10-01T09:23:32.132021+0000","last_fullsized":"2023-10-01T09:23:32.132021+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:24:52.008578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":179,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-10-01T09:23:35.829429+0000","last_change":"2023-10-01T09:21:02.141180+0000","last_active":"2023-10-01T09:23:35.829429+0000","last_peered":"2023-10-01T09:23:35.829429+0000","last_clean":"2023-10-01T09:23:35.829429+0000","last_became_active":"2023-10-01T09:21:02.136796+0000","last_became_peered":"2023-10-01T09:21:02.136796+0000","last_unstale":"2023-10-01T09:23:35.829429+0000","last_undegraded":"2023-10-01T09:23:35.829429+0000","last_fullsized":"2023-10-01T09:23:35.829429+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:03:46.040197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":12,"acting":12,"num_store_stats":3},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":12024,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35793,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430477,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312568,"kb_used_data":7152,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93432584,"statfs":{"total":95995035648,"available":95674966016,"internally_reserved":0,"allocated":7323648,"data_stored":6604003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:15 2023","interfaces":[{"interface":"back","average":{"1min":2.573,"5min":5.2359999999999998,"15min":3.234},"min":{"1min":0.28100000000000003,"5min":0.23400000000000001,"15min":0.219},"max":{"1min":7.5190000000000001,"5min":117.855,"15min":117.855},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.891,"5min":5.2889999999999997,"15min":3.0249999999999999},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":6.1159999999999997,"5min":120.348,"15min":120.348},"last":0.5}]},{"osd":1,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.8600000000000001,"5min":5.3099999999999996,"15min":2.794},"min":{"1min":0.251,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":7.4279999999999999,"5min":135.023,"15min":135.023},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.7719999999999998,"5min":5.2800000000000002,"15min":2.6040000000000001},"min":{"1min":0.29499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":10.034000000000001,"5min":134.51499999999999,"15min":134.51499999999999},"last":0.58499999999999996}]},{"osd":2,"last update":"Sun Oct 1 09:22:54 2023","interfaces":[{"interface":"back","average":{"1min":4.9550000000000001,"5min":4.3860000000000001,"15min":2.4900000000000002},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":59.878,"5min":104.837,"15min":104.837},"last":0.51600000000000001},{"interface":"front","average":{"1min":6.2590000000000003,"5min":4.2640000000000002,"15min":2.4319999999999999},"min":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":60.186,"5min":106.379,"15min":106.379},"last":0.53600000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1739999999999999,"5min":3.4870000000000001,"15min":1.8660000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26400000000000001},"max":{"1min":10.1,"5min":104.797,"15min":104.797},"last":0.56499999999999995},{"interface":"front","average":{"1min":2.8839999999999999,"5min":3.0760000000000001,"15min":1.8260000000000001},"min":{"1min":0.34100000000000003,"5min":0.29399999999999998,"15min":0.27200000000000002},"max":{"1min":7.2039999999999997,"5min":104.774,"15min":104.774},"last":0.59399999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:23:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5870000000000002,"5min":1.726,"15min":1.546},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.23699999999999999},"max":{"1min":6.5510000000000002,"5min":16.062999999999999,"15min":82.992000000000004},"last":0.61099999999999999},{"interface":"front","average":{"1min":2.1859999999999999,"5min":1.7629999999999999,"15min":1.579},"min":{"1min":0.39700000000000002,"5min":0.29499999999999998,"15min":0.252},"max":{"1min":6.2089999999999996,"5min":24.161999999999999,"15min":83.058999999999997},"last":0.63100000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:23:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":3.1339999999999999,"15min":3.1339999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":13.265000000000001,"5min":13.265000000000001,"15min":13.265000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.8940000000000001,"5min":2.8940000000000001,"15min":2.8940000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":13.292999999999999,"5min":13.292999999999999,"15min":13.292999999999999},"last":0.54800000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:23:15 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.4510000000000001,"15min":1.2070000000000001},"min":{"1min":0.35899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.5330000000000004,"5min":11.685,"15min":11.685},"last":0.63800000000000001},{"interface":"front","average":{"1min":2.0750000000000002,"5min":1.6619999999999999,"15min":1.1910000000000001},"min":{"1min":0.38,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.4220000000000006,"5min":11.871,"15min":11.871},"last":0.70999999999999996}]}]},{"osd":6,"up_from":42,"seq":180388626706,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":512960,"kb_used_data":211640,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93232192,"statfs":{"total":95995035648,"available":95469764608,"internally_reserved":0,"allocated":216719360,"data_stored":216014173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8919999999999999,"5min":3.4470000000000001,"15min":3.8959999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20000000000000001},"max":{"1min":6.8200000000000003,"5min":137.941,"15min":178.738},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.3599999999999999,"5min":3.7109999999999999,"15min":4.1600000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":10.992000000000001,"5min":138.20599999999999,"15min":178.684},"last":0.437}]},{"osd":1,"last update":"Sun Oct 1 09:23:30 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":4.6360000000000001,"15min":2.2879999999999998},"min":{"1min":0.253,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.1550000000000002,"5min":115.523,"15min":115.523},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.1699999999999999,"5min":5.0949999999999998,"15min":2.7240000000000002},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":11.042999999999999,"5min":115.681,"15min":115.681},"last":0.56999999999999995}]},{"osd":2,"last update":"Sun Oct 1 09:22:58 2023","interfaces":[{"interface":"back","average":{"1min":3.3330000000000002,"5min":2.3220000000000001,"15min":2.5099999999999998},"min":{"1min":0.34499999999999997,"5min":0.26000000000000001,"15min":0.25900000000000001},"max":{"1min":14.367000000000001,"5min":16.513999999999999,"15min":82.978999999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.532,"5min":1.7749999999999999,"15min":2.1269999999999998},"min":{"1min":0.34300000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.2859999999999996,"5min":17.036999999999999,"15min":82.747},"last":0.47099999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:23:30 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":4.8799999999999999,"15min":2.403},"min":{"1min":0.307,"5min":0.307,"15min":0.27000000000000002},"max":{"1min":7.2050000000000001,"5min":219.32400000000001,"15min":219.32400000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.7769999999999999,"5min":5.1139999999999999,"15min":2.7069999999999999},"min":{"1min":0.37,"5min":0.33600000000000002,"15min":0.30499999999999999},"max":{"1min":8.6989999999999998,"5min":219.77099999999999,"15min":219.77099999999999},"last":0.50800000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:22:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":1.4570000000000001,"15min":1.1220000000000001},"min":{"1min":0.30199999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":14.089,"5min":14.089,"15min":28.271999999999998},"last":0.35499999999999998},{"interface":"front","average":{"1min":2.4039999999999999,"5min":1.3640000000000001,"15min":1.1459999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.26000000000000001},"max":{"1min":9.0440000000000005,"5min":9.907,"15min":28.413},"last":0.56200000000000006}]},{"osd":5,"last update":"Sun Oct 1 09:23:05 2023","interfaces":[{"interface":"back","average":{"1min":2.956,"5min":2.956,"15min":2.956},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.0760000000000005,"5min":9.0760000000000005,"15min":9.0760000000000005},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.1389999999999998,"5min":2.1389999999999998,"15min":2.1389999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":13.999000000000001,"5min":13.999000000000001,"15min":13.999000000000001},"last":0.39900000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:23:12 2023","interfaces":[{"interface":"back","average":{"1min":2.355,"5min":3.1869999999999998,"15min":1.718},"min":{"1min":0.39000000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":11.492000000000001,"5min":145.37299999999999,"15min":145.37299999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.657,"5min":3.012,"15min":1.6579999999999999},"min":{"1min":0.35699999999999998,"5min":0.31,"15min":0.30499999999999999},"max":{"1min":9.1010000000000009,"5min":145.28200000000001,"15min":145.28200000000001},"last":0.53500000000000003}]}]},{"osd":1,"up_from":94,"seq":403726925965,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":236608,"kb_used_data":13048,"kb_used_omap":7,"kb_used_meta":223544,"kb_avail":93508544,"statfs":{"total":95995035648,"available":95752749056,"internally_reserved":0,"allocated":13361152,"data_stored":12178468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":228909479},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":6.3200000000000003,"15min":2.891},"min":{"1min":0.19700000000000001,"5min":0.19400000000000001,"15min":0.17499999999999999},"max":{"1min":7.2770000000000001,"5min":174.40199999999999,"15min":174.40199999999999},"last":0.23599999999999999},{"interface":"front","average":{"1min":2.952,"5min":6.4820000000000002,"15min":3.0430000000000001},"min":{"1min":0.219,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":7.702,"5min":173.77000000000001,"15min":173.77000000000001},"last":0.376}]},{"osd":2,"last update":"Sun Oct 1 09:23:03 2023","interfaces":[{"interface":"back","average":{"1min":8.3699999999999992,"5min":3,"15min":1.673},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":77.325000000000003,"5min":77.325000000000003,"15min":77.325000000000003},"last":0.28000000000000003},{"interface":"front","average":{"1min":9.0679999999999996,"5min":3.222,"15min":1.825},"min":{"1min":0.26900000000000002,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":80.361000000000004,"5min":80.361000000000004,"15min":80.361000000000004},"last":0.32900000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:23:38 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":4.0899999999999999,"15min":1.992},"min":{"1min":0.224,"5min":0.224,"15min":0.218},"max":{"1min":6.593,"5min":99.421000000000006,"15min":99.421000000000006},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.526,"5min":3.8860000000000001,"15min":1.853},"min":{"1min":0.30599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.4409999999999998,"5min":99.364999999999995,"15min":99.364999999999995},"last":0.311}]},{"osd":4,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":1.4079999999999999,"15min":1.2849999999999999},"min":{"1min":0.28799999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.8259999999999996,"5min":9.9169999999999998,"15min":40.673999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":1.7150000000000001,"5min":1.514,"15min":1.417},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.25800000000000001},"max":{"1min":7.2249999999999996,"5min":9.4610000000000003,"15min":40.810000000000002},"last":0.50600000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:23:03 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.403,"15min":1.403},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.734,"5min":4.734,"15min":4.734},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.4369999999999998,"5min":2.4369999999999998,"15min":2.4369999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":5.5090000000000003,"5min":5.5090000000000003,"15min":5.5090000000000003},"last":0.61599999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8260000000000001,"5min":1.5980000000000001,"15min":1.3300000000000001},"min":{"1min":0.35099999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.9860000000000007,"5min":14.452,"15min":14.452},"last":0.57199999999999995},{"interface":"front","average":{"1min":2.29,"5min":1.605,"15min":1.226},"min":{"1min":0.35799999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.9740000000000002,"5min":9.9740000000000002,"15min":10.787000000000001},"last":0.59999999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":1.96,"15min":1.431},"min":{"1min":0.42099999999999999,"5min":0.317,"15min":0.317},"max":{"1min":7.8129999999999997,"5min":23.710999999999999,"15min":23.710999999999999},"last":0.63},{"interface":"front","average":{"1min":1.853,"5min":2.0049999999999999,"15min":1.4450000000000001},"min":{"1min":0.39500000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.3579999999999997,"5min":23.736999999999998,"15min":23.736999999999998},"last":0.58199999999999996}]}]},{"osd":0,"up_from":89,"seq":382252089499,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":363796,"kb_used_data":7692,"kb_used_omap":8,"kb_used_meta":356087,"kb_avail":93381356,"statfs":{"total":95995035648,"available":95622508544,"internally_reserved":0,"allocated":7876608,"data_stored":6672362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":364634050},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:23:28 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":4.202,"15min":1.99},"min":{"1min":0.221,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":13.054,"5min":187.41399999999999,"15min":187.41399999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":3.9529999999999998,"5min":4.6779999999999999,"15min":2.1880000000000002},"min":{"1min":0.23400000000000001,"5min":0.193,"15min":0.193},"max":{"1min":17.088999999999999,"5min":187.78700000000001,"15min":187.78700000000001},"last":0.32700000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:22:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":2.2490000000000001,"15min":1.3939999999999999},"min":{"1min":0.24399999999999999,"5min":0.23300000000000001,"15min":0.23000000000000001},"max":{"1min":4.7699999999999996,"5min":39.048999999999999,"15min":39.048999999999999},"last":0.378},{"interface":"front","average":{"1min":1.514,"5min":2.7719999999999998,"15min":1.591},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.673,"5min":38.945999999999998,"15min":38.945999999999998},"last":0.38800000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:23:28 2023","interfaces":[{"interface":"back","average":{"1min":2.335,"5min":3.2770000000000001,"15min":1.655},"min":{"1min":0.26600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":17.914999999999999,"5min":85.840000000000003,"15min":85.840000000000003},"last":0.40400000000000003},{"interface":"front","average":{"1min":3.5259999999999998,"5min":3.6829999999999998,"15min":1.8500000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23400000000000001},"max":{"1min":11.141999999999999,"5min":86.046000000000006,"15min":86.046000000000006},"last":0.35799999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":3.3420000000000001,"15min":1.8799999999999999},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.6479999999999997,"5min":112.52200000000001,"15min":112.52200000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":2.3439999999999999,"5min":3.2999999999999998,"15min":1.9830000000000001},"min":{"1min":0.33700000000000002,"5min":0.32500000000000001,"15min":0.28599999999999998},"max":{"1min":8.9890000000000008,"5min":113.124,"15min":113.124},"last":0.48699999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:23:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8480000000000001,"5min":1.8480000000000001,"15min":1.8480000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":5.8710000000000004,"5min":5.8710000000000004,"15min":5.8710000000000004},"last":0.754},{"interface":"front","average":{"1min":1.958,"5min":1.958,"15min":1.958},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":5.8879999999999999,"5min":5.8879999999999999,"15min":5.8879999999999999},"last":0.82099999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":2.0910000000000002,"15min":1.577},"min":{"1min":0.375,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.0239999999999991,"5min":34.375999999999998,"15min":34.375999999999998},"last":0.57599999999999996},{"interface":"front","average":{"1min":2.6709999999999998,"5min":1.8899999999999999,"15min":1.379},"min":{"1min":0.39200000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":9.0820000000000007,"5min":31.745000000000001,"15min":31.745000000000001},"last":0.77300000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":3.0150000000000001,"15min":1.867},"min":{"1min":0.44,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":13.096,"5min":98.283000000000001,"15min":98.283000000000001},"last":0.73899999999999999},{"interface":"front","average":{"1min":2.3119999999999998,"5min":2.762,"15min":1.8700000000000001},"min":{"1min":0.36399999999999999,"5min":0.312,"15min":0.312},"max":{"1min":5.7560000000000002,"5min":98.311999999999998,"15min":98.311999999999998},"last":0.54600000000000004}]}]},{"osd":2,"up_from":99,"seq":425201762439,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":346976,"kb_used_data":2968,"kb_used_omap":11,"kb_used_meta":343988,"kb_avail":93398176,"statfs":{"total":95995035648,"available":95639732224,"internally_reserved":0,"allocated":3039232,"data_stored":1886323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":352244599},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":7.5149999999999997,"5min":3.0230000000000001,"15min":2.0899999999999999},"min":{"1min":0.19800000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":119.621,"5min":119.621,"15min":119.621},"last":0.28399999999999997},{"interface":"front","average":{"1min":7.0289999999999999,"5min":3.2010000000000001,"15min":2.3940000000000001},"min":{"1min":0.20999999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":119.411,"5min":119.411,"15min":119.411},"last":0.33100000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.032,"5min":3.3170000000000002,"15min":2.1269999999999998},"min":{"1min":0.19600000000000001,"5min":0.193,"15min":0.193},"max":{"1min":8.4399999999999995,"5min":105.798,"15min":105.798},"last":0.375},{"interface":"front","average":{"1min":1.452,"5min":3.2120000000000002,"15min":2.2669999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.4039999999999999,"5min":105.56399999999999,"15min":107.729},"last":0.30399999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:23:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6059999999999999,"5min":6.2910000000000004,"15min":2.7429999999999999},"min":{"1min":0.217,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":25.975000000000001,"5min":253.238,"15min":253.238},"last":0.32000000000000001},{"interface":"front","average":{"1min":2.8929999999999998,"5min":6.1580000000000004,"15min":2.6970000000000001},"min":{"1min":0.30599999999999999,"5min":0.253,"15min":0.22600000000000001},"max":{"1min":22.498999999999999,"5min":252.54900000000001,"15min":252.54900000000001},"last":0.34399999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1850000000000001,"5min":1.974,"15min":1.321},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27000000000000002},"max":{"1min":8.2530000000000001,"5min":16.059000000000001,"15min":17.751999999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":2.5579999999999998,"5min":2.1030000000000002,"15min":1.409},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":14.359,"5min":14.359,"15min":17.568000000000001},"last":0.56599999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:23:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5289999999999999,"5min":2.5289999999999999,"15min":2.5289999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.2840000000000007,"5min":8.2840000000000007,"15min":8.2840000000000007},"last":0.74399999999999999},{"interface":"front","average":{"1min":2.3580000000000001,"5min":2.3580000000000001,"15min":2.3580000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":9.3160000000000007,"5min":9.3160000000000007,"15min":9.3160000000000007},"last":0.55900000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":3.2320000000000002,"5min":2.1259999999999999,"15min":1.466},"min":{"1min":0.35199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":14.411,"5min":14.411,"15min":17.591000000000001},"last":0.73199999999999998},{"interface":"front","average":{"1min":2.8919999999999999,"5min":1.7629999999999999,"15min":1.2010000000000001},"min":{"1min":0.40400000000000003,"5min":0.36699999999999999,"15min":0.34200000000000003},"max":{"1min":8.6620000000000008,"5min":8.6620000000000008,"15min":17.736000000000001},"last":0.70599999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":3.6739999999999999,"5min":2.0129999999999999,"15min":1.782},"min":{"1min":0.38200000000000001,"5min":0.34699999999999998,"15min":0.32800000000000001},"max":{"1min":16.169,"5min":16.169,"15min":109.506},"last":0.55300000000000005},{"interface":"front","average":{"1min":3.1680000000000001,"5min":2.0329999999999999,"15min":1.8979999999999999},"min":{"1min":0.375,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":16.152000000000001,"5min":16.152000000000001,"15min":109.58799999999999},"last":0.754}]}]},{"osd":3,"up_from":104,"seq":446676598912,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":249124,"kb_used_data":2972,"kb_used_omap":3,"kb_used_meta":246140,"kb_avail":93496028,"statfs":{"total":95995035648,"available":95739932672,"internally_reserved":0,"allocated":3043328,"data_stored":1877832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":252048103},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":2.9199999999999999,"5min":1.9630000000000001,"15min":2.9689999999999999},"min":{"1min":0.20899999999999999,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":18.635000000000002,"5min":38.314,"15min":199.422},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.976,"5min":2.1680000000000001,"15min":3.1509999999999998},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":18.792000000000002,"5min":38.118000000000002,"15min":199.91200000000001},"last":0.45800000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":5.431,"5min":5.702,"15min":3.3940000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":37.011000000000003,"5min":228.60499999999999,"15min":228.60499999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":4.3399999999999999,"5min":5.3099999999999996,"15min":3.2269999999999999},"min":{"1min":0.26500000000000001,"5min":0.23899999999999999,"15min":0.215},"max":{"1min":36.866,"5min":228.28399999999999,"15min":228.28399999999999},"last":0.40200000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":2.6930000000000001,"15min":1.51},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.224},"max":{"1min":7.524,"5min":55.238999999999997,"15min":55.238999999999997},"last":0.375},{"interface":"front","average":{"1min":1.99,"5min":2.9409999999999998,"15min":1.6779999999999999},"min":{"1min":0.23999999999999999,"5min":0.219,"15min":0.20399999999999999},"max":{"1min":6.7919999999999998,"5min":55.692999999999998,"15min":55.692999999999998},"last":0.42699999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5,"5min":4.2729999999999997,"15min":2.274},"min":{"1min":0.32600000000000001,"5min":0.27300000000000002,"15min":0.25900000000000001},"max":{"1min":9.0250000000000004,"5min":154.00399999999999,"15min":154.00399999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":3.8420000000000001,"5min":4.8140000000000001,"15min":2.601},"min":{"1min":0.316,"5min":0.30099999999999999,"15min":0.248},"max":{"1min":36.947000000000003,"5min":156.87700000000001,"15min":156.87700000000001},"last":0.72399999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":1.7769999999999999,"15min":1.7769999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.5670000000000002,"5min":7.5670000000000002,"15min":7.5670000000000002},"last":0.80300000000000005},{"interface":"front","average":{"1min":3.8439999999999999,"5min":3.8439999999999999,"15min":3.8439999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":36.984000000000002,"5min":36.984000000000002,"15min":36.984000000000002},"last":0.76500000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":4.3070000000000004,"5min":3.5739999999999998,"15min":2.0369999999999999},"min":{"1min":0.36499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":36.901000000000003,"5min":72.433999999999997,"15min":72.433999999999997},"last":0.749},{"interface":"front","average":{"1min":4.1470000000000002,"5min":3.1899999999999999,"15min":1.8100000000000001},"min":{"1min":0.33600000000000002,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":37.274999999999999,"5min":72.450999999999993,"15min":72.450999999999993},"last":0.77500000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":4.7910000000000004,"5min":2.2290000000000001,"15min":1.712},"min":{"1min":0.35499999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":37.024000000000001,"5min":38.478999999999999,"15min":79.412999999999997},"last":0.65500000000000003},{"interface":"front","average":{"1min":4.867,"5min":2.415,"15min":1.7490000000000001},"min":{"1min":0.40799999999999997,"5min":0.36399999999999999,"15min":0.308},"max":{"1min":36.762,"5min":40.039999999999999,"15min":79.527000000000001},"last":0.78800000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019162,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303792,"kb_used_data":2472,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93441360,"statfs":{"total":95995035648,"available":95683952640,"internally_reserved":0,"allocated":2531328,"data_stored":1819261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":3.468,"15min":3.3849999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.20999999999999999},"max":{"1min":4.4690000000000003,"5min":111.27800000000001,"15min":133.91499999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":2.581,"5min":3.7410000000000001,"15min":3.3540000000000001},"min":{"1min":0.26600000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.5099999999999998,"5min":111.289,"15min":133.83199999999999},"last":0.5}]},{"osd":1,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":2.6110000000000002,"5min":2.4399999999999999,"15min":2.02},"min":{"1min":0.30599999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.5589999999999993,"5min":37.326999999999998,"15min":133.93600000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.0600000000000001,"5min":2.234,"15min":2.105},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.0009999999999994,"5min":37.335000000000001,"15min":141.06},"last":0.40799999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:23:07 2023","interfaces":[{"interface":"back","average":{"1min":4.367,"5min":3.2690000000000001,"15min":2.9929999999999999},"min":{"1min":0.35299999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":26.545000000000002,"5min":91.474999999999994,"15min":133.96100000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":4.375,"5min":3.4969999999999999,"15min":2.9689999999999999},"min":{"1min":0.33000000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":28.324000000000002,"5min":91.510999999999996,"15min":134.018},"last":0.59199999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.5029999999999999,"15min":1.6439999999999999},"min":{"1min":0.32000000000000001,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":6.0679999999999996,"5min":6.9909999999999997,"15min":134.297},"last":0.56899999999999995},{"interface":"front","average":{"1min":2.9540000000000002,"5min":1.982,"15min":1.9650000000000001},"min":{"1min":0.30399999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":18.370000000000001,"5min":18.370000000000001,"15min":141.089},"last":0.53000000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:23:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.6899999999999999,"15min":1.6899999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.7839999999999998,"5min":5.7839999999999998,"15min":5.7839999999999998},"last":0.63100000000000001},{"interface":"front","average":{"1min":2.681,"5min":2.681,"15min":2.681},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.6539999999999999,"5min":8.6539999999999999,"15min":8.6539999999999999},"last":0.62}]},{"osd":6,"last update":"Sun Oct 1 09:22:50 2023","interfaces":[{"interface":"back","average":{"1min":3.097,"5min":1.728,"15min":1.464},"min":{"1min":0.36799999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.6839999999999993,"5min":9.1370000000000005,"15min":133.90799999999999},"last":0.65400000000000003},{"interface":"front","average":{"1min":2.3719999999999999,"5min":1.595,"15min":1.6819999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":20.622,"5min":20.622,"15min":141.03},"last":0.60399999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:23:08 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":1.401,"15min":1.6100000000000001},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.0579999999999998,"5min":8.8369999999999997,"15min":133.946},"last":0.66600000000000004},{"interface":"front","average":{"1min":2.5859999999999999,"5min":1.5449999999999999,"15min":1.631},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":8.7140000000000004,"5min":11.215,"15min":133.98400000000001},"last":0.82399999999999995}]}]},{"osd":5,"up_from":227,"seq":974957576214,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":473200,"kb_used_data":141672,"kb_used_omap":8,"kb_used_meta":331511,"kb_avail":93271952,"statfs":{"total":95995035648,"available":95510478848,"internally_reserved":0,"allocated":145072128,"data_stored":144371003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":339468036},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":2.0499999999999998,"15min":2.0499999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":7.7699999999999996,"5min":7.7699999999999996,"15min":7.7699999999999996},"last":0.69099999999999995},{"interface":"front","average":{"1min":2.7130000000000001,"5min":2.7130000000000001,"15min":2.7130000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":10.867000000000001,"5min":10.867000000000001,"15min":10.867000000000001},"last":0.45700000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":2.2690000000000001,"15min":2.2690000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.2110000000000003,"5min":6.2110000000000003,"15min":6.2110000000000003},"last":0.54800000000000004},{"interface":"front","average":{"1min":3.3039999999999998,"5min":3.3039999999999998,"15min":3.3039999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.4280000000000008,"5min":9.4280000000000008,"15min":9.4280000000000008},"last":0.71799999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8879999999999999,"5min":2.8879999999999999,"15min":2.8879999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.3049999999999997,"5min":9.3049999999999997,"15min":9.3049999999999997},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.7589999999999999,"15min":1.7589999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.8419999999999996,"5min":4.8419999999999996,"15min":4.8419999999999996},"last":0.623}]},{"osd":3,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":2.2250000000000001,"15min":2.2250000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.5640000000000001,"5min":6.5640000000000001,"15min":6.5640000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":2.1739999999999999,"5min":2.1739999999999999,"15min":2.1739999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.7199999999999998,"5min":7.7199999999999998,"15min":7.7199999999999998},"last":0.63400000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":1.891,"15min":1.891},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":7.532,"5min":7.532,"15min":7.532},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":4.117,"5min":4.117,"15min":4.117},"last":0.66800000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":1.948,"5min":1.948,"15min":1.948},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.7520000000000007,"5min":8.7520000000000007,"15min":8.7520000000000007},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.401,"5min":1.401,"15min":1.401},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.7460000000000004,"5min":7.7460000000000004,"15min":7.7460000000000004},"last":0.76400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":1.6599999999999999,"15min":1.6599999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.6440000000000001,"5min":4.6440000000000001,"15min":4.6440000000000001},"last":0.70399999999999996},{"interface":"front","average":{"1min":2.2330000000000001,"5min":2.2330000000000001,"15min":2.2330000000000001},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":10.836,"5min":10.836,"15min":10.836},"last":0.68400000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:23:48.269 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:23:48.375 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:48.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:48.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:48.479 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:48.505 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:48 smithi138 ceph-mon[192513]: failed to encode map e261 with expected crc 2023-10-01T09:23:48.506 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:48 smithi138 ceph-mon[192513]: failed to encode map e261 with expected crc 2023-10-01T09:23:48.506 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:48 smithi138 ceph-mon[192513]: from='client.35394 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:23:48.506 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:48 smithi138 ceph-mon[192513]: pgmap v655: 89 pgs: 2 creating+peering, 2 unknown, 85 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:23:48.506 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:48 smithi138 ceph-mon[185886]: failed to encode map e261 with expected crc 2023-10-01T09:23:48.506 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:48 smithi138 ceph-mon[185886]: failed to encode map e261 with expected crc 2023-10-01T09:23:48.506 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:48 smithi138 ceph-mon[185886]: from='client.35394 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:23:48.506 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:48 smithi138 ceph-mon[185886]: pgmap v655: 89 pgs: 2 creating+peering, 2 unknown, 85 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:23:48.509 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:48 smithi190 ceph-mon[160435]: failed to encode map e261 with expected crc 2023-10-01T09:23:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:48 smithi190 ceph-mon[160435]: failed to encode map e261 with expected crc 2023-10-01T09:23:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:48 smithi190 ceph-mon[160435]: from='client.35394 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:23:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:48 smithi190 ceph-mon[160435]: pgmap v655: 89 pgs: 2 creating+peering, 2 unknown, 85 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:23:48.762 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:48.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:48.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:48.866 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:23:48.898 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:48.947 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:23:48.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:48.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:23:49.052 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:23:49.079 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:49.189 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:23:49.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:49.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:23:49.293 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:49.319 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:49.713 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:49.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:49.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:49.816 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:49.844 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:50.191 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:50.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:50.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:50.296 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:50.327 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:50.371 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:50.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:50.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:50.431 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:23:50.475 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:23:50.507 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:50.557 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:23:50.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:50.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:23:50.617 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:50.637 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:50 smithi138 ceph-mon[185886]: pgmap v656: 89 pgs: 2 creating+peering, 87 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:23:50.637 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:50 smithi138 ceph-mon[192513]: pgmap v656: 89 pgs: 2 creating+peering, 87 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:23:50.663 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:50.701 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:50.800 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:50.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:50.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:50.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:50 smithi190 ceph-mon[160435]: pgmap v656: 89 pgs: 2 creating+peering, 87 active+clean; 4.6 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:23:50.852 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:23:50.852 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:23:50.852 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-10-01T09:23:50.852 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:23:50.904 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:23:50.936 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:50.984 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:23:50.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:50.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:23:51.092 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:23:51.119 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:51.162 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:23:51.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:51.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:23:51.269 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:51.299 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:51.560 INFO:teuthology.orchestra.run.smithi138.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-10-01T09:23:51.560 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:23:51.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:51 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1896939872' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:23:51.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:51 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3596441579' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:23:51.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:51 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:23:51.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:51 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1896939872' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:23:51.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:51 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3596441579' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:23:51.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:51 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:23:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:51 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1896939872' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:23:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:51 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3596441579' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:23:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:51 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:23:52.392 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:52.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:52.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:52.495 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:52.522 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:52 smithi190 ceph-mon[160435]: pgmap v657: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:23:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:52 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:23:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:52 smithi190 ceph-mon[160435]: failed to encode map e262 with expected crc 2023-10-01T09:23:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:52 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1896939872' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:23:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:52 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:23:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:52 smithi190 ceph-mon[160435]: osdmap e262: 8 total, 8 up, 8 in 2023-10-01T09:23:52.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:52 smithi138 ceph-mon[185886]: pgmap v657: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:23:52.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:52 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:23:52.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:52 smithi138 ceph-mon[185886]: failed to encode map e262 with expected crc 2023-10-01T09:23:52.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:52 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1896939872' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:23:52.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:52 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:23:52.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:52 smithi138 ceph-mon[185886]: osdmap e262: 8 total, 8 up, 8 in 2023-10-01T09:23:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:52 smithi138 ceph-mon[192513]: pgmap v657: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:23:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:52 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:23:52.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:52 smithi138 ceph-mon[192513]: failed to encode map e262 with expected crc 2023-10-01T09:23:52.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:52 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1896939872' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:23:52.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:52 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:23:52.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:52 smithi138 ceph-mon[192513]: osdmap e262: 8 total, 8 up, 8 in 2023-10-01T09:23:52.860 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:23:52.917 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:52.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:52.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:53.021 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:53.048 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:53.244 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:53.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:53.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:53.348 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:53.380 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:53.530 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:53.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:53.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:53.537 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:53.634 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:23:53.662 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:53.765 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:23:53.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:53.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:23:53.789 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:53 smithi138 ceph-mon[185886]: failed to encode map e262 with expected crc 2023-10-01T09:23:53.789 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:53 smithi138 ceph-mon[185886]: failed to encode map e262 with expected crc 2023-10-01T09:23:53.789 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:53 smithi138 ceph-mon[192513]: failed to encode map e262 with expected crc 2023-10-01T09:23:53.789 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:53 smithi138 ceph-mon[192513]: failed to encode map e262 with expected crc 2023-10-01T09:23:53.824 DEBUG:teuthology.orchestra.run.smithi138:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-10-01T09:23:53.826 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T09:23:53.826 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T09:23:53.869 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:53.873 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:hints = 1 2023-10-01T09:23:53.873 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T09:23:53.873 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Object prefix: benchmark_data_smithi138_367058 2023-10-01T09:23:53.873 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:23:53.873 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 0 0 0 0 0 0 - 0 2023-10-01T09:23:53.907 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:23:53.907 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":854,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":262,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152121,"num_in_osds":8,"osd_in_since":1696152061,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":89}],"num_pgs":89,"num_pools":5,"num_objects":28,"data_bytes":4790195,"bytes_used":2784751616,"bytes_avail":765175533568,"bytes_total":767960285184},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":23,"modified":"2023-10-01T09:23:01.474947+0000","services":{"mds":{"daemons":{"summary":"","foo.smithi138.brhzod":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}},"foo.smithi190.weynjc":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}},"mgr":{"daemons":{"summary":"","x":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}},"osd":{"daemons":{"summary":"","6":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-10-01T09:23:53.918 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:54.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:53 smithi190 ceph-mon[160435]: failed to encode map e262 with expected crc 2023-10-01T09:23:54.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:53 smithi190 ceph-mon[160435]: failed to encode map e262 with expected crc 2023-10-01T09:23:54.131 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:54.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:54 smithi138 ceph-mon[185886]: pgmap v659: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:23:54.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:54 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4195389612' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:23:54.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:54 smithi138 ceph-mon[192513]: pgmap v659: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:23:54.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:54 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4195389612' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:23:54.876 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 1 16 3162 3146 196.022 196.625 0.00344939 0.00508089 2023-10-01T09:23:55.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:54 smithi190 ceph-mon[160435]: pgmap v659: 89 pgs: 89 active+clean; 4.6 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-10-01T09:23:55.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:54 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4195389612' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:23:55.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:23:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:23:54] "GET /metrics HTTP/1.1" 200 34041 "" "Prometheus/2.33.4" 2023-10-01T09:23:55.454 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:23:55.753 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:55.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:55.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:55.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:23:55.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:23:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:55.856 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:23:55.876 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 2 16 5370 5354 167.048 138 0.00127725 0.0054995 2023-10-01T09:23:55.883 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:56.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:23:56.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:23:56.266 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:23:56.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:56.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:23:56.373 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:56.377 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-10-01T09:23:56.377 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:23:56.438 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:56.638 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:56.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:56.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:56.752 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:23:56.783 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:56.794 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:56.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:56 smithi138 ceph-mon[185886]: pgmap v660: 89 pgs: 89 active+clean; 49 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s wr, 77 op/s 2023-10-01T09:23:56.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:56 smithi138 ceph-mon[192513]: pgmap v660: 89 pgs: 89 active+clean; 49 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s wr, 77 op/s 2023-10-01T09:23:56.837 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:23:56.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:56.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:23:56.842 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:56 smithi190 ceph-mon[160435]: pgmap v660: 89 pgs: 89 active+clean; 49 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s wr, 77 op/s 2023-10-01T09:23:56.876 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 3 16 7550 7534 156.789 136.25 0.00419656 0.0063668 2023-10-01T09:23:56.942 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:57.005 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:57.174 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:23:57.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:57.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:23:57.281 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:23:57.339 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:57.876 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 4 16 10256 10240 159.865 169.125 0.0036333 0.00624792 2023-10-01T09:23:58.524 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:58 smithi138 ceph-mon[185886]: pgmap v661: 89 pgs: 89 active+clean; 392 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s 2023-10-01T09:23:58.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:58 smithi138 ceph-mon[192513]: pgmap v661: 89 pgs: 89 active+clean; 392 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s 2023-10-01T09:23:58.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:58 smithi190 ceph-mon[160435]: pgmap v661: 89 pgs: 89 active+clean; 392 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s 2023-10-01T09:23:58.731 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:23:58.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:58.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:23:58.836 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:23:58.876 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 5 16 12903 12887 160.976 165.438 0.0213048 0.00619692 2023-10-01T09:23:58.905 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:59.022 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:23:59.070 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:23:59.111 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:23:59.111 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:23:59.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:59.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:23:59.156 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:23:59.193 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:23:59.218 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:23:59.228 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:23:59.254 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:59.263 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:23:59.266 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:23:59.308 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:23:59.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:23:59.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:23:59.319 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:23:59.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:23:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1196784057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:59.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:23:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1196784057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:59.341 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:23:59.342 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:23:59.342 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:23:59.342 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:23:59.342 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:23:59.342 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:23:59.342 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:23:59.342 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:23:59.416 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:23:59.489 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:23:59.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:23:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1196784057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:23:59.878 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 6 16 15003 14987 156.021 131.25 0.01191 0.00640315 2023-10-01T09:24:00.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:00 smithi138 ceph-mon[192513]: pgmap v662: 89 pgs: 89 active+clean; 487 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s wr, 771 op/s 2023-10-01T09:24:00.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:00 smithi138 ceph-mon[185886]: pgmap v662: 89 pgs: 89 active+clean; 487 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s wr, 771 op/s 2023-10-01T09:24:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:00 smithi190 ceph-mon[160435]: pgmap v662: 89 pgs: 89 active+clean; 487 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s wr, 771 op/s 2023-10-01T09:24:00.877 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 7 16 17457 17441 155.641 153.375 0.00170021 0.00637949 2023-10-01T09:24:01.625 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:24:01.625 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:24:01.648 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:24:01.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:01.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:24:01.752 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:01.821 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:01.877 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 8 16 18514 18498 144.447 66.0625 0.0110856 0.00691638 2023-10-01T09:24:02.096 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:02.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:02.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:02.203 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:24:02.274 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:02.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:02 smithi138 ceph-mon[192513]: pgmap v663: 89 pgs: 89 active+clean; 669 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T09:24:02.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:02 smithi138 ceph-mon[185886]: pgmap v663: 89 pgs: 89 active+clean; 669 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T09:24:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:02 smithi190 ceph-mon[160435]: pgmap v663: 89 pgs: 89 active+clean; 669 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T09:24:02.878 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 9 16 20636 20620 143.133 132.625 0.00498599 0.00698162 2023-10-01T09:24:02.985 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:24:02.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:02.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:24:03.093 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:03.121 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:03.162 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:03.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:03.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:03.266 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:03.291 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:03.352 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:03.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:03.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:03.457 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:24:03.513 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:03.877 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 10 16 22193 22177 138.551 97.3125 0.0108843 0.00721095 2023-10-01T09:24:04.319 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:24:04.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:04.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:24:04.383 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:24:04.426 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:24:04.458 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:04.517 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:24:04.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:04.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:24:04.611 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:24:04.623 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:24:04.626 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:04 smithi138 ceph-mon[185886]: pgmap v664: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-01T09:24:04.626 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:04 smithi138 ceph-mon[192513]: pgmap v664: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-01T09:24:04.654 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:04.713 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:24:04.717 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:04 smithi190 ceph-mon[160435]: pgmap v664: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-01T09:24:04.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:04.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:24:04.819 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:04.878 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 11 16 23619 23603 134.05 89.125 0.00716649 0.0074507 2023-10-01T09:24:04.919 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:05.024 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:05.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:05.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:05.128 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:05.158 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:05.211 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:05.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:05.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:05.229 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:24:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:24:04] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.33.4" 2023-10-01T09:24:05.317 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:05.380 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:05.422 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:05.503 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:05.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:05.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:05.595 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:05.609 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:24:05.627 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:24:05.685 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:05.772 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2719103498' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:24:05.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:05 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:24:05.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2719103498' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:24:05.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:05 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:24:05.845 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:24:05.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:05.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:24:05.878 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 12 16 24944 24928 129.781 82.8125 0.0121374 0.00769841 2023-10-01T09:24:05.939 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:24:05.952 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:24:05.953 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:24:06.024 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:06.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2719103498' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:24:06.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:05 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:24:06.878 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 13 16 26708 26692 128.278 110.25 0.00805479 0.00778968 2023-10-01T09:24:07.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:06 smithi190 ceph-mon[160435]: pgmap v665: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-01T09:24:07.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:06 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:24:07.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:06 smithi190 ceph-mon[160435]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:06 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:24:07.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:06 smithi190 ceph-mon[160435]: osdmap e263: 8 total, 8 up, 8 in 2023-10-01T09:24:07.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:06 smithi190 ceph-mon[160435]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:06 smithi190 ceph-mon[160435]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:06 smithi190 ceph-mon[160435]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:06 smithi190 ceph-mon[160435]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[192513]: pgmap v665: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-01T09:24:07.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:24:07.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[192513]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:24:07.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[192513]: osdmap e263: 8 total, 8 up, 8 in 2023-10-01T09:24:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[192513]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[192513]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[192513]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[192513]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[185886]: pgmap v665: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-01T09:24:07.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:24:07.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[185886]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:24:07.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[185886]: osdmap e263: 8 total, 8 up, 8 in 2023-10-01T09:24:07.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[185886]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[185886]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[185886]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:06 smithi138 ceph-mon[185886]: failed to encode map e263 with expected crc 2023-10-01T09:24:07.878 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 14 16 28327 28311 126.34 101.188 0.00756878 0.00790881 2023-10-01T09:24:08.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:07 smithi190 ceph-mon[160435]: from='client.55186 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:24:08.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:07 smithi138 ceph-mon[185886]: from='client.55186 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:24:08.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:07 smithi138 ceph-mon[192513]: from='client.55186 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:24:08.768 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:24:08.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:08.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:24:08.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:08 smithi138 ceph-mon[185886]: pgmap v667: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2023-10-01T09:24:08.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:08 smithi138 ceph-mon[192513]: pgmap v667: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2023-10-01T09:24:08.875 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:08.878 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 15 16 29474 29458 122.698 71.6875 0.00897067 0.00814431 2023-10-01T09:24:08.906 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:08 smithi190 ceph-mon[160435]: pgmap v667: 89 pgs: 89 active+clean; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2023-10-01T09:24:08.909 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:08.973 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:08.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:08.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:09.021 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":666,"stamp":"2023-10-01T09:24:05.595414+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1326520243,"num_objects":20196,"num_object_clones":0,"num_object_copies":60588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20196,"num_whiteouts":0,"num_read":6259,"num_read_kb":12659,"num_write":21274,"num_write_kb":1305893,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":186,"num_bytes_recovered":2373496,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":21338,"ondisk_log_size":21338,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6353400,"kb_used_data":3895608,"kb_used_omap":38,"kb_used_meta":2457689,"kb_avail":743607816,"statfs":{"total":767960285184,"available":761454403584,"internally_reserved":0,"allocated":3989102592,"data_stored":3981493150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39221,"internal_metadata":2516674251},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1274675200,"num_objects":19450,"num_object_clones":0,"num_object_copies":58350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":19450,"num_write_kb":1244800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.100270"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":133,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.770162+0000","last_change":"2023-10-01T09:19:30.091981+0000","last_active":"2023-10-01T09:23:51.770162+0000","last_peered":"2023-10-01T09:23:51.770162+0000","last_clean":"2023-10-01T09:23:51.770162+0000","last_became_active":"2023-10-01T09:19:30.091531+0000","last_became_peered":"2023-10-01T09:19:30.091531+0000","last_unstale":"2023-10-01T09:23:51.770162+0000","last_undegraded":"2023-10-01T09:23:51.770162+0000","last_fullsized":"2023-10-01T09:23:51.770162+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:28:13.442498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":36,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.291791+0000","last_change":"2023-10-01T09:22:40.311894+0000","last_active":"2023-10-01T09:23:46.291791+0000","last_peered":"2023-10-01T09:23:46.291791+0000","last_clean":"2023-10-01T09:23:46.291791+0000","last_became_active":"2023-10-01T09:22:40.311581+0000","last_became_peered":"2023-10-01T09:22:40.311581+0000","last_unstale":"2023-10-01T09:23:46.291791+0000","last_undegraded":"2023-10-01T09:23:46.291791+0000","last_fullsized":"2023-10-01T09:23:46.291791+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:52:27.649535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":208,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.916615+0000","last_change":"2023-10-01T09:13:13.551596+0000","last_active":"2023-10-01T09:23:46.916615+0000","last_peered":"2023-10-01T09:23:46.916615+0000","last_clean":"2023-10-01T09:23:46.916615+0000","last_became_active":"2023-10-01T09:13:13.551300+0000","last_became_peered":"2023-10-01T09:13:13.551300+0000","last_unstale":"2023-10-01T09:23:46.916615+0000","last_undegraded":"2023-10-01T09:23:46.916615+0000","last_fullsized":"2023-10-01T09:23:46.916615+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:57:45.081780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":265,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.918068+0000","last_change":"2023-10-01T09:23:10.611220+0000","last_active":"2023-10-01T09:23:46.918068+0000","last_peered":"2023-10-01T09:23:46.918068+0000","last_clean":"2023-10-01T09:23:46.918068+0000","last_became_active":"2023-10-01T09:23:10.610573+0000","last_became_peered":"2023-10-01T09:23:10.610573+0000","last_unstale":"2023-10-01T09:23:46.918068+0000","last_undegraded":"2023-10-01T09:23:46.918068+0000","last_fullsized":"2023-10-01T09:23:46.918068+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:58:32.020825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":36,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.290767+0000","last_change":"2023-10-01T09:22:40.172183+0000","last_active":"2023-10-01T09:23:46.290767+0000","last_peered":"2023-10-01T09:23:46.290767+0000","last_clean":"2023-10-01T09:23:46.290767+0000","last_became_active":"2023-10-01T09:22:40.170164+0000","last_became_peered":"2023-10-01T09:22:40.170164+0000","last_unstale":"2023-10-01T09:23:46.290767+0000","last_undegraded":"2023-10-01T09:23:46.290767+0000","last_fullsized":"2023-10-01T09:23:46.290767+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:33:10.407999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":208,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919827+0000","last_change":"2023-10-01T09:22:03.153196+0000","last_active":"2023-10-01T09:23:46.919827+0000","last_peered":"2023-10-01T09:23:46.919827+0000","last_clean":"2023-10-01T09:23:46.919827+0000","last_became_active":"2023-10-01T09:22:03.151768+0000","last_became_peered":"2023-10-01T09:22:03.151768+0000","last_unstale":"2023-10-01T09:23:46.919827+0000","last_undegraded":"2023-10-01T09:23:46.919827+0000","last_fullsized":"2023-10-01T09:23:46.919827+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":267,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.917648+0000","last_change":"2023-10-01T09:22:40.175704+0000","last_active":"2023-10-01T09:23:46.917648+0000","last_peered":"2023-10-01T09:23:46.917648+0000","last_clean":"2023-10-01T09:23:46.917648+0000","last_became_active":"2023-10-01T09:22:40.175369+0000","last_became_peered":"2023-10-01T09:22:40.175369+0000","last_unstale":"2023-10-01T09:23:46.917648+0000","last_undegraded":"2023-10-01T09:23:46.917648+0000","last_fullsized":"2023-10-01T09:23:46.917648+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:07.773407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":252,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.290852+0000","last_change":"2023-10-01T09:21:02.139865+0000","last_active":"2023-10-01T09:23:46.290852+0000","last_peered":"2023-10-01T09:23:46.290852+0000","last_clean":"2023-10-01T09:23:46.290852+0000","last_became_active":"2023-10-01T09:21:02.139507+0000","last_became_peered":"2023-10-01T09:21:02.139507+0000","last_unstale":"2023-10-01T09:23:46.290852+0000","last_undegraded":"2023-10-01T09:23:46.290852+0000","last_fullsized":"2023-10-01T09:23:46.290852+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:46:09.214220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":224,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919459+0000","last_change":"2023-10-01T09:23:10.610494+0000","last_active":"2023-10-01T09:23:46.919459+0000","last_peered":"2023-10-01T09:23:46.919459+0000","last_clean":"2023-10-01T09:23:46.919459+0000","last_became_active":"2023-10-01T09:23:10.610037+0000","last_became_peered":"2023-10-01T09:23:10.610037+0000","last_unstale":"2023-10-01T09:23:46.919459+0000","last_undegraded":"2023-10-01T09:23:46.919459+0000","last_fullsized":"2023-10-01T09:23:46.919459+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":278,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.291269+0000","last_change":"2023-10-01T09:23:10.613042+0000","last_active":"2023-10-01T09:23:46.291269+0000","last_peered":"2023-10-01T09:23:46.291269+0000","last_clean":"2023-10-01T09:23:46.291269+0000","last_became_active":"2023-10-01T09:23:10.612686+0000","last_became_peered":"2023-10-01T09:23:10.612686+0000","last_unstale":"2023-10-01T09:23:46.291269+0000","last_undegraded":"2023-10-01T09:23:46.291269+0000","last_fullsized":"2023-10-01T09:23:46.291269+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:12:00.239369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":31,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.918663+0000","last_change":"2023-10-01T09:23:10.616550+0000","last_active":"2023-10-01T09:23:46.918663+0000","last_peered":"2023-10-01T09:23:46.918663+0000","last_clean":"2023-10-01T09:23:46.918663+0000","last_became_active":"2023-10-01T09:23:10.611509+0000","last_became_peered":"2023-10-01T09:23:10.611509+0000","last_unstale":"2023-10-01T09:23:46.918663+0000","last_undegraded":"2023-10-01T09:23:46.918663+0000","last_fullsized":"2023-10-01T09:23:46.918663+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":263,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:48.736768+0000","last_change":"2023-10-01T09:21:02.158958+0000","last_active":"2023-10-01T09:23:48.736768+0000","last_peered":"2023-10-01T09:23:48.736768+0000","last_clean":"2023-10-01T09:23:48.736768+0000","last_became_active":"2023-10-01T09:21:02.158491+0000","last_became_peered":"2023-10-01T09:21:02.158491+0000","last_unstale":"2023-10-01T09:23:48.736768+0000","last_undegraded":"2023-10-01T09:23:48.736768+0000","last_fullsized":"2023-10-01T09:23:48.736768+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:21:48.885982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":207,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.918634+0000","last_change":"2023-10-01T09:21:02.163403+0000","last_active":"2023-10-01T09:23:46.918634+0000","last_peered":"2023-10-01T09:23:46.918634+0000","last_clean":"2023-10-01T09:23:46.918634+0000","last_became_active":"2023-10-01T09:21:02.121680+0000","last_became_peered":"2023-10-01T09:21:02.121680+0000","last_unstale":"2023-10-01T09:23:46.918634+0000","last_undegraded":"2023-10-01T09:23:46.918634+0000","last_fullsized":"2023-10-01T09:23:46.918634+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":230,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919944+0000","last_change":"2023-10-01T09:21:03.349475+0000","last_active":"2023-10-01T09:23:46.919944+0000","last_peered":"2023-10-01T09:23:46.919944+0000","last_clean":"2023-10-01T09:23:46.919944+0000","last_became_active":"2023-10-01T09:21:03.348251+0000","last_became_peered":"2023-10-01T09:21:03.348251+0000","last_unstale":"2023-10-01T09:23:46.919944+0000","last_undegraded":"2023-10-01T09:23:46.919944+0000","last_fullsized":"2023-10-01T09:23:46.919944+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":290,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.918463+0000","last_change":"2023-10-01T09:19:30.182046+0000","last_active":"2023-10-01T09:23:46.918463+0000","last_peered":"2023-10-01T09:23:46.918463+0000","last_clean":"2023-10-01T09:23:46.918463+0000","last_became_active":"2023-10-01T09:19:30.094579+0000","last_became_peered":"2023-10-01T09:19:30.094579+0000","last_unstale":"2023-10-01T09:23:46.918463+0000","last_undegraded":"2023-10-01T09:23:46.918463+0000","last_fullsized":"2023-10-01T09:23:46.918463+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":243,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.291593+0000","last_change":"2023-10-01T09:19:57.493670+0000","last_active":"2023-10-01T09:23:46.291593+0000","last_peered":"2023-10-01T09:23:46.291593+0000","last_clean":"2023-10-01T09:23:46.291593+0000","last_became_active":"2023-10-01T09:19:57.493349+0000","last_became_peered":"2023-10-01T09:19:57.493349+0000","last_unstale":"2023-10-01T09:23:46.291593+0000","last_undegraded":"2023-10-01T09:23:46.291593+0000","last_fullsized":"2023-10-01T09:23:46.291593+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:26:56.468779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":249,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.917280+0000","last_change":"2023-10-01T09:23:10.607388+0000","last_active":"2023-10-01T09:23:46.917280+0000","last_peered":"2023-10-01T09:23:46.917280+0000","last_clean":"2023-10-01T09:23:46.917280+0000","last_became_active":"2023-10-01T09:23:10.607084+0000","last_became_peered":"2023-10-01T09:23:10.607084+0000","last_unstale":"2023-10-01T09:23:46.917280+0000","last_undegraded":"2023-10-01T09:23:46.917280+0000","last_fullsized":"2023-10-01T09:23:46.917280+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:19.669393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":200,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919312+0000","last_change":"2023-10-01T09:22:02.185127+0000","last_active":"2023-10-01T09:23:46.919312+0000","last_peered":"2023-10-01T09:23:46.919312+0000","last_clean":"2023-10-01T09:23:46.919312+0000","last_became_active":"2023-10-01T09:22:02.184059+0000","last_became_peered":"2023-10-01T09:22:02.184059+0000","last_unstale":"2023-10-01T09:23:46.919312+0000","last_undegraded":"2023-10-01T09:23:46.919312+0000","last_fullsized":"2023-10-01T09:23:46.919312+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":231,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.918025+0000","last_change":"2023-10-01T09:21:02.121558+0000","last_active":"2023-10-01T09:23:46.918025+0000","last_peered":"2023-10-01T09:23:46.918025+0000","last_clean":"2023-10-01T09:23:46.918025+0000","last_became_active":"2023-10-01T09:21:02.121153+0000","last_became_peered":"2023-10-01T09:21:02.121153+0000","last_unstale":"2023-10-01T09:23:46.918025+0000","last_undegraded":"2023-10-01T09:23:46.918025+0000","last_fullsized":"2023-10-01T09:23:46.918025+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":231,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.917997+0000","last_change":"2023-10-01T09:23:10.611400+0000","last_active":"2023-10-01T09:23:46.917997+0000","last_peered":"2023-10-01T09:23:46.917997+0000","last_clean":"2023-10-01T09:23:46.917997+0000","last_became_active":"2023-10-01T09:23:10.610928+0000","last_became_peered":"2023-10-01T09:23:10.610928+0000","last_unstale":"2023-10-01T09:23:46.917997+0000","last_undegraded":"2023-10-01T09:23:46.917997+0000","last_fullsized":"2023-10-01T09:23:46.917997+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:17.141540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":208,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919107+0000","last_change":"2023-10-01T09:22:03.163438+0000","last_active":"2023-10-01T09:23:46.919107+0000","last_peered":"2023-10-01T09:23:46.919107+0000","last_clean":"2023-10-01T09:23:46.919107+0000","last_became_active":"2023-10-01T09:22:03.162821+0000","last_became_peered":"2023-10-01T09:22:03.162821+0000","last_unstale":"2023-10-01T09:23:46.919107+0000","last_undegraded":"2023-10-01T09:23:46.919107+0000","last_fullsized":"2023-10-01T09:23:46.919107+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":45,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.291974+0000","last_change":"2023-10-01T09:23:10.613386+0000","last_active":"2023-10-01T09:23:46.291974+0000","last_peered":"2023-10-01T09:23:46.291974+0000","last_clean":"2023-10-01T09:23:46.291974+0000","last_became_active":"2023-10-01T09:23:10.613046+0000","last_became_peered":"2023-10-01T09:23:10.613046+0000","last_unstale":"2023-10-01T09:23:46.291974+0000","last_undegraded":"2023-10-01T09:23:46.291974+0000","last_fullsized":"2023-10-01T09:23:46.291974+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:23:01.448841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"262'1334","reported_seq":1346,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:03.723656+0000","last_change":"2023-10-01T09:23:45.288346+0000","last_active":"2023-10-01T09:24:03.723656+0000","last_peered":"2023-10-01T09:24:03.723656+0000","last_clean":"2023-10-01T09:24:03.723656+0000","last_became_active":"2023-10-01T09:23:45.288016+0000","last_became_peered":"2023-10-01T09:23:45.288016+0000","last_unstale":"2023-10-01T09:24:03.723656+0000","last_undegraded":"2023-10-01T09:24:03.723656+0000","last_fullsized":"2023-10-01T09:24:03.723656+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1334,"log_dups_size":0,"ondisk_log_size":1334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:40.238301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87425024,"num_objects":1334,"num_object_clones":0,"num_object_copies":4002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1334,"num_write_kb":85376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":29,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919113+0000","last_change":"2023-10-01T09:23:10.601126+0000","last_active":"2023-10-01T09:23:46.919113+0000","last_peered":"2023-10-01T09:23:46.919113+0000","last_clean":"2023-10-01T09:23:46.919113+0000","last_became_active":"2023-10-01T09:23:10.598657+0000","last_became_peered":"2023-10-01T09:23:10.598657+0000","last_unstale":"2023-10-01T09:23:46.919113+0000","last_undegraded":"2023-10-01T09:23:46.919113+0000","last_fullsized":"2023-10-01T09:23:46.919113+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":197,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919203+0000","last_change":"2023-10-01T09:23:10.596661+0000","last_active":"2023-10-01T09:23:46.919203+0000","last_peered":"2023-10-01T09:23:46.919203+0000","last_clean":"2023-10-01T09:23:46.919203+0000","last_became_active":"2023-10-01T09:23:10.596299+0000","last_became_peered":"2023-10-01T09:23:10.596299+0000","last_unstale":"2023-10-01T09:23:46.919203+0000","last_undegraded":"2023-10-01T09:23:46.919203+0000","last_fullsized":"2023-10-01T09:23:46.919203+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"262'1163","reported_seq":1175,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:01.975341+0000","last_change":"2023-10-01T09:23:45.286941+0000","last_active":"2023-10-01T09:24:01.975341+0000","last_peered":"2023-10-01T09:24:01.975341+0000","last_clean":"2023-10-01T09:24:01.975341+0000","last_became_active":"2023-10-01T09:23:45.286690+0000","last_became_peered":"2023-10-01T09:23:45.286690+0000","last_unstale":"2023-10-01T09:24:01.975341+0000","last_undegraded":"2023-10-01T09:24:01.975341+0000","last_fullsized":"2023-10-01T09:24:01.975341+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1163,"log_dups_size":0,"ondisk_log_size":1163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:15:15.042008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76218368,"num_objects":1163,"num_object_clones":0,"num_object_copies":3489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1163,"num_write_kb":74432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":230,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.918975+0000","last_change":"2023-10-01T09:12:10.003552+0000","last_active":"2023-10-01T09:23:46.918975+0000","last_peered":"2023-10-01T09:23:46.918975+0000","last_clean":"2023-10-01T09:23:46.918975+0000","last_became_active":"2023-10-01T09:12:10.003131+0000","last_became_peered":"2023-10-01T09:12:10.003131+0000","last_unstale":"2023-10-01T09:23:46.918975+0000","last_undegraded":"2023-10-01T09:23:46.918975+0000","last_fullsized":"2023-10-01T09:23:46.918975+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":263,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.916530+0000","last_change":"2023-10-01T09:22:40.167822+0000","last_active":"2023-10-01T09:23:46.916530+0000","last_peered":"2023-10-01T09:23:46.916530+0000","last_clean":"2023-10-01T09:23:46.916530+0000","last_became_active":"2023-10-01T09:22:40.167502+0000","last_became_peered":"2023-10-01T09:22:40.167502+0000","last_unstale":"2023-10-01T09:23:46.916530+0000","last_undegraded":"2023-10-01T09:23:46.916530+0000","last_fullsized":"2023-10-01T09:23:46.916530+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:40:27.950717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"262'1278","reported_seq":1289,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:02.391281+0000","last_change":"2023-10-01T09:23:45.978822+0000","last_active":"2023-10-01T09:24:02.391281+0000","last_peered":"2023-10-01T09:24:02.391281+0000","last_clean":"2023-10-01T09:24:02.391281+0000","last_became_active":"2023-10-01T09:23:45.978517+0000","last_became_peered":"2023-10-01T09:23:45.978517+0000","last_unstale":"2023-10-01T09:24:02.391281+0000","last_undegraded":"2023-10-01T09:24:02.391281+0000","last_fullsized":"2023-10-01T09:24:02.391281+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1278,"log_dups_size":0,"ondisk_log_size":1278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:52.615311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83755008,"num_objects":1278,"num_object_clones":0,"num_object_copies":3834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1278,"num_write_kb":81792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":254,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.918692+0000","last_change":"2023-10-01T09:22:03.151056+0000","last_active":"2023-10-01T09:23:46.918692+0000","last_peered":"2023-10-01T09:23:46.918692+0000","last_clean":"2023-10-01T09:23:46.918692+0000","last_became_active":"2023-10-01T09:22:03.150643+0000","last_became_peered":"2023-10-01T09:22:03.150643+0000","last_unstale":"2023-10-01T09:23:46.918692+0000","last_undegraded":"2023-10-01T09:23:46.918692+0000","last_fullsized":"2023-10-01T09:23:46.918692+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":223,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919715+0000","last_change":"2023-10-01T09:23:10.610311+0000","last_active":"2023-10-01T09:23:46.919715+0000","last_peered":"2023-10-01T09:23:46.919715+0000","last_clean":"2023-10-01T09:23:46.919715+0000","last_became_active":"2023-10-01T09:23:10.609935+0000","last_became_peered":"2023-10-01T09:23:10.609935+0000","last_unstale":"2023-10-01T09:23:46.919715+0000","last_undegraded":"2023-10-01T09:23:46.919715+0000","last_fullsized":"2023-10-01T09:23:46.919715+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"262'1425","reported_seq":1437,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:04.909718+0000","last_change":"2023-10-01T09:23:45.977931+0000","last_active":"2023-10-01T09:24:04.909718+0000","last_peered":"2023-10-01T09:24:04.909718+0000","last_clean":"2023-10-01T09:24:04.909718+0000","last_became_active":"2023-10-01T09:23:45.977640+0000","last_became_peered":"2023-10-01T09:23:45.977640+0000","last_unstale":"2023-10-01T09:24:04.909718+0000","last_undegraded":"2023-10-01T09:24:04.909718+0000","last_fullsized":"2023-10-01T09:24:04.909718+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1425,"log_dups_size":0,"ondisk_log_size":1425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:33:08.487287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93388800,"num_objects":1425,"num_object_clones":0,"num_object_copies":4275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1425,"num_write_kb":91200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":213,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919591+0000","last_change":"2023-10-01T09:22:40.313521+0000","last_active":"2023-10-01T09:23:46.919591+0000","last_peered":"2023-10-01T09:23:46.919591+0000","last_clean":"2023-10-01T09:23:46.919591+0000","last_became_active":"2023-10-01T09:22:40.312941+0000","last_became_peered":"2023-10-01T09:22:40.312941+0000","last_unstale":"2023-10-01T09:23:46.919591+0000","last_undegraded":"2023-10-01T09:23:46.919591+0000","last_fullsized":"2023-10-01T09:23:46.919591+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":213,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.916832+0000","last_change":"2023-10-01T09:12:10.001565+0000","last_active":"2023-10-01T09:23:46.916832+0000","last_peered":"2023-10-01T09:23:46.916832+0000","last_clean":"2023-10-01T09:23:46.916832+0000","last_became_active":"2023-10-01T09:12:10.001275+0000","last_became_peered":"2023-10-01T09:12:10.001275+0000","last_unstale":"2023-10-01T09:23:46.916832+0000","last_undegraded":"2023-10-01T09:23:46.916832+0000","last_fullsized":"2023-10-01T09:23:46.916832+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:39.067046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"262'1197","reported_seq":1209,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:02.383805+0000","last_change":"2023-10-01T09:23:45.302878+0000","last_active":"2023-10-01T09:24:02.383805+0000","last_peered":"2023-10-01T09:24:02.383805+0000","last_clean":"2023-10-01T09:24:02.383805+0000","last_became_active":"2023-10-01T09:23:45.302301+0000","last_became_peered":"2023-10-01T09:23:45.302301+0000","last_unstale":"2023-10-01T09:24:02.383805+0000","last_undegraded":"2023-10-01T09:24:02.383805+0000","last_fullsized":"2023-10-01T09:24:02.383805+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1197,"log_dups_size":0,"ondisk_log_size":1197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:33:01.482516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78446592,"num_objects":1197,"num_object_clones":0,"num_object_copies":3591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1197,"num_write_kb":76608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":232,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.291443+0000","last_change":"2023-10-01T09:13:13.554321+0000","last_active":"2023-10-01T09:23:46.291443+0000","last_peered":"2023-10-01T09:23:46.291443+0000","last_clean":"2023-10-01T09:23:46.291443+0000","last_became_active":"2023-10-01T09:13:13.553986+0000","last_became_peered":"2023-10-01T09:13:13.553986+0000","last_unstale":"2023-10-01T09:23:46.291443+0000","last_undegraded":"2023-10-01T09:23:46.291443+0000","last_fullsized":"2023-10-01T09:23:46.291443+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:09:12.848269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":193,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.291640+0000","last_change":"2023-10-01T09:21:02.139718+0000","last_active":"2023-10-01T09:23:46.291640+0000","last_peered":"2023-10-01T09:23:46.291640+0000","last_clean":"2023-10-01T09:23:46.291640+0000","last_became_active":"2023-10-01T09:21:02.139406+0000","last_became_peered":"2023-10-01T09:21:02.139406+0000","last_unstale":"2023-10-01T09:23:46.291640+0000","last_undegraded":"2023-10-01T09:23:46.291640+0000","last_fullsized":"2023-10-01T09:23:46.291640+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:10.823093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"262'1497","reported_seq":1509,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:04.918984+0000","last_change":"2023-10-01T09:23:45.304043+0000","last_active":"2023-10-01T09:24:04.918984+0000","last_peered":"2023-10-01T09:24:04.918984+0000","last_clean":"2023-10-01T09:24:04.918984+0000","last_became_active":"2023-10-01T09:23:45.303689+0000","last_became_peered":"2023-10-01T09:23:45.303689+0000","last_unstale":"2023-10-01T09:24:04.918984+0000","last_undegraded":"2023-10-01T09:24:04.918984+0000","last_fullsized":"2023-10-01T09:24:04.918984+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1497,"log_dups_size":0,"ondisk_log_size":1497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:27:28.625644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98107392,"num_objects":1497,"num_object_clones":0,"num_object_copies":4491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1497,"num_write_kb":95808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":189,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.916943+0000","last_change":"2023-10-01T09:21:02.136776+0000","last_active":"2023-10-01T09:23:46.916943+0000","last_peered":"2023-10-01T09:23:46.916943+0000","last_clean":"2023-10-01T09:23:46.916943+0000","last_became_active":"2023-10-01T09:21:02.136476+0000","last_became_peered":"2023-10-01T09:21:02.136476+0000","last_unstale":"2023-10-01T09:23:46.916943+0000","last_undegraded":"2023-10-01T09:23:46.916943+0000","last_fullsized":"2023-10-01T09:23:46.916943+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:44:41.289825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":260,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.416518+0000","last_change":"2023-10-01T09:20:15.505658+0000","last_active":"2023-10-01T09:23:46.416518+0000","last_peered":"2023-10-01T09:23:46.416518+0000","last_clean":"2023-10-01T09:23:46.416518+0000","last_became_active":"2023-10-01T09:20:15.505361+0000","last_became_peered":"2023-10-01T09:20:15.505361+0000","last_unstale":"2023-10-01T09:23:46.416518+0000","last_undegraded":"2023-10-01T09:23:46.416518+0000","last_fullsized":"2023-10-01T09:23:46.416518+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:22:16.176432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.d","version":"262'1325","reported_seq":1337,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:03.736404+0000","last_change":"2023-10-01T09:23:45.303471+0000","last_active":"2023-10-01T09:24:03.736404+0000","last_peered":"2023-10-01T09:24:03.736404+0000","last_clean":"2023-10-01T09:24:03.736404+0000","last_became_active":"2023-10-01T09:23:45.303158+0000","last_became_peered":"2023-10-01T09:23:45.303158+0000","last_unstale":"2023-10-01T09:24:03.736404+0000","last_undegraded":"2023-10-01T09:24:03.736404+0000","last_fullsized":"2023-10-01T09:24:03.736404+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1325,"log_dups_size":0,"ondisk_log_size":1325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:13:38.249949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86835200,"num_objects":1325,"num_object_clones":0,"num_object_copies":3975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1325,"num_write_kb":84800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":130,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.291216+0000","last_change":"2023-10-01T09:19:30.092681+0000","last_active":"2023-10-01T09:23:46.291216+0000","last_peered":"2023-10-01T09:23:46.291216+0000","last_clean":"2023-10-01T09:23:46.291216+0000","last_became_active":"2023-10-01T09:19:30.092394+0000","last_became_peered":"2023-10-01T09:19:30.092394+0000","last_unstale":"2023-10-01T09:23:46.291216+0000","last_undegraded":"2023-10-01T09:23:46.291216+0000","last_fullsized":"2023-10-01T09:23:46.291216+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:41:50.127253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":55,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919533+0000","last_change":"2023-10-01T09:22:03.144199+0000","last_active":"2023-10-01T09:23:46.919533+0000","last_peered":"2023-10-01T09:23:46.919533+0000","last_clean":"2023-10-01T09:23:46.919533+0000","last_became_active":"2023-10-01T09:22:03.143883+0000","last_became_peered":"2023-10-01T09:22:03.143883+0000","last_unstale":"2023-10-01T09:23:46.919533+0000","last_undegraded":"2023-10-01T09:23:46.919533+0000","last_fullsized":"2023-10-01T09:23:46.919533+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.c","version":"262'1144","reported_seq":1156,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:01.985272+0000","last_change":"2023-10-01T09:23:45.288502+0000","last_active":"2023-10-01T09:24:01.985272+0000","last_peered":"2023-10-01T09:24:01.985272+0000","last_clean":"2023-10-01T09:24:01.985272+0000","last_became_active":"2023-10-01T09:23:45.288240+0000","last_became_peered":"2023-10-01T09:23:45.288240+0000","last_unstale":"2023-10-01T09:24:01.985272+0000","last_undegraded":"2023-10-01T09:24:01.985272+0000","last_fullsized":"2023-10-01T09:24:01.985272+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1144,"log_dups_size":0,"ondisk_log_size":1144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:24.693589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74973184,"num_objects":1144,"num_object_clones":0,"num_object_copies":3432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1144,"num_write_kb":73216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":36,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.290820+0000","last_change":"2023-10-01T09:22:40.316008+0000","last_active":"2023-10-01T09:23:46.290820+0000","last_peered":"2023-10-01T09:23:46.290820+0000","last_clean":"2023-10-01T09:23:46.290820+0000","last_became_active":"2023-10-01T09:22:40.315602+0000","last_became_peered":"2023-10-01T09:22:40.315602+0000","last_unstale":"2023-10-01T09:23:46.290820+0000","last_undegraded":"2023-10-01T09:23:46.290820+0000","last_fullsized":"2023-10-01T09:23:46.290820+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:39:37.485241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":127,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.917630+0000","last_change":"2023-10-01T09:18:31.419430+0000","last_active":"2023-10-01T09:23:46.917630+0000","last_peered":"2023-10-01T09:23:46.917630+0000","last_clean":"2023-10-01T09:23:46.917630+0000","last_became_active":"2023-10-01T09:18:31.419189+0000","last_became_peered":"2023-10-01T09:18:31.419189+0000","last_unstale":"2023-10-01T09:23:46.917630+0000","last_undegraded":"2023-10-01T09:23:46.917630+0000","last_fullsized":"2023-10-01T09:23:46.917630+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:08:48.247926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"262'1142","reported_seq":1154,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:01.975631+0000","last_change":"2023-10-01T09:23:45.978783+0000","last_active":"2023-10-01T09:24:01.975631+0000","last_peered":"2023-10-01T09:24:01.975631+0000","last_clean":"2023-10-01T09:24:01.975631+0000","last_became_active":"2023-10-01T09:23:45.978493+0000","last_became_peered":"2023-10-01T09:23:45.978493+0000","last_unstale":"2023-10-01T09:24:01.975631+0000","last_undegraded":"2023-10-01T09:24:01.975631+0000","last_fullsized":"2023-10-01T09:24:01.975631+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1142,"log_dups_size":0,"ondisk_log_size":1142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:46:27.395334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74842112,"num_objects":1142,"num_object_clones":0,"num_object_copies":3426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1142,"num_write_kb":73088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1079,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.930284+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:23:46.930284+0000","last_peered":"2023-10-01T09:23:46.930284+0000","last_clean":"2023-10-01T09:23:46.930284+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:23:46.930284+0000","last_undegraded":"2023-10-01T09:23:46.930284+0000","last_fullsized":"2023-10-01T09:23:46.930284+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":257,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919040+0000","last_change":"2023-10-01T09:22:40.170877+0000","last_active":"2023-10-01T09:23:46.919040+0000","last_peered":"2023-10-01T09:23:46.919040+0000","last_clean":"2023-10-01T09:23:46.919040+0000","last_became_active":"2023-10-01T09:22:40.170578+0000","last_became_peered":"2023-10-01T09:22:40.170578+0000","last_unstale":"2023-10-01T09:23:46.919040+0000","last_undegraded":"2023-10-01T09:23:46.919040+0000","last_fullsized":"2023-10-01T09:23:46.919040+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":209,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.916865+0000","last_change":"2023-10-01T09:22:02.151482+0000","last_active":"2023-10-01T09:23:46.916865+0000","last_peered":"2023-10-01T09:23:46.916865+0000","last_clean":"2023-10-01T09:23:46.916865+0000","last_became_active":"2023-10-01T09:22:02.151180+0000","last_became_peered":"2023-10-01T09:22:02.151180+0000","last_unstale":"2023-10-01T09:23:46.916865+0000","last_undegraded":"2023-10-01T09:23:46.916865+0000","last_fullsized":"2023-10-01T09:23:46.916865+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:11.745198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"262'1073","reported_seq":1084,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:01.170540+0000","last_change":"2023-10-01T09:23:45.303757+0000","last_active":"2023-10-01T09:24:01.170540+0000","last_peered":"2023-10-01T09:24:01.170540+0000","last_clean":"2023-10-01T09:24:01.170540+0000","last_became_active":"2023-10-01T09:23:45.303439+0000","last_became_peered":"2023-10-01T09:23:45.303439+0000","last_unstale":"2023-10-01T09:24:01.170540+0000","last_undegraded":"2023-10-01T09:24:01.170540+0000","last_fullsized":"2023-10-01T09:24:01.170540+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1073,"log_dups_size":0,"ondisk_log_size":1073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70320128,"num_objects":1073,"num_object_clones":0,"num_object_copies":3219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1073,"num_write_kb":68672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":923,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919363+0000","last_change":"2023-10-01T09:22:03.258449+0000","last_active":"2023-10-01T09:23:46.919363+0000","last_peered":"2023-10-01T09:23:46.919363+0000","last_clean":"2023-10-01T09:23:46.919363+0000","last_became_active":"2023-10-01T09:22:03.257766+0000","last_became_peered":"2023-10-01T09:22:03.257766+0000","last_unstale":"2023-10-01T09:23:46.919363+0000","last_undegraded":"2023-10-01T09:23:46.919363+0000","last_fullsized":"2023-10-01T09:23:46.919363+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":180,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.917129+0000","last_change":"2023-10-01T09:21:02.121225+0000","last_active":"2023-10-01T09:23:46.917129+0000","last_peered":"2023-10-01T09:23:46.917129+0000","last_clean":"2023-10-01T09:23:46.917129+0000","last_became_active":"2023-10-01T09:21:02.120911+0000","last_became_peered":"2023-10-01T09:21:02.120911+0000","last_unstale":"2023-10-01T09:23:46.917129+0000","last_undegraded":"2023-10-01T09:23:46.917129+0000","last_fullsized":"2023-10-01T09:23:46.917129+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:34:44.385101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":96,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919417+0000","last_change":"2023-10-01T09:22:40.172426+0000","last_active":"2023-10-01T09:23:46.919417+0000","last_peered":"2023-10-01T09:23:46.919417+0000","last_clean":"2023-10-01T09:23:46.919417+0000","last_became_active":"2023-10-01T09:22:40.172160+0000","last_became_peered":"2023-10-01T09:22:40.172160+0000","last_unstale":"2023-10-01T09:23:46.919417+0000","last_undegraded":"2023-10-01T09:23:46.919417+0000","last_fullsized":"2023-10-01T09:23:46.919417+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"262'1236","reported_seq":1248,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:02.385196+0000","last_change":"2023-10-01T09:23:45.978110+0000","last_active":"2023-10-01T09:24:02.385196+0000","last_peered":"2023-10-01T09:24:02.385196+0000","last_clean":"2023-10-01T09:24:02.385196+0000","last_became_active":"2023-10-01T09:23:45.977724+0000","last_became_peered":"2023-10-01T09:23:45.977724+0000","last_unstale":"2023-10-01T09:24:02.385196+0000","last_undegraded":"2023-10-01T09:24:02.385196+0000","last_fullsized":"2023-10-01T09:24:02.385196+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1236,"log_dups_size":0,"ondisk_log_size":1236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:11:17.874483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81002496,"num_objects":1236,"num_object_clones":0,"num_object_copies":3708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1236,"num_write_kb":79104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":587,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.416378+0000","last_change":"2023-10-01T09:22:40.183431+0000","last_active":"2023-10-01T09:23:46.416378+0000","last_peered":"2023-10-01T09:23:46.416378+0000","last_clean":"2023-10-01T09:23:46.416378+0000","last_became_active":"2023-10-01T09:22:40.171414+0000","last_became_peered":"2023-10-01T09:22:40.171414+0000","last_unstale":"2023-10-01T09:23:46.416378+0000","last_undegraded":"2023-10-01T09:23:46.416378+0000","last_fullsized":"2023-10-01T09:23:46.416378+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:56:42.904846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":263,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.917793+0000","last_change":"2023-10-01T09:21:02.140363+0000","last_active":"2023-10-01T09:23:46.917793+0000","last_peered":"2023-10-01T09:23:46.917793+0000","last_clean":"2023-10-01T09:23:46.917793+0000","last_became_active":"2023-10-01T09:21:02.137198+0000","last_became_peered":"2023-10-01T09:21:02.137198+0000","last_unstale":"2023-10-01T09:23:46.917793+0000","last_undegraded":"2023-10-01T09:23:46.917793+0000","last_fullsized":"2023-10-01T09:23:46.917793+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:38:38.043146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":124,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.917763+0000","last_change":"2023-10-01T09:22:02.156724+0000","last_active":"2023-10-01T09:23:46.917763+0000","last_peered":"2023-10-01T09:23:46.917763+0000","last_clean":"2023-10-01T09:23:46.917763+0000","last_became_active":"2023-10-01T09:22:02.156451+0000","last_became_peered":"2023-10-01T09:22:02.156451+0000","last_unstale":"2023-10-01T09:23:46.917763+0000","last_undegraded":"2023-10-01T09:23:46.917763+0000","last_fullsized":"2023-10-01T09:23:46.917763+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:29:05.232127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"262'1274","reported_seq":1285,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:02.395165+0000","last_change":"2023-10-01T09:23:45.302734+0000","last_active":"2023-10-01T09:24:02.395165+0000","last_peered":"2023-10-01T09:24:02.395165+0000","last_clean":"2023-10-01T09:24:02.395165+0000","last_became_active":"2023-10-01T09:23:45.302228+0000","last_became_peered":"2023-10-01T09:23:45.302228+0000","last_unstale":"2023-10-01T09:24:02.395165+0000","last_undegraded":"2023-10-01T09:24:02.395165+0000","last_fullsized":"2023-10-01T09:24:02.395165+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1274,"log_dups_size":0,"ondisk_log_size":1274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:35.109203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83492864,"num_objects":1274,"num_object_clones":0,"num_object_copies":3822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1274,"num_write_kb":81536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1320,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919280+0000","last_change":"2023-10-01T09:23:13.197266+0000","last_active":"2023-10-01T09:23:46.919280+0000","last_peered":"2023-10-01T09:23:46.919280+0000","last_clean":"2023-10-01T09:23:46.919280+0000","last_became_active":"2023-10-01T09:23:13.196795+0000","last_became_peered":"2023-10-01T09:23:13.196795+0000","last_unstale":"2023-10-01T09:23:46.919280+0000","last_undegraded":"2023-10-01T09:23:46.919280+0000","last_fullsized":"2023-10-01T09:23:46.919280+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":425,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.917970+0000","last_change":"2023-10-01T09:23:32.139254+0000","last_active":"2023-10-01T09:23:46.917970+0000","last_peered":"2023-10-01T09:23:46.917970+0000","last_clean":"2023-10-01T09:23:46.917970+0000","last_became_active":"2023-10-01T09:23:32.138752+0000","last_became_peered":"2023-10-01T09:23:32.138752+0000","last_unstale":"2023-10-01T09:23:46.917970+0000","last_undegraded":"2023-10-01T09:23:46.917970+0000","last_fullsized":"2023-10-01T09:23:46.917970+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":36,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.291444+0000","last_change":"2023-10-01T09:22:40.317678+0000","last_active":"2023-10-01T09:23:46.291444+0000","last_peered":"2023-10-01T09:23:46.291444+0000","last_clean":"2023-10-01T09:23:46.291444+0000","last_became_active":"2023-10-01T09:22:40.317460+0000","last_became_peered":"2023-10-01T09:22:40.317460+0000","last_unstale":"2023-10-01T09:23:46.291444+0000","last_undegraded":"2023-10-01T09:23:46.291444+0000","last_fullsized":"2023-10-01T09:23:46.291444+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:07:41.681679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":134,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.918792+0000","last_change":"2023-10-01T09:21:02.125167+0000","last_active":"2023-10-01T09:23:46.918792+0000","last_peered":"2023-10-01T09:23:46.918792+0000","last_clean":"2023-10-01T09:23:46.918792+0000","last_became_active":"2023-10-01T09:21:02.124791+0000","last_became_peered":"2023-10-01T09:21:02.124791+0000","last_unstale":"2023-10-01T09:23:46.918792+0000","last_undegraded":"2023-10-01T09:23:46.918792+0000","last_fullsized":"2023-10-01T09:23:46.918792+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.5","version":"262'1181","reported_seq":1192,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:01.472772+0000","last_change":"2023-10-01T09:23:45.303660+0000","last_active":"2023-10-01T09:24:01.472772+0000","last_peered":"2023-10-01T09:24:01.472772+0000","last_clean":"2023-10-01T09:24:01.472772+0000","last_became_active":"2023-10-01T09:23:45.303267+0000","last_became_peered":"2023-10-01T09:23:45.303267+0000","last_unstale":"2023-10-01T09:24:01.472772+0000","last_undegraded":"2023-10-01T09:24:01.472772+0000","last_fullsized":"2023-10-01T09:24:01.472772+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1181,"log_dups_size":0,"ondisk_log_size":1181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77398016,"num_objects":1181,"num_object_clones":0,"num_object_copies":3543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1181,"num_write_kb":75584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":358,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.918037+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:23:46.918037+0000","last_peered":"2023-10-01T09:23:46.918037+0000","last_clean":"2023-10-01T09:23:46.918037+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:23:46.918037+0000","last_undegraded":"2023-10-01T09:23:46.918037+0000","last_fullsized":"2023-10-01T09:23:46.918037+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":247,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.916600+0000","last_change":"2023-10-01T09:22:40.171122+0000","last_active":"2023-10-01T09:23:46.916600+0000","last_peered":"2023-10-01T09:23:46.916600+0000","last_clean":"2023-10-01T09:23:46.916600+0000","last_became_active":"2023-10-01T09:22:40.170773+0000","last_became_peered":"2023-10-01T09:22:40.170773+0000","last_unstale":"2023-10-01T09:23:46.916600+0000","last_undegraded":"2023-10-01T09:23:46.916600+0000","last_fullsized":"2023-10-01T09:23:46.916600+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:50:30.349100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":265,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.645442+0000","last_change":"2023-10-01T09:19:30.091481+0000","last_active":"2023-10-01T09:23:52.645442+0000","last_peered":"2023-10-01T09:23:52.645442+0000","last_clean":"2023-10-01T09:23:52.645442+0000","last_became_active":"2023-10-01T09:19:30.091151+0000","last_became_peered":"2023-10-01T09:19:30.091151+0000","last_unstale":"2023-10-01T09:23:52.645442+0000","last_undegraded":"2023-10-01T09:23:52.645442+0000","last_fullsized":"2023-10-01T09:23:52.645442+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:09.127408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"262'1566","reported_seq":1577,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.415353+0000","last_change":"2023-10-01T09:23:45.287796+0000","last_active":"2023-10-01T09:24:05.415353+0000","last_peered":"2023-10-01T09:24:05.415353+0000","last_clean":"2023-10-01T09:24:05.415353+0000","last_became_active":"2023-10-01T09:23:45.287413+0000","last_became_peered":"2023-10-01T09:23:45.287413+0000","last_unstale":"2023-10-01T09:24:05.415353+0000","last_undegraded":"2023-10-01T09:24:05.415353+0000","last_fullsized":"2023-10-01T09:24:05.415353+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1566,"log_dups_size":0,"ondisk_log_size":1566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102629376,"num_objects":1566,"num_object_clones":0,"num_object_copies":4698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1566,"num_write_kb":100224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":622,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.918757+0000","last_change":"2023-10-01T09:10:58.797964+0000","last_active":"2023-10-01T09:23:46.918757+0000","last_peered":"2023-10-01T09:23:46.918757+0000","last_clean":"2023-10-01T09:23:46.918757+0000","last_became_active":"2023-10-01T09:10:58.797447+0000","last_became_peered":"2023-10-01T09:10:58.797447+0000","last_unstale":"2023-10-01T09:23:46.918757+0000","last_undegraded":"2023-10-01T09:23:46.918757+0000","last_fullsized":"2023-10-01T09:23:46.918757+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":260,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.930289+0000","last_change":"2023-10-01T09:22:02.160793+0000","last_active":"2023-10-01T09:23:46.930289+0000","last_peered":"2023-10-01T09:23:46.930289+0000","last_clean":"2023-10-01T09:23:46.930289+0000","last_became_active":"2023-10-01T09:22:02.160579+0000","last_became_peered":"2023-10-01T09:22:02.160579+0000","last_unstale":"2023-10-01T09:23:46.930289+0000","last_undegraded":"2023-10-01T09:23:46.930289+0000","last_fullsized":"2023-10-01T09:23:46.930289+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:34:29.335908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":214,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.917051+0000","last_change":"2023-10-01T09:22:02.151402+0000","last_active":"2023-10-01T09:23:46.917051+0000","last_peered":"2023-10-01T09:23:46.917051+0000","last_clean":"2023-10-01T09:23:46.917051+0000","last_became_active":"2023-10-01T09:22:02.151028+0000","last_became_peered":"2023-10-01T09:22:02.151028+0000","last_unstale":"2023-10-01T09:23:46.917051+0000","last_undegraded":"2023-10-01T09:23:46.917051+0000","last_fullsized":"2023-10-01T09:23:46.917051+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:19.623044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"262'1121","reported_seq":1133,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:01.985609+0000","last_change":"2023-10-01T09:23:45.978785+0000","last_active":"2023-10-01T09:24:01.985609+0000","last_peered":"2023-10-01T09:24:01.985609+0000","last_clean":"2023-10-01T09:24:01.985609+0000","last_became_active":"2023-10-01T09:23:45.978512+0000","last_became_peered":"2023-10-01T09:23:45.978512+0000","last_unstale":"2023-10-01T09:24:01.985609+0000","last_undegraded":"2023-10-01T09:24:01.985609+0000","last_fullsized":"2023-10-01T09:24:01.985609+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1121,"log_dups_size":0,"ondisk_log_size":1121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:25:02.565270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73465856,"num_objects":1121,"num_object_clones":0,"num_object_copies":3363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1121,"num_write_kb":71744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":274,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.556328+0000","last_change":"2023-10-01T09:23:10.615956+0000","last_active":"2023-10-01T09:23:51.556328+0000","last_peered":"2023-10-01T09:23:51.556328+0000","last_clean":"2023-10-01T09:23:51.556328+0000","last_became_active":"2023-10-01T09:23:10.610919+0000","last_became_peered":"2023-10-01T09:23:10.610919+0000","last_unstale":"2023-10-01T09:23:51.556328+0000","last_undegraded":"2023-10-01T09:23:51.556328+0000","last_fullsized":"2023-10-01T09:23:51.556328+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:03:44.294373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":735,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.917293+0000","last_change":"2023-10-01T09:21:20.545328+0000","last_active":"2023-10-01T09:23:46.917293+0000","last_peered":"2023-10-01T09:23:46.917293+0000","last_clean":"2023-10-01T09:23:46.917293+0000","last_became_active":"2023-10-01T09:21:20.544828+0000","last_became_peered":"2023-10-01T09:21:20.544828+0000","last_unstale":"2023-10-01T09:23:46.917293+0000","last_undegraded":"2023-10-01T09:23:46.917293+0000","last_fullsized":"2023-10-01T09:23:46.917293+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:23:52.633471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":45,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.290950+0000","last_change":"2023-10-01T09:23:10.612309+0000","last_active":"2023-10-01T09:23:46.290950+0000","last_peered":"2023-10-01T09:23:46.290950+0000","last_clean":"2023-10-01T09:23:46.290950+0000","last_became_active":"2023-10-01T09:23:10.612034+0000","last_became_peered":"2023-10-01T09:23:10.612034+0000","last_unstale":"2023-10-01T09:23:46.290950+0000","last_undegraded":"2023-10-01T09:23:46.290950+0000","last_fullsized":"2023-10-01T09:23:46.290950+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:09:47.963186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"262'1212","reported_seq":1224,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:24:01.975565+0000","last_change":"2023-10-01T09:23:45.283913+0000","last_active":"2023-10-01T09:24:01.975565+0000","last_peered":"2023-10-01T09:24:01.975565+0000","last_clean":"2023-10-01T09:24:01.975565+0000","last_became_active":"2023-10-01T09:23:45.283600+0000","last_became_peered":"2023-10-01T09:23:45.283600+0000","last_unstale":"2023-10-01T09:24:01.975565+0000","last_undegraded":"2023-10-01T09:24:01.975565+0000","last_fullsized":"2023-10-01T09:24:01.975565+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":1212,"log_dups_size":0,"ondisk_log_size":1212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:24:30.501930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79429632,"num_objects":1212,"num_object_clones":0,"num_object_copies":3636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1212,"num_write_kb":77568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":93,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919754+0000","last_change":"2023-10-01T09:23:35.503601+0000","last_active":"2023-10-01T09:23:46.919754+0000","last_peered":"2023-10-01T09:23:46.919754+0000","last_clean":"2023-10-01T09:23:46.919754+0000","last_became_active":"2023-10-01T09:23:10.613270+0000","last_became_peered":"2023-10-01T09:23:10.613270+0000","last_unstale":"2023-10-01T09:23:46.919754+0000","last_undegraded":"2023-10-01T09:23:46.919754+0000","last_fullsized":"2023-10-01T09:23:46.919754+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":244,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.645363+0000","last_change":"2023-10-01T09:19:05.565472+0000","last_active":"2023-10-01T09:23:52.645363+0000","last_peered":"2023-10-01T09:23:52.645363+0000","last_clean":"2023-10-01T09:23:52.645363+0000","last_became_active":"2023-10-01T09:19:05.561161+0000","last_became_peered":"2023-10-01T09:19:05.561161+0000","last_unstale":"2023-10-01T09:23:52.645363+0000","last_undegraded":"2023-10-01T09:23:52.645363+0000","last_fullsized":"2023-10-01T09:23:52.645363+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:50:29.273388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":57,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.918913+0000","last_change":"2023-10-01T09:22:03.144097+0000","last_active":"2023-10-01T09:23:46.918913+0000","last_peered":"2023-10-01T09:23:46.918913+0000","last_clean":"2023-10-01T09:23:46.918913+0000","last_became_active":"2023-10-01T09:22:03.143800+0000","last_became_peered":"2023-10-01T09:22:03.143800+0000","last_unstale":"2023-10-01T09:23:46.918913+0000","last_undegraded":"2023-10-01T09:23:46.918913+0000","last_fullsized":"2023-10-01T09:23:46.918913+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":202,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.916977+0000","last_change":"2023-10-01T09:10:58.801161+0000","last_active":"2023-10-01T09:23:46.916977+0000","last_peered":"2023-10-01T09:23:46.916977+0000","last_clean":"2023-10-01T09:23:46.916977+0000","last_became_active":"2023-10-01T09:10:58.800794+0000","last_became_peered":"2023-10-01T09:10:58.800794+0000","last_unstale":"2023-10-01T09:23:46.916977+0000","last_undegraded":"2023-10-01T09:23:46.916977+0000","last_fullsized":"2023-10-01T09:23:46.916977+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:35.019445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":289,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.770310+0000","last_change":"2023-10-01T09:22:40.166927+0000","last_active":"2023-10-01T09:23:51.770310+0000","last_peered":"2023-10-01T09:23:51.770310+0000","last_clean":"2023-10-01T09:23:51.770310+0000","last_became_active":"2023-10-01T09:22:40.166606+0000","last_became_peered":"2023-10-01T09:22:40.166606+0000","last_unstale":"2023-10-01T09:23:51.770310+0000","last_undegraded":"2023-10-01T09:23:51.770310+0000","last_fullsized":"2023-10-01T09:23:51.770310+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:54:18.044207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":193,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919563+0000","last_change":"2023-10-01T09:22:02.180112+0000","last_active":"2023-10-01T09:23:46.919563+0000","last_peered":"2023-10-01T09:23:46.919563+0000","last_clean":"2023-10-01T09:23:46.919563+0000","last_became_active":"2023-10-01T09:22:02.161271+0000","last_became_peered":"2023-10-01T09:22:02.161271+0000","last_unstale":"2023-10-01T09:23:46.919563+0000","last_undegraded":"2023-10-01T09:23:46.919563+0000","last_fullsized":"2023-10-01T09:23:46.919563+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":208,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.555747+0000","last_change":"2023-10-01T09:19:05.553200+0000","last_active":"2023-10-01T09:23:51.555747+0000","last_peered":"2023-10-01T09:23:51.555747+0000","last_clean":"2023-10-01T09:23:51.555747+0000","last_became_active":"2023-10-01T09:19:05.552874+0000","last_became_peered":"2023-10-01T09:19:05.552874+0000","last_unstale":"2023-10-01T09:23:51.555747+0000","last_undegraded":"2023-10-01T09:23:51.555747+0000","last_fullsized":"2023-10-01T09:23:51.555747+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:14:31.040887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":182,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.919220+0000","last_change":"2023-10-01T09:22:03.149673+0000","last_active":"2023-10-01T09:23:46.919220+0000","last_peered":"2023-10-01T09:23:46.919220+0000","last_clean":"2023-10-01T09:23:46.919220+0000","last_became_active":"2023-10-01T09:22:03.149236+0000","last_became_peered":"2023-10-01T09:22:03.149236+0000","last_unstale":"2023-10-01T09:23:46.919220+0000","last_undegraded":"2023-10-01T09:23:46.919220+0000","last_fullsized":"2023-10-01T09:23:46.919220+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":190,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.555593+0000","last_change":"2023-10-01T09:18:54.533869+0000","last_active":"2023-10-01T09:23:51.555593+0000","last_peered":"2023-10-01T09:23:51.555593+0000","last_clean":"2023-10-01T09:23:51.555593+0000","last_became_active":"2023-10-01T09:18:54.533542+0000","last_became_peered":"2023-10-01T09:18:54.533542+0000","last_unstale":"2023-10-01T09:23:51.555593+0000","last_undegraded":"2023-10-01T09:23:51.555593+0000","last_fullsized":"2023-10-01T09:23:51.555593+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:18:19.126440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":36,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.291949+0000","last_change":"2023-10-01T09:22:40.313999+0000","last_active":"2023-10-01T09:23:46.291949+0000","last_peered":"2023-10-01T09:23:46.291949+0000","last_clean":"2023-10-01T09:23:46.291949+0000","last_became_active":"2023-10-01T09:22:40.313605+0000","last_became_peered":"2023-10-01T09:22:40.313605+0000","last_unstale":"2023-10-01T09:23:46.291949+0000","last_undegraded":"2023-10-01T09:23:46.291949+0000","last_fullsized":"2023-10-01T09:23:46.291949+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:52:32.154690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":36,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.291174+0000","last_change":"2023-10-01T09:22:40.170676+0000","last_active":"2023-10-01T09:23:46.291174+0000","last_peered":"2023-10-01T09:23:46.291174+0000","last_clean":"2023-10-01T09:23:46.291174+0000","last_became_active":"2023-10-01T09:22:40.170260+0000","last_became_peered":"2023-10-01T09:22:40.170260+0000","last_unstale":"2023-10-01T09:23:46.291174+0000","last_undegraded":"2023-10-01T09:23:46.291174+0000","last_fullsized":"2023-10-01T09:23:46.291174+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:14.693287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":293,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:46.917734+0000","last_change":"2023-10-01T09:22:02.157406+0000","last_active":"2023-10-01T09:23:46.917734+0000","last_peered":"2023-10-01T09:23:46.917734+0000","last_clean":"2023-10-01T09:23:46.917734+0000","last_became_active":"2023-10-01T09:22:02.157041+0000","last_became_peered":"2023-10-01T09:22:02.157041+0000","last_unstale":"2023-10-01T09:23:46.917734+0000","last_undegraded":"2023-10-01T09:23:46.917734+0000","last_fullsized":"2023-10-01T09:23:46.917734+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:24:52.008578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":184,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-10-01T09:23:48.736696+0000","last_change":"2023-10-01T09:21:02.141180+0000","last_active":"2023-10-01T09:23:48.736696+0000","last_peered":"2023-10-01T09:23:48.736696+0000","last_clean":"2023-10-01T09:23:48.736696+0000","last_became_active":"2023-10-01T09:21:02.136796+0000","last_became_peered":"2023-10-01T09:21:02.136796+0000","last_unstale":"2023-10-01T09:23:48.736696+0000","last_undegraded":"2023-10-01T09:23:48.736696+0000","last_fullsized":"2023-10-01T09:23:48.736696+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:03:46.040197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1321730048,"num_objects":20168,"num_object_clones":0,"num_object_copies":60504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20168,"num_write_kb":1290752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3954900992,"data_stored":3954900992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":20168,"ondisk_log_size":20168,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":12024,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35793,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430481,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":674976,"kb_used_data":365464,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93070176,"statfs":{"total":95995035648,"available":95303860224,"internally_reserved":0,"allocated":374235136,"data_stored":373513777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:15 2023","interfaces":[{"interface":"back","average":{"1min":2.573,"5min":5.2359999999999998,"15min":3.234},"min":{"1min":0.28100000000000003,"5min":0.23400000000000001,"15min":0.219},"max":{"1min":7.5190000000000001,"5min":117.855,"15min":117.855},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.891,"5min":5.2889999999999997,"15min":3.0249999999999999},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":6.1159999999999997,"5min":120.348,"15min":120.348},"last":3.157}]},{"osd":1,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.8600000000000001,"5min":5.3099999999999996,"15min":2.794},"min":{"1min":0.251,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":7.4279999999999999,"5min":135.023,"15min":135.023},"last":1.905},{"interface":"front","average":{"1min":2.7719999999999998,"5min":5.2800000000000002,"15min":2.6040000000000001},"min":{"1min":0.29499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":10.034000000000001,"5min":134.51499999999999,"15min":134.51499999999999},"last":0.84699999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":3.9249999999999998,"15min":2.5350000000000001},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.4539999999999997,"5min":104.837,"15min":104.837},"last":0.80300000000000005},{"interface":"front","average":{"1min":1.681,"5min":3.9950000000000001,"15min":2.508},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.9789999999999992,"5min":106.379,"15min":106.379},"last":1.464}]},{"osd":3,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1739999999999999,"5min":3.4870000000000001,"15min":1.8660000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26400000000000001},"max":{"1min":10.1,"5min":104.797,"15min":104.797},"last":0.58699999999999997},{"interface":"front","average":{"1min":2.8839999999999999,"5min":3.0760000000000001,"15min":1.8260000000000001},"min":{"1min":0.34100000000000003,"5min":0.29399999999999998,"15min":0.27200000000000002},"max":{"1min":7.2039999999999997,"5min":104.774,"15min":104.774},"last":0.65400000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:23:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5870000000000002,"5min":1.726,"15min":1.546},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.23699999999999999},"max":{"1min":6.5510000000000002,"5min":16.062999999999999,"15min":82.992000000000004},"last":0.74099999999999999},{"interface":"front","average":{"1min":2.1859999999999999,"5min":1.7629999999999999,"15min":1.579},"min":{"1min":0.39700000000000002,"5min":0.29499999999999998,"15min":0.252},"max":{"1min":6.2089999999999996,"5min":24.161999999999999,"15min":83.058999999999997},"last":1.4139999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:23:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":3.1339999999999999,"15min":3.1339999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":13.265000000000001,"5min":13.265000000000001,"15min":13.265000000000001},"last":0.624},{"interface":"front","average":{"1min":2.8940000000000001,"5min":2.8940000000000001,"15min":2.8940000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":13.292999999999999,"5min":13.292999999999999,"15min":13.292999999999999},"last":0.73399999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:23:15 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.4510000000000001,"15min":1.2070000000000001},"min":{"1min":0.35899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.5330000000000004,"5min":11.685,"15min":11.685},"last":0.77400000000000002},{"interface":"front","average":{"1min":2.0750000000000002,"5min":1.6619999999999999,"15min":1.1910000000000001},"min":{"1min":0.38,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.4220000000000006,"5min":11.871,"15min":11.871},"last":1.252}]}]},{"osd":6,"up_from":42,"seq":180388626709,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":695120,"kb_used_data":389704,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93050032,"statfs":{"total":95995035648,"available":95283232768,"internally_reserved":0,"allocated":399056896,"data_stored":398351164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8919999999999999,"5min":3.4470000000000001,"15min":3.8959999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20000000000000001},"max":{"1min":6.8200000000000003,"5min":137.941,"15min":178.738},"last":0.54000000000000004},{"interface":"front","average":{"1min":2.3599999999999999,"5min":3.7109999999999999,"15min":4.1600000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":10.992000000000001,"5min":138.20599999999999,"15min":178.684},"last":0.44500000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:23:30 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":4.6360000000000001,"15min":2.2879999999999998},"min":{"1min":0.253,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.1550000000000002,"5min":115.523,"15min":115.523},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.1699999999999999,"5min":5.0949999999999998,"15min":2.7240000000000002},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":11.042999999999999,"5min":115.681,"15min":115.681},"last":0.57499999999999996}]},{"osd":2,"last update":"Sun Oct 1 09:24:01 2023","interfaces":[{"interface":"back","average":{"1min":7.0149999999999997,"5min":3.1800000000000002,"15min":2.9430000000000001},"min":{"1min":0.33700000000000002,"5min":0.26500000000000001,"15min":0.25900000000000001},"max":{"1min":122.09399999999999,"5min":122.09399999999999,"15min":122.09399999999999},"last":122.09399999999999},{"interface":"front","average":{"1min":7.3209999999999997,"5min":2.7599999999999998,"15min":2.5800000000000001},"min":{"1min":0.32600000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":121.262,"5min":121.262,"15min":121.262},"last":121.262}]},{"osd":3,"last update":"Sun Oct 1 09:23:30 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":4.8799999999999999,"15min":2.403},"min":{"1min":0.307,"5min":0.307,"15min":0.27000000000000002},"max":{"1min":7.2050000000000001,"5min":219.32400000000001,"15min":219.32400000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.7769999999999999,"5min":5.1139999999999999,"15min":2.7069999999999999},"min":{"1min":0.37,"5min":0.33600000000000002,"15min":0.30499999999999999},"max":{"1min":8.6989999999999998,"5min":219.77099999999999,"15min":219.77099999999999},"last":0.59899999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:24:01 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":1.246,"15min":1.1459999999999999},"min":{"1min":0.28299999999999997,"5min":0.27200000000000002,"15min":0.23699999999999999},"max":{"1min":5.8479999999999999,"5min":14.089,"15min":28.271999999999998},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.22,"15min":1.171},"min":{"1min":0.308,"5min":0.29099999999999998,"15min":0.26000000000000001},"max":{"1min":4.9039999999999999,"5min":9.907,"15min":28.413},"last":0.317}]},{"osd":5,"last update":"Sun Oct 1 09:23:05 2023","interfaces":[{"interface":"back","average":{"1min":2.956,"5min":2.956,"15min":2.956},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.0760000000000005,"5min":9.0760000000000005,"15min":9.0760000000000005},"last":0.28100000000000003},{"interface":"front","average":{"1min":2.1389999999999998,"5min":2.1389999999999998,"15min":2.1389999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":13.999000000000001,"5min":13.999000000000001,"15min":13.999000000000001},"last":0.30299999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:23:12 2023","interfaces":[{"interface":"back","average":{"1min":2.355,"5min":3.1869999999999998,"15min":1.718},"min":{"1min":0.39000000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":11.492000000000001,"5min":145.37299999999999,"15min":145.37299999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":2.657,"5min":3.012,"15min":1.6579999999999999},"min":{"1min":0.35699999999999998,"5min":0.31,"15min":0.30499999999999999},"max":{"1min":9.1010000000000009,"5min":145.28200000000001,"15min":145.28200000000001},"last":0.34100000000000003}]}]},{"osd":1,"up_from":94,"seq":403726925969,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":765680,"kb_used_data":533928,"kb_used_omap":7,"kb_used_meta":231736,"kb_avail":92979472,"statfs":{"total":95995035648,"available":95210979328,"internally_reserved":0,"allocated":546742272,"data_stored":545544752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":237298087},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":6.3200000000000003,"15min":2.891},"min":{"1min":0.19700000000000001,"5min":0.19400000000000001,"15min":0.17499999999999999},"max":{"1min":7.2770000000000001,"5min":174.40199999999999,"15min":174.40199999999999},"last":1.4330000000000001},{"interface":"front","average":{"1min":2.952,"5min":6.4820000000000002,"15min":3.0430000000000001},"min":{"1min":0.219,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":7.702,"5min":173.77000000000001,"15min":173.77000000000001},"last":0.47699999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:23:03 2023","interfaces":[{"interface":"back","average":{"1min":8.3699999999999992,"5min":3,"15min":1.673},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":77.325000000000003,"5min":77.325000000000003,"15min":77.325000000000003},"last":1.351},{"interface":"front","average":{"1min":9.0679999999999996,"5min":3.222,"15min":1.825},"min":{"1min":0.26900000000000002,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":80.361000000000004,"5min":80.361000000000004,"15min":80.361000000000004},"last":0.44500000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:23:38 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":4.0899999999999999,"15min":1.992},"min":{"1min":0.224,"5min":0.224,"15min":0.218},"max":{"1min":6.593,"5min":99.421000000000006,"15min":99.421000000000006},"last":1.5649999999999999},{"interface":"front","average":{"1min":1.526,"5min":3.8860000000000001,"15min":1.853},"min":{"1min":0.30599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.4409999999999998,"5min":99.364999999999995,"15min":99.364999999999995},"last":1.4099999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":1.4079999999999999,"15min":1.2849999999999999},"min":{"1min":0.28799999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.8259999999999996,"5min":9.9169999999999998,"15min":40.673999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.7150000000000001,"5min":1.514,"15min":1.417},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.25800000000000001},"max":{"1min":7.2249999999999996,"5min":9.4610000000000003,"15min":40.810000000000002},"last":1.3799999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:23:03 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.403,"15min":1.403},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.734,"5min":4.734,"15min":4.734},"last":0.48499999999999999},{"interface":"front","average":{"1min":2.4369999999999998,"5min":2.4369999999999998,"15min":2.4369999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":5.5090000000000003,"5min":5.5090000000000003,"15min":5.5090000000000003},"last":1.306}]},{"osd":6,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8260000000000001,"5min":1.5980000000000001,"15min":1.3300000000000001},"min":{"1min":0.35099999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.9860000000000007,"5min":14.452,"15min":14.452},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.29,"5min":1.605,"15min":1.226},"min":{"1min":0.35799999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.9740000000000002,"5min":9.9740000000000002,"15min":10.787000000000001},"last":0.45800000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":1.96,"15min":1.431},"min":{"1min":0.42099999999999999,"5min":0.317,"15min":0.317},"max":{"1min":7.8129999999999997,"5min":23.710999999999999,"15min":23.710999999999999},"last":1.028},{"interface":"front","average":{"1min":1.853,"5min":2.0049999999999999,"15min":1.4450000000000001},"min":{"1min":0.39500000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.3579999999999997,"5min":23.736999999999998,"15min":23.736999999999998},"last":1.4490000000000001}]}]},{"osd":0,"up_from":89,"seq":382252089503,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":985164,"kb_used_data":601156,"kb_used_omap":8,"kb_used_meta":383991,"kb_avail":92759988,"statfs":{"total":95995035648,"available":94986227712,"internally_reserved":0,"allocated":615583744,"data_stored":614363691,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":393207746},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:23:28 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":4.202,"15min":1.99},"min":{"1min":0.221,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":13.054,"5min":187.41399999999999,"15min":187.41399999999999},"last":0.31900000000000001},{"interface":"front","average":{"1min":3.9529999999999998,"5min":4.6779999999999999,"15min":2.1880000000000002},"min":{"1min":0.23400000000000001,"5min":0.193,"15min":0.193},"max":{"1min":17.088999999999999,"5min":187.78700000000001,"15min":187.78700000000001},"last":0.56899999999999995}]},{"osd":2,"last update":"Sun Oct 1 09:22:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":2.2490000000000001,"15min":1.3939999999999999},"min":{"1min":0.24399999999999999,"5min":0.23300000000000001,"15min":0.23000000000000001},"max":{"1min":4.7699999999999996,"5min":39.048999999999999,"15min":39.048999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.514,"5min":2.7719999999999998,"15min":1.591},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.673,"5min":38.945999999999998,"15min":38.945999999999998},"last":0.52400000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:23:28 2023","interfaces":[{"interface":"back","average":{"1min":2.335,"5min":3.2770000000000001,"15min":1.655},"min":{"1min":0.26600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":17.914999999999999,"5min":85.840000000000003,"15min":85.840000000000003},"last":0.39500000000000002},{"interface":"front","average":{"1min":3.5259999999999998,"5min":3.6829999999999998,"15min":1.8500000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23400000000000001},"max":{"1min":11.141999999999999,"5min":86.046000000000006,"15min":86.046000000000006},"last":3.157}]},{"osd":4,"last update":"Sun Oct 1 09:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":3.3420000000000001,"15min":1.8799999999999999},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.6479999999999997,"5min":112.52200000000001,"15min":112.52200000000001},"last":0.623},{"interface":"front","average":{"1min":2.3439999999999999,"5min":3.2999999999999998,"15min":1.9830000000000001},"min":{"1min":0.33700000000000002,"5min":0.32500000000000001,"15min":0.28599999999999998},"max":{"1min":8.9890000000000008,"5min":113.124,"15min":113.124},"last":0.26900000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:23:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8480000000000001,"5min":1.8480000000000001,"15min":1.8480000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":5.8710000000000004,"5min":5.8710000000000004,"15min":5.8710000000000004},"last":0.432},{"interface":"front","average":{"1min":1.958,"5min":1.958,"15min":1.958},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":5.8879999999999999,"5min":5.8879999999999999,"15min":5.8879999999999999},"last":0.47299999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":2.0910000000000002,"15min":1.577},"min":{"1min":0.375,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.0239999999999991,"5min":34.375999999999998,"15min":34.375999999999998},"last":0.55100000000000005},{"interface":"front","average":{"1min":2.6709999999999998,"5min":1.8899999999999999,"15min":1.379},"min":{"1min":0.39200000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":9.0820000000000007,"5min":31.745000000000001,"15min":31.745000000000001},"last":0.44800000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":3.0150000000000001,"15min":1.867},"min":{"1min":0.44,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":13.096,"5min":98.283000000000001,"15min":98.283000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":2.3119999999999998,"5min":2.762,"15min":1.8700000000000001},"min":{"1min":0.36399999999999999,"5min":0.312,"15min":0.312},"max":{"1min":5.7560000000000002,"5min":98.311999999999998,"15min":98.311999999999998},"last":0.59499999999999997}]}]},{"osd":2,"up_from":99,"seq":425201762443,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":881292,"kb_used_data":545476,"kb_used_omap":11,"kb_used_meta":335796,"kb_avail":92863860,"statfs":{"total":95995035648,"available":95092592640,"internally_reserved":0,"allocated":558567424,"data_stored":557403046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":343855991},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":7.5149999999999997,"5min":3.0230000000000001,"15min":2.0899999999999999},"min":{"1min":0.19800000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":119.621,"5min":119.621,"15min":119.621},"last":0.84899999999999998},{"interface":"front","average":{"1min":7.0289999999999999,"5min":3.2010000000000001,"15min":2.3940000000000001},"min":{"1min":0.20999999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":119.411,"5min":119.411,"15min":119.411},"last":3.3250000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.032,"5min":3.3170000000000002,"15min":2.1269999999999998},"min":{"1min":0.19600000000000001,"5min":0.193,"15min":0.193},"max":{"1min":8.4399999999999995,"5min":105.798,"15min":105.798},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.452,"5min":3.2120000000000002,"15min":2.2669999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.4039999999999999,"5min":105.56399999999999,"15min":107.729},"last":1.643}]},{"osd":3,"last update":"Sun Oct 1 09:23:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6059999999999999,"5min":6.2910000000000004,"15min":2.7429999999999999},"min":{"1min":0.217,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":25.975000000000001,"5min":253.238,"15min":253.238},"last":1.04},{"interface":"front","average":{"1min":2.8929999999999998,"5min":6.1580000000000004,"15min":2.6970000000000001},"min":{"1min":0.30599999999999999,"5min":0.253,"15min":0.22600000000000001},"max":{"1min":22.498999999999999,"5min":252.54900000000001,"15min":252.54900000000001},"last":1.901}]},{"osd":4,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1850000000000001,"5min":1.974,"15min":1.321},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27000000000000002},"max":{"1min":8.2530000000000001,"5min":16.059000000000001,"15min":17.751999999999999},"last":1.681},{"interface":"front","average":{"1min":2.5579999999999998,"5min":2.1030000000000002,"15min":1.409},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":14.359,"5min":14.359,"15min":17.568000000000001},"last":0.35299999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:23:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5289999999999999,"5min":2.5289999999999999,"15min":2.5289999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.2840000000000007,"5min":8.2840000000000007,"15min":8.2840000000000007},"last":0.79900000000000004},{"interface":"front","average":{"1min":2.3580000000000001,"5min":2.3580000000000001,"15min":2.3580000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":9.3160000000000007,"5min":9.3160000000000007,"15min":9.3160000000000007},"last":1.7110000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":3.2320000000000002,"5min":2.1259999999999999,"15min":1.466},"min":{"1min":0.35199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":14.411,"5min":14.411,"15min":17.591000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.8919999999999999,"5min":1.7629999999999999,"15min":1.2010000000000001},"min":{"1min":0.40400000000000003,"5min":0.36699999999999999,"15min":0.34200000000000003},"max":{"1min":8.6620000000000008,"5min":8.6620000000000008,"15min":17.736000000000001},"last":0.44800000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:23:06 2023","interfaces":[{"interface":"back","average":{"1min":3.6739999999999999,"5min":2.0129999999999999,"15min":1.782},"min":{"1min":0.38200000000000001,"5min":0.34699999999999998,"15min":0.32800000000000001},"max":{"1min":16.169,"5min":16.169,"15min":109.506},"last":0.47599999999999998},{"interface":"front","average":{"1min":3.1680000000000001,"5min":2.0329999999999999,"15min":1.8979999999999999},"min":{"1min":0.375,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":16.152000000000001,"5min":16.152000000000001,"15min":109.58799999999999},"last":1.742}]}]},{"osd":3,"up_from":104,"seq":446676598916,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":893024,"kb_used_data":650968,"kb_used_omap":3,"kb_used_meta":242044,"kb_avail":92852128,"statfs":{"total":95995035648,"available":95080579072,"internally_reserved":0,"allocated":666591232,"data_stored":665406562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":247853799},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":2.9199999999999999,"5min":1.9630000000000001,"15min":2.9689999999999999},"min":{"1min":0.20899999999999999,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":18.635000000000002,"5min":38.314,"15min":199.422},"last":0.23599999999999999},{"interface":"front","average":{"1min":2.976,"5min":2.1680000000000001,"15min":3.1509999999999998},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":18.792000000000002,"5min":38.118000000000002,"15min":199.91200000000001},"last":0.255}]},{"osd":1,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":5.431,"5min":5.702,"15min":3.3940000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":37.011000000000003,"5min":228.60499999999999,"15min":228.60499999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":4.3399999999999999,"5min":5.3099999999999996,"15min":3.2269999999999999},"min":{"1min":0.26500000000000001,"5min":0.23899999999999999,"15min":0.215},"max":{"1min":36.866,"5min":228.28399999999999,"15min":228.28399999999999},"last":0.29899999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":2.6930000000000001,"15min":1.51},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.224},"max":{"1min":7.524,"5min":55.238999999999997,"15min":55.238999999999997},"last":35.109000000000002},{"interface":"front","average":{"1min":1.99,"5min":2.9409999999999998,"15min":1.6779999999999999},"min":{"1min":0.23999999999999999,"5min":0.219,"15min":0.20399999999999999},"max":{"1min":6.7919999999999998,"5min":55.692999999999998,"15min":55.692999999999998},"last":35.213999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5,"5min":4.2729999999999997,"15min":2.274},"min":{"1min":0.32600000000000001,"5min":0.27300000000000002,"15min":0.25900000000000001},"max":{"1min":9.0250000000000004,"5min":154.00399999999999,"15min":154.00399999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":3.8420000000000001,"5min":4.8140000000000001,"15min":2.601},"min":{"1min":0.316,"5min":0.30099999999999999,"15min":0.248},"max":{"1min":36.947000000000003,"5min":156.87700000000001,"15min":156.87700000000001},"last":0.55300000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":1.7769999999999999,"15min":1.7769999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.5670000000000002,"5min":7.5670000000000002,"15min":7.5670000000000002},"last":0.67300000000000004},{"interface":"front","average":{"1min":3.8439999999999999,"5min":3.8439999999999999,"15min":3.8439999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":36.984000000000002,"5min":36.984000000000002,"15min":36.984000000000002},"last":0.745}]},{"osd":6,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":4.3070000000000004,"5min":3.5739999999999998,"15min":2.0369999999999999},"min":{"1min":0.36499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":36.901000000000003,"5min":72.433999999999997,"15min":72.433999999999997},"last":0.82899999999999996},{"interface":"front","average":{"1min":4.1470000000000002,"5min":3.1899999999999999,"15min":1.8100000000000001},"min":{"1min":0.33600000000000002,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":37.274999999999999,"5min":72.450999999999993,"15min":72.450999999999993},"last":0.81100000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":4.7910000000000004,"5min":2.2290000000000001,"15min":1.712},"min":{"1min":0.35499999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":37.024000000000001,"5min":38.478999999999999,"15min":79.412999999999997},"last":0.71699999999999997},{"interface":"front","average":{"1min":4.867,"5min":2.415,"15min":1.7490000000000001},"min":{"1min":0.40799999999999997,"5min":0.36399999999999999,"15min":0.308},"max":{"1min":36.762,"5min":40.039999999999999,"15min":79.527000000000001},"last":0.69199999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019166,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":678360,"kb_used_data":368848,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93066792,"statfs":{"total":95995035648,"available":95300395008,"internally_reserved":0,"allocated":377700352,"data_stored":376986571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":3.468,"15min":3.3849999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.20999999999999999},"max":{"1min":4.4690000000000003,"5min":111.27800000000001,"15min":133.91499999999999},"last":0.61499999999999999},{"interface":"front","average":{"1min":2.581,"5min":3.7410000000000001,"15min":3.3540000000000001},"min":{"1min":0.26600000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.5099999999999998,"5min":111.289,"15min":133.83199999999999},"last":0.27300000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":2.6110000000000002,"5min":2.4399999999999999,"15min":2.02},"min":{"1min":0.30599999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.5589999999999993,"5min":37.326999999999998,"15min":133.93600000000001},"last":0.37},{"interface":"front","average":{"1min":2.0600000000000001,"5min":2.234,"15min":2.105},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.0009999999999994,"5min":37.335000000000001,"15min":141.06},"last":0.315}]},{"osd":2,"last update":"Sun Oct 1 09:23:07 2023","interfaces":[{"interface":"back","average":{"1min":4.367,"5min":3.2690000000000001,"15min":2.9929999999999999},"min":{"1min":0.35299999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":26.545000000000002,"5min":91.474999999999994,"15min":133.96100000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":4.375,"5min":3.4969999999999999,"15min":2.9689999999999999},"min":{"1min":0.33000000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":28.324000000000002,"5min":91.510999999999996,"15min":134.018},"last":1.161}]},{"osd":3,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.5029999999999999,"15min":1.6439999999999999},"min":{"1min":0.32000000000000001,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":6.0679999999999996,"5min":6.9909999999999997,"15min":134.297},"last":0.32300000000000001},{"interface":"front","average":{"1min":2.9540000000000002,"5min":1.982,"15min":1.9650000000000001},"min":{"1min":0.30399999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":18.370000000000001,"5min":18.370000000000001,"15min":141.089},"last":1.2090000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:23:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.6899999999999999,"15min":1.6899999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.7839999999999998,"5min":5.7839999999999998,"15min":5.7839999999999998},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.681,"5min":2.681,"15min":2.681},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.6539999999999999,"5min":8.6539999999999999,"15min":8.6539999999999999},"last":0.42499999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.502,"15min":1.508},"min":{"1min":0.28799999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.5789999999999997,"5min":8.8900000000000006,"15min":133.90799999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.478,"5min":1.337,"15min":1.744},"min":{"1min":0.38600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.5750000000000002,"5min":20.622,"15min":141.03},"last":0.38900000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:23:08 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":1.401,"15min":1.6100000000000001},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.0579999999999998,"5min":8.8369999999999997,"15min":133.946},"last":0.38100000000000001},{"interface":"front","average":{"1min":2.5859999999999999,"5min":1.5449999999999999,"15min":1.631},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":8.7140000000000004,"5min":11.215,"15min":133.98400000000001},"last":0.51700000000000002}]}]},{"osd":5,"up_from":227,"seq":974957576218,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":779784,"kb_used_data":440064,"kb_used_omap":8,"kb_used_meta":339703,"kb_avail":92965368,"statfs":{"total":95995035648,"available":95196536832,"internally_reserved":0,"allocated":450625536,"data_stored":449923587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":347856644},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":2.0499999999999998,"15min":2.0499999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":7.7699999999999996,"5min":7.7699999999999996,"15min":7.7699999999999996},"last":0.27500000000000002},{"interface":"front","average":{"1min":2.7130000000000001,"5min":2.7130000000000001,"15min":2.7130000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":10.867000000000001,"5min":10.867000000000001,"15min":10.867000000000001},"last":0.60299999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":2.2690000000000001,"15min":2.2690000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.2110000000000003,"5min":6.2110000000000003,"15min":6.2110000000000003},"last":0.55800000000000005},{"interface":"front","average":{"1min":3.3039999999999998,"5min":3.3039999999999998,"15min":3.3039999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.4280000000000008,"5min":9.4280000000000008,"15min":9.4280000000000008},"last":0.39500000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8879999999999999,"5min":2.8879999999999999,"15min":2.8879999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.3049999999999997,"5min":9.3049999999999997,"15min":9.3049999999999997},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.7589999999999999,"15min":1.7589999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.8419999999999996,"5min":4.8419999999999996,"15min":4.8419999999999996},"last":0.48199999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":2.2250000000000001,"15min":2.2250000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.5640000000000001,"5min":6.5640000000000001,"15min":6.5640000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":2.1739999999999999,"5min":2.1739999999999999,"15min":2.1739999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.7199999999999998,"5min":7.7199999999999998,"15min":7.7199999999999998},"last":0.51100000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":1.891,"15min":1.891},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":7.532,"5min":7.532,"15min":7.532},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":4.117,"5min":4.117,"15min":4.117},"last":0.58199999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":1.948,"5min":1.948,"15min":1.948},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.7520000000000007,"5min":8.7520000000000007,"15min":8.7520000000000007},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.401,"5min":1.401,"15min":1.401},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.7460000000000004,"5min":7.7460000000000004,"15min":7.7460000000000004},"last":0.33100000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:23:02 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":1.6599999999999999,"15min":1.6599999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.6440000000000001,"5min":4.6440000000000001,"15min":4.6440000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.2330000000000001,"5min":2.2330000000000001,"15min":2.2330000000000001},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":10.836,"5min":10.836,"15min":10.836},"last":0.56699999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":648806400,"data_stored":648806400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":543752192,"data_stored":543752192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":582680576,"data_stored":582680576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":689242112,"data_stored":689242112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":377618432,"data_stored":377618432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":395771904,"data_stored":395771904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":348913664,"data_stored":348913664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":368115712,"data_stored":368115712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:24:09.079 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:09.112 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:09.168 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:09.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:09.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:09.273 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:09.306 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:09.368 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:09.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:09.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:09.474 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:24:09.507 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:09.566 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:24:09.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:09.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:24:09.676 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:09.759 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:09.883 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 16 16 31209 31193 121.774 108.438 0.010062 0.00820586 2023-10-01T09:24:10.447 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:24:10.748 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:10 smithi138 ceph-mon[185886]: pgmap v668: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-01T09:24:10.748 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:24:10.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:10 smithi138 ceph-mon[192513]: pgmap v668: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-01T09:24:10.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:24:10.762 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:10.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:10.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:10.873 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:24:10.884 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 17 16 32236 32220 118.378 64.1875 0.00798423 0.00844301 2023-10-01T09:24:10.904 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:10 smithi190 ceph-mon[160435]: pgmap v668: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-01T09:24:10.904 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:24:10.906 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:10.960 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:24:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:10.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:24:11.066 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:11.108 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:24:11.153 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:24:11.165 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:11.198 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:24:11.235 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:24:11.279 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:24:11.309 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:24:11.347 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:24:11.382 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:11.384 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:24:11.399 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:24:11.400 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:24:11.400 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:24:11.400 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:24:11.400 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:24:11.400 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:24:11.400 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:24:11.401 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:24:11.415 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:11.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:11.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:11.522 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:11.549 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:24:11.550 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:24:11.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:24:11.550 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:24:11.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.550 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95256 6.1 6:85462a48:::benchmark_data_smithi138_307751_object95255:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-01T09:24:11.550 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:01.373190+0000", 2023-10-01T09:24:11.550 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 190.172508924, 2023-10-01T09:24:11.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.87755441599999995, 2023-10-01T09:24:11.550 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.551 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:24:11.551 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.551 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.551 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.551 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95256 2023-10-01T09:24:11.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.551 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.373190+0000", 2023-10-01T09:24:11.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.373190+0000", 2023-10-01T09:24:11.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.373189+0000", 2023-10-01T09:24:11.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:24:11.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.373243+0000", 2023-10-01T09:24:11.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4184000000000002e-05 2023-10-01T09:24:11.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.373250+0000", 2023-10-01T09:24:11.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.314e-06 2023-10-01T09:24:11.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.373254+0000", 2023-10-01T09:24:11.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7999999999999998e-06 2023-10-01T09:24:11.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.377023+0000", 2023-10-01T09:24:11.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003768764 2023-10-01T09:24:11.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.377179+0000", 2023-10-01T09:24:11.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015610299999999999 2023-10-01T09:24:11.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:24:11.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.377374+0000", 2023-10-01T09:24:11.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019462099999999999 2023-10-01T09:24:11.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.386021+0000", 2023-10-01T09:24:11.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086471499999999993 2023-10-01T09:24:11.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.390887+0000", 2023-10-01T09:24:11.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048662430000000001 2023-10-01T09:24:11.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.406652+0000", 2023-10-01T09:24:11.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015764347000000001 2023-10-01T09:24:11.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.406664+0000", 2023-10-01T09:24:11.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2298e-05 2023-10-01T09:24:11.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.250646+0000", 2023-10-01T09:24:11.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84398162300000001 2023-10-01T09:24:11.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.558 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.558 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.250744+0000", 2023-10-01T09:24:11.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8435000000000002e-05 2023-10-01T09:24:11.558 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.558 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.558 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.558 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95269 6.4 6:22be3102:::benchmark_data_smithi138_307751_object95268:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-01T09:24:11.559 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:01.399198+0000", 2023-10-01T09:24:11.559 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 190.14650072500001, 2023-10-01T09:24:11.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.007680863, 2023-10-01T09:24:11.559 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.559 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.559 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.559 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.559 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.559 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95269 2023-10-01T09:24:11.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.559 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.559 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399198+0000", 2023-10-01T09:24:11.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399198+0000", 2023-10-01T09:24:11.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399196+0000", 2023-10-01T09:24:11.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:24:11.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399248+0000", 2023-10-01T09:24:11.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2265999999999997e-05 2023-10-01T09:24:11.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399266+0000", 2023-10-01T09:24:11.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7453e-05 2023-10-01T09:24:11.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399274+0000", 2023-10-01T09:24:11.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3750000000000003e-06 2023-10-01T09:24:11.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.451332+0000", 2023-10-01T09:24:11.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.052058412999999998 2023-10-01T09:24:11.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.451344+0000", 2023-10-01T09:24:11.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1488000000000001e-05 2023-10-01T09:24:11.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.128553+0000", 2023-10-01T09:24:11.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67720899700000003 2023-10-01T09:24:11.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.128565+0000", 2023-10-01T09:24:11.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1986e-05 2023-10-01T09:24:11.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.370376+0000", 2023-10-01T09:24:11.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2418113319999999 2023-10-01T09:24:11.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.370494+0000", 2023-10-01T09:24:11.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117982 2023-10-01T09:24:11.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:11.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.370621+0000", 2023-10-01T09:24:11.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012717699999999999 2023-10-01T09:24:11.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.393133+0000", 2023-10-01T09:24:11.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.022511258999999999 2023-10-01T09:24:11.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.398001+0000", 2023-10-01T09:24:11.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048688330000000004 2023-10-01T09:24:11.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.402860+0000", 2023-10-01T09:24:11.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048585390000000003 2023-10-01T09:24:11.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.402903+0000", 2023-10-01T09:24:11.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2982999999999998e-05 2023-10-01T09:24:11.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.406879+0000", 2023-10-01T09:24:11.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039757539999999997 2023-10-01T09:24:11.567 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.568 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.568 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.568 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95270 6.4 6:27c5a4cb:::benchmark_data_smithi138_307751_object95269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-01T09:24:11.568 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:01.399284+0000", 2023-10-01T09:24:11.568 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 190.14641477500001, 2023-10-01T09:24:11.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0401369049999998, 2023-10-01T09:24:11.568 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.568 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.569 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.569 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.569 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.569 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95270 2023-10-01T09:24:11.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.569 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399284+0000", 2023-10-01T09:24:11.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399284+0000", 2023-10-01T09:24:11.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399283+0000", 2023-10-01T09:24:11.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:24:11.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399323+0000", 2023-10-01T09:24:11.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9362e-05 2023-10-01T09:24:11.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399330+0000", 2023-10-01T09:24:11.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.99e-06 2023-10-01T09:24:11.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399335+0000", 2023-10-01T09:24:11.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2769999999999996e-06 2023-10-01T09:24:11.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.451365+0000", 2023-10-01T09:24:11.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.052030013 2023-10-01T09:24:11.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.451371+0000", 2023-10-01T09:24:11.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1859999999999997e-06 2023-10-01T09:24:11.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.128588+0000", 2023-10-01T09:24:11.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67721646599999996 2023-10-01T09:24:11.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.128595+0000", 2023-10-01T09:24:11.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0199999999999997e-06 2023-10-01T09:24:11.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.406917+0000", 2023-10-01T09:24:11.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.278322467 2023-10-01T09:24:11.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.407076+0000", 2023-10-01T09:24:11.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015908299999999999 2023-10-01T09:24:11.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:11.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.407217+0000", 2023-10-01T09:24:11.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014033200000000001 2023-10-01T09:24:11.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.413442+0000", 2023-10-01T09:24:11.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0062251240000000003 2023-10-01T09:24:11.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.415175+0000", 2023-10-01T09:24:11.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001733278 2023-10-01T09:24:11.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.436343+0000", 2023-10-01T09:24:11.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.021167605999999999 2023-10-01T09:24:11.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.436383+0000", 2023-10-01T09:24:11.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.074e-05 2023-10-01T09:24:11.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.439421+0000", 2023-10-01T09:24:11.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003037456 2023-10-01T09:24:11.578 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.578 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.578 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.579 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95285 6.4 6:20d08f7b:::benchmark_data_smithi138_307751_object95284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-10-01T09:24:11.579 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:02.212413+0000", 2023-10-01T09:24:11.579 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 189.333285668, 2023-10-01T09:24:11.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2300618379999999, 2023-10-01T09:24:11.579 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.579 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.579 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.579 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.579 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.579 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95285 2023-10-01T09:24:11.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.579 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.212413+0000", 2023-10-01T09:24:11.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.212413+0000", 2023-10-01T09:24:11.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.212407+0000", 2023-10-01T09:24:11.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999938 2023-10-01T09:24:11.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.212455+0000", 2023-10-01T09:24:11.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7827999999999997e-05 2023-10-01T09:24:11.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.212475+0000", 2023-10-01T09:24:11.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0061999999999999e-05 2023-10-01T09:24:11.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.212491+0000", 2023-10-01T09:24:11.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6752000000000002e-05 2023-10-01T09:24:11.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.232186+0000", 2023-10-01T09:24:11.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019694593999999999 2023-10-01T09:24:11.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.232199+0000", 2023-10-01T09:24:11.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2581e-05 2023-10-01T09:24:11.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.409492+0000", 2023-10-01T09:24:11.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.177293527 2023-10-01T09:24:11.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.409581+0000", 2023-10-01T09:24:11.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9182000000000001e-05 2023-10-01T09:24:11.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:11.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.409677+0000", 2023-10-01T09:24:11.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5585e-05 2023-10-01T09:24:11.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.413457+0000", 2023-10-01T09:24:11.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037799370000000001 2023-10-01T09:24:11.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.415218+0000", 2023-10-01T09:24:11.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017614009999999999 2023-10-01T09:24:11.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.439462+0000", 2023-10-01T09:24:11.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024244137999999998 2023-10-01T09:24:11.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.439500+0000", 2023-10-01T09:24:11.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7540999999999999e-05 2023-10-01T09:24:11.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.442475+0000", 2023-10-01T09:24:11.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029749310000000001 2023-10-01T09:24:11.587 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.588 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.588 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.588 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96670 6.1 6:89741715:::benchmark_data_smithi138_307751_object1301:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:24:11.588 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.610842+0000", 2023-10-01T09:24:11.588 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 171.93485647899999, 2023-10-01T09:24:11.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94294599000000001, 2023-10-01T09:24:11.588 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.589 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.589 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.589 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.589 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.589 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96670 2023-10-01T09:24:11.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.589 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610842+0000", 2023-10-01T09:24:11.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610842+0000", 2023-10-01T09:24:11.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610840+0000", 2023-10-01T09:24:11.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:24:11.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610848+0000", 2023-10-01T09:24:11.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6909999999999997e-06 2023-10-01T09:24:11.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610866+0000", 2023-10-01T09:24:11.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7680999999999999e-05 2023-10-01T09:24:11.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610874+0000", 2023-10-01T09:24:11.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8480000000000007e-06 2023-10-01T09:24:11.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610935+0000", 2023-10-01T09:24:11.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0109000000000003e-05 2023-10-01T09:24:11.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610946+0000", 2023-10-01T09:24:11.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.145e-05 2023-10-01T09:24:11.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.545063+0000", 2023-10-01T09:24:11.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93411715900000003 2023-10-01T09:24:11.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.545224+0000", 2023-10-01T09:24:11.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016116500000000001 2023-10-01T09:24:11.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:24:11.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.545369+0000", 2023-10-01T09:24:11.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000144501 2023-10-01T09:24:11.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.553649+0000", 2023-10-01T09:24:11.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0082804039999999999 2023-10-01T09:24:11.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.553661+0000", 2023-10-01T09:24:11.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2086000000000001e-05 2023-10-01T09:24:11.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.553694+0000", 2023-10-01T09:24:11.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2292000000000002e-05 2023-10-01T09:24:11.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.553742+0000", 2023-10-01T09:24:11.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8338999999999999e-05 2023-10-01T09:24:11.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.553788+0000", 2023-10-01T09:24:11.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6186999999999997e-05 2023-10-01T09:24:11.596 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.596 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.596 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.597 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96672 6.1 6:832703e2:::benchmark_data_smithi138_307751_object1303:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:24:11.597 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.611352+0000", 2023-10-01T09:24:11.597 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 171.93434646899999, 2023-10-01T09:24:11.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.948355422, 2023-10-01T09:24:11.597 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.597 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.597 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.597 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.597 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.597 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96672 2023-10-01T09:24:11.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.598 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611352+0000", 2023-10-01T09:24:11.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611352+0000", 2023-10-01T09:24:11.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611351+0000", 2023-10-01T09:24:11.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:24:11.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611356+0000", 2023-10-01T09:24:11.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0869999999999997e-06 2023-10-01T09:24:11.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611371+0000", 2023-10-01T09:24:11.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4251e-05 2023-10-01T09:24:11.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611380+0000", 2023-10-01T09:24:11.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9169999999999995e-06 2023-10-01T09:24:11.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611430+0000", 2023-10-01T09:24:11.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0557999999999999e-05 2023-10-01T09:24:11.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611442+0000", 2023-10-01T09:24:11.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1501000000000001e-05 2023-10-01T09:24:11.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.548561+0000", 2023-10-01T09:24:11.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.937118856 2023-10-01T09:24:11.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.548685+0000", 2023-10-01T09:24:11.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012445499999999999 2023-10-01T09:24:11.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:24:11.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.548760+0000", 2023-10-01T09:24:11.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5332000000000003e-05 2023-10-01T09:24:11.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.604 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:11.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.553926+0000", 2023-10-01T09:24:11.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051653580000000001 2023-10-01T09:24:11.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559605+0000", 2023-10-01T09:24:11.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056791380000000002 2023-10-01T09:24:11.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559634+0000", 2023-10-01T09:24:11.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9139999999999999e-05 2023-10-01T09:24:11.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559667+0000", 2023-10-01T09:24:11.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2948999999999999e-05 2023-10-01T09:24:11.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559708+0000", 2023-10-01T09:24:11.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0606999999999999e-05 2023-10-01T09:24:11.607 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.607 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.607 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.607 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96683 6.1 6:82591597:::benchmark_data_smithi138_307751_object1314:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:24:11.607 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.636038+0000", 2023-10-01T09:24:11.607 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 171.90966063600001, 2023-10-01T09:24:11.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92376125799999997, 2023-10-01T09:24:11.608 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.608 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.608 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.608 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.608 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.608 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96683 2023-10-01T09:24:11.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.608 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636038+0000", 2023-10-01T09:24:11.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636038+0000", 2023-10-01T09:24:11.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636036+0000", 2023-10-01T09:24:11.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:24:11.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636045+0000", 2023-10-01T09:24:11.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.054e-06 2023-10-01T09:24:11.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636063+0000", 2023-10-01T09:24:11.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8025000000000001e-05 2023-10-01T09:24:11.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636072+0000", 2023-10-01T09:24:11.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0829999999999993e-06 2023-10-01T09:24:11.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636127+0000", 2023-10-01T09:24:11.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4907999999999997e-05 2023-10-01T09:24:11.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636137+0000", 2023-10-01T09:24:11.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3589999999999997e-06 2023-10-01T09:24:11.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.550899+0000", 2023-10-01T09:24:11.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91476241400000002 2023-10-01T09:24:11.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.550976+0000", 2023-10-01T09:24:11.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6611000000000002e-05 2023-10-01T09:24:11.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:24:11.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.551047+0000", 2023-10-01T09:24:11.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1879999999999996e-05 2023-10-01T09:24:11.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559596+0000", 2023-10-01T09:24:11.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085482849999999992 2023-10-01T09:24:11.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559611+0000", 2023-10-01T09:24:11.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5514000000000001e-05 2023-10-01T09:24:11.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559740+0000", 2023-10-01T09:24:11.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012912000000000001 2023-10-01T09:24:11.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559766+0000", 2023-10-01T09:24:11.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5973e-05 2023-10-01T09:24:11.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559799+0000", 2023-10-01T09:24:11.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2895000000000003e-05 2023-10-01T09:24:11.617 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.617 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.617 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.617 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96694 6.1 6:823df3f4:::benchmark_data_smithi138_307751_object1325:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:24:11.617 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.638010+0000", 2023-10-01T09:24:11.617 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 171.90768812300001, 2023-10-01T09:24:11.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92454698599999996, 2023-10-01T09:24:11.617 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.617 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.618 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.618 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.618 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.618 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96694 2023-10-01T09:24:11.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.618 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638010+0000", 2023-10-01T09:24:11.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638010+0000", 2023-10-01T09:24:11.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638010+0000", 2023-10-01T09:24:11.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:24:11.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638013+0000", 2023-10-01T09:24:11.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.402e-06 2023-10-01T09:24:11.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638021+0000", 2023-10-01T09:24:11.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6479999999999992e-06 2023-10-01T09:24:11.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638026+0000", 2023-10-01T09:24:11.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1340000000000003e-06 2023-10-01T09:24:11.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638089+0000", 2023-10-01T09:24:11.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2309000000000003e-05 2023-10-01T09:24:11.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638097+0000", 2023-10-01T09:24:11.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6880000000000005e-06 2023-10-01T09:24:11.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.553957+0000", 2023-10-01T09:24:11.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91585984399999998 2023-10-01T09:24:11.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.554068+0000", 2023-10-01T09:24:11.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111307 2023-10-01T09:24:11.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:24:11.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.554154+0000", 2023-10-01T09:24:11.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5555000000000003e-05 2023-10-01T09:24:11.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559811+0000", 2023-10-01T09:24:11.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056574449999999997 2023-10-01T09:24:11.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559846+0000", 2023-10-01T09:24:11.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4576999999999998e-05 2023-10-01T09:24:11.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.562443+0000", 2023-10-01T09:24:11.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025973630000000001 2023-10-01T09:24:11.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.562497+0000", 2023-10-01T09:24:11.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3965000000000003e-05 2023-10-01T09:24:11.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.562557+0000", 2023-10-01T09:24:11.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0052e-05 2023-10-01T09:24:11.626 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.627 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.627 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.627 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96696 6.1 6:85d4c37b:::benchmark_data_smithi138_307751_object1327:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:24:11.627 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.642189+0000", 2023-10-01T09:24:11.627 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 171.90350997300001, 2023-10-01T09:24:11.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92620703800000004, 2023-10-01T09:24:11.627 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.627 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.628 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.628 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.628 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.628 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96696 2023-10-01T09:24:11.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.628 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642189+0000", 2023-10-01T09:24:11.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642189+0000", 2023-10-01T09:24:11.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642187+0000", 2023-10-01T09:24:11.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:24:11.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642193+0000", 2023-10-01T09:24:11.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9179999999999999e-06 2023-10-01T09:24:11.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642215+0000", 2023-10-01T09:24:11.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1492999999999999e-05 2023-10-01T09:24:11.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642222+0000", 2023-10-01T09:24:11.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3180000000000001e-06 2023-10-01T09:24:11.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642274+0000", 2023-10-01T09:24:11.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1579999999999997e-05 2023-10-01T09:24:11.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642283+0000", 2023-10-01T09:24:11.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7010000000000008e-06 2023-10-01T09:24:11.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.557012+0000", 2023-10-01T09:24:11.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91472850299999997 2023-10-01T09:24:11.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.557138+0000", 2023-10-01T09:24:11.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000126604 2023-10-01T09:24:11.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:24:11.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.557224+0000", 2023-10-01T09:24:11.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6123000000000002e-05 2023-10-01T09:24:11.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.562413+0000", 2023-10-01T09:24:11.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051883889999999998 2023-10-01T09:24:11.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.565661+0000", 2023-10-01T09:24:11.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032479039999999998 2023-10-01T09:24:11.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568310+0000", 2023-10-01T09:24:11.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0026492170000000002 2023-10-01T09:24:11.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568342+0000", 2023-10-01T09:24:11.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1946000000000002e-05 2023-10-01T09:24:11.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568396+0000", 2023-10-01T09:24:11.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3701e-05 2023-10-01T09:24:11.636 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.636 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.636 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.636 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96700 6.1 6:86043e7e:::benchmark_data_smithi138_307751_object1331:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:24:11.636 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.649900+0000", 2023-10-01T09:24:11.636 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 171.89579877, 2023-10-01T09:24:11.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91865500700000002, 2023-10-01T09:24:11.637 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.637 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.637 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.637 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.637 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.637 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96700 2023-10-01T09:24:11.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.637 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649900+0000", 2023-10-01T09:24:11.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649900+0000", 2023-10-01T09:24:11.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649899+0000", 2023-10-01T09:24:11.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:24:11.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649905+0000", 2023-10-01T09:24:11.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3450000000000002e-06 2023-10-01T09:24:11.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649919+0000", 2023-10-01T09:24:11.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3854e-05 2023-10-01T09:24:11.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649926+0000", 2023-10-01T09:24:11.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3409999999999997e-06 2023-10-01T09:24:11.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649985+0000", 2023-10-01T09:24:11.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8761999999999997e-05 2023-10-01T09:24:11.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649997+0000", 2023-10-01T09:24:11.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2571999999999999e-05 2023-10-01T09:24:11.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559870+0000", 2023-10-01T09:24:11.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.90987212200000001 2023-10-01T09:24:11.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559945+0000", 2023-10-01T09:24:11.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5581999999999996e-05 2023-10-01T09:24:11.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:24:11.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.560021+0000", 2023-10-01T09:24:11.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6348999999999997e-05 2023-10-01T09:24:11.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568248+0000", 2023-10-01T09:24:11.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0082266019999999995 2023-10-01T09:24:11.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568261+0000", 2023-10-01T09:24:11.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3191e-05 2023-10-01T09:24:11.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568465+0000", 2023-10-01T09:24:11.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020326900000000001 2023-10-01T09:24:11.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568516+0000", 2023-10-01T09:24:11.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1004999999999997e-05 2023-10-01T09:24:11.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568555+0000", 2023-10-01T09:24:11.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9289000000000003e-05 2023-10-01T09:24:11.644 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.644 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.644 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.645 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96724 6.1 6:85bdae1f:::benchmark_data_smithi138_307751_object1355:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:24:11.645 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.681228+0000", 2023-10-01T09:24:11.645 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 171.86447101600001, 2023-10-01T09:24:11.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.89070485099999996, 2023-10-01T09:24:11.645 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.645 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.645 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.645 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.645 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.645 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96724 2023-10-01T09:24:11.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.646 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681228+0000", 2023-10-01T09:24:11.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681228+0000", 2023-10-01T09:24:11.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681226+0000", 2023-10-01T09:24:11.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:24:11.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681232+0000", 2023-10-01T09:24:11.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5030000000000002e-06 2023-10-01T09:24:11.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681244+0000", 2023-10-01T09:24:11.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1861999999999999e-05 2023-10-01T09:24:11.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681252+0000", 2023-10-01T09:24:11.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8790000000000003e-06 2023-10-01T09:24:11.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681312+0000", 2023-10-01T09:24:11.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9669000000000001e-05 2023-10-01T09:24:11.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681322+0000", 2023-10-01T09:24:11.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7170000000000003e-06 2023-10-01T09:24:11.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.562614+0000", 2023-10-01T09:24:11.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88129269799999999 2023-10-01T09:24:11.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.562760+0000", 2023-10-01T09:24:11.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014615200000000001 2023-10-01T09:24:11.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:24:11.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.562888+0000", 2023-10-01T09:24:11.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012731900000000001 2023-10-01T09:24:11.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568571+0000", 2023-10-01T09:24:11.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056834939999999999 2023-10-01T09:24:11.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.571795+0000", 2023-10-01T09:24:11.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003223722 2023-10-01T09:24:11.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.571840+0000", 2023-10-01T09:24:11.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5331999999999999e-05 2023-10-01T09:24:11.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.571895+0000", 2023-10-01T09:24:11.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4256999999999999e-05 2023-10-01T09:24:11.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.571932+0000", 2023-10-01T09:24:11.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7846999999999999e-05 2023-10-01T09:24:11.652 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.652 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.653 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.653 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96735 6.1 6:86696907:::benchmark_data_smithi138_307751_object1366:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:24:11.653 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.695283+0000", 2023-10-01T09:24:11.653 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 171.85041565200001, 2023-10-01T09:24:11.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88571714300000004, 2023-10-01T09:24:11.653 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.653 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.653 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.653 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.654 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.654 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96735 2023-10-01T09:24:11.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.654 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695283+0000", 2023-10-01T09:24:11.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695283+0000", 2023-10-01T09:24:11.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695281+0000", 2023-10-01T09:24:11.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:24:11.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695287+0000", 2023-10-01T09:24:11.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9649999999999996e-06 2023-10-01T09:24:11.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695299+0000", 2023-10-01T09:24:11.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1955e-05 2023-10-01T09:24:11.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695308+0000", 2023-10-01T09:24:11.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4950000000000008e-06 2023-10-01T09:24:11.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695378+0000", 2023-10-01T09:24:11.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0333999999999994e-05 2023-10-01T09:24:11.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695392+0000", 2023-10-01T09:24:11.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3529999999999999e-05 2023-10-01T09:24:11.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.565707+0000", 2023-10-01T09:24:11.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.87031560699999999 2023-10-01T09:24:11.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.565818+0000", 2023-10-01T09:24:11.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111097 2023-10-01T09:24:11.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:24:11.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.565913+0000", 2023-10-01T09:24:11.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4927999999999996e-05 2023-10-01T09:24:11.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.571732+0000", 2023-10-01T09:24:11.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005819243 2023-10-01T09:24:11.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.580894+0000", 2023-10-01T09:24:11.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0091619010000000001 2023-10-01T09:24:11.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.580924+0000", 2023-10-01T09:24:11.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0165999999999999e-05 2023-10-01T09:24:11.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.580956+0000", 2023-10-01T09:24:11.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1526000000000003e-05 2023-10-01T09:24:11.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581000+0000", 2023-10-01T09:24:11.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4057000000000002e-05 2023-10-01T09:24:11.661 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.661 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.661 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.661 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96736 6.1 6:8b2701b9:::benchmark_data_smithi138_307751_object1367:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:24:11.662 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.695846+0000", 2023-10-01T09:24:11.662 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 171.84985246700001, 2023-10-01T09:24:11.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88535111399999999, 2023-10-01T09:24:11.662 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.662 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.662 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.662 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.662 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.662 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96736 2023-10-01T09:24:11.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.663 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695846+0000", 2023-10-01T09:24:11.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695846+0000", 2023-10-01T09:24:11.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695845+0000", 2023-10-01T09:24:11.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:24:11.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695851+0000", 2023-10-01T09:24:11.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1580000000000002e-06 2023-10-01T09:24:11.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695869+0000", 2023-10-01T09:24:11.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7900000000000001e-05 2023-10-01T09:24:11.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695878+0000", 2023-10-01T09:24:11.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6880000000000005e-06 2023-10-01T09:24:11.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695933+0000", 2023-10-01T09:24:11.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5234e-05 2023-10-01T09:24:11.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695946+0000", 2023-10-01T09:24:11.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3251000000000001e-05 2023-10-01T09:24:11.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568597+0000", 2023-10-01T09:24:11.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.87265044899999999 2023-10-01T09:24:11.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568719+0000", 2023-10-01T09:24:11.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012276100000000001 2023-10-01T09:24:11.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:24:11.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568851+0000", 2023-10-01T09:24:11.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131965 2023-10-01T09:24:11.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581034+0000", 2023-10-01T09:24:11.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012183100000000001 2023-10-01T09:24:11.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581042+0000", 2023-10-01T09:24:11.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8259999999999993e-06 2023-10-01T09:24:11.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581117+0000", 2023-10-01T09:24:11.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4518999999999996e-05 2023-10-01T09:24:11.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581163+0000", 2023-10-01T09:24:11.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6390999999999997e-05 2023-10-01T09:24:11.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581197+0000", 2023-10-01T09:24:11.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3998999999999998e-05 2023-10-01T09:24:11.670 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.670 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.670 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.671 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96756 6.1 6:850b9578:::benchmark_data_smithi138_307751_object1387:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:24:11.671 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.726318+0000", 2023-10-01T09:24:11.671 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 171.81938022700001, 2023-10-01T09:24:11.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.855017943, 2023-10-01T09:24:11.671 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.671 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.671 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.671 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.671 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96756 2023-10-01T09:24:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726318+0000", 2023-10-01T09:24:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726318+0000", 2023-10-01T09:24:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726317+0000", 2023-10-01T09:24:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:24:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726323+0000", 2023-10-01T09:24:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2779999999999999e-06 2023-10-01T09:24:11.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726336+0000", 2023-10-01T09:24:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3047000000000001e-05 2023-10-01T09:24:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726344+0000", 2023-10-01T09:24:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2900000000000002e-06 2023-10-01T09:24:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726415+0000", 2023-10-01T09:24:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0298999999999999e-05 2023-10-01T09:24:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726429+0000", 2023-10-01T09:24:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4561e-05 2023-10-01T09:24:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.571965+0000", 2023-10-01T09:24:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84553552099999996 2023-10-01T09:24:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.572048+0000", 2023-10-01T09:24:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3534000000000004e-05 2023-10-01T09:24:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:24:11.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.572116+0000", 2023-10-01T09:24:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7259000000000001e-05 2023-10-01T09:24:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581050+0000", 2023-10-01T09:24:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0089346210000000002 2023-10-01T09:24:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581093+0000", 2023-10-01T09:24:11.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2784000000000002e-05 2023-10-01T09:24:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581269+0000", 2023-10-01T09:24:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017614700000000001 2023-10-01T09:24:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581310+0000", 2023-10-01T09:24:11.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1050999999999997e-05 2023-10-01T09:24:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581336+0000", 2023-10-01T09:24:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6142000000000001e-05 2023-10-01T09:24:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.679 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96761 6.1 6:8f74277c:::benchmark_data_smithi138_307751_object1392:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:24:11.680 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.733089+0000", 2023-10-01T09:24:11.680 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 171.812609307, 2023-10-01T09:24:11.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84834553899999998, 2023-10-01T09:24:11.680 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.680 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.680 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.680 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.680 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.680 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96761 2023-10-01T09:24:11.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.681 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.733089+0000", 2023-10-01T09:24:11.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.733089+0000", 2023-10-01T09:24:11.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.733087+0000", 2023-10-01T09:24:11.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:24:11.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.733095+0000", 2023-10-01T09:24:11.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1189999999999997e-06 2023-10-01T09:24:11.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.733114+0000", 2023-10-01T09:24:11.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8527999999999999e-05 2023-10-01T09:24:11.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.733123+0000", 2023-10-01T09:24:11.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8409999999999995e-06 2023-10-01T09:24:11.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.738263+0000", 2023-10-01T09:24:11.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051396510000000003 2023-10-01T09:24:11.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.738271+0000", 2023-10-01T09:24:11.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6760000000000007e-06 2023-10-01T09:24:11.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.574939+0000", 2023-10-01T09:24:11.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.83666783199999994 2023-10-01T09:24:11.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.575034+0000", 2023-10-01T09:24:11.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5224999999999997e-05 2023-10-01T09:24:11.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:24:11.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.575128+0000", 2023-10-01T09:24:11.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4622000000000003e-05 2023-10-01T09:24:11.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581056+0000", 2023-10-01T09:24:11.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059276659999999998 2023-10-01T09:24:11.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581239+0000", 2023-10-01T09:24:11.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018291999999999999 2023-10-01T09:24:11.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.687 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581374+0000", 2023-10-01T09:24:11.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000135406 2023-10-01T09:24:11.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.687 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581400+0000", 2023-10-01T09:24:11.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5099000000000001e-05 2023-10-01T09:24:11.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581435+0000", 2023-10-01T09:24:11.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5250000000000003e-05 2023-10-01T09:24:11.688 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.688 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.688 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.688 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96769 6.1 6:89cc583e:::benchmark_data_smithi138_307751_object1400:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:24:11.688 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.751200+0000", 2023-10-01T09:24:11.689 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 171.79449837999999, 2023-10-01T09:24:11.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84244887800000001, 2023-10-01T09:24:11.689 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.689 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.689 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.689 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.689 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.689 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96769 2023-10-01T09:24:11.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.689 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751200+0000", 2023-10-01T09:24:11.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751200+0000", 2023-10-01T09:24:11.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751199+0000", 2023-10-01T09:24:11.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:24:11.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751212+0000", 2023-10-01T09:24:11.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3451999999999999e-05 2023-10-01T09:24:11.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751227+0000", 2023-10-01T09:24:11.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4599e-05 2023-10-01T09:24:11.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751235+0000", 2023-10-01T09:24:11.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4309999999999994e-06 2023-10-01T09:24:11.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751291+0000", 2023-10-01T09:24:11.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6378000000000001e-05 2023-10-01T09:24:11.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751301+0000", 2023-10-01T09:24:11.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5149999999999995e-06 2023-10-01T09:24:11.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.577917+0000", 2023-10-01T09:24:11.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.82661580400000001 2023-10-01T09:24:11.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.578029+0000", 2023-10-01T09:24:11.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011215 2023-10-01T09:24:11.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:24:11.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.578137+0000", 2023-10-01T09:24:11.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108558 2023-10-01T09:24:11.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581466+0000", 2023-10-01T09:24:11.695 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003328606 2023-10-01T09:24:11.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.695 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.587294+0000", 2023-10-01T09:24:11.695 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058283420000000002 2023-10-01T09:24:11.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.695 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.593571+0000", 2023-10-01T09:24:11.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0062766829999999999 2023-10-01T09:24:11.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.593614+0000", 2023-10-01T09:24:11.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3208999999999999e-05 2023-10-01T09:24:11.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.593649+0000", 2023-10-01T09:24:11.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4761999999999997e-05 2023-10-01T09:24:11.696 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.697 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.697 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.697 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96799 6.1 6:837f0a28:::benchmark_data_smithi138_307751_object1430:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:24:11.697 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.832042+0000", 2023-10-01T09:24:11.697 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 171.71365620099999, 2023-10-01T09:24:11.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.76482651700000004, 2023-10-01T09:24:11.697 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.697 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.697 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.698 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.698 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.698 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96799 2023-10-01T09:24:11.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.698 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832042+0000", 2023-10-01T09:24:11.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832042+0000", 2023-10-01T09:24:11.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832040+0000", 2023-10-01T09:24:11.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:24:11.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832052+0000", 2023-10-01T09:24:11.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1433999999999999e-05 2023-10-01T09:24:11.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832076+0000", 2023-10-01T09:24:11.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4539999999999999e-05 2023-10-01T09:24:11.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832090+0000", 2023-10-01T09:24:11.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3461999999999999e-05 2023-10-01T09:24:11.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832156+0000", 2023-10-01T09:24:11.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6365000000000006e-05 2023-10-01T09:24:11.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832169+0000", 2023-10-01T09:24:11.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2627e-05 2023-10-01T09:24:11.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581483+0000", 2023-10-01T09:24:11.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.74931435400000002 2023-10-01T09:24:11.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581573+0000", 2023-10-01T09:24:11.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9826000000000004e-05 2023-10-01T09:24:11.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:24:11.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581633+0000", 2023-10-01T09:24:11.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0194e-05 2023-10-01T09:24:11.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.590406+0000", 2023-10-01T09:24:11.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0087729320000000006 2023-10-01T09:24:11.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.593681+0000", 2023-10-01T09:24:11.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032753209999999999 2023-10-01T09:24:11.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.596781+0000", 2023-10-01T09:24:11.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003099662 2023-10-01T09:24:11.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.596828+0000", 2023-10-01T09:24:11.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7064999999999997e-05 2023-10-01T09:24:11.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.596869+0000", 2023-10-01T09:24:11.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0785999999999999e-05 2023-10-01T09:24:11.705 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.705 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.705 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.706 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109293 6.4 6:24545a10:::benchmark_data_smithi138_307751_object13924:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-01T09:24:11.706 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.601748+0000", 2023-10-01T09:24:11.706 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.94395031799999, 2023-10-01T09:24:11.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99822042, 2023-10-01T09:24:11.706 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.706 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:24:11.706 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.706 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.706 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.706 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109293 2023-10-01T09:24:11.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.707 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601748+0000", 2023-10-01T09:24:11.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601748+0000", 2023-10-01T09:24:11.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601748+0000", 2023-10-01T09:24:11.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:24:11.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601751+0000", 2023-10-01T09:24:11.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.994e-06 2023-10-01T09:24:11.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601757+0000", 2023-10-01T09:24:11.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0909999999999998e-06 2023-10-01T09:24:11.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601762+0000", 2023-10-01T09:24:11.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4149999999999998e-06 2023-10-01T09:24:11.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.842483+0000", 2023-10-01T09:24:11.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.240721043 2023-10-01T09:24:11.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.842491+0000", 2023-10-01T09:24:11.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2450000000000001e-06 2023-10-01T09:24:11.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.593662+0000", 2023-10-01T09:24:11.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.75117160199999999 2023-10-01T09:24:11.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:24:11.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.593795+0000", 2023-10-01T09:24:11.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000132822 2023-10-01T09:24:11.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.599877+0000", 2023-10-01T09:24:11.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0060821499999999997 2023-10-01T09:24:11.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.599969+0000", 2023-10-01T09:24:11.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1590999999999998e-05 2023-10-01T09:24:11.712 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.712 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.712 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.713 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109329 6.1 6:81350742:::benchmark_data_smithi138_307751_object13960:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-01T09:24:11.713 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.601783+0000", 2023-10-01T09:24:11.713 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.943915441, 2023-10-01T09:24:11.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.000988641, 2023-10-01T09:24:11.713 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.713 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:24:11.713 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.713 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.713 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.713 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109329 2023-10-01T09:24:11.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.714 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601783+0000", 2023-10-01T09:24:11.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601783+0000", 2023-10-01T09:24:11.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601783+0000", 2023-10-01T09:24:11.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:24:11.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601785+0000", 2023-10-01T09:24:11.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3190000000000002e-06 2023-10-01T09:24:11.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601791+0000", 2023-10-01T09:24:11.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5199999999999997e-06 2023-10-01T09:24:11.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601795+0000", 2023-10-01T09:24:11.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.578e-06 2023-10-01T09:24:11.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.797959+0000", 2023-10-01T09:24:11.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19616351200000001 2023-10-01T09:24:11.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.797969+0000", 2023-10-01T09:24:11.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0542e-05 2023-10-01T09:24:11.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.584103+0000", 2023-10-01T09:24:11.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.78613325999999994 2023-10-01T09:24:11.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:24:11.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.584221+0000", 2023-10-01T09:24:11.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118752 2023-10-01T09:24:11.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.602718+0000", 2023-10-01T09:24:11.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018497202000000001 2023-10-01T09:24:11.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.602772+0000", 2023-10-01T09:24:11.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3303000000000001e-05 2023-10-01T09:24:11.719 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.719 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.719 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.720 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109339 6.4 6:27f60416:::benchmark_data_smithi138_307751_object13970:head [delete] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-10-01T09:24:11.720 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.875846+0000", 2023-10-01T09:24:11.720 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.66985232299999, 2023-10-01T09:24:11.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.78316001899999999, 2023-10-01T09:24:11.720 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:11.720 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:11.720 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:11.720 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:11.720 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:11.720 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109339 2023-10-01T09:24:11.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.721 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:11.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:11.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.875846+0000", 2023-10-01T09:24:11.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:11.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.875846+0000", 2023-10-01T09:24:11.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:11.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:11.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.875845+0000", 2023-10-01T09:24:11.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:24:11.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:11.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.875852+0000", 2023-10-01T09:24:11.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9290000000000001e-06 2023-10-01T09:24:11.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:11.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.875949+0000", 2023-10-01T09:24:11.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6877000000000001e-05 2023-10-01T09:24:11.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:11.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.875959+0000", 2023-10-01T09:24:11.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0346000000000001e-05 2023-10-01T09:24:11.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.883849+0000", 2023-10-01T09:24:11.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0078898479999999997 2023-10-01T09:24:11.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:11.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.883865+0000", 2023-10-01T09:24:11.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5650000000000001e-05 2023-10-01T09:24:11.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.599996+0000", 2023-10-01T09:24:11.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.71613177400000005 2023-10-01T09:24:11.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:24:11.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.600836+0000", 2023-10-01T09:24:11.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000839874 2023-10-01T09:24:11.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:11.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.629704+0000", 2023-10-01T09:24:11.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.028867228000000002 2023-10-01T09:24:11.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:11.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.629811+0000", 2023-10-01T09:24:11.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107219 2023-10-01T09:24:11.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:11.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.630140+0000", 2023-10-01T09:24:11.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00032947299999999999 2023-10-01T09:24:11.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.636495+0000", 2023-10-01T09:24:11.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006354274 2023-10-01T09:24:11.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:11.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.636508+0000", 2023-10-01T09:24:11.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3604999999999999e-05 2023-10-01T09:24:11.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:11.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.657107+0000", 2023-10-01T09:24:11.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020599118999999999 2023-10-01T09:24:11.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:11.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.657143+0000", 2023-10-01T09:24:11.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.54e-05 2023-10-01T09:24:11.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:11.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:11.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:11.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.659006+0000", 2023-10-01T09:24:11.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001863647 2023-10-01T09:24:11.729 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.729 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.729 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.729 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:11.729 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:11.729 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:24:11.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:11.730 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:11.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:11.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2357938454' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:11.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2357938454' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:11.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2357938454' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:11.831 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:11.866 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:11.884 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 18 16 33692 33676 116.857 91 0.00584416 0.00855041 2023-10-01T09:24:11.925 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:11.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:11.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:12.027 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:24:12.033 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:12.066 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:12.123 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:12.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:12.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:12.229 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:24:12.258 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:12.885 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 19 16 34783 34767 114.297 68.1875 0.00634494 0.00874371 2023-10-01T09:24:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:12 smithi190 ceph-mon[160435]: pgmap v669: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-01T09:24:13.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:12 smithi138 ceph-mon[185886]: pgmap v669: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-01T09:24:13.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:12 smithi138 ceph-mon[192513]: pgmap v669: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-01T09:24:13.553 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:24:13.646 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:24:13.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:13.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:24:13.783 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:13.815 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:13.876 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:13.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:13.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:13.885 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:24:13.885307+0000 min lat: 0.0011387 max lat: 0.332925 avg lat: 0.00884956 2023-10-01T09:24:13.885 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:24:13.885 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 20 16 36168 36152 112.909 86.5625 0.00652302 0.00884956 2023-10-01T09:24:13.983 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:14.052 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:14.886 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 21 16 37175 37159 110.525 62.9375 0.00996061 0.00904177 2023-10-01T09:24:14.937 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:14 smithi138 ceph-mon[185886]: pgmap v670: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-01T09:24:14.937 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:14 smithi138 ceph-mon[192513]: pgmap v670: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-01T09:24:15.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:14 smithi190 ceph-mon[160435]: pgmap v670: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-01T09:24:15.118 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:15.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:15.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:15.175 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:15.228 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:24:15.310 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:24:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:24:14] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-01T09:24:15.314 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:15.458 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:24:15.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:15.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:24:15.490 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:24:15.567 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:15.700 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:15.873 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:15.886 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 22 16 38509 38493 109.291 83.375 0.00656409 0.00914349 2023-10-01T09:24:16.570 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:16.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:16.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:16.651 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:24:16.715 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:16.751 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:16.807 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:16.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:16.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:16.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:16 smithi138 ceph-mon[185886]: pgmap v671: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-01T09:24:16.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:16 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3627056402' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:24:16.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:16 smithi138 ceph-mon[192513]: pgmap v671: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-01T09:24:16.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:16 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3627056402' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:24:16.886 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 23 16 39681 39665 107.725 73.25 0.0150298 0.00927807 2023-10-01T09:24:16.913 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:16.994 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:17.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:16 smithi190 ceph-mon[160435]: pgmap v671: 89 pgs: 89 active+clean; 2.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-01T09:24:17.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:16 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3627056402' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:24:17.466 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:17.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:17.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:17.573 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:17.601 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:17.661 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:17.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:17.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:17.768 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:24:17.793 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:24:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[192513]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3627056402' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:24:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[192513]: osdmap e264: 8 total, 8 up, 8 in 2023-10-01T09:24:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[192513]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[192513]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[192513]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[192513]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[192513]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:24:17.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[185886]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3627056402' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:24:17.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[185886]: osdmap e264: 8 total, 8 up, 8 in 2023-10-01T09:24:17.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[185886]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[185886]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[185886]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[185886]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:17 smithi138 ceph-mon[185886]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.835 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:24:17.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:17.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:24:17.886 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 24 16 40820 40804 106.203 71.1875 0.00511863 0.00941114 2023-10-01T09:24:17.942 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:17.977 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:17 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:24:17.977 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:17 smithi190 ceph-mon[160435]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.977 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:17 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3627056402' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:24:17.977 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:17 smithi190 ceph-mon[160435]: osdmap e264: 8 total, 8 up, 8 in 2023-10-01T09:24:17.977 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:17 smithi190 ceph-mon[160435]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.977 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:17 smithi190 ceph-mon[160435]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.977 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:17 smithi190 ceph-mon[160435]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.977 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:17 smithi190 ceph-mon[160435]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.977 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:17 smithi190 ceph-mon[160435]: failed to encode map e264 with expected crc 2023-10-01T09:24:17.979 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:18.036 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:18.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:18.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:18.142 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:18.171 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:18.235 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:18.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:18.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:18.345 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:18.421 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:18.468 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:18.573 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:18.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:18.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:18.657 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:18 smithi138 ceph-mon[185886]: pgmap v673: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-01T09:24:18.678 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:24:18.703 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:18.744 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:24:18.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:18.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:24:18.854 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:18.884 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:18.886 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 25 16 41966 41950 104.82 71.625 0.0121385 0.00953528 2023-10-01T09:24:18.920 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:18 smithi138 ceph-mon[192513]: pgmap v673: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-01T09:24:18.939 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:18.942 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:18 smithi190 ceph-mon[160435]: pgmap v673: 89 pgs: 89 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-01T09:24:18.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:18.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:19.044 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:19.070 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:19.120 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:19.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:19.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:19.227 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:24:19.248 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:24:19.263 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:24:19.263 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:19.316 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:24:19.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:19.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:24:19.420 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:19.483 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:19.886 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 26 16 43142 43126 103.616 73.5 0.00889576 0.0096448 2023-10-01T09:24:20.106 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:24:20.450 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:20.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:20.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:20.560 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:24:20.593 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:20.615 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":673,"stamp":"2023-10-01T09:24:17.507408+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2556106675,"num_objects":38958,"num_object_clones":0,"num_object_copies":116874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38958,"num_whiteouts":0,"num_read":6259,"num_read_kb":12659,"num_write":40036,"num_write_kb":2506661,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":186,"num_bytes_recovered":2373496,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":40100,"ondisk_log_size":40100,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9899624,"kb_used_data":7394216,"kb_used_omap":38,"kb_used_meta":2505305,"kb_avail":740061592,"statfs":{"total":767960285184,"available":757823070208,"internally_reserved":0,"allocated":7571677184,"data_stored":7564042574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39221,"internal_metadata":2565433035},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,5,2,8,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":862584832,"num_objects":13162,"num_object_clones":0,"num_object_copies":39486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13162,"num_write_kb":842368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007821"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":134,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:58.738006+0000","last_change":"2023-10-01T09:19:30.091981+0000","last_active":"2023-10-01T09:23:58.738006+0000","last_peered":"2023-10-01T09:23:58.738006+0000","last_clean":"2023-10-01T09:23:58.738006+0000","last_became_active":"2023-10-01T09:19:30.091531+0000","last_became_peered":"2023-10-01T09:19:30.091531+0000","last_unstale":"2023-10-01T09:23:58.738006+0000","last_undegraded":"2023-10-01T09:23:58.738006+0000","last_fullsized":"2023-10-01T09:23:58.738006+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:28:13.442498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":38,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.606191+0000","last_change":"2023-10-01T09:22:40.311894+0000","last_active":"2023-10-01T09:24:05.606191+0000","last_peered":"2023-10-01T09:24:05.606191+0000","last_clean":"2023-10-01T09:24:05.606191+0000","last_became_active":"2023-10-01T09:22:40.311581+0000","last_became_peered":"2023-10-01T09:22:40.311581+0000","last_unstale":"2023-10-01T09:24:05.606191+0000","last_undegraded":"2023-10-01T09:24:05.606191+0000","last_fullsized":"2023-10-01T09:24:05.606191+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:52:27.649535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":210,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.600871+0000","last_change":"2023-10-01T09:13:13.551596+0000","last_active":"2023-10-01T09:24:05.600871+0000","last_peered":"2023-10-01T09:24:05.600871+0000","last_clean":"2023-10-01T09:24:05.600871+0000","last_became_active":"2023-10-01T09:13:13.551300+0000","last_became_peered":"2023-10-01T09:13:13.551300+0000","last_unstale":"2023-10-01T09:24:05.600871+0000","last_undegraded":"2023-10-01T09:24:05.600871+0000","last_fullsized":"2023-10-01T09:24:05.600871+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:57:45.081780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":267,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.616367+0000","last_change":"2023-10-01T09:23:10.611220+0000","last_active":"2023-10-01T09:24:05.616367+0000","last_peered":"2023-10-01T09:24:05.616367+0000","last_clean":"2023-10-01T09:24:05.616367+0000","last_became_active":"2023-10-01T09:23:10.610573+0000","last_became_peered":"2023-10-01T09:23:10.610573+0000","last_unstale":"2023-10-01T09:24:05.616367+0000","last_undegraded":"2023-10-01T09:24:05.616367+0000","last_fullsized":"2023-10-01T09:24:05.616367+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:58:32.020825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":38,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.605062+0000","last_change":"2023-10-01T09:22:40.172183+0000","last_active":"2023-10-01T09:24:05.605062+0000","last_peered":"2023-10-01T09:24:05.605062+0000","last_clean":"2023-10-01T09:24:05.605062+0000","last_became_active":"2023-10-01T09:22:40.170164+0000","last_became_peered":"2023-10-01T09:22:40.170164+0000","last_unstale":"2023-10-01T09:24:05.605062+0000","last_undegraded":"2023-10-01T09:24:05.605062+0000","last_fullsized":"2023-10-01T09:24:05.605062+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:33:10.407999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":209,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.531561+0000","last_change":"2023-10-01T09:22:03.153196+0000","last_active":"2023-10-01T09:23:52.531561+0000","last_peered":"2023-10-01T09:23:52.531561+0000","last_clean":"2023-10-01T09:23:52.531561+0000","last_became_active":"2023-10-01T09:22:03.151768+0000","last_became_peered":"2023-10-01T09:22:03.151768+0000","last_unstale":"2023-10-01T09:23:52.531561+0000","last_undegraded":"2023-10-01T09:23:52.531561+0000","last_fullsized":"2023-10-01T09:23:52.531561+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":269,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.622295+0000","last_change":"2023-10-01T09:22:40.175704+0000","last_active":"2023-10-01T09:24:05.622295+0000","last_peered":"2023-10-01T09:24:05.622295+0000","last_clean":"2023-10-01T09:24:05.622295+0000","last_became_active":"2023-10-01T09:22:40.175369+0000","last_became_peered":"2023-10-01T09:22:40.175369+0000","last_unstale":"2023-10-01T09:24:05.622295+0000","last_undegraded":"2023-10-01T09:24:05.622295+0000","last_fullsized":"2023-10-01T09:24:05.622295+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:07.773407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":254,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.604893+0000","last_change":"2023-10-01T09:21:02.139865+0000","last_active":"2023-10-01T09:24:05.604893+0000","last_peered":"2023-10-01T09:24:05.604893+0000","last_clean":"2023-10-01T09:24:05.604893+0000","last_became_active":"2023-10-01T09:21:02.139507+0000","last_became_peered":"2023-10-01T09:21:02.139507+0000","last_unstale":"2023-10-01T09:24:05.604893+0000","last_undegraded":"2023-10-01T09:24:05.604893+0000","last_fullsized":"2023-10-01T09:24:05.604893+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:46:09.214220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":225,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.727715+0000","last_change":"2023-10-01T09:23:10.610494+0000","last_active":"2023-10-01T09:23:51.727715+0000","last_peered":"2023-10-01T09:23:51.727715+0000","last_clean":"2023-10-01T09:23:51.727715+0000","last_became_active":"2023-10-01T09:23:10.610037+0000","last_became_peered":"2023-10-01T09:23:10.610037+0000","last_unstale":"2023-10-01T09:23:51.727715+0000","last_undegraded":"2023-10-01T09:23:51.727715+0000","last_fullsized":"2023-10-01T09:23:51.727715+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":280,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.605349+0000","last_change":"2023-10-01T09:23:10.613042+0000","last_active":"2023-10-01T09:24:05.605349+0000","last_peered":"2023-10-01T09:24:05.605349+0000","last_clean":"2023-10-01T09:24:05.605349+0000","last_became_active":"2023-10-01T09:23:10.612686+0000","last_became_peered":"2023-10-01T09:23:10.612686+0000","last_unstale":"2023-10-01T09:24:05.605349+0000","last_undegraded":"2023-10-01T09:24:05.605349+0000","last_fullsized":"2023-10-01T09:24:05.605349+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:12:00.239369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":32,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.398074+0000","last_change":"2023-10-01T09:23:10.616550+0000","last_active":"2023-10-01T09:23:52.398074+0000","last_peered":"2023-10-01T09:23:52.398074+0000","last_clean":"2023-10-01T09:23:52.398074+0000","last_became_active":"2023-10-01T09:23:10.611509+0000","last_became_peered":"2023-10-01T09:23:10.611509+0000","last_unstale":"2023-10-01T09:23:52.398074+0000","last_undegraded":"2023-10-01T09:23:52.398074+0000","last_fullsized":"2023-10-01T09:23:52.398074+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":264,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.769989+0000","last_change":"2023-10-01T09:21:02.158958+0000","last_active":"2023-10-01T09:23:51.769989+0000","last_peered":"2023-10-01T09:23:51.769989+0000","last_clean":"2023-10-01T09:23:51.769989+0000","last_became_active":"2023-10-01T09:21:02.158491+0000","last_became_peered":"2023-10-01T09:21:02.158491+0000","last_unstale":"2023-10-01T09:23:51.769989+0000","last_undegraded":"2023-10-01T09:23:51.769989+0000","last_fullsized":"2023-10-01T09:23:51.769989+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:21:48.885982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":208,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.398059+0000","last_change":"2023-10-01T09:21:02.163403+0000","last_active":"2023-10-01T09:23:52.398059+0000","last_peered":"2023-10-01T09:23:52.398059+0000","last_clean":"2023-10-01T09:23:52.398059+0000","last_became_active":"2023-10-01T09:21:02.121680+0000","last_became_peered":"2023-10-01T09:21:02.121680+0000","last_unstale":"2023-10-01T09:23:52.398059+0000","last_undegraded":"2023-10-01T09:23:52.398059+0000","last_fullsized":"2023-10-01T09:23:52.398059+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":231,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.531119+0000","last_change":"2023-10-01T09:21:03.349475+0000","last_active":"2023-10-01T09:23:52.531119+0000","last_peered":"2023-10-01T09:23:52.531119+0000","last_clean":"2023-10-01T09:23:52.531119+0000","last_became_active":"2023-10-01T09:21:03.348251+0000","last_became_peered":"2023-10-01T09:21:03.348251+0000","last_unstale":"2023-10-01T09:23:52.531119+0000","last_undegraded":"2023-10-01T09:23:52.531119+0000","last_fullsized":"2023-10-01T09:23:52.531119+0000","mapping_epoch":208,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":291,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.397926+0000","last_change":"2023-10-01T09:19:30.182046+0000","last_active":"2023-10-01T09:23:52.397926+0000","last_peered":"2023-10-01T09:23:52.397926+0000","last_clean":"2023-10-01T09:23:52.397926+0000","last_became_active":"2023-10-01T09:19:30.094579+0000","last_became_peered":"2023-10-01T09:19:30.094579+0000","last_unstale":"2023-10-01T09:23:52.397926+0000","last_undegraded":"2023-10-01T09:23:52.397926+0000","last_fullsized":"2023-10-01T09:23:52.397926+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":245,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.605232+0000","last_change":"2023-10-01T09:19:57.493670+0000","last_active":"2023-10-01T09:24:05.605232+0000","last_peered":"2023-10-01T09:24:05.605232+0000","last_clean":"2023-10-01T09:24:05.605232+0000","last_became_active":"2023-10-01T09:19:57.493349+0000","last_became_peered":"2023-10-01T09:19:57.493349+0000","last_unstale":"2023-10-01T09:24:05.605232+0000","last_undegraded":"2023-10-01T09:24:05.605232+0000","last_fullsized":"2023-10-01T09:24:05.605232+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:26:56.468779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":251,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.615370+0000","last_change":"2023-10-01T09:23:10.607388+0000","last_active":"2023-10-01T09:24:05.615370+0000","last_peered":"2023-10-01T09:24:05.615370+0000","last_clean":"2023-10-01T09:24:05.615370+0000","last_became_active":"2023-10-01T09:23:10.607084+0000","last_became_peered":"2023-10-01T09:23:10.607084+0000","last_unstale":"2023-10-01T09:24:05.615370+0000","last_undegraded":"2023-10-01T09:24:05.615370+0000","last_fullsized":"2023-10-01T09:24:05.615370+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:19.669393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":201,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.727765+0000","last_change":"2023-10-01T09:22:02.185127+0000","last_active":"2023-10-01T09:23:51.727765+0000","last_peered":"2023-10-01T09:23:51.727765+0000","last_clean":"2023-10-01T09:23:51.727765+0000","last_became_active":"2023-10-01T09:22:02.184059+0000","last_became_peered":"2023-10-01T09:22:02.184059+0000","last_unstale":"2023-10-01T09:23:51.727765+0000","last_undegraded":"2023-10-01T09:23:51.727765+0000","last_fullsized":"2023-10-01T09:23:51.727765+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":232,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.397827+0000","last_change":"2023-10-01T09:21:02.121558+0000","last_active":"2023-10-01T09:23:52.397827+0000","last_peered":"2023-10-01T09:23:52.397827+0000","last_clean":"2023-10-01T09:23:52.397827+0000","last_became_active":"2023-10-01T09:21:02.121153+0000","last_became_peered":"2023-10-01T09:21:02.121153+0000","last_unstale":"2023-10-01T09:23:52.397827+0000","last_undegraded":"2023-10-01T09:23:52.397827+0000","last_fullsized":"2023-10-01T09:23:52.397827+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":233,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.616298+0000","last_change":"2023-10-01T09:23:10.611400+0000","last_active":"2023-10-01T09:24:05.616298+0000","last_peered":"2023-10-01T09:24:05.616298+0000","last_clean":"2023-10-01T09:24:05.616298+0000","last_became_active":"2023-10-01T09:23:10.610928+0000","last_became_peered":"2023-10-01T09:23:10.610928+0000","last_unstale":"2023-10-01T09:24:05.616298+0000","last_undegraded":"2023-10-01T09:24:05.616298+0000","last_fullsized":"2023-10-01T09:24:05.616298+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:17.141540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":209,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.530744+0000","last_change":"2023-10-01T09:22:03.163438+0000","last_active":"2023-10-01T09:23:52.530744+0000","last_peered":"2023-10-01T09:23:52.530744+0000","last_clean":"2023-10-01T09:23:52.530744+0000","last_became_active":"2023-10-01T09:22:03.162821+0000","last_became_peered":"2023-10-01T09:22:03.162821+0000","last_unstale":"2023-10-01T09:23:52.530744+0000","last_undegraded":"2023-10-01T09:23:52.530744+0000","last_fullsized":"2023-10-01T09:23:52.530744+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":47,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.610388+0000","last_change":"2023-10-01T09:23:10.613386+0000","last_active":"2023-10-01T09:24:05.610388+0000","last_peered":"2023-10-01T09:24:05.610388+0000","last_clean":"2023-10-01T09:24:05.610388+0000","last_became_active":"2023-10-01T09:23:10.613046+0000","last_became_peered":"2023-10-01T09:23:10.613046+0000","last_unstale":"2023-10-01T09:24:05.610388+0000","last_undegraded":"2023-10-01T09:24:05.610388+0000","last_fullsized":"2023-10-01T09:24:05.610388+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:23:01.448841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"263'2187","reported_seq":2198,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:13.720511+0000","last_change":"2023-10-01T09:23:45.288346+0000","last_active":"2023-10-01T09:24:13.720511+0000","last_peered":"2023-10-01T09:24:13.720511+0000","last_clean":"2023-10-01T09:24:13.720511+0000","last_became_active":"2023-10-01T09:23:45.288016+0000","last_became_peered":"2023-10-01T09:23:45.288016+0000","last_unstale":"2023-10-01T09:24:13.720511+0000","last_undegraded":"2023-10-01T09:24:13.720511+0000","last_fullsized":"2023-10-01T09:24:13.720511+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2187,"log_dups_size":0,"ondisk_log_size":2187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:40.238301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143327232,"num_objects":2187,"num_object_clones":0,"num_object_copies":6561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2187,"num_write_kb":139968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":30,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.398336+0000","last_change":"2023-10-01T09:23:10.601126+0000","last_active":"2023-10-01T09:23:52.398336+0000","last_peered":"2023-10-01T09:23:52.398336+0000","last_clean":"2023-10-01T09:23:52.398336+0000","last_became_active":"2023-10-01T09:23:10.598657+0000","last_became_peered":"2023-10-01T09:23:10.598657+0000","last_unstale":"2023-10-01T09:23:52.398336+0000","last_undegraded":"2023-10-01T09:23:52.398336+0000","last_fullsized":"2023-10-01T09:23:52.398336+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":198,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.727364+0000","last_change":"2023-10-01T09:23:10.596661+0000","last_active":"2023-10-01T09:23:51.727364+0000","last_peered":"2023-10-01T09:23:51.727364+0000","last_clean":"2023-10-01T09:23:51.727364+0000","last_became_active":"2023-10-01T09:23:10.596299+0000","last_became_peered":"2023-10-01T09:23:10.596299+0000","last_unstale":"2023-10-01T09:23:51.727364+0000","last_undegraded":"2023-10-01T09:23:51.727364+0000","last_fullsized":"2023-10-01T09:23:51.727364+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"264'2421","reported_seq":2435,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-01T09:24:16.971558+0000","last_change":"2023-10-01T09:23:45.286941+0000","last_active":"2023-10-01T09:24:16.971558+0000","last_peered":"2023-10-01T09:24:16.971558+0000","last_clean":"2023-10-01T09:24:16.971558+0000","last_became_active":"2023-10-01T09:23:45.286690+0000","last_became_peered":"2023-10-01T09:23:45.286690+0000","last_unstale":"2023-10-01T09:24:16.971558+0000","last_undegraded":"2023-10-01T09:24:16.971558+0000","last_fullsized":"2023-10-01T09:24:16.971558+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2421,"log_dups_size":0,"ondisk_log_size":2421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:15:15.042008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158662656,"num_objects":2421,"num_object_clones":0,"num_object_copies":7263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2421,"num_write_kb":154944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":231,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.398381+0000","last_change":"2023-10-01T09:12:10.003552+0000","last_active":"2023-10-01T09:23:52.398381+0000","last_peered":"2023-10-01T09:23:52.398381+0000","last_clean":"2023-10-01T09:23:52.398381+0000","last_became_active":"2023-10-01T09:12:10.003131+0000","last_became_peered":"2023-10-01T09:12:10.003131+0000","last_unstale":"2023-10-01T09:23:52.398381+0000","last_undegraded":"2023-10-01T09:23:52.398381+0000","last_fullsized":"2023-10-01T09:23:52.398381+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":264,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.769910+0000","last_change":"2023-10-01T09:22:40.167822+0000","last_active":"2023-10-01T09:23:51.769910+0000","last_peered":"2023-10-01T09:23:51.769910+0000","last_clean":"2023-10-01T09:23:51.769910+0000","last_became_active":"2023-10-01T09:22:40.167502+0000","last_became_peered":"2023-10-01T09:22:40.167502+0000","last_unstale":"2023-10-01T09:23:51.769910+0000","last_undegraded":"2023-10-01T09:23:51.769910+0000","last_fullsized":"2023-10-01T09:23:51.769910+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:40:27.950717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"264'2604","reported_seq":2617,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-01T09:24:17.404739+0000","last_change":"2023-10-01T09:23:45.978822+0000","last_active":"2023-10-01T09:24:17.404739+0000","last_peered":"2023-10-01T09:24:17.404739+0000","last_clean":"2023-10-01T09:24:17.404739+0000","last_became_active":"2023-10-01T09:23:45.978517+0000","last_became_peered":"2023-10-01T09:23:45.978517+0000","last_unstale":"2023-10-01T09:24:17.404739+0000","last_undegraded":"2023-10-01T09:24:17.404739+0000","last_fullsized":"2023-10-01T09:24:17.404739+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2604,"log_dups_size":0,"ondisk_log_size":2604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:52.615311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170655744,"num_objects":2604,"num_object_clones":0,"num_object_copies":7812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2604,"num_write_kb":166656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":255,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.398158+0000","last_change":"2023-10-01T09:22:03.151056+0000","last_active":"2023-10-01T09:23:52.398158+0000","last_peered":"2023-10-01T09:23:52.398158+0000","last_clean":"2023-10-01T09:23:52.398158+0000","last_became_active":"2023-10-01T09:22:03.150643+0000","last_became_peered":"2023-10-01T09:22:03.150643+0000","last_unstale":"2023-10-01T09:23:52.398158+0000","last_undegraded":"2023-10-01T09:23:52.398158+0000","last_fullsized":"2023-10-01T09:23:52.398158+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":224,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.530987+0000","last_change":"2023-10-01T09:23:10.610311+0000","last_active":"2023-10-01T09:23:52.530987+0000","last_peered":"2023-10-01T09:23:52.530987+0000","last_clean":"2023-10-01T09:23:52.530987+0000","last_became_active":"2023-10-01T09:23:10.609935+0000","last_became_peered":"2023-10-01T09:23:10.609935+0000","last_unstale":"2023-10-01T09:23:52.530987+0000","last_undegraded":"2023-10-01T09:23:52.530987+0000","last_fullsized":"2023-10-01T09:23:52.530987+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"263'2296","reported_seq":2309,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:14.919052+0000","last_change":"2023-10-01T09:23:45.977931+0000","last_active":"2023-10-01T09:24:14.919052+0000","last_peered":"2023-10-01T09:24:14.919052+0000","last_clean":"2023-10-01T09:24:14.919052+0000","last_became_active":"2023-10-01T09:23:45.977640+0000","last_became_peered":"2023-10-01T09:23:45.977640+0000","last_unstale":"2023-10-01T09:24:14.919052+0000","last_undegraded":"2023-10-01T09:24:14.919052+0000","last_fullsized":"2023-10-01T09:24:14.919052+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2296,"log_dups_size":0,"ondisk_log_size":2296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:33:08.487287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150470656,"num_objects":2296,"num_object_clones":0,"num_object_copies":6888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2296,"num_write_kb":146944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":214,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.727686+0000","last_change":"2023-10-01T09:22:40.313521+0000","last_active":"2023-10-01T09:23:51.727686+0000","last_peered":"2023-10-01T09:23:51.727686+0000","last_clean":"2023-10-01T09:23:51.727686+0000","last_became_active":"2023-10-01T09:22:40.312941+0000","last_became_peered":"2023-10-01T09:22:40.312941+0000","last_unstale":"2023-10-01T09:23:51.727686+0000","last_undegraded":"2023-10-01T09:23:51.727686+0000","last_fullsized":"2023-10-01T09:23:51.727686+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":215,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.620320+0000","last_change":"2023-10-01T09:12:10.001565+0000","last_active":"2023-10-01T09:24:05.620320+0000","last_peered":"2023-10-01T09:24:05.620320+0000","last_clean":"2023-10-01T09:24:05.620320+0000","last_became_active":"2023-10-01T09:12:10.001275+0000","last_became_peered":"2023-10-01T09:12:10.001275+0000","last_unstale":"2023-10-01T09:24:05.620320+0000","last_undegraded":"2023-10-01T09:24:05.620320+0000","last_fullsized":"2023-10-01T09:24:05.620320+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:39.067046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"264'2485","reported_seq":2499,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-01T09:24:17.390584+0000","last_change":"2023-10-01T09:23:45.302878+0000","last_active":"2023-10-01T09:24:17.390584+0000","last_peered":"2023-10-01T09:24:17.390584+0000","last_clean":"2023-10-01T09:24:17.390584+0000","last_became_active":"2023-10-01T09:23:45.302301+0000","last_became_peered":"2023-10-01T09:23:45.302301+0000","last_unstale":"2023-10-01T09:24:17.390584+0000","last_undegraded":"2023-10-01T09:24:17.390584+0000","last_fullsized":"2023-10-01T09:24:17.390584+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2485,"log_dups_size":0,"ondisk_log_size":2485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:33:01.482516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162856960,"num_objects":2485,"num_object_clones":0,"num_object_copies":7455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2485,"num_write_kb":159040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":234,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.605097+0000","last_change":"2023-10-01T09:13:13.554321+0000","last_active":"2023-10-01T09:24:05.605097+0000","last_peered":"2023-10-01T09:24:05.605097+0000","last_clean":"2023-10-01T09:24:05.605097+0000","last_became_active":"2023-10-01T09:13:13.553986+0000","last_became_peered":"2023-10-01T09:13:13.553986+0000","last_unstale":"2023-10-01T09:24:05.605097+0000","last_undegraded":"2023-10-01T09:24:05.605097+0000","last_fullsized":"2023-10-01T09:24:05.605097+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:09:12.848269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":195,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.605275+0000","last_change":"2023-10-01T09:21:02.139718+0000","last_active":"2023-10-01T09:24:05.605275+0000","last_peered":"2023-10-01T09:24:05.605275+0000","last_clean":"2023-10-01T09:24:05.605275+0000","last_became_active":"2023-10-01T09:21:02.139406+0000","last_became_peered":"2023-10-01T09:21:02.139406+0000","last_unstale":"2023-10-01T09:24:05.605275+0000","last_undegraded":"2023-10-01T09:24:05.605275+0000","last_fullsized":"2023-10-01T09:24:05.605275+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:10.823093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"263'2318","reported_seq":2331,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:14.910190+0000","last_change":"2023-10-01T09:23:45.304043+0000","last_active":"2023-10-01T09:24:14.910190+0000","last_peered":"2023-10-01T09:24:14.910190+0000","last_clean":"2023-10-01T09:24:14.910190+0000","last_became_active":"2023-10-01T09:23:45.303689+0000","last_became_peered":"2023-10-01T09:23:45.303689+0000","last_unstale":"2023-10-01T09:24:14.910190+0000","last_undegraded":"2023-10-01T09:24:14.910190+0000","last_fullsized":"2023-10-01T09:24:14.910190+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2318,"log_dups_size":0,"ondisk_log_size":2318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:27:28.625644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151912448,"num_objects":2318,"num_object_clones":0,"num_object_copies":6954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2318,"num_write_kb":148352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":190,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.770081+0000","last_change":"2023-10-01T09:21:02.136776+0000","last_active":"2023-10-01T09:23:51.770081+0000","last_peered":"2023-10-01T09:23:51.770081+0000","last_clean":"2023-10-01T09:23:51.770081+0000","last_became_active":"2023-10-01T09:21:02.136476+0000","last_became_peered":"2023-10-01T09:21:02.136476+0000","last_unstale":"2023-10-01T09:23:51.770081+0000","last_undegraded":"2023-10-01T09:23:51.770081+0000","last_fullsized":"2023-10-01T09:23:51.770081+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:44:41.289825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":261,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.644663+0000","last_change":"2023-10-01T09:20:15.505658+0000","last_active":"2023-10-01T09:23:52.644663+0000","last_peered":"2023-10-01T09:23:52.644663+0000","last_clean":"2023-10-01T09:23:52.644663+0000","last_became_active":"2023-10-01T09:20:15.505361+0000","last_became_peered":"2023-10-01T09:20:15.505361+0000","last_unstale":"2023-10-01T09:23:52.644663+0000","last_undegraded":"2023-10-01T09:23:52.644663+0000","last_fullsized":"2023-10-01T09:23:52.644663+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:22:16.176432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.d","version":"263'2194","reported_seq":2205,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:13.723907+0000","last_change":"2023-10-01T09:23:45.303471+0000","last_active":"2023-10-01T09:24:13.723907+0000","last_peered":"2023-10-01T09:24:13.723907+0000","last_clean":"2023-10-01T09:24:13.723907+0000","last_became_active":"2023-10-01T09:23:45.303158+0000","last_became_peered":"2023-10-01T09:23:45.303158+0000","last_unstale":"2023-10-01T09:24:13.723907+0000","last_undegraded":"2023-10-01T09:24:13.723907+0000","last_fullsized":"2023-10-01T09:24:13.723907+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2194,"log_dups_size":0,"ondisk_log_size":2194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:13:38.249949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143785984,"num_objects":2194,"num_object_clones":0,"num_object_copies":6582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2194,"num_write_kb":140416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":132,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.605277+0000","last_change":"2023-10-01T09:19:30.092681+0000","last_active":"2023-10-01T09:24:05.605277+0000","last_peered":"2023-10-01T09:24:05.605277+0000","last_clean":"2023-10-01T09:24:05.605277+0000","last_became_active":"2023-10-01T09:19:30.092394+0000","last_became_peered":"2023-10-01T09:19:30.092394+0000","last_unstale":"2023-10-01T09:24:05.605277+0000","last_undegraded":"2023-10-01T09:24:05.605277+0000","last_fullsized":"2023-10-01T09:24:05.605277+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:41:50.127253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":56,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.727780+0000","last_change":"2023-10-01T09:22:03.144199+0000","last_active":"2023-10-01T09:23:51.727780+0000","last_peered":"2023-10-01T09:23:51.727780+0000","last_clean":"2023-10-01T09:23:51.727780+0000","last_became_active":"2023-10-01T09:22:03.143883+0000","last_became_peered":"2023-10-01T09:22:03.143883+0000","last_unstale":"2023-10-01T09:23:51.727780+0000","last_undegraded":"2023-10-01T09:23:51.727780+0000","last_fullsized":"2023-10-01T09:23:51.727780+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.c","version":"264'2445","reported_seq":2458,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-01T09:24:16.969625+0000","last_change":"2023-10-01T09:23:45.288502+0000","last_active":"2023-10-01T09:24:16.969625+0000","last_peered":"2023-10-01T09:24:16.969625+0000","last_clean":"2023-10-01T09:24:16.969625+0000","last_became_active":"2023-10-01T09:23:45.288240+0000","last_became_peered":"2023-10-01T09:23:45.288240+0000","last_unstale":"2023-10-01T09:24:16.969625+0000","last_undegraded":"2023-10-01T09:24:16.969625+0000","last_fullsized":"2023-10-01T09:24:16.969625+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2445,"log_dups_size":0,"ondisk_log_size":2445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:24.693589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160235520,"num_objects":2445,"num_object_clones":0,"num_object_copies":7335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2445,"num_write_kb":156480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":38,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.604802+0000","last_change":"2023-10-01T09:22:40.316008+0000","last_active":"2023-10-01T09:24:05.604802+0000","last_peered":"2023-10-01T09:24:05.604802+0000","last_clean":"2023-10-01T09:24:05.604802+0000","last_became_active":"2023-10-01T09:22:40.315602+0000","last_became_peered":"2023-10-01T09:22:40.315602+0000","last_unstale":"2023-10-01T09:24:05.604802+0000","last_undegraded":"2023-10-01T09:24:05.604802+0000","last_fullsized":"2023-10-01T09:24:05.604802+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:39:37.485241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":128,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.769759+0000","last_change":"2023-10-01T09:18:31.419430+0000","last_active":"2023-10-01T09:23:51.769759+0000","last_peered":"2023-10-01T09:23:51.769759+0000","last_clean":"2023-10-01T09:23:51.769759+0000","last_became_active":"2023-10-01T09:18:31.419189+0000","last_became_peered":"2023-10-01T09:18:31.419189+0000","last_unstale":"2023-10-01T09:23:51.769759+0000","last_undegraded":"2023-10-01T09:23:51.769759+0000","last_fullsized":"2023-10-01T09:23:51.769759+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:08:48.247926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"264'2438","reported_seq":2452,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-01T09:24:16.969744+0000","last_change":"2023-10-01T09:23:45.978783+0000","last_active":"2023-10-01T09:24:16.969744+0000","last_peered":"2023-10-01T09:24:16.969744+0000","last_clean":"2023-10-01T09:24:16.969744+0000","last_became_active":"2023-10-01T09:23:45.978493+0000","last_became_peered":"2023-10-01T09:23:45.978493+0000","last_unstale":"2023-10-01T09:24:16.969744+0000","last_undegraded":"2023-10-01T09:24:16.969744+0000","last_fullsized":"2023-10-01T09:24:16.969744+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2438,"log_dups_size":0,"ondisk_log_size":2438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:46:27.395334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159776768,"num_objects":2438,"num_object_clones":0,"num_object_copies":7314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2438,"num_write_kb":156032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1080,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.770220+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:23:51.770220+0000","last_peered":"2023-10-01T09:23:51.770220+0000","last_clean":"2023-10-01T09:23:51.770220+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:23:51.770220+0000","last_undegraded":"2023-10-01T09:23:51.770220+0000","last_fullsized":"2023-10-01T09:23:51.770220+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":258,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.727639+0000","last_change":"2023-10-01T09:22:40.170877+0000","last_active":"2023-10-01T09:23:51.727639+0000","last_peered":"2023-10-01T09:23:51.727639+0000","last_clean":"2023-10-01T09:23:51.727639+0000","last_became_active":"2023-10-01T09:22:40.170578+0000","last_became_peered":"2023-10-01T09:22:40.170578+0000","last_unstale":"2023-10-01T09:23:51.727639+0000","last_undegraded":"2023-10-01T09:23:51.727639+0000","last_fullsized":"2023-10-01T09:23:51.727639+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":210,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.769987+0000","last_change":"2023-10-01T09:22:02.151482+0000","last_active":"2023-10-01T09:23:51.769987+0000","last_peered":"2023-10-01T09:23:51.769987+0000","last_clean":"2023-10-01T09:23:51.769987+0000","last_became_active":"2023-10-01T09:22:02.151180+0000","last_became_peered":"2023-10-01T09:22:02.151180+0000","last_unstale":"2023-10-01T09:23:51.769987+0000","last_undegraded":"2023-10-01T09:23:51.769987+0000","last_fullsized":"2023-10-01T09:23:51.769987+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:11.745198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"263'2469","reported_seq":2481,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:16.162116+0000","last_change":"2023-10-01T09:23:45.303757+0000","last_active":"2023-10-01T09:24:16.162116+0000","last_peered":"2023-10-01T09:24:16.162116+0000","last_clean":"2023-10-01T09:24:16.162116+0000","last_became_active":"2023-10-01T09:23:45.303439+0000","last_became_peered":"2023-10-01T09:23:45.303439+0000","last_unstale":"2023-10-01T09:24:16.162116+0000","last_undegraded":"2023-10-01T09:24:16.162116+0000","last_fullsized":"2023-10-01T09:24:16.162116+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2469,"log_dups_size":0,"ondisk_log_size":2469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161808384,"num_objects":2469,"num_object_clones":0,"num_object_copies":7407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2469,"num_write_kb":158016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":924,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.727451+0000","last_change":"2023-10-01T09:22:03.258449+0000","last_active":"2023-10-01T09:23:51.727451+0000","last_peered":"2023-10-01T09:23:51.727451+0000","last_clean":"2023-10-01T09:23:51.727451+0000","last_became_active":"2023-10-01T09:22:03.257766+0000","last_became_peered":"2023-10-01T09:22:03.257766+0000","last_unstale":"2023-10-01T09:23:51.727451+0000","last_undegraded":"2023-10-01T09:23:51.727451+0000","last_fullsized":"2023-10-01T09:23:51.727451+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":182,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.620626+0000","last_change":"2023-10-01T09:21:02.121225+0000","last_active":"2023-10-01T09:24:05.620626+0000","last_peered":"2023-10-01T09:24:05.620626+0000","last_clean":"2023-10-01T09:24:05.620626+0000","last_became_active":"2023-10-01T09:21:02.120911+0000","last_became_peered":"2023-10-01T09:21:02.120911+0000","last_unstale":"2023-10-01T09:24:05.620626+0000","last_undegraded":"2023-10-01T09:24:05.620626+0000","last_fullsized":"2023-10-01T09:24:05.620626+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:34:44.385101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":97,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.727541+0000","last_change":"2023-10-01T09:22:40.172426+0000","last_active":"2023-10-01T09:23:51.727541+0000","last_peered":"2023-10-01T09:23:51.727541+0000","last_clean":"2023-10-01T09:23:51.727541+0000","last_became_active":"2023-10-01T09:22:40.172160+0000","last_became_peered":"2023-10-01T09:22:40.172160+0000","last_unstale":"2023-10-01T09:23:51.727541+0000","last_undegraded":"2023-10-01T09:23:51.727541+0000","last_fullsized":"2023-10-01T09:23:51.727541+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"264'2562","reported_seq":2576,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-01T09:24:17.390820+0000","last_change":"2023-10-01T09:23:45.978110+0000","last_active":"2023-10-01T09:24:17.390820+0000","last_peered":"2023-10-01T09:24:17.390820+0000","last_clean":"2023-10-01T09:24:17.390820+0000","last_became_active":"2023-10-01T09:23:45.977724+0000","last_became_peered":"2023-10-01T09:23:45.977724+0000","last_unstale":"2023-10-01T09:24:17.390820+0000","last_undegraded":"2023-10-01T09:24:17.390820+0000","last_fullsized":"2023-10-01T09:24:17.390820+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2562,"log_dups_size":0,"ondisk_log_size":2562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:11:17.874483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167903232,"num_objects":2562,"num_object_clones":0,"num_object_copies":7686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2562,"num_write_kb":163968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":588,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.643873+0000","last_change":"2023-10-01T09:22:40.183431+0000","last_active":"2023-10-01T09:23:52.643873+0000","last_peered":"2023-10-01T09:23:52.643873+0000","last_clean":"2023-10-01T09:23:52.643873+0000","last_became_active":"2023-10-01T09:22:40.171414+0000","last_became_peered":"2023-10-01T09:22:40.171414+0000","last_unstale":"2023-10-01T09:23:52.643873+0000","last_undegraded":"2023-10-01T09:23:52.643873+0000","last_fullsized":"2023-10-01T09:23:52.643873+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:56:42.904846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":265,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.600604+0000","last_change":"2023-10-01T09:21:02.140363+0000","last_active":"2023-10-01T09:24:05.600604+0000","last_peered":"2023-10-01T09:24:05.600604+0000","last_clean":"2023-10-01T09:24:05.600604+0000","last_became_active":"2023-10-01T09:21:02.137198+0000","last_became_peered":"2023-10-01T09:21:02.137198+0000","last_unstale":"2023-10-01T09:24:05.600604+0000","last_undegraded":"2023-10-01T09:24:05.600604+0000","last_fullsized":"2023-10-01T09:24:05.600604+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:38:38.043146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":125,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.770412+0000","last_change":"2023-10-01T09:22:02.156724+0000","last_active":"2023-10-01T09:23:51.770412+0000","last_peered":"2023-10-01T09:23:51.770412+0000","last_clean":"2023-10-01T09:23:51.770412+0000","last_became_active":"2023-10-01T09:22:02.156451+0000","last_became_peered":"2023-10-01T09:22:02.156451+0000","last_unstale":"2023-10-01T09:23:51.770412+0000","last_undegraded":"2023-10-01T09:23:51.770412+0000","last_fullsized":"2023-10-01T09:23:51.770412+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:29:05.232127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"264'2538","reported_seq":2552,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-01T09:24:17.400231+0000","last_change":"2023-10-01T09:23:45.302734+0000","last_active":"2023-10-01T09:24:17.400231+0000","last_peered":"2023-10-01T09:24:17.400231+0000","last_clean":"2023-10-01T09:24:17.400231+0000","last_became_active":"2023-10-01T09:23:45.302228+0000","last_became_peered":"2023-10-01T09:23:45.302228+0000","last_unstale":"2023-10-01T09:24:17.400231+0000","last_undegraded":"2023-10-01T09:24:17.400231+0000","last_fullsized":"2023-10-01T09:24:17.400231+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2538,"log_dups_size":0,"ondisk_log_size":2538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:35.109203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166330368,"num_objects":2538,"num_object_clones":0,"num_object_copies":7614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2538,"num_write_kb":162432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1321,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.727577+0000","last_change":"2023-10-01T09:23:13.197266+0000","last_active":"2023-10-01T09:23:51.727577+0000","last_peered":"2023-10-01T09:23:51.727577+0000","last_clean":"2023-10-01T09:23:51.727577+0000","last_became_active":"2023-10-01T09:23:13.196795+0000","last_became_peered":"2023-10-01T09:23:13.196795+0000","last_unstale":"2023-10-01T09:23:51.727577+0000","last_undegraded":"2023-10-01T09:23:51.727577+0000","last_fullsized":"2023-10-01T09:23:51.727577+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":426,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.397826+0000","last_change":"2023-10-01T09:23:32.139254+0000","last_active":"2023-10-01T09:23:52.397826+0000","last_peered":"2023-10-01T09:23:52.397826+0000","last_clean":"2023-10-01T09:23:52.397826+0000","last_became_active":"2023-10-01T09:23:32.138752+0000","last_became_peered":"2023-10-01T09:23:32.138752+0000","last_unstale":"2023-10-01T09:23:52.397826+0000","last_undegraded":"2023-10-01T09:23:52.397826+0000","last_fullsized":"2023-10-01T09:23:52.397826+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":38,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.604919+0000","last_change":"2023-10-01T09:22:40.317678+0000","last_active":"2023-10-01T09:24:05.604919+0000","last_peered":"2023-10-01T09:24:05.604919+0000","last_clean":"2023-10-01T09:24:05.604919+0000","last_became_active":"2023-10-01T09:22:40.317460+0000","last_became_peered":"2023-10-01T09:22:40.317460+0000","last_unstale":"2023-10-01T09:24:05.604919+0000","last_undegraded":"2023-10-01T09:24:05.604919+0000","last_fullsized":"2023-10-01T09:24:05.604919+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:07:41.681679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":135,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.398278+0000","last_change":"2023-10-01T09:21:02.125167+0000","last_active":"2023-10-01T09:23:52.398278+0000","last_peered":"2023-10-01T09:23:52.398278+0000","last_clean":"2023-10-01T09:23:52.398278+0000","last_became_active":"2023-10-01T09:21:02.124791+0000","last_became_peered":"2023-10-01T09:21:02.124791+0000","last_unstale":"2023-10-01T09:23:52.398278+0000","last_undegraded":"2023-10-01T09:23:52.398278+0000","last_fullsized":"2023-10-01T09:23:52.398278+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.5","version":"263'2541","reported_seq":2553,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:16.418686+0000","last_change":"2023-10-01T09:23:45.303660+0000","last_active":"2023-10-01T09:24:16.418686+0000","last_peered":"2023-10-01T09:24:16.418686+0000","last_clean":"2023-10-01T09:24:16.418686+0000","last_became_active":"2023-10-01T09:23:45.303267+0000","last_became_peered":"2023-10-01T09:23:45.303267+0000","last_unstale":"2023-10-01T09:24:16.418686+0000","last_undegraded":"2023-10-01T09:24:16.418686+0000","last_fullsized":"2023-10-01T09:24:16.418686+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2541,"log_dups_size":0,"ondisk_log_size":2541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166526976,"num_objects":2541,"num_object_clones":0,"num_object_copies":7623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2541,"num_write_kb":162624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":359,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.398130+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:23:52.398130+0000","last_peered":"2023-10-01T09:23:52.398130+0000","last_clean":"2023-10-01T09:23:52.398130+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:23:52.398130+0000","last_undegraded":"2023-10-01T09:23:52.398130+0000","last_fullsized":"2023-10-01T09:23:52.398130+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":249,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.618414+0000","last_change":"2023-10-01T09:22:40.171122+0000","last_active":"2023-10-01T09:24:05.618414+0000","last_peered":"2023-10-01T09:24:05.618414+0000","last_clean":"2023-10-01T09:24:05.618414+0000","last_became_active":"2023-10-01T09:22:40.170773+0000","last_became_peered":"2023-10-01T09:22:40.170773+0000","last_unstale":"2023-10-01T09:24:05.618414+0000","last_undegraded":"2023-10-01T09:24:05.618414+0000","last_fullsized":"2023-10-01T09:24:05.618414+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:50:30.349100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":267,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.616584+0000","last_change":"2023-10-01T09:19:30.091481+0000","last_active":"2023-10-01T09:24:05.616584+0000","last_peered":"2023-10-01T09:24:05.616584+0000","last_clean":"2023-10-01T09:24:05.616584+0000","last_became_active":"2023-10-01T09:19:30.091151+0000","last_became_peered":"2023-10-01T09:19:30.091151+0000","last_unstale":"2023-10-01T09:24:05.616584+0000","last_undegraded":"2023-10-01T09:24:05.616584+0000","last_fullsized":"2023-10-01T09:24:05.616584+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:09.127408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"263'2385","reported_seq":2397,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:15.404639+0000","last_change":"2023-10-01T09:23:45.287796+0000","last_active":"2023-10-01T09:24:15.404639+0000","last_peered":"2023-10-01T09:24:15.404639+0000","last_clean":"2023-10-01T09:24:15.404639+0000","last_became_active":"2023-10-01T09:23:45.287413+0000","last_became_peered":"2023-10-01T09:23:45.287413+0000","last_unstale":"2023-10-01T09:24:15.404639+0000","last_undegraded":"2023-10-01T09:24:15.404639+0000","last_fullsized":"2023-10-01T09:24:15.404639+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2385,"log_dups_size":0,"ondisk_log_size":2385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156303360,"num_objects":2385,"num_object_clones":0,"num_object_copies":7155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2385,"num_write_kb":152640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":623,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.398218+0000","last_change":"2023-10-01T09:10:58.797964+0000","last_active":"2023-10-01T09:23:52.398218+0000","last_peered":"2023-10-01T09:23:52.398218+0000","last_clean":"2023-10-01T09:23:52.398218+0000","last_became_active":"2023-10-01T09:10:58.797447+0000","last_became_peered":"2023-10-01T09:10:58.797447+0000","last_unstale":"2023-10-01T09:23:52.398218+0000","last_undegraded":"2023-10-01T09:23:52.398218+0000","last_fullsized":"2023-10-01T09:23:52.398218+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":261,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.770277+0000","last_change":"2023-10-01T09:22:02.160793+0000","last_active":"2023-10-01T09:23:51.770277+0000","last_peered":"2023-10-01T09:23:51.770277+0000","last_clean":"2023-10-01T09:23:51.770277+0000","last_became_active":"2023-10-01T09:22:02.160579+0000","last_became_peered":"2023-10-01T09:22:02.160579+0000","last_unstale":"2023-10-01T09:23:51.770277+0000","last_undegraded":"2023-10-01T09:23:51.770277+0000","last_fullsized":"2023-10-01T09:23:51.770277+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:34:29.335908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":216,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.620552+0000","last_change":"2023-10-01T09:22:02.151402+0000","last_active":"2023-10-01T09:24:05.620552+0000","last_peered":"2023-10-01T09:24:05.620552+0000","last_clean":"2023-10-01T09:24:05.620552+0000","last_became_active":"2023-10-01T09:22:02.151028+0000","last_became_peered":"2023-10-01T09:22:02.151028+0000","last_unstale":"2023-10-01T09:24:05.620552+0000","last_undegraded":"2023-10-01T09:24:05.620552+0000","last_fullsized":"2023-10-01T09:24:05.620552+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:19.623044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"264'2456","reported_seq":2470,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-01T09:24:16.982671+0000","last_change":"2023-10-01T09:23:45.978785+0000","last_active":"2023-10-01T09:24:16.982671+0000","last_peered":"2023-10-01T09:24:16.982671+0000","last_clean":"2023-10-01T09:24:16.982671+0000","last_became_active":"2023-10-01T09:23:45.978512+0000","last_became_peered":"2023-10-01T09:23:45.978512+0000","last_unstale":"2023-10-01T09:24:16.982671+0000","last_undegraded":"2023-10-01T09:24:16.982671+0000","last_fullsized":"2023-10-01T09:24:16.982671+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2456,"log_dups_size":0,"ondisk_log_size":2456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:25:02.565270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160956416,"num_objects":2456,"num_object_clones":0,"num_object_copies":7368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2456,"num_write_kb":157184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":276,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.610540+0000","last_change":"2023-10-01T09:23:10.615956+0000","last_active":"2023-10-01T09:24:05.610540+0000","last_peered":"2023-10-01T09:24:05.610540+0000","last_clean":"2023-10-01T09:24:05.610540+0000","last_became_active":"2023-10-01T09:23:10.610919+0000","last_became_peered":"2023-10-01T09:23:10.610919+0000","last_unstale":"2023-10-01T09:24:05.610540+0000","last_undegraded":"2023-10-01T09:24:05.610540+0000","last_fullsized":"2023-10-01T09:24:05.610540+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:03:44.294373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":736,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.770332+0000","last_change":"2023-10-01T09:21:20.545328+0000","last_active":"2023-10-01T09:23:51.770332+0000","last_peered":"2023-10-01T09:23:51.770332+0000","last_clean":"2023-10-01T09:23:51.770332+0000","last_became_active":"2023-10-01T09:21:20.544828+0000","last_became_peered":"2023-10-01T09:21:20.544828+0000","last_unstale":"2023-10-01T09:23:51.770332+0000","last_undegraded":"2023-10-01T09:23:51.770332+0000","last_fullsized":"2023-10-01T09:23:51.770332+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:23:52.633471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":47,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.604975+0000","last_change":"2023-10-01T09:23:10.612309+0000","last_active":"2023-10-01T09:24:05.604975+0000","last_peered":"2023-10-01T09:24:05.604975+0000","last_clean":"2023-10-01T09:24:05.604975+0000","last_became_active":"2023-10-01T09:23:10.612034+0000","last_became_peered":"2023-10-01T09:23:10.612034+0000","last_unstale":"2023-10-01T09:24:05.604975+0000","last_undegraded":"2023-10-01T09:24:05.604975+0000","last_fullsized":"2023-10-01T09:24:05.604975+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:09:47.963186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"264'2591","reported_seq":2604,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-10-01T09:24:16.984890+0000","last_change":"2023-10-01T09:23:45.283913+0000","last_active":"2023-10-01T09:24:16.984890+0000","last_peered":"2023-10-01T09:24:16.984890+0000","last_clean":"2023-10-01T09:24:16.984890+0000","last_became_active":"2023-10-01T09:23:45.283600+0000","last_became_peered":"2023-10-01T09:23:45.283600+0000","last_unstale":"2023-10-01T09:24:16.984890+0000","last_undegraded":"2023-10-01T09:24:16.984890+0000","last_fullsized":"2023-10-01T09:24:16.984890+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":2591,"log_dups_size":0,"ondisk_log_size":2591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:24:30.501930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169803776,"num_objects":2591,"num_object_clones":0,"num_object_copies":7773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2591,"num_write_kb":165824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":94,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:52.531473+0000","last_change":"2023-10-01T09:23:35.503601+0000","last_active":"2023-10-01T09:23:52.531473+0000","last_peered":"2023-10-01T09:23:52.531473+0000","last_clean":"2023-10-01T09:23:52.531473+0000","last_became_active":"2023-10-01T09:23:10.613270+0000","last_became_peered":"2023-10-01T09:23:10.613270+0000","last_unstale":"2023-10-01T09:23:52.531473+0000","last_undegraded":"2023-10-01T09:23:52.531473+0000","last_fullsized":"2023-10-01T09:23:52.531473+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":257,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":246,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.616449+0000","last_change":"2023-10-01T09:19:05.565472+0000","last_active":"2023-10-01T09:24:05.616449+0000","last_peered":"2023-10-01T09:24:05.616449+0000","last_clean":"2023-10-01T09:24:05.616449+0000","last_became_active":"2023-10-01T09:19:05.561161+0000","last_became_peered":"2023-10-01T09:19:05.561161+0000","last_unstale":"2023-10-01T09:24:05.616449+0000","last_undegraded":"2023-10-01T09:24:05.616449+0000","last_fullsized":"2023-10-01T09:24:05.616449+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:50:29.273388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":58,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.727393+0000","last_change":"2023-10-01T09:22:03.144097+0000","last_active":"2023-10-01T09:23:51.727393+0000","last_peered":"2023-10-01T09:23:51.727393+0000","last_clean":"2023-10-01T09:23:51.727393+0000","last_became_active":"2023-10-01T09:22:03.143800+0000","last_became_peered":"2023-10-01T09:22:03.143800+0000","last_unstale":"2023-10-01T09:23:51.727393+0000","last_undegraded":"2023-10-01T09:23:51.727393+0000","last_fullsized":"2023-10-01T09:23:51.727393+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":204,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.620474+0000","last_change":"2023-10-01T09:10:58.801161+0000","last_active":"2023-10-01T09:24:05.620474+0000","last_peered":"2023-10-01T09:24:05.620474+0000","last_clean":"2023-10-01T09:24:05.620474+0000","last_became_active":"2023-10-01T09:10:58.800794+0000","last_became_peered":"2023-10-01T09:10:58.800794+0000","last_unstale":"2023-10-01T09:24:05.620474+0000","last_undegraded":"2023-10-01T09:24:05.620474+0000","last_fullsized":"2023-10-01T09:24:05.620474+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:35.019445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":290,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:53.737294+0000","last_change":"2023-10-01T09:22:40.166927+0000","last_active":"2023-10-01T09:23:53.737294+0000","last_peered":"2023-10-01T09:23:53.737294+0000","last_clean":"2023-10-01T09:23:53.737294+0000","last_became_active":"2023-10-01T09:22:40.166606+0000","last_became_peered":"2023-10-01T09:22:40.166606+0000","last_unstale":"2023-10-01T09:23:53.737294+0000","last_undegraded":"2023-10-01T09:23:53.737294+0000","last_fullsized":"2023-10-01T09:23:53.737294+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:54:18.044207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":194,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.727619+0000","last_change":"2023-10-01T09:22:02.180112+0000","last_active":"2023-10-01T09:23:51.727619+0000","last_peered":"2023-10-01T09:23:51.727619+0000","last_clean":"2023-10-01T09:23:51.727619+0000","last_became_active":"2023-10-01T09:22:02.161271+0000","last_became_peered":"2023-10-01T09:22:02.161271+0000","last_unstale":"2023-10-01T09:23:51.727619+0000","last_undegraded":"2023-10-01T09:23:51.727619+0000","last_fullsized":"2023-10-01T09:23:51.727619+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":210,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.605013+0000","last_change":"2023-10-01T09:19:05.553200+0000","last_active":"2023-10-01T09:24:05.605013+0000","last_peered":"2023-10-01T09:24:05.605013+0000","last_clean":"2023-10-01T09:24:05.605013+0000","last_became_active":"2023-10-01T09:19:05.552874+0000","last_became_peered":"2023-10-01T09:19:05.552874+0000","last_unstale":"2023-10-01T09:24:05.605013+0000","last_undegraded":"2023-10-01T09:24:05.605013+0000","last_fullsized":"2023-10-01T09:24:05.605013+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:14:31.040887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":183,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.727732+0000","last_change":"2023-10-01T09:22:03.149673+0000","last_active":"2023-10-01T09:23:51.727732+0000","last_peered":"2023-10-01T09:23:51.727732+0000","last_clean":"2023-10-01T09:23:51.727732+0000","last_became_active":"2023-10-01T09:22:03.149236+0000","last_became_peered":"2023-10-01T09:22:03.149236+0000","last_unstale":"2023-10-01T09:23:51.727732+0000","last_undegraded":"2023-10-01T09:23:51.727732+0000","last_fullsized":"2023-10-01T09:23:51.727732+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":192,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.604926+0000","last_change":"2023-10-01T09:18:54.533869+0000","last_active":"2023-10-01T09:24:05.604926+0000","last_peered":"2023-10-01T09:24:05.604926+0000","last_clean":"2023-10-01T09:24:05.604926+0000","last_became_active":"2023-10-01T09:18:54.533542+0000","last_became_peered":"2023-10-01T09:18:54.533542+0000","last_unstale":"2023-10-01T09:24:05.604926+0000","last_undegraded":"2023-10-01T09:24:05.604926+0000","last_fullsized":"2023-10-01T09:24:05.604926+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:18:19.126440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":38,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.607912+0000","last_change":"2023-10-01T09:22:40.313999+0000","last_active":"2023-10-01T09:24:05.607912+0000","last_peered":"2023-10-01T09:24:05.607912+0000","last_clean":"2023-10-01T09:24:05.607912+0000","last_became_active":"2023-10-01T09:22:40.313605+0000","last_became_peered":"2023-10-01T09:22:40.313605+0000","last_unstale":"2023-10-01T09:24:05.607912+0000","last_undegraded":"2023-10-01T09:24:05.607912+0000","last_fullsized":"2023-10-01T09:24:05.607912+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:52:32.154690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":38,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.605181+0000","last_change":"2023-10-01T09:22:40.170676+0000","last_active":"2023-10-01T09:24:05.605181+0000","last_peered":"2023-10-01T09:24:05.605181+0000","last_clean":"2023-10-01T09:24:05.605181+0000","last_became_active":"2023-10-01T09:22:40.170260+0000","last_became_peered":"2023-10-01T09:22:40.170260+0000","last_unstale":"2023-10-01T09:24:05.605181+0000","last_undegraded":"2023-10-01T09:24:05.605181+0000","last_fullsized":"2023-10-01T09:24:05.605181+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:14.693287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":295,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-10-01T09:24:05.622371+0000","last_change":"2023-10-01T09:22:02.157406+0000","last_active":"2023-10-01T09:24:05.622371+0000","last_peered":"2023-10-01T09:24:05.622371+0000","last_clean":"2023-10-01T09:24:05.622371+0000","last_became_active":"2023-10-01T09:22:02.157041+0000","last_became_peered":"2023-10-01T09:22:02.157041+0000","last_unstale":"2023-10-01T09:24:05.622371+0000","last_undegraded":"2023-10-01T09:24:05.622371+0000","last_fullsized":"2023-10-01T09:24:05.622371+0000","mapping_epoch":227,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:24:52.008578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":185,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-10-01T09:23:51.769832+0000","last_change":"2023-10-01T09:21:02.141180+0000","last_active":"2023-10-01T09:23:51.769832+0000","last_peered":"2023-10-01T09:23:51.769832+0000","last_clean":"2023-10-01T09:23:51.769832+0000","last_became_active":"2023-10-01T09:21:02.136796+0000","last_became_peered":"2023-10-01T09:21:02.136796+0000","last_unstale":"2023-10-01T09:23:51.769832+0000","last_undegraded":"2023-10-01T09:23:51.769832+0000","last_fullsized":"2023-10-01T09:23:51.769832+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:03:46.040197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2551316480,"num_objects":38930,"num_object_clones":0,"num_object_copies":116790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":38930,"num_write_kb":2491520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7622819840,"data_stored":7622819840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38930,"ondisk_log_size":38930,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":12024,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35793,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430484,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1071080,"kb_used_data":769760,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92674072,"statfs":{"total":95995035648,"available":94898249728,"internally_reserved":0,"allocated":788234240,"data_stored":787512615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:15 2023","interfaces":[{"interface":"back","average":{"1min":2.573,"5min":5.2359999999999998,"15min":3.234},"min":{"1min":0.28100000000000003,"5min":0.23400000000000001,"15min":0.219},"max":{"1min":7.5190000000000001,"5min":117.855,"15min":117.855},"last":4.2009999999999996},{"interface":"front","average":{"1min":1.891,"5min":5.2889999999999997,"15min":3.0249999999999999},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":6.1159999999999997,"5min":120.348,"15min":120.348},"last":1.7230000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.8600000000000001,"5min":5.3099999999999996,"15min":2.794},"min":{"1min":0.251,"5min":0.251,"15min":0.22700000000000001},"max":{"1min":7.4279999999999999,"5min":135.023,"15min":135.023},"last":0.24199999999999999},{"interface":"front","average":{"1min":2.7719999999999998,"5min":5.2800000000000002,"15min":2.6040000000000001},"min":{"1min":0.29499999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":10.034000000000001,"5min":134.51499999999999,"15min":134.51499999999999},"last":2.9249999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":3.9249999999999998,"15min":2.5350000000000001},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.4539999999999997,"5min":104.837,"15min":104.837},"last":3.7170000000000001},{"interface":"front","average":{"1min":1.681,"5min":3.9950000000000001,"15min":2.508},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":9.9789999999999992,"5min":106.379,"15min":106.379},"last":1.9590000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1739999999999999,"5min":3.4870000000000001,"15min":1.8660000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.26400000000000001},"max":{"1min":10.1,"5min":104.797,"15min":104.797},"last":1.0029999999999999},{"interface":"front","average":{"1min":2.8839999999999999,"5min":3.0760000000000001,"15min":1.8260000000000001},"min":{"1min":0.34100000000000003,"5min":0.29399999999999998,"15min":0.27200000000000002},"max":{"1min":7.2039999999999997,"5min":104.774,"15min":104.774},"last":0.997}]},{"osd":4,"last update":"Sun Oct 1 09:23:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5870000000000002,"5min":1.726,"15min":1.546},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.23699999999999999},"max":{"1min":6.5510000000000002,"5min":16.062999999999999,"15min":82.992000000000004},"last":0.28299999999999997},{"interface":"front","average":{"1min":2.1859999999999999,"5min":1.7629999999999999,"15min":1.579},"min":{"1min":0.39700000000000002,"5min":0.29499999999999998,"15min":0.252},"max":{"1min":6.2089999999999996,"5min":24.161999999999999,"15min":83.058999999999997},"last":0.30299999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:24:11 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":2.6539999999999999,"15min":2.9740000000000002},"min":{"1min":0.33500000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.0810000000000004,"5min":13.265000000000001,"15min":13.265000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.161,"5min":2.5470000000000002,"15min":2.778},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":8.2889999999999997,"5min":13.292999999999999,"15min":13.292999999999999},"last":0.38500000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:23:15 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":1.4510000000000001,"15min":1.2070000000000001},"min":{"1min":0.35899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.5330000000000004,"5min":11.685,"15min":11.685},"last":0.41199999999999998},{"interface":"front","average":{"1min":2.0750000000000002,"5min":1.6619999999999999,"15min":1.1910000000000001},"min":{"1min":0.38,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.4220000000000006,"5min":11.871,"15min":11.871},"last":0.33100000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626712,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1055896,"kb_used_data":754576,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92689256,"statfs":{"total":95995035648,"available":94913798144,"internally_reserved":0,"allocated":772685824,"data_stored":771979826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:24:12 2023","interfaces":[{"interface":"back","average":{"1min":2.4380000000000002,"5min":2.008,"15min":4.0170000000000003},"min":{"1min":0.27100000000000002,"5min":0.20699999999999999,"15min":0.20000000000000001},"max":{"1min":28.591000000000001,"5min":52.347000000000001,"15min":178.738},"last":2.8170000000000002},{"interface":"front","average":{"1min":2.0920000000000001,"5min":2.0960000000000001,"15min":4.258},"min":{"1min":0.254,"5min":0.223,"15min":0.223},"max":{"1min":26.748999999999999,"5min":52.375999999999998,"15min":178.684},"last":1.6819999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:23:30 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":4.6360000000000001,"15min":2.2879999999999998},"min":{"1min":0.253,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.1550000000000002,"5min":115.523,"15min":115.523},"last":0.56599999999999995},{"interface":"front","average":{"1min":2.1699999999999999,"5min":5.0949999999999998,"15min":2.7240000000000002},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":11.042999999999999,"5min":115.681,"15min":115.681},"last":0.57199999999999995}]},{"osd":2,"last update":"Sun Oct 1 09:24:01 2023","interfaces":[{"interface":"back","average":{"1min":7.0149999999999997,"5min":3.1800000000000002,"15min":2.9430000000000001},"min":{"1min":0.33700000000000002,"5min":0.26500000000000001,"15min":0.25900000000000001},"max":{"1min":122.09399999999999,"5min":122.09399999999999,"15min":122.09399999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":7.3209999999999997,"5min":2.7599999999999998,"15min":2.5800000000000001},"min":{"1min":0.32600000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":121.262,"5min":121.262,"15min":121.262},"last":0.66700000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:23:30 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":4.8799999999999999,"15min":2.403},"min":{"1min":0.307,"5min":0.307,"15min":0.27000000000000002},"max":{"1min":7.2050000000000001,"5min":219.32400000000001,"15min":219.32400000000001},"last":3.222},{"interface":"front","average":{"1min":1.7769999999999999,"5min":5.1139999999999999,"15min":2.7069999999999999},"min":{"1min":0.37,"5min":0.33600000000000002,"15min":0.30499999999999999},"max":{"1min":8.6989999999999998,"5min":219.77099999999999,"15min":219.77099999999999},"last":5.1909999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:24:01 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":1.246,"15min":1.1459999999999999},"min":{"1min":0.28299999999999997,"5min":0.27200000000000002,"15min":0.23699999999999999},"max":{"1min":5.8479999999999999,"5min":14.089,"15min":28.271999999999998},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":1.22,"15min":1.171},"min":{"1min":0.308,"5min":0.29099999999999998,"15min":0.26000000000000001},"max":{"1min":4.9039999999999999,"5min":9.907,"15min":28.413},"last":0.37}]},{"osd":5,"last update":"Sun Oct 1 09:24:06 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":2.5209999999999999,"15min":2.8109999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.0570000000000004,"5min":9.0760000000000005,"15min":9.0760000000000005},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.92400000000000004,"5min":1.8959999999999999,"15min":2.0579999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.3449999999999998,"5min":13.999000000000001,"15min":13.999000000000001},"last":0.439}]},{"osd":7,"last update":"Sun Oct 1 09:24:12 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.288,"15min":1.7170000000000001},"min":{"1min":0.33400000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.2029999999999998,"5min":12.487,"15min":145.37299999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.63,"5min":1.3180000000000001,"15min":1.661},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.30499999999999999},"max":{"1min":1.4950000000000001,"5min":9.1010000000000009,"15min":145.28200000000001},"last":0.45000000000000001}]}]},{"osd":1,"up_from":94,"seq":403726925971,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1093628,"kb_used_data":843828,"kb_used_omap":7,"kb_used_meta":249784,"kb_avail":92651524,"statfs":{"total":95995035648,"available":94875160576,"internally_reserved":0,"allocated":864079872,"data_stored":862878379,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":255779239},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":6.3200000000000003,"15min":2.891},"min":{"1min":0.19700000000000001,"5min":0.19400000000000001,"15min":0.17499999999999999},"max":{"1min":7.2770000000000001,"5min":174.40199999999999,"15min":174.40199999999999},"last":0.26400000000000001},{"interface":"front","average":{"1min":2.952,"5min":6.4820000000000002,"15min":3.0430000000000001},"min":{"1min":0.219,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":7.702,"5min":173.77000000000001,"15min":173.77000000000001},"last":1.1060000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:24:04 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":2.9089999999999998,"15min":1.734},"min":{"1min":0.26700000000000002,"5min":0.182,"15min":0.182},"max":{"1min":8.327,"5min":77.325000000000003,"15min":77.325000000000003},"last":0.82099999999999995},{"interface":"front","average":{"1min":1.905,"5min":3.331,"15min":1.9299999999999999},"min":{"1min":0.25600000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.994,"5min":80.361000000000004,"15min":80.361000000000004},"last":0.94799999999999995}]},{"osd":3,"last update":"Sun Oct 1 09:23:38 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":4.0899999999999999,"15min":1.992},"min":{"1min":0.224,"5min":0.224,"15min":0.218},"max":{"1min":6.593,"5min":99.421000000000006,"15min":99.421000000000006},"last":1.6679999999999999},{"interface":"front","average":{"1min":1.526,"5min":3.8860000000000001,"15min":1.853},"min":{"1min":0.30599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.4409999999999998,"5min":99.364999999999995,"15min":99.364999999999995},"last":0.45800000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":1.4079999999999999,"15min":1.2849999999999999},"min":{"1min":0.28799999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.8259999999999996,"5min":9.9169999999999998,"15min":40.673999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.7150000000000001,"5min":1.514,"15min":1.417},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.25800000000000001},"max":{"1min":7.2249999999999996,"5min":9.4610000000000003,"15min":40.810000000000002},"last":0.49399999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:24:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.3420000000000001,"15min":1.3819999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":6.4450000000000003,"5min":6.4450000000000003,"15min":6.4450000000000003},"last":0.443},{"interface":"front","average":{"1min":1.2729999999999999,"5min":2.2040000000000002,"15min":2.359},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.3109999999999999,"5min":7.3109999999999999,"15min":7.3109999999999999},"last":0.57899999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8260000000000001,"5min":1.5980000000000001,"15min":1.3300000000000001},"min":{"1min":0.35099999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.9860000000000007,"5min":14.452,"15min":14.452},"last":0.91500000000000004},{"interface":"front","average":{"1min":2.29,"5min":1.605,"15min":1.226},"min":{"1min":0.35799999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.9740000000000002,"5min":9.9740000000000002,"15min":10.787000000000001},"last":0.52900000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":1.96,"15min":1.431},"min":{"1min":0.42099999999999999,"5min":0.317,"15min":0.317},"max":{"1min":7.8129999999999997,"5min":23.710999999999999,"15min":23.710999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.853,"5min":2.0049999999999999,"15min":1.4450000000000001},"min":{"1min":0.39500000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.3579999999999997,"5min":23.736999999999998,"15min":23.736999999999998},"last":0.60999999999999999}]}]},{"osd":0,"up_from":89,"seq":382252089506,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1672420,"kb_used_data":1288412,"kb_used_omap":8,"kb_used_meta":383991,"kb_avail":92072732,"statfs":{"total":95995035648,"available":94282477568,"internally_reserved":0,"allocated":1319333888,"data_stored":1318105889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":393207746},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,4,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:23:28 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":4.202,"15min":1.99},"min":{"1min":0.221,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":13.054,"5min":187.41399999999999,"15min":187.41399999999999},"last":2.4710000000000001},{"interface":"front","average":{"1min":3.9529999999999998,"5min":4.6779999999999999,"15min":2.1880000000000002},"min":{"1min":0.23400000000000001,"5min":0.193,"15min":0.193},"max":{"1min":17.088999999999999,"5min":187.78700000000001,"15min":187.78700000000001},"last":2.4529999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:24:03 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":2.3119999999999998,"15min":1.5049999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":13.209,"5min":39.048999999999999,"15min":39.048999999999999},"last":2.532},{"interface":"front","average":{"1min":1.9490000000000001,"5min":2.5939999999999999,"15min":1.694},"min":{"1min":0.32200000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":12.640000000000001,"5min":38.945999999999998,"15min":38.945999999999998},"last":0.83999999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:23:28 2023","interfaces":[{"interface":"back","average":{"1min":2.335,"5min":3.2770000000000001,"15min":1.655},"min":{"1min":0.26600000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":17.914999999999999,"5min":85.840000000000003,"15min":85.840000000000003},"last":0.28799999999999998},{"interface":"front","average":{"1min":3.5259999999999998,"5min":3.6829999999999998,"15min":1.8500000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23400000000000001},"max":{"1min":11.141999999999999,"5min":86.046000000000006,"15min":86.046000000000006},"last":0.95899999999999996}]},{"osd":4,"last update":"Sun Oct 1 09:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":3.3420000000000001,"15min":1.8799999999999999},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.6479999999999997,"5min":112.52200000000001,"15min":112.52200000000001},"last":0.68700000000000006},{"interface":"front","average":{"1min":2.3439999999999999,"5min":3.2999999999999998,"15min":1.9830000000000001},"min":{"1min":0.33700000000000002,"5min":0.32500000000000001,"15min":0.28599999999999998},"max":{"1min":8.9890000000000008,"5min":113.124,"15min":113.124},"last":0.46300000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:24:07 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.673,"15min":1.7889999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.0270000000000001,"5min":5.8710000000000004,"15min":5.8710000000000004},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.208,"5min":1.8080000000000001,"15min":1.9079999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.8090000000000002,"5min":5.8879999999999999,"15min":5.8879999999999999},"last":2.4369999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":2.0910000000000002,"15min":1.577},"min":{"1min":0.375,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.0239999999999991,"5min":34.375999999999998,"15min":34.375999999999998},"last":0.5},{"interface":"front","average":{"1min":2.6709999999999998,"5min":1.8899999999999999,"15min":1.379},"min":{"1min":0.39200000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":9.0820000000000007,"5min":31.745000000000001,"15min":31.745000000000001},"last":2.395}]},{"osd":7,"last update":"Sun Oct 1 09:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":3.0150000000000001,"15min":1.867},"min":{"1min":0.44,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":13.096,"5min":98.283000000000001,"15min":98.283000000000001},"last":2.5019999999999998},{"interface":"front","average":{"1min":2.3119999999999998,"5min":2.762,"15min":1.8700000000000001},"min":{"1min":0.36399999999999999,"5min":0.312,"15min":0.312},"max":{"1min":5.7560000000000002,"5min":98.311999999999998,"15min":98.311999999999998},"last":2.3719999999999999}]}]},{"osd":2,"up_from":99,"seq":425201762445,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1263704,"kb_used_data":891792,"kb_used_omap":11,"kb_used_meta":371892,"kb_avail":92481448,"statfs":{"total":95995035648,"available":94701002752,"internally_reserved":0,"allocated":913195008,"data_stored":912026657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":380818295},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:24:10 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":2.9700000000000002,"15min":2.129},"min":{"1min":0.23400000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":4.4669999999999996,"5min":119.621,"15min":119.621},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.845,"5min":3.0910000000000002,"15min":2.4940000000000002},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20100000000000001},"max":{"1min":6.9539999999999997,"5min":119.411,"15min":119.411},"last":2.5150000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:24:06 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":3.1379999999999999,"15min":2.153},"min":{"1min":0.22,"5min":0.193,"15min":0.193},"max":{"1min":4.3380000000000001,"5min":105.798,"15min":105.798},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.0900000000000001,"5min":3.1560000000000001,"15min":2.3149999999999999},"min":{"1min":0.22,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":4.7450000000000001,"5min":105.56399999999999,"15min":107.729},"last":1.837}]},{"osd":3,"last update":"Sun Oct 1 09:23:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6059999999999999,"5min":6.2910000000000004,"15min":2.7429999999999999},"min":{"1min":0.217,"5min":0.214,"15min":0.19600000000000001},"max":{"1min":25.975000000000001,"5min":253.238,"15min":253.238},"last":1.97},{"interface":"front","average":{"1min":2.8929999999999998,"5min":6.1580000000000004,"15min":2.6970000000000001},"min":{"1min":0.30599999999999999,"5min":0.253,"15min":0.22600000000000001},"max":{"1min":22.498999999999999,"5min":252.54900000000001,"15min":252.54900000000001},"last":0.79800000000000004}]},{"osd":4,"last update":"Sun Oct 1 09:24:06 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":1.508,"15min":1.339},"min":{"1min":0.32200000000000001,"5min":0.27200000000000002,"15min":0.27000000000000002},"max":{"1min":2.5379999999999998,"5min":8.2530000000000001,"15min":17.751999999999999},"last":1.9570000000000001},{"interface":"front","average":{"1min":0.77900000000000003,"5min":1.754,"15min":1.4239999999999999},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.6469999999999998,"5min":14.359,"15min":17.568000000000001},"last":1.944}]},{"osd":5,"last update":"Sun Oct 1 09:24:05 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":2.1909999999999998,"15min":2.4159999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.4290000000000003,"5min":8.2840000000000007,"15min":8.2840000000000007},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.92600000000000005,"5min":2.0710000000000002,"15min":2.262},"min":{"1min":0.40100000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.7320000000000002,"5min":9.3160000000000007,"15min":9.3160000000000007},"last":1.883}]},{"osd":6,"last update":"Sun Oct 1 09:24:06 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":1.804,"15min":1.48},"min":{"1min":0.317,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.6819999999999999,"5min":14.411,"15min":17.591000000000001},"last":1.9019999999999999},{"interface":"front","average":{"1min":1.091,"5min":1.7549999999999999,"15min":1.232},"min":{"1min":0.376,"5min":0.36699999999999999,"15min":0.34200000000000003},"max":{"1min":6.7859999999999996,"5min":8.6620000000000008,"15min":17.736000000000001},"last":0.56799999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:24:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.9059999999999999,"15min":1.8140000000000001},"min":{"1min":0.34999999999999998,"5min":0.34699999999999998,"15min":0.32800000000000001},"max":{"1min":4.492,"5min":16.169,"15min":109.506},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.04,"5min":1.758,"15min":1.9259999999999999},"min":{"1min":0.438,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.6850000000000001,"5min":16.152000000000001,"15min":109.58799999999999},"last":1.9219999999999999}]}]},{"osd":3,"up_from":104,"seq":446676598919,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1681016,"kb_used_data":1420912,"kb_used_omap":3,"kb_used_meta":260092,"kb_avail":92064136,"statfs":{"total":95995035648,"available":94273675264,"internally_reserved":0,"allocated":1455013888,"data_stored":1453821272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":266334951},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":2.9199999999999999,"5min":1.9630000000000001,"15min":2.9689999999999999},"min":{"1min":0.20899999999999999,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":18.635000000000002,"5min":38.314,"15min":199.422},"last":0.93700000000000006},{"interface":"front","average":{"1min":2.976,"5min":2.1680000000000001,"15min":3.1509999999999998},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":18.792000000000002,"5min":38.118000000000002,"15min":199.91200000000001},"last":0.82199999999999995}]},{"osd":1,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":5.431,"5min":5.702,"15min":3.3940000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":37.011000000000003,"5min":228.60499999999999,"15min":228.60499999999999},"last":2.391},{"interface":"front","average":{"1min":4.3399999999999999,"5min":5.3099999999999996,"15min":3.2269999999999999},"min":{"1min":0.26500000000000001,"5min":0.23899999999999999,"15min":0.215},"max":{"1min":36.866,"5min":228.28399999999999,"15min":228.28399999999999},"last":1.292}]},{"osd":2,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":2.6930000000000001,"15min":1.51},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.224},"max":{"1min":7.524,"5min":55.238999999999997,"15min":55.238999999999997},"last":1.1890000000000001},{"interface":"front","average":{"1min":1.99,"5min":2.9409999999999998,"15min":1.6779999999999999},"min":{"1min":0.23999999999999999,"5min":0.219,"15min":0.20399999999999999},"max":{"1min":6.7919999999999998,"5min":55.692999999999998,"15min":55.692999999999998},"last":0.97499999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5,"5min":4.2729999999999997,"15min":2.274},"min":{"1min":0.32600000000000001,"5min":0.27300000000000002,"15min":0.25900000000000001},"max":{"1min":9.0250000000000004,"5min":154.00399999999999,"15min":154.00399999999999},"last":0.997},{"interface":"front","average":{"1min":3.8420000000000001,"5min":4.8140000000000001,"15min":2.601},"min":{"1min":0.316,"5min":0.30099999999999999,"15min":0.248},"max":{"1min":36.947000000000003,"5min":156.87700000000001,"15min":156.87700000000001},"last":0.85499999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:24:07 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":1.7649999999999999,"15min":1.7729999999999999},"min":{"1min":0.34799999999999998,"5min":0.317,"15min":0.317},"max":{"1min":7.2450000000000001,"5min":7.5670000000000002,"15min":7.5670000000000002},"last":0.89600000000000002},{"interface":"front","average":{"1min":1.7490000000000001,"5min":3.4249999999999998,"15min":3.7040000000000002},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":7.3140000000000001,"5min":36.984000000000002,"15min":36.984000000000002},"last":1.036}]},{"osd":6,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":4.3070000000000004,"5min":3.5739999999999998,"15min":2.0369999999999999},"min":{"1min":0.36499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":36.901000000000003,"5min":72.433999999999997,"15min":72.433999999999997},"last":0.93000000000000005},{"interface":"front","average":{"1min":4.1470000000000002,"5min":3.1899999999999999,"15min":1.8100000000000001},"min":{"1min":0.33600000000000002,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":37.274999999999999,"5min":72.450999999999993,"15min":72.450999999999993},"last":1.147}]},{"osd":7,"last update":"Sun Oct 1 09:23:33 2023","interfaces":[{"interface":"back","average":{"1min":4.7910000000000004,"5min":2.2290000000000001,"15min":1.712},"min":{"1min":0.35499999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":37.024000000000001,"5min":38.478999999999999,"15min":79.412999999999997},"last":1.109},{"interface":"front","average":{"1min":4.867,"5min":2.415,"15min":1.7490000000000001},"min":{"1min":0.40799999999999997,"5min":0.36399999999999999,"15min":0.308},"max":{"1min":36.762,"5min":40.039999999999999,"15min":79.527000000000001},"last":1.024}]}]},{"osd":4,"up_from":30,"seq":128849019169,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1093864,"kb_used_data":792544,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92651288,"statfs":{"total":95995035648,"available":94874918912,"internally_reserved":0,"allocated":811565056,"data_stored":810850743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":3.468,"15min":3.3849999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.20999999999999999},"max":{"1min":4.4690000000000003,"5min":111.27800000000001,"15min":133.91499999999999},"last":0.223},{"interface":"front","average":{"1min":2.581,"5min":3.7410000000000001,"15min":3.3540000000000001},"min":{"1min":0.26600000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":8.5099999999999998,"5min":111.289,"15min":133.83199999999999},"last":1.7589999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":2.6110000000000002,"5min":2.4399999999999999,"15min":2.02},"min":{"1min":0.30599999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.5589999999999993,"5min":37.326999999999998,"15min":133.93600000000001},"last":0.25800000000000001},{"interface":"front","average":{"1min":2.0600000000000001,"5min":2.234,"15min":2.105},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":8.0009999999999994,"5min":37.335000000000001,"15min":141.06},"last":0.33300000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:24:09 2023","interfaces":[{"interface":"back","average":{"1min":1.958,"5min":3.2360000000000002,"15min":3.0880000000000001},"min":{"1min":0.34300000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":10.234,"5min":91.474999999999994,"15min":133.96100000000001},"last":1.827},{"interface":"front","average":{"1min":1.9399999999999999,"5min":3.3519999999999999,"15min":3.0619999999999998},"min":{"1min":0.38,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.1769999999999996,"5min":91.510999999999996,"15min":134.018},"last":0.34899999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.5029999999999999,"15min":1.6439999999999999},"min":{"1min":0.32000000000000001,"5min":0.30299999999999999,"15min":0.26200000000000001},"max":{"1min":6.0679999999999996,"5min":6.9909999999999997,"15min":134.297},"last":0.80700000000000005},{"interface":"front","average":{"1min":2.9540000000000002,"5min":1.982,"15min":1.9650000000000001},"min":{"1min":0.30399999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":18.370000000000001,"5min":18.370000000000001,"15min":141.089},"last":0.49299999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:24:04 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":1.609,"15min":1.663},"min":{"1min":0.33500000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.1310000000000002,"5min":6.1310000000000002,"15min":6.1310000000000002},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.95099999999999996,"5min":2.335,"15min":2.5649999999999999},"min":{"1min":0.29099999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.6180000000000003,"5min":8.6539999999999999,"15min":8.6539999999999999},"last":0.377}]},{"osd":6,"last update":"Sun Oct 1 09:23:52 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.502,"15min":1.508},"min":{"1min":0.28799999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.5789999999999997,"5min":8.8900000000000006,"15min":133.90799999999999},"last":1.8009999999999999},{"interface":"front","average":{"1min":1.478,"5min":1.337,"15min":1.744},"min":{"1min":0.38600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.5750000000000002,"5min":20.622,"15min":141.03},"last":1.8080000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:24:09 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.0429999999999999,"15min":1.6279999999999999},"min":{"1min":0.38100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.4180000000000001,"5min":6.0579999999999998,"15min":133.946},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.357,"15min":1.6619999999999999},"min":{"1min":0.377,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.5380000000000003,"5min":8.7140000000000004,"15min":133.98400000000001},"last":0.32400000000000001}]}]},{"osd":5,"up_from":227,"seq":974957576220,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":968016,"kb_used_data":632392,"kb_used_omap":8,"kb_used_meta":335607,"kb_avail":92777136,"statfs":{"total":95995035648,"available":95003787264,"internally_reserved":0,"allocated":647569408,"data_stored":646867193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":343662340},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:24:08 2023","interfaces":[{"interface":"back","average":{"1min":1.4199999999999999,"5min":1.9239999999999999,"15min":2.008},"min":{"1min":0.254,"5min":0.223,"15min":0.223},"max":{"1min":9.9100000000000001,"5min":9.9100000000000001,"15min":9.9100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":1.5629999999999999,"5min":2.4830000000000001,"15min":2.6360000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.0609999999999999,"5min":10.867000000000001,"15min":10.867000000000001},"last":0.47399999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:24:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":2.04,"15min":2.1920000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.6800000000000002,"5min":6.2110000000000003,"15min":6.2110000000000003},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.0349999999999999,"5min":2.8500000000000001,"15min":3.1520000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":8.8390000000000004,"5min":9.4280000000000008,"15min":9.4280000000000008},"last":0.378}]},{"osd":2,"last update":"Sun Oct 1 09:24:08 2023","interfaces":[{"interface":"back","average":{"1min":3.629,"5min":3.036,"15min":2.9369999999999998},"min":{"1min":0.34599999999999997,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":44.615000000000002,"5min":44.615000000000002,"15min":44.615000000000002},"last":0.30399999999999999},{"interface":"front","average":{"1min":3.8100000000000001,"5min":2.169,"15min":1.895},"min":{"1min":0.32400000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":44.511000000000003,"5min":44.511000000000003,"15min":44.511000000000003},"last":1.681}]},{"osd":3,"last update":"Sun Oct 1 09:24:08 2023","interfaces":[{"interface":"back","average":{"1min":5.5490000000000004,"5min":2.8889999999999998,"15min":2.4460000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":72.844999999999999,"5min":72.844999999999999,"15min":72.844999999999999},"last":1.2470000000000001},{"interface":"front","average":{"1min":5.7300000000000004,"5min":2.8849999999999998,"15min":2.411},"min":{"1min":0.37,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":73.191999999999993,"5min":73.191999999999993,"15min":73.191999999999993},"last":1.389}]},{"osd":4,"last update":"Sun Oct 1 09:24:08 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.7030000000000001,"15min":1.8280000000000001},"min":{"1min":0.27400000000000002,"5min":0.248,"15min":0.248},"max":{"1min":4.9240000000000004,"5min":7.532,"15min":7.532},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.425,"15min":1.4910000000000001},"min":{"1min":0.32800000000000001,"5min":0.316,"15min":0.316},"max":{"1min":3.762,"5min":4.117,"15min":4.117},"last":0.42799999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:24:08 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":1.73,"15min":1.875},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":4.968,"5min":8.7520000000000007,"15min":8.7520000000000007},"last":0.441},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.3109999999999999,"15min":1.371},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.673,"5min":7.7460000000000004,"15min":7.7460000000000004},"last":0.45700000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:24:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":1.5600000000000001,"15min":1.6259999999999999},"min":{"1min":0.36099999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.7190000000000003,"5min":5.7190000000000003,"15min":5.7190000000000003},"last":0.5},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.954,"15min":2.1400000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":5.0019999999999998,"5min":10.836,"15min":10.836},"last":0.40799999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1333723136,"data_stored":1333723136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":890044416,"data_stored":890044416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":916979712,"data_stored":916979712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1464532992,"data_stored":1464532992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":813629440,"data_stored":813629440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":614989824,"data_stored":614989824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":785645568,"data_stored":785645568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":803274752,"data_stored":803274752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:24:20.620 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-01T09:24:20.621 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:24:20.621 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-01T09:24:20.621 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-10-01T09:24:20.649 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:24:20.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:20.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:24:20.752 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:20 smithi138 ceph-mon[192513]: from='client.45383 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:24:20.753 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:20 smithi138 ceph-mon[192513]: pgmap v674: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-01T09:24:20.756 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:20.764 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:20 smithi138 ceph-mon[185886]: from='client.45383 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:24:20.764 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:20 smithi138 ceph-mon[185886]: pgmap v674: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-01T09:24:20.822 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:20.886 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 27 16 44407 44391 102.707 79.0625 0.0134783 0.00972914 2023-10-01T09:24:21.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:20 smithi190 ceph-mon[160435]: from='client.45383 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:24:21.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:20 smithi190 ceph-mon[160435]: pgmap v674: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-01T09:24:21.365 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:24:21.365 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:24:21.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.366 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:44706 7.e 7:72f80b85:::benchmark_data_smithi138_367058_object44705:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-10-01T09:24:21.366 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:24:21.180034+0000", 2023-10-01T09:24:21.366 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0042882320000000003, 2023-10-01T09:24:21.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043252480000000003, 2023-10-01T09:24:21.366 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:21.366 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:24:21.366 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:21.366 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:24:21.366 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:24:21.366 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 44706 2023-10-01T09:24:21.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.367 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:21.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:21.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.180034+0000", 2023-10-01T09:24:21.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:21.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:21.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.180034+0000", 2023-10-01T09:24:21.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:21.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:21.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.180033+0000", 2023-10-01T09:24:21.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:24:21.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:21.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.180055+0000", 2023-10-01T09:24:21.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.211e-05 2023-10-01T09:24:21.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:21.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.180066+0000", 2023-10-01T09:24:21.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0858e-05 2023-10-01T09:24:21.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:21.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.180073+0000", 2023-10-01T09:24:21.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9840000000000004e-06 2023-10-01T09:24:21.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:21.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.180935+0000", 2023-10-01T09:24:21.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00086202299999999996 2023-10-01T09:24:21.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:21.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.181067+0000", 2023-10-01T09:24:21.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131602 2023-10-01T09:24:21.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.370 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:24:21.370 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.181190+0000", 2023-10-01T09:24:21.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000123149 2023-10-01T09:24:21.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:21.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.183968+0000", 2023-10-01T09:24:21.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002778043 2023-10-01T09:24:21.371 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:21.371 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:21.371 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:21.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.371 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35421.0:44703 7.5 e264/259 7:acdf2370:::benchmark_data_smithi138_367058_object44702:head v 264'2864, mlcod=264'2862)", 2023-10-01T09:24:21.371 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:24:21.180910+0000", 2023-10-01T09:24:21.372 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.003412174, 2023-10-01T09:24:21.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035416150000000001, 2023-10-01T09:24:21.372 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:21.372 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:24:21.372 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:21.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:21.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.180910+0000", 2023-10-01T09:24:21.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:21.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:21.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.180910+0000", 2023-10-01T09:24:21.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:21.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:21.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.180910+0000", 2023-10-01T09:24:21.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:24:21.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:21.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.180969+0000", 2023-10-01T09:24:21.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9627999999999999e-05 2023-10-01T09:24:21.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:21.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.180979+0000", 2023-10-01T09:24:21.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.55e-06 2023-10-01T09:24:21.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:21.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.180985+0000", 2023-10-01T09:24:21.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8699999999999997e-06 2023-10-01T09:24:21.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:21.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.181026+0000", 2023-10-01T09:24:21.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0847999999999999e-05 2023-10-01T09:24:21.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:21.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.181047+0000", 2023-10-01T09:24:21.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.141e-05 2023-10-01T09:24:21.375 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:21.375 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:21.375 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:21.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.375 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:44707 7.8 7:1457584c:::benchmark_data_smithi138_367058_object44706:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-10-01T09:24:21.376 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:24:21.181808+0000", 2023-10-01T09:24:21.376 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0025148359999999999, 2023-10-01T09:24:21.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0027157589999999999, 2023-10-01T09:24:21.376 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:21.376 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:24:21.376 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:21.376 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:24:21.376 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:24:21.376 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 44707 2023-10-01T09:24:21.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.376 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:21.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:21.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.181808+0000", 2023-10-01T09:24:21.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:21.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:21.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.181808+0000", 2023-10-01T09:24:21.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:21.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:21.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.181807+0000", 2023-10-01T09:24:21.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:24:21.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:21.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.181831+0000", 2023-10-01T09:24:21.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4632000000000001e-05 2023-10-01T09:24:21.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:21.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.181842+0000", 2023-10-01T09:24:21.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1129e-05 2023-10-01T09:24:21.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:21.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.181849+0000", 2023-10-01T09:24:21.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0960000000000003e-06 2023-10-01T09:24:21.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.379 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:21.379 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.181904+0000", 2023-10-01T09:24:21.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4999e-05 2023-10-01T09:24:21.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:21.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.182052+0000", 2023-10-01T09:24:21.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014773299999999999 2023-10-01T09:24:21.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,7", 2023-10-01T09:24:21.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.182191+0000", 2023-10-01T09:24:21.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000139266 2023-10-01T09:24:21.380 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:21.380 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:21.380 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:21.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.381 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35421.0:44708 7.5 e264/259 7:aa85fdcb:::benchmark_data_smithi138_367058_object44707:head v 264'2865, mlcod=264'2862)", 2023-10-01T09:24:21.381 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:24:21.182552+0000", 2023-10-01T09:24:21.381 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0017700929999999999, 2023-10-01T09:24:21.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020488429999999998, 2023-10-01T09:24:21.381 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:21.381 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:24:21.381 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:21.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:21.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.182552+0000", 2023-10-01T09:24:21.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:21.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:21.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.182552+0000", 2023-10-01T09:24:21.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:21.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:21.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.182552+0000", 2023-10-01T09:24:21.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:24:21.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:21.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.182583+0000", 2023-10-01T09:24:21.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1559000000000001e-05 2023-10-01T09:24:21.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:21.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.182592+0000", 2023-10-01T09:24:21.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2619999999999998e-06 2023-10-01T09:24:21.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:21.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.182598+0000", 2023-10-01T09:24:21.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9070000000000004e-06 2023-10-01T09:24:21.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:21.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.182958+0000", 2023-10-01T09:24:21.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00036081200000000001 2023-10-01T09:24:21.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:21.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:21.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:21.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:24:21.182978+0000", 2023-10-01T09:24:21.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9117999999999999e-05 2023-10-01T09:24:21.384 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:21.384 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:21.384 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:21.385 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:21.385 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:24:21.385 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 4 2023-10-01T09:24:21.385 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:24:21.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:21.386 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:21.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:21.487 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:24:21.525 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:21.572 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:24:21.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:21.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:24:21.676 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:24:21.701 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:21.765 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:24:21.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:21.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:24:21.870 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:21.886 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 28 16 45400 45384 101.255 62.0625 0.0047476 0.00987102 2023-10-01T09:24:21.933 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:22.567 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:22.602 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:24:22.645 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:22.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:22.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:22.754 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:24:22.784 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:22.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:22 smithi138 ceph-mon[185886]: pgmap v675: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T09:24:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:22 smithi138 ceph-mon[192513]: pgmap v675: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T09:24:22.839 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:24:22.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:22.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:24:22.886 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 29 16 46528 46512 100.195 70.5 0.00621164 0.0099729 2023-10-01T09:24:22.947 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:22.976 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:22 smithi190 ceph-mon[160435]: pgmap v675: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T09:24:22.980 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:23.041 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:23.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:23.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:23.103 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:23.147 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:23.178 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:23.230 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:23.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:23.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:23.269 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:24:23.319 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:24:23.336 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:23.369 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:23.373 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:24:23.409 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:24:23.427 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:23.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:23.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:23.450 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:24:23.482 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:24:23.519 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:24:23.536 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:24:23.552 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:24:23.562 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:24:23.562 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:24:23.563 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:24:23.563 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:24:23.563 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:24:23.563 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:24:23.563 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:24:23.563 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:24:23.569 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:23.624 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:24:23.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:23.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:24:23.730 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:24:23.763 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:23.827 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:24:23.831 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:23 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3044820629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:23.831 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:23 smithi190 ceph-mon[160435]: pgmap v676: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T09:24:23.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:23.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:24:23.888 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 30 16 47785 47769 99.4683 78.5625 0.0158371 0.0100481 2023-10-01T09:24:23.934 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:23.959 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:24.000 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:24.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:24.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:24.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:23 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3044820629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:24.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:23 smithi138 ceph-mon[185886]: pgmap v676: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T09:24:24.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:23 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3044820629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:24.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:23 smithi138 ceph-mon[192513]: pgmap v676: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T09:24:24.107 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:24:24.149 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:24.888 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 31 16 48859 48843 98.4249 67.125 0.00458947 0.0101453 2023-10-01T09:24:25.062 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:24:25.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:25.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:24:25.168 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:24:25.231 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:24:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:24:24] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-01T09:24:25.233 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:25.436 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:24:25.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:25.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:24:25.545 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:25.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:24:25.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:24:25.578 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:24:25.582 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:25.639 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:25.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:25.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:25.745 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:25.816 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:25.843 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:25.888 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 32 16 49947 49931 97.4743 68 0.00720189 0.0102539 2023-10-01T09:24:26.078 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:26.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:26.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:26.191 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:26.224 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:26.286 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:26.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:26.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:26.394 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:24:26.420 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:26.461 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:24:26.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:26.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:24:26.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:26 smithi190 ceph-mon[160435]: pgmap v677: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T09:24:26.569 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:24:26.594 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:26 smithi138 ceph-mon[185886]: pgmap v677: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T09:24:26.594 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:26 smithi138 ceph-mon[192513]: pgmap v677: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T09:24:26.597 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:26.890 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 33 16 50943 50927 96.4016 62.25 0.00701677 0.010368 2023-10-01T09:24:26.911 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:24:26.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:26.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:24:27.025 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:24:27.098 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:27.233 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:24:27.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:27.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:24:27.337 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:27.403 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:24:27.411 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:27.430 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:24:27.519 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:27 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3737608678' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:24:27.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:27 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3737608678' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:24:27.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:27 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3737608678' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:24:27.891 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 34 16 51975 51959 95.4634 64.5 0.00482309 0.0104335 2023-10-01T09:24:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:28 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:24:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:28 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:24:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3737608678' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:24:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:28 smithi190 ceph-mon[160435]: osdmap e265: 8 total, 8 up, 8 in 2023-10-01T09:24:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:28 smithi190 ceph-mon[160435]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:28 smithi190 ceph-mon[160435]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:28 smithi190 ceph-mon[160435]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:28 smithi190 ceph-mon[160435]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:28 smithi190 ceph-mon[160435]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:28 smithi190 ceph-mon[160435]: pgmap v679: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T09:24:28.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:24:28.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:24:28.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3737608678' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:24:28.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[185886]: osdmap e265: 8 total, 8 up, 8 in 2023-10-01T09:24:28.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[185886]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[185886]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[185886]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[185886]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[185886]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[185886]: pgmap v679: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T09:24:28.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:24:28.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:24:28.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3737608678' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:24:28.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[192513]: osdmap e265: 8 total, 8 up, 8 in 2023-10-01T09:24:28.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[192513]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[192513]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[192513]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[192513]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[192513]: failed to encode map e265 with expected crc 2023-10-01T09:24:28.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:28 smithi138 ceph-mon[192513]: pgmap v679: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T09:24:28.893 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 35 16 52726 52710 94.07 46.9375 0.00650565 0.0106248 2023-10-01T09:24:29.133 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:29.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:29.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:29.239 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:29.342 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:29.398 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:29.553 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:29.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:29.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:29.663 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:24:29.692 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:29.711 INFO:teuthology.orchestra.run.smithi138.stdout:moved 23 / 267 (8.61423%) 2023-10-01T09:24:29.711 INFO:teuthology.orchestra.run.smithi138.stdout:avg 33.375 2023-10-01T09:24:29.711 INFO:teuthology.orchestra.run.smithi138.stdout:stddev 5.17053 -> 6.92708 (expected baseline 5.40399) 2023-10-01T09:24:29.711 INFO:teuthology.orchestra.run.smithi138.stdout:min osd.2 with 25 -> 28 pgs (0.749064 -> 0.838951 * mean) 2023-10-01T09:24:29.711 INFO:teuthology.orchestra.run.smithi138.stdout:max osd.3 with 41 -> 33 pgs (1.22846 -> 0.988764 * mean) 2023-10-01T09:24:29.711 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:24:29.712 INFO:teuthology.orchestra.run.smithi138.stdout:oload 110 2023-10-01T09:24:29.712 INFO:teuthology.orchestra.run.smithi138.stdout:max_change 3 2023-10-01T09:24:29.712 INFO:teuthology.orchestra.run.smithi138.stdout:max_change_osds 4 2023-10-01T09:24:29.712 INFO:teuthology.orchestra.run.smithi138.stdout:average_utilization 0.0163 2023-10-01T09:24:29.712 INFO:teuthology.orchestra.run.smithi138.stdout:overload_utilization 0.0180 2023-10-01T09:24:29.712 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3 weight 1.0000 -> 0.7395 2023-10-01T09:24:29.712 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0 weight 1.0000 -> 0.7485 2023-10-01T09:24:29.712 INFO:teuthology.orchestra.run.smithi138.stderr:no change 2023-10-01T09:24:29.746 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:24:29.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:29.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:24:29.855 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:29.893 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 36 16 53740 53724 93.2176 63.375 0.01338 0.0107228 2023-10-01T09:24:29.919 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:24:29.919 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:24:29.933 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:30.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:24:29 smithi138 conmon[178630]: 2023-10-01T09:24:29.704+0000 7fa004e48700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-01T09:24:30.743 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:30 smithi138 ceph-mon[192513]: pgmap v680: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T09:24:30.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:30 smithi138 ceph-mon[185886]: pgmap v680: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T09:24:30.894 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 37 16 54498 54482 91.9792 47.375 0.00558688 0.0108669 2023-10-01T09:24:31.024 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:24:31.025 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-01T09:24:31.025 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-10-01T09:24:31.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:30 smithi190 ceph-mon[160435]: pgmap v680: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T09:24:31.894 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 38 16 55206 55190 90.7236 44.25 0.0222372 0.0110167 2023-10-01T09:24:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:31 smithi190 ceph-mon[160435]: from='client.55207 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:24:32.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:31 smithi138 ceph-mon[192513]: from='client.55207 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:24:32.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:31 smithi138 ceph-mon[185886]: from='client.55207 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:24:32.263 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:32.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:32.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:32.321 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:24:32.369 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:24:32.413 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:32.421 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:32.465 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:24:32.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:32.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:24:32.570 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:24:32.697 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:32.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:32 smithi190 ceph-mon[160435]: pgmap v681: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T09:24:32.846 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:32 smithi138 ceph-mon[192513]: pgmap v681: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T09:24:32.846 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:32 smithi138 ceph-mon[185886]: pgmap v681: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T09:24:32.897 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 39 16 56260 56244 90.0796 65.875 0.00866246 0.011094 2023-10-01T09:24:32.955 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:32.983 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:24:32.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:32.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:24:33.098 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:24:33.129 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:24:33.168 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:24:33.170 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:33.209 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:24:33.252 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:24:33.292 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:24:33.326 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:24:33.358 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:24:33.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:33.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:24:33.363 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:24:33.399 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:24:33.406 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:24:33.407 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:24:33.407 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:24:33.407 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:24:33.407 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:24:33.407 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:24:33.407 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:24:33.407 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:24:33.464 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:24:33.502 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:33.569 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:24:33.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:33.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:24:33.674 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:33.703 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:33 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2017044780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:33.706 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:33 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2017044780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:33.707 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:33.760 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:33.763 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:33 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2017044780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:33.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:33.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:33.865 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:33.890 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:33.897 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:24:33.897502+0000 min lat: 0.0011387 max lat: 0.332925 avg lat: 0.0111602 2023-10-01T09:24:33.897 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:24:33.897 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 40 16 57264 57248 89.3966 62.75 0.00284011 0.0111602 2023-10-01T09:24:33.936 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:33.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:33.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:34.045 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:24:34.076 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:34.142 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:24:34.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:34.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:24:34.249 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:24:34.282 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:34.339 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:24:34.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:34.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:24:34.448 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:34.508 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:34.614 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:34 smithi190 ceph-mon[160435]: pgmap v682: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T09:24:34.772 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:34 smithi138 ceph-mon[185886]: pgmap v682: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T09:24:34.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:34 smithi138 ceph-mon[192513]: pgmap v682: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T09:24:34.897 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 41 16 58442 58426 89.0119 73.625 0.00455237 0.0112292 2023-10-01T09:24:34.995 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:35.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:35.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:35.072 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:24:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:24:34] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.33.4" 2023-10-01T09:24:35.102 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:24:35.183 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:35.312 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:35.480 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:24:35.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:35.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:24:35.588 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:35.617 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:35.897 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 42 16 59431 59415 88.3645 61.8125 0.0185649 0.0113106 2023-10-01T09:24:36.094 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:36.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:36.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:36.198 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:36.232 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:36.291 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:36.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:36.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:36.341 INFO:teuthology.orchestra.run.smithi138.stdout:moved 0 / 267 (0%) 2023-10-01T09:24:36.341 INFO:teuthology.orchestra.run.smithi138.stdout:avg 33.375 2023-10-01T09:24:36.341 INFO:teuthology.orchestra.run.smithi138.stdout:stddev 5.17053 -> 5.17053 (expected baseline 5.40399) 2023-10-01T09:24:36.341 INFO:teuthology.orchestra.run.smithi138.stdout:min osd.2 with 25 -> 25 pgs (0.749064 -> 0.749064 * mean) 2023-10-01T09:24:36.341 INFO:teuthology.orchestra.run.smithi138.stdout:max osd.3 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-10-01T09:24:36.341 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:24:36.341 INFO:teuthology.orchestra.run.smithi138.stdout:oload 1000 2023-10-01T09:24:36.341 INFO:teuthology.orchestra.run.smithi138.stdout:max_change 0.05 2023-10-01T09:24:36.342 INFO:teuthology.orchestra.run.smithi138.stdout:max_change_osds 4 2023-10-01T09:24:36.342 INFO:teuthology.orchestra.run.smithi138.stdout:average_utilization 0.0178 2023-10-01T09:24:36.342 INFO:teuthology.orchestra.run.smithi138.stdout:overload_utilization 0.1783 2023-10-01T09:24:36.357 INFO:teuthology.orchestra.run.smithi138.stderr:no change 2023-10-01T09:24:36.401 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:24:36.432 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:36.488 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:24:36.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:36.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:24:36.593 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:24:36.621 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:36.683 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:24:36.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:36.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:24:36.790 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:24:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:36 smithi190 ceph-mon[160435]: pgmap v683: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-01T09:24:36.815 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:36 smithi138 ceph-mon[185886]: pgmap v683: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-01T09:24:36.816 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:36 smithi138 ceph-mon[192513]: pgmap v683: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s 2023-10-01T09:24:36.818 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:36.897 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 43 16 60428 60412 87.7587 62.3125 0.00427274 0.0113737 2023-10-01T09:24:37.169 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:24:37.308 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:24:37.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:37.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:24:37.419 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:37.457 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-01T09:24:37.457 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-10-01T09:24:37.536 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:37.728 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:37.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:37.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:37.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:37 smithi138 ceph-mon[185886]: from='client.45401 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:24:37.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:37 smithi138 ceph-mon[192513]: from='client.45401 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:24:37.839 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:24:37.897 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 44 16 61393 61377 87.1351 60.3125 0.0159425 0.0114702 2023-10-01T09:24:37.946 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:38.033 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:24:38.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:38.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:24:38.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:37 smithi190 ceph-mon[160435]: from='client.45401 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:24:38.137 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:38.214 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:38.734 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:38.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:38.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:38.842 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:24:38.871 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:38.898 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 45 16 62488 62472 86.7196 68.4375 0.00595843 0.011527 2023-10-01T09:24:38.927 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:24:38.932 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:38 smithi190 ceph-mon[160435]: pgmap v684: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T09:24:38.932 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/598757251' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:24:38.932 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:38 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:24:38.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:38.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:24:39.034 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:39.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:38 smithi138 ceph-mon[185886]: pgmap v684: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T09:24:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/598757251' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:24:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:38 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:24:39.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:38 smithi138 ceph-mon[192513]: pgmap v684: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T09:24:39.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/598757251' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:24:39.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:38 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:24:39.099 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:39.218 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:24:39.276 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:39.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:39.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:39.386 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:24:39.448 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:39.596 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:24:39.596 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:24:39.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:24:39.597 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:24:39.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.597 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95302 6.2 6:4510ce89:::benchmark_data_smithi138_307751_object95301:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-01T09:24:39.597 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:03.371974+0000", 2023-10-01T09:24:39.597 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 216.22075266799999, 2023-10-01T09:24:39.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.26466727, 2023-10-01T09:24:39.597 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.597 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.597 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.597 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.598 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.598 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95302 2023-10-01T09:24:39.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.598 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.371974+0000", 2023-10-01T09:24:39.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.371974+0000", 2023-10-01T09:24:39.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.371972+0000", 2023-10-01T09:24:39.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:24:39.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.371998+0000", 2023-10-01T09:24:39.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5545999999999999e-05 2023-10-01T09:24:39.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.372013+0000", 2023-10-01T09:24:39.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5336e-05 2023-10-01T09:24:39.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.372022+0000", 2023-10-01T09:24:39.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8939999999999999e-06 2023-10-01T09:24:39.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.372083+0000", 2023-10-01T09:24:39.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1211999999999996e-05 2023-10-01T09:24:39.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.372216+0000", 2023-10-01T09:24:39.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013330899999999999 2023-10-01T09:24:39.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:39.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.372312+0000", 2023-10-01T09:24:39.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5858000000000003e-05 2023-10-01T09:24:39.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.375920+0000", 2023-10-01T09:24:39.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036081450000000001 2023-10-01T09:24:39.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.376644+0000", 2023-10-01T09:24:39.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000723874 2023-10-01T09:24:39.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.636555+0000", 2023-10-01T09:24:39.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.259910734 2023-10-01T09:24:39.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.636600+0000", 2023-10-01T09:24:39.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5272999999999998e-05 2023-10-01T09:24:39.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.636641+0000", 2023-10-01T09:24:39.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0865999999999998e-05 2023-10-01T09:24:39.605 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.606 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.606 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.606 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95322 6.2 6:456699d3:::benchmark_data_smithi138_307751_object95321:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-01T09:24:39.606 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:03.430718+0000", 2023-10-01T09:24:39.606 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 216.16200827899999, 2023-10-01T09:24:39.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.255276716000001, 2023-10-01T09:24:39.606 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.607 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.607 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.607 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.607 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.607 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95322 2023-10-01T09:24:39.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.607 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430718+0000", 2023-10-01T09:24:39.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430718+0000", 2023-10-01T09:24:39.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430717+0000", 2023-10-01T09:24:39.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:24:39.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430737+0000", 2023-10-01T09:24:39.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9941000000000001e-05 2023-10-01T09:24:39.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430746+0000", 2023-10-01T09:24:39.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2299999999999997e-06 2023-10-01T09:24:39.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430752+0000", 2023-10-01T09:24:39.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4999999999999999e-06 2023-10-01T09:24:39.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430796+0000", 2023-10-01T09:24:39.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3548999999999999e-05 2023-10-01T09:24:39.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430930+0000", 2023-10-01T09:24:39.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013467199999999999 2023-10-01T09:24:39.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:39.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.431086+0000", 2023-10-01T09:24:39.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015616999999999999 2023-10-01T09:24:39.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.434311+0000", 2023-10-01T09:24:39.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032241959999999999 2023-10-01T09:24:39.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.434323+0000", 2023-10-01T09:24:39.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2476999999999999e-05 2023-10-01T09:24:39.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.685898+0000", 2023-10-01T09:24:39.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.251575172999999 2023-10-01T09:24:39.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.685952+0000", 2023-10-01T09:24:39.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4243999999999998e-05 2023-10-01T09:24:39.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.685995+0000", 2023-10-01T09:24:39.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2543999999999998e-05 2023-10-01T09:24:39.615 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.615 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.615 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.615 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95334 6.2 6:42c96aa7:::benchmark_data_smithi138_307751_object95333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-01T09:24:39.615 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:03.454910+0000", 2023-10-01T09:24:39.615 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 216.13781657499999, 2023-10-01T09:24:39.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.238695902, 2023-10-01T09:24:39.615 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.616 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.616 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.616 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.616 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.617 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95334 2023-10-01T09:24:39.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.617 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.454910+0000", 2023-10-01T09:24:39.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.454910+0000", 2023-10-01T09:24:39.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.454909+0000", 2023-10-01T09:24:39.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:24:39.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.454930+0000", 2023-10-01T09:24:39.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1038000000000001e-05 2023-10-01T09:24:39.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.454940+0000", 2023-10-01T09:24:39.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8579999999999995e-06 2023-10-01T09:24:39.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.454946+0000", 2023-10-01T09:24:39.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0120000000000002e-06 2023-10-01T09:24:39.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.454989+0000", 2023-10-01T09:24:39.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3510000000000002e-05 2023-10-01T09:24:39.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.455086+0000", 2023-10-01T09:24:39.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6867000000000006e-05 2023-10-01T09:24:39.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:39.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.455174+0000", 2023-10-01T09:24:39.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7441000000000005e-05 2023-10-01T09:24:39.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.457693+0000", 2023-10-01T09:24:39.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025194079999999999 2023-10-01T09:24:39.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.506165+0000", 2023-10-01T09:24:39.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.048471805 2023-10-01T09:24:39.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.693535+0000", 2023-10-01T09:24:39.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.187370025 2023-10-01T09:24:39.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.693566+0000", 2023-10-01T09:24:39.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1559000000000001e-05 2023-10-01T09:24:39.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.693606+0000", 2023-10-01T09:24:39.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9496999999999998e-05 2023-10-01T09:24:39.624 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.624 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.625 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.625 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95348 6.2 6:46ec8a63:::benchmark_data_smithi138_307751_object95347:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-01T09:24:39.625 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:03.513565+0000", 2023-10-01T09:24:39.625 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 216.07916124900001, 2023-10-01T09:24:39.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.186301394000001, 2023-10-01T09:24:39.625 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.625 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.625 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.625 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.625 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.626 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95348 2023-10-01T09:24:39.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.626 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513565+0000", 2023-10-01T09:24:39.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513565+0000", 2023-10-01T09:24:39.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513564+0000", 2023-10-01T09:24:39.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:24:39.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513584+0000", 2023-10-01T09:24:39.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0169e-05 2023-10-01T09:24:39.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513596+0000", 2023-10-01T09:24:39.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1834000000000001e-05 2023-10-01T09:24:39.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513601+0000", 2023-10-01T09:24:39.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3430000000000003e-06 2023-10-01T09:24:39.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513648+0000", 2023-10-01T09:24:39.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6869999999999997e-05 2023-10-01T09:24:39.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513788+0000", 2023-10-01T09:24:39.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013930999999999999 2023-10-01T09:24:39.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:39.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513937+0000", 2023-10-01T09:24:39.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014913400000000001 2023-10-01T09:24:39.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.516816+0000", 2023-10-01T09:24:39.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028788429999999999 2023-10-01T09:24:39.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.517680+0000", 2023-10-01T09:24:39.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00086450399999999999 2023-10-01T09:24:39.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.699800+0000", 2023-10-01T09:24:39.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.182119651000001 2023-10-01T09:24:39.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.699830+0000", 2023-10-01T09:24:39.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0378e-05 2023-10-01T09:24:39.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.699867+0000", 2023-10-01T09:24:39.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6563000000000002e-05 2023-10-01T09:24:39.632 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.632 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.632 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.632 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95362 6.2 6:4eb5c0df:::benchmark_data_smithi138_307751_object95361:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-01T09:24:39.632 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:03.544575+0000", 2023-10-01T09:24:39.633 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 216.048151174, 2023-10-01T09:24:39.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.171384007, 2023-10-01T09:24:39.633 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.633 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.633 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.633 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.633 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.633 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95362 2023-10-01T09:24:39.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.633 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544575+0000", 2023-10-01T09:24:39.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544575+0000", 2023-10-01T09:24:39.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544574+0000", 2023-10-01T09:24:39.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:24:39.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544594+0000", 2023-10-01T09:24:39.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0352000000000001e-05 2023-10-01T09:24:39.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544605+0000", 2023-10-01T09:24:39.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0407e-05 2023-10-01T09:24:39.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544611+0000", 2023-10-01T09:24:39.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.02e-06 2023-10-01T09:24:39.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544646+0000", 2023-10-01T09:24:39.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4696999999999997e-05 2023-10-01T09:24:39.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544730+0000", 2023-10-01T09:24:39.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4630999999999997e-05 2023-10-01T09:24:39.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:39.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544817+0000", 2023-10-01T09:24:39.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6589999999999996e-05 2023-10-01T09:24:39.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.547214+0000", 2023-10-01T09:24:39.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002397181 2023-10-01T09:24:39.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.548157+0000", 2023-10-01T09:24:39.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000942646 2023-10-01T09:24:39.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.715877+0000", 2023-10-01T09:24:39.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.167720721 2023-10-01T09:24:39.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.715933+0000", 2023-10-01T09:24:39.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5287000000000001e-05 2023-10-01T09:24:39.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.715959+0000", 2023-10-01T09:24:39.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6727e-05 2023-10-01T09:24:39.640 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.640 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.640 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.640 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95366 6.2 6:4a76bb19:::benchmark_data_smithi138_307751_object95365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-01T09:24:39.641 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:03.558126+0000", 2023-10-01T09:24:39.641 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 216.03460085899999, 2023-10-01T09:24:39.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.191781848, 2023-10-01T09:24:39.641 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.641 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.641 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.641 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.641 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.641 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95366 2023-10-01T09:24:39.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.642 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558126+0000", 2023-10-01T09:24:39.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558126+0000", 2023-10-01T09:24:39.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558124+0000", 2023-10-01T09:24:39.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:24:39.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558147+0000", 2023-10-01T09:24:39.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2473999999999999e-05 2023-10-01T09:24:39.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558157+0000", 2023-10-01T09:24:39.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0547000000000001e-05 2023-10-01T09:24:39.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558164+0000", 2023-10-01T09:24:39.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1979999999999998e-06 2023-10-01T09:24:39.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558200+0000", 2023-10-01T09:24:39.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6804000000000002e-05 2023-10-01T09:24:39.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558283+0000", 2023-10-01T09:24:39.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2905999999999995e-05 2023-10-01T09:24:39.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:39.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558365+0000", 2023-10-01T09:24:39.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1786000000000006e-05 2023-10-01T09:24:39.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.560751+0000", 2023-10-01T09:24:39.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023859520000000002 2023-10-01T09:24:39.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.561038+0000", 2023-10-01T09:24:39.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00028716700000000001 2023-10-01T09:24:39.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.749799+0000", 2023-10-01T09:24:39.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.188761356000001 2023-10-01T09:24:39.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.749860+0000", 2023-10-01T09:24:39.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0207999999999997e-05 2023-10-01T09:24:39.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.749908+0000", 2023-10-01T09:24:39.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7851e-05 2023-10-01T09:24:39.648 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.648 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.648 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.648 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96044 6.f 6:f4455201:::benchmark_data_smithi138_307751_object675:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-10-01T09:24:39.648 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:16.567948+0000", 2023-10-01T09:24:39.648 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 203.02477805999999, 2023-10-01T09:24:39.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96977755200000004, 2023-10-01T09:24:39.648 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.649 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.649 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.649 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.649 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.649 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96044 2023-10-01T09:24:39.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.649 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.567948+0000", 2023-10-01T09:24:39.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.567948+0000", 2023-10-01T09:24:39.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.567947+0000", 2023-10-01T09:24:39.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:24:39.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.567952+0000", 2023-10-01T09:24:39.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.045e-06 2023-10-01T09:24:39.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.567965+0000", 2023-10-01T09:24:39.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2472e-05 2023-10-01T09:24:39.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.567972+0000", 2023-10-01T09:24:39.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4449999999999999e-06 2023-10-01T09:24:39.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568042+0000", 2023-10-01T09:24:39.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9713000000000002e-05 2023-10-01T09:24:39.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:39.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568055+0000", 2023-10-01T09:24:39.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2795e-05 2023-10-01T09:24:39.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.530163+0000", 2023-10-01T09:24:39.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96210800900000004 2023-10-01T09:24:39.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.530318+0000", 2023-10-01T09:24:39.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000154797 2023-10-01T09:24:39.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:39.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.530435+0000", 2023-10-01T09:24:39.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117372 2023-10-01T09:24:39.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537484+0000", 2023-10-01T09:24:39.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.007049484 2023-10-01T09:24:39.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537525+0000", 2023-10-01T09:24:39.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0914999999999997e-05 2023-10-01T09:24:39.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537651+0000", 2023-10-01T09:24:39.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012556899999999999 2023-10-01T09:24:39.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537686+0000", 2023-10-01T09:24:39.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5012999999999998e-05 2023-10-01T09:24:39.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537726+0000", 2023-10-01T09:24:39.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0173000000000003e-05 2023-10-01T09:24:39.656 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.656 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.656 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.657 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96045 6.f 6:faa45668:::benchmark_data_smithi138_307751_object676:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-10-01T09:24:39.657 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:16.568148+0000", 2023-10-01T09:24:39.657 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 203.02457847700001, 2023-10-01T09:24:39.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96969213099999996, 2023-10-01T09:24:39.657 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.657 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.657 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.657 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.657 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.658 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96045 2023-10-01T09:24:39.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.658 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568148+0000", 2023-10-01T09:24:39.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568148+0000", 2023-10-01T09:24:39.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568147+0000", 2023-10-01T09:24:39.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:24:39.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568151+0000", 2023-10-01T09:24:39.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0049999999999998e-06 2023-10-01T09:24:39.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568161+0000", 2023-10-01T09:24:39.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0219999999999993e-06 2023-10-01T09:24:39.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568165+0000", 2023-10-01T09:24:39.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4880000000000001e-06 2023-10-01T09:24:39.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568209+0000", 2023-10-01T09:24:39.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4218999999999998e-05 2023-10-01T09:24:39.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:39.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568215+0000", 2023-10-01T09:24:39.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6579999999999999e-06 2023-10-01T09:24:39.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.533101+0000", 2023-10-01T09:24:39.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96488601799999996 2023-10-01T09:24:39.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.533256+0000", 2023-10-01T09:24:39.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000155496 2023-10-01T09:24:39.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:39.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.533361+0000", 2023-10-01T09:24:39.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000104149 2023-10-01T09:24:39.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537559+0000", 2023-10-01T09:24:39.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041986920000000004 2023-10-01T09:24:39.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537618+0000", 2023-10-01T09:24:39.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8689e-05 2023-10-01T09:24:39.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537790+0000", 2023-10-01T09:24:39.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000172199 2023-10-01T09:24:39.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537817+0000", 2023-10-01T09:24:39.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6942999999999999e-05 2023-10-01T09:24:39.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537840+0000", 2023-10-01T09:24:39.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3156999999999999e-05 2023-10-01T09:24:39.665 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.665 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.665 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.665 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96049 6.f 6:fa48c08b:::benchmark_data_smithi138_307751_object680:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-10-01T09:24:39.665 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:16.572299+0000", 2023-10-01T09:24:39.665 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 203.020427289, 2023-10-01T09:24:39.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97246542199999997, 2023-10-01T09:24:39.666 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.666 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.666 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.666 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.666 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.666 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96049 2023-10-01T09:24:39.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.666 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572299+0000", 2023-10-01T09:24:39.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572299+0000", 2023-10-01T09:24:39.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572297+0000", 2023-10-01T09:24:39.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:24:39.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572305+0000", 2023-10-01T09:24:39.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2169999999999994e-06 2023-10-01T09:24:39.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572324+0000", 2023-10-01T09:24:39.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8539999999999999e-05 2023-10-01T09:24:39.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572333+0000", 2023-10-01T09:24:39.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8179999999999993e-06 2023-10-01T09:24:39.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572424+0000", 2023-10-01T09:24:39.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0913000000000002e-05 2023-10-01T09:24:39.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:39.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572438+0000", 2023-10-01T09:24:39.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4127e-05 2023-10-01T09:24:39.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.535622+0000", 2023-10-01T09:24:39.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96318426000000001 2023-10-01T09:24:39.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.535741+0000", 2023-10-01T09:24:39.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011915499999999999 2023-10-01T09:24:39.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:39.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.535863+0000", 2023-10-01T09:24:39.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121599 2023-10-01T09:24:39.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537848+0000", 2023-10-01T09:24:39.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0019855110000000001 2023-10-01T09:24:39.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.541762+0000", 2023-10-01T09:24:39.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039140620000000003 2023-10-01T09:24:39.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544687+0000", 2023-10-01T09:24:39.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029246369999999999 2023-10-01T09:24:39.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544722+0000", 2023-10-01T09:24:39.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4660999999999999e-05 2023-10-01T09:24:39.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544765+0000", 2023-10-01T09:24:39.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3013000000000003e-05 2023-10-01T09:24:39.673 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.673 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.673 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.674 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96051 6.f 6:fad7202f:::benchmark_data_smithi138_307751_object682:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-10-01T09:24:39.674 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:16.572358+0000", 2023-10-01T09:24:39.674 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 203.020368792, 2023-10-01T09:24:39.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97833788099999996, 2023-10-01T09:24:39.674 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.674 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.674 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.674 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.674 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.674 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96051 2023-10-01T09:24:39.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.675 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572358+0000", 2023-10-01T09:24:39.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572358+0000", 2023-10-01T09:24:39.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572357+0000", 2023-10-01T09:24:39.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:24:39.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572360+0000", 2023-10-01T09:24:39.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0809999999999998e-06 2023-10-01T09:24:39.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572369+0000", 2023-10-01T09:24:39.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5790000000000004e-06 2023-10-01T09:24:39.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572375+0000", 2023-10-01T09:24:39.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.037e-06 2023-10-01T09:24:39.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572466+0000", 2023-10-01T09:24:39.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1169999999999996e-05 2023-10-01T09:24:39.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:39.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572477+0000", 2023-10-01T09:24:39.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0981e-05 2023-10-01T09:24:39.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537893+0000", 2023-10-01T09:24:39.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96541566499999998 2023-10-01T09:24:39.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.538017+0000", 2023-10-01T09:24:39.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000124315 2023-10-01T09:24:39.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:39.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.538129+0000", 2023-10-01T09:24:39.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112355 2023-10-01T09:24:39.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544653+0000", 2023-10-01T09:24:39.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0065235960000000004 2023-10-01T09:24:39.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544827+0000", 2023-10-01T09:24:39.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017388700000000001 2023-10-01T09:24:39.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.550629+0000", 2023-10-01T09:24:39.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058019609999999996 2023-10-01T09:24:39.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.550659+0000", 2023-10-01T09:24:39.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0156999999999999e-05 2023-10-01T09:24:39.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.550696+0000", 2023-10-01T09:24:39.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6636e-05 2023-10-01T09:24:39.682 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.682 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.682 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.682 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96072 6.f 6:fdc788ea:::benchmark_data_smithi138_307751_object703:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-10-01T09:24:39.682 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:16.589538+0000", 2023-10-01T09:24:39.682 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 203.00318884199999, 2023-10-01T09:24:39.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96461343300000002, 2023-10-01T09:24:39.683 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.683 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.683 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.683 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.683 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.683 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96072 2023-10-01T09:24:39.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.683 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589538+0000", 2023-10-01T09:24:39.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589538+0000", 2023-10-01T09:24:39.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589535+0000", 2023-10-01T09:24:39.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:24:39.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589544+0000", 2023-10-01T09:24:39.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5469999999999997e-06 2023-10-01T09:24:39.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589567+0000", 2023-10-01T09:24:39.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2816999999999999e-05 2023-10-01T09:24:39.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589578+0000", 2023-10-01T09:24:39.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1002000000000001e-05 2023-10-01T09:24:39.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589639+0000", 2023-10-01T09:24:39.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1264000000000002e-05 2023-10-01T09:24:39.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:39.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589654+0000", 2023-10-01T09:24:39.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5258e-05 2023-10-01T09:24:39.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.687 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.539758+0000", 2023-10-01T09:24:39.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95010384400000003 2023-10-01T09:24:39.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.687 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.539881+0000", 2023-10-01T09:24:39.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012311900000000001 2023-10-01T09:24:39.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:39.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.540000+0000", 2023-10-01T09:24:39.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011898800000000001 2023-10-01T09:24:39.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544837+0000", 2023-10-01T09:24:39.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048369520000000003 2023-10-01T09:24:39.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.547696+0000", 2023-10-01T09:24:39.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002858972 2023-10-01T09:24:39.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.554057+0000", 2023-10-01T09:24:39.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0063608279999999998 2023-10-01T09:24:39.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.554113+0000", 2023-10-01T09:24:39.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5835999999999999e-05 2023-10-01T09:24:39.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.554151+0000", 2023-10-01T09:24:39.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8349000000000002e-05 2023-10-01T09:24:39.690 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.690 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.690 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.690 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96108 6.f 6:ff875b58:::benchmark_data_smithi138_307751_object739:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-10-01T09:24:39.690 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:16.644947+0000", 2023-10-01T09:24:39.690 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 202.947779889, 2023-10-01T09:24:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91276201599999995, 2023-10-01T09:24:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96108 2023-10-01T09:24:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.644947+0000", 2023-10-01T09:24:39.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.644947+0000", 2023-10-01T09:24:39.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.644945+0000", 2023-10-01T09:24:39.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:24:39.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.644954+0000", 2023-10-01T09:24:39.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7639999999999994e-06 2023-10-01T09:24:39.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.644971+0000", 2023-10-01T09:24:39.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7864999999999999e-05 2023-10-01T09:24:39.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.644981+0000", 2023-10-01T09:24:39.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5440000000000004e-06 2023-10-01T09:24:39.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.645035+0000", 2023-10-01T09:24:39.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4039000000000002e-05 2023-10-01T09:24:39.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:39.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.645047+0000", 2023-10-01T09:24:39.695 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2466000000000001e-05 2023-10-01T09:24:39.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.695 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.541802+0000", 2023-10-01T09:24:39.695 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.896754469 2023-10-01T09:24:39.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.695 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.541880+0000", 2023-10-01T09:24:39.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7942999999999996e-05 2023-10-01T09:24:39.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:39.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.541942+0000", 2023-10-01T09:24:39.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2563000000000004e-05 2023-10-01T09:24:39.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544847+0000", 2023-10-01T09:24:39.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029042159999999998 2023-10-01T09:24:39.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.557585+0000", 2023-10-01T09:24:39.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012738289 2023-10-01T09:24:39.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.557658+0000", 2023-10-01T09:24:39.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2776999999999998e-05 2023-10-01T09:24:39.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.557686+0000", 2023-10-01T09:24:39.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8031999999999999e-05 2023-10-01T09:24:39.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.557709+0000", 2023-10-01T09:24:39.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2909e-05 2023-10-01T09:24:39.698 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.698 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.698 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.699 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109321 6.2 6:4665d031:::benchmark_data_smithi138_307751_object13952:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-01T09:24:39.699 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.601615+0000", 2023-10-01T09:24:39.699 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 179.99111114199999, 2023-10-01T09:24:39.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97938183099999998, 2023-10-01T09:24:39.699 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.699 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:24:39.699 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.699 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.699 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.699 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109321 2023-10-01T09:24:39.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.700 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601615+0000", 2023-10-01T09:24:39.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601615+0000", 2023-10-01T09:24:39.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601613+0000", 2023-10-01T09:24:39.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:24:39.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601623+0000", 2023-10-01T09:24:39.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3850000000000008e-06 2023-10-01T09:24:39.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601643+0000", 2023-10-01T09:24:39.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0353e-05 2023-10-01T09:24:39.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601654+0000", 2023-10-01T09:24:39.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1279e-05 2023-10-01T09:24:39.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.823938+0000", 2023-10-01T09:24:39.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.222283589 2023-10-01T09:24:39.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:39.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.823948+0000", 2023-10-01T09:24:39.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7529999999999996e-06 2023-10-01T09:24:39.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.580927+0000", 2023-10-01T09:24:39.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.75697944500000003 2023-10-01T09:24:39.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.580997+0000", 2023-10-01T09:24:39.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0178000000000005e-05 2023-10-01T09:24:39.704 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.704 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.704 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.704 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109328 6.12 6:4e4d45a1:::benchmark_data_smithi138_307751_object13959:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-01T09:24:39.704 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.601663+0000", 2023-10-01T09:24:39.705 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 179.991063854, 2023-10-01T09:24:39.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0018865050000001, 2023-10-01T09:24:39.705 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.705 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.705 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.705 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.705 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.705 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109328 2023-10-01T09:24:39.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.705 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601663+0000", 2023-10-01T09:24:39.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601663+0000", 2023-10-01T09:24:39.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601662+0000", 2023-10-01T09:24:39.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:24:39.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601665+0000", 2023-10-01T09:24:39.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.083e-06 2023-10-01T09:24:39.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601673+0000", 2023-10-01T09:24:39.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7870000000000002e-06 2023-10-01T09:24:39.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601679+0000", 2023-10-01T09:24:39.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.519e-06 2023-10-01T09:24:39.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.823969+0000", 2023-10-01T09:24:39.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.222290083 2023-10-01T09:24:39.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:39.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.823974+0000", 2023-10-01T09:24:39.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.626e-06 2023-10-01T09:24:39.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.594859+0000", 2023-10-01T09:24:39.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.77088483600000002 2023-10-01T09:24:39.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.594988+0000", 2023-10-01T09:24:39.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012879300000000001 2023-10-01T09:24:39.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:39.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.595107+0000", 2023-10-01T09:24:39.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118781 2023-10-01T09:24:39.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.600053+0000", 2023-10-01T09:24:39.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00494627 2023-10-01T09:24:39.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.602542+0000", 2023-10-01T09:24:39.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024894000000000001 2023-10-01T09:24:39.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.603447+0000", 2023-10-01T09:24:39.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00090488299999999999 2023-10-01T09:24:39.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.603501+0000", 2023-10-01T09:24:39.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4141000000000002e-05 2023-10-01T09:24:39.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.603549+0000", 2023-10-01T09:24:39.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7805000000000001e-05 2023-10-01T09:24:39.713 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.713 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.713 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.713 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109334 6.2 6:431726c9:::benchmark_data_smithi138_307751_object13965:head [delete] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-10-01T09:24:39.713 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.641618+0000", 2023-10-01T09:24:39.713 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 179.95110831900001, 2023-10-01T09:24:39.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95538988400000002, 2023-10-01T09:24:39.713 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.714 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.714 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.714 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.714 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.714 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109334 2023-10-01T09:24:39.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.714 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.641618+0000", 2023-10-01T09:24:39.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.641618+0000", 2023-10-01T09:24:39.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.641616+0000", 2023-10-01T09:24:39.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999971 2023-10-01T09:24:39.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.641631+0000", 2023-10-01T09:24:39.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5430999999999998e-05 2023-10-01T09:24:39.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.641661+0000", 2023-10-01T09:24:39.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0103e-05 2023-10-01T09:24:39.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.641677+0000", 2023-10-01T09:24:39.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5546999999999999e-05 2023-10-01T09:24:39.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.823994+0000", 2023-10-01T09:24:39.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18231736500000001 2023-10-01T09:24:39.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:39.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.824001+0000", 2023-10-01T09:24:39.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6989999999999997e-06 2023-10-01T09:24:39.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.584131+0000", 2023-10-01T09:24:39.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.76013001700000005 2023-10-01T09:24:39.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.584664+0000", 2023-10-01T09:24:39.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00053300799999999996 2023-10-01T09:24:39.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:24:39.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.584771+0000", 2023-10-01T09:24:39.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107014 2023-10-01T09:24:39.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.596837+0000", 2023-10-01T09:24:39.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01206572 2023-10-01T09:24:39.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.720 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.720 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.596902+0000", 2023-10-01T09:24:39.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5140000000000003e-05 2023-10-01T09:24:39.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.720 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.720 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.720 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.596931+0000", 2023-10-01T09:24:39.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9320999999999999e-05 2023-10-01T09:24:39.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.720 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.720 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.596974+0000", 2023-10-01T09:24:39.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3328000000000003e-05 2023-10-01T09:24:39.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.597008+0000", 2023-10-01T09:24:39.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3827e-05 2023-10-01T09:24:39.721 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.721 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.721 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.722 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178787 6.18 6:189b3dbf:::benchmark_data_smithi138_307751_object83418:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:24:39.722 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612642+0000", 2023-10-01T09:24:39.722 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 88.980084055999995, 2023-10-01T09:24:39.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4263684350000001, 2023-10-01T09:24:39.722 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.722 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.722 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.722 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.722 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.723 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178787 2023-10-01T09:24:39.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.723 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612642+0000", 2023-10-01T09:24:39.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612642+0000", 2023-10-01T09:24:39.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612641+0000", 2023-10-01T09:24:39.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:24:39.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612647+0000", 2023-10-01T09:24:39.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5709999999999997e-06 2023-10-01T09:24:39.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612658+0000", 2023-10-01T09:24:39.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1528e-05 2023-10-01T09:24:39.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612668+0000", 2023-10-01T09:24:39.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0349999999999999e-05 2023-10-01T09:24:39.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614052+0000", 2023-10-01T09:24:39.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001383188 2023-10-01T09:24:39.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:39.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614062+0000", 2023-10-01T09:24:39.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0437e-05 2023-10-01T09:24:39.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.028763+0000", 2023-10-01T09:24:39.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.414701177 2023-10-01T09:24:39.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.028872+0000", 2023-10-01T09:24:39.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109073 2023-10-01T09:24:39.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:24:39.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.029307+0000", 2023-10-01T09:24:39.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00043503699999999999 2023-10-01T09:24:39.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.032478+0000", 2023-10-01T09:24:39.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031704839999999999 2023-10-01T09:24:39.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.032534+0000", 2023-10-01T09:24:39.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6481999999999999e-05 2023-10-01T09:24:39.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.038405+0000", 2023-10-01T09:24:39.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058706319999999998 2023-10-01T09:24:39.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.038465+0000", 2023-10-01T09:24:39.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0176000000000001e-05 2023-10-01T09:24:39.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039011+0000", 2023-10-01T09:24:39.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00054578000000000005 2023-10-01T09:24:39.730 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.730 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.730 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.730 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178807 6.18 6:1f8d531c:::benchmark_data_smithi138_307751_object83438:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:24:39.731 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612681+0000", 2023-10-01T09:24:39.731 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 88.980045977000003, 2023-10-01T09:24:39.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4300119259999999, 2023-10-01T09:24:39.731 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.731 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.731 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.731 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.731 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.731 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178807 2023-10-01T09:24:39.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.731 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612681+0000", 2023-10-01T09:24:39.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612681+0000", 2023-10-01T09:24:39.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612680+0000", 2023-10-01T09:24:39.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:24:39.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612683+0000", 2023-10-01T09:24:39.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0709999999999999e-06 2023-10-01T09:24:39.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612690+0000", 2023-10-01T09:24:39.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.613e-06 2023-10-01T09:24:39.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612695+0000", 2023-10-01T09:24:39.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4090000000000002e-06 2023-10-01T09:24:39.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614084+0000", 2023-10-01T09:24:39.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001388331 2023-10-01T09:24:39.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:39.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614090+0000", 2023-10-01T09:24:39.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8630000000000002e-06 2023-10-01T09:24:39.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.032562+0000", 2023-10-01T09:24:39.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.41847178 2023-10-01T09:24:39.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.032737+0000", 2023-10-01T09:24:39.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000174477 2023-10-01T09:24:39.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:24:39.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.033304+0000", 2023-10-01T09:24:39.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00056714600000000003 2023-10-01T09:24:39.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039063+0000", 2023-10-01T09:24:39.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057587920000000004 2023-10-01T09:24:39.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039071+0000", 2023-10-01T09:24:39.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5920000000000001e-06 2023-10-01T09:24:39.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042139+0000", 2023-10-01T09:24:39.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030678440000000001 2023-10-01T09:24:39.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042194+0000", 2023-10-01T09:24:39.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5306999999999997e-05 2023-10-01T09:24:39.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042693+0000", 2023-10-01T09:24:39.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00049818099999999999 2023-10-01T09:24:39.738 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.738 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.738 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.739 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178818 6.18 6:19fb958b:::benchmark_data_smithi138_307751_object83449:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:24:39.739 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612700+0000", 2023-10-01T09:24:39.739 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 88.980026326000001, 2023-10-01T09:24:39.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.430431437, 2023-10-01T09:24:39.739 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.739 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.739 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.739 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.739 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.740 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178818 2023-10-01T09:24:39.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.740 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612700+0000", 2023-10-01T09:24:39.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612700+0000", 2023-10-01T09:24:39.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612700+0000", 2023-10-01T09:24:39.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:24:39.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612703+0000", 2023-10-01T09:24:39.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7920000000000001e-06 2023-10-01T09:24:39.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612708+0000", 2023-10-01T09:24:39.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4990000000000002e-06 2023-10-01T09:24:39.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612713+0000", 2023-10-01T09:24:39.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8509999999999998e-06 2023-10-01T09:24:39.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614110+0000", 2023-10-01T09:24:39.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013967039999999999 2023-10-01T09:24:39.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:39.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614116+0000", 2023-10-01T09:24:39.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9349999999999999e-06 2023-10-01T09:24:39.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.035619+0000", 2023-10-01T09:24:39.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.421503172 2023-10-01T09:24:39.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.035767+0000", 2023-10-01T09:24:39.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014777900000000001 2023-10-01T09:24:39.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:24:39.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.036191+0000", 2023-10-01T09:24:39.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000424206 2023-10-01T09:24:39.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042058+0000", 2023-10-01T09:24:39.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005867227 2023-10-01T09:24:39.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042070+0000", 2023-10-01T09:24:39.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2122e-05 2023-10-01T09:24:39.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042747+0000", 2023-10-01T09:24:39.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00067660799999999998 2023-10-01T09:24:39.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.746 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042782+0000", 2023-10-01T09:24:39.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5441000000000002e-05 2023-10-01T09:24:39.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.746 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043132+0000", 2023-10-01T09:24:39.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00034945799999999997 2023-10-01T09:24:39.747 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.747 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.747 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.747 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178848 6.18 6:1e688e02:::benchmark_data_smithi138_307751_object83479:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:24:39.747 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612851+0000", 2023-10-01T09:24:39.747 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 88.979875774000007, 2023-10-01T09:24:39.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.436053759, 2023-10-01T09:24:39.747 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.747 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.747 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.748 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.748 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.748 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178848 2023-10-01T09:24:39.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.748 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612851+0000", 2023-10-01T09:24:39.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612851+0000", 2023-10-01T09:24:39.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612850+0000", 2023-10-01T09:24:39.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:24:39.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612854+0000", 2023-10-01T09:24:39.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5580000000000001e-06 2023-10-01T09:24:39.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612862+0000", 2023-10-01T09:24:39.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9699999999999999e-06 2023-10-01T09:24:39.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612869+0000", 2023-10-01T09:24:39.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0539999999999997e-06 2023-10-01T09:24:39.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614135+0000", 2023-10-01T09:24:39.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012659559999999999 2023-10-01T09:24:39.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:39.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614141+0000", 2023-10-01T09:24:39.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2859999999999996e-06 2023-10-01T09:24:39.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039094+0000", 2023-10-01T09:24:39.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.424953428 2023-10-01T09:24:39.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039262+0000", 2023-10-01T09:24:39.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000167541 2023-10-01T09:24:39.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:24:39.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039703+0000", 2023-10-01T09:24:39.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000441462 2023-10-01T09:24:39.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043192+0000", 2023-10-01T09:24:39.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034885319999999999 2023-10-01T09:24:39.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043218+0000", 2023-10-01T09:24:39.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5754000000000001e-05 2023-10-01T09:24:39.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.048340+0000", 2023-10-01T09:24:39.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051218219999999998 2023-10-01T09:24:39.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.048383+0000", 2023-10-01T09:24:39.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3179000000000001e-05 2023-10-01T09:24:39.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.048905+0000", 2023-10-01T09:24:39.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00052180900000000003 2023-10-01T09:24:39.755 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.755 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.755 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.756 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178861 6.18 6:1fa58ed3:::benchmark_data_smithi138_307751_object83492:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:24:39.756 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612874+0000", 2023-10-01T09:24:39.756 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 88.979852659000002, 2023-10-01T09:24:39.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4449525510000001, 2023-10-01T09:24:39.756 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:24:39.756 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:24:39.756 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:24:39.756 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:24:39.756 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:24:39.757 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178861 2023-10-01T09:24:39.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.757 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:24:39.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:24:39.757 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612874+0000", 2023-10-01T09:24:39.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:24:39.757 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612874+0000", 2023-10-01T09:24:39.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:24:39.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:24:39.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612874+0000", 2023-10-01T09:24:39.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:24:39.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:24:39.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612876+0000", 2023-10-01T09:24:39.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4380000000000002e-06 2023-10-01T09:24:39.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.759 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:24:39.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612882+0000", 2023-10-01T09:24:39.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1179999999999997e-06 2023-10-01T09:24:39.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.759 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:24:39.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612887+0000", 2023-10-01T09:24:39.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0300000000000001e-06 2023-10-01T09:24:39.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614161+0000", 2023-10-01T09:24:39.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012741460000000001 2023-10-01T09:24:39.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:24:39.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614167+0000", 2023-10-01T09:24:39.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6629999999999998e-06 2023-10-01T09:24:39.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:24:39.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043249+0000", 2023-10-01T09:24:39.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.429082395 2023-10-01T09:24:39.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:24:39.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043418+0000", 2023-10-01T09:24:39.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016889300000000001 2023-10-01T09:24:39.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:24:39.761 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043866+0000", 2023-10-01T09:24:39.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00044823600000000002 2023-10-01T09:24:39.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057128+0000", 2023-10-01T09:24:39.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013261854 2023-10-01T09:24:39.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:24:39.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057142+0000", 2023-10-01T09:24:39.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3743e-05 2023-10-01T09:24:39.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:24:39.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057244+0000", 2023-10-01T09:24:39.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010159599999999999 2023-10-01T09:24:39.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:24:39.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057298+0000", 2023-10-01T09:24:39.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.41e-05 2023-10-01T09:24:39.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:24:39.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:24:39.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:24:39.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057826+0000", 2023-10-01T09:24:39.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00052871700000000005 2023-10-01T09:24:39.764 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.764 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.764 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.764 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:24:39.764 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:24:39.764 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:24:39.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:39.765 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:24:39.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:24:39.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:39 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:24:39.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:39 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:24:39.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:39 smithi138 ceph-mon[192513]: osdmap e266: 8 total, 8 up, 8 in 2023-10-01T09:24:39.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:39 smithi138 ceph-mon[192513]: failed to encode map e266 with expected crc 2023-10-01T09:24:39.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:39 smithi138 ceph-mon[192513]: failed to encode map e266 with expected crc 2023-10-01T09:24:39.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:39 smithi138 ceph-mon[192513]: failed to encode map e266 with expected crc 2023-10-01T09:24:39.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:39 smithi138 ceph-mon[192513]: failed to encode map e266 with expected crc 2023-10-01T09:24:39.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:39 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:24:39.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:39 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:24:39.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:39 smithi138 ceph-mon[185886]: osdmap e266: 8 total, 8 up, 8 in 2023-10-01T09:24:39.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:39 smithi138 ceph-mon[185886]: failed to encode map e266 with expected crc 2023-10-01T09:24:39.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:39 smithi138 ceph-mon[185886]: failed to encode map e266 with expected crc 2023-10-01T09:24:39.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:39 smithi138 ceph-mon[185886]: failed to encode map e266 with expected crc 2023-10-01T09:24:39.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:39 smithi138 ceph-mon[185886]: failed to encode map e266 with expected crc 2023-10-01T09:24:39.867 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:24:39.897 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 46 16 63548 63532 86.2747 66.25 0.0141971 0.011585 2023-10-01T09:24:39.954 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:40.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:39 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:24:40.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:39 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:24:40.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:39 smithi190 ceph-mon[160435]: osdmap e266: 8 total, 8 up, 8 in 2023-10-01T09:24:40.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:39 smithi190 ceph-mon[160435]: failed to encode map e266 with expected crc 2023-10-01T09:24:40.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:39 smithi190 ceph-mon[160435]: failed to encode map e266 with expected crc 2023-10-01T09:24:40.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:39 smithi190 ceph-mon[160435]: failed to encode map e266 with expected crc 2023-10-01T09:24:40.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:39 smithi190 ceph-mon[160435]: failed to encode map e266 with expected crc 2023-10-01T09:24:40.673 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:24:40.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:40.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:24:40.777 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:24:40.804 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:40.898 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 47 16 64374 64358 85.5376 51.625 0.00312739 0.0116492 2023-10-01T09:24:41.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:40 smithi190 ceph-mon[160435]: failed to encode map e266 with expected crc 2023-10-01T09:24:41.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:40 smithi190 ceph-mon[160435]: failed to encode map e266 with expected crc 2023-10-01T09:24:41.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:40 smithi190 ceph-mon[160435]: pgmap v686: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-01T09:24:41.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:24:41.061 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:41.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:40 smithi138 ceph-mon[192513]: failed to encode map e266 with expected crc 2023-10-01T09:24:41.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:40 smithi138 ceph-mon[192513]: failed to encode map e266 with expected crc 2023-10-01T09:24:41.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:40 smithi138 ceph-mon[192513]: pgmap v686: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-01T09:24:41.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:24:41.073 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:41.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:40 smithi138 ceph-mon[185886]: failed to encode map e266 with expected crc 2023-10-01T09:24:41.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:40 smithi138 ceph-mon[185886]: failed to encode map e266 with expected crc 2023-10-01T09:24:41.082 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:40 smithi138 ceph-mon[185886]: pgmap v686: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-01T09:24:41.083 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:24:41.237 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:24:41.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:41.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:24:41.340 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:24:41.345 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:41.382 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:41.502 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:41.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:41.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:41.608 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:24:41.659 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:24:41.732 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:41.882 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:24:41.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:41.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:24:41.892 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:24:41.897 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:24:41.902 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 48 16 65302 65286 84.9557 58 0.0198256 0.0117656 2023-10-01T09:24:41.993 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:42.062 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:42.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:42 smithi138 ceph-mon[185886]: pgmap v687: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-01T09:24:42.905 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 49 16 66076 66060 84.2057 48.375 0.0159936 0.0118701 2023-10-01T09:24:42.905 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:42.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:42.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:42.988 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:42 smithi138 ceph-mon[192513]: pgmap v687: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-01T09:24:43.011 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:43.050 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:42 smithi190 ceph-mon[160435]: pgmap v687: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2023-10-01T09:24:43.053 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:43.110 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:24:43.159 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:43.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:43.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:43.265 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:24:43.306 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:43.476 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:24:43.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:43.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:24:43.582 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:43.608 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:43.718 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:43.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:43.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:43.822 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:43.867 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:43.906 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 50 16 66903 66887 83.5534 51.6875 0.0109771 0.0119635 2023-10-01T09:24:43.986 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:43.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:43.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:44.091 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:44.155 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:44.196 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:44.639 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:44.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:44.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:44.746 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:44.767 INFO:teuthology.orchestra.run.smithi138.stdout:moved 0 / 267 (0%) 2023-10-01T09:24:44.767 INFO:teuthology.orchestra.run.smithi138.stdout:avg 33.375 2023-10-01T09:24:44.767 INFO:teuthology.orchestra.run.smithi138.stdout:stddev 5.17053 -> 5.17053 (expected baseline 5.40399) 2023-10-01T09:24:44.767 INFO:teuthology.orchestra.run.smithi138.stdout:min osd.2 with 25 -> 25 pgs (0.749064 -> 0.749064 * mean) 2023-10-01T09:24:44.768 INFO:teuthology.orchestra.run.smithi138.stdout:max osd.3 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-10-01T09:24:44.768 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:24:44.768 INFO:teuthology.orchestra.run.smithi138.stdout:oload 1000 2023-10-01T09:24:44.768 INFO:teuthology.orchestra.run.smithi138.stdout:max_change 3 2023-10-01T09:24:44.768 INFO:teuthology.orchestra.run.smithi138.stdout:max_change_osds 4 2023-10-01T09:24:44.768 INFO:teuthology.orchestra.run.smithi138.stdout:average_utilization 0.0200 2023-10-01T09:24:44.768 INFO:teuthology.orchestra.run.smithi138.stdout:overload_utilization 0.1998 2023-10-01T09:24:44.793 INFO:teuthology.orchestra.run.smithi138.stderr:no change 2023-10-01T09:24:44.851 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:44.906 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 51 16 67734 67718 82.9337 51.9375 0.00803939 0.0120516 2023-10-01T09:24:44.938 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:44 smithi138 ceph-mon[185886]: pgmap v688: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-01T09:24:44.938 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:44 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:44.938 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:44 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:44.939 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:24:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:24:44] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.33.4" 2023-10-01T09:24:44.939 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:44 smithi138 ceph-mon[192513]: pgmap v688: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-01T09:24:44.939 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:44 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:44.939 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:44 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:44.962 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:45.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:44 smithi190 ceph-mon[160435]: pgmap v688: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2023-10-01T09:24:45.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:44 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:45.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:44 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:45.076 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:45.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:45.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:45.091 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:24:45.139 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:24:45.176 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:24:45.187 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:24:45.222 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:24:45.254 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:24:45.258 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:45.286 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:24:45.321 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:24:45.365 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:24:45.380 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:24:45.380 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:24:45.380 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:24:45.381 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:24:45.381 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:24:45.381 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:24:45.381 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:24:45.381 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:24:45.906 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 52 16 68725 68709 82.5301 61.9375 0.0137589 0.0121113 2023-10-01T09:24:46.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:45 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4093166164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:46.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:45 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4093166164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:46.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:45 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4093166164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:46.550 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:24:46.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:46.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:24:46.662 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:46.696 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:46.758 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:46.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:46.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:46.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:46 smithi138 ceph-mon[192513]: from='client.45410 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:24:46.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:46 smithi138 ceph-mon[192513]: pgmap v689: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-01T09:24:46.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:46 smithi138 ceph-mon[185886]: from='client.45410 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:24:46.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:46 smithi138 ceph-mon[185886]: pgmap v689: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-01T09:24:46.864 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:24:46.895 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:46.913 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 53 16 69658 69642 82.0637 58.3125 0.0183177 0.01218 2023-10-01T09:24:46.956 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:24:46.960 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:46 smithi190 ceph-mon[160435]: from='client.45410 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:24:46.960 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:46 smithi190 ceph-mon[160435]: pgmap v689: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-01T09:24:46.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:46.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:24:47.003 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-01T09:24:47.003 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-10-01T09:24:47.062 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:24:47.154 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:47.915 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 54 16 70650 70634 81.6894 62 0.0232943 0.0122364 2023-10-01T09:24:48.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:48 smithi190 ceph-mon[160435]: pgmap v690: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s 2023-10-01T09:24:48.561 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:48 smithi138 ceph-mon[192513]: pgmap v690: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s 2023-10-01T09:24:48.562 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:48 smithi138 ceph-mon[185886]: pgmap v690: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s 2023-10-01T09:24:48.714 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:24:48.714 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:24:48.742 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:24:48.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:48.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:24:48.848 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:48.877 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:48.915 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 55 15 71461 71446 81.1271 50.75 0.00623215 0.0123201 2023-10-01T09:24:49.048 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:49.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:49.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:49.161 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:24:49.196 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:49.692 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:24:49.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:49.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:24:49.801 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:49.819 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:49.825 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:24:49.831 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:49.886 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:49.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:49.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:49.915 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 56 16 72201 72185 80.5035 46.1875 0.0181106 0.0124162 2023-10-01T09:24:49.993 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:50.104 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:50.146 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:24:50.245 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:50.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:50.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:50.354 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:50.474 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:50.723 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:50 smithi138 ceph-mon[192513]: pgmap v691: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-01T09:24:50.723 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:50 smithi138 ceph-mon[185886]: pgmap v691: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-01T09:24:50.797 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:50.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:50.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:50.903 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:50.922 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 57 16 72969 72953 79.923 48 0.0252972 0.0125067 2023-10-01T09:24:50.959 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:51.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:50 smithi190 ceph-mon[160435]: pgmap v691: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-01T09:24:51.638 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:24:51.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:51 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2101157862' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:24:51.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:51 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2101157862' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:24:51.922 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 58 16 73922 73906 79.5721 59.5625 0.0129865 0.0125626 2023-10-01T09:24:51.972 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:51.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:51.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:52.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:51 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2101157862' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:24:52.084 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:52.114 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:52.179 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:52.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:52.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:52.284 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:24:52.311 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:52.743 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:24:52.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:52.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:24:52.857 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:52.912 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[192513]: pgmap v692: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s 2023-10-01T09:24:52.912 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:24:52.912 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[192513]: failed to encode map e267 with expected crc 2023-10-01T09:24:52.912 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2101157862' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:24:52.913 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[192513]: osdmap e267: 8 total, 8 up, 8 in 2023-10-01T09:24:52.913 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[192513]: failed to encode map e267 with expected crc 2023-10-01T09:24:52.913 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[192513]: failed to encode map e267 with expected crc 2023-10-01T09:24:52.913 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[192513]: failed to encode map e267 with expected crc 2023-10-01T09:24:52.913 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[185886]: pgmap v692: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s 2023-10-01T09:24:52.913 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:24:52.913 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[185886]: failed to encode map e267 with expected crc 2023-10-01T09:24:52.913 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2101157862' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:24:52.913 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[185886]: osdmap e267: 8 total, 8 up, 8 in 2023-10-01T09:24:52.913 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[185886]: failed to encode map e267 with expected crc 2023-10-01T09:24:52.914 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[185886]: failed to encode map e267 with expected crc 2023-10-01T09:24:52.914 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:52 smithi138 ceph-mon[185886]: failed to encode map e267 with expected crc 2023-10-01T09:24:52.929 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 59 15 74737 74722 79.0801 51 0.0416221 0.0126381 2023-10-01T09:24:52.976 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:52 smithi190 ceph-mon[160435]: pgmap v692: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s 2023-10-01T09:24:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:52 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:24:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:52 smithi190 ceph-mon[160435]: failed to encode map e267 with expected crc 2023-10-01T09:24:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:52 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2101157862' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:24:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:52 smithi190 ceph-mon[160435]: osdmap e267: 8 total, 8 up, 8 in 2023-10-01T09:24:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:52 smithi190 ceph-mon[160435]: failed to encode map e267 with expected crc 2023-10-01T09:24:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:52 smithi190 ceph-mon[160435]: failed to encode map e267 with expected crc 2023-10-01T09:24:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:52 smithi190 ceph-mon[160435]: failed to encode map e267 with expected crc 2023-10-01T09:24:53.168 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:53.224 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:53.385 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:53.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:53.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:53.494 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:24:53.527 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:53.586 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:24:53.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:53.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:24:53.679 INFO:teuthology.orchestra.run.smithi138.stdout:moved 0 / 267 (0%) 2023-10-01T09:24:53.679 INFO:teuthology.orchestra.run.smithi138.stdout:avg 33.375 2023-10-01T09:24:53.679 INFO:teuthology.orchestra.run.smithi138.stdout:stddev 5.17053 -> 5.17053 (expected baseline 5.40399) 2023-10-01T09:24:53.679 INFO:teuthology.orchestra.run.smithi138.stdout:min osd.2 with 25 -> 25 pgs (0.749064 -> 0.749064 * mean) 2023-10-01T09:24:53.679 INFO:teuthology.orchestra.run.smithi138.stdout:max osd.3 with 41 -> 41 pgs (1.22846 -> 1.22846 * mean) 2023-10-01T09:24:53.679 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:24:53.679 INFO:teuthology.orchestra.run.smithi138.stdout:oload 1000 2023-10-01T09:24:53.679 INFO:teuthology.orchestra.run.smithi138.stdout:max_change 3 2023-10-01T09:24:53.679 INFO:teuthology.orchestra.run.smithi138.stdout:max_change_osds 4 2023-10-01T09:24:53.680 INFO:teuthology.orchestra.run.smithi138.stdout:average_utilization 0.0222 2023-10-01T09:24:53.680 INFO:teuthology.orchestra.run.smithi138.stdout:overload_utilization 0.2225 2023-10-01T09:24:53.687 INFO:teuthology.orchestra.run.smithi138.stderr:no change 2023-10-01T09:24:53.693 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:53.723 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:53.779 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:53.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:53.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:53.885 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:24:53.916 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:53.929 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:24:53.929329+0000 min lat: 0.0011387 max lat: 0.332925 avg lat: 0.0127102 2023-10-01T09:24:53.929 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:24:53.929 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 60 16 75588 75572 78.6478 53.125 0.00956105 0.0127102 2023-10-01T09:24:53.976 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:24:53.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:53.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:24:54.085 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:54.163 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:54.759 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:24:54.930 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 61 16 76392 76376 78.1816 50.25 0.0161216 0.0127861 2023-10-01T09:24:54.937 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:54 smithi138 ceph-mon[192513]: pgmap v694: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2023-10-01T09:24:54.937 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:54 smithi138 ceph-mon[192513]: from='client.35472 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:24:54.937 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:54 smithi138 ceph-mon[185886]: pgmap v694: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2023-10-01T09:24:54.937 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:54 smithi138 ceph-mon[185886]: from='client.35472 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:24:55.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:54 smithi190 ceph-mon[160435]: pgmap v694: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s 2023-10-01T09:24:55.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:54 smithi190 ceph-mon[160435]: from='client.35472 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:24:55.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:24:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:24:54] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.33.4" 2023-10-01T09:24:55.548 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:55.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:55.653 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:55.681 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:55.713 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-01T09:24:55.713 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-10-01T09:24:55.736 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:55.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:55.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:24:55.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:24:55.842 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:55.930 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 62 16 77100 77084 77.6347 44.25 0.00762486 0.0128748 2023-10-01T09:24:55.992 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:56.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:24:56.532 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:56.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:56.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:56.598 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:24:56.598 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:24:56.639 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:56.716 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:56.751 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:56.806 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:56.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:56.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:56 smithi138 ceph-mon[185886]: pgmap v695: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-01T09:24:56.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:56.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:56.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:56 smithi138 ceph-mon[192513]: pgmap v695: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-01T09:24:56.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:56.813 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:56.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:56.915 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:24:56.936 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 63 16 77853 77837 77.1497 47.0625 0.0145254 0.0129573 2023-10-01T09:24:56.979 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:57.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:56 smithi190 ceph-mon[160435]: pgmap v695: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-10-01T09:24:57.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:57.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:57.605 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:24:57.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:57.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:24:57.730 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:24:57.769 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:57.930 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 64 16 78596 78580 76.6701 46.4375 0.0257378 0.0130381 2023-10-01T09:24:58.166 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:24:58.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:58.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:24:58.274 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:24:58.310 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:24:58.343 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:58.447 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:24:58.448 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:24:58.448 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:24:58.448 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:58 smithi138 ceph-mon[185886]: pgmap v696: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2023-10-01T09:24:58.448 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:58.448 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:24:58.448 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:24:58.448 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:58 smithi138 ceph-mon[192513]: pgmap v696: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2023-10-01T09:24:58.448 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:58.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:24:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:24:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:58 smithi190 ceph-mon[160435]: pgmap v696: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2023-10-01T09:24:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:24:58.744 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:24:58.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:58.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:24:58.866 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:58.899 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:58.930 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 65 16 79387 79371 76.2514 49.4375 0.0245067 0.0131075 2023-10-01T09:24:58.951 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:24:58.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:58.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:24:59.056 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:24:59.089 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:59.146 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:24:59.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:59.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:24:59.253 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:24:59.281 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:24:59.669 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:24:59.710 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:24:59.740 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:24:59.796 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:24:59.830 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:24:59.853 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:24:59.863 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:24:59.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:24:59.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:24:59.923 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:24:59.931 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 66 16 80219 80203 75.8824 52 0.0168949 0.0131728 2023-10-01T09:24:59.955 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:24:59.960 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:24:59.960 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:24:59.960 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:24:59.960 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:24:59.961 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:24:59.961 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:24:59.961 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:24:59.961 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:24:59.966 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:24:59.987 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:24:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3381649660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:59.990 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:24:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3381649660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:59.993 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:24:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3381649660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:24:59.995 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:00.054 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:00.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:00.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:00.159 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:00.195 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:00.267 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:00.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:00.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:00.377 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:00.451 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:00.932 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 67 16 81026 81010 75.5028 50.4375 0.0129817 0.0132387 2023-10-01T09:25:00.999 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:00 smithi138 ceph-mon[185886]: pgmap v697: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s 2023-10-01T09:25:01.000 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:00 smithi138 ceph-mon[192513]: pgmap v697: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s 2023-10-01T09:25:01.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:00 smithi190 ceph-mon[160435]: pgmap v697: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s 2023-10-01T09:25:01.554 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:01.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:01.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:01.661 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:01.689 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:01.713 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:01.740 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:01.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:01.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:01.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:01 smithi138 ceph-mon[185886]: pgmap v698: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2023-10-01T09:25:01.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:01 smithi138 ceph-mon[192513]: pgmap v698: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2023-10-01T09:25:01.846 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:25:01.874 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:01.929 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:25:01.932 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 68 16 81803 81787 75.1069 48.5625 0.018173 0.0133092 2023-10-01T09:25:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:01.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:25:02.034 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:02.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:01 smithi190 ceph-mon[160435]: pgmap v698: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2023-10-01T09:25:02.063 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:02.118 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:02.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:02.188 INFO:teuthology.orchestra.run.smithi138.stdout:moved 2 / 267 (0.749064%) 2023-10-01T09:25:02.189 INFO:teuthology.orchestra.run.smithi138.stdout:avg 33.375 2023-10-01T09:25:02.189 INFO:teuthology.orchestra.run.smithi138.stdout:stddev 5.17053 -> 5.21866 (expected baseline 5.40399) 2023-10-01T09:25:02.189 INFO:teuthology.orchestra.run.smithi138.stdout:min osd.2 with 25 -> 25 pgs (0.749064 -> 0.749064 * mean) 2023-10-01T09:25:02.189 INFO:teuthology.orchestra.run.smithi138.stdout:max osd.3 with 41 -> 40 pgs (1.22846 -> 1.1985 * mean) 2023-10-01T09:25:02.189 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:25:02.189 INFO:teuthology.orchestra.run.smithi138.stdout:oload 110 2023-10-01T09:25:02.189 INFO:teuthology.orchestra.run.smithi138.stdout:max_change 0.05 2023-10-01T09:25:02.189 INFO:teuthology.orchestra.run.smithi138.stdout:max_change_osds 4 2023-10-01T09:25:02.189 INFO:teuthology.orchestra.run.smithi138.stdout:average_utilization 0.0238 2023-10-01T09:25:02.190 INFO:teuthology.orchestra.run.smithi138.stdout:overload_utilization 0.0262 2023-10-01T09:25:02.190 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3 weight 1.0000 -> 0.9500 2023-10-01T09:25:02.190 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0 weight 1.0000 -> 0.9500 2023-10-01T09:25:02.202 INFO:teuthology.orchestra.run.smithi138.stderr:no change 2023-10-01T09:25:02.223 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:02.260 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:02.317 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:02.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:02.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:02.424 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:02.456 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:02.512 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:02.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:02.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:02.518 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:25:02 smithi138 conmon[178630]: 2023-10-01T09:25:02.175+0000 7fa004e48700 -1 mgr.server reply reply (2) No such file or directory no change 2023-10-01T09:25:02.618 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:02.698 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:02.932 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 69 16 82548 82532 74.6935 46.5625 0.00298989 0.0133787 2023-10-01T09:25:03.269 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:25:03.488 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:03 smithi138 ceph-mon[185886]: from='client.35478 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:25:03.488 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:03 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/633013054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:25:03.488 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:03 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:25:03.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:03 smithi138 ceph-mon[192513]: from='client.35478 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:25:03.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:03 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/633013054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:25:03.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:03 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:25:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:03 smithi190 ceph-mon[160435]: from='client.35478 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:25:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:03 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/633013054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:25:03.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:03 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:25:03.726 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:25:03.934 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 70 16 83228 83212 74.2322 42.5 0.00841921 0.0134659 2023-10-01T09:25:04.042 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:04.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:04.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:04.147 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:04.187 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:04.247 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:04.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:04.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:04.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:04 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:25:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:04 smithi138 ceph-mon[192513]: failed to encode map e268 with expected crc 2023-10-01T09:25:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:04 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:25:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:04 smithi138 ceph-mon[192513]: osdmap e268: 8 total, 8 up, 8 in 2023-10-01T09:25:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:04 smithi138 ceph-mon[192513]: pgmap v700: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2023-10-01T09:25:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:04 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:25:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:04 smithi138 ceph-mon[185886]: failed to encode map e268 with expected crc 2023-10-01T09:25:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:04 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:25:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:04 smithi138 ceph-mon[185886]: osdmap e268: 8 total, 8 up, 8 in 2023-10-01T09:25:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:04 smithi138 ceph-mon[185886]: pgmap v700: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2023-10-01T09:25:04.355 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:25:04.381 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:04.421 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:25:04.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:04.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:25:04.526 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:04.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:04 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:25:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:04 smithi190 ceph-mon[160435]: failed to encode map e268 with expected crc 2023-10-01T09:25:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:04 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:25:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:04 smithi190 ceph-mon[160435]: osdmap e268: 8 total, 8 up, 8 in 2023-10-01T09:25:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:04 smithi190 ceph-mon[160435]: pgmap v700: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s 2023-10-01T09:25:04.616 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:04.871 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:04.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:04.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:04.934 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 71 16 83962 83946 73.8327 45.875 0.0120758 0.0135383 2023-10-01T09:25:04.975 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:05.014 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:05.070 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:05.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:05.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:05.158 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:25:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:25:04] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.33.4" 2023-10-01T09:25:05.177 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:05.211 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:05.272 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:05.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:05.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:05.335 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:05.378 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:05.410 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:05.467 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:05.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:05.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:05.505 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:25:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:25:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:25:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:25:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96305 6.b 6:d4901a43:::benchmark_data_smithi138_307751_object936:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-01T09:25:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:17.544856+0000", 2023-10-01T09:25:05.505 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 227.95164822500001, 2023-10-01T09:25:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98726735700000001, 2023-10-01T09:25:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96305 2023-10-01T09:25:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544856+0000", 2023-10-01T09:25:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544856+0000", 2023-10-01T09:25:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544854+0000", 2023-10-01T09:25:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:05.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544861+0000", 2023-10-01T09:25:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5010000000000001e-06 2023-10-01T09:25:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544873+0000", 2023-10-01T09:25:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2472e-05 2023-10-01T09:25:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544882+0000", 2023-10-01T09:25:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3170000000000002e-06 2023-10-01T09:25:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544941+0000", 2023-10-01T09:25:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9660999999999997e-05 2023-10-01T09:25:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544951+0000", 2023-10-01T09:25:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6399999999999992e-06 2023-10-01T09:25:05.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.528813+0000", 2023-10-01T09:25:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98386165299999995 2023-10-01T09:25:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.528936+0000", 2023-10-01T09:25:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012322599999999999 2023-10-01T09:25:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:25:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.529043+0000", 2023-10-01T09:25:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107388 2023-10-01T09:25:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.532020+0000", 2023-10-01T09:25:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029768400000000001 2023-10-01T09:25:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.532027+0000", 2023-10-01T09:25:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8079999999999999e-06 2023-10-01T09:25:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.532060+0000", 2023-10-01T09:25:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2759000000000003e-05 2023-10-01T09:25:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.532094+0000", 2023-10-01T09:25:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4204e-05 2023-10-01T09:25:05.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.532123+0000", 2023-10-01T09:25:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9176e-05 2023-10-01T09:25:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96317 6.b 6:d9fd0fa8:::benchmark_data_smithi138_307751_object948:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-01T09:25:05.513 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:17.562823+0000", 2023-10-01T09:25:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 227.93368076300001, 2023-10-01T09:25:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97706624500000006, 2023-10-01T09:25:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96317 2023-10-01T09:25:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.562823+0000", 2023-10-01T09:25:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.562823+0000", 2023-10-01T09:25:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.562822+0000", 2023-10-01T09:25:05.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.562829+0000", 2023-10-01T09:25:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7090000000000002e-06 2023-10-01T09:25:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.562844+0000", 2023-10-01T09:25:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5143999999999999e-05 2023-10-01T09:25:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.562853+0000", 2023-10-01T09:25:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6859999999999995e-06 2023-10-01T09:25:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.562910+0000", 2023-10-01T09:25:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6512999999999999e-05 2023-10-01T09:25:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.562921+0000", 2023-10-01T09:25:05.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1501000000000001e-05 2023-10-01T09:25:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.532172+0000", 2023-10-01T09:25:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.969250689 2023-10-01T09:25:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.532300+0000", 2023-10-01T09:25:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000128553 2023-10-01T09:25:05.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:25:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.532410+0000", 2023-10-01T09:25:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011003 2023-10-01T09:25:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.539695+0000", 2023-10-01T09:25:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0072849880000000001 2023-10-01T09:25:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.539785+0000", 2023-10-01T09:25:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.933e-05 2023-10-01T09:25:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.539796+0000", 2023-10-01T09:25:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0827e-05 2023-10-01T09:25:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.539848+0000", 2023-10-01T09:25:05.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2599000000000002e-05 2023-10-01T09:25:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.539889+0000", 2023-10-01T09:25:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1275e-05 2023-10-01T09:25:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96323 6.b 6:dd827874:::benchmark_data_smithi138_307751_object954:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-01T09:25:05.521 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:17.571308+0000", 2023-10-01T09:25:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 227.92519597200001, 2023-10-01T09:25:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96870355699999999, 2023-10-01T09:25:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96323 2023-10-01T09:25:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.522 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.571308+0000", 2023-10-01T09:25:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.571308+0000", 2023-10-01T09:25:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.571306+0000", 2023-10-01T09:25:05.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.571313+0000", 2023-10-01T09:25:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0940000000000002e-06 2023-10-01T09:25:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.571327+0000", 2023-10-01T09:25:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3576e-05 2023-10-01T09:25:05.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.571334+0000", 2023-10-01T09:25:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5699999999999998e-06 2023-10-01T09:25:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.571402+0000", 2023-10-01T09:25:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8628999999999996e-05 2023-10-01T09:25:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.571416+0000", 2023-10-01T09:25:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4207e-05 2023-10-01T09:25:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.534595+0000", 2023-10-01T09:25:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96317818799999999 2023-10-01T09:25:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.534694+0000", 2023-10-01T09:25:05.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9388000000000005e-05 2023-10-01T09:25:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:25:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.534759+0000", 2023-10-01T09:25:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4853999999999999e-05 2023-10-01T09:25:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.539733+0000", 2023-10-01T09:25:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049737449999999999 2023-10-01T09:25:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.527 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.539903+0000", 2023-10-01T09:25:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017009600000000001 2023-10-01T09:25:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.539941+0000", 2023-10-01T09:25:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8000999999999998e-05 2023-10-01T09:25:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.539979+0000", 2023-10-01T09:25:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8544999999999998e-05 2023-10-01T09:25:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.540012+0000", 2023-10-01T09:25:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2325e-05 2023-10-01T09:25:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.529 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96326 6.b 6:d61928b4:::benchmark_data_smithi138_307751_object957:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-01T09:25:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:17.572487+0000", 2023-10-01T09:25:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 227.92401649300001, 2023-10-01T09:25:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97419134299999999, 2023-10-01T09:25:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96326 2023-10-01T09:25:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.530 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.572487+0000", 2023-10-01T09:25:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.572487+0000", 2023-10-01T09:25:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.572487+0000", 2023-10-01T09:25:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:25:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.572492+0000", 2023-10-01T09:25:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3900000000000001e-06 2023-10-01T09:25:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.572503+0000", 2023-10-01T09:25:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1207e-05 2023-10-01T09:25:05.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.572510+0000", 2023-10-01T09:25:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2989999999999999e-06 2023-10-01T09:25:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.572563+0000", 2023-10-01T09:25:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2904e-05 2023-10-01T09:25:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.572572+0000", 2023-10-01T09:25:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4950000000000008e-06 2023-10-01T09:25:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.536833+0000", 2023-10-01T09:25:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96426089400000004 2023-10-01T09:25:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.536958+0000", 2023-10-01T09:25:05.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012545699999999999 2023-10-01T09:25:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:25:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.537072+0000", 2023-10-01T09:25:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113347 2023-10-01T09:25:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.540053+0000", 2023-10-01T09:25:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029811299999999998 2023-10-01T09:25:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.546566+0000", 2023-10-01T09:25:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0065134030000000001 2023-10-01T09:25:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.546602+0000", 2023-10-01T09:25:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5741999999999998e-05 2023-10-01T09:25:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.546645+0000", 2023-10-01T09:25:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2684e-05 2023-10-01T09:25:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.546679+0000", 2023-10-01T09:25:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4292000000000003e-05 2023-10-01T09:25:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.537 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96328 6.b 6:db1b79e7:::benchmark_data_smithi138_307751_object959:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-01T09:25:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:17.576171+0000", 2023-10-01T09:25:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 227.92033280199999, 2023-10-01T09:25:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98861726599999999, 2023-10-01T09:25:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96328 2023-10-01T09:25:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.576171+0000", 2023-10-01T09:25:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.576171+0000", 2023-10-01T09:25:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.539 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.576170+0000", 2023-10-01T09:25:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.576176+0000", 2023-10-01T09:25:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9349999999999999e-06 2023-10-01T09:25:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.576191+0000", 2023-10-01T09:25:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4904e-05 2023-10-01T09:25:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.576198+0000", 2023-10-01T09:25:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5279999999999998e-06 2023-10-01T09:25:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.576261+0000", 2023-10-01T09:25:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2908000000000001e-05 2023-10-01T09:25:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.576274+0000", 2023-10-01T09:25:05.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2767e-05 2023-10-01T09:25:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.540077+0000", 2023-10-01T09:25:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96380279199999996 2023-10-01T09:25:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.540187+0000", 2023-10-01T09:25:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110742 2023-10-01T09:25:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:25:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.540306+0000", 2023-10-01T09:25:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011872299999999999 2023-10-01T09:25:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.546689+0000", 2023-10-01T09:25:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0063823500000000002 2023-10-01T09:25:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.546719+0000", 2023-10-01T09:25:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0171999999999998e-05 2023-10-01T09:25:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.564715+0000", 2023-10-01T09:25:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017996807 2023-10-01T09:25:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.564755+0000", 2023-10-01T09:25:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9978000000000002e-05 2023-10-01T09:25:05.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.564788+0000", 2023-10-01T09:25:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2978000000000002e-05 2023-10-01T09:25:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96355 6.b 6:dafff694:::benchmark_data_smithi138_307751_object986:head [delete] snapc 0=[] ondisk+write+known_if_redirected e213)", 2023-10-01T09:25:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:17.604055+0000", 2023-10-01T09:25:05.545 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 227.89244858699999, 2023-10-01T09:25:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96085476199999997, 2023-10-01T09:25:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96355 2023-10-01T09:25:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.604055+0000", 2023-10-01T09:25:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.604055+0000", 2023-10-01T09:25:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.604054+0000", 2023-10-01T09:25:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:05.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.604061+0000", 2023-10-01T09:25:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7710000000000001e-06 2023-10-01T09:25:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.604076+0000", 2023-10-01T09:25:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5328e-05 2023-10-01T09:25:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.604085+0000", 2023-10-01T09:25:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1889999999999996e-06 2023-10-01T09:25:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.604150+0000", 2023-10-01T09:25:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5094999999999999e-05 2023-10-01T09:25:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.604160+0000", 2023-10-01T09:25:05.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0113e-05 2023-10-01T09:25:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.543421+0000", 2023-10-01T09:25:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93926045400000002 2023-10-01T09:25:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.543528+0000", 2023-10-01T09:25:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107311 2023-10-01T09:25:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:25:05.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.543633+0000", 2023-10-01T09:25:05.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010480300000000001 2023-10-01T09:25:05.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.549961+0000", 2023-10-01T09:25:05.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0063283489999999996 2023-10-01T09:25:05.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.558978+0000", 2023-10-01T09:25:05.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0090164570000000003 2023-10-01T09:25:05.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.564836+0000", 2023-10-01T09:25:05.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058584529999999996 2023-10-01T09:25:05.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.564877+0000", 2023-10-01T09:25:05.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0772000000000002e-05 2023-10-01T09:25:05.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:18.564910+0000", 2023-10-01T09:25:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3331000000000003e-05 2023-10-01T09:25:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109318 6.18 6:1f2f6c11:::benchmark_data_smithi138_307751_object13949:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-01T09:25:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.601528+0000", 2023-10-01T09:25:05.553 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 205.89497574800001, 2023-10-01T09:25:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.013117303, 2023-10-01T09:25:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:25:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109318 2023-10-01T09:25:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601528+0000", 2023-10-01T09:25:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601528+0000", 2023-10-01T09:25:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601526+0000", 2023-10-01T09:25:05.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:25:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601539+0000", 2023-10-01T09:25:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3653e-05 2023-10-01T09:25:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601558+0000", 2023-10-01T09:25:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9074000000000001e-05 2023-10-01T09:25:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601573+0000", 2023-10-01T09:25:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4681000000000001e-05 2023-10-01T09:25:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.850607+0000", 2023-10-01T09:25:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24903431400000001 2023-10-01T09:25:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.850615+0000", 2023-10-01T09:25:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.892e-06 2023-10-01T09:25:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.614564+0000", 2023-10-01T09:25:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.76394898899999997 2023-10-01T09:25:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.614646+0000", 2023-10-01T09:25:05.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1291000000000005e-05 2023-10-01T09:25:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109324 6.17 6:e9318d33:::benchmark_data_smithi138_307751_object13955:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-01T09:25:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.601582+0000", 2023-10-01T09:25:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 205.894922283, 2023-10-01T09:25:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97088794, 2023-10-01T09:25:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:25:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.559 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109324 2023-10-01T09:25:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601582+0000", 2023-10-01T09:25:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601582+0000", 2023-10-01T09:25:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601581+0000", 2023-10-01T09:25:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:25:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601584+0000", 2023-10-01T09:25:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.819e-06 2023-10-01T09:25:05.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601590+0000", 2023-10-01T09:25:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9190000000000004e-06 2023-10-01T09:25:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601596+0000", 2023-10-01T09:25:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5620000000000003e-06 2023-10-01T09:25:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.836076+0000", 2023-10-01T09:25:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23448064699999999 2023-10-01T09:25:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.836085+0000", 2023-10-01T09:25:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2350000000000005e-06 2023-10-01T09:25:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.572378+0000", 2023-10-01T09:25:05.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.73629289099999995 2023-10-01T09:25:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.572470+0000", 2023-10-01T09:25:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1323999999999995e-05 2023-10-01T09:25:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109325 6.8 6:178cec25:::benchmark_data_smithi138_307751_object13956:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-01T09:25:05.564 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.601605+0000", 2023-10-01T09:25:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 205.894898743, 2023-10-01T09:25:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9548931860000001, 2023-10-01T09:25:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109325 2023-10-01T09:25:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601605+0000", 2023-10-01T09:25:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601605+0000", 2023-10-01T09:25:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601605+0000", 2023-10-01T09:25:05.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:25:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601608+0000", 2023-10-01T09:25:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5950000000000001e-06 2023-10-01T09:25:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601613+0000", 2023-10-01T09:25:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2209999999999996e-06 2023-10-01T09:25:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601617+0000", 2023-10-01T09:25:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7400000000000004e-06 2023-10-01T09:25:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.850638+0000", 2023-10-01T09:25:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24902023000000001 2023-10-01T09:25:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.850645+0000", 2023-10-01T09:25:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6580000000000002e-06 2023-10-01T09:25:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.545635+0000", 2023-10-01T09:25:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.69498998999999995 2023-10-01T09:25:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.545648+0000", 2023-10-01T09:25:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.305e-05 2023-10-01T09:25:05.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:41.550411+0000", 2023-10-01T09:25:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0047621959999999 2023-10-01T09:25:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:41.550539+0000", 2023-10-01T09:25:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000128133 2023-10-01T09:25:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:25:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:41.550631+0000", 2023-10-01T09:25:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2445e-05 2023-10-01T09:25:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:41.556091+0000", 2023-10-01T09:25:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054601509999999999 2023-10-01T09:25:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:41.556159+0000", 2023-10-01T09:25:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7453999999999995e-05 2023-10-01T09:25:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:41.556199+0000", 2023-10-01T09:25:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0556000000000003e-05 2023-10-01T09:25:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:41.556252+0000", 2023-10-01T09:25:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2698999999999998e-05 2023-10-01T09:25:05.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:41.556498+0000", 2023-10-01T09:25:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00024640900000000001 2023-10-01T09:25:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178743 6.b 6:d51fa993:::benchmark_data_smithi138_307751_object83374:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-10-01T09:25:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.547557+0000", 2023-10-01T09:25:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 115.948946599, 2023-10-01T09:25:05.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2844235369999999, 2023-10-01T09:25:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:25:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178743 2023-10-01T09:25:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547557+0000", 2023-10-01T09:25:05.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547557+0000", 2023-10-01T09:25:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547556+0000", 2023-10-01T09:25:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:25:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547562+0000", 2023-10-01T09:25:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1379999999999998e-06 2023-10-01T09:25:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547574+0000", 2023-10-01T09:25:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2531000000000001e-05 2023-10-01T09:25:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547582+0000", 2023-10-01T09:25:05.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4089999999999997e-06 2023-10-01T09:25:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547630+0000", 2023-10-01T09:25:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8316999999999998e-05 2023-10-01T09:25:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547718+0000", 2023-10-01T09:25:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7713999999999994e-05 2023-10-01T09:25:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:25:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547788+0000", 2023-10-01T09:25:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0086999999999995e-05 2023-10-01T09:25:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.550875+0000", 2023-10-01T09:25:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030869299999999999 2023-10-01T09:25:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.551149+0000", 2023-10-01T09:25:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00027397599999999999 2023-10-01T09:25:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.569123+0000", 2023-10-01T09:25:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017974331999999999 2023-10-01T09:25:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.569137+0000", 2023-10-01T09:25:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3746e-05 2023-10-01T09:25:05.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.605915+0000", 2023-10-01T09:25:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0367782169999999 2023-10-01T09:25:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.605928+0000", 2023-10-01T09:25:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2652e-05 2023-10-01T09:25:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.831898+0000", 2023-10-01T09:25:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2259704 2023-10-01T09:25:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.831981+0000", 2023-10-01T09:25:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3023999999999997e-05 2023-10-01T09:25:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.581 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178766 6.b 6:dcd3b8b8:::benchmark_data_smithi138_307751_object83397:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:25:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.597731+0000", 2023-10-01T09:25:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 115.89877253500001, 2023-10-01T09:25:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2475989780000001, 2023-10-01T09:25:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178766 2023-10-01T09:25:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597731+0000", 2023-10-01T09:25:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597731+0000", 2023-10-01T09:25:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597730+0000", 2023-10-01T09:25:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597736+0000", 2023-10-01T09:25:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5930000000000002e-06 2023-10-01T09:25:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597750+0000", 2023-10-01T09:25:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4090000000000001e-05 2023-10-01T09:25:05.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597757+0000", 2023-10-01T09:25:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1169999999999998e-06 2023-10-01T09:25:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597815+0000", 2023-10-01T09:25:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7797000000000002e-05 2023-10-01T09:25:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597825+0000", 2023-10-01T09:25:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9769999999999995e-06 2023-10-01T09:25:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.605953+0000", 2023-10-01T09:25:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.008127888 2023-10-01T09:25:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.605961+0000", 2023-10-01T09:25:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0160000000000008e-06 2023-10-01T09:25:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.832175+0000", 2023-10-01T09:25:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.226213862 2023-10-01T09:25:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.832268+0000", 2023-10-01T09:25:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2755000000000002e-05 2023-10-01T09:25:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:25:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.832346+0000", 2023-10-01T09:25:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8447000000000003e-05 2023-10-01T09:25:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.838284+0000", 2023-10-01T09:25:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059380830000000002 2023-10-01T09:25:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.838432+0000", 2023-10-01T09:25:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014792399999999999 2023-10-01T09:25:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.838497+0000", 2023-10-01T09:25:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4628000000000005e-05 2023-10-01T09:25:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845139+0000", 2023-10-01T09:25:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066418249999999996 2023-10-01T09:25:05.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845196+0000", 2023-10-01T09:25:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7364000000000002e-05 2023-10-01T09:25:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845330+0000", 2023-10-01T09:25:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000134423 2023-10-01T09:25:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.590 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178802 6.b 6:de530af6:::benchmark_data_smithi138_307751_object83433:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:25:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.644385+0000", 2023-10-01T09:25:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 115.85211859499999, 2023-10-01T09:25:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2012870950000001, 2023-10-01T09:25:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.591 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178802 2023-10-01T09:25:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644385+0000", 2023-10-01T09:25:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644385+0000", 2023-10-01T09:25:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644383+0000", 2023-10-01T09:25:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:25:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644391+0000", 2023-10-01T09:25:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5950000000000001e-06 2023-10-01T09:25:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644407+0000", 2023-10-01T09:25:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6685999999999999e-05 2023-10-01T09:25:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644416+0000", 2023-10-01T09:25:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4610000000000008e-06 2023-10-01T09:25:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644488+0000", 2023-10-01T09:25:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2255999999999994e-05 2023-10-01T09:25:05.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644499+0000", 2023-10-01T09:25:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0594999999999999e-05 2023-10-01T09:25:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.605983+0000", 2023-10-01T09:25:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96148462700000004 2023-10-01T09:25:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.605990+0000", 2023-10-01T09:25:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3080000000000004e-06 2023-10-01T09:25:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.834922+0000", 2023-10-01T09:25:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2289324289999999 2023-10-01T09:25:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.835020+0000", 2023-10-01T09:25:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8348000000000001e-05 2023-10-01T09:25:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:25:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.835115+0000", 2023-10-01T09:25:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4214000000000003e-05 2023-10-01T09:25:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.841312+0000", 2023-10-01T09:25:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0061974939999999996 2023-10-01T09:25:05.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.844863+0000", 2023-10-01T09:25:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035506069999999999 2023-10-01T09:25:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845393+0000", 2023-10-01T09:25:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00053041099999999995 2023-10-01T09:25:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845442+0000", 2023-10-01T09:25:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9171000000000001e-05 2023-10-01T09:25:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845506+0000", 2023-10-01T09:25:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3492999999999997e-05 2023-10-01T09:25:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845672+0000", 2023-10-01T09:25:05.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000166751 2023-10-01T09:25:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178832 6.b 6:d8044603:::benchmark_data_smithi138_307751_object83463:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:25:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.679875+0000", 2023-10-01T09:25:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 115.816629144, 2023-10-01T09:25:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1661682290000002, 2023-10-01T09:25:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.600 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178832 2023-10-01T09:25:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679875+0000", 2023-10-01T09:25:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679875+0000", 2023-10-01T09:25:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679873+0000", 2023-10-01T09:25:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679882+0000", 2023-10-01T09:25:05.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6729999999999998e-06 2023-10-01T09:25:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679897+0000", 2023-10-01T09:25:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5526999999999999e-05 2023-10-01T09:25:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679907+0000", 2023-10-01T09:25:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3910000000000004e-06 2023-10-01T09:25:05.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679978+0000", 2023-10-01T09:25:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1153000000000001e-05 2023-10-01T09:25:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679992+0000", 2023-10-01T09:25:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4256e-05 2023-10-01T09:25:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606003+0000", 2023-10-01T09:25:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92601074299999997 2023-10-01T09:25:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606007+0000", 2023-10-01T09:25:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4150000000000001e-06 2023-10-01T09:25:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.838522+0000", 2023-10-01T09:25:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.232515282 2023-10-01T09:25:05.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.838618+0000", 2023-10-01T09:25:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6039000000000007e-05 2023-10-01T09:25:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:25:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.838703+0000", 2023-10-01T09:25:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4345999999999995e-05 2023-10-01T09:25:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.844873+0000", 2023-10-01T09:25:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006170633 2023-10-01T09:25:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845757+0000", 2023-10-01T09:25:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00088383699999999995 2023-10-01T09:25:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845819+0000", 2023-10-01T09:25:05.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1549000000000003e-05 2023-10-01T09:25:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845874+0000", 2023-10-01T09:25:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5065000000000002e-05 2023-10-01T09:25:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845925+0000", 2023-10-01T09:25:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1604000000000003e-05 2023-10-01T09:25:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.846043+0000", 2023-10-01T09:25:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011757899999999999 2023-10-01T09:25:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178834 6.b 6:d604d262:::benchmark_data_smithi138_307751_object83465:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:25:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.683872+0000", 2023-10-01T09:25:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 115.81263205400001, 2023-10-01T09:25:05.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1718884809999999, 2023-10-01T09:25:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178834 2023-10-01T09:25:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683872+0000", 2023-10-01T09:25:05.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683872+0000", 2023-10-01T09:25:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683870+0000", 2023-10-01T09:25:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683877+0000", 2023-10-01T09:25:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6379999999999998e-06 2023-10-01T09:25:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683894+0000", 2023-10-01T09:25:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6860000000000001e-05 2023-10-01T09:25:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683902+0000", 2023-10-01T09:25:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1410000000000005e-06 2023-10-01T09:25:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683968+0000", 2023-10-01T09:25:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6450000000000002e-05 2023-10-01T09:25:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683983+0000", 2023-10-01T09:25:05.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.49e-05 2023-10-01T09:25:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606019+0000", 2023-10-01T09:25:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92203571299999998 2023-10-01T09:25:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606024+0000", 2023-10-01T09:25:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.011e-06 2023-10-01T09:25:05.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.841339+0000", 2023-10-01T09:25:05.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.235314947 2023-10-01T09:25:05.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.841469+0000", 2023-10-01T09:25:05.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013036600000000001 2023-10-01T09:25:05.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:25:05.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.841644+0000", 2023-10-01T09:25:05.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017456700000000001 2023-10-01T09:25:05.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845691+0000", 2023-10-01T09:25:05.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040474459999999997 2023-10-01T09:25:05.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.846103+0000", 2023-10-01T09:25:05.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00041115299999999999 2023-10-01T09:25:05.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855580+0000", 2023-10-01T09:25:05.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094775910000000005 2023-10-01T09:25:05.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855611+0000", 2023-10-01T09:25:05.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0985999999999998e-05 2023-10-01T09:25:05.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855655+0000", 2023-10-01T09:25:05.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3445999999999997e-05 2023-10-01T09:25:05.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855760+0000", 2023-10-01T09:25:05.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010581099999999999 2023-10-01T09:25:05.618 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.618 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.618 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.619 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178842 6.b 6:deac9fef:::benchmark_data_smithi138_307751_object83473:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:25:05.619 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.722803+0000", 2023-10-01T09:25:05.619 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 115.773701094, 2023-10-01T09:25:05.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1395442039999999, 2023-10-01T09:25:05.619 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.619 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.619 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.619 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.619 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.619 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178842 2023-10-01T09:25:05.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.620 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722803+0000", 2023-10-01T09:25:05.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722803+0000", 2023-10-01T09:25:05.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722801+0000", 2023-10-01T09:25:05.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:05.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722809+0000", 2023-10-01T09:25:05.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5399999999999998e-06 2023-10-01T09:25:05.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722824+0000", 2023-10-01T09:25:05.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5231999999999999e-05 2023-10-01T09:25:05.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722832+0000", 2023-10-01T09:25:05.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6990000000000003e-06 2023-10-01T09:25:05.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722883+0000", 2023-10-01T09:25:05.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1439e-05 2023-10-01T09:25:05.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722893+0000", 2023-10-01T09:25:05.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4429999999999992e-06 2023-10-01T09:25:05.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606037+0000", 2023-10-01T09:25:05.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88314473500000001 2023-10-01T09:25:05.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606042+0000", 2023-10-01T09:25:05.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2350000000000001e-06 2023-10-01T09:25:05.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.846136+0000", 2023-10-01T09:25:05.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.240094662 2023-10-01T09:25:05.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.846286+0000", 2023-10-01T09:25:05.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015010300000000001 2023-10-01T09:25:05.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:25:05.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.846430+0000", 2023-10-01T09:25:05.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014343400000000001 2023-10-01T09:25:05.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855502+0000", 2023-10-01T09:25:05.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0090716679999999997 2023-10-01T09:25:05.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855541+0000", 2023-10-01T09:25:05.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9038000000000001e-05 2023-10-01T09:25:05.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855798+0000", 2023-10-01T09:25:05.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025726999999999998 2023-10-01T09:25:05.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862172+0000", 2023-10-01T09:25:05.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0063741739999999998 2023-10-01T09:25:05.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862230+0000", 2023-10-01T09:25:05.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7942000000000002e-05 2023-10-01T09:25:05.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862347+0000", 2023-10-01T09:25:05.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011716 2023-10-01T09:25:05.628 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.628 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.628 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.629 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178859 6.b 6:d9d8473f:::benchmark_data_smithi138_307751_object83490:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:25:05.629 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.882356+0000", 2023-10-01T09:25:05.629 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 115.61414823, 2023-10-01T09:25:05.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9801762709999999, 2023-10-01T09:25:05.629 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.629 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.629 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.629 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.629 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.629 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178859 2023-10-01T09:25:05.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.630 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882356+0000", 2023-10-01T09:25:05.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882356+0000", 2023-10-01T09:25:05.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882355+0000", 2023-10-01T09:25:05.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:25:05.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882360+0000", 2023-10-01T09:25:05.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8579999999999999e-06 2023-10-01T09:25:05.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882372+0000", 2023-10-01T09:25:05.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1674999999999999e-05 2023-10-01T09:25:05.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882378+0000", 2023-10-01T09:25:05.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0630000000000002e-06 2023-10-01T09:25:05.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882441+0000", 2023-10-01T09:25:05.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3232999999999996e-05 2023-10-01T09:25:05.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882458+0000", 2023-10-01T09:25:05.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7087000000000001e-05 2023-10-01T09:25:05.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606055+0000", 2023-10-01T09:25:05.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.72359739000000001 2023-10-01T09:25:05.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606060+0000", 2023-10-01T09:25:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1559999999999997e-06 2023-10-01T09:25:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.849168+0000", 2023-10-01T09:25:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2431087970000001 2023-10-01T09:25:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.849278+0000", 2023-10-01T09:25:05.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109393 2023-10-01T09:25:05.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:25:05.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.849382+0000", 2023-10-01T09:25:05.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000104555 2023-10-01T09:25:05.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855548+0000", 2023-10-01T09:25:05.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0061655649999999996 2023-10-01T09:25:05.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855837+0000", 2023-10-01T09:25:05.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00028885599999999999 2023-10-01T09:25:05.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862102+0000", 2023-10-01T09:25:05.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0062649990000000003 2023-10-01T09:25:05.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862389+0000", 2023-10-01T09:25:05.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00028695199999999999 2023-10-01T09:25:05.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862430+0000", 2023-10-01T09:25:05.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1109000000000002e-05 2023-10-01T09:25:05.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862532+0000", 2023-10-01T09:25:05.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000102305 2023-10-01T09:25:05.637 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.637 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.637 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.638 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178863 6.b 6:d3d535ce:::benchmark_data_smithi138_307751_object83494:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:25:05.638 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.890216+0000", 2023-10-01T09:25:05.638 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 115.60628786700001, 2023-10-01T09:25:05.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9882852390000001, 2023-10-01T09:25:05.638 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.638 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.638 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.638 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.638 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.638 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178863 2023-10-01T09:25:05.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.639 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890216+0000", 2023-10-01T09:25:05.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890216+0000", 2023-10-01T09:25:05.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890215+0000", 2023-10-01T09:25:05.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:05.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890221+0000", 2023-10-01T09:25:05.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5819999999999998e-06 2023-10-01T09:25:05.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890236+0000", 2023-10-01T09:25:05.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4620999999999999e-05 2023-10-01T09:25:05.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890244+0000", 2023-10-01T09:25:05.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8299999999999996e-06 2023-10-01T09:25:05.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890298+0000", 2023-10-01T09:25:05.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4508000000000001e-05 2023-10-01T09:25:05.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890307+0000", 2023-10-01T09:25:05.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7290000000000006e-06 2023-10-01T09:25:05.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606072+0000", 2023-10-01T09:25:05.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.71576492899999999 2023-10-01T09:25:05.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606076+0000", 2023-10-01T09:25:05.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0579999999999999e-06 2023-10-01T09:25:05.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.852232+0000", 2023-10-01T09:25:05.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.246155755 2023-10-01T09:25:05.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.852363+0000", 2023-10-01T09:25:05.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013094699999999999 2023-10-01T09:25:05.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:25:05.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.852489+0000", 2023-10-01T09:25:05.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000126635 2023-10-01T09:25:05.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855805+0000", 2023-10-01T09:25:05.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0033155120000000001 2023-10-01T09:25:05.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862138+0000", 2023-10-01T09:25:05.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006333198 2023-10-01T09:25:05.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869114+0000", 2023-10-01T09:25:05.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069759469999999997 2023-10-01T09:25:05.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878304+0000", 2023-10-01T09:25:05.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0091900300000000001 2023-10-01T09:25:05.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878367+0000", 2023-10-01T09:25:05.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2658999999999998e-05 2023-10-01T09:25:05.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878501+0000", 2023-10-01T09:25:05.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000134609 2023-10-01T09:25:05.647 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.647 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.647 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.647 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178866 6.b 6:d03ed771:::benchmark_data_smithi138_307751_object83497:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:25:05.647 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.903567+0000", 2023-10-01T09:25:05.647 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 115.592936693, 2023-10-01T09:25:05.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9751352579999999, 2023-10-01T09:25:05.647 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.647 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.647 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.648 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.648 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.648 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178866 2023-10-01T09:25:05.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.648 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903567+0000", 2023-10-01T09:25:05.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903567+0000", 2023-10-01T09:25:05.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903565+0000", 2023-10-01T09:25:05.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:05.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903573+0000", 2023-10-01T09:25:05.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9270000000000005e-06 2023-10-01T09:25:05.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903589+0000", 2023-10-01T09:25:05.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5112e-05 2023-10-01T09:25:05.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903598+0000", 2023-10-01T09:25:05.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4050000000000006e-06 2023-10-01T09:25:05.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903655+0000", 2023-10-01T09:25:05.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6694999999999998e-05 2023-10-01T09:25:05.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903670+0000", 2023-10-01T09:25:05.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5192e-05 2023-10-01T09:25:05.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606090+0000", 2023-10-01T09:25:05.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.70241997300000003 2023-10-01T09:25:05.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606097+0000", 2023-10-01T09:25:05.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2300000000000002e-06 2023-10-01T09:25:05.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855860+0000", 2023-10-01T09:25:05.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.249762633 2023-10-01T09:25:05.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855972+0000", 2023-10-01T09:25:05.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112587 2023-10-01T09:25:05.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:25:05.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.856087+0000", 2023-10-01T09:25:05.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115238 2023-10-01T09:25:05.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862571+0000", 2023-10-01T09:25:05.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064834539999999996 2023-10-01T09:25:05.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869173+0000", 2023-10-01T09:25:05.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066022570000000003 2023-10-01T09:25:05.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869318+0000", 2023-10-01T09:25:05.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000145084 2023-10-01T09:25:05.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878567+0000", 2023-10-01T09:25:05.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0092485280000000007 2023-10-01T09:25:05.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878602+0000", 2023-10-01T09:25:05.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5315999999999999e-05 2023-10-01T09:25:05.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878703+0000", 2023-10-01T09:25:05.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000100436 2023-10-01T09:25:05.656 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.656 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.656 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.656 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178873 6.b 6:d1b63b67:::benchmark_data_smithi138_307751_object83504:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:25:05.656 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.923214+0000", 2023-10-01T09:25:05.657 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 115.57328963, 2023-10-01T09:25:05.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.95573526, 2023-10-01T09:25:05.657 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.657 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.657 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.657 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.657 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.657 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178873 2023-10-01T09:25:05.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.657 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923214+0000", 2023-10-01T09:25:05.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923214+0000", 2023-10-01T09:25:05.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923200+0000", 2023-10-01T09:25:05.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999857 2023-10-01T09:25:05.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923221+0000", 2023-10-01T09:25:05.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1022999999999998e-05 2023-10-01T09:25:05.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923238+0000", 2023-10-01T09:25:05.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6948000000000001e-05 2023-10-01T09:25:05.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923246+0000", 2023-10-01T09:25:05.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6830000000000008e-06 2023-10-01T09:25:05.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923317+0000", 2023-10-01T09:25:05.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1519000000000004e-05 2023-10-01T09:25:05.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923330+0000", 2023-10-01T09:25:05.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3159000000000001e-05 2023-10-01T09:25:05.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606118+0000", 2023-10-01T09:25:05.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.68278745900000004 2023-10-01T09:25:05.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606126+0000", 2023-10-01T09:25:05.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2060000000000007e-06 2023-10-01T09:25:05.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.858734+0000", 2023-10-01T09:25:05.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2526076939999999 2023-10-01T09:25:05.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.858881+0000", 2023-10-01T09:25:05.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000147374 2023-10-01T09:25:05.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:25:05.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.859032+0000", 2023-10-01T09:25:05.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015129600000000001 2023-10-01T09:25:05.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862594+0000", 2023-10-01T09:25:05.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035613749999999999 2023-10-01T09:25:05.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869240+0000", 2023-10-01T09:25:05.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066465070000000003 2023-10-01T09:25:05.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869326+0000", 2023-10-01T09:25:05.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5883000000000004e-05 2023-10-01T09:25:05.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878767+0000", 2023-10-01T09:25:05.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094413799999999992 2023-10-01T09:25:05.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878827+0000", 2023-10-01T09:25:05.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9855000000000002e-05 2023-10-01T09:25:05.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878950+0000", 2023-10-01T09:25:05.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000122273 2023-10-01T09:25:05.665 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.665 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.665 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.666 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178875 6.b 6:d0889c71:::benchmark_data_smithi138_307751_object83506:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:25:05.666 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.927144+0000", 2023-10-01T09:25:05.666 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 115.569359682, 2023-10-01T09:25:05.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9520905690000001, 2023-10-01T09:25:05.666 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:05.666 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:05.666 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:05.666 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:25:05.666 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:05.666 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178875 2023-10-01T09:25:05.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.667 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:05.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:05.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927144+0000", 2023-10-01T09:25:05.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:05.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927144+0000", 2023-10-01T09:25:05.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:05.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:05.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927142+0000", 2023-10-01T09:25:05.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:05.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:05.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927150+0000", 2023-10-01T09:25:05.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0209999999999999e-06 2023-10-01T09:25:05.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:05.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927165+0000", 2023-10-01T09:25:05.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4939e-05 2023-10-01T09:25:05.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:05.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927173+0000", 2023-10-01T09:25:05.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2599999999999999e-06 2023-10-01T09:25:05.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927243+0000", 2023-10-01T09:25:05.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0129000000000006e-05 2023-10-01T09:25:05.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927253+0000", 2023-10-01T09:25:05.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0431999999999999e-05 2023-10-01T09:25:05.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606149+0000", 2023-10-01T09:25:05.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67889524400000001 2023-10-01T09:25:05.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:25:05.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606157+0000", 2023-10-01T09:25:05.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0870000000000007e-06 2023-10-01T09:25:05.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:05.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862610+0000", 2023-10-01T09:25:05.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2564529449999999 2023-10-01T09:25:05.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:05.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862693+0000", 2023-10-01T09:25:05.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3590000000000004e-05 2023-10-01T09:25:05.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:25:05.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862765+0000", 2023-10-01T09:25:05.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1478999999999998e-05 2023-10-01T09:25:05.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869306+0000", 2023-10-01T09:25:05.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0065418569999999999 2023-10-01T09:25:05.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:25:05.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869333+0000", 2023-10-01T09:25:05.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6787999999999998e-05 2023-10-01T09:25:05.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869372+0000", 2023-10-01T09:25:05.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9127e-05 2023-10-01T09:25:05.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:05.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.879031+0000", 2023-10-01T09:25:05.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0096583450000000005 2023-10-01T09:25:05.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:25:05.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.879099+0000", 2023-10-01T09:25:05.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7879000000000005e-05 2023-10-01T09:25:05.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:05.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:05.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:25:05.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.879235+0000", 2023-10-01T09:25:05.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000136279 2023-10-01T09:25:05.674 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.675 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.675 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.675 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:05.675 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:05.675 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:25:05.675 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:05.749 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:05.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:05 smithi138 ceph-mon[192513]: pgmap v701: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s 2023-10-01T09:25:05.815 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:05 smithi138 ceph-mon[185886]: pgmap v701: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s 2023-10-01T09:25:05.887 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:05.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:05.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:05.935 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 72 16 84817 84801 73.5482 53.4375 0.0185661 0.0135908 2023-10-01T09:25:05.996 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:06.022 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:05 smithi190 ceph-mon[160435]: pgmap v701: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s 2023-10-01T09:25:06.025 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:06.078 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:06.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:06.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:06.184 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:25:06.254 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:06.400 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:25:06.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:06.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:25:06.506 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:06.546 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:06.606 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:06.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:06.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:06.714 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:06.749 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:06.809 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:06.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:06.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:06.915 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:06.935 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 73 16 85771 85755 73.3576 59.625 0.00904183 0.013626 2023-10-01T09:25:06.951 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:06.980 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:25:07.020 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:07.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:07.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:07.126 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:25:07.153 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:07.195 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:25:07.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:07.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:25:07.300 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:07.387 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:07.462 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:25:07.673 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:07.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:07.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:07.780 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:07.815 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:07.858 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:07.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:07.919 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:07.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:07.932 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:25:07.940 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 74 16 86520 86504 72.9949 46.8125 0.0109189 0.0136945 2023-10-01T09:25:08.020 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:25:08.058 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:08.217 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:25:08.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:08.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:25:08.307 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:08 smithi138 ceph-mon[192513]: pgmap v702: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2023-10-01T09:25:08.308 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:08 smithi138 ceph-mon[185886]: pgmap v702: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2023-10-01T09:25:08.321 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:08.392 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:08.506 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:08 smithi190 ceph-mon[160435]: pgmap v702: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2023-10-01T09:25:08.519 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:25:08.569 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:25:08.592 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:08.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:08.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:08.624 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:25:08.669 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:25:08.705 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:08.718 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:25:08.769 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:25:08.801 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:08.812 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:25:08.848 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:25:08.870 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:25:08.871 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:25:08.871 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:25:08.871 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:25:08.871 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:25:08.871 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:25:08.871 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:25:08.871 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:25:08.885 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:08.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:08.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:08.940 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 75 16 87299 87283 72.671 48.6875 0.0126593 0.0137441 2023-10-01T09:25:08.989 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:09.014 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:09.055 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:09.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:09.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:09.158 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:09.212 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:09 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1696405827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:09.212 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:09 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1696405827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:09.248 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:09.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:09 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1696405827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:09.609 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 7, 2, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 6, 7, 5] 2023-10-01T09:25:09.609 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:25:09.610 INFO:tasks.thrashosds.thrasher:Killing osd 6, live_osds are [0, 1, 2, 3, 4, 6, 7, 5] 2023-10-01T09:25:09.610 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-01T09:25:09.610 DEBUG:teuthology.orchestra.run.smithi190:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.6 2023-10-01T09:25:09.942 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 76 16 87964 87948 72.2601 41.5625 0.0175171 0.0138337 2023-10-01T09:25:10.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:25:09 smithi190 systemd[1]: Stopping Ceph osd.6 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:25:10.108 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:10.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:10.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:10.219 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:10.226 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:10 smithi138 ceph-mon[185886]: pgmap v703: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2023-10-01T09:25:10.259 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:10.506 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:10 smithi138 ceph-mon[192513]: pgmap v703: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2023-10-01T09:25:10.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:10 smithi190 ceph-mon[160435]: pgmap v703: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2023-10-01T09:25:10.805 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:10.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:10.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:10.911 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:10.942 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 77 16 88649 88633 71.8779 42.8125 0.0185224 0.0139062 2023-10-01T09:25:10.995 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:11.126 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:11.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:11.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:11.144 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:25:10 smithi190 conmon[118570]: debug 2023-10-01T09:25:10.936+0000 7fc68f3b8700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T09:25:11.144 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:25:10 smithi190 conmon[118570]: debug 2023-10-01T09:25:10.936+0000 7fc68f3b8700 -1 osd.6 268 *** Got signal Terminated *** 2023-10-01T09:25:11.144 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:25:10 smithi190 conmon[118570]: debug 2023-10-01T09:25:10.936+0000 7fc68f3b8700 -1 osd.6 268 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:25:11.239 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:11.281 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:25:11.298 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:25:11.298 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:11 smithi138 ceph-mon[185886]: osd.6 marked itself down and dead 2023-10-01T09:25:11.298 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1617028685' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:11.298 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:25:11.298 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:11 smithi138 ceph-mon[192513]: osd.6 marked itself down and dead 2023-10-01T09:25:11.298 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1617028685' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:11.306 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:11.387 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:25:11.388 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:25:11.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:25:11.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:11 smithi190 ceph-mon[160435]: osd.6 marked itself down and dead 2023-10-01T09:25:11.411 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1617028685' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:11.500 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:11.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:11.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:11.604 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:11.637 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:11.735 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:11.737 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:25:11 smithi190 bash[237964]: fa45a1c37077d09d029753b7934865af421b63b9720cdbd7f38f8ad5960f93e8 2023-10-01T09:25:11.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:11.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:11.839 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:11.866 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:11.908 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:11.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:11.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:11.942 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 78 16 88958 88942 71.2045 19.3125 0.00427343 0.0139202 2023-10-01T09:25:12.012 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:12.039 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:12.298 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:25:12.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:12 smithi190 ceph-mon[160435]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:25:12.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:12 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:25:12.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:12 smithi190 ceph-mon[160435]: failed to encode map e269 with expected crc 2023-10-01T09:25:12.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:12 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1617028685' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:25:12.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:12 smithi190 ceph-mon[160435]: osdmap e269: 8 total, 7 up, 8 in 2023-10-01T09:25:12.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:12 smithi190 ceph-mon[160435]: failed to encode map e269 with expected crc 2023-10-01T09:25:12.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:12 smithi190 ceph-mon[160435]: failed to encode map e269 with expected crc 2023-10-01T09:25:12.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:12 smithi190 ceph-mon[160435]: pgmap v705: 89 pgs: 6 peering, 5 stale+active+clean, 78 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 666 op/s 2023-10-01T09:25:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[185886]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:25:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:25:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[185886]: failed to encode map e269 with expected crc 2023-10-01T09:25:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1617028685' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:25:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[185886]: osdmap e269: 8 total, 7 up, 8 in 2023-10-01T09:25:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[185886]: failed to encode map e269 with expected crc 2023-10-01T09:25:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[185886]: failed to encode map e269 with expected crc 2023-10-01T09:25:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[185886]: pgmap v705: 89 pgs: 6 peering, 5 stale+active+clean, 78 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 666 op/s 2023-10-01T09:25:12.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[192513]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:25:12.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:25:12.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[192513]: failed to encode map e269 with expected crc 2023-10-01T09:25:12.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1617028685' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:25:12.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[192513]: osdmap e269: 8 total, 7 up, 8 in 2023-10-01T09:25:12.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[192513]: failed to encode map e269 with expected crc 2023-10-01T09:25:12.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[192513]: failed to encode map e269 with expected crc 2023-10-01T09:25:12.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:12 smithi138 ceph-mon[192513]: pgmap v705: 89 pgs: 6 peering, 5 stale+active+clean, 78 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 666 op/s 2023-10-01T09:25:12.787 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:12.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:12.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:12.884 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:25:12.891 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:12.943 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 79 16 89505 89489 70.7355 34.1875 0.0105332 0.0141321 2023-10-01T09:25:12.968 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:13.044 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:13.050 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:13.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:13.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:13.156 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:13.201 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:13 smithi190 ceph-mon[160435]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:25:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:13 smithi190 ceph-mon[160435]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:13 smithi190 ceph-mon[160435]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:13 smithi190 ceph-mon[160435]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:13 smithi190 ceph-mon[160435]: osdmap e270: 8 total, 7 up, 8 in 2023-10-01T09:25:13.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:13 smithi190 ceph-mon[160435]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:13 smithi190 ceph-mon[160435]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:13 smithi190 ceph-mon[160435]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.358 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:13.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:13.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:13.463 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:13.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[185886]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:25:13.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[185886]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[185886]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[185886]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[185886]: osdmap e270: 8 total, 7 up, 8 in 2023-10-01T09:25:13.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[185886]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[185886]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[185886]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.524 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[192513]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:25:13.524 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[192513]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[192513]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[192513]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[192513]: osdmap e270: 8 total, 7 up, 8 in 2023-10-01T09:25:13.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[192513]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[192513]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:13 smithi138 ceph-mon[192513]: failed to encode map e270 with expected crc 2023-10-01T09:25:13.526 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:13.544 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:25:13.640 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:13.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:13.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:13.749 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:13.799 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:13.946 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:25:13.944228+0000 min lat: 0.0011387 max lat: 1.0548 avg lat: 0.0141657 2023-10-01T09:25:13.946 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:25:13.946 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 80 16 90427 90411 70.5713 57.625 0.0302189 0.0141657 2023-10-01T09:25:14.324 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:14.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:14.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:14.340 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:25:14.431 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:25:14.511 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:14 smithi138 ceph-mon[185886]: pgmap v707: 89 pgs: 2 active+undersized+degraded, 1 active+undersized, 13 peering, 3 stale+active+clean, 70 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s; 5589/266091 objects degraded (2.100%) 2023-10-01T09:25:14.511 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:14 smithi138 ceph-mon[192513]: pgmap v707: 89 pgs: 2 active+undersized+degraded, 1 active+undersized, 13 peering, 3 stale+active+clean, 70 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s; 5589/266091 objects degraded (2.100%) 2023-10-01T09:25:14.517 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:14.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:14 smithi190 ceph-mon[160435]: pgmap v707: 89 pgs: 2 active+undersized+degraded, 1 active+undersized, 13 peering, 3 stale+active+clean, 70 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 770 op/s; 5589/266091 objects degraded (2.100%) 2023-10-01T09:25:14.694 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:25:14.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:14.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:25:14.714 DEBUG:teuthology.orchestra.run.smithi190:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-01T09:25:14.761 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:25:14.761 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-01T09:25:14.813 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:14.910 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:14.944 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 81 16 91078 91062 70.2026 40.6875 0.019949 0.0142402 2023-10-01T09:25:15.088 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:15.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:15.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:15.202 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:15.273 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:15 smithi138 ceph-mon[185886]: Health check failed: Degraded data redundancy: 5589/266091 objects degraded (2.100%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:15.275 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:25:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:25:14] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-01T09:25:15.291 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:15.421 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:25:15.441 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:15.448 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:15.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:15.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:15.472 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:25:15.507 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:25:15.535 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:15 smithi190 ceph-mon[160435]: Health check failed: Degraded data redundancy: 5589/266091 objects degraded (2.100%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:15.553 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:25:15.558 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:15.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:15 smithi138 ceph-mon[192513]: Health check failed: Degraded data redundancy: 5589/266091 objects degraded (2.100%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:15.582 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:25:15.628 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:25:15.656 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:15.662 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:25:15.662 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:25:15.662 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:25:15.662 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:25:15.663 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:25:15.663 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:25:15.663 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:25:15.663 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-10-01T09:25:15.663 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:25:15.727 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:25:15.727 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:25:15.728 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:25:15.728 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:25:15.728 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:25:15.919 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:15.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:15.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:15.944 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 82 16 91866 91850 69.9472 49.25 0.0120447 0.0142912 2023-10-01T09:25:16.026 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:16.059 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:16.291 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:16 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2031211283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:16.291 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:16 smithi138 ceph-mon[185886]: pgmap v708: 89 pgs: 6 active+undersized+degraded, 13 active+undersized, 13 peering, 57 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 552 op/s; 16757/269049 objects degraded (6.228%) 2023-10-01T09:25:16.409 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:25:16.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:16 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2031211283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:16.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:16 smithi190 ceph-mon[160435]: pgmap v708: 89 pgs: 6 active+undersized+degraded, 13 active+undersized, 13 peering, 57 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 552 op/s; 16757/269049 objects degraded (6.228%) 2023-10-01T09:25:16.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:16 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2031211283' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:16.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:16 smithi138 ceph-mon[192513]: pgmap v708: 89 pgs: 6 active+undersized+degraded, 13 active+undersized, 13 peering, 57 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 552 op/s; 16757/269049 objects degraded (6.228%) 2023-10-01T09:25:16.842 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:25:16.946 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 83 16 92738 92722 69.7594 54.5 0.0124251 0.0143296 2023-10-01T09:25:16.981 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:16.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:16.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:17.087 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:17.155 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:17.195 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:25:17.195 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:25:17.280 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:17.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:17.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:17.383 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:17.409 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:17.458 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:17.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:17.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:17.561 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:17.592 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:17.656 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:17.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:17.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:17.760 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:17.790 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:17.872 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:17.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:17.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:17.946 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 84 16 93443 93427 69.4537 44.0625 0.0187086 0.0143929 2023-10-01T09:25:17.976 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:18.015 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:18.067 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:18.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:18.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:18.172 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:18.214 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:18.228 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:18 smithi138 ceph-mon[185886]: pgmap v709: 89 pgs: 11 active+undersized+degraded, 21 active+undersized, 57 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 28469/277005 objects degraded (10.277%) 2023-10-01T09:25:18.321 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:18.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:18.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:18.391 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:25:18.391 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:25:18.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:25:18.391 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:25:18.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.392 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:110057 5.e 5:7faf45b8:::benchmark_data_smithi138_240338_object110056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e122)", 2023-10-01T09:25:18.392 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:08.050553+0000", 2023-10-01T09:25:18.392 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 550.33764201400004, 2023-10-01T09:25:18.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22493601599999999, 2023-10-01T09:25:18.392 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.392 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.392 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.392 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.392 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.392 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 110057 2023-10-01T09:25:18.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.393 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050553+0000", 2023-10-01T09:25:18.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050553+0000", 2023-10-01T09:25:18.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:18.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.394 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.394 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050551+0000", 2023-10-01T09:25:18.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000123542 2023-10-01T09:25:18.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.394 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.394 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050674+0000", 2023-10-01T09:25:18.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7989e-05 2023-10-01T09:25:18.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.394 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.394 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050692+0000", 2023-10-01T09:25:18.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4090000000000003e-06 2023-10-01T09:25:18.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.050701+0000", 2023-10-01T09:25:18.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0016255810000000001 2023-10-01T09:25:18.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.052326+0000", 2023-10-01T09:25:18.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000140653 2023-10-01T09:25:18.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.052467+0000", 2023-10-01T09:25:18.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014696699999999999 2023-10-01T09:25:18.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:25:18.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.052614+0000", 2023-10-01T09:25:18.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003798516 2023-10-01T09:25:18.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.056412+0000", 2023-10-01T09:25:18.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0023911510000000002 2023-10-01T09:25:18.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.058804+0000", 2023-10-01T09:25:18.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21657992000000001 2023-10-01T09:25:18.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275383+0000", 2023-10-01T09:25:18.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4166999999999998e-05 2023-10-01T09:25:18.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275438+0000", 2023-10-01T09:25:18.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0992000000000002e-05 2023-10-01T09:25:18.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:08.275489+0000", 2023-10-01T09:25:18.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22493601599999999 2023-10-01T09:25:18.398 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.399 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.399 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.399 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:119705 5.e 5:779a2057:::benchmark_data_smithi138_240338_object4568:head [delete] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-10-01T09:25:18.399 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:20.269601+0000", 2023-10-01T09:25:18.399 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 538.11859381399995, 2023-10-01T09:25:18.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24117385499999999, 2023-10-01T09:25:18.399 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.399 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.399 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.399 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.400 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.400 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 119705 2023-10-01T09:25:18.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.400 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.400 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269601+0000", 2023-10-01T09:25:18.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.400 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269601+0000", 2023-10-01T09:25:18.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:18.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269599+0000", 2023-10-01T09:25:18.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9090000000000003e-06 2023-10-01T09:25:18.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269606+0000", 2023-10-01T09:25:18.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.446e-05 2023-10-01T09:25:18.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.402 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269621+0000", 2023-10-01T09:25:18.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0389999999999999e-06 2023-10-01T09:25:18.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.402 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.269628+0000", 2023-10-01T09:25:18.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0097709400000000005 2023-10-01T09:25:18.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.279399+0000", 2023-10-01T09:25:18.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014551399999999999 2023-10-01T09:25:18.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.279544+0000", 2023-10-01T09:25:18.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000171409 2023-10-01T09:25:18.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:25:18.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.279715+0000", 2023-10-01T09:25:18.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0039263379999999997 2023-10-01T09:25:18.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.283642+0000", 2023-10-01T09:25:18.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012843927 2023-10-01T09:25:18.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.296486+0000", 2023-10-01T09:25:18.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.214182922 2023-10-01T09:25:18.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.510669+0000", 2023-10-01T09:25:18.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4887000000000004e-05 2023-10-01T09:25:18.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.510734+0000", 2023-10-01T09:25:18.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1088999999999999e-05 2023-10-01T09:25:18.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:20.510775+0000", 2023-10-01T09:25:18.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24117385499999999 2023-10-01T09:25:18.406 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.406 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.406 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.406 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:121380 5.a 5:55f57801:::benchmark_data_smithi138_240338_object6243:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:25:18.406 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:22.569549+0000", 2023-10-01T09:25:18.407 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 535.818645188, 2023-10-01T09:25:18.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98289109100000005, 2023-10-01T09:25:18.407 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.407 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.407 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.407 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.407 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.407 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121380 2023-10-01T09:25:18.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.407 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.407 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569549+0000", 2023-10-01T09:25:18.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569549+0000", 2023-10-01T09:25:18.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:18.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569548+0000", 2023-10-01T09:25:18.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7849999999999992e-06 2023-10-01T09:25:18.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569556+0000", 2023-10-01T09:25:18.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5118e-05 2023-10-01T09:25:18.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569571+0000", 2023-10-01T09:25:18.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0104e-05 2023-10-01T09:25:18.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.410 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.410 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569581+0000", 2023-10-01T09:25:18.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1700999999999997e-05 2023-10-01T09:25:18.410 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.410 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.410 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.410 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569653+0000", 2023-10-01T09:25:18.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4584e-05 2023-10-01T09:25:18.410 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.410 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.410 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:18.411 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.569667+0000", 2023-10-01T09:25:18.411 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.96702125500000002 2023-10-01T09:25:18.411 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.411 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.411 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.411 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.536688+0000", 2023-10-01T09:25:18.411 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0026996390000000002 2023-10-01T09:25:18.411 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.411 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.411 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.411 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.539388+0000", 2023-10-01T09:25:18.411 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014500400000000001 2023-10-01T09:25:18.412 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.412 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.412 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:25:18.412 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.539533+0000", 2023-10-01T09:25:18.412 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0075687469999999998 2023-10-01T09:25:18.412 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.412 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.412 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.412 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.547102+0000", 2023-10-01T09:25:18.412 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.005176333 2023-10-01T09:25:18.412 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.412 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.413 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.413 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.552278+0000", 2023-10-01T09:25:18.413 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9648999999999998e-05 2023-10-01T09:25:18.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.413 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.413 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.552328+0000", 2023-10-01T09:25:18.413 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8442000000000006e-05 2023-10-01T09:25:18.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.413 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.413 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.552396+0000", 2023-10-01T09:25:18.414 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4354999999999998e-05 2023-10-01T09:25:18.414 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.414 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.414 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.414 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.552441+0000", 2023-10-01T09:25:18.414 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98289109100000005 2023-10-01T09:25:18.414 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.414 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.414 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.414 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.414 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.414 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:121383 5.a 5:5a1d8215:::benchmark_data_smithi138_240338_object6246:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:25:18.415 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:16:22.574503+0000", 2023-10-01T09:25:18.415 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 535.81369131199995, 2023-10-01T09:25:18.415 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98544083199999999, 2023-10-01T09:25:18.415 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.415 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.415 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.415 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.415 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.415 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121383 2023-10-01T09:25:18.415 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.415 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.415 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.416 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.416 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574503+0000", 2023-10-01T09:25:18.416 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.416 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.416 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.416 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.416 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574503+0000", 2023-10-01T09:25:18.416 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999924 2023-10-01T09:25:18.416 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.416 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.416 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.416 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574496+0000", 2023-10-01T09:25:18.417 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8362000000000001e-05 2023-10-01T09:25:18.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.417 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.417 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574514+0000", 2023-10-01T09:25:18.417 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7633000000000001e-05 2023-10-01T09:25:18.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.417 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.417 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574542+0000", 2023-10-01T09:25:18.417 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.331e-05 2023-10-01T09:25:18.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.418 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.418 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.418 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574555+0000", 2023-10-01T09:25:18.418 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6639999999999996e-05 2023-10-01T09:25:18.418 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.418 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.418 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.418 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574651+0000", 2023-10-01T09:25:18.418 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7617e-05 2023-10-01T09:25:18.418 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.418 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.419 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:18.419 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:22.574679+0000", 2023-10-01T09:25:18.419 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.968492256 2023-10-01T09:25:18.419 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.419 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.419 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.419 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.543171+0000", 2023-10-01T09:25:18.419 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012240600000000001 2023-10-01T09:25:18.419 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.419 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.419 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.419 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.543294+0000", 2023-10-01T09:25:18.420 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016234499999999999 2023-10-01T09:25:18.420 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.420 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.420 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:25:18.420 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.543456+0000", 2023-10-01T09:25:18.420 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003924332 2023-10-01T09:25:18.420 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.420 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.420 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.420 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.547380+0000", 2023-10-01T09:25:18.420 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012435430000000001 2023-10-01T09:25:18.420 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.421 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.421 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.421 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.559816+0000", 2023-10-01T09:25:18.421 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1671000000000001e-05 2023-10-01T09:25:18.421 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.421 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.421 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.421 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.559858+0000", 2023-10-01T09:25:18.421 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9632000000000003e-05 2023-10-01T09:25:18.421 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.421 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.421 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.422 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.559897+0000", 2023-10-01T09:25:18.422 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6953999999999998e-05 2023-10-01T09:25:18.422 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.422 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.422 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.422 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:16:23.559944+0000", 2023-10-01T09:25:18.422 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98544083199999999 2023-10-01T09:25:18.422 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.422 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.422 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.422 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.423 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.423 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:183722 5.e 5:7d170e86:::benchmark_data_smithi138_240338_object68585:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T09:25:18.423 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:17:51.042503+0000", 2023-10-01T09:25:18.423 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 447.34569167900003, 2023-10-01T09:25:18.423 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21231378400000001, 2023-10-01T09:25:18.423 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.423 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.423 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.423 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.423 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.423 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183722 2023-10-01T09:25:18.423 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.424 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.424 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.424 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.424 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.042503+0000", 2023-10-01T09:25:18.424 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.424 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.424 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.424 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.424 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.042503+0000", 2023-10-01T09:25:18.424 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:18.424 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.425 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.425 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.425 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.042501+0000", 2023-10-01T09:25:18.425 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0243e-05 2023-10-01T09:25:18.425 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.425 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.425 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.425 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.042511+0000", 2023-10-01T09:25:18.425 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8862999999999999e-05 2023-10-01T09:25:18.425 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.425 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.426 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.426 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.042530+0000", 2023-10-01T09:25:18.426 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8830000000000006e-06 2023-10-01T09:25:18.426 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.426 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.426 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.426 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.042538+0000", 2023-10-01T09:25:18.426 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2623000000000002e-05 2023-10-01T09:25:18.426 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.426 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.426 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.426 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.042581+0000", 2023-10-01T09:25:18.427 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010574899999999999 2023-10-01T09:25:18.427 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.427 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.427 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.427 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.042686+0000", 2023-10-01T09:25:18.427 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5779000000000006e-05 2023-10-01T09:25:18.427 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.427 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.427 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:25:18.427 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.042782+0000", 2023-10-01T09:25:18.427 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0086101679999999996 2023-10-01T09:25:18.427 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.427 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.428 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.428 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.051392+0000", 2023-10-01T09:25:18.428 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0319000000000001e-05 2023-10-01T09:25:18.428 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.428 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.428 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.428 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.051403+0000", 2023-10-01T09:25:18.428 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20335108299999999 2023-10-01T09:25:18.428 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.428 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.428 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.429 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.254754+0000", 2023-10-01T09:25:18.429 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6742999999999997e-05 2023-10-01T09:25:18.429 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.429 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.429 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.429 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.254791+0000", 2023-10-01T09:25:18.429 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6174999999999999e-05 2023-10-01T09:25:18.429 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.429 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.429 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.429 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.254817+0000", 2023-10-01T09:25:18.429 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21231378400000001 2023-10-01T09:25:18.430 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.430 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.430 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.430 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.430 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.430 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:183724 5.e 5:7e095583:::benchmark_data_smithi138_240338_object68587:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T09:25:18.430 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:17:51.043936+0000", 2023-10-01T09:25:18.430 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 447.34425853900001, 2023-10-01T09:25:18.430 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22755999599999999, 2023-10-01T09:25:18.430 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.430 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.431 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.431 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.431 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.431 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183724 2023-10-01T09:25:18.431 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.431 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.431 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.431 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.431 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.043936+0000", 2023-10-01T09:25:18.431 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.431 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.432 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.432 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.432 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.043936+0000", 2023-10-01T09:25:18.432 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:25:18.432 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.432 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.432 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.432 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.043936+0000", 2023-10-01T09:25:18.432 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.303e-06 2023-10-01T09:25:18.432 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.432 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.432 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.433 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.043939+0000", 2023-10-01T09:25:18.433 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3370000000000002e-06 2023-10-01T09:25:18.433 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.433 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.433 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.433 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.043946+0000", 2023-10-01T09:25:18.433 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5030000000000002e-06 2023-10-01T09:25:18.433 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.433 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.433 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.433 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.043950+0000", 2023-10-01T09:25:18.433 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049412070000000004 2023-10-01T09:25:18.434 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.434 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.434 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.434 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.048891+0000", 2023-10-01T09:25:18.434 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012375400000000001 2023-10-01T09:25:18.434 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.434 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.434 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.434 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.049015+0000", 2023-10-01T09:25:18.434 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014835099999999999 2023-10-01T09:25:18.434 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.434 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.435 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:25:18.435 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.049163+0000", 2023-10-01T09:25:18.435 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0023306030000000001 2023-10-01T09:25:18.435 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.435 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.435 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.435 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.051494+0000", 2023-10-01T09:25:18.435 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000108012 2023-10-01T09:25:18.435 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.435 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.435 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.436 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.051602+0000", 2023-10-01T09:25:18.436 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.219808477 2023-10-01T09:25:18.436 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.436 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.436 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.436 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.271410+0000", 2023-10-01T09:25:18.436 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6455999999999997e-05 2023-10-01T09:25:18.436 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.436 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.436 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.436 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.271457+0000", 2023-10-01T09:25:18.436 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9471999999999997e-05 2023-10-01T09:25:18.437 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.437 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.437 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.437 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:17:51.271496+0000", 2023-10-01T09:25:18.437 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22755999599999999 2023-10-01T09:25:18.437 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.437 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.437 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.437 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.437 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.437 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:196403 5.1 5:8d8835f9:::benchmark_data_smithi138_240338_object81266:head [delete] snapc 0=[] ondisk+write+known_if_redirected e144)", 2023-10-01T09:25:18.437 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:10.956516+0000", 2023-10-01T09:25:18.438 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 427.43167840500001, 2023-10-01T09:25:18.438 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.23079406399999999, 2023-10-01T09:25:18.438 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.438 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.438 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.438 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.438 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.438 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 196403 2023-10-01T09:25:18.438 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.438 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.438 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.438 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.439 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.956516+0000", 2023-10-01T09:25:18.439 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.439 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.439 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.439 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.439 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.956516+0000", 2023-10-01T09:25:18.439 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:25:18.439 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.439 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.439 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.439 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.956516+0000", 2023-10-01T09:25:18.439 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1839999999999999e-06 2023-10-01T09:25:18.440 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.440 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.440 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.440 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.956519+0000", 2023-10-01T09:25:18.440 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1589999999999998e-06 2023-10-01T09:25:18.440 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.440 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.440 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.440 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.956525+0000", 2023-10-01T09:25:18.440 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0600000000000001e-06 2023-10-01T09:25:18.440 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.440 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.441 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.441 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.956529+0000", 2023-10-01T09:25:18.441 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033668600000000002 2023-10-01T09:25:18.441 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.441 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.441 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.441 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.959896+0000", 2023-10-01T09:25:18.441 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000134506 2023-10-01T09:25:18.441 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.441 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.441 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.441 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.960030+0000", 2023-10-01T09:25:18.442 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000156699 2023-10-01T09:25:18.442 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.442 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.442 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,3,7", 2023-10-01T09:25:18.442 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.960187+0000", 2023-10-01T09:25:18.442 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0074670300000000004 2023-10-01T09:25:18.442 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.442 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.442 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.442 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.967654+0000", 2023-10-01T09:25:18.442 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017932399999999999 2023-10-01T09:25:18.443 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.443 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.443 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.443 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.967833+0000", 2023-10-01T09:25:18.443 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00065894399999999996 2023-10-01T09:25:18.443 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.443 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.443 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.443 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:10.968492+0000", 2023-10-01T09:25:18.443 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21862899499999999 2023-10-01T09:25:18.443 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.443 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.444 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.444 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:11.187121+0000", 2023-10-01T09:25:18.444 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7621999999999996e-05 2023-10-01T09:25:18.444 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.444 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.444 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.444 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:11.187199+0000", 2023-10-01T09:25:18.444 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000111229 2023-10-01T09:25:18.444 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.444 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.444 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.444 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:11.187310+0000", 2023-10-01T09:25:18.445 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.23079406399999999 2023-10-01T09:25:18.445 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.445 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.445 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.445 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.445 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.445 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:209442 5.a 5:5b9dfdca:::benchmark_data_smithi138_240338_object94305:head [delete] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-01T09:25:18.445 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:30.410390+0000", 2023-10-01T09:25:18.445 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 407.97780454999997, 2023-10-01T09:25:18.445 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4044485949999999, 2023-10-01T09:25:18.445 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.445 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:25:18.446 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.446 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.446 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.446 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 209442 2023-10-01T09:25:18.446 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.446 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.446 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.446 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.446 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410390+0000", 2023-10-01T09:25:18.446 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.446 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.446 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.447 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.447 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410390+0000", 2023-10-01T09:25:18.447 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:18.447 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.447 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.447 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.447 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410388+0000", 2023-10-01T09:25:18.447 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3309999999999998e-06 2023-10-01T09:25:18.447 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.447 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.447 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.448 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410396+0000", 2023-10-01T09:25:18.448 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8128e-05 2023-10-01T09:25:18.448 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.448 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.448 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.448 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410414+0000", 2023-10-01T09:25:18.448 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5809999999999997e-06 2023-10-01T09:25:18.448 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.448 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.448 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.448 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410422+0000", 2023-10-01T09:25:18.448 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013988600000000001 2023-10-01T09:25:18.449 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.449 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.449 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.449 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410562+0000", 2023-10-01T09:25:18.449 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000141182 2023-10-01T09:25:18.449 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.449 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.449 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.449 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410703+0000", 2023-10-01T09:25:18.449 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016722300000000001 2023-10-01T09:25:18.449 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.449 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.449 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2,3", 2023-10-01T09:25:18.450 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410871+0000", 2023-10-01T09:25:18.450 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0069463420000000003 2023-10-01T09:25:18.450 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.450 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.450 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.450 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.417817+0000", 2023-10-01T09:25:18.450 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0036694050000000001 2023-10-01T09:25:18.450 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.450 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.450 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.450 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.421486+0000", 2023-10-01T09:25:18.451 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002669224 2023-10-01T09:25:18.451 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.451 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.451 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.451 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.424156+0000", 2023-10-01T09:25:18.451 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1552000000000001e-05 2023-10-01T09:25:18.451 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.451 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.451 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:18.451 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.424167+0000", 2023-10-01T09:25:18.451 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98880920299999997 2023-10-01T09:25:18.451 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.452 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.452 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.452 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.412976+0000", 2023-10-01T09:25:18.452 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7869999999999996e-06 2023-10-01T09:25:18.452 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.452 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.452 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:18.452 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.412986+0000", 2023-10-01T09:25:18.452 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.387603428 2023-10-01T09:25:18.452 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.452 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.452 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.453 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.800590+0000", 2023-10-01T09:25:18.453 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012391400000000001 2023-10-01T09:25:18.453 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.453 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.453 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:25:18.453 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.800713+0000", 2023-10-01T09:25:18.453 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014019983999999999 2023-10-01T09:25:18.453 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.453 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.453 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.453 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.814733+0000", 2023-10-01T09:25:18.453 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010518099999999999 2023-10-01T09:25:18.454 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.454 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.454 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.454 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.814839+0000", 2023-10-01T09:25:18.454 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4044485949999999 2023-10-01T09:25:18.454 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.454 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.454 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.454 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.454 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.454 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:209471 5.a 5:5e914fcd:::benchmark_data_smithi138_240338_object94334:head [delete] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-10-01T09:25:18.455 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:32.593876+0000", 2023-10-01T09:25:18.455 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 405.79431869500002, 2023-10-01T09:25:18.455 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22155670199999999, 2023-10-01T09:25:18.455 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.455 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.455 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.455 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.455 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.455 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 209471 2023-10-01T09:25:18.455 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.455 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.456 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.456 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.456 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.593876+0000", 2023-10-01T09:25:18.456 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.456 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.456 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.456 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.456 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.593876+0000", 2023-10-01T09:25:18.456 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:18.456 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.456 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.457 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.457 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.593874+0000", 2023-10-01T09:25:18.457 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0023999999999999e-05 2023-10-01T09:25:18.457 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.457 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.457 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.457 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.593884+0000", 2023-10-01T09:25:18.457 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.1596892 2023-10-01T09:25:18.457 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.457 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.457 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.457 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.753573+0000", 2023-10-01T09:25:18.458 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2102e-05 2023-10-01T09:25:18.458 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.458 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.458 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.458 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.753585+0000", 2023-10-01T09:25:18.458 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.047471116000000001 2023-10-01T09:25:18.458 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.458 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.458 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.458 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.801057+0000", 2023-10-01T09:25:18.458 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012452700000000001 2023-10-01T09:25:18.459 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.459 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.459 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.459 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.801181+0000", 2023-10-01T09:25:18.459 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00060193300000000003 2023-10-01T09:25:18.459 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.459 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.459 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1,2,3", 2023-10-01T09:25:18.459 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.801783+0000", 2023-10-01T09:25:18.459 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012530517999999999 2023-10-01T09:25:18.459 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.459 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.460 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.460 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.814314+0000", 2023-10-01T09:25:18.460 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00056372300000000002 2023-10-01T09:25:18.460 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.460 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.460 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.460 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.814877+0000", 2023-10-01T09:25:18.460 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3500000000000001e-05 2023-10-01T09:25:18.460 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.460 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.460 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.460 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.814911+0000", 2023-10-01T09:25:18.461 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000145752 2023-10-01T09:25:18.461 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.461 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.461 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.461 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.815057+0000", 2023-10-01T09:25:18.461 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016201599999999999 2023-10-01T09:25:18.461 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.461 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.461 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.461 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.815219+0000", 2023-10-01T09:25:18.461 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9009999999999999e-05 2023-10-01T09:25:18.461 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.462 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.462 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.462 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.815278+0000", 2023-10-01T09:25:18.462 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015506499999999999 2023-10-01T09:25:18.462 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.462 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.462 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.462 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.815433+0000", 2023-10-01T09:25:18.462 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22155670199999999 2023-10-01T09:25:18.462 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.462 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.462 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.463 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.463 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.463 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:217879 5.a 5:5a3cd14b:::benchmark_data_smithi138_240338_object102742:head [delete] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-01T09:25:18.463 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:48.025469+0000", 2023-10-01T09:25:18.463 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 390.36272605200003, 2023-10-01T09:25:18.463 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22794552000000001, 2023-10-01T09:25:18.463 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.463 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.463 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.463 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.463 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.463 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 217879 2023-10-01T09:25:18.464 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.464 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.464 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.464 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.464 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025469+0000", 2023-10-01T09:25:18.464 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.464 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.464 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.464 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.464 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025469+0000", 2023-10-01T09:25:18.464 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:18.464 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.465 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.465 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.465 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025467+0000", 2023-10-01T09:25:18.465 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2904e-05 2023-10-01T09:25:18.465 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.465 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.465 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.465 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025480+0000", 2023-10-01T09:25:18.465 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0163999999999999e-05 2023-10-01T09:25:18.465 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.465 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.466 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.466 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025500+0000", 2023-10-01T09:25:18.466 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0749999999999998e-06 2023-10-01T09:25:18.466 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.466 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.466 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.466 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025508+0000", 2023-10-01T09:25:18.466 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.975e-05 2023-10-01T09:25:18.466 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.466 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.466 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.466 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025588+0000", 2023-10-01T09:25:18.467 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013296400000000001 2023-10-01T09:25:18.467 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.467 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.467 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.467 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025721+0000", 2023-10-01T09:25:18.467 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000169004 2023-10-01T09:25:18.467 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.467 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.467 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2,3", 2023-10-01T09:25:18.467 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025890+0000", 2023-10-01T09:25:18.467 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0065930759999999998 2023-10-01T09:25:18.468 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.468 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.468 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.468 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.032483+0000", 2023-10-01T09:25:18.468 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.004438638 2023-10-01T09:25:18.468 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.468 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.468 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.468 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.036922+0000", 2023-10-01T09:25:18.468 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002649907 2023-10-01T09:25:18.468 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.469 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.469 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.469 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.039571+0000", 2023-10-01T09:25:18.469 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21365863500000001 2023-10-01T09:25:18.469 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.469 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.469 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.469 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.253230+0000", 2023-10-01T09:25:18.469 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9175999999999997e-05 2023-10-01T09:25:18.469 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.469 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.469 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.470 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.253289+0000", 2023-10-01T09:25:18.470 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012481100000000001 2023-10-01T09:25:18.470 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.470 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.470 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.470 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.253414+0000", 2023-10-01T09:25:18.470 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22794552000000001 2023-10-01T09:25:18.470 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.470 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.470 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.470 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.471 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.471 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:217883 5.a 5:5e797f10:::benchmark_data_smithi138_240338_object102746:head [delete] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-01T09:25:18.471 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:48.025525+0000", 2023-10-01T09:25:18.471 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 390.36266929200002, 2023-10-01T09:25:18.471 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.233262942, 2023-10-01T09:25:18.471 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.471 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.471 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.471 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.471 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.471 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 217883 2023-10-01T09:25:18.471 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.472 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.472 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.472 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.472 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025525+0000", 2023-10-01T09:25:18.472 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.472 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.472 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.472 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.472 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025525+0000", 2023-10-01T09:25:18.472 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:25:18.472 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.472 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.473 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.473 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025525+0000", 2023-10-01T09:25:18.473 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3440000000000003e-06 2023-10-01T09:25:18.473 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.473 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.473 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.473 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025529+0000", 2023-10-01T09:25:18.473 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4789999999999999e-06 2023-10-01T09:25:18.473 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.473 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.473 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.473 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025537+0000", 2023-10-01T09:25:18.474 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2000000000000002e-06 2023-10-01T09:25:18.474 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.474 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.474 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.474 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.025542+0000", 2023-10-01T09:25:18.474 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035685999999999999 2023-10-01T09:25:18.474 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.474 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.474 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.474 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.029110+0000", 2023-10-01T09:25:18.474 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011079200000000001 2023-10-01T09:25:18.475 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.475 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.475 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.475 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.029221+0000", 2023-10-01T09:25:18.475 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5462000000000002e-05 2023-10-01T09:25:18.475 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.475 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.475 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2,3", 2023-10-01T09:25:18.475 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.029317+0000", 2023-10-01T09:25:18.475 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033038659999999999 2023-10-01T09:25:18.475 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.476 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.476 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.476 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.032620+0000", 2023-10-01T09:25:18.476 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0043369259999999996 2023-10-01T09:25:18.476 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.476 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.476 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.476 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.036957+0000", 2023-10-01T09:25:18.476 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0076043930000000001 2023-10-01T09:25:18.476 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.476 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.476 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.477 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.044562+0000", 2023-10-01T09:25:18.477 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21407159000000001 2023-10-01T09:25:18.477 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.477 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.477 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.477 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.258633+0000", 2023-10-01T09:25:18.477 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1032000000000001e-05 2023-10-01T09:25:18.477 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.477 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.477 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.477 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.258694+0000", 2023-10-01T09:25:18.477 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3832999999999994e-05 2023-10-01T09:25:18.478 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.478 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.478 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.478 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.258788+0000", 2023-10-01T09:25:18.478 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.233262942 2023-10-01T09:25:18.478 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.478 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.478 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.478 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.478 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.478 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:217892 5.1 5:8d886901:::benchmark_data_smithi138_240338_object102755:head [delete] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-01T09:25:18.479 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:48.041798+0000", 2023-10-01T09:25:18.479 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 390.34639709300001, 2023-10-01T09:25:18.479 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21738591099999999, 2023-10-01T09:25:18.479 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.479 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.479 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.479 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.479 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.479 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 217892 2023-10-01T09:25:18.479 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.479 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.479 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.480 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.480 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.041798+0000", 2023-10-01T09:25:18.480 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.480 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.480 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.480 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.480 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.041798+0000", 2023-10-01T09:25:18.480 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:25:18.480 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.480 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.480 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.481 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.041795+0000", 2023-10-01T09:25:18.481 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0363000000000001e-05 2023-10-01T09:25:18.481 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.481 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.481 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.481 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.041806+0000", 2023-10-01T09:25:18.481 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2283e-05 2023-10-01T09:25:18.481 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.481 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.481 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.481 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.041828+0000", 2023-10-01T09:25:18.482 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2959999999999999e-06 2023-10-01T09:25:18.482 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.482 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.482 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.482 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.041836+0000", 2023-10-01T09:25:18.482 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0068073179999999997 2023-10-01T09:25:18.482 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.482 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.482 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.482 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.048643+0000", 2023-10-01T09:25:18.482 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015945 2023-10-01T09:25:18.482 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.483 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.483 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.483 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.048803+0000", 2023-10-01T09:25:18.483 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016377300000000001 2023-10-01T09:25:18.483 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.483 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.483 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,3,7", 2023-10-01T09:25:18.483 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.048967+0000", 2023-10-01T09:25:18.483 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078659520000000007 2023-10-01T09:25:18.483 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.483 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.483 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.484 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.056833+0000", 2023-10-01T09:25:18.484 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0496000000000002e-05 2023-10-01T09:25:18.484 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.484 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.484 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.484 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.056853+0000", 2023-10-01T09:25:18.484 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053033309999999997 2023-10-01T09:25:18.484 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.484 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.484 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.484 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.062156+0000", 2023-10-01T09:25:18.485 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.19680197199999999 2023-10-01T09:25:18.485 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.485 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.485 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.485 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.258958+0000", 2023-10-01T09:25:18.485 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3231999999999994e-05 2023-10-01T09:25:18.485 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.485 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.485 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.485 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.259022+0000", 2023-10-01T09:25:18.485 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016174000000000001 2023-10-01T09:25:18.485 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.486 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.486 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.486 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:48.259183+0000", 2023-10-01T09:25:18.486 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21738591099999999 2023-10-01T09:25:18.486 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.486 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.486 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.486 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.486 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.486 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:220376 5.1 5:8958b2ca:::benchmark_data_smithi138_240338_object105239:head [delete] snapc 0=[] ondisk+write+known_if_redirected e159)", 2023-10-01T09:25:18.486 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:51.780174+0000", 2023-10-01T09:25:18.486 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 386.60802050199999, 2023-10-01T09:25:18.487 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22937580699999999, 2023-10-01T09:25:18.487 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.487 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.487 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.487 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.487 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.487 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 220376 2023-10-01T09:25:18.487 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.487 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.487 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.487 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.487 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.780174+0000", 2023-10-01T09:25:18.488 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.488 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.488 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.488 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.488 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.780174+0000", 2023-10-01T09:25:18.488 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:18.488 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.488 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.488 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.488 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.780173+0000", 2023-10-01T09:25:18.488 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2820000000000002e-06 2023-10-01T09:25:18.489 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.489 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.489 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.489 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.780182+0000", 2023-10-01T09:25:18.489 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7487999999999999e-05 2023-10-01T09:25:18.489 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.489 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.489 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.489 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.780199+0000", 2023-10-01T09:25:18.489 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8260000000000004e-06 2023-10-01T09:25:18.489 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.489 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.490 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.490 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.780206+0000", 2023-10-01T09:25:18.490 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4027999999999999e-05 2023-10-01T09:25:18.490 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.490 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.490 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.490 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.780250+0000", 2023-10-01T09:25:18.490 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013114999999999999 2023-10-01T09:25:18.490 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.490 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.490 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.491 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.780381+0000", 2023-10-01T09:25:18.491 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000178531 2023-10-01T09:25:18.491 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.491 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.491 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,3,7", 2023-10-01T09:25:18.491 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.780560+0000", 2023-10-01T09:25:18.491 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0070060909999999999 2023-10-01T09:25:18.491 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.491 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.491 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.491 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.787566+0000", 2023-10-01T09:25:18.492 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3399999999999997e-06 2023-10-01T09:25:18.492 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.492 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.492 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.492 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.787571+0000", 2023-10-01T09:25:18.492 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011164258 2023-10-01T09:25:18.492 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.492 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.492 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.492 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.798736+0000", 2023-10-01T09:25:18.492 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21058690399999999 2023-10-01T09:25:18.492 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.493 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.493 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.493 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.009323+0000", 2023-10-01T09:25:18.493 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8293000000000004e-05 2023-10-01T09:25:18.493 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.493 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.493 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.493 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.009401+0000", 2023-10-01T09:25:18.493 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000149102 2023-10-01T09:25:18.493 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.493 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.494 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.494 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.009550+0000", 2023-10-01T09:25:18.494 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22937580699999999 2023-10-01T09:25:18.494 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.494 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.494 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.494 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.494 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.494 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:220388 5.a 5:562d32d6:::benchmark_data_smithi138_240338_object105251:head [delete] snapc 0=[] ondisk+write+known_if_redirected e159)", 2023-10-01T09:25:18.494 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:51.786398+0000", 2023-10-01T09:25:18.494 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 386.60179657200001, 2023-10-01T09:25:18.495 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22780567900000001, 2023-10-01T09:25:18.495 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.495 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.495 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.495 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.495 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.495 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 220388 2023-10-01T09:25:18.495 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.495 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.495 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.495 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.495 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.786398+0000", 2023-10-01T09:25:18.496 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.496 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.496 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.496 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.496 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.786398+0000", 2023-10-01T09:25:18.496 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:18.496 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.496 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.496 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.496 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.786397+0000", 2023-10-01T09:25:18.496 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6200000000000004e-06 2023-10-01T09:25:18.496 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.497 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.497 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.497 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.786402+0000", 2023-10-01T09:25:18.497 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7909999999999999e-06 2023-10-01T09:25:18.497 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.497 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.497 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.497 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.786412+0000", 2023-10-01T09:25:18.497 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3590000000000001e-06 2023-10-01T09:25:18.497 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.497 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.497 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.498 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.786417+0000", 2023-10-01T09:25:18.498 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8346999999999997e-05 2023-10-01T09:25:18.498 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.498 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.498 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.498 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.786455+0000", 2023-10-01T09:25:18.498 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000101164 2023-10-01T09:25:18.498 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.498 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.498 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.498 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.786556+0000", 2023-10-01T09:25:18.499 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014020600000000001 2023-10-01T09:25:18.499 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.499 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.499 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2,3", 2023-10-01T09:25:18.499 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.786696+0000", 2023-10-01T09:25:18.499 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050639780000000002 2023-10-01T09:25:18.499 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.499 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.499 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.499 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.791760+0000", 2023-10-01T09:25:18.499 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019407579999999999 2023-10-01T09:25:18.499 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.499 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.500 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.500 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.793701+0000", 2023-10-01T09:25:18.500 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002288258 2023-10-01T09:25:18.500 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.500 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.500 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.500 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.795989+0000", 2023-10-01T09:25:18.500 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21802591599999999 2023-10-01T09:25:18.500 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.500 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.500 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.500 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.014015+0000", 2023-10-01T09:25:18.501 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5146999999999999e-05 2023-10-01T09:25:18.501 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.501 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.501 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.501 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.014070+0000", 2023-10-01T09:25:18.501 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013334299999999999 2023-10-01T09:25:18.501 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.501 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.501 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.501 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.014204+0000", 2023-10-01T09:25:18.501 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22780567900000001 2023-10-01T09:25:18.501 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.501 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.502 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.502 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.502 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.502 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:220389 5.1 5:85ee4571:::benchmark_data_smithi138_240338_object105252:head [delete] snapc 0=[] ondisk+write+known_if_redirected e159)", 2023-10-01T09:25:18.502 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:51.791510+0000", 2023-10-01T09:25:18.502 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 386.59668423599999, 2023-10-01T09:25:18.502 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21830443199999999, 2023-10-01T09:25:18.502 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.502 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.502 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.503 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.503 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.503 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 220389 2023-10-01T09:25:18.503 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.503 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.503 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.503 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.503 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.791510+0000", 2023-10-01T09:25:18.503 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.503 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.503 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.503 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.504 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.791510+0000", 2023-10-01T09:25:18.504 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:18.504 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.504 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.504 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.504 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.791508+0000", 2023-10-01T09:25:18.504 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.713e-06 2023-10-01T09:25:18.504 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.504 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.504 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.504 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.791518+0000", 2023-10-01T09:25:18.505 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1907e-05 2023-10-01T09:25:18.505 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.505 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.505 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.505 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.791530+0000", 2023-10-01T09:25:18.505 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4989999999999999e-06 2023-10-01T09:25:18.505 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.505 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.505 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.505 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.791536+0000", 2023-10-01T09:25:18.505 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000193463 2023-10-01T09:25:18.505 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.506 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.506 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.506 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.791730+0000", 2023-10-01T09:25:18.506 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014087799999999999 2023-10-01T09:25:18.506 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.506 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.506 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.506 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.791871+0000", 2023-10-01T09:25:18.506 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000167064 2023-10-01T09:25:18.506 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.506 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.507 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,3,7", 2023-10-01T09:25:18.507 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.792038+0000", 2023-10-01T09:25:18.507 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0043210310000000004 2023-10-01T09:25:18.507 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.507 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.507 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.507 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.796359+0000", 2023-10-01T09:25:18.507 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001103576 2023-10-01T09:25:18.507 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.507 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.507 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.507 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.797462+0000", 2023-10-01T09:25:18.508 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001367222 2023-10-01T09:25:18.508 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.508 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.508 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.508 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:51.798830+0000", 2023-10-01T09:25:18.508 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.210782635 2023-10-01T09:25:18.508 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.508 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.508 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.508 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.009612+0000", 2023-10-01T09:25:18.508 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7160000000000001e-05 2023-10-01T09:25:18.508 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.509 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.509 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.509 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.009679+0000", 2023-10-01T09:25:18.509 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000135344 2023-10-01T09:25:18.509 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.509 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.509 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.509 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:52.009815+0000", 2023-10-01T09:25:18.509 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21830443199999999 2023-10-01T09:25:18.509 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.509 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.510 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.510 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.510 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.510 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:227511 5.a 5:55308061:::benchmark_data_smithi138_240338_object112374:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T09:25:18.510 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:19:04.539083+0000", 2023-10-01T09:25:18.510 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 373.84911190899999, 2023-10-01T09:25:18.510 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.177536146, 2023-10-01T09:25:18.510 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.510 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:25:18.510 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.510 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.511 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.511 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 227511 2023-10-01T09:25:18.511 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.511 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.511 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.511 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.511 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.539083+0000", 2023-10-01T09:25:18.511 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.511 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.511 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.511 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.511 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.539083+0000", 2023-10-01T09:25:18.512 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:25:18.512 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.512 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.512 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.512 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.539080+0000", 2023-10-01T09:25:18.512 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4380000000000001e-06 2023-10-01T09:25:18.512 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.512 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.512 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.512 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.539090+0000", 2023-10-01T09:25:18.512 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7632e-05 2023-10-01T09:25:18.513 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.513 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.513 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.513 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.539107+0000", 2023-10-01T09:25:18.513 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4159999999999998e-06 2023-10-01T09:25:18.513 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.513 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.513 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.513 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.539115+0000", 2023-10-01T09:25:18.513 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5481000000000005e-05 2023-10-01T09:25:18.513 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.513 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.514 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.514 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.539190+0000", 2023-10-01T09:25:18.514 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016034400000000001 2023-10-01T09:25:18.514 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.514 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.514 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.514 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.539351+0000", 2023-10-01T09:25:18.514 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017084199999999999 2023-10-01T09:25:18.514 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.514 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.514 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2,3", 2023-10-01T09:25:18.515 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.539521+0000", 2023-10-01T09:25:18.515 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050020749999999999 2023-10-01T09:25:18.515 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.515 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.515 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.515 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.544524+0000", 2023-10-01T09:25:18.515 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010673109 2023-10-01T09:25:18.515 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.515 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.515 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.515 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.555197+0000", 2023-10-01T09:25:18.516 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0032218609999999999 2023-10-01T09:25:18.516 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.516 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.516 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.516 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.558418+0000", 2023-10-01T09:25:18.516 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.9609999999999994e-06 2023-10-01T09:25:18.516 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.516 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.516 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:18.516 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.558427+0000", 2023-10-01T09:25:18.516 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1482362669999999 2023-10-01T09:25:18.516 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.517 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.517 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.517 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.706664+0000", 2023-10-01T09:25:18.517 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011240599999999999 2023-10-01T09:25:18.517 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.517 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.517 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:25:18.517 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.706776+0000", 2023-10-01T09:25:18.517 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0097701679999999992 2023-10-01T09:25:18.517 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.517 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.517 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.517 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.716546+0000", 2023-10-01T09:25:18.518 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2569000000000003e-05 2023-10-01T09:25:18.518 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.518 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.518 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.518 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.716619+0000", 2023-10-01T09:25:18.518 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.177536146 2023-10-01T09:25:18.518 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.518 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.518 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.518 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.518 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.519 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:227546 5.a 5:5c089ba1:::benchmark_data_smithi138_240338_object112409:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:25:18.519 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:19:04.603475+0000", 2023-10-01T09:25:18.519 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 373.78471926999998, 2023-10-01T09:25:18.519 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.113392436, 2023-10-01T09:25:18.519 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.519 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.519 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.519 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:25:18.519 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:25:18.519 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 227546 2023-10-01T09:25:18.519 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.520 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.520 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.520 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.520 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.603475+0000", 2023-10-01T09:25:18.520 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.520 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.520 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.520 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.520 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.603475+0000", 2023-10-01T09:25:18.520 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:18.520 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.520 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.521 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.521 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.603474+0000", 2023-10-01T09:25:18.521 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.002e-06 2023-10-01T09:25:18.521 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.521 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.521 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.521 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.603481+0000", 2023-10-01T09:25:18.521 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.26e-05 2023-10-01T09:25:18.521 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.521 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.521 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.521 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.603494+0000", 2023-10-01T09:25:18.522 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3589999999999999e-06 2023-10-01T09:25:18.522 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.522 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.522 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.522 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.603499+0000", 2023-10-01T09:25:18.522 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7911000000000002e-05 2023-10-01T09:25:18.522 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.522 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.522 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.522 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.603557+0000", 2023-10-01T09:25:18.522 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1946999999999999e-05 2023-10-01T09:25:18.523 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.523 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.523 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:18.523 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.603569+0000", 2023-10-01T09:25:18.523 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1032517159999999 2023-10-01T09:25:18.523 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.523 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.523 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.523 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.706821+0000", 2023-10-01T09:25:18.523 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4605000000000006e-05 2023-10-01T09:25:18.523 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.523 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.523 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.524 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.706915+0000", 2023-10-01T09:25:18.524 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00055120600000000005 2023-10-01T09:25:18.524 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.524 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.524 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,1,2,3", 2023-10-01T09:25:18.524 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.707466+0000", 2023-10-01T09:25:18.524 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0048134830000000003 2023-10-01T09:25:18.524 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.524 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.524 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.524 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.712280+0000", 2023-10-01T09:25:18.524 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00060836799999999997 2023-10-01T09:25:18.525 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.525 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.525 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.525 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.712888+0000", 2023-10-01T09:25:18.525 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00062693000000000004 2023-10-01T09:25:18.525 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.525 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.525 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.525 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.713515+0000", 2023-10-01T09:25:18.525 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00038367799999999998 2023-10-01T09:25:18.525 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.526 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.526 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.526 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.713899+0000", 2023-10-01T09:25:18.526 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0028425030000000001 2023-10-01T09:25:18.526 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.526 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.526 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.526 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.716741+0000", 2023-10-01T09:25:18.526 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6937000000000002e-05 2023-10-01T09:25:18.526 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.526 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.526 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.527 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.716778+0000", 2023-10-01T09:25:18.527 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.9543000000000006e-05 2023-10-01T09:25:18.527 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.527 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.527 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.527 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.716868+0000", 2023-10-01T09:25:18.527 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.113392436 2023-10-01T09:25:18.527 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.527 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.527 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.527 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.528 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.528 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:6242 6.2 6:4aaab915:::benchmark_data_smithi138_307751_object6241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-10-01T09:25:18.528 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:19:46.456405+0000", 2023-10-01T09:25:18.528 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 331.93178933600001, 2023-10-01T09:25:18.528 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.32850854299999999, 2023-10-01T09:25:18.528 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.528 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.528 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.528 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:18.528 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:18.528 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 6242 2023-10-01T09:25:18.528 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.529 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.529 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.529 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.529 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.456405+0000", 2023-10-01T09:25:18.529 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.529 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.529 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.529 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.529 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.456405+0000", 2023-10-01T09:25:18.529 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:18.529 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.529 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.530 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.530 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.456403+0000", 2023-10-01T09:25:18.530 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9711000000000001e-05 2023-10-01T09:25:18.530 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.530 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.530 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.530 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.456433+0000", 2023-10-01T09:25:18.530 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6310000000000001e-05 2023-10-01T09:25:18.530 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.530 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.530 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.531 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.456449+0000", 2023-10-01T09:25:18.531 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5389999999999998e-06 2023-10-01T09:25:18.531 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.531 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.531 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.531 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.456455+0000", 2023-10-01T09:25:18.531 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7040999999999999e-05 2023-10-01T09:25:18.531 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.531 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.531 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.531 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.456502+0000", 2023-10-01T09:25:18.531 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011207800000000001 2023-10-01T09:25:18.532 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.532 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.532 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.532 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.456614+0000", 2023-10-01T09:25:18.532 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000148751 2023-10-01T09:25:18.532 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.532 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.532 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:25:18.532 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.456763+0000", 2023-10-01T09:25:18.532 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00098778699999999995 2023-10-01T09:25:18.532 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.532 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.533 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.533 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.457751+0000", 2023-10-01T09:25:18.533 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7686e-05 2023-10-01T09:25:18.533 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.533 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.533 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.533 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.457768+0000", 2023-10-01T09:25:18.533 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.32702989100000002 2023-10-01T09:25:18.533 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.533 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.533 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.534 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.784798+0000", 2023-10-01T09:25:18.534 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7269999999999999e-05 2023-10-01T09:25:18.534 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.534 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.534 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.534 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.784856+0000", 2023-10-01T09:25:18.534 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8312e-05 2023-10-01T09:25:18.534 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.534 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.534 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.534 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.784914+0000", 2023-10-01T09:25:18.534 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.32850854299999999 2023-10-01T09:25:18.535 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.535 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.535 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.535 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.535 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.535 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:6259 6.2 6:447c463e:::benchmark_data_smithi138_307751_object6258:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-10-01T09:25:18.535 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:19:46.462431+0000", 2023-10-01T09:25:18.535 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 331.92576408000002, 2023-10-01T09:25:18.535 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.32333591, 2023-10-01T09:25:18.535 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.535 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.536 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:18.536 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:18.536 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:18.536 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 6259 2023-10-01T09:25:18.536 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.536 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.536 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.536 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.536 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.462431+0000", 2023-10-01T09:25:18.536 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.536 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.536 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.537 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.537 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.462431+0000", 2023-10-01T09:25:18.537 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:18.537 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.537 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.537 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.537 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.462429+0000", 2023-10-01T09:25:18.537 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4493000000000006e-05 2023-10-01T09:25:18.537 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.537 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.537 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.537 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.462513+0000", 2023-10-01T09:25:18.538 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5303999999999999e-05 2023-10-01T09:25:18.538 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.538 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.538 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.538 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.462549+0000", 2023-10-01T09:25:18.538 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2620000000000001e-06 2023-10-01T09:25:18.538 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.538 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.538 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.538 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.462556+0000", 2023-10-01T09:25:18.538 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3851999999999998e-05 2023-10-01T09:25:18.539 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.539 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.539 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.539 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.462620+0000", 2023-10-01T09:25:18.539 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000136229 2023-10-01T09:25:18.539 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.539 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.539 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.539 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.462756+0000", 2023-10-01T09:25:18.539 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015310899999999999 2023-10-01T09:25:18.539 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.539 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.540 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:25:18.540 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.462909+0000", 2023-10-01T09:25:18.540 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00061291199999999996 2023-10-01T09:25:18.540 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.540 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.540 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:18.540 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.463522+0000", 2023-10-01T09:25:18.540 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00028946700000000001 2023-10-01T09:25:18.540 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.540 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.540 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.540 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.463811+0000", 2023-10-01T09:25:18.541 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.32186103799999999 2023-10-01T09:25:18.541 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.541 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.541 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:18.541 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.785672+0000", 2023-10-01T09:25:18.541 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3674999999999997e-05 2023-10-01T09:25:18.541 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.541 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.541 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.541 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.785726+0000", 2023-10-01T09:25:18.541 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0339000000000001e-05 2023-10-01T09:25:18.541 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.542 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.542 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.542 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:46.785766+0000", 2023-10-01T09:25:18.542 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.32333591 2023-10-01T09:25:18.542 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.542 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.542 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.542 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.542 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.542 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:92413 7.4 e270/259 7:20f93266:::benchmark_data_smithi138_367058_object92412:head v 270'5843, mlcod=270'5840)", 2023-10-01T09:25:18.542 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:25:16.608434+0000", 2023-10-01T09:25:18.543 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.7797609599999999, 2023-10-01T09:25:18.543 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0089582480000000003, 2023-10-01T09:25:18.543 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:18.543 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:18.543 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:18.543 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.543 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:18.543 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:16.608434+0000", 2023-10-01T09:25:18.543 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:18.543 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.543 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.543 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:18.544 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:16.608434+0000", 2023-10-01T09:25:18.544 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:25:18.544 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.544 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.544 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:18.544 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:16.608433+0000", 2023-10-01T09:25:18.544 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9521999999999999e-05 2023-10-01T09:25:18.544 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.544 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.544 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:18.544 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:16.608473+0000", 2023-10-01T09:25:18.544 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5679999999999996e-06 2023-10-01T09:25:18.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.545 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:18.545 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:16.608479+0000", 2023-10-01T09:25:18.545 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.529e-06 2023-10-01T09:25:18.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.545 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:18.545 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:16.608483+0000", 2023-10-01T09:25:18.545 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024639110000000001 2023-10-01T09:25:18.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.546 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:18.546 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:16.610947+0000", 2023-10-01T09:25:18.546 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9531e-05 2023-10-01T09:25:18.546 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.546 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:18.546 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:16.610966+0000", 2023-10-01T09:25:18.546 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0063875340000000003 2023-10-01T09:25:18.546 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.546 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:18.547 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:16.617354+0000", 2023-10-01T09:25:18.547 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7987000000000001e-05 2023-10-01T09:25:18.547 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:18.547 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:18.547 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:18.547 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:16.617392+0000", 2023-10-01T09:25:18.547 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0089582480000000003 2023-10-01T09:25:18.547 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.547 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.547 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.547 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:18.547 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:18.548 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:25:18.548 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:25:18.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:18 smithi190 ceph-mon[160435]: pgmap v709: 89 pgs: 11 active+undersized+degraded, 21 active+undersized, 57 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 28469/277005 objects degraded (10.277%) 2023-10-01T09:25:18.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:18 smithi138 ceph-mon[192513]: pgmap v709: 89 pgs: 11 active+undersized+degraded, 21 active+undersized, 57 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 28469/277005 objects degraded (10.277%) 2023-10-01T09:25:18.633 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:18.730 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:25:18.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:18.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:25:18.839 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:18.906 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:18.946 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 85 16 94259 94243 69.2364 51 0.0377356 0.0144379 2023-10-01T09:25:19.045 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:19.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:19.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:19.154 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:25:19.155 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:25:19.196 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:25:19.226 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:19.265 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:25:19.472 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:25:19.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:19.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:25:19.479 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:19 smithi138 ceph-mon[192513]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:25:19.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:19 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/188041138' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:25:19.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:19 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:25:19.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:19 smithi138 ceph-mon[185886]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:25:19.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/188041138' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:25:19.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:19 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:25:19.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:19 smithi190 ceph-mon[160435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:25:19.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:19 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/188041138' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:25:19.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:19 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:25:19.574 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:25:19.580 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:19.623 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:19.719 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:19.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:19.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:19.762 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 5, 7, 2, 6] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 7, 5] 2023-10-01T09:25:19.762 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:25:19.762 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 3, 4, 5, 7, 2, 6] 2023-10-01T09:25:19.762 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd out 5 2023-10-01T09:25:19.823 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:19.917 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:19.947 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 86 16 95038 95022 68.9976 48.6875 0.0045905 0.0144819 2023-10-01T09:25:20.241 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:20 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:25:20.241 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:20 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:25:20.241 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:20 smithi138 ceph-mon[185886]: osdmap e271: 8 total, 7 up, 8 in 2023-10-01T09:25:20.241 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:20 smithi138 ceph-mon[185886]: failed to encode map e271 with expected crc 2023-10-01T09:25:20.241 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:20 smithi138 ceph-mon[185886]: failed to encode map e271 with expected crc 2023-10-01T09:25:20.241 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:20 smithi138 ceph-mon[185886]: pgmap v711: 89 pgs: 11 active+undersized+degraded, 21 active+undersized, 57 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 728 op/s; 28948/278442 objects degraded (10.396%); 0 B/s, 0 objects/s recovering 2023-10-01T09:25:20.471 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:20.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:20.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:20.530 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:20 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:25:20.530 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:20 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:25:20.530 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:20 smithi138 ceph-mon[192513]: osdmap e271: 8 total, 7 up, 8 in 2023-10-01T09:25:20.530 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:20 smithi138 ceph-mon[192513]: failed to encode map e271 with expected crc 2023-10-01T09:25:20.530 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:20 smithi138 ceph-mon[192513]: failed to encode map e271 with expected crc 2023-10-01T09:25:20.530 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:20 smithi138 ceph-mon[192513]: pgmap v711: 89 pgs: 11 active+undersized+degraded, 21 active+undersized, 57 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 728 op/s; 28948/278442 objects degraded (10.396%); 0 B/s, 0 objects/s recovering 2023-10-01T09:25:20.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:20 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:25:20.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:20 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:25:20.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:20 smithi190 ceph-mon[160435]: osdmap e271: 8 total, 7 up, 8 in 2023-10-01T09:25:20.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:20 smithi190 ceph-mon[160435]: failed to encode map e271 with expected crc 2023-10-01T09:25:20.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:20 smithi190 ceph-mon[160435]: failed to encode map e271 with expected crc 2023-10-01T09:25:20.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:20 smithi190 ceph-mon[160435]: pgmap v711: 89 pgs: 11 active+undersized+degraded, 21 active+undersized, 57 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 728 op/s; 28948/278442 objects degraded (10.396%); 0 B/s, 0 objects/s recovering 2023-10-01T09:25:20.577 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:20.634 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:20.731 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:20.742 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:20.814 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:20.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:20.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:20.921 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:20.947 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 87 16 95635 95619 68.6336 37.3125 0.00831571 0.0145655 2023-10-01T09:25:20.958 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:21.010 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:21.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:21.115 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:21.145 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:25:21.145 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:25:21.145 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 0 2023-10-01T09:25:21.145 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:25:21.194 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:21.550 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:21.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:21 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 28948/278442 objects degraded (10.396%), 11 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:21.563 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:21 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 28948/278442 objects degraded (10.396%), 11 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:21.563 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:21 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 28948/278442 objects degraded (10.396%), 11 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:21.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:21.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:21.665 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:21.697 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:21.844 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:25:21.847 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:21.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:21.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:21.947 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 88 16 96439 96423 68.4248 50.25 0.00814039 0.0146085 2023-10-01T09:25:21.956 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:21.989 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:22.053 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:22.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:22.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:22.158 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:22.186 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:22.233 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:22.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:22.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:22.300 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:25:22.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:22 smithi138 ceph-mon[192513]: pgmap v712: 89 pgs: 11 active+undersized+degraded, 21 active+undersized, 57 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 28948/281259 objects degraded (10.292%); 0 B/s, 0 objects/s recovering 2023-10-01T09:25:22.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:22 smithi138 ceph-mon[185886]: pgmap v712: 89 pgs: 11 active+undersized+degraded, 21 active+undersized, 57 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 28948/281259 objects degraded (10.292%); 0 B/s, 0 objects/s recovering 2023-10-01T09:25:22.338 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:22.370 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:22.439 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:22.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:22.512 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:22 smithi190 ceph-mon[160435]: pgmap v712: 89 pgs: 11 active+undersized+degraded, 21 active+undersized, 57 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 39 MiB/s wr, 619 op/s; 28948/281259 objects degraded (10.292%); 0 B/s, 0 objects/s recovering 2023-10-01T09:25:22.544 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:22.564 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:25:22.575 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:22.683 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:22.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:22.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:22.787 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:22.845 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:22.948 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 89 16 97121 97105 68.1338 42.625 0.0186735 0.0146717 2023-10-01T09:25:23.237 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:23.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:23.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:23.346 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:23.391 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:23.475 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:23.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:23.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:23.579 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:23.692 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:23.798 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:25:23.798 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:25:23.798 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:25:23.798 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:25:23.799 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:25:23.855 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:23.866 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:23 smithi138 ceph-mon[192513]: pgmap v713: 89 pgs: 11 active+undersized+degraded, 21 active+undersized, 57 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 741 op/s; 29657/288351 objects degraded (10.285%) 2023-10-01T09:25:23.866 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:23 smithi138 ceph-mon[185886]: pgmap v713: 89 pgs: 11 active+undersized+degraded, 21 active+undersized, 57 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 741 op/s; 29657/288351 objects degraded (10.285%) 2023-10-01T09:25:23.949 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Total time run: 90.0162 2023-10-01T09:25:23.949 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Total writes made: 97792 2023-10-01T09:25:23.949 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Write size: 65536 2023-10-01T09:25:23.949 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Object size: 65536 2023-10-01T09:25:23.949 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Bandwidth (MB/sec): 67.8989 2023-10-01T09:25:23.949 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev Bandwidth: 31.7832 2023-10-01T09:25:23.950 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max bandwidth (MB/sec): 196.625 2023-10-01T09:25:23.950 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min bandwidth (MB/sec): 19.3125 2023-10-01T09:25:23.950 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Average IOPS: 1086 2023-10-01T09:25:23.950 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev IOPS: 508.531 2023-10-01T09:25:23.950 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max IOPS: 3146 2023-10-01T09:25:23.950 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min IOPS: 309 2023-10-01T09:25:23.950 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Average Latency(s): 0.014724 2023-10-01T09:25:23.950 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev Latency(s): 0.0224058 2023-10-01T09:25:23.950 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max latency(s): 1.0548 2023-10-01T09:25:23.950 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min latency(s): 0.0011387 2023-10-01T09:25:23.953 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:25:23.954 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T09:25:23.985 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:25:23.985 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:23 smithi190 ceph-mon[160435]: pgmap v713: 89 pgs: 11 active+undersized+degraded, 21 active+undersized, 57 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 741 op/s; 29657/288351 objects degraded (10.285%) 2023-10-01T09:25:24.006 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:24.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:24.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:24.058 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:25:24.107 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:25:24.122 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:24.155 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:24.164 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:25:24.198 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:25:24.209 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:24.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:24.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:24.228 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:25:24.228 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:25:24.228 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:25:24.228 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:25:24.228 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:25:24.228 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:25:24.228 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:25:24.229 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-10-01T09:25:24.229 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:25:24.312 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:24.400 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:24.523 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:24 smithi138 conmon[185862]: 2023-10-01T09:25:24.439+0000 7f70d64ec700 -1 mon.a@0(leader).osd e271 definitely_dead 0 2023-10-01T09:25:24.569 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:25:24.720 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:24.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:24.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:24.785 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1650328410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:24.785 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3790401035' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-01T09:25:24.785 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:24 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-01T09:25:24.785 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:24 smithi138 ceph-mon[185886]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-01T09:25:24.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1650328410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:24.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3790401035' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-01T09:25:24.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:24 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-01T09:25:24.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:24 smithi138 ceph-mon[192513]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-01T09:25:24.804 INFO:teuthology.orchestra.run.smithi138.stderr:marked out osd.5. 2023-10-01T09:25:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1650328410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3790401035' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-01T09:25:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:24 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-10-01T09:25:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:24 smithi190 ceph-mon[160435]: Client client.admin marked osd.5 out, while it was still marked up 2023-10-01T09:25:24.824 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:24.877 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:24.948 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:25:25.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:25:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:25:24] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-01T09:25:25.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-01T09:25:25.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[185886]: osdmap e272: 8 total, 7 up, 7 in 2023-10-01T09:25:25.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[185886]: failed to encode map e272 with expected crc 2023-10-01T09:25:25.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:25:25.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:25:25.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:25:25.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[185886]: pgmap v715: 89 pgs: 1 peering, 1 unknown, 11 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 35 MiB/s wr, 573 op/s; 30065/290286 objects degraded (10.357%) 2023-10-01T09:25:25.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[185886]: osdmap e273: 8 total, 7 up, 7 in 2023-10-01T09:25:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:25 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-01T09:25:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:25 smithi190 ceph-mon[160435]: osdmap e272: 8 total, 7 up, 7 in 2023-10-01T09:25:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:25 smithi190 ceph-mon[160435]: failed to encode map e272 with expected crc 2023-10-01T09:25:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:25:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:25:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:25:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:25 smithi190 ceph-mon[160435]: pgmap v715: 89 pgs: 1 peering, 1 unknown, 11 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 35 MiB/s wr, 573 op/s; 30065/290286 objects degraded (10.357%) 2023-10-01T09:25:26.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:25 smithi190 ceph-mon[160435]: osdmap e273: 8 total, 7 up, 7 in 2023-10-01T09:25:26.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-10-01T09:25:26.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[192513]: osdmap e272: 8 total, 7 up, 7 in 2023-10-01T09:25:26.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[192513]: failed to encode map e272 with expected crc 2023-10-01T09:25:26.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:25:26.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:25:26.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:25:26.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[192513]: pgmap v715: 89 pgs: 1 peering, 1 unknown, 11 active+undersized+degraded, 21 active+undersized, 55 active+clean; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 35 MiB/s wr, 573 op/s; 30065/290286 objects degraded (10.357%) 2023-10-01T09:25:26.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:25 smithi138 ceph-mon[192513]: osdmap e273: 8 total, 7 up, 7 in 2023-10-01T09:25:26.252 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:25:26.253 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:25:26.394 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:25:26.394 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:25:26.447 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:26.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:26.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:26.551 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:25:26.579 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:26.726 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:25:26.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:26.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:25:26.800 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:26 smithi138 ceph-mon[185886]: failed to encode map e273 with expected crc 2023-10-01T09:25:26.800 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:26 smithi138 ceph-mon[185886]: failed to encode map e273 with expected crc 2023-10-01T09:25:26.801 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:26 smithi138 ceph-mon[185886]: failed to encode map e273 with expected crc 2023-10-01T09:25:26.801 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:26 smithi138 ceph-mon[185886]: failed to encode map e273 with expected crc 2023-10-01T09:25:26.801 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:26 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 30065/290286 objects degraded (10.357%), 11 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:26.801 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:26 smithi138 ceph-mon[185886]: osdmap e274: 8 total, 7 up, 7 in 2023-10-01T09:25:26.830 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:26.860 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:26 smithi190 ceph-mon[160435]: failed to encode map e273 with expected crc 2023-10-01T09:25:26.860 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:26 smithi190 ceph-mon[160435]: failed to encode map e273 with expected crc 2023-10-01T09:25:26.860 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:26 smithi190 ceph-mon[160435]: failed to encode map e273 with expected crc 2023-10-01T09:25:26.860 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:26 smithi190 ceph-mon[160435]: failed to encode map e273 with expected crc 2023-10-01T09:25:26.860 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:26 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 30065/290286 objects degraded (10.357%), 11 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:26.861 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:26 smithi190 ceph-mon[160435]: osdmap e274: 8 total, 7 up, 7 in 2023-10-01T09:25:26.864 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:26.978 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:26.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:26.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:27.033 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:25:27.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:26 smithi138 ceph-mon[192513]: failed to encode map e273 with expected crc 2023-10-01T09:25:27.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:26 smithi138 ceph-mon[192513]: failed to encode map e273 with expected crc 2023-10-01T09:25:27.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:26 smithi138 ceph-mon[192513]: failed to encode map e273 with expected crc 2023-10-01T09:25:27.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:26 smithi138 ceph-mon[192513]: failed to encode map e273 with expected crc 2023-10-01T09:25:27.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:26 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 30065/290286 objects degraded (10.357%), 11 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:27.080 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:26 smithi138 ceph-mon[192513]: osdmap e274: 8 total, 7 up, 7 in 2023-10-01T09:25:27.082 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:27.182 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:27.393 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:27.407 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:25:27.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:27.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:27.512 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:27.593 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:27.710 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:27.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:27.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:27.828 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:27.905 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[192513]: failed to encode map e274 with expected crc 2023-10-01T09:25:27.906 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[192513]: failed to encode map e274 with expected crc 2023-10-01T09:25:27.906 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[192513]: failed to encode map e274 with expected crc 2023-10-01T09:25:27.906 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[192513]: failed to encode map e274 with expected crc 2023-10-01T09:25:27.906 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[192513]: osdmap e275: 8 total, 7 up, 7 in 2023-10-01T09:25:27.906 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[192513]: failed to encode map e275 with expected crc 2023-10-01T09:25:27.906 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[192513]: failed to encode map e275 with expected crc 2023-10-01T09:25:27.906 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3893531749' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:27.906 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:27.906 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[192513]: pgmap v719: 89 pgs: 7 active+clean+remapped, 3 remapped+peering, 1 peering, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 17 active+undersized, 50 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 24 MiB/s wr, 533 op/s; 30157/291189 objects degraded (10.357%); 6038/291189 objects misplaced (2.074%); 48 KiB/s, 0 objects/s recovering 2023-10-01T09:25:27.907 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[185886]: failed to encode map e274 with expected crc 2023-10-01T09:25:27.907 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[185886]: failed to encode map e274 with expected crc 2023-10-01T09:25:27.907 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[185886]: failed to encode map e274 with expected crc 2023-10-01T09:25:27.907 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[185886]: failed to encode map e274 with expected crc 2023-10-01T09:25:27.907 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[185886]: osdmap e275: 8 total, 7 up, 7 in 2023-10-01T09:25:27.907 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[185886]: failed to encode map e275 with expected crc 2023-10-01T09:25:27.907 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[185886]: failed to encode map e275 with expected crc 2023-10-01T09:25:27.907 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3893531749' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:27.907 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:27.907 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:27 smithi138 ceph-mon[185886]: pgmap v719: 89 pgs: 7 active+clean+remapped, 3 remapped+peering, 1 peering, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 17 active+undersized, 50 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 24 MiB/s wr, 533 op/s; 30157/291189 objects degraded (10.357%); 6038/291189 objects misplaced (2.074%); 48 KiB/s, 0 objects/s recovering 2023-10-01T09:25:27.919 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:27.985 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:27 smithi190 ceph-mon[160435]: failed to encode map e274 with expected crc 2023-10-01T09:25:27.985 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:27 smithi190 ceph-mon[160435]: failed to encode map e274 with expected crc 2023-10-01T09:25:27.985 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:27 smithi190 ceph-mon[160435]: failed to encode map e274 with expected crc 2023-10-01T09:25:27.985 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:27 smithi190 ceph-mon[160435]: failed to encode map e274 with expected crc 2023-10-01T09:25:27.986 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:27 smithi190 ceph-mon[160435]: osdmap e275: 8 total, 7 up, 7 in 2023-10-01T09:25:27.986 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:27 smithi190 ceph-mon[160435]: failed to encode map e275 with expected crc 2023-10-01T09:25:27.986 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:27 smithi190 ceph-mon[160435]: failed to encode map e275 with expected crc 2023-10-01T09:25:27.986 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:27 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3893531749' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:27.986 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:27 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:27.986 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:27 smithi190 ceph-mon[160435]: pgmap v719: 89 pgs: 7 active+clean+remapped, 3 remapped+peering, 1 peering, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 17 active+undersized, 50 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 24 MiB/s wr, 533 op/s; 30157/291189 objects degraded (10.357%); 6038/291189 objects misplaced (2.074%); 48 KiB/s, 0 objects/s recovering 2023-10-01T09:25:28.040 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:28.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:28.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:28.156 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:28.242 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:28.254 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:25:28.291 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:25:28.365 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:28.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:28.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:28.490 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:28.635 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:28.778 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:28.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:28.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:28.889 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:28.930 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:25:28.931 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:25:28.987 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:28 smithi190 ceph-mon[160435]: failed to encode map e275 with expected crc 2023-10-01T09:25:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:28 smithi190 ceph-mon[160435]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:25:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:28 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:25:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:28 smithi190 ceph-mon[160435]: failed to encode map e276 with expected crc 2023-10-01T09:25:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:28 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:25:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:28 smithi190 ceph-mon[160435]: osdmap e276: 8 total, 7 up, 7 in 2023-10-01T09:25:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:28 smithi190 ceph-mon[160435]: failed to encode map e276 with expected crc 2023-10-01T09:25:29.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:28 smithi138 ceph-mon[185886]: failed to encode map e275 with expected crc 2023-10-01T09:25:29.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:28 smithi138 ceph-mon[185886]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:25:29.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:28 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:25:29.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:28 smithi138 ceph-mon[185886]: failed to encode map e276 with expected crc 2023-10-01T09:25:29.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:28 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:25:29.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:28 smithi138 ceph-mon[185886]: osdmap e276: 8 total, 7 up, 7 in 2023-10-01T09:25:29.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:28 smithi138 ceph-mon[185886]: failed to encode map e276 with expected crc 2023-10-01T09:25:29.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:28 smithi138 ceph-mon[192513]: failed to encode map e275 with expected crc 2023-10-01T09:25:29.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:28 smithi138 ceph-mon[192513]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:25:29.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:28 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:25:29.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:28 smithi138 ceph-mon[192513]: failed to encode map e276 with expected crc 2023-10-01T09:25:29.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:28 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:25:29.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:28 smithi138 ceph-mon[192513]: osdmap e276: 8 total, 7 up, 7 in 2023-10-01T09:25:29.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:28 smithi138 ceph-mon[192513]: failed to encode map e276 with expected crc 2023-10-01T09:25:29.624 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T09:25:29.965 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:29.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:29.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:30.009 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:25:30.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:29 smithi190 ceph-mon[160435]: failed to encode map e276 with expected crc 2023-10-01T09:25:30.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:29 smithi190 ceph-mon[160435]: pgmap v721: 89 pgs: 12 active+clean+remapped, 3 remapped+peering, 1 peering, 1 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 48 active+clean; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 255 B/s rd, 15 MiB/s wr, 437 op/s; 36002/290901 objects degraded (12.376%); 12010/290901 objects misplaced (4.129%); 48 KiB/s, 7 objects/s recovering 2023-10-01T09:25:30.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:29 smithi138 ceph-mon[185886]: failed to encode map e276 with expected crc 2023-10-01T09:25:30.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:29 smithi138 ceph-mon[185886]: pgmap v721: 89 pgs: 12 active+clean+remapped, 3 remapped+peering, 1 peering, 1 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 48 active+clean; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 255 B/s rd, 15 MiB/s wr, 437 op/s; 36002/290901 objects degraded (12.376%); 12010/290901 objects misplaced (4.129%); 48 KiB/s, 7 objects/s recovering 2023-10-01T09:25:30.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:29 smithi138 ceph-mon[192513]: failed to encode map e276 with expected crc 2023-10-01T09:25:30.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:29 smithi138 ceph-mon[192513]: pgmap v721: 89 pgs: 12 active+clean+remapped, 3 remapped+peering, 1 peering, 1 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 48 active+clean; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 255 B/s rd, 15 MiB/s wr, 437 op/s; 36002/290901 objects degraded (12.376%); 12010/290901 objects misplaced (4.129%); 48 KiB/s, 7 objects/s recovering 2023-10-01T09:25:30.083 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:30.112 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:30.157 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:30.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:30.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:30.261 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:30.309 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:30.397 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:30.617 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:30.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:30.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:30.736 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:30.765 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:30.810 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:30.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:30.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:30.914 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:30.942 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:31.107 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:31.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:31.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:31.211 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:31.243 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:31.308 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:31.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:31.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:31.411 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:31.443 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:31.520 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 7, 2, 6] out_osds: [5] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 7, 5] 2023-10-01T09:25:31.521 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:25:31.521 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-01T09:25:31.521 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:25:31.521 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:25:31.521 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-01T09:25:31.538 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:31.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:31.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:31.626 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:25:31.626 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:25:31.642 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:31.732 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:31.900 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:31.904 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:31 smithi138 ceph-mon[185886]: pgmap v722: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 172 B/s rd, 10 MiB/s wr, 348 op/s; 36040/289944 objects degraded (12.430%); 18029/289944 objects misplaced (6.218%); 755 KiB/s, 18 objects/s recovering 2023-10-01T09:25:31.904 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:31 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 36040/289944 objects degraded (12.430%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:31.904 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:31 smithi138 ceph-mon[185886]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:25:31.904 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:31 smithi138 ceph-mon[192513]: pgmap v722: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 172 B/s rd, 10 MiB/s wr, 348 op/s; 36040/289944 objects degraded (12.430%); 18029/289944 objects misplaced (6.218%); 755 KiB/s, 18 objects/s recovering 2023-10-01T09:25:31.905 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:31 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 36040/289944 objects degraded (12.430%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:31.905 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:31 smithi138 ceph-mon[192513]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:25:31.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:31.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:32.006 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:32.031 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:31 smithi190 ceph-mon[160435]: pgmap v722: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 172 B/s rd, 10 MiB/s wr, 348 op/s; 36040/289944 objects degraded (12.430%); 18029/289944 objects misplaced (6.218%); 755 KiB/s, 18 objects/s recovering 2023-10-01T09:25:32.031 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:31 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 36040/289944 objects degraded (12.430%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:32.031 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:31 smithi190 ceph-mon[160435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:25:32.033 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:32.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:25:32.311 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T09:25:32.322 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:32.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:32.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:32.427 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:32.455 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:32.504 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:32.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:32.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:32.609 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:32.700 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:25:32.753 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:32.847 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:32.993 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:33.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:33.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:33.114 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:33.215 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:33.267 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:33.424 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:25:33.428 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:33.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:33.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:33.458 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:25:33.503 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:25:33.549 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:33.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:33 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2069996481' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:33.557 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:25:33.578 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:33.601 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:25:33.624 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:33 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2069996481' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:33.624 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:33 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2069996481' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:33.656 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:25:33.707 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:25:33.707 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:25:33.707 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:25:33.707 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:25:33.707 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:25:33.708 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:25:33.708 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:25:33.708 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-10-01T09:25:33.708 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:25:33.770 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:33.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:33.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:33.876 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:33.991 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:34.198 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:25:34.198 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:25:34.199 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:25:34.199 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:25:34.199 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.199 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:125418 6.d 6.2bc0704d (undecoded) ondisk+retry+write+known_if_redirected e227)", 2023-10-01T09:25:34.199 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:01.365555+0000", 2023-10-01T09:25:34.199 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 212.82990270499999, 2023-10-01T09:25:34.199 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81812904099999995, 2023-10-01T09:25:34.199 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.199 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:25:34.199 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.199 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.200 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.200 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 125418 2023-10-01T09:25:34.200 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.200 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.200 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.200 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.200 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365555+0000", 2023-10-01T09:25:34.200 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.200 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.200 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.200 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.200 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365555+0000", 2023-10-01T09:25:34.201 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:34.201 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.201 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.201 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.201 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365553+0000", 2023-10-01T09:25:34.201 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8529999999999992e-06 2023-10-01T09:25:34.201 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.201 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.201 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.201 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365561+0000", 2023-10-01T09:25:34.201 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7089999999999997e-06 2023-10-01T09:25:34.202 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.202 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.202 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.202 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365571+0000", 2023-10-01T09:25:34.202 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.075e-06 2023-10-01T09:25:34.202 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.202 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.202 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.202 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365578+0000", 2023-10-01T09:25:34.202 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000104657 2023-10-01T09:25:34.202 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.203 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.203 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.203 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365683+0000", 2023-10-01T09:25:34.203 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5913000000000002e-05 2023-10-01T09:25:34.203 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.203 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.203 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.203 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365699+0000", 2023-10-01T09:25:34.203 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.817972484 2023-10-01T09:25:34.203 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.203 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.203 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.204 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183671+0000", 2023-10-01T09:25:34.204 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3343000000000001e-05 2023-10-01T09:25:34.204 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.204 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.204 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.204 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183684+0000", 2023-10-01T09:25:34.204 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81812904099999995 2023-10-01T09:25:34.204 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.204 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.204 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.204 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.204 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.205 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:125430 6.d 6.d1d166cd (undecoded) ondisk+write+known_if_redirected e227)", 2023-10-01T09:25:34.205 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:01.397994+0000", 2023-10-01T09:25:34.205 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 212.79746400799999, 2023-10-01T09:25:34.205 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78591710299999995, 2023-10-01T09:25:34.205 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.205 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:25:34.205 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.205 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.205 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.205 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 125430 2023-10-01T09:25:34.205 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.206 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.206 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.206 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.206 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.397994+0000", 2023-10-01T09:25:34.206 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.206 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.206 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.206 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.206 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.397994+0000", 2023-10-01T09:25:34.206 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:34.206 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.206 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.207 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.207 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.397992+0000", 2023-10-01T09:25:34.207 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0169999999999999e-05 2023-10-01T09:25:34.207 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.207 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.207 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.207 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398002+0000", 2023-10-01T09:25:34.207 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6186999999999999e-05 2023-10-01T09:25:34.207 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.207 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.207 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.207 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398019+0000", 2023-10-01T09:25:34.208 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.17e-06 2023-10-01T09:25:34.208 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.208 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.208 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.208 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398026+0000", 2023-10-01T09:25:34.208 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2902000000000002e-05 2023-10-01T09:25:34.208 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.208 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.208 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.208 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398089+0000", 2023-10-01T09:25:34.208 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1385e-05 2023-10-01T09:25:34.208 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.208 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.209 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.209 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398100+0000", 2023-10-01T09:25:34.209 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78580161999999998 2023-10-01T09:25:34.209 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.209 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.209 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.209 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183902+0000", 2023-10-01T09:25:34.209 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3659999999999998e-06 2023-10-01T09:25:34.209 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.209 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.209 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.209 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183911+0000", 2023-10-01T09:25:34.210 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78591710299999995 2023-10-01T09:25:34.210 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.210 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.210 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.210 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.210 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.210 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:125431 6.d 6.45b3c99d (undecoded) ondisk+write+known_if_redirected e227)", 2023-10-01T09:25:34.210 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:01.399470+0000", 2023-10-01T09:25:34.210 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 212.795988292, 2023-10-01T09:25:34.210 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78447003999999998, 2023-10-01T09:25:34.210 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.210 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:25:34.211 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.211 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.211 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.211 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 125431 2023-10-01T09:25:34.211 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.211 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.211 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.211 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.211 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399470+0000", 2023-10-01T09:25:34.211 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.211 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.211 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.212 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.212 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399470+0000", 2023-10-01T09:25:34.212 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:34.212 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.212 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.212 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.212 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399468+0000", 2023-10-01T09:25:34.212 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8000000000000001e-06 2023-10-01T09:25:34.212 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.212 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.212 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.212 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399475+0000", 2023-10-01T09:25:34.212 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6650000000000002e-05 2023-10-01T09:25:34.213 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.213 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.213 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.213 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399492+0000", 2023-10-01T09:25:34.213 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5230000000000001e-06 2023-10-01T09:25:34.213 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.213 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.213 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.213 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399498+0000", 2023-10-01T09:25:34.213 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1703999999999998e-05 2023-10-01T09:25:34.213 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.213 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.214 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.214 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399550+0000", 2023-10-01T09:25:34.214 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8299999999999998e-06 2023-10-01T09:25:34.214 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.214 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.214 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.214 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399557+0000", 2023-10-01T09:25:34.214 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78437442400000001 2023-10-01T09:25:34.214 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.214 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.214 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.215 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183931+0000", 2023-10-01T09:25:34.215 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4589999999999998e-06 2023-10-01T09:25:34.215 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.215 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.215 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.215 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183940+0000", 2023-10-01T09:25:34.215 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78447003999999998 2023-10-01T09:25:34.215 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.215 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.215 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.215 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.216 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.216 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:125447 6.7 6.c1000b47 (undecoded) ondisk+write+known_if_redirected e227)", 2023-10-01T09:25:34.216 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:01.419192+0000", 2023-10-01T09:25:34.216 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 212.77626564799999, 2023-10-01T09:25:34.216 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.73714969699999999, 2023-10-01T09:25:34.216 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.216 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:25:34.216 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.216 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.216 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.216 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 125447 2023-10-01T09:25:34.216 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.217 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.217 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.217 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.217 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419192+0000", 2023-10-01T09:25:34.217 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.217 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.217 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.217 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.217 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419192+0000", 2023-10-01T09:25:34.217 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:34.217 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.217 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.218 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.218 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419191+0000", 2023-10-01T09:25:34.218 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1951e-05 2023-10-01T09:25:34.218 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.218 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.218 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.218 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419203+0000", 2023-10-01T09:25:34.218 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8039000000000001e-05 2023-10-01T09:25:34.218 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.218 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.218 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.219 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419221+0000", 2023-10-01T09:25:34.219 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6539999999999999e-06 2023-10-01T09:25:34.219 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.219 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.219 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.219 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419229+0000", 2023-10-01T09:25:34.219 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2092999999999997e-05 2023-10-01T09:25:34.219 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.219 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.219 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.219 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419281+0000", 2023-10-01T09:25:34.219 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.994e-06 2023-10-01T09:25:34.220 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.220 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.220 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.220 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.419289+0000", 2023-10-01T09:25:34.220 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.73703525000000003 2023-10-01T09:25:34.220 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.220 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.220 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.220 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.156324+0000", 2023-10-01T09:25:34.220 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8236e-05 2023-10-01T09:25:34.220 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.220 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.221 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.221 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.156342+0000", 2023-10-01T09:25:34.221 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.73714969699999999 2023-10-01T09:25:34.221 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.221 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.221 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.221 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.221 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.221 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:125450 6.7 6.376de267 (undecoded) ondisk+write+known_if_redirected e227)", 2023-10-01T09:25:34.221 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:01.421544+0000", 2023-10-01T09:25:34.221 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 212.773914039, 2023-10-01T09:25:34.222 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.73483475099999995, 2023-10-01T09:25:34.222 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.222 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:25:34.222 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.222 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.222 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.222 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 125450 2023-10-01T09:25:34.222 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.222 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.222 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.222 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.222 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421544+0000", 2023-10-01T09:25:34.223 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.223 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.223 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.223 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.223 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421544+0000", 2023-10-01T09:25:34.223 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:25:34.223 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.223 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.223 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.223 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421543+0000", 2023-10-01T09:25:34.223 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4030000000000007e-06 2023-10-01T09:25:34.223 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.223 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.224 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.224 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421551+0000", 2023-10-01T09:25:34.224 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7561999999999999e-05 2023-10-01T09:25:34.224 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.224 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.224 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.224 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421569+0000", 2023-10-01T09:25:34.224 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2339999999999997e-06 2023-10-01T09:25:34.224 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.224 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.224 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.225 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421576+0000", 2023-10-01T09:25:34.225 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7411999999999999e-05 2023-10-01T09:25:34.225 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.225 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.225 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.225 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421623+0000", 2023-10-01T09:25:34.225 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3320000000000003e-06 2023-10-01T09:25:34.225 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.225 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.225 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.225 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.421631+0000", 2023-10-01T09:25:34.225 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.734737784 2023-10-01T09:25:34.226 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.226 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.226 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.226 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.156369+0000", 2023-10-01T09:25:34.226 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0196e-05 2023-10-01T09:25:34.226 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.226 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.226 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.226 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.156379+0000", 2023-10-01T09:25:34.226 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.73483475099999995 2023-10-01T09:25:34.226 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.227 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.227 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.227 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.227 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.227 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166458 6.7 6:e2130551:::benchmark_data_smithi138_307751_object71089:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.227 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.189221+0000", 2023-10-01T09:25:34.227 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 162.00623727799999, 2023-10-01T09:25:34.227 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.07166782, 2023-10-01T09:25:34.227 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.227 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.227 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.227 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.228 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.228 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166458 2023-10-01T09:25:34.228 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.228 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.228 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.228 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.228 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189221+0000", 2023-10-01T09:25:34.228 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.228 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.228 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.228 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.228 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189221+0000", 2023-10-01T09:25:34.228 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:34.229 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.229 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.229 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.229 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189219+0000", 2023-10-01T09:25:34.229 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5189999999999992e-06 2023-10-01T09:25:34.229 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.229 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.229 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.229 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189228+0000", 2023-10-01T09:25:34.229 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6232e-05 2023-10-01T09:25:34.229 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.229 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.230 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.230 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189244+0000", 2023-10-01T09:25:34.230 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3189999999999997e-06 2023-10-01T09:25:34.230 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.230 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.230 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.230 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189251+0000", 2023-10-01T09:25:34.230 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0047771130000000004 2023-10-01T09:25:34.230 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.230 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.230 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.231 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.194029+0000", 2023-10-01T09:25:34.231 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1163e-05 2023-10-01T09:25:34.231 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.231 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.231 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.231 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.194040+0000", 2023-10-01T09:25:34.231 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0557915819999999 2023-10-01T09:25:34.231 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.231 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.231 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.231 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.249831+0000", 2023-10-01T09:25:34.231 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010584799999999999 2023-10-01T09:25:34.232 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.232 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.232 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.232 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.249937+0000", 2023-10-01T09:25:34.232 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6229000000000003e-05 2023-10-01T09:25:34.232 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.232 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.232 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.232 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.250033+0000", 2023-10-01T09:25:34.232 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049686640000000002 2023-10-01T09:25:34.232 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.233 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.233 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.233 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.255002+0000", 2023-10-01T09:25:34.233 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001218654 2023-10-01T09:25:34.233 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.233 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.233 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.233 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.256221+0000", 2023-10-01T09:25:34.233 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045970969999999996 2023-10-01T09:25:34.233 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.233 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.233 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.234 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260818+0000", 2023-10-01T09:25:34.234 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9246999999999999e-05 2023-10-01T09:25:34.234 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.234 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.234 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.234 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260857+0000", 2023-10-01T09:25:34.234 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1477999999999999e-05 2023-10-01T09:25:34.234 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.234 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.234 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.234 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260889+0000", 2023-10-01T09:25:34.235 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.07166782 2023-10-01T09:25:34.235 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.235 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.235 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.235 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.235 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.235 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166487 6.7 6:e01c315f:::benchmark_data_smithi138_307751_object71118:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.235 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.213314+0000", 2023-10-01T09:25:34.235 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 161.982143995, 2023-10-01T09:25:34.235 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0525543909999999, 2023-10-01T09:25:34.235 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.235 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.236 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.236 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.236 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.236 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166487 2023-10-01T09:25:34.236 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.236 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.236 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.236 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.236 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213314+0000", 2023-10-01T09:25:34.236 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.236 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.237 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.237 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.237 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213314+0000", 2023-10-01T09:25:34.237 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:34.237 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.237 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.237 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.237 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213312+0000", 2023-10-01T09:25:34.237 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5779999999999999e-06 2023-10-01T09:25:34.237 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.237 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.237 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.238 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213321+0000", 2023-10-01T09:25:34.238 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5784e-05 2023-10-01T09:25:34.238 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.238 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.238 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.238 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213337+0000", 2023-10-01T09:25:34.238 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2225000000000002e-05 2023-10-01T09:25:34.238 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.238 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.238 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.238 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213359+0000", 2023-10-01T09:25:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000137766 2023-10-01T09:25:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213497+0000", 2023-10-01T09:25:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2595999999999999e-05 2023-10-01T09:25:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213509+0000", 2023-10-01T09:25:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0382323870000001 2023-10-01T09:25:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.251742+0000", 2023-10-01T09:25:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0978999999999998e-05 2023-10-01T09:25:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.251823+0000", 2023-10-01T09:25:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7830999999999995e-05 2023-10-01T09:25:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.251890+0000", 2023-10-01T09:25:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0056844479999999999 2023-10-01T09:25:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257575+0000", 2023-10-01T09:25:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003355055 2023-10-01T09:25:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260930+0000", 2023-10-01T09:25:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0048781969999999999 2023-10-01T09:25:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265808+0000", 2023-10-01T09:25:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2873000000000002e-05 2023-10-01T09:25:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265841+0000", 2023-10-01T09:25:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7410999999999999e-05 2023-10-01T09:25:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265868+0000", 2023-10-01T09:25:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0525543909999999 2023-10-01T09:25:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166491 6.7 6:e6d8849d:::benchmark_data_smithi138_307751_object71122:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.219132+0000", 2023-10-01T09:25:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 161.97632558800001, 2023-10-01T09:25:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0468224770000001, 2023-10-01T09:25:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166491 2023-10-01T09:25:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219132+0000", 2023-10-01T09:25:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219132+0000", 2023-10-01T09:25:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219131+0000", 2023-10-01T09:25:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1768e-05 2023-10-01T09:25:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219143+0000", 2023-10-01T09:25:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2629000000000001e-05 2023-10-01T09:25:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219155+0000", 2023-10-01T09:25:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.942e-06 2023-10-01T09:25:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219161+0000", 2023-10-01T09:25:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5195000000000003e-05 2023-10-01T09:25:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219216+0000", 2023-10-01T09:25:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0199e-05 2023-10-01T09:25:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219226+0000", 2023-10-01T09:25:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.033648492 2023-10-01T09:25:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.252875+0000", 2023-10-01T09:25:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9799999999999997e-05 2023-10-01T09:25:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.252935+0000", 2023-10-01T09:25:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5607999999999999e-05 2023-10-01T09:25:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.253000+0000", 2023-10-01T09:25:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00458422 2023-10-01T09:25:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257585+0000", 2023-10-01T09:25:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033770720000000001 2023-10-01T09:25:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260962+0000", 2023-10-01T09:25:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049325590000000004 2023-10-01T09:25:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265894+0000", 2023-10-01T09:25:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6339e-05 2023-10-01T09:25:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265921+0000", 2023-10-01T09:25:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4357999999999999e-05 2023-10-01T09:25:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265955+0000", 2023-10-01T09:25:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0468224770000001 2023-10-01T09:25:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166496 6.7 6:e4e4b310:::benchmark_data_smithi138_307751_object71127:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.230525+0000", 2023-10-01T09:25:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 161.964932906, 2023-10-01T09:25:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0393795619999999, 2023-10-01T09:25:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166496 2023-10-01T09:25:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230525+0000", 2023-10-01T09:25:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230525+0000", 2023-10-01T09:25:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:25:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230524+0000", 2023-10-01T09:25:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8950000000000003e-06 2023-10-01T09:25:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230530+0000", 2023-10-01T09:25:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2658e-05 2023-10-01T09:25:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230543+0000", 2023-10-01T09:25:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2159999999999997e-06 2023-10-01T09:25:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230548+0000", 2023-10-01T09:25:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9517e-05 2023-10-01T09:25:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230597+0000", 2023-10-01T09:25:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1009000000000001e-05 2023-10-01T09:25:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230608+0000", 2023-10-01T09:25:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.023332213 2023-10-01T09:25:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.253940+0000", 2023-10-01T09:25:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0046000000000001e-05 2023-10-01T09:25:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.254001+0000", 2023-10-01T09:25:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6650999999999996e-05 2023-10-01T09:25:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.254067+0000", 2023-10-01T09:25:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035276270000000002 2023-10-01T09:25:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257595+0000", 2023-10-01T09:25:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0084019709999999994 2023-10-01T09:25:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265997+0000", 2023-10-01T09:25:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038161779999999999 2023-10-01T09:25:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.269813+0000", 2023-10-01T09:25:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0507999999999998e-05 2023-10-01T09:25:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.269863+0000", 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1207e-05 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.269905+0000", 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0393795619999999 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166506 6.7 6:e0267cc2:::benchmark_data_smithi138_307751_object71137:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.230586+0000", 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 161.96487194299999, 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.039509367, 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166506 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230586+0000", 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230586+0000", 2023-10-01T09:25:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:25:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230586+0000", 2023-10-01T09:25:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7310000000000002e-06 2023-10-01T09:25:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230588+0000", 2023-10-01T09:25:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6399999999999997e-06 2023-10-01T09:25:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230596+0000", 2023-10-01T09:25:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9950000000000001e-06 2023-10-01T09:25:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230599+0000", 2023-10-01T09:25:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4839000000000003e-05 2023-10-01T09:25:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230634+0000", 2023-10-01T09:25:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6569999999999997e-06 2023-10-01T09:25:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230641+0000", 2023-10-01T09:25:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0243796119999999 2023-10-01T09:25:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.255021+0000", 2023-10-01T09:25:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5718999999999999e-05 2023-10-01T09:25:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.255087+0000", 2023-10-01T09:25:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1915999999999994e-05 2023-10-01T09:25:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.255159+0000", 2023-10-01T09:25:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0056119830000000001 2023-10-01T09:25:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260771+0000", 2023-10-01T09:25:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0052694889999999996 2023-10-01T09:25:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.266040+0000", 2023-10-01T09:25:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003962826 2023-10-01T09:25:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270003+0000", 2023-10-01T09:25:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2723999999999999e-05 2023-10-01T09:25:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270056+0000", 2023-10-01T09:25:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9711e-05 2023-10-01T09:25:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270095+0000", 2023-10-01T09:25:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.039509367 2023-10-01T09:25:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166513 6.7 6:e69cb40c:::benchmark_data_smithi138_307751_object71144:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.247029+0000", 2023-10-01T09:25:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 161.948429318, 2023-10-01T09:25:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0288884, 2023-10-01T09:25:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166513 2023-10-01T09:25:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247029+0000", 2023-10-01T09:25:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247029+0000", 2023-10-01T09:25:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247027+0000", 2023-10-01T09:25:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0600000000000002e-06 2023-10-01T09:25:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247034+0000", 2023-10-01T09:25:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2286000000000001e-05 2023-10-01T09:25:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247046+0000", 2023-10-01T09:25:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7830000000000001e-06 2023-10-01T09:25:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247053+0000", 2023-10-01T09:25:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7210000000000003e-05 2023-10-01T09:25:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247110+0000", 2023-10-01T09:25:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1158999999999999e-05 2023-10-01T09:25:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247122+0000", 2023-10-01T09:25:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.009139901 2023-10-01T09:25:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.256262+0000", 2023-10-01T09:25:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6122e-05 2023-10-01T09:25:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.256348+0000", 2023-10-01T09:25:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6933000000000004e-05 2023-10-01T09:25:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.256425+0000", 2023-10-01T09:25:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045112299999999998 2023-10-01T09:25:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260936+0000", 2023-10-01T09:25:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0090185890000000005 2023-10-01T09:25:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.269954+0000", 2023-10-01T09:25:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0058642859999999998 2023-10-01T09:25:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.275819+0000", 2023-10-01T09:25:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7589999999999997e-05 2023-10-01T09:25:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.275866+0000", 2023-10-01T09:25:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0816999999999998e-05 2023-10-01T09:25:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.275917+0000", 2023-10-01T09:25:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0288884 2023-10-01T09:25:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166528 6.7 6:e73602e0:::benchmark_data_smithi138_307751_object71159:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.261428+0000", 2023-10-01T09:25:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 161.934029584, 2023-10-01T09:25:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.014679273, 2023-10-01T09:25:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166528 2023-10-01T09:25:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261428+0000", 2023-10-01T09:25:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261428+0000", 2023-10-01T09:25:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261426+0000", 2023-10-01T09:25:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3349999999999996e-06 2023-10-01T09:25:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261436+0000", 2023-10-01T09:25:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4049000000000001e-05 2023-10-01T09:25:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261450+0000", 2023-10-01T09:25:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5750000000000003e-06 2023-10-01T09:25:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261457+0000", 2023-10-01T09:25:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8458999999999997e-05 2023-10-01T09:25:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261516+0000", 2023-10-01T09:25:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1588e-05 2023-10-01T09:25:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261527+0000", 2023-10-01T09:25:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99610873700000002 2023-10-01T09:25:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257636+0000", 2023-10-01T09:25:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000114474 2023-10-01T09:25:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257751+0000", 2023-10-01T09:25:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014375799999999999 2023-10-01T09:25:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257894+0000", 2023-10-01T09:25:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0030463740000000001 2023-10-01T09:25:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260941+0000", 2023-10-01T09:25:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.009213377 2023-10-01T09:25:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270154+0000", 2023-10-01T09:25:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0058627649999999998 2023-10-01T09:25:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.276017+0000", 2023-10-01T09:25:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0591999999999999e-05 2023-10-01T09:25:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.276067+0000", 2023-10-01T09:25:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0309999999999999e-05 2023-10-01T09:25:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.276108+0000", 2023-10-01T09:25:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.014679273 2023-10-01T09:25:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166545 6.7 6:e60e3404:::benchmark_data_smithi138_307751_object71176:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.284403+0000", 2023-10-01T09:25:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 161.91105505100001, 2023-10-01T09:25:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99383394000000003, 2023-10-01T09:25:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166545 2023-10-01T09:25:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284403+0000", 2023-10-01T09:25:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284403+0000", 2023-10-01T09:25:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:25:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284401+0000", 2023-10-01T09:25:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0525e-05 2023-10-01T09:25:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284411+0000", 2023-10-01T09:25:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8612999999999999e-05 2023-10-01T09:25:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284430+0000", 2023-10-01T09:25:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7730000000000002e-06 2023-10-01T09:25:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284436+0000", 2023-10-01T09:25:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4496000000000001e-05 2023-10-01T09:25:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284491+0000", 2023-10-01T09:25:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1708999999999999e-05 2023-10-01T09:25:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284503+0000", 2023-10-01T09:25:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97494659800000005 2023-10-01T09:25:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.259449+0000", 2023-10-01T09:25:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000104481 2023-10-01T09:25:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.259554+0000", 2023-10-01T09:25:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2804000000000002e-05 2023-10-01T09:25:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.259637+0000", 2023-10-01T09:25:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002740433 2023-10-01T09:25:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.262377+0000", 2023-10-01T09:25:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078248339999999993 2023-10-01T09:25:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270202+0000", 2023-10-01T09:25:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0079470840000000001 2023-10-01T09:25:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278149+0000", 2023-10-01T09:25:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8479999999999997e-05 2023-10-01T09:25:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278197+0000", 2023-10-01T09:25:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9505000000000002e-05 2023-10-01T09:25:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278237+0000", 2023-10-01T09:25:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99383394000000003 2023-10-01T09:25:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166600 6.7 6:e23e27f5:::benchmark_data_smithi138_307751_object71231:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.346015+0000", 2023-10-01T09:25:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 161.849442759, 2023-10-01T09:25:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.93235052399999996, 2023-10-01T09:25:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166600 2023-10-01T09:25:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346015+0000", 2023-10-01T09:25:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346015+0000", 2023-10-01T09:25:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346014+0000", 2023-10-01T09:25:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1130000000000001e-06 2023-10-01T09:25:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346022+0000", 2023-10-01T09:25:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4234e-05 2023-10-01T09:25:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346036+0000", 2023-10-01T09:25:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9259999999999997e-06 2023-10-01T09:25:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346042+0000", 2023-10-01T09:25:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9398999999999998e-05 2023-10-01T09:25:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346102+0000", 2023-10-01T09:25:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.202e-05 2023-10-01T09:25:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346114+0000", 2023-10-01T09:25:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.91486791099999998 2023-10-01T09:25:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260982+0000", 2023-10-01T09:25:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3937999999999997e-05 2023-10-01T09:25:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.261045+0000", 2023-10-01T09:25:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9540000000000001e-05 2023-10-01T09:25:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.261125+0000", 2023-10-01T09:25:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0046514310000000001 2023-10-01T09:25:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265776+0000", 2023-10-01T09:25:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0077202750000000004 2023-10-01T09:25:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.273497+0000", 2023-10-01T09:25:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0047860460000000004 2023-10-01T09:25:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278283+0000", 2023-10-01T09:25:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6359000000000002e-05 2023-10-01T09:25:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278319+0000", 2023-10-01T09:25:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6647000000000002e-05 2023-10-01T09:25:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278366+0000", 2023-10-01T09:25:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.93235052399999996 2023-10-01T09:25:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166608 6.7 6:e3e1df92:::benchmark_data_smithi138_307751_object71239:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.493260+0000", 2023-10-01T09:25:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 161.70219759599999, 2023-10-01T09:25:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78524043099999996, 2023-10-01T09:25:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166608 2023-10-01T09:25:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493260+0000", 2023-10-01T09:25:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493260+0000", 2023-10-01T09:25:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:25:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493260+0000", 2023-10-01T09:25:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6130000000000003e-06 2023-10-01T09:25:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493265+0000", 2023-10-01T09:25:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1015e-05 2023-10-01T09:25:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493276+0000", 2023-10-01T09:25:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5259999999999999e-06 2023-10-01T09:25:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493283+0000", 2023-10-01T09:25:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9543999999999997e-05 2023-10-01T09:25:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493352+0000", 2023-10-01T09:25:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.118e-05 2023-10-01T09:25:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493364+0000", 2023-10-01T09:25:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.76903849099999999 2023-10-01T09:25:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.262402+0000", 2023-10-01T09:25:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7131000000000002e-05 2023-10-01T09:25:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.262499+0000", 2023-10-01T09:25:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000120406 2023-10-01T09:25:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.262620+0000", 2023-10-01T09:25:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0032544620000000001 2023-10-01T09:25:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265874+0000", 2023-10-01T09:25:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010088079 2023-10-01T09:25:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.275962+0000", 2023-10-01T09:25:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024570550000000001 2023-10-01T09:25:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278419+0000", 2023-10-01T09:25:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5637999999999999e-05 2023-10-01T09:25:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278465+0000", 2023-10-01T09:25:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6021e-05 2023-10-01T09:25:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278501+0000", 2023-10-01T09:25:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78524043099999996 2023-10-01T09:25:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166718 6.7 6:e4a2c62d:::benchmark_data_smithi138_307751_object71349:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.665722+0000", 2023-10-01T09:25:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 161.52973647799999, 2023-10-01T09:25:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.61779461199999997, 2023-10-01T09:25:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166718 2023-10-01T09:25:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665722+0000", 2023-10-01T09:25:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665722+0000", 2023-10-01T09:25:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665720+0000", 2023-10-01T09:25:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5480000000000002e-06 2023-10-01T09:25:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665728+0000", 2023-10-01T09:25:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6147999999999999e-05 2023-10-01T09:25:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665745+0000", 2023-10-01T09:25:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2089999999999999e-06 2023-10-01T09:25:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665752+0000", 2023-10-01T09:25:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4418999999999994e-05 2023-10-01T09:25:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665816+0000", 2023-10-01T09:25:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0353999999999999e-05 2023-10-01T09:25:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.665827+0000", 2023-10-01T09:25:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.59826122199999998 2023-10-01T09:25:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.264088+0000", 2023-10-01T09:25:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0003e-05 2023-10-01T09:25:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.264158+0000", 2023-10-01T09:25:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7170999999999996e-05 2023-10-01T09:25:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.264245+0000", 2023-10-01T09:25:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001758414 2023-10-01T09:25:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.266003+0000", 2023-10-01T09:25:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016782042 2023-10-01T09:25:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.282785+0000", 2023-10-01T09:25:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00063452500000000004 2023-10-01T09:25:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.283420+0000", 2023-10-01T09:25:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0348e-05 2023-10-01T09:25:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.283470+0000", 2023-10-01T09:25:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5849000000000001e-05 2023-10-01T09:25:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.283516+0000", 2023-10-01T09:25:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.61779461199999997 2023-10-01T09:25:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166770 6.7 6:e5b59087:::benchmark_data_smithi138_307751_object71401:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.753490+0000", 2023-10-01T09:25:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 161.44196785099999, 2023-10-01T09:25:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.53017308900000004, 2023-10-01T09:25:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166770 2023-10-01T09:25:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753490+0000", 2023-10-01T09:25:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753490+0000", 2023-10-01T09:25:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:25:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753489+0000", 2023-10-01T09:25:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9090000000000003e-06 2023-10-01T09:25:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753496+0000", 2023-10-01T09:25:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3482000000000001e-05 2023-10-01T09:25:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753509+0000", 2023-10-01T09:25:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4609999999999999e-06 2023-10-01T09:25:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753515+0000", 2023-10-01T09:25:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8496999999999999e-05 2023-10-01T09:25:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753573+0000", 2023-10-01T09:25:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0601999999999999e-05 2023-10-01T09:25:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.753584+0000", 2023-10-01T09:25:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.51248056099999995 2023-10-01T09:25:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.266064+0000", 2023-10-01T09:25:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000108893 2023-10-01T09:25:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.266173+0000", 2023-10-01T09:25:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000125909 2023-10-01T09:25:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.266299+0000", 2023-10-01T09:25:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003810857 2023-10-01T09:25:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270110+0000", 2023-10-01T09:25:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012703811000000001 2023-10-01T09:25:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.282814+0000", 2023-10-01T09:25:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00075106700000000001 2023-10-01T09:25:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.283565+0000", 2023-10-01T09:25:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6557000000000001e-05 2023-10-01T09:25:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.283621+0000", 2023-10-01T09:25:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1903000000000002e-05 2023-10-01T09:25:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.283663+0000", 2023-10-01T09:25:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.53017308900000004 2023-10-01T09:25:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166817 6.7 6:e32b8a7a:::benchmark_data_smithi138_307751_object71448:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.827383+0000", 2023-10-01T09:25:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 161.368075116, 2023-10-01T09:25:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.45803607899999998, 2023-10-01T09:25:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166817 2023-10-01T09:25:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827383+0000", 2023-10-01T09:25:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827383+0000", 2023-10-01T09:25:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827381+0000", 2023-10-01T09:25:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0366999999999999e-05 2023-10-01T09:25:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827391+0000", 2023-10-01T09:25:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5305999999999999e-05 2023-10-01T09:25:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827406+0000", 2023-10-01T09:25:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6420000000000007e-06 2023-10-01T09:25:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827414+0000", 2023-10-01T09:25:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1822999999999994e-05 2023-10-01T09:25:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827476+0000", 2023-10-01T09:25:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0706e-05 2023-10-01T09:25:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.827487+0000", 2023-10-01T09:25:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.44055778499999998 2023-10-01T09:25:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.268044+0000", 2023-10-01T09:25:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4426999999999997e-05 2023-10-01T09:25:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.268129+0000", 2023-10-01T09:25:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7678999999999999e-05 2023-10-01T09:25:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.268217+0000", 2023-10-01T09:25:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019031390000000001 2023-10-01T09:25:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270120+0000", 2023-10-01T09:25:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014529134000000001 2023-10-01T09:25:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.284649+0000", 2023-10-01T09:25:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00067139599999999999 2023-10-01T09:25:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285320+0000", 2023-10-01T09:25:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0686000000000002e-05 2023-10-01T09:25:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285371+0000", 2023-10-01T09:25:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8123e-05 2023-10-01T09:25:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285419+0000", 2023-10-01T09:25:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.45803607899999998 2023-10-01T09:25:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166820 6.7 6:e587feea:::benchmark_data_smithi138_307751_object71451:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.832585+0000", 2023-10-01T09:25:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 161.36287294600001, 2023-10-01T09:25:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.45297539100000001, 2023-10-01T09:25:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166820 2023-10-01T09:25:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832585+0000", 2023-10-01T09:25:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832585+0000", 2023-10-01T09:25:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832583+0000", 2023-10-01T09:25:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1138000000000001e-05 2023-10-01T09:25:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832594+0000", 2023-10-01T09:25:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3779e-05 2023-10-01T09:25:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832608+0000", 2023-10-01T09:25:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7260000000000002e-06 2023-10-01T09:25:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832616+0000", 2023-10-01T09:25:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8177000000000002e-05 2023-10-01T09:25:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832674+0000", 2023-10-01T09:25:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2645e-05 2023-10-01T09:25:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.832687+0000", 2023-10-01T09:25:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.43753456800000001 2023-10-01T09:25:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270221+0000", 2023-10-01T09:25:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4250000000000002e-05 2023-10-01T09:25:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270295+0000", 2023-10-01T09:25:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4099999999999998e-05 2023-10-01T09:25:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270380+0000", 2023-10-01T09:25:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0031290200000000002 2023-10-01T09:25:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.273509+0000", 2023-10-01T09:25:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011158936 2023-10-01T09:25:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.284667+0000", 2023-10-01T09:25:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00081128500000000002 2023-10-01T09:25:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285479+0000", 2023-10-01T09:25:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6978999999999999e-05 2023-10-01T09:25:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285526+0000", 2023-10-01T09:25:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4718000000000002e-05 2023-10-01T09:25:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285560+0000", 2023-10-01T09:25:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.45297539100000001 2023-10-01T09:25:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166823 6.7 6:e67d4cd1:::benchmark_data_smithi138_307751_object71454:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:25:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.838568+0000", 2023-10-01T09:25:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 161.35689024800001, 2023-10-01T09:25:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.44710524600000001, 2023-10-01T09:25:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:25:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:25:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:25:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:25:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:25:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166823 2023-10-01T09:25:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:25:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:25:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838568+0000", 2023-10-01T09:25:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:25:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:25:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838568+0000", 2023-10-01T09:25:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:25:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838566+0000", 2023-10-01T09:25:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1300000000000001e-06 2023-10-01T09:25:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:25:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838574+0000", 2023-10-01T09:25:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6716000000000001e-05 2023-10-01T09:25:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:25:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838591+0000", 2023-10-01T09:25:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8700000000000003e-06 2023-10-01T09:25:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:25:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838597+0000", 2023-10-01T09:25:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9017000000000002e-05 2023-10-01T09:25:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838647+0000", 2023-10-01T09:25:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.986e-06 2023-10-01T09:25:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:25:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.838655+0000", 2023-10-01T09:25:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.43301372500000002 2023-10-01T09:25:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:25:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.271669+0000", 2023-10-01T09:25:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8678999999999998e-05 2023-10-01T09:25:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:25:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.271738+0000", 2023-10-01T09:25:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9672999999999995e-05 2023-10-01T09:25:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:25:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.271808+0000", 2023-10-01T09:25:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003960547 2023-10-01T09:25:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:25:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.275768+0000", 2023-10-01T09:25:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0089158359999999999 2023-10-01T09:25:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.284684+0000", 2023-10-01T09:25:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000918366 2023-10-01T09:25:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285602+0000", 2023-10-01T09:25:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0027999999999997e-05 2023-10-01T09:25:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:25:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285642+0000", 2023-10-01T09:25:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0675e-05 2023-10-01T09:25:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:25:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:25:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:25:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.285673+0000", 2023-10-01T09:25:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.44710524600000001 2023-10-01T09:25:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:25:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:25:34.350 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:25:34.585 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:34.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:34.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:34.690 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:34.716 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:34 smithi190 ceph-mon[160435]: pgmap v723: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 147 B/s rd, 7.8 MiB/s wr, 492 op/s; 35641/285483 objects degraded (12.484%); 17897/285483 objects misplaced (6.269%); 1.3 MiB/s, 26 objects/s recovering 2023-10-01T09:25:34.720 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:34 smithi138 ceph-mon[192513]: pgmap v723: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 147 B/s rd, 7.8 MiB/s wr, 492 op/s; 35641/285483 objects degraded (12.484%); 17897/285483 objects misplaced (6.269%); 1.3 MiB/s, 26 objects/s recovering 2023-10-01T09:25:34.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:34 smithi138 ceph-mon[185886]: pgmap v723: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 147 B/s rd, 7.8 MiB/s wr, 492 op/s; 35641/285483 objects degraded (12.484%); 17897/285483 objects misplaced (6.269%); 1.3 MiB/s, 26 objects/s recovering 2023-10-01T09:25:34.855 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:34.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:34.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:34.908 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:25:34.958 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:34.998 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:35.325 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:25:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:25:34] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-01T09:25:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:35 smithi190 ceph-mon[160435]: pgmap v724: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 120 B/s rd, 6.3 MiB/s wr, 481 op/s; 35444/283437 objects degraded (12.505%); 17766/283437 objects misplaced (6.268%); 1.7 MiB/s, 31 objects/s recovering 2023-10-01T09:25:36.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:35 smithi138 ceph-mon[192513]: pgmap v724: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 120 B/s rd, 6.3 MiB/s wr, 481 op/s; 35444/283437 objects degraded (12.505%); 17766/283437 objects misplaced (6.268%); 1.7 MiB/s, 31 objects/s recovering 2023-10-01T09:25:36.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:35 smithi138 ceph-mon[185886]: pgmap v724: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 120 B/s rd, 6.3 MiB/s wr, 481 op/s; 35444/283437 objects degraded (12.505%); 17766/283437 objects misplaced (6.268%); 1.7 MiB/s, 31 objects/s recovering 2023-10-01T09:25:36.825 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:25:36.826 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:25:36.960 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:36.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:36.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:37.076 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:37.162 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:37.192 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:37.385 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:37 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 35444/283437 objects degraded (12.505%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:37.385 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:37 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 35444/283437 objects degraded (12.505%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:37.553 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:37.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:37 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 35444/283437 objects degraded (12.505%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:37.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:37.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:37.666 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:37.823 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:37.840 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:25:37.840 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T09:25:37.840 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:25:38.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:38 smithi190 ceph-mon[160435]: pgmap v725: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 1.0 MiB/s wr, 447 op/s; 35105/278739 objects degraded (12.594%); 17625/278739 objects misplaced (6.323%); 1.9 MiB/s, 33 objects/s recovering 2023-10-01T09:25:38.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:38 smithi138 ceph-mon[185886]: pgmap v725: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 1.0 MiB/s wr, 447 op/s; 35105/278739 objects degraded (12.594%); 17625/278739 objects misplaced (6.323%); 1.9 MiB/s, 33 objects/s recovering 2023-10-01T09:25:38.630 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:38 smithi138 ceph-mon[192513]: pgmap v725: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 1.0 MiB/s wr, 447 op/s; 35105/278739 objects degraded (12.594%); 17625/278739 objects misplaced (6.323%); 1.9 MiB/s, 33 objects/s recovering 2023-10-01T09:25:38.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:38.631 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:38.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:38.732 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:38.760 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:38.808 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:38.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:38.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:38.827 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:25:38.916 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:25:38.944 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:39.340 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:25:39.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:39 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2070959617' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:25:39.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:39 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:25:39.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:39 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2070959617' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:25:39.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:39 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:25:39.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:39 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2070959617' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:25:39.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:39 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:25:40.081 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:25:40.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:40.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:25:40.188 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:40.277 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:25:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:25:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[192513]: osdmap e277: 8 total, 7 up, 7 in 2023-10-01T09:25:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[192513]: failed to encode map e277 with expected crc 2023-10-01T09:25:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[192513]: failed to encode map e277 with expected crc 2023-10-01T09:25:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[192513]: failed to encode map e277 with expected crc 2023-10-01T09:25:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[192513]: pgmap v727: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 34839/277464 objects degraded (12.556%); 17625/277464 objects misplaced (6.352%); 1.9 MiB/s, 31 objects/s recovering 2023-10-01T09:25:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:25:40.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:25:40.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:25:40.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[185886]: osdmap e277: 8 total, 7 up, 7 in 2023-10-01T09:25:40.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[185886]: failed to encode map e277 with expected crc 2023-10-01T09:25:40.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[185886]: failed to encode map e277 with expected crc 2023-10-01T09:25:40.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[185886]: failed to encode map e277 with expected crc 2023-10-01T09:25:40.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[185886]: pgmap v727: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 34839/277464 objects degraded (12.556%); 17625/277464 objects misplaced (6.352%); 1.9 MiB/s, 31 objects/s recovering 2023-10-01T09:25:40.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:25:40.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:40 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:25:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:40 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:25:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:40 smithi190 ceph-mon[160435]: osdmap e277: 8 total, 7 up, 7 in 2023-10-01T09:25:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:40 smithi190 ceph-mon[160435]: failed to encode map e277 with expected crc 2023-10-01T09:25:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:40 smithi190 ceph-mon[160435]: failed to encode map e277 with expected crc 2023-10-01T09:25:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:40 smithi190 ceph-mon[160435]: failed to encode map e277 with expected crc 2023-10-01T09:25:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:40 smithi190 ceph-mon[160435]: pgmap v727: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 34839/277464 objects degraded (12.556%); 17625/277464 objects misplaced (6.352%); 1.9 MiB/s, 31 objects/s recovering 2023-10-01T09:25:40.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:25:41.163 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:41.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:41.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:41.285 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:41.392 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:41.497 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:41.578 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:41.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:41.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:41.612 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:25:41.612 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:25:41.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.612 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:104978 7.7 7:e07c7747:::benchmark_data_smithi138_367058_object7183:head [delete] snapc 0=[] ondisk+write+known_if_redirected e277)", 2023-10-01T09:25:41.612 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:25:41.583549+0000", 2023-10-01T09:25:41.612 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.015162343999999999, 2023-10-01T09:25:41.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015191542000000001, 2023-10-01T09:25:41.613 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:41.613 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:25:41.613 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:41.613 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:25:41.613 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:25:41.613 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 104978 2023-10-01T09:25:41.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.613 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:41.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:41.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583549+0000", 2023-10-01T09:25:41.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:41.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:41.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583549+0000", 2023-10-01T09:25:41.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:41.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:41.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583547+0000", 2023-10-01T09:25:41.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:41.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:41.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583557+0000", 2023-10-01T09:25:41.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0611000000000001e-05 2023-10-01T09:25:41.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:41.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583584+0000", 2023-10-01T09:25:41.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6517999999999999e-05 2023-10-01T09:25:41.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:41.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583595+0000", 2023-10-01T09:25:41.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1353e-05 2023-10-01T09:25:41.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:41.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583727+0000", 2023-10-01T09:25:41.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131657 2023-10-01T09:25:41.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:41.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583863+0000", 2023-10-01T09:25:41.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013638100000000001 2023-10-01T09:25:41.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3", 2023-10-01T09:25:41.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583987+0000", 2023-10-01T09:25:41.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000123707 2023-10-01T09:25:41.617 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:41.617 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:41.617 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:41.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.618 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:104979 7.9 7:98f59c9c:::benchmark_data_smithi138_367058_object7184:head [delete] snapc 0=[] ondisk+write+known_if_redirected e277)", 2023-10-01T09:25:41.618 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:25:41.583615+0000", 2023-10-01T09:25:41.618 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.015095661, 2023-10-01T09:25:41.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015179583, 2023-10-01T09:25:41.618 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:41.618 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:25:41.618 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:41.618 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:25:41.618 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:25:41.619 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 104979 2023-10-01T09:25:41.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.619 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:41.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:41.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583615+0000", 2023-10-01T09:25:41.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:41.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:41.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583615+0000", 2023-10-01T09:25:41.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:41.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:41.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583615+0000", 2023-10-01T09:25:41.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:25:41.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:41.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583619+0000", 2023-10-01T09:25:41.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.591e-06 2023-10-01T09:25:41.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:41.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583626+0000", 2023-10-01T09:25:41.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7230000000000004e-06 2023-10-01T09:25:41.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:41.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.583633+0000", 2023-10-01T09:25:41.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3949999999999998e-06 2023-10-01T09:25:41.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:41.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.585839+0000", 2023-10-01T09:25:41.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022061759999999998 2023-10-01T09:25:41.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:41.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.585984+0000", 2023-10-01T09:25:41.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014477199999999999 2023-10-01T09:25:41.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,4", 2023-10-01T09:25:41.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.586109+0000", 2023-10-01T09:25:41.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012576199999999999 2023-10-01T09:25:41.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:25:41.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.593625+0000", 2023-10-01T09:25:41.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0075152780000000002 2023-10-01T09:25:41.623 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:41.623 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:41.623 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:41.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.624 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35421.0:104980 7.5 e277/259 7:a8e4fe44:::benchmark_data_smithi138_367058_object7185:head v 277'6732, mlcod=277'6730)", 2023-10-01T09:25:41.624 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:25:41.585878+0000", 2023-10-01T09:25:41.624 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.012833008999999999, 2023-10-01T09:25:41.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012967977, 2023-10-01T09:25:41.624 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:41.624 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:25:41.624 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:41.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:41.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.585878+0000", 2023-10-01T09:25:41.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:41.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:41.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.585878+0000", 2023-10-01T09:25:41.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:41.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:41.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.585877+0000", 2023-10-01T09:25:41.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:25:41.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:41.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.585905+0000", 2023-10-01T09:25:41.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8175e-05 2023-10-01T09:25:41.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:41.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.585919+0000", 2023-10-01T09:25:41.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4061e-05 2023-10-01T09:25:41.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:41.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.585927+0000", 2023-10-01T09:25:41.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0879999999999995e-06 2023-10-01T09:25:41.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:41.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.585996+0000", 2023-10-01T09:25:41.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8730000000000001e-05 2023-10-01T09:25:41.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:41.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.586017+0000", 2023-10-01T09:25:41.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1192e-05 2023-10-01T09:25:41.628 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:41.628 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:41.628 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:41.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.628 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35421.0:104984 7.d e277/269 7:b1dd053b:::benchmark_data_smithi138_367058_object7189:head v 277'6553, mlcod=277'6551)", 2023-10-01T09:25:41.628 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:25:41.589774+0000", 2023-10-01T09:25:41.628 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0089375289999999996, 2023-10-01T09:25:41.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0091094179999999993, 2023-10-01T09:25:41.629 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:41.629 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:25:41.629 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:41.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:41.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.589774+0000", 2023-10-01T09:25:41.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:41.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:41.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.589774+0000", 2023-10-01T09:25:41.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:41.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:41.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.589773+0000", 2023-10-01T09:25:41.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:25:41.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:41.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.589783+0000", 2023-10-01T09:25:41.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0023999999999999e-05 2023-10-01T09:25:41.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:41.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.589796+0000", 2023-10-01T09:25:41.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3005e-05 2023-10-01T09:25:41.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:41.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.589803+0000", 2023-10-01T09:25:41.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6059999999999997e-06 2023-10-01T09:25:41.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:41.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.589901+0000", 2023-10-01T09:25:41.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7851999999999998e-05 2023-10-01T09:25:41.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:41.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.589925+0000", 2023-10-01T09:25:41.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3845e-05 2023-10-01T09:25:41.633 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:41.633 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:41.633 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:41.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.633 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:104985 7.f 7:fb6d4bf1:::benchmark_data_smithi138_367058_object7190:head [delete] snapc 0=[] ondisk+write+known_if_redirected e277)", 2023-10-01T09:25:41.633 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:25:41.596479+0000", 2023-10-01T09:25:41.633 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0022316889999999998, 2023-10-01T09:25:41.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002439447, 2023-10-01T09:25:41.633 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:25:41.633 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:25:41.634 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:25:41.634 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:25:41.634 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:25:41.634 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 104985 2023-10-01T09:25:41.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.634 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:25:41.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:25:41.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.596479+0000", 2023-10-01T09:25:41.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:41.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:25:41.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.596479+0000", 2023-10-01T09:25:41.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:25:41.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:25:41.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.596478+0000", 2023-10-01T09:25:41.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:25:41.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:25:41.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.596487+0000", 2023-10-01T09:25:41.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8570000000000001e-06 2023-10-01T09:25:41.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:25:41.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.596510+0000", 2023-10-01T09:25:41.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3886000000000001e-05 2023-10-01T09:25:41.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:25:41.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.596520+0000", 2023-10-01T09:25:41.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0057999999999999e-05 2023-10-01T09:25:41.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:25:41.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.596577+0000", 2023-10-01T09:25:41.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6263e-05 2023-10-01T09:25:41.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:25:41.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.596726+0000", 2023-10-01T09:25:41.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000148892 2023-10-01T09:25:41.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:25:41.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:25:41.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:25:41.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:25:41.596855+0000", 2023-10-01T09:25:41.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012949700000000001 2023-10-01T09:25:41.638 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:41.638 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:25:41.638 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:41.638 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:25:41.638 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:25:41.638 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 5 2023-10-01T09:25:41.639 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:25:41.683 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:25:41.770 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:41.976 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:41 smithi138 ceph-mon[185886]: pgmap v728: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 34901/276252 objects degraded (12.634%); 17487/276252 objects misplaced (6.330%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T09:25:41.976 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:41 smithi138 ceph-mon[192513]: pgmap v728: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 34901/276252 objects degraded (12.634%); 17487/276252 objects misplaced (6.330%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T09:25:42.028 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:25:42.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:42.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:25:42.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:41 smithi190 ceph-mon[160435]: pgmap v728: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 34901/276252 objects degraded (12.634%); 17487/276252 objects misplaced (6.330%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T09:25:42.154 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:25:42.233 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:42.790 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:25:42.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:42.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:25:42.898 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:42.928 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:43.027 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:25:43.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:42 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 34901/276252 objects degraded (12.634%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:43.081 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:42 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 34901/276252 objects degraded (12.634%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:43.081 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:42 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 34901/276252 objects degraded (12.634%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:44.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:43 smithi190 ceph-mon[160435]: pgmap v729: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 34581/272085 objects degraded (12.710%); 17348/272085 objects misplaced (6.376%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T09:25:44.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:43 smithi138 ceph-mon[185886]: pgmap v729: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 34581/272085 objects degraded (12.710%); 17348/272085 objects misplaced (6.376%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T09:25:44.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:43 smithi138 ceph-mon[192513]: pgmap v729: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 34581/272085 objects degraded (12.710%); 17348/272085 objects misplaced (6.376%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T09:25:44.209 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:25:44.236 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:44.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:44.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:44.364 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:44.392 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:44.436 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:44.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:44.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:44.544 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:44.570 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:44.615 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:44.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:44.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:44.722 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:44.748 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:44.792 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:44.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:44.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:44.875 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:44.899 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:44.942 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:25:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:25:44] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.33.4" 2023-10-01T09:25:44.998 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:45.118 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:25:45.165 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:25:45.172 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:45.208 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:25:45.255 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:25:45.272 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:45 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2064038581' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:45.272 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:45 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2064038581' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:45.273 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:45.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:45.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:45.301 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:25:45.347 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:25:45.375 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:25:45.375 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:25:45.375 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:25:45.375 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:25:45.375 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:25:45.376 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:25:45.376 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:25:45.376 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-10-01T09:25:45.376 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:25:45.386 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:45.529 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:45.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:45 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2064038581' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:45.732 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 7, 2, 6] out_osds: [5] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 7, 5] 2023-10-01T09:25:45.732 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:25:45.732 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 3, 4, 7, 2, 6] 2023-10-01T09:25:45.732 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-01T09:25:46.378 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:46.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:46.390 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:46 smithi138 ceph-mon[185886]: pgmap v730: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 450 op/s; 34330/269913 objects degraded (12.719%); 17209/269913 objects misplaced (6.376%); 1.9 MiB/s, 31 objects/s recovering 2023-10-01T09:25:46.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:46.394 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:46 smithi138 ceph-mon[192513]: pgmap v730: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 450 op/s; 34330/269913 objects degraded (12.719%); 17209/269913 objects misplaced (6.376%); 1.9 MiB/s, 31 objects/s recovering 2023-10-01T09:25:46.492 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:46.516 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:46 smithi190 ceph-mon[160435]: pgmap v730: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 450 op/s; 34330/269913 objects degraded (12.719%); 17209/269913 objects misplaced (6.376%); 1.9 MiB/s, 31 objects/s recovering 2023-10-01T09:25:46.518 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:46.559 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:46.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:46.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:46.665 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:46.692 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:46.736 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:46.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:46.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:46.841 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:46.867 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:46.914 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:46.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:46.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:47.019 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:25:47.105 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:47.323 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:25:47.324 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:25:47.350 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:25:47.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:47.362 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:47 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 34330/269913 objects degraded (12.719%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:47.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:25:47.363 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:47 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 34330/269913 objects degraded (12.719%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:47.463 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:47.490 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:47.534 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:47.537 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:47 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 34330/269913 objects degraded (12.719%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:47.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:47.639 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:25:47.715 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:48.397 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:25:48.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:48.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:25:48.509 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:48.535 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:48.578 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:48.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:48.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:48.683 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:48.709 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:48.758 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:48.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:48.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:48.863 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:48.969 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:49.194 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:49.194 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:49.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:49.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:49.304 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:49.325 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:25:49.330 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:49.374 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:49.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:49.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:49.464 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:49 smithi138 ceph-mon[185886]: pgmap v731: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 34171/267543 objects degraded (12.772%); 17063/267543 objects misplaced (6.378%); 2.0 MiB/s, 31 objects/s recovering 2023-10-01T09:25:49.464 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:49 smithi138 ceph-mon[192513]: pgmap v731: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 34171/267543 objects degraded (12.772%); 17063/267543 objects misplaced (6.378%); 2.0 MiB/s, 31 objects/s recovering 2023-10-01T09:25:49.481 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:49.507 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:49.550 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:49.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:49 smithi190 ceph-mon[160435]: pgmap v731: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 372 op/s; 34171/267543 objects degraded (12.772%); 17063/267543 objects misplaced (6.378%); 2.0 MiB/s, 31 objects/s recovering 2023-10-01T09:25:49.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:49.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:49.658 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:49.781 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:50.016 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:50.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:50.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:50.129 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:50.206 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:50.303 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:50 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2448317887' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:50.303 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:50 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:50.312 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:25:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:50 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2448317887' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:50 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:50.584 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:50 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2448317887' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:50.585 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:50 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:25:50.988 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:25:50.988 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:25:51.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:51 smithi190 ceph-mon[160435]: pgmap v732: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 33733/264225 objects degraded (12.767%); 17063/264225 objects misplaced (6.458%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T09:25:51.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:51 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:25:51.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:51 smithi190 ceph-mon[160435]: failed to encode map e278 with expected crc 2023-10-01T09:25:51.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:51 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:25:51.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:51 smithi190 ceph-mon[160435]: osdmap e278: 8 total, 7 up, 7 in 2023-10-01T09:25:51.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:51 smithi138 ceph-mon[185886]: pgmap v732: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 33733/264225 objects degraded (12.767%); 17063/264225 objects misplaced (6.458%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T09:25:51.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:51 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:25:51.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:51 smithi138 ceph-mon[185886]: failed to encode map e278 with expected crc 2023-10-01T09:25:51.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:51 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:25:51.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:51 smithi138 ceph-mon[185886]: osdmap e278: 8 total, 7 up, 7 in 2023-10-01T09:25:51.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:51 smithi138 ceph-mon[192513]: pgmap v732: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 423 op/s; 33733/264225 objects degraded (12.767%); 17063/264225 objects misplaced (6.458%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T09:25:51.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:51 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:25:51.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:51 smithi138 ceph-mon[192513]: failed to encode map e278 with expected crc 2023-10-01T09:25:51.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:51 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:25:51.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:51 smithi138 ceph-mon[192513]: osdmap e278: 8 total, 7 up, 7 in 2023-10-01T09:25:52.019 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:52.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:52.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:52.128 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:52.155 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:52.200 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:52.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:52.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:52.308 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:52.456 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:52.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:52 smithi190 ceph-mon[160435]: pgmap v734: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 443 op/s; 33786/262920 objects degraded (12.850%); 16929/262920 objects misplaced (6.439%); 2.0 MiB/s, 31 objects/s recovering 2023-10-01T09:25:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:52 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 33733/264225 objects degraded (12.767%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:52.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:52 smithi138 ceph-mon[192513]: pgmap v734: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 443 op/s; 33786/262920 objects degraded (12.850%); 16929/262920 objects misplaced (6.439%); 2.0 MiB/s, 31 objects/s recovering 2023-10-01T09:25:52.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:52 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 33733/264225 objects degraded (12.767%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:52.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:52 smithi138 ceph-mon[185886]: pgmap v734: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 443 op/s; 33786/262920 objects degraded (12.850%); 16929/262920 objects misplaced (6.439%); 2.0 MiB/s, 31 objects/s recovering 2023-10-01T09:25:52.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:52 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 33733/264225 objects degraded (12.767%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:53.272 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:53.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:53.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:53.382 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:25:53.398 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:25:53.434 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:53.523 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:53.819 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:25:53.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:53.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:25:53.937 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:53.963 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:54.008 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:54.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:54.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:54.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:53 smithi190 ceph-mon[160435]: pgmap v735: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 33429/258711 objects degraded (12.921%); 16797/258711 objects misplaced (6.493%); 2.0 MiB/s, 32 objects/s recovering 2023-10-01T09:25:54.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:53 smithi138 ceph-mon[185886]: pgmap v735: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 33429/258711 objects degraded (12.921%); 16797/258711 objects misplaced (6.493%); 2.0 MiB/s, 32 objects/s recovering 2023-10-01T09:25:54.076 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:53 smithi138 ceph-mon[192513]: pgmap v735: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 33429/258711 objects degraded (12.921%); 16797/258711 objects misplaced (6.493%); 2.0 MiB/s, 32 objects/s recovering 2023-10-01T09:25:54.116 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:54.142 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:54.186 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:54.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:54.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:54.292 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:54.342 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:25:54.379 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:54 smithi138 conmon[185862]: 2023-10-01T09:25:54.108+0000 7f70d64ec700 -1 mon.a@0(leader).osd e278 definitely_dead 0 2023-10-01T09:25:54.380 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:54.494 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:54.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:54.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:54.613 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:54.720 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:54.842 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:54.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:54.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:54.865 INFO:teuthology.orchestra.run.smithi138.stderr:marked out osd.6. 2023-10-01T09:25:54.953 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:54.979 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:55.022 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:55.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:55.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:55.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:54 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1977216032' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T09:25:55.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:54 smithi138 ceph-mon[192513]: Client client.admin marked osd.6 out, after it was down for 42 seconds 2023-10-01T09:25:55.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:54 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1977216032' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T09:25:55.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:54 smithi138 ceph-mon[185886]: Client client.admin marked osd.6 out, after it was down for 42 seconds 2023-10-01T09:25:55.071 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:25:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:25:54] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.33.4" 2023-10-01T09:25:55.129 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:55.152 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:54 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1977216032' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T09:25:55.152 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:54 smithi190 ceph-mon[160435]: Client client.admin marked osd.6 out, after it was down for 42 seconds 2023-10-01T09:25:55.154 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:55.198 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:55.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:55.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:55.306 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:25:55.332 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:55.373 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:25:55.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:55.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:25:55.479 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:25:55.505 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:55.546 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:25:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:55.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:25:55.653 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:55.684 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:56.253 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:56.255 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[192513]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:25:56.255 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1977216032' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T09:25:56.255 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[192513]: osdmap e279: 8 total, 7 up, 6 in 2023-10-01T09:25:56.256 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[192513]: failed to encode map e279 with expected crc 2023-10-01T09:25:56.256 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[192513]: failed to encode map e279 with expected crc 2023-10-01T09:25:56.256 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[192513]: failed to encode map e279 with expected crc 2023-10-01T09:25:56.257 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:25:56.257 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:25:56.257 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[192513]: pgmap v737: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 450 op/s; 33224/256728 objects degraded (12.941%); 16654/256728 objects misplaced (6.487%); 2.0 MiB/s, 31 objects/s recovering 2023-10-01T09:25:56.258 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[185886]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:25:56.258 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1977216032' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T09:25:56.258 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[185886]: osdmap e279: 8 total, 7 up, 6 in 2023-10-01T09:25:56.258 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[185886]: failed to encode map e279 with expected crc 2023-10-01T09:25:56.258 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[185886]: failed to encode map e279 with expected crc 2023-10-01T09:25:56.258 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[185886]: failed to encode map e279 with expected crc 2023-10-01T09:25:56.258 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:25:56.258 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:25:56.258 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:55 smithi138 ceph-mon[185886]: pgmap v737: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 450 op/s; 33224/256728 objects degraded (12.941%); 16654/256728 objects misplaced (6.487%); 2.0 MiB/s, 31 objects/s recovering 2023-10-01T09:25:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:55 smithi190 ceph-mon[160435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:25:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:55 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1977216032' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T09:25:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:55 smithi190 ceph-mon[160435]: osdmap e279: 8 total, 7 up, 6 in 2023-10-01T09:25:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:55 smithi190 ceph-mon[160435]: failed to encode map e279 with expected crc 2023-10-01T09:25:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:55 smithi190 ceph-mon[160435]: failed to encode map e279 with expected crc 2023-10-01T09:25:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:55 smithi190 ceph-mon[160435]: failed to encode map e279 with expected crc 2023-10-01T09:25:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:25:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:25:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:55 smithi190 ceph-mon[160435]: pgmap v737: 89 pgs: 15 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 15 active+undersized, 49 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 450 op/s; 33224/256728 objects degraded (12.941%); 16654/256728 objects misplaced (6.487%); 2.0 MiB/s, 31 objects/s recovering 2023-10-01T09:25:56.462 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:25:56.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:56.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:25:56.565 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:25:56.596 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:56.647 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:25:56.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:56.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:25:56.755 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:25:56.788 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:56.924 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:56 smithi138 ceph-mon[185886]: osdmap e280: 8 total, 7 up, 6 in 2023-10-01T09:25:56.924 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:56 smithi138 ceph-mon[185886]: failed to encode map e280 with expected crc 2023-10-01T09:25:56.924 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:56 smithi138 ceph-mon[185886]: failed to encode map e280 with expected crc 2023-10-01T09:25:56.924 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:56 smithi138 ceph-mon[192513]: osdmap e280: 8 total, 7 up, 6 in 2023-10-01T09:25:56.924 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:56 smithi138 ceph-mon[192513]: failed to encode map e280 with expected crc 2023-10-01T09:25:56.924 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:56 smithi138 ceph-mon[192513]: failed to encode map e280 with expected crc 2023-10-01T09:25:56.948 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:25:57.139 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:25:57.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:57.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:25:57.247 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:25:57.272 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:25:57.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:56 smithi190 ceph-mon[160435]: osdmap e280: 8 total, 7 up, 6 in 2023-10-01T09:25:57.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:56 smithi190 ceph-mon[160435]: failed to encode map e280 with expected crc 2023-10-01T09:25:57.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:56 smithi190 ceph-mon[160435]: failed to encode map e280 with expected crc 2023-10-01T09:25:57.316 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:25:57.350 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:57.366 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:25:57.409 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:25:57.467 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:25:57.492 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:25:57.554 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:25:57.554 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:25:57.554 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:25:57.554 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:25:57.554 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:25:57.555 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:25:57.555 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:25:57.555 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-10-01T09:25:57.555 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:25:57.858 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[185886]: failed to encode map e281 with expected crc 2023-10-01T09:25:57.858 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[185886]: failed to encode map e281 with expected crc 2023-10-01T09:25:57.858 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[185886]: osdmap e281: 8 total, 7 up, 6 in 2023-10-01T09:25:57.858 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[185886]: failed to encode map e281 with expected crc 2023-10-01T09:25:57.858 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[185886]: failed to encode map e281 with expected crc 2023-10-01T09:25:57.858 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[185886]: 7.1 starting backfill to osd.1 from (0'0,0'0] MAX to 278'6801 2023-10-01T09:25:57.859 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[185886]: 7.c starting backfill to osd.2 from (0'0,0'0] MAX to 278'6843 2023-10-01T09:25:57.859 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1417706337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:57.859 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 33224/256728 objects degraded (12.941%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:57.859 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[185886]: pgmap v740: 89 pgs: 3 peering, 1 active+recovery_wait+degraded, 1 remapped+peering, 9 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 8 active+undersized, 59 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 427 op/s; 33096/255225 objects degraded (12.967%); 16522/255225 objects misplaced (6.474%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T09:25:57.859 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:25:57.859 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[192513]: failed to encode map e281 with expected crc 2023-10-01T09:25:57.859 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[192513]: failed to encode map e281 with expected crc 2023-10-01T09:25:57.859 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[192513]: osdmap e281: 8 total, 7 up, 6 in 2023-10-01T09:25:57.859 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[192513]: failed to encode map e281 with expected crc 2023-10-01T09:25:57.859 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[192513]: failed to encode map e281 with expected crc 2023-10-01T09:25:57.860 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[192513]: 7.1 starting backfill to osd.1 from (0'0,0'0] MAX to 278'6801 2023-10-01T09:25:57.860 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[192513]: 7.c starting backfill to osd.2 from (0'0,0'0] MAX to 278'6843 2023-10-01T09:25:57.860 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1417706337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:57 smithi190 ceph-mon[160435]: failed to encode map e281 with expected crc 2023-10-01T09:25:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:57 smithi190 ceph-mon[160435]: failed to encode map e281 with expected crc 2023-10-01T09:25:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:57 smithi190 ceph-mon[160435]: osdmap e281: 8 total, 7 up, 6 in 2023-10-01T09:25:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:57 smithi190 ceph-mon[160435]: failed to encode map e281 with expected crc 2023-10-01T09:25:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:57 smithi190 ceph-mon[160435]: failed to encode map e281 with expected crc 2023-10-01T09:25:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:57 smithi190 ceph-mon[160435]: 7.1 starting backfill to osd.1 from (0'0,0'0] MAX to 278'6801 2023-10-01T09:25:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:57 smithi190 ceph-mon[160435]: 7.c starting backfill to osd.2 from (0'0,0'0] MAX to 278'6843 2023-10-01T09:25:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1417706337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:25:58.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:57 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 33224/256728 objects degraded (12.941%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:58.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:57 smithi190 ceph-mon[160435]: pgmap v740: 89 pgs: 3 peering, 1 active+recovery_wait+degraded, 1 remapped+peering, 9 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 8 active+undersized, 59 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 427 op/s; 33096/255225 objects degraded (12.967%); 16522/255225 objects misplaced (6.474%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T09:25:58.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:57 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:25:58.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 33224/256728 objects degraded (12.941%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T09:25:58.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[192513]: pgmap v740: 89 pgs: 3 peering, 1 active+recovery_wait+degraded, 1 remapped+peering, 9 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 8 active+undersized, 59 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 427 op/s; 33096/255225 objects degraded (12.967%); 16522/255225 objects misplaced (6.474%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T09:25:58.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:57 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:25:59.515 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:25:59.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:25:59.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:25:59.627 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:25:59.765 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:25:59.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:25:59 smithi138 ceph-mon[185886]: pgmap v741: 89 pgs: 3 peering, 2 active+recovery_wait+degraded, 1 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 74 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 294 op/s; 36111/253407 objects degraded (14.250%); 15641/253407 objects misplaced (6.172%); 2.7 MiB/s, 44 objects/s recovering 2023-10-01T09:25:59.883 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:25:59.883 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:26:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:25:59 smithi138 ceph-mon[192513]: pgmap v741: 89 pgs: 3 peering, 2 active+recovery_wait+degraded, 1 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 74 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 294 op/s; 36111/253407 objects degraded (14.250%); 15641/253407 objects misplaced (6.172%); 2.7 MiB/s, 44 objects/s recovering 2023-10-01T09:26:00.137 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:26:00.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:00.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:26:00.247 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:26:00.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:25:59 smithi190 ceph-mon[160435]: pgmap v741: 89 pgs: 3 peering, 2 active+recovery_wait+degraded, 1 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 74 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 294 op/s; 36111/253407 objects degraded (14.250%); 15641/253407 objects misplaced (6.172%); 2.7 MiB/s, 44 objects/s recovering 2023-10-01T09:26:00.361 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:00.509 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:26:00.510 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:26:00.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:26:00.510 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:26:00.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.510 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:121341 5.b 5:db80cfa5:::benchmark_data_smithi138_240338_object6204:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e125)", 2023-10-01T09:26:00.510 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:16:22.525767+0000", 2023-10-01T09:26:00.510 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 577.97889718500005, 2023-10-01T09:26:00.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.010709842, 2023-10-01T09:26:00.510 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.511 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:26:00.511 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.511 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.511 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.511 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121341 2023-10-01T09:26:00.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.511 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525767+0000", 2023-10-01T09:26:00.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525767+0000", 2023-10-01T09:26:00.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525766+0000", 2023-10-01T09:26:00.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:00.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525769+0000", 2023-10-01T09:26:00.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.993e-06 2023-10-01T09:26:00.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525776+0000", 2023-10-01T09:26:00.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3829999999999997e-06 2023-10-01T09:26:00.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525782+0000", 2023-10-01T09:26:00.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7440000000000001e-06 2023-10-01T09:26:00.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.514 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525829+0000", 2023-10-01T09:26:00.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7169999999999997e-05 2023-10-01T09:26:00.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.514 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525840+0000", 2023-10-01T09:26:00.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0494e-05 2023-10-01T09:26:00.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.534631+0000", 2023-10-01T09:26:00.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0087906170000001 2023-10-01T09:26:00.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.515 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.536476+0000", 2023-10-01T09:26:00.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001845867 2023-10-01T09:26:00.515 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.515 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.515 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.516 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:121349 5.b 5:dee15a14:::benchmark_data_smithi138_240338_object6212:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e125)", 2023-10-01T09:26:00.516 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:16:22.525794+0000", 2023-10-01T09:26:00.516 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 577.97887029799995, 2023-10-01T09:26:00.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0346446920000001, 2023-10-01T09:26:00.516 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.516 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.516 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.516 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.516 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.516 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121349 2023-10-01T09:26:00.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.517 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525794+0000", 2023-10-01T09:26:00.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525794+0000", 2023-10-01T09:26:00.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.518 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525793+0000", 2023-10-01T09:26:00.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:00.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.518 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525796+0000", 2023-10-01T09:26:00.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4490000000000002e-06 2023-10-01T09:26:00.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.518 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525801+0000", 2023-10-01T09:26:00.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.536e-06 2023-10-01T09:26:00.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.519 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525806+0000", 2023-10-01T09:26:00.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7330000000000003e-06 2023-10-01T09:26:00.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.519 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525866+0000", 2023-10-01T09:26:00.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0538999999999998e-05 2023-10-01T09:26:00.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.525875+0000", 2023-10-01T09:26:00.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.772e-06 2023-10-01T09:26:00.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.536750+0000", 2023-10-01T09:26:00.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0108747680000001 2023-10-01T09:26:00.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.537437+0000", 2023-10-01T09:26:00.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00068692000000000004 2023-10-01T09:26:00.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.521 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:26:00.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.537550+0000", 2023-10-01T09:26:00.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011352 2023-10-01T09:26:00.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.521 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.547975+0000", 2023-10-01T09:26:00.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010424638999999999 2023-10-01T09:26:00.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.522 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.522 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.548036+0000", 2023-10-01T09:26:00.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0909000000000002e-05 2023-10-01T09:26:00.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.522 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.522 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560353+0000", 2023-10-01T09:26:00.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012316720999999999 2023-10-01T09:26:00.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560397+0000", 2023-10-01T09:26:00.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4697999999999998e-05 2023-10-01T09:26:00.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560438+0000", 2023-10-01T09:26:00.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0871000000000002e-05 2023-10-01T09:26:00.523 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.523 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.523 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.524 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:121358 5.8 5:166d68c4:::benchmark_data_smithi138_240338_object6221:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:26:00.524 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:16:22.539405+0000", 2023-10-01T09:26:00.524 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 577.96525877099998, 2023-10-01T09:26:00.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0166601280000001, 2023-10-01T09:26:00.524 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.524 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.524 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.524 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.524 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.524 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121358 2023-10-01T09:26:00.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.525 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539405+0000", 2023-10-01T09:26:00.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539405+0000", 2023-10-01T09:26:00.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539403+0000", 2023-10-01T09:26:00.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:26:00.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539413+0000", 2023-10-01T09:26:00.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8060000000000006e-06 2023-10-01T09:26:00.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539434+0000", 2023-10-01T09:26:00.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0557e-05 2023-10-01T09:26:00.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.527 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539444+0000", 2023-10-01T09:26:00.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0197000000000001e-05 2023-10-01T09:26:00.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.527 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539526+0000", 2023-10-01T09:26:00.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2010999999999998e-05 2023-10-01T09:26:00.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.539539+0000", 2023-10-01T09:26:00.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3654e-05 2023-10-01T09:26:00.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.535296+0000", 2023-10-01T09:26:00.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99575626900000003 2023-10-01T09:26:00.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.535893+0000", 2023-10-01T09:26:00.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00059725200000000005 2023-10-01T09:26:00.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.529 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2", 2023-10-01T09:26:00.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.535973+0000", 2023-10-01T09:26:00.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0136000000000007e-05 2023-10-01T09:26:00.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.529 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.546652+0000", 2023-10-01T09:26:00.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01067912 2023-10-01T09:26:00.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.530 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.530 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.530 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.546866+0000", 2023-10-01T09:26:00.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021356899999999999 2023-10-01T09:26:00.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.530 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.530 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.530 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.555975+0000", 2023-10-01T09:26:00.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0091089359999999998 2023-10-01T09:26:00.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.556025+0000", 2023-10-01T09:26:00.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0182999999999997e-05 2023-10-01T09:26:00.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.556065+0000", 2023-10-01T09:26:00.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0333000000000002e-05 2023-10-01T09:26:00.532 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.532 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.532 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.532 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:121363 5.b 5:d08c45c8:::benchmark_data_smithi138_240338_object6226:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:26:00.532 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:16:22.544661+0000", 2023-10-01T09:26:00.532 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 577.96000242100001, 2023-10-01T09:26:00.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.015913072, 2023-10-01T09:26:00.532 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.532 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.532 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.533 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.533 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.533 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121363 2023-10-01T09:26:00.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.533 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544661+0000", 2023-10-01T09:26:00.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544661+0000", 2023-10-01T09:26:00.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.534 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544660+0000", 2023-10-01T09:26:00.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:26:00.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.534 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544666+0000", 2023-10-01T09:26:00.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.366e-06 2023-10-01T09:26:00.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544677+0000", 2023-10-01T09:26:00.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1012000000000001e-05 2023-10-01T09:26:00.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544683+0000", 2023-10-01T09:26:00.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.178e-06 2023-10-01T09:26:00.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544729+0000", 2023-10-01T09:26:00.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6149999999999997e-05 2023-10-01T09:26:00.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.544739+0000", 2023-10-01T09:26:00.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0042e-05 2023-10-01T09:26:00.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.541089+0000", 2023-10-01T09:26:00.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99635048000000004 2023-10-01T09:26:00.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.541181+0000", 2023-10-01T09:26:00.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1774000000000006e-05 2023-10-01T09:26:00.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:26:00.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.541279+0000", 2023-10-01T09:26:00.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7918999999999995e-05 2023-10-01T09:26:00.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.548094+0000", 2023-10-01T09:26:00.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0068148089999999998 2023-10-01T09:26:00.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.548102+0000", 2023-10-01T09:26:00.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.875e-06 2023-10-01T09:26:00.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.539 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.539 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560506+0000", 2023-10-01T09:26:00.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012403765000000001 2023-10-01T09:26:00.539 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.539 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.539 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560543+0000", 2023-10-01T09:26:00.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7039000000000002e-05 2023-10-01T09:26:00.539 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.539 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.539 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560574+0000", 2023-10-01T09:26:00.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1794000000000001e-05 2023-10-01T09:26:00.540 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.540 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.540 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.540 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:121366 5.b 5:dacdb783:::benchmark_data_smithi138_240338_object6229:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:26:00.540 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:16:22.553337+0000", 2023-10-01T09:26:00.540 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 577.95132685800002, 2023-10-01T09:26:00.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.012110101, 2023-10-01T09:26:00.540 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.540 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.541 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.541 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.541 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121366 2023-10-01T09:26:00.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.541 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553337+0000", 2023-10-01T09:26:00.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553337+0000", 2023-10-01T09:26:00.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553335+0000", 2023-10-01T09:26:00.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:26:00.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553342+0000", 2023-10-01T09:26:00.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0210000000000002e-06 2023-10-01T09:26:00.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553356+0000", 2023-10-01T09:26:00.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3891e-05 2023-10-01T09:26:00.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553363+0000", 2023-10-01T09:26:00.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8549999999999996e-06 2023-10-01T09:26:00.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553411+0000", 2023-10-01T09:26:00.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8300999999999997e-05 2023-10-01T09:26:00.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.553420+0000", 2023-10-01T09:26:00.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8859999999999993e-06 2023-10-01T09:26:00.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.544670+0000", 2023-10-01T09:26:00.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99124969399999996 2023-10-01T09:26:00.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.544794+0000", 2023-10-01T09:26:00.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000124003 2023-10-01T09:26:00.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:26:00.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.544915+0000", 2023-10-01T09:26:00.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120844 2023-10-01T09:26:00.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560476+0000", 2023-10-01T09:26:00.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015561339 2023-10-01T09:26:00.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.560612+0000", 2023-10-01T09:26:00.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013593200000000001 2023-10-01T09:26:00.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.565305+0000", 2023-10-01T09:26:00.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0046935630000000004 2023-10-01T09:26:00.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.565403+0000", 2023-10-01T09:26:00.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7567999999999998e-05 2023-10-01T09:26:00.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.565447+0000", 2023-10-01T09:26:00.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4085999999999998e-05 2023-10-01T09:26:00.548 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.548 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.548 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.548 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:121371 5.b 5:db764039:::benchmark_data_smithi138_240338_object6234:head [delete] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-10-01T09:26:00.548 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:16:22.554473+0000", 2023-10-01T09:26:00.548 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 577.95019083600005, 2023-10-01T09:26:00.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0121766640000001, 2023-10-01T09:26:00.549 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.549 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.549 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.549 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.549 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.549 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121371 2023-10-01T09:26:00.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.549 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554473+0000", 2023-10-01T09:26:00.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554473+0000", 2023-10-01T09:26:00.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554472+0000", 2023-10-01T09:26:00.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:00.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554476+0000", 2023-10-01T09:26:00.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.726e-06 2023-10-01T09:26:00.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554487+0000", 2023-10-01T09:26:00.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0890000000000001e-05 2023-10-01T09:26:00.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554493+0000", 2023-10-01T09:26:00.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6350000000000003e-06 2023-10-01T09:26:00.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554527+0000", 2023-10-01T09:26:00.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.392e-05 2023-10-01T09:26:00.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:22.554534+0000", 2023-10-01T09:26:00.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8010000000000004e-06 2023-10-01T09:26:00.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.548141+0000", 2023-10-01T09:26:00.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99360651499999997 2023-10-01T09:26:00.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.548274+0000", 2023-10-01T09:26:00.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013291900000000001 2023-10-01T09:26:00.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:26:00.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.548391+0000", 2023-10-01T09:26:00.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117315 2023-10-01T09:26:00.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.565456+0000", 2023-10-01T09:26:00.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017065104000000001 2023-10-01T09:26:00.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.566498+0000", 2023-10-01T09:26:00.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001041464 2023-10-01T09:26:00.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.566556+0000", 2023-10-01T09:26:00.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8624e-05 2023-10-01T09:26:00.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.566601+0000", 2023-10-01T09:26:00.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5129000000000001e-05 2023-10-01T09:26:00.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:16:23.566650+0000", 2023-10-01T09:26:00.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8253e-05 2023-10-01T09:26:00.556 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.556 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.556 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.556 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:209439 5.b 5:dfe595a2:::benchmark_data_smithi138_240338_object94302:head [delete] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-01T09:26:00.556 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:18:30.402237+0000", 2023-10-01T09:26:00.556 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 450.10242713999997, 2023-10-01T09:26:00.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1907945639999999, 2023-10-01T09:26:00.557 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.557 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:26:00.557 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.557 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.557 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.557 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 209439 2023-10-01T09:26:00.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.557 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402237+0000", 2023-10-01T09:26:00.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.558 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.558 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402237+0000", 2023-10-01T09:26:00.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.558 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.558 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402235+0000", 2023-10-01T09:26:00.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:00.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402243+0000", 2023-10-01T09:26:00.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5229999999999998e-06 2023-10-01T09:26:00.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.559 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402254+0000", 2023-10-01T09:26:00.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1741e-05 2023-10-01T09:26:00.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.559 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402262+0000", 2023-10-01T09:26:00.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8019999999999992e-06 2023-10-01T09:26:00.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402309+0000", 2023-10-01T09:26:00.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6397000000000003e-05 2023-10-01T09:26:00.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402441+0000", 2023-10-01T09:26:00.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013244 2023-10-01T09:26:00.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:26:00.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.402560+0000", 2023-10-01T09:26:00.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011885800000000001 2023-10-01T09:26:00.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.560276+0000", 2023-10-01T09:26:00.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.15771628900000001 2023-10-01T09:26:00.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.560286+0000", 2023-10-01T09:26:00.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5850000000000006e-06 2023-10-01T09:26:00.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404172+0000", 2023-10-01T09:26:00.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.843886358 2023-10-01T09:26:00.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404187+0000", 2023-10-01T09:26:00.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4751e-05 2023-10-01T09:26:00.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.592961+0000", 2023-10-01T09:26:00.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1887738130000001 2023-10-01T09:26:00.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.593031+0000", 2023-10-01T09:26:00.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0541000000000001e-05 2023-10-01T09:26:00.563 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.563 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.563 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.564 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:209448 5.8 5:1e06e43d:::benchmark_data_smithi138_240338_object94311:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-01T09:26:00.564 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:18:30.550675+0000", 2023-10-01T09:26:00.564 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 449.95398834899999, 2023-10-01T09:26:00.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2198924120000001, 2023-10-01T09:26:00.564 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.564 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.564 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.564 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.564 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.564 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 209448 2023-10-01T09:26:00.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.565 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.550675+0000", 2023-10-01T09:26:00.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.550675+0000", 2023-10-01T09:26:00.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.550675+0000", 2023-10-01T09:26:00.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:00.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.550678+0000", 2023-10-01T09:26:00.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0529999999999998e-06 2023-10-01T09:26:00.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.550684+0000", 2023-10-01T09:26:00.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6749999999999999e-06 2023-10-01T09:26:00.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.550689+0000", 2023-10-01T09:26:00.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9599999999999999e-06 2023-10-01T09:26:00.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.559523+0000", 2023-10-01T09:26:00.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088342590000000006 2023-10-01T09:26:00.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.559534+0000", 2023-10-01T09:26:00.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1218e-05 2023-10-01T09:26:00.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404403+0000", 2023-10-01T09:26:00.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84486879400000003 2023-10-01T09:26:00.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404413+0000", 2023-10-01T09:26:00.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9769999999999995e-06 2023-10-01T09:26:00.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.632637+0000", 2023-10-01T09:26:00.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.228223751 2023-10-01T09:26:00.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.632784+0000", 2023-10-01T09:26:00.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000147377 2023-10-01T09:26:00.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:26:00.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.633399+0000", 2023-10-01T09:26:00.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00061477500000000002 2023-10-01T09:26:00.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.648265+0000", 2023-10-01T09:26:00.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014866084999999999 2023-10-01T09:26:00.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.648340+0000", 2023-10-01T09:26:00.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4616999999999994e-05 2023-10-01T09:26:00.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.652364+0000", 2023-10-01T09:26:00.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040248239999999998 2023-10-01T09:26:00.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770452+0000", 2023-10-01T09:26:00.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.118087964 2023-10-01T09:26:00.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770486+0000", 2023-10-01T09:26:00.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3207000000000002e-05 2023-10-01T09:26:00.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770568+0000", 2023-10-01T09:26:00.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2259e-05 2023-10-01T09:26:00.572 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.573 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.573 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.573 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:209453 5.b 5:d1ec2937:::benchmark_data_smithi138_240338_object94316:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-01T09:26:00.573 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:18:30.567968+0000", 2023-10-01T09:26:00.573 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 449.93669552599999, 2023-10-01T09:26:00.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2009798319999998, 2023-10-01T09:26:00.573 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.573 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.573 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.573 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.574 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.574 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 209453 2023-10-01T09:26:00.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.574 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.567968+0000", 2023-10-01T09:26:00.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.567968+0000", 2023-10-01T09:26:00.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.567967+0000", 2023-10-01T09:26:00.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:00.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.567973+0000", 2023-10-01T09:26:00.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7300000000000002e-06 2023-10-01T09:26:00.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.567985+0000", 2023-10-01T09:26:00.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2026e-05 2023-10-01T09:26:00.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.567993+0000", 2023-10-01T09:26:00.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9540000000000004e-06 2023-10-01T09:26:00.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.569152+0000", 2023-10-01T09:26:00.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011594540000000001 2023-10-01T09:26:00.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.569159+0000", 2023-10-01T09:26:00.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5860000000000001e-06 2023-10-01T09:26:00.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404618+0000", 2023-10-01T09:26:00.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.83545915000000004 2023-10-01T09:26:00.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404626+0000", 2023-10-01T09:26:00.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3089999999999996e-06 2023-10-01T09:26:00.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.593313+0000", 2023-10-01T09:26:00.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.188686565 2023-10-01T09:26:00.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.593397+0000", 2023-10-01T09:26:00.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4592999999999995e-05 2023-10-01T09:26:00.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5,7", 2023-10-01T09:26:00.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.593957+0000", 2023-10-01T09:26:00.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00055995199999999995 2023-10-01T09:26:00.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.603958+0000", 2023-10-01T09:26:00.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010000754000000001 2023-10-01T09:26:00.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.604212+0000", 2023-10-01T09:26:00.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025370699999999999 2023-10-01T09:26:00.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.765406+0000", 2023-10-01T09:26:00.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16119386699999999 2023-10-01T09:26:00.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.768761+0000", 2023-10-01T09:26:00.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003355369 2023-10-01T09:26:00.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.768822+0000", 2023-10-01T09:26:00.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1327000000000004e-05 2023-10-01T09:26:00.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.768948+0000", 2023-10-01T09:26:00.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000125759 2023-10-01T09:26:00.582 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.582 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.582 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.582 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:209456 5.8 5:180e28dd:::benchmark_data_smithi138_240338_object94319:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-01T09:26:00.582 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:18:30.571789+0000", 2023-10-01T09:26:00.583 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 449.93287499299998, 2023-10-01T09:26:00.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1990186619999998, 2023-10-01T09:26:00.583 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.583 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.583 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.583 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.583 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.583 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 209456 2023-10-01T09:26:00.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.583 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571789+0000", 2023-10-01T09:26:00.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571789+0000", 2023-10-01T09:26:00.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571788+0000", 2023-10-01T09:26:00.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:00.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571792+0000", 2023-10-01T09:26:00.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7000000000000002e-06 2023-10-01T09:26:00.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571800+0000", 2023-10-01T09:26:00.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9719999999999992e-06 2023-10-01T09:26:00.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571805+0000", 2023-10-01T09:26:00.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9099999999999996e-06 2023-10-01T09:26:00.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571839+0000", 2023-10-01T09:26:00.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4468999999999998e-05 2023-10-01T09:26:00.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.571845+0000", 2023-10-01T09:26:00.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1120000000000001e-06 2023-10-01T09:26:00.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404441+0000", 2023-10-01T09:26:00.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.83259549099999997 2023-10-01T09:26:00.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404449+0000", 2023-10-01T09:26:00.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7400000000000004e-06 2023-10-01T09:26:00.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.642769+0000", 2023-10-01T09:26:00.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.238320885 2023-10-01T09:26:00.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.642923+0000", 2023-10-01T09:26:00.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015377899999999999 2023-10-01T09:26:00.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:26:00.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.643084+0000", 2023-10-01T09:26:00.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016034900000000001 2023-10-01T09:26:00.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.648400+0000", 2023-10-01T09:26:00.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0053166079999999996 2023-10-01T09:26:00.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.648409+0000", 2023-10-01T09:26:00.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.789e-06 2023-10-01T09:26:00.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.652411+0000", 2023-10-01T09:26:00.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040017539999999997 2023-10-01T09:26:00.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770628+0000", 2023-10-01T09:26:00.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11821683700000001 2023-10-01T09:26:00.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770682+0000", 2023-10-01T09:26:00.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4422999999999997e-05 2023-10-01T09:26:00.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770807+0000", 2023-10-01T09:26:00.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012551500000000001 2023-10-01T09:26:00.591 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.591 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.591 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.592 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:209458 5.8 5:156e792b:::benchmark_data_smithi138_240338_object94321:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-01T09:26:00.592 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:18:30.576504+0000", 2023-10-01T09:26:00.592 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 449.928159623, 2023-10-01T09:26:00.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1944605589999999, 2023-10-01T09:26:00.592 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.592 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.592 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.592 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.592 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.592 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 209458 2023-10-01T09:26:00.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.593 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576504+0000", 2023-10-01T09:26:00.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576504+0000", 2023-10-01T09:26:00.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576503+0000", 2023-10-01T09:26:00.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:00.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576509+0000", 2023-10-01T09:26:00.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3899999999999998e-06 2023-10-01T09:26:00.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576520+0000", 2023-10-01T09:26:00.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0701e-05 2023-10-01T09:26:00.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576525+0000", 2023-10-01T09:26:00.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4659999999999998e-06 2023-10-01T09:26:00.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576571+0000", 2023-10-01T09:26:00.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6239999999999998e-05 2023-10-01T09:26:00.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:30.576580+0000", 2023-10-01T09:26:00.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0629999999999995e-06 2023-10-01T09:26:00.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404546+0000", 2023-10-01T09:26:00.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.82796542900000003 2023-10-01T09:26:00.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:31.404556+0000", 2023-10-01T09:26:00.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8879999999999992e-06 2023-10-01T09:26:00.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.645642+0000", 2023-10-01T09:26:00.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.241086329 2023-10-01T09:26:00.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.645749+0000", 2023-10-01T09:26:00.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106863 2023-10-01T09:26:00.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:26:00.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.645871+0000", 2023-10-01T09:26:00.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121784 2023-10-01T09:26:00.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.648417+0000", 2023-10-01T09:26:00.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002546169 2023-10-01T09:26:00.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.648632+0000", 2023-10-01T09:26:00.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021547800000000001 2023-10-01T09:26:00.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.652454+0000", 2023-10-01T09:26:00.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0038215530000000001 2023-10-01T09:26:00.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770853+0000", 2023-10-01T09:26:00.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.118399108 2023-10-01T09:26:00.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770890+0000", 2023-10-01T09:26:00.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7047e-05 2023-10-01T09:26:00.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:18:32.770965+0000", 2023-10-01T09:26:00.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4610999999999994e-05 2023-10-01T09:26:00.601 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.601 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.601 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.601 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227518 5.b 5:dd303cc1:::benchmark_data_smithi138_240338_object112381:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T09:26:00.601 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.540699+0000", 2023-10-01T09:26:00.601 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 415.96396500399999, 2023-10-01T09:26:00.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2259289409999998, 2023-10-01T09:26:00.601 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.601 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:26:00.601 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.601 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.602 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.602 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227518 2023-10-01T09:26:00.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.602 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540699+0000", 2023-10-01T09:26:00.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540699+0000", 2023-10-01T09:26:00.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540698+0000", 2023-10-01T09:26:00.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:00.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540702+0000", 2023-10-01T09:26:00.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5999999999999998e-06 2023-10-01T09:26:00.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540708+0000", 2023-10-01T09:26:00.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7790000000000001e-06 2023-10-01T09:26:00.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540713+0000", 2023-10-01T09:26:00.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0540000000000002e-06 2023-10-01T09:26:00.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540759+0000", 2023-10-01T09:26:00.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6001000000000002e-05 2023-10-01T09:26:00.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540886+0000", 2023-10-01T09:26:00.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012746099999999999 2023-10-01T09:26:00.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:26:00.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540997+0000", 2023-10-01T09:26:00.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110389 2023-10-01T09:26:00.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.553848+0000", 2023-10-01T09:26:00.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012850996 2023-10-01T09:26:00.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.559796+0000", 2023-10-01T09:26:00.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059481589999999997 2023-10-01T09:26:00.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.559807+0000", 2023-10-01T09:26:00.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0993e-05 2023-10-01T09:26:00.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547860+0000", 2023-10-01T09:26:00.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98805305899999996 2023-10-01T09:26:00.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547873+0000", 2023-10-01T09:26:00.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.322e-05 2023-10-01T09:26:00.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.750500+0000", 2023-10-01T09:26:00.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.202626795 2023-10-01T09:26:00.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:26:00.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.750659+0000", 2023-10-01T09:26:00.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015917500000000001 2023-10-01T09:26:00.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.766393+0000", 2023-10-01T09:26:00.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015734056999999999 2023-10-01T09:26:00.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.766628+0000", 2023-10-01T09:26:00.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00023459500000000001 2023-10-01T09:26:00.609 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.609 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.609 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.610 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227520 5.b 5:d8712104:::benchmark_data_smithi138_240338_object112383:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T09:26:00.610 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.547004+0000", 2023-10-01T09:26:00.610 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 415.95765981400001, 2023-10-01T09:26:00.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.227945434, 2023-10-01T09:26:00.610 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.610 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:26:00.610 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.610 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.610 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.610 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227520 2023-10-01T09:26:00.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.611 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547004+0000", 2023-10-01T09:26:00.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547004+0000", 2023-10-01T09:26:00.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547002+0000", 2023-10-01T09:26:00.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:26:00.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547011+0000", 2023-10-01T09:26:00.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3419999999999997e-06 2023-10-01T09:26:00.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547026+0000", 2023-10-01T09:26:00.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4453e-05 2023-10-01T09:26:00.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547042+0000", 2023-10-01T09:26:00.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5999999999999999e-05 2023-10-01T09:26:00.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.553877+0000", 2023-10-01T09:26:00.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006835163 2023-10-01T09:26:00.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.554019+0000", 2023-10-01T09:26:00.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014198699999999999 2023-10-01T09:26:00.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:26:00.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.554141+0000", 2023-10-01T09:26:00.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000122032 2023-10-01T09:26:00.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.559831+0000", 2023-10-01T09:26:00.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005690339 2023-10-01T09:26:00.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.559838+0000", 2023-10-01T09:26:00.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1690000000000003e-06 2023-10-01T09:26:00.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547900+0000", 2023-10-01T09:26:00.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98806163599999997 2023-10-01T09:26:00.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547905+0000", 2023-10-01T09:26:00.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2580000000000003e-06 2023-10-01T09:26:00.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.750939+0000", 2023-10-01T09:26:00.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2030342000000001 2023-10-01T09:26:00.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:26:00.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751046+0000", 2023-10-01T09:26:00.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106037 2023-10-01T09:26:00.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.774856+0000", 2023-10-01T09:26:00.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.023810120000000001 2023-10-01T09:26:00.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.774949+0000", 2023-10-01T09:26:00.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3781000000000002e-05 2023-10-01T09:26:00.618 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.618 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.618 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.618 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227542 5.8 5:163a5025:::benchmark_data_smithi138_240338_object112405:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:26:00.618 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.593796+0000", 2023-10-01T09:26:00.618 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 415.91086736099999, 2023-10-01T09:26:00.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.158330061, 2023-10-01T09:26:00.618 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.618 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.618 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.619 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.619 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.619 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227542 2023-10-01T09:26:00.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.619 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593796+0000", 2023-10-01T09:26:00.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593796+0000", 2023-10-01T09:26:00.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593795+0000", 2023-10-01T09:26:00.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:00.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593802+0000", 2023-10-01T09:26:00.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2359999999999998e-06 2023-10-01T09:26:00.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593815+0000", 2023-10-01T09:26:00.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2765000000000001e-05 2023-10-01T09:26:00.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593823+0000", 2023-10-01T09:26:00.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0220000000000004e-06 2023-10-01T09:26:00.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.596113+0000", 2023-10-01T09:26:00.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002289999 2023-10-01T09:26:00.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.596128+0000", 2023-10-01T09:26:00.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4827000000000001e-05 2023-10-01T09:26:00.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.549844+0000", 2023-10-01T09:26:00.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95371578099999998 2023-10-01T09:26:00.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.549855+0000", 2023-10-01T09:26:00.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.163e-05 2023-10-01T09:26:00.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.734943+0000", 2023-10-01T09:26:00.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1850879670000001 2023-10-01T09:26:00.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.735123+0000", 2023-10-01T09:26:00.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000179347 2023-10-01T09:26:00.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:26:00.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.735771+0000", 2023-10-01T09:26:00.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00064860999999999999 2023-10-01T09:26:00.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751876+0000", 2023-10-01T09:26:00.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016104579000000001 2023-10-01T09:26:00.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751917+0000", 2023-10-01T09:26:00.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1201000000000001e-05 2023-10-01T09:26:00.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751925+0000", 2023-10-01T09:26:00.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8830000000000006e-06 2023-10-01T09:26:00.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751962+0000", 2023-10-01T09:26:00.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7521999999999998e-05 2023-10-01T09:26:00.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.752015+0000", 2023-10-01T09:26:00.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2407999999999997e-05 2023-10-01T09:26:00.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.752127+0000", 2023-10-01T09:26:00.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111671 2023-10-01T09:26:00.627 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.627 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.627 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.627 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227543 5.b 5:d7088ca7:::benchmark_data_smithi138_240338_object112406:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:26:00.627 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.593966+0000", 2023-10-01T09:26:00.627 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 415.91069792899998, 2023-10-01T09:26:00.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2140065830000002, 2023-10-01T09:26:00.627 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.628 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.628 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.628 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.628 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.628 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227543 2023-10-01T09:26:00.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.628 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593966+0000", 2023-10-01T09:26:00.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593966+0000", 2023-10-01T09:26:00.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593965+0000", 2023-10-01T09:26:00.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:26:00.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593971+0000", 2023-10-01T09:26:00.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5830000000000003e-06 2023-10-01T09:26:00.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593981+0000", 2023-10-01T09:26:00.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4229999999999994e-06 2023-10-01T09:26:00.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593989+0000", 2023-10-01T09:26:00.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2039999999999997e-06 2023-10-01T09:26:00.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.594069+0000", 2023-10-01T09:26:00.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0267999999999998e-05 2023-10-01T09:26:00.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.594082+0000", 2023-10-01T09:26:00.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2901000000000001e-05 2023-10-01T09:26:00.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547917+0000", 2023-10-01T09:26:00.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95383516800000001 2023-10-01T09:26:00.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547921+0000", 2023-10-01T09:26:00.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9430000000000002e-06 2023-10-01T09:26:00.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751073+0000", 2023-10-01T09:26:00.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2031520040000001 2023-10-01T09:26:00.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751182+0000", 2023-10-01T09:26:00.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108553 2023-10-01T09:26:00.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5,7", 2023-10-01T09:26:00.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751816+0000", 2023-10-01T09:26:00.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00063413800000000002 2023-10-01T09:26:00.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.771765+0000", 2023-10-01T09:26:00.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019948585000000001 2023-10-01T09:26:00.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.771773+0000", 2023-10-01T09:26:00.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8589999999999994e-06 2023-10-01T09:26:00.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.789897+0000", 2023-10-01T09:26:00.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018123378999999998 2023-10-01T09:26:00.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.807826+0000", 2023-10-01T09:26:00.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017929269000000001 2023-10-01T09:26:00.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.807867+0000", 2023-10-01T09:26:00.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1131000000000003e-05 2023-10-01T09:26:00.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.807972+0000", 2023-10-01T09:26:00.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105274 2023-10-01T09:26:00.636 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.636 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.636 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.636 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227549 5.b 5:dca5cded:::benchmark_data_smithi138_240338_object112412:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:26:00.636 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.607851+0000", 2023-10-01T09:26:00.636 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 415.89681294399998, 2023-10-01T09:26:00.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2003441270000001, 2023-10-01T09:26:00.636 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.637 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.637 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.637 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.637 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.637 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227549 2023-10-01T09:26:00.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.637 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607851+0000", 2023-10-01T09:26:00.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607851+0000", 2023-10-01T09:26:00.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607849+0000", 2023-10-01T09:26:00.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:26:00.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607857+0000", 2023-10-01T09:26:00.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0649999999999999e-06 2023-10-01T09:26:00.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607872+0000", 2023-10-01T09:26:00.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5038000000000001e-05 2023-10-01T09:26:00.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607880+0000", 2023-10-01T09:26:00.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8690000000000004e-06 2023-10-01T09:26:00.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607947+0000", 2023-10-01T09:26:00.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6847000000000006e-05 2023-10-01T09:26:00.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607959+0000", 2023-10-01T09:26:00.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2575000000000001e-05 2023-10-01T09:26:00.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547932+0000", 2023-10-01T09:26:00.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93997244999999996 2023-10-01T09:26:00.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547936+0000", 2023-10-01T09:26:00.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.878e-06 2023-10-01T09:26:00.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.762267+0000", 2023-10-01T09:26:00.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.21433116 2023-10-01T09:26:00.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.762399+0000", 2023-10-01T09:26:00.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013192999999999999 2023-10-01T09:26:00.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5,7", 2023-10-01T09:26:00.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.762536+0000", 2023-10-01T09:26:00.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000137439 2023-10-01T09:26:00.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.771780+0000", 2023-10-01T09:26:00.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0092435450000000006 2023-10-01T09:26:00.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.776336+0000", 2023-10-01T09:26:00.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004556264 2023-10-01T09:26:00.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.789967+0000", 2023-10-01T09:26:00.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013631228 2023-10-01T09:26:00.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808023+0000", 2023-10-01T09:26:00.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018056274000000001 2023-10-01T09:26:00.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808075+0000", 2023-10-01T09:26:00.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1910000000000003e-05 2023-10-01T09:26:00.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808195+0000", 2023-10-01T09:26:00.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011964 2023-10-01T09:26:00.645 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.645 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.645 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.645 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227553 5.8 5:1337bdd0:::benchmark_data_smithi138_240338_object112416:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:26:00.645 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.622592+0000", 2023-10-01T09:26:00.645 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 415.88207156800001, 2023-10-01T09:26:00.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.142155657, 2023-10-01T09:26:00.646 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.646 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.646 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.646 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.646 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.646 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227553 2023-10-01T09:26:00.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.646 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622592+0000", 2023-10-01T09:26:00.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622592+0000", 2023-10-01T09:26:00.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622590+0000", 2023-10-01T09:26:00.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:26:00.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622599+0000", 2023-10-01T09:26:00.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9509999999999995e-06 2023-10-01T09:26:00.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622616+0000", 2023-10-01T09:26:00.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6458000000000001e-05 2023-10-01T09:26:00.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622626+0000", 2023-10-01T09:26:00.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0059e-05 2023-10-01T09:26:00.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622691+0000", 2023-10-01T09:26:00.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4848e-05 2023-10-01T09:26:00.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622705+0000", 2023-10-01T09:26:00.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4041e-05 2023-10-01T09:26:00.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.550082+0000", 2023-10-01T09:26:00.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92737758299999995 2023-10-01T09:26:00.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.550091+0000", 2023-10-01T09:26:00.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0710000000000001e-06 2023-10-01T09:26:00.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.744482+0000", 2023-10-01T09:26:00.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1943907620000001 2023-10-01T09:26:00.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.744576+0000", 2023-10-01T09:26:00.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4320999999999994e-05 2023-10-01T09:26:00.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:26:00.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.744661+0000", 2023-10-01T09:26:00.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4855e-05 2023-10-01T09:26:00.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751931+0000", 2023-10-01T09:26:00.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.007269574 2023-10-01T09:26:00.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.759742+0000", 2023-10-01T09:26:00.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0078116840000000002 2023-10-01T09:26:00.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764514+0000", 2023-10-01T09:26:00.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047718409999999998 2023-10-01T09:26:00.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764594+0000", 2023-10-01T09:26:00.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9671000000000003e-05 2023-10-01T09:26:00.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764645+0000", 2023-10-01T09:26:00.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0713e-05 2023-10-01T09:26:00.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764748+0000", 2023-10-01T09:26:00.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103174 2023-10-01T09:26:00.654 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.654 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.654 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.655 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227561 5.8 5:1c2dcf03:::benchmark_data_smithi138_240338_object112424:head [delete] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-01T09:26:00.655 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:05.744421+0000", 2023-10-01T09:26:00.655 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 414.76024301799998, 2023-10-01T09:26:00.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.020494499, 2023-10-01T09:26:00.655 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.655 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.655 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.655 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.655 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.655 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227561 2023-10-01T09:26:00.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.656 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744421+0000", 2023-10-01T09:26:00.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744421+0000", 2023-10-01T09:26:00.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744419+0000", 2023-10-01T09:26:00.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:26:00.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744427+0000", 2023-10-01T09:26:00.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6160000000000002e-06 2023-10-01T09:26:00.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744445+0000", 2023-10-01T09:26:00.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7246999999999999e-05 2023-10-01T09:26:00.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744453+0000", 2023-10-01T09:26:00.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6740000000000003e-06 2023-10-01T09:26:00.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744541+0000", 2023-10-01T09:26:00.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7819999999999996e-05 2023-10-01T09:26:00.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744554+0000", 2023-10-01T09:26:00.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.237e-05 2023-10-01T09:26:00.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.748068+0000", 2023-10-01T09:26:00.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0035148250000001 2023-10-01T09:26:00.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.748233+0000", 2023-10-01T09:26:00.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016479900000000001 2023-10-01T09:26:00.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:26:00.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.748378+0000", 2023-10-01T09:26:00.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014517999999999999 2023-10-01T09:26:00.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.752176+0000", 2023-10-01T09:26:00.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037973389999999998 2023-10-01T09:26:00.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.759790+0000", 2023-10-01T09:26:00.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0076139179999999999 2023-10-01T09:26:00.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764554+0000", 2023-10-01T09:26:00.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047645869999999998 2023-10-01T09:26:00.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764783+0000", 2023-10-01T09:26:00.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00022926399999999999 2023-10-01T09:26:00.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764818+0000", 2023-10-01T09:26:00.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4400999999999998e-05 2023-10-01T09:26:00.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764915+0000", 2023-10-01T09:26:00.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7405000000000007e-05 2023-10-01T09:26:00.663 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.663 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.663 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.664 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227562 5.b 5:d7e9814e:::benchmark_data_smithi138_240338_object112425:head [delete] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-01T09:26:00.664 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:05.744494+0000", 2023-10-01T09:26:00.664 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 414.76016968699997, 2023-10-01T09:26:00.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.064026162, 2023-10-01T09:26:00.664 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.664 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:00.664 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.664 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:26:00.664 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:26:00.664 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227562 2023-10-01T09:26:00.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.664 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744494+0000", 2023-10-01T09:26:00.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744494+0000", 2023-10-01T09:26:00.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744494+0000", 2023-10-01T09:26:00.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:00.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744497+0000", 2023-10-01T09:26:00.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3390000000000001e-06 2023-10-01T09:26:00.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744507+0000", 2023-10-01T09:26:00.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0276e-05 2023-10-01T09:26:00.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744511+0000", 2023-10-01T09:26:00.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9160000000000003e-06 2023-10-01T09:26:00.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744548+0000", 2023-10-01T09:26:00.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7197999999999999e-05 2023-10-01T09:26:00.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744558+0000", 2023-10-01T09:26:00.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.55e-06 2023-10-01T09:26:00.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.767699+0000", 2023-10-01T09:26:00.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0231407809999999 2023-10-01T09:26:00.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:00.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.767807+0000", 2023-10-01T09:26:00.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010877199999999999 2023-10-01T09:26:00.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5,7", 2023-10-01T09:26:00.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.767928+0000", 2023-10-01T09:26:00.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120122 2023-10-01T09:26:00.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.777766+0000", 2023-10-01T09:26:00.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0098384030000000008 2023-10-01T09:26:00.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:00.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.782314+0000", 2023-10-01T09:26:00.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045480240000000003 2023-10-01T09:26:00.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.789999+0000", 2023-10-01T09:26:00.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0076853340000000003 2023-10-01T09:26:00.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:00.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808355+0000", 2023-10-01T09:26:00.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018356006000000001 2023-10-01T09:26:00.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:00.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808402+0000", 2023-10-01T09:26:00.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6980000000000001e-05 2023-10-01T09:26:00.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808520+0000", 2023-10-01T09:26:00.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118026 2023-10-01T09:26:00.673 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.673 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.673 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.673 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109327 6.19 6:9b6cdeca:::benchmark_data_smithi138_307751_object13958:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-01T09:26:00.673 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.601576+0000", 2023-10-01T09:26:00.674 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 260.90308784000001, 2023-10-01T09:26:00.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0306919560000001, 2023-10-01T09:26:00.674 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:00.674 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:26:00.674 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:00.674 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:00.674 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:00.674 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109327 2023-10-01T09:26:00.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.674 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:00.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:00.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601576+0000", 2023-10-01T09:26:00.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:00.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601576+0000", 2023-10-01T09:26:00.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:00.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:00.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601574+0000", 2023-10-01T09:26:00.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:26:00.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:00.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601584+0000", 2023-10-01T09:26:00.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0159e-05 2023-10-01T09:26:00.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:00.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601610+0000", 2023-10-01T09:26:00.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5582999999999999e-05 2023-10-01T09:26:00.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:00.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601621+0000", 2023-10-01T09:26:00.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1435e-05 2023-10-01T09:26:00.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.767017+0000", 2023-10-01T09:26:00.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16539611800000001 2023-10-01T09:26:00.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:00.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.767027+0000", 2023-10-01T09:26:00.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6609999999999995e-06 2023-10-01T09:26:00.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.616116+0000", 2023-10-01T09:26:00.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84908936300000004 2023-10-01T09:26:00.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:26:00.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.616242+0000", 2023-10-01T09:26:00.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012627800000000001 2023-10-01T09:26:00.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:00.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.632186+0000", 2023-10-01T09:26:00.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015943382999999998 2023-10-01T09:26:00.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:00.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:00.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:00.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.632268+0000", 2023-10-01T09:26:00.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2056000000000002e-05 2023-10-01T09:26:00.679 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.680 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.680 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.680 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:00.680 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:00.680 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:26:00.977 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:26:00.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:00.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:26:01.087 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:01.128 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:01.328 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:01.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:01.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:01.434 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:01.461 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:01.504 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:01.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:01.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:01.610 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:01.617 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 4, 7, 2] out_osds: [5, 6] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 7, 5] 2023-10-01T09:26:01.617 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:26:01.617 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 3, 4, 7, 2] 2023-10-01T09:26:01.617 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd out 4 2023-10-01T09:26:01.677 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:01.885 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:26:02.585 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:26:02.599 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:02.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:02.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:02.723 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:26:02.747 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:02 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:02.747 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:02 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:02.747 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:02 smithi190 ceph-mon[160435]: pgmap v742: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 233 op/s; 41376/252678 objects degraded (16.375%); 10308/252678 objects misplaced (4.080%); 3.3 MiB/s, 54 objects/s recovering 2023-10-01T09:26:02.747 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:02 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 36111/253407 objects degraded (14.250%), 11 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:02.751 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:02.795 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:26:02.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:02.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:26:02.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:02 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:02.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:02 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:02.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:02 smithi138 ceph-mon[185886]: pgmap v742: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 233 op/s; 41376/252678 objects degraded (16.375%); 10308/252678 objects misplaced (4.080%); 3.3 MiB/s, 54 objects/s recovering 2023-10-01T09:26:02.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:02 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 36111/253407 objects degraded (14.250%), 11 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:02.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:02 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:02.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:02 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:02.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:02 smithi138 ceph-mon[192513]: pgmap v742: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 233 op/s; 41376/252678 objects degraded (16.375%); 10308/252678 objects misplaced (4.080%); 3.3 MiB/s, 54 objects/s recovering 2023-10-01T09:26:02.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:02 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 36111/253407 objects degraded (14.250%), 11 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:02.903 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:26:03.048 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:03.364 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:26:03.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:03.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:26:03.478 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:03.493 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:03.561 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:03.589 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:03.780 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:03.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:03.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:03.860 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:03.913 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:04.012 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:04.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:03 smithi138 ceph-mon[185886]: pgmap v743: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 301 op/s; 41150/249465 objects degraded (16.495%); 10228/249465 objects misplaced (4.100%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T09:26:04.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:03 smithi138 ceph-mon[192513]: pgmap v743: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 301 op/s; 41150/249465 objects degraded (16.495%); 10228/249465 objects misplaced (4.100%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T09:26:04.246 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:04.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:04.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:03 smithi190 ceph-mon[160435]: pgmap v743: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 301 op/s; 41150/249465 objects degraded (16.495%); 10228/249465 objects misplaced (4.100%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T09:26:04.361 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:04.450 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:04.958 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:26:05.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3716230247' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:26:05.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3716230247' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:26:05.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3716230247' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:26:05.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:26:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:26:04] "GET /metrics HTTP/1.1" 200 36415 "" "Prometheus/2.33.4" 2023-10-01T09:26:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:05 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:26:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:05 smithi190 ceph-mon[160435]: failed to encode map e282 with expected crc 2023-10-01T09:26:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3716230247' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:26:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:05 smithi190 ceph-mon[160435]: osdmap e282: 8 total, 7 up, 6 in 2023-10-01T09:26:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:05 smithi190 ceph-mon[160435]: pgmap v745: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 321 op/s; 40960/247839 objects degraded (16.527%); 10102/247839 objects misplaced (4.076%); 3.9 MiB/s, 63 objects/s recovering 2023-10-01T09:26:06.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:05 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:26:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:05 smithi138 ceph-mon[185886]: failed to encode map e282 with expected crc 2023-10-01T09:26:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3716230247' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:26:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:05 smithi138 ceph-mon[185886]: osdmap e282: 8 total, 7 up, 6 in 2023-10-01T09:26:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:05 smithi138 ceph-mon[185886]: pgmap v745: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 321 op/s; 40960/247839 objects degraded (16.527%); 10102/247839 objects misplaced (4.076%); 3.9 MiB/s, 63 objects/s recovering 2023-10-01T09:26:06.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:05 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:26:06.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:05 smithi138 ceph-mon[192513]: failed to encode map e282 with expected crc 2023-10-01T09:26:06.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3716230247' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:26:06.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:05 smithi138 ceph-mon[192513]: osdmap e282: 8 total, 7 up, 6 in 2023-10-01T09:26:06.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:05 smithi138 ceph-mon[192513]: pgmap v745: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 321 op/s; 40960/247839 objects degraded (16.527%); 10102/247839 objects misplaced (4.076%); 3.9 MiB/s, 63 objects/s recovering 2023-10-01T09:26:07.221 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:07.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:07.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:07.332 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:07.358 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:07.401 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:07.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:07.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:07.509 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:07.536 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:07.582 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:07.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:07.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:07.688 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:26:07.717 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:08.420 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:26:08.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:08.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:26:08.540 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:26:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:08 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 40960/247839 objects degraded (16.527%), 12 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:08 smithi190 ceph-mon[160435]: pgmap v746: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 376 op/s; 40758/243906 objects degraded (16.711%); 9985/243906 objects misplaced (4.094%); 3.4 MiB/s, 55 objects/s recovering 2023-10-01T09:26:08.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:08 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 40960/247839 objects degraded (16.527%), 12 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:08.567 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:08 smithi138 ceph-mon[192513]: pgmap v746: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 376 op/s; 40758/243906 objects degraded (16.711%); 9985/243906 objects misplaced (4.094%); 3.4 MiB/s, 55 objects/s recovering 2023-10-01T09:26:08.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:08 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 40960/247839 objects degraded (16.527%), 12 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:08.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:08 smithi138 ceph-mon[185886]: pgmap v746: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 376 op/s; 40758/243906 objects degraded (16.711%); 9985/243906 objects misplaced (4.094%); 3.4 MiB/s, 55 objects/s recovering 2023-10-01T09:26:08.569 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:09.332 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:26:09.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:09.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:26:09.442 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:09.468 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:09.514 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:09.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:09.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:09.620 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:26:09.852 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:10.017 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:09 smithi138 ceph-mon[185886]: pgmap v747: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 352 op/s; 40546/242805 objects degraded (16.699%); 9858/242805 objects misplaced (4.060%); 4.3 MiB/s, 69 objects/s recovering 2023-10-01T09:26:10.017 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:09 smithi138 ceph-mon[192513]: pgmap v747: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 352 op/s; 40546/242805 objects degraded (16.699%); 9858/242805 objects misplaced (4.060%); 4.3 MiB/s, 69 objects/s recovering 2023-10-01T09:26:10.031 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:26:10.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:10.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:26:10.148 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:10.171 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:09 smithi190 ceph-mon[160435]: pgmap v747: 89 pgs: 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 352 op/s; 40546/242805 objects degraded (16.699%); 9858/242805 objects misplaced (4.060%); 4.3 MiB/s, 69 objects/s recovering 2023-10-01T09:26:10.174 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:10.218 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:10.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:10.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:10.325 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:10.350 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:10.393 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:10.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:10.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:10.420 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:26:10.420 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:26:10.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.421 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:115681 7.b 7:d5cab0ff:::benchmark_data_smithi138_367058_object17886:head [delete] snapc 0=[] ondisk+write+known_if_redirected e282)", 2023-10-01T09:26:10.421 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:10.375789+0000", 2023-10-01T09:26:10.421 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.041875778000000002, 2023-10-01T09:26:10.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.041909492, 2023-10-01T09:26:10.421 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:10.421 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "delayed", 2023-10-01T09:26:10.421 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:10.421 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:26:10.421 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:26:10.422 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 115681 2023-10-01T09:26:10.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.422 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:10.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:10.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.375789+0000", 2023-10-01T09:26:10.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:10.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.375789+0000", 2023-10-01T09:26:10.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:10.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.375787+0000", 2023-10-01T09:26:10.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:10.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:10.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.375792+0000", 2023-10-01T09:26:10.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8420000000000004e-06 2023-10-01T09:26:10.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:10.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.375802+0000", 2023-10-01T09:26:10.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0189e-05 2023-10-01T09:26:10.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:10.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.375810+0000", 2023-10-01T09:26:10.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2779999999999996e-06 2023-10-01T09:26:10.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:10.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.381410+0000", 2023-10-01T09:26:10.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056006299999999997 2023-10-01T09:26:10.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:26:10.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.382330+0000", 2023-10-01T09:26:10.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000920081 2023-10-01T09:26:10.426 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.426 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:10.426 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.427 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:115683 7.0 7:055863e1:::benchmark_data_smithi138_367058_object17888:head [delete] snapc 0=[] ondisk+write+known_if_redirected e282)", 2023-10-01T09:26:10.427 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:10.375819+0000", 2023-10-01T09:26:10.427 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.041845707000000003, 2023-10-01T09:26:10.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.041961090999999999, 2023-10-01T09:26:10.427 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:10.427 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:26:10.427 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:10.427 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:26:10.427 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:26:10.427 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 115683 2023-10-01T09:26:10.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.428 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:10.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.428 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:10.428 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.375819+0000", 2023-10-01T09:26:10.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.428 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:10.428 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.375819+0000", 2023-10-01T09:26:10.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:10.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.375818+0000", 2023-10-01T09:26:10.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:10.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:10.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.375821+0000", 2023-10-01T09:26:10.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1750000000000001e-06 2023-10-01T09:26:10.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:10.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.375827+0000", 2023-10-01T09:26:10.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8470000000000001e-06 2023-10-01T09:26:10.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:10.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.375832+0000", 2023-10-01T09:26:10.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0180000000000001e-06 2023-10-01T09:26:10.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.431 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:10.431 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.375883+0000", 2023-10-01T09:26:10.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0989000000000003e-05 2023-10-01T09:26:10.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.431 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:10.431 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.375999+0000", 2023-10-01T09:26:10.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011545299999999999 2023-10-01T09:26:10.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:26:10.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.376161+0000", 2023-10-01T09:26:10.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016264200000000001 2023-10-01T09:26:10.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:10.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.385669+0000", 2023-10-01T09:26:10.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0095074410000000002 2023-10-01T09:26:10.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:10.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.389345+0000", 2023-10-01T09:26:10.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036761989999999998 2023-10-01T09:26:10.433 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.433 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:10.433 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.434 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:115692 7.d 7:bdaaa96c:::benchmark_data_smithi138_367058_object17897:head [delete] snapc 0=[] ondisk+write+known_if_redirected e282)", 2023-10-01T09:26:10.434 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:10.395377+0000", 2023-10-01T09:26:10.434 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.022287789999999998, 2023-10-01T09:26:10.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.022453163000000002, 2023-10-01T09:26:10.434 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:10.434 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:26:10.434 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:10.434 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:26:10.434 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:26:10.435 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 115692 2023-10-01T09:26:10.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.435 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:10.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:10.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.395377+0000", 2023-10-01T09:26:10.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:10.436 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.395377+0000", 2023-10-01T09:26:10.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.436 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:10.436 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.395374+0000", 2023-10-01T09:26:10.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:26:10.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:10.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.395385+0000", 2023-10-01T09:26:10.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0413e-05 2023-10-01T09:26:10.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:10.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.395399+0000", 2023-10-01T09:26:10.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3681e-05 2023-10-01T09:26:10.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.438 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:10.438 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.395409+0000", 2023-10-01T09:26:10.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0837e-05 2023-10-01T09:26:10.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.438 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:10.438 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.395719+0000", 2023-10-01T09:26:10.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00030954000000000002 2023-10-01T09:26:10.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:10.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.395949+0000", 2023-10-01T09:26:10.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000230106 2023-10-01T09:26:10.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,3", 2023-10-01T09:26:10.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.396170+0000", 2023-10-01T09:26:10.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00022118999999999999 2023-10-01T09:26:10.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:10.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.410390+0000", 2023-10-01T09:26:10.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014219379000000001 2023-10-01T09:26:10.440 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.440 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:10.440 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.440 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.35421.0:115683 7.0 e282/273)", 2023-10-01T09:26:10.441 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:10.406235+0000", 2023-10-01T09:26:10.441 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.011429112999999999, 2023-10-01T09:26:10.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011642356, 2023-10-01T09:26:10.441 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:10.441 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:26:10.441 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:10.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:10.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.406235+0000", 2023-10-01T09:26:10.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:10.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.406235+0000", 2023-10-01T09:26:10.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:10.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.406234+0000", 2023-10-01T09:26:10.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:26:10.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:10.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.406242+0000", 2023-10-01T09:26:10.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6389999999999992e-06 2023-10-01T09:26:10.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:10.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.406256+0000", 2023-10-01T09:26:10.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4071e-05 2023-10-01T09:26:10.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:10.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.406264+0000", 2023-10-01T09:26:10.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1980000000000001e-06 2023-10-01T09:26:10.444 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.444 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:10.444 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.445 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35421.0:115691 7.1 e282/280 7:869dbdcd:::benchmark_data_smithi138_367058_object17896:head v 282'7120, mlcod=282'7118)", 2023-10-01T09:26:10.445 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:10.407337+0000", 2023-10-01T09:26:10.445 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.010327203, 2023-10-01T09:26:10.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010588658000000001, 2023-10-01T09:26:10.445 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:10.445 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:26:10.445 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:10.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:10.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.407337+0000", 2023-10-01T09:26:10.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:10.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.407337+0000", 2023-10-01T09:26:10.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:10.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.407336+0000", 2023-10-01T09:26:10.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:26:10.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:10.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.407346+0000", 2023-10-01T09:26:10.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0052e-05 2023-10-01T09:26:10.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:10.448 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.407358+0000", 2023-10-01T09:26:10.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2086000000000001e-05 2023-10-01T09:26:10.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.448 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:10.448 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.407367+0000", 2023-10-01T09:26:10.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0609999999999996e-06 2023-10-01T09:26:10.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.448 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:10.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.416428+0000", 2023-10-01T09:26:10.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0090617619999999992 2023-10-01T09:26:10.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:10.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.416453+0000", 2023-10-01T09:26:10.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4754999999999999e-05 2023-10-01T09:26:10.449 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.449 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:10.449 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.450 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.35421.0:115689 7.7 e282/280 7:e73314cd:::benchmark_data_smithi138_367058_object17894:head v 282'7232, mlcod=262'814)", 2023-10-01T09:26:10.450 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:10.407388+0000", 2023-10-01T09:26:10.450 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.010276375000000001, 2023-10-01T09:26:10.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010592476999999999, 2023-10-01T09:26:10.450 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:10.450 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:26:10.450 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:10.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:10.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.407388+0000", 2023-10-01T09:26:10.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:10.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.407388+0000", 2023-10-01T09:26:10.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:10.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.407388+0000", 2023-10-01T09:26:10.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:10.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:10.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.407393+0000", 2023-10-01T09:26:10.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1599999999999997e-06 2023-10-01T09:26:10.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:10.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.407401+0000", 2023-10-01T09:26:10.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.824e-06 2023-10-01T09:26:10.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:10.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.407406+0000", 2023-10-01T09:26:10.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3469999999999998e-06 2023-10-01T09:26:10.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:10.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.412438+0000", 2023-10-01T09:26:10.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050318730000000001 2023-10-01T09:26:10.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:10.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.412466+0000", 2023-10-01T09:26:10.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8354999999999999e-05 2023-10-01T09:26:10.455 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.455 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:10.455 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.455 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPushReply(7.b 282/280 [PushReplyOp(7:d5cab0ff:::benchmark_data_smithi138_367058_object17886:head)])", 2023-10-01T09:26:10.455 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:10.411096+0000", 2023-10-01T09:26:10.455 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0065687870000000004, 2023-10-01T09:26:10.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069489759999999999, 2023-10-01T09:26:10.456 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:10.456 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:26:10.456 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:10.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:10.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.411096+0000", 2023-10-01T09:26:10.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:10.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.411096+0000", 2023-10-01T09:26:10.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:10.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.411095+0000", 2023-10-01T09:26:10.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:26:10.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:10.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.411103+0000", 2023-10-01T09:26:10.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5599999999999994e-06 2023-10-01T09:26:10.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:10.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.411120+0000", 2023-10-01T09:26:10.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6381000000000001e-05 2023-10-01T09:26:10.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:10.459 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.411128+0000", 2023-10-01T09:26:10.459 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8920000000000006e-06 2023-10-01T09:26:10.459 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.459 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:10.459 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.411224+0000", 2023-10-01T09:26:10.459 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5643999999999994e-05 2023-10-01T09:26:10.459 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.459 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:10.460 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.460 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.460 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.460 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(7.a 282/273 [PushOp(7:517d1aa4:::benchmark_data_smithi138_367058_object17901:head, version: 262'1117, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:517d1aa4:::benchmark_data_smithi138_367058_object17901:head@262'1117, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-01T09:26:10.460 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:10.415537+0000", 2023-10-01T09:26:10.460 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0021272359999999998, 2023-10-01T09:26:10.460 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025706079999999998, 2023-10-01T09:26:10.460 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:10.460 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:26:10.461 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:10.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:10.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.415537+0000", 2023-10-01T09:26:10.461 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.461 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:10.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.415537+0000", 2023-10-01T09:26:10.461 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:10.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.462 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:10.462 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.415535+0000", 2023-10-01T09:26:10.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:26:10.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.462 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:10.462 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.415590+0000", 2023-10-01T09:26:10.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4163000000000003e-05 2023-10-01T09:26:10.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:10.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.415616+0000", 2023-10-01T09:26:10.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6143e-05 2023-10-01T09:26:10.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:10.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.415634+0000", 2023-10-01T09:26:10.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8556999999999999e-05 2023-10-01T09:26:10.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:10.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.415743+0000", 2023-10-01T09:26:10.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109101 2023-10-01T09:26:10.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:10.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:10.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:10.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:10.415757+0000", 2023-10-01T09:26:10.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3722e-05 2023-10-01T09:26:10.465 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.465 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:10.465 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.465 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:10.465 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:26:10.465 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 8 2023-10-01T09:26:10.465 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:26:10.501 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:10.526 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:10.573 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:10.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:10.584 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:26:10.635 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:26:10.644 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:10 smithi138 conmon[185862]: 2023-10-01T09:26:10.348+0000 7f70d64ec700 -1 mon.a@0(leader).osd e282 definitely_dead 0 2023-10-01T09:26:10.678 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:26:10.685 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:26:10.739 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:26:10.769 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:26:10.777 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:10.808 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:26:10.864 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:26:10.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:26:10.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:26:10.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:26:10.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:26:10.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:26:10.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:26:10.865 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-10-01T09:26:10.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:26:11.051 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:26:11.414 INFO:teuthology.orchestra.run.smithi138.stderr:marked out osd.4. 2023-10-01T09:26:11.438 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:11.438 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:26:11.438 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2199610881' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-01T09:26:11.438 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:11 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-01T09:26:11.438 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:11 smithi138 ceph-mon[185886]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-01T09:26:11.439 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2505701622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:26:11.439 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:11.439 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:26:11.439 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2199610881' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-01T09:26:11.439 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:11 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-01T09:26:11.439 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:11 smithi138 ceph-mon[192513]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-01T09:26:11.439 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2505701622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:26:11.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:26:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2199610881' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-01T09:26:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:11 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-10-01T09:26:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:11 smithi190 ceph-mon[160435]: Client client.admin marked osd.4 out, while it was still marked up 2023-10-01T09:26:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2505701622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:26:12.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:12 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-01T09:26:12.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:12 smithi138 ceph-mon[192513]: osdmap e283: 8 total, 7 up, 5 in 2023-10-01T09:26:12.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:12 smithi138 ceph-mon[192513]: failed to encode map e283 with expected crc 2023-10-01T09:26:12.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:12 smithi138 ceph-mon[192513]: pgmap v749: 89 pgs: 10 peering, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 67 active+clean; 4.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 318 op/s; 40546/241800 objects degraded (16.768%); 9858/241800 objects misplaced (4.077%); 3.9 MiB/s, 63 objects/s recovering 2023-10-01T09:26:12.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:12 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 40546/242805 objects degraded (16.699%), 12 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:12.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:12 smithi138 ceph-mon[192513]: osdmap e284: 8 total, 7 up, 5 in 2023-10-01T09:26:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:12 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-01T09:26:12.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:12 smithi138 ceph-mon[185886]: osdmap e283: 8 total, 7 up, 5 in 2023-10-01T09:26:12.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:12 smithi138 ceph-mon[185886]: failed to encode map e283 with expected crc 2023-10-01T09:26:12.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:12 smithi138 ceph-mon[185886]: pgmap v749: 89 pgs: 10 peering, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 67 active+clean; 4.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 318 op/s; 40546/241800 objects degraded (16.768%); 9858/241800 objects misplaced (4.077%); 3.9 MiB/s, 63 objects/s recovering 2023-10-01T09:26:12.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:12 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 40546/242805 objects degraded (16.699%), 12 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:12.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:12 smithi138 ceph-mon[185886]: osdmap e284: 8 total, 7 up, 5 in 2023-10-01T09:26:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:12 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-10-01T09:26:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:12 smithi190 ceph-mon[160435]: osdmap e283: 8 total, 7 up, 5 in 2023-10-01T09:26:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:12 smithi190 ceph-mon[160435]: failed to encode map e283 with expected crc 2023-10-01T09:26:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:12 smithi190 ceph-mon[160435]: pgmap v749: 89 pgs: 10 peering, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 67 active+clean; 4.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 318 op/s; 40546/241800 objects degraded (16.768%); 9858/241800 objects misplaced (4.077%); 3.9 MiB/s, 63 objects/s recovering 2023-10-01T09:26:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:12 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 40546/242805 objects degraded (16.699%), 12 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:12 smithi190 ceph-mon[160435]: osdmap e284: 8 total, 7 up, 5 in 2023-10-01T09:26:13.487 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:26:13.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:13.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:26:13.587 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:26:13.600 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:26:13.609 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:26:13.610 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:26:13.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:13 smithi138 ceph-mon[185886]: failed to encode map e284 with expected crc 2023-10-01T09:26:13.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:13 smithi138 ceph-mon[185886]: failed to encode map e284 with expected crc 2023-10-01T09:26:13.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:13 smithi138 ceph-mon[185886]: failed to encode map e284 with expected crc 2023-10-01T09:26:13.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:13 smithi138 ceph-mon[185886]: 7.1 starting backfill to osd.0 from (0'0,0'0] MAX to 282'7155 2023-10-01T09:26:13.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:13 smithi138 ceph-mon[185886]: failed to encode map e284 with expected crc 2023-10-01T09:26:13.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:13 smithi138 ceph-mon[185886]: 7.1 continuing backfill to osd.1 from (263'1744,282'7155] MIN to 282'7155 2023-10-01T09:26:13.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:13 smithi138 ceph-mon[185886]: osdmap e285: 8 total, 7 up, 5 in 2023-10-01T09:26:13.678 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:13 smithi138 ceph-mon[192513]: failed to encode map e284 with expected crc 2023-10-01T09:26:13.678 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:13 smithi138 ceph-mon[192513]: failed to encode map e284 with expected crc 2023-10-01T09:26:13.678 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:13 smithi138 ceph-mon[192513]: failed to encode map e284 with expected crc 2023-10-01T09:26:13.678 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:13 smithi138 ceph-mon[192513]: 7.1 starting backfill to osd.0 from (0'0,0'0] MAX to 282'7155 2023-10-01T09:26:13.679 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:13 smithi138 ceph-mon[192513]: failed to encode map e284 with expected crc 2023-10-01T09:26:13.679 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:13 smithi138 ceph-mon[192513]: 7.1 continuing backfill to osd.1 from (263'1744,282'7155] MIN to 282'7155 2023-10-01T09:26:13.679 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:13 smithi138 ceph-mon[192513]: osdmap e285: 8 total, 7 up, 5 in 2023-10-01T09:26:13.738 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:13.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:13 smithi190 ceph-mon[160435]: failed to encode map e284 with expected crc 2023-10-01T09:26:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:13 smithi190 ceph-mon[160435]: failed to encode map e284 with expected crc 2023-10-01T09:26:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:13 smithi190 ceph-mon[160435]: failed to encode map e284 with expected crc 2023-10-01T09:26:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:13 smithi190 ceph-mon[160435]: 7.1 starting backfill to osd.0 from (0'0,0'0] MAX to 282'7155 2023-10-01T09:26:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:13 smithi190 ceph-mon[160435]: failed to encode map e284 with expected crc 2023-10-01T09:26:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:13 smithi190 ceph-mon[160435]: 7.1 continuing backfill to osd.1 from (263'1744,282'7155] MIN to 282'7155 2023-10-01T09:26:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:13 smithi190 ceph-mon[160435]: osdmap e285: 8 total, 7 up, 5 in 2023-10-01T09:26:14.447 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:26:14.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:14.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:26:14.554 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:26:14.581 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:14.625 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:26:14.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:14.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:26:14.732 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:14.794 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[192513]: 7.b starting backfill to osd.0 from (0'0,0'0] MAX to 282'7076 2023-10-01T09:26:14.794 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[192513]: failed to encode map e285 with expected crc 2023-10-01T09:26:14.794 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[192513]: failed to encode map e285 with expected crc 2023-10-01T09:26:14.794 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[192513]: 7.9 starting backfill to osd.7 from (0'0,0'0] MAX to 282'7455 2023-10-01T09:26:14.794 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[192513]: 7.4 starting backfill to osd.3 from (0'0,0'0] MAX to 282'7356 2023-10-01T09:26:14.794 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[192513]: failed to encode map e285 with expected crc 2023-10-01T09:26:14.795 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[192513]: 7.3 starting backfill to osd.7 from (0'0,0'0] MAX to 282'7173 2023-10-01T09:26:14.795 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[192513]: failed to encode map e285 with expected crc 2023-10-01T09:26:14.795 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[192513]: pgmap v752: 89 pgs: 3 remapped+peering, 10 peering, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 64 active+clean; 4.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 262 op/s; 44433/239175 objects degraded (18.578%); 9777/239175 objects misplaced (4.088%); 3.7 MiB/s, 60 objects/s recovering 2023-10-01T09:26:14.795 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[192513]: osdmap e286: 8 total, 7 up, 5 in 2023-10-01T09:26:14.795 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[185886]: 7.b starting backfill to osd.0 from (0'0,0'0] MAX to 282'7076 2023-10-01T09:26:14.795 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[185886]: failed to encode map e285 with expected crc 2023-10-01T09:26:14.795 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[185886]: failed to encode map e285 with expected crc 2023-10-01T09:26:14.795 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[185886]: 7.9 starting backfill to osd.7 from (0'0,0'0] MAX to 282'7455 2023-10-01T09:26:14.795 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[185886]: 7.4 starting backfill to osd.3 from (0'0,0'0] MAX to 282'7356 2023-10-01T09:26:14.796 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[185886]: failed to encode map e285 with expected crc 2023-10-01T09:26:14.796 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[185886]: 7.3 starting backfill to osd.7 from (0'0,0'0] MAX to 282'7173 2023-10-01T09:26:14.796 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[185886]: failed to encode map e285 with expected crc 2023-10-01T09:26:14.796 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[185886]: pgmap v752: 89 pgs: 3 remapped+peering, 10 peering, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 64 active+clean; 4.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 262 op/s; 44433/239175 objects degraded (18.578%); 9777/239175 objects misplaced (4.088%); 3.7 MiB/s, 60 objects/s recovering 2023-10-01T09:26:14.796 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:14 smithi138 ceph-mon[185886]: osdmap e286: 8 total, 7 up, 5 in 2023-10-01T09:26:14.797 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:14.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:14 smithi190 ceph-mon[160435]: 7.b starting backfill to osd.0 from (0'0,0'0] MAX to 282'7076 2023-10-01T09:26:14.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:14 smithi190 ceph-mon[160435]: failed to encode map e285 with expected crc 2023-10-01T09:26:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:14 smithi190 ceph-mon[160435]: failed to encode map e285 with expected crc 2023-10-01T09:26:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:14 smithi190 ceph-mon[160435]: 7.9 starting backfill to osd.7 from (0'0,0'0] MAX to 282'7455 2023-10-01T09:26:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:14 smithi190 ceph-mon[160435]: 7.4 starting backfill to osd.3 from (0'0,0'0] MAX to 282'7356 2023-10-01T09:26:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:14 smithi190 ceph-mon[160435]: failed to encode map e285 with expected crc 2023-10-01T09:26:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:14 smithi190 ceph-mon[160435]: 7.3 starting backfill to osd.7 from (0'0,0'0] MAX to 282'7173 2023-10-01T09:26:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:14 smithi190 ceph-mon[160435]: failed to encode map e285 with expected crc 2023-10-01T09:26:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:14 smithi190 ceph-mon[160435]: pgmap v752: 89 pgs: 3 remapped+peering, 10 peering, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 64 active+clean; 4.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 262 op/s; 44433/239175 objects degraded (18.578%); 9777/239175 objects misplaced (4.088%); 3.7 MiB/s, 60 objects/s recovering 2023-10-01T09:26:14.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:14 smithi190 ceph-mon[160435]: osdmap e286: 8 total, 7 up, 5 in 2023-10-01T09:26:15.076 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:26:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:26:14] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.33.4" 2023-10-01T09:26:15.148 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:15.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:15.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:15.257 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:15.370 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:15.431 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:15 smithi138 ceph-mon[185886]: failed to encode map e286 with expected crc 2023-10-01T09:26:15.565 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:15.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:15.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:15.611 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:26:15.643 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:15.685 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:15.711 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:15.722 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:15 smithi138 ceph-mon[192513]: failed to encode map e286 with expected crc 2023-10-01T09:26:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:15 smithi190 ceph-mon[160435]: failed to encode map e286 with expected crc 2023-10-01T09:26:15.828 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:16.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:16 smithi138 ceph-mon[185886]: pgmap v754: 89 pgs: 3 remapped+peering, 10 peering, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 63 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 298 op/s; 54376/238161 objects degraded (22.832%); 4933/238161 objects misplaced (2.071%); 4.2 MiB/s, 69 objects/s recovering 2023-10-01T09:26:16.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:16 smithi138 ceph-mon[192513]: pgmap v754: 89 pgs: 3 remapped+peering, 10 peering, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 63 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 298 op/s; 54376/238161 objects degraded (22.832%); 4933/238161 objects misplaced (2.071%); 4.2 MiB/s, 69 objects/s recovering 2023-10-01T09:26:16.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:16 smithi190 ceph-mon[160435]: pgmap v754: 89 pgs: 3 remapped+peering, 10 peering, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 63 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 298 op/s; 54376/238161 objects degraded (22.832%); 4933/238161 objects misplaced (2.071%); 4.2 MiB/s, 69 objects/s recovering 2023-10-01T09:26:16.869 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:16.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:16.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:16.989 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:17.110 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:17.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:17 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 54376/238161 objects degraded (22.832%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:17.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:17 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 54376/238161 objects degraded (22.832%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:17.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:17 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 54376/238161 objects degraded (22.832%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:18.409 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:18.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:18.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:18.523 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:18.550 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:18.594 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:18.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:18.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:18.675 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:18 smithi138 ceph-mon[192513]: pgmap v755: 89 pgs: 2 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 285 op/s; 54216/236649 objects degraded (22.910%); 9740/236649 objects misplaced (4.116%); 3.2 MiB/s, 1 keys/s, 53 objects/s recovering 2023-10-01T09:26:18.678 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:18 smithi138 ceph-mon[185886]: pgmap v755: 89 pgs: 2 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 285 op/s; 54216/236649 objects degraded (22.910%); 9740/236649 objects misplaced (4.116%); 3.2 MiB/s, 1 keys/s, 53 objects/s recovering 2023-10-01T09:26:18.699 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:26:18.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:18 smithi190 ceph-mon[160435]: pgmap v755: 89 pgs: 2 remapped+peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 285 op/s; 54216/236649 objects degraded (22.910%); 9740/236649 objects misplaced (4.116%); 3.2 MiB/s, 1 keys/s, 53 objects/s recovering 2023-10-01T09:26:18.853 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:19.032 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:19.396 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:26:19.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:19.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:26:19.504 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:19.530 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:19.573 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:19.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:19.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:19.680 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:19.785 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:19.793 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:26:19.794 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:26:20.023 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:19 smithi138 ceph-mon[192513]: pgmap v756: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 348 op/s; 53824/233910 objects degraded (23.011%); 19624/233910 objects misplaced (8.390%); 4.8 MiB/s, 1 keys/s, 79 objects/s recovering 2023-10-01T09:26:20.023 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:19 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:20.023 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:19 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:20.024 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:19 smithi138 ceph-mon[185886]: pgmap v756: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 348 op/s; 53824/233910 objects degraded (23.011%); 19624/233910 objects misplaced (8.390%); 4.8 MiB/s, 1 keys/s, 79 objects/s recovering 2023-10-01T09:26:20.024 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:19 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:20.024 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:19 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:20.275 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:20.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:20.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:20.306 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2] out_osds: [5, 6, 4] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 7, 5] 2023-10-01T09:26:20.306 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:26:20.306 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T09:26:20.307 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.218664 2023-10-01T09:26:20.307 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 0.21866380554980136 2023-10-01T09:26:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:19 smithi190 ceph-mon[160435]: pgmap v756: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 348 op/s; 53824/233910 objects degraded (23.011%); 19624/233910 objects misplaced (8.390%); 4.8 MiB/s, 1 keys/s, 79 objects/s recovering 2023-10-01T09:26:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:19 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:20.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:19 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:20.380 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:20.407 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:20.451 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:20.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:20.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:20.558 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:26:20.584 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:20.631 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:26:20.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:20.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:26:20.738 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:20.764 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:20.909 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:26:21.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:20 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2939549562' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:26:21.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:20 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:26:21.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:20 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2939549562' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:26:21.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:20 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:26:21.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:20 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2939549562' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:26:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:20 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:26:22.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:21 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:26:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:21 smithi190 ceph-mon[160435]: failed to encode map e287 with expected crc 2023-10-01T09:26:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:21 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:26:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:21 smithi190 ceph-mon[160435]: osdmap e287: 8 total, 7 up, 5 in 2023-10-01T09:26:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:21 smithi190 ceph-mon[160435]: failed to encode map e287 with expected crc 2023-10-01T09:26:22.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:21 smithi190 ceph-mon[160435]: pgmap v758: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 260 op/s; 53824/232902 objects degraded (23.110%); 19624/232902 objects misplaced (8.426%); 3.5 MiB/s, 1 keys/s, 56 objects/s recovering 2023-10-01T09:26:22.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:21 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:26:22.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:21 smithi138 ceph-mon[192513]: failed to encode map e287 with expected crc 2023-10-01T09:26:22.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:21 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:26:22.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:21 smithi138 ceph-mon[192513]: osdmap e287: 8 total, 7 up, 5 in 2023-10-01T09:26:22.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:21 smithi138 ceph-mon[192513]: failed to encode map e287 with expected crc 2023-10-01T09:26:22.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:21 smithi138 ceph-mon[192513]: pgmap v758: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 260 op/s; 53824/232902 objects degraded (23.110%); 19624/232902 objects misplaced (8.426%); 3.5 MiB/s, 1 keys/s, 56 objects/s recovering 2023-10-01T09:26:22.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:21 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:26:22.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:21 smithi138 ceph-mon[185886]: failed to encode map e287 with expected crc 2023-10-01T09:26:22.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:21 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:26:22.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:21 smithi138 ceph-mon[185886]: osdmap e287: 8 total, 7 up, 5 in 2023-10-01T09:26:22.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:21 smithi138 ceph-mon[185886]: failed to encode map e287 with expected crc 2023-10-01T09:26:22.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:21 smithi138 ceph-mon[185886]: pgmap v758: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 260 op/s; 53824/232902 objects degraded (23.110%); 19624/232902 objects misplaced (8.426%); 3.5 MiB/s, 1 keys/s, 56 objects/s recovering 2023-10-01T09:26:22.539 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:22.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:22.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:22.653 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:26:22.795 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:22.807 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:26:22.936 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:26:22.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:22.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:26:23.017 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:23.043 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:23.070 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:23.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:22 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 53824/232902 objects degraded (23.110%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:23.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:26:23.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:26:23.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:22 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 53824/232902 objects degraded (23.110%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:26:23.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:26:23.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:23.114 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:23.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:23.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:23.221 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:23.305 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:23.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:22 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 53824/232902 objects degraded (23.110%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:23.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:26:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:26:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:26:23.582 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:26:23.588 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:23.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:23.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:23.652 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:26:23.701 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:23.707 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:26:23.728 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:23.761 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:26:23.772 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:23.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:23.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:23.789 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:26:23.836 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:26:23.880 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:26:23.889 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:26:23.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:26:23.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:26:23.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:26:23.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:26:23.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:26:23.889 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:26:23.890 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-10-01T09:26:23.890 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:26:23.906 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:23.949 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:26:23.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:23.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:26:24.055 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:26:24.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:23 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3114271762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:26:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:23 smithi190 ceph-mon[160435]: pgmap v759: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 323 op/s; 53509/229368 objects degraded (23.329%); 19169/229368 objects misplaced (8.357%); 2.4 MiB/s, 1 keys/s, 39 objects/s recovering 2023-10-01T09:26:24.133 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:23 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3114271762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:26:24.133 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:23 smithi138 ceph-mon[185886]: pgmap v759: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 323 op/s; 53509/229368 objects degraded (23.329%); 19169/229368 objects misplaced (8.357%); 2.4 MiB/s, 1 keys/s, 39 objects/s recovering 2023-10-01T09:26:24.133 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:23 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3114271762' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:26:24.133 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:23 smithi138 ceph-mon[192513]: pgmap v759: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 323 op/s; 53509/229368 objects degraded (23.329%); 19169/229368 objects misplaced (8.357%); 2.4 MiB/s, 1 keys/s, 39 objects/s recovering 2023-10-01T09:26:24.167 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:24.671 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:26:24.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:24.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:26:24.709 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:26:24.782 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:24.821 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:24.866 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:24.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:24.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:24.970 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:24.998 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:25.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:26:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:26:24] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.33.4" 2023-10-01T09:26:25.676 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:25.682 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:26:25.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:25.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:26:25.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:25.785 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:25.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:26:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:26:25.899 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:26:25.904 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:25.930 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:26.071 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:26.075 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:26.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:26.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:26.180 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:26.314 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:26.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:26 smithi138 ceph-mon[192513]: pgmap v760: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 351 op/s; 53270/227613 objects degraded (23.404%); 19169/227613 objects misplaced (8.422%); 3.1 MiB/s, 0 keys/s, 49 objects/s recovering 2023-10-01T09:26:26.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:26 smithi138 ceph-mon[185886]: pgmap v760: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 351 op/s; 53270/227613 objects degraded (23.404%); 19169/227613 objects misplaced (8.422%); 3.1 MiB/s, 0 keys/s, 49 objects/s recovering 2023-10-01T09:26:26.624 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:26.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:26.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:26.734 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:26.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:26 smithi190 ceph-mon[160435]: pgmap v760: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 351 op/s; 53270/227613 objects degraded (23.404%); 19169/227613 objects misplaced (8.422%); 3.1 MiB/s, 0 keys/s, 49 objects/s recovering 2023-10-01T09:26:26.813 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:27.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:27 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 53270/227613 objects degraded (23.404%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:27.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:27 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 53270/227613 objects degraded (23.404%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:27.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:27 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 53270/227613 objects degraded (23.404%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:26:27.900 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:26:28.366 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.2 primary-affinity to 0.218664 (8143302) 2023-10-01T09:26:28.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3229275881' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.21866380554980136}]: dispatch 2023-10-01T09:26:28.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:28 smithi138 ceph-mon[192513]: pgmap v761: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 374 op/s; 53034/225417 objects degraded (23.527%); 18913/225417 objects misplaced (8.390%); 3.0 MiB/s, 48 objects/s recovering 2023-10-01T09:26:28.578 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3229275881' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.21866380554980136}]: dispatch 2023-10-01T09:26:28.579 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:28 smithi138 ceph-mon[185886]: pgmap v761: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 374 op/s; 53034/225417 objects degraded (23.527%); 18913/225417 objects misplaced (8.390%); 3.0 MiB/s, 48 objects/s recovering 2023-10-01T09:26:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3229275881' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.21866380554980136}]: dispatch 2023-10-01T09:26:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:28 smithi190 ceph-mon[160435]: pgmap v761: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 374 op/s; 53034/225417 objects degraded (23.527%); 18913/225417 objects misplaced (8.390%); 3.0 MiB/s, 48 objects/s recovering 2023-10-01T09:26:28.871 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:28.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:28.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:28.982 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:29.012 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:29.754 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:29 smithi138 ceph-mon[185886]: failed to encode map e288 with expected crc 2023-10-01T09:26:29.754 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:29 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3229275881' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.21866380554980136}]': finished 2023-10-01T09:26:29.754 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:29 smithi138 ceph-mon[185886]: osdmap e288: 8 total, 7 up, 5 in 2023-10-01T09:26:29.754 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:29 smithi138 ceph-mon[185886]: osdmap e289: 8 total, 7 up, 5 in 2023-10-01T09:26:29.754 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:29 smithi138 ceph-mon[192513]: failed to encode map e288 with expected crc 2023-10-01T09:26:29.754 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:29 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3229275881' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.21866380554980136}]': finished 2023-10-01T09:26:29.755 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:29 smithi138 ceph-mon[192513]: osdmap e288: 8 total, 7 up, 5 in 2023-10-01T09:26:29.755 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:29 smithi138 ceph-mon[192513]: osdmap e289: 8 total, 7 up, 5 in 2023-10-01T09:26:29.778 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:29.918 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:29 smithi190 ceph-mon[160435]: failed to encode map e288 with expected crc 2023-10-01T09:26:29.918 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:29 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3229275881' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.21866380554980136}]': finished 2023-10-01T09:26:29.918 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:29 smithi190 ceph-mon[160435]: osdmap e288: 8 total, 7 up, 5 in 2023-10-01T09:26:29.919 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:29 smithi190 ceph-mon[160435]: osdmap e289: 8 total, 7 up, 5 in 2023-10-01T09:26:29.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:29.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:29.920 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:30.021 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:30.058 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:30.105 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:30.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:30.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:30.213 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:30.242 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:30.288 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:30.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:30.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:30.393 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:30.430 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:30.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:30 smithi138 ceph-mon[192513]: failed to encode map e289 with expected crc 2023-10-01T09:26:30.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:30 smithi138 ceph-mon[192513]: failed to encode map e289 with expected crc 2023-10-01T09:26:30.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:30 smithi138 ceph-mon[192513]: failed to encode map e289 with expected crc 2023-10-01T09:26:30.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:30 smithi138 ceph-mon[192513]: failed to encode map e289 with expected crc 2023-10-01T09:26:30.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:30 smithi138 ceph-mon[192513]: pgmap v764: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 4.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 425 op/s; 52741/222696 objects degraded (23.683%); 18676/222696 objects misplaced (8.386%); 3.2 MiB/s, 50 objects/s recovering 2023-10-01T09:26:30.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:30 smithi138 ceph-mon[192513]: osdmap e290: 8 total, 7 up, 5 in 2023-10-01T09:26:30.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:30 smithi138 ceph-mon[185886]: failed to encode map e289 with expected crc 2023-10-01T09:26:30.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:30 smithi138 ceph-mon[185886]: failed to encode map e289 with expected crc 2023-10-01T09:26:30.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:30 smithi138 ceph-mon[185886]: failed to encode map e289 with expected crc 2023-10-01T09:26:30.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:30 smithi138 ceph-mon[185886]: failed to encode map e289 with expected crc 2023-10-01T09:26:30.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:30 smithi138 ceph-mon[185886]: pgmap v764: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 4.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 425 op/s; 52741/222696 objects degraded (23.683%); 18676/222696 objects misplaced (8.386%); 3.2 MiB/s, 50 objects/s recovering 2023-10-01T09:26:30.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:30 smithi138 ceph-mon[185886]: osdmap e290: 8 total, 7 up, 5 in 2023-10-01T09:26:30.579 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:26:30.579 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:26:30.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:26:30.580 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:26:30.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.580 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95256 6.1 6:85462a48:::benchmark_data_smithi138_307751_object95255:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-01T09:26:30.580 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:01.373190+0000", 2023-10-01T09:26:30.580 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 329.202042941, 2023-10-01T09:26:30.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.87755441599999995, 2023-10-01T09:26:30.580 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.580 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:26:30.580 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.580 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.580 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.581 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95256 2023-10-01T09:26:30.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.581 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.373190+0000", 2023-10-01T09:26:30.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.373190+0000", 2023-10-01T09:26:30.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.373189+0000", 2023-10-01T09:26:30.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:30.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.373243+0000", 2023-10-01T09:26:30.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4184000000000002e-05 2023-10-01T09:26:30.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.373250+0000", 2023-10-01T09:26:30.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.314e-06 2023-10-01T09:26:30.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.373254+0000", 2023-10-01T09:26:30.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7999999999999998e-06 2023-10-01T09:26:30.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.377023+0000", 2023-10-01T09:26:30.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003768764 2023-10-01T09:26:30.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.377179+0000", 2023-10-01T09:26:30.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015610299999999999 2023-10-01T09:26:30.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:26:30.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.377374+0000", 2023-10-01T09:26:30.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019462099999999999 2023-10-01T09:26:30.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.386021+0000", 2023-10-01T09:26:30.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086471499999999993 2023-10-01T09:26:30.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.390887+0000", 2023-10-01T09:26:30.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048662430000000001 2023-10-01T09:26:30.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.406652+0000", 2023-10-01T09:26:30.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015764347000000001 2023-10-01T09:26:30.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.406664+0000", 2023-10-01T09:26:30.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2298e-05 2023-10-01T09:26:30.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.250646+0000", 2023-10-01T09:26:30.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84398162300000001 2023-10-01T09:26:30.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.250744+0000", 2023-10-01T09:26:30.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8435000000000002e-05 2023-10-01T09:26:30.587 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.587 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.587 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.587 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95269 6.4 6:22be3102:::benchmark_data_smithi138_307751_object95268:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-01T09:26:30.587 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:01.399198+0000", 2023-10-01T09:26:30.587 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 329.17603474200001, 2023-10-01T09:26:30.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.007680863, 2023-10-01T09:26:30.588 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.588 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.588 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.588 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.588 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.588 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95269 2023-10-01T09:26:30.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.588 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399198+0000", 2023-10-01T09:26:30.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399198+0000", 2023-10-01T09:26:30.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399196+0000", 2023-10-01T09:26:30.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:26:30.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399248+0000", 2023-10-01T09:26:30.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2265999999999997e-05 2023-10-01T09:26:30.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399266+0000", 2023-10-01T09:26:30.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7453e-05 2023-10-01T09:26:30.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399274+0000", 2023-10-01T09:26:30.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3750000000000003e-06 2023-10-01T09:26:30.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.451332+0000", 2023-10-01T09:26:30.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.052058412999999998 2023-10-01T09:26:30.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.451344+0000", 2023-10-01T09:26:30.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1488000000000001e-05 2023-10-01T09:26:30.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.128553+0000", 2023-10-01T09:26:30.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67720899700000003 2023-10-01T09:26:30.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.128565+0000", 2023-10-01T09:26:30.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1986e-05 2023-10-01T09:26:30.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.370376+0000", 2023-10-01T09:26:30.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2418113319999999 2023-10-01T09:26:30.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.370494+0000", 2023-10-01T09:26:30.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117982 2023-10-01T09:26:30.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:30.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.370621+0000", 2023-10-01T09:26:30.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012717699999999999 2023-10-01T09:26:30.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.393133+0000", 2023-10-01T09:26:30.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.022511258999999999 2023-10-01T09:26:30.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.398001+0000", 2023-10-01T09:26:30.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048688330000000004 2023-10-01T09:26:30.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.402860+0000", 2023-10-01T09:26:30.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048585390000000003 2023-10-01T09:26:30.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.402903+0000", 2023-10-01T09:26:30.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2982999999999998e-05 2023-10-01T09:26:30.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.406879+0000", 2023-10-01T09:26:30.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039757539999999997 2023-10-01T09:26:30.596 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.596 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.596 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.596 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95270 6.4 6:27c5a4cb:::benchmark_data_smithi138_307751_object95269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-10-01T09:26:30.596 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:01.399284+0000", 2023-10-01T09:26:30.596 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 329.17594879199999, 2023-10-01T09:26:30.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0401369049999998, 2023-10-01T09:26:30.596 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.597 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.597 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.597 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.597 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.597 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95270 2023-10-01T09:26:30.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.597 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399284+0000", 2023-10-01T09:26:30.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399284+0000", 2023-10-01T09:26:30.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399283+0000", 2023-10-01T09:26:30.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:30.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399323+0000", 2023-10-01T09:26:30.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9362e-05 2023-10-01T09:26:30.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399330+0000", 2023-10-01T09:26:30.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.99e-06 2023-10-01T09:26:30.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.399335+0000", 2023-10-01T09:26:30.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2769999999999996e-06 2023-10-01T09:26:30.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.451365+0000", 2023-10-01T09:26:30.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.052030013 2023-10-01T09:26:30.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:01.451371+0000", 2023-10-01T09:26:30.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1859999999999997e-06 2023-10-01T09:26:30.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.128588+0000", 2023-10-01T09:26:30.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67721646599999996 2023-10-01T09:26:30.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.128595+0000", 2023-10-01T09:26:30.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0199999999999997e-06 2023-10-01T09:26:30.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.406917+0000", 2023-10-01T09:26:30.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.278322467 2023-10-01T09:26:30.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.407076+0000", 2023-10-01T09:26:30.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015908299999999999 2023-10-01T09:26:30.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:30.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.407217+0000", 2023-10-01T09:26:30.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014033200000000001 2023-10-01T09:26:30.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.413442+0000", 2023-10-01T09:26:30.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0062251240000000003 2023-10-01T09:26:30.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.415175+0000", 2023-10-01T09:26:30.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001733278 2023-10-01T09:26:30.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.436343+0000", 2023-10-01T09:26:30.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.021167605999999999 2023-10-01T09:26:30.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.436383+0000", 2023-10-01T09:26:30.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.074e-05 2023-10-01T09:26:30.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.439421+0000", 2023-10-01T09:26:30.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003037456 2023-10-01T09:26:30.605 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.605 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.605 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.605 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95285 6.4 6:20d08f7b:::benchmark_data_smithi138_307751_object95284:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e208)", 2023-10-01T09:26:30.605 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:02.212413+0000", 2023-10-01T09:26:30.605 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 328.36281968499998, 2023-10-01T09:26:30.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2300618379999999, 2023-10-01T09:26:30.605 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.605 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.605 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.606 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.606 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.606 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95285 2023-10-01T09:26:30.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.606 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.212413+0000", 2023-10-01T09:26:30.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.212413+0000", 2023-10-01T09:26:30.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.212407+0000", 2023-10-01T09:26:30.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999938 2023-10-01T09:26:30.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.212455+0000", 2023-10-01T09:26:30.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7827999999999997e-05 2023-10-01T09:26:30.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.212475+0000", 2023-10-01T09:26:30.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0061999999999999e-05 2023-10-01T09:26:30.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.212491+0000", 2023-10-01T09:26:30.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6752000000000002e-05 2023-10-01T09:26:30.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.232186+0000", 2023-10-01T09:26:30.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019694593999999999 2023-10-01T09:26:30.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:02.232199+0000", 2023-10-01T09:26:30.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2581e-05 2023-10-01T09:26:30.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.409492+0000", 2023-10-01T09:26:30.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.177293527 2023-10-01T09:26:30.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.409581+0000", 2023-10-01T09:26:30.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9182000000000001e-05 2023-10-01T09:26:30.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:30.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.409677+0000", 2023-10-01T09:26:30.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5585e-05 2023-10-01T09:26:30.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.413457+0000", 2023-10-01T09:26:30.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037799370000000001 2023-10-01T09:26:30.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.415218+0000", 2023-10-01T09:26:30.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017614009999999999 2023-10-01T09:26:30.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.439462+0000", 2023-10-01T09:26:30.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024244137999999998 2023-10-01T09:26:30.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.439500+0000", 2023-10-01T09:26:30.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7540999999999999e-05 2023-10-01T09:26:30.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.442475+0000", 2023-10-01T09:26:30.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029749310000000001 2023-10-01T09:26:30.613 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.613 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.613 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.613 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96670 6.1 6:89741715:::benchmark_data_smithi138_307751_object1301:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:26:30.613 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.610842+0000", 2023-10-01T09:26:30.613 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 310.96439049600002, 2023-10-01T09:26:30.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94294599000000001, 2023-10-01T09:26:30.613 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.613 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.614 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.614 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.614 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.614 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96670 2023-10-01T09:26:30.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.614 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610842+0000", 2023-10-01T09:26:30.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610842+0000", 2023-10-01T09:26:30.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610840+0000", 2023-10-01T09:26:30.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:26:30.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610848+0000", 2023-10-01T09:26:30.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6909999999999997e-06 2023-10-01T09:26:30.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610866+0000", 2023-10-01T09:26:30.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7680999999999999e-05 2023-10-01T09:26:30.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610874+0000", 2023-10-01T09:26:30.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8480000000000007e-06 2023-10-01T09:26:30.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610935+0000", 2023-10-01T09:26:30.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0109000000000003e-05 2023-10-01T09:26:30.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.610946+0000", 2023-10-01T09:26:30.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.145e-05 2023-10-01T09:26:30.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.545063+0000", 2023-10-01T09:26:30.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93411715900000003 2023-10-01T09:26:30.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.545224+0000", 2023-10-01T09:26:30.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016116500000000001 2023-10-01T09:26:30.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:26:30.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.545369+0000", 2023-10-01T09:26:30.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000144501 2023-10-01T09:26:30.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.553649+0000", 2023-10-01T09:26:30.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0082804039999999999 2023-10-01T09:26:30.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.553661+0000", 2023-10-01T09:26:30.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2086000000000001e-05 2023-10-01T09:26:30.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.553694+0000", 2023-10-01T09:26:30.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2292000000000002e-05 2023-10-01T09:26:30.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.553742+0000", 2023-10-01T09:26:30.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8338999999999999e-05 2023-10-01T09:26:30.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.553788+0000", 2023-10-01T09:26:30.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6186999999999997e-05 2023-10-01T09:26:30.621 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.621 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.621 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.621 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96672 6.1 6:832703e2:::benchmark_data_smithi138_307751_object1303:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:26:30.621 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.611352+0000", 2023-10-01T09:26:30.621 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 310.96388048599999, 2023-10-01T09:26:30.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.948355422, 2023-10-01T09:26:30.622 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.622 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.622 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.622 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.622 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.622 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96672 2023-10-01T09:26:30.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.622 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611352+0000", 2023-10-01T09:26:30.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611352+0000", 2023-10-01T09:26:30.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611351+0000", 2023-10-01T09:26:30.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:26:30.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611356+0000", 2023-10-01T09:26:30.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0869999999999997e-06 2023-10-01T09:26:30.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611371+0000", 2023-10-01T09:26:30.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4251e-05 2023-10-01T09:26:30.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611380+0000", 2023-10-01T09:26:30.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9169999999999995e-06 2023-10-01T09:26:30.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611430+0000", 2023-10-01T09:26:30.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0557999999999999e-05 2023-10-01T09:26:30.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.611442+0000", 2023-10-01T09:26:30.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1501000000000001e-05 2023-10-01T09:26:30.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.548561+0000", 2023-10-01T09:26:30.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.937118856 2023-10-01T09:26:30.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.548685+0000", 2023-10-01T09:26:30.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012445499999999999 2023-10-01T09:26:30.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:26:30.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.548760+0000", 2023-10-01T09:26:30.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5332000000000003e-05 2023-10-01T09:26:30.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.553926+0000", 2023-10-01T09:26:30.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051653580000000001 2023-10-01T09:26:30.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559605+0000", 2023-10-01T09:26:30.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056791380000000002 2023-10-01T09:26:30.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559634+0000", 2023-10-01T09:26:30.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9139999999999999e-05 2023-10-01T09:26:30.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559667+0000", 2023-10-01T09:26:30.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2948999999999999e-05 2023-10-01T09:26:30.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559708+0000", 2023-10-01T09:26:30.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0606999999999999e-05 2023-10-01T09:26:30.629 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.629 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.629 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.630 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96683 6.1 6:82591597:::benchmark_data_smithi138_307751_object1314:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:26:30.630 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.636038+0000", 2023-10-01T09:26:30.630 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 310.93919465300002, 2023-10-01T09:26:30.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92376125799999997, 2023-10-01T09:26:30.630 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.630 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.630 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.630 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.630 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.630 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96683 2023-10-01T09:26:30.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.631 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636038+0000", 2023-10-01T09:26:30.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636038+0000", 2023-10-01T09:26:30.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636036+0000", 2023-10-01T09:26:30.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:26:30.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636045+0000", 2023-10-01T09:26:30.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.054e-06 2023-10-01T09:26:30.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636063+0000", 2023-10-01T09:26:30.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8025000000000001e-05 2023-10-01T09:26:30.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636072+0000", 2023-10-01T09:26:30.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0829999999999993e-06 2023-10-01T09:26:30.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636127+0000", 2023-10-01T09:26:30.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4907999999999997e-05 2023-10-01T09:26:30.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.636137+0000", 2023-10-01T09:26:30.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3589999999999997e-06 2023-10-01T09:26:30.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.550899+0000", 2023-10-01T09:26:30.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91476241400000002 2023-10-01T09:26:30.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.550976+0000", 2023-10-01T09:26:30.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6611000000000002e-05 2023-10-01T09:26:30.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:26:30.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.551047+0000", 2023-10-01T09:26:30.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1879999999999996e-05 2023-10-01T09:26:30.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559596+0000", 2023-10-01T09:26:30.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085482849999999992 2023-10-01T09:26:30.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559611+0000", 2023-10-01T09:26:30.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5514000000000001e-05 2023-10-01T09:26:30.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559740+0000", 2023-10-01T09:26:30.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012912000000000001 2023-10-01T09:26:30.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559766+0000", 2023-10-01T09:26:30.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5973e-05 2023-10-01T09:26:30.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559799+0000", 2023-10-01T09:26:30.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2895000000000003e-05 2023-10-01T09:26:30.637 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.637 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.638 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.638 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96694 6.1 6:823df3f4:::benchmark_data_smithi138_307751_object1325:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:26:30.638 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.638010+0000", 2023-10-01T09:26:30.638 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 310.93722214000002, 2023-10-01T09:26:30.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92454698599999996, 2023-10-01T09:26:30.638 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.638 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.638 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.638 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.639 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.639 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96694 2023-10-01T09:26:30.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.639 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638010+0000", 2023-10-01T09:26:30.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638010+0000", 2023-10-01T09:26:30.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638010+0000", 2023-10-01T09:26:30.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:30.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638013+0000", 2023-10-01T09:26:30.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.402e-06 2023-10-01T09:26:30.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638021+0000", 2023-10-01T09:26:30.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6479999999999992e-06 2023-10-01T09:26:30.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638026+0000", 2023-10-01T09:26:30.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1340000000000003e-06 2023-10-01T09:26:30.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638089+0000", 2023-10-01T09:26:30.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2309000000000003e-05 2023-10-01T09:26:30.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.638097+0000", 2023-10-01T09:26:30.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6880000000000005e-06 2023-10-01T09:26:30.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.553957+0000", 2023-10-01T09:26:30.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91585984399999998 2023-10-01T09:26:30.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.554068+0000", 2023-10-01T09:26:30.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111307 2023-10-01T09:26:30.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:26:30.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.554154+0000", 2023-10-01T09:26:30.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5555000000000003e-05 2023-10-01T09:26:30.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559811+0000", 2023-10-01T09:26:30.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056574449999999997 2023-10-01T09:26:30.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559846+0000", 2023-10-01T09:26:30.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4576999999999998e-05 2023-10-01T09:26:30.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.562443+0000", 2023-10-01T09:26:30.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025973630000000001 2023-10-01T09:26:30.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.562497+0000", 2023-10-01T09:26:30.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3965000000000003e-05 2023-10-01T09:26:30.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.562557+0000", 2023-10-01T09:26:30.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0052e-05 2023-10-01T09:26:30.646 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.646 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.646 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.646 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96696 6.1 6:85d4c37b:::benchmark_data_smithi138_307751_object1327:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:26:30.646 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.642189+0000", 2023-10-01T09:26:30.646 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 310.93304398999999, 2023-10-01T09:26:30.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92620703800000004, 2023-10-01T09:26:30.646 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.646 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.647 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.647 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.647 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.647 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96696 2023-10-01T09:26:30.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.647 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642189+0000", 2023-10-01T09:26:30.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642189+0000", 2023-10-01T09:26:30.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642187+0000", 2023-10-01T09:26:30.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:30.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642193+0000", 2023-10-01T09:26:30.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9179999999999999e-06 2023-10-01T09:26:30.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642215+0000", 2023-10-01T09:26:30.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1492999999999999e-05 2023-10-01T09:26:30.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642222+0000", 2023-10-01T09:26:30.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3180000000000001e-06 2023-10-01T09:26:30.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642274+0000", 2023-10-01T09:26:30.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1579999999999997e-05 2023-10-01T09:26:30.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.642283+0000", 2023-10-01T09:26:30.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7010000000000008e-06 2023-10-01T09:26:30.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.557012+0000", 2023-10-01T09:26:30.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91472850299999997 2023-10-01T09:26:30.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.557138+0000", 2023-10-01T09:26:30.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000126604 2023-10-01T09:26:30.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:26:30.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.557224+0000", 2023-10-01T09:26:30.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6123000000000002e-05 2023-10-01T09:26:30.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.562413+0000", 2023-10-01T09:26:30.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051883889999999998 2023-10-01T09:26:30.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.565661+0000", 2023-10-01T09:26:30.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032479039999999998 2023-10-01T09:26:30.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568310+0000", 2023-10-01T09:26:30.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0026492170000000002 2023-10-01T09:26:30.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568342+0000", 2023-10-01T09:26:30.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1946000000000002e-05 2023-10-01T09:26:30.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568396+0000", 2023-10-01T09:26:30.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3701e-05 2023-10-01T09:26:30.654 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.654 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.654 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.654 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96700 6.1 6:86043e7e:::benchmark_data_smithi138_307751_object1331:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:26:30.654 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.649900+0000", 2023-10-01T09:26:30.654 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 310.925332787, 2023-10-01T09:26:30.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.91865500700000002, 2023-10-01T09:26:30.654 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.655 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.655 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.655 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.655 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.655 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96700 2023-10-01T09:26:30.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.655 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649900+0000", 2023-10-01T09:26:30.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649900+0000", 2023-10-01T09:26:30.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649899+0000", 2023-10-01T09:26:30.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:30.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649905+0000", 2023-10-01T09:26:30.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3450000000000002e-06 2023-10-01T09:26:30.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649919+0000", 2023-10-01T09:26:30.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3854e-05 2023-10-01T09:26:30.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649926+0000", 2023-10-01T09:26:30.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3409999999999997e-06 2023-10-01T09:26:30.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649985+0000", 2023-10-01T09:26:30.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8761999999999997e-05 2023-10-01T09:26:30.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.649997+0000", 2023-10-01T09:26:30.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2571999999999999e-05 2023-10-01T09:26:30.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559870+0000", 2023-10-01T09:26:30.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.90987212200000001 2023-10-01T09:26:30.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.559945+0000", 2023-10-01T09:26:30.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5581999999999996e-05 2023-10-01T09:26:30.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:26:30.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.560021+0000", 2023-10-01T09:26:30.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6348999999999997e-05 2023-10-01T09:26:30.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568248+0000", 2023-10-01T09:26:30.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0082266019999999995 2023-10-01T09:26:30.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568261+0000", 2023-10-01T09:26:30.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3191e-05 2023-10-01T09:26:30.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568465+0000", 2023-10-01T09:26:30.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020326900000000001 2023-10-01T09:26:30.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568516+0000", 2023-10-01T09:26:30.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1004999999999997e-05 2023-10-01T09:26:30.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568555+0000", 2023-10-01T09:26:30.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9289000000000003e-05 2023-10-01T09:26:30.663 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.663 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.663 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.663 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96724 6.1 6:85bdae1f:::benchmark_data_smithi138_307751_object1355:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:26:30.663 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.681228+0000", 2023-10-01T09:26:30.663 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 310.89400503299998, 2023-10-01T09:26:30.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.89070485099999996, 2023-10-01T09:26:30.663 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.663 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.664 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.664 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.664 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.664 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96724 2023-10-01T09:26:30.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.664 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681228+0000", 2023-10-01T09:26:30.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681228+0000", 2023-10-01T09:26:30.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681226+0000", 2023-10-01T09:26:30.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:30.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681232+0000", 2023-10-01T09:26:30.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5030000000000002e-06 2023-10-01T09:26:30.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681244+0000", 2023-10-01T09:26:30.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1861999999999999e-05 2023-10-01T09:26:30.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681252+0000", 2023-10-01T09:26:30.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8790000000000003e-06 2023-10-01T09:26:30.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681312+0000", 2023-10-01T09:26:30.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9669000000000001e-05 2023-10-01T09:26:30.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.681322+0000", 2023-10-01T09:26:30.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7170000000000003e-06 2023-10-01T09:26:30.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.562614+0000", 2023-10-01T09:26:30.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88129269799999999 2023-10-01T09:26:30.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.562760+0000", 2023-10-01T09:26:30.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014615200000000001 2023-10-01T09:26:30.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:26:30.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.562888+0000", 2023-10-01T09:26:30.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012731900000000001 2023-10-01T09:26:30.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568571+0000", 2023-10-01T09:26:30.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056834939999999999 2023-10-01T09:26:30.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.571795+0000", 2023-10-01T09:26:30.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003223722 2023-10-01T09:26:30.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.571840+0000", 2023-10-01T09:26:30.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5331999999999999e-05 2023-10-01T09:26:30.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.571895+0000", 2023-10-01T09:26:30.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4256999999999999e-05 2023-10-01T09:26:30.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.571932+0000", 2023-10-01T09:26:30.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7846999999999999e-05 2023-10-01T09:26:30.671 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.671 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.671 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.671 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96735 6.1 6:86696907:::benchmark_data_smithi138_307751_object1366:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:26:30.671 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.695283+0000", 2023-10-01T09:26:30.671 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 310.87994966899998, 2023-10-01T09:26:30.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88571714300000004, 2023-10-01T09:26:30.672 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.672 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.672 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.672 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.672 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.672 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96735 2023-10-01T09:26:30.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.672 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695283+0000", 2023-10-01T09:26:30.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695283+0000", 2023-10-01T09:26:30.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695281+0000", 2023-10-01T09:26:30.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:30.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695287+0000", 2023-10-01T09:26:30.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9649999999999996e-06 2023-10-01T09:26:30.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695299+0000", 2023-10-01T09:26:30.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1955e-05 2023-10-01T09:26:30.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695308+0000", 2023-10-01T09:26:30.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4950000000000008e-06 2023-10-01T09:26:30.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695378+0000", 2023-10-01T09:26:30.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0333999999999994e-05 2023-10-01T09:26:30.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695392+0000", 2023-10-01T09:26:30.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3529999999999999e-05 2023-10-01T09:26:30.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.565707+0000", 2023-10-01T09:26:30.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.87031560699999999 2023-10-01T09:26:30.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.565818+0000", 2023-10-01T09:26:30.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111097 2023-10-01T09:26:30.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:26:30.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.565913+0000", 2023-10-01T09:26:30.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4927999999999996e-05 2023-10-01T09:26:30.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.571732+0000", 2023-10-01T09:26:30.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005819243 2023-10-01T09:26:30.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.580894+0000", 2023-10-01T09:26:30.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0091619010000000001 2023-10-01T09:26:30.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.580924+0000", 2023-10-01T09:26:30.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0165999999999999e-05 2023-10-01T09:26:30.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.580956+0000", 2023-10-01T09:26:30.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1526000000000003e-05 2023-10-01T09:26:30.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581000+0000", 2023-10-01T09:26:30.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4057000000000002e-05 2023-10-01T09:26:30.679 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.679 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.679 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.679 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96736 6.1 6:8b2701b9:::benchmark_data_smithi138_307751_object1367:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:26:30.679 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.695846+0000", 2023-10-01T09:26:30.679 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 310.87938648400001, 2023-10-01T09:26:30.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88535111399999999, 2023-10-01T09:26:30.679 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.680 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.680 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.680 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.680 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.680 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96736 2023-10-01T09:26:30.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.680 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695846+0000", 2023-10-01T09:26:30.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695846+0000", 2023-10-01T09:26:30.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695845+0000", 2023-10-01T09:26:30.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:26:30.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695851+0000", 2023-10-01T09:26:30.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1580000000000002e-06 2023-10-01T09:26:30.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695869+0000", 2023-10-01T09:26:30.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7900000000000001e-05 2023-10-01T09:26:30.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695878+0000", 2023-10-01T09:26:30.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6880000000000005e-06 2023-10-01T09:26:30.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695933+0000", 2023-10-01T09:26:30.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5234e-05 2023-10-01T09:26:30.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.695946+0000", 2023-10-01T09:26:30.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3251000000000001e-05 2023-10-01T09:26:30.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568597+0000", 2023-10-01T09:26:30.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.87265044899999999 2023-10-01T09:26:30.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568719+0000", 2023-10-01T09:26:30.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012276100000000001 2023-10-01T09:26:30.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:26:30.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.568851+0000", 2023-10-01T09:26:30.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131965 2023-10-01T09:26:30.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581034+0000", 2023-10-01T09:26:30.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012183100000000001 2023-10-01T09:26:30.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581042+0000", 2023-10-01T09:26:30.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8259999999999993e-06 2023-10-01T09:26:30.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581117+0000", 2023-10-01T09:26:30.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4518999999999996e-05 2023-10-01T09:26:30.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581163+0000", 2023-10-01T09:26:30.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6390999999999997e-05 2023-10-01T09:26:30.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.687 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581197+0000", 2023-10-01T09:26:30.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3998999999999998e-05 2023-10-01T09:26:30.687 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.687 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.687 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.687 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96756 6.1 6:850b9578:::benchmark_data_smithi138_307751_object1387:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:26:30.687 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.726318+0000", 2023-10-01T09:26:30.687 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 310.84891424400001, 2023-10-01T09:26:30.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.855017943, 2023-10-01T09:26:30.688 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.688 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.688 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.688 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.688 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.688 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96756 2023-10-01T09:26:30.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.688 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726318+0000", 2023-10-01T09:26:30.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726318+0000", 2023-10-01T09:26:30.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726317+0000", 2023-10-01T09:26:30.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:30.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726323+0000", 2023-10-01T09:26:30.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2779999999999999e-06 2023-10-01T09:26:30.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726336+0000", 2023-10-01T09:26:30.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3047000000000001e-05 2023-10-01T09:26:30.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726344+0000", 2023-10-01T09:26:30.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2900000000000002e-06 2023-10-01T09:26:30.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726415+0000", 2023-10-01T09:26:30.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0298999999999999e-05 2023-10-01T09:26:30.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.726429+0000", 2023-10-01T09:26:30.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4561e-05 2023-10-01T09:26:30.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.571965+0000", 2023-10-01T09:26:30.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84553552099999996 2023-10-01T09:26:30.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.572048+0000", 2023-10-01T09:26:30.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3534000000000004e-05 2023-10-01T09:26:30.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:26:30.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.572116+0000", 2023-10-01T09:26:30.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7259000000000001e-05 2023-10-01T09:26:30.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581050+0000", 2023-10-01T09:26:30.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0089346210000000002 2023-10-01T09:26:30.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581093+0000", 2023-10-01T09:26:30.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2784000000000002e-05 2023-10-01T09:26:30.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581269+0000", 2023-10-01T09:26:30.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017614700000000001 2023-10-01T09:26:30.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581310+0000", 2023-10-01T09:26:30.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1050999999999997e-05 2023-10-01T09:26:30.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.695 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581336+0000", 2023-10-01T09:26:30.695 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6142000000000001e-05 2023-10-01T09:26:30.695 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.695 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.695 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.695 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96761 6.1 6:8f74277c:::benchmark_data_smithi138_307751_object1392:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:26:30.695 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.733089+0000", 2023-10-01T09:26:30.695 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 310.84214332400001, 2023-10-01T09:26:30.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84834553899999998, 2023-10-01T09:26:30.696 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.696 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.696 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.696 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.696 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.696 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96761 2023-10-01T09:26:30.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.696 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.733089+0000", 2023-10-01T09:26:30.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.733089+0000", 2023-10-01T09:26:30.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.733087+0000", 2023-10-01T09:26:30.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:26:30.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.733095+0000", 2023-10-01T09:26:30.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1189999999999997e-06 2023-10-01T09:26:30.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.733114+0000", 2023-10-01T09:26:30.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8527999999999999e-05 2023-10-01T09:26:30.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.733123+0000", 2023-10-01T09:26:30.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8409999999999995e-06 2023-10-01T09:26:30.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.738263+0000", 2023-10-01T09:26:30.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051396510000000003 2023-10-01T09:26:30.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.738271+0000", 2023-10-01T09:26:30.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6760000000000007e-06 2023-10-01T09:26:30.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.574939+0000", 2023-10-01T09:26:30.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.83666783199999994 2023-10-01T09:26:30.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.575034+0000", 2023-10-01T09:26:30.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5224999999999997e-05 2023-10-01T09:26:30.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:26:30.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.575128+0000", 2023-10-01T09:26:30.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4622000000000003e-05 2023-10-01T09:26:30.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581056+0000", 2023-10-01T09:26:30.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059276659999999998 2023-10-01T09:26:30.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581239+0000", 2023-10-01T09:26:30.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018291999999999999 2023-10-01T09:26:30.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581374+0000", 2023-10-01T09:26:30.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000135406 2023-10-01T09:26:30.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581400+0000", 2023-10-01T09:26:30.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5099000000000001e-05 2023-10-01T09:26:30.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581435+0000", 2023-10-01T09:26:30.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5250000000000003e-05 2023-10-01T09:26:30.703 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.703 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.703 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.703 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96769 6.1 6:89cc583e:::benchmark_data_smithi138_307751_object1400:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:26:30.704 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.751200+0000", 2023-10-01T09:26:30.704 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 310.824032397, 2023-10-01T09:26:30.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84244887800000001, 2023-10-01T09:26:30.704 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.704 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.704 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.704 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.704 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.704 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96769 2023-10-01T09:26:30.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.704 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751200+0000", 2023-10-01T09:26:30.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751200+0000", 2023-10-01T09:26:30.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751199+0000", 2023-10-01T09:26:30.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:30.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751212+0000", 2023-10-01T09:26:30.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3451999999999999e-05 2023-10-01T09:26:30.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751227+0000", 2023-10-01T09:26:30.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4599e-05 2023-10-01T09:26:30.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751235+0000", 2023-10-01T09:26:30.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4309999999999994e-06 2023-10-01T09:26:30.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751291+0000", 2023-10-01T09:26:30.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6378000000000001e-05 2023-10-01T09:26:30.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.751301+0000", 2023-10-01T09:26:30.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5149999999999995e-06 2023-10-01T09:26:30.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.577917+0000", 2023-10-01T09:26:30.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.82661580400000001 2023-10-01T09:26:30.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.578029+0000", 2023-10-01T09:26:30.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011215 2023-10-01T09:26:30.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:26:30.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.578137+0000", 2023-10-01T09:26:30.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108558 2023-10-01T09:26:30.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581466+0000", 2023-10-01T09:26:30.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003328606 2023-10-01T09:26:30.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.587294+0000", 2023-10-01T09:26:30.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058283420000000002 2023-10-01T09:26:30.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.593571+0000", 2023-10-01T09:26:30.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0062766829999999999 2023-10-01T09:26:30.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.593614+0000", 2023-10-01T09:26:30.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3208999999999999e-05 2023-10-01T09:26:30.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.593649+0000", 2023-10-01T09:26:30.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4761999999999997e-05 2023-10-01T09:26:30.711 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.711 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.711 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.712 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96799 6.1 6:837f0a28:::benchmark_data_smithi138_307751_object1430:head [delete] snapc 0=[] ondisk+write+known_if_redirected e215)", 2023-10-01T09:26:30.712 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:19.832042+0000", 2023-10-01T09:26:30.712 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 310.743190218, 2023-10-01T09:26:30.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.76482651700000004, 2023-10-01T09:26:30.712 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.712 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.712 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.712 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.712 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.712 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96799 2023-10-01T09:26:30.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.713 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832042+0000", 2023-10-01T09:26:30.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832042+0000", 2023-10-01T09:26:30.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832040+0000", 2023-10-01T09:26:30.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:26:30.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832052+0000", 2023-10-01T09:26:30.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1433999999999999e-05 2023-10-01T09:26:30.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832076+0000", 2023-10-01T09:26:30.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4539999999999999e-05 2023-10-01T09:26:30.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832090+0000", 2023-10-01T09:26:30.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3461999999999999e-05 2023-10-01T09:26:30.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832156+0000", 2023-10-01T09:26:30.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6365000000000006e-05 2023-10-01T09:26:30.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:19.832169+0000", 2023-10-01T09:26:30.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2627e-05 2023-10-01T09:26:30.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581483+0000", 2023-10-01T09:26:30.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.74931435400000002 2023-10-01T09:26:30.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581573+0000", 2023-10-01T09:26:30.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9826000000000004e-05 2023-10-01T09:26:30.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:26:30.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.581633+0000", 2023-10-01T09:26:30.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0194e-05 2023-10-01T09:26:30.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.590406+0000", 2023-10-01T09:26:30.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0087729320000000006 2023-10-01T09:26:30.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.593681+0000", 2023-10-01T09:26:30.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032753209999999999 2023-10-01T09:26:30.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.720 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.720 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.720 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.596781+0000", 2023-10-01T09:26:30.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003099662 2023-10-01T09:26:30.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.720 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.720 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.720 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.596828+0000", 2023-10-01T09:26:30.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7064999999999997e-05 2023-10-01T09:26:30.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:20.596869+0000", 2023-10-01T09:26:30.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0785999999999999e-05 2023-10-01T09:26:30.721 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.721 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.721 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.721 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109293 6.4 6:24545a10:::benchmark_data_smithi138_307751_object13924:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-01T09:26:30.721 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.601748+0000", 2023-10-01T09:26:30.722 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 290.97348433500002, 2023-10-01T09:26:30.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99822042, 2023-10-01T09:26:30.722 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.722 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:26:30.722 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.722 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.722 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.722 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109293 2023-10-01T09:26:30.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.722 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601748+0000", 2023-10-01T09:26:30.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601748+0000", 2023-10-01T09:26:30.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601748+0000", 2023-10-01T09:26:30.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:30.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601751+0000", 2023-10-01T09:26:30.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.994e-06 2023-10-01T09:26:30.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601757+0000", 2023-10-01T09:26:30.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0909999999999998e-06 2023-10-01T09:26:30.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601762+0000", 2023-10-01T09:26:30.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4149999999999998e-06 2023-10-01T09:26:30.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.842483+0000", 2023-10-01T09:26:30.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.240721043 2023-10-01T09:26:30.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.842491+0000", 2023-10-01T09:26:30.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2450000000000001e-06 2023-10-01T09:26:30.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.593662+0000", 2023-10-01T09:26:30.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.75117160199999999 2023-10-01T09:26:30.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:26:30.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.593795+0000", 2023-10-01T09:26:30.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000132822 2023-10-01T09:26:30.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.599877+0000", 2023-10-01T09:26:30.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0060821499999999997 2023-10-01T09:26:30.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.599969+0000", 2023-10-01T09:26:30.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1590999999999998e-05 2023-10-01T09:26:30.727 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.728 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.728 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.728 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109329 6.1 6:81350742:::benchmark_data_smithi138_307751_object13960:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-01T09:26:30.728 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.601783+0000", 2023-10-01T09:26:30.728 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 290.973449458, 2023-10-01T09:26:30.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.000988641, 2023-10-01T09:26:30.728 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.728 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:26:30.728 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.728 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.729 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.729 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109329 2023-10-01T09:26:30.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.729 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601783+0000", 2023-10-01T09:26:30.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601783+0000", 2023-10-01T09:26:30.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601783+0000", 2023-10-01T09:26:30.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:30.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601785+0000", 2023-10-01T09:26:30.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3190000000000002e-06 2023-10-01T09:26:30.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601791+0000", 2023-10-01T09:26:30.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5199999999999997e-06 2023-10-01T09:26:30.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601795+0000", 2023-10-01T09:26:30.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.578e-06 2023-10-01T09:26:30.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.797959+0000", 2023-10-01T09:26:30.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19616351200000001 2023-10-01T09:26:30.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.797969+0000", 2023-10-01T09:26:30.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0542e-05 2023-10-01T09:26:30.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.584103+0000", 2023-10-01T09:26:30.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.78613325999999994 2023-10-01T09:26:30.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:26:30.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.584221+0000", 2023-10-01T09:26:30.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118752 2023-10-01T09:26:30.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.602718+0000", 2023-10-01T09:26:30.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018497202000000001 2023-10-01T09:26:30.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.602772+0000", 2023-10-01T09:26:30.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3303000000000001e-05 2023-10-01T09:26:30.734 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.734 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.734 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.734 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109339 6.4 6:27f60416:::benchmark_data_smithi138_307751_object13970:head [delete] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-10-01T09:26:30.735 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.875846+0000", 2023-10-01T09:26:30.735 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 290.69938633999999, 2023-10-01T09:26:30.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.78316001899999999, 2023-10-01T09:26:30.735 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:30.735 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:30.735 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:30.735 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:30.735 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:30.735 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109339 2023-10-01T09:26:30.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.735 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:30.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:30.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.875846+0000", 2023-10-01T09:26:30.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:30.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.875846+0000", 2023-10-01T09:26:30.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:30.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:30.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.875845+0000", 2023-10-01T09:26:30.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:30.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:30.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.875852+0000", 2023-10-01T09:26:30.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9290000000000001e-06 2023-10-01T09:26:30.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:30.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.875949+0000", 2023-10-01T09:26:30.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6877000000000001e-05 2023-10-01T09:26:30.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:30.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.875959+0000", 2023-10-01T09:26:30.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0346000000000001e-05 2023-10-01T09:26:30.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.883849+0000", 2023-10-01T09:26:30.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0078898479999999997 2023-10-01T09:26:30.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:30.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.883865+0000", 2023-10-01T09:26:30.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5650000000000001e-05 2023-10-01T09:26:30.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.599996+0000", 2023-10-01T09:26:30.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.71613177400000005 2023-10-01T09:26:30.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:26:30.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.600836+0000", 2023-10-01T09:26:30.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000839874 2023-10-01T09:26:30.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:30.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.629704+0000", 2023-10-01T09:26:30.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.028867228000000002 2023-10-01T09:26:30.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:30.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.629811+0000", 2023-10-01T09:26:30.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107219 2023-10-01T09:26:30.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:30.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.630140+0000", 2023-10-01T09:26:30.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00032947299999999999 2023-10-01T09:26:30.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.636495+0000", 2023-10-01T09:26:30.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006354274 2023-10-01T09:26:30.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:30.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.636508+0000", 2023-10-01T09:26:30.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3604999999999999e-05 2023-10-01T09:26:30.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:30.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.657107+0000", 2023-10-01T09:26:30.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020599118999999999 2023-10-01T09:26:30.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:30.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.657143+0000", 2023-10-01T09:26:30.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.54e-05 2023-10-01T09:26:30.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:30.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:30.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:30.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.659006+0000", 2023-10-01T09:26:30.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001863647 2023-10-01T09:26:30.743 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.743 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.743 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.743 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:30.743 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:30.743 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:26:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:30 smithi190 ceph-mon[160435]: failed to encode map e289 with expected crc 2023-10-01T09:26:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:30 smithi190 ceph-mon[160435]: failed to encode map e289 with expected crc 2023-10-01T09:26:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:30 smithi190 ceph-mon[160435]: failed to encode map e289 with expected crc 2023-10-01T09:26:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:30 smithi190 ceph-mon[160435]: failed to encode map e289 with expected crc 2023-10-01T09:26:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:30 smithi190 ceph-mon[160435]: pgmap v764: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 4.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 425 op/s; 52741/222696 objects degraded (23.683%); 18676/222696 objects misplaced (8.386%); 3.2 MiB/s, 50 objects/s recovering 2023-10-01T09:26:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:30 smithi190 ceph-mon[160435]: osdmap e290: 8 total, 7 up, 5 in 2023-10-01T09:26:31.391 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:31 smithi190 ceph-mon[160435]: failed to encode map e289 with expected crc 2023-10-01T09:26:31.391 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:31 smithi190 ceph-mon[160435]: failed to encode map e289 with expected crc 2023-10-01T09:26:31.391 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:31 smithi190 ceph-mon[160435]: failed to encode map e290 with expected crc 2023-10-01T09:26:31.391 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:31 smithi190 ceph-mon[160435]: failed to encode map e290 with expected crc 2023-10-01T09:26:31.392 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/242431472' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:26:31.443 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:26:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:31 smithi138 ceph-mon[192513]: failed to encode map e289 with expected crc 2023-10-01T09:26:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:31 smithi138 ceph-mon[192513]: failed to encode map e289 with expected crc 2023-10-01T09:26:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:31 smithi138 ceph-mon[192513]: failed to encode map e290 with expected crc 2023-10-01T09:26:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:31 smithi138 ceph-mon[192513]: failed to encode map e290 with expected crc 2023-10-01T09:26:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/242431472' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:26:31.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:31 smithi138 ceph-mon[185886]: failed to encode map e289 with expected crc 2023-10-01T09:26:31.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:31 smithi138 ceph-mon[185886]: failed to encode map e289 with expected crc 2023-10-01T09:26:31.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:31 smithi138 ceph-mon[185886]: failed to encode map e290 with expected crc 2023-10-01T09:26:31.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:31 smithi138 ceph-mon[185886]: failed to encode map e290 with expected crc 2023-10-01T09:26:31.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/242431472' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:26:32.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:32 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:26:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:32 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/242431472' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:26:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:32 smithi190 ceph-mon[160435]: osdmap e291: 8 total, 7 up, 5 in 2023-10-01T09:26:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:32 smithi190 ceph-mon[160435]: failed to encode map e291 with expected crc 2023-10-01T09:26:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:32 smithi190 ceph-mon[160435]: failed to encode map e291 with expected crc 2023-10-01T09:26:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:32 smithi190 ceph-mon[160435]: failed to encode map e291 with expected crc 2023-10-01T09:26:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:32 smithi190 ceph-mon[160435]: pgmap v767: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 4.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 295 op/s; 52705/221871 objects degraded (23.755%); 18676/221871 objects misplaced (8.418%); 3.6 MiB/s, 54 objects/s recovering 2023-10-01T09:26:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:32 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 52741/222696 objects degraded (23.683%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:32.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:26:32.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/242431472' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:26:32.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[192513]: osdmap e291: 8 total, 7 up, 5 in 2023-10-01T09:26:32.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[192513]: failed to encode map e291 with expected crc 2023-10-01T09:26:32.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[192513]: failed to encode map e291 with expected crc 2023-10-01T09:26:32.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[192513]: failed to encode map e291 with expected crc 2023-10-01T09:26:32.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[192513]: pgmap v767: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 4.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 295 op/s; 52705/221871 objects degraded (23.755%); 18676/221871 objects misplaced (8.418%); 3.6 MiB/s, 54 objects/s recovering 2023-10-01T09:26:32.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 52741/222696 objects degraded (23.683%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:32.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:26:32.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/242431472' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:26:32.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[185886]: osdmap e291: 8 total, 7 up, 5 in 2023-10-01T09:26:32.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[185886]: failed to encode map e291 with expected crc 2023-10-01T09:26:32.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[185886]: failed to encode map e291 with expected crc 2023-10-01T09:26:32.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[185886]: failed to encode map e291 with expected crc 2023-10-01T09:26:32.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[185886]: pgmap v767: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 4.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 295 op/s; 52705/221871 objects degraded (23.755%); 18676/221871 objects misplaced (8.418%); 3.6 MiB/s, 54 objects/s recovering 2023-10-01T09:26:32.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:32 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 52741/222696 objects degraded (23.683%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:32.994 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:33.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:33.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:33.109 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:33.218 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:33.559 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:26:33.657 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:33.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:33.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:33.768 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:33.794 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:33.841 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:33.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:33.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:33.947 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:26:34.043 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:34.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:33 smithi190 ceph-mon[160435]: pgmap v768: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 4.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 276 op/s; 52319/219198 objects degraded (23.868%); 18301/219198 objects misplaced (8.349%); 3.9 MiB/s, 58 objects/s recovering 2023-10-01T09:26:34.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:33 smithi138 ceph-mon[185886]: pgmap v768: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 4.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 276 op/s; 52319/219198 objects degraded (23.868%); 18301/219198 objects misplaced (8.349%); 3.9 MiB/s, 58 objects/s recovering 2023-10-01T09:26:34.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:33 smithi138 ceph-mon[192513]: pgmap v768: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 4.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 276 op/s; 52319/219198 objects degraded (23.868%); 18301/219198 objects misplaced (8.349%); 3.9 MiB/s, 58 objects/s recovering 2023-10-01T09:26:34.211 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:26:34.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:34.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:26:34.324 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:26:34.359 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:26:34.400 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:26:34.408 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:34.447 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:34.481 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:26:34.536 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:26:34.577 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:26:34.592 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:26:34.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:34.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:26:34.614 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:26:34.672 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:26:34.672 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:26:34.672 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:26:34.673 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:26:34.673 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:26:34.673 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:26:34.673 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:26:34.673 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-10-01T09:26:34.673 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:26:34.704 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:34.731 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:34.775 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:34.779 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2] out_osds: [5, 6, 4] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 7, 5] 2023-10-01T09:26:34.779 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:26:34.779 INFO:tasks.thrashosds.thrasher:Reviving osd 6 2023-10-01T09:26:34.779 INFO:tasks.cephadm.osd.6:Restarting osd.6 (starting--it wasn't running)... 2023-10-01T09:26:34.780 DEBUG:teuthology.orchestra.run.smithi190:osd.6> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-01T09:26:34.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:34.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:34.782 DEBUG:teuthology.orchestra.run.smithi190:> sudo systemctl start ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.6 2023-10-01T09:26:34.810 INFO:journalctl@ceph.osd.6.smithi190.stdout:-- Logs begin at Sun 2023-10-01 08:38:18 UTC. -- 2023-10-01T09:26:34.870 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:34 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1993328906' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:26:34.882 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:34.978 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:35.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:26:34 smithi190 systemd[1]: Starting Ceph osd.6 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:26:35.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:34 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1993328906' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:26:35.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:34 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1993328906' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:26:35.324 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:26:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:26:34] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.33.4" 2023-10-01T09:26:35.654 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:35.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:35.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:35.767 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:35.810 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:26:35.859 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:35.921 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:35 smithi138 ceph-mon[185886]: pgmap v769: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 238 op/s; 52136/218103 objects degraded (23.904%); 18301/218103 objects misplaced (8.391%); 3.2 MiB/s, 47 objects/s recovering 2023-10-01T09:26:35.924 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:35 smithi138 ceph-mon[192513]: pgmap v769: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 238 op/s; 52136/218103 objects degraded (23.904%); 18301/218103 objects misplaced (8.391%); 3.2 MiB/s, 47 objects/s recovering 2023-10-01T09:26:36.202 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:36.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:36.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:36.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:35 smithi190 ceph-mon[160435]: pgmap v769: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 238 op/s; 52136/218103 objects degraded (23.904%); 18301/218103 objects misplaced (8.391%); 3.2 MiB/s, 47 objects/s recovering 2023-10-01T09:26:36.308 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:26:35 smithi190 bash[247096]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-10-01T09:26:36.308 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:26:35 smithi190 bash[247096]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2023-10-01T09:26:36.308 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:26:35 smithi190 bash[247096]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2023-10-01T09:26:36.308 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:26:35 smithi190 bash[247096]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-01T09:26:36.308 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:26:35 smithi190 bash[247096]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2023-10-01T09:26:36.308 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:26:35 smithi190 bash[247096]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-10-01T09:26:36.308 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:26:35 smithi190 bash[247096]: --> ceph-volume raw activate successful for osd ID: 6 2023-10-01T09:26:36.319 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:26:36.350 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:37.051 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:26:37.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:37.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:26:37.082 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:26:37.082 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:26:37.182 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:37.230 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:26:37.274 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:37.310 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:37 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 52136/218103 objects degraded (23.904%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:37.459 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:37.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:37.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:37.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:37 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 52136/218103 objects degraded (23.904%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:37.557 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:26:37 smithi190 conmon[247554]: debug 2023-10-01T09:26:37.213+0000 7fcf2256e700 1 -- 172.21.15.190:0/1340445282 <== mon.1 v2:172.21.15.190:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ceb35aa680 con 0x55ceb3571400 2023-10-01T09:26:37.557 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:26:37 smithi190 bash[247513]: 37f530693c7fc2237ac08f17677754d220c69eabee9d845254186ce0581bc5d0 2023-10-01T09:26:37.557 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:26:37 smithi190 systemd[1]: Started Ceph osd.6 for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:26:37.570 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:37 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 52136/218103 objects degraded (23.904%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:37.623 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:37.631 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:26:37.699 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:37.782 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:26:37.782 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:26:37.860 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:37.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:37.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:37.981 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:26:38.011 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:26:37 smithi190 conmon[247554]: debug 2023-10-01T09:26:37.733+0000 7fcf29fe8200 -1 Falling back to public interface 2023-10-01T09:26:38.018 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:38.119 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:26:38.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:38.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:26:38.223 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:26:38.339 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:38.399 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:26:38.401 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:38 smithi138 ceph-mon[185886]: pgmap v770: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 289 op/s; 51813/215748 objects degraded (24.016%); 18093/215748 objects misplaced (8.386%); 3.5 MiB/s, 52 objects/s recovering 2023-10-01T09:26:38.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:38.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:26:38.402 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:38 smithi138 ceph-mon[192513]: pgmap v770: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 289 op/s; 51813/215748 objects degraded (24.016%); 18093/215748 objects misplaced (8.386%); 3.5 MiB/s, 52 objects/s recovering 2023-10-01T09:26:38.503 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:26:38.537 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:38.686 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:26:38.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:38.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:26:38.795 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:26:38.799 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:38 smithi190 ceph-mon[160435]: pgmap v770: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 289 op/s; 51813/215748 objects degraded (24.016%); 18093/215748 objects misplaced (8.386%); 3.5 MiB/s, 52 objects/s recovering 2023-10-01T09:26:38.837 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:38.892 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:26:38.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:38.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:26:39.000 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:26:39.033 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:39.084 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:26:39.094 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:26:39.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:39.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:26:39.199 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:39.280 INFO:teuthology.orchestra.run.smithi190.stderr:no valid command found; 10 closest matches: 2023-10-01T09:26:39.280 INFO:teuthology.orchestra.run.smithi190.stderr:0 2023-10-01T09:26:39.280 INFO:teuthology.orchestra.run.smithi190.stderr:1 2023-10-01T09:26:39.280 INFO:teuthology.orchestra.run.smithi190.stderr:2 2023-10-01T09:26:39.280 INFO:teuthology.orchestra.run.smithi190.stderr:abort 2023-10-01T09:26:39.280 INFO:teuthology.orchestra.run.smithi190.stderr:assert 2023-10-01T09:26:39.280 INFO:teuthology.orchestra.run.smithi190.stderr:config diff 2023-10-01T09:26:39.281 INFO:teuthology.orchestra.run.smithi190.stderr:config diff get 2023-10-01T09:26:39.281 INFO:teuthology.orchestra.run.smithi190.stderr:config get 2023-10-01T09:26:39.281 INFO:teuthology.orchestra.run.smithi190.stderr:config help [] 2023-10-01T09:26:39.281 INFO:teuthology.orchestra.run.smithi190.stderr:config set ... 2023-10-01T09:26:39.281 INFO:teuthology.orchestra.run.smithi190.stderr:admin_socket: invalid command 2023-10-01T09:26:39.283 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:39.452 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:39.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:39.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:39.563 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:26:39.565 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:26:39.627 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:39.789 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:26:39.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:39.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:26:39.909 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:39.940 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:39 smithi138 ceph-mon[185886]: pgmap v771: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 338 op/s; 51494/212862 objects degraded (24.191%); 17855/212862 objects misplaced (8.388%); 4.3 MiB/s, 67 objects/s recovering 2023-10-01T09:26:39.942 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:39 smithi138 ceph-mon[192513]: pgmap v771: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 338 op/s; 51494/212862 objects degraded (24.191%); 17855/212862 objects misplaced (8.388%); 4.3 MiB/s, 67 objects/s recovering 2023-10-01T09:26:39.958 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:39.971 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-01T09:26:39.971 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-6, ['dump_ops_in_flight'] 2023-10-01T09:26:40.030 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:40.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:40.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:40.133 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:40.194 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:40.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:39 smithi190 ceph-mon[160435]: pgmap v771: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 338 op/s; 51494/212862 objects degraded (24.191%); 17855/212862 objects misplaced (8.388%); 4.3 MiB/s, 67 objects/s recovering 2023-10-01T09:26:40.418 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:40.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:40.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:40.531 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:26:40.564 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:40.614 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:26:40.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:40.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:26:40.719 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:26:40.806 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:40.847 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:40.847 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:40.890 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:26:40.984 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:26:40.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:40.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:26:41.053 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:26:41.053 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:26:41.095 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:41.125 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:41.174 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:41.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:41.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:41.281 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:26:41.307 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:41.356 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:26:41.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:41.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:26:41.461 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:41.648 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:42.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:41 smithi190 ceph-mon[160435]: pgmap v772: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 344 op/s; 51372/211728 objects degraded (24.263%); 17855/211728 objects misplaced (8.433%); 4.7 MiB/s, 72 objects/s recovering 2023-10-01T09:26:42.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:41 smithi138 ceph-mon[185886]: pgmap v772: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 344 op/s; 51372/211728 objects degraded (24.263%); 17855/211728 objects misplaced (8.433%); 4.7 MiB/s, 72 objects/s recovering 2023-10-01T09:26:42.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:41 smithi138 ceph-mon[192513]: pgmap v772: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 344 op/s; 51372/211728 objects degraded (24.263%); 17855/211728 objects misplaced (8.433%); 4.7 MiB/s, 72 objects/s recovering 2023-10-01T09:26:42.999 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:26:43.194 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:42 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/205920167' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:26:43.194 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:42 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:26:43.194 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:42 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 51372/211728 objects degraded (24.263%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:43.195 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:42 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/205920167' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:26:43.195 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:42 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:26:43.195 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:42 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 51372/211728 objects degraded (24.263%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:43.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:42 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/205920167' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:26:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:42 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:26:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:42 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 51372/211728 objects degraded (24.263%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:43.722 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:43.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:43.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:43.835 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:43.866 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:43.918 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:43.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:43.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:44.024 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:44.059 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:43 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:26:44.059 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:43 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:26:44.059 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:43 smithi138 ceph-mon[185886]: osdmap e292: 8 total, 7 up, 5 in 2023-10-01T09:26:44.059 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:43 smithi138 ceph-mon[185886]: failed to encode map e292 with expected crc 2023-10-01T09:26:44.059 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:43 smithi138 ceph-mon[185886]: failed to encode map e292 with expected crc 2023-10-01T09:26:44.059 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:43 smithi138 ceph-mon[185886]: failed to encode map e292 with expected crc 2023-10-01T09:26:44.059 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:43 smithi138 ceph-mon[185886]: pgmap v774: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 370 op/s; 51018/208068 objects degraded (24.520%); 17389/208068 objects misplaced (8.357%); 4.1 MiB/s, 66 objects/s recovering 2023-10-01T09:26:44.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:43 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:26:44.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:43 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:26:44.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:43 smithi138 ceph-mon[192513]: osdmap e292: 8 total, 7 up, 5 in 2023-10-01T09:26:44.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:43 smithi138 ceph-mon[192513]: failed to encode map e292 with expected crc 2023-10-01T09:26:44.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:43 smithi138 ceph-mon[192513]: failed to encode map e292 with expected crc 2023-10-01T09:26:44.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:43 smithi138 ceph-mon[192513]: failed to encode map e292 with expected crc 2023-10-01T09:26:44.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:43 smithi138 ceph-mon[192513]: pgmap v774: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 370 op/s; 51018/208068 objects degraded (24.520%); 17389/208068 objects misplaced (8.357%); 4.1 MiB/s, 66 objects/s recovering 2023-10-01T09:26:44.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:43 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:26:44.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:43 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:26:44.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:43 smithi190 ceph-mon[160435]: osdmap e292: 8 total, 7 up, 5 in 2023-10-01T09:26:44.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:43 smithi190 ceph-mon[160435]: failed to encode map e292 with expected crc 2023-10-01T09:26:44.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:43 smithi190 ceph-mon[160435]: failed to encode map e292 with expected crc 2023-10-01T09:26:44.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:43 smithi190 ceph-mon[160435]: failed to encode map e292 with expected crc 2023-10-01T09:26:44.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:43 smithi190 ceph-mon[160435]: pgmap v774: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 370 op/s; 51018/208068 objects degraded (24.520%); 17389/208068 objects misplaced (8.357%); 4.1 MiB/s, 66 objects/s recovering 2023-10-01T09:26:44.621 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:44.798 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:44.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:44.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:44.920 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:44.948 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:44.972 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:26:45.002 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:45.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:45.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:45.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:26:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:26:44] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.33.4" 2023-10-01T09:26:45.109 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:26:45.215 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:45.330 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:26:45.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:45.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:26:45.394 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:26:45.444 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:45.478 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:45.543 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:45.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:45.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:45.646 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:26:45.755 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:46.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:45 smithi138 ceph-mon[185886]: pgmap v775: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 383 op/s; 50794/206604 objects degraded (24.585%); 17389/206604 objects misplaced (8.417%); 4.3 MiB/s, 68 objects/s recovering 2023-10-01T09:26:46.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:45 smithi138 ceph-mon[192513]: pgmap v775: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 383 op/s; 50794/206604 objects degraded (24.585%); 17389/206604 objects misplaced (8.417%); 4.3 MiB/s, 68 objects/s recovering 2023-10-01T09:26:46.165 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:26:46.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:46.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:26:46.307 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:26:45 smithi190 conmon[247554]: debug 2023-10-01T09:26:45.947+0000 7fcf29fe8200 -1 osd.6 268 log_to_monitors {default=true} 2023-10-01T09:26:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:45 smithi190 ceph-mon[160435]: pgmap v775: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 383 op/s; 50794/206604 objects degraded (24.585%); 17389/206604 objects misplaced (8.417%); 4.3 MiB/s, 68 objects/s recovering 2023-10-01T09:26:46.308 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:46.362 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:26:46.362 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:26:46.363 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:46.622 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:26:46.724 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:46.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:46.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:46.837 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:46.869 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:46.986 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:46.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:46.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:46.996 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:26:46 smithi190 conmon[247554]: debug 2023-10-01T09:26:46.891+0000 7fcf1b4e3700 -1 osd.6 268 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T09:26:46.996 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:46 smithi190 ceph-mon[160435]: from='osd.6 [v2:172.21.15.190:6816/1281430772,v1:172.21.15.190:6817/1281430772]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T09:26:46.997 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:46 smithi190 ceph-mon[160435]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T09:26:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:46 smithi138 ceph-mon[185886]: from='osd.6 [v2:172.21.15.190:6816/1281430772,v1:172.21.15.190:6817/1281430772]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T09:26:47.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:46 smithi138 ceph-mon[185886]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T09:26:47.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:46 smithi138 ceph-mon[192513]: from='osd.6 [v2:172.21.15.190:6816/1281430772,v1:172.21.15.190:6817/1281430772]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T09:26:47.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:46 smithi138 ceph-mon[192513]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T09:26:47.090 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:47.201 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:26:47.201 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:26:47.201 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 0 2023-10-01T09:26:47.201 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:26:47.377 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:48.154 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:48.157 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:26:48.161 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[185886]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-01T09:26:48.161 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[185886]: osdmap e293: 8 total, 7 up, 5 in 2023-10-01T09:26:48.161 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[185886]: from='osd.6 [v2:172.21.15.190:6816/1281430772,v1:172.21.15.190:6817/1281430772]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:26:48.161 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[185886]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:26:48.162 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[185886]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.162 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[185886]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.162 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[185886]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.162 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[185886]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.162 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 50794/206604 objects degraded (24.585%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:48.162 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[185886]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.162 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[185886]: pgmap v777: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 371 op/s; 50419/203946 objects degraded (24.722%); 17144/203946 objects misplaced (8.406%); 3.7 MiB/s, 59 objects/s recovering 2023-10-01T09:26:48.162 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[192513]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-01T09:26:48.163 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[192513]: osdmap e293: 8 total, 7 up, 5 in 2023-10-01T09:26:48.163 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[192513]: from='osd.6 [v2:172.21.15.190:6816/1281430772,v1:172.21.15.190:6817/1281430772]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:26:48.163 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[192513]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:26:48.163 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[192513]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.163 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[192513]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.163 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[192513]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.163 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[192513]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.163 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 50794/206604 objects degraded (24.585%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:48.163 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[192513]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.163 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:47 smithi138 ceph-mon[192513]: pgmap v777: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 371 op/s; 50419/203946 objects degraded (24.722%); 17144/203946 objects misplaced (8.406%); 3.7 MiB/s, 59 objects/s recovering 2023-10-01T09:26:48.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:48.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:48.192 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:26:48.250 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:26:48.265 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:26:48.304 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:26:48.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:47 smithi190 ceph-mon[160435]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-01T09:26:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:47 smithi190 ceph-mon[160435]: osdmap e293: 8 total, 7 up, 5 in 2023-10-01T09:26:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:47 smithi190 ceph-mon[160435]: from='osd.6 [v2:172.21.15.190:6816/1281430772,v1:172.21.15.190:6817/1281430772]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:26:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:47 smithi190 ceph-mon[160435]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:26:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:47 smithi190 ceph-mon[160435]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:47 smithi190 ceph-mon[160435]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:47 smithi190 ceph-mon[160435]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:47 smithi190 ceph-mon[160435]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:47 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 50794/206604 objects degraded (24.585%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:48.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:47 smithi190 ceph-mon[160435]: failed to encode map e293 with expected crc 2023-10-01T09:26:48.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:47 smithi190 ceph-mon[160435]: pgmap v777: 89 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 371 op/s; 50419/203946 objects degraded (24.722%); 17144/203946 objects misplaced (8.406%); 3.7 MiB/s, 59 objects/s recovering 2023-10-01T09:26:48.364 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:48.365 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:26:48.393 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:26:48.433 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:26:48.480 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:26:48.480 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:26:48.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:48.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:26:48.510 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:26:48.519 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:26:48.519 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:26:48.519 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:26:48.519 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:26:48.519 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:26:48.520 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:26:48.520 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:26:48.520 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:26:48.593 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:48.709 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:49.161 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:49.172 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:48 smithi138 ceph-mon[185886]: osd.6 [v2:172.21.15.190:6816/1281430772,v1:172.21.15.190:6817/1281430772] boot 2023-10-01T09:26:49.172 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:48 smithi138 ceph-mon[185886]: osdmap e294: 8 total, 8 up, 5 in 2023-10-01T09:26:49.172 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:48 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:26:49.172 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:48 smithi138 ceph-mon[185886]: failed to encode map e294 with expected crc 2023-10-01T09:26:49.172 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:48 smithi138 ceph-mon[185886]: failed to encode map e294 with expected crc 2023-10-01T09:26:49.172 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:48 smithi138 ceph-mon[185886]: failed to encode map e294 with expected crc 2023-10-01T09:26:49.173 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:48 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1029576692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:26:49.173 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:48 smithi138 ceph-mon[192513]: osd.6 [v2:172.21.15.190:6816/1281430772,v1:172.21.15.190:6817/1281430772] boot 2023-10-01T09:26:49.173 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:48 smithi138 ceph-mon[192513]: osdmap e294: 8 total, 8 up, 5 in 2023-10-01T09:26:49.173 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:48 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:26:49.173 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:48 smithi138 ceph-mon[192513]: failed to encode map e294 with expected crc 2023-10-01T09:26:49.173 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:48 smithi138 ceph-mon[192513]: failed to encode map e294 with expected crc 2023-10-01T09:26:49.173 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:48 smithi138 ceph-mon[192513]: failed to encode map e294 with expected crc 2023-10-01T09:26:49.173 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:48 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1029576692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:26:49.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:49.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:49.275 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:26:49.301 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:49.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:48 smithi190 ceph-mon[160435]: osd.6 [v2:172.21.15.190:6816/1281430772,v1:172.21.15.190:6817/1281430772] boot 2023-10-01T09:26:49.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:48 smithi190 ceph-mon[160435]: osdmap e294: 8 total, 8 up, 5 in 2023-10-01T09:26:49.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:48 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T09:26:49.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:48 smithi190 ceph-mon[160435]: failed to encode map e294 with expected crc 2023-10-01T09:26:49.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:48 smithi190 ceph-mon[160435]: failed to encode map e294 with expected crc 2023-10-01T09:26:49.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:48 smithi190 ceph-mon[160435]: failed to encode map e294 with expected crc 2023-10-01T09:26:49.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:48 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1029576692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:26:49.745 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:26:49.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:49.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:26:49.848 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:49.886 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:49.915 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:49.975 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:50.014 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:50.017 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[185886]: failed to encode map e269 with expected crc 2023-10-01T09:26:50.017 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[185886]: failed to encode map e294 with expected crc 2023-10-01T09:26:50.017 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[185886]: failed to encode map e294 with expected crc 2023-10-01T09:26:50.018 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[185886]: failed to encode map e294 with expected crc 2023-10-01T09:26:50.018 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[185886]: failed to encode map e295 with expected crc 2023-10-01T09:26:50.018 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[185886]: osdmap e295: 8 total, 8 up, 5 in 2023-10-01T09:26:50.018 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[185886]: failed to encode map e295 with expected crc 2023-10-01T09:26:50.018 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[185886]: pgmap v780: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 394 op/s; 50057/200955 objects degraded (24.910%); 16867/200955 objects misplaced (8.393%); 4.9 MiB/s, 78 objects/s recovering 2023-10-01T09:26:50.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:50.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:50.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[192513]: failed to encode map e269 with expected crc 2023-10-01T09:26:50.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[192513]: failed to encode map e294 with expected crc 2023-10-01T09:26:50.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[192513]: failed to encode map e294 with expected crc 2023-10-01T09:26:50.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[192513]: failed to encode map e294 with expected crc 2023-10-01T09:26:50.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[192513]: failed to encode map e295 with expected crc 2023-10-01T09:26:50.019 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[192513]: osdmap e295: 8 total, 8 up, 5 in 2023-10-01T09:26:50.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[192513]: failed to encode map e295 with expected crc 2023-10-01T09:26:50.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:49 smithi138 ceph-mon[192513]: pgmap v780: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 394 op/s; 50057/200955 objects degraded (24.910%); 16867/200955 objects misplaced (8.393%); 4.9 MiB/s, 78 objects/s recovering 2023-10-01T09:26:50.119 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:26:50.254 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:50.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:49 smithi190 ceph-mon[160435]: failed to encode map e269 with expected crc 2023-10-01T09:26:50.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:49 smithi190 ceph-mon[160435]: failed to encode map e294 with expected crc 2023-10-01T09:26:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:49 smithi190 ceph-mon[160435]: failed to encode map e294 with expected crc 2023-10-01T09:26:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:49 smithi190 ceph-mon[160435]: failed to encode map e294 with expected crc 2023-10-01T09:26:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:49 smithi190 ceph-mon[160435]: failed to encode map e295 with expected crc 2023-10-01T09:26:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:49 smithi190 ceph-mon[160435]: osdmap e295: 8 total, 8 up, 5 in 2023-10-01T09:26:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:49 smithi190 ceph-mon[160435]: failed to encode map e295 with expected crc 2023-10-01T09:26:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:49 smithi190 ceph-mon[160435]: pgmap v780: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 394 op/s; 50057/200955 objects degraded (24.910%); 16867/200955 objects misplaced (8.393%); 4.9 MiB/s, 78 objects/s recovering 2023-10-01T09:26:50.706 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:26:50.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:50.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:26:50.817 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:50.912 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:51.155 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:50 smithi138 ceph-mon[185886]: failed to encode map e296 with expected crc 2023-10-01T09:26:51.155 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:50 smithi138 ceph-mon[185886]: osdmap e296: 8 total, 8 up, 5 in 2023-10-01T09:26:51.155 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:50 smithi138 ceph-mon[185886]: 7.1 continuing backfill to osd.0 from (264'3017,294'7922] MIN to 294'7922 2023-10-01T09:26:51.155 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:50 smithi138 ceph-mon[185886]: 7.1 continuing backfill to osd.1 from (264'3017,294'7922] MIN to 294'7922 2023-10-01T09:26:51.155 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:50 smithi138 ceph-mon[185886]: 7.c continuing backfill to osd.2 from (264'3025,294'7958] MIN to 294'7958 2023-10-01T09:26:51.156 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:50 smithi138 ceph-mon[185886]: failed to encode map e296 with expected crc 2023-10-01T09:26:51.156 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:50 smithi138 ceph-mon[192513]: failed to encode map e296 with expected crc 2023-10-01T09:26:51.156 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:50 smithi138 ceph-mon[192513]: osdmap e296: 8 total, 8 up, 5 in 2023-10-01T09:26:51.156 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:50 smithi138 ceph-mon[192513]: 7.1 continuing backfill to osd.0 from (264'3017,294'7922] MIN to 294'7922 2023-10-01T09:26:51.156 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:50 smithi138 ceph-mon[192513]: 7.1 continuing backfill to osd.1 from (264'3017,294'7922] MIN to 294'7922 2023-10-01T09:26:51.156 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:50 smithi138 ceph-mon[192513]: 7.c continuing backfill to osd.2 from (264'3025,294'7958] MIN to 294'7958 2023-10-01T09:26:51.156 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:50 smithi138 ceph-mon[192513]: failed to encode map e296 with expected crc 2023-10-01T09:26:51.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:50 smithi190 ceph-mon[160435]: failed to encode map e296 with expected crc 2023-10-01T09:26:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:50 smithi190 ceph-mon[160435]: osdmap e296: 8 total, 8 up, 5 in 2023-10-01T09:26:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:50 smithi190 ceph-mon[160435]: 7.1 continuing backfill to osd.0 from (264'3017,294'7922] MIN to 294'7922 2023-10-01T09:26:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:50 smithi190 ceph-mon[160435]: 7.1 continuing backfill to osd.1 from (264'3017,294'7922] MIN to 294'7922 2023-10-01T09:26:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:50 smithi190 ceph-mon[160435]: 7.c continuing backfill to osd.2 from (264'3025,294'7958] MIN to 294'7958 2023-10-01T09:26:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:50 smithi190 ceph-mon[160435]: failed to encode map e296 with expected crc 2023-10-01T09:26:51.337 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:51.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:51.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:51.447 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:26:51.476 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:51.520 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:26:51.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:51.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:26:51.628 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:26:51.656 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:51.874 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:26:51.985 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:51 smithi138 ceph-mon[185886]: failed to encode map e296 with expected crc 2023-10-01T09:26:51.985 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:51 smithi138 ceph-mon[185886]: pgmap v782: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 394 op/s; 49946/200049 objects degraded (24.967%); 16867/200049 objects misplaced (8.431%); 4.4 MiB/s, 70 objects/s recovering 2023-10-01T09:26:52.175 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:26:52.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:52.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:26:52.265 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:51 smithi138 ceph-mon[192513]: failed to encode map e296 with expected crc 2023-10-01T09:26:52.265 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:51 smithi138 ceph-mon[192513]: pgmap v782: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 394 op/s; 49946/200049 objects degraded (24.967%); 16867/200049 objects misplaced (8.431%); 4.4 MiB/s, 70 objects/s recovering 2023-10-01T09:26:52.297 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:52.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:51 smithi190 ceph-mon[160435]: failed to encode map e296 with expected crc 2023-10-01T09:26:52.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:51 smithi190 ceph-mon[160435]: pgmap v782: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 73 active+clean; 4.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 394 op/s; 49946/200049 objects degraded (24.967%); 16867/200049 objects misplaced (8.431%); 4.4 MiB/s, 70 objects/s recovering 2023-10-01T09:26:52.326 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:52.376 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:52.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:52.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:52.483 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:52.510 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:52.557 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:52.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:52.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:52.664 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:52.690 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:52.731 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:52.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:52.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:52.838 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:26:52.866 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:52.914 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:26:52.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:52.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:26:53.019 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2] out_osds: [5, 6, 4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2023-10-01T09:26:53.019 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:26:53.019 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-10-01T09:26:53.019 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd in 5 2023-10-01T09:26:53.022 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:26:53.173 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:53.179 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:52 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 49946/200049 objects degraded (24.967%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:53.180 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:52 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 49946/200049 objects degraded (24.967%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:53.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:52 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 49946/200049 objects degraded (24.967%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:53.386 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:26:53.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:53.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:26:53.495 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:53.524 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:53.572 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:53.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:53.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:53.680 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:26:53.709 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:53.753 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:26:53.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:53.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:26:53.863 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:53.892 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:53.907 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:53.936 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:53.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:53.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:54.043 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:54.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:54 smithi138 ceph-mon[185886]: pgmap v783: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.0 GiB data, 11 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 386 op/s; 43022/196986 objects degraded (21.840%); 23036/196986 objects misplaced (11.694%); 5.0 MiB/s, 79 objects/s recovering 2023-10-01T09:26:54.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:54 smithi138 ceph-mon[192513]: pgmap v783: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.0 GiB data, 11 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 386 op/s; 43022/196986 objects degraded (21.840%); 23036/196986 objects misplaced (11.694%); 5.0 MiB/s, 79 objects/s recovering 2023-10-01T09:26:54.201 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:26:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:26:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:26:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:26:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95302 6.2 6:4510ce89:::benchmark_data_smithi138_307751_object95301:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-01T09:26:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:03.371974+0000", 2023-10-01T09:26:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 350.81859042299999, 2023-10-01T09:26:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.26466727, 2023-10-01T09:26:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95302 2023-10-01T09:26:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.371974+0000", 2023-10-01T09:26:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.371974+0000", 2023-10-01T09:26:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.371972+0000", 2023-10-01T09:26:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:26:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.371998+0000", 2023-10-01T09:26:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5545999999999999e-05 2023-10-01T09:26:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.372013+0000", 2023-10-01T09:26:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5336e-05 2023-10-01T09:26:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.372022+0000", 2023-10-01T09:26:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8939999999999999e-06 2023-10-01T09:26:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.372083+0000", 2023-10-01T09:26:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1211999999999996e-05 2023-10-01T09:26:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.372216+0000", 2023-10-01T09:26:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013330899999999999 2023-10-01T09:26:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.372312+0000", 2023-10-01T09:26:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5858000000000003e-05 2023-10-01T09:26:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.375920+0000", 2023-10-01T09:26:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036081450000000001 2023-10-01T09:26:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.376644+0000", 2023-10-01T09:26:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000723874 2023-10-01T09:26:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.636555+0000", 2023-10-01T09:26:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.259910734 2023-10-01T09:26:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.636600+0000", 2023-10-01T09:26:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5272999999999998e-05 2023-10-01T09:26:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.636641+0000", 2023-10-01T09:26:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0865999999999998e-05 2023-10-01T09:26:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95322 6.2 6:456699d3:::benchmark_data_smithi138_307751_object95321:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-01T09:26:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:03.430718+0000", 2023-10-01T09:26:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 350.75984603400002, 2023-10-01T09:26:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.255276716000001, 2023-10-01T09:26:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95322 2023-10-01T09:26:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430718+0000", 2023-10-01T09:26:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430718+0000", 2023-10-01T09:26:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430717+0000", 2023-10-01T09:26:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:26:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430737+0000", 2023-10-01T09:26:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9941000000000001e-05 2023-10-01T09:26:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430746+0000", 2023-10-01T09:26:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2299999999999997e-06 2023-10-01T09:26:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430752+0000", 2023-10-01T09:26:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4999999999999999e-06 2023-10-01T09:26:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430796+0000", 2023-10-01T09:26:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3548999999999999e-05 2023-10-01T09:26:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.430930+0000", 2023-10-01T09:26:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013467199999999999 2023-10-01T09:26:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.431086+0000", 2023-10-01T09:26:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015616999999999999 2023-10-01T09:26:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.434311+0000", 2023-10-01T09:26:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032241959999999999 2023-10-01T09:26:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.434323+0000", 2023-10-01T09:26:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2476999999999999e-05 2023-10-01T09:26:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.685898+0000", 2023-10-01T09:26:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.251575172999999 2023-10-01T09:26:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.685952+0000", 2023-10-01T09:26:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4243999999999998e-05 2023-10-01T09:26:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.685995+0000", 2023-10-01T09:26:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2543999999999998e-05 2023-10-01T09:26:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95334 6.2 6:42c96aa7:::benchmark_data_smithi138_307751_object95333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-01T09:26:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:03.454910+0000", 2023-10-01T09:26:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 350.73565432999999, 2023-10-01T09:26:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.238695902, 2023-10-01T09:26:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95334 2023-10-01T09:26:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.454910+0000", 2023-10-01T09:26:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.454910+0000", 2023-10-01T09:26:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.454909+0000", 2023-10-01T09:26:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:26:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.454930+0000", 2023-10-01T09:26:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1038000000000001e-05 2023-10-01T09:26:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.454940+0000", 2023-10-01T09:26:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8579999999999995e-06 2023-10-01T09:26:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.454946+0000", 2023-10-01T09:26:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0120000000000002e-06 2023-10-01T09:26:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.454989+0000", 2023-10-01T09:26:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3510000000000002e-05 2023-10-01T09:26:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.455086+0000", 2023-10-01T09:26:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6867000000000006e-05 2023-10-01T09:26:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.455174+0000", 2023-10-01T09:26:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7441000000000005e-05 2023-10-01T09:26:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.457693+0000", 2023-10-01T09:26:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025194079999999999 2023-10-01T09:26:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.506165+0000", 2023-10-01T09:26:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.048471805 2023-10-01T09:26:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.693535+0000", 2023-10-01T09:26:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.187370025 2023-10-01T09:26:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.693566+0000", 2023-10-01T09:26:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1559000000000001e-05 2023-10-01T09:26:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.693606+0000", 2023-10-01T09:26:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9496999999999998e-05 2023-10-01T09:26:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95348 6.2 6:46ec8a63:::benchmark_data_smithi138_307751_object95347:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-01T09:26:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:03.513565+0000", 2023-10-01T09:26:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 350.67699900399998, 2023-10-01T09:26:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.186301394000001, 2023-10-01T09:26:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95348 2023-10-01T09:26:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513565+0000", 2023-10-01T09:26:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513565+0000", 2023-10-01T09:26:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513564+0000", 2023-10-01T09:26:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513584+0000", 2023-10-01T09:26:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0169e-05 2023-10-01T09:26:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513596+0000", 2023-10-01T09:26:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1834000000000001e-05 2023-10-01T09:26:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513601+0000", 2023-10-01T09:26:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3430000000000003e-06 2023-10-01T09:26:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513648+0000", 2023-10-01T09:26:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6869999999999997e-05 2023-10-01T09:26:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513788+0000", 2023-10-01T09:26:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013930999999999999 2023-10-01T09:26:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.513937+0000", 2023-10-01T09:26:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014913400000000001 2023-10-01T09:26:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.516816+0000", 2023-10-01T09:26:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028788429999999999 2023-10-01T09:26:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.517680+0000", 2023-10-01T09:26:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00086450399999999999 2023-10-01T09:26:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.699800+0000", 2023-10-01T09:26:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.182119651000001 2023-10-01T09:26:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.699830+0000", 2023-10-01T09:26:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0378e-05 2023-10-01T09:26:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.699867+0000", 2023-10-01T09:26:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6563000000000002e-05 2023-10-01T09:26:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95362 6.2 6:4eb5c0df:::benchmark_data_smithi138_307751_object95361:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-01T09:26:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:03.544575+0000", 2023-10-01T09:26:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 350.645988929, 2023-10-01T09:26:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.171384007, 2023-10-01T09:26:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95362 2023-10-01T09:26:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544575+0000", 2023-10-01T09:26:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544575+0000", 2023-10-01T09:26:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544574+0000", 2023-10-01T09:26:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544594+0000", 2023-10-01T09:26:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0352000000000001e-05 2023-10-01T09:26:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544605+0000", 2023-10-01T09:26:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0407e-05 2023-10-01T09:26:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544611+0000", 2023-10-01T09:26:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.02e-06 2023-10-01T09:26:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544646+0000", 2023-10-01T09:26:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4696999999999997e-05 2023-10-01T09:26:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544730+0000", 2023-10-01T09:26:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4630999999999997e-05 2023-10-01T09:26:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.544817+0000", 2023-10-01T09:26:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6589999999999996e-05 2023-10-01T09:26:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.547214+0000", 2023-10-01T09:26:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002397181 2023-10-01T09:26:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.548157+0000", 2023-10-01T09:26:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000942646 2023-10-01T09:26:54.237 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.237 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.237 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.237 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.715877+0000", 2023-10-01T09:26:54.237 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.167720721 2023-10-01T09:26:54.237 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.237 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.237 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.238 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.715933+0000", 2023-10-01T09:26:54.238 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5287000000000001e-05 2023-10-01T09:26:54.238 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.238 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.238 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.239 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.715959+0000", 2023-10-01T09:26:54.239 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6727e-05 2023-10-01T09:26:54.239 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.239 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.239 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.239 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.239 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.239 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:95366 6.2 6:4a76bb19:::benchmark_data_smithi138_307751_object95365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e209)", 2023-10-01T09:26:54.239 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:03.558126+0000", 2023-10-01T09:26:54.239 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 350.63243861400002, 2023-10-01T09:26:54.239 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.191781848, 2023-10-01T09:26:54.239 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.240 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.240 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.240 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.240 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.240 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 95366 2023-10-01T09:26:54.240 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.240 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.240 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.240 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558126+0000", 2023-10-01T09:26:54.240 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.240 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.241 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.241 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558126+0000", 2023-10-01T09:26:54.241 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.241 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.241 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.241 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558124+0000", 2023-10-01T09:26:54.241 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:54.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.241 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.241 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558147+0000", 2023-10-01T09:26:54.242 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2473999999999999e-05 2023-10-01T09:26:54.242 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.242 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558157+0000", 2023-10-01T09:26:54.242 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0547000000000001e-05 2023-10-01T09:26:54.242 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.242 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558164+0000", 2023-10-01T09:26:54.242 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1979999999999998e-06 2023-10-01T09:26:54.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.243 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.243 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558200+0000", 2023-10-01T09:26:54.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6804000000000002e-05 2023-10-01T09:26:54.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.243 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.243 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558283+0000", 2023-10-01T09:26:54.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2905999999999995e-05 2023-10-01T09:26:54.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:54.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.558365+0000", 2023-10-01T09:26:54.244 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1786000000000006e-05 2023-10-01T09:26:54.244 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.560751+0000", 2023-10-01T09:26:54.244 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023859520000000002 2023-10-01T09:26:54.244 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:03.561038+0000", 2023-10-01T09:26:54.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00028716700000000001 2023-10-01T09:26:54.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.245 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.245 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.245 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.749799+0000", 2023-10-01T09:26:54.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 11.188761356000001 2023-10-01T09:26:54.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.245 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.245 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.245 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.749860+0000", 2023-10-01T09:26:54.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0207999999999997e-05 2023-10-01T09:26:54.245 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.246 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.246 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.246 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:14.749908+0000", 2023-10-01T09:26:54.246 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7851e-05 2023-10-01T09:26:54.246 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.246 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.246 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.246 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.246 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.246 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96044 6.f 6:f4455201:::benchmark_data_smithi138_307751_object675:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-10-01T09:26:54.246 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:16.567948+0000", 2023-10-01T09:26:54.246 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 337.62261581500002, 2023-10-01T09:26:54.246 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96977755200000004, 2023-10-01T09:26:54.247 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.247 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.247 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.247 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.247 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.247 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96044 2023-10-01T09:26:54.247 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.247 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.247 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.247 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.247 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.567948+0000", 2023-10-01T09:26:54.247 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.248 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.248 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.248 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.567948+0000", 2023-10-01T09:26:54.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.248 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.248 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.248 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.567947+0000", 2023-10-01T09:26:54.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:54.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.248 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.249 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.567952+0000", 2023-10-01T09:26:54.249 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.045e-06 2023-10-01T09:26:54.249 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.249 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.567965+0000", 2023-10-01T09:26:54.249 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2472e-05 2023-10-01T09:26:54.249 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.249 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.567972+0000", 2023-10-01T09:26:54.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4449999999999999e-06 2023-10-01T09:26:54.250 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.250 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568042+0000", 2023-10-01T09:26:54.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9713000000000002e-05 2023-10-01T09:26:54.250 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.250 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:54.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568055+0000", 2023-10-01T09:26:54.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2795e-05 2023-10-01T09:26:54.251 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.251 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.251 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.251 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.530163+0000", 2023-10-01T09:26:54.251 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96210800900000004 2023-10-01T09:26:54.251 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.251 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.251 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.251 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.530318+0000", 2023-10-01T09:26:54.251 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000154797 2023-10-01T09:26:54.251 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.251 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.252 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:54.252 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.530435+0000", 2023-10-01T09:26:54.252 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117372 2023-10-01T09:26:54.252 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.252 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.252 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.252 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537484+0000", 2023-10-01T09:26:54.252 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.007049484 2023-10-01T09:26:54.252 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.252 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.252 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.252 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537525+0000", 2023-10-01T09:26:54.253 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0914999999999997e-05 2023-10-01T09:26:54.253 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.253 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.253 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.253 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537651+0000", 2023-10-01T09:26:54.253 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012556899999999999 2023-10-01T09:26:54.253 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.253 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.253 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.253 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537686+0000", 2023-10-01T09:26:54.253 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5012999999999998e-05 2023-10-01T09:26:54.253 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.254 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.254 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.254 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537726+0000", 2023-10-01T09:26:54.254 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0173000000000003e-05 2023-10-01T09:26:54.254 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.254 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.254 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.254 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.254 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.254 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96045 6.f 6:faa45668:::benchmark_data_smithi138_307751_object676:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-10-01T09:26:54.254 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:16.568148+0000", 2023-10-01T09:26:54.254 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 337.62241623199998, 2023-10-01T09:26:54.255 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96969213099999996, 2023-10-01T09:26:54.255 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.255 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.255 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.255 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.255 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.255 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96045 2023-10-01T09:26:54.255 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.255 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.255 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.255 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.255 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568148+0000", 2023-10-01T09:26:54.256 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.256 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568148+0000", 2023-10-01T09:26:54.256 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.256 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568147+0000", 2023-10-01T09:26:54.256 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:54.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.257 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568151+0000", 2023-10-01T09:26:54.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0049999999999998e-06 2023-10-01T09:26:54.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.257 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.257 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568161+0000", 2023-10-01T09:26:54.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0219999999999993e-06 2023-10-01T09:26:54.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.257 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.257 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568165+0000", 2023-10-01T09:26:54.258 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4880000000000001e-06 2023-10-01T09:26:54.258 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.258 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.258 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.258 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568209+0000", 2023-10-01T09:26:54.258 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4218999999999998e-05 2023-10-01T09:26:54.258 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.258 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.258 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:54.258 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.568215+0000", 2023-10-01T09:26:54.258 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6579999999999999e-06 2023-10-01T09:26:54.258 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.259 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.533101+0000", 2023-10-01T09:26:54.259 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96488601799999996 2023-10-01T09:26:54.259 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.259 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.533256+0000", 2023-10-01T09:26:54.259 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000155496 2023-10-01T09:26:54.259 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:54.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.533361+0000", 2023-10-01T09:26:54.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000104149 2023-10-01T09:26:54.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537559+0000", 2023-10-01T09:26:54.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041986920000000004 2023-10-01T09:26:54.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537618+0000", 2023-10-01T09:26:54.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8689e-05 2023-10-01T09:26:54.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537790+0000", 2023-10-01T09:26:54.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000172199 2023-10-01T09:26:54.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537817+0000", 2023-10-01T09:26:54.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6942999999999999e-05 2023-10-01T09:26:54.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.262 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.262 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537840+0000", 2023-10-01T09:26:54.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3156999999999999e-05 2023-10-01T09:26:54.262 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.262 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.262 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.262 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.262 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96049 6.f 6:fa48c08b:::benchmark_data_smithi138_307751_object680:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-10-01T09:26:54.262 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:16.572299+0000", 2023-10-01T09:26:54.262 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 337.618265044, 2023-10-01T09:26:54.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97246542199999997, 2023-10-01T09:26:54.263 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.263 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.263 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.263 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.263 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.263 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96049 2023-10-01T09:26:54.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.263 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572299+0000", 2023-10-01T09:26:54.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572299+0000", 2023-10-01T09:26:54.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572297+0000", 2023-10-01T09:26:54.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:26:54.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572305+0000", 2023-10-01T09:26:54.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2169999999999994e-06 2023-10-01T09:26:54.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572324+0000", 2023-10-01T09:26:54.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8539999999999999e-05 2023-10-01T09:26:54.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572333+0000", 2023-10-01T09:26:54.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8179999999999993e-06 2023-10-01T09:26:54.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572424+0000", 2023-10-01T09:26:54.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0913000000000002e-05 2023-10-01T09:26:54.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:54.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572438+0000", 2023-10-01T09:26:54.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4127e-05 2023-10-01T09:26:54.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.535622+0000", 2023-10-01T09:26:54.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96318426000000001 2023-10-01T09:26:54.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.535741+0000", 2023-10-01T09:26:54.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011915499999999999 2023-10-01T09:26:54.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:54.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.535863+0000", 2023-10-01T09:26:54.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121599 2023-10-01T09:26:54.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537848+0000", 2023-10-01T09:26:54.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0019855110000000001 2023-10-01T09:26:54.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.541762+0000", 2023-10-01T09:26:54.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039140620000000003 2023-10-01T09:26:54.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544687+0000", 2023-10-01T09:26:54.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029246369999999999 2023-10-01T09:26:54.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544722+0000", 2023-10-01T09:26:54.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4660999999999999e-05 2023-10-01T09:26:54.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544765+0000", 2023-10-01T09:26:54.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3013000000000003e-05 2023-10-01T09:26:54.270 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.270 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.270 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.270 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96051 6.f 6:fad7202f:::benchmark_data_smithi138_307751_object682:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-10-01T09:26:54.270 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:16.572358+0000", 2023-10-01T09:26:54.270 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 337.618206547, 2023-10-01T09:26:54.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97833788099999996, 2023-10-01T09:26:54.271 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.271 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.271 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.271 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.271 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.271 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96051 2023-10-01T09:26:54.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.271 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.271 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.271 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.271 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572358+0000", 2023-10-01T09:26:54.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.272 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.272 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572358+0000", 2023-10-01T09:26:54.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.272 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.272 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572357+0000", 2023-10-01T09:26:54.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:54.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572360+0000", 2023-10-01T09:26:54.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0809999999999998e-06 2023-10-01T09:26:54.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572369+0000", 2023-10-01T09:26:54.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5790000000000004e-06 2023-10-01T09:26:54.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572375+0000", 2023-10-01T09:26:54.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.037e-06 2023-10-01T09:26:54.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572466+0000", 2023-10-01T09:26:54.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1169999999999996e-05 2023-10-01T09:26:54.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:54.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.572477+0000", 2023-10-01T09:26:54.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0981e-05 2023-10-01T09:26:54.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.537893+0000", 2023-10-01T09:26:54.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96541566499999998 2023-10-01T09:26:54.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.538017+0000", 2023-10-01T09:26:54.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000124315 2023-10-01T09:26:54.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:54.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.538129+0000", 2023-10-01T09:26:54.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112355 2023-10-01T09:26:54.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544653+0000", 2023-10-01T09:26:54.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0065235960000000004 2023-10-01T09:26:54.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544827+0000", 2023-10-01T09:26:54.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017388700000000001 2023-10-01T09:26:54.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.550629+0000", 2023-10-01T09:26:54.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058019609999999996 2023-10-01T09:26:54.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.550659+0000", 2023-10-01T09:26:54.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0156999999999999e-05 2023-10-01T09:26:54.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.550696+0000", 2023-10-01T09:26:54.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6636e-05 2023-10-01T09:26:54.278 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.278 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.278 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.278 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:96072 6.f 6:fdc788ea:::benchmark_data_smithi138_307751_object703:head [delete] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-10-01T09:26:54.278 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:16.589538+0000", 2023-10-01T09:26:54.279 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 337.60102659699999, 2023-10-01T09:26:54.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96461343300000002, 2023-10-01T09:26:54.279 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.279 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.279 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.279 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.279 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.279 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96072 2023-10-01T09:26:54.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.279 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589538+0000", 2023-10-01T09:26:54.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.280 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.280 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589538+0000", 2023-10-01T09:26:54.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.280 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.280 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589535+0000", 2023-10-01T09:26:54.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:26:54.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589544+0000", 2023-10-01T09:26:54.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5469999999999997e-06 2023-10-01T09:26:54.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589567+0000", 2023-10-01T09:26:54.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2816999999999999e-05 2023-10-01T09:26:54.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589578+0000", 2023-10-01T09:26:54.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1002000000000001e-05 2023-10-01T09:26:54.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.282 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589639+0000", 2023-10-01T09:26:54.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1264000000000002e-05 2023-10-01T09:26:54.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.282 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:54.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:16.589654+0000", 2023-10-01T09:26:54.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5258e-05 2023-10-01T09:26:54.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.539758+0000", 2023-10-01T09:26:54.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95010384400000003 2023-10-01T09:26:54.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.539881+0000", 2023-10-01T09:26:54.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012311900000000001 2023-10-01T09:26:54.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:54.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.540000+0000", 2023-10-01T09:26:54.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011898800000000001 2023-10-01T09:26:54.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.544837+0000", 2023-10-01T09:26:54.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048369520000000003 2023-10-01T09:26:54.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.547696+0000", 2023-10-01T09:26:54.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002858972 2023-10-01T09:26:54.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.554057+0000", 2023-10-01T09:26:54.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0063608279999999998 2023-10-01T09:26:54.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.554113+0000", 2023-10-01T09:26:54.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5835999999999999e-05 2023-10-01T09:26:54.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:17.554151+0000", 2023-10-01T09:26:54.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8349000000000002e-05 2023-10-01T09:26:54.286 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.286 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.286 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.286 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109321 6.2 6:4665d031:::benchmark_data_smithi138_307751_object13952:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-01T09:26:54.286 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.601615+0000", 2023-10-01T09:26:54.287 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 314.58894889700002, 2023-10-01T09:26:54.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97938183099999998, 2023-10-01T09:26:54.287 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.287 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:26:54.287 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.287 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.287 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.287 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109321 2023-10-01T09:26:54.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.287 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601615+0000", 2023-10-01T09:26:54.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601615+0000", 2023-10-01T09:26:54.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.288 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.288 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601613+0000", 2023-10-01T09:26:54.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:26:54.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.289 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601623+0000", 2023-10-01T09:26:54.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3850000000000008e-06 2023-10-01T09:26:54.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.289 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601643+0000", 2023-10-01T09:26:54.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0353e-05 2023-10-01T09:26:54.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601654+0000", 2023-10-01T09:26:54.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1279e-05 2023-10-01T09:26:54.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.823938+0000", 2023-10-01T09:26:54.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.222283589 2023-10-01T09:26:54.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:54.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.823948+0000", 2023-10-01T09:26:54.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7529999999999996e-06 2023-10-01T09:26:54.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.580927+0000", 2023-10-01T09:26:54.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.75697944500000003 2023-10-01T09:26:54.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.580997+0000", 2023-10-01T09:26:54.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0178000000000005e-05 2023-10-01T09:26:54.291 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.292 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.292 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.292 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109328 6.12 6:4e4d45a1:::benchmark_data_smithi138_307751_object13959:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-01T09:26:54.292 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.601663+0000", 2023-10-01T09:26:54.292 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 314.588901609, 2023-10-01T09:26:54.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0018865050000001, 2023-10-01T09:26:54.292 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.292 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.292 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.292 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.293 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.293 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109328 2023-10-01T09:26:54.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.293 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601663+0000", 2023-10-01T09:26:54.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601663+0000", 2023-10-01T09:26:54.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601662+0000", 2023-10-01T09:26:54.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:54.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601665+0000", 2023-10-01T09:26:54.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.083e-06 2023-10-01T09:26:54.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.295 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601673+0000", 2023-10-01T09:26:54.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7870000000000002e-06 2023-10-01T09:26:54.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.295 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601679+0000", 2023-10-01T09:26:54.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.519e-06 2023-10-01T09:26:54.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.823969+0000", 2023-10-01T09:26:54.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.222290083 2023-10-01T09:26:54.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:54.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.823974+0000", 2023-10-01T09:26:54.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.626e-06 2023-10-01T09:26:54.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.594859+0000", 2023-10-01T09:26:54.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.77088483600000002 2023-10-01T09:26:54.297 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.297 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.297 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.297 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.594988+0000", 2023-10-01T09:26:54.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012879300000000001 2023-10-01T09:26:54.297 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.297 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.297 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:54.297 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.595107+0000", 2023-10-01T09:26:54.297 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118781 2023-10-01T09:26:54.298 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.298 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.298 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.298 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.600053+0000", 2023-10-01T09:26:54.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00494627 2023-10-01T09:26:54.298 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.298 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.298 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.298 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.602542+0000", 2023-10-01T09:26:54.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024894000000000001 2023-10-01T09:26:54.298 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.603447+0000", 2023-10-01T09:26:54.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00090488299999999999 2023-10-01T09:26:54.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.603501+0000", 2023-10-01T09:26:54.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4141000000000002e-05 2023-10-01T09:26:54.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.603549+0000", 2023-10-01T09:26:54.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7805000000000001e-05 2023-10-01T09:26:54.300 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.300 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.300 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.300 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109334 6.2 6:431726c9:::benchmark_data_smithi138_307751_object13965:head [delete] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-10-01T09:26:54.300 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.641618+0000", 2023-10-01T09:26:54.300 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 314.54894607400001, 2023-10-01T09:26:54.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95538988400000002, 2023-10-01T09:26:54.300 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.301 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.301 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.301 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.301 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.301 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109334 2023-10-01T09:26:54.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.301 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.301 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.301 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.301 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.641618+0000", 2023-10-01T09:26:54.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.302 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.302 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.641618+0000", 2023-10-01T09:26:54.302 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.302 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.302 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.302 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.641616+0000", 2023-10-01T09:26:54.302 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999971 2023-10-01T09:26:54.302 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.302 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.641631+0000", 2023-10-01T09:26:54.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5430999999999998e-05 2023-10-01T09:26:54.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.303 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.303 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.641661+0000", 2023-10-01T09:26:54.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0103e-05 2023-10-01T09:26:54.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.303 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.303 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.641677+0000", 2023-10-01T09:26:54.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5546999999999999e-05 2023-10-01T09:26:54.304 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.304 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.304 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.304 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.823994+0000", 2023-10-01T09:26:54.304 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18231736500000001 2023-10-01T09:26:54.304 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.304 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.304 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:54.304 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.824001+0000", 2023-10-01T09:26:54.304 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6989999999999997e-06 2023-10-01T09:26:54.304 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.305 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.305 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.584131+0000", 2023-10-01T09:26:54.305 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.76013001700000005 2023-10-01T09:26:54.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.305 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.305 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.584664+0000", 2023-10-01T09:26:54.305 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00053300799999999996 2023-10-01T09:26:54.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.305 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:26:54.306 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.584771+0000", 2023-10-01T09:26:54.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107014 2023-10-01T09:26:54.306 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.306 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.306 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.306 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.596837+0000", 2023-10-01T09:26:54.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01206572 2023-10-01T09:26:54.306 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.306 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.306 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.306 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.596902+0000", 2023-10-01T09:26:54.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5140000000000003e-05 2023-10-01T09:26:54.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.307 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.596931+0000", 2023-10-01T09:26:54.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9320999999999999e-05 2023-10-01T09:26:54.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.307 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.596974+0000", 2023-10-01T09:26:54.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3328000000000003e-05 2023-10-01T09:26:54.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.597008+0000", 2023-10-01T09:26:54.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3827e-05 2023-10-01T09:26:54.308 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.308 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.308 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.308 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178787 6.18 6:189b3dbf:::benchmark_data_smithi138_307751_object83418:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:26:54.308 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612642+0000", 2023-10-01T09:26:54.308 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 223.57792181100001, 2023-10-01T09:26:54.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4263684350000001, 2023-10-01T09:26:54.309 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.309 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.309 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.309 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.309 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.309 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178787 2023-10-01T09:26:54.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.309 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612642+0000", 2023-10-01T09:26:54.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612642+0000", 2023-10-01T09:26:54.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.310 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.310 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612641+0000", 2023-10-01T09:26:54.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:54.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612647+0000", 2023-10-01T09:26:54.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5709999999999997e-06 2023-10-01T09:26:54.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612658+0000", 2023-10-01T09:26:54.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1528e-05 2023-10-01T09:26:54.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612668+0000", 2023-10-01T09:26:54.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0349999999999999e-05 2023-10-01T09:26:54.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614052+0000", 2023-10-01T09:26:54.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001383188 2023-10-01T09:26:54.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:54.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614062+0000", 2023-10-01T09:26:54.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0437e-05 2023-10-01T09:26:54.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.028763+0000", 2023-10-01T09:26:54.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.414701177 2023-10-01T09:26:54.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.028872+0000", 2023-10-01T09:26:54.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109073 2023-10-01T09:26:54.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:26:54.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.029307+0000", 2023-10-01T09:26:54.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00043503699999999999 2023-10-01T09:26:54.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.032478+0000", 2023-10-01T09:26:54.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031704839999999999 2023-10-01T09:26:54.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.032534+0000", 2023-10-01T09:26:54.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6481999999999999e-05 2023-10-01T09:26:54.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.038405+0000", 2023-10-01T09:26:54.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058706319999999998 2023-10-01T09:26:54.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.038465+0000", 2023-10-01T09:26:54.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0176000000000001e-05 2023-10-01T09:26:54.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.316 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.316 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039011+0000", 2023-10-01T09:26:54.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00054578000000000005 2023-10-01T09:26:54.316 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.316 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.316 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.316 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178807 6.18 6:1f8d531c:::benchmark_data_smithi138_307751_object83438:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:26:54.316 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612681+0000", 2023-10-01T09:26:54.317 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 223.577883732, 2023-10-01T09:26:54.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4300119259999999, 2023-10-01T09:26:54.317 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.317 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.317 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.317 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.317 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.317 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178807 2023-10-01T09:26:54.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.317 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612681+0000", 2023-10-01T09:26:54.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.318 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612681+0000", 2023-10-01T09:26:54.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.318 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612680+0000", 2023-10-01T09:26:54.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:54.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612683+0000", 2023-10-01T09:26:54.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0709999999999999e-06 2023-10-01T09:26:54.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612690+0000", 2023-10-01T09:26:54.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.613e-06 2023-10-01T09:26:54.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612695+0000", 2023-10-01T09:26:54.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4090000000000002e-06 2023-10-01T09:26:54.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614084+0000", 2023-10-01T09:26:54.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001388331 2023-10-01T09:26:54.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:54.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614090+0000", 2023-10-01T09:26:54.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8630000000000002e-06 2023-10-01T09:26:54.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.032562+0000", 2023-10-01T09:26:54.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.41847178 2023-10-01T09:26:54.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.032737+0000", 2023-10-01T09:26:54.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000174477 2023-10-01T09:26:54.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:26:54.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.033304+0000", 2023-10-01T09:26:54.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00056714600000000003 2023-10-01T09:26:54.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039063+0000", 2023-10-01T09:26:54.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057587920000000004 2023-10-01T09:26:54.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039071+0000", 2023-10-01T09:26:54.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5920000000000001e-06 2023-10-01T09:26:54.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.323 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042139+0000", 2023-10-01T09:26:54.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030678440000000001 2023-10-01T09:26:54.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.324 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042194+0000", 2023-10-01T09:26:54.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5306999999999997e-05 2023-10-01T09:26:54.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.324 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.324 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042693+0000", 2023-10-01T09:26:54.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00049818099999999999 2023-10-01T09:26:54.324 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.324 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.324 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.325 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178818 6.18 6:19fb958b:::benchmark_data_smithi138_307751_object83449:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:26:54.325 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612700+0000", 2023-10-01T09:26:54.325 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 223.577864081, 2023-10-01T09:26:54.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.430431437, 2023-10-01T09:26:54.325 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.325 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.325 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.325 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.325 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.325 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178818 2023-10-01T09:26:54.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.326 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612700+0000", 2023-10-01T09:26:54.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612700+0000", 2023-10-01T09:26:54.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612700+0000", 2023-10-01T09:26:54.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:54.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612703+0000", 2023-10-01T09:26:54.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7920000000000001e-06 2023-10-01T09:26:54.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612708+0000", 2023-10-01T09:26:54.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4990000000000002e-06 2023-10-01T09:26:54.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612713+0000", 2023-10-01T09:26:54.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8509999999999998e-06 2023-10-01T09:26:54.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.328 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.328 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614110+0000", 2023-10-01T09:26:54.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013967039999999999 2023-10-01T09:26:54.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:54.329 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614116+0000", 2023-10-01T09:26:54.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9349999999999999e-06 2023-10-01T09:26:54.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.329 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.035619+0000", 2023-10-01T09:26:54.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.421503172 2023-10-01T09:26:54.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.035767+0000", 2023-10-01T09:26:54.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014777900000000001 2023-10-01T09:26:54.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:26:54.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.036191+0000", 2023-10-01T09:26:54.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000424206 2023-10-01T09:26:54.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042058+0000", 2023-10-01T09:26:54.331 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:54 smithi190 ceph-mon[160435]: pgmap v783: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.0 GiB data, 11 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 386 op/s; 43022/196986 objects degraded (21.840%); 23036/196986 objects misplaced (11.694%); 5.0 MiB/s, 79 objects/s recovering 2023-10-01T09:26:54.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005867227 2023-10-01T09:26:54.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042070+0000", 2023-10-01T09:26:54.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2122e-05 2023-10-01T09:26:54.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042747+0000", 2023-10-01T09:26:54.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00067660799999999998 2023-10-01T09:26:54.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042782+0000", 2023-10-01T09:26:54.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5441000000000002e-05 2023-10-01T09:26:54.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.333 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043132+0000", 2023-10-01T09:26:54.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00034945799999999997 2023-10-01T09:26:54.333 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.333 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.333 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.334 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178848 6.18 6:1e688e02:::benchmark_data_smithi138_307751_object83479:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:26:54.334 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612851+0000", 2023-10-01T09:26:54.334 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 223.57771352899999, 2023-10-01T09:26:54.334 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.436053759, 2023-10-01T09:26:54.334 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.334 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.334 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.334 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.335 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.335 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178848 2023-10-01T09:26:54.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.335 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612851+0000", 2023-10-01T09:26:54.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612851+0000", 2023-10-01T09:26:54.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612850+0000", 2023-10-01T09:26:54.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:54.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612854+0000", 2023-10-01T09:26:54.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5580000000000001e-06 2023-10-01T09:26:54.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612862+0000", 2023-10-01T09:26:54.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9699999999999999e-06 2023-10-01T09:26:54.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612869+0000", 2023-10-01T09:26:54.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0539999999999997e-06 2023-10-01T09:26:54.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614135+0000", 2023-10-01T09:26:54.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012659559999999999 2023-10-01T09:26:54.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:54.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614141+0000", 2023-10-01T09:26:54.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2859999999999996e-06 2023-10-01T09:26:54.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039094+0000", 2023-10-01T09:26:54.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.424953428 2023-10-01T09:26:54.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039262+0000", 2023-10-01T09:26:54.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000167541 2023-10-01T09:26:54.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:26:54.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039703+0000", 2023-10-01T09:26:54.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000441462 2023-10-01T09:26:54.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043192+0000", 2023-10-01T09:26:54.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034885319999999999 2023-10-01T09:26:54.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043218+0000", 2023-10-01T09:26:54.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5754000000000001e-05 2023-10-01T09:26:54.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.048340+0000", 2023-10-01T09:26:54.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051218219999999998 2023-10-01T09:26:54.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.341 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.048383+0000", 2023-10-01T09:26:54.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3179000000000001e-05 2023-10-01T09:26:54.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.341 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.048905+0000", 2023-10-01T09:26:54.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00052180900000000003 2023-10-01T09:26:54.342 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.342 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.342 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.342 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178861 6.18 6:1fa58ed3:::benchmark_data_smithi138_307751_object83492:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:26:54.342 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612874+0000", 2023-10-01T09:26:54.342 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 223.57769041399999, 2023-10-01T09:26:54.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4449525510000001, 2023-10-01T09:26:54.343 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.343 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:26:54.343 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.343 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:26:54.343 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:26:54.343 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178861 2023-10-01T09:26:54.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.343 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612874+0000", 2023-10-01T09:26:54.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612874+0000", 2023-10-01T09:26:54.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612874+0000", 2023-10-01T09:26:54.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:26:54.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612876+0000", 2023-10-01T09:26:54.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4380000000000002e-06 2023-10-01T09:26:54.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612882+0000", 2023-10-01T09:26:54.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1179999999999997e-06 2023-10-01T09:26:54.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612887+0000", 2023-10-01T09:26:54.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0300000000000001e-06 2023-10-01T09:26:54.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614161+0000", 2023-10-01T09:26:54.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012741460000000001 2023-10-01T09:26:54.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:54.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614167+0000", 2023-10-01T09:26:54.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6629999999999998e-06 2023-10-01T09:26:54.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043249+0000", 2023-10-01T09:26:54.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.429082395 2023-10-01T09:26:54.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043418+0000", 2023-10-01T09:26:54.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016889300000000001 2023-10-01T09:26:54.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.348 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:26:54.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043866+0000", 2023-10-01T09:26:54.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00044823600000000002 2023-10-01T09:26:54.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.348 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057128+0000", 2023-10-01T09:26:54.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013261854 2023-10-01T09:26:54.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057142+0000", 2023-10-01T09:26:54.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3743e-05 2023-10-01T09:26:54.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057244+0000", 2023-10-01T09:26:54.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010159599999999999 2023-10-01T09:26:54.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:26:54.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057298+0000", 2023-10-01T09:26:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.41e-05 2023-10-01T09:26:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057826+0000", 2023-10-01T09:26:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00052871700000000005 2023-10-01T09:26:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:121701 7.8 7:123e4a57:::benchmark_data_smithi138_367058_object23906:head [delete] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-01T09:26:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:28.301494+0000", 2023-10-01T09:26:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 25.889070228000001, 2023-10-01T09:26:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0627196029999999, 2023-10-01T09:26:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:26:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:26:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:26:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:26:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:26:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121701 2023-10-01T09:26:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:26:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:26:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301494+0000", 2023-10-01T09:26:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:26:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301494+0000", 2023-10-01T09:26:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:26:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:26:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301493+0000", 2023-10-01T09:26:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:26:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:26:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301497+0000", 2023-10-01T09:26:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7670000000000003e-06 2023-10-01T09:26:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:26:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301506+0000", 2023-10-01T09:26:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3489999999999992e-06 2023-10-01T09:26:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:26:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301512+0000", 2023-10-01T09:26:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0290000000000002e-06 2023-10-01T09:26:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301562+0000", 2023-10-01T09:26:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0515e-05 2023-10-01T09:26:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:26:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301721+0000", 2023-10-01T09:26:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015924799999999999 2023-10-01T09:26:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:26:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301875+0000", 2023-10-01T09:26:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000154046 2023-10-01T09:26:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:26:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.310228+0000", 2023-10-01T09:26:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0083525619999999991 2023-10-01T09:26:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:26:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.318729+0000", 2023-10-01T09:26:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085004699999999996 2023-10-01T09:26:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.416615+0000", 2023-10-01T09:26:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.097886033999999997 2023-10-01T09:26:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:26:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.416630+0000", 2023-10-01T09:26:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5301000000000001e-05 2023-10-01T09:26:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:26:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:29.364189+0000", 2023-10-01T09:26:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94755891999999997 2023-10-01T09:26:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:26:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:26:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:26:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:29.364214+0000", 2023-10-01T09:26:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4902999999999999e-05 2023-10-01T09:26:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:26:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:26:54.358 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:26:54.359 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:54.359 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:26:54.379 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:54.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:54.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:54.496 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:26:54.526 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:54.578 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:26:54.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:54.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:26:54.685 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:54.713 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:54.759 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:54.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:54.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:54.866 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:26:54.966 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:55.068 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:26:55.212 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:55 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/558520891' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:26:55.212 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:55 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:26:55.212 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:55 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/558520891' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:26:55.212 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:55 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:26:55.218 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:26:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:26:54] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.33.4" 2023-10-01T09:26:55.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:55 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/558520891' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:26:55.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:55 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:26:55.767 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:26:55.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:55.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:26:55.881 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:55.908 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:55.958 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:55.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:55.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:56.010 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:56 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:26:56.010 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:56 smithi138 ceph-mon[185886]: failed to encode map e297 with expected crc 2023-10-01T09:26:56.010 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:56 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:26:56.010 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:56 smithi138 ceph-mon[185886]: osdmap e297: 8 total, 8 up, 5 in 2023-10-01T09:26:56.011 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:56 smithi138 ceph-mon[185886]: failed to encode map e297 with expected crc 2023-10-01T09:26:56.011 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:26:56.011 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:56 smithi138 ceph-mon[185886]: pgmap v785: 89 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 298 op/s; 42850/195864 objects degraded (21.877%); 23036/195864 objects misplaced (11.761%); 4.0 MiB/s, 64 objects/s recovering 2023-10-01T09:26:56.012 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:56 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:26:56.012 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:56 smithi138 ceph-mon[192513]: failed to encode map e297 with expected crc 2023-10-01T09:26:56.012 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:56 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:26:56.012 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:56 smithi138 ceph-mon[192513]: osdmap e297: 8 total, 8 up, 5 in 2023-10-01T09:26:56.012 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:56 smithi138 ceph-mon[192513]: failed to encode map e297 with expected crc 2023-10-01T09:26:56.012 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:26:56.012 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:56 smithi138 ceph-mon[192513]: pgmap v785: 89 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 298 op/s; 42850/195864 objects degraded (21.877%); 23036/195864 objects misplaced (11.761%); 4.0 MiB/s, 64 objects/s recovering 2023-10-01T09:26:56.064 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:56.288 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:56.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:56 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:26:56.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:56 smithi190 ceph-mon[160435]: failed to encode map e297 with expected crc 2023-10-01T09:26:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:56 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:26:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:56 smithi190 ceph-mon[160435]: osdmap e297: 8 total, 8 up, 5 in 2023-10-01T09:26:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:56 smithi190 ceph-mon[160435]: failed to encode map e297 with expected crc 2023-10-01T09:26:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:26:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:56 smithi190 ceph-mon[160435]: pgmap v785: 89 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 298 op/s; 42850/195864 objects degraded (21.877%); 23036/195864 objects misplaced (11.761%); 4.0 MiB/s, 64 objects/s recovering 2023-10-01T09:26:57.506 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:57.530 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:57 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 42850/195864 objects degraded (21.877%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:57.530 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:57 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 42850/195864 objects degraded (21.877%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:57.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:57.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:57.634 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:26:57.659 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:57 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 42850/195864 objects degraded (21.877%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:26:57.661 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:57.706 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:26:57.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:57.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:26:57.812 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:57.840 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:57.890 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:57.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:57.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:57.997 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:26:58.024 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:58.076 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:26:58.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:58.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:26:58.182 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:26:58.210 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:58.242 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:26:58.254 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:26:58.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:58.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:26:58.361 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:26:58.390 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:58.436 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:26:58.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:58.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:26:58.545 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:26:58.562 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:58 smithi138 ceph-mon[192513]: pgmap v786: 89 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 3.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 316 op/s; 42726/193353 objects degraded (22.097%); 22550/193353 objects misplaced (11.663%); 3.6 MiB/s, 57 objects/s recovering 2023-10-01T09:26:58.562 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:58 smithi138 ceph-mon[185886]: pgmap v786: 89 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 3.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 316 op/s; 42726/193353 objects degraded (22.097%); 22550/193353 objects misplaced (11.663%); 3.6 MiB/s, 57 objects/s recovering 2023-10-01T09:26:58.663 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:58.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:58 smithi190 ceph-mon[160435]: pgmap v786: 89 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 3.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 316 op/s; 42726/193353 objects degraded (22.097%); 22550/193353 objects misplaced (11.663%); 3.6 MiB/s, 57 objects/s recovering 2023-10-01T09:26:58.820 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:26:58.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:58.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:26:58.932 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:26:58.958 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:59.004 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:26:59.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:59.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:26:59.111 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:26:59.138 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:59.644 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:26:59.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:59.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:26:59.752 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:26:59.780 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:26:59.830 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:26:59.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:26:59.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:26:59.954 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:27:00.051 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:26:59 smithi138 ceph-mon[185886]: pgmap v787: 89 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 3.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 350 op/s; 42470/190572 objects degraded (22.286%); 22191/190572 objects misplaced (11.644%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T09:27:00.051 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:26:59 smithi138 ceph-mon[192513]: pgmap v787: 89 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 3.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 350 op/s; 42470/190572 objects degraded (22.286%); 22191/190572 objects misplaced (11.644%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T09:27:00.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:26:59 smithi190 ceph-mon[160435]: pgmap v787: 89 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 3.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 350 op/s; 42470/190572 objects degraded (22.286%); 22191/190572 objects misplaced (11.644%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T09:27:00.073 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:00.329 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:27:00.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:00.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:27:00.443 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:00.471 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:00.514 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:00.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:00.548 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:00.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:00.597 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:27:00.649 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:27:00.725 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:27:00.790 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:27:00.794 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:00.879 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:27:00.912 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:00 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1795017324' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-01T09:27:00.912 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:00 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/921024507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:00.914 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:00 smithi138 conmon[185862]: 2023-10-01T09:27:00.581+0000 7f70d64ec700 -1 mon.a@0(leader).osd e297 definitely_dead 0 2023-10-01T09:27:00.914 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:00 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1795017324' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-01T09:27:00.914 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:00 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/921024507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:00.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:00.959 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:27:00.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:27:00.959 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:27:00.978 INFO:teuthology.orchestra.run.smithi138.stderr:marked in osd.5. 2023-10-01T09:27:01.032 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:27:01.060 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:01.080 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:27:01.122 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:27:01.151 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:27:01.159 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:01.164 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:27:01.164 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:27:01.164 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:27:01.164 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:27:01.165 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:27:01.165 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:27:01.165 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:27:01.165 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:27:01.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:00 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1795017324' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-10-01T09:27:01.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:00 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/921024507' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:01.579 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:01.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:01.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:01.684 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:01.710 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:01.754 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:01.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:01.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:01.860 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:27:01.886 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:01.932 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:27:01.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:01.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:27:01.958 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:01 smithi138 ceph-mon[185886]: failed to encode map e298 with expected crc 2023-10-01T09:27:01.958 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1795017324' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-01T09:27:01.958 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:01 smithi138 ceph-mon[185886]: osdmap e298: 8 total, 8 up, 6 in 2023-10-01T09:27:01.958 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:01 smithi138 ceph-mon[185886]: pgmap v789: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 3.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 305 op/s; 42368/189654 objects degraded (22.340%); 22191/189654 objects misplaced (11.701%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T09:27:02.037 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:02.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:01 smithi190 ceph-mon[160435]: failed to encode map e298 with expected crc 2023-10-01T09:27:02.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1795017324' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-01T09:27:02.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:01 smithi190 ceph-mon[160435]: osdmap e298: 8 total, 8 up, 6 in 2023-10-01T09:27:02.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:01 smithi190 ceph-mon[160435]: pgmap v789: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 3.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 305 op/s; 42368/189654 objects degraded (22.340%); 22191/189654 objects misplaced (11.701%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T09:27:02.208 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:01 smithi138 ceph-mon[192513]: failed to encode map e298 with expected crc 2023-10-01T09:27:02.208 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1795017324' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-10-01T09:27:02.208 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:01 smithi138 ceph-mon[192513]: osdmap e298: 8 total, 8 up, 6 in 2023-10-01T09:27:02.208 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:01 smithi138 ceph-mon[192513]: pgmap v789: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 72 active+clean; 3.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 305 op/s; 42368/189654 objects degraded (22.340%); 22191/189654 objects misplaced (11.701%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T09:27:02.210 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:02.790 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:02.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:02.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:02.895 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:02.921 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:02.966 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:02.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:02.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[192513]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[192513]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[192513]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[192513]: osdmap e299: 8 total, 8 up, 6 in 2023-10-01T09:27:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[192513]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[192513]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[192513]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[192513]: 7.4 starting backfill to osd.5 from (0'0,0'0] MAX to 297'8455 2023-10-01T09:27:03.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 42368/189654 objects degraded (22.340%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:03.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[185886]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[185886]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[185886]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[185886]: osdmap e299: 8 total, 8 up, 6 in 2023-10-01T09:27:03.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[185886]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[185886]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[185886]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[185886]: 7.4 starting backfill to osd.5 from (0'0,0'0] MAX to 297'8455 2023-10-01T09:27:03.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:02 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 42368/189654 objects degraded (22.340%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:03.073 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:27:03.113 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:02 smithi190 ceph-mon[160435]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:02 smithi190 ceph-mon[160435]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:02 smithi190 ceph-mon[160435]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:02 smithi190 ceph-mon[160435]: osdmap e299: 8 total, 8 up, 6 in 2023-10-01T09:27:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:02 smithi190 ceph-mon[160435]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:02 smithi190 ceph-mon[160435]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:02 smithi190 ceph-mon[160435]: failed to encode map e299 with expected crc 2023-10-01T09:27:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:02 smithi190 ceph-mon[160435]: 7.4 starting backfill to osd.5 from (0'0,0'0] MAX to 297'8455 2023-10-01T09:27:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:02 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 42368/189654 objects degraded (22.340%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:03 smithi190 ceph-mon[160435]: failed to encode map e300 with expected crc 2023-10-01T09:27:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:03 smithi190 ceph-mon[160435]: osdmap e300: 8 total, 8 up, 6 in 2023-10-01T09:27:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:03 smithi190 ceph-mon[160435]: failed to encode map e300 with expected crc 2023-10-01T09:27:04.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:03 smithi190 ceph-mon[160435]: failed to encode map e300 with expected crc 2023-10-01T09:27:04.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:03 smithi190 ceph-mon[160435]: failed to encode map e300 with expected crc 2023-10-01T09:27:04.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:03 smithi190 ceph-mon[160435]: pgmap v792: 89 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean; 3.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 343 op/s; 37103/187158 objects degraded (19.824%); 19769/187158 objects misplaced (10.563%); 13 MiB/s, 201 objects/s recovering 2023-10-01T09:27:04.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:03 smithi138 ceph-mon[185886]: failed to encode map e300 with expected crc 2023-10-01T09:27:04.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:03 smithi138 ceph-mon[185886]: osdmap e300: 8 total, 8 up, 6 in 2023-10-01T09:27:04.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:03 smithi138 ceph-mon[185886]: failed to encode map e300 with expected crc 2023-10-01T09:27:04.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:03 smithi138 ceph-mon[185886]: failed to encode map e300 with expected crc 2023-10-01T09:27:04.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:03 smithi138 ceph-mon[185886]: failed to encode map e300 with expected crc 2023-10-01T09:27:04.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:03 smithi138 ceph-mon[185886]: pgmap v792: 89 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean; 3.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 343 op/s; 37103/187158 objects degraded (19.824%); 19769/187158 objects misplaced (10.563%); 13 MiB/s, 201 objects/s recovering 2023-10-01T09:27:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:03 smithi138 ceph-mon[192513]: failed to encode map e300 with expected crc 2023-10-01T09:27:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:03 smithi138 ceph-mon[192513]: osdmap e300: 8 total, 8 up, 6 in 2023-10-01T09:27:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:03 smithi138 ceph-mon[192513]: failed to encode map e300 with expected crc 2023-10-01T09:27:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:03 smithi138 ceph-mon[192513]: failed to encode map e300 with expected crc 2023-10-01T09:27:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:03 smithi138 ceph-mon[192513]: failed to encode map e300 with expected crc 2023-10-01T09:27:04.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:03 smithi138 ceph-mon[192513]: pgmap v792: 89 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 72 active+clean; 3.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 343 op/s; 37103/187158 objects degraded (19.824%); 19769/187158 objects misplaced (10.563%); 13 MiB/s, 201 objects/s recovering 2023-10-01T09:27:04.909 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:27:04.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:04.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:27:05.019 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:05.050 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:05.094 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:05.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:05.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:05.202 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:05.236 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:05.306 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:05.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:05.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:05.322 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:27:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:27:04] "GET /metrics HTTP/1.1" 200 36436 "" "Prometheus/2.33.4" 2023-10-01T09:27:05.412 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:05.491 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:06.279 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:05 smithi138 ceph-mon[192513]: pgmap v793: 89 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 247 op/s; 27656/186117 objects degraded (14.859%); 19769/186117 objects misplaced (10.622%); 10 MiB/s, 169 objects/s recovering 2023-10-01T09:27:06.287 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:05 smithi138 ceph-mon[185886]: pgmap v793: 89 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 247 op/s; 27656/186117 objects degraded (14.859%); 19769/186117 objects misplaced (10.622%); 10 MiB/s, 169 objects/s recovering 2023-10-01T09:27:06.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:05 smithi190 ceph-mon[160435]: pgmap v793: 89 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 247 op/s; 27656/186117 objects degraded (14.859%); 19769/186117 objects misplaced (10.622%); 10 MiB/s, 169 objects/s recovering 2023-10-01T09:27:06.526 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:06.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:06.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:06.632 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:06.733 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:07.497 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-10-01T09:27:07.587 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:07 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 27656/186117 objects degraded (14.859%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:07.588 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:07 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 27656/186117 objects degraded (14.859%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:07.624 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:07.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:07.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:07.734 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:07.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:07 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 27656/186117 objects degraded (14.859%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:07.815 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:07.836 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:07.962 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:07.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:07.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:08.071 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:27:08.103 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:08.157 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:27:08.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:08.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:27:08.264 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:08.294 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:08.346 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:08.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:08.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:08.398 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:27:08.398 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:27:08.451 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:08.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:08 smithi190 ceph-mon[160435]: pgmap v794: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 347 op/s; 27105/183072 objects degraded (14.806%); 19321/183072 objects misplaced (10.554%); 14 MiB/s, 228 objects/s recovering 2023-10-01T09:27:08.560 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:08.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:08 smithi138 ceph-mon[192513]: pgmap v794: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 347 op/s; 27105/183072 objects degraded (14.806%); 19321/183072 objects misplaced (10.554%); 14 MiB/s, 228 objects/s recovering 2023-10-01T09:27:08.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:08 smithi138 ceph-mon[185886]: pgmap v794: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 347 op/s; 27105/183072 objects degraded (14.806%); 19321/183072 objects misplaced (10.554%); 14 MiB/s, 228 objects/s recovering 2023-10-01T09:27:09.627 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:27:09.926 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:09.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:09.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:09.941 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:09 smithi138 ceph-mon[192513]: pgmap v795: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 412 op/s; 27105/179733 objects degraded (15.081%); 18891/179733 objects misplaced (10.511%); 12 MiB/s, 202 objects/s recovering 2023-10-01T09:27:09.942 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:09 smithi138 ceph-mon[185886]: pgmap v795: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 412 op/s; 27105/179733 objects degraded (15.081%); 18891/179733 objects misplaced (10.511%); 12 MiB/s, 202 objects/s recovering 2023-10-01T09:27:10.040 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:10.072 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:10.134 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:10.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:10.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:10.243 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:27:10.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:09 smithi190 ceph-mon[160435]: pgmap v795: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 412 op/s; 27105/179733 objects degraded (15.081%); 18891/179733 objects misplaced (10.511%); 12 MiB/s, 202 objects/s recovering 2023-10-01T09:27:10.322 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:11.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:27:11.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:27:11.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:27:11.506 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:27:11.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:11.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:27:11.615 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:27:11.644 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:11.701 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:27:11.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:11.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:27:11.806 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:27:11.832 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:11.889 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:27:11.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:11.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:27:11.998 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:27:12.024 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:11 smithi190 ceph-mon[160435]: pgmap v796: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 389 op/s; 26929/178140 objects degraded (15.117%); 18891/178140 objects misplaced (10.605%); 11 MiB/s, 182 objects/s recovering 2023-10-01T09:27:12.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:11 smithi138 ceph-mon[192513]: pgmap v796: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 389 op/s; 26929/178140 objects degraded (15.117%); 18891/178140 objects misplaced (10.605%); 11 MiB/s, 182 objects/s recovering 2023-10-01T09:27:12.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:11 smithi138 ceph-mon[185886]: pgmap v796: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 389 op/s; 26929/178140 objects degraded (15.117%); 18891/178140 objects misplaced (10.605%); 11 MiB/s, 182 objects/s recovering 2023-10-01T09:27:12.498 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5] out_osds: [6, 4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2023-10-01T09:27:12.498 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:27:12.498 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-01T09:27:12.498 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:27:12.563 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:27:12.571 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:27:12.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:12.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:27:12.683 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:12.711 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:12.757 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:12.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:12.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:12.838 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:12.862 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:12.889 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:12.945 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:12.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:12.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:13.051 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:27:13.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:12 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 26929/178140 objects degraded (15.117%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:13.078 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:13.130 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:27:13.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:13.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:27:13.237 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:27:13.252 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:12 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 26929/178140 objects degraded (15.117%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:13.252 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:12 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 26929/178140 objects degraded (15.117%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:13.341 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:13.710 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:27:13.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:13.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:27:13.839 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:27:13.868 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:13.926 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:27:13.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:13.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:27:14.032 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:14.119 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:14.257 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:14.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:14.269 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:13 smithi138 ceph-mon[185886]: pgmap v797: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 465 op/s; 26830/172005 objects degraded (15.598%); 17978/172005 objects misplaced (10.452%); 6.6 MiB/s, 140 objects/s recovering 2023-10-01T09:27:14.269 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:13 smithi138 ceph-mon[192513]: pgmap v797: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 465 op/s; 26830/172005 objects degraded (15.598%); 17978/172005 objects misplaced (10.452%); 6.6 MiB/s, 140 objects/s recovering 2023-10-01T09:27:14.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:14.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:13 smithi190 ceph-mon[160435]: pgmap v797: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 465 op/s; 26830/172005 objects degraded (15.598%); 17978/172005 objects misplaced (10.452%); 6.6 MiB/s, 140 objects/s recovering 2023-10-01T09:27:14.370 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:27:14.402 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:14.456 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:27:14.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:14.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:27:14.563 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:14.592 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:14.652 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:14.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:14.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:14.758 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:14.813 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:14.964 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/150078626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:27:14.964 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:14 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:27:14.964 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:14 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:27:14.964 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:14 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:27:14.964 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:14 smithi190 ceph-mon[160435]: osdmap e301: 8 total, 8 up, 6 in 2023-10-01T09:27:14.999 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:15.004 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:27:15.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:15.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:15.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/150078626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:27:15.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:14 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:27:15.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:14 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:27:15.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:14 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:27:15.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:14 smithi138 ceph-mon[185886]: osdmap e301: 8 total, 8 up, 6 in 2023-10-01T09:27:15.072 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:27:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:27:14] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.33.4" 2023-10-01T09:27:15.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/150078626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:27:15.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:14 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:27:15.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:14 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:27:15.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:14 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:27:15.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:14 smithi138 ceph-mon[192513]: osdmap e301: 8 total, 8 up, 6 in 2023-10-01T09:27:15.126 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:15.154 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:15.195 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:15.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:15.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:15.300 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:27:15.333 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:15.397 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:27:15.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:15.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:27:15.503 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:15.533 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:15.586 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:15.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:15.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:15.692 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:15.809 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:16.001 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:15 smithi190 ceph-mon[160435]: failed to encode map e301 with expected crc 2023-10-01T09:27:16.002 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:15 smithi190 ceph-mon[160435]: failed to encode map e301 with expected crc 2023-10-01T09:27:16.002 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:15 smithi190 ceph-mon[160435]: pgmap v799: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 486 op/s; 26830/171519 objects degraded (15.643%); 17978/171519 objects misplaced (10.482%); 6.9 MiB/s, 145 objects/s recovering 2023-10-01T09:27:16.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:15 smithi138 ceph-mon[185886]: failed to encode map e301 with expected crc 2023-10-01T09:27:16.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:15 smithi138 ceph-mon[185886]: failed to encode map e301 with expected crc 2023-10-01T09:27:16.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:15 smithi138 ceph-mon[185886]: pgmap v799: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 486 op/s; 26830/171519 objects degraded (15.643%); 17978/171519 objects misplaced (10.482%); 6.9 MiB/s, 145 objects/s recovering 2023-10-01T09:27:16.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:15 smithi138 ceph-mon[192513]: failed to encode map e301 with expected crc 2023-10-01T09:27:16.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:15 smithi138 ceph-mon[192513]: failed to encode map e301 with expected crc 2023-10-01T09:27:16.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:15 smithi138 ceph-mon[192513]: pgmap v799: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 486 op/s; 26830/171519 objects degraded (15.643%); 17978/171519 objects misplaced (10.482%); 6.9 MiB/s, 145 objects/s recovering 2023-10-01T09:27:16.986 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:17.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:17.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:17.074 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:17.103 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:27:17.212 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:17.515 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:27:17.532 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:17 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 26830/171519 objects degraded (15.643%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:17.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:17.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:27:17.537 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:17 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 26830/171519 objects degraded (15.643%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:17.602 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:17.633 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:17.661 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:17 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 26830/171519 objects degraded (15.643%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:17.663 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:17.685 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:27:17.712 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:17.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:17.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:17.753 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:27:17.798 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:27:17.818 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:17.846 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:17.855 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:27:17.899 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:27:17.900 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:17.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:17.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:17.941 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:27:17.985 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:27:18.006 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:18.019 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:27:18.040 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:27:18.040 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:27:18.040 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:27:18.040 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:27:18.040 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:27:18.040 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:27:18.041 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:27:18.041 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:27:18.091 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:18.354 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:18 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1331079566' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:18.355 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:18 smithi138 ceph-mon[185886]: pgmap v800: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 26684/168132 objects degraded (15.871%); 17488/168132 objects misplaced (10.401%); 4.4 MiB/s, 155 objects/s recovering 2023-10-01T09:27:18.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:18 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1331079566' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:18 smithi190 ceph-mon[160435]: pgmap v800: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 26684/168132 objects degraded (15.871%); 17488/168132 objects misplaced (10.401%); 4.4 MiB/s, 155 objects/s recovering 2023-10-01T09:27:18.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:18 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1331079566' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:18.825 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:18 smithi138 ceph-mon[192513]: pgmap v800: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 26684/168132 objects degraded (15.871%); 17488/168132 objects misplaced (10.401%); 4.4 MiB/s, 155 objects/s recovering 2023-10-01T09:27:19.104 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:27:19.945 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:19.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:19.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:20.059 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:27:20.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:19 smithi138 ceph-mon[192513]: pgmap v801: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 512 op/s; 26684/164349 objects degraded (16.236%); 17072/164349 objects misplaced (10.388%); 4.2 MiB/s, 152 objects/s recovering 2023-10-01T09:27:20.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:19 smithi138 ceph-mon[185886]: pgmap v801: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 512 op/s; 26684/164349 objects degraded (16.236%); 17072/164349 objects misplaced (10.388%); 4.2 MiB/s, 152 objects/s recovering 2023-10-01T09:27:20.090 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:20.133 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:27:20.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:20.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:27:20.239 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:20.268 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:19 smithi190 ceph-mon[160435]: pgmap v801: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 512 op/s; 26684/164349 objects degraded (16.236%); 17072/164349 objects misplaced (10.388%); 4.2 MiB/s, 152 objects/s recovering 2023-10-01T09:27:20.271 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:20.320 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:20.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:20.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:20.426 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:20.459 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:20.505 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:20.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:20.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:20.612 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:27:20.638 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:20.679 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:27:20.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:20.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:27:20.784 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:27:20.821 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:22.176 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:21 smithi138 ceph-mon[185886]: pgmap v802: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 26549/163083 objects degraded (16.279%); 17072/163083 objects misplaced (10.468%); 3.9 MiB/s, 148 objects/s recovering 2023-10-01T09:27:22.187 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:21 smithi138 ceph-mon[192513]: pgmap v802: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 26549/163083 objects degraded (16.279%); 17072/163083 objects misplaced (10.468%); 3.9 MiB/s, 148 objects/s recovering 2023-10-01T09:27:22.236 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:27:22.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:22.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:27:22.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:21 smithi190 ceph-mon[160435]: pgmap v802: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 26549/163083 objects degraded (16.279%); 17072/163083 objects misplaced (10.468%); 3.9 MiB/s, 148 objects/s recovering 2023-10-01T09:27:22.343 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:22.456 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:22.818 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:22.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:22.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:22.851 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:27:22.851 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:27:22.852 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:27:22.852 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:27:22.852 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:27:22.942 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:23.058 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:22 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 26549/163083 objects degraded (16.279%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:23.058 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:27:23.058 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:22 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 26549/163083 objects degraded (16.279%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:23.059 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:27:23.066 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:22 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 26549/163083 objects degraded (16.279%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:27:23.501 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:27:23.524 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:27:24.196 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:27:24.220 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:23 smithi138 ceph-mon[192513]: from='client.55372 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:27:24.221 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:23 smithi138 ceph-mon[192513]: pgmap v803: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 26549/157293 objects degraded (16.879%); 16192/157293 objects misplaced (10.294%); 3.4 MiB/s, 165 objects/s recovering 2023-10-01T09:27:24.221 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:23 smithi138 ceph-mon[185886]: from='client.55372 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:27:24.221 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:23 smithi138 ceph-mon[185886]: pgmap v803: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 26549/157293 objects degraded (16.879%); 16192/157293 objects misplaced (10.294%); 3.4 MiB/s, 165 objects/s recovering 2023-10-01T09:27:24.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:23 smithi190 ceph-mon[160435]: from='client.55372 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:27:24.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:23 smithi190 ceph-mon[160435]: pgmap v803: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 26549/157293 objects degraded (16.879%); 16192/157293 objects misplaced (10.294%); 3.4 MiB/s, 165 objects/s recovering 2023-10-01T09:27:24.592 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:24.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:24.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:24.700 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:24.743 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:24.822 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:24.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:24.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:24.927 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:27:24.941 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:27:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:27:24] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.33.4" 2023-10-01T09:27:24.960 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:25.065 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:27:25.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:25.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:27:25.176 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:25.202 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:25.335 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:27:25.419 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:27:25.543 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:27:25.544 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:27:25.602 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:25.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:25.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:25.646 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:25.704 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":802,"stamp":"2023-10-01T09:27:21.820827+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3565492171,"num_objects":54361,"num_object_clones":0,"num_object_copies":163083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26549,"num_objects_misplaced":17072,"num_objects_unfound":0,"num_objects_dirty":54361,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":142359,"num_write_kb":6273830,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8406,"num_bytes_recovered":480667343,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90111,"ondisk_log_size":90111,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":290,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11190404,"kb_used_data":8823828,"kb_used_omap":38,"kb_used_meta":2366297,"kb_avail":551280508,"statfs":{"total":575970213888,"available":564511240192,"internally_reserved":0,"allocated":9035599872,"data_stored":9028326103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39221,"internal_metadata":2423088843},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,2,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-328925184,"num_objects":-5019,"num_object_clones":0,"num_object_copies":-15057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-380,"num_objects_misplaced":-1819,"num_objects_unfound":0,"num_objects_dirty":-5019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5019,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1486,"num_bytes_recovered":41025536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003605"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":182,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.961799+0000","last_change":"2023-10-01T09:26:12.273466+0000","last_active":"2023-10-01T09:27:02.961799+0000","last_peered":"2023-10-01T09:27:02.961799+0000","last_clean":"2023-10-01T09:27:02.961799+0000","last_became_active":"2023-10-01T09:26:12.272944+0000","last_became_peered":"2023-10-01T09:26:12.272944+0000","last_unstale":"2023-10-01T09:27:02.961799+0000","last_undegraded":"2023-10-01T09:27:02.961799+0000","last_fullsized":"2023-10-01T09:27:02.961799+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:40:41.939800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":94,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.972133+0000","last_change":"2023-10-01T09:27:01.993008+0000","last_active":"2023-10-01T09:27:02.972133+0000","last_peered":"2023-10-01T09:27:02.972133+0000","last_clean":"2023-10-01T09:27:02.972133+0000","last_became_active":"2023-10-01T09:27:01.992687+0000","last_became_peered":"2023-10-01T09:27:01.992687+0000","last_unstale":"2023-10-01T09:27:02.972133+0000","last_undegraded":"2023-10-01T09:27:02.972133+0000","last_fullsized":"2023-10-01T09:27:02.972133+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:20:18.067760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":258,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.960828+0000","last_change":"2023-10-01T09:26:12.316287+0000","last_active":"2023-10-01T09:27:02.960828+0000","last_peered":"2023-10-01T09:27:02.960828+0000","last_clean":"2023-10-01T09:27:02.960828+0000","last_became_active":"2023-10-01T09:26:12.315708+0000","last_became_peered":"2023-10-01T09:26:12.315708+0000","last_unstale":"2023-10-01T09:27:02.960828+0000","last_undegraded":"2023-10-01T09:27:02.960828+0000","last_fullsized":"2023-10-01T09:27:02.960828+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:41:36.142856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":306,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.974584+0000","last_change":"2023-10-01T09:23:10.611220+0000","last_active":"2023-10-01T09:27:02.974584+0000","last_peered":"2023-10-01T09:27:02.974584+0000","last_clean":"2023-10-01T09:27:02.974584+0000","last_became_active":"2023-10-01T09:23:10.610573+0000","last_became_peered":"2023-10-01T09:23:10.610573+0000","last_unstale":"2023-10-01T09:27:02.974584+0000","last_undegraded":"2023-10-01T09:27:02.974584+0000","last_fullsized":"2023-10-01T09:27:02.974584+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:58:32.020825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":76,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.970541+0000","last_change":"2023-10-01T09:22:40.172183+0000","last_active":"2023-10-01T09:27:02.970541+0000","last_peered":"2023-10-01T09:27:02.970541+0000","last_clean":"2023-10-01T09:27:02.970541+0000","last_became_active":"2023-10-01T09:22:40.170164+0000","last_became_peered":"2023-10-01T09:22:40.170164+0000","last_unstale":"2023-10-01T09:27:02.970541+0000","last_undegraded":"2023-10-01T09:27:02.970541+0000","last_fullsized":"2023-10-01T09:27:02.970541+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:33:10.407999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.963385+0000","last_change":"2023-10-01T09:27:02.003207+0000","last_active":"2023-10-01T09:27:02.963385+0000","last_peered":"2023-10-01T09:27:02.963385+0000","last_clean":"2023-10-01T09:27:02.963385+0000","last_became_active":"2023-10-01T09:27:02.002872+0000","last_became_peered":"2023-10-01T09:27:02.002872+0000","last_unstale":"2023-10-01T09:27:02.963385+0000","last_undegraded":"2023-10-01T09:27:02.963385+0000","last_fullsized":"2023-10-01T09:27:02.963385+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":336,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.978536+0000","last_change":"2023-10-01T09:27:01.992338+0000","last_active":"2023-10-01T09:27:02.978536+0000","last_peered":"2023-10-01T09:27:02.978536+0000","last_clean":"2023-10-01T09:27:02.978536+0000","last_became_active":"2023-10-01T09:27:01.989050+0000","last_became_peered":"2023-10-01T09:27:01.989050+0000","last_unstale":"2023-10-01T09:27:02.978536+0000","last_undegraded":"2023-10-01T09:27:02.978536+0000","last_fullsized":"2023-10-01T09:27:02.978536+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:43:29.856456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":326,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:12.571626+0000","last_change":"2023-10-01T09:27:01.984577+0000","last_active":"2023-10-01T09:27:12.571626+0000","last_peered":"2023-10-01T09:27:12.571626+0000","last_clean":"2023-10-01T09:27:12.571626+0000","last_became_active":"2023-10-01T09:27:01.984262+0000","last_became_peered":"2023-10-01T09:27:01.984262+0000","last_unstale":"2023-10-01T09:27:12.571626+0000","last_undegraded":"2023-10-01T09:27:12.571626+0000","last_fullsized":"2023-10-01T09:27:12.571626+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:16:49.798047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.963231+0000","last_change":"2023-10-01T09:27:01.970412+0000","last_active":"2023-10-01T09:27:02.963231+0000","last_peered":"2023-10-01T09:27:02.963231+0000","last_clean":"2023-10-01T09:27:02.963231+0000","last_became_active":"2023-10-01T09:27:01.970108+0000","last_became_peered":"2023-10-01T09:27:01.970108+0000","last_unstale":"2023-10-01T09:27:02.963231+0000","last_undegraded":"2023-10-01T09:27:02.963231+0000","last_fullsized":"2023-10-01T09:27:02.963231+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":343,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:12.571684+0000","last_change":"2023-10-01T09:27:01.988876+0000","last_active":"2023-10-01T09:27:12.571684+0000","last_peered":"2023-10-01T09:27:12.571684+0000","last_clean":"2023-10-01T09:27:12.571684+0000","last_became_active":"2023-10-01T09:27:01.979001+0000","last_became_peered":"2023-10-01T09:27:01.979001+0000","last_unstale":"2023-10-01T09:27:12.571684+0000","last_undegraded":"2023-10-01T09:27:12.571684+0000","last_fullsized":"2023-10-01T09:27:12.571684+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:01:31.161101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":85,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.972548+0000","last_change":"2023-10-01T09:26:12.415219+0000","last_active":"2023-10-01T09:27:02.972548+0000","last_peered":"2023-10-01T09:27:02.972548+0000","last_clean":"2023-10-01T09:27:02.972548+0000","last_became_active":"2023-10-01T09:26:12.286669+0000","last_became_peered":"2023-10-01T09:26:12.286669+0000","last_unstale":"2023-10-01T09:27:02.972548+0000","last_undegraded":"2023-10-01T09:27:02.972548+0000","last_fullsized":"2023-10-01T09:27:02.972548+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":318,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.961845+0000","last_change":"2023-10-01T09:25:56.039970+0000","last_active":"2023-10-01T09:27:02.961845+0000","last_peered":"2023-10-01T09:27:02.961845+0000","last_clean":"2023-10-01T09:27:02.961845+0000","last_became_active":"2023-10-01T09:25:56.039514+0000","last_became_peered":"2023-10-01T09:25:56.039514+0000","last_unstale":"2023-10-01T09:27:02.961845+0000","last_undegraded":"2023-10-01T09:27:02.961845+0000","last_fullsized":"2023-10-01T09:27:02.961845+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:57:43.841503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":290,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.972179+0000","last_change":"2023-10-01T09:26:12.404331+0000","last_active":"2023-10-01T09:27:02.972179+0000","last_peered":"2023-10-01T09:27:02.972179+0000","last_clean":"2023-10-01T09:27:02.972179+0000","last_became_active":"2023-10-01T09:26:12.270400+0000","last_became_peered":"2023-10-01T09:26:12.270400+0000","last_unstale":"2023-10-01T09:27:02.972179+0000","last_undegraded":"2023-10-01T09:27:02.972179+0000","last_fullsized":"2023-10-01T09:27:02.972179+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":244,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.959562+0000","last_change":"2023-10-01T09:27:02.010624+0000","last_active":"2023-10-01T09:27:02.959562+0000","last_peered":"2023-10-01T09:27:02.959562+0000","last_clean":"2023-10-01T09:27:02.959562+0000","last_became_active":"2023-10-01T09:27:02.010352+0000","last_became_peered":"2023-10-01T09:27:02.010352+0000","last_unstale":"2023-10-01T09:27:02.959562+0000","last_undegraded":"2023-10-01T09:27:02.959562+0000","last_fullsized":"2023-10-01T09:27:02.959562+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:58:47.050369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":344,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.971882+0000","last_change":"2023-10-01T09:26:12.384447+0000","last_active":"2023-10-01T09:27:02.971882+0000","last_peered":"2023-10-01T09:27:02.971882+0000","last_clean":"2023-10-01T09:27:02.971882+0000","last_became_active":"2023-10-01T09:26:12.249109+0000","last_became_peered":"2023-10-01T09:26:12.249109+0000","last_unstale":"2023-10-01T09:27:02.971882+0000","last_undegraded":"2023-10-01T09:27:02.971882+0000","last_fullsized":"2023-10-01T09:27:02.971882+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":284,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:12.574945+0000","last_change":"2023-10-01T09:19:57.493670+0000","last_active":"2023-10-01T09:27:12.574945+0000","last_peered":"2023-10-01T09:27:12.574945+0000","last_clean":"2023-10-01T09:27:12.574945+0000","last_became_active":"2023-10-01T09:19:57.493349+0000","last_became_peered":"2023-10-01T09:19:57.493349+0000","last_unstale":"2023-10-01T09:27:12.574945+0000","last_undegraded":"2023-10-01T09:27:12.574945+0000","last_fullsized":"2023-10-01T09:27:12.574945+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:26:56.468779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":290,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-01T09:27:14.948776+0000","last_change":"2023-10-01T09:23:10.607388+0000","last_active":"2023-10-01T09:27:14.948776+0000","last_peered":"2023-10-01T09:27:14.948776+0000","last_clean":"2023-10-01T09:27:14.948776+0000","last_became_active":"2023-10-01T09:23:10.607084+0000","last_became_peered":"2023-10-01T09:23:10.607084+0000","last_unstale":"2023-10-01T09:27:14.948776+0000","last_undegraded":"2023-10-01T09:27:14.948776+0000","last_fullsized":"2023-10-01T09:27:14.948776+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:19.669393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.963100+0000","last_change":"2023-10-01T09:27:01.978738+0000","last_active":"2023-10-01T09:27:02.963100+0000","last_peered":"2023-10-01T09:27:02.963100+0000","last_clean":"2023-10-01T09:27:02.963100+0000","last_became_active":"2023-10-01T09:27:01.978024+0000","last_became_peered":"2023-10-01T09:27:01.978024+0000","last_unstale":"2023-10-01T09:27:02.963100+0000","last_undegraded":"2023-10-01T09:27:02.963100+0000","last_fullsized":"2023-10-01T09:27:02.963100+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":294,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.971727+0000","last_change":"2023-10-01T09:26:12.294122+0000","last_active":"2023-10-01T09:27:02.971727+0000","last_peered":"2023-10-01T09:27:02.971727+0000","last_clean":"2023-10-01T09:27:02.971727+0000","last_became_active":"2023-10-01T09:26:12.293760+0000","last_became_peered":"2023-10-01T09:26:12.293760+0000","last_unstale":"2023-10-01T09:27:02.971727+0000","last_undegraded":"2023-10-01T09:27:02.971727+0000","last_fullsized":"2023-10-01T09:27:02.971727+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":271,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.974629+0000","last_change":"2023-10-01T09:23:10.611400+0000","last_active":"2023-10-01T09:27:02.974629+0000","last_peered":"2023-10-01T09:27:02.974629+0000","last_clean":"2023-10-01T09:27:02.974629+0000","last_became_active":"2023-10-01T09:23:10.610928+0000","last_became_peered":"2023-10-01T09:23:10.610928+0000","last_unstale":"2023-10-01T09:27:02.974629+0000","last_undegraded":"2023-10-01T09:27:02.974629+0000","last_fullsized":"2023-10-01T09:27:02.974629+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:17.141540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":202,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.972839+0000","last_change":"2023-10-01T09:27:02.093853+0000","last_active":"2023-10-01T09:27:02.972839+0000","last_peered":"2023-10-01T09:27:02.972839+0000","last_clean":"2023-10-01T09:27:02.972839+0000","last_became_active":"2023-10-01T09:27:02.052168+0000","last_became_peered":"2023-10-01T09:27:02.052168+0000","last_unstale":"2023-10-01T09:27:02.972839+0000","last_undegraded":"2023-10-01T09:27:02.972839+0000","last_fullsized":"2023-10-01T09:27:02.972839+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:02.761711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":85,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.973033+0000","last_change":"2023-10-01T09:23:10.613386+0000","last_active":"2023-10-01T09:27:02.973033+0000","last_peered":"2023-10-01T09:27:02.973033+0000","last_clean":"2023-10-01T09:27:02.973033+0000","last_became_active":"2023-10-01T09:23:10.613046+0000","last_became_peered":"2023-10-01T09:23:10.613046+0000","last_unstale":"2023-10-01T09:27:02.973033+0000","last_undegraded":"2023-10-01T09:27:02.973033+0000","last_fullsized":"2023-10-01T09:27:02.973033+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:23:01.448841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"301'8618","reported_seq":12802,"reported_epoch":301,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:27:17.986512+0000","last_change":"2023-10-01T09:27:03.171939+0000","last_active":"2023-10-01T09:27:17.986512+0000","last_peered":"2023-10-01T09:27:17.986512+0000","last_clean":"2023-10-01T09:25:11.244862+0000","last_became_active":"2023-10-01T09:27:03.098878+0000","last_became_peered":"2023-10-01T09:27:03.098878+0000","last_unstale":"2023-10-01T09:27:17.986512+0000","last_undegraded":"2023-10-01T09:27:03.087334+0000","last_fullsized":"2023-10-01T09:27:02.959716+0000","mapping_epoch":299,"log_start":"265'3404","ondisk_log_start":"265'3404","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":5214,"log_dups_size":0,"ondisk_log_size":5214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:08:32.490412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217710592,"num_objects":3322,"num_object_clones":0,"num_object_copies":9966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5967,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8618,"num_write_kb":382080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1367,"num_bytes_recovered":89325568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5],"avail_no_missing":["1","0","6"],"object_location_counts":[{"shards":"1,5","objects":-503},{"shards":"1,5,6","objects":2805}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":81,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.972069+0000","last_change":"2023-10-01T09:26:12.395230+0000","last_active":"2023-10-01T09:27:02.972069+0000","last_peered":"2023-10-01T09:27:02.972069+0000","last_clean":"2023-10-01T09:27:02.972069+0000","last_became_active":"2023-10-01T09:26:12.269960+0000","last_became_peered":"2023-10-01T09:26:12.269960+0000","last_unstale":"2023-10-01T09:27:02.972069+0000","last_undegraded":"2023-10-01T09:27:02.972069+0000","last_fullsized":"2023-10-01T09:27:02.972069+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.962388+0000","last_change":"2023-10-01T09:27:01.978449+0000","last_active":"2023-10-01T09:27:02.962388+0000","last_peered":"2023-10-01T09:27:02.962388+0000","last_clean":"2023-10-01T09:27:02.962388+0000","last_became_active":"2023-10-01T09:27:01.977942+0000","last_became_peered":"2023-10-01T09:27:01.977942+0000","last_unstale":"2023-10-01T09:27:02.962388+0000","last_undegraded":"2023-10-01T09:27:02.962388+0000","last_fullsized":"2023-10-01T09:27:02.962388+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"301'8695","reported_seq":10301,"reported_epoch":301,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:27:17.465583+0000","last_change":"2023-10-01T09:27:02.124392+0000","last_active":"2023-10-01T09:27:17.465583+0000","last_peered":"2023-10-01T09:27:17.465583+0000","last_clean":"2023-10-01T09:25:24.756344+0000","last_became_active":"2023-10-01T09:27:02.100836+0000","last_became_peered":"2023-10-01T09:27:02.100836+0000","last_unstale":"2023-10-01T09:27:17.465583+0000","last_undegraded":"2023-10-01T09:27:01.996532+0000","last_fullsized":"2023-10-01T09:27:01.973450+0000","mapping_epoch":298,"log_start":"265'3765","ondisk_log_start":"265'3765","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":4930,"log_dups_size":0,"ondisk_log_size":4930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:17:25.121610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224329728,"num_objects":3423,"num_object_clones":0,"num_object_copies":10269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2124,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8695,"num_write_kb":387776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":765,"num_bytes_recovered":50069504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":3423}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":282,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.972508+0000","last_change":"2023-10-01T09:26:12.408240+0000","last_active":"2023-10-01T09:27:02.972508+0000","last_peered":"2023-10-01T09:27:02.972508+0000","last_clean":"2023-10-01T09:27:02.972508+0000","last_became_active":"2023-10-01T09:26:12.272093+0000","last_became_peered":"2023-10-01T09:26:12.272093+0000","last_unstale":"2023-10-01T09:27:02.972508+0000","last_undegraded":"2023-10-01T09:27:02.972508+0000","last_fullsized":"2023-10-01T09:27:02.972508+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":312,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.960735+0000","last_change":"2023-10-01T09:26:12.316647+0000","last_active":"2023-10-01T09:27:02.960735+0000","last_peered":"2023-10-01T09:27:02.960735+0000","last_clean":"2023-10-01T09:27:02.960735+0000","last_became_active":"2023-10-01T09:26:12.316106+0000","last_became_peered":"2023-10-01T09:26:12.316106+0000","last_unstale":"2023-10-01T09:27:02.960735+0000","last_undegraded":"2023-10-01T09:27:02.960735+0000","last_fullsized":"2023-10-01T09:27:02.960735+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:33:10.358187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"301'9026","reported_seq":10055,"reported_epoch":301,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:27:17.887046+0000","last_change":"2023-10-01T09:27:01.986121+0000","last_active":"2023-10-01T09:27:17.887046+0000","last_peered":"2023-10-01T09:27:17.887046+0000","last_clean":"2023-10-01T09:26:11.265557+0000","last_became_active":"2023-10-01T09:26:13.355183+0000","last_became_peered":"2023-10-01T09:26:13.355183+0000","last_unstale":"2023-10-01T09:27:17.887046+0000","last_undegraded":"2023-10-01T09:27:17.887046+0000","last_fullsized":"2023-10-01T09:27:17.887046+0000","mapping_epoch":284,"log_start":"265'3383","ondisk_log_start":"265'3383","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":5643,"log_dups_size":0,"ondisk_log_size":5643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:00:49.109260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228720640,"num_objects":3490,"num_object_clones":0,"num_object_copies":10470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3073,"num_objects_unfound":0,"num_objects_dirty":3490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9026,"num_write_kb":400512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":480,"num_bytes_recovered":31457280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3490}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":353,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.972538+0000","last_change":"2023-10-01T09:27:01.985586+0000","last_active":"2023-10-01T09:27:02.972538+0000","last_peered":"2023-10-01T09:27:02.972538+0000","last_clean":"2023-10-01T09:27:02.972538+0000","last_became_active":"2023-10-01T09:27:01.978021+0000","last_became_peered":"2023-10-01T09:27:01.978021+0000","last_unstale":"2023-10-01T09:27:02.972538+0000","last_undegraded":"2023-10-01T09:27:02.972538+0000","last_fullsized":"2023-10-01T09:27:02.972538+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":257,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.961717+0000","last_change":"2023-10-01T09:27:01.977339+0000","last_active":"2023-10-01T09:27:02.961717+0000","last_peered":"2023-10-01T09:27:02.961717+0000","last_clean":"2023-10-01T09:27:02.961717+0000","last_became_active":"2023-10-01T09:27:01.976216+0000","last_became_peered":"2023-10-01T09:27:01.976216+0000","last_unstale":"2023-10-01T09:27:02.961717+0000","last_undegraded":"2023-10-01T09:27:02.961717+0000","last_fullsized":"2023-10-01T09:27:02.961717+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:38.984633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"301'8817","reported_seq":11522,"reported_epoch":301,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:27:21.488901+0000","last_change":"2023-10-01T09:26:30.448302+0000","last_active":"2023-10-01T09:27:21.488901+0000","last_peered":"2023-10-01T09:27:21.488901+0000","last_clean":"2023-10-01T09:26:11.334917+0000","last_became_active":"2023-10-01T09:26:30.389442+0000","last_became_peered":"2023-10-01T09:26:30.389442+0000","last_unstale":"2023-10-01T09:27:21.488901+0000","last_undegraded":"2023-10-01T09:26:30.373994+0000","last_fullsized":"2023-10-01T09:27:21.488901+0000","mapping_epoch":289,"log_start":"264'2918","ondisk_log_start":"264'2918","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":5899,"log_dups_size":0,"ondisk_log_size":5899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215678976,"num_objects":3291,"num_object_clones":0,"num_object_copies":9873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4734,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8817,"num_write_kb":387456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1320,"num_bytes_recovered":86376448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,3,7","objects":1},{"shards":"2,4,7","objects":3290}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.962730+0000","last_change":"2023-10-01T09:27:01.978212+0000","last_active":"2023-10-01T09:27:02.962730+0000","last_peered":"2023-10-01T09:27:02.962730+0000","last_clean":"2023-10-01T09:27:02.962730+0000","last_became_active":"2023-10-01T09:27:01.977875+0000","last_became_peered":"2023-10-01T09:27:01.977875+0000","last_unstale":"2023-10-01T09:27:02.962730+0000","last_undegraded":"2023-10-01T09:27:02.962730+0000","last_fullsized":"2023-10-01T09:27:02.962730+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":263,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.973296+0000","last_change":"2023-10-01T09:26:12.308350+0000","last_active":"2023-10-01T09:27:02.973296+0000","last_peered":"2023-10-01T09:27:02.973296+0000","last_clean":"2023-10-01T09:27:02.973296+0000","last_became_active":"2023-10-01T09:26:12.307689+0000","last_became_peered":"2023-10-01T09:26:12.307689+0000","last_unstale":"2023-10-01T09:27:02.973296+0000","last_undegraded":"2023-10-01T09:27:02.973296+0000","last_fullsized":"2023-10-01T09:27:02.973296+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:34:57.116449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"301'8710","reported_seq":8762,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-01T09:27:17.867587+0000","last_change":"2023-10-01T09:23:45.302878+0000","last_active":"2023-10-01T09:27:17.867587+0000","last_peered":"2023-10-01T09:27:17.867587+0000","last_clean":"2023-10-01T09:27:17.867587+0000","last_became_active":"2023-10-01T09:23:45.302301+0000","last_became_peered":"2023-10-01T09:23:45.302301+0000","last_unstale":"2023-10-01T09:27:17.867587+0000","last_undegraded":"2023-10-01T09:27:17.867587+0000","last_fullsized":"2023-10-01T09:27:17.867587+0000","mapping_epoch":259,"log_start":"264'2996","ondisk_log_start":"264'2996","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":5714,"log_dups_size":0,"ondisk_log_size":5714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:33:01.482516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220987392,"num_objects":3372,"num_object_clones":0,"num_object_copies":10116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8710,"num_write_kb":386624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":286,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.970668+0000","last_change":"2023-10-01T09:26:12.387826+0000","last_active":"2023-10-01T09:27:02.970668+0000","last_peered":"2023-10-01T09:27:02.970668+0000","last_clean":"2023-10-01T09:27:02.970668+0000","last_became_active":"2023-10-01T09:26:12.301770+0000","last_became_peered":"2023-10-01T09:26:12.301770+0000","last_unstale":"2023-10-01T09:27:02.970668+0000","last_undegraded":"2023-10-01T09:27:02.970668+0000","last_fullsized":"2023-10-01T09:27:02.970668+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:09:42.617904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":257,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.971844+0000","last_change":"2023-10-01T09:26:12.335185+0000","last_active":"2023-10-01T09:27:02.971844+0000","last_peered":"2023-10-01T09:27:02.971844+0000","last_clean":"2023-10-01T09:27:02.971844+0000","last_became_active":"2023-10-01T09:26:12.334909+0000","last_became_peered":"2023-10-01T09:26:12.334909+0000","last_unstale":"2023-10-01T09:27:02.971844+0000","last_undegraded":"2023-10-01T09:27:02.971844+0000","last_fullsized":"2023-10-01T09:27:02.971844+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:50:54.679196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"301'8926","reported_seq":10156,"reported_epoch":301,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:27:20.289435+0000","last_change":"2023-10-01T09:27:02.127232+0000","last_active":"2023-10-01T09:27:20.289435+0000","last_peered":"2023-10-01T09:27:20.289435+0000","last_clean":"2023-10-01T09:25:24.757312+0000","last_became_active":"2023-10-01T09:27:02.105435+0000","last_became_peered":"2023-10-01T09:27:02.105435+0000","last_unstale":"2023-10-01T09:27:20.289435+0000","last_undegraded":"2023-10-01T09:27:01.997504+0000","last_fullsized":"2023-10-01T09:27:01.978288+0000","mapping_epoch":298,"log_start":"262'400","ondisk_log_start":"262'400","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":8526,"log_dups_size":0,"ondisk_log_size":8526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:48:58.966743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222298112,"num_objects":3392,"num_object_clones":0,"num_object_copies":10176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2185,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8926,"num_write_kb":394176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":580,"num_bytes_recovered":37945344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":3392}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":262,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.961007+0000","last_change":"2023-10-01T09:27:02.009333+0000","last_active":"2023-10-01T09:27:02.961007+0000","last_peered":"2023-10-01T09:27:02.961007+0000","last_clean":"2023-10-01T09:27:02.961007+0000","last_became_active":"2023-10-01T09:27:02.009036+0000","last_became_peered":"2023-10-01T09:27:02.009036+0000","last_unstale":"2023-10-01T09:27:02.961007+0000","last_undegraded":"2023-10-01T09:27:02.961007+0000","last_fullsized":"2023-10-01T09:27:02.961007+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:35.440855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":307,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.960857+0000","last_change":"2023-10-01T09:26:29.382524+0000","last_active":"2023-10-01T09:27:02.960857+0000","last_peered":"2023-10-01T09:27:02.960857+0000","last_clean":"2023-10-01T09:27:02.960857+0000","last_became_active":"2023-10-01T09:26:29.382184+0000","last_became_peered":"2023-10-01T09:26:29.382184+0000","last_unstale":"2023-10-01T09:27:02.960857+0000","last_undegraded":"2023-10-01T09:27:02.960857+0000","last_fullsized":"2023-10-01T09:27:02.960857+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:44:04.822662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"301'8744","reported_seq":8814,"reported_epoch":301,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:27:17.959220+0000","last_change":"2023-10-01T09:27:01.996562+0000","last_active":"2023-10-01T09:27:17.959220+0000","last_peered":"2023-10-01T09:27:17.959220+0000","last_clean":"2023-10-01T09:25:11.222435+0000","last_became_active":"2023-10-01T09:25:57.191969+0000","last_became_peered":"2023-10-01T09:25:57.191969+0000","last_unstale":"2023-10-01T09:27:17.959220+0000","last_undegraded":"2023-10-01T09:25:56.900420+0000","last_fullsized":"2023-10-01T09:25:56.858326+0000","mapping_epoch":280,"log_start":"265'3540","ondisk_log_start":"265'3540","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":5204,"log_dups_size":0,"ondisk_log_size":5204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:22:36.271195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225902616,"num_objects":3448,"num_object_clones":0,"num_object_copies":10344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6096,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3448,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8744,"num_write_kb":390081,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","6"],"object_location_counts":[{"shards":"0,1","objects":-1840},{"shards":"0,1,6","objects":2903}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":164,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.971750+0000","last_change":"2023-10-01T09:26:29.372326+0000","last_active":"2023-10-01T09:27:02.971750+0000","last_peered":"2023-10-01T09:27:02.971750+0000","last_clean":"2023-10-01T09:27:02.971750+0000","last_became_active":"2023-10-01T09:26:29.372035+0000","last_became_peered":"2023-10-01T09:26:29.372035+0000","last_unstale":"2023-10-01T09:27:02.971750+0000","last_undegraded":"2023-10-01T09:27:02.971750+0000","last_fullsized":"2023-10-01T09:27:02.971750+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":68,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.959915+0000","last_change":"2023-10-01T09:27:02.009243+0000","last_active":"2023-10-01T09:27:02.959915+0000","last_peered":"2023-10-01T09:27:02.959915+0000","last_clean":"2023-10-01T09:27:02.959915+0000","last_became_active":"2023-10-01T09:27:02.008951+0000","last_became_peered":"2023-10-01T09:27:02.008951+0000","last_unstale":"2023-10-01T09:27:02.959915+0000","last_undegraded":"2023-10-01T09:27:02.959915+0000","last_fullsized":"2023-10-01T09:27:02.959915+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:59:12.820478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.c","version":"301'8676","reported_seq":11618,"reported_epoch":301,"state":"active+recovering+remapped","last_fresh":"2023-10-01T09:27:17.573511+0000","last_change":"2023-10-01T09:27:09.108564+0000","last_active":"2023-10-01T09:27:17.573511+0000","last_peered":"2023-10-01T09:27:17.573511+0000","last_clean":"2023-10-01T09:25:11.244632+0000","last_became_active":"2023-10-01T09:26:50.058339+0000","last_became_peered":"2023-10-01T09:26:50.058339+0000","last_unstale":"2023-10-01T09:27:17.573511+0000","last_undegraded":"2023-10-01T09:27:17.573511+0000","last_fullsized":"2023-10-01T09:27:17.573511+0000","mapping_epoch":295,"log_start":"265'3825","ondisk_log_start":"265'3825","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":4851,"log_dups_size":0,"ondisk_log_size":4851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:33:36.180269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224133120,"num_objects":3420,"num_object_clones":0,"num_object_copies":10260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8676,"num_write_kb":387072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1434,"num_bytes_recovered":37617664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,6],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,6,7","objects":3420}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":77,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.970667+0000","last_change":"2023-10-01T09:22:40.316008+0000","last_active":"2023-10-01T09:27:02.970667+0000","last_peered":"2023-10-01T09:27:02.970667+0000","last_clean":"2023-10-01T09:27:02.970667+0000","last_became_active":"2023-10-01T09:22:40.315602+0000","last_became_peered":"2023-10-01T09:22:40.315602+0000","last_unstale":"2023-10-01T09:27:02.970667+0000","last_undegraded":"2023-10-01T09:27:02.970667+0000","last_fullsized":"2023-10-01T09:27:02.970667+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:39:37.485241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":176,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.961050+0000","last_change":"2023-10-01T09:26:12.320680+0000","last_active":"2023-10-01T09:27:02.961050+0000","last_peered":"2023-10-01T09:27:02.961050+0000","last_clean":"2023-10-01T09:27:02.961050+0000","last_became_active":"2023-10-01T09:26:12.320340+0000","last_became_peered":"2023-10-01T09:26:12.320340+0000","last_unstale":"2023-10-01T09:27:02.961050+0000","last_undegraded":"2023-10-01T09:27:02.961050+0000","last_fullsized":"2023-10-01T09:27:02.961050+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:11:35.796015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"301'8674","reported_seq":9014,"reported_epoch":301,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-01T09:27:17.454713+0000","last_change":"2023-10-01T09:27:01.991618+0000","last_active":"2023-10-01T09:27:17.454713+0000","last_peered":"2023-10-01T09:27:17.454713+0000","last_clean":"2023-10-01T09:25:11.215077+0000","last_became_active":"2023-10-01T09:26:50.053084+0000","last_became_peered":"2023-10-01T09:26:50.053084+0000","last_unstale":"2023-10-01T09:27:17.454713+0000","last_undegraded":"2023-10-01T09:26:50.050534+0000","last_fullsized":"2023-10-01T09:27:17.454713+0000","mapping_epoch":295,"log_start":"265'3817","ondisk_log_start":"265'3817","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":4857,"log_dups_size":0,"ondisk_log_size":4857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:49:48.898183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223739904,"num_objects":3414,"num_object_clones":0,"num_object_copies":10242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2339,"num_objects_misplaced":4489,"num_objects_unfound":0,"num_objects_dirty":3414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8674,"num_write_kb":386816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":8323072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,4,6],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":461},{"shards":"3,4,6","objects":2953}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1120,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.959682+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:27:02.959682+0000","last_peered":"2023-10-01T09:27:02.959682+0000","last_clean":"2023-10-01T09:27:02.959682+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:27:02.959682+0000","last_undegraded":"2023-10-01T09:27:02.959682+0000","last_fullsized":"2023-10-01T09:27:02.959682+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.963170+0000","last_change":"2023-10-01T09:27:01.989188+0000","last_active":"2023-10-01T09:27:02.963170+0000","last_peered":"2023-10-01T09:27:02.963170+0000","last_clean":"2023-10-01T09:27:02.963170+0000","last_became_active":"2023-10-01T09:27:01.988878+0000","last_became_peered":"2023-10-01T09:27:01.988878+0000","last_unstale":"2023-10-01T09:27:02.963170+0000","last_undegraded":"2023-10-01T09:27:02.963170+0000","last_fullsized":"2023-10-01T09:27:02.963170+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":277,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.960910+0000","last_change":"2023-10-01T09:27:01.977155+0000","last_active":"2023-10-01T09:27:02.960910+0000","last_peered":"2023-10-01T09:27:02.960910+0000","last_clean":"2023-10-01T09:27:02.960910+0000","last_became_active":"2023-10-01T09:27:01.975803+0000","last_became_peered":"2023-10-01T09:27:01.975803+0000","last_unstale":"2023-10-01T09:27:02.960910+0000","last_undegraded":"2023-10-01T09:27:02.960910+0000","last_fullsized":"2023-10-01T09:27:02.960910+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:00:55.170662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"301'8865","reported_seq":12460,"reported_epoch":301,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:27:17.925955+0000","last_change":"2023-10-01T09:27:02.203568+0000","last_active":"2023-10-01T09:27:17.925955+0000","last_peered":"2023-10-01T09:27:17.925955+0000","last_clean":"2023-10-01T09:25:11.198588+0000","last_became_active":"2023-10-01T09:25:57.204076+0000","last_became_peered":"2023-10-01T09:25:57.204076+0000","last_unstale":"2023-10-01T09:27:17.925955+0000","last_undegraded":"2023-10-01T09:25:56.915901+0000","last_fullsized":"2023-10-01T09:25:56.855288+0000","mapping_epoch":280,"log_start":"264'3185","ondisk_log_start":"264'3185","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":5680,"log_dups_size":0,"ondisk_log_size":5680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:16:13.273952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223019008,"num_objects":3403,"num_object_clones":0,"num_object_copies":10209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":889,"num_objects_misplaced":2668,"num_objects_unfound":0,"num_objects_dirty":3403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8865,"num_write_kb":392576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1763,"num_bytes_recovered":115539968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3","objects":-154},{"shards":"0,3,6","objects":2668}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":953,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.962571+0000","last_change":"2023-10-01T09:27:01.982280+0000","last_active":"2023-10-01T09:27:02.962571+0000","last_peered":"2023-10-01T09:27:02.962571+0000","last_clean":"2023-10-01T09:27:02.962571+0000","last_became_active":"2023-10-01T09:27:01.981784+0000","last_became_peered":"2023-10-01T09:27:01.981784+0000","last_unstale":"2023-10-01T09:27:02.962571+0000","last_undegraded":"2023-10-01T09:27:02.962571+0000","last_fullsized":"2023-10-01T09:27:02.962571+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":254,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-01T09:27:14.949452+0000","last_change":"2023-10-01T09:27:01.986949+0000","last_active":"2023-10-01T09:27:14.949452+0000","last_peered":"2023-10-01T09:27:14.949452+0000","last_clean":"2023-10-01T09:27:14.949452+0000","last_became_active":"2023-10-01T09:27:01.986655+0000","last_became_peered":"2023-10-01T09:27:01.986655+0000","last_unstale":"2023-10-01T09:27:14.949452+0000","last_undegraded":"2023-10-01T09:27:14.949452+0000","last_fullsized":"2023-10-01T09:27:14.949452+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:26:09.591722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":39,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.959601+0000","last_change":"2023-10-01T09:25:25.784640+0000","last_active":"2023-10-01T09:27:02.959601+0000","last_peered":"2023-10-01T09:27:02.959601+0000","last_clean":"2023-10-01T09:27:02.959601+0000","last_became_active":"2023-10-01T09:25:25.784135+0000","last_became_peered":"2023-10-01T09:25:25.784135+0000","last_unstale":"2023-10-01T09:27:02.959601+0000","last_undegraded":"2023-10-01T09:27:02.959601+0000","last_fullsized":"2023-10-01T09:27:02.959601+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:52:33.685988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"301'8946","reported_seq":9018,"reported_epoch":301,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:27:17.927140+0000","last_change":"2023-10-01T09:27:02.098305+0000","last_active":"2023-10-01T09:27:17.927140+0000","last_peered":"2023-10-01T09:27:17.927140+0000","last_clean":"2023-10-01T09:26:11.380982+0000","last_became_active":"2023-10-01T09:27:02.067633+0000","last_became_peered":"2023-10-01T09:27:02.067633+0000","last_unstale":"2023-10-01T09:27:17.927140+0000","last_undegraded":"2023-10-01T09:27:17.927140+0000","last_fullsized":"2023-10-01T09:27:17.927140+0000","mapping_epoch":298,"log_start":"264'3184","ondisk_log_start":"264'3184","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":5762,"log_dups_size":0,"ondisk_log_size":5762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:13:40.130142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226099200,"num_objects":3450,"num_object_clones":0,"num_object_copies":10350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3450,"num_objects_unfound":0,"num_objects_dirty":3450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8946,"num_write_kb":396672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":3450}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":555,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.971574+0000","last_change":"2023-10-01T09:26:29.385302+0000","last_active":"2023-10-01T09:27:02.971574+0000","last_peered":"2023-10-01T09:27:02.971574+0000","last_clean":"2023-10-01T09:27:02.971574+0000","last_became_active":"2023-10-01T09:26:29.384738+0000","last_became_peered":"2023-10-01T09:26:29.384738+0000","last_unstale":"2023-10-01T09:27:02.971574+0000","last_undegraded":"2023-10-01T09:27:02.971574+0000","last_fullsized":"2023-10-01T09:27:02.971574+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:22:00.127736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":319,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.959358+0000","last_change":"2023-10-01T09:25:56.034502+0000","last_active":"2023-10-01T09:27:02.959358+0000","last_peered":"2023-10-01T09:27:02.959358+0000","last_clean":"2023-10-01T09:27:02.959358+0000","last_became_active":"2023-10-01T09:25:56.033935+0000","last_became_peered":"2023-10-01T09:25:56.033935+0000","last_unstale":"2023-10-01T09:27:02.959358+0000","last_undegraded":"2023-10-01T09:27:02.959358+0000","last_fullsized":"2023-10-01T09:27:02.959358+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:08:10.291396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":182,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.962438+0000","last_change":"2023-10-01T09:27:02.004900+0000","last_active":"2023-10-01T09:27:02.962438+0000","last_peered":"2023-10-01T09:27:02.962438+0000","last_clean":"2023-10-01T09:27:02.962438+0000","last_became_active":"2023-10-01T09:27:02.004501+0000","last_became_peered":"2023-10-01T09:27:02.004501+0000","last_unstale":"2023-10-01T09:27:02.962438+0000","last_undegraded":"2023-10-01T09:27:02.962438+0000","last_fullsized":"2023-10-01T09:27:02.962438+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:50:52.617833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"301'8930","reported_seq":8982,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-01T09:27:17.928130+0000","last_change":"2023-10-01T09:23:45.302734+0000","last_active":"2023-10-01T09:27:17.928130+0000","last_peered":"2023-10-01T09:27:17.928130+0000","last_clean":"2023-10-01T09:27:17.928130+0000","last_became_active":"2023-10-01T09:23:45.302228+0000","last_became_peered":"2023-10-01T09:23:45.302228+0000","last_unstale":"2023-10-01T09:27:17.928130+0000","last_undegraded":"2023-10-01T09:27:17.928130+0000","last_fullsized":"2023-10-01T09:27:17.928130+0000","mapping_epoch":259,"log_start":"264'3209","ondisk_log_start":"264'3209","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":5721,"log_dups_size":0,"ondisk_log_size":5721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:35.109203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228065280,"num_objects":3480,"num_object_clones":0,"num_object_copies":10440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8930,"num_write_kb":397120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1236,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:27:02.978010+0000","last_change":"2023-10-01T09:27:02.978010+0000","last_active":"2023-10-01T09:27:02.978010+0000","last_peered":"2023-10-01T09:27:02.978010+0000","last_clean":"2023-10-01T09:26:20.907959+0000","last_became_active":"2023-10-01T09:27:02.977431+0000","last_became_peered":"2023-10-01T09:27:02.977431+0000","last_unstale":"2023-10-01T09:27:02.978010+0000","last_undegraded":"2023-10-01T09:27:02.964350+0000","last_fullsized":"2023-10-01T09:27:02.962927+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:07:02.499097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":496,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.971677+0000","last_change":"2023-10-01T09:26:14.269228+0000","last_active":"2023-10-01T09:27:02.971677+0000","last_peered":"2023-10-01T09:27:02.971677+0000","last_clean":"2023-10-01T09:27:02.971677+0000","last_became_active":"2023-10-01T09:26:14.268773+0000","last_became_peered":"2023-10-01T09:26:14.268773+0000","last_unstale":"2023-10-01T09:27:02.971677+0000","last_undegraded":"2023-10-01T09:27:02.971677+0000","last_fullsized":"2023-10-01T09:27:02.971677+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":121,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.970360+0000","last_change":"2023-10-01T09:27:02.062269+0000","last_active":"2023-10-01T09:27:02.970360+0000","last_peered":"2023-10-01T09:27:02.970360+0000","last_clean":"2023-10-01T09:27:02.970360+0000","last_became_active":"2023-10-01T09:27:02.061945+0000","last_became_peered":"2023-10-01T09:27:02.061945+0000","last_unstale":"2023-10-01T09:27:02.970360+0000","last_undegraded":"2023-10-01T09:27:02.970360+0000","last_fullsized":"2023-10-01T09:27:02.970360+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:11:54.465425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":197,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.972621+0000","last_change":"2023-10-01T09:26:12.287335+0000","last_active":"2023-10-01T09:27:02.972621+0000","last_peered":"2023-10-01T09:27:02.972621+0000","last_clean":"2023-10-01T09:27:02.972621+0000","last_became_active":"2023-10-01T09:26:12.287087+0000","last_became_peered":"2023-10-01T09:26:12.287087+0000","last_unstale":"2023-10-01T09:27:02.972621+0000","last_undegraded":"2023-10-01T09:27:02.972621+0000","last_fullsized":"2023-10-01T09:27:02.972621+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.5","version":"301'9141","reported_seq":9191,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-01T09:27:21.502797+0000","last_change":"2023-10-01T09:23:45.303660+0000","last_active":"2023-10-01T09:27:21.502797+0000","last_peered":"2023-10-01T09:27:21.502797+0000","last_clean":"2023-10-01T09:27:21.502797+0000","last_became_active":"2023-10-01T09:23:45.303267+0000","last_became_peered":"2023-10-01T09:23:45.303267+0000","last_unstale":"2023-10-01T09:27:21.502797+0000","last_undegraded":"2023-10-01T09:27:21.502797+0000","last_fullsized":"2023-10-01T09:27:21.502797+0000","mapping_epoch":259,"log_start":"264'3185","ondisk_log_start":"264'3185","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":5956,"log_dups_size":0,"ondisk_log_size":5956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221708288,"num_objects":3383,"num_object_clones":0,"num_object_copies":10149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9141,"num_write_kb":400768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":397,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.971767+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:27:02.971767+0000","last_peered":"2023-10-01T09:27:02.971767+0000","last_clean":"2023-10-01T09:27:02.971767+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:27:02.971767+0000","last_undegraded":"2023-10-01T09:27:02.971767+0000","last_fullsized":"2023-10-01T09:27:02.971767+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":305,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.973899+0000","last_change":"2023-10-01T09:27:01.986786+0000","last_active":"2023-10-01T09:27:02.973899+0000","last_peered":"2023-10-01T09:27:02.973899+0000","last_clean":"2023-10-01T09:27:02.973899+0000","last_became_active":"2023-10-01T09:27:01.986466+0000","last_became_peered":"2023-10-01T09:27:01.986466+0000","last_unstale":"2023-10-01T09:27:02.973899+0000","last_undegraded":"2023-10-01T09:27:02.973899+0000","last_fullsized":"2023-10-01T09:27:02.973899+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:55:51.244125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":305,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.974338+0000","last_change":"2023-10-01T09:19:30.091481+0000","last_active":"2023-10-01T09:27:02.974338+0000","last_peered":"2023-10-01T09:27:02.974338+0000","last_clean":"2023-10-01T09:27:02.974338+0000","last_became_active":"2023-10-01T09:19:30.091151+0000","last_became_peered":"2023-10-01T09:19:30.091151+0000","last_unstale":"2023-10-01T09:27:02.974338+0000","last_undegraded":"2023-10-01T09:27:02.974338+0000","last_fullsized":"2023-10-01T09:27:02.974338+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:09.127408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"301'8722","reported_seq":9432,"reported_epoch":301,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:27:17.924566+0000","last_change":"2023-10-01T09:27:02.091065+0000","last_active":"2023-10-01T09:27:17.924566+0000","last_peered":"2023-10-01T09:27:17.924566+0000","last_clean":"2023-10-01T09:25:24.649292+0000","last_became_active":"2023-10-01T09:27:02.068125+0000","last_became_peered":"2023-10-01T09:27:02.068125+0000","last_unstale":"2023-10-01T09:27:17.924566+0000","last_undegraded":"2023-10-01T09:27:01.974722+0000","last_fullsized":"2023-10-01T09:27:01.959698+0000","mapping_epoch":298,"log_start":"265'3523","ondisk_log_start":"265'3523","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":5199,"log_dups_size":0,"ondisk_log_size":5199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:10:32.870297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214040576,"num_objects":3266,"num_object_clones":0,"num_object_copies":9798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2192,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8722,"num_write_kb":383616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":321,"num_bytes_recovered":21037056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":3266}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":669,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:27:02.972604+0000","last_change":"2023-10-01T09:26:13.263177+0000","last_active":"2023-10-01T09:27:02.972604+0000","last_peered":"2023-10-01T09:27:02.972604+0000","last_clean":"2023-10-01T09:26:04.908507+0000","last_became_active":"2023-10-01T09:26:13.262853+0000","last_became_peered":"2023-10-01T09:26:13.262853+0000","last_unstale":"2023-10-01T09:27:02.972604+0000","last_undegraded":"2023-10-01T09:26:13.248686+0000","last_fullsized":"2023-10-01T09:26:13.247841+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":319,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.959732+0000","last_change":"2023-10-01T09:27:01.999110+0000","last_active":"2023-10-01T09:27:02.959732+0000","last_peered":"2023-10-01T09:27:02.959732+0000","last_clean":"2023-10-01T09:27:02.959732+0000","last_became_active":"2023-10-01T09:27:01.998740+0000","last_became_peered":"2023-10-01T09:27:01.998740+0000","last_unstale":"2023-10-01T09:27:02.959732+0000","last_undegraded":"2023-10-01T09:27:02.959732+0000","last_fullsized":"2023-10-01T09:27:02.959732+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:40:08.795106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":272,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.973500+0000","last_change":"2023-10-01T09:27:01.998986+0000","last_active":"2023-10-01T09:27:02.973500+0000","last_peered":"2023-10-01T09:27:02.973500+0000","last_clean":"2023-10-01T09:27:02.973500+0000","last_became_active":"2023-10-01T09:27:01.998709+0000","last_became_peered":"2023-10-01T09:27:01.998709+0000","last_unstale":"2023-10-01T09:27:02.973500+0000","last_undegraded":"2023-10-01T09:27:02.973500+0000","last_fullsized":"2023-10-01T09:27:02.973500+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:01:14.296638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"301'8770","reported_seq":8831,"reported_epoch":301,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:27:17.537764+0000","last_change":"2023-10-01T09:26:13.383046+0000","last_active":"2023-10-01T09:27:17.537764+0000","last_peered":"2023-10-01T09:27:17.537764+0000","last_clean":"2023-10-01T09:26:11.349719+0000","last_became_active":"2023-10-01T09:26:13.328405+0000","last_became_peered":"2023-10-01T09:26:13.328405+0000","last_unstale":"2023-10-01T09:27:17.537764+0000","last_undegraded":"2023-10-01T09:27:17.537764+0000","last_fullsized":"2023-10-01T09:27:17.537764+0000","mapping_epoch":284,"log_start":"265'3835","ondisk_log_start":"265'3835","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":4935,"log_dups_size":0,"ondisk_log_size":4935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:33:03.511436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222298112,"num_objects":3392,"num_object_clones":0,"num_object_copies":10176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3392,"num_objects_unfound":0,"num_objects_dirty":3392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8770,"num_write_kb":389184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,4],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3,4","objects":3392}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":314,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.973144+0000","last_change":"2023-10-01T09:23:10.615956+0000","last_active":"2023-10-01T09:27:02.973144+0000","last_peered":"2023-10-01T09:27:02.973144+0000","last_clean":"2023-10-01T09:27:02.973144+0000","last_became_active":"2023-10-01T09:23:10.610919+0000","last_became_peered":"2023-10-01T09:23:10.610919+0000","last_unstale":"2023-10-01T09:27:02.973144+0000","last_undegraded":"2023-10-01T09:27:02.973144+0000","last_fullsized":"2023-10-01T09:27:02.973144+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:03:44.294373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":844,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-01T09:27:14.965031+0000","last_change":"2023-10-01T09:27:02.199746+0000","last_active":"2023-10-01T09:27:14.965031+0000","last_peered":"2023-10-01T09:27:14.965031+0000","last_clean":"2023-10-01T09:27:14.965031+0000","last_became_active":"2023-10-01T09:26:49.954999+0000","last_became_peered":"2023-10-01T09:26:49.954999+0000","last_unstale":"2023-10-01T09:27:14.965031+0000","last_undegraded":"2023-10-01T09:27:14.965031+0000","last_fullsized":"2023-10-01T09:27:14.965031+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:54.062991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":86,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.970808+0000","last_change":"2023-10-01T09:23:10.612309+0000","last_active":"2023-10-01T09:27:02.970808+0000","last_peered":"2023-10-01T09:27:02.970808+0000","last_clean":"2023-10-01T09:27:02.970808+0000","last_became_active":"2023-10-01T09:23:10.612034+0000","last_became_peered":"2023-10-01T09:23:10.612034+0000","last_unstale":"2023-10-01T09:27:02.970808+0000","last_undegraded":"2023-10-01T09:27:02.970808+0000","last_fullsized":"2023-10-01T09:27:02.970808+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:09:47.963186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"301'8993","reported_seq":9043,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-10-01T09:27:17.537519+0000","last_change":"2023-10-01T09:23:45.283913+0000","last_active":"2023-10-01T09:27:17.537519+0000","last_peered":"2023-10-01T09:27:17.537519+0000","last_clean":"2023-10-01T09:27:17.537519+0000","last_became_active":"2023-10-01T09:23:45.283600+0000","last_became_peered":"2023-10-01T09:23:45.283600+0000","last_unstale":"2023-10-01T09:27:17.537519+0000","last_undegraded":"2023-10-01T09:27:17.537519+0000","last_fullsized":"2023-10-01T09:27:17.537519+0000","mapping_epoch":259,"log_start":"266'4143","ondisk_log_start":"266'4143","created":259,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:23:44.270133+0000","last_clean_scrub_stamp":"2023-10-01T09:23:44.270133+0000","objects_scrubbed":0,"log_size":4850,"log_dups_size":0,"ondisk_log_size":4850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:24:30.501930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221970432,"num_objects":3387,"num_object_clones":0,"num_object_copies":10161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8993,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":121,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.972095+0000","last_change":"2023-10-01T09:27:02.012076+0000","last_active":"2023-10-01T09:27:02.972095+0000","last_peered":"2023-10-01T09:27:02.972095+0000","last_clean":"2023-10-01T09:27:02.972095+0000","last_became_active":"2023-10-01T09:25:55.868591+0000","last_became_peered":"2023-10-01T09:25:55.868591+0000","last_unstale":"2023-10-01T09:27:02.972095+0000","last_undegraded":"2023-10-01T09:27:02.972095+0000","last_fullsized":"2023-10-01T09:27:02.972095+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":297,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.974457+0000","last_change":"2023-10-01T09:26:12.291583+0000","last_active":"2023-10-01T09:27:02.974457+0000","last_peered":"2023-10-01T09:27:02.974457+0000","last_clean":"2023-10-01T09:27:02.974457+0000","last_became_active":"2023-10-01T09:26:12.271891+0000","last_became_peered":"2023-10-01T09:26:12.271891+0000","last_unstale":"2023-10-01T09:27:02.974457+0000","last_undegraded":"2023-10-01T09:27:02.974457+0000","last_fullsized":"2023-10-01T09:27:02.974457+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:53:06.889203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.962515+0000","last_change":"2023-10-01T09:27:01.993414+0000","last_active":"2023-10-01T09:27:02.962515+0000","last_peered":"2023-10-01T09:27:02.962515+0000","last_clean":"2023-10-01T09:27:02.962515+0000","last_became_active":"2023-10-01T09:27:01.993070+0000","last_became_peered":"2023-10-01T09:27:01.993070+0000","last_unstale":"2023-10-01T09:27:02.962515+0000","last_undegraded":"2023-10-01T09:27:02.962515+0000","last_fullsized":"2023-10-01T09:27:02.962515+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":251,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.973429+0000","last_change":"2023-10-01T09:26:12.271469+0000","last_active":"2023-10-01T09:27:02.973429+0000","last_peered":"2023-10-01T09:27:02.973429+0000","last_clean":"2023-10-01T09:27:02.973429+0000","last_became_active":"2023-10-01T09:26:12.270972+0000","last_became_peered":"2023-10-01T09:26:12.270972+0000","last_unstale":"2023-10-01T09:27:02.973429+0000","last_undegraded":"2023-10-01T09:27:02.973429+0000","last_fullsized":"2023-10-01T09:27:02.973429+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:45:03.822952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":329,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.959825+0000","last_change":"2023-10-01T09:22:40.166927+0000","last_active":"2023-10-01T09:27:02.959825+0000","last_peered":"2023-10-01T09:27:02.959825+0000","last_clean":"2023-10-01T09:27:02.959825+0000","last_became_active":"2023-10-01T09:22:40.166606+0000","last_became_peered":"2023-10-01T09:22:40.166606+0000","last_unstale":"2023-10-01T09:27:02.959825+0000","last_undegraded":"2023-10-01T09:27:02.959825+0000","last_fullsized":"2023-10-01T09:27:02.959825+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:54:18.044207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.962782+0000","last_change":"2023-10-01T09:27:01.973141+0000","last_active":"2023-10-01T09:27:02.962782+0000","last_peered":"2023-10-01T09:27:02.962782+0000","last_clean":"2023-10-01T09:27:02.962782+0000","last_became_active":"2023-10-01T09:27:01.970684+0000","last_became_peered":"2023-10-01T09:27:01.970684+0000","last_unstale":"2023-10-01T09:27:02.962782+0000","last_undegraded":"2023-10-01T09:27:02.962782+0000","last_fullsized":"2023-10-01T09:27:02.962782+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":257,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.970518+0000","last_change":"2023-10-01T09:26:12.368909+0000","last_active":"2023-10-01T09:27:02.970518+0000","last_peered":"2023-10-01T09:27:02.970518+0000","last_clean":"2023-10-01T09:27:02.970518+0000","last_became_active":"2023-10-01T09:26:12.368628+0000","last_became_peered":"2023-10-01T09:26:12.368628+0000","last_unstale":"2023-10-01T09:27:02.970518+0000","last_undegraded":"2023-10-01T09:27:02.970518+0000","last_fullsized":"2023-10-01T09:27:02.970518+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:01:09.686206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.963176+0000","last_change":"2023-10-01T09:27:01.993628+0000","last_active":"2023-10-01T09:27:02.963176+0000","last_peered":"2023-10-01T09:27:02.963176+0000","last_clean":"2023-10-01T09:27:02.963176+0000","last_became_active":"2023-10-01T09:27:01.978497+0000","last_became_peered":"2023-10-01T09:27:01.978497+0000","last_unstale":"2023-10-01T09:27:02.963176+0000","last_undegraded":"2023-10-01T09:27:02.963176+0000","last_fullsized":"2023-10-01T09:27:02.963176+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":239,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.970635+0000","last_change":"2023-10-01T09:26:12.309345+0000","last_active":"2023-10-01T09:27:02.970635+0000","last_peered":"2023-10-01T09:27:02.970635+0000","last_clean":"2023-10-01T09:27:02.970635+0000","last_became_active":"2023-10-01T09:26:12.304299+0000","last_became_peered":"2023-10-01T09:26:12.304299+0000","last_unstale":"2023-10-01T09:27:02.970635+0000","last_undegraded":"2023-10-01T09:27:02.970635+0000","last_fullsized":"2023-10-01T09:27:02.970635+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:35:59.915530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":92,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.972414+0000","last_change":"2023-10-01T09:25:56.039185+0000","last_active":"2023-10-01T09:27:02.972414+0000","last_peered":"2023-10-01T09:27:02.972414+0000","last_clean":"2023-10-01T09:27:02.972414+0000","last_became_active":"2023-10-01T09:25:56.038857+0000","last_became_peered":"2023-10-01T09:25:56.038857+0000","last_unstale":"2023-10-01T09:27:02.972414+0000","last_undegraded":"2023-10-01T09:27:02.972414+0000","last_fullsized":"2023-10-01T09:27:02.972414+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:09:04.070820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":76,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.971699+0000","last_change":"2023-10-01T09:22:40.170676+0000","last_active":"2023-10-01T09:27:02.971699+0000","last_peered":"2023-10-01T09:27:02.971699+0000","last_clean":"2023-10-01T09:27:02.971699+0000","last_became_active":"2023-10-01T09:22:40.170260+0000","last_became_peered":"2023-10-01T09:22:40.170260+0000","last_unstale":"2023-10-01T09:27:02.971699+0000","last_undegraded":"2023-10-01T09:27:02.971699+0000","last_fullsized":"2023-10-01T09:27:02.971699+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:14.693287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":389,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.978606+0000","last_change":"2023-10-01T09:27:02.019266+0000","last_active":"2023-10-01T09:27:02.978606+0000","last_peered":"2023-10-01T09:27:02.978606+0000","last_clean":"2023-10-01T09:27:02.978606+0000","last_became_active":"2023-10-01T09:27:02.006744+0000","last_became_peered":"2023-10-01T09:27:02.006744+0000","last_unstale":"2023-10-01T09:27:02.978606+0000","last_undegraded":"2023-10-01T09:27:02.978606+0000","last_fullsized":"2023-10-01T09:27:02.978606+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:39:21.899359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":248,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-10-01T09:27:02.959168+0000","last_change":"2023-10-01T09:26:12.316644+0000","last_active":"2023-10-01T09:27:02.959168+0000","last_peered":"2023-10-01T09:27:02.959168+0000","last_clean":"2023-10-01T09:27:02.959168+0000","last_became_active":"2023-10-01T09:26:12.316335+0000","last_became_peered":"2023-10-01T09:26:12.316335+0000","last_unstale":"2023-10-01T09:27:02.959168+0000","last_undegraded":"2023-10-01T09:27:02.959168+0000","last_fullsized":"2023-10-01T09:27:02.959168+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:42:21.184699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3560701976,"num_objects":54333,"num_object_clones":0,"num_object_copies":162999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26526,"num_objects_misplaced":17072,"num_objects_unfound":0,"num_objects_dirty":54333,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":141253,"num_write_kb":6258689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8159,"num_bytes_recovered":477691904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11899248640,"data_stored":11899240496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88941,"ondisk_log_size":88941,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":23580,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42916,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":155,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12603392,"data_stored":12583007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430521,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1404028,"kb_used_data":1102708,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92341124,"statfs":{"total":95995035648,"available":94557310976,"internally_reserved":0,"allocated":1129172992,"data_stored":1128349179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:27:21 2023","interfaces":[{"interface":"back","average":{"1min":1.893,"5min":2.903,"15min":3.8780000000000001},"min":{"1min":0.23899999999999999,"5min":0.22800000000000001,"15min":0.219},"max":{"1min":9.0920000000000005,"5min":73.284000000000006,"15min":117.855},"last":3.972},{"interface":"front","average":{"1min":3.5939999999999999,"5min":3.3340000000000001,"15min":3.8639999999999999},"min":{"1min":0.32000000000000001,"5min":0.249,"15min":0.22500000000000001},"max":{"1min":31.466999999999999,"5min":73.269000000000005,"15min":120.348},"last":6.3170000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:26:36 2023","interfaces":[{"interface":"back","average":{"1min":2.1269999999999998,"5min":2.0030000000000001,"15min":3.0409999999999999},"min":{"1min":0.248,"5min":0.24199999999999999,"15min":0.22700000000000001},"max":{"1min":9.4589999999999996,"5min":9.4589999999999996,"15min":135.023},"last":1.5049999999999999},{"interface":"front","average":{"1min":1.774,"5min":2.012,"15min":2.8260000000000001},"min":{"1min":0.33000000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.574,"5min":11.574,"15min":134.51499999999999},"last":1.3460000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:27:04 2023","interfaces":[{"interface":"back","average":{"1min":2.3809999999999998,"5min":2.5800000000000001,"15min":2.8730000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26400000000000001},"max":{"1min":7.6680000000000001,"5min":59.878,"15min":104.837},"last":2.9180000000000001},{"interface":"front","average":{"1min":1.8029999999999999,"5min":2.827,"15min":2.8140000000000001},"min":{"1min":0.27400000000000002,"5min":0.247,"15min":0.247},"max":{"1min":7.2210000000000001,"5min":60.186,"15min":106.379},"last":3.0289999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:26:36 2023","interfaces":[{"interface":"back","average":{"1min":10.952,"5min":4.1239999999999997,"15min":2.694},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26400000000000001},"max":{"1min":164.595,"5min":164.595,"15min":164.595},"last":2.9420000000000002},{"interface":"front","average":{"1min":10.960000000000001,"5min":4.0190000000000001,"15min":2.6629999999999998},"min":{"1min":0.32200000000000001,"5min":0.308,"15min":0.27200000000000002},"max":{"1min":164.60499999999999,"5min":164.60499999999999,"15min":164.60499999999999},"last":2.9279999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:27:21 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.97399999999999998,"15min":1.569},"min":{"1min":0.254,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.7000000000000002,"5min":7.4989999999999997,"15min":82.992000000000004},"last":2.7000000000000002},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.0049999999999999,"15min":1.631},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.252},"max":{"1min":3.25,"5min":7.1299999999999999,"15min":83.058999999999997},"last":0.64100000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:27:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":1.1259999999999999,"15min":2.464},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.5800000000000001,"5min":13.265000000000001,"15min":13.265000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":1.171,"15min":2.319},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.93000000000000005,"5min":13.292999999999999,"15min":13.292999999999999},"last":0.628}]},{"osd":6,"last update":"Sun Oct 1 09:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.335,"15min":1.2749999999999999},"min":{"1min":0.36899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":17.962,"5min":17.962,"15min":17.962},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.042,"5min":1.3899999999999999,"15min":1.2210000000000001},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.0389999999999997,"5min":9.4120000000000008,"15min":11.871},"last":0.504}]}]},{"osd":6,"up_from":0,"seq":1262720385031,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":94,"seq":403726926008,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1708940,"kb_used_data":1324420,"kb_used_omap":7,"kb_used_meta":384312,"kb_avail":92036212,"statfs":{"total":95995035648,"available":94245081088,"internally_reserved":0,"allocated":1356206080,"data_stored":1354789261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":393535911},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:26:31 2023","interfaces":[{"interface":"back","average":{"1min":4.4020000000000001,"5min":2.726,"15min":3.4279999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.17499999999999999},"max":{"1min":20.065999999999999,"5min":37.101999999999997,"15min":174.40199999999999},"last":2.0499999999999998},{"interface":"front","average":{"1min":5.3499999999999996,"5min":3.4449999999999998,"15min":3.7210000000000001},"min":{"1min":0.24299999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":20.945,"5min":36.795999999999999,"15min":173.77000000000001},"last":3.165}]},{"osd":2,"last update":"Sun Oct 1 09:27:07 2023","interfaces":[{"interface":"back","average":{"1min":10.702999999999999,"5min":5.218,"15min":2.766},"min":{"1min":0.27400000000000002,"5min":0.182,"15min":0.182},"max":{"1min":44.915999999999997,"5min":77.325000000000003,"15min":77.325000000000003},"last":0.23799999999999999},{"interface":"front","average":{"1min":10.473000000000001,"5min":5.7300000000000004,"15min":3.0409999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.19800000000000001},"max":{"1min":48.590000000000003,"5min":80.361000000000004,"15min":80.361000000000004},"last":2.8610000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:26:41 2023","interfaces":[{"interface":"back","average":{"1min":6.2030000000000003,"5min":4.3639999999999999,"15min":2.8119999999999998},"min":{"1min":0.312,"5min":0.224,"15min":0.218},"max":{"1min":29.66,"5min":63.435000000000002,"15min":99.421000000000006},"last":2.8980000000000001},{"interface":"front","average":{"1min":4.9370000000000003,"5min":3.9470000000000001,"15min":2.5699999999999998},"min":{"1min":0.32700000000000001,"5min":0.21199999999999999,"15min":0.20999999999999999},"max":{"1min":17.648,"5min":62.945,"15min":99.364999999999995},"last":0.86899999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:26:31 2023","interfaces":[{"interface":"back","average":{"1min":2.5590000000000002,"5min":1.5760000000000001,"15min":1.4710000000000001},"min":{"1min":0.34100000000000003,"5min":0.28799999999999998,"15min":0.27800000000000002},"max":{"1min":10.231,"5min":10.231,"15min":40.673999999999999},"last":1.4810000000000001},{"interface":"front","average":{"1min":2.5110000000000001,"5min":1.8069999999999999,"15min":1.6259999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.25800000000000001},"max":{"1min":14.167,"5min":14.167,"15min":40.810000000000002},"last":0.29799999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:27:07 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":1.601,"15min":1.4690000000000001},"min":{"1min":0.39700000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.273,"5min":10.273,"15min":10.273},"last":1.395},{"interface":"front","average":{"1min":3.6659999999999999,"5min":2.089,"15min":2.3210000000000002},"min":{"1min":0.35499999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":18.265000000000001,"5min":18.265000000000001,"15min":18.265000000000001},"last":0.32400000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:24:26 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":1.7030000000000001,"15min":1.3740000000000001},"min":{"1min":0.47299999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.0880000000000001,"5min":14.452,"15min":14.452},"last":2.9489999999999998},{"interface":"front","average":{"1min":0.68600000000000005,"5min":1.5720000000000001,"15min":1.2290000000000001},"min":{"1min":0.36099999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.093,"5min":9.9740000000000002,"15min":10.787000000000001},"last":1.4330000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:26:31 2023","interfaces":[{"interface":"back","average":{"1min":2.5779999999999998,"5min":1.992,"15min":1.639},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":10.305999999999999,"5min":21.399000000000001,"15min":23.710999999999999},"last":1.458},{"interface":"front","average":{"1min":2.4350000000000001,"5min":2.0310000000000001,"15min":1.6399999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":14.217000000000001,"5min":22.318999999999999,"15min":23.736999999999998},"last":0.34599999999999997}]}]},{"osd":0,"up_from":89,"seq":382252089542,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2250484,"kb_used_data":1774380,"kb_used_omap":8,"kb_used_meta":476087,"kb_avail":91494668,"statfs":{"total":95995035648,"available":93690540032,"internally_reserved":0,"allocated":1816965120,"data_stored":1815517971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":487514050},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:26:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":2.2949999999999999,"15min":2.448},"min":{"1min":0.22800000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":8.0399999999999991,"5min":47.244999999999997,"15min":187.41399999999999},"last":2.8340000000000001},{"interface":"front","average":{"1min":4.532,"5min":3.2519999999999998,"15min":2.8479999999999999},"min":{"1min":0.21199999999999999,"5min":0.20200000000000001,"15min":0.193},"max":{"1min":16.815000000000001,"5min":48.042000000000002,"15min":187.78700000000001},"last":2.9169999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:27:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7689999999999999,"5min":2.1549999999999998,"15min":1.923},"min":{"1min":0.26000000000000001,"5min":0.224,"15min":0.224},"max":{"1min":11.468999999999999,"5min":17.731000000000002,"15min":39.048999999999999},"last":2.8519999999999999},{"interface":"front","average":{"1min":4.1040000000000001,"5min":3.1869999999999998,"15min":2.444},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":19.945,"5min":21.611000000000001,"15min":38.945999999999998},"last":2.9620000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:26:34 2023","interfaces":[{"interface":"back","average":{"1min":3.8220000000000001,"5min":2.7869999999999999,"15min":2.1240000000000001},"min":{"1min":0.249,"5min":0.24299999999999999,"15min":0.23400000000000001},"max":{"1min":19.850999999999999,"5min":41.840000000000003,"15min":85.840000000000003},"last":2.7109999999999999},{"interface":"front","average":{"1min":4.8799999999999999,"5min":3.323,"15min":2.379},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":13.555999999999999,"5min":42.023000000000003,"15min":86.046000000000006},"last":3.387}]},{"osd":4,"last update":"Sun Oct 1 09:26:26 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":1.722,"15min":2.093},"min":{"1min":0.29199999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":9.0299999999999994,"5min":9.0299999999999994,"15min":112.52200000000001},"last":2.9950000000000001},{"interface":"front","average":{"1min":2.1280000000000001,"5min":1.772,"15min":2.1400000000000001},"min":{"1min":0.33400000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":9.5440000000000005,"5min":9.5440000000000005,"15min":113.124},"last":1.3919999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:27:16 2023","interfaces":[{"interface":"back","average":{"1min":5.0060000000000002,"5min":2.1059999999999999,"15min":1.9339999999999999},"min":{"1min":0.46200000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":28.074000000000002,"5min":28.074000000000002,"15min":28.074000000000002},"last":1.419},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.3939999999999999,"15min":1.77},"min":{"1min":0.40699999999999997,"5min":0.31,"15min":0.31},"max":{"1min":11.351000000000001,"5min":11.351000000000001,"15min":11.351000000000001},"last":2.8769999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":1.5209999999999999,"15min":1.585},"min":{"1min":0.44600000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.2570000000000001,"5min":9.0239999999999991,"15min":34.375999999999998},"last":2.9039999999999999},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.472,"15min":1.4119999999999999},"min":{"1min":0.38300000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.7440000000000002,"5min":9.0820000000000007,"15min":31.745000000000001},"last":3.02}]},{"osd":7,"last update":"Sun Oct 1 09:26:26 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":1.468,"15min":1.9610000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.29199999999999998},"max":{"1min":4.4660000000000002,"5min":13.096,"15min":98.283000000000001},"last":2.952},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.8520000000000001,"15min":2.1040000000000001},"min":{"1min":0.41599999999999998,"5min":0.36099999999999999,"15min":0.312},"max":{"1min":9.0630000000000006,"5min":9.0630000000000006,"15min":98.311999999999998},"last":3.0449999999999999}]}]},{"osd":2,"up_from":99,"seq":425201762482,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1761952,"kb_used_data":1293336,"kb_used_omap":11,"kb_used_meta":468596,"kb_avail":91983200,"statfs":{"total":95995035648,"available":94190796800,"internally_reserved":0,"allocated":1324376064,"data_stored":1323005323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":479843191},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:27:17 2023","interfaces":[{"interface":"back","average":{"1min":16.838000000000001,"5min":7.7469999999999999,"15min":4.0839999999999996},"min":{"1min":0.251,"5min":0.19800000000000001,"15min":0.17599999999999999},"max":{"1min":138.107,"5min":138.107,"15min":138.107},"last":0.29799999999999999},{"interface":"front","average":{"1min":18.317,"5min":8.6959999999999997,"15min":4.7329999999999997},"min":{"1min":0.23899999999999999,"5min":0.20699999999999999,"15min":0.20100000000000001},"max":{"1min":138.20599999999999,"5min":138.20599999999999,"15min":138.20599999999999},"last":7.8949999999999996}]},{"osd":1,"last update":"Sun Oct 1 09:27:12 2023","interfaces":[{"interface":"back","average":{"1min":26.169,"5min":8.9199999999999999,"15min":4.867},"min":{"1min":0.252,"5min":0.19600000000000001,"15min":0.193},"max":{"1min":209.922,"5min":209.922,"15min":209.922},"last":10.715},{"interface":"front","average":{"1min":28.276,"5min":10.066000000000001,"15min":5.4269999999999996},"min":{"1min":0.222,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":212.714,"5min":212.952,"15min":212.952},"last":7.9260000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:26:35 2023","interfaces":[{"interface":"back","average":{"1min":11.483000000000001,"5min":5.6760000000000002,"15min":4.1630000000000003},"min":{"1min":0.53700000000000003,"5min":0.21099999999999999,"15min":0.19600000000000001},"max":{"1min":87.715999999999994,"5min":138.85900000000001,"15min":253.238},"last":1.722},{"interface":"front","average":{"1min":11.507,"5min":5.133,"15min":3.9929999999999999},"min":{"1min":0.33100000000000002,"5min":0.26400000000000001,"15min":0.22600000000000001},"max":{"1min":95.275999999999996,"5min":136.66399999999999,"15min":252.54900000000001},"last":10.669}]},{"osd":4,"last update":"Sun Oct 1 09:27:12 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":2.6259999999999999,"15min":1.8899999999999999},"min":{"1min":0.32800000000000001,"5min":0.27200000000000002,"15min":0.27000000000000002},"max":{"1min":5.5739999999999998,"5min":54.231000000000002,"15min":54.231000000000002},"last":3.9060000000000001},{"interface":"front","average":{"1min":2.4569999999999999,"5min":2.4470000000000001,"15min":1.905},"min":{"1min":0.33400000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.7949999999999999,"5min":53.133000000000003,"15min":53.133000000000003},"last":1.6499999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:27:12 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":1.5309999999999999,"15min":2.1960000000000002},"min":{"1min":0.39300000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.8490000000000002,"5min":13.895,"15min":13.895},"last":0.33300000000000002},{"interface":"front","average":{"1min":2.0209999999999999,"5min":2.0419999999999998,"15min":2.2519999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":5.5999999999999996,"5min":16.943000000000001,"15min":16.943000000000001},"last":3.9889999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:25:10 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":1.7869999999999999,"15min":1.5489999999999999},"min":{"1min":0.442,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.282,"5min":14.411,"15min":17.591000000000001},"last":1.6870000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.637,"15min":1.2270000000000001},"min":{"1min":0.42099999999999999,"5min":0.374,"15min":0.34200000000000003},"max":{"1min":0.89600000000000002,"5min":8.6620000000000008,"15min":17.736000000000001},"last":0.36599999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:27:12 2023","interfaces":[{"interface":"back","average":{"1min":3.8340000000000001,"5min":2.7549999999999999,"15min":2.2949999999999999},"min":{"1min":0.41399999999999998,"5min":0.34999999999999998,"15min":0.32800000000000001},"max":{"1min":41.164999999999999,"5min":51.840000000000003,"15min":109.506},"last":0.39400000000000002},{"interface":"front","average":{"1min":3.8740000000000001,"5min":3.1509999999999998,"15min":2.573},"min":{"1min":0.35299999999999998,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":41.188000000000002,"5min":51.908999999999999,"15min":109.58799999999999},"last":3.9580000000000002}]}]},{"osd":3,"up_from":104,"seq":446676598955,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2389508,"kb_used_data":1980924,"kb_used_omap":3,"kb_used_meta":408572,"kb_avail":91355644,"statfs":{"total":95995035648,"available":93548179456,"internally_reserved":0,"allocated":2028466176,"data_stored":2027055332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":418378471},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:26:38 2023","interfaces":[{"interface":"back","average":{"1min":4.8470000000000004,"5min":4.3440000000000003,"15min":4.0499999999999998},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.17100000000000001},"max":{"1min":18.981999999999999,"5min":93.509,"15min":199.422},"last":0.29699999999999999},{"interface":"front","average":{"1min":3.625,"5min":4.2279999999999998,"15min":4.1879999999999997},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":17.029,"5min":91.537999999999997,"15min":199.91200000000001},"last":0.625}]},{"osd":1,"last update":"Sun Oct 1 09:26:38 2023","interfaces":[{"interface":"back","average":{"1min":5.6890000000000001,"5min":5.3109999999999999,"15min":4.5119999999999996},"min":{"1min":0.23999999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":28.277000000000001,"5min":120.79300000000001,"15min":228.60499999999999},"last":0.85099999999999998},{"interface":"front","average":{"1min":8.3160000000000007,"5min":5.6680000000000001,"15min":4.5300000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.215},"max":{"1min":33.982999999999997,"5min":121.455,"15min":228.28399999999999},"last":0.51500000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:26:38 2023","interfaces":[{"interface":"back","average":{"1min":6.7329999999999997,"5min":4.5330000000000004,"15min":2.5390000000000001},"min":{"1min":0.34100000000000003,"5min":0.22500000000000001,"15min":0.224},"max":{"1min":23.489000000000001,"5min":62.909999999999997,"15min":62.909999999999997},"last":5.6449999999999996},{"interface":"front","average":{"1min":6.524,"5min":4.6189999999999998,"15min":2.746},"min":{"1min":0.30299999999999999,"5min":0.23999999999999999,"15min":0.20399999999999999},"max":{"1min":16.306000000000001,"5min":62.802,"15min":62.802},"last":3.3420000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:26:38 2023","interfaces":[{"interface":"back","average":{"1min":3.9449999999999998,"5min":3.1520000000000001,"15min":2.6269999999999998},"min":{"1min":0.35999999999999999,"5min":0.26600000000000001,"15min":0.25900000000000001},"max":{"1min":16,"5min":108.896,"15min":154.00399999999999},"last":0.371},{"interface":"front","average":{"1min":2.528,"5min":3.3340000000000001,"15min":2.903},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":15.257999999999999,"5min":108.916,"15min":156.87700000000001},"last":0.70099999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:27:11 2023","interfaces":[{"interface":"back","average":{"1min":2.1720000000000002,"5min":2.012,"15min":1.855},"min":{"1min":0.39200000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":15.351000000000001,"5min":15.351000000000001,"15min":15.351000000000001},"last":0.67200000000000004},{"interface":"front","average":{"1min":2.198,"5min":2.4500000000000002,"15min":3.379},"min":{"1min":0.40000000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":15.382,"5min":36.984000000000002,"15min":36.984000000000002},"last":0.72899999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:24:33 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":3.198,"15min":2.052},"min":{"1min":0.36499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.9620000000000002,"5min":72.433999999999997,"15min":72.433999999999997},"last":5.7130000000000001},{"interface":"front","average":{"1min":1.1459999999999999,"5min":2.9430000000000001,"15min":1.8440000000000001},"min":{"1min":0.41999999999999998,"5min":0.32600000000000001,"15min":0.32100000000000001},"max":{"1min":3.3260000000000001,"5min":72.450999999999993,"15min":72.450999999999993},"last":5.7830000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:26:38 2023","interfaces":[{"interface":"back","average":{"1min":4.4589999999999996,"5min":2.746,"15min":2.0819999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":23.536000000000001,"5min":37.024000000000001,"15min":79.412999999999997},"last":5.6859999999999999},{"interface":"front","average":{"1min":4.0380000000000003,"5min":2.7160000000000002,"15min":2.0990000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.308},"max":{"1min":16.053999999999998,"5min":36.762,"15min":79.527000000000001},"last":0.54100000000000004}]}]},{"osd":4,"up_from":0,"seq":128849019205,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":227,"seq":974957576257,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1675492,"kb_used_data":1348060,"kb_used_omap":8,"kb_used_meta":327415,"kb_avail":92069660,"statfs":{"total":95995035648,"available":94279331840,"internally_reserved":0,"allocated":1380413440,"data_stored":1379609037,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":335273732},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":14.468,"5min":4.5149999999999997,"15min":2.871},"min":{"1min":0.23300000000000001,"5min":0.223,"15min":0.223},"max":{"1min":169.517,"5min":169.517,"15min":169.517},"last":0.23200000000000001},{"interface":"front","average":{"1min":15.448,"5min":4.6150000000000002,"15min":3.347},"min":{"1min":0.28799999999999998,"5min":0.219,"15min":0.219},"max":{"1min":169.69900000000001,"5min":169.69900000000001,"15min":169.69900000000001},"last":3.6699999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":2.617,"5min":4.3760000000000003,"15min":2.9710000000000001},"min":{"1min":0.30099999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":19.550999999999998,"5min":115.976,"15min":115.976},"last":0.28299999999999997},{"interface":"front","average":{"1min":2.4489999999999998,"5min":4.2880000000000003,"15min":3.6320000000000001},"min":{"1min":0.28699999999999998,"5min":0.253,"15min":0.253},"max":{"1min":19.439,"5min":115.732,"15min":115.732},"last":0.34300000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":2.3610000000000002,"5min":2.5760000000000001,"15min":2.7839999999999998},"min":{"1min":0.30599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.234999999999999,"5min":44.615000000000002,"15min":44.615000000000002},"last":3.569},{"interface":"front","average":{"1min":2.6120000000000001,"5min":2.7250000000000001,"15min":2.081},"min":{"1min":0.26700000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.7809999999999997,"5min":44.511000000000003,"15min":44.511000000000003},"last":0.40699999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":15.691000000000001,"5min":7.1020000000000003,"15min":3.8500000000000001},"min":{"1min":0.40699999999999997,"5min":0.313,"15min":0.313},"max":{"1min":214.73599999999999,"5min":214.73599999999999,"15min":214.73599999999999},"last":2.3679999999999999},{"interface":"front","average":{"1min":16.024000000000001,"5min":7.2480000000000002,"15min":3.8650000000000002},"min":{"1min":0.32500000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":214.86699999999999,"5min":214.86699999999999,"15min":214.86699999999999},"last":1.716}]},{"osd":4,"last update":"Sun Oct 1 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.80600000000000005,"15min":1.5289999999999999},"min":{"1min":0.26800000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.66500000000000004,"5min":7.532,"15min":7.532},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.443,"5min":0.78800000000000003,"15min":1.2789999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4710000000000001,"5min":4.117,"15min":4.117},"last":0.311}]},{"osd":6,"last update":"Sun Oct 1 09:25:08 2023","interfaces":[{"interface":"back","average":{"1min":9.6859999999999999,"5min":3.278,"15min":2.391},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":176.66,"5min":176.66,"15min":176.66},"last":0.33400000000000002},{"interface":"front","average":{"1min":9.7769999999999992,"5min":2.9860000000000002,"15min":1.929},"min":{"1min":0.33200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":176.68000000000001,"5min":176.68000000000001,"15min":176.68000000000001},"last":0.53500000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:27:10 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.88100000000000001,"15min":1.3999999999999999},"min":{"1min":0.30099999999999999,"5min":0.252,"15min":0.252},"max":{"1min":1.5249999999999999,"5min":5.7190000000000003,"15min":5.7190000000000003},"last":0.372},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.876,"15min":1.78},"min":{"1min":0.307,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.63500000000000001,"5min":10.836,"15min":10.836},"last":0.56999999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1807552512,"data_stored":1807548440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1344147456,"data_stored":1344143384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1314914304,"data_stored":1314914304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2005860352,"data_stored":2005860352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1231355904,"data_stored":1231355904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1377435648,"data_stored":1377435648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1702559744,"data_stored":1702559744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1115422720,"data_stored":1115422720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:27:25.711 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 1 2023-10-01T09:27:25.711 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:25.743 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:25.801 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:25.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:25.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:25.912 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:27:25.946 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:26.051 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:27:26.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:26.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:27:26.158 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:26.189 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:26.241 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:26.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:26.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:26.348 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:26.357 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:26 smithi138 ceph-mon[185886]: pgmap v804: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 464 op/s; 26549/156843 objects degraded (16.927%); 16192/156843 objects misplaced (10.324%); 3.1 MiB/s, 152 objects/s recovering 2023-10-01T09:27:26.357 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:26.357 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:26.378 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:26.444 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:26 smithi190 ceph-mon[160435]: pgmap v804: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 464 op/s; 26549/156843 objects degraded (16.927%); 16192/156843 objects misplaced (10.324%); 3.1 MiB/s, 152 objects/s recovering 2023-10-01T09:27:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:26.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:26 smithi138 ceph-mon[192513]: pgmap v804: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 464 op/s; 26549/156843 objects degraded (16.927%); 16192/156843 objects misplaced (10.324%); 3.1 MiB/s, 152 objects/s recovering 2023-10-01T09:27:26.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:26.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:27.657 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:27.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:27.674 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:27 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 26549/156843 objects degraded (16.927%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:27.677 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:27 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 26549/156843 objects degraded (16.927%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:27.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:27.778 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:27.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:27 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 26549/156843 objects degraded (16.927%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:27.850 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:28.028 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:28.125 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:28.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:28.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:28.242 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:27:28.369 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:28.502 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:28.653 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:28 smithi138 ceph-mon[185886]: pgmap v805: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 492 op/s; 26404/153780 objects degraded (17.170%); 18877/153780 objects misplaced (12.275%); 3.6 MiB/s, 186 objects/s recovering 2023-10-01T09:27:28.654 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:28 smithi138 ceph-mon[192513]: pgmap v805: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 492 op/s; 26404/153780 objects degraded (17.170%); 18877/153780 objects misplaced (12.275%); 3.6 MiB/s, 186 objects/s recovering 2023-10-01T09:27:28.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:28 smithi190 ceph-mon[160435]: pgmap v805: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 492 op/s; 26404/153780 objects degraded (17.170%); 18877/153780 objects misplaced (12.275%); 3.6 MiB/s, 186 objects/s recovering 2023-10-01T09:27:28.833 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:27:28.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:28.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:27:28.948 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:29.068 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:30.115 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:30.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:30.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:30.222 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:27:30.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:29 smithi190 ceph-mon[160435]: pgmap v806: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 26404/150264 objects degraded (17.572%); 18170/150264 objects misplaced (12.092%); 5.1 MiB/s, 169 objects/s recovering 2023-10-01T09:27:30.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:29 smithi138 ceph-mon[185886]: pgmap v806: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 26404/150264 objects degraded (17.572%); 18170/150264 objects misplaced (12.092%); 5.1 MiB/s, 169 objects/s recovering 2023-10-01T09:27:30.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:29 smithi138 ceph-mon[192513]: pgmap v806: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 26404/150264 objects degraded (17.572%); 18170/150264 objects misplaced (12.092%); 5.1 MiB/s, 169 objects/s recovering 2023-10-01T09:27:30.337 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:30.754 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:27:30.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:30.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:27:30.865 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:30.895 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:30.956 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:30.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:30.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:30.966 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:27:31.066 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:27:31.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:30 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3682298133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:27:31.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:30 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:27:31.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:30 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3682298133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:27:31.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:30 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:27:31.097 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:31.146 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:27:31.152 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:30 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3682298133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:27:31.152 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:30 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:27:31.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:31.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:27:31.254 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:31.283 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:31.348 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:31.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:31.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:31.451 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:31.551 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:31 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:27:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:31 smithi190 ceph-mon[160435]: failed to encode map e302 with expected crc 2023-10-01T09:27:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:31 smithi190 ceph-mon[160435]: failed to encode map e302 with expected crc 2023-10-01T09:27:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:31 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:27:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:31 smithi190 ceph-mon[160435]: osdmap e302: 8 total, 8 up, 6 in 2023-10-01T09:27:32.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:31 smithi190 ceph-mon[160435]: pgmap v808: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 473 op/s; 26243/148854 objects degraded (17.630%); 18170/148854 objects misplaced (12.207%); 5.6 MiB/s, 194 objects/s recovering 2023-10-01T09:27:32.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:31 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:27:32.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:31 smithi138 ceph-mon[192513]: failed to encode map e302 with expected crc 2023-10-01T09:27:32.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:31 smithi138 ceph-mon[192513]: failed to encode map e302 with expected crc 2023-10-01T09:27:32.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:31 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:27:32.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:31 smithi138 ceph-mon[192513]: osdmap e302: 8 total, 8 up, 6 in 2023-10-01T09:27:32.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:31 smithi138 ceph-mon[192513]: pgmap v808: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 473 op/s; 26243/148854 objects degraded (17.630%); 18170/148854 objects misplaced (12.207%); 5.6 MiB/s, 194 objects/s recovering 2023-10-01T09:27:32.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:31 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:27:32.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:31 smithi138 ceph-mon[185886]: failed to encode map e302 with expected crc 2023-10-01T09:27:32.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:31 smithi138 ceph-mon[185886]: failed to encode map e302 with expected crc 2023-10-01T09:27:32.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:31 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:27:32.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:31 smithi138 ceph-mon[185886]: osdmap e302: 8 total, 8 up, 6 in 2023-10-01T09:27:32.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:31 smithi138 ceph-mon[185886]: pgmap v808: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 3.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 473 op/s; 26243/148854 objects degraded (17.630%); 18170/148854 objects misplaced (12.207%); 5.6 MiB/s, 194 objects/s recovering 2023-10-01T09:27:33.202 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:33.215 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:32 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 26243/148854 objects degraded (17.630%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:33.216 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:32 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 26243/148854 objects degraded (17.630%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:33.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:33.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:33.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:32 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 26243/148854 objects degraded (17.630%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:33.317 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:33.412 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:34.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:33 smithi190 ceph-mon[160435]: pgmap v809: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.9 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 26243/142773 objects degraded (18.381%); 16380/142773 objects misplaced (11.473%); 10 MiB/s, 209 objects/s recovering 2023-10-01T09:27:34.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:33 smithi138 ceph-mon[185886]: pgmap v809: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.9 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 26243/142773 objects degraded (18.381%); 16380/142773 objects misplaced (11.473%); 10 MiB/s, 209 objects/s recovering 2023-10-01T09:27:34.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:34 smithi138 ceph-mon[192513]: pgmap v809: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.9 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 26243/142773 objects degraded (18.381%); 16380/142773 objects misplaced (11.473%); 10 MiB/s, 209 objects/s recovering 2023-10-01T09:27:34.524 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:34.544 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:27:34.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:34.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:34.605 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:27:34.646 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:34.647 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:27:34.704 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:27:34.742 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:27:34.757 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:34.782 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:27:34.823 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:27:34.872 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:27:34.899 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:27:34.899 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:27:34.899 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:27:34.899 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:27:34.900 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:27:34.900 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:27:34.900 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:27:34.900 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:27:34.947 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:34.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:34.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:35.064 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:35.084 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:27:35.085 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:27:35.145 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:35.203 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/506490257' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:35.204 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:27:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:27:34] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.33.4" 2023-10-01T09:27:35.204 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/506490257' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:35.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/506490257' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:36.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:36 smithi190 ceph-mon[160435]: pgmap v810: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.9 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 26243/142305 objects degraded (18.441%); 16380/142305 objects misplaced (11.510%); 10 MiB/s, 209 objects/s recovering 2023-10-01T09:27:36.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:36 smithi138 ceph-mon[185886]: pgmap v810: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.9 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 26243/142305 objects degraded (18.441%); 16380/142305 objects misplaced (11.510%); 10 MiB/s, 209 objects/s recovering 2023-10-01T09:27:36.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:36 smithi138 ceph-mon[192513]: pgmap v810: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.9 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 26243/142305 objects degraded (18.441%); 16380/142305 objects misplaced (11.510%); 10 MiB/s, 209 objects/s recovering 2023-10-01T09:27:36.576 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:27:37.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:37 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 26243/142305 objects degraded (18.441%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:37.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:37 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 26243/142305 objects degraded (18.441%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:37.767 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:37.771 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:27:37.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:37.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:37.797 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:37 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 26243/142305 objects degraded (18.441%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:37.878 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:38.056 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:38.192 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:27:38.289 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T09:27:38.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:38.359 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:38.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:38.460 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:27:38.497 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:38.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:38 smithi138 ceph-mon[185886]: pgmap v811: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 494 op/s; 26078/138942 objects degraded (18.769%); 15732/138942 objects misplaced (11.323%); 10 MiB/s, 167 objects/s recovering 2023-10-01T09:27:38.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/751426875' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:27:38.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:38 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:27:38.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:38 smithi138 ceph-mon[192513]: pgmap v811: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 494 op/s; 26078/138942 objects degraded (18.769%); 15732/138942 objects misplaced (11.323%); 10 MiB/s, 167 objects/s recovering 2023-10-01T09:27:38.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/751426875' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:27:38.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:38 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:27:38.782 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:38 smithi190 ceph-mon[160435]: pgmap v811: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.8 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 494 op/s; 26078/138942 objects degraded (18.769%); 15732/138942 objects misplaced (11.323%); 10 MiB/s, 167 objects/s recovering 2023-10-01T09:27:38.783 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/751426875' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:27:38.783 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:38 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-10-01T09:27:38.811 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:27:38.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:38.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:27:38.915 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:39.026 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:39.445 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:27:39.730 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:39.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:39.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:39.840 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:39.855 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:39.892 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T09:27:39.964 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:40.222 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:40.225 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:39 smithi138 ceph-mon[192513]: pgmap v812: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.8 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 26078/135351 objects degraded (19.267%); 14568/135351 objects misplaced (10.763%); 14 MiB/s, 216 objects/s recovering 2023-10-01T09:27:40.225 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:39 smithi138 ceph-mon[185886]: pgmap v812: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.8 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 26078/135351 objects degraded (19.267%); 14568/135351 objects misplaced (10.763%); 14 MiB/s, 216 objects/s recovering 2023-10-01T09:27:40.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:40.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:40.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:39 smithi190 ceph-mon[160435]: pgmap v812: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.8 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 26078/135351 objects degraded (19.267%); 14568/135351 objects misplaced (10.763%); 14 MiB/s, 216 objects/s recovering 2023-10-01T09:27:40.327 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:27:40.413 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:40.806 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:27:40.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:40.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:27:40.909 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:27:40.974 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:41.116 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:27:41.117 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:27:41.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:27:41.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:27:41.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:27:41.852 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:27:42.112 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:27:42.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:42.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:27:42.222 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:27:42.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:41 smithi138 ceph-mon[192513]: pgmap v813: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.7 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 25927/133980 objects degraded (19.351%); 14568/133980 objects misplaced (10.873%); 13 MiB/s, 212 objects/s recovering 2023-10-01T09:27:42.223 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:42.223 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:41 smithi138 ceph-mon[185886]: pgmap v813: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.7 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 25927/133980 objects degraded (19.351%); 14568/133980 objects misplaced (10.873%); 13 MiB/s, 212 objects/s recovering 2023-10-01T09:27:42.251 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:41 smithi190 ceph-mon[160435]: pgmap v813: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.7 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 25927/133980 objects degraded (19.351%); 14568/133980 objects misplaced (10.873%); 13 MiB/s, 212 objects/s recovering 2023-10-01T09:27:42.254 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:42.308 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:27:42.316 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:27:42.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:42.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:27:42.422 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:42.520 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:42.711 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:42.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:42.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:42.819 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:27:42.925 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:43.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:42 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 25927/133980 objects degraded (19.351%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:43.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:42 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 25927/133980 objects degraded (19.351%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:43.204 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:27:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:43.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:27:43.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:42 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 25927/133980 objects degraded (19.351%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:43.310 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:43.346 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:43.552 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:43.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:43.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:43.658 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:43.745 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:44.011 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:27:44.012 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:27:44.036 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:43 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1346245975' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:27:44.037 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:43 smithi190 ceph-mon[160435]: pgmap v814: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.6 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 589 op/s; 25927/127623 objects degraded (20.315%); 12909/127623 objects misplaced (10.115%); 16 MiB/s, 248 objects/s recovering 2023-10-01T09:27:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:43 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1346245975' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:27:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:43 smithi138 ceph-mon[185886]: pgmap v814: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.6 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 589 op/s; 25927/127623 objects degraded (20.315%); 12909/127623 objects misplaced (10.115%); 16 MiB/s, 248 objects/s recovering 2023-10-01T09:27:44.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:43 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1346245975' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:27:44.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:43 smithi138 ceph-mon[192513]: pgmap v814: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.6 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 589 op/s; 25927/127623 objects degraded (20.315%); 12909/127623 objects misplaced (10.115%); 16 MiB/s, 248 objects/s recovering 2023-10-01T09:27:44.569 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:27:44.677 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:27:44.894 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5] out_osds: [6, 4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2023-10-01T09:27:44.894 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:27:44.894 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-01T09:27:44.895 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:27:44.895 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:27:44.895 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-01T09:27:45.023 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:27:45.107 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:45.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:45.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:45.219 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:27:45.223 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:44 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:27:45.223 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:44 smithi190 ceph-mon[160435]: failed to encode map e303 with expected crc 2023-10-01T09:27:45.223 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:44 smithi190 ceph-mon[160435]: failed to encode map e303 with expected crc 2023-10-01T09:27:45.223 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:44 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1346245975' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:27:45.223 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:44 smithi190 ceph-mon[160435]: osdmap e303: 8 total, 8 up, 6 in 2023-10-01T09:27:45.223 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:44 smithi190 ceph-mon[160435]: failed to encode map e303 with expected crc 2023-10-01T09:27:45.224 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:44 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:45.224 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:44 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:45.276 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:27:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:27:44] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.33.4" 2023-10-01T09:27:45.277 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:27:45.277 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[192513]: failed to encode map e303 with expected crc 2023-10-01T09:27:45.277 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[192513]: failed to encode map e303 with expected crc 2023-10-01T09:27:45.277 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1346245975' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:27:45.277 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[192513]: osdmap e303: 8 total, 8 up, 6 in 2023-10-01T09:27:45.277 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[192513]: failed to encode map e303 with expected crc 2023-10-01T09:27:45.277 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:45.277 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:45.278 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:27:45.278 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[185886]: failed to encode map e303 with expected crc 2023-10-01T09:27:45.278 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[185886]: failed to encode map e303 with expected crc 2023-10-01T09:27:45.278 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1346245975' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:27:45.278 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[185886]: osdmap e303: 8 total, 8 up, 6 in 2023-10-01T09:27:45.278 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[185886]: failed to encode map e303 with expected crc 2023-10-01T09:27:45.278 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:45.278 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:44 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:45.280 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:45.822 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:27:45.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:45.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:27:45.934 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:46.048 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:46.221 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:46 smithi138 ceph-mon[185886]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:46 smithi138 ceph-mon[185886]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:46 smithi138 ceph-mon[185886]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:46 smithi138 ceph-mon[185886]: osdmap e304: 8 total, 8 up, 6 in 2023-10-01T09:27:46.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:46 smithi138 ceph-mon[185886]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:46 smithi138 ceph-mon[185886]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:46 smithi138 ceph-mon[185886]: pgmap v817: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.6 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 25927/127152 objects degraded (20.391%); 12909/127152 objects misplaced (10.152%); 15 MiB/s, 239 objects/s recovering 2023-10-01T09:27:46.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:46 smithi138 ceph-mon[192513]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:46 smithi138 ceph-mon[192513]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:46 smithi138 ceph-mon[192513]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:46 smithi138 ceph-mon[192513]: osdmap e304: 8 total, 8 up, 6 in 2023-10-01T09:27:46.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:46 smithi138 ceph-mon[192513]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:46 smithi138 ceph-mon[192513]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:46 smithi138 ceph-mon[192513]: pgmap v817: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.6 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 25927/127152 objects degraded (20.391%); 12909/127152 objects misplaced (10.152%); 15 MiB/s, 239 objects/s recovering 2023-10-01T09:27:46.253 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:46.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:46.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:46 smithi190 ceph-mon[160435]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:46 smithi190 ceph-mon[160435]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:46 smithi190 ceph-mon[160435]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:46 smithi190 ceph-mon[160435]: osdmap e304: 8 total, 8 up, 6 in 2023-10-01T09:27:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:46 smithi190 ceph-mon[160435]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:46 smithi190 ceph-mon[160435]: failed to encode map e304 with expected crc 2023-10-01T09:27:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:46 smithi190 ceph-mon[160435]: pgmap v817: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.6 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 25927/127152 objects degraded (20.391%); 12909/127152 objects misplaced (10.152%); 15 MiB/s, 239 objects/s recovering 2023-10-01T09:27:46.367 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:46.464 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:46.479 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:46.599 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:46.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:46.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:46.635 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:27:46.635 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:27:46.663 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:27:46.716 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:46.723 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:27:46.753 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:46.777 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:27:46.823 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:27:46.857 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:46.859 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:27:46.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:46.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:46.901 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:27:46.937 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:27:46.962 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:46.992 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:47.082 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:27:47.105 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:27:47.105 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:27:47.105 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:27:47.105 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:27:47.105 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:27:47.105 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:27:47.105 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:27:47.105 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:27:47.138 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:27:47.138 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:27:47.138 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:47.138 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:47 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2660731959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:47.142 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:27:47.143 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:27:47.143 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:47.143 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:47 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2660731959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:47.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:27:47.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:27:47.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:27:47.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:47 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2660731959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:47.317 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:47.321 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:27:47.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:47.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:47.435 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:47.472 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:47.534 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:47.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:47.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:47.641 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:27:47.727 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:47.757 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:27:47.974 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:27:48.035 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:48 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 25927/127152 objects degraded (20.391%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:48.035 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:48 smithi138 ceph-mon[185886]: pgmap v818: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.5 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 468 op/s; 25789/124101 objects degraded (20.781%); 12360/124101 objects misplaced (9.960%); 8.7 MiB/s, 139 objects/s recovering 2023-10-01T09:27:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:48 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 25927/127152 objects degraded (20.391%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:48 smithi190 ceph-mon[160435]: pgmap v818: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.5 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 468 op/s; 25789/124101 objects degraded (20.781%); 12360/124101 objects misplaced (9.960%); 8.7 MiB/s, 139 objects/s recovering 2023-10-01T09:27:48.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:48 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 25927/127152 objects degraded (20.391%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:48.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:48 smithi138 ceph-mon[192513]: pgmap v818: 89 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 2.5 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 468 op/s; 25789/124101 objects degraded (20.781%); 12360/124101 objects misplaced (9.960%); 8.7 MiB/s, 139 objects/s recovering 2023-10-01T09:27:49.436 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:27:49.436 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:27:49.437 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.437 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:156274 7.5 7:abc9fd22:::benchmark_data_smithi138_367058_object58479:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-01T09:27:49.437 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:27:49.422370+0000", 2023-10-01T09:27:49.437 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.013477675, 2023-10-01T09:27:49.437 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013497673999999999, 2023-10-01T09:27:49.437 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:49.438 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:27:49.438 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:49.438 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:27:49.438 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:27:49.438 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 156274 2023-10-01T09:27:49.438 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.439 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:49.439 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.439 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:49.439 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422370+0000", 2023-10-01T09:27:49.439 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:49.439 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.440 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.440 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:49.440 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422370+0000", 2023-10-01T09:27:49.440 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:27:49.440 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.440 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.440 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:49.441 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422368+0000", 2023-10-01T09:27:49.441 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3350000000000007e-06 2023-10-01T09:27:49.441 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.441 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.441 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:49.441 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422376+0000", 2023-10-01T09:27:49.442 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4778e-05 2023-10-01T09:27:49.442 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.442 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.442 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:49.442 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422391+0000", 2023-10-01T09:27:49.443 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9580000000000004e-06 2023-10-01T09:27:49.443 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.443 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.443 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:49.443 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422397+0000", 2023-10-01T09:27:49.443 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0599000000000002e-05 2023-10-01T09:27:49.444 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.444 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.444 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:49.444 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422438+0000", 2023-10-01T09:27:49.444 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2293999999999994e-05 2023-10-01T09:27:49.444 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.444 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.445 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:49.445 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422530+0000", 2023-10-01T09:27:49.445 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6594000000000005e-05 2023-10-01T09:27:49.445 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.445 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.445 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:27:49.445 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422617+0000", 2023-10-01T09:27:49.446 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.004853492 2023-10-01T09:27:49.446 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.446 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.446 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:49.446 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.427470+0000", 2023-10-01T09:27:49.446 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003440022 2023-10-01T09:27:49.446 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.446 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.447 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:49.447 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.430910+0000", 2023-10-01T09:27:49.447 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0085403730000000004 2023-10-01T09:27:49.447 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:49.447 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:49.447 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:49.447 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.448 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.448 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:156276 7.5 7:ac016666:::benchmark_data_smithi138_367058_object58481:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-01T09:27:49.448 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:27:49.422408+0000", 2023-10-01T09:27:49.448 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.013439065, 2023-10-01T09:27:49.448 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013514600999999999, 2023-10-01T09:27:49.448 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:49.449 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:27:49.449 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:49.449 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:27:49.449 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:27:49.449 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 156276 2023-10-01T09:27:49.449 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.449 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:49.449 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.449 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:49.449 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422408+0000", 2023-10-01T09:27:49.449 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:49.450 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.450 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.450 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:49.450 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422408+0000", 2023-10-01T09:27:49.450 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:27:49.450 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.450 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.450 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:49.450 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422408+0000", 2023-10-01T09:27:49.451 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2449999999999998e-06 2023-10-01T09:27:49.451 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.451 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.451 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:49.451 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422411+0000", 2023-10-01T09:27:49.451 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.105e-06 2023-10-01T09:27:49.452 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.452 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.452 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:49.452 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422417+0000", 2023-10-01T09:27:49.452 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4659999999999999e-06 2023-10-01T09:27:49.452 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.453 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.453 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:49.453 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.422421+0000", 2023-10-01T09:27:49.453 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002579591 2023-10-01T09:27:49.453 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.453 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.454 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:49.454 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.425000+0000", 2023-10-01T09:27:49.454 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000103965 2023-10-01T09:27:49.454 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.454 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.454 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:49.455 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.425104+0000", 2023-10-01T09:27:49.455 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000117631 2023-10-01T09:27:49.455 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.455 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.455 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:27:49.455 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.425222+0000", 2023-10-01T09:27:49.456 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024043839999999999 2023-10-01T09:27:49.456 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.456 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.456 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:49.456 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.427626+0000", 2023-10-01T09:27:49.456 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0052178950000000002 2023-10-01T09:27:49.456 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:49.456 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:49.456 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:49.456 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.457 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.457 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:156286 7.5 7:a69db716:::benchmark_data_smithi138_367058_object58491:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-10-01T09:27:49.457 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:27:49.432168+0000", 2023-10-01T09:27:49.457 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0036794660000000002, 2023-10-01T09:27:49.457 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003799845, 2023-10-01T09:27:49.457 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:49.457 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:27:49.457 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:49.457 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:27:49.457 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:27:49.457 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 156286 2023-10-01T09:27:49.457 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.458 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:49.458 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.458 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:49.458 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.432168+0000", 2023-10-01T09:27:49.458 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:49.458 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.458 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.458 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:49.458 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.432168+0000", 2023-10-01T09:27:49.458 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:27:49.458 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.459 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.459 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:49.459 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.432166+0000", 2023-10-01T09:27:49.459 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7049999999999996e-06 2023-10-01T09:27:49.459 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.459 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.459 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:49.459 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.432172+0000", 2023-10-01T09:27:49.459 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0971e-05 2023-10-01T09:27:49.459 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.459 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.459 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:49.459 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.432183+0000", 2023-10-01T09:27:49.460 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6090000000000003e-06 2023-10-01T09:27:49.460 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.460 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.460 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:49.460 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.432188+0000", 2023-10-01T09:27:49.460 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9881000000000001e-05 2023-10-01T09:27:49.460 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.460 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.460 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:49.460 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.432218+0000", 2023-10-01T09:27:49.460 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6724999999999995e-05 2023-10-01T09:27:49.460 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.461 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.461 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:49.461 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.432294+0000", 2023-10-01T09:27:49.461 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0521999999999998e-05 2023-10-01T09:27:49.461 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.461 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.461 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:27:49.461 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.432385+0000", 2023-10-01T09:27:49.461 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0022207250000000002 2023-10-01T09:27:49.462 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:49.462 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:49.462 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:49.462 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:27:49.434606+0000", 2023-10-01T09:27:49.462 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024375820000000002 2023-10-01T09:27:49.462 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:49.462 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:49.462 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:49.462 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:49.462 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:27:49.462 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 3 2023-10-01T09:27:49.463 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:27:49.587 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:27:49.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:49.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:27:49.712 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:49.743 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:49.786 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:49.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:49.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:49.856 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:49.890 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:50.007 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:50.159 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:27:50.187 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:49 smithi190 ceph-mon[160435]: pgmap v819: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.5 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 554 op/s; 25789/120666 objects degraded (21.372%); 11975/120666 objects misplaced (9.924%); 9.5 MiB/s, 151 objects/s recovering 2023-10-01T09:27:50.190 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:50.202 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:49 smithi138 ceph-mon[185886]: pgmap v819: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.5 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 554 op/s; 25789/120666 objects degraded (21.372%); 11975/120666 objects misplaced (9.924%); 9.5 MiB/s, 151 objects/s recovering 2023-10-01T09:27:50.202 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:49 smithi138 ceph-mon[192513]: pgmap v819: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.5 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 554 op/s; 25789/120666 objects degraded (21.372%); 11975/120666 objects misplaced (9.924%); 9.5 MiB/s, 151 objects/s recovering 2023-10-01T09:27:50.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:50.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:50.259 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:27:50.260 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "" 2023-10-01T09:27:50.260 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:27:50.303 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:50.331 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:50.394 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:50.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:50.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:50.500 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:27:50.587 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:27:50.598 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:50.796 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:27:50.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:50.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:27:50.904 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:27:50.946 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:51.245 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:27:51.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:51.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:27:51.350 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:27:51.378 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:51.620 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:27:52.055 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:51 smithi190 ceph-mon[160435]: pgmap v820: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.4 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 25654/119373 objects degraded (21.491%); 11975/119373 objects misplaced (10.032%); 4.1 MiB/s, 65 objects/s recovering 2023-10-01T09:27:52.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:51 smithi138 ceph-mon[185886]: pgmap v820: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.4 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 25654/119373 objects degraded (21.491%); 11975/119373 objects misplaced (10.032%); 4.1 MiB/s, 65 objects/s recovering 2023-10-01T09:27:52.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:51 smithi138 ceph-mon[192513]: pgmap v820: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.4 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 25654/119373 objects degraded (21.491%); 11975/119373 objects misplaced (10.032%); 4.1 MiB/s, 65 objects/s recovering 2023-10-01T09:27:52.265 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:27:52.265 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:27:52.266 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:27:52.266 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:27:52.266 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:27:52.531 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:27:52.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:52.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:27:52.646 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:27:52.740 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:52.860 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:27:52.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:52.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:27:52.895 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:52.928 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:27:52.974 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:53.006 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:53.075 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:53.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:53.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:53.134 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:52 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 25654/119373 objects degraded (21.491%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:53.134 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:52 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 25654/119373 objects degraded (21.491%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:53.180 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:27:53.211 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:52 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 25654/119373 objects degraded (21.491%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:53.217 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:53.330 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:27:53.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:53.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:27:53.364 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:27:53.439 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:27:53.468 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:53.517 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:27:53.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:53.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:27:53.623 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:27:53.757 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:53.924 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:27:53.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:53.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:27:53.975 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:27:54.012 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/581577382' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:27:54.012 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:53 smithi138 ceph-mon[185886]: pgmap v821: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 478 op/s; 25654/113439 objects degraded (22.615%); 11028/113439 objects misplaced (9.722%); 4.4 MiB/s, 69 objects/s recovering 2023-10-01T09:27:54.012 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:53 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:27:54.013 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/581577382' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:27:54.013 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:53 smithi138 ceph-mon[192513]: pgmap v821: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 478 op/s; 25654/113439 objects degraded (22.615%); 11028/113439 objects misplaced (9.722%); 4.4 MiB/s, 69 objects/s recovering 2023-10-01T09:27:54.013 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:53 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:27:54.035 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:27:54.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/581577382' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:27:54.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:53 smithi190 ceph-mon[160435]: pgmap v821: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 478 op/s; 25654/113439 objects degraded (22.615%); 11028/113439 objects misplaced (9.722%); 4.4 MiB/s, 69 objects/s recovering 2023-10-01T09:27:54.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:53 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:27:54.112 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:54.186 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:27:54.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:54.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:27:54.291 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:27:54.321 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:54.561 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:27:54.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:54.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:27:54.668 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:54.696 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:54.769 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:54.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:54.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:54.873 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:55.066 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:55.094 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:27:55.095 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:27:55.095 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:27:55.095 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:27:55.095 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.095 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:209431 5.4 5:29b24467:::benchmark_data_smithi138_240338_object94294:head [delete] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-01T09:27:55.095 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:30.394489+0000", 2023-10-01T09:27:55.095 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 564.69649975000004, 2023-10-01T09:27:55.095 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4265964819999999, 2023-10-01T09:27:55.096 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.096 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:27:55.096 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.096 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:27:55.096 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:27:55.096 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 209431 2023-10-01T09:27:55.096 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.096 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.096 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.096 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.096 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394489+0000", 2023-10-01T09:27:55.096 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.097 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.097 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.097 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.097 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394489+0000", 2023-10-01T09:27:55.097 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:27:55.097 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.097 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.097 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.097 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394487+0000", 2023-10-01T09:27:55.097 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0932e-05 2023-10-01T09:27:55.097 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.097 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.098 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.098 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394498+0000", 2023-10-01T09:27:55.098 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7770000000000001e-05 2023-10-01T09:27:55.098 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.098 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.098 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.098 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394516+0000", 2023-10-01T09:27:55.098 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0899999999999999e-06 2023-10-01T09:27:55.098 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.098 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.098 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.099 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394523+0000", 2023-10-01T09:27:55.099 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3308999999999998e-05 2023-10-01T09:27:55.099 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.099 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.099 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.099 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394606+0000", 2023-10-01T09:27:55.099 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016228 2023-10-01T09:27:55.099 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.099 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.099 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.099 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394768+0000", 2023-10-01T09:27:55.099 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00018086799999999999 2023-10-01T09:27:55.100 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.100 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.100 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:27:55.100 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394949+0000", 2023-10-01T09:27:55.100 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013612196 2023-10-01T09:27:55.100 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.100 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.100 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.100 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.408561+0000", 2023-10-01T09:27:55.101 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4705999999999999e-05 2023-10-01T09:27:55.101 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.101 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.101 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.101 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.408576+0000", 2023-10-01T09:27:55.101 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013021654000000001 2023-10-01T09:27:55.102 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.102 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.102 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.102 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.421598+0000", 2023-10-01T09:27:55.102 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5470000000000004e-06 2023-10-01T09:27:55.102 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.102 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.102 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.103 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.421603+0000", 2023-10-01T09:27:55.103 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99164297499999998 2023-10-01T09:27:55.103 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.103 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.103 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.103 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.413246+0000", 2023-10-01T09:27:55.103 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.24e-05 2023-10-01T09:27:55.103 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.103 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.103 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.103 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.413259+0000", 2023-10-01T09:27:55.104 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.407740548 2023-10-01T09:27:55.104 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.104 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.104 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.104 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.820999+0000", 2023-10-01T09:27:55.104 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6007999999999994e-05 2023-10-01T09:27:55.104 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.104 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.104 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.104 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.821085+0000", 2023-10-01T09:27:55.105 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4265964819999999 2023-10-01T09:27:55.105 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.105 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.105 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.105 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.105 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.105 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:209433 5.4 5:28629f7e:::benchmark_data_smithi138_240338_object94296:head [delete] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-01T09:27:55.105 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:30.394538+0000", 2023-10-01T09:27:55.105 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 564.69645037199996, 2023-10-01T09:27:55.105 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4268786050000002, 2023-10-01T09:27:55.105 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.105 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:27:55.106 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.106 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:27:55.106 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:27:55.106 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 209433 2023-10-01T09:27:55.106 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.106 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.106 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.106 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.106 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394538+0000", 2023-10-01T09:27:55.106 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.107 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.107 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.107 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.107 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394538+0000", 2023-10-01T09:27:55.107 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:27:55.107 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.107 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.107 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.107 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394537+0000", 2023-10-01T09:27:55.107 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0980000000000004e-06 2023-10-01T09:27:55.107 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.108 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.108 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.108 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394541+0000", 2023-10-01T09:27:55.108 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7070000000000004e-06 2023-10-01T09:27:55.108 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.108 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.108 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.108 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394551+0000", 2023-10-01T09:27:55.108 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3129999999999998e-06 2023-10-01T09:27:55.108 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.108 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.108 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.109 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.394556+0000", 2023-10-01T09:27:55.109 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045311270000000002 2023-10-01T09:27:55.109 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.109 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.109 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.109 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.399087+0000", 2023-10-01T09:27:55.109 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011880899999999999 2023-10-01T09:27:55.109 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.109 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.109 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.109 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.399206+0000", 2023-10-01T09:27:55.109 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014506400000000001 2023-10-01T09:27:55.110 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.110 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.110 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:27:55.110 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.399351+0000", 2023-10-01T09:27:55.110 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.022264551 2023-10-01T09:27:55.110 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.110 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.110 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.110 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.421616+0000", 2023-10-01T09:27:55.110 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7929999999999999e-06 2023-10-01T09:27:55.111 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.111 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.111 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.111 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.421620+0000", 2023-10-01T09:27:55.111 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99211701900000004 2023-10-01T09:27:55.111 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.111 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.111 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.111 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.413737+0000", 2023-10-01T09:27:55.111 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.371e-06 2023-10-01T09:27:55.111 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.112 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.112 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.112 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.413745+0000", 2023-10-01T09:27:55.112 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.407613403 2023-10-01T09:27:55.112 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.112 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.112 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.112 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.821358+0000", 2023-10-01T09:27:55.112 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8146999999999997e-05 2023-10-01T09:27:55.112 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.113 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.113 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.113 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.821417+0000", 2023-10-01T09:27:55.113 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4268786050000002 2023-10-01T09:27:55.113 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.113 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.113 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.113 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.113 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.114 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:209434 5.d 5:bc8b9902:::benchmark_data_smithi138_240338_object94297:head [delete] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-01T09:27:55.114 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:30.398823+0000", 2023-10-01T09:27:55.114 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 564.69216500899995, 2023-10-01T09:27:55.114 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4233056020000001, 2023-10-01T09:27:55.114 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.114 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:27:55.114 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.115 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:27:55.115 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:27:55.115 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 209434 2023-10-01T09:27:55.115 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.115 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.115 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.116 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.116 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.398823+0000", 2023-10-01T09:27:55.116 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.116 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.116 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.116 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.116 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.398823+0000", 2023-10-01T09:27:55.116 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:27:55.117 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.117 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.117 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.117 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.398822+0000", 2023-10-01T09:27:55.117 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7499999999999992e-06 2023-10-01T09:27:55.117 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.118 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.118 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.118 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.398830+0000", 2023-10-01T09:27:55.118 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7166000000000001e-05 2023-10-01T09:27:55.118 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.118 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.119 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.119 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.398847+0000", 2023-10-01T09:27:55.119 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6990000000000003e-06 2023-10-01T09:27:55.119 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.119 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.119 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.120 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.398855+0000", 2023-10-01T09:27:55.120 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00025465900000000002 2023-10-01T09:27:55.120 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.120 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.120 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.120 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.399110+0000", 2023-10-01T09:27:55.120 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000102122 2023-10-01T09:27:55.121 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.121 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.121 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.121 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.399212+0000", 2023-10-01T09:27:55.121 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012519699999999999 2023-10-01T09:27:55.121 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.121 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.122 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:27:55.122 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.399337+0000", 2023-10-01T09:27:55.122 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0098970570000000008 2023-10-01T09:27:55.122 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.122 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.122 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.122 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.409234+0000", 2023-10-01T09:27:55.122 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0057004780000000001 2023-10-01T09:27:55.122 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.122 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.123 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.123 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.414935+0000", 2023-10-01T09:27:55.123 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7669999999999999e-05 2023-10-01T09:27:55.123 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.123 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.123 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.123 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.414952+0000", 2023-10-01T09:27:55.123 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99873159099999997 2023-10-01T09:27:55.124 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.124 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.124 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.124 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.413684+0000", 2023-10-01T09:27:55.124 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1194e-05 2023-10-01T09:27:55.124 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.124 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.124 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.124 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.413695+0000", 2023-10-01T09:27:55.124 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.40835344 2023-10-01T09:27:55.124 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.125 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.125 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.125 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.822049+0000", 2023-10-01T09:27:55.125 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0395000000000005e-05 2023-10-01T09:27:55.125 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.125 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.125 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.125 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.822129+0000", 2023-10-01T09:27:55.125 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4233056020000001 2023-10-01T09:27:55.125 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.126 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.126 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.126 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.126 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.126 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:209435 5.d 5:ba0d91a5:::benchmark_data_smithi138_240338_object94298:head [delete] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-01T09:27:55.126 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:30.399081+0000", 2023-10-01T09:27:55.126 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 564.691906915, 2023-10-01T09:27:55.126 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.423434613, 2023-10-01T09:27:55.126 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.126 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:27:55.126 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.127 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:27:55.127 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:27:55.127 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 209435 2023-10-01T09:27:55.127 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.127 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.127 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.127 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.127 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.399081+0000", 2023-10-01T09:27:55.127 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.127 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.127 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.128 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.128 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.399081+0000", 2023-10-01T09:27:55.128 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:27:55.128 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.128 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.128 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.128 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.399080+0000", 2023-10-01T09:27:55.128 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7409999999999998e-06 2023-10-01T09:27:55.128 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.128 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.128 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.128 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.399086+0000", 2023-10-01T09:27:55.128 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2356e-05 2023-10-01T09:27:55.129 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.129 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.129 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.129 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.399099+0000", 2023-10-01T09:27:55.129 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5570000000000001e-06 2023-10-01T09:27:55.129 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.129 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.129 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.129 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.399105+0000", 2023-10-01T09:27:55.130 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0047300440000000001 2023-10-01T09:27:55.130 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.130 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.130 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.130 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.403835+0000", 2023-10-01T09:27:55.130 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014357700000000001 2023-10-01T09:27:55.130 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.130 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.130 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.130 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.403979+0000", 2023-10-01T09:27:55.130 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00062671999999999999 2023-10-01T09:27:55.131 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.131 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.131 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:27:55.131 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.404605+0000", 2023-10-01T09:27:55.131 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.005446846 2023-10-01T09:27:55.131 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.131 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.131 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.131 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.410052+0000", 2023-10-01T09:27:55.131 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0051148740000000002 2023-10-01T09:27:55.132 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.132 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.132 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.132 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.415167+0000", 2023-10-01T09:27:55.132 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2799999999999992e-06 2023-10-01T09:27:55.132 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.132 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.132 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.132 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.415176+0000", 2023-10-01T09:27:55.132 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99915752199999996 2023-10-01T09:27:55.133 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.133 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.133 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.133 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.414334+0000", 2023-10-01T09:27:55.133 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8907e-05 2023-10-01T09:27:55.133 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.133 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.133 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.133 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.414353+0000", 2023-10-01T09:27:55.133 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.408095849 2023-10-01T09:27:55.134 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.134 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.134 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.134 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.822449+0000", 2023-10-01T09:27:55.134 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7341999999999993e-05 2023-10-01T09:27:55.134 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.134 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.134 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.134 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.822516+0000", 2023-10-01T09:27:55.134 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.423434613 2023-10-01T09:27:55.135 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.135 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.135 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.135 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.135 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.135 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:209437 5.4 5:2b89126e:::benchmark_data_smithi138_240338_object94300:head [delete] snapc 0=[] ondisk+write+known_if_redirected e146)", 2023-10-01T09:27:55.135 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:30.402325+0000", 2023-10-01T09:27:55.135 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 564.68866307400003, 2023-10-01T09:27:55.136 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4498408330000001, 2023-10-01T09:27:55.136 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.136 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:27:55.136 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.136 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:27:55.136 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:27:55.136 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 209437 2023-10-01T09:27:55.136 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.136 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.137 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.137 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.137 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.402325+0000", 2023-10-01T09:27:55.137 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.137 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.137 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.137 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.137 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.402325+0000", 2023-10-01T09:27:55.137 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:27:55.137 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.137 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.138 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.138 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.402323+0000", 2023-10-01T09:27:55.138 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0273e-05 2023-10-01T09:27:55.138 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.138 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.138 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.138 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.402334+0000", 2023-10-01T09:27:55.138 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2481000000000003e-05 2023-10-01T09:27:55.138 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.139 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.139 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.139 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.402366+0000", 2023-10-01T09:27:55.139 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3670000000000003e-06 2023-10-01T09:27:55.139 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.139 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.139 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.139 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.402375+0000", 2023-10-01T09:27:55.139 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0066150330000000002 2023-10-01T09:27:55.140 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.140 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.140 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.140 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.408990+0000", 2023-10-01T09:27:55.140 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000122793 2023-10-01T09:27:55.140 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.140 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.140 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.140 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.409113+0000", 2023-10-01T09:27:55.140 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014080200000000001 2023-10-01T09:27:55.140 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.140 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.141 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:27:55.141 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.409254+0000", 2023-10-01T09:27:55.141 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012378202 2023-10-01T09:27:55.141 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.141 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.141 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.141 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.421632+0000", 2023-10-01T09:27:55.141 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.456e-06 2023-10-01T09:27:55.141 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.141 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.141 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.141 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.421636+0000", 2023-10-01T09:27:55.142 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99217018999999995 2023-10-01T09:27:55.142 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.142 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.142 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.142 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.413806+0000", 2023-10-01T09:27:55.142 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6270000000000006e-06 2023-10-01T09:27:55.142 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.142 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.142 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.142 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.413815+0000", 2023-10-01T09:27:55.142 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.407626673 2023-10-01T09:27:55.142 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.143 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.143 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.143 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.821441+0000", 2023-10-01T09:27:55.143 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000112858 2023-10-01T09:27:55.143 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.143 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.143 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:27:55.143 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.821554+0000", 2023-10-01T09:27:55.143 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.03048758 2023-10-01T09:27:55.143 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.143 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.143 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.144 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.852042+0000", 2023-10-01T09:27:55.144 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012448599999999999 2023-10-01T09:27:55.144 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.144 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.144 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.144 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.852166+0000", 2023-10-01T09:27:55.144 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4498408330000001 2023-10-01T09:27:55.144 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.144 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.144 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.144 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.144 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.144 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:209449 5.d 5:b9a4a2ca:::benchmark_data_smithi138_240338_object94312:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-01T09:27:55.145 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:30.562464+0000", 2023-10-01T09:27:55.145 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 564.52852426599998, 2023-10-01T09:27:55.145 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2976982430000001, 2023-10-01T09:27:55.145 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.145 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:27:55.145 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.145 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:27:55.145 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:27:55.145 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 209449 2023-10-01T09:27:55.145 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.145 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.145 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.146 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.146 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.562464+0000", 2023-10-01T09:27:55.146 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.146 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.146 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.146 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.146 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.562464+0000", 2023-10-01T09:27:55.146 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:27:55.146 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.146 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.146 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.147 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.562463+0000", 2023-10-01T09:27:55.147 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8560000000000001e-06 2023-10-01T09:27:55.147 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.147 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.147 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.147 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.562469+0000", 2023-10-01T09:27:55.147 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2901999999999999e-05 2023-10-01T09:27:55.147 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.147 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.147 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.147 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.562482+0000", 2023-10-01T09:27:55.147 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2490000000000004e-06 2023-10-01T09:27:55.148 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.148 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.148 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.148 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.562490+0000", 2023-10-01T09:27:55.148 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0038999999999999e-05 2023-10-01T09:27:55.148 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.148 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.148 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.148 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.562550+0000", 2023-10-01T09:27:55.148 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5517000000000001e-05 2023-10-01T09:27:55.148 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.149 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.149 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.149 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.562565+0000", 2023-10-01T09:27:55.149 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.85181179900000004 2023-10-01T09:27:55.149 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.149 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.149 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.149 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.414377+0000", 2023-10-01T09:27:55.149 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.9060000000000008e-06 2023-10-01T09:27:55.149 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.149 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.149 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.150 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.414386+0000", 2023-10-01T09:27:55.150 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4081740659999999 2023-10-01T09:27:55.150 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.150 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.150 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.150 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.822560+0000", 2023-10-01T09:27:55.150 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000111488 2023-10-01T09:27:55.150 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.150 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.150 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.150 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.822671+0000", 2023-10-01T09:27:55.151 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00059953600000000001 2023-10-01T09:27:55.151 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.151 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.151 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2,5", 2023-10-01T09:27:55.151 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.823271+0000", 2023-10-01T09:27:55.151 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017160424000000001 2023-10-01T09:27:55.151 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.151 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.151 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.151 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.840431+0000", 2023-10-01T09:27:55.151 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0090143879999999999 2023-10-01T09:27:55.151 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.152 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.152 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.152 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.849446+0000", 2023-10-01T09:27:55.152 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0022591600000000001 2023-10-01T09:27:55.152 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.152 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.152 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.152 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.851705+0000", 2023-10-01T09:27:55.152 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.008257337 2023-10-01T09:27:55.152 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.152 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.153 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.153 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.859962+0000", 2023-10-01T09:27:55.153 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0168999999999999e-05 2023-10-01T09:27:55.153 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.153 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.153 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:27:55.153 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.860022+0000", 2023-10-01T09:27:55.153 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013997400000000001 2023-10-01T09:27:55.153 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.153 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.153 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.153 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.860162+0000", 2023-10-01T09:27:55.154 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2976982430000001 2023-10-01T09:27:55.154 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.154 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.154 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.154 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.154 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.154 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:209461 5.4 5:267ce15a:::benchmark_data_smithi138_240338_object94324:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-01T09:27:55.154 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:30.581879+0000", 2023-10-01T09:27:55.154 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 564.50910945299995, 2023-10-01T09:27:55.154 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2706903829999998, 2023-10-01T09:27:55.155 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.155 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:27:55.155 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.155 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:27:55.155 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:27:55.155 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 209461 2023-10-01T09:27:55.155 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.155 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.155 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.155 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.155 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.581879+0000", 2023-10-01T09:27:55.156 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.156 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.156 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.156 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.156 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.581879+0000", 2023-10-01T09:27:55.156 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:27:55.156 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.156 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.156 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.156 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.581877+0000", 2023-10-01T09:27:55.156 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.132e-05 2023-10-01T09:27:55.156 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.157 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.157 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.157 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.581888+0000", 2023-10-01T09:27:55.157 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2515e-05 2023-10-01T09:27:55.157 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.157 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.157 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.157 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.581910+0000", 2023-10-01T09:27:55.157 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4700000000000008e-06 2023-10-01T09:27:55.157 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.157 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.158 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.158 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.581920+0000", 2023-10-01T09:27:55.158 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1536000000000003e-05 2023-10-01T09:27:55.158 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.158 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.158 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.158 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.581971+0000", 2023-10-01T09:27:55.158 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2400000000000007e-06 2023-10-01T09:27:55.158 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.158 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.159 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.159 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.581980+0000", 2023-10-01T09:27:55.159 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.83187468799999997 2023-10-01T09:27:55.159 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.159 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.159 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.159 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.413854+0000", 2023-10-01T09:27:55.159 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0769999999999997e-06 2023-10-01T09:27:55.159 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.159 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.160 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.160 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.413863+0000", 2023-10-01T09:27:55.160 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4077159720000001 2023-10-01T09:27:55.160 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.160 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.160 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.160 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.821579+0000", 2023-10-01T09:27:55.160 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000121524 2023-10-01T09:27:55.160 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.160 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.160 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.160 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.821701+0000", 2023-10-01T09:27:55.161 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00066204199999999995 2023-10-01T09:27:55.161 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.161 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.161 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2,5", 2023-10-01T09:27:55.161 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.822363+0000", 2023-10-01T09:27:55.161 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.029167234 2023-10-01T09:27:55.161 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.161 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.161 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.161 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.851530+0000", 2023-10-01T09:27:55.162 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6545999999999999e-05 2023-10-01T09:27:55.162 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.162 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.162 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.162 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.851567+0000", 2023-10-01T09:27:55.162 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6582000000000001e-05 2023-10-01T09:27:55.162 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.163 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.163 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.163 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.851623+0000", 2023-10-01T09:27:55.163 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00068734099999999995 2023-10-01T09:27:55.163 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.163 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.163 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.163 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.852311+0000", 2023-10-01T09:27:55.163 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3295999999999997e-05 2023-10-01T09:27:55.163 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.163 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.163 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:27:55.164 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.852394+0000", 2023-10-01T09:27:55.164 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017530399999999999 2023-10-01T09:27:55.164 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.164 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.164 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.164 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.852569+0000", 2023-10-01T09:27:55.164 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2706903829999998 2023-10-01T09:27:55.164 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.164 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.164 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.164 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.165 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.165 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:209465 5.4 5:2898f988:::benchmark_data_smithi138_240338_object94328:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-10-01T09:27:55.165 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:18:30.589443+0000", 2023-10-01T09:27:55.165 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 564.50154505700004, 2023-10-01T09:27:55.165 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2723432809999999, 2023-10-01T09:27:55.165 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.165 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:27:55.165 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.165 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:27:55.165 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:27:55.165 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 209465 2023-10-01T09:27:55.165 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.165 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.166 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.166 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.166 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.589443+0000", 2023-10-01T09:27:55.166 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.166 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.166 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.166 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.166 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.589443+0000", 2023-10-01T09:27:55.166 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:27:55.166 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.166 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.167 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.167 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.589443+0000", 2023-10-01T09:27:55.167 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0339999999999998e-06 2023-10-01T09:27:55.167 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.167 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.167 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.167 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.589447+0000", 2023-10-01T09:27:55.167 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.515e-06 2023-10-01T09:27:55.167 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.167 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.167 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.168 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.589454+0000", 2023-10-01T09:27:55.168 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1619999999999999e-06 2023-10-01T09:27:55.168 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.168 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.168 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.168 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.589459+0000", 2023-10-01T09:27:55.168 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6175999999999998e-05 2023-10-01T09:27:55.168 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.168 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.168 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.168 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.589526+0000", 2023-10-01T09:27:55.168 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3459999999999994e-06 2023-10-01T09:27:55.168 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.169 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.169 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.169 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:30.589534+0000", 2023-10-01T09:27:55.169 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.82479759100000005 2023-10-01T09:27:55.169 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.169 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.169 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.169 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.414331+0000", 2023-10-01T09:27:55.169 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8025000000000001e-05 2023-10-01T09:27:55.169 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.169 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.170 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.170 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:31.414349+0000", 2023-10-01T09:27:55.170 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.432523529 2023-10-01T09:27:55.170 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.170 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.170 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.170 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.846873+0000", 2023-10-01T09:27:55.170 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000146513 2023-10-01T09:27:55.170 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.170 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.170 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.170 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.847020+0000", 2023-10-01T09:27:55.171 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000195152 2023-10-01T09:27:55.171 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.171 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.171 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2,5", 2023-10-01T09:27:55.171 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.847215+0000", 2023-10-01T09:27:55.171 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050129240000000002 2023-10-01T09:27:55.171 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.171 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.171 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.171 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.852228+0000", 2023-10-01T09:27:55.171 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8740999999999998e-05 2023-10-01T09:27:55.171 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.172 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.172 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.172 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.852296+0000", 2023-10-01T09:27:55.172 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00028460999999999997 2023-10-01T09:27:55.172 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.172 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.172 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.172 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.852581+0000", 2023-10-01T09:27:55.172 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0090297699999999995 2023-10-01T09:27:55.172 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.172 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.172 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.173 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.861611+0000", 2023-10-01T09:27:55.173 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4014000000000002e-05 2023-10-01T09:27:55.173 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.173 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.173 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:27:55.173 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.861665+0000", 2023-10-01T09:27:55.173 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000121841 2023-10-01T09:27:55.173 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.173 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.173 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.173 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:18:32.861787+0000", 2023-10-01T09:27:55.174 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2723432809999999 2023-10-01T09:27:55.174 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.174 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.174 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.174 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.174 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.174 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:227516 5.4 5:22e31b5f:::benchmark_data_smithi138_240338_object112379:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T09:27:55.174 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:19:04.544057+0000", 2023-10-01T09:27:55.174 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 530.54693115800001, 2023-10-01T09:27:55.174 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2305909800000001, 2023-10-01T09:27:55.175 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.175 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:27:55.175 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.175 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:27:55.175 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:27:55.175 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 227516 2023-10-01T09:27:55.175 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.175 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.175 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.175 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.175 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.544057+0000", 2023-10-01T09:27:55.175 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.176 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.176 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.176 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.176 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.544057+0000", 2023-10-01T09:27:55.176 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:27:55.176 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.176 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.176 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.176 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.544055+0000", 2023-10-01T09:27:55.176 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.9250000000000001e-06 2023-10-01T09:27:55.176 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.176 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.177 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.177 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.544064+0000", 2023-10-01T09:27:55.177 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1943000000000001e-05 2023-10-01T09:27:55.177 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.177 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.177 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.177 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.544076+0000", 2023-10-01T09:27:55.177 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8220000000000001e-06 2023-10-01T09:27:55.177 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.177 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.177 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.178 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.544083+0000", 2023-10-01T09:27:55.178 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000221444 2023-10-01T09:27:55.178 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.178 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.178 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.178 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.544305+0000", 2023-10-01T09:27:55.178 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012834000000000001 2023-10-01T09:27:55.178 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.178 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.178 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.178 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.544433+0000", 2023-10-01T09:27:55.178 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000112026 2023-10-01T09:27:55.179 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.179 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.179 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:27:55.179 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.544545+0000", 2023-10-01T09:27:55.179 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0056554539999999999 2023-10-01T09:27:55.179 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.179 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.179 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.179 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.550200+0000", 2023-10-01T09:27:55.179 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0434e-05 2023-10-01T09:27:55.179 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.180 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.180 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.180 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.550211+0000", 2023-10-01T09:27:55.180 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99980519099999998 2023-10-01T09:27:55.180 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.180 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.180 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.180 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.550016+0000", 2023-10-01T09:27:55.180 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0138e-05 2023-10-01T09:27:55.180 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.180 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.180 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.181 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.550026+0000", 2023-10-01T09:27:55.181 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.224533031 2023-10-01T09:27:55.181 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.181 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.181 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.181 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.774559+0000", 2023-10-01T09:27:55.181 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.9027e-05 2023-10-01T09:27:55.181 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.181 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.181 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.181 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.774648+0000", 2023-10-01T09:27:55.181 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2305909800000001 2023-10-01T09:27:55.182 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.182 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.182 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.182 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.182 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.182 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:227519 5.d 5:bab0bbb3:::benchmark_data_smithi138_240338_object112382:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T09:27:55.182 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:19:04.546987+0000", 2023-10-01T09:27:55.182 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 530.54400150699996, 2023-10-01T09:27:55.182 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2104705490000001, 2023-10-01T09:27:55.182 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.182 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:27:55.182 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.183 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:27:55.183 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:27:55.183 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 227519 2023-10-01T09:27:55.183 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.183 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.183 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.183 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.183 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.546987+0000", 2023-10-01T09:27:55.183 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.183 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.183 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.183 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.183 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.546987+0000", 2023-10-01T09:27:55.184 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:27:55.184 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.184 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.184 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.184 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.546985+0000", 2023-10-01T09:27:55.184 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4730000000000003e-06 2023-10-01T09:27:55.184 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.184 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.184 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.184 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.546993+0000", 2023-10-01T09:27:55.184 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.149e-05 2023-10-01T09:27:55.184 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.185 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.185 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.185 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.547004+0000", 2023-10-01T09:27:55.185 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7139999999999996e-06 2023-10-01T09:27:55.185 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.185 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.185 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.185 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.547011+0000", 2023-10-01T09:27:55.185 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1955e-05 2023-10-01T09:27:55.186 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.186 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.186 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.186 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.547063+0000", 2023-10-01T09:27:55.186 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000127732 2023-10-01T09:27:55.186 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.186 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.186 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.186 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.547191+0000", 2023-10-01T09:27:55.186 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013481699999999999 2023-10-01T09:27:55.186 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.186 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.187 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:27:55.187 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.547326+0000", 2023-10-01T09:27:55.187 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0062892360000000001 2023-10-01T09:27:55.187 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.187 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.187 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.187 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.553615+0000", 2023-10-01T09:27:55.187 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5060000000000001e-06 2023-10-01T09:27:55.187 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.187 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.187 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.187 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.553624+0000", 2023-10-01T09:27:55.187 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99670451299999996 2023-10-01T09:27:55.188 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.188 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.188 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.188 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.550329+0000", 2023-10-01T09:27:55.188 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1960999999999999e-05 2023-10-01T09:27:55.188 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.188 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.188 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.188 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.550351+0000", 2023-10-01T09:27:55.188 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2070263649999999 2023-10-01T09:27:55.188 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.189 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.189 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.189 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.757377+0000", 2023-10-01T09:27:55.189 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0227000000000003e-05 2023-10-01T09:27:55.189 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.189 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.189 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.189 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.757457+0000", 2023-10-01T09:27:55.189 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2104705490000001 2023-10-01T09:27:55.189 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.189 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.190 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.190 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.190 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.190 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:227531 5.d 5:b317658c:::benchmark_data_smithi138_240338_object112394:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:27:55.190 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:19:04.576709+0000", 2023-10-01T09:27:55.190 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 530.51427954300004, 2023-10-01T09:27:55.190 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2054773939999999, 2023-10-01T09:27:55.190 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.190 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:27:55.190 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.190 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:27:55.190 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:27:55.191 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 227531 2023-10-01T09:27:55.191 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.191 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.191 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.191 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.191 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.576709+0000", 2023-10-01T09:27:55.191 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.191 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.191 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.191 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.191 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.576709+0000", 2023-10-01T09:27:55.191 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:27:55.191 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.192 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.192 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.192 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.576707+0000", 2023-10-01T09:27:55.192 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.339e-06 2023-10-01T09:27:55.192 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.192 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.192 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.192 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.576712+0000", 2023-10-01T09:27:55.192 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1158999999999999e-05 2023-10-01T09:27:55.192 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.192 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.192 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.192 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.576724+0000", 2023-10-01T09:27:55.193 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1740000000000002e-06 2023-10-01T09:27:55.193 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.193 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.193 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.193 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.576728+0000", 2023-10-01T09:27:55.193 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0951000000000001e-05 2023-10-01T09:27:55.193 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.193 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.193 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.193 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.576779+0000", 2023-10-01T09:27:55.193 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7079999999999997e-06 2023-10-01T09:27:55.193 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.194 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.194 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.194 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.576786+0000", 2023-10-01T09:27:55.194 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97359638699999995 2023-10-01T09:27:55.194 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.194 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.194 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.194 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.550383+0000", 2023-10-01T09:27:55.194 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.536e-06 2023-10-01T09:27:55.194 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.194 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.194 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.195 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.550388+0000", 2023-10-01T09:27:55.195 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2072948050000001 2023-10-01T09:27:55.195 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.195 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.195 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.195 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.757683+0000", 2023-10-01T09:27:55.195 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5127000000000001e-05 2023-10-01T09:27:55.195 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.195 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.195 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.195 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.757758+0000", 2023-10-01T09:27:55.196 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00053407200000000002 2023-10-01T09:27:55.196 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.196 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.196 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2,5", 2023-10-01T09:27:55.196 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.758292+0000", 2023-10-01T09:27:55.196 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.01323828 2023-10-01T09:27:55.196 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.196 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.196 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.196 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.771531+0000", 2023-10-01T09:27:55.196 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00078861600000000001 2023-10-01T09:27:55.196 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.196 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.197 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.197 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.772319+0000", 2023-10-01T09:27:55.197 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0013807380000000001 2023-10-01T09:27:55.197 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.197 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.197 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.197 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.773700+0000", 2023-10-01T09:27:55.197 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0083077159999999997 2023-10-01T09:27:55.197 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.197 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.197 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.197 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.782008+0000", 2023-10-01T09:27:55.198 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4234999999999998e-05 2023-10-01T09:27:55.198 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.198 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.198 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:27:55.198 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.782062+0000", 2023-10-01T09:27:55.198 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000124206 2023-10-01T09:27:55.198 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.198 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.198 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.198 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.782186+0000", 2023-10-01T09:27:55.198 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2054773939999999 2023-10-01T09:27:55.199 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.199 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.199 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.199 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.199 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.199 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:227533 5.4 5:2eb1ef1c:::benchmark_data_smithi138_240338_object112396:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:27:55.199 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:19:04.577866+0000", 2023-10-01T09:27:55.199 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 530.51312202500003, 2023-10-01T09:27:55.199 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2188770770000001, 2023-10-01T09:27:55.199 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.199 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:27:55.199 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.199 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:27:55.200 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:27:55.200 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 227533 2023-10-01T09:27:55.200 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.200 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.200 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.200 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.200 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.577866+0000", 2023-10-01T09:27:55.200 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.200 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.200 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.200 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.200 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.577866+0000", 2023-10-01T09:27:55.201 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:27:55.201 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.201 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.201 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.201 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.577865+0000", 2023-10-01T09:27:55.201 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3250000000000008e-06 2023-10-01T09:27:55.201 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.201 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.201 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.201 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.577873+0000", 2023-10-01T09:27:55.201 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9468000000000001e-05 2023-10-01T09:27:55.201 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.201 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.202 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.202 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.577892+0000", 2023-10-01T09:27:55.202 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2239999999999998e-06 2023-10-01T09:27:55.202 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.202 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.202 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.202 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.577900+0000", 2023-10-01T09:27:55.202 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1898999999999998e-05 2023-10-01T09:27:55.202 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.202 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.202 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.202 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.577962+0000", 2023-10-01T09:27:55.203 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2981e-05 2023-10-01T09:27:55.203 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.203 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.203 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.203 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.577974+0000", 2023-10-01T09:27:55.203 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97208099199999998 2023-10-01T09:27:55.203 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.203 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.203 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.203 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.550055+0000", 2023-10-01T09:27:55.203 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.8789999999999992e-06 2023-10-01T09:27:55.204 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.204 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.204 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.204 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.550064+0000", 2023-10-01T09:27:55.204 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2248798329999999 2023-10-01T09:27:55.204 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.204 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.204 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.204 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.774944+0000", 2023-10-01T09:27:55.204 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013290900000000001 2023-10-01T09:27:55.204 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.204 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.205 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.205 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.775077+0000", 2023-10-01T09:27:55.205 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00066310299999999996 2023-10-01T09:27:55.205 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.205 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.205 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2,5", 2023-10-01T09:27:55.205 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.775740+0000", 2023-10-01T09:27:55.205 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013832386 2023-10-01T09:27:55.205 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.205 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.205 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.205 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.789573+0000", 2023-10-01T09:27:55.205 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00078280699999999995 2023-10-01T09:27:55.206 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.206 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.206 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.206 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.790355+0000", 2023-10-01T09:27:55.206 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0042971559999999999 2023-10-01T09:27:55.206 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.206 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.206 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.206 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.794653+0000", 2023-10-01T09:27:55.206 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001920092 2023-10-01T09:27:55.206 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.206 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.207 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.207 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.796573+0000", 2023-10-01T09:27:55.207 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4035e-05 2023-10-01T09:27:55.207 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.207 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.207 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:27:55.207 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.796627+0000", 2023-10-01T09:27:55.207 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000116727 2023-10-01T09:27:55.207 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.207 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.207 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.207 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.796743+0000", 2023-10-01T09:27:55.207 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2188770770000001 2023-10-01T09:27:55.208 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.208 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.208 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.208 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.208 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.208 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.34683.0:227541 5.d 5:b6c4db5e:::benchmark_data_smithi138_240338_object112404:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:27:55.208 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:19:04.590428+0000", 2023-10-01T09:27:55.208 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 530.50056002099996, 2023-10-01T09:27:55.208 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2046087399999998, 2023-10-01T09:27:55.208 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.208 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:27:55.209 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.209 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.34683", 2023-10-01T09:27:55.209 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:27:55.209 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 227541 2023-10-01T09:27:55.209 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.209 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.209 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.209 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.209 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.590428+0000", 2023-10-01T09:27:55.209 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.209 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.209 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.209 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.210 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.590428+0000", 2023-10-01T09:27:55.210 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:27:55.210 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.210 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.210 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.210 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.590427+0000", 2023-10-01T09:27:55.210 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6340000000000003e-06 2023-10-01T09:27:55.210 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.210 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.210 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.210 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.590433+0000", 2023-10-01T09:27:55.210 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1999e-05 2023-10-01T09:27:55.211 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.211 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.211 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.211 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.590445+0000", 2023-10-01T09:27:55.211 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4260000000000002e-06 2023-10-01T09:27:55.211 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.211 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.211 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.211 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.590451+0000", 2023-10-01T09:27:55.211 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7330999999999997e-05 2023-10-01T09:27:55.211 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.211 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.212 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.212 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.590508+0000", 2023-10-01T09:27:55.212 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0288999999999999e-05 2023-10-01T09:27:55.212 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.212 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.212 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.212 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:04.590518+0000", 2023-10-01T09:27:55.212 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.95989403600000001 2023-10-01T09:27:55.212 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.212 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.212 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.212 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.550412+0000", 2023-10-01T09:27:55.212 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7089999999999997e-06 2023-10-01T09:27:55.213 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.213 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.213 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.213 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:05.550422+0000", 2023-10-01T09:27:55.213 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.216798485 2023-10-01T09:27:55.213 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.213 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.213 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.213 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.767221+0000", 2023-10-01T09:27:55.213 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015531599999999999 2023-10-01T09:27:55.213 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.213 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.214 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.214 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.767376+0000", 2023-10-01T09:27:55.214 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014835099999999999 2023-10-01T09:27:55.214 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.214 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.214 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,2,5", 2023-10-01T09:27:55.214 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.767524+0000", 2023-10-01T09:27:55.214 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0062782910000000001 2023-10-01T09:27:55.214 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.214 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.214 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.214 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.773803+0000", 2023-10-01T09:27:55.215 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00246722 2023-10-01T09:27:55.215 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.215 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.215 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.215 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.776270+0000", 2023-10-01T09:27:55.215 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010348384 2023-10-01T09:27:55.215 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.215 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.215 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.215 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.786618+0000", 2023-10-01T09:27:55.215 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0082576179999999996 2023-10-01T09:27:55.215 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.216 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.216 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.216 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.794876+0000", 2023-10-01T09:27:55.216 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5889999999999994e-05 2023-10-01T09:27:55.216 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.216 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.216 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:27:55.216 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.794942+0000", 2023-10-01T09:27:55.216 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5447999999999998e-05 2023-10-01T09:27:55.216 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.216 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.216 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.217 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:19:06.795037+0000", 2023-10-01T09:27:55.217 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2046087399999998 2023-10-01T09:27:55.217 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.217 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.217 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.217 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.217 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.217 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:121883 7.8 7:1d9910e1:::benchmark_data_smithi138_367058_object24088:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-10-01T09:27:55.217 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:26:29.364699+0000", 2023-10-01T09:27:55.217 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 85.726289010000002, 2023-10-01T09:27:55.217 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2421492629999999, 2023-10-01T09:27:55.218 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.218 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:27:55.218 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.218 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:27:55.218 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:27:55.218 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121883 2023-10-01T09:27:55.218 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.218 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.218 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.218 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.218 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364699+0000", 2023-10-01T09:27:55.219 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.219 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.219 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.219 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.219 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364699+0000", 2023-10-01T09:27:55.219 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:27:55.219 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.219 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.219 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.219 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364699+0000", 2023-10-01T09:27:55.219 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.618e-06 2023-10-01T09:27:55.219 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.220 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.220 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.220 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364701+0000", 2023-10-01T09:27:55.220 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1879999999999998e-06 2023-10-01T09:27:55.220 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.220 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.220 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.220 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364704+0000", 2023-10-01T09:27:55.220 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2629999999999999e-06 2023-10-01T09:27:55.220 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.220 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.220 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.221 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364707+0000", 2023-10-01T09:27:55.221 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00023008100000000001 2023-10-01T09:27:55.221 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.221 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.221 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.221 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364937+0000", 2023-10-01T09:27:55.221 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7380000000000002e-06 2023-10-01T09:27:55.221 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.221 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.221 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting_for_map not empty", 2023-10-01T09:27:55.221 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364942+0000", 2023-10-01T09:27:55.221 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0022245450000000001 2023-10-01T09:27:55.222 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.222 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.222 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.222 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.367167+0000", 2023-10-01T09:27:55.222 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6610000000000002e-06 2023-10-01T09:27:55.222 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.222 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.222 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.222 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.367170+0000", 2023-10-01T09:27:55.222 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.092279784 2023-10-01T09:27:55.222 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.222 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.223 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.223 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.459450+0000", 2023-10-01T09:27:55.223 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001133841 2023-10-01T09:27:55.223 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.223 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.223 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for degraded object", 2023-10-01T09:27:55.223 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.460584+0000", 2023-10-01T09:27:55.223 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.078443947 2023-10-01T09:27:55.223 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.223 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.223 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.223 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.539028+0000", 2023-10-01T09:27:55.223 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0853999999999994e-05 2023-10-01T09:27:55.224 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.224 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.224 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.224 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.539119+0000", 2023-10-01T09:27:55.224 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016501499999999999 2023-10-01T09:27:55.224 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.224 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.224 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:27:55.224 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.539284+0000", 2023-10-01T09:27:55.224 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0021284239999999999 2023-10-01T09:27:55.224 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.224 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.225 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.225 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.541412+0000", 2023-10-01T09:27:55.225 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0020002430000000001 2023-10-01T09:27:55.225 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.225 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.225 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.225 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.543412+0000", 2023-10-01T09:27:55.225 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.061766067000000001 2023-10-01T09:27:55.225 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.225 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.225 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.225 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.605178+0000", 2023-10-01T09:27:55.226 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8877000000000002e-05 2023-10-01T09:27:55.226 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.226 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.226 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:27:55.226 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.605207+0000", 2023-10-01T09:27:55.226 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001641361 2023-10-01T09:27:55.226 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.226 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.226 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.226 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.606849+0000", 2023-10-01T09:27:55.226 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2421492629999999 2023-10-01T09:27:55.226 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.227 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.227 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.227 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.227 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.227 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:121885 7.8 7:123ac357:::benchmark_data_smithi138_367058_object24090:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-10-01T09:27:55.227 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:26:29.364721+0000", 2023-10-01T09:27:55.227 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 85.726267179000004, 2023-10-01T09:27:55.227 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2440128500000001, 2023-10-01T09:27:55.227 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.227 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:27:55.228 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.228 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:27:55.228 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:27:55.228 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121885 2023-10-01T09:27:55.228 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.228 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.228 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.228 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.228 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364721+0000", 2023-10-01T09:27:55.228 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.228 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.228 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.229 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.229 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364721+0000", 2023-10-01T09:27:55.229 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:27:55.229 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.229 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.229 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.229 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364721+0000", 2023-10-01T09:27:55.229 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5629999999999999e-06 2023-10-01T09:27:55.229 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.229 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.229 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.230 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364722+0000", 2023-10-01T09:27:55.230 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.063e-06 2023-10-01T09:27:55.230 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.230 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.230 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.230 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364730+0000", 2023-10-01T09:27:55.230 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5000000000000002e-06 2023-10-01T09:27:55.230 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.230 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.230 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.230 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364732+0000", 2023-10-01T09:27:55.231 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00023374299999999999 2023-10-01T09:27:55.231 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.231 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.231 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.231 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364966+0000", 2023-10-01T09:27:55.231 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0210000000000001e-06 2023-10-01T09:27:55.231 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.231 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.231 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting_for_map not empty", 2023-10-01T09:27:55.231 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364970+0000", 2023-10-01T09:27:55.231 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0022192000000000002 2023-10-01T09:27:55.232 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.232 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.232 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.232 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.367189+0000", 2023-10-01T09:27:55.232 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7469999999999999e-06 2023-10-01T09:27:55.232 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.232 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.232 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.232 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.367193+0000", 2023-10-01T09:27:55.232 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0934347339999999 2023-10-01T09:27:55.232 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.232 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.233 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.233 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.460627+0000", 2023-10-01T09:27:55.233 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00055894799999999998 2023-10-01T09:27:55.233 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.233 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.233 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for degraded object", 2023-10-01T09:27:55.233 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.461186+0000", 2023-10-01T09:27:55.233 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.094048734999999994 2023-10-01T09:27:55.233 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.233 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.234 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.234 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.555235+0000", 2023-10-01T09:27:55.234 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6341000000000005e-05 2023-10-01T09:27:55.234 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.234 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.234 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.234 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.555332+0000", 2023-10-01T09:27:55.234 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017159099999999999 2023-10-01T09:27:55.234 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.234 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.234 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:27:55.235 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.555503+0000", 2023-10-01T09:27:55.235 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0027180009999999998 2023-10-01T09:27:55.235 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.235 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.235 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.235 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.558221+0000", 2023-10-01T09:27:55.235 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0014135420000000001 2023-10-01T09:27:55.235 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.235 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.235 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.236 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.559635+0000", 2023-10-01T09:27:55.236 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.047262169999999999 2023-10-01T09:27:55.236 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.236 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.236 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.236 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.606897+0000", 2023-10-01T09:27:55.236 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2144000000000003e-05 2023-10-01T09:27:55.236 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.236 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.236 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:27:55.236 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.606929+0000", 2023-10-01T09:27:55.236 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0018050690000000001 2023-10-01T09:27:55.237 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.237 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.237 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.237 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.608734+0000", 2023-10-01T09:27:55.237 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2440128500000001 2023-10-01T09:27:55.237 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.237 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.237 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.237 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.237 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.237 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:121889 7.8 7:15207cc8:::benchmark_data_smithi138_367058_object24094:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-10-01T09:27:55.237 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:26:29.364738+0000", 2023-10-01T09:27:55.238 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 85.726249969999998, 2023-10-01T09:27:55.238 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.245699254, 2023-10-01T09:27:55.238 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.238 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:27:55.238 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.238 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:27:55.238 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:27:55.238 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121889 2023-10-01T09:27:55.238 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.238 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.238 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.238 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.239 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364738+0000", 2023-10-01T09:27:55.239 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.239 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.239 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.239 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.239 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364738+0000", 2023-10-01T09:27:55.239 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:27:55.239 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.239 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.239 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.239 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364738+0000", 2023-10-01T09:27:55.239 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.668e-06 2023-10-01T09:27:55.239 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.240 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.240 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.240 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364740+0000", 2023-10-01T09:27:55.240 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1130000000000001e-06 2023-10-01T09:27:55.240 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.240 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.240 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.240 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364743+0000", 2023-10-01T09:27:55.240 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5809999999999999e-06 2023-10-01T09:27:55.240 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.240 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.240 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.241 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364745+0000", 2023-10-01T09:27:55.241 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000251394 2023-10-01T09:27:55.241 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.241 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.241 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.241 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364997+0000", 2023-10-01T09:27:55.241 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9779999999999999e-06 2023-10-01T09:27:55.241 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.241 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.241 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting_for_map not empty", 2023-10-01T09:27:55.241 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.365001+0000", 2023-10-01T09:27:55.241 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002210895 2023-10-01T09:27:55.242 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.242 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.242 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.242 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.367212+0000", 2023-10-01T09:27:55.242 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6509999999999999e-06 2023-10-01T09:27:55.242 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.242 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.242 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.242 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.367215+0000", 2023-10-01T09:27:55.242 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0940297409999999 2023-10-01T09:27:55.242 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.243 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.243 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.243 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.461245+0000", 2023-10-01T09:27:55.243 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00056327299999999999 2023-10-01T09:27:55.243 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.243 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.243 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for degraded object", 2023-10-01T09:27:55.243 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.461808+0000", 2023-10-01T09:27:55.243 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.108540409 2023-10-01T09:27:55.243 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.243 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.244 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.244 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.570349+0000", 2023-10-01T09:27:55.244 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1264000000000002e-05 2023-10-01T09:27:55.244 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.244 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.244 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.244 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.570410+0000", 2023-10-01T09:27:55.244 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3289999999999996e-05 2023-10-01T09:27:55.244 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.244 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.244 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:27:55.244 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.570503+0000", 2023-10-01T09:27:55.245 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.006449237 2023-10-01T09:27:55.245 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.245 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.245 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.245 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.576953+0000", 2023-10-01T09:27:55.245 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016146470999999999 2023-10-01T09:27:55.245 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.245 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.245 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.245 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.593099+0000", 2023-10-01T09:27:55.245 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015687586 2023-10-01T09:27:55.245 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.246 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.246 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.246 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.608787+0000", 2023-10-01T09:27:55.246 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0846e-05 2023-10-01T09:27:55.246 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.246 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.246 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:27:55.246 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.608848+0000", 2023-10-01T09:27:55.246 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0015901050000000001 2023-10-01T09:27:55.246 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.246 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.246 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.247 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.610438+0000", 2023-10-01T09:27:55.247 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.245699254 2023-10-01T09:27:55.247 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.247 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.247 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.247 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.247 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.247 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:121890 7.8 7:15040ec4:::benchmark_data_smithi138_367058_object24095:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-10-01T09:27:55.247 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:26:29.364748+0000", 2023-10-01T09:27:55.247 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 85.726240606999994, 2023-10-01T09:27:55.247 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.247185129, 2023-10-01T09:27:55.247 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.248 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:27:55.248 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.248 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:27:55.248 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:27:55.248 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121890 2023-10-01T09:27:55.248 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.248 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.248 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.248 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.248 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364748+0000", 2023-10-01T09:27:55.248 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.248 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.249 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.249 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.249 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364748+0000", 2023-10-01T09:27:55.249 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:27:55.249 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.249 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.249 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.249 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364748+0000", 2023-10-01T09:27:55.249 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.59e-06 2023-10-01T09:27:55.249 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.249 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.249 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.249 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364749+0000", 2023-10-01T09:27:55.250 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.937e-06 2023-10-01T09:27:55.250 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.250 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.250 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.250 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364752+0000", 2023-10-01T09:27:55.250 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.12e-06 2023-10-01T09:27:55.250 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.250 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.250 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.250 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364754+0000", 2023-10-01T09:27:55.250 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00027553199999999999 2023-10-01T09:27:55.250 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.251 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.251 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.251 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.365030+0000", 2023-10-01T09:27:55.251 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9040000000000002e-06 2023-10-01T09:27:55.251 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.251 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.251 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting_for_map not empty", 2023-10-01T09:27:55.251 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.365034+0000", 2023-10-01T09:27:55.251 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002200788 2023-10-01T09:27:55.251 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.251 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.252 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.252 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.367234+0000", 2023-10-01T09:27:55.252 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.597e-06 2023-10-01T09:27:55.252 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.252 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.252 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.252 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.367238+0000", 2023-10-01T09:27:55.252 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0946307420000001 2023-10-01T09:27:55.252 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.252 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.252 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.252 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.461869+0000", 2023-10-01T09:27:55.252 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00044317299999999999 2023-10-01T09:27:55.253 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.253 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.253 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for degraded object", 2023-10-01T09:27:55.253 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.462312+0000", 2023-10-01T09:27:55.253 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.112725586 2023-10-01T09:27:55.253 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.253 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.253 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.253 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.575037+0000", 2023-10-01T09:27:55.253 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7856999999999998e-05 2023-10-01T09:27:55.253 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.253 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.254 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.254 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.575095+0000", 2023-10-01T09:27:55.254 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7907999999999998e-05 2023-10-01T09:27:55.254 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.254 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.254 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:27:55.254 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.575193+0000", 2023-10-01T09:27:55.254 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019565816 2023-10-01T09:27:55.254 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.254 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.254 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.254 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.594759+0000", 2023-10-01T09:27:55.255 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3133999999999995e-05 2023-10-01T09:27:55.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.255 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.255 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.594832+0000", 2023-10-01T09:27:55.255 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015652741000000001 2023-10-01T09:27:55.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.255 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.255 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.610485+0000", 2023-10-01T09:27:55.255 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9743999999999998e-05 2023-10-01T09:27:55.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.256 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.256 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:27:55.256 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.610525+0000", 2023-10-01T09:27:55.256 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0014082000000000001 2023-10-01T09:27:55.256 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.256 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.256 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.256 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.611933+0000", 2023-10-01T09:27:55.256 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.247185129 2023-10-01T09:27:55.256 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.256 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.256 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.257 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.257 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.257 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:121892 7.8 7:1db79fc6:::benchmark_data_smithi138_367058_object24097:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-10-01T09:27:55.257 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:26:29.364770+0000", 2023-10-01T09:27:55.257 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 85.726218652, 2023-10-01T09:27:55.257 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.248471213, 2023-10-01T09:27:55.257 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.257 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:27:55.257 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.257 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:27:55.257 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:27:55.257 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121892 2023-10-01T09:27:55.258 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.258 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.258 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.258 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.258 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364770+0000", 2023-10-01T09:27:55.258 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.258 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.258 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.258 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.258 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364770+0000", 2023-10-01T09:27:55.258 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:27:55.259 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.259 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.259 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.259 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364769+0000", 2023-10-01T09:27:55.259 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.759e-06 2023-10-01T09:27:55.259 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.259 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.259 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.259 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364771+0000", 2023-10-01T09:27:55.259 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8683000000000001e-05 2023-10-01T09:27:55.259 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.259 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.260 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.260 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364790+0000", 2023-10-01T09:27:55.260 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2079999999999999e-06 2023-10-01T09:27:55.260 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.260 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.260 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.260 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364792+0000", 2023-10-01T09:27:55.260 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00026696000000000001 2023-10-01T09:27:55.260 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.260 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.260 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.261 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.365059+0000", 2023-10-01T09:27:55.261 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.604e-06 2023-10-01T09:27:55.261 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.261 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.261 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting_for_map not empty", 2023-10-01T09:27:55.261 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.365065+0000", 2023-10-01T09:27:55.261 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0021921599999999999 2023-10-01T09:27:55.261 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.261 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.261 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.262 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.367257+0000", 2023-10-01T09:27:55.262 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7100000000000001e-06 2023-10-01T09:27:55.262 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.262 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.262 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.262 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.367261+0000", 2023-10-01T09:27:55.262 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0950948439999999 2023-10-01T09:27:55.262 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.262 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.262 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.262 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.462355+0000", 2023-10-01T09:27:55.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00065737 2023-10-01T09:27:55.263 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.263 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.263 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for degraded object", 2023-10-01T09:27:55.263 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.463013+0000", 2023-10-01T09:27:55.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.116815619 2023-10-01T09:27:55.263 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.263 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.263 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.263 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.579828+0000", 2023-10-01T09:27:55.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1589999999999998e-05 2023-10-01T09:27:55.263 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.264 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.264 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.264 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.579890+0000", 2023-10-01T09:27:55.264 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000101096 2023-10-01T09:27:55.264 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.264 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.264 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:27:55.264 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.579991+0000", 2023-10-01T09:27:55.264 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014772410999999999 2023-10-01T09:27:55.264 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.265 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.265 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.265 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.594763+0000", 2023-10-01T09:27:55.265 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0016449749999999999 2023-10-01T09:27:55.265 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.265 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.265 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.265 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.596408+0000", 2023-10-01T09:27:55.265 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015571323999999999 2023-10-01T09:27:55.265 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.266 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.266 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.266 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.611980+0000", 2023-10-01T09:27:55.266 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1693999999999998e-05 2023-10-01T09:27:55.266 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.266 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.266 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:27:55.266 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.612011+0000", 2023-10-01T09:27:55.266 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001229464 2023-10-01T09:27:55.266 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.267 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.267 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.267 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.613241+0000", 2023-10-01T09:27:55.267 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.248471213 2023-10-01T09:27:55.267 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.267 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.267 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.267 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.267 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.267 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:121902 7.8 7:16028f31:::benchmark_data_smithi138_367058_object24107:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-10-01T09:27:55.267 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:26:29.364794+0000", 2023-10-01T09:27:55.267 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 85.726193898999995, 2023-10-01T09:27:55.268 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2497131669999999, 2023-10-01T09:27:55.268 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.268 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:27:55.268 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.268 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:27:55.268 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:27:55.268 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121902 2023-10-01T09:27:55.268 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.268 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.268 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.268 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.269 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364794+0000", 2023-10-01T09:27:55.269 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.269 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.269 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364794+0000", 2023-10-01T09:27:55.269 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:27:55.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.269 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.269 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364794+0000", 2023-10-01T09:27:55.270 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.561e-06 2023-10-01T09:27:55.270 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.270 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.270 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364796+0000", 2023-10-01T09:27:55.270 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9809999999999999e-06 2023-10-01T09:27:55.270 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.270 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.270 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364799+0000", 2023-10-01T09:27:55.270 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9469999999999999e-06 2023-10-01T09:27:55.270 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.271 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364802+0000", 2023-10-01T09:27:55.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00031120600000000002 2023-10-01T09:27:55.271 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.271 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.365113+0000", 2023-10-01T09:27:55.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5900000000000002e-06 2023-10-01T09:27:55.271 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.271 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting_for_map not empty", 2023-10-01T09:27:55.272 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.365120+0000", 2023-10-01T09:27:55.272 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0021590020000000001 2023-10-01T09:27:55.272 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.272 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.272 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.272 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.367280+0000", 2023-10-01T09:27:55.272 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6689999999999999e-06 2023-10-01T09:27:55.272 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.272 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.272 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.272 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.367283+0000", 2023-10-01T09:27:55.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.095789685 2023-10-01T09:27:55.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.273 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.273 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.463073+0000", 2023-10-01T09:27:55.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00059084899999999995 2023-10-01T09:27:55.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.273 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for degraded object", 2023-10-01T09:27:55.273 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.463664+0000", 2023-10-01T09:27:55.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.120648115 2023-10-01T09:27:55.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.584312+0000", 2023-10-01T09:27:55.274 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3280000000000003e-05 2023-10-01T09:27:55.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.584385+0000", 2023-10-01T09:27:55.274 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6377000000000002e-05 2023-10-01T09:27:55.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:27:55.275 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.584481+0000", 2023-10-01T09:27:55.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010285346000000001 2023-10-01T09:27:55.275 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.275 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.275 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.275 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.594767+0000", 2023-10-01T09:27:55.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003414692 2023-10-01T09:27:55.275 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.275 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.275 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.276 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.598182+0000", 2023-10-01T09:27:55.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015105560000000001 2023-10-01T09:27:55.276 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.276 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.276 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.276 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.613287+0000", 2023-10-01T09:27:55.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0715e-05 2023-10-01T09:27:55.276 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.276 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.276 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:27:55.276 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.613318+0000", 2023-10-01T09:27:55.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0011898340000000001 2023-10-01T09:27:55.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.277 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.277 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.277 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.614508+0000", 2023-10-01T09:27:55.277 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2497131669999999 2023-10-01T09:27:55.277 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.277 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.277 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.277 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.277 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:121920 7.8 7:1afe117c:::benchmark_data_smithi138_367058_object24125:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-10-01T09:27:55.278 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:26:29.364805+0000", 2023-10-01T09:27:55.278 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 85.726183407999997, 2023-10-01T09:27:55.278 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2618444630000001, 2023-10-01T09:27:55.278 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:27:55.278 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:27:55.278 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:27:55.278 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:27:55.278 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:27:55.278 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 121920 2023-10-01T09:27:55.278 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.278 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:27:55.278 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:27:55.279 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364805+0000", 2023-10-01T09:27:55.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:27:55.279 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.279 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:27:55.279 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364805+0000", 2023-10-01T09:27:55.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:27:55.279 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.279 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:27:55.279 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364805+0000", 2023-10-01T09:27:55.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3089999999999998e-06 2023-10-01T09:27:55.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:27:55.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364807+0000", 2023-10-01T09:27:55.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0220000000000004e-06 2023-10-01T09:27:55.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:27:55.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364812+0000", 2023-10-01T09:27:55.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8979999999999998e-06 2023-10-01T09:27:55.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:27:55.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.364816+0000", 2023-10-01T09:27:55.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00033698300000000001 2023-10-01T09:27:55.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.365153+0000", 2023-10-01T09:27:55.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.8090000000000004e-06 2023-10-01T09:27:55.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting_for_map not empty", 2023-10-01T09:27:55.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.365163+0000", 2023-10-01T09:27:55.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002132648 2023-10-01T09:27:55.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.282 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.367295+0000", 2023-10-01T09:27:55.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9840000000000003e-06 2023-10-01T09:27:55.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.282 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:27:55.283 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:29.367299+0000", 2023-10-01T09:27:55.283 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0964230939999999 2023-10-01T09:27:55.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.283 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.283 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.463722+0000", 2023-10-01T09:27:55.283 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00064681500000000002 2023-10-01T09:27:55.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.283 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for degraded object", 2023-10-01T09:27:55.283 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.464369+0000", 2023-10-01T09:27:55.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.12461100999999999 2023-10-01T09:27:55.284 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:27:55.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.588980+0000", 2023-10-01T09:27:55.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6649000000000003e-05 2023-10-01T09:27:55.284 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:27:55.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.589067+0000", 2023-10-01T09:27:55.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000144163 2023-10-01T09:27:55.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.285 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:27:55.285 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.589211+0000", 2023-10-01T09:27:55.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0055608799999999998 2023-10-01T09:27:55.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.285 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:27:55.285 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.594772+0000", 2023-10-01T09:27:55.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0051975249999999997 2023-10-01T09:27:55.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.286 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.286 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.599969+0000", 2023-10-01T09:27:55.286 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.023902783 2023-10-01T09:27:55.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.286 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.286 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:27:55.286 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.623872+0000", 2023-10-01T09:27:55.286 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2179999999999999e-05 2023-10-01T09:27:55.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.286 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.286 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:27:55.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.623914+0000", 2023-10-01T09:27:55.287 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002735033 2023-10-01T09:27:55.287 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:27:55.287 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:27:55.287 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:27:55.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:26:30.626649+0000", 2023-10-01T09:27:55.287 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2618444630000001 2023-10-01T09:27:55.287 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.287 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.287 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.287 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:27:55.287 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:27:55.288 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:27:55.288 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:27:55.288 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[185886]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.288 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[185886]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:27:55.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[185886]: osdmap e305: 8 total, 8 up, 6 in 2023-10-01T09:27:55.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[185886]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[185886]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[185886]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[185886]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[185886]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.289 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:54 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:27:55.290 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:54 smithi190 ceph-mon[160435]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.290 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:54 smithi190 ceph-mon[160435]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.290 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:54 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:27:55.290 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:54 smithi190 ceph-mon[160435]: osdmap e305: 8 total, 8 up, 6 in 2023-10-01T09:27:55.290 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:54 smithi190 ceph-mon[160435]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.290 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:54 smithi190 ceph-mon[160435]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.290 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:54 smithi190 ceph-mon[160435]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.290 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:54 smithi190 ceph-mon[160435]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.290 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:54 smithi190 ceph-mon[160435]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.290 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:27:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:27:54] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.33.4" 2023-10-01T09:27:55.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:27:55.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[192513]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[192513]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:27:55.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[192513]: osdmap e305: 8 total, 8 up, 6 in 2023-10-01T09:27:55.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[192513]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[192513]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[192513]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[192513]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:54 smithi138 ceph-mon[192513]: failed to encode map e305 with expected crc 2023-10-01T09:27:55.722 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:55.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:55.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:55.818 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:27:55.834 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:27:55.922 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:56.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:27:56.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:56 smithi190 ceph-mon[160435]: pgmap v823: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.3 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 471 op/s; 25654/112998 objects degraded (22.703%); 11028/112998 objects misplaced (9.759%); 4.3 MiB/s, 68 objects/s recovering 2023-10-01T09:27:56.064 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:27:56.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:56.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:27:56.180 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:27:56.214 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:56.249 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:27:56.258 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:27:56.258 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:56 smithi138 ceph-mon[192513]: pgmap v823: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.3 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 471 op/s; 25654/112998 objects degraded (22.703%); 11028/112998 objects misplaced (9.759%); 4.3 MiB/s, 68 objects/s recovering 2023-10-01T09:27:56.258 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:27:56.258 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:56 smithi138 ceph-mon[185886]: pgmap v823: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.3 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 471 op/s; 25654/112998 objects degraded (22.703%); 11028/112998 objects misplaced (9.759%); 4.3 MiB/s, 68 objects/s recovering 2023-10-01T09:27:56.275 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:27:56.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:56.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:27:56.384 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:27:56.481 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:56.599 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:27:56.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:56.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:27:56.647 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:27:56.701 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:27:56.703 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:27:56.738 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:56.749 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:27:56.802 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:27:56.843 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:27:56.893 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:27:56.896 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:27:56.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:56.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:27:56.927 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:27:56.966 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:27:56.988 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:27:56.988 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:27:56.988 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:27:56.988 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:27:56.988 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:27:56.988 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:27:56.989 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:27:56.989 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:27:57.001 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:27:57.031 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/196597159' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:57.031 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:57.081 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:27:57.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:57.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:27:57.185 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:27:57.213 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:57.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/196597159' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:57.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/196597159' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:27:57.444 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:27:57.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:57.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:27:57.549 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:57.637 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:57.714 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5] out_osds: [6, 4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2023-10-01T09:27:57.714 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:27:57.714 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-01T09:27:57.715 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-01T09:27:57.815 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:57.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:57.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:57.905 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:27:57.905 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:27:57.925 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:57.989 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:58.048 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:58.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:58.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:58.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:27:58.152 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:27:58.183 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:58 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 25654/112998 objects degraded (22.703%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:58.183 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:58 smithi190 ceph-mon[160435]: pgmap v824: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.2 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 478 op/s; 25355/109728 objects degraded (23.107%); 10546/109728 objects misplaced (9.611%); 4.4 MiB/s, 70 objects/s recovering 2023-10-01T09:27:58.186 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:58.235 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:27:58.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:58.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:27:58.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:58 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 25654/112998 objects degraded (22.703%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:58.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:58 smithi138 ceph-mon[192513]: pgmap v824: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.2 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 478 op/s; 25355/109728 objects degraded (23.107%); 10546/109728 objects misplaced (9.611%); 4.4 MiB/s, 70 objects/s recovering 2023-10-01T09:27:58.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:58 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 25654/112998 objects degraded (22.703%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:27:58.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:58 smithi138 ceph-mon[185886]: pgmap v824: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.2 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 478 op/s; 25355/109728 objects degraded (23.107%); 10546/109728 objects misplaced (9.611%); 4.4 MiB/s, 70 objects/s recovering 2023-10-01T09:27:58.339 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:27:58.437 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:58.572 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T09:27:58.591 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:27:58.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:58.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:27:58.701 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:27:58.708 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:58.731 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:58.785 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:27:58.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:58.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:27:58.895 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:27:58.980 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:58.997 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:27:59.126 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:27:59.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:59.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:27:59.240 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:27:59.328 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:59.369 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:27:59.520 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:27:59.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:27:59.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:27:59.631 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:27:59.690 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:27:59.929 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:27:59 smithi190 ceph-mon[160435]: pgmap v825: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.2 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 489 op/s; 25355/105990 objects degraded (23.922%); 10226/105990 objects misplaced (9.648%); 3.9 MiB/s, 63 objects/s recovering 2023-10-01T09:28:00.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:27:59 smithi138 ceph-mon[185886]: pgmap v825: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.2 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 489 op/s; 25355/105990 objects degraded (23.922%); 10226/105990 objects misplaced (9.648%); 3.9 MiB/s, 63 objects/s recovering 2023-10-01T09:28:00.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:27:59 smithi138 ceph-mon[192513]: pgmap v825: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.2 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 489 op/s; 25355/105990 objects degraded (23.922%); 10226/105990 objects misplaced (9.648%); 3.9 MiB/s, 63 objects/s recovering 2023-10-01T09:28:00.395 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:00.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:00.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:00.501 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:28:00.534 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:00.578 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:28:00.586 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:28:00.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:00.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:28:00.636 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:28:00.636 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:28:00.692 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:00.807 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:01.008 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:01.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:01.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:01.117 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:01.149 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:01.334 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T09:28:01.337 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:01.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:01.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:01.443 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:01.474 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:01.524 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:01.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:01.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:01.630 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:01.665 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:01.715 INFO:teuthology.orchestra.run.smithi138.stderr:marked in osd.6. 2023-10-01T09:28:01.749 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:01.751 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/747188517' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T09:28:01.751 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:01 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T09:28:01.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:01.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:01.797 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:28:01.854 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:02.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:01 smithi138 conmon[185862]: 2023-10-01T09:28:01.610+0000 7f70d64ec700 -1 mon.a@0(leader).osd e305 definitely_dead 0 2023-10-01T09:28:02.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/747188517' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T09:28:02.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:01 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T09:28:02.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/747188517' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T09:28:02.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:01 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T09:28:02.075 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:03.026 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:03.028 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T09:28:03.028 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: osdmap e306: 8 total, 8 up, 7 in 2023-10-01T09:28:03.028 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: pgmap v827: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 373 op/s; 25191/104475 objects degraded (24.112%); 10226/104475 objects misplaced (9.788%); 3.8 MiB/s, 61 objects/s recovering 2023-10-01T09:28:03.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 25355/105990 objects degraded (23.922%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:03.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: osdmap e307: 8 total, 8 up, 7 in 2023-10-01T09:28:03.030 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.030 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[192513]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T09:28:03.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: osdmap e306: 8 total, 8 up, 7 in 2023-10-01T09:28:03.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.031 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: pgmap v827: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 373 op/s; 25191/104475 objects degraded (24.112%); 10226/104475 objects misplaced (9.788%); 3.8 MiB/s, 61 objects/s recovering 2023-10-01T09:28:03.031 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 25355/105990 objects degraded (23.922%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:03.031 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.031 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.031 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.031 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.031 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: osdmap e307: 8 total, 8 up, 7 in 2023-10-01T09:28:03.031 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.031 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:02 smithi138 ceph-mon[185886]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:03.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:03.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T09:28:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: osdmap e306: 8 total, 8 up, 7 in 2023-10-01T09:28:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: failed to encode map e306 with expected crc 2023-10-01T09:28:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: pgmap v827: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 2.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 373 op/s; 25191/104475 objects degraded (24.112%); 10226/104475 objects misplaced (9.788%); 3.8 MiB/s, 61 objects/s recovering 2023-10-01T09:28:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 25355/105990 objects degraded (23.922%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: osdmap e307: 8 total, 8 up, 7 in 2023-10-01T09:28:03.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:02 smithi190 ceph-mon[160435]: failed to encode map e307 with expected crc 2023-10-01T09:28:03.132 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:03.165 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:03.246 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:03.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:03.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:03.350 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:03.400 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:03.520 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:03.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:03.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:03.625 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:28:03.630 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:28:03.631 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:28:03.712 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:04.028 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[185886]: 7.3 starting backfill to osd.6 from (0'0,0'0] MAX to 305'10112 2023-10-01T09:28:04.028 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[185886]: failed to encode map e307 with expected crc 2023-10-01T09:28:04.028 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[185886]: failed to encode map e307 with expected crc 2023-10-01T09:28:04.028 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[185886]: failed to encode map e307 with expected crc 2023-10-01T09:28:04.028 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[185886]: 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 305'10246 2023-10-01T09:28:04.029 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[185886]: failed to encode map e308 with expected crc 2023-10-01T09:28:04.029 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[185886]: failed to encode map e308 with expected crc 2023-10-01T09:28:04.029 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[185886]: osdmap e308: 8 total, 8 up, 7 in 2023-10-01T09:28:04.029 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[185886]: failed to encode map e308 with expected crc 2023-10-01T09:28:04.029 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[185886]: failed to encode map e308 with expected crc 2023-10-01T09:28:04.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[192513]: 7.3 starting backfill to osd.6 from (0'0,0'0] MAX to 305'10112 2023-10-01T09:28:04.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[192513]: failed to encode map e307 with expected crc 2023-10-01T09:28:04.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[192513]: failed to encode map e307 with expected crc 2023-10-01T09:28:04.029 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[192513]: failed to encode map e307 with expected crc 2023-10-01T09:28:04.030 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[192513]: 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 305'10246 2023-10-01T09:28:04.030 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[192513]: failed to encode map e308 with expected crc 2023-10-01T09:28:04.030 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[192513]: failed to encode map e308 with expected crc 2023-10-01T09:28:04.030 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[192513]: osdmap e308: 8 total, 8 up, 7 in 2023-10-01T09:28:04.030 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[192513]: failed to encode map e308 with expected crc 2023-10-01T09:28:04.030 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:03 smithi138 ceph-mon[192513]: failed to encode map e308 with expected crc 2023-10-01T09:28:04.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:03 smithi190 ceph-mon[160435]: 7.3 starting backfill to osd.6 from (0'0,0'0] MAX to 305'10112 2023-10-01T09:28:04.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:03 smithi190 ceph-mon[160435]: failed to encode map e307 with expected crc 2023-10-01T09:28:04.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:03 smithi190 ceph-mon[160435]: failed to encode map e307 with expected crc 2023-10-01T09:28:04.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:03 smithi190 ceph-mon[160435]: failed to encode map e307 with expected crc 2023-10-01T09:28:04.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:03 smithi190 ceph-mon[160435]: 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 305'10246 2023-10-01T09:28:04.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:03 smithi190 ceph-mon[160435]: failed to encode map e308 with expected crc 2023-10-01T09:28:04.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:03 smithi190 ceph-mon[160435]: failed to encode map e308 with expected crc 2023-10-01T09:28:04.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:03 smithi190 ceph-mon[160435]: osdmap e308: 8 total, 8 up, 7 in 2023-10-01T09:28:04.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:03 smithi190 ceph-mon[160435]: failed to encode map e308 with expected crc 2023-10-01T09:28:04.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:03 smithi190 ceph-mon[160435]: failed to encode map e308 with expected crc 2023-10-01T09:28:04.101 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-01T09:28:04.101 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:28:04.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:04.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:28:04.208 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:04.265 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:28:04.276 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:04.286 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:28:04.390 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:04.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:04.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:04.495 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:04.581 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:04.708 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:04.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:04.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:04.775 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:28:04.811 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:28:04.879 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:04.939 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:28:04.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:04.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:28:05.012 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:04 smithi190 ceph-mon[160435]: failed to encode map e308 with expected crc 2023-10-01T09:28:05.013 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:04 smithi190 ceph-mon[160435]: 7.1 continuing backfill to osd.1 from (268'5117,305'10026] MIN to 305'10026 2023-10-01T09:28:05.013 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:04 smithi190 ceph-mon[160435]: pgmap v830: 89 pgs: 2 remapped+peering, 6 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 2.0 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 20844/99411 objects degraded (20.967%); 4184/99411 objects misplaced (4.209%); 5.1 MiB/s, 90 objects/s recovering 2023-10-01T09:28:05.015 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:28:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:28:04] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.33.4" 2023-10-01T09:28:05.016 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:04 smithi138 ceph-mon[192513]: failed to encode map e308 with expected crc 2023-10-01T09:28:05.016 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:04 smithi138 ceph-mon[192513]: 7.1 continuing backfill to osd.1 from (268'5117,305'10026] MIN to 305'10026 2023-10-01T09:28:05.017 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:04 smithi138 ceph-mon[192513]: pgmap v830: 89 pgs: 2 remapped+peering, 6 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 2.0 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 20844/99411 objects degraded (20.967%); 4184/99411 objects misplaced (4.209%); 5.1 MiB/s, 90 objects/s recovering 2023-10-01T09:28:05.017 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:04 smithi138 ceph-mon[185886]: failed to encode map e308 with expected crc 2023-10-01T09:28:05.017 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:04 smithi138 ceph-mon[185886]: 7.1 continuing backfill to osd.1 from (268'5117,305'10026] MIN to 305'10026 2023-10-01T09:28:05.017 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:04 smithi138 ceph-mon[185886]: pgmap v830: 89 pgs: 2 remapped+peering, 6 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 2.0 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 20844/99411 objects degraded (20.967%); 4184/99411 objects misplaced (4.209%); 5.1 MiB/s, 90 objects/s recovering 2023-10-01T09:28:05.044 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:05.141 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:05.264 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:05.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:05.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:05.375 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:28:05.450 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:05.534 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:28:05.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:05.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:28:05.638 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:05.690 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:05.785 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:28:05.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1168222067' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:28:05.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:05 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:28:05.815 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1168222067' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:28:05.815 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:05 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:28:05.959 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:05.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:05.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:05.966 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1168222067' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:28:05.966 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:05 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:28:06.067 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:06.121 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:06.528 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:28:06.529 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:28:06.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:06 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:28:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:06 smithi190 ceph-mon[160435]: failed to encode map e309 with expected crc 2023-10-01T09:28:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:06 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:28:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:06 smithi190 ceph-mon[160435]: osdmap e309: 8 total, 8 up, 7 in 2023-10-01T09:28:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:06 smithi190 ceph-mon[160435]: failed to encode map e309 with expected crc 2023-10-01T09:28:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:06 smithi190 ceph-mon[160435]: pgmap v832: 89 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 2.0 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 18659/99063 objects degraded (18.835%); 4184/99063 objects misplaced (4.224%); 2.3 MiB/s, 108 objects/s recovering 2023-10-01T09:28:07.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:06 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:28:07.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:06 smithi138 ceph-mon[192513]: failed to encode map e309 with expected crc 2023-10-01T09:28:07.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:06 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:28:07.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:06 smithi138 ceph-mon[192513]: osdmap e309: 8 total, 8 up, 7 in 2023-10-01T09:28:07.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:06 smithi138 ceph-mon[192513]: failed to encode map e309 with expected crc 2023-10-01T09:28:07.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:06 smithi138 ceph-mon[192513]: pgmap v832: 89 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 2.0 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 18659/99063 objects degraded (18.835%); 4184/99063 objects misplaced (4.224%); 2.3 MiB/s, 108 objects/s recovering 2023-10-01T09:28:07.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:06 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:28:07.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:06 smithi138 ceph-mon[185886]: failed to encode map e309 with expected crc 2023-10-01T09:28:07.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:06 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:28:07.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:06 smithi138 ceph-mon[185886]: osdmap e309: 8 total, 8 up, 7 in 2023-10-01T09:28:07.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:06 smithi138 ceph-mon[185886]: failed to encode map e309 with expected crc 2023-10-01T09:28:07.075 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:06 smithi138 ceph-mon[185886]: pgmap v832: 89 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 6 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 2.0 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 18659/99063 objects degraded (18.835%); 4184/99063 objects misplaced (4.224%); 2.3 MiB/s, 108 objects/s recovering 2023-10-01T09:28:07.335 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:28:07.418 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:07.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:07.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:07.528 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:07.598 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:07.731 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:07.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:07.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:07.840 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:28:07.896 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:28:07.913 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:08.027 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:28:08.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:08.036 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:07 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 18659/99063 objects degraded (18.835%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:08.037 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:07 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 18659/99063 objects degraded (18.835%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:08.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:28:08.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:07 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 18659/99063 objects degraded (18.835%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:08.139 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:28:08.195 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:08.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:08.276 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:28:08.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:28:08.377 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:08.441 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:08.618 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:08.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:08.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:08.726 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:08.758 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:08.894 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:08.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:08.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:09.003 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:28:09.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:08 smithi190 ceph-mon[160435]: pgmap v833: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 20387/95652 objects degraded (21.314%); 6352/95652 objects misplaced (6.641%); 3.1 MiB/s, 1 keys/s, 107 objects/s recovering 2023-10-01T09:28:09.060 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:09.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:08 smithi138 ceph-mon[192513]: pgmap v833: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 20387/95652 objects degraded (21.314%); 6352/95652 objects misplaced (6.641%); 3.1 MiB/s, 1 keys/s, 107 objects/s recovering 2023-10-01T09:28:09.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:08 smithi138 ceph-mon[185886]: pgmap v833: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 490 op/s; 20387/95652 objects degraded (21.314%); 6352/95652 objects misplaced (6.641%); 3.1 MiB/s, 1 keys/s, 107 objects/s recovering 2023-10-01T09:28:09.102 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5, 6] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2023-10-01T09:28:09.102 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:28:09.103 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-10-01T09:28:09.103 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:28:09.103 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:28:09.103 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-10-01T09:28:09.254 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:28:09.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:09.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:28:09.365 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:09.405 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:09.543 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:09.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:09.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:09.654 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:09.680 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:09.830 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:28:09.837 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:28:09.837 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:28:09.913 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:09.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:09.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:10.032 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:28:10.096 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:10.225 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:09 smithi138 ceph-mon[185886]: pgmap v834: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 277 KiB/s rd, 273 KiB/s wr, 594 op/s; 20216/91314 objects degraded (22.139%); 6162/91314 objects misplaced (6.748%); 3.9 MiB/s, 1 keys/s, 114 objects/s recovering 2023-10-01T09:28:10.225 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:09 smithi138 ceph-mon[192513]: pgmap v834: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 277 KiB/s rd, 273 KiB/s wr, 594 op/s; 20216/91314 objects degraded (22.139%); 6162/91314 objects misplaced (6.748%); 3.9 MiB/s, 1 keys/s, 114 objects/s recovering 2023-10-01T09:28:10.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:09 smithi190 ceph-mon[160435]: pgmap v834: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.9 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 277 KiB/s rd, 273 KiB/s wr, 594 op/s; 20216/91314 objects degraded (22.139%); 6162/91314 objects misplaced (6.748%); 3.9 MiB/s, 1 keys/s, 114 objects/s recovering 2023-10-01T09:28:10.553 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:28:10.599 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:28:10.631 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:28:10.667 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:28:10.700 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:28:10.737 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:28:10.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:10.747 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:28:10.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:28:10.748 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:28:10.775 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:28:10.814 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:28:10.822 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:28:10.822 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:28:10.822 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:28:10.823 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:28:10.823 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:28:10.823 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:28:10.823 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:28:10.823 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:28:10.848 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:10.905 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:11.152 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:11.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:11.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:11.257 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:11.288 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:28:11.288 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:10 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3859979821' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:28:11.291 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:11.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:28:11.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:10 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3859979821' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:28:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:28:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:10 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3859979821' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:28:11.541 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:11.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:11.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:11.647 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:11.706 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:11.759 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:28:11.829 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:11.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:11.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:11.937 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:11.973 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:28:11.996 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:11 smithi138 ceph-mon[185886]: pgmap v835: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.8 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 245 KiB/s rd, 241 KiB/s wr, 492 op/s; 19874/87126 objects degraded (22.811%); 6004/87126 objects misplaced (6.891%); 3.8 MiB/s, 1 keys/s, 139 objects/s recovering 2023-10-01T09:28:12.021 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:12.179 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:12.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:12.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:12.242 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:28:12.242 INFO:teuthology.orchestra.run.smithi190.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T09:28:12.242 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:28:12.252 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:11 smithi190 ceph-mon[160435]: pgmap v835: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.8 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 245 KiB/s rd, 241 KiB/s wr, 492 op/s; 19874/87126 objects degraded (22.811%); 6004/87126 objects misplaced (6.891%); 3.8 MiB/s, 1 keys/s, 139 objects/s recovering 2023-10-01T09:28:12.287 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:28:12.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:11 smithi138 ceph-mon[192513]: pgmap v835: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.8 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 245 KiB/s rd, 241 KiB/s wr, 492 op/s; 19874/87126 objects degraded (22.811%); 6004/87126 objects misplaced (6.891%); 3.8 MiB/s, 1 keys/s, 139 objects/s recovering 2023-10-01T09:28:12.349 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:12.990 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:28:12.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:12.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:28:13.094 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:28:13.138 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:13 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 19874/87126 objects degraded (22.811%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:13.144 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:13.225 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:28:13.254 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:28:13.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:13.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:28:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:12 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 19874/87126 objects degraded (22.811%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:13.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:13 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 19874/87126 objects degraded (22.811%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:13.360 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:13.394 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:13.560 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:13.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:13.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:13.675 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:28:13.754 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:14.010 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:14 smithi138 ceph-mon[185886]: pgmap v836: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.7 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 209 KiB/s rd, 206 KiB/s wr, 573 op/s; 19689/82536 objects degraded (23.855%); 5836/82536 objects misplaced (7.071%); 4.4 MiB/s, 0 keys/s, 137 objects/s recovering 2023-10-01T09:28:14.040 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:28:14.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:14.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:28:14.150 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:28:14.228 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:14.266 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:28:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:28:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:28:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:28:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:167309 7.b e309/306 7:dbd907bd:::benchmark_data_smithi138_367058_object69514:head v 309'10215, mlcod=268'5453)", 2023-10-01T09:28:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:10.919073+0000", 2023-10-01T09:28:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 3.3319917009999998, 2023-10-01T09:28:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.042684385999999998, 2023-10-01T09:28:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.919073+0000", 2023-10-01T09:28:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.919073+0000", 2023-10-01T09:28:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:28:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.919072+0000", 2023-10-01T09:28:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1319e-05 2023-10-01T09:28:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.919083+0000", 2023-10-01T09:28:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6597e-05 2023-10-01T09:28:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.919110+0000", 2023-10-01T09:28:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3129999999999999e-06 2023-10-01T09:28:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.919118+0000", 2023-10-01T09:28:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0042848169999999998 2023-10-01T09:28:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923403+0000", 2023-10-01T09:28:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5381e-05 2023-10-01T09:28:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923428+0000", 2023-10-01T09:28:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.038277948999999999 2023-10-01T09:28:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.961706+0000", 2023-10-01T09:28:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1412000000000002e-05 2023-10-01T09:28:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.961758+0000", 2023-10-01T09:28:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.042684385999999998 2023-10-01T09:28:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:167313 7.3 e309/306 7:c42b35c1:::benchmark_data_smithi138_367058_object69518:head v 309'10427, mlcod=309'10423)", 2023-10-01T09:28:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:10.925535+0000", 2023-10-01T09:28:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 3.325529924, 2023-10-01T09:28:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.036253401999999997, 2023-10-01T09:28:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.925535+0000", 2023-10-01T09:28:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.925535+0000", 2023-10-01T09:28:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:28:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.925535+0000", 2023-10-01T09:28:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.993e-06 2023-10-01T09:28:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.925541+0000", 2023-10-01T09:28:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3480000000000003e-06 2023-10-01T09:28:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.925546+0000", 2023-10-01T09:28:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4709999999999999e-06 2023-10-01T09:28:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.925550+0000", 2023-10-01T09:28:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0048790400000000003 2023-10-01T09:28:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.930429+0000", 2023-10-01T09:28:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6702e-05 2023-10-01T09:28:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.930445+0000", 2023-10-01T09:28:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.031317013999999997 2023-10-01T09:28:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.961762+0000", 2023-10-01T09:28:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6210000000000001e-05 2023-10-01T09:28:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.961789+0000", 2023-10-01T09:28:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.036253401999999997 2023-10-01T09:28:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:167562 7.3 e309/306 7:c6012f67:::benchmark_data_smithi138_367058_object69767:head v 309'10439, mlcod=309'10437)", 2023-10-01T09:28:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:11.277243+0000", 2023-10-01T09:28:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 2.9738221940000003, 2023-10-01T09:28:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.055625041, 2023-10-01T09:28:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277243+0000", 2023-10-01T09:28:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277243+0000", 2023-10-01T09:28:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:28:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277242+0000", 2023-10-01T09:28:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0468e-05 2023-10-01T09:28:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277252+0000", 2023-10-01T09:28:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0624e-05 2023-10-01T09:28:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277263+0000", 2023-10-01T09:28:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1290000000000001e-06 2023-10-01T09:28:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277269+0000", 2023-10-01T09:28:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3198000000000002e-05 2023-10-01T09:28:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277312+0000", 2023-10-01T09:28:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2291000000000001e-05 2023-10-01T09:28:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277334+0000", 2023-10-01T09:28:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.055484780999999997 2023-10-01T09:28:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.332819+0000", 2023-10-01T09:28:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8730000000000003e-05 2023-10-01T09:28:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.332868+0000", 2023-10-01T09:28:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.055625041 2023-10-01T09:28:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:167561 7.b e309/306 7:d4a5ccbb:::benchmark_data_smithi138_367058_object69766:head v 309'10232, mlcod=268'5453)", 2023-10-01T09:28:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:11.277332+0000", 2023-10-01T09:28:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 2.9737328139999999, 2023-10-01T09:28:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.055564628999999997, 2023-10-01T09:28:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277332+0000", 2023-10-01T09:28:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277332+0000", 2023-10-01T09:28:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:28:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277332+0000", 2023-10-01T09:28:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8286000000000001e-05 2023-10-01T09:28:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277350+0000", 2023-10-01T09:28:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4839999999999993e-06 2023-10-01T09:28:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277359+0000", 2023-10-01T09:28:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2360000000000004e-06 2023-10-01T09:28:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277365+0000", 2023-10-01T09:28:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9563e-05 2023-10-01T09:28:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277414+0000", 2023-10-01T09:28:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5274e-05 2023-10-01T09:28:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.277430+0000", 2023-10-01T09:28:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.055442934999999999 2023-10-01T09:28:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.332872+0000", 2023-10-01T09:28:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4431999999999999e-05 2023-10-01T09:28:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.332897+0000", 2023-10-01T09:28:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.055564628999999997 2023-10-01T09:28:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "MOSDPGPush(7.7 309/306 [PushOp(7:e14e15f3:::benchmark_data_smithi138_367058_object94261:head, version: 270'5895, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:e14e15f3:::benchmark_data_smithi138_367058_object94261:head@270'5895, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-01T09:28:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:11.281883+0000", 2023-10-01T09:28:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 2.96918171, 2023-10-01T09:28:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.052906627999999997, 2023-10-01T09:28:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:28:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.281883+0000", 2023-10-01T09:28:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.281883+0000", 2023-10-01T09:28:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:28:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.281882+0000", 2023-10-01T09:28:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1987000000000002e-05 2023-10-01T09:28:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.281934+0000", 2023-10-01T09:28:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2316000000000001e-05 2023-10-01T09:28:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.281956+0000", 2023-10-01T09:28:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2614e-05 2023-10-01T09:28:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.281969+0000", 2023-10-01T09:28:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6042000000000001e-05 2023-10-01T09:28:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.282045+0000", 2023-10-01T09:28:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.8980000000000008e-06 2023-10-01T09:28:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.282055+0000", 2023-10-01T09:28:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.052735255000000002 2023-10-01T09:28:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.334790+0000", 2023-10-01T09:28:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.052906627999999997 2023-10-01T09:28:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:167566 7.b e309/306 7:d57a4c6f:::benchmark_data_smithi138_367058_object69771:head v 309'10233, mlcod=268'5453)", 2023-10-01T09:28:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:11.283768+0000", 2023-10-01T09:28:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 2.967296685, 2023-10-01T09:28:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.058675684999999998, 2023-10-01T09:28:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.283768+0000", 2023-10-01T09:28:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.283768+0000", 2023-10-01T09:28:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:28:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.283767+0000", 2023-10-01T09:28:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9820000000000003e-06 2023-10-01T09:28:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.283777+0000", 2023-10-01T09:28:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.050822491999999997 2023-10-01T09:28:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.334600+0000", 2023-10-01T09:28:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2719999999999997e-06 2023-10-01T09:28:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.334608+0000", 2023-10-01T09:28:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00029840900000000002 2023-10-01T09:28:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.334907+0000", 2023-10-01T09:28:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3728999999999999e-05 2023-10-01T09:28:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.334930+0000", 2023-10-01T09:28:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0074715939999999998 2023-10-01T09:28:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.342402+0000", 2023-10-01T09:28:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2126000000000003e-05 2023-10-01T09:28:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.342444+0000", 2023-10-01T09:28:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.058675684999999998 2023-10-01T09:28:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168393 7.b e309/306 7:de1e549c:::benchmark_data_smithi138_367058_object70598:head v 309'10279, mlcod=268'5453)", 2023-10-01T09:28:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:12.658391+0000", 2023-10-01T09:28:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.5926741149999999, 2023-10-01T09:28:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.101860468, 2023-10-01T09:28:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658391+0000", 2023-10-01T09:28:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658391+0000", 2023-10-01T09:28:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:28:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658390+0000", 2023-10-01T09:28:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2609999999999999e-06 2023-10-01T09:28:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658399+0000", 2023-10-01T09:28:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0547000000000001e-05 2023-10-01T09:28:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658410+0000", 2023-10-01T09:28:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0090000000000004e-06 2023-10-01T09:28:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658416+0000", 2023-10-01T09:28:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0454999999999999e-05 2023-10-01T09:28:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658446+0000", 2023-10-01T09:28:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4567e-05 2023-10-01T09:28:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658461+0000", 2023-10-01T09:28:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.101753844 2023-10-01T09:28:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760215+0000", 2023-10-01T09:28:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6927000000000001e-05 2023-10-01T09:28:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760251+0000", 2023-10-01T09:28:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.101860468 2023-10-01T09:28:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "MOSDPGPush(7.7 309/306 [PushOp(7:e9fc44f3:::benchmark_data_smithi138_367058_object95165:head, version: 271'5956, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:e9fc44f3:::benchmark_data_smithi138_367058_object95165:head@271'5956, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-01T09:28:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:12.663731+0000", 2023-10-01T09:28:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.587333603, 2023-10-01T09:28:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.099798983999999993, 2023-10-01T09:28:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:28:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.663731+0000", 2023-10-01T09:28:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.663731+0000", 2023-10-01T09:28:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:28:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.663731+0000", 2023-10-01T09:28:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4078e-05 2023-10-01T09:28:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.663785+0000", 2023-10-01T09:28:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7159e-05 2023-10-01T09:28:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.663802+0000", 2023-10-01T09:28:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5820000000000004e-06 2023-10-01T09:28:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.663809+0000", 2023-10-01T09:28:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.096452888000000001 2023-10-01T09:28:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760262+0000", 2023-10-01T09:28:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6060000000000003e-06 2023-10-01T09:28:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760271+0000", 2023-10-01T09:28:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0032596389999999999 2023-10-01T09:28:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.763530+0000", 2023-10-01T09:28:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.099798983999999993 2023-10-01T09:28:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168396 7.7 e309/306 7:eb13eb3e:::benchmark_data_smithi138_367058_object70601:head v 309'10561, mlcod=271'5955)", 2023-10-01T09:28:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:12.665871+0000", 2023-10-01T09:28:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.5851937710000001, 2023-10-01T09:28:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.128762455, 2023-10-01T09:28:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.665871+0000", 2023-10-01T09:28:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.665871+0000", 2023-10-01T09:28:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:28:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.665871+0000", 2023-10-01T09:28:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7300000000000005e-06 2023-10-01T09:28:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.665878+0000", 2023-10-01T09:28:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.094338721 2023-10-01T09:28:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760217+0000", 2023-10-01T09:28:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5829999999999996e-06 2023-10-01T09:28:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760227+0000", 2023-10-01T09:28:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033480960000000001 2023-10-01T09:28:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.763575+0000", 2023-10-01T09:28:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0735000000000001e-05 2023-10-01T09:28:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.763595+0000", 2023-10-01T09:28:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.031017712999999999 2023-10-01T09:28:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.794613+0000", 2023-10-01T09:28:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0622999999999999e-05 2023-10-01T09:28:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.794634+0000", 2023-10-01T09:28:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.128762455 2023-10-01T09:28:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168395 7.3 e309/306 7:c3660f7d:::benchmark_data_smithi138_367058_object70600:head v 309'10502, mlcod=309'10500)", 2023-10-01T09:28:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:12.669034+0000", 2023-10-01T09:28:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.5820315730000001, 2023-10-01T09:28:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.097437389999999999, 2023-10-01T09:28:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.669034+0000", 2023-10-01T09:28:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.669034+0000", 2023-10-01T09:28:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:28:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.669033+0000", 2023-10-01T09:28:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.8289999999999997e-06 2023-10-01T09:28:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.669041+0000", 2023-10-01T09:28:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.091159113999999999 2023-10-01T09:28:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760201+0000", 2023-10-01T09:28:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6405e-05 2023-10-01T09:28:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760217+0000", 2023-10-01T09:28:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1770999999999997e-05 2023-10-01T09:28:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760259+0000", 2023-10-01T09:28:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2144e-05 2023-10-01T09:28:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760281+0000", 2023-10-01T09:28:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0061497269999999998 2023-10-01T09:28:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.766431+0000", 2023-10-01T09:28:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0278000000000003e-05 2023-10-01T09:28:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.766471+0000", 2023-10-01T09:28:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.097437389999999999 2023-10-01T09:28:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168402 7.b e309/306 7:dd739fb6:::benchmark_data_smithi138_367058_object70607:head v 309'10280, mlcod=268'5453)", 2023-10-01T09:28:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:12.672435+0000", 2023-10-01T09:28:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.5786301790000001, 2023-10-01T09:28:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.095632628999999997, 2023-10-01T09:28:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.672435+0000", 2023-10-01T09:28:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.672435+0000", 2023-10-01T09:28:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:28:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.672434+0000", 2023-10-01T09:28:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9410000000000007e-06 2023-10-01T09:28:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.672442+0000", 2023-10-01T09:28:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.087759411999999995 2023-10-01T09:28:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760201+0000", 2023-10-01T09:28:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5786999999999999e-05 2023-10-01T09:28:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760227+0000", 2023-10-01T09:28:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7932000000000001e-05 2023-10-01T09:28:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760275+0000", 2023-10-01T09:28:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5676e-05 2023-10-01T09:28:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760291+0000", 2023-10-01T09:28:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.007732793 2023-10-01T09:28:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.768024+0000", 2023-10-01T09:28:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3903999999999998e-05 2023-10-01T09:28:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.768068+0000", 2023-10-01T09:28:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.095632628999999997 2023-10-01T09:28:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168404 7.7 e309/306 7:e127e794:::benchmark_data_smithi138_367058_object70609:head v 309'10562, mlcod=271'5955)", 2023-10-01T09:28:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:12.760275+0000", 2023-10-01T09:28:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.490789637, 2023-10-01T09:28:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0343708, 2023-10-01T09:28:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760275+0000", 2023-10-01T09:28:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760275+0000", 2023-10-01T09:28:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:28:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760275+0000", 2023-10-01T09:28:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2224e-05 2023-10-01T09:28:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760287+0000", 2023-10-01T09:28:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4460000000000001e-06 2023-10-01T09:28:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760295+0000", 2023-10-01T09:28:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4150000000000001e-06 2023-10-01T09:28:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760300+0000", 2023-10-01T09:28:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010893959999999999 2023-10-01T09:28:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.771194+0000", 2023-10-01T09:28:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2087e-05 2023-10-01T09:28:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.771216+0000", 2023-10-01T09:28:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.023420545000000001 2023-10-01T09:28:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.794636+0000", 2023-10-01T09:28:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7750000000000004e-06 2023-10-01T09:28:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.794646+0000", 2023-10-01T09:28:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0343708 2023-10-01T09:28:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168405 7.7 e309/306 7:ed4ac155:::benchmark_data_smithi138_367058_object70610:head v 309'10563, mlcod=271'5955)", 2023-10-01T09:28:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:12.760305+0000", 2023-10-01T09:28:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.4907598630000001, 2023-10-01T09:28:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.034355943999999999, 2023-10-01T09:28:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760305+0000", 2023-10-01T09:28:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760305+0000", 2023-10-01T09:28:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:28:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760305+0000", 2023-10-01T09:28:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5000000000000001e-06 2023-10-01T09:28:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760309+0000", 2023-10-01T09:28:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7570000000000004e-06 2023-10-01T09:28:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760314+0000", 2023-10-01T09:28:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1700000000000001e-06 2023-10-01T09:28:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760317+0000", 2023-10-01T09:28:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020318850999999999 2023-10-01T09:28:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.780636+0000", 2023-10-01T09:28:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8665000000000001e-05 2023-10-01T09:28:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.780665+0000", 2023-10-01T09:28:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013984619 2023-10-01T09:28:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.794650+0000", 2023-10-01T09:28:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1644000000000001e-05 2023-10-01T09:28:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.794661+0000", 2023-10-01T09:28:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.034355943999999999 2023-10-01T09:28:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168530 7.3 e309/306 7:c522aeb0:::benchmark_data_smithi138_367058_object70735:head v 309'10513, mlcod=309'10509)", 2023-10-01T09:28:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:12.928594+0000", 2023-10-01T09:28:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.3224712009999999, 2023-10-01T09:28:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.036846797000000001, 2023-10-01T09:28:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.928594+0000", 2023-10-01T09:28:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.928594+0000", 2023-10-01T09:28:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:28:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.928593+0000", 2023-10-01T09:28:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6400000000000003e-06 2023-10-01T09:28:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.928602+0000", 2023-10-01T09:28:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0179e-05 2023-10-01T09:28:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.928612+0000", 2023-10-01T09:28:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8309999999999997e-06 2023-10-01T09:28:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.928618+0000", 2023-10-01T09:28:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011304628000000001 2023-10-01T09:28:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.939922+0000", 2023-10-01T09:28:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9488999999999999e-05 2023-10-01T09:28:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.939942+0000", 2023-10-01T09:28:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.025451254 2023-10-01T09:28:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.965393+0000", 2023-10-01T09:28:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7475999999999997e-05 2023-10-01T09:28:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.965441+0000", 2023-10-01T09:28:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.036846797000000001 2023-10-01T09:28:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168533 7.b e309/306 7:d8a14568:::benchmark_data_smithi138_367058_object70738:head v 309'10290, mlcod=268'5453)", 2023-10-01T09:28:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:12.931727+0000", 2023-10-01T09:28:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.3193385229999999, 2023-10-01T09:28:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.040747272000000001, 2023-10-01T09:28:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.931727+0000", 2023-10-01T09:28:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.931727+0000", 2023-10-01T09:28:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:28:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.931726+0000", 2023-10-01T09:28:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4975999999999999e-05 2023-10-01T09:28:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.931751+0000", 2023-10-01T09:28:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.118e-05 2023-10-01T09:28:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.931762+0000", 2023-10-01T09:28:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6819999999999997e-06 2023-10-01T09:28:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.931769+0000", 2023-10-01T09:28:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0081091050000000001 2023-10-01T09:28:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.939878+0000", 2023-10-01T09:28:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8415000000000002e-05 2023-10-01T09:28:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.939906+0000", 2023-10-01T09:28:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.032525414000000002 2023-10-01T09:28:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.972431+0000", 2023-10-01T09:28:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2351999999999997e-05 2023-10-01T09:28:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.972474+0000", 2023-10-01T09:28:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.040747272000000001 2023-10-01T09:28:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168536 7.b e309/306 7:d22c43d5:::benchmark_data_smithi138_367058_object70741:head v 309'10291, mlcod=268'5453)", 2023-10-01T09:28:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:12.936457+0000", 2023-10-01T09:28:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.3146080840000001, 2023-10-01T09:28:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.036038522000000003, 2023-10-01T09:28:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.936457+0000", 2023-10-01T09:28:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.936457+0000", 2023-10-01T09:28:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:28:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.936456+0000", 2023-10-01T09:28:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0121e-05 2023-10-01T09:28:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.936466+0000", 2023-10-01T09:28:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0946e-05 2023-10-01T09:28:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.936477+0000", 2023-10-01T09:28:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4160000000000003e-06 2023-10-01T09:28:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.936483+0000", 2023-10-01T09:28:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0097255439999999992 2023-10-01T09:28:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.946208+0000", 2023-10-01T09:28:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2742999999999999e-05 2023-10-01T09:28:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.946231+0000", 2023-10-01T09:28:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.026246782 2023-10-01T09:28:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.972478+0000", 2023-10-01T09:28:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7762999999999999e-05 2023-10-01T09:28:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.972496+0000", 2023-10-01T09:28:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.036038522000000003 2023-10-01T09:28:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168818 7.7 e309/306 7:e9a8d8e1:::benchmark_data_smithi138_367058_object71023:head v 309'10589, mlcod=271'5977)", 2023-10-01T09:28:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:13.359054+0000", 2023-10-01T09:28:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.89201092100000001, 2023-10-01T09:28:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.073086986000000007, 2023-10-01T09:28:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.359054+0000", 2023-10-01T09:28:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.359054+0000", 2023-10-01T09:28:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:28:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.359053+0000", 2023-10-01T09:28:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5759999999999996e-06 2023-10-01T09:28:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.359059+0000", 2023-10-01T09:28:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.064e-05 2023-10-01T09:28:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.359070+0000", 2023-10-01T09:28:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4649999999999996e-06 2023-10-01T09:28:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.359074+0000", 2023-10-01T09:28:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2088999999999997e-05 2023-10-01T09:28:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.359106+0000", 2023-10-01T09:28:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4841000000000001e-05 2023-10-01T09:28:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.359121+0000", 2023-10-01T09:28:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.072979780999999994 2023-10-01T09:28:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.432101+0000", 2023-10-01T09:28:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0389000000000002e-05 2023-10-01T09:28:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.432141+0000", 2023-10-01T09:28:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.073086986000000007 2023-10-01T09:28:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168815 7.b e309/306 7:d60f5039:::benchmark_data_smithi138_367058_object71020:head v 309'10316, mlcod=268'5453)", 2023-10-01T09:28:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:13.363501+0000", 2023-10-01T09:28:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.88756391499999998, 2023-10-01T09:28:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.051632674000000003, 2023-10-01T09:28:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.363501+0000", 2023-10-01T09:28:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.363501+0000", 2023-10-01T09:28:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:28:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.363500+0000", 2023-10-01T09:28:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0329999999999997e-06 2023-10-01T09:28:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.363509+0000", 2023-10-01T09:28:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.044502829000000001 2023-10-01T09:28:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.408012+0000", 2023-10-01T09:28:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0413e-05 2023-10-01T09:28:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.408023+0000", 2023-10-01T09:28:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7621999999999996e-05 2023-10-01T09:28:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.408100+0000", 2023-10-01T09:28:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5369000000000001e-05 2023-10-01T09:28:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.408126+0000", 2023-10-01T09:28:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0069645180000000003 2023-10-01T09:28:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.415090+0000", 2023-10-01T09:28:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3798000000000003e-05 2023-10-01T09:28:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.415134+0000", 2023-10-01T09:28:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.051632674000000003 2023-10-01T09:28:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168821 7.4 e309/306 7:296454da:::benchmark_data_smithi138_367058_object71026:head v 309'10631, mlcod=309'10629)", 2023-10-01T09:28:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:13.367887+0000", 2023-10-01T09:28:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.88317838299999996, 2023-10-01T09:28:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.044119742000000003, 2023-10-01T09:28:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.367887+0000", 2023-10-01T09:28:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.367887+0000", 2023-10-01T09:28:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:28:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.367886+0000", 2023-10-01T09:28:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0809999999999996e-06 2023-10-01T09:28:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.367893+0000", 2023-10-01T09:28:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.040124948000000001 2023-10-01T09:28:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.408018+0000", 2023-10-01T09:28:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9569999999999999e-06 2023-10-01T09:28:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.408024+0000", 2023-10-01T09:28:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000799564 2023-10-01T09:28:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.408824+0000", 2023-10-01T09:28:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9089e-05 2023-10-01T09:28:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.408843+0000", 2023-10-01T09:28:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0031220660000000002 2023-10-01T09:28:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.411965+0000", 2023-10-01T09:28:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1703999999999999e-05 2023-10-01T09:28:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.412006+0000", 2023-10-01T09:28:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.044119742000000003 2023-10-01T09:28:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168820 7.c e309/307 7:3888ac35:::benchmark_data_smithi138_367058_object71025:head v 309'10461, mlcod=309'10458)", 2023-10-01T09:28:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:13.370572+0000", 2023-10-01T09:28:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.88049330199999998, 2023-10-01T09:28:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.040087113000000001, 2023-10-01T09:28:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:28:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:28:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:28:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.370572+0000", 2023-10-01T09:28:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:28:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:28:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.370572+0000", 2023-10-01T09:28:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:28:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:28:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.370571+0000", 2023-10-01T09:28:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2133000000000001e-05 2023-10-01T09:28:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:28:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.370583+0000", 2023-10-01T09:28:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.037413941999999999 2023-10-01T09:28:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:28:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.407997+0000", 2023-10-01T09:28:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2245e-05 2023-10-01T09:28:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:28:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.408009+0000", 2023-10-01T09:28:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0084000000000002e-05 2023-10-01T09:28:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:28:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.408079+0000", 2023-10-01T09:28:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.44e-05 2023-10-01T09:28:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:28:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.408104+0000", 2023-10-01T09:28:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0025097980000000002 2023-10-01T09:28:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:28:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.410613+0000", 2023-10-01T09:28:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5537999999999997e-05 2023-10-01T09:28:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:28:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:28:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:28:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:13.410659+0000", 2023-10-01T09:28:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.040087113000000001 2023-10-01T09:28:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:28:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:28:14.375 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:28:14.376 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:14 smithi138 ceph-mon[192513]: pgmap v836: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.7 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 209 KiB/s rd, 206 KiB/s wr, 573 op/s; 19689/82536 objects degraded (23.855%); 5836/82536 objects misplaced (7.071%); 4.4 MiB/s, 0 keys/s, 137 objects/s recovering 2023-10-01T09:28:14.376 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:14 smithi190 ceph-mon[160435]: pgmap v836: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.7 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 209 KiB/s rd, 206 KiB/s wr, 573 op/s; 19689/82536 objects degraded (23.855%); 5836/82536 objects misplaced (7.071%); 4.4 MiB/s, 0 keys/s, 137 objects/s recovering 2023-10-01T09:28:14.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:14.377 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:28:14.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:28:14.478 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:14.562 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:14.806 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:14.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:14.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:14.922 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:14.967 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:15.006 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:28:15.056 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:15.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:28:15.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:15.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:15.134 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:28:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:28:14] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.33.4" 2023-10-01T09:28:15.162 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:15.201 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:15.257 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:15.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:15.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:15.367 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:28:15.413 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:15.472 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:28:15.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:15.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:28:15.578 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:28:15.648 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:15.948 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:15 smithi138 ceph-mon[185886]: pgmap v837: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.7 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 207 KiB/s rd, 204 KiB/s wr, 574 op/s; 19689/81960 objects degraded (24.023%); 5836/81960 objects misplaced (7.121%); 4.4 MiB/s, 0 keys/s, 151 objects/s recovering 2023-10-01T09:28:15.948 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:15 smithi138 ceph-mon[192513]: pgmap v837: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.7 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 207 KiB/s rd, 204 KiB/s wr, 574 op/s; 19689/81960 objects degraded (24.023%); 5836/81960 objects misplaced (7.121%); 4.4 MiB/s, 0 keys/s, 151 objects/s recovering 2023-10-01T09:28:16.089 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:28:16.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:16.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:28:16.197 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:28:16.223 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:15 smithi190 ceph-mon[160435]: pgmap v837: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.7 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 207 KiB/s rd, 204 KiB/s wr, 574 op/s; 19689/81960 objects degraded (24.023%); 5836/81960 objects misplaced (7.121%); 4.4 MiB/s, 0 keys/s, 151 objects/s recovering 2023-10-01T09:28:16.227 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:16.296 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:28:16.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:16.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:28:16.351 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:28:16.378 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:28:16.405 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:16.442 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:16.502 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:16.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:16.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:16.607 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:16.644 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:16.728 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:16.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:16.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:16.839 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:28:16.939 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:17.123 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:28:17.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:17.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:28:17.234 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:28:17.264 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:17.321 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:28:17.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:17.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:28:17.426 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:28:17.479 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:17.537 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:28:17.538 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:17 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 19689/81960 objects degraded (24.023%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:17.539 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:17 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 19689/81960 objects degraded (24.023%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:17.544 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:17 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 19689/81960 objects degraded (24.023%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:17.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:17.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:28:17.646 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:17.729 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:18.384 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:28:18.531 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5, 6] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2023-10-01T09:28:18.532 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:28:18.532 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-01T09:28:18.532 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:28:18.532 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:28:18.532 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-01T09:28:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:18 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1364030052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:28:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:18 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:28:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:18 smithi190 ceph-mon[160435]: pgmap v838: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.6 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 174 KiB/s rd, 172 KiB/s wr, 599 op/s; 19298/77805 objects degraded (24.803%); 5632/77805 objects misplaced (7.239%); 4.8 MiB/s, 0 keys/s, 144 objects/s recovering 2023-10-01T09:28:18.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:18 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1364030052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:28:18.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:18 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:28:18.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:18 smithi138 ceph-mon[192513]: pgmap v838: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.6 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 174 KiB/s rd, 172 KiB/s wr, 599 op/s; 19298/77805 objects degraded (24.803%); 5632/77805 objects misplaced (7.239%); 4.8 MiB/s, 0 keys/s, 144 objects/s recovering 2023-10-01T09:28:18.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:18 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1364030052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:28:18.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:18 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:28:18.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:18 smithi138 ceph-mon[185886]: pgmap v838: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.6 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 174 KiB/s rd, 172 KiB/s wr, 599 op/s; 19298/77805 objects degraded (24.803%); 5632/77805 objects misplaced (7.239%); 4.8 MiB/s, 0 keys/s, 144 objects/s recovering 2023-10-01T09:28:18.940 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:18.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:18.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:19.050 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:19.089 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:19.169 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:19.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:19.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:19.280 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:28:19.354 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:19.379 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:19 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:28:19.379 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:19 smithi190 ceph-mon[160435]: failed to encode map e310 with expected crc 2023-10-01T09:28:19.379 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:19 smithi190 ceph-mon[160435]: failed to encode map e310 with expected crc 2023-10-01T09:28:19.379 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:19 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:28:19.379 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:19 smithi190 ceph-mon[160435]: osdmap e310: 8 total, 8 up, 7 in 2023-10-01T09:28:19.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:19 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:28:19.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:19 smithi138 ceph-mon[185886]: failed to encode map e310 with expected crc 2023-10-01T09:28:19.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:19 smithi138 ceph-mon[185886]: failed to encode map e310 with expected crc 2023-10-01T09:28:19.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:19 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:28:19.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:19 smithi138 ceph-mon[185886]: osdmap e310: 8 total, 8 up, 7 in 2023-10-01T09:28:19.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:19 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:28:19.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:19 smithi138 ceph-mon[192513]: failed to encode map e310 with expected crc 2023-10-01T09:28:19.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:19 smithi138 ceph-mon[192513]: failed to encode map e310 with expected crc 2023-10-01T09:28:19.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:19 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:28:19.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:19 smithi138 ceph-mon[192513]: osdmap e310: 8 total, 8 up, 7 in 2023-10-01T09:28:20.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:20 smithi138 ceph-mon[185886]: pgmap v840: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.5 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 19132/73146 objects degraded (26.156%); 5426/73146 objects misplaced (7.418%); 4.7 MiB/s, 156 objects/s recovering 2023-10-01T09:28:20.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:20 smithi138 ceph-mon[192513]: pgmap v840: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.5 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 19132/73146 objects degraded (26.156%); 5426/73146 objects misplaced (7.418%); 4.7 MiB/s, 156 objects/s recovering 2023-10-01T09:28:20.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:20 smithi190 ceph-mon[160435]: pgmap v840: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.5 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 19132/73146 objects degraded (26.156%); 5426/73146 objects misplaced (7.418%); 4.7 MiB/s, 156 objects/s recovering 2023-10-01T09:28:21.591 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:28:21.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:21.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:28:21.698 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:28:21.772 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:22.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:21 smithi190 ceph-mon[160435]: pgmap v841: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.4 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 18754/68613 objects degraded (27.333%); 5263/68613 objects misplaced (7.671%); 4.8 MiB/s, 166 objects/s recovering 2023-10-01T09:28:22.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:21 smithi138 ceph-mon[185886]: pgmap v841: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.4 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 18754/68613 objects degraded (27.333%); 5263/68613 objects misplaced (7.671%); 4.8 MiB/s, 166 objects/s recovering 2023-10-01T09:28:22.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:21 smithi138 ceph-mon[192513]: pgmap v841: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.4 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 18754/68613 objects degraded (27.333%); 5263/68613 objects misplaced (7.671%); 4.8 MiB/s, 166 objects/s recovering 2023-10-01T09:28:22.635 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:28:22.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:22.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:28:22.755 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:22.790 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:22.845 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:22.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:22.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:22.953 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:22 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 18754/68613 objects degraded (27.333%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:22.954 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:28:23.034 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:23.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:22 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 18754/68613 objects degraded (27.333%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:23.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:22 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 18754/68613 objects degraded (27.333%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:23.223 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:28:23.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:23.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:28:23.328 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:28:23.367 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:23.404 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:28:23.414 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:28:23.420 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:28:23.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:23.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:28:23.456 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:28:23.495 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:28:23.528 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:23.546 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:28:23.560 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:23.564 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:28:23.564 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:28:23.572 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:28:23.609 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:28:23.621 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:23.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:23.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:23.659 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:28:23.694 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:28:23.711 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:28:23.712 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:28:23.713 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:28:23.713 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:28:23.713 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:28:23.713 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:28:23.713 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:28:23.713 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:28:23.731 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:28:23.808 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:23.976 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:23 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2219089015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:28:23.976 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:23 smithi190 ceph-mon[160435]: pgmap v842: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.3 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 18754/63852 objects degraded (29.371%); 5041/63852 objects misplaced (7.895%); 3.6 MiB/s, 171 objects/s recovering 2023-10-01T09:28:24.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:23 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2219089015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:28:24.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:23 smithi138 ceph-mon[185886]: pgmap v842: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.3 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 18754/63852 objects degraded (29.371%); 5041/63852 objects misplaced (7.895%); 3.6 MiB/s, 171 objects/s recovering 2023-10-01T09:28:24.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:23 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2219089015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:28:24.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:23 smithi138 ceph-mon[192513]: pgmap v842: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.3 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 18754/63852 objects degraded (29.371%); 5041/63852 objects misplaced (7.895%); 3.6 MiB/s, 171 objects/s recovering 2023-10-01T09:28:24.688 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:28:24.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:24.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:28:24.799 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:24.826 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:28:24.855 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:25.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:28:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:28:24] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.33.4" 2023-10-01T09:28:25.750 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:28:25.750 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:28:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:28:26.708 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:26.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:26.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:26.716 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:26 smithi138 ceph-mon[192513]: pgmap v843: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.3 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 18754/63168 objects degraded (29.689%); 5041/63168 objects misplaced (7.980%); 3.6 MiB/s, 183 objects/s recovering 2023-10-01T09:28:26.716 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:26 smithi138 ceph-mon[185886]: pgmap v843: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.3 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 18754/63168 objects degraded (29.689%); 5041/63168 objects misplaced (7.980%); 3.6 MiB/s, 183 objects/s recovering 2023-10-01T09:28:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:26 smithi190 ceph-mon[160435]: pgmap v843: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.3 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 18754/63168 objects degraded (29.689%); 5041/63168 objects misplaced (7.980%); 3.6 MiB/s, 183 objects/s recovering 2023-10-01T09:28:26.817 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:28:26.911 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:27.099 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:28:27.273 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:28:27.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:27.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:28:27.389 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:27.425 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:27.490 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:27.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:27.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:27.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:27 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 18754/63168 objects degraded (29.689%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:27.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:27 smithi138 ceph-mon[185886]: osdmap e311: 8 total, 8 up, 7 in 2023-10-01T09:28:27.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:27 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 18754/63168 objects degraded (29.689%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:27.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:27 smithi138 ceph-mon[192513]: osdmap e311: 8 total, 8 up, 7 in 2023-10-01T09:28:27.595 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:28:27.653 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:27.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:27 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 18754/63168 objects degraded (29.689%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:27.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:27 smithi190 ceph-mon[160435]: osdmap e311: 8 total, 8 up, 7 in 2023-10-01T09:28:28.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:28 smithi190 ceph-mon[160435]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:28 smithi190 ceph-mon[160435]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:28 smithi190 ceph-mon[160435]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:28 smithi190 ceph-mon[160435]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:28 smithi190 ceph-mon[160435]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:28 smithi190 ceph-mon[160435]: pgmap v845: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.2 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 18382/59226 objects degraded (31.037%); 4854/59226 objects misplaced (8.196%); 2.6 MiB/s, 170 objects/s recovering 2023-10-01T09:28:28.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:28 smithi190 ceph-mon[160435]: osdmap e312: 8 total, 8 up, 7 in 2023-10-01T09:28:28.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:28 smithi138 ceph-mon[185886]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:28 smithi138 ceph-mon[185886]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:28 smithi138 ceph-mon[185886]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:28 smithi138 ceph-mon[185886]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:28 smithi138 ceph-mon[185886]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:28 smithi138 ceph-mon[185886]: pgmap v845: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.2 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 18382/59226 objects degraded (31.037%); 4854/59226 objects misplaced (8.196%); 2.6 MiB/s, 170 objects/s recovering 2023-10-01T09:28:28.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:28 smithi138 ceph-mon[185886]: osdmap e312: 8 total, 8 up, 7 in 2023-10-01T09:28:28.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:28 smithi138 ceph-mon[192513]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:28 smithi138 ceph-mon[192513]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:28 smithi138 ceph-mon[192513]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:28 smithi138 ceph-mon[192513]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:28 smithi138 ceph-mon[192513]: failed to encode map e311 with expected crc 2023-10-01T09:28:28.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:28 smithi138 ceph-mon[192513]: pgmap v845: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.2 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 18382/59226 objects degraded (31.037%); 4854/59226 objects misplaced (8.196%); 2.6 MiB/s, 170 objects/s recovering 2023-10-01T09:28:28.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:28 smithi138 ceph-mon[192513]: osdmap e312: 8 total, 8 up, 7 in 2023-10-01T09:28:29.090 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:28:29.484 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:28:29.493 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:29 smithi138 ceph-mon[192513]: failed to encode map e312 with expected crc 2023-10-01T09:28:29.493 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:29 smithi138 ceph-mon[192513]: failed to encode map e312 with expected crc 2023-10-01T09:28:29.493 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:29 smithi138 ceph-mon[192513]: failed to encode map e312 with expected crc 2023-10-01T09:28:29.493 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:29 smithi138 ceph-mon[192513]: failed to encode map e312 with expected crc 2023-10-01T09:28:29.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:29.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:28:29.495 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:29 smithi138 ceph-mon[185886]: failed to encode map e312 with expected crc 2023-10-01T09:28:29.495 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:29 smithi138 ceph-mon[185886]: failed to encode map e312 with expected crc 2023-10-01T09:28:29.495 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:29 smithi138 ceph-mon[185886]: failed to encode map e312 with expected crc 2023-10-01T09:28:29.495 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:29 smithi138 ceph-mon[185886]: failed to encode map e312 with expected crc 2023-10-01T09:28:29.543 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:28:29.595 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:29.700 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:29 smithi190 ceph-mon[160435]: failed to encode map e312 with expected crc 2023-10-01T09:28:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:29 smithi190 ceph-mon[160435]: failed to encode map e312 with expected crc 2023-10-01T09:28:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:29 smithi190 ceph-mon[160435]: failed to encode map e312 with expected crc 2023-10-01T09:28:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:29 smithi190 ceph-mon[160435]: failed to encode map e312 with expected crc 2023-10-01T09:28:30.016 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:30.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:30.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:30.131 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:30.174 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:30.227 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:28:30.239 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:30.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:30.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:30.346 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:30.409 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:30 smithi138 ceph-mon[192513]: pgmap v847: 89 pgs: 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.1 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 16190/54927 objects degraded (29.475%); 4675/54927 objects misplaced (8.511%); 1.4 MiB/s, 164 objects/s recovering 2023-10-01T09:28:30.409 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:30 smithi138 ceph-mon[185886]: pgmap v847: 89 pgs: 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.1 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 16190/54927 objects degraded (29.475%); 4675/54927 objects misplaced (8.511%); 1.4 MiB/s, 164 objects/s recovering 2023-10-01T09:28:30.425 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:30.521 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:28:30.522 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "" 2023-10-01T09:28:30.522 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:28:30.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:30 smithi190 ceph-mon[160435]: pgmap v847: 89 pgs: 3 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 1.1 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 16190/54927 objects degraded (29.475%); 4675/54927 objects misplaced (8.511%); 1.4 MiB/s, 164 objects/s recovering 2023-10-01T09:28:31.826 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:28:31.872 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:31.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:31.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:31.980 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:32.023 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:32.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:31 smithi190 ceph-mon[160435]: pgmap v848: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 1.0 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 15868/50946 objects degraded (31.147%); 4511/50946 objects misplaced (8.854%); 2.6 MiB/s, 177 objects/s recovering 2023-10-01T09:28:32.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:31 smithi138 ceph-mon[185886]: pgmap v848: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 1.0 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 15868/50946 objects degraded (31.147%); 4511/50946 objects misplaced (8.854%); 2.6 MiB/s, 177 objects/s recovering 2023-10-01T09:28:32.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:31 smithi138 ceph-mon[192513]: pgmap v848: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 1.0 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 15868/50946 objects degraded (31.147%); 4511/50946 objects misplaced (8.854%); 2.6 MiB/s, 177 objects/s recovering 2023-10-01T09:28:32.961 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:32.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:32.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:33.070 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:28:33.099 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:33.213 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:32 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 15868/50946 objects degraded (31.147%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:33.213 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:32 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 15868/50946 objects degraded (31.147%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:33.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:32 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 15868/50946 objects degraded (31.147%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:33.873 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:28:33.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:33.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:28:33.989 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:34.030 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:34.110 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:34.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:34.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:34.217 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:34.288 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:34 smithi138 ceph-mon[185886]: pgmap v849: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 976 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 15868/46737 objects degraded (33.952%); 4351/46737 objects misplaced (9.310%); 2.6 MiB/s, 206 objects/s recovering 2023-10-01T09:28:34.289 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:34 smithi138 ceph-mon[192513]: pgmap v849: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 976 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 15868/46737 objects degraded (33.952%); 4351/46737 objects misplaced (9.310%); 2.6 MiB/s, 206 objects/s recovering 2023-10-01T09:28:34.302 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:34.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:34 smithi190 ceph-mon[160435]: pgmap v849: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 976 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 15868/46737 objects degraded (33.952%); 4351/46737 objects misplaced (9.310%); 2.6 MiB/s, 206 objects/s recovering 2023-10-01T09:28:35.322 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:28:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:28:34] "GET /metrics HTTP/1.1" 200 36427 "" "Prometheus/2.33.4" 2023-10-01T09:28:35.344 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:35.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:35.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:35.457 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:35.550 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:36.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:35 smithi138 ceph-mon[192513]: pgmap v850: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 965 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 15868/46170 objects degraded (34.369%); 4351/46170 objects misplaced (9.424%); 1.1 MiB/s, 172 objects/s recovering 2023-10-01T09:28:36.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:35 smithi138 ceph-mon[185886]: pgmap v850: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 965 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 15868/46170 objects degraded (34.369%); 4351/46170 objects misplaced (9.424%); 1.1 MiB/s, 172 objects/s recovering 2023-10-01T09:28:36.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:35 smithi190 ceph-mon[160435]: pgmap v850: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 965 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 15868/46170 objects degraded (34.369%); 4351/46170 objects misplaced (9.424%); 1.1 MiB/s, 172 objects/s recovering 2023-10-01T09:28:36.878 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:36.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:36.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:36.985 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:37.023 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:37.100 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:37.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:37.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:37.212 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:28:37.257 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:37.329 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:28:37.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:37.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:28:37.435 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:28:37.443 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:37 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 15868/46170 objects degraded (34.369%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:37.444 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:37 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 15868/46170 objects degraded (34.369%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:37.471 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:37.526 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:28:37.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:37.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:28:37.632 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:28:37.698 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:37.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:37 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 15868/46170 objects degraded (34.369%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:38.034 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:28:38.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:38.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:28:38.140 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:38.175 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:38.245 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:38.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:38.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:38.356 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:38.423 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:28:38.434 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:38.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:38 smithi190 ceph-mon[160435]: pgmap v851: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 881 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 15491/42162 objects degraded (36.742%); 4167/42162 objects misplaced (9.883%); 2.2 MiB/s, 166 objects/s recovering 2023-10-01T09:28:38.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2711079397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:28:38.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:38 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:28:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:38 smithi138 ceph-mon[192513]: pgmap v851: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 881 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 15491/42162 objects degraded (36.742%); 4167/42162 objects misplaced (9.883%); 2.2 MiB/s, 166 objects/s recovering 2023-10-01T09:28:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2711079397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:28:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:38 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:28:38.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:38 smithi138 ceph-mon[185886]: pgmap v851: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 881 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 15491/42162 objects degraded (36.742%); 4167/42162 objects misplaced (9.883%); 2.2 MiB/s, 166 objects/s recovering 2023-10-01T09:28:38.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2711079397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:28:38.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:38 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:28:39.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:39 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:28:39.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:39 smithi190 ceph-mon[160435]: failed to encode map e313 with expected crc 2023-10-01T09:28:39.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:39 smithi190 ceph-mon[160435]: failed to encode map e313 with expected crc 2023-10-01T09:28:39.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:39 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:28:39.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:39 smithi190 ceph-mon[160435]: osdmap e313: 8 total, 8 up, 7 in 2023-10-01T09:28:39.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:39 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:28:39.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:39 smithi138 ceph-mon[192513]: failed to encode map e313 with expected crc 2023-10-01T09:28:39.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:39 smithi138 ceph-mon[192513]: failed to encode map e313 with expected crc 2023-10-01T09:28:39.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:39 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:28:39.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:39 smithi138 ceph-mon[192513]: osdmap e313: 8 total, 8 up, 7 in 2023-10-01T09:28:39.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:39 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:28:39.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:39 smithi138 ceph-mon[185886]: failed to encode map e313 with expected crc 2023-10-01T09:28:39.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:39 smithi138 ceph-mon[185886]: failed to encode map e313 with expected crc 2023-10-01T09:28:39.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:39 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:28:39.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:39 smithi138 ceph-mon[185886]: osdmap e313: 8 total, 8 up, 7 in 2023-10-01T09:28:40.115 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:40.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:40.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:40.228 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:40.270 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:40.328 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:40.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:40.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:40.416 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:40 smithi138 ceph-mon[185886]: pgmap v853: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 788 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 15491/37668 objects degraded (41.125%); 3962/37668 objects misplaced (10.518%); 2.2 MiB/s, 209 objects/s recovering 2023-10-01T09:28:40.417 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:40.417 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:28:40.433 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:28:40.478 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:40.539 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:28:40.543 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:40 smithi190 ceph-mon[160435]: pgmap v853: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 788 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 15491/37668 objects degraded (41.125%); 3962/37668 objects misplaced (10.518%); 2.2 MiB/s, 209 objects/s recovering 2023-10-01T09:28:40.543 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:40.543 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:28:40.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:40.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:28:40.645 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:40.690 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:40.744 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:40.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:40.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:40 smithi138 ceph-mon[192513]: pgmap v853: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 788 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 15491/37668 objects degraded (41.125%); 3962/37668 objects misplaced (10.518%); 2.2 MiB/s, 209 objects/s recovering 2023-10-01T09:28:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:28:40.854 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:28:40.894 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:40.965 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:28:40.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:40.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:28:41.079 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:41.105 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:41.146 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:41.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:41.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:41.251 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:28:41.285 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:41.353 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:28:41.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:41.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:28:41.463 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:41.497 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:41.559 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:41.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:41.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:41.667 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:28:41.710 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:41.727 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:28:41.727 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:28:41.776 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:28:41.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:41.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:28:41.884 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:28:41.954 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:42.044 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:41 smithi138 ceph-mon[185886]: pgmap v854: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 694 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 15116/33198 objects degraded (45.533%); 3792/33198 objects misplaced (11.422%); 2.4 MiB/s, 195 objects/s recovering 2023-10-01T09:28:42.044 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:41 smithi138 ceph-mon[192513]: pgmap v854: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 694 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 15116/33198 objects degraded (45.533%); 3792/33198 objects misplaced (11.422%); 2.4 MiB/s, 195 objects/s recovering 2023-10-01T09:28:42.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:41 smithi190 ceph-mon[160435]: pgmap v854: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 694 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 15116/33198 objects degraded (45.533%); 3792/33198 objects misplaced (11.422%); 2.4 MiB/s, 195 objects/s recovering 2023-10-01T09:28:42.534 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5, 6] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2023-10-01T09:28:42.535 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:28:42.535 INFO:tasks.thrashosds.thrasher:Reweighting osd 5 to 0.920966428629851 2023-10-01T09:28:42.535 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 0.920966428629851 2023-10-01T09:28:42.554 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:28:42.554 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:28:42.555 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:28:42.555 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:28:42.560 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:28:42.598 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:28:42.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:42.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:28:42.615 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:28:42.653 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:28:42.699 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:28:42.710 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:42.714 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:28:42.714 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:28:42.715 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:28:42.715 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:28:42.715 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:28:42.715 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:28:42.715 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:28:42.715 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:28:42.747 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:42.815 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:42.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:42.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:42.922 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:28:42.955 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:43 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2240761236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:28:43.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:43 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 15116/33198 objects degraded (45.533%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:43.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:43 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2240761236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:28:43.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:43 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 15116/33198 objects degraded (45.533%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:43.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:43 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2240761236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:28:43.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:43 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 15116/33198 objects degraded (45.533%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:43.488 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:28:43.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:43.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:28:43.514 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:28:43.598 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:28:43.727 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:44.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:44 smithi190 ceph-mon[160435]: pgmap v855: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 596 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 15116/28482 objects degraded (53.072%); 3587/28482 objects misplaced (12.594%); 2.4 MiB/s, 199 objects/s recovering 2023-10-01T09:28:44.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:44 smithi138 ceph-mon[192513]: pgmap v855: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 596 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 15116/28482 objects degraded (53.072%); 3587/28482 objects misplaced (12.594%); 2.4 MiB/s, 199 objects/s recovering 2023-10-01T09:28:44.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:44 smithi138 ceph-mon[185886]: pgmap v855: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 596 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 15116/28482 objects degraded (53.072%); 3587/28482 objects misplaced (12.594%); 2.4 MiB/s, 199 objects/s recovering 2023-10-01T09:28:44.737 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:28:44.887 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:28:44.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:44.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:28:44.999 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:28:45.040 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:28:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:28:44] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.33.4" 2023-10-01T09:28:45.144 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:45.159 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:28:45.297 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:28:45.334 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:28:45.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:45.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:28:45.446 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:28:45.489 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:45.554 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:28:45.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:45.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:28:45.664 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:28:45.701 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:45.764 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:28:45.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:45.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:28:45.870 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:28:45.906 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:45.963 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:28:45.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:45.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:28:45.987 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:45 smithi138 ceph-mon[185886]: pgmap v856: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 585 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 15116/27960 objects degraded (54.063%); 3587/27960 objects misplaced (12.829%); 2.4 MiB/s, 199 objects/s recovering 2023-10-01T09:28:45.987 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:45 smithi138 ceph-mon[192513]: pgmap v856: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 585 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 15116/27960 objects degraded (54.063%); 3587/27960 objects misplaced (12.829%); 2.4 MiB/s, 199 objects/s recovering 2023-10-01T09:28:46.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:45 smithi190 ceph-mon[160435]: pgmap v856: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 585 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 15116/27960 objects degraded (54.063%); 3587/27960 objects misplaced (12.829%); 2.4 MiB/s, 199 objects/s recovering 2023-10-01T09:28:46.072 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:46.157 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:46.479 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:46.484 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:28:46.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:46.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:46.589 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:28:46.691 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:47.076 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:28:47.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:28:47.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:47.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:47.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:28:47.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:47.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:47.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:28:47.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:47.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:48.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:48 smithi190 ceph-mon[160435]: failed to encode map e314 with expected crc 2023-10-01T09:28:48.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:48 smithi190 ceph-mon[160435]: osdmap e314: 8 total, 8 up, 7 in 2023-10-01T09:28:48.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:48 smithi190 ceph-mon[160435]: failed to encode map e314 with expected crc 2023-10-01T09:28:48.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:48 smithi190 ceph-mon[160435]: failed to encode map e314 with expected crc 2023-10-01T09:28:48.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:48 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 15116/27960 objects degraded (54.063%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:48.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:48 smithi190 ceph-mon[160435]: pgmap v858: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 505 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 14727/24114 objects degraded (61.072%); 3411/24114 objects misplaced (14.145%); 2.6 MiB/s, 124 objects/s recovering 2023-10-01T09:28:48.310 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:48 smithi190 ceph-mon[160435]: osdmap e315: 8 total, 8 up, 7 in 2023-10-01T09:28:48.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:48 smithi138 ceph-mon[192513]: failed to encode map e314 with expected crc 2023-10-01T09:28:48.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:48 smithi138 ceph-mon[192513]: osdmap e314: 8 total, 8 up, 7 in 2023-10-01T09:28:48.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:48 smithi138 ceph-mon[192513]: failed to encode map e314 with expected crc 2023-10-01T09:28:48.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:48 smithi138 ceph-mon[192513]: failed to encode map e314 with expected crc 2023-10-01T09:28:48.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:48 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 15116/27960 objects degraded (54.063%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:48.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:48 smithi138 ceph-mon[192513]: pgmap v858: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 505 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 14727/24114 objects degraded (61.072%); 3411/24114 objects misplaced (14.145%); 2.6 MiB/s, 124 objects/s recovering 2023-10-01T09:28:48.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:48 smithi138 ceph-mon[192513]: osdmap e315: 8 total, 8 up, 7 in 2023-10-01T09:28:48.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:48 smithi138 ceph-mon[185886]: failed to encode map e314 with expected crc 2023-10-01T09:28:48.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:48 smithi138 ceph-mon[185886]: osdmap e314: 8 total, 8 up, 7 in 2023-10-01T09:28:48.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:48 smithi138 ceph-mon[185886]: failed to encode map e314 with expected crc 2023-10-01T09:28:48.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:48 smithi138 ceph-mon[185886]: failed to encode map e314 with expected crc 2023-10-01T09:28:48.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:48 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 15116/27960 objects degraded (54.063%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:48.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:48 smithi138 ceph-mon[185886]: pgmap v858: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 505 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 14727/24114 objects degraded (61.072%); 3411/24114 objects misplaced (14.145%); 2.6 MiB/s, 124 objects/s recovering 2023-10-01T09:28:48.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:48 smithi138 ceph-mon[185886]: osdmap e315: 8 total, 8 up, 7 in 2023-10-01T09:28:48.686 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:28:48.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:48.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:28:48.794 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:48.840 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:48.915 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:48.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:48.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:49.021 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:28:49.049 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:49.206 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:28:49.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:49.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:28:49.316 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:49.351 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:49.468 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:49.469 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:49 smithi138 ceph-mon[185886]: failed to encode map e315 with expected crc 2023-10-01T09:28:49.469 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:49 smithi138 ceph-mon[185886]: failed to encode map e315 with expected crc 2023-10-01T09:28:49.469 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:49 smithi138 ceph-mon[185886]: failed to encode map e315 with expected crc 2023-10-01T09:28:49.470 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:49 smithi138 ceph-mon[185886]: failed to encode map e315 with expected crc 2023-10-01T09:28:49.470 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:49 smithi138 ceph-mon[192513]: failed to encode map e315 with expected crc 2023-10-01T09:28:49.470 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:49 smithi138 ceph-mon[192513]: failed to encode map e315 with expected crc 2023-10-01T09:28:49.470 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:49 smithi138 ceph-mon[192513]: failed to encode map e315 with expected crc 2023-10-01T09:28:49.470 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:49 smithi138 ceph-mon[192513]: failed to encode map e315 with expected crc 2023-10-01T09:28:49.476 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:49 smithi190 ceph-mon[160435]: failed to encode map e315 with expected crc 2023-10-01T09:28:49.476 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:49 smithi190 ceph-mon[160435]: failed to encode map e315 with expected crc 2023-10-01T09:28:49.476 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:49 smithi190 ceph-mon[160435]: failed to encode map e315 with expected crc 2023-10-01T09:28:49.476 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:49 smithi190 ceph-mon[160435]: failed to encode map e315 with expected crc 2023-10-01T09:28:49.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:49.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:49.578 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:49.621 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:49.791 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:49.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:49.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:49.896 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:28:49.984 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:50.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:50 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:50.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:50 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:50.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:50 smithi190 ceph-mon[160435]: pgmap v860: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 433 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 14727/20658 objects degraded (71.290%); 3229/20658 objects misplaced (15.631%); 1.5 MiB/s, 173 objects/s recovering 2023-10-01T09:28:50.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:50 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:50.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:50 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:50.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:50 smithi138 ceph-mon[185886]: pgmap v860: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 433 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 14727/20658 objects degraded (71.290%); 3229/20658 objects misplaced (15.631%); 1.5 MiB/s, 173 objects/s recovering 2023-10-01T09:28:50.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:50 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:50.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:50 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:50.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:50 smithi138 ceph-mon[192513]: pgmap v860: 89 pgs: 2 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 76 active+clean; 433 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 14727/20658 objects degraded (71.290%); 3229/20658 objects misplaced (15.631%); 1.5 MiB/s, 173 objects/s recovering 2023-10-01T09:28:50.952 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:28:50.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:50.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:28:51.062 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:51.172 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:51.254 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:51.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:51.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:51.358 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:28:51.384 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:51.393 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:28:51.406 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.5 to 0.920966 (ebc4) 2023-10-01T09:28:51.522 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:28:51.530 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:51 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/729863711' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:28:51.530 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:51 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:28:51.530 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:51 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:51.531 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:51 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:51.531 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:51 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2226915754' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.920966428629851}]: dispatch 2023-10-01T09:28:51.531 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:51 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.920966428629851}]: dispatch 2023-10-01T09:28:51.531 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:51 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/729863711' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:28:51.531 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:51 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:28:51.531 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:51 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:51.531 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:51 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:51.531 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:51 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2226915754' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.920966428629851}]: dispatch 2023-10-01T09:28:51.531 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:51 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.920966428629851}]: dispatch 2023-10-01T09:28:51.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:51.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:28:51.633 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:51.723 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:51.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:51 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/729863711' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:28:51.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:51 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:28:51.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:51 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:51.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:51 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:51.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:51 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2226915754' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.920966428629851}]: dispatch 2023-10-01T09:28:51.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:51 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.920966428629851}]: dispatch 2023-10-01T09:28:52.104 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:52.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:52.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:52.214 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:52.241 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:52.283 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:52.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:52.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:52.388 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:52.461 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:52.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:52 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:28:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:52 smithi190 ceph-mon[160435]: failed to encode map e316 with expected crc 2023-10-01T09:28:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:52 smithi190 ceph-mon[160435]: failed to encode map e316 with expected crc 2023-10-01T09:28:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:52 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:28:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:52 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.920966428629851}]': finished 2023-10-01T09:28:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:52 smithi190 ceph-mon[160435]: osdmap e316: 8 total, 8 up, 7 in 2023-10-01T09:28:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:52 smithi190 ceph-mon[160435]: failed to encode map e316 with expected crc 2023-10-01T09:28:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:52 smithi190 ceph-mon[160435]: failed to encode map e316 with expected crc 2023-10-01T09:28:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:52 smithi190 ceph-mon[160435]: pgmap v862: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 337 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 14409/16032 objects degraded (89.876%); 2890/16032 objects misplaced (18.026%); 5.2 MiB/s, 245 objects/s recovering 2023-10-01T09:28:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:52 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 14727/20658 objects degraded (71.290%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:52.654 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:28:52.654 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[192513]: failed to encode map e316 with expected crc 2023-10-01T09:28:52.654 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[192513]: failed to encode map e316 with expected crc 2023-10-01T09:28:52.655 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:28:52.655 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.920966428629851}]': finished 2023-10-01T09:28:52.655 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[192513]: osdmap e316: 8 total, 8 up, 7 in 2023-10-01T09:28:52.655 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[192513]: failed to encode map e316 with expected crc 2023-10-01T09:28:52.655 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[192513]: failed to encode map e316 with expected crc 2023-10-01T09:28:52.655 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[192513]: pgmap v862: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 337 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 14409/16032 objects degraded (89.876%); 2890/16032 objects misplaced (18.026%); 5.2 MiB/s, 245 objects/s recovering 2023-10-01T09:28:52.655 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 14727/20658 objects degraded (71.290%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:52.656 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:28:52.656 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[185886]: failed to encode map e316 with expected crc 2023-10-01T09:28:52.656 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[185886]: failed to encode map e316 with expected crc 2023-10-01T09:28:52.656 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:28:52.656 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.920966428629851}]': finished 2023-10-01T09:28:52.656 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[185886]: osdmap e316: 8 total, 8 up, 7 in 2023-10-01T09:28:52.656 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[185886]: failed to encode map e316 with expected crc 2023-10-01T09:28:52.656 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[185886]: failed to encode map e316 with expected crc 2023-10-01T09:28:52.657 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[185886]: pgmap v862: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 337 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 14409/16032 objects degraded (89.876%); 2890/16032 objects misplaced (18.026%); 5.2 MiB/s, 245 objects/s recovering 2023-10-01T09:28:52.657 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:52 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 14727/20658 objects degraded (71.290%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:28:52.984 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:52.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:53.092 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:53.157 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:53.604 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:53.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:53.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:53.704 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:28:53.709 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:28:53.725 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:53 smithi138 ceph-mon[192513]: failed to encode map e317 with expected crc 2023-10-01T09:28:53.725 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:53 smithi138 ceph-mon[192513]: osdmap e317: 8 total, 8 up, 7 in 2023-10-01T09:28:53.725 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:53 smithi138 ceph-mon[192513]: failed to encode map e317 with expected crc 2023-10-01T09:28:53.725 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:53 smithi138 ceph-mon[192513]: failed to encode map e317 with expected crc 2023-10-01T09:28:53.725 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:53 smithi138 ceph-mon[192513]: failed to encode map e317 with expected crc 2023-10-01T09:28:53.725 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:53 smithi138 ceph-mon[185886]: failed to encode map e317 with expected crc 2023-10-01T09:28:53.725 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:53 smithi138 ceph-mon[185886]: osdmap e317: 8 total, 8 up, 7 in 2023-10-01T09:28:53.725 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:53 smithi138 ceph-mon[185886]: failed to encode map e317 with expected crc 2023-10-01T09:28:53.726 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:53 smithi138 ceph-mon[185886]: failed to encode map e317 with expected crc 2023-10-01T09:28:53.726 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:53 smithi138 ceph-mon[185886]: failed to encode map e317 with expected crc 2023-10-01T09:28:53.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:53 smithi190 ceph-mon[160435]: failed to encode map e317 with expected crc 2023-10-01T09:28:53.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:53 smithi190 ceph-mon[160435]: osdmap e317: 8 total, 8 up, 7 in 2023-10-01T09:28:53.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:53 smithi190 ceph-mon[160435]: failed to encode map e317 with expected crc 2023-10-01T09:28:53.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:53 smithi190 ceph-mon[160435]: failed to encode map e317 with expected crc 2023-10-01T09:28:53.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:53 smithi190 ceph-mon[160435]: failed to encode map e317 with expected crc 2023-10-01T09:28:53.820 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:54.224 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:28:54.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:54.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:28:54.332 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:54.349 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:28:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:28:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:28:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:28:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227518 5.b 5:dd303cc1:::benchmark_data_smithi138_240338_object112381:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T09:28:54.350 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.540699+0000", 2023-10-01T09:28:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 589.80416517499998, 2023-10-01T09:28:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2259289409999998, 2023-10-01T09:28:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:28:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:28:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:28:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227518 2023-10-01T09:28:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540699+0000", 2023-10-01T09:28:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540699+0000", 2023-10-01T09:28:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540698+0000", 2023-10-01T09:28:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:28:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540702+0000", 2023-10-01T09:28:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5999999999999998e-06 2023-10-01T09:28:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540708+0000", 2023-10-01T09:28:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7790000000000001e-06 2023-10-01T09:28:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540713+0000", 2023-10-01T09:28:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0540000000000002e-06 2023-10-01T09:28:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540759+0000", 2023-10-01T09:28:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6001000000000002e-05 2023-10-01T09:28:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540886+0000", 2023-10-01T09:28:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012746099999999999 2023-10-01T09:28:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:28:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.540997+0000", 2023-10-01T09:28:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110389 2023-10-01T09:28:54.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.553848+0000", 2023-10-01T09:28:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012850996 2023-10-01T09:28:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.559796+0000", 2023-10-01T09:28:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059481589999999997 2023-10-01T09:28:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.559807+0000", 2023-10-01T09:28:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0993e-05 2023-10-01T09:28:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547860+0000", 2023-10-01T09:28:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98805305899999996 2023-10-01T09:28:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547873+0000", 2023-10-01T09:28:54.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.322e-05 2023-10-01T09:28:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.750500+0000", 2023-10-01T09:28:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.202626795 2023-10-01T09:28:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:28:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.750659+0000", 2023-10-01T09:28:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015917500000000001 2023-10-01T09:28:54.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.359 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.766393+0000", 2023-10-01T09:28:54.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015734056999999999 2023-10-01T09:28:54.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.359 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.766628+0000", 2023-10-01T09:28:54.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00023459500000000001 2023-10-01T09:28:54.359 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.359 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.359 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.361 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227520 5.b 5:d8712104:::benchmark_data_smithi138_240338_object112383:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T09:28:54.361 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.547004+0000", 2023-10-01T09:28:54.361 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 589.79785998499995, 2023-10-01T09:28:54.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.227945434, 2023-10-01T09:28:54.361 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.361 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:28:54.362 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.362 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:28:54.362 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:28:54.362 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227520 2023-10-01T09:28:54.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.362 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547004+0000", 2023-10-01T09:28:54.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547004+0000", 2023-10-01T09:28:54.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547002+0000", 2023-10-01T09:28:54.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:28:54.363 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547011+0000", 2023-10-01T09:28:54.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3419999999999997e-06 2023-10-01T09:28:54.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547026+0000", 2023-10-01T09:28:54.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4453e-05 2023-10-01T09:28:54.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.364 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.364 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.547042+0000", 2023-10-01T09:28:54.364 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5999999999999999e-05 2023-10-01T09:28:54.364 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.364 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.553877+0000", 2023-10-01T09:28:54.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006835163 2023-10-01T09:28:54.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.365 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.554019+0000", 2023-10-01T09:28:54.365 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014198699999999999 2023-10-01T09:28:54.365 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.365 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.365 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:28:54.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.554141+0000", 2023-10-01T09:28:54.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000122032 2023-10-01T09:28:54.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.559831+0000", 2023-10-01T09:28:54.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005690339 2023-10-01T09:28:54.366 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.366 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.366 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.366 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.559838+0000", 2023-10-01T09:28:54.366 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1690000000000003e-06 2023-10-01T09:28:54.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547900+0000", 2023-10-01T09:28:54.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.98806163599999997 2023-10-01T09:28:54.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.367 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.367 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547905+0000", 2023-10-01T09:28:54.367 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2580000000000003e-06 2023-10-01T09:28:54.367 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.367 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.750939+0000", 2023-10-01T09:28:54.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2030342000000001 2023-10-01T09:28:54.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:28:54.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751046+0000", 2023-10-01T09:28:54.368 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106037 2023-10-01T09:28:54.368 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.368 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.368 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.368 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.774856+0000", 2023-10-01T09:28:54.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.023810120000000001 2023-10-01T09:28:54.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.369 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.369 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.774949+0000", 2023-10-01T09:28:54.369 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3781000000000002e-05 2023-10-01T09:28:54.369 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.369 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.369 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.369 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.369 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.370 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227542 5.8 5:163a5025:::benchmark_data_smithi138_240338_object112405:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:28:54.370 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.593796+0000", 2023-10-01T09:28:54.370 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 589.75106753199998, 2023-10-01T09:28:54.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.158330061, 2023-10-01T09:28:54.370 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.370 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.370 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.370 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:28:54.370 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:28:54.370 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227542 2023-10-01T09:28:54.370 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.370 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593796+0000", 2023-10-01T09:28:54.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593796+0000", 2023-10-01T09:28:54.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593795+0000", 2023-10-01T09:28:54.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:28:54.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593802+0000", 2023-10-01T09:28:54.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2359999999999998e-06 2023-10-01T09:28:54.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593815+0000", 2023-10-01T09:28:54.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2765000000000001e-05 2023-10-01T09:28:54.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593823+0000", 2023-10-01T09:28:54.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0220000000000004e-06 2023-10-01T09:28:54.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.596113+0000", 2023-10-01T09:28:54.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002289999 2023-10-01T09:28:54.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.596128+0000", 2023-10-01T09:28:54.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4827000000000001e-05 2023-10-01T09:28:54.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.374 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.549844+0000", 2023-10-01T09:28:54.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95371578099999998 2023-10-01T09:28:54.374 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.549855+0000", 2023-10-01T09:28:54.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.163e-05 2023-10-01T09:28:54.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.375 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.734943+0000", 2023-10-01T09:28:54.375 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1850879670000001 2023-10-01T09:28:54.375 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.375 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.375 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.735123+0000", 2023-10-01T09:28:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000179347 2023-10-01T09:28:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:28:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.735771+0000", 2023-10-01T09:28:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00064860999999999999 2023-10-01T09:28:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751876+0000", 2023-10-01T09:28:54.376 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016104579000000001 2023-10-01T09:28:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751917+0000", 2023-10-01T09:28:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1201000000000001e-05 2023-10-01T09:28:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751925+0000", 2023-10-01T09:28:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8830000000000006e-06 2023-10-01T09:28:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.377 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751962+0000", 2023-10-01T09:28:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7521999999999998e-05 2023-10-01T09:28:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.752015+0000", 2023-10-01T09:28:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2407999999999997e-05 2023-10-01T09:28:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.378 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.752127+0000", 2023-10-01T09:28:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111671 2023-10-01T09:28:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227543 5.b 5:d7088ca7:::benchmark_data_smithi138_240338_object112406:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:28:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.593966+0000", 2023-10-01T09:28:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 589.75089809999997, 2023-10-01T09:28:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2140065830000002, 2023-10-01T09:28:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.379 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:28:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:28:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227543 2023-10-01T09:28:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593966+0000", 2023-10-01T09:28:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.380 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593966+0000", 2023-10-01T09:28:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593965+0000", 2023-10-01T09:28:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:28:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593971+0000", 2023-10-01T09:28:54.381 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5830000000000003e-06 2023-10-01T09:28:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593981+0000", 2023-10-01T09:28:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4229999999999994e-06 2023-10-01T09:28:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.593989+0000", 2023-10-01T09:28:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2039999999999997e-06 2023-10-01T09:28:54.382 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.594069+0000", 2023-10-01T09:28:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0267999999999998e-05 2023-10-01T09:28:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.594082+0000", 2023-10-01T09:28:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2901000000000001e-05 2023-10-01T09:28:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.383 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547917+0000", 2023-10-01T09:28:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.95383516800000001 2023-10-01T09:28:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547921+0000", 2023-10-01T09:28:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9430000000000002e-06 2023-10-01T09:28:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751073+0000", 2023-10-01T09:28:54.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2031520040000001 2023-10-01T09:28:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751182+0000", 2023-10-01T09:28:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108553 2023-10-01T09:28:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5,7", 2023-10-01T09:28:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751816+0000", 2023-10-01T09:28:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00063413800000000002 2023-10-01T09:28:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.771765+0000", 2023-10-01T09:28:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019948585000000001 2023-10-01T09:28:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.771773+0000", 2023-10-01T09:28:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8589999999999994e-06 2023-10-01T09:28:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.789897+0000", 2023-10-01T09:28:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018123378999999998 2023-10-01T09:28:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.807826+0000", 2023-10-01T09:28:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017929269000000001 2023-10-01T09:28:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.807867+0000", 2023-10-01T09:28:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1131000000000003e-05 2023-10-01T09:28:54.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.807972+0000", 2023-10-01T09:28:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105274 2023-10-01T09:28:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227549 5.b 5:dca5cded:::benchmark_data_smithi138_240338_object112412:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:28:54.388 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.607851+0000", 2023-10-01T09:28:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 589.73701311499997, 2023-10-01T09:28:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2003441270000001, 2023-10-01T09:28:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:28:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:28:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227549 2023-10-01T09:28:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607851+0000", 2023-10-01T09:28:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607851+0000", 2023-10-01T09:28:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607849+0000", 2023-10-01T09:28:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:28:54.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607857+0000", 2023-10-01T09:28:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0649999999999999e-06 2023-10-01T09:28:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607872+0000", 2023-10-01T09:28:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5038000000000001e-05 2023-10-01T09:28:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607880+0000", 2023-10-01T09:28:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8690000000000004e-06 2023-10-01T09:28:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607947+0000", 2023-10-01T09:28:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6847000000000006e-05 2023-10-01T09:28:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.607959+0000", 2023-10-01T09:28:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2575000000000001e-05 2023-10-01T09:28:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547932+0000", 2023-10-01T09:28:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.93997244999999996 2023-10-01T09:28:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.547936+0000", 2023-10-01T09:28:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.878e-06 2023-10-01T09:28:54.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.762267+0000", 2023-10-01T09:28:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.21433116 2023-10-01T09:28:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.762399+0000", 2023-10-01T09:28:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013192999999999999 2023-10-01T09:28:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5,7", 2023-10-01T09:28:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.762536+0000", 2023-10-01T09:28:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000137439 2023-10-01T09:28:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.771780+0000", 2023-10-01T09:28:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0092435450000000006 2023-10-01T09:28:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.776336+0000", 2023-10-01T09:28:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004556264 2023-10-01T09:28:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.789967+0000", 2023-10-01T09:28:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013631228 2023-10-01T09:28:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808023+0000", 2023-10-01T09:28:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018056274000000001 2023-10-01T09:28:54.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808075+0000", 2023-10-01T09:28:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1910000000000003e-05 2023-10-01T09:28:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808195+0000", 2023-10-01T09:28:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011964 2023-10-01T09:28:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227553 5.8 5:1337bdd0:::benchmark_data_smithi138_240338_object112416:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T09:28:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:04.622592+0000", 2023-10-01T09:28:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 589.72227173900001, 2023-10-01T09:28:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.142155657, 2023-10-01T09:28:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:28:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:28:54.398 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227553 2023-10-01T09:28:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622592+0000", 2023-10-01T09:28:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622592+0000", 2023-10-01T09:28:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622590+0000", 2023-10-01T09:28:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:28:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622599+0000", 2023-10-01T09:28:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9509999999999995e-06 2023-10-01T09:28:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622616+0000", 2023-10-01T09:28:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6458000000000001e-05 2023-10-01T09:28:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622626+0000", 2023-10-01T09:28:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0059e-05 2023-10-01T09:28:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622691+0000", 2023-10-01T09:28:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4848e-05 2023-10-01T09:28:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:04.622705+0000", 2023-10-01T09:28:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4041e-05 2023-10-01T09:28:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.550082+0000", 2023-10-01T09:28:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92737758299999995 2023-10-01T09:28:54.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.550091+0000", 2023-10-01T09:28:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0710000000000001e-06 2023-10-01T09:28:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.744482+0000", 2023-10-01T09:28:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1943907620000001 2023-10-01T09:28:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.744576+0000", 2023-10-01T09:28:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4320999999999994e-05 2023-10-01T09:28:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:28:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.744661+0000", 2023-10-01T09:28:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4855e-05 2023-10-01T09:28:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.751931+0000", 2023-10-01T09:28:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.007269574 2023-10-01T09:28:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.759742+0000", 2023-10-01T09:28:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0078116840000000002 2023-10-01T09:28:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764514+0000", 2023-10-01T09:28:54.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047718409999999998 2023-10-01T09:28:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764594+0000", 2023-10-01T09:28:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9671000000000003e-05 2023-10-01T09:28:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764645+0000", 2023-10-01T09:28:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0713e-05 2023-10-01T09:28:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764748+0000", 2023-10-01T09:28:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103174 2023-10-01T09:28:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227561 5.8 5:1c2dcf03:::benchmark_data_smithi138_240338_object112424:head [delete] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-01T09:28:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:05.744421+0000", 2023-10-01T09:28:54.407 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 588.60044318899998, 2023-10-01T09:28:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.020494499, 2023-10-01T09:28:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:28:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:28:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227561 2023-10-01T09:28:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744421+0000", 2023-10-01T09:28:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744421+0000", 2023-10-01T09:28:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744419+0000", 2023-10-01T09:28:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:28:54.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744427+0000", 2023-10-01T09:28:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6160000000000002e-06 2023-10-01T09:28:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744445+0000", 2023-10-01T09:28:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7246999999999999e-05 2023-10-01T09:28:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744453+0000", 2023-10-01T09:28:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6740000000000003e-06 2023-10-01T09:28:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744541+0000", 2023-10-01T09:28:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7819999999999996e-05 2023-10-01T09:28:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744554+0000", 2023-10-01T09:28:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.237e-05 2023-10-01T09:28:54.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.748068+0000", 2023-10-01T09:28:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0035148250000001 2023-10-01T09:28:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.748233+0000", 2023-10-01T09:28:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016479900000000001 2023-10-01T09:28:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,2,5", 2023-10-01T09:28:54.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.748378+0000", 2023-10-01T09:28:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014517999999999999 2023-10-01T09:28:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.752176+0000", 2023-10-01T09:28:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037973389999999998 2023-10-01T09:28:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.759790+0000", 2023-10-01T09:28:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0076139179999999999 2023-10-01T09:28:54.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764554+0000", 2023-10-01T09:28:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047645869999999998 2023-10-01T09:28:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764783+0000", 2023-10-01T09:28:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00022926399999999999 2023-10-01T09:28:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764818+0000", 2023-10-01T09:28:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4400999999999998e-05 2023-10-01T09:28:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.764915+0000", 2023-10-01T09:28:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7405000000000007e-05 2023-10-01T09:28:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.34683.0:227562 5.b 5:d7e9814e:::benchmark_data_smithi138_240338_object112425:head [delete] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-10-01T09:28:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:19:05.744494+0000", 2023-10-01T09:28:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 588.60036985800002, 2023-10-01T09:28:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.064026162, 2023-10-01T09:28:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.34683", 2023-10-01T09:28:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/340903342", 2023-10-01T09:28:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 227562 2023-10-01T09:28:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.416 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744494+0000", 2023-10-01T09:28:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744494+0000", 2023-10-01T09:28:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744494+0000", 2023-10-01T09:28:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:28:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744497+0000", 2023-10-01T09:28:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3390000000000001e-06 2023-10-01T09:28:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744507+0000", 2023-10-01T09:28:54.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0276e-05 2023-10-01T09:28:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744511+0000", 2023-10-01T09:28:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9160000000000003e-06 2023-10-01T09:28:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744548+0000", 2023-10-01T09:28:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7197999999999999e-05 2023-10-01T09:28:54.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:05.744558+0000", 2023-10-01T09:28:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.55e-06 2023-10-01T09:28:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.767699+0000", 2023-10-01T09:28:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0231407809999999 2023-10-01T09:28:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.767807+0000", 2023-10-01T09:28:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010877199999999999 2023-10-01T09:28:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5,7", 2023-10-01T09:28:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.767928+0000", 2023-10-01T09:28:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120122 2023-10-01T09:28:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.777766+0000", 2023-10-01T09:28:54.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0098384030000000008 2023-10-01T09:28:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.782314+0000", 2023-10-01T09:28:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045480240000000003 2023-10-01T09:28:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.789999+0000", 2023-10-01T09:28:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0076853340000000003 2023-10-01T09:28:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808355+0000", 2023-10-01T09:28:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018356006000000001 2023-10-01T09:28:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808402+0000", 2023-10-01T09:28:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6980000000000001e-05 2023-10-01T09:28:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:19:06.808520+0000", 2023-10-01T09:28:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118026 2023-10-01T09:28:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109327 6.19 6:9b6cdeca:::benchmark_data_smithi138_307751_object13958:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-01T09:28:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.601576+0000", 2023-10-01T09:28:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 434.743288011, 2023-10-01T09:28:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0306919560000001, 2023-10-01T09:28:54.424 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:28:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:28:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:28:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109327 2023-10-01T09:28:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601576+0000", 2023-10-01T09:28:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601576+0000", 2023-10-01T09:28:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601574+0000", 2023-10-01T09:28:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:28:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601584+0000", 2023-10-01T09:28:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0159e-05 2023-10-01T09:28:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601610+0000", 2023-10-01T09:28:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5582999999999999e-05 2023-10-01T09:28:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601621+0000", 2023-10-01T09:28:54.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1435e-05 2023-10-01T09:28:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.767017+0000", 2023-10-01T09:28:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16539611800000001 2023-10-01T09:28:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:28:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.767027+0000", 2023-10-01T09:28:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6609999999999995e-06 2023-10-01T09:28:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.616116+0000", 2023-10-01T09:28:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84908936300000004 2023-10-01T09:28:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:28:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.616242+0000", 2023-10-01T09:28:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012627800000000001 2023-10-01T09:28:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.632186+0000", 2023-10-01T09:28:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015943382999999998 2023-10-01T09:28:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.632268+0000", 2023-10-01T09:28:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2056000000000002e-05 2023-10-01T09:28:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.430 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129799 7.4 7:2a987c36:::benchmark_data_smithi138_367058_object32004:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-10-01T09:28:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:51.394453+0000", 2023-10-01T09:28:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 122.950410983, 2023-10-01T09:28:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.31166278200000003, 2023-10-01T09:28:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:28:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:28:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129799 2023-10-01T09:28:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394453+0000", 2023-10-01T09:28:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394453+0000", 2023-10-01T09:28:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394451+0000", 2023-10-01T09:28:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:28:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394459+0000", 2023-10-01T09:28:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0700000000000007e-06 2023-10-01T09:28:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394472+0000", 2023-10-01T09:28:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2882e-05 2023-10-01T09:28:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394481+0000", 2023-10-01T09:28:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6340000000000007e-06 2023-10-01T09:28:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394584+0000", 2023-10-01T09:28:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010345599999999999 2023-10-01T09:28:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394716+0000", 2023-10-01T09:28:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013240800000000001 2023-10-01T09:28:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3,4", 2023-10-01T09:28:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394846+0000", 2023-10-01T09:28:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012953300000000001 2023-10-01T09:28:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.408545+0000", 2023-10-01T09:28:54.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013699413000000001 2023-10-01T09:28:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.408614+0000", 2023-10-01T09:28:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8595000000000003e-05 2023-10-01T09:28:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.636954+0000", 2023-10-01T09:28:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22833962399999999 2023-10-01T09:28:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.705924+0000", 2023-10-01T09:28:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.068970630000000005 2023-10-01T09:28:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.705984+0000", 2023-10-01T09:28:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0136999999999997e-05 2023-10-01T09:28:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.706116+0000", 2023-10-01T09:28:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013135999999999999 2023-10-01T09:28:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129803 7.4 7:21dfca7d:::benchmark_data_smithi138_367058_object32008:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-10-01T09:28:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:51.394498+0000", 2023-10-01T09:28:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 122.950366374, 2023-10-01T09:28:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.31185276299999998, 2023-10-01T09:28:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.438 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:28:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:28:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129803 2023-10-01T09:28:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394498+0000", 2023-10-01T09:28:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394498+0000", 2023-10-01T09:28:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394497+0000", 2023-10-01T09:28:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:28:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394500+0000", 2023-10-01T09:28:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5519999999999999e-06 2023-10-01T09:28:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394506+0000", 2023-10-01T09:28:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0399999999999998e-06 2023-10-01T09:28:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394511+0000", 2023-10-01T09:28:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8189999999999999e-06 2023-10-01T09:28:54.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.397846+0000", 2023-10-01T09:28:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003335363 2023-10-01T09:28:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.397980+0000", 2023-10-01T09:28:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013349100000000001 2023-10-01T09:28:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3,4", 2023-10-01T09:28:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.398108+0000", 2023-10-01T09:28:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012803200000000001 2023-10-01T09:28:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.408655+0000", 2023-10-01T09:28:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010547575999999999 2023-10-01T09:28:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.408665+0000", 2023-10-01T09:28:54.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7310000000000005e-06 2023-10-01T09:28:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.637002+0000", 2023-10-01T09:28:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22833692 2023-10-01T09:28:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.706176+0000", 2023-10-01T09:28:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.069173945000000001 2023-10-01T09:28:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.706243+0000", 2023-10-01T09:28:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7479999999999998e-05 2023-10-01T09:28:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.706350+0000", 2023-10-01T09:28:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107153 2023-10-01T09:28:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.445 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129812 7.4 7:24dcedab:::benchmark_data_smithi138_367058_object32017:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-10-01T09:28:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:51.411647+0000", 2023-10-01T09:28:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 122.933217167, 2023-10-01T09:28:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.31493354499999998, 2023-10-01T09:28:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.446 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:28:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:28:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129812 2023-10-01T09:28:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411647+0000", 2023-10-01T09:28:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411647+0000", 2023-10-01T09:28:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411645+0000", 2023-10-01T09:28:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:28:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411655+0000", 2023-10-01T09:28:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0538e-05 2023-10-01T09:28:54.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411670+0000", 2023-10-01T09:28:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4566000000000001e-05 2023-10-01T09:28:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411680+0000", 2023-10-01T09:28:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9380000000000004e-06 2023-10-01T09:28:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411729+0000", 2023-10-01T09:28:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9330999999999999e-05 2023-10-01T09:28:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411864+0000", 2023-10-01T09:28:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013525899999999999 2023-10-01T09:28:54.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3,4", 2023-10-01T09:28:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.412012+0000", 2023-10-01T09:28:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014787900000000001 2023-10-01T09:28:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.636906+0000", 2023-10-01T09:28:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22489326600000001 2023-10-01T09:28:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.637082+0000", 2023-10-01T09:28:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000176603 2023-10-01T09:28:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.637093+0000", 2023-10-01T09:28:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0428000000000001e-05 2023-10-01T09:28:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.726416+0000", 2023-10-01T09:28:54.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.089322953999999996 2023-10-01T09:28:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.726470+0000", 2023-10-01T09:28:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4666999999999997e-05 2023-10-01T09:28:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.726580+0000", 2023-10-01T09:28:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110179 2023-10-01T09:28:54.453 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:142310 7.f 7:fadc5338:::benchmark_data_smithi138_367058_object44515:head [delete] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T09:28:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:27:20.468221+0000", 2023-10-01T09:28:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 93.876642828000001, 2023-10-01T09:28:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.34216545700000001, 2023-10-01T09:28:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.454 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:28:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:28:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 142310 2023-10-01T09:28:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.468221+0000", 2023-10-01T09:28:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.468221+0000", 2023-10-01T09:28:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.468220+0000", 2023-10-01T09:28:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:28:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.468227+0000", 2023-10-01T09:28:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1670000000000002e-06 2023-10-01T09:28:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.468239+0000", 2023-10-01T09:28:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1844999999999999e-05 2023-10-01T09:28:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.468246+0000", 2023-10-01T09:28:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.255e-06 2023-10-01T09:28:54.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.471974+0000", 2023-10-01T09:28:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003727756 2023-10-01T09:28:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.472092+0000", 2023-10-01T09:28:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118267 2023-10-01T09:28:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:28:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.472201+0000", 2023-10-01T09:28:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109379 2023-10-01T09:28:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478845+0000", 2023-10-01T09:28:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066437090000000002 2023-10-01T09:28:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.459 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.479233+0000", 2023-10-01T09:28:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00038774099999999999 2023-10-01T09:28:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.810288+0000", 2023-10-01T09:28:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33105508900000002 2023-10-01T09:28:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.810341+0000", 2023-10-01T09:28:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3573999999999999e-05 2023-10-01T09:28:54.460 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.810387+0000", 2023-10-01T09:28:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5324000000000002e-05 2023-10-01T09:28:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:142313 7.6 7:6a412c14:::benchmark_data_smithi138_367058_object44518:head [delete] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T09:28:54.461 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:27:20.474654+0000", 2023-10-01T09:28:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 93.870209557999999, 2023-10-01T09:28:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33643319399999999, 2023-10-01T09:28:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:28:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:28:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 142313 2023-10-01T09:28:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.462 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474654+0000", 2023-10-01T09:28:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474654+0000", 2023-10-01T09:28:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474653+0000", 2023-10-01T09:28:54.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:28:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474660+0000", 2023-10-01T09:28:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.939e-06 2023-10-01T09:28:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474674+0000", 2023-10-01T09:28:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.437e-05 2023-10-01T09:28:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474682+0000", 2023-10-01T09:28:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6620000000000005e-06 2023-10-01T09:28:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474732+0000", 2023-10-01T09:28:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0380999999999998e-05 2023-10-01T09:28:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474860+0000", 2023-10-01T09:28:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012723500000000001 2023-10-01T09:28:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:28:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474974+0000", 2023-10-01T09:28:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011427899999999999 2023-10-01T09:28:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.466 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478152+0000", 2023-10-01T09:28:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031778779999999999 2023-10-01T09:28:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478781+0000", 2023-10-01T09:28:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00062936100000000005 2023-10-01T09:28:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.811011+0000", 2023-10-01T09:28:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33222958899999999 2023-10-01T09:28:54.467 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.811054+0000", 2023-10-01T09:28:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3208999999999999e-05 2023-10-01T09:28:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.811088+0000", 2023-10-01T09:28:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3680000000000003e-05 2023-10-01T09:28:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.468 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:142315 7.6 7:62a92d28:::benchmark_data_smithi138_367058_object44520:head [delete] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T09:28:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:27:20.478706+0000", 2023-10-01T09:28:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 93.866157756999996, 2023-10-01T09:28:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33247613999999998, 2023-10-01T09:28:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:28:54.469 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:28:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 142315 2023-10-01T09:28:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478706+0000", 2023-10-01T09:28:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478706+0000", 2023-10-01T09:28:54.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478705+0000", 2023-10-01T09:28:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:28:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478711+0000", 2023-10-01T09:28:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3049999999999998e-06 2023-10-01T09:28:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478723+0000", 2023-10-01T09:28:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1358999999999999e-05 2023-10-01T09:28:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478730+0000", 2023-10-01T09:28:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.464e-06 2023-10-01T09:28:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478802+0000", 2023-10-01T09:28:54.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1975000000000001e-05 2023-10-01T09:28:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478895+0000", 2023-10-01T09:28:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3499999999999996e-05 2023-10-01T09:28:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:28:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478976+0000", 2023-10-01T09:28:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0068000000000007e-05 2023-10-01T09:28:54.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.481966+0000", 2023-10-01T09:28:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029905190000000001 2023-10-01T09:28:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.482079+0000", 2023-10-01T09:28:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011309799999999999 2023-10-01T09:28:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.811120+0000", 2023-10-01T09:28:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.32904090000000003 2023-10-01T09:28:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.811153+0000", 2023-10-01T09:28:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3448000000000002e-05 2023-10-01T09:28:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.811182+0000", 2023-10-01T09:28:54.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8864e-05 2023-10-01T09:28:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:191021 7.7 7:e778b9f7:::benchmark_data_smithi138_367058_object93226:head [delete] snapc 0=[] ondisk+write+known_if_redirected e315)", 2023-10-01T09:28:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:28:51.211552+0000", 2023-10-01T09:28:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 3.133312428, 2023-10-01T09:28:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.198400145, 2023-10-01T09:28:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.476 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:28:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:28:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 191021 2023-10-01T09:28:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.211552+0000", 2023-10-01T09:28:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.211552+0000", 2023-10-01T09:28:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.211550+0000", 2023-10-01T09:28:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:28:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.211554+0000", 2023-10-01T09:28:54.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0099999999999997e-06 2023-10-01T09:28:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.211562+0000", 2023-10-01T09:28:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5639999999999999e-06 2023-10-01T09:28:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.211567+0000", 2023-10-01T09:28:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.344e-06 2023-10-01T09:28:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.216369+0000", 2023-10-01T09:28:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048019810000000003 2023-10-01T09:28:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.216497+0000", 2023-10-01T09:28:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012783200000000001 2023-10-01T09:28:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:28:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.216626+0000", 2023-10-01T09:28:54.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012919799999999999 2023-10-01T09:28:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.225223+0000", 2023-10-01T09:28:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085961930000000002 2023-10-01T09:28:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.232221+0000", 2023-10-01T09:28:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069983609999999998 2023-10-01T09:28:54.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.409632+0000", 2023-10-01T09:28:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17741085100000001 2023-10-01T09:28:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.409668+0000", 2023-10-01T09:28:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5924999999999999e-05 2023-10-01T09:28:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.409952+0000", 2023-10-01T09:28:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00028396900000000002 2023-10-01T09:28:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:191031 7.f 7:f83baa13:::benchmark_data_smithi138_367058_object93236:head [delete] snapc 0=[] ondisk+write+known_if_redirected e315)", 2023-10-01T09:28:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:28:51.232450+0000", 2023-10-01T09:28:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 3.1124139519999998, 2023-10-01T09:28:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.189737512, 2023-10-01T09:28:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.483 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:28:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:28:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 191031 2023-10-01T09:28:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.232450+0000", 2023-10-01T09:28:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.232450+0000", 2023-10-01T09:28:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.232448+0000", 2023-10-01T09:28:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:28:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.232455+0000", 2023-10-01T09:28:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5769999999999999e-06 2023-10-01T09:28:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.232469+0000", 2023-10-01T09:28:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4034e-05 2023-10-01T09:28:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.232477+0000", 2023-10-01T09:28:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4800000000000001e-06 2023-10-01T09:28:54.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.232527+0000", 2023-10-01T09:28:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9394000000000002e-05 2023-10-01T09:28:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.232617+0000", 2023-10-01T09:28:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0264000000000002e-05 2023-10-01T09:28:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:28:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.232695+0000", 2023-10-01T09:28:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7668000000000002e-05 2023-10-01T09:28:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.236321+0000", 2023-10-01T09:28:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003625985 2023-10-01T09:28:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.238880+0000", 2023-10-01T09:28:54.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025591419999999999 2023-10-01T09:28:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.422112+0000", 2023-10-01T09:28:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.183232331 2023-10-01T09:28:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.422153+0000", 2023-10-01T09:28:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1105e-05 2023-10-01T09:28:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:51.422188+0000", 2023-10-01T09:28:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4152000000000001e-05 2023-10-01T09:28:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:191550 7.7 7:ea7a6236:::benchmark_data_smithi138_367058_object93755:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-01T09:28:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:28:52.162240+0000", 2023-10-01T09:28:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.182624224, 2023-10-01T09:28:54.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.197148085, 2023-10-01T09:28:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:28:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:28:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 191550 2023-10-01T09:28:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.162240+0000", 2023-10-01T09:28:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.162240+0000", 2023-10-01T09:28:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.162238+0000", 2023-10-01T09:28:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:28:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.162246+0000", 2023-10-01T09:28:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5560000000000008e-06 2023-10-01T09:28:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.162262+0000", 2023-10-01T09:28:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5262000000000001e-05 2023-10-01T09:28:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.162271+0000", 2023-10-01T09:28:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2490000000000007e-06 2023-10-01T09:28:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.164418+0000", 2023-10-01T09:28:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002147047 2023-10-01T09:28:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.164538+0000", 2023-10-01T09:28:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120029 2023-10-01T09:28:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:28:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.164655+0000", 2023-10-01T09:28:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000116891 2023-10-01T09:28:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.172654+0000", 2023-10-01T09:28:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0079993340000000003 2023-10-01T09:28:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.176231+0000", 2023-10-01T09:28:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035764659999999999 2023-10-01T09:28:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.359027+0000", 2023-10-01T09:28:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.182796612 2023-10-01T09:28:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.359072+0000", 2023-10-01T09:28:54.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4249000000000003e-05 2023-10-01T09:28:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.359388+0000", 2023-10-01T09:28:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00031623000000000003 2023-10-01T09:28:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.497 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:191557 7.7 7:e206c561:::benchmark_data_smithi138_367058_object93762:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-01T09:28:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:28:52.168291+0000", 2023-10-01T09:28:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.1765734390000002, 2023-10-01T09:28:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19134055899999999, 2023-10-01T09:28:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:28:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:28:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 191557 2023-10-01T09:28:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.168291+0000", 2023-10-01T09:28:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.168291+0000", 2023-10-01T09:28:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.168289+0000", 2023-10-01T09:28:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:28:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.168296+0000", 2023-10-01T09:28:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6340000000000001e-06 2023-10-01T09:28:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.168307+0000", 2023-10-01T09:28:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0891e-05 2023-10-01T09:28:54.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.168315+0000", 2023-10-01T09:28:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3679999999999996e-06 2023-10-01T09:28:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.172686+0000", 2023-10-01T09:28:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004370896 2023-10-01T09:28:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.172822+0000", 2023-10-01T09:28:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013637799999999999 2023-10-01T09:28:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:28:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.172955+0000", 2023-10-01T09:28:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013281100000000001 2023-10-01T09:28:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.176305+0000", 2023-10-01T09:28:54.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003350282 2023-10-01T09:28:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.298505+0000", 2023-10-01T09:28:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.12220020600000001 2023-10-01T09:28:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.359433+0000", 2023-10-01T09:28:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.060927874999999999 2023-10-01T09:28:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.359475+0000", 2023-10-01T09:28:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1609000000000001e-05 2023-10-01T09:28:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.359631+0000", 2023-10-01T09:28:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015636099999999999 2023-10-01T09:28:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:191560 7.6 7:645d3573:::benchmark_data_smithi138_367058_object93765:head [delete] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-10-01T09:28:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:28:52.169307+0000", 2023-10-01T09:28:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.1755565900000002, 2023-10-01T09:28:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18623858200000001, 2023-10-01T09:28:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:28:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:28:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:28:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:28:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:28:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 191560 2023-10-01T09:28:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.505 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:28:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:28:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.169307+0000", 2023-10-01T09:28:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:28:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.169307+0000", 2023-10-01T09:28:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:28:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:28:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.169307+0000", 2023-10-01T09:28:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:28:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:28:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.169312+0000", 2023-10-01T09:28:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9810000000000003e-06 2023-10-01T09:28:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:28:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.169322+0000", 2023-10-01T09:28:54.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0383999999999999e-05 2023-10-01T09:28:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:28:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.169329+0000", 2023-10-01T09:28:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8199999999999999e-06 2023-10-01T09:28:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:28:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.169380+0000", 2023-10-01T09:28:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1377e-05 2023-10-01T09:28:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:28:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.169511+0000", 2023-10-01T09:28:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131156 2023-10-01T09:28:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:28:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.169631+0000", 2023-10-01T09:28:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000119675 2023-10-01T09:28:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.173607+0000", 2023-10-01T09:28:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039757619999999999 2023-10-01T09:28:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:28:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.175763+0000", 2023-10-01T09:28:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021563860000000002 2023-10-01T09:28:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:28:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.355448+0000", 2023-10-01T09:28:54.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17968466 2023-10-01T09:28:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:28:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.355500+0000", 2023-10-01T09:28:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2315000000000003e-05 2023-10-01T09:28:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:28:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:28:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:28:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:52.355546+0000", 2023-10-01T09:28:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5825000000000003e-05 2023-10-01T09:28:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.511 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:28:54.512 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:28:54.512 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:28:54.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:54.513 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:54.513 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:54.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:54.614 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:54.649 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:54 smithi190 ceph-mon[160435]: pgmap v864: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 258 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 14409/12264 objects degraded (117.490%); 2690/12264 objects misplaced (21.934%); 3.2 MiB/s, 263 objects/s recovering 2023-10-01T09:28:54.653 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:54.718 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:54.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:54.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:54.727 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:54 smithi138 ceph-mon[185886]: pgmap v864: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 258 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 14409/12264 objects degraded (117.490%); 2690/12264 objects misplaced (21.934%); 3.2 MiB/s, 263 objects/s recovering 2023-10-01T09:28:54.728 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:54 smithi138 ceph-mon[192513]: pgmap v864: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 258 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 14409/12264 objects degraded (117.490%); 2690/12264 objects misplaced (21.934%); 3.2 MiB/s, 263 objects/s recovering 2023-10-01T09:28:54.824 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:54.900 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:55.066 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:55.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:28:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:28:54] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.33.4" 2023-10-01T09:28:55.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:55.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:55.180 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:28:55.219 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:55.284 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:28:55.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:55.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:28:55.397 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:55.447 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:28:55.466 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:56.304 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:56.304 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:28:56.305 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:56 smithi138 ceph-mon[185886]: pgmap v865: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 235 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 14409/11148 objects degraded (129.252%); 2690/11148 objects misplaced (24.130%); 2.4 MiB/s, 200 objects/s recovering 2023-10-01T09:28:56.423 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:56.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:56.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:56.527 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:56.553 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:56.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:56.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:28:56.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:56 smithi190 ceph-mon[160435]: pgmap v865: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 235 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 14409/11148 objects degraded (129.252%); 2690/11148 objects misplaced (24.130%); 2.4 MiB/s, 200 objects/s recovering 2023-10-01T09:28:56.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:28:56.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:28:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:56 smithi138 ceph-mon[192513]: pgmap v865: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 235 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 14409/11148 objects degraded (129.252%); 2690/11148 objects misplaced (24.130%); 2.4 MiB/s, 200 objects/s recovering 2023-10-01T09:28:56.604 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:56.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:56.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:56.719 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:28:56.804 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:57.015 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:28:57.029 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:28:57.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:57.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:28:57.136 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:57.290 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:57.401 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:28:57.638 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:57.644 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:57 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 14409/11148 objects degraded (129.252%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-01T09:28:57.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:57.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:57.649 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:57 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 14409/11148 objects degraded (129.252%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-01T09:28:57.746 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:28:57.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:57 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 14409/11148 objects degraded (129.252%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-01T09:28:57.848 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:57.955 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:28:58.017 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:28:58.045 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:28:58.061 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:28:58.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:58.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:28:58.105 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:28:58.145 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:28:58.172 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:58.182 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:28:58.213 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:28:58.240 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:58.249 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:28:58.261 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:28:58.262 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:28:58.262 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:28:58.262 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:28:58.262 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:28:58.262 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:28:58.262 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:28:58.262 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:28:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:58 smithi138 ceph-mon[185886]: pgmap v866: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 152 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 14023/7152 objects degraded (196.071%); 2283/7152 objects misplaced (31.921%); 5.4 MiB/s, 196 objects/s recovering 2023-10-01T09:28:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:28:58 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4014340151' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:28:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:58 smithi138 ceph-mon[192513]: pgmap v866: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 152 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 14023/7152 objects degraded (196.071%); 2283/7152 objects misplaced (31.921%); 5.4 MiB/s, 196 objects/s recovering 2023-10-01T09:28:58.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:28:58 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4014340151' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:28:58.750 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5, 6] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2023-10-01T09:28:58.750 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-01T09:28:58.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:58 smithi190 ceph-mon[160435]: pgmap v866: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 152 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 14023/7152 objects degraded (196.071%); 2283/7152 objects misplaced (31.921%); 5.4 MiB/s, 196 objects/s recovering 2023-10-01T09:28:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:28:58 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4014340151' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:28:58.883 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Removed 97792 objects 2023-10-01T09:28:58.883 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Clean up completed and total clean up time :214.929 2023-10-01T09:28:58.895 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-10-01T09:28:58.896 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-10-01T09:28:59.365 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:28:59.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:59.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:28:59.469 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:28:59.495 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:28:59.537 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:28:59.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:28:59.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:28:59.642 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:28:59.668 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:00.666 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:29:00.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:00.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:29:00.769 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:29:00.795 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:01.068 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:29:01.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:01.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:29:01.138 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:01.173 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:01.197 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:00 smithi138 ceph-mon[192513]: pgmap v867: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 69 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 14023/3168 objects degraded (442.645%); 2086/3168 objects misplaced (65.846%); 5.1 MiB/s, 164 objects/s recovering 2023-10-01T09:29:01.197 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:00 smithi138 ceph-mon[185886]: pgmap v867: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 69 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 14023/3168 objects degraded (442.645%); 2086/3168 objects misplaced (65.846%); 5.1 MiB/s, 164 objects/s recovering 2023-10-01T09:29:01.200 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:01.268 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:01.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:01.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:01.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:00 smithi190 ceph-mon[160435]: pgmap v867: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 69 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 14023/3168 objects degraded (442.645%); 2086/3168 objects misplaced (65.846%); 5.1 MiB/s, 164 objects/s recovering 2023-10-01T09:29:01.371 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:01.403 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:01.659 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:01.660 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:01.757 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:01.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:01.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:01.861 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:01.895 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:01.947 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:01.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:01.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:02.054 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:29:02.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:01 smithi138 ceph-mon[192513]: pgmap v868: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 15 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 13849/585 objects degraded (2367.350%); 1901/585 objects misplaced (324.957%); 3.5 MiB/s, 126 objects/s recovering 2023-10-01T09:29:02.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:01 smithi138 ceph-mon[185886]: pgmap v868: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 15 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 13849/585 objects degraded (2367.350%); 1901/585 objects misplaced (324.957%); 3.5 MiB/s, 126 objects/s recovering 2023-10-01T09:29:02.083 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:02.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:01 smithi190 ceph-mon[160435]: pgmap v868: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 15 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 13849/585 objects degraded (2367.350%); 1901/585 objects misplaced (324.957%); 3.5 MiB/s, 126 objects/s recovering 2023-10-01T09:29:02.955 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:29:03.167 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:29:03.193 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:02 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1606180172' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:03.194 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:02 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:03.194 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:02 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 13849/585 objects degraded (2367.350%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-01T09:29:03.194 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:02 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1606180172' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:03.194 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:02 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:03.194 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:02 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 13849/585 objects degraded (2367.350%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-01T09:29:03.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:02 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1606180172' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:02 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:02 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 13849/585 objects degraded (2367.350%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-01T09:29:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:03 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:29:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:03 smithi190 ceph-mon[160435]: failed to encode map e318 with expected crc 2023-10-01T09:29:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:03 smithi190 ceph-mon[160435]: failed to encode map e318 with expected crc 2023-10-01T09:29:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:03 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:29:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:03 smithi190 ceph-mon[160435]: osdmap e318: 8 total, 8 up, 7 in 2023-10-01T09:29:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:03 smithi190 ceph-mon[160435]: pgmap v870: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 13849/84 objects degraded (16486.905%); 1878/84 objects misplaced (2235.714%); 3.5 MiB/s, 235 objects/s recovering 2023-10-01T09:29:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:03 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:29:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:03 smithi138 ceph-mon[185886]: failed to encode map e318 with expected crc 2023-10-01T09:29:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:03 smithi138 ceph-mon[185886]: failed to encode map e318 with expected crc 2023-10-01T09:29:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:03 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:29:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:03 smithi138 ceph-mon[185886]: osdmap e318: 8 total, 8 up, 7 in 2023-10-01T09:29:04.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:03 smithi138 ceph-mon[185886]: pgmap v870: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 13849/84 objects degraded (16486.905%); 1878/84 objects misplaced (2235.714%); 3.5 MiB/s, 235 objects/s recovering 2023-10-01T09:29:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:03 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:29:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:03 smithi138 ceph-mon[192513]: failed to encode map e318 with expected crc 2023-10-01T09:29:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:03 smithi138 ceph-mon[192513]: failed to encode map e318 with expected crc 2023-10-01T09:29:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:03 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:29:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:03 smithi138 ceph-mon[192513]: osdmap e318: 8 total, 8 up, 7 in 2023-10-01T09:29:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:03 smithi138 ceph-mon[192513]: pgmap v870: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 405 op/s; 13849/84 objects degraded (16486.905%); 1878/84 objects misplaced (2235.714%); 3.5 MiB/s, 235 objects/s recovering 2023-10-01T09:29:04.566 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:29:04.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:04.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:29:04.670 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:29:04.809 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:05.298 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:29:05.300 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:04 smithi138 ceph-mon[192513]: failed to encode map e319 with expected crc 2023-10-01T09:29:05.300 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:04 smithi138 ceph-mon[192513]: osdmap e319: 8 total, 8 up, 7 in 2023-10-01T09:29:05.300 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:04 smithi138 ceph-mon[192513]: failed to encode map e319 with expected crc 2023-10-01T09:29:05.300 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:04 smithi138 ceph-mon[192513]: failed to encode map e319 with expected crc 2023-10-01T09:29:05.300 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:04 smithi138 ceph-mon[192513]: failed to encode map e319 with expected crc 2023-10-01T09:29:05.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:04 smithi138 ceph-mon[185886]: failed to encode map e319 with expected crc 2023-10-01T09:29:05.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:04 smithi138 ceph-mon[185886]: osdmap e319: 8 total, 8 up, 7 in 2023-10-01T09:29:05.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:04 smithi138 ceph-mon[185886]: failed to encode map e319 with expected crc 2023-10-01T09:29:05.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:04 smithi138 ceph-mon[185886]: failed to encode map e319 with expected crc 2023-10-01T09:29:05.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:04 smithi138 ceph-mon[185886]: failed to encode map e319 with expected crc 2023-10-01T09:29:05.302 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:29:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:29:04] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.33.4" 2023-10-01T09:29:05.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:05.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:29:05.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:04 smithi190 ceph-mon[160435]: failed to encode map e319 with expected crc 2023-10-01T09:29:05.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:04 smithi190 ceph-mon[160435]: osdmap e319: 8 total, 8 up, 7 in 2023-10-01T09:29:05.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:04 smithi190 ceph-mon[160435]: failed to encode map e319 with expected crc 2023-10-01T09:29:05.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:04 smithi190 ceph-mon[160435]: failed to encode map e319 with expected crc 2023-10-01T09:29:05.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:04 smithi190 ceph-mon[160435]: failed to encode map e319 with expected crc 2023-10-01T09:29:05.403 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:05.433 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:06.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:05 smithi190 ceph-mon[160435]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:05 smithi190 ceph-mon[160435]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:05 smithi190 ceph-mon[160435]: osdmap e320: 8 total, 8 up, 7 in 2023-10-01T09:29:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:05 smithi190 ceph-mon[160435]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:05 smithi190 ceph-mon[160435]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:05 smithi190 ceph-mon[160435]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:05 smithi190 ceph-mon[160435]: pgmap v873: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 171 op/s; 13849/84 objects degraded (16486.905%); 1878/84 objects misplaced (2235.714%); 1.8 MiB/s, 261 objects/s recovering 2023-10-01T09:29:06.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:05 smithi138 ceph-mon[192513]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:05 smithi138 ceph-mon[192513]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:05 smithi138 ceph-mon[192513]: osdmap e320: 8 total, 8 up, 7 in 2023-10-01T09:29:06.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:05 smithi138 ceph-mon[192513]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:05 smithi138 ceph-mon[192513]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:05 smithi138 ceph-mon[192513]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:05 smithi138 ceph-mon[192513]: pgmap v873: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 171 op/s; 13849/84 objects degraded (16486.905%); 1878/84 objects misplaced (2235.714%); 1.8 MiB/s, 261 objects/s recovering 2023-10-01T09:29:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:05 smithi138 ceph-mon[185886]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:05 smithi138 ceph-mon[185886]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:05 smithi138 ceph-mon[185886]: osdmap e320: 8 total, 8 up, 7 in 2023-10-01T09:29:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:05 smithi138 ceph-mon[185886]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:05 smithi138 ceph-mon[185886]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:05 smithi138 ceph-mon[185886]: failed to encode map e320 with expected crc 2023-10-01T09:29:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:05 smithi138 ceph-mon[185886]: pgmap v873: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 171 op/s; 13849/84 objects degraded (16486.905%); 1878/84 objects misplaced (2235.714%); 1.8 MiB/s, 261 objects/s recovering 2023-10-01T09:29:06.329 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:06.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:06.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:06.434 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:06.460 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:06.506 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:06.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:06.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:06.611 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:29:06.645 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:06.898 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:29:06.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:06.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:29:07.002 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:29:07.084 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:07.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:07 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 13849/84 objects degraded (16486.905%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T09:29:07.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:07 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 13849/84 objects degraded (16486.905%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T09:29:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:07 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 13849/84 objects degraded (16486.905%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T09:29:08.374 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:29:08.736 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:08 smithi138 ceph-mon[192513]: pgmap v874: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 27 op/s; 12403/84 objects degraded (14765.476%); 1878/84 objects misplaced (2235.714%); 96 KiB/s, 465 objects/s recovering 2023-10-01T09:29:08.737 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:08 smithi138 ceph-mon[185886]: pgmap v874: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 27 op/s; 12403/84 objects degraded (14765.476%); 1878/84 objects misplaced (2235.714%); 96 KiB/s, 465 objects/s recovering 2023-10-01T09:29:08.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:08 smithi190 ceph-mon[160435]: pgmap v874: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 27 op/s; 12403/84 objects degraded (14765.476%); 1878/84 objects misplaced (2235.714%); 96 KiB/s, 465 objects/s recovering 2023-10-01T09:29:09.792 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:29:09.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:09.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:29:09.897 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:09.924 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:09.968 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:09.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:09.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:10.045 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:09 smithi138 ceph-mon[185886]: pgmap v875: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 24 op/s; 8411/84 objects degraded (10013.095%); 1878/84 objects misplaced (2235.714%); 83 KiB/s, 488 objects/s recovering 2023-10-01T09:29:10.045 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:09 smithi138 ceph-mon[192513]: pgmap v875: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 24 op/s; 8411/84 objects degraded (10013.095%); 1878/84 objects misplaced (2235.714%); 83 KiB/s, 488 objects/s recovering 2023-10-01T09:29:10.074 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:29:10.149 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:10.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:09 smithi190 ceph-mon[160435]: pgmap v875: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 24 op/s; 8411/84 objects degraded (10013.095%); 1878/84 objects misplaced (2235.714%); 83 KiB/s, 488 objects/s recovering 2023-10-01T09:29:10.609 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:29:10.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:10.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:29:10.637 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:29:10.637 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:29:10.667 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:10.713 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:10.739 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:10.785 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:10.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:10.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:10.822 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:10.890 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:10.916 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:10.960 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:10.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:10.995 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:29:10.996 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:10 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2435724639' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:29:10.996 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:10 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:29:10.996 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:10 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/797745760' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:29:10.996 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:10 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:29:10.996 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:29:10.996 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:10 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2435724639' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:29:10.996 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:10 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:29:10.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:10 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/797745760' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:29:10.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:10 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:29:11.007 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.0 to 1 (10000) 2023-10-01T09:29:11.018 INFO:teuthology.orchestra.run.smithi138.stderr:pool 'unique_pool_2' removed 2023-10-01T09:29:11.067 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:29:11.093 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:11.135 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:29:11.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:11.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:29:11.239 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:11.268 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:29:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:10 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2435724639' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:29:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:10 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:29:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:10 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/797745760' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:29:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:10 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:29:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:11 smithi190 ceph-mon[160435]: failed to encode map e321 with expected crc 2023-10-01T09:29:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:11 smithi190 ceph-mon[160435]: failed to encode map e321 with expected crc 2023-10-01T09:29:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:11 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:29:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:11 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:29:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:11 smithi190 ceph-mon[160435]: osdmap e321: 8 total, 8 up, 7 in 2023-10-01T09:29:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:11 smithi190 ceph-mon[160435]: failed to encode map e321 with expected crc 2023-10-01T09:29:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:11 smithi190 ceph-mon[160435]: failed to encode map e321 with expected crc 2023-10-01T09:29:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:11 smithi190 ceph-mon[160435]: pgmap v877: 73 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 15/84 objects degraded (17.857%) 2023-10-01T09:29:12.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[185886]: failed to encode map e321 with expected crc 2023-10-01T09:29:12.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[185886]: failed to encode map e321 with expected crc 2023-10-01T09:29:12.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:29:12.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:29:12.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[185886]: osdmap e321: 8 total, 8 up, 7 in 2023-10-01T09:29:12.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[185886]: failed to encode map e321 with expected crc 2023-10-01T09:29:12.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[185886]: failed to encode map e321 with expected crc 2023-10-01T09:29:12.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[185886]: pgmap v877: 73 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 15/84 objects degraded (17.857%) 2023-10-01T09:29:12.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[192513]: failed to encode map e321 with expected crc 2023-10-01T09:29:12.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[192513]: failed to encode map e321 with expected crc 2023-10-01T09:29:12.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:29:12.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:29:12.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[192513]: osdmap e321: 8 total, 8 up, 7 in 2023-10-01T09:29:12.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[192513]: failed to encode map e321 with expected crc 2023-10-01T09:29:12.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[192513]: failed to encode map e321 with expected crc 2023-10-01T09:29:12.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:11 smithi138 ceph-mon[192513]: pgmap v877: 73 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 15/84 objects degraded (17.857%) 2023-10-01T09:29:13.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:12 smithi190 ceph-mon[160435]: failed to encode map e322 with expected crc 2023-10-01T09:29:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:12 smithi190 ceph-mon[160435]: failed to encode map e322 with expected crc 2023-10-01T09:29:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:12 smithi190 ceph-mon[160435]: osdmap e322: 8 total, 8 up, 7 in 2023-10-01T09:29:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:12 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 15/84 objects degraded (17.857%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-01T09:29:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:12 smithi138 ceph-mon[192513]: failed to encode map e322 with expected crc 2023-10-01T09:29:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:12 smithi138 ceph-mon[192513]: failed to encode map e322 with expected crc 2023-10-01T09:29:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:12 smithi138 ceph-mon[192513]: osdmap e322: 8 total, 8 up, 7 in 2023-10-01T09:29:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:12 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 15/84 objects degraded (17.857%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-01T09:29:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:12 smithi138 ceph-mon[185886]: failed to encode map e322 with expected crc 2023-10-01T09:29:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:12 smithi138 ceph-mon[185886]: failed to encode map e322 with expected crc 2023-10-01T09:29:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:12 smithi138 ceph-mon[185886]: osdmap e322: 8 total, 8 up, 7 in 2023-10-01T09:29:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:12 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 15/84 objects degraded (17.857%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-10-01T09:29:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:14 smithi190 ceph-mon[160435]: failed to encode map e323 with expected crc 2023-10-01T09:29:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:14 smithi190 ceph-mon[160435]: failed to encode map e323 with expected crc 2023-10-01T09:29:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:14 smithi190 ceph-mon[160435]: osdmap e323: 8 total, 8 up, 7 in 2023-10-01T09:29:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:14 smithi190 ceph-mon[160435]: failed to encode map e323 with expected crc 2023-10-01T09:29:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:14 smithi190 ceph-mon[160435]: failed to encode map e323 with expected crc 2023-10-01T09:29:14.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:14 smithi190 ceph-mon[160435]: pgmap v880: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:14 smithi138 ceph-mon[192513]: failed to encode map e323 with expected crc 2023-10-01T09:29:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:14 smithi138 ceph-mon[192513]: failed to encode map e323 with expected crc 2023-10-01T09:29:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:14 smithi138 ceph-mon[192513]: osdmap e323: 8 total, 8 up, 7 in 2023-10-01T09:29:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:14 smithi138 ceph-mon[192513]: failed to encode map e323 with expected crc 2023-10-01T09:29:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:14 smithi138 ceph-mon[192513]: failed to encode map e323 with expected crc 2023-10-01T09:29:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:14 smithi138 ceph-mon[192513]: pgmap v880: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:14.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:14 smithi138 ceph-mon[185886]: failed to encode map e323 with expected crc 2023-10-01T09:29:14.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:14 smithi138 ceph-mon[185886]: failed to encode map e323 with expected crc 2023-10-01T09:29:14.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:14 smithi138 ceph-mon[185886]: osdmap e323: 8 total, 8 up, 7 in 2023-10-01T09:29:14.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:14 smithi138 ceph-mon[185886]: failed to encode map e323 with expected crc 2023-10-01T09:29:14.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:14 smithi138 ceph-mon[185886]: failed to encode map e323 with expected crc 2023-10-01T09:29:14.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:14 smithi138 ceph-mon[185886]: pgmap v880: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:15.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:15 smithi138 ceph-mon[185886]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/84 objects degraded (17.857%), 2 pgs degraded, 1 pg undersized) 2023-10-01T09:29:15.262 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:15 smithi138 ceph-mon[192513]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/84 objects degraded (17.857%), 2 pgs degraded, 1 pg undersized) 2023-10-01T09:29:15.262 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:29:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:29:14] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.33.4" 2023-10-01T09:29:15.279 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:15.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:15.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:15.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:15 smithi190 ceph-mon[160435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/84 objects degraded (17.857%), 2 pgs degraded, 1 pg undersized) 2023-10-01T09:29:15.385 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:15.445 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:15.569 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-01T09:29:15.629 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:29:15.680 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T09:29:15.681 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T09:29:15.681 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-10-01T09:29:15.681 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-10-01T09:29:15.796 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:15.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:15.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:15.902 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:15.934 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:15.982 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:15.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:15.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:16.088 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:16.117 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:16.161 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:16.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:16.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:16.269 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:16.295 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:16 smithi138 ceph-mon[192513]: pgmap v881: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:16.295 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:16 smithi138 ceph-mon[185886]: pgmap v881: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:16.297 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:16.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:16 smithi190 ceph-mon[160435]: pgmap v881: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:16.494 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:16.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:16.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:16.598 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:16.626 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:16.794 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:16.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:16.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:16.898 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:16.927 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:16.979 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:16.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:16.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:17.084 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:17.089 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:17.119 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:17.172 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:17.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:17.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:17.277 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:29:17.309 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:17.357 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:29:17.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:17.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:29:17.464 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:17.492 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:17.537 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:17.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:17.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:17.642 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:17.669 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:17.711 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:17.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:17.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:17.816 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:17.841 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:17.885 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:17.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:17.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:17.934 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:17.992 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:29:18.001 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:29:18.022 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:18.182 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:18.271 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:17 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:18.271 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:17 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:18.272 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:17 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4028219950' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:29:18.272 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:17 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:29:18.272 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:17 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:29:18.272 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:17 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:18.272 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:17 smithi138 ceph-mon[192513]: pgmap v882: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-01T09:29:18.272 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:17 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:18.272 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:17 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:18.272 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:17 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4028219950' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:29:18.272 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:17 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:29:18.273 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:17 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:29:18.273 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:17 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:18.273 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:17 smithi138 ceph-mon[185886]: pgmap v882: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-01T09:29:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:17 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:17 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:17 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4028219950' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:29:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:17 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:29:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:17 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:29:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:17 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:17 smithi190 ceph-mon[160435]: pgmap v882: 73 pgs: 73 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 2 objects/s recovering 2023-10-01T09:29:18.526 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:29:18.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:18.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:29:18.630 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:18.658 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:19.022 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:19.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:19.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:19.126 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:19.158 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:19.207 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:19.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:19.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:19.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:18 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:29:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:18 smithi190 ceph-mon[160435]: failed to encode map e324 with expected crc 2023-10-01T09:29:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:18 smithi190 ceph-mon[160435]: failed to encode map e324 with expected crc 2023-10-01T09:29:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:18 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4028219950' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:29:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:18 smithi190 ceph-mon[160435]: osdmap e324: 8 total, 8 up, 7 in 2023-10-01T09:29:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:18 smithi190 ceph-mon[160435]: failed to encode map e324 with expected crc 2023-10-01T09:29:19.314 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:19.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:18 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:29:19.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:18 smithi138 ceph-mon[185886]: failed to encode map e324 with expected crc 2023-10-01T09:29:19.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:18 smithi138 ceph-mon[185886]: failed to encode map e324 with expected crc 2023-10-01T09:29:19.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:18 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4028219950' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:29:19.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:18 smithi138 ceph-mon[185886]: osdmap e324: 8 total, 8 up, 7 in 2023-10-01T09:29:19.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:18 smithi138 ceph-mon[185886]: failed to encode map e324 with expected crc 2023-10-01T09:29:19.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:18 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:29:19.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:18 smithi138 ceph-mon[192513]: failed to encode map e324 with expected crc 2023-10-01T09:29:19.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:18 smithi138 ceph-mon[192513]: failed to encode map e324 with expected crc 2023-10-01T09:29:19.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:18 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4028219950' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:29:19.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:18 smithi138 ceph-mon[192513]: osdmap e324: 8 total, 8 up, 7 in 2023-10-01T09:29:19.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:18 smithi138 ceph-mon[192513]: failed to encode map e324 with expected crc 2023-10-01T09:29:19.343 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:19.387 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:19.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:19.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:19.493 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:19.520 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:19.816 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:19.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:19.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:19.921 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:29:19.952 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:19.989 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:19 smithi138 ceph-mon[192513]: pgmap v884: 73 pgs: 73 active+clean; 4.6 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:29:20.003 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:29:20.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:20.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:29:20.077 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:29:20.095 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:29:20.110 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:20.112 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:29:20.131 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:29:20.141 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:20.150 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:29:20.168 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:29:20.187 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:29:20.205 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:29:20.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:29:20.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:29:20.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:29:20.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:29:20.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:29:20.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:29:20.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:29:20.206 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:29:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:19 smithi190 ceph-mon[160435]: pgmap v884: 73 pgs: 73 active+clean; 4.6 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:29:20.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:19 smithi138 ceph-mon[185886]: pgmap v884: 73 pgs: 73 active+clean; 4.6 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:29:21.280 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:20 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/893218222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:29:21.280 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:20 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:21.280 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:20 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/893218222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:29:21.280 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:20 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:21.290 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:21.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:21.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:21.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:20 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/893218222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:29:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:20 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:21.395 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:21.424 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:21.680 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:21.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:21.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:21.784 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:29:21.806 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:29:21.812 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:22.008 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.1 to 1 (10000) 2023-10-01T09:29:22.172 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:29:22.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:22.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:29:22.277 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:22.280 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:22 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/338015846' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:29:22.280 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:22 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:29:22.280 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:22 smithi138 ceph-mon[192513]: pgmap v885: 73 pgs: 73 active+clean; 4.6 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:29:22.280 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:22 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/338015846' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:29:22.281 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:22 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:29:22.281 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:22 smithi138 ceph-mon[185886]: pgmap v885: 73 pgs: 73 active+clean; 4.6 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:29:22.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:21 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/338015846' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:29:22.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:21 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:29:22.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:21 smithi190 ceph-mon[160435]: pgmap v885: 73 pgs: 73 active+clean; 4.6 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:29:22.308 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:22.360 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:22.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:22.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:22.465 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:22.492 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:22.535 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:22.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:22.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:22.641 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:29:22.666 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:22.702 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:29:22.702 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:29:22.702 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:29:22.703 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:29:22.703 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:29:22.708 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:29:22.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:22.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:29:22.813 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:22.842 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:22.937 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:22.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:22.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:23.041 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:29:23.067 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:23.108 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:29:23.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:23.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:29:23.216 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:23.240 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:23 smithi138 ceph-mon[192513]: failed to encode map e325 with expected crc 2023-10-01T09:29:23.241 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:23 smithi138 ceph-mon[192513]: failed to encode map e325 with expected crc 2023-10-01T09:29:23.241 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:23 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:29:23.241 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:23 smithi138 ceph-mon[192513]: osdmap e325: 8 total, 8 up, 7 in 2023-10-01T09:29:23.241 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:23 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4208514911' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-01T09:29:23.241 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:22 smithi138 ceph-mon[185886]: failed to encode map e325 with expected crc 2023-10-01T09:29:23.241 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:22 smithi138 ceph-mon[185886]: failed to encode map e325 with expected crc 2023-10-01T09:29:23.241 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:22 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:29:23.242 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:22 smithi138 ceph-mon[185886]: osdmap e325: 8 total, 8 up, 7 in 2023-10-01T09:29:23.242 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:22 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4208514911' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-01T09:29:23.243 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:23.286 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-01T09:29:23.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:22 smithi190 ceph-mon[160435]: failed to encode map e325 with expected crc 2023-10-01T09:29:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:22 smithi190 ceph-mon[160435]: failed to encode map e325 with expected crc 2023-10-01T09:29:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:22 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:29:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:22 smithi190 ceph-mon[160435]: osdmap e325: 8 total, 8 up, 7 in 2023-10-01T09:29:23.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:22 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4208514911' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-01T09:29:23.352 INFO:teuthology.orchestra.run.smithi138.stderr:pool 'unique_pool_3' created 2023-10-01T09:29:23.555 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:23.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:23.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:23.659 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:29:23.685 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:23.728 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:29:23.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:23.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:29:23.805 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:29:23.827 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:29:23.837 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:29:23.895 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:24.056 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:29:24.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:24.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:29:24.161 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:29:24.187 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:24.240 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:29:24.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:24.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:29:24.250 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-10-01T09:29:24.267 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:24.292 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:24.344 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:29:24.374 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:24.418 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:29:24.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:24.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:29:24.479 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:24.479 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[192513]: failed to encode map e325 with expected crc 2023-10-01T09:29:24.479 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[192513]: failed to encode map e325 with expected crc 2023-10-01T09:29:24.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[192513]: failed to encode map e326 with expected crc 2023-10-01T09:29:24.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[192513]: failed to encode map e326 with expected crc 2023-10-01T09:29:24.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4208514911' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-01T09:29:24.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[192513]: osdmap e326: 8 total, 8 up, 7 in 2023-10-01T09:29:24.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[192513]: failed to encode map e326 with expected crc 2023-10-01T09:29:24.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[192513]: pgmap v888: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:24.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[185886]: failed to encode map e325 with expected crc 2023-10-01T09:29:24.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[185886]: failed to encode map e325 with expected crc 2023-10-01T09:29:24.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[185886]: failed to encode map e326 with expected crc 2023-10-01T09:29:24.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[185886]: failed to encode map e326 with expected crc 2023-10-01T09:29:24.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4208514911' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-01T09:29:24.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[185886]: osdmap e326: 8 total, 8 up, 7 in 2023-10-01T09:29:24.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[185886]: failed to encode map e326 with expected crc 2023-10-01T09:29:24.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:24 smithi138 ceph-mon[185886]: pgmap v888: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:24.525 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:24.555 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:24.708 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:24.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:24.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:24 smithi190 ceph-mon[160435]: failed to encode map e325 with expected crc 2023-10-01T09:29:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:24 smithi190 ceph-mon[160435]: failed to encode map e325 with expected crc 2023-10-01T09:29:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:24 smithi190 ceph-mon[160435]: failed to encode map e326 with expected crc 2023-10-01T09:29:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:24 smithi190 ceph-mon[160435]: failed to encode map e326 with expected crc 2023-10-01T09:29:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4208514911' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-01T09:29:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:24 smithi190 ceph-mon[160435]: osdmap e326: 8 total, 8 up, 7 in 2023-10-01T09:29:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:24 smithi190 ceph-mon[160435]: failed to encode map e326 with expected crc 2023-10-01T09:29:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:24 smithi190 ceph-mon[160435]: pgmap v888: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:24.812 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:24.831 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:24.842 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:25.220 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:29:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:29:24] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.33.4" 2023-10-01T09:29:25.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:25 smithi138 ceph-mon[192513]: failed to encode map e327 with expected crc 2023-10-01T09:29:25.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:25 smithi138 ceph-mon[192513]: osdmap e327: 8 total, 8 up, 7 in 2023-10-01T09:29:25.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:25 smithi138 ceph-mon[192513]: failed to encode map e327 with expected crc 2023-10-01T09:29:25.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:25 smithi138 ceph-mon[192513]: failed to encode map e327 with expected crc 2023-10-01T09:29:25.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:25.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:29:25.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:25 smithi138 ceph-mon[185886]: failed to encode map e327 with expected crc 2023-10-01T09:29:25.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:25 smithi138 ceph-mon[185886]: osdmap e327: 8 total, 8 up, 7 in 2023-10-01T09:29:25.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:25 smithi138 ceph-mon[185886]: failed to encode map e327 with expected crc 2023-10-01T09:29:25.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:25 smithi138 ceph-mon[185886]: failed to encode map e327 with expected crc 2023-10-01T09:29:25.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:25.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:29:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:25 smithi190 ceph-mon[160435]: failed to encode map e327 with expected crc 2023-10-01T09:29:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:25 smithi190 ceph-mon[160435]: osdmap e327: 8 total, 8 up, 7 in 2023-10-01T09:29:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:25 smithi190 ceph-mon[160435]: failed to encode map e327 with expected crc 2023-10-01T09:29:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:25 smithi190 ceph-mon[160435]: failed to encode map e327 with expected crc 2023-10-01T09:29:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:29:26.146 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:26.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:26.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:26.251 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:26.267 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:26.285 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:26.335 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:26.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:26.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:26.442 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:26.469 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:26.513 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:26.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:26.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:26.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:26 smithi190 ceph-mon[160435]: failed to encode map e328 with expected crc 2023-10-01T09:29:26.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:26 smithi190 ceph-mon[160435]: failed to encode map e328 with expected crc 2023-10-01T09:29:26.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:26 smithi190 ceph-mon[160435]: osdmap e328: 8 total, 8 up, 7 in 2023-10-01T09:29:26.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:26 smithi190 ceph-mon[160435]: pgmap v891: 89 pgs: 12 unknown, 77 active+clean; 4.6 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:26.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:26 smithi138 ceph-mon[192513]: failed to encode map e328 with expected crc 2023-10-01T09:29:26.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:26 smithi138 ceph-mon[192513]: failed to encode map e328 with expected crc 2023-10-01T09:29:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:26 smithi138 ceph-mon[192513]: osdmap e328: 8 total, 8 up, 7 in 2023-10-01T09:29:26.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:26 smithi138 ceph-mon[192513]: pgmap v891: 89 pgs: 12 unknown, 77 active+clean; 4.6 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:26 smithi138 ceph-mon[185886]: failed to encode map e328 with expected crc 2023-10-01T09:29:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:26 smithi138 ceph-mon[185886]: failed to encode map e328 with expected crc 2023-10-01T09:29:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:26 smithi138 ceph-mon[185886]: osdmap e328: 8 total, 8 up, 7 in 2023-10-01T09:29:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:26 smithi138 ceph-mon[185886]: pgmap v891: 89 pgs: 12 unknown, 77 active+clean; 4.6 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:26.620 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:26.646 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:26.688 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:26.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:26.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:26.795 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:26.820 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:26.862 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:26.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:26.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:26.967 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:29:26.993 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:27.035 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:29:27.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:27.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:29:27.141 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:29:27.175 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:27.474 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:29:27.493 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:29:27.513 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:29:27.538 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:29:27.564 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:29:27.583 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:29:27.601 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:29:27.618 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:29:27.619 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:29:27.619 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:29:27.620 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:29:27.620 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:29:27.620 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:29:27.620 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:29:27.620 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:29:27.620 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:29:27.625 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:29:27.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:27.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:29:27.732 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:29:27.765 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:27.917 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:29:27.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:27.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:29:28.020 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:28.047 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:28.348 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:29:28.348 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.3 to 1 (10000) 2023-10-01T09:29:28.624 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1797271304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:29:28.624 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1951793117' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:28.624 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:28 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:28.624 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:28 smithi138 ceph-mon[185886]: pgmap v892: 89 pgs: 8 unknown, 81 active+clean; 4.6 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:28.625 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4122067770' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:29:28.625 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:28 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:29:28.625 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1797271304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:29:28.625 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1951793117' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:28.625 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:28 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:28.625 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:28 smithi138 ceph-mon[192513]: pgmap v892: 89 pgs: 8 unknown, 81 active+clean; 4.6 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:28.625 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4122067770' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:29:28.626 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:28 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:29:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1797271304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:29:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1951793117' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:28 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:28 smithi190 ceph-mon[160435]: pgmap v892: 89 pgs: 8 unknown, 81 active+clean; 4.6 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4122067770' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:29:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:28 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:29:29.499 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:29.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:29.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:29.598 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:29 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:29:29.598 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:29 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:29:29.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:29 smithi138 ceph-mon[192513]: failed to encode map e329 with expected crc 2023-10-01T09:29:29.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:29 smithi138 ceph-mon[192513]: failed to encode map e329 with expected crc 2023-10-01T09:29:29.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:29 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:29:29.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:29 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:29:29.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:29 smithi138 ceph-mon[192513]: osdmap e329: 8 total, 8 up, 7 in 2023-10-01T09:29:29.599 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:29 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:29:29.599 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:29 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:29:29.599 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:29 smithi138 ceph-mon[185886]: failed to encode map e329 with expected crc 2023-10-01T09:29:29.599 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:29 smithi138 ceph-mon[185886]: failed to encode map e329 with expected crc 2023-10-01T09:29:29.599 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:29 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:29:29.600 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:29 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:29:29.600 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:29 smithi138 ceph-mon[185886]: osdmap e329: 8 total, 8 up, 7 in 2023-10-01T09:29:29.604 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:29.632 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:29 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:29:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:29 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:29:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:29 smithi190 ceph-mon[160435]: failed to encode map e329 with expected crc 2023-10-01T09:29:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:29 smithi190 ceph-mon[160435]: failed to encode map e329 with expected crc 2023-10-01T09:29:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:29 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:29:29.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:29 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:29:29.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:29 smithi190 ceph-mon[160435]: osdmap e329: 8 total, 8 up, 7 in 2023-10-01T09:29:30.176 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-01T09:29:30.278 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:30.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:30.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:30.382 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:30.412 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:30.461 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:30.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:30.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:30.567 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:29:30.597 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:30.601 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:29:30.601 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:29:30.627 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:30 smithi138 ceph-mon[192513]: pgmap v894: 89 pgs: 89 active+clean; 4.6 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:30.628 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:30 smithi138 ceph-mon[185886]: pgmap v894: 89 pgs: 89 active+clean; 4.6 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:30.643 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:29:30.651 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:30 smithi190 ceph-mon[160435]: pgmap v894: 89 pgs: 89 active+clean; 4.6 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:30.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:30.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:29:30.754 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:29:30.781 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:30.824 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:29:30.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:30.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:29:30.933 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:30.966 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:31.009 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:31.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:31.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:31.116 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:29:31.145 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:31.187 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:29:31.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:31.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:29:31.294 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:31.324 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:31.334 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:29:31.357 INFO:teuthology.orchestra.run.smithi138.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-10-01T09:29:31.367 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:31.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:31.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:31.375 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:31.474 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:29:31.503 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:31.544 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:29:31.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:31.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:29:31.586 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1312786837' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:29:31.586 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:31 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:29:31.586 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1312786837' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:29:31.586 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:31 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:29:31.651 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:31.675 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1312786837' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:29:31.676 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:31 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:29:31.677 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:31.719 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:31.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:31.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:31.826 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:31.856 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:32.086 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:29:32.097 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:32.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:32.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:32.203 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:32.222 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:29:32.230 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:32.272 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:32.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:32.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:32.359 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:32 smithi138 ceph-mon[192513]: failed to encode map e330 with expected crc 2023-10-01T09:29:32.360 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:32 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:29:32.360 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:32 smithi138 ceph-mon[192513]: osdmap e330: 8 total, 8 up, 7 in 2023-10-01T09:29:32.360 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:32 smithi138 ceph-mon[192513]: pgmap v896: 89 pgs: 89 active+clean; 4.6 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:32.377 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:32.402 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:32.444 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:32.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:32.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:32.550 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:29:32.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:32 smithi190 ceph-mon[160435]: failed to encode map e330 with expected crc 2023-10-01T09:29:32.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:32 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:29:32.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:32 smithi190 ceph-mon[160435]: osdmap e330: 8 total, 8 up, 7 in 2023-10-01T09:29:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:32 smithi190 ceph-mon[160435]: pgmap v896: 89 pgs: 89 active+clean; 4.6 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:32.575 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:32.618 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:29:32.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:32.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:29:32.724 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:32.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:32 smithi138 ceph-mon[185886]: failed to encode map e330 with expected crc 2023-10-01T09:29:32.748 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:32 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:29:32.748 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:32 smithi138 ceph-mon[185886]: osdmap e330: 8 total, 8 up, 7 in 2023-10-01T09:29:32.748 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:32 smithi138 ceph-mon[185886]: pgmap v896: 89 pgs: 89 active+clean; 4.6 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail 2023-10-01T09:29:32.750 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:33.009 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:33.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:33.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:33.076 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:33.105 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:33.113 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:33.144 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:33.162 DEBUG:teuthology.orchestra.run.smithi138:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-10-01T09:29:33.164 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T09:29:33.164 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T09:29:33.348 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:hints = 1 2023-10-01T09:29:33.348 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T09:29:33.348 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Object prefix: benchmark_data_smithi138_431617 2023-10-01T09:29:33.349 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:29:33.349 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 0 0 0 0 0 0 - 0 2023-10-01T09:29:33.376 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:33.378 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:33.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:33.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:33.490 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:29:33.601 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:33.693 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:33 smithi138 ceph-mon[192513]: failed to encode map e330 with expected crc 2023-10-01T09:29:33.693 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:33 smithi138 ceph-mon[192513]: failed to encode map e330 with expected crc 2023-10-01T09:29:33.693 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:33 smithi138 ceph-mon[185886]: failed to encode map e330 with expected crc 2023-10-01T09:29:33.693 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:33 smithi138 ceph-mon[185886]: failed to encode map e330 with expected crc 2023-10-01T09:29:33.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:33 smithi190 ceph-mon[160435]: failed to encode map e330 with expected crc 2023-10-01T09:29:33.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:33 smithi190 ceph-mon[160435]: failed to encode map e330 with expected crc 2023-10-01T09:29:34.349 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 1 16 2844 2828 176.719 176.75 0.00401456 0.00564251 2023-10-01T09:29:34.379 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:34 smithi138 ceph-mon[185886]: failed to encode map e330 with expected crc 2023-10-01T09:29:34.379 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:34 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1334022864' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:29:34.379 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:34 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:29:34.380 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:34 smithi138 ceph-mon[185886]: pgmap v897: 89 pgs: 89 active+clean; 4.6 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2023-10-01T09:29:34.382 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:34 smithi138 ceph-mon[192513]: failed to encode map e330 with expected crc 2023-10-01T09:29:34.383 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:34 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1334022864' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:29:34.383 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:34 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:29:34.383 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:34 smithi138 ceph-mon[192513]: pgmap v897: 89 pgs: 89 active+clean; 4.6 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2023-10-01T09:29:34.402 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.7 to 1 (10000) 2023-10-01T09:29:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:34 smithi190 ceph-mon[160435]: failed to encode map e330 with expected crc 2023-10-01T09:29:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:34 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1334022864' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:29:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:34 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:29:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:34 smithi190 ceph-mon[160435]: pgmap v897: 89 pgs: 89 active+clean; 4.6 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2023-10-01T09:29:35.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:29:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:29:34] "GET /metrics HTTP/1.1" 200 36332 "" "Prometheus/2.33.4" 2023-10-01T09:29:35.348 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 2 16 5895 5879 183.695 190.688 0.00438372 0.00543427 2023-10-01T09:29:35.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:35 smithi190 ceph-mon[160435]: failed to encode map e331 with expected crc 2023-10-01T09:29:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:35 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:29:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:35 smithi190 ceph-mon[160435]: osdmap e331: 8 total, 8 up, 7 in 2023-10-01T09:29:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:35 smithi190 ceph-mon[160435]: failed to encode map e331 with expected crc 2023-10-01T09:29:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:35 smithi190 ceph-mon[160435]: failed to encode map e331 with expected crc 2023-10-01T09:29:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:35 smithi190 ceph-mon[160435]: failed to encode map e331 with expected crc 2023-10-01T09:29:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:35 smithi190 ceph-mon[160435]: failed to encode map e331 with expected crc 2023-10-01T09:29:35.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:35 smithi138 ceph-mon[185886]: failed to encode map e331 with expected crc 2023-10-01T09:29:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:35 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:29:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:35 smithi138 ceph-mon[185886]: osdmap e331: 8 total, 8 up, 7 in 2023-10-01T09:29:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:35 smithi138 ceph-mon[185886]: failed to encode map e331 with expected crc 2023-10-01T09:29:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:35 smithi138 ceph-mon[185886]: failed to encode map e331 with expected crc 2023-10-01T09:29:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:35 smithi138 ceph-mon[185886]: failed to encode map e331 with expected crc 2023-10-01T09:29:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:35 smithi138 ceph-mon[185886]: failed to encode map e331 with expected crc 2023-10-01T09:29:35.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:35 smithi138 ceph-mon[192513]: failed to encode map e331 with expected crc 2023-10-01T09:29:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:35 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:29:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:35 smithi138 ceph-mon[192513]: osdmap e331: 8 total, 8 up, 7 in 2023-10-01T09:29:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:35 smithi138 ceph-mon[192513]: failed to encode map e331 with expected crc 2023-10-01T09:29:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:35 smithi138 ceph-mon[192513]: failed to encode map e331 with expected crc 2023-10-01T09:29:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:35 smithi138 ceph-mon[192513]: failed to encode map e331 with expected crc 2023-10-01T09:29:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:35 smithi138 ceph-mon[192513]: failed to encode map e331 with expected crc 2023-10-01T09:29:36.351 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 3 16 8586 8570 178.41 168.188 0.00814307 0.0055937 2023-10-01T09:29:36.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:36 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:36.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:36 smithi138 ceph-mon[192513]: pgmap v899: 89 pgs: 89 active+clean; 102 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 13 MiB/s wr, 205 op/s 2023-10-01T09:29:36.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:36.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:36 smithi138 ceph-mon[185886]: pgmap v899: 89 pgs: 89 active+clean; 102 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 13 MiB/s wr, 205 op/s 2023-10-01T09:29:37.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:37.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:36 smithi190 ceph-mon[160435]: pgmap v899: 89 pgs: 89 active+clean; 102 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 13 MiB/s wr, 205 op/s 2023-10-01T09:29:37.350 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 4 16 11616 11600 181.146 189.375 0.00715324 0.00551101 2023-10-01T09:29:37.657 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:29:37.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:37.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:29:37.764 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:37.821 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:37.886 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:37.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:37.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:37.992 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:29:38.183 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:38.350 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 5 16 14362 14346 179.239 171.625 0.00502318 0.00557344 2023-10-01T09:29:38.500 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:29:38.501 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:38 smithi138 ceph-mon[192513]: pgmap v900: 89 pgs: 89 active+clean; 251 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 31 MiB/s wr, 492 op/s 2023-10-01T09:29:38.502 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:38 smithi138 ceph-mon[185886]: pgmap v900: 89 pgs: 89 active+clean; 251 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 31 MiB/s wr, 492 op/s 2023-10-01T09:29:38.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:38.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:29:38.603 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:38.630 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:38.672 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:38.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:38.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:38.776 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:38 smithi190 ceph-mon[160435]: pgmap v900: 89 pgs: 89 active+clean; 251 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 31 MiB/s wr, 492 op/s 2023-10-01T09:29:38.829 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:39.351 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 6 16 16365 16349 170.221 125.188 0.00687687 0.00586841 2023-10-01T09:29:39.427 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:39.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:39.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:39.532 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:39.561 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:39.630 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:39.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:39.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:39.734 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:29:39.843 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:39.863 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-01T09:29:40.156 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:39 smithi138 ceph-mon[185886]: pgmap v901: 89 pgs: 89 active+clean; 691 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-01T09:29:40.156 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:39 smithi138 ceph-mon[192513]: pgmap v901: 89 pgs: 89 active+clean; 691 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-01T09:29:40.160 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:29:40.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:40.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:29:40.268 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:40.300 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:39 smithi190 ceph-mon[160435]: pgmap v901: 89 pgs: 89 active+clean; 691 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 80 MiB/s wr, 1.29k op/s 2023-10-01T09:29:40.303 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:40.351 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 7 16 18262 18246 162.842 118.562 0.00977955 0.00613344 2023-10-01T09:29:40.362 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:40.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:40.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:40.468 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:29:40.491 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:29:40.491 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:29:40.544 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:41.076 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:29:41.076 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:29:41.076 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:29:41.103 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:29:41.137 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:29:41.156 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:29:41.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:41.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:29:41.176 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:29:41.213 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:29:41.246 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:29:41.258 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:29:41.258 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:29:41.258 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:29:41.258 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:29:41.258 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:29:41.259 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:29:41.259 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:29:41.259 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:29:41.262 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:41.320 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:41.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:41.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:29:41.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:41 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1732053657' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:29:41.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:41 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/408151201' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:29:41.343 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:41 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:29:41.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:41.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:29:41.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:41 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1732053657' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:29:41.344 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:41 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/408151201' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:29:41.344 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:41 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:29:41.351 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 8 16 19983 19967 155.933 107.562 0.004716 0.00637839 2023-10-01T09:29:41.355 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:29:41.379 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:41.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:41.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:41.486 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:41.526 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:41.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:29:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:41 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1732053657' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:29:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:41 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/408151201' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:29:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:41 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:29:42.351 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 9 16 21921 21905 152.065 121.125 0.00482308 0.00657064 2023-10-01T09:29:42.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:42 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:29:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:42 smithi190 ceph-mon[160435]: failed to encode map e332 with expected crc 2023-10-01T09:29:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:42 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:29:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:42 smithi190 ceph-mon[160435]: osdmap e332: 8 total, 8 up, 7 in 2023-10-01T09:29:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:42 smithi190 ceph-mon[160435]: failed to encode map e332 with expected crc 2023-10-01T09:29:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:42 smithi190 ceph-mon[160435]: failed to encode map e332 with expected crc 2023-10-01T09:29:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:42 smithi190 ceph-mon[160435]: pgmap v903: 89 pgs: 89 active+clean; 1012 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-01T09:29:42.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:42 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:29:42.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:42 smithi138 ceph-mon[192513]: failed to encode map e332 with expected crc 2023-10-01T09:29:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:42 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:29:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:42 smithi138 ceph-mon[192513]: osdmap e332: 8 total, 8 up, 7 in 2023-10-01T09:29:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:42 smithi138 ceph-mon[192513]: failed to encode map e332 with expected crc 2023-10-01T09:29:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:42 smithi138 ceph-mon[192513]: failed to encode map e332 with expected crc 2023-10-01T09:29:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:42 smithi138 ceph-mon[192513]: pgmap v903: 89 pgs: 89 active+clean; 1012 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-01T09:29:42.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:42 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:29:42.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:42 smithi138 ceph-mon[185886]: failed to encode map e332 with expected crc 2023-10-01T09:29:42.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:42 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:29:42.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:42 smithi138 ceph-mon[185886]: osdmap e332: 8 total, 8 up, 7 in 2023-10-01T09:29:42.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:42 smithi138 ceph-mon[185886]: failed to encode map e332 with expected crc 2023-10-01T09:29:42.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:42 smithi138 ceph-mon[185886]: failed to encode map e332 with expected crc 2023-10-01T09:29:42.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:42 smithi138 ceph-mon[185886]: pgmap v903: 89 pgs: 89 active+clean; 1012 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-01T09:29:43.352 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 10 16 23523 23507 146.871 100.125 0.00727224 0.0068022 2023-10-01T09:29:43.432 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:29:43.512 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:43.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:43.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:43.625 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:43.696 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:43.992 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:43 smithi138 ceph-mon[185886]: pgmap v904: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-10-01T09:29:44.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:43 smithi190 ceph-mon[160435]: pgmap v904: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-10-01T09:29:44.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:43 smithi138 ceph-mon[192513]: pgmap v904: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-10-01T09:29:44.324 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:44.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:44.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:44.352 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 11 16 25331 25315 143.791 113 0.00637194 0.00694908 2023-10-01T09:29:44.371 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:44.434 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:44.464 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:44.521 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:44.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:44.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:44.598 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:44.627 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:44.664 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:44.722 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:44.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:44.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:44.828 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:44.866 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:44.931 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:44.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:44.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:45.036 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:45.042 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:29:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:29:44] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-01T09:29:45.103 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:45.352 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 12 16 27144 27128 141.251 113.312 0.00549974 0.00707364 2023-10-01T09:29:45.825 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:45.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:45.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:45.932 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:29:46.009 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:46.352 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 13 16 28707 28691 137.9 97.6875 0.00188302 0.00724369 2023-10-01T09:29:46.353 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:29:46.406 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:29:47.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:46 smithi190 ceph-mon[160435]: pgmap v905: 89 pgs: 89 active+clean; 1.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-01T09:29:47.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:46 smithi138 ceph-mon[192513]: pgmap v905: 89 pgs: 89 active+clean; 1.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-01T09:29:47.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:46 smithi138 ceph-mon[185886]: pgmap v905: 89 pgs: 89 active+clean; 1.5 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-10-01T09:29:47.352 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 14 16 29979 29963 133.729 79.5 0.00290657 0.00745197 2023-10-01T09:29:47.738 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:29:47.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:47.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:29:47.847 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:47.946 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:48.294 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:48.315 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:48.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:48.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:48.355 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 15 16 31359 31343 130.564 86.25 0.00689824 0.00764779 2023-10-01T09:29:48.365 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:48.431 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:48.525 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:48 smithi138 ceph-mon[185886]: pgmap v906: 89 pgs: 89 active+clean; 1.6 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 137 MiB/s wr, 2.18k op/s 2023-10-01T09:29:48.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:48 smithi138 ceph-mon[192513]: pgmap v906: 89 pgs: 89 active+clean; 1.6 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 137 MiB/s wr, 2.18k op/s 2023-10-01T09:29:48.530 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:29:48.530 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:29:48.530 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:29:48.530 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:29:48.530 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:29:48.532 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:48.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:48 smithi190 ceph-mon[160435]: pgmap v906: 89 pgs: 89 active+clean; 1.6 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 137 MiB/s wr, 2.18k op/s 2023-10-01T09:29:49.352 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 16 16 32881 32865 128.349 95.125 0.00783572 0.00778623 2023-10-01T09:29:49.376 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1520787654' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:29:49.391 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.2 to 1 (10000) 2023-10-01T09:29:49.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1520787654' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:29:49.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1520787654' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:29:50.062 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:29:50.088 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:50.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:50.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:50.194 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:29:50.225 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:50.278 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:29:50.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:50.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:29:50.352 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 17 16 34511 34495 126.791 101.875 0.0176758 0.00788059 2023-10-01T09:29:50.385 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:29:50.418 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:50.481 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:29:50.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:50.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:29:50.486 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:50 smithi190 ceph-mon[160435]: failed to encode map e333 with expected crc 2023-10-01T09:29:50.486 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:50 smithi190 ceph-mon[160435]: failed to encode map e333 with expected crc 2023-10-01T09:29:50.486 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:50 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1520787654' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:29:50.486 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:50 smithi190 ceph-mon[160435]: osdmap e333: 8 total, 8 up, 7 in 2023-10-01T09:29:50.487 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:50 smithi190 ceph-mon[160435]: pgmap v908: 89 pgs: 89 active+clean; 1.8 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T09:29:50.586 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:50.622 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:50.678 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:50.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:50.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:50.783 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:29:50.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:50 smithi138 ceph-mon[192513]: failed to encode map e333 with expected crc 2023-10-01T09:29:50.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:50 smithi138 ceph-mon[192513]: failed to encode map e333 with expected crc 2023-10-01T09:29:50.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:50 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1520787654' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:29:50.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:50 smithi138 ceph-mon[192513]: osdmap e333: 8 total, 8 up, 7 in 2023-10-01T09:29:50.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:50 smithi138 ceph-mon[192513]: pgmap v908: 89 pgs: 89 active+clean; 1.8 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T09:29:50.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:50 smithi138 ceph-mon[185886]: failed to encode map e333 with expected crc 2023-10-01T09:29:50.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:50 smithi138 ceph-mon[185886]: failed to encode map e333 with expected crc 2023-10-01T09:29:50.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:50 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1520787654' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:29:50.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:50 smithi138 ceph-mon[185886]: osdmap e333: 8 total, 8 up, 7 in 2023-10-01T09:29:50.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:50 smithi138 ceph-mon[185886]: pgmap v908: 89 pgs: 89 active+clean; 1.8 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T09:29:50.847 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:51.230 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:29:51.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:51.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:29:51.338 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:29:51.352 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 18 16 36066 36050 125.146 97.1875 0.0129529 0.00798511 2023-10-01T09:29:51.377 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:51.440 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:29:51.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:51.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:29:51.507 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-01T09:29:51.546 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:51.647 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:52.352 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 19 16 37107 37091 121.984 65.0625 0.00295043 0.00814335 2023-10-01T09:29:52.603 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:52.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:52.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:52.711 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:52.779 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:52.940 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:52.945 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:52 smithi138 ceph-mon[185886]: pgmap v909: 89 pgs: 89 active+clean; 2.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-01T09:29:52.945 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:52 smithi138 ceph-mon[192513]: pgmap v909: 89 pgs: 89 active+clean; 2.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-01T09:29:52.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:52.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:53.048 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:29:53.049 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:53.073 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:29:53.085 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:53.142 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:29:53.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:53.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:29:53.249 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:53.282 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:52 smithi190 ceph-mon[160435]: pgmap v909: 89 pgs: 89 active+clean; 2.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-01T09:29:53.285 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:53.339 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:53.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:53.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:53.352 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:29:53.352967+0000 min lat: 0.00115441 max lat: 0.234182 avg lat: 0.00830525 2023-10-01T09:29:53.352 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:29:53.353 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 20 16 38518 38502 120.294 88.1875 0.00208134 0.00830525 2023-10-01T09:29:53.446 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:29:53.478 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:53.540 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:29:53.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:53.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:29:53.646 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:29:53.678 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:53.727 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:29:53.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:53.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:29:53.827 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:29:53.834 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:53.851 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:29:53.889 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:29:53.939 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:53.957 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:29:53.985 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:29:54.033 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:29:54.070 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:29:54.101 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:29:54.110 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:29:54.110 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:29:54.111 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:29:54.111 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:29:54.111 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:29:54.111 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:29:54.111 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:29:54.111 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:29:54.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1454093588' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:29:54.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:53 smithi190 ceph-mon[160435]: pgmap v910: 89 pgs: 89 active+clean; 2.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-01T09:29:54.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1454093588' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:29:54.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:53 smithi138 ceph-mon[185886]: pgmap v910: 89 pgs: 89 active+clean; 2.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-01T09:29:54.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1454093588' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:29:54.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:53 smithi138 ceph-mon[192513]: pgmap v910: 89 pgs: 89 active+clean; 2.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-01T09:29:54.352 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 21 16 39798 39782 118.375 80 0.00536029 0.00844272 2023-10-01T09:29:54.484 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:54.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:54.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:54.591 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:54.618 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:54.659 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:54.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:54.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:54.765 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:29:54.822 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:54.963 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:54 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4107136996' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:54.978 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:29:55.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:54 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4107136996' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:55.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:54 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:55.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:54 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4107136996' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:55.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:54 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:55.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:29:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:29:54] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-01T09:29:55.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:54 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:29:55.353 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 22 16 40839 40823 115.952 65.0625 0.0082316 0.00861862 2023-10-01T09:29:56.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:55 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:29:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:55 smithi190 ceph-mon[160435]: failed to encode map e334 with expected crc 2023-10-01T09:29:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:55 smithi190 ceph-mon[160435]: failed to encode map e334 with expected crc 2023-10-01T09:29:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:55 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:29:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:55 smithi190 ceph-mon[160435]: osdmap e334: 8 total, 8 up, 7 in 2023-10-01T09:29:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:55 smithi190 ceph-mon[160435]: failed to encode map e334 with expected crc 2023-10-01T09:29:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:55 smithi190 ceph-mon[160435]: failed to encode map e334 with expected crc 2023-10-01T09:29:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:29:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:55 smithi190 ceph-mon[160435]: pgmap v912: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-01T09:29:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:29:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[192513]: failed to encode map e334 with expected crc 2023-10-01T09:29:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[192513]: failed to encode map e334 with expected crc 2023-10-01T09:29:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:29:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[192513]: osdmap e334: 8 total, 8 up, 7 in 2023-10-01T09:29:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[192513]: failed to encode map e334 with expected crc 2023-10-01T09:29:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[192513]: failed to encode map e334 with expected crc 2023-10-01T09:29:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:29:56.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[192513]: pgmap v912: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-01T09:29:56.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:29:56.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[185886]: failed to encode map e334 with expected crc 2023-10-01T09:29:56.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[185886]: failed to encode map e334 with expected crc 2023-10-01T09:29:56.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:29:56.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[185886]: osdmap e334: 8 total, 8 up, 7 in 2023-10-01T09:29:56.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[185886]: failed to encode map e334 with expected crc 2023-10-01T09:29:56.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[185886]: failed to encode map e334 with expected crc 2023-10-01T09:29:56.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:29:56.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:29:56.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:55 smithi138 ceph-mon[185886]: pgmap v912: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-01T09:29:56.352 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 23 16 42132 42116 114.424 80.8125 0.00285445 0.00870689 2023-10-01T09:29:56.460 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:29:56.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:56.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:29:56.568 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:56.607 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:57.353 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 24 16 43339 43323 112.799 75.4375 0.0166346 0.00886091 2023-10-01T09:29:57.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:57 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-01T09:29:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:57 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]: dispatch 2023-10-01T09:29:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:57 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]: dispatch 2023-10-01T09:29:57.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:57 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-01T09:29:57.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:57 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]: dispatch 2023-10-01T09:29:57.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:57 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]: dispatch 2023-10-01T09:29:57.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:57 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-10-01T09:29:57.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:57 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]: dispatch 2023-10-01T09:29:57.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:57 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]: dispatch 2023-10-01T09:29:57.769 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:57.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:57.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:57.877 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:57.909 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:57.974 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:57.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:57.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:58.082 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:58.159 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:58.353 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 25 16 44677 44661 111.632 83.625 0.0110038 0.00895259 2023-10-01T09:29:58.432 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:58.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:58.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:58.441 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:58 smithi138 ceph-mon[192513]: failed to encode map e335 with expected crc 2023-10-01T09:29:58.441 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-01T09:29:58.441 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]': finished 2023-10-01T09:29:58.441 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2023-10-01T09:29:58.441 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:58 smithi138 ceph-mon[192513]: osdmap e335: 8 total, 8 up, 7 in 2023-10-01T09:29:58.441 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:58 smithi138 ceph-mon[192513]: failed to encode map e335 with expected crc 2023-10-01T09:29:58.441 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:58 smithi138 ceph-mon[192513]: pgmap v914: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-01T09:29:58.442 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:58 smithi138 ceph-mon[185886]: failed to encode map e335 with expected crc 2023-10-01T09:29:58.442 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-01T09:29:58.442 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]': finished 2023-10-01T09:29:58.442 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2023-10-01T09:29:58.442 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:58 smithi138 ceph-mon[185886]: osdmap e335: 8 total, 8 up, 7 in 2023-10-01T09:29:58.442 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:58 smithi138 ceph-mon[185886]: failed to encode map e335 with expected crc 2023-10-01T09:29:58.442 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:58 smithi138 ceph-mon[185886]: pgmap v914: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-01T09:29:58.502 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:29:58.502 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:29:58.541 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:58.609 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:58.736 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:58.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:58.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:58.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:58 smithi190 ceph-mon[160435]: failed to encode map e335 with expected crc 2023-10-01T09:29:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-10-01T09:29:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]': finished 2023-10-01T09:29:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2023-10-01T09:29:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:58 smithi190 ceph-mon[160435]: osdmap e335: 8 total, 8 up, 7 in 2023-10-01T09:29:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:58 smithi190 ceph-mon[160435]: failed to encode map e335 with expected crc 2023-10-01T09:29:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:58 smithi190 ceph-mon[160435]: pgmap v914: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-01T09:29:58.847 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:29:58.874 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:58.945 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:29:58.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:58.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:29:59.051 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:59.112 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:59.201 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:59.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:59.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:29:59.312 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:59.351 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:59.353 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 26 16 45811 45795 110.064 70.875 0.0080624 0.00908105 2023-10-01T09:29:59.392 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.5 to 1 (10000) 2023-10-01T09:29:59.421 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:59 smithi138 ceph-mon[192513]: failed to encode map e336 with expected crc 2023-10-01T09:29:59.421 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:59 smithi138 ceph-mon[192513]: failed to encode map e336 with expected crc 2023-10-01T09:29:59.421 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:59 smithi138 ceph-mon[192513]: osdmap e336: 8 total, 8 up, 7 in 2023-10-01T09:29:59.421 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:59 smithi138 ceph-mon[192513]: failed to encode map e336 with expected crc 2023-10-01T09:29:59.421 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:29:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4103663338' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:29:59.422 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:59 smithi138 ceph-mon[185886]: failed to encode map e336 with expected crc 2023-10-01T09:29:59.422 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:59 smithi138 ceph-mon[185886]: failed to encode map e336 with expected crc 2023-10-01T09:29:59.422 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:59 smithi138 ceph-mon[185886]: osdmap e336: 8 total, 8 up, 7 in 2023-10-01T09:29:59.422 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:59 smithi138 ceph-mon[185886]: failed to encode map e336 with expected crc 2023-10-01T09:29:59.422 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:29:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4103663338' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:29:59.423 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:59.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:59.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:59.528 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:29:59.568 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:59.624 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:29:59.624 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:29:59.627 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:59 smithi190 ceph-mon[160435]: failed to encode map e336 with expected crc 2023-10-01T09:29:59.627 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:59 smithi190 ceph-mon[160435]: failed to encode map e336 with expected crc 2023-10-01T09:29:59.627 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:59 smithi190 ceph-mon[160435]: osdmap e336: 8 total, 8 up, 7 in 2023-10-01T09:29:59.627 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:59 smithi190 ceph-mon[160435]: failed to encode map e336 with expected crc 2023-10-01T09:29:59.627 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:29:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4103663338' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:29:59.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:59.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:29:59.729 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:29:59.832 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:29:59.854 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:29:59.935 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:29:59.949 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:29:59.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:29:59.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:00.055 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:00.081 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:00.357 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 27 16 46848 46832 108.373 64.8125 0.0106915 0.00922242 2023-10-01T09:30:00.498 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:00.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:00.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:00.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:00 smithi138 ceph-mon[192513]: failed to encode map e336 with expected crc 2023-10-01T09:30:00.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:00 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4103663338' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:30:00.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:00 smithi138 ceph-mon[192513]: osdmap e337: 8 total, 8 up, 7 in 2023-10-01T09:30:00.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:00 smithi138 ceph-mon[192513]: failed to encode map e337 with expected crc 2023-10-01T09:30:00.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:00 smithi138 ceph-mon[192513]: pgmap v917: 89 pgs: 1 peering, 88 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-01T09:30:00.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:00 smithi138 ceph-mon[192513]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-10-01T09:30:00.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:00 smithi138 ceph-mon[192513]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-01T09:30:00.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:00 smithi138 ceph-mon[185886]: failed to encode map e336 with expected crc 2023-10-01T09:30:00.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:00 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4103663338' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:30:00.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:00 smithi138 ceph-mon[185886]: osdmap e337: 8 total, 8 up, 7 in 2023-10-01T09:30:00.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:00 smithi138 ceph-mon[185886]: failed to encode map e337 with expected crc 2023-10-01T09:30:00.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:00 smithi138 ceph-mon[185886]: pgmap v917: 89 pgs: 1 peering, 88 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-01T09:30:00.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:00 smithi138 ceph-mon[185886]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-10-01T09:30:00.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:00 smithi138 ceph-mon[185886]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-01T09:30:00.605 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:00.621 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-01T09:30:00.633 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:00 smithi190 ceph-mon[160435]: failed to encode map e336 with expected crc 2023-10-01T09:30:00.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:00 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4103663338' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:30:00.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:00 smithi190 ceph-mon[160435]: osdmap e337: 8 total, 8 up, 7 in 2023-10-01T09:30:00.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:00 smithi190 ceph-mon[160435]: failed to encode map e337 with expected crc 2023-10-01T09:30:00.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:00 smithi190 ceph-mon[160435]: pgmap v917: 89 pgs: 1 peering, 88 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-01T09:30:00.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:00 smithi190 ceph-mon[160435]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-10-01T09:30:00.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:00 smithi190 ceph-mon[160435]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-10-01T09:30:00.637 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:00.699 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:00.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:00.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:00.784 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:00.805 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:00.841 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:00.865 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:00.971 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:01.138 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:01.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:01.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:01.245 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:01.288 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:01.349 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:01.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:01.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:01.358 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 28 16 48027 48011 107.135 73.6875 0.00734704 0.00932812 2023-10-01T09:30:01.455 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:01.496 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:01.551 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:01.555 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:01 smithi190 ceph-mon[160435]: failed to encode map e338 with expected crc 2023-10-01T09:30:01.555 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:01 smithi190 ceph-mon[160435]: failed to encode map e338 with expected crc 2023-10-01T09:30:01.555 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:01 smithi190 ceph-mon[160435]: osdmap e338: 8 total, 8 up, 7 in 2023-10-01T09:30:01.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:01.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:01.656 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:01.657 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:01 smithi138 ceph-mon[185886]: failed to encode map e338 with expected crc 2023-10-01T09:30:01.657 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:01 smithi138 ceph-mon[185886]: failed to encode map e338 with expected crc 2023-10-01T09:30:01.658 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:01 smithi138 ceph-mon[185886]: osdmap e338: 8 total, 8 up, 7 in 2023-10-01T09:30:01.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:01 smithi138 ceph-mon[192513]: failed to encode map e338 with expected crc 2023-10-01T09:30:01.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:01 smithi138 ceph-mon[192513]: failed to encode map e338 with expected crc 2023-10-01T09:30:01.659 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:01 smithi138 ceph-mon[192513]: osdmap e338: 8 total, 8 up, 7 in 2023-10-01T09:30:01.752 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:02.358 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 29 16 49266 49250 106.105 77.4375 0.0143462 0.00941985 2023-10-01T09:30:02.520 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:02.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:02.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:02.628 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:02.691 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:02 smithi138 ceph-mon[185886]: failed to encode map e338 with expected crc 2023-10-01T09:30:02.692 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:02 smithi138 ceph-mon[185886]: pgmap v919: 89 pgs: 1 peering, 88 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-01T09:30:02.692 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:02 smithi138 ceph-mon[192513]: failed to encode map e338 with expected crc 2023-10-01T09:30:02.692 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:02 smithi138 ceph-mon[192513]: pgmap v919: 89 pgs: 1 peering, 88 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-01T09:30:02.694 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:02.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:02 smithi190 ceph-mon[160435]: failed to encode map e338 with expected crc 2023-10-01T09:30:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:02 smithi190 ceph-mon[160435]: pgmap v919: 89 pgs: 1 peering, 88 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-01T09:30:03.358 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 30 16 50539 50523 105.221 79.5625 0.0145602 0.00949915 2023-10-01T09:30:04.359 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 31 16 51554 51538 103.873 63.4375 0.00234494 0.00961114 2023-10-01T09:30:04.719 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:04.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:04.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:04.788 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:04.835 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:04.864 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:04.920 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:04.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:04.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:05.032 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:05.062 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:05.122 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:05.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:05.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:05.132 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:04 smithi190 ceph-mon[160435]: pgmap v920: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:05.207 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:04 smithi138 ceph-mon[192513]: pgmap v920: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:05.207 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:04 smithi138 ceph-mon[185886]: pgmap v920: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:05.211 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:30:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:30:04] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-10-01T09:30:05.232 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:05.321 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:05.359 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 32 16 52697 52681 102.859 71.4375 0.00976624 0.00971472 2023-10-01T09:30:05.371 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:30:05.371 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:30:05.992 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:30:06.116 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:06.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:06.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:06.228 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:06.253 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/486024424' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:30:06.254 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:05 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:30:06.256 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:06.297 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:06.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:06.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/486024424' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:30:06.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:05 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:30:06.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/486024424' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:30:06.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:05 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:30:06.359 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 33 16 53745 53729 101.727 65.5 0.00929555 0.00982454 2023-10-01T09:30:06.403 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:30:06.747 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:07.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:06 smithi138 ceph-mon[192513]: pgmap v921: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 61 MiB/s wr, 977 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:07.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:06 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:30:07.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:06 smithi138 ceph-mon[192513]: failed to encode map e339 with expected crc 2023-10-01T09:30:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:06 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:30:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:06 smithi138 ceph-mon[192513]: osdmap e339: 8 total, 8 up, 7 in 2023-10-01T09:30:07.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:06 smithi138 ceph-mon[192513]: failed to encode map e339 with expected crc 2023-10-01T09:30:07.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:06 smithi138 ceph-mon[185886]: pgmap v921: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 61 MiB/s wr, 977 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:07.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:06 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:30:07.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:06 smithi138 ceph-mon[185886]: failed to encode map e339 with expected crc 2023-10-01T09:30:07.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:06 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:30:07.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:06 smithi138 ceph-mon[185886]: osdmap e339: 8 total, 8 up, 7 in 2023-10-01T09:30:07.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:06 smithi138 ceph-mon[185886]: failed to encode map e339 with expected crc 2023-10-01T09:30:07.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:06 smithi190 ceph-mon[160435]: pgmap v921: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 61 MiB/s wr, 977 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:06 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:30:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:06 smithi190 ceph-mon[160435]: failed to encode map e339 with expected crc 2023-10-01T09:30:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:06 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:30:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:06 smithi190 ceph-mon[160435]: osdmap e339: 8 total, 8 up, 7 in 2023-10-01T09:30:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:06 smithi190 ceph-mon[160435]: failed to encode map e339 with expected crc 2023-10-01T09:30:07.360 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 34 16 55044 55028 101.119 81.1875 0.0137799 0.00988453 2023-10-01T09:30:07.389 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:30:07.390 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:30:07.390 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:30:07.390 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:30:07.390 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:30:07.390 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:30:07.394 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:30:07.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:07.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:30:07.417 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:30:07.453 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:30:07.467 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:30:07.467 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:30:07.467 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:30:07.467 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:30:07.467 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:30:07.468 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:30:07.468 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:30:07.468 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:30:07.496 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:30:07.504 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:07.598 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:08.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:07 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2930529123' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:08.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:07 smithi138 ceph-mon[185886]: pgmap v923: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:08.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:07 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2930529123' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:08.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:07 smithi138 ceph-mon[192513]: pgmap v923: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:08.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:07 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2930529123' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:07 smithi190 ceph-mon[160435]: pgmap v923: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:08.360 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 35 16 56033 56017 99.9962 61.8125 0.0116396 0.00999553 2023-10-01T09:30:09.026 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:09.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:09.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:09.133 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:09.163 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:09.218 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:09.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:09.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:09.324 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:09.360 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 36 16 57244 57228 99.3209 75.6875 0.00982139 0.0100622 2023-10-01T09:30:09.414 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:09.554 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:09.617 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:09.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:09.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:09.723 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:09.755 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:09.807 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:09.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:09.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:09.915 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:09.980 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:10.087 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:10.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:10.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:10.196 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:10.231 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:10.295 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:10.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:10.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:10.360 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 37 16 58364 58348 98.5285 70 0.0118206 0.0101439 2023-10-01T09:30:10.401 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:10.438 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:10.505 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:10.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:10.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:10.614 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:10.652 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:10.708 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:10.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:10.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:10.816 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:10.896 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:10.993 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.6 to 1 (10000) 2023-10-01T09:30:11.147 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:30:11.189 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:30:11.203 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:10 smithi138 ceph-mon[192513]: pgmap v924: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:11.203 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:10 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/912457080' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:30:11.203 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:30:11.203 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:11.204 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:10 smithi138 ceph-mon[185886]: pgmap v924: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:11.204 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:10 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/912457080' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:30:11.204 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:30:11.204 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:11.263 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:11.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:11.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:11.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:10 smithi190 ceph-mon[160435]: pgmap v924: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:11.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:10 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/912457080' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:30:11.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:30:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:11.360 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 38 16 59602 59586 97.9718 77.375 0.00245475 0.0101983 2023-10-01T09:30:11.378 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:11.403 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:11.448 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:11.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:11.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:11.558 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:11.626 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:11.960 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:11.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:11.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:12.068 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:12.083 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-01T09:30:12.083 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:30:12.179 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:12.290 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:12.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:12.300 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:11 smithi138 ceph-mon[185886]: failed to encode map e340 with expected crc 2023-10-01T09:30:12.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/912457080' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:30:12.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:11 smithi138 ceph-mon[185886]: osdmap e340: 8 total, 8 up, 7 in 2023-10-01T09:30:12.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:11 smithi138 ceph-mon[185886]: failed to encode map e340 with expected crc 2023-10-01T09:30:12.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:11 smithi138 ceph-mon[185886]: pgmap v926: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-01T09:30:12.301 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:11 smithi138 ceph-mon[192513]: failed to encode map e340 with expected crc 2023-10-01T09:30:12.301 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/912457080' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:30:12.301 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:11 smithi138 ceph-mon[192513]: osdmap e340: 8 total, 8 up, 7 in 2023-10-01T09:30:12.301 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:11 smithi138 ceph-mon[192513]: failed to encode map e340 with expected crc 2023-10-01T09:30:12.301 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:11 smithi138 ceph-mon[192513]: pgmap v926: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-01T09:30:12.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:12.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:11 smithi190 ceph-mon[160435]: failed to encode map e340 with expected crc 2023-10-01T09:30:12.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/912457080' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:30:12.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:11 smithi190 ceph-mon[160435]: osdmap e340: 8 total, 8 up, 7 in 2023-10-01T09:30:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:11 smithi190 ceph-mon[160435]: failed to encode map e340 with expected crc 2023-10-01T09:30:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:11 smithi190 ceph-mon[160435]: pgmap v926: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-01T09:30:12.362 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 39 16 60578 60562 97.0187 61 0.0141722 0.0103008 2023-10-01T09:30:12.403 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:12.443 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:12.496 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:12.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:12.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:12.538 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:12.539 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:12.601 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:12.683 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:30:12.683 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:30:12.683 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:30:12.683 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:30:12.683 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:30:12.700 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:13.363 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:30:13.363322+0000 min lat: 0.00115441 max lat: 0.275388 avg lat: 0.0103745 2023-10-01T09:30:13.363 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:30:13.363 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 40 16 61698 61682 96.3432 70 0.0127663 0.0103745 2023-10-01T09:30:14.087 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:14.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:14.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:14.195 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:14.229 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:14.282 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:14.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:14.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:14.324 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:30:14.363 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 41 16 62726 62710 95.5597 64.25 0.00746623 0.0104588 2023-10-01T09:30:14.387 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:14.421 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:14.502 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:14.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:14.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:14.615 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:14.642 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:14.790 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:15.256 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:14 smithi138 ceph-mon[185886]: pgmap v927: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-01T09:30:15.256 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:30:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:30:14] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.33.4" 2023-10-01T09:30:15.257 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:14 smithi138 ceph-mon[192513]: pgmap v927: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-01T09:30:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:14 smithi190 ceph-mon[160435]: pgmap v927: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-10-01T09:30:15.363 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 42 16 63622 63606 94.6179 56 0.00509485 0.0105628 2023-10-01T09:30:15.847 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:15.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:15.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:15.969 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:15.999 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:16.057 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:16.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:16.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:16.163 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:16.229 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:16.363 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 43 16 64541 64525 93.7534 57.4375 0.00753346 0.0106612 2023-10-01T09:30:16.540 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:16.563 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:16.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:16.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:16.676 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:16.704 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:16.747 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:16.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:16.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:16.852 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:16.881 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:16.937 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:30:16.939 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:16.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:16.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:16.993 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:30:17.029 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:30:17.047 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:17.048 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:30:17.087 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:30:17.118 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:30:17.132 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:17.150 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:30:17.187 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:30:17.218 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:30:17.218 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:30:17.218 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:30:17.218 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:30:17.218 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:30:17.218 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:30:17.218 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:30:17.219 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:30:17.235 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:16 smithi138 ceph-mon[192513]: pgmap v928: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T09:30:17.235 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:16 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/502391423' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:17.235 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:16 smithi138 ceph-mon[185886]: pgmap v928: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T09:30:17.235 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:16 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/502391423' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:17.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:16 smithi190 ceph-mon[160435]: pgmap v928: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T09:30:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:16 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/502391423' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:17.363 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 44 16 65470 65454 92.9423 58.0625 0.0123878 0.0107542 2023-10-01T09:30:17.970 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:17.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:17.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:18.013 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:30:18.081 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:18.111 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:18.154 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:18.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:18.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:18.259 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:18.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:17 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3620085984' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:18.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:17 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:18.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:17 smithi138 ceph-mon[185886]: pgmap v929: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T09:30:18.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:17 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:30:18.289 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:17 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3620085984' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:18.290 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:17 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:18.290 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:17 smithi138 ceph-mon[192513]: pgmap v929: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T09:30:18.290 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:17 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:30:18.291 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:18.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:17 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3620085984' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:18.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:17 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:18.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:17 smithi190 ceph-mon[160435]: pgmap v929: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T09:30:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:17 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:30:18.363 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 45 16 66445 66429 92.2312 60.9375 0.00704997 0.0108373 2023-10-01T09:30:18.979 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:18 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:30:18.979 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:18 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:30:18.979 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:18 smithi138 ceph-mon[185886]: failed to encode map e341 with expected crc 2023-10-01T09:30:18.979 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:18 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:30:18.979 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:18 smithi138 ceph-mon[185886]: osdmap e341: 8 total, 8 up, 7 in 2023-10-01T09:30:18.979 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:18 smithi138 ceph-mon[185886]: failed to encode map e341 with expected crc 2023-10-01T09:30:19.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:18 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:30:19.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:18 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:30:19.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:18 smithi190 ceph-mon[160435]: failed to encode map e341 with expected crc 2023-10-01T09:30:19.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:18 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:30:19.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:18 smithi190 ceph-mon[160435]: osdmap e341: 8 total, 8 up, 7 in 2023-10-01T09:30:19.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:18 smithi190 ceph-mon[160435]: failed to encode map e341 with expected crc 2023-10-01T09:30:19.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:18 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:30:19.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:18 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:30:19.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:18 smithi138 ceph-mon[192513]: failed to encode map e341 with expected crc 2023-10-01T09:30:19.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:18 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:30:19.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:18 smithi138 ceph-mon[192513]: osdmap e341: 8 total, 8 up, 7 in 2023-10-01T09:30:19.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:18 smithi138 ceph-mon[192513]: failed to encode map e341 with expected crc 2023-10-01T09:30:19.365 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 46 16 67509 67493 91.6717 66.5 0.00702249 0.0109041 2023-10-01T09:30:19.483 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:19.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:19.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:19.596 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:19.637 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:19.700 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:19.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:19.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:19.805 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:19.848 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:19.922 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:19.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:19.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:20.027 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:20.068 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:20.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:19 smithi138 ceph-mon[185886]: pgmap v931: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T09:30:20.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:19 smithi138 ceph-mon[192513]: pgmap v931: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T09:30:20.132 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:20.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:20.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:20.241 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:30:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:19 smithi190 ceph-mon[160435]: pgmap v931: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T09:30:20.331 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:20.364 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 47 16 68560 68544 91.1189 65.6875 0.00822817 0.0109693 2023-10-01T09:30:20.866 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:30:20.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:20.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:30:20.976 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:21.007 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:21.165 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:21.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:21.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:21.274 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:21.308 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:21.324 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:30:21.350 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:30:21.365 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 48 16 69534 69518 90.4855 60.875 0.0242817 0.0110449 2023-10-01T09:30:21.418 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:30:21.427 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:30:21.427 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:30:21.469 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:21.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:21.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:21.572 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:21.603 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:21.671 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:21.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:21.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:21.779 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:21.814 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:30:21.819 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:21.903 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:21.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:21.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:22.011 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:22.101 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:22.366 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 49 16 70401 70385 89.7448 54.1875 0.00340618 0.0111155 2023-10-01T09:30:22.798 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:22 smithi138 ceph-mon[185886]: from='client.35754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:30:22.799 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:22.799 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:22.799 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:22 smithi138 ceph-mon[185886]: pgmap v932: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-01T09:30:23.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:22 smithi190 ceph-mon[160435]: from='client.35754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:30:23.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:23.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:23.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:22 smithi190 ceph-mon[160435]: pgmap v932: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-01T09:30:23.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:22 smithi138 ceph-mon[192513]: from='client.35754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:30:23.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:23.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:23.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:22 smithi138 ceph-mon[192513]: pgmap v932: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-01T09:30:23.263 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:23.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:23.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:23.307 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:30:23.361 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":931,"stamp":"2023-10-01T09:30:19.895324+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4288944051,"num_objects":65399,"num_object_clones":0,"num_object_copies":196197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":65399,"num_whiteouts":0,"num_read":6259,"num_read_kb":12659,"num_write":66477,"num_write_kb":4198885,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":310,"num_bytes_recovered":3570664,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":66541,"ondisk_log_size":66541,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15699704,"kb_used_data":12642368,"kb_used_omap":45,"kb_used_meta":3057234,"kb_avail":640516360,"statfs":{"total":671965249536,"available":655888752640,"internally_reserved":0,"allocated":12945784832,"data_stored":12936959542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46344,"internal_metadata":3130608376},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,2,1,10,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":594804736,"num_objects":9076,"num_object_clones":0,"num_object_copies":27228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9076,"num_write_kb":580864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.935903"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":223,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974403+0000","last_change":"2023-10-01T09:26:12.273466+0000","last_active":"2023-10-01T09:30:10.974403+0000","last_peered":"2023-10-01T09:30:10.974403+0000","last_clean":"2023-10-01T09:30:10.974403+0000","last_became_active":"2023-10-01T09:26:12.272944+0000","last_became_peered":"2023-10-01T09:26:12.272944+0000","last_unstale":"2023-10-01T09:30:10.974403+0000","last_undegraded":"2023-10-01T09:30:10.974403+0000","last_fullsized":"2023-10-01T09:30:10.974403+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:40:41.939800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":144,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964138+0000","last_change":"2023-10-01T09:28:02.324629+0000","last_active":"2023-10-01T09:30:10.964138+0000","last_peered":"2023-10-01T09:30:10.964138+0000","last_clean":"2023-10-01T09:30:10.964138+0000","last_became_active":"2023-10-01T09:28:02.324019+0000","last_became_peered":"2023-10-01T09:28:02.324019+0000","last_unstale":"2023-10-01T09:30:10.964138+0000","last_undegraded":"2023-10-01T09:30:10.964138+0000","last_fullsized":"2023-10-01T09:30:10.964138+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:00:43.440938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":308,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.973895+0000","last_change":"2023-10-01T09:28:02.338915+0000","last_active":"2023-10-01T09:30:10.973895+0000","last_peered":"2023-10-01T09:30:10.973895+0000","last_clean":"2023-10-01T09:30:10.973895+0000","last_became_active":"2023-10-01T09:28:02.338589+0000","last_became_peered":"2023-10-01T09:28:02.338589+0000","last_unstale":"2023-10-01T09:30:10.973895+0000","last_undegraded":"2023-10-01T09:30:10.973895+0000","last_fullsized":"2023-10-01T09:30:10.973895+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:18:19.992078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":356,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964371+0000","last_change":"2023-10-01T09:29:58.372577+0000","last_active":"2023-10-01T09:30:10.964371+0000","last_peered":"2023-10-01T09:30:10.964371+0000","last_clean":"2023-10-01T09:30:10.964371+0000","last_became_active":"2023-10-01T09:29:58.372218+0000","last_became_peered":"2023-10-01T09:29:58.372218+0000","last_unstale":"2023-10-01T09:30:10.964371+0000","last_undegraded":"2023-10-01T09:30:10.964371+0000","last_fullsized":"2023-10-01T09:30:10.964371+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:04:08.376752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":117,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.975370+0000","last_change":"2023-10-01T09:22:40.172183+0000","last_active":"2023-10-01T09:30:05.975370+0000","last_peered":"2023-10-01T09:30:05.975370+0000","last_clean":"2023-10-01T09:30:05.975370+0000","last_became_active":"2023-10-01T09:22:40.170164+0000","last_became_peered":"2023-10-01T09:22:40.170164+0000","last_unstale":"2023-10-01T09:30:05.975370+0000","last_undegraded":"2023-10-01T09:30:05.975370+0000","last_fullsized":"2023-10-01T09:30:05.975370+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:33:10.407999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":44,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.967616+0000","last_change":"2023-10-01T09:28:02.362383+0000","last_active":"2023-10-01T09:30:05.967616+0000","last_peered":"2023-10-01T09:30:05.967616+0000","last_clean":"2023-10-01T09:30:05.967616+0000","last_became_active":"2023-10-01T09:28:02.362044+0000","last_became_peered":"2023-10-01T09:28:02.362044+0000","last_unstale":"2023-10-01T09:30:05.967616+0000","last_undegraded":"2023-10-01T09:30:05.967616+0000","last_fullsized":"2023-10-01T09:30:05.967616+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":378,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:17.991997+0000","last_change":"2023-10-01T09:27:01.992338+0000","last_active":"2023-10-01T09:30:17.991997+0000","last_peered":"2023-10-01T09:30:17.991997+0000","last_clean":"2023-10-01T09:30:17.991997+0000","last_became_active":"2023-10-01T09:27:01.989050+0000","last_became_peered":"2023-10-01T09:27:01.989050+0000","last_unstale":"2023-10-01T09:30:17.991997+0000","last_undegraded":"2023-10-01T09:30:17.991997+0000","last_fullsized":"2023-10-01T09:30:17.991997+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:43:29.856456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":375,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.975494+0000","last_change":"2023-10-01T09:28:02.347509+0000","last_active":"2023-10-01T09:30:05.975494+0000","last_peered":"2023-10-01T09:30:05.975494+0000","last_clean":"2023-10-01T09:30:05.975494+0000","last_became_active":"2023-10-01T09:28:02.344509+0000","last_became_peered":"2023-10-01T09:28:02.344509+0000","last_unstale":"2023-10-01T09:30:05.975494+0000","last_undegraded":"2023-10-01T09:30:05.975494+0000","last_fullsized":"2023-10-01T09:30:05.975494+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:04:00.983057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":51,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.978656+0000","last_change":"2023-10-01T09:27:01.970412+0000","last_active":"2023-10-01T09:30:05.978656+0000","last_peered":"2023-10-01T09:30:05.978656+0000","last_clean":"2023-10-01T09:30:05.978656+0000","last_became_active":"2023-10-01T09:27:01.970108+0000","last_became_peered":"2023-10-01T09:27:01.970108+0000","last_unstale":"2023-10-01T09:30:05.978656+0000","last_undegraded":"2023-10-01T09:30:05.978656+0000","last_fullsized":"2023-10-01T09:30:05.978656+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":392,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.974162+0000","last_change":"2023-10-01T09:28:02.322837+0000","last_active":"2023-10-01T09:30:05.974162+0000","last_peered":"2023-10-01T09:30:05.974162+0000","last_clean":"2023-10-01T09:30:05.974162+0000","last_became_active":"2023-10-01T09:28:02.322536+0000","last_became_peered":"2023-10-01T09:28:02.322536+0000","last_unstale":"2023-10-01T09:30:05.974162+0000","last_undegraded":"2023-10-01T09:30:05.974162+0000","last_fullsized":"2023-10-01T09:30:05.974162+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:48:11.306512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":124,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.971097+0000","last_change":"2023-10-01T09:26:12.415219+0000","last_active":"2023-10-01T09:30:05.971097+0000","last_peered":"2023-10-01T09:30:05.971097+0000","last_clean":"2023-10-01T09:30:05.971097+0000","last_became_active":"2023-10-01T09:26:12.286669+0000","last_became_peered":"2023-10-01T09:26:12.286669+0000","last_unstale":"2023-10-01T09:30:05.971097+0000","last_undegraded":"2023-10-01T09:30:05.971097+0000","last_fullsized":"2023-10-01T09:30:05.971097+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":368,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.973972+0000","last_change":"2023-10-01T09:28:02.342661+0000","last_active":"2023-10-01T09:30:10.973972+0000","last_peered":"2023-10-01T09:30:10.973972+0000","last_clean":"2023-10-01T09:30:10.973972+0000","last_became_active":"2023-10-01T09:28:02.342358+0000","last_became_peered":"2023-10-01T09:28:02.342358+0000","last_unstale":"2023-10-01T09:30:10.973972+0000","last_undegraded":"2023-10-01T09:30:10.973972+0000","last_fullsized":"2023-10-01T09:30:10.973972+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:29:30.624568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":348,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.970372+0000","last_change":"2023-10-01T09:28:02.511577+0000","last_active":"2023-10-01T09:30:05.970372+0000","last_peered":"2023-10-01T09:30:05.970372+0000","last_clean":"2023-10-01T09:30:05.970372+0000","last_became_active":"2023-10-01T09:28:02.318805+0000","last_became_peered":"2023-10-01T09:28:02.318805+0000","last_unstale":"2023-10-01T09:30:05.970372+0000","last_undegraded":"2023-10-01T09:30:05.970372+0000","last_fullsized":"2023-10-01T09:30:05.970372+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":44,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.967150+0000","last_change":"2023-10-01T09:28:02.454516+0000","last_active":"2023-10-01T09:30:05.967150+0000","last_peered":"2023-10-01T09:30:05.967150+0000","last_clean":"2023-10-01T09:30:05.967150+0000","last_became_active":"2023-10-01T09:28:02.454098+0000","last_became_peered":"2023-10-01T09:28:02.454098+0000","last_unstale":"2023-10-01T09:30:05.967150+0000","last_undegraded":"2023-10-01T09:30:05.967150+0000","last_fullsized":"2023-10-01T09:30:05.967150+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":398,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.970241+0000","last_change":"2023-10-01T09:28:02.505692+0000","last_active":"2023-10-01T09:30:05.970241+0000","last_peered":"2023-10-01T09:30:05.970241+0000","last_clean":"2023-10-01T09:30:05.970241+0000","last_became_active":"2023-10-01T09:28:02.318771+0000","last_became_peered":"2023-10-01T09:28:02.318771+0000","last_unstale":"2023-10-01T09:30:05.970241+0000","last_undegraded":"2023-10-01T09:30:05.970241+0000","last_fullsized":"2023-10-01T09:30:05.970241+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":324,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.974963+0000","last_change":"2023-10-01T09:19:57.493670+0000","last_active":"2023-10-01T09:30:05.974963+0000","last_peered":"2023-10-01T09:30:05.974963+0000","last_clean":"2023-10-01T09:30:05.974963+0000","last_became_active":"2023-10-01T09:19:57.493349+0000","last_became_peered":"2023-10-01T09:19:57.493349+0000","last_unstale":"2023-10-01T09:30:05.974963+0000","last_undegraded":"2023-10-01T09:30:05.974963+0000","last_fullsized":"2023-10-01T09:30:05.974963+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:26:56.468779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":331,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964896+0000","last_change":"2023-10-01T09:23:10.607388+0000","last_active":"2023-10-01T09:30:10.964896+0000","last_peered":"2023-10-01T09:30:10.964896+0000","last_clean":"2023-10-01T09:30:10.964896+0000","last_became_active":"2023-10-01T09:23:10.607084+0000","last_became_peered":"2023-10-01T09:23:10.607084+0000","last_unstale":"2023-10-01T09:30:10.964896+0000","last_undegraded":"2023-10-01T09:30:10.964896+0000","last_fullsized":"2023-10-01T09:30:10.964896+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:19.669393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":51,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.979058+0000","last_change":"2023-10-01T09:27:01.978738+0000","last_active":"2023-10-01T09:30:05.979058+0000","last_peered":"2023-10-01T09:30:05.979058+0000","last_clean":"2023-10-01T09:30:05.979058+0000","last_became_active":"2023-10-01T09:27:01.978024+0000","last_became_peered":"2023-10-01T09:27:01.978024+0000","last_unstale":"2023-10-01T09:30:05.979058+0000","last_undegraded":"2023-10-01T09:30:05.979058+0000","last_fullsized":"2023-10-01T09:30:05.979058+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":342,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.970047+0000","last_change":"2023-10-01T09:28:02.314047+0000","last_active":"2023-10-01T09:30:05.970047+0000","last_peered":"2023-10-01T09:30:05.970047+0000","last_clean":"2023-10-01T09:30:05.970047+0000","last_became_active":"2023-10-01T09:28:02.313729+0000","last_became_peered":"2023-10-01T09:28:02.313729+0000","last_unstale":"2023-10-01T09:30:05.970047+0000","last_undegraded":"2023-10-01T09:30:05.970047+0000","last_fullsized":"2023-10-01T09:30:05.970047+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":313,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:17.992871+0000","last_change":"2023-10-01T09:23:10.611400+0000","last_active":"2023-10-01T09:30:17.992871+0000","last_peered":"2023-10-01T09:30:17.992871+0000","last_clean":"2023-10-01T09:30:17.992871+0000","last_became_active":"2023-10-01T09:23:10.610928+0000","last_became_peered":"2023-10-01T09:23:10.610928+0000","last_unstale":"2023-10-01T09:30:17.992871+0000","last_undegraded":"2023-10-01T09:30:17.992871+0000","last_fullsized":"2023-10-01T09:30:17.992871+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:17.141540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":81,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.966970+0000","last_change":"2023-10-01T09:30:00.577886+0000","last_active":"2023-10-01T09:30:05.966970+0000","last_peered":"2023-10-01T09:30:05.966970+0000","last_clean":"2023-10-01T09:30:05.966970+0000","last_became_active":"2023-10-01T09:30:00.394121+0000","last_became_peered":"2023-10-01T09:30:00.394121+0000","last_unstale":"2023-10-01T09:30:05.966970+0000","last_undegraded":"2023-10-01T09:30:05.966970+0000","last_fullsized":"2023-10-01T09:30:05.966970+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":126,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.980629+0000","last_change":"2023-10-01T09:23:10.613386+0000","last_active":"2023-10-01T09:30:05.980629+0000","last_peered":"2023-10-01T09:30:05.980629+0000","last_clean":"2023-10-01T09:30:05.980629+0000","last_became_active":"2023-10-01T09:23:10.613046+0000","last_became_peered":"2023-10-01T09:23:10.613046+0000","last_unstale":"2023-10-01T09:30:05.980629+0000","last_undegraded":"2023-10-01T09:30:05.980629+0000","last_fullsized":"2023-10-01T09:30:05.980629+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:23:01.448841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"340'3977","reported_seq":3999,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:15.602391+0000","last_change":"2023-10-01T09:29:24.378970+0000","last_active":"2023-10-01T09:30:15.602391+0000","last_peered":"2023-10-01T09:30:15.602391+0000","last_clean":"2023-10-01T09:30:15.602391+0000","last_became_active":"2023-10-01T09:29:24.378703+0000","last_became_peered":"2023-10-01T09:29:24.378703+0000","last_unstale":"2023-10-01T09:30:15.602391+0000","last_undegraded":"2023-10-01T09:30:15.602391+0000","last_fullsized":"2023-10-01T09:30:15.602391+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":3977,"log_dups_size":0,"ondisk_log_size":3977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260636672,"num_objects":3977,"num_object_clones":0,"num_object_copies":11931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3977,"num_write_kb":254528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":120,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.970599+0000","last_change":"2023-10-01T09:26:12.395230+0000","last_active":"2023-10-01T09:30:05.970599+0000","last_peered":"2023-10-01T09:30:05.970599+0000","last_clean":"2023-10-01T09:30:05.970599+0000","last_became_active":"2023-10-01T09:26:12.269960+0000","last_became_peered":"2023-10-01T09:26:12.269960+0000","last_unstale":"2023-10-01T09:30:05.970599+0000","last_undegraded":"2023-10-01T09:30:05.970599+0000","last_fullsized":"2023-10-01T09:30:05.970599+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":60,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.980456+0000","last_change":"2023-10-01T09:28:02.315483+0000","last_active":"2023-10-01T09:30:05.980456+0000","last_peered":"2023-10-01T09:30:05.980456+0000","last_clean":"2023-10-01T09:30:05.980456+0000","last_became_active":"2023-10-01T09:28:02.315121+0000","last_became_peered":"2023-10-01T09:28:02.315121+0000","last_unstale":"2023-10-01T09:30:05.980456+0000","last_undegraded":"2023-10-01T09:30:05.980456+0000","last_fullsized":"2023-10-01T09:30:05.980456+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"340'4135","reported_seq":4157,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:16.898654+0000","last_change":"2023-10-01T09:29:24.376565+0000","last_active":"2023-10-01T09:30:16.898654+0000","last_peered":"2023-10-01T09:30:16.898654+0000","last_clean":"2023-10-01T09:30:16.898654+0000","last_became_active":"2023-10-01T09:29:24.376238+0000","last_became_peered":"2023-10-01T09:29:24.376238+0000","last_unstale":"2023-10-01T09:30:16.898654+0000","last_undegraded":"2023-10-01T09:30:16.898654+0000","last_fullsized":"2023-10-01T09:30:16.898654+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4135,"log_dups_size":0,"ondisk_log_size":4135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270991360,"num_objects":4135,"num_object_clones":0,"num_object_copies":12405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4135,"num_write_kb":264640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":321,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.970612+0000","last_change":"2023-10-01T09:26:12.408240+0000","last_active":"2023-10-01T09:30:05.970612+0000","last_peered":"2023-10-01T09:30:05.970612+0000","last_clean":"2023-10-01T09:30:05.970612+0000","last_became_active":"2023-10-01T09:26:12.272093+0000","last_became_peered":"2023-10-01T09:26:12.272093+0000","last_unstale":"2023-10-01T09:30:05.970612+0000","last_undegraded":"2023-10-01T09:30:05.970612+0000","last_fullsized":"2023-10-01T09:30:05.970612+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":363,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.973845+0000","last_change":"2023-10-01T09:28:02.338346+0000","last_active":"2023-10-01T09:30:10.973845+0000","last_peered":"2023-10-01T09:30:10.973845+0000","last_clean":"2023-10-01T09:30:10.973845+0000","last_became_active":"2023-10-01T09:28:02.338038+0000","last_became_peered":"2023-10-01T09:28:02.338038+0000","last_unstale":"2023-10-01T09:30:10.973845+0000","last_undegraded":"2023-10-01T09:30:10.973845+0000","last_fullsized":"2023-10-01T09:30:10.973845+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:56:32.734028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"341'4094","reported_seq":4118,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:18.267606+0000","last_change":"2023-10-01T09:29:24.379084+0000","last_active":"2023-10-01T09:30:18.267606+0000","last_peered":"2023-10-01T09:30:18.267606+0000","last_clean":"2023-10-01T09:30:18.267606+0000","last_became_active":"2023-10-01T09:29:24.378789+0000","last_became_peered":"2023-10-01T09:29:24.378789+0000","last_unstale":"2023-10-01T09:30:18.267606+0000","last_undegraded":"2023-10-01T09:30:18.267606+0000","last_fullsized":"2023-10-01T09:30:18.267606+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4094,"log_dups_size":0,"ondisk_log_size":4094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:00:21.161819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268304384,"num_objects":4094,"num_object_clones":0,"num_object_copies":12282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4094,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":405,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.971156+0000","last_change":"2023-10-01T09:28:02.318966+0000","last_active":"2023-10-01T09:30:05.971156+0000","last_peered":"2023-10-01T09:30:05.971156+0000","last_clean":"2023-10-01T09:30:05.971156+0000","last_became_active":"2023-10-01T09:28:02.316075+0000","last_became_peered":"2023-10-01T09:28:02.316075+0000","last_unstale":"2023-10-01T09:30:05.971156+0000","last_undegraded":"2023-10-01T09:30:05.971156+0000","last_fullsized":"2023-10-01T09:30:05.971156+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":43,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.967275+0000","last_change":"2023-10-01T09:28:02.361663+0000","last_active":"2023-10-01T09:30:05.967275+0000","last_peered":"2023-10-01T09:30:05.967275+0000","last_clean":"2023-10-01T09:30:05.967275+0000","last_became_active":"2023-10-01T09:28:02.361093+0000","last_became_peered":"2023-10-01T09:28:02.361093+0000","last_unstale":"2023-10-01T09:30:05.967275+0000","last_undegraded":"2023-10-01T09:30:05.967275+0000","last_fullsized":"2023-10-01T09:30:05.967275+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"341'4094","reported_seq":4118,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:18.049365+0000","last_change":"2023-10-01T09:29:24.356954+0000","last_active":"2023-10-01T09:30:18.049365+0000","last_peered":"2023-10-01T09:30:18.049365+0000","last_clean":"2023-10-01T09:30:18.049365+0000","last_became_active":"2023-10-01T09:29:24.356643+0000","last_became_peered":"2023-10-01T09:29:24.356643+0000","last_unstale":"2023-10-01T09:30:18.049365+0000","last_undegraded":"2023-10-01T09:30:18.049365+0000","last_fullsized":"2023-10-01T09:30:18.049365+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4094,"log_dups_size":0,"ondisk_log_size":4094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:56:34.140751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268304384,"num_objects":4094,"num_object_clones":0,"num_object_copies":12282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4094,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":60,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.979066+0000","last_change":"2023-10-01T09:28:02.314625+0000","last_active":"2023-10-01T09:30:05.979066+0000","last_peered":"2023-10-01T09:30:05.979066+0000","last_clean":"2023-10-01T09:30:05.979066+0000","last_became_active":"2023-10-01T09:28:02.314315+0000","last_became_peered":"2023-10-01T09:28:02.314315+0000","last_unstale":"2023-10-01T09:30:05.979066+0000","last_undegraded":"2023-10-01T09:30:05.979066+0000","last_fullsized":"2023-10-01T09:30:05.979066+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":304,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963213+0000","last_change":"2023-10-01T09:26:12.308350+0000","last_active":"2023-10-01T09:30:10.963213+0000","last_peered":"2023-10-01T09:30:10.963213+0000","last_clean":"2023-10-01T09:30:10.963213+0000","last_became_active":"2023-10-01T09:26:12.307689+0000","last_became_peered":"2023-10-01T09:26:12.307689+0000","last_unstale":"2023-10-01T09:30:10.963213+0000","last_undegraded":"2023-10-01T09:30:10.963213+0000","last_fullsized":"2023-10-01T09:30:10.963213+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:34:57.116449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"340'4037","reported_seq":4059,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:16.660022+0000","last_change":"2023-10-01T09:29:24.360118+0000","last_active":"2023-10-01T09:30:16.660022+0000","last_peered":"2023-10-01T09:30:16.660022+0000","last_clean":"2023-10-01T09:30:16.660022+0000","last_became_active":"2023-10-01T09:29:24.359817+0000","last_became_peered":"2023-10-01T09:29:24.359817+0000","last_unstale":"2023-10-01T09:30:16.660022+0000","last_undegraded":"2023-10-01T09:30:16.660022+0000","last_fullsized":"2023-10-01T09:30:16.660022+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4037,"log_dups_size":0,"ondisk_log_size":4037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:52:42.958518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264568832,"num_objects":4037,"num_object_clones":0,"num_object_copies":12111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4037,"num_write_kb":258368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":326,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.974803+0000","last_change":"2023-10-01T09:26:12.387826+0000","last_active":"2023-10-01T09:30:05.974803+0000","last_peered":"2023-10-01T09:30:05.974803+0000","last_clean":"2023-10-01T09:30:05.974803+0000","last_became_active":"2023-10-01T09:26:12.301770+0000","last_became_peered":"2023-10-01T09:26:12.301770+0000","last_unstale":"2023-10-01T09:30:05.974803+0000","last_undegraded":"2023-10-01T09:30:05.974803+0000","last_fullsized":"2023-10-01T09:30:05.974803+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:09:42.617904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":307,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964928+0000","last_change":"2023-10-01T09:28:02.349347+0000","last_active":"2023-10-01T09:30:10.964928+0000","last_peered":"2023-10-01T09:30:10.964928+0000","last_clean":"2023-10-01T09:30:10.964928+0000","last_became_active":"2023-10-01T09:28:02.337322+0000","last_became_peered":"2023-10-01T09:28:02.337322+0000","last_unstale":"2023-10-01T09:30:10.964928+0000","last_undegraded":"2023-10-01T09:30:10.964928+0000","last_fullsized":"2023-10-01T09:30:10.964928+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:46:56.505421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"341'4136","reported_seq":4158,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:18.060428+0000","last_change":"2023-10-01T09:29:24.380240+0000","last_active":"2023-10-01T09:30:18.060428+0000","last_peered":"2023-10-01T09:30:18.060428+0000","last_clean":"2023-10-01T09:30:18.060428+0000","last_became_active":"2023-10-01T09:29:24.379922+0000","last_became_peered":"2023-10-01T09:29:24.379922+0000","last_unstale":"2023-10-01T09:30:18.060428+0000","last_undegraded":"2023-10-01T09:30:18.060428+0000","last_fullsized":"2023-10-01T09:30:18.060428+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4136,"log_dups_size":0,"ondisk_log_size":4136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:34:10.363596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271056896,"num_objects":4136,"num_object_clones":0,"num_object_copies":12408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4136,"num_write_kb":264704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":313,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974171+0000","last_change":"2023-10-01T09:28:02.342548+0000","last_active":"2023-10-01T09:30:10.974171+0000","last_peered":"2023-10-01T09:30:10.974171+0000","last_clean":"2023-10-01T09:30:10.974171+0000","last_became_active":"2023-10-01T09:28:02.342162+0000","last_became_peered":"2023-10-01T09:28:02.342162+0000","last_unstale":"2023-10-01T09:30:10.974171+0000","last_undegraded":"2023-10-01T09:30:10.974171+0000","last_fullsized":"2023-10-01T09:30:10.974171+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:49:07.163031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":357,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.975533+0000","last_change":"2023-10-01T09:28:02.342750+0000","last_active":"2023-10-01T09:30:10.975533+0000","last_peered":"2023-10-01T09:30:10.975533+0000","last_clean":"2023-10-01T09:30:10.975533+0000","last_became_active":"2023-10-01T09:28:02.342487+0000","last_became_peered":"2023-10-01T09:28:02.342487+0000","last_unstale":"2023-10-01T09:30:10.975533+0000","last_undegraded":"2023-10-01T09:30:10.975533+0000","last_fullsized":"2023-10-01T09:30:10.975533+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:23:46.051719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"341'4100","reported_seq":4124,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:18.249248+0000","last_change":"2023-10-01T09:29:24.359090+0000","last_active":"2023-10-01T09:30:18.249248+0000","last_peered":"2023-10-01T09:30:18.249248+0000","last_clean":"2023-10-01T09:30:18.249248+0000","last_became_active":"2023-10-01T09:29:24.358822+0000","last_became_peered":"2023-10-01T09:29:24.358822+0000","last_unstale":"2023-10-01T09:30:18.249248+0000","last_undegraded":"2023-10-01T09:30:18.249248+0000","last_fullsized":"2023-10-01T09:30:18.249248+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4100,"log_dups_size":0,"ondisk_log_size":4100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:29:47.448230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268697600,"num_objects":4100,"num_object_clones":0,"num_object_copies":12300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4100,"num_write_kb":262400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":203,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.970086+0000","last_change":"2023-10-01T09:26:29.372326+0000","last_active":"2023-10-01T09:30:05.970086+0000","last_peered":"2023-10-01T09:30:05.970086+0000","last_clean":"2023-10-01T09:30:05.970086+0000","last_became_active":"2023-10-01T09:26:29.372035+0000","last_became_peered":"2023-10-01T09:26:29.372035+0000","last_unstale":"2023-10-01T09:30:05.970086+0000","last_undegraded":"2023-10-01T09:30:05.970086+0000","last_fullsized":"2023-10-01T09:30:05.970086+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":119,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974858+0000","last_change":"2023-10-01T09:28:02.339901+0000","last_active":"2023-10-01T09:30:10.974858+0000","last_peered":"2023-10-01T09:30:10.974858+0000","last_clean":"2023-10-01T09:30:10.974858+0000","last_became_active":"2023-10-01T09:28:02.339601+0000","last_became_peered":"2023-10-01T09:28:02.339601+0000","last_unstale":"2023-10-01T09:30:10.974858+0000","last_undegraded":"2023-10-01T09:30:10.974858+0000","last_fullsized":"2023-10-01T09:30:10.974858+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:30:16.497879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"341'4133","reported_seq":4157,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:18.059865+0000","last_change":"2023-10-01T09:29:24.351403+0000","last_active":"2023-10-01T09:30:18.059865+0000","last_peered":"2023-10-01T09:30:18.059865+0000","last_clean":"2023-10-01T09:30:18.059865+0000","last_became_active":"2023-10-01T09:29:24.351113+0000","last_became_peered":"2023-10-01T09:29:24.351113+0000","last_unstale":"2023-10-01T09:30:18.059865+0000","last_undegraded":"2023-10-01T09:30:18.059865+0000","last_fullsized":"2023-10-01T09:30:18.059865+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4133,"log_dups_size":0,"ondisk_log_size":4133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:07:56.443376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270860288,"num_objects":4133,"num_object_clones":0,"num_object_copies":12399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4133,"num_write_kb":264512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":117,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.975591+0000","last_change":"2023-10-01T09:22:40.316008+0000","last_active":"2023-10-01T09:30:05.975591+0000","last_peered":"2023-10-01T09:30:05.975591+0000","last_clean":"2023-10-01T09:30:05.975591+0000","last_became_active":"2023-10-01T09:22:40.315602+0000","last_became_peered":"2023-10-01T09:22:40.315602+0000","last_unstale":"2023-10-01T09:30:05.975591+0000","last_undegraded":"2023-10-01T09:30:05.975591+0000","last_fullsized":"2023-10-01T09:30:05.975591+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:39:37.485241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":218,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.975086+0000","last_change":"2023-10-01T09:26:12.320680+0000","last_active":"2023-10-01T09:30:10.975086+0000","last_peered":"2023-10-01T09:30:10.975086+0000","last_clean":"2023-10-01T09:30:10.975086+0000","last_became_active":"2023-10-01T09:26:12.320340+0000","last_became_peered":"2023-10-01T09:26:12.320340+0000","last_unstale":"2023-10-01T09:30:10.975086+0000","last_undegraded":"2023-10-01T09:30:10.975086+0000","last_fullsized":"2023-10-01T09:30:10.975086+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:11:35.796015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"341'4218","reported_seq":4242,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:18.198229+0000","last_change":"2023-10-01T09:29:24.382969+0000","last_active":"2023-10-01T09:30:18.198229+0000","last_peered":"2023-10-01T09:30:18.198229+0000","last_clean":"2023-10-01T09:30:18.198229+0000","last_became_active":"2023-10-01T09:29:24.382653+0000","last_became_peered":"2023-10-01T09:29:24.382653+0000","last_unstale":"2023-10-01T09:30:18.198229+0000","last_undegraded":"2023-10-01T09:30:18.198229+0000","last_fullsized":"2023-10-01T09:30:18.198229+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4218,"log_dups_size":0,"ondisk_log_size":4218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:37:56.666282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276430848,"num_objects":4218,"num_object_clones":0,"num_object_copies":12654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4218,"num_write_kb":269952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1161,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974682+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:30:10.974682+0000","last_peered":"2023-10-01T09:30:10.974682+0000","last_clean":"2023-10-01T09:30:10.974682+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:30:10.974682+0000","last_undegraded":"2023-10-01T09:30:10.974682+0000","last_fullsized":"2023-10-01T09:30:10.974682+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":51,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.979215+0000","last_change":"2023-10-01T09:27:01.989188+0000","last_active":"2023-10-01T09:30:05.979215+0000","last_peered":"2023-10-01T09:30:05.979215+0000","last_clean":"2023-10-01T09:30:05.979215+0000","last_became_active":"2023-10-01T09:27:01.988878+0000","last_became_peered":"2023-10-01T09:27:01.988878+0000","last_unstale":"2023-10-01T09:30:05.979215+0000","last_undegraded":"2023-10-01T09:30:05.979215+0000","last_fullsized":"2023-10-01T09:30:05.979215+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":318,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974045+0000","last_change":"2023-10-01T09:27:01.977155+0000","last_active":"2023-10-01T09:30:10.974045+0000","last_peered":"2023-10-01T09:30:10.974045+0000","last_clean":"2023-10-01T09:30:10.974045+0000","last_became_active":"2023-10-01T09:27:01.975803+0000","last_became_peered":"2023-10-01T09:27:01.975803+0000","last_unstale":"2023-10-01T09:30:10.974045+0000","last_undegraded":"2023-10-01T09:30:10.974045+0000","last_fullsized":"2023-10-01T09:30:10.974045+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:00:55.170662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"340'3926","reported_seq":3948,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:15.571212+0000","last_change":"2023-10-01T09:29:24.376062+0000","last_active":"2023-10-01T09:30:15.571212+0000","last_peered":"2023-10-01T09:30:15.571212+0000","last_clean":"2023-10-01T09:30:15.571212+0000","last_became_active":"2023-10-01T09:29:24.375795+0000","last_became_peered":"2023-10-01T09:29:24.375795+0000","last_unstale":"2023-10-01T09:30:15.571212+0000","last_undegraded":"2023-10-01T09:30:15.571212+0000","last_fullsized":"2023-10-01T09:30:15.571212+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":3926,"log_dups_size":0,"ondisk_log_size":3926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257294336,"num_objects":3926,"num_object_clones":0,"num_object_copies":11778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3926,"num_write_kb":251264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1023,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.979039+0000","last_change":"2023-10-01T09:28:02.595215+0000","last_active":"2023-10-01T09:30:05.979039+0000","last_peered":"2023-10-01T09:30:05.979039+0000","last_clean":"2023-10-01T09:30:05.979039+0000","last_became_active":"2023-10-01T09:28:02.346777+0000","last_became_peered":"2023-10-01T09:28:02.346777+0000","last_unstale":"2023-10-01T09:30:05.979039+0000","last_undegraded":"2023-10-01T09:30:05.979039+0000","last_fullsized":"2023-10-01T09:30:05.979039+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":304,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963570+0000","last_change":"2023-10-01T09:28:02.457735+0000","last_active":"2023-10-01T09:30:10.963570+0000","last_peered":"2023-10-01T09:30:10.963570+0000","last_clean":"2023-10-01T09:30:10.963570+0000","last_became_active":"2023-10-01T09:28:02.457495+0000","last_became_peered":"2023-10-01T09:28:02.457495+0000","last_unstale":"2023-10-01T09:30:10.963570+0000","last_undegraded":"2023-10-01T09:30:10.963570+0000","last_fullsized":"2023-10-01T09:30:10.963570+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:48:53.316760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.978819+0000","last_change":"2023-10-01T09:29:58.362019+0000","last_active":"2023-10-01T09:30:05.978819+0000","last_peered":"2023-10-01T09:30:05.978819+0000","last_clean":"2023-10-01T09:30:05.978819+0000","last_became_active":"2023-10-01T09:29:58.361762+0000","last_became_peered":"2023-10-01T09:29:58.361762+0000","last_unstale":"2023-10-01T09:30:05.978819+0000","last_undegraded":"2023-10-01T09:30:05.978819+0000","last_fullsized":"2023-10-01T09:30:05.978819+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"340'4043","reported_seq":4066,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:16.597962+0000","last_change":"2023-10-01T09:29:24.379882+0000","last_active":"2023-10-01T09:30:16.597962+0000","last_peered":"2023-10-01T09:30:16.597962+0000","last_clean":"2023-10-01T09:30:16.597962+0000","last_became_active":"2023-10-01T09:29:24.379572+0000","last_became_peered":"2023-10-01T09:29:24.379572+0000","last_unstale":"2023-10-01T09:30:16.597962+0000","last_undegraded":"2023-10-01T09:30:16.597962+0000","last_fullsized":"2023-10-01T09:30:16.597962+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4043,"log_dups_size":0,"ondisk_log_size":4043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:39:28.119558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264962048,"num_objects":4043,"num_object_clones":0,"num_object_copies":12129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4043,"num_write_kb":258752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":595,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.973944+0000","last_change":"2023-10-01T09:26:29.385302+0000","last_active":"2023-10-01T09:30:05.973944+0000","last_peered":"2023-10-01T09:30:05.973944+0000","last_clean":"2023-10-01T09:30:05.973944+0000","last_became_active":"2023-10-01T09:26:29.384738+0000","last_became_peered":"2023-10-01T09:26:29.384738+0000","last_unstale":"2023-10-01T09:30:05.973944+0000","last_undegraded":"2023-10-01T09:30:05.973944+0000","last_fullsized":"2023-10-01T09:30:05.973944+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:22:00.127736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":370,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:13.168721+0000","last_change":"2023-10-01T09:28:02.346596+0000","last_active":"2023-10-01T09:30:13.168721+0000","last_peered":"2023-10-01T09:30:13.168721+0000","last_clean":"2023-10-01T09:30:13.168721+0000","last_became_active":"2023-10-01T09:28:02.346318+0000","last_became_peered":"2023-10-01T09:28:02.346318+0000","last_unstale":"2023-10-01T09:30:13.168721+0000","last_undegraded":"2023-10-01T09:30:13.168721+0000","last_fullsized":"2023-10-01T09:30:13.168721+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:27:45.840901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":224,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.975261+0000","last_change":"2023-10-01T09:27:02.004900+0000","last_active":"2023-10-01T09:30:10.975261+0000","last_peered":"2023-10-01T09:30:10.975261+0000","last_clean":"2023-10-01T09:30:10.975261+0000","last_became_active":"2023-10-01T09:27:02.004501+0000","last_became_peered":"2023-10-01T09:27:02.004501+0000","last_unstale":"2023-10-01T09:30:10.975261+0000","last_undegraded":"2023-10-01T09:30:10.975261+0000","last_fullsized":"2023-10-01T09:30:10.975261+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:50:52.617833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"340'3982","reported_seq":4004,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:15.593448+0000","last_change":"2023-10-01T09:29:24.359284+0000","last_active":"2023-10-01T09:30:15.593448+0000","last_peered":"2023-10-01T09:30:15.593448+0000","last_clean":"2023-10-01T09:30:15.593448+0000","last_became_active":"2023-10-01T09:29:24.358932+0000","last_became_peered":"2023-10-01T09:29:24.358932+0000","last_unstale":"2023-10-01T09:30:15.593448+0000","last_undegraded":"2023-10-01T09:30:15.593448+0000","last_fullsized":"2023-10-01T09:30:15.593448+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":3982,"log_dups_size":0,"ondisk_log_size":3982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260964352,"num_objects":3982,"num_object_clones":0,"num_object_copies":11946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3982,"num_write_kb":254848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1266,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.979037+0000","last_change":"2023-10-01T09:28:48.006973+0000","last_active":"2023-10-01T09:30:05.979037+0000","last_peered":"2023-10-01T09:30:05.979037+0000","last_clean":"2023-10-01T09:30:05.979037+0000","last_became_active":"2023-10-01T09:28:48.006289+0000","last_became_peered":"2023-10-01T09:28:48.006289+0000","last_unstale":"2023-10-01T09:30:05.979037+0000","last_undegraded":"2023-10-01T09:30:05.979037+0000","last_fullsized":"2023-10-01T09:30:05.979037+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":558,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.970068+0000","last_change":"2023-10-01T09:29:04.962927+0000","last_active":"2023-10-01T09:30:05.970068+0000","last_peered":"2023-10-01T09:30:05.970068+0000","last_clean":"2023-10-01T09:30:05.970068+0000","last_became_active":"2023-10-01T09:29:04.962542+0000","last_became_peered":"2023-10-01T09:29:04.962542+0000","last_unstale":"2023-10-01T09:30:05.970068+0000","last_undegraded":"2023-10-01T09:30:05.970068+0000","last_fullsized":"2023-10-01T09:30:05.970068+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":170,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.974468+0000","last_change":"2023-10-01T09:28:02.320192+0000","last_active":"2023-10-01T09:30:05.974468+0000","last_peered":"2023-10-01T09:30:05.974468+0000","last_clean":"2023-10-01T09:30:05.974468+0000","last_became_active":"2023-10-01T09:28:02.319852+0000","last_became_peered":"2023-10-01T09:28:02.319852+0000","last_unstale":"2023-10-01T09:30:05.974468+0000","last_undegraded":"2023-10-01T09:30:05.974468+0000","last_fullsized":"2023-10-01T09:30:05.974468+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:16:46.792585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":245,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.971190+0000","last_change":"2023-10-01T09:28:02.312356+0000","last_active":"2023-10-01T09:30:05.971190+0000","last_peered":"2023-10-01T09:30:05.971190+0000","last_clean":"2023-10-01T09:30:05.971190+0000","last_became_active":"2023-10-01T09:28:02.312014+0000","last_became_peered":"2023-10-01T09:28:02.312014+0000","last_unstale":"2023-10-01T09:30:05.971190+0000","last_undegraded":"2023-10-01T09:30:05.971190+0000","last_fullsized":"2023-10-01T09:30:05.971190+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"341'4205","reported_seq":4229,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:18.043175+0000","last_change":"2023-10-01T09:29:24.380724+0000","last_active":"2023-10-01T09:30:18.043175+0000","last_peered":"2023-10-01T09:30:18.043175+0000","last_clean":"2023-10-01T09:30:18.043175+0000","last_became_active":"2023-10-01T09:29:24.380523+0000","last_became_peered":"2023-10-01T09:29:24.380523+0000","last_unstale":"2023-10-01T09:30:18.043175+0000","last_undegraded":"2023-10-01T09:30:18.043175+0000","last_fullsized":"2023-10-01T09:30:18.043175+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4205,"log_dups_size":0,"ondisk_log_size":4205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:16.268423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275578880,"num_objects":4205,"num_object_clones":0,"num_object_copies":12615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4205,"num_write_kb":269120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":436,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.970118+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:30:05.970118+0000","last_peered":"2023-10-01T09:30:05.970118+0000","last_clean":"2023-10-01T09:30:05.970118+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:30:05.970118+0000","last_undegraded":"2023-10-01T09:30:05.970118+0000","last_fullsized":"2023-10-01T09:30:05.970118+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":347,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963933+0000","last_change":"2023-10-01T09:27:01.986786+0000","last_active":"2023-10-01T09:30:10.963933+0000","last_peered":"2023-10-01T09:30:10.963933+0000","last_clean":"2023-10-01T09:30:10.963933+0000","last_became_active":"2023-10-01T09:27:01.986466+0000","last_became_peered":"2023-10-01T09:27:01.986466+0000","last_unstale":"2023-10-01T09:30:10.963933+0000","last_undegraded":"2023-10-01T09:30:10.963933+0000","last_fullsized":"2023-10-01T09:30:10.963933+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:55:51.244125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":346,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964214+0000","last_change":"2023-10-01T09:19:30.091481+0000","last_active":"2023-10-01T09:30:10.964214+0000","last_peered":"2023-10-01T09:30:10.964214+0000","last_clean":"2023-10-01T09:30:10.964214+0000","last_became_active":"2023-10-01T09:19:30.091151+0000","last_became_peered":"2023-10-01T09:19:30.091151+0000","last_unstale":"2023-10-01T09:30:10.964214+0000","last_undegraded":"2023-10-01T09:30:10.964214+0000","last_fullsized":"2023-10-01T09:30:10.964214+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:09.127408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"340'4054","reported_seq":4074,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:16.653424+0000","last_change":"2023-10-01T09:29:24.381082+0000","last_active":"2023-10-01T09:30:16.653424+0000","last_peered":"2023-10-01T09:30:16.653424+0000","last_clean":"2023-10-01T09:30:16.653424+0000","last_became_active":"2023-10-01T09:29:24.380733+0000","last_became_peered":"2023-10-01T09:29:24.380733+0000","last_unstale":"2023-10-01T09:30:16.653424+0000","last_undegraded":"2023-10-01T09:30:16.653424+0000","last_fullsized":"2023-10-01T09:30:16.653424+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4054,"log_dups_size":0,"ondisk_log_size":4054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:27:26.636652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265682944,"num_objects":4054,"num_object_clones":0,"num_object_copies":12162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4054,"num_write_kb":259456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":739,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.971145+0000","last_change":"2023-10-01T09:28:03.317559+0000","last_active":"2023-10-01T09:30:05.971145+0000","last_peered":"2023-10-01T09:30:05.971145+0000","last_clean":"2023-10-01T09:30:05.971145+0000","last_became_active":"2023-10-01T09:28:03.317237+0000","last_became_peered":"2023-10-01T09:28:03.317237+0000","last_unstale":"2023-10-01T09:30:05.971145+0000","last_undegraded":"2023-10-01T09:30:05.971145+0000","last_fullsized":"2023-10-01T09:30:05.971145+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":360,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974766+0000","last_change":"2023-10-01T09:27:01.999110+0000","last_active":"2023-10-01T09:30:10.974766+0000","last_peered":"2023-10-01T09:30:10.974766+0000","last_clean":"2023-10-01T09:30:10.974766+0000","last_became_active":"2023-10-01T09:27:01.998740+0000","last_became_peered":"2023-10-01T09:27:01.998740+0000","last_unstale":"2023-10-01T09:30:10.974766+0000","last_undegraded":"2023-10-01T09:30:10.974766+0000","last_fullsized":"2023-10-01T09:30:10.974766+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:40:08.795106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":314,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963435+0000","last_change":"2023-10-01T09:27:01.998986+0000","last_active":"2023-10-01T09:30:10.963435+0000","last_peered":"2023-10-01T09:30:10.963435+0000","last_clean":"2023-10-01T09:30:10.963435+0000","last_became_active":"2023-10-01T09:27:01.998709+0000","last_became_peered":"2023-10-01T09:27:01.998709+0000","last_unstale":"2023-10-01T09:30:10.963435+0000","last_undegraded":"2023-10-01T09:30:10.963435+0000","last_fullsized":"2023-10-01T09:30:10.963435+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:01:14.296638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"341'4248","reported_seq":4272,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:18.260947+0000","last_change":"2023-10-01T09:29:24.378898+0000","last_active":"2023-10-01T09:30:18.260947+0000","last_peered":"2023-10-01T09:30:18.260947+0000","last_clean":"2023-10-01T09:30:18.260947+0000","last_became_active":"2023-10-01T09:29:24.378624+0000","last_became_peered":"2023-10-01T09:29:24.378624+0000","last_unstale":"2023-10-01T09:30:18.260947+0000","last_undegraded":"2023-10-01T09:30:18.260947+0000","last_fullsized":"2023-10-01T09:30:18.260947+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4248,"log_dups_size":0,"ondisk_log_size":4248,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:12:28.028735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278396928,"num_objects":4248,"num_object_clones":0,"num_object_copies":12744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4248,"num_write_kb":271872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":354,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.980786+0000","last_change":"2023-10-01T09:23:10.615956+0000","last_active":"2023-10-01T09:30:05.980786+0000","last_peered":"2023-10-01T09:30:05.980786+0000","last_clean":"2023-10-01T09:30:05.980786+0000","last_became_active":"2023-10-01T09:23:10.610919+0000","last_became_peered":"2023-10-01T09:23:10.610919+0000","last_unstale":"2023-10-01T09:30:05.980786+0000","last_undegraded":"2023-10-01T09:30:05.980786+0000","last_fullsized":"2023-10-01T09:30:05.980786+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:03:44.294373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":926,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974990+0000","last_change":"2023-10-01T09:29:13.014839+0000","last_active":"2023-10-01T09:30:10.974990+0000","last_peered":"2023-10-01T09:30:10.974990+0000","last_clean":"2023-10-01T09:30:10.974990+0000","last_became_active":"2023-10-01T09:29:13.014139+0000","last_became_peered":"2023-10-01T09:29:13.014139+0000","last_unstale":"2023-10-01T09:30:10.974990+0000","last_undegraded":"2023-10-01T09:30:10.974990+0000","last_fullsized":"2023-10-01T09:30:10.974990+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:23:10.370435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":126,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.975585+0000","last_change":"2023-10-01T09:23:10.612309+0000","last_active":"2023-10-01T09:30:05.975585+0000","last_peered":"2023-10-01T09:30:05.975585+0000","last_clean":"2023-10-01T09:30:05.975585+0000","last_became_active":"2023-10-01T09:23:10.612034+0000","last_became_peered":"2023-10-01T09:23:10.612034+0000","last_unstale":"2023-10-01T09:30:05.975585+0000","last_undegraded":"2023-10-01T09:30:05.975585+0000","last_fullsized":"2023-10-01T09:30:05.975585+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:09:47.963186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"340'3989","reported_seq":4011,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:15.571295+0000","last_change":"2023-10-01T09:29:24.379832+0000","last_active":"2023-10-01T09:30:15.571295+0000","last_peered":"2023-10-01T09:30:15.571295+0000","last_clean":"2023-10-01T09:30:15.571295+0000","last_became_active":"2023-10-01T09:29:24.379540+0000","last_became_peered":"2023-10-01T09:29:24.379540+0000","last_unstale":"2023-10-01T09:30:15.571295+0000","last_undegraded":"2023-10-01T09:30:15.571295+0000","last_fullsized":"2023-10-01T09:30:15.571295+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":3989,"log_dups_size":0,"ondisk_log_size":3989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261423104,"num_objects":3989,"num_object_clones":0,"num_object_copies":11967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3989,"num_write_kb":255296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":53,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.967672+0000","last_change":"2023-10-01T09:29:11.374086+0000","last_active":"2023-10-01T09:30:05.967672+0000","last_peered":"2023-10-01T09:30:05.967672+0000","last_clean":"2023-10-01T09:30:05.967672+0000","last_became_active":"2023-10-01T09:28:02.319106+0000","last_became_peered":"2023-10-01T09:28:02.319106+0000","last_unstale":"2023-10-01T09:30:05.967672+0000","last_undegraded":"2023-10-01T09:30:05.967672+0000","last_fullsized":"2023-10-01T09:30:05.967672+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":338,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964299+0000","last_change":"2023-10-01T09:26:12.291583+0000","last_active":"2023-10-01T09:30:10.964299+0000","last_peered":"2023-10-01T09:30:10.964299+0000","last_clean":"2023-10-01T09:30:10.964299+0000","last_became_active":"2023-10-01T09:26:12.271891+0000","last_became_peered":"2023-10-01T09:26:12.271891+0000","last_unstale":"2023-10-01T09:30:10.964299+0000","last_undegraded":"2023-10-01T09:30:10.964299+0000","last_fullsized":"2023-10-01T09:30:10.964299+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:53:06.889203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":60,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.978787+0000","last_change":"2023-10-01T09:28:02.314867+0000","last_active":"2023-10-01T09:30:05.978787+0000","last_peered":"2023-10-01T09:30:05.978787+0000","last_clean":"2023-10-01T09:30:05.978787+0000","last_became_active":"2023-10-01T09:28:02.314480+0000","last_became_peered":"2023-10-01T09:28:02.314480+0000","last_unstale":"2023-10-01T09:30:05.978787+0000","last_undegraded":"2023-10-01T09:30:05.978787+0000","last_fullsized":"2023-10-01T09:30:05.978787+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":292,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963352+0000","last_change":"2023-10-01T09:26:12.271469+0000","last_active":"2023-10-01T09:30:10.963352+0000","last_peered":"2023-10-01T09:30:10.963352+0000","last_clean":"2023-10-01T09:30:10.963352+0000","last_became_active":"2023-10-01T09:26:12.270972+0000","last_became_peered":"2023-10-01T09:26:12.270972+0000","last_unstale":"2023-10-01T09:30:10.963352+0000","last_undegraded":"2023-10-01T09:30:10.963352+0000","last_fullsized":"2023-10-01T09:30:10.963352+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:45:03.822952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":370,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974922+0000","last_change":"2023-10-01T09:22:40.166927+0000","last_active":"2023-10-01T09:30:10.974922+0000","last_peered":"2023-10-01T09:30:10.974922+0000","last_clean":"2023-10-01T09:30:10.974922+0000","last_became_active":"2023-10-01T09:22:40.166606+0000","last_became_peered":"2023-10-01T09:22:40.166606+0000","last_unstale":"2023-10-01T09:30:10.974922+0000","last_undegraded":"2023-10-01T09:30:10.974922+0000","last_fullsized":"2023-10-01T09:30:10.974922+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:54:18.044207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":68,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.979154+0000","last_change":"2023-10-01T09:28:02.507232+0000","last_active":"2023-10-01T09:30:05.979154+0000","last_peered":"2023-10-01T09:30:05.979154+0000","last_clean":"2023-10-01T09:30:05.979154+0000","last_became_active":"2023-10-01T09:28:02.340776+0000","last_became_peered":"2023-10-01T09:28:02.340776+0000","last_unstale":"2023-10-01T09:30:05.979154+0000","last_undegraded":"2023-10-01T09:30:05.979154+0000","last_fullsized":"2023-10-01T09:30:05.979154+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.978776+0000","last_change":"2023-10-01T09:29:58.359570+0000","last_active":"2023-10-01T09:30:05.978776+0000","last_peered":"2023-10-01T09:30:05.978776+0000","last_clean":"2023-10-01T09:30:05.978776+0000","last_became_active":"2023-10-01T09:29:58.359181+0000","last_became_peered":"2023-10-01T09:29:58.359181+0000","last_unstale":"2023-10-01T09:30:05.978776+0000","last_undegraded":"2023-10-01T09:30:05.978776+0000","last_fullsized":"2023-10-01T09:30:05.978776+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":68,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.979144+0000","last_change":"2023-10-01T09:28:02.551019+0000","last_active":"2023-10-01T09:30:05.979144+0000","last_peered":"2023-10-01T09:30:05.979144+0000","last_clean":"2023-10-01T09:30:05.979144+0000","last_became_active":"2023-10-01T09:28:02.342509+0000","last_became_peered":"2023-10-01T09:28:02.342509+0000","last_unstale":"2023-10-01T09:30:05.979144+0000","last_undegraded":"2023-10-01T09:30:05.979144+0000","last_fullsized":"2023-10-01T09:30:05.979144+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":279,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.974605+0000","last_change":"2023-10-01T09:26:12.309345+0000","last_active":"2023-10-01T09:30:05.974605+0000","last_peered":"2023-10-01T09:30:05.974605+0000","last_clean":"2023-10-01T09:30:05.974605+0000","last_became_active":"2023-10-01T09:26:12.304299+0000","last_became_peered":"2023-10-01T09:26:12.304299+0000","last_unstale":"2023-10-01T09:30:05.974605+0000","last_undegraded":"2023-10-01T09:30:05.974605+0000","last_fullsized":"2023-10-01T09:30:05.974605+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:35:59.915530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":141,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.975043+0000","last_change":"2023-10-01T09:28:02.321395+0000","last_active":"2023-10-01T09:30:05.975043+0000","last_peered":"2023-10-01T09:30:05.975043+0000","last_clean":"2023-10-01T09:30:05.975043+0000","last_became_active":"2023-10-01T09:28:02.321071+0000","last_became_peered":"2023-10-01T09:28:02.321071+0000","last_unstale":"2023-10-01T09:30:05.975043+0000","last_undegraded":"2023-10-01T09:30:05.975043+0000","last_fullsized":"2023-10-01T09:30:05.975043+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:12:45.734920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":117,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-10-01T09:30:05.974079+0000","last_change":"2023-10-01T09:22:40.170676+0000","last_active":"2023-10-01T09:30:05.974079+0000","last_peered":"2023-10-01T09:30:05.974079+0000","last_clean":"2023-10-01T09:30:05.974079+0000","last_became_active":"2023-10-01T09:22:40.170260+0000","last_became_peered":"2023-10-01T09:22:40.170260+0000","last_unstale":"2023-10-01T09:30:05.974079+0000","last_undegraded":"2023-10-01T09:30:05.974079+0000","last_fullsized":"2023-10-01T09:30:05.974079+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:14.693287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":431,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964041+0000","last_change":"2023-10-01T09:27:02.019266+0000","last_active":"2023-10-01T09:30:10.964041+0000","last_peered":"2023-10-01T09:30:10.964041+0000","last_clean":"2023-10-01T09:30:10.964041+0000","last_became_active":"2023-10-01T09:27:02.006744+0000","last_became_peered":"2023-10-01T09:27:02.006744+0000","last_unstale":"2023-10-01T09:30:10.964041+0000","last_undegraded":"2023-10-01T09:30:10.964041+0000","last_fullsized":"2023-10-01T09:30:10.964041+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:39:21.899359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":298,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.973550+0000","last_change":"2023-10-01T09:28:02.343724+0000","last_active":"2023-10-01T09:30:10.973550+0000","last_peered":"2023-10-01T09:30:10.973550+0000","last_clean":"2023-10-01T09:30:10.973550+0000","last_became_active":"2023-10-01T09:28:02.339485+0000","last_became_peered":"2023-10-01T09:28:02.339485+0000","last_unstale":"2023-10-01T09:30:10.973550+0000","last_undegraded":"2023-10-01T09:30:10.973550+0000","last_fullsized":"2023-10-01T09:30:10.973550+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:54:52.086601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":28418,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":22016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42916,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4284153856,"num_objects":65371,"num_object_clones":0,"num_object_copies":196113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":65371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":65371,"num_write_kb":4183744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12773228544,"data_stored":12773228544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":65371,"ondisk_log_size":65371,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":199,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430556,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1867740,"kb_used_data":1566420,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91877412,"statfs":{"total":95995035648,"available":94082469888,"internally_reserved":0,"allocated":1604014080,"data_stored":1603140614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:29:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":3.3220000000000001,"15min":4.2069999999999999},"min":{"1min":0.23999999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":8.8170000000000002,"5min":73.284000000000006,"15min":117.855},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.264,"5min":3.9820000000000002,"15min":4.2229999999999999},"min":{"1min":0.311,"5min":0.249,"15min":0.22500000000000001},"max":{"1min":7.9710000000000001,"5min":73.269000000000005,"15min":120.348},"last":0.88600000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:29:43 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":2.0249999999999999,"15min":3.302},"min":{"1min":0.25800000000000001,"5min":0.246,"15min":0.22700000000000001},"max":{"1min":3.835,"5min":10.611000000000001,"15min":135.023},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.81000000000000005,"5min":2.0470000000000002,"15min":3.1299999999999999},"min":{"1min":0.34100000000000003,"5min":0.28599999999999998,"15min":0.23599999999999999},"max":{"1min":3.1709999999999998,"5min":13.688000000000001,"15min":134.51499999999999},"last":0.81000000000000005}]},{"osd":2,"last update":"Sun Oct 1 09:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":2.1179999999999999,"15min":3.1549999999999998},"min":{"1min":0.28000000000000003,"5min":0.249,"15min":0.249},"max":{"1min":7.9029999999999996,"5min":10.452,"15min":104.837},"last":7.9029999999999996},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.8440000000000001,"15min":3.0259999999999998},"min":{"1min":0.35999999999999999,"5min":0.247,"15min":0.247},"max":{"1min":3.3839999999999999,"5min":12.388999999999999,"15min":106.379},"last":0.52100000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:29:43 2023","interfaces":[{"interface":"back","average":{"1min":4.9340000000000002,"5min":4.7000000000000002,"15min":3.27},"min":{"1min":0.34799999999999998,"5min":0.28199999999999997,"15min":0.26400000000000001},"max":{"1min":80.352999999999994,"5min":164.595,"15min":164.595},"last":0.61099999999999999},{"interface":"front","average":{"1min":4.9740000000000002,"5min":4.7610000000000001,"15min":3.238},"min":{"1min":0.33100000000000002,"5min":0.32200000000000001,"15min":0.27200000000000002},"max":{"1min":80.364000000000004,"5min":164.60499999999999,"15min":164.60499999999999},"last":0.56799999999999995}]},{"osd":4,"last update":"Sun Oct 1 09:29:23 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.53500000000000003,"15min":1.5940000000000001},"min":{"1min":0.30599999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.8420000000000001,"5min":3.8420000000000001,"15min":82.992000000000004},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.752,"5min":0.63500000000000001,"15min":1.6539999999999999},"min":{"1min":0.31900000000000001,"5min":0.26400000000000001,"15min":0.252},"max":{"1min":2.5310000000000001,"5min":3.25,"15min":83.058999999999997},"last":0.57999999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:29:23 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.69399999999999995,"15min":2.1600000000000001},"min":{"1min":0.29599999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.6989999999999998,"5min":4.6989999999999998,"15min":13.265000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.091,"5min":0.752,"15min":2.0640000000000001},"min":{"1min":0.35699999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.7639999999999998,"5min":3.7639999999999998,"15min":13.292999999999999},"last":0.46899999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:29:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.76700000000000002,"15min":0.78900000000000003},"min":{"1min":0.371,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.97099999999999997,"5min":3.306,"15min":3.306},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.69899999999999995,"15min":0.63700000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.87,"5min":2.8300000000000001,"15min":2.8300000000000001},"last":0.501}]}]},{"osd":6,"up_from":294,"seq":1262720385067,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2086352,"kb_used_data":1744648,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":91658800,"statfs":{"total":95995035648,"available":93858611200,"internally_reserved":0,"allocated":1786519552,"data_stored":1785657651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":1.73,"15min":1.9239999999999999},"min":{"1min":0.23799999999999999,"5min":0.222,"15min":0.222},"max":{"1min":3.7999999999999998,"5min":8.9960000000000004,"15min":8.9960000000000004},"last":1.1319999999999999},{"interface":"front","average":{"1min":1.369,"5min":1.6699999999999999,"15min":1.524},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.1470000000000002,"5min":10.329000000000001,"15min":10.329000000000001},"last":3.1880000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.619,"15min":1.7549999999999999},"min":{"1min":0.30299999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.878,"5min":7.7560000000000002,"15min":7.7560000000000002},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.58399999999999996,"5min":1.516,"15min":1.734},"min":{"1min":0.38500000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.3859999999999999,"5min":4.9749999999999996,"15min":4.9749999999999996},"last":0.80900000000000005}]},{"osd":2,"last update":"Sun Oct 1 09:29:51 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":2.4340000000000002,"15min":2.8119999999999998},"min":{"1min":0.29999999999999999,"5min":0.254,"15min":0.254},"max":{"1min":6.1100000000000003,"5min":26.289999999999999,"15min":26.289999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.82299999999999995,"5min":2.5419999999999998,"15min":2.9900000000000002},"min":{"1min":0.371,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.1850000000000001,"5min":26.388999999999999,"15min":26.388999999999999},"last":7.8170000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:29:51 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":2.3370000000000002,"15min":2.4239999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":3.7509999999999999,"5min":10.632999999999999,"15min":10.632999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.498,"15min":1.458},"min":{"1min":0.33400000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.911,"5min":8.8680000000000003,"15min":8.8680000000000003},"last":0.52800000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.44900000000000001,"15min":0.39700000000000002},"min":{"1min":0.28899999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.3100000000000001,"5min":3.1560000000000001,"15min":3.1560000000000001},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.627,"15min":0.56299999999999994},"min":{"1min":0.30099999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.0609999999999999,"5min":4.1319999999999997,"15min":4.1319999999999997},"last":0.41899999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.53500000000000003,"15min":0.46000000000000002},"min":{"1min":0.28799999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.3600000000000001,"5min":4.6200000000000001,"15min":4.6200000000000001},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.65200000000000002,"15min":0.59599999999999997},"min":{"1min":0.31900000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.6440000000000001,"5min":4.0919999999999996,"15min":4.0919999999999996},"last":0.36799999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":4.1740000000000004,"15min":5.1449999999999996},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.93600000000000005,"5min":115.11,"15min":115.11},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.49399999999999999,"5min":4.1159999999999997,"15min":5.0819999999999999},"min":{"1min":0.35399999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.67300000000000004,"5min":114.666,"15min":114.666},"last":0.40500000000000003}]}]},{"osd":1,"up_from":94,"seq":403726926044,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2316500,"kb_used_data":1853132,"kb_used_omap":7,"kb_used_meta":463352,"kb_avail":91428652,"statfs":{"total":95995035648,"available":93622939648,"internally_reserved":0,"allocated":1897607168,"data_stored":1896044547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":474472871},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":3.6000000000000001,"15min":4.0110000000000001},"min":{"1min":0.19600000000000001,"5min":0.191,"15min":0.17499999999999999},"max":{"1min":5.8789999999999996,"5min":37.101999999999997,"15min":174.40199999999999},"last":5.673},{"interface":"front","average":{"1min":1.4279999999999999,"5min":3.3929999999999998,"15min":4.1210000000000004},"min":{"1min":0.188,"5min":0.188,"15min":0.17599999999999999},"max":{"1min":8.4420000000000002,"5min":36.795999999999999,"15min":173.77000000000001},"last":0.78000000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:30:14 2023","interfaces":[{"interface":"back","average":{"1min":3.661,"5min":4.7080000000000002,"15min":3.274},"min":{"1min":0.28499999999999998,"5min":0.215,"15min":0.182},"max":{"1min":56.573,"5min":56.573,"15min":77.325000000000003},"last":5.6989999999999998},{"interface":"front","average":{"1min":4.0460000000000003,"5min":5.2069999999999999,"15min":3.71},"min":{"1min":0.27800000000000002,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":55.930999999999997,"5min":55.930999999999997,"15min":80.361000000000004},"last":0.80100000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:29:46 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":4.649,"15min":3.5019999999999998},"min":{"1min":0.28499999999999998,"5min":0.254,"15min":0.218},"max":{"1min":2.7429999999999999,"5min":66.334000000000003,"15min":99.421000000000006},"last":2.7709999999999999},{"interface":"front","average":{"1min":0.77700000000000002,"5min":4.673,"15min":3.3620000000000001},"min":{"1min":0.27900000000000003,"5min":0.21199999999999999,"15min":0.20999999999999999},"max":{"1min":3.4129999999999998,"5min":69.641000000000005,"15min":99.364999999999995},"last":5.5739999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.6599999999999999,"15min":1.663},"min":{"1min":0.36399999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.234,"5min":10.231,"15min":40.673999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.3080000000000001,"5min":2.2530000000000001,"15min":2.0030000000000001},"min":{"1min":0.29999999999999999,"5min":0.27500000000000002,"15min":0.25800000000000001},"max":{"1min":5.4640000000000004,"5min":18.219000000000001,"15min":40.810000000000002},"last":0.44500000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:30:14 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.6839999999999999,"15min":1.444},"min":{"1min":0.35499999999999998,"5min":0.314,"15min":0.29599999999999999},"max":{"1min":2.665,"5min":10.273,"15min":10.273},"last":0.495},{"interface":"front","average":{"1min":1.0189999999999999,"5min":2.298,"15min":2.2170000000000001},"min":{"1min":0.39000000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.871,"5min":18.265000000000001,"15min":18.265000000000001},"last":0.371}]},{"osd":6,"last update":"Sun Oct 1 09:29:54 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":2.5289999999999999,"15min":2.9889999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.0099999999999998,"5min":10.196999999999999,"15min":10.196999999999999},"last":0.749},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.7150000000000001,"15min":1.9239999999999999},"min":{"1min":0.40300000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.98299999999999998,"5min":7.2679999999999998,"15min":7.2679999999999998},"last":0.52000000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.7669999999999999,"15min":1.829},"min":{"1min":0.45600000000000002,"5min":0.316,"15min":0.316},"max":{"1min":6.2610000000000001,"5min":10.305999999999999,"15min":23.710999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.0329999999999999,"5min":2.1419999999999999,"15min":1.9790000000000001},"min":{"1min":0.436,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.2599999999999998,"5min":18.291,"15min":23.736999999999998},"last":5.5259999999999998}]}]},{"osd":0,"up_from":89,"seq":382252089578,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2424672,"kb_used_data":1862424,"kb_used_omap":8,"kb_used_meta":562231,"kb_avail":91320480,"statfs":{"total":95995035648,"available":93512171520,"internally_reserved":0,"allocated":1907122176,"data_stored":1905551758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":575725506},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:29:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":2.5539999999999998,"15min":2.7890000000000001},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":7.1529999999999996,"5min":47.244999999999997,"15min":187.41399999999999},"last":3.052},{"interface":"front","average":{"1min":2.5569999999999999,"5min":3.98,"15min":3.4630000000000001},"min":{"1min":0.25900000000000001,"5min":0.21199999999999999,"15min":0.193},"max":{"1min":21.968,"5min":48.042000000000002,"15min":187.78700000000001},"last":5.6219999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:30:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":1.974,"15min":2.113},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":4.6539999999999999,"5min":17.731000000000002,"15min":39.048999999999999},"last":3.0840000000000001},{"interface":"front","average":{"1min":1.8959999999999999,"5min":3.8540000000000001,"15min":2.9790000000000001},"min":{"1min":0.27200000000000002,"5min":0.26600000000000001,"15min":0.24299999999999999},"max":{"1min":6.3979999999999997,"5min":22.07,"15min":38.945999999999998},"last":5.4649999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:29:38 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":5.1539999999999999,"15min":3.2829999999999999},"min":{"1min":0.32700000000000001,"5min":0.249,"15min":0.23400000000000001},"max":{"1min":11.241,"5min":139.85300000000001,"15min":139.85300000000001},"last":3.194},{"interface":"front","average":{"1min":2.294,"5min":6.3200000000000003,"15min":3.8799999999999999},"min":{"1min":0.313,"5min":0.219,"15min":0.219},"max":{"1min":7.048,"5min":139.70099999999999,"15min":139.70099999999999},"last":2.823}]},{"osd":4,"last update":"Sun Oct 1 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2120000000000002,"5min":2.3660000000000001,"15min":2.488},"min":{"1min":0.32100000000000001,"5min":0.28199999999999997,"15min":0.27300000000000002},"max":{"1min":18.600999999999999,"5min":18.600999999999999,"15min":112.52200000000001},"last":5.5880000000000001},{"interface":"front","average":{"1min":1.2929999999999999,"5min":2.387,"15min":2.573},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26900000000000002},"max":{"1min":5.7130000000000001,"5min":28.035,"15min":113.124},"last":2.8450000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:29:19 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":2.4359999999999999,"15min":1.9850000000000001},"min":{"1min":0.32000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":6.1890000000000001,"5min":28.074000000000002,"15min":28.074000000000002},"last":2.8919999999999999},{"interface":"front","average":{"1min":0.69299999999999995,"5min":1.1259999999999999,"15min":1.6299999999999999},"min":{"1min":0.312,"5min":0.31,"15min":0.31},"max":{"1min":2.6629999999999998,"5min":11.351000000000001,"15min":11.351000000000001},"last":3.109}]},{"osd":6,"last update":"Sun Oct 1 09:29:58 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.2669999999999999,"15min":1.4079999999999999},"min":{"1min":0.42099999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.0219999999999998,"5min":11.387,"15min":11.387},"last":3.1419999999999999},{"interface":"front","average":{"1min":1.034,"5min":2.4060000000000001,"15min":2.472},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.8370000000000002,"5min":18.521000000000001,"15min":18.521000000000001},"last":5.524}]},{"osd":7,"last update":"Sun Oct 1 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":1.258,"15min":2.0939999999999999},"min":{"1min":0.35699999999999998,"5min":0.34499999999999997,"15min":0.29199999999999998},"max":{"1min":2.597,"5min":11.413,"15min":98.283000000000001},"last":3.1619999999999999},{"interface":"front","average":{"1min":2.29,"5min":2.6789999999999998,"15min":2.5680000000000001},"min":{"1min":0.373,"5min":0.36099999999999999,"15min":0.312},"max":{"1min":18.565999999999999,"5min":18.565999999999999,"15min":98.311999999999998},"last":5.5540000000000003}]}]},{"osd":2,"up_from":99,"seq":425201762517,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2327152,"kb_used_data":1790440,"kb_used_omap":11,"kb_used_meta":536692,"kb_avail":91418000,"statfs":{"total":95995035648,"available":93612032000,"internally_reserved":0,"allocated":1833410560,"data_stored":1831886314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":549573495},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:29:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":7.5069999999999997,"15min":4.5220000000000002},"min":{"1min":0.23499999999999999,"5min":0.20000000000000001,"15min":0.17599999999999999},"max":{"1min":8.2720000000000002,"5min":138.107,"15min":138.107},"last":12.837999999999999},{"interface":"front","average":{"1min":1.675,"5min":8.8759999999999994,"15min":5.3390000000000004},"min":{"1min":0.26600000000000001,"5min":0.20699999999999999,"15min":0.20100000000000001},"max":{"1min":8.3230000000000004,"5min":138.20599999999999,"15min":138.20599999999999},"last":0.57599999999999996}]},{"osd":1,"last update":"Sun Oct 1 09:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":8.8900000000000006,"15min":4.9950000000000001},"min":{"1min":0.214,"5min":0.19500000000000001,"15min":0.193},"max":{"1min":7.7939999999999996,"5min":209.922,"15min":209.922},"last":5.8710000000000004},{"interface":"front","average":{"1min":3.0299999999999998,"5min":10.895,"15min":5.8230000000000004},"min":{"1min":0.36199999999999999,"5min":0.222,"15min":0.20899999999999999},"max":{"1min":13.172000000000001,"5min":212.952,"15min":212.952},"last":0.70399999999999996}]},{"osd":3,"last update":"Sun Oct 1 09:29:41 2023","interfaces":[{"interface":"back","average":{"1min":3.718,"5min":7.5350000000000001,"15min":5.2619999999999996},"min":{"1min":0.26100000000000001,"5min":0.218,"15min":0.19600000000000001},"max":{"1min":52.115000000000002,"5min":138.85900000000001,"15min":253.238},"last":1.0389999999999999},{"interface":"front","average":{"1min":4.0720000000000001,"5min":7.2210000000000001,"15min":5.0510000000000002},"min":{"1min":0.32300000000000001,"5min":0.245,"15min":0.22600000000000001},"max":{"1min":53.067,"5min":136.66399999999999,"15min":252.54900000000001},"last":6.0519999999999996}]},{"osd":4,"last update":"Sun Oct 1 09:29:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":2.6840000000000002,"15min":2.0329999999999999},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.27000000000000002},"max":{"1min":6.9669999999999996,"5min":54.231000000000002,"15min":54.231000000000002},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.7929999999999999,"5min":2.468,"15min":2.0600000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":6.8319999999999999,"5min":53.133000000000003,"15min":53.133000000000003},"last":1}]},{"osd":5,"last update":"Sun Oct 1 09:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.2989999999999999,"15min":2.0059999999999998},"min":{"1min":0.38600000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6579999999999999,"5min":13.895,"15min":13.895},"last":5.774},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.1230000000000002,"15min":2.1840000000000002},"min":{"1min":0.371,"5min":0.309,"15min":0.309},"max":{"1min":6.9059999999999997,"5min":16.943000000000001,"15min":16.943000000000001},"last":0.61299999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:29:50 2023","interfaces":[{"interface":"back","average":{"1min":12.066000000000001,"5min":4.0650000000000004,"15min":2.7469999999999999},"min":{"1min":0.38200000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":225.41499999999999,"5min":225.41499999999999,"15min":225.41499999999999},"last":0.95999999999999996},{"interface":"front","average":{"1min":11.970000000000001,"5min":3.4119999999999999,"15min":2.0579999999999998},"min":{"1min":0.46899999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":225.02000000000001,"5min":225.02000000000001,"15min":225.02000000000001},"last":5.8179999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:29:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":2.2570000000000001,"15min":2.3660000000000001},"min":{"1min":0.40400000000000003,"5min":0.308,"15min":0.308},"max":{"1min":3.6920000000000002,"5min":51.840000000000003,"15min":109.506},"last":5.8460000000000001},{"interface":"front","average":{"1min":2.222,"5min":3.1219999999999999,"15min":2.762},"min":{"1min":0.39800000000000002,"5min":0.34499999999999997,"15min":0.33600000000000002},"max":{"1min":6.9370000000000003,"5min":51.908999999999999,"15min":109.58799999999999},"last":0.67400000000000004}]}]},{"osd":3,"up_from":104,"seq":446676598991,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2334052,"kb_used_data":1809500,"kb_used_omap":3,"kb_used_meta":524540,"kb_avail":91411100,"statfs":{"total":95995035648,"available":93604966400,"internally_reserved":0,"allocated":1852928000,"data_stored":1851349418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":537129703},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":6.5069999999999997,"15min":5.2270000000000003},"min":{"1min":0.29999999999999999,"5min":0.18099999999999999,"15min":0.17100000000000001},"max":{"1min":3.6419999999999999,"5min":128.47800000000001,"15min":199.422},"last":1.9950000000000001},{"interface":"front","average":{"1min":0.96599999999999997,"5min":6.1029999999999998,"15min":5.2610000000000001},"min":{"1min":0.23200000000000001,"5min":0.184,"15min":0.184},"max":{"1min":4.0839999999999996,"5min":127.15600000000001,"15min":199.91200000000001},"last":2.0289999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":4.6109999999999998,"15min":4.8600000000000003},"min":{"1min":0.26600000000000001,"5min":0.23999999999999999,"15min":0.21099999999999999},"max":{"1min":9.0630000000000006,"5min":120.79300000000001,"15min":228.60499999999999},"last":2.5870000000000002},{"interface":"front","average":{"1min":1.1759999999999999,"5min":5.8259999999999996,"15min":5.1189999999999998},"min":{"1min":0.29099999999999998,"5min":0.23400000000000001,"15min":0.215},"max":{"1min":3.9359999999999999,"5min":121.455,"15min":228.28399999999999},"last":2.5}]},{"osd":2,"last update":"Sun Oct 1 09:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":5.343,"15min":3.3700000000000001},"min":{"1min":0.27500000000000002,"5min":0.22800000000000001,"15min":0.224},"max":{"1min":6.0579999999999998,"5min":62.909999999999997,"15min":62.909999999999997},"last":2.266},{"interface":"front","average":{"1min":0.84599999999999997,"5min":6.165,"15min":3.8100000000000001},"min":{"1min":0.35599999999999998,"5min":0.255,"15min":0.20399999999999999},"max":{"1min":3.1850000000000001,"5min":78.594999999999999,"15min":78.594999999999999},"last":1.698}]},{"osd":4,"last update":"Sun Oct 1 09:29:45 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":1.97,"15min":2.7989999999999999},"min":{"1min":0.46700000000000003,"5min":0.29799999999999999,"15min":0.25900000000000001},"max":{"1min":3.1040000000000001,"5min":16,"15min":154.00399999999999},"last":0.44},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.8859999999999999,"15min":3.04},"min":{"1min":0.371,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.3079999999999998,"5min":15.257999999999999,"15min":156.87700000000001},"last":2.0609999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:30:14 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":1.9330000000000001,"15min":1.8089999999999999},"min":{"1min":0.33200000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.3529999999999998,"5min":15.351000000000001,"15min":15.351000000000001},"last":2.0880000000000001},{"interface":"front","average":{"1min":2.024,"5min":1.962,"15min":2.9260000000000002},"min":{"1min":0.39800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.4269999999999996,"5min":15.382,"15min":36.984000000000002},"last":2.1070000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":2.9180000000000001,"15min":3.0720000000000001},"min":{"1min":0.39900000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.0960000000000001,"5min":18.361000000000001,"15min":18.361000000000001},"last":1.7549999999999999},{"interface":"front","average":{"1min":0.85299999999999998,"5min":2.7000000000000002,"15min":3.028},"min":{"1min":0.40899999999999997,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.948,"5min":18.332000000000001,"15min":18.332000000000001},"last":1.8260000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:29:45 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":3.5,"15min":2.698},"min":{"1min":0.42999999999999999,"5min":0.309,"15min":0.309},"max":{"1min":2.8580000000000001,"5min":52.006,"15min":79.412999999999997},"last":1.79},{"interface":"front","average":{"1min":0.94499999999999995,"5min":2.8130000000000002,"15min":2.5289999999999999},"min":{"1min":0.503,"5min":0.34200000000000003,"15min":0.308},"max":{"1min":3.1459999999999999,"5min":51.890999999999998,"15min":79.527000000000001},"last":0.40500000000000003}]}]},{"osd":4,"up_from":0,"seq":128849019241,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":227,"seq":974957576292,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2343236,"kb_used_data":2015804,"kb_used_omap":8,"kb_used_meta":327415,"kb_avail":91401916,"statfs":{"total":95995035648,"available":93595561984,"internally_reserved":0,"allocated":2064183296,"data_stored":2063329240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":335273732},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:29:14 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":4.3330000000000002,"15min":2.7690000000000001},"min":{"1min":0.248,"5min":0.219,"15min":0.219},"max":{"1min":3.5990000000000002,"5min":169.517,"15min":169.517},"last":1.1879999999999999},{"interface":"front","average":{"1min":2.4540000000000002,"5min":4.617,"15min":3.2709999999999999},"min":{"1min":0.25,"5min":0.219,"15min":0.219},"max":{"1min":16.678999999999998,"5min":169.69900000000001,"15min":169.69900000000001},"last":1.077}]},{"osd":1,"last update":"Sun Oct 1 09:29:14 2023","interfaces":[{"interface":"back","average":{"1min":1.6890000000000001,"5min":4.375,"15min":2.8940000000000001},"min":{"1min":0.26700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":9.4390000000000001,"5min":115.976,"15min":115.976},"last":5.6159999999999997},{"interface":"front","average":{"1min":1.252,"5min":3.899,"15min":3.351},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.253},"max":{"1min":9.9610000000000003,"5min":115.732,"15min":115.732},"last":0.26200000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:29:14 2023","interfaces":[{"interface":"back","average":{"1min":1.5129999999999999,"5min":2.0529999999999999,"15min":2.6589999999999998},"min":{"1min":0.32100000000000001,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":9.0540000000000003,"5min":17.998000000000001,"15min":44.615000000000002},"last":1.887},{"interface":"front","average":{"1min":1.8839999999999999,"5min":2.3540000000000001,"15min":2.0939999999999999},"min":{"1min":0.29899999999999999,"5min":0.26700000000000002,"15min":0.26600000000000001},"max":{"1min":9.9440000000000008,"5min":17.652000000000001,"15min":44.511000000000003},"last":1.9790000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:29:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":6.3070000000000004,"15min":3.8069999999999999},"min":{"1min":0.34100000000000003,"5min":0.307,"15min":0.307},"max":{"1min":8.4700000000000006,"5min":214.73599999999999,"15min":214.73599999999999},"last":0.97499999999999998},{"interface":"front","average":{"1min":2.0409999999999999,"5min":6.6230000000000002,"15min":3.8940000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27900000000000003},"max":{"1min":7.7999999999999998,"5min":214.86699999999999,"15min":214.86699999999999},"last":0.35699999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:29:14 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.48299999999999998,"15min":1.359},"min":{"1min":0.30599999999999999,"5min":0.26200000000000001,"15min":0.248},"max":{"1min":2.8029999999999999,"5min":2.8029999999999999,"15min":7.532},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48999999999999999,"15min":1.147},"min":{"1min":0.33100000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.163,"5min":2.2810000000000001,"15min":4.117},"last":0.316}]},{"osd":6,"last update":"Sun Oct 1 09:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.54800000000000004,"15min":0.502},"min":{"1min":0.32000000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.778,"5min":2.778,"15min":2.778},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.59699999999999998,"15min":0.51100000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.72,"5min":4.806,"15min":4.806},"last":0.32400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:29:14 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":1.3320000000000001,"15min":1.5169999999999999},"min":{"1min":0.35799999999999998,"5min":0.252,"15min":0.252},"max":{"1min":1.2370000000000001,"5min":83.683999999999997,"15min":83.683999999999997},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":1.3200000000000001,"15min":1.835},"min":{"1min":0.39300000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.1949999999999998,"5min":83.495000000000005,"15min":83.495000000000005},"last":0.42299999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1911029760,"data_stored":1911029760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1899364352,"data_stored":1899364352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1829371904,"data_stored":1829371904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1854799872,"data_stored":1854799872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2085486592,"data_stored":2085486592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1592983552,"data_stored":1592983552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1600192512,"data_stored":1600192512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:30:23.369 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:30:23.370 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 50 16 71254 71238 89.0133 53.3125 0.0215002 0.0112293 2023-10-01T09:30:23.395 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:23.427 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:23.494 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:23.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:23.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:23.601 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:23.709 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:23.745 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:23.780 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:30:23.813 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:23.837 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:23.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:23.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:23.953 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:24.036 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:24.245 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:24.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:24.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:24.354 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:24.367 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 51 16 72128 72112 88.3391 54.625 0.0115753 0.0113142 2023-10-01T09:30:24.389 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:24.456 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:24.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:24.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:24.561 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:24.603 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:24.666 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:24.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:24.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:24.720 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:24.770 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:24.875 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:24.957 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:30:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:30:24] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.33.4" 2023-10-01T09:30:24.958 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:24 smithi190 ceph-mon[160435]: pgmap v933: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 988 op/s 2023-10-01T09:30:24.960 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:24 smithi138 ceph-mon[185886]: pgmap v933: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 988 op/s 2023-10-01T09:30:25.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:24 smithi138 ceph-mon[192513]: pgmap v933: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 988 op/s 2023-10-01T09:30:25.368 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 52 16 72950 72934 87.6283 51.375 0.00350589 0.0114001 2023-10-01T09:30:25.937 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:25.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:25.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:26.048 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:26.063 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:30:26.063 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:30:26.136 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:26.299 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:26.299 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:30:26.299 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:26 smithi138 ceph-mon[185886]: pgmap v934: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-01T09:30:26.368 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 53 16 73739 73723 86.9042 49.3125 0.0174707 0.0115012 2023-10-01T09:30:26.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:26.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:30:26.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:26 smithi190 ceph-mon[160435]: pgmap v934: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-01T09:30:26.564 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:26.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:26.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:30:26.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:26 smithi138 ceph-mon[192513]: pgmap v934: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 60 MiB/s wr, 960 op/s 2023-10-01T09:30:26.672 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:26.711 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:26.809 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:30:26.824 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:26.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:26.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:26.933 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:27.036 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:27.296 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:30:27.297 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:27 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1558153953' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:27.369 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 54 16 74535 74519 86.2163 49.75 0.00663208 0.0115905 2023-10-01T09:30:27.554 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:27 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1558153953' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:27.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:27 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1558153953' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:27.784 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:30:27.785 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:30:27.785 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:30:27.785 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:30:27.785 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:30:27.785 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:30:27.785 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:30:27.785 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:30:27.785 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:30:27.785 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:30:27.786 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:30:27.786 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:30:27.786 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:30:27.786 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:30:27.786 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:30:27.786 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:30:28.107 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:28.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:28.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:28.215 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:28.248 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:28.372 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 55 16 75309 75293 85.5239 48.375 0.00745817 0.0116866 2023-10-01T09:30:28.430 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:28.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:28.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:28.538 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:28.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:28 smithi190 ceph-mon[160435]: pgmap v935: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 940 op/s 2023-10-01T09:30:28.624 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:28.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:28 smithi138 ceph-mon[192513]: pgmap v935: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 940 op/s 2023-10-01T09:30:28.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:28 smithi138 ceph-mon[185886]: pgmap v935: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 59 MiB/s wr, 940 op/s 2023-10-01T09:30:28.878 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:30:28.878 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:30:28.879 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:30:28.879 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:30:28.879 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:30:29.055 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:29.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:29.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:29.166 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:29.218 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:29.372 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 56 16 76045 76029 84.8182 46 0.0140708 0.0117859 2023-10-01T09:30:29.417 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:29.435 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:30:29.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:29.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:29.538 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:29.577 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:29.633 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:30:29.653 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:29.655 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:29 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2992330719' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:30:29.655 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:29 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:30:29.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:29.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:29.758 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:29.790 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:29.802 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:30:29.804 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:30:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:29 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2992330719' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:30:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:29 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:30:29.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:29 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2992330719' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:30:29.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:29 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:30:29.878 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:29.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:29.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:29.985 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:30:30.071 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:30.135 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:30:30.375 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 57 16 76926 76910 84.2911 55.0625 0.0161382 0.011858 2023-10-01T09:30:30.528 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:30:30.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:30.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:30:30.633 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:30.667 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:30 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:30:30.668 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:30 smithi190 ceph-mon[160435]: failed to encode map e342 with expected crc 2023-10-01T09:30:30.668 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:30 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:30:30.668 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:30 smithi190 ceph-mon[160435]: osdmap e342: 8 total, 8 up, 7 in 2023-10-01T09:30:30.668 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:30 smithi190 ceph-mon[160435]: failed to encode map e342 with expected crc 2023-10-01T09:30:30.668 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:30 smithi190 ceph-mon[160435]: failed to encode map e342 with expected crc 2023-10-01T09:30:30.668 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:30 smithi190 ceph-mon[160435]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:30:30.668 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:30 smithi190 ceph-mon[160435]: pgmap v937: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-01T09:30:30.670 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:30.714 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:30.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:30.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:30.818 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:30.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:30:30.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[185886]: failed to encode map e342 with expected crc 2023-10-01T09:30:30.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:30:30.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[185886]: osdmap e342: 8 total, 8 up, 7 in 2023-10-01T09:30:30.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[185886]: failed to encode map e342 with expected crc 2023-10-01T09:30:30.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[185886]: failed to encode map e342 with expected crc 2023-10-01T09:30:30.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[185886]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:30:30.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[185886]: pgmap v937: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-01T09:30:30.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:30:30.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[192513]: failed to encode map e342 with expected crc 2023-10-01T09:30:30.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:30:30.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[192513]: osdmap e342: 8 total, 8 up, 7 in 2023-10-01T09:30:30.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[192513]: failed to encode map e342 with expected crc 2023-10-01T09:30:30.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[192513]: failed to encode map e342 with expected crc 2023-10-01T09:30:30.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[192513]: from='client.35763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:30:30.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:30 smithi138 ceph-mon[192513]: pgmap v937: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 901 op/s 2023-10-01T09:30:30.858 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:31.132 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:30:31.177 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:31.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:31.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:31.211 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":936,"stamp":"2023-10-01T09:30:29.410507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4880013235,"num_objects":74418,"num_object_clones":0,"num_object_copies":223254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74418,"num_whiteouts":0,"num_read":6259,"num_read_kb":12659,"num_write":75496,"num_write_kb":4776101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":310,"num_bytes_recovered":3570664,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75560,"ondisk_log_size":75560,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17465784,"kb_used_data":14344448,"kb_used_omap":45,"kb_used_meta":3121234,"kb_avail":638750280,"statfs":{"total":671965249536,"available":654080286720,"internally_reserved":0,"allocated":14688714752,"data_stored":14679880186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46344,"internal_metadata":3196144376},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":751566848,"num_objects":11468,"num_object_clones":0,"num_object_copies":34404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11468,"num_write_kb":733952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.431492"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":223,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974403+0000","last_change":"2023-10-01T09:26:12.273466+0000","last_active":"2023-10-01T09:30:10.974403+0000","last_peered":"2023-10-01T09:30:10.974403+0000","last_clean":"2023-10-01T09:30:10.974403+0000","last_became_active":"2023-10-01T09:26:12.272944+0000","last_became_peered":"2023-10-01T09:26:12.272944+0000","last_unstale":"2023-10-01T09:30:10.974403+0000","last_undegraded":"2023-10-01T09:30:10.974403+0000","last_fullsized":"2023-10-01T09:30:10.974403+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:40:41.939800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":145,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:11.676552+0000","last_change":"2023-10-01T09:28:02.324629+0000","last_active":"2023-10-01T09:30:11.676552+0000","last_peered":"2023-10-01T09:30:11.676552+0000","last_clean":"2023-10-01T09:30:11.676552+0000","last_became_active":"2023-10-01T09:28:02.324019+0000","last_became_peered":"2023-10-01T09:28:02.324019+0000","last_unstale":"2023-10-01T09:30:11.676552+0000","last_undegraded":"2023-10-01T09:30:11.676552+0000","last_fullsized":"2023-10-01T09:30:11.676552+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:00:43.440938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":308,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.973895+0000","last_change":"2023-10-01T09:28:02.338915+0000","last_active":"2023-10-01T09:30:10.973895+0000","last_peered":"2023-10-01T09:30:10.973895+0000","last_clean":"2023-10-01T09:30:10.973895+0000","last_became_active":"2023-10-01T09:28:02.338589+0000","last_became_peered":"2023-10-01T09:28:02.338589+0000","last_unstale":"2023-10-01T09:30:10.973895+0000","last_undegraded":"2023-10-01T09:30:10.973895+0000","last_fullsized":"2023-10-01T09:30:10.973895+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:18:19.992078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":356,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964371+0000","last_change":"2023-10-01T09:29:58.372577+0000","last_active":"2023-10-01T09:30:10.964371+0000","last_peered":"2023-10-01T09:30:10.964371+0000","last_clean":"2023-10-01T09:30:10.964371+0000","last_became_active":"2023-10-01T09:29:58.372218+0000","last_became_peered":"2023-10-01T09:29:58.372218+0000","last_unstale":"2023-10-01T09:30:10.964371+0000","last_undegraded":"2023-10-01T09:30:10.964371+0000","last_fullsized":"2023-10-01T09:30:10.964371+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:04:08.376752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":118,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.966224+0000","last_change":"2023-10-01T09:22:40.172183+0000","last_active":"2023-10-01T09:30:10.966224+0000","last_peered":"2023-10-01T09:30:10.966224+0000","last_clean":"2023-10-01T09:30:10.966224+0000","last_became_active":"2023-10-01T09:22:40.170164+0000","last_became_peered":"2023-10-01T09:22:40.170164+0000","last_unstale":"2023-10-01T09:30:10.966224+0000","last_undegraded":"2023-10-01T09:30:10.966224+0000","last_fullsized":"2023-10-01T09:30:10.966224+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:33:10.407999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":45,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.961141+0000","last_change":"2023-10-01T09:28:02.362383+0000","last_active":"2023-10-01T09:30:10.961141+0000","last_peered":"2023-10-01T09:30:10.961141+0000","last_clean":"2023-10-01T09:30:10.961141+0000","last_became_active":"2023-10-01T09:28:02.362044+0000","last_became_peered":"2023-10-01T09:28:02.362044+0000","last_unstale":"2023-10-01T09:30:10.961141+0000","last_undegraded":"2023-10-01T09:30:10.961141+0000","last_fullsized":"2023-10-01T09:30:10.961141+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":378,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:17.991997+0000","last_change":"2023-10-01T09:27:01.992338+0000","last_active":"2023-10-01T09:30:17.991997+0000","last_peered":"2023-10-01T09:30:17.991997+0000","last_clean":"2023-10-01T09:30:17.991997+0000","last_became_active":"2023-10-01T09:27:01.989050+0000","last_became_peered":"2023-10-01T09:27:01.989050+0000","last_unstale":"2023-10-01T09:30:17.991997+0000","last_undegraded":"2023-10-01T09:30:17.991997+0000","last_fullsized":"2023-10-01T09:30:17.991997+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:43:29.856456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":376,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.969900+0000","last_change":"2023-10-01T09:28:02.347509+0000","last_active":"2023-10-01T09:30:10.969900+0000","last_peered":"2023-10-01T09:30:10.969900+0000","last_clean":"2023-10-01T09:30:10.969900+0000","last_became_active":"2023-10-01T09:28:02.344509+0000","last_became_peered":"2023-10-01T09:28:02.344509+0000","last_unstale":"2023-10-01T09:30:10.969900+0000","last_undegraded":"2023-10-01T09:30:10.969900+0000","last_fullsized":"2023-10-01T09:30:10.969900+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:04:00.983057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":52,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.970536+0000","last_change":"2023-10-01T09:27:01.970412+0000","last_active":"2023-10-01T09:30:10.970536+0000","last_peered":"2023-10-01T09:30:10.970536+0000","last_clean":"2023-10-01T09:30:10.970536+0000","last_became_active":"2023-10-01T09:27:01.970108+0000","last_became_peered":"2023-10-01T09:27:01.970108+0000","last_unstale":"2023-10-01T09:30:10.970536+0000","last_undegraded":"2023-10-01T09:30:10.970536+0000","last_fullsized":"2023-10-01T09:30:10.970536+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":393,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964981+0000","last_change":"2023-10-01T09:28:02.322837+0000","last_active":"2023-10-01T09:30:10.964981+0000","last_peered":"2023-10-01T09:30:10.964981+0000","last_clean":"2023-10-01T09:30:10.964981+0000","last_became_active":"2023-10-01T09:28:02.322536+0000","last_became_peered":"2023-10-01T09:28:02.322536+0000","last_unstale":"2023-10-01T09:30:10.964981+0000","last_undegraded":"2023-10-01T09:30:10.964981+0000","last_fullsized":"2023-10-01T09:30:10.964981+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:48:11.306512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":125,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963035+0000","last_change":"2023-10-01T09:26:12.415219+0000","last_active":"2023-10-01T09:30:10.963035+0000","last_peered":"2023-10-01T09:30:10.963035+0000","last_clean":"2023-10-01T09:30:10.963035+0000","last_became_active":"2023-10-01T09:26:12.286669+0000","last_became_peered":"2023-10-01T09:26:12.286669+0000","last_unstale":"2023-10-01T09:30:10.963035+0000","last_undegraded":"2023-10-01T09:30:10.963035+0000","last_fullsized":"2023-10-01T09:30:10.963035+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":368,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.973972+0000","last_change":"2023-10-01T09:28:02.342661+0000","last_active":"2023-10-01T09:30:10.973972+0000","last_peered":"2023-10-01T09:30:10.973972+0000","last_clean":"2023-10-01T09:30:10.973972+0000","last_became_active":"2023-10-01T09:28:02.342358+0000","last_became_peered":"2023-10-01T09:28:02.342358+0000","last_unstale":"2023-10-01T09:30:10.973972+0000","last_undegraded":"2023-10-01T09:30:10.973972+0000","last_fullsized":"2023-10-01T09:30:10.973972+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:29:30.624568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":349,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.962944+0000","last_change":"2023-10-01T09:28:02.511577+0000","last_active":"2023-10-01T09:30:10.962944+0000","last_peered":"2023-10-01T09:30:10.962944+0000","last_clean":"2023-10-01T09:30:10.962944+0000","last_became_active":"2023-10-01T09:28:02.318805+0000","last_became_peered":"2023-10-01T09:28:02.318805+0000","last_unstale":"2023-10-01T09:30:10.962944+0000","last_undegraded":"2023-10-01T09:30:10.962944+0000","last_fullsized":"2023-10-01T09:30:10.962944+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":45,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.961076+0000","last_change":"2023-10-01T09:28:02.454516+0000","last_active":"2023-10-01T09:30:10.961076+0000","last_peered":"2023-10-01T09:30:10.961076+0000","last_clean":"2023-10-01T09:30:10.961076+0000","last_became_active":"2023-10-01T09:28:02.454098+0000","last_became_peered":"2023-10-01T09:28:02.454098+0000","last_unstale":"2023-10-01T09:30:10.961076+0000","last_undegraded":"2023-10-01T09:30:10.961076+0000","last_fullsized":"2023-10-01T09:30:10.961076+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":399,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.962826+0000","last_change":"2023-10-01T09:28:02.505692+0000","last_active":"2023-10-01T09:30:10.962826+0000","last_peered":"2023-10-01T09:30:10.962826+0000","last_clean":"2023-10-01T09:30:10.962826+0000","last_became_active":"2023-10-01T09:28:02.318771+0000","last_became_peered":"2023-10-01T09:28:02.318771+0000","last_unstale":"2023-10-01T09:30:10.962826+0000","last_undegraded":"2023-10-01T09:30:10.962826+0000","last_fullsized":"2023-10-01T09:30:10.962826+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":325,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964853+0000","last_change":"2023-10-01T09:19:57.493670+0000","last_active":"2023-10-01T09:30:10.964853+0000","last_peered":"2023-10-01T09:30:10.964853+0000","last_clean":"2023-10-01T09:30:10.964853+0000","last_became_active":"2023-10-01T09:19:57.493349+0000","last_became_peered":"2023-10-01T09:19:57.493349+0000","last_unstale":"2023-10-01T09:30:10.964853+0000","last_undegraded":"2023-10-01T09:30:10.964853+0000","last_fullsized":"2023-10-01T09:30:10.964853+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:26:56.468779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":331,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964896+0000","last_change":"2023-10-01T09:23:10.607388+0000","last_active":"2023-10-01T09:30:10.964896+0000","last_peered":"2023-10-01T09:30:10.964896+0000","last_clean":"2023-10-01T09:30:10.964896+0000","last_became_active":"2023-10-01T09:23:10.607084+0000","last_became_peered":"2023-10-01T09:23:10.607084+0000","last_unstale":"2023-10-01T09:30:10.964896+0000","last_undegraded":"2023-10-01T09:30:10.964896+0000","last_fullsized":"2023-10-01T09:30:10.964896+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:19.669393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":52,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.970631+0000","last_change":"2023-10-01T09:27:01.978738+0000","last_active":"2023-10-01T09:30:10.970631+0000","last_peered":"2023-10-01T09:30:10.970631+0000","last_clean":"2023-10-01T09:30:10.970631+0000","last_became_active":"2023-10-01T09:27:01.978024+0000","last_became_peered":"2023-10-01T09:27:01.978024+0000","last_unstale":"2023-10-01T09:30:10.970631+0000","last_undegraded":"2023-10-01T09:30:10.970631+0000","last_fullsized":"2023-10-01T09:30:10.970631+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":343,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.962508+0000","last_change":"2023-10-01T09:28:02.314047+0000","last_active":"2023-10-01T09:30:10.962508+0000","last_peered":"2023-10-01T09:30:10.962508+0000","last_clean":"2023-10-01T09:30:10.962508+0000","last_became_active":"2023-10-01T09:28:02.313729+0000","last_became_peered":"2023-10-01T09:28:02.313729+0000","last_unstale":"2023-10-01T09:30:10.962508+0000","last_undegraded":"2023-10-01T09:30:10.962508+0000","last_fullsized":"2023-10-01T09:30:10.962508+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":313,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:17.992871+0000","last_change":"2023-10-01T09:23:10.611400+0000","last_active":"2023-10-01T09:30:17.992871+0000","last_peered":"2023-10-01T09:30:17.992871+0000","last_clean":"2023-10-01T09:30:17.992871+0000","last_became_active":"2023-10-01T09:23:10.610928+0000","last_became_peered":"2023-10-01T09:23:10.610928+0000","last_unstale":"2023-10-01T09:30:17.992871+0000","last_undegraded":"2023-10-01T09:30:17.992871+0000","last_fullsized":"2023-10-01T09:30:17.992871+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:17.141540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":82,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.960703+0000","last_change":"2023-10-01T09:30:00.577886+0000","last_active":"2023-10-01T09:30:10.960703+0000","last_peered":"2023-10-01T09:30:10.960703+0000","last_clean":"2023-10-01T09:30:10.960703+0000","last_became_active":"2023-10-01T09:30:00.394121+0000","last_became_peered":"2023-10-01T09:30:00.394121+0000","last_unstale":"2023-10-01T09:30:10.960703+0000","last_undegraded":"2023-10-01T09:30:10.960703+0000","last_fullsized":"2023-10-01T09:30:10.960703+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":127,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.965878+0000","last_change":"2023-10-01T09:23:10.613386+0000","last_active":"2023-10-01T09:30:10.965878+0000","last_peered":"2023-10-01T09:30:10.965878+0000","last_clean":"2023-10-01T09:30:10.965878+0000","last_became_active":"2023-10-01T09:23:10.613046+0000","last_became_peered":"2023-10-01T09:23:10.613046+0000","last_unstale":"2023-10-01T09:30:10.965878+0000","last_undegraded":"2023-10-01T09:30:10.965878+0000","last_fullsized":"2023-10-01T09:30:10.965878+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:23:01.448841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"341'4539","reported_seq":4562,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:25.583988+0000","last_change":"2023-10-01T09:29:24.378970+0000","last_active":"2023-10-01T09:30:25.583988+0000","last_peered":"2023-10-01T09:30:25.583988+0000","last_clean":"2023-10-01T09:30:25.583988+0000","last_became_active":"2023-10-01T09:29:24.378703+0000","last_became_peered":"2023-10-01T09:29:24.378703+0000","last_unstale":"2023-10-01T09:30:25.583988+0000","last_undegraded":"2023-10-01T09:30:25.583988+0000","last_fullsized":"2023-10-01T09:30:25.583988+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4539,"log_dups_size":0,"ondisk_log_size":4539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297467904,"num_objects":4539,"num_object_clones":0,"num_object_copies":13617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4539,"num_write_kb":290496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":121,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963163+0000","last_change":"2023-10-01T09:26:12.395230+0000","last_active":"2023-10-01T09:30:10.963163+0000","last_peered":"2023-10-01T09:30:10.963163+0000","last_clean":"2023-10-01T09:30:10.963163+0000","last_became_active":"2023-10-01T09:26:12.269960+0000","last_became_peered":"2023-10-01T09:26:12.269960+0000","last_unstale":"2023-10-01T09:30:10.963163+0000","last_undegraded":"2023-10-01T09:30:10.963163+0000","last_fullsized":"2023-10-01T09:30:10.963163+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":61,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.970141+0000","last_change":"2023-10-01T09:28:02.315483+0000","last_active":"2023-10-01T09:30:10.970141+0000","last_peered":"2023-10-01T09:30:10.970141+0000","last_clean":"2023-10-01T09:30:10.970141+0000","last_became_active":"2023-10-01T09:28:02.315121+0000","last_became_peered":"2023-10-01T09:28:02.315121+0000","last_unstale":"2023-10-01T09:30:10.970141+0000","last_undegraded":"2023-10-01T09:30:10.970141+0000","last_fullsized":"2023-10-01T09:30:10.970141+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"341'4690","reported_seq":4711,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:26.952115+0000","last_change":"2023-10-01T09:29:24.376565+0000","last_active":"2023-10-01T09:30:26.952115+0000","last_peered":"2023-10-01T09:30:26.952115+0000","last_clean":"2023-10-01T09:30:26.952115+0000","last_became_active":"2023-10-01T09:29:24.376238+0000","last_became_peered":"2023-10-01T09:29:24.376238+0000","last_unstale":"2023-10-01T09:30:26.952115+0000","last_undegraded":"2023-10-01T09:30:26.952115+0000","last_fullsized":"2023-10-01T09:30:26.952115+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4690,"log_dups_size":0,"ondisk_log_size":4690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307363840,"num_objects":4690,"num_object_clones":0,"num_object_copies":14070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4690,"num_write_kb":300160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":322,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963210+0000","last_change":"2023-10-01T09:26:12.408240+0000","last_active":"2023-10-01T09:30:10.963210+0000","last_peered":"2023-10-01T09:30:10.963210+0000","last_clean":"2023-10-01T09:30:10.963210+0000","last_became_active":"2023-10-01T09:26:12.272093+0000","last_became_peered":"2023-10-01T09:26:12.272093+0000","last_unstale":"2023-10-01T09:30:10.963210+0000","last_undegraded":"2023-10-01T09:30:10.963210+0000","last_fullsized":"2023-10-01T09:30:10.963210+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":363,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.973845+0000","last_change":"2023-10-01T09:28:02.338346+0000","last_active":"2023-10-01T09:30:10.973845+0000","last_peered":"2023-10-01T09:30:10.973845+0000","last_clean":"2023-10-01T09:30:10.973845+0000","last_became_active":"2023-10-01T09:28:02.338038+0000","last_became_peered":"2023-10-01T09:28:02.338038+0000","last_unstale":"2023-10-01T09:30:10.973845+0000","last_undegraded":"2023-10-01T09:30:10.973845+0000","last_fullsized":"2023-10-01T09:30:10.973845+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:56:32.734028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"341'4651","reported_seq":4674,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:28.339528+0000","last_change":"2023-10-01T09:29:24.379084+0000","last_active":"2023-10-01T09:30:28.339528+0000","last_peered":"2023-10-01T09:30:28.339528+0000","last_clean":"2023-10-01T09:30:28.339528+0000","last_became_active":"2023-10-01T09:29:24.378789+0000","last_became_peered":"2023-10-01T09:29:24.378789+0000","last_unstale":"2023-10-01T09:30:28.339528+0000","last_undegraded":"2023-10-01T09:30:28.339528+0000","last_fullsized":"2023-10-01T09:30:28.339528+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4651,"log_dups_size":0,"ondisk_log_size":4651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:00:21.161819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304807936,"num_objects":4651,"num_object_clones":0,"num_object_copies":13953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4651,"num_write_kb":297664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":406,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963108+0000","last_change":"2023-10-01T09:28:02.318966+0000","last_active":"2023-10-01T09:30:10.963108+0000","last_peered":"2023-10-01T09:30:10.963108+0000","last_clean":"2023-10-01T09:30:10.963108+0000","last_became_active":"2023-10-01T09:28:02.316075+0000","last_became_peered":"2023-10-01T09:28:02.316075+0000","last_unstale":"2023-10-01T09:30:10.963108+0000","last_undegraded":"2023-10-01T09:30:10.963108+0000","last_fullsized":"2023-10-01T09:30:10.963108+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":44,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.960869+0000","last_change":"2023-10-01T09:28:02.361663+0000","last_active":"2023-10-01T09:30:10.960869+0000","last_peered":"2023-10-01T09:30:10.960869+0000","last_clean":"2023-10-01T09:30:10.960869+0000","last_became_active":"2023-10-01T09:28:02.361093+0000","last_became_peered":"2023-10-01T09:28:02.361093+0000","last_unstale":"2023-10-01T09:30:10.960869+0000","last_undegraded":"2023-10-01T09:30:10.960869+0000","last_fullsized":"2023-10-01T09:30:10.960869+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"341'4633","reported_seq":4657,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:28.145858+0000","last_change":"2023-10-01T09:29:24.356954+0000","last_active":"2023-10-01T09:30:28.145858+0000","last_peered":"2023-10-01T09:30:28.145858+0000","last_clean":"2023-10-01T09:30:28.145858+0000","last_became_active":"2023-10-01T09:29:24.356643+0000","last_became_peered":"2023-10-01T09:29:24.356643+0000","last_unstale":"2023-10-01T09:30:28.145858+0000","last_undegraded":"2023-10-01T09:30:28.145858+0000","last_fullsized":"2023-10-01T09:30:28.145858+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4633,"log_dups_size":0,"ondisk_log_size":4633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:56:34.140751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303628288,"num_objects":4633,"num_object_clones":0,"num_object_copies":13899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4633,"num_write_kb":296512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":61,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.970752+0000","last_change":"2023-10-01T09:28:02.314625+0000","last_active":"2023-10-01T09:30:10.970752+0000","last_peered":"2023-10-01T09:30:10.970752+0000","last_clean":"2023-10-01T09:30:10.970752+0000","last_became_active":"2023-10-01T09:28:02.314315+0000","last_became_peered":"2023-10-01T09:28:02.314315+0000","last_unstale":"2023-10-01T09:30:10.970752+0000","last_undegraded":"2023-10-01T09:30:10.970752+0000","last_fullsized":"2023-10-01T09:30:10.970752+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":304,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963213+0000","last_change":"2023-10-01T09:26:12.308350+0000","last_active":"2023-10-01T09:30:10.963213+0000","last_peered":"2023-10-01T09:30:10.963213+0000","last_clean":"2023-10-01T09:30:10.963213+0000","last_became_active":"2023-10-01T09:26:12.307689+0000","last_became_peered":"2023-10-01T09:26:12.307689+0000","last_unstale":"2023-10-01T09:30:10.963213+0000","last_undegraded":"2023-10-01T09:30:10.963213+0000","last_fullsized":"2023-10-01T09:30:10.963213+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:34:57.116449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"341'4611","reported_seq":4635,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:26.675464+0000","last_change":"2023-10-01T09:29:24.360118+0000","last_active":"2023-10-01T09:30:26.675464+0000","last_peered":"2023-10-01T09:30:26.675464+0000","last_clean":"2023-10-01T09:30:26.675464+0000","last_became_active":"2023-10-01T09:29:24.359817+0000","last_became_peered":"2023-10-01T09:29:24.359817+0000","last_unstale":"2023-10-01T09:30:26.675464+0000","last_undegraded":"2023-10-01T09:30:26.675464+0000","last_fullsized":"2023-10-01T09:30:26.675464+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4611,"log_dups_size":0,"ondisk_log_size":4611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:52:42.958518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302186496,"num_objects":4611,"num_object_clones":0,"num_object_copies":13833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4611,"num_write_kb":295104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":327,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964054+0000","last_change":"2023-10-01T09:26:12.387826+0000","last_active":"2023-10-01T09:30:10.964054+0000","last_peered":"2023-10-01T09:30:10.964054+0000","last_clean":"2023-10-01T09:30:10.964054+0000","last_became_active":"2023-10-01T09:26:12.301770+0000","last_became_peered":"2023-10-01T09:26:12.301770+0000","last_unstale":"2023-10-01T09:30:10.964054+0000","last_undegraded":"2023-10-01T09:30:10.964054+0000","last_fullsized":"2023-10-01T09:30:10.964054+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:09:42.617904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":308,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:11.676532+0000","last_change":"2023-10-01T09:28:02.349347+0000","last_active":"2023-10-01T09:30:11.676532+0000","last_peered":"2023-10-01T09:30:11.676532+0000","last_clean":"2023-10-01T09:30:11.676532+0000","last_became_active":"2023-10-01T09:28:02.337322+0000","last_became_peered":"2023-10-01T09:28:02.337322+0000","last_unstale":"2023-10-01T09:30:11.676532+0000","last_undegraded":"2023-10-01T09:30:11.676532+0000","last_fullsized":"2023-10-01T09:30:11.676532+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:46:56.505421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"341'4720","reported_seq":4744,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:28.020480+0000","last_change":"2023-10-01T09:29:24.380240+0000","last_active":"2023-10-01T09:30:28.020480+0000","last_peered":"2023-10-01T09:30:28.020480+0000","last_clean":"2023-10-01T09:30:28.020480+0000","last_became_active":"2023-10-01T09:29:24.379922+0000","last_became_peered":"2023-10-01T09:29:24.379922+0000","last_unstale":"2023-10-01T09:30:28.020480+0000","last_undegraded":"2023-10-01T09:30:28.020480+0000","last_fullsized":"2023-10-01T09:30:28.020480+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4720,"log_dups_size":0,"ondisk_log_size":4720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:34:10.363596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309329920,"num_objects":4720,"num_object_clones":0,"num_object_copies":14160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4720,"num_write_kb":302080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":313,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974171+0000","last_change":"2023-10-01T09:28:02.342548+0000","last_active":"2023-10-01T09:30:10.974171+0000","last_peered":"2023-10-01T09:30:10.974171+0000","last_clean":"2023-10-01T09:30:10.974171+0000","last_became_active":"2023-10-01T09:28:02.342162+0000","last_became_peered":"2023-10-01T09:28:02.342162+0000","last_unstale":"2023-10-01T09:30:10.974171+0000","last_undegraded":"2023-10-01T09:30:10.974171+0000","last_fullsized":"2023-10-01T09:30:10.974171+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:49:07.163031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":357,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.975533+0000","last_change":"2023-10-01T09:28:02.342750+0000","last_active":"2023-10-01T09:30:10.975533+0000","last_peered":"2023-10-01T09:30:10.975533+0000","last_clean":"2023-10-01T09:30:10.975533+0000","last_became_active":"2023-10-01T09:28:02.342487+0000","last_became_peered":"2023-10-01T09:28:02.342487+0000","last_unstale":"2023-10-01T09:30:10.975533+0000","last_undegraded":"2023-10-01T09:30:10.975533+0000","last_fullsized":"2023-10-01T09:30:10.975533+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:23:46.051719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"341'4670","reported_seq":4694,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:28.327626+0000","last_change":"2023-10-01T09:29:24.359090+0000","last_active":"2023-10-01T09:30:28.327626+0000","last_peered":"2023-10-01T09:30:28.327626+0000","last_clean":"2023-10-01T09:30:28.327626+0000","last_became_active":"2023-10-01T09:29:24.358822+0000","last_became_peered":"2023-10-01T09:29:24.358822+0000","last_unstale":"2023-10-01T09:30:28.327626+0000","last_undegraded":"2023-10-01T09:30:28.327626+0000","last_fullsized":"2023-10-01T09:30:28.327626+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4670,"log_dups_size":0,"ondisk_log_size":4670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:29:47.448230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306053120,"num_objects":4670,"num_object_clones":0,"num_object_copies":14010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4670,"num_write_kb":298880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":204,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.962518+0000","last_change":"2023-10-01T09:26:29.372326+0000","last_active":"2023-10-01T09:30:10.962518+0000","last_peered":"2023-10-01T09:30:10.962518+0000","last_clean":"2023-10-01T09:30:10.962518+0000","last_became_active":"2023-10-01T09:26:29.372035+0000","last_became_peered":"2023-10-01T09:26:29.372035+0000","last_unstale":"2023-10-01T09:30:10.962518+0000","last_undegraded":"2023-10-01T09:30:10.962518+0000","last_fullsized":"2023-10-01T09:30:10.962518+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":119,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974858+0000","last_change":"2023-10-01T09:28:02.339901+0000","last_active":"2023-10-01T09:30:10.974858+0000","last_peered":"2023-10-01T09:30:10.974858+0000","last_clean":"2023-10-01T09:30:10.974858+0000","last_became_active":"2023-10-01T09:28:02.339601+0000","last_became_peered":"2023-10-01T09:28:02.339601+0000","last_unstale":"2023-10-01T09:30:10.974858+0000","last_undegraded":"2023-10-01T09:30:10.974858+0000","last_fullsized":"2023-10-01T09:30:10.974858+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:30:16.497879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"341'4661","reported_seq":4685,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:28.151338+0000","last_change":"2023-10-01T09:29:24.351403+0000","last_active":"2023-10-01T09:30:28.151338+0000","last_peered":"2023-10-01T09:30:28.151338+0000","last_clean":"2023-10-01T09:30:28.151338+0000","last_became_active":"2023-10-01T09:29:24.351113+0000","last_became_peered":"2023-10-01T09:29:24.351113+0000","last_unstale":"2023-10-01T09:30:28.151338+0000","last_undegraded":"2023-10-01T09:30:28.151338+0000","last_fullsized":"2023-10-01T09:30:28.151338+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4661,"log_dups_size":0,"ondisk_log_size":4661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:07:56.443376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305463296,"num_objects":4661,"num_object_clones":0,"num_object_copies":13983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4661,"num_write_kb":298304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":118,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.966339+0000","last_change":"2023-10-01T09:22:40.316008+0000","last_active":"2023-10-01T09:30:10.966339+0000","last_peered":"2023-10-01T09:30:10.966339+0000","last_clean":"2023-10-01T09:30:10.966339+0000","last_became_active":"2023-10-01T09:22:40.315602+0000","last_became_peered":"2023-10-01T09:22:40.315602+0000","last_unstale":"2023-10-01T09:30:10.966339+0000","last_undegraded":"2023-10-01T09:30:10.966339+0000","last_fullsized":"2023-10-01T09:30:10.966339+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:39:37.485241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":218,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.975086+0000","last_change":"2023-10-01T09:26:12.320680+0000","last_active":"2023-10-01T09:30:10.975086+0000","last_peered":"2023-10-01T09:30:10.975086+0000","last_clean":"2023-10-01T09:30:10.975086+0000","last_became_active":"2023-10-01T09:26:12.320340+0000","last_became_peered":"2023-10-01T09:26:12.320340+0000","last_unstale":"2023-10-01T09:30:10.975086+0000","last_undegraded":"2023-10-01T09:30:10.975086+0000","last_fullsized":"2023-10-01T09:30:10.975086+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:11:35.796015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"341'4820","reported_seq":4844,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:28.318248+0000","last_change":"2023-10-01T09:29:24.382969+0000","last_active":"2023-10-01T09:30:28.318248+0000","last_peered":"2023-10-01T09:30:28.318248+0000","last_clean":"2023-10-01T09:30:28.318248+0000","last_became_active":"2023-10-01T09:29:24.382653+0000","last_became_peered":"2023-10-01T09:29:24.382653+0000","last_unstale":"2023-10-01T09:30:28.318248+0000","last_undegraded":"2023-10-01T09:30:28.318248+0000","last_fullsized":"2023-10-01T09:30:28.318248+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4820,"log_dups_size":0,"ondisk_log_size":4820,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:37:56.666282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315883520,"num_objects":4820,"num_object_clones":0,"num_object_copies":14460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4820,"num_write_kb":308480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1161,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974682+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:30:10.974682+0000","last_peered":"2023-10-01T09:30:10.974682+0000","last_clean":"2023-10-01T09:30:10.974682+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:30:10.974682+0000","last_undegraded":"2023-10-01T09:30:10.974682+0000","last_fullsized":"2023-10-01T09:30:10.974682+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":52,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.970676+0000","last_change":"2023-10-01T09:27:01.989188+0000","last_active":"2023-10-01T09:30:10.970676+0000","last_peered":"2023-10-01T09:30:10.970676+0000","last_clean":"2023-10-01T09:30:10.970676+0000","last_became_active":"2023-10-01T09:27:01.988878+0000","last_became_peered":"2023-10-01T09:27:01.988878+0000","last_unstale":"2023-10-01T09:30:10.970676+0000","last_undegraded":"2023-10-01T09:30:10.970676+0000","last_fullsized":"2023-10-01T09:30:10.970676+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":318,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974045+0000","last_change":"2023-10-01T09:27:01.977155+0000","last_active":"2023-10-01T09:30:10.974045+0000","last_peered":"2023-10-01T09:30:10.974045+0000","last_clean":"2023-10-01T09:30:10.974045+0000","last_became_active":"2023-10-01T09:27:01.975803+0000","last_became_peered":"2023-10-01T09:27:01.975803+0000","last_unstale":"2023-10-01T09:30:10.974045+0000","last_undegraded":"2023-10-01T09:30:10.974045+0000","last_fullsized":"2023-10-01T09:30:10.974045+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:00:55.170662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"341'4524","reported_seq":4547,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:25.581617+0000","last_change":"2023-10-01T09:29:24.376062+0000","last_active":"2023-10-01T09:30:25.581617+0000","last_peered":"2023-10-01T09:30:25.581617+0000","last_clean":"2023-10-01T09:30:25.581617+0000","last_became_active":"2023-10-01T09:29:24.375795+0000","last_became_peered":"2023-10-01T09:29:24.375795+0000","last_unstale":"2023-10-01T09:30:25.581617+0000","last_undegraded":"2023-10-01T09:30:25.581617+0000","last_fullsized":"2023-10-01T09:30:25.581617+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4524,"log_dups_size":0,"ondisk_log_size":4524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296484864,"num_objects":4524,"num_object_clones":0,"num_object_copies":13572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4524,"num_write_kb":289536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1024,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.970212+0000","last_change":"2023-10-01T09:28:02.595215+0000","last_active":"2023-10-01T09:30:10.970212+0000","last_peered":"2023-10-01T09:30:10.970212+0000","last_clean":"2023-10-01T09:30:10.970212+0000","last_became_active":"2023-10-01T09:28:02.346777+0000","last_became_peered":"2023-10-01T09:28:02.346777+0000","last_unstale":"2023-10-01T09:30:10.970212+0000","last_undegraded":"2023-10-01T09:30:10.970212+0000","last_fullsized":"2023-10-01T09:30:10.970212+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":304,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963570+0000","last_change":"2023-10-01T09:28:02.457735+0000","last_active":"2023-10-01T09:30:10.963570+0000","last_peered":"2023-10-01T09:30:10.963570+0000","last_clean":"2023-10-01T09:30:10.963570+0000","last_became_active":"2023-10-01T09:28:02.457495+0000","last_became_peered":"2023-10-01T09:28:02.457495+0000","last_unstale":"2023-10-01T09:30:10.963570+0000","last_undegraded":"2023-10-01T09:30:10.963570+0000","last_fullsized":"2023-10-01T09:30:10.963570+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:48:53.316760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":15,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.970408+0000","last_change":"2023-10-01T09:29:58.362019+0000","last_active":"2023-10-01T09:30:10.970408+0000","last_peered":"2023-10-01T09:30:10.970408+0000","last_clean":"2023-10-01T09:30:10.970408+0000","last_became_active":"2023-10-01T09:29:58.361762+0000","last_became_peered":"2023-10-01T09:29:58.361762+0000","last_unstale":"2023-10-01T09:30:10.970408+0000","last_undegraded":"2023-10-01T09:30:10.970408+0000","last_fullsized":"2023-10-01T09:30:10.970408+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"341'4583","reported_seq":4607,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:26.656005+0000","last_change":"2023-10-01T09:29:24.379882+0000","last_active":"2023-10-01T09:30:26.656005+0000","last_peered":"2023-10-01T09:30:26.656005+0000","last_clean":"2023-10-01T09:30:26.656005+0000","last_became_active":"2023-10-01T09:29:24.379572+0000","last_became_peered":"2023-10-01T09:29:24.379572+0000","last_unstale":"2023-10-01T09:30:26.656005+0000","last_undegraded":"2023-10-01T09:30:26.656005+0000","last_fullsized":"2023-10-01T09:30:26.656005+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4583,"log_dups_size":0,"ondisk_log_size":4583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:39:28.119558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300351488,"num_objects":4583,"num_object_clones":0,"num_object_copies":13749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4583,"num_write_kb":293312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":596,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964760+0000","last_change":"2023-10-01T09:26:29.385302+0000","last_active":"2023-10-01T09:30:10.964760+0000","last_peered":"2023-10-01T09:30:10.964760+0000","last_clean":"2023-10-01T09:30:10.964760+0000","last_became_active":"2023-10-01T09:26:29.384738+0000","last_became_peered":"2023-10-01T09:26:29.384738+0000","last_unstale":"2023-10-01T09:30:10.964760+0000","last_undegraded":"2023-10-01T09:30:10.964760+0000","last_fullsized":"2023-10-01T09:30:10.964760+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:22:00.127736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":370,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:13.168721+0000","last_change":"2023-10-01T09:28:02.346596+0000","last_active":"2023-10-01T09:30:13.168721+0000","last_peered":"2023-10-01T09:30:13.168721+0000","last_clean":"2023-10-01T09:30:13.168721+0000","last_became_active":"2023-10-01T09:28:02.346318+0000","last_became_peered":"2023-10-01T09:28:02.346318+0000","last_unstale":"2023-10-01T09:30:13.168721+0000","last_undegraded":"2023-10-01T09:30:13.168721+0000","last_fullsized":"2023-10-01T09:30:13.168721+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:27:45.840901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":224,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.975261+0000","last_change":"2023-10-01T09:27:02.004900+0000","last_active":"2023-10-01T09:30:10.975261+0000","last_peered":"2023-10-01T09:30:10.975261+0000","last_clean":"2023-10-01T09:30:10.975261+0000","last_became_active":"2023-10-01T09:27:02.004501+0000","last_became_peered":"2023-10-01T09:27:02.004501+0000","last_unstale":"2023-10-01T09:30:10.975261+0000","last_undegraded":"2023-10-01T09:30:10.975261+0000","last_fullsized":"2023-10-01T09:30:10.975261+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:50:52.617833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"341'4550","reported_seq":4573,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:25.468183+0000","last_change":"2023-10-01T09:29:24.359284+0000","last_active":"2023-10-01T09:30:25.468183+0000","last_peered":"2023-10-01T09:30:25.468183+0000","last_clean":"2023-10-01T09:30:25.468183+0000","last_became_active":"2023-10-01T09:29:24.358932+0000","last_became_peered":"2023-10-01T09:29:24.358932+0000","last_unstale":"2023-10-01T09:30:25.468183+0000","last_undegraded":"2023-10-01T09:30:25.468183+0000","last_fullsized":"2023-10-01T09:30:25.468183+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4550,"log_dups_size":0,"ondisk_log_size":4550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298188800,"num_objects":4550,"num_object_clones":0,"num_object_copies":13650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4550,"num_write_kb":291200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1267,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.970458+0000","last_change":"2023-10-01T09:28:48.006973+0000","last_active":"2023-10-01T09:30:10.970458+0000","last_peered":"2023-10-01T09:30:10.970458+0000","last_clean":"2023-10-01T09:30:10.970458+0000","last_became_active":"2023-10-01T09:28:48.006289+0000","last_became_peered":"2023-10-01T09:28:48.006289+0000","last_unstale":"2023-10-01T09:30:10.970458+0000","last_undegraded":"2023-10-01T09:30:10.970458+0000","last_fullsized":"2023-10-01T09:30:10.970458+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":559,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.962570+0000","last_change":"2023-10-01T09:29:04.962927+0000","last_active":"2023-10-01T09:30:10.962570+0000","last_peered":"2023-10-01T09:30:10.962570+0000","last_clean":"2023-10-01T09:30:10.962570+0000","last_became_active":"2023-10-01T09:29:04.962542+0000","last_became_peered":"2023-10-01T09:29:04.962542+0000","last_unstale":"2023-10-01T09:30:10.962570+0000","last_undegraded":"2023-10-01T09:30:10.962570+0000","last_fullsized":"2023-10-01T09:30:10.962570+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":171,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.968702+0000","last_change":"2023-10-01T09:28:02.320192+0000","last_active":"2023-10-01T09:30:10.968702+0000","last_peered":"2023-10-01T09:30:10.968702+0000","last_clean":"2023-10-01T09:30:10.968702+0000","last_became_active":"2023-10-01T09:28:02.319852+0000","last_became_peered":"2023-10-01T09:28:02.319852+0000","last_unstale":"2023-10-01T09:30:10.968702+0000","last_undegraded":"2023-10-01T09:30:10.968702+0000","last_fullsized":"2023-10-01T09:30:10.968702+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:16:46.792585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":246,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963198+0000","last_change":"2023-10-01T09:28:02.312356+0000","last_active":"2023-10-01T09:30:10.963198+0000","last_peered":"2023-10-01T09:30:10.963198+0000","last_clean":"2023-10-01T09:30:10.963198+0000","last_became_active":"2023-10-01T09:28:02.312014+0000","last_became_peered":"2023-10-01T09:28:02.312014+0000","last_unstale":"2023-10-01T09:30:10.963198+0000","last_undegraded":"2023-10-01T09:30:10.963198+0000","last_fullsized":"2023-10-01T09:30:10.963198+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"341'4753","reported_seq":4776,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:28.209589+0000","last_change":"2023-10-01T09:29:24.380724+0000","last_active":"2023-10-01T09:30:28.209589+0000","last_peered":"2023-10-01T09:30:28.209589+0000","last_clean":"2023-10-01T09:30:28.209589+0000","last_became_active":"2023-10-01T09:29:24.380523+0000","last_became_peered":"2023-10-01T09:29:24.380523+0000","last_unstale":"2023-10-01T09:30:28.209589+0000","last_undegraded":"2023-10-01T09:30:28.209589+0000","last_fullsized":"2023-10-01T09:30:28.209589+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4753,"log_dups_size":0,"ondisk_log_size":4753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:16.268423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311492608,"num_objects":4753,"num_object_clones":0,"num_object_copies":14259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4753,"num_write_kb":304192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":437,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.962664+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:30:10.962664+0000","last_peered":"2023-10-01T09:30:10.962664+0000","last_clean":"2023-10-01T09:30:10.962664+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:30:10.962664+0000","last_undegraded":"2023-10-01T09:30:10.962664+0000","last_fullsized":"2023-10-01T09:30:10.962664+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":347,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963933+0000","last_change":"2023-10-01T09:27:01.986786+0000","last_active":"2023-10-01T09:30:10.963933+0000","last_peered":"2023-10-01T09:30:10.963933+0000","last_clean":"2023-10-01T09:30:10.963933+0000","last_became_active":"2023-10-01T09:27:01.986466+0000","last_became_peered":"2023-10-01T09:27:01.986466+0000","last_unstale":"2023-10-01T09:30:10.963933+0000","last_undegraded":"2023-10-01T09:30:10.963933+0000","last_fullsized":"2023-10-01T09:30:10.963933+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:55:51.244125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":347,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:17.992646+0000","last_change":"2023-10-01T09:19:30.091481+0000","last_active":"2023-10-01T09:30:17.992646+0000","last_peered":"2023-10-01T09:30:17.992646+0000","last_clean":"2023-10-01T09:30:17.992646+0000","last_became_active":"2023-10-01T09:19:30.091151+0000","last_became_peered":"2023-10-01T09:19:30.091151+0000","last_unstale":"2023-10-01T09:30:17.992646+0000","last_undegraded":"2023-10-01T09:30:17.992646+0000","last_fullsized":"2023-10-01T09:30:17.992646+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:09.127408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"341'4632","reported_seq":4656,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:26.638583+0000","last_change":"2023-10-01T09:29:24.381082+0000","last_active":"2023-10-01T09:30:26.638583+0000","last_peered":"2023-10-01T09:30:26.638583+0000","last_clean":"2023-10-01T09:30:26.638583+0000","last_became_active":"2023-10-01T09:29:24.380733+0000","last_became_peered":"2023-10-01T09:29:24.380733+0000","last_unstale":"2023-10-01T09:30:26.638583+0000","last_undegraded":"2023-10-01T09:30:26.638583+0000","last_fullsized":"2023-10-01T09:30:26.638583+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4632,"log_dups_size":0,"ondisk_log_size":4632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:27:26.636652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303562752,"num_objects":4632,"num_object_clones":0,"num_object_copies":13896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4632,"num_write_kb":296448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":740,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963106+0000","last_change":"2023-10-01T09:28:03.317559+0000","last_active":"2023-10-01T09:30:10.963106+0000","last_peered":"2023-10-01T09:30:10.963106+0000","last_clean":"2023-10-01T09:30:10.963106+0000","last_became_active":"2023-10-01T09:28:03.317237+0000","last_became_peered":"2023-10-01T09:28:03.317237+0000","last_unstale":"2023-10-01T09:30:10.963106+0000","last_undegraded":"2023-10-01T09:30:10.963106+0000","last_fullsized":"2023-10-01T09:30:10.963106+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":360,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974766+0000","last_change":"2023-10-01T09:27:01.999110+0000","last_active":"2023-10-01T09:30:10.974766+0000","last_peered":"2023-10-01T09:30:10.974766+0000","last_clean":"2023-10-01T09:30:10.974766+0000","last_became_active":"2023-10-01T09:27:01.998740+0000","last_became_peered":"2023-10-01T09:27:01.998740+0000","last_unstale":"2023-10-01T09:30:10.974766+0000","last_undegraded":"2023-10-01T09:30:10.974766+0000","last_fullsized":"2023-10-01T09:30:10.974766+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:40:08.795106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":314,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963435+0000","last_change":"2023-10-01T09:27:01.998986+0000","last_active":"2023-10-01T09:30:10.963435+0000","last_peered":"2023-10-01T09:30:10.963435+0000","last_clean":"2023-10-01T09:30:10.963435+0000","last_became_active":"2023-10-01T09:27:01.998709+0000","last_became_peered":"2023-10-01T09:27:01.998709+0000","last_unstale":"2023-10-01T09:30:10.963435+0000","last_undegraded":"2023-10-01T09:30:10.963435+0000","last_fullsized":"2023-10-01T09:30:10.963435+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:01:14.296638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"341'4837","reported_seq":4861,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:28.331838+0000","last_change":"2023-10-01T09:29:24.378898+0000","last_active":"2023-10-01T09:30:28.331838+0000","last_peered":"2023-10-01T09:30:28.331838+0000","last_clean":"2023-10-01T09:30:28.331838+0000","last_became_active":"2023-10-01T09:29:24.378624+0000","last_became_peered":"2023-10-01T09:29:24.378624+0000","last_unstale":"2023-10-01T09:30:28.331838+0000","last_undegraded":"2023-10-01T09:30:28.331838+0000","last_fullsized":"2023-10-01T09:30:28.331838+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4837,"log_dups_size":0,"ondisk_log_size":4837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:12:28.028735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316997632,"num_objects":4837,"num_object_clones":0,"num_object_copies":14511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4837,"num_write_kb":309568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":355,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.966001+0000","last_change":"2023-10-01T09:23:10.615956+0000","last_active":"2023-10-01T09:30:10.966001+0000","last_peered":"2023-10-01T09:30:10.966001+0000","last_clean":"2023-10-01T09:30:10.966001+0000","last_became_active":"2023-10-01T09:23:10.610919+0000","last_became_peered":"2023-10-01T09:23:10.610919+0000","last_unstale":"2023-10-01T09:30:10.966001+0000","last_undegraded":"2023-10-01T09:30:10.966001+0000","last_fullsized":"2023-10-01T09:30:10.966001+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:03:44.294373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":926,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974990+0000","last_change":"2023-10-01T09:29:13.014839+0000","last_active":"2023-10-01T09:30:10.974990+0000","last_peered":"2023-10-01T09:30:10.974990+0000","last_clean":"2023-10-01T09:30:10.974990+0000","last_became_active":"2023-10-01T09:29:13.014139+0000","last_became_peered":"2023-10-01T09:29:13.014139+0000","last_unstale":"2023-10-01T09:30:10.974990+0000","last_undegraded":"2023-10-01T09:30:10.974990+0000","last_fullsized":"2023-10-01T09:30:10.974990+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:23:10.370435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":127,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.969988+0000","last_change":"2023-10-01T09:23:10.612309+0000","last_active":"2023-10-01T09:30:10.969988+0000","last_peered":"2023-10-01T09:30:10.969988+0000","last_clean":"2023-10-01T09:30:10.969988+0000","last_became_active":"2023-10-01T09:23:10.612034+0000","last_became_peered":"2023-10-01T09:23:10.612034+0000","last_unstale":"2023-10-01T09:30:10.969988+0000","last_undegraded":"2023-10-01T09:30:10.969988+0000","last_fullsized":"2023-10-01T09:30:10.969988+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:09:47.963186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"341'4516","reported_seq":4539,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:25.477683+0000","last_change":"2023-10-01T09:29:24.379832+0000","last_active":"2023-10-01T09:30:25.477683+0000","last_peered":"2023-10-01T09:30:25.477683+0000","last_clean":"2023-10-01T09:30:25.477683+0000","last_became_active":"2023-10-01T09:29:24.379540+0000","last_became_peered":"2023-10-01T09:29:24.379540+0000","last_unstale":"2023-10-01T09:30:25.477683+0000","last_undegraded":"2023-10-01T09:30:25.477683+0000","last_fullsized":"2023-10-01T09:30:25.477683+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":4516,"log_dups_size":0,"ondisk_log_size":4516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295960576,"num_objects":4516,"num_object_clones":0,"num_object_copies":13548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4516,"num_write_kb":289024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":54,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.961223+0000","last_change":"2023-10-01T09:29:11.374086+0000","last_active":"2023-10-01T09:30:10.961223+0000","last_peered":"2023-10-01T09:30:10.961223+0000","last_clean":"2023-10-01T09:30:10.961223+0000","last_became_active":"2023-10-01T09:28:02.319106+0000","last_became_peered":"2023-10-01T09:28:02.319106+0000","last_unstale":"2023-10-01T09:30:10.961223+0000","last_undegraded":"2023-10-01T09:30:10.961223+0000","last_fullsized":"2023-10-01T09:30:10.961223+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":339,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-10-01T09:30:17.992770+0000","last_change":"2023-10-01T09:26:12.291583+0000","last_active":"2023-10-01T09:30:17.992770+0000","last_peered":"2023-10-01T09:30:17.992770+0000","last_clean":"2023-10-01T09:30:17.992770+0000","last_became_active":"2023-10-01T09:26:12.271891+0000","last_became_peered":"2023-10-01T09:26:12.271891+0000","last_unstale":"2023-10-01T09:30:17.992770+0000","last_undegraded":"2023-10-01T09:30:17.992770+0000","last_fullsized":"2023-10-01T09:30:17.992770+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:53:06.889203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":61,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.970138+0000","last_change":"2023-10-01T09:28:02.314867+0000","last_active":"2023-10-01T09:30:10.970138+0000","last_peered":"2023-10-01T09:30:10.970138+0000","last_clean":"2023-10-01T09:30:10.970138+0000","last_became_active":"2023-10-01T09:28:02.314480+0000","last_became_peered":"2023-10-01T09:28:02.314480+0000","last_unstale":"2023-10-01T09:30:10.970138+0000","last_undegraded":"2023-10-01T09:30:10.970138+0000","last_fullsized":"2023-10-01T09:30:10.970138+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":292,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.963352+0000","last_change":"2023-10-01T09:26:12.271469+0000","last_active":"2023-10-01T09:30:10.963352+0000","last_peered":"2023-10-01T09:30:10.963352+0000","last_clean":"2023-10-01T09:30:10.963352+0000","last_became_active":"2023-10-01T09:26:12.270972+0000","last_became_peered":"2023-10-01T09:26:12.270972+0000","last_unstale":"2023-10-01T09:30:10.963352+0000","last_undegraded":"2023-10-01T09:30:10.963352+0000","last_fullsized":"2023-10-01T09:30:10.963352+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:45:03.822952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":370,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.974922+0000","last_change":"2023-10-01T09:22:40.166927+0000","last_active":"2023-10-01T09:30:10.974922+0000","last_peered":"2023-10-01T09:30:10.974922+0000","last_clean":"2023-10-01T09:30:10.974922+0000","last_became_active":"2023-10-01T09:22:40.166606+0000","last_became_peered":"2023-10-01T09:22:40.166606+0000","last_unstale":"2023-10-01T09:30:10.974922+0000","last_undegraded":"2023-10-01T09:30:10.974922+0000","last_fullsized":"2023-10-01T09:30:10.974922+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:54:18.044207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":69,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.970841+0000","last_change":"2023-10-01T09:28:02.507232+0000","last_active":"2023-10-01T09:30:10.970841+0000","last_peered":"2023-10-01T09:30:10.970841+0000","last_clean":"2023-10-01T09:30:10.970841+0000","last_became_active":"2023-10-01T09:28:02.340776+0000","last_became_peered":"2023-10-01T09:28:02.340776+0000","last_unstale":"2023-10-01T09:30:10.970841+0000","last_undegraded":"2023-10-01T09:30:10.970841+0000","last_fullsized":"2023-10-01T09:30:10.970841+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":15,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.970180+0000","last_change":"2023-10-01T09:29:58.359570+0000","last_active":"2023-10-01T09:30:10.970180+0000","last_peered":"2023-10-01T09:30:10.970180+0000","last_clean":"2023-10-01T09:30:10.970180+0000","last_became_active":"2023-10-01T09:29:58.359181+0000","last_became_peered":"2023-10-01T09:29:58.359181+0000","last_unstale":"2023-10-01T09:30:10.970180+0000","last_undegraded":"2023-10-01T09:30:10.970180+0000","last_fullsized":"2023-10-01T09:30:10.970180+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":69,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.970598+0000","last_change":"2023-10-01T09:28:02.551019+0000","last_active":"2023-10-01T09:30:10.970598+0000","last_peered":"2023-10-01T09:30:10.970598+0000","last_clean":"2023-10-01T09:30:10.970598+0000","last_became_active":"2023-10-01T09:28:02.342509+0000","last_became_peered":"2023-10-01T09:28:02.342509+0000","last_unstale":"2023-10-01T09:30:10.970598+0000","last_undegraded":"2023-10-01T09:30:10.970598+0000","last_fullsized":"2023-10-01T09:30:10.970598+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":280,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.968855+0000","last_change":"2023-10-01T09:26:12.309345+0000","last_active":"2023-10-01T09:30:10.968855+0000","last_peered":"2023-10-01T09:30:10.968855+0000","last_clean":"2023-10-01T09:30:10.968855+0000","last_became_active":"2023-10-01T09:26:12.304299+0000","last_became_peered":"2023-10-01T09:26:12.304299+0000","last_unstale":"2023-10-01T09:30:10.968855+0000","last_undegraded":"2023-10-01T09:30:10.968855+0000","last_fullsized":"2023-10-01T09:30:10.968855+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:35:59.915530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":142,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964381+0000","last_change":"2023-10-01T09:28:02.321395+0000","last_active":"2023-10-01T09:30:10.964381+0000","last_peered":"2023-10-01T09:30:10.964381+0000","last_clean":"2023-10-01T09:30:10.964381+0000","last_became_active":"2023-10-01T09:28:02.321071+0000","last_became_peered":"2023-10-01T09:28:02.321071+0000","last_unstale":"2023-10-01T09:30:10.964381+0000","last_undegraded":"2023-10-01T09:30:10.964381+0000","last_fullsized":"2023-10-01T09:30:10.964381+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:12:45.734920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":118,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964904+0000","last_change":"2023-10-01T09:22:40.170676+0000","last_active":"2023-10-01T09:30:10.964904+0000","last_peered":"2023-10-01T09:30:10.964904+0000","last_clean":"2023-10-01T09:30:10.964904+0000","last_became_active":"2023-10-01T09:22:40.170260+0000","last_became_peered":"2023-10-01T09:22:40.170260+0000","last_unstale":"2023-10-01T09:30:10.964904+0000","last_undegraded":"2023-10-01T09:30:10.964904+0000","last_fullsized":"2023-10-01T09:30:10.964904+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:14:14.693287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":431,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.964041+0000","last_change":"2023-10-01T09:27:02.019266+0000","last_active":"2023-10-01T09:30:10.964041+0000","last_peered":"2023-10-01T09:30:10.964041+0000","last_clean":"2023-10-01T09:30:10.964041+0000","last_became_active":"2023-10-01T09:27:02.006744+0000","last_became_peered":"2023-10-01T09:27:02.006744+0000","last_unstale":"2023-10-01T09:30:10.964041+0000","last_undegraded":"2023-10-01T09:30:10.964041+0000","last_fullsized":"2023-10-01T09:30:10.964041+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:39:21.899359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":298,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-10-01T09:30:10.973550+0000","last_change":"2023-10-01T09:28:02.343724+0000","last_active":"2023-10-01T09:30:10.973550+0000","last_peered":"2023-10-01T09:30:10.973550+0000","last_clean":"2023-10-01T09:30:10.973550+0000","last_became_active":"2023-10-01T09:28:02.339485+0000","last_became_peered":"2023-10-01T09:28:02.339485+0000","last_unstale":"2023-10-01T09:30:10.973550+0000","last_undegraded":"2023-10-01T09:30:10.973550+0000","last_fullsized":"2023-10-01T09:30:10.973550+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:54:52.086601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":28418,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":22016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42916,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4875223040,"num_objects":74390,"num_object_clones":0,"num_object_copies":223170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":74390,"num_write_kb":4760960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14560329728,"data_stored":14560329728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74390,"ondisk_log_size":74390,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":199,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430558,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2088164,"kb_used_data":1790940,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91656988,"statfs":{"total":95995035648,"available":93856755712,"internally_reserved":0,"allocated":1833922560,"data_stored":1833048828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:30:24 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":3.2160000000000002,"15min":3.9129999999999998},"min":{"1min":0.215,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":5.1879999999999997,"5min":73.284000000000006,"15min":117.855},"last":7.056},{"interface":"front","average":{"1min":1.2110000000000001,"5min":3.8290000000000002,"15min":3.9350000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.22500000000000001},"max":{"1min":5.3170000000000002,"5min":73.269000000000005,"15min":120.348},"last":0.58499999999999996}]},{"osd":1,"last update":"Sun Oct 1 09:29:43 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":2.0249999999999999,"15min":3.302},"min":{"1min":0.25800000000000001,"5min":0.246,"15min":0.22700000000000001},"max":{"1min":3.835,"5min":10.611000000000001,"15min":135.023},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.81000000000000005,"5min":2.0470000000000002,"15min":3.1299999999999999},"min":{"1min":0.34100000000000003,"5min":0.28599999999999998,"15min":0.23599999999999999},"max":{"1min":3.1709999999999998,"5min":13.688000000000001,"15min":134.51499999999999},"last":0.41999999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:30:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":2.1179999999999999,"15min":3.1549999999999998},"min":{"1min":0.28000000000000003,"5min":0.249,"15min":0.249},"max":{"1min":7.9029999999999996,"5min":10.452,"15min":104.837},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.85499999999999998,"5min":1.8440000000000001,"15min":3.0259999999999998},"min":{"1min":0.35999999999999999,"5min":0.247,"15min":0.247},"max":{"1min":3.3839999999999999,"5min":12.388999999999999,"15min":106.379},"last":4.726}]},{"osd":3,"last update":"Sun Oct 1 09:29:43 2023","interfaces":[{"interface":"back","average":{"1min":4.9340000000000002,"5min":4.7000000000000002,"15min":3.27},"min":{"1min":0.34799999999999998,"5min":0.28199999999999997,"15min":0.26400000000000001},"max":{"1min":80.352999999999994,"5min":164.595,"15min":164.595},"last":4.694},{"interface":"front","average":{"1min":4.9740000000000002,"5min":4.7610000000000001,"15min":3.238},"min":{"1min":0.33100000000000002,"5min":0.32200000000000001,"15min":0.27200000000000002},"max":{"1min":80.364000000000004,"5min":164.60499999999999,"15min":164.60499999999999},"last":6.835}]},{"osd":4,"last update":"Sun Oct 1 09:30:24 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.53000000000000003,"15min":1.595},"min":{"1min":0.29999999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.1559999999999999,"5min":3.8420000000000001,"15min":82.992000000000004},"last":0.371},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.59099999999999997,"15min":1.6559999999999999},"min":{"1min":0.32500000000000001,"5min":0.26400000000000001,"15min":0.252},"max":{"1min":2.677,"5min":3.25,"15min":83.058999999999997},"last":0.45900000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:30:24 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.64700000000000002,"15min":1.984},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.8300000000000001,"5min":4.6989999999999998,"15min":13.265000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.68400000000000005,"15min":1.909},"min":{"1min":0.311,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.4860000000000002,"5min":3.7639999999999998,"15min":13.292999999999999},"last":0.40100000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:29:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.76700000000000002,"15min":0.78900000000000003},"min":{"1min":0.371,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.97099999999999997,"5min":3.306,"15min":3.306},"last":0.749},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.69899999999999995,"15min":0.63700000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.87,"5min":2.8300000000000001,"15min":2.8300000000000001},"last":0.44900000000000001}]}]},{"osd":6,"up_from":294,"seq":1262720385069,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2257944,"kb_used_data":1920336,"kb_used_omap":6,"kb_used_meta":337593,"kb_avail":91487208,"statfs":{"total":95995035648,"available":93682900992,"internally_reserved":0,"allocated":1966424064,"data_stored":1965561897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":345695277},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":1.73,"15min":1.9239999999999999},"min":{"1min":0.23799999999999999,"5min":0.222,"15min":0.222},"max":{"1min":3.7999999999999998,"5min":8.9960000000000004,"15min":8.9960000000000004},"last":62.569000000000003},{"interface":"front","average":{"1min":1.369,"5min":1.6699999999999999,"15min":1.524},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.1470000000000002,"5min":10.329000000000001,"15min":10.329000000000001},"last":62.912999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.619,"15min":1.7549999999999999},"min":{"1min":0.30299999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.878,"5min":7.7560000000000002,"15min":7.7560000000000002},"last":62.698999999999998},{"interface":"front","average":{"1min":0.58399999999999996,"5min":1.516,"15min":1.734},"min":{"1min":0.38500000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.3859999999999999,"5min":4.9749999999999996,"15min":4.9749999999999996},"last":62.704000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:29:51 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":2.4340000000000002,"15min":2.8119999999999998},"min":{"1min":0.29999999999999999,"5min":0.254,"15min":0.254},"max":{"1min":6.1100000000000003,"5min":26.289999999999999,"15min":26.289999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.82299999999999995,"5min":2.5419999999999998,"15min":2.9900000000000002},"min":{"1min":0.371,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.1850000000000001,"5min":26.388999999999999,"15min":26.388999999999999},"last":0.38700000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:29:51 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":2.3370000000000002,"15min":2.4239999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":3.7509999999999999,"5min":10.632999999999999,"15min":10.632999999999999},"last":62.546999999999997},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.498,"15min":1.458},"min":{"1min":0.33400000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.911,"5min":8.8680000000000003,"15min":8.8680000000000003},"last":62.465000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.44900000000000001,"15min":0.39700000000000002},"min":{"1min":0.28899999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.3100000000000001,"5min":3.1560000000000001,"15min":3.1560000000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.627,"15min":0.56299999999999994},"min":{"1min":0.30099999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.0609999999999999,"5min":4.1319999999999997,"15min":4.1319999999999997},"last":0.36799999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.53500000000000003,"15min":0.46000000000000002},"min":{"1min":0.28799999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.3600000000000001,"5min":4.6200000000000001,"15min":4.6200000000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.65200000000000002,"15min":0.59599999999999997},"min":{"1min":0.31900000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.6440000000000001,"5min":4.0919999999999996,"15min":4.0919999999999996},"last":0.33200000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":4.1740000000000004,"15min":5.1449999999999996},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.93600000000000005,"5min":115.11,"15min":115.11},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.49399999999999999,"5min":4.1159999999999997,"15min":5.0819999999999999},"min":{"1min":0.35399999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.67300000000000004,"5min":114.666,"15min":114.666},"last":0.34499999999999997}]}]},{"osd":1,"up_from":94,"seq":403726926046,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2556572,"kb_used_data":2097300,"kb_used_omap":7,"kb_used_meta":459256,"kb_avail":91188580,"statfs":{"total":95995035648,"available":93377105920,"internally_reserved":0,"allocated":2147635200,"data_stored":2146072313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":470278567},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":3.6000000000000001,"15min":4.0110000000000001},"min":{"1min":0.19600000000000001,"5min":0.191,"15min":0.17499999999999999},"max":{"1min":5.8789999999999996,"5min":37.101999999999997,"15min":174.40199999999999},"last":4.5110000000000001},{"interface":"front","average":{"1min":1.4279999999999999,"5min":3.3929999999999998,"15min":4.1210000000000004},"min":{"1min":0.188,"5min":0.188,"15min":0.17599999999999999},"max":{"1min":8.4420000000000002,"5min":36.795999999999999,"15min":173.77000000000001},"last":5.3360000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:30:14 2023","interfaces":[{"interface":"back","average":{"1min":3.661,"5min":4.7080000000000002,"15min":3.274},"min":{"1min":0.28499999999999998,"5min":0.215,"15min":0.182},"max":{"1min":56.573,"5min":56.573,"15min":77.325000000000003},"last":4.4480000000000004},{"interface":"front","average":{"1min":4.0460000000000003,"5min":5.2069999999999999,"15min":3.71},"min":{"1min":0.27800000000000002,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":55.930999999999997,"5min":55.930999999999997,"15min":80.361000000000004},"last":5.3639999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:29:46 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":4.649,"15min":3.5019999999999998},"min":{"1min":0.28499999999999998,"5min":0.254,"15min":0.218},"max":{"1min":2.7429999999999999,"5min":66.334000000000003,"15min":99.421000000000006},"last":5.3040000000000003},{"interface":"front","average":{"1min":0.77700000000000002,"5min":4.673,"15min":3.3620000000000001},"min":{"1min":0.27900000000000003,"5min":0.21199999999999999,"15min":0.20999999999999999},"max":{"1min":3.4129999999999998,"5min":69.641000000000005,"15min":99.364999999999995},"last":4.1429999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2789999999999999,"5min":1.6599999999999999,"15min":1.663},"min":{"1min":0.36399999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.234,"5min":10.231,"15min":40.673999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.3080000000000001,"5min":2.2530000000000001,"15min":2.0030000000000001},"min":{"1min":0.29999999999999999,"5min":0.27500000000000002,"15min":0.25800000000000001},"max":{"1min":5.4640000000000004,"5min":18.219000000000001,"15min":40.810000000000002},"last":4.266}]},{"osd":5,"last update":"Sun Oct 1 09:30:14 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.6839999999999999,"15min":1.444},"min":{"1min":0.35499999999999998,"5min":0.314,"15min":0.29599999999999999},"max":{"1min":2.665,"5min":10.273,"15min":10.273},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.0189999999999999,"5min":2.298,"15min":2.2170000000000001},"min":{"1min":0.39000000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.871,"5min":18.265000000000001,"15min":18.265000000000001},"last":4.3609999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:29:54 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":2.5289999999999999,"15min":2.9889999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.0099999999999998,"5min":10.196999999999999,"15min":10.196999999999999},"last":5.2690000000000001},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.7150000000000001,"15min":1.9239999999999999},"min":{"1min":0.40300000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.98299999999999998,"5min":7.2679999999999998,"15min":7.2679999999999998},"last":1.7150000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.7669999999999999,"15min":1.829},"min":{"1min":0.45600000000000002,"5min":0.316,"15min":0.316},"max":{"1min":6.2610000000000001,"5min":10.305999999999999,"15min":23.710999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.0329999999999999,"5min":2.1419999999999999,"15min":1.9790000000000001},"min":{"1min":0.436,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.2599999999999998,"5min":18.291,"15min":23.736999999999998},"last":4.476}]}]},{"osd":0,"up_from":89,"seq":382252089580,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2744492,"kb_used_data":2115812,"kb_used_omap":8,"kb_used_meta":628663,"kb_avail":91000660,"statfs":{"total":95995035648,"available":93184675840,"internally_reserved":0,"allocated":2166591488,"data_stored":2165017097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":643751874},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:29:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":2.5539999999999998,"15min":2.7890000000000001},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":7.1529999999999996,"5min":47.244999999999997,"15min":187.41399999999999},"last":1.871},{"interface":"front","average":{"1min":2.5569999999999999,"5min":3.98,"15min":3.4630000000000001},"min":{"1min":0.25900000000000001,"5min":0.21199999999999999,"15min":0.193},"max":{"1min":21.968,"5min":48.042000000000002,"15min":187.78700000000001},"last":2.952}]},{"osd":2,"last update":"Sun Oct 1 09:30:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":1.974,"15min":2.113},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":4.6539999999999999,"5min":17.731000000000002,"15min":39.048999999999999},"last":1.3160000000000001},{"interface":"front","average":{"1min":1.8959999999999999,"5min":3.8540000000000001,"15min":2.9790000000000001},"min":{"1min":0.27200000000000002,"5min":0.26600000000000001,"15min":0.24299999999999999},"max":{"1min":6.3979999999999997,"5min":22.07,"15min":38.945999999999998},"last":3.0670000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:29:38 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":5.1539999999999999,"15min":3.2829999999999999},"min":{"1min":0.32700000000000001,"5min":0.249,"15min":0.23400000000000001},"max":{"1min":11.241,"5min":139.85300000000001,"15min":139.85300000000001},"last":3.7069999999999999},{"interface":"front","average":{"1min":2.294,"5min":6.3200000000000003,"15min":3.8799999999999999},"min":{"1min":0.313,"5min":0.219,"15min":0.219},"max":{"1min":7.048,"5min":139.70099999999999,"15min":139.70099999999999},"last":2.246}]},{"osd":4,"last update":"Sun Oct 1 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2120000000000002,"5min":2.3660000000000001,"15min":2.488},"min":{"1min":0.32100000000000001,"5min":0.28199999999999997,"15min":0.27300000000000002},"max":{"1min":18.600999999999999,"5min":18.600999999999999,"15min":112.52200000000001},"last":3.0059999999999998},{"interface":"front","average":{"1min":1.2929999999999999,"5min":2.387,"15min":2.573},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26900000000000002},"max":{"1min":5.7130000000000001,"5min":28.035,"15min":113.124},"last":0.96799999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:30:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":2.54,"15min":1.9810000000000001},"min":{"1min":0.35399999999999998,"5min":0.317,"15min":0.317},"max":{"1min":5.2999999999999998,"5min":28.074000000000002,"15min":28.074000000000002},"last":0.997},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.109,"15min":1.569},"min":{"1min":0.379,"5min":0.312,"15min":0.31},"max":{"1min":3.109,"5min":11.351000000000001,"15min":11.351000000000001},"last":1.022}]},{"osd":6,"last update":"Sun Oct 1 09:29:58 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.2669999999999999,"15min":1.4079999999999999},"min":{"1min":0.42099999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.0219999999999998,"5min":11.387,"15min":11.387},"last":2.3849999999999998},{"interface":"front","average":{"1min":1.034,"5min":2.4060000000000001,"15min":2.472},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.8370000000000002,"5min":18.521000000000001,"15min":18.521000000000001},"last":3.0379999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:29:32 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":1.258,"15min":2.0939999999999999},"min":{"1min":0.35699999999999998,"5min":0.34499999999999997,"15min":0.29199999999999998},"max":{"1min":2.597,"5min":11.413,"15min":98.283000000000001},"last":1.109},{"interface":"front","average":{"1min":2.29,"5min":2.6789999999999998,"15min":2.5680000000000001},"min":{"1min":0.373,"5min":0.36099999999999999,"15min":0.312},"max":{"1min":18.565999999999999,"5min":18.565999999999999,"15min":98.311999999999998},"last":3.1030000000000002}]}]},{"osd":2,"up_from":99,"seq":425201762519,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2563644,"kb_used_data":2031028,"kb_used_omap":11,"kb_used_meta":532596,"kb_avail":91181508,"statfs":{"total":95995035648,"available":93369864192,"internally_reserved":0,"allocated":2079772672,"data_stored":2078244453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":545379191},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:30:23 2023","interfaces":[{"interface":"back","average":{"1min":12.760999999999999,"5min":9.8350000000000009,"15min":5.3479999999999999},"min":{"1min":0.249,"5min":0.20499999999999999,"15min":0.17599999999999999},"max":{"1min":224.56,"5min":224.56,"15min":224.56},"last":3.9209999999999998},{"interface":"front","average":{"1min":11.824,"5min":10.765000000000001,"15min":6.0999999999999996},"min":{"1min":0.215,"5min":0.20699999999999999,"15min":0.20100000000000001},"max":{"1min":225.66900000000001,"5min":225.66900000000001,"15min":225.66900000000001},"last":0.41999999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:30:16 2023","interfaces":[{"interface":"back","average":{"1min":13.784000000000001,"5min":11.163,"15min":5.8879999999999999},"min":{"1min":0.26200000000000001,"5min":0.19500000000000001,"15min":0.193},"max":{"1min":224.917,"5min":224.917,"15min":224.917},"last":3.9609999999999999},{"interface":"front","average":{"1min":13.742000000000001,"5min":12.964,"15min":6.7110000000000003},"min":{"1min":0.27900000000000003,"5min":0.222,"15min":0.20899999999999999},"max":{"1min":225.625,"5min":225.625,"15min":225.625},"last":0.74299999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:29:41 2023","interfaces":[{"interface":"back","average":{"1min":3.718,"5min":7.5350000000000001,"15min":5.2619999999999996},"min":{"1min":0.26100000000000001,"5min":0.218,"15min":0.19600000000000001},"max":{"1min":52.115000000000002,"5min":138.85900000000001,"15min":253.238},"last":3.895},{"interface":"front","average":{"1min":4.0720000000000001,"5min":7.2210000000000001,"15min":5.0510000000000002},"min":{"1min":0.32300000000000001,"5min":0.245,"15min":0.22600000000000001},"max":{"1min":53.067,"5min":136.66399999999999,"15min":252.54900000000001},"last":3.9900000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:30:16 2023","interfaces":[{"interface":"back","average":{"1min":11.555,"5min":4.7110000000000003,"15min":2.762},"min":{"1min":0.36499999999999999,"5min":0.317,"15min":0.27000000000000002},"max":{"1min":224.90199999999999,"5min":224.90199999999999,"15min":224.90199999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":11.621,"5min":4.4589999999999996,"15min":2.7959999999999998},"min":{"1min":0.33200000000000002,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":225.386,"5min":225.386,"15min":225.386},"last":2.4220000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:30:16 2023","interfaces":[{"interface":"back","average":{"1min":11.669,"5min":3.528,"15min":2.6150000000000002},"min":{"1min":0.40000000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":224.989,"5min":224.989,"15min":224.989},"last":4.024},{"interface":"front","average":{"1min":11.519,"5min":4.0999999999999996,"15min":2.7949999999999999},"min":{"1min":0.33700000000000002,"5min":0.309,"15min":0.309},"max":{"1min":224.81,"5min":224.81,"15min":224.81},"last":4.0890000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:29:50 2023","interfaces":[{"interface":"back","average":{"1min":12.066000000000001,"5min":4.0650000000000004,"15min":2.7469999999999999},"min":{"1min":0.38200000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":225.41499999999999,"5min":225.41499999999999,"15min":225.41499999999999},"last":4.3899999999999997},{"interface":"front","average":{"1min":11.970000000000001,"5min":3.4119999999999999,"15min":2.0579999999999998},"min":{"1min":0.46899999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":225.02000000000001,"5min":225.02000000000001,"15min":225.02000000000001},"last":4.0540000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:30:16 2023","interfaces":[{"interface":"back","average":{"1min":11.705,"5min":4.4870000000000001,"15min":3.1030000000000002},"min":{"1min":0.47099999999999997,"5min":0.308,"15min":0.308},"max":{"1min":224.94999999999999,"5min":224.94999999999999,"15min":224.94999999999999},"last":5.0099999999999998},{"interface":"front","average":{"1min":11.569000000000001,"5min":5.1029999999999998,"15min":3.4900000000000002},"min":{"1min":0.40799999999999997,"5min":0.35299999999999998,"15min":0.33600000000000002},"max":{"1min":224.92599999999999,"5min":224.92599999999999,"15min":224.92599999999999},"last":0.58999999999999997}]}]},{"osd":3,"up_from":104,"seq":446676598993,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2606124,"kb_used_data":2067620,"kb_used_omap":3,"kb_used_meta":538492,"kb_avail":91139028,"statfs":{"total":95995035648,"available":93326364672,"internally_reserved":0,"allocated":2117242880,"data_stored":2115664032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":551416551},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":6.5069999999999997,"15min":5.2270000000000003},"min":{"1min":0.29999999999999999,"5min":0.18099999999999999,"15min":0.17100000000000001},"max":{"1min":3.6419999999999999,"5min":128.47800000000001,"15min":199.422},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.96599999999999997,"5min":6.1029999999999998,"15min":5.2610000000000001},"min":{"1min":0.23200000000000001,"5min":0.184,"15min":0.184},"max":{"1min":4.0839999999999996,"5min":127.15600000000001,"15min":199.91200000000001},"last":0.38800000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":4.6109999999999998,"15min":4.8600000000000003},"min":{"1min":0.26600000000000001,"5min":0.23999999999999999,"15min":0.21099999999999999},"max":{"1min":9.0630000000000006,"5min":120.79300000000001,"15min":228.60499999999999},"last":0.65200000000000002},{"interface":"front","average":{"1min":1.1759999999999999,"5min":5.8259999999999996,"15min":5.1189999999999998},"min":{"1min":0.29099999999999998,"5min":0.23400000000000001,"15min":0.215},"max":{"1min":3.9359999999999999,"5min":121.455,"15min":228.28399999999999},"last":0.253}]},{"osd":2,"last update":"Sun Oct 1 09:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":5.343,"15min":3.3700000000000001},"min":{"1min":0.27500000000000002,"5min":0.22800000000000001,"15min":0.224},"max":{"1min":6.0579999999999998,"5min":62.909999999999997,"15min":62.909999999999997},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.84599999999999997,"5min":6.165,"15min":3.8100000000000001},"min":{"1min":0.35599999999999998,"5min":0.255,"15min":0.20399999999999999},"max":{"1min":3.1850000000000001,"5min":78.594999999999999,"15min":78.594999999999999},"last":0.52500000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:29:45 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":1.97,"15min":2.7989999999999999},"min":{"1min":0.46700000000000003,"5min":0.29799999999999999,"15min":0.25900000000000001},"max":{"1min":3.1040000000000001,"5min":16,"15min":154.00399999999999},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.8859999999999999,"15min":3.04},"min":{"1min":0.371,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.3079999999999998,"5min":15.257999999999999,"15min":156.87700000000001},"last":0.76600000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:30:14 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":1.9330000000000001,"15min":1.8089999999999999},"min":{"1min":0.33200000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.3529999999999998,"5min":15.351000000000001,"15min":15.351000000000001},"last":0.82299999999999995},{"interface":"front","average":{"1min":2.024,"5min":1.962,"15min":2.9260000000000002},"min":{"1min":0.39800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.4269999999999996,"5min":15.382,"15min":36.984000000000002},"last":0.79700000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:29:55 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":2.9180000000000001,"15min":3.0720000000000001},"min":{"1min":0.39900000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.0960000000000001,"5min":18.361000000000001,"15min":18.361000000000001},"last":1.2270000000000001},{"interface":"front","average":{"1min":0.85299999999999998,"5min":2.7000000000000002,"15min":3.028},"min":{"1min":0.40899999999999997,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.948,"5min":18.332000000000001,"15min":18.332000000000001},"last":1.2050000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:29:45 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":3.5,"15min":2.698},"min":{"1min":0.42999999999999999,"5min":0.309,"15min":0.309},"max":{"1min":2.8580000000000001,"5min":52.006,"15min":79.412999999999997},"last":1.173},{"interface":"front","average":{"1min":0.94499999999999995,"5min":2.8130000000000002,"15min":2.5289999999999999},"min":{"1min":0.503,"5min":0.34200000000000003,"15min":0.308},"max":{"1min":3.1459999999999999,"5min":51.890999999999998,"15min":79.527000000000001},"last":1.056}]}]},{"osd":4,"up_from":0,"seq":128849019243,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":227,"seq":974957576294,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2648844,"kb_used_data":2321412,"kb_used_omap":8,"kb_used_meta":327415,"kb_avail":91096308,"statfs":{"total":95995035648,"available":93282619392,"internally_reserved":0,"allocated":2377125888,"data_stored":2376271566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":335273732},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":4.1609999999999996,"15min":2.6909999999999998},"min":{"1min":0.223,"5min":0.219,"15min":0.219},"max":{"1min":1.8480000000000001,"5min":169.517,"15min":169.517},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.91800000000000004,"5min":4.5259999999999998,"15min":3.1509999999999998},"min":{"1min":0.30499999999999999,"5min":0.219,"15min":0.219},"max":{"1min":2.9729999999999999,"5min":169.69900000000001,"15min":169.69900000000001},"last":0.59399999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:30:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":3.1419999999999999,"15min":2.8140000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":5.6159999999999997,"5min":115.976,"15min":115.976},"last":92.899000000000001},{"interface":"front","average":{"1min":1.3700000000000001,"5min":2.871,"15min":3.222},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":10.08,"5min":115.732,"15min":115.732},"last":92.180999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:30:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.7709999999999999,"15min":2.5590000000000002},"min":{"1min":0.36199999999999999,"5min":0.27600000000000002,"15min":0.25900000000000001},"max":{"1min":10.288,"5min":10.288,"15min":44.615000000000002},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.1779999999999999,"5min":2.0680000000000001,"15min":2.0550000000000002},"min":{"1min":0.37,"5min":0.26700000000000002,"15min":0.26600000000000001},"max":{"1min":4.6909999999999998,"5min":14.83,"15min":44.511000000000003},"last":0.624}]},{"osd":3,"last update":"Sun Oct 1 09:30:16 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":5.4249999999999998,"15min":3.7570000000000001},"min":{"1min":0.33900000000000002,"5min":0.307,"15min":0.307},"max":{"1min":5.5060000000000002,"5min":214.73599999999999,"15min":214.73599999999999},"last":85.387},{"interface":"front","average":{"1min":1.4099999999999999,"5min":5.7229999999999999,"15min":3.843},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.6600000000000001,"5min":214.86699999999999,"15min":214.86699999999999},"last":85.346999999999994}]},{"osd":4,"last update":"Sun Oct 1 09:30:16 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.49099999999999999,"15min":1.2629999999999999},"min":{"1min":0.27500000000000002,"5min":0.26200000000000001,"15min":0.248},"max":{"1min":1.1140000000000001,"5min":2.8029999999999999,"15min":7.532},"last":1.6819999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.495,"15min":1.0840000000000001},"min":{"1min":0.28999999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.7290000000000001,"5min":2.2810000000000001,"15min":4.117},"last":0.64600000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:29:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.54800000000000004,"15min":0.502},"min":{"1min":0.32000000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.778,"5min":2.778,"15min":2.778},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.59699999999999998,"15min":0.51100000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.72,"5min":4.806,"15min":4.806},"last":1.6299999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:30:16 2023","interfaces":[{"interface":"back","average":{"1min":4.7649999999999997,"5min":2.1589999999999998,"15min":1.724},"min":{"1min":0.32300000000000001,"5min":0.252,"15min":0.252},"max":{"1min":84.105000000000004,"5min":84.105000000000004,"15min":84.105000000000004},"last":0.66900000000000004},{"interface":"front","average":{"1min":4.6120000000000001,"5min":2.1589999999999998,"15min":1.994},"min":{"1min":0.30099999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":82.069000000000003,"5min":83.495000000000005,"15min":83.495000000000005},"last":1.6599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2172452864,"data_stored":2172452864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2151219200,"data_stored":2151219200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2095710208,"data_stored":2095710208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2115829760,"data_stored":2115829760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2382495744,"data_stored":2382495744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1815216128,"data_stored":1815216128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1827405824,"data_stored":1827405824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:30:31.218 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-01T09:30:31.219 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:30:31.219 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T09:30:31.219 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.000000 2023-10-01T09:30:31.219 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 0 2023-10-01T09:30:31.293 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:31.324 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:31.378 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 58 16 77715 77699 83.6835 49.3125 0.0126038 0.0119441 2023-10-01T09:30:31.435 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:31.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:31.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:31.541 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:31.634 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:31.879 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:30:31.879 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:30:31.880 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:30:31.880 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:30:31.880 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.880 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:95124 7.1 e271/269 7:84c3d6a9:::benchmark_data_smithi138_367058_object95123:head v 271'5868, mlcod=271'5866)", 2023-10-01T09:30:31.880 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:25:20.262562+0000", 2023-10-01T09:30:31.880 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 311.61217332699999, 2023-10-01T09:30:31.880 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.079727882999999999, 2023-10-01T09:30:31.880 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.880 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.880 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.880 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.881 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.881 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.262562+0000", 2023-10-01T09:30:31.881 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.881 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.881 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.881 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.881 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.262562+0000", 2023-10-01T09:30:31.881 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:30:31.881 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.881 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.881 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.881 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.262561+0000", 2023-10-01T09:30:31.882 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5702000000000003e-05 2023-10-01T09:30:31.882 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.882 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.882 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.882 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.262617+0000", 2023-10-01T09:30:31.882 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0469999999999994e-06 2023-10-01T09:30:31.882 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.882 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.882 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.882 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.262625+0000", 2023-10-01T09:30:31.882 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4139999999999996e-06 2023-10-01T09:30:31.883 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.883 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.883 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.883 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.262629+0000", 2023-10-01T09:30:31.883 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.070540481000000002 2023-10-01T09:30:31.883 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.883 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.883 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.883 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.333170+0000", 2023-10-01T09:30:31.883 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1875e-05 2023-10-01T09:30:31.883 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.883 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.884 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.884 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.333231+0000", 2023-10-01T09:30:31.884 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0090221510000000008 2023-10-01T09:30:31.884 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.884 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.884 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.884 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.342254+0000", 2023-10-01T09:30:31.884 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5887999999999999e-05 2023-10-01T09:30:31.884 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.884 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.884 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.884 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.342289+0000", 2023-10-01T09:30:31.885 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.079727882999999999 2023-10-01T09:30:31.885 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.885 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.885 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.885 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.885 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.885 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:167311 7.1 e309/307 7:8b26f75e:::benchmark_data_smithi138_367058_object69516:head v 309'10334, mlcod=270'5583)", 2023-10-01T09:30:31.885 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:10.923353+0000", 2023-10-01T09:30:31.885 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 140.951382362, 2023-10-01T09:30:31.885 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016899226999999999, 2023-10-01T09:30:31.885 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.885 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.886 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.886 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.886 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.886 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923353+0000", 2023-10-01T09:30:31.886 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.886 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.886 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.886 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.886 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923353+0000", 2023-10-01T09:30:31.886 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:30:31.886 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.886 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.887 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.887 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923350+0000", 2023-10-01T09:30:31.887 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.552e-05 2023-10-01T09:30:31.887 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.887 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.887 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.887 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923366+0000", 2023-10-01T09:30:31.887 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1384000000000001e-05 2023-10-01T09:30:31.887 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.887 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.887 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.887 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923387+0000", 2023-10-01T09:30:31.888 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3260000000000002e-06 2023-10-01T09:30:31.888 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.888 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.888 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.888 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923396+0000", 2023-10-01T09:30:31.888 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0259000000000005e-05 2023-10-01T09:30:31.888 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.888 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.888 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.888 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923477+0000", 2023-10-01T09:30:31.888 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0988000000000002e-05 2023-10-01T09:30:31.889 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.889 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.889 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.889 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923508+0000", 2023-10-01T09:30:31.889 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016684719000000001 2023-10-01T09:30:31.889 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.889 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.889 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.889 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.940192+0000", 2023-10-01T09:30:31.889 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9469999999999998e-05 2023-10-01T09:30:31.889 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.889 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.890 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.890 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.940252+0000", 2023-10-01T09:30:31.890 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016899226999999999 2023-10-01T09:30:31.890 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.890 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.890 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.890 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.890 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.890 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:167312 7.1 e309/307 7:8c38bd60:::benchmark_data_smithi138_367058_object69517:head v 309'10335, mlcod=270'5583)", 2023-10-01T09:30:31.890 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:10.923440+0000", 2023-10-01T09:30:31.890 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 140.95129462200001, 2023-10-01T09:30:31.890 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016832646, 2023-10-01T09:30:31.891 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.891 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.891 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.891 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.891 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.891 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923440+0000", 2023-10-01T09:30:31.891 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.891 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.891 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.891 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.892 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923440+0000", 2023-10-01T09:30:31.892 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967296 2023-10-01T09:30:31.892 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.892 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.892 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.892 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923440+0000", 2023-10-01T09:30:31.892 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2470000000000008e-06 2023-10-01T09:30:31.892 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.892 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.892 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.892 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923448+0000", 2023-10-01T09:30:31.892 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1432e-05 2023-10-01T09:30:31.893 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.893 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.893 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.893 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923460+0000", 2023-10-01T09:30:31.893 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5899999999999999e-06 2023-10-01T09:30:31.893 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.893 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.893 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.893 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923463+0000", 2023-10-01T09:30:31.893 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0031189730000000001 2023-10-01T09:30:31.893 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.894 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.894 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.894 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.926582+0000", 2023-10-01T09:30:31.894 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8634999999999999e-05 2023-10-01T09:30:31.894 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.894 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.894 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.894 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.926611+0000", 2023-10-01T09:30:31.894 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013645309 2023-10-01T09:30:31.894 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.894 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.895 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.895 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.940256+0000", 2023-10-01T09:30:31.895 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6685999999999999e-05 2023-10-01T09:30:31.895 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.895 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.895 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.895 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.940273+0000", 2023-10-01T09:30:31.895 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016832646 2023-10-01T09:30:31.895 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.895 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.895 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.895 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.896 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.896 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:167314 7.1 e309/307 7:8eca8555:::benchmark_data_smithi138_367058_object69519:head v 309'10336, mlcod=270'5583)", 2023-10-01T09:30:31.896 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:10.923558+0000", 2023-10-01T09:30:31.896 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 140.95117684499999, 2023-10-01T09:30:31.896 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016731647999999998, 2023-10-01T09:30:31.896 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.896 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.896 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.896 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.896 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.896 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923558+0000", 2023-10-01T09:30:31.896 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.897 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.897 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.897 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.897 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923558+0000", 2023-10-01T09:30:31.897 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:30:31.897 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.897 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.897 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.897 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.923557+0000", 2023-10-01T09:30:31.897 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010673619000000001 2023-10-01T09:30:31.897 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.897 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.898 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.898 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.934231+0000", 2023-10-01T09:30:31.898 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5715e-05 2023-10-01T09:30:31.898 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.898 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.898 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.898 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.934257+0000", 2023-10-01T09:30:31.898 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5739999999999998e-06 2023-10-01T09:30:31.898 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.898 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.898 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.898 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.934264+0000", 2023-10-01T09:30:31.899 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013063199999999999 2023-10-01T09:30:31.899 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.899 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.899 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.899 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.934395+0000", 2023-10-01T09:30:31.899 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6699e-05 2023-10-01T09:30:31.899 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.899 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.899 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.899 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.934422+0000", 2023-10-01T09:30:31.899 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0058543620000000001 2023-10-01T09:30:31.899 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.900 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.900 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.900 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.940276+0000", 2023-10-01T09:30:31.900 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3626e-05 2023-10-01T09:30:31.900 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.900 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.900 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.900 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:10.940290+0000", 2023-10-01T09:30:31.900 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016731647999999998 2023-10-01T09:30:31.900 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.900 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.900 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.901 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.901 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.901 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:167748 7.1 e309/307 7:8e571860:::benchmark_data_smithi138_367058_object69953:head v 309'10359, mlcod=270'5583)", 2023-10-01T09:30:31.901 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:11.590318+0000", 2023-10-01T09:30:31.901 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 140.28441645800001, 2023-10-01T09:30:31.901 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017835182000000002, 2023-10-01T09:30:31.901 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.901 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.901 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.901 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.901 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.902 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590318+0000", 2023-10-01T09:30:31.902 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.902 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.902 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.902 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.902 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590318+0000", 2023-10-01T09:30:31.902 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:30:31.902 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.902 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.902 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.902 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590317+0000", 2023-10-01T09:30:31.902 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7569000000000001e-05 2023-10-01T09:30:31.902 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.903 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.903 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.903 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590334+0000", 2023-10-01T09:30:31.903 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8146000000000003e-05 2023-10-01T09:30:31.903 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.903 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.903 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.903 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590382+0000", 2023-10-01T09:30:31.903 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9840000000000001e-06 2023-10-01T09:30:31.903 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.903 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.903 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.904 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590390+0000", 2023-10-01T09:30:31.904 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003199726 2023-10-01T09:30:31.904 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.904 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.904 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.904 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.593590+0000", 2023-10-01T09:30:31.904 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9781000000000002e-05 2023-10-01T09:30:31.904 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.904 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.904 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.904 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.593620+0000", 2023-10-01T09:30:31.904 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014491779999999999 2023-10-01T09:30:31.904 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.905 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.905 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.905 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.608112+0000", 2023-10-01T09:30:31.905 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1876999999999999e-05 2023-10-01T09:30:31.905 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.905 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.905 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.905 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.608154+0000", 2023-10-01T09:30:31.905 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017835182000000002 2023-10-01T09:30:31.905 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.906 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.906 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.906 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.906 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.906 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:167751 7.1 e309/307 7:87025a43:::benchmark_data_smithi138_367058_object69956:head v 309'10360, mlcod=270'5583)", 2023-10-01T09:30:31.906 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:11.590423+0000", 2023-10-01T09:30:31.906 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 140.28431175, 2023-10-01T09:30:31.906 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017751335, 2023-10-01T09:30:31.906 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.906 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.906 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.906 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.907 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.907 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590423+0000", 2023-10-01T09:30:31.907 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.907 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.907 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.907 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.907 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590423+0000", 2023-10-01T09:30:31.907 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:30:31.907 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.907 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.907 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.907 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590423+0000", 2023-10-01T09:30:31.908 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0379999999999996e-06 2023-10-01T09:30:31.908 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.908 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.908 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.908 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590429+0000", 2023-10-01T09:30:31.908 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8630000000000002e-06 2023-10-01T09:30:31.908 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.908 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.908 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.908 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590436+0000", 2023-10-01T09:30:31.908 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4360000000000004e-06 2023-10-01T09:30:31.908 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.909 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.909 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.909 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590440+0000", 2023-10-01T09:30:31.909 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0063771230000000002 2023-10-01T09:30:31.909 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.909 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.909 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.909 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.596817+0000", 2023-10-01T09:30:31.909 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5887000000000001e-05 2023-10-01T09:30:31.909 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.909 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.909 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.910 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.596843+0000", 2023-10-01T09:30:31.910 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011313986 2023-10-01T09:30:31.910 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.910 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.910 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.910 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.608157+0000", 2023-10-01T09:30:31.910 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7524e-05 2023-10-01T09:30:31.910 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.910 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.910 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.910 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.608174+0000", 2023-10-01T09:30:31.911 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017751335 2023-10-01T09:30:31.911 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.911 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.911 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.911 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.911 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.911 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:171330 7.4 e309/306 7:2e0ad103:::benchmark_data_smithi138_367058_object73535:head v 309'10800, mlcod=309'10798)", 2023-10-01T09:30:31.911 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:17.502720+0000", 2023-10-01T09:30:31.911 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 134.37201447499999, 2023-10-01T09:30:31.911 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014946232, 2023-10-01T09:30:31.911 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.911 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.912 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.912 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.912 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.912 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:17.502720+0000", 2023-10-01T09:30:31.912 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.912 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.912 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.912 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.912 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:17.502720+0000", 2023-10-01T09:30:31.912 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:30:31.912 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.913 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.913 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.913 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:17.502719+0000", 2023-10-01T09:30:31.913 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4698000000000001e-05 2023-10-01T09:30:31.913 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.913 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.913 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.913 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:17.502733+0000", 2023-10-01T09:30:31.913 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1339e-05 2023-10-01T09:30:31.913 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.913 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.913 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.914 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:17.502755+0000", 2023-10-01T09:30:31.914 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1219999999999995e-06 2023-10-01T09:30:31.914 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.914 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.914 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.914 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:17.502763+0000", 2023-10-01T09:30:31.914 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0041534529999999997 2023-10-01T09:30:31.914 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.914 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.914 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.914 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:17.506916+0000", 2023-10-01T09:30:31.914 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4403999999999999e-05 2023-10-01T09:30:31.915 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.915 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.915 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.915 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:17.506941+0000", 2023-10-01T09:30:31.915 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010684568 2023-10-01T09:30:31.915 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.915 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.915 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.915 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:17.517625+0000", 2023-10-01T09:30:31.915 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1335000000000003e-05 2023-10-01T09:30:31.915 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.915 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.916 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.916 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:17.517667+0000", 2023-10-01T09:30:31.916 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014946232 2023-10-01T09:30:31.916 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.916 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.916 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.916 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.916 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.916 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:172367 7.3 e310/306 7:ced4301a:::benchmark_data_smithi138_367058_object74572:head v 310'10724, mlcod=310'10721)", 2023-10-01T09:30:31.916 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:19.134310+0000", 2023-10-01T09:30:31.916 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 132.74042467999999, 2023-10-01T09:30:31.916 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015907355000000001, 2023-10-01T09:30:31.917 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.917 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.917 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.917 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.917 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.917 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:19.134310+0000", 2023-10-01T09:30:31.917 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.917 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.917 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.917 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.917 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:19.134310+0000", 2023-10-01T09:30:31.917 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:30:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:19.134308+0000", 2023-10-01T09:30:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4918e-05 2023-10-01T09:30:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:19.134323+0000", 2023-10-01T09:30:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9369999999999998e-05 2023-10-01T09:30:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.918 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.919 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:19.134353+0000", 2023-10-01T09:30:31.919 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2799999999999992e-06 2023-10-01T09:30:31.919 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.919 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.919 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.921 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:19.134362+0000", 2023-10-01T09:30:31.921 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0037175369999999999 2023-10-01T09:30:31.921 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.921 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.921 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.921 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:19.138079+0000", 2023-10-01T09:30:31.922 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6466e-05 2023-10-01T09:30:31.922 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.922 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.922 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.922 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:19.138106+0000", 2023-10-01T09:30:31.922 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012069295000000001 2023-10-01T09:30:31.922 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.922 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.922 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.922 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:19.150175+0000", 2023-10-01T09:30:31.922 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2434000000000001e-05 2023-10-01T09:30:31.923 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.923 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.923 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.923 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:19.150218+0000", 2023-10-01T09:30:31.923 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015907355000000001 2023-10-01T09:30:31.923 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.923 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.923 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.923 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.923 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.923 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:187338 7.1 e313/307 7:8ac8f53d:::benchmark_data_smithi138_367058_object89543:head v 313'11542, mlcod=270'5583)", 2023-10-01T09:30:31.923 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:44.449109+0000", 2023-10-01T09:30:31.924 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 107.425626041, 2023-10-01T09:30:31.924 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013764587999999999, 2023-10-01T09:30:31.924 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.924 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.924 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.924 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.924 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.924 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:44.449109+0000", 2023-10-01T09:30:31.924 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.924 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.924 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.925 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.925 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:44.449109+0000", 2023-10-01T09:30:31.925 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:31.925 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.925 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.925 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.925 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:44.449108+0000", 2023-10-01T09:30:31.925 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2010000000000002e-06 2023-10-01T09:30:31.925 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.925 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.925 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.925 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:44.449115+0000", 2023-10-01T09:30:31.925 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.594e-06 2023-10-01T09:30:31.926 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.926 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.926 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.926 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:44.449125+0000", 2023-10-01T09:30:31.926 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5099999999999998e-06 2023-10-01T09:30:31.926 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.926 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.926 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.926 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:44.449130+0000", 2023-10-01T09:30:31.926 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6028999999999997e-05 2023-10-01T09:30:31.926 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.927 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.927 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.927 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:44.449166+0000", 2023-10-01T09:30:31.927 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8448e-05 2023-10-01T09:30:31.927 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.927 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.927 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.927 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:44.449185+0000", 2023-10-01T09:30:31.927 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0136534 2023-10-01T09:30:31.927 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.927 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.927 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.928 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:44.462838+0000", 2023-10-01T09:30:31.928 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5370000000000002e-05 2023-10-01T09:30:31.928 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.928 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.928 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.928 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:44.462873+0000", 2023-10-01T09:30:31.928 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013764587999999999 2023-10-01T09:30:31.928 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.928 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.928 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.928 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.928 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.929 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:189270 7.4 e315/306 7:291edf05:::benchmark_data_smithi138_367058_object91475:head v 315'11973, mlcod=315'11970)", 2023-10-01T09:30:31.929 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:48.177534+0000", 2023-10-01T09:30:31.929 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 103.697201011, 2023-10-01T09:30:31.929 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016393863000000002, 2023-10-01T09:30:31.929 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.929 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.929 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.929 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.929 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.929 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.177534+0000", 2023-10-01T09:30:31.929 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.929 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.930 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.930 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.930 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.177534+0000", 2023-10-01T09:30:31.930 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:31.930 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.930 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.930 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.930 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.177533+0000", 2023-10-01T09:30:31.930 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3344e-05 2023-10-01T09:30:31.930 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.931 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.931 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.931 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.177546+0000", 2023-10-01T09:30:31.931 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3343000000000001e-05 2023-10-01T09:30:31.931 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.931 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.931 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.931 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.177560+0000", 2023-10-01T09:30:31.931 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1230000000000002e-06 2023-10-01T09:30:31.931 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.931 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.931 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.932 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.177567+0000", 2023-10-01T09:30:31.932 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003066718 2023-10-01T09:30:31.932 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.932 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.932 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.932 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.180633+0000", 2023-10-01T09:30:31.932 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1790999999999999e-05 2023-10-01T09:30:31.932 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.932 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.932 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.932 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.180655+0000", 2023-10-01T09:30:31.932 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013230842 2023-10-01T09:30:31.933 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.933 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.933 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.933 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.193886+0000", 2023-10-01T09:30:31.933 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1692e-05 2023-10-01T09:30:31.933 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.933 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.933 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.933 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.193928+0000", 2023-10-01T09:30:31.933 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016393863000000002 2023-10-01T09:30:31.933 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.933 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.933 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.934 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.934 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.934 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:189271 7.4 e315/306 7:25f62095:::benchmark_data_smithi138_367058_object91476:head v 315'11974, mlcod=315'11970)", 2023-10-01T09:30:31.934 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:48.180541+0000", 2023-10-01T09:30:31.934 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 103.694194399, 2023-10-01T09:30:31.934 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013409730999999999, 2023-10-01T09:30:31.934 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.934 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.934 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.934 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.934 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.934 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.180541+0000", 2023-10-01T09:30:31.935 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.935 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.935 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.935 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.935 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.180541+0000", 2023-10-01T09:30:31.935 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:30:31.935 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.935 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.935 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.935 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.180539+0000", 2023-10-01T09:30:31.935 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3268000000000001e-05 2023-10-01T09:30:31.935 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.936 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.936 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.936 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.180552+0000", 2023-10-01T09:30:31.936 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5420000000000001e-05 2023-10-01T09:30:31.936 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.936 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.936 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.936 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.180568+0000", 2023-10-01T09:30:31.936 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5479999999999998e-06 2023-10-01T09:30:31.936 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.936 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.936 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.937 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.180574+0000", 2023-10-01T09:30:31.937 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033048579999999999 2023-10-01T09:30:31.937 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.937 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.937 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.937 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.183879+0000", 2023-10-01T09:30:31.937 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5765000000000001e-05 2023-10-01T09:30:31.937 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.937 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.937 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.937 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.183905+0000", 2023-10-01T09:30:31.937 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010027971 2023-10-01T09:30:31.938 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.938 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.938 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.938 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.193933+0000", 2023-10-01T09:30:31.938 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7269999999999999e-05 2023-10-01T09:30:31.938 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.938 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.938 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.938 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.193950+0000", 2023-10-01T09:30:31.938 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013409730999999999 2023-10-01T09:30:31.938 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.939 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.939 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.939 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.939 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.939 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:192041 7.4 e317/306 7:2b0b166b:::benchmark_data_smithi138_367058_object94246:head v 317'12161, mlcod=317'12156)", 2023-10-01T09:30:31.939 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:53.028112+0000", 2023-10-01T09:30:31.939 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 98.846622562999997, 2023-10-01T09:30:31.939 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014727831, 2023-10-01T09:30:31.939 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.939 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.939 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.939 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.940 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.940 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.028112+0000", 2023-10-01T09:30:31.940 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.940 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.940 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.940 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.940 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.028112+0000", 2023-10-01T09:30:31.940 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:30:31.940 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.940 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.941 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.941 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.028111+0000", 2023-10-01T09:30:31.941 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1168e-05 2023-10-01T09:30:31.941 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.941 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.941 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.941 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.028122+0000", 2023-10-01T09:30:31.941 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2472e-05 2023-10-01T09:30:31.941 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.941 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.942 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.942 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.028134+0000", 2023-10-01T09:30:31.942 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3720000000000001e-06 2023-10-01T09:30:31.942 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.942 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.942 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.942 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.028141+0000", 2023-10-01T09:30:31.942 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0046608910000000003 2023-10-01T09:30:31.942 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.942 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.942 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.943 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.032802+0000", 2023-10-01T09:30:31.943 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5068000000000001e-05 2023-10-01T09:30:31.943 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.943 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.943 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.943 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.032827+0000", 2023-10-01T09:30:31.943 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0099827709999999997 2023-10-01T09:30:31.943 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.943 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.943 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.943 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.042810+0000", 2023-10-01T09:30:31.943 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0648999999999997e-05 2023-10-01T09:30:31.944 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.944 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.944 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.944 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.042840+0000", 2023-10-01T09:30:31.944 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014727831 2023-10-01T09:30:31.944 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.944 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.944 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.944 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.944 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.944 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:192136 7.4 e317/306 7:23280b10:::benchmark_data_smithi138_367058_object94341:head v 317'12172, mlcod=317'12169)", 2023-10-01T09:30:31.945 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:53.176657+0000", 2023-10-01T09:30:31.945 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 98.698078201000001, 2023-10-01T09:30:31.945 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017663564999999999, 2023-10-01T09:30:31.945 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.945 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.945 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.945 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.945 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.945 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.176657+0000", 2023-10-01T09:30:31.945 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.945 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.946 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.946 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.946 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.176657+0000", 2023-10-01T09:30:31.946 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:30:31.946 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.946 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.946 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.946 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.176656+0000", 2023-10-01T09:30:31.946 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8289999999999998e-06 2023-10-01T09:30:31.946 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.946 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.946 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.947 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.176661+0000", 2023-10-01T09:30:31.947 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2320000000000001e-06 2023-10-01T09:30:31.947 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.947 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.947 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.947 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.176666+0000", 2023-10-01T09:30:31.947 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.467e-06 2023-10-01T09:30:31.947 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.947 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.947 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.947 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.176670+0000", 2023-10-01T09:30:31.948 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0054083050000000004 2023-10-01T09:30:31.948 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.948 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.948 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.948 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.182078+0000", 2023-10-01T09:30:31.948 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2277e-05 2023-10-01T09:30:31.948 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.948 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.948 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.948 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.182100+0000", 2023-10-01T09:30:31.948 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012181777 2023-10-01T09:30:31.948 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.949 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.949 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.949 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.194282+0000", 2023-10-01T09:30:31.949 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8040000000000002e-05 2023-10-01T09:30:31.949 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.949 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.949 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.949 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.194320+0000", 2023-10-01T09:30:31.949 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017663564999999999 2023-10-01T09:30:31.949 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.949 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.950 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.950 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.950 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.950 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:192975 7.1 e317/307 7:822d7b80:::benchmark_data_smithi138_367058_object95180:head v 317'11920, mlcod=270'5583)", 2023-10-01T09:30:31.950 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:54.541771+0000", 2023-10-01T09:30:31.950 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 97.332963550000002, 2023-10-01T09:30:31.950 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014686121, 2023-10-01T09:30:31.950 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.950 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.950 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.950 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.950 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.951 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.541771+0000", 2023-10-01T09:30:31.951 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.951 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.951 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.951 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.951 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.541771+0000", 2023-10-01T09:30:31.951 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:30:31.951 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.951 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.951 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.951 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.541771+0000", 2023-10-01T09:30:31.952 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7710000000000006e-06 2023-10-01T09:30:31.952 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.952 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.952 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.952 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.541778+0000", 2023-10-01T09:30:31.952 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0326000000000001e-05 2023-10-01T09:30:31.952 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.952 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.952 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.952 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.541789+0000", 2023-10-01T09:30:31.952 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7859999999999996e-06 2023-10-01T09:30:31.952 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.953 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.953 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.953 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.541794+0000", 2023-10-01T09:30:31.953 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0048861490000000002 2023-10-01T09:30:31.953 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.953 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.953 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.953 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.546680+0000", 2023-10-01T09:30:31.953 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.728e-05 2023-10-01T09:30:31.953 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.953 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.953 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.954 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.546707+0000", 2023-10-01T09:30:31.954 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0096946789999999994 2023-10-01T09:30:31.954 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.954 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.954 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.954 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.556402+0000", 2023-10-01T09:30:31.954 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5829999999999999e-05 2023-10-01T09:30:31.954 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.954 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.954 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.954 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.556457+0000", 2023-10-01T09:30:31.955 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014686121 2023-10-01T09:30:31.955 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.955 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.955 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.955 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.955 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.955 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:193239 7.4 e317/306 7:25c6a0ff:::benchmark_data_smithi138_367058_object95444:head v 317'12241, mlcod=317'12237)", 2023-10-01T09:30:31.955 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:54.875209+0000", 2023-10-01T09:30:31.955 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 96.999525786999996, 2023-10-01T09:30:31.955 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012723541999999999, 2023-10-01T09:30:31.956 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.956 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.956 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.956 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.956 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.956 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.875209+0000", 2023-10-01T09:30:31.956 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.956 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.956 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.956 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.956 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.875209+0000", 2023-10-01T09:30:31.957 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:30:31.957 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.957 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.957 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.957 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.875209+0000", 2023-10-01T09:30:31.957 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6930000000000007e-06 2023-10-01T09:30:31.957 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.957 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.957 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.957 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.875217+0000", 2023-10-01T09:30:31.958 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3109999999999995e-06 2023-10-01T09:30:31.958 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.958 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.958 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.958 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.875226+0000", 2023-10-01T09:30:31.958 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1749999999999998e-06 2023-10-01T09:30:31.958 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.958 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.958 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.959 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.875230+0000", 2023-10-01T09:30:31.959 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0064054089999999999 2023-10-01T09:30:31.959 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.959 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.959 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.959 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.881635+0000", 2023-10-01T09:30:31.959 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6074000000000001e-05 2023-10-01T09:30:31.959 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.959 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.959 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.959 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.881661+0000", 2023-10-01T09:30:31.959 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0062379289999999997 2023-10-01T09:30:31.960 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.960 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.960 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.960 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.887899+0000", 2023-10-01T09:30:31.960 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3254999999999999e-05 2023-10-01T09:30:31.960 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.960 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.960 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.960 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:54.887933+0000", 2023-10-01T09:30:31.960 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012723541999999999 2023-10-01T09:30:31.960 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.960 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.961 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.961 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.961 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.961 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:193507 7.4 e317/306 7:2b95b67a:::benchmark_data_smithi138_367058_object95712:head v 317'12265, mlcod=317'12263)", 2023-10-01T09:30:31.961 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:55.252396+0000", 2023-10-01T09:30:31.961 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 96.622338928000005, 2023-10-01T09:30:31.961 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013909008, 2023-10-01T09:30:31.961 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:31.961 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:31.961 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:31.961 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.961 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:31.962 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:55.252396+0000", 2023-10-01T09:30:31.962 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:31.962 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.962 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.962 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:31.962 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:55.252396+0000", 2023-10-01T09:30:31.962 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:31.962 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.962 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.962 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:31.962 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:55.252395+0000", 2023-10-01T09:30:31.962 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2870000000000004e-06 2023-10-01T09:30:31.963 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.963 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.963 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:31.963 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:55.252403+0000", 2023-10-01T09:30:31.963 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3922e-05 2023-10-01T09:30:31.963 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.963 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.963 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:31.963 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:55.252417+0000", 2023-10-01T09:30:31.963 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9480000000000002e-06 2023-10-01T09:30:31.963 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.963 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.964 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:31.964 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:55.252424+0000", 2023-10-01T09:30:31.964 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6316999999999997e-05 2023-10-01T09:30:31.964 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.964 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.964 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:31.964 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:55.252470+0000", 2023-10-01T09:30:31.964 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7220999999999999e-05 2023-10-01T09:30:31.964 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.964 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.964 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:31.964 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:55.252498+0000", 2023-10-01T09:30:31.965 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013754911 2023-10-01T09:30:31.965 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.965 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.965 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:31.965 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:55.266253+0000", 2023-10-01T09:30:31.965 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2437e-05 2023-10-01T09:30:31.965 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:31.965 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:31.965 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:31.965 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:55.266305+0000", 2023-10-01T09:30:31.965 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013909008 2023-10-01T09:30:31.965 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.965 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.966 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.966 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:31.966 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:31.966 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:30:31.966 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:31.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:31.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:32.076 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:32.091 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:32.176 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:32.296 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:32.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:32.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:32.382 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 59 16 78493 78477 83.0855 48.625 0.0109963 0.0120305 2023-10-01T09:30:32.399 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:30:32.499 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:32.604 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:30:32.891 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:33.026 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:30:33.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:33.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:30:33.119 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:30:33.135 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:33.159 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:30:33.222 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:33.271 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:32 smithi138 ceph-mon[185886]: pgmap v938: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-01T09:30:33.271 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:32 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:33.271 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:32 smithi138 ceph-mon[192513]: pgmap v938: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-01T09:30:33.271 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:32 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:33.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:32 smithi190 ceph-mon[160435]: pgmap v938: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-01T09:30:33.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:32 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:33.382 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:30:33.382276+0000 min lat: 0.00115441 max lat: 0.275388 avg lat: 0.0120952 2023-10-01T09:30:33.382 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:30:33.382 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 60 16 79398 79382 82.6435 56.5625 0.0139511 0.0120952 2023-10-01T09:30:34.019 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:34.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:34.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:34.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:33 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:34.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:33 smithi190 ceph-mon[160435]: pgmap v939: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-01T09:30:34.123 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:34.156 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:34.280 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:33 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:34.281 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:33 smithi138 ceph-mon[185886]: pgmap v939: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-01T09:30:34.281 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:33 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:34.281 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:33 smithi138 ceph-mon[192513]: pgmap v939: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-01T09:30:34.382 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 61 16 80162 80146 82.0717 47.75 0.0139759 0.0121801 2023-10-01T09:30:34.443 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:34.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:34.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:34.553 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:34.590 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:34.625 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:34.710 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:34.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:34.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:34.816 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:30:34.864 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:30:34.888 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:34.917 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:30:34.949 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:30:34.953 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:30:34.953 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:30:34.954 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.954 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:80654 8.5 e342/326 8:aa98cbb3:::benchmark_data_smithi138_431617_object80653:head v 342'5102, mlcod=342'5100)", 2023-10-01T09:30:34.954 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:34.943303+0000", 2023-10-01T09:30:34.954 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0077380219999999998, 2023-10-01T09:30:34.954 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0077684700000000004, 2023-10-01T09:30:34.954 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:34.954 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:34.954 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:34.954 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.954 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:34.955 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.943303+0000", 2023-10-01T09:30:34.955 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:34.955 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.955 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.955 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:34.955 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.943303+0000", 2023-10-01T09:30:34.955 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:34.955 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.955 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.956 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:34.956 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.943302+0000", 2023-10-01T09:30:34.956 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6069999999999999e-05 2023-10-01T09:30:34.956 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.956 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.956 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:34.956 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.943328+0000", 2023-10-01T09:30:34.956 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9904000000000001e-05 2023-10-01T09:30:34.956 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.957 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.957 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:34.957 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.943348+0000", 2023-10-01T09:30:34.957 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.798e-06 2023-10-01T09:30:34.957 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.957 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.957 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:34.957 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.943355+0000", 2023-10-01T09:30:34.957 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00276261 2023-10-01T09:30:34.957 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.958 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.958 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:34.958 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.946118+0000", 2023-10-01T09:30:34.958 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8067000000000002e-05 2023-10-01T09:30:34.958 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.958 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.958 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:34.958 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.946136+0000", 2023-10-01T09:30:34.958 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049302030000000002 2023-10-01T09:30:34.959 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.959 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.959 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:34.959 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.951066+0000", 2023-10-01T09:30:34.959 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0077627310000000001 2023-10-01T09:30:34.959 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:34.959 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:34.959 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:34.959 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.959 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.960 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:80660 8.5 e342/326 8:a59db8bf:::benchmark_data_smithi138_431617_object80659:head v 342'5103, mlcod=342'5100)", 2023-10-01T09:30:34.960 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:34.946352+0000", 2023-10-01T09:30:34.960 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0046897220000000003, 2023-10-01T09:30:34.960 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0047997530000000004, 2023-10-01T09:30:34.960 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:34.960 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:30:34.960 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:34.960 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.960 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:34.960 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.946352+0000", 2023-10-01T09:30:34.961 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:34.961 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.961 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.961 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:34.961 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.946352+0000", 2023-10-01T09:30:34.961 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:34.961 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.961 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.961 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:34.961 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.946351+0000", 2023-10-01T09:30:34.962 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7382e-05 2023-10-01T09:30:34.962 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.962 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.962 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:34.962 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.946378+0000", 2023-10-01T09:30:34.962 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1838000000000001e-05 2023-10-01T09:30:34.962 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.962 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.962 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:34.962 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.946390+0000", 2023-10-01T09:30:34.963 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3080000000000004e-06 2023-10-01T09:30:34.963 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.963 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.963 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:34.963 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.946396+0000", 2023-10-01T09:30:34.963 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0022028870000000002 2023-10-01T09:30:34.963 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.963 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.963 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:34.963 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.948599+0000", 2023-10-01T09:30:34.964 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.368e-05 2023-10-01T09:30:34.964 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.964 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.964 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:34.964 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.948623+0000", 2023-10-01T09:30:34.964 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0022712570000000001 2023-10-01T09:30:34.964 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:34.964 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:34.964 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:34.965 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.965 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.965 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:80663 8.8 e342/326 8:11cc5cd2:::benchmark_data_smithi138_431617_object80662:head v 342'5025, mlcod=342'5023)", 2023-10-01T09:30:34.965 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:34.949957+0000", 2023-10-01T09:30:34.965 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.00108479, 2023-10-01T09:30:34.965 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0012566369999999999, 2023-10-01T09:30:34.965 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:34.965 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:30:34.965 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:34.965 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.966 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:34.966 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.949957+0000", 2023-10-01T09:30:34.966 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:34.966 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.966 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.966 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:34.966 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.949957+0000", 2023-10-01T09:30:34.966 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:34.966 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.966 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.967 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:34.967 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.949955+0000", 2023-10-01T09:30:34.967 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3198e-05 2023-10-01T09:30:34.967 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.967 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.967 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:34.967 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.949979+0000", 2023-10-01T09:30:34.967 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.452e-05 2023-10-01T09:30:34.967 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.967 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.968 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:34.968 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.949993+0000", 2023-10-01T09:30:34.968 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1630000000000001e-06 2023-10-01T09:30:34.968 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.968 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.968 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:34.968 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.949999+0000", 2023-10-01T09:30:34.968 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8462999999999997e-05 2023-10-01T09:30:34.968 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.968 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.969 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:34.969 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.950078+0000", 2023-10-01T09:30:34.969 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7907999999999999e-05 2023-10-01T09:30:34.969 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.969 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.969 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:34.969 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.950096+0000", 2023-10-01T09:30:34.969 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013922199999999999 2023-10-01T09:30:34.969 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:34.969 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:34.970 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:34.970 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.970 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.970 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:80667 8.c e342/326)", 2023-10-01T09:30:34.970 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:34.950375+0000", 2023-10-01T09:30:34.970 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.00066649899999999998, 2023-10-01T09:30:34.970 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00089474400000000005, 2023-10-01T09:30:34.970 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:34.970 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:30:34.970 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:34.971 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.971 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:34.971 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.950375+0000", 2023-10-01T09:30:34.971 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:34.971 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.971 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.971 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:34.971 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.950375+0000", 2023-10-01T09:30:34.971 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:34.971 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.971 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.972 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:34.972 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.950374+0000", 2023-10-01T09:30:34.972 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7444000000000001e-05 2023-10-01T09:30:34.972 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.972 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.972 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:34.972 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.950432+0000", 2023-10-01T09:30:34.972 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0462000000000001e-05 2023-10-01T09:30:34.972 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.972 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.973 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:34.973 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.950442+0000", 2023-10-01T09:30:34.973 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7479999999999999e-06 2023-10-01T09:30:34.973 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:34.973 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:34.973 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:34.973 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:34.950448+0000", 2023-10-01T09:30:34.973 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2902000000000001e-05 2023-10-01T09:30:34.973 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:34.973 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:34.974 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:34.974 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:34.974 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:30:34.974 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 4 2023-10-01T09:30:34.974 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:30:34.992 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:30:35.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:35.014 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:30:35.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:30:35.023 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:30:35.063 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:30:35.090 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:30:35.116 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:35.120 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:30:35.140 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:30:35.140 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:30:35.140 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:30:35.140 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:30:35.140 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:30:35.140 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:30:35.140 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:30:35.141 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:30:35.175 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:34 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:30:35.175 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:34 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:30:35.175 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:34 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:35.175 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:34 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2518422249' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:35.175 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:34 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:30:35.175 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:34 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:30:35.176 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:34 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:35.176 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:34 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2518422249' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:35.176 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:30:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:30:34] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-01T09:30:35.200 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:35.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:34 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:30:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:34 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:30:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:34 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:34 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2518422249' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:35.382 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 62 16 81076 81060 81.6694 57.125 0.0136655 0.0122395 2023-10-01T09:30:35.773 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:30:36.023 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:36.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:36.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:36.130 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:30:36.156 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:36.256 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:30:36.296 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:36 smithi138 ceph-mon[185886]: pgmap v940: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-01T09:30:36.296 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:36 smithi138 ceph-mon[192513]: pgmap v940: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-01T09:30:36.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:36 smithi190 ceph-mon[160435]: pgmap v940: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-01T09:30:36.382 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 63 15 81853 81838 81.145 48.625 0.0155546 0.0123182 2023-10-01T09:30:36.893 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:30:36.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:36.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:30:37.003 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:37.048 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:37.385 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 64 16 82716 82700 80.7159 53.875 0.00744679 0.0123845 2023-10-01T09:30:37.429 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:37.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:37.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:37.536 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:37.586 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:37.648 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:37.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:37.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:37.753 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:37.796 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:37.911 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:37.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:37.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:37.992 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:30:37.992 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:30:38.015 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:38.119 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:38.305 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:38.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:38.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:38.385 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 65 16 83555 83539 80.2801 52.4375 0.00523768 0.0124373 2023-10-01T09:30:38.395 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.3 primary-affinity to 0 (802) 2023-10-01T09:30:38.411 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:38.445 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:38.612 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:38.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:38.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:38.716 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:38.742 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:38 smithi190 ceph-mon[160435]: pgmap v941: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-01T09:30:38.743 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3123413316' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-01T09:30:38.743 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:38 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-01T09:30:38.746 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:38.772 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:30:38.775 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:30:38.790 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:38.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:38.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:38.799 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:38 smithi138 ceph-mon[185886]: pgmap v941: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-01T09:30:38.799 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3123413316' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-01T09:30:38.799 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:38 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-01T09:30:38.799 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:38 smithi138 ceph-mon[192513]: pgmap v941: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s 2023-10-01T09:30:38.800 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3123413316' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-01T09:30:38.800 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:38 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-10-01T09:30:38.897 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:38.922 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:38.965 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:38.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:38.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:39.072 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:39.101 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:39.144 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:30:39.145 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:39.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:39.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:39.255 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:39.281 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:39.375 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:39.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:39.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:39.385 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 66 16 83625 83609 79.1305 4.375 0.00404535 0.0124574 2023-10-01T09:30:39.398 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:30:39.478 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:39.572 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:39.729 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:39.731 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[192513]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.731 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-01T09:30:39.731 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[192513]: osdmap e343: 8 total, 8 up, 7 in 2023-10-01T09:30:39.731 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[192513]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.731 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[192513]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.731 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[192513]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.731 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[192513]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.731 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[192513]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.731 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/176539529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:39.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[185886]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-01T09:30:39.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[185886]: osdmap e343: 8 total, 8 up, 7 in 2023-10-01T09:30:39.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[185886]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[185886]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[185886]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[185886]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[185886]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:39 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/176539529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:39.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:39.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:39.736 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:39 smithi190 ceph-mon[160435]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.736 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:39 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-10-01T09:30:39.736 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:39 smithi190 ceph-mon[160435]: osdmap e343: 8 total, 8 up, 7 in 2023-10-01T09:30:39.736 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:39 smithi190 ceph-mon[160435]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.736 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:39 smithi190 ceph-mon[160435]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.736 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:39 smithi190 ceph-mon[160435]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.736 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:39 smithi190 ceph-mon[160435]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.736 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:39 smithi190 ceph-mon[160435]: failed to encode map e343 with expected crc 2023-10-01T09:30:39.736 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:39 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/176539529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:39.834 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:39.873 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:39.940 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:39.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:39.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:40.043 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:40.079 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:40.242 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:40.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:40.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:40.307 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:40.347 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:40.386 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 67 16 84427 84411 78.6977 50.125 0.0205486 0.0127013 2023-10-01T09:30:40.399 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:40.478 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:40.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:40.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:40.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:30:40.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[192513]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[192513]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[192513]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/176539529' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:30:40.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[192513]: osdmap e344: 8 total, 8 up, 7 in 2023-10-01T09:30:40.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[192513]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[192513]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[192513]: pgmap v944: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s 2023-10-01T09:30:40.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:40.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:30:40.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:30:40.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[185886]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[185886]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[185886]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/176539529' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:30:40.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[185886]: osdmap e344: 8 total, 8 up, 7 in 2023-10-01T09:30:40.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[185886]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[185886]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[185886]: pgmap v944: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s 2023-10-01T09:30:40.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:40.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:30:40.582 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:40.663 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:40.781 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:40.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:40.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:40 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:30:40.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:40 smithi190 ceph-mon[160435]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:40 smithi190 ceph-mon[160435]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:40 smithi190 ceph-mon[160435]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:40 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/176539529' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:30:40.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:40 smithi190 ceph-mon[160435]: osdmap e344: 8 total, 8 up, 7 in 2023-10-01T09:30:40.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:40 smithi190 ceph-mon[160435]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:40 smithi190 ceph-mon[160435]: failed to encode map e344 with expected crc 2023-10-01T09:30:40.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:40 smithi190 ceph-mon[160435]: pgmap v944: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s 2023-10-01T09:30:40.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:30:40.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:30:40.817 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:30:40.817 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:30:40.886 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:30:40.969 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:41.096 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:30:41.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:41.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:30:41.208 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:30:41.249 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:41.296 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:30:41.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:41.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:30:41.386 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 68 16 85253 85237 78.2997 51.625 0.00350158 0.0127413 2023-10-01T09:30:41.400 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:41.429 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:41.661 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:30:41.661 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:41.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:41.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:41.770 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:41.846 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:41.986 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:42.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:42.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:42.102 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:30:42.177 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:42.200 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:30:42.213 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:30:42.274 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:30:42.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:42.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:30:42.380 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:42.385 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 69 16 86058 86042 77.8942 50.3125 0.0235546 0.0128324 2023-10-01T09:30:42.420 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:42.492 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:42.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:42.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:42.573 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:30:42.596 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:42.607 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:30:42.643 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:30:42.664 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:42.684 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:30:42.724 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:30:42.760 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:30:42.774 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:42.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:42.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:42.799 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:30:42.834 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:30:42.863 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:30:42.863 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:30:42.863 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:30:42.863 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:30:42.863 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:30:42.863 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:30:42.864 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:30:42.864 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:30:42.883 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:30:42.957 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:43.077 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:30:43.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:43.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:30:43.181 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:43.232 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:42 smithi138 ceph-mon[192513]: pgmap v945: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s 2023-10-01T09:30:43.232 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:42 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3013266358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:43.233 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:42 smithi138 ceph-mon[185886]: pgmap v945: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s 2023-10-01T09:30:43.233 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:42 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3013266358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:43.239 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:43.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:42 smithi190 ceph-mon[160435]: pgmap v945: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s 2023-10-01T09:30:43.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:42 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3013266358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:43.328 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:43.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:43.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:43.386 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 70 16 86741 86725 77.391 42.6875 0.0657053 0.0129066 2023-10-01T09:30:43.433 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:43.502 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:43.577 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:43.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:43.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:43.686 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:43.757 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:43.997 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:44.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:44.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:44.012 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:30:44.012 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:30:44.012 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.012 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:87342 8.9 8:997d18d5:::benchmark_data_smithi138_431617_object87341:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-01T09:30:44.012 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:43.997373+0000", 2023-10-01T09:30:44.012 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.01390545, 2023-10-01T09:30:44.012 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013935474, 2023-10-01T09:30:44.012 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:44.013 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:30:44.013 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:44.013 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:44.013 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:44.013 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 87342 2023-10-01T09:30:44.013 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.013 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:44.013 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.013 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:44.013 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:43.997373+0000", 2023-10-01T09:30:44.013 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:44.013 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.014 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.014 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:44.014 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:43.997373+0000", 2023-10-01T09:30:44.014 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:30:44.014 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.014 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.014 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:44.014 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:43.997372+0000", 2023-10-01T09:30:44.014 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9866e-05 2023-10-01T09:30:44.014 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.014 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.014 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:44.015 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:43.997422+0000", 2023-10-01T09:30:44.015 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1127999999999999e-05 2023-10-01T09:30:44.015 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.015 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.015 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:44.015 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:43.997433+0000", 2023-10-01T09:30:44.015 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4590000000000003e-06 2023-10-01T09:30:44.015 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.015 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.015 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:44.015 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:43.997440+0000", 2023-10-01T09:30:44.015 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2095000000000002e-05 2023-10-01T09:30:44.016 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.016 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.016 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:44.016 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:43.997492+0000", 2023-10-01T09:30:44.016 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000190509 2023-10-01T09:30:44.016 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.016 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.016 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:44.016 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:43.997682+0000", 2023-10-01T09:30:44.016 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00019065 2023-10-01T09:30:44.016 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.016 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.017 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:30:44.017 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:43.997873+0000", 2023-10-01T09:30:44.017 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0031470249999999999 2023-10-01T09:30:44.017 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.017 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.017 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:44.017 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001020+0000", 2023-10-01T09:30:44.017 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016672599999999999 2023-10-01T09:30:44.017 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.017 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.017 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:44.017 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001187+0000", 2023-10-01T09:30:44.017 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038131409999999999 2023-10-01T09:30:44.018 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:44.018 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:44.018 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:44.018 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.018 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.018 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:87347 8.4 8:29dc9f63:::benchmark_data_smithi138_431617_object87346:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-01T09:30:44.018 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:44.001003+0000", 2023-10-01T09:30:44.018 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.010276321, 2023-10-01T09:30:44.018 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010374003, 2023-10-01T09:30:44.018 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:44.018 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:30:44.018 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:44.019 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:44.019 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:44.019 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 87347 2023-10-01T09:30:44.019 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.019 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:44.019 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.019 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:44.019 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001003+0000", 2023-10-01T09:30:44.019 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:44.019 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.019 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.019 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:44.020 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001003+0000", 2023-10-01T09:30:44.020 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:30:44.020 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.020 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.020 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:44.020 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001001+0000", 2023-10-01T09:30:44.020 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9606000000000001e-05 2023-10-01T09:30:44.020 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.020 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.020 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:44.020 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001031+0000", 2023-10-01T09:30:44.020 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3289999999999996e-06 2023-10-01T09:30:44.021 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.021 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.021 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:44.021 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001038+0000", 2023-10-01T09:30:44.021 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2170000000000002e-06 2023-10-01T09:30:44.021 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.021 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.021 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:44.021 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001043+0000", 2023-10-01T09:30:44.021 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000215281 2023-10-01T09:30:44.021 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.021 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.021 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:44.022 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001259+0000", 2023-10-01T09:30:44.022 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014191099999999999 2023-10-01T09:30:44.022 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.022 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.022 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:44.022 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001400+0000", 2023-10-01T09:30:44.022 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015701699999999999 2023-10-01T09:30:44.022 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.022 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.022 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:30:44.022 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001557+0000", 2023-10-01T09:30:44.022 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0023485540000000001 2023-10-01T09:30:44.023 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.023 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.023 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:44.023 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.003906+0000", 2023-10-01T09:30:44.023 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6934000000000002e-05 2023-10-01T09:30:44.023 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.023 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.023 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:44.023 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.003943+0000", 2023-10-01T09:30:44.023 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0029404240000000001 2023-10-01T09:30:44.023 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:44.025 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:44.025 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:44.025 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.025 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.025 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:87350 8.0 8:02e90da6:::benchmark_data_smithi138_431617_object87349:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-01T09:30:44.025 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:44.001047+0000", 2023-10-01T09:30:44.025 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.010231672000000001, 2023-10-01T09:30:44.025 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010401159, 2023-10-01T09:30:44.025 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:44.026 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:30:44.026 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:44.026 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:44.026 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:44.026 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 87350 2023-10-01T09:30:44.026 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.026 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:44.026 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.026 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:44.026 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001047+0000", 2023-10-01T09:30:44.027 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:44.027 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.027 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.027 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:44.027 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001047+0000", 2023-10-01T09:30:44.027 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:30:44.027 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.027 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.027 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:44.027 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001047+0000", 2023-10-01T09:30:44.028 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9369999999999999e-05 2023-10-01T09:30:44.028 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.028 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.028 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:44.028 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001066+0000", 2023-10-01T09:30:44.028 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.788e-06 2023-10-01T09:30:44.028 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.028 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.028 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:44.028 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001070+0000", 2023-10-01T09:30:44.028 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9560000000000002e-06 2023-10-01T09:30:44.028 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.028 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.029 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:44.029 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001073+0000", 2023-10-01T09:30:44.029 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6022000000000001e-05 2023-10-01T09:30:44.029 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.029 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.029 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:44.029 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001119+0000", 2023-10-01T09:30:44.029 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016706100000000001 2023-10-01T09:30:44.029 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.029 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.029 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:44.029 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001286+0000", 2023-10-01T09:30:44.030 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000205097 2023-10-01T09:30:44.030 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.030 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.030 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:30:44.030 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.001491+0000", 2023-10-01T09:30:44.030 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0030772130000000001 2023-10-01T09:30:44.030 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:44.030 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:44.030 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:44.030 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:44.004568+0000", 2023-10-01T09:30:44.030 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035210380000000002 2023-10-01T09:30:44.030 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:44.031 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:44.031 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:44.031 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:44.031 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:30:44.031 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 3 2023-10-01T09:30:44.031 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:30:44.034 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:44.106 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:44.149 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:44.226 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:44.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:44.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:44.331 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:44.361 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:44.386 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 71 16 87609 87593 77.0652 54.25 0.00324475 0.0129514 2023-10-01T09:30:44.403 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:44.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:44.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:44.507 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:44.524 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5, 6] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2023-10-01T09:30:44.524 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-01T09:30:44.594 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:44.783 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:44.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:44.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:44.809 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T09:30:44.890 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:44.938 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:30:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:30:44] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.33.4" 2023-10-01T09:30:44.943 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:45.039 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:45.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:45.051 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:44 smithi190 ceph-mon[160435]: pgmap v946: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:45.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:45.152 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:45.246 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:44 smithi138 ceph-mon[192513]: pgmap v946: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:45.248 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:44 smithi138 ceph-mon[185886]: pgmap v946: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:45.263 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:45.291 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:30:45.386 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 72 16 88398 88382 76.6798 49.3125 0.0251993 0.013037 2023-10-01T09:30:45.425 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:45.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:45.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:45.543 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:45.629 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:45.664 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:45.798 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:45.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:45.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:45.909 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:45.955 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:46.018 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:46.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:46.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:46.032 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:30:46.121 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:46.148 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:46.190 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:46.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:46.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:46.293 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:46.326 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:46.386 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 73 16 89087 89071 76.2195 43.0625 0.0323227 0.0131128 2023-10-01T09:30:46.532 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:46.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:46.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:46.640 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:46.711 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:46.763 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:46.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:46.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:46.866 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:46.895 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:46.944 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:46.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:46.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:46.992 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:46 smithi138 ceph-mon[185886]: pgmap v947: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 834 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:46.992 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:46 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1142583637' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:30:46.992 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:46 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:30:47.005 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:30:47.005 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:30:47.005 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:30:47.005 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:30:47.005 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:30:47.014 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:30:47.029 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:46 smithi190 ceph-mon[160435]: pgmap v947: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 834 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:47.029 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:46 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1142583637' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:30:47.029 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:46 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:30:47.052 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:47.102 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:47.263 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:47.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:47.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:47.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:46 smithi138 ceph-mon[192513]: pgmap v947: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 834 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:47.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:46 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1142583637' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:30:47.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:46 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:30:47.366 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:47.386 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 74 16 89823 89807 75.8112 46 0.00387165 0.0131802 2023-10-01T09:30:47.392 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:47.434 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:47.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:47.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:47.537 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:47.580 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:47.821 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:47.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:47.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:47.920 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T09:30:47.925 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:47.996 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:48.027 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:48.219 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:48.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:48.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:48.229 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:47 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:30:48.229 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:47 smithi138 ceph-mon[185886]: failed to encode map e345 with expected crc 2023-10-01T09:30:48.229 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:47 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:30:48.229 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:47 smithi138 ceph-mon[185886]: osdmap e345: 8 total, 8 up, 7 in 2023-10-01T09:30:48.230 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:47 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:30:48.230 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:47 smithi138 ceph-mon[192513]: failed to encode map e345 with expected crc 2023-10-01T09:30:48.230 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:47 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:30:48.230 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:47 smithi138 ceph-mon[192513]: osdmap e345: 8 total, 8 up, 7 in 2023-10-01T09:30:48.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:47 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:30:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:47 smithi190 ceph-mon[160435]: failed to encode map e345 with expected crc 2023-10-01T09:30:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:47 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:30:48.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:47 smithi190 ceph-mon[160435]: osdmap e345: 8 total, 8 up, 7 in 2023-10-01T09:30:48.330 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:48.385 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:30:48.387 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 75 16 90505 90489 75.3679 42.625 0.0164896 0.0132631 2023-10-01T09:30:48.393 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:48.628 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:48.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:48.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:48.736 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:48.781 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:48.946 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:48.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:48.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:49.050 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:49.121 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:49.230 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:49.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:49.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:49.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:49 smithi190 ceph-mon[160435]: pgmap v949: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 35 MiB/s wr, 562 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:49.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1146008192' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:30:49.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:49 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:30:49.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:49 smithi138 ceph-mon[192513]: pgmap v949: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 35 MiB/s wr, 562 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:49.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1146008192' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:30:49.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:49 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:30:49.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:49 smithi138 ceph-mon[185886]: pgmap v949: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 35 MiB/s wr, 562 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T09:30:49.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1146008192' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:30:49.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:49 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:30:49.339 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:49.364 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:49.388 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 76 16 91310 91294 75.0383 50.3125 0.00501977 0.0133124 2023-10-01T09:30:49.415 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.0 to 1 (10000) 2023-10-01T09:30:49.431 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:49.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:49.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:49.541 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:49.584 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:49.743 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:49.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:49.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:49.852 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:49.888 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:50.047 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:30:50.059 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:50.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:50.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:50.166 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:50.198 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:50.279 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:30:50.280 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:30:50.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:30:50.280 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:30:50.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.280 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:125418 6.d 6.2bc0704d (undecoded) ondisk+retry+write+known_if_redirected e227)", 2023-10-01T09:30:50.280 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:01.365555+0000", 2023-10-01T09:30:50.280 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 528.91007708899997, 2023-10-01T09:30:50.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81812904099999995, 2023-10-01T09:30:50.281 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.281 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:30:50.281 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.281 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:30:50.281 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:30:50.281 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 125418 2023-10-01T09:30:50.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.281 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365555+0000", 2023-10-01T09:30:50.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.282 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365555+0000", 2023-10-01T09:30:50.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:30:50.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.282 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365553+0000", 2023-10-01T09:30:50.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8529999999999992e-06 2023-10-01T09:30:50.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.283 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.283 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365561+0000", 2023-10-01T09:30:50.283 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7089999999999997e-06 2023-10-01T09:30:50.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.283 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.283 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365571+0000", 2023-10-01T09:30:50.283 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.075e-06 2023-10-01T09:30:50.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365578+0000", 2023-10-01T09:30:50.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000104657 2023-10-01T09:30:50.284 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365683+0000", 2023-10-01T09:30:50.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5913000000000002e-05 2023-10-01T09:30:50.284 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.365699+0000", 2023-10-01T09:30:50.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.817972484 2023-10-01T09:30:50.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.285 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.285 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183671+0000", 2023-10-01T09:30:50.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3343000000000001e-05 2023-10-01T09:30:50.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.285 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.285 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183684+0000", 2023-10-01T09:30:50.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81812904099999995 2023-10-01T09:30:50.285 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.286 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.286 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.286 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.286 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:125430 6.d 6.d1d166cd (undecoded) ondisk+write+known_if_redirected e227)", 2023-10-01T09:30:50.286 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:01.397994+0000", 2023-10-01T09:30:50.286 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 528.87763839199999, 2023-10-01T09:30:50.286 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78591710299999995, 2023-10-01T09:30:50.286 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.286 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:30:50.287 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.287 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:30:50.287 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:30:50.287 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 125430 2023-10-01T09:30:50.287 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.287 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.287 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.287 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.397994+0000", 2023-10-01T09:30:50.287 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.288 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.288 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.288 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.397994+0000", 2023-10-01T09:30:50.288 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:30:50.288 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.288 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.288 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.397992+0000", 2023-10-01T09:30:50.288 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0169999999999999e-05 2023-10-01T09:30:50.288 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.289 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.289 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398002+0000", 2023-10-01T09:30:50.289 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6186999999999999e-05 2023-10-01T09:30:50.289 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.289 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.289 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.289 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398019+0000", 2023-10-01T09:30:50.289 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.17e-06 2023-10-01T09:30:50.289 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.289 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.289 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.289 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398026+0000", 2023-10-01T09:30:50.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2902000000000002e-05 2023-10-01T09:30:50.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398089+0000", 2023-10-01T09:30:50.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1385e-05 2023-10-01T09:30:50.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.398100+0000", 2023-10-01T09:30:50.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78580161999999998 2023-10-01T09:30:50.291 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.291 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.291 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183902+0000", 2023-10-01T09:30:50.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3659999999999998e-06 2023-10-01T09:30:50.291 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.291 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.291 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183911+0000", 2023-10-01T09:30:50.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78591710299999995 2023-10-01T09:30:50.291 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.291 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.292 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.292 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.292 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.292 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:125431 6.d 6.45b3c99d (undecoded) ondisk+write+known_if_redirected e227)", 2023-10-01T09:30:50.292 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:01.399470+0000", 2023-10-01T09:30:50.292 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 528.87616267600004, 2023-10-01T09:30:50.292 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78447003999999998, 2023-10-01T09:30:50.292 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.292 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:30:50.292 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.292 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:30:50.292 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:30:50.292 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 125431 2023-10-01T09:30:50.293 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.293 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.293 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.293 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399470+0000", 2023-10-01T09:30:50.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.293 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.293 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.293 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399470+0000", 2023-10-01T09:30:50.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:30:50.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.294 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.294 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399468+0000", 2023-10-01T09:30:50.294 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8000000000000001e-06 2023-10-01T09:30:50.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.294 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.294 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399475+0000", 2023-10-01T09:30:50.295 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6650000000000002e-05 2023-10-01T09:30:50.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.295 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.295 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399492+0000", 2023-10-01T09:30:50.295 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5230000000000001e-06 2023-10-01T09:30:50.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.295 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399498+0000", 2023-10-01T09:30:50.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1703999999999998e-05 2023-10-01T09:30:50.296 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399550+0000", 2023-10-01T09:30:50.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8299999999999998e-06 2023-10-01T09:30:50.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.297 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.297 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:01.399557+0000", 2023-10-01T09:30:50.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78437442400000001 2023-10-01T09:30:50.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.298 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.298 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183931+0000", 2023-10-01T09:30:50.298 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4589999999999998e-06 2023-10-01T09:30:50.298 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.298 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.298 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.298 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:02.183940+0000", 2023-10-01T09:30:50.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78447003999999998 2023-10-01T09:30:50.299 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.299 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.299 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.299 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.300 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166458 6.7 6:e2130551:::benchmark_data_smithi138_307751_object71089:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:30:50.300 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.189221+0000", 2023-10-01T09:30:50.300 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 478.08641166199999, 2023-10-01T09:30:50.300 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.07166782, 2023-10-01T09:30:50.300 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.300 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.301 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.301 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:30:50.301 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:30:50.301 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166458 2023-10-01T09:30:50.301 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.301 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.302 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.302 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189221+0000", 2023-10-01T09:30:50.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.302 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.302 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.302 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.302 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189221+0000", 2023-10-01T09:30:50.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:30:50.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.303 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189219+0000", 2023-10-01T09:30:50.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5189999999999992e-06 2023-10-01T09:30:50.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189228+0000", 2023-10-01T09:30:50.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6232e-05 2023-10-01T09:30:50.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189244+0000", 2023-10-01T09:30:50.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3189999999999997e-06 2023-10-01T09:30:50.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.189251+0000", 2023-10-01T09:30:50.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0047771130000000004 2023-10-01T09:30:50.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.194029+0000", 2023-10-01T09:30:50.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1163e-05 2023-10-01T09:30:50.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.194040+0000", 2023-10-01T09:30:50.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0557915819999999 2023-10-01T09:30:50.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.249831+0000", 2023-10-01T09:30:50.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010584799999999999 2023-10-01T09:30:50.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.249937+0000", 2023-10-01T09:30:50.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6229000000000003e-05 2023-10-01T09:30:50.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:30:50.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.250033+0000", 2023-10-01T09:30:50.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049686640000000002 2023-10-01T09:30:50.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.255002+0000", 2023-10-01T09:30:50.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001218654 2023-10-01T09:30:50.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.256221+0000", 2023-10-01T09:30:50.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045970969999999996 2023-10-01T09:30:50.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260818+0000", 2023-10-01T09:30:50.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9246999999999999e-05 2023-10-01T09:30:50.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.310 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260857+0000", 2023-10-01T09:30:50.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1477999999999999e-05 2023-10-01T09:30:50.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.310 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.310 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260889+0000", 2023-10-01T09:30:50.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.07166782 2023-10-01T09:30:50.310 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.310 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.310 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.311 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166487 6.7 6:e01c315f:::benchmark_data_smithi138_307751_object71118:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:30:50.311 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.213314+0000", 2023-10-01T09:30:50.311 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 478.06231837899998, 2023-10-01T09:30:50.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0525543909999999, 2023-10-01T09:30:50.311 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.311 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.311 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.311 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:30:50.311 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:30:50.311 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166487 2023-10-01T09:30:50.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.311 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213314+0000", 2023-10-01T09:30:50.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213314+0000", 2023-10-01T09:30:50.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:30:50.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213312+0000", 2023-10-01T09:30:50.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5779999999999999e-06 2023-10-01T09:30:50.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213321+0000", 2023-10-01T09:30:50.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5784e-05 2023-10-01T09:30:50.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213337+0000", 2023-10-01T09:30:50.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2225000000000002e-05 2023-10-01T09:30:50.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213359+0000", 2023-10-01T09:30:50.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000137766 2023-10-01T09:30:50.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213497+0000", 2023-10-01T09:30:50.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2595999999999999e-05 2023-10-01T09:30:50.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.213509+0000", 2023-10-01T09:30:50.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0382323870000001 2023-10-01T09:30:50.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.251742+0000", 2023-10-01T09:30:50.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0978999999999998e-05 2023-10-01T09:30:50.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.251823+0000", 2023-10-01T09:30:50.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7830999999999995e-05 2023-10-01T09:30:50.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:30:50.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.251890+0000", 2023-10-01T09:30:50.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0056844479999999999 2023-10-01T09:30:50.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257575+0000", 2023-10-01T09:30:50.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003355055 2023-10-01T09:30:50.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260930+0000", 2023-10-01T09:30:50.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0048781969999999999 2023-10-01T09:30:50.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265808+0000", 2023-10-01T09:30:50.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2873000000000002e-05 2023-10-01T09:30:50.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265841+0000", 2023-10-01T09:30:50.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7410999999999999e-05 2023-10-01T09:30:50.318 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.318 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.318 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.318 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265868+0000", 2023-10-01T09:30:50.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0525543909999999 2023-10-01T09:30:50.318 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.318 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.318 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.318 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.318 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.318 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166491 6.7 6:e6d8849d:::benchmark_data_smithi138_307751_object71122:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:30:50.319 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.219132+0000", 2023-10-01T09:30:50.319 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 478.05649997199998, 2023-10-01T09:30:50.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0468224770000001, 2023-10-01T09:30:50.319 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.319 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.319 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.319 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:30:50.319 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:30:50.319 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166491 2023-10-01T09:30:50.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.319 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.320 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.320 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219132+0000", 2023-10-01T09:30:50.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.320 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.320 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.320 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219132+0000", 2023-10-01T09:30:50.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:30:50.320 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.320 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219131+0000", 2023-10-01T09:30:50.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1768e-05 2023-10-01T09:30:50.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219143+0000", 2023-10-01T09:30:50.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2629000000000001e-05 2023-10-01T09:30:50.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219155+0000", 2023-10-01T09:30:50.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.942e-06 2023-10-01T09:30:50.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219161+0000", 2023-10-01T09:30:50.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5195000000000003e-05 2023-10-01T09:30:50.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219216+0000", 2023-10-01T09:30:50.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0199e-05 2023-10-01T09:30:50.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.219226+0000", 2023-10-01T09:30:50.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.033648492 2023-10-01T09:30:50.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.252875+0000", 2023-10-01T09:30:50.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9799999999999997e-05 2023-10-01T09:30:50.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.252935+0000", 2023-10-01T09:30:50.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5607999999999999e-05 2023-10-01T09:30:50.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:30:50.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.253000+0000", 2023-10-01T09:30:50.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00458422 2023-10-01T09:30:50.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257585+0000", 2023-10-01T09:30:50.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033770720000000001 2023-10-01T09:30:50.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.325 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260962+0000", 2023-10-01T09:30:50.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049325590000000004 2023-10-01T09:30:50.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.326 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.326 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265894+0000", 2023-10-01T09:30:50.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6339e-05 2023-10-01T09:30:50.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.326 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.326 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265921+0000", 2023-10-01T09:30:50.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4357999999999999e-05 2023-10-01T09:30:50.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.327 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.327 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265955+0000", 2023-10-01T09:30:50.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0468224770000001 2023-10-01T09:30:50.327 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.327 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.327 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.327 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166496 6.7 6:e4e4b310:::benchmark_data_smithi138_307751_object71127:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:30:50.328 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.230525+0000", 2023-10-01T09:30:50.328 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 478.04510728999998, 2023-10-01T09:30:50.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0393795619999999, 2023-10-01T09:30:50.328 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.328 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.328 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.328 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:30:50.328 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:30:50.328 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166496 2023-10-01T09:30:50.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.328 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230525+0000", 2023-10-01T09:30:50.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230525+0000", 2023-10-01T09:30:50.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:50.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230524+0000", 2023-10-01T09:30:50.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8950000000000003e-06 2023-10-01T09:30:50.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230530+0000", 2023-10-01T09:30:50.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2658e-05 2023-10-01T09:30:50.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230543+0000", 2023-10-01T09:30:50.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2159999999999997e-06 2023-10-01T09:30:50.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230548+0000", 2023-10-01T09:30:50.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9517e-05 2023-10-01T09:30:50.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230597+0000", 2023-10-01T09:30:50.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1009000000000001e-05 2023-10-01T09:30:50.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230608+0000", 2023-10-01T09:30:50.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.023332213 2023-10-01T09:30:50.332 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.253940+0000", 2023-10-01T09:30:50.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0046000000000001e-05 2023-10-01T09:30:50.332 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.254001+0000", 2023-10-01T09:30:50.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6650999999999996e-05 2023-10-01T09:30:50.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:30:50.333 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.254067+0000", 2023-10-01T09:30:50.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035276270000000002 2023-10-01T09:30:50.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.333 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257595+0000", 2023-10-01T09:30:50.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0084019709999999994 2023-10-01T09:30:50.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.334 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265997+0000", 2023-10-01T09:30:50.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038161779999999999 2023-10-01T09:30:50.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.334 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.269813+0000", 2023-10-01T09:30:50.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0507999999999998e-05 2023-10-01T09:30:50.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.334 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.335 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.269863+0000", 2023-10-01T09:30:50.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1207e-05 2023-10-01T09:30:50.335 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.335 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.335 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.269905+0000", 2023-10-01T09:30:50.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0393795619999999 2023-10-01T09:30:50.335 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.335 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.335 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.335 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.336 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166506 6.7 6:e0267cc2:::benchmark_data_smithi138_307751_object71137:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:30:50.336 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.230586+0000", 2023-10-01T09:30:50.336 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 478.04504632700002, 2023-10-01T09:30:50.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.039509367, 2023-10-01T09:30:50.336 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.336 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.336 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.336 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:30:50.336 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:30:50.336 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166506 2023-10-01T09:30:50.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.336 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.337 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.337 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230586+0000", 2023-10-01T09:30:50.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.337 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.337 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230586+0000", 2023-10-01T09:30:50.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:30:50.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.337 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.337 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230586+0000", 2023-10-01T09:30:50.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7310000000000002e-06 2023-10-01T09:30:50.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230588+0000", 2023-10-01T09:30:50.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6399999999999997e-06 2023-10-01T09:30:50.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230596+0000", 2023-10-01T09:30:50.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9950000000000001e-06 2023-10-01T09:30:50.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230599+0000", 2023-10-01T09:30:50.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4839000000000003e-05 2023-10-01T09:30:50.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230634+0000", 2023-10-01T09:30:50.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6569999999999997e-06 2023-10-01T09:30:50.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.230641+0000", 2023-10-01T09:30:50.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0243796119999999 2023-10-01T09:30:50.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.255021+0000", 2023-10-01T09:30:50.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5718999999999999e-05 2023-10-01T09:30:50.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.255087+0000", 2023-10-01T09:30:50.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1915999999999994e-05 2023-10-01T09:30:50.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:30:50.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.255159+0000", 2023-10-01T09:30:50.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0056119830000000001 2023-10-01T09:30:50.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260771+0000", 2023-10-01T09:30:50.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0052694889999999996 2023-10-01T09:30:50.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.342 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.266040+0000", 2023-10-01T09:30:50.342 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003962826 2023-10-01T09:30:50.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.342 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.342 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270003+0000", 2023-10-01T09:30:50.342 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2723999999999999e-05 2023-10-01T09:30:50.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.342 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.342 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270056+0000", 2023-10-01T09:30:50.342 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9711e-05 2023-10-01T09:30:50.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270095+0000", 2023-10-01T09:30:50.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.039509367 2023-10-01T09:30:50.343 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.343 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.343 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.343 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166513 6.7 6:e69cb40c:::benchmark_data_smithi138_307751_object71144:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:30:50.343 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.247029+0000", 2023-10-01T09:30:50.343 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 478.028603702, 2023-10-01T09:30:50.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0288884, 2023-10-01T09:30:50.344 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.344 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.344 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.344 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:30:50.344 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:30:50.344 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166513 2023-10-01T09:30:50.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.344 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247029+0000", 2023-10-01T09:30:50.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.345 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247029+0000", 2023-10-01T09:30:50.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:30:50.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.345 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247027+0000", 2023-10-01T09:30:50.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0600000000000002e-06 2023-10-01T09:30:50.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247034+0000", 2023-10-01T09:30:50.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2286000000000001e-05 2023-10-01T09:30:50.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247046+0000", 2023-10-01T09:30:50.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7830000000000001e-06 2023-10-01T09:30:50.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247053+0000", 2023-10-01T09:30:50.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7210000000000003e-05 2023-10-01T09:30:50.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247110+0000", 2023-10-01T09:30:50.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1158999999999999e-05 2023-10-01T09:30:50.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.247122+0000", 2023-10-01T09:30:50.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.009139901 2023-10-01T09:30:50.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.256262+0000", 2023-10-01T09:30:50.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6122e-05 2023-10-01T09:30:50.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.256348+0000", 2023-10-01T09:30:50.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6933000000000004e-05 2023-10-01T09:30:50.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:30:50.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.256425+0000", 2023-10-01T09:30:50.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045112299999999998 2023-10-01T09:30:50.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260936+0000", 2023-10-01T09:30:50.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0090185890000000005 2023-10-01T09:30:50.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.269954+0000", 2023-10-01T09:30:50.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0058642859999999998 2023-10-01T09:30:50.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.275819+0000", 2023-10-01T09:30:50.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7589999999999997e-05 2023-10-01T09:30:50.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.275866+0000", 2023-10-01T09:30:50.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0816999999999998e-05 2023-10-01T09:30:50.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.275917+0000", 2023-10-01T09:30:50.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0288884 2023-10-01T09:30:50.351 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.351 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.351 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.351 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166528 6.7 6:e73602e0:::benchmark_data_smithi138_307751_object71159:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:30:50.351 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.261428+0000", 2023-10-01T09:30:50.351 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 478.014203968, 2023-10-01T09:30:50.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.014679273, 2023-10-01T09:30:50.352 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.352 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.352 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.352 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:30:50.352 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:30:50.352 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166528 2023-10-01T09:30:50.352 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.352 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.352 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.352 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261428+0000", 2023-10-01T09:30:50.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.353 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.353 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261428+0000", 2023-10-01T09:30:50.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:30:50.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.353 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.353 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261426+0000", 2023-10-01T09:30:50.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3349999999999996e-06 2023-10-01T09:30:50.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261436+0000", 2023-10-01T09:30:50.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4049000000000001e-05 2023-10-01T09:30:50.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261450+0000", 2023-10-01T09:30:50.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5750000000000003e-06 2023-10-01T09:30:50.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261457+0000", 2023-10-01T09:30:50.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8458999999999997e-05 2023-10-01T09:30:50.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261516+0000", 2023-10-01T09:30:50.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1588e-05 2023-10-01T09:30:50.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.261527+0000", 2023-10-01T09:30:50.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99610873700000002 2023-10-01T09:30:50.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257636+0000", 2023-10-01T09:30:50.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000114474 2023-10-01T09:30:50.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257751+0000", 2023-10-01T09:30:50.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014375799999999999 2023-10-01T09:30:50.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:30:50.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.257894+0000", 2023-10-01T09:30:50.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0030463740000000001 2023-10-01T09:30:50.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260941+0000", 2023-10-01T09:30:50.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.009213377 2023-10-01T09:30:50.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270154+0000", 2023-10-01T09:30:50.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0058627649999999998 2023-10-01T09:30:50.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.276017+0000", 2023-10-01T09:30:50.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0591999999999999e-05 2023-10-01T09:30:50.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.276067+0000", 2023-10-01T09:30:50.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0309999999999999e-05 2023-10-01T09:30:50.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.359 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.276108+0000", 2023-10-01T09:30:50.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.014679273 2023-10-01T09:30:50.359 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.359 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.359 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.359 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.359 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.359 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166545 6.7 6:e60e3404:::benchmark_data_smithi138_307751_object71176:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:30:50.359 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.284403+0000", 2023-10-01T09:30:50.359 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 477.99122943499998, 2023-10-01T09:30:50.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99383394000000003, 2023-10-01T09:30:50.359 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.359 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.360 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.360 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:30:50.360 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:30:50.360 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166545 2023-10-01T09:30:50.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.360 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.360 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.360 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284403+0000", 2023-10-01T09:30:50.360 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284403+0000", 2023-10-01T09:30:50.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:30:50.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284401+0000", 2023-10-01T09:30:50.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0525e-05 2023-10-01T09:30:50.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284411+0000", 2023-10-01T09:30:50.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8612999999999999e-05 2023-10-01T09:30:50.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284430+0000", 2023-10-01T09:30:50.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7730000000000002e-06 2023-10-01T09:30:50.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284436+0000", 2023-10-01T09:30:50.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4496000000000001e-05 2023-10-01T09:30:50.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284491+0000", 2023-10-01T09:30:50.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1708999999999999e-05 2023-10-01T09:30:50.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.284503+0000", 2023-10-01T09:30:50.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97494659800000005 2023-10-01T09:30:50.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.259449+0000", 2023-10-01T09:30:50.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000104481 2023-10-01T09:30:50.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.259554+0000", 2023-10-01T09:30:50.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2804000000000002e-05 2023-10-01T09:30:50.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:30:50.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.259637+0000", 2023-10-01T09:30:50.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002740433 2023-10-01T09:30:50.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.262377+0000", 2023-10-01T09:30:50.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078248339999999993 2023-10-01T09:30:50.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.270202+0000", 2023-10-01T09:30:50.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0079470840000000001 2023-10-01T09:30:50.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278149+0000", 2023-10-01T09:30:50.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8479999999999997e-05 2023-10-01T09:30:50.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278197+0000", 2023-10-01T09:30:50.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9505000000000002e-05 2023-10-01T09:30:50.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278237+0000", 2023-10-01T09:30:50.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99383394000000003 2023-10-01T09:30:50.367 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.367 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.367 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.367 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166600 6.7 6:e23e27f5:::benchmark_data_smithi138_307751_object71231:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:30:50.367 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.346015+0000", 2023-10-01T09:30:50.367 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 477.92961714299997, 2023-10-01T09:30:50.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.93235052399999996, 2023-10-01T09:30:50.367 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.367 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.367 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.368 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:30:50.368 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:30:50.368 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166600 2023-10-01T09:30:50.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.368 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.368 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.368 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346015+0000", 2023-10-01T09:30:50.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.368 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346015+0000", 2023-10-01T09:30:50.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:30:50.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346014+0000", 2023-10-01T09:30:50.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1130000000000001e-06 2023-10-01T09:30:50.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346022+0000", 2023-10-01T09:30:50.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4234e-05 2023-10-01T09:30:50.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346036+0000", 2023-10-01T09:30:50.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9259999999999997e-06 2023-10-01T09:30:50.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346042+0000", 2023-10-01T09:30:50.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9398999999999998e-05 2023-10-01T09:30:50.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346102+0000", 2023-10-01T09:30:50.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.202e-05 2023-10-01T09:30:50.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.346114+0000", 2023-10-01T09:30:50.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.91486791099999998 2023-10-01T09:30:50.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.260982+0000", 2023-10-01T09:30:50.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3937999999999997e-05 2023-10-01T09:30:50.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.261045+0000", 2023-10-01T09:30:50.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9540000000000001e-05 2023-10-01T09:30:50.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:30:50.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.261125+0000", 2023-10-01T09:30:50.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0046514310000000001 2023-10-01T09:30:50.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265776+0000", 2023-10-01T09:30:50.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0077202750000000004 2023-10-01T09:30:50.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.273497+0000", 2023-10-01T09:30:50.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0047860460000000004 2023-10-01T09:30:50.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278283+0000", 2023-10-01T09:30:50.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6359000000000002e-05 2023-10-01T09:30:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278319+0000", 2023-10-01T09:30:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6647000000000002e-05 2023-10-01T09:30:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278366+0000", 2023-10-01T09:30:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.93235052399999996 2023-10-01T09:30:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.375 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.375 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.375 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.375 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35070.0:166608 6.7 6:e3e1df92:::benchmark_data_smithi138_307751_object71239:head [delete] snapc 0=[] ondisk+write+known_if_redirected e241)", 2023-10-01T09:30:50.375 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:22:52.493260+0000", 2023-10-01T09:30:50.375 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 477.78237197999999, 2023-10-01T09:30:50.375 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78524043099999996, 2023-10-01T09:30:50.375 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.375 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.375 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.375 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35070", 2023-10-01T09:30:50.375 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:30:50.376 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 166608 2023-10-01T09:30:50.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.376 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.376 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493260+0000", 2023-10-01T09:30:50.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.376 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493260+0000", 2023-10-01T09:30:50.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:50.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493260+0000", 2023-10-01T09:30:50.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6130000000000003e-06 2023-10-01T09:30:50.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493265+0000", 2023-10-01T09:30:50.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1015e-05 2023-10-01T09:30:50.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493276+0000", 2023-10-01T09:30:50.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5259999999999999e-06 2023-10-01T09:30:50.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493283+0000", 2023-10-01T09:30:50.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9543999999999997e-05 2023-10-01T09:30:50.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493352+0000", 2023-10-01T09:30:50.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.118e-05 2023-10-01T09:30:50.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:52.493364+0000", 2023-10-01T09:30:50.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.76903849099999999 2023-10-01T09:30:50.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.262402+0000", 2023-10-01T09:30:50.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7131000000000002e-05 2023-10-01T09:30:50.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.262499+0000", 2023-10-01T09:30:50.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000120406 2023-10-01T09:30:50.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.380 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:30:50.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.262620+0000", 2023-10-01T09:30:50.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0032544620000000001 2023-10-01T09:30:50.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.380 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.265874+0000", 2023-10-01T09:30:50.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010088079 2023-10-01T09:30:50.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.275962+0000", 2023-10-01T09:30:50.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024570550000000001 2023-10-01T09:30:50.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278419+0000", 2023-10-01T09:30:50.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5637999999999999e-05 2023-10-01T09:30:50.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.382 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278465+0000", 2023-10-01T09:30:50.382 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6021e-05 2023-10-01T09:30:50.382 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.382 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.382 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.382 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:22:53.278501+0000", 2023-10-01T09:30:50.382 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.78524043099999996 2023-10-01T09:30:50.382 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.382 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.382 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.382 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.382 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.383 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:188912 7.0 7:0620eb53:::benchmark_data_smithi138_367058_object91117:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-01T09:30:50.383 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:47.042424+0000", 2023-10-01T09:30:50.383 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 123.233208198, 2023-10-01T09:30:50.383 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.002489907, 2023-10-01T09:30:50.383 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.383 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:30:50.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:30:50.383 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 188912 2023-10-01T09:30:50.383 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.383 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.042424+0000", 2023-10-01T09:30:50.384 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.042424+0000", 2023-10-01T09:30:50.384 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999971 2023-10-01T09:30:50.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.042422+0000", 2023-10-01T09:30:50.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2092999999999999e-05 2023-10-01T09:30:50.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.042434+0000", 2023-10-01T09:30:50.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0454000000000001e-05 2023-10-01T09:30:50.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.042454+0000", 2023-10-01T09:30:50.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2390000000000002e-06 2023-10-01T09:30:50.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.386 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.042462+0000", 2023-10-01T09:30:50.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024505270000000001 2023-10-01T09:30:50.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.386 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.044913+0000", 2023-10-01T09:30:50.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3760999999999999e-05 2023-10-01T09:30:50.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.044927+0000", 2023-10-01T09:30:50.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99333189899999996 2023-10-01T09:30:50.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.038258+0000", 2023-10-01T09:30:50.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000156415 2023-10-01T09:30:50.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.038415+0000", 2023-10-01T09:30:50.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5783000000000001e-05 2023-10-01T09:30:50.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.388 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:30:50.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.038511+0000", 2023-10-01T09:30:50.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0061929730000000001 2023-10-01T09:30:50.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.388 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.044704+0000", 2023-10-01T09:30:50.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8519999999999998e-06 2023-10-01T09:30:50.389 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.389 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.389 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.389 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.044711+0000", 2023-10-01T09:30:50.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000112176 2023-10-01T09:30:50.389 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.389 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.389 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.389 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.044823+0000", 2023-10-01T09:30:50.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0776000000000003e-05 2023-10-01T09:30:50.389 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.389 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.389 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.390 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.044873+0000", 2023-10-01T09:30:50.390 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0661000000000003e-05 2023-10-01T09:30:50.390 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.390 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.390 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.390 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.044914+0000", 2023-10-01T09:30:50.390 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.002489907 2023-10-01T09:30:50.390 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.390 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.390 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.390 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.391 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:188917 7.0 7:0a638b47:::benchmark_data_smithi138_367058_object91122:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-01T09:30:50.391 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:47.059441+0000", 2023-10-01T09:30:50.391 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 123.216191481, 2023-10-01T09:30:50.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99312395099999995, 2023-10-01T09:30:50.391 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.391 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.391 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.391 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:30:50.391 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:30:50.391 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 188917 2023-10-01T09:30:50.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.392 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.059441+0000", 2023-10-01T09:30:50.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.059441+0000", 2023-10-01T09:30:50.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999952 2023-10-01T09:30:50.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.059436+0000", 2023-10-01T09:30:50.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4661000000000001e-05 2023-10-01T09:30:50.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.059451+0000", 2023-10-01T09:30:50.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3621e-05 2023-10-01T09:30:50.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.394 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.394 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.059474+0000", 2023-10-01T09:30:50.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.755e-06 2023-10-01T09:30:50.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.394 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.394 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.059483+0000", 2023-10-01T09:30:50.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8213999999999994e-05 2023-10-01T09:30:50.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.059551+0000", 2023-10-01T09:30:50.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2285e-05 2023-10-01T09:30:50.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.059564+0000", 2023-10-01T09:30:50.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98137222800000001 2023-10-01T09:30:50.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.040936+0000", 2023-10-01T09:30:50.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000103253 2023-10-01T09:30:50.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.041039+0000", 2023-10-01T09:30:50.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4078000000000003e-05 2023-10-01T09:30:50.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:30:50.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.041133+0000", 2023-10-01T09:30:50.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0055998990000000002 2023-10-01T09:30:50.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.046733+0000", 2023-10-01T09:30:50.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00174259 2023-10-01T09:30:50.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.048476+0000", 2023-10-01T09:30:50.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0039918469999999998 2023-10-01T09:30:50.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.052467+0000", 2023-10-01T09:30:50.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5952999999999998e-05 2023-10-01T09:30:50.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.052523+0000", 2023-10-01T09:30:50.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1474999999999998e-05 2023-10-01T09:30:50.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.052565+0000", 2023-10-01T09:30:50.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99312395099999995 2023-10-01T09:30:50.398 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.399 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.399 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.399 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:188939 7.0 7:0a19c7ca:::benchmark_data_smithi138_367058_object91144:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-01T09:30:50.399 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:47.084636+0000", 2023-10-01T09:30:50.399 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 123.190996802, 2023-10-01T09:30:50.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97319148300000002, 2023-10-01T09:30:50.399 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.399 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.399 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.399 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:30:50.400 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:30:50.400 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 188939 2023-10-01T09:30:50.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.400 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.400 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.084636+0000", 2023-10-01T09:30:50.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.400 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.084636+0000", 2023-10-01T09:30:50.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:30:50.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.084633+0000", 2023-10-01T09:30:50.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0603e-05 2023-10-01T09:30:50.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.084644+0000", 2023-10-01T09:30:50.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8547e-05 2023-10-01T09:30:50.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.402 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.084663+0000", 2023-10-01T09:30:50.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3859999999999996e-06 2023-10-01T09:30:50.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.402 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.084672+0000", 2023-10-01T09:30:50.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8133999999999995e-05 2023-10-01T09:30:50.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.084740+0000", 2023-10-01T09:30:50.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1034e-05 2023-10-01T09:30:50.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.084751+0000", 2023-10-01T09:30:50.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.95818252299999995 2023-10-01T09:30:50.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.042934+0000", 2023-10-01T09:30:50.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012296900000000001 2023-10-01T09:30:50.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.043057+0000", 2023-10-01T09:30:50.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014132500000000001 2023-10-01T09:30:50.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:30:50.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.043198+0000", 2023-10-01T09:30:50.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0052841190000000003 2023-10-01T09:30:50.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.048482+0000", 2023-10-01T09:30:50.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8133999999999998e-05 2023-10-01T09:30:50.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.048520+0000", 2023-10-01T09:30:50.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0092112240000000005 2023-10-01T09:30:50.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.057731+0000", 2023-10-01T09:30:50.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7274000000000001e-05 2023-10-01T09:30:50.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.057789+0000", 2023-10-01T09:30:50.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8353999999999999e-05 2023-10-01T09:30:50.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.407 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.057827+0000", 2023-10-01T09:30:50.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97319148300000002 2023-10-01T09:30:50.407 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.407 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.407 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.407 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.407 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:188949 7.0 7:0c2a01b7:::benchmark_data_smithi138_367058_object91154:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-01T09:30:50.407 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:47.103372+0000", 2023-10-01T09:30:50.407 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 123.172260079, 2023-10-01T09:30:50.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.95460666999999999, 2023-10-01T09:30:50.408 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.408 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.408 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.408 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:30:50.408 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:30:50.408 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 188949 2023-10-01T09:30:50.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.408 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.103372+0000", 2023-10-01T09:30:50.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.103372+0000", 2023-10-01T09:30:50.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:30:50.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.103370+0000", 2023-10-01T09:30:50.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.8970000000000003e-06 2023-10-01T09:30:50.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.410 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.103380+0000", 2023-10-01T09:30:50.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5753e-05 2023-10-01T09:30:50.410 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.410 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.410 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.410 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.103396+0000", 2023-10-01T09:30:50.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1702000000000001e-05 2023-10-01T09:30:50.410 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.410 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.410 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.410 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.103407+0000", 2023-10-01T09:30:50.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5026999999999998e-05 2023-10-01T09:30:50.410 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.411 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.411 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.411 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.103482+0000", 2023-10-01T09:30:50.411 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.009e-05 2023-10-01T09:30:50.411 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.411 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.411 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.411 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.103492+0000", 2023-10-01T09:30:50.411 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.94144534099999999 2023-10-01T09:30:50.411 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.411 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.411 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.412 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.044938+0000", 2023-10-01T09:30:50.412 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3458e-05 2023-10-01T09:30:50.412 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.412 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.412 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.412 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.045011+0000", 2023-10-01T09:30:50.412 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8026000000000001e-05 2023-10-01T09:30:50.412 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.412 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.412 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:30:50.412 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.045089+0000", 2023-10-01T09:30:50.412 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0052646230000000004 2023-10-01T09:30:50.412 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.413 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.413 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.050354+0000", 2023-10-01T09:30:50.413 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0022991819999999999 2023-10-01T09:30:50.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.413 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.413 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.052653+0000", 2023-10-01T09:30:50.413 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0052354469999999998 2023-10-01T09:30:50.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.413 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.414 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.057888+0000", 2023-10-01T09:30:50.414 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0957e-05 2023-10-01T09:30:50.414 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.414 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.414 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.414 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.057939+0000", 2023-10-01T09:30:50.414 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9555999999999999e-05 2023-10-01T09:30:50.414 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.414 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.414 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.414 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.057979+0000", 2023-10-01T09:30:50.414 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.95460666999999999 2023-10-01T09:30:50.415 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.415 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.415 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.415 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.415 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.415 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:189018 7.0 7:04033c4d:::benchmark_data_smithi138_367058_object91223:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-01T09:30:50.415 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:47.250575+0000", 2023-10-01T09:30:50.415 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 123.025057633, 2023-10-01T09:30:50.415 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81934733599999998, 2023-10-01T09:30:50.415 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.416 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.416 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.416 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:30:50.416 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:30:50.416 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 189018 2023-10-01T09:30:50.416 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.416 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.416 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.416 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.416 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.250575+0000", 2023-10-01T09:30:50.416 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.416 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.417 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.417 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.250575+0000", 2023-10-01T09:30:50.417 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:30:50.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.417 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.417 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.250572+0000", 2023-10-01T09:30:50.417 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0752e-05 2023-10-01T09:30:50.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.417 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.418 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.250583+0000", 2023-10-01T09:30:50.418 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3360999999999999e-05 2023-10-01T09:30:50.418 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.418 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.418 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.418 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.250606+0000", 2023-10-01T09:30:50.418 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.1880000000000008e-06 2023-10-01T09:30:50.418 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.418 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.418 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.418 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.250616+0000", 2023-10-01T09:30:50.418 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2875999999999999e-05 2023-10-01T09:30:50.419 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.419 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.419 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.419 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.250679+0000", 2023-10-01T09:30:50.419 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0581000000000001e-05 2023-10-01T09:30:50.419 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.419 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.419 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.419 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.250689+0000", 2023-10-01T09:30:50.419 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.79608477 2023-10-01T09:30:50.419 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.419 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.420 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.420 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.046774+0000", 2023-10-01T09:30:50.420 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8827999999999998e-05 2023-10-01T09:30:50.420 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.420 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.420 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.420 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.046853+0000", 2023-10-01T09:30:50.420 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.9196000000000005e-05 2023-10-01T09:30:50.420 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.420 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.420 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:30:50.420 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.046942+0000", 2023-10-01T09:30:50.420 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0056402209999999999 2023-10-01T09:30:50.421 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.421 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.421 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.421 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.052582+0000", 2023-10-01T09:30:50.421 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0054300570000000003 2023-10-01T09:30:50.421 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.421 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.421 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.421 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.058012+0000", 2023-10-01T09:30:50.421 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.01182457 2023-10-01T09:30:50.421 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.421 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.422 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.422 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.069837+0000", 2023-10-01T09:30:50.422 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4233000000000002e-05 2023-10-01T09:30:50.422 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.422 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.422 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.422 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.069881+0000", 2023-10-01T09:30:50.422 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1094000000000003e-05 2023-10-01T09:30:50.422 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.422 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.423 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.423 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.069922+0000", 2023-10-01T09:30:50.423 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81934733599999998 2023-10-01T09:30:50.423 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.423 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.423 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.423 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.423 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.423 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:189020 7.0 7:06ada931:::benchmark_data_smithi138_367058_object91225:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-01T09:30:50.423 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:47.254364+0000", 2023-10-01T09:30:50.423 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 123.02126816800001, 2023-10-01T09:30:50.423 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81572159099999997, 2023-10-01T09:30:50.424 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.424 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.424 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.424 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:30:50.424 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:30:50.424 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 189020 2023-10-01T09:30:50.424 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.424 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.424 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.424 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.424 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.254364+0000", 2023-10-01T09:30:50.424 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.425 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.425 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.425 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.425 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.254364+0000", 2023-10-01T09:30:50.425 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:30:50.425 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.425 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.425 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.425 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.254362+0000", 2023-10-01T09:30:50.425 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2346e-05 2023-10-01T09:30:50.425 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.425 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.425 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.426 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.254374+0000", 2023-10-01T09:30:50.426 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3139999999999999e-05 2023-10-01T09:30:50.426 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.426 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.426 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.426 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.254397+0000", 2023-10-01T09:30:50.426 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.9539999999999993e-06 2023-10-01T09:30:50.426 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.426 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.426 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.426 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.254406+0000", 2023-10-01T09:30:50.427 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3011999999999998e-05 2023-10-01T09:30:50.427 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.427 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.427 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.427 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.254479+0000", 2023-10-01T09:30:50.427 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6700000000000006e-06 2023-10-01T09:30:50.427 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.427 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.427 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.427 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.254489+0000", 2023-10-01T09:30:50.427 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.79404673599999998 2023-10-01T09:30:50.427 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.427 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.428 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.428 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.048536+0000", 2023-10-01T09:30:50.428 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8984000000000001e-05 2023-10-01T09:30:50.428 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.428 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.428 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.428 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.048614+0000", 2023-10-01T09:30:50.428 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2358e-05 2023-10-01T09:30:50.428 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.428 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.428 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:30:50.428 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.048697+0000", 2023-10-01T09:30:50.429 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0061936439999999999 2023-10-01T09:30:50.429 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.429 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.429 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.429 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.054890+0000", 2023-10-01T09:30:50.429 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0075884400000000001 2023-10-01T09:30:50.429 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.429 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.429 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.429 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.062479+0000", 2023-10-01T09:30:50.429 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0075343709999999998 2023-10-01T09:30:50.429 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.430 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.430 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.430 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.070013+0000", 2023-10-01T09:30:50.430 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7327999999999999e-05 2023-10-01T09:30:50.430 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.430 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.430 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.430 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.070051+0000", 2023-10-01T09:30:50.430 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5197999999999997e-05 2023-10-01T09:30:50.430 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.430 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.430 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.431 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.070086+0000", 2023-10-01T09:30:50.431 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81572159099999997 2023-10-01T09:30:50.431 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.431 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.431 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.431 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.431 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.431 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.35421.0:189033 7.0 7:00870e7d:::benchmark_data_smithi138_367058_object91238:head [delete] snapc 0=[] ondisk+write+known_if_redirected e314)", 2023-10-01T09:30:50.431 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:47.275421+0000", 2023-10-01T09:30:50.431 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 123.000211471, 2023-10-01T09:30:50.431 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.794778502, 2023-10-01T09:30:50.432 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:50.432 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:50.432 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:50.432 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.35421", 2023-10-01T09:30:50.432 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:30:50.432 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 189033 2023-10-01T09:30:50.432 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.432 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:50.432 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.432 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:50.432 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.275421+0000", 2023-10-01T09:30:50.433 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:50.433 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.433 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.433 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:50.433 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.275421+0000", 2023-10-01T09:30:50.433 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:30:50.433 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.433 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.433 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:50.433 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.275419+0000", 2023-10-01T09:30:50.433 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1775e-05 2023-10-01T09:30:50.433 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.434 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.434 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:50.434 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.275430+0000", 2023-10-01T09:30:50.434 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3121000000000001e-05 2023-10-01T09:30:50.434 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.434 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.434 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:50.434 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.275454+0000", 2023-10-01T09:30:50.434 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4439999999999997e-06 2023-10-01T09:30:50.434 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.434 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.434 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:50.435 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.275463+0000", 2023-10-01T09:30:50.435 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0658e-05 2023-10-01T09:30:50.435 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.435 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.435 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.435 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.275534+0000", 2023-10-01T09:30:50.435 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1640999999999999e-05 2023-10-01T09:30:50.435 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.435 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.435 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:30:50.435 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:47.275545+0000", 2023-10-01T09:30:50.435 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.77485320199999996 2023-10-01T09:30:50.435 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.436 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.436 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:50.436 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.050398+0000", 2023-10-01T09:30:50.436 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000121303 2023-10-01T09:30:50.436 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.436 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.436 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:50.436 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.050520+0000", 2023-10-01T09:30:50.436 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000107304 2023-10-01T09:30:50.436 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.436 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.436 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:30:50.437 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.050627+0000", 2023-10-01T09:30:50.437 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0042694220000000001 2023-10-01T09:30:50.437 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.437 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.437 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:50.437 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.054896+0000", 2023-10-01T09:30:50.437 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0076335439999999999 2023-10-01T09:30:50.437 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.437 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.437 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.437 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.062530+0000", 2023-10-01T09:30:50.437 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0075906130000000004 2023-10-01T09:30:50.438 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.438 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.438 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:50.438 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.070121+0000", 2023-10-01T09:30:50.438 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6220000000000001e-05 2023-10-01T09:30:50.438 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.438 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.438 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:50.438 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.070167+0000", 2023-10-01T09:30:50.438 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2568999999999999e-05 2023-10-01T09:30:50.438 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:50.438 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:50.439 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:50.439 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:48.070199+0000", 2023-10-01T09:30:50.439 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.794778502 2023-10-01T09:30:50.439 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.439 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.439 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.439 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:50.439 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:50.439 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:30:50.440 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 77 16 92081 92065 74.6897 48.1875 0.020243 0.0133833 2023-10-01T09:30:50.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:50.441 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:50.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:50.541 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:50.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:50 smithi190 ceph-mon[160435]: failed to encode map e346 with expected crc 2023-10-01T09:30:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:50 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:30:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:50 smithi190 ceph-mon[160435]: osdmap e346: 8 total, 8 up, 7 in 2023-10-01T09:30:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:50 smithi190 ceph-mon[160435]: failed to encode map e346 with expected crc 2023-10-01T09:30:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:50 smithi190 ceph-mon[160435]: pgmap v951: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s; 0 B/s, 1 objects/s recovering 2023-10-01T09:30:50.576 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:50.616 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-01T09:30:50.619 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:50.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:50.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:50.699 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:50 smithi138 ceph-mon[192513]: failed to encode map e346 with expected crc 2023-10-01T09:30:50.699 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:50 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:30:50.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:50 smithi138 ceph-mon[192513]: osdmap e346: 8 total, 8 up, 7 in 2023-10-01T09:30:50.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:50 smithi138 ceph-mon[192513]: failed to encode map e346 with expected crc 2023-10-01T09:30:50.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:50 smithi138 ceph-mon[192513]: pgmap v951: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s; 0 B/s, 1 objects/s recovering 2023-10-01T09:30:50.704 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:50 smithi138 ceph-mon[185886]: failed to encode map e346 with expected crc 2023-10-01T09:30:50.704 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:50 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:30:50.704 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:50 smithi138 ceph-mon[185886]: osdmap e346: 8 total, 8 up, 7 in 2023-10-01T09:30:50.704 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:50 smithi138 ceph-mon[185886]: failed to encode map e346 with expected crc 2023-10-01T09:30:50.704 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:50 smithi138 ceph-mon[185886]: pgmap v951: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s; 0 B/s, 1 objects/s recovering 2023-10-01T09:30:50.722 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:50.851 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:50.974 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:51.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:51.008 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:30:51.009 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:51.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:51.038 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:30:51.048 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:30:51.094 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:30:51.110 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:30:51.142 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:30:51.149 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:51.172 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:30:51.207 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:30:51.234 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:30:51.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:51.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:30:51.252 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:30:51.299 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:30:51.332 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:30:51.332 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:30:51.332 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:30:51.332 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:30:51.333 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:30:51.333 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:30:51.333 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:30:51.333 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:30:51.339 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:51.374 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:51.388 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 78 16 92772 92756 74.286 43.1875 0.0294793 0.0134558 2023-10-01T09:30:51.489 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:51.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:51.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:51.553 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:30:51.596 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:51.636 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:51 smithi190 ceph-mon[160435]: failed to encode map e346 with expected crc 2023-10-01T09:30:51.636 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:51 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3739754832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:51.640 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:51.720 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:51.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:51.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:51.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:51 smithi138 ceph-mon[192513]: failed to encode map e346 with expected crc 2023-10-01T09:30:51.811 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:51 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3739754832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:51.811 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:51 smithi138 ceph-mon[185886]: failed to encode map e346 with expected crc 2023-10-01T09:30:51.812 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:51 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3739754832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:51.825 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:51.858 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:51.904 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:51.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:51.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:52.008 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:30:52.086 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:52.136 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:52.360 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:30:52.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:52.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:30:52.390 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 79 16 93556 93540 73.9639 49 0.0178824 0.0135138 2023-10-01T09:30:52.467 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:52.494 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:52.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:52 smithi190 ceph-mon[160435]: pgmap v952: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 38 MiB/s wr, 613 op/s 2023-10-01T09:30:52.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:52 smithi138 ceph-mon[185886]: pgmap v952: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 38 MiB/s wr, 613 op/s 2023-10-01T09:30:52.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:52 smithi138 ceph-mon[192513]: pgmap v952: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 38 MiB/s wr, 613 op/s 2023-10-01T09:30:53.313 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:53.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:53.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:53.391 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:30:53.390822+0000 min lat: 0.00115441 max lat: 1.11695 avg lat: 0.013568 2023-10-01T09:30:53.392 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:30:53.392 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 80 16 94372 94356 73.677 51 0.0164616 0.013568 2023-10-01T09:30:53.425 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:53.485 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:53.495 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:30:53.495 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:30:53.567 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:53.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:53.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:53.684 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:53.728 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:53.791 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:53.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:53.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:53.895 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:30:53.921 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:53.971 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:30:53.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:53.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:30:54.078 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:54.104 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:54.303 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:30:54.312 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:54.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:54.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:54.393 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 81 16 95008 94992 73.2554 39.75 0.00877337 0.0136462 2023-10-01T09:30:54.419 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:54.499 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:54.547 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:30:54.684 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:54.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:54.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:54.762 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:30:54.791 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:54.827 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:54.902 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:54.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:54.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:55.008 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:55.052 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:55.143 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:54 smithi190 ceph-mon[160435]: pgmap v953: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s 2023-10-01T09:30:55.166 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:55.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:55.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:55.270 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:55.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:54 smithi138 ceph-mon[192513]: pgmap v953: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s 2023-10-01T09:30:55.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:54 smithi138 ceph-mon[185886]: pgmap v953: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s 2023-10-01T09:30:55.323 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:30:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:30:54] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.33.4" 2023-10-01T09:30:55.357 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:55.393 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 82 16 95726 95710 72.9094 44.875 0.0166444 0.0137097 2023-10-01T09:30:55.437 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:55.462 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:55.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:55.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:55.573 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:30:55.618 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:55.904 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:30:55.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:55.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:30:56.012 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:30:56.015 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:30:56.042 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.1 to 1 (10000) 2023-10-01T09:30:56.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:56 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1167789533' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:56.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:56 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:56.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:30:56.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:56 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1765360115' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:30:56.073 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:56 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:30:56.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:56 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1167789533' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:56.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:56 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:56.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:30:56.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:56 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1765360115' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:30:56.073 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:56 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:30:56.116 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:56.173 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:56 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1167789533' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:56.173 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:56 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:30:56.173 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:30:56.173 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:56 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1765360115' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:30:56.173 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:56 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:30:56.245 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:30:56.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:56.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:30:56.353 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:56.393 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 83 16 96521 96505 72.6297 49.6875 0.0164056 0.0137638 2023-10-01T09:30:56.430 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:30:56.430 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:30:56.435 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:57.266 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:30:57.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:57 smithi190 ceph-mon[160435]: pgmap v954: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-01T09:30:57.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:57 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:30:57.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:57 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:30:57.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:57 smithi190 ceph-mon[160435]: failed to encode map e347 with expected crc 2023-10-01T09:30:57.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:57 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:30:57.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:57 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:30:57.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:57 smithi190 ceph-mon[160435]: osdmap e347: 8 total, 8 up, 7 in 2023-10-01T09:30:57.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:57 smithi190 ceph-mon[160435]: failed to encode map e347 with expected crc 2023-10-01T09:30:57.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[185886]: pgmap v954: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-01T09:30:57.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:30:57.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:30:57.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[185886]: failed to encode map e347 with expected crc 2023-10-01T09:30:57.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:30:57.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:30:57.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[185886]: osdmap e347: 8 total, 8 up, 7 in 2023-10-01T09:30:57.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[185886]: failed to encode map e347 with expected crc 2023-10-01T09:30:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[192513]: pgmap v954: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-01T09:30:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:30:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:30:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[192513]: failed to encode map e347 with expected crc 2023-10-01T09:30:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:30:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:30:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[192513]: osdmap e347: 8 total, 8 up, 7 in 2023-10-01T09:30:57.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:57 smithi138 ceph-mon[192513]: failed to encode map e347 with expected crc 2023-10-01T09:30:57.396 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 84 16 97254 97238 72.3085 45.8125 0.023122 0.0138252 2023-10-01T09:30:57.782 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:30:58.123 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:58.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:58.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:58.234 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:30:58.305 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:58.396 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 85 16 98040 98024 72.0358 49.125 0.0321746 0.0138769 2023-10-01T09:30:58.422 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-01T09:30:58.494 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:30:58.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:58.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:30:58.554 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:58 smithi138 ceph-mon[185886]: pgmap v956: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 38 MiB/s wr, 611 op/s 2023-10-01T09:30:58.555 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:58 smithi138 ceph-mon[192513]: pgmap v956: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 38 MiB/s wr, 611 op/s 2023-10-01T09:30:58.605 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:58.690 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:58.804 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:58.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:58 smithi190 ceph-mon[160435]: pgmap v956: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 38 MiB/s wr, 611 op/s 2023-10-01T09:30:58.816 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:30:58.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:58.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:58.862 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:30:58.907 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:30:58.921 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:58.973 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:30:59.028 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:30:59.030 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:59.058 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:30:59.090 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:30:59.131 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:30:59.137 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:30:59.146 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:30:59.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:59.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:30:59.165 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:30:59.166 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:30:59.166 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:30:59.166 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:30:59.166 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:30:59.166 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:30:59.166 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:30:59.166 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:30:59.258 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:30:59.312 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:59.401 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 86 16 98721 98705 71.6896 42.5625 0.0144009 0.0139437 2023-10-01T09:30:59.405 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:30:59.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:59.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:30:59.501 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:30:59.502 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:30:59.502 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:30:59.502 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:30:59.502 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.502 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168393 7.b e309/306 7:de1e549c:::benchmark_data_smithi138_367058_object70598:head v 309'10279, mlcod=268'5453)", 2023-10-01T09:30:59.502 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:12.658391+0000", 2023-10-01T09:30:59.502 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 166.83952722800001, 2023-10-01T09:30:59.502 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.101860468, 2023-10-01T09:30:59.503 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.503 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.503 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.503 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.503 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.503 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658391+0000", 2023-10-01T09:30:59.503 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.503 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.504 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.504 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.504 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658391+0000", 2023-10-01T09:30:59.504 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:59.504 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.504 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.504 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.504 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658390+0000", 2023-10-01T09:30:59.504 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2609999999999999e-06 2023-10-01T09:30:59.505 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.505 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.505 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.505 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658399+0000", 2023-10-01T09:30:59.505 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0547000000000001e-05 2023-10-01T09:30:59.505 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.505 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.505 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.505 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658410+0000", 2023-10-01T09:30:59.506 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0090000000000004e-06 2023-10-01T09:30:59.506 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.506 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.506 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.506 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658416+0000", 2023-10-01T09:30:59.506 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0454999999999999e-05 2023-10-01T09:30:59.506 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.507 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.507 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.507 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658446+0000", 2023-10-01T09:30:59.507 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4567e-05 2023-10-01T09:30:59.507 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.507 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.507 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.507 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.658461+0000", 2023-10-01T09:30:59.507 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.101753844 2023-10-01T09:30:59.508 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.508 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.508 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.508 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760215+0000", 2023-10-01T09:30:59.508 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6927000000000001e-05 2023-10-01T09:30:59.508 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.508 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.508 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.508 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760251+0000", 2023-10-01T09:30:59.508 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.101860468 2023-10-01T09:30:59.509 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.509 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.509 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.509 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.509 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.509 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:168396 7.7 e309/306 7:eb13eb3e:::benchmark_data_smithi138_367058_object70601:head v 309'10561, mlcod=271'5955)", 2023-10-01T09:30:59.509 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:12.665871+0000", 2023-10-01T09:30:59.509 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 166.83204688399999, 2023-10-01T09:30:59.510 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.128762455, 2023-10-01T09:30:59.510 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.510 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.510 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.510 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.510 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.510 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.665871+0000", 2023-10-01T09:30:59.510 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.510 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.511 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.511 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.511 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.665871+0000", 2023-10-01T09:30:59.511 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:59.511 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.511 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.511 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.511 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.665871+0000", 2023-10-01T09:30:59.511 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7300000000000005e-06 2023-10-01T09:30:59.511 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.512 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.512 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.512 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.665878+0000", 2023-10-01T09:30:59.512 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.094338721 2023-10-01T09:30:59.512 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.512 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.512 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.512 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760217+0000", 2023-10-01T09:30:59.512 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5829999999999996e-06 2023-10-01T09:30:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.760227+0000", 2023-10-01T09:30:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033480960000000001 2023-10-01T09:30:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.513 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.763575+0000", 2023-10-01T09:30:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0735000000000001e-05 2023-10-01T09:30:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.763595+0000", 2023-10-01T09:30:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.031017712999999999 2023-10-01T09:30:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.514 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.794613+0000", 2023-10-01T09:30:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0622999999999999e-05 2023-10-01T09:30:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:12.794634+0000", 2023-10-01T09:30:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.128762455 2023-10-01T09:30:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.515 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:21860 8.5 8:a2e4c89f:::benchmark_data_smithi138_431617_object21859:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e332)", 2023-10-01T09:30:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:29:42.223816+0000", 2023-10-01T09:30:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 77.274101857000005, 2023-10-01T09:30:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.110348153, 2023-10-01T09:30:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.516 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 21860 2023-10-01T09:30:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.223816+0000", 2023-10-01T09:30:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.517 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.223816+0000", 2023-10-01T09:30:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:30:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.223814+0000", 2023-10-01T09:30:59.518 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.8806000000000002e-05 2023-10-01T09:30:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.223913+0000", 2023-10-01T09:30:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6614999999999999e-05 2023-10-01T09:30:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.519 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.223930+0000", 2023-10-01T09:30:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1960000000000002e-06 2023-10-01T09:30:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.223937+0000", 2023-10-01T09:30:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0327000000000001e-05 2023-10-01T09:30:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.520 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.223987+0000", 2023-10-01T09:30:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014336399999999999 2023-10-01T09:30:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.224131+0000", 2023-10-01T09:30:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016779499999999999 2023-10-01T09:30:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.521 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.224299+0000", 2023-10-01T09:30:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0010614229999999999 2023-10-01T09:30:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.225360+0000", 2023-10-01T09:30:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001253047 2023-10-01T09:30:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.522 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.226613+0000", 2023-10-01T09:30:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.107437647 2023-10-01T09:30:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.334051+0000", 2023-10-01T09:30:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6308000000000003e-05 2023-10-01T09:30:59.523 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.334117+0000", 2023-10-01T09:30:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7545e-05 2023-10-01T09:30:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.334164+0000", 2023-10-01T09:30:59.524 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.110348153 2023-10-01T09:30:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.525 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:21874 8.5 8:a43903aa:::benchmark_data_smithi138_431617_object21873:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e332)", 2023-10-01T09:30:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:29:42.230696+0000", 2023-10-01T09:30:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 77.267222016000005, 2023-10-01T09:30:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.10392075000000001, 2023-10-01T09:30:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.526 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 21874 2023-10-01T09:30:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.230696+0000", 2023-10-01T09:30:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.527 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.230696+0000", 2023-10-01T09:30:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999928 2023-10-01T09:30:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.230689+0000", 2023-10-01T09:30:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2361999999999998e-05 2023-10-01T09:30:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.528 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.230741+0000", 2023-10-01T09:30:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2802000000000002e-05 2023-10-01T09:30:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.230774+0000", 2023-10-01T09:30:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3332999999999999e-05 2023-10-01T09:30:59.529 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.230787+0000", 2023-10-01T09:30:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6680000000000005e-05 2023-10-01T09:30:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.230854+0000", 2023-10-01T09:30:59.530 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000148784 2023-10-01T09:30:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.231003+0000", 2023-10-01T09:30:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017789300000000001 2023-10-01T09:30:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.231181+0000", 2023-10-01T09:30:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00099224899999999995 2023-10-01T09:30:59.531 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.232173+0000", 2023-10-01T09:30:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015145499999999999 2023-10-01T09:30:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.232324+0000", 2023-10-01T09:30:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.102212384 2023-10-01T09:30:59.532 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.334537+0000", 2023-10-01T09:30:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5473000000000003e-05 2023-10-01T09:30:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.334582+0000", 2023-10-01T09:30:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4595000000000003e-05 2023-10-01T09:30:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.533 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.334617+0000", 2023-10-01T09:30:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.10392075000000001 2023-10-01T09:30:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:21876 8.5 8:a365956b:::benchmark_data_smithi138_431617_object21875:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e332)", 2023-10-01T09:30:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:29:42.233785+0000", 2023-10-01T09:30:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 77.264132817999993, 2023-10-01T09:30:59.534 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.100935055, 2023-10-01T09:30:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 21876 2023-10-01T09:30:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.535 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.233785+0000", 2023-10-01T09:30:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.233785+0000", 2023-10-01T09:30:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.233784+0000", 2023-10-01T09:30:59.536 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8875999999999997e-05 2023-10-01T09:30:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.233843+0000", 2023-10-01T09:30:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1759e-05 2023-10-01T09:30:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.233855+0000", 2023-10-01T09:30:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7879999999999997e-06 2023-10-01T09:30:59.537 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.233860+0000", 2023-10-01T09:30:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5522999999999999e-05 2023-10-01T09:30:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.233895+0000", 2023-10-01T09:30:59.538 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6566999999999999e-05 2023-10-01T09:30:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.233992+0000", 2023-10-01T09:30:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000101933 2023-10-01T09:30:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.234094+0000", 2023-10-01T09:30:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00071760300000000003 2023-10-01T09:30:59.539 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.234811+0000", 2023-10-01T09:30:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000240067 2023-10-01T09:30:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.235051+0000", 2023-10-01T09:30:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.099596737000000005 2023-10-01T09:30:59.540 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.334648+0000", 2023-10-01T09:30:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7567000000000001e-05 2023-10-01T09:30:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.334686+0000", 2023-10-01T09:30:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4669999999999998e-05 2023-10-01T09:30:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.541 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:42.334720+0000", 2023-10-01T09:30:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.100935055 2023-10-01T09:30:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.542 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:29959 8.5 8:ac02a9b6:::benchmark_data_smithi138_431617_object29958:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e332)", 2023-10-01T09:30:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:29:47.307113+0000", 2023-10-01T09:30:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 72.190805314000002, 2023-10-01T09:30:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16153841799999999, 2023-10-01T09:30:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 29959 2023-10-01T09:30:59.543 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.307113+0000", 2023-10-01T09:30:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.307113+0000", 2023-10-01T09:30:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:30:59.544 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.307111+0000", 2023-10-01T09:30:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4925999999999997e-05 2023-10-01T09:30:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.307146+0000", 2023-10-01T09:30:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7503000000000002e-05 2023-10-01T09:30:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.307164+0000", 2023-10-01T09:30:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7569999999999998e-06 2023-10-01T09:30:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.307170+0000", 2023-10-01T09:30:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1662999999999997e-05 2023-10-01T09:30:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.307212+0000", 2023-10-01T09:30:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000105073 2023-10-01T09:30:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.307317+0000", 2023-10-01T09:30:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000166861 2023-10-01T09:30:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.547 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.307484+0000", 2023-10-01T09:30:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001254872 2023-10-01T09:30:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.308739+0000", 2023-10-01T09:30:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00030904100000000002 2023-10-01T09:30:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.309048+0000", 2023-10-01T09:30:59.548 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15950238 2023-10-01T09:30:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.468550+0000", 2023-10-01T09:30:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7853000000000003e-05 2023-10-01T09:30:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.468608+0000", 2023-10-01T09:30:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3139000000000002e-05 2023-10-01T09:30:59.549 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.468651+0000", 2023-10-01T09:30:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16153841799999999 2023-10-01T09:30:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:29970 8.5 8:aef8768d:::benchmark_data_smithi138_431617_object29969:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e332)", 2023-10-01T09:30:59.550 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:29:47.312373+0000", 2023-10-01T09:30:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 72.185545367000003, 2023-10-01T09:30:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15862991400000001, 2023-10-01T09:30:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 29970 2023-10-01T09:30:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.551 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.312373+0000", 2023-10-01T09:30:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.312373+0000", 2023-10-01T09:30:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:30:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.312371+0000", 2023-10-01T09:30:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6927e-05 2023-10-01T09:30:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.312418+0000", 2023-10-01T09:30:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6200000000000001e-05 2023-10-01T09:30:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.312434+0000", 2023-10-01T09:30:59.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8569999999999997e-06 2023-10-01T09:30:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.312441+0000", 2023-10-01T09:30:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9667999999999999e-05 2023-10-01T09:30:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.312491+0000", 2023-10-01T09:30:59.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013784800000000001 2023-10-01T09:30:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.312628+0000", 2023-10-01T09:30:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017208800000000001 2023-10-01T09:30:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.312800+0000", 2023-10-01T09:30:59.555 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0020022410000000001 2023-10-01T09:30:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.314803+0000", 2023-10-01T09:30:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00094958 2023-10-01T09:30:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.315752+0000", 2023-10-01T09:30:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15515389399999999 2023-10-01T09:30:59.556 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.470906+0000", 2023-10-01T09:30:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5315999999999997e-05 2023-10-01T09:30:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.470961+0000", 2023-10-01T09:30:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1255999999999999e-05 2023-10-01T09:30:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:29:47.471003+0000", 2023-10-01T09:30:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15862991400000001 2023-10-01T09:30:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:55933 8.5 8:a802442f:::benchmark_data_smithi138_431617_object55932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e339)", 2023-10-01T09:30:59.558 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:08.171462+0000", 2023-10-01T09:30:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 51.326456446000002, 2023-10-01T09:30:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.133043998, 2023-10-01T09:30:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 55933 2023-10-01T09:30:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.559 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.560 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.560 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.560 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:08.171462+0000", 2023-10-01T09:30:59.560 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.560 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.560 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.560 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.560 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:08.171462+0000", 2023-10-01T09:30:59.560 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:30:59.560 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:08.171460+0000", 2023-10-01T09:30:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3126000000000001e-05 2023-10-01T09:30:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:08.171493+0000", 2023-10-01T09:30:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2658e-05 2023-10-01T09:30:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.561 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:08.171506+0000", 2023-10-01T09:30:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.661e-06 2023-10-01T09:30:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:08.171513+0000", 2023-10-01T09:30:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5578999999999998e-05 2023-10-01T09:30:59.562 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:08.171559+0000", 2023-10-01T09:30:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012531900000000001 2023-10-01T09:30:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:08.171684+0000", 2023-10-01T09:30:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000142948 2023-10-01T09:30:59.563 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:08.171827+0000", 2023-10-01T09:30:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0042629429999999999 2023-10-01T09:30:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:08.176090+0000", 2023-10-01T09:30:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00047082100000000001 2023-10-01T09:30:59.564 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:08.176561+0000", 2023-10-01T09:30:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.12785481300000001 2023-10-01T09:30:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:08.304416+0000", 2023-10-01T09:30:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0914e-05 2023-10-01T09:30:59.565 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:08.304477+0000", 2023-10-01T09:30:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9014e-05 2023-10-01T09:30:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:08.304506+0000", 2023-10-01T09:30:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.133043998 2023-10-01T09:30:59.566 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:61933 8.5 8:ae340eae:::benchmark_data_smithi138_431617_object61932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e340)", 2023-10-01T09:30:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:13.546596+0000", 2023-10-01T09:30:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 45.951322114999996, 2023-10-01T09:30:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.170064412, 2023-10-01T09:30:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.567 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 61933 2023-10-01T09:30:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546596+0000", 2023-10-01T09:30:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.568 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546596+0000", 2023-10-01T09:30:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546595+0000", 2023-10-01T09:30:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0971999999999999e-05 2023-10-01T09:30:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.569 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546646+0000", 2023-10-01T09:30:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7750000000000004e-06 2023-10-01T09:30:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546656+0000", 2023-10-01T09:30:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8340000000000001e-06 2023-10-01T09:30:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.570 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546663+0000", 2023-10-01T09:30:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019821679999999999 2023-10-01T09:30:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.548645+0000", 2023-10-01T09:30:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010491400000000001 2023-10-01T09:30:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.571 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.548750+0000", 2023-10-01T09:30:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011701099999999999 2023-10-01T09:30:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.548867+0000", 2023-10-01T09:30:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0020976710000000002 2023-10-01T09:30:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.572 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.550964+0000", 2023-10-01T09:30:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0018446389999999999 2023-10-01T09:30:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.552809+0000", 2023-10-01T09:30:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16371334600000001 2023-10-01T09:30:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.573 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716522+0000", 2023-10-01T09:30:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3269999999999994e-05 2023-10-01T09:30:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716606+0000", 2023-10-01T09:30:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4738999999999999e-05 2023-10-01T09:30:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.574 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716660+0000", 2023-10-01T09:30:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.170064412 2023-10-01T09:30:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:61939 8.5 8:afb0542b:::benchmark_data_smithi138_431617_object61938:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e340)", 2023-10-01T09:30:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:13.554085+0000", 2023-10-01T09:30:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 45.943832784000001, 2023-10-01T09:30:59.575 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16270126800000001, 2023-10-01T09:30:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 61939 2023-10-01T09:30:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554085+0000", 2023-10-01T09:30:59.576 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554085+0000", 2023-10-01T09:30:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:30:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554084+0000", 2023-10-01T09:30:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3692000000000002e-05 2023-10-01T09:30:59.577 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554118+0000", 2023-10-01T09:30:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2537999999999999e-05 2023-10-01T09:30:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554130+0000", 2023-10-01T09:30:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1240000000000002e-06 2023-10-01T09:30:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.578 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554136+0000", 2023-10-01T09:30:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2286000000000001e-05 2023-10-01T09:30:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554179+0000", 2023-10-01T09:30:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000149696 2023-10-01T09:30:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.579 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554328+0000", 2023-10-01T09:30:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00020244499999999999 2023-10-01T09:30:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.581 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554531+0000", 2023-10-01T09:30:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0036724459999999998 2023-10-01T09:30:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.558203+0000", 2023-10-01T09:30:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0352e-05 2023-10-01T09:30:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.582 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.558214+0000", 2023-10-01T09:30:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15850223199999999 2023-10-01T09:30:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716716+0000", 2023-10-01T09:30:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5321000000000003e-05 2023-10-01T09:30:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716751+0000", 2023-10-01T09:30:59.583 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5450000000000001e-05 2023-10-01T09:30:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716787+0000", 2023-10-01T09:30:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16270126800000001 2023-10-01T09:30:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:61940 8.5 8:a5fb8062:::benchmark_data_smithi138_431617_object61939:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e340)", 2023-10-01T09:30:59.584 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:13.554147+0000", 2023-10-01T09:30:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 45.943771243, 2023-10-01T09:30:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16273934700000001, 2023-10-01T09:30:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 61940 2023-10-01T09:30:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.585 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554147+0000", 2023-10-01T09:30:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554147+0000", 2023-10-01T09:30:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:30:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.586 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554146+0000", 2023-10-01T09:30:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5114999999999998e-05 2023-10-01T09:30:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554181+0000", 2023-10-01T09:30:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0229999999999998e-06 2023-10-01T09:30:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.587 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554190+0000", 2023-10-01T09:30:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8210000000000001e-06 2023-10-01T09:30:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.554195+0000", 2023-10-01T09:30:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0021983049999999998 2023-10-01T09:30:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.556394+0000", 2023-10-01T09:30:59.588 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010834499999999999 2023-10-01T09:30:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.556502+0000", 2023-10-01T09:30:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011230600000000001 2023-10-01T09:30:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.556614+0000", 2023-10-01T09:30:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0017141420000000001 2023-10-01T09:30:59.589 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.558328+0000", 2023-10-01T09:30:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00059152 2023-10-01T09:30:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.558920+0000", 2023-10-01T09:30:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15790080200000001 2023-10-01T09:30:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.590 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716821+0000", 2023-10-01T09:30:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1440999999999999e-05 2023-10-01T09:30:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716852+0000", 2023-10-01T09:30:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4181999999999999e-05 2023-10-01T09:30:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.591 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716886+0000", 2023-10-01T09:30:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16273934700000001 2023-10-01T09:30:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:75080 8.5 8:a500d559:::benchmark_data_smithi138_431617_object75079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-10-01T09:30:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:28.018267+0000", 2023-10-01T09:30:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 31.479651285999999, 2023-10-01T09:30:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.120543808, 2023-10-01T09:30:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.592 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 75080 2023-10-01T09:30:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.018267+0000", 2023-10-01T09:30:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.593 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.018267+0000", 2023-10-01T09:30:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:30:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.018266+0000", 2023-10-01T09:30:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0967000000000001e-05 2023-10-01T09:30:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.594 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.018327+0000", 2023-10-01T09:30:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5969999999999992e-06 2023-10-01T09:30:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.018335+0000", 2023-10-01T09:30:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4709999999999996e-06 2023-10-01T09:30:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.595 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.018345+0000", 2023-10-01T09:30:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9292000000000003e-05 2023-10-01T09:30:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.596 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.018384+0000", 2023-10-01T09:30:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000123316 2023-10-01T09:30:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.018507+0000", 2023-10-01T09:30:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014364600000000001 2023-10-01T09:30:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.018651+0000", 2023-10-01T09:30:59.597 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024003179999999998 2023-10-01T09:30:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.021051+0000", 2023-10-01T09:30:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0023385760000000002 2023-10-01T09:30:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.023390+0000", 2023-10-01T09:30:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.115321542 2023-10-01T09:30:59.598 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.138711+0000", 2023-10-01T09:30:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5257999999999998e-05 2023-10-01T09:30:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.138767+0000", 2023-10-01T09:30:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4066000000000001e-05 2023-10-01T09:30:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.599 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.138811+0000", 2023-10-01T09:30:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.120543808 2023-10-01T09:30:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop_reply(client.45674.0:75080 8.5 e341/326 ondisk, result = 0)", 2023-10-01T09:30:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:28.024350+0000", 2023-10-01T09:30:59.600 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 31.473568177000001, 2023-10-01T09:30:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.11446988299999999, 2023-10-01T09:30:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:30:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.024350+0000", 2023-10-01T09:30:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.601 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.024350+0000", 2023-10-01T09:30:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:30:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.024349+0000", 2023-10-01T09:30:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3690000000000003e-06 2023-10-01T09:30:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.602 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.024356+0000", 2023-10-01T09:30:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6539999999999999e-06 2023-10-01T09:30:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.024363+0000", 2023-10-01T09:30:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0600000000000001e-06 2023-10-01T09:30:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.603 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.024368+0000", 2023-10-01T09:30:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.114331107 2023-10-01T09:30:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.138699+0000", 2023-10-01T09:30:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8430000000000004e-06 2023-10-01T09:30:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.604 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.138705+0000", 2023-10-01T09:30:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011443900000000001 2023-10-01T09:30:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.138820+0000", 2023-10-01T09:30:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.11446988299999999 2023-10-01T09:30:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.605 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:75090 8.5 8:a63ad20e:::benchmark_data_smithi138_431617_object75089:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-10-01T09:30:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:28.025672+0000", 2023-10-01T09:30:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 31.472245916999999, 2023-10-01T09:30:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.121740347, 2023-10-01T09:30:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 75090 2023-10-01T09:30:59.606 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.025672+0000", 2023-10-01T09:30:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.025672+0000", 2023-10-01T09:30:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:30:59.607 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.025671+0000", 2023-10-01T09:30:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3162999999999999e-05 2023-10-01T09:30:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.025724+0000", 2023-10-01T09:30:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.112948857 2023-10-01T09:30:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.608 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.138673+0000", 2023-10-01T09:30:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1325000000000001e-05 2023-10-01T09:30:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.138684+0000", 2023-10-01T09:30:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00018072699999999999 2023-10-01T09:30:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.609 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.138865+0000", 2023-10-01T09:30:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000165487 2023-10-01T09:30:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.139030+0000", 2023-10-01T09:30:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00018111099999999999 2023-10-01T09:30:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.610 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.139211+0000", 2023-10-01T09:30:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078981320000000004 2023-10-01T09:30:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.147109+0000", 2023-10-01T09:30:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.1575000000000003e-05 2023-10-01T09:30:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.147201+0000", 2023-10-01T09:30:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000103526 2023-10-01T09:30:59.611 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.147305+0000", 2023-10-01T09:30:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0260000000000002e-05 2023-10-01T09:30:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.147365+0000", 2023-10-01T09:30:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7855000000000002e-05 2023-10-01T09:30:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.612 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.147413+0000", 2023-10-01T09:30:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.121740347 2023-10-01T09:30:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:75087 8.3 e341/326 8:c100d0aa:::benchmark_data_smithi138_431617_object75086:head v 341'4658, mlcod=341'4656)", 2023-10-01T09:30:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:28.026373+0000", 2023-10-01T09:30:59.613 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 31.47154493, 2023-10-01T09:30:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.119218293, 2023-10-01T09:30:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.026373+0000", 2023-10-01T09:30:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.614 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.026373+0000", 2023-10-01T09:30:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.026372+0000", 2023-10-01T09:30:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4542000000000001e-05 2023-10-01T09:30:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.615 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.026417+0000", 2023-10-01T09:30:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.112273764 2023-10-01T09:30:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.138691+0000", 2023-10-01T09:30:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7780000000000002e-06 2023-10-01T09:30:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.138701+0000", 2023-10-01T09:30:59.616 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7086999999999998e-05 2023-10-01T09:30:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.138748+0000", 2023-10-01T09:30:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9724999999999998e-05 2023-10-01T09:30:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.138767+0000", 2023-10-01T09:30:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0067818899999999996 2023-10-01T09:30:59.617 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.145549+0000", 2023-10-01T09:30:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2324999999999999e-05 2023-10-01T09:30:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.145592+0000", 2023-10-01T09:30:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.119218293 2023-10-01T09:30:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.618 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:75567 8.5 8:aef5727f:::benchmark_data_smithi138_431617_object75566:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-10-01T09:30:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:28.647427+0000", 2023-10-01T09:30:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 30.850491065, 2023-10-01T09:30:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17919886700000001, 2023-10-01T09:30:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.619 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 75567 2023-10-01T09:30:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647427+0000", 2023-10-01T09:30:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.620 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647427+0000", 2023-10-01T09:30:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:30:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647425+0000", 2023-10-01T09:30:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8278e-05 2023-10-01T09:30:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647483+0000", 2023-10-01T09:30:59.621 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4720999999999999e-05 2023-10-01T09:30:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647508+0000", 2023-10-01T09:30:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8720000000000002e-06 2023-10-01T09:30:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647516+0000", 2023-10-01T09:30:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00034716700000000001 2023-10-01T09:30:59.622 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647863+0000", 2023-10-01T09:30:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5581999999999994e-05 2023-10-01T09:30:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647959+0000", 2023-10-01T09:30:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000119572 2023-10-01T09:30:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.623 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.648078+0000", 2023-10-01T09:30:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078048329999999997 2023-10-01T09:30:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.655883+0000", 2023-10-01T09:30:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013701100000000001 2023-10-01T09:30:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.624 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.656020+0000", 2023-10-01T09:30:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17050426699999999 2023-10-01T09:30:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.826524+0000", 2023-10-01T09:30:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8119999999999999e-05 2023-10-01T09:30:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.625 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.826582+0000", 2023-10-01T09:30:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3550999999999997e-05 2023-10-01T09:30:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.826626+0000", 2023-10-01T09:30:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17919886700000001 2023-10-01T09:30:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.626 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:75568 8.5 8:ad74c15b:::benchmark_data_smithi138_431617_object75567:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-10-01T09:30:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:28.647524+0000", 2023-10-01T09:30:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 30.850394678000001, 2023-10-01T09:30:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18103444699999999, 2023-10-01T09:30:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.627 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 75568 2023-10-01T09:30:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647524+0000", 2023-10-01T09:30:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.628 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647524+0000", 2023-10-01T09:30:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:30:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647523+0000", 2023-10-01T09:30:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4457999999999998e-05 2023-10-01T09:30:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647597+0000", 2023-10-01T09:30:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6279999999999994e-06 2023-10-01T09:30:59.629 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647606+0000", 2023-10-01T09:30:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0679999999999996e-06 2023-10-01T09:30:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647611+0000", 2023-10-01T09:30:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0030153350000000001 2023-10-01T09:30:59.630 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.650626+0000", 2023-10-01T09:30:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013843899999999999 2023-10-01T09:30:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.650765+0000", 2023-10-01T09:30:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017521299999999999 2023-10-01T09:30:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.631 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.650940+0000", 2023-10-01T09:30:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049486560000000001 2023-10-01T09:30:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.655889+0000", 2023-10-01T09:30:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00030954499999999999 2023-10-01T09:30:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.632 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.656198+0000", 2023-10-01T09:30:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.172268646 2023-10-01T09:30:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828467+0000", 2023-10-01T09:30:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0330000000000001e-05 2023-10-01T09:30:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828517+0000", 2023-10-01T09:30:59.633 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0725000000000001e-05 2023-10-01T09:30:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828558+0000", 2023-10-01T09:30:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18103444699999999 2023-10-01T09:30:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.634 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:75571 8.5 8:a2fb1dba:::benchmark_data_smithi138_431617_object75570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-10-01T09:30:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:28.647622+0000", 2023-10-01T09:30:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 30.850296050000001, 2023-10-01T09:30:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18107846399999999, 2023-10-01T09:30:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 75571 2023-10-01T09:30:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.635 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647622+0000", 2023-10-01T09:30:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647622+0000", 2023-10-01T09:30:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:30:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.636 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647621+0000", 2023-10-01T09:30:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8969e-05 2023-10-01T09:30:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647650+0000", 2023-10-01T09:30:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0261000000000001e-05 2023-10-01T09:30:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.637 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647661+0000", 2023-10-01T09:30:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8960000000000002e-06 2023-10-01T09:30:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647666+0000", 2023-10-01T09:30:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053841219999999999 2023-10-01T09:30:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.638 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.653050+0000", 2023-10-01T09:30:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015679600000000001 2023-10-01T09:30:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.653206+0000", 2023-10-01T09:30:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00019690199999999999 2023-10-01T09:30:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.653403+0000", 2023-10-01T09:30:59.639 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002801993 2023-10-01T09:30:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.656205+0000", 2023-10-01T09:30:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00025078999999999998 2023-10-01T09:30:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.656456+0000", 2023-10-01T09:30:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.172176575 2023-10-01T09:30:59.640 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828633+0000", 2023-10-01T09:30:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1740999999999999e-05 2023-10-01T09:30:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828674+0000", 2023-10-01T09:30:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6143999999999999e-05 2023-10-01T09:30:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.641 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828701+0000", 2023-10-01T09:30:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18107846399999999 2023-10-01T09:30:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:89387 8.5 8:ac2bd163:::benchmark_data_smithi138_431617_object89386:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-01T09:30:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:46.701591+0000", 2023-10-01T09:30:59.642 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 12.796326970999999, 2023-10-01T09:30:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.242286687, 2023-10-01T09:30:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 89387 2023-10-01T09:30:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.643 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701591+0000", 2023-10-01T09:30:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701591+0000", 2023-10-01T09:30:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:30:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701590+0000", 2023-10-01T09:30:59.644 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2069e-05 2023-10-01T09:30:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701632+0000", 2023-10-01T09:30:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.005e-05 2023-10-01T09:30:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701642+0000", 2023-10-01T09:30:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2100000000000001e-06 2023-10-01T09:30:59.645 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701647+0000", 2023-10-01T09:30:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012216000000000001 2023-10-01T09:30:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701769+0000", 2023-10-01T09:30:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015872299999999999 2023-10-01T09:30:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.646 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701928+0000", 2023-10-01T09:30:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017488099999999999 2023-10-01T09:30:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.702103+0000", 2023-10-01T09:30:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0026736849999999999 2023-10-01T09:30:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.647 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.704776+0000", 2023-10-01T09:30:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00095142499999999999 2023-10-01T09:30:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.705728+0000", 2023-10-01T09:30:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.238030466 2023-10-01T09:30:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.943758+0000", 2023-10-01T09:30:59.648 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1157999999999999e-05 2023-10-01T09:30:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.943829+0000", 2023-10-01T09:30:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8421000000000003e-05 2023-10-01T09:30:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.943878+0000", 2023-10-01T09:30:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.242286687 2023-10-01T09:30:59.649 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.650 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.650 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.650 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.650 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.650 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:89417 8.5 8:ac07dffe:::benchmark_data_smithi138_431617_object89416:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-01T09:30:59.650 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:46.731169+0000", 2023-10-01T09:30:59.650 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 12.766748859, 2023-10-01T09:30:59.650 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21775186899999999, 2023-10-01T09:30:59.650 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:30:59.650 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:30:59.650 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:30:59.650 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:30:59.651 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:30:59.651 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 89417 2023-10-01T09:30:59.651 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.651 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:30:59.651 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.651 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:30:59.651 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731169+0000", 2023-10-01T09:30:59.651 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:30:59.651 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.651 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.651 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:30:59.651 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731169+0000", 2023-10-01T09:30:59.652 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:30:59.652 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.652 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.652 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:30:59.652 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731168+0000", 2023-10-01T09:30:59.652 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0485000000000003e-05 2023-10-01T09:30:59.652 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.652 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.652 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:30:59.652 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731208+0000", 2023-10-01T09:30:59.652 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5139e-05 2023-10-01T09:30:59.653 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.653 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.653 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:30:59.653 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731223+0000", 2023-10-01T09:30:59.653 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4389999999999997e-06 2023-10-01T09:30:59.653 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.653 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.653 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:30:59.653 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731230+0000", 2023-10-01T09:30:59.653 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8512e-05 2023-10-01T09:30:59.653 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.653 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.654 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:30:59.654 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731268+0000", 2023-10-01T09:30:59.654 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012213799999999999 2023-10-01T09:30:59.654 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.654 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.654 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:30:59.654 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731390+0000", 2023-10-01T09:30:59.654 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000120797 2023-10-01T09:30:59.654 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.654 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.654 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:30:59.655 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731511+0000", 2023-10-01T09:30:59.655 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002402623 2023-10-01T09:30:59.655 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.655 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.655 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:30:59.655 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.733914+0000", 2023-10-01T09:30:59.655 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00025578500000000002 2023-10-01T09:30:59.655 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.655 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.655 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.655 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.734170+0000", 2023-10-01T09:30:59.656 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21464570199999999 2023-10-01T09:30:59.656 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.656 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.656 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:30:59.656 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.948815+0000", 2023-10-01T09:30:59.656 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2032999999999994e-05 2023-10-01T09:30:59.656 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.656 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.656 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:30:59.656 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.948877+0000", 2023-10-01T09:30:59.656 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3804999999999998e-05 2023-10-01T09:30:59.657 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:30:59.657 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:30:59.657 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:30:59.657 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.948921+0000", 2023-10-01T09:30:59.657 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21775186899999999 2023-10-01T09:30:59.657 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.657 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.657 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.657 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:30:59.657 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:30:59.657 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:30:59.658 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:30:59.659 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:30:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/766398965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:59.682 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:30:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/766398965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:59.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:30:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/766398965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:30:59.685 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:30:59.728 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:30:59.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:30:59.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:30:59.831 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:30:59.913 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:00.257 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:00.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:00.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:00.296 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:31:00.328 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:31:00.373 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:00.407 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 87 16 99315 99299 71.2891 37.125 0.0258606 0.0140215 2023-10-01T09:31:00.464 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:00.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:00 smithi190 ceph-mon[160435]: pgmap v957: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 752 op/s 2023-10-01T09:31:00.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:00 smithi138 ceph-mon[185886]: pgmap v957: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 752 op/s 2023-10-01T09:31:00.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:00 smithi138 ceph-mon[192513]: pgmap v957: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 752 op/s 2023-10-01T09:31:00.712 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:00.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:00.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:00.829 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:00.864 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:00.928 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:00.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:00.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:01.039 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:31:01.065 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:01.107 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:31:01.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:01.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:31:01.215 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:01.294 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:01.405 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 88 16 99961 99945 70.938 40.375 0.00630047 0.0140921 2023-10-01T09:31:01.636 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:01.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:01.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:01.720 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:01.758 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:01.768 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:01.854 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:02.109 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:02.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:02.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:02.214 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:31:02.285 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:02.291 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:31:02.416 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 89 16 100714 100698 70.6693 47.0625 0.0304912 0.0141451 2023-10-01T09:31:02.462 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:31:02.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:02.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:31:02.579 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:31:02.616 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:02.678 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:31:02.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:02.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:31:02.783 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:02.858 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:02.986 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:02 smithi190 ceph-mon[160435]: pgmap v958: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s 2023-10-01T09:31:02.987 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:02 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2714864489' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:31:03.309 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:03.315 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:02 smithi138 ceph-mon[185886]: pgmap v958: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s 2023-10-01T09:31:03.316 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:02 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2714864489' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:31:03.316 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:02 smithi138 ceph-mon[192513]: pgmap v958: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 47 MiB/s wr, 744 op/s 2023-10-01T09:31:03.316 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:02 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2714864489' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:31:03.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:03.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:03.399 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.3 to 1 (10000) 2023-10-01T09:31:03.405 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 90 16 101408 101392 70.3662 43.375 0.00352075 0.0141941 2023-10-01T09:31:03.418 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:03.446 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:03.487 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:03.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:03.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:03.595 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:03.621 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:03.662 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:03.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:03.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:03.767 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:31:03.792 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:03.833 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:31:03.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:03.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:31:03.941 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:31:03.969 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:04.317 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:31:04.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:04.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:31:04.406 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Total time run: 90.1975 2023-10-01T09:31:04.406 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Total writes made: 101408 2023-10-01T09:31:04.406 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Write size: 65536 2023-10-01T09:31:04.406 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Object size: 65536 2023-10-01T09:31:04.406 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Bandwidth (MB/sec): 70.268 2023-10-01T09:31:04.406 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev Bandwidth: 34.1941 2023-10-01T09:31:04.407 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max bandwidth (MB/sec): 190.688 2023-10-01T09:31:04.407 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min bandwidth (MB/sec): 4.375 2023-10-01T09:31:04.407 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Average IOPS: 1124 2023-10-01T09:31:04.407 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev IOPS: 547.106 2023-10-01T09:31:04.407 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max IOPS: 3051 2023-10-01T09:31:04.407 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min IOPS: 70 2023-10-01T09:31:04.407 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Average Latency(s): 0.0142273 2023-10-01T09:31:04.407 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev Latency(s): 0.0210203 2023-10-01T09:31:04.407 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max latency(s): 1.11695 2023-10-01T09:31:04.407 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min latency(s): 0.00115441 2023-10-01T09:31:04.411 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T09:31:04.425 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:04.517 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:04 smithi138 ceph-mon[192513]: failed to encode map e348 with expected crc 2023-10-01T09:31:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:04 smithi138 ceph-mon[192513]: failed to encode map e348 with expected crc 2023-10-01T09:31:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2714864489' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:31:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:04 smithi138 ceph-mon[192513]: osdmap e348: 8 total, 8 up, 7 in 2023-10-01T09:31:04.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:04 smithi138 ceph-mon[192513]: pgmap v960: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 47 MiB/s wr, 747 op/s 2023-10-01T09:31:04.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:04 smithi138 ceph-mon[185886]: failed to encode map e348 with expected crc 2023-10-01T09:31:04.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:04 smithi138 ceph-mon[185886]: failed to encode map e348 with expected crc 2023-10-01T09:31:04.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2714864489' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:31:04.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:04 smithi138 ceph-mon[185886]: osdmap e348: 8 total, 8 up, 7 in 2023-10-01T09:31:04.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:04 smithi138 ceph-mon[185886]: pgmap v960: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 47 MiB/s wr, 747 op/s 2023-10-01T09:31:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:04 smithi190 ceph-mon[160435]: failed to encode map e348 with expected crc 2023-10-01T09:31:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:04 smithi190 ceph-mon[160435]: failed to encode map e348 with expected crc 2023-10-01T09:31:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2714864489' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:31:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:04 smithi190 ceph-mon[160435]: osdmap e348: 8 total, 8 up, 7 in 2023-10-01T09:31:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:04 smithi190 ceph-mon[160435]: pgmap v960: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 47 MiB/s wr, 747 op/s 2023-10-01T09:31:04.936 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:31:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:31:04] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.33.4" 2023-10-01T09:31:05.213 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:05.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:05.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:05.271 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:05.308 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-01T09:31:05.328 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:31:05.489 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:05 smithi138 ceph-mon[185886]: failed to encode map e348 with expected crc 2023-10-01T09:31:05.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:05 smithi138 ceph-mon[192513]: failed to encode map e348 with expected crc 2023-10-01T09:31:05.496 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:05.791 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:31:05.800 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:31:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:31:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:102324 8.2 8:41a940a2:::benchmark_data_smithi138_431617_object913:head [delete] snapc 0=[] ondisk+write+known_if_redirected e348)", 2023-10-01T09:31:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:05.770877+0000", 2023-10-01T09:31:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.026542109000000001, 2023-10-01T09:31:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.02658168, 2023-10-01T09:31:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:31:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:05.801 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 102324 2023-10-01T09:31:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:05.770877+0000", 2023-10-01T09:31:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:05.802 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:05.770877+0000", 2023-10-01T09:31:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:05.770876+0000", 2023-10-01T09:31:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:31:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:05.770886+0000", 2023-10-01T09:31:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0779e-05 2023-10-01T09:31:05.803 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:05.770908+0000", 2023-10-01T09:31:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1081e-05 2023-10-01T09:31:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:05.770919+0000", 2023-10-01T09:31:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1765e-05 2023-10-01T09:31:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:05.804 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:05.774403+0000", 2023-10-01T09:31:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034833099999999999 2023-10-01T09:31:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:05.774538+0000", 2023-10-01T09:31:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013582199999999999 2023-10-01T09:31:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-10-01T09:31:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:05.774662+0000", 2023-10-01T09:31:05.805 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000123208 2023-10-01T09:31:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:05.782055+0000", 2023-10-01T09:31:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0073929310000000002 2023-10-01T09:31:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:05.787027+0000", 2023-10-01T09:31:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049722569999999999 2023-10-01T09:31:05.806 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:31:05.807 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 1 2023-10-01T09:31:05.807 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:31:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:05 smithi190 ceph-mon[160435]: failed to encode map e348 with expected crc 2023-10-01T09:31:05.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:05.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:31:05.911 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:31:05.947 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:06.015 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:31:06.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:06.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:31:06.125 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:06.184 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:06.421 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:31:06.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:06 smithi190 ceph-mon[160435]: pgmap v961: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 103 B/s rd, 41 MiB/s wr, 686 op/s 2023-10-01T09:31:06.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:06 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3504832972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:31:06.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:06 smithi138 ceph-mon[192513]: pgmap v961: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 103 B/s rd, 41 MiB/s wr, 686 op/s 2023-10-01T09:31:06.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:06 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3504832972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:31:06.830 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:06 smithi138 ceph-mon[185886]: pgmap v961: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 103 B/s rd, 41 MiB/s wr, 686 op/s 2023-10-01T09:31:06.830 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:06 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3504832972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:31:07.467 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:07.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:07.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:07.574 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:07.601 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:07.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:07 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:31:07.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:07 smithi190 ceph-mon[160435]: failed to encode map e349 with expected crc 2023-10-01T09:31:07.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:07 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3504832972' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:31:07.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:07 smithi190 ceph-mon[160435]: osdmap e349: 8 total, 8 up, 7 in 2023-10-01T09:31:07.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:07 smithi190 ceph-mon[160435]: failed to encode map e349 with expected crc 2023-10-01T09:31:07.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:07 smithi190 ceph-mon[160435]: failed to encode map e349 with expected crc 2023-10-01T09:31:07.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:07 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:31:07.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:07 smithi138 ceph-mon[192513]: failed to encode map e349 with expected crc 2023-10-01T09:31:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:07 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3504832972' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:31:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:07 smithi138 ceph-mon[192513]: osdmap e349: 8 total, 8 up, 7 in 2023-10-01T09:31:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:07 smithi138 ceph-mon[192513]: failed to encode map e349 with expected crc 2023-10-01T09:31:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:07 smithi138 ceph-mon[192513]: failed to encode map e349 with expected crc 2023-10-01T09:31:07.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:07 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:31:07.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:07 smithi138 ceph-mon[185886]: failed to encode map e349 with expected crc 2023-10-01T09:31:07.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:07 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3504832972' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:31:07.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:07 smithi138 ceph-mon[185886]: osdmap e349: 8 total, 8 up, 7 in 2023-10-01T09:31:07.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:07 smithi138 ceph-mon[185886]: failed to encode map e349 with expected crc 2023-10-01T09:31:07.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:07 smithi138 ceph-mon[185886]: failed to encode map e349 with expected crc 2023-10-01T09:31:08.079 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:08.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:08.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:08.156 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:08.185 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:08.194 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:31:08.218 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:08.280 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:08.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:08.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:08.436 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:08.519 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:08.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:08 smithi190 ceph-mon[160435]: pgmap v963: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 127 B/s rd, 32 MiB/s wr, 558 op/s 2023-10-01T09:31:08.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:08 smithi138 ceph-mon[192513]: pgmap v963: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 127 B/s rd, 32 MiB/s wr, 558 op/s 2023-10-01T09:31:08.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:08 smithi138 ceph-mon[185886]: pgmap v963: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 127 B/s rd, 32 MiB/s wr, 558 op/s 2023-10-01T09:31:08.709 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:08.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:08.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:08.822 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:08.852 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:08.908 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:08.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:08.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:08.972 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:31:09.013 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:31:09.016 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:31:09.046 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:09.057 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:31:09.101 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:31:09.113 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:31:09.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:09.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:31:09.143 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:31:09.172 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:31:09.199 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:31:09.224 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:09.230 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:31:09.249 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:31:09.249 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:31:09.249 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:31:09.250 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:31:09.250 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:31:09.250 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:31:09.250 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:31:09.250 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:31:09.301 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:09.395 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:09.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:09.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:09.520 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:09.552 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:09 smithi138 ceph-mon[192513]: failed to encode map e349 with expected crc 2023-10-01T09:31:09.553 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:09 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4067100260' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:09.553 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:09 smithi138 ceph-mon[185886]: failed to encode map e349 with expected crc 2023-10-01T09:31:09.553 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:09 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4067100260' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:09.555 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:09.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:09 smithi190 ceph-mon[160435]: failed to encode map e349 with expected crc 2023-10-01T09:31:09.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:09 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4067100260' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:10.058 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:31:10.569 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:10.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:10.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:10.695 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:31:10.722 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:10 smithi190 ceph-mon[160435]: pgmap v964: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 127 B/s rd, 24 MiB/s wr, 648 op/s 2023-10-01T09:31:10.722 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:31:10.724 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:10.734 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:10.789 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:31:10.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:10.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:31:10.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:10 smithi138 ceph-mon[192513]: pgmap v964: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 127 B/s rd, 24 MiB/s wr, 648 op/s 2023-10-01T09:31:10.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:31:10.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:10 smithi138 ceph-mon[185886]: pgmap v964: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 127 B/s rd, 24 MiB/s wr, 648 op/s 2023-10-01T09:31:10.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:31:10.895 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:10.931 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:10.983 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:10.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:10.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:11.091 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:11.121 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:11.173 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:11.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:11.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:11.279 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:11.363 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:11.422 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:11.617 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:11.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:11.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:11.678 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4188684024' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:31:11.681 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4188684024' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:31:11.681 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:11 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:31:11.695 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.7 to 1 (10000) 2023-10-01T09:31:11.728 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:31:11.784 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:12.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4188684024' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:31:12.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:11 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:31:12.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:11 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:31:12.381 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:31:12.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:12.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:31:12.493 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:12.520 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:12.575 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:31:12.580 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:12.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:12.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:12.685 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:12.698 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:12 smithi138 ceph-mon[185886]: failed to encode map e350 with expected crc 2023-10-01T09:31:12.698 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:12 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:31:12.725 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:12.772 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:12.775 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:12 smithi190 ceph-mon[160435]: failed to encode map e350 with expected crc 2023-10-01T09:31:12.775 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:12 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:31:12.775 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:12 smithi190 ceph-mon[160435]: osdmap e350: 8 total, 8 up, 7 in 2023-10-01T09:31:12.775 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:12 smithi190 ceph-mon[160435]: failed to encode map e350 with expected crc 2023-10-01T09:31:12.776 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:12 smithi190 ceph-mon[160435]: pgmap v966: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 127 B/s rd, 5.3 MiB/s wr, 499 op/s 2023-10-01T09:31:12.776 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:12 smithi190 ceph-mon[160435]: failed to encode map e350 with expected crc 2023-10-01T09:31:12.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:12.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:12.877 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:12.936 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:13.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:12 smithi138 ceph-mon[185886]: osdmap e350: 8 total, 8 up, 7 in 2023-10-01T09:31:13.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:12 smithi138 ceph-mon[185886]: failed to encode map e350 with expected crc 2023-10-01T09:31:13.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:12 smithi138 ceph-mon[185886]: pgmap v966: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 127 B/s rd, 5.3 MiB/s wr, 499 op/s 2023-10-01T09:31:13.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:12 smithi138 ceph-mon[185886]: failed to encode map e350 with expected crc 2023-10-01T09:31:13.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:12 smithi138 ceph-mon[192513]: failed to encode map e350 with expected crc 2023-10-01T09:31:13.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:12 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:31:13.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:12 smithi138 ceph-mon[192513]: osdmap e350: 8 total, 8 up, 7 in 2023-10-01T09:31:13.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:12 smithi138 ceph-mon[192513]: failed to encode map e350 with expected crc 2023-10-01T09:31:13.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:12 smithi138 ceph-mon[192513]: pgmap v966: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 127 B/s rd, 5.3 MiB/s wr, 499 op/s 2023-10-01T09:31:13.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:12 smithi138 ceph-mon[192513]: failed to encode map e350 with expected crc 2023-10-01T09:31:13.279 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-01T09:31:13.417 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:13.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:13.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:13.532 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:13.669 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:14.269 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:31:14.270 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:31:14.894 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:14.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:14.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:14.999 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:15.036 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:15.198 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:31:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:31:14] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.33.4" 2023-10-01T09:31:15.198 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:14 smithi138 ceph-mon[185886]: pgmap v967: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 751 KiB/s wr, 720 op/s 2023-10-01T09:31:15.202 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:14 smithi138 ceph-mon[192513]: pgmap v967: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 751 KiB/s wr, 720 op/s 2023-10-01T09:31:15.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:14 smithi190 ceph-mon[160435]: pgmap v967: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 751 KiB/s wr, 720 op/s 2023-10-01T09:31:15.319 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:15.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:15.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:15.432 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:15.476 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:15.512 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:15.513 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:31:15.530 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:15.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:15.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:15.641 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:15.674 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:15.744 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:15.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:15.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:15.851 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:15.872 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:15.880 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:15.937 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:15.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:15.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:16.044 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:16.076 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:16.142 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:16.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:16.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:16.253 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:16.349 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:17.009 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:31:17.128 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:17.137 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:17 smithi138 ceph-mon[185886]: pgmap v968: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 632 KiB/s wr, 725 op/s 2023-10-01T09:31:17.137 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:17 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4038174939' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:31:17.138 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:17 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:31:17.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:17.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:17.141 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:17 smithi138 ceph-mon[192513]: pgmap v968: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 632 KiB/s wr, 725 op/s 2023-10-01T09:31:17.141 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:17 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4038174939' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:31:17.141 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:17 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:31:17.188 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:17.239 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:17 smithi190 ceph-mon[160435]: pgmap v968: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 632 KiB/s wr, 725 op/s 2023-10-01T09:31:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:17 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4038174939' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:31:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:17 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:31:17.316 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:18 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:31:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:18 smithi190 ceph-mon[160435]: failed to encode map e351 with expected crc 2023-10-01T09:31:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:18 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:31:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:18 smithi190 ceph-mon[160435]: osdmap e351: 8 total, 8 up, 7 in 2023-10-01T09:31:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:18 smithi190 ceph-mon[160435]: failed to encode map e351 with expected crc 2023-10-01T09:31:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:18 smithi190 ceph-mon[160435]: failed to encode map e351 with expected crc 2023-10-01T09:31:18.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:18 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:31:18.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:18 smithi138 ceph-mon[192513]: failed to encode map e351 with expected crc 2023-10-01T09:31:18.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:18 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:31:18.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:18 smithi138 ceph-mon[192513]: osdmap e351: 8 total, 8 up, 7 in 2023-10-01T09:31:18.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:18 smithi138 ceph-mon[192513]: failed to encode map e351 with expected crc 2023-10-01T09:31:18.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:18 smithi138 ceph-mon[192513]: failed to encode map e351 with expected crc 2023-10-01T09:31:18.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:18 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:31:18.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:18 smithi138 ceph-mon[185886]: failed to encode map e351 with expected crc 2023-10-01T09:31:18.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:18 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:31:18.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:18 smithi138 ceph-mon[185886]: osdmap e351: 8 total, 8 up, 7 in 2023-10-01T09:31:18.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:18 smithi138 ceph-mon[185886]: failed to encode map e351 with expected crc 2023-10-01T09:31:18.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:18 smithi138 ceph-mon[185886]: failed to encode map e351 with expected crc 2023-10-01T09:31:18.938 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:18.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:18.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:19.046 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:31:19.076 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:19.132 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:31:19.133 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:19 smithi138 ceph-mon[192513]: pgmap v970: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 641 op/s 2023-10-01T09:31:19.135 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:19 smithi138 ceph-mon[185886]: pgmap v970: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 641 op/s 2023-10-01T09:31:19.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:19.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:31:19.242 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:19.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:19 smithi190 ceph-mon[160435]: pgmap v970: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 641 op/s 2023-10-01T09:31:19.356 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:19.527 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:19.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:19.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:19.636 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:19.665 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:19.725 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:19.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:19.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:19.831 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:19.936 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:20.029 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:31:20.029 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:31:20.029 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:31:20.029 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:31:20.029 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:31:20.029 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:31:20.030 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:31:20.030 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:31:20.030 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:31:20.030 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:31:20.030 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:31:20.030 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:31:20.030 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:31:20.030 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:31:20.030 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:31:20.030 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:31:20.038 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:20 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1081352742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:20.038 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:20 smithi190 ceph-mon[160435]: pgmap v971: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 771 op/s 2023-10-01T09:31:20.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:20 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1081352742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:20.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:20 smithi138 ceph-mon[185886]: pgmap v971: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 771 op/s 2023-10-01T09:31:20.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:20 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1081352742' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:20.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:20 smithi138 ceph-mon[192513]: pgmap v971: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 771 op/s 2023-10-01T09:31:20.474 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:20.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:20.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:20.588 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:31:20.616 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:20.665 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:31:20.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:20.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:31:20.777 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:20.802 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:20.851 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:20.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:20.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:20.956 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:31:21.011 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:31:21.055 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:21.093 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.2 to 1 (10000) 2023-10-01T09:31:21.224 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:21 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1097803922' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:31:21.224 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:21 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:31:21.224 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:21 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1097803922' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:31:21.224 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:21 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:31:21.548 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:31:21.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:21.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:31:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:21 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1097803922' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:31:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:21 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:31:21.655 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:21.681 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:21.722 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:21.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:21.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:21.827 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:21.859 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:21.911 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:21.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:21.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:22.017 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:22.113 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:22.194 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-01T09:31:22.275 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:22.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:22.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:22.329 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:22 smithi138 ceph-mon[185886]: failed to encode map e352 with expected crc 2023-10-01T09:31:22.329 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:22 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:31:22.329 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:22 smithi138 ceph-mon[185886]: osdmap e352: 8 total, 8 up, 7 in 2023-10-01T09:31:22.329 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:22 smithi138 ceph-mon[185886]: failed to encode map e352 with expected crc 2023-10-01T09:31:22.329 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:22 smithi138 ceph-mon[185886]: pgmap v973: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T09:31:22.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:22 smithi138 ceph-mon[192513]: failed to encode map e352 with expected crc 2023-10-01T09:31:22.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:22 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:31:22.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:22 smithi138 ceph-mon[192513]: osdmap e352: 8 total, 8 up, 7 in 2023-10-01T09:31:22.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:22 smithi138 ceph-mon[192513]: failed to encode map e352 with expected crc 2023-10-01T09:31:22.330 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:22 smithi138 ceph-mon[192513]: pgmap v973: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T09:31:22.388 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:31:22.518 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:31:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:31:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:31:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:31:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:109327 6.19 6:9b6cdeca:::benchmark_data_smithi138_307751_object13958:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e220)", 2023-10-01T09:31:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:21:39.601576+0000", 2023-10-01T09:31:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 582.90401274800001, 2023-10-01T09:31:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0306919560000001, 2023-10-01T09:31:22.519 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:31:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:31:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:31:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 109327 2023-10-01T09:31:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601576+0000", 2023-10-01T09:31:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601576+0000", 2023-10-01T09:31:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601574+0000", 2023-10-01T09:31:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:31:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601584+0000", 2023-10-01T09:31:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0159e-05 2023-10-01T09:31:22.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601610+0000", 2023-10-01T09:31:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5582999999999999e-05 2023-10-01T09:31:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.601621+0000", 2023-10-01T09:31:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1435e-05 2023-10-01T09:31:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.767017+0000", 2023-10-01T09:31:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16539611800000001 2023-10-01T09:31:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:31:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:39.767027+0000", 2023-10-01T09:31:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6609999999999995e-06 2023-10-01T09:31:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.616116+0000", 2023-10-01T09:31:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.84908936300000004 2023-10-01T09:31:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:31:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.616242+0000", 2023-10-01T09:31:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012627800000000001 2023-10-01T09:31:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.632186+0000", 2023-10-01T09:31:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015943382999999998 2023-10-01T09:31:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:21:40.632268+0000", 2023-10-01T09:31:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2056000000000002e-05 2023-10-01T09:31:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.527 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129799 7.4 7:2a987c36:::benchmark_data_smithi138_367058_object32004:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-10-01T09:31:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:51.394453+0000", 2023-10-01T09:31:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 271.11113571999999, 2023-10-01T09:31:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.31166278200000003, 2023-10-01T09:31:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.528 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:31:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:31:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129799 2023-10-01T09:31:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394453+0000", 2023-10-01T09:31:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394453+0000", 2023-10-01T09:31:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394451+0000", 2023-10-01T09:31:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:31:22.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394459+0000", 2023-10-01T09:31:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0700000000000007e-06 2023-10-01T09:31:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394472+0000", 2023-10-01T09:31:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2882e-05 2023-10-01T09:31:22.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394481+0000", 2023-10-01T09:31:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6340000000000007e-06 2023-10-01T09:31:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394584+0000", 2023-10-01T09:31:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010345599999999999 2023-10-01T09:31:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394716+0000", 2023-10-01T09:31:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013240800000000001 2023-10-01T09:31:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3,4", 2023-10-01T09:31:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394846+0000", 2023-10-01T09:31:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012953300000000001 2023-10-01T09:31:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.408545+0000", 2023-10-01T09:31:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013699413000000001 2023-10-01T09:31:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.408614+0000", 2023-10-01T09:31:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8595000000000003e-05 2023-10-01T09:31:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.636954+0000", 2023-10-01T09:31:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22833962399999999 2023-10-01T09:31:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.705924+0000", 2023-10-01T09:31:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.068970630000000005 2023-10-01T09:31:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.705984+0000", 2023-10-01T09:31:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0136999999999997e-05 2023-10-01T09:31:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.706116+0000", 2023-10-01T09:31:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013135999999999999 2023-10-01T09:31:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.536 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129803 7.4 7:21dfca7d:::benchmark_data_smithi138_367058_object32008:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-10-01T09:31:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:51.394498+0000", 2023-10-01T09:31:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 271.11109111100001, 2023-10-01T09:31:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.31185276299999998, 2023-10-01T09:31:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.537 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:31:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:31:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129803 2023-10-01T09:31:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394498+0000", 2023-10-01T09:31:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394498+0000", 2023-10-01T09:31:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394497+0000", 2023-10-01T09:31:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394500+0000", 2023-10-01T09:31:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5519999999999999e-06 2023-10-01T09:31:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394506+0000", 2023-10-01T09:31:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0399999999999998e-06 2023-10-01T09:31:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.394511+0000", 2023-10-01T09:31:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8189999999999999e-06 2023-10-01T09:31:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.397846+0000", 2023-10-01T09:31:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003335363 2023-10-01T09:31:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.397980+0000", 2023-10-01T09:31:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013349100000000001 2023-10-01T09:31:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3,4", 2023-10-01T09:31:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.398108+0000", 2023-10-01T09:31:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012803200000000001 2023-10-01T09:31:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.408655+0000", 2023-10-01T09:31:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010547575999999999 2023-10-01T09:31:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.408665+0000", 2023-10-01T09:31:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7310000000000005e-06 2023-10-01T09:31:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.637002+0000", 2023-10-01T09:31:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22833692 2023-10-01T09:31:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.706176+0000", 2023-10-01T09:31:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.069173945000000001 2023-10-01T09:31:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.706243+0000", 2023-10-01T09:31:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7479999999999998e-05 2023-10-01T09:31:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.706350+0000", 2023-10-01T09:31:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107153 2023-10-01T09:31:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.545 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129812 7.4 7:24dcedab:::benchmark_data_smithi138_367058_object32017:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-10-01T09:31:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:51.411647+0000", 2023-10-01T09:31:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 271.09394190400002, 2023-10-01T09:31:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.31493354499999998, 2023-10-01T09:31:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.546 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:31:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:31:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129812 2023-10-01T09:31:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411647+0000", 2023-10-01T09:31:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411647+0000", 2023-10-01T09:31:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411645+0000", 2023-10-01T09:31:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:31:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411655+0000", 2023-10-01T09:31:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0538e-05 2023-10-01T09:31:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411670+0000", 2023-10-01T09:31:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4566000000000001e-05 2023-10-01T09:31:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411680+0000", 2023-10-01T09:31:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9380000000000004e-06 2023-10-01T09:31:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411729+0000", 2023-10-01T09:31:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9330999999999999e-05 2023-10-01T09:31:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.411864+0000", 2023-10-01T09:31:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013525899999999999 2023-10-01T09:31:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,3,4", 2023-10-01T09:31:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.412012+0000", 2023-10-01T09:31:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014787900000000001 2023-10-01T09:31:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.636906+0000", 2023-10-01T09:31:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22489326600000001 2023-10-01T09:31:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.637082+0000", 2023-10-01T09:31:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000176603 2023-10-01T09:31:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.637093+0000", 2023-10-01T09:31:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0428000000000001e-05 2023-10-01T09:31:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.726416+0000", 2023-10-01T09:31:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.089322953999999996 2023-10-01T09:31:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.726470+0000", 2023-10-01T09:31:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4666999999999997e-05 2023-10-01T09:31:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:51.726580+0000", 2023-10-01T09:31:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110179 2023-10-01T09:31:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.554 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:142310 7.f 7:fadc5338:::benchmark_data_smithi138_367058_object44515:head [delete] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T09:31:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:27:20.468221+0000", 2023-10-01T09:31:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 242.03736756500001, 2023-10-01T09:31:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.34216545700000001, 2023-10-01T09:31:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.555 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:31:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:31:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 142310 2023-10-01T09:31:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.468221+0000", 2023-10-01T09:31:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.468221+0000", 2023-10-01T09:31:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.468220+0000", 2023-10-01T09:31:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:31:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.558 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.468227+0000", 2023-10-01T09:31:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1670000000000002e-06 2023-10-01T09:31:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.468239+0000", 2023-10-01T09:31:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1844999999999999e-05 2023-10-01T09:31:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.468246+0000", 2023-10-01T09:31:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.255e-06 2023-10-01T09:31:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.471974+0000", 2023-10-01T09:31:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003727756 2023-10-01T09:31:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.472092+0000", 2023-10-01T09:31:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118267 2023-10-01T09:31:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:31:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.472201+0000", 2023-10-01T09:31:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109379 2023-10-01T09:31:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478845+0000", 2023-10-01T09:31:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066437090000000002 2023-10-01T09:31:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.479233+0000", 2023-10-01T09:31:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00038774099999999999 2023-10-01T09:31:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.810288+0000", 2023-10-01T09:31:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33105508900000002 2023-10-01T09:31:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.810341+0000", 2023-10-01T09:31:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3573999999999999e-05 2023-10-01T09:31:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.810387+0000", 2023-10-01T09:31:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5324000000000002e-05 2023-10-01T09:31:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.564 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:142313 7.6 7:6a412c14:::benchmark_data_smithi138_367058_object44518:head [delete] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T09:31:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:27:20.474654+0000", 2023-10-01T09:31:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 242.03093429500001, 2023-10-01T09:31:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33643319399999999, 2023-10-01T09:31:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.565 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:31:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:31:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 142313 2023-10-01T09:31:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474654+0000", 2023-10-01T09:31:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474654+0000", 2023-10-01T09:31:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474653+0000", 2023-10-01T09:31:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:31:22.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474660+0000", 2023-10-01T09:31:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.939e-06 2023-10-01T09:31:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474674+0000", 2023-10-01T09:31:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.437e-05 2023-10-01T09:31:22.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474682+0000", 2023-10-01T09:31:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6620000000000005e-06 2023-10-01T09:31:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474732+0000", 2023-10-01T09:31:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0380999999999998e-05 2023-10-01T09:31:22.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474860+0000", 2023-10-01T09:31:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012723500000000001 2023-10-01T09:31:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:31:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.474974+0000", 2023-10-01T09:31:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011427899999999999 2023-10-01T09:31:22.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478152+0000", 2023-10-01T09:31:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031778779999999999 2023-10-01T09:31:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478781+0000", 2023-10-01T09:31:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00062936100000000005 2023-10-01T09:31:22.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.811011+0000", 2023-10-01T09:31:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33222958899999999 2023-10-01T09:31:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.811054+0000", 2023-10-01T09:31:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3208999999999999e-05 2023-10-01T09:31:22.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.811088+0000", 2023-10-01T09:31:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3680000000000003e-05 2023-10-01T09:31:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:142315 7.6 7:62a92d28:::benchmark_data_smithi138_367058_object44520:head [delete] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T09:31:22.573 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:27:20.478706+0000", 2023-10-01T09:31:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 242.02688249400001, 2023-10-01T09:31:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.33247613999999998, 2023-10-01T09:31:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:31:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:31:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 142315 2023-10-01T09:31:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.574 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478706+0000", 2023-10-01T09:31:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478706+0000", 2023-10-01T09:31:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478705+0000", 2023-10-01T09:31:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:31:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478711+0000", 2023-10-01T09:31:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3049999999999998e-06 2023-10-01T09:31:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478723+0000", 2023-10-01T09:31:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1358999999999999e-05 2023-10-01T09:31:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478730+0000", 2023-10-01T09:31:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.464e-06 2023-10-01T09:31:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478802+0000", 2023-10-01T09:31:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1975000000000001e-05 2023-10-01T09:31:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478895+0000", 2023-10-01T09:31:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3499999999999996e-05 2023-10-01T09:31:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:31:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.478976+0000", 2023-10-01T09:31:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0068000000000007e-05 2023-10-01T09:31:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.481966+0000", 2023-10-01T09:31:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029905190000000001 2023-10-01T09:31:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.482079+0000", 2023-10-01T09:31:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011309799999999999 2023-10-01T09:31:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.811120+0000", 2023-10-01T09:31:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.32904090000000003 2023-10-01T09:31:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.811153+0000", 2023-10-01T09:31:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3448000000000002e-05 2023-10-01T09:31:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:27:20.811182+0000", 2023-10-01T09:31:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8864e-05 2023-10-01T09:31:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.581 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:194222 7.7 7:e92362a8:::benchmark_data_smithi138_367058_object96427:head [delete] snapc 0=[] ondisk+write+known_if_redirected e317)", 2023-10-01T09:31:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:28:56.358219+0000", 2023-10-01T09:31:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 146.14736951899999, 2023-10-01T09:31:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24070214600000001, 2023-10-01T09:31:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.582 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:31:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:31:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 194222 2023-10-01T09:31:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358219+0000", 2023-10-01T09:31:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358219+0000", 2023-10-01T09:31:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358219+0000", 2023-10-01T09:31:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:22.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358221+0000", 2023-10-01T09:31:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4499999999999998e-06 2023-10-01T09:31:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358228+0000", 2023-10-01T09:31:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1359999999999999e-06 2023-10-01T09:31:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358233+0000", 2023-10-01T09:31:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5299999999999998e-06 2023-10-01T09:31:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358294+0000", 2023-10-01T09:31:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0801999999999998e-05 2023-10-01T09:31:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358410+0000", 2023-10-01T09:31:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011609599999999999 2023-10-01T09:31:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:31:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358529+0000", 2023-10-01T09:31:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000119413 2023-10-01T09:31:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.362274+0000", 2023-10-01T09:31:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037445040000000001 2023-10-01T09:31:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.369771+0000", 2023-10-01T09:31:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0074968869999999998 2023-10-01T09:31:22.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.598612+0000", 2023-10-01T09:31:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.228840878 2023-10-01T09:31:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.598656+0000", 2023-10-01T09:31:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4131999999999997e-05 2023-10-01T09:31:22.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.598921+0000", 2023-10-01T09:31:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00026565900000000001 2023-10-01T09:31:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.590 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:194226 7.7 7:e60142e1:::benchmark_data_smithi138_367058_object96431:head [delete] snapc 0=[] ondisk+write+known_if_redirected e317)", 2023-10-01T09:31:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:28:56.358246+0000", 2023-10-01T09:31:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 146.14734319900001, 2023-10-01T09:31:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24091088999999999, 2023-10-01T09:31:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:31:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:31:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 194226 2023-10-01T09:31:22.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358246+0000", 2023-10-01T09:31:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358246+0000", 2023-10-01T09:31:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358245+0000", 2023-10-01T09:31:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358247+0000", 2023-10-01T09:31:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2230000000000001e-06 2023-10-01T09:31:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358253+0000", 2023-10-01T09:31:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.395e-06 2023-10-01T09:31:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.358257+0000", 2023-10-01T09:31:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.16e-06 2023-10-01T09:31:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.364723+0000", 2023-10-01T09:31:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064660810000000003 2023-10-01T09:31:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.364848+0000", 2023-10-01T09:31:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012459 2023-10-01T09:31:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:31:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.364974+0000", 2023-10-01T09:31:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012670499999999999 2023-10-01T09:31:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:22.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.368254+0000", 2023-10-01T09:31:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032795290000000002 2023-10-01T09:31:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.369807+0000", 2023-10-01T09:31:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001552912 2023-10-01T09:31:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.598967+0000", 2023-10-01T09:31:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22916010000000001 2023-10-01T09:31:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.599018+0000", 2023-10-01T09:31:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1039999999999999e-05 2023-10-01T09:31:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:28:56.599156+0000", 2023-10-01T09:31:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013849399999999999 2023-10-01T09:31:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.599 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:89391 8.c 8:39281457:::benchmark_data_smithi138_431617_object89390:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-01T09:31:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:30:46.701767+0000", 2023-10-01T09:31:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 35.803822082000003, 2023-10-01T09:31:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24559731400000001, 2023-10-01T09:31:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 89391 2023-10-01T09:31:22.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.701767+0000", 2023-10-01T09:31:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.701767+0000", 2023-10-01T09:31:22.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.701765+0000", 2023-10-01T09:31:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:31:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.701817+0000", 2023-10-01T09:31:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2172000000000001e-05 2023-10-01T09:31:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.701828+0000", 2023-10-01T09:31:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0812000000000001e-05 2023-10-01T09:31:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.701835+0000", 2023-10-01T09:31:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.72e-06 2023-10-01T09:31:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.701880+0000", 2023-10-01T09:31:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5198999999999999e-05 2023-10-01T09:31:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.702027+0000", 2023-10-01T09:31:22.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014672599999999999 2023-10-01T09:31:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:31:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.702171+0000", 2023-10-01T09:31:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014410700000000001 2023-10-01T09:31:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.705350+0000", 2023-10-01T09:31:22.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031794979999999998 2023-10-01T09:31:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.943442+0000", 2023-10-01T09:31:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.238091889 2023-10-01T09:31:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.947280+0000", 2023-10-01T09:31:22.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0038380380000000002 2023-10-01T09:31:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.947324+0000", 2023-10-01T09:31:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3683999999999997e-05 2023-10-01T09:31:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.947364+0000", 2023-10-01T09:31:22.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0219000000000002e-05 2023-10-01T09:31:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:89387 8.5 e344/326 8:ac2bd163:::benchmark_data_smithi138_431617_object89386:head v 344'5637, mlcod=344'5635)", 2023-10-01T09:31:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:30:46.702181+0000", 2023-10-01T09:31:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 35.803408028, 2023-10-01T09:31:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.241234644, 2023-10-01T09:31:22.608 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.702181+0000", 2023-10-01T09:31:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.702181+0000", 2023-10-01T09:31:22.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.702180+0000", 2023-10-01T09:31:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:31:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.702299+0000", 2023-10-01T09:31:22.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011935200000000001 2023-10-01T09:31:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.702310+0000", 2023-10-01T09:31:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1195e-05 2023-10-01T09:31:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.702317+0000", 2023-10-01T09:31:22.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0099999999999998e-06 2023-10-01T09:31:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.706419+0000", 2023-10-01T09:31:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041017149999999997 2023-10-01T09:31:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.706440+0000", 2023-10-01T09:31:22.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1189e-05 2023-10-01T09:31:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.943376+0000", 2023-10-01T09:31:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23693572800000001 2023-10-01T09:31:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.943415+0000", 2023-10-01T09:31:22.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9240999999999999e-05 2023-10-01T09:31:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:89396 8.6 8:64a579e5:::benchmark_data_smithi138_431617_object89395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-01T09:31:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:30:46.709135+0000", 2023-10-01T09:31:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 35.796453931999999, 2023-10-01T09:31:22.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24098256800000001, 2023-10-01T09:31:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 89396 2023-10-01T09:31:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709135+0000", 2023-10-01T09:31:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709135+0000", 2023-10-01T09:31:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709133+0000", 2023-10-01T09:31:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:31:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709156+0000", 2023-10-01T09:31:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3039000000000001e-05 2023-10-01T09:31:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709171+0000", 2023-10-01T09:31:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4724e-05 2023-10-01T09:31:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709178+0000", 2023-10-01T09:31:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0480000000000001e-06 2023-10-01T09:31:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709239+0000", 2023-10-01T09:31:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1198999999999995e-05 2023-10-01T09:31:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709382+0000", 2023-10-01T09:31:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014248900000000001 2023-10-01T09:31:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 6,7", 2023-10-01T09:31:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709529+0000", 2023-10-01T09:31:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014745399999999999 2023-10-01T09:31:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.946557+0000", 2023-10-01T09:31:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23702794599999999 2023-10-01T09:31:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.949992+0000", 2023-10-01T09:31:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003434856 2023-10-01T09:31:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.950030+0000", 2023-10-01T09:31:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8444e-05 2023-10-01T09:31:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.950077+0000", 2023-10-01T09:31:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7086000000000003e-05 2023-10-01T09:31:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.950117+0000", 2023-10-01T09:31:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9996999999999997e-05 2023-10-01T09:31:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.622 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:89397 8.e 8:71d5e78a:::benchmark_data_smithi138_431617_object89396:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-01T09:31:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:30:46.709193+0000", 2023-10-01T09:31:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 35.796395326000003, 2023-10-01T09:31:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24594287500000001, 2023-10-01T09:31:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.623 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 89397 2023-10-01T09:31:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709193+0000", 2023-10-01T09:31:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709193+0000", 2023-10-01T09:31:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709193+0000", 2023-10-01T09:31:22.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709231+0000", 2023-10-01T09:31:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8473000000000002e-05 2023-10-01T09:31:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709239+0000", 2023-10-01T09:31:22.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6660000000000008e-06 2023-10-01T09:31:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709244+0000", 2023-10-01T09:31:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1950000000000002e-06 2023-10-01T09:31:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709281+0000", 2023-10-01T09:31:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6990999999999999e-05 2023-10-01T09:31:22.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709394+0000", 2023-10-01T09:31:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113157 2023-10-01T09:31:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-10-01T09:31:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.709587+0000", 2023-10-01T09:31:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000192239 2023-10-01T09:31:22.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.949998+0000", 2023-10-01T09:31:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24041110500000001 2023-10-01T09:31:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.955027+0000", 2023-10-01T09:31:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050295369999999997 2023-10-01T09:31:22.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.955058+0000", 2023-10-01T09:31:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0368000000000001e-05 2023-10-01T09:31:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.955101+0000", 2023-10-01T09:31:22.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3828000000000001e-05 2023-10-01T09:31:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:46.955136+0000", 2023-10-01T09:31:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4813e-05 2023-10-01T09:31:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:98805 8.e 8:77edef46:::benchmark_data_smithi138_431617_object98804:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-01T09:31:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:30:59.489388+0000", 2023-10-01T09:31:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 23.016200763000001, 2023-10-01T09:31:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27872444200000002, 2023-10-01T09:31:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:22.632 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 98805 2023-10-01T09:31:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489388+0000", 2023-10-01T09:31:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489388+0000", 2023-10-01T09:31:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489386+0000", 2023-10-01T09:31:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:31:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489429+0000", 2023-10-01T09:31:22.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3362999999999998e-05 2023-10-01T09:31:22.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489441+0000", 2023-10-01T09:31:22.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1618e-05 2023-10-01T09:31:22.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489449+0000", 2023-10-01T09:31:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9370000000000004e-06 2023-10-01T09:31:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489542+0000", 2023-10-01T09:31:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3416000000000001e-05 2023-10-01T09:31:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489706+0000", 2023-10-01T09:31:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016358099999999999 2023-10-01T09:31:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-10-01T09:31:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489913+0000", 2023-10-01T09:31:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020767 2023-10-01T09:31:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.496860+0000", 2023-10-01T09:31:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069463939999999998 2023-10-01T09:31:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.764680+0000", 2023-10-01T09:31:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26782008200000001 2023-10-01T09:31:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.768039+0000", 2023-10-01T09:31:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003358944 2023-10-01T09:31:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.768079+0000", 2023-10-01T09:31:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0182999999999998e-05 2023-10-01T09:31:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.768112+0000", 2023-10-01T09:31:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3417999999999998e-05 2023-10-01T09:31:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:98806 8.2 8:4e84589e:::benchmark_data_smithi138_431617_object98805:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-01T09:31:22.640 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:30:59.489463+0000", 2023-10-01T09:31:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 23.016126092, 2023-10-01T09:31:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27864754400000002, 2023-10-01T09:31:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 98806 2023-10-01T09:31:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.641 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489463+0000", 2023-10-01T09:31:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489463+0000", 2023-10-01T09:31:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489462+0000", 2023-10-01T09:31:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489484+0000", 2023-10-01T09:31:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1795000000000001e-05 2023-10-01T09:31:22.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489493+0000", 2023-10-01T09:31:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0210000000000005e-06 2023-10-01T09:31:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489499+0000", 2023-10-01T09:31:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.657e-06 2023-10-01T09:31:22.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489542+0000", 2023-10-01T09:31:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2355999999999999e-05 2023-10-01T09:31:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489706+0000", 2023-10-01T09:31:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016383000000000001 2023-10-01T09:31:22.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-10-01T09:31:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.489843+0000", 2023-10-01T09:31:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013694999999999999 2023-10-01T09:31:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.496848+0000", 2023-10-01T09:31:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0070051239999999997 2023-10-01T09:31:22.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.767991+0000", 2023-10-01T09:31:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27114338100000002 2023-10-01T09:31:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.768031+0000", 2023-10-01T09:31:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9972999999999998e-05 2023-10-01T09:31:22.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.768078+0000", 2023-10-01T09:31:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6616000000000002e-05 2023-10-01T09:31:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.768110+0000", 2023-10-01T09:31:22.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2511000000000001e-05 2023-10-01T09:31:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:98812 8.2 8:43a2ed4b:::benchmark_data_smithi138_431617_object98811:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-01T09:31:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:30:59.496922+0000", 2023-10-01T09:31:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 23.008666974, 2023-10-01T09:31:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27576615999999998, 2023-10-01T09:31:22.649 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 98812 2023-10-01T09:31:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.496922+0000", 2023-10-01T09:31:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.496922+0000", 2023-10-01T09:31:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.496920+0000", 2023-10-01T09:31:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:31:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.496952+0000", 2023-10-01T09:31:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1615999999999997e-05 2023-10-01T09:31:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.496963+0000", 2023-10-01T09:31:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1406000000000001e-05 2023-10-01T09:31:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.496971+0000", 2023-10-01T09:31:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4719999999999998e-06 2023-10-01T09:31:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.497112+0000", 2023-10-01T09:31:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000141521 2023-10-01T09:31:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.764644+0000", 2023-10-01T09:31:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26753147700000002 2023-10-01T09:31:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-10-01T09:31:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.764819+0000", 2023-10-01T09:31:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017522499999999999 2023-10-01T09:31:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.771985+0000", 2023-10-01T09:31:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0071666029999999997 2023-10-01T09:31:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.772016+0000", 2023-10-01T09:31:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0796999999999997e-05 2023-10-01T09:31:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.772597+0000", 2023-10-01T09:31:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00058093100000000003 2023-10-01T09:31:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.772648+0000", 2023-10-01T09:31:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0447e-05 2023-10-01T09:31:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.772688+0000", 2023-10-01T09:31:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0240000000000001e-05 2023-10-01T09:31:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.657 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.658 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:98811 8.5 e347/326 8:aea6ced9:::benchmark_data_smithi138_431617_object98810:head v 347'6225, mlcod=347'6223)", 2023-10-01T09:31:22.658 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:30:59.496999+0000", 2023-10-01T09:31:22.658 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 23.008590013999999, 2023-10-01T09:31:22.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27603860000000002, 2023-10-01T09:31:22.658 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.658 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:22.658 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.496999+0000", 2023-10-01T09:31:22.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.496999+0000", 2023-10-01T09:31:22.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.496998+0000", 2023-10-01T09:31:22.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:22.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.497035+0000", 2023-10-01T09:31:22.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7058000000000003e-05 2023-10-01T09:31:22.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.497043+0000", 2023-10-01T09:31:22.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8800000000000008e-06 2023-10-01T09:31:22.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.497049+0000", 2023-10-01T09:31:22.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9270000000000002e-06 2023-10-01T09:31:22.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.764639+0000", 2023-10-01T09:31:22.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26759045999999997 2023-10-01T09:31:22.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.764670+0000", 2023-10-01T09:31:22.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0951000000000002e-05 2023-10-01T09:31:22.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:22.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.773007+0000", 2023-10-01T09:31:22.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0083370170000000004 2023-10-01T09:31:22.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.773037+0000", 2023-10-01T09:31:22.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0014000000000001e-05 2023-10-01T09:31:22.664 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.664 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.664 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.664 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.45674.0:98805 8.e e347/326 ondisk, result = 0)", 2023-10-01T09:31:22.664 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:30:59.499411+0000", 2023-10-01T09:31:22.664 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 23.006177645000001, 2023-10-01T09:31:22.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26527490799999998, 2023-10-01T09:31:22.664 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.665 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:31:22.665 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.499411+0000", 2023-10-01T09:31:22.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.499411+0000", 2023-10-01T09:31:22.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.499410+0000", 2023-10-01T09:31:22.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:31:22.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.499415+0000", 2023-10-01T09:31:22.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4399999999999998e-06 2023-10-01T09:31:22.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.764543+0000", 2023-10-01T09:31:22.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.265128525 2023-10-01T09:31:22.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.764567+0000", 2023-10-01T09:31:22.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.419e-05 2023-10-01T09:31:22.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.764662+0000", 2023-10-01T09:31:22.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4396999999999998e-05 2023-10-01T09:31:22.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.764672+0000", 2023-10-01T09:31:22.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0087e-05 2023-10-01T09:31:22.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.764686+0000", 2023-10-01T09:31:22.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4133e-05 2023-10-01T09:31:22.669 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.669 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.669 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.670 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.45674.0:98806 8.2 e347/326 ondisk, result = 0)", 2023-10-01T09:31:22.670 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:30:59.500836+0000", 2023-10-01T09:31:22.670 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 23.004752761999999, 2023-10-01T09:31:22.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26728137600000001, 2023-10-01T09:31:22.670 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.670 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:31:22.670 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.500836+0000", 2023-10-01T09:31:22.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.500836+0000", 2023-10-01T09:31:22.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.500835+0000", 2023-10-01T09:31:22.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:31:22.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.500844+0000", 2023-10-01T09:31:22.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7539999999999995e-06 2023-10-01T09:31:22.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.764537+0000", 2023-10-01T09:31:22.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26369364299999998 2023-10-01T09:31:22.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.764554+0000", 2023-10-01T09:31:22.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6212999999999999e-05 2023-10-01T09:31:22.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.768020+0000", 2023-10-01T09:31:22.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034665830000000001 2023-10-01T09:31:22.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.768026+0000", 2023-10-01T09:31:22.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9830000000000001e-06 2023-10-01T09:31:22.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.768117+0000", 2023-10-01T09:31:22.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1079e-05 2023-10-01T09:31:22.675 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.675 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.675 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.675 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.45674.0:98806 8.2 e347/326 ondisk, result = 0)", 2023-10-01T09:31:22.675 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:30:59.502142+0000", 2023-10-01T09:31:22.675 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 23.00344625, 2023-10-01T09:31:22.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26585477299999999, 2023-10-01T09:31:22.676 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:22.676 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:31:22.676 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:22.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:22.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.502142+0000", 2023-10-01T09:31:22.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:22.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.502142+0000", 2023-10-01T09:31:22.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:22.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:22.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.502142+0000", 2023-10-01T09:31:22.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:31:22.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:22.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.502148+0000", 2023-10-01T09:31:22.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2910000000000004e-06 2023-10-01T09:31:22.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:22.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.764523+0000", 2023-10-01T09:31:22.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26237493499999998 2023-10-01T09:31:22.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:22.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.764552+0000", 2023-10-01T09:31:22.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9275999999999999e-05 2023-10-01T09:31:22.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:22.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.767979+0000", 2023-10-01T09:31:22.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003426859 2023-10-01T09:31:22.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:22.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.767986+0000", 2023-10-01T09:31:22.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7839999999999998e-06 2023-10-01T09:31:22.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:22.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:22.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:22.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:30:59.767997+0000", 2023-10-01T09:31:22.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1396999999999999e-05 2023-10-01T09:31:22.680 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.680 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.680 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.680 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:22.681 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:22.681 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:31:22.681 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:22.681 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:22 smithi190 ceph-mon[160435]: failed to encode map e352 with expected crc 2023-10-01T09:31:22.682 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:22 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:31:22.682 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:22 smithi190 ceph-mon[160435]: osdmap e352: 8 total, 8 up, 7 in 2023-10-01T09:31:22.682 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:22 smithi190 ceph-mon[160435]: failed to encode map e352 with expected crc 2023-10-01T09:31:22.682 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:22 smithi190 ceph-mon[160435]: pgmap v973: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T09:31:22.797 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:31:22.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:22.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:31:22.901 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:31:22.929 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:23.254 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:31:23.337 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:31:23.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:23.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:31:23.446 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:31:23.529 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:23.561 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:31:23.696 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:31:23.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:23.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:31:23.801 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:23.830 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:23.886 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:23.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:23.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:23.968 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:23.991 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:31:24.023 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:24.077 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:31:24.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:24.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:31:24.185 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:31:24.264 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:25.034 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:31:25.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:25.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:31:25.143 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:25.155 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:25.175 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:25.208 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:25 smithi138 ceph-mon[185886]: pgmap v974: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 786 op/s 2023-10-01T09:31:25.208 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:25 smithi138 ceph-mon[192513]: pgmap v974: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 786 op/s 2023-10-01T09:31:25.209 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:31:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:31:24] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.33.4" 2023-10-01T09:31:25.223 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:25.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:25.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:25.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:25 smithi190 ceph-mon[160435]: pgmap v974: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 786 op/s 2023-10-01T09:31:25.332 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:31:25.429 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:25.684 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:31:25.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:25.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:31:25.793 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:25.843 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:25.943 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:25.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:25.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:26.018 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:31:26.053 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:31:26.123 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:26.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:31:26.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:31:26.584 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:31:26.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:26.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:31:26.698 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:26.713 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:26.782 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:27.042 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:27.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:27.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:27.152 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:27.217 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:27 smithi138 ceph-mon[192513]: pgmap v975: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 797 op/s 2023-10-01T09:31:27.217 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:27 smithi138 ceph-mon[185886]: pgmap v975: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 797 op/s 2023-10-01T09:31:27.225 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:27.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:27 smithi190 ceph-mon[160435]: pgmap v975: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 797 op/s 2023-10-01T09:31:28.022 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:28.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:28.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:28.137 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:31:28.173 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:28.242 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:31:28.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:28.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:31:28.348 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:31:28.382 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:28.443 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:31:28.445 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3238950049' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:31:28.445 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:28 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:31:28.445 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:28 smithi190 ceph-mon[160435]: pgmap v976: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T09:31:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:28.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:31:28.483 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:31:28.551 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:28.579 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:28.653 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:28.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:28.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:28.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3238950049' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:31:28.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:28 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:31:28.732 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:28 smithi138 ceph-mon[185886]: pgmap v976: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T09:31:28.733 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3238950049' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:31:28.733 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:28 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:31:28.733 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:28 smithi138 ceph-mon[192513]: pgmap v976: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T09:31:28.759 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:28.814 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:29.471 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:29.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:29 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:31:29.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:29 smithi138 ceph-mon[192513]: failed to encode map e353 with expected crc 2023-10-01T09:31:29.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:29 smithi138 ceph-mon[192513]: failed to encode map e353 with expected crc 2023-10-01T09:31:29.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:29 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:31:29.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:29 smithi138 ceph-mon[192513]: osdmap e353: 8 total, 8 up, 7 in 2023-10-01T09:31:29.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:29 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/167937600' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:31:29.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:29 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:31:29.473 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:29 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:31:29.473 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:29 smithi138 ceph-mon[185886]: failed to encode map e353 with expected crc 2023-10-01T09:31:29.474 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:29 smithi138 ceph-mon[185886]: failed to encode map e353 with expected crc 2023-10-01T09:31:29.474 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:29 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:31:29.474 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:29 smithi138 ceph-mon[185886]: osdmap e353: 8 total, 8 up, 7 in 2023-10-01T09:31:29.474 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:29 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/167937600' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:31:29.474 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:29 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:31:29.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:29.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:29.495 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.5 to 1 (10000) 2023-10-01T09:31:29.589 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:29.681 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:29.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:29 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:31:29.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:29 smithi190 ceph-mon[160435]: failed to encode map e353 with expected crc 2023-10-01T09:31:29.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:29 smithi190 ceph-mon[160435]: failed to encode map e353 with expected crc 2023-10-01T09:31:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:29 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:31:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:29 smithi190 ceph-mon[160435]: osdmap e353: 8 total, 8 up, 7 in 2023-10-01T09:31:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:29 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/167937600' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:31:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:29 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:31:29.964 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:29.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:29.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:30.071 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:31:30.125 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:30.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:30 smithi138 ceph-mon[192513]: failed to encode map e354 with expected crc 2023-10-01T09:31:30.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:30 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:31:30.474 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:30 smithi138 ceph-mon[185886]: failed to encode map e354 with expected crc 2023-10-01T09:31:30.474 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:30 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:31:30.474 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:30 smithi138 ceph-mon[185886]: osdmap e354: 8 total, 8 up, 7 in 2023-10-01T09:31:30.474 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:30 smithi138 ceph-mon[185886]: failed to encode map e354 with expected crc 2023-10-01T09:31:30.474 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:30 smithi138 ceph-mon[185886]: pgmap v979: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 770 op/s 2023-10-01T09:31:30.553 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:31:30.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:30.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:31:30.667 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:30.695 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:30.702 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-01T09:31:30.748 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:30.751 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:30 smithi190 ceph-mon[160435]: failed to encode map e354 with expected crc 2023-10-01T09:31:30.751 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:30 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:31:30.751 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:30 smithi190 ceph-mon[160435]: osdmap e354: 8 total, 8 up, 7 in 2023-10-01T09:31:30.751 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:30 smithi190 ceph-mon[160435]: failed to encode map e354 with expected crc 2023-10-01T09:31:30.752 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:30 smithi190 ceph-mon[160435]: pgmap v979: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 770 op/s 2023-10-01T09:31:30.752 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:30 smithi138 ceph-mon[192513]: osdmap e354: 8 total, 8 up, 7 in 2023-10-01T09:31:30.752 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:30 smithi138 ceph-mon[192513]: failed to encode map e354 with expected crc 2023-10-01T09:31:30.752 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:30 smithi138 ceph-mon[192513]: pgmap v979: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 770 op/s 2023-10-01T09:31:30.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:30.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:30.830 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:31:30.854 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:31:30.886 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:30.894 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:31:30.940 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:31:30.940 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:31:30.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:30.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:31:30.981 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:31:31.009 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:31:31.047 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:31.048 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:31:31.087 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:31.092 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:31:31.123 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:31:31.123 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:31:31.127 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:31:31.132 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:31:31.132 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:31:31.132 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:31:31.132 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:31:31.132 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:31:31.132 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:31:31.132 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:31:31.132 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:31:31.158 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:31.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:31.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:31.269 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:31.363 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:31.481 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/382641212' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:31.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/382641212' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:31.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/382641212' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:31.875 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:31:32.187 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:32.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:32.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:32.292 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:31:32.328 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:32.399 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:31:32.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:32.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:31:32.508 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:31:33.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:32 smithi190 ceph-mon[160435]: pgmap v980: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 612 op/s 2023-10-01T09:31:33.066 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:31:33.066 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:32 smithi138 ceph-mon[192513]: pgmap v980: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 612 op/s 2023-10-01T09:31:33.066 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:32 smithi138 ceph-mon[185886]: pgmap v980: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 612 op/s 2023-10-01T09:31:33.067 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:33.091 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:33.169 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:33.282 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:31:33.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:33.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:31:33.394 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:33.425 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:33.481 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:33.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:33.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:33.593 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:31:33.689 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:34.828 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:31:34.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:34.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:31:34.938 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:34.987 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:35.068 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:31:35.116 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:35 smithi138 ceph-mon[185886]: pgmap v981: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 771 op/s 2023-10-01T09:31:35.116 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:35 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:31:35.116 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:35 smithi138 ceph-mon[192513]: pgmap v981: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 771 op/s 2023-10-01T09:31:35.116 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:35 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:31:35.119 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:31:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:31:34] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.33.4" 2023-10-01T09:31:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:35 smithi190 ceph-mon[160435]: pgmap v981: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 771 op/s 2023-10-01T09:31:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:35 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:31:35.372 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:35.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:35.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:35.481 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:31:35.533 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:35.599 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:31:35.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:35.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:31:35.706 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:31:35.800 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:35.993 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:36.145 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:31:36.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:36.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:31:36.262 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:31:36.302 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:36.374 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:31:36.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:36.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:31:36.482 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:31:36.575 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:36.661 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:36.697 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:31:36.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:36.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:31:36.803 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:36.846 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:36.916 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:36.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:36.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:37.023 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:31:37.050 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:37 smithi138 ceph-mon[185886]: pgmap v982: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 870 op/s 2023-10-01T09:31:37.050 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:37 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1644618969' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:31:37.050 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:37 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/39233171' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:31:37.059 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:31:37.071 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.6 to 1 (10000) 2023-10-01T09:31:37.085 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:37.205 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:31:37.214 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:37 smithi190 ceph-mon[160435]: pgmap v982: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 870 op/s 2023-10-01T09:31:37.214 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:37 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1644618969' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:31:37.214 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:37 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/39233171' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:31:37.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:37.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:31:37.316 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:31:37.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:37 smithi138 ceph-mon[192513]: pgmap v982: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 870 op/s 2023-10-01T09:31:37.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:37 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1644618969' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:31:37.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:37 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/39233171' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:31:37.373 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:38 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:31:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:38 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:31:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:38 smithi190 ceph-mon[160435]: failed to encode map e355 with expected crc 2023-10-01T09:31:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1644618969' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:31:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/39233171' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:31:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:38 smithi190 ceph-mon[160435]: osdmap e355: 8 total, 8 up, 7 in 2023-10-01T09:31:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:38 smithi190 ceph-mon[160435]: failed to encode map e355 with expected crc 2023-10-01T09:31:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:38 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:38 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:38 smithi190 ceph-mon[160435]: pgmap v984: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 564 op/s 2023-10-01T09:31:38.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:31:38.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:31:38.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[185886]: failed to encode map e355 with expected crc 2023-10-01T09:31:38.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1644618969' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:31:38.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/39233171' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:31:38.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[185886]: osdmap e355: 8 total, 8 up, 7 in 2023-10-01T09:31:38.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[185886]: failed to encode map e355 with expected crc 2023-10-01T09:31:38.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:38.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:38.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[185886]: pgmap v984: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 564 op/s 2023-10-01T09:31:38.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:31:38.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:31:38.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[192513]: failed to encode map e355 with expected crc 2023-10-01T09:31:38.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1644618969' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:31:38.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/39233171' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:31:38.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[192513]: osdmap e355: 8 total, 8 up, 7 in 2023-10-01T09:31:38.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[192513]: failed to encode map e355 with expected crc 2023-10-01T09:31:38.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:38.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:38.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:38 smithi138 ceph-mon[192513]: pgmap v984: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 564 op/s 2023-10-01T09:31:38.571 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:31:38.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:38.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:31:38.681 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:38.736 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:39.188 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-01T09:31:39.189 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:31:39.497 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:39.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:39.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:39.605 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:39.638 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:39.707 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:39.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:39.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:39.816 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:31:39.929 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:40.375 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:31:40.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:40.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:31:40.498 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:31:40.529 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:40.587 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:31:40.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:40.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:31:40.640 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:31:40.641 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:31:40.696 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:40.832 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:40.888 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:41.023 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:41 smithi138 ceph-mon[185886]: pgmap v985: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T09:31:41.024 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:41.024 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:31:41.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:41 smithi190 ceph-mon[160435]: pgmap v985: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T09:31:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:41.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:31:41.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:41 smithi138 ceph-mon[192513]: pgmap v985: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T09:31:41.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:41.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:31:41.386 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:31:41.808 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:41.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:41.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:41.914 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:31:41.978 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:42.058 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:31:42.278 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:31:42.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:42.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:31:42.385 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:31:42.450 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:42 smithi138 ceph-mon[185886]: pgmap v986: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T09:31:42.450 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:42 smithi138 ceph-mon[192513]: pgmap v986: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T09:31:42.513 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:42.540 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:31:42.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:42 smithi190 ceph-mon[160435]: pgmap v986: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T09:31:42.575 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:31:42.623 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:31:42.667 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:31:42.707 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:31:42.748 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:31:42.777 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:31:42.805 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:31:42.809 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:31:42.810 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:31:42.810 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:31:42.810 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:31:42.810 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:31:42.810 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:31:42.810 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:31:42.810 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:31:43.542 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:31:43.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:43.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:31:43.547 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:43 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3353644084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:43.551 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:43 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3353644084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:43.647 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:43.673 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:43 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3353644084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:43.678 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:43.738 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:43.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:43.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:43.844 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:31:43.882 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:43.924 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:31:43.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:43.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:31:44.031 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:31:44.059 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:44.381 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:31:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:44.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:31:44.492 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:44.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:44 smithi138 ceph-mon[185886]: pgmap v987: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 703 op/s 2023-10-01T09:31:44.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:44 smithi138 ceph-mon[192513]: pgmap v987: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 703 op/s 2023-10-01T09:31:44.580 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:44.736 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:44.760 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:44.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:44.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:44.796 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:44.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:44 smithi190 ceph-mon[160435]: pgmap v987: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 703 op/s 2023-10-01T09:31:44.869 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:44.899 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:44.951 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:44.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:44.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:45.057 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:31:45.094 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:45.158 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:31:45.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:45.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:31:45.264 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:45.266 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:31:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:31:44] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.33.4" 2023-10-01T09:31:45.291 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:45.349 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:45.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:45.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:45.412 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:31:45.447 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:31:45.455 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:45.485 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:45.542 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:45.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:45.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:45.651 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:45.694 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:45.752 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:45.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:45.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:45.861 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:45.903 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:45.975 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:45.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:45.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:46.083 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:46.129 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:46.193 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:46.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:46.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:46.297 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:46.362 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:46.627 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:31:46.785 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:46.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:46.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:46.901 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:46.934 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:46.960 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":987,"stamp":"2023-10-01T09:31:43.956485+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4838856651,"num_objects":73791,"num_object_clones":0,"num_object_copies":221373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73791,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":130161,"num_write_kb":6505254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":318,"num_bytes_recovered":3570664,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117425,"ondisk_log_size":117425,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17888568,"kb_used_data":14326016,"kb_used_omap":45,"kb_used_meta":3562194,"kb_avail":638327496,"statfs":{"total":671965249536,"available":653647355904,"internally_reserved":0,"allocated":14669840384,"data_stored":14660815835,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46344,"internal_metadata":3647687416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-461176832,"num_objects":-7037,"num_object_clones":0,"num_object_copies":-21111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7037,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005475"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":238,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.472639+0000","last_change":"2023-10-01T09:26:12.273466+0000","last_active":"2023-10-01T09:31:29.472639+0000","last_peered":"2023-10-01T09:31:29.472639+0000","last_clean":"2023-10-01T09:31:29.472639+0000","last_became_active":"2023-10-01T09:26:12.272944+0000","last_became_peered":"2023-10-01T09:26:12.272944+0000","last_unstale":"2023-10-01T09:31:29.472639+0000","last_undegraded":"2023-10-01T09:31:29.472639+0000","last_fullsized":"2023-10-01T09:31:29.472639+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:40:41.939800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":159,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.465487+0000","last_change":"2023-10-01T09:28:02.324629+0000","last_active":"2023-10-01T09:31:29.465487+0000","last_peered":"2023-10-01T09:31:29.465487+0000","last_clean":"2023-10-01T09:31:29.465487+0000","last_became_active":"2023-10-01T09:28:02.324019+0000","last_became_peered":"2023-10-01T09:28:02.324019+0000","last_unstale":"2023-10-01T09:31:29.465487+0000","last_undegraded":"2023-10-01T09:31:29.465487+0000","last_fullsized":"2023-10-01T09:31:29.465487+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:00:43.440938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":323,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:33.412401+0000","last_change":"2023-10-01T09:28:02.338915+0000","last_active":"2023-10-01T09:31:33.412401+0000","last_peered":"2023-10-01T09:31:33.412401+0000","last_clean":"2023-10-01T09:31:33.412401+0000","last_became_active":"2023-10-01T09:28:02.338589+0000","last_became_peered":"2023-10-01T09:28:02.338589+0000","last_unstale":"2023-10-01T09:31:33.412401+0000","last_undegraded":"2023-10-01T09:31:33.412401+0000","last_fullsized":"2023-10-01T09:31:33.412401+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:18:19.992078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":371,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.459529+0000","last_change":"2023-10-01T09:29:58.372577+0000","last_active":"2023-10-01T09:31:29.459529+0000","last_peered":"2023-10-01T09:31:29.459529+0000","last_clean":"2023-10-01T09:31:29.459529+0000","last_became_active":"2023-10-01T09:29:58.372218+0000","last_became_peered":"2023-10-01T09:29:58.372218+0000","last_unstale":"2023-10-01T09:31:29.459529+0000","last_undegraded":"2023-10-01T09:31:29.459529+0000","last_fullsized":"2023-10-01T09:31:29.459529+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:04:08.376752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":244,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.457614+0000","last_change":"2023-10-01T09:30:39.394083+0000","last_active":"2023-10-01T09:31:29.457614+0000","last_peered":"2023-10-01T09:31:29.457614+0000","last_clean":"2023-10-01T09:31:29.457614+0000","last_became_active":"2023-10-01T09:30:39.393797+0000","last_became_peered":"2023-10-01T09:30:39.393797+0000","last_unstale":"2023-10-01T09:31:29.457614+0000","last_undegraded":"2023-10-01T09:31:29.457614+0000","last_fullsized":"2023-10-01T09:31:29.457614+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:22.625624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":59,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.455308+0000","last_change":"2023-10-01T09:28:02.362383+0000","last_active":"2023-10-01T09:31:29.455308+0000","last_peered":"2023-10-01T09:31:29.455308+0000","last_clean":"2023-10-01T09:31:29.455308+0000","last_became_active":"2023-10-01T09:28:02.362044+0000","last_became_peered":"2023-10-01T09:28:02.362044+0000","last_unstale":"2023-10-01T09:31:29.455308+0000","last_undegraded":"2023-10-01T09:31:29.455308+0000","last_fullsized":"2023-10-01T09:31:29.455308+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":392,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.458272+0000","last_change":"2023-10-01T09:27:01.992338+0000","last_active":"2023-10-01T09:31:29.458272+0000","last_peered":"2023-10-01T09:31:29.458272+0000","last_clean":"2023-10-01T09:31:29.458272+0000","last_became_active":"2023-10-01T09:27:01.989050+0000","last_became_peered":"2023-10-01T09:27:01.989050+0000","last_unstale":"2023-10-01T09:31:29.458272+0000","last_undegraded":"2023-10-01T09:31:29.458272+0000","last_fullsized":"2023-10-01T09:31:29.458272+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:43:29.856456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":355,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.465161+0000","last_change":"2023-10-01T09:30:39.395270+0000","last_active":"2023-10-01T09:31:29.465161+0000","last_peered":"2023-10-01T09:31:29.465161+0000","last_clean":"2023-10-01T09:31:29.465161+0000","last_became_active":"2023-10-01T09:30:39.394888+0000","last_became_peered":"2023-10-01T09:30:39.394888+0000","last_unstale":"2023-10-01T09:31:29.465161+0000","last_undegraded":"2023-10-01T09:31:29.465161+0000","last_fullsized":"2023-10-01T09:31:29.465161+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:35:40.846102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":66,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.463800+0000","last_change":"2023-10-01T09:27:01.970412+0000","last_active":"2023-10-01T09:31:29.463800+0000","last_peered":"2023-10-01T09:31:29.463800+0000","last_clean":"2023-10-01T09:31:29.463800+0000","last_became_active":"2023-10-01T09:27:01.970108+0000","last_became_peered":"2023-10-01T09:27:01.970108+0000","last_unstale":"2023-10-01T09:31:29.463800+0000","last_undegraded":"2023-10-01T09:31:29.463800+0000","last_fullsized":"2023-10-01T09:31:29.463800+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":372,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.455662+0000","last_change":"2023-10-01T09:30:39.398262+0000","last_active":"2023-10-01T09:31:29.455662+0000","last_peered":"2023-10-01T09:31:29.455662+0000","last_clean":"2023-10-01T09:31:29.455662+0000","last_became_active":"2023-10-01T09:30:39.397934+0000","last_became_peered":"2023-10-01T09:30:39.397934+0000","last_unstale":"2023-10-01T09:31:29.455662+0000","last_undegraded":"2023-10-01T09:31:29.455662+0000","last_fullsized":"2023-10-01T09:31:29.455662+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":139,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.455816+0000","last_change":"2023-10-01T09:26:12.415219+0000","last_active":"2023-10-01T09:31:29.455816+0000","last_peered":"2023-10-01T09:31:29.455816+0000","last_clean":"2023-10-01T09:31:29.455816+0000","last_became_active":"2023-10-01T09:26:12.286669+0000","last_became_peered":"2023-10-01T09:26:12.286669+0000","last_unstale":"2023-10-01T09:31:29.455816+0000","last_undegraded":"2023-10-01T09:31:29.455816+0000","last_fullsized":"2023-10-01T09:31:29.455816+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":383,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.472327+0000","last_change":"2023-10-01T09:28:02.342661+0000","last_active":"2023-10-01T09:31:29.472327+0000","last_peered":"2023-10-01T09:31:29.472327+0000","last_clean":"2023-10-01T09:31:29.472327+0000","last_became_active":"2023-10-01T09:28:02.342358+0000","last_became_peered":"2023-10-01T09:28:02.342358+0000","last_unstale":"2023-10-01T09:31:29.472327+0000","last_undegraded":"2023-10-01T09:31:29.472327+0000","last_fullsized":"2023-10-01T09:31:29.472327+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:29:30.624568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":363,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.455879+0000","last_change":"2023-10-01T09:28:02.511577+0000","last_active":"2023-10-01T09:31:29.455879+0000","last_peered":"2023-10-01T09:31:29.455879+0000","last_clean":"2023-10-01T09:31:29.455879+0000","last_became_active":"2023-10-01T09:28:02.318805+0000","last_became_peered":"2023-10-01T09:28:02.318805+0000","last_unstale":"2023-10-01T09:31:29.455879+0000","last_undegraded":"2023-10-01T09:31:29.455879+0000","last_fullsized":"2023-10-01T09:31:29.455879+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":59,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.454238+0000","last_change":"2023-10-01T09:28:02.454516+0000","last_active":"2023-10-01T09:31:29.454238+0000","last_peered":"2023-10-01T09:31:29.454238+0000","last_clean":"2023-10-01T09:31:29.454238+0000","last_became_active":"2023-10-01T09:28:02.454098+0000","last_became_peered":"2023-10-01T09:28:02.454098+0000","last_unstale":"2023-10-01T09:31:29.454238+0000","last_undegraded":"2023-10-01T09:31:29.454238+0000","last_fullsized":"2023-10-01T09:31:29.454238+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":413,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.455807+0000","last_change":"2023-10-01T09:28:02.505692+0000","last_active":"2023-10-01T09:31:29.455807+0000","last_peered":"2023-10-01T09:31:29.455807+0000","last_clean":"2023-10-01T09:31:29.455807+0000","last_became_active":"2023-10-01T09:28:02.318771+0000","last_became_peered":"2023-10-01T09:28:02.318771+0000","last_unstale":"2023-10-01T09:31:29.455807+0000","last_undegraded":"2023-10-01T09:31:29.455807+0000","last_fullsized":"2023-10-01T09:31:29.455807+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":183,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.455979+0000","last_change":"2023-10-01T09:30:39.399483+0000","last_active":"2023-10-01T09:31:29.455979+0000","last_peered":"2023-10-01T09:31:29.455979+0000","last_clean":"2023-10-01T09:31:29.455979+0000","last_became_active":"2023-10-01T09:30:39.399266+0000","last_became_peered":"2023-10-01T09:30:39.399266+0000","last_unstale":"2023-10-01T09:31:29.455979+0000","last_undegraded":"2023-10-01T09:31:29.455979+0000","last_fullsized":"2023-10-01T09:31:29.455979+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":345,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.458794+0000","last_change":"2023-10-01T09:23:10.607388+0000","last_active":"2023-10-01T09:31:29.458794+0000","last_peered":"2023-10-01T09:31:29.458794+0000","last_clean":"2023-10-01T09:31:29.458794+0000","last_became_active":"2023-10-01T09:23:10.607084+0000","last_became_peered":"2023-10-01T09:23:10.607084+0000","last_unstale":"2023-10-01T09:31:29.458794+0000","last_undegraded":"2023-10-01T09:31:29.458794+0000","last_fullsized":"2023-10-01T09:31:29.458794+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:19.669393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":66,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.464009+0000","last_change":"2023-10-01T09:27:01.978738+0000","last_active":"2023-10-01T09:31:29.464009+0000","last_peered":"2023-10-01T09:31:29.464009+0000","last_clean":"2023-10-01T09:31:29.464009+0000","last_became_active":"2023-10-01T09:27:01.978024+0000","last_became_peered":"2023-10-01T09:27:01.978024+0000","last_unstale":"2023-10-01T09:31:29.464009+0000","last_undegraded":"2023-10-01T09:31:29.464009+0000","last_fullsized":"2023-10-01T09:31:29.464009+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":357,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.455570+0000","last_change":"2023-10-01T09:28:02.314047+0000","last_active":"2023-10-01T09:31:29.455570+0000","last_peered":"2023-10-01T09:31:29.455570+0000","last_clean":"2023-10-01T09:31:29.455570+0000","last_became_active":"2023-10-01T09:28:02.313729+0000","last_became_peered":"2023-10-01T09:28:02.313729+0000","last_unstale":"2023-10-01T09:31:29.455570+0000","last_undegraded":"2023-10-01T09:31:29.455570+0000","last_fullsized":"2023-10-01T09:31:29.455570+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":327,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.459600+0000","last_change":"2023-10-01T09:23:10.611400+0000","last_active":"2023-10-01T09:31:29.459600+0000","last_peered":"2023-10-01T09:31:29.459600+0000","last_clean":"2023-10-01T09:31:29.459600+0000","last_became_active":"2023-10-01T09:23:10.610928+0000","last_became_peered":"2023-10-01T09:23:10.610928+0000","last_unstale":"2023-10-01T09:31:29.459600+0000","last_undegraded":"2023-10-01T09:31:29.459600+0000","last_fullsized":"2023-10-01T09:31:29.459600+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:17.141540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":96,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.453693+0000","last_change":"2023-10-01T09:30:00.577886+0000","last_active":"2023-10-01T09:31:29.453693+0000","last_peered":"2023-10-01T09:31:29.453693+0000","last_clean":"2023-10-01T09:31:29.453693+0000","last_became_active":"2023-10-01T09:30:00.394121+0000","last_became_peered":"2023-10-01T09:30:00.394121+0000","last_unstale":"2023-10-01T09:31:29.453693+0000","last_undegraded":"2023-10-01T09:31:29.453693+0000","last_fullsized":"2023-10-01T09:31:29.453693+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":44,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.455872+0000","last_change":"2023-10-01T09:30:39.398326+0000","last_active":"2023-10-01T09:31:29.455872+0000","last_peered":"2023-10-01T09:31:29.455872+0000","last_clean":"2023-10-01T09:31:29.455872+0000","last_became_active":"2023-10-01T09:30:39.398021+0000","last_became_peered":"2023-10-01T09:30:39.398021+0000","last_unstale":"2023-10-01T09:31:29.455872+0000","last_undegraded":"2023-10-01T09:31:29.455872+0000","last_fullsized":"2023-10-01T09:31:29.455872+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"355'7902","reported_seq":7939,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:40.581915+0000","last_change":"2023-10-01T09:29:24.378970+0000","last_active":"2023-10-01T09:31:40.581915+0000","last_peered":"2023-10-01T09:31:40.581915+0000","last_clean":"2023-10-01T09:31:40.581915+0000","last_became_active":"2023-10-01T09:29:24.378703+0000","last_became_peered":"2023-10-01T09:29:24.378703+0000","last_unstale":"2023-10-01T09:31:40.581915+0000","last_undegraded":"2023-10-01T09:31:40.581915+0000","last_fullsized":"2023-10-01T09:31:40.581915+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":7902,"log_dups_size":0,"ondisk_log_size":7902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301989888,"num_objects":4608,"num_object_clones":0,"num_object_copies":13824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7902,"num_write_kb":400320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":135,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.455938+0000","last_change":"2023-10-01T09:26:12.395230+0000","last_active":"2023-10-01T09:31:29.455938+0000","last_peered":"2023-10-01T09:31:29.455938+0000","last_clean":"2023-10-01T09:31:29.455938+0000","last_became_active":"2023-10-01T09:26:12.269960+0000","last_became_peered":"2023-10-01T09:26:12.269960+0000","last_unstale":"2023-10-01T09:31:29.455938+0000","last_undegraded":"2023-10-01T09:31:29.455938+0000","last_fullsized":"2023-10-01T09:31:29.455938+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":75,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.463400+0000","last_change":"2023-10-01T09:28:02.315483+0000","last_active":"2023-10-01T09:31:29.463400+0000","last_peered":"2023-10-01T09:31:29.463400+0000","last_clean":"2023-10-01T09:31:29.463400+0000","last_became_active":"2023-10-01T09:28:02.315121+0000","last_became_peered":"2023-10-01T09:28:02.315121+0000","last_unstale":"2023-10-01T09:31:29.463400+0000","last_undegraded":"2023-10-01T09:31:29.463400+0000","last_fullsized":"2023-10-01T09:31:29.463400+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"355'8134","reported_seq":8171,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:41.895039+0000","last_change":"2023-10-01T09:29:24.376565+0000","last_active":"2023-10-01T09:31:41.895039+0000","last_peered":"2023-10-01T09:31:41.895039+0000","last_clean":"2023-10-01T09:31:41.895039+0000","last_became_active":"2023-10-01T09:29:24.376238+0000","last_became_peered":"2023-10-01T09:29:24.376238+0000","last_unstale":"2023-10-01T09:31:41.895039+0000","last_undegraded":"2023-10-01T09:31:41.895039+0000","last_fullsized":"2023-10-01T09:31:41.895039+0000","mapping_epoch":326,"log_start":"331'400","ondisk_log_start":"331'400","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":7734,"log_dups_size":0,"ondisk_log_size":7734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304480256,"num_objects":4646,"num_object_clones":0,"num_object_copies":13938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8134,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":336,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.457292+0000","last_change":"2023-10-01T09:26:12.408240+0000","last_active":"2023-10-01T09:31:29.457292+0000","last_peered":"2023-10-01T09:31:29.457292+0000","last_clean":"2023-10-01T09:31:29.457292+0000","last_became_active":"2023-10-01T09:26:12.272093+0000","last_became_peered":"2023-10-01T09:26:12.272093+0000","last_unstale":"2023-10-01T09:31:29.457292+0000","last_undegraded":"2023-10-01T09:31:29.457292+0000","last_fullsized":"2023-10-01T09:31:29.457292+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":377,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.469643+0000","last_change":"2023-10-01T09:28:02.338346+0000","last_active":"2023-10-01T09:31:29.469643+0000","last_peered":"2023-10-01T09:31:29.469643+0000","last_clean":"2023-10-01T09:31:29.469643+0000","last_became_active":"2023-10-01T09:28:02.338038+0000","last_became_peered":"2023-10-01T09:28:02.338038+0000","last_unstale":"2023-10-01T09:31:29.469643+0000","last_undegraded":"2023-10-01T09:31:29.469643+0000","last_fullsized":"2023-10-01T09:31:29.469643+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:56:32.734028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"355'8071","reported_seq":8110,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:43.533493+0000","last_change":"2023-10-01T09:29:24.379084+0000","last_active":"2023-10-01T09:31:43.533493+0000","last_peered":"2023-10-01T09:31:43.533493+0000","last_clean":"2023-10-01T09:31:43.533493+0000","last_became_active":"2023-10-01T09:29:24.378789+0000","last_became_peered":"2023-10-01T09:29:24.378789+0000","last_unstale":"2023-10-01T09:31:43.533493+0000","last_undegraded":"2023-10-01T09:31:43.533493+0000","last_fullsized":"2023-10-01T09:31:43.533493+0000","mapping_epoch":326,"log_start":"331'500","ondisk_log_start":"331'500","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":7571,"log_dups_size":0,"ondisk_log_size":7571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:00:21.161819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296681472,"num_objects":4527,"num_object_clones":0,"num_object_copies":13581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8071,"num_write_kb":403136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":420,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.455918+0000","last_change":"2023-10-01T09:28:02.318966+0000","last_active":"2023-10-01T09:31:29.455918+0000","last_peered":"2023-10-01T09:31:29.455918+0000","last_clean":"2023-10-01T09:31:29.455918+0000","last_became_active":"2023-10-01T09:28:02.316075+0000","last_became_peered":"2023-10-01T09:28:02.316075+0000","last_unstale":"2023-10-01T09:31:29.455918+0000","last_undegraded":"2023-10-01T09:31:29.455918+0000","last_fullsized":"2023-10-01T09:31:29.455918+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":58,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.453460+0000","last_change":"2023-10-01T09:28:02.361663+0000","last_active":"2023-10-01T09:31:29.453460+0000","last_peered":"2023-10-01T09:31:29.453460+0000","last_clean":"2023-10-01T09:31:29.453460+0000","last_became_active":"2023-10-01T09:28:02.361093+0000","last_became_peered":"2023-10-01T09:28:02.361093+0000","last_unstale":"2023-10-01T09:31:29.453460+0000","last_undegraded":"2023-10-01T09:31:29.453460+0000","last_fullsized":"2023-10-01T09:31:29.453460+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"355'8128","reported_seq":8167,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:43.344828+0000","last_change":"2023-10-01T09:29:24.356954+0000","last_active":"2023-10-01T09:31:43.344828+0000","last_peered":"2023-10-01T09:31:43.344828+0000","last_clean":"2023-10-01T09:31:43.344828+0000","last_became_active":"2023-10-01T09:29:24.356643+0000","last_became_peered":"2023-10-01T09:29:24.356643+0000","last_unstale":"2023-10-01T09:31:43.344828+0000","last_undegraded":"2023-10-01T09:31:43.344828+0000","last_fullsized":"2023-10-01T09:31:43.344828+0000","mapping_epoch":326,"log_start":"332'1700","ondisk_log_start":"332'1700","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":6428,"log_dups_size":0,"ondisk_log_size":6428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:56:34.140751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301203456,"num_objects":4596,"num_object_clones":0,"num_object_copies":13788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8128,"num_write_kb":407168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":75,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.464056+0000","last_change":"2023-10-01T09:28:02.314625+0000","last_active":"2023-10-01T09:31:29.464056+0000","last_peered":"2023-10-01T09:31:29.464056+0000","last_clean":"2023-10-01T09:31:29.464056+0000","last_became_active":"2023-10-01T09:28:02.314315+0000","last_became_peered":"2023-10-01T09:28:02.314315+0000","last_unstale":"2023-10-01T09:31:29.464056+0000","last_undegraded":"2023-10-01T09:31:29.464056+0000","last_fullsized":"2023-10-01T09:31:29.464056+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":319,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.457727+0000","last_change":"2023-10-01T09:26:12.308350+0000","last_active":"2023-10-01T09:31:29.457727+0000","last_peered":"2023-10-01T09:31:29.457727+0000","last_clean":"2023-10-01T09:31:29.457727+0000","last_became_active":"2023-10-01T09:26:12.307689+0000","last_became_peered":"2023-10-01T09:26:12.307689+0000","last_unstale":"2023-10-01T09:31:29.457727+0000","last_undegraded":"2023-10-01T09:31:29.457727+0000","last_fullsized":"2023-10-01T09:31:29.457727+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:34:57.116449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"355'7948","reported_seq":7993,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:40.680543+0000","last_change":"2023-10-01T09:30:39.402464+0000","last_active":"2023-10-01T09:31:40.680543+0000","last_peered":"2023-10-01T09:31:40.680543+0000","last_clean":"2023-10-01T09:31:40.680543+0000","last_became_active":"2023-10-01T09:30:39.397122+0000","last_became_peered":"2023-10-01T09:30:39.397122+0000","last_unstale":"2023-10-01T09:31:40.680543+0000","last_undegraded":"2023-10-01T09:31:40.680543+0000","last_fullsized":"2023-10-01T09:31:40.680543+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":7948,"log_dups_size":0,"ondisk_log_size":7948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306839552,"num_objects":4682,"num_object_clones":0,"num_object_copies":14046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7948,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":42,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.457836+0000","last_change":"2023-10-01T09:30:39.395785+0000","last_active":"2023-10-01T09:31:29.457836+0000","last_peered":"2023-10-01T09:31:29.457836+0000","last_clean":"2023-10-01T09:31:29.457836+0000","last_became_active":"2023-10-01T09:30:39.395449+0000","last_became_peered":"2023-10-01T09:30:39.395449+0000","last_unstale":"2023-10-01T09:31:29.457836+0000","last_undegraded":"2023-10-01T09:31:29.457836+0000","last_fullsized":"2023-10-01T09:31:29.457836+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:23:32.908527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":287,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.463574+0000","last_change":"2023-10-01T09:30:39.398279+0000","last_active":"2023-10-01T09:31:29.463574+0000","last_peered":"2023-10-01T09:31:29.463574+0000","last_clean":"2023-10-01T09:31:29.463574+0000","last_became_active":"2023-10-01T09:30:39.397946+0000","last_became_peered":"2023-10-01T09:30:39.397946+0000","last_unstale":"2023-10-01T09:31:29.463574+0000","last_undegraded":"2023-10-01T09:31:29.463574+0000","last_fullsized":"2023-10-01T09:31:29.463574+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:30:40.348097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1","version":"355'8142","reported_seq":8181,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:43.396871+0000","last_change":"2023-10-01T09:29:24.380240+0000","last_active":"2023-10-01T09:31:43.396871+0000","last_peered":"2023-10-01T09:31:43.396871+0000","last_clean":"2023-10-01T09:31:43.396871+0000","last_became_active":"2023-10-01T09:29:24.379922+0000","last_became_peered":"2023-10-01T09:29:24.379922+0000","last_unstale":"2023-10-01T09:31:43.396871+0000","last_undegraded":"2023-10-01T09:31:43.396871+0000","last_fullsized":"2023-10-01T09:31:43.396871+0000","mapping_epoch":326,"log_start":"332'1700","ondisk_log_start":"332'1700","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":6442,"log_dups_size":0,"ondisk_log_size":6442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:34:10.363596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299237376,"num_objects":4566,"num_object_clones":0,"num_object_copies":13698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8142,"num_write_kb":406656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":327,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.472471+0000","last_change":"2023-10-01T09:28:02.342548+0000","last_active":"2023-10-01T09:31:29.472471+0000","last_peered":"2023-10-01T09:31:29.472471+0000","last_clean":"2023-10-01T09:31:29.472471+0000","last_became_active":"2023-10-01T09:28:02.342162+0000","last_became_peered":"2023-10-01T09:28:02.342162+0000","last_unstale":"2023-10-01T09:31:29.472471+0000","last_undegraded":"2023-10-01T09:31:29.472471+0000","last_fullsized":"2023-10-01T09:31:29.472471+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:49:07.163031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":372,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.464087+0000","last_change":"2023-10-01T09:28:02.342750+0000","last_active":"2023-10-01T09:31:29.464087+0000","last_peered":"2023-10-01T09:31:29.464087+0000","last_clean":"2023-10-01T09:31:29.464087+0000","last_became_active":"2023-10-01T09:28:02.342487+0000","last_became_peered":"2023-10-01T09:28:02.342487+0000","last_unstale":"2023-10-01T09:31:29.464087+0000","last_undegraded":"2023-10-01T09:31:29.464087+0000","last_fullsized":"2023-10-01T09:31:29.464087+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:23:46.051719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"355'8046","reported_seq":8085,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:43.563361+0000","last_change":"2023-10-01T09:29:24.359090+0000","last_active":"2023-10-01T09:31:43.563361+0000","last_peered":"2023-10-01T09:31:43.563361+0000","last_clean":"2023-10-01T09:31:43.563361+0000","last_became_active":"2023-10-01T09:29:24.358822+0000","last_became_peered":"2023-10-01T09:29:24.358822+0000","last_unstale":"2023-10-01T09:31:43.563361+0000","last_undegraded":"2023-10-01T09:31:43.563361+0000","last_fullsized":"2023-10-01T09:31:43.563361+0000","mapping_epoch":326,"log_start":"331'500","ondisk_log_start":"331'500","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":7546,"log_dups_size":0,"ondisk_log_size":7546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:29:47.448230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294125568,"num_objects":4488,"num_object_clones":0,"num_object_copies":13464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8046,"num_write_kb":401088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":218,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.455570+0000","last_change":"2023-10-01T09:26:29.372326+0000","last_active":"2023-10-01T09:31:29.455570+0000","last_peered":"2023-10-01T09:31:29.455570+0000","last_clean":"2023-10-01T09:31:29.455570+0000","last_became_active":"2023-10-01T09:26:29.372035+0000","last_became_peered":"2023-10-01T09:26:29.372035+0000","last_unstale":"2023-10-01T09:31:29.455570+0000","last_undegraded":"2023-10-01T09:31:29.455570+0000","last_fullsized":"2023-10-01T09:31:29.455570+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":133,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.464433+0000","last_change":"2023-10-01T09:28:02.339901+0000","last_active":"2023-10-01T09:31:29.464433+0000","last_peered":"2023-10-01T09:31:29.464433+0000","last_clean":"2023-10-01T09:31:29.464433+0000","last_became_active":"2023-10-01T09:28:02.339601+0000","last_became_peered":"2023-10-01T09:28:02.339601+0000","last_unstale":"2023-10-01T09:31:29.464433+0000","last_undegraded":"2023-10-01T09:31:29.464433+0000","last_fullsized":"2023-10-01T09:31:29.464433+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:30:16.497879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"355'7970","reported_seq":8009,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:43.357649+0000","last_change":"2023-10-01T09:29:24.351403+0000","last_active":"2023-10-01T09:31:43.357649+0000","last_peered":"2023-10-01T09:31:43.357649+0000","last_clean":"2023-10-01T09:31:43.357649+0000","last_became_active":"2023-10-01T09:29:24.351113+0000","last_became_peered":"2023-10-01T09:29:24.351113+0000","last_unstale":"2023-10-01T09:31:43.357649+0000","last_undegraded":"2023-10-01T09:31:43.357649+0000","last_fullsized":"2023-10-01T09:31:43.357649+0000","mapping_epoch":326,"log_start":"332'1500","ondisk_log_start":"332'1500","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":6470,"log_dups_size":0,"ondisk_log_size":6470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:07:56.443376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298844160,"num_objects":4560,"num_object_clones":0,"num_object_copies":13680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7970,"num_write_kb":400960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":194,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.464976+0000","last_change":"2023-10-01T09:30:39.396141+0000","last_active":"2023-10-01T09:31:29.464976+0000","last_peered":"2023-10-01T09:31:29.464976+0000","last_clean":"2023-10-01T09:31:29.464976+0000","last_became_active":"2023-10-01T09:30:39.395843+0000","last_became_peered":"2023-10-01T09:30:39.395843+0000","last_unstale":"2023-10-01T09:31:29.464976+0000","last_undegraded":"2023-10-01T09:31:29.464976+0000","last_fullsized":"2023-10-01T09:31:29.464976+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:13:32.932438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":232,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.464018+0000","last_change":"2023-10-01T09:26:12.320680+0000","last_active":"2023-10-01T09:31:29.464018+0000","last_peered":"2023-10-01T09:31:29.464018+0000","last_clean":"2023-10-01T09:31:29.464018+0000","last_became_active":"2023-10-01T09:26:12.320340+0000","last_became_peered":"2023-10-01T09:26:12.320340+0000","last_unstale":"2023-10-01T09:31:29.464018+0000","last_undegraded":"2023-10-01T09:31:29.464018+0000","last_fullsized":"2023-10-01T09:31:29.464018+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:11:35.796015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"355'8282","reported_seq":8320,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:43.573336+0000","last_change":"2023-10-01T09:29:24.382969+0000","last_active":"2023-10-01T09:31:43.573336+0000","last_peered":"2023-10-01T09:31:43.573336+0000","last_clean":"2023-10-01T09:31:43.573336+0000","last_became_active":"2023-10-01T09:29:24.382653+0000","last_became_peered":"2023-10-01T09:29:24.382653+0000","last_unstale":"2023-10-01T09:31:43.573336+0000","last_undegraded":"2023-10-01T09:31:43.573336+0000","last_fullsized":"2023-10-01T09:31:43.573336+0000","mapping_epoch":326,"log_start":"331'700","ondisk_log_start":"331'700","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":7582,"log_dups_size":0,"ondisk_log_size":7582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:37:56.666282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307625984,"num_objects":4694,"num_object_clones":0,"num_object_copies":14082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8282,"num_write_kb":415232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1176,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.463659+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:31:29.463659+0000","last_peered":"2023-10-01T09:31:29.463659+0000","last_clean":"2023-10-01T09:31:29.463659+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:31:29.463659+0000","last_undegraded":"2023-10-01T09:31:29.463659+0000","last_fullsized":"2023-10-01T09:31:29.463659+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":66,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.464113+0000","last_change":"2023-10-01T09:27:01.989188+0000","last_active":"2023-10-01T09:31:29.464113+0000","last_peered":"2023-10-01T09:31:29.464113+0000","last_clean":"2023-10-01T09:31:29.464113+0000","last_became_active":"2023-10-01T09:27:01.988878+0000","last_became_peered":"2023-10-01T09:27:01.988878+0000","last_unstale":"2023-10-01T09:31:29.464113+0000","last_undegraded":"2023-10-01T09:31:29.464113+0000","last_fullsized":"2023-10-01T09:31:29.464113+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":333,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.472392+0000","last_change":"2023-10-01T09:27:01.977155+0000","last_active":"2023-10-01T09:31:29.472392+0000","last_peered":"2023-10-01T09:31:29.472392+0000","last_clean":"2023-10-01T09:31:29.472392+0000","last_became_active":"2023-10-01T09:27:01.975803+0000","last_became_peered":"2023-10-01T09:27:01.975803+0000","last_unstale":"2023-10-01T09:31:29.472392+0000","last_undegraded":"2023-10-01T09:31:29.472392+0000","last_fullsized":"2023-10-01T09:31:29.472392+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:00:55.170662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"355'8001","reported_seq":8038,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:40.691030+0000","last_change":"2023-10-01T09:29:24.376062+0000","last_active":"2023-10-01T09:31:40.691030+0000","last_peered":"2023-10-01T09:31:40.691030+0000","last_clean":"2023-10-01T09:31:40.691030+0000","last_became_active":"2023-10-01T09:29:24.375795+0000","last_became_peered":"2023-10-01T09:29:24.375795+0000","last_unstale":"2023-10-01T09:31:40.691030+0000","last_undegraded":"2023-10-01T09:31:40.691030+0000","last_fullsized":"2023-10-01T09:31:40.691030+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":8001,"log_dups_size":0,"ondisk_log_size":8001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309526528,"num_objects":4723,"num_object_clones":0,"num_object_copies":14169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8001,"num_write_kb":407168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1038,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.463859+0000","last_change":"2023-10-01T09:28:02.595215+0000","last_active":"2023-10-01T09:31:29.463859+0000","last_peered":"2023-10-01T09:31:29.463859+0000","last_clean":"2023-10-01T09:31:29.463859+0000","last_became_active":"2023-10-01T09:28:02.346777+0000","last_became_peered":"2023-10-01T09:28:02.346777+0000","last_unstale":"2023-10-01T09:31:29.463859+0000","last_undegraded":"2023-10-01T09:31:29.463859+0000","last_fullsized":"2023-10-01T09:31:29.463859+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":319,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.458084+0000","last_change":"2023-10-01T09:28:02.457735+0000","last_active":"2023-10-01T09:31:29.458084+0000","last_peered":"2023-10-01T09:31:29.458084+0000","last_clean":"2023-10-01T09:31:29.458084+0000","last_became_active":"2023-10-01T09:28:02.457495+0000","last_became_peered":"2023-10-01T09:28:02.457495+0000","last_unstale":"2023-10-01T09:31:29.458084+0000","last_undegraded":"2023-10-01T09:31:29.458084+0000","last_fullsized":"2023-10-01T09:31:29.458084+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:48:53.316760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":29,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.463823+0000","last_change":"2023-10-01T09:29:58.362019+0000","last_active":"2023-10-01T09:31:29.463823+0000","last_peered":"2023-10-01T09:31:29.463823+0000","last_clean":"2023-10-01T09:31:29.463823+0000","last_became_active":"2023-10-01T09:29:58.361762+0000","last_became_peered":"2023-10-01T09:29:58.361762+0000","last_unstale":"2023-10-01T09:31:29.463823+0000","last_undegraded":"2023-10-01T09:31:29.463823+0000","last_fullsized":"2023-10-01T09:31:29.463823+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"355'8006","reported_seq":8052,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:43.361875+0000","last_change":"2023-10-01T09:30:39.412038+0000","last_active":"2023-10-01T09:31:43.361875+0000","last_peered":"2023-10-01T09:31:43.361875+0000","last_clean":"2023-10-01T09:31:43.361875+0000","last_became_active":"2023-10-01T09:30:39.400515+0000","last_became_peered":"2023-10-01T09:30:39.400515+0000","last_unstale":"2023-10-01T09:31:43.361875+0000","last_undegraded":"2023-10-01T09:31:43.361875+0000","last_fullsized":"2023-10-01T09:31:43.361875+0000","mapping_epoch":343,"log_start":"332'1600","ondisk_log_start":"332'1600","created":326,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":6406,"log_dups_size":0,"ondisk_log_size":6406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:17:03.280627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300417024,"num_objects":4584,"num_object_clones":0,"num_object_copies":13752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8006,"num_write_kb":402880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":546,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.457910+0000","last_change":"2023-10-01T09:30:39.393231+0000","last_active":"2023-10-01T09:31:29.457910+0000","last_peered":"2023-10-01T09:31:29.457910+0000","last_clean":"2023-10-01T09:31:29.457910+0000","last_became_active":"2023-10-01T09:30:39.392700+0000","last_became_peered":"2023-10-01T09:30:39.392700+0000","last_unstale":"2023-10-01T09:31:29.457910+0000","last_undegraded":"2023-10-01T09:31:29.457910+0000","last_fullsized":"2023-10-01T09:31:29.457910+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:38:26.873688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":384,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.465107+0000","last_change":"2023-10-01T09:28:02.346596+0000","last_active":"2023-10-01T09:31:29.465107+0000","last_peered":"2023-10-01T09:31:29.465107+0000","last_clean":"2023-10-01T09:31:29.465107+0000","last_became_active":"2023-10-01T09:28:02.346318+0000","last_became_peered":"2023-10-01T09:28:02.346318+0000","last_unstale":"2023-10-01T09:31:29.465107+0000","last_undegraded":"2023-10-01T09:31:29.465107+0000","last_fullsized":"2023-10-01T09:31:29.465107+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:27:45.840901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":238,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.464141+0000","last_change":"2023-10-01T09:27:02.004900+0000","last_active":"2023-10-01T09:31:29.464141+0000","last_peered":"2023-10-01T09:31:29.464141+0000","last_clean":"2023-10-01T09:31:29.464141+0000","last_became_active":"2023-10-01T09:27:02.004501+0000","last_became_peered":"2023-10-01T09:27:02.004501+0000","last_unstale":"2023-10-01T09:31:29.464141+0000","last_undegraded":"2023-10-01T09:31:29.464141+0000","last_fullsized":"2023-10-01T09:31:29.464141+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:50:52.617833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"355'7924","reported_seq":7961,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:40.684725+0000","last_change":"2023-10-01T09:29:24.359284+0000","last_active":"2023-10-01T09:31:40.684725+0000","last_peered":"2023-10-01T09:31:40.684725+0000","last_clean":"2023-10-01T09:31:40.684725+0000","last_became_active":"2023-10-01T09:29:24.358932+0000","last_became_peered":"2023-10-01T09:29:24.358932+0000","last_unstale":"2023-10-01T09:31:40.684725+0000","last_undegraded":"2023-10-01T09:31:40.684725+0000","last_fullsized":"2023-10-01T09:31:40.684725+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":7924,"log_dups_size":0,"ondisk_log_size":7924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310509568,"num_objects":4738,"num_object_clones":0,"num_object_copies":14214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7924,"num_write_kb":405184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1281,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.463949+0000","last_change":"2023-10-01T09:28:48.006973+0000","last_active":"2023-10-01T09:31:29.463949+0000","last_peered":"2023-10-01T09:31:29.463949+0000","last_clean":"2023-10-01T09:31:29.463949+0000","last_became_active":"2023-10-01T09:28:48.006289+0000","last_became_peered":"2023-10-01T09:28:48.006289+0000","last_unstale":"2023-10-01T09:31:29.463949+0000","last_undegraded":"2023-10-01T09:31:29.463949+0000","last_fullsized":"2023-10-01T09:31:29.463949+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":573,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.456013+0000","last_change":"2023-10-01T09:29:04.962927+0000","last_active":"2023-10-01T09:31:29.456013+0000","last_peered":"2023-10-01T09:31:29.456013+0000","last_clean":"2023-10-01T09:31:29.456013+0000","last_became_active":"2023-10-01T09:29:04.962542+0000","last_became_peered":"2023-10-01T09:29:04.962542+0000","last_unstale":"2023-10-01T09:31:29.456013+0000","last_undegraded":"2023-10-01T09:31:29.456013+0000","last_fullsized":"2023-10-01T09:31:29.456013+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":150,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.463760+0000","last_change":"2023-10-01T09:30:39.397227+0000","last_active":"2023-10-01T09:31:29.463760+0000","last_peered":"2023-10-01T09:31:29.463760+0000","last_clean":"2023-10-01T09:31:29.463760+0000","last_became_active":"2023-10-01T09:30:39.396936+0000","last_became_peered":"2023-10-01T09:30:39.396936+0000","last_unstale":"2023-10-01T09:31:29.463760+0000","last_undegraded":"2023-10-01T09:31:29.463760+0000","last_fullsized":"2023-10-01T09:31:29.463760+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":260,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.456094+0000","last_change":"2023-10-01T09:28:02.312356+0000","last_active":"2023-10-01T09:31:29.456094+0000","last_peered":"2023-10-01T09:31:29.456094+0000","last_clean":"2023-10-01T09:31:29.456094+0000","last_became_active":"2023-10-01T09:28:02.312014+0000","last_became_peered":"2023-10-01T09:28:02.312014+0000","last_unstale":"2023-10-01T09:31:29.456094+0000","last_undegraded":"2023-10-01T09:31:29.456094+0000","last_fullsized":"2023-10-01T09:31:29.456094+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"355'8175","reported_seq":8214,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:43.405559+0000","last_change":"2023-10-01T09:29:24.380724+0000","last_active":"2023-10-01T09:31:43.405559+0000","last_peered":"2023-10-01T09:31:43.405559+0000","last_clean":"2023-10-01T09:31:43.405559+0000","last_became_active":"2023-10-01T09:29:24.380523+0000","last_became_peered":"2023-10-01T09:29:24.380523+0000","last_unstale":"2023-10-01T09:31:43.405559+0000","last_undegraded":"2023-10-01T09:31:43.405559+0000","last_fullsized":"2023-10-01T09:31:43.405559+0000","mapping_epoch":326,"log_start":"332'1700","ondisk_log_start":"332'1700","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":6475,"log_dups_size":0,"ondisk_log_size":6475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:16.268423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293666816,"num_objects":4481,"num_object_clones":0,"num_object_copies":13443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8175,"num_write_kb":404992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":451,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.455613+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:31:29.455613+0000","last_peered":"2023-10-01T09:31:29.455613+0000","last_clean":"2023-10-01T09:31:29.455613+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:31:29.455613+0000","last_undegraded":"2023-10-01T09:31:29.455613+0000","last_fullsized":"2023-10-01T09:31:29.455613+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":361,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.458047+0000","last_change":"2023-10-01T09:27:01.986786+0000","last_active":"2023-10-01T09:31:29.458047+0000","last_peered":"2023-10-01T09:31:29.458047+0000","last_clean":"2023-10-01T09:31:29.458047+0000","last_became_active":"2023-10-01T09:27:01.986466+0000","last_became_peered":"2023-10-01T09:27:01.986466+0000","last_unstale":"2023-10-01T09:31:29.458047+0000","last_undegraded":"2023-10-01T09:31:29.458047+0000","last_fullsized":"2023-10-01T09:31:29.458047+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:55:51.244125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":361,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.459353+0000","last_change":"2023-10-01T09:19:30.091481+0000","last_active":"2023-10-01T09:31:29.459353+0000","last_peered":"2023-10-01T09:31:29.459353+0000","last_clean":"2023-10-01T09:31:29.459353+0000","last_became_active":"2023-10-01T09:19:30.091151+0000","last_became_peered":"2023-10-01T09:19:30.091151+0000","last_unstale":"2023-10-01T09:31:29.459353+0000","last_undegraded":"2023-10-01T09:31:29.459353+0000","last_fullsized":"2023-10-01T09:31:29.459353+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:09.127408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"355'8112","reported_seq":8158,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:43.395541+0000","last_change":"2023-10-01T09:30:39.412276+0000","last_active":"2023-10-01T09:31:43.395541+0000","last_peered":"2023-10-01T09:31:43.395541+0000","last_clean":"2023-10-01T09:31:43.395541+0000","last_became_active":"2023-10-01T09:30:39.400521+0000","last_became_peered":"2023-10-01T09:30:39.400521+0000","last_unstale":"2023-10-01T09:31:43.395541+0000","last_undegraded":"2023-10-01T09:31:43.395541+0000","last_fullsized":"2023-10-01T09:31:43.395541+0000","mapping_epoch":343,"log_start":"332'1700","ondisk_log_start":"332'1700","created":326,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":6412,"log_dups_size":0,"ondisk_log_size":6412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:35.349863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299237376,"num_objects":4566,"num_object_clones":0,"num_object_copies":13698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8112,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":754,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.456046+0000","last_change":"2023-10-01T09:28:03.317559+0000","last_active":"2023-10-01T09:31:29.456046+0000","last_peered":"2023-10-01T09:31:29.456046+0000","last_clean":"2023-10-01T09:31:29.456046+0000","last_became_active":"2023-10-01T09:28:03.317237+0000","last_became_peered":"2023-10-01T09:28:03.317237+0000","last_unstale":"2023-10-01T09:31:29.456046+0000","last_undegraded":"2023-10-01T09:31:29.456046+0000","last_fullsized":"2023-10-01T09:31:29.456046+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":375,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.463740+0000","last_change":"2023-10-01T09:27:01.999110+0000","last_active":"2023-10-01T09:31:29.463740+0000","last_peered":"2023-10-01T09:31:29.463740+0000","last_clean":"2023-10-01T09:31:29.463740+0000","last_became_active":"2023-10-01T09:27:01.998740+0000","last_became_peered":"2023-10-01T09:27:01.998740+0000","last_unstale":"2023-10-01T09:31:29.463740+0000","last_undegraded":"2023-10-01T09:31:29.463740+0000","last_fullsized":"2023-10-01T09:31:29.463740+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:40:08.795106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":328,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.458001+0000","last_change":"2023-10-01T09:27:01.998986+0000","last_active":"2023-10-01T09:31:29.458001+0000","last_peered":"2023-10-01T09:31:29.458001+0000","last_clean":"2023-10-01T09:31:29.458001+0000","last_became_active":"2023-10-01T09:27:01.998709+0000","last_became_peered":"2023-10-01T09:27:01.998709+0000","last_unstale":"2023-10-01T09:31:29.458001+0000","last_undegraded":"2023-10-01T09:31:29.458001+0000","last_fullsized":"2023-10-01T09:31:29.458001+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:01:14.296638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"355'8347","reported_seq":8386,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:43.596931+0000","last_change":"2023-10-01T09:29:24.378898+0000","last_active":"2023-10-01T09:31:43.596931+0000","last_peered":"2023-10-01T09:31:43.596931+0000","last_clean":"2023-10-01T09:31:43.596931+0000","last_became_active":"2023-10-01T09:29:24.378624+0000","last_became_peered":"2023-10-01T09:29:24.378624+0000","last_unstale":"2023-10-01T09:31:43.596931+0000","last_undegraded":"2023-10-01T09:31:43.596931+0000","last_fullsized":"2023-10-01T09:31:43.596931+0000","mapping_epoch":326,"log_start":"331'800","ondisk_log_start":"331'800","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":7547,"log_dups_size":0,"ondisk_log_size":7547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:12:28.028735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308084736,"num_objects":4701,"num_object_clones":0,"num_object_copies":14103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8347,"num_write_kb":417536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":268,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.456052+0000","last_change":"2023-10-01T09:30:39.402924+0000","last_active":"2023-10-01T09:31:29.456052+0000","last_peered":"2023-10-01T09:31:29.456052+0000","last_clean":"2023-10-01T09:31:29.456052+0000","last_became_active":"2023-10-01T09:30:39.402594+0000","last_became_peered":"2023-10-01T09:30:39.402594+0000","last_unstale":"2023-10-01T09:31:29.456052+0000","last_undegraded":"2023-10-01T09:31:29.456052+0000","last_fullsized":"2023-10-01T09:31:29.456052+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":941,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.048362+0000","last_change":"2023-10-01T09:29:13.014839+0000","last_active":"2023-10-01T09:31:37.048362+0000","last_peered":"2023-10-01T09:31:37.048362+0000","last_clean":"2023-10-01T09:31:37.048362+0000","last_became_active":"2023-10-01T09:29:13.014139+0000","last_became_peered":"2023-10-01T09:29:13.014139+0000","last_unstale":"2023-10-01T09:31:37.048362+0000","last_undegraded":"2023-10-01T09:31:37.048362+0000","last_fullsized":"2023-10-01T09:31:37.048362+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:23:10.370435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":44,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.455509+0000","last_change":"2023-10-01T09:30:39.418085+0000","last_active":"2023-10-01T09:31:29.455509+0000","last_peered":"2023-10-01T09:31:29.455509+0000","last_clean":"2023-10-01T09:31:29.455509+0000","last_became_active":"2023-10-01T09:30:39.417846+0000","last_became_peered":"2023-10-01T09:30:39.417846+0000","last_unstale":"2023-10-01T09:31:29.455509+0000","last_undegraded":"2023-10-01T09:31:29.455509+0000","last_fullsized":"2023-10-01T09:31:29.455509+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"355'7867","reported_seq":7905,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:40.718007+0000","last_change":"2023-10-01T09:29:24.379832+0000","last_active":"2023-10-01T09:31:40.718007+0000","last_peered":"2023-10-01T09:31:40.718007+0000","last_clean":"2023-10-01T09:31:40.718007+0000","last_became_active":"2023-10-01T09:29:24.379540+0000","last_became_peered":"2023-10-01T09:29:24.379540+0000","last_unstale":"2023-10-01T09:31:40.718007+0000","last_undegraded":"2023-10-01T09:31:40.718007+0000","last_fullsized":"2023-10-01T09:31:40.718007+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":7867,"log_dups_size":0,"ondisk_log_size":7867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301596696,"num_objects":4603,"num_object_clones":0,"num_object_copies":13809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4603,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7867,"num_write_kb":398977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":68,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.455364+0000","last_change":"2023-10-01T09:29:11.374086+0000","last_active":"2023-10-01T09:31:29.455364+0000","last_peered":"2023-10-01T09:31:29.455364+0000","last_clean":"2023-10-01T09:31:29.455364+0000","last_became_active":"2023-10-01T09:28:02.319106+0000","last_became_peered":"2023-10-01T09:28:02.319106+0000","last_unstale":"2023-10-01T09:31:29.455364+0000","last_undegraded":"2023-10-01T09:31:29.455364+0000","last_fullsized":"2023-10-01T09:31:29.455364+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":353,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.459441+0000","last_change":"2023-10-01T09:26:12.291583+0000","last_active":"2023-10-01T09:31:29.459441+0000","last_peered":"2023-10-01T09:31:29.459441+0000","last_clean":"2023-10-01T09:31:29.459441+0000","last_became_active":"2023-10-01T09:26:12.271891+0000","last_became_peered":"2023-10-01T09:26:12.271891+0000","last_unstale":"2023-10-01T09:31:29.459441+0000","last_undegraded":"2023-10-01T09:31:29.459441+0000","last_fullsized":"2023-10-01T09:31:29.459441+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:53:06.889203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":75,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.463690+0000","last_change":"2023-10-01T09:28:02.314867+0000","last_active":"2023-10-01T09:31:29.463690+0000","last_peered":"2023-10-01T09:31:29.463690+0000","last_clean":"2023-10-01T09:31:29.463690+0000","last_became_active":"2023-10-01T09:28:02.314480+0000","last_became_peered":"2023-10-01T09:28:02.314480+0000","last_unstale":"2023-10-01T09:31:29.463690+0000","last_undegraded":"2023-10-01T09:31:29.463690+0000","last_fullsized":"2023-10-01T09:31:29.463690+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":307,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.457925+0000","last_change":"2023-10-01T09:26:12.271469+0000","last_active":"2023-10-01T09:31:29.457925+0000","last_peered":"2023-10-01T09:31:29.457925+0000","last_clean":"2023-10-01T09:31:29.457925+0000","last_became_active":"2023-10-01T09:26:12.270972+0000","last_became_peered":"2023-10-01T09:26:12.270972+0000","last_unstale":"2023-10-01T09:31:29.457925+0000","last_undegraded":"2023-10-01T09:31:29.457925+0000","last_fullsized":"2023-10-01T09:31:29.457925+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:45:03.822952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":385,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.463812+0000","last_change":"2023-10-01T09:22:40.166927+0000","last_active":"2023-10-01T09:31:29.463812+0000","last_peered":"2023-10-01T09:31:29.463812+0000","last_clean":"2023-10-01T09:31:29.463812+0000","last_became_active":"2023-10-01T09:22:40.166606+0000","last_became_peered":"2023-10-01T09:22:40.166606+0000","last_unstale":"2023-10-01T09:31:29.463812+0000","last_undegraded":"2023-10-01T09:31:29.463812+0000","last_fullsized":"2023-10-01T09:31:29.463812+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:54:18.044207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":83,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.464101+0000","last_change":"2023-10-01T09:28:02.507232+0000","last_active":"2023-10-01T09:31:29.464101+0000","last_peered":"2023-10-01T09:31:29.464101+0000","last_clean":"2023-10-01T09:31:29.464101+0000","last_became_active":"2023-10-01T09:28:02.340776+0000","last_became_peered":"2023-10-01T09:28:02.340776+0000","last_unstale":"2023-10-01T09:31:29.464101+0000","last_undegraded":"2023-10-01T09:31:29.464101+0000","last_fullsized":"2023-10-01T09:31:29.464101+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":29,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.463687+0000","last_change":"2023-10-01T09:29:58.359570+0000","last_active":"2023-10-01T09:31:29.463687+0000","last_peered":"2023-10-01T09:31:29.463687+0000","last_clean":"2023-10-01T09:31:29.463687+0000","last_became_active":"2023-10-01T09:29:58.359181+0000","last_became_peered":"2023-10-01T09:29:58.359181+0000","last_unstale":"2023-10-01T09:31:29.463687+0000","last_undegraded":"2023-10-01T09:31:29.463687+0000","last_fullsized":"2023-10-01T09:31:29.463687+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":83,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.464107+0000","last_change":"2023-10-01T09:28:02.551019+0000","last_active":"2023-10-01T09:31:29.464107+0000","last_peered":"2023-10-01T09:31:29.464107+0000","last_clean":"2023-10-01T09:31:29.464107+0000","last_became_active":"2023-10-01T09:28:02.342509+0000","last_became_peered":"2023-10-01T09:28:02.342509+0000","last_unstale":"2023-10-01T09:31:29.464107+0000","last_undegraded":"2023-10-01T09:31:29.464107+0000","last_fullsized":"2023-10-01T09:31:29.464107+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":235,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.458653+0000","last_change":"2023-10-01T09:30:39.397184+0000","last_active":"2023-10-01T09:31:29.458653+0000","last_peered":"2023-10-01T09:31:29.458653+0000","last_clean":"2023-10-01T09:31:29.458653+0000","last_became_active":"2023-10-01T09:30:39.396907+0000","last_became_peered":"2023-10-01T09:30:39.396907+0000","last_unstale":"2023-10-01T09:31:29.458653+0000","last_undegraded":"2023-10-01T09:31:29.458653+0000","last_fullsized":"2023-10-01T09:31:29.458653+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:16:14.113046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":157,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.464134+0000","last_change":"2023-10-01T09:28:02.321395+0000","last_active":"2023-10-01T09:31:29.464134+0000","last_peered":"2023-10-01T09:31:29.464134+0000","last_clean":"2023-10-01T09:31:29.464134+0000","last_became_active":"2023-10-01T09:28:02.321071+0000","last_became_peered":"2023-10-01T09:28:02.321071+0000","last_unstale":"2023-10-01T09:31:29.464134+0000","last_undegraded":"2023-10-01T09:31:29.464134+0000","last_fullsized":"2023-10-01T09:31:29.464134+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:12:45.734920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":262,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.457979+0000","last_change":"2023-10-01T09:30:39.392942+0000","last_active":"2023-10-01T09:31:29.457979+0000","last_peered":"2023-10-01T09:31:29.457979+0000","last_clean":"2023-10-01T09:31:29.457979+0000","last_became_active":"2023-10-01T09:30:39.392663+0000","last_became_peered":"2023-10-01T09:30:39.392663+0000","last_unstale":"2023-10-01T09:31:29.457979+0000","last_undegraded":"2023-10-01T09:31:29.457979+0000","last_fullsized":"2023-10-01T09:31:29.457979+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:19:39.644244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":445,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.458348+0000","last_change":"2023-10-01T09:27:02.019266+0000","last_active":"2023-10-01T09:31:29.458348+0000","last_peered":"2023-10-01T09:31:29.458348+0000","last_clean":"2023-10-01T09:31:29.458348+0000","last_became_active":"2023-10-01T09:27:02.006744+0000","last_became_peered":"2023-10-01T09:27:02.006744+0000","last_unstale":"2023-10-01T09:31:29.458348+0000","last_undegraded":"2023-10-01T09:31:29.458348+0000","last_fullsized":"2023-10-01T09:31:29.458348+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:39:21.899359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":313,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-10-01T09:31:29.464902+0000","last_change":"2023-10-01T09:28:02.343724+0000","last_active":"2023-10-01T09:31:29.464902+0000","last_peered":"2023-10-01T09:31:29.464902+0000","last_clean":"2023-10-01T09:31:29.464902+0000","last_became_active":"2023-10-01T09:28:02.339485+0000","last_became_peered":"2023-10-01T09:28:02.339485+0000","last_unstale":"2023-10-01T09:31:29.464902+0000","last_undegraded":"2023-10-01T09:31:29.464902+0000","last_fullsized":"2023-10-01T09:31:29.464902+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:54:52.086601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":97,"num_bytes_recovered":28418,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":22016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42916,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4834066456,"num_objects":73763,"num_object_clones":0,"num_object_copies":221289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73763,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":129055,"num_write_kb":6490113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14575087616,"data_stored":14575075400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116255,"ondisk_log_size":116255,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":209,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430573,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2402904,"kb_used_data":1805008,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":91342248,"statfs":{"total":95995035648,"available":93534461952,"internally_reserved":0,"allocated":1848328192,"data_stored":1847446664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:31:27 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":2.1850000000000001,"15min":3.903},"min":{"1min":0.249,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":7.056,"5min":64.411000000000001,"15min":117.855},"last":3.1480000000000001},{"interface":"front","average":{"1min":1.804,"5min":2.6139999999999999,"15min":3.9590000000000001},"min":{"1min":0.27300000000000002,"5min":0.23899999999999999,"15min":0.22500000000000001},"max":{"1min":3.4420000000000002,"5min":64.519000000000005,"15min":120.348},"last":0.90600000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:30:45 2023","interfaces":[{"interface":"back","average":{"1min":1.665,"5min":1.881,"15min":3.3260000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":6.899,"5min":10.611000000000001,"15min":135.023},"last":4.181},{"interface":"front","average":{"1min":1.4410000000000001,"5min":1.881,"15min":3.1320000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.23599999999999999},"max":{"1min":5.6619999999999999,"5min":13.688000000000001,"15min":134.51499999999999},"last":0.61199999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:31:18 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.8380000000000001,"15min":2.899},"min":{"1min":0.27500000000000002,"5min":0.249,"15min":0.249},"max":{"1min":3.3199999999999998,"5min":10.452,"15min":104.837},"last":3.8530000000000002},{"interface":"front","average":{"1min":2.581,"5min":1.841,"15min":2.8500000000000001},"min":{"1min":0.308,"5min":0.27400000000000002,"15min":0.247},"max":{"1min":10.026,"5min":10.026,"15min":106.379},"last":0.64200000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:30:45 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":4.641,"15min":3.3500000000000001},"min":{"1min":0.30199999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.1870000000000003,"5min":164.595,"15min":164.595},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.6479999999999999,"5min":4.577,"15min":3.3069999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.29399999999999998},"max":{"1min":7.1909999999999998,"5min":164.60499999999999,"15min":164.60499999999999},"last":0.44500000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:31:27 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":0.60699999999999998,"15min":1.542},"min":{"1min":0.28399999999999997,"5min":0.254,"15min":0.23499999999999999},"max":{"1min":3.9740000000000002,"5min":3.9740000000000002,"15min":82.992000000000004},"last":3.109},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.65400000000000003,"15min":1.603},"min":{"1min":0.29199999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.1859999999999999,"5min":3.25,"15min":83.058999999999997},"last":0.56399999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:31:27 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.70599999999999996,"15min":1.8240000000000001},"min":{"1min":0.27800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.3820000000000001,"5min":4.6989999999999998,"15min":13.265000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.72699999999999998,"15min":1.762},"min":{"1min":0.317,"5min":0.311,"15min":0.27600000000000002},"max":{"1min":3.1070000000000002,"5min":3.7639999999999998,"15min":13.292999999999999},"last":0.46800000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:30:59 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.77600000000000002,"15min":0.79200000000000004},"min":{"1min":0.34599999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.5779999999999998,"5min":3.5779999999999998,"15min":3.5779999999999998},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.75900000000000001,"15min":0.65700000000000003},"min":{"1min":0.37,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.3860000000000001,"5min":6.3860000000000001,"15min":6.3860000000000001},"last":0.498}]}]},{"osd":6,"up_from":294,"seq":1262720385084,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2117640,"kb_used_data":1784128,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":91627512,"statfs":{"total":95995035648,"available":93826572288,"internally_reserved":0,"allocated":1826947072,"data_stored":1826081181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":4.8410000000000002,"5min":2.294,"15min":2.1120000000000001},"min":{"1min":0.23799999999999999,"5min":0.222,"15min":0.222},"max":{"1min":62.569000000000003,"5min":62.569000000000003,"15min":62.569000000000003},"last":1.173},{"interface":"front","average":{"1min":4.7130000000000001,"5min":2.3220000000000001,"15min":1.7410000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":62.912999999999997,"5min":62.912999999999997,"15min":62.912999999999997},"last":4.1550000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":5.0270000000000001,"5min":2.2599999999999998,"15min":1.968},"min":{"1min":0.29299999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":62.698999999999998,"5min":62.698999999999998,"15min":62.698999999999998},"last":1.071},{"interface":"front","average":{"1min":4.8769999999999998,"5min":2.1230000000000002,"15min":1.9359999999999999},"min":{"1min":0.34899999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":62.704000000000001,"5min":62.704000000000001,"15min":62.704000000000001},"last":1.081}]},{"osd":2,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8799999999999999,"5min":2.21,"15min":2.738},"min":{"1min":0.29799999999999999,"5min":0.254,"15min":0.254},"max":{"1min":6.9889999999999999,"5min":26.289999999999999,"15min":26.289999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.6830000000000001,"5min":2.2360000000000002,"15min":2.8879999999999999},"min":{"1min":0.315,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.8170000000000002,"5min":26.388999999999999,"15min":26.388999999999999},"last":0.371}]},{"osd":3,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":4.3470000000000004,"5min":2.7130000000000001,"15min":2.5489999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":62.546999999999997,"5min":62.546999999999997,"15min":62.546999999999997},"last":0.72899999999999998},{"interface":"front","average":{"1min":4.7709999999999999,"5min":2.1640000000000001,"15min":1.6799999999999999},"min":{"1min":0.35599999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":62.465000000000003,"5min":62.465000000000003,"15min":62.465000000000003},"last":1.05}]},{"osd":4,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.496,"15min":0.41299999999999998},"min":{"1min":0.30099999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.9059999999999999,"5min":3.1560000000000001,"15min":3.1560000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.621,"15min":0.56100000000000005},"min":{"1min":0.35599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.75,"5min":4.1319999999999997,"15min":4.1319999999999997},"last":0.42399999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.59199999999999997,"15min":0.47899999999999998},"min":{"1min":0.28999999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.9450000000000001,"5min":4.6200000000000001,"15min":4.6200000000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.73699999999999999,"15min":0.624},"min":{"1min":0.30299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.4900000000000002,"5min":4.4900000000000002,"15min":4.4900000000000002},"last":0.34200000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":3.1499999999999999,"15min":4.8040000000000003},"min":{"1min":0.33100000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.861,"5min":115.11,"15min":115.11},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.873,"5min":3.177,"15min":4.7690000000000001},"min":{"1min":0.34499999999999997,"5min":0.309,"15min":0.309},"max":{"1min":3.3599999999999999,"5min":114.666,"15min":114.666},"last":0.40899999999999997}]}]},{"osd":1,"up_from":94,"seq":403726926061,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2557552,"kb_used_data":2058088,"kb_used_omap":7,"kb_used_meta":499448,"kb_avail":91187600,"statfs":{"total":95995035648,"available":93376102400,"internally_reserved":0,"allocated":2107482112,"data_stored":2105882202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":511435175},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:31:36 2023","interfaces":[{"interface":"back","average":{"1min":3.6000000000000001,"5min":3.8570000000000002,"15min":4.327},"min":{"1min":0.19900000000000001,"5min":0.19600000000000001,"15min":0.17499999999999999},"max":{"1min":11.266,"5min":82.359999999999999,"15min":174.40199999999999},"last":5.9580000000000002},{"interface":"front","average":{"1min":3.9380000000000002,"5min":3.3530000000000002,"15min":4.4790000000000001},"min":{"1min":0.25800000000000001,"5min":0.188,"15min":0.17599999999999999},"max":{"1min":12.978,"5min":82.078999999999994,"15min":173.77000000000001},"last":5.4109999999999996}]},{"osd":2,"last update":"Sun Oct 1 09:31:18 2023","interfaces":[{"interface":"back","average":{"1min":4.1070000000000002,"5min":4.6920000000000002,"15min":3.4079999999999999},"min":{"1min":0.218,"5min":0.215,"15min":0.182},"max":{"1min":13.468,"5min":56.573,"15min":77.325000000000003},"last":1.4870000000000001},{"interface":"front","average":{"1min":3.3260000000000001,"5min":4.9790000000000001,"15min":3.7730000000000001},"min":{"1min":0.28599999999999998,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":12.878,"5min":55.930999999999997,"15min":80.361000000000004},"last":5.5860000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:30:49 2023","interfaces":[{"interface":"back","average":{"1min":12.676,"5min":6.0679999999999996,"15min":4.2830000000000004},"min":{"1min":0.30399999999999999,"5min":0.28499999999999998,"15min":0.222},"max":{"1min":200.56800000000001,"5min":200.56800000000001,"15min":200.56800000000001},"last":5.3289999999999997},{"interface":"front","average":{"1min":12.613,"5min":6.117,"15min":4.0860000000000003},"min":{"1min":0.316,"5min":0.25600000000000001,"15min":0.20999999999999999},"max":{"1min":200.63800000000001,"5min":200.63800000000001,"15min":200.63800000000001},"last":5.4909999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:31:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.3300000000000001,"15min":1.6379999999999999},"min":{"1min":0.33000000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.6900000000000004,"5min":7.1669999999999998,"15min":40.673999999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.5950000000000002,"5min":2.2490000000000001,"15min":2.0329999999999999},"min":{"1min":0.44500000000000001,"5min":0.29799999999999999,"15min":0.25800000000000001},"max":{"1min":9.298,"5min":18.219000000000001,"15min":40.810000000000002},"last":5.4500000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:31:18 2023","interfaces":[{"interface":"back","average":{"1min":3.8410000000000002,"5min":2.0099999999999998,"15min":1.6060000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":47.106000000000002,"5min":47.106000000000002,"15min":47.106000000000002},"last":1.782},{"interface":"front","average":{"1min":4.5519999999999996,"5min":2.794,"15min":2.3580000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":47.164999999999999,"5min":47.164999999999999,"15min":47.164999999999999},"last":1.585}]},{"osd":6,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":2.2989999999999999,"15min":2.9119999999999999},"min":{"1min":0.35699999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.3019999999999996,"5min":10.196999999999999,"15min":10.196999999999999},"last":1.9419999999999999},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.661,"15min":1.9059999999999999},"min":{"1min":0.41399999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.8109999999999999,"5min":7.2679999999999998,"15min":7.2679999999999998},"last":0.33300000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:31:36 2023","interfaces":[{"interface":"back","average":{"1min":2.1389999999999998,"5min":1.5840000000000001,"15min":1.8620000000000001},"min":{"1min":0.39500000000000002,"5min":0.36799999999999999,"15min":0.316},"max":{"1min":14.717000000000001,"5min":14.717000000000001,"15min":23.710999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.4750000000000001,"5min":2.2010000000000001,"15min":2.0190000000000001},"min":{"1min":0.42899999999999999,"5min":0.34599999999999997,"15min":0.30199999999999999},"max":{"1min":9.25,"5min":18.291,"15min":23.736999999999998},"last":1.8200000000000001}]}]},{"osd":0,"up_from":89,"seq":382252089595,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2708036,"kb_used_data":2069372,"kb_used_omap":8,"kb_used_meta":638519,"kb_avail":91037116,"statfs":{"total":95995035648,"available":93222006784,"internally_reserved":0,"allocated":2119036928,"data_stored":2117421815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":653844418},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":2.081,"15min":2.8319999999999999},"min":{"1min":0.20499999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":8.3260000000000005,"5min":11.683,"15min":187.41399999999999},"last":10.597},{"interface":"front","average":{"1min":3.016,"5min":3.5670000000000002,"15min":3.6179999999999999},"min":{"1min":0.22500000000000001,"5min":0.21199999999999999,"15min":0.193},"max":{"1min":9.2750000000000004,"5min":21.968,"15min":187.78700000000001},"last":0.873}]},{"osd":2,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.792,"5min":1.51,"15min":2.0920000000000001},"min":{"1min":0.23000000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":11.109,"5min":11.468999999999999,"15min":39.048999999999999},"last":10.555},{"interface":"front","average":{"1min":2.1579999999999999,"5min":3.093,"15min":3.0299999999999998},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":8.2149999999999999,"5min":22.07,"15min":38.945999999999998},"last":10.853999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:30:41 2023","interfaces":[{"interface":"back","average":{"1min":4.0830000000000002,"5min":5.282,"15min":3.4780000000000002},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":38.953000000000003,"5min":139.85300000000001,"15min":139.85300000000001},"last":3.931},{"interface":"front","average":{"1min":4.5359999999999996,"5min":6.6050000000000004,"15min":4.1429999999999998},"min":{"1min":0.27000000000000002,"5min":0.219,"15min":0.219},"max":{"1min":38.997,"5min":139.70099999999999,"15min":139.70099999999999},"last":7.5330000000000004}]},{"osd":4,"last update":"Sun Oct 1 09:31:33 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":2.302,"15min":2.5219999999999998},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.27300000000000002},"max":{"1min":4.8710000000000004,"5min":18.600999999999999,"15min":112.52200000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.5680000000000001,"5min":2.3250000000000002,"15min":2.5459999999999998},"min":{"1min":0.33700000000000002,"5min":0.28799999999999998,"15min":0.26900000000000002},"max":{"1min":5.2249999999999996,"5min":28.035,"15min":113.124},"last":3.2109999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:31:23 2023","interfaces":[{"interface":"back","average":{"1min":8.8879999999999999,"5min":4.0289999999999999,"15min":2.4500000000000002},"min":{"1min":0.32900000000000001,"5min":0.317,"15min":0.317},"max":{"1min":122.825,"5min":122.825,"15min":122.825},"last":3.1789999999999998},{"interface":"front","average":{"1min":9.4090000000000007,"5min":2.806,"15min":2.0659999999999998},"min":{"1min":0.42599999999999999,"5min":0.312,"15min":0.31},"max":{"1min":122.837,"5min":122.837,"15min":122.837},"last":3.2469999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:31:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2969999999999999,"5min":1.23,"15min":1.3959999999999999},"min":{"1min":0.39300000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":5.3600000000000003,"5min":11.387,"15min":11.387},"last":10.452},{"interface":"front","average":{"1min":1.8069999999999999,"5min":2.266,"15min":2.4260000000000002},"min":{"1min":0.32700000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.1419999999999995,"5min":18.521000000000001,"15min":18.521000000000001},"last":0.372}]},{"osd":7,"last update":"Sun Oct 1 09:31:33 2023","interfaces":[{"interface":"back","average":{"1min":2.2290000000000001,"5min":1.456,"15min":2.0339999999999998},"min":{"1min":0.41199999999999998,"5min":0.34499999999999997,"15min":0.29199999999999998},"max":{"1min":10.991,"5min":11.413,"15min":98.283000000000001},"last":3.3170000000000002},{"interface":"front","average":{"1min":1.71,"5min":2.52,"15min":2.573},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.312},"max":{"1min":4.835,"5min":18.565999999999999,"15min":98.311999999999998},"last":0.436}]}]},{"osd":2,"up_from":99,"seq":425201762534,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2735324,"kb_used_data":2118164,"kb_used_omap":11,"kb_used_meta":617076,"kb_avail":91009828,"statfs":{"total":95995035648,"available":93194063872,"internally_reserved":0,"allocated":2168999936,"data_stored":2167423216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":631886711},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:31:27 2023","interfaces":[{"interface":"back","average":{"1min":3.242,"5min":8.0139999999999993,"15min":5.2130000000000001},"min":{"1min":0.221,"5min":0.20499999999999999,"15min":0.17599999999999999},"max":{"1min":10.510999999999999,"5min":224.56,"15min":224.56},"last":3.2440000000000002},{"interface":"front","average":{"1min":2.6499999999999999,"5min":8.5129999999999999,"15min":5.8369999999999997},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20100000000000001},"max":{"1min":6.702,"5min":225.66900000000001,"15min":225.66900000000001},"last":0.95499999999999996}]},{"osd":1,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":3.3839999999999999,"5min":9.1959999999999997,"15min":5.6710000000000003},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":14.653,"5min":224.917,"15min":224.917},"last":1.8480000000000001},{"interface":"front","average":{"1min":3.9420000000000002,"5min":10.529999999999999,"15min":6.4569999999999999},"min":{"1min":0.375,"5min":0.222,"15min":0.20899999999999999},"max":{"1min":13.664,"5min":225.625,"15min":225.625},"last":0.57999999999999996}]},{"osd":3,"last update":"Sun Oct 1 09:30:46 2023","interfaces":[{"interface":"back","average":{"1min":11.696,"5min":8.1530000000000005,"15min":5.9980000000000002},"min":{"1min":0.23699999999999999,"5min":0.218,"15min":0.19600000000000001},"max":{"1min":226.124,"5min":226.124,"15min":253.238},"last":1.3919999999999999},{"interface":"front","average":{"1min":12.888999999999999,"5min":8.3019999999999996,"15min":5.8570000000000002},"min":{"1min":0.39400000000000002,"5min":0.245,"15min":0.22600000000000001},"max":{"1min":226.32900000000001,"5min":226.32900000000001,"15min":252.54900000000001},"last":1.9159999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":3.6539999999999999,"15min":2.7719999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.27200000000000002},"max":{"1min":5.9349999999999996,"5min":224.90199999999999,"15min":224.90199999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":3.3330000000000002,"5min":4.1710000000000003,"15min":2.903},"min":{"1min":0.33900000000000002,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":17.594999999999999,"5min":225.386,"15min":225.386},"last":3.8109999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":4.2830000000000004,"5min":3.9980000000000002,"15min":2.7320000000000002},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":40.542000000000002,"5min":224.989,"15min":224.989},"last":1.8089999999999999},{"interface":"front","average":{"1min":3.5680000000000001,"5min":4.1589999999999998,"15min":2.875},"min":{"1min":0.34399999999999997,"5min":0.309,"15min":0.309},"max":{"1min":40.436,"5min":224.81,"15min":224.81},"last":0.48799999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:30:52 2023","interfaces":[{"interface":"back","average":{"1min":4.4829999999999997,"5min":4.5439999999999996,"15min":2.907},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":40.582999999999998,"5min":225.41499999999999,"15min":225.41499999999999},"last":4.2220000000000004},{"interface":"front","average":{"1min":4.6239999999999997,"5min":4.0609999999999999,"15min":2.274},"min":{"1min":0.42199999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":40.210999999999999,"5min":225.02000000000001,"15min":225.02000000000001},"last":1.879}]},{"osd":7,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":4.9530000000000003,"5min":4.5499999999999998,"15min":3.3359999999999999},"min":{"1min":0.38900000000000001,"5min":0.308,"15min":0.308},"max":{"1min":38.802,"5min":224.94999999999999,"15min":224.94999999999999},"last":2.8290000000000002},{"interface":"front","average":{"1min":3.3730000000000002,"5min":4.5750000000000002,"15min":3.5739999999999998},"min":{"1min":0.37,"5min":0.35299999999999998,"15min":0.33600000000000002},"max":{"1min":38.817,"5min":224.92599999999999,"15min":224.92599999999999},"last":0.59999999999999998}]}]},{"osd":3,"up_from":104,"seq":446676599008,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2642632,"kb_used_data":2090176,"kb_used_omap":3,"kb_used_meta":552444,"kb_avail":91102520,"statfs":{"total":95995035648,"available":93288980480,"internally_reserved":0,"allocated":2140340224,"data_stored":2138716955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":565703399},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.5819999999999999,"5min":5.2370000000000001,"15min":4.2229999999999999},"min":{"1min":0.26400000000000001,"5min":0.18099999999999999,"15min":0.17100000000000001},"max":{"1min":6.399,"5min":128.47800000000001,"15min":128.47800000000001},"last":6.8490000000000002},{"interface":"front","average":{"1min":3.964,"5min":4.9459999999999997,"15min":4.3579999999999997},"min":{"1min":0.28699999999999998,"5min":0.184,"15min":0.184},"max":{"1min":10.539,"5min":127.15600000000001,"15min":127.15600000000001},"last":0.68600000000000005}]},{"osd":1,"last update":"Sun Oct 1 09:30:45 2023","interfaces":[{"interface":"back","average":{"1min":3.407,"5min":3.0899999999999999,"15min":4.5350000000000001},"min":{"1min":0.251,"5min":0.23999999999999999,"15min":0.21099999999999999},"max":{"1min":11.000999999999999,"5min":28.277000000000001,"15min":228.60499999999999},"last":6.6779999999999999},{"interface":"front","average":{"1min":3.4329999999999998,"5min":4.343,"15min":4.7889999999999997},"min":{"1min":0.253,"5min":0.23400000000000001,"15min":0.215},"max":{"1min":12.958,"5min":33.982999999999997,"15min":228.28399999999999},"last":6.8239999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":4.4800000000000004,"15min":3.4460000000000002},"min":{"1min":0.25600000000000001,"5min":0.253,"15min":0.22500000000000001},"max":{"1min":6.1449999999999996,"5min":43.222999999999999,"15min":62.909999999999997},"last":0.42999999999999999},{"interface":"front","average":{"1min":3.1920000000000002,"5min":5.3840000000000003,"15min":3.9710000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.215},"max":{"1min":12.913,"5min":78.594999999999999,"15min":78.594999999999999},"last":6.8019999999999996}]},{"osd":4,"last update":"Sun Oct 1 09:30:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":1.9450000000000001,"15min":2.8359999999999999},"min":{"1min":0.372,"5min":0.29799999999999999,"15min":0.25900000000000001},"max":{"1min":6.5149999999999997,"5min":16,"15min":154.00399999999999},"last":6.7770000000000001},{"interface":"front","average":{"1min":2.7349999999999999,"5min":1.8300000000000001,"15min":3.1680000000000001},"min":{"1min":0.502,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.5069999999999997,"5min":15.257999999999999,"15min":156.87700000000001},"last":0.72299999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":7.516,"5min":2.867,"15min":2.1920000000000002},"min":{"1min":0.33300000000000002,"5min":0.29899999999999999,"15min":0.27400000000000002},"max":{"1min":79.822999999999993,"5min":79.822999999999993,"15min":79.822999999999993},"last":0.75},{"interface":"front","average":{"1min":7.5369999999999999,"5min":2.895,"15min":3.1720000000000002},"min":{"1min":0.41099999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":79.783000000000001,"5min":79.783000000000001,"15min":79.783000000000001},"last":0.79300000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:30:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":2.7240000000000002,"15min":3.008},"min":{"1min":0.38900000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.952,"5min":18.361000000000001,"15min":18.361000000000001},"last":0.77200000000000002},{"interface":"front","average":{"1min":1.9410000000000001,"5min":2.4489999999999998,"15min":2.9449999999999998},"min":{"1min":0.41999999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.1779999999999999,"5min":18.332000000000001,"15min":18.332000000000001},"last":0.83199999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:30:45 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":3.448,"15min":2.7240000000000002},"min":{"1min":0.42499999999999999,"5min":0.309,"15min":0.309},"max":{"1min":3.6760000000000002,"5min":52.006,"15min":79.412999999999997},"last":0.80500000000000005},{"interface":"front","average":{"1min":1.9990000000000001,"5min":2.8809999999999998,"15min":2.597},"min":{"1min":0.40500000000000003,"5min":0.34200000000000003,"15min":0.308},"max":{"1min":13.077999999999999,"5min":51.890999999999998,"15min":79.527000000000001},"last":6.7370000000000001}]}]},{"osd":4,"up_from":0,"seq":128849019258,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":227,"seq":974957576309,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2724480,"kb_used_data":2401080,"kb_used_omap":8,"kb_used_meta":323319,"kb_avail":91020672,"statfs":{"total":95995035648,"available":93205168128,"internally_reserved":0,"allocated":2458705920,"data_stored":2457843802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":331079428},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":12.558,"5min":6.0940000000000003,"15min":3.391},"min":{"1min":0.246,"5min":0.219,"15min":0.219},"max":{"1min":108.437,"5min":169.517,"15min":169.517},"last":1.8},{"interface":"front","average":{"1min":11.718,"5min":6.4729999999999999,"15min":3.7509999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.219},"max":{"1min":104.69199999999999,"5min":169.69900000000001,"15min":169.69900000000001},"last":1.952}]},{"osd":1,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":22.428000000000001,"5min":5.9000000000000004,"15min":4.1580000000000004},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":122.29000000000001,"5min":122.29000000000001,"15min":122.29000000000001},"last":0.83399999999999996},{"interface":"front","average":{"1min":22.638999999999999,"5min":5.7699999999999996,"15min":4.5110000000000001},"min":{"1min":0.26700000000000002,"5min":0.251,"15min":0.251},"max":{"1min":122.13500000000001,"5min":122.13500000000001,"15min":122.13500000000001},"last":0.28199999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.8400000000000001,"15min":2.4700000000000002},"min":{"1min":0.33500000000000002,"5min":0.27600000000000002,"15min":0.25900000000000001},"max":{"1min":4.2320000000000002,"5min":10.288,"15min":44.615000000000002},"last":2.7519999999999998},{"interface":"front","average":{"1min":1.804,"5min":1.861,"15min":2.0579999999999998},"min":{"1min":0.27900000000000003,"5min":0.26700000000000002,"15min":0.26600000000000001},"max":{"1min":6.3499999999999996,"5min":9.9440000000000008,"15min":44.511000000000003},"last":0.38500000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":6.7690000000000001,"5min":5.548,"15min":4.0599999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":85.387,"5min":214.73599999999999,"15min":214.73599999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":6.5979999999999999,"5min":5.7610000000000001,"15min":4.1379999999999999},"min":{"1min":0.33300000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":85.346999999999994,"5min":214.86699999999999,"15min":214.86699999999999},"last":0.40000000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":0.61699999999999999,"15min":1.2070000000000001},"min":{"1min":0.31900000000000001,"5min":0.26800000000000002,"15min":0.248},"max":{"1min":3.6989999999999998,"5min":3.6989999999999998,"15min":7.532},"last":0.316},{"interface":"front","average":{"1min":1.3200000000000001,"5min":0.68200000000000005,"15min":1.0700000000000001},"min":{"1min":0.29599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.3200000000000003,"5min":6.3200000000000003,"15min":6.3200000000000003},"last":0.55700000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:30:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":0.66500000000000004,"15min":0.54100000000000004},"min":{"1min":0.34399999999999997,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.4279999999999999,"5min":6.4279999999999999,"15min":6.4279999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.032,"5min":0.70999999999999996,"15min":0.54900000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.6419999999999999,"5min":4.806,"15min":4.806},"last":0.34300000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":2.3250000000000002,"15min":1.702},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.252},"max":{"1min":6.3769999999999998,"5min":84.105000000000004,"15min":84.105000000000004},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.3540000000000001,"5min":2.3319999999999999,"15min":1.9350000000000001},"min":{"1min":0.39800000000000002,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":4.4889999999999999,"5min":83.495000000000005,"15min":83.495000000000005},"last":0.36699999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2106589184,"data_stored":2106589184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2091646976,"data_stored":2091646976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2145849344,"data_stored":2145845272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2135293952,"data_stored":2135293952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2440171520,"data_stored":2440167448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1818099712,"data_stored":1818099712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1837436928,"data_stored":1837432856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:31:46.993 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:31:46.998 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:47.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:47.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:47.105 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:47.151 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:47.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:47 smithi190 ceph-mon[160435]: from='client.35850 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:31:47.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:47 smithi190 ceph-mon[160435]: pgmap v988: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 707 op/s 2023-10-01T09:31:47.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:47 smithi138 ceph-mon[192513]: from='client.35850 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:31:47.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:47 smithi138 ceph-mon[192513]: pgmap v988: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 707 op/s 2023-10-01T09:31:47.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:47 smithi138 ceph-mon[185886]: from='client.35850 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:31:47.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:47 smithi138 ceph-mon[185886]: pgmap v988: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 707 op/s 2023-10-01T09:31:47.631 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:47.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:47.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:47.743 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:31:47.782 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:47.841 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:31:47.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:47.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:31:47.947 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:48.007 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:48.501 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:48.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:48.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:48.612 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:48.655 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:48.722 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:48.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:48.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:48.727 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:48 smithi190 ceph-mon[160435]: pgmap v989: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 646 op/s 2023-10-01T09:31:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:48 smithi138 ceph-mon[192513]: pgmap v989: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 646 op/s 2023-10-01T09:31:48.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:48 smithi138 ceph-mon[185886]: pgmap v989: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 646 op/s 2023-10-01T09:31:48.827 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:48.908 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:49.109 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:49.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:49.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:49.222 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:31:49.257 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:49.341 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:49.452 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:49.620 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:31:49.620 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:31:49.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:31:49.621 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:31:49.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.621 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:101381 8.1 8:8aec2f48:::benchmark_data_smithi138_431617_object101380:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-01T09:31:49.621 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:03.305573+0000", 2023-10-01T09:31:49.621 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 46.301088606, 2023-10-01T09:31:49.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23178528400000001, 2023-10-01T09:31:49.621 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.621 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.621 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.621 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.622 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.622 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 101381 2023-10-01T09:31:49.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.622 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305573+0000", 2023-10-01T09:31:49.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305573+0000", 2023-10-01T09:31:49.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305572+0000", 2023-10-01T09:31:49.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:49.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305593+0000", 2023-10-01T09:31:49.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0383000000000001e-05 2023-10-01T09:31:49.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305600+0000", 2023-10-01T09:31:49.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.759e-06 2023-10-01T09:31:49.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305605+0000", 2023-10-01T09:31:49.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0039999999999999e-06 2023-10-01T09:31:49.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305833+0000", 2023-10-01T09:31:49.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00022808 2023-10-01T09:31:49.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305954+0000", 2023-10-01T09:31:49.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012141700000000001 2023-10-01T09:31:49.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:31:49.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.306082+0000", 2023-10-01T09:31:49.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012771700000000001 2023-10-01T09:31:49.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.317868+0000", 2023-10-01T09:31:49.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011785899000000001 2023-10-01T09:31:49.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.533371+0000", 2023-10-01T09:31:49.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21550293300000001 2023-10-01T09:31:49.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.537276+0000", 2023-10-01T09:31:49.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039056300000000002 2023-10-01T09:31:49.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.537321+0000", 2023-10-01T09:31:49.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4435999999999999e-05 2023-10-01T09:31:49.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.537358+0000", 2023-10-01T09:31:49.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7450999999999998e-05 2023-10-01T09:31:49.628 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.628 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.628 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.628 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:101383 8.f 8:f24c59be:::benchmark_data_smithi138_431617_object101382:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-01T09:31:49.628 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:03.305654+0000", 2023-10-01T09:31:49.628 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 46.301007411999997, 2023-10-01T09:31:49.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23159395999999999, 2023-10-01T09:31:49.628 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.629 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.629 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.629 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.629 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.629 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 101383 2023-10-01T09:31:49.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.629 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305654+0000", 2023-10-01T09:31:49.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305654+0000", 2023-10-01T09:31:49.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305654+0000", 2023-10-01T09:31:49.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:49.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305673+0000", 2023-10-01T09:31:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9423999999999999e-05 2023-10-01T09:31:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305679+0000", 2023-10-01T09:31:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9780000000000002e-06 2023-10-01T09:31:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305684+0000", 2023-10-01T09:31:49.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8659999999999997e-06 2023-10-01T09:31:49.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305889+0000", 2023-10-01T09:31:49.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000205317 2023-10-01T09:31:49.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.306043+0000", 2023-10-01T09:31:49.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015352900000000001 2023-10-01T09:31:49.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:31:49.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.306195+0000", 2023-10-01T09:31:49.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015257899999999999 2023-10-01T09:31:49.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.309094+0000", 2023-10-01T09:31:49.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028984470000000002 2023-10-01T09:31:49.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.533094+0000", 2023-10-01T09:31:49.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.224000541 2023-10-01T09:31:49.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.537168+0000", 2023-10-01T09:31:49.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040739460000000002 2023-10-01T09:31:49.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.537213+0000", 2023-10-01T09:31:49.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5178e-05 2023-10-01T09:31:49.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.537248+0000", 2023-10-01T09:31:49.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4584e-05 2023-10-01T09:31:49.635 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.635 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.635 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.635 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:101385 8.1 8:8315235e:::benchmark_data_smithi138_431617_object101384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-01T09:31:49.635 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:03.305689+0000", 2023-10-01T09:31:49.636 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 46.300972338000001, 2023-10-01T09:31:49.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.231793527, 2023-10-01T09:31:49.636 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.636 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.636 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.636 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.636 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.636 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 101385 2023-10-01T09:31:49.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.636 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305689+0000", 2023-10-01T09:31:49.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305689+0000", 2023-10-01T09:31:49.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305689+0000", 2023-10-01T09:31:49.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:49.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305708+0000", 2023-10-01T09:31:49.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9003000000000001e-05 2023-10-01T09:31:49.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305714+0000", 2023-10-01T09:31:49.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9100000000000002e-06 2023-10-01T09:31:49.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305719+0000", 2023-10-01T09:31:49.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1970000000000004e-06 2023-10-01T09:31:49.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.309396+0000", 2023-10-01T09:31:49.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036767340000000001 2023-10-01T09:31:49.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.310261+0000", 2023-10-01T09:31:49.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00086579199999999999 2023-10-01T09:31:49.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:31:49.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.310405+0000", 2023-10-01T09:31:49.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000143411 2023-10-01T09:31:49.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.317874+0000", 2023-10-01T09:31:49.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0074687579999999998 2023-10-01T09:31:49.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.533409+0000", 2023-10-01T09:31:49.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21553525800000001 2023-10-01T09:31:49.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.537396+0000", 2023-10-01T09:31:49.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039867110000000004 2023-10-01T09:31:49.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.537451+0000", 2023-10-01T09:31:49.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5046000000000001e-05 2023-10-01T09:31:49.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.537483+0000", 2023-10-01T09:31:49.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2061999999999999e-05 2023-10-01T09:31:49.642 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.642 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.642 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.643 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:101388 8.1 8:80241c4b:::benchmark_data_smithi138_431617_object101387:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-01T09:31:49.643 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:03.305766+0000", 2023-10-01T09:31:49.643 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 46.300895232000002, 2023-10-01T09:31:49.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23182377500000001, 2023-10-01T09:31:49.643 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.643 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.643 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.643 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.643 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.643 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 101388 2023-10-01T09:31:49.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.643 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305766+0000", 2023-10-01T09:31:49.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305766+0000", 2023-10-01T09:31:49.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305766+0000", 2023-10-01T09:31:49.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:49.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305785+0000", 2023-10-01T09:31:49.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.969e-05 2023-10-01T09:31:49.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305792+0000", 2023-10-01T09:31:49.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6989999999999997e-06 2023-10-01T09:31:49.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.305798+0000", 2023-10-01T09:31:49.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1800000000000001e-06 2023-10-01T09:31:49.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.314108+0000", 2023-10-01T09:31:49.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0083096369999999999 2023-10-01T09:31:49.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.314263+0000", 2023-10-01T09:31:49.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015540399999999999 2023-10-01T09:31:49.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:31:49.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.314404+0000", 2023-10-01T09:31:49.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000140901 2023-10-01T09:31:49.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.318123+0000", 2023-10-01T09:31:49.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037183680000000001 2023-10-01T09:31:49.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.533439+0000", 2023-10-01T09:31:49.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21531642600000001 2023-10-01T09:31:49.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.537519+0000", 2023-10-01T09:31:49.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040798070000000004 2023-10-01T09:31:49.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.537556+0000", 2023-10-01T09:31:49.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7357999999999997e-05 2023-10-01T09:31:49.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:03.537590+0000", 2023-10-01T09:31:49.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3787e-05 2023-10-01T09:31:49.649 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.649 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.649 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.650 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:115044 8.1 8:88246067:::benchmark_data_smithi138_431617_object13633:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-10-01T09:31:49.650 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:22.833666+0000", 2023-10-01T09:31:49.650 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 26.772995657999999, 2023-10-01T09:31:49.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.260692072, 2023-10-01T09:31:49.650 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.650 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.650 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.650 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.650 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.651 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 115044 2023-10-01T09:31:49.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.651 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.833666+0000", 2023-10-01T09:31:49.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.833666+0000", 2023-10-01T09:31:49.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.833665+0000", 2023-10-01T09:31:49.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:31:49.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.833670+0000", 2023-10-01T09:31:49.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6609999999999997e-06 2023-10-01T09:31:49.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.833684+0000", 2023-10-01T09:31:49.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.398e-05 2023-10-01T09:31:49.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.833691+0000", 2023-10-01T09:31:49.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8560000000000001e-06 2023-10-01T09:31:49.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.833784+0000", 2023-10-01T09:31:49.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3157000000000003e-05 2023-10-01T09:31:49.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.833938+0000", 2023-10-01T09:31:49.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015405200000000001 2023-10-01T09:31:49.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:31:49.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.834065+0000", 2023-10-01T09:31:49.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012623199999999999 2023-10-01T09:31:49.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.842787+0000", 2023-10-01T09:31:49.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0087222810000000001 2023-10-01T09:31:49.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.842803+0000", 2023-10-01T09:31:49.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6141999999999999e-05 2023-10-01T09:31:49.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:23.094265+0000", 2023-10-01T09:31:49.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25146233400000001 2023-10-01T09:31:49.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:23.094315+0000", 2023-10-01T09:31:49.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0010999999999999e-05 2023-10-01T09:31:49.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:23.094358+0000", 2023-10-01T09:31:49.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2505e-05 2023-10-01T09:31:49.657 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.657 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.657 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.657 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:115045 8.1 8:80ef9860:::benchmark_data_smithi138_431617_object13634:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-10-01T09:31:49.657 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:22.833706+0000", 2023-10-01T09:31:49.657 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 26.772955620000001, 2023-10-01T09:31:49.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.27075566099999998, 2023-10-01T09:31:49.657 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.657 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.658 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.658 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.658 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.658 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 115045 2023-10-01T09:31:49.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.658 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.833706+0000", 2023-10-01T09:31:49.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.833706+0000", 2023-10-01T09:31:49.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.833705+0000", 2023-10-01T09:31:49.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:49.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.833708+0000", 2023-10-01T09:31:49.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.497e-06 2023-10-01T09:31:49.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.833717+0000", 2023-10-01T09:31:49.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0420000000000008e-06 2023-10-01T09:31:49.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.833722+0000", 2023-10-01T09:31:49.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8280000000000002e-06 2023-10-01T09:31:49.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.839523+0000", 2023-10-01T09:31:49.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058016129999999997 2023-10-01T09:31:49.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.839626+0000", 2023-10-01T09:31:49.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010231899999999999 2023-10-01T09:31:49.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:31:49.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.839729+0000", 2023-10-01T09:31:49.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010287 2023-10-01T09:31:49.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.848439+0000", 2023-10-01T09:31:49.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0087103660000000006 2023-10-01T09:31:49.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.851764+0000", 2023-10-01T09:31:49.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003324961 2023-10-01T09:31:49.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:23.104373+0000", 2023-10-01T09:31:49.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25260878799999997 2023-10-01T09:31:49.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:23.104421+0000", 2023-10-01T09:31:49.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8062999999999997e-05 2023-10-01T09:31:49.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:23.104462+0000", 2023-10-01T09:31:49.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0741999999999997e-05 2023-10-01T09:31:49.664 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.664 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.664 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.664 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:115053 8.1 8:87ce8e9d:::benchmark_data_smithi138_431617_object13642:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-10-01T09:31:49.664 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:22.838259+0000", 2023-10-01T09:31:49.664 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 26.768402802000001, 2023-10-01T09:31:49.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26630721000000002, 2023-10-01T09:31:49.665 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.665 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.665 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.665 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.665 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.665 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 115053 2023-10-01T09:31:49.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.665 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.838259+0000", 2023-10-01T09:31:49.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.838259+0000", 2023-10-01T09:31:49.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.838258+0000", 2023-10-01T09:31:49.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:31:49.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.838262+0000", 2023-10-01T09:31:49.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8199999999999996e-06 2023-10-01T09:31:49.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.838282+0000", 2023-10-01T09:31:49.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0004999999999999e-05 2023-10-01T09:31:49.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.838290+0000", 2023-10-01T09:31:49.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5800000000000003e-06 2023-10-01T09:31:49.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.845251+0000", 2023-10-01T09:31:49.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069608710000000004 2023-10-01T09:31:49.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.845387+0000", 2023-10-01T09:31:49.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000135912 2023-10-01T09:31:49.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:31:49.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.845499+0000", 2023-10-01T09:31:49.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112081 2023-10-01T09:31:49.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.851777+0000", 2023-10-01T09:31:49.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0062778130000000001 2023-10-01T09:31:49.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.851800+0000", 2023-10-01T09:31:49.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3400999999999998e-05 2023-10-01T09:31:49.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:23.104498+0000", 2023-10-01T09:31:49.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25269768399999998 2023-10-01T09:31:49.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:23.104533+0000", 2023-10-01T09:31:49.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5682000000000002e-05 2023-10-01T09:31:49.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:23.104566+0000", 2023-10-01T09:31:49.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2484000000000003e-05 2023-10-01T09:31:49.671 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.671 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.671 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.672 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:115064 8.1 8:8f12f642:::benchmark_data_smithi138_431617_object13653:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-10-01T09:31:49.672 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:22.847360+0000", 2023-10-01T09:31:49.672 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 26.759301608000001, 2023-10-01T09:31:49.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.264997076, 2023-10-01T09:31:49.672 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.672 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.672 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.672 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.672 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.672 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 115064 2023-10-01T09:31:49.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.672 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.847360+0000", 2023-10-01T09:31:49.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.847360+0000", 2023-10-01T09:31:49.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.847359+0000", 2023-10-01T09:31:49.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:31:49.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.847363+0000", 2023-10-01T09:31:49.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3100000000000002e-06 2023-10-01T09:31:49.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.847373+0000", 2023-10-01T09:31:49.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0059e-05 2023-10-01T09:31:49.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.847380+0000", 2023-10-01T09:31:49.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8650000000000003e-06 2023-10-01T09:31:49.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.848460+0000", 2023-10-01T09:31:49.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010800740000000001 2023-10-01T09:31:49.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.848594+0000", 2023-10-01T09:31:49.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013376999999999999 2023-10-01T09:31:49.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:31:49.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.848730+0000", 2023-10-01T09:31:49.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013553600000000001 2023-10-01T09:31:49.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.851875+0000", 2023-10-01T09:31:49.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031450470000000002 2023-10-01T09:31:49.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:22.853143+0000", 2023-10-01T09:31:49.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012687760000000001 2023-10-01T09:31:49.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:23.112278+0000", 2023-10-01T09:31:49.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.25913457000000001 2023-10-01T09:31:49.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:23.112321+0000", 2023-10-01T09:31:49.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3290000000000001e-05 2023-10-01T09:31:49.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:23.112357+0000", 2023-10-01T09:31:49.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5663000000000001e-05 2023-10-01T09:31:49.678 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.678 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.679 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.679 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:119469 8.7 8:efcbd356:::benchmark_data_smithi138_431617_object18058:head [delete] snapc 0=[] ondisk+write+known_if_redirected e353)", 2023-10-01T09:31:49.679 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:28.756692+0000", 2023-10-01T09:31:49.679 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 20.849969772000001, 2023-10-01T09:31:49.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23298775799999999, 2023-10-01T09:31:49.679 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.679 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.679 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.679 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.679 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.679 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 119469 2023-10-01T09:31:49.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.680 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.756692+0000", 2023-10-01T09:31:49.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.756692+0000", 2023-10-01T09:31:49.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.756691+0000", 2023-10-01T09:31:49.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:49.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.756695+0000", 2023-10-01T09:31:49.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.014e-06 2023-10-01T09:31:49.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.756703+0000", 2023-10-01T09:31:49.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3750000000000003e-06 2023-10-01T09:31:49.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.756709+0000", 2023-10-01T09:31:49.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8309999999999997e-06 2023-10-01T09:31:49.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.772263+0000", 2023-10-01T09:31:49.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015554114000000001 2023-10-01T09:31:49.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.772393+0000", 2023-10-01T09:31:49.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013000300000000001 2023-10-01T09:31:49.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:31:49.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.772517+0000", 2023-10-01T09:31:49.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000123783 2023-10-01T09:31:49.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.776420+0000", 2023-10-01T09:31:49.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039023809999999999 2023-10-01T09:31:49.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.778328+0000", 2023-10-01T09:31:49.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0019088810000000001 2023-10-01T09:31:49.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989596+0000", 2023-10-01T09:31:49.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21126785000000001 2023-10-01T09:31:49.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989640+0000", 2023-10-01T09:31:49.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3275000000000002e-05 2023-10-01T09:31:49.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989679+0000", 2023-10-01T09:31:49.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9923999999999999e-05 2023-10-01T09:31:49.685 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.686 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.686 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.686 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:119495 8.a 8:5bfa501f:::benchmark_data_smithi138_431617_object18084:head [delete] snapc 0=[] ondisk+write+known_if_redirected e353)", 2023-10-01T09:31:49.686 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:28.788391+0000", 2023-10-01T09:31:49.686 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 20.818270623, 2023-10-01T09:31:49.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21253694100000001, 2023-10-01T09:31:49.686 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.686 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.686 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.686 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.687 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.687 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 119495 2023-10-01T09:31:49.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.687 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.687 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.788391+0000", 2023-10-01T09:31:49.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.687 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.788391+0000", 2023-10-01T09:31:49.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.788390+0000", 2023-10-01T09:31:49.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:31:49.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.788395+0000", 2023-10-01T09:31:49.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6710000000000004e-06 2023-10-01T09:31:49.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.788408+0000", 2023-10-01T09:31:49.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2597e-05 2023-10-01T09:31:49.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.788416+0000", 2023-10-01T09:31:49.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5499999999999995e-06 2023-10-01T09:31:49.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989576+0000", 2023-10-01T09:31:49.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.20115960599999999 2023-10-01T09:31:49.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989721+0000", 2023-10-01T09:31:49.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000145439 2023-10-01T09:31:49.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:31:49.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989860+0000", 2023-10-01T09:31:49.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013891899999999999 2023-10-01T09:31:49.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.993881+0000", 2023-10-01T09:31:49.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040206540000000002 2023-10-01T09:31:49.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.998840+0000", 2023-10-01T09:31:49.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049589940000000004 2023-10-01T09:31:49.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:29.000849+0000", 2023-10-01T09:31:49.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002009177 2023-10-01T09:31:49.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:29.000889+0000", 2023-10-01T09:31:49.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9625000000000001e-05 2023-10-01T09:31:49.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:29.000928+0000", 2023-10-01T09:31:49.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9019e-05 2023-10-01T09:31:49.693 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.693 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.693 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.693 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:119497 8.0 e353/343 8:05d8371f:::benchmark_data_smithi138_431617_object18086:head v 353'7458, mlcod=353'7455)", 2023-10-01T09:31:49.693 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:28.789692+0000", 2023-10-01T09:31:49.693 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 20.816969161999999, 2023-10-01T09:31:49.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21103181400000001, 2023-10-01T09:31:49.693 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.693 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.694 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.789692+0000", 2023-10-01T09:31:49.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.789692+0000", 2023-10-01T09:31:49.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.695 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.789691+0000", 2023-10-01T09:31:49.695 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:31:49.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.695 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.789701+0000", 2023-10-01T09:31:49.695 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2010000000000005e-06 2023-10-01T09:31:49.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.695 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989583+0000", 2023-10-01T09:31:49.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.199882487 2023-10-01T09:31:49.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989591+0000", 2023-10-01T09:31:49.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9359999999999999e-06 2023-10-01T09:31:49.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989802+0000", 2023-10-01T09:31:49.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000211246 2023-10-01T09:31:49.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989843+0000", 2023-10-01T09:31:49.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1100999999999998e-05 2023-10-01T09:31:49.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:29.000692+0000", 2023-10-01T09:31:49.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010848688 2023-10-01T09:31:49.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:29.000724+0000", 2023-10-01T09:31:49.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2084e-05 2023-10-01T09:31:49.698 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.698 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.698 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.698 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:119501 8.7 8:e75bed09:::benchmark_data_smithi138_431617_object18090:head [delete] snapc 0=[] ondisk+write+known_if_redirected e353)", 2023-10-01T09:31:49.698 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:28.791359+0000", 2023-10-01T09:31:49.698 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 20.815302848000002, 2023-10-01T09:31:49.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21209781599999999, 2023-10-01T09:31:49.698 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.699 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.699 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.699 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.699 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.699 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 119501 2023-10-01T09:31:49.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.699 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.791359+0000", 2023-10-01T09:31:49.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.791359+0000", 2023-10-01T09:31:49.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.791357+0000", 2023-10-01T09:31:49.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:31:49.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.791366+0000", 2023-10-01T09:31:49.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5909999999999996e-06 2023-10-01T09:31:49.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989689+0000", 2023-10-01T09:31:49.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19832307699999999 2023-10-01T09:31:49.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989697+0000", 2023-10-01T09:31:49.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6710000000000005e-06 2023-10-01T09:31:49.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989733+0000", 2023-10-01T09:31:49.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6081999999999998e-05 2023-10-01T09:31:49.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989847+0000", 2023-10-01T09:31:49.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113711 2023-10-01T09:31:49.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:31:49.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.989953+0000", 2023-10-01T09:31:49.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010552600000000001 2023-10-01T09:31:49.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.997893+0000", 2023-10-01T09:31:49.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0079402080000000007 2023-10-01T09:31:49.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:28.998882+0000", 2023-10-01T09:31:49.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00098872900000000004 2023-10-01T09:31:49.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:29.003368+0000", 2023-10-01T09:31:49.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044864869999999999 2023-10-01T09:31:49.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:29.003413+0000", 2023-10-01T09:31:49.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4731999999999998e-05 2023-10-01T09:31:49.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:29.003456+0000", 2023-10-01T09:31:49.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3717000000000002e-05 2023-10-01T09:31:49.705 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.705 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.705 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.706 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:130010 8.b 8:dd52852e:::benchmark_data_smithi138_431617_object28599:head [delete] snapc 0=[] ondisk+write+known_if_redirected e355)", 2023-10-01T09:31:49.706 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:43.811901+0000", 2023-10-01T09:31:49.706 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 5.7947603690000005, 2023-10-01T09:31:49.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.26073928600000001, 2023-10-01T09:31:49.706 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.706 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.706 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.706 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.706 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.706 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 130010 2023-10-01T09:31:49.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.706 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.811901+0000", 2023-10-01T09:31:49.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.811901+0000", 2023-10-01T09:31:49.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.811901+0000", 2023-10-01T09:31:49.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:49.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.811903+0000", 2023-10-01T09:31:49.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.774e-06 2023-10-01T09:31:49.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.811910+0000", 2023-10-01T09:31:49.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8770000000000004e-06 2023-10-01T09:31:49.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.811916+0000", 2023-10-01T09:31:49.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3319999999999999e-06 2023-10-01T09:31:49.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.839723+0000", 2023-10-01T09:31:49.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.027807100000000001 2023-10-01T09:31:49.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.839854+0000", 2023-10-01T09:31:49.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131297 2023-10-01T09:31:49.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:31:49.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.839975+0000", 2023-10-01T09:31:49.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121277 2023-10-01T09:31:49.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.846452+0000", 2023-10-01T09:31:49.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006476229 2023-10-01T09:31:49.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847511+0000", 2023-10-01T09:31:49.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001059422 2023-10-01T09:31:49.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072564+0000", 2023-10-01T09:31:49.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22505257400000001 2023-10-01T09:31:49.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072601+0000", 2023-10-01T09:31:49.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7753000000000003e-05 2023-10-01T09:31:49.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072640+0000", 2023-10-01T09:31:49.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9150999999999998e-05 2023-10-01T09:31:49.712 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.712 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.713 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.713 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:130018 8.7 8:e4934da9:::benchmark_data_smithi138_431617_object28607:head [delete] snapc 0=[] ondisk+write+known_if_redirected e355)", 2023-10-01T09:31:49.713 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:43.837069+0000", 2023-10-01T09:31:49.713 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 5.7695928209999998, 2023-10-01T09:31:49.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23938279100000001, 2023-10-01T09:31:49.713 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.713 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.713 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.713 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.713 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.714 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 130018 2023-10-01T09:31:49.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.714 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.837069+0000", 2023-10-01T09:31:49.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.837069+0000", 2023-10-01T09:31:49.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.837068+0000", 2023-10-01T09:31:49.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:49.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.837071+0000", 2023-10-01T09:31:49.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2639999999999998e-06 2023-10-01T09:31:49.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.837077+0000", 2023-10-01T09:31:49.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8479999999999997e-06 2023-10-01T09:31:49.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.837090+0000", 2023-10-01T09:31:49.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3190000000000001e-05 2023-10-01T09:31:49.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.848108+0000", 2023-10-01T09:31:49.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011018 2023-10-01T09:31:49.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.848178+0000", 2023-10-01T09:31:49.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0407999999999994e-05 2023-10-01T09:31:49.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:31:49.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.848270+0000", 2023-10-01T09:31:49.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2258999999999999e-05 2023-10-01T09:31:49.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.851064+0000", 2023-10-01T09:31:49.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0027932090000000001 2023-10-01T09:31:49.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072722+0000", 2023-10-01T09:31:49.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.2216581 2023-10-01T09:31:49.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.076385+0000", 2023-10-01T09:31:49.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036629010000000001 2023-10-01T09:31:49.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.076420+0000", 2023-10-01T09:31:49.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4992e-05 2023-10-01T09:31:49.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.076451+0000", 2023-10-01T09:31:49.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1890999999999997e-05 2023-10-01T09:31:49.720 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.720 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.720 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.720 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.720 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:130028 8.f 8:f1438df4:::benchmark_data_smithi138_431617_object28617:head [delete] snapc 0=[] ondisk+write+known_if_redirected e355)", 2023-10-01T09:31:49.720 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:43.847486+0000", 2023-10-01T09:31:49.720 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 5.7591754110000002, 2023-10-01T09:31:49.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23617080700000001, 2023-10-01T09:31:49.720 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.720 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.721 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.721 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.721 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.721 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 130028 2023-10-01T09:31:49.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.721 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847486+0000", 2023-10-01T09:31:49.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847486+0000", 2023-10-01T09:31:49.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847485+0000", 2023-10-01T09:31:49.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:31:49.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847490+0000", 2023-10-01T09:31:49.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6330000000000001e-06 2023-10-01T09:31:49.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847499+0000", 2023-10-01T09:31:49.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2989999999999997e-06 2023-10-01T09:31:49.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847506+0000", 2023-10-01T09:31:49.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0559999999999999e-06 2023-10-01T09:31:49.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.850926+0000", 2023-10-01T09:31:49.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034207349999999998 2023-10-01T09:31:49.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.851029+0000", 2023-10-01T09:31:49.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010300400000000001 2023-10-01T09:31:49.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:31:49.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.851109+0000", 2023-10-01T09:31:49.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9938999999999995e-05 2023-10-01T09:31:49.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.076354+0000", 2023-10-01T09:31:49.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22524497499999999 2023-10-01T09:31:49.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.076481+0000", 2023-10-01T09:31:49.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000126458 2023-10-01T09:31:49.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.083591+0000", 2023-10-01T09:31:49.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0071101180000000003 2023-10-01T09:31:49.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.083626+0000", 2023-10-01T09:31:49.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5166999999999997e-05 2023-10-01T09:31:49.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.083657+0000", 2023-10-01T09:31:49.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0852000000000002e-05 2023-10-01T09:31:49.727 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.727 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.727 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.727 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:130031 8.a 8:5c32d136:::benchmark_data_smithi138_431617_object28620:head [delete] snapc 0=[] ondisk+write+known_if_redirected e355)", 2023-10-01T09:31:49.727 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:43.847516+0000", 2023-10-01T09:31:49.727 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 5.7591457039999998, 2023-10-01T09:31:49.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22795003899999999, 2023-10-01T09:31:49.728 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.728 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.728 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.728 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.728 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.728 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 130031 2023-10-01T09:31:49.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.728 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.728 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.728 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847516+0000", 2023-10-01T09:31:49.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847516+0000", 2023-10-01T09:31:49.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847515+0000", 2023-10-01T09:31:49.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:49.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847518+0000", 2023-10-01T09:31:49.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3259999999999998e-06 2023-10-01T09:31:49.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847523+0000", 2023-10-01T09:31:49.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5439999999999998e-06 2023-10-01T09:31:49.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847528+0000", 2023-10-01T09:31:49.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5850000000000001e-06 2023-10-01T09:31:49.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847563+0000", 2023-10-01T09:31:49.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5335e-05 2023-10-01T09:31:49.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847672+0000", 2023-10-01T09:31:49.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108633 2023-10-01T09:31:49.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:31:49.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.847779+0000", 2023-10-01T09:31:49.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106673 2023-10-01T09:31:49.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.851272+0000", 2023-10-01T09:31:49.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034936250000000002 2023-10-01T09:31:49.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852504+0000", 2023-10-01T09:31:49.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001231469 2023-10-01T09:31:49.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.075390+0000", 2023-10-01T09:31:49.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22288601 2023-10-01T09:31:49.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.075429+0000", 2023-10-01T09:31:49.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9603e-05 2023-10-01T09:31:49.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.075466+0000", 2023-10-01T09:31:49.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6575999999999997e-05 2023-10-01T09:31:49.734 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.734 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.734 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.734 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:130035 8.1 8:8625a435:::benchmark_data_smithi138_431617_object28624:head [delete] snapc 0=[] ondisk+write+known_if_redirected e355)", 2023-10-01T09:31:49.734 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:43.852267+0000", 2023-10-01T09:31:49.735 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 5.7543946850000003, 2023-10-01T09:31:49.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23800200199999999, 2023-10-01T09:31:49.735 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.735 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.735 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.735 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.735 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.735 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 130035 2023-10-01T09:31:49.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.735 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852267+0000", 2023-10-01T09:31:49.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852267+0000", 2023-10-01T09:31:49.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.736 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.736 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852266+0000", 2023-10-01T09:31:49.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:31:49.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852270+0000", 2023-10-01T09:31:49.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1420000000000003e-06 2023-10-01T09:31:49.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852278+0000", 2023-10-01T09:31:49.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3750000000000003e-06 2023-10-01T09:31:49.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852284+0000", 2023-10-01T09:31:49.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5809999999999996e-06 2023-10-01T09:31:49.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852329+0000", 2023-10-01T09:31:49.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5537999999999997e-05 2023-10-01T09:31:49.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852444+0000", 2023-10-01T09:31:49.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000114874 2023-10-01T09:31:49.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:31:49.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072147+0000", 2023-10-01T09:31:49.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21970250799999999 2023-10-01T09:31:49.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.079491+0000", 2023-10-01T09:31:49.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0073444440000000003 2023-10-01T09:31:49.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.083843+0000", 2023-10-01T09:31:49.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043519509999999997 2023-10-01T09:31:49.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.090174+0000", 2023-10-01T09:31:49.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0063306899999999999 2023-10-01T09:31:49.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.090227+0000", 2023-10-01T09:31:49.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2871999999999998e-05 2023-10-01T09:31:49.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.090269+0000", 2023-10-01T09:31:49.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2030000000000002e-05 2023-10-01T09:31:49.741 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.741 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.741 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.742 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.45674.0:130010 8.b e355/343 ondisk, result = 0)", 2023-10-01T09:31:49.742 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:43.852415+0000", 2023-10-01T09:31:49.742 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 5.7542460880000004, 2023-10-01T09:31:49.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22023026800000001, 2023-10-01T09:31:49.742 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.742 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:31:49.742 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852415+0000", 2023-10-01T09:31:49.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852415+0000", 2023-10-01T09:31:49.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852415+0000", 2023-10-01T09:31:49.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:31:49.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852418+0000", 2023-10-01T09:31:49.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7410000000000001e-06 2023-10-01T09:31:49.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852425+0000", 2023-10-01T09:31:49.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9199999999999998e-06 2023-10-01T09:31:49.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.744 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.744 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.852430+0000", 2023-10-01T09:31:49.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8099999999999997e-06 2023-10-01T09:31:49.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072548+0000", 2023-10-01T09:31:49.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22011850099999999 2023-10-01T09:31:49.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072557+0000", 2023-10-01T09:31:49.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9020000000000005e-06 2023-10-01T09:31:49.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.746 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072646+0000", 2023-10-01T09:31:49.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8760999999999999e-05 2023-10-01T09:31:49.746 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.746 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.746 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.746 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.45674.0:130018 8.7 e355/326 ondisk, result = 0)", 2023-10-01T09:31:49.746 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:43.855517+0000", 2023-10-01T09:31:49.747 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 5.7511440289999998, 2023-10-01T09:31:49.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21720650799999999, 2023-10-01T09:31:49.747 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.747 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:31:49.747 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.747 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.747 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.855517+0000", 2023-10-01T09:31:49.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.747 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.855517+0000", 2023-10-01T09:31:49.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.855517+0000", 2023-10-01T09:31:49.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:31:49.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.855527+0000", 2023-10-01T09:31:49.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0046e-05 2023-10-01T09:31:49.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072456+0000", 2023-10-01T09:31:49.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.216929011 2023-10-01T09:31:49.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072461+0000", 2023-10-01T09:31:49.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4269999999999999e-06 2023-10-01T09:31:49.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072709+0000", 2023-10-01T09:31:49.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00024808600000000001 2023-10-01T09:31:49.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072716+0000", 2023-10-01T09:31:49.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5490000000000003e-06 2023-10-01T09:31:49.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072724+0000", 2023-10-01T09:31:49.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3110000000000006e-06 2023-10-01T09:31:49.751 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.751 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.751 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.751 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:130038 8.f 8:fab6e4b4:::benchmark_data_smithi138_431617_object28627:head [delete] snapc 0=[] ondisk+write+known_if_redirected e355)", 2023-10-01T09:31:49.751 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:43.855992+0000", 2023-10-01T09:31:49.751 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 5.7506699470000004, 2023-10-01T09:31:49.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24442292400000001, 2023-10-01T09:31:49.751 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:31:49.752 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:31:49.752 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:31:49.752 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:31:49.752 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:31:49.752 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 130038 2023-10-01T09:31:49.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.752 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:31:49.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.752 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:31:49.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.855992+0000", 2023-10-01T09:31:49.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:31:49.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.855992+0000", 2023-10-01T09:31:49.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:31:49.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:31:49.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.855990+0000", 2023-10-01T09:31:49.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:31:49.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:31:49.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:43.855996+0000", 2023-10-01T09:31:49.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8710000000000002e-06 2023-10-01T09:31:49.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:31:49.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072382+0000", 2023-10-01T09:31:49.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.21638645200000001 2023-10-01T09:31:49.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:31:49.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072393+0000", 2023-10-01T09:31:49.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1398e-05 2023-10-01T09:31:49.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:31:49.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072743+0000", 2023-10-01T09:31:49.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00034939699999999999 2023-10-01T09:31:49.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:31:49.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072856+0000", 2023-10-01T09:31:49.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000113028 2023-10-01T09:31:49.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:31:49.756 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.072963+0000", 2023-10-01T09:31:49.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106716 2023-10-01T09:31:49.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.756 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:31:49.756 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.088411+0000", 2023-10-01T09:31:49.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015448865000000001 2023-10-01T09:31:49.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.756 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.756 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.089960+0000", 2023-10-01T09:31:49.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001548511 2023-10-01T09:31:49.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:31:49.757 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.100319+0000", 2023-10-01T09:31:49.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01035933 2023-10-01T09:31:49.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:31:49.757 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.100370+0000", 2023-10-01T09:31:49.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0453e-05 2023-10-01T09:31:49.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:31:49.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:31:49.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:31:49.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.100414+0000", 2023-10-01T09:31:49.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4705e-05 2023-10-01T09:31:49.758 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.758 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.758 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.758 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:31:49.758 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:31:49.758 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:31:49.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:49.759 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:31:49.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:31:49.860 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:31:49.876 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:31:49.894 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:49.908 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:49.908 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:49.908 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1441285925' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:31:49.908 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:31:49.908 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:31:49.908 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:49.909 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:49.909 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:49.909 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1441285925' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:31:49.909 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:31:49.909 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:31:49.910 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:49.964 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:31:49.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:49.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:31:50.070 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:31:50.142 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:50.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1441285925' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:31:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:31:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:31:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:50.782 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:31:50.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:50.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:31:50.891 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:50.944 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:51.011 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:51.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:51.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:51.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:50 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:31:51.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:50 smithi138 ceph-mon[185886]: failed to encode map e356 with expected crc 2023-10-01T09:31:51.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:50 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1441285925' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:31:51.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:50 smithi138 ceph-mon[185886]: osdmap e356: 8 total, 8 up, 7 in 2023-10-01T09:31:51.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:50 smithi138 ceph-mon[185886]: failed to encode map e356 with expected crc 2023-10-01T09:31:51.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:50 smithi138 ceph-mon[185886]: pgmap v991: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T09:31:51.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:50 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:31:51.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:50 smithi138 ceph-mon[192513]: failed to encode map e356 with expected crc 2023-10-01T09:31:51.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:50 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1441285925' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:31:51.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:50 smithi138 ceph-mon[192513]: osdmap e356: 8 total, 8 up, 7 in 2023-10-01T09:31:51.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:50 smithi138 ceph-mon[192513]: failed to encode map e356 with expected crc 2023-10-01T09:31:51.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:50 smithi138 ceph-mon[192513]: pgmap v991: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T09:31:51.122 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:31:51.130 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:31:51.212 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:51.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:50 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:31:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:50 smithi190 ceph-mon[160435]: failed to encode map e356 with expected crc 2023-10-01T09:31:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:50 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1441285925' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:31:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:50 smithi190 ceph-mon[160435]: osdmap e356: 8 total, 8 up, 7 in 2023-10-01T09:31:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:50 smithi190 ceph-mon[160435]: failed to encode map e356 with expected crc 2023-10-01T09:31:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:50 smithi190 ceph-mon[160435]: pgmap v991: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T09:31:51.843 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:31:51.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:51.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:31:51.952 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:31:51.990 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:52.052 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:31:52.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:52.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:31:52.158 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:52.200 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:52.274 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:52.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:52.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:52.382 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:52.464 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:52.603 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:52.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:52.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:52.709 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:31:52.788 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:52.907 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:31:52.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:52.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:31:53.014 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:31:53.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:53 smithi138 ceph-mon[192513]: pgmap v992: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 687 op/s 2023-10-01T09:31:53.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:53 smithi138 ceph-mon[185886]: pgmap v992: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 687 op/s 2023-10-01T09:31:53.081 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:53.140 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:31:53.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:53.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:31:53.251 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:31:53.287 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:53 smithi190 ceph-mon[160435]: pgmap v992: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 687 op/s 2023-10-01T09:31:53.290 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:53.364 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:31:53.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:53.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:31:53.473 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:53.571 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:53.711 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:53.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:53.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:53.783 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:53.819 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:31:53.850 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:53.909 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:31:53.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:53.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:31:54.016 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:31:54.064 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:54.125 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:31:54.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:54.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:31:54.191 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:31:54.199 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:31:54.231 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:31:54.268 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:54.322 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:31:54.323 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:31:54.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:54.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:31:54.360 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:31:54.399 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:31:54.429 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:54.455 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:31:54.489 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:31:54.520 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:54.525 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:31:54.554 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:31:54.589 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:31:54.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:31:54.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:31:54.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:31:54.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:31:54.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:31:54.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:31:54.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:31:54.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:31:54.595 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:31:55.041 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:31:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:31:54] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.33.4" 2023-10-01T09:31:55.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:55 smithi190 ceph-mon[160435]: pgmap v993: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-10-01T09:31:55.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:55 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2930965864' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:55.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:55 smithi138 ceph-mon[185886]: pgmap v993: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-10-01T09:31:55.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:55 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2930965864' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:55.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:55 smithi138 ceph-mon[192513]: pgmap v993: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-10-01T09:31:55.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:55 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2930965864' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:31:55.381 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:55.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:55.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:55.485 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:31:55.515 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:56.066 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:31:56.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:56.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:31:56.180 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:56.206 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:56.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:56 smithi190 ceph-mon[160435]: from='client.35862 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:31:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:56.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:31:56.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:56 smithi138 ceph-mon[192513]: from='client.35862 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:31:56.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:56.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:31:56.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:56 smithi138 ceph-mon[185886]: from='client.35862 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:31:56.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:31:56.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:31:56.455 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":993,"stamp":"2023-10-01T09:31:53.966059+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4388886475,"num_objects":66925,"num_object_clones":0,"num_object_copies":200775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66925,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":137027,"num_write_kb":6505254,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":318,"num_bytes_recovered":3570664,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119491,"ondisk_log_size":119491,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16591220,"kb_used_data":13020284,"kb_used_omap":45,"kb_used_meta":3570386,"kb_avail":639624844,"statfs":{"total":671965249536,"available":654975840256,"internally_reserved":0,"allocated":13332770816,"data_stored":13323733550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":46344,"internal_metadata":3656076024},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,0,2,6,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-449970176,"num_objects":-6866,"num_object_clones":0,"num_object_copies":-20598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6866,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009574"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":239,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.048475+0000","last_change":"2023-10-01T09:26:12.273466+0000","last_active":"2023-10-01T09:31:37.048475+0000","last_peered":"2023-10-01T09:31:37.048475+0000","last_clean":"2023-10-01T09:31:37.048475+0000","last_became_active":"2023-10-01T09:26:12.272944+0000","last_became_peered":"2023-10-01T09:26:12.272944+0000","last_unstale":"2023-10-01T09:31:37.048475+0000","last_undegraded":"2023-10-01T09:31:37.048475+0000","last_fullsized":"2023-10-01T09:31:37.048475+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:40:41.939800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":160,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.049765+0000","last_change":"2023-10-01T09:28:02.324629+0000","last_active":"2023-10-01T09:31:37.049765+0000","last_peered":"2023-10-01T09:31:37.049765+0000","last_clean":"2023-10-01T09:31:37.049765+0000","last_became_active":"2023-10-01T09:28:02.324019+0000","last_became_peered":"2023-10-01T09:28:02.324019+0000","last_unstale":"2023-10-01T09:31:37.049765+0000","last_undegraded":"2023-10-01T09:31:37.049765+0000","last_fullsized":"2023-10-01T09:31:37.049765+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:00:43.440938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":324,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.046474+0000","last_change":"2023-10-01T09:28:02.338915+0000","last_active":"2023-10-01T09:31:37.046474+0000","last_peered":"2023-10-01T09:31:37.046474+0000","last_clean":"2023-10-01T09:31:37.046474+0000","last_became_active":"2023-10-01T09:28:02.338589+0000","last_became_peered":"2023-10-01T09:28:02.338589+0000","last_unstale":"2023-10-01T09:31:37.046474+0000","last_undegraded":"2023-10-01T09:31:37.046474+0000","last_fullsized":"2023-10-01T09:31:37.046474+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:18:19.992078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":372,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041897+0000","last_change":"2023-10-01T09:29:58.372577+0000","last_active":"2023-10-01T09:31:37.041897+0000","last_peered":"2023-10-01T09:31:37.041897+0000","last_clean":"2023-10-01T09:31:37.041897+0000","last_became_active":"2023-10-01T09:29:58.372218+0000","last_became_peered":"2023-10-01T09:29:58.372218+0000","last_unstale":"2023-10-01T09:31:37.041897+0000","last_undegraded":"2023-10-01T09:31:37.041897+0000","last_fullsized":"2023-10-01T09:31:37.041897+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:04:08.376752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":245,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041350+0000","last_change":"2023-10-01T09:30:39.394083+0000","last_active":"2023-10-01T09:31:37.041350+0000","last_peered":"2023-10-01T09:31:37.041350+0000","last_clean":"2023-10-01T09:31:37.041350+0000","last_became_active":"2023-10-01T09:30:39.393797+0000","last_became_peered":"2023-10-01T09:30:39.393797+0000","last_unstale":"2023-10-01T09:31:37.041350+0000","last_undegraded":"2023-10-01T09:31:37.041350+0000","last_fullsized":"2023-10-01T09:31:37.041350+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:22.625624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041017+0000","last_change":"2023-10-01T09:28:02.362383+0000","last_active":"2023-10-01T09:31:37.041017+0000","last_peered":"2023-10-01T09:31:37.041017+0000","last_clean":"2023-10-01T09:31:37.041017+0000","last_became_active":"2023-10-01T09:28:02.362044+0000","last_became_peered":"2023-10-01T09:28:02.362044+0000","last_unstale":"2023-10-01T09:31:37.041017+0000","last_undegraded":"2023-10-01T09:31:37.041017+0000","last_fullsized":"2023-10-01T09:31:37.041017+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":393,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041501+0000","last_change":"2023-10-01T09:27:01.992338+0000","last_active":"2023-10-01T09:31:37.041501+0000","last_peered":"2023-10-01T09:31:37.041501+0000","last_clean":"2023-10-01T09:31:37.041501+0000","last_became_active":"2023-10-01T09:27:01.989050+0000","last_became_peered":"2023-10-01T09:27:01.989050+0000","last_unstale":"2023-10-01T09:31:37.041501+0000","last_undegraded":"2023-10-01T09:31:37.041501+0000","last_fullsized":"2023-10-01T09:31:37.041501+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:43:29.856456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":356,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.042314+0000","last_change":"2023-10-01T09:30:39.395270+0000","last_active":"2023-10-01T09:31:37.042314+0000","last_peered":"2023-10-01T09:31:37.042314+0000","last_clean":"2023-10-01T09:31:37.042314+0000","last_became_active":"2023-10-01T09:30:39.394888+0000","last_became_peered":"2023-10-01T09:30:39.394888+0000","last_unstale":"2023-10-01T09:31:37.042314+0000","last_undegraded":"2023-10-01T09:31:37.042314+0000","last_fullsized":"2023-10-01T09:31:37.042314+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:35:40.846102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":67,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041591+0000","last_change":"2023-10-01T09:27:01.970412+0000","last_active":"2023-10-01T09:31:37.041591+0000","last_peered":"2023-10-01T09:31:37.041591+0000","last_clean":"2023-10-01T09:31:37.041591+0000","last_became_active":"2023-10-01T09:27:01.970108+0000","last_became_peered":"2023-10-01T09:27:01.970108+0000","last_unstale":"2023-10-01T09:31:37.041591+0000","last_undegraded":"2023-10-01T09:31:37.041591+0000","last_fullsized":"2023-10-01T09:31:37.041591+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":373,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.050190+0000","last_change":"2023-10-01T09:30:39.398262+0000","last_active":"2023-10-01T09:31:37.050190+0000","last_peered":"2023-10-01T09:31:37.050190+0000","last_clean":"2023-10-01T09:31:37.050190+0000","last_became_active":"2023-10-01T09:30:39.397934+0000","last_became_peered":"2023-10-01T09:30:39.397934+0000","last_unstale":"2023-10-01T09:31:37.050190+0000","last_undegraded":"2023-10-01T09:31:37.050190+0000","last_fullsized":"2023-10-01T09:31:37.050190+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":140,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.050180+0000","last_change":"2023-10-01T09:26:12.415219+0000","last_active":"2023-10-01T09:31:37.050180+0000","last_peered":"2023-10-01T09:31:37.050180+0000","last_clean":"2023-10-01T09:31:37.050180+0000","last_became_active":"2023-10-01T09:26:12.286669+0000","last_became_peered":"2023-10-01T09:26:12.286669+0000","last_unstale":"2023-10-01T09:31:37.050180+0000","last_undegraded":"2023-10-01T09:31:37.050180+0000","last_fullsized":"2023-10-01T09:31:37.050180+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":384,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.047939+0000","last_change":"2023-10-01T09:28:02.342661+0000","last_active":"2023-10-01T09:31:37.047939+0000","last_peered":"2023-10-01T09:31:37.047939+0000","last_clean":"2023-10-01T09:31:37.047939+0000","last_became_active":"2023-10-01T09:28:02.342358+0000","last_became_peered":"2023-10-01T09:28:02.342358+0000","last_unstale":"2023-10-01T09:31:37.047939+0000","last_undegraded":"2023-10-01T09:31:37.047939+0000","last_fullsized":"2023-10-01T09:31:37.047939+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:29:30.624568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":364,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.050028+0000","last_change":"2023-10-01T09:28:02.511577+0000","last_active":"2023-10-01T09:31:37.050028+0000","last_peered":"2023-10-01T09:31:37.050028+0000","last_clean":"2023-10-01T09:31:37.050028+0000","last_became_active":"2023-10-01T09:28:02.318805+0000","last_became_peered":"2023-10-01T09:28:02.318805+0000","last_unstale":"2023-10-01T09:31:37.050028+0000","last_undegraded":"2023-10-01T09:31:37.050028+0000","last_fullsized":"2023-10-01T09:31:37.050028+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.039306+0000","last_change":"2023-10-01T09:28:02.454516+0000","last_active":"2023-10-01T09:31:37.039306+0000","last_peered":"2023-10-01T09:31:37.039306+0000","last_clean":"2023-10-01T09:31:37.039306+0000","last_became_active":"2023-10-01T09:28:02.454098+0000","last_became_peered":"2023-10-01T09:28:02.454098+0000","last_unstale":"2023-10-01T09:31:37.039306+0000","last_undegraded":"2023-10-01T09:31:37.039306+0000","last_fullsized":"2023-10-01T09:31:37.039306+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":414,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.049689+0000","last_change":"2023-10-01T09:28:02.505692+0000","last_active":"2023-10-01T09:31:37.049689+0000","last_peered":"2023-10-01T09:31:37.049689+0000","last_clean":"2023-10-01T09:31:37.049689+0000","last_became_active":"2023-10-01T09:28:02.318771+0000","last_became_peered":"2023-10-01T09:28:02.318771+0000","last_unstale":"2023-10-01T09:31:37.049689+0000","last_undegraded":"2023-10-01T09:31:37.049689+0000","last_fullsized":"2023-10-01T09:31:37.049689+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":184,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.050126+0000","last_change":"2023-10-01T09:30:39.399483+0000","last_active":"2023-10-01T09:31:37.050126+0000","last_peered":"2023-10-01T09:31:37.050126+0000","last_clean":"2023-10-01T09:31:37.050126+0000","last_became_active":"2023-10-01T09:30:39.399266+0000","last_became_peered":"2023-10-01T09:30:39.399266+0000","last_unstale":"2023-10-01T09:31:37.050126+0000","last_undegraded":"2023-10-01T09:31:37.050126+0000","last_fullsized":"2023-10-01T09:31:37.050126+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":346,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041276+0000","last_change":"2023-10-01T09:23:10.607388+0000","last_active":"2023-10-01T09:31:37.041276+0000","last_peered":"2023-10-01T09:31:37.041276+0000","last_clean":"2023-10-01T09:31:37.041276+0000","last_became_active":"2023-10-01T09:23:10.607084+0000","last_became_peered":"2023-10-01T09:23:10.607084+0000","last_unstale":"2023-10-01T09:31:37.041276+0000","last_undegraded":"2023-10-01T09:31:37.041276+0000","last_fullsized":"2023-10-01T09:31:37.041276+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:19.669393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":67,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041868+0000","last_change":"2023-10-01T09:27:01.978738+0000","last_active":"2023-10-01T09:31:37.041868+0000","last_peered":"2023-10-01T09:31:37.041868+0000","last_clean":"2023-10-01T09:31:37.041868+0000","last_became_active":"2023-10-01T09:27:01.978024+0000","last_became_peered":"2023-10-01T09:27:01.978024+0000","last_unstale":"2023-10-01T09:31:37.041868+0000","last_undegraded":"2023-10-01T09:31:37.041868+0000","last_fullsized":"2023-10-01T09:31:37.041868+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":358,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.049923+0000","last_change":"2023-10-01T09:28:02.314047+0000","last_active":"2023-10-01T09:31:37.049923+0000","last_peered":"2023-10-01T09:31:37.049923+0000","last_clean":"2023-10-01T09:31:37.049923+0000","last_became_active":"2023-10-01T09:28:02.313729+0000","last_became_peered":"2023-10-01T09:28:02.313729+0000","last_unstale":"2023-10-01T09:31:37.049923+0000","last_undegraded":"2023-10-01T09:31:37.049923+0000","last_fullsized":"2023-10-01T09:31:37.049923+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":328,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041964+0000","last_change":"2023-10-01T09:23:10.611400+0000","last_active":"2023-10-01T09:31:37.041964+0000","last_peered":"2023-10-01T09:31:37.041964+0000","last_clean":"2023-10-01T09:31:37.041964+0000","last_became_active":"2023-10-01T09:23:10.610928+0000","last_became_peered":"2023-10-01T09:23:10.610928+0000","last_unstale":"2023-10-01T09:31:37.041964+0000","last_undegraded":"2023-10-01T09:31:37.041964+0000","last_fullsized":"2023-10-01T09:31:37.041964+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:17.141540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":97,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.039167+0000","last_change":"2023-10-01T09:30:00.577886+0000","last_active":"2023-10-01T09:31:37.039167+0000","last_peered":"2023-10-01T09:31:37.039167+0000","last_clean":"2023-10-01T09:31:37.039167+0000","last_became_active":"2023-10-01T09:30:00.394121+0000","last_became_peered":"2023-10-01T09:30:00.394121+0000","last_unstale":"2023-10-01T09:31:37.039167+0000","last_undegraded":"2023-10-01T09:31:37.039167+0000","last_fullsized":"2023-10-01T09:31:37.039167+0000","mapping_epoch":337,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":338,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":45,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.049896+0000","last_change":"2023-10-01T09:30:39.398326+0000","last_active":"2023-10-01T09:31:37.049896+0000","last_peered":"2023-10-01T09:31:37.049896+0000","last_clean":"2023-10-01T09:31:37.049896+0000","last_became_active":"2023-10-01T09:30:39.398021+0000","last_became_peered":"2023-10-01T09:30:39.398021+0000","last_unstale":"2023-10-01T09:31:37.049896+0000","last_undegraded":"2023-10-01T09:31:37.049896+0000","last_fullsized":"2023-10-01T09:31:37.049896+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"356'8322","reported_seq":8360,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:50.716003+0000","last_change":"2023-10-01T09:29:24.378970+0000","last_active":"2023-10-01T09:31:50.716003+0000","last_peered":"2023-10-01T09:31:50.716003+0000","last_clean":"2023-10-01T09:31:50.716003+0000","last_became_active":"2023-10-01T09:29:24.378703+0000","last_became_peered":"2023-10-01T09:29:24.378703+0000","last_unstale":"2023-10-01T09:31:50.716003+0000","last_undegraded":"2023-10-01T09:31:50.716003+0000","last_fullsized":"2023-10-01T09:31:50.716003+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":8322,"log_dups_size":0,"ondisk_log_size":8322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274464768,"num_objects":4188,"num_object_clones":0,"num_object_copies":12564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8322,"num_write_kb":400320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":136,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.049988+0000","last_change":"2023-10-01T09:26:12.395230+0000","last_active":"2023-10-01T09:31:37.049988+0000","last_peered":"2023-10-01T09:31:37.049988+0000","last_clean":"2023-10-01T09:31:37.049988+0000","last_became_active":"2023-10-01T09:26:12.269960+0000","last_became_peered":"2023-10-01T09:26:12.269960+0000","last_unstale":"2023-10-01T09:31:37.049988+0000","last_undegraded":"2023-10-01T09:31:37.049988+0000","last_fullsized":"2023-10-01T09:31:37.049988+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":76,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041937+0000","last_change":"2023-10-01T09:28:02.315483+0000","last_active":"2023-10-01T09:31:37.041937+0000","last_peered":"2023-10-01T09:31:37.041937+0000","last_clean":"2023-10-01T09:31:37.041937+0000","last_became_active":"2023-10-01T09:28:02.315121+0000","last_became_peered":"2023-10-01T09:28:02.315121+0000","last_unstale":"2023-10-01T09:31:37.041937+0000","last_undegraded":"2023-10-01T09:31:37.041937+0000","last_fullsized":"2023-10-01T09:31:37.041937+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"356'8565","reported_seq":8603,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:51.951525+0000","last_change":"2023-10-01T09:29:24.376565+0000","last_active":"2023-10-01T09:31:51.951525+0000","last_peered":"2023-10-01T09:31:51.951525+0000","last_clean":"2023-10-01T09:31:51.951525+0000","last_became_active":"2023-10-01T09:29:24.376238+0000","last_became_peered":"2023-10-01T09:29:24.376238+0000","last_unstale":"2023-10-01T09:31:51.951525+0000","last_undegraded":"2023-10-01T09:31:51.951525+0000","last_fullsized":"2023-10-01T09:31:51.951525+0000","mapping_epoch":326,"log_start":"331'800","ondisk_log_start":"331'800","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":7765,"log_dups_size":0,"ondisk_log_size":7765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276234240,"num_objects":4215,"num_object_clones":0,"num_object_copies":12645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8565,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":337,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.050611+0000","last_change":"2023-10-01T09:26:12.408240+0000","last_active":"2023-10-01T09:31:37.050611+0000","last_peered":"2023-10-01T09:31:37.050611+0000","last_clean":"2023-10-01T09:31:37.050611+0000","last_became_active":"2023-10-01T09:26:12.272093+0000","last_became_peered":"2023-10-01T09:26:12.272093+0000","last_unstale":"2023-10-01T09:31:37.050611+0000","last_undegraded":"2023-10-01T09:31:37.050611+0000","last_fullsized":"2023-10-01T09:31:37.050611+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":378,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.046436+0000","last_change":"2023-10-01T09:28:02.338346+0000","last_active":"2023-10-01T09:31:37.046436+0000","last_peered":"2023-10-01T09:31:37.046436+0000","last_clean":"2023-10-01T09:31:37.046436+0000","last_became_active":"2023-10-01T09:28:02.338038+0000","last_became_peered":"2023-10-01T09:28:02.338038+0000","last_unstale":"2023-10-01T09:31:37.046436+0000","last_undegraded":"2023-10-01T09:31:37.046436+0000","last_fullsized":"2023-10-01T09:31:37.046436+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:56:32.734028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"356'8505","reported_seq":8545,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:53.558418+0000","last_change":"2023-10-01T09:29:24.379084+0000","last_active":"2023-10-01T09:31:53.558418+0000","last_peered":"2023-10-01T09:31:53.558418+0000","last_clean":"2023-10-01T09:31:53.558418+0000","last_became_active":"2023-10-01T09:29:24.378789+0000","last_became_peered":"2023-10-01T09:29:24.378789+0000","last_unstale":"2023-10-01T09:31:53.558418+0000","last_undegraded":"2023-10-01T09:31:53.558418+0000","last_fullsized":"2023-10-01T09:31:53.558418+0000","mapping_epoch":326,"log_start":"331'1000","ondisk_log_start":"331'1000","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":7505,"log_dups_size":0,"ondisk_log_size":7505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:00:21.161819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268238848,"num_objects":4093,"num_object_clones":0,"num_object_copies":12279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8505,"num_write_kb":403136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":421,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.050234+0000","last_change":"2023-10-01T09:28:02.318966+0000","last_active":"2023-10-01T09:31:37.050234+0000","last_peered":"2023-10-01T09:31:37.050234+0000","last_clean":"2023-10-01T09:31:37.050234+0000","last_became_active":"2023-10-01T09:28:02.316075+0000","last_became_peered":"2023-10-01T09:28:02.316075+0000","last_unstale":"2023-10-01T09:31:37.050234+0000","last_undegraded":"2023-10-01T09:31:37.050234+0000","last_fullsized":"2023-10-01T09:31:37.050234+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":59,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.039200+0000","last_change":"2023-10-01T09:28:02.361663+0000","last_active":"2023-10-01T09:31:37.039200+0000","last_peered":"2023-10-01T09:31:37.039200+0000","last_clean":"2023-10-01T09:31:37.039200+0000","last_became_active":"2023-10-01T09:28:02.361093+0000","last_became_peered":"2023-10-01T09:28:02.361093+0000","last_unstale":"2023-10-01T09:31:37.039200+0000","last_undegraded":"2023-10-01T09:31:37.039200+0000","last_fullsized":"2023-10-01T09:31:37.039200+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"356'8519","reported_seq":8559,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:53.416093+0000","last_change":"2023-10-01T09:29:24.356954+0000","last_active":"2023-10-01T09:31:53.416093+0000","last_peered":"2023-10-01T09:31:53.416093+0000","last_clean":"2023-10-01T09:31:53.416093+0000","last_became_active":"2023-10-01T09:29:24.356643+0000","last_became_peered":"2023-10-01T09:29:24.356643+0000","last_unstale":"2023-10-01T09:31:53.416093+0000","last_undegraded":"2023-10-01T09:31:53.416093+0000","last_fullsized":"2023-10-01T09:31:53.416093+0000","mapping_epoch":326,"log_start":"333'2100","ondisk_log_start":"333'2100","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":6419,"log_dups_size":0,"ondisk_log_size":6419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:56:34.140751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275578880,"num_objects":4205,"num_object_clones":0,"num_object_copies":12615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8519,"num_write_kb":407168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":76,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041881+0000","last_change":"2023-10-01T09:28:02.314625+0000","last_active":"2023-10-01T09:31:37.041881+0000","last_peered":"2023-10-01T09:31:37.041881+0000","last_clean":"2023-10-01T09:31:37.041881+0000","last_became_active":"2023-10-01T09:28:02.314315+0000","last_became_peered":"2023-10-01T09:28:02.314315+0000","last_unstale":"2023-10-01T09:31:37.041881+0000","last_undegraded":"2023-10-01T09:31:37.041881+0000","last_fullsized":"2023-10-01T09:31:37.041881+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":320,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041386+0000","last_change":"2023-10-01T09:26:12.308350+0000","last_active":"2023-10-01T09:31:37.041386+0000","last_peered":"2023-10-01T09:31:37.041386+0000","last_clean":"2023-10-01T09:31:37.041386+0000","last_became_active":"2023-10-01T09:26:12.307689+0000","last_became_peered":"2023-10-01T09:26:12.307689+0000","last_unstale":"2023-10-01T09:31:37.041386+0000","last_undegraded":"2023-10-01T09:31:37.041386+0000","last_fullsized":"2023-10-01T09:31:37.041386+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:34:57.116449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"356'8385","reported_seq":8431,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:50.683946+0000","last_change":"2023-10-01T09:30:39.402464+0000","last_active":"2023-10-01T09:31:50.683946+0000","last_peered":"2023-10-01T09:31:50.683946+0000","last_clean":"2023-10-01T09:31:50.683946+0000","last_became_active":"2023-10-01T09:30:39.397122+0000","last_became_peered":"2023-10-01T09:30:39.397122+0000","last_unstale":"2023-10-01T09:31:50.683946+0000","last_undegraded":"2023-10-01T09:31:50.683946+0000","last_fullsized":"2023-10-01T09:31:50.683946+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":8385,"log_dups_size":0,"ondisk_log_size":8385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278200320,"num_objects":4245,"num_object_clones":0,"num_object_copies":12735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8385,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":43,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041417+0000","last_change":"2023-10-01T09:30:39.395785+0000","last_active":"2023-10-01T09:31:37.041417+0000","last_peered":"2023-10-01T09:31:37.041417+0000","last_clean":"2023-10-01T09:31:37.041417+0000","last_became_active":"2023-10-01T09:30:39.395449+0000","last_became_peered":"2023-10-01T09:30:39.395449+0000","last_unstale":"2023-10-01T09:31:37.041417+0000","last_undegraded":"2023-10-01T09:31:37.041417+0000","last_fullsized":"2023-10-01T09:31:37.041417+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:23:32.908527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":288,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.048180+0000","last_change":"2023-10-01T09:30:39.398279+0000","last_active":"2023-10-01T09:31:37.048180+0000","last_peered":"2023-10-01T09:31:37.048180+0000","last_clean":"2023-10-01T09:31:37.048180+0000","last_became_active":"2023-10-01T09:30:39.397946+0000","last_became_peered":"2023-10-01T09:30:39.397946+0000","last_unstale":"2023-10-01T09:31:37.048180+0000","last_undegraded":"2023-10-01T09:31:37.048180+0000","last_fullsized":"2023-10-01T09:31:37.048180+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:30:40.348097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.1","version":"356'8567","reported_seq":8607,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:53.415830+0000","last_change":"2023-10-01T09:29:24.380240+0000","last_active":"2023-10-01T09:31:53.415830+0000","last_peered":"2023-10-01T09:31:53.415830+0000","last_clean":"2023-10-01T09:31:53.415830+0000","last_became_active":"2023-10-01T09:29:24.379922+0000","last_became_peered":"2023-10-01T09:29:24.379922+0000","last_unstale":"2023-10-01T09:31:53.415830+0000","last_undegraded":"2023-10-01T09:31:53.415830+0000","last_fullsized":"2023-10-01T09:31:53.415830+0000","mapping_epoch":326,"log_start":"333'2100","ondisk_log_start":"333'2100","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":6467,"log_dups_size":0,"ondisk_log_size":6467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:34:10.363596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271384576,"num_objects":4141,"num_object_clones":0,"num_object_copies":12423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8567,"num_write_kb":406656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":328,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.048120+0000","last_change":"2023-10-01T09:28:02.342548+0000","last_active":"2023-10-01T09:31:37.048120+0000","last_peered":"2023-10-01T09:31:37.048120+0000","last_clean":"2023-10-01T09:31:37.048120+0000","last_became_active":"2023-10-01T09:28:02.342162+0000","last_became_peered":"2023-10-01T09:28:02.342162+0000","last_unstale":"2023-10-01T09:31:37.048120+0000","last_undegraded":"2023-10-01T09:31:37.048120+0000","last_fullsized":"2023-10-01T09:31:37.048120+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:49:07.163031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":373,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.048113+0000","last_change":"2023-10-01T09:28:02.342750+0000","last_active":"2023-10-01T09:31:37.048113+0000","last_peered":"2023-10-01T09:31:37.048113+0000","last_clean":"2023-10-01T09:31:37.048113+0000","last_became_active":"2023-10-01T09:28:02.342487+0000","last_became_peered":"2023-10-01T09:28:02.342487+0000","last_unstale":"2023-10-01T09:31:37.048113+0000","last_undegraded":"2023-10-01T09:31:37.048113+0000","last_fullsized":"2023-10-01T09:31:37.048113+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:23:46.051719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"356'8469","reported_seq":8508,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:53.594313+0000","last_change":"2023-10-01T09:29:24.359090+0000","last_active":"2023-10-01T09:31:53.594313+0000","last_peered":"2023-10-01T09:31:53.594313+0000","last_clean":"2023-10-01T09:31:53.594313+0000","last_became_active":"2023-10-01T09:29:24.358822+0000","last_became_peered":"2023-10-01T09:29:24.358822+0000","last_unstale":"2023-10-01T09:31:53.594313+0000","last_undegraded":"2023-10-01T09:31:53.594313+0000","last_fullsized":"2023-10-01T09:31:53.594313+0000","mapping_epoch":326,"log_start":"331'900","ondisk_log_start":"331'900","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":7569,"log_dups_size":0,"ondisk_log_size":7569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:29:47.448230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266403840,"num_objects":4065,"num_object_clones":0,"num_object_copies":12195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8469,"num_write_kb":401088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":219,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.050007+0000","last_change":"2023-10-01T09:26:29.372326+0000","last_active":"2023-10-01T09:31:37.050007+0000","last_peered":"2023-10-01T09:31:37.050007+0000","last_clean":"2023-10-01T09:31:37.050007+0000","last_became_active":"2023-10-01T09:26:29.372035+0000","last_became_peered":"2023-10-01T09:26:29.372035+0000","last_unstale":"2023-10-01T09:31:37.050007+0000","last_undegraded":"2023-10-01T09:31:37.050007+0000","last_fullsized":"2023-10-01T09:31:37.050007+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":134,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.048214+0000","last_change":"2023-10-01T09:28:02.339901+0000","last_active":"2023-10-01T09:31:37.048214+0000","last_peered":"2023-10-01T09:31:37.048214+0000","last_clean":"2023-10-01T09:31:37.048214+0000","last_became_active":"2023-10-01T09:28:02.339601+0000","last_became_peered":"2023-10-01T09:28:02.339601+0000","last_unstale":"2023-10-01T09:31:37.048214+0000","last_undegraded":"2023-10-01T09:31:37.048214+0000","last_fullsized":"2023-10-01T09:31:37.048214+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:30:16.497879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.3","version":"356'8407","reported_seq":8447,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:53.373275+0000","last_change":"2023-10-01T09:29:24.351403+0000","last_active":"2023-10-01T09:31:53.373275+0000","last_peered":"2023-10-01T09:31:53.373275+0000","last_clean":"2023-10-01T09:31:53.373275+0000","last_became_active":"2023-10-01T09:29:24.351113+0000","last_became_peered":"2023-10-01T09:29:24.351113+0000","last_unstale":"2023-10-01T09:31:53.373275+0000","last_undegraded":"2023-10-01T09:31:53.373275+0000","last_fullsized":"2023-10-01T09:31:53.373275+0000","mapping_epoch":326,"log_start":"333'2000","ondisk_log_start":"333'2000","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":6407,"log_dups_size":0,"ondisk_log_size":6407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:07:56.443376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270204928,"num_objects":4123,"num_object_clones":0,"num_object_copies":12369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8407,"num_write_kb":400960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":195,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.042159+0000","last_change":"2023-10-01T09:30:39.396141+0000","last_active":"2023-10-01T09:31:37.042159+0000","last_peered":"2023-10-01T09:31:37.042159+0000","last_clean":"2023-10-01T09:31:37.042159+0000","last_became_active":"2023-10-01T09:30:39.395843+0000","last_became_peered":"2023-10-01T09:30:39.395843+0000","last_unstale":"2023-10-01T09:31:37.042159+0000","last_undegraded":"2023-10-01T09:31:37.042159+0000","last_fullsized":"2023-10-01T09:31:37.042159+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:13:32.932438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":233,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.048711+0000","last_change":"2023-10-01T09:26:12.320680+0000","last_active":"2023-10-01T09:31:37.048711+0000","last_peered":"2023-10-01T09:31:37.048711+0000","last_clean":"2023-10-01T09:31:37.048711+0000","last_became_active":"2023-10-01T09:26:12.320340+0000","last_became_peered":"2023-10-01T09:26:12.320340+0000","last_unstale":"2023-10-01T09:31:37.048711+0000","last_undegraded":"2023-10-01T09:31:37.048711+0000","last_fullsized":"2023-10-01T09:31:37.048711+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:11:35.796015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"356'8721","reported_seq":8761,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:53.558324+0000","last_change":"2023-10-01T09:29:24.382969+0000","last_active":"2023-10-01T09:31:53.558324+0000","last_peered":"2023-10-01T09:31:53.558324+0000","last_clean":"2023-10-01T09:31:53.558324+0000","last_became_active":"2023-10-01T09:29:24.382653+0000","last_became_peered":"2023-10-01T09:29:24.382653+0000","last_unstale":"2023-10-01T09:31:53.558324+0000","last_undegraded":"2023-10-01T09:31:53.558324+0000","last_fullsized":"2023-10-01T09:31:53.558324+0000","mapping_epoch":326,"log_start":"331'1200","ondisk_log_start":"331'1200","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":7521,"log_dups_size":0,"ondisk_log_size":7521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:37:56.666282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278855680,"num_objects":4255,"num_object_clones":0,"num_object_copies":12765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8721,"num_write_kb":415232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1177,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.048553+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:31:37.048553+0000","last_peered":"2023-10-01T09:31:37.048553+0000","last_clean":"2023-10-01T09:31:37.048553+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:31:37.048553+0000","last_undegraded":"2023-10-01T09:31:37.048553+0000","last_fullsized":"2023-10-01T09:31:37.048553+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":67,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.042008+0000","last_change":"2023-10-01T09:27:01.989188+0000","last_active":"2023-10-01T09:31:37.042008+0000","last_peered":"2023-10-01T09:31:37.042008+0000","last_clean":"2023-10-01T09:31:37.042008+0000","last_became_active":"2023-10-01T09:27:01.988878+0000","last_became_peered":"2023-10-01T09:27:01.988878+0000","last_unstale":"2023-10-01T09:31:37.042008+0000","last_undegraded":"2023-10-01T09:31:37.042008+0000","last_fullsized":"2023-10-01T09:31:37.042008+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":334,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.048018+0000","last_change":"2023-10-01T09:27:01.977155+0000","last_active":"2023-10-01T09:31:37.048018+0000","last_peered":"2023-10-01T09:31:37.048018+0000","last_clean":"2023-10-01T09:31:37.048018+0000","last_became_active":"2023-10-01T09:27:01.975803+0000","last_became_peered":"2023-10-01T09:27:01.975803+0000","last_unstale":"2023-10-01T09:31:37.048018+0000","last_undegraded":"2023-10-01T09:31:37.048018+0000","last_fullsized":"2023-10-01T09:31:37.048018+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:00:55.170662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"356'8419","reported_seq":8457,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:50.701887+0000","last_change":"2023-10-01T09:29:24.376062+0000","last_active":"2023-10-01T09:31:50.701887+0000","last_peered":"2023-10-01T09:31:50.701887+0000","last_clean":"2023-10-01T09:31:50.701887+0000","last_became_active":"2023-10-01T09:29:24.375795+0000","last_became_peered":"2023-10-01T09:29:24.375795+0000","last_unstale":"2023-10-01T09:31:50.701887+0000","last_undegraded":"2023-10-01T09:31:50.701887+0000","last_fullsized":"2023-10-01T09:31:50.701887+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":8419,"log_dups_size":0,"ondisk_log_size":8419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282132480,"num_objects":4305,"num_object_clones":0,"num_object_copies":12915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8419,"num_write_kb":407168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1039,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.043528+0000","last_change":"2023-10-01T09:28:02.595215+0000","last_active":"2023-10-01T09:31:37.043528+0000","last_peered":"2023-10-01T09:31:37.043528+0000","last_clean":"2023-10-01T09:31:37.043528+0000","last_became_active":"2023-10-01T09:28:02.346777+0000","last_became_peered":"2023-10-01T09:28:02.346777+0000","last_unstale":"2023-10-01T09:31:37.043528+0000","last_undegraded":"2023-10-01T09:31:37.043528+0000","last_fullsized":"2023-10-01T09:31:37.043528+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":320,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041576+0000","last_change":"2023-10-01T09:28:02.457735+0000","last_active":"2023-10-01T09:31:37.041576+0000","last_peered":"2023-10-01T09:31:37.041576+0000","last_clean":"2023-10-01T09:31:37.041576+0000","last_became_active":"2023-10-01T09:28:02.457495+0000","last_became_peered":"2023-10-01T09:28:02.457495+0000","last_unstale":"2023-10-01T09:31:37.041576+0000","last_undegraded":"2023-10-01T09:31:37.041576+0000","last_fullsized":"2023-10-01T09:31:37.041576+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:48:53.316760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":30,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041672+0000","last_change":"2023-10-01T09:29:58.362019+0000","last_active":"2023-10-01T09:31:37.041672+0000","last_peered":"2023-10-01T09:31:37.041672+0000","last_clean":"2023-10-01T09:31:37.041672+0000","last_became_active":"2023-10-01T09:29:58.361762+0000","last_became_peered":"2023-10-01T09:29:58.361762+0000","last_unstale":"2023-10-01T09:31:37.041672+0000","last_undegraded":"2023-10-01T09:31:37.041672+0000","last_fullsized":"2023-10-01T09:31:37.041672+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"356'8457","reported_seq":8503,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:53.425251+0000","last_change":"2023-10-01T09:30:39.412038+0000","last_active":"2023-10-01T09:31:53.425251+0000","last_peered":"2023-10-01T09:31:53.425251+0000","last_clean":"2023-10-01T09:31:53.425251+0000","last_became_active":"2023-10-01T09:30:39.400515+0000","last_became_peered":"2023-10-01T09:30:39.400515+0000","last_unstale":"2023-10-01T09:31:53.425251+0000","last_undegraded":"2023-10-01T09:31:53.425251+0000","last_fullsized":"2023-10-01T09:31:53.425251+0000","mapping_epoch":343,"log_start":"332'2000","ondisk_log_start":"332'2000","created":326,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":6457,"log_dups_size":0,"ondisk_log_size":6457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:17:03.280627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270860288,"num_objects":4133,"num_object_clones":0,"num_object_copies":12399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8457,"num_write_kb":402880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":547,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.047762+0000","last_change":"2023-10-01T09:30:39.393231+0000","last_active":"2023-10-01T09:31:37.047762+0000","last_peered":"2023-10-01T09:31:37.047762+0000","last_clean":"2023-10-01T09:31:37.047762+0000","last_became_active":"2023-10-01T09:30:39.392700+0000","last_became_peered":"2023-10-01T09:30:39.392700+0000","last_unstale":"2023-10-01T09:31:37.047762+0000","last_undegraded":"2023-10-01T09:31:37.047762+0000","last_fullsized":"2023-10-01T09:31:37.047762+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:38:26.873688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":385,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.042281+0000","last_change":"2023-10-01T09:28:02.346596+0000","last_active":"2023-10-01T09:31:37.042281+0000","last_peered":"2023-10-01T09:31:37.042281+0000","last_clean":"2023-10-01T09:31:37.042281+0000","last_became_active":"2023-10-01T09:28:02.346318+0000","last_became_peered":"2023-10-01T09:28:02.346318+0000","last_unstale":"2023-10-01T09:31:37.042281+0000","last_undegraded":"2023-10-01T09:31:37.042281+0000","last_fullsized":"2023-10-01T09:31:37.042281+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:27:45.840901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":239,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.048826+0000","last_change":"2023-10-01T09:27:02.004900+0000","last_active":"2023-10-01T09:31:37.048826+0000","last_peered":"2023-10-01T09:31:37.048826+0000","last_clean":"2023-10-01T09:31:37.048826+0000","last_became_active":"2023-10-01T09:27:02.004501+0000","last_became_peered":"2023-10-01T09:27:02.004501+0000","last_unstale":"2023-10-01T09:31:37.048826+0000","last_undegraded":"2023-10-01T09:31:37.048826+0000","last_fullsized":"2023-10-01T09:31:37.048826+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:50:52.617833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"356'8341","reported_seq":8379,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:50.658797+0000","last_change":"2023-10-01T09:29:24.359284+0000","last_active":"2023-10-01T09:31:50.658797+0000","last_peered":"2023-10-01T09:31:50.658797+0000","last_clean":"2023-10-01T09:31:50.658797+0000","last_became_active":"2023-10-01T09:29:24.358932+0000","last_became_peered":"2023-10-01T09:29:24.358932+0000","last_unstale":"2023-10-01T09:31:50.658797+0000","last_undegraded":"2023-10-01T09:31:50.658797+0000","last_fullsized":"2023-10-01T09:31:50.658797+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":8341,"log_dups_size":0,"ondisk_log_size":8341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283181056,"num_objects":4321,"num_object_clones":0,"num_object_copies":12963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8341,"num_write_kb":405184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1282,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.044765+0000","last_change":"2023-10-01T09:28:48.006973+0000","last_active":"2023-10-01T09:31:37.044765+0000","last_peered":"2023-10-01T09:31:37.044765+0000","last_clean":"2023-10-01T09:31:37.044765+0000","last_became_active":"2023-10-01T09:28:48.006289+0000","last_became_peered":"2023-10-01T09:28:48.006289+0000","last_unstale":"2023-10-01T09:31:37.044765+0000","last_undegraded":"2023-10-01T09:31:37.044765+0000","last_fullsized":"2023-10-01T09:31:37.044765+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":315,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":574,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.049546+0000","last_change":"2023-10-01T09:29:04.962927+0000","last_active":"2023-10-01T09:31:37.049546+0000","last_peered":"2023-10-01T09:31:37.049546+0000","last_clean":"2023-10-01T09:31:37.049546+0000","last_became_active":"2023-10-01T09:29:04.962542+0000","last_became_peered":"2023-10-01T09:29:04.962542+0000","last_unstale":"2023-10-01T09:31:37.049546+0000","last_undegraded":"2023-10-01T09:31:37.049546+0000","last_fullsized":"2023-10-01T09:31:37.049546+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":151,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.043449+0000","last_change":"2023-10-01T09:30:39.397227+0000","last_active":"2023-10-01T09:31:37.043449+0000","last_peered":"2023-10-01T09:31:37.043449+0000","last_clean":"2023-10-01T09:31:37.043449+0000","last_became_active":"2023-10-01T09:30:39.396936+0000","last_became_peered":"2023-10-01T09:30:39.396936+0000","last_unstale":"2023-10-01T09:31:37.043449+0000","last_undegraded":"2023-10-01T09:31:37.043449+0000","last_fullsized":"2023-10-01T09:31:37.043449+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":261,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.050318+0000","last_change":"2023-10-01T09:28:02.312356+0000","last_active":"2023-10-01T09:31:37.050318+0000","last_peered":"2023-10-01T09:31:37.050318+0000","last_clean":"2023-10-01T09:31:37.050318+0000","last_became_active":"2023-10-01T09:28:02.312014+0000","last_became_peered":"2023-10-01T09:28:02.312014+0000","last_unstale":"2023-10-01T09:31:37.050318+0000","last_undegraded":"2023-10-01T09:31:37.050318+0000","last_fullsized":"2023-10-01T09:31:37.050318+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"356'8616","reported_seq":8655,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:53.411513+0000","last_change":"2023-10-01T09:29:24.380724+0000","last_active":"2023-10-01T09:31:53.411513+0000","last_peered":"2023-10-01T09:31:53.411513+0000","last_clean":"2023-10-01T09:31:53.411513+0000","last_became_active":"2023-10-01T09:29:24.380523+0000","last_became_peered":"2023-10-01T09:29:24.380523+0000","last_unstale":"2023-10-01T09:31:53.411513+0000","last_undegraded":"2023-10-01T09:31:53.411513+0000","last_fullsized":"2023-10-01T09:31:53.411513+0000","mapping_epoch":326,"log_start":"333'2200","ondisk_log_start":"333'2200","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":6416,"log_dups_size":0,"ondisk_log_size":6416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:16.268423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264765440,"num_objects":4040,"num_object_clones":0,"num_object_copies":12120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8616,"num_write_kb":404992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":452,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.049629+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:31:37.049629+0000","last_peered":"2023-10-01T09:31:37.049629+0000","last_clean":"2023-10-01T09:31:37.049629+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:31:37.049629+0000","last_undegraded":"2023-10-01T09:31:37.049629+0000","last_fullsized":"2023-10-01T09:31:37.049629+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":363,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:48.608996+0000","last_change":"2023-10-01T09:27:01.986786+0000","last_active":"2023-10-01T09:31:48.608996+0000","last_peered":"2023-10-01T09:31:48.608996+0000","last_clean":"2023-10-01T09:31:48.608996+0000","last_became_active":"2023-10-01T09:27:01.986466+0000","last_became_peered":"2023-10-01T09:27:01.986466+0000","last_unstale":"2023-10-01T09:31:48.608996+0000","last_undegraded":"2023-10-01T09:31:48.608996+0000","last_fullsized":"2023-10-01T09:31:48.608996+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:55:51.244125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":362,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041789+0000","last_change":"2023-10-01T09:19:30.091481+0000","last_active":"2023-10-01T09:31:37.041789+0000","last_peered":"2023-10-01T09:31:37.041789+0000","last_clean":"2023-10-01T09:31:37.041789+0000","last_became_active":"2023-10-01T09:19:30.091151+0000","last_became_peered":"2023-10-01T09:19:30.091151+0000","last_unstale":"2023-10-01T09:31:37.041789+0000","last_undegraded":"2023-10-01T09:31:37.041789+0000","last_fullsized":"2023-10-01T09:31:37.041789+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:09.127408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"356'8568","reported_seq":8614,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:53.425329+0000","last_change":"2023-10-01T09:30:39.412276+0000","last_active":"2023-10-01T09:31:53.425329+0000","last_peered":"2023-10-01T09:31:53.425329+0000","last_clean":"2023-10-01T09:31:53.425329+0000","last_became_active":"2023-10-01T09:30:39.400521+0000","last_became_peered":"2023-10-01T09:30:39.400521+0000","last_unstale":"2023-10-01T09:31:53.425329+0000","last_undegraded":"2023-10-01T09:31:53.425329+0000","last_fullsized":"2023-10-01T09:31:53.425329+0000","mapping_epoch":343,"log_start":"333'2100","ondisk_log_start":"333'2100","created":326,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":6468,"log_dups_size":0,"ondisk_log_size":6468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:35.349863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269352960,"num_objects":4110,"num_object_clones":0,"num_object_copies":12330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8568,"num_write_kb":405696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":755,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.050269+0000","last_change":"2023-10-01T09:28:03.317559+0000","last_active":"2023-10-01T09:31:37.050269+0000","last_peered":"2023-10-01T09:31:37.050269+0000","last_clean":"2023-10-01T09:31:37.050269+0000","last_became_active":"2023-10-01T09:28:03.317237+0000","last_became_peered":"2023-10-01T09:28:03.317237+0000","last_unstale":"2023-10-01T09:31:37.050269+0000","last_undegraded":"2023-10-01T09:31:37.050269+0000","last_fullsized":"2023-10-01T09:31:37.050269+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":376,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.048360+0000","last_change":"2023-10-01T09:27:01.999110+0000","last_active":"2023-10-01T09:31:37.048360+0000","last_peered":"2023-10-01T09:31:37.048360+0000","last_clean":"2023-10-01T09:31:37.048360+0000","last_became_active":"2023-10-01T09:27:01.998740+0000","last_became_peered":"2023-10-01T09:27:01.998740+0000","last_unstale":"2023-10-01T09:31:37.048360+0000","last_undegraded":"2023-10-01T09:31:37.048360+0000","last_fullsized":"2023-10-01T09:31:37.048360+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:40:08.795106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":330,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:48.609050+0000","last_change":"2023-10-01T09:27:01.998986+0000","last_active":"2023-10-01T09:31:48.609050+0000","last_peered":"2023-10-01T09:31:48.609050+0000","last_clean":"2023-10-01T09:31:48.609050+0000","last_became_active":"2023-10-01T09:27:01.998709+0000","last_became_peered":"2023-10-01T09:27:01.998709+0000","last_unstale":"2023-10-01T09:31:48.609050+0000","last_undegraded":"2023-10-01T09:31:48.609050+0000","last_fullsized":"2023-10-01T09:31:48.609050+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:01:14.296638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"356'8797","reported_seq":8837,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:53.517420+0000","last_change":"2023-10-01T09:29:24.378898+0000","last_active":"2023-10-01T09:31:53.517420+0000","last_peered":"2023-10-01T09:31:53.517420+0000","last_clean":"2023-10-01T09:31:53.517420+0000","last_became_active":"2023-10-01T09:29:24.378624+0000","last_became_peered":"2023-10-01T09:29:24.378624+0000","last_unstale":"2023-10-01T09:31:53.517420+0000","last_undegraded":"2023-10-01T09:31:53.517420+0000","last_fullsized":"2023-10-01T09:31:53.517420+0000","mapping_epoch":326,"log_start":"331'1200","ondisk_log_start":"331'1200","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":7597,"log_dups_size":0,"ondisk_log_size":7597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:12:28.028735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278593536,"num_objects":4251,"num_object_clones":0,"num_object_copies":12753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8797,"num_write_kb":417536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":269,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.050131+0000","last_change":"2023-10-01T09:30:39.402924+0000","last_active":"2023-10-01T09:31:37.050131+0000","last_peered":"2023-10-01T09:31:37.050131+0000","last_clean":"2023-10-01T09:31:37.050131+0000","last_became_active":"2023-10-01T09:30:39.402594+0000","last_became_peered":"2023-10-01T09:30:39.402594+0000","last_unstale":"2023-10-01T09:31:37.050131+0000","last_undegraded":"2023-10-01T09:31:37.050131+0000","last_fullsized":"2023-10-01T09:31:37.050131+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":942,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:38.417561+0000","last_change":"2023-10-01T09:29:13.014839+0000","last_active":"2023-10-01T09:31:38.417561+0000","last_peered":"2023-10-01T09:31:38.417561+0000","last_clean":"2023-10-01T09:31:38.417561+0000","last_became_active":"2023-10-01T09:29:13.014139+0000","last_became_peered":"2023-10-01T09:29:13.014139+0000","last_unstale":"2023-10-01T09:31:38.417561+0000","last_undegraded":"2023-10-01T09:31:38.417561+0000","last_fullsized":"2023-10-01T09:31:38.417561+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:23:10.370435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":45,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.046924+0000","last_change":"2023-10-01T09:30:39.418085+0000","last_active":"2023-10-01T09:31:37.046924+0000","last_peered":"2023-10-01T09:31:37.046924+0000","last_clean":"2023-10-01T09:31:37.046924+0000","last_became_active":"2023-10-01T09:30:39.417846+0000","last_became_peered":"2023-10-01T09:30:39.417846+0000","last_unstale":"2023-10-01T09:31:37.046924+0000","last_undegraded":"2023-10-01T09:31:37.046924+0000","last_fullsized":"2023-10-01T09:31:37.046924+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"356'8263","reported_seq":8302,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-10-01T09:31:50.696274+0000","last_change":"2023-10-01T09:29:24.379832+0000","last_active":"2023-10-01T09:31:50.696274+0000","last_peered":"2023-10-01T09:31:50.696274+0000","last_clean":"2023-10-01T09:31:50.696274+0000","last_became_active":"2023-10-01T09:29:24.379540+0000","last_became_peered":"2023-10-01T09:29:24.379540+0000","last_unstale":"2023-10-01T09:31:50.696274+0000","last_undegraded":"2023-10-01T09:31:50.696274+0000","last_fullsized":"2023-10-01T09:31:50.696274+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":326,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:29:23.333335+0000","last_clean_scrub_stamp":"2023-10-01T09:29:23.333335+0000","objects_scrubbed":0,"log_size":8263,"log_dups_size":0,"ondisk_log_size":8263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275644440,"num_objects":4207,"num_object_clones":0,"num_object_copies":12621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4207,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8263,"num_write_kb":398977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":69,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041100+0000","last_change":"2023-10-01T09:29:11.374086+0000","last_active":"2023-10-01T09:31:37.041100+0000","last_peered":"2023-10-01T09:31:37.041100+0000","last_clean":"2023-10-01T09:31:37.041100+0000","last_became_active":"2023-10-01T09:28:02.319106+0000","last_became_peered":"2023-10-01T09:28:02.319106+0000","last_unstale":"2023-10-01T09:31:37.041100+0000","last_undegraded":"2023-10-01T09:31:37.041100+0000","last_fullsized":"2023-10-01T09:31:37.041100+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":354,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041846+0000","last_change":"2023-10-01T09:26:12.291583+0000","last_active":"2023-10-01T09:31:37.041846+0000","last_peered":"2023-10-01T09:31:37.041846+0000","last_clean":"2023-10-01T09:31:37.041846+0000","last_became_active":"2023-10-01T09:26:12.271891+0000","last_became_peered":"2023-10-01T09:26:12.271891+0000","last_unstale":"2023-10-01T09:31:37.041846+0000","last_undegraded":"2023-10-01T09:31:37.041846+0000","last_fullsized":"2023-10-01T09:31:37.041846+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:53:06.889203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":76,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041673+0000","last_change":"2023-10-01T09:28:02.314867+0000","last_active":"2023-10-01T09:31:37.041673+0000","last_peered":"2023-10-01T09:31:37.041673+0000","last_clean":"2023-10-01T09:31:37.041673+0000","last_became_active":"2023-10-01T09:28:02.314480+0000","last_became_peered":"2023-10-01T09:28:02.314480+0000","last_unstale":"2023-10-01T09:31:37.041673+0000","last_undegraded":"2023-10-01T09:31:37.041673+0000","last_fullsized":"2023-10-01T09:31:37.041673+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":308,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041647+0000","last_change":"2023-10-01T09:26:12.271469+0000","last_active":"2023-10-01T09:31:37.041647+0000","last_peered":"2023-10-01T09:31:37.041647+0000","last_clean":"2023-10-01T09:31:37.041647+0000","last_became_active":"2023-10-01T09:26:12.270972+0000","last_became_peered":"2023-10-01T09:26:12.270972+0000","last_unstale":"2023-10-01T09:31:37.041647+0000","last_undegraded":"2023-10-01T09:31:37.041647+0000","last_fullsized":"2023-10-01T09:31:37.041647+0000","mapping_epoch":283,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:45:03.822952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":386,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.048447+0000","last_change":"2023-10-01T09:22:40.166927+0000","last_active":"2023-10-01T09:31:37.048447+0000","last_peered":"2023-10-01T09:31:37.048447+0000","last_clean":"2023-10-01T09:31:37.048447+0000","last_became_active":"2023-10-01T09:22:40.166606+0000","last_became_peered":"2023-10-01T09:22:40.166606+0000","last_unstale":"2023-10-01T09:31:37.048447+0000","last_undegraded":"2023-10-01T09:31:37.048447+0000","last_fullsized":"2023-10-01T09:31:37.048447+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:54:18.044207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":84,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041968+0000","last_change":"2023-10-01T09:28:02.507232+0000","last_active":"2023-10-01T09:31:37.041968+0000","last_peered":"2023-10-01T09:31:37.041968+0000","last_clean":"2023-10-01T09:31:37.041968+0000","last_became_active":"2023-10-01T09:28:02.340776+0000","last_became_peered":"2023-10-01T09:28:02.340776+0000","last_unstale":"2023-10-01T09:31:37.041968+0000","last_undegraded":"2023-10-01T09:31:37.041968+0000","last_fullsized":"2023-10-01T09:31:37.041968+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":30,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.042130+0000","last_change":"2023-10-01T09:29:58.359570+0000","last_active":"2023-10-01T09:31:37.042130+0000","last_peered":"2023-10-01T09:31:37.042130+0000","last_clean":"2023-10-01T09:31:37.042130+0000","last_became_active":"2023-10-01T09:29:58.359181+0000","last_became_peered":"2023-10-01T09:29:58.359181+0000","last_unstale":"2023-10-01T09:31:37.042130+0000","last_undegraded":"2023-10-01T09:31:37.042130+0000","last_fullsized":"2023-10-01T09:31:37.042130+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":84,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041955+0000","last_change":"2023-10-01T09:28:02.551019+0000","last_active":"2023-10-01T09:31:37.041955+0000","last_peered":"2023-10-01T09:31:37.041955+0000","last_clean":"2023-10-01T09:31:37.041955+0000","last_became_active":"2023-10-01T09:28:02.342509+0000","last_became_peered":"2023-10-01T09:28:02.342509+0000","last_unstale":"2023-10-01T09:31:37.041955+0000","last_undegraded":"2023-10-01T09:31:37.041955+0000","last_fullsized":"2023-10-01T09:31:37.041955+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":236,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.041173+0000","last_change":"2023-10-01T09:30:39.397184+0000","last_active":"2023-10-01T09:31:37.041173+0000","last_peered":"2023-10-01T09:31:37.041173+0000","last_clean":"2023-10-01T09:31:37.041173+0000","last_became_active":"2023-10-01T09:30:39.396907+0000","last_became_peered":"2023-10-01T09:30:39.396907+0000","last_unstale":"2023-10-01T09:31:37.041173+0000","last_undegraded":"2023-10-01T09:31:37.041173+0000","last_fullsized":"2023-10-01T09:31:37.041173+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:16:14.113046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":158,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.049630+0000","last_change":"2023-10-01T09:28:02.321395+0000","last_active":"2023-10-01T09:31:37.049630+0000","last_peered":"2023-10-01T09:31:37.049630+0000","last_clean":"2023-10-01T09:31:37.049630+0000","last_became_active":"2023-10-01T09:28:02.321071+0000","last_became_peered":"2023-10-01T09:28:02.321071+0000","last_unstale":"2023-10-01T09:31:37.049630+0000","last_undegraded":"2023-10-01T09:31:37.049630+0000","last_fullsized":"2023-10-01T09:31:37.049630+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:12:45.734920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":263,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.047835+0000","last_change":"2023-10-01T09:30:39.392942+0000","last_active":"2023-10-01T09:31:37.047835+0000","last_peered":"2023-10-01T09:31:37.047835+0000","last_clean":"2023-10-01T09:31:37.047835+0000","last_became_active":"2023-10-01T09:30:39.392663+0000","last_became_peered":"2023-10-01T09:30:39.392663+0000","last_unstale":"2023-10-01T09:31:37.047835+0000","last_undegraded":"2023-10-01T09:31:37.047835+0000","last_fullsized":"2023-10-01T09:31:37.047835+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:19:39.644244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":447,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:48.609146+0000","last_change":"2023-10-01T09:27:02.019266+0000","last_active":"2023-10-01T09:31:48.609146+0000","last_peered":"2023-10-01T09:31:48.609146+0000","last_clean":"2023-10-01T09:31:48.609146+0000","last_became_active":"2023-10-01T09:27:02.006744+0000","last_became_peered":"2023-10-01T09:27:02.006744+0000","last_unstale":"2023-10-01T09:31:48.609146+0000","last_undegraded":"2023-10-01T09:31:48.609146+0000","last_fullsized":"2023-10-01T09:31:48.609146+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:39:21.899359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":314,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-10-01T09:31:37.042154+0000","last_change":"2023-10-01T09:28:02.343724+0000","last_active":"2023-10-01T09:31:37.042154+0000","last_peered":"2023-10-01T09:31:37.042154+0000","last_clean":"2023-10-01T09:31:37.042154+0000","last_became_active":"2023-10-01T09:28:02.339485+0000","last_became_peered":"2023-10-01T09:28:02.339485+0000","last_unstale":"2023-10-01T09:31:37.042154+0000","last_undegraded":"2023-10-01T09:31:37.042154+0000","last_fullsized":"2023-10-01T09:31:37.042154+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:54:52.086601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":97,"num_bytes_recovered":28418,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":22016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42916,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4384096280,"num_objects":66897,"num_object_clones":0,"num_object_copies":200691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66897,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135921,"num_write_kb":6490113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13247393792,"data_stored":13247381576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118321,"ondisk_log_size":118321,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":209,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430575,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2238048,"kb_used_data":1644248,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":91507104,"statfs":{"total":95995035648,"available":93703274496,"internally_reserved":0,"allocated":1683709952,"data_stored":1682828158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:31:27 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":2.1850000000000001,"15min":3.903},"min":{"1min":0.249,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":7.056,"5min":64.411000000000001,"15min":117.855},"last":2.7589999999999999},{"interface":"front","average":{"1min":1.804,"5min":2.6139999999999999,"15min":3.9590000000000001},"min":{"1min":0.27300000000000002,"5min":0.23899999999999999,"15min":0.22500000000000001},"max":{"1min":3.4420000000000002,"5min":64.519000000000005,"15min":120.348},"last":0.61399999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:31:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7290000000000001,"5min":2.0009999999999999,"15min":3.1930000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":6.8040000000000003,"5min":10.611000000000001,"15min":135.023},"last":6.4080000000000004},{"interface":"front","average":{"1min":2.5880000000000001,"5min":2.0430000000000001,"15min":3.0499999999999998},"min":{"1min":0.27200000000000002,"5min":0.25800000000000001,"15min":0.23599999999999999},"max":{"1min":7.7649999999999997,"5min":13.688000000000001,"15min":134.51499999999999},"last":4.4740000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:31:18 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.8380000000000001,"15min":2.899},"min":{"1min":0.27500000000000002,"5min":0.249,"15min":0.249},"max":{"1min":3.3199999999999998,"5min":10.452,"15min":104.837},"last":0.86899999999999999},{"interface":"front","average":{"1min":2.581,"5min":1.841,"15min":2.8500000000000001},"min":{"1min":0.308,"5min":0.27400000000000002,"15min":0.247},"max":{"1min":10.026,"5min":10.026,"15min":106.379},"last":4.4390000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:31:48 2023","interfaces":[{"interface":"back","average":{"1min":2.044,"5min":2.859,"15min":3.3540000000000001},"min":{"1min":0.379,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":7.2210000000000001,"5min":80.352999999999994,"15min":164.595},"last":7.2060000000000004},{"interface":"front","average":{"1min":1.859,"5min":2.7570000000000001,"15min":3.3109999999999999},"min":{"1min":0.39800000000000002,"5min":0.313,"15min":0.29399999999999998},"max":{"1min":7.2320000000000002,"5min":80.364000000000004,"15min":164.60499999999999},"last":4.2450000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:31:27 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":0.60699999999999998,"15min":1.542},"min":{"1min":0.28399999999999997,"5min":0.254,"15min":0.23499999999999999},"max":{"1min":3.9740000000000002,"5min":3.9740000000000002,"15min":82.992000000000004},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.65400000000000003,"15min":1.603},"min":{"1min":0.29199999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.1859999999999999,"5min":3.25,"15min":83.058999999999997},"last":0.65700000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:31:27 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.70599999999999996,"15min":1.8240000000000001},"min":{"1min":0.27800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.3820000000000001,"5min":4.6989999999999998,"15min":13.265000000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.72699999999999998,"15min":1.762},"min":{"1min":0.317,"5min":0.311,"15min":0.27600000000000002},"max":{"1min":3.1070000000000002,"5min":3.7639999999999998,"15min":13.292999999999999},"last":0.59799999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:30:59 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.77600000000000002,"15min":0.79200000000000004},"min":{"1min":0.34599999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.5779999999999998,"5min":3.5779999999999998,"15min":3.5779999999999998},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.75900000000000001,"15min":0.65700000000000003},"min":{"1min":0.37,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.3860000000000001,"5min":6.3860000000000001,"15min":6.3860000000000001},"last":0.64300000000000002}]}]},{"osd":6,"up_from":294,"seq":1262720385086,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1969936,"kb_used_data":1628232,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":91775216,"statfs":{"total":95995035648,"available":93977821184,"internally_reserved":0,"allocated":1667309568,"data_stored":1666443411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":4.8410000000000002,"5min":2.294,"15min":2.1120000000000001},"min":{"1min":0.23799999999999999,"5min":0.222,"15min":0.222},"max":{"1min":62.569000000000003,"5min":62.569000000000003,"15min":62.569000000000003},"last":3.3359999999999999},{"interface":"front","average":{"1min":4.7130000000000001,"5min":2.3220000000000001,"15min":1.7410000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":62.912999999999997,"5min":62.912999999999997,"15min":62.912999999999997},"last":0.27700000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":5.0270000000000001,"5min":2.2599999999999998,"15min":1.968},"min":{"1min":0.29299999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":62.698999999999998,"5min":62.698999999999998,"15min":62.698999999999998},"last":0.82999999999999996},{"interface":"front","average":{"1min":4.8769999999999998,"5min":2.1230000000000002,"15min":1.9359999999999999},"min":{"1min":0.34899999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":62.704000000000001,"5min":62.704000000000001,"15min":62.704000000000001},"last":1.0409999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8799999999999999,"5min":2.21,"15min":2.738},"min":{"1min":0.29799999999999999,"5min":0.254,"15min":0.254},"max":{"1min":6.9889999999999999,"5min":26.289999999999999,"15min":26.289999999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.6830000000000001,"5min":2.2360000000000002,"15min":2.8879999999999999},"min":{"1min":0.315,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.8170000000000002,"5min":26.388999999999999,"15min":26.388999999999999},"last":1.8009999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":4.3470000000000004,"5min":2.7130000000000001,"15min":2.5489999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":62.546999999999997,"5min":62.546999999999997,"15min":62.546999999999997},"last":4.8639999999999999},{"interface":"front","average":{"1min":4.7709999999999999,"5min":2.1640000000000001,"15min":1.6799999999999999},"min":{"1min":0.35599999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":62.465000000000003,"5min":62.465000000000003,"15min":62.465000000000003},"last":1.7410000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.496,"15min":0.41299999999999998},"min":{"1min":0.30099999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.9059999999999999,"5min":3.1560000000000001,"15min":3.1560000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.621,"15min":0.56100000000000005},"min":{"1min":0.35599999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.75,"5min":4.1319999999999997,"15min":4.1319999999999997},"last":3.556}]},{"osd":5,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.59199999999999997,"15min":0.47899999999999998},"min":{"1min":0.28999999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.9450000000000001,"5min":4.6200000000000001,"15min":4.6200000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.73699999999999999,"15min":0.624},"min":{"1min":0.30299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.4900000000000002,"5min":4.4900000000000002,"15min":4.4900000000000002},"last":3.6589999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":3.1499999999999999,"15min":4.8040000000000003},"min":{"1min":0.33100000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.861,"5min":115.11,"15min":115.11},"last":3.5030000000000001},{"interface":"front","average":{"1min":0.873,"5min":3.177,"15min":4.7690000000000001},"min":{"1min":0.34499999999999997,"5min":0.309,"15min":0.309},"max":{"1min":3.3599999999999999,"5min":114.666,"15min":114.666},"last":0.47699999999999998}]}]},{"osd":1,"up_from":94,"seq":403726926063,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2355772,"kb_used_data":1860340,"kb_used_omap":7,"kb_used_meta":495352,"kb_avail":91389380,"statfs":{"total":95995035648,"available":93582725120,"internally_reserved":0,"allocated":1904988160,"data_stored":1903384277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":507240871},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:31:36 2023","interfaces":[{"interface":"back","average":{"1min":3.6000000000000001,"5min":3.8570000000000002,"15min":4.327},"min":{"1min":0.19900000000000001,"5min":0.19600000000000001,"15min":0.17499999999999999},"max":{"1min":11.266,"5min":82.359999999999999,"15min":174.40199999999999},"last":7.5099999999999998},{"interface":"front","average":{"1min":3.9380000000000002,"5min":3.3530000000000002,"15min":4.4790000000000001},"min":{"1min":0.25800000000000001,"5min":0.188,"15min":0.17599999999999999},"max":{"1min":12.978,"5min":82.078999999999994,"15min":173.77000000000001},"last":7.7919999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:31:18 2023","interfaces":[{"interface":"back","average":{"1min":4.1070000000000002,"5min":4.6920000000000002,"15min":3.4079999999999999},"min":{"1min":0.218,"5min":0.215,"15min":0.182},"max":{"1min":13.468,"5min":56.573,"15min":77.325000000000003},"last":3.1139999999999999},{"interface":"front","average":{"1min":3.3260000000000001,"5min":4.9790000000000001,"15min":3.7730000000000001},"min":{"1min":0.28599999999999998,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":12.878,"5min":55.930999999999997,"15min":80.361000000000004},"last":7.7670000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:31:51 2023","interfaces":[{"interface":"back","average":{"1min":4.181,"5min":5.6639999999999997,"15min":4.4409999999999998},"min":{"1min":0.29399999999999998,"5min":0.28499999999999998,"15min":0.222},"max":{"1min":13.284000000000001,"5min":200.56800000000001,"15min":200.56800000000001},"last":7.6890000000000001},{"interface":"front","average":{"1min":5.7709999999999999,"5min":6.2839999999999998,"15min":4.3959999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.20999999999999999},"max":{"1min":30.231999999999999,"5min":200.63800000000001,"15min":200.63800000000001},"last":0.25}]},{"osd":4,"last update":"Sun Oct 1 09:31:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.3300000000000001,"15min":1.6379999999999999},"min":{"1min":0.33000000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.6900000000000004,"5min":7.1669999999999998,"15min":40.673999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.5950000000000002,"5min":2.2490000000000001,"15min":2.0329999999999999},"min":{"1min":0.44500000000000001,"5min":0.29799999999999999,"15min":0.25800000000000001},"max":{"1min":9.298,"5min":18.219000000000001,"15min":40.810000000000002},"last":0.34999999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:31:18 2023","interfaces":[{"interface":"back","average":{"1min":3.8410000000000002,"5min":2.0099999999999998,"15min":1.6060000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":47.106000000000002,"5min":47.106000000000002,"15min":47.106000000000002},"last":0.53200000000000003},{"interface":"front","average":{"1min":4.5519999999999996,"5min":2.794,"15min":2.3580000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":47.164999999999999,"5min":47.164999999999999,"15min":47.164999999999999},"last":0.318}]},{"osd":6,"last update":"Sun Oct 1 09:30:56 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":2.2989999999999999,"15min":2.9119999999999999},"min":{"1min":0.35699999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.3019999999999996,"5min":10.196999999999999,"15min":10.196999999999999},"last":7.7309999999999999},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.661,"15min":1.9059999999999999},"min":{"1min":0.41399999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.8109999999999999,"5min":7.2679999999999998,"15min":7.2679999999999998},"last":1.3839999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:31:36 2023","interfaces":[{"interface":"back","average":{"1min":2.1389999999999998,"5min":1.5840000000000001,"15min":1.8620000000000001},"min":{"1min":0.39500000000000002,"5min":0.36799999999999999,"15min":0.316},"max":{"1min":14.717000000000001,"5min":14.717000000000001,"15min":23.710999999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.4750000000000001,"5min":2.2010000000000001,"15min":2.0190000000000001},"min":{"1min":0.42899999999999999,"5min":0.34599999999999997,"15min":0.30199999999999999},"max":{"1min":9.25,"5min":18.291,"15min":23.736999999999998},"last":0.437}]}]},{"osd":0,"up_from":89,"seq":382252089597,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2505680,"kb_used_data":1871240,"kb_used_omap":8,"kb_used_meta":634423,"kb_avail":91239472,"statfs":{"total":95995035648,"available":93429219328,"internally_reserved":0,"allocated":1916149760,"data_stored":1914530674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":649650114},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:31:45 2023","interfaces":[{"interface":"back","average":{"1min":5.4859999999999998,"5min":2.6739999999999999,"15min":3.0790000000000002},"min":{"1min":0.19700000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":61.173000000000002,"5min":61.173000000000002,"15min":187.41399999999999},"last":5.2290000000000001},{"interface":"front","average":{"1min":7.0220000000000002,"5min":4.0650000000000004,"15min":3.9399999999999999},"min":{"1min":0.23999999999999999,"5min":0.22500000000000001,"15min":0.193},"max":{"1min":68.587999999999994,"5min":68.587999999999994,"15min":187.78700000000001},"last":11.680999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.792,"5min":1.51,"15min":2.0920000000000001},"min":{"1min":0.23000000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":11.109,"5min":11.468999999999999,"15min":39.048999999999999},"last":4.9690000000000003},{"interface":"front","average":{"1min":2.1579999999999999,"5min":3.093,"15min":3.0299999999999998},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":8.2149999999999999,"5min":22.07,"15min":38.945999999999998},"last":3.9009999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:31:45 2023","interfaces":[{"interface":"back","average":{"1min":6.0019999999999998,"5min":5.718,"15min":3.7610000000000001},"min":{"1min":0.28799999999999998,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":59.216000000000001,"5min":139.85300000000001,"15min":139.85300000000001},"last":5.391},{"interface":"front","average":{"1min":6.3239999999999998,"5min":6.8940000000000001,"15min":4.4100000000000001},"min":{"1min":0.26300000000000001,"5min":0.25800000000000001,"15min":0.219},"max":{"1min":59.177999999999997,"5min":139.70099999999999,"15min":139.70099999999999},"last":2.0139999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:31:33 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":2.302,"15min":2.5219999999999998},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.27300000000000002},"max":{"1min":4.8710000000000004,"5min":18.600999999999999,"15min":112.52200000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.5680000000000001,"5min":2.3250000000000002,"15min":2.5459999999999998},"min":{"1min":0.33700000000000002,"5min":0.28799999999999998,"15min":0.26900000000000002},"max":{"1min":5.2249999999999996,"5min":28.035,"15min":113.124},"last":2.109}]},{"osd":5,"last update":"Sun Oct 1 09:31:23 2023","interfaces":[{"interface":"back","average":{"1min":8.8879999999999999,"5min":4.0289999999999999,"15min":2.4500000000000002},"min":{"1min":0.32900000000000001,"5min":0.317,"15min":0.317},"max":{"1min":122.825,"5min":122.825,"15min":122.825},"last":2.1739999999999999},{"interface":"front","average":{"1min":9.4090000000000007,"5min":2.806,"15min":2.0659999999999998},"min":{"1min":0.42599999999999999,"5min":0.312,"15min":0.31},"max":{"1min":122.837,"5min":122.837,"15min":122.837},"last":5.0170000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:31:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2969999999999999,"5min":1.23,"15min":1.3959999999999999},"min":{"1min":0.39300000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":5.3600000000000003,"5min":11.387,"15min":11.387},"last":5.0709999999999997},{"interface":"front","average":{"1min":1.8069999999999999,"5min":2.266,"15min":2.4260000000000002},"min":{"1min":0.32700000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.1419999999999995,"5min":18.521000000000001,"15min":18.521000000000001},"last":0.48499999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:31:33 2023","interfaces":[{"interface":"back","average":{"1min":2.2290000000000001,"5min":1.456,"15min":2.0339999999999998},"min":{"1min":0.41199999999999998,"5min":0.34499999999999997,"15min":0.29199999999999998},"max":{"1min":10.991,"5min":11.413,"15min":98.283000000000001},"last":5.101},{"interface":"front","average":{"1min":1.71,"5min":2.52,"15min":2.573},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.312},"max":{"1min":4.835,"5min":18.565999999999999,"15min":98.311999999999998},"last":0.44500000000000001}]}]},{"osd":2,"up_from":99,"seq":425201762536,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2539164,"kb_used_data":1930132,"kb_used_omap":11,"kb_used_meta":608884,"kb_avail":91205988,"statfs":{"total":95995035648,"available":93394931712,"internally_reserved":0,"allocated":1976455168,"data_stored":1974878448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":623498103},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:31:27 2023","interfaces":[{"interface":"back","average":{"1min":3.242,"5min":8.0139999999999993,"15min":5.2130000000000001},"min":{"1min":0.221,"5min":0.20499999999999999,"15min":0.17599999999999999},"max":{"1min":10.510999999999999,"5min":224.56,"15min":224.56},"last":1.508},{"interface":"front","average":{"1min":2.6499999999999999,"5min":8.5129999999999999,"15min":5.8369999999999997},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20100000000000001},"max":{"1min":6.702,"5min":225.66900000000001,"15min":225.66900000000001},"last":1.7969999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":3.3839999999999999,"5min":9.1959999999999997,"15min":5.6710000000000003},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":14.653,"5min":224.917,"15min":224.917},"last":1.345},{"interface":"front","average":{"1min":3.9420000000000002,"5min":10.529999999999999,"15min":6.4569999999999999},"min":{"1min":0.375,"5min":0.222,"15min":0.20899999999999999},"max":{"1min":13.664,"5min":225.625,"15min":225.625},"last":0.23599999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:31:47 2023","interfaces":[{"interface":"back","average":{"1min":3.198,"5min":6.4960000000000004,"15min":6.1210000000000004},"min":{"1min":0.26800000000000002,"5min":0.218,"15min":0.19600000000000001},"max":{"1min":17.625,"5min":226.124,"15min":253.238},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.6480000000000001,"5min":6.5300000000000002,"15min":5.9359999999999999},"min":{"1min":0.251,"5min":0.245,"15min":0.22600000000000001},"max":{"1min":13.672000000000001,"5min":226.32900000000001,"15min":252.54900000000001},"last":1.2310000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":3.6539999999999999,"15min":2.7719999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.27200000000000002},"max":{"1min":5.9349999999999996,"5min":224.90199999999999,"15min":224.90199999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":3.3330000000000002,"5min":4.1710000000000003,"15min":2.903},"min":{"1min":0.33900000000000002,"5min":0.31,"15min":0.27500000000000002},"max":{"1min":17.594999999999999,"5min":225.386,"15min":225.386},"last":0.52600000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":4.2830000000000004,"5min":3.9980000000000002,"15min":2.7320000000000002},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":40.542000000000002,"5min":224.989,"15min":224.989},"last":0.41599999999999998},{"interface":"front","average":{"1min":3.5680000000000001,"5min":4.1589999999999998,"15min":2.875},"min":{"1min":0.34399999999999997,"5min":0.309,"15min":0.309},"max":{"1min":40.436,"5min":224.81,"15min":224.81},"last":0.46200000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:30:52 2023","interfaces":[{"interface":"back","average":{"1min":4.4829999999999997,"5min":4.5439999999999996,"15min":2.907},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":40.582999999999998,"5min":225.41499999999999,"15min":225.41499999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":4.6239999999999997,"5min":4.0609999999999999,"15min":2.274},"min":{"1min":0.42199999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":40.210999999999999,"5min":225.02000000000001,"15min":225.02000000000001},"last":0.44900000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":4.9530000000000003,"5min":4.5499999999999998,"15min":3.3359999999999999},"min":{"1min":0.38900000000000001,"5min":0.308,"15min":0.308},"max":{"1min":38.802,"5min":224.94999999999999,"15min":224.94999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":3.3730000000000002,"5min":4.5750000000000002,"15min":3.5739999999999998},"min":{"1min":0.37,"5min":0.35299999999999998,"15min":0.33600000000000002},"max":{"1min":38.817,"5min":224.92599999999999,"15min":224.92599999999999},"last":0.441}]}]},{"osd":3,"up_from":104,"seq":446676599010,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2465748,"kb_used_data":1905100,"kb_used_omap":3,"kb_used_meta":560636,"kb_avail":91279404,"statfs":{"total":95995035648,"available":93470109696,"internally_reserved":0,"allocated":1950822400,"data_stored":1949195158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":574092007},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:31:45 2023","interfaces":[{"interface":"back","average":{"1min":18.123000000000001,"5min":7.8920000000000003,"15min":5.1900000000000004},"min":{"1min":0.252,"5min":0.20899999999999999,"15min":0.17100000000000001},"max":{"1min":109.446,"5min":128.47800000000001,"15min":128.47800000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":17.632999999999999,"5min":7.7480000000000002,"15min":5.226},"min":{"1min":0.33800000000000002,"5min":0.218,"15min":0.184},"max":{"1min":109.304,"5min":127.15600000000001,"15min":127.15600000000001},"last":0.70699999999999996}]},{"osd":1,"last update":"Sun Oct 1 09:31:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3029999999999999,"5min":2.4119999999999999,"15min":4.5709999999999997},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":7.0129999999999999,"5min":11.359,"15min":228.60499999999999},"last":0.23300000000000001},{"interface":"front","average":{"1min":3.1779999999999999,"5min":3.3149999999999999,"15min":4.8849999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.215},"max":{"1min":11.516,"5min":23.129000000000001,"15min":228.28399999999999},"last":0.82099999999999995}]},{"osd":2,"last update":"Sun Oct 1 09:31:45 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":3.286,"15min":3.3199999999999998},"min":{"1min":0.249,"5min":0.249,"15min":0.22500000000000001},"max":{"1min":3.0379999999999998,"5min":43.222999999999999,"15min":62.909999999999997},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.915,"5min":4.6619999999999999,"15min":3.9620000000000002},"min":{"1min":0.26000000000000001,"5min":0.252,"15min":0.219},"max":{"1min":7.7350000000000003,"5min":78.594999999999999,"15min":78.594999999999999},"last":6.9770000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:31:45 2023","interfaces":[{"interface":"back","average":{"1min":1.6639999999999999,"5min":1.4890000000000001,"15min":2.8769999999999998},"min":{"1min":0.32700000000000001,"5min":0.29799999999999999,"15min":0.25900000000000001},"max":{"1min":7.8310000000000004,"5min":8.8330000000000002,"15min":154.00399999999999},"last":0.621},{"interface":"front","average":{"1min":1.6970000000000001,"5min":1.6639999999999999,"15min":3.1760000000000002},"min":{"1min":0.35799999999999998,"5min":0.28999999999999998,"15min":0.23899999999999999},"max":{"1min":5.7169999999999996,"5min":7.8070000000000004,"15min":156.87700000000001},"last":2.1299999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":7.516,"5min":2.867,"15min":2.1920000000000002},"min":{"1min":0.33300000000000002,"5min":0.29899999999999999,"15min":0.27400000000000002},"max":{"1min":79.822999999999993,"5min":79.822999999999993,"15min":79.822999999999993},"last":0.53600000000000003},{"interface":"front","average":{"1min":7.5369999999999999,"5min":2.895,"15min":3.1720000000000002},"min":{"1min":0.41099999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":79.783000000000001,"5min":79.783000000000001,"15min":79.783000000000001},"last":0.57299999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:30:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":2.7240000000000002,"15min":3.008},"min":{"1min":0.38900000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.952,"5min":18.361000000000001,"15min":18.361000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.9410000000000001,"5min":2.4489999999999998,"15min":2.9449999999999998},"min":{"1min":0.41999999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.1779999999999999,"5min":18.332000000000001,"15min":18.332000000000001},"last":0.55500000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:31:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":2.8170000000000002,"15min":2.7040000000000002},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.309},"max":{"1min":5.915,"5min":52.006,"15min":79.412999999999997},"last":1.1799999999999999},{"interface":"front","average":{"1min":2.3999999999999999,"5min":2.5529999999999999,"15min":2.6269999999999998},"min":{"1min":0.41699999999999998,"5min":0.34200000000000003,"15min":0.308},"max":{"1min":7.8529999999999998,"5min":51.890999999999998,"15min":79.527000000000001},"last":1.2130000000000001}]}]},{"osd":4,"up_from":0,"seq":128849019260,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":227,"seq":974957576311,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2516872,"kb_used_data":2180992,"kb_used_omap":8,"kb_used_meta":335607,"kb_avail":91228280,"statfs":{"total":95995035648,"available":93417758720,"internally_reserved":0,"allocated":2233335808,"data_stored":2232473424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":343662340},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":12.558,"5min":6.0940000000000003,"15min":3.391},"min":{"1min":0.246,"5min":0.219,"15min":0.219},"max":{"1min":108.437,"5min":169.517,"15min":169.517},"last":104.27200000000001},{"interface":"front","average":{"1min":11.718,"5min":6.4729999999999999,"15min":3.7509999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.219},"max":{"1min":104.69199999999999,"5min":169.69900000000001,"15min":169.69900000000001},"last":104.333}]},{"osd":1,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":22.428000000000001,"5min":5.9000000000000004,"15min":4.1580000000000004},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":122.29000000000001,"5min":122.29000000000001,"15min":122.29000000000001},"last":116.483},{"interface":"front","average":{"1min":22.638999999999999,"5min":5.7699999999999996,"15min":4.5110000000000001},"min":{"1min":0.26700000000000002,"5min":0.251,"15min":0.251},"max":{"1min":122.13500000000001,"5min":122.13500000000001,"15min":122.13500000000001},"last":116.196}]},{"osd":2,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.8400000000000001,"15min":2.4700000000000002},"min":{"1min":0.33500000000000002,"5min":0.27600000000000002,"15min":0.25900000000000001},"max":{"1min":4.2320000000000002,"5min":10.288,"15min":44.615000000000002},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.804,"5min":1.861,"15min":2.0579999999999998},"min":{"1min":0.27900000000000003,"5min":0.26700000000000002,"15min":0.26600000000000001},"max":{"1min":6.3499999999999996,"5min":9.9440000000000008,"15min":44.511000000000003},"last":0.59999999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":6.7690000000000001,"5min":5.548,"15min":4.0599999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":85.387,"5min":214.73599999999999,"15min":214.73599999999999},"last":0.58399999999999996},{"interface":"front","average":{"1min":6.5979999999999999,"5min":5.7610000000000001,"15min":4.1379999999999999},"min":{"1min":0.33300000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":85.346999999999994,"5min":214.86699999999999,"15min":214.86699999999999},"last":0.61499999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":0.61699999999999999,"15min":1.2070000000000001},"min":{"1min":0.31900000000000001,"5min":0.26800000000000002,"15min":0.248},"max":{"1min":3.6989999999999998,"5min":3.6989999999999998,"15min":7.532},"last":0.499},{"interface":"front","average":{"1min":1.3200000000000001,"5min":0.68200000000000005,"15min":1.0700000000000001},"min":{"1min":0.29599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.3200000000000003,"5min":6.3200000000000003,"15min":6.3200000000000003},"last":0.626}]},{"osd":6,"last update":"Sun Oct 1 09:30:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":0.66500000000000004,"15min":0.54100000000000004},"min":{"1min":0.34399999999999997,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.4279999999999999,"5min":6.4279999999999999,"15min":6.4279999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.032,"5min":0.70999999999999996,"15min":0.54900000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.6419999999999999,"5min":4.806,"15min":4.806},"last":0.54100000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":2.3250000000000002,"15min":1.702},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.252},"max":{"1min":6.3769999999999998,"5min":84.105000000000004,"15min":84.105000000000004},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.3540000000000001,"5min":2.3319999999999999,"15min":1.9350000000000001},"min":{"1min":0.39800000000000002,"5min":0.30099999999999999,"15min":0.29599999999999999},"max":{"1min":4.4889999999999999,"5min":83.495000000000005,"15min":83.495000000000005},"last":0.56599999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8444,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1906376704,"data_stored":1906376704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1891631104,"data_stored":1891631104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1955860480,"data_stored":1955856408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1944715264,"data_stored":1944715264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2221346816,"data_stored":2221342744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1655046144,"data_stored":1655046144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1672417280,"data_stored":1672413208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:31:56.461 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-01T09:31:56.461 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:31:56.461 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-01T09:31:56.462 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:31:56.462 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:31:56.462 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-01T09:31:56.610 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:56.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:56.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:56.717 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:31:56.792 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:56.828 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:56.966 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:31:56.967 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:31:56.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:56.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:31:57.076 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:57.157 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:57.286 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:57 smithi138 ceph-mon[185886]: pgmap v994: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 686 op/s 2023-10-01T09:31:57.287 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:57 smithi138 ceph-mon[192513]: pgmap v994: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 686 op/s 2023-10-01T09:31:57.379 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:57 smithi190 ceph-mon[160435]: pgmap v994: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 686 op/s 2023-10-01T09:31:57.426 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:57.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:57.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:57.535 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:31:57.633 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:57.682 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:31:57.682 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:31:57.823 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:31:57.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:57.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:31:57.930 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:31:57.997 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:58.448 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:31:58.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:31:58 smithi190 ceph-mon[160435]: pgmap v995: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T09:31:58.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:31:58 smithi138 ceph-mon[192513]: pgmap v995: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T09:31:58.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:31:58 smithi138 ceph-mon[185886]: pgmap v995: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T09:31:59.124 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:31:59.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:59.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:31:59.171 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:31:59.240 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:59.278 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:59.344 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:59.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:59.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:59.451 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:31:59.484 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:59.558 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:31:59.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:59.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:31:59.661 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:59.692 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:59.753 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:59.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:59.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:31:59.819 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:31:59.856 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:31:59.883 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:31:59.946 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:31:59.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:31:59.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:00.018 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:00.049 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:00.078 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:00.174 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:00.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:00.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:00.281 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:32:00.347 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:00.522 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:32:00.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:00.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:32:00.637 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:00.720 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:00.992 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:32:00.992 INFO:teuthology.orchestra.run.smithi190.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T09:32:00.992 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:32:01.048 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:01 smithi138 ceph-mon[185886]: pgmap v996: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T09:32:01.048 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2974055360' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:01.048 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:01 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:01.082 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:32:01.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:01 smithi190 ceph-mon[160435]: pgmap v996: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T09:32:01.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2974055360' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:01.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:01 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:01.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:01 smithi138 ceph-mon[192513]: pgmap v996: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T09:32:01.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2974055360' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:01.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:01 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:02.131 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:02.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:02.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:02.237 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:02.285 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:02.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:02 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:02.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:02 smithi190 ceph-mon[160435]: failed to encode map e357 with expected crc 2023-10-01T09:32:02.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:02 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:32:02.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:02 smithi190 ceph-mon[160435]: osdmap e357: 8 total, 8 up, 7 in 2023-10-01T09:32:02.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:02 smithi190 ceph-mon[160435]: failed to encode map e357 with expected crc 2023-10-01T09:32:02.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:02 smithi190 ceph-mon[160435]: failed to encode map e357 with expected crc 2023-10-01T09:32:02.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:02 smithi190 ceph-mon[160435]: failed to encode map e357 with expected crc 2023-10-01T09:32:02.312 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:02 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:02.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:02 smithi138 ceph-mon[185886]: failed to encode map e357 with expected crc 2023-10-01T09:32:02.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:02 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:32:02.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:02 smithi138 ceph-mon[185886]: osdmap e357: 8 total, 8 up, 7 in 2023-10-01T09:32:02.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:02 smithi138 ceph-mon[185886]: failed to encode map e357 with expected crc 2023-10-01T09:32:02.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:02 smithi138 ceph-mon[185886]: failed to encode map e357 with expected crc 2023-10-01T09:32:02.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:02 smithi138 ceph-mon[185886]: failed to encode map e357 with expected crc 2023-10-01T09:32:02.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:02 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:02.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:02 smithi138 ceph-mon[192513]: failed to encode map e357 with expected crc 2023-10-01T09:32:02.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:02 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:32:02.313 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:02 smithi138 ceph-mon[192513]: osdmap e357: 8 total, 8 up, 7 in 2023-10-01T09:32:02.314 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:02 smithi138 ceph-mon[192513]: failed to encode map e357 with expected crc 2023-10-01T09:32:02.314 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:02 smithi138 ceph-mon[192513]: failed to encode map e357 with expected crc 2023-10-01T09:32:02.314 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:02 smithi138 ceph-mon[192513]: failed to encode map e357 with expected crc 2023-10-01T09:32:02.345 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:02.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:02.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:02.462 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:02.488 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:02.530 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:02.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:02.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:02.636 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:02.738 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:02.925 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:02.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:02.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:03.048 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:03.093 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:03.150 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:03.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:03.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:03.256 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:03.260 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:03 smithi138 ceph-mon[185886]: pgmap v998: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 691 op/s 2023-10-01T09:32:03.260 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:03 smithi138 ceph-mon[192513]: pgmap v998: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 691 op/s 2023-10-01T09:32:03.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:03 smithi190 ceph-mon[160435]: pgmap v998: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 691 op/s 2023-10-01T09:32:03.340 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:03.548 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:03.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:03.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:03.627 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:32:03.665 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:03.676 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:32:03.706 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:03.718 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:32:03.758 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:32:03.768 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:03.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:03.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:03.793 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:32:03.825 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:32:03.825 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:32:03.825 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:32:03.825 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:32:03.826 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:32:03.840 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:32:03.872 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:03.880 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:32:03.920 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:32:03.926 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:32:03.926 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:32:03.926 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:32:03.926 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:32:03.926 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:32:03.926 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:32:03.926 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:32:03.926 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:32:03.958 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:04.311 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:32:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2831600950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:04.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:04 smithi138 ceph-mon[192513]: pgmap v999: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-01T09:32:04.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2831600950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:04.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:04 smithi138 ceph-mon[185886]: pgmap v999: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-01T09:32:04.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2831600950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:04.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:04 smithi190 ceph-mon[160435]: pgmap v999: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-01T09:32:05.028 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:05.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:05.038 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:32:05.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:05.140 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:05.241 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:32:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:32:04] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.33.4" 2023-10-01T09:32:05.248 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:05.629 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:05.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:05.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:05.746 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:05.791 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:05.851 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:05.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:05.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:05.884 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:05.957 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:05.974 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:06.067 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:06.447 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:06.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:06.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:06.554 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:06.599 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:06.665 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:06.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:06.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:06.772 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:06.805 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5, 6] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2023-10-01T09:32:06.805 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:32:06.805 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-01T09:32:06.805 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:32:06.805 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:32:06.806 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-01T09:32:06.806 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:06.869 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:06.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:06.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:06.977 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:07.021 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:07.054 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:07 smithi138 ceph-mon[185886]: pgmap v1000: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 694 op/s 2023-10-01T09:32:07.097 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:07.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:07.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:07.205 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:07.243 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:07.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:07 smithi190 ceph-mon[160435]: pgmap v1000: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 694 op/s 2023-10-01T09:32:07.328 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:07 smithi138 ceph-mon[192513]: pgmap v1000: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 694 op/s 2023-10-01T09:32:07.645 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:32:07.850 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:07.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:07.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:07.973 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:08.007 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:08.087 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:08.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:08.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:08.192 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:08.281 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:08.448 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:08 smithi138 ceph-mon[192513]: pgmap v1001: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 693 op/s 2023-10-01T09:32:08.449 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:08.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:08.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:08.571 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:08.683 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:08.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:08 smithi190 ceph-mon[160435]: pgmap v1001: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 693 op/s 2023-10-01T09:32:08.827 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:08 smithi138 ceph-mon[185886]: pgmap v1001: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 693 op/s 2023-10-01T09:32:08.845 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:08.849 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:08.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:08.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:08.970 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:09.003 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:09.055 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:09.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:09.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:09.166 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:09.196 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:09.254 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:09.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:09.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:09.359 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:09.451 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:09.539 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:32:09.539 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:32:09.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:32:09.539 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:32:09.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.539 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178787 6.18 6:189b3dbf:::benchmark_data_smithi138_307751_object83418:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:32:09.539 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612642+0000", 2023-10-01T09:32:09.539 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 538.92271433200006, 2023-10-01T09:32:09.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4263684350000001, 2023-10-01T09:32:09.540 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.540 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:09.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:09.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:09.540 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178787 2023-10-01T09:32:09.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.540 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612642+0000", 2023-10-01T09:32:09.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612642+0000", 2023-10-01T09:32:09.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612641+0000", 2023-10-01T09:32:09.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:32:09.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612647+0000", 2023-10-01T09:32:09.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5709999999999997e-06 2023-10-01T09:32:09.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612658+0000", 2023-10-01T09:32:09.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1528e-05 2023-10-01T09:32:09.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612668+0000", 2023-10-01T09:32:09.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0349999999999999e-05 2023-10-01T09:32:09.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614052+0000", 2023-10-01T09:32:09.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001383188 2023-10-01T09:32:09.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:09.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614062+0000", 2023-10-01T09:32:09.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0437e-05 2023-10-01T09:32:09.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.028763+0000", 2023-10-01T09:32:09.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.414701177 2023-10-01T09:32:09.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.028872+0000", 2023-10-01T09:32:09.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109073 2023-10-01T09:32:09.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:32:09.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.029307+0000", 2023-10-01T09:32:09.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00043503699999999999 2023-10-01T09:32:09.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:09.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.032478+0000", 2023-10-01T09:32:09.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031704839999999999 2023-10-01T09:32:09.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:09.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.032534+0000", 2023-10-01T09:32:09.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6481999999999999e-05 2023-10-01T09:32:09.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:09.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.038405+0000", 2023-10-01T09:32:09.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058706319999999998 2023-10-01T09:32:09.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.038465+0000", 2023-10-01T09:32:09.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0176000000000001e-05 2023-10-01T09:32:09.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039011+0000", 2023-10-01T09:32:09.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00054578000000000005 2023-10-01T09:32:09.547 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.547 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.547 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.547 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178807 6.18 6:1f8d531c:::benchmark_data_smithi138_307751_object83438:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:32:09.548 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612681+0000", 2023-10-01T09:32:09.548 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 538.92267625299996, 2023-10-01T09:32:09.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4300119259999999, 2023-10-01T09:32:09.548 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.548 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.548 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:09.548 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:09.548 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:09.548 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178807 2023-10-01T09:32:09.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.548 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612681+0000", 2023-10-01T09:32:09.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612681+0000", 2023-10-01T09:32:09.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612680+0000", 2023-10-01T09:32:09.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:09.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612683+0000", 2023-10-01T09:32:09.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0709999999999999e-06 2023-10-01T09:32:09.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612690+0000", 2023-10-01T09:32:09.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.613e-06 2023-10-01T09:32:09.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612695+0000", 2023-10-01T09:32:09.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4090000000000002e-06 2023-10-01T09:32:09.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614084+0000", 2023-10-01T09:32:09.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001388331 2023-10-01T09:32:09.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:09.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614090+0000", 2023-10-01T09:32:09.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8630000000000002e-06 2023-10-01T09:32:09.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.032562+0000", 2023-10-01T09:32:09.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.41847178 2023-10-01T09:32:09.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.032737+0000", 2023-10-01T09:32:09.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000174477 2023-10-01T09:32:09.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:32:09.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.033304+0000", 2023-10-01T09:32:09.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00056714600000000003 2023-10-01T09:32:09.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:09.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039063+0000", 2023-10-01T09:32:09.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057587920000000004 2023-10-01T09:32:09.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:09.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039071+0000", 2023-10-01T09:32:09.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5920000000000001e-06 2023-10-01T09:32:09.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:09.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042139+0000", 2023-10-01T09:32:09.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030678440000000001 2023-10-01T09:32:09.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042194+0000", 2023-10-01T09:32:09.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5306999999999997e-05 2023-10-01T09:32:09.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042693+0000", 2023-10-01T09:32:09.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00049818099999999999 2023-10-01T09:32:09.555 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.555 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.556 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.556 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178818 6.18 6:19fb958b:::benchmark_data_smithi138_307751_object83449:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:32:09.556 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612700+0000", 2023-10-01T09:32:09.556 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 538.92265660199996, 2023-10-01T09:32:09.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.430431437, 2023-10-01T09:32:09.556 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.556 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.556 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:09.556 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:09.557 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:09.557 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178818 2023-10-01T09:32:09.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.557 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612700+0000", 2023-10-01T09:32:09.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.558 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612700+0000", 2023-10-01T09:32:09.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.558 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.558 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612700+0000", 2023-10-01T09:32:09.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:09.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.559 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612703+0000", 2023-10-01T09:32:09.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7920000000000001e-06 2023-10-01T09:32:09.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.559 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612708+0000", 2023-10-01T09:32:09.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4990000000000002e-06 2023-10-01T09:32:09.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.559 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612713+0000", 2023-10-01T09:32:09.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8509999999999998e-06 2023-10-01T09:32:09.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614110+0000", 2023-10-01T09:32:09.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0013967039999999999 2023-10-01T09:32:09.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:09.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614116+0000", 2023-10-01T09:32:09.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9349999999999999e-06 2023-10-01T09:32:09.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.035619+0000", 2023-10-01T09:32:09.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.421503172 2023-10-01T09:32:09.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.035767+0000", 2023-10-01T09:32:09.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014777900000000001 2023-10-01T09:32:09.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:32:09.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.036191+0000", 2023-10-01T09:32:09.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000424206 2023-10-01T09:32:09.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:09.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042058+0000", 2023-10-01T09:32:09.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005867227 2023-10-01T09:32:09.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:09.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042070+0000", 2023-10-01T09:32:09.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2122e-05 2023-10-01T09:32:09.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:09.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042747+0000", 2023-10-01T09:32:09.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00067660799999999998 2023-10-01T09:32:09.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.042782+0000", 2023-10-01T09:32:09.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5441000000000002e-05 2023-10-01T09:32:09.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043132+0000", 2023-10-01T09:32:09.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00034945799999999997 2023-10-01T09:32:09.564 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.564 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.564 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.565 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178848 6.18 6:1e688e02:::benchmark_data_smithi138_307751_object83479:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:32:09.565 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612851+0000", 2023-10-01T09:32:09.565 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 538.92250605000004, 2023-10-01T09:32:09.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.436053759, 2023-10-01T09:32:09.565 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.565 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.565 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:09.565 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:09.565 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:09.565 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178848 2023-10-01T09:32:09.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.566 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612851+0000", 2023-10-01T09:32:09.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612851+0000", 2023-10-01T09:32:09.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612850+0000", 2023-10-01T09:32:09.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:09.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612854+0000", 2023-10-01T09:32:09.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5580000000000001e-06 2023-10-01T09:32:09.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612862+0000", 2023-10-01T09:32:09.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9699999999999999e-06 2023-10-01T09:32:09.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612869+0000", 2023-10-01T09:32:09.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0539999999999997e-06 2023-10-01T09:32:09.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614135+0000", 2023-10-01T09:32:09.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012659559999999999 2023-10-01T09:32:09.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:09.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614141+0000", 2023-10-01T09:32:09.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2859999999999996e-06 2023-10-01T09:32:09.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039094+0000", 2023-10-01T09:32:09.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.424953428 2023-10-01T09:32:09.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039262+0000", 2023-10-01T09:32:09.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000167541 2023-10-01T09:32:09.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:32:09.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.039703+0000", 2023-10-01T09:32:09.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000441462 2023-10-01T09:32:09.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:09.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043192+0000", 2023-10-01T09:32:09.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034885319999999999 2023-10-01T09:32:09.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:09.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043218+0000", 2023-10-01T09:32:09.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5754000000000001e-05 2023-10-01T09:32:09.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:09.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.048340+0000", 2023-10-01T09:32:09.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051218219999999998 2023-10-01T09:32:09.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.048383+0000", 2023-10-01T09:32:09.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3179000000000001e-05 2023-10-01T09:32:09.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.048905+0000", 2023-10-01T09:32:09.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00052180900000000003 2023-10-01T09:32:09.572 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.573 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.573 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.573 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178861 6.18 6:1fa58ed3:::benchmark_data_smithi138_307751_object83492:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e247)", 2023-10-01T09:32:09.573 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:10.612874+0000", 2023-10-01T09:32:09.573 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 538.92248293499995, 2023-10-01T09:32:09.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4449525510000001, 2023-10-01T09:32:09.573 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.573 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.573 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:09.573 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:09.574 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:09.574 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178861 2023-10-01T09:32:09.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.574 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612874+0000", 2023-10-01T09:32:09.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612874+0000", 2023-10-01T09:32:09.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612874+0000", 2023-10-01T09:32:09.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:09.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612876+0000", 2023-10-01T09:32:09.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4380000000000002e-06 2023-10-01T09:32:09.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612882+0000", 2023-10-01T09:32:09.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1179999999999997e-06 2023-10-01T09:32:09.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.612887+0000", 2023-10-01T09:32:09.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0300000000000001e-06 2023-10-01T09:32:09.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614161+0000", 2023-10-01T09:32:09.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0012741460000000001 2023-10-01T09:32:09.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:09.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.614167+0000", 2023-10-01T09:32:09.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6629999999999998e-06 2023-10-01T09:32:09.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043249+0000", 2023-10-01T09:32:09.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.429082395 2023-10-01T09:32:09.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043418+0000", 2023-10-01T09:32:09.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016889300000000001 2023-10-01T09:32:09.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:32:09.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.043866+0000", 2023-10-01T09:32:09.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00044823600000000002 2023-10-01T09:32:09.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:09.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057128+0000", 2023-10-01T09:32:09.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013261854 2023-10-01T09:32:09.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:09.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057142+0000", 2023-10-01T09:32:09.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3743e-05 2023-10-01T09:32:09.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:09.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057244+0000", 2023-10-01T09:32:09.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010159599999999999 2023-10-01T09:32:09.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057298+0000", 2023-10-01T09:32:09.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.41e-05 2023-10-01T09:32:09.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:12.057826+0000", 2023-10-01T09:32:09.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00052871700000000005 2023-10-01T09:32:09.581 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.581 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.581 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.581 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:121701 7.8 7:123e4a57:::benchmark_data_smithi138_367058_object23906:head [delete] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-01T09:32:09.581 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:28.301494+0000", 2023-10-01T09:32:09.581 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.23386274900002, 2023-10-01T09:32:09.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0627196029999999, 2023-10-01T09:32:09.582 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.582 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:32:09.582 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:09.582 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:32:09.582 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:32:09.582 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121701 2023-10-01T09:32:09.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.582 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301494+0000", 2023-10-01T09:32:09.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301494+0000", 2023-10-01T09:32:09.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301493+0000", 2023-10-01T09:32:09.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:32:09.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301497+0000", 2023-10-01T09:32:09.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7670000000000003e-06 2023-10-01T09:32:09.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301506+0000", 2023-10-01T09:32:09.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3489999999999992e-06 2023-10-01T09:32:09.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301512+0000", 2023-10-01T09:32:09.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0290000000000002e-06 2023-10-01T09:32:09.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301562+0000", 2023-10-01T09:32:09.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0515e-05 2023-10-01T09:32:09.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301721+0000", 2023-10-01T09:32:09.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015924799999999999 2023-10-01T09:32:09.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:32:09.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301875+0000", 2023-10-01T09:32:09.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000154046 2023-10-01T09:32:09.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:09.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.310228+0000", 2023-10-01T09:32:09.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0083525619999999991 2023-10-01T09:32:09.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:09.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.318729+0000", 2023-10-01T09:32:09.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085004699999999996 2023-10-01T09:32:09.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.416615+0000", 2023-10-01T09:32:09.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.097886033999999997 2023-10-01T09:32:09.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:09.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.416630+0000", 2023-10-01T09:32:09.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5301000000000001e-05 2023-10-01T09:32:09.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:29.364189+0000", 2023-10-01T09:32:09.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94755891999999997 2023-10-01T09:32:09.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:29.364214+0000", 2023-10-01T09:32:09.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4902999999999999e-05 2023-10-01T09:32:09.588 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.589 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.589 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.589 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:129237 8.b e355/343 8:dab60112:::benchmark_data_smithi138_431617_object27826:head v 355'7976, mlcod=355'7974)", 2023-10-01T09:32:09.589 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:42.779899+0000", 2023-10-01T09:32:09.589 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 26.755457822, 2023-10-01T09:32:09.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.085791627999999995, 2023-10-01T09:32:09.589 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.589 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.589 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.779899+0000", 2023-10-01T09:32:09.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.779899+0000", 2023-10-01T09:32:09.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.779898+0000", 2023-10-01T09:32:09.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:32:09.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.779906+0000", 2023-10-01T09:32:09.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7300000000000005e-06 2023-10-01T09:32:09.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.779918+0000", 2023-10-01T09:32:09.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1888e-05 2023-10-01T09:32:09.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.779925+0000", 2023-10-01T09:32:09.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4279999999999999e-06 2023-10-01T09:32:09.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.779987+0000", 2023-10-01T09:32:09.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1978999999999997e-05 2023-10-01T09:32:09.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.780008+0000", 2023-10-01T09:32:09.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0840999999999999e-05 2023-10-01T09:32:09.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.784173+0000", 2023-10-01T09:32:09.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041654429999999996 2023-10-01T09:32:09.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.865691+0000", 2023-10-01T09:32:09.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.081517210000000007 2023-10-01T09:32:09.594 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.594 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.594 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.594 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:129250 8.a e355/326 8:5c5bc3fc:::benchmark_data_smithi138_431617_object27839:head v 355'8152, mlcod=355'8150)", 2023-10-01T09:32:09.594 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:42.787650+0000", 2023-10-01T09:32:09.594 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 26.747706395000002, 2023-10-01T09:32:09.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.087764171000000002, 2023-10-01T09:32:09.595 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.595 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.595 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.787650+0000", 2023-10-01T09:32:09.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.787650+0000", 2023-10-01T09:32:09.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.787649+0000", 2023-10-01T09:32:09.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:32:09.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.787659+0000", 2023-10-01T09:32:09.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6630000000000005e-06 2023-10-01T09:32:09.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.865599+0000", 2023-10-01T09:32:09.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.077939513000000002 2023-10-01T09:32:09.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.865625+0000", 2023-10-01T09:32:09.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.658e-05 2023-10-01T09:32:09.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.865703+0000", 2023-10-01T09:32:09.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7577000000000006e-05 2023-10-01T09:32:09.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.865726+0000", 2023-10-01T09:32:09.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3232999999999999e-05 2023-10-01T09:32:09.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.875378+0000", 2023-10-01T09:32:09.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0096521880000000008 2023-10-01T09:32:09.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:42.875415+0000", 2023-10-01T09:32:09.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6359999999999997e-05 2023-10-01T09:32:09.599 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.599 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.600 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.600 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:130577 8.4 e355/326 8:26913b77:::benchmark_data_smithi138_431617_object29166:head v 355'8071, mlcod=355'8068)", 2023-10-01T09:32:09.600 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:44.755039+0000", 2023-10-01T09:32:09.600 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 24.780317570000001, 2023-10-01T09:32:09.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.039778108999999999, 2023-10-01T09:32:09.600 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.600 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.600 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.755039+0000", 2023-10-01T09:32:09.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.755039+0000", 2023-10-01T09:32:09.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.755039+0000", 2023-10-01T09:32:09.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:09.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.755043+0000", 2023-10-01T09:32:09.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4220000000000002e-06 2023-10-01T09:32:09.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.755057+0000", 2023-10-01T09:32:09.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3884e-05 2023-10-01T09:32:09.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.755062+0000", 2023-10-01T09:32:09.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4610000000000001e-06 2023-10-01T09:32:09.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.762574+0000", 2023-10-01T09:32:09.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0075126769999999997 2023-10-01T09:32:09.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.762599+0000", 2023-10-01T09:32:09.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4266000000000001e-05 2023-10-01T09:32:09.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.794772+0000", 2023-10-01T09:32:09.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.032173871 2023-10-01T09:32:09.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:44.794817+0000", 2023-10-01T09:32:09.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4935999999999998e-05 2023-10-01T09:32:09.605 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.605 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.605 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.605 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:132632 8.b e355/343 8:d20a922b:::benchmark_data_smithi138_431617_object31221:head v 355'8187, mlcod=355'8182)", 2023-10-01T09:32:09.605 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:47.728965+0000", 2023-10-01T09:32:09.605 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 21.806391850000001, 2023-10-01T09:32:09.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.038479891000000002, 2023-10-01T09:32:09.605 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.605 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.606 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:47.728965+0000", 2023-10-01T09:32:09.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:47.728965+0000", 2023-10-01T09:32:09.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:47.728964+0000", 2023-10-01T09:32:09.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:09.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:47.728972+0000", 2023-10-01T09:32:09.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5970000000000003e-06 2023-10-01T09:32:09.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:47.728983+0000", 2023-10-01T09:32:09.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1419e-05 2023-10-01T09:32:09.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:47.728991+0000", 2023-10-01T09:32:09.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1359999999999999e-06 2023-10-01T09:32:09.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:47.735562+0000", 2023-10-01T09:32:09.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0065719799999999998 2023-10-01T09:32:09.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:47.735580+0000", 2023-10-01T09:32:09.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7595999999999999e-05 2023-10-01T09:32:09.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:47.767415+0000", 2023-10-01T09:32:09.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.031834751000000001 2023-10-01T09:32:09.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:47.767445+0000", 2023-10-01T09:32:09.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0043999999999999e-05 2023-10-01T09:32:09.610 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.610 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.610 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.610 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:133590 8.4 e355/326 8:28c6850c:::benchmark_data_smithi138_431617_object32179:head v 355'8269, mlcod=355'8265)", 2023-10-01T09:32:09.610 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:49.199242+0000", 2023-10-01T09:32:09.610 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 20.336114928000001, 2023-10-01T09:32:09.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.038397657000000002, 2023-10-01T09:32:09.611 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.611 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.611 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.199242+0000", 2023-10-01T09:32:09.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.199242+0000", 2023-10-01T09:32:09.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.199241+0000", 2023-10-01T09:32:09.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:32:09.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.199253+0000", 2023-10-01T09:32:09.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1856e-05 2023-10-01T09:32:09.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.199266+0000", 2023-10-01T09:32:09.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3653e-05 2023-10-01T09:32:09.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.199275+0000", 2023-10-01T09:32:09.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1999999999999994e-06 2023-10-01T09:32:09.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.218601+0000", 2023-10-01T09:32:09.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019326283 2023-10-01T09:32:09.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.218622+0000", 2023-10-01T09:32:09.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1212999999999998e-05 2023-10-01T09:32:09.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.237589+0000", 2023-10-01T09:32:09.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018967047000000001 2023-10-01T09:32:09.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.237640+0000", 2023-10-01T09:32:09.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0442999999999998e-05 2023-10-01T09:32:09.615 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.615 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.615 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.616 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:133657 8.b e355/343 8:daf615b4:::benchmark_data_smithi138_431617_object32246:head v 355'8262, mlcod=355'8257)", 2023-10-01T09:32:09.616 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:49.335119+0000", 2023-10-01T09:32:09.616 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 20.200237585, 2023-10-01T09:32:09.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.046295430999999998, 2023-10-01T09:32:09.616 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.616 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.616 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.335119+0000", 2023-10-01T09:32:09.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.335119+0000", 2023-10-01T09:32:09.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.335119+0000", 2023-10-01T09:32:09.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967296 2023-10-01T09:32:09.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.335122+0000", 2023-10-01T09:32:09.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1319999999999998e-06 2023-10-01T09:32:09.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.335127+0000", 2023-10-01T09:32:09.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.549e-06 2023-10-01T09:32:09.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.335131+0000", 2023-10-01T09:32:09.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2969999999999997e-06 2023-10-01T09:32:09.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.354828+0000", 2023-10-01T09:32:09.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019697065999999999 2023-10-01T09:32:09.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.354855+0000", 2023-10-01T09:32:09.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6553000000000001e-05 2023-10-01T09:32:09.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.381374+0000", 2023-10-01T09:32:09.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.026519504999999999 2023-10-01T09:32:09.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.381415+0000", 2023-10-01T09:32:09.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0459e-05 2023-10-01T09:32:09.620 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.621 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.621 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.621 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:133807 8.e e355/326 8:7daebeec:::benchmark_data_smithi138_431617_object32396:head v 355'8531, mlcod=355'8529)", 2023-10-01T09:32:09.621 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:49.666020+0000", 2023-10-01T09:32:09.621 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 19.869336443000002, 2023-10-01T09:32:09.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.045742272, 2023-10-01T09:32:09.621 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.621 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.621 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666020+0000", 2023-10-01T09:32:09.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666020+0000", 2023-10-01T09:32:09.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666019+0000", 2023-10-01T09:32:09.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:32:09.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666031+0000", 2023-10-01T09:32:09.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2126e-05 2023-10-01T09:32:09.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666055+0000", 2023-10-01T09:32:09.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3600000000000001e-05 2023-10-01T09:32:09.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666065+0000", 2023-10-01T09:32:09.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6989999999999998e-06 2023-10-01T09:32:09.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666260+0000", 2023-10-01T09:32:09.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000194878 2023-10-01T09:32:09.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666295+0000", 2023-10-01T09:32:09.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5735999999999998e-05 2023-10-01T09:32:09.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.711727+0000", 2023-10-01T09:32:09.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.045431307999999997 2023-10-01T09:32:09.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.711763+0000", 2023-10-01T09:32:09.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6078999999999998e-05 2023-10-01T09:32:09.628 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.628 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.628 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.629 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:133809 8.e e355/326 8:7f8b63ab:::benchmark_data_smithi138_431617_object32398:head v 355'8532, mlcod=355'8529)", 2023-10-01T09:32:09.629 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:49.666082+0000", 2023-10-01T09:32:09.629 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 19.869275178999999, 2023-10-01T09:32:09.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.045700686999999997, 2023-10-01T09:32:09.629 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.629 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.629 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666082+0000", 2023-10-01T09:32:09.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666082+0000", 2023-10-01T09:32:09.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666081+0000", 2023-10-01T09:32:09.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:09.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666087+0000", 2023-10-01T09:32:09.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8259999999999998e-06 2023-10-01T09:32:09.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666094+0000", 2023-10-01T09:32:09.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5289999999999997e-06 2023-10-01T09:32:09.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666098+0000", 2023-10-01T09:32:09.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.702e-06 2023-10-01T09:32:09.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.670230+0000", 2023-10-01T09:32:09.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041318309999999999 2023-10-01T09:32:09.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.670251+0000", 2023-10-01T09:32:09.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0554999999999999e-05 2023-10-01T09:32:09.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.711765+0000", 2023-10-01T09:32:09.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.041514612999999999 2023-10-01T09:32:09.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.711782+0000", 2023-10-01T09:32:09.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.696e-05 2023-10-01T09:32:09.635 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.635 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.635 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.635 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:133818 8.e e355/326 8:7be3addc:::benchmark_data_smithi138_431617_object32407:head v 355'8533, mlcod=355'8529)", 2023-10-01T09:32:09.635 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:31:49.666103+0000", 2023-10-01T09:32:09.636 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 19.869254201, 2023-10-01T09:32:09.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.045700400000000002, 2023-10-01T09:32:09.636 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.636 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.636 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666103+0000", 2023-10-01T09:32:09.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666103+0000", 2023-10-01T09:32:09.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666102+0000", 2023-10-01T09:32:09.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967296 2023-10-01T09:32:09.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666111+0000", 2023-10-01T09:32:09.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4910000000000005e-06 2023-10-01T09:32:09.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666116+0000", 2023-10-01T09:32:09.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4179999999999996e-06 2023-10-01T09:32:09.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.666121+0000", 2023-10-01T09:32:09.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3710000000000002e-06 2023-10-01T09:32:09.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.704502+0000", 2023-10-01T09:32:09.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.038381707000000001 2023-10-01T09:32:09.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.704527+0000", 2023-10-01T09:32:09.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4445000000000001e-05 2023-10-01T09:32:09.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.711784+0000", 2023-10-01T09:32:09.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0072573409999999996 2023-10-01T09:32:09.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:31:49.711803+0000", 2023-10-01T09:32:09.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8859999999999999e-05 2023-10-01T09:32:09.641 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.641 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.641 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.642 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:146254 8.b e357/343 8:d46214da:::benchmark_data_smithi138_431617_object44843:head v 357'9083, mlcod=357'9080)", 2023-10-01T09:32:09.642 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:07.359577+0000", 2023-10-01T09:32:09.642 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.175779737, 2023-10-01T09:32:09.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.059411651000000003, 2023-10-01T09:32:09.642 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.642 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.642 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359577+0000", 2023-10-01T09:32:09.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359577+0000", 2023-10-01T09:32:09.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359576+0000", 2023-10-01T09:32:09.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:32:09.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359583+0000", 2023-10-01T09:32:09.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8630000000000002e-06 2023-10-01T09:32:09.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359594+0000", 2023-10-01T09:32:09.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0817e-05 2023-10-01T09:32:09.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359601+0000", 2023-10-01T09:32:09.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4850000000000003e-06 2023-10-01T09:32:09.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359638+0000", 2023-10-01T09:32:09.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6755999999999998e-05 2023-10-01T09:32:09.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359652+0000", 2023-10-01T09:32:09.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4059999999999999e-05 2023-10-01T09:32:09.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.418960+0000", 2023-10-01T09:32:09.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.059308147999999998 2023-10-01T09:32:09.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.418989+0000", 2023-10-01T09:32:09.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8374999999999999e-05 2023-10-01T09:32:09.648 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.648 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.648 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.648 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:146264 8.b e357/343 8:d2218005:::benchmark_data_smithi138_431617_object44853:head v 357'9084, mlcod=357'9080)", 2023-10-01T09:32:09.648 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:07.359616+0000", 2023-10-01T09:32:09.649 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.1757405059999999, 2023-10-01T09:32:09.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.059391287000000001, 2023-10-01T09:32:09.649 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.649 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.649 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359616+0000", 2023-10-01T09:32:09.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359616+0000", 2023-10-01T09:32:09.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359616+0000", 2023-10-01T09:32:09.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:09.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359620+0000", 2023-10-01T09:32:09.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6279999999999998e-06 2023-10-01T09:32:09.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359625+0000", 2023-10-01T09:32:09.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1420000000000001e-06 2023-10-01T09:32:09.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359629+0000", 2023-10-01T09:32:09.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4800000000000003e-06 2023-10-01T09:32:09.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.406767+0000", 2023-10-01T09:32:09.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.047138148999999997 2023-10-01T09:32:09.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.406789+0000", 2023-10-01T09:32:09.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.192e-05 2023-10-01T09:32:09.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.418991+0000", 2023-10-01T09:32:09.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012201694000000001 2023-10-01T09:32:09.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.419008+0000", 2023-10-01T09:32:09.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6532e-05 2023-10-01T09:32:09.655 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.655 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.655 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.655 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:146266 8.b e357/343 8:d5083670:::benchmark_data_smithi138_431617_object44855:head v 357'9085, mlcod=357'9080)", 2023-10-01T09:32:09.655 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:07.359634+0000", 2023-10-01T09:32:09.655 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.1757230339999998, 2023-10-01T09:32:09.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.059390871999999997, 2023-10-01T09:32:09.655 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.656 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.656 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359634+0000", 2023-10-01T09:32:09.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359634+0000", 2023-10-01T09:32:09.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359634+0000", 2023-10-01T09:32:09.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967296 2023-10-01T09:32:09.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359639+0000", 2023-10-01T09:32:09.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3619999999999997e-06 2023-10-01T09:32:09.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359644+0000", 2023-10-01T09:32:09.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2499999999999997e-06 2023-10-01T09:32:09.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.359648+0000", 2023-10-01T09:32:09.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.681e-06 2023-10-01T09:32:09.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.409756+0000", 2023-10-01T09:32:09.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.050108236 2023-10-01T09:32:09.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.409772+0000", 2023-10-01T09:32:09.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6279000000000001e-05 2023-10-01T09:32:09.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.419009+0000", 2023-10-01T09:32:09.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0092367490000000007 2023-10-01T09:32:09.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.419025+0000", 2023-10-01T09:32:09.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5468000000000002e-05 2023-10-01T09:32:09.661 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.661 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.661 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.661 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:146268 8.1 e357/326 8:80542ac7:::benchmark_data_smithi138_431617_object44857:head v 357'9180, mlcod=357'9178)", 2023-10-01T09:32:09.661 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:07.362481+0000", 2023-10-01T09:32:09.662 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 2.1728763230000001, 2023-10-01T09:32:09.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.048481512999999997, 2023-10-01T09:32:09.662 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.662 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.662 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.362481+0000", 2023-10-01T09:32:09.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.362481+0000", 2023-10-01T09:32:09.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.362480+0000", 2023-10-01T09:32:09.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:32:09.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.362487+0000", 2023-10-01T09:32:09.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9380000000000003e-06 2023-10-01T09:32:09.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.399710+0000", 2023-10-01T09:32:09.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.037223203000000003 2023-10-01T09:32:09.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.399734+0000", 2023-10-01T09:32:09.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.385e-05 2023-10-01T09:32:09.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.399846+0000", 2023-10-01T09:32:09.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111908 2023-10-01T09:32:09.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.399902+0000", 2023-10-01T09:32:09.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5858e-05 2023-10-01T09:32:09.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.410920+0000", 2023-10-01T09:32:09.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011018956999999999 2023-10-01T09:32:09.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:07.410962+0000", 2023-10-01T09:32:09.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1548000000000003e-05 2023-10-01T09:32:09.668 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.668 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.668 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.668 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:147744 8.4 e357/326 8:220cb7bd:::benchmark_data_smithi138_431617_object46333:head v 357'9109, mlcod=357'9106)", 2023-10-01T09:32:09.668 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:09.446658+0000", 2023-10-01T09:32:09.669 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.088698610999999997, 2023-10-01T09:32:09.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.042467655999999999, 2023-10-01T09:32:09.669 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:09.669 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:09.669 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:09.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:09.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:09.446658+0000", 2023-10-01T09:32:09.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:09.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:09.446658+0000", 2023-10-01T09:32:09.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:09.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:09.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:09.446657+0000", 2023-10-01T09:32:09.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:32:09.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:09.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:09.446668+0000", 2023-10-01T09:32:09.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0895e-05 2023-10-01T09:32:09.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:09.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:09.446681+0000", 2023-10-01T09:32:09.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3175e-05 2023-10-01T09:32:09.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:09.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:09.446689+0000", 2023-10-01T09:32:09.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7260000000000002e-06 2023-10-01T09:32:09.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:09.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:09.473635+0000", 2023-10-01T09:32:09.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.026946327999999999 2023-10-01T09:32:09.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:09.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:09.473657+0000", 2023-10-01T09:32:09.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1999000000000001e-05 2023-10-01T09:32:09.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:09.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:09.489094+0000", 2023-10-01T09:32:09.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015436675 2023-10-01T09:32:09.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:09.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:09.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:09.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:09.489126+0000", 2023-10-01T09:32:09.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1856000000000001e-05 2023-10-01T09:32:09.675 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.675 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.675 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.675 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:09.675 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:09.675 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:32:09.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:09.677 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:09.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:09.677 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:32:09.778 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:09.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:09 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2771977659' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:32:09.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:09 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:32:09.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:09 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2771977659' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:32:09.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:09 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:32:09.876 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:10.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:09 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2771977659' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:32:10.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:09 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:32:10.738 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:10.858 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:10 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:10.858 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:10 smithi138 ceph-mon[192513]: failed to encode map e358 with expected crc 2023-10-01T09:32:10.858 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:10 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:32:10.858 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:10 smithi138 ceph-mon[192513]: osdmap e358: 8 total, 8 up, 7 in 2023-10-01T09:32:10.858 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:10 smithi138 ceph-mon[192513]: failed to encode map e358 with expected crc 2023-10-01T09:32:10.858 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:10 smithi138 ceph-mon[192513]: pgmap v1003: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-01T09:32:10.858 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:32:10.859 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:10 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:10.859 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:10 smithi138 ceph-mon[185886]: failed to encode map e358 with expected crc 2023-10-01T09:32:10.859 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:10 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:32:10.859 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:10 smithi138 ceph-mon[185886]: osdmap e358: 8 total, 8 up, 7 in 2023-10-01T09:32:10.859 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:10 smithi138 ceph-mon[185886]: failed to encode map e358 with expected crc 2023-10-01T09:32:10.859 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:10 smithi138 ceph-mon[185886]: pgmap v1003: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-01T09:32:10.859 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:32:11.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:10 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:11.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:10 smithi190 ceph-mon[160435]: failed to encode map e358 with expected crc 2023-10-01T09:32:11.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:10 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:32:11.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:10 smithi190 ceph-mon[160435]: osdmap e358: 8 total, 8 up, 7 in 2023-10-01T09:32:11.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:10 smithi190 ceph-mon[160435]: failed to encode map e358 with expected crc 2023-10-01T09:32:11.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:10 smithi190 ceph-mon[160435]: pgmap v1003: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 671 op/s 2023-10-01T09:32:11.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:32:11.595 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:11.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:11.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:11.643 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:32:11.704 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:11.756 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:11.797 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:11.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:11.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:11.905 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:11.942 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:12.357 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:12.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:12.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:12.469 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:12.503 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:12.588 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:12.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:12.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:12.695 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:32:12.804 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:12.873 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:32:12.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:12.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:32:12.977 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:13.004 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:13.046 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:13.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:13.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:13.149 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:13.181 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:13.244 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:13.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:13.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:13.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:13 smithi190 ceph-mon[160435]: pgmap v1004: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 732 op/s 2023-10-01T09:32:13.307 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:32:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:13 smithi138 ceph-mon[192513]: pgmap v1004: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 732 op/s 2023-10-01T09:32:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:13 smithi138 ceph-mon[185886]: pgmap v1004: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 732 op/s 2023-10-01T09:32:13.356 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:13.393 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:13.466 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:13.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:13.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:13.484 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:13.573 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:13.696 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:13.791 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:13.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:13.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:13.905 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:14.018 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:14.101 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:32:14.102 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "" 2023-10-01T09:32:14.102 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:32:14.185 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:14 smithi138 ceph-mon[192513]: pgmap v1005: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-01T09:32:14.185 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:14 smithi138 ceph-mon[185886]: pgmap v1005: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-01T09:32:14.334 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:14.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:14.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:14.441 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:14.466 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:14 smithi190 ceph-mon[160435]: pgmap v1005: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 690 op/s 2023-10-01T09:32:14.469 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:14.496 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:32:14.538 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:32:14.540 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:14.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:14.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:14.564 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:32:14.597 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:32:14.637 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:32:14.648 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:14.667 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:32:14.686 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:14.719 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:32:14.753 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:32:14.759 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:14.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:14.781 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:32:14.781 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:32:14.781 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:32:14.781 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:32:14.781 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:32:14.781 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:32:14.781 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:32:14.781 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:32:14.866 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:14.894 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:15.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:32:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:32:14] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.33.4" 2023-10-01T09:32:15.442 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:15 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1053718519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:15.442 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:15 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1053718519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:15.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:15 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1053718519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:16.137 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:16 smithi138 ceph-mon[192513]: pgmap v1006: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-01T09:32:16.381 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:16.387 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:16 smithi138 ceph-mon[185886]: pgmap v1006: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-01T09:32:16.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:16.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:16.446 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:16.489 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:16.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:16 smithi190 ceph-mon[160435]: pgmap v1006: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-01T09:32:16.566 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:16.783 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:16.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:16.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:16.900 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:16.930 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:16.992 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:17.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:17.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:17.101 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:17.140 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:17.217 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:17.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:17.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:17.328 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:17.414 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:17.478 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:32:17.479 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:32:17.524 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:17.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:17.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:17.632 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:17.714 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:18.284 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:18.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:18.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:18.394 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:18.419 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:32:18.421 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:18.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:18 smithi138 ceph-mon[192513]: pgmap v1007: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-01T09:32:18.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:18 smithi138 ceph-mon[185886]: pgmap v1007: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-01T09:32:18.702 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:18.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:18.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:18.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:18 smithi190 ceph-mon[160435]: pgmap v1007: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-10-01T09:32:18.815 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:18.856 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:32:18.920 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:19.077 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:19.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:19.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:19.199 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:19.227 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:19.286 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:19.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:19.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:19.396 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:19.429 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:19.485 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:19.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:19.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:19.571 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:19.593 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:19.625 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:19.696 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:19.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:19.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:19.805 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:19.806 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:19.844 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:19.914 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:19.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:19.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:20.022 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:32:20.045 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:32:20.117 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:20.191 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:20 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/996571412' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:20.191 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/996571412' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:20 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/996571412' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:20.826 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:32:20.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:20.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:32:20.880 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5, 6] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2023-10-01T09:32:20.881 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:32:20.881 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-10-01T09:32:20.881 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd in 4 2023-10-01T09:32:20.941 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:20.966 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:21.008 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:21.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:21.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:21.118 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:21.146 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:21.218 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:21.228 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:21 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:32:21.228 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:21 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:32:21.228 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:21 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/996571412' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:32:21.229 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:21 smithi190 ceph-mon[160435]: osdmap e359: 8 total, 8 up, 7 in 2023-10-01T09:32:21.229 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:21 smithi190 ceph-mon[160435]: failed to encode map e359 with expected crc 2023-10-01T09:32:21.229 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:21 smithi190 ceph-mon[160435]: failed to encode map e359 with expected crc 2023-10-01T09:32:21.229 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:21 smithi190 ceph-mon[160435]: failed to encode map e359 with expected crc 2023-10-01T09:32:21.229 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:21 smithi190 ceph-mon[160435]: pgmap v1009: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T09:32:21.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:21.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:21.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:32:21.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:32:21.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/996571412' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:32:21.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[185886]: osdmap e359: 8 total, 8 up, 7 in 2023-10-01T09:32:21.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[185886]: failed to encode map e359 with expected crc 2023-10-01T09:32:21.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[185886]: failed to encode map e359 with expected crc 2023-10-01T09:32:21.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[185886]: failed to encode map e359 with expected crc 2023-10-01T09:32:21.326 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[185886]: pgmap v1009: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T09:32:21.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:32:21.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:32:21.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/996571412' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:32:21.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[192513]: osdmap e359: 8 total, 8 up, 7 in 2023-10-01T09:32:21.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[192513]: failed to encode map e359 with expected crc 2023-10-01T09:32:21.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[192513]: failed to encode map e359 with expected crc 2023-10-01T09:32:21.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[192513]: failed to encode map e359 with expected crc 2023-10-01T09:32:21.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:21 smithi138 ceph-mon[192513]: pgmap v1009: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T09:32:21.335 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:21.380 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:21.486 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:21.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:21.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:21.593 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:21.656 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:21.698 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:21.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:21.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:21.804 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:21.879 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:22.255 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:22 smithi138 ceph-mon[185886]: pgmap v1010: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 666 op/s 2023-10-01T09:32:22.255 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:22 smithi138 ceph-mon[192513]: pgmap v1010: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 666 op/s 2023-10-01T09:32:22.278 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:22.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:22.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:22.389 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:22.415 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:22.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:22 smithi190 ceph-mon[160435]: pgmap v1010: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 666 op/s 2023-10-01T09:32:22.726 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:22.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:22.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:22.840 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:32:22.911 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:23.052 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:32:23.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:23.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:32:23.170 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:23.209 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:23.283 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:23.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:23.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:23.337 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:23.392 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:23.496 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:23.749 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:23.755 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:32:23.755 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:32:23.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:23.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:23.773 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:32:23.808 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:32:23.853 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:32:23.867 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:23.906 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:32:23.952 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:32:23.958 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:23.993 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:32:24.025 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:32:24.026 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:32:24.028 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:23 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2991017728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:24.035 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:23 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2991017728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:24.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:23 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2991017728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:24.060 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:32:24.077 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:32:24.077 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:32:24.077 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:32:24.077 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:32:24.077 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:32:24.077 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:32:24.077 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:32:24.077 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:32:25.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:24 smithi190 ceph-mon[160435]: pgmap v1011: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 672 op/s 2023-10-01T09:32:25.059 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:24 smithi138 ceph-mon[185886]: pgmap v1011: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 672 op/s 2023-10-01T09:32:25.059 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:32:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:32:24] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.33.4" 2023-10-01T09:32:25.060 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:24 smithi138 ceph-mon[192513]: pgmap v1011: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 672 op/s 2023-10-01T09:32:25.749 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:25.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:25.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:25.854 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:32:25.864 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:25.999 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:26.201 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:26.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:26.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:26.307 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:26.312 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:26.312 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:32:26.312 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:26 smithi138 ceph-mon[185886]: pgmap v1012: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T09:32:26.345 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:26.407 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:26.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:26.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:26.475 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:26.516 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:32:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:26 smithi138 ceph-mon[192513]: pgmap v1012: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T09:32:26.601 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:26.796 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:26.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:26.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:32:26.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:26 smithi190 ceph-mon[160435]: pgmap v1012: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T09:32:26.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:26.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:26.917 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:27.009 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:27.153 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:27.174 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:27 smithi138 conmon[185862]: 2023-10-01T09:32:27.055+0000 7f70d64ec700 -1 mon.a@0(leader).osd e359 definitely_dead 0 2023-10-01T09:32:27.180 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:27.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:27.285 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:27.373 INFO:teuthology.orchestra.run.smithi138.stderr:marked in osd.4. 2023-10-01T09:32:27.378 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:27.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:27 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1824116292' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-01T09:32:27.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:27 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1824116292' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-01T09:32:27.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:27 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1824116292' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-10-01T09:32:28.096 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:32:28.446 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:28.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: failed to encode map e360 with expected crc 2023-10-01T09:32:28.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: failed to encode map e360 with expected crc 2023-10-01T09:32:28.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1824116292' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-01T09:32:28.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: osdmap e360: 8 total, 8 up, 8 in 2023-10-01T09:32:28.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: osdmap e361: 8 total, 8 up, 8 in 2023-10-01T09:32:28.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.454 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[192513]: pgmap v1015: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 576 op/s 2023-10-01T09:32:28.454 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: failed to encode map e360 with expected crc 2023-10-01T09:32:28.454 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: failed to encode map e360 with expected crc 2023-10-01T09:32:28.454 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1824116292' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-01T09:32:28.454 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: osdmap e360: 8 total, 8 up, 8 in 2023-10-01T09:32:28.454 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.454 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.454 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: osdmap e361: 8 total, 8 up, 8 in 2023-10-01T09:32:28.454 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.454 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:28 smithi138 ceph-mon[185886]: pgmap v1015: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 576 op/s 2023-10-01T09:32:28.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:28.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:28.557 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:28.595 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: failed to encode map e360 with expected crc 2023-10-01T09:32:28.595 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: failed to encode map e360 with expected crc 2023-10-01T09:32:28.595 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1824116292' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-10-01T09:32:28.595 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: osdmap e360: 8 total, 8 up, 8 in 2023-10-01T09:32:28.595 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.595 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.595 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: osdmap e361: 8 total, 8 up, 8 in 2023-10-01T09:32:28.596 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.596 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.596 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.596 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.596 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.596 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.596 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.596 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.596 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.596 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: failed to encode map e361 with expected crc 2023-10-01T09:32:28.596 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:28 smithi190 ceph-mon[160435]: pgmap v1015: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 576 op/s 2023-10-01T09:32:28.598 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:28.659 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:28.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:28.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:28.771 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:28.818 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:28.883 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:28.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:28.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:28.900 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-10-01T09:32:28.989 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:29.056 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:29.134 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:29.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:29.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:29.240 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:29.278 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:29.388 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:29.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:29.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:29.499 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:29.559 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:29.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[192513]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[192513]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[192513]: osdmap e362: 8 total, 8 up, 8 in 2023-10-01T09:32:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[192513]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[192513]: 8.e starting backfill to osd.4 from (0'0,0'0] MAX to 359'10200 2023-10-01T09:32:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[192513]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[192513]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[192513]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[192513]: 8.5 starting backfill to osd.4 from (0'0,0'0] MAX to 359'10126 2023-10-01T09:32:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[192513]: 8.b starting backfill to osd.4 from (0'0,0'0] MAX to 359'9935 2023-10-01T09:32:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[192513]: 8.2 starting backfill to osd.4 from (0'0,0'0] MAX to 359'9864 2023-10-01T09:32:29.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[192513]: 8.4 starting backfill to osd.4 from (0'0,0'0] MAX to 359'9880 2023-10-01T09:32:29.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[185886]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[185886]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[185886]: osdmap e362: 8 total, 8 up, 8 in 2023-10-01T09:32:29.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[185886]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[185886]: 8.e starting backfill to osd.4 from (0'0,0'0] MAX to 359'10200 2023-10-01T09:32:29.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[185886]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[185886]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[185886]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[185886]: 8.5 starting backfill to osd.4 from (0'0,0'0] MAX to 359'10126 2023-10-01T09:32:29.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[185886]: 8.b starting backfill to osd.4 from (0'0,0'0] MAX to 359'9935 2023-10-01T09:32:29.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[185886]: 8.2 starting backfill to osd.4 from (0'0,0'0] MAX to 359'9864 2023-10-01T09:32:29.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:29 smithi138 ceph-mon[185886]: 8.4 starting backfill to osd.4 from (0'0,0'0] MAX to 359'9880 2023-10-01T09:32:29.674 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:29.679 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:29 smithi190 ceph-mon[160435]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.679 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:29 smithi190 ceph-mon[160435]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.679 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:29 smithi190 ceph-mon[160435]: osdmap e362: 8 total, 8 up, 8 in 2023-10-01T09:32:29.679 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:29 smithi190 ceph-mon[160435]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.679 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:29 smithi190 ceph-mon[160435]: 8.e starting backfill to osd.4 from (0'0,0'0] MAX to 359'10200 2023-10-01T09:32:29.680 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:29 smithi190 ceph-mon[160435]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.680 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:29 smithi190 ceph-mon[160435]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.680 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:29 smithi190 ceph-mon[160435]: failed to encode map e362 with expected crc 2023-10-01T09:32:29.680 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:29 smithi190 ceph-mon[160435]: 8.5 starting backfill to osd.4 from (0'0,0'0] MAX to 359'10126 2023-10-01T09:32:29.680 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:29 smithi190 ceph-mon[160435]: 8.b starting backfill to osd.4 from (0'0,0'0] MAX to 359'9935 2023-10-01T09:32:29.680 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:29 smithi190 ceph-mon[160435]: 8.2 starting backfill to osd.4 from (0'0,0'0] MAX to 359'9864 2023-10-01T09:32:29.680 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:29 smithi190 ceph-mon[160435]: 8.4 starting backfill to osd.4 from (0'0,0'0] MAX to 359'9880 2023-10-01T09:32:29.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:29.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:29.782 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:32:29.870 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:30.056 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:32:30.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:30.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:32:30.167 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:30.226 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:30.256 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:30.386 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:30.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:30.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:30.492 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:30.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:30 smithi138 ceph-mon[185886]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:30 smithi138 ceph-mon[185886]: osdmap e363: 8 total, 8 up, 8 in 2023-10-01T09:32:30.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:30 smithi138 ceph-mon[185886]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:30 smithi138 ceph-mon[185886]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:30 smithi138 ceph-mon[185886]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:30 smithi138 ceph-mon[185886]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.519 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:30 smithi138 ceph-mon[185886]: pgmap v1018: 89 pgs: 1 activating+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 13/130095 objects degraded (0.010%); 8097/130095 objects misplaced (6.224%); 0 B/s, 0 objects/s recovering 2023-10-01T09:32:30.529 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:30.608 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:30.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:30.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:30.726 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:30.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:30 smithi138 ceph-mon[192513]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:30 smithi138 ceph-mon[192513]: osdmap e363: 8 total, 8 up, 8 in 2023-10-01T09:32:30.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:30 smithi138 ceph-mon[192513]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:30 smithi138 ceph-mon[192513]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:30 smithi138 ceph-mon[192513]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:30 smithi138 ceph-mon[192513]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:30 smithi138 ceph-mon[192513]: pgmap v1018: 89 pgs: 1 activating+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 13/130095 objects degraded (0.010%); 8097/130095 objects misplaced (6.224%); 0 B/s, 0 objects/s recovering 2023-10-01T09:32:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:30 smithi190 ceph-mon[160435]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:30 smithi190 ceph-mon[160435]: osdmap e363: 8 total, 8 up, 8 in 2023-10-01T09:32:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:30 smithi190 ceph-mon[160435]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:30 smithi190 ceph-mon[160435]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:30 smithi190 ceph-mon[160435]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:30 smithi190 ceph-mon[160435]: failed to encode map e363 with expected crc 2023-10-01T09:32:30.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:30 smithi190 ceph-mon[160435]: pgmap v1018: 89 pgs: 1 activating+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 85 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 13/130095 objects degraded (0.010%); 8097/130095 objects misplaced (6.224%); 0 B/s, 0 objects/s recovering 2023-10-01T09:32:30.811 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:30.934 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:32:30.935 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:32:30.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:32:30.935 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:32:30.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.935 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178743 6.b 6:d51fa993:::benchmark_data_smithi138_307751_object83374:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-10-01T09:32:30.935 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.547557+0000", 2023-10-01T09:32:30.935 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 561.37956264699994, 2023-10-01T09:32:30.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2844235369999999, 2023-10-01T09:32:30.935 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:30.935 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:32:30.936 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:30.936 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:30.936 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:30.936 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178743 2023-10-01T09:32:30.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.936 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:30.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:30.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547557+0000", 2023-10-01T09:32:30.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:30.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:30.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547557+0000", 2023-10-01T09:32:30.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:30.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:30.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547556+0000", 2023-10-01T09:32:30.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:32:30.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:30.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547562+0000", 2023-10-01T09:32:30.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1379999999999998e-06 2023-10-01T09:32:30.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:30.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547574+0000", 2023-10-01T09:32:30.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2531000000000001e-05 2023-10-01T09:32:30.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:30.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547582+0000", 2023-10-01T09:32:30.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4089999999999997e-06 2023-10-01T09:32:30.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547630+0000", 2023-10-01T09:32:30.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8316999999999998e-05 2023-10-01T09:32:30.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:30.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547718+0000", 2023-10-01T09:32:30.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7713999999999994e-05 2023-10-01T09:32:30.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6", 2023-10-01T09:32:30.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.547788+0000", 2023-10-01T09:32:30.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0086999999999995e-05 2023-10-01T09:32:30.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:30.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.550875+0000", 2023-10-01T09:32:30.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030869299999999999 2023-10-01T09:32:30.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.551149+0000", 2023-10-01T09:32:30.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00027397599999999999 2023-10-01T09:32:30.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.569123+0000", 2023-10-01T09:32:30.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017974331999999999 2023-10-01T09:32:30.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:30.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.569137+0000", 2023-10-01T09:32:30.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3746e-05 2023-10-01T09:32:30.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.942 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.942 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.605915+0000", 2023-10-01T09:32:30.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0367782169999999 2023-10-01T09:32:30.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.942 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:30.942 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.605928+0000", 2023-10-01T09:32:30.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2652e-05 2023-10-01T09:32:30.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.831898+0000", 2023-10-01T09:32:30.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2259704 2023-10-01T09:32:30.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:30.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.831981+0000", 2023-10-01T09:32:30.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3023999999999997e-05 2023-10-01T09:32:30.943 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:30.943 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:30.943 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:30.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.944 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178766 6.b 6:dcd3b8b8:::benchmark_data_smithi138_307751_object83397:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:32:30.944 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.597731+0000", 2023-10-01T09:32:30.944 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 561.32938858299997, 2023-10-01T09:32:30.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2475989780000001, 2023-10-01T09:32:30.944 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:30.944 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:30.944 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:30.944 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:30.944 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:30.944 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178766 2023-10-01T09:32:30.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.945 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:30.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:30.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597731+0000", 2023-10-01T09:32:30.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:30.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:30.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597731+0000", 2023-10-01T09:32:30.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:30.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:30.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597730+0000", 2023-10-01T09:32:30.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:32:30.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:30.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597736+0000", 2023-10-01T09:32:30.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5930000000000002e-06 2023-10-01T09:32:30.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:30.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597750+0000", 2023-10-01T09:32:30.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4090000000000001e-05 2023-10-01T09:32:30.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:30.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597757+0000", 2023-10-01T09:32:30.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1169999999999998e-06 2023-10-01T09:32:30.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597815+0000", 2023-10-01T09:32:30.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7797000000000002e-05 2023-10-01T09:32:30.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:30.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.597825+0000", 2023-10-01T09:32:30.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9769999999999995e-06 2023-10-01T09:32:30.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.605953+0000", 2023-10-01T09:32:30.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.008127888 2023-10-01T09:32:30.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:30.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.605961+0000", 2023-10-01T09:32:30.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0160000000000008e-06 2023-10-01T09:32:30.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.832175+0000", 2023-10-01T09:32:30.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.226213862 2023-10-01T09:32:30.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:30.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.832268+0000", 2023-10-01T09:32:30.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2755000000000002e-05 2023-10-01T09:32:30.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.950 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:32:30.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.832346+0000", 2023-10-01T09:32:30.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8447000000000003e-05 2023-10-01T09:32:30.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.950 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.838284+0000", 2023-10-01T09:32:30.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059380830000000002 2023-10-01T09:32:30.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:30.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.838432+0000", 2023-10-01T09:32:30.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014792399999999999 2023-10-01T09:32:30.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.838497+0000", 2023-10-01T09:32:30.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4628000000000005e-05 2023-10-01T09:32:30.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845139+0000", 2023-10-01T09:32:30.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066418249999999996 2023-10-01T09:32:30.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:30.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845196+0000", 2023-10-01T09:32:30.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7364000000000002e-05 2023-10-01T09:32:30.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:30.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845330+0000", 2023-10-01T09:32:30.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000134423 2023-10-01T09:32:30.953 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:30.953 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:30.953 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:30.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.953 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178802 6.b 6:de530af6:::benchmark_data_smithi138_307751_object83433:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:32:30.953 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.644385+0000", 2023-10-01T09:32:30.953 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 561.28273464300003, 2023-10-01T09:32:30.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2012870950000001, 2023-10-01T09:32:30.954 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:30.954 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:30.954 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:30.954 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:30.954 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:30.954 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178802 2023-10-01T09:32:30.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.954 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:30.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:30.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644385+0000", 2023-10-01T09:32:30.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:30.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:30.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644385+0000", 2023-10-01T09:32:30.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:30.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:30.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644383+0000", 2023-10-01T09:32:30.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:32:30.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:30.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644391+0000", 2023-10-01T09:32:30.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5950000000000001e-06 2023-10-01T09:32:30.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:30.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644407+0000", 2023-10-01T09:32:30.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6685999999999999e-05 2023-10-01T09:32:30.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:30.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644416+0000", 2023-10-01T09:32:30.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4610000000000008e-06 2023-10-01T09:32:30.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644488+0000", 2023-10-01T09:32:30.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2255999999999994e-05 2023-10-01T09:32:30.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:30.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.644499+0000", 2023-10-01T09:32:30.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0594999999999999e-05 2023-10-01T09:32:30.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.605983+0000", 2023-10-01T09:32:30.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96148462700000004 2023-10-01T09:32:30.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:30.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.605990+0000", 2023-10-01T09:32:30.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3080000000000004e-06 2023-10-01T09:32:30.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.959 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.959 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.834922+0000", 2023-10-01T09:32:30.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2289324289999999 2023-10-01T09:32:30.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.959 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:30.959 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.835020+0000", 2023-10-01T09:32:30.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8348000000000001e-05 2023-10-01T09:32:30.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:32:30.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.835115+0000", 2023-10-01T09:32:30.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4214000000000003e-05 2023-10-01T09:32:30.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:30.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.841312+0000", 2023-10-01T09:32:30.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0061974939999999996 2023-10-01T09:32:30.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.844863+0000", 2023-10-01T09:32:30.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035506069999999999 2023-10-01T09:32:30.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845393+0000", 2023-10-01T09:32:30.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00053041099999999995 2023-10-01T09:32:30.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845442+0000", 2023-10-01T09:32:30.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9171000000000001e-05 2023-10-01T09:32:30.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:30.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845506+0000", 2023-10-01T09:32:30.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3492999999999997e-05 2023-10-01T09:32:30.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:30.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845672+0000", 2023-10-01T09:32:30.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000166751 2023-10-01T09:32:30.962 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:30.962 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:30.962 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:30.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.963 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178832 6.b 6:d8044603:::benchmark_data_smithi138_307751_object83463:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:32:30.965 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.679875+0000", 2023-10-01T09:32:30.965 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 561.24724519200004, 2023-10-01T09:32:30.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1661682290000002, 2023-10-01T09:32:30.965 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:30.965 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:30.965 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:30.965 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:30.965 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:30.966 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178832 2023-10-01T09:32:30.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.966 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:30.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.966 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:30.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679875+0000", 2023-10-01T09:32:30.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:30.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.966 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:30.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679875+0000", 2023-10-01T09:32:30.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:30.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:30.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679873+0000", 2023-10-01T09:32:30.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:32:30.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:30.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679882+0000", 2023-10-01T09:32:30.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6729999999999998e-06 2023-10-01T09:32:30.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:30.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679897+0000", 2023-10-01T09:32:30.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5526999999999999e-05 2023-10-01T09:32:30.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:30.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679907+0000", 2023-10-01T09:32:30.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3910000000000004e-06 2023-10-01T09:32:30.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679978+0000", 2023-10-01T09:32:30.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1153000000000001e-05 2023-10-01T09:32:30.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:30.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.679992+0000", 2023-10-01T09:32:30.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4256e-05 2023-10-01T09:32:30.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606003+0000", 2023-10-01T09:32:30.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92601074299999997 2023-10-01T09:32:30.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:30.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606007+0000", 2023-10-01T09:32:30.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4150000000000001e-06 2023-10-01T09:32:30.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.838522+0000", 2023-10-01T09:32:30.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.232515282 2023-10-01T09:32:30.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:30.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.838618+0000", 2023-10-01T09:32:30.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6039000000000007e-05 2023-10-01T09:32:30.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:32:30.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.838703+0000", 2023-10-01T09:32:30.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4345999999999995e-05 2023-10-01T09:32:30.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:30.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.844873+0000", 2023-10-01T09:32:30.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006170633 2023-10-01T09:32:30.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845757+0000", 2023-10-01T09:32:30.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00088383699999999995 2023-10-01T09:32:30.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845819+0000", 2023-10-01T09:32:30.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1549000000000003e-05 2023-10-01T09:32:30.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845874+0000", 2023-10-01T09:32:30.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5065000000000002e-05 2023-10-01T09:32:30.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:30.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845925+0000", 2023-10-01T09:32:30.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1604000000000003e-05 2023-10-01T09:32:30.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:30.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.846043+0000", 2023-10-01T09:32:30.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011757899999999999 2023-10-01T09:32:30.974 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:30.974 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:30.974 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:30.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.974 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178834 6.b 6:d604d262:::benchmark_data_smithi138_307751_object83465:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:32:30.975 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.683872+0000", 2023-10-01T09:32:30.975 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 561.24324810200005, 2023-10-01T09:32:30.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1718884809999999, 2023-10-01T09:32:30.975 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:30.975 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:30.975 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:30.975 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:30.975 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:30.975 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178834 2023-10-01T09:32:30.975 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.975 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:30.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:30.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683872+0000", 2023-10-01T09:32:30.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:30.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:30.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683872+0000", 2023-10-01T09:32:30.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:30.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:30.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683870+0000", 2023-10-01T09:32:30.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:32:30.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:30.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683877+0000", 2023-10-01T09:32:30.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6379999999999998e-06 2023-10-01T09:32:30.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:30.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683894+0000", 2023-10-01T09:32:30.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6860000000000001e-05 2023-10-01T09:32:30.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:30.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683902+0000", 2023-10-01T09:32:30.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1410000000000005e-06 2023-10-01T09:32:30.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683968+0000", 2023-10-01T09:32:30.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6450000000000002e-05 2023-10-01T09:32:30.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:30.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.683983+0000", 2023-10-01T09:32:30.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.49e-05 2023-10-01T09:32:30.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606019+0000", 2023-10-01T09:32:30.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.92203571299999998 2023-10-01T09:32:30.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:30.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606024+0000", 2023-10-01T09:32:30.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.011e-06 2023-10-01T09:32:30.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.841339+0000", 2023-10-01T09:32:30.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.235314947 2023-10-01T09:32:30.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:30.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.841469+0000", 2023-10-01T09:32:30.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013036600000000001 2023-10-01T09:32:30.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:32:30.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.841644+0000", 2023-10-01T09:32:30.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017456700000000001 2023-10-01T09:32:30.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:30.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.845691+0000", 2023-10-01T09:32:30.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040474459999999997 2023-10-01T09:32:30.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.846103+0000", 2023-10-01T09:32:30.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00041115299999999999 2023-10-01T09:32:30.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.982 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855580+0000", 2023-10-01T09:32:30.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094775910000000005 2023-10-01T09:32:30.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.982 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855611+0000", 2023-10-01T09:32:30.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0985999999999998e-05 2023-10-01T09:32:30.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.983 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.983 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:30.983 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855655+0000", 2023-10-01T09:32:30.983 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3445999999999997e-05 2023-10-01T09:32:30.983 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.983 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.983 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:30.983 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855760+0000", 2023-10-01T09:32:30.983 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010581099999999999 2023-10-01T09:32:30.983 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:30.983 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:30.983 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:30.984 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.984 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178842 6.b 6:deac9fef:::benchmark_data_smithi138_307751_object83473:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:32:30.984 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.722803+0000", 2023-10-01T09:32:30.984 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 561.20431714200004, 2023-10-01T09:32:30.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1395442039999999, 2023-10-01T09:32:30.984 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:30.984 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:30.984 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:30.984 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:30.984 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:30.984 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178842 2023-10-01T09:32:30.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.985 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:30.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:30.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722803+0000", 2023-10-01T09:32:30.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:30.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:30.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722803+0000", 2023-10-01T09:32:30.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:30.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.986 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:30.986 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722801+0000", 2023-10-01T09:32:30.986 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:32:30.986 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.986 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:30.986 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722809+0000", 2023-10-01T09:32:30.986 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5399999999999998e-06 2023-10-01T09:32:30.986 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:30.987 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722824+0000", 2023-10-01T09:32:30.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5231999999999999e-05 2023-10-01T09:32:30.987 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.987 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:30.987 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722832+0000", 2023-10-01T09:32:30.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6990000000000003e-06 2023-10-01T09:32:30.987 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.987 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.987 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722883+0000", 2023-10-01T09:32:30.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1439e-05 2023-10-01T09:32:30.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:30.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.722893+0000", 2023-10-01T09:32:30.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4429999999999992e-06 2023-10-01T09:32:30.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606037+0000", 2023-10-01T09:32:30.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.88314473500000001 2023-10-01T09:32:30.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:30.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606042+0000", 2023-10-01T09:32:30.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2350000000000001e-06 2023-10-01T09:32:30.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.846136+0000", 2023-10-01T09:32:30.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.240094662 2023-10-01T09:32:30.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:30.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.846286+0000", 2023-10-01T09:32:30.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015010300000000001 2023-10-01T09:32:30.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:32:30.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.846430+0000", 2023-10-01T09:32:30.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014343400000000001 2023-10-01T09:32:30.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:30.991 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855502+0000", 2023-10-01T09:32:30.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0090716679999999997 2023-10-01T09:32:30.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.991 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.991 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855541+0000", 2023-10-01T09:32:30.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9038000000000001e-05 2023-10-01T09:32:30.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.991 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.991 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855798+0000", 2023-10-01T09:32:30.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025726999999999998 2023-10-01T09:32:30.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:30.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862172+0000", 2023-10-01T09:32:30.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0063741739999999998 2023-10-01T09:32:30.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:30.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862230+0000", 2023-10-01T09:32:30.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7942000000000002e-05 2023-10-01T09:32:30.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:30.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862347+0000", 2023-10-01T09:32:30.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011716 2023-10-01T09:32:30.993 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:30.993 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:30.993 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:30.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.993 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178859 6.b 6:d9d8473f:::benchmark_data_smithi138_307751_object83490:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:32:30.993 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.882356+0000", 2023-10-01T09:32:30.994 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 561.044764278, 2023-10-01T09:32:30.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9801762709999999, 2023-10-01T09:32:30.994 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:30.994 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:30.994 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:30.994 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:30.994 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:30.994 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178859 2023-10-01T09:32:30.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.994 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:30.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:30.995 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882356+0000", 2023-10-01T09:32:30.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:30.995 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:30.995 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882356+0000", 2023-10-01T09:32:30.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:30.995 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:30.995 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882355+0000", 2023-10-01T09:32:30.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:32:30.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:30.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882360+0000", 2023-10-01T09:32:30.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8579999999999999e-06 2023-10-01T09:32:30.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:30.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882372+0000", 2023-10-01T09:32:30.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1674999999999999e-05 2023-10-01T09:32:30.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:30.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882378+0000", 2023-10-01T09:32:30.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0630000000000002e-06 2023-10-01T09:32:30.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882441+0000", 2023-10-01T09:32:30.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3232999999999996e-05 2023-10-01T09:32:30.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:30.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.882458+0000", 2023-10-01T09:32:30.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7087000000000001e-05 2023-10-01T09:32:30.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606055+0000", 2023-10-01T09:32:30.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.72359739000000001 2023-10-01T09:32:30.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:30.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606060+0000", 2023-10-01T09:32:30.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1559999999999997e-06 2023-10-01T09:32:30.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.999 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:30.999 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.849168+0000", 2023-10-01T09:32:30.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2431087970000001 2023-10-01T09:32:30.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:30.999 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:30.999 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.849278+0000", 2023-10-01T09:32:30.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109393 2023-10-01T09:32:30.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:30.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.000 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:32:31.000 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.849382+0000", 2023-10-01T09:32:31.000 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000104555 2023-10-01T09:32:31.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.000 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:31.000 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855548+0000", 2023-10-01T09:32:31.000 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0061655649999999996 2023-10-01T09:32:31.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.000 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.000 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855837+0000", 2023-10-01T09:32:31.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00028885599999999999 2023-10-01T09:32:31.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862102+0000", 2023-10-01T09:32:31.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0062649990000000003 2023-10-01T09:32:31.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862389+0000", 2023-10-01T09:32:31.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00028695199999999999 2023-10-01T09:32:31.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:31.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862430+0000", 2023-10-01T09:32:31.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1109000000000002e-05 2023-10-01T09:32:31.002 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:31.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862532+0000", 2023-10-01T09:32:31.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000102305 2023-10-01T09:32:31.002 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.002 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.002 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.003 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178863 6.b 6:d3d535ce:::benchmark_data_smithi138_307751_object83494:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:32:31.003 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.890216+0000", 2023-10-01T09:32:31.003 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 561.03690391500004, 2023-10-01T09:32:31.003 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9882852390000001, 2023-10-01T09:32:31.003 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:31.003 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:31.003 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:31.003 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:31.003 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:31.003 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178863 2023-10-01T09:32:31.004 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.004 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:31.004 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.004 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:31.004 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890216+0000", 2023-10-01T09:32:31.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.004 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.004 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.004 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:31.004 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890216+0000", 2023-10-01T09:32:31.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.004 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:31.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890215+0000", 2023-10-01T09:32:31.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:32:31.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:31.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890221+0000", 2023-10-01T09:32:31.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5819999999999998e-06 2023-10-01T09:32:31.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:31.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890236+0000", 2023-10-01T09:32:31.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4620999999999999e-05 2023-10-01T09:32:31.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:31.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890244+0000", 2023-10-01T09:32:31.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8299999999999996e-06 2023-10-01T09:32:31.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890298+0000", 2023-10-01T09:32:31.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4508000000000001e-05 2023-10-01T09:32:31.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.007 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.007 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.890307+0000", 2023-10-01T09:32:31.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7290000000000006e-06 2023-10-01T09:32:31.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.007 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.007 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606072+0000", 2023-10-01T09:32:31.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.71576492899999999 2023-10-01T09:32:31.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606076+0000", 2023-10-01T09:32:31.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0579999999999999e-06 2023-10-01T09:32:31.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.852232+0000", 2023-10-01T09:32:31.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.246155755 2023-10-01T09:32:31.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:31.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.852363+0000", 2023-10-01T09:32:31.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013094699999999999 2023-10-01T09:32:31.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:32:31.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.852489+0000", 2023-10-01T09:32:31.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000126635 2023-10-01T09:32:31.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:31.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855805+0000", 2023-10-01T09:32:31.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0033155120000000001 2023-10-01T09:32:31.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862138+0000", 2023-10-01T09:32:31.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006333198 2023-10-01T09:32:31.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869114+0000", 2023-10-01T09:32:31.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069759469999999997 2023-10-01T09:32:31.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878304+0000", 2023-10-01T09:32:31.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0091900300000000001 2023-10-01T09:32:31.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:31.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878367+0000", 2023-10-01T09:32:31.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2658999999999998e-05 2023-10-01T09:32:31.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.012 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:31.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878501+0000", 2023-10-01T09:32:31.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000134609 2023-10-01T09:32:31.012 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.012 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.012 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.012 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178866 6.b 6:d03ed771:::benchmark_data_smithi138_307751_object83497:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:32:31.012 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.903567+0000", 2023-10-01T09:32:31.012 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 561.023552741, 2023-10-01T09:32:31.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9751352579999999, 2023-10-01T09:32:31.013 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:31.013 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:31.013 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:31.013 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:31.013 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:31.013 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178866 2023-10-01T09:32:31.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.013 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:31.013 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:31.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903567+0000", 2023-10-01T09:32:31.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:31.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903567+0000", 2023-10-01T09:32:31.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:31.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903565+0000", 2023-10-01T09:32:31.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:32:31.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:31.015 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903573+0000", 2023-10-01T09:32:31.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9270000000000005e-06 2023-10-01T09:32:31.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:31.015 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903589+0000", 2023-10-01T09:32:31.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5112e-05 2023-10-01T09:32:31.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:31.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903598+0000", 2023-10-01T09:32:31.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4050000000000006e-06 2023-10-01T09:32:31.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903655+0000", 2023-10-01T09:32:31.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6694999999999998e-05 2023-10-01T09:32:31.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.903670+0000", 2023-10-01T09:32:31.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5192e-05 2023-10-01T09:32:31.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606090+0000", 2023-10-01T09:32:31.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.70241997300000003 2023-10-01T09:32:31.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606097+0000", 2023-10-01T09:32:31.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2300000000000002e-06 2023-10-01T09:32:31.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.018 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855860+0000", 2023-10-01T09:32:31.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.249762633 2023-10-01T09:32:31.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:31.018 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.855972+0000", 2023-10-01T09:32:31.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112587 2023-10-01T09:32:31.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:32:31.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.856087+0000", 2023-10-01T09:32:31.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115238 2023-10-01T09:32:31.019 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862571+0000", 2023-10-01T09:32:31.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064834539999999996 2023-10-01T09:32:31.019 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869173+0000", 2023-10-01T09:32:31.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066022570000000003 2023-10-01T09:32:31.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:31.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869318+0000", 2023-10-01T09:32:31.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000145084 2023-10-01T09:32:31.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878567+0000", 2023-10-01T09:32:31.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0092485280000000007 2023-10-01T09:32:31.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:31.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878602+0000", 2023-10-01T09:32:31.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5315999999999999e-05 2023-10-01T09:32:31.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:31.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878703+0000", 2023-10-01T09:32:31.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000100436 2023-10-01T09:32:31.021 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.021 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.021 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.022 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178873 6.b 6:d1b63b67:::benchmark_data_smithi138_307751_object83504:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:32:31.022 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.923214+0000", 2023-10-01T09:32:31.022 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 561.00390567800002, 2023-10-01T09:32:31.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.95573526, 2023-10-01T09:32:31.022 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:31.022 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:31.022 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:31.022 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:31.022 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:31.022 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178873 2023-10-01T09:32:31.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.023 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:31.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:31.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923214+0000", 2023-10-01T09:32:31.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:31.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923214+0000", 2023-10-01T09:32:31.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:31.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923200+0000", 2023-10-01T09:32:31.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999857 2023-10-01T09:32:31.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:31.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923221+0000", 2023-10-01T09:32:31.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1022999999999998e-05 2023-10-01T09:32:31.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.025 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:31.025 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923238+0000", 2023-10-01T09:32:31.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6948000000000001e-05 2023-10-01T09:32:31.025 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.025 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.025 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:31.025 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923246+0000", 2023-10-01T09:32:31.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6830000000000008e-06 2023-10-01T09:32:31.025 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.025 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.025 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.025 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923317+0000", 2023-10-01T09:32:31.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1519000000000004e-05 2023-10-01T09:32:31.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.923330+0000", 2023-10-01T09:32:31.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3159000000000001e-05 2023-10-01T09:32:31.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606118+0000", 2023-10-01T09:32:31.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.68278745900000004 2023-10-01T09:32:31.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606126+0000", 2023-10-01T09:32:31.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2060000000000007e-06 2023-10-01T09:32:31.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.858734+0000", 2023-10-01T09:32:31.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2526076939999999 2023-10-01T09:32:31.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.028 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:31.028 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.858881+0000", 2023-10-01T09:32:31.028 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000147374 2023-10-01T09:32:31.028 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.028 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.028 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:32:31.028 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.859032+0000", 2023-10-01T09:32:31.028 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015129600000000001 2023-10-01T09:32:31.028 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.028 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.028 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.028 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862594+0000", 2023-10-01T09:32:31.029 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035613749999999999 2023-10-01T09:32:31.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.029 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.029 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869240+0000", 2023-10-01T09:32:31.029 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066465070000000003 2023-10-01T09:32:31.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.029 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:31.029 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869326+0000", 2023-10-01T09:32:31.029 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5883000000000004e-05 2023-10-01T09:32:31.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878767+0000", 2023-10-01T09:32:31.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094413799999999992 2023-10-01T09:32:31.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:31.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878827+0000", 2023-10-01T09:32:31.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9855000000000002e-05 2023-10-01T09:32:31.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:31.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.878950+0000", 2023-10-01T09:32:31.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000122273 2023-10-01T09:32:31.031 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.031 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.031 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.031 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35070.0:178875 6.b 6:d0889c71:::benchmark_data_smithi138_307751_object83506:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-10-01T09:32:31.031 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:23:09.927144+0000", 2023-10-01T09:32:31.031 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 560.99997572999996, 2023-10-01T09:32:31.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9520905690000001, 2023-10-01T09:32:31.032 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:31.032 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:31.032 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:31.032 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35070", 2023-10-01T09:32:31.032 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1822629737", 2023-10-01T09:32:31.032 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 178875 2023-10-01T09:32:31.032 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.032 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:31.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.032 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:31.032 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927144+0000", 2023-10-01T09:32:31.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:31.033 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927144+0000", 2023-10-01T09:32:31.033 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:31.033 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927142+0000", 2023-10-01T09:32:31.033 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:32:31.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.034 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:31.034 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927150+0000", 2023-10-01T09:32:31.034 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0209999999999999e-06 2023-10-01T09:32:31.034 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.034 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.034 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:31.034 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927165+0000", 2023-10-01T09:32:31.034 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4939e-05 2023-10-01T09:32:31.034 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.034 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.034 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:31.035 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927173+0000", 2023-10-01T09:32:31.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2599999999999999e-06 2023-10-01T09:32:31.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.035 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927243+0000", 2023-10-01T09:32:31.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0129000000000006e-05 2023-10-01T09:32:31.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.035 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:09.927253+0000", 2023-10-01T09:32:31.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0431999999999999e-05 2023-10-01T09:32:31.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.036 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606149+0000", 2023-10-01T09:32:31.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.67889524400000001 2023-10-01T09:32:31.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.036 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:10.606157+0000", 2023-10-01T09:32:31.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0870000000000007e-06 2023-10-01T09:32:31.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.036 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862610+0000", 2023-10-01T09:32:31.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2564529449999999 2023-10-01T09:32:31.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.037 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:31.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862693+0000", 2023-10-01T09:32:31.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3590000000000004e-05 2023-10-01T09:32:31.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.037 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,6,7", 2023-10-01T09:32:31.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.862765+0000", 2023-10-01T09:32:31.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1478999999999998e-05 2023-10-01T09:32:31.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.038 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.038 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.038 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869306+0000", 2023-10-01T09:32:31.038 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0065418569999999999 2023-10-01T09:32:31.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.038 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.038 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:31.038 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869333+0000", 2023-10-01T09:32:31.038 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6787999999999998e-05 2023-10-01T09:32:31.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.038 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.039 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.039 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.869372+0000", 2023-10-01T09:32:31.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9127e-05 2023-10-01T09:32:31.039 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.039 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.039 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.039 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.879031+0000", 2023-10-01T09:32:31.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0096583450000000005 2023-10-01T09:32:31.039 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.039 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.039 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:31.039 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.879099+0000", 2023-10-01T09:32:31.040 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7879000000000005e-05 2023-10-01T09:32:31.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.040 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:31.040 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:23:11.879235+0000", 2023-10-01T09:32:31.040 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000136279 2023-10-01T09:32:31.040 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.040 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.040 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.040 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129093 7.c 7:3fad7313:::benchmark_data_smithi138_367058_object31298:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:32:31.040 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:48.990163+0000", 2023-10-01T09:32:31.041 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.93695743900003, 2023-10-01T09:32:31.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.108214129, 2023-10-01T09:32:31.041 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:31.041 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:31.041 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:31.041 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:32:31.041 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:32:31.041 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129093 2023-10-01T09:32:31.041 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.041 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:31.041 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.042 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:31.042 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:48.990163+0000", 2023-10-01T09:32:31.042 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.042 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.042 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.042 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:31.042 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:48.990163+0000", 2023-10-01T09:32:31.042 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.042 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.042 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.042 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:31.042 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:48.990161+0000", 2023-10-01T09:32:31.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:32:31.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.043 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.043 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:31.043 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:48.990167+0000", 2023-10-01T09:32:31.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3000000000000001e-06 2023-10-01T09:32:31.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.043 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.043 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:31.043 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:48.990175+0000", 2023-10-01T09:32:31.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.585e-06 2023-10-01T09:32:31.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:31.044 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:48.990182+0000", 2023-10-01T09:32:31.044 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2929999999999997e-06 2023-10-01T09:32:31.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.044 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.021171+0000", 2023-10-01T09:32:31.044 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.030989045999999999 2023-10-01T09:32:31.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.021179+0000", 2023-10-01T09:32:31.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3869999999999995e-06 2023-10-01T09:32:31.045 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.045 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.045 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.073219+0000", 2023-10-01T09:32:31.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0520403949999999 2023-10-01T09:32:31.045 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.045 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.045 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:31.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.073363+0000", 2023-10-01T09:32:31.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000143091 2023-10-01T09:32:31.046 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.046 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.046 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6,7", 2023-10-01T09:32:31.046 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.073536+0000", 2023-10-01T09:32:31.046 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017386100000000001 2023-10-01T09:32:31.046 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.046 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.046 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.046 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.076964+0000", 2023-10-01T09:32:31.046 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034275680000000002 2023-10-01T09:32:31.046 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.046 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.047 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:31.047 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.076971+0000", 2023-10-01T09:32:31.047 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6150000000000002e-06 2023-10-01T09:32:31.047 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.047 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.047 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.047 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.077028+0000", 2023-10-01T09:32:31.047 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7528000000000002e-05 2023-10-01T09:32:31.047 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.047 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.047 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.047 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.097535+0000", 2023-10-01T09:32:31.048 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.02050714 2023-10-01T09:32:31.048 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.048 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.048 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:31.048 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.097590+0000", 2023-10-01T09:32:31.048 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4523e-05 2023-10-01T09:32:31.048 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.048 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.048 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:31.048 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.098377+0000", 2023-10-01T09:32:31.048 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000786983 2023-10-01T09:32:31.048 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.049 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.049 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.049 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.049 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.049 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129107 7.1 7:86d994f7:::benchmark_data_smithi138_367058_object31312:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:32:31.049 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:49.013753+0000", 2023-10-01T09:32:31.049 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.91336678800002, 2023-10-01T09:32:31.049 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.064642614, 2023-10-01T09:32:31.049 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:31.049 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:31.049 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:31.049 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:32:31.050 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:32:31.050 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129107 2023-10-01T09:32:31.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.050 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:31.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:31.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013753+0000", 2023-10-01T09:32:31.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:31.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013753+0000", 2023-10-01T09:32:31.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.051 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:31.051 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013752+0000", 2023-10-01T09:32:31.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:32:31.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.051 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:31.051 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013758+0000", 2023-10-01T09:32:31.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.908e-06 2023-10-01T09:32:31.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.052 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:31.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013767+0000", 2023-10-01T09:32:31.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7820000000000005e-06 2023-10-01T09:32:31.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.052 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:31.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013774+0000", 2023-10-01T09:32:31.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6279999999999999e-06 2023-10-01T09:32:31.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.052 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.053 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013826+0000", 2023-10-01T09:32:31.053 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1839999999999998e-05 2023-10-01T09:32:31.053 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.053 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013839+0000", 2023-10-01T09:32:31.053 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2704000000000001e-05 2023-10-01T09:32:31.053 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.053 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.067857+0000", 2023-10-01T09:32:31.053 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.054017991 2023-10-01T09:32:31.054 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.054 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.054 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:31.054 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.068023+0000", 2023-10-01T09:32:31.054 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016644699999999999 2023-10-01T09:32:31.054 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.054 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.054 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,4,6", 2023-10-01T09:32:31.054 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.068242+0000", 2023-10-01T09:32:31.054 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000219445 2023-10-01T09:32:31.054 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.054 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.055 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.055 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.071910+0000", 2023-10-01T09:32:31.055 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036673579999999999 2023-10-01T09:32:31.055 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.055 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.055 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.055 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.071944+0000", 2023-10-01T09:32:31.055 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4419999999999999e-05 2023-10-01T09:32:31.055 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.055 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.055 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:31.055 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.071954+0000", 2023-10-01T09:32:31.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4590000000000004e-06 2023-10-01T09:32:31.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.072007+0000", 2023-10-01T09:32:31.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2777e-05 2023-10-01T09:32:31.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.077735+0000", 2023-10-01T09:32:31.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057282039999999998 2023-10-01T09:32:31.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:31.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.077771+0000", 2023-10-01T09:32:31.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5790000000000001e-05 2023-10-01T09:32:31.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:31.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078396+0000", 2023-10-01T09:32:31.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00062535800000000001 2023-10-01T09:32:31.057 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.057 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.057 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.058 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129109 7.c 7:3aabb763:::benchmark_data_smithi138_367058_object31314:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:32:31.058 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:49.014366+0000", 2023-10-01T09:32:31.058 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.91275369300001, 2023-10-01T09:32:31.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.084557913, 2023-10-01T09:32:31.058 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:31.058 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:31.058 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:31.058 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:32:31.058 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:32:31.058 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129109 2023-10-01T09:32:31.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.059 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:31.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:31.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.014366+0000", 2023-10-01T09:32:31.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:31.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.014366+0000", 2023-10-01T09:32:31.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:31.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.014365+0000", 2023-10-01T09:32:31.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:32:31.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:31.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.014371+0000", 2023-10-01T09:32:31.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1129999999999997e-06 2023-10-01T09:32:31.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:31.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.014387+0000", 2023-10-01T09:32:31.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5548000000000001e-05 2023-10-01T09:32:31.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:31.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.014396+0000", 2023-10-01T09:32:31.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6589999999999996e-06 2023-10-01T09:32:31.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.021197+0000", 2023-10-01T09:32:31.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0068016149999999996 2023-10-01T09:32:31.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.062 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.062 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.062 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.021207+0000", 2023-10-01T09:32:31.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7140000000000005e-06 2023-10-01T09:32:31.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.062 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.062 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.062 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.077079+0000", 2023-10-01T09:32:31.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.055872071 2023-10-01T09:32:31.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.063 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:31.063 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.077198+0000", 2023-10-01T09:32:31.063 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118742 2023-10-01T09:32:31.063 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.063 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6,7", 2023-10-01T09:32:31.063 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.077322+0000", 2023-10-01T09:32:31.063 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012386499999999999 2023-10-01T09:32:31.063 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.063 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.085284+0000", 2023-10-01T09:32:31.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0079620330000000003 2023-10-01T09:32:31.064 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:31.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.085298+0000", 2023-10-01T09:32:31.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4591e-05 2023-10-01T09:32:31.064 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.085366+0000", 2023-10-01T09:32:31.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8028000000000007e-05 2023-10-01T09:32:31.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.098440+0000", 2023-10-01T09:32:31.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013073664 2023-10-01T09:32:31.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:31.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.098492+0000", 2023-10-01T09:32:31.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1774000000000003e-05 2023-10-01T09:32:31.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:31.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.098924+0000", 2023-10-01T09:32:31.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00043260299999999999 2023-10-01T09:32:31.066 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.066 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.066 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.066 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129118 7.1 7:8edf0bff:::benchmark_data_smithi138_367058_object31323:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:32:31.066 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:49.038527+0000", 2023-10-01T09:32:31.066 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.88859303800001, 2023-10-01T09:32:31.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0404381739999999, 2023-10-01T09:32:31.067 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:31.067 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:31.067 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:31.067 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:32:31.067 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:32:31.067 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129118 2023-10-01T09:32:31.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.067 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:31.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:31.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.038527+0000", 2023-10-01T09:32:31.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:31.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.038527+0000", 2023-10-01T09:32:31.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:31.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.038525+0000", 2023-10-01T09:32:31.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:32:31.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:31.069 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.038536+0000", 2023-10-01T09:32:31.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1782e-05 2023-10-01T09:32:31.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:31.069 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.038549+0000", 2023-10-01T09:32:31.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2228999999999999e-05 2023-10-01T09:32:31.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:31.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.038557+0000", 2023-10-01T09:32:31.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5220000000000007e-06 2023-10-01T09:32:31.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.039875+0000", 2023-10-01T09:32:31.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001317384 2023-10-01T09:32:31.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.039884+0000", 2023-10-01T09:32:31.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3789999999999995e-06 2023-10-01T09:32:31.071 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.071 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.071 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.071 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.072039+0000", 2023-10-01T09:32:31.071 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.032154623 2023-10-01T09:32:31.071 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.071 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.071 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:31.071 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.072146+0000", 2023-10-01T09:32:31.071 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107936 2023-10-01T09:32:31.071 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.071 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.072 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,4,6", 2023-10-01T09:32:31.072 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.072264+0000", 2023-10-01T09:32:31.072 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011767799999999999 2023-10-01T09:32:31.072 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.072 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.072 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078438+0000", 2023-10-01T09:32:31.072 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006174194 2023-10-01T09:32:31.072 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.072 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078468+0000", 2023-10-01T09:32:31.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9221999999999999e-05 2023-10-01T09:32:31.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:31.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078476+0000", 2023-10-01T09:32:31.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7029999999999995e-06 2023-10-01T09:32:31.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078511+0000", 2023-10-01T09:32:31.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4972999999999999e-05 2023-10-01T09:32:31.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078537+0000", 2023-10-01T09:32:31.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5562e-05 2023-10-01T09:32:31.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:31.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078571+0000", 2023-10-01T09:32:31.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4350999999999997e-05 2023-10-01T09:32:31.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:31.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078965+0000", 2023-10-01T09:32:31.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000394001 2023-10-01T09:32:31.075 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.075 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.075 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.075 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129121 7.c 7:33e29068:::benchmark_data_smithi138_367058_object31326:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:32:31.075 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:49.051338+0000", 2023-10-01T09:32:31.075 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.87578186399998, 2023-10-01T09:32:31.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0484849009999999, 2023-10-01T09:32:31.076 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:31.076 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:31.076 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:31.076 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:32:31.076 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:32:31.076 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129121 2023-10-01T09:32:31.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.076 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:31.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:31.076 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051338+0000", 2023-10-01T09:32:31.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.077 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:31.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051338+0000", 2023-10-01T09:32:31.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.077 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:31.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051337+0000", 2023-10-01T09:32:31.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:32:31.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:31.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051342+0000", 2023-10-01T09:32:31.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6739999999999996e-06 2023-10-01T09:32:31.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:31.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051350+0000", 2023-10-01T09:32:31.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5480000000000002e-06 2023-10-01T09:32:31.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:31.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051356+0000", 2023-10-01T09:32:31.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.896e-06 2023-10-01T09:32:31.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051399+0000", 2023-10-01T09:32:31.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2916e-05 2023-10-01T09:32:31.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051408+0000", 2023-10-01T09:32:31.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.969e-06 2023-10-01T09:32:31.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.080 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.080 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079606+0000", 2023-10-01T09:32:31.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.028198366 2023-10-01T09:32:31.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.080 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:31.080 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079762+0000", 2023-10-01T09:32:31.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015603000000000001 2023-10-01T09:32:31.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6,7", 2023-10-01T09:32:31.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079923+0000", 2023-10-01T09:32:31.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016086299999999999 2023-10-01T09:32:31.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:31.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.097463+0000", 2023-10-01T09:32:31.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017540224 2023-10-01T09:32:31.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.098992+0000", 2023-10-01T09:32:31.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015289769999999999 2023-10-01T09:32:31.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099153+0000", 2023-10-01T09:32:31.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000160178 2023-10-01T09:32:31.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099285+0000", 2023-10-01T09:32:31.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013285099999999999 2023-10-01T09:32:31.083 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.083 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:31.083 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099345+0000", 2023-10-01T09:32:31.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9145999999999999e-05 2023-10-01T09:32:31.083 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.083 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:31.083 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099823+0000", 2023-10-01T09:32:31.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00047857500000000002 2023-10-01T09:32:31.083 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.083 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.084 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.084 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.084 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129122 7.1 7:8ace0e85:::benchmark_data_smithi138_367058_object31327:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:32:31.084 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:49.051372+0000", 2023-10-01T09:32:31.084 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.87574847500002, 2023-10-01T09:32:31.084 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0346654209999999, 2023-10-01T09:32:31.084 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:31.084 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:31.084 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:31.084 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:32:31.084 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:32:31.084 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129122 2023-10-01T09:32:31.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.085 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:31.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.085 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:31.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051372+0000", 2023-10-01T09:32:31.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.085 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:31.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051372+0000", 2023-10-01T09:32:31.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:31.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051371+0000", 2023-10-01T09:32:31.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:31.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:31.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051374+0000", 2023-10-01T09:32:31.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.666e-06 2023-10-01T09:32:31.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:31.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051379+0000", 2023-10-01T09:32:31.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5389999999999998e-06 2023-10-01T09:32:31.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:31.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051384+0000", 2023-10-01T09:32:31.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6789999999999996e-06 2023-10-01T09:32:31.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051524+0000", 2023-10-01T09:32:31.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0001404 2023-10-01T09:32:31.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051534+0000", 2023-10-01T09:32:31.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4919999999999999e-06 2023-10-01T09:32:31.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.074933+0000", 2023-10-01T09:32:31.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.023398606 2023-10-01T09:32:31.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:31.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.075080+0000", 2023-10-01T09:32:31.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014698599999999999 2023-10-01T09:32:31.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,4,6", 2023-10-01T09:32:31.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.075169+0000", 2023-10-01T09:32:31.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8976999999999999e-05 2023-10-01T09:32:31.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:31.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078483+0000", 2023-10-01T09:32:31.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0033149120000000002 2023-10-01T09:32:31.090 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079044+0000", 2023-10-01T09:32:31.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00056057299999999997 2023-10-01T09:32:31.090 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.091 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079081+0000", 2023-10-01T09:32:31.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7175000000000002e-05 2023-10-01T09:32:31.091 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.091 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.091 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.091 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079108+0000", 2023-10-01T09:32:31.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6740000000000001e-05 2023-10-01T09:32:31.091 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.091 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.091 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.091 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.085270+0000", 2023-10-01T09:32:31.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0061619159999999999 2023-10-01T09:32:31.092 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.092 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.092 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:31.092 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.085329+0000", 2023-10-01T09:32:31.092 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9414999999999999e-05 2023-10-01T09:32:31.092 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.092 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.092 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:31.092 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086037+0000", 2023-10-01T09:32:31.092 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00070771199999999999 2023-10-01T09:32:31.092 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.092 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.093 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.093 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.093 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.093 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129124 7.1 7:89d65aee:::benchmark_data_smithi138_367058_object31329:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:32:31.093 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:49.051416+0000", 2023-10-01T09:32:31.093 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.87570398600002, 2023-10-01T09:32:31.093 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0353429519999999, 2023-10-01T09:32:31.093 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:31.093 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:31.093 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:31.093 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:32:31.093 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:32:31.094 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129124 2023-10-01T09:32:31.094 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.094 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:31.094 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.094 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:31.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051416+0000", 2023-10-01T09:32:31.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.094 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.094 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.094 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:31.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051416+0000", 2023-10-01T09:32:31.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:31.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051416+0000", 2023-10-01T09:32:31.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:31.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:31.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051418+0000", 2023-10-01T09:32:31.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1799999999999999e-06 2023-10-01T09:32:31.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:31.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051423+0000", 2023-10-01T09:32:31.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9509999999999997e-06 2023-10-01T09:32:31.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:31.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051427+0000", 2023-10-01T09:32:31.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9609999999999999e-06 2023-10-01T09:32:31.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051553+0000", 2023-10-01T09:32:31.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000125981 2023-10-01T09:32:31.097 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.097 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051559+0000", 2023-10-01T09:32:31.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2040000000000002e-06 2023-10-01T09:32:31.097 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.097 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079137+0000", 2023-10-01T09:32:31.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0275780130000001 2023-10-01T09:32:31.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.098 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:31.098 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079280+0000", 2023-10-01T09:32:31.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014338799999999999 2023-10-01T09:32:31.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.098 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,4,6", 2023-10-01T09:32:31.098 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079462+0000", 2023-10-01T09:32:31.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018144600000000001 2023-10-01T09:32:31.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.099 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.099 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086085+0000", 2023-10-01T09:32:31.099 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066228499999999996 2023-10-01T09:32:31.099 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.099 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.099 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086145+0000", 2023-10-01T09:32:31.099 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0649999999999997e-05 2023-10-01T09:32:31.099 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.099 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:31.099 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086161+0000", 2023-10-01T09:32:31.100 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5865000000000001e-05 2023-10-01T09:32:31.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.100 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.100 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.100 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086238+0000", 2023-10-01T09:32:31.100 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6648000000000002e-05 2023-10-01T09:32:31.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.100 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.100 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.100 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086273+0000", 2023-10-01T09:32:31.100 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5553999999999999e-05 2023-10-01T09:32:31.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.101 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.101 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:31.101 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086324+0000", 2023-10-01T09:32:31.101 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0556000000000002e-05 2023-10-01T09:32:31.101 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.101 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.101 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:31.101 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086759+0000", 2023-10-01T09:32:31.101 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00043499599999999999 2023-10-01T09:32:31.101 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.101 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.101 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.102 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.102 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.102 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129131 7.c 7:3f7b5b4e:::benchmark_data_smithi138_367058_object31336:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:32:31.102 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:49.062625+0000", 2023-10-01T09:32:31.102 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 341.86449500100002, 2023-10-01T09:32:31.102 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0376771250000001, 2023-10-01T09:32:31.102 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:31.102 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:31.102 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:32:31.102 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:32:31.102 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:32:31.103 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129131 2023-10-01T09:32:31.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.103 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:31.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:31.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062625+0000", 2023-10-01T09:32:31.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:31.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062625+0000", 2023-10-01T09:32:31.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:31.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062624+0000", 2023-10-01T09:32:31.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:32:31.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:31.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062630+0000", 2023-10-01T09:32:31.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.823e-06 2023-10-01T09:32:31.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:31.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062640+0000", 2023-10-01T09:32:31.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0767999999999999e-05 2023-10-01T09:32:31.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:31.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062648+0000", 2023-10-01T09:32:31.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1199999999999996e-06 2023-10-01T09:32:31.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062697+0000", 2023-10-01T09:32:31.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.901e-05 2023-10-01T09:32:31.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.106 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:32:31.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062706+0000", 2023-10-01T09:32:31.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8449999999999998e-06 2023-10-01T09:32:31.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.106 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.082149+0000", 2023-10-01T09:32:31.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0194428250000001 2023-10-01T09:32:31.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.107 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:31.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.082294+0000", 2023-10-01T09:32:31.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014442799999999999 2023-10-01T09:32:31.107 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.107 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6,7", 2023-10-01T09:32:31.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.082448+0000", 2023-10-01T09:32:31.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015422400000000001 2023-10-01T09:32:31.107 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.107 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:32:31.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099008+0000", 2023-10-01T09:32:31.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016560099000000002 2023-10-01T09:32:31.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.108 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.108 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.108 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099072+0000", 2023-10-01T09:32:31.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3619000000000002e-05 2023-10-01T09:32:31.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.108 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.108 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.108 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099193+0000", 2023-10-01T09:32:31.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121525 2023-10-01T09:32:31.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:32:31.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099886+0000", 2023-10-01T09:32:31.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00069325300000000001 2023-10-01T09:32:31.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:31.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099947+0000", 2023-10-01T09:32:31.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0306000000000002e-05 2023-10-01T09:32:31.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.110 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:31.110 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.100302+0000", 2023-10-01T09:32:31.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00035552000000000001 2023-10-01T09:32:31.110 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.110 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.110 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.110 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.110 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:149669 8.7 e358/326 8:e840c2c7:::benchmark_data_smithi138_431617_object48258:head v 358'9349, mlcod=358'9346)", 2023-10-01T09:32:31.110 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:12.345262+0000", 2023-10-01T09:32:31.110 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.581857840000001, 2023-10-01T09:32:31.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.044421335999999999, 2023-10-01T09:32:31.111 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:32:31.111 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:31.111 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:32:31.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:32:31.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:12.345262+0000", 2023-10-01T09:32:31.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:32:31.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:12.345262+0000", 2023-10-01T09:32:31.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:32:31.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:32:31.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:12.345262+0000", 2023-10-01T09:32:31.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967296 2023-10-01T09:32:31.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:32:31.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:12.345266+0000", 2023-10-01T09:32:31.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6959999999999998e-06 2023-10-01T09:32:31.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:32:31.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:12.345273+0000", 2023-10-01T09:32:31.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3239999999999997e-06 2023-10-01T09:32:31.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:32:31.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:12.345277+0000", 2023-10-01T09:32:31.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7440000000000001e-06 2023-10-01T09:32:31.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:32:31.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:12.352884+0000", 2023-10-01T09:32:31.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0076072819999999999 2023-10-01T09:32:31.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:32:31.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:12.352907+0000", 2023-10-01T09:32:31.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3326999999999999e-05 2023-10-01T09:32:31.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:32:31.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:12.389648+0000", 2023-10-01T09:32:31.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.036740676 2023-10-01T09:32:31.115 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:32:31.115 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:32:31.115 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:32:31.115 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:12.389684+0000", 2023-10-01T09:32:31.115 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5457999999999998e-05 2023-10-01T09:32:31.115 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.115 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.115 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.115 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:32:31.115 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:32:31.115 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:32:31.125 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:31.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:31.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:31.233 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:31.261 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:31.307 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:31.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:31.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:31.413 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:31.455 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:31.502 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:31.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:31.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:31.545 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:32:31.608 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:31.693 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:31.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:31 smithi190 ceph-mon[160435]: Health check failed: Degraded data redundancy: 13/130095 objects degraded (0.010%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:32:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:31 smithi190 ceph-mon[160435]: failed to encode map e364 with expected crc 2023-10-01T09:32:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:31 smithi190 ceph-mon[160435]: failed to encode map e364 with expected crc 2023-10-01T09:32:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:31 smithi190 ceph-mon[160435]: osdmap e364: 8 total, 8 up, 8 in 2023-10-01T09:32:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:31 smithi190 ceph-mon[160435]: failed to encode map e364 with expected crc 2023-10-01T09:32:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1308365310' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:32:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:31 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:32:31.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:31 smithi138 ceph-mon[185886]: Health check failed: Degraded data redundancy: 13/130095 objects degraded (0.010%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:32:31.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:31 smithi138 ceph-mon[185886]: failed to encode map e364 with expected crc 2023-10-01T09:32:31.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:31 smithi138 ceph-mon[185886]: failed to encode map e364 with expected crc 2023-10-01T09:32:31.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:31 smithi138 ceph-mon[185886]: osdmap e364: 8 total, 8 up, 8 in 2023-10-01T09:32:31.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:31 smithi138 ceph-mon[185886]: failed to encode map e364 with expected crc 2023-10-01T09:32:31.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1308365310' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:32:31.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:31 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:32:31.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:31 smithi138 ceph-mon[192513]: Health check failed: Degraded data redundancy: 13/130095 objects degraded (0.010%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:32:31.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:31 smithi138 ceph-mon[192513]: failed to encode map e364 with expected crc 2023-10-01T09:32:31.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:31 smithi138 ceph-mon[192513]: failed to encode map e364 with expected crc 2023-10-01T09:32:31.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:31 smithi138 ceph-mon[192513]: osdmap e364: 8 total, 8 up, 8 in 2023-10-01T09:32:31.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:31 smithi138 ceph-mon[192513]: failed to encode map e364 with expected crc 2023-10-01T09:32:31.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1308365310' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:32:31.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:31 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:32:32.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:32 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:32.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:32 smithi138 ceph-mon[192513]: failed to encode map e365 with expected crc 2023-10-01T09:32:32.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:32 smithi138 ceph-mon[192513]: failed to encode map e365 with expected crc 2023-10-01T09:32:32.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:32 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:32:32.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:32 smithi138 ceph-mon[192513]: osdmap e365: 8 total, 8 up, 8 in 2023-10-01T09:32:32.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:32 smithi138 ceph-mon[192513]: failed to encode map e365 with expected crc 2023-10-01T09:32:32.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:32 smithi138 ceph-mon[192513]: pgmap v1021: 89 pgs: 1 active+remapped+backfilling, 1 activating+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 13/127173 objects degraded (0.010%); 12713/127173 objects misplaced (9.997%); 6.6 MiB/s, 2 keys/s, 111 objects/s recovering 2023-10-01T09:32:32.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:32 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:32.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:32 smithi138 ceph-mon[185886]: failed to encode map e365 with expected crc 2023-10-01T09:32:32.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:32 smithi138 ceph-mon[185886]: failed to encode map e365 with expected crc 2023-10-01T09:32:32.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:32 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:32:32.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:32 smithi138 ceph-mon[185886]: osdmap e365: 8 total, 8 up, 8 in 2023-10-01T09:32:32.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:32 smithi138 ceph-mon[185886]: failed to encode map e365 with expected crc 2023-10-01T09:32:32.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:32 smithi138 ceph-mon[185886]: pgmap v1021: 89 pgs: 1 active+remapped+backfilling, 1 activating+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 13/127173 objects degraded (0.010%); 12713/127173 objects misplaced (9.997%); 6.6 MiB/s, 2 keys/s, 111 objects/s recovering 2023-10-01T09:32:32.806 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:32:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:32 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:32 smithi190 ceph-mon[160435]: failed to encode map e365 with expected crc 2023-10-01T09:32:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:32 smithi190 ceph-mon[160435]: failed to encode map e365 with expected crc 2023-10-01T09:32:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:32 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:32:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:32 smithi190 ceph-mon[160435]: osdmap e365: 8 total, 8 up, 8 in 2023-10-01T09:32:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:32 smithi190 ceph-mon[160435]: failed to encode map e365 with expected crc 2023-10-01T09:32:32.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:32 smithi190 ceph-mon[160435]: pgmap v1021: 89 pgs: 1 active+remapped+backfilling, 1 activating+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 83 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 13/127173 objects degraded (0.010%); 12713/127173 objects misplaced (9.997%); 6.6 MiB/s, 2 keys/s, 111 objects/s recovering 2023-10-01T09:32:32.814 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:32.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:32.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:32.924 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:33.015 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:33.185 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:33.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:33.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:33.293 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:33.381 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:33.452 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:32:33.523 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:33.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:33.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:33.632 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:33.663 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:32:33.677 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:33.719 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:32:33.766 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:32:33.814 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:32:33.852 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:32:33.901 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2023-10-01T09:32:33.901 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:32:33.901 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T09:32:33.902 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2023-10-01T09:32:33.902 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 0 2023-10-01T09:32:33.904 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:32:33.939 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:32:33.964 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:33 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3782545012' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:33.964 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:33 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3782545012' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:33.973 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:32:34.031 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:32:34.031 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:32:34.031 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:32:34.031 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:32:34.031 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:32:34.031 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:32:34.031 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:32:34.031 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:32:34.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:33 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3782545012' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:34.230 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:34.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:34.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:34.333 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:34.387 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:34.839 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:32:35.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:34 smithi190 ceph-mon[160435]: pgmap v1022: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 11931/121332 objects misplaced (9.833%); 6.9 MiB/s, 1 keys/s, 117 objects/s recovering 2023-10-01T09:32:35.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:32:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:32:34] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.33.4" 2023-10-01T09:32:35.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:34 smithi138 ceph-mon[192513]: pgmap v1022: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 11931/121332 objects misplaced (9.833%); 6.9 MiB/s, 1 keys/s, 117 objects/s recovering 2023-10-01T09:32:35.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:34 smithi138 ceph-mon[185886]: pgmap v1022: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 11931/121332 objects misplaced (9.833%); 6.9 MiB/s, 1 keys/s, 117 objects/s recovering 2023-10-01T09:32:35.149 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:35.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:35.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:35.253 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:35.272 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:35.316 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:35.717 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:35.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:35.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:35.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:35 smithi190 ceph-mon[160435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/127173 objects degraded (0.010%), 2 pgs degraded) 2023-10-01T09:32:35.823 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:35.870 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:35.889 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:35.950 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:35.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:35.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:36.023 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:35 smithi138 ceph-mon[192513]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/127173 objects degraded (0.010%), 2 pgs degraded) 2023-10-01T09:32:36.023 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:35 smithi138 ceph-mon[185886]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/127173 objects degraded (0.010%), 2 pgs degraded) 2023-10-01T09:32:36.054 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:36.125 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:36.630 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:36.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:36.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:36.739 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:36.765 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:36.809 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:36.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:36.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:36.890 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:36 smithi138 ceph-mon[192513]: pgmap v1023: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11720/118236 objects misplaced (9.912%); 5.9 MiB/s, 1 keys/s, 99 objects/s recovering 2023-10-01T09:32:36.890 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:36 smithi138 ceph-mon[185886]: pgmap v1023: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11720/118236 objects misplaced (9.912%); 5.9 MiB/s, 1 keys/s, 99 objects/s recovering 2023-10-01T09:32:36.912 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:32:36.939 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:37.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:36 smithi190 ceph-mon[160435]: pgmap v1023: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 11720/118236 objects misplaced (9.912%); 5.9 MiB/s, 1 keys/s, 99 objects/s recovering 2023-10-01T09:32:37.086 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:32:37.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:37.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:32:37.120 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:32:37.191 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:37.276 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:37.453 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:37.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:37.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:37.563 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:32:37.623 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:37.885 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:32:37.885 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:32:38.013 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:32:38.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:38.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:32:38.118 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:38.194 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:38.309 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:38.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:38.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:38.398 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:38.436 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:38.448 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1108551386' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-01T09:32:38.448 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:38 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-01T09:32:38.448 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:38 smithi138 ceph-mon[185886]: pgmap v1024: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 10486/117708 objects misplaced (8.908%); 14 MiB/s, 1 keys/s, 221 objects/s recovering 2023-10-01T09:32:38.454 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.4 primary-affinity to 0 (802) 2023-10-01T09:32:38.493 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:38.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1108551386' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-01T09:32:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:38 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-01T09:32:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:38 smithi190 ceph-mon[160435]: pgmap v1024: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 10486/117708 objects misplaced (8.908%); 14 MiB/s, 1 keys/s, 221 objects/s recovering 2023-10-01T09:32:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1108551386' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-01T09:32:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:38 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-10-01T09:32:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:38 smithi138 ceph-mon[192513]: pgmap v1024: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 10486/117708 objects misplaced (8.908%); 14 MiB/s, 1 keys/s, 221 objects/s recovering 2023-10-01T09:32:38.843 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:32:39.424 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:32:39.528 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:39.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:39.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:39.638 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:32:39.711 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:39 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-01T09:32:39.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:39 smithi190 ceph-mon[160435]: osdmap e366: 8 total, 8 up, 8 in 2023-10-01T09:32:39.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:39 smithi190 ceph-mon[160435]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:39 smithi190 ceph-mon[160435]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:39 smithi190 ceph-mon[160435]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:39 smithi190 ceph-mon[160435]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:39 smithi190 ceph-mon[160435]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:39 smithi190 ceph-mon[160435]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:39 smithi190 ceph-mon[160435]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:39 smithi190 ceph-mon[160435]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:39 smithi190 ceph-mon[160435]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.713 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-01T09:32:39.713 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[185886]: osdmap e366: 8 total, 8 up, 8 in 2023-10-01T09:32:39.713 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[185886]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.713 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[185886]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.713 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[185886]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.713 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[185886]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.713 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[185886]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.714 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[185886]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.714 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[185886]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.714 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[185886]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.714 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[185886]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-10-01T09:32:39.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[192513]: osdmap e366: 8 total, 8 up, 8 in 2023-10-01T09:32:39.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[192513]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[192513]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[192513]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.715 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[192513]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.715 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[192513]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.715 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[192513]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.715 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[192513]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.715 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[192513]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.715 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:39 smithi138 ceph-mon[192513]: failed to encode map e366 with expected crc 2023-10-01T09:32:39.734 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:39.856 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:32:39.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:39.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:32:39.966 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:32:40.021 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:40.126 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:32:40.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:40.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:32:40.232 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:40.273 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:40.320 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:40.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:40.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:40.423 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:40.452 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:40.463 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:32:40.497 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:40.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:40.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:40.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:40 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1367709618' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:40.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:40 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:40.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:40 smithi138 ceph-mon[192513]: pgmap v1026: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 9930/111969 objects misplaced (8.869%); 13 MiB/s, 1 keys/s, 208 objects/s recovering 2023-10-01T09:32:40.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:40.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:32:40.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:40 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1367709618' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:40.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:40 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:40.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:40 smithi138 ceph-mon[185886]: pgmap v1026: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 9930/111969 objects misplaced (8.869%); 13 MiB/s, 1 keys/s, 208 objects/s recovering 2023-10-01T09:32:40.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:40.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:32:40.600 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:40.639 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:40 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1367709618' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:40 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:40 smithi190 ceph-mon[160435]: pgmap v1026: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 9930/111969 objects misplaced (8.869%); 13 MiB/s, 1 keys/s, 208 objects/s recovering 2023-10-01T09:32:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:32:40.906 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:40.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:40.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:41.009 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:41.036 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:41.383 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:41.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:41.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:41.447 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:32:41.447 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:32:41.486 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:41.526 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:41.726 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:41 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:41.726 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:41 smithi138 ceph-mon[192513]: failed to encode map e367 with expected crc 2023-10-01T09:32:41.726 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:41 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:32:41.726 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:41 smithi138 ceph-mon[192513]: osdmap e367: 8 total, 8 up, 8 in 2023-10-01T09:32:41.726 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:41 smithi138 ceph-mon[192513]: failed to encode map e367 with expected crc 2023-10-01T09:32:41.727 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:41.727 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:41 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:41.727 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:41 smithi138 ceph-mon[185886]: failed to encode map e367 with expected crc 2023-10-01T09:32:41.727 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:41 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:32:41.727 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:41 smithi138 ceph-mon[185886]: osdmap e367: 8 total, 8 up, 8 in 2023-10-01T09:32:41.727 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:41 smithi138 ceph-mon[185886]: failed to encode map e367 with expected crc 2023-10-01T09:32:41.727 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:41.730 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:41.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:41.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:41.744 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:41 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:41.744 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:41 smithi190 ceph-mon[160435]: failed to encode map e367 with expected crc 2023-10-01T09:32:41.744 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:41 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:32:41.744 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:41 smithi190 ceph-mon[160435]: osdmap e367: 8 total, 8 up, 8 in 2023-10-01T09:32:41.744 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:41 smithi190 ceph-mon[160435]: failed to encode map e367 with expected crc 2023-10-01T09:32:41.744 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:41.844 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:41.936 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:42.011 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:32:42.040 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:42.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:42.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:42.066 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:32:42.090 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:32:42.109 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:32:42.152 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:42.179 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:32:42.230 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:32:42.236 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:42.267 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:32:42.300 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:32:42.311 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:32:42.312 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:32:42.312 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:32:42.312 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:32:42.312 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:32:42.312 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:32:42.312 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:32:42.312 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:32:42.353 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:42.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:42.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:42.459 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:42.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:42 smithi138 ceph-mon[192513]: failed to encode map e368 with expected crc 2023-10-01T09:32:42.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:42 smithi138 ceph-mon[192513]: failed to encode map e368 with expected crc 2023-10-01T09:32:42.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:42 smithi138 ceph-mon[192513]: osdmap e368: 8 total, 8 up, 8 in 2023-10-01T09:32:42.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:42 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2962612248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:42.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:42 smithi138 ceph-mon[192513]: pgmap v1029: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 8802/109008 objects misplaced (8.075%); 22 MiB/s, 345 objects/s recovering 2023-10-01T09:32:42.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:42 smithi138 ceph-mon[185886]: failed to encode map e368 with expected crc 2023-10-01T09:32:42.480 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:42 smithi138 ceph-mon[185886]: failed to encode map e368 with expected crc 2023-10-01T09:32:42.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:42 smithi138 ceph-mon[185886]: osdmap e368: 8 total, 8 up, 8 in 2023-10-01T09:32:42.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:42 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2962612248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:42.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:42 smithi138 ceph-mon[185886]: pgmap v1029: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 8802/109008 objects misplaced (8.075%); 22 MiB/s, 345 objects/s recovering 2023-10-01T09:32:42.527 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:32:42.548 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:42.624 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:42.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:42.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:42.728 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:42.754 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:42.809 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:42.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:42 smithi190 ceph-mon[160435]: failed to encode map e368 with expected crc 2023-10-01T09:32:42.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:42 smithi190 ceph-mon[160435]: failed to encode map e368 with expected crc 2023-10-01T09:32:42.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:42 smithi190 ceph-mon[160435]: osdmap e368: 8 total, 8 up, 8 in 2023-10-01T09:32:42.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:42 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2962612248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:42.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:42 smithi190 ceph-mon[160435]: pgmap v1029: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 8802/109008 objects misplaced (8.075%); 22 MiB/s, 345 objects/s recovering 2023-10-01T09:32:42.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:42.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:42.921 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:32:42.990 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:43.160 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:32:43.190 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:32:43.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:43.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:32:43.305 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:43.381 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:43.480 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:43.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:43.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:43.549 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:32:43.586 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:43.696 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:43.844 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:43.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:43.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:43.959 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:32:44.016 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:44.106 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:32:44.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:44.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:32:44.190 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:44.209 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:44.237 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:44.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:44 smithi190 ceph-mon[160435]: pgmap v1030: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 8316/104220 objects misplaced (7.979%); 9.9 MiB/s, 158 objects/s recovering 2023-10-01T09:32:44.396 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:44.404 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:44 smithi138 ceph-mon[192513]: pgmap v1030: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 8316/104220 objects misplaced (7.979%); 9.9 MiB/s, 158 objects/s recovering 2023-10-01T09:32:44.404 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:44 smithi138 ceph-mon[185886]: pgmap v1030: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 85 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 8316/104220 objects misplaced (7.979%); 9.9 MiB/s, 158 objects/s recovering 2023-10-01T09:32:44.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:44.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:44.511 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:44.574 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 5, 6] 2023-10-01T09:32:44.574 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:32:44.574 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [0, 1, 2, 3, 4, 7, 5, 6] 2023-10-01T09:32:44.574 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-01T09:32:44.574 DEBUG:teuthology.orchestra.run.smithi190:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.5 2023-10-01T09:32:44.582 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:44.890 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:44.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:44.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:44.980 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:44 smithi190 systemd[1]: Stopping Ceph osd.5 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:32:44.995 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:45.039 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:45.080 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:45.082 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:32:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:32:44] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.33.4" 2023-10-01T09:32:45.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:45.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:45.183 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:45.189 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:32:45.219 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:45.241 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:32:45.241 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:32:45.241 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:32:45.241 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:32:45.241 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.241 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:95124 7.1 e271/269 7:84c3d6a9:::benchmark_data_smithi138_367058_object95123:head v 271'5868, mlcod=271'5866)", 2023-10-01T09:32:45.242 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:25:20.262562+0000", 2023-10-01T09:32:45.242 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 444.97527640999999, 2023-10-01T09:32:45.242 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.079727882999999999, 2023-10-01T09:32:45.242 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.242 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.242 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.242 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.242 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.242 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.262562+0000", 2023-10-01T09:32:45.242 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.242 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.243 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.243 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.243 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.262562+0000", 2023-10-01T09:32:45.243 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:45.243 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.243 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.243 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.243 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.262561+0000", 2023-10-01T09:32:45.243 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5702000000000003e-05 2023-10-01T09:32:45.243 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.243 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.243 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.244 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.262617+0000", 2023-10-01T09:32:45.244 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0469999999999994e-06 2023-10-01T09:32:45.244 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.244 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.244 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.244 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.262625+0000", 2023-10-01T09:32:45.244 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4139999999999996e-06 2023-10-01T09:32:45.244 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.244 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.244 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.244 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.262629+0000", 2023-10-01T09:32:45.245 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.070540481000000002 2023-10-01T09:32:45.245 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.245 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.245 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.245 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.333170+0000", 2023-10-01T09:32:45.245 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1875e-05 2023-10-01T09:32:45.245 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.245 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.245 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.245 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.333231+0000", 2023-10-01T09:32:45.245 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0090221510000000008 2023-10-01T09:32:45.245 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.246 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.246 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.246 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.342254+0000", 2023-10-01T09:32:45.246 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5887999999999999e-05 2023-10-01T09:32:45.246 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.246 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.246 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.246 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:25:20.342289+0000", 2023-10-01T09:32:45.246 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.079727882999999999 2023-10-01T09:32:45.246 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.246 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.246 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.247 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.247 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.247 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:167748 7.1 e309/307 7:8e571860:::benchmark_data_smithi138_367058_object69953:head v 309'10359, mlcod=270'5583)", 2023-10-01T09:32:45.247 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:11.590318+0000", 2023-10-01T09:32:45.247 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 273.64751954100001, 2023-10-01T09:32:45.247 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017835182000000002, 2023-10-01T09:32:45.247 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.247 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.247 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.247 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.247 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.248 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590318+0000", 2023-10-01T09:32:45.248 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.248 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.248 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.248 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.248 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590318+0000", 2023-10-01T09:32:45.248 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:32:45.248 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.248 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.248 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.248 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590317+0000", 2023-10-01T09:32:45.248 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7569000000000001e-05 2023-10-01T09:32:45.249 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.249 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.249 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.249 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590334+0000", 2023-10-01T09:32:45.249 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8146000000000003e-05 2023-10-01T09:32:45.249 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.249 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.249 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.249 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590382+0000", 2023-10-01T09:32:45.249 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9840000000000001e-06 2023-10-01T09:32:45.249 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.249 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.250 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.250 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590390+0000", 2023-10-01T09:32:45.250 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003199726 2023-10-01T09:32:45.250 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.250 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.250 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.250 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.593590+0000", 2023-10-01T09:32:45.250 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9781000000000002e-05 2023-10-01T09:32:45.250 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.250 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.250 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.250 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.593620+0000", 2023-10-01T09:32:45.251 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014491779999999999 2023-10-01T09:32:45.251 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.251 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.251 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.251 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.608112+0000", 2023-10-01T09:32:45.251 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1876999999999999e-05 2023-10-01T09:32:45.251 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.251 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.251 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.251 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.608154+0000", 2023-10-01T09:32:45.251 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017835182000000002 2023-10-01T09:32:45.251 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.252 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.252 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.252 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.252 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.252 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:167751 7.1 e309/307 7:87025a43:::benchmark_data_smithi138_367058_object69956:head v 309'10360, mlcod=270'5583)", 2023-10-01T09:32:45.252 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:11.590423+0000", 2023-10-01T09:32:45.252 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 273.64741483300003, 2023-10-01T09:32:45.252 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017751335, 2023-10-01T09:32:45.252 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.252 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.252 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.252 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.253 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.253 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590423+0000", 2023-10-01T09:32:45.253 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.253 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.253 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.253 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.253 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590423+0000", 2023-10-01T09:32:45.253 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:45.253 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.253 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.253 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.253 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590423+0000", 2023-10-01T09:32:45.254 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0379999999999996e-06 2023-10-01T09:32:45.254 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.254 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.254 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.254 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590429+0000", 2023-10-01T09:32:45.254 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8630000000000002e-06 2023-10-01T09:32:45.254 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.254 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.254 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.254 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590436+0000", 2023-10-01T09:32:45.254 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4360000000000004e-06 2023-10-01T09:32:45.254 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.255 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.255 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.590440+0000", 2023-10-01T09:32:45.255 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0063771230000000002 2023-10-01T09:32:45.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.255 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.255 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.596817+0000", 2023-10-01T09:32:45.255 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5887000000000001e-05 2023-10-01T09:32:45.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.256 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.256 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.596843+0000", 2023-10-01T09:32:45.256 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011313986 2023-10-01T09:32:45.256 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.256 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.256 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.256 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.608157+0000", 2023-10-01T09:32:45.256 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7524e-05 2023-10-01T09:32:45.256 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.256 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.256 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.256 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:11.608174+0000", 2023-10-01T09:32:45.257 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017751335 2023-10-01T09:32:45.257 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.257 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.257 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.257 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.257 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.257 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.35421.0:192136 7.4 e317/306 7:23280b10:::benchmark_data_smithi138_367058_object94341:head v 317'12172, mlcod=317'12169)", 2023-10-01T09:32:45.257 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:28:53.176657+0000", 2023-10-01T09:32:45.257 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 232.06118128400001, 2023-10-01T09:32:45.257 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017663564999999999, 2023-10-01T09:32:45.257 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.257 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.258 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.258 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.258 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.258 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.176657+0000", 2023-10-01T09:32:45.258 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.258 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.258 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.258 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.258 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.176657+0000", 2023-10-01T09:32:45.258 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:45.258 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.259 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.259 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.259 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.176656+0000", 2023-10-01T09:32:45.259 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8289999999999998e-06 2023-10-01T09:32:45.259 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.259 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.259 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.259 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.176661+0000", 2023-10-01T09:32:45.259 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2320000000000001e-06 2023-10-01T09:32:45.259 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.259 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.260 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.260 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.176666+0000", 2023-10-01T09:32:45.260 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.467e-06 2023-10-01T09:32:45.260 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.260 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.260 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.260 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.176670+0000", 2023-10-01T09:32:45.260 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0054083050000000004 2023-10-01T09:32:45.260 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.260 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.260 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.260 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.182078+0000", 2023-10-01T09:32:45.261 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2277e-05 2023-10-01T09:32:45.261 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.261 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.261 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.261 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.182100+0000", 2023-10-01T09:32:45.261 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012181777 2023-10-01T09:32:45.261 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.261 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.261 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.261 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.194282+0000", 2023-10-01T09:32:45.261 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8040000000000002e-05 2023-10-01T09:32:45.261 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.262 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.262 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.262 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:28:53.194320+0000", 2023-10-01T09:32:45.262 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017663564999999999 2023-10-01T09:32:45.262 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.262 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.262 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.262 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.262 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.262 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:159843 8.4 e362/361 8:2a7bc9e3:::benchmark_data_smithi138_431617_object58432:head v 362'9883, mlcod=0'0)", 2023-10-01T09:32:45.262 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:28.682390+0000", 2023-10-01T09:32:45.263 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 16.555447877999999, 2023-10-01T09:32:45.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019058354999999999, 2023-10-01T09:32:45.263 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.263 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.263 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.263 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.263 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.263 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.682390+0000", 2023-10-01T09:32:45.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.263 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.263 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.263 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.264 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.682390+0000", 2023-10-01T09:32:45.264 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:32:45.264 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.264 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.264 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.264 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.682389+0000", 2023-10-01T09:32:45.264 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.9439999999999994e-06 2023-10-01T09:32:45.264 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.264 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.264 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.264 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.682398+0000", 2023-10-01T09:32:45.264 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1326e-05 2023-10-01T09:32:45.265 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.265 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.265 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.265 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.682409+0000", 2023-10-01T09:32:45.265 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7179999999999999e-06 2023-10-01T09:32:45.265 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.265 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.265 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.265 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.682416+0000", 2023-10-01T09:32:45.265 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2339000000000002e-05 2023-10-01T09:32:45.265 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.266 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.266 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.266 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.682458+0000", 2023-10-01T09:32:45.266 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2495000000000001e-05 2023-10-01T09:32:45.266 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.266 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.266 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.266 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.682481+0000", 2023-10-01T09:32:45.266 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018931491000000002 2023-10-01T09:32:45.266 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.266 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.266 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.267 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.701412+0000", 2023-10-01T09:32:45.267 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6015e-05 2023-10-01T09:32:45.267 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.267 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.267 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.267 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.701448+0000", 2023-10-01T09:32:45.267 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019058354999999999 2023-10-01T09:32:45.267 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.267 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.267 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.267 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.268 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.268 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:159846 8.4 e362/361 8:2d514052:::benchmark_data_smithi138_431617_object58435:head v 362'9884, mlcod=0'0)", 2023-10-01T09:32:45.268 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:28.682431+0000", 2023-10-01T09:32:45.268 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 16.555407365000001, 2023-10-01T09:32:45.268 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019041224999999998, 2023-10-01T09:32:45.268 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.268 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.268 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.268 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.268 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.268 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.682431+0000", 2023-10-01T09:32:45.269 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.269 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.269 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.682431+0000", 2023-10-01T09:32:45.269 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:45.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.269 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.269 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.682430+0000", 2023-10-01T09:32:45.269 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7190000000000001e-06 2023-10-01T09:32:45.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.270 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.270 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.682435+0000", 2023-10-01T09:32:45.270 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2020000000000001e-06 2023-10-01T09:32:45.270 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.270 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.270 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.682441+0000", 2023-10-01T09:32:45.270 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3869999999999997e-06 2023-10-01T09:32:45.270 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.682446+0000", 2023-10-01T09:32:45.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0040097910000000004 2023-10-01T09:32:45.271 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.271 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.686455+0000", 2023-10-01T09:32:45.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2328e-05 2023-10-01T09:32:45.271 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.271 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.686478+0000", 2023-10-01T09:32:45.272 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014974612 2023-10-01T09:32:45.272 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.272 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.272 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.272 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.701452+0000", 2023-10-01T09:32:45.272 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9584000000000001e-05 2023-10-01T09:32:45.272 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.272 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.272 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.272 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.701472+0000", 2023-10-01T09:32:45.272 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019041224999999998 2023-10-01T09:32:45.272 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.273 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.273 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.273 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:160540 8.b e363/361 8:d4056573:::benchmark_data_smithi138_431617_object59129:head v 363'9991, mlcod=363'9989)", 2023-10-01T09:32:45.273 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:29.603563+0000", 2023-10-01T09:32:45.273 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 15.634274983999999, 2023-10-01T09:32:45.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018795975999999999, 2023-10-01T09:32:45.273 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.273 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.273 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:29.603563+0000", 2023-10-01T09:32:45.274 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:29.603563+0000", 2023-10-01T09:32:45.274 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:45.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.275 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.275 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:29.603562+0000", 2023-10-01T09:32:45.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9550000000000003e-06 2023-10-01T09:32:45.275 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.275 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.275 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.275 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:29.603569+0000", 2023-10-01T09:32:45.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4739999999999994e-06 2023-10-01T09:32:45.275 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.275 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.275 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.275 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:29.603579+0000", 2023-10-01T09:32:45.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3739999999999997e-06 2023-10-01T09:32:45.276 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.276 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.276 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.276 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:29.603584+0000", 2023-10-01T09:32:45.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7874000000000003e-05 2023-10-01T09:32:45.276 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.276 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.276 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.276 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:29.603632+0000", 2023-10-01T09:32:45.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7859e-05 2023-10-01T09:32:45.276 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.277 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.277 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.277 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:29.603650+0000", 2023-10-01T09:32:45.277 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018682537999999999 2023-10-01T09:32:45.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.277 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.277 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.277 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:29.622332+0000", 2023-10-01T09:32:45.277 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6571e-05 2023-10-01T09:32:45.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.277 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.277 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.278 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:29.622359+0000", 2023-10-01T09:32:45.278 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018795975999999999 2023-10-01T09:32:45.278 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.278 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.278 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.278 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.278 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.278 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:161081 8.5 e363/361 8:aa866e2e:::benchmark_data_smithi138_431617_object59670:head v 363'10204, mlcod=363'10202)", 2023-10-01T09:32:45.278 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:30.392423+0000", 2023-10-01T09:32:45.278 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 14.845415269, 2023-10-01T09:32:45.278 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017871768999999999, 2023-10-01T09:32:45.279 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.279 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.279 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.279 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.279 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:30.392423+0000", 2023-10-01T09:32:45.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.279 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.279 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.279 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:30.392423+0000", 2023-10-01T09:32:45.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:32:45.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:30.392421+0000", 2023-10-01T09:32:45.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2669e-05 2023-10-01T09:32:45.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:30.392434+0000", 2023-10-01T09:32:45.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1648999999999999e-05 2023-10-01T09:32:45.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:30.392456+0000", 2023-10-01T09:32:45.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0080000000000002e-06 2023-10-01T09:32:45.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:30.392464+0000", 2023-10-01T09:32:45.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5508999999999998e-05 2023-10-01T09:32:45.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:30.392539+0000", 2023-10-01T09:32:45.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2337999999999999e-05 2023-10-01T09:32:45.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.282 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:30.392561+0000", 2023-10-01T09:32:45.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017695313000000001 2023-10-01T09:32:45.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.282 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:30.410257+0000", 2023-10-01T09:32:45.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7741000000000003e-05 2023-10-01T09:32:45.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.283 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.283 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:30.410294+0000", 2023-10-01T09:32:45.283 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017871768999999999 2023-10-01T09:32:45.283 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.283 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.283 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.283 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:162084 8.2 e365/361 8:4f4d8e7e:::benchmark_data_smithi138_431617_object60673:head v 365'10003, mlcod=365'9999)", 2023-10-01T09:32:45.283 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:32.110858+0000", 2023-10-01T09:32:45.283 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 13.126980211999999, 2023-10-01T09:32:45.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.024562178, 2023-10-01T09:32:45.284 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.284 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.284 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.110858+0000", 2023-10-01T09:32:45.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.284 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.110858+0000", 2023-10-01T09:32:45.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:32:45.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.285 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.285 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.110856+0000", 2023-10-01T09:32:45.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0988000000000001e-05 2023-10-01T09:32:45.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.285 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.285 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.110867+0000", 2023-10-01T09:32:45.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2289e-05 2023-10-01T09:32:45.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.286 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.286 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.286 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.110880+0000", 2023-10-01T09:32:45.286 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1849999999999998e-06 2023-10-01T09:32:45.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.286 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.286 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.286 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.110887+0000", 2023-10-01T09:32:45.286 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0061008470000000004 2023-10-01T09:32:45.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.286 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.287 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.116988+0000", 2023-10-01T09:32:45.287 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0709000000000001e-05 2023-10-01T09:32:45.287 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.287 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.287 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.117008+0000", 2023-10-01T09:32:45.287 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018360004999999999 2023-10-01T09:32:45.287 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.287 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.287 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.135368+0000", 2023-10-01T09:32:45.288 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1480999999999997e-05 2023-10-01T09:32:45.288 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.288 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.288 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.135420+0000", 2023-10-01T09:32:45.288 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.024562178 2023-10-01T09:32:45.288 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.288 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.288 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.288 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.288 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:162415 8.5 e365/361 8:a761002f:::benchmark_data_smithi138_431617_object61004:head v 365'10308, mlcod=365'10305)", 2023-10-01T09:32:45.289 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:32.605164+0000", 2023-10-01T09:32:45.289 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 12.632673759999999, 2023-10-01T09:32:45.289 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020216728, 2023-10-01T09:32:45.289 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.289 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.289 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.289 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.289 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.289 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.605164+0000", 2023-10-01T09:32:45.289 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.289 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.289 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.605164+0000", 2023-10-01T09:32:45.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:32:45.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.605163+0000", 2023-10-01T09:32:45.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5070000000000003e-06 2023-10-01T09:32:45.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.605171+0000", 2023-10-01T09:32:45.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0611999999999999e-05 2023-10-01T09:32:45.291 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.291 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.291 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.605182+0000", 2023-10-01T09:32:45.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0249999999999999e-06 2023-10-01T09:32:45.291 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.291 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.291 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.605188+0000", 2023-10-01T09:32:45.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024395609999999998 2023-10-01T09:32:45.291 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.292 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.292 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.292 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.607627+0000", 2023-10-01T09:32:45.292 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5556000000000001e-05 2023-10-01T09:32:45.292 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.292 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.292 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.292 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.607653+0000", 2023-10-01T09:32:45.292 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017681017 2023-10-01T09:32:45.292 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.293 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.293 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.625334+0000", 2023-10-01T09:32:45.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7185000000000003e-05 2023-10-01T09:32:45.293 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.293 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.293 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.625381+0000", 2023-10-01T09:32:45.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020216728 2023-10-01T09:32:45.293 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.293 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.293 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.294 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:162684 8.4 e365/361 8:25f69c37:::benchmark_data_smithi138_431617_object61273:head v 365'10073, mlcod=365'10069)", 2023-10-01T09:32:45.294 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:32.994780+0000", 2023-10-01T09:32:45.294 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 12.24305829, 2023-10-01T09:32:45.294 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019085318, 2023-10-01T09:32:45.294 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.294 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.294 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.294 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.295 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.994780+0000", 2023-10-01T09:32:45.295 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.297 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.297 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.994780+0000", 2023-10-01T09:32:45.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:32:45.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.297 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.298 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.994779+0000", 2023-10-01T09:32:45.298 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0329999999999999e-05 2023-10-01T09:32:45.298 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.298 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.298 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.298 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.994789+0000", 2023-10-01T09:32:45.298 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2510000000000001e-05 2023-10-01T09:32:45.298 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.298 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.298 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.298 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.994802+0000", 2023-10-01T09:32:45.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3929999999999996e-06 2023-10-01T09:32:45.299 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.299 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.299 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.299 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.994808+0000", 2023-10-01T09:32:45.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00063574599999999997 2023-10-01T09:32:45.299 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.299 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.299 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.299 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.995444+0000", 2023-10-01T09:32:45.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0256999999999999e-05 2023-10-01T09:32:45.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.300 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.301 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:32.995464+0000", 2023-10-01T09:32:45.301 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018355831999999999 2023-10-01T09:32:45.301 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.301 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.013820+0000", 2023-10-01T09:32:45.301 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5089000000000001e-05 2023-10-01T09:32:45.301 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.301 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.013865+0000", 2023-10-01T09:32:45.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019085318 2023-10-01T09:32:45.302 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.302 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.302 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.302 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.302 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.302 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:162760 8.5 e365/361 8:a70ae832:::benchmark_data_smithi138_431617_object61349:head v 365'10330, mlcod=365'10326)", 2023-10-01T09:32:45.302 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:33.096168+0000", 2023-10-01T09:32:45.302 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 12.141669614, 2023-10-01T09:32:45.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021266145, 2023-10-01T09:32:45.302 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.303 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.303 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.303 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.096168+0000", 2023-10-01T09:32:45.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.303 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.096168+0000", 2023-10-01T09:32:45.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967296 2023-10-01T09:32:45.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.096168+0000", 2023-10-01T09:32:45.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1499999999999999e-06 2023-10-01T09:32:45.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.096171+0000", 2023-10-01T09:32:45.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3370000000000001e-06 2023-10-01T09:32:45.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.096176+0000", 2023-10-01T09:32:45.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8700000000000002e-06 2023-10-01T09:32:45.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.096180+0000", 2023-10-01T09:32:45.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0063064039999999998 2023-10-01T09:32:45.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.102486+0000", 2023-10-01T09:32:45.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9145000000000001e-05 2023-10-01T09:32:45.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.102505+0000", 2023-10-01T09:32:45.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0148682 2023-10-01T09:32:45.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.117373+0000", 2023-10-01T09:32:45.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1198999999999995e-05 2023-10-01T09:32:45.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.117435+0000", 2023-10-01T09:32:45.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021266145 2023-10-01T09:32:45.307 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.307 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.307 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.308 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:163055 8.4 e365/361 8:2381f5ed:::benchmark_data_smithi138_431617_object61644:head v 365'10102, mlcod=365'10098)", 2023-10-01T09:32:45.308 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:33.532754+0000", 2023-10-01T09:32:45.308 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 11.705084296999999, 2023-10-01T09:32:45.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021060342999999999, 2023-10-01T09:32:45.308 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.308 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.308 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.532754+0000", 2023-10-01T09:32:45.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.532754+0000", 2023-10-01T09:32:45.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:32:45.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.532752+0000", 2023-10-01T09:32:45.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9119999999999993e-06 2023-10-01T09:32:45.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.310 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.310 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.532762+0000", 2023-10-01T09:32:45.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4076e-05 2023-10-01T09:32:45.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.310 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.310 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.532776+0000", 2023-10-01T09:32:45.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9929999999999998e-06 2023-10-01T09:32:45.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.310 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.532783+0000", 2023-10-01T09:32:45.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8801999999999996e-05 2023-10-01T09:32:45.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.532852+0000", 2023-10-01T09:32:45.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2751e-05 2023-10-01T09:32:45.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.532875+0000", 2023-10-01T09:32:45.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020909003999999998 2023-10-01T09:32:45.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.553784+0000", 2023-10-01T09:32:45.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0026999999999999e-05 2023-10-01T09:32:45.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.553814+0000", 2023-10-01T09:32:45.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021060342999999999 2023-10-01T09:32:45.312 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.313 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.313 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.313 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:163063 8.4 e365/361 8:27edc2fc:::benchmark_data_smithi138_431617_object61652:head v 365'10104, mlcod=365'10098)", 2023-10-01T09:32:45.313 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:33.542441+0000", 2023-10-01T09:32:45.313 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 11.695396922, 2023-10-01T09:32:45.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019425075999999999, 2023-10-01T09:32:45.313 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.313 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.313 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.542441+0000", 2023-10-01T09:32:45.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.542441+0000", 2023-10-01T09:32:45.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:45.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.542440+0000", 2023-10-01T09:32:45.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.81e-06 2023-10-01T09:32:45.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.542447+0000", 2023-10-01T09:32:45.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.8089999999999999e-06 2023-10-01T09:32:45.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.542456+0000", 2023-10-01T09:32:45.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9409999999999998e-06 2023-10-01T09:32:45.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.542461+0000", 2023-10-01T09:32:45.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0077105460000000004 2023-10-01T09:32:45.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.550172+0000", 2023-10-01T09:32:45.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9913000000000001e-05 2023-10-01T09:32:45.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.550191+0000", 2023-10-01T09:32:45.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.01163177 2023-10-01T09:32:45.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.561823+0000", 2023-10-01T09:32:45.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2948000000000003e-05 2023-10-01T09:32:45.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.318 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.318 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.318 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:33.561866+0000", 2023-10-01T09:32:45.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019425075999999999 2023-10-01T09:32:45.318 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.318 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.318 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.319 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:168056 8.4 e368/361 8:2239cc6c:::benchmark_data_smithi138_431617_object66645:head v 368'10400, mlcod=368'10398)", 2023-10-01T09:32:45.319 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:42.826615+0000", 2023-10-01T09:32:45.319 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 2.4112229379999999, 2023-10-01T09:32:45.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019283046000000002, 2023-10-01T09:32:45.319 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.319 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.319 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.319 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.320 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.826615+0000", 2023-10-01T09:32:45.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.320 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.320 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.320 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.826615+0000", 2023-10-01T09:32:45.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:45.320 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.826614+0000", 2023-10-01T09:32:45.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2139999999999999e-06 2023-10-01T09:32:45.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.826622+0000", 2023-10-01T09:32:45.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9459999999999993e-06 2023-10-01T09:32:45.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.826632+0000", 2023-10-01T09:32:45.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3790000000000003e-06 2023-10-01T09:32:45.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.826638+0000", 2023-10-01T09:32:45.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3018e-05 2023-10-01T09:32:45.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.826681+0000", 2023-10-01T09:32:45.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8074e-05 2023-10-01T09:32:45.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.826699+0000", 2023-10-01T09:32:45.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019152022000000001 2023-10-01T09:32:45.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.845851+0000", 2023-10-01T09:32:45.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7086999999999998e-05 2023-10-01T09:32:45.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.845898+0000", 2023-10-01T09:32:45.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019283046000000002 2023-10-01T09:32:45.324 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.324 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.324 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.325 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:168077 8.2 e368/361 8:498f03de:::benchmark_data_smithi138_431617_object66666:head v 368'10386, mlcod=368'10383)", 2023-10-01T09:32:45.325 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:42.849542+0000", 2023-10-01T09:32:45.325 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 2.3882959709999998, 2023-10-01T09:32:45.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019865615, 2023-10-01T09:32:45.325 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.325 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.325 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.325 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.849542+0000", 2023-10-01T09:32:45.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.326 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.326 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.849542+0000", 2023-10-01T09:32:45.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:32:45.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.326 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.326 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.849541+0000", 2023-10-01T09:32:45.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6290000000000005e-06 2023-10-01T09:32:45.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.327 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.327 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.849551+0000", 2023-10-01T09:32:45.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2208999999999999e-05 2023-10-01T09:32:45.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.327 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.327 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.849563+0000", 2023-10-01T09:32:45.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7689999999999999e-06 2023-10-01T09:32:45.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.328 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.849570+0000", 2023-10-01T09:32:45.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3639999999999994e-05 2023-10-01T09:32:45.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.328 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.849633+0000", 2023-10-01T09:32:45.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.022e-05 2023-10-01T09:32:45.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.849654+0000", 2023-10-01T09:32:45.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019708755000000001 2023-10-01T09:32:45.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.869362+0000", 2023-10-01T09:32:45.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5241000000000003e-05 2023-10-01T09:32:45.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:42.869408+0000", 2023-10-01T09:32:45.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019865615 2023-10-01T09:32:45.330 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.330 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.330 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.330 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:168231 8.4 e368/361 8:278336f9:::benchmark_data_smithi138_431617_object66820:head v 368'10412, mlcod=368'10409)", 2023-10-01T09:32:45.330 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:43.065712+0000", 2023-10-01T09:32:45.330 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 2.1721263940000002, 2023-10-01T09:32:45.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.023429388999999998, 2023-10-01T09:32:45.331 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.331 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.331 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:43.065712+0000", 2023-10-01T09:32:45.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:43.065712+0000", 2023-10-01T09:32:45.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:45.332 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:43.065711+0000", 2023-10-01T09:32:45.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3780000000000003e-06 2023-10-01T09:32:45.332 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:43.065716+0000", 2023-10-01T09:32:45.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3901e-05 2023-10-01T09:32:45.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.333 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:43.065730+0000", 2023-10-01T09:32:45.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5410000000000002e-06 2023-10-01T09:32:45.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.333 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:43.065734+0000", 2023-10-01T09:32:45.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038218589999999999 2023-10-01T09:32:45.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.334 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:43.069556+0000", 2023-10-01T09:32:45.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9928e-05 2023-10-01T09:32:45.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.334 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:43.069576+0000", 2023-10-01T09:32:45.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019532266999999999 2023-10-01T09:32:45.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.335 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.335 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:43.089108+0000", 2023-10-01T09:32:45.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2812999999999999e-05 2023-10-01T09:32:45.335 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.335 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.335 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:43.089141+0000", 2023-10-01T09:32:45.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.023429388999999998 2023-10-01T09:32:45.335 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.335 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.335 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.336 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:168947 8.e e368/361 8:71b46346:::benchmark_data_smithi138_431617_object67536:head v 368'10790, mlcod=368'10788)", 2023-10-01T09:32:45.336 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:44.048330+0000", 2023-10-01T09:32:45.336 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.189508056, 2023-10-01T09:32:45.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021785390000000002, 2023-10-01T09:32:45.336 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.336 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.336 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.336 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.048330+0000", 2023-10-01T09:32:45.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.337 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.337 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.048330+0000", 2023-10-01T09:32:45.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:32:45.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.337 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.337 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.048329+0000", 2023-10-01T09:32:45.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2184e-05 2023-10-01T09:32:45.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.048351+0000", 2023-10-01T09:32:45.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2002e-05 2023-10-01T09:32:45.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.048363+0000", 2023-10-01T09:32:45.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6290000000000004e-06 2023-10-01T09:32:45.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.048370+0000", 2023-10-01T09:32:45.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0039884409999999997 2023-10-01T09:32:45.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.052358+0000", 2023-10-01T09:32:45.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2734e-05 2023-10-01T09:32:45.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.052381+0000", 2023-10-01T09:32:45.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017686423 2023-10-01T09:32:45.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.070067+0000", 2023-10-01T09:32:45.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7843999999999998e-05 2023-10-01T09:32:45.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.070115+0000", 2023-10-01T09:32:45.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021785390000000002 2023-10-01T09:32:45.341 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.341 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.341 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.341 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:169234 8.5 e368/367 8:aa59099e:::benchmark_data_smithi138_431617_object67823:head v 368'10718, mlcod=368'10714)", 2023-10-01T09:32:45.341 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:44.610700+0000", 2023-10-01T09:32:45.341 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.62713839599999999, 2023-10-01T09:32:45.342 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018230596000000002, 2023-10-01T09:32:45.342 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.342 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.342 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.342 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.342 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.610700+0000", 2023-10-01T09:32:45.342 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.342 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.610700+0000", 2023-10-01T09:32:45.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:32:45.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.610699+0000", 2023-10-01T09:32:45.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2132e-05 2023-10-01T09:32:45.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.610711+0000", 2023-10-01T09:32:45.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.824e-06 2023-10-01T09:32:45.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.610719+0000", 2023-10-01T09:32:45.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6839999999999998e-06 2023-10-01T09:32:45.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.610723+0000", 2023-10-01T09:32:45.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0079545580000000005 2023-10-01T09:32:45.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.345 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.618677+0000", 2023-10-01T09:32:45.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2849000000000001e-05 2023-10-01T09:32:45.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.345 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.618700+0000", 2023-10-01T09:32:45.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010190721999999999 2023-10-01T09:32:45.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.345 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.628891+0000", 2023-10-01T09:32:45.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9459000000000003e-05 2023-10-01T09:32:45.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.628930+0000", 2023-10-01T09:32:45.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018230596000000002 2023-10-01T09:32:45.346 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.346 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.346 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.347 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:169306 8.5 e368/367 8:a4aaff24:::benchmark_data_smithi138_431617_object67895:head v 368'10721, mlcod=368'10719)", 2023-10-01T09:32:45.347 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:44.709246+0000", 2023-10-01T09:32:45.347 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.52859155599999996, 2023-10-01T09:32:45.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020147841, 2023-10-01T09:32:45.347 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:32:45.347 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:32:45.347 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:32:45.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:32:45.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.709246+0000", 2023-10-01T09:32:45.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:32:45.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:32:45.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.709246+0000", 2023-10-01T09:32:45.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:32:45.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:32:45.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.709246+0000", 2023-10-01T09:32:45.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7200000000000001e-06 2023-10-01T09:32:45.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:32:45.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.709255+0000", 2023-10-01T09:32:45.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5245000000000001e-05 2023-10-01T09:32:45.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:32:45.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.709271+0000", 2023-10-01T09:32:45.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5169999999999996e-06 2023-10-01T09:32:45.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:32:45.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.709277+0000", 2023-10-01T09:32:45.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019276499999999999 2023-10-01T09:32:45.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:32:45.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.711205+0000", 2023-10-01T09:32:45.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0979e-05 2023-10-01T09:32:45.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:32:45.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.711226+0000", 2023-10-01T09:32:45.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018125274 2023-10-01T09:32:45.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:32:45.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.729351+0000", 2023-10-01T09:32:45.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3306000000000002e-05 2023-10-01T09:32:45.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:32:45.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:32:45.352 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:32:45.352 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:44.729394+0000", 2023-10-01T09:32:45.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020147841 2023-10-01T09:32:45.352 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.352 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.352 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.352 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:32:45.352 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:32:45.352 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:32:45.353 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:45 smithi190 conmon[216583]: debug 2023-10-01T09:32:45.182+0000 7f4b78c47700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T09:32:45.353 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:45 smithi190 conmon[216583]: debug 2023-10-01T09:32:45.182+0000 7f4b78c47700 -1 osd.5 368 *** Got signal Terminated *** 2023-10-01T09:32:45.353 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:45 smithi190 conmon[216583]: debug 2023-10-01T09:32:45.182+0000 7f4b78c47700 -1 osd.5 368 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:32:45.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:45.354 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:45.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:45.455 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:45.485 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:45 smithi190 ceph-mon[160435]: osd.5 marked itself down and dead 2023-10-01T09:32:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:45 smithi190 ceph-mon[160435]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:32:45.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:45 smithi138 ceph-mon[192513]: osd.5 marked itself down and dead 2023-10-01T09:32:45.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:45 smithi138 ceph-mon[192513]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:32:45.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:45 smithi138 ceph-mon[185886]: osd.5 marked itself down and dead 2023-10-01T09:32:45.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:45 smithi138 ceph-mon[185886]: Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T09:32:45.810 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:46.135 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:46.137 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:45 smithi190 bash[286447]: 48f5e8c26c664763985f42da4753ecf61a6cd038e8f27e260ce0b3b34c04b1f8 2023-10-01T09:32:46.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:46.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:46.239 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:46.337 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:46.498 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:46 smithi138 ceph-mon[192513]: failed to encode map e369 with expected crc 2023-10-01T09:32:46.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:46 smithi138 ceph-mon[192513]: osdmap e369: 8 total, 7 up, 8 in 2023-10-01T09:32:46.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:46 smithi138 ceph-mon[192513]: failed to encode map e369 with expected crc 2023-10-01T09:32:46.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:46 smithi138 ceph-mon[192513]: failed to encode map e369 with expected crc 2023-10-01T09:32:46.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:46 smithi138 ceph-mon[192513]: pgmap v1032: 89 pgs: 1 remapped+peering, 1 peering, 16 stale+active+clean, 3 active+remapped+backfill_wait, 68 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 6273/103794 objects misplaced (6.044%); 20 MiB/s, 322 objects/s recovering 2023-10-01T09:32:46.499 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:46 smithi138 ceph-mon[185886]: failed to encode map e369 with expected crc 2023-10-01T09:32:46.499 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:46 smithi138 ceph-mon[185886]: osdmap e369: 8 total, 7 up, 8 in 2023-10-01T09:32:46.499 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:46 smithi138 ceph-mon[185886]: failed to encode map e369 with expected crc 2023-10-01T09:32:46.500 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:46 smithi138 ceph-mon[185886]: failed to encode map e369 with expected crc 2023-10-01T09:32:46.500 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:46 smithi138 ceph-mon[185886]: pgmap v1032: 89 pgs: 1 remapped+peering, 1 peering, 16 stale+active+clean, 3 active+remapped+backfill_wait, 68 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 6273/103794 objects misplaced (6.044%); 20 MiB/s, 322 objects/s recovering 2023-10-01T09:32:46.510 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:46.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:46.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:46.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:46 smithi190 ceph-mon[160435]: failed to encode map e369 with expected crc 2023-10-01T09:32:46.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:46 smithi190 ceph-mon[160435]: osdmap e369: 8 total, 7 up, 8 in 2023-10-01T09:32:46.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:46 smithi190 ceph-mon[160435]: failed to encode map e369 with expected crc 2023-10-01T09:32:46.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:46 smithi190 ceph-mon[160435]: failed to encode map e369 with expected crc 2023-10-01T09:32:46.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:46 smithi190 ceph-mon[160435]: pgmap v1032: 89 pgs: 1 remapped+peering, 1 peering, 16 stale+active+clean, 3 active+remapped+backfill_wait, 68 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 6273/103794 objects misplaced (6.044%); 20 MiB/s, 322 objects/s recovering 2023-10-01T09:32:46.571 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:32:46.620 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:46.708 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:46.800 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:46.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:46.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:46.913 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:46.945 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:46.995 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:47.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:47.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:47.050 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:32:47.101 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:47.164 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:47.313 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:32:47.414 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[185886]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:32:47.414 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[185886]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.414 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[185886]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.414 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[185886]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.414 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[185886]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.414 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[185886]: osdmap e370: 8 total, 7 up, 8 in 2023-10-01T09:32:47.414 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[185886]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.414 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[185886]: 8.2 starting backfill to osd.4 from (333'2300,368'10456] MIN to 368'10457 2023-10-01T09:32:47.415 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[185886]: 8.4 continuing backfill to osd.4 from (331'500,368'10489] 8:291fd5c9:::benchmark_data_smithi138_431617_object87457:head to 368'10489 2023-10-01T09:32:47.415 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1400967840' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:32:47.415 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:32:47.415 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[192513]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:32:47.415 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[192513]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.416 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[192513]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.416 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[192513]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.416 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[192513]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.416 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[192513]: osdmap e370: 8 total, 7 up, 8 in 2023-10-01T09:32:47.416 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[192513]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.416 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[192513]: 8.2 starting backfill to osd.4 from (333'2300,368'10456] MIN to 368'10457 2023-10-01T09:32:47.416 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[192513]: 8.4 continuing backfill to osd.4 from (331'500,368'10489] 8:291fd5c9:::benchmark_data_smithi138_431617_object87457:head to 368'10489 2023-10-01T09:32:47.416 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1400967840' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:32:47.416 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:47 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:32:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:47 smithi190 ceph-mon[160435]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:32:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:47 smithi190 ceph-mon[160435]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:47 smithi190 ceph-mon[160435]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:47 smithi190 ceph-mon[160435]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:47 smithi190 ceph-mon[160435]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:47 smithi190 ceph-mon[160435]: osdmap e370: 8 total, 7 up, 8 in 2023-10-01T09:32:47.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:47 smithi190 ceph-mon[160435]: failed to encode map e370 with expected crc 2023-10-01T09:32:47.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:47 smithi190 ceph-mon[160435]: 8.2 starting backfill to osd.4 from (333'2300,368'10456] MIN to 368'10457 2023-10-01T09:32:47.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:47 smithi190 ceph-mon[160435]: 8.4 continuing backfill to osd.4 from (331'500,368'10489] 8:291fd5c9:::benchmark_data_smithi138_431617_object87457:head to 368'10489 2023-10-01T09:32:47.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:47 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1400967840' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:32:47.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:47 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:32:47.591 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:47.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:47.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:47.697 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:47.727 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:48.381 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:48.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:48.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:48.492 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:48.536 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:48 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:48.536 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:48 smithi190 ceph-mon[160435]: failed to encode map e371 with expected crc 2023-10-01T09:32:48.536 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:48 smithi190 ceph-mon[160435]: failed to encode map e371 with expected crc 2023-10-01T09:32:48.536 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:48 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:32:48.537 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:48 smithi190 ceph-mon[160435]: osdmap e371: 8 total, 7 up, 8 in 2023-10-01T09:32:48.537 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:48 smithi190 ceph-mon[160435]: pgmap v1035: 89 pgs: 8 active+undersized, 1 remapped+peering, 1 peering, 5 stale+active+clean, 6 active+undersized+degraded, 3 active+remapped+backfill_wait, 65 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 4135/102411 objects degraded (4.038%); 6273/102411 objects misplaced (6.125%); 10 MiB/s, 159 objects/s recovering 2023-10-01T09:32:48.538 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:48 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:48 smithi138 ceph-mon[185886]: failed to encode map e371 with expected crc 2023-10-01T09:32:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:48 smithi138 ceph-mon[185886]: failed to encode map e371 with expected crc 2023-10-01T09:32:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:48 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:32:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:48 smithi138 ceph-mon[185886]: osdmap e371: 8 total, 7 up, 8 in 2023-10-01T09:32:48.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:48 smithi138 ceph-mon[185886]: pgmap v1035: 89 pgs: 8 active+undersized, 1 remapped+peering, 1 peering, 5 stale+active+clean, 6 active+undersized+degraded, 3 active+remapped+backfill_wait, 65 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 4135/102411 objects degraded (4.038%); 6273/102411 objects misplaced (6.125%); 10 MiB/s, 159 objects/s recovering 2023-10-01T09:32:48.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:48 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:32:48.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:48 smithi138 ceph-mon[192513]: failed to encode map e371 with expected crc 2023-10-01T09:32:48.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:48 smithi138 ceph-mon[192513]: failed to encode map e371 with expected crc 2023-10-01T09:32:48.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:48 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:32:48.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:48 smithi138 ceph-mon[192513]: osdmap e371: 8 total, 7 up, 8 in 2023-10-01T09:32:48.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:48 smithi138 ceph-mon[192513]: pgmap v1035: 89 pgs: 8 active+undersized, 1 remapped+peering, 1 peering, 5 stale+active+clean, 6 active+undersized+degraded, 3 active+remapped+backfill_wait, 65 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 366 op/s; 4135/102411 objects degraded (4.038%); 6273/102411 objects misplaced (6.125%); 10 MiB/s, 159 objects/s recovering 2023-10-01T09:32:48.620 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:48.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:48.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:48.725 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:48.768 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:32:48.799 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:48.816 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:32:48.842 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:32:48.884 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:32:48.919 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:32:48.963 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:32:48.999 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:32:48.999 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:32:48.999 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:32:48.999 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:32:48.999 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:32:49.000 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:32:49.000 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-01T09:32:49.000 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:32:49.000 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:32:49.061 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:32:49.061 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:32:49.082 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:49.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:49.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:49.187 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:49.277 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:49.374 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:49.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:49.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:49.485 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:49.536 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:49 smithi138 ceph-mon[185886]: Health check failed: Degraded data redundancy: 4135/102411 objects degraded (4.038%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:32:49.536 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2298286882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:49.537 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:49 smithi138 ceph-mon[192513]: Health check failed: Degraded data redundancy: 4135/102411 objects degraded (4.038%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:32:49.538 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2298286882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:49.541 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:49.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:49 smithi190 ceph-mon[160435]: Health check failed: Degraded data redundancy: 4135/102411 objects degraded (4.038%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:32:49.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2298286882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:49.589 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:49.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:49.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:49.693 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:49.723 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:50.067 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:32:50.102 DEBUG:teuthology.orchestra.run.smithi190:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-01T09:32:50.137 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:50.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:50.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:50.205 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:32:50.205 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-01T09:32:50.243 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:32:50.266 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:32:50.266 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:32:50.329 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:50.329 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:32:50.474 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:32:50.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:50.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:32:50.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:50 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:32:50.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:50 smithi190 ceph-mon[160435]: pgmap v1036: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 1 remapped+peering, 1 peering, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 14024/96792 objects degraded (14.489%); 5389/96792 objects misplaced (5.568%); 15 MiB/s, 242 objects/s recovering 2023-10-01T09:32:50.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:50 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:32:50.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:50 smithi138 ceph-mon[192513]: pgmap v1036: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 1 remapped+peering, 1 peering, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 14024/96792 objects degraded (14.489%); 5389/96792 objects misplaced (5.568%); 15 MiB/s, 242 objects/s recovering 2023-10-01T09:32:50.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:50 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:32:50.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:50 smithi138 ceph-mon[185886]: pgmap v1036: 89 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 1 remapped+peering, 1 peering, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 412 op/s; 14024/96792 objects degraded (14.489%); 5389/96792 objects misplaced (5.568%); 15 MiB/s, 242 objects/s recovering 2023-10-01T09:32:50.585 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:50.642 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:32:50.717 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:50.835 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:50.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:50.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:50.952 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:50.986 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:51.065 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:51.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:51.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:51.119 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:51.174 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:32:51.245 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:51.384 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:32:51.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:51.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:32:51.490 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:51.602 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:51.763 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:51.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:51.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:51.873 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:51.908 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:52.128 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:51 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:52.267 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:32:52.283 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:51 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:52.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:51 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:52.345 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:52.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:52.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:52.449 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:52.482 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:52.526 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:52.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:52.630 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:52.660 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:52.720 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:52.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:52.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:52.826 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:52.885 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:52 smithi190 ceph-mon[160435]: pgmap v1037: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 13917/94704 objects degraded (14.695%); 7263/94704 objects misplaced (7.669%); 19 MiB/s, 297 objects/s recovering 2023-10-01T09:32:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:52 smithi190 ceph-mon[160435]: failed to encode map e372 with expected crc 2023-10-01T09:32:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:52 smithi190 ceph-mon[160435]: osdmap e372: 8 total, 7 up, 8 in 2023-10-01T09:32:53.058 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:32:53.059 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:32:53.175 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:53.177 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:52 smithi138 ceph-mon[192513]: pgmap v1037: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 13917/94704 objects degraded (14.695%); 7263/94704 objects misplaced (7.669%); 19 MiB/s, 297 objects/s recovering 2023-10-01T09:32:53.178 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:52 smithi138 ceph-mon[192513]: failed to encode map e372 with expected crc 2023-10-01T09:32:53.178 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:52 smithi138 ceph-mon[192513]: osdmap e372: 8 total, 7 up, 8 in 2023-10-01T09:32:53.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:53.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:53.181 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:52 smithi138 ceph-mon[185886]: pgmap v1037: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 467 op/s; 13917/94704 objects degraded (14.695%); 7263/94704 objects misplaced (7.669%); 19 MiB/s, 297 objects/s recovering 2023-10-01T09:32:53.181 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:52 smithi138 ceph-mon[185886]: failed to encode map e372 with expected crc 2023-10-01T09:32:53.182 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:52 smithi138 ceph-mon[185886]: osdmap e372: 8 total, 7 up, 8 in 2023-10-01T09:32:53.279 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:53.313 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:53.472 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:53.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:53.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:53.480 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:53.577 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:32:53.611 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:53.786 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:32:53.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:53.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:32:53.901 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:32:53.904 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:32:53.957 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:54.067 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:32:54.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:53 smithi138 ceph-mon[192513]: failed to encode map e372 with expected crc 2023-10-01T09:32:54.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:53 smithi138 ceph-mon[192513]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:32:54.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:53 smithi138 ceph-mon[192513]: osdmap e373: 8 total, 7 up, 8 in 2023-10-01T09:32:54.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:53 smithi138 ceph-mon[192513]: failed to encode map e373 with expected crc 2023-10-01T09:32:54.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:53 smithi138 ceph-mon[192513]: failed to encode map e373 with expected crc 2023-10-01T09:32:54.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:53 smithi138 ceph-mon[185886]: failed to encode map e372 with expected crc 2023-10-01T09:32:54.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:53 smithi138 ceph-mon[185886]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:32:54.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:53 smithi138 ceph-mon[185886]: osdmap e373: 8 total, 7 up, 8 in 2023-10-01T09:32:54.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:53 smithi138 ceph-mon[185886]: failed to encode map e373 with expected crc 2023-10-01T09:32:54.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:53 smithi138 ceph-mon[185886]: failed to encode map e373 with expected crc 2023-10-01T09:32:54.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:54.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:32:54.177 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:54.211 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:53 smithi190 ceph-mon[160435]: failed to encode map e372 with expected crc 2023-10-01T09:32:54.211 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:53 smithi190 ceph-mon[160435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T09:32:54.212 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:53 smithi190 ceph-mon[160435]: osdmap e373: 8 total, 7 up, 8 in 2023-10-01T09:32:54.212 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:53 smithi190 ceph-mon[160435]: failed to encode map e373 with expected crc 2023-10-01T09:32:54.212 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:53 smithi190 ceph-mon[160435]: failed to encode map e373 with expected crc 2023-10-01T09:32:54.220 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:54.281 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:54.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:54.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:54.390 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:54.490 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:54.513 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:32:54.810 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:54.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:54.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:54.901 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:32:54.927 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:32:54.954 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:54.997 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:32:55.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:55.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:32:55.108 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:55.148 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:54 smithi190 ceph-mon[160435]: failed to encode map e373 with expected crc 2023-10-01T09:32:55.148 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:54 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3792843131' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:55.148 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:54 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:55.148 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:54 smithi190 ceph-mon[160435]: pgmap v1040: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 12652/88212 objects degraded (14.343%); 6787/88212 objects misplaced (7.694%); 13 MiB/s, 210 objects/s recovering 2023-10-01T09:32:55.152 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:55.206 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5, 6, 4] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 3, 4, 7, 6] 2023-10-01T09:32:55.206 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:32:55.207 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2023-10-01T09:32:55.207 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2023-10-01T09:32:55.207 DEBUG:teuthology.orchestra.run.smithi190:osd.5> sudo journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-01T09:32:55.210 DEBUG:teuthology.orchestra.run.smithi190:> sudo systemctl start ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.5 2023-10-01T09:32:55.233 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:32:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:32:54] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.33.4" 2023-10-01T09:32:55.233 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:54 smithi138 ceph-mon[185886]: failed to encode map e373 with expected crc 2023-10-01T09:32:55.233 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:54 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3792843131' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:55.234 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:54 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:55.234 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:54 smithi138 ceph-mon[185886]: pgmap v1040: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 12652/88212 objects degraded (14.343%); 6787/88212 objects misplaced (7.694%); 13 MiB/s, 210 objects/s recovering 2023-10-01T09:32:55.235 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:54 smithi138 ceph-mon[192513]: failed to encode map e373 with expected crc 2023-10-01T09:32:55.235 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:54 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3792843131' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:55.236 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:54 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:32:55.236 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:54 smithi138 ceph-mon[192513]: pgmap v1040: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 12652/88212 objects degraded (14.343%); 6787/88212 objects misplaced (7.694%); 13 MiB/s, 210 objects/s recovering 2023-10-01T09:32:55.259 INFO:journalctl@ceph.osd.5.smithi190.stdout:-- Logs begin at Sun 2023-10-01 08:38:18 UTC. -- 2023-10-01T09:32:55.557 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:55 smithi190 systemd[1]: Starting Ceph osd.5 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:32:55.899 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:55 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 12652/88212 objects degraded (14.343%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T09:32:55.899 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:55 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:32:55.899 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:55 smithi190 ceph-mon[160435]: failed to encode map e374 with expected crc 2023-10-01T09:32:55.899 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:55 smithi190 ceph-mon[160435]: failed to encode map e374 with expected crc 2023-10-01T09:32:55.899 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:55 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:32:55.899 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:55 smithi190 ceph-mon[160435]: osdmap e374: 8 total, 7 up, 8 in 2023-10-01T09:32:55.900 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:55.900 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:32:56.120 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:56.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:56.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:56.195 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 12652/88212 objects degraded (14.343%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T09:32:56.196 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:32:56.196 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[192513]: failed to encode map e374 with expected crc 2023-10-01T09:32:56.196 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[192513]: failed to encode map e374 with expected crc 2023-10-01T09:32:56.196 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:32:56.196 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[192513]: osdmap e374: 8 total, 7 up, 8 in 2023-10-01T09:32:56.196 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:56.196 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:32:56.200 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 12652/88212 objects degraded (14.343%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T09:32:56.200 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:32:56.200 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[185886]: failed to encode map e374 with expected crc 2023-10-01T09:32:56.200 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[185886]: failed to encode map e374 with expected crc 2023-10-01T09:32:56.200 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:32:56.200 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[185886]: osdmap e374: 8 total, 7 up, 8 in 2023-10-01T09:32:56.200 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:56.200 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:32:56.227 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:56.276 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:56.402 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:56.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:56.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:56.512 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:32:56.623 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:56.711 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:32:56.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:56.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:32:56.816 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:56.827 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:32:56.827 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:32:56.859 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:56.912 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:56 smithi138 ceph-mon[185886]: pgmap v1042: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 12652/88212 objects degraded (14.343%); 6787/88212 objects misplaced (7.694%); 9.6 MiB/s, 153 objects/s recovering 2023-10-01T09:32:56.912 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:56.912 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:57.009 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:32:57.054 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:32:57.105 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:32:57.113 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:57.116 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:56 smithi190 ceph-mon[160435]: pgmap v1042: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 12652/88212 objects degraded (14.343%); 6787/88212 objects misplaced (7.694%); 9.6 MiB/s, 153 objects/s recovering 2023-10-01T09:32:57.116 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:57.116 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:57.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:57.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:57.140 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:32:57.172 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:32:57.219 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:32:57.228 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:32:57.257 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:32:57.257 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:32:57.257 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:32:57.257 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:32:57.257 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:32:57.257 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:32:57.257 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-01T09:32:57.258 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:32:57.258 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:32:57.284 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:56 smithi138 ceph-mon[192513]: pgmap v1042: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 12652/88212 objects degraded (14.343%); 6787/88212 objects misplaced (7.694%); 9.6 MiB/s, 153 objects/s recovering 2023-10-01T09:32:57.284 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:57.284 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:57.308 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:57.562 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:32:57.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:57.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:32:57.674 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:57.746 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:57.802 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:32:57.927 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2083764512' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:58.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2083764512' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:58.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:57 smithi190 bash[290285]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-10-01T09:32:58.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:57 smithi190 bash[290285]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2023-10-01T09:32:58.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:57 smithi190 bash[290285]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2023-10-01T09:32:58.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:57 smithi190 bash[290285]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-01T09:32:58.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:57 smithi190 bash[290285]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2023-10-01T09:32:58.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:57 smithi190 bash[290285]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-10-01T09:32:58.058 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:57 smithi190 bash[290285]: --> ceph-volume raw activate successful for osd ID: 5 2023-10-01T09:32:58.234 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:32:58.280 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2083764512' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:32:58.550 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:58.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:58.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:58.658 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:32:58.722 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:32:58.723 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:32:58.739 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:58.747 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:32:58.840 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:32:58.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:58.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:32:58.952 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:32:59.006 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:59.026 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:32:59.088 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:32:59.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:59.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:32:59.193 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:59.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:58 smithi138 ceph-mon[185886]: pgmap v1043: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 12052/85983 objects degraded (14.017%); 6535/85983 objects misplaced (7.600%); 6.4 MiB/s, 101 objects/s recovering 2023-10-01T09:32:59.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:59.301 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:59.301 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:58 smithi138 ceph-mon[192513]: pgmap v1043: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 12052/85983 objects degraded (14.017%); 6535/85983 objects misplaced (7.600%); 6.4 MiB/s, 101 objects/s recovering 2023-10-01T09:32:59.301 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:59.301 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:59.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:58 smithi190 ceph-mon[160435]: pgmap v1043: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 12052/85983 objects degraded (14.017%); 6535/85983 objects misplaced (7.600%); 6.4 MiB/s, 101 objects/s recovering 2023-10-01T09:32:59.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:59.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:32:59.315 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:59.479 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:59.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:59.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:32:59.586 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:32:59.683 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:32:59.904 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:32:59.904 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:32:59.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:32:59.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:00.032 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:00.097 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:00.153 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:00.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:00.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:00.165 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:59 smithi190 conmon[291240]: debug 2023-10-01T09:32:59.842+0000 7fed6da04700 1 -- 172.21.15.190:0/1477645499 <== mon.1 v2:172.21.15.190:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x565529dee680 con 0x565529db5400 2023-10-01T09:33:00.165 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:59 smithi190 bash[291155]: 6377b4c138ce70222ecbae6de4699f733362da09c749352bd7838513f16e4913 2023-10-01T09:33:00.165 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:32:59 smithi190 systemd[1]: Started Ceph osd.5 for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:33:00.166 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:33:00.166 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:33:00.166 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:32:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:00.257 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:00.305 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:00.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:33:00.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:33:00.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:32:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:00.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:33:00.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:33:00.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:32:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:00.419 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:33:00 smithi190 conmon[291240]: debug 2023-10-01T09:33:00.364+0000 7fed7547e200 -1 Falling back to public interface 2023-10-01T09:33:00.445 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:00.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:00.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:00.549 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:00.594 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:00.657 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:33:00.686 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:00.693 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:33:00.693 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:33:00.693 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.694 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:178086 8.5 8:a83fe641:::benchmark_data_smithi138_431617_object76675:head [delete] snapc 0=[] ondisk+write+known_if_redirected e374)", 2023-10-01T09:33:00.694 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:00.671403+0000", 2023-10-01T09:33:00.694 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.020612518999999999, 2023-10-01T09:33:00.694 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020636578999999999, 2023-10-01T09:33:00.695 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:33:00.695 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:33:00.695 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:33:00.695 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:33:00.695 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:00.696 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 178086 2023-10-01T09:33:00.696 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.696 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:33:00.696 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.696 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:33:00.697 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.671403+0000", 2023-10-01T09:33:00.697 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:33:00.697 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.697 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.697 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:33:00.698 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.671403+0000", 2023-10-01T09:33:00.698 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:33:00.698 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.698 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.698 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:33:00.699 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.671401+0000", 2023-10-01T09:33:00.699 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.905e-06 2023-10-01T09:33:00.699 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.699 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.700 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:33:00.701 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.671408+0000", 2023-10-01T09:33:00.701 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7918e-05 2023-10-01T09:33:00.701 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.701 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.702 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:33:00.702 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.671426+0000", 2023-10-01T09:33:00.702 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8370000000000002e-06 2023-10-01T09:33:00.702 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.703 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.703 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:33:00.703 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.671431+0000", 2023-10-01T09:33:00.703 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5674999999999999e-05 2023-10-01T09:33:00.703 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.703 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.704 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:33:00.704 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.671477+0000", 2023-10-01T09:33:00.704 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013499399999999999 2023-10-01T09:33:00.704 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.704 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.704 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:33:00.705 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.671612+0000", 2023-10-01T09:33:00.705 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015180700000000001 2023-10-01T09:33:00.705 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.705 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.705 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 4,7", 2023-10-01T09:33:00.705 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.671764+0000", 2023-10-01T09:33:00.705 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045071119999999998 2023-10-01T09:33:00.706 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.706 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.706 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:33:00.706 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.676271+0000", 2023-10-01T09:33:00.706 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0044617299999999997 2023-10-01T09:33:00.706 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.706 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.706 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:33:00.706 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.680733+0000", 2023-10-01T09:33:00.706 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0093301630000000007 2023-10-01T09:33:00.706 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:00.707 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:33:00.707 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:00.707 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.707 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.707 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:178089 8.5 8:ad9abe12:::benchmark_data_smithi138_431617_object76678:head [delete] snapc 0=[] ondisk+write+known_if_redirected e374)", 2023-10-01T09:33:00.707 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:00.676301+0000", 2023-10-01T09:33:00.707 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.015714004, 2023-10-01T09:33:00.707 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015798774000000002, 2023-10-01T09:33:00.707 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:33:00.708 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:33:00.708 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:33:00.708 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:33:00.708 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:00.708 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 178089 2023-10-01T09:33:00.708 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.708 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:33:00.708 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.709 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:33:00.709 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.676301+0000", 2023-10-01T09:33:00.709 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:33:00.709 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.709 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.709 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:33:00.710 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.676301+0000", 2023-10-01T09:33:00.710 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:33:00.710 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.710 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.710 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:33:00.710 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.676300+0000", 2023-10-01T09:33:00.711 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2929999999999997e-06 2023-10-01T09:33:00.711 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.711 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.711 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:33:00.711 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.676306+0000", 2023-10-01T09:33:00.711 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3597e-05 2023-10-01T09:33:00.711 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.711 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.712 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:33:00.712 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.676320+0000", 2023-10-01T09:33:00.712 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1589999999999998e-06 2023-10-01T09:33:00.712 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.712 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.712 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:33:00.712 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.676326+0000", 2023-10-01T09:33:00.713 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013175 2023-10-01T09:33:00.713 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.713 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.713 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:33:00.713 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.676458+0000", 2023-10-01T09:33:00.713 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012609599999999999 2023-10-01T09:33:00.713 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.714 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.714 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:33:00.714 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.676584+0000", 2023-10-01T09:33:00.714 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000121981 2023-10-01T09:33:00.714 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.714 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.715 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 4,7", 2023-10-01T09:33:00.715 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.676706+0000", 2023-10-01T09:33:00.715 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049186689999999996 2023-10-01T09:33:00.715 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.715 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.715 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:33:00.715 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.681624+0000", 2023-10-01T09:33:00.715 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0040518539999999997 2023-10-01T09:33:00.716 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.716 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.716 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:33:00.716 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.685676+0000", 2023-10-01T09:33:00.716 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0093751259999999993 2023-10-01T09:33:00.716 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:00.716 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:33:00.717 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:00.717 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.717 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.717 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:178097 8.9 e374/369 8:91e5c533:::benchmark_data_smithi138_431617_object76686:head v 374'11113, mlcod=374'11111)", 2023-10-01T09:33:00.717 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:00.688137+0000", 2023-10-01T09:33:00.717 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0038783519999999998, 2023-10-01T09:33:00.717 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0040150400000000001, 2023-10-01T09:33:00.718 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:33:00.718 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:33:00.718 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:33:00.718 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.718 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:33:00.718 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688137+0000", 2023-10-01T09:33:00.718 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:33:00.719 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.719 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.719 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:33:00.719 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688137+0000", 2023-10-01T09:33:00.719 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:33:00.719 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.719 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.719 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:33:00.720 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688136+0000", 2023-10-01T09:33:00.720 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2101e-05 2023-10-01T09:33:00.720 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.720 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.720 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:33:00.720 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688148+0000", 2023-10-01T09:33:00.720 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4781e-05 2023-10-01T09:33:00.721 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.721 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.721 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:33:00.721 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688163+0000", 2023-10-01T09:33:00.721 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6279999999999999e-06 2023-10-01T09:33:00.721 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.721 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.722 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:33:00.722 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688169+0000", 2023-10-01T09:33:00.722 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7700000000000006e-05 2023-10-01T09:33:00.722 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.722 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.722 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:33:00.722 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688237+0000", 2023-10-01T09:33:00.723 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3638e-05 2023-10-01T09:33:00.723 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.723 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.723 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:33:00.723 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688261+0000", 2023-10-01T09:33:00.723 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012412100000000001 2023-10-01T09:33:00.724 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:00.724 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:33:00.724 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:00.724 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.724 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.724 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:178101 8.4 e374/372 8:29adbe5b:::benchmark_data_smithi138_431617_object76690:head v 374'11020, mlcod=374'11018)", 2023-10-01T09:33:00.724 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:00.688792+0000", 2023-10-01T09:33:00.724 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0032229260000000001, 2023-10-01T09:33:00.724 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033986810000000002, 2023-10-01T09:33:00.725 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:33:00.726 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:33:00.726 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:33:00.726 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.726 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:33:00.726 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688792+0000", 2023-10-01T09:33:00.726 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:33:00.726 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.726 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.726 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:33:00.726 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688792+0000", 2023-10-01T09:33:00.727 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:33:00.727 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.727 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.727 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:33:00.727 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688791+0000", 2023-10-01T09:33:00.727 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.126e-06 2023-10-01T09:33:00.727 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.727 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.727 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:33:00.727 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688799+0000", 2023-10-01T09:33:00.727 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4039999999999999e-05 2023-10-01T09:33:00.727 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.728 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.728 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:33:00.728 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688813+0000", 2023-10-01T09:33:00.728 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7529999999999998e-06 2023-10-01T09:33:00.728 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.728 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.728 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:33:00.728 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688818+0000", 2023-10-01T09:33:00.728 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1103999999999998e-05 2023-10-01T09:33:00.728 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.728 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.728 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:33:00.729 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688859+0000", 2023-10-01T09:33:00.729 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5364000000000002e-05 2023-10-01T09:33:00.729 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.729 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.729 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:33:00.729 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.688895+0000", 2023-10-01T09:33:00.729 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000102709 2023-10-01T09:33:00.729 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:00.729 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:33:00.729 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:00.729 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.729 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.729 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:178096 8.3 e374/369 8:c586b326:::benchmark_data_smithi138_431617_object76685:head v 374'11015, mlcod=374'11013)", 2023-10-01T09:33:00.730 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:00.690875+0000", 2023-10-01T09:33:00.730 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0011403030000000001, 2023-10-01T09:33:00.730 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0013509559999999999, 2023-10-01T09:33:00.730 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:33:00.730 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:33:00.730 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:33:00.730 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.730 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:33:00.730 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.690875+0000", 2023-10-01T09:33:00.730 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:33:00.730 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.730 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.731 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:33:00.731 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.690875+0000", 2023-10-01T09:33:00.731 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:33:00.731 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.731 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.731 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:33:00.731 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.690874+0000", 2023-10-01T09:33:00.731 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2259999999999994e-06 2023-10-01T09:33:00.731 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.731 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.731 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:33:00.731 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.690883+0000", 2023-10-01T09:33:00.732 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6124999999999999e-05 2023-10-01T09:33:00.732 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.732 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.732 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:33:00.732 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.690899+0000", 2023-10-01T09:33:00.732 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0990000000000001e-06 2023-10-01T09:33:00.732 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.732 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.732 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:33:00.732 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.690906+0000", 2023-10-01T09:33:00.732 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1890000000000005e-05 2023-10-01T09:33:00.732 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.732 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.733 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:33:00.733 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.690968+0000", 2023-10-01T09:33:00.733 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4314000000000001e-05 2023-10-01T09:33:00.733 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:00.733 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:00.733 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:33:00.733 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:00.690993+0000", 2023-10-01T09:33:00.733 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011780799999999999 2023-10-01T09:33:00.733 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:00.733 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:33:00.733 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:00.733 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:00.734 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:33:00.734 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 5 2023-10-01T09:33:00.734 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:33:00.734 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:33:00.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:00.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:00.837 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:00.867 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:00.983 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:00.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:00.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:01.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:00 smithi138 ceph-mon[185886]: pgmap v1044: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.6 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 10587/79080 objects degraded (13.388%); 6007/79080 objects misplaced (7.596%); 9.9 MiB/s, 158 objects/s recovering 2023-10-01T09:33:01.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:00 smithi138 ceph-mon[192513]: pgmap v1044: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.6 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 10587/79080 objects degraded (13.388%); 6007/79080 objects misplaced (7.596%); 9.9 MiB/s, 158 objects/s recovering 2023-10-01T09:33:01.089 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:01.172 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:01.218 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:01.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:01.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:01.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:00 smithi190 ceph-mon[160435]: pgmap v1044: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.6 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 10587/79080 objects degraded (13.388%); 6007/79080 objects misplaced (7.596%); 9.9 MiB/s, 158 objects/s recovering 2023-10-01T09:33:01.322 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:01.427 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:01.576 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:01.580 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:01.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:01.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:01.691 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:01.738 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:33:01.776 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:01.939 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:01.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:01.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:01.992 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:01 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 10587/79080 objects degraded (13.388%), 14 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:01.992 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4190132244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:33:01.992 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:01 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:33:02.006 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:33:02.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:01 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 10587/79080 objects degraded (13.388%), 14 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:02.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4190132244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:33:02.030 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:01 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:33:02.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:01 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 10587/79080 objects degraded (13.388%), 14 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:02.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4190132244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:33:02.031 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:01 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:33:02.047 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:02.081 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:02.481 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:02.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:02.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:02.588 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:02.670 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:02.701 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:33:02.996 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:02.996 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[192513]: failed to encode map e375 with expected crc 2023-10-01T09:33:02.996 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:33:02.996 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[192513]: osdmap e375: 8 total, 7 up, 8 in 2023-10-01T09:33:02.996 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[192513]: failed to encode map e375 with expected crc 2023-10-01T09:33:02.996 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[192513]: failed to encode map e375 with expected crc 2023-10-01T09:33:02.996 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[192513]: failed to encode map e375 with expected crc 2023-10-01T09:33:02.996 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[192513]: pgmap v1046: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 10217/77412 objects degraded (13.198%); 6007/77412 objects misplaced (7.760%); 6.6 MiB/s, 105 objects/s recovering 2023-10-01T09:33:02.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:02.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[185886]: failed to encode map e375 with expected crc 2023-10-01T09:33:02.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:33:02.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[185886]: osdmap e375: 8 total, 7 up, 8 in 2023-10-01T09:33:02.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[185886]: failed to encode map e375 with expected crc 2023-10-01T09:33:02.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[185886]: failed to encode map e375 with expected crc 2023-10-01T09:33:02.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[185886]: failed to encode map e375 with expected crc 2023-10-01T09:33:02.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:02 smithi138 ceph-mon[185886]: pgmap v1046: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 10217/77412 objects degraded (13.198%); 6007/77412 objects misplaced (7.760%); 6.6 MiB/s, 105 objects/s recovering 2023-10-01T09:33:03.084 INFO:teuthology.orchestra.run.smithi190.stderr:no valid command found; 10 closest matches: 2023-10-01T09:33:03.084 INFO:teuthology.orchestra.run.smithi190.stderr:0 2023-10-01T09:33:03.084 INFO:teuthology.orchestra.run.smithi190.stderr:1 2023-10-01T09:33:03.084 INFO:teuthology.orchestra.run.smithi190.stderr:2 2023-10-01T09:33:03.085 INFO:teuthology.orchestra.run.smithi190.stderr:abort 2023-10-01T09:33:03.085 INFO:teuthology.orchestra.run.smithi190.stderr:assert 2023-10-01T09:33:03.085 INFO:teuthology.orchestra.run.smithi190.stderr:bluefs debug_inject_read_zeros 2023-10-01T09:33:03.085 INFO:teuthology.orchestra.run.smithi190.stderr:bluefs files list 2023-10-01T09:33:03.085 INFO:teuthology.orchestra.run.smithi190.stderr:bluefs stats 2023-10-01T09:33:03.085 INFO:teuthology.orchestra.run.smithi190.stderr:bluestore bluefs device info [] 2023-10-01T09:33:03.085 INFO:teuthology.orchestra.run.smithi190.stderr:config diff 2023-10-01T09:33:03.085 INFO:teuthology.orchestra.run.smithi190.stderr:admin_socket: invalid command 2023-10-01T09:33:03.107 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:02 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:03.107 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:02 smithi190 ceph-mon[160435]: failed to encode map e375 with expected crc 2023-10-01T09:33:03.107 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:02 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:33:03.107 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:02 smithi190 ceph-mon[160435]: osdmap e375: 8 total, 7 up, 8 in 2023-10-01T09:33:03.107 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:02 smithi190 ceph-mon[160435]: failed to encode map e375 with expected crc 2023-10-01T09:33:03.107 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:02 smithi190 ceph-mon[160435]: failed to encode map e375 with expected crc 2023-10-01T09:33:03.107 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:02 smithi190 ceph-mon[160435]: failed to encode map e375 with expected crc 2023-10-01T09:33:03.107 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:02 smithi190 ceph-mon[160435]: pgmap v1046: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 10217/77412 objects degraded (13.198%); 6007/77412 objects misplaced (7.760%); 6.6 MiB/s, 105 objects/s recovering 2023-10-01T09:33:03.374 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:03.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:03.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:03.490 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:03.541 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:03.747 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:03.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:03.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:03.853 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:03.884 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:04.072 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:04.075 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-10-01T09:33:04.076 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2023-10-01T09:33:04.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:04.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:04.180 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:04.217 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:04.294 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:04.299 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:04 smithi190 ceph-mon[160435]: pgmap v1047: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 8777/69543 objects degraded (12.621%); 5306/69543 objects misplaced (7.630%); 9.8 MiB/s, 156 objects/s recovering 2023-10-01T09:33:04.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:04.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:04.400 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:04.429 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:04 smithi138 ceph-mon[185886]: pgmap v1047: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 8777/69543 objects degraded (12.621%); 5306/69543 objects misplaced (7.630%); 9.8 MiB/s, 156 objects/s recovering 2023-10-01T09:33:04.429 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:04 smithi138 ceph-mon[192513]: pgmap v1047: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 8777/69543 objects degraded (12.621%); 5306/69543 objects misplaced (7.630%); 9.8 MiB/s, 156 objects/s recovering 2023-10-01T09:33:04.432 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:04.565 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:04.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:04.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:04.675 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:04.714 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:04.800 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:33:04.837 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:33:04.854 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:04.871 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:33:04.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:04.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:04.909 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:33:04.953 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:33:04.972 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:05.004 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:33:05.049 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:33:05.049 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:33:05.049 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:33:05.050 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:33:05.050 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:33:05.050 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:33:05.050 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-10-01T09:33:05.050 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:33:05.050 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:33:05.051 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:05.077 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:33:05.078 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:33:05.157 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2459725966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:05.272 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2459725966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:05.272 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:33:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:33:04] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.33.4" 2023-10-01T09:33:05.272 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2459725966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:05.281 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:05.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:05.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:05.385 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:05.415 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:05.461 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:05.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:05.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:05.564 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:05.598 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:05.644 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:05.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:05.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:05.751 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:05.777 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:05.860 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:33:05.907 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:33:05.917 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:05.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:05.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:06.022 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:06.050 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:06.152 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-10-01T09:33:06.152 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-10-01T09:33:06.161 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:06.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:06.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:06.267 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:06.325 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:06.338 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:33:06.401 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:06.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:06.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:06.433 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:06.507 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:06.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:06 smithi190 ceph-mon[160435]: osdmap e376: 8 total, 7 up, 8 in 2023-10-01T09:33:06.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:06 smithi190 ceph-mon[160435]: failed to encode map e376 with expected crc 2023-10-01T09:33:06.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:06 smithi190 ceph-mon[160435]: failed to encode map e376 with expected crc 2023-10-01T09:33:06.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:06 smithi190 ceph-mon[160435]: failed to encode map e376 with expected crc 2023-10-01T09:33:06.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:06 smithi190 ceph-mon[160435]: pgmap v1049: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 8777/69543 objects degraded (12.621%); 5306/69543 objects misplaced (7.630%); 9.2 MiB/s, 146 objects/s recovering 2023-10-01T09:33:06.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:06 smithi138 ceph-mon[185886]: osdmap e376: 8 total, 7 up, 8 in 2023-10-01T09:33:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:06 smithi138 ceph-mon[185886]: failed to encode map e376 with expected crc 2023-10-01T09:33:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:06 smithi138 ceph-mon[185886]: failed to encode map e376 with expected crc 2023-10-01T09:33:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:06 smithi138 ceph-mon[185886]: failed to encode map e376 with expected crc 2023-10-01T09:33:06.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:06 smithi138 ceph-mon[185886]: pgmap v1049: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 8777/69543 objects degraded (12.621%); 5306/69543 objects misplaced (7.630%); 9.2 MiB/s, 146 objects/s recovering 2023-10-01T09:33:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:06 smithi138 ceph-mon[192513]: osdmap e376: 8 total, 7 up, 8 in 2023-10-01T09:33:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:06 smithi138 ceph-mon[192513]: failed to encode map e376 with expected crc 2023-10-01T09:33:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:06 smithi138 ceph-mon[192513]: failed to encode map e376 with expected crc 2023-10-01T09:33:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:06 smithi138 ceph-mon[192513]: failed to encode map e376 with expected crc 2023-10-01T09:33:06.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:06 smithi138 ceph-mon[192513]: pgmap v1049: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 8777/69543 objects degraded (12.621%); 5306/69543 objects misplaced (7.630%); 9.2 MiB/s, 146 objects/s recovering 2023-10-01T09:33:06.599 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:06.769 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:06.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:06.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:06.877 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:06.973 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:07.039 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:07.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:07.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:07.142 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:07.197 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:07.282 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:07.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:07.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:07.289 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:07 smithi190 ceph-mon[160435]: failed to encode map e377 with expected crc 2023-10-01T09:33:07.289 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:07 smithi190 ceph-mon[160435]: failed to encode map e377 with expected crc 2023-10-01T09:33:07.289 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:07 smithi190 ceph-mon[160435]: failed to encode map e377 with expected crc 2023-10-01T09:33:07.289 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:07 smithi190 ceph-mon[160435]: osdmap e377: 8 total, 7 up, 8 in 2023-10-01T09:33:07.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:07 smithi138 ceph-mon[185886]: failed to encode map e377 with expected crc 2023-10-01T09:33:07.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:07 smithi138 ceph-mon[185886]: failed to encode map e377 with expected crc 2023-10-01T09:33:07.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:07 smithi138 ceph-mon[185886]: failed to encode map e377 with expected crc 2023-10-01T09:33:07.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:07 smithi138 ceph-mon[185886]: osdmap e377: 8 total, 7 up, 8 in 2023-10-01T09:33:07.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:07 smithi138 ceph-mon[192513]: failed to encode map e377 with expected crc 2023-10-01T09:33:07.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:07 smithi138 ceph-mon[192513]: failed to encode map e377 with expected crc 2023-10-01T09:33:07.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:07 smithi138 ceph-mon[192513]: failed to encode map e377 with expected crc 2023-10-01T09:33:07.327 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:07 smithi138 ceph-mon[192513]: osdmap e377: 8 total, 7 up, 8 in 2023-10-01T09:33:07.389 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:07.423 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:07.629 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:07.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:07.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:07.735 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:07.776 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:07.881 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:07.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:07.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:07.987 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:08.083 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:08.153 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:33:08.180 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:33:08.180 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:33:08.183 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:08 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8777/69543 objects degraded (12.621%), 14 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:08.184 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:08 smithi190 ceph-mon[160435]: pgmap v1051: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 8395/67272 objects degraded (12.479%); 5120/67272 objects misplaced (7.611%); 6.0 MiB/s, 96 objects/s recovering 2023-10-01T09:33:08.223 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:08.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:08.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:08.341 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:08.467 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:08 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8777/69543 objects degraded (12.621%), 14 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:08.467 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:08 smithi138 ceph-mon[192513]: pgmap v1051: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 8395/67272 objects degraded (12.479%); 5120/67272 objects misplaced (7.611%); 6.0 MiB/s, 96 objects/s recovering 2023-10-01T09:33:08.468 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:08 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8777/69543 objects degraded (12.621%), 14 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:08.469 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:08 smithi138 ceph-mon[185886]: pgmap v1051: 89 pgs: 1 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 15 active+undersized, 13 active+undersized+degraded, 2 active+remapped+backfill_wait, 57 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 8395/67272 objects degraded (12.479%); 5120/67272 objects misplaced (7.611%); 6.0 MiB/s, 96 objects/s recovering 2023-10-01T09:33:08.490 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:08.630 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:08.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:08.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:08.742 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:08.817 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:08.955 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:08.975 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:08.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:08.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:09.076 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:33:09.088 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:09.179 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:09.220 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:33:09.323 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:09.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:09.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:09.443 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:09.485 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:09 smithi138 ceph-mon[192513]: from='osd.5 [v2:172.21.15.190:6808/3401418768,v1:172.21.15.190:6809/3401418768]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:33:09.485 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:09 smithi138 ceph-mon[192513]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:33:09.485 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:09 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4047404289' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:33:09.485 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:09 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:33:09.486 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:09 smithi138 ceph-mon[185886]: from='osd.5 [v2:172.21.15.190:6808/3401418768,v1:172.21.15.190:6809/3401418768]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:33:09.486 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:09 smithi138 ceph-mon[185886]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:33:09.486 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:09 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4047404289' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:33:09.486 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:09 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:33:09.487 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:33:09 smithi190 conmon[291240]: debug 2023-10-01T09:33:09.095+0000 7fed7547e200 -1 osd.5 368 log_to_monitors {default=true} 2023-10-01T09:33:09.488 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:09 smithi190 ceph-mon[160435]: from='osd.5 [v2:172.21.15.190:6808/3401418768,v1:172.21.15.190:6809/3401418768]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:33:09.488 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:09 smithi190 ceph-mon[160435]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T09:33:09.516 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:33:09.532 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:33:09.591 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:09.740 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:33:09 smithi190 conmon[291240]: debug 2023-10-01T09:33:09.535+0000 7fed64174700 -1 osd.5 368 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T09:33:09.740 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:09 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4047404289' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:33:09.740 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:09 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:33:09.925 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:09.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:09.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:10.032 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:10.089 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:10.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:10.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[185886]: failed to encode map e378 with expected crc 2023-10-01T09:33:10.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[185886]: failed to encode map e378 with expected crc 2023-10-01T09:33:10.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[185886]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T09:33:10.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:33:10.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[185886]: osdmap e378: 8 total, 7 up, 8 in 2023-10-01T09:33:10.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[185886]: from='osd.5 [v2:172.21.15.190:6808/3401418768,v1:172.21.15.190:6809/3401418768]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:33:10.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[185886]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:33:10.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[185886]: pgmap v1053: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 15 active+undersized, 1 active+remapped+backfill_wait, 13 active+undersized+degraded, 57 active+clean; 1.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 7632/61137 objects degraded (12.483%); 4095/61137 objects misplaced (6.698%); 7.4 MiB/s, 118 objects/s recovering 2023-10-01T09:33:10.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:10.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:33:10.747 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:10.747 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[192513]: failed to encode map e378 with expected crc 2023-10-01T09:33:10.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[192513]: failed to encode map e378 with expected crc 2023-10-01T09:33:10.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[192513]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T09:33:10.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:33:10.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[192513]: osdmap e378: 8 total, 7 up, 8 in 2023-10-01T09:33:10.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[192513]: from='osd.5 [v2:172.21.15.190:6808/3401418768,v1:172.21.15.190:6809/3401418768]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:33:10.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[192513]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:33:10.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[192513]: pgmap v1053: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 15 active+undersized, 1 active+remapped+backfill_wait, 13 active+undersized+degraded, 57 active+clean; 1.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 7632/61137 objects degraded (12.483%); 4095/61137 objects misplaced (6.698%); 7.4 MiB/s, 118 objects/s recovering 2023-10-01T09:33:10.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:10.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:33:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:10 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:10 smithi190 ceph-mon[160435]: failed to encode map e378 with expected crc 2023-10-01T09:33:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:10 smithi190 ceph-mon[160435]: failed to encode map e378 with expected crc 2023-10-01T09:33:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:10 smithi190 ceph-mon[160435]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T09:33:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:10 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:33:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:10 smithi190 ceph-mon[160435]: osdmap e378: 8 total, 7 up, 8 in 2023-10-01T09:33:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:10 smithi190 ceph-mon[160435]: from='osd.5 [v2:172.21.15.190:6808/3401418768,v1:172.21.15.190:6809/3401418768]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:33:10.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:10 smithi190 ceph-mon[160435]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi190", "root=default"]}]: dispatch 2023-10-01T09:33:10.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:10 smithi190 ceph-mon[160435]: pgmap v1053: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 15 active+undersized, 1 active+remapped+backfill_wait, 13 active+undersized+degraded, 57 active+clean; 1.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 7632/61137 objects degraded (12.483%); 4095/61137 objects misplaced (6.698%); 7.4 MiB/s, 118 objects/s recovering 2023-10-01T09:33:10.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:10.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:33:11.155 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:11.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:11.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:11.259 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:11.288 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:11.292 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:33:11.292 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:33:11.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.293 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:183790 8.9 8.e1f9c39 (undecoded) ondisk+retry+write+known_if_redirected e379)", 2023-10-01T09:33:11.293 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:10.516097+0000", 2023-10-01T09:33:11.293 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.77547838300000005, 2023-10-01T09:33:11.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.77550104600000003, 2023-10-01T09:33:11.293 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:33:11.293 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:33:11.293 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:33:11.293 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:33:11.293 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:11.293 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183790 2023-10-01T09:33:11.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.294 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:33:11.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.294 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:33:11.294 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516097+0000", 2023-10-01T09:33:11.294 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:33:11.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.294 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:33:11.294 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516097+0000", 2023-10-01T09:33:11.294 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:33:11.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.295 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:33:11.295 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516095+0000", 2023-10-01T09:33:11.295 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.1649999999999995e-06 2023-10-01T09:33:11.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.295 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:33:11.295 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516104+0000", 2023-10-01T09:33:11.295 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4233000000000001e-05 2023-10-01T09:33:11.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:33:11.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516118+0000", 2023-10-01T09:33:11.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2600000000000005e-06 2023-10-01T09:33:11.296 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:33:11.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516126+0000", 2023-10-01T09:33:11.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7398999999999997e-05 2023-10-01T09:33:11.296 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:33:11.297 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516184+0000", 2023-10-01T09:33:11.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8756999999999999e-05 2023-10-01T09:33:11.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.297 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:33:11.297 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516213+0000", 2023-10-01T09:33:11.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000115995 2023-10-01T09:33:11.297 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.297 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:33:11.297 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.298 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:183811 8.8 8.36abb188 (undecoded) ondisk+write+known_if_redirected e379)", 2023-10-01T09:33:11.298 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:10.519813+0000", 2023-10-01T09:33:11.298 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.77176245899999996, 2023-10-01T09:33:11.298 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.77183139999999995, 2023-10-01T09:33:11.298 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:33:11.298 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:33:11.298 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:33:11.298 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:33:11.298 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:11.298 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183811 2023-10-01T09:33:11.298 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.299 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:33:11.299 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.299 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:33:11.299 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.519813+0000", 2023-10-01T09:33:11.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:33:11.299 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.299 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.299 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:33:11.299 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.519813+0000", 2023-10-01T09:33:11.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:33:11.299 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.300 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:33:11.300 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.519811+0000", 2023-10-01T09:33:11.300 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.238e-06 2023-10-01T09:33:11.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.300 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:33:11.300 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.519818+0000", 2023-10-01T09:33:11.300 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1250000000000001e-05 2023-10-01T09:33:11.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:33:11.301 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.519829+0000", 2023-10-01T09:33:11.301 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9419999999999998e-06 2023-10-01T09:33:11.301 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:33:11.301 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.519836+0000", 2023-10-01T09:33:11.301 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00046759699999999999 2023-10-01T09:33:11.301 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.302 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:33:11.302 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.520304+0000", 2023-10-01T09:33:11.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9529999999999994e-06 2023-10-01T09:33:11.302 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.302 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.302 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:33:11.302 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.520314+0000", 2023-10-01T09:33:11.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000500977 2023-10-01T09:33:11.302 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.302 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:33:11.303 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.303 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:183813 8.9 8.3ad4cfa9 (undecoded) ondisk+write+known_if_redirected e379)", 2023-10-01T09:33:11.303 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:10.525835+0000", 2023-10-01T09:33:11.303 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.76574011200000003, 2023-10-01T09:33:11.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.76585789500000001, 2023-10-01T09:33:11.303 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:33:11.303 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:33:11.304 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:33:11.304 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:33:11.304 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:11.304 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183813 2023-10-01T09:33:11.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.304 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:33:11.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:33:11.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525835+0000", 2023-10-01T09:33:11.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:33:11.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:33:11.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525835+0000", 2023-10-01T09:33:11.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:33:11.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:33:11.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525834+0000", 2023-10-01T09:33:11.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2539999999999998e-06 2023-10-01T09:33:11.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:33:11.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525840+0000", 2023-10-01T09:33:11.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2208000000000001e-05 2023-10-01T09:33:11.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:33:11.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525852+0000", 2023-10-01T09:33:11.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8560000000000003e-06 2023-10-01T09:33:11.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:33:11.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525858+0000", 2023-10-01T09:33:11.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1612e-05 2023-10-01T09:33:11.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:33:11.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525909+0000", 2023-10-01T09:33:11.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9480000000000003e-06 2023-10-01T09:33:11.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:33:11.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525919+0000", 2023-10-01T09:33:11.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4523000000000004e-05 2023-10-01T09:33:11.308 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.308 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:33:11.308 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.308 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:183814 8.8 8.db050808 (undecoded) ondisk+write+known_if_redirected e379)", 2023-10-01T09:33:11.309 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:10.593434+0000", 2023-10-01T09:33:11.309 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.698140808, 2023-10-01T09:33:11.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.69829634399999996, 2023-10-01T09:33:11.309 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:33:11.309 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:33:11.309 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:33:11.309 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:33:11.309 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:11.309 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183814 2023-10-01T09:33:11.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.309 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:33:11.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.310 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:33:11.310 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.593434+0000", 2023-10-01T09:33:11.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:33:11.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.310 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:33:11.310 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.593434+0000", 2023-10-01T09:33:11.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:33:11.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.310 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:33:11.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.593433+0000", 2023-10-01T09:33:11.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8989999999999998e-06 2023-10-01T09:33:11.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:33:11.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.593439+0000", 2023-10-01T09:33:11.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.8599999999999999e-06 2023-10-01T09:33:11.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:33:11.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.593448+0000", 2023-10-01T09:33:11.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2760000000000003e-06 2023-10-01T09:33:11.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:33:11.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.593455+0000", 2023-10-01T09:33:11.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4953000000000001e-05 2023-10-01T09:33:11.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:33:11.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.593500+0000", 2023-10-01T09:33:11.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.1679999999999993e-06 2023-10-01T09:33:11.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:33:11.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.593509+0000", 2023-10-01T09:33:11.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4758999999999994e-05 2023-10-01T09:33:11.313 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.313 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:33:11.313 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.314 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:183817 8.0 8.590ebaa0 (undecoded) ondisk+write+known_if_redirected e379)", 2023-10-01T09:33:11.314 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:10.607211+0000", 2023-10-01T09:33:11.314 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.68436379899999999, 2023-10-01T09:33:11.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.684564693, 2023-10-01T09:33:11.314 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:33:11.314 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:33:11.314 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:33:11.314 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:33:11.314 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:11.314 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183817 2023-10-01T09:33:11.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.315 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:33:11.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:33:11.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607211+0000", 2023-10-01T09:33:11.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:33:11.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:33:11.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607211+0000", 2023-10-01T09:33:11.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:33:11.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:33:11.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607210+0000", 2023-10-01T09:33:11.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2490000000000001e-06 2023-10-01T09:33:11.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:33:11.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607218+0000", 2023-10-01T09:33:11.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5078e-05 2023-10-01T09:33:11.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:33:11.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607233+0000", 2023-10-01T09:33:11.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0129999999999996e-06 2023-10-01T09:33:11.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:33:11.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607240+0000", 2023-10-01T09:33:11.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3373999999999993e-05 2023-10-01T09:33:11.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.318 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.318 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:33:11.318 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607303+0000", 2023-10-01T09:33:11.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1518999999999999e-05 2023-10-01T09:33:11.318 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.318 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.318 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:33:11.318 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607315+0000", 2023-10-01T09:33:11.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010360400000000001 2023-10-01T09:33:11.318 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.318 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:33:11.319 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.319 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:183822 8.8 8.f647178 (undecoded) ondisk+write+known_if_redirected e379)", 2023-10-01T09:33:11.319 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:10.620236+0000", 2023-10-01T09:33:11.319 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.671338659, 2023-10-01T09:33:11.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.671576442, 2023-10-01T09:33:11.319 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:33:11.319 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:33:11.319 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:33:11.319 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:33:11.320 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:11.320 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183822 2023-10-01T09:33:11.320 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.320 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:33:11.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.320 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:33:11.320 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.620236+0000", 2023-10-01T09:33:11.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:33:11.320 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.320 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:33:11.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.620236+0000", 2023-10-01T09:33:11.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:33:11.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:33:11.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.620235+0000", 2023-10-01T09:33:11.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0910000000000003e-06 2023-10-01T09:33:11.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:33:11.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.620239+0000", 2023-10-01T09:33:11.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.8389999999999996e-06 2023-10-01T09:33:11.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:33:11.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.620248+0000", 2023-10-01T09:33:11.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1410000000000002e-06 2023-10-01T09:33:11.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:33:11.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.620254+0000", 2023-10-01T09:33:11.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0810999999999999e-05 2023-10-01T09:33:11.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:33:11.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.620305+0000", 2023-10-01T09:33:11.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5700000000000004e-06 2023-10-01T09:33:11.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:33:11.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.620313+0000", 2023-10-01T09:33:11.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6341999999999995e-05 2023-10-01T09:33:11.323 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.324 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:33:11.324 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.324 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:183832 8.d 8.de11130d (undecoded) ondisk+write+known_if_redirected e379)", 2023-10-01T09:33:11.324 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:10.640122+0000", 2023-10-01T09:33:11.324 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.65145295599999997, 2023-10-01T09:33:11.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.65172802500000004, 2023-10-01T09:33:11.324 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:33:11.324 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:33:11.325 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:33:11.325 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:33:11.325 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:11.325 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183832 2023-10-01T09:33:11.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.325 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:33:11.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.325 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:33:11.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640122+0000", 2023-10-01T09:33:11.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:33:11.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.326 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:33:11.326 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640122+0000", 2023-10-01T09:33:11.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:33:11.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.326 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:33:11.326 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640120+0000", 2023-10-01T09:33:11.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0611000000000001e-05 2023-10-01T09:33:11.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.326 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:33:11.327 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640131+0000", 2023-10-01T09:33:11.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8621999999999999e-05 2023-10-01T09:33:11.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.327 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:33:11.327 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640149+0000", 2023-10-01T09:33:11.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1699999999999997e-06 2023-10-01T09:33:11.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.327 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:33:11.327 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640158+0000", 2023-10-01T09:33:11.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3267000000000002e-05 2023-10-01T09:33:11.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:33:11.328 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640231+0000", 2023-10-01T09:33:11.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3482999999999999e-05 2023-10-01T09:33:11.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:33:11.328 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640244+0000", 2023-10-01T09:33:11.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000122299 2023-10-01T09:33:11.328 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.329 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:33:11.329 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.329 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:183834 8.8 8.80949338 (undecoded) ondisk+write+known_if_redirected e379)", 2023-10-01T09:33:11.329 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:10.642470+0000", 2023-10-01T09:33:11.329 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.64910545799999997, 2023-10-01T09:33:11.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.64941854799999998, 2023-10-01T09:33:11.329 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:33:11.329 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:33:11.329 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:33:11.330 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:33:11.330 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:11.330 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183834 2023-10-01T09:33:11.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.330 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:33:11.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:33:11.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642470+0000", 2023-10-01T09:33:11.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:33:11.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:33:11.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642470+0000", 2023-10-01T09:33:11.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:33:11.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:33:11.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642469+0000", 2023-10-01T09:33:11.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1830000000000002e-06 2023-10-01T09:33:11.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:33:11.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642474+0000", 2023-10-01T09:33:11.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4800000000000001e-05 2023-10-01T09:33:11.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:33:11.333 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642489+0000", 2023-10-01T09:33:11.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6120000000000004e-06 2023-10-01T09:33:11.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:33:11.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642495+0000", 2023-10-01T09:33:11.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9181999999999998e-05 2023-10-01T09:33:11.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.334 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:33:11.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642545+0000", 2023-10-01T09:33:11.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3410000000000003e-06 2023-10-01T09:33:11.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:33:11.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:33:11.334 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:33:11.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642553+0000", 2023-10-01T09:33:11.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3316e-05 2023-10-01T09:33:11.335 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.335 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:33:11.335 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.335 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:33:11.335 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:33:11.335 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 8 2023-10-01T09:33:11.335 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:33:11.537 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:11.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:11.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:11.617 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:11.642 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:11.685 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:11 smithi190 ceph-mon[160435]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:33:11.685 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:11 smithi190 ceph-mon[160435]: osd.5 [v2:172.21.15.190:6808/3401418768,v1:172.21.15.190:6809/3401418768] boot 2023-10-01T09:33:11.685 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:11 smithi190 ceph-mon[160435]: osdmap e379: 8 total, 8 up, 8 in 2023-10-01T09:33:11.685 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:33:11.685 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:11 smithi190 ceph-mon[160435]: failed to encode map e379 with expected crc 2023-10-01T09:33:11.685 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:11 smithi190 ceph-mon[160435]: failed to encode map e379 with expected crc 2023-10-01T09:33:11.685 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:11 smithi190 ceph-mon[160435]: failed to encode map e379 with expected crc 2023-10-01T09:33:11.689 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:11.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:11 smithi138 ceph-mon[192513]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:33:11.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:11 smithi138 ceph-mon[192513]: osd.5 [v2:172.21.15.190:6808/3401418768,v1:172.21.15.190:6809/3401418768] boot 2023-10-01T09:33:11.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:11 smithi138 ceph-mon[192513]: osdmap e379: 8 total, 8 up, 8 in 2023-10-01T09:33:11.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:33:11.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:11 smithi138 ceph-mon[192513]: failed to encode map e379 with expected crc 2023-10-01T09:33:11.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:11 smithi138 ceph-mon[192513]: failed to encode map e379 with expected crc 2023-10-01T09:33:11.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:11 smithi138 ceph-mon[192513]: failed to encode map e379 with expected crc 2023-10-01T09:33:11.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:11 smithi138 ceph-mon[185886]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T09:33:11.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:11 smithi138 ceph-mon[185886]: osd.5 [v2:172.21.15.190:6808/3401418768,v1:172.21.15.190:6809/3401418768] boot 2023-10-01T09:33:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:11 smithi138 ceph-mon[185886]: osdmap e379: 8 total, 8 up, 8 in 2023-10-01T09:33:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T09:33:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:11 smithi138 ceph-mon[185886]: failed to encode map e379 with expected crc 2023-10-01T09:33:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:11 smithi138 ceph-mon[185886]: failed to encode map e379 with expected crc 2023-10-01T09:33:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:11 smithi138 ceph-mon[185886]: failed to encode map e379 with expected crc 2023-10-01T09:33:11.967 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:11.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:11.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:12.072 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:33:12.072 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:12.094 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:33:12.104 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:12.117 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:33:12.142 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:33:12.159 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:12.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:12.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:12.165 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:33:12.184 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:33:12.201 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:33:12.222 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:33:12.224 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:33:12.224 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:33:12.224 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:33:12.224 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:33:12.224 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:33:12.224 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:33:12.224 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:33:12.224 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:33:12.266 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:12.297 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:12.347 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:12.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:12.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:12.454 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:12.483 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:12 smithi190 ceph-mon[160435]: failed to encode map e369 with expected crc 2023-10-01T09:33:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:12 smithi190 ceph-mon[160435]: failed to encode map e379 with expected crc 2023-10-01T09:33:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:12 smithi190 ceph-mon[160435]: failed to encode map e380 with expected crc 2023-10-01T09:33:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:12 smithi190 ceph-mon[160435]: failed to encode map e380 with expected crc 2023-10-01T09:33:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:12 smithi190 ceph-mon[160435]: failed to encode map e380 with expected crc 2023-10-01T09:33:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:12 smithi190 ceph-mon[160435]: osdmap e380: 8 total, 8 up, 8 in 2023-10-01T09:33:12.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:12 smithi190 ceph-mon[160435]: failed to encode map e380 with expected crc 2023-10-01T09:33:12.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:12 smithi190 ceph-mon[160435]: pgmap v1056: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 2 active+clean+remapped, 14 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 60 active+clean; 1.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 6249/60297 objects degraded (10.364%); 4095/60297 objects misplaced (6.791%); 7.6 MiB/s, 121 objects/s recovering 2023-10-01T09:33:12.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:12 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1942363931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:12.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:12 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7632/61137 objects degraded (12.483%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:12.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[192513]: failed to encode map e369 with expected crc 2023-10-01T09:33:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[192513]: failed to encode map e379 with expected crc 2023-10-01T09:33:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[192513]: failed to encode map e380 with expected crc 2023-10-01T09:33:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[192513]: failed to encode map e380 with expected crc 2023-10-01T09:33:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[192513]: failed to encode map e380 with expected crc 2023-10-01T09:33:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[192513]: osdmap e380: 8 total, 8 up, 8 in 2023-10-01T09:33:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[192513]: failed to encode map e380 with expected crc 2023-10-01T09:33:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[192513]: pgmap v1056: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 2 active+clean+remapped, 14 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 60 active+clean; 1.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 6249/60297 objects degraded (10.364%); 4095/60297 objects misplaced (6.791%); 7.6 MiB/s, 121 objects/s recovering 2023-10-01T09:33:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1942363931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7632/61137 objects degraded (12.483%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:12.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[185886]: failed to encode map e369 with expected crc 2023-10-01T09:33:12.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[185886]: failed to encode map e379 with expected crc 2023-10-01T09:33:12.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[185886]: failed to encode map e380 with expected crc 2023-10-01T09:33:12.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[185886]: failed to encode map e380 with expected crc 2023-10-01T09:33:12.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[185886]: failed to encode map e380 with expected crc 2023-10-01T09:33:12.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[185886]: osdmap e380: 8 total, 8 up, 8 in 2023-10-01T09:33:12.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[185886]: failed to encode map e380 with expected crc 2023-10-01T09:33:12.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[185886]: pgmap v1056: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 2 active+clean+remapped, 14 active+undersized, 1 active+remapped+backfill_wait, 10 active+undersized+degraded, 60 active+clean; 1.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 6249/60297 objects degraded (10.364%); 4095/60297 objects misplaced (6.791%); 7.6 MiB/s, 121 objects/s recovering 2023-10-01T09:33:12.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1942363931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:12.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:12 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7632/61137 objects degraded (12.483%), 13 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:13.533 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:33:13.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:13 smithi190 ceph-mon[160435]: failed to encode map e381 with expected crc 2023-10-01T09:33:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:13 smithi190 ceph-mon[160435]: failed to encode map e381 with expected crc 2023-10-01T09:33:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:13 smithi190 ceph-mon[160435]: failed to encode map e381 with expected crc 2023-10-01T09:33:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:13 smithi190 ceph-mon[160435]: osdmap e381: 8 total, 8 up, 8 in 2023-10-01T09:33:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:13 smithi190 ceph-mon[160435]: failed to encode map e381 with expected crc 2023-10-01T09:33:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:13 smithi190 ceph-mon[160435]: failed to encode map e381 with expected crc 2023-10-01T09:33:13.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:13 smithi138 ceph-mon[185886]: failed to encode map e381 with expected crc 2023-10-01T09:33:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:13 smithi138 ceph-mon[185886]: failed to encode map e381 with expected crc 2023-10-01T09:33:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:13 smithi138 ceph-mon[185886]: failed to encode map e381 with expected crc 2023-10-01T09:33:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:13 smithi138 ceph-mon[185886]: osdmap e381: 8 total, 8 up, 8 in 2023-10-01T09:33:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:13 smithi138 ceph-mon[185886]: failed to encode map e381 with expected crc 2023-10-01T09:33:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:13 smithi138 ceph-mon[185886]: failed to encode map e381 with expected crc 2023-10-01T09:33:13.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:13 smithi138 ceph-mon[192513]: failed to encode map e381 with expected crc 2023-10-01T09:33:13.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:13 smithi138 ceph-mon[192513]: failed to encode map e381 with expected crc 2023-10-01T09:33:13.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:13 smithi138 ceph-mon[192513]: failed to encode map e381 with expected crc 2023-10-01T09:33:13.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:13 smithi138 ceph-mon[192513]: osdmap e381: 8 total, 8 up, 8 in 2023-10-01T09:33:13.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:13 smithi138 ceph-mon[192513]: failed to encode map e381 with expected crc 2023-10-01T09:33:13.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:13 smithi138 ceph-mon[192513]: failed to encode map e381 with expected crc 2023-10-01T09:33:14.296 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:14.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:14.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:14.401 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:14.429 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:14.462 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:14.475 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:14.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:14.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:14.524 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[185886]: failed to encode map e381 with expected crc 2023-10-01T09:33:14.524 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[185886]: osdmap e382: 8 total, 8 up, 8 in 2023-10-01T09:33:14.524 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[185886]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.524 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[185886]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.525 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[185886]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.525 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[185886]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.525 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[185886]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.525 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[185886]: pgmap v1059: 89 pgs: 1 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 9 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 67 active+clean; 1.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 327 op/s; 4818/57204 objects degraded (8.422%); 23 MiB/s, 383 objects/s recovering 2023-10-01T09:33:14.585 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:33:14.616 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:14.685 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:33:14.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:14.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:33:14.695 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:14 smithi190 ceph-mon[160435]: failed to encode map e381 with expected crc 2023-10-01T09:33:14.695 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:14 smithi190 ceph-mon[160435]: osdmap e382: 8 total, 8 up, 8 in 2023-10-01T09:33:14.695 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:14 smithi190 ceph-mon[160435]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.695 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:14 smithi190 ceph-mon[160435]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.695 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:14 smithi190 ceph-mon[160435]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.696 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:14 smithi190 ceph-mon[160435]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.696 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:14 smithi190 ceph-mon[160435]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.696 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:14 smithi190 ceph-mon[160435]: pgmap v1059: 89 pgs: 1 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 9 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 67 active+clean; 1.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 327 op/s; 4818/57204 objects degraded (8.422%); 23 MiB/s, 383 objects/s recovering 2023-10-01T09:33:14.795 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:14.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[192513]: failed to encode map e381 with expected crc 2023-10-01T09:33:14.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[192513]: osdmap e382: 8 total, 8 up, 8 in 2023-10-01T09:33:14.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[192513]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[192513]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[192513]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[192513]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[192513]: failed to encode map e382 with expected crc 2023-10-01T09:33:14.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:14 smithi138 ceph-mon[192513]: pgmap v1059: 89 pgs: 1 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 9 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 67 active+clean; 1.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 327 op/s; 4818/57204 objects degraded (8.422%); 23 MiB/s, 383 objects/s recovering 2023-10-01T09:33:14.885 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:14.896 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:33:14.896 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:33:14.896 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.896 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:184375 8.9 8:96ddfb23:::benchmark_data_smithi138_431617_object82964:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-10-01T09:33:14.896 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:14.871615+0000", 2023-10-01T09:33:14.896 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0048972709999999999, 2023-10-01T09:33:14.896 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049389289999999999, 2023-10-01T09:33:14.897 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:14.897 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:33:14.897 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:33:14.897 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:33:14.897 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:14.897 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 184375 2023-10-01T09:33:14.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.897 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:14.897 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:14.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871615+0000", 2023-10-01T09:33:14.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:14.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:14.898 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871615+0000", 2023-10-01T09:33:14.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:14.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:14.898 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871613+0000", 2023-10-01T09:33:14.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:33:14.899 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.899 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.899 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:14.899 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871623+0000", 2023-10-01T09:33:14.899 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0302999999999999e-05 2023-10-01T09:33:14.899 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.899 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.899 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:14.899 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871647+0000", 2023-10-01T09:33:14.900 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3940000000000001e-05 2023-10-01T09:33:14.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.900 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:14.900 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871658+0000", 2023-10-01T09:33:14.900 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1028e-05 2023-10-01T09:33:14.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.900 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:33:14.900 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871795+0000", 2023-10-01T09:33:14.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013745599999999999 2023-10-01T09:33:14.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:33:14.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871920+0000", 2023-10-01T09:33:14.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012444800000000001 2023-10-01T09:33:14.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,6", 2023-10-01T09:33:14.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.872042+0000", 2023-10-01T09:33:14.902 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000122361 2023-10-01T09:33:14.902 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:14.902 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:14.902 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:14.902 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.902 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:184377 8.9 8.93863c79 (undecoded) ondisk+write+known_if_redirected e383)", 2023-10-01T09:33:14.902 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:14.871676+0000", 2023-10-01T09:33:14.902 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0048354940000000001, 2023-10-01T09:33:14.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049571249999999997, 2023-10-01T09:33:14.903 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:14.903 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:33:14.903 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:33:14.903 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:33:14.903 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:14.903 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 184377 2023-10-01T09:33:14.903 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.903 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:14.903 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.904 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:14.904 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871676+0000", 2023-10-01T09:33:14.904 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:14.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.904 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.904 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:14.904 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871676+0000", 2023-10-01T09:33:14.904 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:14.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:14.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871676+0000", 2023-10-01T09:33:14.905 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:33:14.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:14.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871679+0000", 2023-10-01T09:33:14.905 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1530000000000001e-06 2023-10-01T09:33:14.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.906 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:14.906 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871686+0000", 2023-10-01T09:33:14.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1849999999999998e-06 2023-10-01T09:33:14.906 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.906 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:14.906 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871692+0000", 2023-10-01T09:33:14.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7740000000000001e-06 2023-10-01T09:33:14.907 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:14.907 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:14.907 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:14.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.907 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:184378 8.c 8:312d218e:::benchmark_data_smithi138_431617_object82967:head [delete] snapc 0=[] ondisk+write+known_if_redirected e383)", 2023-10-01T09:33:14.907 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:14.871697+0000", 2023-10-01T09:33:14.907 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0048146650000000001, 2023-10-01T09:33:14.907 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004993234, 2023-10-01T09:33:14.908 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:14.908 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:33:14.908 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:33:14.908 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:33:14.908 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:14.908 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 184378 2023-10-01T09:33:14.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.908 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:14.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.909 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:14.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871697+0000", 2023-10-01T09:33:14.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:14.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.909 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:14.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871697+0000", 2023-10-01T09:33:14.910 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:14.910 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.910 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:14.910 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871697+0000", 2023-10-01T09:33:14.910 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:33:14.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:14.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871700+0000", 2023-10-01T09:33:14.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0060000000000001e-06 2023-10-01T09:33:14.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:14.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871706+0000", 2023-10-01T09:33:14.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8529999999999997e-06 2023-10-01T09:33:14.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:14.912 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871711+0000", 2023-10-01T09:33:14.912 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0590000000000002e-06 2023-10-01T09:33:14.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.913 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:33:14.913 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871793+0000", 2023-10-01T09:33:14.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2737000000000004e-05 2023-10-01T09:33:14.913 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.913 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.913 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:33:14.913 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871919+0000", 2023-10-01T09:33:14.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012583700000000001 2023-10-01T09:33:14.913 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.913 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.913 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:33:14.913 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.872042+0000", 2023-10-01T09:33:14.914 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000122908 2023-10-01T09:33:14.914 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:14.914 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:14.914 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:14.914 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.914 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.914 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:184381 8.c 8.d849a87c (undecoded) ondisk+write+known_if_redirected e383)", 2023-10-01T09:33:14.914 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:14.871723+0000", 2023-10-01T09:33:14.914 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0047884609999999999, 2023-10-01T09:33:14.914 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050364010000000002, 2023-10-01T09:33:14.914 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:14.915 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:33:14.915 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:33:14.915 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:33:14.915 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:14.915 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 184381 2023-10-01T09:33:14.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.915 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:14.915 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.915 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:14.915 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871723+0000", 2023-10-01T09:33:14.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:14.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.916 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.916 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:14.916 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871723+0000", 2023-10-01T09:33:14.916 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:14.916 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.916 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.916 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:14.916 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871723+0000", 2023-10-01T09:33:14.916 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:33:14.916 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.916 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:14.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871726+0000", 2023-10-01T09:33:14.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5909999999999998e-06 2023-10-01T09:33:14.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:14.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871731+0000", 2023-10-01T09:33:14.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8420000000000001e-06 2023-10-01T09:33:14.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:14.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871736+0000", 2023-10-01T09:33:14.918 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0329999999999999e-06 2023-10-01T09:33:14.918 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:14.918 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:14.918 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:14.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.918 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:184383 8.9 8.cb1e9239 (undecoded) ondisk+write+known_if_redirected e383)", 2023-10-01T09:33:14.918 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:14.871741+0000", 2023-10-01T09:33:14.919 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0047706440000000001, 2023-10-01T09:33:14.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050719830000000004, 2023-10-01T09:33:14.919 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:14.919 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:33:14.919 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:33:14.919 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:33:14.919 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:14.919 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 184383 2023-10-01T09:33:14.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.919 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:14.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:14.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871741+0000", 2023-10-01T09:33:14.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:14.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:14.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871741+0000", 2023-10-01T09:33:14.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:14.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:14.921 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871741+0000", 2023-10-01T09:33:14.921 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:33:14.921 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.921 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.921 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:14.921 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871744+0000", 2023-10-01T09:33:14.921 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8669999999999998e-06 2023-10-01T09:33:14.922 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.922 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.922 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:14.922 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871749+0000", 2023-10-01T09:33:14.922 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.75e-06 2023-10-01T09:33:14.922 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:14.922 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:14.923 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:14.923 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.871755+0000", 2023-10-01T09:33:14.923 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1669999999999998e-06 2023-10-01T09:33:14.923 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:14.923 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:14.923 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:14.923 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:14.923 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:33:14.923 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 5 2023-10-01T09:33:14.923 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:33:15.323 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:33:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:33:14] "GET /metrics HTTP/1.1" 200 36268 "" "Prometheus/2.33.4" 2023-10-01T09:33:15.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:15 smithi190 ceph-mon[160435]: failed to encode map e383 with expected crc 2023-10-01T09:33:15.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:15 smithi190 ceph-mon[160435]: osdmap e383: 8 total, 8 up, 8 in 2023-10-01T09:33:15.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:15 smithi190 ceph-mon[160435]: failed to encode map e383 with expected crc 2023-10-01T09:33:15.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:15 smithi190 ceph-mon[160435]: failed to encode map e383 with expected crc 2023-10-01T09:33:15.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:15 smithi138 ceph-mon[192513]: failed to encode map e383 with expected crc 2023-10-01T09:33:15.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:15 smithi138 ceph-mon[192513]: osdmap e383: 8 total, 8 up, 8 in 2023-10-01T09:33:15.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:15 smithi138 ceph-mon[192513]: failed to encode map e383 with expected crc 2023-10-01T09:33:15.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:15 smithi138 ceph-mon[192513]: failed to encode map e383 with expected crc 2023-10-01T09:33:15.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:15 smithi138 ceph-mon[185886]: failed to encode map e383 with expected crc 2023-10-01T09:33:15.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:15 smithi138 ceph-mon[185886]: osdmap e383: 8 total, 8 up, 8 in 2023-10-01T09:33:15.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:15 smithi138 ceph-mon[185886]: failed to encode map e383 with expected crc 2023-10-01T09:33:15.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:15 smithi138 ceph-mon[185886]: failed to encode map e383 with expected crc 2023-10-01T09:33:15.841 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:15.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:15.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:15.955 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:16.013 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:16.218 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:16.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:16.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:16.267 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:33:16.339 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:16.378 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:16.420 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:16.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:16.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:16.527 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:16.639 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:16.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:16 smithi190 ceph-mon[160435]: failed to encode map e383 with expected crc 2023-10-01T09:33:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:16 smithi190 ceph-mon[160435]: pgmap v1061: 89 pgs: 1 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 329 op/s; 4346/56634 objects degraded (7.674%); 20 MiB/s, 342 objects/s recovering 2023-10-01T09:33:16.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:16 smithi138 ceph-mon[192513]: failed to encode map e383 with expected crc 2023-10-01T09:33:16.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:16 smithi138 ceph-mon[192513]: pgmap v1061: 89 pgs: 1 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 329 op/s; 4346/56634 objects degraded (7.674%); 20 MiB/s, 342 objects/s recovering 2023-10-01T09:33:16.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:16 smithi138 ceph-mon[185886]: failed to encode map e383 with expected crc 2023-10-01T09:33:16.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:16 smithi138 ceph-mon[185886]: pgmap v1061: 89 pgs: 1 peering, 2 remapped+peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 329 op/s; 4346/56634 objects degraded (7.674%); 20 MiB/s, 342 objects/s recovering 2023-10-01T09:33:16.879 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:33:16.912 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:16.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:16.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:16.963 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:33:16.963 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:33:16.963 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-01T09:33:16.963 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:33:16.964 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:33:16.964 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-01T09:33:17.019 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:17.143 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:17.325 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:17.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:17.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:17.437 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:17.484 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:17.500 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:33:17.583 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:17.593 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:17.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:17.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:17.701 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:17.706 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:17.795 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:17 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 4346/56634 objects degraded (7.674%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:17.804 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:17 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 4346/56634 objects degraded (7.674%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:17.815 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:17.816 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:17 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 4346/56634 objects degraded (7.674%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:17.992 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:18.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:18.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:18.107 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:18.174 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:18.254 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:18.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:18.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:18.358 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:33:18.401 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:18.473 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:33:18.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:18.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:33:18.585 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:33:18.636 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:33:18.650 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:18.706 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:33:18.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:18.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:33:18.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:18 smithi190 ceph-mon[160435]: pgmap v1062: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 268 op/s; 6154/55455 objects degraded (11.097%); 16 MiB/s, 260 objects/s recovering 2023-10-01T09:33:18.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:18 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/764615321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:33:18.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:18 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:33:18.810 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:18.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:18 smithi138 ceph-mon[185886]: pgmap v1062: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 268 op/s; 6154/55455 objects degraded (11.097%); 16 MiB/s, 260 objects/s recovering 2023-10-01T09:33:18.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:18 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/764615321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:33:18.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:18 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:33:18.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:18 smithi138 ceph-mon[192513]: pgmap v1062: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 268 op/s; 6154/55455 objects degraded (11.097%); 16 MiB/s, 260 objects/s recovering 2023-10-01T09:33:18.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:18 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/764615321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:33:18.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:18 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:33:18.847 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:18.900 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:18.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:18.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:19.008 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:33:19.033 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:19.074 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:33:19.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:19.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:33:19.177 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:19.215 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:19.560 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:19.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:19.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:19.668 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:19.713 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:19.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:19 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:19 smithi190 ceph-mon[160435]: failed to encode map e384 with expected crc 2023-10-01T09:33:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:19 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:33:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:19 smithi190 ceph-mon[160435]: osdmap e384: 8 total, 8 up, 8 in 2023-10-01T09:33:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:19 smithi190 ceph-mon[160435]: failed to encode map e384 with expected crc 2023-10-01T09:33:19.897 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:19 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:19.897 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:19 smithi138 ceph-mon[185886]: failed to encode map e384 with expected crc 2023-10-01T09:33:19.897 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:19 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:33:19.897 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:19 smithi138 ceph-mon[185886]: osdmap e384: 8 total, 8 up, 8 in 2023-10-01T09:33:19.897 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:19 smithi138 ceph-mon[185886]: failed to encode map e384 with expected crc 2023-10-01T09:33:19.897 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:19 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:19.898 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:19 smithi138 ceph-mon[192513]: failed to encode map e384 with expected crc 2023-10-01T09:33:19.898 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:19 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:33:19.898 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:19 smithi138 ceph-mon[192513]: osdmap e384: 8 total, 8 up, 8 in 2023-10-01T09:33:19.898 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:19 smithi138 ceph-mon[192513]: failed to encode map e384 with expected crc 2023-10-01T09:33:20.144 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:33:20.144 INFO:teuthology.orchestra.run.smithi190.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T09:33:20.145 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:33:20.518 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:20.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:20.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:20.623 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:33:20.660 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:20.901 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:20 smithi190 ceph-mon[160435]: pgmap v1064: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1017 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 6154/48696 objects degraded (12.638%); 14 MiB/s, 321 objects/s recovering 2023-10-01T09:33:20.926 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:33:20.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:20.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:33:21.033 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:21.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:20 smithi138 ceph-mon[185886]: pgmap v1064: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1017 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 6154/48696 objects degraded (12.638%); 14 MiB/s, 321 objects/s recovering 2023-10-01T09:33:21.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:20 smithi138 ceph-mon[192513]: pgmap v1064: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1017 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 6154/48696 objects degraded (12.638%); 14 MiB/s, 321 objects/s recovering 2023-10-01T09:33:21.095 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:21.216 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:21.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:21.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:21.328 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:21.404 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:21.630 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:21.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:21.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:21.737 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:21.768 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:33:21.810 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:33:21.824 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:21.871 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:33:21.913 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:33:21.922 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:21 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/572123357' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:21.923 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:21 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/572123357' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:21.955 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:33:21.982 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:21.983 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:33:21.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:21.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:22.012 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:33:22.046 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:33:22.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:21 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/572123357' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:22.059 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:33:22.060 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:33:22.060 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:33:22.060 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:33:22.060 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:33:22.060 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:33:22.060 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:33:22.060 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:33:22.099 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:22.190 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:33:22.191 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:33:22.229 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:22.540 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:33:23.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:22 smithi190 ceph-mon[160435]: pgmap v1065: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 982 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 6154/47025 objects degraded (13.087%); 0 B/s, 71 objects/s recovering 2023-10-01T09:33:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:22 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6154/48696 objects degraded (12.638%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:22 smithi190 ceph-mon[160435]: osdmap e385: 8 total, 8 up, 8 in 2023-10-01T09:33:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:22 smithi190 ceph-mon[160435]: failed to encode map e385 with expected crc 2023-10-01T09:33:23.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:22 smithi190 ceph-mon[160435]: failed to encode map e385 with expected crc 2023-10-01T09:33:23.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:22 smithi138 ceph-mon[192513]: pgmap v1065: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 982 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 6154/47025 objects degraded (13.087%); 0 B/s, 71 objects/s recovering 2023-10-01T09:33:23.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:22 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6154/48696 objects degraded (12.638%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:23.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:22 smithi138 ceph-mon[192513]: osdmap e385: 8 total, 8 up, 8 in 2023-10-01T09:33:23.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:22 smithi138 ceph-mon[192513]: failed to encode map e385 with expected crc 2023-10-01T09:33:23.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:22 smithi138 ceph-mon[192513]: failed to encode map e385 with expected crc 2023-10-01T09:33:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:22 smithi138 ceph-mon[185886]: pgmap v1065: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 982 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 6154/47025 objects degraded (13.087%); 0 B/s, 71 objects/s recovering 2023-10-01T09:33:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:22 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6154/48696 objects degraded (12.638%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:22 smithi138 ceph-mon[185886]: osdmap e385: 8 total, 8 up, 8 in 2023-10-01T09:33:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:22 smithi138 ceph-mon[185886]: failed to encode map e385 with expected crc 2023-10-01T09:33:23.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:22 smithi138 ceph-mon[185886]: failed to encode map e385 with expected crc 2023-10-01T09:33:23.659 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:23.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:23.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:23.762 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:23.840 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:24.016 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:23 smithi138 ceph-mon[192513]: failed to encode map e385 with expected crc 2023-10-01T09:33:24.016 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:23 smithi138 ceph-mon[192513]: failed to encode map e385 with expected crc 2023-10-01T09:33:24.016 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:23 smithi138 ceph-mon[192513]: failed to encode map e385 with expected crc 2023-10-01T09:33:24.017 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:23 smithi138 ceph-mon[192513]: failed to encode map e385 with expected crc 2023-10-01T09:33:24.017 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:23 smithi138 ceph-mon[192513]: osdmap e386: 8 total, 8 up, 8 in 2023-10-01T09:33:24.017 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:23 smithi138 ceph-mon[192513]: failed to encode map e386 with expected crc 2023-10-01T09:33:24.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:23 smithi138 ceph-mon[185886]: failed to encode map e385 with expected crc 2023-10-01T09:33:24.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:23 smithi138 ceph-mon[185886]: failed to encode map e385 with expected crc 2023-10-01T09:33:24.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:23 smithi138 ceph-mon[185886]: failed to encode map e385 with expected crc 2023-10-01T09:33:24.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:23 smithi138 ceph-mon[185886]: failed to encode map e385 with expected crc 2023-10-01T09:33:24.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:23 smithi138 ceph-mon[185886]: osdmap e386: 8 total, 8 up, 8 in 2023-10-01T09:33:24.022 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:23 smithi138 ceph-mon[185886]: failed to encode map e386 with expected crc 2023-10-01T09:33:24.041 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:24.045 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:33:24.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:24.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:24.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:23 smithi190 ceph-mon[160435]: failed to encode map e385 with expected crc 2023-10-01T09:33:24.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:23 smithi190 ceph-mon[160435]: failed to encode map e385 with expected crc 2023-10-01T09:33:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:23 smithi190 ceph-mon[160435]: failed to encode map e385 with expected crc 2023-10-01T09:33:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:23 smithi190 ceph-mon[160435]: failed to encode map e385 with expected crc 2023-10-01T09:33:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:23 smithi190 ceph-mon[160435]: osdmap e386: 8 total, 8 up, 8 in 2023-10-01T09:33:24.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:23 smithi190 ceph-mon[160435]: failed to encode map e386 with expected crc 2023-10-01T09:33:24.147 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:24.228 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:24.373 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:24.378 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:24.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:24.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:24.489 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:24.521 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:24.618 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:24.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:24.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:24.732 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:24.781 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:24.862 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:24.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:24.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:24.971 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:25.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:24 smithi138 ceph-mon[185886]: failed to encode map e386 with expected crc 2023-10-01T09:33:25.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:24 smithi138 ceph-mon[185886]: failed to encode map e386 with expected crc 2023-10-01T09:33:25.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:24 smithi138 ceph-mon[185886]: pgmap v1068: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 851 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 6154/40698 objects degraded (15.121%); 0 B/s, 140 objects/s recovering 2023-10-01T09:33:25.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:24 smithi138 ceph-mon[192513]: failed to encode map e386 with expected crc 2023-10-01T09:33:25.021 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:24 smithi138 ceph-mon[192513]: failed to encode map e386 with expected crc 2023-10-01T09:33:25.021 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:24 smithi138 ceph-mon[192513]: pgmap v1068: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 851 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 6154/40698 objects degraded (15.121%); 0 B/s, 140 objects/s recovering 2023-10-01T09:33:25.021 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:33:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:33:24] "GET /metrics HTTP/1.1" 200 36268 "" "Prometheus/2.33.4" 2023-10-01T09:33:25.030 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:25.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:24 smithi190 ceph-mon[160435]: failed to encode map e386 with expected crc 2023-10-01T09:33:25.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:24 smithi190 ceph-mon[160435]: failed to encode map e386 with expected crc 2023-10-01T09:33:25.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:24 smithi190 ceph-mon[160435]: pgmap v1068: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 851 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 6154/40698 objects degraded (15.121%); 0 B/s, 140 objects/s recovering 2023-10-01T09:33:25.255 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:25.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:25.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:25.364 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:25.417 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:25.535 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:25.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:25.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:25.645 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:25.671 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:25.713 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:25.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:25.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:25.817 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:25.862 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:25.918 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:33:25.918 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-01T09:33:25.947 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:25.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:25.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:26.017 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:33:26.072 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:26.198 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:26.227 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:26.339 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:26.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:26.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:26.448 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:26.514 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:26.515 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:33:26.515 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:26 smithi138 ceph-mon[185886]: pgmap v1069: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 841 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 5331/40215 objects degraded (13.256%); 0 B/s, 105 objects/s recovering 2023-10-01T09:33:26.516 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:26.516 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:33:26.516 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:26 smithi138 ceph-mon[192513]: pgmap v1069: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 841 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 5331/40215 objects degraded (13.256%); 0 B/s, 105 objects/s recovering 2023-10-01T09:33:26.520 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:26.717 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:26.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:26.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:26.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:33:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:26 smithi190 ceph-mon[160435]: pgmap v1069: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 841 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 5331/40215 objects degraded (13.256%); 0 B/s, 105 objects/s recovering 2023-10-01T09:33:26.821 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:26.935 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:27.125 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:27.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:27.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:27.245 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:27.306 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:27.379 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:27.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:27.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:27.486 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:27.522 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:27.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:27 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5331/40215 objects degraded (13.256%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:27.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:27 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/884030803' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:33:27.563 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:27.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:27.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:27.668 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:27.698 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:27.741 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:27.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:27.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:27.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:27 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5331/40215 objects degraded (13.256%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:27.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:27 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/884030803' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:33:27.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:27 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5331/40215 objects degraded (13.256%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:27.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:27 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/884030803' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:33:27.845 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:27.908 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:28.184 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:28.191 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:28.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:28.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:28.244 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:28.293 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:28.364 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:28.509 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:33:28.636 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:28.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:28.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:28.732 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:28 smithi138 ceph-mon[192513]: pgmap v1070: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 807 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 5331/38604 objects degraded (13.809%); 0 B/s, 98 objects/s recovering 2023-10-01T09:33:28.732 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:28 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:33:28.732 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/884030803' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:33:28.733 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:28 smithi138 ceph-mon[192513]: osdmap e387: 8 total, 8 up, 8 in 2023-10-01T09:33:28.733 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:28 smithi138 ceph-mon[185886]: pgmap v1070: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 807 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 5331/38604 objects degraded (13.809%); 0 B/s, 98 objects/s recovering 2023-10-01T09:33:28.733 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:28 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:33:28.733 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/884030803' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:33:28.733 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:28 smithi138 ceph-mon[185886]: osdmap e387: 8 total, 8 up, 8 in 2023-10-01T09:33:28.752 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:28.785 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:28 smithi190 ceph-mon[160435]: pgmap v1070: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 807 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 5331/38604 objects degraded (13.809%); 0 B/s, 98 objects/s recovering 2023-10-01T09:33:28.785 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:28 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:33:28.785 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/884030803' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:33:28.785 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:28 smithi190 ceph-mon[160435]: osdmap e387: 8 total, 8 up, 8 in 2023-10-01T09:33:28.786 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:28.858 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:28.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:28.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:28.963 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:29.019 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:29.220 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:33:29.221 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:33:29.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:29 smithi190 ceph-mon[160435]: failed to encode map e387 with expected crc 2023-10-01T09:33:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:29 smithi190 ceph-mon[160435]: failed to encode map e387 with expected crc 2023-10-01T09:33:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:29 smithi190 ceph-mon[160435]: failed to encode map e387 with expected crc 2023-10-01T09:33:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:29 smithi190 ceph-mon[160435]: failed to encode map e387 with expected crc 2023-10-01T09:33:29.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:29 smithi138 ceph-mon[192513]: failed to encode map e387 with expected crc 2023-10-01T09:33:29.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:29 smithi138 ceph-mon[192513]: failed to encode map e387 with expected crc 2023-10-01T09:33:29.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:29 smithi138 ceph-mon[192513]: failed to encode map e387 with expected crc 2023-10-01T09:33:29.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:29 smithi138 ceph-mon[192513]: failed to encode map e387 with expected crc 2023-10-01T09:33:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:29 smithi138 ceph-mon[185886]: failed to encode map e387 with expected crc 2023-10-01T09:33:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:29 smithi138 ceph-mon[185886]: failed to encode map e387 with expected crc 2023-10-01T09:33:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:29 smithi138 ceph-mon[185886]: failed to encode map e387 with expected crc 2023-10-01T09:33:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:29 smithi138 ceph-mon[185886]: failed to encode map e387 with expected crc 2023-10-01T09:33:30.773 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:30.779 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:30 smithi138 ceph-mon[192513]: failed to encode map e388 with expected crc 2023-10-01T09:33:30.780 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:30 smithi138 ceph-mon[192513]: osdmap e388: 8 total, 8 up, 8 in 2023-10-01T09:33:30.780 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:30 smithi138 ceph-mon[192513]: failed to encode map e388 with expected crc 2023-10-01T09:33:30.780 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:30 smithi138 ceph-mon[192513]: failed to encode map e388 with expected crc 2023-10-01T09:33:30.780 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:30 smithi138 ceph-mon[192513]: pgmap v1073: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 663 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 4436/31695 objects degraded (13.996%); 0 B/s, 119 objects/s recovering 2023-10-01T09:33:30.780 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:30 smithi138 ceph-mon[185886]: failed to encode map e388 with expected crc 2023-10-01T09:33:30.780 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:30 smithi138 ceph-mon[185886]: osdmap e388: 8 total, 8 up, 8 in 2023-10-01T09:33:30.781 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:30 smithi138 ceph-mon[185886]: failed to encode map e388 with expected crc 2023-10-01T09:33:30.781 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:30 smithi138 ceph-mon[185886]: failed to encode map e388 with expected crc 2023-10-01T09:33:30.781 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:30 smithi138 ceph-mon[185886]: pgmap v1073: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 663 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 4436/31695 objects degraded (13.996%); 0 B/s, 119 objects/s recovering 2023-10-01T09:33:30.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:30.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:30 smithi190 ceph-mon[160435]: failed to encode map e388 with expected crc 2023-10-01T09:33:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:30 smithi190 ceph-mon[160435]: osdmap e388: 8 total, 8 up, 8 in 2023-10-01T09:33:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:30 smithi190 ceph-mon[160435]: failed to encode map e388 with expected crc 2023-10-01T09:33:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:30 smithi190 ceph-mon[160435]: failed to encode map e388 with expected crc 2023-10-01T09:33:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:30 smithi190 ceph-mon[160435]: pgmap v1073: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 663 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 4436/31695 objects degraded (13.996%); 0 B/s, 119 objects/s recovering 2023-10-01T09:33:30.882 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:30.939 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:31.019 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:31.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:31.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:31.135 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:31.213 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:31 smithi190 ceph-mon[160435]: failed to encode map e389 with expected crc 2023-10-01T09:33:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:31 smithi190 ceph-mon[160435]: osdmap e389: 8 total, 8 up, 8 in 2023-10-01T09:33:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:31 smithi190 ceph-mon[160435]: failed to encode map e389 with expected crc 2023-10-01T09:33:31.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:31 smithi138 ceph-mon[192513]: failed to encode map e389 with expected crc 2023-10-01T09:33:31.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:31 smithi138 ceph-mon[192513]: osdmap e389: 8 total, 8 up, 8 in 2023-10-01T09:33:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:31 smithi138 ceph-mon[192513]: failed to encode map e389 with expected crc 2023-10-01T09:33:31.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:31 smithi138 ceph-mon[185886]: failed to encode map e389 with expected crc 2023-10-01T09:33:31.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:31 smithi138 ceph-mon[185886]: osdmap e389: 8 total, 8 up, 8 in 2023-10-01T09:33:31.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:31 smithi138 ceph-mon[185886]: failed to encode map e389 with expected crc 2023-10-01T09:33:32.435 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:32.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:32.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:32.519 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:33:32.550 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:32.609 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:32 smithi138 ceph-mon[192513]: failed to encode map e389 with expected crc 2023-10-01T09:33:32.610 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:32 smithi138 ceph-mon[192513]: pgmap v1075: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 643 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 4436/30738 objects degraded (14.432%); 0 B/s, 0 objects/s recovering 2023-10-01T09:33:32.610 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:32 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 4436/31695 objects degraded (13.996%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:32.610 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:32 smithi138 ceph-mon[185886]: failed to encode map e389 with expected crc 2023-10-01T09:33:32.610 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:32 smithi138 ceph-mon[185886]: pgmap v1075: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 643 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 4436/30738 objects degraded (14.432%); 0 B/s, 0 objects/s recovering 2023-10-01T09:33:32.610 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:32 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 4436/31695 objects degraded (13.996%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:32.627 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:32.871 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:32.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:32.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:32.976 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:33:33.012 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:32 smithi190 ceph-mon[160435]: failed to encode map e389 with expected crc 2023-10-01T09:33:33.012 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:32 smithi190 ceph-mon[160435]: pgmap v1075: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 643 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 4436/30738 objects degraded (14.432%); 0 B/s, 0 objects/s recovering 2023-10-01T09:33:33.013 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:32 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 4436/31695 objects degraded (13.996%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:33.018 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:33.093 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:33:33.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:33.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:33:33.198 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:33.281 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:33.344 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:33:33.364 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:33.394 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:33:33.421 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:33.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:33.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:33.435 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:33:33.491 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:33:33.527 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:33:33.535 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:33.574 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:33:33.609 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:33.610 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:33:33.616 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.0 to 1 (10000) 2023-10-01T09:33:33.640 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:33:33.646 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:33:33.646 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:33:33.647 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:33:33.647 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:33:33.647 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:33:33.647 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:33:33.647 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:33:33.647 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:33:33.954 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:33 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4073933037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:33.954 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:33 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2262040888' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:33:33.954 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:33 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:33:33.954 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:33 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4073933037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:33.954 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:33 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2262040888' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:33:33.954 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:33 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:33:34.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:33 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4073933037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:34.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:33 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2262040888' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:33:34.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:33 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:33:34.419 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:33:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:34 smithi138 ceph-mon[192513]: failed to encode map e390 with expected crc 2023-10-01T09:33:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:34 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:33:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:34 smithi138 ceph-mon[192513]: osdmap e390: 8 total, 8 up, 8 in 2023-10-01T09:33:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:34 smithi138 ceph-mon[192513]: failed to encode map e390 with expected crc 2023-10-01T09:33:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:34 smithi138 ceph-mon[192513]: failed to encode map e390 with expected crc 2023-10-01T09:33:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:34 smithi138 ceph-mon[192513]: failed to encode map e390 with expected crc 2023-10-01T09:33:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:34 smithi138 ceph-mon[192513]: pgmap v1077: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 510 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 4436/24330 objects degraded (18.233%); 0 B/s, 103 objects/s recovering 2023-10-01T09:33:34.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:34 smithi138 ceph-mon[185886]: failed to encode map e390 with expected crc 2023-10-01T09:33:34.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:34 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:33:34.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:34 smithi138 ceph-mon[185886]: osdmap e390: 8 total, 8 up, 8 in 2023-10-01T09:33:34.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:34 smithi138 ceph-mon[185886]: failed to encode map e390 with expected crc 2023-10-01T09:33:34.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:34 smithi138 ceph-mon[185886]: failed to encode map e390 with expected crc 2023-10-01T09:33:34.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:34 smithi138 ceph-mon[185886]: failed to encode map e390 with expected crc 2023-10-01T09:33:34.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:34 smithi138 ceph-mon[185886]: pgmap v1077: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 510 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 4436/24330 objects degraded (18.233%); 0 B/s, 103 objects/s recovering 2023-10-01T09:33:35.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:34 smithi190 ceph-mon[160435]: failed to encode map e390 with expected crc 2023-10-01T09:33:35.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:34 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:33:35.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:34 smithi190 ceph-mon[160435]: osdmap e390: 8 total, 8 up, 8 in 2023-10-01T09:33:35.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:34 smithi190 ceph-mon[160435]: failed to encode map e390 with expected crc 2023-10-01T09:33:35.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:34 smithi190 ceph-mon[160435]: failed to encode map e390 with expected crc 2023-10-01T09:33:35.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:34 smithi190 ceph-mon[160435]: failed to encode map e390 with expected crc 2023-10-01T09:33:35.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:34 smithi190 ceph-mon[160435]: pgmap v1077: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 510 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 4436/24330 objects degraded (18.233%); 0 B/s, 103 objects/s recovering 2023-10-01T09:33:35.322 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:33:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:33:34] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.33.4" 2023-10-01T09:33:35.330 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:35.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:35.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:35.440 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:35.467 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:36.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:36 smithi190 ceph-mon[160435]: pgmap v1078: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 489 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 426 op/s; 4436/23349 objects degraded (18.999%); 0 B/s, 140 objects/s recovering 2023-10-01T09:33:36.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:36 smithi138 ceph-mon[185886]: pgmap v1078: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 489 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 426 op/s; 4436/23349 objects degraded (18.999%); 0 B/s, 140 objects/s recovering 2023-10-01T09:33:36.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:36 smithi138 ceph-mon[192513]: pgmap v1078: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 489 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 426 op/s; 4436/23349 objects degraded (18.999%); 0 B/s, 140 objects/s recovering 2023-10-01T09:33:36.729 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:36.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:36.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:36.762 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-01T09:33:36.844 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:36.877 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:36.906 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:36.948 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:36.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:36.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:37.055 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:37.148 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:37.377 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:37.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:37.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:37.466 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:33:37.466 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:33:37.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.466 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:197657 8.1 8:82d5b874:::benchmark_data_smithi138_431617_object96246:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-01T09:33:37.467 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:37.446112+0000", 2023-10-01T09:33:37.467 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.010566936000000001, 2023-10-01T09:33:37.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010597895, 2023-10-01T09:33:37.467 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:37.467 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:33:37.467 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:33:37.467 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:33:37.467 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:37.467 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 197657 2023-10-01T09:33:37.467 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.467 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:37.467 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:37.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.446112+0000", 2023-10-01T09:33:37.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:37.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.446112+0000", 2023-10-01T09:33:37.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:37.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.446110+0000", 2023-10-01T09:33:37.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:33:37.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.469 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:37.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.446116+0000", 2023-10-01T09:33:37.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6309999999999997e-06 2023-10-01T09:33:37.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.469 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:37.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.446132+0000", 2023-10-01T09:33:37.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5556000000000001e-05 2023-10-01T09:33:37.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:37.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.446140+0000", 2023-10-01T09:33:37.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1950000000000003e-06 2023-10-01T09:33:37.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:33:37.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.446192+0000", 2023-10-01T09:33:37.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1518999999999999e-05 2023-10-01T09:33:37.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:33:37.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.446313+0000", 2023-10-01T09:33:37.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121184 2023-10-01T09:33:37.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:33:37.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.446420+0000", 2023-10-01T09:33:37.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106779 2023-10-01T09:33:37.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:33:37.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451125+0000", 2023-10-01T09:33:37.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047052680000000003 2023-10-01T09:33:37.472 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.472 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:37.472 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.472 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:197659 8.7 8:e8f8aef7:::benchmark_data_smithi138_431617_object96248:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-01T09:33:37.472 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:37.451530+0000", 2023-10-01T09:33:37.472 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0051481740000000002, 2023-10-01T09:33:37.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0052743369999999996, 2023-10-01T09:33:37.472 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:37.472 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:33:37.473 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:33:37.473 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:33:37.473 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:37.473 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 197659 2023-10-01T09:33:37.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.473 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:37.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:37.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451530+0000", 2023-10-01T09:33:37.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:37.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451530+0000", 2023-10-01T09:33:37.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:37.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451529+0000", 2023-10-01T09:33:37.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:33:37.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:37.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451537+0000", 2023-10-01T09:33:37.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2539999999999992e-06 2023-10-01T09:33:37.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.475 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:37.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451554+0000", 2023-10-01T09:33:37.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6917000000000001e-05 2023-10-01T09:33:37.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:37.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451564+0000", 2023-10-01T09:33:37.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8039999999999996e-06 2023-10-01T09:33:37.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:33:37.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451653+0000", 2023-10-01T09:33:37.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8962999999999995e-05 2023-10-01T09:33:37.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:33:37.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451789+0000", 2023-10-01T09:33:37.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013596699999999999 2023-10-01T09:33:37.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:33:37.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451913+0000", 2023-10-01T09:33:37.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012403800000000001 2023-10-01T09:33:37.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:33:37.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.456459+0000", 2023-10-01T09:33:37.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004546502 2023-10-01T09:33:37.479 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.479 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:37.479 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.479 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:197660 8.0 8:03d8004c:::benchmark_data_smithi138_431617_object96249:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-01T09:33:37.480 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:37.451583+0000", 2023-10-01T09:33:37.480 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0050958829999999998, 2023-10-01T09:33:37.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0053024309999999998, 2023-10-01T09:33:37.480 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:37.480 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:33:37.480 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:33:37.480 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:33:37.480 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:37.480 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 197660 2023-10-01T09:33:37.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.480 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:37.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:37.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451583+0000", 2023-10-01T09:33:37.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:37.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451583+0000", 2023-10-01T09:33:37.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:37.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451582+0000", 2023-10-01T09:33:37.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:33:37.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:37.482 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451585+0000", 2023-10-01T09:33:37.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.118e-06 2023-10-01T09:33:37.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:37.482 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451592+0000", 2023-10-01T09:33:37.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6930000000000001e-06 2023-10-01T09:33:37.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:37.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451597+0000", 2023-10-01T09:33:37.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1379999999999998e-06 2023-10-01T09:33:37.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:33:37.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.456456+0000", 2023-10-01T09:33:37.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004858612 2023-10-01T09:33:37.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:33:37.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.456576+0000", 2023-10-01T09:33:37.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012023099999999999 2023-10-01T09:33:37.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-10-01T09:33:37.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.456692+0000", 2023-10-01T09:33:37.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011602700000000001 2023-10-01T09:33:37.484 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.484 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:37.484 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.485 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:197663 8.1 8:8da289e0:::benchmark_data_smithi138_431617_object96252:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-01T09:33:37.485 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:37.451610+0000", 2023-10-01T09:33:37.485 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0050687129999999999, 2023-10-01T09:33:37.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0053479720000000003, 2023-10-01T09:33:37.485 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:37.485 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:33:37.485 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:33:37.485 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:33:37.485 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:37.485 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 197663 2023-10-01T09:33:37.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.485 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:37.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:37.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451610+0000", 2023-10-01T09:33:37.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:37.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451610+0000", 2023-10-01T09:33:37.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:37.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451609+0000", 2023-10-01T09:33:37.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:33:37.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:37.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451612+0000", 2023-10-01T09:33:37.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6649999999999999e-06 2023-10-01T09:33:37.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:37.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451619+0000", 2023-10-01T09:33:37.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5359999999999998e-06 2023-10-01T09:33:37.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:37.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451625+0000", 2023-10-01T09:33:37.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.981e-06 2023-10-01T09:33:37.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:33:37.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.456341+0000", 2023-10-01T09:33:37.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047163869999999998 2023-10-01T09:33:37.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:33:37.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.456575+0000", 2023-10-01T09:33:37.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00023398399999999999 2023-10-01T09:33:37.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,2", 2023-10-01T09:33:37.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.456686+0000", 2023-10-01T09:33:37.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110914 2023-10-01T09:33:37.489 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.489 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:37.489 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.490 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:197665 8.1 8.3d493491 (undecoded) ondisk+write+known_if_redirected e390)", 2023-10-01T09:33:37.490 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:37.451635+0000", 2023-10-01T09:33:37.490 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0050439780000000002, 2023-10-01T09:33:37.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054007270000000001, 2023-10-01T09:33:37.490 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:37.490 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:33:37.490 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:33:37.490 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:33:37.490 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:37.490 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 197665 2023-10-01T09:33:37.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.491 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:37.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:37.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451635+0000", 2023-10-01T09:33:37.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:37.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451635+0000", 2023-10-01T09:33:37.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:37.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451634+0000", 2023-10-01T09:33:37.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:33:37.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:37.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451637+0000", 2023-10-01T09:33:37.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5840000000000002e-06 2023-10-01T09:33:37.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:37.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451643+0000", 2023-10-01T09:33:37.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4309999999999999e-06 2023-10-01T09:33:37.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:37.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451648+0000", 2023-10-01T09:33:37.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.51e-06 2023-10-01T09:33:37.493 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.493 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:37.493 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.493 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:197667 8.b 8:d8a71ad9:::benchmark_data_smithi138_431617_object96256:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-01T09:33:37.494 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:37.451652+0000", 2023-10-01T09:33:37.494 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0050262989999999997, 2023-10-01T09:33:37.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054408140000000004, 2023-10-01T09:33:37.494 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:37.494 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:33:37.494 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:33:37.494 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:33:37.494 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:37.494 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 197667 2023-10-01T09:33:37.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.494 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:37.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:37.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451652+0000", 2023-10-01T09:33:37.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:37.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451652+0000", 2023-10-01T09:33:37.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:37.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451652+0000", 2023-10-01T09:33:37.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:33:37.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:37.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451654+0000", 2023-10-01T09:33:37.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2290000000000002e-06 2023-10-01T09:33:37.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:37.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451659+0000", 2023-10-01T09:33:37.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9840000000000001e-06 2023-10-01T09:33:37.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:37.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451664+0000", 2023-10-01T09:33:37.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2349999999999999e-06 2023-10-01T09:33:37.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:33:37.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451765+0000", 2023-10-01T09:33:37.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000100559 2023-10-01T09:33:37.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:33:37.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451877+0000", 2023-10-01T09:33:37.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011254799999999999 2023-10-01T09:33:37.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:33:37.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451989+0000", 2023-10-01T09:33:37.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111072 2023-10-01T09:33:37.498 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.498 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:37.498 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.499 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:197668 8.1 8.27980001 (undecoded) ondisk+write+known_if_redirected e390)", 2023-10-01T09:33:37.499 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:37.451677+0000", 2023-10-01T09:33:37.499 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0050019060000000004, 2023-10-01T09:33:37.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054947700000000004, 2023-10-01T09:33:37.499 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:37.499 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:33:37.499 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:33:37.499 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:33:37.499 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:37.499 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 197668 2023-10-01T09:33:37.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.500 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:37.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:37.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451677+0000", 2023-10-01T09:33:37.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:37.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451677+0000", 2023-10-01T09:33:37.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:37.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451676+0000", 2023-10-01T09:33:37.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:33:37.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:37.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451679+0000", 2023-10-01T09:33:37.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5890000000000001e-06 2023-10-01T09:33:37.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:37.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451685+0000", 2023-10-01T09:33:37.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9179999999999999e-06 2023-10-01T09:33:37.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:37.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451690+0000", 2023-10-01T09:33:37.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0209999999999999e-06 2023-10-01T09:33:37.502 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.502 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:37.502 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.503 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:197669 8.b 8:d2434b79:::benchmark_data_smithi138_431617_object96258:head [delete] snapc 0=[] ondisk+write+known_if_redirected e390)", 2023-10-01T09:33:37.503 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:37.451694+0000", 2023-10-01T09:33:37.503 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0049844249999999998, 2023-10-01T09:33:37.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0055477010000000004, 2023-10-01T09:33:37.503 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:37.503 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:33:37.503 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:33:37.503 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:33:37.503 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:37.503 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 197669 2023-10-01T09:33:37.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.504 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:37.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:37.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451694+0000", 2023-10-01T09:33:37.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:37.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451694+0000", 2023-10-01T09:33:37.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:37.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451694+0000", 2023-10-01T09:33:37.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:33:37.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:37.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451696+0000", 2023-10-01T09:33:37.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3599999999999999e-06 2023-10-01T09:33:37.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:37.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451701+0000", 2023-10-01T09:33:37.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3650000000000003e-06 2023-10-01T09:33:37.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:37.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451705+0000", 2023-10-01T09:33:37.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0160000000000002e-06 2023-10-01T09:33:37.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:33:37.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.456160+0000", 2023-10-01T09:33:37.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044545380000000001 2023-10-01T09:33:37.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:33:37.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.456322+0000", 2023-10-01T09:33:37.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016166600000000001 2023-10-01T09:33:37.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:33:37.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.456458+0000", 2023-10-01T09:33:37.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000135833 2023-10-01T09:33:37.507 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.507 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:37.507 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.508 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:197672 8.b 8.c606a0bb (undecoded) ondisk+write+known_if_redirected e390)", 2023-10-01T09:33:37.508 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:37.451709+0000", 2023-10-01T09:33:37.508 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0049691179999999998, 2023-10-01T09:33:37.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056047309999999999, 2023-10-01T09:33:37.508 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:37.508 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:33:37.508 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:33:37.508 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:33:37.508 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:33:37.508 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 197672 2023-10-01T09:33:37.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.509 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:37.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:37.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451709+0000", 2023-10-01T09:33:37.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:37.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451709+0000", 2023-10-01T09:33:37.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:37.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451709+0000", 2023-10-01T09:33:37.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:33:37.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:37.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451711+0000", 2023-10-01T09:33:37.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3250000000000002e-06 2023-10-01T09:33:37.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:37.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451717+0000", 2023-10-01T09:33:37.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6670000000000001e-06 2023-10-01T09:33:37.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:37.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.451722+0000", 2023-10-01T09:33:37.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8949999999999997e-06 2023-10-01T09:33:37.511 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.511 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:37.511 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.512 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.45674.0:197657 8.1 e390/326)", 2023-10-01T09:33:37.512 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:37.457148+0000", 2023-10-01T09:33:37.512 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 4294967295.9995308, 2023-10-01T09:33:37.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00022076800000000001, 2023-10-01T09:33:37.512 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:37.512 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:33:37.512 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:37.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:37.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.457148+0000", 2023-10-01T09:33:37.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:37.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.457148+0000", 2023-10-01T09:33:37.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:37.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.457146+0000", 2023-10-01T09:33:37.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:33:37.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:37.514 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.457155+0000", 2023-10-01T09:33:37.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2800000000000003e-06 2023-10-01T09:33:37.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:37.514 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.457169+0000", 2023-10-01T09:33:37.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.454e-05 2023-10-01T09:33:37.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:37.514 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.457178+0000", 2023-10-01T09:33:37.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7029999999999995e-06 2023-10-01T09:33:37.515 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.515 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:37.515 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.515 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.45674.0:197657 8.1 e390/326)", 2023-10-01T09:33:37.515 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:37.457368+0000", 2023-10-01T09:33:37.515 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 4294967295.9993105, 2023-10-01T09:33:37.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8109999999999998e-05, 2023-10-01T09:33:37.515 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:33:37.515 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:33:37.515 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:33:37.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:33:37.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.457368+0000", 2023-10-01T09:33:37.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:33:37.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.457368+0000", 2023-10-01T09:33:37.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:33:37.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:33:37.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.457367+0000", 2023-10-01T09:33:37.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:33:37.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:33:37.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.457372+0000", 2023-10-01T09:33:37.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0390000000000004e-06 2023-10-01T09:33:37.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:33:37.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.457381+0000", 2023-10-01T09:33:37.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0629999999999995e-06 2023-10-01T09:33:37.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:33:37.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:33:37.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:33:37.518 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:37.457388+0000", 2023-10-01T09:33:37.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3659999999999997e-06 2023-10-01T09:33:37.518 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.518 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:33:37.518 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.518 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:33:37.518 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:33:37.518 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 11 2023-10-01T09:33:37.518 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:33:37.519 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:33:37.550 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:37.621 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:33:37.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:37.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:33:37.699 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:37 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 4436/23349 objects degraded (18.999%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:37.699 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:37 smithi138 ceph-mon[185886]: osdmap e391: 8 total, 8 up, 8 in 2023-10-01T09:33:37.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:37 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 4436/23349 objects degraded (18.999%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:37.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:37 smithi138 ceph-mon[192513]: osdmap e391: 8 total, 8 up, 8 in 2023-10-01T09:33:37.729 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:37.758 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:37.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:37 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 4436/23349 objects degraded (18.999%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:37.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:37 smithi190 ceph-mon[160435]: osdmap e391: 8 total, 8 up, 8 in 2023-10-01T09:33:37.938 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:33:38.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:38 smithi190 ceph-mon[160435]: failed to encode map e391 with expected crc 2023-10-01T09:33:38.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:38 smithi190 ceph-mon[160435]: failed to encode map e391 with expected crc 2023-10-01T09:33:38.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:38 smithi190 ceph-mon[160435]: failed to encode map e391 with expected crc 2023-10-01T09:33:38.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:38 smithi190 ceph-mon[160435]: pgmap v1080: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 452 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 4436/21567 objects degraded (20.568%); 0 B/s, 121 objects/s recovering 2023-10-01T09:33:38.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:38 smithi190 ceph-mon[160435]: osdmap e392: 8 total, 8 up, 8 in 2023-10-01T09:33:38.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:38 smithi138 ceph-mon[192513]: failed to encode map e391 with expected crc 2023-10-01T09:33:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:38 smithi138 ceph-mon[192513]: failed to encode map e391 with expected crc 2023-10-01T09:33:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:38 smithi138 ceph-mon[192513]: failed to encode map e391 with expected crc 2023-10-01T09:33:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:38 smithi138 ceph-mon[192513]: pgmap v1080: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 452 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 4436/21567 objects degraded (20.568%); 0 B/s, 121 objects/s recovering 2023-10-01T09:33:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:38 smithi138 ceph-mon[192513]: osdmap e392: 8 total, 8 up, 8 in 2023-10-01T09:33:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:38 smithi138 ceph-mon[185886]: failed to encode map e391 with expected crc 2023-10-01T09:33:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:38 smithi138 ceph-mon[185886]: failed to encode map e391 with expected crc 2023-10-01T09:33:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:38 smithi138 ceph-mon[185886]: failed to encode map e391 with expected crc 2023-10-01T09:33:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:38 smithi138 ceph-mon[185886]: pgmap v1080: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 452 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 449 op/s; 4436/21567 objects degraded (20.568%); 0 B/s, 121 objects/s recovering 2023-10-01T09:33:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:38 smithi138 ceph-mon[185886]: osdmap e392: 8 total, 8 up, 8 in 2023-10-01T09:33:39.015 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:33:39.161 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:39.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:39.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:39.280 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:39.309 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:39.370 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:39.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:39.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:39.394 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:39.476 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:39.480 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:39.518 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:39.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:39 smithi138 ceph-mon[185886]: failed to encode map e392 with expected crc 2023-10-01T09:33:39.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:39 smithi138 ceph-mon[185886]: failed to encode map e392 with expected crc 2023-10-01T09:33:39.573 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:39.579 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:39 smithi190 ceph-mon[160435]: failed to encode map e392 with expected crc 2023-10-01T09:33:39.579 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:39 smithi190 ceph-mon[160435]: failed to encode map e392 with expected crc 2023-10-01T09:33:39.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:39.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:39.681 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:39.733 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:39.800 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:39.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:39.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:39.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:39 smithi138 ceph-mon[192513]: failed to encode map e392 with expected crc 2023-10-01T09:33:39.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:39 smithi138 ceph-mon[192513]: failed to encode map e392 with expected crc 2023-10-01T09:33:39.907 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:40.023 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:40.142 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:40.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:40.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:40.260 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:40.345 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:40.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:40 smithi138 ceph-mon[185886]: pgmap v1082: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 312 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 4436/14865 objects degraded (29.842%); 0 B/s, 190 objects/s recovering 2023-10-01T09:33:40.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:40 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1426305855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:33:40.601 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:33:40.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:40 smithi138 ceph-mon[192513]: pgmap v1082: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 312 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 4436/14865 objects degraded (29.842%); 0 B/s, 190 objects/s recovering 2023-10-01T09:33:40.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:40 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1426305855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:33:40.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:40.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:33:40.828 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:40.828 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:33:41.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:40 smithi190 ceph-mon[160435]: pgmap v1082: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 312 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 4436/14865 objects degraded (29.842%); 0 B/s, 190 objects/s recovering 2023-10-01T09:33:41.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:40 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1426305855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:33:41.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:41.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:33:41.625 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:41.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:41.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:41.730 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:41.798 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:41.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:41 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:41.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:41 smithi138 ceph-mon[192513]: failed to encode map e393 with expected crc 2023-10-01T09:33:41.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:41 smithi138 ceph-mon[192513]: failed to encode map e393 with expected crc 2023-10-01T09:33:41.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:41 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1426305855' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:33:41.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:41 smithi138 ceph-mon[192513]: osdmap e393: 8 total, 8 up, 8 in 2023-10-01T09:33:41.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:41 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:41.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:41 smithi138 ceph-mon[185886]: failed to encode map e393 with expected crc 2023-10-01T09:33:41.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:41 smithi138 ceph-mon[185886]: failed to encode map e393 with expected crc 2023-10-01T09:33:41.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:41 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1426305855' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:33:41.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:41 smithi138 ceph-mon[185886]: osdmap e393: 8 total, 8 up, 8 in 2023-10-01T09:33:42.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:41 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:42.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:41 smithi190 ceph-mon[160435]: failed to encode map e393 with expected crc 2023-10-01T09:33:42.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:41 smithi190 ceph-mon[160435]: failed to encode map e393 with expected crc 2023-10-01T09:33:42.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:41 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1426305855' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:33:42.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:41 smithi190 ceph-mon[160435]: osdmap e393: 8 total, 8 up, 8 in 2023-10-01T09:33:42.609 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:42.714 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:42.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:42.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:42.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:42 smithi138 ceph-mon[185886]: pgmap v1084: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 290 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 4436/13806 objects degraded (32.131%); 0 B/s, 52 objects/s recovering 2023-10-01T09:33:42.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:42 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 4436/14865 objects degraded (29.842%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:42.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:42 smithi138 ceph-mon[192513]: pgmap v1084: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 290 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 4436/13806 objects degraded (32.131%); 0 B/s, 52 objects/s recovering 2023-10-01T09:33:42.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:42 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 4436/14865 objects degraded (29.842%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:42.835 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:42.866 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:42 smithi190 ceph-mon[160435]: pgmap v1084: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 290 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 4436/13806 objects degraded (32.131%); 0 B/s, 52 objects/s recovering 2023-10-01T09:33:42.866 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:42 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 4436/14865 objects degraded (29.842%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:42.871 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:42.966 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:42.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:42.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:43.075 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:43.152 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:43.326 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:43.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:43.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:43.358 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:33:43.405 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:33:43.436 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:43.439 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:33:43.498 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:33:43.528 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:43.552 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:33:43.601 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:33:43.645 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:33:43.684 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:33:43.699 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:33:43.699 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:33:43.700 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:33:43.700 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:33:43.700 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:33:43.700 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:33:43.700 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:33:43.700 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:33:43.702 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:43.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:43.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:43.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:43 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/930827681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:43.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:43 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/930827681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:43.826 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:43.863 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:43 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/930827681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:43.866 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:43.961 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:43.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:43.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:44.072 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:44.123 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:44.625 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:33:44.645 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.1 to 1 (10000) 2023-10-01T09:33:44.941 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:44 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3848035653' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:33:44.941 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:44 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:33:44.941 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:44 smithi138 ceph-mon[192513]: pgmap v1085: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 137 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 3560/6462 objects degraded (55.091%); 0 B/s, 65 objects/s recovering 2023-10-01T09:33:44.942 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:44 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3848035653' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:33:44.942 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:44 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:33:44.942 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:44 smithi138 ceph-mon[185886]: pgmap v1085: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 137 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 3560/6462 objects degraded (55.091%); 0 B/s, 65 objects/s recovering 2023-10-01T09:33:45.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:44 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3848035653' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:33:45.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:44 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:33:45.066 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:44 smithi190 ceph-mon[160435]: pgmap v1085: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 137 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 3560/6462 objects degraded (55.091%); 0 B/s, 65 objects/s recovering 2023-10-01T09:33:45.236 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:33:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:33:44] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.33.4" 2023-10-01T09:33:45.467 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:45.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:45.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:45.582 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:45.609 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:46.013 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Removed 101408 objects 2023-10-01T09:33:46.013 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Clean up completed and total clean up time :161.601 2023-10-01T09:33:46.052 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:33:46.052 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[192513]: osdmap e394: 8 total, 8 up, 8 in 2023-10-01T09:33:46.052 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[192513]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.052 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[192513]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.052 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[192513]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.052 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[192513]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.052 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[192513]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.052 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[192513]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.052 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[192513]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.052 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[192513]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.053 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:33:46.053 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[185886]: osdmap e394: 8 total, 8 up, 8 in 2023-10-01T09:33:46.053 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[185886]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.053 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[185886]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.053 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[185886]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.053 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[185886]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.053 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[185886]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.053 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[185886]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.053 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[185886]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.053 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:45 smithi138 ceph-mon[185886]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.055 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:46.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:45 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:33:46.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:45 smithi190 ceph-mon[160435]: osdmap e394: 8 total, 8 up, 8 in 2023-10-01T09:33:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:45 smithi190 ceph-mon[160435]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:45 smithi190 ceph-mon[160435]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:45 smithi190 ceph-mon[160435]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:45 smithi190 ceph-mon[160435]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:45 smithi190 ceph-mon[160435]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:45 smithi190 ceph-mon[160435]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:45 smithi190 ceph-mon[160435]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:45 smithi190 ceph-mon[160435]: failed to encode map e394 with expected crc 2023-10-01T09:33:46.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:46.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:46.072 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-01T09:33:46.159 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:46.184 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:46.232 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:46.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:46.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:46.287 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:46.336 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:46.369 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:46.383 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:33:46.384 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:33:46.945 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:46.947 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:46 smithi138 ceph-mon[185886]: pgmap v1087: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 111 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 3560/5190 objects degraded (68.593%); 0 B/s, 56 objects/s recovering 2023-10-01T09:33:46.948 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:46 smithi138 ceph-mon[192513]: pgmap v1087: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 111 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 3560/5190 objects degraded (68.593%); 0 B/s, 56 objects/s recovering 2023-10-01T09:33:46.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:46.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:47.050 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:47.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:46 smithi190 ceph-mon[160435]: pgmap v1087: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 111 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 3560/5190 objects degraded (68.593%); 0 B/s, 56 objects/s recovering 2023-10-01T09:33:47.075 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:47.117 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:47.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:47.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:47.222 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:47.247 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:47.290 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:47.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:47.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:47.314 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:33:47.395 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:47.420 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:47.464 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:47.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:47.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:47.570 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:33:47.596 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:47.639 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:33:47.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:47.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:33:47.748 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:47.778 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:47 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 3560/5190 objects degraded (68.593%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:47.781 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:47.814 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:33:47.822 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:47.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:47.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:47.847 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:47 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 3560/5190 objects degraded (68.593%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:47.847 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:47 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 3560/5190 objects degraded (68.593%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:47.927 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:47.952 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:47.996 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:48.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:48.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:48.104 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:48.110 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:48.137 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:48.200 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:48.208 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:48.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:48.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:48.313 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:48.349 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:48.882 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:48 smithi138 ceph-mon[185886]: pgmap v1088: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 79 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 3560/3675 objects degraded (96.871%); 0 B/s, 28 objects/s recovering 2023-10-01T09:33:48.883 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:48 smithi138 ceph-mon[192513]: pgmap v1088: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 79 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 3560/3675 objects degraded (96.871%); 0 B/s, 28 objects/s recovering 2023-10-01T09:33:48.924 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:48.997 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:49.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:49.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:49.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:48 smithi190 ceph-mon[160435]: pgmap v1088: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 79 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 3560/3675 objects degraded (96.871%); 0 B/s, 28 objects/s recovering 2023-10-01T09:33:49.101 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:49.132 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:49.428 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:49.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:49.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:49.533 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:49.544 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-10-01T09:33:49.545 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-10-01T09:33:49.564 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:49.655 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:33:49.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1929528447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:33:49.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1929528447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:33:49.825 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:49.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:49.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:49.928 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:49.962 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:50.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1929528447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:33:50.066 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:50.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:50.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:50.171 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:50.199 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:50.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:50 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:50.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:50 smithi138 ceph-mon[185886]: failed to encode map e395 with expected crc 2023-10-01T09:33:50.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:50 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1929528447' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:33:50.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:50 smithi138 ceph-mon[185886]: osdmap e395: 8 total, 8 up, 8 in 2023-10-01T09:33:50.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:50 smithi138 ceph-mon[185886]: pgmap v1090: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 3560/84 objects degraded (4238.095%); 0 B/s, 28 objects/s recovering 2023-10-01T09:33:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:50 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:50 smithi138 ceph-mon[192513]: failed to encode map e395 with expected crc 2023-10-01T09:33:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:50 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1929528447' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:33:50.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:50 smithi138 ceph-mon[192513]: osdmap e395: 8 total, 8 up, 8 in 2023-10-01T09:33:50.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:50 smithi138 ceph-mon[192513]: pgmap v1090: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 3560/84 objects degraded (4238.095%); 0 B/s, 28 objects/s recovering 2023-10-01T09:33:51.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:50 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:51.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:50 smithi190 ceph-mon[160435]: failed to encode map e395 with expected crc 2023-10-01T09:33:51.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:50 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1929528447' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:33:51.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:50 smithi190 ceph-mon[160435]: osdmap e395: 8 total, 8 up, 8 in 2023-10-01T09:33:51.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:50 smithi190 ceph-mon[160435]: pgmap v1090: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 3560/84 objects degraded (4238.095%); 0 B/s, 28 objects/s recovering 2023-10-01T09:33:51.451 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:33:51.451 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:33:51.533 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:51.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:51.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:51.639 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:51.667 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:51.708 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:51.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:51.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:51.815 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:51.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:51 smithi138 ceph-mon[185886]: failed to encode map e395 with expected crc 2023-10-01T09:33:51.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:51 smithi138 ceph-mon[192513]: failed to encode map e395 with expected crc 2023-10-01T09:33:51.846 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:52.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:51 smithi190 ceph-mon[160435]: failed to encode map e395 with expected crc 2023-10-01T09:33:52.171 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:52.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:52.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:52.261 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:33:52.274 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:52.333 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:52.465 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:52.487 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:52.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:52.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:52.567 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:33:52.585 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:33:52.591 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:33:52.604 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:33:52.617 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:52.623 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:33:52.647 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:33:52.658 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.3 to 1 (10000) 2023-10-01T09:33:52.664 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:33:52.666 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:33:52.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:52.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:33:52.687 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:33:52.710 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:33:52.712 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:33:52.713 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:33:52.713 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:33:52.713 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:33:52.713 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:33:52.713 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:33:52.713 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:33:52.713 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:33:52.771 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:52.798 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:52 smithi138 ceph-mon[192513]: pgmap v1091: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 3560/84 objects degraded (4238.095%); 0 B/s, 14 objects/s recovering 2023-10-01T09:33:52.799 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:52 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3761591115' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:33:52.799 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:52 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:33:52.799 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:52 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 3560/84 objects degraded (4238.095%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:52.799 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:52 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4258615841' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:52.799 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:52 smithi138 ceph-mon[185886]: pgmap v1091: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 3560/84 objects degraded (4238.095%); 0 B/s, 14 objects/s recovering 2023-10-01T09:33:52.799 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:52 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3761591115' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:33:52.800 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:52 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:33:52.800 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:52 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 3560/84 objects degraded (4238.095%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:52.800 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:52 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4258615841' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:52.802 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:53.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:52 smithi190 ceph-mon[160435]: pgmap v1091: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 265 op/s; 3560/84 objects degraded (4238.095%); 0 B/s, 14 objects/s recovering 2023-10-01T09:33:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:52 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3761591115' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:33:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:52 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:33:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:52 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 3560/84 objects degraded (4238.095%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T09:33:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:52 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4258615841' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:33:53.148 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:53.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:53.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:53.322 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:53.349 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:53.441 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:33:53.444 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:53.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:53.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:53.547 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:53.572 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:53.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:53 smithi138 ceph-mon[192513]: failed to encode map e396 with expected crc 2023-10-01T09:33:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:53 smithi138 ceph-mon[192513]: failed to encode map e396 with expected crc 2023-10-01T09:33:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:53 smithi138 ceph-mon[192513]: failed to encode map e396 with expected crc 2023-10-01T09:33:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:53 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:33:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:53 smithi138 ceph-mon[192513]: osdmap e396: 8 total, 8 up, 8 in 2023-10-01T09:33:53.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:53 smithi138 ceph-mon[185886]: failed to encode map e396 with expected crc 2023-10-01T09:33:53.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:53 smithi138 ceph-mon[185886]: failed to encode map e396 with expected crc 2023-10-01T09:33:53.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:53 smithi138 ceph-mon[185886]: failed to encode map e396 with expected crc 2023-10-01T09:33:53.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:53 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:33:53.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:53 smithi138 ceph-mon[185886]: osdmap e396: 8 total, 8 up, 8 in 2023-10-01T09:33:53.866 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:53.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:53.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:53.970 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:33:53.993 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:53 smithi190 ceph-mon[160435]: failed to encode map e396 with expected crc 2023-10-01T09:33:53.993 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:53 smithi190 ceph-mon[160435]: failed to encode map e396 with expected crc 2023-10-01T09:33:53.993 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:53 smithi190 ceph-mon[160435]: failed to encode map e396 with expected crc 2023-10-01T09:33:53.993 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:53 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:33:53.993 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:53 smithi190 ceph-mon[160435]: osdmap e396: 8 total, 8 up, 8 in 2023-10-01T09:33:53.995 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:54.040 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:33:54.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:54.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:33:54.145 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:54.170 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:54.213 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:54.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:54.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:54.318 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:33:54.342 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:54.388 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:33:54.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:54.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:33:54.497 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:54.527 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:54.532 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:54.561 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-01T09:33:54.677 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:54.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:54.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:54.695 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:54.780 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:54.806 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:54.848 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:54.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:54.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:54.877 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:54 smithi138 ceph-mon[192513]: pgmap v1093: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 3560/84 objects degraded (4238.095%); 0 B/s, 29 objects/s recovering 2023-10-01T09:33:54.878 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:54 smithi138 ceph-mon[185886]: pgmap v1093: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 3560/84 objects degraded (4238.095%); 0 B/s, 29 objects/s recovering 2023-10-01T09:33:54.953 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:33:54.977 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:54 smithi190 ceph-mon[160435]: pgmap v1093: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 3560/84 objects degraded (4238.095%); 0 B/s, 29 objects/s recovering 2023-10-01T09:33:54.979 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:55.022 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:33:55.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:55.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:33:55.128 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:55.129 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:33:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:33:54] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.33.4" 2023-10-01T09:33:55.162 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:55.207 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:55.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:55.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:55.311 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:55.336 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:55.382 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:55.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:55.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:55.488 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:33:55.514 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:55.555 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:33:55.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:55.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:33:55.661 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:33:55.686 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:55.729 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:33:55.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:55.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:33:55.838 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:33:55.863 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:55.908 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:33:55.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:55.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:33:56.013 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:56.040 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:56.331 INFO:teuthology.orchestra.run.smithi138.stderr:pool 'unique_pool_3' removed 2023-10-01T09:33:56.365 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:33:56.373 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:56.394 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:56.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:56.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:56.501 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:56.526 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:33:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:56 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3812220688' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:33:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:56 smithi190 ceph-mon[160435]: pgmap v1094: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 3560/84 objects degraded (4238.095%); 0 B/s, 14 objects/s recovering 2023-10-01T09:33:56.569 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:56.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:56.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:33:56.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:56 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3812220688' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:33:56.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:56 smithi138 ceph-mon[185886]: pgmap v1094: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 3560/84 objects degraded (4238.095%); 0 B/s, 14 objects/s recovering 2023-10-01T09:33:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:33:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:33:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:56 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3812220688' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:33:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:56 smithi138 ceph-mon[192513]: pgmap v1094: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 149 op/s; 3560/84 objects degraded (4238.095%); 0 B/s, 14 objects/s recovering 2023-10-01T09:33:56.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:56.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:56.673 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:33:56.699 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:56.741 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:33:56.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:56.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:33:56.849 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:56.914 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:56.999 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:57.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:57.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:57.103 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:33:57.130 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:57.152 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T09:33:57.153 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T09:33:57.153 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-10-01T09:33:57.153 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-10-01T09:33:57.169 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:33:57.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:57.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:33:57.276 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:57.305 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:57.504 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:33:57.529 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:57.534 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:57 smithi138 ceph-mon[192513]: failed to encode map e397 with expected crc 2023-10-01T09:33:57.534 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:57 smithi138 ceph-mon[192513]: failed to encode map e397 with expected crc 2023-10-01T09:33:57.534 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3812220688' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:33:57.534 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:57 smithi138 ceph-mon[192513]: osdmap e397: 8 total, 8 up, 8 in 2023-10-01T09:33:57.534 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:57 smithi138 ceph-mon[192513]: failed to encode map e397 with expected crc 2023-10-01T09:33:57.535 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:57 smithi138 ceph-mon[185886]: failed to encode map e397 with expected crc 2023-10-01T09:33:57.535 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:57 smithi138 ceph-mon[185886]: failed to encode map e397 with expected crc 2023-10-01T09:33:57.535 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3812220688' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:33:57.536 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:57 smithi138 ceph-mon[185886]: osdmap e397: 8 total, 8 up, 8 in 2023-10-01T09:33:57.536 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:57 smithi138 ceph-mon[185886]: failed to encode map e397 with expected crc 2023-10-01T09:33:57.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:57.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:57.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:57 smithi190 ceph-mon[160435]: failed to encode map e397 with expected crc 2023-10-01T09:33:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:57 smithi190 ceph-mon[160435]: failed to encode map e397 with expected crc 2023-10-01T09:33:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3812220688' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:33:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:57 smithi190 ceph-mon[160435]: osdmap e397: 8 total, 8 up, 8 in 2023-10-01T09:33:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:57 smithi190 ceph-mon[160435]: failed to encode map e397 with expected crc 2023-10-01T09:33:57.625 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:33:57.626 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:33:57.626 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-10-01T09:33:57.626 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:33:57.638 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:33:57.669 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:58.067 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:33:58.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:58.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:33:58.170 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:33:58.197 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:58.484 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:33:58.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:58 smithi190 ceph-mon[160435]: failed to encode map e397 with expected crc 2023-10-01T09:33:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:58 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3547738307' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:33:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:58 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:33:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:58 smithi190 ceph-mon[160435]: pgmap v1096: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:33:58.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:58 smithi138 ceph-mon[192513]: failed to encode map e397 with expected crc 2023-10-01T09:33:58.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:58 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3547738307' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:33:58.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:58 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:33:58.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:58 smithi138 ceph-mon[192513]: pgmap v1096: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:33:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:58 smithi138 ceph-mon[185886]: failed to encode map e397 with expected crc 2023-10-01T09:33:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:58 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3547738307' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:33:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:58 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:33:58.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:58 smithi138 ceph-mon[185886]: pgmap v1096: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:33:59.117 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:33:59.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:59.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:33:59.222 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:33:59.250 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:59.370 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:33:59.399 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:33:59.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:59.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:33:59.503 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:33:59.528 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:59 smithi138 ceph-mon[185886]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3560/84 objects degraded (4238.095%), 4 pgs degraded) 2023-10-01T09:33:59.528 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:59 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:59.528 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:59 smithi138 ceph-mon[185886]: failed to encode map e398 with expected crc 2023-10-01T09:33:59.528 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:59 smithi138 ceph-mon[185886]: failed to encode map e398 with expected crc 2023-10-01T09:33:59.528 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:59 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:33:59.528 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:59 smithi138 ceph-mon[185886]: osdmap e398: 8 total, 8 up, 8 in 2023-10-01T09:33:59.528 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:33:59 smithi138 ceph-mon[185886]: failed to encode map e398 with expected crc 2023-10-01T09:33:59.528 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:59 smithi138 ceph-mon[192513]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3560/84 objects degraded (4238.095%), 4 pgs degraded) 2023-10-01T09:33:59.529 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:59 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:59.529 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:59 smithi138 ceph-mon[192513]: failed to encode map e398 with expected crc 2023-10-01T09:33:59.529 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:59 smithi138 ceph-mon[192513]: failed to encode map e398 with expected crc 2023-10-01T09:33:59.529 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:59 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:33:59.529 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:59 smithi138 ceph-mon[192513]: osdmap e398: 8 total, 8 up, 8 in 2023-10-01T09:33:59.529 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:33:59 smithi138 ceph-mon[192513]: failed to encode map e398 with expected crc 2023-10-01T09:33:59.530 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:33:59.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:59 smithi190 ceph-mon[160435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3560/84 objects degraded (4238.095%), 4 pgs degraded) 2023-10-01T09:33:59.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:59 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:33:59.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:59 smithi190 ceph-mon[160435]: failed to encode map e398 with expected crc 2023-10-01T09:33:59.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:59 smithi190 ceph-mon[160435]: failed to encode map e398 with expected crc 2023-10-01T09:33:59.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:59 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:33:59.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:59 smithi190 ceph-mon[160435]: osdmap e398: 8 total, 8 up, 8 in 2023-10-01T09:33:59.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:33:59 smithi190 ceph-mon[160435]: failed to encode map e398 with expected crc 2023-10-01T09:33:59.881 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:33:59.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:33:59.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:33:59.985 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:00.014 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:00.061 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:00.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:00.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:00.166 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:00.193 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:00.237 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:00.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:00.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:00.343 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:00.370 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:00.415 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:00.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:00.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:00.491 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:00.521 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:00.547 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:00.606 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:00.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:00.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:00.711 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:34:00.739 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:34:00.739 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:00 smithi138 ceph-mon[192513]: pgmap v1098: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:00.740 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:34:00.740 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:00 smithi138 ceph-mon[185886]: pgmap v1098: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:00.743 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:00.797 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:00.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:34:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:00 smithi190 ceph-mon[160435]: pgmap v1098: 73 pgs: 73 active+clean; 4.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:00.929 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:34:00.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:00.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:34:01.033 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:01.060 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:01.103 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:01.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:01.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:01.207 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:01.237 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:01.277 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:34:01.298 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:34:01.317 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:34:01.337 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:34:01.366 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:34:01.385 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:34:01.403 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:34:01.421 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:34:01.422 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:34:01.422 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:34:01.423 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:34:01.423 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:34:01.423 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:34:01.423 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:34:01.423 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:34:01.423 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:34:01.497 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.7 to 1 (10000) 2023-10-01T09:34:01.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3209453672' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:34:01.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:01 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:34:01.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3013869809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:01.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3209453672' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:34:01.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:01 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:34:01.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3013869809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3209453672' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:34:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:01 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:34:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3013869809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:02.194 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:02.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:02.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:02.298 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:02.325 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:02.469 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:02.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:02.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:02.493 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:34:02.577 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:34:02.602 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:02 smithi138 ceph-mon[192513]: failed to encode map e399 with expected crc 2023-10-01T09:34:02.602 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:02 smithi138 ceph-mon[192513]: failed to encode map e399 with expected crc 2023-10-01T09:34:02.602 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:02 smithi138 ceph-mon[192513]: failed to encode map e399 with expected crc 2023-10-01T09:34:02.602 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:02 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:34:02.603 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:02 smithi138 ceph-mon[192513]: osdmap e399: 8 total, 8 up, 8 in 2023-10-01T09:34:02.603 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:02 smithi138 ceph-mon[192513]: pgmap v1100: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:02.603 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:02 smithi138 ceph-mon[185886]: failed to encode map e399 with expected crc 2023-10-01T09:34:02.603 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:02 smithi138 ceph-mon[185886]: failed to encode map e399 with expected crc 2023-10-01T09:34:02.603 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:02 smithi138 ceph-mon[185886]: failed to encode map e399 with expected crc 2023-10-01T09:34:02.603 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:02 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:34:02.603 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:02 smithi138 ceph-mon[185886]: osdmap e399: 8 total, 8 up, 8 in 2023-10-01T09:34:02.603 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:02 smithi138 ceph-mon[185886]: pgmap v1100: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:02.604 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:02.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:02 smithi190 ceph-mon[160435]: failed to encode map e399 with expected crc 2023-10-01T09:34:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:02 smithi190 ceph-mon[160435]: failed to encode map e399 with expected crc 2023-10-01T09:34:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:02 smithi190 ceph-mon[160435]: failed to encode map e399 with expected crc 2023-10-01T09:34:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:02 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:34:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:02 smithi190 ceph-mon[160435]: osdmap e399: 8 total, 8 up, 8 in 2023-10-01T09:34:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:02 smithi190 ceph-mon[160435]: pgmap v1100: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:02.903 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:34:02.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:02.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:34:03.000 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-01T09:34:03.006 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:03.038 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:03.082 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:03.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:03.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:03.189 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:03.215 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:03.259 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:03.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:03.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:03.364 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:03.390 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:03.434 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:03.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:03.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:03.541 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:03.571 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:03.612 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:03.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:03.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:03.717 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:03.749 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:03.850 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:03.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:03.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:03.954 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:03.981 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:04.023 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:04.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:04.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:04.118 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:04.129 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:04.156 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:04.198 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:04.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:04.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:04.305 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:04.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:04 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:04 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:04.337 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:04.432 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:04.478 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:34:04.479 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:34:04.501 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:04 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:04.501 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:04 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:04.501 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:04 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:04.501 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:04 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:04.540 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:04.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:04.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:04.646 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:04.676 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:04.871 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:04.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:04.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:04.974 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:05.005 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:05.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:34:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:34:04] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.33.4" 2023-10-01T09:34:05.121 INFO:teuthology.orchestra.run.smithi138.stderr:pool 'unique_pool_4' created 2023-10-01T09:34:05.534 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:05 smithi138 ceph-mon[192513]: pgmap v1101: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:05.534 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1227594191' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-01T09:34:05.534 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:05 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-01T09:34:05.535 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:05 smithi138 ceph-mon[185886]: pgmap v1101: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:05.535 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1227594191' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-01T09:34:05.535 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:05 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-01T09:34:05.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:05 smithi190 ceph-mon[160435]: pgmap v1101: 73 pgs: 73 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:05.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1227594191' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-01T09:34:05.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:05 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-01T09:34:05.812 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:34:05.902 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:05.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:05.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:06.006 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:06.038 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:06.085 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:06.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:06.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:06.191 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:06.218 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:06 smithi190 ceph-mon[160435]: failed to encode map e400 with expected crc 2023-10-01T09:34:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:06 smithi190 ceph-mon[160435]: failed to encode map e400 with expected crc 2023-10-01T09:34:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:06 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-01T09:34:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:06 smithi190 ceph-mon[160435]: osdmap e400: 8 total, 8 up, 8 in 2023-10-01T09:34:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:06 smithi190 ceph-mon[160435]: failed to encode map e400 with expected crc 2023-10-01T09:34:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:06 smithi190 ceph-mon[160435]: failed to encode map e400 with expected crc 2023-10-01T09:34:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:06 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:06 smithi190 ceph-mon[160435]: pgmap v1103: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:06.439 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[192513]: failed to encode map e400 with expected crc 2023-10-01T09:34:06.439 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[192513]: failed to encode map e400 with expected crc 2023-10-01T09:34:06.440 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-01T09:34:06.440 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[192513]: osdmap e400: 8 total, 8 up, 8 in 2023-10-01T09:34:06.440 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[192513]: failed to encode map e400 with expected crc 2023-10-01T09:34:06.440 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[192513]: failed to encode map e400 with expected crc 2023-10-01T09:34:06.440 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:06.441 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[192513]: pgmap v1103: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:06.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[185886]: failed to encode map e400 with expected crc 2023-10-01T09:34:06.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[185886]: failed to encode map e400 with expected crc 2023-10-01T09:34:06.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-01T09:34:06.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[185886]: osdmap e400: 8 total, 8 up, 8 in 2023-10-01T09:34:06.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[185886]: failed to encode map e400 with expected crc 2023-10-01T09:34:06.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[185886]: failed to encode map e400 with expected crc 2023-10-01T09:34:06.441 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:06.442 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:06 smithi138 ceph-mon[185886]: pgmap v1103: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:06.512 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:34:06.569 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:06.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:06.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:06.580 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-10-01T09:34:06.673 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:06.705 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:06.754 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:06.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:06.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:06.859 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:06.887 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:06.930 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:06.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:06.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:07.036 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:07.063 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:07.142 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:07.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:07.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:07.246 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:07.280 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:07.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:07 smithi138 ceph-mon[192513]: osdmap e401: 8 total, 8 up, 8 in 2023-10-01T09:34:07.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:07 smithi138 ceph-mon[192513]: failed to encode map e401 with expected crc 2023-10-01T09:34:07.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:07 smithi138 ceph-mon[192513]: failed to encode map e401 with expected crc 2023-10-01T09:34:07.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:07 smithi138 ceph-mon[192513]: failed to encode map e401 with expected crc 2023-10-01T09:34:07.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:07 smithi138 ceph-mon[192513]: failed to encode map e401 with expected crc 2023-10-01T09:34:07.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:07 smithi138 ceph-mon[185886]: osdmap e401: 8 total, 8 up, 8 in 2023-10-01T09:34:07.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:07 smithi138 ceph-mon[185886]: failed to encode map e401 with expected crc 2023-10-01T09:34:07.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:07 smithi138 ceph-mon[185886]: failed to encode map e401 with expected crc 2023-10-01T09:34:07.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:07 smithi138 ceph-mon[185886]: failed to encode map e401 with expected crc 2023-10-01T09:34:07.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:07 smithi138 ceph-mon[185886]: failed to encode map e401 with expected crc 2023-10-01T09:34:07.339 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:07.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:07.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:07.422 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:07.443 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:07.450 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:07.478 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:07.537 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:07.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:07.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:07 smithi190 ceph-mon[160435]: osdmap e401: 8 total, 8 up, 8 in 2023-10-01T09:34:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:07 smithi190 ceph-mon[160435]: failed to encode map e401 with expected crc 2023-10-01T09:34:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:07 smithi190 ceph-mon[160435]: failed to encode map e401 with expected crc 2023-10-01T09:34:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:07 smithi190 ceph-mon[160435]: failed to encode map e401 with expected crc 2023-10-01T09:34:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:07 smithi190 ceph-mon[160435]: failed to encode map e401 with expected crc 2023-10-01T09:34:07.643 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:07.673 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:07.821 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:07.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:07.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:07.925 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:07.953 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:07.997 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:08.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:08.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:08.045 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:08.104 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:08.130 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:08.172 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:08.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:08.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:08.279 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:34:08.305 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[192513]: failed to encode map e402 with expected crc 2023-10-01T09:34:08.305 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[192513]: osdmap e402: 8 total, 8 up, 8 in 2023-10-01T09:34:08.305 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1718981167' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:34:08.305 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:34:08.305 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[192513]: failed to encode map e402 with expected crc 2023-10-01T09:34:08.305 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1604842429' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:08.305 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:08.305 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[192513]: pgmap v1106: 89 pgs: 11 unknown, 78 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:08.305 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[185886]: failed to encode map e402 with expected crc 2023-10-01T09:34:08.306 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[185886]: osdmap e402: 8 total, 8 up, 8 in 2023-10-01T09:34:08.306 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1718981167' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:34:08.306 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:34:08.306 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[185886]: failed to encode map e402 with expected crc 2023-10-01T09:34:08.306 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1604842429' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:08.306 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:08.306 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:08 smithi138 ceph-mon[185886]: pgmap v1106: 89 pgs: 11 unknown, 78 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:08.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:08 smithi190 ceph-mon[160435]: failed to encode map e402 with expected crc 2023-10-01T09:34:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:08 smithi190 ceph-mon[160435]: osdmap e402: 8 total, 8 up, 8 in 2023-10-01T09:34:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:08 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1718981167' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:34:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:08 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:34:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:08 smithi190 ceph-mon[160435]: failed to encode map e402 with expected crc 2023-10-01T09:34:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:08 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1604842429' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:08 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:08.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:08 smithi190 ceph-mon[160435]: pgmap v1106: 89 pgs: 11 unknown, 78 active+clean; 4.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:08.307 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:08.521 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.2 to 1 (10000) 2023-10-01T09:34:08.522 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:34:08.857 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:34:08.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:08.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:34:08.963 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:08.994 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:09.042 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:09.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:09.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:09.147 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:09.174 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:09.217 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:09.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:09.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:09.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:09 smithi190 ceph-mon[160435]: failed to encode map e402 with expected crc 2023-10-01T09:34:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:09 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:34:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:09 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:34:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:09 smithi190 ceph-mon[160435]: failed to encode map e403 with expected crc 2023-10-01T09:34:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:09 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:34:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:09 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:34:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:09 smithi190 ceph-mon[160435]: osdmap e403: 8 total, 8 up, 8 in 2023-10-01T09:34:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:09 smithi190 ceph-mon[160435]: failed to encode map e403 with expected crc 2023-10-01T09:34:09.323 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:09.351 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:09.528 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[192513]: failed to encode map e402 with expected crc 2023-10-01T09:34:09.528 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:34:09.528 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:34:09.528 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[192513]: failed to encode map e403 with expected crc 2023-10-01T09:34:09.528 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:34:09.528 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:34:09.528 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[192513]: osdmap e403: 8 total, 8 up, 8 in 2023-10-01T09:34:09.528 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[192513]: failed to encode map e403 with expected crc 2023-10-01T09:34:09.528 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[185886]: failed to encode map e402 with expected crc 2023-10-01T09:34:09.529 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:34:09.529 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:34:09.529 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[185886]: failed to encode map e403 with expected crc 2023-10-01T09:34:09.529 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:34:09.529 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:34:09.529 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[185886]: osdmap e403: 8 total, 8 up, 8 in 2023-10-01T09:34:09.529 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:09 smithi138 ceph-mon[185886]: failed to encode map e403 with expected crc 2023-10-01T09:34:09.547 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-01T09:34:10.347 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:10.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:10.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:10.451 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:10.476 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:10 smithi138 ceph-mon[192513]: failed to encode map e403 with expected crc 2023-10-01T09:34:10.476 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:10 smithi138 ceph-mon[192513]: failed to encode map e403 with expected crc 2023-10-01T09:34:10.476 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:10 smithi138 ceph-mon[192513]: pgmap v1108: 89 pgs: 89 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:10.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:10 smithi138 ceph-mon[185886]: failed to encode map e403 with expected crc 2023-10-01T09:34:10.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:10 smithi138 ceph-mon[185886]: failed to encode map e403 with expected crc 2023-10-01T09:34:10.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:10 smithi138 ceph-mon[185886]: pgmap v1108: 89 pgs: 89 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:10.478 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:10 smithi190 ceph-mon[160435]: failed to encode map e403 with expected crc 2023-10-01T09:34:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:10 smithi190 ceph-mon[160435]: failed to encode map e403 with expected crc 2023-10-01T09:34:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:10 smithi190 ceph-mon[160435]: pgmap v1108: 89 pgs: 89 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:10.825 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:10.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:10.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:10.929 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:10.962 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:11.009 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:11.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:11.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:11.115 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:11.144 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:11.148 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:34:11.167 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:34:11.187 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:34:11.190 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:11.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:11.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:11.206 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:34:11.223 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:34:11.241 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:34:11.259 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:34:11.277 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:34:11.278 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:34:11.278 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:34:11.278 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:34:11.278 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:34:11.278 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:34:11.279 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:34:11.279 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:34:11.279 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:34:11.296 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:11.326 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:11.384 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:34:11.384 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:34:11.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:34:11.384 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:34:11.384 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.384 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:121701 7.8 7:123e4a57:::benchmark_data_smithi138_367058_object23906:head [delete] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-10-01T09:34:11.384 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:28.301494+0000", 2023-10-01T09:34:11.384 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 463.08052056899999, 2023-10-01T09:34:11.384 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0627196029999999, 2023-10-01T09:34:11.385 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.385 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:34:11.385 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:11.385 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:34:11.385 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:34:11.385 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 121701 2023-10-01T09:34:11.385 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.385 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.385 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.385 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.385 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301494+0000", 2023-10-01T09:34:11.385 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301494+0000", 2023-10-01T09:34:11.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.386 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.386 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301493+0000", 2023-10-01T09:34:11.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:34:11.386 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301497+0000", 2023-10-01T09:34:11.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7670000000000003e-06 2023-10-01T09:34:11.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301506+0000", 2023-10-01T09:34:11.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3489999999999992e-06 2023-10-01T09:34:11.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301512+0000", 2023-10-01T09:34:11.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0290000000000002e-06 2023-10-01T09:34:11.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301562+0000", 2023-10-01T09:34:11.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0515e-05 2023-10-01T09:34:11.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301721+0000", 2023-10-01T09:34:11.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015924799999999999 2023-10-01T09:34:11.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:34:11.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.301875+0000", 2023-10-01T09:34:11.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000154046 2023-10-01T09:34:11.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:11.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.310228+0000", 2023-10-01T09:34:11.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0083525619999999991 2023-10-01T09:34:11.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:11.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.318729+0000", 2023-10-01T09:34:11.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085004699999999996 2023-10-01T09:34:11.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.416615+0000", 2023-10-01T09:34:11.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.097886033999999997 2023-10-01T09:34:11.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:34:11.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:28.416630+0000", 2023-10-01T09:34:11.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5301000000000001e-05 2023-10-01T09:34:11.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:29.364189+0000", 2023-10-01T09:34:11.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94755891999999997 2023-10-01T09:34:11.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:29.364214+0000", 2023-10-01T09:34:11.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4902999999999999e-05 2023-10-01T09:34:11.392 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.392 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.392 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.392 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169548 8.4 8:2b69f497:::benchmark_data_smithi138_431617_object68137:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-10-01T09:34:11.392 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.299289+0000", 2023-10-01T09:34:11.392 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 86.082725468000007, 2023-10-01T09:34:11.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0099249669999999, 2023-10-01T09:34:11.392 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.392 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:11.393 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:11.393 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:34:11.393 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:11.393 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169548 2023-10-01T09:34:11.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.393 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.393 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299289+0000", 2023-10-01T09:34:11.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299289+0000", 2023-10-01T09:34:11.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299288+0000", 2023-10-01T09:34:11.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:34:11.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299294+0000", 2023-10-01T09:34:11.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4699999999999999e-06 2023-10-01T09:34:11.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299309+0000", 2023-10-01T09:34:11.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4881e-05 2023-10-01T09:34:11.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299319+0000", 2023-10-01T09:34:11.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6339999999999996e-06 2023-10-01T09:34:11.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299372+0000", 2023-10-01T09:34:11.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3783999999999999e-05 2023-10-01T09:34:11.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:34:11.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299382+0000", 2023-10-01T09:34:11.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6840000000000008e-06 2023-10-01T09:34:11.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.298379+0000", 2023-10-01T09:34:11.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99899664200000005 2023-10-01T09:34:11.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.298527+0000", 2023-10-01T09:34:11.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000148095 2023-10-01T09:34:11.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.398 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:34:11.398 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.298668+0000", 2023-10-01T09:34:11.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014150400000000001 2023-10-01T09:34:11.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.398 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:11.398 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.306474+0000", 2023-10-01T09:34:11.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0078054810000000004 2023-10-01T09:34:11.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.398 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:11.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.307019+0000", 2023-10-01T09:34:11.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00054510799999999999 2023-10-01T09:34:11.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:11.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.309059+0000", 2023-10-01T09:34:11.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020396699999999999 2023-10-01T09:34:11.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:11.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.309113+0000", 2023-10-01T09:34:11.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3981999999999999e-05 2023-10-01T09:34:11.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.309214+0000", 2023-10-01T09:34:11.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101505 2023-10-01T09:34:11.400 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.400 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.400 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.400 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169877 8.4 8:268b69d6:::benchmark_data_smithi138_431617_object68466:head [delete] snapc 0=[] ondisk+write+known_if_redirected e370)", 2023-10-01T09:34:11.400 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:46.867343+0000", 2023-10-01T09:34:11.401 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 84.514671450999998, 2023-10-01T09:34:11.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.212218399, 2023-10-01T09:34:11.401 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.401 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:11.401 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:11.401 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:34:11.401 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:11.401 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169877 2023-10-01T09:34:11.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.401 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.867343+0000", 2023-10-01T09:34:11.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.867343+0000", 2023-10-01T09:34:11.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.867343+0000", 2023-10-01T09:34:11.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:11.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.403 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.867345+0000", 2023-10-01T09:34:11.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5000000000000002e-06 2023-10-01T09:34:11.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.403 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.867351+0000", 2023-10-01T09:34:11.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0440000000000001e-06 2023-10-01T09:34:11.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.867356+0000", 2023-10-01T09:34:11.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.04e-06 2023-10-01T09:34:11.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.872450+0000", 2023-10-01T09:34:11.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050940350000000002 2023-10-01T09:34:11.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.872580+0000", 2023-10-01T09:34:11.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012970599999999999 2023-10-01T09:34:11.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:34:11.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.872711+0000", 2023-10-01T09:34:11.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013086599999999999 2023-10-01T09:34:11.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:11.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.882024+0000", 2023-10-01T09:34:11.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0093126000000000007 2023-10-01T09:34:11.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:11.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.882032+0000", 2023-10-01T09:34:11.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7020000000000007e-06 2023-10-01T09:34:11.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:11.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079417+0000", 2023-10-01T09:34:11.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.197384798 2023-10-01T09:34:11.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:11.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079476+0000", 2023-10-01T09:34:11.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8541000000000001e-05 2023-10-01T09:34:11.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079562+0000", 2023-10-01T09:34:11.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5928999999999997e-05 2023-10-01T09:34:11.407 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.407 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.407 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.408 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169882 8.4 8:21c9a911:::benchmark_data_smithi138_431617_object68471:head [delete] snapc 0=[] ondisk+write+known_if_redirected e370)", 2023-10-01T09:34:11.408 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:46.875767+0000", 2023-10-01T09:34:11.408 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 84.506247865999995, 2023-10-01T09:34:11.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.204057719, 2023-10-01T09:34:11.408 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.408 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:11.408 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:11.408 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:34:11.408 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:11.408 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169882 2023-10-01T09:34:11.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.409 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.875767+0000", 2023-10-01T09:34:11.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.875767+0000", 2023-10-01T09:34:11.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.875766+0000", 2023-10-01T09:34:11.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:34:11.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.875771+0000", 2023-10-01T09:34:11.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8720000000000001e-06 2023-10-01T09:34:11.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.875779+0000", 2023-10-01T09:34:11.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3189999999999995e-06 2023-10-01T09:34:11.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.875784+0000", 2023-10-01T09:34:11.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3730000000000001e-06 2023-10-01T09:34:11.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.877573+0000", 2023-10-01T09:34:11.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017888719999999999 2023-10-01T09:34:11.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.877695+0000", 2023-10-01T09:34:11.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121889 2023-10-01T09:34:11.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:34:11.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.877787+0000", 2023-10-01T09:34:11.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1780000000000006e-05 2023-10-01T09:34:11.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:11.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.886356+0000", 2023-10-01T09:34:11.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0085696350000000008 2023-10-01T09:34:11.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:11.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079313+0000", 2023-10-01T09:34:11.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19295651 2023-10-01T09:34:11.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:11.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079681+0000", 2023-10-01T09:34:11.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00036812599999999998 2023-10-01T09:34:11.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:11.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079728+0000", 2023-10-01T09:34:11.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6575e-05 2023-10-01T09:34:11.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079825+0000", 2023-10-01T09:34:11.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6838999999999999e-05 2023-10-01T09:34:11.416 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.416 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.416 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.416 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169886 8.4 8:23fe4aea:::benchmark_data_smithi138_431617_object68475:head [delete] snapc 0=[] ondisk+write+known_if_redirected e370)", 2023-10-01T09:34:11.416 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:46.880789+0000", 2023-10-01T09:34:11.416 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 84.501226075000005, 2023-10-01T09:34:11.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19920389599999999, 2023-10-01T09:34:11.417 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.417 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:11.417 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:11.417 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:34:11.417 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:11.417 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169886 2023-10-01T09:34:11.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.417 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.417 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.417 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.880789+0000", 2023-10-01T09:34:11.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.880789+0000", 2023-10-01T09:34:11.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.880787+0000", 2023-10-01T09:34:11.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:34:11.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.880795+0000", 2023-10-01T09:34:11.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6079999999999998e-06 2023-10-01T09:34:11.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.880808+0000", 2023-10-01T09:34:11.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3417999999999999e-05 2023-10-01T09:34:11.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.880817+0000", 2023-10-01T09:34:11.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8019999999999998e-06 2023-10-01T09:34:11.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.882051+0000", 2023-10-01T09:34:11.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001234326 2023-10-01T09:34:11.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.882136+0000", 2023-10-01T09:34:11.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4766999999999997e-05 2023-10-01T09:34:11.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:34:11.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.882221+0000", 2023-10-01T09:34:11.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5023000000000002e-05 2023-10-01T09:34:11.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:11.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079574+0000", 2023-10-01T09:34:11.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19735292600000001 2023-10-01T09:34:11.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:11.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079611+0000", 2023-10-01T09:34:11.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7163000000000003e-05 2023-10-01T09:34:11.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:11.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079866+0000", 2023-10-01T09:34:11.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025479700000000002 2023-10-01T09:34:11.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:11.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079904+0000", 2023-10-01T09:34:11.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7514e-05 2023-10-01T09:34:11.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079992+0000", 2023-10-01T09:34:11.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8891e-05 2023-10-01T09:34:11.424 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.424 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.425 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.425 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.45674.0:169882 8.4 e370/369 ondisk, result = 0)", 2023-10-01T09:34:11.425 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:46.883713+0000", 2023-10-01T09:34:11.425 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 84.498301901999994, 2023-10-01T09:34:11.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19561103499999999, 2023-10-01T09:34:11.425 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.425 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:34:11.425 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.883713+0000", 2023-10-01T09:34:11.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.883713+0000", 2023-10-01T09:34:11.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.883712+0000", 2023-10-01T09:34:11.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:34:11.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.883717+0000", 2023-10-01T09:34:11.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0069999999999997e-06 2023-10-01T09:34:11.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.428 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.428 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.883726+0000", 2023-10-01T09:34:11.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3360000000000001e-06 2023-10-01T09:34:11.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.428 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.428 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.883732+0000", 2023-10-01T09:34:11.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2900000000000002e-06 2023-10-01T09:34:11.428 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079285+0000", 2023-10-01T09:34:11.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.195553058 2023-10-01T09:34:11.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079301+0000", 2023-10-01T09:34:11.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6779999999999999e-05 2023-10-01T09:34:11.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.431 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.431 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079324+0000", 2023-10-01T09:34:11.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2408999999999998e-05 2023-10-01T09:34:11.431 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.431 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.431 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.431 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.45674.0:169877 8.4 e370/369 ondisk, result = 0)", 2023-10-01T09:34:11.431 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:46.883855+0000", 2023-10-01T09:34:11.432 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 84.498159582, 2023-10-01T09:34:11.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19572051200000001, 2023-10-01T09:34:11.432 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.432 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:34:11.432 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.883855+0000", 2023-10-01T09:34:11.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.883855+0000", 2023-10-01T09:34:11.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.883854+0000", 2023-10-01T09:34:11.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:11.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.883862+0000", 2023-10-01T09:34:11.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0979999999999994e-06 2023-10-01T09:34:11.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.883871+0000", 2023-10-01T09:34:11.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6829999999999997e-06 2023-10-01T09:34:11.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.883877+0000", 2023-10-01T09:34:11.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4689999999999996e-06 2023-10-01T09:34:11.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079395+0000", 2023-10-01T09:34:11.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.195518094 2023-10-01T09:34:11.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079408+0000", 2023-10-01T09:34:11.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2836e-05 2023-10-01T09:34:11.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.435 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.435 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079576+0000", 2023-10-01T09:34:11.435 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000168042 2023-10-01T09:34:11.435 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.436 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.436 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.436 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.436 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.45674.0:169886 8.4 e370/369 ondisk, result = 0)", 2023-10-01T09:34:11.436 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:46.888824+0000", 2023-10-01T09:34:11.436 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 84.493190557999995, 2023-10-01T09:34:11.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.190789396, 2023-10-01T09:34:11.436 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.436 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:34:11.436 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.888824+0000", 2023-10-01T09:34:11.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.888824+0000", 2023-10-01T09:34:11.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.888824+0000", 2023-10-01T09:34:11.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:11.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.438 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.438 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.888828+0000", 2023-10-01T09:34:11.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4179999999999999e-06 2023-10-01T09:34:11.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.438 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.438 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079247+0000", 2023-10-01T09:34:11.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19041897299999999 2023-10-01T09:34:11.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079267+0000", 2023-10-01T09:34:11.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9966000000000002e-05 2023-10-01T09:34:11.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079597+0000", 2023-10-01T09:34:11.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00033025300000000002 2023-10-01T09:34:11.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079604+0000", 2023-10-01T09:34:11.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9720000000000003e-06 2023-10-01T09:34:11.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079613+0000", 2023-10-01T09:34:11.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3090000000000001e-06 2023-10-01T09:34:11.440 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.440 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.440 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.441 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.45674.0:169882 8.4 e370/369 ondisk, result = 0)", 2023-10-01T09:34:11.441 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:46.889567+0000", 2023-10-01T09:34:11.441 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 84.492447329000001, 2023-10-01T09:34:11.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19026179600000001, 2023-10-01T09:34:11.441 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.441 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:34:11.441 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.889567+0000", 2023-10-01T09:34:11.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.889567+0000", 2023-10-01T09:34:11.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.889567+0000", 2023-10-01T09:34:11.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:11.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.889571+0000", 2023-10-01T09:34:11.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3800000000000004e-06 2023-10-01T09:34:11.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079240+0000", 2023-10-01T09:34:11.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18966872500000001 2023-10-01T09:34:11.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.443 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079271+0000", 2023-10-01T09:34:11.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0998999999999999e-05 2023-10-01T09:34:11.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079667+0000", 2023-10-01T09:34:11.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000396005 2023-10-01T09:34:11.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079675+0000", 2023-10-01T09:34:11.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.943e-06 2023-10-01T09:34:11.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.079829+0000", 2023-10-01T09:34:11.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015433800000000001 2023-10-01T09:34:11.445 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.445 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.445 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.445 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:170342 8.d e371/369 8:b3ff9153:::benchmark_data_smithi138_431617_object68931:head v 371'10519, mlcod=371'10517)", 2023-10-01T09:34:11.445 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:47.673355+0000", 2023-10-01T09:34:11.445 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 83.708659908000001, 2023-10-01T09:34:11.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.185272875, 2023-10-01T09:34:11.446 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.446 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:11.446 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.673355+0000", 2023-10-01T09:34:11.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.673355+0000", 2023-10-01T09:34:11.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.673354+0000", 2023-10-01T09:34:11.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:34:11.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.673362+0000", 2023-10-01T09:34:11.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7279999999999995e-06 2023-10-01T09:34:11.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.448 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.448 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.673373+0000", 2023-10-01T09:34:11.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.082e-05 2023-10-01T09:34:11.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.448 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.448 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.673379+0000", 2023-10-01T09:34:11.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5400000000000001e-06 2023-10-01T09:34:11.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.448 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.673443+0000", 2023-10-01T09:34:11.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4269000000000004e-05 2023-10-01T09:34:11.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.673464+0000", 2023-10-01T09:34:11.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0441e-05 2023-10-01T09:34:11.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:11.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.683518+0000", 2023-10-01T09:34:11.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010053984 2023-10-01T09:34:11.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.858628+0000", 2023-10-01T09:34:11.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.175109872 2023-10-01T09:34:11.450 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.450 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.450 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.450 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:170343 8.d e371/369 8:b8b8e284:::benchmark_data_smithi138_431617_object68932:head v 371'10520, mlcod=371'10517)", 2023-10-01T09:34:11.450 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:47.677815+0000", 2023-10-01T09:34:11.450 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 83.704199518999999, 2023-10-01T09:34:11.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.180839369, 2023-10-01T09:34:11.451 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.451 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:11.451 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.677815+0000", 2023-10-01T09:34:11.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.677815+0000", 2023-10-01T09:34:11.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.677814+0000", 2023-10-01T09:34:11.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:34:11.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.677823+0000", 2023-10-01T09:34:11.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2930000000000001e-06 2023-10-01T09:34:11.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.677833+0000", 2023-10-01T09:34:11.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0749e-05 2023-10-01T09:34:11.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.677841+0000", 2023-10-01T09:34:11.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6890000000000004e-06 2023-10-01T09:34:11.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.677897+0000", 2023-10-01T09:34:11.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6407999999999999e-05 2023-10-01T09:34:11.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.677919+0000", 2023-10-01T09:34:11.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1665e-05 2023-10-01T09:34:11.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:11.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.858633+0000", 2023-10-01T09:34:11.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18071367299999999 2023-10-01T09:34:11.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.858655+0000", 2023-10-01T09:34:11.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1787e-05 2023-10-01T09:34:11.455 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.455 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.455 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.455 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:170341 8.1 e371/326 8:8a0e53b3:::benchmark_data_smithi138_431617_object68930:head v 371'10660, mlcod=371'10657)", 2023-10-01T09:34:11.455 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:47.680333+0000", 2023-10-01T09:34:11.455 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 83.701682148000003, 2023-10-01T09:34:11.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17826524199999999, 2023-10-01T09:34:11.456 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.456 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:11.456 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.680333+0000", 2023-10-01T09:34:11.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.680333+0000", 2023-10-01T09:34:11.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.680332+0000", 2023-10-01T09:34:11.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:11.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.680338+0000", 2023-10-01T09:34:11.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5829999999999997e-06 2023-10-01T09:34:11.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.680345+0000", 2023-10-01T09:34:11.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1080000000000004e-06 2023-10-01T09:34:11.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.462 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.680350+0000", 2023-10-01T09:34:11.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5300000000000004e-06 2023-10-01T09:34:11.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.462 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.462 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.680389+0000", 2023-10-01T09:34:11.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8368000000000003e-05 2023-10-01T09:34:11.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.680407+0000", 2023-10-01T09:34:11.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.791e-05 2023-10-01T09:34:11.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:11.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.858561+0000", 2023-10-01T09:34:11.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.178154008 2023-10-01T09:34:11.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.858598+0000", 2023-10-01T09:34:11.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7218999999999997e-05 2023-10-01T09:34:11.464 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.464 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.464 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.464 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:170345 8.b e371/361 8:dba22418:::benchmark_data_smithi138_431617_object68934:head v 371'10595, mlcod=371'10593)", 2023-10-01T09:34:11.464 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:47.681429+0000", 2023-10-01T09:34:11.465 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 83.700585919000005, 2023-10-01T09:34:11.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18147788300000001, 2023-10-01T09:34:11.465 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.465 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:11.465 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.465 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.465 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.681429+0000", 2023-10-01T09:34:11.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.465 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.465 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.466 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.466 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.681429+0000", 2023-10-01T09:34:11.466 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.466 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.466 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.466 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.681428+0000", 2023-10-01T09:34:11.466 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:11.466 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.467 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.467 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.467 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.681433+0000", 2023-10-01T09:34:11.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7060000000000003e-06 2023-10-01T09:34:11.467 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.467 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.467 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.467 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.681442+0000", 2023-10-01T09:34:11.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2739999999999996e-06 2023-10-01T09:34:11.467 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.467 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.681449+0000", 2023-10-01T09:34:11.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5189999999999998e-06 2023-10-01T09:34:11.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.681493+0000", 2023-10-01T09:34:11.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4165999999999997e-05 2023-10-01T09:34:11.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.469 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.681517+0000", 2023-10-01T09:34:11.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4239999999999998e-05 2023-10-01T09:34:11.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.469 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:11.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.862872+0000", 2023-10-01T09:34:11.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18135507000000001 2023-10-01T09:34:11.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.862907+0000", 2023-10-01T09:34:11.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4387000000000001e-05 2023-10-01T09:34:11.470 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.470 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.470 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.470 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPushReply(8.4 371/369 [PushReplyOp(8:2a5a16fa:::benchmark_data_smithi138_431617_object94550:head)])", 2023-10-01T09:34:11.470 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:47.682662+0000", 2023-10-01T09:34:11.470 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 83.699352513999997, 2023-10-01T09:34:11.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.176233484, 2023-10-01T09:34:11.471 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.471 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:34:11.471 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.682662+0000", 2023-10-01T09:34:11.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.682662+0000", 2023-10-01T09:34:11.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.682662+0000", 2023-10-01T09:34:11.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967296 2023-10-01T09:34:11.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.682666+0000", 2023-10-01T09:34:11.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6500000000000002e-06 2023-10-01T09:34:11.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.682673+0000", 2023-10-01T09:34:11.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2359999999999998e-06 2023-10-01T09:34:11.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.473 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.682680+0000", 2023-10-01T09:34:11.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9920000000000001e-06 2023-10-01T09:34:11.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.473 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.682724+0000", 2023-10-01T09:34:11.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4567999999999997e-05 2023-10-01T09:34:11.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.858896+0000", 2023-10-01T09:34:11.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.176171244 2023-10-01T09:34:11.474 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.475 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.475 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.475 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:170352 8.b e371/361 8:d12d7b32:::benchmark_data_smithi138_431617_object68941:head v 371'10596, mlcod=371'10593)", 2023-10-01T09:34:11.475 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:47.689506+0000", 2023-10-01T09:34:11.475 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 83.692509138999995, 2023-10-01T09:34:11.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.178523242, 2023-10-01T09:34:11.475 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.475 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:11.476 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.476 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.476 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.689506+0000", 2023-10-01T09:34:11.476 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.476 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.476 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.476 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.689506+0000", 2023-10-01T09:34:11.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.689505+0000", 2023-10-01T09:34:11.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:11.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.689512+0000", 2023-10-01T09:34:11.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.503e-06 2023-10-01T09:34:11.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.858983+0000", 2023-10-01T09:34:11.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16947113 2023-10-01T09:34:11.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.858993+0000", 2023-10-01T09:34:11.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6549999999999999e-06 2023-10-01T09:34:11.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.859149+0000", 2023-10-01T09:34:11.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015588700000000001 2023-10-01T09:34:11.479 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.859173+0000", 2023-10-01T09:34:11.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4229000000000001e-05 2023-10-01T09:34:11.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:11.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.867987+0000", 2023-10-01T09:34:11.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0088136900000000008 2023-10-01T09:34:11.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:47.868029+0000", 2023-10-01T09:34:11.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1832000000000002e-05 2023-10-01T09:34:11.481 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.481 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.481 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.482 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:186977 8.e e384/382 8:73c56b87:::benchmark_data_smithi138_431617_object85566:head v 384'11937, mlcod=384'11933)", 2023-10-01T09:34:11.482 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:18.709634+0000", 2023-10-01T09:34:11.482 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 52.672380191999999, 2023-10-01T09:34:11.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.061048482000000001, 2023-10-01T09:34:11.482 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.482 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:11.482 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.709634+0000", 2023-10-01T09:34:11.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.709634+0000", 2023-10-01T09:34:11.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.709634+0000", 2023-10-01T09:34:11.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:11.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.709641+0000", 2023-10-01T09:34:11.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4769999999999997e-06 2023-10-01T09:34:11.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.484 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.484 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.709652+0000", 2023-10-01T09:34:11.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0437e-05 2023-10-01T09:34:11.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.709660+0000", 2023-10-01T09:34:11.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9219999999999997e-06 2023-10-01T09:34:11.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.740270+0000", 2023-10-01T09:34:11.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.030610180000000001 2023-10-01T09:34:11.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.740285+0000", 2023-10-01T09:34:11.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4808e-05 2023-10-01T09:34:11.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:11.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.770644+0000", 2023-10-01T09:34:11.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.030359442 2023-10-01T09:34:11.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.770683+0000", 2023-10-01T09:34:11.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8923e-05 2023-10-01T09:34:11.487 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.487 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.487 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.487 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:187117 8.d 8:b22cc3e7:::benchmark_data_smithi138_431617_object85706:head [delete] snapc 0=[] ondisk+write+known_if_redirected e384)", 2023-10-01T09:34:11.487 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:18.920356+0000", 2023-10-01T09:34:11.487 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 52.461658428, 2023-10-01T09:34:11.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19144409300000001, 2023-10-01T09:34:11.487 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.487 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:11.487 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:11.488 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:34:11.488 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:11.488 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 187117 2023-10-01T09:34:11.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.488 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.920356+0000", 2023-10-01T09:34:11.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.920356+0000", 2023-10-01T09:34:11.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.920354+0000", 2023-10-01T09:34:11.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:34:11.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.920365+0000", 2023-10-01T09:34:11.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0438000000000001e-05 2023-10-01T09:34:11.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.920387+0000", 2023-10-01T09:34:11.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2127999999999999e-05 2023-10-01T09:34:11.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.490 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.920399+0000", 2023-10-01T09:34:11.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1574e-05 2023-10-01T09:34:11.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.490 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.920537+0000", 2023-10-01T09:34:11.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013871200000000001 2023-10-01T09:34:11.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.920676+0000", 2023-10-01T09:34:11.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013891000000000001 2023-10-01T09:34:11.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:34:11.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.920824+0000", 2023-10-01T09:34:11.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014758499999999999 2023-10-01T09:34:11.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:11.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925255+0000", 2023-10-01T09:34:11.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044314410000000004 2023-10-01T09:34:11.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:11.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.929915+0000", 2023-10-01T09:34:11.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0046594330000000002 2023-10-01T09:34:11.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:11.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111531+0000", 2023-10-01T09:34:11.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18161682000000001 2023-10-01T09:34:11.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:11.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111592+0000", 2023-10-01T09:34:11.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0594999999999998e-05 2023-10-01T09:34:11.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111800+0000", 2023-10-01T09:34:11.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000208259 2023-10-01T09:34:11.494 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.494 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.494 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.494 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:195984 8.1 e390/326 8:8d3eb30c:::benchmark_data_smithi138_431617_object94573:head v 390'12248, mlcod=390'12246)", 2023-10-01T09:34:11.494 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:34.789539+0000", 2023-10-01T09:34:11.494 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 36.592475385, 2023-10-01T09:34:11.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.069125814999999993, 2023-10-01T09:34:11.494 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.494 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:11.495 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.789539+0000", 2023-10-01T09:34:11.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.789539+0000", 2023-10-01T09:34:11.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.789538+0000", 2023-10-01T09:34:11.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:34:11.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.789549+0000", 2023-10-01T09:34:11.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0854e-05 2023-10-01T09:34:11.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.789563+0000", 2023-10-01T09:34:11.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4293e-05 2023-10-01T09:34:11.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.789571+0000", 2023-10-01T09:34:11.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.089e-06 2023-10-01T09:34:11.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.789614+0000", 2023-10-01T09:34:11.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2681e-05 2023-10-01T09:34:11.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.789635+0000", 2023-10-01T09:34:11.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0721e-05 2023-10-01T09:34:11.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:11.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.858615+0000", 2023-10-01T09:34:11.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.068980279000000005 2023-10-01T09:34:11.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.858665+0000", 2023-10-01T09:34:11.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0022999999999999e-05 2023-10-01T09:34:11.499 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.499 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.499 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.499 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:195985 8.f e390/343 8:fb2e44fe:::benchmark_data_smithi138_431617_object94574:head v 390'12272, mlcod=390'12267)", 2023-10-01T09:34:11.499 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:34.792914+0000", 2023-10-01T09:34:11.499 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 36.589100762999998, 2023-10-01T09:34:11.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.070874997999999995, 2023-10-01T09:34:11.500 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.500 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:11.500 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.792914+0000", 2023-10-01T09:34:11.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.792914+0000", 2023-10-01T09:34:11.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.792913+0000", 2023-10-01T09:34:11.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:34:11.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.792921+0000", 2023-10-01T09:34:11.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8120000000000008e-06 2023-10-01T09:34:11.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.792932+0000", 2023-10-01T09:34:11.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1232e-05 2023-10-01T09:34:11.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.792940+0000", 2023-10-01T09:34:11.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9969999999999998e-06 2023-10-01T09:34:11.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.855832+0000", 2023-10-01T09:34:11.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.062892142999999998 2023-10-01T09:34:11.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.855869+0000", 2023-10-01T09:34:11.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6602e-05 2023-10-01T09:34:11.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:11.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.863750+0000", 2023-10-01T09:34:11.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0078811060000000006 2023-10-01T09:34:11.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.863789+0000", 2023-10-01T09:34:11.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9020999999999998e-05 2023-10-01T09:34:11.505 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.505 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.505 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.505 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.45674.0:195988 8.4 e390/380 8:2a7f75b1:::benchmark_data_smithi138_431617_object94577:head v 390'12137, mlcod=368'10489)", 2023-10-01T09:34:11.505 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:34.797879+0000", 2023-10-01T09:34:11.505 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 36.584135285999999, 2023-10-01T09:34:11.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.062918513999999995, 2023-10-01T09:34:11.505 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:11.505 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:11.505 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:11.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:11.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.797879+0000", 2023-10-01T09:34:11.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:11.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.797879+0000", 2023-10-01T09:34:11.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:11.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:11.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.797878+0000", 2023-10-01T09:34:11.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:34:11.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:11.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.797887+0000", 2023-10-01T09:34:11.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8030000000000003e-06 2023-10-01T09:34:11.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:11.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.856581+0000", 2023-10-01T09:34:11.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.058693496999999997 2023-10-01T09:34:11.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:11.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.856592+0000", 2023-10-01T09:34:11.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0767999999999999e-05 2023-10-01T09:34:11.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:11.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.856728+0000", 2023-10-01T09:34:11.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000136167 2023-10-01T09:34:11.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:11.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.856752+0000", 2023-10-01T09:34:11.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4505999999999999e-05 2023-10-01T09:34:11.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:11.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.860767+0000", 2023-10-01T09:34:11.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040145379999999998 2023-10-01T09:34:11.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:11.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:11.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:11.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:34.860798+0000", 2023-10-01T09:34:11.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1111000000000001e-05 2023-10-01T09:34:11.511 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.511 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.511 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.511 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:11.511 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:11.511 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:34:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:34:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3567056645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:11.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:11.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:34:11.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3567056645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:34:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3567056645' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:11.727 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:11.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:11.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:11.830 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:11.859 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:11.905 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:11.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:11.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:12.010 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:12.037 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:12.079 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:12.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:12.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:12.187 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:34:12.217 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:12.311 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:34:12.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:12.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:34:12.415 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:12.442 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:12.485 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:12.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:12.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:12.496 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:12 smithi138 ceph-mon[192513]: pgmap v1109: 89 pgs: 89 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:12.497 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:12 smithi138 ceph-mon[185886]: pgmap v1109: 89 pgs: 89 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:12.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:12 smithi190 ceph-mon[160435]: pgmap v1109: 89 pgs: 89 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:12.591 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:12.618 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:12.692 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:34:13.185 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:13.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:13.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:13.288 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:13.320 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:13.335 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:34:13.368 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:13.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:13.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:13.477 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:34:13.480 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:13.517 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:13.664 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:34:13.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:13.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:34:13.768 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:13.800 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:13.845 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:13.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:13.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:13.950 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:13.978 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:14.046 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:14.046 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:14.177 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:14.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:14.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:14.281 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:14.308 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:14.541 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:14.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:14.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:14.636 INFO:teuthology.orchestra.run.smithi138.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-10-01T09:34:14.645 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:14.672 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:14.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:14 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:14 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2903814881' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:34:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:14 smithi138 ceph-mon[192513]: pgmap v1110: 89 pgs: 89 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:14 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:34:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:14 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:34:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:14 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:14.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:14 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:14.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:14 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2903814881' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:34:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:14 smithi138 ceph-mon[185886]: pgmap v1110: 89 pgs: 89 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:14 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:34:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:14 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:34:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:14 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:14.923 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:14.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:14.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:15.027 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:15.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:14 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:14 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2903814881' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:34:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:14 smithi190 ceph-mon[160435]: pgmap v1110: 89 pgs: 89 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:14 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:34:15.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:14 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:34:15.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:14 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:15.059 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:15.108 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:15.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:15.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:15.182 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:34:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:34:14] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.33.4" 2023-10-01T09:34:15.214 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:34:15.246 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:15.400 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:34:15.746 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:34:15.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:15.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:34:15.851 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:15.878 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:15.981 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:15 smithi138 ceph-mon[185886]: failed to encode map e404 with expected crc 2023-10-01T09:34:15.981 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:15 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2903814881' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:34:15.982 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:15 smithi138 ceph-mon[185886]: osdmap e404: 8 total, 8 up, 8 in 2023-10-01T09:34:15.982 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:15 smithi138 ceph-mon[185886]: failed to encode map e404 with expected crc 2023-10-01T09:34:15.982 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:15 smithi138 ceph-mon[192513]: failed to encode map e404 with expected crc 2023-10-01T09:34:15.982 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:15 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2903814881' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:34:15.982 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:15 smithi138 ceph-mon[192513]: osdmap e404: 8 total, 8 up, 8 in 2023-10-01T09:34:15.983 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:15 smithi138 ceph-mon[192513]: failed to encode map e404 with expected crc 2023-10-01T09:34:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:15 smithi190 ceph-mon[160435]: failed to encode map e404 with expected crc 2023-10-01T09:34:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:15 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2903814881' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:34:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:15 smithi190 ceph-mon[160435]: osdmap e404: 8 total, 8 up, 8 in 2023-10-01T09:34:16.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:15 smithi190 ceph-mon[160435]: failed to encode map e404 with expected crc 2023-10-01T09:34:16.692 DEBUG:teuthology.orchestra.run.smithi138:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-10-01T09:34:16.694 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T09:34:16.694 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T09:34:16.742 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:hints = 1 2023-10-01T09:34:16.742 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T09:34:16.742 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Object prefix: benchmark_data_smithi138_497829 2023-10-01T09:34:16.743 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:34:16.743 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 0 0 0 0 0 0 - 0 2023-10-01T09:34:16.831 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:16.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:16.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:16.914 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:16 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:16.914 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:16 smithi138 ceph-mon[192513]: pgmap v1112: 89 pgs: 89 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:16.919 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:16 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:16.919 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:16 smithi138 ceph-mon[185886]: pgmap v1112: 89 pgs: 89 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:16.940 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:16.982 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:17.071 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:17.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:17.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:17.093 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:17.178 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:17.247 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:17.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:16 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:16 smithi190 ceph-mon[160435]: pgmap v1112: 89 pgs: 89 active+clean; 4.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-10-01T09:34:17.525 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:17.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:17.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:17.631 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:17.663 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:34:17.663 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:34:17.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:17.663 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:2516 9.4 e404/400)", 2023-10-01T09:34:17.663 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:34:17.655664+0000", 2023-10-01T09:34:17.664 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 9.6529999999999999e-05, 2023-10-01T09:34:17.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012230699999999999, 2023-10-01T09:34:17.664 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:17.664 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:34:17.664 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:17.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:17.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:17.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:17.655664+0000", 2023-10-01T09:34:17.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:17.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:17.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:17.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:17.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:17.655664+0000", 2023-10-01T09:34:17.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:17.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:17.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:17.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:17.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:17.655663+0000", 2023-10-01T09:34:17.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:34:17.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:17.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:17.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:17.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:17.655685+0000", 2023-10-01T09:34:17.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2027000000000001e-05 2023-10-01T09:34:17.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:17.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:17.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:17.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:17.655699+0000", 2023-10-01T09:34:17.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4724e-05 2023-10-01T09:34:17.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:17.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:17.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:17.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:17.655707+0000", 2023-10-01T09:34:17.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2690000000000002e-06 2023-10-01T09:34:17.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:17.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:17.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:17.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:17.655757+0000", 2023-10-01T09:34:17.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0158000000000003e-05 2023-10-01T09:34:17.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:17.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:17.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:17.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:17.655776+0000", 2023-10-01T09:34:17.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9334000000000002e-05 2023-10-01T09:34:17.667 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:17.668 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:17.668 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:17.668 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:17.668 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:34:17.668 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 1 2023-10-01T09:34:17.668 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:34:17.687 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:17.742 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 1 16 2742 2726 170.366 170.375 0.00379237 0.00584176 2023-10-01T09:34:18.506 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.5 to 1 (10000) 2023-10-01T09:34:18.512 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:34:18.547 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:18 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1485438306' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:34:18.547 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:18 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:34:18.547 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:18 smithi138 ceph-mon[192513]: pgmap v1113: 89 pgs: 89 active+clean; 6.1 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s wr, 2 op/s 2023-10-01T09:34:18.547 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:18 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/653552658' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:34:18.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:18 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1485438306' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:34:18.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:18 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:34:18.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:18 smithi138 ceph-mon[185886]: pgmap v1113: 89 pgs: 89 active+clean; 6.1 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s wr, 2 op/s 2023-10-01T09:34:18.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:18 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/653552658' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:34:18.743 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 2 16 6319 6303 196.912 223.562 0.00301886 0.0050697 2023-10-01T09:34:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:18 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1485438306' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:34:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:18 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:34:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:18 smithi190 ceph-mon[160435]: pgmap v1113: 89 pgs: 89 active+clean; 6.1 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s wr, 2 op/s 2023-10-01T09:34:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:18 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/653552658' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:34:18.901 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:18.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:18.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:19.008 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:19.061 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:19.743 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 3 16 9399 9383 195.436 192.5 0.00432511 0.00510887 2023-10-01T09:34:19.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:19 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:34:19.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:19 smithi138 ceph-mon[185886]: failed to encode map e405 with expected crc 2023-10-01T09:34:19.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:19 smithi138 ceph-mon[185886]: failed to encode map e405 with expected crc 2023-10-01T09:34:19.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:19 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:34:19.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/653552658' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:34:19.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:19 smithi138 ceph-mon[185886]: osdmap e405: 8 total, 8 up, 8 in 2023-10-01T09:34:19.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:19 smithi138 ceph-mon[185886]: failed to encode map e405 with expected crc 2023-10-01T09:34:19.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:19 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:34:19.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:19 smithi138 ceph-mon[192513]: failed to encode map e405 with expected crc 2023-10-01T09:34:19.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:19 smithi138 ceph-mon[192513]: failed to encode map e405 with expected crc 2023-10-01T09:34:19.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:19 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:34:19.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:19 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/653552658' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:34:19.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:19 smithi138 ceph-mon[192513]: osdmap e405: 8 total, 8 up, 8 in 2023-10-01T09:34:19.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:19 smithi138 ceph-mon[192513]: failed to encode map e405 with expected crc 2023-10-01T09:34:19.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:19 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:34:19.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:19 smithi190 ceph-mon[160435]: failed to encode map e405 with expected crc 2023-10-01T09:34:19.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:19 smithi190 ceph-mon[160435]: failed to encode map e405 with expected crc 2023-10-01T09:34:19.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:19 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:34:19.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:19 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/653552658' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:34:19.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:19 smithi190 ceph-mon[160435]: osdmap e405: 8 total, 8 up, 8 in 2023-10-01T09:34:19.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:19 smithi190 ceph-mon[160435]: failed to encode map e405 with expected crc 2023-10-01T09:34:20.114 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:20.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:20.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:20.220 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:20.267 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:20.743 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 4 16 11802 11786 184.12 150.188 0.00129858 0.00532892 2023-10-01T09:34:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:20 smithi190 ceph-mon[160435]: pgmap v1115: 89 pgs: 89 active+clean; 320 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s wr, 631 op/s 2023-10-01T09:34:20.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:20 smithi138 ceph-mon[192513]: pgmap v1115: 89 pgs: 89 active+clean; 320 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s wr, 631 op/s 2023-10-01T09:34:20.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:20 smithi138 ceph-mon[185886]: pgmap v1115: 89 pgs: 89 active+clean; 320 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 39 MiB/s wr, 631 op/s 2023-10-01T09:34:21.743 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 5 16 14284 14268 178.318 155.125 0.00739841 0.00560188 2023-10-01T09:34:21.907 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:21.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:21.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:22.013 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:22.039 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:22.404 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:22.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:22.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:22.419 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:22 smithi138 ceph-mon[192513]: pgmap v1116: 89 pgs: 89 active+clean; 599 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T09:34:22.420 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:22 smithi138 ceph-mon[185886]: pgmap v1116: 89 pgs: 89 active+clean; 599 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T09:34:22.496 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:34:22.510 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:22.532 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-01T09:34:22.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:22 smithi190 ceph-mon[160435]: pgmap v1116: 89 pgs: 89 active+clean; 599 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T09:34:22.556 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:22.633 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:22.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:22.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:22.738 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:22.746 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 6 16 16152 16136 168.056 116.75 0.00347094 0.00594506 2023-10-01T09:34:22.874 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:22.974 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:22.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:22.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:23.084 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:23.123 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:23.153 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:34:23.183 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:34:23.187 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:23.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:23.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:23.233 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:34:23.248 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:23.271 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:34:23.293 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:23.306 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:34:23.314 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:23.329 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:23.341 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:34:23.380 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:34:23.402 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:23.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:23.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:23.415 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:34:23.426 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:34:23.426 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:34:23.426 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:34:23.426 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:34:23.426 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:34:23.426 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:34:23.426 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:34:23.426 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:34:23.509 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:23.551 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:23.630 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:23.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:23.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:23.738 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:23.743 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 7 16 18237 18221 162.663 130.312 0.00523142 0.00614218 2023-10-01T09:34:23.783 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:23 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2116428592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:23.787 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:23.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:23 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2116428592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:23.807 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:23 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2116428592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:23.875 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:23.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:23.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:23.981 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:24.416 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:24.460 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:34:24.500 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:24.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:24.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:24.609 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:24.654 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:24.726 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:24.729 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:24 smithi190 ceph-mon[160435]: pgmap v1117: 89 pgs: 89 active+clean; 925 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-10-01T09:34:24.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:24.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:24.744 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 8 16 20782 20766 162.212 159.062 0.00363979 0.0061589 2023-10-01T09:34:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:24 smithi138 ceph-mon[185886]: pgmap v1117: 89 pgs: 89 active+clean; 925 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-10-01T09:34:24.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:24 smithi138 ceph-mon[192513]: pgmap v1117: 89 pgs: 89 active+clean; 925 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-10-01T09:34:24.831 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:24.887 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:24.973 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:24.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:24.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:25.082 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:25.108 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:25.150 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:25.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:25.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:25.258 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:25.300 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:25.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:34:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:34:24] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.33.4" 2023-10-01T09:34:25.389 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:25.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:25.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:25.496 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:25.553 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:25.744 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 9 16 22689 22673 157.43 119.188 0.0017417 0.00631571 2023-10-01T09:34:26.231 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:26.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:26.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:26.344 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:26.383 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:26.455 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:26.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:26.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:26.549 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:26.549 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:34:26.549 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:26 smithi138 ceph-mon[192513]: pgmap v1118: 89 pgs: 89 active+clean; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s 2023-10-01T09:34:26.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:26.550 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:34:26.550 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:26 smithi138 ceph-mon[185886]: pgmap v1118: 89 pgs: 89 active+clean; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s 2023-10-01T09:34:26.565 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:26.607 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:26.607 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:34:26.607 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:26 smithi190 ceph-mon[160435]: pgmap v1118: 89 pgs: 89 active+clean; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s 2023-10-01T09:34:26.610 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:26.702 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:26.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:26.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:26.744 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 10 16 24786 24770 154.792 131.062 0.0031996 0.00645434 2023-10-01T09:34:26.809 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:26.835 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:26.877 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:26.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:26.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:26.983 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:27.030 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:27.352 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:27.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:27.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:27.458 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:27.466 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:27.533 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:27.744 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 11 16 26534 26518 150.651 109.25 0.00312431 0.00663236 2023-10-01T09:34:27.918 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:27.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:27.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:28.026 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:28.069 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:28.154 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:28.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:28.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:28.248 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:34:28.260 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:28.368 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:28.551 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:28 smithi138 ceph-mon[185886]: pgmap v1119: 89 pgs: 89 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-01T09:34:28.551 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:28 smithi138 ceph-mon[192513]: pgmap v1119: 89 pgs: 89 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-01T09:34:28.575 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:28.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:28.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:28.603 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:34:28.604 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:34:28.682 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:28.717 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:28.745 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 12 16 28425 28409 147.932 118.188 0.00278993 0.0067547 2023-10-01T09:34:28.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:28 smithi190 ceph-mon[160435]: pgmap v1119: 89 pgs: 89 active+clean; 1.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-01T09:34:28.810 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:28.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:28.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:28.916 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:28.973 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:29.080 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:29.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:29.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:29.190 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:29.242 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:29.516 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.6 to 1 (10000) 2023-10-01T09:34:29.745 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 13 16 29819 29803 143.253 87.125 0.0014658 0.0069615 2023-10-01T09:34:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:29 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1926924148' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:34:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:29 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:34:29.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:29 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1926924148' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:34:29.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:29 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:34:29.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:29 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1926924148' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:34:29.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:29 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:34:30.217 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:30.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:30.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:30.328 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:30.385 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:30.479 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:30.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:30.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:30.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:30 smithi138 ceph-mon[192513]: failed to encode map e406 with expected crc 2023-10-01T09:34:30.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:30 smithi138 ceph-mon[192513]: failed to encode map e406 with expected crc 2023-10-01T09:34:30.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:30 smithi138 ceph-mon[192513]: failed to encode map e406 with expected crc 2023-10-01T09:34:30.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:30 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:34:30.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:30 smithi138 ceph-mon[192513]: osdmap e406: 8 total, 8 up, 8 in 2023-10-01T09:34:30.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:30 smithi138 ceph-mon[192513]: pgmap v1121: 89 pgs: 89 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-01T09:34:30.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:30 smithi138 ceph-mon[185886]: failed to encode map e406 with expected crc 2023-10-01T09:34:30.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:30 smithi138 ceph-mon[185886]: failed to encode map e406 with expected crc 2023-10-01T09:34:30.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:30 smithi138 ceph-mon[185886]: failed to encode map e406 with expected crc 2023-10-01T09:34:30.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:30 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:34:30.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:30 smithi138 ceph-mon[185886]: osdmap e406: 8 total, 8 up, 8 in 2023-10-01T09:34:30.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:30 smithi138 ceph-mon[185886]: pgmap v1121: 89 pgs: 89 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-01T09:34:30.587 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:30.646 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:30.746 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 14 16 31401 31385 140.073 98.875 0.0040042 0.00713435 2023-10-01T09:34:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:30 smithi190 ceph-mon[160435]: failed to encode map e406 with expected crc 2023-10-01T09:34:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:30 smithi190 ceph-mon[160435]: failed to encode map e406 with expected crc 2023-10-01T09:34:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:30 smithi190 ceph-mon[160435]: failed to encode map e406 with expected crc 2023-10-01T09:34:30.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:30 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:34:30.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:30 smithi190 ceph-mon[160435]: osdmap e406: 8 total, 8 up, 8 in 2023-10-01T09:34:30.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:30 smithi190 ceph-mon[160435]: pgmap v1121: 89 pgs: 89 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2023-10-01T09:34:31.469 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-01T09:34:31.556 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:31.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:31.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:31.583 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:34:31.663 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:31.684 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:31.747 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 15 16 33122 33106 137.906 107.562 0.00633761 0.00724676 2023-10-01T09:34:31.800 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:31.878 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:31.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:31.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:31.983 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:32.071 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:32.274 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:32.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:32.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:32.288 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:32 smithi138 ceph-mon[185886]: pgmap v1122: 89 pgs: 89 active+clean; 1.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-01T09:34:32.289 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:32 smithi138 ceph-mon[192513]: pgmap v1122: 89 pgs: 89 active+clean; 1.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-01T09:34:32.389 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:32.468 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:32.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:32 smithi190 ceph-mon[160435]: pgmap v1122: 89 pgs: 89 active+clean; 1.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-10-01T09:34:32.573 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:32.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:32.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:32.678 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:32.724 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:32.746 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 16 16 34687 34671 135.4 97.8125 0.00527192 0.00738031 2023-10-01T09:34:32.793 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:32.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:32.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:32.901 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:32.950 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:32.991 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:32.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:32.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:33.097 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:33.151 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:33.437 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:33.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:33.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:33.520 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:34:33.533 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:33.548 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:33.595 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:33.633 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:33.748 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 17 16 35982 35966 132.189 80.9375 0.00571595 0.00756009 2023-10-01T09:34:33.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:33 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3456278042' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:33.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:33 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:33.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:33 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3456278042' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:33.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:33 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:33.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:33 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3456278042' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:33.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:33 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:33.823 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:33.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:33.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:33.929 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:33.978 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:34.073 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:34.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:34.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:34.181 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:34.209 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:34.749 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 18 16 37382 37366 129.702 87.5 0.00555631 0.00770435 2023-10-01T09:34:34.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:34 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:34:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:34 smithi190 ceph-mon[160435]: failed to encode map e407 with expected crc 2023-10-01T09:34:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:34 smithi190 ceph-mon[160435]: failed to encode map e407 with expected crc 2023-10-01T09:34:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:34 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:34:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:34 smithi190 ceph-mon[160435]: osdmap e407: 8 total, 8 up, 8 in 2023-10-01T09:34:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:34 smithi190 ceph-mon[160435]: pgmap v1124: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-01T09:34:34.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:34 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:34:34.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:34 smithi138 ceph-mon[185886]: failed to encode map e407 with expected crc 2023-10-01T09:34:34.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:34 smithi138 ceph-mon[185886]: failed to encode map e407 with expected crc 2023-10-01T09:34:34.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:34 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:34:34.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:34 smithi138 ceph-mon[185886]: osdmap e407: 8 total, 8 up, 8 in 2023-10-01T09:34:34.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:34 smithi138 ceph-mon[185886]: pgmap v1124: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-01T09:34:34.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:34 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:34:34.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:34 smithi138 ceph-mon[192513]: failed to encode map e407 with expected crc 2023-10-01T09:34:34.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:34 smithi138 ceph-mon[192513]: failed to encode map e407 with expected crc 2023-10-01T09:34:34.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:34 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:34:34.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:34 smithi138 ceph-mon[192513]: osdmap e407: 8 total, 8 up, 8 in 2023-10-01T09:34:34.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:34 smithi138 ceph-mon[192513]: pgmap v1124: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-01T09:34:35.157 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:35.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:35.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:35.189 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:34:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:34:34] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.33.4" 2023-10-01T09:34:35.272 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:35.334 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:35.442 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:35.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:35.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:35.472 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:34:35.516 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:34:35.551 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:35.554 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:34:35.592 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:34:35.605 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2956545270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:35.606 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2956545270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:35.622 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:35.633 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:34:35.672 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:34:35.717 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:34:35.748 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:34:35.750 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 19 16 39069 39053 128.413 105.438 0.00737745 0.00778125 2023-10-01T09:34:35.768 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:34:35.768 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:34:35.768 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:34:35.768 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:34:35.768 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:34:35.768 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:34:35.768 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:34:35.769 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:34:35.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2956545270' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:36.750 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:34:36.750713+0000 min lat: 0.00117708 max lat: 0.286267 avg lat: 0.00793032 2023-10-01T09:34:36.750 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:34:36.750 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 20 15 40354 40339 126.012 80.375 0.0102264 0.00793032 2023-10-01T09:34:36.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:36 smithi190 ceph-mon[160435]: pgmap v1125: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T09:34:36.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:36 smithi138 ceph-mon[192513]: pgmap v1125: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T09:34:36.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:36 smithi138 ceph-mon[185886]: pgmap v1125: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T09:34:37.128 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:37.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:37.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:37.170 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:34:37.232 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:37.281 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:37.356 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:37.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:37.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:37.463 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:37.496 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:37.750 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 21 16 41449 41433 123.267 68.375 0.0109069 0.0081073 2023-10-01T09:34:38.178 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:38.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:38.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:38.282 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:38.345 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:38.459 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:38.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:38.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:38.606 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:38.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:38 smithi138 ceph-mon[185886]: pgmap v1126: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T09:34:38.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:38 smithi138 ceph-mon[192513]: pgmap v1126: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T09:34:38.681 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:38.750 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 22 16 43063 43047 122.25 100.875 0.00703702 0.00817591 2023-10-01T09:34:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:38 smithi190 ceph-mon[160435]: pgmap v1126: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T09:34:38.830 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:38.881 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:38.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:38.905 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:34:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:34:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:34:38.905 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:34:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129093 7.c 7:3fad7313:::benchmark_data_smithi138_367058_object31298:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:34:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:48.990163+0000", 2023-10-01T09:34:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 469.91080544300002, 2023-10-01T09:34:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.108214129, 2023-10-01T09:34:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:34:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:34:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129093 2023-10-01T09:34:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:48.990163+0000", 2023-10-01T09:34:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:48.990163+0000", 2023-10-01T09:34:38.907 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:48.990161+0000", 2023-10-01T09:34:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:34:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:38.908 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.054 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.054 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:48.990167+0000", 2023-10-01T09:34:39.054 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3000000000000001e-06 2023-10-01T09:34:39.054 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.054 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.055 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.055 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:48.990175+0000", 2023-10-01T09:34:39.055 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.585e-06 2023-10-01T09:34:39.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:48.990182+0000", 2023-10-01T09:34:39.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2929999999999997e-06 2023-10-01T09:34:39.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.021171+0000", 2023-10-01T09:34:39.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.030989045999999999 2023-10-01T09:34:39.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:34:39.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.021179+0000", 2023-10-01T09:34:39.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3869999999999995e-06 2023-10-01T09:34:39.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.073219+0000", 2023-10-01T09:34:39.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0520403949999999 2023-10-01T09:34:39.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.073363+0000", 2023-10-01T09:34:39.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000143091 2023-10-01T09:34:39.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6,7", 2023-10-01T09:34:39.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.073536+0000", 2023-10-01T09:34:39.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017386100000000001 2023-10-01T09:34:39.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.076964+0000", 2023-10-01T09:34:39.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034275680000000002 2023-10-01T09:34:39.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:39.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.076971+0000", 2023-10-01T09:34:39.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6150000000000002e-06 2023-10-01T09:34:39.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.077028+0000", 2023-10-01T09:34:39.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7528000000000002e-05 2023-10-01T09:34:39.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.062 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.062 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.097535+0000", 2023-10-01T09:34:39.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.02050714 2023-10-01T09:34:39.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.062 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.062 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.062 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.097590+0000", 2023-10-01T09:34:39.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4523e-05 2023-10-01T09:34:39.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.062 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.062 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.063 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.098377+0000", 2023-10-01T09:34:39.063 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000786983 2023-10-01T09:34:39.063 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.063 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.063 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.063 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.063 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129107 7.1 7:86d994f7:::benchmark_data_smithi138_367058_object31312:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:34:39.063 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:49.013753+0000", 2023-10-01T09:34:39.063 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 469.88721479200001, 2023-10-01T09:34:39.063 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.064642614, 2023-10-01T09:34:39.063 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.064 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.064 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:39.064 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:34:39.064 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:34:39.064 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129107 2023-10-01T09:34:39.064 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.064 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013753+0000", 2023-10-01T09:34:39.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013753+0000", 2023-10-01T09:34:39.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013752+0000", 2023-10-01T09:34:39.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:34:39.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013758+0000", 2023-10-01T09:34:39.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.908e-06 2023-10-01T09:34:39.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013767+0000", 2023-10-01T09:34:39.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7820000000000005e-06 2023-10-01T09:34:39.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013774+0000", 2023-10-01T09:34:39.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6279999999999999e-06 2023-10-01T09:34:39.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013826+0000", 2023-10-01T09:34:39.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1839999999999998e-05 2023-10-01T09:34:39.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:34:39.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.013839+0000", 2023-10-01T09:34:39.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2704000000000001e-05 2023-10-01T09:34:39.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.067857+0000", 2023-10-01T09:34:39.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.054017991 2023-10-01T09:34:39.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.068023+0000", 2023-10-01T09:34:39.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016644699999999999 2023-10-01T09:34:39.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,4,6", 2023-10-01T09:34:39.069 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.068242+0000", 2023-10-01T09:34:39.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000219445 2023-10-01T09:34:39.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.069 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.071910+0000", 2023-10-01T09:34:39.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036673579999999999 2023-10-01T09:34:39.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.069 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.071944+0000", 2023-10-01T09:34:39.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4419999999999999e-05 2023-10-01T09:34:39.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:39.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.071954+0000", 2023-10-01T09:34:39.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4590000000000004e-06 2023-10-01T09:34:39.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.072007+0000", 2023-10-01T09:34:39.071 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:34:39.072 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2777e-05 2023-10-01T09:34:39.072 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.077735+0000", 2023-10-01T09:34:39.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057282039999999998 2023-10-01T09:34:39.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.077771+0000", 2023-10-01T09:34:39.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5790000000000001e-05 2023-10-01T09:34:39.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078396+0000", 2023-10-01T09:34:39.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00062535800000000001 2023-10-01T09:34:39.074 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.074 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.074 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.074 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129109 7.c 7:3aabb763:::benchmark_data_smithi138_367058_object31314:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:34:39.074 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:49.014366+0000", 2023-10-01T09:34:39.074 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 469.886601697, 2023-10-01T09:34:39.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.084557913, 2023-10-01T09:34:39.074 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.075 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.075 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:39.075 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:34:39.075 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:34:39.075 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129109 2023-10-01T09:34:39.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.075 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.014366+0000", 2023-10-01T09:34:39.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.076 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.014366+0000", 2023-10-01T09:34:39.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.076 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.014365+0000", 2023-10-01T09:34:39.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:34:39.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.014371+0000", 2023-10-01T09:34:39.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1129999999999997e-06 2023-10-01T09:34:39.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.077 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.014387+0000", 2023-10-01T09:34:39.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5548000000000001e-05 2023-10-01T09:34:39.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.077 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.014396+0000", 2023-10-01T09:34:39.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6589999999999996e-06 2023-10-01T09:34:39.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.021197+0000", 2023-10-01T09:34:39.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0068016149999999996 2023-10-01T09:34:39.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:34:39.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.021207+0000", 2023-10-01T09:34:39.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7140000000000005e-06 2023-10-01T09:34:39.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.077079+0000", 2023-10-01T09:34:39.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.055872071 2023-10-01T09:34:39.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.077198+0000", 2023-10-01T09:34:39.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000118742 2023-10-01T09:34:39.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6,7", 2023-10-01T09:34:39.080 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.077322+0000", 2023-10-01T09:34:39.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012386499999999999 2023-10-01T09:34:39.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.080 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.080 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.085284+0000", 2023-10-01T09:34:39.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0079620330000000003 2023-10-01T09:34:39.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.080 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:39.080 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.085298+0000", 2023-10-01T09:34:39.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4591e-05 2023-10-01T09:34:39.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.085366+0000", 2023-10-01T09:34:39.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8028000000000007e-05 2023-10-01T09:34:39.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.098440+0000", 2023-10-01T09:34:39.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013073664 2023-10-01T09:34:39.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.098492+0000", 2023-10-01T09:34:39.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1774000000000003e-05 2023-10-01T09:34:39.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.098924+0000", 2023-10-01T09:34:39.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00043260299999999999 2023-10-01T09:34:39.082 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.082 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.082 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.083 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.083 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129118 7.1 7:8edf0bff:::benchmark_data_smithi138_367058_object31323:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:34:39.083 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:49.038527+0000", 2023-10-01T09:34:39.083 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 469.862441042, 2023-10-01T09:34:39.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0404381739999999, 2023-10-01T09:34:39.083 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.083 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.083 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:39.083 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:34:39.083 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:34:39.083 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129118 2023-10-01T09:34:39.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.084 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.084 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.084 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.084 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.038527+0000", 2023-10-01T09:34:39.084 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.084 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.084 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.084 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.038527+0000", 2023-10-01T09:34:39.084 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.085 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.038525+0000", 2023-10-01T09:34:39.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:34:39.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.085 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.038536+0000", 2023-10-01T09:34:39.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1782e-05 2023-10-01T09:34:39.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.085 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.038549+0000", 2023-10-01T09:34:39.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2228999999999999e-05 2023-10-01T09:34:39.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.038557+0000", 2023-10-01T09:34:39.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5220000000000007e-06 2023-10-01T09:34:39.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.039875+0000", 2023-10-01T09:34:39.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001317384 2023-10-01T09:34:39.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:34:39.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.039884+0000", 2023-10-01T09:34:39.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3789999999999995e-06 2023-10-01T09:34:39.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.072039+0000", 2023-10-01T09:34:39.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.032154623 2023-10-01T09:34:39.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.072146+0000", 2023-10-01T09:34:39.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107936 2023-10-01T09:34:39.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,4,6", 2023-10-01T09:34:39.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.072264+0000", 2023-10-01T09:34:39.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011767799999999999 2023-10-01T09:34:39.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078438+0000", 2023-10-01T09:34:39.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.006174194 2023-10-01T09:34:39.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078468+0000", 2023-10-01T09:34:39.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9221999999999999e-05 2023-10-01T09:34:39.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:39.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078476+0000", 2023-10-01T09:34:39.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7029999999999995e-06 2023-10-01T09:34:39.090 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078511+0000", 2023-10-01T09:34:39.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4972999999999999e-05 2023-10-01T09:34:39.090 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078537+0000", 2023-10-01T09:34:39.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5562e-05 2023-10-01T09:34:39.091 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.091 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.091 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.091 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078571+0000", 2023-10-01T09:34:39.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4350999999999997e-05 2023-10-01T09:34:39.091 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.091 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.091 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.091 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078965+0000", 2023-10-01T09:34:39.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000394001 2023-10-01T09:34:39.091 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.092 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.092 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.092 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.092 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.092 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129121 7.c 7:33e29068:::benchmark_data_smithi138_367058_object31326:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:34:39.092 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:49.051338+0000", 2023-10-01T09:34:39.092 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 469.84962986800002, 2023-10-01T09:34:39.092 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0484849009999999, 2023-10-01T09:34:39.092 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.092 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.092 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:39.092 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:34:39.093 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:34:39.093 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129121 2023-10-01T09:34:39.093 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.093 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.093 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.093 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.093 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051338+0000", 2023-10-01T09:34:39.093 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.093 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.093 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.093 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051338+0000", 2023-10-01T09:34:39.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.094 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.094 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.094 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051337+0000", 2023-10-01T09:34:39.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:34:39.094 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.094 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.094 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051342+0000", 2023-10-01T09:34:39.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6739999999999996e-06 2023-10-01T09:34:39.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051350+0000", 2023-10-01T09:34:39.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5480000000000002e-06 2023-10-01T09:34:39.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051356+0000", 2023-10-01T09:34:39.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.896e-06 2023-10-01T09:34:39.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051399+0000", 2023-10-01T09:34:39.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2916e-05 2023-10-01T09:34:39.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:34:39.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051408+0000", 2023-10-01T09:34:39.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.969e-06 2023-10-01T09:34:39.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079606+0000", 2023-10-01T09:34:39.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.028198366 2023-10-01T09:34:39.097 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.097 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079762+0000", 2023-10-01T09:34:39.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015603000000000001 2023-10-01T09:34:39.097 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.097 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6,7", 2023-10-01T09:34:39.097 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079923+0000", 2023-10-01T09:34:39.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016086299999999999 2023-10-01T09:34:39.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.098 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:39.098 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.097463+0000", 2023-10-01T09:34:39.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017540224 2023-10-01T09:34:39.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.098 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.098 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.098992+0000", 2023-10-01T09:34:39.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015289769999999999 2023-10-01T09:34:39.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.099 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.099 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099153+0000", 2023-10-01T09:34:39.099 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000160178 2023-10-01T09:34:39.099 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.099 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.099 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099285+0000", 2023-10-01T09:34:39.099 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013285099999999999 2023-10-01T09:34:39.099 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.099 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.099 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099345+0000", 2023-10-01T09:34:39.100 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9145999999999999e-05 2023-10-01T09:34:39.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.100 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.100 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.100 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099823+0000", 2023-10-01T09:34:39.100 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00047857500000000002 2023-10-01T09:34:39.100 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.100 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.100 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.100 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.100 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129122 7.1 7:8ace0e85:::benchmark_data_smithi138_367058_object31327:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:34:39.101 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:49.051372+0000", 2023-10-01T09:34:39.101 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 469.84959647900001, 2023-10-01T09:34:39.101 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0346654209999999, 2023-10-01T09:34:39.101 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.101 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.101 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:39.101 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:34:39.101 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:34:39.101 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129122 2023-10-01T09:34:39.101 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.101 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.102 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.102 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.102 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051372+0000", 2023-10-01T09:34:39.102 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.102 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.102 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.102 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.102 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051372+0000", 2023-10-01T09:34:39.102 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.102 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.102 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.102 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051371+0000", 2023-10-01T09:34:39.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:39.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051374+0000", 2023-10-01T09:34:39.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.666e-06 2023-10-01T09:34:39.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051379+0000", 2023-10-01T09:34:39.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5389999999999998e-06 2023-10-01T09:34:39.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051384+0000", 2023-10-01T09:34:39.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6789999999999996e-06 2023-10-01T09:34:39.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051524+0000", 2023-10-01T09:34:39.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0001404 2023-10-01T09:34:39.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:34:39.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051534+0000", 2023-10-01T09:34:39.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4919999999999999e-06 2023-10-01T09:34:39.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.074933+0000", 2023-10-01T09:34:39.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.023398606 2023-10-01T09:34:39.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.075080+0000", 2023-10-01T09:34:39.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014698599999999999 2023-10-01T09:34:39.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.106 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,4,6", 2023-10-01T09:34:39.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.075169+0000", 2023-10-01T09:34:39.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8976999999999999e-05 2023-10-01T09:34:39.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.106 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.106 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:39.106 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.078483+0000", 2023-10-01T09:34:39.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0033149120000000002 2023-10-01T09:34:39.107 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.107 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079044+0000", 2023-10-01T09:34:39.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00056057299999999997 2023-10-01T09:34:39.107 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.107 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079081+0000", 2023-10-01T09:34:39.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7175000000000002e-05 2023-10-01T09:34:39.107 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.108 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.108 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.108 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079108+0000", 2023-10-01T09:34:39.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6740000000000001e-05 2023-10-01T09:34:39.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.108 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.108 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.108 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.085270+0000", 2023-10-01T09:34:39.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0061619159999999999 2023-10-01T09:34:39.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.108 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.108 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.085329+0000", 2023-10-01T09:34:39.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9414999999999999e-05 2023-10-01T09:34:39.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086037+0000", 2023-10-01T09:34:39.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00070771199999999999 2023-10-01T09:34:39.109 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.109 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.109 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.110 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129124 7.1 7:89d65aee:::benchmark_data_smithi138_367058_object31329:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:34:39.110 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:49.051416+0000", 2023-10-01T09:34:39.110 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 469.84955199000001, 2023-10-01T09:34:39.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0353429519999999, 2023-10-01T09:34:39.110 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.110 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.110 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:39.110 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:34:39.110 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:34:39.110 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129124 2023-10-01T09:34:39.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.110 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051416+0000", 2023-10-01T09:34:39.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051416+0000", 2023-10-01T09:34:39.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051416+0000", 2023-10-01T09:34:39.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:39.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051418+0000", 2023-10-01T09:34:39.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1799999999999999e-06 2023-10-01T09:34:39.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051423+0000", 2023-10-01T09:34:39.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9509999999999997e-06 2023-10-01T09:34:39.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051427+0000", 2023-10-01T09:34:39.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9609999999999999e-06 2023-10-01T09:34:39.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051553+0000", 2023-10-01T09:34:39.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000125981 2023-10-01T09:34:39.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:34:39.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.051559+0000", 2023-10-01T09:34:39.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2040000000000002e-06 2023-10-01T09:34:39.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079137+0000", 2023-10-01T09:34:39.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0275780130000001 2023-10-01T09:34:39.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.115 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.115 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079280+0000", 2023-10-01T09:34:39.115 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014338799999999999 2023-10-01T09:34:39.115 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.115 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.115 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,1,4,6", 2023-10-01T09:34:39.115 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.079462+0000", 2023-10-01T09:34:39.115 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018144600000000001 2023-10-01T09:34:39.115 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.115 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.115 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.116 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086085+0000", 2023-10-01T09:34:39.116 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066228499999999996 2023-10-01T09:34:39.116 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.116 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.116 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.116 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086145+0000", 2023-10-01T09:34:39.116 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0649999999999997e-05 2023-10-01T09:34:39.116 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.116 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.116 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:39.116 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086161+0000", 2023-10-01T09:34:39.116 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5865000000000001e-05 2023-10-01T09:34:39.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.117 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.117 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086238+0000", 2023-10-01T09:34:39.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6648000000000002e-05 2023-10-01T09:34:39.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.117 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.117 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086273+0000", 2023-10-01T09:34:39.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5553999999999999e-05 2023-10-01T09:34:39.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.118 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.118 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086324+0000", 2023-10-01T09:34:39.118 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0556000000000002e-05 2023-10-01T09:34:39.118 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.118 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.118 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.118 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.086759+0000", 2023-10-01T09:34:39.118 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00043499599999999999 2023-10-01T09:34:39.118 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.118 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.118 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.119 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.35421.0:129131 7.c 7:3f7b5b4e:::benchmark_data_smithi138_367058_object31336:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-10-01T09:34:39.119 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:26:49.062625+0000", 2023-10-01T09:34:39.119 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 469.83834300500001, 2023-10-01T09:34:39.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0376771250000001, 2023-10-01T09:34:39.119 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.119 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.119 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:39.119 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.35421", 2023-10-01T09:34:39.119 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/140519564", 2023-10-01T09:34:39.119 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 129131 2023-10-01T09:34:39.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.120 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062625+0000", 2023-10-01T09:34:39.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062625+0000", 2023-10-01T09:34:39.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062624+0000", 2023-10-01T09:34:39.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:34:39.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062630+0000", 2023-10-01T09:34:39.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.823e-06 2023-10-01T09:34:39.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062640+0000", 2023-10-01T09:34:39.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0767999999999999e-05 2023-10-01T09:34:39.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062648+0000", 2023-10-01T09:34:39.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1199999999999996e-06 2023-10-01T09:34:39.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062697+0000", 2023-10-01T09:34:39.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.901e-05 2023-10-01T09:34:39.123 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.123 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:34:39.123 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:49.062706+0000", 2023-10-01T09:34:39.123 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8449999999999998e-06 2023-10-01T09:34:39.123 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.123 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.123 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.082149+0000", 2023-10-01T09:34:39.124 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0194428250000001 2023-10-01T09:34:39.124 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.124 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.125 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.125 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.082294+0000", 2023-10-01T09:34:39.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014442799999999999 2023-10-01T09:34:39.125 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.125 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.125 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6,7", 2023-10-01T09:34:39.126 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.082448+0000", 2023-10-01T09:34:39.126 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015422400000000001 2023-10-01T09:34:39.126 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.126 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:39.126 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099008+0000", 2023-10-01T09:34:39.126 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016560099000000002 2023-10-01T09:34:39.126 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.126 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.126 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099072+0000", 2023-10-01T09:34:39.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3619000000000002e-05 2023-10-01T09:34:39.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.127 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099193+0000", 2023-10-01T09:34:39.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121525 2023-10-01T09:34:39.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.127 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099886+0000", 2023-10-01T09:34:39.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00069325300000000001 2023-10-01T09:34:39.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.099947+0000", 2023-10-01T09:34:39.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0306000000000002e-05 2023-10-01T09:34:39.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:26:50.100302+0000", 2023-10-01T09:34:39.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00035552000000000001 2023-10-01T09:34:39.128 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.128 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.128 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.129 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.129 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:183858 8.8 8:11f91ba9:::benchmark_data_smithi138_431617_object82447:head [delete] snapc 0=[] ondisk+write+known_if_redirected e381)", 2023-10-01T09:34:39.129 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:13.384316+0000", 2023-10-01T09:34:39.129 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 85.516651640999996, 2023-10-01T09:34:39.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.153157033, 2023-10-01T09:34:39.129 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.129 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.129 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:39.129 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:34:39.129 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:39.129 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 183858 2023-10-01T09:34:39.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.130 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.130 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.130 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.384316+0000", 2023-10-01T09:34:39.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.130 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.130 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.384316+0000", 2023-10-01T09:34:39.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.131 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.131 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.131 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.384315+0000", 2023-10-01T09:34:39.131 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:34:39.131 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.131 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.131 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.131 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.384323+0000", 2023-10-01T09:34:39.131 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6140000000000003e-06 2023-10-01T09:34:39.131 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.131 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.131 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.384339+0000", 2023-10-01T09:34:39.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6490999999999998e-05 2023-10-01T09:34:39.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.384347+0000", 2023-10-01T09:34:39.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2179999999999999e-06 2023-10-01T09:34:39.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.384398+0000", 2023-10-01T09:34:39.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0547000000000002e-05 2023-10-01T09:34:39.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:34:39.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.384412+0000", 2023-10-01T09:34:39.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3760999999999999e-05 2023-10-01T09:34:39.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.493767+0000", 2023-10-01T09:34:39.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.109354982 2023-10-01T09:34:39.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.493888+0000", 2023-10-01T09:34:39.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000120974 2023-10-01T09:34:39.134 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.134 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.134 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:34:39.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.494018+0000", 2023-10-01T09:34:39.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013063699999999999 2023-10-01T09:34:39.134 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.134 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.134 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.519413+0000", 2023-10-01T09:34:39.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.025394710000000001 2023-10-01T09:34:39.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:39.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.524355+0000", 2023-10-01T09:34:39.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004941829 2023-10-01T09:34:39.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.537164+0000", 2023-10-01T09:34:39.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012809210999999999 2023-10-01T09:34:39.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.537231+0000", 2023-10-01T09:34:39.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7007000000000004e-05 2023-10-01T09:34:39.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.537473+0000", 2023-10-01T09:34:39.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000242413 2023-10-01T09:34:39.136 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.136 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.136 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.137 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:183863 8.8 8:144f1bf9:::benchmark_data_smithi138_431617_object82452:head [delete] snapc 0=[] ondisk+write+known_if_redirected e381)", 2023-10-01T09:34:39.137 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:13.400054+0000", 2023-10-01T09:34:39.137 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 85.500913711999999, 2023-10-01T09:34:39.137 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.15037605600000001, 2023-10-01T09:34:39.137 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.137 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.137 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:39.137 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:34:39.137 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:39.137 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 183863 2023-10-01T09:34:39.137 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.137 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.138 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.138 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.400054+0000", 2023-10-01T09:34:39.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.138 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.138 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.400054+0000", 2023-10-01T09:34:39.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.138 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.139 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.400053+0000", 2023-10-01T09:34:39.139 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:34:39.139 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.139 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.139 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.139 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.400060+0000", 2023-10-01T09:34:39.139 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0330000000000003e-06 2023-10-01T09:34:39.139 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.139 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.139 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.139 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.400073+0000", 2023-10-01T09:34:39.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2869e-05 2023-10-01T09:34:39.140 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.140 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.140 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.400079+0000", 2023-10-01T09:34:39.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5429999999999999e-06 2023-10-01T09:34:39.140 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.140 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.140 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.400119+0000", 2023-10-01T09:34:39.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9579000000000002e-05 2023-10-01T09:34:39.140 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.141 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:34:39.141 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.400126+0000", 2023-10-01T09:34:39.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7179999999999996e-06 2023-10-01T09:34:39.141 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.141 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.141 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.497805+0000", 2023-10-01T09:34:39.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.097678531999999998 2023-10-01T09:34:39.141 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.141 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.142 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.497929+0000", 2023-10-01T09:34:39.142 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000124549 2023-10-01T09:34:39.142 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.142 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.142 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:34:39.142 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.498061+0000", 2023-10-01T09:34:39.142 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131872 2023-10-01T09:34:39.142 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.142 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.142 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.142 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.519455+0000", 2023-10-01T09:34:39.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.021393645999999999 2023-10-01T09:34:39.143 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.143 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.143 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:39.143 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.524406+0000", 2023-10-01T09:34:39.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049506200000000002 2023-10-01T09:34:39.143 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.143 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.143 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.143 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.550167+0000", 2023-10-01T09:34:39.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.025761661000000002 2023-10-01T09:34:39.144 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.144 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.144 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.144 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.550220+0000", 2023-10-01T09:34:39.144 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2933999999999998e-05 2023-10-01T09:34:39.144 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.144 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.144 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.144 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.550430+0000", 2023-10-01T09:34:39.144 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021017599999999999 2023-10-01T09:34:39.144 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.144 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.145 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.145 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.145 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.145 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:187123 8.8 8:1531d290:::benchmark_data_smithi138_431617_object85712:head [delete] snapc 0=[] ondisk+write+known_if_redirected e384)", 2023-10-01T09:34:39.145 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:18.925346+0000", 2023-10-01T09:34:39.145 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 79.975621911999994, 2023-10-01T09:34:39.145 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18660890799999999, 2023-10-01T09:34:39.145 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.145 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.145 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:39.145 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:34:39.145 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:39.146 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 187123 2023-10-01T09:34:39.146 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.146 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.146 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.146 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.146 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925346+0000", 2023-10-01T09:34:39.146 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.146 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.146 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.146 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.146 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925346+0000", 2023-10-01T09:34:39.146 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.147 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.147 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.147 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.147 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925344+0000", 2023-10-01T09:34:39.147 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:34:39.147 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.147 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.147 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.147 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925353+0000", 2023-10-01T09:34:39.147 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3270000000000001e-06 2023-10-01T09:34:39.147 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.147 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.148 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.148 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925373+0000", 2023-10-01T09:34:39.148 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9842000000000001e-05 2023-10-01T09:34:39.148 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.148 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.148 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.148 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925381+0000", 2023-10-01T09:34:39.148 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5310000000000001e-06 2023-10-01T09:34:39.148 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.148 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.148 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.148 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925432+0000", 2023-10-01T09:34:39.149 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1103999999999997e-05 2023-10-01T09:34:39.149 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.149 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.149 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.149 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925571+0000", 2023-10-01T09:34:39.149 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013912700000000001 2023-10-01T09:34:39.149 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.149 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.149 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:34:39.149 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925723+0000", 2023-10-01T09:34:39.149 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000151879 2023-10-01T09:34:39.149 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.150 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.150 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.150 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.931279+0000", 2023-10-01T09:34:39.150 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0055557790000000003 2023-10-01T09:34:39.150 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.150 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.150 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:39.150 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.934123+0000", 2023-10-01T09:34:39.150 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028444490000000002 2023-10-01T09:34:39.150 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.150 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.150 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.151 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111698+0000", 2023-10-01T09:34:39.151 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.177574809 2023-10-01T09:34:39.151 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.151 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.151 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.151 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111751+0000", 2023-10-01T09:34:39.151 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2756000000000001e-05 2023-10-01T09:34:39.151 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.151 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.151 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.151 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111955+0000", 2023-10-01T09:34:39.151 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020406600000000001 2023-10-01T09:34:39.152 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.152 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.152 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.152 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.152 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.152 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:187129 8.8 8:155c408a:::benchmark_data_smithi138_431617_object85718:head [delete] snapc 0=[] ondisk+write+known_if_redirected e384)", 2023-10-01T09:34:39.152 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:18.936343+0000", 2023-10-01T09:34:39.152 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 79.964624951999994, 2023-10-01T09:34:39.152 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17579768900000001, 2023-10-01T09:34:39.152 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.152 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.152 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:39.152 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:34:39.153 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:39.153 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 187129 2023-10-01T09:34:39.153 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.153 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.153 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.153 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.153 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936343+0000", 2023-10-01T09:34:39.153 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.153 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.153 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.153 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.153 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936343+0000", 2023-10-01T09:34:39.154 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.154 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.154 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.154 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.154 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936341+0000", 2023-10-01T09:34:39.154 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:34:39.154 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.154 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.154 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.154 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936350+0000", 2023-10-01T09:34:39.154 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6170000000000007e-06 2023-10-01T09:34:39.154 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.155 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.155 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.155 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936369+0000", 2023-10-01T09:34:39.155 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9264999999999999e-05 2023-10-01T09:34:39.155 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.155 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.155 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.155 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936379+0000", 2023-10-01T09:34:39.155 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7550000000000006e-06 2023-10-01T09:34:39.155 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.155 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.156 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.156 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936435+0000", 2023-10-01T09:34:39.156 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5951e-05 2023-10-01T09:34:39.156 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.156 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.156 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.156 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936602+0000", 2023-10-01T09:34:39.156 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000167446 2023-10-01T09:34:39.156 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.156 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.156 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:34:39.156 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936694+0000", 2023-10-01T09:34:39.157 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2208999999999998e-05 2023-10-01T09:34:39.157 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.157 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.157 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:39.157 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.940157+0000", 2023-10-01T09:34:39.157 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034625760000000002 2023-10-01T09:34:39.157 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.157 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.157 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.157 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.941169+0000", 2023-10-01T09:34:39.157 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001012352 2023-10-01T09:34:39.157 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.158 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.158 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.158 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111993+0000", 2023-10-01T09:34:39.158 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17082314400000001 2023-10-01T09:34:39.158 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.158 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.158 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.158 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.112030+0000", 2023-10-01T09:34:39.158 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7453000000000002e-05 2023-10-01T09:34:39.158 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.158 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.158 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.159 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.112141+0000", 2023-10-01T09:34:39.159 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110803 2023-10-01T09:34:39.159 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.159 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.159 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.159 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.159 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.159 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:187964 8.8 8:17b0d3d9:::benchmark_data_smithi138_431617_object86553:head [delete] snapc 0=[] ondisk+write+known_if_redirected e384)", 2023-10-01T09:34:39.159 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:20.443813+0000", 2023-10-01T09:34:39.159 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 78.457155528000001, 2023-10-01T09:34:39.159 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.149733054, 2023-10-01T09:34:39.159 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.160 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.160 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:39.160 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:34:39.160 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:39.160 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 187964 2023-10-01T09:34:39.160 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.160 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.160 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.160 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.160 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.443813+0000", 2023-10-01T09:34:39.160 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.160 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.161 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.161 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.161 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.443813+0000", 2023-10-01T09:34:39.161 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.161 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.161 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.161 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.161 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.443811+0000", 2023-10-01T09:34:39.161 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:34:39.161 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.161 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.161 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.162 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.443819+0000", 2023-10-01T09:34:39.162 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1790000000000008e-06 2023-10-01T09:34:39.162 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.162 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.162 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.162 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.443833+0000", 2023-10-01T09:34:39.162 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4232e-05 2023-10-01T09:34:39.162 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.162 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.162 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.162 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.443841+0000", 2023-10-01T09:34:39.163 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6510000000000001e-06 2023-10-01T09:34:39.163 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.163 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.163 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.163 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.443895+0000", 2023-10-01T09:34:39.163 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4137e-05 2023-10-01T09:34:39.163 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.163 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.163 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.163 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.444039+0000", 2023-10-01T09:34:39.163 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014346300000000001 2023-10-01T09:34:39.163 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.164 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.164 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:34:39.164 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.444164+0000", 2023-10-01T09:34:39.164 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012482500000000001 2023-10-01T09:34:39.164 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.164 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.164 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:39.164 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.447720+0000", 2023-10-01T09:34:39.164 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035569489999999998 2023-10-01T09:34:39.164 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.164 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.164 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.165 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.449154+0000", 2023-10-01T09:34:39.165 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0014339940000000001 2023-10-01T09:34:39.165 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.165 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.165 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.165 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.593268+0000", 2023-10-01T09:34:39.165 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.14411342999999999 2023-10-01T09:34:39.165 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.165 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.165 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.165 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.593329+0000", 2023-10-01T09:34:39.165 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0649000000000002e-05 2023-10-01T09:34:39.165 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.166 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.166 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.166 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.593546+0000", 2023-10-01T09:34:39.166 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000217018 2023-10-01T09:34:39.166 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.166 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.166 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.166 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.166 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.166 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:187980 8.8 8:1fc8acc0:::benchmark_data_smithi138_431617_object86569:head [delete] snapc 0=[] ondisk+write+known_if_redirected e384)", 2023-10-01T09:34:39.166 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:20.475551+0000", 2023-10-01T09:34:39.167 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 78.425417249000006, 2023-10-01T09:34:39.167 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11820293900000001, 2023-10-01T09:34:39.167 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.167 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.167 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:34:39.167 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:34:39.167 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:39.167 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 187980 2023-10-01T09:34:39.167 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.167 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.167 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.167 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.167 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.475551+0000", 2023-10-01T09:34:39.168 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.168 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.168 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.168 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.168 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.475551+0000", 2023-10-01T09:34:39.168 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.168 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.168 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.168 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.168 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.475550+0000", 2023-10-01T09:34:39.168 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:34:39.168 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.169 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.169 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.169 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.475555+0000", 2023-10-01T09:34:39.169 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7710000000000003e-06 2023-10-01T09:34:39.169 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.169 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.169 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.169 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.475570+0000", 2023-10-01T09:34:39.169 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4895000000000001e-05 2023-10-01T09:34:39.169 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.169 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.169 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.170 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.475578+0000", 2023-10-01T09:34:39.170 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6040000000000004e-06 2023-10-01T09:34:39.170 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.170 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.170 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.170 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.475623+0000", 2023-10-01T09:34:39.170 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5537000000000001e-05 2023-10-01T09:34:39.170 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.170 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.170 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.170 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.475770+0000", 2023-10-01T09:34:39.170 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000146329 2023-10-01T09:34:39.171 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.171 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.171 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:34:39.171 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.475932+0000", 2023-10-01T09:34:39.171 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016224699999999999 2023-10-01T09:34:39.171 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.171 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.171 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.171 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.480488+0000", 2023-10-01T09:34:39.171 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045563169999999998 2023-10-01T09:34:39.171 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.171 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.172 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:34:39.172 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.480539+0000", 2023-10-01T09:34:39.172 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1091999999999998e-05 2023-10-01T09:34:39.172 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.172 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.172 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:39.172 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.593592+0000", 2023-10-01T09:34:39.172 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11305272600000001 2023-10-01T09:34:39.172 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.172 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.172 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.172 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.593639+0000", 2023-10-01T09:34:39.173 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6862e-05 2023-10-01T09:34:39.173 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.173 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.173 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.173 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:20.593754+0000", 2023-10-01T09:34:39.173 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000114809 2023-10-01T09:34:39.173 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.173 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.173 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.173 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.173 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.173 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:11070 9.3 e405/400 9:c4f40228:::benchmark_data_smithi138_497829_object11069:head v 405'688, mlcod=405'686)", 2023-10-01T09:34:39.174 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:34:20.254346+0000", 2023-10-01T09:34:39.174 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.646622391000001, 2023-10-01T09:34:39.174 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.120122777, 2023-10-01T09:34:39.174 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.174 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.174 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.174 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.174 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.174 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.254346+0000", 2023-10-01T09:34:39.174 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.174 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.174 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.175 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.175 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.254346+0000", 2023-10-01T09:34:39.175 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.175 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.175 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.175 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.254345+0000", 2023-10-01T09:34:39.175 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:34:39.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.175 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.175 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.175 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255057+0000", 2023-10-01T09:34:39.176 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00071242300000000001 2023-10-01T09:34:39.176 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.176 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.176 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.176 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255072+0000", 2023-10-01T09:34:39.176 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4861999999999999e-05 2023-10-01T09:34:39.176 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.176 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.176 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.176 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255080+0000", 2023-10-01T09:34:39.176 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4039999999999995e-06 2023-10-01T09:34:39.176 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.177 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.177 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.177 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255129+0000", 2023-10-01T09:34:39.177 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9064999999999999e-05 2023-10-01T09:34:39.177 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.177 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.177 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.177 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255150+0000", 2023-10-01T09:34:39.177 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1056999999999998e-05 2023-10-01T09:34:39.177 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.177 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.177 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.177 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.374430+0000", 2023-10-01T09:34:39.178 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.119279603 2023-10-01T09:34:39.178 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.178 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.178 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.178 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.374468+0000", 2023-10-01T09:34:39.178 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8441e-05 2023-10-01T09:34:39.178 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.178 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.178 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.178 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.178 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.179 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:11076 9.c e405/400 9:35958ca6:::benchmark_data_smithi138_497829_object11075:head v 405'715, mlcod=405'713)", 2023-10-01T09:34:39.179 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:34:20.255098+0000", 2023-10-01T09:34:39.179 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.645870025000001, 2023-10-01T09:34:39.179 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.118873326, 2023-10-01T09:34:39.179 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.179 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.179 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.179 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.179 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.179 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255098+0000", 2023-10-01T09:34:39.179 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.179 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.180 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.180 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.180 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255098+0000", 2023-10-01T09:34:39.180 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.180 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.180 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.180 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.180 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255098+0000", 2023-10-01T09:34:39.180 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:39.180 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.180 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.180 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.181 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255134+0000", 2023-10-01T09:34:39.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6822000000000001e-05 2023-10-01T09:34:39.181 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.181 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.181 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.181 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255156+0000", 2023-10-01T09:34:39.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1797000000000002e-05 2023-10-01T09:34:39.181 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.181 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.181 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.181 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255163+0000", 2023-10-01T09:34:39.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3609999999999997e-06 2023-10-01T09:34:39.182 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.182 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.182 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255250+0000", 2023-10-01T09:34:39.182 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7748000000000001e-05 2023-10-01T09:34:39.182 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.182 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.182 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255264+0000", 2023-10-01T09:34:39.182 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3383999999999999e-05 2023-10-01T09:34:39.182 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.183 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.183 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.373929+0000", 2023-10-01T09:34:39.183 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.118665146 2023-10-01T09:34:39.183 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.183 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.183 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.183 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.373971+0000", 2023-10-01T09:34:39.183 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2558000000000001e-05 2023-10-01T09:34:39.183 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.183 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.183 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.183 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.184 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:11077 9.2 e405/400 9:431614af:::benchmark_data_smithi138_497829_object11076:head v 405'733, mlcod=405'731)", 2023-10-01T09:34:39.184 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:34:20.255830+0000", 2023-10-01T09:34:39.184 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.645138078999999, 2023-10-01T09:34:39.184 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.117895128, 2023-10-01T09:34:39.184 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.184 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.184 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.184 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.184 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255830+0000", 2023-10-01T09:34:39.184 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.185 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.185 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.185 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.185 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255830+0000", 2023-10-01T09:34:39.185 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.185 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.185 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.185 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.185 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255829+0000", 2023-10-01T09:34:39.185 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:34:39.185 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.186 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255853+0000", 2023-10-01T09:34:39.186 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3405999999999999e-05 2023-10-01T09:34:39.186 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.186 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255864+0000", 2023-10-01T09:34:39.186 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1955e-05 2023-10-01T09:34:39.186 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255871+0000", 2023-10-01T09:34:39.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0190000000000001e-06 2023-10-01T09:34:39.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255933+0000", 2023-10-01T09:34:39.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1227000000000002e-05 2023-10-01T09:34:39.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.255952+0000", 2023-10-01T09:34:39.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9338999999999999e-05 2023-10-01T09:34:39.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.373677+0000", 2023-10-01T09:34:39.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11772495600000001 2023-10-01T09:34:39.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.373725+0000", 2023-10-01T09:34:39.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8093000000000002e-05 2023-10-01T09:34:39.188 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.188 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.188 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.189 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:11086 9.3 e405/400 9:c83681f9:::benchmark_data_smithi138_497829_object11085:head v 405'689, mlcod=405'686)", 2023-10-01T09:34:39.189 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:34:20.256196+0000", 2023-10-01T09:34:39.189 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.644771773999999, 2023-10-01T09:34:39.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.118294345, 2023-10-01T09:34:39.189 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.189 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.189 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.189 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.189 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.256196+0000", 2023-10-01T09:34:39.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.190 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.190 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.190 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.256196+0000", 2023-10-01T09:34:39.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.190 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.190 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.190 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.256195+0000", 2023-10-01T09:34:39.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:34:39.190 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.256263+0000", 2023-10-01T09:34:39.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7152999999999999e-05 2023-10-01T09:34:39.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.256278+0000", 2023-10-01T09:34:39.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5255e-05 2023-10-01T09:34:39.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.256285+0000", 2023-10-01T09:34:39.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4359999999999996e-06 2023-10-01T09:34:39.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.256327+0000", 2023-10-01T09:34:39.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2014999999999997e-05 2023-10-01T09:34:39.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.256345+0000", 2023-10-01T09:34:39.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8159e-05 2023-10-01T09:34:39.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.374472+0000", 2023-10-01T09:34:39.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11812700199999999 2023-10-01T09:34:39.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.374491+0000", 2023-10-01T09:34:39.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8151999999999998e-05 2023-10-01T09:34:39.193 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.194 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.194 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.194 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.194 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.194 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:11083 9.c e405/400 9:3db4b374:::benchmark_data_smithi138_497829_object11082:head v 405'717, mlcod=405'713)", 2023-10-01T09:34:39.194 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:34:20.256662+0000", 2023-10-01T09:34:39.194 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.644305712000001, 2023-10-01T09:34:39.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11774111900000001, 2023-10-01T09:34:39.194 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.194 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.194 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.194 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.256662+0000", 2023-10-01T09:34:39.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.256662+0000", 2023-10-01T09:34:39.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.256662+0000", 2023-10-01T09:34:39.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:39.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.256757+0000", 2023-10-01T09:34:39.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5132999999999999e-05 2023-10-01T09:34:39.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.372832+0000", 2023-10-01T09:34:39.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.116074776 2023-10-01T09:34:39.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.372850+0000", 2023-10-01T09:34:39.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8749999999999998e-05 2023-10-01T09:34:39.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.373484+0000", 2023-10-01T09:34:39.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00063379799999999998 2023-10-01T09:34:39.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.198 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.373499+0000", 2023-10-01T09:34:39.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4695000000000001e-05 2023-10-01T09:34:39.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.198 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.374375+0000", 2023-10-01T09:34:39.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00087569799999999995 2023-10-01T09:34:39.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.198 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:20.374403+0000", 2023-10-01T09:34:39.199 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8856999999999998e-05 2023-10-01T09:34:39.199 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.199 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.199 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.199 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.199 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:14779 9.1 e405/400 9:8d99d6e9:::benchmark_data_smithi138_497829_object14778:head v 405'892, mlcod=405'890)", 2023-10-01T09:34:39.199 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:34:22.068325+0000", 2023-10-01T09:34:39.199 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 16.83264303, 2023-10-01T09:34:39.199 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.118346144, 2023-10-01T09:34:39.199 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:34:39.200 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:39.200 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:34:39.200 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.200 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:34:39.200 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:22.068325+0000", 2023-10-01T09:34:39.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.200 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.200 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.200 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:34:39.200 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:22.068325+0000", 2023-10-01T09:34:39.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:34:39.200 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.201 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.201 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:34:39.201 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:22.068324+0000", 2023-10-01T09:34:39.201 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:34:39.201 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.201 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.201 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:34:39.201 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:22.068372+0000", 2023-10-01T09:34:39.201 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7803999999999998e-05 2023-10-01T09:34:39.201 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.201 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.201 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:34:39.202 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:22.180014+0000", 2023-10-01T09:34:39.202 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.111641513 2023-10-01T09:34:39.202 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.202 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.202 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:34:39.202 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:22.180028+0000", 2023-10-01T09:34:39.202 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4041e-05 2023-10-01T09:34:39.202 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.202 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.202 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:34:39.202 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:22.180092+0000", 2023-10-01T09:34:39.202 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4010000000000005e-05 2023-10-01T09:34:39.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.203 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.203 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:34:39.203 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:22.180112+0000", 2023-10-01T09:34:39.203 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0486e-05 2023-10-01T09:34:39.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.203 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.203 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:34:39.203 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:22.186629+0000", 2023-10-01T09:34:39.203 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0065171960000000003 2023-10-01T09:34:39.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:34:39.203 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:34:39.204 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:34:39.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:22.186671+0000", 2023-10-01T09:34:39.204 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1884000000000001e-05 2023-10-01T09:34:39.204 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.204 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.204 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.204 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:34:39.204 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:34:39.204 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:34:39.297 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:39.448 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:34:39.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:39.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:34:39.519 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.4 to 1 (10000) 2023-10-01T09:34:39.556 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:39.635 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:39.750 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 23 16 44576 44560 121.045 94.5625 0.00955903 0.00825612 2023-10-01T09:34:39.779 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:39.785 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:39 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2405180180' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:34:39.786 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:39 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:34:39.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:39.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:39.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:39 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2405180180' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:34:39.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:39 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:34:39.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:39 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2405180180' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:34:39.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:39 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:34:39.887 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:39.914 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:40.115 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:34:40.750 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 24 16 45629 45613 118.744 65.8125 0.00549953 0.00840456 2023-10-01T09:34:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:40 smithi190 ceph-mon[160435]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:40 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:34:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:40 smithi190 ceph-mon[160435]: osdmap e408: 8 total, 8 up, 8 in 2023-10-01T09:34:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:40 smithi190 ceph-mon[160435]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:40 smithi190 ceph-mon[160435]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:40 smithi190 ceph-mon[160435]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:40 smithi190 ceph-mon[160435]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:40 smithi190 ceph-mon[160435]: pgmap v1128: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-01T09:34:40.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:34:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[192513]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:34:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[192513]: osdmap e408: 8 total, 8 up, 8 in 2023-10-01T09:34:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[192513]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[192513]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[192513]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[192513]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[192513]: pgmap v1128: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-01T09:34:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:34:40.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[185886]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:34:40.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[185886]: osdmap e408: 8 total, 8 up, 8 in 2023-10-01T09:34:40.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[185886]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[185886]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[185886]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[185886]: failed to encode map e408 with expected crc 2023-10-01T09:34:40.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[185886]: pgmap v1128: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-01T09:34:40.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:34:41.024 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:34:41.062 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:41.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:41.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:41.169 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:41.223 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:41.319 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:41.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:41.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:41.337 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-01T09:34:41.337 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:34:41.431 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:41.478 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:41.521 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:41.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:41.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:41.625 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:41.664 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:41.751 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:41.751 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 25 16 46828 46812 116.992 74.9375 0.00915601 0.00854175 2023-10-01T09:34:41.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:41.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:41.861 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:41.918 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:42.063 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:42.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:42.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:42.177 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:42 smithi138 ceph-mon[185886]: pgmap v1129: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-01T09:34:42.177 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:42.178 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:42.212 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:42.279 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:42.286 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:42.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:42.395 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:42.455 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:42 smithi138 ceph-mon[192513]: pgmap v1129: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-01T09:34:42.463 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:42 smithi190 ceph-mon[160435]: pgmap v1129: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-10-01T09:34:42.626 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:42.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:42.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:42.733 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:42.755 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 26 16 48249 48233 115.891 88.8125 0.00995731 0.0086234 2023-10-01T09:34:42.797 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:43.508 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:34:43.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:43 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1229446848' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:34:43.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:43 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:34:43.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:43 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1229446848' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:34:43.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:43 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:34:43.755 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 27 16 49463 49447 114.409 75.875 0.00984944 0.00873539 2023-10-01T09:34:43.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:43 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1229446848' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:34:43.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:43 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:34:43.916 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:43.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:43.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:44.022 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:44.059 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:44.085 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:34:44.150 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:44.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:44.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:44.258 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:44.309 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:44.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:44 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:34:44.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:44 smithi138 ceph-mon[192513]: failed to encode map e409 with expected crc 2023-10-01T09:34:44.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:44 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:34:44.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:44 smithi138 ceph-mon[192513]: osdmap e409: 8 total, 8 up, 8 in 2023-10-01T09:34:44.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:44 smithi138 ceph-mon[192513]: failed to encode map e409 with expected crc 2023-10-01T09:34:44.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:44 smithi138 ceph-mon[192513]: pgmap v1131: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-01T09:34:44.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:44 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:34:44.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:44 smithi138 ceph-mon[185886]: failed to encode map e409 with expected crc 2023-10-01T09:34:44.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:44 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:34:44.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:44 smithi138 ceph-mon[185886]: osdmap e409: 8 total, 8 up, 8 in 2023-10-01T09:34:44.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:44 smithi138 ceph-mon[185886]: failed to encode map e409 with expected crc 2023-10-01T09:34:44.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:44 smithi138 ceph-mon[185886]: pgmap v1131: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-01T09:34:44.755 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 28 16 50809 50793 113.327 84.125 0.00379616 0.00881014 2023-10-01T09:34:44.790 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:44.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:44.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:44.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:44 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:34:44.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:44 smithi190 ceph-mon[160435]: failed to encode map e409 with expected crc 2023-10-01T09:34:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:44 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:34:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:44 smithi190 ceph-mon[160435]: osdmap e409: 8 total, 8 up, 8 in 2023-10-01T09:34:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:44 smithi190 ceph-mon[160435]: failed to encode map e409 with expected crc 2023-10-01T09:34:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:44 smithi190 ceph-mon[160435]: pgmap v1131: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-01T09:34:44.899 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:44.925 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:45.175 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:45.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:45.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:45.279 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:45.304 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:34:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:34:44] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.33.4" 2023-10-01T09:34:45.307 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:45.425 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:45.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:45.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:45.532 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:45.584 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:45.755 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 29 16 51941 51925 111.859 70.75 0.00852685 0.00893362 2023-10-01T09:34:45.966 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:45.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:45.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:46.035 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:34:46.069 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:46.076 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:34:46.116 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:46.116 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:34:46.155 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:34:46.190 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:46.192 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:34:46.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:46.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:46.232 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:34:46.286 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:34:46.287 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:34:46.293 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:46.300 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:34:46.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:46 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3211665467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:46.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:46 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3211665467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:46.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:46 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3211665467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:46.324 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:34:46.343 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:34:46.343 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:34:46.343 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:34:46.343 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:34:46.343 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:34:46.344 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:34:46.344 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:34:46.344 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:34:46.364 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:46.563 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:46.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:46.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:46.679 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:46.730 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:46.755 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 30 15 53309 53294 110.983 85.5625 0.00579366 0.00900595 2023-10-01T09:34:46.840 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:46.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:46.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:46.946 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:46.985 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:47.028 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:47.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:47.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:47.131 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:47.168 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:47.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:47 smithi190 ceph-mon[160435]: pgmap v1132: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-01T09:34:47.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:47 smithi138 ceph-mon[192513]: pgmap v1132: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-01T09:34:47.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:47 smithi138 ceph-mon[185886]: pgmap v1132: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-01T09:34:47.425 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:47.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:47.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:47.450 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:34:47.529 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:47.563 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:47.661 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:47.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:47.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:47.755 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 31 16 54109 54093 109.014 49.9375 0.0040827 0.00914287 2023-10-01T09:34:47.768 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:47.829 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:47.915 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:47.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:47.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:48.020 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:48.022 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1132,"stamp":"2023-10-01T09:34:46.083525+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3216119731,"num_objects":49029,"num_object_clones":0,"num_object_copies":147087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":49029,"num_whiteouts":0,"num_read":6259,"num_read_kb":12659,"num_write":50107,"num_write_kb":3151205,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":350,"num_bytes_recovered":3575563,"num_keys_recovered":70,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":50171,"ondisk_log_size":50171,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13781720,"kb_used_data":9304856,"kb_used_omap":50,"kb_used_meta":4476749,"kb_avail":736179496,"statfs":{"total":767960285184,"available":753847803904,"internally_reserved":0,"allocated":9528172544,"data_stored":9517682599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51738,"internal_metadata":4584191462},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,6,3,7,2,0,0,0,5],"upper_bound":512},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":738918400,"num_objects":11275,"num_object_clones":0,"num_object_copies":33825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11275,"num_write_kb":721600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003212"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":302,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.506102+0000","last_change":"2023-10-01T09:32:27.452345+0000","last_active":"2023-10-01T09:34:39.506102+0000","last_peered":"2023-10-01T09:34:39.506102+0000","last_clean":"2023-10-01T09:34:39.506102+0000","last_became_active":"2023-10-01T09:32:27.452033+0000","last_became_peered":"2023-10-01T09:32:27.452033+0000","last_unstale":"2023-10-01T09:34:39.506102+0000","last_undegraded":"2023-10-01T09:34:39.506102+0000","last_fullsized":"2023-10-01T09:34:39.506102+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:44:10.031691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":222,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.510693+0000","last_change":"2023-10-01T09:32:27.450725+0000","last_active":"2023-10-01T09:34:33.510693+0000","last_peered":"2023-10-01T09:34:33.510693+0000","last_clean":"2023-10-01T09:34:33.510693+0000","last_became_active":"2023-10-01T09:32:27.450419+0000","last_became_peered":"2023-10-01T09:32:27.450419+0000","last_unstale":"2023-10-01T09:34:33.510693+0000","last_undegraded":"2023-10-01T09:34:33.510693+0000","last_fullsized":"2023-10-01T09:34:33.510693+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:55:11.692362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":387,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504285+0000","last_change":"2023-10-01T09:32:27.445629+0000","last_active":"2023-10-01T09:34:39.504285+0000","last_peered":"2023-10-01T09:34:39.504285+0000","last_clean":"2023-10-01T09:34:39.504285+0000","last_became_active":"2023-10-01T09:32:27.445365+0000","last_became_peered":"2023-10-01T09:32:27.445365+0000","last_unstale":"2023-10-01T09:34:39.504285+0000","last_undegraded":"2023-10-01T09:34:39.504285+0000","last_fullsized":"2023-10-01T09:34:39.504285+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:13:21.018961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":426,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513757+0000","last_change":"2023-10-01T09:29:58.372577+0000","last_active":"2023-10-01T09:34:39.513757+0000","last_peered":"2023-10-01T09:34:39.513757+0000","last_clean":"2023-10-01T09:34:39.513757+0000","last_became_active":"2023-10-01T09:29:58.372218+0000","last_became_peered":"2023-10-01T09:29:58.372218+0000","last_unstale":"2023-10-01T09:34:39.513757+0000","last_undegraded":"2023-10-01T09:34:39.513757+0000","last_fullsized":"2023-10-01T09:34:39.513757+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:04:08.376752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":299,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513225+0000","last_change":"2023-10-01T09:30:39.394083+0000","last_active":"2023-10-01T09:34:39.513225+0000","last_peered":"2023-10-01T09:34:39.513225+0000","last_clean":"2023-10-01T09:34:39.513225+0000","last_became_active":"2023-10-01T09:30:39.393797+0000","last_became_peered":"2023-10-01T09:30:39.393797+0000","last_unstale":"2023-10-01T09:34:39.513225+0000","last_undegraded":"2023-10-01T09:34:39.513225+0000","last_fullsized":"2023-10-01T09:34:39.513225+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:22.625624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":127,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.505993+0000","last_change":"2023-10-01T09:33:11.511556+0000","last_active":"2023-10-01T09:34:33.505993+0000","last_peered":"2023-10-01T09:34:33.505993+0000","last_clean":"2023-10-01T09:34:33.505993+0000","last_became_active":"2023-10-01T09:33:11.511318+0000","last_became_peered":"2023-10-01T09:33:11.511318+0000","last_unstale":"2023-10-01T09:34:33.505993+0000","last_undegraded":"2023-10-01T09:34:33.505993+0000","last_fullsized":"2023-10-01T09:34:33.505993+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":462,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513380+0000","last_change":"2023-10-01T09:33:11.508780+0000","last_active":"2023-10-01T09:34:39.513380+0000","last_peered":"2023-10-01T09:34:39.513380+0000","last_clean":"2023-10-01T09:34:39.513380+0000","last_became_active":"2023-10-01T09:33:11.508451+0000","last_became_peered":"2023-10-01T09:33:11.508451+0000","last_unstale":"2023-10-01T09:34:39.513380+0000","last_undegraded":"2023-10-01T09:34:39.513380+0000","last_fullsized":"2023-10-01T09:34:39.513380+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:55:02.323934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":410,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.507878+0000","last_change":"2023-10-01T09:30:39.395270+0000","last_active":"2023-10-01T09:34:39.507878+0000","last_peered":"2023-10-01T09:34:39.507878+0000","last_clean":"2023-10-01T09:34:39.507878+0000","last_became_active":"2023-10-01T09:30:39.394888+0000","last_became_peered":"2023-10-01T09:30:39.394888+0000","last_unstale":"2023-10-01T09:34:39.507878+0000","last_undegraded":"2023-10-01T09:34:39.507878+0000","last_fullsized":"2023-10-01T09:34:39.507878+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:35:40.846102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":94,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.512173+0000","last_change":"2023-10-01T09:33:11.667007+0000","last_active":"2023-10-01T09:34:39.512173+0000","last_peered":"2023-10-01T09:34:39.512173+0000","last_clean":"2023-10-01T09:34:39.512173+0000","last_became_active":"2023-10-01T09:33:11.666693+0000","last_became_peered":"2023-10-01T09:33:11.666693+0000","last_unstale":"2023-10-01T09:34:39.512173+0000","last_undegraded":"2023-10-01T09:34:39.512173+0000","last_fullsized":"2023-10-01T09:34:39.512173+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":425,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.508913+0000","last_change":"2023-10-01T09:30:39.398262+0000","last_active":"2023-10-01T09:34:33.508913+0000","last_peered":"2023-10-01T09:34:33.508913+0000","last_clean":"2023-10-01T09:34:33.508913+0000","last_became_active":"2023-10-01T09:30:39.397934+0000","last_became_peered":"2023-10-01T09:30:39.397934+0000","last_unstale":"2023-10-01T09:34:33.508913+0000","last_undegraded":"2023-10-01T09:34:33.508913+0000","last_fullsized":"2023-10-01T09:34:33.508913+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":207,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.509007+0000","last_change":"2023-10-01T09:32:27.492859+0000","last_active":"2023-10-01T09:34:33.509007+0000","last_peered":"2023-10-01T09:34:33.509007+0000","last_clean":"2023-10-01T09:34:33.509007+0000","last_became_active":"2023-10-01T09:32:27.454334+0000","last_became_peered":"2023-10-01T09:32:27.454334+0000","last_unstale":"2023-10-01T09:34:33.509007+0000","last_undegraded":"2023-10-01T09:34:33.509007+0000","last_fullsized":"2023-10-01T09:34:33.509007+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":438,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.505793+0000","last_change":"2023-10-01T09:28:02.342661+0000","last_active":"2023-10-01T09:34:39.505793+0000","last_peered":"2023-10-01T09:34:39.505793+0000","last_clean":"2023-10-01T09:34:39.505793+0000","last_became_active":"2023-10-01T09:28:02.342358+0000","last_became_peered":"2023-10-01T09:28:02.342358+0000","last_unstale":"2023-10-01T09:34:39.505793+0000","last_undegraded":"2023-10-01T09:34:39.505793+0000","last_fullsized":"2023-10-01T09:34:39.505793+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:29:30.624568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":435,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.509074+0000","last_change":"2023-10-01T09:32:27.498041+0000","last_active":"2023-10-01T09:34:33.509074+0000","last_peered":"2023-10-01T09:34:33.509074+0000","last_clean":"2023-10-01T09:34:33.509074+0000","last_became_active":"2023-10-01T09:32:27.454394+0000","last_became_peered":"2023-10-01T09:32:27.454394+0000","last_unstale":"2023-10-01T09:34:33.509074+0000","last_undegraded":"2023-10-01T09:34:33.509074+0000","last_fullsized":"2023-10-01T09:34:33.509074+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":121,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.506355+0000","last_change":"2023-10-01T09:32:27.454453+0000","last_active":"2023-10-01T09:34:33.506355+0000","last_peered":"2023-10-01T09:34:33.506355+0000","last_clean":"2023-10-01T09:34:33.506355+0000","last_became_active":"2023-10-01T09:32:27.454169+0000","last_became_peered":"2023-10-01T09:32:27.454169+0000","last_unstale":"2023-10-01T09:34:33.506355+0000","last_undegraded":"2023-10-01T09:34:33.506355+0000","last_fullsized":"2023-10-01T09:34:33.506355+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":481,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.508979+0000","last_change":"2023-10-01T09:32:27.592277+0000","last_active":"2023-10-01T09:34:33.508979+0000","last_peered":"2023-10-01T09:34:33.508979+0000","last_clean":"2023-10-01T09:34:33.508979+0000","last_became_active":"2023-10-01T09:32:27.467110+0000","last_became_peered":"2023-10-01T09:32:27.467110+0000","last_unstale":"2023-10-01T09:34:33.508979+0000","last_undegraded":"2023-10-01T09:34:33.508979+0000","last_fullsized":"2023-10-01T09:34:33.508979+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":236,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.509157+0000","last_change":"2023-10-01T09:30:39.399483+0000","last_active":"2023-10-01T09:34:33.509157+0000","last_peered":"2023-10-01T09:34:33.509157+0000","last_clean":"2023-10-01T09:34:33.509157+0000","last_became_active":"2023-10-01T09:30:39.399266+0000","last_became_peered":"2023-10-01T09:30:39.399266+0000","last_unstale":"2023-10-01T09:34:33.509157+0000","last_undegraded":"2023-10-01T09:34:33.509157+0000","last_fullsized":"2023-10-01T09:34:33.509157+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":401,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513042+0000","last_change":"2023-10-01T09:23:10.607388+0000","last_active":"2023-10-01T09:34:39.513042+0000","last_peered":"2023-10-01T09:34:39.513042+0000","last_clean":"2023-10-01T09:34:39.513042+0000","last_became_active":"2023-10-01T09:23:10.607084+0000","last_became_peered":"2023-10-01T09:23:10.607084+0000","last_unstale":"2023-10-01T09:34:39.513042+0000","last_undegraded":"2023-10-01T09:34:39.513042+0000","last_fullsized":"2023-10-01T09:34:39.513042+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:19.669393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":94,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511945+0000","last_change":"2023-10-01T09:33:11.663325+0000","last_active":"2023-10-01T09:34:39.511945+0000","last_peered":"2023-10-01T09:34:39.511945+0000","last_clean":"2023-10-01T09:34:39.511945+0000","last_became_active":"2023-10-01T09:33:11.662818+0000","last_became_peered":"2023-10-01T09:33:11.662818+0000","last_unstale":"2023-10-01T09:34:39.511945+0000","last_undegraded":"2023-10-01T09:34:39.511945+0000","last_fullsized":"2023-10-01T09:34:39.511945+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":419,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.508853+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:34:33.508853+0000","last_peered":"2023-10-01T09:34:33.508853+0000","last_clean":"2023-10-01T09:34:33.508853+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:34:33.508853+0000","last_undegraded":"2023-10-01T09:34:33.508853+0000","last_fullsized":"2023-10-01T09:34:33.508853+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":382,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513829+0000","last_change":"2023-10-01T09:23:10.611400+0000","last_active":"2023-10-01T09:34:39.513829+0000","last_peered":"2023-10-01T09:34:39.513829+0000","last_clean":"2023-10-01T09:34:39.513829+0000","last_became_active":"2023-10-01T09:23:10.610928+0000","last_became_peered":"2023-10-01T09:23:10.610928+0000","last_unstale":"2023-10-01T09:34:39.513829+0000","last_undegraded":"2023-10-01T09:34:39.513829+0000","last_fullsized":"2023-10-01T09:34:39.513829+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:17.141540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":164,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.505724+0000","last_change":"2023-10-01T09:33:11.658471+0000","last_active":"2023-10-01T09:34:33.505724+0000","last_peered":"2023-10-01T09:34:33.505724+0000","last_clean":"2023-10-01T09:34:33.505724+0000","last_became_active":"2023-10-01T09:33:11.657960+0000","last_became_peered":"2023-10-01T09:33:11.657960+0000","last_unstale":"2023-10-01T09:34:33.505724+0000","last_undegraded":"2023-10-01T09:34:33.505724+0000","last_fullsized":"2023-10-01T09:34:33.505724+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":97,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.509082+0000","last_change":"2023-10-01T09:30:39.398326+0000","last_active":"2023-10-01T09:34:33.509082+0000","last_peered":"2023-10-01T09:34:33.509082+0000","last_clean":"2023-10-01T09:34:33.509082+0000","last_became_active":"2023-10-01T09:30:39.398021+0000","last_became_peered":"2023-10-01T09:30:39.398021+0000","last_unstale":"2023-10-01T09:34:33.509082+0000","last_undegraded":"2023-10-01T09:34:33.509082+0000","last_fullsized":"2023-10-01T09:34:33.509082+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.5","version":"409'3153","reported_seq":3171,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:43.931022+0000","last_change":"2023-10-01T09:34:06.140161+0000","last_active":"2023-10-01T09:34:43.931022+0000","last_peered":"2023-10-01T09:34:43.931022+0000","last_clean":"2023-10-01T09:34:43.931022+0000","last_became_active":"2023-10-01T09:34:06.139856+0000","last_became_peered":"2023-10-01T09:34:06.139856+0000","last_unstale":"2023-10-01T09:34:43.931022+0000","last_undegraded":"2023-10-01T09:34:43.931022+0000","last_fullsized":"2023-10-01T09:34:43.931022+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3153,"log_dups_size":0,"ondisk_log_size":3153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:58:29.822074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206635008,"num_objects":3153,"num_object_clones":0,"num_object_copies":9459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3153,"num_write_kb":201792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":201,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.509199+0000","last_change":"2023-10-01T09:32:27.502868+0000","last_active":"2023-10-01T09:34:33.509199+0000","last_peered":"2023-10-01T09:34:33.509199+0000","last_clean":"2023-10-01T09:34:33.509199+0000","last_became_active":"2023-10-01T09:32:27.455662+0000","last_became_peered":"2023-10-01T09:32:27.455662+0000","last_unstale":"2023-10-01T09:34:33.509199+0000","last_undegraded":"2023-10-01T09:34:33.509199+0000","last_fullsized":"2023-10-01T09:34:33.509199+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":111,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511596+0000","last_change":"2023-10-01T09:33:11.656404+0000","last_active":"2023-10-01T09:34:39.511596+0000","last_peered":"2023-10-01T09:34:39.511596+0000","last_clean":"2023-10-01T09:34:39.511596+0000","last_became_active":"2023-10-01T09:33:11.656093+0000","last_became_peered":"2023-10-01T09:33:11.656093+0000","last_unstale":"2023-10-01T09:34:39.511596+0000","last_undegraded":"2023-10-01T09:34:39.511596+0000","last_fullsized":"2023-10-01T09:34:39.511596+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"409'3076","reported_seq":3094,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:43.665324+0000","last_change":"2023-10-01T09:34:06.168002+0000","last_active":"2023-10-01T09:34:43.665324+0000","last_peered":"2023-10-01T09:34:43.665324+0000","last_clean":"2023-10-01T09:34:43.665324+0000","last_became_active":"2023-10-01T09:34:06.167679+0000","last_became_peered":"2023-10-01T09:34:06.167679+0000","last_unstale":"2023-10-01T09:34:43.665324+0000","last_undegraded":"2023-10-01T09:34:43.665324+0000","last_fullsized":"2023-10-01T09:34:43.665324+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3076,"log_dups_size":0,"ondisk_log_size":3076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:21:42.118174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201588736,"num_objects":3076,"num_object_clones":0,"num_object_copies":9228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3076,"num_write_kb":196864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":402,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.509042+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:34:33.509042+0000","last_peered":"2023-10-01T09:34:33.509042+0000","last_clean":"2023-10-01T09:34:33.509042+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:34:33.509042+0000","last_undegraded":"2023-10-01T09:34:33.509042+0000","last_fullsized":"2023-10-01T09:34:33.509042+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":442,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504161+0000","last_change":"2023-10-01T09:32:27.445399+0000","last_active":"2023-10-01T09:34:39.504161+0000","last_peered":"2023-10-01T09:34:39.504161+0000","last_clean":"2023-10-01T09:34:39.504161+0000","last_became_active":"2023-10-01T09:32:27.445067+0000","last_became_peered":"2023-10-01T09:32:27.445067+0000","last_unstale":"2023-10-01T09:34:39.504161+0000","last_undegraded":"2023-10-01T09:34:39.504161+0000","last_fullsized":"2023-10-01T09:34:39.504161+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:22:23.844066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"409'3163","reported_seq":3180,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:44.491395+0000","last_change":"2023-10-01T09:34:06.141263+0000","last_active":"2023-10-01T09:34:44.491395+0000","last_peered":"2023-10-01T09:34:44.491395+0000","last_clean":"2023-10-01T09:34:44.491395+0000","last_became_active":"2023-10-01T09:34:06.140720+0000","last_became_peered":"2023-10-01T09:34:06.140720+0000","last_unstale":"2023-10-01T09:34:44.491395+0000","last_undegraded":"2023-10-01T09:34:44.491395+0000","last_fullsized":"2023-10-01T09:34:44.491395+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3163,"log_dups_size":0,"ondisk_log_size":3163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207290368,"num_objects":3163,"num_object_clones":0,"num_object_copies":9489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3163,"num_write_kb":202432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":488,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.509126+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:34:33.509126+0000","last_peered":"2023-10-01T09:34:33.509126+0000","last_clean":"2023-10-01T09:34:33.509126+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:34:33.509126+0000","last_undegraded":"2023-10-01T09:34:33.509126+0000","last_fullsized":"2023-10-01T09:34:33.509126+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":111,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.506072+0000","last_change":"2023-10-01T09:28:02.361663+0000","last_active":"2023-10-01T09:34:33.506072+0000","last_peered":"2023-10-01T09:34:33.506072+0000","last_clean":"2023-10-01T09:34:33.506072+0000","last_became_active":"2023-10-01T09:28:02.361093+0000","last_became_peered":"2023-10-01T09:28:02.361093+0000","last_unstale":"2023-10-01T09:34:33.506072+0000","last_undegraded":"2023-10-01T09:34:33.506072+0000","last_fullsized":"2023-10-01T09:34:33.506072+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"409'3114","reported_seq":3131,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:44.562757+0000","last_change":"2023-10-01T09:34:06.140196+0000","last_active":"2023-10-01T09:34:44.562757+0000","last_peered":"2023-10-01T09:34:44.562757+0000","last_clean":"2023-10-01T09:34:44.562757+0000","last_became_active":"2023-10-01T09:34:06.139888+0000","last_became_peered":"2023-10-01T09:34:06.139888+0000","last_unstale":"2023-10-01T09:34:44.562757+0000","last_undegraded":"2023-10-01T09:34:44.562757+0000","last_fullsized":"2023-10-01T09:34:44.562757+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3114,"log_dups_size":0,"ondisk_log_size":3114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204079104,"num_objects":3114,"num_object_clones":0,"num_object_copies":9342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3114,"num_write_kb":199296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":111,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511787+0000","last_change":"2023-10-01T09:33:11.656845+0000","last_active":"2023-10-01T09:34:39.511787+0000","last_peered":"2023-10-01T09:34:39.511787+0000","last_clean":"2023-10-01T09:34:39.511787+0000","last_became_active":"2023-10-01T09:33:11.655365+0000","last_became_peered":"2023-10-01T09:33:11.655365+0000","last_unstale":"2023-10-01T09:34:39.511787+0000","last_undegraded":"2023-10-01T09:34:39.511787+0000","last_fullsized":"2023-10-01T09:34:39.511787+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":383,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513060+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:34:39.513060+0000","last_peered":"2023-10-01T09:34:39.513060+0000","last_clean":"2023-10-01T09:34:39.513060+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:34:39.513060+0000","last_undegraded":"2023-10-01T09:34:39.513060+0000","last_fullsized":"2023-10-01T09:34:39.513060+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"408'2814","reported_seq":2830,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:41.749724+0000","last_change":"2023-10-01T09:34:06.135421+0000","last_active":"2023-10-01T09:34:41.749724+0000","last_peered":"2023-10-01T09:34:41.749724+0000","last_clean":"2023-10-01T09:34:41.749724+0000","last_became_active":"2023-10-01T09:34:06.135086+0000","last_became_peered":"2023-10-01T09:34:06.135086+0000","last_unstale":"2023-10-01T09:34:41.749724+0000","last_undegraded":"2023-10-01T09:34:41.749724+0000","last_fullsized":"2023-10-01T09:34:41.749724+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":2814,"log_dups_size":0,"ondisk_log_size":2814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184418304,"num_objects":2814,"num_object_clones":0,"num_object_copies":8442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2814,"num_write_kb":180096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":110,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513160+0000","last_change":"2023-10-01T09:32:27.489306+0000","last_active":"2023-10-01T09:34:39.513160+0000","last_peered":"2023-10-01T09:34:39.513160+0000","last_clean":"2023-10-01T09:34:39.513160+0000","last_became_active":"2023-10-01T09:32:27.449850+0000","last_became_peered":"2023-10-01T09:32:27.449850+0000","last_unstale":"2023-10-01T09:34:39.513160+0000","last_undegraded":"2023-10-01T09:34:39.513160+0000","last_fullsized":"2023-10-01T09:34:39.513160+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:26:15.712006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":57,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.517285+0000","last_change":"2023-10-01T09:32:27.449970+0000","last_active":"2023-10-01T09:34:33.517285+0000","last_peered":"2023-10-01T09:34:33.517285+0000","last_clean":"2023-10-01T09:34:33.517285+0000","last_became_active":"2023-10-01T09:32:27.449665+0000","last_became_peered":"2023-10-01T09:32:27.449665+0000","last_unstale":"2023-10-01T09:34:33.517285+0000","last_undegraded":"2023-10-01T09:34:33.517285+0000","last_fullsized":"2023-10-01T09:34:33.517285+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"409'3123","reported_seq":3141,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:43.652181+0000","last_change":"2023-10-01T09:34:06.135399+0000","last_active":"2023-10-01T09:34:43.652181+0000","last_peered":"2023-10-01T09:34:43.652181+0000","last_clean":"2023-10-01T09:34:43.652181+0000","last_became_active":"2023-10-01T09:34:06.135073+0000","last_became_peered":"2023-10-01T09:34:06.135073+0000","last_unstale":"2023-10-01T09:34:43.652181+0000","last_undegraded":"2023-10-01T09:34:43.652181+0000","last_fullsized":"2023-10-01T09:34:43.652181+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3123,"log_dups_size":0,"ondisk_log_size":3123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:40:11.547974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204668928,"num_objects":3123,"num_object_clones":0,"num_object_copies":9369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3123,"num_write_kb":199872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":392,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.505930+0000","last_change":"2023-10-01T09:32:27.447519+0000","last_active":"2023-10-01T09:34:39.505930+0000","last_peered":"2023-10-01T09:34:39.505930+0000","last_clean":"2023-10-01T09:34:39.505930+0000","last_became_active":"2023-10-01T09:32:27.447142+0000","last_became_peered":"2023-10-01T09:32:27.447142+0000","last_unstale":"2023-10-01T09:34:39.505930+0000","last_undegraded":"2023-10-01T09:34:39.505930+0000","last_fullsized":"2023-10-01T09:34:39.505930+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:28:48.014433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":436,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504208+0000","last_change":"2023-10-01T09:32:27.452641+0000","last_active":"2023-10-01T09:34:39.504208+0000","last_peered":"2023-10-01T09:34:39.504208+0000","last_clean":"2023-10-01T09:34:39.504208+0000","last_became_active":"2023-10-01T09:32:27.452411+0000","last_became_peered":"2023-10-01T09:32:27.452411+0000","last_unstale":"2023-10-01T09:34:39.504208+0000","last_undegraded":"2023-10-01T09:34:39.504208+0000","last_fullsized":"2023-10-01T09:34:39.504208+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:47:43.031520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"408'2921","reported_seq":2937,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:41.765755+0000","last_change":"2023-10-01T09:34:06.128656+0000","last_active":"2023-10-01T09:34:41.765755+0000","last_peered":"2023-10-01T09:34:41.765755+0000","last_clean":"2023-10-01T09:34:41.765755+0000","last_became_active":"2023-10-01T09:34:06.128364+0000","last_became_peered":"2023-10-01T09:34:06.128364+0000","last_unstale":"2023-10-01T09:34:41.765755+0000","last_undegraded":"2023-10-01T09:34:41.765755+0000","last_fullsized":"2023-10-01T09:34:41.765755+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":2921,"log_dups_size":0,"ondisk_log_size":2921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191430656,"num_objects":2921,"num_object_clones":0,"num_object_copies":8763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2921,"num_write_kb":186944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":280,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.508769+0000","last_change":"2023-10-01T09:32:27.458143+0000","last_active":"2023-10-01T09:34:33.508769+0000","last_peered":"2023-10-01T09:34:33.508769+0000","last_clean":"2023-10-01T09:34:33.508769+0000","last_became_active":"2023-10-01T09:32:27.457802+0000","last_became_peered":"2023-10-01T09:32:27.457802+0000","last_unstale":"2023-10-01T09:34:33.508769+0000","last_undegraded":"2023-10-01T09:34:33.508769+0000","last_fullsized":"2023-10-01T09:34:33.508769+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":198,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504419+0000","last_change":"2023-10-01T09:32:27.446005+0000","last_active":"2023-10-01T09:34:39.504419+0000","last_peered":"2023-10-01T09:34:39.504419+0000","last_clean":"2023-10-01T09:34:39.504419+0000","last_became_active":"2023-10-01T09:32:27.445757+0000","last_became_peered":"2023-10-01T09:32:27.445757+0000","last_unstale":"2023-10-01T09:34:39.504419+0000","last_undegraded":"2023-10-01T09:34:39.504419+0000","last_fullsized":"2023-10-01T09:34:39.504419+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:27:23.521543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.2","version":"409'3155","reported_seq":3173,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:43.640833+0000","last_change":"2023-10-01T09:34:06.130449+0000","last_active":"2023-10-01T09:34:43.640833+0000","last_peered":"2023-10-01T09:34:43.640833+0000","last_clean":"2023-10-01T09:34:43.640833+0000","last_became_active":"2023-10-01T09:34:06.130193+0000","last_became_peered":"2023-10-01T09:34:06.130193+0000","last_unstale":"2023-10-01T09:34:43.640833+0000","last_undegraded":"2023-10-01T09:34:43.640833+0000","last_fullsized":"2023-10-01T09:34:43.640833+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3155,"log_dups_size":0,"ondisk_log_size":3155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:23:07.736775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206766080,"num_objects":3155,"num_object_clones":0,"num_object_copies":9465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3155,"num_write_kb":201920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":249,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.507637+0000","last_change":"2023-10-01T09:30:39.396141+0000","last_active":"2023-10-01T09:34:39.507637+0000","last_peered":"2023-10-01T09:34:39.507637+0000","last_clean":"2023-10-01T09:34:39.507637+0000","last_became_active":"2023-10-01T09:30:39.395843+0000","last_became_peered":"2023-10-01T09:30:39.395843+0000","last_unstale":"2023-10-01T09:34:39.507637+0000","last_undegraded":"2023-10-01T09:34:39.507637+0000","last_fullsized":"2023-10-01T09:34:39.507637+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:13:32.932438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":297,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.503966+0000","last_change":"2023-10-01T09:32:27.445484+0000","last_active":"2023-10-01T09:34:39.503966+0000","last_peered":"2023-10-01T09:34:39.503966+0000","last_clean":"2023-10-01T09:34:39.503966+0000","last_became_active":"2023-10-01T09:32:27.445198+0000","last_became_peered":"2023-10-01T09:32:27.445198+0000","last_unstale":"2023-10-01T09:34:39.503966+0000","last_undegraded":"2023-10-01T09:34:39.503966+0000","last_fullsized":"2023-10-01T09:34:39.503966+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:29:19.880130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"409'3156","reported_seq":3172,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:44.547489+0000","last_change":"2023-10-01T09:34:06.141056+0000","last_active":"2023-10-01T09:34:44.547489+0000","last_peered":"2023-10-01T09:34:44.547489+0000","last_clean":"2023-10-01T09:34:44.547489+0000","last_became_active":"2023-10-01T09:34:06.140163+0000","last_became_peered":"2023-10-01T09:34:06.140163+0000","last_unstale":"2023-10-01T09:34:44.547489+0000","last_undegraded":"2023-10-01T09:34:44.547489+0000","last_fullsized":"2023-10-01T09:34:44.547489+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3156,"log_dups_size":0,"ondisk_log_size":3156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206831616,"num_objects":3156,"num_object_clones":0,"num_object_copies":9468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3156,"num_write_kb":201984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1231,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504217+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:34:39.504217+0000","last_peered":"2023-10-01T09:34:39.504217+0000","last_clean":"2023-10-01T09:34:39.504217+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:34:39.504217+0000","last_undegraded":"2023-10-01T09:34:39.504217+0000","last_fullsized":"2023-10-01T09:34:39.504217+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":94,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.512016+0000","last_change":"2023-10-01T09:33:11.663163+0000","last_active":"2023-10-01T09:34:39.512016+0000","last_peered":"2023-10-01T09:34:39.512016+0000","last_clean":"2023-10-01T09:34:39.512016+0000","last_became_active":"2023-10-01T09:33:11.662771+0000","last_became_peered":"2023-10-01T09:33:11.662771+0000","last_unstale":"2023-10-01T09:34:39.512016+0000","last_undegraded":"2023-10-01T09:34:39.512016+0000","last_fullsized":"2023-10-01T09:34:39.512016+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":412,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.505859+0000","last_change":"2023-10-01T09:33:11.664415+0000","last_active":"2023-10-01T09:34:39.505859+0000","last_peered":"2023-10-01T09:34:39.505859+0000","last_clean":"2023-10-01T09:34:39.505859+0000","last_became_active":"2023-10-01T09:33:11.664177+0000","last_became_peered":"2023-10-01T09:33:11.664177+0000","last_unstale":"2023-10-01T09:34:39.505859+0000","last_undegraded":"2023-10-01T09:34:39.505859+0000","last_fullsized":"2023-10-01T09:34:39.505859+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:44:01.229405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"409'3176","reported_seq":3194,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:43.927697+0000","last_change":"2023-10-01T09:34:06.140052+0000","last_active":"2023-10-01T09:34:43.927697+0000","last_peered":"2023-10-01T09:34:43.927697+0000","last_clean":"2023-10-01T09:34:43.927697+0000","last_became_active":"2023-10-01T09:34:06.139672+0000","last_became_peered":"2023-10-01T09:34:06.139672+0000","last_unstale":"2023-10-01T09:34:43.927697+0000","last_undegraded":"2023-10-01T09:34:43.927697+0000","last_fullsized":"2023-10-01T09:34:43.927697+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3176,"log_dups_size":0,"ondisk_log_size":3176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:39:55.000830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208142336,"num_objects":3176,"num_object_clones":0,"num_object_copies":9528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3176,"num_write_kb":203264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1074,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511762+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:34:39.511762+0000","last_peered":"2023-10-01T09:34:39.511762+0000","last_clean":"2023-10-01T09:34:39.511762+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:34:39.511762+0000","last_undegraded":"2023-10-01T09:34:39.511762+0000","last_fullsized":"2023-10-01T09:34:39.511762+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":383,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513358+0000","last_change":"2023-10-01T09:32:27.451746+0000","last_active":"2023-10-01T09:34:39.513358+0000","last_peered":"2023-10-01T09:34:39.513358+0000","last_clean":"2023-10-01T09:34:39.513358+0000","last_became_active":"2023-10-01T09:32:27.451069+0000","last_became_peered":"2023-10-01T09:32:27.451069+0000","last_unstale":"2023-10-01T09:34:39.513358+0000","last_undegraded":"2023-10-01T09:34:39.513358+0000","last_fullsized":"2023-10-01T09:34:39.513358+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:50.586397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":52,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511914+0000","last_change":"2023-10-01T09:33:11.655679+0000","last_active":"2023-10-01T09:34:39.511914+0000","last_peered":"2023-10-01T09:34:39.511914+0000","last_clean":"2023-10-01T09:34:39.511914+0000","last_became_active":"2023-10-01T09:33:11.653576+0000","last_became_peered":"2023-10-01T09:33:11.653576+0000","last_unstale":"2023-10-01T09:34:39.511914+0000","last_undegraded":"2023-10-01T09:34:39.511914+0000","last_fullsized":"2023-10-01T09:34:39.511914+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"409'3139","reported_seq":3156,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:44.546551+0000","last_change":"2023-10-01T09:34:06.140732+0000","last_active":"2023-10-01T09:34:44.546551+0000","last_peered":"2023-10-01T09:34:44.546551+0000","last_clean":"2023-10-01T09:34:44.546551+0000","last_became_active":"2023-10-01T09:34:06.140433+0000","last_became_peered":"2023-10-01T09:34:06.140433+0000","last_unstale":"2023-10-01T09:34:44.546551+0000","last_undegraded":"2023-10-01T09:34:44.546551+0000","last_fullsized":"2023-10-01T09:34:44.546551+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3139,"log_dups_size":0,"ondisk_log_size":3139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205717504,"num_objects":3139,"num_object_clones":0,"num_object_copies":9417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3139,"num_write_kb":200896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":601,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513045+0000","last_change":"2023-10-01T09:30:39.393231+0000","last_active":"2023-10-01T09:34:39.513045+0000","last_peered":"2023-10-01T09:34:39.513045+0000","last_clean":"2023-10-01T09:34:39.513045+0000","last_became_active":"2023-10-01T09:30:39.392700+0000","last_became_peered":"2023-10-01T09:30:39.392700+0000","last_unstale":"2023-10-01T09:34:39.513045+0000","last_undegraded":"2023-10-01T09:34:39.513045+0000","last_fullsized":"2023-10-01T09:34:39.513045+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:38:26.873688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":440,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.507797+0000","last_change":"2023-10-01T09:28:02.346596+0000","last_active":"2023-10-01T09:34:39.507797+0000","last_peered":"2023-10-01T09:34:39.507797+0000","last_clean":"2023-10-01T09:34:39.507797+0000","last_became_active":"2023-10-01T09:28:02.346318+0000","last_became_peered":"2023-10-01T09:28:02.346318+0000","last_unstale":"2023-10-01T09:34:39.507797+0000","last_undegraded":"2023-10-01T09:34:39.507797+0000","last_fullsized":"2023-10-01T09:34:39.507797+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:27:45.840901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":309,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504104+0000","last_change":"2023-10-01T09:33:11.665777+0000","last_active":"2023-10-01T09:34:39.504104+0000","last_peered":"2023-10-01T09:34:39.504104+0000","last_clean":"2023-10-01T09:34:39.504104+0000","last_became_active":"2023-10-01T09:33:11.665459+0000","last_became_peered":"2023-10-01T09:33:11.665459+0000","last_unstale":"2023-10-01T09:34:39.504104+0000","last_undegraded":"2023-10-01T09:34:39.504104+0000","last_fullsized":"2023-10-01T09:34:39.504104+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:40:34.693521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"409'3114","reported_seq":3129,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:43.936953+0000","last_change":"2023-10-01T09:34:06.140010+0000","last_active":"2023-10-01T09:34:43.936953+0000","last_peered":"2023-10-01T09:34:43.936953+0000","last_clean":"2023-10-01T09:34:43.936953+0000","last_became_active":"2023-10-01T09:34:06.139718+0000","last_became_peered":"2023-10-01T09:34:06.139718+0000","last_unstale":"2023-10-01T09:34:43.936953+0000","last_undegraded":"2023-10-01T09:34:43.936953+0000","last_fullsized":"2023-10-01T09:34:43.936953+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3114,"log_dups_size":0,"ondisk_log_size":3114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:58.151539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204079104,"num_objects":3114,"num_object_clones":0,"num_object_copies":9342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3114,"num_write_kb":199296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1325,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511613+0000","last_change":"2023-10-01T09:33:11.665240+0000","last_active":"2023-10-01T09:34:39.511613+0000","last_peered":"2023-10-01T09:34:39.511613+0000","last_clean":"2023-10-01T09:34:39.511613+0000","last_became_active":"2023-10-01T09:33:11.664702+0000","last_became_peered":"2023-10-01T09:33:11.664702+0000","last_unstale":"2023-10-01T09:34:39.511613+0000","last_undegraded":"2023-10-01T09:34:39.511613+0000","last_fullsized":"2023-10-01T09:34:39.511613+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":626,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.508662+0000","last_change":"2023-10-01T09:29:04.962927+0000","last_active":"2023-10-01T09:34:33.508662+0000","last_peered":"2023-10-01T09:34:33.508662+0000","last_clean":"2023-10-01T09:34:33.508662+0000","last_became_active":"2023-10-01T09:29:04.962542+0000","last_became_peered":"2023-10-01T09:29:04.962542+0000","last_unstale":"2023-10-01T09:34:33.508662+0000","last_undegraded":"2023-10-01T09:34:33.508662+0000","last_fullsized":"2023-10-01T09:34:33.508662+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":181,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511683+0000","last_change":"2023-10-01T09:33:11.663444+0000","last_active":"2023-10-01T09:34:39.511683+0000","last_peered":"2023-10-01T09:34:39.511683+0000","last_clean":"2023-10-01T09:34:39.511683+0000","last_became_active":"2023-10-01T09:33:11.660310+0000","last_became_peered":"2023-10-01T09:33:11.660310+0000","last_unstale":"2023-10-01T09:34:39.511683+0000","last_undegraded":"2023-10-01T09:34:39.511683+0000","last_fullsized":"2023-10-01T09:34:39.511683+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":322,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.509200+0000","last_change":"2023-10-01T09:32:27.453291+0000","last_active":"2023-10-01T09:34:33.509200+0000","last_peered":"2023-10-01T09:34:33.509200+0000","last_clean":"2023-10-01T09:34:33.509200+0000","last_became_active":"2023-10-01T09:32:27.453026+0000","last_became_peered":"2023-10-01T09:32:27.453026+0000","last_unstale":"2023-10-01T09:34:33.509200+0000","last_undegraded":"2023-10-01T09:34:33.509200+0000","last_fullsized":"2023-10-01T09:34:33.509200+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"409'3097","reported_seq":3115,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:43.929855+0000","last_change":"2023-10-01T09:34:06.140844+0000","last_active":"2023-10-01T09:34:43.929855+0000","last_peered":"2023-10-01T09:34:43.929855+0000","last_clean":"2023-10-01T09:34:43.929855+0000","last_became_active":"2023-10-01T09:34:06.139812+0000","last_became_peered":"2023-10-01T09:34:06.139812+0000","last_unstale":"2023-10-01T09:34:43.929855+0000","last_undegraded":"2023-10-01T09:34:43.929855+0000","last_fullsized":"2023-10-01T09:34:43.929855+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3097,"log_dups_size":0,"ondisk_log_size":3097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:31:05.105491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202964992,"num_objects":3097,"num_object_clones":0,"num_object_copies":9291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3097,"num_write_kb":198208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":504,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.508674+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:34:33.508674+0000","last_peered":"2023-10-01T09:34:33.508674+0000","last_clean":"2023-10-01T09:34:33.508674+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:34:33.508674+0000","last_undegraded":"2023-10-01T09:34:33.508674+0000","last_fullsized":"2023-10-01T09:34:33.508674+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":426,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513150+0000","last_change":"2023-10-01T09:32:27.450723+0000","last_active":"2023-10-01T09:34:39.513150+0000","last_peered":"2023-10-01T09:34:39.513150+0000","last_clean":"2023-10-01T09:34:39.513150+0000","last_became_active":"2023-10-01T09:32:27.450037+0000","last_became_peered":"2023-10-01T09:32:27.450037+0000","last_unstale":"2023-10-01T09:34:39.513150+0000","last_undegraded":"2023-10-01T09:34:39.513150+0000","last_fullsized":"2023-10-01T09:34:39.513150+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:37.260098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":416,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513650+0000","last_change":"2023-10-01T09:19:30.091481+0000","last_active":"2023-10-01T09:34:39.513650+0000","last_peered":"2023-10-01T09:34:39.513650+0000","last_clean":"2023-10-01T09:34:39.513650+0000","last_became_active":"2023-10-01T09:19:30.091151+0000","last_became_peered":"2023-10-01T09:19:30.091151+0000","last_unstale":"2023-10-01T09:34:39.513650+0000","last_undegraded":"2023-10-01T09:34:39.513650+0000","last_fullsized":"2023-10-01T09:34:39.513650+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:09.127408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"408'3003","reported_seq":3019,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:41.766079+0000","last_change":"2023-10-01T09:34:06.167577+0000","last_active":"2023-10-01T09:34:41.766079+0000","last_peered":"2023-10-01T09:34:41.766079+0000","last_clean":"2023-10-01T09:34:41.766079+0000","last_became_active":"2023-10-01T09:34:06.167267+0000","last_became_peered":"2023-10-01T09:34:06.167267+0000","last_unstale":"2023-10-01T09:34:41.766079+0000","last_undegraded":"2023-10-01T09:34:41.766079+0000","last_fullsized":"2023-10-01T09:34:41.766079+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3003,"log_dups_size":0,"ondisk_log_size":3003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196804608,"num_objects":3003,"num_object_clones":0,"num_object_copies":9009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3003,"num_write_kb":192192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":846,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.509181+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:34:33.509181+0000","last_peered":"2023-10-01T09:34:33.509181+0000","last_clean":"2023-10-01T09:34:33.509181+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:34:33.509181+0000","last_undegraded":"2023-10-01T09:34:33.509181+0000","last_fullsized":"2023-10-01T09:34:33.509181+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":445,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504320+0000","last_change":"2023-10-01T09:33:11.665789+0000","last_active":"2023-10-01T09:34:39.504320+0000","last_peered":"2023-10-01T09:34:39.504320+0000","last_clean":"2023-10-01T09:34:39.504320+0000","last_became_active":"2023-10-01T09:33:11.665477+0000","last_became_peered":"2023-10-01T09:33:11.665477+0000","last_unstale":"2023-10-01T09:34:39.504320+0000","last_undegraded":"2023-10-01T09:34:39.504320+0000","last_fullsized":"2023-10-01T09:34:39.504320+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:18:38.766268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":399,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513274+0000","last_change":"2023-10-01T09:33:11.656981+0000","last_active":"2023-10-01T09:34:39.513274+0000","last_peered":"2023-10-01T09:34:39.513274+0000","last_clean":"2023-10-01T09:34:39.513274+0000","last_became_active":"2023-10-01T09:33:11.656334+0000","last_became_peered":"2023-10-01T09:33:11.656334+0000","last_unstale":"2023-10-01T09:34:39.513274+0000","last_undegraded":"2023-10-01T09:34:39.513274+0000","last_fullsized":"2023-10-01T09:34:39.513274+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:50:15.562113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"408'2839","reported_seq":2855,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:41.708207+0000","last_change":"2023-10-01T09:34:06.140475+0000","last_active":"2023-10-01T09:34:41.708207+0000","last_peered":"2023-10-01T09:34:41.708207+0000","last_clean":"2023-10-01T09:34:41.708207+0000","last_became_active":"2023-10-01T09:34:06.140142+0000","last_became_peered":"2023-10-01T09:34:06.140142+0000","last_unstale":"2023-10-01T09:34:41.708207+0000","last_undegraded":"2023-10-01T09:34:41.708207+0000","last_fullsized":"2023-10-01T09:34:41.708207+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":2839,"log_dups_size":0,"ondisk_log_size":2839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186056704,"num_objects":2839,"num_object_clones":0,"num_object_copies":8517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2839,"num_write_kb":181696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":321,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.509284+0000","last_change":"2023-10-01T09:30:39.402924+0000","last_active":"2023-10-01T09:34:33.509284+0000","last_peered":"2023-10-01T09:34:33.509284+0000","last_clean":"2023-10-01T09:34:33.509284+0000","last_became_active":"2023-10-01T09:30:39.402594+0000","last_became_peered":"2023-10-01T09:30:39.402594+0000","last_unstale":"2023-10-01T09:34:33.509284+0000","last_undegraded":"2023-10-01T09:34:33.509284+0000","last_fullsized":"2023-10-01T09:34:33.509284+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1037,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504786+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:34:39.504786+0000","last_peered":"2023-10-01T09:34:39.504786+0000","last_clean":"2023-10-01T09:34:39.504786+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:34:39.504786+0000","last_undegraded":"2023-10-01T09:34:39.504786+0000","last_fullsized":"2023-10-01T09:34:39.504786+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":97,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.508567+0000","last_change":"2023-10-01T09:30:39.418085+0000","last_active":"2023-10-01T09:34:33.508567+0000","last_peered":"2023-10-01T09:34:33.508567+0000","last_clean":"2023-10-01T09:34:33.508567+0000","last_became_active":"2023-10-01T09:30:39.417846+0000","last_became_peered":"2023-10-01T09:30:39.417846+0000","last_unstale":"2023-10-01T09:34:33.508567+0000","last_undegraded":"2023-10-01T09:34:33.508567+0000","last_fullsized":"2023-10-01T09:34:33.508567+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"408'2958","reported_seq":2974,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:41.760351+0000","last_change":"2023-10-01T09:34:06.167610+0000","last_active":"2023-10-01T09:34:41.760351+0000","last_peered":"2023-10-01T09:34:41.760351+0000","last_clean":"2023-10-01T09:34:41.760351+0000","last_became_active":"2023-10-01T09:34:06.167324+0000","last_became_peered":"2023-10-01T09:34:06.167324+0000","last_unstale":"2023-10-01T09:34:41.760351+0000","last_undegraded":"2023-10-01T09:34:41.760351+0000","last_fullsized":"2023-10-01T09:34:41.760351+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":2958,"log_dups_size":0,"ondisk_log_size":2958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193855488,"num_objects":2958,"num_object_clones":0,"num_object_copies":8874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2958,"num_write_kb":189312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":121,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.506085+0000","last_change":"2023-10-01T09:29:11.374086+0000","last_active":"2023-10-01T09:34:33.506085+0000","last_peered":"2023-10-01T09:34:33.506085+0000","last_clean":"2023-10-01T09:34:33.506085+0000","last_became_active":"2023-10-01T09:28:02.319106+0000","last_became_peered":"2023-10-01T09:28:02.319106+0000","last_unstale":"2023-10-01T09:34:33.506085+0000","last_undegraded":"2023-10-01T09:34:33.506085+0000","last_fullsized":"2023-10-01T09:34:33.506085+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":421,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513842+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:34:39.513842+0000","last_peered":"2023-10-01T09:34:39.513842+0000","last_clean":"2023-10-01T09:34:39.513842+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:34:39.513842+0000","last_undegraded":"2023-10-01T09:34:39.513842+0000","last_fullsized":"2023-10-01T09:34:39.513842+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":132,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511599+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:34:39.511599+0000","last_peered":"2023-10-01T09:34:39.511599+0000","last_clean":"2023-10-01T09:34:39.511599+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:34:39.511599+0000","last_undegraded":"2023-10-01T09:34:39.511599+0000","last_fullsized":"2023-10-01T09:34:39.511599+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":371,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513293+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:34:39.513293+0000","last_peered":"2023-10-01T09:34:39.513293+0000","last_clean":"2023-10-01T09:34:39.513293+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:34:39.513293+0000","last_undegraded":"2023-10-01T09:34:39.513293+0000","last_fullsized":"2023-10-01T09:34:39.513293+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":440,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504388+0000","last_change":"2023-10-01T09:22:40.166927+0000","last_active":"2023-10-01T09:34:39.504388+0000","last_peered":"2023-10-01T09:34:39.504388+0000","last_clean":"2023-10-01T09:34:39.504388+0000","last_became_active":"2023-10-01T09:22:40.166606+0000","last_became_peered":"2023-10-01T09:22:40.166606+0000","last_unstale":"2023-10-01T09:34:39.504388+0000","last_undegraded":"2023-10-01T09:34:39.504388+0000","last_fullsized":"2023-10-01T09:34:39.504388+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:54:18.044207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":144,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511709+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:34:39.511709+0000","last_peered":"2023-10-01T09:34:39.511709+0000","last_clean":"2023-10-01T09:34:39.511709+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:34:39.511709+0000","last_undegraded":"2023-10-01T09:34:39.511709+0000","last_fullsized":"2023-10-01T09:34:39.511709+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511527+0000","last_change":"2023-10-01T09:33:11.660645+0000","last_active":"2023-10-01T09:34:39.511527+0000","last_peered":"2023-10-01T09:34:39.511527+0000","last_clean":"2023-10-01T09:34:39.511527+0000","last_became_active":"2023-10-01T09:33:11.660059+0000","last_became_peered":"2023-10-01T09:33:11.660059+0000","last_unstale":"2023-10-01T09:34:39.511527+0000","last_undegraded":"2023-10-01T09:34:39.511527+0000","last_fullsized":"2023-10-01T09:34:39.511527+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":144,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.512092+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:34:39.512092+0000","last_peered":"2023-10-01T09:34:39.512092+0000","last_clean":"2023-10-01T09:34:39.512092+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:34:39.512092+0000","last_undegraded":"2023-10-01T09:34:39.512092+0000","last_fullsized":"2023-10-01T09:34:39.512092+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":299,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513132+0000","last_change":"2023-10-01T09:32:27.456398+0000","last_active":"2023-10-01T09:34:39.513132+0000","last_peered":"2023-10-01T09:34:39.513132+0000","last_clean":"2023-10-01T09:34:39.513132+0000","last_became_active":"2023-10-01T09:32:27.456106+0000","last_became_peered":"2023-10-01T09:32:27.456106+0000","last_unstale":"2023-10-01T09:34:39.513132+0000","last_undegraded":"2023-10-01T09:34:39.513132+0000","last_fullsized":"2023-10-01T09:34:39.513132+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:10:29.553023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":211,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-10-01T09:34:33.510836+0000","last_change":"2023-10-01T09:28:02.321395+0000","last_active":"2023-10-01T09:34:33.510836+0000","last_peered":"2023-10-01T09:34:33.510836+0000","last_clean":"2023-10-01T09:34:33.510836+0000","last_became_active":"2023-10-01T09:28:02.321071+0000","last_became_peered":"2023-10-01T09:28:02.321071+0000","last_unstale":"2023-10-01T09:34:33.510836+0000","last_undegraded":"2023-10-01T09:34:33.510836+0000","last_fullsized":"2023-10-01T09:34:33.510836+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:12:45.734920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":317,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513140+0000","last_change":"2023-10-01T09:30:39.392942+0000","last_active":"2023-10-01T09:34:39.513140+0000","last_peered":"2023-10-01T09:34:39.513140+0000","last_clean":"2023-10-01T09:34:39.513140+0000","last_became_active":"2023-10-01T09:30:39.392663+0000","last_became_peered":"2023-10-01T09:30:39.392663+0000","last_unstale":"2023-10-01T09:34:39.513140+0000","last_undegraded":"2023-10-01T09:34:39.513140+0000","last_fullsized":"2023-10-01T09:34:39.513140+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:19:39.644244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":516,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513665+0000","last_change":"2023-10-01T09:33:11.656642+0000","last_active":"2023-10-01T09:34:39.513665+0000","last_peered":"2023-10-01T09:34:39.513665+0000","last_clean":"2023-10-01T09:34:39.513665+0000","last_became_active":"2023-10-01T09:33:11.656325+0000","last_became_peered":"2023-10-01T09:33:11.656325+0000","last_unstale":"2023-10-01T09:34:39.513665+0000","last_undegraded":"2023-10-01T09:34:39.513665+0000","last_fullsized":"2023-10-01T09:34:39.513665+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:01:05.809978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":377,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.507541+0000","last_change":"2023-10-01T09:32:27.445299+0000","last_active":"2023-10-01T09:34:39.507541+0000","last_peered":"2023-10-01T09:34:39.507541+0000","last_clean":"2023-10-01T09:34:39.507541+0000","last_became_active":"2023-10-01T09:32:27.444956+0000","last_became_peered":"2023-10-01T09:32:27.444956+0000","last_unstale":"2023-10-01T09:34:39.507541+0000","last_undegraded":"2023-10-01T09:34:39.507541+0000","last_fullsized":"2023-10-01T09:34:39.507541+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:49:07.862356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":33298,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3211329536,"num_objects":49001,"num_object_clones":0,"num_object_copies":147003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":49001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":49001,"num_write_kb":3136064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9567731712,"data_stored":9567731712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":49001,"ondisk_log_size":49001,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430609,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2274496,"kb_used_data":1688888,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":91470656,"statfs":{"total":95995035648,"available":93665951744,"internally_reserved":0,"allocated":1729421312,"data_stored":1728530949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,5],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:34:34 2023","interfaces":[{"interface":"back","average":{"1min":10.117000000000001,"5min":3.4329999999999998,"15min":3.02},"min":{"1min":0.26800000000000002,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":108.90000000000001,"5min":108.90000000000001,"15min":108.90000000000001},"last":46.042999999999999},{"interface":"front","average":{"1min":9.8490000000000002,"5min":3.48,"15min":3.2429999999999999},"min":{"1min":0.40200000000000002,"5min":0.23899999999999999,"15min":0.22500000000000001},"max":{"1min":109.187,"5min":109.187,"15min":109.187},"last":46.057000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:33:51 2023","interfaces":[{"interface":"back","average":{"1min":5.3789999999999996,"5min":4.915,"15min":4.0860000000000003},"min":{"1min":0.26100000000000001,"5min":0.222,"15min":0.222},"max":{"1min":56.694000000000003,"5min":205.34200000000001,"15min":205.34200000000001},"last":49.100000000000001},{"interface":"front","average":{"1min":5.9710000000000001,"5min":4.8159999999999998,"15min":4.0570000000000004},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.23599999999999999},"max":{"1min":57.816000000000003,"5min":205.44499999999999,"15min":205.44499999999999},"last":49.560000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:34:24 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.7250000000000001,"15min":2.653},"min":{"1min":0.32400000000000001,"5min":0.27100000000000002,"15min":0.249},"max":{"1min":5.3099999999999996,"5min":10.523,"15min":104.837},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.7270000000000001,"15min":2.585},"min":{"1min":0.374,"5min":0.27700000000000002,"15min":0.247},"max":{"1min":4.7400000000000002,"5min":10.026,"15min":106.379},"last":0.433}]},{"osd":3,"last update":"Sun Oct 1 09:33:51 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":3.3220000000000001,"15min":3.5720000000000001},"min":{"1min":0.32900000000000001,"5min":0.30199999999999999,"15min":0.28199999999999997},"max":{"1min":3.9649999999999999,"5min":80.352999999999994,"15min":164.595},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.4059999999999999,"5min":3.0019999999999998,"15min":3.3250000000000002},"min":{"1min":0.33200000000000002,"5min":0.313,"15min":0.29399999999999998},"max":{"1min":3.7349999999999999,"5min":80.364000000000004,"15min":164.60499999999999},"last":0.378}]},{"osd":4,"last update":"Sun Oct 1 09:34:34 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.046,"15min":0.98399999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.23499999999999999},"max":{"1min":5.3490000000000002,"5min":7.7039999999999997,"15min":9.2349999999999994},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.97799999999999998,"5min":1.075,"15min":0.98899999999999999},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":4.6230000000000002,"5min":8.0009999999999994,"15min":9.4339999999999993},"last":0.34399999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:34:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.4100000000000001,"5min":4.4100000000000001,"15min":4.4100000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.3839999999999999,"5min":3.3839999999999999,"15min":3.3839999999999999},"last":0.42399999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:34:04 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.91500000000000004,"15min":0.84299999999999997},"min":{"1min":0.38900000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.3490000000000002,"5min":7.75,"15min":7.75},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.88200000000000001,"15min":0.73399999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.5430000000000001,"5min":6.3860000000000001,"15min":6.3860000000000001},"last":0.40200000000000002}]}]},{"osd":6,"up_from":294,"seq":1262720385120,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899824,"kb_used_data":562216,"kb_used_omap":6,"kb_used_meta":337593,"kb_avail":92845328,"statfs":{"total":95995035648,"available":95073615872,"internally_reserved":0,"allocated":575709184,"data_stored":574834448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":345695277},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:34:01 2023","interfaces":[{"interface":"back","average":{"1min":4.6840000000000002,"5min":6.8559999999999999,"15min":3.6179999999999999},"min":{"1min":0.31900000000000001,"5min":0.23200000000000001,"15min":0.222},"max":{"1min":40.426000000000002,"5min":130.02699999999999,"15min":130.02699999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":3.3889999999999998,"5min":6.5449999999999999,"15min":3.2269999999999999},"min":{"1min":0.248,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":40.447000000000003,"5min":133.72900000000001,"15min":133.72900000000001},"last":0.29299999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":4.4450000000000003,"15min":2.718},"min":{"1min":0.28299999999999997,"5min":0.24199999999999999,"15min":0.23899999999999999},"max":{"1min":5.5039999999999996,"5min":97.561000000000007,"15min":97.561000000000007},"last":84.378},{"interface":"front","average":{"1min":1.284,"5min":4.3959999999999999,"15min":2.669},"min":{"1min":0.33900000000000002,"5min":0.25600000000000001,"15min":0.24399999999999999},"max":{"1min":5.4260000000000002,"5min":97.585999999999999,"15min":97.585999999999999},"last":84.567999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.7989999999999999,"15min":2.5419999999999998},"min":{"1min":0.28100000000000003,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":5.7679999999999998,"5min":10.92,"15min":26.289999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.645,"5min":1.8240000000000001,"15min":2.6859999999999999},"min":{"1min":0.309,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.9000000000000004,"5min":12.804,"15min":26.388999999999999},"last":0.41699999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:34:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0680000000000001,"5min":2.5649999999999999,"15min":2.5539999999999998},"min":{"1min":0.39600000000000002,"5min":0.316,"15min":0.316},"max":{"1min":8.0730000000000004,"5min":62.546999999999997,"15min":62.546999999999997},"last":0.318},{"interface":"front","average":{"1min":1.8,"5min":2.2799999999999998,"15min":1.79},"min":{"1min":0.37,"5min":0.32300000000000001,"15min":0.29299999999999998},"max":{"1min":6.6600000000000001,"5min":62.465000000000003,"15min":62.465000000000003},"last":0.35499999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":0.93799999999999994,"15min":0.57599999999999996},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.25700000000000001},"max":{"1min":6.0640000000000001,"5min":6.0640000000000001,"15min":6.0640000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.5820000000000001,"5min":1.077,"15min":0.74099999999999999},"min":{"1min":0.315,"5min":0.30099999999999999,"15min":0.28299999999999997},"max":{"1min":8.8249999999999993,"5min":9.4090000000000007,"15min":9.4090000000000007},"last":0.40200000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:34:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.629,"5min":3.629,"15min":3.629},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.131,"5min":1.131,"15min":1.131},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.5309999999999997,"5min":4.5309999999999997,"15min":4.5309999999999997},"last":0.46000000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":0.995,"15min":3.9399999999999999},"min":{"1min":0.31900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.3710000000000004,"5min":7.4589999999999996,"15min":115.11},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.069,"5min":0.78300000000000003,"15min":3.8260000000000001},"min":{"1min":0.308,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.8789999999999996,"5min":4.8789999999999996,"15min":114.666},"last":0.53600000000000003}]}]},{"osd":1,"up_from":94,"seq":403726926097,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2369048,"kb_used_data":1757200,"kb_used_omap":7,"kb_used_meta":611832,"kb_avail":91376104,"statfs":{"total":95995035648,"available":93569130496,"internally_reserved":0,"allocated":1799372800,"data_stored":1797646917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":626516391},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:34:42 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":4.1449999999999996,"15min":4.1589999999999998},"min":{"1min":0.24399999999999999,"5min":0.183,"15min":0.183},"max":{"1min":7.1210000000000004,"5min":82.359999999999999,"15min":134.84999999999999},"last":6.8339999999999996},{"interface":"front","average":{"1min":0.97399999999999998,"5min":4.0140000000000002,"15min":4.1520000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":4.9829999999999997,"5min":82.078999999999994,"15min":134.77099999999999},"last":1.413}]},{"osd":2,"last update":"Sun Oct 1 09:34:20 2023","interfaces":[{"interface":"back","average":{"1min":1.599,"5min":3.8399999999999999,"15min":3.681},"min":{"1min":0.27300000000000002,"5min":0.20499999999999999,"15min":0.182},"max":{"1min":9.5370000000000008,"5min":56.573,"15min":77.325000000000003},"last":13.836},{"interface":"front","average":{"1min":1.9450000000000001,"5min":3.9809999999999999,"15min":4.0860000000000003},"min":{"1min":0.28299999999999997,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":7.9699999999999998,"5min":55.930999999999997,"15min":80.361000000000004},"last":2.75}]},{"osd":3,"last update":"Sun Oct 1 09:33:57 2023","interfaces":[{"interface":"back","average":{"1min":3.5049999999999999,"5min":5.0359999999999996,"15min":4.6500000000000004},"min":{"1min":0.26800000000000002,"5min":0.245,"15min":0.224},"max":{"1min":13.281000000000001,"5min":200.56800000000001,"15min":200.56800000000001},"last":1.4530000000000001},{"interface":"front","average":{"1min":3.3780000000000001,"5min":5.2469999999999999,"15min":4.6550000000000002},"min":{"1min":0.30199999999999999,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":25.190999999999999,"5min":200.63800000000001,"15min":200.63800000000001},"last":7.0140000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:34:42 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.302,"15min":1.4339999999999999},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":3.3239999999999998,"5min":13.045,"15min":13.045},"last":0.312},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.9870000000000001,"15min":1.901},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.27500000000000002},"max":{"1min":6.9500000000000002,"5min":12.023999999999999,"15min":18.219000000000001},"last":6.9500000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:34:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.0190000000000001,"5min":4.0190000000000001,"15min":4.0190000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.0230000000000001,"5min":2.0230000000000001,"15min":2.0230000000000001},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":11.218999999999999,"5min":11.218999999999999,"15min":11.218999999999999},"last":6.9199999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:33:57 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":2.2789999999999999,"15min":2.8460000000000001},"min":{"1min":0.39500000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.9429999999999996,"5min":12.842000000000001,"15min":12.842000000000001},"last":1.554},{"interface":"front","average":{"1min":1.3360000000000001,"5min":1.7150000000000001,"15min":1.9179999999999999},"min":{"1min":0.375,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.2469999999999999,"5min":14.772,"15min":14.772},"last":0.82499999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:34:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.9390000000000001,"15min":1.7929999999999999},"min":{"1min":0.442,"5min":0.35899999999999999,"15min":0.316},"max":{"1min":3.3919999999999999,"5min":14.717000000000001,"15min":21.399000000000001},"last":1.845},{"interface":"front","average":{"1min":1.546,"5min":2.1539999999999999,"15min":2.008},"min":{"1min":0.45200000000000001,"5min":0.35399999999999998,"15min":0.30199999999999999},"max":{"1min":7.1529999999999996,"5min":11.177,"15min":22.318999999999999},"last":6.9820000000000002}]}]},{"osd":0,"up_from":89,"seq":382252089631,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1484236,"kb_used_data":811012,"kb_used_omap":8,"kb_used_meta":673207,"kb_avail":92260916,"statfs":{"total":95995035648,"available":94475177984,"internally_reserved":0,"allocated":830476288,"data_stored":828714266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":689364930},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:33:46 2023","interfaces":[{"interface":"back","average":{"1min":3.8290000000000002,"5min":3.0600000000000001,"15min":3.282},"min":{"1min":0.23200000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":30.263999999999999,"5min":61.173000000000002,"15min":187.41399999999999},"last":3.0960000000000001},{"interface":"front","average":{"1min":4.9489999999999998,"5min":4.1500000000000004,"15min":4.1929999999999996},"min":{"1min":0.30499999999999999,"5min":0.22500000000000001,"15min":0.193},"max":{"1min":30.248999999999999,"5min":68.587999999999994,"15min":187.78700000000001},"last":0.39100000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:34:20 2023","interfaces":[{"interface":"back","average":{"1min":2.6709999999999998,"5min":2.1600000000000001,"15min":2.1749999999999998},"min":{"1min":0.25600000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":13.494,"5min":16.033000000000001,"15min":39.048999999999999},"last":3.1829999999999998},{"interface":"front","average":{"1min":1.9199999999999999,"5min":2.3799999999999999,"15min":2.9769999999999999},"min":{"1min":0.30099999999999999,"5min":0.189,"15min":0.189},"max":{"1min":13.003,"5min":14.319000000000001,"15min":38.945999999999998},"last":0.433}]},{"osd":3,"last update":"Sun Oct 1 09:33:46 2023","interfaces":[{"interface":"back","average":{"1min":3.149,"5min":3.9359999999999999,"15min":4.0670000000000002},"min":{"1min":0.26600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":13.189,"5min":59.216000000000001,"15min":139.85300000000001},"last":3.2090000000000001},{"interface":"front","average":{"1min":3.8340000000000001,"5min":3.8460000000000001,"15min":4.5330000000000004},"min":{"1min":0.29299999999999998,"5min":0.26300000000000001,"15min":0.219},"max":{"1min":14.442,"5min":59.177999999999997,"15min":139.70099999999999},"last":4.5380000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":2.5369999999999999,"15min":2.1259999999999999},"min":{"1min":0.318,"5min":0.27600000000000002,"15min":0.27300000000000002},"max":{"1min":8.7569999999999997,"5min":62.206000000000003,"15min":62.206000000000003},"last":2.6440000000000001},{"interface":"front","average":{"1min":1.0209999999999999,"5min":2.3149999999999999,"15min":2.0299999999999998},"min":{"1min":0.38,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":4.0800000000000001,"5min":62.073999999999998,"15min":62.073999999999998},"last":2.5750000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:34:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7609999999999999,"5min":1.7609999999999999,"15min":1.7609999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":9.4779999999999998,"5min":9.4779999999999998,"15min":9.4779999999999998},"last":2.6269999999999998},{"interface":"front","average":{"1min":1.6319999999999999,"5min":1.6319999999999999,"15min":1.6319999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":7.2130000000000001,"5min":7.2130000000000001,"15min":7.2130000000000001},"last":3.2360000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:34:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7330000000000001,"5min":1.8819999999999999,"15min":1.591},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.36899999999999999},"max":{"1min":7.2869999999999999,"5min":15.904,"15min":15.904},"last":3.2589999999999999},{"interface":"front","average":{"1min":1.831,"5min":1.988,"15min":2.3980000000000001},"min":{"1min":0.41499999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.8089999999999993,"5min":14.215,"15min":18.521000000000001},"last":0.53000000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":2.085,"15min":1.5780000000000001},"min":{"1min":0.41999999999999998,"5min":0.38100000000000001,"15min":0.29199999999999998},"max":{"1min":7.3250000000000002,"5min":19.402999999999999,"15min":19.402999999999999},"last":3.2810000000000001},{"interface":"front","average":{"1min":1.367,"5min":2.1299999999999999,"15min":2.0710000000000002},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.312},"max":{"1min":8.8439999999999994,"5min":19.373000000000001,"15min":19.373000000000001},"last":0.45700000000000002}]}]},{"osd":2,"up_from":99,"seq":425201762571,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1290048,"kb_used_data":622904,"kb_used_omap":11,"kb_used_meta":667124,"kb_avail":92455104,"statfs":{"total":95995035648,"available":94674026496,"internally_reserved":0,"allocated":637853696,"data_stored":636159198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":683135863},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:34:36 2023","interfaces":[{"interface":"back","average":{"1min":10.503,"5min":9.6669999999999998,"15min":6.7149999999999999},"min":{"1min":0.22700000000000001,"5min":0.191,"15min":0.17599999999999999},"max":{"1min":95.597999999999999,"5min":224.56,"15min":224.56},"last":0.58099999999999996},{"interface":"front","average":{"1min":10.513999999999999,"5min":9.4510000000000005,"15min":7.1390000000000002},"min":{"1min":0.23999999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":95.558000000000007,"5min":225.66900000000001,"15min":225.66900000000001},"last":0.23699999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:34:26 2023","interfaces":[{"interface":"back","average":{"1min":5.5,"5min":9.5709999999999997,"15min":7.2000000000000002},"min":{"1min":0.218,"5min":0.191,"15min":0.191},"max":{"1min":92.584999999999994,"5min":224.917,"15min":224.917},"last":0.33200000000000002},{"interface":"front","average":{"1min":5.7809999999999997,"5min":9.8789999999999996,"15min":7.9770000000000003},"min":{"1min":0.28799999999999998,"5min":0.23599999999999999,"15min":0.20899999999999999},"max":{"1min":94.441000000000003,"5min":225.625,"15min":225.625},"last":0.55600000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:33:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1219999999999999,"5min":4.6970000000000001,"15min":6.0810000000000004},"min":{"1min":0.251,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":9.766,"5min":226.124,"15min":253.238},"last":0.34799999999999998},{"interface":"front","average":{"1min":2.097,"5min":4.8579999999999997,"15min":5.9199999999999999},"min":{"1min":0.28899999999999998,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":9.5969999999999995,"5min":226.32900000000001,"15min":252.54900000000001},"last":1.7370000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:34:26 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":3.6589999999999998,"15min":2.617},"min":{"1min":0.29999999999999999,"5min":0.28599999999999998,"15min":0.27200000000000002},"max":{"1min":5.5599999999999996,"5min":224.90199999999999,"15min":224.90199999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.90200000000000002,"5min":4.2050000000000001,"15min":2.8090000000000002},"min":{"1min":0.32500000000000001,"5min":0.309,"15min":0.27500000000000002},"max":{"1min":5.4240000000000004,"5min":225.386,"15min":225.386},"last":0.42099999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:34:11 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":1.661,"15min":1.661},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":12.500999999999999,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.73499999999999999},{"interface":"front","average":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.194000000000001,"5min":10.194000000000001,"15min":10.194000000000001},"last":0.72399999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:33:57 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":4.6440000000000001,"15min":2.9340000000000002},"min":{"1min":0.36399999999999999,"5min":0.317,"15min":0.317},"max":{"1min":6.3780000000000001,"5min":225.41499999999999,"15min":225.41499999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.1829999999999998,"5min":4.5960000000000001,"15min":2.4239999999999999},"min":{"1min":0.374,"5min":0.33700000000000002,"15min":0.33100000000000002},"max":{"1min":13.614000000000001,"5min":225.02000000000001,"15min":225.02000000000001},"last":0.628}]},{"osd":7,"last update":"Sun Oct 1 09:34:26 2023","interfaces":[{"interface":"back","average":{"1min":1.548,"5min":4.3639999999999999,"15min":2.8420000000000001},"min":{"1min":0.36799999999999999,"5min":0.31,"15min":0.308},"max":{"1min":12.544,"5min":224.94999999999999,"15min":224.94999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.89900000000000002,"5min":3.7869999999999999,"15min":2.8889999999999998},"min":{"1min":0.42599999999999999,"5min":0.37,"15min":0.33600000000000002},"max":{"1min":4.6660000000000004,"5min":224.92599999999999,"15min":224.92599999999999},"last":0.57099999999999995}]}]},{"osd":3,"up_from":104,"seq":446676599044,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2135260,"kb_used_data":1496276,"kb_used_omap":3,"kb_used_meta":638972,"kb_avail":91609892,"statfs":{"total":95995035648,"available":93808529408,"internally_reserved":0,"allocated":1532186624,"data_stored":1530398185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":654308071},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:33:51 2023","interfaces":[{"interface":"back","average":{"1min":4.6870000000000003,"5min":6.3239999999999998,"15min":5.0960000000000001},"min":{"1min":0.186,"5min":0.186,"15min":0.17100000000000001},"max":{"1min":24.023,"5min":109.446,"15min":128.47800000000001},"last":1.514},{"interface":"front","average":{"1min":2.391,"5min":6.0069999999999997,"15min":4.9109999999999996},"min":{"1min":0.224,"5min":0.224,"15min":0.184},"max":{"1min":11.784000000000001,"5min":109.304,"15min":127.15600000000001},"last":3.2650000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:33:51 2023","interfaces":[{"interface":"back","average":{"1min":3.1669999999999998,"5min":3.7829999999999999,"15min":4.6760000000000002},"min":{"1min":0.24299999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.782999999999999,"5min":68.962000000000003,"15min":228.60499999999999},"last":3.23},{"interface":"front","average":{"1min":5.391,"5min":4.173,"15min":5.125},"min":{"1min":0.27500000000000002,"5min":0.221,"15min":0.221},"max":{"1min":24.196000000000002,"5min":68.819999999999993,"15min":228.28399999999999},"last":0.46999999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:33:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7589999999999999,"5min":1.4710000000000001,"15min":3.2989999999999999},"min":{"1min":0.28899999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":4.609,"5min":7.0919999999999996,"15min":62.909999999999997},"last":0.41899999999999998},{"interface":"front","average":{"1min":4.0339999999999998,"5min":2.7719999999999998,"15min":4.1459999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.219},"max":{"1min":13.226000000000001,"5min":13.226000000000001,"15min":78.594999999999999},"last":0.622}]},{"osd":4,"last update":"Sun Oct 1 09:33:51 2023","interfaces":[{"interface":"back","average":{"1min":4.5830000000000002,"5min":2.093,"15min":2.8130000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":29.419,"5min":29.419,"15min":154.00399999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":4.0949999999999998,"5min":2.2210000000000001,"15min":2.9700000000000002},"min":{"1min":0.33600000000000002,"5min":0.309,"15min":0.23899999999999999},"max":{"1min":33.002000000000002,"5min":33.002000000000002,"15min":156.87700000000001},"last":1.1499999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:34:10 2023","interfaces":[{"interface":"back","average":{"1min":3.883,"5min":3.883,"15min":3.883},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":28.117999999999999,"5min":28.117999999999999,"15min":28.117999999999999},"last":1.1830000000000001},{"interface":"front","average":{"1min":3.7250000000000001,"5min":3.7250000000000001,"15min":3.7250000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":28.34,"5min":28.34,"15min":28.34},"last":0.38800000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:34:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":1.4179999999999999,"15min":2.6419999999999999},"min":{"1min":0.50800000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.4480000000000004,"5min":7.6929999999999996,"15min":18.361000000000001},"last":0.441},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.381,"15min":2.5800000000000001},"min":{"1min":0.497,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.5289999999999999,"5min":7.625,"15min":18.332000000000001},"last":0.48199999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:33:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2909999999999999,"5min":1.548,"15min":2.4329999999999998},"min":{"1min":0.48199999999999998,"5min":0.31,"15min":0.309},"max":{"1min":7.657,"5min":7.657,"15min":52.006},"last":0.46000000000000002},{"interface":"front","average":{"1min":3.3759999999999999,"5min":2.1099999999999999,"15min":2.4820000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":13.191000000000001,"5min":13.191000000000001,"15min":51.890999999999998},"last":0.498}]}]},{"osd":4,"up_from":30,"seq":128849019294,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1169720,"kb_used_data":580016,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":92575432,"statfs":{"total":95995035648,"available":94797242368,"internally_reserved":0,"allocated":593936384,"data_stored":593053599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:33:39 2023","interfaces":[{"interface":"back","average":{"1min":19.279,"5min":7.3949999999999996,"15min":4.8979999999999997},"min":{"1min":0.248,"5min":0.224,"15min":0.223},"max":{"1min":155.315,"5min":155.315,"15min":155.315},"last":1.5289999999999999},{"interface":"front","average":{"1min":20.324999999999999,"5min":8.1519999999999992,"15min":5.0229999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.20699999999999999},"max":{"1min":155.334,"5min":155.334,"15min":155.334},"last":0.93000000000000005}]},{"osd":1,"last update":"Sun Oct 1 09:33:46 2023","interfaces":[{"interface":"back","average":{"1min":14.359,"5min":7.665,"15min":5.3680000000000003},"min":{"1min":0.25,"5min":0.25,"15min":0.22600000000000001},"max":{"1min":190.61199999999999,"5min":190.61199999999999,"15min":190.61199999999999},"last":0.32200000000000001},{"interface":"front","average":{"1min":26.035,"5min":9.9540000000000006,"15min":6.1130000000000004},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":253.273,"5min":253.273,"15min":253.273},"last":0.46400000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:34:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":4.2430000000000003,"15min":3.2370000000000001},"min":{"1min":0.316,"5min":0.28299999999999997,"15min":0.26700000000000002},"max":{"1min":3.6099999999999999,"5min":170.45400000000001,"15min":170.45400000000001},"last":0.442},{"interface":"front","average":{"1min":1.2689999999999999,"5min":3.0630000000000002,"15min":2.806},"min":{"1min":0.33300000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.8179999999999996,"5min":58.771000000000001,"15min":91.510999999999996},"last":0.49099999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:33:46 2023","interfaces":[{"interface":"back","average":{"1min":2.673,"5min":2.3069999999999999,"15min":2.7610000000000001},"min":{"1min":0.38500000000000001,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":11.32,"5min":48.177,"15min":92.218000000000004},"last":2.504},{"interface":"front","average":{"1min":2.605,"5min":2.6070000000000002,"15min":3.004},"min":{"1min":0.317,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":23.231000000000002,"5min":48.518999999999998,"15min":92.766999999999996},"last":1.254}]},{"osd":5,"last update":"Sun Oct 1 09:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.085,"15min":1.085},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.367,"5min":5.367,"15min":5.367},"last":0.373},{"interface":"front","average":{"1min":0.75,"5min":0.75,"15min":0.75},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.3929999999999998,"5min":3.3929999999999998,"15min":3.3929999999999998},"last":0.73599999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:34:00 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.04,"15min":0.73899999999999999},"min":{"1min":0.32300000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.9710000000000001,"5min":11.358000000000001,"15min":11.358000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.5469999999999999,"5min":1.0329999999999999,"15min":0.73999999999999999},"min":{"1min":0.32500000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.5229999999999997,"5min":10.076000000000001,"15min":10.076000000000001},"last":0.40300000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:34:28 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":1.0660000000000001,"15min":0.91000000000000003},"min":{"1min":0.39100000000000001,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":3.4630000000000001,"5min":11.384,"15min":11.384},"last":1.3959999999999999},{"interface":"front","average":{"1min":1.028,"5min":1.244,"15min":1.0740000000000001},"min":{"1min":0.42099999999999999,"5min":0.308,"15min":0.28699999999999998},"max":{"1min":3.585,"5min":10.103999999999999,"15min":10.103999999999999},"last":0.78500000000000003}]}]},{"osd":5,"up_from":379,"seq":1627792605204,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2159088,"kb_used_data":1786344,"kb_used_omap":13,"kb_used_meta":372722,"kb_avail":91586064,"statfs":{"total":95995035648,"available":93784129536,"internally_reserved":0,"allocated":1829216256,"data_stored":1828345037,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":381667826},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:34:14 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.5049999999999999,"15min":1.5049999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.3550000000000004,"5min":8.3550000000000004,"15min":8.3550000000000004},"last":0.26500000000000001},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.2310000000000001,"15min":1.2310000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.0869999999999997,"5min":5.0869999999999997,"15min":5.0869999999999997},"last":1.696}]},{"osd":1,"last update":"Sun Oct 1 09:34:14 2023","interfaces":[{"interface":"back","average":{"1min":1.698,"5min":1.698,"15min":1.698},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.773999999999999,"5min":10.773999999999999,"15min":10.773999999999999},"last":1.5349999999999999},{"interface":"front","average":{"1min":1.395,"5min":1.395,"15min":1.395},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.3239999999999998,"5min":9.3239999999999998,"15min":9.3239999999999998},"last":0.32900000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:34:14 2023","interfaces":[{"interface":"back","average":{"1min":1.304,"5min":1.304,"15min":1.304},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.6790000000000003,"5min":4.6790000000000003,"15min":4.6790000000000003},"last":1.657},{"interface":"front","average":{"1min":1.6220000000000001,"5min":1.6220000000000001,"15min":1.6220000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":11.221,"5min":11.221,"15min":11.221},"last":1.5509999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:34:14 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.1619999999999999,"15min":1.1619999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":3.6269999999999998,"5min":3.6269999999999998,"15min":3.6269999999999998},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.1779999999999999,"5min":1.1779999999999999,"15min":1.1779999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":4.4950000000000001,"5min":4.4950000000000001,"15min":4.4950000000000001},"last":1.734}]},{"osd":4,"last update":"Sun Oct 1 09:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":4.5279999999999996,"5min":4.5279999999999996,"15min":4.5279999999999996},"last":1.8089999999999999},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.6659999999999999,"15min":1.6659999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":14.68,"5min":14.68,"15min":14.68},"last":1.466}]},{"osd":6,"last update":"Sun Oct 1 09:34:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":3.7170000000000001,"5min":3.7170000000000001,"15min":3.7170000000000001},"last":0.84199999999999997},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":12.699999999999999,"5min":12.699999999999999,"15min":12.699999999999999},"last":1.6040000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:34:14 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":1.369,"15min":1.369},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":10.813000000000001,"5min":10.813000000000001,"15min":10.813000000000001},"last":1.5089999999999999},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":3.6000000000000001,"5min":3.6000000000000001,"15min":3.6000000000000001},"last":0.35299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":822214656,"data_stored":822214656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1818755072,"data_stored":1818755072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":636485632,"data_stored":636485632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1536557056,"data_stored":1536557056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":590938112,"data_stored":590938112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1859321856,"data_stored":1859321856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":575537152,"data_stored":575537152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1727922176,"data_stored":1727922176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:34:48.029 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:34:48.066 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:48.132 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:48.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:48.137 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:48 smithi190 ceph-mon[160435]: from='client.46103 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:34:48.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:48.238 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:48.263 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:48.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:48 smithi138 ceph-mon[192513]: from='client.46103 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:34:48.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:48 smithi138 ceph-mon[185886]: from='client.46103 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:34:48.341 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:48.533 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:48.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:48.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:48.639 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:48.683 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:48.755 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 32 16 55403 55387 108.135 80.875 0.00772165 0.009243 2023-10-01T09:34:48.758 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:48.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:48.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:48.866 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:34:48.941 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:49.109 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:34:49.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:49.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:34:49.221 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:34:49.275 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:49 smithi138 ceph-mon[192513]: pgmap v1133: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-01T09:34:49.277 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:49 smithi138 ceph-mon[185886]: pgmap v1133: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-01T09:34:49.288 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:49.368 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:34:49.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:49.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:34:49.418 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:49 smithi190 ceph-mon[160435]: pgmap v1133: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-01T09:34:49.474 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:49.524 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:49.750 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:49.755 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 33 16 56818 56802 107.538 88.4375 0.0108593 0.0092945 2023-10-01T09:34:49.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:49.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:49.860 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:49.867 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:49.914 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:49.957 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:34:49.997 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:50.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:50.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:50.102 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:50.124 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:34:50.124 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:34:50.238 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:50.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:50 smithi138 ceph-mon[192513]: pgmap v1134: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-01T09:34:50.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:50 smithi138 ceph-mon[185886]: pgmap v1134: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-01T09:34:50.459 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:50.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:50.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:50.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:50 smithi190 ceph-mon[160435]: pgmap v1134: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-01T09:34:50.564 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:50.639 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:50.755 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 34 16 58009 57993 106.564 74.4375 0.00738738 0.00936719 2023-10-01T09:34:50.803 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:50.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:50.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:50.906 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:50.961 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:51.033 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:51.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:51.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:51.136 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:51.162 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:51.203 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:34:51.204 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:34:51.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:51.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:34:51.307 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:51.364 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:51.453 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:51.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:51.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:51.559 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:51.639 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:51.737 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:51.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:51.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:51.756 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 35 16 58941 58925 105.184 58.25 0.0110139 0.00950157 2023-10-01T09:34:51.845 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:51.874 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:34:51.893 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:51.975 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:51.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:51.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:52.016 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:52.084 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:52.129 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:52.172 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:52 smithi138 ceph-mon[185886]: pgmap v1135: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-01T09:34:52.172 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:52 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2850932468' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:52.225 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:52.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:52.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:52.331 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:52.401 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:52.431 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:52 smithi138 ceph-mon[192513]: pgmap v1135: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-01T09:34:52.432 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:52 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2850932468' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:52.503 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:52.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:52.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:52.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:52 smithi190 ceph-mon[160435]: pgmap v1135: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-01T09:34:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:52 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2850932468' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:34:52.609 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:52.671 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:52.756 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 36 16 60055 60039 104.197 69.625 0.031852 0.00959168 2023-10-01T09:34:52.867 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:34:52.871 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:34:52.915 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:52.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:52.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:53.022 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:34:53.068 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:53.180 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:34:53.499 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:53 smithi138 ceph-mon[185886]: from='client.55954 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:34:53.499 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:53 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:34:53.499 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:53 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:34:53.499 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2850932468' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:34:53.499 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:53 smithi138 ceph-mon[185886]: osdmap e410: 8 total, 8 up, 8 in 2023-10-01T09:34:53.748 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:53 smithi190 ceph-mon[160435]: from='client.55954 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:34:53.748 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:53 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:34:53.748 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:53 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:34:53.748 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2850932468' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:34:53.748 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:53 smithi190 ceph-mon[160435]: osdmap e410: 8 total, 8 up, 8 in 2023-10-01T09:34:53.755 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 37 16 61116 61100 103.173 66.3125 0.0056236 0.00968712 2023-10-01T09:34:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:53 smithi138 ceph-mon[192513]: from='client.55954 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:34:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:53 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:34:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:53 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:34:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2850932468' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:34:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:53 smithi138 ceph-mon[192513]: osdmap e410: 8 total, 8 up, 8 in 2023-10-01T09:34:54.074 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:34:54.075 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:34:54.756 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 38 16 62111 62095 102.094 62.1875 0.0779731 0.00977821 2023-10-01T09:34:54.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:54 smithi190 ceph-mon[160435]: failed to encode map e410 with expected crc 2023-10-01T09:34:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:54 smithi190 ceph-mon[160435]: failed to encode map e410 with expected crc 2023-10-01T09:34:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:54 smithi190 ceph-mon[160435]: failed to encode map e410 with expected crc 2023-10-01T09:34:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:54 smithi190 ceph-mon[160435]: pgmap v1137: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T09:34:54.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:54 smithi138 ceph-mon[192513]: failed to encode map e410 with expected crc 2023-10-01T09:34:54.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:54 smithi138 ceph-mon[192513]: failed to encode map e410 with expected crc 2023-10-01T09:34:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:54 smithi138 ceph-mon[192513]: failed to encode map e410 with expected crc 2023-10-01T09:34:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:54 smithi138 ceph-mon[192513]: pgmap v1137: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T09:34:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:54 smithi138 ceph-mon[185886]: failed to encode map e410 with expected crc 2023-10-01T09:34:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:54 smithi138 ceph-mon[185886]: failed to encode map e410 with expected crc 2023-10-01T09:34:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:54 smithi138 ceph-mon[185886]: failed to encode map e410 with expected crc 2023-10-01T09:34:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:54 smithi138 ceph-mon[185886]: pgmap v1137: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T09:34:55.048 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:34:55.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:55.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:34:55.160 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:34:55.175 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:34:55.198 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1135,"stamp":"2023-10-01T09:34:52.085250+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3729004467,"num_objects":56855,"num_object_clones":0,"num_object_copies":170565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":56855,"num_whiteouts":0,"num_read":6259,"num_read_kb":12659,"num_write":57933,"num_write_kb":3652069,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":350,"num_bytes_recovered":3575563,"num_keys_recovered":70,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":57997,"ondisk_log_size":57997,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15419840,"kb_used_data":10915072,"kb_used_omap":50,"kb_used_meta":4504653,"kb_avail":734541376,"statfs":{"total":767960285184,"available":752170369024,"internally_reserved":0,"allocated":11177033728,"data_stored":11166542453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51738,"internal_metadata":4612765158},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":804126720,"num_objects":12270,"num_object_clones":0,"num_object_copies":36810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12270,"num_write_kb":785280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002852"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":302,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.506102+0000","last_change":"2023-10-01T09:32:27.452345+0000","last_active":"2023-10-01T09:34:39.506102+0000","last_peered":"2023-10-01T09:34:39.506102+0000","last_clean":"2023-10-01T09:34:39.506102+0000","last_became_active":"2023-10-01T09:32:27.452033+0000","last_became_peered":"2023-10-01T09:32:27.452033+0000","last_unstale":"2023-10-01T09:34:39.506102+0000","last_undegraded":"2023-10-01T09:34:39.506102+0000","last_fullsized":"2023-10-01T09:34:39.506102+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:44:10.031691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":223,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513784+0000","last_change":"2023-10-01T09:32:27.450725+0000","last_active":"2023-10-01T09:34:39.513784+0000","last_peered":"2023-10-01T09:34:39.513784+0000","last_clean":"2023-10-01T09:34:39.513784+0000","last_became_active":"2023-10-01T09:32:27.450419+0000","last_became_peered":"2023-10-01T09:32:27.450419+0000","last_unstale":"2023-10-01T09:34:39.513784+0000","last_undegraded":"2023-10-01T09:34:39.513784+0000","last_fullsized":"2023-10-01T09:34:39.513784+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:55:11.692362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":387,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504285+0000","last_change":"2023-10-01T09:32:27.445629+0000","last_active":"2023-10-01T09:34:39.504285+0000","last_peered":"2023-10-01T09:34:39.504285+0000","last_clean":"2023-10-01T09:34:39.504285+0000","last_became_active":"2023-10-01T09:32:27.445365+0000","last_became_peered":"2023-10-01T09:32:27.445365+0000","last_unstale":"2023-10-01T09:34:39.504285+0000","last_undegraded":"2023-10-01T09:34:39.504285+0000","last_fullsized":"2023-10-01T09:34:39.504285+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:13:21.018961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":426,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513757+0000","last_change":"2023-10-01T09:29:58.372577+0000","last_active":"2023-10-01T09:34:39.513757+0000","last_peered":"2023-10-01T09:34:39.513757+0000","last_clean":"2023-10-01T09:34:39.513757+0000","last_became_active":"2023-10-01T09:29:58.372218+0000","last_became_peered":"2023-10-01T09:29:58.372218+0000","last_unstale":"2023-10-01T09:34:39.513757+0000","last_undegraded":"2023-10-01T09:34:39.513757+0000","last_fullsized":"2023-10-01T09:34:39.513757+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:04:08.376752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":299,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513225+0000","last_change":"2023-10-01T09:30:39.394083+0000","last_active":"2023-10-01T09:34:39.513225+0000","last_peered":"2023-10-01T09:34:39.513225+0000","last_clean":"2023-10-01T09:34:39.513225+0000","last_became_active":"2023-10-01T09:30:39.393797+0000","last_became_peered":"2023-10-01T09:30:39.393797+0000","last_unstale":"2023-10-01T09:34:39.513225+0000","last_undegraded":"2023-10-01T09:34:39.513225+0000","last_fullsized":"2023-10-01T09:34:39.513225+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:22.625624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":128,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.503130+0000","last_change":"2023-10-01T09:33:11.511556+0000","last_active":"2023-10-01T09:34:39.503130+0000","last_peered":"2023-10-01T09:34:39.503130+0000","last_clean":"2023-10-01T09:34:39.503130+0000","last_became_active":"2023-10-01T09:33:11.511318+0000","last_became_peered":"2023-10-01T09:33:11.511318+0000","last_unstale":"2023-10-01T09:34:39.503130+0000","last_undegraded":"2023-10-01T09:34:39.503130+0000","last_fullsized":"2023-10-01T09:34:39.503130+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":462,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513380+0000","last_change":"2023-10-01T09:33:11.508780+0000","last_active":"2023-10-01T09:34:39.513380+0000","last_peered":"2023-10-01T09:34:39.513380+0000","last_clean":"2023-10-01T09:34:39.513380+0000","last_became_active":"2023-10-01T09:33:11.508451+0000","last_became_peered":"2023-10-01T09:33:11.508451+0000","last_unstale":"2023-10-01T09:34:39.513380+0000","last_undegraded":"2023-10-01T09:34:39.513380+0000","last_fullsized":"2023-10-01T09:34:39.513380+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:55:02.323934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":410,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.507878+0000","last_change":"2023-10-01T09:30:39.395270+0000","last_active":"2023-10-01T09:34:39.507878+0000","last_peered":"2023-10-01T09:34:39.507878+0000","last_clean":"2023-10-01T09:34:39.507878+0000","last_became_active":"2023-10-01T09:30:39.394888+0000","last_became_peered":"2023-10-01T09:30:39.394888+0000","last_unstale":"2023-10-01T09:34:39.507878+0000","last_undegraded":"2023-10-01T09:34:39.507878+0000","last_fullsized":"2023-10-01T09:34:39.507878+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:35:40.846102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":94,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.512173+0000","last_change":"2023-10-01T09:33:11.667007+0000","last_active":"2023-10-01T09:34:39.512173+0000","last_peered":"2023-10-01T09:34:39.512173+0000","last_clean":"2023-10-01T09:34:39.512173+0000","last_became_active":"2023-10-01T09:33:11.666693+0000","last_became_peered":"2023-10-01T09:33:11.666693+0000","last_unstale":"2023-10-01T09:34:39.512173+0000","last_undegraded":"2023-10-01T09:34:39.512173+0000","last_fullsized":"2023-10-01T09:34:39.512173+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":426,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.508922+0000","last_change":"2023-10-01T09:30:39.398262+0000","last_active":"2023-10-01T09:34:39.508922+0000","last_peered":"2023-10-01T09:34:39.508922+0000","last_clean":"2023-10-01T09:34:39.508922+0000","last_became_active":"2023-10-01T09:30:39.397934+0000","last_became_peered":"2023-10-01T09:30:39.397934+0000","last_unstale":"2023-10-01T09:34:39.508922+0000","last_undegraded":"2023-10-01T09:34:39.508922+0000","last_fullsized":"2023-10-01T09:34:39.508922+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":208,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.509229+0000","last_change":"2023-10-01T09:32:27.492859+0000","last_active":"2023-10-01T09:34:39.509229+0000","last_peered":"2023-10-01T09:34:39.509229+0000","last_clean":"2023-10-01T09:34:39.509229+0000","last_became_active":"2023-10-01T09:32:27.454334+0000","last_became_peered":"2023-10-01T09:32:27.454334+0000","last_unstale":"2023-10-01T09:34:39.509229+0000","last_undegraded":"2023-10-01T09:34:39.509229+0000","last_fullsized":"2023-10-01T09:34:39.509229+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":438,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.505793+0000","last_change":"2023-10-01T09:28:02.342661+0000","last_active":"2023-10-01T09:34:39.505793+0000","last_peered":"2023-10-01T09:34:39.505793+0000","last_clean":"2023-10-01T09:34:39.505793+0000","last_became_active":"2023-10-01T09:28:02.342358+0000","last_became_peered":"2023-10-01T09:28:02.342358+0000","last_unstale":"2023-10-01T09:34:39.505793+0000","last_undegraded":"2023-10-01T09:34:39.505793+0000","last_fullsized":"2023-10-01T09:34:39.505793+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:29:30.624568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":436,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.509408+0000","last_change":"2023-10-01T09:32:27.498041+0000","last_active":"2023-10-01T09:34:39.509408+0000","last_peered":"2023-10-01T09:34:39.509408+0000","last_clean":"2023-10-01T09:34:39.509408+0000","last_became_active":"2023-10-01T09:32:27.454394+0000","last_became_peered":"2023-10-01T09:32:27.454394+0000","last_unstale":"2023-10-01T09:34:39.509408+0000","last_undegraded":"2023-10-01T09:34:39.509408+0000","last_fullsized":"2023-10-01T09:34:39.509408+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":122,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.503063+0000","last_change":"2023-10-01T09:32:27.454453+0000","last_active":"2023-10-01T09:34:39.503063+0000","last_peered":"2023-10-01T09:34:39.503063+0000","last_clean":"2023-10-01T09:34:39.503063+0000","last_became_active":"2023-10-01T09:32:27.454169+0000","last_became_peered":"2023-10-01T09:32:27.454169+0000","last_unstale":"2023-10-01T09:34:39.503063+0000","last_undegraded":"2023-10-01T09:34:39.503063+0000","last_fullsized":"2023-10-01T09:34:39.503063+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":482,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.510038+0000","last_change":"2023-10-01T09:32:27.592277+0000","last_active":"2023-10-01T09:34:39.510038+0000","last_peered":"2023-10-01T09:34:39.510038+0000","last_clean":"2023-10-01T09:34:39.510038+0000","last_became_active":"2023-10-01T09:32:27.467110+0000","last_became_peered":"2023-10-01T09:32:27.467110+0000","last_unstale":"2023-10-01T09:34:39.510038+0000","last_undegraded":"2023-10-01T09:34:39.510038+0000","last_fullsized":"2023-10-01T09:34:39.510038+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":237,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.509486+0000","last_change":"2023-10-01T09:30:39.399483+0000","last_active":"2023-10-01T09:34:39.509486+0000","last_peered":"2023-10-01T09:34:39.509486+0000","last_clean":"2023-10-01T09:34:39.509486+0000","last_became_active":"2023-10-01T09:30:39.399266+0000","last_became_peered":"2023-10-01T09:30:39.399266+0000","last_unstale":"2023-10-01T09:34:39.509486+0000","last_undegraded":"2023-10-01T09:34:39.509486+0000","last_fullsized":"2023-10-01T09:34:39.509486+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":401,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513042+0000","last_change":"2023-10-01T09:23:10.607388+0000","last_active":"2023-10-01T09:34:39.513042+0000","last_peered":"2023-10-01T09:34:39.513042+0000","last_clean":"2023-10-01T09:34:39.513042+0000","last_became_active":"2023-10-01T09:23:10.607084+0000","last_became_peered":"2023-10-01T09:23:10.607084+0000","last_unstale":"2023-10-01T09:34:39.513042+0000","last_undegraded":"2023-10-01T09:34:39.513042+0000","last_fullsized":"2023-10-01T09:34:39.513042+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:19.669393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":94,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511945+0000","last_change":"2023-10-01T09:33:11.663325+0000","last_active":"2023-10-01T09:34:39.511945+0000","last_peered":"2023-10-01T09:34:39.511945+0000","last_clean":"2023-10-01T09:34:39.511945+0000","last_became_active":"2023-10-01T09:33:11.662818+0000","last_became_peered":"2023-10-01T09:33:11.662818+0000","last_unstale":"2023-10-01T09:34:39.511945+0000","last_undegraded":"2023-10-01T09:34:39.511945+0000","last_fullsized":"2023-10-01T09:34:39.511945+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":420,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.508767+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:34:39.508767+0000","last_peered":"2023-10-01T09:34:39.508767+0000","last_clean":"2023-10-01T09:34:39.508767+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:34:39.508767+0000","last_undegraded":"2023-10-01T09:34:39.508767+0000","last_fullsized":"2023-10-01T09:34:39.508767+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":382,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513829+0000","last_change":"2023-10-01T09:23:10.611400+0000","last_active":"2023-10-01T09:34:39.513829+0000","last_peered":"2023-10-01T09:34:39.513829+0000","last_clean":"2023-10-01T09:34:39.513829+0000","last_became_active":"2023-10-01T09:23:10.610928+0000","last_became_peered":"2023-10-01T09:23:10.610928+0000","last_unstale":"2023-10-01T09:34:39.513829+0000","last_undegraded":"2023-10-01T09:34:39.513829+0000","last_fullsized":"2023-10-01T09:34:39.513829+0000","mapping_epoch":246,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":247,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:17.141540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":165,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.502542+0000","last_change":"2023-10-01T09:33:11.658471+0000","last_active":"2023-10-01T09:34:39.502542+0000","last_peered":"2023-10-01T09:34:39.502542+0000","last_clean":"2023-10-01T09:34:39.502542+0000","last_became_active":"2023-10-01T09:33:11.657960+0000","last_became_peered":"2023-10-01T09:33:11.657960+0000","last_unstale":"2023-10-01T09:34:39.502542+0000","last_undegraded":"2023-10-01T09:34:39.502542+0000","last_fullsized":"2023-10-01T09:34:39.502542+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":98,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.508733+0000","last_change":"2023-10-01T09:30:39.398326+0000","last_active":"2023-10-01T09:34:39.508733+0000","last_peered":"2023-10-01T09:34:39.508733+0000","last_clean":"2023-10-01T09:34:39.508733+0000","last_became_active":"2023-10-01T09:30:39.398021+0000","last_became_peered":"2023-10-01T09:30:39.398021+0000","last_unstale":"2023-10-01T09:34:39.508733+0000","last_undegraded":"2023-10-01T09:34:39.508733+0000","last_fullsized":"2023-10-01T09:34:39.508733+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.5","version":"409'3521","reported_seq":3539,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:48.917864+0000","last_change":"2023-10-01T09:34:06.140161+0000","last_active":"2023-10-01T09:34:48.917864+0000","last_peered":"2023-10-01T09:34:48.917864+0000","last_clean":"2023-10-01T09:34:48.917864+0000","last_became_active":"2023-10-01T09:34:06.139856+0000","last_became_peered":"2023-10-01T09:34:06.139856+0000","last_unstale":"2023-10-01T09:34:48.917864+0000","last_undegraded":"2023-10-01T09:34:48.917864+0000","last_fullsized":"2023-10-01T09:34:48.917864+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3521,"log_dups_size":0,"ondisk_log_size":3521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:58:29.822074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230752256,"num_objects":3521,"num_object_clones":0,"num_object_copies":10563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3521,"num_write_kb":225344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":202,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.508817+0000","last_change":"2023-10-01T09:32:27.502868+0000","last_active":"2023-10-01T09:34:39.508817+0000","last_peered":"2023-10-01T09:34:39.508817+0000","last_clean":"2023-10-01T09:34:39.508817+0000","last_became_active":"2023-10-01T09:32:27.455662+0000","last_became_peered":"2023-10-01T09:32:27.455662+0000","last_unstale":"2023-10-01T09:34:39.508817+0000","last_undegraded":"2023-10-01T09:34:39.508817+0000","last_fullsized":"2023-10-01T09:34:39.508817+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":111,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511596+0000","last_change":"2023-10-01T09:33:11.656404+0000","last_active":"2023-10-01T09:34:39.511596+0000","last_peered":"2023-10-01T09:34:39.511596+0000","last_clean":"2023-10-01T09:34:39.511596+0000","last_became_active":"2023-10-01T09:33:11.656093+0000","last_became_peered":"2023-10-01T09:33:11.656093+0000","last_unstale":"2023-10-01T09:34:39.511596+0000","last_undegraded":"2023-10-01T09:34:39.511596+0000","last_fullsized":"2023-10-01T09:34:39.511596+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"409'3497","reported_seq":3514,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:48.683108+0000","last_change":"2023-10-01T09:34:06.168002+0000","last_active":"2023-10-01T09:34:48.683108+0000","last_peered":"2023-10-01T09:34:48.683108+0000","last_clean":"2023-10-01T09:34:48.683108+0000","last_became_active":"2023-10-01T09:34:06.167679+0000","last_became_peered":"2023-10-01T09:34:06.167679+0000","last_unstale":"2023-10-01T09:34:48.683108+0000","last_undegraded":"2023-10-01T09:34:48.683108+0000","last_fullsized":"2023-10-01T09:34:48.683108+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3497,"log_dups_size":0,"ondisk_log_size":3497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:21:42.118174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229179392,"num_objects":3497,"num_object_clones":0,"num_object_copies":10491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3497,"num_write_kb":223808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":403,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.509660+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:34:39.509660+0000","last_peered":"2023-10-01T09:34:39.509660+0000","last_clean":"2023-10-01T09:34:39.509660+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:34:39.509660+0000","last_undegraded":"2023-10-01T09:34:39.509660+0000","last_fullsized":"2023-10-01T09:34:39.509660+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":442,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504161+0000","last_change":"2023-10-01T09:32:27.445399+0000","last_active":"2023-10-01T09:34:39.504161+0000","last_peered":"2023-10-01T09:34:39.504161+0000","last_clean":"2023-10-01T09:34:39.504161+0000","last_became_active":"2023-10-01T09:32:27.445067+0000","last_became_peered":"2023-10-01T09:32:27.445067+0000","last_unstale":"2023-10-01T09:34:39.504161+0000","last_undegraded":"2023-10-01T09:34:39.504161+0000","last_fullsized":"2023-10-01T09:34:39.504161+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:22:23.844066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"409'3515","reported_seq":3532,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:49.550902+0000","last_change":"2023-10-01T09:34:06.141263+0000","last_active":"2023-10-01T09:34:49.550902+0000","last_peered":"2023-10-01T09:34:49.550902+0000","last_clean":"2023-10-01T09:34:49.550902+0000","last_became_active":"2023-10-01T09:34:06.140720+0000","last_became_peered":"2023-10-01T09:34:06.140720+0000","last_unstale":"2023-10-01T09:34:49.550902+0000","last_undegraded":"2023-10-01T09:34:49.550902+0000","last_fullsized":"2023-10-01T09:34:49.550902+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3515,"log_dups_size":0,"ondisk_log_size":3515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230359040,"num_objects":3515,"num_object_clones":0,"num_object_copies":10545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3515,"num_write_kb":224960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":489,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.509252+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:34:39.509252+0000","last_peered":"2023-10-01T09:34:39.509252+0000","last_clean":"2023-10-01T09:34:39.509252+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:34:39.509252+0000","last_undegraded":"2023-10-01T09:34:39.509252+0000","last_fullsized":"2023-10-01T09:34:39.509252+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":112,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.503061+0000","last_change":"2023-10-01T09:28:02.361663+0000","last_active":"2023-10-01T09:34:39.503061+0000","last_peered":"2023-10-01T09:34:39.503061+0000","last_clean":"2023-10-01T09:34:39.503061+0000","last_became_active":"2023-10-01T09:28:02.361093+0000","last_became_peered":"2023-10-01T09:28:02.361093+0000","last_unstale":"2023-10-01T09:34:39.503061+0000","last_undegraded":"2023-10-01T09:34:39.503061+0000","last_fullsized":"2023-10-01T09:34:39.503061+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"409'3461","reported_seq":3478,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:49.558182+0000","last_change":"2023-10-01T09:34:06.140196+0000","last_active":"2023-10-01T09:34:49.558182+0000","last_peered":"2023-10-01T09:34:49.558182+0000","last_clean":"2023-10-01T09:34:49.558182+0000","last_became_active":"2023-10-01T09:34:06.139888+0000","last_became_peered":"2023-10-01T09:34:06.139888+0000","last_unstale":"2023-10-01T09:34:49.558182+0000","last_undegraded":"2023-10-01T09:34:49.558182+0000","last_fullsized":"2023-10-01T09:34:49.558182+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3461,"log_dups_size":0,"ondisk_log_size":3461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226820096,"num_objects":3461,"num_object_clones":0,"num_object_copies":10383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3461,"num_write_kb":221504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":111,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511787+0000","last_change":"2023-10-01T09:33:11.656845+0000","last_active":"2023-10-01T09:34:39.511787+0000","last_peered":"2023-10-01T09:34:39.511787+0000","last_clean":"2023-10-01T09:34:39.511787+0000","last_became_active":"2023-10-01T09:33:11.655365+0000","last_became_peered":"2023-10-01T09:33:11.655365+0000","last_unstale":"2023-10-01T09:34:39.511787+0000","last_undegraded":"2023-10-01T09:34:39.511787+0000","last_fullsized":"2023-10-01T09:34:39.511787+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":383,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513060+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:34:39.513060+0000","last_peered":"2023-10-01T09:34:39.513060+0000","last_clean":"2023-10-01T09:34:39.513060+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:34:39.513060+0000","last_undegraded":"2023-10-01T09:34:39.513060+0000","last_fullsized":"2023-10-01T09:34:39.513060+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"409'3594","reported_seq":3611,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:51.765198+0000","last_change":"2023-10-01T09:34:06.135421+0000","last_active":"2023-10-01T09:34:51.765198+0000","last_peered":"2023-10-01T09:34:51.765198+0000","last_clean":"2023-10-01T09:34:51.765198+0000","last_became_active":"2023-10-01T09:34:06.135086+0000","last_became_peered":"2023-10-01T09:34:06.135086+0000","last_unstale":"2023-10-01T09:34:51.765198+0000","last_undegraded":"2023-10-01T09:34:51.765198+0000","last_fullsized":"2023-10-01T09:34:51.765198+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3594,"log_dups_size":0,"ondisk_log_size":3594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235536384,"num_objects":3594,"num_object_clones":0,"num_object_copies":10782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3594,"num_write_kb":230016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":110,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513160+0000","last_change":"2023-10-01T09:32:27.489306+0000","last_active":"2023-10-01T09:34:39.513160+0000","last_peered":"2023-10-01T09:34:39.513160+0000","last_clean":"2023-10-01T09:34:39.513160+0000","last_became_active":"2023-10-01T09:32:27.449850+0000","last_became_peered":"2023-10-01T09:32:27.449850+0000","last_unstale":"2023-10-01T09:34:39.513160+0000","last_undegraded":"2023-10-01T09:34:39.513160+0000","last_fullsized":"2023-10-01T09:34:39.513160+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:26:15.712006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":58,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.519778+0000","last_change":"2023-10-01T09:32:27.449970+0000","last_active":"2023-10-01T09:34:39.519778+0000","last_peered":"2023-10-01T09:34:39.519778+0000","last_clean":"2023-10-01T09:34:39.519778+0000","last_became_active":"2023-10-01T09:32:27.449665+0000","last_became_peered":"2023-10-01T09:32:27.449665+0000","last_unstale":"2023-10-01T09:34:39.519778+0000","last_undegraded":"2023-10-01T09:34:39.519778+0000","last_fullsized":"2023-10-01T09:34:39.519778+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"409'3472","reported_seq":3490,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:48.672599+0000","last_change":"2023-10-01T09:34:06.135399+0000","last_active":"2023-10-01T09:34:48.672599+0000","last_peered":"2023-10-01T09:34:48.672599+0000","last_clean":"2023-10-01T09:34:48.672599+0000","last_became_active":"2023-10-01T09:34:06.135073+0000","last_became_peered":"2023-10-01T09:34:06.135073+0000","last_unstale":"2023-10-01T09:34:48.672599+0000","last_undegraded":"2023-10-01T09:34:48.672599+0000","last_fullsized":"2023-10-01T09:34:48.672599+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3472,"log_dups_size":0,"ondisk_log_size":3472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:40:11.547974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227540992,"num_objects":3472,"num_object_clones":0,"num_object_copies":10416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3472,"num_write_kb":222208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":392,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.505930+0000","last_change":"2023-10-01T09:32:27.447519+0000","last_active":"2023-10-01T09:34:39.505930+0000","last_peered":"2023-10-01T09:34:39.505930+0000","last_clean":"2023-10-01T09:34:39.505930+0000","last_became_active":"2023-10-01T09:32:27.447142+0000","last_became_peered":"2023-10-01T09:32:27.447142+0000","last_unstale":"2023-10-01T09:34:39.505930+0000","last_undegraded":"2023-10-01T09:34:39.505930+0000","last_fullsized":"2023-10-01T09:34:39.505930+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:28:48.014433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":436,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504208+0000","last_change":"2023-10-01T09:32:27.452641+0000","last_active":"2023-10-01T09:34:39.504208+0000","last_peered":"2023-10-01T09:34:39.504208+0000","last_clean":"2023-10-01T09:34:39.504208+0000","last_became_active":"2023-10-01T09:32:27.452411+0000","last_became_peered":"2023-10-01T09:32:27.452411+0000","last_unstale":"2023-10-01T09:34:39.504208+0000","last_undegraded":"2023-10-01T09:34:39.504208+0000","last_fullsized":"2023-10-01T09:34:39.504208+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:47:43.031520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"409'3697","reported_seq":3714,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:51.746750+0000","last_change":"2023-10-01T09:34:06.128656+0000","last_active":"2023-10-01T09:34:51.746750+0000","last_peered":"2023-10-01T09:34:51.746750+0000","last_clean":"2023-10-01T09:34:51.746750+0000","last_became_active":"2023-10-01T09:34:06.128364+0000","last_became_peered":"2023-10-01T09:34:06.128364+0000","last_unstale":"2023-10-01T09:34:51.746750+0000","last_undegraded":"2023-10-01T09:34:51.746750+0000","last_fullsized":"2023-10-01T09:34:51.746750+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3697,"log_dups_size":0,"ondisk_log_size":3697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242286592,"num_objects":3697,"num_object_clones":0,"num_object_copies":11091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3697,"num_write_kb":236608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":281,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.508850+0000","last_change":"2023-10-01T09:32:27.458143+0000","last_active":"2023-10-01T09:34:39.508850+0000","last_peered":"2023-10-01T09:34:39.508850+0000","last_clean":"2023-10-01T09:34:39.508850+0000","last_became_active":"2023-10-01T09:32:27.457802+0000","last_became_peered":"2023-10-01T09:32:27.457802+0000","last_unstale":"2023-10-01T09:34:39.508850+0000","last_undegraded":"2023-10-01T09:34:39.508850+0000","last_fullsized":"2023-10-01T09:34:39.508850+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":198,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504419+0000","last_change":"2023-10-01T09:32:27.446005+0000","last_active":"2023-10-01T09:34:39.504419+0000","last_peered":"2023-10-01T09:34:39.504419+0000","last_clean":"2023-10-01T09:34:39.504419+0000","last_became_active":"2023-10-01T09:32:27.445757+0000","last_became_peered":"2023-10-01T09:32:27.445757+0000","last_unstale":"2023-10-01T09:34:39.504419+0000","last_undegraded":"2023-10-01T09:34:39.504419+0000","last_fullsized":"2023-10-01T09:34:39.504419+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:27:23.521543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.2","version":"409'3506","reported_seq":3524,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:48.656463+0000","last_change":"2023-10-01T09:34:06.130449+0000","last_active":"2023-10-01T09:34:48.656463+0000","last_peered":"2023-10-01T09:34:48.656463+0000","last_clean":"2023-10-01T09:34:48.656463+0000","last_became_active":"2023-10-01T09:34:06.130193+0000","last_became_peered":"2023-10-01T09:34:06.130193+0000","last_unstale":"2023-10-01T09:34:48.656463+0000","last_undegraded":"2023-10-01T09:34:48.656463+0000","last_fullsized":"2023-10-01T09:34:48.656463+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3506,"log_dups_size":0,"ondisk_log_size":3506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:23:07.736775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229769216,"num_objects":3506,"num_object_clones":0,"num_object_copies":10518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3506,"num_write_kb":224384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":249,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.507637+0000","last_change":"2023-10-01T09:30:39.396141+0000","last_active":"2023-10-01T09:34:39.507637+0000","last_peered":"2023-10-01T09:34:39.507637+0000","last_clean":"2023-10-01T09:34:39.507637+0000","last_became_active":"2023-10-01T09:30:39.395843+0000","last_became_peered":"2023-10-01T09:30:39.395843+0000","last_unstale":"2023-10-01T09:34:39.507637+0000","last_undegraded":"2023-10-01T09:34:39.507637+0000","last_fullsized":"2023-10-01T09:34:39.507637+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:13:32.932438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":297,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.503966+0000","last_change":"2023-10-01T09:32:27.445484+0000","last_active":"2023-10-01T09:34:39.503966+0000","last_peered":"2023-10-01T09:34:39.503966+0000","last_clean":"2023-10-01T09:34:39.503966+0000","last_became_active":"2023-10-01T09:32:27.445198+0000","last_became_peered":"2023-10-01T09:32:27.445198+0000","last_unstale":"2023-10-01T09:34:39.503966+0000","last_undegraded":"2023-10-01T09:34:39.503966+0000","last_fullsized":"2023-10-01T09:34:39.503966+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:29:19.880130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"409'3525","reported_seq":3542,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:49.558099+0000","last_change":"2023-10-01T09:34:06.141056+0000","last_active":"2023-10-01T09:34:49.558099+0000","last_peered":"2023-10-01T09:34:49.558099+0000","last_clean":"2023-10-01T09:34:49.558099+0000","last_became_active":"2023-10-01T09:34:06.140163+0000","last_became_peered":"2023-10-01T09:34:06.140163+0000","last_unstale":"2023-10-01T09:34:49.558099+0000","last_undegraded":"2023-10-01T09:34:49.558099+0000","last_fullsized":"2023-10-01T09:34:49.558099+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3525,"log_dups_size":0,"ondisk_log_size":3525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231014400,"num_objects":3525,"num_object_clones":0,"num_object_copies":10575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3525,"num_write_kb":225600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1231,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504217+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:34:39.504217+0000","last_peered":"2023-10-01T09:34:39.504217+0000","last_clean":"2023-10-01T09:34:39.504217+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:34:39.504217+0000","last_undegraded":"2023-10-01T09:34:39.504217+0000","last_fullsized":"2023-10-01T09:34:39.504217+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":94,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.512016+0000","last_change":"2023-10-01T09:33:11.663163+0000","last_active":"2023-10-01T09:34:39.512016+0000","last_peered":"2023-10-01T09:34:39.512016+0000","last_clean":"2023-10-01T09:34:39.512016+0000","last_became_active":"2023-10-01T09:33:11.662771+0000","last_became_peered":"2023-10-01T09:33:11.662771+0000","last_unstale":"2023-10-01T09:34:39.512016+0000","last_undegraded":"2023-10-01T09:34:39.512016+0000","last_fullsized":"2023-10-01T09:34:39.512016+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":412,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.505859+0000","last_change":"2023-10-01T09:33:11.664415+0000","last_active":"2023-10-01T09:34:39.505859+0000","last_peered":"2023-10-01T09:34:39.505859+0000","last_clean":"2023-10-01T09:34:39.505859+0000","last_became_active":"2023-10-01T09:33:11.664177+0000","last_became_peered":"2023-10-01T09:33:11.664177+0000","last_unstale":"2023-10-01T09:34:39.505859+0000","last_undegraded":"2023-10-01T09:34:39.505859+0000","last_fullsized":"2023-10-01T09:34:39.505859+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:44:01.229405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"409'3504","reported_seq":3522,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:48.936393+0000","last_change":"2023-10-01T09:34:06.140052+0000","last_active":"2023-10-01T09:34:48.936393+0000","last_peered":"2023-10-01T09:34:48.936393+0000","last_clean":"2023-10-01T09:34:48.936393+0000","last_became_active":"2023-10-01T09:34:06.139672+0000","last_became_peered":"2023-10-01T09:34:06.139672+0000","last_unstale":"2023-10-01T09:34:48.936393+0000","last_undegraded":"2023-10-01T09:34:48.936393+0000","last_fullsized":"2023-10-01T09:34:48.936393+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3504,"log_dups_size":0,"ondisk_log_size":3504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:39:55.000830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229638144,"num_objects":3504,"num_object_clones":0,"num_object_copies":10512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3504,"num_write_kb":224256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1074,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511762+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:34:39.511762+0000","last_peered":"2023-10-01T09:34:39.511762+0000","last_clean":"2023-10-01T09:34:39.511762+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:34:39.511762+0000","last_undegraded":"2023-10-01T09:34:39.511762+0000","last_fullsized":"2023-10-01T09:34:39.511762+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":383,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513358+0000","last_change":"2023-10-01T09:32:27.451746+0000","last_active":"2023-10-01T09:34:39.513358+0000","last_peered":"2023-10-01T09:34:39.513358+0000","last_clean":"2023-10-01T09:34:39.513358+0000","last_became_active":"2023-10-01T09:32:27.451069+0000","last_became_peered":"2023-10-01T09:32:27.451069+0000","last_unstale":"2023-10-01T09:34:39.513358+0000","last_undegraded":"2023-10-01T09:34:39.513358+0000","last_fullsized":"2023-10-01T09:34:39.513358+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:50.586397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":52,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511914+0000","last_change":"2023-10-01T09:33:11.655679+0000","last_active":"2023-10-01T09:34:39.511914+0000","last_peered":"2023-10-01T09:34:39.511914+0000","last_clean":"2023-10-01T09:34:39.511914+0000","last_became_active":"2023-10-01T09:33:11.653576+0000","last_became_peered":"2023-10-01T09:33:11.653576+0000","last_unstale":"2023-10-01T09:34:39.511914+0000","last_undegraded":"2023-10-01T09:34:39.511914+0000","last_fullsized":"2023-10-01T09:34:39.511914+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"409'3514","reported_seq":3531,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:49.531517+0000","last_change":"2023-10-01T09:34:06.140732+0000","last_active":"2023-10-01T09:34:49.531517+0000","last_peered":"2023-10-01T09:34:49.531517+0000","last_clean":"2023-10-01T09:34:49.531517+0000","last_became_active":"2023-10-01T09:34:06.140433+0000","last_became_peered":"2023-10-01T09:34:06.140433+0000","last_unstale":"2023-10-01T09:34:49.531517+0000","last_undegraded":"2023-10-01T09:34:49.531517+0000","last_fullsized":"2023-10-01T09:34:49.531517+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3514,"log_dups_size":0,"ondisk_log_size":3514,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230293504,"num_objects":3514,"num_object_clones":0,"num_object_copies":10542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3514,"num_write_kb":224896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":601,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513045+0000","last_change":"2023-10-01T09:30:39.393231+0000","last_active":"2023-10-01T09:34:39.513045+0000","last_peered":"2023-10-01T09:34:39.513045+0000","last_clean":"2023-10-01T09:34:39.513045+0000","last_became_active":"2023-10-01T09:30:39.392700+0000","last_became_peered":"2023-10-01T09:30:39.392700+0000","last_unstale":"2023-10-01T09:34:39.513045+0000","last_undegraded":"2023-10-01T09:34:39.513045+0000","last_fullsized":"2023-10-01T09:34:39.513045+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:38:26.873688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":440,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.507797+0000","last_change":"2023-10-01T09:28:02.346596+0000","last_active":"2023-10-01T09:34:39.507797+0000","last_peered":"2023-10-01T09:34:39.507797+0000","last_clean":"2023-10-01T09:34:39.507797+0000","last_became_active":"2023-10-01T09:28:02.346318+0000","last_became_peered":"2023-10-01T09:28:02.346318+0000","last_unstale":"2023-10-01T09:34:39.507797+0000","last_undegraded":"2023-10-01T09:34:39.507797+0000","last_fullsized":"2023-10-01T09:34:39.507797+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:27:45.840901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":309,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504104+0000","last_change":"2023-10-01T09:33:11.665777+0000","last_active":"2023-10-01T09:34:39.504104+0000","last_peered":"2023-10-01T09:34:39.504104+0000","last_clean":"2023-10-01T09:34:39.504104+0000","last_became_active":"2023-10-01T09:33:11.665459+0000","last_became_peered":"2023-10-01T09:33:11.665459+0000","last_unstale":"2023-10-01T09:34:39.504104+0000","last_undegraded":"2023-10-01T09:34:39.504104+0000","last_fullsized":"2023-10-01T09:34:39.504104+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:40:34.693521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"409'3478","reported_seq":3496,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:48.922439+0000","last_change":"2023-10-01T09:34:06.140010+0000","last_active":"2023-10-01T09:34:48.922439+0000","last_peered":"2023-10-01T09:34:48.922439+0000","last_clean":"2023-10-01T09:34:48.922439+0000","last_became_active":"2023-10-01T09:34:06.139718+0000","last_became_peered":"2023-10-01T09:34:06.139718+0000","last_unstale":"2023-10-01T09:34:48.922439+0000","last_undegraded":"2023-10-01T09:34:48.922439+0000","last_fullsized":"2023-10-01T09:34:48.922439+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3478,"log_dups_size":0,"ondisk_log_size":3478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:58.151539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227934208,"num_objects":3478,"num_object_clones":0,"num_object_copies":10434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3478,"num_write_kb":222592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1325,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511613+0000","last_change":"2023-10-01T09:33:11.665240+0000","last_active":"2023-10-01T09:34:39.511613+0000","last_peered":"2023-10-01T09:34:39.511613+0000","last_clean":"2023-10-01T09:34:39.511613+0000","last_became_active":"2023-10-01T09:33:11.664702+0000","last_became_peered":"2023-10-01T09:33:11.664702+0000","last_unstale":"2023-10-01T09:34:39.511613+0000","last_undegraded":"2023-10-01T09:34:39.511613+0000","last_fullsized":"2023-10-01T09:34:39.511613+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":627,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.508523+0000","last_change":"2023-10-01T09:29:04.962927+0000","last_active":"2023-10-01T09:34:39.508523+0000","last_peered":"2023-10-01T09:34:39.508523+0000","last_clean":"2023-10-01T09:34:39.508523+0000","last_became_active":"2023-10-01T09:29:04.962542+0000","last_became_peered":"2023-10-01T09:29:04.962542+0000","last_unstale":"2023-10-01T09:34:39.508523+0000","last_undegraded":"2023-10-01T09:34:39.508523+0000","last_fullsized":"2023-10-01T09:34:39.508523+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":181,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511683+0000","last_change":"2023-10-01T09:33:11.663444+0000","last_active":"2023-10-01T09:34:39.511683+0000","last_peered":"2023-10-01T09:34:39.511683+0000","last_clean":"2023-10-01T09:34:39.511683+0000","last_became_active":"2023-10-01T09:33:11.660310+0000","last_became_peered":"2023-10-01T09:33:11.660310+0000","last_unstale":"2023-10-01T09:34:39.511683+0000","last_undegraded":"2023-10-01T09:34:39.511683+0000","last_fullsized":"2023-10-01T09:34:39.511683+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":323,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.509344+0000","last_change":"2023-10-01T09:32:27.453291+0000","last_active":"2023-10-01T09:34:39.509344+0000","last_peered":"2023-10-01T09:34:39.509344+0000","last_clean":"2023-10-01T09:34:39.509344+0000","last_became_active":"2023-10-01T09:32:27.453026+0000","last_became_peered":"2023-10-01T09:32:27.453026+0000","last_unstale":"2023-10-01T09:34:39.509344+0000","last_undegraded":"2023-10-01T09:34:39.509344+0000","last_fullsized":"2023-10-01T09:34:39.509344+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"409'3470","reported_seq":3487,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:48.949335+0000","last_change":"2023-10-01T09:34:06.140844+0000","last_active":"2023-10-01T09:34:48.949335+0000","last_peered":"2023-10-01T09:34:48.949335+0000","last_clean":"2023-10-01T09:34:48.949335+0000","last_became_active":"2023-10-01T09:34:06.139812+0000","last_became_peered":"2023-10-01T09:34:06.139812+0000","last_unstale":"2023-10-01T09:34:48.949335+0000","last_undegraded":"2023-10-01T09:34:48.949335+0000","last_fullsized":"2023-10-01T09:34:48.949335+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3470,"log_dups_size":0,"ondisk_log_size":3470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:31:05.105491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227409920,"num_objects":3470,"num_object_clones":0,"num_object_copies":10410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3470,"num_write_kb":222080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":505,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.508526+0000","last_change":"2023-10-01T09:12:10.001932+0000","last_active":"2023-10-01T09:34:39.508526+0000","last_peered":"2023-10-01T09:34:39.508526+0000","last_clean":"2023-10-01T09:34:39.508526+0000","last_became_active":"2023-10-01T09:12:10.001421+0000","last_became_peered":"2023-10-01T09:12:10.001421+0000","last_unstale":"2023-10-01T09:34:39.508526+0000","last_undegraded":"2023-10-01T09:34:39.508526+0000","last_fullsized":"2023-10-01T09:34:39.508526+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":426,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513150+0000","last_change":"2023-10-01T09:32:27.450723+0000","last_active":"2023-10-01T09:34:39.513150+0000","last_peered":"2023-10-01T09:34:39.513150+0000","last_clean":"2023-10-01T09:34:39.513150+0000","last_became_active":"2023-10-01T09:32:27.450037+0000","last_became_peered":"2023-10-01T09:32:27.450037+0000","last_unstale":"2023-10-01T09:34:39.513150+0000","last_undegraded":"2023-10-01T09:34:39.513150+0000","last_fullsized":"2023-10-01T09:34:39.513150+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:37.260098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":417,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:43.609563+0000","last_change":"2023-10-01T09:19:30.091481+0000","last_active":"2023-10-01T09:34:43.609563+0000","last_peered":"2023-10-01T09:34:43.609563+0000","last_clean":"2023-10-01T09:34:43.609563+0000","last_became_active":"2023-10-01T09:19:30.091151+0000","last_became_peered":"2023-10-01T09:19:30.091151+0000","last_unstale":"2023-10-01T09:34:43.609563+0000","last_undegraded":"2023-10-01T09:34:43.609563+0000","last_fullsized":"2023-10-01T09:34:43.609563+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:09.127408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"409'3776","reported_seq":3793,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:51.773224+0000","last_change":"2023-10-01T09:34:06.167577+0000","last_active":"2023-10-01T09:34:51.773224+0000","last_peered":"2023-10-01T09:34:51.773224+0000","last_clean":"2023-10-01T09:34:51.773224+0000","last_became_active":"2023-10-01T09:34:06.167267+0000","last_became_peered":"2023-10-01T09:34:06.167267+0000","last_unstale":"2023-10-01T09:34:51.773224+0000","last_undegraded":"2023-10-01T09:34:51.773224+0000","last_fullsized":"2023-10-01T09:34:51.773224+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3776,"log_dups_size":0,"ondisk_log_size":3776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247463936,"num_objects":3776,"num_object_clones":0,"num_object_copies":11328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3776,"num_write_kb":241664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":847,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.509323+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:34:39.509323+0000","last_peered":"2023-10-01T09:34:39.509323+0000","last_clean":"2023-10-01T09:34:39.509323+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:34:39.509323+0000","last_undegraded":"2023-10-01T09:34:39.509323+0000","last_fullsized":"2023-10-01T09:34:39.509323+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":445,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504320+0000","last_change":"2023-10-01T09:33:11.665789+0000","last_active":"2023-10-01T09:34:39.504320+0000","last_peered":"2023-10-01T09:34:39.504320+0000","last_clean":"2023-10-01T09:34:39.504320+0000","last_became_active":"2023-10-01T09:33:11.665477+0000","last_became_peered":"2023-10-01T09:33:11.665477+0000","last_unstale":"2023-10-01T09:34:39.504320+0000","last_undegraded":"2023-10-01T09:34:39.504320+0000","last_fullsized":"2023-10-01T09:34:39.504320+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:18:38.766268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":399,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513274+0000","last_change":"2023-10-01T09:33:11.656981+0000","last_active":"2023-10-01T09:34:39.513274+0000","last_peered":"2023-10-01T09:34:39.513274+0000","last_clean":"2023-10-01T09:34:39.513274+0000","last_became_active":"2023-10-01T09:33:11.656334+0000","last_became_peered":"2023-10-01T09:33:11.656334+0000","last_unstale":"2023-10-01T09:34:39.513274+0000","last_undegraded":"2023-10-01T09:34:39.513274+0000","last_fullsized":"2023-10-01T09:34:39.513274+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:50:15.562113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"409'3583","reported_seq":3600,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:51.753286+0000","last_change":"2023-10-01T09:34:06.140475+0000","last_active":"2023-10-01T09:34:51.753286+0000","last_peered":"2023-10-01T09:34:51.753286+0000","last_clean":"2023-10-01T09:34:51.753286+0000","last_became_active":"2023-10-01T09:34:06.140142+0000","last_became_peered":"2023-10-01T09:34:06.140142+0000","last_unstale":"2023-10-01T09:34:51.753286+0000","last_undegraded":"2023-10-01T09:34:51.753286+0000","last_fullsized":"2023-10-01T09:34:51.753286+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3583,"log_dups_size":0,"ondisk_log_size":3583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234815488,"num_objects":3583,"num_object_clones":0,"num_object_copies":10749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3583,"num_write_kb":229312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":322,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.508956+0000","last_change":"2023-10-01T09:30:39.402924+0000","last_active":"2023-10-01T09:34:39.508956+0000","last_peered":"2023-10-01T09:34:39.508956+0000","last_clean":"2023-10-01T09:34:39.508956+0000","last_became_active":"2023-10-01T09:30:39.402594+0000","last_became_peered":"2023-10-01T09:30:39.402594+0000","last_unstale":"2023-10-01T09:34:39.508956+0000","last_undegraded":"2023-10-01T09:34:39.508956+0000","last_fullsized":"2023-10-01T09:34:39.508956+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1037,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504786+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:34:39.504786+0000","last_peered":"2023-10-01T09:34:39.504786+0000","last_clean":"2023-10-01T09:34:39.504786+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:34:39.504786+0000","last_undegraded":"2023-10-01T09:34:39.504786+0000","last_fullsized":"2023-10-01T09:34:39.504786+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":98,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.508434+0000","last_change":"2023-10-01T09:30:39.418085+0000","last_active":"2023-10-01T09:34:39.508434+0000","last_peered":"2023-10-01T09:34:39.508434+0000","last_clean":"2023-10-01T09:34:39.508434+0000","last_became_active":"2023-10-01T09:30:39.417846+0000","last_became_peered":"2023-10-01T09:30:39.417846+0000","last_unstale":"2023-10-01T09:34:39.508434+0000","last_undegraded":"2023-10-01T09:34:39.508434+0000","last_fullsized":"2023-10-01T09:34:39.508434+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"409'3714","reported_seq":3731,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:51.741027+0000","last_change":"2023-10-01T09:34:06.167610+0000","last_active":"2023-10-01T09:34:51.741027+0000","last_peered":"2023-10-01T09:34:51.741027+0000","last_clean":"2023-10-01T09:34:51.741027+0000","last_became_active":"2023-10-01T09:34:06.167324+0000","last_became_peered":"2023-10-01T09:34:06.167324+0000","last_unstale":"2023-10-01T09:34:51.741027+0000","last_undegraded":"2023-10-01T09:34:51.741027+0000","last_fullsized":"2023-10-01T09:34:51.741027+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":3714,"log_dups_size":0,"ondisk_log_size":3714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243400704,"num_objects":3714,"num_object_clones":0,"num_object_copies":11142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3714,"num_write_kb":237696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":122,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.503221+0000","last_change":"2023-10-01T09:29:11.374086+0000","last_active":"2023-10-01T09:34:39.503221+0000","last_peered":"2023-10-01T09:34:39.503221+0000","last_clean":"2023-10-01T09:34:39.503221+0000","last_became_active":"2023-10-01T09:28:02.319106+0000","last_became_peered":"2023-10-01T09:28:02.319106+0000","last_unstale":"2023-10-01T09:34:39.503221+0000","last_undegraded":"2023-10-01T09:34:39.503221+0000","last_fullsized":"2023-10-01T09:34:39.503221+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":422,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:43.609645+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:34:43.609645+0000","last_peered":"2023-10-01T09:34:43.609645+0000","last_clean":"2023-10-01T09:34:43.609645+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:34:43.609645+0000","last_undegraded":"2023-10-01T09:34:43.609645+0000","last_fullsized":"2023-10-01T09:34:43.609645+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":132,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511599+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:34:39.511599+0000","last_peered":"2023-10-01T09:34:39.511599+0000","last_clean":"2023-10-01T09:34:39.511599+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:34:39.511599+0000","last_undegraded":"2023-10-01T09:34:39.511599+0000","last_fullsized":"2023-10-01T09:34:39.511599+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":371,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513293+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:34:39.513293+0000","last_peered":"2023-10-01T09:34:39.513293+0000","last_clean":"2023-10-01T09:34:39.513293+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:34:39.513293+0000","last_undegraded":"2023-10-01T09:34:39.513293+0000","last_fullsized":"2023-10-01T09:34:39.513293+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":440,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.504388+0000","last_change":"2023-10-01T09:22:40.166927+0000","last_active":"2023-10-01T09:34:39.504388+0000","last_peered":"2023-10-01T09:34:39.504388+0000","last_clean":"2023-10-01T09:34:39.504388+0000","last_became_active":"2023-10-01T09:22:40.166606+0000","last_became_peered":"2023-10-01T09:22:40.166606+0000","last_unstale":"2023-10-01T09:34:39.504388+0000","last_undegraded":"2023-10-01T09:34:39.504388+0000","last_fullsized":"2023-10-01T09:34:39.504388+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:54:18.044207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":144,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511709+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:34:39.511709+0000","last_peered":"2023-10-01T09:34:39.511709+0000","last_clean":"2023-10-01T09:34:39.511709+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:34:39.511709+0000","last_undegraded":"2023-10-01T09:34:39.511709+0000","last_fullsized":"2023-10-01T09:34:39.511709+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.511527+0000","last_change":"2023-10-01T09:33:11.660645+0000","last_active":"2023-10-01T09:34:39.511527+0000","last_peered":"2023-10-01T09:34:39.511527+0000","last_clean":"2023-10-01T09:34:39.511527+0000","last_became_active":"2023-10-01T09:33:11.660059+0000","last_became_peered":"2023-10-01T09:33:11.660059+0000","last_unstale":"2023-10-01T09:34:39.511527+0000","last_undegraded":"2023-10-01T09:34:39.511527+0000","last_fullsized":"2023-10-01T09:34:39.511527+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":144,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.512092+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:34:39.512092+0000","last_peered":"2023-10-01T09:34:39.512092+0000","last_clean":"2023-10-01T09:34:39.512092+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:34:39.512092+0000","last_undegraded":"2023-10-01T09:34:39.512092+0000","last_fullsized":"2023-10-01T09:34:39.512092+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":299,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513132+0000","last_change":"2023-10-01T09:32:27.456398+0000","last_active":"2023-10-01T09:34:39.513132+0000","last_peered":"2023-10-01T09:34:39.513132+0000","last_clean":"2023-10-01T09:34:39.513132+0000","last_became_active":"2023-10-01T09:32:27.456106+0000","last_became_peered":"2023-10-01T09:32:27.456106+0000","last_unstale":"2023-10-01T09:34:39.513132+0000","last_undegraded":"2023-10-01T09:34:39.513132+0000","last_fullsized":"2023-10-01T09:34:39.513132+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:10:29.553023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":213,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-10-01T09:34:43.558871+0000","last_change":"2023-10-01T09:28:02.321395+0000","last_active":"2023-10-01T09:34:43.558871+0000","last_peered":"2023-10-01T09:34:43.558871+0000","last_clean":"2023-10-01T09:34:43.558871+0000","last_became_active":"2023-10-01T09:28:02.321071+0000","last_became_peered":"2023-10-01T09:28:02.321071+0000","last_unstale":"2023-10-01T09:34:43.558871+0000","last_undegraded":"2023-10-01T09:34:43.558871+0000","last_fullsized":"2023-10-01T09:34:43.558871+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:12:45.734920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":317,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513140+0000","last_change":"2023-10-01T09:30:39.392942+0000","last_active":"2023-10-01T09:34:39.513140+0000","last_peered":"2023-10-01T09:34:39.513140+0000","last_clean":"2023-10-01T09:34:39.513140+0000","last_became_active":"2023-10-01T09:30:39.392663+0000","last_became_peered":"2023-10-01T09:30:39.392663+0000","last_unstale":"2023-10-01T09:34:39.513140+0000","last_undegraded":"2023-10-01T09:34:39.513140+0000","last_fullsized":"2023-10-01T09:34:39.513140+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:19:39.644244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":516,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.513665+0000","last_change":"2023-10-01T09:33:11.656642+0000","last_active":"2023-10-01T09:34:39.513665+0000","last_peered":"2023-10-01T09:34:39.513665+0000","last_clean":"2023-10-01T09:34:39.513665+0000","last_became_active":"2023-10-01T09:33:11.656325+0000","last_became_peered":"2023-10-01T09:33:11.656325+0000","last_unstale":"2023-10-01T09:34:39.513665+0000","last_undegraded":"2023-10-01T09:34:39.513665+0000","last_fullsized":"2023-10-01T09:34:39.513665+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:01:05.809978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":377,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-10-01T09:34:39.507541+0000","last_change":"2023-10-01T09:32:27.445299+0000","last_active":"2023-10-01T09:34:39.507541+0000","last_peered":"2023-10-01T09:34:39.507541+0000","last_clean":"2023-10-01T09:34:39.507541+0000","last_became_active":"2023-10-01T09:32:27.444956+0000","last_became_peered":"2023-10-01T09:32:27.444956+0000","last_unstale":"2023-10-01T09:34:39.507541+0000","last_undegraded":"2023-10-01T09:34:39.507541+0000","last_fullsized":"2023-10-01T09:34:39.507541+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:49:07.862356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":33298,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3724214272,"num_objects":56827,"num_object_clones":0,"num_object_copies":170481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":56827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":56827,"num_write_kb":3636928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11243487232,"data_stored":11243487232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":56827,"ondisk_log_size":56827,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430611,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2708296,"kb_used_data":2118592,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":91036856,"statfs":{"total":95995035648,"available":93221740544,"internally_reserved":0,"allocated":2169438208,"data_stored":2168547579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:34:34 2023","interfaces":[{"interface":"back","average":{"1min":10.117000000000001,"5min":3.4329999999999998,"15min":3.02},"min":{"1min":0.26800000000000002,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":108.90000000000001,"5min":108.90000000000001,"15min":108.90000000000001},"last":0.436},{"interface":"front","average":{"1min":9.8490000000000002,"5min":3.48,"15min":3.2429999999999999},"min":{"1min":0.40200000000000002,"5min":0.23899999999999999,"15min":0.22500000000000001},"max":{"1min":109.187,"5min":109.187,"15min":109.187},"last":0.23300000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:33:51 2023","interfaces":[{"interface":"back","average":{"1min":5.3789999999999996,"5min":4.915,"15min":4.0860000000000003},"min":{"1min":0.26100000000000001,"5min":0.222,"15min":0.222},"max":{"1min":56.694000000000003,"5min":205.34200000000001,"15min":205.34200000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":5.9710000000000001,"5min":4.8159999999999998,"15min":4.0570000000000004},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.23599999999999999},"max":{"1min":57.816000000000003,"5min":205.44499999999999,"15min":205.44499999999999},"last":1.548}]},{"osd":2,"last update":"Sun Oct 1 09:34:24 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.7250000000000001,"15min":2.653},"min":{"1min":0.32400000000000001,"5min":0.27100000000000002,"15min":0.249},"max":{"1min":5.3099999999999996,"5min":10.523,"15min":104.837},"last":0.499},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.7270000000000001,"15min":2.585},"min":{"1min":0.374,"5min":0.27700000000000002,"15min":0.247},"max":{"1min":4.7400000000000002,"5min":10.026,"15min":106.379},"last":0.29099999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:33:51 2023","interfaces":[{"interface":"back","average":{"1min":1.407,"5min":3.3220000000000001,"15min":3.5720000000000001},"min":{"1min":0.32900000000000001,"5min":0.30199999999999999,"15min":0.28199999999999997},"max":{"1min":3.9649999999999999,"5min":80.352999999999994,"15min":164.595},"last":1.5840000000000001},{"interface":"front","average":{"1min":1.4059999999999999,"5min":3.0019999999999998,"15min":3.3250000000000002},"min":{"1min":0.33200000000000002,"5min":0.313,"15min":0.29399999999999998},"max":{"1min":3.7349999999999999,"5min":80.364000000000004,"15min":164.60499999999999},"last":1.5980000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:34:34 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.046,"15min":0.98399999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.23499999999999999},"max":{"1min":5.3490000000000002,"5min":7.7039999999999997,"15min":9.2349999999999994},"last":4.1829999999999998},{"interface":"front","average":{"1min":0.97799999999999998,"5min":1.075,"15min":0.98899999999999999},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":4.6230000000000002,"5min":8.0009999999999994,"15min":9.4339999999999993},"last":0.312}]},{"osd":5,"last update":"Sun Oct 1 09:34:11 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.4100000000000001,"5min":4.4100000000000001,"15min":4.4100000000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.3839999999999999,"5min":3.3839999999999999,"15min":3.3839999999999999},"last":0.60299999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:34:04 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.91500000000000004,"15min":0.84299999999999997},"min":{"1min":0.38900000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.3490000000000002,"5min":7.75,"15min":7.75},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.88200000000000001,"15min":0.73399999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.5430000000000001,"5min":6.3860000000000001,"15min":6.3860000000000001},"last":0.624}]}]},{"osd":6,"up_from":294,"seq":1262720385122,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1036344,"kb_used_data":706928,"kb_used_omap":6,"kb_used_meta":329401,"kb_avail":92708808,"statfs":{"total":95995035648,"available":94933819392,"internally_reserved":0,"allocated":723894272,"data_stored":723019270,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":337306669},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:34:01 2023","interfaces":[{"interface":"back","average":{"1min":4.6840000000000002,"5min":6.8559999999999999,"15min":3.6179999999999999},"min":{"1min":0.31900000000000001,"5min":0.23200000000000001,"15min":0.222},"max":{"1min":40.426000000000002,"5min":130.02699999999999,"15min":130.02699999999999},"last":1.0049999999999999},{"interface":"front","average":{"1min":3.3889999999999998,"5min":6.5449999999999999,"15min":3.2269999999999999},"min":{"1min":0.248,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":40.447000000000003,"5min":133.72900000000001,"15min":133.72900000000001},"last":0.998}]},{"osd":1,"last update":"Sun Oct 1 09:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":4.4450000000000003,"15min":2.718},"min":{"1min":0.28299999999999997,"5min":0.24199999999999999,"15min":0.23899999999999999},"max":{"1min":5.5039999999999996,"5min":97.561000000000007,"15min":97.561000000000007},"last":0.97099999999999997},{"interface":"front","average":{"1min":1.284,"5min":4.3959999999999999,"15min":2.669},"min":{"1min":0.33900000000000002,"5min":0.25600000000000001,"15min":0.24399999999999999},"max":{"1min":5.4260000000000002,"5min":97.585999999999999,"15min":97.585999999999999},"last":1.1140000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.7989999999999999,"15min":2.5419999999999998},"min":{"1min":0.28100000000000003,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":5.7679999999999998,"5min":10.92,"15min":26.289999999999999},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.645,"5min":1.8240000000000001,"15min":2.6859999999999999},"min":{"1min":0.309,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.9000000000000004,"5min":12.804,"15min":26.388999999999999},"last":1.1799999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:34:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0680000000000001,"5min":2.5649999999999999,"15min":2.5539999999999998},"min":{"1min":0.39600000000000002,"5min":0.316,"15min":0.316},"max":{"1min":8.0730000000000004,"5min":62.546999999999997,"15min":62.546999999999997},"last":0.65400000000000003},{"interface":"front","average":{"1min":1.8,"5min":2.2799999999999998,"15min":1.79},"min":{"1min":0.37,"5min":0.32300000000000001,"15min":0.29299999999999998},"max":{"1min":6.6600000000000001,"5min":62.465000000000003,"15min":62.465000000000003},"last":1.286}]},{"osd":4,"last update":"Sun Oct 1 09:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":0.93799999999999994,"15min":0.57599999999999996},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.25700000000000001},"max":{"1min":6.0640000000000001,"5min":6.0640000000000001,"15min":6.0640000000000001},"last":4.5750000000000002},{"interface":"front","average":{"1min":1.5820000000000001,"5min":1.077,"15min":0.74099999999999999},"min":{"1min":0.315,"5min":0.30099999999999999,"15min":0.28299999999999997},"max":{"1min":8.8249999999999993,"5min":9.4090000000000007,"15min":9.4090000000000007},"last":0.58699999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:34:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.629,"5min":3.629,"15min":3.629},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.131,"5min":1.131,"15min":1.131},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.5309999999999997,"5min":4.5309999999999997,"15min":4.5309999999999997},"last":1.147}]},{"osd":7,"last update":"Sun Oct 1 09:34:01 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":0.995,"15min":3.9399999999999999},"min":{"1min":0.31900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.3710000000000004,"5min":7.4589999999999996,"15min":115.11},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.069,"5min":0.78300000000000003,"15min":3.8260000000000001},"min":{"1min":0.308,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.8789999999999996,"5min":4.8789999999999996,"15min":114.666},"last":0.93500000000000005}]}]},{"osd":1,"up_from":94,"seq":403726926098,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2569824,"kb_used_data":1966168,"kb_used_omap":7,"kb_used_meta":603640,"kb_avail":91175328,"statfs":{"total":95995035648,"available":93363535872,"internally_reserved":0,"allocated":2013356032,"data_stored":2011629883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":618127783},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:34:42 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":4.1449999999999996,"15min":4.1589999999999998},"min":{"1min":0.24399999999999999,"5min":0.183,"15min":0.183},"max":{"1min":7.1210000000000004,"5min":82.359999999999999,"15min":134.84999999999999},"last":2.3580000000000001},{"interface":"front","average":{"1min":0.97399999999999998,"5min":4.0140000000000002,"15min":4.1520000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":4.9829999999999997,"5min":82.078999999999994,"15min":134.77099999999999},"last":0.44600000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:34:20 2023","interfaces":[{"interface":"back","average":{"1min":1.599,"5min":3.8399999999999999,"15min":3.681},"min":{"1min":0.27300000000000002,"5min":0.20499999999999999,"15min":0.182},"max":{"1min":9.5370000000000008,"5min":56.573,"15min":77.325000000000003},"last":2.3290000000000002},{"interface":"front","average":{"1min":1.9450000000000001,"5min":3.9809999999999999,"15min":4.0860000000000003},"min":{"1min":0.28299999999999997,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":7.9699999999999998,"5min":55.930999999999997,"15min":80.361000000000004},"last":0.41199999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:33:57 2023","interfaces":[{"interface":"back","average":{"1min":3.5049999999999999,"5min":5.0359999999999996,"15min":4.6500000000000004},"min":{"1min":0.26800000000000002,"5min":0.245,"15min":0.224},"max":{"1min":13.281000000000001,"5min":200.56800000000001,"15min":200.56800000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":3.3780000000000001,"5min":5.2469999999999999,"15min":4.6550000000000002},"min":{"1min":0.30199999999999999,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":25.190999999999999,"5min":200.63800000000001,"15min":200.63800000000001},"last":1.9059999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:34:42 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.302,"15min":1.4339999999999999},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":3.3239999999999998,"5min":13.045,"15min":13.045},"last":1.2050000000000001},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.9870000000000001,"15min":1.901},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.27500000000000002},"max":{"1min":6.9500000000000002,"5min":12.023999999999999,"15min":18.219000000000001},"last":4.0819999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:34:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.0190000000000001,"5min":4.0190000000000001,"15min":4.0190000000000001},"last":5.1550000000000002},{"interface":"front","average":{"1min":2.0230000000000001,"5min":2.0230000000000001,"15min":2.0230000000000001},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":11.218999999999999,"5min":11.218999999999999,"15min":11.218999999999999},"last":3.0430000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:33:57 2023","interfaces":[{"interface":"back","average":{"1min":2.238,"5min":2.2789999999999999,"15min":2.8460000000000001},"min":{"1min":0.39500000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.9429999999999996,"5min":12.842000000000001,"15min":12.842000000000001},"last":2.839},{"interface":"front","average":{"1min":1.3360000000000001,"5min":1.7150000000000001,"15min":1.9179999999999999},"min":{"1min":0.375,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.2469999999999999,"5min":14.772,"15min":14.772},"last":1.2450000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:34:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.9390000000000001,"15min":1.7929999999999999},"min":{"1min":0.442,"5min":0.35899999999999999,"15min":0.316},"max":{"1min":3.3919999999999999,"5min":14.717000000000001,"15min":21.399000000000001},"last":1.2769999999999999},{"interface":"front","average":{"1min":1.546,"5min":2.1539999999999999,"15min":2.008},"min":{"1min":0.45200000000000001,"5min":0.35399999999999998,"15min":0.30199999999999999},"max":{"1min":7.1529999999999996,"5min":11.177,"15min":22.318999999999999},"last":2.2869999999999999}]}]},{"osd":0,"up_from":89,"seq":382252089632,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1580812,"kb_used_data":903492,"kb_used_omap":8,"kb_used_meta":677303,"kb_avail":92164340,"statfs":{"total":95995035648,"available":94376284160,"internally_reserved":0,"allocated":925175808,"data_stored":923413786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":693559234},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:33:46 2023","interfaces":[{"interface":"back","average":{"1min":3.8290000000000002,"5min":3.0600000000000001,"15min":3.282},"min":{"1min":0.23200000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":30.263999999999999,"5min":61.173000000000002,"15min":187.41399999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":4.9489999999999998,"5min":4.1500000000000004,"15min":4.1929999999999996},"min":{"1min":0.30499999999999999,"5min":0.22500000000000001,"15min":0.193},"max":{"1min":30.248999999999999,"5min":68.587999999999994,"15min":187.78700000000001},"last":2.9660000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:34:20 2023","interfaces":[{"interface":"back","average":{"1min":2.6709999999999998,"5min":2.1600000000000001,"15min":2.1749999999999998},"min":{"1min":0.25600000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":13.494,"5min":16.033000000000001,"15min":39.048999999999999},"last":1.1519999999999999},{"interface":"front","average":{"1min":1.9199999999999999,"5min":2.3799999999999999,"15min":2.9769999999999999},"min":{"1min":0.30099999999999999,"5min":0.189,"15min":0.189},"max":{"1min":13.003,"5min":14.319000000000001,"15min":38.945999999999998},"last":3.0009999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:33:46 2023","interfaces":[{"interface":"back","average":{"1min":3.149,"5min":3.9359999999999999,"15min":4.0670000000000002},"min":{"1min":0.26600000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":13.189,"5min":59.216000000000001,"15min":139.85300000000001},"last":0.64100000000000001},{"interface":"front","average":{"1min":3.8340000000000001,"5min":3.8460000000000001,"15min":4.5330000000000004},"min":{"1min":0.29299999999999998,"5min":0.26300000000000001,"15min":0.219},"max":{"1min":14.442,"5min":59.177999999999997,"15min":139.70099999999999},"last":2.419}]},{"osd":4,"last update":"Sun Oct 1 09:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":2.5369999999999999,"15min":2.1259999999999999},"min":{"1min":0.318,"5min":0.27600000000000002,"15min":0.27300000000000002},"max":{"1min":8.7569999999999997,"5min":62.206000000000003,"15min":62.206000000000003},"last":3.0299999999999998},{"interface":"front","average":{"1min":1.0209999999999999,"5min":2.3149999999999999,"15min":2.0299999999999998},"min":{"1min":0.38,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":4.0800000000000001,"5min":62.073999999999998,"15min":62.073999999999998},"last":1.335}]},{"osd":5,"last update":"Sun Oct 1 09:34:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7609999999999999,"5min":1.7609999999999999,"15min":1.7609999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":9.4779999999999998,"5min":9.4779999999999998,"15min":9.4779999999999998},"last":1.286},{"interface":"front","average":{"1min":1.6319999999999999,"5min":1.6319999999999999,"15min":1.6319999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":7.2130000000000001,"5min":7.2130000000000001,"15min":7.2130000000000001},"last":0.52500000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:34:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7330000000000001,"5min":1.8819999999999999,"15min":1.591},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.36899999999999999},"max":{"1min":7.2869999999999999,"5min":15.904,"15min":15.904},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.831,"5min":1.988,"15min":2.3980000000000001},"min":{"1min":0.41499999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.8089999999999993,"5min":14.215,"15min":18.521000000000001},"last":3.0550000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":2.085,"15min":1.5780000000000001},"min":{"1min":0.41999999999999998,"5min":0.38100000000000001,"15min":0.29199999999999998},"max":{"1min":7.3250000000000002,"5min":19.402999999999999,"15min":19.402999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.367,"5min":2.1299999999999999,"15min":2.0710000000000002},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.312},"max":{"1min":8.8439999999999994,"5min":19.373000000000001,"15min":19.373000000000001},"last":3.0779999999999998}]}]},{"osd":2,"up_from":99,"seq":425201762572,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1364224,"kb_used_data":692984,"kb_used_omap":11,"kb_used_meta":671220,"kb_avail":92380928,"statfs":{"total":95995035648,"available":94598070272,"internally_reserved":0,"allocated":709615616,"data_stored":707921118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":687330167},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:34:36 2023","interfaces":[{"interface":"back","average":{"1min":10.503,"5min":9.6669999999999998,"15min":6.7149999999999999},"min":{"1min":0.22700000000000001,"5min":0.191,"15min":0.17599999999999999},"max":{"1min":95.597999999999999,"5min":224.56,"15min":224.56},"last":0.29099999999999998},{"interface":"front","average":{"1min":10.513999999999999,"5min":9.4510000000000005,"15min":7.1390000000000002},"min":{"1min":0.23999999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":95.558000000000007,"5min":225.66900000000001,"15min":225.66900000000001},"last":0.26900000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:34:26 2023","interfaces":[{"interface":"back","average":{"1min":5.5,"5min":9.5709999999999997,"15min":7.2000000000000002},"min":{"1min":0.218,"5min":0.191,"15min":0.191},"max":{"1min":92.584999999999994,"5min":224.917,"15min":224.917},"last":0.25},{"interface":"front","average":{"1min":5.7809999999999997,"5min":9.8789999999999996,"15min":7.9770000000000003},"min":{"1min":0.28799999999999998,"5min":0.23599999999999999,"15min":0.20899999999999999},"max":{"1min":94.441000000000003,"5min":225.625,"15min":225.625},"last":0.22900000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:33:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1219999999999999,"5min":4.6970000000000001,"15min":6.0810000000000004},"min":{"1min":0.251,"5min":0.218,"15min":0.21099999999999999},"max":{"1min":9.766,"5min":226.124,"15min":253.238},"last":0.312},{"interface":"front","average":{"1min":2.097,"5min":4.8579999999999997,"15min":5.9199999999999999},"min":{"1min":0.28899999999999998,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":9.5969999999999995,"5min":226.32900000000001,"15min":252.54900000000001},"last":0.41099999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:34:26 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":3.6589999999999998,"15min":2.617},"min":{"1min":0.29999999999999999,"5min":0.28599999999999998,"15min":0.27200000000000002},"max":{"1min":5.5599999999999996,"5min":224.90199999999999,"15min":224.90199999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.90200000000000002,"5min":4.2050000000000001,"15min":2.8090000000000002},"min":{"1min":0.32500000000000001,"5min":0.309,"15min":0.27500000000000002},"max":{"1min":5.4240000000000004,"5min":225.386,"15min":225.386},"last":0.34499999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:34:11 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":1.661,"15min":1.661},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":12.500999999999999,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.194000000000001,"5min":10.194000000000001,"15min":10.194000000000001},"last":0.502}]},{"osd":6,"last update":"Sun Oct 1 09:33:57 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":4.6440000000000001,"15min":2.9340000000000002},"min":{"1min":0.36399999999999999,"5min":0.317,"15min":0.317},"max":{"1min":6.3780000000000001,"5min":225.41499999999999,"15min":225.41499999999999},"last":0.374},{"interface":"front","average":{"1min":2.1829999999999998,"5min":4.5960000000000001,"15min":2.4239999999999999},"min":{"1min":0.374,"5min":0.33700000000000002,"15min":0.33100000000000002},"max":{"1min":13.614000000000001,"5min":225.02000000000001,"15min":225.02000000000001},"last":0.48399999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:34:26 2023","interfaces":[{"interface":"back","average":{"1min":1.548,"5min":4.3639999999999999,"15min":2.8420000000000001},"min":{"1min":0.36799999999999999,"5min":0.31,"15min":0.308},"max":{"1min":12.544,"5min":224.94999999999999,"15min":224.94999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.89900000000000002,"5min":3.7869999999999999,"15min":2.8889999999999998},"min":{"1min":0.42599999999999999,"5min":0.37,"15min":0.33600000000000002},"max":{"1min":4.6660000000000004,"5min":224.92599999999999,"15min":224.92599999999999},"last":0.52800000000000002}]}]},{"osd":3,"up_from":104,"seq":446676599046,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2556964,"kb_used_data":1881884,"kb_used_omap":3,"kb_used_meta":675068,"kb_avail":91188188,"statfs":{"total":95995035648,"available":93376704512,"internally_reserved":0,"allocated":1927049216,"data_stored":1925260511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":691270375},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:33:51 2023","interfaces":[{"interface":"back","average":{"1min":4.6870000000000003,"5min":6.3239999999999998,"15min":5.0960000000000001},"min":{"1min":0.186,"5min":0.186,"15min":0.17100000000000001},"max":{"1min":24.023,"5min":109.446,"15min":128.47800000000001},"last":0.78500000000000003},{"interface":"front","average":{"1min":2.391,"5min":6.0069999999999997,"15min":4.9109999999999996},"min":{"1min":0.224,"5min":0.224,"15min":0.184},"max":{"1min":11.784000000000001,"5min":109.304,"15min":127.15600000000001},"last":0.27500000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:33:51 2023","interfaces":[{"interface":"back","average":{"1min":3.1669999999999998,"5min":3.7829999999999999,"15min":4.6760000000000002},"min":{"1min":0.24299999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.782999999999999,"5min":68.962000000000003,"15min":228.60499999999999},"last":1.153},{"interface":"front","average":{"1min":5.391,"5min":4.173,"15min":5.125},"min":{"1min":0.27500000000000002,"5min":0.221,"15min":0.221},"max":{"1min":24.196000000000002,"5min":68.819999999999993,"15min":228.28399999999999},"last":0.75900000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:33:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7589999999999999,"5min":1.4710000000000001,"15min":3.2989999999999999},"min":{"1min":0.28899999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":4.609,"5min":7.0919999999999996,"15min":62.909999999999997},"last":0.56599999999999995},{"interface":"front","average":{"1min":4.0339999999999998,"5min":2.7719999999999998,"15min":4.1459999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.219},"max":{"1min":13.226000000000001,"5min":13.226000000000001,"15min":78.594999999999999},"last":0.81399999999999995}]},{"osd":4,"last update":"Sun Oct 1 09:33:51 2023","interfaces":[{"interface":"back","average":{"1min":4.5830000000000002,"5min":2.093,"15min":2.8130000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":29.419,"5min":29.419,"15min":154.00399999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":4.0949999999999998,"5min":2.2210000000000001,"15min":2.9700000000000002},"min":{"1min":0.33600000000000002,"5min":0.309,"15min":0.23899999999999999},"max":{"1min":33.002000000000002,"5min":33.002000000000002,"15min":156.87700000000001},"last":0.34000000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:34:10 2023","interfaces":[{"interface":"back","average":{"1min":3.883,"5min":3.883,"15min":3.883},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":28.117999999999999,"5min":28.117999999999999,"15min":28.117999999999999},"last":0.313},{"interface":"front","average":{"1min":3.7250000000000001,"5min":3.7250000000000001,"15min":3.7250000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":28.34,"5min":28.34,"15min":28.34},"last":5.4589999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:34:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":1.4179999999999999,"15min":2.6419999999999999},"min":{"1min":0.50800000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.4480000000000004,"5min":7.6929999999999996,"15min":18.361000000000001},"last":3.4950000000000001},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.381,"15min":2.5800000000000001},"min":{"1min":0.497,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.5289999999999999,"5min":7.625,"15min":18.332000000000001},"last":3.4590000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:33:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2909999999999999,"5min":1.548,"15min":2.4329999999999998},"min":{"1min":0.48199999999999998,"5min":0.31,"15min":0.309},"max":{"1min":7.657,"5min":7.657,"15min":52.006},"last":0.92300000000000004},{"interface":"front","average":{"1min":3.3759999999999999,"5min":2.1099999999999999,"15min":2.4820000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":13.191000000000001,"5min":13.191000000000001,"15min":51.890999999999998},"last":0.86099999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019295,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1238656,"kb_used_data":644856,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":92506496,"statfs":{"total":95995035648,"available":94726651904,"internally_reserved":0,"allocated":660332544,"data_stored":659449493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:34:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":7.1550000000000002,"15min":4.7169999999999996},"min":{"1min":0.22600000000000001,"5min":0.224,"15min":0.223},"max":{"1min":4.8609999999999998,"5min":155.315,"15min":155.315},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.3100000000000001,"5min":7.7569999999999997,"15min":4.8730000000000002},"min":{"1min":0.29699999999999999,"5min":0.27900000000000003,"15min":0.23499999999999999},"max":{"1min":5.048,"5min":155.334,"15min":155.334},"last":1.96}]},{"osd":1,"last update":"Sun Oct 1 09:34:46 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":7.4119999999999999,"15min":5.3390000000000004},"min":{"1min":0.307,"5min":0.25,"15min":0.22600000000000001},"max":{"1min":14.456,"5min":190.61199999999999,"15min":190.61199999999999},"last":1.7050000000000001},{"interface":"front","average":{"1min":1.899,"5min":9.7439999999999998,"15min":6.1059999999999999},"min":{"1min":0.371,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":14.42,"5min":253.273,"15min":253.273},"last":3.4169999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:34:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":4.2430000000000003,"15min":3.2370000000000001},"min":{"1min":0.316,"5min":0.28299999999999997,"15min":0.26700000000000002},"max":{"1min":3.6099999999999999,"5min":170.45400000000001,"15min":170.45400000000001},"last":1.6890000000000001},{"interface":"front","average":{"1min":1.2689999999999999,"5min":3.0630000000000002,"15min":2.806},"min":{"1min":0.33300000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.8179999999999996,"5min":58.771000000000001,"15min":91.510999999999996},"last":2.6120000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:34:46 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":1.8560000000000001,"15min":2.7250000000000001},"min":{"1min":0.36799999999999999,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":2.9630000000000001,"5min":14.597,"15min":92.218000000000004},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.0069999999999999,"5min":2.1909999999999998,"15min":2.9649999999999999},"min":{"1min":0.39600000000000002,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":3.9209999999999998,"5min":23.231000000000002,"15min":92.766999999999996},"last":1.9870000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.085,"15min":1.085},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.367,"5min":5.367,"15min":5.367},"last":1.6180000000000001},{"interface":"front","average":{"1min":0.75,"5min":0.75,"15min":0.75},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.3929999999999998,"5min":3.3929999999999998,"15min":3.3929999999999998},"last":2.0270000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:34:00 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.04,"15min":0.73899999999999999},"min":{"1min":0.32300000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.9710000000000001,"5min":11.358000000000001,"15min":11.358000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.5469999999999999,"5min":1.0329999999999999,"15min":0.73999999999999999},"min":{"1min":0.32500000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.5229999999999997,"5min":10.076000000000001,"15min":10.076000000000001},"last":1.657}]},{"osd":7,"last update":"Sun Oct 1 09:34:28 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":1.0660000000000001,"15min":0.91000000000000003},"min":{"1min":0.39100000000000001,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":3.4630000000000001,"5min":11.384,"15min":11.384},"last":2.4820000000000002},{"interface":"front","average":{"1min":1.028,"5min":1.244,"15min":1.0740000000000001},"min":{"1min":0.42099999999999999,"5min":0.308,"15min":0.28699999999999998},"max":{"1min":3.585,"5min":10.103999999999999,"15min":10.103999999999999},"last":1.7210000000000001}]}]},{"osd":5,"up_from":379,"seq":1627792605205,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2364720,"kb_used_data":2000168,"kb_used_omap":13,"kb_used_meta":364530,"kb_avail":91380432,"statfs":{"total":95995035648,"available":93573562368,"internally_reserved":0,"allocated":2048172032,"data_stored":2047300813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":373279218},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:34:14 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.5049999999999999,"15min":1.5049999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.3550000000000004,"5min":8.3550000000000004,"15min":8.3550000000000004},"last":1.891},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.2310000000000001,"15min":1.2310000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.0869999999999997,"5min":5.0869999999999997,"15min":5.0869999999999997},"last":0.39500000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:34:14 2023","interfaces":[{"interface":"back","average":{"1min":1.698,"5min":1.698,"15min":1.698},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.773999999999999,"5min":10.773999999999999,"15min":10.773999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.395,"5min":1.395,"15min":1.395},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.3239999999999998,"5min":9.3239999999999998,"15min":9.3239999999999998},"last":0.24199999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:34:14 2023","interfaces":[{"interface":"back","average":{"1min":1.304,"5min":1.304,"15min":1.304},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.6790000000000003,"5min":4.6790000000000003,"15min":4.6790000000000003},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.6220000000000001,"5min":1.6220000000000001,"15min":1.6220000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":11.221,"5min":11.221,"15min":11.221},"last":0.55000000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:34:14 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.1619999999999999,"15min":1.1619999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":3.6269999999999998,"5min":3.6269999999999998,"15min":3.6269999999999998},"last":1.4830000000000001},{"interface":"front","average":{"1min":1.1779999999999999,"5min":1.1779999999999999,"15min":1.1779999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":4.4950000000000001,"5min":4.4950000000000001,"15min":4.4950000000000001},"last":3.2959999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":4.5279999999999996,"5min":4.5279999999999996,"15min":4.5279999999999996},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.6659999999999999,"15min":1.6659999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":14.68,"5min":14.68,"15min":14.68},"last":0.52700000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:34:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":3.7170000000000001,"5min":3.7170000000000001,"15min":3.7170000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":12.699999999999999,"5min":12.699999999999999,"15min":12.699999999999999},"last":0.46100000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:34:14 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":1.369,"15min":1.369},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":10.813000000000001,"5min":10.813000000000001,"15min":10.813000000000001},"last":7.7469999999999999},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":3.6000000000000001,"5min":3.6000000000000001,"15min":3.6000000000000001},"last":1.5129999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":915865600,"data_stored":915865600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2039611392,"data_stored":2039611392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":708313088,"data_stored":708313088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1935540224,"data_stored":1935540224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":672006144,"data_stored":672006144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2075590656,"data_stored":2075590656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":722665472,"data_stored":722665472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2173894656,"data_stored":2173894656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:34:55.205 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-01T09:34:55.205 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:34:55.205 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-10-01T09:34:55.205 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:34:55.205 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:34:55.206 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-10-01T09:34:55.206 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:34:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:34:54] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.33.4" 2023-10-01T09:34:55.252 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:55.366 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:34:55.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:55.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:34:55.525 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:55.571 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:55.679 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:55.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:55.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:55.702 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:34:55.737 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:34:55.756 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 39 16 63238 63222 101.282 70.4375 0.0110939 0.00986737 2023-10-01T09:34:55.769 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:34:55.783 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:55.791 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:34:55.797 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:34:55.816 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:55.822 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:34:55.865 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:34:55.872 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:55.899 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:55.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:55.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:55.908 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:34:55.949 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:34:55.961 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:34:55.961 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:34:55.962 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:34:55.962 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:34:55.962 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:34:55.962 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:34:55.962 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:34:55.962 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:34:56.004 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:56.042 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:56.206 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:56.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:56.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:56.304 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:56.304 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:34:56.304 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:56 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1904902389' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:56.304 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:56.304 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:34:56.305 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:56 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1904902389' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:56.305 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:56 smithi138 ceph-mon[185886]: pgmap v1138: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T09:34:56.311 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:34:56.364 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:34:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:34:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:56 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1904902389' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:34:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:56 smithi190 ceph-mon[160435]: pgmap v1138: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T09:34:56.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:56 smithi138 ceph-mon[192513]: pgmap v1138: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T09:34:56.756 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:34:56.756882+0000 min lat: 0.00117708 max lat: 0.300916 avg lat: 0.0099261 2023-10-01T09:34:56.756 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:34:56.756 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 40 16 64406 64390 100.575 73 0.00916932 0.0099261 2023-10-01T09:34:57.051 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:34:57.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:57.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:34:57.155 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:57.200 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:57.240 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:34:57.387 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:57.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:57.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:57.500 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:34:57.570 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:57.662 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:34:57.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:57.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:34:57.756 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 41 16 65621 65605 99.9738 75.9375 0.0140439 0.00999745 2023-10-01T09:34:57.769 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:57.835 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:57.941 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:57.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:57.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:57.970 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:34:57.970 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:34:57.970 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:34:57.970 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:34:57.970 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.970 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "MOSDPGPush(8.e 369/361 [PushOp(8:71da5c83:::benchmark_data_smithi138_431617_object81342:head, version: 342'5187, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:71da5c83:::benchmark_data_smithi138_431617_object81342:head@342'5187, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-01T09:34:57.970 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:45.778621+0000", 2023-10-01T09:34:57.971 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 132.18886833600001, 2023-10-01T09:34:57.971 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.101742287, 2023-10-01T09:34:57.971 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:57.971 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:34:57.971 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:57.971 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.971 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:57.971 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778621+0000", 2023-10-01T09:34:57.971 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:57.971 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.971 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.972 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:57.972 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778621+0000", 2023-10-01T09:34:57.972 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:34:57.972 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.972 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.972 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:57.972 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778620+0000", 2023-10-01T09:34:57.972 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9847999999999999e-05 2023-10-01T09:34:57.972 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.972 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.972 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:57.973 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778690+0000", 2023-10-01T09:34:57.973 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.265e-05 2023-10-01T09:34:57.973 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.973 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.973 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:57.973 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778733+0000", 2023-10-01T09:34:57.973 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9190000000000004e-06 2023-10-01T09:34:57.973 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.973 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.973 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:57.974 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778739+0000", 2023-10-01T09:34:57.974 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.101418614 2023-10-01T09:34:57.974 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.974 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.974 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:57.974 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.880157+0000", 2023-10-01T09:34:57.974 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0553e-05 2023-10-01T09:34:57.974 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.974 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.974 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:57.975 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.880168+0000", 2023-10-01T09:34:57.975 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00019595600000000001 2023-10-01T09:34:57.975 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.975 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.975 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:57.975 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.880364+0000", 2023-10-01T09:34:57.975 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.101742287 2023-10-01T09:34:57.975 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:57.975 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:57.975 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:57.976 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.976 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.976 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "pg_backfill(progress 8.e e 369/369 lb 8:71d9e500:::benchmark_data_smithi138_431617_object97540:head)", 2023-10-01T09:34:57.976 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:45.778749+0000", 2023-10-01T09:34:57.976 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 132.18874038300001, 2023-10-01T09:34:57.976 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.10167633199999999, 2023-10-01T09:34:57.976 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:57.976 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:34:57.976 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:57.976 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.976 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:57.976 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778749+0000", 2023-10-01T09:34:57.977 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:57.977 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.977 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.977 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:57.977 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778749+0000", 2023-10-01T09:34:57.977 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:57.979 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.980 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.980 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:57.980 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778749+0000", 2023-10-01T09:34:57.980 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3109999999999996e-06 2023-10-01T09:34:57.980 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.980 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.980 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:57.980 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778754+0000", 2023-10-01T09:34:57.980 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2940000000000002e-06 2023-10-01T09:34:57.980 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.981 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.981 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:57.981 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778761+0000", 2023-10-01T09:34:57.981 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9700000000000001e-06 2023-10-01T09:34:57.981 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.981 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.981 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:57.981 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778765+0000", 2023-10-01T09:34:57.981 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.10161822099999999 2023-10-01T09:34:57.981 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.981 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.982 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:57.982 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.880383+0000", 2023-10-01T09:34:57.982 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9059999999999997e-06 2023-10-01T09:34:57.982 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.982 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.982 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:57.982 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.880393+0000", 2023-10-01T09:34:57.982 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2981000000000002e-05 2023-10-01T09:34:57.982 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.982 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.982 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:57.983 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.880426+0000", 2023-10-01T09:34:57.983 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.10167633199999999 2023-10-01T09:34:57.983 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:57.983 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:57.983 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:57.983 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.983 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.983 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:177431 8.4 8:24757737:::benchmark_data_smithi138_431617_object76020:head [delete] snapc 0=[] ondisk+write+known_if_redirected e374)", 2023-10-01T09:34:57.983 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:59.731362+0000", 2023-10-01T09:34:57.983 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 118.236127477, 2023-10-01T09:34:57.983 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.091568272000000006, 2023-10-01T09:34:57.983 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:57.984 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:57.984 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:34:57.984 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:34:57.984 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:57.984 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 177431 2023-10-01T09:34:57.984 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.984 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:57.984 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.984 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:57.984 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731362+0000", 2023-10-01T09:34:57.984 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:57.985 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.985 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.985 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:57.985 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731362+0000", 2023-10-01T09:34:57.985 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:34:57.985 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.985 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.985 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:57.986 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731361+0000", 2023-10-01T09:34:57.986 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0410000000000003e-06 2023-10-01T09:34:57.986 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.986 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.986 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:57.986 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731367+0000", 2023-10-01T09:34:57.987 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2029999999999992e-06 2023-10-01T09:34:57.987 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.987 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.987 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:57.987 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731375+0000", 2023-10-01T09:34:57.990 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4909999999999999e-06 2023-10-01T09:34:57.990 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.990 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.990 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:57.990 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731380+0000", 2023-10-01T09:34:57.991 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7421e-05 2023-10-01T09:34:57.991 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.991 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.991 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:57.991 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731417+0000", 2023-10-01T09:34:57.991 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3317000000000003e-05 2023-10-01T09:34:57.992 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.992 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.992 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:57.992 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731491+0000", 2023-10-01T09:34:57.992 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.073553882000000001 2023-10-01T09:34:57.992 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.992 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.993 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:34:57.993 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.805044+0000", 2023-10-01T09:34:57.993 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0070079729999999998 2023-10-01T09:34:57.993 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.993 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.993 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:57.994 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.812052+0000", 2023-10-01T09:34:57.994 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0210999999999999e-05 2023-10-01T09:34:57.994 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.994 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.994 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:34:57.995 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.812063+0000", 2023-10-01T09:34:57.995 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010705756 2023-10-01T09:34:57.995 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.995 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.995 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:57.995 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.822768+0000", 2023-10-01T09:34:57.996 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3562e-05 2023-10-01T09:34:57.996 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.996 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.996 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:57.996 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.822822+0000", 2023-10-01T09:34:57.996 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010863500000000001 2023-10-01T09:34:57.996 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.996 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.996 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:57.997 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.822931+0000", 2023-10-01T09:34:57.997 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.091568272000000006 2023-10-01T09:34:57.997 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:57.997 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:57.997 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:57.997 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.997 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.997 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:177437 8.4 8:218da7f3:::benchmark_data_smithi138_431617_object76026:head [delete] snapc 0=[] ondisk+write+known_if_redirected e374)", 2023-10-01T09:34:57.997 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:59.736172+0000", 2023-10-01T09:34:57.997 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 118.231318184, 2023-10-01T09:34:57.997 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.087005369999999999, 2023-10-01T09:34:57.998 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:57.998 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:57.998 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:34:57.998 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:34:57.998 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:57.998 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 177437 2023-10-01T09:34:57.998 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.998 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:57.998 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.998 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:57.999 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.736172+0000", 2023-10-01T09:34:57.999 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:57.999 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.999 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.999 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:57.999 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.736172+0000", 2023-10-01T09:34:57.999 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:34:57.999 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:57.999 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:57.999 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:57.999 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.736170+0000", 2023-10-01T09:34:58.000 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9699999999999999e-06 2023-10-01T09:34:58.000 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.000 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.000 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.000 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.736178+0000", 2023-10-01T09:34:58.000 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.068732363000000005 2023-10-01T09:34:58.000 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.000 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.000 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.000 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.804910+0000", 2023-10-01T09:34:58.000 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2921999999999998e-05 2023-10-01T09:34:58.001 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.001 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.001 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.001 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.804933+0000", 2023-10-01T09:34:58.001 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003639083 2023-10-01T09:34:58.001 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.001 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.001 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.001 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.808572+0000", 2023-10-01T09:34:58.001 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000132459 2023-10-01T09:34:58.001 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.002 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.002 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.002 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.808705+0000", 2023-10-01T09:34:58.002 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000144124 2023-10-01T09:34:58.002 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.002 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.002 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:34:58.002 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.808849+0000", 2023-10-01T09:34:58.002 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014097115 2023-10-01T09:34:58.002 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.002 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.003 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:34:58.003 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.822946+0000", 2023-10-01T09:34:58.003 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8383000000000002e-05 2023-10-01T09:34:58.003 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.003 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.003 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:58.003 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.822984+0000", 2023-10-01T09:34:58.003 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8476000000000001e-05 2023-10-01T09:34:58.003 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.003 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.003 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:58.003 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.823033+0000", 2023-10-01T09:34:58.004 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3470000000000002e-05 2023-10-01T09:34:58.004 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.004 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.004 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.004 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.823076+0000", 2023-10-01T09:34:58.004 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000100905 2023-10-01T09:34:58.004 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.004 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.004 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.004 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.823177+0000", 2023-10-01T09:34:58.004 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.087005369999999999 2023-10-01T09:34:58.005 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.005 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.005 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.005 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.005 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.005 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:178396 8.4 8:230afef1:::benchmark_data_smithi138_431617_object76985:head [delete] snapc 0=[] ondisk+write+known_if_redirected e374)", 2023-10-01T09:34:58.005 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:01.128189+0000", 2023-10-01T09:34:58.005 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.83930117, 2023-10-01T09:34:58.005 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17065782800000001, 2023-10-01T09:34:58.005 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.005 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.005 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:34:58.006 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:34:58.006 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:58.006 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 178396 2023-10-01T09:34:58.006 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.006 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.006 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.006 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.006 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128189+0000", 2023-10-01T09:34:58.006 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.006 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.006 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.007 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.007 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128189+0000", 2023-10-01T09:34:58.007 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:34:58.007 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.007 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.007 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.007 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128187+0000", 2023-10-01T09:34:58.007 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3019999999999995e-06 2023-10-01T09:34:58.007 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.007 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.007 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.008 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128195+0000", 2023-10-01T09:34:58.008 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4068e-05 2023-10-01T09:34:58.008 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.008 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.008 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.008 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128209+0000", 2023-10-01T09:34:58.008 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0709999999999995e-06 2023-10-01T09:34:58.009 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.009 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.009 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.009 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128217+0000", 2023-10-01T09:34:58.009 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3971999999999998e-05 2023-10-01T09:34:58.009 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.009 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.009 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.009 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128271+0000", 2023-10-01T09:34:58.009 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000141423 2023-10-01T09:34:58.010 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.010 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.010 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.010 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128413+0000", 2023-10-01T09:34:58.010 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000146489 2023-10-01T09:34:58.010 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.010 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.010 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:34:58.010 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128559+0000", 2023-10-01T09:34:58.010 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16657522299999999 2023-10-01T09:34:58.011 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.011 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.011 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:34:58.011 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295134+0000", 2023-10-01T09:34:58.011 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0034864190000000001 2023-10-01T09:34:58.011 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.011 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.011 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:58.011 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.298621+0000", 2023-10-01T09:34:58.011 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1519999999999994e-05 2023-10-01T09:34:58.011 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.012 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.012 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:58.012 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.298682+0000", 2023-10-01T09:34:58.012 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2095999999999997e-05 2023-10-01T09:34:58.012 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.012 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.012 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.012 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.298734+0000", 2023-10-01T09:34:58.012 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000111989 2023-10-01T09:34:58.012 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.012 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.012 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.013 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.298846+0000", 2023-10-01T09:34:58.013 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17065782800000001 2023-10-01T09:34:58.013 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.013 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.013 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.013 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.013 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.013 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:178400 8.4 8:2fa69b17:::benchmark_data_smithi138_431617_object76989:head [delete] snapc 0=[] ondisk+write+known_if_redirected e374)", 2023-10-01T09:34:58.013 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:01.128235+0000", 2023-10-01T09:34:58.013 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.83925479, 2023-10-01T09:34:58.013 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.170802483, 2023-10-01T09:34:58.014 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.014 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.014 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:34:58.014 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:34:58.014 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:58.014 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 178400 2023-10-01T09:34:58.014 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.014 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.014 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.014 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.015 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128235+0000", 2023-10-01T09:34:58.015 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.015 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.015 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.015 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.015 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128235+0000", 2023-10-01T09:34:58.015 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:58.015 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.015 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.015 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.015 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128234+0000", 2023-10-01T09:34:58.016 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9519999999999999e-06 2023-10-01T09:34:58.016 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.016 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.016 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.016 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128237+0000", 2023-10-01T09:34:58.016 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0060000000000003e-06 2023-10-01T09:34:58.016 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.016 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.016 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.016 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128244+0000", 2023-10-01T09:34:58.016 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4630000000000003e-06 2023-10-01T09:34:58.016 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.017 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.017 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.017 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128249+0000", 2023-10-01T09:34:58.017 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024535020000000002 2023-10-01T09:34:58.017 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.017 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.017 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.017 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.130702+0000", 2023-10-01T09:34:58.017 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4159999999999994e-05 2023-10-01T09:34:58.017 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.017 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.018 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.018 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.130787+0000", 2023-10-01T09:34:58.018 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000105208 2023-10-01T09:34:58.018 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.018 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.018 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:34:58.018 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.130892+0000", 2023-10-01T09:34:58.018 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16425838400000001 2023-10-01T09:34:58.018 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.019 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.019 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:34:58.019 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295150+0000", 2023-10-01T09:34:58.019 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035032010000000001 2023-10-01T09:34:58.019 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.019 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.019 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:58.019 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.298653+0000", 2023-10-01T09:34:58.019 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00023980900000000001 2023-10-01T09:34:58.019 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.019 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.019 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:58.020 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.298893+0000", 2023-10-01T09:34:58.020 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.269e-05 2023-10-01T09:34:58.020 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.020 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.020 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.020 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.298936+0000", 2023-10-01T09:34:58.020 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010165 2023-10-01T09:34:58.020 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.020 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.020 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.020 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.299038+0000", 2023-10-01T09:34:58.021 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.170802483 2023-10-01T09:34:58.021 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.021 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.021 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.021 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.021 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.021 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:178405 8.e e374/361 8:7705199a:::benchmark_data_smithi138_431617_object76994:head v 374'11414, mlcod=374'11412)", 2023-10-01T09:34:58.021 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:01.133009+0000", 2023-10-01T09:34:58.021 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.834480934, 2023-10-01T09:34:58.021 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17295598600000001, 2023-10-01T09:34:58.021 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.021 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.022 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.022 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.022 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.022 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.133009+0000", 2023-10-01T09:34:58.022 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.022 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.022 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.022 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.022 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.133009+0000", 2023-10-01T09:34:58.022 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:58.023 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.023 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.023 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.023 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.133008+0000", 2023-10-01T09:34:58.023 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5589999999999999e-06 2023-10-01T09:34:58.023 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.023 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.023 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.023 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.133013+0000", 2023-10-01T09:34:58.023 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2519999999999996e-06 2023-10-01T09:34:58.023 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.023 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.024 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.024 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.133019+0000", 2023-10-01T09:34:58.024 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1579999999999998e-06 2023-10-01T09:34:58.024 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.024 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.024 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.024 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.133023+0000", 2023-10-01T09:34:58.024 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.162030965 2023-10-01T09:34:58.024 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.024 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.024 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.025 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295054+0000", 2023-10-01T09:34:58.025 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7665999999999999e-05 2023-10-01T09:34:58.025 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.025 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.025 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.025 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295122+0000", 2023-10-01T09:34:58.025 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010807071 2023-10-01T09:34:58.025 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.026 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.026 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.026 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.305929+0000", 2023-10-01T09:34:58.026 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5861000000000001e-05 2023-10-01T09:34:58.026 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.026 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.026 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.027 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.305965+0000", 2023-10-01T09:34:58.027 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17295598600000001 2023-10-01T09:34:58.027 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.027 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.027 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.027 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.027 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.028 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "MOSDPGPush(8.2 374/369 [PushOp(8:4bc097d3:::benchmark_data_smithi138_431617_object83895:head, version: 344'5152, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:4bc097d3:::benchmark_data_smithi138_431617_object83895:head@344'5152, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-01T09:34:58.028 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:01.134252+0000", 2023-10-01T09:34:58.028 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.83323754600001, 2023-10-01T09:34:58.028 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16124813499999999, 2023-10-01T09:34:58.028 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.028 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:34:58.029 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.029 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.029 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.029 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.134252+0000", 2023-10-01T09:34:58.029 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.029 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.030 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.030 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.030 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.134252+0000", 2023-10-01T09:34:58.030 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:34:58.030 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.030 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.031 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.031 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.134252+0000", 2023-10-01T09:34:58.031 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9670000000000001e-05 2023-10-01T09:34:58.031 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.031 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.031 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.031 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.134331+0000", 2023-10-01T09:34:58.032 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.160729709 2023-10-01T09:34:58.032 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.032 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.032 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.032 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295061+0000", 2023-10-01T09:34:58.032 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.515e-05 2023-10-01T09:34:58.032 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.032 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.033 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.033 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295106+0000", 2023-10-01T09:34:58.033 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011285099999999999 2023-10-01T09:34:58.033 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.033 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.033 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.033 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295219+0000", 2023-10-01T09:34:58.034 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4350000000000003e-06 2023-10-01T09:34:58.034 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.034 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.034 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.034 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295228+0000", 2023-10-01T09:34:58.034 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00027193200000000001 2023-10-01T09:34:58.034 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.034 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.034 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.035 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295500+0000", 2023-10-01T09:34:58.035 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16124813499999999 2023-10-01T09:34:58.035 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.035 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.035 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.035 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.035 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.035 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:178408 8.4 8:22b10d42:::benchmark_data_smithi138_431617_object76997:head [delete] snapc 0=[] ondisk+write+known_if_redirected e374)", 2023-10-01T09:34:58.035 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:01.136744+0000", 2023-10-01T09:34:58.035 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.830745756, 2023-10-01T09:34:58.035 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17711885099999999, 2023-10-01T09:34:58.036 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.036 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.036 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:34:58.036 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:34:58.036 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:58.036 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 178408 2023-10-01T09:34:58.036 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.036 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.036 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.036 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.036 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.136744+0000", 2023-10-01T09:34:58.037 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.037 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.037 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.037 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.037 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.136744+0000", 2023-10-01T09:34:58.037 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:34:58.037 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.037 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.037 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.037 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.136743+0000", 2023-10-01T09:34:58.037 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7950000000000002e-06 2023-10-01T09:34:58.037 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.038 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.038 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.038 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.136753+0000", 2023-10-01T09:34:58.038 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.158290036 2023-10-01T09:34:58.038 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.039 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.039 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.039 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295043+0000", 2023-10-01T09:34:58.039 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8039e-05 2023-10-01T09:34:58.039 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.039 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.039 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.040 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295081+0000", 2023-10-01T09:34:58.040 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0966999999999998e-05 2023-10-01T09:34:58.040 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.040 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.040 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.040 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295172+0000", 2023-10-01T09:34:58.040 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014039400000000001 2023-10-01T09:34:58.040 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.040 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.040 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.040 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295312+0000", 2023-10-01T09:34:58.041 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000162378 2023-10-01T09:34:58.041 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.041 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.041 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:34:58.041 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295474+0000", 2023-10-01T09:34:58.041 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010599005 2023-10-01T09:34:58.041 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.041 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.041 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:58.041 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.306073+0000", 2023-10-01T09:34:58.041 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1469e-05 2023-10-01T09:34:58.041 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.042 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.042 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:34:58.042 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.306085+0000", 2023-10-01T09:34:58.042 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0076060090000000004 2023-10-01T09:34:58.042 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.042 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.042 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:58.042 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.313691+0000", 2023-10-01T09:34:58.042 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7948999999999997e-05 2023-10-01T09:34:58.042 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.043 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.043 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.043 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.313749+0000", 2023-10-01T09:34:58.043 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011403100000000001 2023-10-01T09:34:58.043 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.043 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.043 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.043 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.313863+0000", 2023-10-01T09:34:58.043 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17711885099999999 2023-10-01T09:34:58.043 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.043 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.044 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.044 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.044 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.044 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:178401 8.b e374/361 8:de46d23f:::benchmark_data_smithi138_431617_object76990:head v 374'11065, mlcod=374'11063)", 2023-10-01T09:34:58.044 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:01.137716+0000", 2023-10-01T09:34:58.044 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.82977386500001, 2023-10-01T09:34:58.044 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.166376951, 2023-10-01T09:34:58.044 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.044 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.044 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.044 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.044 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.045 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.137716+0000", 2023-10-01T09:34:58.045 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.045 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.045 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.045 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.045 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.137716+0000", 2023-10-01T09:34:58.045 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:34:58.045 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.045 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.045 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.045 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.137715+0000", 2023-10-01T09:34:58.046 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.8770000000000005e-06 2023-10-01T09:34:58.046 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.046 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.046 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.046 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.137725+0000", 2023-10-01T09:34:58.046 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15732311399999999 2023-10-01T09:34:58.046 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.046 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.046 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.046 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295048+0000", 2023-10-01T09:34:58.046 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3589000000000001e-05 2023-10-01T09:34:58.047 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.047 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.047 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.047 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295072+0000", 2023-10-01T09:34:58.047 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1915000000000005e-05 2023-10-01T09:34:58.047 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.047 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.047 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.047 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295144+0000", 2023-10-01T09:34:58.047 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2493e-05 2023-10-01T09:34:58.048 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.048 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.048 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.048 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295166+0000", 2023-10-01T09:34:58.048 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0088856999999999998 2023-10-01T09:34:58.048 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.048 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.048 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.048 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.304052+0000", 2023-10-01T09:34:58.048 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1014999999999999e-05 2023-10-01T09:34:58.048 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.048 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.049 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.049 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.304093+0000", 2023-10-01T09:34:58.049 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.166376951 2023-10-01T09:34:58.049 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.049 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.049 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.049 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.049 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.049 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:178948 8.4 8:206e4418:::benchmark_data_smithi138_431617_object77537:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-10-01T09:34:58.049 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:02.137484+0000", 2023-10-01T09:34:58.050 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 115.830005637, 2023-10-01T09:34:58.050 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.117356637, 2023-10-01T09:34:58.050 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.050 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.050 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:34:58.050 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:34:58.050 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:58.050 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 178948 2023-10-01T09:34:58.050 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.050 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.051 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.051 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.051 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.137484+0000", 2023-10-01T09:34:58.051 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.051 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.051 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.051 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.051 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.137484+0000", 2023-10-01T09:34:58.051 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:34:58.051 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.052 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.052 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.052 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.137483+0000", 2023-10-01T09:34:58.052 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4009999999999996e-06 2023-10-01T09:34:58.052 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.052 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.052 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.052 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.137488+0000", 2023-10-01T09:34:58.052 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.102937139 2023-10-01T09:34:58.052 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.052 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.052 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.053 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240426+0000", 2023-10-01T09:34:58.053 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7660000000000002e-05 2023-10-01T09:34:58.053 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.053 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.053 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.053 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240463+0000", 2023-10-01T09:34:58.053 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6388999999999998e-05 2023-10-01T09:34:58.053 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.053 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.053 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.053 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240520+0000", 2023-10-01T09:34:58.054 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000149646 2023-10-01T09:34:58.054 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.054 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.054 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.054 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240669+0000", 2023-10-01T09:34:58.054 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016274799999999999 2023-10-01T09:34:58.054 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.054 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.054 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:34:58.055 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240832+0000", 2023-10-01T09:34:58.055 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.01014233 2023-10-01T09:34:58.055 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.055 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.055 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:34:58.055 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.250974+0000", 2023-10-01T09:34:58.055 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9999000000000001e-05 2023-10-01T09:34:58.055 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.055 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.055 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:58.055 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.251014+0000", 2023-10-01T09:34:58.056 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00366593 2023-10-01T09:34:58.056 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.056 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.056 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:58.056 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.254680+0000", 2023-10-01T09:34:58.056 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3010999999999998e-05 2023-10-01T09:34:58.056 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.056 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.056 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.056 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.254733+0000", 2023-10-01T09:34:58.056 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010754700000000001 2023-10-01T09:34:58.057 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.057 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.057 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.057 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.254841+0000", 2023-10-01T09:34:58.057 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.117356637 2023-10-01T09:34:58.057 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.057 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.057 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.057 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.057 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.058 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:178947 8.b e375/361 8:d41f59f4:::benchmark_data_smithi138_431617_object77536:head v 375'11098, mlcod=375'11095)", 2023-10-01T09:34:58.058 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:02.138023+0000", 2023-10-01T09:34:58.058 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 115.829466748, 2023-10-01T09:34:58.058 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.115564793, 2023-10-01T09:34:58.058 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.058 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.058 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.058 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.058 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.058 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.138023+0000", 2023-10-01T09:34:58.058 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.059 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.059 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.059 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.059 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.138023+0000", 2023-10-01T09:34:58.059 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:34:58.059 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.059 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.059 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.059 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.138022+0000", 2023-10-01T09:34:58.059 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5499999999999995e-06 2023-10-01T09:34:58.059 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.060 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.060 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.060 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.138031+0000", 2023-10-01T09:34:58.060 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.102400555 2023-10-01T09:34:58.060 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.060 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.060 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.060 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240431+0000", 2023-10-01T09:34:58.060 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6390000000000004e-06 2023-10-01T09:34:58.060 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.060 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.061 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.061 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240441+0000", 2023-10-01T09:34:58.061 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4252999999999997e-05 2023-10-01T09:34:58.061 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.061 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.061 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.061 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240495+0000", 2023-10-01T09:34:58.061 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6033999999999998e-05 2023-10-01T09:34:58.061 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.061 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.061 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.061 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240521+0000", 2023-10-01T09:34:58.062 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013027031 2023-10-01T09:34:58.062 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.062 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.062 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.062 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.253548+0000", 2023-10-01T09:34:58.062 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9552999999999999e-05 2023-10-01T09:34:58.062 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.062 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.062 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.062 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.253588+0000", 2023-10-01T09:34:58.063 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.115564793 2023-10-01T09:34:58.063 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.063 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.063 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.063 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.063 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.063 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:178951 8.e e375/361 8:76df1f5b:::benchmark_data_smithi138_431617_object77540:head v 375'11448, mlcod=375'11446)", 2023-10-01T09:34:58.063 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:02.148362+0000", 2023-10-01T09:34:58.063 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 115.81912807099999, 2023-10-01T09:34:58.063 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.099382195000000007, 2023-10-01T09:34:58.063 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.064 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.064 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.064 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.064 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.064 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.148362+0000", 2023-10-01T09:34:58.064 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.064 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.064 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.064 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.064 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.148362+0000", 2023-10-01T09:34:58.065 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:34:58.065 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.065 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.065 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.065 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.148361+0000", 2023-10-01T09:34:58.065 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8310000000000003e-06 2023-10-01T09:34:58.065 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.065 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.065 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.065 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.148368+0000", 2023-10-01T09:34:58.065 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.092058944000000004 2023-10-01T09:34:58.066 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.066 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.066 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.066 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240427+0000", 2023-10-01T09:34:58.066 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4754e-05 2023-10-01T09:34:58.066 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.066 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.066 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.066 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240441+0000", 2023-10-01T09:34:58.066 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3739000000000002e-05 2023-10-01T09:34:58.066 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.067 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.067 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.067 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240495+0000", 2023-10-01T09:34:58.067 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5982999999999998e-05 2023-10-01T09:34:58.067 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.068 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.068 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.068 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240521+0000", 2023-10-01T09:34:58.068 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.007186011 2023-10-01T09:34:58.068 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.068 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.068 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.069 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.247707+0000", 2023-10-01T09:34:58.069 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6711999999999997e-05 2023-10-01T09:34:58.069 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.069 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.069 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.069 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.247744+0000", 2023-10-01T09:34:58.069 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.099382195000000007 2023-10-01T09:34:58.069 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.069 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.069 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.069 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.070 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.070 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:187125 8.4 8:24541909:::benchmark_data_smithi138_431617_object85714:head [delete] snapc 0=[] ondisk+write+known_if_redirected e384)", 2023-10-01T09:34:58.070 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:18.925404+0000", 2023-10-01T09:34:58.070 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 99.042085513000004, 2023-10-01T09:34:58.070 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18668991800000001, 2023-10-01T09:34:58.071 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.071 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.071 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:34:58.071 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:34:58.071 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:58.072 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 187125 2023-10-01T09:34:58.072 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.072 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.072 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.072 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.072 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925404+0000", 2023-10-01T09:34:58.072 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925404+0000", 2023-10-01T09:34:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:34:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925402+0000", 2023-10-01T09:34:58.073 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.568e-06 2023-10-01T09:34:58.074 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.074 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.074 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.074 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925411+0000", 2023-10-01T09:34:58.074 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3375e-05 2023-10-01T09:34:58.074 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.075 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.075 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.075 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925424+0000", 2023-10-01T09:34:58.075 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4150000000000004e-06 2023-10-01T09:34:58.075 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.075 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.075 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.075 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925431+0000", 2023-10-01T09:34:58.075 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9450000000000002e-05 2023-10-01T09:34:58.075 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.076 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.076 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.076 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925490+0000", 2023-10-01T09:34:58.076 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000144986 2023-10-01T09:34:58.076 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.076 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.076 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.076 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925635+0000", 2023-10-01T09:34:58.076 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016538100000000001 2023-10-01T09:34:58.076 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.076 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.076 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:34:58.077 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925801+0000", 2023-10-01T09:34:58.077 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033733119999999998 2023-10-01T09:34:58.077 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.077 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.077 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:34:58.077 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.929174+0000", 2023-10-01T09:34:58.077 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019933059999999998 2023-10-01T09:34:58.077 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.077 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.077 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:58.077 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.931167+0000", 2023-10-01T09:34:58.078 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18060246999999999 2023-10-01T09:34:58.078 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.078 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.078 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:58.078 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.111770+0000", 2023-10-01T09:34:58.078 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3547000000000006e-05 2023-10-01T09:34:58.078 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.078 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.078 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.078 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.111843+0000", 2023-10-01T09:34:58.079 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00025091799999999999 2023-10-01T09:34:58.079 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.079 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.079 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.079 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.112094+0000", 2023-10-01T09:34:58.079 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18668991800000001 2023-10-01T09:34:58.079 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.079 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.079 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.079 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.079 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.079 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:187134 8.4 8:2f1ba86e:::benchmark_data_smithi138_431617_object85723:head [delete] snapc 0=[] ondisk+write+known_if_redirected e384)", 2023-10-01T09:34:58.080 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:18.945154+0000", 2023-10-01T09:34:58.080 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 99.022335823000006, 2023-10-01T09:34:58.080 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.167213626, 2023-10-01T09:34:58.080 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.080 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.080 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:34:58.080 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:34:58.080 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:34:58.080 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 187134 2023-10-01T09:34:58.080 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.080 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.081 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.081 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.081 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945154+0000", 2023-10-01T09:34:58.081 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.081 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.081 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.081 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.081 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945154+0000", 2023-10-01T09:34:58.081 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:34:58.081 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.082 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.082 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.082 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945152+0000", 2023-10-01T09:34:58.082 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2859e-05 2023-10-01T09:34:58.082 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.082 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.082 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.082 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945165+0000", 2023-10-01T09:34:58.082 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3360000000000001e-05 2023-10-01T09:34:58.082 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.082 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.083 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.083 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945178+0000", 2023-10-01T09:34:58.083 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2300000000000002e-06 2023-10-01T09:34:58.083 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.083 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.083 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.083 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945186+0000", 2023-10-01T09:34:58.083 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9648000000000003e-05 2023-10-01T09:34:58.083 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.083 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.084 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.084 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945235+0000", 2023-10-01T09:34:58.084 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9283000000000005e-05 2023-10-01T09:34:58.084 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.084 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.085 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.085 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945334+0000", 2023-10-01T09:34:58.085 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000114163 2023-10-01T09:34:58.085 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.085 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.085 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:34:58.085 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945449+0000", 2023-10-01T09:34:58.085 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0023364940000000002 2023-10-01T09:34:58.085 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.085 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.086 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:34:58.086 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.947785+0000", 2023-10-01T09:34:58.086 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0018238289999999999 2023-10-01T09:34:58.086 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.086 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.086 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:58.086 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.949609+0000", 2023-10-01T09:34:58.086 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.1625383 2023-10-01T09:34:58.086 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.086 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.086 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:34:58.087 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.112147+0000", 2023-10-01T09:34:58.087 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7358000000000002e-05 2023-10-01T09:34:58.087 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.087 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.087 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.087 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.112205+0000", 2023-10-01T09:34:58.087 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016299000000000001 2023-10-01T09:34:58.087 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.087 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.087 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.087 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.112368+0000", 2023-10-01T09:34:58.088 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.167213626 2023-10-01T09:34:58.088 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.088 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.088 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.088 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.088 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.088 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:187138 8.b e384/380 8:d4b7dda1:::benchmark_data_smithi138_431617_object85727:head v 384'11643, mlcod=384'11641)", 2023-10-01T09:34:58.088 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:18.957268+0000", 2023-10-01T09:34:58.088 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 99.010222271999993, 2023-10-01T09:34:58.088 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14524411300000001, 2023-10-01T09:34:58.088 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.089 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.089 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.089 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.089 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.089 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957268+0000", 2023-10-01T09:34:58.089 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.089 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.089 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.089 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.089 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957268+0000", 2023-10-01T09:34:58.089 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:34:58.089 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.090 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.090 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.090 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957266+0000", 2023-10-01T09:34:58.090 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4919000000000001e-05 2023-10-01T09:34:58.090 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.090 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.090 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.090 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957281+0000", 2023-10-01T09:34:58.090 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2416e-05 2023-10-01T09:34:58.090 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.091 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.091 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.091 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957303+0000", 2023-10-01T09:34:58.091 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2563e-05 2023-10-01T09:34:58.091 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.091 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.091 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.091 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957316+0000", 2023-10-01T09:34:58.091 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000103028 2023-10-01T09:34:58.091 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.091 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.091 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.092 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957419+0000", 2023-10-01T09:34:58.092 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4715999999999998e-05 2023-10-01T09:34:58.092 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.092 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.092 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.092 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957453+0000", 2023-10-01T09:34:58.092 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.145028781 2023-10-01T09:34:58.092 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.092 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.092 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.092 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.102482+0000", 2023-10-01T09:34:58.093 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9589000000000001e-05 2023-10-01T09:34:58.093 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.093 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.093 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.093 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.102512+0000", 2023-10-01T09:34:58.093 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14524411300000001 2023-10-01T09:34:58.093 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.093 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.093 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.093 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.093 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.094 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:187139 8.2 e384/380 8:45fc0a90:::benchmark_data_smithi138_431617_object85728:head v 384'11540, mlcod=368'10456)", 2023-10-01T09:34:58.094 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:18.957444+0000", 2023-10-01T09:34:58.094 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 99.010045796, 2023-10-01T09:34:58.094 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14461728400000001, 2023-10-01T09:34:58.094 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.094 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.094 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.094 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.094 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.094 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957444+0000", 2023-10-01T09:34:58.094 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.095 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.095 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.095 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.095 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957444+0000", 2023-10-01T09:34:58.095 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:34:58.095 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.095 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.095 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.095 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957443+0000", 2023-10-01T09:34:58.095 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8043e-05 2023-10-01T09:34:58.095 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.095 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.096 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.096 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957461+0000", 2023-10-01T09:34:58.096 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14196397799999999 2023-10-01T09:34:58.096 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.096 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.096 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.096 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.099425+0000", 2023-10-01T09:34:58.096 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4381e-05 2023-10-01T09:34:58.096 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.096 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.096 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.097 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.099439+0000", 2023-10-01T09:34:58.097 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3649999999999997e-05 2023-10-01T09:34:58.097 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.097 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.097 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.097 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.099483+0000", 2023-10-01T09:34:58.097 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2201999999999999e-05 2023-10-01T09:34:58.097 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.097 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.097 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.097 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.099505+0000", 2023-10-01T09:34:58.098 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002511629 2023-10-01T09:34:58.098 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.098 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.098 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.098 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.102016+0000", 2023-10-01T09:34:58.098 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4898000000000003e-05 2023-10-01T09:34:58.098 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.098 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.098 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.098 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.102061+0000", 2023-10-01T09:34:58.098 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14461728400000001 2023-10-01T09:34:58.098 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.099 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.099 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.099 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.099 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.099 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:187962 8.2 e384/380 8:4458b8ad:::benchmark_data_smithi138_431617_object86551:head v 384'11600, mlcod=368'10456)", 2023-10-01T09:34:58.099 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:20.442352+0000", 2023-10-01T09:34:58.099 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 97.525138224000003, 2023-10-01T09:34:58.099 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14790603499999999, 2023-10-01T09:34:58.099 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.099 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.099 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.100 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.100 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.100 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442352+0000", 2023-10-01T09:34:58.100 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.100 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.100 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.100 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.100 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442352+0000", 2023-10-01T09:34:58.100 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:34:58.101 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.101 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.101 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.101 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442350+0000", 2023-10-01T09:34:58.101 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5109000000000001e-05 2023-10-01T09:34:58.101 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.101 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.101 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.101 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442365+0000", 2023-10-01T09:34:58.101 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.383e-05 2023-10-01T09:34:58.101 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.101 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.102 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.102 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442379+0000", 2023-10-01T09:34:58.102 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8100000000000003e-06 2023-10-01T09:34:58.102 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.102 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.102 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.102 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442385+0000", 2023-10-01T09:34:58.102 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3603000000000002e-05 2023-10-01T09:34:58.102 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.102 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.102 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.103 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442429+0000", 2023-10-01T09:34:58.103 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7490999999999999e-05 2023-10-01T09:34:58.103 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.103 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.103 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.103 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442446+0000", 2023-10-01T09:34:58.103 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.147771334 2023-10-01T09:34:58.103 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.103 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.103 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.103 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.590217+0000", 2023-10-01T09:34:58.104 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0160000000000002e-05 2023-10-01T09:34:58.104 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.104 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.104 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.104 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.590258+0000", 2023-10-01T09:34:58.104 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14790603499999999 2023-10-01T09:34:58.104 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.104 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.104 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.104 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.104 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.105 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:187966 8.5 e384/367 8:a88a80c6:::benchmark_data_smithi138_431617_object86555:head v 384'11853, mlcod=384'11851)", 2023-10-01T09:34:58.105 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:20.448350+0000", 2023-10-01T09:34:58.105 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 97.519139777000007, 2023-10-01T09:34:58.105 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14374200000000001, 2023-10-01T09:34:58.105 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.105 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.105 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.105 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.105 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.105 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448350+0000", 2023-10-01T09:34:58.105 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.105 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.106 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.106 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.106 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448350+0000", 2023-10-01T09:34:58.106 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:34:58.106 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.106 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.106 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.106 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448349+0000", 2023-10-01T09:34:58.106 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5560000000000008e-06 2023-10-01T09:34:58.106 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.106 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.107 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.107 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448358+0000", 2023-10-01T09:34:58.107 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13449048599999999 2023-10-01T09:34:58.107 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.107 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.107 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.107 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.582848+0000", 2023-10-01T09:34:58.107 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0599e-05 2023-10-01T09:34:58.108 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.108 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.108 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.108 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.582859+0000", 2023-10-01T09:34:58.108 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4080999999999999e-05 2023-10-01T09:34:58.108 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.108 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.108 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.109 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.582913+0000", 2023-10-01T09:34:58.109 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4927999999999999e-05 2023-10-01T09:34:58.109 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.109 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.109 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.109 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.582938+0000", 2023-10-01T09:34:58.109 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0091225230000000004 2023-10-01T09:34:58.109 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.109 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.109 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.109 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.592060+0000", 2023-10-01T09:34:58.110 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1761000000000003e-05 2023-10-01T09:34:58.110 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.110 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.110 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.110 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.592092+0000", 2023-10-01T09:34:58.110 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14374200000000001 2023-10-01T09:34:58.110 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.110 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.110 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.110 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.110 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.111 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:187967 8.b e384/380 8:d518ca2e:::benchmark_data_smithi138_431617_object86556:head v 384'11699, mlcod=384'11697)", 2023-10-01T09:34:58.111 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:20.448508+0000", 2023-10-01T09:34:58.111 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 97.518981331000006, 2023-10-01T09:34:58.111 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13899778400000001, 2023-10-01T09:34:58.111 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:34:58.111 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:34:58.111 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:34:58.111 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.111 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:34:58.111 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448508+0000", 2023-10-01T09:34:58.111 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:34:58.112 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.112 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.112 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:34:58.112 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448508+0000", 2023-10-01T09:34:58.112 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:34:58.112 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.112 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.112 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:34:58.112 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448508+0000", 2023-10-01T09:34:58.112 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1119e-05 2023-10-01T09:34:58.112 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.112 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.113 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:34:58.113 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448519+0000", 2023-10-01T09:34:58.113 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13449881799999999 2023-10-01T09:34:58.113 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.113 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.113 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:34:58.113 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.583018+0000", 2023-10-01T09:34:58.113 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8060000000000002e-06 2023-10-01T09:34:58.113 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.113 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.113 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:34:58.114 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.583022+0000", 2023-10-01T09:34:58.114 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010691099999999999 2023-10-01T09:34:58.114 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.114 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.114 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:34:58.114 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.583129+0000", 2023-10-01T09:34:58.114 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9769999999999999e-05 2023-10-01T09:34:58.114 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.114 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.114 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:34:58.114 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.583149+0000", 2023-10-01T09:34:58.115 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0043129889999999997 2023-10-01T09:34:58.115 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.115 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.115 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:34:58.115 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.587462+0000", 2023-10-01T09:34:58.115 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4264000000000002e-05 2023-10-01T09:34:58.115 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:34:58.115 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:34:58.115 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:34:58.115 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.587506+0000", 2023-10-01T09:34:58.116 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13899778400000001 2023-10-01T09:34:58.116 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.116 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.116 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.116 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:34:58.116 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:34:58.116 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:34:58.116 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:34:58.176 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:58.307 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:34:58.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:58.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:34:58.327 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:34:58.418 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:58.465 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:58.536 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:58.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:58.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:58.641 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:34:58.684 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:34:58 smithi190 ceph-mon[160435]: pgmap v1139: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T09:34:58.689 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:58.756 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 42 16 66821 66805 99.3792 75 0.0108086 0.0100559 2023-10-01T09:34:58.767 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:34:58 smithi138 ceph-mon[192513]: pgmap v1139: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T09:34:58.769 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:34:58 smithi138 ceph-mon[185886]: pgmap v1139: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T09:34:58.795 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:34:58.795 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "" 2023-10-01T09:34:58.795 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:34:59.042 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:34:59.048 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:34:59.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:59.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:34:59.152 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:34:59.175 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:34:59.209 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:59.402 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:34:59.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:59.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:34:59.507 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:34:59.552 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:34:59.643 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:34:59.658 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:34:59.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:34:59.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:34:59.756 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 43 16 67894 67878 98.6277 67.0625 0.00824966 0.0101334 2023-10-01T09:34:59.769 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:34:59.833 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:00.182 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:00 smithi190 ceph-mon[160435]: pgmap v1140: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T09:35:00.290 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:00.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:00.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:00.406 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:00.463 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:00 smithi138 ceph-mon[185886]: pgmap v1140: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T09:35:00.463 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:00 smithi138 ceph-mon[192513]: pgmap v1140: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T09:35:00.468 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:00.757 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 44 16 69122 69106 98.1302 76.75 0.0141949 0.0101854 2023-10-01T09:35:00.767 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:00.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:00.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:00.876 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:00.958 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:01.076 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:01.273 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:01.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:01.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:01.380 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:01.443 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:01.582 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:01.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:01.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:01.692 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:01.759 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 45 16 70276 70260 97.5474 72.125 0.017138 0.0102462 2023-10-01T09:35:01.765 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:01.834 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:35:01.834 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:35:01.834 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.834 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.46067.0:70347 9.5 e410/400 9:a3f660d6:::benchmark_data_smithi138_497829_object70346:head v 410'4445, mlcod=410'4443)", 2023-10-01T09:35:01.834 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:35:01.830978+0000", 2023-10-01T09:35:01.834 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0019720459999999999, 2023-10-01T09:35:01.834 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019991179999999998, 2023-10-01T09:35:01.834 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:01.834 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:35:01.835 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:01.835 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.835 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:01.835 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.830978+0000", 2023-10-01T09:35:01.835 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:01.835 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.835 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.835 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:01.835 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.830978+0000", 2023-10-01T09:35:01.835 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:35:01.835 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.835 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.836 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:01.836 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.830977+0000", 2023-10-01T09:35:01.836 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4501000000000001e-05 2023-10-01T09:35:01.836 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.836 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.836 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:01.836 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.831011+0000", 2023-10-01T09:35:01.836 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5438e-05 2023-10-01T09:35:01.836 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.836 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.836 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:01.836 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.831027+0000", 2023-10-01T09:35:01.837 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8279999999999997e-06 2023-10-01T09:35:01.837 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.837 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.837 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:01.837 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.831034+0000", 2023-10-01T09:35:01.837 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6603000000000001e-05 2023-10-01T09:35:01.837 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.837 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.837 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:01.837 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.831080+0000", 2023-10-01T09:35:01.837 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4121000000000002e-05 2023-10-01T09:35:01.837 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.838 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.838 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:01.838 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.831104+0000", 2023-10-01T09:35:01.838 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000126546 2023-10-01T09:35:01.838 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:01.838 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:01.838 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:01.838 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.838 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.838 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:70351 9.c 9:3280215a:::benchmark_data_smithi138_497829_object70350:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e410)", 2023-10-01T09:35:01.838 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:35:01.831892+0000", 2023-10-01T09:35:01.839 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0010577989999999999, 2023-10-01T09:35:01.839 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0011585580000000001, 2023-10-01T09:35:01.839 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:01.839 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:35:01.839 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:01.839 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:35:01.839 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:35:01.839 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 70351 2023-10-01T09:35:01.839 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.839 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:01.839 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.839 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:01.839 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.831892+0000", 2023-10-01T09:35:01.840 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:01.840 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.840 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.840 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:01.840 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.831892+0000", 2023-10-01T09:35:01.840 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:01.840 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.840 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.840 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:01.840 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.831891+0000", 2023-10-01T09:35:01.840 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1295000000000002e-05 2023-10-01T09:35:01.840 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.841 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.841 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:01.841 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.831952+0000", 2023-10-01T09:35:01.841 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3189e-05 2023-10-01T09:35:01.841 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.841 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.841 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:01.841 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.831965+0000", 2023-10-01T09:35:01.841 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0249999999999999e-06 2023-10-01T09:35:01.841 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.841 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.841 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:01.841 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.831971+0000", 2023-10-01T09:35:01.842 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7423000000000003e-05 2023-10-01T09:35:01.842 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.842 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.842 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:01.842 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.832018+0000", 2023-10-01T09:35:01.842 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013101399999999999 2023-10-01T09:35:01.842 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.842 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.842 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:01.842 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.832149+0000", 2023-10-01T09:35:01.842 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015956600000000001 2023-10-01T09:35:01.843 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.843 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.843 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:35:01.843 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.832309+0000", 2023-10-01T09:35:01.843 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00041709499999999998 2023-10-01T09:35:01.843 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:01.843 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:01.843 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:01.843 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.843 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.843 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:70340 9.1 9:886fa110:::benchmark_data_smithi138_497829_object70339:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e410)", 2023-10-01T09:35:01.843 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:35:01.818445+0000", 2023-10-01T09:35:01.844 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.014505113, 2023-10-01T09:35:01.844 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014688521, 2023-10-01T09:35:01.844 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:01.844 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:35:01.844 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:01.844 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:35:01.844 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:35:01.844 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 70340 2023-10-01T09:35:01.844 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.844 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:01.844 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.844 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:01.845 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.818445+0000", 2023-10-01T09:35:01.845 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:01.845 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.845 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.845 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:01.845 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.818445+0000", 2023-10-01T09:35:01.845 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:35:01.845 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.845 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.845 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:01.845 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.818444+0000", 2023-10-01T09:35:01.845 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6372999999999999e-05 2023-10-01T09:35:01.846 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.846 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.846 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:01.846 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.818480+0000", 2023-10-01T09:35:01.846 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5889999999999992e-06 2023-10-01T09:35:01.846 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.846 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.846 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:01.846 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.818490+0000", 2023-10-01T09:35:01.846 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1220000000000003e-06 2023-10-01T09:35:01.846 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.846 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.847 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:01.847 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.818495+0000", 2023-10-01T09:35:01.847 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1792999999999997e-05 2023-10-01T09:35:01.847 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.847 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.847 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:01.847 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.818547+0000", 2023-10-01T09:35:01.847 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012326399999999999 2023-10-01T09:35:01.847 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.847 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.847 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:01.847 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.818670+0000", 2023-10-01T09:35:01.847 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000159486 2023-10-01T09:35:01.848 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.848 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:35:01.848 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.818830+0000", 2023-10-01T09:35:01.848 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002236399 2023-10-01T09:35:01.848 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.848 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:01.848 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.821066+0000", 2023-10-01T09:35:01.848 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0040592600000000003 2023-10-01T09:35:01.848 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.849 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:01.849 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.825125+0000", 2023-10-01T09:35:01.849 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0066806160000000003 2023-10-01T09:35:01.849 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:01.849 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:01.849 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:01.849 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.849 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.849 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:70345 9.d 9:b2c70810:::benchmark_data_smithi138_497829_object70344:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e410)", 2023-10-01T09:35:01.849 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:35:01.824669+0000", 2023-10-01T09:35:01.849 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0082807999999999996, 2023-10-01T09:35:01.849 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0085629899999999995, 2023-10-01T09:35:01.850 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:01.850 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:35:01.850 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:01.850 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:35:01.850 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:35:01.850 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 70345 2023-10-01T09:35:01.850 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.850 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:01.850 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.850 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:01.850 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.824669+0000", 2023-10-01T09:35:01.851 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:01.851 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.851 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.851 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:01.851 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.824669+0000", 2023-10-01T09:35:01.851 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:01.851 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.851 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.851 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:01.851 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.824667+0000", 2023-10-01T09:35:01.851 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2674000000000002e-05 2023-10-01T09:35:01.851 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.852 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.852 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:01.852 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.824740+0000", 2023-10-01T09:35:01.852 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5094e-05 2023-10-01T09:35:01.852 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.852 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.852 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:01.852 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.824755+0000", 2023-10-01T09:35:01.852 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9209999999999997e-06 2023-10-01T09:35:01.852 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.852 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.852 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:01.853 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.824761+0000", 2023-10-01T09:35:01.853 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2373999999999998e-05 2023-10-01T09:35:01.853 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.853 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.853 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:01.853 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.824803+0000", 2023-10-01T09:35:01.853 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012330899999999999 2023-10-01T09:35:01.853 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.853 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.853 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:01.853 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.824927+0000", 2023-10-01T09:35:01.853 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014468799999999999 2023-10-01T09:35:01.854 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.854 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.854 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,5", 2023-10-01T09:35:01.854 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.825071+0000", 2023-10-01T09:35:01.854 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035168339999999999 2023-10-01T09:35:01.854 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:01.854 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:01.854 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:01.854 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:35:01.828588+0000", 2023-10-01T09:35:01.854 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0039191549999999997 2023-10-01T09:35:01.854 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:01.855 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:01.855 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:01.855 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:01.855 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:35:01.855 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 4 2023-10-01T09:35:01.855 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:35:02.054 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:35:02.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:02 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/941932853' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:35:02.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:02 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/941932853' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:35:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:02 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/941932853' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:35:02.471 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:02.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:02.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:02.576 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:02.649 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:02.759 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 46 16 71157 71141 96.6239 55.0625 0.0116813 0.0103445 2023-10-01T09:35:02.808 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:02.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:02.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:02.849 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:35:02.926 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:02.973 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:03.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:03 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:03 smithi190 ceph-mon[160435]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:03 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/941932853' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:35:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:03 smithi190 ceph-mon[160435]: osdmap e411: 8 total, 8 up, 8 in 2023-10-01T09:35:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:03 smithi190 ceph-mon[160435]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:03 smithi190 ceph-mon[160435]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:03 smithi190 ceph-mon[160435]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:03 smithi190 ceph-mon[160435]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:03 smithi190 ceph-mon[160435]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:03 smithi190 ceph-mon[160435]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:03 smithi190 ceph-mon[160435]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:03 smithi190 ceph-mon[160435]: pgmap v1142: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-01T09:35:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[185886]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/941932853' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:35:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[185886]: osdmap e411: 8 total, 8 up, 8 in 2023-10-01T09:35:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[185886]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[185886]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[185886]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[185886]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[185886]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[185886]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[185886]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[185886]: pgmap v1142: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-01T09:35:03.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:03.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[192513]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/941932853' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:35:03.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[192513]: osdmap e411: 8 total, 8 up, 8 in 2023-10-01T09:35:03.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[192513]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[192513]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[192513]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[192513]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[192513]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[192513]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[192513]: failed to encode map e411 with expected crc 2023-10-01T09:35:03.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:03 smithi138 ceph-mon[192513]: pgmap v1142: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-01T09:35:03.334 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:35:03.588 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:03.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:03.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:03.693 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:03.756 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:03.759 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 47 16 72080 72064 95.7954 57.6875 0.00886849 0.0104327 2023-10-01T09:35:03.870 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:03.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:03.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:03.977 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:04.040 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:04.149 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:04.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:04.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:04.259 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:04.296 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:35:04.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:04 smithi190 ceph-mon[160435]: pgmap v1143: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T09:35:04.319 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:04 smithi138 ceph-mon[185886]: pgmap v1143: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T09:35:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:04 smithi138 ceph-mon[192513]: pgmap v1143: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T09:35:04.330 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:35:04.360 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:35:04.383 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:04.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:04.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:04.386 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:35:04.418 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:35:04.453 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:35:04.486 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:04.490 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:35:04.531 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:35:04.538 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:04.546 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:35:04.547 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:35:04.547 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:35:04.547 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:35:04.547 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:35:04.547 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:35:04.547 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:35:04.547 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:35:04.759 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 48 16 73193 73177 95.249 69.5625 0.0146174 0.0104937 2023-10-01T09:35:04.806 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:04.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:04.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:04.913 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:04.966 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:05.013 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:05.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:05.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:05.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:35:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:35:04] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-10-01T09:35:05.116 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:05.144 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:05.189 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:05.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:05.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:05.293 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:05.307 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:35:05.308 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:35:05.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3626382316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:05.320 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 2, 5, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:35:05.321 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:35:05.321 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 1, 3, 7, 2, 5, 6, 4] 2023-10-01T09:35:05.321 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-01T09:35:05.357 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:05.544 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:05.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:05.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:05.551 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3626382316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:05.552 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3626382316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:05.653 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:05.678 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:35:05.695 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:05.759 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 49 16 74068 74052 94.421 54.6875 0.0111764 0.010586 2023-10-01T09:35:05.770 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:05.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:05.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:05.879 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:05.908 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:05.978 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:05.984 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:05.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:05.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:06.088 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:06.147 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:06.305 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:06 smithi190 ceph-mon[160435]: pgmap v1144: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T09:35:06.324 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:06.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:06.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:06.337 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:35:06.431 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:06.475 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:06 smithi138 ceph-mon[185886]: pgmap v1144: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T09:35:06.476 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:06 smithi138 ceph-mon[192513]: pgmap v1144: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T09:35:06.481 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:06.599 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:06.636 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:06.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:06.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:06.743 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:06.760 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 50 16 75080 75064 93.7974 63.25 0.013545 0.0106557 2023-10-01T09:35:06.800 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:06.902 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:06.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:06.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:06.984 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:35:07.006 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:07.057 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:07.136 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:07.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:07.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:07.245 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:07.300 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:07.400 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:07.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:07.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:07.504 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:07.559 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:07.663 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:35:07.725 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:07.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:07.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:07.764 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 51 16 76217 76201 93.3479 71.0625 0.0252225 0.0107068 2023-10-01T09:35:07.834 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:07.881 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:07.977 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:07.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:07.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:08.084 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:08.126 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:08.200 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:08.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:08.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:08.304 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:08.349 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:08.516 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:08 smithi138 ceph-mon[192513]: pgmap v1145: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-01T09:35:08.561 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:08.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:08.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:08.668 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:08.692 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:08 smithi190 ceph-mon[160435]: pgmap v1145: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-01T09:35:08.695 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:08.762 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 52 16 77135 77119 92.6562 57.375 0.0144633 0.0107872 2023-10-01T09:35:08.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:08 smithi138 ceph-mon[185886]: pgmap v1145: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-10-01T09:35:08.826 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:08.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:08.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:08.942 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:09.008 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:09.134 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:35:09.134 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:35:09.324 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:09.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:09.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:09.408 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:09.434 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:09.495 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:09.593 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:09.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:09.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:09.697 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:09.746 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:09.765 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 53 16 77956 77940 91.8715 51.3125 0.0166775 0.0108791 2023-10-01T09:35:09.887 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:09 smithi138 conmon[185862]: 2023-10-01T09:35:09.881+0000 7f70d64ec700 -1 mon.a@0(leader).osd e411 definitely_dead 0 2023-10-01T09:35:09.978 INFO:teuthology.orchestra.run.smithi138.stderr:marked out osd.2. 2023-10-01T09:35:10.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:09 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1590929061' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:35:10.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:09 smithi190 ceph-mon[160435]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-01T09:35:10.083 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:35:10.084 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:10.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:10.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:10.190 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:10.216 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:10.224 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:09 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1590929061' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:35:10.224 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:09 smithi138 ceph-mon[192513]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-01T09:35:10.224 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:09 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1590929061' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:35:10.225 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:09 smithi138 ceph-mon[185886]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-01T09:35:10.259 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:10.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:10.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:10.364 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:10.391 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:10.436 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:10.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:10.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:10.457 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:35:10.542 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:10.571 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:10.765 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 54 16 78247 78231 90.5073 18.1875 0.00309865 0.0108924 2023-10-01T09:35:10.962 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:10 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1590929061' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-01T09:35:10.962 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:10 smithi190 ceph-mon[160435]: osdmap e412: 8 total, 8 up, 7 in 2023-10-01T09:35:10.962 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:10 smithi190 ceph-mon[160435]: failed to encode map e412 with expected crc 2023-10-01T09:35:10.962 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:10 smithi190 ceph-mon[160435]: failed to encode map e412 with expected crc 2023-10-01T09:35:10.963 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:10 smithi190 ceph-mon[160435]: failed to encode map e412 with expected crc 2023-10-01T09:35:10.963 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:10 smithi190 ceph-mon[160435]: pgmap v1147: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T09:35:10.963 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:10 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2744437474' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:35:10.963 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:10 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:35:10.963 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:10.963 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:35:10.975 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:35:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1590929061' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-01T09:35:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[185886]: osdmap e412: 8 total, 8 up, 7 in 2023-10-01T09:35:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[185886]: failed to encode map e412 with expected crc 2023-10-01T09:35:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[185886]: failed to encode map e412 with expected crc 2023-10-01T09:35:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[185886]: failed to encode map e412 with expected crc 2023-10-01T09:35:11.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[185886]: pgmap v1147: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T09:35:11.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2744437474' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:35:11.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:35:11.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:11.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:35:11.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1590929061' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-01T09:35:11.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[192513]: osdmap e412: 8 total, 8 up, 7 in 2023-10-01T09:35:11.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[192513]: failed to encode map e412 with expected crc 2023-10-01T09:35:11.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[192513]: failed to encode map e412 with expected crc 2023-10-01T09:35:11.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[192513]: failed to encode map e412 with expected crc 2023-10-01T09:35:11.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[192513]: pgmap v1147: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T09:35:11.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2744437474' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:35:11.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:35:11.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:11.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:35:11.375 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:11.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:11.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:11.479 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:11.508 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:11.655 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:11.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:11.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:11.763 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:11.766 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 55 16 78247 78231 88.8623 0 - 0.0108924 2023-10-01T09:35:11.791 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:12.047 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:12.048 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[192513]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.048 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[192513]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.048 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:35:12.048 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[192513]: osdmap e413: 8 total, 8 up, 7 in 2023-10-01T09:35:12.048 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[192513]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.048 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[192513]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.048 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[192513]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.048 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[192513]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.048 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[192513]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.049 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:12.049 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[185886]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.049 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[185886]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.049 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:35:12.049 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[185886]: osdmap e413: 8 total, 8 up, 7 in 2023-10-01T09:35:12.049 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[185886]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.049 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[185886]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.049 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[185886]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.049 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[185886]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.050 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:11 smithi138 ceph-mon[185886]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.070 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:12.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:12.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:12.116 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:35:12.117 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:35:12.159 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:11 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:12.159 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:11 smithi190 ceph-mon[160435]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.159 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:11 smithi190 ceph-mon[160435]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.159 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:11 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:35:12.159 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:11 smithi190 ceph-mon[160435]: osdmap e413: 8 total, 8 up, 7 in 2023-10-01T09:35:12.160 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:11 smithi190 ceph-mon[160435]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.160 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:11 smithi190 ceph-mon[160435]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.160 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:11 smithi190 ceph-mon[160435]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.160 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:11 smithi190 ceph-mon[160435]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.160 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:11 smithi190 ceph-mon[160435]: failed to encode map e413 with expected crc 2023-10-01T09:35:12.174 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:12.231 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:12.301 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:12.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:12.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:12.405 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:12.485 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:12.543 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:35:12.549 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:12.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:12.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:12.563 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:35:12.581 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:35:12.599 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:35:12.616 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:35:12.654 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:12.665 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:35:12.699 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:35:12.715 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:12.732 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:35:12.738 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:35:12.738 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:35:12.738 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:35:12.738 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:35:12.738 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:35:12.738 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:35:12.738 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:35:12.738 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:35:12.765 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 56 16 78578 78562 87.6452 10.3438 0.0116498 0.0114024 2023-10-01T09:35:12.809 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:12.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:12.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:12.912 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:12.980 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:13.000 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:35:13.071 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:13.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:13.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:13.177 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:13.210 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:13.260 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:13.271 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:12 smithi190 ceph-mon[160435]: failed to encode map e414 with expected crc 2023-10-01T09:35:13.271 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:12 smithi190 ceph-mon[160435]: failed to encode map e414 with expected crc 2023-10-01T09:35:13.271 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:12 smithi190 ceph-mon[160435]: osdmap e414: 8 total, 8 up, 7 in 2023-10-01T09:35:13.272 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:12 smithi190 ceph-mon[160435]: failed to encode map e414 with expected crc 2023-10-01T09:35:13.272 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:12 smithi190 ceph-mon[160435]: pgmap v1150: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 990 op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-01T09:35:13.272 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:12 smithi190 ceph-mon[160435]: failed to encode map e414 with expected crc 2023-10-01T09:35:13.272 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:12 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4000096772' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:13.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:13.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:13.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:12 smithi138 ceph-mon[185886]: failed to encode map e414 with expected crc 2023-10-01T09:35:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:12 smithi138 ceph-mon[185886]: failed to encode map e414 with expected crc 2023-10-01T09:35:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:12 smithi138 ceph-mon[185886]: osdmap e414: 8 total, 8 up, 7 in 2023-10-01T09:35:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:12 smithi138 ceph-mon[185886]: failed to encode map e414 with expected crc 2023-10-01T09:35:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:12 smithi138 ceph-mon[185886]: pgmap v1150: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 990 op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-01T09:35:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:12 smithi138 ceph-mon[185886]: failed to encode map e414 with expected crc 2023-10-01T09:35:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:12 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4000096772' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:13.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:12 smithi138 ceph-mon[192513]: failed to encode map e414 with expected crc 2023-10-01T09:35:13.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:12 smithi138 ceph-mon[192513]: failed to encode map e414 with expected crc 2023-10-01T09:35:13.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:12 smithi138 ceph-mon[192513]: osdmap e414: 8 total, 8 up, 7 in 2023-10-01T09:35:13.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:12 smithi138 ceph-mon[192513]: failed to encode map e414 with expected crc 2023-10-01T09:35:13.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:12 smithi138 ceph-mon[192513]: pgmap v1150: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 990 op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-01T09:35:13.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:12 smithi138 ceph-mon[192513]: failed to encode map e414 with expected crc 2023-10-01T09:35:13.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:12 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4000096772' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:13.374 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:13.429 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:13.676 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:13.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:13.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:13.691 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:35:13.765 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 57 16 79331 79315 86.9334 47.0625 0.0224915 0.0114963 2023-10-01T09:35:13.790 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:13.864 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:14.048 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:14.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:14.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:14.067 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:35:14.155 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:14.247 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:14.364 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:14.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:14.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:14.475 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:14.532 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:14 smithi190 ceph-mon[160435]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.532 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:14 smithi190 ceph-mon[160435]: osdmap e415: 8 total, 8 up, 7 in 2023-10-01T09:35:14.533 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:14 smithi190 ceph-mon[160435]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.533 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:14 smithi190 ceph-mon[160435]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.533 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:14 smithi190 ceph-mon[160435]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.533 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:14 smithi190 ceph-mon[160435]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.533 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:14 smithi190 ceph-mon[160435]: pgmap v1152: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 29 MiB/s wr, 467 op/s; 4884/235995 objects degraded (2.070%); 4907/235995 objects misplaced (2.079%); 540 KiB/s, 2 keys/s, 12 objects/s recovering 2023-10-01T09:35:14.533 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:14 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:35:14.569 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:14.720 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:14.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:14.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:14.765 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 58 16 80108 80092 86.272 48.5625 0.0122095 0.011585 2023-10-01T09:35:14.789 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:14.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[192513]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[192513]: osdmap e415: 8 total, 8 up, 7 in 2023-10-01T09:35:14.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[192513]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[192513]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[192513]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.828 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[192513]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[192513]: pgmap v1152: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 29 MiB/s wr, 467 op/s; 4884/235995 objects degraded (2.070%); 4907/235995 objects misplaced (2.079%); 540 KiB/s, 2 keys/s, 12 objects/s recovering 2023-10-01T09:35:14.829 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:35:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[185886]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[185886]: osdmap e415: 8 total, 8 up, 7 in 2023-10-01T09:35:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[185886]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[185886]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[185886]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[185886]: failed to encode map e415 with expected crc 2023-10-01T09:35:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[185886]: pgmap v1152: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 29 MiB/s wr, 467 op/s; 4884/235995 objects degraded (2.070%); 4907/235995 objects misplaced (2.079%); 540 KiB/s, 2 keys/s, 12 objects/s recovering 2023-10-01T09:35:14.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:14 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:35:14.836 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:14.928 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:15.127 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:15.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:15.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:15.168 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:35:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:35:14] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-01T09:35:15.233 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:15.292 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:15.373 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:15.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:15.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:15.482 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:15.533 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:15.699 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:35:15.765 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 59 16 80908 80892 85.6573 50 0.0216901 0.0116693 2023-10-01T09:35:15.783 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:15 smithi190 ceph-mon[160435]: Health check failed: Degraded data redundancy: 4884/235995 objects degraded (2.070%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:15.784 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:15 smithi190 ceph-mon[160435]: failed to encode map e416 with expected crc 2023-10-01T09:35:15.784 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:15 smithi190 ceph-mon[160435]: failed to encode map e416 with expected crc 2023-10-01T09:35:15.784 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:15 smithi190 ceph-mon[160435]: osdmap e416: 8 total, 8 up, 7 in 2023-10-01T09:35:15.784 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:15 smithi190 ceph-mon[160435]: failed to encode map e416 with expected crc 2023-10-01T09:35:15.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:15 smithi138 ceph-mon[192513]: Health check failed: Degraded data redundancy: 4884/235995 objects degraded (2.070%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:15.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:15 smithi138 ceph-mon[192513]: failed to encode map e416 with expected crc 2023-10-01T09:35:15.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:15 smithi138 ceph-mon[192513]: failed to encode map e416 with expected crc 2023-10-01T09:35:15.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:15 smithi138 ceph-mon[192513]: osdmap e416: 8 total, 8 up, 7 in 2023-10-01T09:35:15.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:15 smithi138 ceph-mon[192513]: failed to encode map e416 with expected crc 2023-10-01T09:35:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:15 smithi138 ceph-mon[185886]: Health check failed: Degraded data redundancy: 4884/235995 objects degraded (2.070%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:15 smithi138 ceph-mon[185886]: failed to encode map e416 with expected crc 2023-10-01T09:35:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:15 smithi138 ceph-mon[185886]: failed to encode map e416 with expected crc 2023-10-01T09:35:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:15 smithi138 ceph-mon[185886]: osdmap e416: 8 total, 8 up, 7 in 2023-10-01T09:35:15.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:15 smithi138 ceph-mon[185886]: failed to encode map e416 with expected crc 2023-10-01T09:35:16.109 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:35:16.110 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:35:16.110 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:35:16.110 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:35:16.110 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:35:16.428 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 5, 6, 4] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:35:16.428 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:35:16.429 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T09:35:16.429 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.000000 2023-10-01T09:35:16.429 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 0 2023-10-01T09:35:16.552 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:16 smithi190 ceph-mon[160435]: pgmap v1154: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 30 MiB/s wr, 485 op/s; 4884/237654 objects degraded (2.055%); 9688/237654 objects misplaced (4.077%); 985 KiB/s, 1 keys/s, 21 objects/s recovering 2023-10-01T09:35:16.629 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:16.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:16.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:16.739 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:16.765 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:35:16.766216+0000 min lat: 0.00117708 max lat: 2.39797 avg lat: 0.0117802 2023-10-01T09:35:16.766 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:35:16.766 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 60 16 81505 81489 84.8517 37.3125 0.00893017 0.0117802 2023-10-01T09:35:16.793 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:16.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:16 smithi138 ceph-mon[192513]: pgmap v1154: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 30 MiB/s wr, 485 op/s; 4884/237654 objects degraded (2.055%); 9688/237654 objects misplaced (4.077%); 985 KiB/s, 1 keys/s, 21 objects/s recovering 2023-10-01T09:35:16.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:16 smithi138 ceph-mon[185886]: pgmap v1154: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 30 MiB/s wr, 485 op/s; 4884/237654 objects degraded (2.055%); 9688/237654 objects misplaced (4.077%); 985 KiB/s, 1 keys/s, 21 objects/s recovering 2023-10-01T09:35:17.240 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:17.281 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:17.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:17.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:17.391 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:17.450 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:17.489 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:35:17.631 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:17.742 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:17.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:17.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:17.765 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 61 16 82253 82237 84.2272 46.75 0.0175738 0.0118654 2023-10-01T09:35:17.857 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:17.951 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:18.286 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:18.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:18.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:18.396 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:18.436 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:18.437 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:35:18.524 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:35:18.766 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 62 16 82979 82963 83.6006 45.375 0.023962 0.0119563 2023-10-01T09:35:18.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:18 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/312563845' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:35:18.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:18 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:35:18.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:18 smithi190 ceph-mon[160435]: pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 26 MiB/s wr, 413 op/s; 4884/240834 objects degraded (2.028%); 9688/240834 objects misplaced (4.023%); 825 KiB/s, 17 objects/s recovering 2023-10-01T09:35:18.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:18 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/312563845' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:35:18.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:18 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:35:18.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:18 smithi138 ceph-mon[192513]: pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 26 MiB/s wr, 413 op/s; 4884/240834 objects degraded (2.028%); 9688/240834 objects misplaced (4.023%); 825 KiB/s, 17 objects/s recovering 2023-10-01T09:35:18.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:18 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/312563845' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:35:18.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:18 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:35:18.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:18 smithi138 ceph-mon[185886]: pgmap v1155: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 26 MiB/s wr, 413 op/s; 4884/240834 objects degraded (2.028%); 9688/240834 objects misplaced (4.023%); 825 KiB/s, 17 objects/s recovering 2023-10-01T09:35:19.531 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:19 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:19.531 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:19 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:35:19.532 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:19 smithi190 ceph-mon[160435]: osdmap e417: 8 total, 8 up, 7 in 2023-10-01T09:35:19.532 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:19 smithi190 ceph-mon[160435]: failed to encode map e417 with expected crc 2023-10-01T09:35:19.532 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:19 smithi190 ceph-mon[160435]: failed to encode map e417 with expected crc 2023-10-01T09:35:19.532 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:19 smithi190 ceph-mon[160435]: failed to encode map e417 with expected crc 2023-10-01T09:35:19.532 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:19 smithi190 ceph-mon[160435]: failed to encode map e417 with expected crc 2023-10-01T09:35:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:19 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:19 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:35:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:19 smithi138 ceph-mon[185886]: osdmap e417: 8 total, 8 up, 7 in 2023-10-01T09:35:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:19 smithi138 ceph-mon[185886]: failed to encode map e417 with expected crc 2023-10-01T09:35:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:19 smithi138 ceph-mon[185886]: failed to encode map e417 with expected crc 2023-10-01T09:35:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:19 smithi138 ceph-mon[185886]: failed to encode map e417 with expected crc 2023-10-01T09:35:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:19 smithi138 ceph-mon[185886]: failed to encode map e417 with expected crc 2023-10-01T09:35:19.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:19 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:19.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:19 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:35:19.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:19 smithi138 ceph-mon[192513]: osdmap e417: 8 total, 8 up, 7 in 2023-10-01T09:35:19.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:19 smithi138 ceph-mon[192513]: failed to encode map e417 with expected crc 2023-10-01T09:35:19.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:19 smithi138 ceph-mon[192513]: failed to encode map e417 with expected crc 2023-10-01T09:35:19.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:19 smithi138 ceph-mon[192513]: failed to encode map e417 with expected crc 2023-10-01T09:35:19.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:19 smithi138 ceph-mon[192513]: failed to encode map e417 with expected crc 2023-10-01T09:35:19.766 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 63 16 83641 83625 82.9305 41.375 0.0177453 0.0120523 2023-10-01T09:35:20.508 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:20.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:20.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:20.613 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:20.704 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:20.766 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 64 16 84328 84312 82.3057 42.9375 0.0228624 0.0121446 2023-10-01T09:35:20.803 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:20 smithi138 ceph-mon[185886]: pgmap v1157: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 755 op/s; 4884/248160 objects degraded (1.968%); 9462/248160 objects misplaced (3.813%); 2.9 MiB/s, 50 objects/s recovering 2023-10-01T09:35:20.803 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:20 smithi138 ceph-mon[192513]: pgmap v1157: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 755 op/s; 4884/248160 objects degraded (1.968%); 9462/248160 objects misplaced (3.813%); 2.9 MiB/s, 50 objects/s recovering 2023-10-01T09:35:20.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:20 smithi190 ceph-mon[160435]: pgmap v1157: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 755 op/s; 4884/248160 objects degraded (1.968%); 9462/248160 objects misplaced (3.813%); 2.9 MiB/s, 50 objects/s recovering 2023-10-01T09:35:20.827 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:20.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:20.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:20.939 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:21.035 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:21.135 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:21.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:21.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:21.254 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:21.312 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:21.420 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:21.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:21.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:21.433 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:35:21.467 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:35:21.522 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:35:21.534 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:21.553 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:21 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 4884/248160 objects degraded (1.968%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:21.553 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:21 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1899696219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:21.554 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:21 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 4884/248160 objects degraded (1.968%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:21.555 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:21 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1899696219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:21.565 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:35:21.604 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:35:21.610 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:21.656 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:35:21.677 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:35:21.677 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:35:21.677 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:35:21.677 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:35:21.678 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.678 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:61933 8.5 8:ae340eae:::benchmark_data_smithi138_431617_object61932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e340)", 2023-10-01T09:35:21.678 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:13.546596+0000", 2023-10-01T09:35:21.678 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 308.127940948, 2023-10-01T09:35:21.678 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.170064412, 2023-10-01T09:35:21.678 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.678 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.678 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.678 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.678 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.679 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 61933 2023-10-01T09:35:21.679 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.679 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.679 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.679 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.679 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546596+0000", 2023-10-01T09:35:21.679 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.679 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.680 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.680 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.680 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546596+0000", 2023-10-01T09:35:21.680 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:35:21.680 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.680 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.680 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.680 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546595+0000", 2023-10-01T09:35:21.680 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0971999999999999e-05 2023-10-01T09:35:21.680 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.681 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.681 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.681 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546646+0000", 2023-10-01T09:35:21.681 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7750000000000004e-06 2023-10-01T09:35:21.681 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.681 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.681 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.681 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546656+0000", 2023-10-01T09:35:21.681 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8340000000000001e-06 2023-10-01T09:35:21.681 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.682 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.682 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.682 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546663+0000", 2023-10-01T09:35:21.682 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019821679999999999 2023-10-01T09:35:21.682 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.682 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.682 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.682 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.548645+0000", 2023-10-01T09:35:21.682 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010491400000000001 2023-10-01T09:35:21.683 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.683 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.683 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.683 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.548750+0000", 2023-10-01T09:35:21.683 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011701099999999999 2023-10-01T09:35:21.683 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.683 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.683 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.683 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.548867+0000", 2023-10-01T09:35:21.683 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0020976710000000002 2023-10-01T09:35:21.683 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.684 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.684 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.684 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.550964+0000", 2023-10-01T09:35:21.684 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0018446389999999999 2023-10-01T09:35:21.684 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.684 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.684 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.684 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.552809+0000", 2023-10-01T09:35:21.684 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16371334600000001 2023-10-01T09:35:21.684 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.685 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.685 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.685 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716522+0000", 2023-10-01T09:35:21.685 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3269999999999994e-05 2023-10-01T09:35:21.685 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.685 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.685 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.685 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716606+0000", 2023-10-01T09:35:21.685 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4738999999999999e-05 2023-10-01T09:35:21.685 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.686 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.686 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.686 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716660+0000", 2023-10-01T09:35:21.686 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.170064412 2023-10-01T09:35:21.686 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.686 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.686 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.686 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.686 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.686 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:75567 8.5 8:aef5727f:::benchmark_data_smithi138_431617_object75566:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-10-01T09:35:21.687 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:28.647427+0000", 2023-10-01T09:35:21.687 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 293.02710989799999, 2023-10-01T09:35:21.687 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17919886700000001, 2023-10-01T09:35:21.687 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.687 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.687 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.687 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.687 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.687 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 75567 2023-10-01T09:35:21.687 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.688 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.688 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.688 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.688 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647427+0000", 2023-10-01T09:35:21.688 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.688 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.688 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.688 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.688 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647427+0000", 2023-10-01T09:35:21.688 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:21.689 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.689 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.689 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.689 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647425+0000", 2023-10-01T09:35:21.689 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8278e-05 2023-10-01T09:35:21.689 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.689 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.689 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.689 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647483+0000", 2023-10-01T09:35:21.689 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4720999999999999e-05 2023-10-01T09:35:21.690 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.690 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.690 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.690 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647508+0000", 2023-10-01T09:35:21.690 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8720000000000002e-06 2023-10-01T09:35:21.690 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.690 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.690 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.691 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647516+0000", 2023-10-01T09:35:21.691 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00034716700000000001 2023-10-01T09:35:21.691 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.691 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.691 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.691 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647863+0000", 2023-10-01T09:35:21.691 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5581999999999994e-05 2023-10-01T09:35:21.691 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.691 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.691 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.692 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647959+0000", 2023-10-01T09:35:21.692 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000119572 2023-10-01T09:35:21.692 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.692 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.692 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.692 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.648078+0000", 2023-10-01T09:35:21.692 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078048329999999997 2023-10-01T09:35:21.692 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.692 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.692 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.692 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.655883+0000", 2023-10-01T09:35:21.693 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013701100000000001 2023-10-01T09:35:21.693 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.693 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.693 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.693 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.656020+0000", 2023-10-01T09:35:21.693 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17050426699999999 2023-10-01T09:35:21.693 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.693 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.693 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.693 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.826524+0000", 2023-10-01T09:35:21.693 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8119999999999999e-05 2023-10-01T09:35:21.694 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.694 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.694 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.694 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.826582+0000", 2023-10-01T09:35:21.694 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3550999999999997e-05 2023-10-01T09:35:21.694 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.694 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.694 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.694 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.826626+0000", 2023-10-01T09:35:21.694 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17919886700000001 2023-10-01T09:35:21.695 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.695 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.695 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.695 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.695 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.695 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:75568 8.5 8:ad74c15b:::benchmark_data_smithi138_431617_object75567:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-10-01T09:35:21.695 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:28.647524+0000", 2023-10-01T09:35:21.695 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 293.02701351100001, 2023-10-01T09:35:21.695 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18103444699999999, 2023-10-01T09:35:21.695 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.696 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.696 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.696 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.696 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.696 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 75568 2023-10-01T09:35:21.696 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.696 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.696 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.696 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.696 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647524+0000", 2023-10-01T09:35:21.697 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.697 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.697 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.697 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.697 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647524+0000", 2023-10-01T09:35:21.697 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:35:21.697 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.697 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.697 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.697 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647523+0000", 2023-10-01T09:35:21.697 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4457999999999998e-05 2023-10-01T09:35:21.698 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.698 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.698 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.698 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647597+0000", 2023-10-01T09:35:21.698 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6279999999999994e-06 2023-10-01T09:35:21.698 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.698 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.698 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.698 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647606+0000", 2023-10-01T09:35:21.699 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0679999999999996e-06 2023-10-01T09:35:21.699 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.699 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.699 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.699 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647611+0000", 2023-10-01T09:35:21.699 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0030153350000000001 2023-10-01T09:35:21.699 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.699 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.699 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.699 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.650626+0000", 2023-10-01T09:35:21.699 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013843899999999999 2023-10-01T09:35:21.700 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.700 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.700 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.700 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.650765+0000", 2023-10-01T09:35:21.700 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017521299999999999 2023-10-01T09:35:21.700 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.700 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.700 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.700 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.650940+0000", 2023-10-01T09:35:21.700 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049486560000000001 2023-10-01T09:35:21.700 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.701 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.701 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.701 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.655889+0000", 2023-10-01T09:35:21.701 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00030954499999999999 2023-10-01T09:35:21.701 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.701 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.701 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.701 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.656198+0000", 2023-10-01T09:35:21.701 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.172268646 2023-10-01T09:35:21.701 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.702 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.702 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.702 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828467+0000", 2023-10-01T09:35:21.702 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0330000000000001e-05 2023-10-01T09:35:21.702 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.702 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.702 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.702 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828517+0000", 2023-10-01T09:35:21.702 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0725000000000001e-05 2023-10-01T09:35:21.702 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.702 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.703 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.703 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828558+0000", 2023-10-01T09:35:21.703 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18103444699999999 2023-10-01T09:35:21.703 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.703 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.703 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.703 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.703 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.703 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:75571 8.5 8:a2fb1dba:::benchmark_data_smithi138_431617_object75570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-10-01T09:35:21.703 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:28.647622+0000", 2023-10-01T09:35:21.703 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 293.02691488300002, 2023-10-01T09:35:21.704 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18107846399999999, 2023-10-01T09:35:21.704 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.704 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.704 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.704 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.704 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.704 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 75571 2023-10-01T09:35:21.704 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.704 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.704 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.705 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.705 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647622+0000", 2023-10-01T09:35:21.705 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.705 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.705 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.705 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.705 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647622+0000", 2023-10-01T09:35:21.705 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:35:21.705 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.705 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.705 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.706 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647621+0000", 2023-10-01T09:35:21.706 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8969e-05 2023-10-01T09:35:21.706 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.706 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.706 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.706 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647650+0000", 2023-10-01T09:35:21.706 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0261000000000001e-05 2023-10-01T09:35:21.706 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.706 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.706 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.707 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647661+0000", 2023-10-01T09:35:21.707 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8960000000000002e-06 2023-10-01T09:35:21.707 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.707 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.707 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.707 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647666+0000", 2023-10-01T09:35:21.707 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053841219999999999 2023-10-01T09:35:21.707 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.707 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.707 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.707 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.653050+0000", 2023-10-01T09:35:21.708 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015679600000000001 2023-10-01T09:35:21.708 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.708 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.708 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.708 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.653206+0000", 2023-10-01T09:35:21.708 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00019690199999999999 2023-10-01T09:35:21.708 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.708 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.708 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.708 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.653403+0000", 2023-10-01T09:35:21.709 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002801993 2023-10-01T09:35:21.709 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.709 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.709 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.709 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.656205+0000", 2023-10-01T09:35:21.709 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00025078999999999998 2023-10-01T09:35:21.709 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.709 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.709 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.709 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.656456+0000", 2023-10-01T09:35:21.709 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.172176575 2023-10-01T09:35:21.710 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.710 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.710 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.710 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828633+0000", 2023-10-01T09:35:21.710 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1740999999999999e-05 2023-10-01T09:35:21.710 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.710 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.710 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.710 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828674+0000", 2023-10-01T09:35:21.710 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6143999999999999e-05 2023-10-01T09:35:21.711 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.711 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.711 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.711 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828701+0000", 2023-10-01T09:35:21.711 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18107846399999999 2023-10-01T09:35:21.711 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.711 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.711 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.711 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.711 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.711 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:89387 8.5 8:ac2bd163:::benchmark_data_smithi138_431617_object89386:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-01T09:35:21.712 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:46.701591+0000", 2023-10-01T09:35:21.712 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 274.97294580400001, 2023-10-01T09:35:21.712 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.242286687, 2023-10-01T09:35:21.712 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.712 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.712 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.712 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.712 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.712 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 89387 2023-10-01T09:35:21.712 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.713 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.713 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.713 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.713 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701591+0000", 2023-10-01T09:35:21.714 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.714 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.714 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.714 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.714 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701591+0000", 2023-10-01T09:35:21.714 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:21.714 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.714 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.715 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.715 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701590+0000", 2023-10-01T09:35:21.715 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2069e-05 2023-10-01T09:35:21.715 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.715 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.715 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.715 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701632+0000", 2023-10-01T09:35:21.715 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.005e-05 2023-10-01T09:35:21.716 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.716 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.716 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.716 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701642+0000", 2023-10-01T09:35:21.716 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2100000000000001e-06 2023-10-01T09:35:21.716 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.716 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.716 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.716 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701647+0000", 2023-10-01T09:35:21.716 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012216000000000001 2023-10-01T09:35:21.717 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.717 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.717 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.717 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701769+0000", 2023-10-01T09:35:21.717 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015872299999999999 2023-10-01T09:35:21.717 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.717 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.717 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.717 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701928+0000", 2023-10-01T09:35:21.718 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017488099999999999 2023-10-01T09:35:21.718 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.718 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.718 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.718 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.702103+0000", 2023-10-01T09:35:21.718 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0026736849999999999 2023-10-01T09:35:21.718 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.718 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.718 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.719 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.704776+0000", 2023-10-01T09:35:21.719 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00095142499999999999 2023-10-01T09:35:21.719 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.719 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.719 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.719 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.705728+0000", 2023-10-01T09:35:21.719 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.238030466 2023-10-01T09:35:21.719 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.719 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.719 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.720 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.943758+0000", 2023-10-01T09:35:21.720 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1157999999999999e-05 2023-10-01T09:35:21.720 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.720 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.720 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.720 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.943829+0000", 2023-10-01T09:35:21.720 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8421000000000003e-05 2023-10-01T09:35:21.720 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.720 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.720 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.720 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.943878+0000", 2023-10-01T09:35:21.721 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.242286687 2023-10-01T09:35:21.721 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.721 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.721 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.721 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.721 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.721 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:89417 8.5 8:ac07dffe:::benchmark_data_smithi138_431617_object89416:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-01T09:35:21.721 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:46.731169+0000", 2023-10-01T09:35:21.721 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 274.94336769199998, 2023-10-01T09:35:21.721 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21775186899999999, 2023-10-01T09:35:21.722 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.722 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.722 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.722 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.722 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.722 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 89417 2023-10-01T09:35:21.722 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.722 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.722 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.722 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.722 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731169+0000", 2023-10-01T09:35:21.723 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.723 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.723 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.723 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.723 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731169+0000", 2023-10-01T09:35:21.723 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:21.723 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.723 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.723 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.723 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731168+0000", 2023-10-01T09:35:21.723 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0485000000000003e-05 2023-10-01T09:35:21.724 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.724 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.724 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.724 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731208+0000", 2023-10-01T09:35:21.724 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5139e-05 2023-10-01T09:35:21.724 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.724 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.724 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.724 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731223+0000", 2023-10-01T09:35:21.724 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4389999999999997e-06 2023-10-01T09:35:21.724 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.725 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.725 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.725 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731230+0000", 2023-10-01T09:35:21.725 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8512e-05 2023-10-01T09:35:21.725 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.725 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.725 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.725 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731268+0000", 2023-10-01T09:35:21.725 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012213799999999999 2023-10-01T09:35:21.725 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.725 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.726 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.726 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731390+0000", 2023-10-01T09:35:21.726 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000120797 2023-10-01T09:35:21.726 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.726 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.726 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.726 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731511+0000", 2023-10-01T09:35:21.726 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002402623 2023-10-01T09:35:21.726 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.726 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.727 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.727 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.733914+0000", 2023-10-01T09:35:21.727 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00025578500000000002 2023-10-01T09:35:21.727 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.727 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.727 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.727 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.734170+0000", 2023-10-01T09:35:21.727 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21464570199999999 2023-10-01T09:35:21.727 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.727 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.728 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.728 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.948815+0000", 2023-10-01T09:35:21.728 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2032999999999994e-05 2023-10-01T09:35:21.728 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.728 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.728 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.728 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.948877+0000", 2023-10-01T09:35:21.728 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3804999999999998e-05 2023-10-01T09:35:21.728 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.728 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.728 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.729 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.948921+0000", 2023-10-01T09:35:21.729 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21775186899999999 2023-10-01T09:35:21.729 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.729 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.729 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.729 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.729 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.729 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:98811 8.5 8:aea6ced9:::benchmark_data_smithi138_431617_object98810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-01T09:35:21.729 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:59.492826+0000", 2023-10-01T09:35:21.729 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 262.18171115799998, 2023-10-01T09:35:21.729 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28100355799999999, 2023-10-01T09:35:21.730 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.730 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.730 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.730 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.730 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.730 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 98811 2023-10-01T09:35:21.730 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.730 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.730 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.730 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.731 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.492826+0000", 2023-10-01T09:35:21.731 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.731 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.731 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.731 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.731 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.492826+0000", 2023-10-01T09:35:21.731 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:21.731 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.731 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.731 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.731 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.492825+0000", 2023-10-01T09:35:21.732 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9044000000000001e-05 2023-10-01T09:35:21.732 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.732 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.732 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.732 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.492864+0000", 2023-10-01T09:35:21.732 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4671000000000001e-05 2023-10-01T09:35:21.732 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.732 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.732 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.732 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.492878+0000", 2023-10-01T09:35:21.732 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2079999999999997e-06 2023-10-01T09:35:21.733 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.733 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.733 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.733 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.492885+0000", 2023-10-01T09:35:21.734 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:35:21.735 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0015081719999999999 2023-10-01T09:35:21.735 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.735 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.735 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.735 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.494393+0000", 2023-10-01T09:35:21.735 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000132967 2023-10-01T09:35:21.736 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.736 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.736 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.736 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.494526+0000", 2023-10-01T09:35:21.736 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014614700000000001 2023-10-01T09:35:21.736 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.736 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.736 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.736 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.494672+0000", 2023-10-01T09:35:21.736 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0034588140000000002 2023-10-01T09:35:21.736 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.737 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.737 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.737 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.498131+0000", 2023-10-01T09:35:21.737 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001235756 2023-10-01T09:35:21.737 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.737 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.737 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.737 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.499366+0000", 2023-10-01T09:35:21.737 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27437294899999998 2023-10-01T09:35:21.737 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.738 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.738 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.738 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.773739+0000", 2023-10-01T09:35:21.738 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0510000000000003e-05 2023-10-01T09:35:21.738 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.738 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.738 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.738 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.773790+0000", 2023-10-01T09:35:21.738 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9598999999999998e-05 2023-10-01T09:35:21.738 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.738 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.739 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.739 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.773829+0000", 2023-10-01T09:35:21.739 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28100355799999999 2023-10-01T09:35:21.739 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.739 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.739 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.739 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.739 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.739 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:98822 8.5 8:a281756c:::benchmark_data_smithi138_431617_object98821:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-01T09:35:21.739 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:59.506300+0000", 2023-10-01T09:35:21.740 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 262.16823690899997, 2023-10-01T09:35:21.740 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.267638088, 2023-10-01T09:35:21.740 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.740 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.740 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.740 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.740 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.740 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 98822 2023-10-01T09:35:21.740 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.740 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.741 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.741 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.741 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506300+0000", 2023-10-01T09:35:21.741 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.741 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.741 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.741 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.741 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506300+0000", 2023-10-01T09:35:21.741 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:21.741 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.741 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.742 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.742 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506298+0000", 2023-10-01T09:35:21.742 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1570999999999998e-05 2023-10-01T09:35:21.742 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.742 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.742 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.742 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506350+0000", 2023-10-01T09:35:21.742 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8213e-05 2023-10-01T09:35:21.742 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.742 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.743 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.743 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506368+0000", 2023-10-01T09:35:21.743 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4329999999999998e-06 2023-10-01T09:35:21.743 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.743 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.743 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.743 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506376+0000", 2023-10-01T09:35:21.743 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2095999999999997e-05 2023-10-01T09:35:21.743 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.743 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.743 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.744 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506428+0000", 2023-10-01T09:35:21.744 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016378599999999999 2023-10-01T09:35:21.744 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.744 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.744 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.744 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506591+0000", 2023-10-01T09:35:21.744 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000195434 2023-10-01T09:35:21.744 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.744 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.744 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.745 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506787+0000", 2023-10-01T09:35:21.745 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0027607909999999999 2023-10-01T09:35:21.745 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.745 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.745 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.745 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.509548+0000", 2023-10-01T09:35:21.745 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001199079 2023-10-01T09:35:21.745 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.745 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.745 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.745 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.510747+0000", 2023-10-01T09:35:21.746 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26312249300000001 2023-10-01T09:35:21.746 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.746 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.746 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.746 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.773869+0000", 2023-10-01T09:35:21.746 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8377999999999997e-05 2023-10-01T09:35:21.746 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.746 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.746 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.746 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.773908+0000", 2023-10-01T09:35:21.747 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0613e-05 2023-10-01T09:35:21.747 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.747 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.747 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.747 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.773938+0000", 2023-10-01T09:35:21.747 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.267638088 2023-10-01T09:35:21.747 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.747 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.747 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.747 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.747 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.749 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:102764 8.5 8:ab6d8cb6:::benchmark_data_smithi138_431617_object1353:head [delete] snapc 0=[] ondisk+write+known_if_redirected e348)", 2023-10-01T09:35:21.749 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:31:06.313439+0000", 2023-10-01T09:35:21.749 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 255.36109824100001, 2023-10-01T09:35:21.749 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18351769100000001, 2023-10-01T09:35:21.749 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.749 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.749 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.749 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.750 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.750 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 102764 2023-10-01T09:35:21.750 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.750 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.750 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.750 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.750 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313439+0000", 2023-10-01T09:35:21.750 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.750 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.750 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.751 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.751 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313439+0000", 2023-10-01T09:35:21.751 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:21.751 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.751 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.751 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.751 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313437+0000", 2023-10-01T09:35:21.751 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0399999999999993e-06 2023-10-01T09:35:21.751 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.751 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.751 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.752 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313445+0000", 2023-10-01T09:35:21.752 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.738e-05 2023-10-01T09:35:21.752 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.752 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.752 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.752 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313463+0000", 2023-10-01T09:35:21.752 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7819999999999996e-06 2023-10-01T09:35:21.752 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.752 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.752 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.752 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313469+0000", 2023-10-01T09:35:21.753 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4065000000000004e-05 2023-10-01T09:35:21.753 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.753 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.753 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.753 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313533+0000", 2023-10-01T09:35:21.753 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014046600000000001 2023-10-01T09:35:21.753 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.753 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.753 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.753 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313674+0000", 2023-10-01T09:35:21.754 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013492699999999999 2023-10-01T09:35:21.754 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.754 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.754 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.754 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313809+0000", 2023-10-01T09:35:21.754 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0067771769999999997 2023-10-01T09:35:21.754 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.754 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.754 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.754 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.320586+0000", 2023-10-01T09:35:21.755 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0048310360000000004 2023-10-01T09:35:21.755 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.755 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.755 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.755 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.325417+0000", 2023-10-01T09:35:21.755 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17142922099999999 2023-10-01T09:35:21.755 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.755 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.755 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.755 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.496846+0000", 2023-10-01T09:35:21.756 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0718999999999999e-05 2023-10-01T09:35:21.756 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.756 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.756 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.756 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.496907+0000", 2023-10-01T09:35:21.756 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9557e-05 2023-10-01T09:35:21.756 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.756 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.756 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.756 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.496956+0000", 2023-10-01T09:35:21.757 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18351769100000001 2023-10-01T09:35:21.757 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.757 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.757 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.757 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.757 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.757 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:102767 8.5 8:aad588e2:::benchmark_data_smithi138_431617_object1356:head [delete] snapc 0=[] ondisk+write+known_if_redirected e348)", 2023-10-01T09:35:21.757 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:31:06.313484+0000", 2023-10-01T09:35:21.757 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 255.361053263, 2023-10-01T09:35:21.757 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18457310299999999, 2023-10-01T09:35:21.758 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.758 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.758 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.758 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.758 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.758 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 102767 2023-10-01T09:35:21.758 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.758 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.758 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.758 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.758 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313484+0000", 2023-10-01T09:35:21.759 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.759 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.759 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.759 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.759 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313484+0000", 2023-10-01T09:35:21.759 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:35:21.759 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.759 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.759 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.759 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313483+0000", 2023-10-01T09:35:21.759 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9440000000000001e-06 2023-10-01T09:35:21.760 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.760 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.760 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.760 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313486+0000", 2023-10-01T09:35:21.760 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5590000000000006e-06 2023-10-01T09:35:21.760 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.760 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.760 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.760 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313495+0000", 2023-10-01T09:35:21.760 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0080000000000004e-06 2023-10-01T09:35:21.760 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.761 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.761 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.761 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313499+0000", 2023-10-01T09:35:21.761 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0036149279999999999 2023-10-01T09:35:21.761 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.761 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.761 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.761 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.317114+0000", 2023-10-01T09:35:21.761 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000124485 2023-10-01T09:35:21.761 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.762 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.762 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.762 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.317238+0000", 2023-10-01T09:35:21.762 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013227399999999999 2023-10-01T09:35:21.762 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.762 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.762 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.762 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.317370+0000", 2023-10-01T09:35:21.762 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035185339999999998 2023-10-01T09:35:21.762 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.763 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.763 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.763 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.320889+0000", 2023-10-01T09:35:21.763 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045892579999999997 2023-10-01T09:35:21.763 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.763 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.763 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.763 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.325478+0000", 2023-10-01T09:35:21.763 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.172485942 2023-10-01T09:35:21.764 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.764 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.764 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.764 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.497964+0000", 2023-10-01T09:35:21.764 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8517999999999999e-05 2023-10-01T09:35:21.764 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.764 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.764 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.764 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.498013+0000", 2023-10-01T09:35:21.764 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4134000000000001e-05 2023-10-01T09:35:21.765 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.765 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.765 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.765 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.498057+0000", 2023-10-01T09:35:21.765 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18457310299999999 2023-10-01T09:35:21.765 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.765 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.765 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.765 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.765 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.766 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:115052 8.5 8:a3e965a9:::benchmark_data_smithi138_431617_object13641:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-10-01T09:35:21.766 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:31:22.838052+0000", 2023-10-01T09:35:21.766 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 238.83648461799999, 2023-10-01T09:35:21.766 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26344810600000002, 2023-10-01T09:35:21.766 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.766 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.766 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.766 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.766 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.766 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 115052 2023-10-01T09:35:21.767 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.767 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.767 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.767 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.767 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838052+0000", 2023-10-01T09:35:21.767 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.767 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.767 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.767 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.767 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838052+0000", 2023-10-01T09:35:21.768 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:21.768 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.768 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.768 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.768 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838051+0000", 2023-10-01T09:35:21.768 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6899999999999997e-06 2023-10-01T09:35:21.768 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.768 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.768 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.768 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838057+0000", 2023-10-01T09:35:21.769 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3438000000000001e-05 2023-10-01T09:35:21.769 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.769 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.769 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.769 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838070+0000", 2023-10-01T09:35:21.769 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2419999999999997e-06 2023-10-01T09:35:21.769 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.769 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.769 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.769 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838076+0000", 2023-10-01T09:35:21.770 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2471e-05 2023-10-01T09:35:21.770 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.770 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.770 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.770 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838119+0000", 2023-10-01T09:35:21.770 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012397600000000001 2023-10-01T09:35:21.770 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.770 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.770 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.770 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838243+0000", 2023-10-01T09:35:21.771 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000152304 2023-10-01T09:35:21.771 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.771 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.771 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.771 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838395+0000", 2023-10-01T09:35:21.771 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0071079480000000002 2023-10-01T09:35:21.771 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.771 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.771 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.771 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.845503+0000", 2023-10-01T09:35:21.771 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0060857350000000001 2023-10-01T09:35:21.772 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.772 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.772 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.772 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.851589+0000", 2023-10-01T09:35:21.772 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.249801365 2023-10-01T09:35:21.772 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.772 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.772 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.772 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101390+0000", 2023-10-01T09:35:21.772 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2990000000000005e-05 2023-10-01T09:35:21.772 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.773 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.773 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.773 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101453+0000", 2023-10-01T09:35:21.773 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7327000000000003e-05 2023-10-01T09:35:21.773 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.773 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.773 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.773 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101501+0000", 2023-10-01T09:35:21.773 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26344810600000002 2023-10-01T09:35:21.773 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.773 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.774 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.774 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.774 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.774 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:115054 8.5 8:ac73bb40:::benchmark_data_smithi138_431617_object13643:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-10-01T09:35:21.774 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:31:22.838091+0000", 2023-10-01T09:35:21.774 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 238.83644645800001, 2023-10-01T09:35:21.774 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26356659199999999, 2023-10-01T09:35:21.774 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.774 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.774 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.774 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.775 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.775 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 115054 2023-10-01T09:35:21.775 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.775 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.775 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.775 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.775 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838091+0000", 2023-10-01T09:35:21.775 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.775 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.775 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.776 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.776 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838091+0000", 2023-10-01T09:35:21.776 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:35:21.776 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.776 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.776 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.776 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838090+0000", 2023-10-01T09:35:21.776 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7269999999999999e-06 2023-10-01T09:35:21.776 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.776 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.776 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.776 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838093+0000", 2023-10-01T09:35:21.777 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7969999999999997e-06 2023-10-01T09:35:21.777 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.777 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.777 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.777 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838099+0000", 2023-10-01T09:35:21.777 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7689999999999998e-06 2023-10-01T09:35:21.777 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.777 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.777 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.777 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838102+0000", 2023-10-01T09:35:21.778 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0041852180000000001 2023-10-01T09:35:21.778 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.778 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.778 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.778 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.842288+0000", 2023-10-01T09:35:21.778 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012376100000000001 2023-10-01T09:35:21.778 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.778 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.778 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.778 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.842411+0000", 2023-10-01T09:35:21.778 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000119407 2023-10-01T09:35:21.779 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.779 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.779 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.779 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.842531+0000", 2023-10-01T09:35:21.779 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0031142069999999999 2023-10-01T09:35:21.779 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.779 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.779 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.779 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.845645+0000", 2023-10-01T09:35:21.779 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0059780909999999996 2023-10-01T09:35:21.779 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.780 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.780 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.780 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.851623+0000", 2023-10-01T09:35:21.780 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24993852699999999 2023-10-01T09:35:21.780 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.780 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.780 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.780 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101562+0000", 2023-10-01T09:35:21.780 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2067000000000001e-05 2023-10-01T09:35:21.780 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.780 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.781 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.781 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101614+0000", 2023-10-01T09:35:21.781 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3449999999999999e-05 2023-10-01T09:35:21.781 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.781 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.781 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.781 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101657+0000", 2023-10-01T09:35:21.781 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26356659199999999 2023-10-01T09:35:21.781 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.781 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.781 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.782 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.782 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.782 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:115065 8.5 8:a4841ad2:::benchmark_data_smithi138_431617_object13654:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-10-01T09:35:21.782 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:31:22.847762+0000", 2023-10-01T09:35:21.782 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 238.82677542799999, 2023-10-01T09:35:21.782 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.25401542300000002, 2023-10-01T09:35:21.782 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.782 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.782 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.782 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.782 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.783 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 115065 2023-10-01T09:35:21.783 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.783 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.783 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.783 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.783 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847762+0000", 2023-10-01T09:35:21.783 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.783 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.783 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.783 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.783 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847762+0000", 2023-10-01T09:35:21.784 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:21.784 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.784 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.784 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.784 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847760+0000", 2023-10-01T09:35:21.784 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9060000000000002e-06 2023-10-01T09:35:21.784 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.784 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.784 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.784 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847768+0000", 2023-10-01T09:35:21.785 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6707000000000001e-05 2023-10-01T09:35:21.785 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.785 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.785 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.785 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847784+0000", 2023-10-01T09:35:21.785 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0369999999999997e-06 2023-10-01T09:35:21.785 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.785 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.785 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.785 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847791+0000", 2023-10-01T09:35:21.785 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5365999999999999e-05 2023-10-01T09:35:21.786 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.786 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.786 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.786 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847837+0000", 2023-10-01T09:35:21.786 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012898299999999999 2023-10-01T09:35:21.786 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.786 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.786 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.786 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847966+0000", 2023-10-01T09:35:21.786 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014580100000000001 2023-10-01T09:35:21.786 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.787 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.787 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.787 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.848112+0000", 2023-10-01T09:35:21.787 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0036017150000000001 2023-10-01T09:35:21.787 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.787 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.787 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.787 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.851713+0000", 2023-10-01T09:35:21.787 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4385000000000003e-05 2023-10-01T09:35:21.787 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.788 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.788 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.788 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.851758+0000", 2023-10-01T09:35:21.788 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24993739000000001 2023-10-01T09:35:21.788 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.788 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.788 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.788 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101695+0000", 2023-10-01T09:35:21.788 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4341000000000001e-05 2023-10-01T09:35:21.788 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.788 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.788 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.789 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101739+0000", 2023-10-01T09:35:21.789 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7678000000000001e-05 2023-10-01T09:35:21.789 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.789 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.789 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.789 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101777+0000", 2023-10-01T09:35:21.789 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.25401542300000002 2023-10-01T09:35:21.789 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.789 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.789 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.789 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.790 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.790 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:149332 8.5 8:ad5fd0df:::benchmark_data_smithi138_431617_object47921:head [delete] snapc 0=[] ondisk+write+known_if_redirected e358)", 2023-10-01T09:35:21.790 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:11.735369+0000", 2023-10-01T09:35:21.790 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 189.93916760299999, 2023-10-01T09:35:21.790 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.19642474700000001, 2023-10-01T09:35:21.790 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.790 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.790 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.790 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.790 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.791 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 149332 2023-10-01T09:35:21.791 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.791 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.791 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.791 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.791 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735369+0000", 2023-10-01T09:35:21.791 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.791 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.791 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.791 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.792 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735369+0000", 2023-10-01T09:35:21.792 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:21.792 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.792 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.792 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.792 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735367+0000", 2023-10-01T09:35:21.792 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7149999999999993e-06 2023-10-01T09:35:21.792 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.792 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.792 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.792 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735377+0000", 2023-10-01T09:35:21.793 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8277000000000001e-05 2023-10-01T09:35:21.793 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.793 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.793 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.793 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735395+0000", 2023-10-01T09:35:21.793 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8809999999999996e-06 2023-10-01T09:35:21.793 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.793 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.793 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.793 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735403+0000", 2023-10-01T09:35:21.793 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3931000000000003e-05 2023-10-01T09:35:21.793 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.794 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.794 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.794 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735457+0000", 2023-10-01T09:35:21.794 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000136833 2023-10-01T09:35:21.794 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.794 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.794 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.794 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735594+0000", 2023-10-01T09:35:21.794 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016526299999999999 2023-10-01T09:35:21.794 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.795 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.795 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.795 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735759+0000", 2023-10-01T09:35:21.795 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0076574440000000002 2023-10-01T09:35:21.795 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.795 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.795 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.795 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.743417+0000", 2023-10-01T09:35:21.795 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053083100000000001 2023-10-01T09:35:21.795 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.795 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.796 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.796 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.748725+0000", 2023-10-01T09:35:21.796 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18300225000000001 2023-10-01T09:35:21.796 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.796 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.796 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.796 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.931727+0000", 2023-10-01T09:35:21.796 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6539999999999999e-05 2023-10-01T09:35:21.796 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.796 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.796 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.797 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.931764+0000", 2023-10-01T09:35:21.797 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0328000000000002e-05 2023-10-01T09:35:21.797 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.797 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.797 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.797 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.931794+0000", 2023-10-01T09:35:21.797 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.19642474700000001 2023-10-01T09:35:21.797 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.797 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.797 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.797 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.798 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.798 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:149342 8.5 8:a6fdf006:::benchmark_data_smithi138_431617_object47931:head [delete] snapc 0=[] ondisk+write+known_if_redirected e358)", 2023-10-01T09:35:21.798 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:11.736386+0000", 2023-10-01T09:35:21.798 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 189.93815053399999, 2023-10-01T09:35:21.798 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.19553300500000001, 2023-10-01T09:35:21.798 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.798 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.798 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.798 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.798 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.798 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 149342 2023-10-01T09:35:21.799 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.799 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.799 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.799 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.799 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.736386+0000", 2023-10-01T09:35:21.799 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.799 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.799 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.799 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.799 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.736386+0000", 2023-10-01T09:35:21.799 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:21.800 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.800 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.800 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.800 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.736385+0000", 2023-10-01T09:35:21.800 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1870000000000002e-06 2023-10-01T09:35:21.800 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.800 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.800 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.800 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.736391+0000", 2023-10-01T09:35:21.800 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5701000000000001e-05 2023-10-01T09:35:21.800 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.800 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.801 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.801 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.736407+0000", 2023-10-01T09:35:21.801 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5139999999999998e-06 2023-10-01T09:35:21.801 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.801 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.801 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.801 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.736414+0000", 2023-10-01T09:35:21.801 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0031694340000000001 2023-10-01T09:35:21.801 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.801 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.801 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.802 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.739583+0000", 2023-10-01T09:35:21.802 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000117611 2023-10-01T09:35:21.802 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.802 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.802 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.802 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.739701+0000", 2023-10-01T09:35:21.802 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5253000000000004e-05 2023-10-01T09:35:21.802 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.802 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.802 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.802 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.739796+0000", 2023-10-01T09:35:21.803 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0040039790000000004 2023-10-01T09:35:21.803 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.803 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.803 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.803 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.743800+0000", 2023-10-01T09:35:21.803 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050418490000000002 2023-10-01T09:35:21.803 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.803 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.803 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.803 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.748842+0000", 2023-10-01T09:35:21.803 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.182993609 2023-10-01T09:35:21.804 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.804 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.804 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.804 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.931835+0000", 2023-10-01T09:35:21.804 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7147000000000001e-05 2023-10-01T09:35:21.804 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.804 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.804 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.804 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.931883+0000", 2023-10-01T09:35:21.804 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6977000000000002e-05 2023-10-01T09:35:21.804 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.805 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.805 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.805 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.931919+0000", 2023-10-01T09:35:21.805 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.19553300500000001 2023-10-01T09:35:21.805 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.805 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.805 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.805 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.805 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.805 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:159796 8.5 8:ae0f494c:::benchmark_data_smithi138_431617_object58385:head [delete] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-10-01T09:35:21.805 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:27.321194+0000", 2023-10-01T09:35:21.806 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 174.35334274300001, 2023-10-01T09:35:21.806 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2966308610000001, 2023-10-01T09:35:21.806 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.806 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:35:21.806 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.806 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.806 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.806 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 159796 2023-10-01T09:35:21.806 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.806 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.806 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.807 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.807 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321194+0000", 2023-10-01T09:35:21.807 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.807 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.807 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.807 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.807 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321194+0000", 2023-10-01T09:35:21.807 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:21.807 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.807 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.807 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.808 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321192+0000", 2023-10-01T09:35:21.808 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3389999999999998e-06 2023-10-01T09:35:21.808 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.808 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.808 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.808 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321199+0000", 2023-10-01T09:35:21.808 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2383e-05 2023-10-01T09:35:21.808 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.808 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.808 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.808 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321211+0000", 2023-10-01T09:35:21.808 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0760000000000002e-06 2023-10-01T09:35:21.809 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.809 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.809 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.809 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321216+0000", 2023-10-01T09:35:21.809 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3895e-05 2023-10-01T09:35:21.809 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.809 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.809 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.809 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321250+0000", 2023-10-01T09:35:21.809 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000108306 2023-10-01T09:35:21.809 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.809 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.810 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.810 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321358+0000", 2023-10-01T09:35:21.810 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7575e-05 2023-10-01T09:35:21.810 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.810 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.810 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:35:21.810 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321456+0000", 2023-10-01T09:35:21.810 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0041519549999999997 2023-10-01T09:35:21.810 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.810 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.810 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.811 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.325608+0000", 2023-10-01T09:35:21.811 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00030447800000000001 2023-10-01T09:35:21.811 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.811 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.811 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.811 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.325912+0000", 2023-10-01T09:35:21.811 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017190989 2023-10-01T09:35:21.811 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.811 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.811 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.811 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.343103+0000", 2023-10-01T09:35:21.811 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3421000000000001e-05 2023-10-01T09:35:21.812 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.812 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.812 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:21.812 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.343117+0000", 2023-10-01T09:35:21.812 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.100368394 2023-10-01T09:35:21.812 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.812 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.812 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.812 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.443485+0000", 2023-10-01T09:35:21.812 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1826e-05 2023-10-01T09:35:21.812 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.813 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.813 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:21.813 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.443497+0000", 2023-10-01T09:35:21.813 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.174244764 2023-10-01T09:35:21.813 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.813 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.813 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.813 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.617742+0000", 2023-10-01T09:35:21.813 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3264999999999996e-05 2023-10-01T09:35:21.813 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.813 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.813 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.814 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.617825+0000", 2023-10-01T09:35:21.814 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2966308610000001 2023-10-01T09:35:21.814 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.814 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.814 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.814 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.814 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.814 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:159825 8.5 8:ac107179:::benchmark_data_smithi138_431617_object58414:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-10-01T09:35:21.814 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:27.380723+0000", 2023-10-01T09:35:21.814 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 174.29381368700001, 2023-10-01T09:35:21.814 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2513447710000001, 2023-10-01T09:35:21.815 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.815 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.815 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.815 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.815 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.815 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 159825 2023-10-01T09:35:21.815 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.815 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.815 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.815 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.815 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380723+0000", 2023-10-01T09:35:21.816 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.816 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.816 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.816 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.816 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380723+0000", 2023-10-01T09:35:21.816 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:21.816 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.816 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.816 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.816 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380722+0000", 2023-10-01T09:35:21.816 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9830000000000001e-06 2023-10-01T09:35:21.816 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.817 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.817 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.817 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380728+0000", 2023-10-01T09:35:21.817 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.329e-05 2023-10-01T09:35:21.817 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.817 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.817 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.817 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380741+0000", 2023-10-01T09:35:21.817 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3140000000000003e-06 2023-10-01T09:35:21.817 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.817 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.817 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.818 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380747+0000", 2023-10-01T09:35:21.818 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2110000000000002e-05 2023-10-01T09:35:21.818 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.818 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.818 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.818 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380789+0000", 2023-10-01T09:35:21.818 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0670000000000003e-06 2023-10-01T09:35:21.818 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.818 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.818 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:21.818 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380796+0000", 2023-10-01T09:35:21.819 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.062726183000000005 2023-10-01T09:35:21.819 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.819 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.819 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.819 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.443522+0000", 2023-10-01T09:35:21.819 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5750000000000001e-06 2023-10-01T09:35:21.819 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.819 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.819 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:21.819 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.443531+0000", 2023-10-01T09:35:21.819 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1746131310000001 2023-10-01T09:35:21.819 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.819 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.820 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.820 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.618144+0000", 2023-10-01T09:35:21.820 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00018594600000000001 2023-10-01T09:35:21.820 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.820 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.820 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.820 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.618330+0000", 2023-10-01T09:35:21.820 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000157885 2023-10-01T09:35:21.820 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.820 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.820 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,4,7", 2023-10-01T09:35:21.821 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.618488+0000", 2023-10-01T09:35:21.821 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0091334640000000009 2023-10-01T09:35:21.821 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.821 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.821 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.821 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.627621+0000", 2023-10-01T09:35:21.821 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000203346 2023-10-01T09:35:21.821 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.821 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.821 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.821 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.627824+0000", 2023-10-01T09:35:21.822 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024199270000000001 2023-10-01T09:35:21.822 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.822 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.822 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.822 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.630244+0000", 2023-10-01T09:35:21.822 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0016233860000000001 2023-10-01T09:35:21.822 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.822 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.822 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.822 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.631868+0000", 2023-10-01T09:35:21.822 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6780999999999996e-05 2023-10-01T09:35:21.822 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.823 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.823 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.823 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.631934+0000", 2023-10-01T09:35:21.823 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013365799999999999 2023-10-01T09:35:21.823 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.823 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.823 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.823 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.632068+0000", 2023-10-01T09:35:21.823 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2513447710000001 2023-10-01T09:35:21.823 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.823 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.823 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.824 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.824 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.824 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:159837 8.5 8:a337ee5f:::benchmark_data_smithi138_431617_object58426:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-10-01T09:35:21.824 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:27.399363+0000", 2023-10-01T09:35:21.824 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 174.275173604, 2023-10-01T09:35:21.824 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2329211799999999, 2023-10-01T09:35:21.824 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.824 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.824 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.824 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.824 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.825 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 159837 2023-10-01T09:35:21.825 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.825 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.825 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.825 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.825 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399363+0000", 2023-10-01T09:35:21.825 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.825 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.825 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.825 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.825 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399363+0000", 2023-10-01T09:35:21.826 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:21.826 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.826 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.826 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.826 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399362+0000", 2023-10-01T09:35:21.826 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1049999999999995e-06 2023-10-01T09:35:21.826 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.826 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.826 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.826 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399370+0000", 2023-10-01T09:35:21.826 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3217999999999999e-05 2023-10-01T09:35:21.826 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.827 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.827 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.827 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399383+0000", 2023-10-01T09:35:21.827 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.575e-06 2023-10-01T09:35:21.827 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.827 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.827 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.827 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399389+0000", 2023-10-01T09:35:21.827 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6829000000000002e-05 2023-10-01T09:35:21.827 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.827 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.828 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.828 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399436+0000", 2023-10-01T09:35:21.828 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4390000000000003e-06 2023-10-01T09:35:21.828 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.828 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.828 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:21.828 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399443+0000", 2023-10-01T09:35:21.828 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.044108631000000002 2023-10-01T09:35:21.828 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.828 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.828 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.828 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.443552+0000", 2023-10-01T09:35:21.828 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6140000000000003e-06 2023-10-01T09:35:21.829 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.829 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.829 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:21.829 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.443560+0000", 2023-10-01T09:35:21.829 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1789550630000001 2023-10-01T09:35:21.829 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.829 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.829 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.829 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.622515+0000", 2023-10-01T09:35:21.829 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012748499999999999 2023-10-01T09:35:21.829 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.829 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.830 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.830 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.622642+0000", 2023-10-01T09:35:21.830 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015045100000000001 2023-10-01T09:35:21.830 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.830 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.830 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,4,7", 2023-10-01T09:35:21.830 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.622793+0000", 2023-10-01T09:35:21.830 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050188949999999998 2023-10-01T09:35:21.830 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.830 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.830 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.831 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.627811+0000", 2023-10-01T09:35:21.831 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002583288 2023-10-01T09:35:21.831 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.831 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.831 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.831 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.630395+0000", 2023-10-01T09:35:21.831 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00032070599999999998 2023-10-01T09:35:21.831 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.831 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.831 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.831 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.630715+0000", 2023-10-01T09:35:21.831 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0013941660000000001 2023-10-01T09:35:21.832 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.832 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.832 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.832 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.632110+0000", 2023-10-01T09:35:21.832 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2788999999999999e-05 2023-10-01T09:35:21.832 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.832 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.832 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.832 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.632162+0000", 2023-10-01T09:35:21.832 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000122216 2023-10-01T09:35:21.832 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.832 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.833 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.833 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.632285+0000", 2023-10-01T09:35:21.833 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2329211799999999 2023-10-01T09:35:21.833 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.833 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.833 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.833 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.833 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.833 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:161648 8.5 8:a4b0b645:::benchmark_data_smithi138_431617_object60237:head [delete] snapc 0=[] ondisk+write+known_if_redirected e364)", 2023-10-01T09:35:21.833 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:31.211170+0000", 2023-10-01T09:35:21.833 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 170.463366694, 2023-10-01T09:35:21.833 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17941784, 2023-10-01T09:35:21.834 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.834 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.834 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.834 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.834 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.834 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 161648 2023-10-01T09:35:21.834 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.834 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.834 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.834 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.834 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211170+0000", 2023-10-01T09:35:21.834 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.835 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.835 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.835 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.835 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211170+0000", 2023-10-01T09:35:21.835 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:35:21.835 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.835 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.835 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.835 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211168+0000", 2023-10-01T09:35:21.835 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4460000000000001e-06 2023-10-01T09:35:21.836 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.836 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.836 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.836 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211176+0000", 2023-10-01T09:35:21.836 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1656e-05 2023-10-01T09:35:21.836 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.836 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.836 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.836 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211188+0000", 2023-10-01T09:35:21.836 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6570000000000002e-06 2023-10-01T09:35:21.836 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.836 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.837 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.837 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211194+0000", 2023-10-01T09:35:21.837 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00048738600000000003 2023-10-01T09:35:21.837 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.837 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.837 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.837 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211681+0000", 2023-10-01T09:35:21.837 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000131089 2023-10-01T09:35:21.837 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.837 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.837 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.837 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211812+0000", 2023-10-01T09:35:21.838 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000138503 2023-10-01T09:35:21.838 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.838 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.838 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,4,7", 2023-10-01T09:35:21.838 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211951+0000", 2023-10-01T09:35:21.838 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013556732 2023-10-01T09:35:21.838 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.838 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.838 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.838 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.225507+0000", 2023-10-01T09:35:21.838 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0031414619999999998 2023-10-01T09:35:21.839 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.839 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.839 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.839 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.228649+0000", 2023-10-01T09:35:21.839 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.009401487 2023-10-01T09:35:21.839 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.839 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.839 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.839 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.238050+0000", 2023-10-01T09:35:21.839 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15239425700000001 2023-10-01T09:35:21.839 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.839 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.840 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.840 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.390445+0000", 2023-10-01T09:35:21.840 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9906e-05 2023-10-01T09:35:21.840 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.840 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.840 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.840 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.390495+0000", 2023-10-01T09:35:21.840 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3560999999999994e-05 2023-10-01T09:35:21.840 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.840 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.840 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.841 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.390588+0000", 2023-10-01T09:35:21.841 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17941784 2023-10-01T09:35:21.841 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.841 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.841 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.841 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.841 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.841 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:161661 8.5 8:aec7fe18:::benchmark_data_smithi138_431617_object60250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e364)", 2023-10-01T09:35:21.841 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:31.232803+0000", 2023-10-01T09:35:21.841 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 170.44173432400001, 2023-10-01T09:35:21.841 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16878474500000001, 2023-10-01T09:35:21.841 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:21.842 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:21.842 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:21.842 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:21.842 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:21.842 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 161661 2023-10-01T09:35:21.842 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.842 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:21.842 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.842 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:21.842 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.232803+0000", 2023-10-01T09:35:21.842 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:21.843 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.843 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.843 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:21.843 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.232803+0000", 2023-10-01T09:35:21.843 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:35:21.843 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.843 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.843 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:21.843 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.232802+0000", 2023-10-01T09:35:21.843 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9800000000000003e-06 2023-10-01T09:35:21.843 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.844 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.844 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:21.844 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.232808+0000", 2023-10-01T09:35:21.844 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0929999999999992e-06 2023-10-01T09:35:21.844 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.844 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.844 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:21.844 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.232817+0000", 2023-10-01T09:35:21.844 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5669999999999996e-06 2023-10-01T09:35:21.844 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.844 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.844 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:21.845 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.232821+0000", 2023-10-01T09:35:21.845 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00095037599999999996 2023-10-01T09:35:21.845 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.845 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.845 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:21.845 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.233772+0000", 2023-10-01T09:35:21.845 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014877 2023-10-01T09:35:21.845 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.845 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.845 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:21.845 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.233920+0000", 2023-10-01T09:35:21.846 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00018008600000000001 2023-10-01T09:35:21.846 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.846 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.846 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,4,7", 2023-10-01T09:35:21.846 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.234100+0000", 2023-10-01T09:35:21.846 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0072926019999999996 2023-10-01T09:35:21.846 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.846 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.846 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.846 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.241393+0000", 2023-10-01T09:35:21.846 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2293000000000001e-05 2023-10-01T09:35:21.847 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.847 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.847 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:21.847 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.241405+0000", 2023-10-01T09:35:21.847 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001707376 2023-10-01T09:35:21.847 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.847 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.847 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.847 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.243113+0000", 2023-10-01T09:35:21.847 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15829549900000001 2023-10-01T09:35:21.847 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.847 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.848 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:21.848 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.401408+0000", 2023-10-01T09:35:21.848 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8941000000000003e-05 2023-10-01T09:35:21.848 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.848 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:21.848 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.401477+0000", 2023-10-01T09:35:21.848 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000110337 2023-10-01T09:35:21.848 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:21.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:21.848 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:21.849 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.401587+0000", 2023-10-01T09:35:21.849 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16878474500000001 2023-10-01T09:35:21.849 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.849 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.849 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.849 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:21.849 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:21.849 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:35:21.849 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:21 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 4884/248160 objects degraded (1.968%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:21.849 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:21 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1899696219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:21.850 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:35:21.850 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:35:21.850 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:35:21.850 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:35:21.851 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:35:21.851 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:35:21.851 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:35:21.851 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:35:21.851 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:35:21.851 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 65 16 85034 85018 81.7136 44.125 0.015265 0.0122317 2023-10-01T09:35:21.986 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:21.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:21.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:22.092 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:22.171 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:22.565 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.6 primary-affinity to 0 (802) 2023-10-01T09:35:22.658 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:35:22.770 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 66 16 85577 85561 80.9899 33.9375 0.0327165 0.0123398 2023-10-01T09:35:22.781 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:22.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:22.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:22.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:22 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3304408681' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-01T09:35:22.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:22 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-01T09:35:22.826 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:22 smithi138 ceph-mon[185886]: pgmap v1158: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 642 op/s; 4884/251424 objects degraded (1.943%); 9462/251424 objects misplaced (3.763%); 2.1 MiB/s, 35 objects/s recovering 2023-10-01T09:35:22.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:22 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3304408681' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-01T09:35:22.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:22 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-01T09:35:22.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:22 smithi138 ceph-mon[192513]: pgmap v1158: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 642 op/s; 4884/251424 objects degraded (1.943%); 9462/251424 objects misplaced (3.763%); 2.1 MiB/s, 35 objects/s recovering 2023-10-01T09:35:22.884 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:22 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3304408681' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-01T09:35:22.885 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:22 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]: dispatch 2023-10-01T09:35:22.885 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:22 smithi190 ceph-mon[160435]: pgmap v1158: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 642 op/s; 4884/251424 objects degraded (1.943%); 9462/251424 objects misplaced (3.763%); 2.1 MiB/s, 35 objects/s recovering 2023-10-01T09:35:22.887 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:22.899 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:35:22.955 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:23.145 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:23.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:23.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:23.256 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:23.291 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:23.689 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:23.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:23.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:23.726 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:35:23.770 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 67 16 86167 86151 80.3317 36.875 0.0169929 0.0124425 2023-10-01T09:35:23.794 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:23.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:23 smithi138 ceph-mon[192513]: failed to encode map e418 with expected crc 2023-10-01T09:35:23.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:23 smithi138 ceph-mon[192513]: failed to encode map e418 with expected crc 2023-10-01T09:35:23.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:23 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-01T09:35:23.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:23 smithi138 ceph-mon[192513]: osdmap e418: 8 total, 8 up, 7 in 2023-10-01T09:35:23.815 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:23 smithi138 ceph-mon[192513]: failed to encode map e418 with expected crc 2023-10-01T09:35:23.815 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:23 smithi138 ceph-mon[185886]: failed to encode map e418 with expected crc 2023-10-01T09:35:23.815 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:23 smithi138 ceph-mon[185886]: failed to encode map e418 with expected crc 2023-10-01T09:35:23.816 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:23 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-01T09:35:23.816 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:23 smithi138 ceph-mon[185886]: osdmap e418: 8 total, 8 up, 7 in 2023-10-01T09:35:23.816 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:23 smithi138 ceph-mon[185886]: failed to encode map e418 with expected crc 2023-10-01T09:35:23.920 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:23 smithi190 ceph-mon[160435]: failed to encode map e418 with expected crc 2023-10-01T09:35:23.920 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:23 smithi190 ceph-mon[160435]: failed to encode map e418 with expected crc 2023-10-01T09:35:23.920 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:23 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.0}]': finished 2023-10-01T09:35:23.920 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:23 smithi190 ceph-mon[160435]: osdmap e418: 8 total, 8 up, 7 in 2023-10-01T09:35:23.920 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:23 smithi190 ceph-mon[160435]: failed to encode map e418 with expected crc 2023-10-01T09:35:23.925 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:24.053 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:24.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:24.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:24.156 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:24.164 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:24.249 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:24.375 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:24.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:24.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:24.485 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:24.560 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:24.586 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:24 smithi190 ceph-mon[160435]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.586 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:24 smithi190 ceph-mon[160435]: osdmap e419: 8 total, 8 up, 7 in 2023-10-01T09:35:24.586 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:24 smithi190 ceph-mon[160435]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.587 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:24 smithi190 ceph-mon[160435]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.587 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:24 smithi190 ceph-mon[160435]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.587 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:24 smithi190 ceph-mon[160435]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.587 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:24 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:24.587 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:24 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:24.587 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:24 smithi190 ceph-mon[160435]: pgmap v1161: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 824 op/s; 4884/255684 objects degraded (1.910%); 9362/255684 objects misplaced (3.662%); 3.4 MiB/s, 52 objects/s recovering 2023-10-01T09:35:24.770 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 68 16 86912 86896 79.8352 46.5625 0.0157132 0.0125201 2023-10-01T09:35:24.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[185886]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[185886]: osdmap e419: 8 total, 8 up, 7 in 2023-10-01T09:35:24.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[185886]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.773 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[185886]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[185886]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[185886]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:24.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:24.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[185886]: pgmap v1161: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 824 op/s; 4884/255684 objects degraded (1.910%); 9362/255684 objects misplaced (3.662%); 3.4 MiB/s, 52 objects/s recovering 2023-10-01T09:35:24.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[192513]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[192513]: osdmap e419: 8 total, 8 up, 7 in 2023-10-01T09:35:24.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[192513]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[192513]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[192513]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[192513]: failed to encode map e419 with expected crc 2023-10-01T09:35:24.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:24.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:24.775 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:24 smithi138 ceph-mon[192513]: pgmap v1161: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 52 MiB/s wr, 824 op/s; 4884/255684 objects degraded (1.910%); 9362/255684 objects misplaced (3.662%); 3.4 MiB/s, 52 objects/s recovering 2023-10-01T09:35:24.825 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:24.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:24.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:24.939 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:24.968 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:25.012 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:25.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:25.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:25.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:35:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:35:24] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.33.4" 2023-10-01T09:35:25.116 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:25.145 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:25.592 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:35:25.770 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 69 16 87487 87471 79.1991 35.9375 0.0232945 0.0126206 2023-10-01T09:35:26.101 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:35:26.101 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:35:26.198 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:26.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:26.317 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:26.389 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:26.453 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:26.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:26.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:26.462 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:26.462 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:35:26.462 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:26.462 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:26.462 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:26.462 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:26 smithi190 ceph-mon[160435]: pgmap v1162: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 765 op/s; 4884/258234 objects degraded (1.891%); 9252/258234 objects misplaced (3.583%); 3.6 MiB/s, 56 objects/s recovering 2023-10-01T09:35:26.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:26.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:35:26.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:26.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:26.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:26.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:26 smithi138 ceph-mon[185886]: pgmap v1162: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 765 op/s; 4884/258234 objects degraded (1.891%); 9252/258234 objects misplaced (3.583%); 3.6 MiB/s, 56 objects/s recovering 2023-10-01T09:35:26.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:26.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:35:26.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:26.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:26.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:26.543 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:26 smithi138 ceph-mon[192513]: pgmap v1162: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 765 op/s; 4884/258234 objects degraded (1.891%); 9252/258234 objects misplaced (3.583%); 3.6 MiB/s, 56 objects/s recovering 2023-10-01T09:35:26.562 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:26.627 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:26.705 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:26.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:26.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:26.709 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:26.770 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 70 16 88174 88158 78.6812 42.9375 0.0135177 0.0126993 2023-10-01T09:35:26.808 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:26.862 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:26.971 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:26.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:26.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:27.078 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:27.159 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:27.308 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T09:35:27.374 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:27.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:27.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:27.480 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:27.522 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:27 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:35:27.522 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:27 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:35:27.522 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:27 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:27.522 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:27 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 4884/258234 objects degraded (1.891%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:27.522 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:27 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2580124730' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:35:27.522 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:27 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:35:27.523 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:27 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:35:27.523 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:27 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:27.523 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:27 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 4884/258234 objects degraded (1.891%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:27.523 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:27 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2580124730' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:35:27.526 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:27.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:27 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:35:27.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:27 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:35:27.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:27 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:35:27.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:27 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 4884/258234 objects degraded (1.891%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:27.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:27 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2580124730' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:35:27.597 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:27.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:27.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:27.702 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:27.770 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 71 16 88961 88945 78.2659 49.1875 0.015382 0.012771 2023-10-01T09:35:27.776 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:27.878 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:27.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:27.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:27.941 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:35:27.949 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:35:27.988 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:28.029 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:28.570 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:28.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:28.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:28.677 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:28.730 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:28.770 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 72 16 89695 89679 77.8161 45.875 0.0190664 0.0128447 2023-10-01T09:35:28.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:28 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:35:28.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:28 smithi138 ceph-mon[192513]: failed to encode map e420 with expected crc 2023-10-01T09:35:28.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2580124730' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:35:28.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:28 smithi138 ceph-mon[192513]: osdmap e420: 8 total, 8 up, 7 in 2023-10-01T09:35:28.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:28 smithi138 ceph-mon[192513]: failed to encode map e420 with expected crc 2023-10-01T09:35:28.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:28 smithi138 ceph-mon[192513]: pgmap v1164: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 34 MiB/s wr, 536 op/s; 4884/261087 objects degraded (1.871%); 9252/261087 objects misplaced (3.544%); 2.2 MiB/s, 32 objects/s recovering 2023-10-01T09:35:28.808 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:28 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:35:28.808 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:28 smithi138 ceph-mon[185886]: failed to encode map e420 with expected crc 2023-10-01T09:35:28.808 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2580124730' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:35:28.808 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:28 smithi138 ceph-mon[185886]: osdmap e420: 8 total, 8 up, 7 in 2023-10-01T09:35:28.808 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:28 smithi138 ceph-mon[185886]: failed to encode map e420 with expected crc 2023-10-01T09:35:28.808 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:28 smithi138 ceph-mon[185886]: pgmap v1164: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 34 MiB/s wr, 536 op/s; 4884/261087 objects degraded (1.871%); 9252/261087 objects misplaced (3.544%); 2.2 MiB/s, 32 objects/s recovering 2023-10-01T09:35:28.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:28 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:35:28.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:28 smithi190 ceph-mon[160435]: failed to encode map e420 with expected crc 2023-10-01T09:35:28.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2580124730' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:35:28.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:28 smithi190 ceph-mon[160435]: osdmap e420: 8 total, 8 up, 7 in 2023-10-01T09:35:28.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:28 smithi190 ceph-mon[160435]: failed to encode map e420 with expected crc 2023-10-01T09:35:28.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:28 smithi190 ceph-mon[160435]: pgmap v1164: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 34 MiB/s wr, 536 op/s; 4884/261087 objects degraded (1.871%); 9252/261087 objects misplaced (3.544%); 2.2 MiB/s, 32 objects/s recovering 2023-10-01T09:35:28.815 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 5, 6, 4] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:35:28.815 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:35:28.815 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-01T09:35:28.816 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-01T09:35:29.015 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:29.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:29.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:29.130 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:29.165 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:29.266 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:29.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:29.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:29.371 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:29.451 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:29.520 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:29.565 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:29.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:29.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:29.672 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:29.701 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:35:29.730 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:29.735 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:35:29.767 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:35:29.770 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 73 16 90269 90253 77.2417 35.875 0.0153121 0.0129404 2023-10-01T09:35:29.808 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:35:29.844 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:35:29.849 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:29.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:29.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:29.891 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:35:29.926 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:35:29.963 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:29.989 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:35:30.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:35:30.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:35:30.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:35:30.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:35:30.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:35:30.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:35:30.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:35:30.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:35:30.033 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:30.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:29 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/711610418' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:30.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:29 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/711610418' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:30.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:29 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/711610418' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:30.418 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:35:30.419 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:35:30.419 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:35:30.419 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:35:30.419 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:35:30.472 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:30.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:30.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:30.581 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:30.642 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:30.773 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 74 16 91020 91004 76.8322 46.9375 0.0167898 0.013002 2023-10-01T09:35:30.876 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:30.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:30.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:30.988 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:31.017 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:31.060 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:31.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:31.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:31.119 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:35:31.161 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:30 smithi138 ceph-mon[192513]: pgmap v1165: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 743 op/s; 4884/268242 objects degraded (1.821%); 9013/268242 objects misplaced (3.360%); 3.7 MiB/s, 57 objects/s recovering 2023-10-01T09:35:31.161 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:30 smithi138 ceph-mon[185886]: pgmap v1165: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 743 op/s; 4884/268242 objects degraded (1.821%); 9013/268242 objects misplaced (3.360%); 3.7 MiB/s, 57 objects/s recovering 2023-10-01T09:35:31.163 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:31.189 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:31.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:30 smithi190 ceph-mon[160435]: pgmap v1165: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 743 op/s; 4884/268242 objects degraded (1.821%); 9013/268242 objects misplaced (3.360%); 3.7 MiB/s, 57 objects/s recovering 2023-10-01T09:35:31.497 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T09:35:31.499 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:31.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:31.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:31.616 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:31.662 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:31.771 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 75 16 91512 91496 76.2179 30.75 0.0117072 0.0131135 2023-10-01T09:35:31.876 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:31.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:31.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:31.987 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:32.059 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:32.144 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:35:32.160 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:32.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:32.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:32.277 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:32.328 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:32 smithi138 ceph-mon[185886]: pgmap v1166: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 49 MiB/s wr, 784 op/s; 4884/271485 objects degraded (1.799%); 9013/271485 objects misplaced (3.320%); 3.3 MiB/s, 51 objects/s recovering 2023-10-01T09:35:32.328 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:32 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 4884/271485 objects degraded (1.799%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:32.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:32 smithi138 ceph-mon[192513]: pgmap v1166: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 49 MiB/s wr, 784 op/s; 4884/271485 objects degraded (1.799%); 9013/271485 objects misplaced (3.320%); 3.3 MiB/s, 51 objects/s recovering 2023-10-01T09:35:32.329 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:32 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 4884/271485 objects degraded (1.799%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:32.330 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:32.368 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:32.452 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:32.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:32.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:32.475 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:32 smithi190 ceph-mon[160435]: pgmap v1166: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 49 MiB/s wr, 784 op/s; 4884/271485 objects degraded (1.799%); 9013/271485 objects misplaced (3.320%); 3.3 MiB/s, 51 objects/s recovering 2023-10-01T09:35:32.475 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:32 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 4884/271485 objects degraded (1.799%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:32.569 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:32.623 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:32.771 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 76 16 92239 92223 75.8129 45.4375 0.0151048 0.0131852 2023-10-01T09:35:32.895 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:32.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:32.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:33.005 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:33.067 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:33.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:32 smithi138 conmon[185862]: 2023-10-01T09:35:32.904+0000 7f70d64ec700 -1 mon.a@0(leader).osd e420 definitely_dead 0 2023-10-01T09:35:33.156 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:33.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:33.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:33.265 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:33.313 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:33.359 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:33.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:33.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:33.463 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:33.527 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:33 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3723665415' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:35:33.546 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:33.563 INFO:teuthology.orchestra.run.smithi138.stderr:marked in osd.2. 2023-10-01T09:35:33.771 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 77 16 92832 92816 75.3098 37.0625 0.00335075 0.0132478 2023-10-01T09:35:33.815 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:33.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:33.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:33.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:33 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3723665415' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:35:33.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:33 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3723665415' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:35:33.861 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:35:33.920 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:33.949 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:34.095 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:34.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:34.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:34.200 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:34.227 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:34.239 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:35:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:35:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:35:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:35:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:183790 8.9 8.e1f9c39 (undecoded) ondisk+retry+write+known_if_redirected e379)", 2023-10-01T09:35:34.239 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:10.516097+0000", 2023-10-01T09:35:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 143.72069119299999, 2023-10-01T09:35:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98694263999999998, 2023-10-01T09:35:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:35:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183790 2023-10-01T09:35:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.240 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516097+0000", 2023-10-01T09:35:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516097+0000", 2023-10-01T09:35:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.241 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516095+0000", 2023-10-01T09:35:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.1649999999999995e-06 2023-10-01T09:35:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516104+0000", 2023-10-01T09:35:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4233000000000001e-05 2023-10-01T09:35:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.242 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516118+0000", 2023-10-01T09:35:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2600000000000005e-06 2023-10-01T09:35:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516126+0000", 2023-10-01T09:35:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7398999999999997e-05 2023-10-01T09:35:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.243 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516184+0000", 2023-10-01T09:35:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8756999999999999e-05 2023-10-01T09:35:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.516213+0000", 2023-10-01T09:35:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98681300500000002 2023-10-01T09:35:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.244 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:11.503026+0000", 2023-10-01T09:35:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.364e-05 2023-10-01T09:35:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:11.503039+0000", 2023-10-01T09:35:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98694263999999998 2023-10-01T09:35:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.245 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:183811 8.8 8.36abb188 (undecoded) ondisk+write+known_if_redirected e379)", 2023-10-01T09:35:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:10.519813+0000", 2023-10-01T09:35:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 143.71697526899999, 2023-10-01T09:35:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98341005000000004, 2023-10-01T09:35:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:35:34.246 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183811 2023-10-01T09:35:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.247 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.519813+0000", 2023-10-01T09:35:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.519813+0000", 2023-10-01T09:35:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.248 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.519811+0000", 2023-10-01T09:35:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.238e-06 2023-10-01T09:35:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.519818+0000", 2023-10-01T09:35:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1250000000000001e-05 2023-10-01T09:35:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.249 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.519829+0000", 2023-10-01T09:35:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9419999999999998e-06 2023-10-01T09:35:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.519836+0000", 2023-10-01T09:35:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00046759699999999999 2023-10-01T09:35:34.250 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.520304+0000", 2023-10-01T09:35:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9529999999999994e-06 2023-10-01T09:35:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.520314+0000", 2023-10-01T09:35:34.251 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98288837200000001 2023-10-01T09:35:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:11.503202+0000", 2023-10-01T09:35:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0701000000000001e-05 2023-10-01T09:35:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:11.503223+0000", 2023-10-01T09:35:34.252 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98341005000000004 2023-10-01T09:35:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:183813 8.9 8.3ad4cfa9 (undecoded) ondisk+write+known_if_redirected e379)", 2023-10-01T09:35:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:10.525835+0000", 2023-10-01T09:35:34.253 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 143.71095292199999, 2023-10-01T09:35:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97725383600000004, 2023-10-01T09:35:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:35:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183813 2023-10-01T09:35:34.254 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525835+0000", 2023-10-01T09:35:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.255 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525835+0000", 2023-10-01T09:35:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525834+0000", 2023-10-01T09:35:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2539999999999998e-06 2023-10-01T09:35:34.256 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525840+0000", 2023-10-01T09:35:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2208000000000001e-05 2023-10-01T09:35:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525852+0000", 2023-10-01T09:35:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8560000000000003e-06 2023-10-01T09:35:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.257 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525858+0000", 2023-10-01T09:35:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1612e-05 2023-10-01T09:35:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525909+0000", 2023-10-01T09:35:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9480000000000003e-06 2023-10-01T09:35:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.258 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.525919+0000", 2023-10-01T09:35:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97715963400000005 2023-10-01T09:35:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:11.503079+0000", 2023-10-01T09:35:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.679e-06 2023-10-01T09:35:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.259 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:11.503089+0000", 2023-10-01T09:35:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97725383600000004 2023-10-01T09:35:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:183817 8.0 8.590ebaa0 (undecoded) ondisk+write+known_if_redirected e379)", 2023-10-01T09:35:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:10.607211+0000", 2023-10-01T09:35:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 143.629576609, 2023-10-01T09:35:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0483890790000001, 2023-10-01T09:35:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:35:34.260 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183817 2023-10-01T09:35:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607211+0000", 2023-10-01T09:35:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.261 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607211+0000", 2023-10-01T09:35:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607210+0000", 2023-10-01T09:35:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2490000000000001e-06 2023-10-01T09:35:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.262 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607218+0000", 2023-10-01T09:35:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5078e-05 2023-10-01T09:35:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607233+0000", 2023-10-01T09:35:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0129999999999996e-06 2023-10-01T09:35:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607240+0000", 2023-10-01T09:35:34.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3373999999999993e-05 2023-10-01T09:35:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607303+0000", 2023-10-01T09:35:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1518999999999999e-05 2023-10-01T09:35:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.607315+0000", 2023-10-01T09:35:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0482726630000001 2023-10-01T09:35:34.264 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:11.655587+0000", 2023-10-01T09:35:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2812e-05 2023-10-01T09:35:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:11.655600+0000", 2023-10-01T09:35:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0483890790000001 2023-10-01T09:35:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.265 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:183832 8.d 8.de11130d (undecoded) ondisk+write+known_if_redirected e379)", 2023-10-01T09:35:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:10.640122+0000", 2023-10-01T09:35:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 143.596665766, 2023-10-01T09:35:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.021125837, 2023-10-01T09:35:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:35:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.266 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183832 2023-10-01T09:35:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640122+0000", 2023-10-01T09:35:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.267 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640122+0000", 2023-10-01T09:35:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640120+0000", 2023-10-01T09:35:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0611000000000001e-05 2023-10-01T09:35:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640131+0000", 2023-10-01T09:35:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8621999999999999e-05 2023-10-01T09:35:34.268 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640149+0000", 2023-10-01T09:35:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1699999999999997e-06 2023-10-01T09:35:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640158+0000", 2023-10-01T09:35:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3267000000000002e-05 2023-10-01T09:35:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.269 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640231+0000", 2023-10-01T09:35:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3482999999999999e-05 2023-10-01T09:35:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.640244+0000", 2023-10-01T09:35:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0209910099999999 2023-10-01T09:35:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.270 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:11.661235+0000", 2023-10-01T09:35:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2527999999999999e-05 2023-10-01T09:35:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:11.661248+0000", 2023-10-01T09:35:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.021125837 2023-10-01T09:35:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.271 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:183834 8.8 8.80949338 (undecoded) ondisk+write+known_if_redirected e379)", 2023-10-01T09:35:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:10.642470+0000", 2023-10-01T09:35:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 143.59431826799999, 2023-10-01T09:35:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.011054621, 2023-10-01T09:35:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:35:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 183834 2023-10-01T09:35:34.272 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642470+0000", 2023-10-01T09:35:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642470+0000", 2023-10-01T09:35:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:35:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642469+0000", 2023-10-01T09:35:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1830000000000002e-06 2023-10-01T09:35:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642474+0000", 2023-10-01T09:35:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4800000000000001e-05 2023-10-01T09:35:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642489+0000", 2023-10-01T09:35:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6120000000000004e-06 2023-10-01T09:35:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642495+0000", 2023-10-01T09:35:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9181999999999998e-05 2023-10-01T09:35:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.275 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642545+0000", 2023-10-01T09:35:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3410000000000003e-06 2023-10-01T09:35:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:10.642553+0000", 2023-10-01T09:35:34.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.010515136 2023-10-01T09:35:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:11.653068+0000", 2023-10-01T09:35:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00045616899999999998 2023-10-01T09:35:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.277 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:11.653524+0000", 2023-10-01T09:35:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.011054621 2023-10-01T09:35:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.278 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:197664 8.9 8:99d82d28:::benchmark_data_smithi138_431617_object96253:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e391)", 2023-10-01T09:35:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:37.485463+0000", 2023-10-01T09:35:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.751324461, 2023-10-01T09:35:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0488023500000001, 2023-10-01T09:35:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:35:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 197664 2023-10-01T09:35:34.279 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485463+0000", 2023-10-01T09:35:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485463+0000", 2023-10-01T09:35:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485462+0000", 2023-10-01T09:35:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8970000000000008e-06 2023-10-01T09:35:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485469+0000", 2023-10-01T09:35:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7881e-05 2023-10-01T09:35:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485487+0000", 2023-10-01T09:35:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4780000000000008e-06 2023-10-01T09:35:34.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485496+0000", 2023-10-01T09:35:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9155999999999999e-05 2023-10-01T09:35:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.283 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485575+0000", 2023-10-01T09:35:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.081e-06 2023-10-01T09:35:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op must wait for map", 2023-10-01T09:35:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485584+0000", 2023-10-01T09:35:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0016865459999999999 2023-10-01T09:35:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.487271+0000", 2023-10-01T09:35:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.1090000000000004e-06 2023-10-01T09:35:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.487280+0000", 2023-10-01T09:35:34.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0468957400000001 2023-10-01T09:35:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.534175+0000", 2023-10-01T09:35:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0236999999999997e-05 2023-10-01T09:35:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.286 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.534266+0000", 2023-10-01T09:35:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0488023500000001 2023-10-01T09:35:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.287 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:197679 8.9 8:9cc4637e:::benchmark_data_smithi138_431617_object96268:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e391)", 2023-10-01T09:35:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:37.485512+0000", 2023-10-01T09:35:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.75127545799999, 2023-10-01T09:35:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.092398395, 2023-10-01T09:35:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.288 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 197679 2023-10-01T09:35:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485512+0000", 2023-10-01T09:35:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485512+0000", 2023-10-01T09:35:34.289 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:35:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485512+0000", 2023-10-01T09:35:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4860000000000002e-06 2023-10-01T09:35:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485515+0000", 2023-10-01T09:35:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6669999999999999e-06 2023-10-01T09:35:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485522+0000", 2023-10-01T09:35:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.763e-06 2023-10-01T09:35:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485527+0000", 2023-10-01T09:35:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00047059499999999998 2023-10-01T09:35:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.291 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.485997+0000", 2023-10-01T09:35:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5530000000000004e-06 2023-10-01T09:35:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting_for_map not empty", 2023-10-01T09:35:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.486005+0000", 2023-10-01T09:35:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001357914 2023-10-01T09:35:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.292 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.487363+0000", 2023-10-01T09:35:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.8149999999999995e-06 2023-10-01T09:35:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.487372+0000", 2023-10-01T09:35:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0469181299999999 2023-10-01T09:35:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.534290+0000", 2023-10-01T09:35:34.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.8895999999999996e-05 2023-10-01T09:35:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.534389+0000", 2023-10-01T09:35:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4265000000000006e-05 2023-10-01T09:35:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:35:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.534483+0000", 2023-10-01T09:35:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.01246893 2023-10-01T09:35:34.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.546952+0000", 2023-10-01T09:35:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.030874755 2023-10-01T09:35:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.577827+0000", 2023-10-01T09:35:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.694e-05 2023-10-01T09:35:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.577853+0000", 2023-10-01T09:35:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0256999999999998e-05 2023-10-01T09:35:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.577884+0000", 2023-10-01T09:35:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7035000000000001e-05 2023-10-01T09:35:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.577911+0000", 2023-10-01T09:35:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.092398395 2023-10-01T09:35:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:197684 8.9 8:9da19773:::benchmark_data_smithi138_431617_object96273:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-01T09:35:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:37.496081+0000", 2023-10-01T09:35:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.740706765, 2023-10-01T09:35:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.081947725, 2023-10-01T09:35:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.297 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 197684 2023-10-01T09:35:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.496081+0000", 2023-10-01T09:35:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.298 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.496081+0000", 2023-10-01T09:35:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:35:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.496080+0000", 2023-10-01T09:35:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9529999999999996e-06 2023-10-01T09:35:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.299 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.496086+0000", 2023-10-01T09:35:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3808000000000001e-05 2023-10-01T09:35:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.496100+0000", 2023-10-01T09:35:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6340000000000003e-06 2023-10-01T09:35:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.496106+0000", 2023-10-01T09:35:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6042000000000003e-05 2023-10-01T09:35:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.496162+0000", 2023-10-01T09:35:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3217000000000001e-05 2023-10-01T09:35:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.496176+0000", 2023-10-01T09:35:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0407086190000001 2023-10-01T09:35:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.536884+0000", 2023-10-01T09:35:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012582899999999999 2023-10-01T09:35:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.302 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.537010+0000", 2023-10-01T09:35:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000152225 2023-10-01T09:35:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:35:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.537162+0000", 2023-10-01T09:35:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013462765 2023-10-01T09:35:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:34.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.550625+0000", 2023-10-01T09:35:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.027322152999999998 2023-10-01T09:35:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.577947+0000", 2023-10-01T09:35:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5405999999999999e-05 2023-10-01T09:35:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.577983+0000", 2023-10-01T09:35:34.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5680000000000001e-05 2023-10-01T09:35:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578008+0000", 2023-10-01T09:35:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0471000000000001e-05 2023-10-01T09:35:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578029+0000", 2023-10-01T09:35:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.081947725 2023-10-01T09:35:34.305 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:197685 8.9 8:9e551f2d:::benchmark_data_smithi138_431617_object96274:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-01T09:35:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:37.496822+0000", 2023-10-01T09:35:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.73996565500001, 2023-10-01T09:35:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0813339829999999, 2023-10-01T09:35:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.306 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 197685 2023-10-01T09:35:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.496822+0000", 2023-10-01T09:35:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.496822+0000", 2023-10-01T09:35:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:35:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.496821+0000", 2023-10-01T09:35:34.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0010000000000001e-06 2023-10-01T09:35:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.496826+0000", 2023-10-01T09:35:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5310000000000004e-06 2023-10-01T09:35:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.496834+0000", 2023-10-01T09:35:34.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1160000000000004e-06 2023-10-01T09:35:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.496840+0000", 2023-10-01T09:35:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0039740089999999997 2023-10-01T09:35:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.500814+0000", 2023-10-01T09:35:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0149999999999999e-05 2023-10-01T09:35:34.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.500824+0000", 2023-10-01T09:35:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0390505990000001 2023-10-01T09:35:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.539875+0000", 2023-10-01T09:35:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000127024 2023-10-01T09:35:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.540002+0000", 2023-10-01T09:35:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015855700000000001 2023-10-01T09:35:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:35:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.540160+0000", 2023-10-01T09:35:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010474542999999999 2023-10-01T09:35:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.550635+0000", 2023-10-01T09:35:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.027426412000000001 2023-10-01T09:35:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578061+0000", 2023-10-01T09:35:34.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4008999999999998e-05 2023-10-01T09:35:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578105+0000", 2023-10-01T09:35:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6557999999999999e-05 2023-10-01T09:35:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:34.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578132+0000", 2023-10-01T09:35:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4261999999999999e-05 2023-10-01T09:35:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578156+0000", 2023-10-01T09:35:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0813339829999999 2023-10-01T09:35:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:197691 8.9 8:9e78963a:::benchmark_data_smithi138_431617_object96280:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-01T09:35:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:37.502445+0000", 2023-10-01T09:35:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.73434272999999, 2023-10-01T09:35:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0758567859999999, 2023-10-01T09:35:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.316 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 197691 2023-10-01T09:35:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.502445+0000", 2023-10-01T09:35:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.502445+0000", 2023-10-01T09:35:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.318 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.502444+0000", 2023-10-01T09:35:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0068000000000001e-05 2023-10-01T09:35:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.502454+0000", 2023-10-01T09:35:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1263999999999999e-05 2023-10-01T09:35:34.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.502465+0000", 2023-10-01T09:35:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.015e-06 2023-10-01T09:35:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.320 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.502471+0000", 2023-10-01T09:35:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8229000000000001e-05 2023-10-01T09:35:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.502529+0000", 2023-10-01T09:35:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0349e-05 2023-10-01T09:35:34.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.502540+0000", 2023-10-01T09:35:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0407272510000001 2023-10-01T09:35:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.543267+0000", 2023-10-01T09:35:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012748999999999999 2023-10-01T09:35:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.543394+0000", 2023-10-01T09:35:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014132000000000001 2023-10-01T09:35:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:35:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.543536+0000", 2023-10-01T09:35:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012798884 2023-10-01T09:35:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.556334+0000", 2023-10-01T09:35:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021753572999999998 2023-10-01T09:35:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578088+0000", 2023-10-01T09:35:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016616100000000001 2023-10-01T09:35:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578254+0000", 2023-10-01T09:35:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6308999999999998e-05 2023-10-01T09:35:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:34.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578281+0000", 2023-10-01T09:35:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1359999999999999e-05 2023-10-01T09:35:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578302+0000", 2023-10-01T09:35:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0758567859999999 2023-10-01T09:35:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:197694 8.9 8:96c94afe:::benchmark_data_smithi138_431617_object96283:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-01T09:35:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:37.513138+0000", 2023-10-01T09:35:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.723649761, 2023-10-01T09:35:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0652498720000001, 2023-10-01T09:35:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 197694 2023-10-01T09:35:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.327 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.513138+0000", 2023-10-01T09:35:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.513138+0000", 2023-10-01T09:35:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.328 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.513137+0000", 2023-10-01T09:35:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3969999999999999e-06 2023-10-01T09:35:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.513143+0000", 2023-10-01T09:35:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3572999999999999e-05 2023-10-01T09:35:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.513167+0000", 2023-10-01T09:35:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.587e-06 2023-10-01T09:35:34.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.513172+0000", 2023-10-01T09:35:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8690000000000002e-05 2023-10-01T09:35:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.513231+0000", 2023-10-01T09:35:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2343e-05 2023-10-01T09:35:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.513243+0000", 2023-10-01T09:35:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0337376599999999 2023-10-01T09:35:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.546981+0000", 2023-10-01T09:35:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013429699999999999 2023-10-01T09:35:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.547115+0000", 2023-10-01T09:35:34.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014562200000000001 2023-10-01T09:35:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:35:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.547261+0000", 2023-10-01T09:35:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014126536 2023-10-01T09:35:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.561387+0000", 2023-10-01T09:35:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016796236999999999 2023-10-01T09:35:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578184+0000", 2023-10-01T09:35:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000150471 2023-10-01T09:35:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578334+0000", 2023-10-01T09:35:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3130000000000003e-05 2023-10-01T09:35:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578367+0000", 2023-10-01T09:35:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.075e-05 2023-10-01T09:35:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578388+0000", 2023-10-01T09:35:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0652498720000001 2023-10-01T09:35:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:197701 8.9 8:980755a4:::benchmark_data_smithi138_431617_object96290:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-01T09:35:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:37.525954+0000", 2023-10-01T09:35:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.71083344199999, 2023-10-01T09:35:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0525059800000001, 2023-10-01T09:35:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.335 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 197701 2023-10-01T09:35:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.525954+0000", 2023-10-01T09:35:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.525954+0000", 2023-10-01T09:35:34.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.525952+0000", 2023-10-01T09:35:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.8190000000000003e-06 2023-10-01T09:35:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.525962+0000", 2023-10-01T09:35:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9089e-05 2023-10-01T09:35:34.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.525981+0000", 2023-10-01T09:35:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6899999999999998e-06 2023-10-01T09:35:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.525990+0000", 2023-10-01T09:35:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1540999999999998e-05 2023-10-01T09:35:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.526062+0000", 2023-10-01T09:35:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4443999999999999e-05 2023-10-01T09:35:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.526076+0000", 2023-10-01T09:35:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.024600065 2023-10-01T09:35:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.550676+0000", 2023-10-01T09:35:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000111476 2023-10-01T09:35:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.550788+0000", 2023-10-01T09:35:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5885999999999996e-05 2023-10-01T09:35:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:35:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.550883+0000", 2023-10-01T09:35:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010510118000000001 2023-10-01T09:35:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.561394+0000", 2023-10-01T09:35:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016836490999999999 2023-10-01T09:35:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578230+0000", 2023-10-01T09:35:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000186162 2023-10-01T09:35:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578416+0000", 2023-10-01T09:35:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4567e-05 2023-10-01T09:35:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578441+0000", 2023-10-01T09:35:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9541000000000001e-05 2023-10-01T09:35:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578460+0000", 2023-10-01T09:35:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0525059800000001 2023-10-01T09:35:34.343 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:197703 8.9 8:9f03eca5:::benchmark_data_smithi138_431617_object96292:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-01T09:35:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:37.530987+0000", 2023-10-01T09:35:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.705800584, 2023-10-01T09:35:34.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0476586880000001, 2023-10-01T09:35:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 197703 2023-10-01T09:35:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.530987+0000", 2023-10-01T09:35:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.530987+0000", 2023-10-01T09:35:34.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.530986+0000", 2023-10-01T09:35:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0169e-05 2023-10-01T09:35:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.530996+0000", 2023-10-01T09:35:34.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3104000000000001e-05 2023-10-01T09:35:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.531019+0000", 2023-10-01T09:35:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4089999999999992e-06 2023-10-01T09:35:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.531028+0000", 2023-10-01T09:35:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5628000000000003e-05 2023-10-01T09:35:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.531094+0000", 2023-10-01T09:35:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6191000000000001e-05 2023-10-01T09:35:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.531110+0000", 2023-10-01T09:35:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0219309670000001 2023-10-01T09:35:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.553041+0000", 2023-10-01T09:35:34.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000120464 2023-10-01T09:35:34.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:34.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.553161+0000", 2023-10-01T09:35:34.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000151779 2023-10-01T09:35:34.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:35:34.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.553313+0000", 2023-10-01T09:35:34.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014585832999999999 2023-10-01T09:35:34.352 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.352 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:34.352 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.567899+0000", 2023-10-01T09:35:34.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.01059059 2023-10-01T09:35:34.352 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.352 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.353 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578490+0000", 2023-10-01T09:35:34.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4957999999999996e-05 2023-10-01T09:35:34.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.353 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.353 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578565+0000", 2023-10-01T09:35:34.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3810999999999998e-05 2023-10-01T09:35:34.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:34.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578608+0000", 2023-10-01T09:35:34.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7455e-05 2023-10-01T09:35:34.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578646+0000", 2023-10-01T09:35:34.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0476586880000001 2023-10-01T09:35:34.355 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.355 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.355 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.355 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:197737 8.9 8:9e4d188f:::benchmark_data_smithi138_431617_object96326:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-01T09:35:34.355 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:37.586577+0000", 2023-10-01T09:35:34.356 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.650210335, 2023-10-01T09:35:34.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99222085800000004, 2023-10-01T09:35:34.356 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.356 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:34.356 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.356 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.356 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.356 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 197737 2023-10-01T09:35:34.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.356 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.586577+0000", 2023-10-01T09:35:34.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.586577+0000", 2023-10-01T09:35:34.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:34.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.586576+0000", 2023-10-01T09:35:34.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9510000000000006e-06 2023-10-01T09:35:34.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.586584+0000", 2023-10-01T09:35:34.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4109e-05 2023-10-01T09:35:34.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.586598+0000", 2023-10-01T09:35:34.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1469999999999998e-06 2023-10-01T09:35:34.359 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.359 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.359 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.359 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.586604+0000", 2023-10-01T09:35:34.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0468000000000004e-05 2023-10-01T09:35:34.359 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.359 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.359 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.359 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.586674+0000", 2023-10-01T09:35:34.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3482999999999999e-05 2023-10-01T09:35:34.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.360 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.360 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.586688+0000", 2023-10-01T09:35:34.360 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.96970416000000004 2023-10-01T09:35:34.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.360 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.360 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.556392+0000", 2023-10-01T09:35:34.360 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000119802 2023-10-01T09:35:34.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:34.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.556512+0000", 2023-10-01T09:35:34.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4883000000000006e-05 2023-10-01T09:35:34.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:35:34.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.556607+0000", 2023-10-01T09:35:34.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011302304000000001 2023-10-01T09:35:34.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:34.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.567909+0000", 2023-10-01T09:35:34.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010623143 2023-10-01T09:35:34.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578532+0000", 2023-10-01T09:35:34.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00018031000000000001 2023-10-01T09:35:34.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578712+0000", 2023-10-01T09:35:34.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6536999999999998e-05 2023-10-01T09:35:34.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:34.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578759+0000", 2023-10-01T09:35:34.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9459000000000003e-05 2023-10-01T09:35:34.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.578798+0000", 2023-10-01T09:35:34.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99222085800000004 2023-10-01T09:35:34.364 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.364 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.364 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.364 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:197741 8.9 8:90501378:::benchmark_data_smithi138_431617_object96330:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-01T09:35:34.364 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:37.589903+0000", 2023-10-01T09:35:34.364 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.646885055, 2023-10-01T09:35:34.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99282904500000002, 2023-10-01T09:35:34.364 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.364 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:34.365 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.365 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.365 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.365 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 197741 2023-10-01T09:35:34.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.365 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.589903+0000", 2023-10-01T09:35:34.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.589903+0000", 2023-10-01T09:35:34.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:34.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.589901+0000", 2023-10-01T09:35:34.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3350000000000001e-06 2023-10-01T09:35:34.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.367 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.589908+0000", 2023-10-01T09:35:34.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2244e-05 2023-10-01T09:35:34.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.367 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.367 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.589920+0000", 2023-10-01T09:35:34.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.613e-06 2023-10-01T09:35:34.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.367 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.367 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.589927+0000", 2023-10-01T09:35:34.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9760999999999999e-05 2023-10-01T09:35:34.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.368 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.368 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.589987+0000", 2023-10-01T09:35:34.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0009e-05 2023-10-01T09:35:34.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.368 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.368 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.589997+0000", 2023-10-01T09:35:34.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.96887634199999995 2023-10-01T09:35:34.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.558873+0000", 2023-10-01T09:35:34.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000123626 2023-10-01T09:35:34.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:34.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.558997+0000", 2023-10-01T09:35:34.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013617800000000001 2023-10-01T09:35:34.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:35:34.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.559133+0000", 2023-10-01T09:35:34.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013265056000000001 2023-10-01T09:35:34.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:34.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.572398+0000", 2023-10-01T09:35:34.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010059492999999999 2023-10-01T09:35:34.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.582458+0000", 2023-10-01T09:35:34.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017782299999999999 2023-10-01T09:35:34.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.582635+0000", 2023-10-01T09:35:34.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3851000000000003e-05 2023-10-01T09:35:34.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:34.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.582689+0000", 2023-10-01T09:35:34.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2568999999999998e-05 2023-10-01T09:35:34.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.582732+0000", 2023-10-01T09:35:34.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99282904500000002 2023-10-01T09:35:34.372 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.372 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.372 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.373 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:197751 8.9 8:917b1a0e:::benchmark_data_smithi138_431617_object96340:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-01T09:35:34.373 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:37.607673+0000", 2023-10-01T09:35:34.373 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.629115078, 2023-10-01T09:35:34.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98421412200000002, 2023-10-01T09:35:34.373 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.373 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:34.373 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.373 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.373 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.374 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 197751 2023-10-01T09:35:34.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.374 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.374 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.607673+0000", 2023-10-01T09:35:34.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.375 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.375 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.607673+0000", 2023-10-01T09:35:34.375 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:34.375 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.375 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.375 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.375 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.607671+0000", 2023-10-01T09:35:34.375 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6490000000000008e-06 2023-10-01T09:35:34.375 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.375 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.375 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.607679+0000", 2023-10-01T09:35:34.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6718000000000002e-05 2023-10-01T09:35:34.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.376 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.607695+0000", 2023-10-01T09:35:34.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9829999999999996e-06 2023-10-01T09:35:34.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.607703+0000", 2023-10-01T09:35:34.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2570999999999994e-05 2023-10-01T09:35:34.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.607766+0000", 2023-10-01T09:35:34.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2248e-05 2023-10-01T09:35:34.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.607778+0000", 2023-10-01T09:35:34.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.95365683700000003 2023-10-01T09:35:34.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.561435+0000", 2023-10-01T09:35:34.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000130591 2023-10-01T09:35:34.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:34.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.561566+0000", 2023-10-01T09:35:34.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00020872100000000001 2023-10-01T09:35:34.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:35:34.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.561774+0000", 2023-10-01T09:35:34.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010629753 2023-10-01T09:35:34.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.380 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:34.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.572404+0000", 2023-10-01T09:35:34.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010107799000000001 2023-10-01T09:35:34.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.380 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.582512+0000", 2023-10-01T09:35:34.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0092889640000000002 2023-10-01T09:35:34.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.591801+0000", 2023-10-01T09:35:34.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1096000000000001e-05 2023-10-01T09:35:34.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:34.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.591842+0000", 2023-10-01T09:35:34.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4969000000000003e-05 2023-10-01T09:35:34.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.382 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.382 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.591887+0000", 2023-10-01T09:35:34.382 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98421412200000002 2023-10-01T09:35:34.382 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.382 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.382 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.382 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.382 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.383 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:197757 8.9 8:929a0704:::benchmark_data_smithi138_431617_object96346:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-01T09:35:34.383 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:37.620166+0000", 2023-10-01T09:35:34.383 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.61662181, 2023-10-01T09:35:34.383 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97191588200000001, 2023-10-01T09:35:34.383 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.383 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:34.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.384 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 197757 2023-10-01T09:35:34.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.384 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.620166+0000", 2023-10-01T09:35:34.384 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.620166+0000", 2023-10-01T09:35:34.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:35:34.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.620164+0000", 2023-10-01T09:35:34.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0604e-05 2023-10-01T09:35:34.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.386 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.620174+0000", 2023-10-01T09:35:34.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8624999999999999e-05 2023-10-01T09:35:34.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.620193+0000", 2023-10-01T09:35:34.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1307e-05 2023-10-01T09:35:34.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.620204+0000", 2023-10-01T09:35:34.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3812000000000004e-05 2023-10-01T09:35:34.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.388 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.620278+0000", 2023-10-01T09:35:34.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4575e-05 2023-10-01T09:35:34.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.388 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.389 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.620293+0000", 2023-10-01T09:35:34.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.94487681400000001 2023-10-01T09:35:34.389 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.389 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.389 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.389 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.565170+0000", 2023-10-01T09:35:34.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012593899999999999 2023-10-01T09:35:34.389 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.389 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.390 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:34.390 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.565296+0000", 2023-10-01T09:35:34.390 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016369099999999999 2023-10-01T09:35:34.390 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.390 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.390 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:35:34.390 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.565459+0000", 2023-10-01T09:35:34.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012065660000000001 2023-10-01T09:35:34.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.391 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:34.391 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.577525+0000", 2023-10-01T09:35:34.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014402346999999999 2023-10-01T09:35:34.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.391 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.391 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.591927+0000", 2023-10-01T09:35:34.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7041999999999997e-05 2023-10-01T09:35:34.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.592024+0000", 2023-10-01T09:35:34.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1968999999999998e-05 2023-10-01T09:35:34.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:34.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.592056+0000", 2023-10-01T09:35:34.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5681e-05 2023-10-01T09:35:34.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.592082+0000", 2023-10-01T09:35:34.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97191588200000001 2023-10-01T09:35:34.393 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.393 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.393 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.393 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:197758 8.9 8:93321d53:::benchmark_data_smithi138_431617_object96347:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-01T09:35:34.393 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:37.620966+0000", 2023-10-01T09:35:34.394 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.61582223800001, 2023-10-01T09:35:34.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97125771500000002, 2023-10-01T09:35:34.394 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.394 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:34.394 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.394 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.394 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.394 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 197758 2023-10-01T09:35:34.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.394 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.620966+0000", 2023-10-01T09:35:34.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.620966+0000", 2023-10-01T09:35:34.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:35:34.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.620964+0000", 2023-10-01T09:35:34.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7789999999999998e-06 2023-10-01T09:35:34.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.620971+0000", 2023-10-01T09:35:34.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5634e-05 2023-10-01T09:35:34.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.620987+0000", 2023-10-01T09:35:34.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3149999999999997e-06 2023-10-01T09:35:34.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.620993+0000", 2023-10-01T09:35:34.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3131000000000003e-05 2023-10-01T09:35:34.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.621056+0000", 2023-10-01T09:35:34.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2741e-05 2023-10-01T09:35:34.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.621069+0000", 2023-10-01T09:35:34.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.94688227199999997 2023-10-01T09:35:34.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.567951+0000", 2023-10-01T09:35:34.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4444000000000007e-05 2023-10-01T09:35:34.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:34.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.568036+0000", 2023-10-01T09:35:34.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1518000000000001e-05 2023-10-01T09:35:34.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:35:34.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.568117+0000", 2023-10-01T09:35:34.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0094125519999999994 2023-10-01T09:35:34.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:34.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.577530+0000", 2023-10-01T09:35:34.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014447088 2023-10-01T09:35:34.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.400 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.591977+0000", 2023-10-01T09:35:34.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015000199999999999 2023-10-01T09:35:34.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.400 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.592127+0000", 2023-10-01T09:35:34.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6981999999999998e-05 2023-10-01T09:35:34.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:34.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.592184+0000", 2023-10-01T09:35:34.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9406999999999997e-05 2023-10-01T09:35:34.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.592223+0000", 2023-10-01T09:35:34.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.97125771500000002 2023-10-01T09:35:34.401 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.401 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.401 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.402 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:197770 8.9 8:993c3be8:::benchmark_data_smithi138_431617_object96359:head [delete] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-10-01T09:35:34.402 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:37.644350+0000", 2023-10-01T09:35:34.402 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 116.592437351, 2023-10-01T09:35:34.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.95553890600000002, 2023-10-01T09:35:34.402 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:35:34.402 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:34.402 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:35:34.402 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:35:34.402 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:34.403 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 197770 2023-10-01T09:35:34.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.403 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:35:34.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:35:34.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.644350+0000", 2023-10-01T09:35:34.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:35:34.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:35:34.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.644350+0000", 2023-10-01T09:35:34.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:35:34.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:35:34.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.644349+0000", 2023-10-01T09:35:34.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2859999999999999e-06 2023-10-01T09:35:34.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:35:34.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.644357+0000", 2023-10-01T09:35:34.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5002e-05 2023-10-01T09:35:34.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:35:34.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.644372+0000", 2023-10-01T09:35:34.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4869999999999999e-06 2023-10-01T09:35:34.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:35:34.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.644379+0000", 2023-10-01T09:35:34.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8325000000000001e-05 2023-10-01T09:35:34.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.644437+0000", 2023-10-01T09:35:34.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.1630000000000002e-06 2023-10-01T09:35:34.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:35:34.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:37.644446+0000", 2023-10-01T09:35:34.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.92579455799999999 2023-10-01T09:35:34.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:35:34.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.570241+0000", 2023-10-01T09:35:34.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0939000000000006e-05 2023-10-01T09:35:34.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.407 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:35:34.407 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.570322+0000", 2023-10-01T09:35:34.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9382999999999994e-05 2023-10-01T09:35:34.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.407 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:35:34.407 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.570421+0000", 2023-10-01T09:35:34.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0071979879999999998 2023-10-01T09:35:34.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.407 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:35:34.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.577619+0000", 2023-10-01T09:35:34.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019633248999999998 2023-10-01T09:35:34.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.597252+0000", 2023-10-01T09:35:34.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0025324409999999999 2023-10-01T09:35:34.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:34.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.599785+0000", 2023-10-01T09:35:34.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9042000000000002e-05 2023-10-01T09:35:34.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:35:34.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.599844+0000", 2023-10-01T09:35:34.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5729000000000002e-05 2023-10-01T09:35:34.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:35:34.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:35:34.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:35:34.410 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:38.599889+0000", 2023-10-01T09:35:34.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.95553890600000002 2023-10-01T09:35:34.410 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.410 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.410 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.410 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:35:34.410 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:35:34.410 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:35:34.523 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:34.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:34.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:34.627 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:34.690 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:34.737 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-01T09:35:34.771 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 78 16 92919 92903 74.4142 5.4375 0.0124153 0.0134338 2023-10-01T09:35:34.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:34 smithi190 ceph-mon[160435]: failed to encode map e421 with expected crc 2023-10-01T09:35:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:34 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3723665415' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-01T09:35:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:34 smithi190 ceph-mon[160435]: osdmap e421: 8 total, 8 up, 8 in 2023-10-01T09:35:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:34 smithi190 ceph-mon[160435]: failed to encode map e421 with expected crc 2023-10-01T09:35:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:34 smithi190 ceph-mon[160435]: failed to encode map e421 with expected crc 2023-10-01T09:35:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:34 smithi190 ceph-mon[160435]: pgmap v1168: 89 pgs: 2 remapped+peering, 10 peering, 1 active+recovering+undersized+remapped, 76 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 45 MiB/s wr, 713 op/s; 4449/275352 objects misplaced (1.616%); 2.6 MiB/s, 41 objects/s recovering 2023-10-01T09:35:34.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:34 smithi138 ceph-mon[192513]: failed to encode map e421 with expected crc 2023-10-01T09:35:34.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:34 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3723665415' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-01T09:35:34.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:34 smithi138 ceph-mon[192513]: osdmap e421: 8 total, 8 up, 8 in 2023-10-01T09:35:34.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:34 smithi138 ceph-mon[192513]: failed to encode map e421 with expected crc 2023-10-01T09:35:34.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:34 smithi138 ceph-mon[192513]: failed to encode map e421 with expected crc 2023-10-01T09:35:34.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:34 smithi138 ceph-mon[192513]: pgmap v1168: 89 pgs: 2 remapped+peering, 10 peering, 1 active+recovering+undersized+remapped, 76 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 45 MiB/s wr, 713 op/s; 4449/275352 objects misplaced (1.616%); 2.6 MiB/s, 41 objects/s recovering 2023-10-01T09:35:34.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:34 smithi138 ceph-mon[185886]: failed to encode map e421 with expected crc 2023-10-01T09:35:34.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:34 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3723665415' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-01T09:35:34.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:34 smithi138 ceph-mon[185886]: osdmap e421: 8 total, 8 up, 8 in 2023-10-01T09:35:34.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:34 smithi138 ceph-mon[185886]: failed to encode map e421 with expected crc 2023-10-01T09:35:34.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:34 smithi138 ceph-mon[185886]: failed to encode map e421 with expected crc 2023-10-01T09:35:34.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:34 smithi138 ceph-mon[185886]: pgmap v1168: 89 pgs: 2 remapped+peering, 10 peering, 1 active+recovering+undersized+remapped, 76 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 45 MiB/s wr, 713 op/s; 4449/275352 objects misplaced (1.616%); 2.6 MiB/s, 41 objects/s recovering 2023-10-01T09:35:34.882 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:34.887 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:34.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:34.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:34.954 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:35:34.993 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:35.048 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:35.110 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:35.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:35.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:35.215 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:35:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:35:34] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.33.4" 2023-10-01T09:35:35.223 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:35.265 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:35.330 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:35.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:35.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:35.434 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:35.474 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:35.550 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:35.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:35.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:35.587 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:35:35.663 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:35.737 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[185886]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4884/271485 objects degraded (1.799%), 1 pg degraded) 2023-10-01T09:35:35.737 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:35:35.737 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[185886]: failed to encode map e422 with expected crc 2023-10-01T09:35:35.738 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[185886]: failed to encode map e422 with expected crc 2023-10-01T09:35:35.738 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[185886]: failed to encode map e422 with expected crc 2023-10-01T09:35:35.738 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[185886]: osdmap e422: 8 total, 8 up, 8 in 2023-10-01T09:35:35.738 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[185886]: failed to encode map e422 with expected crc 2023-10-01T09:35:35.738 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[185886]: failed to encode map e422 with expected crc 2023-10-01T09:35:35.738 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2766865169' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:35:35.738 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:35:35.739 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[192513]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4884/271485 objects degraded (1.799%), 1 pg degraded) 2023-10-01T09:35:35.739 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:35:35.739 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[192513]: failed to encode map e422 with expected crc 2023-10-01T09:35:35.739 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[192513]: failed to encode map e422 with expected crc 2023-10-01T09:35:35.739 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[192513]: failed to encode map e422 with expected crc 2023-10-01T09:35:35.739 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[192513]: osdmap e422: 8 total, 8 up, 8 in 2023-10-01T09:35:35.740 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[192513]: failed to encode map e422 with expected crc 2023-10-01T09:35:35.740 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[192513]: failed to encode map e422 with expected crc 2023-10-01T09:35:35.740 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2766865169' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:35:35.740 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:35 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:35:35.740 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:35.771 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 79 16 93684 93668 74.0775 47.8125 0.0135306 0.0134946 2023-10-01T09:35:36.023 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:36.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:36.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:36.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:35 smithi190 ceph-mon[160435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4884/271485 objects degraded (1.799%), 1 pg degraded) 2023-10-01T09:35:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:35 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:35:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:35 smithi190 ceph-mon[160435]: failed to encode map e422 with expected crc 2023-10-01T09:35:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:35 smithi190 ceph-mon[160435]: failed to encode map e422 with expected crc 2023-10-01T09:35:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:35 smithi190 ceph-mon[160435]: failed to encode map e422 with expected crc 2023-10-01T09:35:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:35 smithi190 ceph-mon[160435]: osdmap e422: 8 total, 8 up, 8 in 2023-10-01T09:35:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:35 smithi190 ceph-mon[160435]: failed to encode map e422 with expected crc 2023-10-01T09:35:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:35 smithi190 ceph-mon[160435]: failed to encode map e422 with expected crc 2023-10-01T09:35:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2766865169' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:35:36.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:35 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:35:36.128 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:36.154 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:36.197 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:36.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:36.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:36.305 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:36.376 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:36.503 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:36.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:36.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:36.613 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:36.659 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:36.739 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:36.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:36.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:36.777 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:35:36.774920+0000 min lat: 0.00117708 max lat: 2.39797 avg lat: 0.0135505 2023-10-01T09:35:36.777 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:35:36.777 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 80 16 94477 94461 73.7683 49.5625 0.0241383 0.0135505 2023-10-01T09:35:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:36 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:36 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:35:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:36 smithi138 ceph-mon[192513]: osdmap e423: 8 total, 8 up, 8 in 2023-10-01T09:35:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:36 smithi138 ceph-mon[192513]: failed to encode map e423 with expected crc 2023-10-01T09:35:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:36 smithi138 ceph-mon[192513]: failed to encode map e423 with expected crc 2023-10-01T09:35:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:36 smithi138 ceph-mon[192513]: failed to encode map e423 with expected crc 2023-10-01T09:35:36.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:36 smithi138 ceph-mon[192513]: pgmap v1171: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 10 peering, 75 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 31 MiB/s wr, 503 op/s; 914/277302 objects degraded (0.330%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T09:35:36.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:36 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:36.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:36 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:35:36.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:36 smithi138 ceph-mon[185886]: osdmap e423: 8 total, 8 up, 8 in 2023-10-01T09:35:36.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:36 smithi138 ceph-mon[185886]: failed to encode map e423 with expected crc 2023-10-01T09:35:36.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:36 smithi138 ceph-mon[185886]: failed to encode map e423 with expected crc 2023-10-01T09:35:36.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:36 smithi138 ceph-mon[185886]: failed to encode map e423 with expected crc 2023-10-01T09:35:36.825 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:36 smithi138 ceph-mon[185886]: pgmap v1171: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 10 peering, 75 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 31 MiB/s wr, 503 op/s; 914/277302 objects degraded (0.330%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T09:35:36.849 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:36.886 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:36 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:36.886 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:36 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:35:36.886 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:36 smithi190 ceph-mon[160435]: osdmap e423: 8 total, 8 up, 8 in 2023-10-01T09:35:36.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:36 smithi190 ceph-mon[160435]: failed to encode map e423 with expected crc 2023-10-01T09:35:36.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:36 smithi190 ceph-mon[160435]: failed to encode map e423 with expected crc 2023-10-01T09:35:36.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:36 smithi190 ceph-mon[160435]: failed to encode map e423 with expected crc 2023-10-01T09:35:36.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:36 smithi190 ceph-mon[160435]: pgmap v1171: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 10 peering, 75 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 31 MiB/s wr, 503 op/s; 914/277302 objects degraded (0.330%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T09:35:36.889 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:36.963 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:36.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:36.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:37.070 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:37.103 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:37.181 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:37.444 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:37.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:37.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:37.559 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:37.602 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:37.671 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:37.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:37.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:37.721 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:37 smithi138 ceph-mon[185886]: Health check failed: Degraded data redundancy: 914/277302 objects degraded (0.330%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:37.726 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:37 smithi138 ceph-mon[192513]: Health check failed: Degraded data redundancy: 914/277302 objects degraded (0.330%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:37.769 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:35:37.774 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 81 16 95323 95307 73.5104 52.875 0.0156355 0.0135974 2023-10-01T09:35:37.777 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:37.815 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:35:37.846 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:35:37.883 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:35:37.887 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:37.933 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:35:37.964 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:35:38.007 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:35:38.008 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:38.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:38.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:38.039 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:35:38.040 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:35:38.041 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:35:38.041 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:35:38.041 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:35:38.041 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:35:38.041 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:35:38.041 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:35:38.041 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:35:38.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:37 smithi190 ceph-mon[160435]: Health check failed: Degraded data redundancy: 914/277302 objects degraded (0.330%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:35:38.116 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:38.156 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:38.266 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:38.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:38.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:38.375 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:38.417 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:38.504 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:38.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:38.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:38.614 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:38.695 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:38.774 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 82 16 96137 96121 73.2344 50.875 0.0190499 0.0136503 2023-10-01T09:35:38.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2286264225' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:38 smithi190 ceph-mon[160435]: pgmap v1172: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 10 peering, 74 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 30 MiB/s wr, 483 op/s; 917/280197 objects degraded (0.327%); 1.9 MiB/s, 1 keys/s, 30 objects/s recovering 2023-10-01T09:35:38.938 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2286264225' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:38.939 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:38 smithi138 ceph-mon[192513]: pgmap v1172: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 10 peering, 74 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 30 MiB/s wr, 483 op/s; 917/280197 objects degraded (0.327%); 1.9 MiB/s, 1 keys/s, 30 objects/s recovering 2023-10-01T09:35:38.939 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2286264225' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:38.940 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:38 smithi138 ceph-mon[185886]: pgmap v1172: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 10 peering, 74 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 30 MiB/s wr, 483 op/s; 917/280197 objects degraded (0.327%); 1.9 MiB/s, 1 keys/s, 30 objects/s recovering 2023-10-01T09:35:39.012 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:35:39.512 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:39.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:39.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:39.620 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:39.686 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:39.738 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 5, 6, 4, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:35:39.738 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:35:39.738 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-10-01T09:35:39.738 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-10-01T09:35:39.782 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 83 16 96901 96885 72.9208 47.75 0.021981 0.0137076 2023-10-01T09:35:39.977 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:39.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:39.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:40.084 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:40.177 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:40.219 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:40.318 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:40.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:40.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:40.443 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:40.493 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:40.549 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:40 smithi138 ceph-mon[192513]: pgmap v1173: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 1858/287655 objects degraded (0.646%); 636/287655 objects misplaced (0.221%); 4.3 MiB/s, 1 keys/s, 69 objects/s recovering 2023-10-01T09:35:40.552 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:40 smithi138 ceph-mon[185886]: pgmap v1173: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 1858/287655 objects degraded (0.646%); 636/287655 objects misplaced (0.221%); 4.3 MiB/s, 1 keys/s, 69 objects/s recovering 2023-10-01T09:35:40.553 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:40.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:40 smithi190 ceph-mon[160435]: pgmap v1173: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 1858/287655 objects degraded (0.646%); 636/287655 objects misplaced (0.221%); 4.3 MiB/s, 1 keys/s, 69 objects/s recovering 2023-10-01T09:35:40.669 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:35:40.669 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:35:40.687 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:40.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:40.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:40.782 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 84 16 97658 97642 72.616 47.3125 0.0198885 0.0137656 2023-10-01T09:35:40.793 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:40.839 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:40.932 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:40.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:40.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:41.039 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:41.097 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:41.196 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:41.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:41.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:41.309 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:41.382 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:41.488 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:35:41.490 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:35:41.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:35:41.622 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:41.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:41.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:41.666 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:35:41.726 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:41.782 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 85 16 98540 98524 72.4101 55.125 0.0276723 0.0138047 2023-10-01T09:35:41.859 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:42.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:42 smithi138 ceph-mon[185886]: pgmap v1174: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s; 1858/291558 objects degraded (0.637%); 636/291558 objects misplaced (0.218%); 2.8 MiB/s, 1 keys/s, 45 objects/s recovering 2023-10-01T09:35:42.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:42 smithi138 ceph-mon[192513]: pgmap v1174: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s; 1858/291558 objects degraded (0.637%); 636/291558 objects misplaced (0.218%); 2.8 MiB/s, 1 keys/s, 45 objects/s recovering 2023-10-01T09:35:42.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:42 smithi190 ceph-mon[160435]: pgmap v1174: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s; 1858/291558 objects degraded (0.637%); 636/291558 objects misplaced (0.218%); 2.8 MiB/s, 1 keys/s, 45 objects/s recovering 2023-10-01T09:35:42.782 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 86 16 99272 99256 72.1002 45.75 0.0262756 0.0138633 2023-10-01T09:35:43.012 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:43.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:43.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:43.122 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:43.175 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:43.251 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:43.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:43.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:43.295 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:35:43.361 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:43.481 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:43 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 1858/291558 objects degraded (0.637%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:35:43.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:43 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 1858/291558 objects degraded (0.637%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:35:43.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:43 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 1858/291558 objects degraded (0.637%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:35:43.670 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:43.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:43.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:43.780 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:43.784 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 87 16 100075 100059 71.8476 50.1875 0.0269678 0.0139122 2023-10-01T09:35:43.879 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:44.008 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:44.101 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:44.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:44.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:44.220 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:44.263 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:44.354 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:44.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:44.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:44.388 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:44 smithi138 ceph-mon[185886]: pgmap v1175: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 1858/296718 objects degraded (0.626%); 301/296718 objects misplaced (0.101%); 4.5 MiB/s, 1 keys/s, 72 objects/s recovering 2023-10-01T09:35:44.390 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:44 smithi138 ceph-mon[192513]: pgmap v1175: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 1858/296718 objects degraded (0.626%); 301/296718 objects misplaced (0.101%); 4.5 MiB/s, 1 keys/s, 72 objects/s recovering 2023-10-01T09:35:44.459 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:44.501 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:44.553 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:44.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:44 smithi190 ceph-mon[160435]: pgmap v1175: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 1858/296718 objects degraded (0.626%); 301/296718 objects misplaced (0.101%); 4.5 MiB/s, 1 keys/s, 72 objects/s recovering 2023-10-01T09:35:44.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:44.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:44.609 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:44.661 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:35:44.696 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:44.757 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:35:44.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:44.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:35:44.783 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 88 16 100821 100805 71.5611 46.625 0.043856 0.0139681 2023-10-01T09:35:44.863 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:44.950 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:45.002 INFO:teuthology.orchestra.run.smithi138.stdout:moved 0 / 267 (0%) 2023-10-01T09:35:45.002 INFO:teuthology.orchestra.run.smithi138.stdout:avg 33.375 2023-10-01T09:35:45.002 INFO:teuthology.orchestra.run.smithi138.stdout:stddev 6.70704 -> 6.70704 (expected baseline 5.40399) 2023-10-01T09:35:45.002 INFO:teuthology.orchestra.run.smithi138.stdout:min osd.2 with 20 -> 20 pgs (0.599251 -> 0.599251 * mean) 2023-10-01T09:35:45.002 INFO:teuthology.orchestra.run.smithi138.stdout:max osd.1 with 43 -> 43 pgs (1.28839 -> 1.28839 * mean) 2023-10-01T09:35:45.002 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:35:45.002 INFO:teuthology.orchestra.run.smithi138.stdout:oload 1000 2023-10-01T09:35:45.003 INFO:teuthology.orchestra.run.smithi138.stdout:max_change 0.05 2023-10-01T09:35:45.003 INFO:teuthology.orchestra.run.smithi138.stdout:max_change_osds 4 2023-10-01T09:35:45.003 INFO:teuthology.orchestra.run.smithi138.stdout:average_utilization 0.0319 2023-10-01T09:35:45.003 INFO:teuthology.orchestra.run.smithi138.stdout:overload_utilization 0.3190 2023-10-01T09:35:45.018 INFO:teuthology.orchestra.run.smithi138.stderr:no change 2023-10-01T09:35:45.180 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:35:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:35:44] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.33.4" 2023-10-01T09:35:45.288 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:35:45.511 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:45 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/518330739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:35:45.511 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:45 smithi138 ceph-mon[192513]: from='client.36156 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:35:45.512 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:45 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/518330739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:35:45.513 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:45 smithi138 ceph-mon[185886]: from='client.36156 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:35:45.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:45 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/518330739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:35:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:45 smithi190 ceph-mon[160435]: from='client.36156 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:35:45.784 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 89 16 101610 101594 71.3111 49.3125 0.0255774 0.0140177 2023-10-01T09:35:46.090 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:46.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:46.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:46.205 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:46.269 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:46.488 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:46.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[192513]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/518330739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:35:46.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[192513]: osdmap e424: 8 total, 8 up, 8 in 2023-10-01T09:35:46.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[192513]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[192513]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[192513]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[192513]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[192513]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[192513]: pgmap v1177: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 1858/299529 objects degraded (0.620%); 301/299529 objects misplaced (0.100%); 3.8 MiB/s, 0 keys/s, 60 objects/s recovering 2023-10-01T09:35:46.490 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:46.490 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[185886]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.490 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/518330739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:35:46.490 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[185886]: osdmap e424: 8 total, 8 up, 8 in 2023-10-01T09:35:46.490 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[185886]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.490 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[185886]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.490 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[185886]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.490 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[185886]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.491 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[185886]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.491 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:46 smithi138 ceph-mon[185886]: pgmap v1177: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 1858/299529 objects degraded (0.620%); 301/299529 objects misplaced (0.100%); 3.8 MiB/s, 0 keys/s, 60 objects/s recovering 2023-10-01T09:35:46.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:46 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:46.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:46 smithi190 ceph-mon[160435]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:46 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/518330739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:35:46.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:46 smithi190 ceph-mon[160435]: osdmap e424: 8 total, 8 up, 8 in 2023-10-01T09:35:46.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:46 smithi190 ceph-mon[160435]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:46 smithi190 ceph-mon[160435]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:46 smithi190 ceph-mon[160435]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:46 smithi190 ceph-mon[160435]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:46 smithi190 ceph-mon[160435]: failed to encode map e424 with expected crc 2023-10-01T09:35:46.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:46 smithi190 ceph-mon[160435]: pgmap v1177: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 1858/299529 objects degraded (0.620%); 301/299529 objects misplaced (0.100%); 3.8 MiB/s, 0 keys/s, 60 objects/s recovering 2023-10-01T09:35:46.784 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Total time run: 90.0053 2023-10-01T09:35:46.784 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Total writes made: 102382 2023-10-01T09:35:46.784 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Write size: 65536 2023-10-01T09:35:46.784 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Object size: 65536 2023-10-01T09:35:46.785 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Bandwidth (MB/sec): 71.0944 2023-10-01T09:35:46.785 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev Bandwidth: 38.8117 2023-10-01T09:35:46.785 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max bandwidth (MB/sec): 223.562 2023-10-01T09:35:46.785 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min bandwidth (MB/sec): 0 2023-10-01T09:35:46.785 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Average IOPS: 1137 2023-10-01T09:35:46.785 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev IOPS: 620.997 2023-10-01T09:35:46.785 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max IOPS: 3577 2023-10-01T09:35:46.785 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min IOPS: 0 2023-10-01T09:35:46.786 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Average Latency(s): 0.0140618 2023-10-01T09:35:46.786 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev Latency(s): 0.0353435 2023-10-01T09:35:46.786 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max latency(s): 2.39797 2023-10-01T09:35:46.786 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min latency(s): 0.00117708 2023-10-01T09:35:46.790 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T09:35:47.094 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-01T09:35:47.094 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-10-01T09:35:47.162 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:47.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:47.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:47.264 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:47.290 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:47.354 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:47.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:47.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:47.468 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:47.533 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:47.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:47 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 1858/299529 objects degraded (0.620%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:35:47.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:47 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 1858/299529 objects degraded (0.620%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:35:48.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:47 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 1858/299529 objects degraded (0.620%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T09:35:48.137 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:48.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:48.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:48.246 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:48.279 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:48.350 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:48.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:48.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:48.455 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:48.546 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:48.685 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:48.783 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:48.786 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:48 smithi138 ceph-mon[192513]: pgmap v1178: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 763 op/s; 1858/303099 objects degraded (0.613%); 301/303099 objects misplaced (0.099%); 3.8 MiB/s, 60 objects/s recovering 2023-10-01T09:35:48.786 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:48 smithi138 ceph-mon[185886]: pgmap v1178: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 763 op/s; 1858/303099 objects degraded (0.613%); 301/303099 objects misplaced (0.099%); 3.8 MiB/s, 60 objects/s recovering 2023-10-01T09:35:48.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:48.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:48.888 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:48.920 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:48.982 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:35:49.002 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:35:49.021 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:35:49.041 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:35:49.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:48 smithi190 ceph-mon[160435]: pgmap v1178: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 48 MiB/s wr, 763 op/s; 1858/303099 objects degraded (0.613%); 301/303099 objects misplaced (0.099%); 3.8 MiB/s, 60 objects/s recovering 2023-10-01T09:35:49.059 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:35:49.068 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:49.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:49.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:49.079 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:35:49.097 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:35:49.115 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:35:49.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:35:49.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:35:49.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:35:49.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:35:49.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:35:49.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:35:49.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:35:49.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:35:49.119 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:35:49.140 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:35:49.140 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:35:49.140 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:35:49.141 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:35:49.141 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:35:49.172 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:49.207 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:49.764 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:49.767 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:49 smithi138 ceph-mon[185886]: failed to encode map e425 with expected crc 2023-10-01T09:35:49.767 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:49 smithi138 ceph-mon[185886]: osdmap e425: 8 total, 8 up, 8 in 2023-10-01T09:35:49.768 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:49 smithi138 ceph-mon[185886]: failed to encode map e425 with expected crc 2023-10-01T09:35:49.768 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/579153748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:49.768 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:49 smithi138 ceph-mon[192513]: failed to encode map e425 with expected crc 2023-10-01T09:35:49.768 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:49 smithi138 ceph-mon[192513]: osdmap e425: 8 total, 8 up, 8 in 2023-10-01T09:35:49.768 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:49 smithi138 ceph-mon[192513]: failed to encode map e425 with expected crc 2023-10-01T09:35:49.768 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/579153748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:49.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:49.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:49.870 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:49.925 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:49 smithi190 ceph-mon[160435]: failed to encode map e425 with expected crc 2023-10-01T09:35:49.925 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:49 smithi190 ceph-mon[160435]: osdmap e425: 8 total, 8 up, 8 in 2023-10-01T09:35:49.925 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:49 smithi190 ceph-mon[160435]: failed to encode map e425 with expected crc 2023-10-01T09:35:49.925 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/579153748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:49.929 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:50.015 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:50.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:50.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:50.128 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:50.199 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:51.001 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:50 smithi138 ceph-mon[192513]: failed to encode map e426 with expected crc 2023-10-01T09:35:51.001 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:50 smithi138 ceph-mon[192513]: osdmap e426: 8 total, 8 up, 8 in 2023-10-01T09:35:51.001 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:50 smithi138 ceph-mon[192513]: failed to encode map e426 with expected crc 2023-10-01T09:35:51.001 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:50 smithi138 ceph-mon[192513]: pgmap v1181: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 36 MiB/s wr, 758 op/s; 934/304095 objects degraded (0.307%); 875/304095 objects misplaced (0.288%); 3.5 MiB/s, 58 objects/s recovering 2023-10-01T09:35:51.007 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:50 smithi138 ceph-mon[185886]: failed to encode map e426 with expected crc 2023-10-01T09:35:51.007 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:50 smithi138 ceph-mon[185886]: osdmap e426: 8 total, 8 up, 8 in 2023-10-01T09:35:51.007 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:50 smithi138 ceph-mon[185886]: failed to encode map e426 with expected crc 2023-10-01T09:35:51.007 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:50 smithi138 ceph-mon[185886]: pgmap v1181: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 36 MiB/s wr, 758 op/s; 934/304095 objects degraded (0.307%); 875/304095 objects misplaced (0.288%); 3.5 MiB/s, 58 objects/s recovering 2023-10-01T09:35:51.039 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:35:51.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:50 smithi190 ceph-mon[160435]: failed to encode map e426 with expected crc 2023-10-01T09:35:51.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:50 smithi190 ceph-mon[160435]: osdmap e426: 8 total, 8 up, 8 in 2023-10-01T09:35:51.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:50 smithi190 ceph-mon[160435]: failed to encode map e426 with expected crc 2023-10-01T09:35:51.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:50 smithi190 ceph-mon[160435]: pgmap v1181: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 170 B/s rd, 36 MiB/s wr, 758 op/s; 934/304095 objects degraded (0.307%); 875/304095 objects misplaced (0.288%); 3.5 MiB/s, 58 objects/s recovering 2023-10-01T09:35:51.153 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:51.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:51.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:51.259 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:51.305 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:51.367 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:51.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:51.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:51.476 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:51.527 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:51.566 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:51.708 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:51.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:51.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:51.820 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:51.908 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:52.050 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:52.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:52.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:52.162 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:52.226 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:52.277 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:52.397 INFO:teuthology.orchestra.run.smithi138.stdout:moved 9 / 267 (3.37079%) 2023-10-01T09:35:52.397 INFO:teuthology.orchestra.run.smithi138.stdout:avg 33.375 2023-10-01T09:35:52.397 INFO:teuthology.orchestra.run.smithi138.stdout:stddev 6.70704 -> 6.72565 (expected baseline 5.40399) 2023-10-01T09:35:52.397 INFO:teuthology.orchestra.run.smithi138.stdout:min osd.2 with 20 -> 20 pgs (0.599251 -> 0.599251 * mean) 2023-10-01T09:35:52.398 INFO:teuthology.orchestra.run.smithi138.stdout:max osd.1 with 43 -> 41 pgs (1.28839 -> 1.22846 * mean) 2023-10-01T09:35:52.398 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:35:52.398 INFO:teuthology.orchestra.run.smithi138.stdout:oload 110 2023-10-01T09:35:52.398 INFO:teuthology.orchestra.run.smithi138.stdout:max_change 0.05 2023-10-01T09:35:52.398 INFO:teuthology.orchestra.run.smithi138.stdout:max_change_osds 4 2023-10-01T09:35:52.398 INFO:teuthology.orchestra.run.smithi138.stdout:average_utilization 0.0330 2023-10-01T09:35:52.398 INFO:teuthology.orchestra.run.smithi138.stdout:overload_utilization 0.0363 2023-10-01T09:35:52.398 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7 weight 1.0000 -> 0.9500 2023-10-01T09:35:52.398 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1 weight 1.0000 -> 0.9500 2023-10-01T09:35:52.398 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5 weight 1.0000 -> 0.9500 2023-10-01T09:35:52.398 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3 weight 1.0000 -> 0.9500 2023-10-01T09:35:52.406 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:52.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:52.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:52.423 INFO:teuthology.orchestra.run.smithi138.stderr:no change 2023-10-01T09:35:52.519 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:52.569 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:35:52 smithi138 conmon[178630]: 2023-10-01T09:35:52.395+0000 7fa004e48700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-10-01T09:35:52.575 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:52.669 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:52.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:52.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:52.778 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:52.804 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:53.036 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:35:53.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:53 smithi190 ceph-mon[160435]: pgmap v1182: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 149 B/s rd, 23 MiB/s wr, 679 op/s; 931/301011 objects degraded (0.309%); 875/301011 objects misplaced (0.291%); 3.1 MiB/s, 51 objects/s recovering 2023-10-01T09:35:53.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:53 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 931/301011 objects degraded (0.309%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:35:53.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:53 smithi138 ceph-mon[192513]: pgmap v1182: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 149 B/s rd, 23 MiB/s wr, 679 op/s; 931/301011 objects degraded (0.309%); 875/301011 objects misplaced (0.291%); 3.1 MiB/s, 51 objects/s recovering 2023-10-01T09:35:53.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:53 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 931/301011 objects degraded (0.309%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:35:53.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:53 smithi138 ceph-mon[185886]: pgmap v1182: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 149 B/s rd, 23 MiB/s wr, 679 op/s; 931/301011 objects degraded (0.309%); 875/301011 objects misplaced (0.291%); 3.1 MiB/s, 51 objects/s recovering 2023-10-01T09:35:53.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:53 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 931/301011 objects degraded (0.309%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:35:53.662 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:53.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:53.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:53.766 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:53.838 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:53.900 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-10-01T09:35:53.900 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-10-01T09:35:54.001 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:54.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:54.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:54.110 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:54.164 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:54.263 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:54.288 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:54 smithi138 ceph-mon[185886]: from='client.36168 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:35:54.288 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:54 smithi138 ceph-mon[192513]: from='client.36168 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:35:54.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:54.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:54.390 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:54.431 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:54 smithi190 ceph-mon[160435]: from='client.36168 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:35:54.432 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:54.518 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:54.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:54.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:54.628 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:54.681 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:54.787 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:54.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:54.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:54.897 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:35:54.940 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:35:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:35:54] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.33.4" 2023-10-01T09:35:54.989 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:55.201 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:55 smithi138 ceph-mon[185886]: pgmap v1183: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 20 MiB/s wr, 764 op/s; 931/296580 objects degraded (0.314%); 875/296580 objects misplaced (0.295%); 2.7 MiB/s, 44 objects/s recovering 2023-10-01T09:35:55.202 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:55 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1915906204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:35:55.202 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:55 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:35:55.202 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:55 smithi138 ceph-mon[192513]: pgmap v1183: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 20 MiB/s wr, 764 op/s; 931/296580 objects degraded (0.314%); 875/296580 objects misplaced (0.295%); 2.7 MiB/s, 44 objects/s recovering 2023-10-01T09:35:55.202 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:55 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1915906204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:35:55.202 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:55 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:35:55.243 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:35:55.553 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:35:55.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:55 smithi190 ceph-mon[160435]: pgmap v1183: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 20 MiB/s wr, 764 op/s; 931/296580 objects degraded (0.314%); 875/296580 objects misplaced (0.295%); 2.7 MiB/s, 44 objects/s recovering 2023-10-01T09:35:55.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:55 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1915906204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:35:55.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:55 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:35:55.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:55.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:35:55.662 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:35:55.689 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:56.023 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:35:56.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:56.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:35:56.123 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:56.129 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:56.210 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:56.529 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:56.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:56.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:56.537 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:56 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:56.537 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:56 smithi138 ceph-mon[185886]: failed to encode map e427 with expected crc 2023-10-01T09:35:56.538 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:56 smithi138 ceph-mon[185886]: failed to encode map e427 with expected crc 2023-10-01T09:35:56.538 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:56 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:35:56.538 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:56 smithi138 ceph-mon[185886]: osdmap e427: 8 total, 8 up, 8 in 2023-10-01T09:35:56.538 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:35:56.538 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:56 smithi138 ceph-mon[185886]: pgmap v1185: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 11 MiB/s wr, 777 op/s; 931/293817 objects degraded (0.317%); 759/293817 objects misplaced (0.258%); 3.8 MiB/s, 62 objects/s recovering 2023-10-01T09:35:56.538 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:56 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:56.538 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:56 smithi138 ceph-mon[192513]: failed to encode map e427 with expected crc 2023-10-01T09:35:56.538 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:56 smithi138 ceph-mon[192513]: failed to encode map e427 with expected crc 2023-10-01T09:35:56.538 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:56 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:35:56.538 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:56 smithi138 ceph-mon[192513]: osdmap e427: 8 total, 8 up, 8 in 2023-10-01T09:35:56.539 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:35:56.539 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:56 smithi138 ceph-mon[192513]: pgmap v1185: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 11 MiB/s wr, 777 op/s; 931/293817 objects degraded (0.317%); 759/293817 objects misplaced (0.258%); 3.8 MiB/s, 62 objects/s recovering 2023-10-01T09:35:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:56 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:35:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:56 smithi190 ceph-mon[160435]: failed to encode map e427 with expected crc 2023-10-01T09:35:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:56 smithi190 ceph-mon[160435]: failed to encode map e427 with expected crc 2023-10-01T09:35:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:56 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:35:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:56 smithi190 ceph-mon[160435]: osdmap e427: 8 total, 8 up, 8 in 2023-10-01T09:35:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:35:56.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:56 smithi190 ceph-mon[160435]: pgmap v1185: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 11 MiB/s wr, 777 op/s; 931/293817 objects degraded (0.317%); 759/293817 objects misplaced (0.258%); 3.8 MiB/s, 62 objects/s recovering 2023-10-01T09:35:56.597 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:35:56.598 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:35:56.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:35:56.598 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:35:56.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.598 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:159797 8.2 8:454d3afa:::benchmark_data_smithi138_431617_object58386:head [delete] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-10-01T09:35:56.598 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:27.321419+0000", 2023-10-01T09:35:56.598 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 209.27195198199999, 2023-10-01T09:35:56.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.378759509, 2023-10-01T09:35:56.598 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.599 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:35:56.599 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.599 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.599 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.599 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 159797 2023-10-01T09:35:56.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.599 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.321419+0000", 2023-10-01T09:35:56.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.321419+0000", 2023-10-01T09:35:56.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.601 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.601 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.601 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.321418+0000", 2023-10-01T09:35:56.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:35:56.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.321421+0000", 2023-10-01T09:35:56.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6079999999999998e-06 2023-10-01T09:35:56.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.321427+0000", 2023-10-01T09:35:56.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4989999999999999e-06 2023-10-01T09:35:56.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.321432+0000", 2023-10-01T09:35:56.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9699999999999998e-06 2023-10-01T09:35:56.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.321475+0000", 2023-10-01T09:35:56.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2957999999999998e-05 2023-10-01T09:35:56.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.321590+0000", 2023-10-01T09:35:56.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000114936 2023-10-01T09:35:56.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-10-01T09:35:56.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.321676+0000", 2023-10-01T09:35:56.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5671e-05 2023-10-01T09:35:56.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.331021+0000", 2023-10-01T09:35:56.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0093451669999999997 2023-10-01T09:35:56.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.331053+0000", 2023-10-01T09:35:56.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.205e-05 2023-10-01T09:35:56.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.350108+0000", 2023-10-01T09:35:56.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019054628000000001 2023-10-01T09:35:56.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.350119+0000", 2023-10-01T09:35:56.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1525000000000001e-05 2023-10-01T09:35:56.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.433540+0000", 2023-10-01T09:35:56.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.083421062000000004 2023-10-01T09:35:56.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.433560+0000", 2023-10-01T09:35:56.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9324999999999999e-05 2023-10-01T09:35:56.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.666039+0000", 2023-10-01T09:35:56.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.232479772 2023-10-01T09:35:56.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:35:56.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.666145+0000", 2023-10-01T09:35:56.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010569900000000001 2023-10-01T09:35:56.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.700129+0000", 2023-10-01T09:35:56.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.033983815000000001 2023-10-01T09:35:56.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.700178+0000", 2023-10-01T09:35:56.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9212999999999998e-05 2023-10-01T09:35:56.608 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.608 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.608 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.609 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:159801 8.2 8:49d7537e:::benchmark_data_smithi138_431617_object58390:head [delete] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-10-01T09:35:56.609 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:27.326118+0000", 2023-10-01T09:35:56.609 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 209.26725282199999, 2023-10-01T09:35:56.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3739902370000001, 2023-10-01T09:35:56.609 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.609 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:35:56.609 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.609 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.609 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.609 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 159801 2023-10-01T09:35:56.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.610 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.326118+0000", 2023-10-01T09:35:56.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.326118+0000", 2023-10-01T09:35:56.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.326116+0000", 2023-10-01T09:35:56.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:56.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.326124+0000", 2023-10-01T09:35:56.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0010000000000001e-06 2023-10-01T09:35:56.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.326141+0000", 2023-10-01T09:35:56.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6484e-05 2023-10-01T09:35:56.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.326149+0000", 2023-10-01T09:35:56.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0360000000000006e-06 2023-10-01T09:35:56.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.326216+0000", 2023-10-01T09:35:56.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6804000000000006e-05 2023-10-01T09:35:56.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.326356+0000", 2023-10-01T09:35:56.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014037300000000001 2023-10-01T09:35:56.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,5", 2023-10-01T09:35:56.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.326481+0000", 2023-10-01T09:35:56.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000125265 2023-10-01T09:35:56.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.331222+0000", 2023-10-01T09:35:56.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047411040000000003 2023-10-01T09:35:56.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.331892+0000", 2023-10-01T09:35:56.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00066913599999999997 2023-10-01T09:35:56.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.352000+0000", 2023-10-01T09:35:56.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020108629999999999 2023-10-01T09:35:56.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.352012+0000", 2023-10-01T09:35:56.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1528999999999999e-05 2023-10-01T09:35:56.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.434717+0000", 2023-10-01T09:35:56.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.082705230000000005 2023-10-01T09:35:56.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.434724+0000", 2023-10-01T09:35:56.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3130000000000001e-06 2023-10-01T09:35:56.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.666393+0000", 2023-10-01T09:35:56.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2316683909999999 2023-10-01T09:35:56.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:35:56.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.666483+0000", 2023-10-01T09:35:56.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9870999999999994e-05 2023-10-01T09:35:56.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.700043+0000", 2023-10-01T09:35:56.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.033560589000000002 2023-10-01T09:35:56.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.700108+0000", 2023-10-01T09:35:56.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4948000000000002e-05 2023-10-01T09:35:56.618 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.618 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.618 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.618 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:159803 8.e 8:7dca3e12:::benchmark_data_smithi138_431617_object58392:head [delete] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-10-01T09:35:56.618 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:27.332697+0000", 2023-10-01T09:35:56.618 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 209.26067326800001, 2023-10-01T09:35:56.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.295072515, 2023-10-01T09:35:56.619 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.619 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:35:56.619 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.619 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.619 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.619 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 159803 2023-10-01T09:35:56.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.619 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.332697+0000", 2023-10-01T09:35:56.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.332697+0000", 2023-10-01T09:35:56.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.332696+0000", 2023-10-01T09:35:56.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:56.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.332702+0000", 2023-10-01T09:35:56.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7570000000000001e-06 2023-10-01T09:35:56.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.332716+0000", 2023-10-01T09:35:56.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4012999999999999e-05 2023-10-01T09:35:56.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.332725+0000", 2023-10-01T09:35:56.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7930000000000003e-06 2023-10-01T09:35:56.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.333563+0000", 2023-10-01T09:35:56.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00083844699999999998 2023-10-01T09:35:56.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.333676+0000", 2023-10-01T09:35:56.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011338200000000001 2023-10-01T09:35:56.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-10-01T09:35:56.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.333779+0000", 2023-10-01T09:35:56.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000102772 2023-10-01T09:35:56.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.340919+0000", 2023-10-01T09:35:56.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0071400209999999999 2023-10-01T09:35:56.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.340980+0000", 2023-10-01T09:35:56.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0797000000000001e-05 2023-10-01T09:35:56.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.353128+0000", 2023-10-01T09:35:56.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012148071999999999 2023-10-01T09:35:56.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.353139+0000", 2023-10-01T09:35:56.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0584000000000001e-05 2023-10-01T09:35:56.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.434957+0000", 2023-10-01T09:35:56.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.081818368000000002 2023-10-01T09:35:56.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.434967+0000", 2023-10-01T09:35:56.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0484e-05 2023-10-01T09:35:56.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.627706+0000", 2023-10-01T09:35:56.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1927386659999999 2023-10-01T09:35:56.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.627770+0000", 2023-10-01T09:35:56.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3779e-05 2023-10-01T09:35:56.626 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.627 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.627 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.627 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:159823 8.2 8:4060dd7e:::benchmark_data_smithi138_431617_object58412:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-10-01T09:35:56.627 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:27.373574+0000", 2023-10-01T09:35:56.627 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 209.21979690399999, 2023-10-01T09:35:56.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3255837109999999, 2023-10-01T09:35:56.627 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.627 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.627 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.628 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.628 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.628 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 159823 2023-10-01T09:35:56.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.628 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.373574+0000", 2023-10-01T09:35:56.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.373574+0000", 2023-10-01T09:35:56.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.373573+0000", 2023-10-01T09:35:56.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:35:56.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.373579+0000", 2023-10-01T09:35:56.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3999999999999997e-06 2023-10-01T09:35:56.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.373594+0000", 2023-10-01T09:35:56.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4636e-05 2023-10-01T09:35:56.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.373601+0000", 2023-10-01T09:35:56.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4189999999999996e-06 2023-10-01T09:35:56.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.373803+0000", 2023-10-01T09:35:56.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020155300000000001 2023-10-01T09:35:56.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.373813+0000", 2023-10-01T09:35:56.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0047000000000001e-05 2023-10-01T09:35:56.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.434787+0000", 2023-10-01T09:35:56.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.060974367000000002 2023-10-01T09:35:56.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.434792+0000", 2023-10-01T09:35:56.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6589999999999998e-06 2023-10-01T09:35:56.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.666507+0000", 2023-10-01T09:35:56.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.231715296 2023-10-01T09:35:56.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.666620+0000", 2023-10-01T09:35:56.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112833 2023-10-01T09:35:56.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4,5", 2023-10-01T09:35:56.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.666735+0000", 2023-10-01T09:35:56.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000115523 2023-10-01T09:35:56.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.675729+0000", 2023-10-01T09:35:56.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0089939310000000001 2023-10-01T09:35:56.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.680483+0000", 2023-10-01T09:35:56.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047542249999999999 2023-10-01T09:35:56.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.681611+0000", 2023-10-01T09:35:56.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011275160000000001 2023-10-01T09:35:56.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.698991+0000", 2023-10-01T09:35:56.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017379786000000001 2023-10-01T09:35:56.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.699046+0000", 2023-10-01T09:35:56.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4839000000000001e-05 2023-10-01T09:35:56.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.699157+0000", 2023-10-01T09:35:56.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111858 2023-10-01T09:35:56.636 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.636 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.636 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.637 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:159829 8.2 8:474c1185:::benchmark_data_smithi138_431617_object58418:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-10-01T09:35:56.637 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:27.381278+0000", 2023-10-01T09:35:56.637 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 209.212092227, 2023-10-01T09:35:56.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.318052134, 2023-10-01T09:35:56.637 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.637 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.637 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.637 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.637 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.637 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 159829 2023-10-01T09:35:56.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.637 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.381278+0000", 2023-10-01T09:35:56.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.381278+0000", 2023-10-01T09:35:56.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.381277+0000", 2023-10-01T09:35:56.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:35:56.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.381284+0000", 2023-10-01T09:35:56.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4860000000000002e-06 2023-10-01T09:35:56.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.381293+0000", 2023-10-01T09:35:56.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5380000000000008e-06 2023-10-01T09:35:56.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.381299+0000", 2023-10-01T09:35:56.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.857e-06 2023-10-01T09:35:56.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.381347+0000", 2023-10-01T09:35:56.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7494000000000003e-05 2023-10-01T09:35:56.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.381356+0000", 2023-10-01T09:35:56.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9279999999999999e-06 2023-10-01T09:35:56.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.434872+0000", 2023-10-01T09:35:56.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.053516708000000003 2023-10-01T09:35:56.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.434877+0000", 2023-10-01T09:35:56.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8659999999999997e-06 2023-10-01T09:35:56.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.671539+0000", 2023-10-01T09:35:56.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2366622039999999 2023-10-01T09:35:56.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.671680+0000", 2023-10-01T09:35:56.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000140526 2023-10-01T09:35:56.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4,5", 2023-10-01T09:35:56.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.671817+0000", 2023-10-01T09:35:56.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013730599999999999 2023-10-01T09:35:56.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.679979+0000", 2023-10-01T09:35:56.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0081617140000000005 2023-10-01T09:35:56.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.681619+0000", 2023-10-01T09:35:56.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0016400169999999999 2023-10-01T09:35:56.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.682351+0000", 2023-10-01T09:35:56.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00073242399999999997 2023-10-01T09:35:56.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.699192+0000", 2023-10-01T09:35:56.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016841009000000001 2023-10-01T09:35:56.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.699237+0000", 2023-10-01T09:35:56.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4462999999999998e-05 2023-10-01T09:35:56.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.699331+0000", 2023-10-01T09:35:56.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3703999999999996e-05 2023-10-01T09:35:56.646 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.646 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.646 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.646 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169544 8.2 8:47fdaf48:::benchmark_data_smithi138_431617_object68133:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-01T09:35:56.646 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.284090+0000", 2023-10-01T09:35:56.646 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 191.309280301, 2023-10-01T09:35:56.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.208749836, 2023-10-01T09:35:56.647 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.647 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:35:56.647 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.647 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.647 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.647 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169544 2023-10-01T09:35:56.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.647 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.647 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.647 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.284090+0000", 2023-10-01T09:35:56.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.284090+0000", 2023-10-01T09:35:56.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.284089+0000", 2023-10-01T09:35:56.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:56.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.284096+0000", 2023-10-01T09:35:56.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9609999999999999e-06 2023-10-01T09:35:56.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.284107+0000", 2023-10-01T09:35:56.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1294999999999999e-05 2023-10-01T09:35:56.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.284113+0000", 2023-10-01T09:35:56.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.711e-06 2023-10-01T09:35:56.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.287032+0000", 2023-10-01T09:35:56.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029190940000000001 2023-10-01T09:35:56.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.287182+0000", 2023-10-01T09:35:56.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000150433 2023-10-01T09:35:56.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4,5", 2023-10-01T09:35:56.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.287325+0000", 2023-10-01T09:35:56.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014276499999999999 2023-10-01T09:35:56.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.300430+0000", 2023-10-01T09:35:56.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01310474 2023-10-01T09:35:56.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.300442+0000", 2023-10-01T09:35:56.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2018000000000001e-05 2023-10-01T09:35:56.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.472858+0000", 2023-10-01T09:35:56.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1724165200000001 2023-10-01T09:35:56.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:35:56.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.472987+0000", 2023-10-01T09:35:56.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000128033 2023-10-01T09:35:56.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.492767+0000", 2023-10-01T09:35:56.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019780731999999999 2023-10-01T09:35:56.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.492840+0000", 2023-10-01T09:35:56.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2949999999999998e-05 2023-10-01T09:35:56.654 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.654 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.654 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.654 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169551 8.9 8:997a2a84:::benchmark_data_smithi138_431617_object68140:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-10-01T09:35:56.654 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.297515+0000", 2023-10-01T09:35:56.654 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 191.29585544899999, 2023-10-01T09:35:56.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1763150630000001, 2023-10-01T09:35:56.655 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.655 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.655 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.655 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.655 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.655 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169551 2023-10-01T09:35:56.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.655 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.297515+0000", 2023-10-01T09:35:56.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.297515+0000", 2023-10-01T09:35:56.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.297515+0000", 2023-10-01T09:35:56.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:35:56.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.297518+0000", 2023-10-01T09:35:56.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9100000000000001e-06 2023-10-01T09:35:56.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.297525+0000", 2023-10-01T09:35:56.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0879999999999997e-06 2023-10-01T09:35:56.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.297532+0000", 2023-10-01T09:35:56.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1949999999999997e-06 2023-10-01T09:35:56.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.297698+0000", 2023-10-01T09:35:56.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016572399999999999 2023-10-01T09:35:56.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.297707+0000", 2023-10-01T09:35:56.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0229999999999998e-06 2023-10-01T09:35:56.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.318926+0000", 2023-10-01T09:35:56.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.021219375 2023-10-01T09:35:56.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.323699+0000", 2023-10-01T09:35:56.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004772536 2023-10-01T09:35:56.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 6", 2023-10-01T09:35:56.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.324299+0000", 2023-10-01T09:35:56.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00060033700000000005 2023-10-01T09:35:56.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.359549+0000", 2023-10-01T09:35:56.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.035250162000000002 2023-10-01T09:35:56.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.473705+0000", 2023-10-01T09:35:56.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.114155417 2023-10-01T09:35:56.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.473753+0000", 2023-10-01T09:35:56.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8872999999999998e-05 2023-10-01T09:35:56.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.473830+0000", 2023-10-01T09:35:56.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6806999999999998e-05 2023-10-01T09:35:56.662 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.662 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.662 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.662 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169553 8.2 8:4c000137:::benchmark_data_smithi138_431617_object68142:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-10-01T09:35:56.662 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.297540+0000", 2023-10-01T09:35:56.662 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 191.29583038000001, 2023-10-01T09:35:56.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1954954799999999, 2023-10-01T09:35:56.662 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.662 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.663 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.663 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.663 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.663 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169553 2023-10-01T09:35:56.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.663 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.297540+0000", 2023-10-01T09:35:56.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.297540+0000", 2023-10-01T09:35:56.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.297540+0000", 2023-10-01T09:35:56.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:35:56.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.297542+0000", 2023-10-01T09:35:56.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.52e-06 2023-10-01T09:35:56.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.297548+0000", 2023-10-01T09:35:56.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8619999999999999e-06 2023-10-01T09:35:56.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.297553+0000", 2023-10-01T09:35:56.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7049999999999998e-06 2023-10-01T09:35:56.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.301400+0000", 2023-10-01T09:35:56.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0038472390000000001 2023-10-01T09:35:56.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.301414+0000", 2023-10-01T09:35:56.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3614999999999999e-05 2023-10-01T09:35:56.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.473262+0000", 2023-10-01T09:35:56.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1718480490000001 2023-10-01T09:35:56.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.473399+0000", 2023-10-01T09:35:56.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013700799999999999 2023-10-01T09:35:56.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:35:56.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.473534+0000", 2023-10-01T09:35:56.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013524000000000001 2023-10-01T09:35:56.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.488021+0000", 2023-10-01T09:35:56.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014487119999999999 2023-10-01T09:35:56.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.488090+0000", 2023-10-01T09:35:56.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8548999999999997e-05 2023-10-01T09:35:56.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.492902+0000", 2023-10-01T09:35:56.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048123289999999997 2023-10-01T09:35:56.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.492943+0000", 2023-10-01T09:35:56.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0714999999999999e-05 2023-10-01T09:35:56.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.493036+0000", 2023-10-01T09:35:56.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3034000000000004e-05 2023-10-01T09:35:56.670 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.670 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.670 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.670 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169557 8.9 8:92eb7ef5:::benchmark_data_smithi138_431617_object68146:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-10-01T09:35:56.670 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.301064+0000", 2023-10-01T09:35:56.670 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 191.29230689799999, 2023-10-01T09:35:56.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1728980469999999, 2023-10-01T09:35:56.671 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.671 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.671 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.671 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.671 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.671 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169557 2023-10-01T09:35:56.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.671 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.301064+0000", 2023-10-01T09:35:56.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.301064+0000", 2023-10-01T09:35:56.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.301063+0000", 2023-10-01T09:35:56.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:35:56.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.301076+0000", 2023-10-01T09:35:56.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3305000000000001e-05 2023-10-01T09:35:56.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302264+0000", 2023-10-01T09:35:56.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0011877159999999999 2023-10-01T09:35:56.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302274+0000", 2023-10-01T09:35:56.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0318e-05 2023-10-01T09:35:56.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302318+0000", 2023-10-01T09:35:56.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3862000000000001e-05 2023-10-01T09:35:56.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302326+0000", 2023-10-01T09:35:56.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1680000000000004e-06 2023-10-01T09:35:56.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.334345+0000", 2023-10-01T09:35:56.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.032018992 2023-10-01T09:35:56.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.334466+0000", 2023-10-01T09:35:56.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012147200000000001 2023-10-01T09:35:56.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 6", 2023-10-01T09:35:56.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.334591+0000", 2023-10-01T09:35:56.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012439800000000001 2023-10-01T09:35:56.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.359563+0000", 2023-10-01T09:35:56.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.024971851 2023-10-01T09:35:56.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.473865+0000", 2023-10-01T09:35:56.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.114302393 2023-10-01T09:35:56.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.473898+0000", 2023-10-01T09:35:56.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2851000000000001e-05 2023-10-01T09:35:56.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.473962+0000", 2023-10-01T09:35:56.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3886000000000005e-05 2023-10-01T09:35:56.678 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.678 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.678 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.678 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169558 8.2 8:43d3ba96:::benchmark_data_smithi138_431617_object68147:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-10-01T09:35:56.678 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.302296+0000", 2023-10-01T09:35:56.678 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 191.291074627, 2023-10-01T09:35:56.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.199435509, 2023-10-01T09:35:56.678 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.678 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.679 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.679 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.679 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.679 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169558 2023-10-01T09:35:56.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.679 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302296+0000", 2023-10-01T09:35:56.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302296+0000", 2023-10-01T09:35:56.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302295+0000", 2023-10-01T09:35:56.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:35:56.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302300+0000", 2023-10-01T09:35:56.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.515e-06 2023-10-01T09:35:56.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302308+0000", 2023-10-01T09:35:56.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2430000000000005e-06 2023-10-01T09:35:56.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302314+0000", 2023-10-01T09:35:56.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4240000000000001e-06 2023-10-01T09:35:56.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302354+0000", 2023-10-01T09:35:56.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0179000000000003e-05 2023-10-01T09:35:56.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302361+0000", 2023-10-01T09:35:56.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7570000000000004e-06 2023-10-01T09:35:56.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.477882+0000", 2023-10-01T09:35:56.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.175520742 2023-10-01T09:35:56.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.477986+0000", 2023-10-01T09:35:56.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000104066 2023-10-01T09:35:56.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:35:56.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.478067+0000", 2023-10-01T09:35:56.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0499000000000003e-05 2023-10-01T09:35:56.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.488081+0000", 2023-10-01T09:35:56.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01001408 2023-10-01T09:35:56.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.493074+0000", 2023-10-01T09:35:56.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049926679999999996 2023-10-01T09:35:56.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.501558+0000", 2023-10-01T09:35:56.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0084845149999999998 2023-10-01T09:35:56.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.501614+0000", 2023-10-01T09:35:56.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5810999999999998e-05 2023-10-01T09:35:56.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.686 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.686 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.501732+0000", 2023-10-01T09:35:56.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117719 2023-10-01T09:35:56.686 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.686 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.686 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.686 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169559 8.9 8:948e4df0:::benchmark_data_smithi138_431617_object68148:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-10-01T09:35:56.686 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.302327+0000", 2023-10-01T09:35:56.687 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 191.29104398499999, 2023-10-01T09:35:56.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1717774560000001, 2023-10-01T09:35:56.687 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.687 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.687 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.687 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.687 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.687 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169559 2023-10-01T09:35:56.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.687 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302327+0000", 2023-10-01T09:35:56.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302327+0000", 2023-10-01T09:35:56.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302326+0000", 2023-10-01T09:35:56.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:35:56.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302329+0000", 2023-10-01T09:35:56.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6900000000000001e-06 2023-10-01T09:35:56.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302335+0000", 2023-10-01T09:35:56.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0079999999999999e-06 2023-10-01T09:35:56.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302340+0000", 2023-10-01T09:35:56.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0209999999999999e-06 2023-10-01T09:35:56.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302371+0000", 2023-10-01T09:35:56.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0548e-05 2023-10-01T09:35:56.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302376+0000", 2023-10-01T09:35:56.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5060000000000003e-06 2023-10-01T09:35:56.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.339440+0000", 2023-10-01T09:35:56.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0370637659999999 2023-10-01T09:35:56.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.339567+0000", 2023-10-01T09:35:56.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000126892 2023-10-01T09:35:56.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 6", 2023-10-01T09:35:56.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.339696+0000", 2023-10-01T09:35:56.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012969500000000001 2023-10-01T09:35:56.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.359568+0000", 2023-10-01T09:35:56.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019872009 2023-10-01T09:35:56.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.473985+0000", 2023-10-01T09:35:56.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11441641 2023-10-01T09:35:56.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.474022+0000", 2023-10-01T09:35:56.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6683000000000001e-05 2023-10-01T09:35:56.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.474104+0000", 2023-10-01T09:35:56.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2628999999999998e-05 2023-10-01T09:35:56.694 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.694 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.694 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.694 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169560 8.2 8:49a59265:::benchmark_data_smithi138_431617_object68149:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-10-01T09:35:56.694 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.302351+0000", 2023-10-01T09:35:56.694 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 191.291020051, 2023-10-01T09:35:56.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.209456468, 2023-10-01T09:35:56.695 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.695 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.695 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.695 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.695 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.695 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169560 2023-10-01T09:35:56.695 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.695 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.695 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302351+0000", 2023-10-01T09:35:56.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302351+0000", 2023-10-01T09:35:56.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302350+0000", 2023-10-01T09:35:56.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:35:56.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302353+0000", 2023-10-01T09:35:56.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3970000000000001e-06 2023-10-01T09:35:56.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302359+0000", 2023-10-01T09:35:56.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9109999999999998e-06 2023-10-01T09:35:56.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302364+0000", 2023-10-01T09:35:56.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.908e-06 2023-10-01T09:35:56.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302394+0000", 2023-10-01T09:35:56.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9294000000000001e-05 2023-10-01T09:35:56.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.302397+0000", 2023-10-01T09:35:56.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7230000000000002e-06 2023-10-01T09:35:56.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.488110+0000", 2023-10-01T09:35:56.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1857123869999999 2023-10-01T09:35:56.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.488238+0000", 2023-10-01T09:35:56.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000128021 2023-10-01T09:35:56.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:35:56.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.488366+0000", 2023-10-01T09:35:56.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012808399999999999 2023-10-01T09:35:56.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.501778+0000", 2023-10-01T09:35:56.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013411963000000001 2023-10-01T09:35:56.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.501786+0000", 2023-10-01T09:35:56.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7889999999999995e-06 2023-10-01T09:35:56.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.511654+0000", 2023-10-01T09:35:56.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0098684270000000008 2023-10-01T09:35:56.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.701 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.701 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.511701+0000", 2023-10-01T09:35:56.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7202e-05 2023-10-01T09:35:56.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.511807+0000", 2023-10-01T09:35:56.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105726 2023-10-01T09:35:56.702 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.702 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.702 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.702 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169563 8.9 8:970ce6af:::benchmark_data_smithi138_431617_object68152:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-10-01T09:35:56.702 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.320200+0000", 2023-10-01T09:35:56.703 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 191.27317020000001, 2023-10-01T09:35:56.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.154073849, 2023-10-01T09:35:56.703 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.703 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.703 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.703 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.703 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.703 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169563 2023-10-01T09:35:56.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.703 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.320200+0000", 2023-10-01T09:35:56.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.320200+0000", 2023-10-01T09:35:56.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.320199+0000", 2023-10-01T09:35:56.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:56.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.320214+0000", 2023-10-01T09:35:56.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4589e-05 2023-10-01T09:35:56.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.320223+0000", 2023-10-01T09:35:56.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1779999999999992e-06 2023-10-01T09:35:56.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.320229+0000", 2023-10-01T09:35:56.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4030000000000004e-06 2023-10-01T09:35:56.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.320281+0000", 2023-10-01T09:35:56.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1795000000000001e-05 2023-10-01T09:35:56.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.320293+0000", 2023-10-01T09:35:56.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2619000000000001e-05 2023-10-01T09:35:56.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.344684+0000", 2023-10-01T09:35:56.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0243901630000001 2023-10-01T09:35:56.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.707 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.707 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.344805+0000", 2023-10-01T09:35:56.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121178 2023-10-01T09:35:56.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 6", 2023-10-01T09:35:56.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.344945+0000", 2023-10-01T09:35:56.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000139863 2023-10-01T09:35:56.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.359577+0000", 2023-10-01T09:35:56.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014632078 2023-10-01T09:35:56.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.474148+0000", 2023-10-01T09:35:56.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11457126500000001 2023-10-01T09:35:56.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.474189+0000", 2023-10-01T09:35:56.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0509999999999997e-05 2023-10-01T09:35:56.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.474274+0000", 2023-10-01T09:35:56.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5767999999999996e-05 2023-10-01T09:35:56.710 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.710 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.710 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.710 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169564 8.2 8:49e252fe:::benchmark_data_smithi138_431617_object68153:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-10-01T09:35:56.710 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.321134+0000", 2023-10-01T09:35:56.710 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 191.272237107, 2023-10-01T09:35:56.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.19083863, 2023-10-01T09:35:56.710 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.711 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.711 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.711 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.711 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.711 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169564 2023-10-01T09:35:56.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.711 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.321134+0000", 2023-10-01T09:35:56.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.321134+0000", 2023-10-01T09:35:56.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.321133+0000", 2023-10-01T09:35:56.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:35:56.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.321137+0000", 2023-10-01T09:35:56.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1289999999999998e-06 2023-10-01T09:35:56.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.321146+0000", 2023-10-01T09:35:56.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3440000000000007e-06 2023-10-01T09:35:56.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.321152+0000", 2023-10-01T09:35:56.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2159999999999997e-06 2023-10-01T09:35:56.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.321213+0000", 2023-10-01T09:35:56.714 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:56.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1828999999999994e-05 2023-10-01T09:35:56.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.321223+0000", 2023-10-01T09:35:56.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4849999999999998e-06 2023-10-01T09:35:56.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.493097+0000", 2023-10-01T09:35:56.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1718743620000001 2023-10-01T09:35:56.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.493198+0000", 2023-10-01T09:35:56.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000100485 2023-10-01T09:35:56.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:35:56.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.493310+0000", 2023-10-01T09:35:56.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0001125 2023-10-01T09:35:56.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.501792+0000", 2023-10-01T09:35:56.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0084818319999999999 2023-10-01T09:35:56.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.506419+0000", 2023-10-01T09:35:56.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0046268070000000001 2023-10-01T09:35:56.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.511842+0000", 2023-10-01T09:35:56.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054231219999999998 2023-10-01T09:35:56.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.511885+0000", 2023-10-01T09:35:56.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2678e-05 2023-10-01T09:35:56.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.511972+0000", 2023-10-01T09:35:56.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7457e-05 2023-10-01T09:35:56.719 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.719 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.719 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.720 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.720 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:183900 8.e 8:732ee076:::benchmark_data_smithi138_431617_object82489:head [delete] snapc 0=[] ondisk+write+known_if_redirected e382)", 2023-10-01T09:35:56.720 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:13.529929+0000", 2023-10-01T09:35:56.720 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 163.06344216599999, 2023-10-01T09:35:56.720 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.049326564, 2023-10-01T09:35:56.720 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.720 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.720 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.720 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.720 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.720 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 183900 2023-10-01T09:35:56.720 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.721 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.529929+0000", 2023-10-01T09:35:56.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.721 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.721 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.529929+0000", 2023-10-01T09:35:56.721 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.721 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.721 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.529927+0000", 2023-10-01T09:35:56.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:56.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.529935+0000", 2023-10-01T09:35:56.722 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6059999999999997e-06 2023-10-01T09:35:56.722 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.722 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.722 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.722 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.529951+0000", 2023-10-01T09:35:56.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6157000000000001e-05 2023-10-01T09:35:56.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.529960+0000", 2023-10-01T09:35:56.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2199999999999998e-06 2023-10-01T09:35:56.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.723 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.723 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.723 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.530021+0000", 2023-10-01T09:35:56.723 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1136000000000006e-05 2023-10-01T09:35:56.723 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.530028+0000", 2023-10-01T09:35:56.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0990000000000001e-06 2023-10-01T09:35:56.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.724 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.565654+0000", 2023-10-01T09:35:56.724 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.035625585 2023-10-01T09:35:56.724 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.724 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.724 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.565797+0000", 2023-10-01T09:35:56.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014305700000000001 2023-10-01T09:35:56.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,4", 2023-10-01T09:35:56.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.565947+0000", 2023-10-01T09:35:56.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015052899999999999 2023-10-01T09:35:56.725 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.725 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.725 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.725 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.570577+0000", 2023-10-01T09:35:56.725 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004629218 2023-10-01T09:35:56.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.574982+0000", 2023-10-01T09:35:56.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044054020000000001 2023-10-01T09:35:56.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.726 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.726 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.726 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579152+0000", 2023-10-01T09:35:56.726 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041696930000000004 2023-10-01T09:35:56.726 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579220+0000", 2023-10-01T09:35:56.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7838999999999999e-05 2023-10-01T09:35:56.727 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.727 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.727 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.727 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579255+0000", 2023-10-01T09:35:56.727 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5438999999999997e-05 2023-10-01T09:35:56.727 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.727 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.727 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.728 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.728 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.728 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:183908 8.e 8:79ef37d1:::benchmark_data_smithi138_431617_object82497:head [delete] snapc 0=[] ondisk+write+known_if_redirected e382)", 2023-10-01T09:35:56.728 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:13.538218+0000", 2023-10-01T09:35:56.728 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 163.05515229700001, 2023-10-01T09:35:56.728 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0412136089999999, 2023-10-01T09:35:56.728 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.728 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.728 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.728 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.728 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.728 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 183908 2023-10-01T09:35:56.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.729 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.538218+0000", 2023-10-01T09:35:56.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.729 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.729 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.729 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.729 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.538218+0000", 2023-10-01T09:35:56.729 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.538218+0000", 2023-10-01T09:35:56.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:35:56.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.730 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.730 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.538222+0000", 2023-10-01T09:35:56.730 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5719999999999996e-06 2023-10-01T09:35:56.730 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.730 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.538234+0000", 2023-10-01T09:35:56.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1761e-05 2023-10-01T09:35:56.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.538241+0000", 2023-10-01T09:35:56.731 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.764e-06 2023-10-01T09:35:56.731 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.731 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.731 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.731 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.542304+0000", 2023-10-01T09:35:56.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004063489 2023-10-01T09:35:56.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.542315+0000", 2023-10-01T09:35:56.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0777e-05 2023-10-01T09:35:56.732 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.732 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.732 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.732 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.570620+0000", 2023-10-01T09:35:56.732 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.028304935 2023-10-01T09:35:56.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.570755+0000", 2023-10-01T09:35:56.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000134695 2023-10-01T09:35:56.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.733 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,4", 2023-10-01T09:35:56.733 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.570882+0000", 2023-10-01T09:35:56.733 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000127506 2023-10-01T09:35:56.733 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.733 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579110+0000", 2023-10-01T09:35:56.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0082281530000000002 2023-10-01T09:35:56.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579327+0000", 2023-10-01T09:35:56.734 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000216493 2023-10-01T09:35:56.734 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.734 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.734 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.734 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579341+0000", 2023-10-01T09:35:56.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4041e-05 2023-10-01T09:35:56.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579391+0000", 2023-10-01T09:35:56.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9747999999999999e-05 2023-10-01T09:35:56.735 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.735 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.735 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.735 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579432+0000", 2023-10-01T09:35:56.735 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1428999999999999e-05 2023-10-01T09:35:56.736 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.736 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.736 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.736 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.736 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.736 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:183925 8.e 8:72bc7946:::benchmark_data_smithi138_431617_object82514:head [delete] snapc 0=[] ondisk+write+known_if_redirected e382)", 2023-10-01T09:35:56.736 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:13.560550+0000", 2023-10-01T09:35:56.736 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 163.032820988, 2023-10-01T09:35:56.736 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0190212350000001, 2023-10-01T09:35:56.736 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.736 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.736 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.737 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.737 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.737 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 183925 2023-10-01T09:35:56.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.737 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.737 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.560550+0000", 2023-10-01T09:35:56.737 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.737 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.737 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.737 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.560550+0000", 2023-10-01T09:35:56.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.560549+0000", 2023-10-01T09:35:56.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:35:56.738 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.738 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.738 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.738 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.560552+0000", 2023-10-01T09:35:56.738 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3119999999999998e-06 2023-10-01T09:35:56.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.560563+0000", 2023-10-01T09:35:56.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0817e-05 2023-10-01T09:35:56.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.739 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.739 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.560569+0000", 2023-10-01T09:35:56.739 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7230000000000001e-06 2023-10-01T09:35:56.739 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.739 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.560725+0000", 2023-10-01T09:35:56.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015579600000000001 2023-10-01T09:35:56.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.560735+0000", 2023-10-01T09:35:56.740 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0068000000000001e-05 2023-10-01T09:35:56.740 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.740 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.740 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.740 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.575012+0000", 2023-10-01T09:35:56.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.01427745 2023-10-01T09:35:56.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.575126+0000", 2023-10-01T09:35:56.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011335100000000001 2023-10-01T09:35:56.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.741 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.741 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,4", 2023-10-01T09:35:56.741 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.575201+0000", 2023-10-01T09:35:56.741 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5438999999999994e-05 2023-10-01T09:35:56.741 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579479+0000", 2023-10-01T09:35:56.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042782979999999998 2023-10-01T09:35:56.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.742 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.742 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579513+0000", 2023-10-01T09:35:56.742 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3285999999999999e-05 2023-10-01T09:35:56.742 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.742 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579520+0000", 2023-10-01T09:35:56.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3379999999999999e-06 2023-10-01T09:35:56.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579546+0000", 2023-10-01T09:35:56.743 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5673e-05 2023-10-01T09:35:56.743 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.743 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.743 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.743 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579571+0000", 2023-10-01T09:35:56.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5205e-05 2023-10-01T09:35:56.744 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.744 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.744 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.744 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.744 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.744 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:183937 8.e 8:7e11f5b6:::benchmark_data_smithi138_431617_object82526:head [delete] snapc 0=[] ondisk+write+known_if_redirected e382)", 2023-10-01T09:35:56.744 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:13.578863+0000", 2023-10-01T09:35:56.744 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 163.014507369, 2023-10-01T09:35:56.744 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.04065995, 2023-10-01T09:35:56.744 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.744 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.745 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.745 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.745 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.745 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 183937 2023-10-01T09:35:56.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.745 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.745 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.745 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.578863+0000", 2023-10-01T09:35:56.745 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.745 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.745 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.746 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.578863+0000", 2023-10-01T09:35:56.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.746 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.746 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.578862+0000", 2023-10-01T09:35:56.746 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:56.746 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.746 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.746 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.747 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.578869+0000", 2023-10-01T09:35:56.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7120000000000003e-06 2023-10-01T09:35:56.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.747 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.747 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.578884+0000", 2023-10-01T09:35:56.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5136e-05 2023-10-01T09:35:56.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.747 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.747 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.747 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.578891+0000", 2023-10-01T09:35:56.747 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5820000000000004e-06 2023-10-01T09:35:56.747 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.578940+0000", 2023-10-01T09:35:56.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8878999999999997e-05 2023-10-01T09:35:56.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.748 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.578952+0000", 2023-10-01T09:35:56.748 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1584e-05 2023-10-01T09:35:56.748 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.748 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.748 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579604+0000", 2023-10-01T09:35:56.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0006522840000001 2023-10-01T09:35:56.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579699+0000", 2023-10-01T09:35:56.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4381000000000003e-05 2023-10-01T09:35:56.749 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.749 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.749 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,4", 2023-10-01T09:35:56.749 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.579801+0000", 2023-10-01T09:35:56.749 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101896 2023-10-01T09:35:56.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.600222+0000", 2023-10-01T09:35:56.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020421357000000001 2023-10-01T09:35:56.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.750 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.750 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.750 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.600253+0000", 2023-10-01T09:35:56.750 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1092e-05 2023-10-01T09:35:56.750 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.619430+0000", 2023-10-01T09:35:56.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019176727000000001 2023-10-01T09:35:56.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.751 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.619482+0000", 2023-10-01T09:35:56.751 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2305000000000002e-05 2023-10-01T09:35:56.751 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.751 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.751 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.752 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.619523+0000", 2023-10-01T09:35:56.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1254000000000002e-05 2023-10-01T09:35:56.752 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.752 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.752 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.752 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.752 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.752 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:183956 8.e 8:7d60244a:::benchmark_data_smithi138_431617_object82545:head [delete] snapc 0=[] ondisk+write+known_if_redirected e382)", 2023-10-01T09:35:56.752 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:13.609335+0000", 2023-10-01T09:35:56.752 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 162.984035904, 2023-10-01T09:35:56.752 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0102767130000001, 2023-10-01T09:35:56.752 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.753 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.753 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.753 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.753 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.753 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 183956 2023-10-01T09:35:56.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.753 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.753 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.753 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.753 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.609335+0000", 2023-10-01T09:35:56.753 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.753 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.609335+0000", 2023-10-01T09:35:56.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.754 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.754 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.609333+0000", 2023-10-01T09:35:56.754 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:35:56.754 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.754 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.609342+0000", 2023-10-01T09:35:56.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5690000000000005e-06 2023-10-01T09:35:56.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.609354+0000", 2023-10-01T09:35:56.755 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2394e-05 2023-10-01T09:35:56.755 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.755 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.755 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.755 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.609363+0000", 2023-10-01T09:35:56.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1540000000000008e-06 2023-10-01T09:35:56.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.756 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.756 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.609407+0000", 2023-10-01T09:35:56.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3800000000000001e-05 2023-10-01T09:35:56.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.756 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.756 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.756 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.609419+0000", 2023-10-01T09:35:56.756 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.167e-05 2023-10-01T09:35:56.756 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.757 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.583882+0000", 2023-10-01T09:35:56.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.97446337599999999 2023-10-01T09:35:56.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.757 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.757 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.584018+0000", 2023-10-01T09:35:56.757 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013541700000000001 2023-10-01T09:35:56.757 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.757 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,4", 2023-10-01T09:35:56.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.584139+0000", 2023-10-01T09:35:56.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012097800000000001 2023-10-01T09:35:56.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.600281+0000", 2023-10-01T09:35:56.758 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016142479000000001 2023-10-01T09:35:56.758 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.758 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.758 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.758 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.600308+0000", 2023-10-01T09:35:56.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7449000000000001e-05 2023-10-01T09:35:56.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.759 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.619534+0000", 2023-10-01T09:35:56.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019225399000000001 2023-10-01T09:35:56.759 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.759 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.759 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.759 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.619576+0000", 2023-10-01T09:35:56.759 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2301e-05 2023-10-01T09:35:56.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.760 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.760 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.619611+0000", 2023-10-01T09:35:56.760 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5293999999999998e-05 2023-10-01T09:35:56.760 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.760 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.760 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.760 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.760 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.760 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:183969 8.e 8:73527eab:::benchmark_data_smithi138_431617_object82558:head [delete] snapc 0=[] ondisk+write+known_if_redirected e382)", 2023-10-01T09:35:56.760 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:13.623369+0000", 2023-10-01T09:35:56.761 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 162.97000144, 2023-10-01T09:35:56.761 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99632954799999995, 2023-10-01T09:35:56.761 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:35:56.761 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:35:56.761 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:35:56.761 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:35:56.761 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:35:56.761 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 183969 2023-10-01T09:35:56.761 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.761 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:35:56.761 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.761 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:35:56.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.623369+0000", 2023-10-01T09:35:56.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:35:56.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.623369+0000", 2023-10-01T09:35:56.762 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:35:56.762 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.762 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.762 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:35:56.762 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.623369+0000", 2023-10-01T09:35:56.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:35:56.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:35:56.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.623371+0000", 2023-10-01T09:35:56.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2340000000000001e-06 2023-10-01T09:35:56.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.763 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.763 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:35:56.763 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.623377+0000", 2023-10-01T09:35:56.763 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7409999999999998e-06 2023-10-01T09:35:56.763 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:35:56.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.623382+0000", 2023-10-01T09:35:56.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6659999999999999e-06 2023-10-01T09:35:56.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.764 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.623557+0000", 2023-10-01T09:35:56.764 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017585499999999999 2023-10-01T09:35:56.764 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.764 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.764 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:35:56.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:13.623565+0000", 2023-10-01T09:35:56.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6909999999999997e-06 2023-10-01T09:35:56.765 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:35:56.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.589044+0000", 2023-10-01T09:35:56.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96547897100000002 2023-10-01T09:35:56.765 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.765 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.765 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:35:56.765 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.589140+0000", 2023-10-01T09:35:56.765 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5599000000000004e-05 2023-10-01T09:35:56.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.766 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,4", 2023-10-01T09:35:56.766 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.589238+0000", 2023-10-01T09:35:56.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8170000000000004e-05 2023-10-01T09:35:56.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.766 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.766 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.600337+0000", 2023-10-01T09:35:56.766 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011099108 2023-10-01T09:35:56.766 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.767 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:35:56.767 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.600364+0000", 2023-10-01T09:35:56.767 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6951e-05 2023-10-01T09:35:56.767 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.767 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:35:56.767 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.619621+0000", 2023-10-01T09:35:56.767 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019257153999999999 2023-10-01T09:35:56.767 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.767 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:35:56.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.619661+0000", 2023-10-01T09:35:56.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9699000000000001e-05 2023-10-01T09:35:56.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:35:56.768 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:35:56.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:35:56.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:14.619699+0000", 2023-10-01T09:35:56.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8040999999999997e-05 2023-10-01T09:35:56.768 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.768 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.768 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.768 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:35:56.768 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:35:56.768 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:35:56.856 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:56.962 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:56.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:56.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:57.072 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:35:57.099 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:57.672 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:35:57.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:57.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:35:57.779 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:35:57.831 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:57.927 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:35:57.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:57.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:35:57.948 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:57 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 931/293817 objects degraded (0.317%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:35:58.048 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:58.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:57 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 931/293817 objects degraded (0.317%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:35:58.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:57 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 931/293817 objects degraded (0.317%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:35:58.106 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:58.167 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:35:58.183 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:35:58.225 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:35:58.249 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:35:58.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:58.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:35:58.373 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:58.416 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:58.495 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:58.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:58.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:58.606 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:35:58.656 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:58.734 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:35:58.738 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:58 smithi190 ceph-mon[160435]: pgmap v1186: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 2.9 MiB/s wr, 634 op/s; 931/290217 objects degraded (0.321%); 759/290217 objects misplaced (0.262%); 1.1 MiB/s, 18 objects/s recovering 2023-10-01T09:35:58.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:58.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:35:58.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:58 smithi138 ceph-mon[185886]: pgmap v1186: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 2.9 MiB/s wr, 634 op/s; 931/290217 objects degraded (0.321%); 759/290217 objects misplaced (0.262%); 1.1 MiB/s, 18 objects/s recovering 2023-10-01T09:35:58.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:58 smithi138 ceph-mon[192513]: pgmap v1186: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 2.9 MiB/s wr, 634 op/s; 931/290217 objects degraded (0.321%); 759/290217 objects misplaced (0.262%); 1.1 MiB/s, 18 objects/s recovering 2023-10-01T09:35:58.840 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:58.867 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:58.956 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:35:58.962 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:58.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:58.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:58.999 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:35:59.034 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:35:59.067 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:35:59.069 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:59.092 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:35:59.126 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:35:59.136 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:59.156 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:35:59.187 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:35:59.189 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:35:59.190 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:35:59.190 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:35:59.190 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:35:59.190 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:35:59.190 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:35:59.190 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:35:59.190 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:35:59.201 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:59.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:59.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:59.307 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:35:59.387 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:59.512 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:35:59.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:35:59.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:35:59.617 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:35:59.688 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:35:59.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:35:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1494956053' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:59.901 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:35:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1494956053' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:35:59.902 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:35:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1494956053' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:00.530 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:00.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:00.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:00.639 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:36:00.647 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:00.722 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:00.746 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:00 smithi138 ceph-mon[192513]: pgmap v1187: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 931/282384 objects degraded (0.330%); 643/282384 objects misplaced (0.228%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:36:00.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:00 smithi138 ceph-mon[185886]: pgmap v1187: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 931/282384 objects degraded (0.330%); 643/282384 objects misplaced (0.228%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:36:00.760 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:00.862 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:00.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:00.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:00.975 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:36:01.053 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:01.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:00 smithi190 ceph-mon[160435]: pgmap v1187: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 931/282384 objects degraded (0.330%); 643/282384 objects misplaced (0.228%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:36:01.392 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:36:01.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:01.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:36:01.499 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:36:01.541 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:01.652 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:36:01.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:01.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:36:01.758 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:36:01.783 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:01.826 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:36:01.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:01.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:36:01.874 INFO:teuthology.orchestra.run.smithi138.stdout:moved 0 / 267 (0%) 2023-10-01T09:36:01.874 INFO:teuthology.orchestra.run.smithi138.stdout:avg 33.375 2023-10-01T09:36:01.874 INFO:teuthology.orchestra.run.smithi138.stdout:stddev 6.70704 -> 6.70704 (expected baseline 5.40399) 2023-10-01T09:36:01.874 INFO:teuthology.orchestra.run.smithi138.stdout:min osd.2 with 20 -> 20 pgs (0.599251 -> 0.599251 * mean) 2023-10-01T09:36:01.874 INFO:teuthology.orchestra.run.smithi138.stdout:max osd.1 with 43 -> 43 pgs (1.28839 -> 1.28839 * mean) 2023-10-01T09:36:01.874 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:36:01.874 INFO:teuthology.orchestra.run.smithi138.stdout:oload 1000 2023-10-01T09:36:01.875 INFO:teuthology.orchestra.run.smithi138.stdout:max_change 0.05 2023-10-01T09:36:01.875 INFO:teuthology.orchestra.run.smithi138.stdout:max_change_osds 4 2023-10-01T09:36:01.875 INFO:teuthology.orchestra.run.smithi138.stdout:average_utilization 0.0315 2023-10-01T09:36:01.875 INFO:teuthology.orchestra.run.smithi138.stdout:overload_utilization 0.3155 2023-10-01T09:36:01.876 INFO:teuthology.orchestra.run.smithi138.stderr:no change 2023-10-01T09:36:01.933 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:01.960 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:02.250 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:02.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:02.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:02.362 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:36:02.424 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:02.457 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:36:03.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:03 smithi138 ceph-mon[192513]: from='client.46199 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:36:03.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:03 smithi138 ceph-mon[192513]: pgmap v1188: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 931/278883 objects degraded (0.334%); 643/278883 objects misplaced (0.231%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:36:03.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:03 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 931/278883 objects degraded (0.334%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:03 smithi138 ceph-mon[185886]: from='client.46199 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:36:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:03 smithi138 ceph-mon[185886]: pgmap v1188: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 931/278883 objects degraded (0.334%); 643/278883 objects misplaced (0.231%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:36:03.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:03 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 931/278883 objects degraded (0.334%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:03.551 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-10-01T09:36:03.551 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-10-01T09:36:03.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:03 smithi190 ceph-mon[160435]: from='client.46199 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:36:03.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:03 smithi190 ceph-mon[160435]: pgmap v1188: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 931/278883 objects degraded (0.334%); 643/278883 objects misplaced (0.231%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:36:03.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:03 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 931/278883 objects degraded (0.334%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:03.626 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:03.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:03.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:03.741 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:36:03.783 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:03.873 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:36:03.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:03.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:36:03.980 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:36:04.045 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:04.162 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:04.372 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:04.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:04.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:04.430 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:36:04.430 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:36:04.485 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:36:04.579 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:04.758 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:04.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:04.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:04.869 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:36:04.913 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:05.038 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:36:05.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:05.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:36:05.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:36:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:36:04] "GET /metrics HTTP/1.1" 200 36443 "" "Prometheus/2.33.4" 2023-10-01T09:36:05.148 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:05.197 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:05 smithi190 ceph-mon[160435]: pgmap v1189: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 931/273927 objects degraded (0.340%); 643/273927 objects misplaced (0.235%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:36:05.197 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2003617548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:36:05.198 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:05 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:36:05.214 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:36:05.226 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:05.347 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:05.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:05.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:05.455 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:36:05.505 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:05 smithi138 ceph-mon[192513]: pgmap v1189: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 931/273927 objects degraded (0.340%); 643/273927 objects misplaced (0.235%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:36:05.505 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2003617548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:36:05.505 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:05 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:36:05.507 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:05 smithi138 ceph-mon[185886]: pgmap v1189: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 931/273927 objects degraded (0.340%); 643/273927 objects misplaced (0.235%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:36:05.507 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2003617548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:36:05.507 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:05 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:36:05.517 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:06.239 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:36:06.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:06.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:36:06.349 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:36:06.409 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:06.449 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:36:06.468 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:06 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:36:06.468 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:06 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:36:06.468 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:06 smithi138 ceph-mon[185886]: osdmap e428: 8 total, 8 up, 8 in 2023-10-01T09:36:06.468 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:06 smithi138 ceph-mon[185886]: failed to encode map e428 with expected crc 2023-10-01T09:36:06.468 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:06 smithi138 ceph-mon[185886]: failed to encode map e428 with expected crc 2023-10-01T09:36:06.468 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:06 smithi138 ceph-mon[185886]: failed to encode map e428 with expected crc 2023-10-01T09:36:06.468 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:06 smithi138 ceph-mon[185886]: pgmap v1191: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 931/271014 objects degraded (0.344%); 526/271014 objects misplaced (0.194%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:06.470 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:06 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:36:06.470 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:06 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:36:06.470 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:06 smithi138 ceph-mon[192513]: osdmap e428: 8 total, 8 up, 8 in 2023-10-01T09:36:06.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:06 smithi138 ceph-mon[192513]: failed to encode map e428 with expected crc 2023-10-01T09:36:06.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:06 smithi138 ceph-mon[192513]: failed to encode map e428 with expected crc 2023-10-01T09:36:06.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:06 smithi138 ceph-mon[192513]: failed to encode map e428 with expected crc 2023-10-01T09:36:06.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:06 smithi138 ceph-mon[192513]: pgmap v1191: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 931/271014 objects degraded (0.344%); 526/271014 objects misplaced (0.194%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:06.493 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:36:06.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:06.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:36:06.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:06 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:36:06.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:06 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:36:06.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:06 smithi190 ceph-mon[160435]: osdmap e428: 8 total, 8 up, 8 in 2023-10-01T09:36:06.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:06 smithi190 ceph-mon[160435]: failed to encode map e428 with expected crc 2023-10-01T09:36:06.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:06 smithi190 ceph-mon[160435]: failed to encode map e428 with expected crc 2023-10-01T09:36:06.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:06 smithi190 ceph-mon[160435]: failed to encode map e428 with expected crc 2023-10-01T09:36:06.501 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:06 smithi190 ceph-mon[160435]: pgmap v1191: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 931/271014 objects degraded (0.344%); 526/271014 objects misplaced (0.194%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:06.601 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:06.692 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:07.090 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:07.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:07.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:07.194 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:07.209 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:07.282 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:07.389 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:07.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:07.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:07.507 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:07.555 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:07.614 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:07.695 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:07 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 931/271014 objects degraded (0.344%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:07.696 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:07 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 931/271014 objects degraded (0.344%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:07.726 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:07.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:07.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:07.837 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:36:07.857 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:36:07.894 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:36:07.894 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:07 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 931/271014 objects degraded (0.344%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:07.899 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:07.923 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:36:07.958 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:36:07.984 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:36:07.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:07.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:36:08.002 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:36:08.044 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:36:08.095 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:36:08.098 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:36:08.135 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:36:08.139 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:36:08.139 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:36:08.140 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:36:08.140 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:36:08.140 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:36:08.140 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:36:08.140 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:36:08.140 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:36:08.174 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:08.252 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:36:08.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:08.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:36:08.362 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:36:08.407 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:08.471 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:36:08.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:08.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:36:08.577 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:08.648 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:08.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:08 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1990012504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:08.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:08 smithi190 ceph-mon[160435]: pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 931/267453 objects degraded (0.348%); 526/267453 objects misplaced (0.197%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:08.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:08 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1990012504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:08.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:08 smithi138 ceph-mon[185886]: pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 931/267453 objects degraded (0.348%); 526/267453 objects misplaced (0.197%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:08.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:08 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1990012504' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:08.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:08 smithi138 ceph-mon[192513]: pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 931/267453 objects degraded (0.348%); 526/267453 objects misplaced (0.197%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:09.233 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:36:10.290 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:10.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:10.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:10.406 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:36:10.514 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:10.697 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:10.837 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:36:10.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:10.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:36:10.948 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:36:11.014 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:11.085 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:36:11.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:11.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:36:11.168 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:36:11.168 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:36:11.168 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:36:11.168 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:36:11.168 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:36:11.198 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:36:11.257 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:11.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:11 smithi190 ceph-mon[160435]: pgmap v1193: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 931/259257 objects degraded (0.359%); 410/259257 objects misplaced (0.158%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:11.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:11.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:36:11.344 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:36:11.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:11.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:36:11.443 INFO:teuthology.orchestra.run.smithi138.stdout:moved 9 / 267 (3.37079%) 2023-10-01T09:36:11.443 INFO:teuthology.orchestra.run.smithi138.stdout:avg 33.375 2023-10-01T09:36:11.443 INFO:teuthology.orchestra.run.smithi138.stdout:stddev 6.70704 -> 6.72565 (expected baseline 5.40399) 2023-10-01T09:36:11.443 INFO:teuthology.orchestra.run.smithi138.stdout:min osd.2 with 20 -> 20 pgs (0.599251 -> 0.599251 * mean) 2023-10-01T09:36:11.443 INFO:teuthology.orchestra.run.smithi138.stdout:max osd.1 with 43 -> 41 pgs (1.28839 -> 1.22846 * mean) 2023-10-01T09:36:11.444 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:36:11.444 INFO:teuthology.orchestra.run.smithi138.stdout:oload 110 2023-10-01T09:36:11.444 INFO:teuthology.orchestra.run.smithi138.stdout:max_change 0.05 2023-10-01T09:36:11.444 INFO:teuthology.orchestra.run.smithi138.stdout:max_change_osds 4 2023-10-01T09:36:11.444 INFO:teuthology.orchestra.run.smithi138.stdout:average_utilization 0.0297 2023-10-01T09:36:11.444 INFO:teuthology.orchestra.run.smithi138.stdout:overload_utilization 0.0327 2023-10-01T09:36:11.444 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7 weight 1.0000 -> 0.9500 2023-10-01T09:36:11.444 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1 weight 1.0000 -> 0.9500 2023-10-01T09:36:11.444 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3 weight 1.0000 -> 0.9500 2023-10-01T09:36:11.444 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5 weight 1.0000 -> 0.9500 2023-10-01T09:36:11.448 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:11.449 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:36:11 smithi138 conmon[178630]: 2023-10-01T09:36:11.441+0000 7fa004e48700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-10-01T09:36:11.449 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:11 smithi138 ceph-mon[192513]: pgmap v1193: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 931/259257 objects degraded (0.359%); 410/259257 objects misplaced (0.158%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:11.449 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:11.449 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:36:11.450 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:11 smithi138 ceph-mon[185886]: pgmap v1193: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 931/259257 objects degraded (0.359%); 410/259257 objects misplaced (0.158%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:11.450 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:11.450 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:36:11.451 INFO:teuthology.orchestra.run.smithi138.stderr:no change 2023-10-01T09:36:11.515 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:12.060 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:36:12.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:12 smithi138 ceph-mon[185886]: from='client.46211 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:36:12.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:12 smithi138 ceph-mon[185886]: pgmap v1194: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 931/255873 objects degraded (0.364%); 410/255873 objects misplaced (0.160%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:12.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:12 smithi138 ceph-mon[192513]: from='client.46211 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:36:12.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:12 smithi138 ceph-mon[192513]: pgmap v1194: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 931/255873 objects degraded (0.364%); 410/255873 objects misplaced (0.160%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:12.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:12 smithi190 ceph-mon[160435]: from='client.46211 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:36:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:12 smithi190 ceph-mon[160435]: pgmap v1194: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 931/255873 objects degraded (0.364%); 410/255873 objects misplaced (0.160%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:12.770 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:36:13.049 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:13.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:13.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:13.121 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-10-01T09:36:13.122 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-10-01T09:36:13.161 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:36:13.230 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:13.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:13 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 931/255873 objects degraded (0.364%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:13.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:13 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 931/255873 objects degraded (0.364%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:13.346 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:36:13.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:13.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:36:13.460 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:13.531 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:13.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:13 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 931/255873 objects degraded (0.364%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:13.656 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:13.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:13.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:13.764 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:13.832 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:13.851 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:13.913 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:14.136 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:14.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:14.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:14.242 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:36:14.298 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:14.344 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:14.400 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:14 smithi138 ceph-mon[192513]: pgmap v1195: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 931/251043 objects degraded (0.371%); 410/251043 objects misplaced (0.163%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:14.406 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:14 smithi138 ceph-mon[185886]: pgmap v1195: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 931/251043 objects degraded (0.371%); 410/251043 objects misplaced (0.163%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:14.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:14 smithi190 ceph-mon[160435]: pgmap v1195: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 931/251043 objects degraded (0.371%); 410/251043 objects misplaced (0.163%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:15.282 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:15 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1826236906' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:36:15.283 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:36:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:36:14] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.33.4" 2023-10-01T09:36:15.304 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:36:15.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:15 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1826236906' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:36:15.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:15 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1826236906' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:36:16.283 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:16 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:36:16.283 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:16 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1826236906' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:36:16.283 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:16 smithi138 ceph-mon[192513]: osdmap e429: 8 total, 8 up, 8 in 2023-10-01T09:36:16.283 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:16 smithi138 ceph-mon[192513]: failed to encode map e429 with expected crc 2023-10-01T09:36:16.283 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:16 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:36:16.284 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:16 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1826236906' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:36:16.284 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:16 smithi138 ceph-mon[185886]: osdmap e429: 8 total, 8 up, 8 in 2023-10-01T09:36:16.284 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:16 smithi138 ceph-mon[185886]: failed to encode map e429 with expected crc 2023-10-01T09:36:16.284 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:16 smithi138 ceph-mon[185886]: failed to encode map e429 with expected crc 2023-10-01T09:36:16.284 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:16 smithi138 ceph-mon[185886]: failed to encode map e429 with expected crc 2023-10-01T09:36:16.284 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:16 smithi138 ceph-mon[185886]: pgmap v1197: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 931/248217 objects degraded (0.375%); 293/248217 objects misplaced (0.118%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:16.458 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:16.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:16.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:16.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:16 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:36:16.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:16 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1826236906' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:36:16.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:16 smithi190 ceph-mon[160435]: osdmap e429: 8 total, 8 up, 8 in 2023-10-01T09:36:16.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:16 smithi190 ceph-mon[160435]: failed to encode map e429 with expected crc 2023-10-01T09:36:16.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:16 smithi190 ceph-mon[160435]: failed to encode map e429 with expected crc 2023-10-01T09:36:16.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:16 smithi190 ceph-mon[160435]: failed to encode map e429 with expected crc 2023-10-01T09:36:16.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:16 smithi190 ceph-mon[160435]: pgmap v1197: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 931/248217 objects degraded (0.375%); 293/248217 objects misplaced (0.118%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:16.562 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:16 smithi138 ceph-mon[192513]: failed to encode map e429 with expected crc 2023-10-01T09:36:16.563 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:16 smithi138 ceph-mon[192513]: failed to encode map e429 with expected crc 2023-10-01T09:36:16.563 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:16 smithi138 ceph-mon[192513]: pgmap v1197: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 931/248217 objects degraded (0.375%); 293/248217 objects misplaced (0.118%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:16.572 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:36:16.644 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:16.849 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:36:16.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:16.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:36:16.960 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:36:17.000 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:17.111 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:36:17.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:17.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:36:17.227 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:17.312 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:17.472 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:17.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:17.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:17.536 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:36:17.579 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:36:17.586 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:17.612 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:36:17.672 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:36:17.687 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:36:17.687 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:36:17.694 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:17.702 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:36:17.736 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:36:17.766 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:36:17.771 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:17.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:17.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:17.779 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:17 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2367268928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:17.780 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:17 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2367268928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:17.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:17 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2367268928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:17.824 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:36:17.829 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:36:17.829 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:36:17.829 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:36:17.830 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:36:17.830 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:36:17.830 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:36:17.830 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:36:17.830 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:36:17.880 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:36:17.957 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:18.091 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:36:18.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:18.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:36:18.216 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:36:18.321 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:18.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:18 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 931/248217 objects degraded (0.375%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:18.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:18 smithi138 ceph-mon[192513]: pgmap v1198: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 931/244875 objects degraded (0.380%); 293/244875 objects misplaced (0.120%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:18.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:18 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 931/248217 objects degraded (0.375%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:18.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:18 smithi138 ceph-mon[185886]: pgmap v1198: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 931/244875 objects degraded (0.380%); 293/244875 objects misplaced (0.120%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:18.656 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:36:18.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:18 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 931/248217 objects degraded (0.375%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:18.816 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:18 smithi190 ceph-mon[160435]: pgmap v1198: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 931/244875 objects degraded (0.380%); 293/244875 objects misplaced (0.120%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:19.197 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:36:19.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:19.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:36:19.301 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:36:19.346 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:19.434 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:36:19.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:19.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:36:19.473 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:36:19.539 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:36:19.628 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:19.835 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:36:19.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:19.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:36:19.904 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:19.947 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:20.020 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:20.229 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:20.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:20.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:20.334 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:20.393 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:20.515 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:20.536 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:20.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:20.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:20.643 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:36:20.697 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:20.789 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:36:20.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:20.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:36:20.915 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:36:20.940 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:20.987 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:36:20.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:20.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:36:21.094 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:36:21.160 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:21.195 INFO:teuthology.orchestra.run.smithi138.stdout:moved 44 / 267 (16.4794%) 2023-10-01T09:36:21.195 INFO:teuthology.orchestra.run.smithi138.stdout:avg 33.375 2023-10-01T09:36:21.196 INFO:teuthology.orchestra.run.smithi138.stdout:stddev 6.70704 -> 5.07291 (expected baseline 5.40399) 2023-10-01T09:36:21.196 INFO:teuthology.orchestra.run.smithi138.stdout:min osd.2 with 20 -> 28 pgs (0.599251 -> 0.838951 * mean) 2023-10-01T09:36:21.196 INFO:teuthology.orchestra.run.smithi138.stdout:max osd.1 with 43 -> 37 pgs (1.28839 -> 1.10861 * mean) 2023-10-01T09:36:21.196 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:36:21.196 INFO:teuthology.orchestra.run.smithi138.stdout:oload 110 2023-10-01T09:36:21.196 INFO:teuthology.orchestra.run.smithi138.stdout:max_change 3 2023-10-01T09:36:21.196 INFO:teuthology.orchestra.run.smithi138.stdout:max_change_osds 4 2023-10-01T09:36:21.196 INFO:teuthology.orchestra.run.smithi138.stdout:average_utilization 0.0278 2023-10-01T09:36:21.196 INFO:teuthology.orchestra.run.smithi138.stdout:overload_utilization 0.0306 2023-10-01T09:36:21.196 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7 weight 1.0000 -> 0.6651 2023-10-01T09:36:21.196 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1 weight 1.0000 -> 0.7370 2023-10-01T09:36:21.196 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3 weight 1.0000 -> 0.7847 2023-10-01T09:36:21.197 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5 weight 1.0000 -> 0.8196 2023-10-01T09:36:21.235 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:36:21.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:21.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:36:21.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:21 smithi190 ceph-mon[160435]: pgmap v1199: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 931/237291 objects degraded (0.392%); 177/237291 objects misplaced (0.075%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:21.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:21 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'48302','3':'51425','5':'53711','7':'43588'}"}]: dispatch 2023-10-01T09:36:21.340 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:21.368 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:21.423 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:36:21.457 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:21 smithi138 ceph-mon[185886]: pgmap v1199: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 931/237291 objects degraded (0.392%); 177/237291 objects misplaced (0.075%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:21.457 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:21 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'48302','3':'51425','5':'53711','7':'43588'}"}]: dispatch 2023-10-01T09:36:21.458 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:21 smithi138 ceph-mon[192513]: pgmap v1199: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 931/237291 objects degraded (0.392%); 177/237291 objects misplaced (0.075%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:36:21.458 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:21 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'48302','3':'51425','5':'53711','7':'43588'}"}]: dispatch 2023-10-01T09:36:21.989 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:21.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:21.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:22.094 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:36:22.119 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:22.161 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:36:22.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:22.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:36:22.266 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:36:22.291 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:22.340 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:36:22.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:22.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:36:22.448 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:36:22.471 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:22 smithi190 ceph-mon[160435]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:36:22.471 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'48302','3':'51425','5':'53711','7':'43588'}"}]': finished 2023-10-01T09:36:22.471 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:22 smithi190 ceph-mon[160435]: osdmap e430: 8 total, 8 up, 8 in 2023-10-01T09:36:22.471 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:22 smithi190 ceph-mon[160435]: failed to encode map e430 with expected crc 2023-10-01T09:36:22.471 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:22 smithi190 ceph-mon[160435]: failed to encode map e430 with expected crc 2023-10-01T09:36:22.471 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:22 smithi190 ceph-mon[160435]: pgmap v1201: 89 pgs: 1 unknown, 3 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 931/235479 objects degraded (0.395%); 177/235479 objects misplaced (0.075%); 1.8 MiB/s, 29 objects/s recovering 2023-10-01T09:36:22.473 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:22.515 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:36:22.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:22.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:36:22.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:22 smithi138 ceph-mon[192513]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:36:22.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'48302','3':'51425','5':'53711','7':'43588'}"}]': finished 2023-10-01T09:36:22.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:22 smithi138 ceph-mon[192513]: osdmap e430: 8 total, 8 up, 8 in 2023-10-01T09:36:22.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:22 smithi138 ceph-mon[192513]: failed to encode map e430 with expected crc 2023-10-01T09:36:22.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:22 smithi138 ceph-mon[192513]: failed to encode map e430 with expected crc 2023-10-01T09:36:22.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:22 smithi138 ceph-mon[192513]: pgmap v1201: 89 pgs: 1 unknown, 3 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 931/235479 objects degraded (0.395%); 177/235479 objects misplaced (0.075%); 1.8 MiB/s, 29 objects/s recovering 2023-10-01T09:36:22.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:22 smithi138 ceph-mon[185886]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T09:36:22.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'48302','3':'51425','5':'53711','7':'43588'}"}]': finished 2023-10-01T09:36:22.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:22 smithi138 ceph-mon[185886]: osdmap e430: 8 total, 8 up, 8 in 2023-10-01T09:36:22.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:22 smithi138 ceph-mon[185886]: failed to encode map e430 with expected crc 2023-10-01T09:36:22.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:22 smithi138 ceph-mon[185886]: failed to encode map e430 with expected crc 2023-10-01T09:36:22.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:22 smithi138 ceph-mon[185886]: pgmap v1201: 89 pgs: 1 unknown, 3 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 931/235479 objects degraded (0.395%); 177/235479 objects misplaced (0.075%); 1.8 MiB/s, 29 objects/s recovering 2023-10-01T09:36:22.613 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:22.620 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:36:22.657 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:22.803 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:22.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:22.906 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:22.935 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:23.270 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:36:23.349 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:23.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:23.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:23.460 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:36:23.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T09:36:23.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: osdmap e431: 8 total, 8 up, 8 in 2023-10-01T09:36:23.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 931/235479 objects degraded (0.395%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:23.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:23 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3776935129' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:36:23.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T09:36:23.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: osdmap e431: 8 total, 8 up, 8 in 2023-10-01T09:36:23.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 931/235479 objects degraded (0.395%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:23.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3776935129' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:36:23.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T09:36:23.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: osdmap e431: 8 total, 8 up, 8 in 2023-10-01T09:36:23.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: failed to encode map e431 with expected crc 2023-10-01T09:36:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 931/235479 objects degraded (0.395%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:23 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3776935129' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:36:23.575 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:24.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:36:24.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3776935129' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:36:24.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: osdmap e432: 8 total, 8 up, 8 in 2023-10-01T09:36:24.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 429'7772 2023-10-01T09:36:24.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 429'8062 2023-10-01T09:36:24.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 429'7967 2023-10-01T09:36:24.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: 9.d starting backfill to osd.6 from (0'0,0'0] MAX to 429'7967 2023-10-01T09:36:24.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:24 smithi190 ceph-mon[160435]: pgmap v1204: 89 pgs: 1 active+recovery_wait+remapped, 1 unknown, 5 peering, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 31936/232512 objects degraded (13.735%); 10556/232512 objects misplaced (4.540%); 1.2 MiB/s, 23 objects/s recovering 2023-10-01T09:36:24.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:36:24.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3776935129' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:36:24.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: osdmap e432: 8 total, 8 up, 8 in 2023-10-01T09:36:24.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 429'7772 2023-10-01T09:36:24.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 429'8062 2023-10-01T09:36:24.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 429'7967 2023-10-01T09:36:24.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: 9.d starting backfill to osd.6 from (0'0,0'0] MAX to 429'7967 2023-10-01T09:36:24.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[192513]: pgmap v1204: 89 pgs: 1 active+recovery_wait+remapped, 1 unknown, 5 peering, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 31936/232512 objects degraded (13.735%); 10556/232512 objects misplaced (4.540%); 1.2 MiB/s, 23 objects/s recovering 2023-10-01T09:36:24.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:36:24.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3776935129' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:36:24.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: osdmap e432: 8 total, 8 up, 8 in 2023-10-01T09:36:24.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 429'7772 2023-10-01T09:36:24.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 429'8062 2023-10-01T09:36:24.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 429'7967 2023-10-01T09:36:24.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: 9.d starting backfill to osd.6 from (0'0,0'0] MAX to 429'7967 2023-10-01T09:36:24.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: failed to encode map e432 with expected crc 2023-10-01T09:36:24.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:24 smithi138 ceph-mon[185886]: pgmap v1204: 89 pgs: 1 active+recovery_wait+remapped, 1 unknown, 5 peering, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 31936/232512 objects degraded (13.735%); 10556/232512 objects misplaced (4.540%); 1.2 MiB/s, 23 objects/s recovering 2023-10-01T09:36:24.857 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:36:24.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:24.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:36:24.966 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:36:25.021 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:25.087 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:36:25.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:25.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:36:25.124 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:36:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:36:24] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.33.4" 2023-10-01T09:36:25.196 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:36:25.255 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:25.324 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:36:25.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:25.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:36:25.429 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:25.561 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:25.671 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:25.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:25.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:25.785 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:25.812 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:25.855 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:25.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:25.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:25.959 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:36:25.985 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:26.028 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:36:26.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:26.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:36:26.133 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:36:26.167 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:26.234 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:36:26.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:26.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:36:26.339 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:26.454 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:26.464 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:36:26.464 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:36:26.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:26.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:36:26.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:26 smithi190 ceph-mon[160435]: pgmap v1205: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 unknown, 5 peering, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 44662/231117 objects degraded (19.324%); 20050/231117 objects misplaced (8.675%); 1.4 MiB/s, 26 objects/s recovering 2023-10-01T09:36:26.672 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:26.672 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:36:26.672 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:26 smithi138 ceph-mon[192513]: pgmap v1205: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 unknown, 5 peering, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 44662/231117 objects degraded (19.324%); 20050/231117 objects misplaced (8.675%); 1.4 MiB/s, 26 objects/s recovering 2023-10-01T09:36:26.672 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:26.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:26.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:36:26.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:26 smithi138 ceph-mon[185886]: pgmap v1205: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 unknown, 5 peering, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 44662/231117 objects degraded (19.324%); 20050/231117 objects misplaced (8.675%); 1.4 MiB/s, 26 objects/s recovering 2023-10-01T09:36:26.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:26.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:26.782 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:26.861 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:27.027 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:36:27.124 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 5, 6, 4, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:36:27.124 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:36:27.124 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-10-01T09:36:27.124 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:36:27.226 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:27.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:27.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:27.339 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:36:27.383 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:27 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:36:27.392 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:27 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:36:27.414 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:36:27.552 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:27.691 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:27.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:27.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:27.803 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:27.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:27 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:36:27.936 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:28.001 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:36:28.033 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:28.045 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:28.054 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:36:28.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:28.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:28.096 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:28.101 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:36:28.143 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:36:28.165 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:36:28.182 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:36:28.239 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:36:28.249 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:28.275 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:36:28.311 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:36:28.354 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:36:28.354 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:36:28.355 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:36:28.355 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:36:28.355 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:36:28.355 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:36:28.355 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:36:28.355 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:36:28.569 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:36:28.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:28.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:36:28.678 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:36:28.753 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[192513]: Health check update: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-01T09:36:28.753 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 44662/231117 objects degraded (19.324%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:28.753 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[192513]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.753 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[192513]: osdmap e433: 8 total, 8 up, 8 in 2023-10-01T09:36:28.754 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[192513]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.754 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[192513]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.754 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[192513]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.754 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[192513]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.754 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/397377840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:28.754 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[192513]: pgmap v1207: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 283 op/s; 44666/230370 objects degraded (19.389%); 24223/230370 objects misplaced (10.515%); 7.3 MiB/s, 111 objects/s recovering 2023-10-01T09:36:28.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[185886]: Health check update: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-01T09:36:28.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 44662/231117 objects degraded (19.324%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:28.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[185886]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[185886]: osdmap e433: 8 total, 8 up, 8 in 2023-10-01T09:36:28.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[185886]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.755 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[185886]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.756 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[185886]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.756 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[185886]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.756 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/397377840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:28.756 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:28 smithi138 ceph-mon[185886]: pgmap v1207: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 283 op/s; 44666/230370 objects degraded (19.389%); 24223/230370 objects misplaced (10.515%); 7.3 MiB/s, 111 objects/s recovering 2023-10-01T09:36:28.770 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:28 smithi190 ceph-mon[160435]: Health check update: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-01T09:36:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:28 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 44662/231117 objects degraded (19.324%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:28 smithi190 ceph-mon[160435]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:28 smithi190 ceph-mon[160435]: osdmap e433: 8 total, 8 up, 8 in 2023-10-01T09:36:28.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:28 smithi190 ceph-mon[160435]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:28 smithi190 ceph-mon[160435]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:28 smithi190 ceph-mon[160435]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:28 smithi190 ceph-mon[160435]: failed to encode map e433 with expected crc 2023-10-01T09:36:28.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/397377840' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:28.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:28 smithi190 ceph-mon[160435]: pgmap v1207: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 283 op/s; 44666/230370 objects degraded (19.389%); 24223/230370 objects misplaced (10.515%); 7.3 MiB/s, 111 objects/s recovering 2023-10-01T09:36:29.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:29 smithi190 ceph-mon[160435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-10-01T09:36:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:29 smithi190 ceph-mon[160435]: failed to encode map e434 with expected crc 2023-10-01T09:36:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:29 smithi190 ceph-mon[160435]: failed to encode map e434 with expected crc 2023-10-01T09:36:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:29 smithi190 ceph-mon[160435]: failed to encode map e434 with expected crc 2023-10-01T09:36:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:29 smithi190 ceph-mon[160435]: osdmap e434: 8 total, 8 up, 8 in 2023-10-01T09:36:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:29 smithi190 ceph-mon[160435]: failed to encode map e434 with expected crc 2023-10-01T09:36:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:29 smithi190 ceph-mon[160435]: failed to encode map e434 with expected crc 2023-10-01T09:36:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:29 smithi138 ceph-mon[192513]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-10-01T09:36:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:29 smithi138 ceph-mon[192513]: failed to encode map e434 with expected crc 2023-10-01T09:36:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:29 smithi138 ceph-mon[192513]: failed to encode map e434 with expected crc 2023-10-01T09:36:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:29 smithi138 ceph-mon[192513]: failed to encode map e434 with expected crc 2023-10-01T09:36:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:29 smithi138 ceph-mon[192513]: osdmap e434: 8 total, 8 up, 8 in 2023-10-01T09:36:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:29 smithi138 ceph-mon[192513]: failed to encode map e434 with expected crc 2023-10-01T09:36:29.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:29 smithi138 ceph-mon[192513]: failed to encode map e434 with expected crc 2023-10-01T09:36:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:29 smithi138 ceph-mon[185886]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-10-01T09:36:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:29 smithi138 ceph-mon[185886]: failed to encode map e434 with expected crc 2023-10-01T09:36:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:29 smithi138 ceph-mon[185886]: failed to encode map e434 with expected crc 2023-10-01T09:36:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:29 smithi138 ceph-mon[185886]: failed to encode map e434 with expected crc 2023-10-01T09:36:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:29 smithi138 ceph-mon[185886]: osdmap e434: 8 total, 8 up, 8 in 2023-10-01T09:36:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:29 smithi138 ceph-mon[185886]: failed to encode map e434 with expected crc 2023-10-01T09:36:29.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:29 smithi138 ceph-mon[185886]: failed to encode map e434 with expected crc 2023-10-01T09:36:30.169 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:31.052 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:30 smithi138 ceph-mon[185886]: failed to encode map e435 with expected crc 2023-10-01T09:36:31.052 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:30 smithi138 ceph-mon[185886]: failed to encode map e435 with expected crc 2023-10-01T09:36:31.052 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:30 smithi138 ceph-mon[185886]: osdmap e435: 8 total, 8 up, 8 in 2023-10-01T09:36:31.052 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:30 smithi138 ceph-mon[185886]: failed to encode map e435 with expected crc 2023-10-01T09:36:31.052 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:30 smithi138 ceph-mon[185886]: failed to encode map e435 with expected crc 2023-10-01T09:36:31.053 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:30 smithi138 ceph-mon[185886]: pgmap v1210: 89 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 353 op/s; 37863/226137 objects degraded (16.743%); 26864/226137 objects misplaced (11.880%); 11 MiB/s, 163 objects/s recovering 2023-10-01T09:36:31.053 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:30 smithi138 ceph-mon[192513]: failed to encode map e435 with expected crc 2023-10-01T09:36:31.053 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:30 smithi138 ceph-mon[192513]: failed to encode map e435 with expected crc 2023-10-01T09:36:31.053 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:30 smithi138 ceph-mon[192513]: osdmap e435: 8 total, 8 up, 8 in 2023-10-01T09:36:31.053 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:30 smithi138 ceph-mon[192513]: failed to encode map e435 with expected crc 2023-10-01T09:36:31.053 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:30 smithi138 ceph-mon[192513]: failed to encode map e435 with expected crc 2023-10-01T09:36:31.053 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:30 smithi138 ceph-mon[192513]: pgmap v1210: 89 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 353 op/s; 37863/226137 objects degraded (16.743%); 26864/226137 objects misplaced (11.880%); 11 MiB/s, 163 objects/s recovering 2023-10-01T09:36:31.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:30 smithi190 ceph-mon[160435]: failed to encode map e435 with expected crc 2023-10-01T09:36:31.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:30 smithi190 ceph-mon[160435]: failed to encode map e435 with expected crc 2023-10-01T09:36:31.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:30 smithi190 ceph-mon[160435]: osdmap e435: 8 total, 8 up, 8 in 2023-10-01T09:36:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:30 smithi190 ceph-mon[160435]: failed to encode map e435 with expected crc 2023-10-01T09:36:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:30 smithi190 ceph-mon[160435]: failed to encode map e435 with expected crc 2023-10-01T09:36:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:30 smithi190 ceph-mon[160435]: pgmap v1210: 89 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 353 op/s; 37863/226137 objects degraded (16.743%); 26864/226137 objects misplaced (11.880%); 11 MiB/s, 163 objects/s recovering 2023-10-01T09:36:31.060 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:31.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:31.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:31.167 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:36:31.211 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:31.274 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:36:31.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:31.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:36:31.385 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:36:31.426 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:31.482 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:36:31.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:31.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:36:31.590 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:36:31.628 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:31.693 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:36:31.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:31.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:36:31.799 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:31.896 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:32.227 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:36:32.251 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:31 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:32.252 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:31 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:32.252 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:31 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:32.252 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:31 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:32.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:31 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:32.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:31 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:32.677 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:32.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:32.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:32.792 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:32.912 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:32.987 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:32 smithi138 ceph-mon[185886]: pgmap v1211: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 37863/225279 objects degraded (16.807%); 25952/225279 objects misplaced (11.520%); 18 MiB/s, 280 objects/s recovering 2023-10-01T09:36:32.987 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:32 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 37863/226137 objects degraded (16.743%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:32.989 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:32 smithi138 ceph-mon[192513]: pgmap v1211: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 37863/225279 objects degraded (16.807%); 25952/225279 objects misplaced (11.520%); 18 MiB/s, 280 objects/s recovering 2023-10-01T09:36:32.989 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:32 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 37863/226137 objects degraded (16.743%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:33.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:32 smithi190 ceph-mon[160435]: pgmap v1211: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 37863/225279 objects degraded (16.807%); 25952/225279 objects misplaced (11.520%); 18 MiB/s, 280 objects/s recovering 2023-10-01T09:36:33.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:32 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 37863/226137 objects degraded (16.743%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:33.372 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:33.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:33.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:33.495 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:33.607 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:33.702 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:36:33.717 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:36:34.617 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:34.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:34.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:34.726 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:36:34.851 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:35.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:36:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:36:34] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.33.4" 2023-10-01T09:36:35.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:35 smithi138 ceph-mon[185886]: from='client.36213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:36:35.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:35 smithi138 ceph-mon[185886]: pgmap v1212: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 37859/223608 objects degraded (16.931%); 25797/223608 objects misplaced (11.537%); 12 MiB/s, 187 objects/s recovering 2023-10-01T09:36:35.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:35 smithi138 ceph-mon[192513]: from='client.36213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:36:35.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:35 smithi138 ceph-mon[192513]: pgmap v1212: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 37859/223608 objects degraded (16.931%); 25797/223608 objects misplaced (11.537%); 12 MiB/s, 187 objects/s recovering 2023-10-01T09:36:35.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:35 smithi190 ceph-mon[160435]: from='client.36213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:36:35.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:35 smithi190 ceph-mon[160435]: pgmap v1212: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 37859/223608 objects degraded (16.931%); 25797/223608 objects misplaced (11.537%); 12 MiB/s, 187 objects/s recovering 2023-10-01T09:36:35.733 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:35.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:35.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:35.852 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:36:35.881 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:35.898 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1211,"stamp":"2023-10-01T09:36:32.118237+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4924184523,"num_objects":75093,"num_object_clones":0,"num_object_copies":225279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":37863,"num_objects_misplaced":25952,"num_objects_unfound":0,"num_objects_dirty":75093,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":130807,"num_write_kb":6567590,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4806,"num_bytes_recovered":295801085,"num_keys_recovered":90,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125016,"ondisk_log_size":125016,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":284,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20046884,"kb_used_data":14748836,"kb_used_omap":50,"kb_used_meta":5297805,"kb_avail":729914332,"statfs":{"total":767960285184,"available":747432275968,"internally_reserved":0,"allocated":15102808064,"data_stored":15092045028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51738,"internal_metadata":5424952806},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-127533056,"num_objects":-1946,"num_object_clones":0,"num_object_copies":-5838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-6799,"num_objects_misplaced":5902,"num_objects_unfound":0,"num_objects_dirty":-1946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1946,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1681,"num_bytes_recovered":114295328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.003468"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":346,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.607638+0000","last_change":"2023-10-01T09:35:34.561455+0000","last_active":"2023-10-01T09:36:28.607638+0000","last_peered":"2023-10-01T09:36:28.607638+0000","last_clean":"2023-10-01T09:36:28.607638+0000","last_became_active":"2023-10-01T09:35:34.560978+0000","last_became_peered":"2023-10-01T09:35:34.560978+0000","last_unstale":"2023-10-01T09:36:28.607638+0000","last_undegraded":"2023-10-01T09:36:28.607638+0000","last_fullsized":"2023-10-01T09:36:28.607638+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:02:58.965198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":32,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.612391+0000","last_change":"2023-10-01T09:35:10.971375+0000","last_active":"2023-10-01T09:36:27.612391+0000","last_peered":"2023-10-01T09:36:27.612391+0000","last_clean":"2023-10-01T09:36:27.612391+0000","last_became_active":"2023-10-01T09:35:10.970710+0000","last_became_peered":"2023-10-01T09:35:10.970710+0000","last_unstale":"2023-10-01T09:36:27.612391+0000","last_undegraded":"2023-10-01T09:36:27.612391+0000","last_fullsized":"2023-10-01T09:36:27.612391+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:52:50.407889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":413,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.585434+0000","last_change":"2023-10-01T09:32:27.445629+0000","last_active":"2023-10-01T09:36:27.585434+0000","last_peered":"2023-10-01T09:36:27.585434+0000","last_clean":"2023-10-01T09:36:27.585434+0000","last_became_active":"2023-10-01T09:32:27.445365+0000","last_became_peered":"2023-10-01T09:32:27.445365+0000","last_unstale":"2023-10-01T09:36:27.585434+0000","last_undegraded":"2023-10-01T09:36:27.585434+0000","last_fullsized":"2023-10-01T09:36:27.585434+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:13:21.018961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":470,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.618455+0000","last_change":"2023-10-01T09:36:22.227558+0000","last_active":"2023-10-01T09:36:27.618455+0000","last_peered":"2023-10-01T09:36:27.618455+0000","last_clean":"2023-10-01T09:36:27.618455+0000","last_became_active":"2023-10-01T09:36:22.226550+0000","last_became_peered":"2023-10-01T09:36:22.226550+0000","last_unstale":"2023-10-01T09:36:27.618455+0000","last_undegraded":"2023-10-01T09:36:27.618455+0000","last_fullsized":"2023-10-01T09:36:27.618455+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:35:45.988649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":334,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.597876+0000","last_change":"2023-10-01T09:36:22.219906+0000","last_active":"2023-10-01T09:36:27.597876+0000","last_peered":"2023-10-01T09:36:27.597876+0000","last_clean":"2023-10-01T09:36:27.597876+0000","last_became_active":"2023-10-01T09:36:22.219587+0000","last_became_peered":"2023-10-01T09:36:22.219587+0000","last_unstale":"2023-10-01T09:36:27.597876+0000","last_undegraded":"2023-10-01T09:36:27.597876+0000","last_fullsized":"2023-10-01T09:36:27.597876+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:22:36.199122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":118,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.593526+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:36:28.593526+0000","last_peered":"2023-10-01T09:36:28.593526+0000","last_clean":"2023-10-01T09:36:28.593526+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:36:28.593526+0000","last_undegraded":"2023-10-01T09:36:28.593526+0000","last_fullsized":"2023-10-01T09:36:28.593526+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":488,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.594635+0000","last_change":"2023-10-01T09:33:11.508780+0000","last_active":"2023-10-01T09:36:27.594635+0000","last_peered":"2023-10-01T09:36:27.594635+0000","last_clean":"2023-10-01T09:36:27.594635+0000","last_became_active":"2023-10-01T09:33:11.508451+0000","last_became_peered":"2023-10-01T09:33:11.508451+0000","last_unstale":"2023-10-01T09:36:27.594635+0000","last_undegraded":"2023-10-01T09:36:27.594635+0000","last_fullsized":"2023-10-01T09:36:27.594635+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:55:02.323934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":436,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.594478+0000","last_change":"2023-10-01T09:30:39.395270+0000","last_active":"2023-10-01T09:36:27.594478+0000","last_peered":"2023-10-01T09:36:27.594478+0000","last_clean":"2023-10-01T09:36:27.594478+0000","last_became_active":"2023-10-01T09:30:39.394888+0000","last_became_peered":"2023-10-01T09:30:39.394888+0000","last_unstale":"2023-10-01T09:36:27.594478+0000","last_undegraded":"2023-10-01T09:36:27.594478+0000","last_fullsized":"2023-10-01T09:36:27.594478+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:35:40.846102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":129,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.592689+0000","last_change":"2023-10-01T09:36:22.254817+0000","last_active":"2023-10-01T09:36:28.592689+0000","last_peered":"2023-10-01T09:36:28.592689+0000","last_clean":"2023-10-01T09:36:28.592689+0000","last_became_active":"2023-10-01T09:36:22.254460+0000","last_became_peered":"2023-10-01T09:36:22.254460+0000","last_unstale":"2023-10-01T09:36:28.592689+0000","last_undegraded":"2023-10-01T09:36:28.592689+0000","last_fullsized":"2023-10-01T09:36:28.592689+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.592634+0000","last_change":"2023-10-01T09:36:22.286771+0000","last_active":"2023-10-01T09:36:28.592634+0000","last_peered":"2023-10-01T09:36:28.592634+0000","last_clean":"2023-10-01T09:36:28.592634+0000","last_became_active":"2023-10-01T09:36:22.286319+0000","last_became_peered":"2023-10-01T09:36:22.286319+0000","last_unstale":"2023-10-01T09:36:28.592634+0000","last_undegraded":"2023-10-01T09:36:28.592634+0000","last_fullsized":"2023-10-01T09:36:28.592634+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":20,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.594883+0000","last_change":"2023-10-01T09:36:22.255487+0000","last_active":"2023-10-01T09:36:27.594883+0000","last_peered":"2023-10-01T09:36:27.594883+0000","last_clean":"2023-10-01T09:36:27.594883+0000","last_became_active":"2023-10-01T09:36:22.220605+0000","last_became_peered":"2023-10-01T09:36:22.220605+0000","last_unstale":"2023-10-01T09:36:27.594883+0000","last_undegraded":"2023-10-01T09:36:27.594883+0000","last_fullsized":"2023-10-01T09:36:27.594883+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:15:28.488096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.593679+0000","last_change":"2023-10-01T09:36:22.223469+0000","last_active":"2023-10-01T09:36:27.593679+0000","last_peered":"2023-10-01T09:36:27.593679+0000","last_clean":"2023-10-01T09:36:27.593679+0000","last_became_active":"2023-10-01T09:36:22.222681+0000","last_became_peered":"2023-10-01T09:36:22.222681+0000","last_unstale":"2023-10-01T09:36:27.593679+0000","last_undegraded":"2023-10-01T09:36:27.593679+0000","last_fullsized":"2023-10-01T09:36:27.593679+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:19:28.367922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":462,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.599943+0000","last_change":"2023-10-01T09:32:27.498041+0000","last_active":"2023-10-01T09:36:28.599943+0000","last_peered":"2023-10-01T09:36:28.599943+0000","last_clean":"2023-10-01T09:36:28.599943+0000","last_became_active":"2023-10-01T09:32:27.454394+0000","last_became_peered":"2023-10-01T09:32:27.454394+0000","last_unstale":"2023-10-01T09:36:28.599943+0000","last_undegraded":"2023-10-01T09:36:28.599943+0000","last_fullsized":"2023-10-01T09:36:28.599943+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":275,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.586402+0000","last_change":"2023-10-01T09:35:23.610660+0000","last_active":"2023-10-01T09:36:27.586402+0000","last_peered":"2023-10-01T09:36:27.586402+0000","last_clean":"2023-10-01T09:36:27.586402+0000","last_became_active":"2023-10-01T09:35:23.610273+0000","last_became_peered":"2023-10-01T09:35:23.610273+0000","last_unstale":"2023-10-01T09:36:27.586402+0000","last_undegraded":"2023-10-01T09:36:27.586402+0000","last_fullsized":"2023-10-01T09:36:27.586402+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:43.996644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":538,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.600013+0000","last_change":"2023-10-01T09:35:34.640669+0000","last_active":"2023-10-01T09:36:28.600013+0000","last_peered":"2023-10-01T09:36:28.600013+0000","last_clean":"2023-10-01T09:36:28.600013+0000","last_became_active":"2023-10-01T09:35:34.552977+0000","last_became_peered":"2023-10-01T09:35:34.552977+0000","last_unstale":"2023-10-01T09:36:28.600013+0000","last_undegraded":"2023-10-01T09:36:28.600013+0000","last_fullsized":"2023-10-01T09:36:28.600013+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":306,"num_keys_recovered":90,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":254,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.599976+0000","last_change":"2023-10-01T09:36:22.223347+0000","last_active":"2023-10-01T09:36:28.599976+0000","last_peered":"2023-10-01T09:36:28.599976+0000","last_clean":"2023-10-01T09:36:28.599976+0000","last_became_active":"2023-10-01T09:36:22.222974+0000","last_became_peered":"2023-10-01T09:36:22.222974+0000","last_unstale":"2023-10-01T09:36:28.599976+0000","last_undegraded":"2023-10-01T09:36:28.599976+0000","last_fullsized":"2023-10-01T09:36:28.599976+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":435,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.593841+0000","last_change":"2023-10-01T09:36:22.223650+0000","last_active":"2023-10-01T09:36:27.593841+0000","last_peered":"2023-10-01T09:36:27.593841+0000","last_clean":"2023-10-01T09:36:27.593841+0000","last_became_active":"2023-10-01T09:36:22.223087+0000","last_became_peered":"2023-10-01T09:36:22.223087+0000","last_unstale":"2023-10-01T09:36:27.593841+0000","last_undegraded":"2023-10-01T09:36:27.593841+0000","last_fullsized":"2023-10-01T09:36:27.593841+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:52:21.962744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":235,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.600150+0000","last_change":"2023-10-01T09:36:22.222958+0000","last_active":"2023-10-01T09:36:28.600150+0000","last_peered":"2023-10-01T09:36:28.600150+0000","last_clean":"2023-10-01T09:36:28.600150+0000","last_became_active":"2023-10-01T09:36:22.222585+0000","last_became_peered":"2023-10-01T09:36:22.222585+0000","last_unstale":"2023-10-01T09:36:28.600150+0000","last_undegraded":"2023-10-01T09:36:28.600150+0000","last_fullsized":"2023-10-01T09:36:28.600150+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":446,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.599746+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:36:28.599746+0000","last_peered":"2023-10-01T09:36:28.599746+0000","last_clean":"2023-10-01T09:36:28.599746+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:36:28.599746+0000","last_undegraded":"2023-10-01T09:36:28.599746+0000","last_fullsized":"2023-10-01T09:36:28.599746+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":417,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.618528+0000","last_change":"2023-10-01T09:36:22.218977+0000","last_active":"2023-10-01T09:36:27.618528+0000","last_peered":"2023-10-01T09:36:27.618528+0000","last_clean":"2023-10-01T09:36:27.618528+0000","last_became_active":"2023-10-01T09:36:22.218653+0000","last_became_peered":"2023-10-01T09:36:22.218653+0000","last_unstale":"2023-10-01T09:36:27.618528+0000","last_undegraded":"2023-10-01T09:36:27.618528+0000","last_fullsized":"2023-10-01T09:36:27.618528+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:28:14.311288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":252,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.597723+0000","last_change":"2023-10-01T09:36:22.385457+0000","last_active":"2023-10-01T09:36:27.597723+0000","last_peered":"2023-10-01T09:36:27.597723+0000","last_clean":"2023-10-01T09:36:27.597723+0000","last_became_active":"2023-10-01T09:36:22.222584+0000","last_became_peered":"2023-10-01T09:36:22.222584+0000","last_unstale":"2023-10-01T09:36:27.597723+0000","last_undegraded":"2023-10-01T09:36:27.597723+0000","last_fullsized":"2023-10-01T09:36:27.597723+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:57:37.538709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":234,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.612686+0000","last_change":"2023-10-01T09:36:22.223054+0000","last_active":"2023-10-01T09:36:27.612686+0000","last_peered":"2023-10-01T09:36:27.612686+0000","last_clean":"2023-10-01T09:36:27.612686+0000","last_became_active":"2023-10-01T09:36:22.222776+0000","last_became_peered":"2023-10-01T09:36:22.222776+0000","last_unstale":"2023-10-01T09:36:27.612686+0000","last_undegraded":"2023-10-01T09:36:27.612686+0000","last_fullsized":"2023-10-01T09:36:27.612686+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:09:23.536588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"434'8165","reported_seq":8271,"reported_epoch":434,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:36:29.033236+0000","last_change":"2023-10-01T09:36:27.702239+0000","last_active":"2023-10-01T09:36:29.033236+0000","last_peered":"2023-10-01T09:36:29.033236+0000","last_clean":"2023-10-01T09:36:21.212087+0000","last_became_active":"2023-10-01T09:36:23.799500+0000","last_became_peered":"2023-10-01T09:36:23.799500+0000","last_unstale":"2023-10-01T09:36:29.033236+0000","last_undegraded":"2023-10-01T09:36:23.263244+0000","last_fullsized":"2023-10-01T09:36:23.212896+0000","mapping_epoch":431,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":8165,"log_dups_size":0,"ondisk_log_size":8165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:09:34.039273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310968320,"num_objects":4745,"num_object_clones":0,"num_object_copies":14235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":102,"num_objects_misplaced":4733,"num_objects_unfound":0,"num_objects_dirty":4745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8165,"num_write_kb":413120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":1703936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":-90},{"shards":"0,5,7","objects":4733}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":107,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.594504+0000","last_change":"2023-10-01T09:36:22.387631+0000","last_active":"2023-10-01T09:36:27.594504+0000","last_peered":"2023-10-01T09:36:27.594504+0000","last_clean":"2023-10-01T09:36:27.594504+0000","last_became_active":"2023-10-01T09:36:22.237417+0000","last_became_peered":"2023-10-01T09:36:22.237417+0000","last_unstale":"2023-10-01T09:36:27.594504+0000","last_undegraded":"2023-10-01T09:36:27.594504+0000","last_fullsized":"2023-10-01T09:36:27.594504+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:06:04.026880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":146,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.592262+0000","last_change":"2023-10-01T09:36:22.236445+0000","last_active":"2023-10-01T09:36:28.592262+0000","last_peered":"2023-10-01T09:36:28.592262+0000","last_clean":"2023-10-01T09:36:28.592262+0000","last_became_active":"2023-10-01T09:36:22.236081+0000","last_became_peered":"2023-10-01T09:36:22.236081+0000","last_unstale":"2023-10-01T09:36:28.592262+0000","last_undegraded":"2023-10-01T09:36:28.592262+0000","last_fullsized":"2023-10-01T09:36:28.592262+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"434'8153","reported_seq":8197,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.895394+0000","last_change":"2023-10-01T09:34:06.168002+0000","last_active":"2023-10-01T09:36:28.895394+0000","last_peered":"2023-10-01T09:36:28.895394+0000","last_clean":"2023-10-01T09:36:28.895394+0000","last_became_active":"2023-10-01T09:34:06.167679+0000","last_became_peered":"2023-10-01T09:34:06.167679+0000","last_unstale":"2023-10-01T09:36:28.895394+0000","last_undegraded":"2023-10-01T09:36:28.895394+0000","last_fullsized":"2023-10-01T09:36:28.895394+0000","mapping_epoch":400,"log_start":"405'1200","ondisk_log_start":"405'1200","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6953,"log_dups_size":0,"ondisk_log_size":6953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:21:42.118174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315424768,"num_objects":4813,"num_object_clones":0,"num_object_copies":14439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8153,"num_write_kb":414912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":429,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.600195+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:36:28.600195+0000","last_peered":"2023-10-01T09:36:28.600195+0000","last_clean":"2023-10-01T09:36:28.600195+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:36:28.600195+0000","last_undegraded":"2023-10-01T09:36:28.600195+0000","last_fullsized":"2023-10-01T09:36:28.600195+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":477,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.585346+0000","last_change":"2023-10-01T09:36:22.223799+0000","last_active":"2023-10-01T09:36:27.585346+0000","last_peered":"2023-10-01T09:36:27.585346+0000","last_clean":"2023-10-01T09:36:27.585346+0000","last_became_active":"2023-10-01T09:36:22.223546+0000","last_became_peered":"2023-10-01T09:36:22.223546+0000","last_unstale":"2023-10-01T09:36:27.585346+0000","last_undegraded":"2023-10-01T09:36:27.585346+0000","last_fullsized":"2023-10-01T09:36:27.585346+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:54:07.013864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"435'8100","reported_seq":8553,"reported_epoch":435,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:36:29.763970+0000","last_change":"2023-10-01T09:36:23.821080+0000","last_active":"2023-10-01T09:36:29.763970+0000","last_peered":"2023-10-01T09:36:29.763970+0000","last_clean":"2023-10-01T09:36:21.192430+0000","last_became_active":"2023-10-01T09:36:23.791409+0000","last_became_peered":"2023-10-01T09:36:23.791409+0000","last_unstale":"2023-10-01T09:36:29.763970+0000","last_undegraded":"2023-10-01T09:36:23.398631+0000","last_fullsized":"2023-10-01T09:36:23.217545+0000","mapping_epoch":431,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":8100,"log_dups_size":0,"ondisk_log_size":8100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305659904,"num_objects":4664,"num_object_clones":0,"num_object_copies":13992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12496,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8100,"num_write_kb":408448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":134,"num_bytes_recovered":8716288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,1],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":4664}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":515,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.600042+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:36:28.600042+0000","last_peered":"2023-10-01T09:36:28.600042+0000","last_clean":"2023-10-01T09:36:28.600042+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:36:28.600042+0000","last_undegraded":"2023-10-01T09:36:28.600042+0000","last_fullsized":"2023-10-01T09:36:28.600042+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":297,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.672889+0000","last_change":"2023-10-01T09:36:22.247103+0000","last_active":"2023-10-01T09:36:27.672889+0000","last_peered":"2023-10-01T09:36:27.672889+0000","last_clean":"2023-10-01T09:36:27.672889+0000","last_became_active":"2023-10-01T09:36:22.246793+0000","last_became_peered":"2023-10-01T09:36:22.246793+0000","last_unstale":"2023-10-01T09:36:27.672889+0000","last_undegraded":"2023-10-01T09:36:27.672889+0000","last_fullsized":"2023-10-01T09:36:27.672889+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:25:05.754730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"435'8058","reported_seq":10883,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-01T09:36:29.669219+0000","last_change":"2023-10-01T09:36:28.672838+0000","last_active":"2023-10-01T09:36:29.669219+0000","last_peered":"2023-10-01T09:36:29.669219+0000","last_clean":"2023-10-01T09:36:29.669219+0000","last_became_active":"2023-10-01T09:36:28.606609+0000","last_became_peered":"2023-10-01T09:36:28.606609+0000","last_unstale":"2023-10-01T09:36:29.669219+0000","last_undegraded":"2023-10-01T09:36:29.669219+0000","last_fullsized":"2023-10-01T09:36:29.669219+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":8058,"log_dups_size":0,"ondisk_log_size":8058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300154880,"num_objects":4580,"num_object_clones":0,"num_object_copies":13740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8058,"num_write_kb":404416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1377,"num_bytes_recovered":90243072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":137,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.592874+0000","last_change":"2023-10-01T09:33:11.656845+0000","last_active":"2023-10-01T09:36:28.592874+0000","last_peered":"2023-10-01T09:36:28.592874+0000","last_clean":"2023-10-01T09:36:28.592874+0000","last_became_active":"2023-10-01T09:33:11.655365+0000","last_became_peered":"2023-10-01T09:33:11.655365+0000","last_unstale":"2023-10-01T09:36:28.592874+0000","last_undegraded":"2023-10-01T09:36:28.592874+0000","last_fullsized":"2023-10-01T09:36:28.592874+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":409,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.594007+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:36:27.594007+0000","last_peered":"2023-10-01T09:36:27.594007+0000","last_clean":"2023-10-01T09:36:27.594007+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:36:27.594007+0000","last_undegraded":"2023-10-01T09:36:27.594007+0000","last_fullsized":"2023-10-01T09:36:27.594007+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"435'7944","reported_seq":10672,"reported_epoch":435,"state":"active+remapped+backfilling","last_fresh":"2023-10-01T09:36:31.967246+0000","last_change":"2023-10-01T09:36:23.643697+0000","last_active":"2023-10-01T09:36:31.967246+0000","last_peered":"2023-10-01T09:36:31.967246+0000","last_clean":"2023-10-01T09:36:21.184462+0000","last_became_active":"2023-10-01T09:36:23.621395+0000","last_became_peered":"2023-10-01T09:36:23.621395+0000","last_unstale":"2023-10-01T09:36:31.967246+0000","last_undegraded":"2023-10-01T09:36:31.967246+0000","last_fullsized":"2023-10-01T09:36:31.967246+0000","mapping_epoch":431,"log_start":"404'200","ondisk_log_start":"404'200","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7744,"log_dups_size":0,"ondisk_log_size":7744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:16:18.125086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299630592,"num_objects":4572,"num_object_clones":0,"num_object_copies":13716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3261,"num_objects_unfound":0,"num_objects_dirty":4572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7944,"num_write_kb":400512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1339,"num_bytes_recovered":87752704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,7],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6,7","objects":4572}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":136,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.594087+0000","last_change":"2023-10-01T09:32:27.489306+0000","last_active":"2023-10-01T09:36:27.594087+0000","last_peered":"2023-10-01T09:36:27.594087+0000","last_clean":"2023-10-01T09:36:27.594087+0000","last_became_active":"2023-10-01T09:32:27.449850+0000","last_became_peered":"2023-10-01T09:32:27.449850+0000","last_unstale":"2023-10-01T09:36:27.594087+0000","last_undegraded":"2023-10-01T09:36:27.594087+0000","last_fullsized":"2023-10-01T09:36:27.594087+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:26:15.712006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":81,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-01T09:36:22.217053+0000","last_change":"2023-10-01T09:32:27.449970+0000","last_active":"2023-10-01T09:36:22.217053+0000","last_peered":"2023-10-01T09:36:22.217053+0000","last_clean":"2023-10-01T09:36:22.217053+0000","last_became_active":"2023-10-01T09:32:27.449665+0000","last_became_peered":"2023-10-01T09:32:27.449665+0000","last_unstale":"2023-10-01T09:36:22.217053+0000","last_undegraded":"2023-10-01T09:36:22.217053+0000","last_fullsized":"2023-10-01T09:36:22.217053+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"434'8097","reported_seq":10028,"reported_epoch":434,"state":"active+recovering+remapped","last_fresh":"2023-10-01T09:36:28.879522+0000","last_change":"2023-10-01T09:36:27.674693+0000","last_active":"2023-10-01T09:36:28.879522+0000","last_peered":"2023-10-01T09:36:28.879522+0000","last_clean":"2023-10-01T09:35:09.959309+0000","last_became_active":"2023-10-01T09:36:23.678618+0000","last_became_peered":"2023-10-01T09:36:23.678618+0000","last_unstale":"2023-10-01T09:36:28.879522+0000","last_undegraded":"2023-10-01T09:36:28.879522+0000","last_fullsized":"2023-10-01T09:36:28.879522+0000","mapping_epoch":431,"log_start":"405'1100","ondisk_log_start":"405'1100","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6997,"log_dups_size":0,"ondisk_log_size":6997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:32:05.325065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301137920,"num_objects":4595,"num_object_clones":0,"num_object_copies":13785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3774,"num_objects_unfound":0,"num_objects_dirty":4595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8097,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":929,"num_bytes_recovered":60817408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[1,6,2],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":821}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":347,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-01T09:36:22.228498+0000","last_change":"2023-10-01T09:36:22.226949+0000","last_active":"2023-10-01T09:36:22.228498+0000","last_peered":"2023-10-01T09:36:22.228498+0000","last_clean":"2023-10-01T09:36:22.228498+0000","last_became_active":"2023-10-01T09:36:22.226624+0000","last_became_peered":"2023-10-01T09:36:22.226624+0000","last_unstale":"2023-10-01T09:36:22.228498+0000","last_undegraded":"2023-10-01T09:36:22.228498+0000","last_fullsized":"2023-10-01T09:36:22.228498+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":480,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.612531+0000","last_change":"2023-10-01T09:35:34.558802+0000","last_active":"2023-10-01T09:36:27.612531+0000","last_peered":"2023-10-01T09:36:27.612531+0000","last_clean":"2023-10-01T09:36:27.612531+0000","last_became_active":"2023-10-01T09:35:34.558511+0000","last_became_peered":"2023-10-01T09:35:34.558511+0000","last_unstale":"2023-10-01T09:36:27.612531+0000","last_undegraded":"2023-10-01T09:36:27.612531+0000","last_fullsized":"2023-10-01T09:36:27.612531+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:40:52.864568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"435'8147","reported_seq":8189,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-01T09:36:31.808559+0000","last_change":"2023-10-01T09:34:06.128656+0000","last_active":"2023-10-01T09:36:31.808559+0000","last_peered":"2023-10-01T09:36:31.808559+0000","last_clean":"2023-10-01T09:36:31.808559+0000","last_became_active":"2023-10-01T09:34:06.128364+0000","last_became_peered":"2023-10-01T09:34:06.128364+0000","last_unstale":"2023-10-01T09:36:31.808559+0000","last_undegraded":"2023-10-01T09:36:31.808559+0000","last_fullsized":"2023-10-01T09:36:31.808559+0000","mapping_epoch":400,"log_start":"405'400","ondisk_log_start":"405'400","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7747,"log_dups_size":0,"ondisk_log_size":7747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303104000,"num_objects":4625,"num_object_clones":0,"num_object_copies":13875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8147,"num_write_kb":408704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":325,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.599806+0000","last_change":"2023-10-01T09:35:34.551196+0000","last_active":"2023-10-01T09:36:28.599806+0000","last_peered":"2023-10-01T09:36:28.599806+0000","last_clean":"2023-10-01T09:36:28.599806+0000","last_became_active":"2023-10-01T09:35:34.550803+0000","last_became_peered":"2023-10-01T09:35:34.550803+0000","last_unstale":"2023-10-01T09:36:28.599806+0000","last_undegraded":"2023-10-01T09:36:28.599806+0000","last_fullsized":"2023-10-01T09:36:28.599806+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":153,"reported_epoch":431,"state":"active+clean","last_fresh":"2023-10-01T09:36:22.221994+0000","last_change":"2023-10-01T09:36:22.220899+0000","last_active":"2023-10-01T09:36:22.221994+0000","last_peered":"2023-10-01T09:36:22.221994+0000","last_clean":"2023-10-01T09:36:22.221994+0000","last_became_active":"2023-10-01T09:36:22.220537+0000","last_became_peered":"2023-10-01T09:36:22.220537+0000","last_unstale":"2023-10-01T09:36:22.221994+0000","last_undegraded":"2023-10-01T09:36:22.221994+0000","last_fullsized":"2023-10-01T09:36:22.221994+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"434'8142","reported_seq":9160,"reported_epoch":434,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:36:29.028440+0000","last_change":"2023-10-01T09:36:23.771443+0000","last_active":"2023-10-01T09:36:29.028440+0000","last_peered":"2023-10-01T09:36:29.028440+0000","last_clean":"2023-10-01T09:35:09.934020+0000","last_became_active":"2023-10-01T09:36:23.749995+0000","last_became_peered":"2023-10-01T09:36:23.749995+0000","last_unstale":"2023-10-01T09:36:29.028440+0000","last_undegraded":"2023-10-01T09:36:23.566811+0000","last_fullsized":"2023-10-01T09:36:23.220185+0000","mapping_epoch":431,"log_start":"405'1200","ondisk_log_start":"405'1200","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6942,"log_dups_size":0,"ondisk_log_size":6942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:23:09.158739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312606720,"num_objects":4770,"num_object_clones":0,"num_object_copies":14310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8142,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":476,"num_bytes_recovered":30998528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2","objects":-117},{"shards":"1,2,3","objects":3852},{"shards":"1,3,7","objects":918}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":284,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.594360+0000","last_change":"2023-10-01T09:36:22.223384+0000","last_active":"2023-10-01T09:36:27.594360+0000","last_peered":"2023-10-01T09:36:27.594360+0000","last_clean":"2023-10-01T09:36:27.594360+0000","last_became_active":"2023-10-01T09:36:22.223098+0000","last_became_peered":"2023-10-01T09:36:22.223098+0000","last_unstale":"2023-10-01T09:36:27.594360+0000","last_undegraded":"2023-10-01T09:36:27.594360+0000","last_fullsized":"2023-10-01T09:36:27.594360+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:15:40.489055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":322,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.612758+0000","last_change":"2023-10-01T09:32:27.445484+0000","last_active":"2023-10-01T09:36:27.612758+0000","last_peered":"2023-10-01T09:36:27.612758+0000","last_clean":"2023-10-01T09:36:27.612758+0000","last_became_active":"2023-10-01T09:32:27.445198+0000","last_became_peered":"2023-10-01T09:32:27.445198+0000","last_unstale":"2023-10-01T09:36:27.612758+0000","last_undegraded":"2023-10-01T09:36:27.612758+0000","last_fullsized":"2023-10-01T09:36:27.612758+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:29:19.880130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"435'8116","reported_seq":8159,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-01T09:36:29.637548+0000","last_change":"2023-10-01T09:34:06.141056+0000","last_active":"2023-10-01T09:36:29.637548+0000","last_peered":"2023-10-01T09:36:29.637548+0000","last_clean":"2023-10-01T09:36:29.637548+0000","last_became_active":"2023-10-01T09:34:06.140163+0000","last_became_peered":"2023-10-01T09:34:06.140163+0000","last_unstale":"2023-10-01T09:36:29.637548+0000","last_undegraded":"2023-10-01T09:36:29.637548+0000","last_fullsized":"2023-10-01T09:36:29.637548+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":8116,"log_dups_size":0,"ondisk_log_size":8116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309592064,"num_objects":4724,"num_object_clones":0,"num_object_copies":14172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8116,"num_write_kb":410880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1257,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.585597+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:36:27.585597+0000","last_peered":"2023-10-01T09:36:27.585597+0000","last_clean":"2023-10-01T09:36:27.585597+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:36:27.585597+0000","last_undegraded":"2023-10-01T09:36:27.585597+0000","last_fullsized":"2023-10-01T09:36:27.585597+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":138,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.593287+0000","last_change":"2023-10-01T09:35:34.552480+0000","last_active":"2023-10-01T09:36:28.593287+0000","last_peered":"2023-10-01T09:36:28.593287+0000","last_clean":"2023-10-01T09:36:28.593287+0000","last_became_active":"2023-10-01T09:35:34.552189+0000","last_became_peered":"2023-10-01T09:35:34.552189+0000","last_unstale":"2023-10-01T09:36:28.593287+0000","last_undegraded":"2023-10-01T09:36:28.593287+0000","last_fullsized":"2023-10-01T09:36:28.593287+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":390,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.592742+0000","last_change":"2023-10-01T09:36:22.236067+0000","last_active":"2023-10-01T09:36:28.592742+0000","last_peered":"2023-10-01T09:36:28.592742+0000","last_clean":"2023-10-01T09:36:28.592742+0000","last_became_active":"2023-10-01T09:36:22.235732+0000","last_became_peered":"2023-10-01T09:36:22.235732+0000","last_unstale":"2023-10-01T09:36:28.592742+0000","last_undegraded":"2023-10-01T09:36:28.592742+0000","last_fullsized":"2023-10-01T09:36:28.592742+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"434'8178","reported_seq":8222,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.896420+0000","last_change":"2023-10-01T09:34:06.140052+0000","last_active":"2023-10-01T09:36:28.896420+0000","last_peered":"2023-10-01T09:36:28.896420+0000","last_clean":"2023-10-01T09:36:28.896420+0000","last_became_active":"2023-10-01T09:34:06.139672+0000","last_became_peered":"2023-10-01T09:34:06.139672+0000","last_unstale":"2023-10-01T09:36:28.896420+0000","last_undegraded":"2023-10-01T09:36:28.896420+0000","last_fullsized":"2023-10-01T09:36:28.896420+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":8178,"log_dups_size":0,"ondisk_log_size":8178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:39:55.000830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306708480,"num_objects":4680,"num_object_clones":0,"num_object_copies":14040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8178,"num_write_kb":411456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1100,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.592916+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:36:28.592916+0000","last_peered":"2023-10-01T09:36:28.592916+0000","last_clean":"2023-10-01T09:36:28.592916+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:36:28.592916+0000","last_undegraded":"2023-10-01T09:36:28.592916+0000","last_fullsized":"2023-10-01T09:36:28.592916+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":409,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.594328+0000","last_change":"2023-10-01T09:32:27.451746+0000","last_active":"2023-10-01T09:36:27.594328+0000","last_peered":"2023-10-01T09:36:27.594328+0000","last_clean":"2023-10-01T09:36:27.594328+0000","last_became_active":"2023-10-01T09:32:27.451069+0000","last_became_peered":"2023-10-01T09:32:27.451069+0000","last_unstale":"2023-10-01T09:36:27.594328+0000","last_undegraded":"2023-10-01T09:36:27.594328+0000","last_fullsized":"2023-10-01T09:36:27.594328+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:50.586397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":96,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.592636+0000","last_change":"2023-10-01T09:35:34.552611+0000","last_active":"2023-10-01T09:36:28.592636+0000","last_peered":"2023-10-01T09:36:28.592636+0000","last_clean":"2023-10-01T09:36:28.592636+0000","last_became_active":"2023-10-01T09:35:34.552295+0000","last_became_peered":"2023-10-01T09:35:34.552295+0000","last_unstale":"2023-10-01T09:36:28.592636+0000","last_undegraded":"2023-10-01T09:36:28.592636+0000","last_fullsized":"2023-10-01T09:36:28.592636+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"434'7962","reported_seq":8011,"reported_epoch":434,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:36:28.722123+0000","last_change":"2023-10-01T09:36:23.747246+0000","last_active":"2023-10-01T09:36:28.722123+0000","last_peered":"2023-10-01T09:36:28.722123+0000","last_clean":"2023-10-01T09:36:21.153294+0000","last_became_active":"2023-10-01T09:36:23.691564+0000","last_became_peered":"2023-10-01T09:36:23.691564+0000","last_unstale":"2023-10-01T09:36:28.722123+0000","last_undegraded":"2023-10-01T09:36:23.314548+0000","last_fullsized":"2023-10-01T09:36:23.222285+0000","mapping_epoch":431,"log_start":"404'255","ondisk_log_start":"404'255","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7707,"log_dups_size":0,"ondisk_log_size":7707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:22:50.742097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301072384,"num_objects":4594,"num_object_clones":0,"num_object_copies":13782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6278,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7962,"num_write_kb":401792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":4594},{"shards":"1,7","objects":-115}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":689,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.593610+0000","last_change":"2023-10-01T09:35:48.089027+0000","last_active":"2023-10-01T09:36:27.593610+0000","last_peered":"2023-10-01T09:36:27.593610+0000","last_clean":"2023-10-01T09:36:27.593610+0000","last_became_active":"2023-10-01T09:35:34.556960+0000","last_became_peered":"2023-10-01T09:35:34.556960+0000","last_unstale":"2023-10-01T09:36:27.593610+0000","last_undegraded":"2023-10-01T09:36:27.593610+0000","last_fullsized":"2023-10-01T09:36:27.593610+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:21:15.204374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":484,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.594421+0000","last_change":"2023-10-01T09:35:34.562777+0000","last_active":"2023-10-01T09:36:27.594421+0000","last_peered":"2023-10-01T09:36:27.594421+0000","last_clean":"2023-10-01T09:36:27.594421+0000","last_became_active":"2023-10-01T09:35:34.562492+0000","last_became_peered":"2023-10-01T09:35:34.562492+0000","last_unstale":"2023-10-01T09:36:27.594421+0000","last_undegraded":"2023-10-01T09:36:27.594421+0000","last_fullsized":"2023-10-01T09:36:27.594421+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:26:15.751778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":286,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.593657+0000","last_change":"2023-10-01T09:36:22.384073+0000","last_active":"2023-10-01T09:36:28.593657+0000","last_peered":"2023-10-01T09:36:28.593657+0000","last_clean":"2023-10-01T09:36:28.593657+0000","last_became_active":"2023-10-01T09:36:22.383799+0000","last_became_peered":"2023-10-01T09:36:22.383799+0000","last_unstale":"2023-10-01T09:36:28.593657+0000","last_undegraded":"2023-10-01T09:36:28.593657+0000","last_fullsized":"2023-10-01T09:36:28.593657+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"434'8021","reported_seq":8420,"reported_epoch":434,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:36:29.050227+0000","last_change":"2023-10-01T09:36:23.839061+0000","last_active":"2023-10-01T09:36:29.050227+0000","last_peered":"2023-10-01T09:36:29.050227+0000","last_clean":"2023-10-01T09:36:21.170373+0000","last_became_active":"2023-10-01T09:36:23.812646+0000","last_became_peered":"2023-10-01T09:36:23.812646+0000","last_unstale":"2023-10-01T09:36:29.050227+0000","last_undegraded":"2023-10-01T09:36:23.487783+0000","last_fullsized":"2023-10-01T09:36:23.211397+0000","mapping_epoch":431,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":8021,"log_dups_size":0,"ondisk_log_size":8021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:34:58.841399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310050816,"num_objects":4731,"num_object_clones":0,"num_object_copies":14193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12517,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8021,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":7471104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":4728},{"shards":"0,3,5,7","objects":3}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1413,"reported_epoch":434,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:36:28.592912+0000","last_change":"2023-10-01T09:36:23.230289+0000","last_active":"2023-10-01T09:36:28.592912+0000","last_peered":"2023-10-01T09:36:28.592912+0000","last_clean":"2023-10-01T09:35:27.946455+0000","last_became_active":"2023-10-01T09:36:23.229490+0000","last_became_peered":"2023-10-01T09:36:23.229490+0000","last_unstale":"2023-10-01T09:36:28.592912+0000","last_undegraded":"2023-10-01T09:36:23.217725+0000","last_fullsized":"2023-10-01T09:36:28.592912+0000","mapping_epoch":431,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":445,"reported_epoch":434,"state":"peering","last_fresh":"2023-10-01T09:36:28.595187+0000","last_change":"2023-10-01T09:36:28.595365+0000","last_active":"2023-10-01T09:36:28.595187+0000","last_peered":"2023-10-01T09:36:27.668047+0000","last_clean":"2023-10-01T09:23:05.577819+0000","last_became_active":"2023-10-01T09:36:23.496350+0000","last_became_peered":"2023-10-01T09:36:23.496350+0000","last_unstale":"2023-10-01T09:36:28.595187+0000","last_undegraded":"2023-10-01T09:36:28.595187+0000","last_fullsized":"2023-10-01T09:36:28.595187+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:36:45.002669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,6,7],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":207,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.592468+0000","last_change":"2023-10-01T09:33:11.663444+0000","last_active":"2023-10-01T09:36:28.592468+0000","last_peered":"2023-10-01T09:36:28.592468+0000","last_clean":"2023-10-01T09:36:28.592468+0000","last_became_active":"2023-10-01T09:33:11.660310+0000","last_became_peered":"2023-10-01T09:33:11.660310+0000","last_unstale":"2023-10-01T09:36:28.592468+0000","last_undegraded":"2023-10-01T09:36:28.592468+0000","last_fullsized":"2023-10-01T09:36:28.592468+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":349,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.600139+0000","last_change":"2023-10-01T09:32:27.453291+0000","last_active":"2023-10-01T09:36:28.600139+0000","last_peered":"2023-10-01T09:36:28.600139+0000","last_clean":"2023-10-01T09:36:28.600139+0000","last_became_active":"2023-10-01T09:32:27.453026+0000","last_became_peered":"2023-10-01T09:32:27.453026+0000","last_unstale":"2023-10-01T09:36:28.600139+0000","last_undegraded":"2023-10-01T09:36:28.600139+0000","last_fullsized":"2023-10-01T09:36:28.600139+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"434'8023","reported_seq":8067,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.699794+0000","last_change":"2023-10-01T09:34:06.140844+0000","last_active":"2023-10-01T09:36:28.699794+0000","last_peered":"2023-10-01T09:36:28.699794+0000","last_clean":"2023-10-01T09:36:28.699794+0000","last_became_active":"2023-10-01T09:34:06.139812+0000","last_became_peered":"2023-10-01T09:34:06.139812+0000","last_unstale":"2023-10-01T09:36:28.699794+0000","last_undegraded":"2023-10-01T09:36:28.699794+0000","last_fullsized":"2023-10-01T09:36:28.699794+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":8023,"log_dups_size":0,"ondisk_log_size":8023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:31:05.105491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310312960,"num_objects":4735,"num_object_clones":0,"num_object_copies":14205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8023,"num_write_kb":408256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":561,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.600493+0000","last_change":"2023-10-01T09:36:22.405817+0000","last_active":"2023-10-01T09:36:28.600493+0000","last_peered":"2023-10-01T09:36:28.600493+0000","last_clean":"2023-10-01T09:36:28.600493+0000","last_became_active":"2023-10-01T09:36:22.226606+0000","last_became_peered":"2023-10-01T09:36:22.226606+0000","last_unstale":"2023-10-01T09:36:28.600493+0000","last_undegraded":"2023-10-01T09:36:28.600493+0000","last_fullsized":"2023-10-01T09:36:28.600493+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":461,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.593765+0000","last_change":"2023-10-01T09:36:22.225663+0000","last_active":"2023-10-01T09:36:27.593765+0000","last_peered":"2023-10-01T09:36:27.593765+0000","last_clean":"2023-10-01T09:36:27.593765+0000","last_became_active":"2023-10-01T09:36:22.225382+0000","last_became_peered":"2023-10-01T09:36:22.225382+0000","last_unstale":"2023-10-01T09:36:27.593765+0000","last_undegraded":"2023-10-01T09:36:27.593765+0000","last_fullsized":"2023-10-01T09:36:27.593765+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:19:41.386221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":460,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.618200+0000","last_change":"2023-10-01T09:35:34.551948+0000","last_active":"2023-10-01T09:36:27.618200+0000","last_peered":"2023-10-01T09:36:27.618200+0000","last_clean":"2023-10-01T09:36:27.618200+0000","last_became_active":"2023-10-01T09:35:34.551665+0000","last_became_peered":"2023-10-01T09:35:34.551665+0000","last_unstale":"2023-10-01T09:36:27.618200+0000","last_undegraded":"2023-10-01T09:36:27.618200+0000","last_fullsized":"2023-10-01T09:36:27.618200+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:31:30.655381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"434'8187","reported_seq":8275,"reported_epoch":434,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:36:28.887803+0000","last_change":"2023-10-01T09:36:23.705972+0000","last_active":"2023-10-01T09:36:28.887803+0000","last_peered":"2023-10-01T09:36:28.887803+0000","last_clean":"2023-10-01T09:36:21.165407+0000","last_became_active":"2023-10-01T09:36:23.529481+0000","last_became_peered":"2023-10-01T09:36:23.529481+0000","last_unstale":"2023-10-01T09:36:28.887803+0000","last_undegraded":"2023-10-01T09:36:28.887803+0000","last_fullsized":"2023-10-01T09:36:28.887803+0000","mapping_epoch":431,"log_start":"405'500","ondisk_log_start":"405'500","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7687,"log_dups_size":0,"ondisk_log_size":7687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:01:56.211865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316997632,"num_objects":4837,"num_object_clones":0,"num_object_copies":14511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4824,"num_objects_unfound":0,"num_objects_dirty":4837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8187,"num_write_kb":416768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":4837}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":873,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.600068+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:36:28.600068+0000","last_peered":"2023-10-01T09:36:28.600068+0000","last_clean":"2023-10-01T09:36:28.600068+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:36:28.600068+0000","last_undegraded":"2023-10-01T09:36:28.600068+0000","last_fullsized":"2023-10-01T09:36:28.600068+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":489,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.585667+0000","last_change":"2023-10-01T09:35:34.559426+0000","last_active":"2023-10-01T09:36:27.585667+0000","last_peered":"2023-10-01T09:36:27.585667+0000","last_clean":"2023-10-01T09:36:27.585667+0000","last_became_active":"2023-10-01T09:35:34.558890+0000","last_became_peered":"2023-10-01T09:35:34.558890+0000","last_unstale":"2023-10-01T09:36:27.585667+0000","last_undegraded":"2023-10-01T09:36:27.585667+0000","last_fullsized":"2023-10-01T09:36:27.585667+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:55:37.464032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":425,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.594247+0000","last_change":"2023-10-01T09:33:11.656981+0000","last_active":"2023-10-01T09:36:27.594247+0000","last_peered":"2023-10-01T09:36:27.594247+0000","last_clean":"2023-10-01T09:36:27.594247+0000","last_became_active":"2023-10-01T09:33:11.656334+0000","last_became_peered":"2023-10-01T09:33:11.656334+0000","last_unstale":"2023-10-01T09:36:27.594247+0000","last_undegraded":"2023-10-01T09:36:27.594247+0000","last_fullsized":"2023-10-01T09:36:27.594247+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:50:15.562113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"435'8116","reported_seq":8168,"reported_epoch":435,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:36:29.616524+0000","last_change":"2023-10-01T09:36:23.622968+0000","last_active":"2023-10-01T09:36:29.616524+0000","last_peered":"2023-10-01T09:36:29.616524+0000","last_clean":"2023-10-01T09:36:21.149981+0000","last_became_active":"2023-10-01T09:36:23.603244+0000","last_became_peered":"2023-10-01T09:36:23.603244+0000","last_unstale":"2023-10-01T09:36:29.616524+0000","last_undegraded":"2023-10-01T09:36:29.616524+0000","last_fullsized":"2023-10-01T09:36:29.616524+0000","mapping_epoch":431,"log_start":"405'400","ondisk_log_start":"405'400","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7716,"log_dups_size":0,"ondisk_log_size":7716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306642968,"num_objects":4680,"num_object_clones":0,"num_object_copies":14040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9360,"num_objects_unfound":0,"num_objects_dirty":4680,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8116,"num_write_kb":409409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,1,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":4680}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":348,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.599987+0000","last_change":"2023-10-01T09:30:39.402924+0000","last_active":"2023-10-01T09:36:28.599987+0000","last_peered":"2023-10-01T09:36:28.599987+0000","last_clean":"2023-10-01T09:36:28.599987+0000","last_became_active":"2023-10-01T09:30:39.402594+0000","last_became_peered":"2023-10-01T09:30:39.402594+0000","last_unstale":"2023-10-01T09:36:28.599987+0000","last_undegraded":"2023-10-01T09:36:28.599987+0000","last_fullsized":"2023-10-01T09:36:28.599987+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1063,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.612835+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:36:27.612835+0000","last_peered":"2023-10-01T09:36:27.612835+0000","last_clean":"2023-10-01T09:36:27.612835+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:36:27.612835+0000","last_undegraded":"2023-10-01T09:36:27.612835+0000","last_fullsized":"2023-10-01T09:36:27.612835+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":155,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.597354+0000","last_change":"2023-10-01T09:36:22.236397+0000","last_active":"2023-10-01T09:36:27.597354+0000","last_peered":"2023-10-01T09:36:27.597354+0000","last_clean":"2023-10-01T09:36:27.597354+0000","last_became_active":"2023-10-01T09:36:22.236066+0000","last_became_peered":"2023-10-01T09:36:22.236066+0000","last_unstale":"2023-10-01T09:36:27.597354+0000","last_undegraded":"2023-10-01T09:36:27.597354+0000","last_fullsized":"2023-10-01T09:36:27.597354+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:04:06.876501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"435'8292","reported_seq":8335,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-10-01T09:36:31.738042+0000","last_change":"2023-10-01T09:34:06.167610+0000","last_active":"2023-10-01T09:36:31.738042+0000","last_peered":"2023-10-01T09:36:31.738042+0000","last_clean":"2023-10-01T09:36:31.738042+0000","last_became_active":"2023-10-01T09:34:06.167324+0000","last_became_peered":"2023-10-01T09:34:06.167324+0000","last_unstale":"2023-10-01T09:36:31.738042+0000","last_undegraded":"2023-10-01T09:36:31.738042+0000","last_fullsized":"2023-10-01T09:36:31.738042+0000","mapping_epoch":400,"log_start":"405'600","ondisk_log_start":"405'600","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7692,"log_dups_size":0,"ondisk_log_size":7692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309329920,"num_objects":4720,"num_object_clones":0,"num_object_copies":14160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8292,"num_write_kb":416384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":159,"reported_epoch":431,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:36:22.226985+0000","last_change":"2023-10-01T09:36:22.226985+0000","last_active":"2023-10-01T09:36:22.226985+0000","last_peered":"2023-10-01T09:36:22.226985+0000","last_clean":"2023-10-01T09:35:18.526882+0000","last_became_active":"2023-10-01T09:36:22.226622+0000","last_became_peered":"2023-10-01T09:36:22.226622+0000","last_unstale":"2023-10-01T09:36:22.226985+0000","last_undegraded":"2023-10-01T09:36:22.220430+0000","last_fullsized":"2023-10-01T09:36:22.226985+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":["6","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":447,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.618372+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:36:27.618372+0000","last_peered":"2023-10-01T09:36:27.618372+0000","last_clean":"2023-10-01T09:36:27.618372+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:36:27.618372+0000","last_undegraded":"2023-10-01T09:36:27.618372+0000","last_fullsized":"2023-10-01T09:36:27.618372+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":158,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.592777+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:36:28.592777+0000","last_peered":"2023-10-01T09:36:28.592777+0000","last_clean":"2023-10-01T09:36:28.592777+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:36:28.592777+0000","last_undegraded":"2023-10-01T09:36:28.592777+0000","last_fullsized":"2023-10-01T09:36:28.592777+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":397,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.594169+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:36:27.594169+0000","last_peered":"2023-10-01T09:36:27.594169+0000","last_clean":"2023-10-01T09:36:27.594169+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:36:27.594169+0000","last_undegraded":"2023-10-01T09:36:27.594169+0000","last_fullsized":"2023-10-01T09:36:27.594169+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":484,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.585730+0000","last_change":"2023-10-01T09:35:34.559546+0000","last_active":"2023-10-01T09:36:27.585730+0000","last_peered":"2023-10-01T09:36:27.585730+0000","last_clean":"2023-10-01T09:36:27.585730+0000","last_became_active":"2023-10-01T09:35:34.559268+0000","last_became_peered":"2023-10-01T09:35:34.559268+0000","last_unstale":"2023-10-01T09:36:27.585730+0000","last_undegraded":"2023-10-01T09:36:27.585730+0000","last_fullsized":"2023-10-01T09:36:27.585730+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:25.870831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":170,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.592459+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:36:28.592459+0000","last_peered":"2023-10-01T09:36:28.592459+0000","last_clean":"2023-10-01T09:36:28.592459+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:36:28.592459+0000","last_undegraded":"2023-10-01T09:36:28.592459+0000","last_fullsized":"2023-10-01T09:36:28.592459+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":121,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.592786+0000","last_change":"2023-10-01T09:36:22.236882+0000","last_active":"2023-10-01T09:36:28.592786+0000","last_peered":"2023-10-01T09:36:28.592786+0000","last_clean":"2023-10-01T09:36:28.592786+0000","last_became_active":"2023-10-01T09:36:22.236598+0000","last_became_peered":"2023-10-01T09:36:22.236598+0000","last_unstale":"2023-10-01T09:36:28.592786+0000","last_undegraded":"2023-10-01T09:36:28.592786+0000","last_fullsized":"2023-10-01T09:36:28.592786+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":170,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.593334+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:36:28.593334+0000","last_peered":"2023-10-01T09:36:28.593334+0000","last_clean":"2023-10-01T09:36:28.593334+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:36:28.593334+0000","last_undegraded":"2023-10-01T09:36:28.593334+0000","last_fullsized":"2023-10-01T09:36:28.593334+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":325,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.593759+0000","last_change":"2023-10-01T09:32:27.456398+0000","last_active":"2023-10-01T09:36:27.593759+0000","last_peered":"2023-10-01T09:36:27.593759+0000","last_clean":"2023-10-01T09:36:27.593759+0000","last_became_active":"2023-10-01T09:32:27.456106+0000","last_became_peered":"2023-10-01T09:32:27.456106+0000","last_unstale":"2023-10-01T09:36:27.593759+0000","last_undegraded":"2023-10-01T09:36:27.593759+0000","last_fullsized":"2023-10-01T09:36:27.593759+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:10:29.553023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":238,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-10-01T09:36:28.597375+0000","last_change":"2023-10-01T09:35:34.552254+0000","last_active":"2023-10-01T09:36:28.597375+0000","last_peered":"2023-10-01T09:36:28.597375+0000","last_clean":"2023-10-01T09:36:28.597375+0000","last_became_active":"2023-10-01T09:35:34.551892+0000","last_became_peered":"2023-10-01T09:35:34.551892+0000","last_unstale":"2023-10-01T09:36:28.597375+0000","last_undegraded":"2023-10-01T09:36:28.597375+0000","last_fullsized":"2023-10-01T09:36:28.597375+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:09:40.683817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":361,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.593684+0000","last_change":"2023-10-01T09:35:34.557516+0000","last_active":"2023-10-01T09:36:27.593684+0000","last_peered":"2023-10-01T09:36:27.593684+0000","last_clean":"2023-10-01T09:36:27.593684+0000","last_became_active":"2023-10-01T09:35:34.556528+0000","last_became_peered":"2023-10-01T09:35:34.556528+0000","last_unstale":"2023-10-01T09:36:27.593684+0000","last_undegraded":"2023-10-01T09:36:27.593684+0000","last_fullsized":"2023-10-01T09:36:27.593684+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:33:56.582462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":572,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.594716+0000","last_change":"2023-10-01T09:35:34.658812+0000","last_active":"2023-10-01T09:36:27.594716+0000","last_peered":"2023-10-01T09:36:27.594716+0000","last_clean":"2023-10-01T09:36:27.594716+0000","last_became_active":"2023-10-01T09:35:34.550952+0000","last_became_peered":"2023-10-01T09:35:34.550952+0000","last_unstale":"2023-10-01T09:36:27.594716+0000","last_undegraded":"2023-10-01T09:36:27.594716+0000","last_fullsized":"2023-10-01T09:36:27.594716+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:24:17.304117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":403,"reported_epoch":433,"state":"active+clean","last_fresh":"2023-10-01T09:36:27.594284+0000","last_change":"2023-10-01T09:32:27.445299+0000","last_active":"2023-10-01T09:36:27.594284+0000","last_peered":"2023-10-01T09:36:27.594284+0000","last_clean":"2023-10-01T09:36:27.594284+0000","last_became_active":"2023-10-01T09:32:27.444956+0000","last_became_peered":"2023-10-01T09:32:27.444956+0000","last_unstale":"2023-10-01T09:36:27.594284+0000","last_undegraded":"2023-10-01T09:36:27.594284+0000","last_fullsized":"2023-10-01T09:36:27.594284+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:49:07.862356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":34340,"num_keys_recovered":90,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4919394328,"num_objects":75065,"num_object_clones":0,"num_object_copies":225195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":37846,"num_objects_misplaced":25952,"num_objects_unfound":0,"num_objects_dirty":75065,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":129701,"num_write_kb":6552449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4413,"num_bytes_recovered":288620544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15026106368,"data_stored":15026094152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123846,"ondisk_log_size":123846,"up":48,"acting":43,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":276,"num_bytes_recovered":4194361,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430631,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3697680,"kb_used_data":2794568,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":90047472,"statfs":{"total":95995035648,"available":92208611328,"internally_reserved":0,"allocated":2861637632,"data_stored":2860748800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:35:36 2023","interfaces":[{"interface":"back","average":{"1min":5.6790000000000003,"5min":4.343,"15min":3.0569999999999999},"min":{"1min":0.218,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":46.042999999999999,"5min":108.90000000000001,"15min":108.90000000000001},"last":3.278},{"interface":"front","average":{"1min":6.0460000000000003,"5min":4.4470000000000001,"15min":3.3130000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.22500000000000001},"max":{"1min":46.057000000000002,"5min":109.187,"15min":109.187},"last":2.327}]},{"osd":1,"last update":"Sun Oct 1 09:35:55 2023","interfaces":[{"interface":"back","average":{"1min":6.899,"5min":8.3000000000000007,"15min":4.5780000000000003},"min":{"1min":0.23300000000000001,"5min":0.222,"15min":0.222},"max":{"1min":110.05500000000001,"5min":205.34200000000001,"15min":205.34200000000001},"last":1.0660000000000001},{"interface":"front","average":{"1min":6.8869999999999996,"5min":8.3219999999999992,"15min":4.5629999999999997},"min":{"1min":0.33600000000000002,"5min":0.24399999999999999,"15min":0.23599999999999999},"max":{"1min":112.23999999999999,"5min":205.44499999999999,"15min":205.44499999999999},"last":4.4000000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:36:28 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":2.0430000000000001,"15min":2.1549999999999998},"min":{"1min":0.32100000000000001,"5min":0.27100000000000002,"15min":0.249},"max":{"1min":14.901999999999999,"5min":14.901999999999999,"15min":59.878},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.9330000000000001,"5min":1.71,"15min":2.0979999999999999},"min":{"1min":0.377,"5min":0.27700000000000002,"15min":0.247},"max":{"1min":5.4630000000000001,"5min":7.407,"15min":60.186},"last":2.3540000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:35:55 2023","interfaces":[{"interface":"back","average":{"1min":5.5609999999999999,"5min":3.2730000000000001,"15min":3.367},"min":{"1min":0.33300000000000002,"5min":0.32900000000000001,"15min":0.28199999999999997},"max":{"1min":76.555000000000007,"5min":76.555000000000007,"15min":164.595},"last":4.3029999999999999},{"interface":"front","average":{"1min":6.5430000000000001,"5min":3.1859999999999999,"15min":3.2989999999999999},"min":{"1min":0.40600000000000003,"5min":0.33200000000000002,"15min":0.29399999999999998},"max":{"1min":76.534000000000006,"5min":76.534000000000006,"15min":164.60499999999999},"last":0.435}]},{"osd":4,"last update":"Sun Oct 1 09:35:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":1.226,"15min":0.997},"min":{"1min":0.27900000000000003,"5min":0.26700000000000002,"15min":0.23499999999999999},"max":{"1min":5.2050000000000001,"5min":7.7039999999999997,"15min":7.7039999999999997},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.2110000000000001,"15min":1.0249999999999999},"min":{"1min":0.311,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":5.016,"5min":8.0009999999999994,"15min":9.4339999999999993},"last":2.2869999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:36:17 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":1.0720000000000001,"15min":1.012},"min":{"1min":0.35899999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.1159999999999997,"5min":6.1159999999999997,"15min":6.1159999999999997},"last":2.383},{"interface":"front","average":{"1min":2.577,"5min":1.2010000000000001,"15min":0.92900000000000005},"min":{"1min":0.372,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":13.932,"5min":13.932,"15min":13.932},"last":0.45200000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:36:08 2023","interfaces":[{"interface":"back","average":{"1min":1.948,"5min":1.272,"15min":0.94899999999999995},"min":{"1min":0.35699999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":11.853999999999999,"5min":11.853999999999999,"15min":11.853999999999999},"last":2.4089999999999998},{"interface":"front","average":{"1min":2.214,"5min":1.2889999999999999,"15min":0.88500000000000001},"min":{"1min":0.32400000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.266,"5min":7.266,"15min":7.266},"last":0.39900000000000002}]}]},{"osd":6,"up_from":294,"seq":1262720385141,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1327728,"kb_used_data":994216,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":92417424,"statfs":{"total":95995035648,"available":94635442176,"internally_reserved":0,"allocated":1018077184,"data_stored":1017196618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:36:04 2023","interfaces":[{"interface":"back","average":{"1min":3.6150000000000002,"5min":6.6660000000000004,"15min":3.6600000000000001},"min":{"1min":0.252,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":27.949999999999999,"5min":130.02699999999999,"15min":130.02699999999999},"last":5.7569999999999997},{"interface":"front","average":{"1min":3.177,"5min":6.1619999999999999,"15min":3.3119999999999998},"min":{"1min":0.26800000000000002,"5min":0.221,"15min":0.221},"max":{"1min":29.405000000000001,"5min":133.72900000000001,"15min":133.72900000000001},"last":4.1950000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:36:04 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":5.9240000000000004,"15min":3.335},"min":{"1min":0.35699999999999998,"5min":0.24199999999999999,"15min":0.23899999999999999},"max":{"1min":5.9029999999999996,"5min":97.561000000000007,"15min":97.561000000000007},"last":1.665},{"interface":"front","average":{"1min":1.728,"5min":5.8659999999999997,"15min":3.2770000000000001},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.24399999999999999},"max":{"1min":6.4370000000000003,"5min":97.585999999999999,"15min":97.585999999999999},"last":1.6839999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:36:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1840000000000002,"5min":1.827,"15min":2.3460000000000001},"min":{"1min":0.27600000000000002,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":5.8179999999999996,"5min":10.92,"15min":26.289999999999999},"last":1.3109999999999999},{"interface":"front","average":{"1min":2.0129999999999999,"5min":1.873,"15min":2.4409999999999998},"min":{"1min":0.35199999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.454000000000001,"5min":12.804,"15min":26.388999999999999},"last":2.8610000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:36:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":2.9470000000000001,"15min":2.7090000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":9.5679999999999996,"5min":89.753,"15min":89.753},"last":1.119},{"interface":"front","average":{"1min":2.024,"5min":2.5800000000000001,"15min":2.0600000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.29299999999999998},"max":{"1min":5.8710000000000004,"5min":89.849999999999994,"15min":89.849999999999994},"last":1.8109999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:36:04 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.0389999999999999,"15min":0.63500000000000001},"min":{"1min":0.29599999999999999,"5min":0.26400000000000001,"15min":0.25700000000000001},"max":{"1min":3.419,"5min":6.0640000000000001,"15min":6.0640000000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.84699999999999998,"5min":1.151,"15min":0.76800000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":3.984,"5min":9.4090000000000007,"15min":9.4090000000000007},"last":0.39700000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.1040000000000001,"15min":1.0660000000000001},"min":{"1min":0.33200000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":12.023999999999999,"5min":12.023999999999999,"15min":12.023999999999999},"last":1.1970000000000001},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.159,"15min":1.1399999999999999},"min":{"1min":0.33200000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.141,"5min":4.5309999999999997,"15min":4.5309999999999997},"last":0.35199999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:36:04 2023","interfaces":[{"interface":"back","average":{"1min":7.3319999999999999,"5min":2.3919999999999999,"15min":3.7240000000000002},"min":{"1min":0.438,"5min":0.30499999999999999,"15min":0.27900000000000003},"max":{"1min":94.245999999999995,"5min":94.245999999999995,"15min":115.11},"last":0.51700000000000002},{"interface":"front","average":{"1min":7.5540000000000003,"5min":2.2050000000000001,"15min":3.649},"min":{"1min":0.33000000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":94.715000000000003,"5min":94.715000000000003,"15min":114.666},"last":0.48599999999999999}]}]},{"osd":1,"up_from":94,"seq":403726926118,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3400804,"kb_used_data":2723868,"kb_used_omap":7,"kb_used_meta":676856,"kb_avail":90344348,"statfs":{"total":95995035648,"available":92512612352,"internally_reserved":0,"allocated":2789240832,"data_stored":2787445889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":693100967},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:35:43 2023","interfaces":[{"interface":"back","average":{"1min":2.3119999999999998,"5min":3.4180000000000001,"15min":3.21},"min":{"1min":0.20100000000000001,"5min":0.183,"15min":0.183},"max":{"1min":8.5389999999999997,"5min":46.497999999999998,"15min":82.359999999999999},"last":4.415},{"interface":"front","average":{"1min":2.5979999999999999,"5min":3.383,"15min":3.2280000000000002},"min":{"1min":0.20799999999999999,"5min":0.20200000000000001,"15min":0.17599999999999999},"max":{"1min":9.5510000000000002,"5min":46.353999999999999,"15min":82.078999999999994},"last":0.27300000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:36:27 2023","interfaces":[{"interface":"back","average":{"1min":1.792,"5min":3.048,"15min":3.7919999999999998},"min":{"1min":0.23400000000000001,"5min":0.20000000000000001,"15min":0.182},"max":{"1min":6.7450000000000001,"5min":51.640000000000001,"15min":77.325000000000003},"last":3.714},{"interface":"front","average":{"1min":9.7409999999999997,"5min":4.8170000000000002,"15min":4.7009999999999996},"min":{"1min":0.254,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":154.197,"5min":154.197,"15min":154.197},"last":3.5859999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:36:01 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":3.496,"15min":4.3780000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":9.4090000000000007,"5min":52.417999999999999,"15min":200.56800000000001},"last":27.486000000000001},{"interface":"front","average":{"1min":2.1230000000000002,"5min":3.7959999999999998,"15min":4.4160000000000004},"min":{"1min":0.249,"5min":0.216,"15min":0.21199999999999999},"max":{"1min":8.7759999999999998,"5min":52.469999999999999,"15min":200.63800000000001},"last":4.2990000000000004}]},{"osd":4,"last update":"Sun Oct 1 09:35:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2050000000000001,"5min":1.546,"15min":1.5009999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":11.036,"5min":13.045,"15min":13.045},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.5860000000000001,"5min":2.0470000000000002,"15min":1.9179999999999999},"min":{"1min":0.33800000000000002,"5min":0.32800000000000001,"15min":0.27500000000000002},"max":{"1min":4.6660000000000004,"5min":12.023999999999999,"15min":18.219000000000001},"last":3.6739999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:36:22 2023","interfaces":[{"interface":"back","average":{"1min":10.359999999999999,"5min":3.0609999999999999,"15min":1.6240000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":154.22999999999999,"5min":154.22999999999999,"15min":154.22999999999999},"last":3.8999999999999999},{"interface":"front","average":{"1min":10.117000000000001,"5min":3.637,"15min":2.5609999999999999},"min":{"1min":0.46600000000000003,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":154.54400000000001,"5min":154.54400000000001,"15min":154.54400000000001},"last":3.7799999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:36:01 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":2.4980000000000002,"15min":2.6720000000000002},"min":{"1min":0.313,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":10.961,"5min":12.842000000000001,"15min":12.842000000000001},"last":3.5329999999999999},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.9159999999999999,"15min":1.8680000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.7270000000000003,"5min":14.772,"15min":14.772},"last":0.44600000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:35:43 2023","interfaces":[{"interface":"back","average":{"1min":3.0179999999999998,"5min":2.3100000000000001,"15min":1.9219999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.316},"max":{"1min":20.756,"5min":20.756,"15min":21.399000000000001},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.8010000000000002,"5min":2.391,"15min":2.117},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.30199999999999999},"max":{"1min":20.765000000000001,"5min":20.765000000000001,"15min":22.318999999999999},"last":3.7480000000000002}]}]},{"osd":0,"up_from":89,"seq":382252089652,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1995472,"kb_used_data":1277960,"kb_used_omap":8,"kb_used_meta":717495,"kb_avail":91749680,"statfs":{"total":95995035648,"available":93951672320,"internally_reserved":0,"allocated":1308631040,"data_stored":1306807510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":734715842},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:35:51 2023","interfaces":[{"interface":"back","average":{"1min":6.2869999999999999,"5min":3.9580000000000002,"15min":3.4169999999999998},"min":{"1min":0.224,"5min":0.19500000000000001,"15min":0.17399999999999999},"max":{"1min":73.135999999999996,"5min":73.135999999999996,"15min":187.41399999999999},"last":216.821},{"interface":"front","average":{"1min":6.2409999999999997,"5min":4.4859999999999998,"15min":4.2770000000000001},"min":{"1min":0.28000000000000003,"5min":0.23999999999999999,"15min":0.193},"max":{"1min":73.683000000000007,"5min":73.683000000000007,"15min":187.78700000000001},"last":217.02699999999999}]},{"osd":2,"last update":"Sun Oct 1 09:36:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":2.4900000000000002,"15min":2.1970000000000001},"min":{"1min":0.28899999999999998,"5min":0.21299999999999999,"15min":0.20100000000000001},"max":{"1min":9.6639999999999997,"5min":16.033000000000001,"15min":39.048999999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":2.0139999999999998,"5min":2.4950000000000001,"15min":2.9209999999999998},"min":{"1min":0.27500000000000002,"5min":0.222,"15min":0.189},"max":{"1min":15.472,"5min":15.472,"15min":38.945999999999998},"last":0.54200000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":3.5289999999999999,"15min":4.069},"min":{"1min":0.26300000000000001,"5min":0.25,"15min":0.20899999999999999},"max":{"1min":16.978000000000002,"5min":59.216000000000001,"15min":139.85300000000001},"last":222.16499999999999},{"interface":"front","average":{"1min":1.9019999999999999,"5min":3.0840000000000001,"15min":4.4420000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.219},"max":{"1min":9.5899999999999999,"5min":59.177999999999997,"15min":139.70099999999999},"last":222.227}]},{"osd":4,"last update":"Sun Oct 1 09:35:44 2023","interfaces":[{"interface":"back","average":{"1min":1.6870000000000001,"5min":2.4700000000000002,"15min":2.177},"min":{"1min":0.31,"5min":0.27600000000000002,"15min":0.27300000000000002},"max":{"1min":4.6139999999999999,"5min":62.206000000000003,"15min":62.206000000000003},"last":0.622},{"interface":"front","average":{"1min":1.7090000000000001,"5min":2.3239999999999998,"15min":2.0910000000000002},"min":{"1min":0.39100000000000001,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":9.4570000000000007,"5min":62.073999999999998,"15min":62.073999999999998},"last":0.60399999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:36:15 2023","interfaces":[{"interface":"back","average":{"1min":2.77,"5min":2.0390000000000001,"15min":1.853},"min":{"1min":0.375,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":16.808,"5min":16.808,"15min":16.808},"last":0.57099999999999995},{"interface":"front","average":{"1min":2.4740000000000002,"5min":1.8580000000000001,"15min":1.7070000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":13.162000000000001,"5min":13.162000000000001,"15min":13.162000000000001},"last":0.63600000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:36:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9630000000000001,"5min":2.2090000000000001,"15min":1.639},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":7.9779999999999998,"5min":15.904,"15min":15.904},"last":0.70399999999999996},{"interface":"front","average":{"1min":2.883,"5min":2.363,"15min":2.3780000000000001},"min":{"1min":0.38100000000000001,"5min":0.34200000000000003,"15min":0.32100000000000001},"max":{"1min":15.651999999999999,"5min":15.651999999999999,"15min":18.521000000000001},"last":0.65800000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:35:44 2023","interfaces":[{"interface":"back","average":{"1min":3.5350000000000001,"5min":2.573,"15min":1.758},"min":{"1min":0.40000000000000002,"5min":0.39400000000000002,"15min":0.29199999999999998},"max":{"1min":35.130000000000003,"5min":35.130000000000003,"15min":35.130000000000003},"last":0.56200000000000006},{"interface":"front","average":{"1min":3.847,"5min":2.504,"15min":2.274},"min":{"1min":0.38600000000000001,"5min":0.35099999999999998,"15min":0.312},"max":{"1min":38.590000000000003,"5min":38.590000000000003,"15min":38.590000000000003},"last":0.69099999999999995}]}]},{"osd":2,"up_from":99,"seq":425201762592,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1533640,"kb_used_data":836736,"kb_used_omap":11,"kb_used_meta":696884,"kb_avail":92211512,"statfs":{"total":95995035648,"available":94424588288,"internally_reserved":0,"allocated":856817664,"data_stored":855065537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":713610103},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:35:37 2023","interfaces":[{"interface":"back","average":{"1min":4.0899999999999999,"5min":7.9329999999999998,"15min":6.7880000000000003},"min":{"1min":0.216,"5min":0.191,"15min":0.17599999999999999},"max":{"1min":45.688000000000002,"5min":201.892,"15min":224.56},"last":6.3449999999999998},{"interface":"front","average":{"1min":3.9060000000000001,"5min":7.867,"15min":7.1909999999999998},"min":{"1min":0.23100000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":45.512999999999998,"5min":200.089,"15min":225.66900000000001},"last":5.3529999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:36:28 2023","interfaces":[{"interface":"back","average":{"1min":10.864000000000001,"5min":11.503,"15min":8.3239999999999998},"min":{"1min":0.27600000000000002,"5min":0.214,"15min":0.191},"max":{"1min":89.593000000000004,"5min":196.34399999999999,"15min":224.917},"last":5.5179999999999998},{"interface":"front","average":{"1min":11.606,"5min":11.936,"15min":9.1859999999999999},"min":{"1min":0.38300000000000001,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":92.969999999999999,"5min":196.542,"15min":225.625},"last":6.2789999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:35:58 2023","interfaces":[{"interface":"back","average":{"1min":1.4299999999999999,"5min":2.0030000000000001,"15min":5.0460000000000003},"min":{"1min":0.26800000000000002,"5min":0.216,"15min":0.21099999999999999},"max":{"1min":5.431,"5min":17.625,"15min":226.124},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.167,"5min":1.8939999999999999,"15min":4.923},"min":{"1min":0.25700000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":4.7300000000000004,"5min":13.672000000000001,"15min":226.32900000000001},"last":5.452}]},{"osd":4,"last update":"Sun Oct 1 09:36:28 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.583,"15min":2.6160000000000001},"min":{"1min":0.32500000000000001,"5min":0.28599999999999998,"15min":0.27200000000000002},"max":{"1min":5.3899999999999997,"5min":22.084,"15min":224.90199999999999},"last":5.3899999999999997},{"interface":"front","average":{"1min":2.2120000000000002,"5min":1.9990000000000001,"15min":2.8279999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27500000000000002},"max":{"1min":6.4450000000000003,"5min":22.266999999999999,"15min":225.386},"last":0.36499999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:36:14 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.466,"15min":1.5960000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.0019999999999998,"5min":12.500999999999999,"15min":12.500999999999999},"last":5.5419999999999998},{"interface":"front","average":{"1min":1.29,"5min":1.232,"15min":1.29},"min":{"1min":0.35499999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.8529999999999998,"5min":10.194000000000001,"15min":10.194000000000001},"last":5.569}]},{"osd":6,"last update":"Sun Oct 1 09:36:04 2023","interfaces":[{"interface":"back","average":{"1min":1.698,"5min":1.8109999999999999,"15min":2.8149999999999999},"min":{"1min":0.34100000000000003,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":5.6040000000000001,"5min":16.184999999999999,"15min":225.41499999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.6419999999999999,"15min":2.3610000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.8109999999999999,"5min":13.724,"15min":225.02000000000001},"last":5.8239999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:36:28 2023","interfaces":[{"interface":"back","average":{"1min":10.976000000000001,"5min":4.5830000000000002,"15min":3.8039999999999998},"min":{"1min":0.38700000000000001,"5min":0.31,"15min":0.308},"max":{"1min":139.03299999999999,"5min":139.03299999999999,"15min":224.94999999999999},"last":5.476},{"interface":"front","average":{"1min":11.333,"5min":4.4539999999999997,"15min":3.9180000000000001},"min":{"1min":0.39400000000000002,"5min":0.375,"15min":0.33600000000000002},"max":{"1min":139.065,"5min":139.065,"15min":224.92599999999999},"last":5.4240000000000004}]}]},{"osd":3,"up_from":104,"seq":446676599066,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3153712,"kb_used_data":2438440,"kb_used_omap":3,"kb_used_meta":715260,"kb_avail":90591440,"statfs":{"total":95995035648,"available":92765634560,"internally_reserved":0,"allocated":2496962560,"data_stored":2495113189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":732426983},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:35:57 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":7.5839999999999996,"15min":5.484},"min":{"1min":0.19900000000000001,"5min":0.186,"15min":0.18099999999999999},"max":{"1min":3.4009999999999998,"5min":137.34399999999999,"15min":137.34399999999999},"last":1.256},{"interface":"front","average":{"1min":2.3140000000000001,"5min":7.2169999999999996,"15min":5.3479999999999999},"min":{"1min":0.29899999999999999,"5min":0.215,"15min":0.184},"max":{"1min":7.7560000000000002,"5min":137.73400000000001,"15min":137.73400000000001},"last":0.44600000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:35:57 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":5.7130000000000001,"15min":5.3540000000000001},"min":{"1min":0.28000000000000003,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":12.965999999999999,"5min":162.22999999999999,"15min":228.60499999999999},"last":1.4450000000000001},{"interface":"front","average":{"1min":1.7849999999999999,"5min":5.8860000000000001,"15min":5.7350000000000003},"min":{"1min":0.23300000000000001,"5min":0.221,"15min":0.221},"max":{"1min":12.818,"5min":162.70699999999999,"15min":228.28399999999999},"last":0.29499999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:35:57 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.161,"15min":3.048},"min":{"1min":0.23200000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.5229999999999997,"5min":7.0919999999999996,"15min":62.909999999999997},"last":3.1160000000000001},{"interface":"front","average":{"1min":2.044,"5min":2.528,"15min":3.8809999999999998},"min":{"1min":0.27800000000000002,"5min":0.222,"15min":0.219},"max":{"1min":12.645,"5min":13.226000000000001,"15min":78.594999999999999},"last":0.67800000000000005}]},{"osd":4,"last update":"Sun Oct 1 09:35:57 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":2.0579999999999998,"15min":2.1890000000000001},"min":{"1min":0.316,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.4020000000000001,"5min":29.419,"15min":108.896},"last":0.34000000000000002},{"interface":"front","average":{"1min":2.0739999999999998,"5min":2.1259999999999999,"15min":2.415},"min":{"1min":0.34200000000000003,"5min":0.309,"15min":0.23899999999999999},"max":{"1min":9.8079999999999998,"5min":33.002000000000002,"15min":108.916},"last":6.4210000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:36:14 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":2.9660000000000002,"15min":3.577},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":7.4109999999999996,"5min":28.117999999999999,"15min":28.117999999999999},"last":1.3460000000000001},{"interface":"front","average":{"1min":2.0899999999999999,"5min":2.9380000000000002,"15min":3.4620000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":7.016,"5min":28.34,"15min":28.34},"last":3.2069999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:36:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5429999999999999,"5min":1.337,"15min":2.403},"min":{"1min":0.377,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.9950000000000001,"5min":7.6929999999999996,"15min":18.361000000000001},"last":3.2280000000000002},{"interface":"front","average":{"1min":1.5920000000000001,"5min":1.399,"15min":2.347},"min":{"1min":0.33600000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.8600000000000003,"5min":7.625,"15min":18.332000000000001},"last":3.181}]},{"osd":7,"last update":"Sun Oct 1 09:35:57 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":1.611,"15min":2.3660000000000001},"min":{"1min":0.39900000000000002,"5min":0.31,"15min":0.309},"max":{"1min":5.6580000000000004,"5min":7.657,"15min":52.006},"last":3.625},{"interface":"front","average":{"1min":1.5780000000000001,"5min":2.0630000000000002,"15min":2.3559999999999999},"min":{"1min":0.36399999999999999,"5min":0.311,"15min":0.311},"max":{"1min":3.3250000000000002,"5min":13.191000000000001,"15min":51.890999999999998},"last":1.3140000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019315,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1849912,"kb_used_data":963824,"kb_used_omap":0,"kb_used_meta":886080,"kb_avail":91895240,"statfs":{"total":95995035648,"available":94100725760,"internally_reserved":0,"allocated":986955776,"data_stored":986067161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":907345920},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:35:44 2023","interfaces":[{"interface":"back","average":{"1min":4.3529999999999998,"5min":7.3540000000000001,"15min":4.3710000000000004},"min":{"1min":0.23799999999999999,"5min":0.22600000000000001,"15min":0.223},"max":{"1min":57.328000000000003,"5min":155.315,"15min":155.315},"last":7.6529999999999996},{"interface":"front","average":{"1min":4.5819999999999999,"5min":7.9379999999999997,"15min":4.5359999999999996},"min":{"1min":0.28499999999999998,"5min":0.27900000000000003,"15min":0.23499999999999999},"max":{"1min":57.508000000000003,"5min":155.334,"15min":155.334},"last":5.7880000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:35:50 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":4.5179999999999998,"15min":5.4059999999999997},"min":{"1min":0.30399999999999999,"5min":0.25,"15min":0.25},"max":{"1min":7.657,"5min":190.61199999999999,"15min":190.61199999999999},"last":5.7000000000000002},{"interface":"front","average":{"1min":1.8939999999999999,"5min":6.8929999999999998,"15min":6.1769999999999996},"min":{"1min":0.35299999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.1829999999999998,"5min":253.273,"15min":253.273},"last":5.7080000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:35:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":3.633,"15min":3.2890000000000001},"min":{"1min":0.35399999999999998,"5min":0.28299999999999997,"15min":0.26700000000000002},"max":{"1min":5.7649999999999997,"5min":170.45400000000001,"15min":170.45400000000001},"last":4.7450000000000001},{"interface":"front","average":{"1min":1.929,"5min":2.5960000000000001,"15min":2.8879999999999999},"min":{"1min":0.34699999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":12.045,"5min":36.515999999999998,"15min":91.510999999999996},"last":5.6630000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:35:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6970000000000001,"5min":2.0419999999999998,"15min":2.7549999999999999},"min":{"1min":0.30299999999999999,"5min":0.28799999999999998,"15min":0.28299999999999997},"max":{"1min":9.2569999999999997,"5min":14.597,"15min":92.218000000000004},"last":3.2789999999999999},{"interface":"front","average":{"1min":1.9450000000000001,"5min":2.2530000000000001,"15min":3.036},"min":{"1min":0.36799999999999999,"5min":0.317,"15min":0.30399999999999999},"max":{"1min":6.6150000000000002,"5min":23.231000000000002,"15min":92.766999999999996},"last":6.0789999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:36:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0939999999999999,"5min":1.4059999999999999,"15min":1.1919999999999999},"min":{"1min":0.27400000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.6600000000000001,"5min":20.457000000000001,"15min":20.457000000000001},"last":4.8929999999999998},{"interface":"front","average":{"1min":2.206,"5min":1.2310000000000001,"15min":0.91000000000000003},"min":{"1min":0.35499999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.7210000000000001,"5min":20.475000000000001,"15min":20.475000000000001},"last":4.899}]},{"osd":6,"last update":"Sun Oct 1 09:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.1539999999999999,"15min":0.79500000000000004},"min":{"1min":0.29699999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.8070000000000004,"5min":11.358000000000001,"15min":11.358000000000001},"last":1.22},{"interface":"front","average":{"1min":1.607,"5min":1.2330000000000001,"15min":0.83799999999999997},"min":{"1min":0.35099999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.6219999999999999,"5min":10.076000000000001,"15min":10.076000000000001},"last":4.7060000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:35:30 2023","interfaces":[{"interface":"back","average":{"1min":2.4399999999999999,"5min":1.4079999999999999,"15min":1.026},"min":{"1min":0.376,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":22.256,"5min":22.256,"15min":22.256},"last":1.1919999999999999},{"interface":"front","average":{"1min":2.8889999999999998,"5min":1.6819999999999999,"15min":1.196},"min":{"1min":0.377,"5min":0.308,"15min":0.28699999999999998},"max":{"1min":22.635999999999999,"5min":22.635999999999999,"15min":22.635999999999999},"last":4.766}]}]},{"osd":5,"up_from":379,"seq":1627792605225,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3087936,"kb_used_data":2719224,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":90657216,"statfs":{"total":95995035648,"available":92832989184,"internally_reserved":0,"allocated":2784485376,"data_stored":2783600324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:36:16 2023","interfaces":[{"interface":"back","average":{"1min":4.4509999999999996,"5min":2.9910000000000001,"15min":2},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":60.168999999999997,"5min":84.340999999999994,"15min":84.340999999999994},"last":0.434},{"interface":"front","average":{"1min":6.3639999999999999,"5min":3.3399999999999999,"15min":1.9339999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":60.521000000000001,"5min":84.356999999999999,"15min":84.356999999999999},"last":0.33800000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:36:16 2023","interfaces":[{"interface":"back","average":{"1min":14.837999999999999,"5min":6.2939999999999996,"15min":3.23},"min":{"1min":0.28999999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":134.465,"5min":134.465,"15min":134.465},"last":0.64000000000000001},{"interface":"front","average":{"1min":14.263,"5min":6.0579999999999998,"15min":2.9489999999999998},"min":{"1min":0.29299999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":134.22900000000001,"5min":134.22900000000001,"15min":134.22900000000001},"last":1.1459999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:36:16 2023","interfaces":[{"interface":"back","average":{"1min":2.4689999999999999,"5min":1.649,"15min":1.419},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":8.3469999999999995,"5min":8.3469999999999995,"15min":8.3469999999999995},"last":3.4980000000000002},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.524,"15min":1.589},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.1529999999999996,"5min":11.221,"15min":11.221},"last":0.66700000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":1.274,"15min":1.1990000000000001},"min":{"1min":0.308,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.032,"5min":5.032,"15min":5.032},"last":0.54100000000000004},{"interface":"front","average":{"1min":2.9319999999999999,"5min":1.6060000000000001,"15min":1.3200000000000001},"min":{"1min":0.36099999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":8.3740000000000006,"5min":8.3740000000000006,"15min":8.3740000000000006},"last":0.53300000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9770000000000001,"5min":1.232,"15min":1.036},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.808999999999999,"5min":12.808999999999999,"15min":12.808999999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.5980000000000001,"5min":1.609,"15min":1.647},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.9420000000000002,"5min":14.68,"15min":14.68},"last":0.35199999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.97999999999999998,"15min":1.016},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.016,"5min":4.3810000000000002,"15min":4.3810000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":2.9580000000000002,"5min":1.744,"15min":1.579},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":14.193,"5min":14.193,"15min":14.193},"last":0.48199999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":1.53,"15min":1.4219999999999999},"min":{"1min":0.39500000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":9.984,"5min":10.813000000000001,"15min":10.813000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.716,"5min":1.095,"15min":0.96099999999999997},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":12.045999999999999,"5min":12.045999999999999,"15min":12.045999999999999},"last":0.39400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1298595840,"data_stored":1298595840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2775846912,"data_stored":2775842840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":848232448,"data_stored":848232448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2480996352,"data_stored":2480996352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":986251264,"data_stored":986251264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2773356544,"data_stored":2773352472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1010565120,"data_stored":1010565120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2852261888,"data_stored":2852257816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:36:35.906 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-10-01T09:36:35.936 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:36:35.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:35.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:36:36.045 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:36:36.075 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:36.131 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:36:36.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:36.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:36:36.239 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:36:36.252 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:36:36.293 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:36 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/84717081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:36:36.294 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:36 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/84717081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:36:36.377 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:36.399 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:36.495 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:36:36.495 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:36:36.495 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:36:36.495 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:36:36.495 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:36:36.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:36 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/84717081' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:36:37.133 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:36:37.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:37.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:36:37.248 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:36:37.276 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:37.494 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[185886]: pgmap v1213: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 37378/220977 objects degraded (16.915%); 25513/220977 objects misplaced (11.546%); 12 MiB/s, 195 objects/s recovering 2023-10-01T09:36:37.494 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:36:37.494 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[185886]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.494 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/84717081' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:36:37.494 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[185886]: osdmap e436: 8 total, 8 up, 8 in 2023-10-01T09:36:37.494 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[185886]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.494 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[185886]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.494 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[185886]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.495 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[185886]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.495 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[192513]: pgmap v1213: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 37378/220977 objects degraded (16.915%); 25513/220977 objects misplaced (11.546%); 12 MiB/s, 195 objects/s recovering 2023-10-01T09:36:37.495 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:36:37.495 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[192513]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.495 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/84717081' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:36:37.495 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[192513]: osdmap e436: 8 total, 8 up, 8 in 2023-10-01T09:36:37.495 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[192513]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.495 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[192513]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.495 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[192513]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.496 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:37 smithi138 ceph-mon[192513]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:37 smithi190 ceph-mon[160435]: pgmap v1213: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 37378/220977 objects degraded (16.915%); 25513/220977 objects misplaced (11.546%); 12 MiB/s, 195 objects/s recovering 2023-10-01T09:36:37.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:37 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:36:37.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:37 smithi190 ceph-mon[160435]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:37 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/84717081' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:36:37.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:37 smithi190 ceph-mon[160435]: osdmap e436: 8 total, 8 up, 8 in 2023-10-01T09:36:37.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:37 smithi190 ceph-mon[160435]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:37 smithi190 ceph-mon[160435]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:37 smithi190 ceph-mon[160435]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:37 smithi190 ceph-mon[160435]: failed to encode map e436 with expected crc 2023-10-01T09:36:37.668 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:37.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:37.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:37.780 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:37.903 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:38.212 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:38 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 37378/220977 objects degraded (16.915%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:38.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:38 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 37378/220977 objects degraded (16.915%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:38.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:38 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 37378/220977 objects degraded (16.915%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:38.835 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:38.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:38.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:38.955 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:36:38.984 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:38.985 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:36:39.041 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:36:39.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:39.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:36:39.148 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:36:39.178 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:39.238 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:36:39.241 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:39 smithi190 ceph-mon[160435]: pgmap v1215: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 37378/220086 objects degraded (16.983%); 24605/220086 objects misplaced (11.180%); 16 MiB/s, 262 objects/s recovering 2023-10-01T09:36:39.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:39.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:36:39.276 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:39.343 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:36:39.374 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:39.427 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:36:39.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:39.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:36:39.533 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:39.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:39 smithi138 ceph-mon[185886]: pgmap v1215: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 37378/220086 objects degraded (16.983%); 24605/220086 objects misplaced (11.180%); 16 MiB/s, 262 objects/s recovering 2023-10-01T09:36:39.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:39 smithi138 ceph-mon[192513]: pgmap v1215: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 37378/220086 objects degraded (16.983%); 24605/220086 objects misplaced (11.180%); 16 MiB/s, 262 objects/s recovering 2023-10-01T09:36:39.657 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:39.708 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:36:40.524 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:40.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:40.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:40.528 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:40 smithi138 ceph-mon[192513]: pgmap v1216: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 36925/215913 objects degraded (17.102%); 24131/215913 objects misplaced (11.176%); 15 MiB/s, 244 objects/s recovering 2023-10-01T09:36:40.528 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:40 smithi138 ceph-mon[185886]: pgmap v1216: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 36925/215913 objects degraded (17.102%); 24131/215913 objects misplaced (11.176%); 15 MiB/s, 244 objects/s recovering 2023-10-01T09:36:40.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:40 smithi190 ceph-mon[160435]: pgmap v1216: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 36925/215913 objects degraded (17.102%); 24131/215913 objects misplaced (11.176%); 15 MiB/s, 244 objects/s recovering 2023-10-01T09:36:40.629 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:36:40.657 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:40.715 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:36:40.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:40.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:36:40.827 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:36:40.950 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:41.135 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:41.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:41.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:41.249 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:41.387 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:41.528 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:41.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:41.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:36:41.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:41.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:36:41.575 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:41.595 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:36:41.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:41.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:41.632 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:36:41.699 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:36:41.712 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:41.746 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:36:41.790 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:36:41.800 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:41.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:41.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:36:41.833 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:36:41.863 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:36:41.952 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:36:41.953 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:36:41.953 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:36:41.953 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:36:41.953 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:36:41.953 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:36:41.953 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:36:41.954 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:36:41.954 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:36:42.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:42 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1728210349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:42 smithi190 ceph-mon[160435]: pgmap v1217: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 36925/215022 objects degraded (17.173%); 23182/215022 objects misplaced (10.781%); 16 MiB/s, 252 objects/s recovering 2023-10-01T09:36:42.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:42 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1728210349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:42.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:42 smithi138 ceph-mon[185886]: pgmap v1217: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 36925/215022 objects degraded (17.173%); 23182/215022 objects misplaced (10.781%); 16 MiB/s, 252 objects/s recovering 2023-10-01T09:36:42.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:42 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1728210349' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:42.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:42 smithi138 ceph-mon[192513]: pgmap v1217: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 36925/215022 objects degraded (17.173%); 23182/215022 objects misplaced (10.781%); 16 MiB/s, 252 objects/s recovering 2023-10-01T09:36:43.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:43 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 36925/215022 objects degraded (17.173%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:43.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:43 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 36925/215022 objects degraded (17.173%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:43.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:43 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 36925/215022 objects degraded (17.173%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:43.909 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:43.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:43.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:44.020 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:36:44.058 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:44.116 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:36:44.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:44.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:36:44.223 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:36:44.251 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:44.285 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:44.298 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:36:44.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:44.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:36:44.405 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:44.496 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:44 smithi138 ceph-mon[192513]: pgmap v1218: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 36925/215022 objects degraded (17.173%); 23182/215022 objects misplaced (10.781%); 15 MiB/s, 246 objects/s recovering 2023-10-01T09:36:44.496 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:44 smithi138 ceph-mon[185886]: pgmap v1218: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 36925/215022 objects degraded (17.173%); 23182/215022 objects misplaced (10.781%); 15 MiB/s, 246 objects/s recovering 2023-10-01T09:36:44.522 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:44 smithi190 ceph-mon[160435]: pgmap v1218: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 36925/215022 objects degraded (17.173%); 23182/215022 objects misplaced (10.781%); 15 MiB/s, 246 objects/s recovering 2023-10-01T09:36:44.906 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:44.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:44.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:45.019 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:36:45.050 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:45.100 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:36:45.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:45.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:36:45.135 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:36:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:36:44] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.33.4" 2023-10-01T09:36:45.205 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:36:45.235 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:45.291 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:36:45.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:45.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:36:45.400 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:36:45.475 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:46.278 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:36:47.176 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:36:47.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:47.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:36:47.282 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:36:47.442 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:47.442 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:47 smithi138 ceph-mon[192513]: pgmap v1219: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 36526/210720 objects degraded (17.334%); 22690/210720 objects misplaced (10.768%); 16 MiB/s, 248 objects/s recovering 2023-10-01T09:36:47.442 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:47.442 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:47 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:47.452 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:47 smithi138 ceph-mon[185886]: pgmap v1219: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 36526/210720 objects degraded (17.334%); 22690/210720 objects misplaced (10.768%); 16 MiB/s, 248 objects/s recovering 2023-10-01T09:36:47.453 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:47.453 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:47 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:47.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:47 smithi190 ceph-mon[160435]: pgmap v1219: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 36526/210720 objects degraded (17.334%); 22690/210720 objects misplaced (10.768%); 16 MiB/s, 248 objects/s recovering 2023-10-01T09:36:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:47 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:47.621 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:36:47.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:47.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:36:47.738 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:47.766 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:48.304 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:48.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:48.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:48.414 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:48 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 36526/210720 objects degraded (17.334%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:48.415 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:48 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 36526/210720 objects degraded (17.334%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:48.431 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:36:48.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:48 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 36526/210720 objects degraded (17.334%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:48.599 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:48.921 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:36:48.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:48.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:36:49.007 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:49.054 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:36:49.084 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:49.142 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:36:49.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:49.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:36:49.214 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:36:49.215 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:36:49.220 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:49 smithi138 ceph-mon[185886]: pgmap v1220: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 312 op/s; 36526/209814 objects degraded (17.409%); 21732/209814 objects misplaced (10.358%); 18 MiB/s, 288 objects/s recovering 2023-10-01T09:36:49.220 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:36:49.221 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:36:49.221 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:49.221 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4015920024' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T09:36:49.221 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:49 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T09:36:49.228 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:49 smithi138 ceph-mon[192513]: pgmap v1220: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 312 op/s; 36526/209814 objects degraded (17.409%); 21732/209814 objects misplaced (10.358%); 18 MiB/s, 288 objects/s recovering 2023-10-01T09:36:49.228 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:36:49.228 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:36:49.228 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:49.228 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4015920024' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T09:36:49.228 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:49 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T09:36:49.249 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:36:49.286 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:49.348 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:36:49.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:49.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:36:49.360 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:49 smithi190 ceph-mon[160435]: pgmap v1220: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 312 op/s; 36526/209814 objects degraded (17.409%); 21732/209814 objects misplaced (10.358%); 18 MiB/s, 288 objects/s recovering 2023-10-01T09:36:49.361 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:36:49.361 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:36:49.361 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:49.361 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4015920024' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T09:36:49.362 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:49 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T09:36:49.461 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:36:49.490 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:49.543 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:36:49.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:49.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:36:49.649 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:49.688 INFO:teuthology.orchestra.run.smithi138.stderr:set pool 9 pgp_num to 16 2023-10-01T09:36:49.800 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:50.767 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:36:50.817 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:50.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:50.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:50.922 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:36:50.959 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:50 smithi190 ceph-mon[160435]: failed to encode map e437 with expected crc 2023-10-01T09:36:50.959 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:50 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T09:36:50.959 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:50 smithi190 ceph-mon[160435]: osdmap e437: 8 total, 8 up, 8 in 2023-10-01T09:36:50.959 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:50 smithi190 ceph-mon[160435]: failed to encode map e437 with expected crc 2023-10-01T09:36:50.959 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:50 smithi190 ceph-mon[160435]: failed to encode map e437 with expected crc 2023-10-01T09:36:50.959 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:50 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3069416438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:36:50.959 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:50 smithi190 ceph-mon[160435]: pgmap v1222: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 36116/205458 objects degraded (17.578%); 21220/205458 objects misplaced (10.328%); 16 MiB/s, 258 objects/s recovering 2023-10-01T09:36:50.960 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:50.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:50 smithi138 ceph-mon[185886]: failed to encode map e437 with expected crc 2023-10-01T09:36:50.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:50 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T09:36:50.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:50 smithi138 ceph-mon[185886]: osdmap e437: 8 total, 8 up, 8 in 2023-10-01T09:36:50.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:50 smithi138 ceph-mon[185886]: failed to encode map e437 with expected crc 2023-10-01T09:36:50.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:50 smithi138 ceph-mon[185886]: failed to encode map e437 with expected crc 2023-10-01T09:36:50.997 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:50 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3069416438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:36:50.998 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:50 smithi138 ceph-mon[185886]: pgmap v1222: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 36116/205458 objects degraded (17.578%); 21220/205458 objects misplaced (10.328%); 16 MiB/s, 258 objects/s recovering 2023-10-01T09:36:50.998 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:50 smithi138 ceph-mon[192513]: failed to encode map e437 with expected crc 2023-10-01T09:36:50.998 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:50 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T09:36:50.998 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:50 smithi138 ceph-mon[192513]: osdmap e437: 8 total, 8 up, 8 in 2023-10-01T09:36:50.998 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:50 smithi138 ceph-mon[192513]: failed to encode map e437 with expected crc 2023-10-01T09:36:50.998 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:50 smithi138 ceph-mon[192513]: failed to encode map e437 with expected crc 2023-10-01T09:36:50.998 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:50 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3069416438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:36:50.998 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:50 smithi138 ceph-mon[192513]: pgmap v1222: 89 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 36116/205458 objects degraded (17.578%); 21220/205458 objects misplaced (10.328%); 16 MiB/s, 258 objects/s recovering 2023-10-01T09:36:51.022 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:36:51.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:51.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:36:51.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:36:51.201 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:51.601 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:36:52.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:51 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:36:52.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:51 smithi190 ceph-mon[160435]: failed to encode map e438 with expected crc 2023-10-01T09:36:52.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:51 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3069416438' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:36:52.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:51 smithi190 ceph-mon[160435]: osdmap e438: 8 total, 8 up, 8 in 2023-10-01T09:36:52.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:51 smithi190 ceph-mon[160435]: failed to encode map e438 with expected crc 2023-10-01T09:36:52.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:51 smithi190 ceph-mon[160435]: failed to encode map e438 with expected crc 2023-10-01T09:36:52.061 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:51 smithi190 ceph-mon[160435]: failed to encode map e438 with expected crc 2023-10-01T09:36:52.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:51 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:36:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:51 smithi138 ceph-mon[192513]: failed to encode map e438 with expected crc 2023-10-01T09:36:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:51 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3069416438' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:36:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:51 smithi138 ceph-mon[192513]: osdmap e438: 8 total, 8 up, 8 in 2023-10-01T09:36:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:51 smithi138 ceph-mon[192513]: failed to encode map e438 with expected crc 2023-10-01T09:36:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:51 smithi138 ceph-mon[192513]: failed to encode map e438 with expected crc 2023-10-01T09:36:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:51 smithi138 ceph-mon[192513]: failed to encode map e438 with expected crc 2023-10-01T09:36:52.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:51 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:36:52.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:51 smithi138 ceph-mon[185886]: failed to encode map e438 with expected crc 2023-10-01T09:36:52.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:51 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3069416438' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:36:52.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:51 smithi138 ceph-mon[185886]: osdmap e438: 8 total, 8 up, 8 in 2023-10-01T09:36:52.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:51 smithi138 ceph-mon[185886]: failed to encode map e438 with expected crc 2023-10-01T09:36:52.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:51 smithi138 ceph-mon[185886]: failed to encode map e438 with expected crc 2023-10-01T09:36:52.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:51 smithi138 ceph-mon[185886]: failed to encode map e438 with expected crc 2023-10-01T09:36:52.557 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:52.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:52.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:52.677 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:52.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:52 smithi138 ceph-mon[192513]: failed to encode map e439 with expected crc 2023-10-01T09:36:52.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:52 smithi138 ceph-mon[192513]: osdmap e439: 8 total, 8 up, 8 in 2023-10-01T09:36:52.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:52 smithi138 ceph-mon[192513]: failed to encode map e439 with expected crc 2023-10-01T09:36:52.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:52 smithi138 ceph-mon[192513]: pgmap v1225: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 36116/204591 objects degraded (17.653%); 20774/204591 objects misplaced (10.154%); 18 MiB/s, 292 objects/s recovering 2023-10-01T09:36:52.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:52 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 36116/205458 objects degraded (17.578%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:52.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:52 smithi138 ceph-mon[185886]: failed to encode map e439 with expected crc 2023-10-01T09:36:52.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:52 smithi138 ceph-mon[185886]: osdmap e439: 8 total, 8 up, 8 in 2023-10-01T09:36:52.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:52 smithi138 ceph-mon[185886]: failed to encode map e439 with expected crc 2023-10-01T09:36:52.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:52 smithi138 ceph-mon[185886]: pgmap v1225: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 36116/204591 objects degraded (17.653%); 20774/204591 objects misplaced (10.154%); 18 MiB/s, 292 objects/s recovering 2023-10-01T09:36:52.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:52 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 36116/205458 objects degraded (17.578%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:52.795 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:52.973 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:52.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:52.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:53.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:52 smithi190 ceph-mon[160435]: failed to encode map e439 with expected crc 2023-10-01T09:36:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:52 smithi190 ceph-mon[160435]: osdmap e439: 8 total, 8 up, 8 in 2023-10-01T09:36:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:52 smithi190 ceph-mon[160435]: failed to encode map e439 with expected crc 2023-10-01T09:36:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:52 smithi190 ceph-mon[160435]: pgmap v1225: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 340 op/s; 36116/204591 objects degraded (17.653%); 20774/204591 objects misplaced (10.154%); 18 MiB/s, 292 objects/s recovering 2023-10-01T09:36:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:52 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 36116/205458 objects degraded (17.578%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:53.080 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:36:53.113 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:53.155 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:53.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:53.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:53.259 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:36:53.296 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:53.361 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:36:53.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:53.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:36:53.470 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:36:53.552 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:53.747 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:36:53.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:53.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:36:53.854 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:53.868 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:54.020 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:54.354 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:54.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:54.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:54.482 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:36:54.513 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:36:54.514 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:54.558 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:36:54.579 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:36:54.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:54.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:36:54.605 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:36:54.648 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:36:54.685 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:36:54.691 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:36:54.713 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:54.736 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:36:54.752 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:36:54.771 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:36:54.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:54.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:36:54.779 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:36:54.820 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:36:54.833 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:36:54.833 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:36:54.834 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:36:54.834 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:36:54.834 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:36:54.834 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:36:54.834 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:36:54.834 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:36:54.876 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:36:54.911 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:54.965 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:36:54.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:54.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:36:55.074 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:55.188 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:55 smithi138 ceph-mon[185886]: pgmap v1226: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 290 op/s; 36116/204591 objects degraded (17.653%); 20774/204591 objects misplaced (10.154%); 8.5 MiB/s, 135 objects/s recovering 2023-10-01T09:36:55.188 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:55 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2634986842' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:55.188 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:36:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:36:54] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.33.4" 2023-10-01T09:36:55.188 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:55 smithi138 ceph-mon[192513]: pgmap v1226: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 290 op/s; 36116/204591 objects degraded (17.653%); 20774/204591 objects misplaced (10.154%); 8.5 MiB/s, 135 objects/s recovering 2023-10-01T09:36:55.188 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:55 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2634986842' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:55.207 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:55.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:55 smithi190 ceph-mon[160435]: pgmap v1226: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 290 op/s; 36116/204591 objects degraded (17.653%); 20774/204591 objects misplaced (10.154%); 8.5 MiB/s, 135 objects/s recovering 2023-10-01T09:36:55.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:55 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2634986842' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:36:55.885 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:55.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:55.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:55.993 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:36:56.071 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:56.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:56.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:36:56.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:56 smithi138 ceph-mon[185886]: pgmap v1227: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 35725/200406 objects degraded (17.826%); 20249/200406 objects misplaced (10.104%); 9.0 MiB/s, 144 objects/s recovering 2023-10-01T09:36:56.547 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:36:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:36:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:56 smithi138 ceph-mon[192513]: pgmap v1227: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 35725/200406 objects degraded (17.826%); 20249/200406 objects misplaced (10.104%); 9.0 MiB/s, 144 objects/s recovering 2023-10-01T09:36:56.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:56.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:36:56.677 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:56.777 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:56.778 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:36:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:36:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:56 smithi190 ceph-mon[160435]: pgmap v1227: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 35725/200406 objects degraded (17.826%); 20249/200406 objects misplaced (10.104%); 9.0 MiB/s, 144 objects/s recovering 2023-10-01T09:36:56.932 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:56.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:56.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:57.044 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:36:57.083 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:57.150 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:36:57.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:57.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:36:57.257 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:36:57.295 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:57.355 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:36:57.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:57.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:36:57.369 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:36:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:36:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:36:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:139271 9.4 e439/400 9:2c78c914:::benchmark_data_smithi138_497829_object36886:head v 439'8777, mlcod=439'8775)", 2023-10-01T09:36:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:36:57.349159+0000", 2023-10-01T09:36:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.010085462, 2023-10-01T09:36:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010117941, 2023-10-01T09:36:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:36:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:36:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:36:57.370 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:36:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:36:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.349159+0000", 2023-10-01T09:36:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:36:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:36:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:36:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:36:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.349159+0000", 2023-10-01T09:36:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:36:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:36:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:36:57.371 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:36:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.349158+0000", 2023-10-01T09:36:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:36:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:36:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:36:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:36:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.349162+0000", 2023-10-01T09:36:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7689999999999998e-06 2023-10-01T09:36:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:36:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:36:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:36:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.349168+0000", 2023-10-01T09:36:57.372 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4149999999999998e-06 2023-10-01T09:36:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:36:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:36:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:36:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.349172+0000", 2023-10-01T09:36:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0629999999999999e-06 2023-10-01T09:36:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:36:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:36:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:36:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.358473+0000", 2023-10-01T09:36:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0093011090000000001 2023-10-01T09:36:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:36:57.373 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:36:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:36:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.358495+0000", 2023-10-01T09:36:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2198999999999999e-05 2023-10-01T09:36:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:36:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:36:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:36:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:36:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:36:57.374 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 1 2023-10-01T09:36:57.374 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:36:57.460 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:36:57.489 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:57.555 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:36:57.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:57.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:36:57.566 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 3, 7, 5, 6, 4, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:36:57.567 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:36:57.567 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 3, 7, 5, 6, 4, 2] 2023-10-01T09:36:57.567 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd out 3 2023-10-01T09:36:57.661 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:36:57.692 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:57.756 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:36:57.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:57.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:36:57.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:57 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 35725/200406 objects degraded (17.826%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:57.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:57 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 35725/200406 objects degraded (17.826%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:57.864 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:36:57.891 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:57.956 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:36:57.962 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:57 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 35725/200406 objects degraded (17.826%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:36:57.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:57.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:36:58.064 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:36:58.167 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:58.582 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:36:59.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:36:58 smithi190 ceph-mon[160435]: pgmap v1228: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 35725/199383 objects degraded (17.918%); 20249/199383 objects misplaced (10.156%); 6.4 MiB/s, 101 objects/s recovering 2023-10-01T09:36:59.076 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:36:58 smithi138 ceph-mon[185886]: pgmap v1228: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 35725/199383 objects degraded (17.918%); 20249/199383 objects misplaced (10.156%); 6.4 MiB/s, 101 objects/s recovering 2023-10-01T09:36:59.076 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:36:58 smithi138 ceph-mon[192513]: pgmap v1228: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 35725/199383 objects degraded (17.918%); 20249/199383 objects misplaced (10.156%); 6.4 MiB/s, 101 objects/s recovering 2023-10-01T09:36:59.236 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:36:59.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:59.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:36:59.342 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:36:59.371 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:36:59.372 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:59.424 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:36:59.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:59.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:36:59.530 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:36:59.624 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:59.657 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:36:59.685 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:36:59.867 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:36:59.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:36:59.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:36:59.971 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:37:00.095 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:00.252 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:37:00.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:00.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:37:00.371 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:37:00.399 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:00.453 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:37:00.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:00.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:37:00.558 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:37:00.663 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:01.306 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:37:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:01 smithi190 ceph-mon[160435]: pgmap v1229: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 372 op/s; 35241/194859 objects degraded (18.085%); 19748/194859 objects misplaced (10.135%); 8.0 MiB/s, 128 objects/s recovering 2023-10-01T09:37:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1768473935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:37:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:01 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:37:01.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:01 smithi138 ceph-mon[192513]: pgmap v1229: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 372 op/s; 35241/194859 objects degraded (18.085%); 19748/194859 objects misplaced (10.135%); 8.0 MiB/s, 128 objects/s recovering 2023-10-01T09:37:01.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1768473935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:37:01.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:01 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:37:01.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:01 smithi138 ceph-mon[185886]: pgmap v1229: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 372 op/s; 35241/194859 objects degraded (18.085%); 19748/194859 objects misplaced (10.135%); 8.0 MiB/s, 128 objects/s recovering 2023-10-01T09:37:01.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1768473935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:37:01.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:01 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:37:02.472 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:37:02.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:02.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:37:02.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:02 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:37:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:02 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:37:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:02 smithi190 ceph-mon[160435]: failed to encode map e440 with expected crc 2023-10-01T09:37:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:02 smithi190 ceph-mon[160435]: osdmap e440: 8 total, 8 up, 8 in 2023-10-01T09:37:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:02 smithi190 ceph-mon[160435]: failed to encode map e440 with expected crc 2023-10-01T09:37:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:02 smithi190 ceph-mon[160435]: pgmap v1231: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 35241/193896 objects degraded (18.175%); 19748/193896 objects misplaced (10.185%); 4.8 MiB/s, 77 objects/s recovering 2023-10-01T09:37:02.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:02 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:37:02.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:02 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:37:02.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:02 smithi138 ceph-mon[185886]: failed to encode map e440 with expected crc 2023-10-01T09:37:02.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:02 smithi138 ceph-mon[185886]: osdmap e440: 8 total, 8 up, 8 in 2023-10-01T09:37:02.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:02 smithi138 ceph-mon[185886]: failed to encode map e440 with expected crc 2023-10-01T09:37:02.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:02 smithi138 ceph-mon[185886]: pgmap v1231: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 35241/193896 objects degraded (18.175%); 19748/193896 objects misplaced (10.185%); 4.8 MiB/s, 77 objects/s recovering 2023-10-01T09:37:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:02 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:37:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:02 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:37:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:02 smithi138 ceph-mon[192513]: failed to encode map e440 with expected crc 2023-10-01T09:37:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:02 smithi138 ceph-mon[192513]: osdmap e440: 8 total, 8 up, 8 in 2023-10-01T09:37:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:02 smithi138 ceph-mon[192513]: failed to encode map e440 with expected crc 2023-10-01T09:37:02.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:02 smithi138 ceph-mon[192513]: pgmap v1231: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 35241/193896 objects degraded (18.175%); 19748/193896 objects misplaced (10.185%); 4.8 MiB/s, 77 objects/s recovering 2023-10-01T09:37:02.588 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:02.616 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:02.683 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:02.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:02.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:02.789 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:37:02.885 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:03.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:03 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 35241/193896 objects degraded (18.175%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:03.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:03 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 35241/193896 objects degraded (18.175%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:03.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:03 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 35241/193896 objects degraded (18.175%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:03.890 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:37:03.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:03.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:37:03.998 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:04.033 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:04.093 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:04.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:04.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:04.201 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:04.262 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:04.372 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:04.450 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:37:04.490 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:37:04.537 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:37:04.581 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:37:04.613 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:37:04.625 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:04 smithi138 ceph-mon[192513]: pgmap v1232: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 35241/193896 objects degraded (18.175%); 19748/193896 objects misplaced (10.185%); 4.8 MiB/s, 77 objects/s recovering 2023-10-01T09:37:04.626 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:04 smithi138 ceph-mon[185886]: pgmap v1232: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 35241/193896 objects degraded (18.175%); 19748/193896 objects misplaced (10.185%); 4.8 MiB/s, 77 objects/s recovering 2023-10-01T09:37:04.626 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:04 smithi138 conmon[185862]: 2023-10-01T09:37:04.621+0000 7f70d64ec700 -1 mon.a@0(leader).osd e440 definitely_dead 0 2023-10-01T09:37:04.659 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:04.667 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:37:04.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:04.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:04.707 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:37:04.774 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:37:04.775 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:37:04.805 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:37:04.805 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:37:04.805 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:37:04.805 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:37:04.805 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:37:04.805 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:37:04.805 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:37:04.806 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:37:04.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:04 smithi190 ceph-mon[160435]: pgmap v1232: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 35241/193896 objects degraded (18.175%); 19748/193896 objects misplaced (10.185%); 4.8 MiB/s, 77 objects/s recovering 2023-10-01T09:37:04.884 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:04.936 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:37:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:37:04] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.33.4" 2023-10-01T09:37:05.391 INFO:teuthology.orchestra.run.smithi138.stderr:marked out osd.3. 2023-10-01T09:37:05.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3507005145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1506466504' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-01T09:37:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:05 smithi190 ceph-mon[160435]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-01T09:37:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3507005145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1506466504' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-01T09:37:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:05 smithi138 ceph-mon[185886]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-01T09:37:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3507005145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1506466504' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-10-01T09:37:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:05 smithi138 ceph-mon[192513]: Client client.admin marked osd.3 out, while it was still marked up 2023-10-01T09:37:05.952 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:37:06.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:06 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1506466504' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-01T09:37:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:06 smithi190 ceph-mon[160435]: osdmap e441: 8 total, 8 up, 7 in 2023-10-01T09:37:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:06 smithi190 ceph-mon[160435]: failed to encode map e441 with expected crc 2023-10-01T09:37:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:06 smithi190 ceph-mon[160435]: failed to encode map e441 with expected crc 2023-10-01T09:37:06.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:06 smithi190 ceph-mon[160435]: pgmap v1234: 89 pgs: 1 unknown, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 34814/177555 objects degraded (19.607%); 19249/177555 objects misplaced (10.841%) 2023-10-01T09:37:06.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:06 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1506466504' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-01T09:37:06.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:06 smithi138 ceph-mon[185886]: osdmap e441: 8 total, 8 up, 7 in 2023-10-01T09:37:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:06 smithi138 ceph-mon[185886]: failed to encode map e441 with expected crc 2023-10-01T09:37:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:06 smithi138 ceph-mon[185886]: failed to encode map e441 with expected crc 2023-10-01T09:37:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:06 smithi138 ceph-mon[185886]: pgmap v1234: 89 pgs: 1 unknown, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 34814/177555 objects degraded (19.607%); 19249/177555 objects misplaced (10.841%) 2023-10-01T09:37:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:06 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1506466504' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-10-01T09:37:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:06 smithi138 ceph-mon[192513]: osdmap e441: 8 total, 8 up, 7 in 2023-10-01T09:37:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:06 smithi138 ceph-mon[192513]: failed to encode map e441 with expected crc 2023-10-01T09:37:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:06 smithi138 ceph-mon[192513]: failed to encode map e441 with expected crc 2023-10-01T09:37:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:06 smithi138 ceph-mon[192513]: pgmap v1234: 89 pgs: 1 unknown, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 34814/177555 objects degraded (19.607%); 19249/177555 objects misplaced (10.841%) 2023-10-01T09:37:07.800 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:37:07.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:07 smithi190 ceph-mon[160435]: failed to encode map e442 with expected crc 2023-10-01T09:37:07.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:07 smithi190 ceph-mon[160435]: failed to encode map e442 with expected crc 2023-10-01T09:37:07.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:07 smithi190 ceph-mon[160435]: osdmap e442: 8 total, 8 up, 7 in 2023-10-01T09:37:07.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:07 smithi190 ceph-mon[160435]: failed to encode map e442 with expected crc 2023-10-01T09:37:07.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:07 smithi190 ceph-mon[160435]: failed to encode map e442 with expected crc 2023-10-01T09:37:07.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:07.809 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:07 smithi138 ceph-mon[185886]: failed to encode map e442 with expected crc 2023-10-01T09:37:07.809 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:07 smithi138 ceph-mon[185886]: failed to encode map e442 with expected crc 2023-10-01T09:37:07.810 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:07 smithi138 ceph-mon[185886]: osdmap e442: 8 total, 8 up, 7 in 2023-10-01T09:37:07.810 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:07 smithi138 ceph-mon[185886]: failed to encode map e442 with expected crc 2023-10-01T09:37:07.810 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:07 smithi138 ceph-mon[185886]: failed to encode map e442 with expected crc 2023-10-01T09:37:07.810 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:07 smithi138 ceph-mon[192513]: failed to encode map e442 with expected crc 2023-10-01T09:37:07.810 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:07 smithi138 ceph-mon[192513]: failed to encode map e442 with expected crc 2023-10-01T09:37:07.810 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:07 smithi138 ceph-mon[192513]: osdmap e442: 8 total, 8 up, 7 in 2023-10-01T09:37:07.810 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:07 smithi138 ceph-mon[192513]: failed to encode map e442 with expected crc 2023-10-01T09:37:07.810 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:07 smithi138 ceph-mon[192513]: failed to encode map e442 with expected crc 2023-10-01T09:37:07.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:37:07.912 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:07.941 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:08.018 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:08.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:08.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:08.124 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:37:08.164 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:08.240 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:37:08.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:08.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:37:08.347 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:08.477 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: osdmap e443: 8 total, 8 up, 7 in 2023-10-01T09:37:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: 9.1 starting backfill to osd.2 from (0'0,0'0] MAX to 440'8642 2023-10-01T09:37:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 440'8952 2023-10-01T09:37:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 440'8725 2023-10-01T09:37:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 440'8998 2023-10-01T09:37:08.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 440'8844 2023-10-01T09:37:08.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: 9.f starting backfill to osd.2 from (0'0,0'0] MAX to 440'8880 2023-10-01T09:37:08.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 34814/177555 objects degraded (19.607%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:08.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:08 smithi190 ceph-mon[160435]: pgmap v1237: 89 pgs: 3 remapped+peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 34817/188664 objects degraded (18.455%); 19249/188664 objects misplaced (10.203%); 4.5 MiB/s, 63 objects/s recovering 2023-10-01T09:37:08.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: osdmap e443: 8 total, 8 up, 7 in 2023-10-01T09:37:08.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: 9.1 starting backfill to osd.2 from (0'0,0'0] MAX to 440'8642 2023-10-01T09:37:08.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 440'8952 2023-10-01T09:37:08.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 440'8725 2023-10-01T09:37:08.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 440'8998 2023-10-01T09:37:08.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 440'8844 2023-10-01T09:37:08.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: 9.f starting backfill to osd.2 from (0'0,0'0] MAX to 440'8880 2023-10-01T09:37:08.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 34814/177555 objects degraded (19.607%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:08.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[192513]: pgmap v1237: 89 pgs: 3 remapped+peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 34817/188664 objects degraded (18.455%); 19249/188664 objects misplaced (10.203%); 4.5 MiB/s, 63 objects/s recovering 2023-10-01T09:37:08.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: osdmap e443: 8 total, 8 up, 7 in 2023-10-01T09:37:08.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: 9.1 starting backfill to osd.2 from (0'0,0'0] MAX to 440'8642 2023-10-01T09:37:08.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 440'8952 2023-10-01T09:37:08.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 440'8725 2023-10-01T09:37:08.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: 9.c starting backfill to osd.0 from (0'0,0'0] MAX to 440'8998 2023-10-01T09:37:08.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 440'8844 2023-10-01T09:37:08.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: failed to encode map e443 with expected crc 2023-10-01T09:37:08.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: 9.f starting backfill to osd.2 from (0'0,0'0] MAX to 440'8880 2023-10-01T09:37:08.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 34814/177555 objects degraded (19.607%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:08.824 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:08 smithi138 ceph-mon[185886]: pgmap v1237: 89 pgs: 3 remapped+peering, 1 active+recovering+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 290 op/s; 34817/188664 objects degraded (18.455%); 19249/188664 objects misplaced (10.203%); 4.5 MiB/s, 63 objects/s recovering 2023-10-01T09:37:09.281 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:09.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:09.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:09.393 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:37:09.519 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:09.530 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:09.708 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:37:09.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:09.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:37:09.822 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:37:09.931 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:10.047 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:37:10.047 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:37:10.085 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:37:10.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:10.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:37:10.190 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:10.295 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:11.129 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:37:11.512 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:11 smithi138 ceph-mon[185886]: pgmap v1238: 89 pgs: 2 active+recovering+undersized+remapped, 3 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 16023/186084 objects degraded (8.611%); 39732/186084 objects misplaced (21.352%); 8.6 MiB/s, 129 objects/s recovering 2023-10-01T09:37:11.512 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:37:11.512 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:37:11.512 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:11 smithi138 ceph-mon[192513]: pgmap v1238: 89 pgs: 2 active+recovering+undersized+remapped, 3 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 16023/186084 objects degraded (8.611%); 39732/186084 objects misplaced (21.352%); 8.6 MiB/s, 129 objects/s recovering 2023-10-01T09:37:11.512 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:37:11.512 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:37:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:11 smithi190 ceph-mon[160435]: pgmap v1238: 89 pgs: 2 active+recovering+undersized+remapped, 3 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 16023/186084 objects degraded (8.611%); 39732/186084 objects misplaced (21.352%); 8.6 MiB/s, 129 objects/s recovering 2023-10-01T09:37:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:37:11.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:37:12.250 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:12.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:12.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:12.297 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:37:12.366 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:12.459 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:12.518 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:12.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:12.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:12.570 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:12.624 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:12.751 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:12.935 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:12.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:12.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:13.045 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:37:13.086 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:13.149 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:37:13.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:13.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:37:13.255 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:13.284 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:13.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:13 smithi138 ceph-mon[185886]: pgmap v1239: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 16023/185307 objects degraded (8.647%); 47315/185307 objects misplaced (25.533%); 5.1 MiB/s, 75 objects/s recovering 2023-10-01T09:37:13.325 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:13 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 16023/185307 objects degraded (8.647%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:13.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:13 smithi138 ceph-mon[192513]: pgmap v1239: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 16023/185307 objects degraded (8.647%); 47315/185307 objects misplaced (25.533%); 5.1 MiB/s, 75 objects/s recovering 2023-10-01T09:37:13.325 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:13 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 16023/185307 objects degraded (8.647%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:13.352 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:13.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:13.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:13.414 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:13 smithi190 ceph-mon[160435]: pgmap v1239: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 16023/185307 objects degraded (8.647%); 47315/185307 objects misplaced (25.533%); 5.1 MiB/s, 75 objects/s recovering 2023-10-01T09:37:13.414 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:13 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 16023/185307 objects degraded (8.647%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:13.515 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:37:13.549 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:13.615 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:37:13.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:13.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:37:13.721 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:37:13.803 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:13.885 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:37:13.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:13.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:37:13.996 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:14.046 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:14.111 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:14.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:14.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:14.217 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:37:14.293 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:14.294 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:37:14.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3924928430' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:14.402 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 2] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:37:14.402 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:37:14.402 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-10-01T09:37:14.402 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd in 3 2023-10-01T09:37:14.514 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:14.544 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3924928430' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:14.544 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3924928430' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:14.655 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:37:14.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:14.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:37:14.771 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:37:14.808 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:14.851 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:37:14.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:14.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:37:14.958 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:14.996 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:15.078 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:15.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:15.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:15.204 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:37:15.243 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:15.261 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:37:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:37:14] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.33.4" 2023-10-01T09:37:15.264 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:15 smithi138 ceph-mon[185886]: pgmap v1240: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 49 MiB/s wr, 1.25k op/s; 16023/184962 objects degraded (8.663%); 51074/184962 objects misplaced (27.613%); 39 MiB/s, 622 objects/s recovering 2023-10-01T09:37:15.264 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:15 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:37:15.264 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:15 smithi138 ceph-mon[185886]: failed to encode map e444 with expected crc 2023-10-01T09:37:15.264 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:15 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3924928430' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:37:15.264 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:15 smithi138 ceph-mon[185886]: osdmap e444: 8 total, 8 up, 7 in 2023-10-01T09:37:15.264 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:15 smithi138 ceph-mon[185886]: failed to encode map e444 with expected crc 2023-10-01T09:37:15.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:15 smithi138 ceph-mon[185886]: failed to encode map e444 with expected crc 2023-10-01T09:37:15.341 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:37:15.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:15.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:37:15.452 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:15.556 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:15 smithi138 ceph-mon[192513]: pgmap v1240: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 49 MiB/s wr, 1.25k op/s; 16023/184962 objects degraded (8.663%); 51074/184962 objects misplaced (27.613%); 39 MiB/s, 622 objects/s recovering 2023-10-01T09:37:15.556 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:15 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:37:15.556 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:15 smithi138 ceph-mon[192513]: failed to encode map e444 with expected crc 2023-10-01T09:37:15.556 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:15 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3924928430' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:37:15.556 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:15 smithi138 ceph-mon[192513]: osdmap e444: 8 total, 8 up, 7 in 2023-10-01T09:37:15.557 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:15 smithi138 ceph-mon[192513]: failed to encode map e444 with expected crc 2023-10-01T09:37:15.557 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:15 smithi138 ceph-mon[192513]: failed to encode map e444 with expected crc 2023-10-01T09:37:15.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:15 smithi190 ceph-mon[160435]: pgmap v1240: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 49 MiB/s wr, 1.25k op/s; 16023/184962 objects degraded (8.663%); 51074/184962 objects misplaced (27.613%); 39 MiB/s, 622 objects/s recovering 2023-10-01T09:37:15.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:15 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:37:15.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:15 smithi190 ceph-mon[160435]: failed to encode map e444 with expected crc 2023-10-01T09:37:15.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:15 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3924928430' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:37:15.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:15 smithi190 ceph-mon[160435]: osdmap e444: 8 total, 8 up, 7 in 2023-10-01T09:37:15.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:15 smithi190 ceph-mon[160435]: failed to encode map e444 with expected crc 2023-10-01T09:37:15.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:15 smithi190 ceph-mon[160435]: failed to encode map e444 with expected crc 2023-10-01T09:37:15.702 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:15.791 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:15.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:15.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:15.898 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:37:15.933 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:15.962 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:16.000 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:37:16.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:16.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:37:16.056 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:37:16.092 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:37:16.107 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:37:16.141 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:16.158 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:37:16.202 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:37:16.207 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:37:16.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:16.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:37:16.237 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:37:16.285 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:37:16.298 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:16 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1329934801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:16.298 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:16 smithi138 ceph-mon[192513]: pgmap v1242: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 326 op/s; 15888/180048 objects degraded (8.824%); 50053/180048 objects misplaced (27.800%); 5.9 MiB/s, 95 objects/s recovering 2023-10-01T09:37:16.298 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:16 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1329934801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:16.298 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:16 smithi138 ceph-mon[185886]: pgmap v1242: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 326 op/s; 15888/180048 objects degraded (8.824%); 50053/180048 objects misplaced (27.800%); 5.9 MiB/s, 95 objects/s recovering 2023-10-01T09:37:16.315 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:16.321 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:37:16.350 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:16.363 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:37:16.379 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:37:16.379 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:37:16.380 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:37:16.380 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:37:16.380 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:37:16.380 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:37:16.380 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:37:16.380 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:37:16.427 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:16.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:16.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:16.536 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:37:16.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:16 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1329934801' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:16.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:16 smithi190 ceph-mon[160435]: pgmap v1242: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 326 op/s; 15888/180048 objects degraded (8.824%); 50053/180048 objects misplaced (27.800%); 5.9 MiB/s, 95 objects/s recovering 2023-10-01T09:37:16.568 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:16.613 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:37:16.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:16.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:37:16.718 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:16.743 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:16.796 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:16.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:16.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:16.902 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:16.956 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:17.019 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:17.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:17.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:17.130 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:37:17.167 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:17.234 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:37:17.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:17.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:37:17.344 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:37:17.400 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:17.450 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:37:17.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:17 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 15888/180048 objects degraded (8.824%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:17.690 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:17 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 15888/180048 objects degraded (8.824%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:18.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:17 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 15888/180048 objects degraded (8.824%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:18.093 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:37:18.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:18.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:37:18.201 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:37:18.280 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:18.601 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:37:18.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:18.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:37:18.691 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:18 smithi138 ceph-mon[185886]: pgmap v1243: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 320 op/s; 15888/179025 objects degraded (8.875%); 49602/179025 objects misplaced (27.707%); 5.9 MiB/s, 95 objects/s recovering 2023-10-01T09:37:18.692 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:18 smithi138 ceph-mon[192513]: pgmap v1243: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 320 op/s; 15888/179025 objects degraded (8.875%); 49602/179025 objects misplaced (27.707%); 5.9 MiB/s, 95 objects/s recovering 2023-10-01T09:37:18.709 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:37:18.798 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:19.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:18 smithi190 ceph-mon[160435]: pgmap v1243: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 320 op/s; 15888/179025 objects degraded (8.875%); 49602/179025 objects misplaced (27.707%); 5.9 MiB/s, 95 objects/s recovering 2023-10-01T09:37:19.385 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:37:19.385 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:37:19.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:37:19.386 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:37:19.386 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.386 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:187123 8.8 8:1531d290:::benchmark_data_smithi138_431617_object85712:head [delete] snapc 0=[] ondisk+write+known_if_redirected e384)", 2023-10-01T09:37:19.386 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:18.925346+0000", 2023-10-01T09:37:19.386 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 240.456934504, 2023-10-01T09:37:19.386 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18660890799999999, 2023-10-01T09:37:19.386 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.386 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.386 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:37:19.387 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:37:19.387 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:19.387 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 187123 2023-10-01T09:37:19.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.387 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.387 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925346+0000", 2023-10-01T09:37:19.387 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.387 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.387 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.387 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925346+0000", 2023-10-01T09:37:19.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925344+0000", 2023-10-01T09:37:19.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:19.388 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.388 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.388 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.388 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925353+0000", 2023-10-01T09:37:19.388 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3270000000000001e-06 2023-10-01T09:37:19.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925373+0000", 2023-10-01T09:37:19.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9842000000000001e-05 2023-10-01T09:37:19.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.389 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.389 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925381+0000", 2023-10-01T09:37:19.389 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5310000000000001e-06 2023-10-01T09:37:19.389 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.389 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925432+0000", 2023-10-01T09:37:19.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1103999999999997e-05 2023-10-01T09:37:19.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925571+0000", 2023-10-01T09:37:19.390 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013912700000000001 2023-10-01T09:37:19.390 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.390 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.390 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:37:19.390 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925723+0000", 2023-10-01T09:37:19.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000151879 2023-10-01T09:37:19.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:19.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.931279+0000", 2023-10-01T09:37:19.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0055557790000000003 2023-10-01T09:37:19.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.391 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.391 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:37:19.391 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.934123+0000", 2023-10-01T09:37:19.391 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028444490000000002 2023-10-01T09:37:19.391 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:19.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111698+0000", 2023-10-01T09:37:19.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.177574809 2023-10-01T09:37:19.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.392 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111751+0000", 2023-10-01T09:37:19.392 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2756000000000001e-05 2023-10-01T09:37:19.392 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.392 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.392 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.393 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111955+0000", 2023-10-01T09:37:19.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020406600000000001 2023-10-01T09:37:19.393 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.393 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.393 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.393 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.393 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.393 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:187129 8.8 8:155c408a:::benchmark_data_smithi138_431617_object85718:head [delete] snapc 0=[] ondisk+write+known_if_redirected e384)", 2023-10-01T09:37:19.393 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:18.936343+0000", 2023-10-01T09:37:19.393 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 240.445937544, 2023-10-01T09:37:19.393 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17579768900000001, 2023-10-01T09:37:19.393 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.394 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.394 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:37:19.394 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:37:19.394 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:19.394 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 187129 2023-10-01T09:37:19.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.394 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.394 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.394 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.394 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936343+0000", 2023-10-01T09:37:19.394 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.394 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936343+0000", 2023-10-01T09:37:19.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.395 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.395 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936341+0000", 2023-10-01T09:37:19.395 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:19.395 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.395 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936350+0000", 2023-10-01T09:37:19.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6170000000000007e-06 2023-10-01T09:37:19.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936369+0000", 2023-10-01T09:37:19.396 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9264999999999999e-05 2023-10-01T09:37:19.396 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.396 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.396 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.396 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936379+0000", 2023-10-01T09:37:19.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7550000000000006e-06 2023-10-01T09:37:19.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936435+0000", 2023-10-01T09:37:19.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5951e-05 2023-10-01T09:37:19.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.397 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.397 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.397 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936602+0000", 2023-10-01T09:37:19.397 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000167446 2023-10-01T09:37:19.397 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.398 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:37:19.398 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936694+0000", 2023-10-01T09:37:19.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2208999999999998e-05 2023-10-01T09:37:19.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.398 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:37:19.398 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.940157+0000", 2023-10-01T09:37:19.398 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034625760000000002 2023-10-01T09:37:19.398 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.398 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:19.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.941169+0000", 2023-10-01T09:37:19.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001012352 2023-10-01T09:37:19.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:19.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111993+0000", 2023-10-01T09:37:19.399 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17082314400000001 2023-10-01T09:37:19.399 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.399 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.399 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.399 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.112030+0000", 2023-10-01T09:37:19.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7453000000000002e-05 2023-10-01T09:37:19.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.400 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.400 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.112141+0000", 2023-10-01T09:37:19.400 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110803 2023-10-01T09:37:19.400 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.400 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.400 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.400 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.400 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.400 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:50898 9.7 e409/400 9:e780c3ef:::benchmark_data_smithi138_497829_object50897:head v 409'3182, mlcod=409'3180)", 2023-10-01T09:37:19.401 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:34:44.818857+0000", 2023-10-01T09:37:19.401 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 154.56342379700001, 2023-10-01T09:37:19.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.175987492, 2023-10-01T09:37:19.401 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.401 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.401 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.401 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.401 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818857+0000", 2023-10-01T09:37:19.401 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.401 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.401 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818857+0000", 2023-10-01T09:37:19.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.402 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818856+0000", 2023-10-01T09:37:19.402 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:37:19.402 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.402 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.402 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818901+0000", 2023-10-01T09:37:19.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4372000000000001e-05 2023-10-01T09:37:19.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.403 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818909+0000", 2023-10-01T09:37:19.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6240000000000008e-06 2023-10-01T09:37:19.403 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.403 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.403 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.403 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818915+0000", 2023-10-01T09:37:19.403 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0249999999999999e-06 2023-10-01T09:37:19.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818976+0000", 2023-10-01T09:37:19.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1261999999999997e-05 2023-10-01T09:37:19.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.404 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.404 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818992+0000", 2023-10-01T09:37:19.404 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5268e-05 2023-10-01T09:37:19.404 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.404 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.405 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.405 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.994803+0000", 2023-10-01T09:37:19.405 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17581116399999999 2023-10-01T09:37:19.405 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.405 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.406 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.406 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.994844+0000", 2023-10-01T09:37:19.406 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1482e-05 2023-10-01T09:37:19.406 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.406 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.406 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.406 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.406 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.406 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:50900 9.1 e409/400 9:8a8ebdf9:::benchmark_data_smithi138_497829_object50899:head v 409'3075, mlcod=409'3073)", 2023-10-01T09:37:19.406 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:34:44.818933+0000", 2023-10-01T09:37:19.406 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 154.563347287, 2023-10-01T09:37:19.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.176232109, 2023-10-01T09:37:19.407 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.407 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.407 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.407 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818933+0000", 2023-10-01T09:37:19.407 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.407 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.407 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.407 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818933+0000", 2023-10-01T09:37:19.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818933+0000", 2023-10-01T09:37:19.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:37:19.408 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.408 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.408 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.408 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818985+0000", 2023-10-01T09:37:19.408 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2595e-05 2023-10-01T09:37:19.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818996+0000", 2023-10-01T09:37:19.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0823e-05 2023-10-01T09:37:19.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.409 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.409 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.819003+0000", 2023-10-01T09:37:19.409 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9539999999999998e-06 2023-10-01T09:37:19.409 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.409 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.992729+0000", 2023-10-01T09:37:19.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.173726357 2023-10-01T09:37:19.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.410 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.992748+0000", 2023-10-01T09:37:19.410 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8726e-05 2023-10-01T09:37:19.410 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.410 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.410 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.995125+0000", 2023-10-01T09:37:19.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002376534 2023-10-01T09:37:19.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.411 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.411 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.995165+0000", 2023-10-01T09:37:19.411 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0775000000000002e-05 2023-10-01T09:37:19.411 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.411 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.411 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.411 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.411 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.412 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:50903 9.4 e409/400 9:2f73ff33:::benchmark_data_smithi138_497829_object50902:head v 409'3191, mlcod=409'3189)", 2023-10-01T09:37:19.412 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:34:44.823126+0000", 2023-10-01T09:37:19.412 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 154.55915442899999, 2023-10-01T09:37:19.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17174679700000001, 2023-10-01T09:37:19.412 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.412 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.412 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.412 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.412 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.412 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.823126+0000", 2023-10-01T09:37:19.412 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.412 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.823126+0000", 2023-10-01T09:37:19.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.413 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.413 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.413 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.823125+0000", 2023-10-01T09:37:19.413 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:37:19.413 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.823150+0000", 2023-10-01T09:37:19.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4986000000000001e-05 2023-10-01T09:37:19.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.414 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.414 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.992731+0000", 2023-10-01T09:37:19.414 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16958084700000001 2023-10-01T09:37:19.414 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.414 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.992741+0000", 2023-10-01T09:37:19.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7769999999999997e-06 2023-10-01T09:37:19.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.992783+0000", 2023-10-01T09:37:19.415 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2511e-05 2023-10-01T09:37:19.415 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.415 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.415 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.415 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.992804+0000", 2023-10-01T09:37:19.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0214999999999999e-05 2023-10-01T09:37:19.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.994840+0000", 2023-10-01T09:37:19.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020361759999999998 2023-10-01T09:37:19.416 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.416 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.416 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.416 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.994873+0000", 2023-10-01T09:37:19.416 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3142000000000002e-05 2023-10-01T09:37:19.417 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.417 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.417 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.417 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.417 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.417 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:108564 9.1 e427/400 9:895d5137:::benchmark_data_smithi138_497829_object6179:head v 427'6627, mlcod=427'6625)", 2023-10-01T09:37:19.417 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:55.637242+0000", 2023-10-01T09:37:19.417 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 83.745038140000005, 2023-10-01T09:37:19.417 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.167424715, 2023-10-01T09:37:19.417 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.417 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.418 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.637242+0000", 2023-10-01T09:37:19.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.418 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.418 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.418 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.637242+0000", 2023-10-01T09:37:19.418 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.418 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.637242+0000", 2023-10-01T09:37:19.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:37:19.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.419 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.419 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.637250+0000", 2023-10-01T09:37:19.419 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3389999999999993e-06 2023-10-01T09:37:19.419 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.419 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.637260+0000", 2023-10-01T09:37:19.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6840000000000008e-06 2023-10-01T09:37:19.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.637266+0000", 2023-10-01T09:37:19.420 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7059999999999998e-06 2023-10-01T09:37:19.420 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.420 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.420 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.420 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.638253+0000", 2023-10-01T09:37:19.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00098667299999999998 2023-10-01T09:37:19.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.638273+0000", 2023-10-01T09:37:19.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0242999999999999e-05 2023-10-01T09:37:19.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.421 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.421 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.421 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.804633+0000", 2023-10-01T09:37:19.421 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16636009900000001 2023-10-01T09:37:19.421 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.422 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.422 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.804667+0000", 2023-10-01T09:37:19.422 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3862999999999997e-05 2023-10-01T09:37:19.422 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.422 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.422 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.422 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.422 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.422 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:108569 9.1 e427/400 9:8f07d951:::benchmark_data_smithi138_497829_object6184:head v 427'6628, mlcod=427'6625)", 2023-10-01T09:37:19.422 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:55.638919+0000", 2023-10-01T09:37:19.423 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 83.743361831000001, 2023-10-01T09:37:19.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.165768103, 2023-10-01T09:37:19.423 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.423 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.423 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.423 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.423 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.638919+0000", 2023-10-01T09:37:19.423 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.423 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.423 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.638919+0000", 2023-10-01T09:37:19.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.638918+0000", 2023-10-01T09:37:19.424 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:37:19.424 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.424 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.424 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.424 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.638928+0000", 2023-10-01T09:37:19.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7720000000000006e-06 2023-10-01T09:37:19.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.791286+0000", 2023-10-01T09:37:19.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.15235870800000001 2023-10-01T09:37:19.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.425 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.425 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.425 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.791299+0000", 2023-10-01T09:37:19.425 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2269999999999999e-05 2023-10-01T09:37:19.425 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.794740+0000", 2023-10-01T09:37:19.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034412140000000002 2023-10-01T09:37:19.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.426 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.794759+0000", 2023-10-01T09:37:19.426 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8938000000000001e-05 2023-10-01T09:37:19.426 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.426 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.426 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.804670+0000", 2023-10-01T09:37:19.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0099107410000000007 2023-10-01T09:37:19.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.427 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.427 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.427 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.804687+0000", 2023-10-01T09:37:19.427 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7297999999999999e-05 2023-10-01T09:37:19.427 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.427 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.427 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.427 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.428 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:132658 9.a e436/431 9:549aa041:::benchmark_data_smithi138_497829_object30273:head v 436'8159, mlcod=405'1569)", 2023-10-01T09:37:19.428 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:36:38.289459+0000", 2023-10-01T09:37:19.428 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 41.092821962999999, 2023-10-01T09:37:19.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.247702749, 2023-10-01T09:37:19.428 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.428 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.428 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.428 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.428 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.428 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289459+0000", 2023-10-01T09:37:19.428 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289459+0000", 2023-10-01T09:37:19.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.429 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.429 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289458+0000", 2023-10-01T09:37:19.429 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:37:19.429 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.429 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289466+0000", 2023-10-01T09:37:19.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5660000000000007e-06 2023-10-01T09:37:19.430 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.430 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.430 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.430 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289479+0000", 2023-10-01T09:37:19.430 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2714000000000001e-05 2023-10-01T09:37:19.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.431 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.431 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.431 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289486+0000", 2023-10-01T09:37:19.431 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.391e-06 2023-10-01T09:37:19.431 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.432 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289548+0000", 2023-10-01T09:37:19.432 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2012000000000002e-05 2023-10-01T09:37:19.432 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.432 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.432 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289575+0000", 2023-10-01T09:37:19.433 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6438e-05 2023-10-01T09:37:19.433 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.433 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.433 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.433 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537129+0000", 2023-10-01T09:37:19.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.247554408 2023-10-01T09:37:19.434 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.434 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.434 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.434 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537161+0000", 2023-10-01T09:37:19.434 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2141000000000003e-05 2023-10-01T09:37:19.435 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.435 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.435 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.435 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.435 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.435 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:132659 9.a e436/431 9:5345b610:::benchmark_data_smithi138_497829_object30274:head v 436'8160, mlcod=405'1569)", 2023-10-01T09:37:19.435 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:36:38.291551+0000", 2023-10-01T09:37:19.435 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 41.090729768999999, 2023-10-01T09:37:19.436 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24563221299999999, 2023-10-01T09:37:19.436 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.436 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.436 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.436 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.436 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.291551+0000", 2023-10-01T09:37:19.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.437 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.437 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.437 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.437 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.291551+0000", 2023-10-01T09:37:19.437 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.438 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.438 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.438 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.291551+0000", 2023-10-01T09:37:19.438 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:37:19.438 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.439 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.439 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.439 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.291554+0000", 2023-10-01T09:37:19.439 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7299999999999999e-06 2023-10-01T09:37:19.439 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.291560+0000", 2023-10-01T09:37:19.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4029999999999998e-06 2023-10-01T09:37:19.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.440 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.440 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.440 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.291564+0000", 2023-10-01T09:37:19.440 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8999999999999999e-06 2023-10-01T09:37:19.440 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.298221+0000", 2023-10-01T09:37:19.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066577329999999999 2023-10-01T09:37:19.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.441 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.441 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.441 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.298245+0000", 2023-10-01T09:37:19.441 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3175000000000001e-05 2023-10-01T09:37:19.441 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537164+0000", 2023-10-01T09:37:19.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.238919627 2023-10-01T09:37:19.442 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.442 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.442 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537183+0000", 2023-10-01T09:37:19.442 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8912000000000001e-05 2023-10-01T09:37:19.443 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.443 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.443 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.443 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:132664 9.4 e436/400 9:24a98379:::benchmark_data_smithi138_497829_object30279:head v 436'8340, mlcod=436'8338)", 2023-10-01T09:37:19.443 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:36:38.303519+0000", 2023-10-01T09:37:19.443 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 41.078761176, 2023-10-01T09:37:19.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23341430599999999, 2023-10-01T09:37:19.443 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.443 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.444 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.303519+0000", 2023-10-01T09:37:19.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.303519+0000", 2023-10-01T09:37:19.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.303518+0000", 2023-10-01T09:37:19.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:19.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.303528+0000", 2023-10-01T09:37:19.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6739999999999992e-06 2023-10-01T09:37:19.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532367+0000", 2023-10-01T09:37:19.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22883935999999999 2023-10-01T09:37:19.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532376+0000", 2023-10-01T09:37:19.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5339999999999999e-06 2023-10-01T09:37:19.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532538+0000", 2023-10-01T09:37:19.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016260299999999999 2023-10-01T09:37:19.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532561+0000", 2023-10-01T09:37:19.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2904e-05 2023-10-01T09:37:19.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.448 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.448 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.536896+0000", 2023-10-01T09:37:19.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043344639999999997 2023-10-01T09:37:19.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.448 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.448 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.536934+0000", 2023-10-01T09:37:19.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8022000000000003e-05 2023-10-01T09:37:19.448 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.449 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.449 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.449 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:132665 9.3 e436/400 9:c75edcf0:::benchmark_data_smithi138_497829_object30280:head v 436'8280, mlcod=436'8278)", 2023-10-01T09:37:19.449 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:36:38.306765+0000", 2023-10-01T09:37:19.449 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 41.075515897000002, 2023-10-01T09:37:19.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24329358100000001, 2023-10-01T09:37:19.449 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.449 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.449 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.306765+0000", 2023-10-01T09:37:19.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.306765+0000", 2023-10-01T09:37:19.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.306763+0000", 2023-10-01T09:37:19.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:19.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.306775+0000", 2023-10-01T09:37:19.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1369000000000001e-05 2023-10-01T09:37:19.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537194+0000", 2023-10-01T09:37:19.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23041979100000001 2023-10-01T09:37:19.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537208+0000", 2023-10-01T09:37:19.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3708e-05 2023-10-01T09:37:19.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537351+0000", 2023-10-01T09:37:19.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014312999999999999 2023-10-01T09:37:19.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537373+0000", 2023-10-01T09:37:19.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2087e-05 2023-10-01T09:37:19.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.453 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.453 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.549826+0000", 2023-10-01T09:37:19.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012452420000000001 2023-10-01T09:37:19.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.550058+0000", 2023-10-01T09:37:19.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000232527 2023-10-01T09:37:19.454 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.454 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.454 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.454 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:132647 9.f e436/400 9:f2582533:::benchmark_data_smithi138_497829_object30262:head v 436'8314, mlcod=436'8312)", 2023-10-01T09:37:19.455 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:36:38.318653+0000", 2023-10-01T09:37:19.455 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 41.063627769, 2023-10-01T09:37:19.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22333981, 2023-10-01T09:37:19.455 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.455 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.455 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.318653+0000", 2023-10-01T09:37:19.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.318653+0000", 2023-10-01T09:37:19.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.318652+0000", 2023-10-01T09:37:19.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:19.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.318661+0000", 2023-10-01T09:37:19.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8020000000000003e-06 2023-10-01T09:37:19.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532259+0000", 2023-10-01T09:37:19.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.213598017 2023-10-01T09:37:19.457 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532275+0000", 2023-10-01T09:37:19.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5883e-05 2023-10-01T09:37:19.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.458 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532434+0000", 2023-10-01T09:37:19.458 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000158286 2023-10-01T09:37:19.458 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.458 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.458 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.459 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532462+0000", 2023-10-01T09:37:19.459 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8693000000000001e-05 2023-10-01T09:37:19.459 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.459 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.459 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.541945+0000", 2023-10-01T09:37:19.459 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094831150000000003 2023-10-01T09:37:19.459 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.459 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.459 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.460 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.541993+0000", 2023-10-01T09:37:19.460 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7264e-05 2023-10-01T09:37:19.460 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.460 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.460 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.460 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.460 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.460 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:138340 9.a e439/431 9:5d96b39f:::benchmark_data_smithi138_497829_object35955:head v 439'8490, mlcod=405'1569)", 2023-10-01T09:37:19.460 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:36:54.739387+0000", 2023-10-01T09:37:19.460 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 24.642893758, 2023-10-01T09:37:19.461 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.049180310999999997, 2023-10-01T09:37:19.461 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.461 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.461 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:54.739387+0000", 2023-10-01T09:37:19.461 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.461 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.461 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.461 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.461 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:54.739387+0000", 2023-10-01T09:37:19.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.462 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.462 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:54.739386+0000", 2023-10-01T09:37:19.462 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:19.462 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.462 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.462 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:54.739409+0000", 2023-10-01T09:37:19.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3496e-05 2023-10-01T09:37:19.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.463 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.463 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:54.739431+0000", 2023-10-01T09:37:19.463 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2092000000000001e-05 2023-10-01T09:37:19.463 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.463 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:54.739441+0000", 2023-10-01T09:37:19.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5889999999999992e-06 2023-10-01T09:37:19.464 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.464 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.464 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.464 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:54.765070+0000", 2023-10-01T09:37:19.464 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.025629567999999998 2023-10-01T09:37:19.465 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.465 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.465 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:54.765101+0000", 2023-10-01T09:37:19.465 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0913e-05 2023-10-01T09:37:19.465 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.465 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.465 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.465 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:54.788509+0000", 2023-10-01T09:37:19.466 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.023407322000000001 2023-10-01T09:37:19.466 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.466 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.466 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.466 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:54.788567+0000", 2023-10-01T09:37:19.466 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8594999999999997e-05 2023-10-01T09:37:19.466 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.466 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.467 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.467 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.467 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.467 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:138843 9.a e439/431 9:509cf16c:::benchmark_data_smithi138_497829_object36458:head v 439'8516, mlcod=405'1569)", 2023-10-01T09:37:19.467 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:36:56.327215+0000", 2023-10-01T09:37:19.467 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 23.055065828, 2023-10-01T09:37:19.467 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.042182218, 2023-10-01T09:37:19.467 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.467 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.468 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:56.327215+0000", 2023-10-01T09:37:19.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.468 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.468 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.468 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.468 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:56.327215+0000", 2023-10-01T09:37:19.468 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.469 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:56.327213+0000", 2023-10-01T09:37:19.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:19.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.469 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.469 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:56.327224+0000", 2023-10-01T09:37:19.469 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0835999999999999e-05 2023-10-01T09:37:19.469 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.469 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:56.327246+0000", 2023-10-01T09:37:19.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1466000000000001e-05 2023-10-01T09:37:19.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.470 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.470 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.470 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:56.327255+0000", 2023-10-01T09:37:19.470 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0149999999999992e-06 2023-10-01T09:37:19.470 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:56.327351+0000", 2023-10-01T09:37:19.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6218999999999995e-05 2023-10-01T09:37:19.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.471 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.471 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:56.327378+0000", 2023-10-01T09:37:19.471 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7117000000000001e-05 2023-10-01T09:37:19.471 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.471 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:56.369349+0000", 2023-10-01T09:37:19.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.041970690999999997 2023-10-01T09:37:19.472 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.472 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.472 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.472 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:56.369397+0000", 2023-10-01T09:37:19.472 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8214000000000003e-05 2023-10-01T09:37:19.472 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.473 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.473 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.473 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.473 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.473 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:139289 9.c e439/400 9:3992cde0:::benchmark_data_smithi138_497829_object36904:head v 439'8819, mlcod=439'8816)", 2023-10-01T09:37:19.473 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:36:57.401968+0000", 2023-10-01T09:37:19.473 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 21.980312266999999, 2023-10-01T09:37:19.473 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.04243094, 2023-10-01T09:37:19.473 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.473 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.473 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.401968+0000", 2023-10-01T09:37:19.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.474 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.474 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.474 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.474 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.401968+0000", 2023-10-01T09:37:19.474 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.475 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.401967+0000", 2023-10-01T09:37:19.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:37:19.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.475 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.475 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.401977+0000", 2023-10-01T09:37:19.475 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4539999999999996e-06 2023-10-01T09:37:19.475 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.475 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.476 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.476 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.401991+0000", 2023-10-01T09:37:19.476 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3939e-05 2023-10-01T09:37:19.476 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.476 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.476 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.401998+0000", 2023-10-01T09:37:19.476 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5800000000000003e-06 2023-10-01T09:37:19.476 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.476 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.406282+0000", 2023-10-01T09:37:19.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004283518 2023-10-01T09:37:19.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.477 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.406308+0000", 2023-10-01T09:37:19.477 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5780000000000001e-05 2023-10-01T09:37:19.477 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.477 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.477 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.444358+0000", 2023-10-01T09:37:19.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.038050162999999998 2023-10-01T09:37:19.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.478 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.478 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:57.444399+0000", 2023-10-01T09:37:19.478 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1440000000000003e-05 2023-10-01T09:37:19.478 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.478 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.478 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.478 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.478 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.479 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:140370 9.f e439/400 9:fa4dd3c1:::benchmark_data_smithi138_497829_object37985:head v 439'8773, mlcod=439'8771)", 2023-10-01T09:37:19.479 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:00.556124+0000", 2023-10-01T09:37:19.479 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.826156231999999, 2023-10-01T09:37:19.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.046254339999999998, 2023-10-01T09:37:19.479 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.479 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.479 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.479 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.479 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.479 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.556124+0000", 2023-10-01T09:37:19.479 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.556124+0000", 2023-10-01T09:37:19.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.480 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.480 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.480 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.556115+0000", 2023-10-01T09:37:19.480 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999909 2023-10-01T09:37:19.480 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.556140+0000", 2023-10-01T09:37:19.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4939e-05 2023-10-01T09:37:19.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.481 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.481 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.556165+0000", 2023-10-01T09:37:19.481 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5096000000000002e-05 2023-10-01T09:37:19.481 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.481 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.482 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.556178+0000", 2023-10-01T09:37:19.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3159000000000001e-05 2023-10-01T09:37:19.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.482 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.556266+0000", 2023-10-01T09:37:19.482 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7292000000000004e-05 2023-10-01T09:37:19.482 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.482 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.482 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.556295+0000", 2023-10-01T09:37:19.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8938999999999999e-05 2023-10-01T09:37:19.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.602334+0000", 2023-10-01T09:37:19.483 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.046039432999999998 2023-10-01T09:37:19.483 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.483 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.483 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.483 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.602379+0000", 2023-10-01T09:37:19.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.477e-05 2023-10-01T09:37:19.484 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.484 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.484 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.484 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.484 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.484 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:140377 9.f e439/400 9:fa80ec40:::benchmark_data_smithi138_497829_object37992:head v 439'8774, mlcod=439'8771)", 2023-10-01T09:37:19.484 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:00.561458+0000", 2023-10-01T09:37:19.484 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.820823075, 2023-10-01T09:37:19.484 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.040961554999999997, 2023-10-01T09:37:19.484 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.484 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.485 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.561458+0000", 2023-10-01T09:37:19.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.485 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.485 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.561458+0000", 2023-10-01T09:37:19.485 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.485 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.485 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.561456+0000", 2023-10-01T09:37:19.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:19.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.486 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.561470+0000", 2023-10-01T09:37:19.486 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4239999999999999e-05 2023-10-01T09:37:19.486 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.486 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.486 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.561487+0000", 2023-10-01T09:37:19.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6062e-05 2023-10-01T09:37:19.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.561496+0000", 2023-10-01T09:37:19.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2089999999999994e-06 2023-10-01T09:37:19.487 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.487 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.487 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.487 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.561778+0000", 2023-10-01T09:37:19.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00028182400000000002 2023-10-01T09:37:19.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.561799+0000", 2023-10-01T09:37:19.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1403000000000002e-05 2023-10-01T09:37:19.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.488 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.488 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.602382+0000", 2023-10-01T09:37:19.488 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.040583441999999997 2023-10-01T09:37:19.488 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.489 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.489 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.602419+0000", 2023-10-01T09:37:19.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6711999999999997e-05 2023-10-01T09:37:19.489 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.489 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.489 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.489 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.489 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.489 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:140427 9.1 9:8e414f8b:::benchmark_data_smithi138_497829_object38042:head [delete] snapc 0=[] ondisk+write+known_if_redirected e439)", 2023-10-01T09:37:19.489 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:00.723832+0000", 2023-10-01T09:37:19.489 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.658448277000002, 2023-10-01T09:37:19.490 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.037503617000000003, 2023-10-01T09:37:19.490 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.490 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.490 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:37:19.490 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:37:19.490 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:19.490 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 140427 2023-10-01T09:37:19.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.490 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.723832+0000", 2023-10-01T09:37:19.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.723832+0000", 2023-10-01T09:37:19.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.723830+0000", 2023-10-01T09:37:19.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:19.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.723840+0000", 2023-10-01T09:37:19.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3170000000000008e-06 2023-10-01T09:37:19.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.723853+0000", 2023-10-01T09:37:19.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3701e-05 2023-10-01T09:37:19.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.723862+0000", 2023-10-01T09:37:19.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6859999999999995e-06 2023-10-01T09:37:19.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.723911+0000", 2023-10-01T09:37:19.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.897e-05 2023-10-01T09:37:19.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.724050+0000", 2023-10-01T09:37:19.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013905800000000001 2023-10-01T09:37:19.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:37:19.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.724174+0000", 2023-10-01T09:37:19.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012333900000000001 2023-10-01T09:37:19.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:19.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.731094+0000", 2023-10-01T09:37:19.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069204469999999997 2023-10-01T09:37:19.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:19.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.734329+0000", 2023-10-01T09:37:19.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032350909999999998 2023-10-01T09:37:19.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:37:19.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.761244+0000", 2023-10-01T09:37:19.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.026914784000000001 2023-10-01T09:37:19.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.761298+0000", 2023-10-01T09:37:19.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4049999999999999e-05 2023-10-01T09:37:19.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.761336+0000", 2023-10-01T09:37:19.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.807e-05 2023-10-01T09:37:19.497 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.497 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.497 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.498 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:140425 9.f e439/400 9:ffe61e90:::benchmark_data_smithi138_497829_object38040:head v 439'8777, mlcod=439'8774)", 2023-10-01T09:37:19.498 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:00.726238+0000", 2023-10-01T09:37:19.498 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.656042666000001, 2023-10-01T09:37:19.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.038432283999999997, 2023-10-01T09:37:19.498 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.498 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.498 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.726238+0000", 2023-10-01T09:37:19.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.726238+0000", 2023-10-01T09:37:19.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.726236+0000", 2023-10-01T09:37:19.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:19.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.726253+0000", 2023-10-01T09:37:19.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6439999999999998e-05 2023-10-01T09:37:19.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.726268+0000", 2023-10-01T09:37:19.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5158e-05 2023-10-01T09:37:19.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.726278+0000", 2023-10-01T09:37:19.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7040000000000006e-06 2023-10-01T09:37:19.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.726378+0000", 2023-10-01T09:37:19.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010053800000000001 2023-10-01T09:37:19.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.726403+0000", 2023-10-01T09:37:19.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4899e-05 2023-10-01T09:37:19.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.764626+0000", 2023-10-01T09:37:19.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.038223335999999997 2023-10-01T09:37:19.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:00.764670+0000", 2023-10-01T09:37:19.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3891999999999999e-05 2023-10-01T09:37:19.503 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.503 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.503 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.503 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:141759 9.a e440/431 9:5ba0636d:::benchmark_data_smithi138_497829_object39374:head v 440'8698, mlcod=405'1569)", 2023-10-01T09:37:19.503 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:04.201981+0000", 2023-10-01T09:37:19.503 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 15.180299384, 2023-10-01T09:37:19.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.039529625999999998, 2023-10-01T09:37:19.504 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:37:19.504 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:19.504 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:37:19.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:37:19.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:04.201981+0000", 2023-10-01T09:37:19.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:37:19.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:04.201981+0000", 2023-10-01T09:37:19.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:37:19.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:37:19.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:04.201980+0000", 2023-10-01T09:37:19.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:19.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:37:19.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:04.201997+0000", 2023-10-01T09:37:19.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6816e-05 2023-10-01T09:37:19.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:37:19.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:04.202009+0000", 2023-10-01T09:37:19.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2802999999999999e-05 2023-10-01T09:37:19.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:37:19.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:04.202022+0000", 2023-10-01T09:37:19.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3129000000000001e-05 2023-10-01T09:37:19.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:37:19.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:04.202074+0000", 2023-10-01T09:37:19.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1328000000000001e-05 2023-10-01T09:37:19.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:37:19.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:04.202103+0000", 2023-10-01T09:37:19.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9533999999999999e-05 2023-10-01T09:37:19.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:37:19.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:04.241464+0000", 2023-10-01T09:37:19.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.039360974 2023-10-01T09:37:19.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:37:19.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:37:19.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:37:19.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:04.241511+0000", 2023-10-01T09:37:19.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6560000000000001e-05 2023-10-01T09:37:19.508 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.508 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.508 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.508 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:37:19.509 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:37:19.509 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:37:19.758 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:37:19.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:19.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:37:19.866 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:37:19.992 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:20.059 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:37:20.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:20.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:37:20.142 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:37:20.170 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:20.207 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:20.269 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:20.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:20.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:20.378 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:20.413 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:20.482 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:20.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:20.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:20.512 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:37:20.590 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:20.633 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:20.703 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:20.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:20.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:20.812 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:20.900 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:20.996 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:21.122 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:21.241 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:21 smithi138 ceph-mon[185886]: pgmap v1244: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 15763/174471 objects degraded (9.035%); 48555/174471 objects misplaced (27.830%); 5.6 MiB/s, 90 objects/s recovering 2023-10-01T09:37:21.245 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:21.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:21.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:21.361 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:21.420 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:21.500 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:21 smithi138 conmon[185862]: 2023-10-01T09:37:21.424+0000 7f70d64ec700 -1 mon.a@0(leader).osd e444 definitely_dead 0 2023-10-01T09:37:21.502 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:21 smithi138 ceph-mon[192513]: pgmap v1244: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 15763/174471 objects degraded (9.035%); 48555/174471 objects misplaced (27.830%); 5.6 MiB/s, 90 objects/s recovering 2023-10-01T09:37:21.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:21 smithi190 ceph-mon[160435]: pgmap v1244: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 15763/174471 objects degraded (9.035%); 48555/174471 objects misplaced (27.830%); 5.6 MiB/s, 90 objects/s recovering 2023-10-01T09:37:21.823 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:21.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:21.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:21.931 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:37:21.999 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:22.042 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:37:22.261 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:22 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/86171416' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-01T09:37:22.261 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:22 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-01T09:37:22.308 INFO:teuthology.orchestra.run.smithi138.stderr:marked in osd.3. 2023-10-01T09:37:22.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:22 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/86171416' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-01T09:37:22.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:22 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-01T09:37:22.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:22 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/86171416' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-01T09:37:22.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:22 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-10-01T09:37:23.034 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:37:23.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:23.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:37:23.137 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:23.169 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:23.219 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:23.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:23.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:23.259 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: pgmap v1245: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 387 op/s; 15763/173676 objects degraded (9.076%); 48185/173676 objects misplaced (27.744%); 5.7 MiB/s, 91 objects/s recovering 2023-10-01T09:37:23.260 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-01T09:37:23.260 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: osdmap e445: 8 total, 8 up, 8 in 2023-10-01T09:37:23.260 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.260 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.260 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.260 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.260 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.260 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 15763/173676 objects degraded (9.076%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:23.260 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.260 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: osdmap e446: 8 total, 8 up, 8 in 2023-10-01T09:37:23.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.261 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.262 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[185886]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.323 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:23.349 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:23.392 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:23.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:23.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:23.496 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:37:23.519 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: pgmap v1245: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 387 op/s; 15763/173676 objects degraded (9.076%); 48185/173676 objects misplaced (27.744%); 5.7 MiB/s, 91 objects/s recovering 2023-10-01T09:37:23.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-01T09:37:23.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: osdmap e445: 8 total, 8 up, 8 in 2023-10-01T09:37:23.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 15763/173676 objects degraded (9.076%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:23.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.520 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: osdmap e446: 8 total, 8 up, 8 in 2023-10-01T09:37:23.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.521 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:23 smithi190 ceph-mon[160435]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.522 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:23.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: pgmap v1245: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 73 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 387 op/s; 15763/173676 objects degraded (9.076%); 48185/173676 objects misplaced (27.744%); 5.7 MiB/s, 91 objects/s recovering 2023-10-01T09:37:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-10-01T09:37:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: osdmap e445: 8 total, 8 up, 8 in 2023-10-01T09:37:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e445 with expected crc 2023-10-01T09:37:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 15763/173676 objects degraded (9.076%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T09:37:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: osdmap e446: 8 total, 8 up, 8 in 2023-10-01T09:37:23.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:23 smithi138 ceph-mon[192513]: failed to encode map e446 with expected crc 2023-10-01T09:37:23.711 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:37:23.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:23.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:37:23.816 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:37:23.916 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:24.180 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:37:24.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:24.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:37:24.251 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:37:24.252 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:37:24.252 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.252 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.46067.0:148375 9.4 e447/445 9:25f63120:::benchmark_data_smithi138_497829_object45990:head v 447'9351, mlcod=447'9349)", 2023-10-01T09:37:24.252 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:24.249375+0000", 2023-10-01T09:37:24.252 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0020636719999999999, 2023-10-01T09:37:24.252 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0020949250000000001, 2023-10-01T09:37:24.252 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:24.252 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:37:24.252 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:24.253 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.253 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:24.253 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249375+0000", 2023-10-01T09:37:24.253 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:24.253 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.253 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.253 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:24.253 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249375+0000", 2023-10-01T09:37:24.253 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:37:24.253 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.253 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.253 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:24.254 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249374+0000", 2023-10-01T09:37:24.254 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.8529999999999998e-06 2023-10-01T09:37:24.254 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.254 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.254 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:24.254 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249383+0000", 2023-10-01T09:37:24.254 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2492000000000001e-05 2023-10-01T09:37:24.254 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.254 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.254 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:24.254 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249405+0000", 2023-10-01T09:37:24.254 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0900000000000005e-06 2023-10-01T09:37:24.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.255 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:24.255 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249413+0000", 2023-10-01T09:37:24.255 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.9240000000000006e-05 2023-10-01T09:37:24.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.255 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:24.255 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249502+0000", 2023-10-01T09:37:24.255 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9969999999999999e-05 2023-10-01T09:37:24.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.256 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:24.256 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249532+0000", 2023-10-01T09:37:24.256 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015752 2023-10-01T09:37:24.256 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:24.256 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:24.256 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:24.256 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.256 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.256 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.46067.0:148380 9.e e447/431 9:76630267:::benchmark_data_smithi138_497829_object45995:head v 447'9466, mlcod=447'9462)", 2023-10-01T09:37:24.256 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:24.249433+0000", 2023-10-01T09:37:24.256 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0020056560000000002, 2023-10-01T09:37:24.256 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0020797329999999998, 2023-10-01T09:37:24.257 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:24.257 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:37:24.257 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:24.257 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.257 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:24.257 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249433+0000", 2023-10-01T09:37:24.257 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:24.257 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.257 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.257 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:24.257 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249433+0000", 2023-10-01T09:37:24.257 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:37:24.258 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.258 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.258 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:24.258 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249433+0000", 2023-10-01T09:37:24.258 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6140000000000002e-06 2023-10-01T09:37:24.258 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.258 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.258 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:24.258 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249437+0000", 2023-10-01T09:37:24.258 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.728e-06 2023-10-01T09:37:24.258 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.258 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.259 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:24.259 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249443+0000", 2023-10-01T09:37:24.259 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0899999999999998e-06 2023-10-01T09:37:24.259 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.259 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.259 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:24.259 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249447+0000", 2023-10-01T09:37:24.259 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6301000000000002e-05 2023-10-01T09:37:24.259 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.259 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.259 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:24.259 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249503+0000", 2023-10-01T09:37:24.259 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4078999999999997e-05 2023-10-01T09:37:24.260 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.260 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.260 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:24.260 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249537+0000", 2023-10-01T09:37:24.260 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000104409 2023-10-01T09:37:24.260 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:24.260 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:24.260 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:24.260 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.260 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.260 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.46067.0:148376 9.4 e447/445)", 2023-10-01T09:37:24.261 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:24.249457+0000", 2023-10-01T09:37:24.261 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.001981109, 2023-10-01T09:37:24.261 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0020896370000000001, 2023-10-01T09:37:24.261 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:24.261 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:37:24.261 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:24.261 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.261 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:24.261 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249457+0000", 2023-10-01T09:37:24.261 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:24.261 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.262 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.262 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:24.262 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249457+0000", 2023-10-01T09:37:24.262 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:37:24.262 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.262 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.262 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:24.262 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249457+0000", 2023-10-01T09:37:24.262 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3920000000000003e-06 2023-10-01T09:37:24.262 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.262 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.262 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:24.263 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249466+0000", 2023-10-01T09:37:24.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2959999999999998e-06 2023-10-01T09:37:24.263 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.263 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.263 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:24.263 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249471+0000", 2023-10-01T09:37:24.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7599999999999999e-06 2023-10-01T09:37:24.263 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:24.263 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:24.263 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:24.263 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:24.249477+0000", 2023-10-01T09:37:24.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9171e-05 2023-10-01T09:37:24.264 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:24.264 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:24.264 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:24.264 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:24.264 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:37:24.264 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 3 2023-10-01T09:37:24.264 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:37:24.296 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:37:24.389 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:24.941 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:24 smithi138 ceph-mon[192513]: failed to encode map e447 with expected crc 2023-10-01T09:37:24.941 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:24 smithi138 ceph-mon[192513]: failed to encode map e447 with expected crc 2023-10-01T09:37:24.942 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:24 smithi138 ceph-mon[192513]: osdmap e447: 8 total, 8 up, 8 in 2023-10-01T09:37:24.942 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:24 smithi138 ceph-mon[192513]: failed to encode map e447 with expected crc 2023-10-01T09:37:24.942 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:24 smithi138 ceph-mon[192513]: failed to encode map e447 with expected crc 2023-10-01T09:37:24.942 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:24 smithi138 ceph-mon[192513]: pgmap v1249: 89 pgs: 3 peering, 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 15763/173352 objects degraded (9.093%); 48077/173352 objects misplaced (27.734%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T09:37:24.942 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:24 smithi138 ceph-mon[185886]: failed to encode map e447 with expected crc 2023-10-01T09:37:24.942 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:24 smithi138 ceph-mon[185886]: failed to encode map e447 with expected crc 2023-10-01T09:37:24.942 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:24 smithi138 ceph-mon[185886]: osdmap e447: 8 total, 8 up, 8 in 2023-10-01T09:37:24.942 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:24 smithi138 ceph-mon[185886]: failed to encode map e447 with expected crc 2023-10-01T09:37:24.943 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:24 smithi138 ceph-mon[185886]: failed to encode map e447 with expected crc 2023-10-01T09:37:24.943 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:24 smithi138 ceph-mon[185886]: pgmap v1249: 89 pgs: 3 peering, 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 15763/173352 objects degraded (9.093%); 48077/173352 objects misplaced (27.734%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T09:37:24.943 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:37:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:37:24] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.33.4" 2023-10-01T09:37:24.996 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:37:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:25.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:37:25.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:24 smithi190 ceph-mon[160435]: failed to encode map e447 with expected crc 2023-10-01T09:37:25.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:24 smithi190 ceph-mon[160435]: failed to encode map e447 with expected crc 2023-10-01T09:37:25.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:24 smithi190 ceph-mon[160435]: osdmap e447: 8 total, 8 up, 8 in 2023-10-01T09:37:25.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:24 smithi190 ceph-mon[160435]: failed to encode map e447 with expected crc 2023-10-01T09:37:25.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:24 smithi190 ceph-mon[160435]: failed to encode map e447 with expected crc 2023-10-01T09:37:25.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:24 smithi190 ceph-mon[160435]: pgmap v1249: 89 pgs: 3 peering, 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 70 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 315 op/s; 15763/173352 objects degraded (9.093%); 48077/173352 objects misplaced (27.734%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T09:37:25.120 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:37:25.214 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:25.235 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:37:25.242 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-10-01T09:37:25.428 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:37:25.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:25.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:37:25.541 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:25.573 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:25.643 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:25.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:25.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:25.746 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:37:25.753 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:25.801 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:37:25.857 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:37:25.888 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:25.924 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:37:25.961 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:37:25.966 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:25 smithi190 ceph-mon[160435]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-01T09:37:25.966 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:37:25.966 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:37:25.991 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:37:25.994 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:26.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:26.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:26.010 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:25 smithi138 ceph-mon[192513]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-01T09:37:26.010 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:37:26.011 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:37:26.011 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:25 smithi138 ceph-mon[185886]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-01T09:37:26.011 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:37:26.011 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:37:26.030 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:37:26.071 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:37:26.110 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:26.120 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:37:26.134 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:37:26.135 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:37:26.135 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:37:26.135 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:37:26.135 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:37:26.135 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:37:26.135 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:37:26.136 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:37:26.146 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:26.215 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:26.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:26.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:26.319 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:26.398 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:26.485 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:26.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:26.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:26.596 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:37:26.685 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:26.730 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:37:26.768 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:37:26.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:26.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:37:26.872 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:26.909 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:26.979 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:26.984 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:26 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2934554294' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:26.984 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:26 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2358734295' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:37:26.984 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:26 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:37:26.984 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:26 smithi190 ceph-mon[160435]: pgmap v1250: 89 pgs: 3 peering, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 282 op/s; 22997/169380 objects degraded (13.577%); 36501/169380 objects misplaced (21.550%); 5.0 MiB/s, 81 objects/s recovering 2023-10-01T09:37:26.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:26.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:27.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:26 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2934554294' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:27.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:26 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2358734295' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:37:27.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:26 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:37:27.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:26 smithi138 ceph-mon[192513]: pgmap v1250: 89 pgs: 3 peering, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 282 op/s; 22997/169380 objects degraded (13.577%); 36501/169380 objects misplaced (21.550%); 5.0 MiB/s, 81 objects/s recovering 2023-10-01T09:37:27.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:26 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2934554294' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:27.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:26 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2358734295' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:37:27.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:26 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:37:27.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:26 smithi138 ceph-mon[185886]: pgmap v1250: 89 pgs: 3 peering, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 282 op/s; 22997/169380 objects degraded (13.577%); 36501/169380 objects misplaced (21.550%); 5.0 MiB/s, 81 objects/s recovering 2023-10-01T09:37:27.087 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:27.112 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:27.329 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:27.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:27.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:27.440 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:37:27.505 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:27.738 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:37:27.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:27.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:37:27.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:27 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:37:27.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:27 smithi138 ceph-mon[185886]: failed to encode map e448 with expected crc 2023-10-01T09:37:27.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:27 smithi138 ceph-mon[185886]: failed to encode map e448 with expected crc 2023-10-01T09:37:27.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:27 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:37:27.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:27 smithi138 ceph-mon[185886]: osdmap e448: 8 total, 8 up, 8 in 2023-10-01T09:37:27.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:27 smithi138 ceph-mon[185886]: failed to encode map e448 with expected crc 2023-10-01T09:37:27.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:27 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 22997/169380 objects degraded (13.577%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:27.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:27 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:37:27.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:27 smithi138 ceph-mon[192513]: failed to encode map e448 with expected crc 2023-10-01T09:37:27.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:27 smithi138 ceph-mon[192513]: failed to encode map e448 with expected crc 2023-10-01T09:37:27.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:27 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:37:27.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:27 smithi138 ceph-mon[192513]: osdmap e448: 8 total, 8 up, 8 in 2023-10-01T09:37:27.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:27 smithi138 ceph-mon[192513]: failed to encode map e448 with expected crc 2023-10-01T09:37:27.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:27 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 22997/169380 objects degraded (13.577%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:27.844 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:37:27.844 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:37:27.849 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:27.868 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:27 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:37:27.868 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:27 smithi190 ceph-mon[160435]: failed to encode map e448 with expected crc 2023-10-01T09:37:27.869 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:27 smithi190 ceph-mon[160435]: failed to encode map e448 with expected crc 2023-10-01T09:37:27.869 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:27 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:37:27.869 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:27 smithi190 ceph-mon[160435]: osdmap e448: 8 total, 8 up, 8 in 2023-10-01T09:37:27.869 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:27 smithi190 ceph-mon[160435]: failed to encode map e448 with expected crc 2023-10-01T09:37:27.869 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:27 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 22997/169380 objects degraded (13.577%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:27.955 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:28.078 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:28.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:28.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:28.190 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:37:28.221 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:28.277 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:37:28.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:28.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:37:28.383 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:28.414 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:28.655 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:28.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:28.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:28.774 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:37:28.812 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:37:28.822 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:28.908 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:37:28.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:28.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:37:29.016 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:37:29.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:28 smithi190 ceph-mon[160435]: pgmap v1252: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 274 op/s; 22994/168504 objects degraded (13.646%); 26088/168504 objects misplaced (15.482%); 4.7 MiB/s, 66 objects/s recovering 2023-10-01T09:37:29.077 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:28 smithi138 ceph-mon[192513]: pgmap v1252: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 274 op/s; 22994/168504 objects degraded (13.646%); 26088/168504 objects misplaced (15.482%); 4.7 MiB/s, 66 objects/s recovering 2023-10-01T09:37:29.077 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:29.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:28 smithi138 ceph-mon[185886]: pgmap v1252: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 274 op/s; 22994/168504 objects degraded (13.646%); 26088/168504 objects misplaced (15.482%); 4.7 MiB/s, 66 objects/s recovering 2023-10-01T09:37:29.143 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:37:29.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:29.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:37:29.252 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:29.334 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:29.349 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:37:29.440 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:29.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:29.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:29.560 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:37:29.654 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:29.760 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:37:29.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:29.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:37:29.878 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:37:29.882 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:37:29.947 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:29 smithi190 ceph-mon[160435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-01T09:37:29.949 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:29 smithi138 ceph-mon[192513]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-01T09:37:29.950 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:29 smithi138 ceph-mon[185886]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-01T09:37:29.983 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:30.134 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:37:30.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:30.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:37:30.243 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 2, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:37:30.243 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:37:30.243 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-01T09:37:30.243 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:37:30.267 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:37:30.303 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:30.382 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:37:30.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:30.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:37:30.486 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:30.520 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:30.586 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:30.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:30.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:30.681 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:37:30.690 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:37:30.698 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:30.799 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:31.001 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:30 smithi190 ceph-mon[160435]: pgmap v1253: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 426 op/s; 22633/163704 objects degraded (13.826%); 25381/163704 objects misplaced (15.504%); 7.5 MiB/s, 113 objects/s recovering 2023-10-01T09:37:31.066 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:37:31.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:31.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:30 smithi138 ceph-mon[192513]: pgmap v1253: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 426 op/s; 22633/163704 objects degraded (13.826%); 25381/163704 objects misplaced (15.504%); 7.5 MiB/s, 113 objects/s recovering 2023-10-01T09:37:31.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:37:31.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:30 smithi138 ceph-mon[185886]: pgmap v1253: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 426 op/s; 22633/163704 objects degraded (13.826%); 25381/163704 objects misplaced (15.504%); 7.5 MiB/s, 113 objects/s recovering 2023-10-01T09:37:31.173 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:37:31.315 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:31.484 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:31.535 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:37:31.535 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:37:31.537 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:37:31.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:31.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:37:31.655 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:37:31.690 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:31.761 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:37:31.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:31.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:37:31.865 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:31.907 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:31.980 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:31.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:31.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:32.096 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:37:32.197 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:32.331 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:37:32.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:32.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:37:32.435 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:37:32.524 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:32.622 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:37:32.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:32.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:37:32.653 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:37:32.729 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:37:32.755 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:32.798 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:37:32.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:32.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:37:32.901 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:32.951 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:33.025 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:33.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:33.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:33.115 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:37:33.138 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:37:33.184 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:33.268 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:37:33.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:33.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:37:33.371 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:37:33.450 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:33.503 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:33 smithi138 ceph-mon[192513]: pgmap v1254: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 22633/162915 objects degraded (13.893%); 25381/162915 objects misplaced (15.579%); 6.6 MiB/s, 100 objects/s recovering 2023-10-01T09:37:33.503 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:33 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 22633/162915 objects degraded (13.893%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:33.503 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:33 smithi138 ceph-mon[185886]: pgmap v1254: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 22633/162915 objects degraded (13.893%); 25381/162915 objects misplaced (15.579%); 6.6 MiB/s, 100 objects/s recovering 2023-10-01T09:37:33.503 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:33 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 22633/162915 objects degraded (13.893%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:33.543 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:33 smithi190 ceph-mon[160435]: pgmap v1254: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 22633/162915 objects degraded (13.893%); 25381/162915 objects misplaced (15.579%); 6.6 MiB/s, 100 objects/s recovering 2023-10-01T09:37:33.543 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:33 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 22633/162915 objects degraded (13.893%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:33.590 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:33.627 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:37:33.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:33.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:37:33.742 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:37:33.784 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:33.849 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:37:33.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:33.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:37:33.953 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:34.069 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:34.767 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:34.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:34.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:34.883 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:34.930 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:35.054 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:35.056 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:37:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:37:34] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.33.4" 2023-10-01T09:37:35.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:35.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:35.121 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:37:35.122 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:37:35.163 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:35.196 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:35.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:35 smithi138 ceph-mon[192513]: pgmap v1255: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 22633/162477 objects degraded (13.930%); 25381/162477 objects misplaced (15.621%); 5.7 MiB/s, 85 objects/s recovering 2023-10-01T09:37:35.331 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:35 smithi138 ceph-mon[185886]: pgmap v1255: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 22633/162477 objects degraded (13.930%); 25381/162477 objects misplaced (15.621%); 5.7 MiB/s, 85 objects/s recovering 2023-10-01T09:37:35.360 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:35.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:35.464 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:37:35.495 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:35 smithi190 ceph-mon[160435]: pgmap v1255: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 22633/162477 objects degraded (13.930%); 25381/162477 objects misplaced (15.621%); 5.7 MiB/s, 85 objects/s recovering 2023-10-01T09:37:35.497 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:37:35.502 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:35.549 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:37:35.556 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:37:35.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:35.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:37:35.602 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:37:35.658 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:37:35.661 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:35.684 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:37:35.687 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:35.729 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:35.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:35.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:35.802 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:37:35.832 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:37:35.841 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:37:35.875 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:37:35.896 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:37:35.897 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:37:35.897 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:37:35.897 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:37:35.897 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:37:35.897 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:37:35.897 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:37:35.897 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:37:35.949 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:36.218 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:37:36.389 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:37:36.393 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:37:36.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:36 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2541803457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:36.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:36 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2541803457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:36.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:36 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2541803457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:36.655 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:37:37.424 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:37:37.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:37.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:37:37.547 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:37.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:37 smithi190 ceph-mon[160435]: pgmap v1256: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 22235/157611 objects degraded (14.108%); 24645/157611 objects misplaced (15.637%); 6.4 MiB/s, 97 objects/s recovering 2023-10-01T09:37:37.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:37 smithi138 ceph-mon[192513]: pgmap v1256: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 22235/157611 objects degraded (14.108%); 24645/157611 objects misplaced (15.637%); 6.4 MiB/s, 97 objects/s recovering 2023-10-01T09:37:37.579 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:37 smithi138 ceph-mon[185886]: pgmap v1256: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 22235/157611 objects degraded (14.108%); 24645/157611 objects misplaced (15.637%); 6.4 MiB/s, 97 objects/s recovering 2023-10-01T09:37:37.584 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:37.652 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:37.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:37.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:37.756 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:37.796 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:37.869 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:37.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:37.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:37.976 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:38.055 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:38.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:38 smithi138 ceph-mon[192513]: from='client.36264 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:37:38.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:38 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 22235/157611 objects degraded (14.108%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:38.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:38 smithi138 ceph-mon[192513]: pgmap v1257: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 22235/156501 objects degraded (14.208%); 24645/156501 objects misplaced (15.748%); 5.6 MiB/s, 84 objects/s recovering 2023-10-01T09:37:38.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:38 smithi138 ceph-mon[185886]: from='client.36264 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:37:38.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:38 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 22235/157611 objects degraded (14.108%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:38.324 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:38 smithi138 ceph-mon[185886]: pgmap v1257: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 22235/156501 objects degraded (14.208%); 24645/156501 objects misplaced (15.748%); 5.6 MiB/s, 84 objects/s recovering 2023-10-01T09:37:38.402 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:38.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:38.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:38.440 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1256,"stamp":"2023-10-01T09:37:36.160471+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3445954507,"num_objects":52537,"num_object_clones":0,"num_object_copies":157611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22235,"num_objects_misplaced":24645,"num_objects_unfound":0,"num_objects_dirty":52537,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":153363,"num_write_kb":6567590,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13371,"num_bytes_recovered":862818737,"num_keys_recovered":90,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118685,"ondisk_log_size":118685,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16647332,"kb_used_data":11240996,"kb_used_omap":50,"kb_used_meta":5406093,"kb_avail":733313884,"statfs":{"total":767960285184,"available":750913417216,"internally_reserved":0,"allocated":11510779904,"data_stored":11499777927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51738,"internal_metadata":5535839718},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-257097728,"num_objects":-3923,"num_object_clones":0,"num_object_copies":-11769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-762,"num_objects_misplaced":-11856,"num_objects_unfound":0,"num_objects_dirty":-3923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3923,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":974,"num_bytes_recovered":67436544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.010818"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":360,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.637380+0000","last_change":"2023-10-01T09:35:34.561455+0000","last_active":"2023-10-01T09:37:23.637380+0000","last_peered":"2023-10-01T09:37:23.637380+0000","last_clean":"2023-10-01T09:37:23.637380+0000","last_became_active":"2023-10-01T09:35:34.560978+0000","last_became_peered":"2023-10-01T09:35:34.560978+0000","last_unstale":"2023-10-01T09:37:23.637380+0000","last_undegraded":"2023-10-01T09:37:23.637380+0000","last_fullsized":"2023-10-01T09:37:23.637380+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:02:58.965198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":64,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630826+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:37:23.630826+0000","last_peered":"2023-10-01T09:37:23.630826+0000","last_clean":"2023-10-01T09:37:23.630826+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:37:23.630826+0000","last_undegraded":"2023-10-01T09:37:23.630826+0000","last_fullsized":"2023-10-01T09:37:23.630826+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":445,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.631044+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:37:23.631044+0000","last_peered":"2023-10-01T09:37:23.631044+0000","last_clean":"2023-10-01T09:37:23.631044+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:37:23.631044+0000","last_undegraded":"2023-10-01T09:37:23.631044+0000","last_fullsized":"2023-10-01T09:37:23.631044+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":485,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.639182+0000","last_change":"2023-10-01T09:36:22.227558+0000","last_active":"2023-10-01T09:37:23.639182+0000","last_peered":"2023-10-01T09:37:23.639182+0000","last_clean":"2023-10-01T09:37:23.639182+0000","last_became_active":"2023-10-01T09:36:22.226550+0000","last_became_peered":"2023-10-01T09:36:22.226550+0000","last_unstale":"2023-10-01T09:37:23.639182+0000","last_undegraded":"2023-10-01T09:37:23.639182+0000","last_fullsized":"2023-10-01T09:37:23.639182+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:35:45.988649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":366,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.638771+0000","last_change":"2023-10-01T09:37:22.672668+0000","last_active":"2023-10-01T09:37:23.638771+0000","last_peered":"2023-10-01T09:37:23.638771+0000","last_clean":"2023-10-01T09:37:23.638771+0000","last_became_active":"2023-10-01T09:37:22.671882+0000","last_became_peered":"2023-10-01T09:37:22.671882+0000","last_unstale":"2023-10-01T09:37:23.638771+0000","last_undegraded":"2023-10-01T09:37:23.638771+0000","last_fullsized":"2023-10-01T09:37:23.638771+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:42:45.864800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":131,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.632905+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:37:23.632905+0000","last_peered":"2023-10-01T09:37:23.632905+0000","last_clean":"2023-10-01T09:37:23.632905+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:37:23.632905+0000","last_undegraded":"2023-10-01T09:37:23.632905+0000","last_fullsized":"2023-10-01T09:37:23.632905+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":521,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-01T09:37:26.706052+0000","last_change":"2023-10-01T09:37:22.671362+0000","last_active":"2023-10-01T09:37:26.706052+0000","last_peered":"2023-10-01T09:37:26.706052+0000","last_clean":"2023-10-01T09:37:26.706052+0000","last_became_active":"2023-10-01T09:37:22.671053+0000","last_became_peered":"2023-10-01T09:37:22.671053+0000","last_unstale":"2023-10-01T09:37:26.706052+0000","last_undegraded":"2023-10-01T09:37:26.706052+0000","last_fullsized":"2023-10-01T09:37:26.706052+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:42.476593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":468,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.631109+0000","last_change":"2023-10-01T09:37:22.667841+0000","last_active":"2023-10-01T09:37:23.631109+0000","last_peered":"2023-10-01T09:37:23.631109+0000","last_clean":"2023-10-01T09:37:23.631109+0000","last_became_active":"2023-10-01T09:37:22.667525+0000","last_became_peered":"2023-10-01T09:37:22.667525+0000","last_unstale":"2023-10-01T09:37:23.631109+0000","last_undegraded":"2023-10-01T09:37:23.631109+0000","last_fullsized":"2023-10-01T09:37:23.631109+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:03:37.695850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":142,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.631830+0000","last_change":"2023-10-01T09:36:22.254817+0000","last_active":"2023-10-01T09:37:23.631830+0000","last_peered":"2023-10-01T09:37:23.631830+0000","last_clean":"2023-10-01T09:37:23.631830+0000","last_became_active":"2023-10-01T09:36:22.254460+0000","last_became_peered":"2023-10-01T09:36:22.254460+0000","last_unstale":"2023-10-01T09:37:23.631830+0000","last_undegraded":"2023-10-01T09:37:23.631830+0000","last_fullsized":"2023-10-01T09:37:23.631830+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":45,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.631846+0000","last_change":"2023-10-01T09:37:22.680561+0000","last_active":"2023-10-01T09:37:23.631846+0000","last_peered":"2023-10-01T09:37:23.631846+0000","last_clean":"2023-10-01T09:37:23.631846+0000","last_became_active":"2023-10-01T09:37:22.680228+0000","last_became_peered":"2023-10-01T09:37:22.680228+0000","last_unstale":"2023-10-01T09:37:23.631846+0000","last_undegraded":"2023-10-01T09:37:23.631846+0000","last_fullsized":"2023-10-01T09:37:23.631846+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":64,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.640081+0000","last_change":"2023-10-01T09:37:22.697476+0000","last_active":"2023-10-01T09:37:23.640081+0000","last_peered":"2023-10-01T09:37:23.640081+0000","last_clean":"2023-10-01T09:37:23.640081+0000","last_became_active":"2023-10-01T09:37:22.671750+0000","last_became_peered":"2023-10-01T09:37:22.671750+0000","last_unstale":"2023-10-01T09:37:23.640081+0000","last_undegraded":"2023-10-01T09:37:23.640081+0000","last_fullsized":"2023-10-01T09:37:23.640081+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:36:00.363327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":28,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.638622+0000","last_change":"2023-10-01T09:36:22.223469+0000","last_active":"2023-10-01T09:37:23.638622+0000","last_peered":"2023-10-01T09:37:23.638622+0000","last_clean":"2023-10-01T09:37:23.638622+0000","last_became_active":"2023-10-01T09:36:22.222681+0000","last_became_peered":"2023-10-01T09:36:22.222681+0000","last_unstale":"2023-10-01T09:37:23.638622+0000","last_undegraded":"2023-10-01T09:37:23.638622+0000","last_fullsized":"2023-10-01T09:37:23.638622+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:19:28.367922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":475,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630983+0000","last_change":"2023-10-01T09:32:27.498041+0000","last_active":"2023-10-01T09:37:23.630983+0000","last_peered":"2023-10-01T09:37:23.630983+0000","last_clean":"2023-10-01T09:37:23.630983+0000","last_became_active":"2023-10-01T09:32:27.454394+0000","last_became_peered":"2023-10-01T09:32:27.454394+0000","last_unstale":"2023-10-01T09:37:23.630983+0000","last_undegraded":"2023-10-01T09:37:23.630983+0000","last_fullsized":"2023-10-01T09:37:23.630983+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":289,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630486+0000","last_change":"2023-10-01T09:35:23.610660+0000","last_active":"2023-10-01T09:37:23.630486+0000","last_peered":"2023-10-01T09:37:23.630486+0000","last_clean":"2023-10-01T09:37:23.630486+0000","last_became_active":"2023-10-01T09:35:23.610273+0000","last_became_peered":"2023-10-01T09:35:23.610273+0000","last_unstale":"2023-10-01T09:37:23.630486+0000","last_undegraded":"2023-10-01T09:37:23.630486+0000","last_fullsized":"2023-10-01T09:37:23.630486+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:43.996644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":551,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630825+0000","last_change":"2023-10-01T09:35:34.640669+0000","last_active":"2023-10-01T09:37:23.630825+0000","last_peered":"2023-10-01T09:37:23.630825+0000","last_clean":"2023-10-01T09:37:23.630825+0000","last_became_active":"2023-10-01T09:35:34.552977+0000","last_became_peered":"2023-10-01T09:35:34.552977+0000","last_unstale":"2023-10-01T09:37:23.630825+0000","last_undegraded":"2023-10-01T09:37:23.630825+0000","last_fullsized":"2023-10-01T09:37:23.630825+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":306,"num_keys_recovered":90,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":285,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630996+0000","last_change":"2023-10-01T09:37:22.681548+0000","last_active":"2023-10-01T09:37:23.630996+0000","last_peered":"2023-10-01T09:37:23.630996+0000","last_clean":"2023-10-01T09:37:23.630996+0000","last_became_active":"2023-10-01T09:37:22.681195+0000","last_became_peered":"2023-10-01T09:37:22.681195+0000","last_unstale":"2023-10-01T09:37:23.630996+0000","last_undegraded":"2023-10-01T09:37:23.630996+0000","last_fullsized":"2023-10-01T09:37:23.630996+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":450,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.638737+0000","last_change":"2023-10-01T09:36:22.223650+0000","last_active":"2023-10-01T09:37:23.638737+0000","last_peered":"2023-10-01T09:37:23.638737+0000","last_clean":"2023-10-01T09:37:23.638737+0000","last_became_active":"2023-10-01T09:36:22.223087+0000","last_became_peered":"2023-10-01T09:36:22.223087+0000","last_unstale":"2023-10-01T09:37:23.638737+0000","last_undegraded":"2023-10-01T09:37:23.638737+0000","last_fullsized":"2023-10-01T09:37:23.638737+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:52:21.962744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":266,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.631016+0000","last_change":"2023-10-01T09:37:22.670521+0000","last_active":"2023-10-01T09:37:23.631016+0000","last_peered":"2023-10-01T09:37:23.631016+0000","last_clean":"2023-10-01T09:37:23.631016+0000","last_became_active":"2023-10-01T09:37:22.670154+0000","last_became_peered":"2023-10-01T09:37:22.670154+0000","last_unstale":"2023-10-01T09:37:23.631016+0000","last_undegraded":"2023-10-01T09:37:23.631016+0000","last_fullsized":"2023-10-01T09:37:23.631016+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":459,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630582+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:37:23.630582+0000","last_peered":"2023-10-01T09:37:23.630582+0000","last_clean":"2023-10-01T09:37:23.630582+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:37:23.630582+0000","last_undegraded":"2023-10-01T09:37:23.630582+0000","last_fullsized":"2023-10-01T09:37:23.630582+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":450,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-01T09:37:26.705806+0000","last_change":"2023-10-01T09:37:22.681587+0000","last_active":"2023-10-01T09:37:26.705806+0000","last_peered":"2023-10-01T09:37:26.705806+0000","last_clean":"2023-10-01T09:37:26.705806+0000","last_became_active":"2023-10-01T09:37:22.681290+0000","last_became_peered":"2023-10-01T09:37:22.681290+0000","last_unstale":"2023-10-01T09:37:26.705806+0000","last_undegraded":"2023-10-01T09:37:26.705806+0000","last_fullsized":"2023-10-01T09:37:26.705806+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:13:27.218691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":304,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.638636+0000","last_change":"2023-10-01T09:37:22.717074+0000","last_active":"2023-10-01T09:37:23.638636+0000","last_peered":"2023-10-01T09:37:23.638636+0000","last_clean":"2023-10-01T09:37:23.638636+0000","last_became_active":"2023-10-01T09:37:22.672350+0000","last_became_peered":"2023-10-01T09:37:22.672350+0000","last_unstale":"2023-10-01T09:37:23.638636+0000","last_undegraded":"2023-10-01T09:37:23.638636+0000","last_fullsized":"2023-10-01T09:37:23.638636+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:44:43.921813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":248,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.631493+0000","last_change":"2023-10-01T09:36:22.223054+0000","last_active":"2023-10-01T09:37:23.631493+0000","last_peered":"2023-10-01T09:37:23.631493+0000","last_clean":"2023-10-01T09:37:23.631493+0000","last_became_active":"2023-10-01T09:36:22.222776+0000","last_became_peered":"2023-10-01T09:36:22.222776+0000","last_unstale":"2023-10-01T09:37:23.631493+0000","last_undegraded":"2023-10-01T09:37:23.631493+0000","last_fullsized":"2023-10-01T09:37:23.631493+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:09:23.536588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"448'9620","reported_seq":11712,"reported_epoch":448,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:37:34.430998+0000","last_change":"2023-10-01T09:37:22.729877+0000","last_active":"2023-10-01T09:37:34.430998+0000","last_peered":"2023-10-01T09:37:34.430998+0000","last_clean":"2023-10-01T09:36:21.212087+0000","last_became_active":"2023-10-01T09:36:23.799500+0000","last_became_peered":"2023-10-01T09:36:23.799500+0000","last_unstale":"2023-10-01T09:37:34.430998+0000","last_undegraded":"2023-10-01T09:36:23.263244+0000","last_fullsized":"2023-10-01T09:36:23.212896+0000","mapping_epoch":431,"log_start":"406'2098","ondisk_log_start":"406'2098","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7522,"log_dups_size":0,"ondisk_log_size":7522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:09:34.039273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215613440,"num_objects":3290,"num_object_clones":0,"num_object_copies":9870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":577,"num_objects_misplaced":3273,"num_objects_unfound":0,"num_objects_dirty":3290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9620,"num_write_kb":413120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1012,"num_bytes_recovered":66256896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":-560},{"shards":"0,5,7","objects":3273}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":121,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.639143+0000","last_change":"2023-10-01T09:36:22.387631+0000","last_active":"2023-10-01T09:37:23.639143+0000","last_peered":"2023-10-01T09:37:23.639143+0000","last_clean":"2023-10-01T09:37:23.639143+0000","last_became_active":"2023-10-01T09:36:22.237417+0000","last_became_peered":"2023-10-01T09:36:22.237417+0000","last_unstale":"2023-10-01T09:37:23.639143+0000","last_undegraded":"2023-10-01T09:37:23.639143+0000","last_fullsized":"2023-10-01T09:37:23.639143+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:06:04.026880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":159,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.631615+0000","last_change":"2023-10-01T09:36:22.236445+0000","last_active":"2023-10-01T09:37:23.631615+0000","last_peered":"2023-10-01T09:37:23.631615+0000","last_clean":"2023-10-01T09:37:23.631615+0000","last_became_active":"2023-10-01T09:36:22.236081+0000","last_became_peered":"2023-10-01T09:36:22.236081+0000","last_unstale":"2023-10-01T09:37:23.631615+0000","last_undegraded":"2023-10-01T09:37:23.631615+0000","last_fullsized":"2023-10-01T09:37:23.631615+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"448'9604","reported_seq":9684,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-01T09:37:34.349821+0000","last_change":"2023-10-01T09:37:22.701634+0000","last_active":"2023-10-01T09:37:34.349821+0000","last_peered":"2023-10-01T09:37:34.349821+0000","last_clean":"2023-10-01T09:37:34.349821+0000","last_became_active":"2023-10-01T09:37:22.671424+0000","last_became_peered":"2023-10-01T09:37:22.671424+0000","last_unstale":"2023-10-01T09:37:34.349821+0000","last_undegraded":"2023-10-01T09:37:34.349821+0000","last_fullsized":"2023-10-01T09:37:34.349821+0000","mapping_epoch":445,"log_start":"407'2734","ondisk_log_start":"407'2734","created":400,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6870,"log_dups_size":0,"ondisk_log_size":6870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:07:45.393494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220332032,"num_objects":3362,"num_object_clones":0,"num_object_copies":10086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9604,"num_write_kb":414912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":442,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.631198+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:37:23.631198+0000","last_peered":"2023-10-01T09:37:23.631198+0000","last_clean":"2023-10-01T09:37:23.631198+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:37:23.631198+0000","last_undegraded":"2023-10-01T09:37:23.631198+0000","last_fullsized":"2023-10-01T09:37:23.631198+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":491,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630462+0000","last_change":"2023-10-01T09:36:22.223799+0000","last_active":"2023-10-01T09:37:23.630462+0000","last_peered":"2023-10-01T09:37:23.630462+0000","last_clean":"2023-10-01T09:37:23.630462+0000","last_became_active":"2023-10-01T09:36:22.223546+0000","last_became_peered":"2023-10-01T09:36:22.223546+0000","last_unstale":"2023-10-01T09:37:23.630462+0000","last_undegraded":"2023-10-01T09:37:23.630462+0000","last_fullsized":"2023-10-01T09:37:23.630462+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:54:07.013864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"448'9527","reported_seq":14277,"reported_epoch":448,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-01T09:37:34.898724+0000","last_change":"2023-10-01T09:37:06.460981+0000","last_active":"2023-10-01T09:37:34.898724+0000","last_peered":"2023-10-01T09:37:34.898724+0000","last_clean":"2023-10-01T09:36:21.192430+0000","last_became_active":"2023-10-01T09:36:23.791409+0000","last_became_peered":"2023-10-01T09:36:23.791409+0000","last_unstale":"2023-10-01T09:37:34.898724+0000","last_undegraded":"2023-10-01T09:37:34.898724+0000","last_fullsized":"2023-10-01T09:36:23.217545+0000","mapping_epoch":431,"log_start":"407'2707","ondisk_log_start":"407'2707","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6820,"log_dups_size":0,"ondisk_log_size":6820,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212140032,"num_objects":3237,"num_object_clones":0,"num_object_copies":9711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6472,"num_objects_unfound":0,"num_objects_dirty":3237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9527,"num_write_kb":408448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1562,"num_bytes_recovered":102301696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,1],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"1,5","objects":1},{"shards":"3,5,7","objects":3236}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":528,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630780+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:37:23.630780+0000","last_peered":"2023-10-01T09:37:23.630780+0000","last_clean":"2023-10-01T09:37:23.630780+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:37:23.630780+0000","last_undegraded":"2023-10-01T09:37:23.630780+0000","last_fullsized":"2023-10-01T09:37:23.630780+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":312,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-01T09:37:26.730370+0000","last_change":"2023-10-01T09:36:22.247103+0000","last_active":"2023-10-01T09:37:26.730370+0000","last_peered":"2023-10-01T09:37:26.730370+0000","last_clean":"2023-10-01T09:37:26.730370+0000","last_became_active":"2023-10-01T09:36:22.246793+0000","last_became_peered":"2023-10-01T09:36:22.246793+0000","last_unstale":"2023-10-01T09:37:26.730370+0000","last_undegraded":"2023-10-01T09:37:26.730370+0000","last_fullsized":"2023-10-01T09:37:26.730370+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:25:05.754730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"448'9409","reported_seq":12247,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-01T09:37:34.889581+0000","last_change":"2023-10-01T09:36:28.672838+0000","last_active":"2023-10-01T09:37:34.889581+0000","last_peered":"2023-10-01T09:37:34.889581+0000","last_clean":"2023-10-01T09:37:34.889581+0000","last_became_active":"2023-10-01T09:36:28.606609+0000","last_became_peered":"2023-10-01T09:36:28.606609+0000","last_unstale":"2023-10-01T09:37:34.889581+0000","last_undegraded":"2023-10-01T09:37:34.889581+0000","last_fullsized":"2023-10-01T09:37:34.889581+0000","mapping_epoch":433,"log_start":"407'2498","ondisk_log_start":"407'2498","created":400,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6911,"log_dups_size":0,"ondisk_log_size":6911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211615744,"num_objects":3229,"num_object_clones":0,"num_object_copies":9687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9409,"num_write_kb":404416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1377,"num_bytes_recovered":90243072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":168,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.632608+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:37:23.632608+0000","last_peered":"2023-10-01T09:37:23.632608+0000","last_clean":"2023-10-01T09:37:23.632608+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:37:23.632608+0000","last_undegraded":"2023-10-01T09:37:23.632608+0000","last_fullsized":"2023-10-01T09:37:23.632608+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":423,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.638853+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:37:23.638853+0000","last_peered":"2023-10-01T09:37:23.638853+0000","last_clean":"2023-10-01T09:37:23.638853+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:37:23.638853+0000","last_undegraded":"2023-10-01T09:37:23.638853+0000","last_fullsized":"2023-10-01T09:37:23.638853+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"448'9227","reported_seq":18273,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-01T09:37:32.215892+0000","last_change":"2023-10-01T09:37:22.694633+0000","last_active":"2023-10-01T09:37:32.215892+0000","last_peered":"2023-10-01T09:37:32.215892+0000","last_clean":"2023-10-01T09:37:32.215892+0000","last_became_active":"2023-10-01T09:37:22.668427+0000","last_became_peered":"2023-10-01T09:37:22.668427+0000","last_unstale":"2023-10-01T09:37:32.215892+0000","last_undegraded":"2023-10-01T09:37:32.215892+0000","last_fullsized":"2023-10-01T09:37:32.215892+0000","mapping_epoch":445,"log_start":"405'1679","ondisk_log_start":"405'1679","created":400,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7548,"log_dups_size":0,"ondisk_log_size":7548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:50:23.248095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215547904,"num_objects":3289,"num_object_clones":0,"num_object_copies":9867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9227,"num_write_kb":400512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4478,"num_bytes_recovered":293339136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":177,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-01T09:37:26.705000+0000","last_change":"2023-10-01T09:37:22.723456+0000","last_active":"2023-10-01T09:37:26.705000+0000","last_peered":"2023-10-01T09:37:26.705000+0000","last_clean":"2023-10-01T09:37:26.705000+0000","last_became_active":"2023-10-01T09:37:22.677109+0000","last_became_peered":"2023-10-01T09:37:22.677109+0000","last_unstale":"2023-10-01T09:37:26.705000+0000","last_undegraded":"2023-10-01T09:37:26.705000+0000","last_fullsized":"2023-10-01T09:37:26.705000+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:04:02.812669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":102,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.634326+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:37:23.634326+0000","last_peered":"2023-10-01T09:37:23.634326+0000","last_clean":"2023-10-01T09:37:23.634326+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:37:23.634326+0000","last_undegraded":"2023-10-01T09:37:23.634326+0000","last_fullsized":"2023-10-01T09:37:23.634326+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"448'9492","reported_seq":12453,"reported_epoch":448,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-01T09:37:34.083425+0000","last_change":"2023-10-01T09:37:06.423304+0000","last_active":"2023-10-01T09:37:34.083425+0000","last_peered":"2023-10-01T09:37:34.083425+0000","last_clean":"2023-10-01T09:35:09.959309+0000","last_became_active":"2023-10-01T09:36:23.678618+0000","last_became_peered":"2023-10-01T09:36:23.678618+0000","last_unstale":"2023-10-01T09:37:34.083425+0000","last_undegraded":"2023-10-01T09:37:34.083425+0000","last_fullsized":"2023-10-01T09:37:34.083425+0000","mapping_epoch":431,"log_start":"407'2614","ondisk_log_start":"407'2614","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6878,"log_dups_size":0,"ondisk_log_size":6878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:32:05.325065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209715200,"num_objects":3200,"num_object_clones":0,"num_object_copies":9600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4912,"num_objects_unfound":0,"num_objects_dirty":3200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9492,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1435,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[1,6,2],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":-68}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":368,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.629516+0000","last_change":"2023-10-01T09:37:22.684537+0000","last_active":"2023-10-01T09:37:23.629516+0000","last_peered":"2023-10-01T09:37:23.629516+0000","last_clean":"2023-10-01T09:37:23.629516+0000","last_became_active":"2023-10-01T09:37:22.683692+0000","last_became_peered":"2023-10-01T09:37:22.683692+0000","last_unstale":"2023-10-01T09:37:23.629516+0000","last_undegraded":"2023-10-01T09:37:23.629516+0000","last_fullsized":"2023-10-01T09:37:23.629516+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":494,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630968+0000","last_change":"2023-10-01T09:35:34.558802+0000","last_active":"2023-10-01T09:37:23.630968+0000","last_peered":"2023-10-01T09:37:23.630968+0000","last_clean":"2023-10-01T09:37:23.630968+0000","last_became_active":"2023-10-01T09:35:34.558511+0000","last_became_peered":"2023-10-01T09:35:34.558511+0000","last_unstale":"2023-10-01T09:37:23.630968+0000","last_undegraded":"2023-10-01T09:37:23.630968+0000","last_fullsized":"2023-10-01T09:37:23.630968+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:40:52.864568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"448'9440","reported_seq":10202,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-01T09:37:31.871408+0000","last_change":"2023-10-01T09:37:23.700088+0000","last_active":"2023-10-01T09:37:31.871408+0000","last_peered":"2023-10-01T09:37:31.871408+0000","last_clean":"2023-10-01T09:37:31.871408+0000","last_became_active":"2023-10-01T09:37:23.635822+0000","last_became_peered":"2023-10-01T09:37:23.635822+0000","last_unstale":"2023-10-01T09:37:31.871408+0000","last_undegraded":"2023-10-01T09:37:31.871408+0000","last_fullsized":"2023-10-01T09:37:31.871408+0000","mapping_epoch":446,"log_start":"405'400","ondisk_log_start":"405'400","created":400,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":9040,"log_dups_size":0,"ondisk_log_size":9040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218365952,"num_objects":3332,"num_object_clones":0,"num_object_copies":9996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9440,"num_write_kb":408704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":343,"num_bytes_recovered":22478848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":338,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630607+0000","last_change":"2023-10-01T09:35:34.551196+0000","last_active":"2023-10-01T09:37:23.630607+0000","last_peered":"2023-10-01T09:37:23.630607+0000","last_clean":"2023-10-01T09:37:23.630607+0000","last_became_active":"2023-10-01T09:35:34.550803+0000","last_became_peered":"2023-10-01T09:35:34.550803+0000","last_unstale":"2023-10-01T09:37:23.630607+0000","last_undegraded":"2023-10-01T09:37:23.630607+0000","last_fullsized":"2023-10-01T09:37:23.630607+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":174,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.634546+0000","last_change":"2023-10-01T09:37:22.681270+0000","last_active":"2023-10-01T09:37:23.634546+0000","last_peered":"2023-10-01T09:37:23.634546+0000","last_clean":"2023-10-01T09:37:23.634546+0000","last_became_active":"2023-10-01T09:37:22.680951+0000","last_became_peered":"2023-10-01T09:37:22.680951+0000","last_unstale":"2023-10-01T09:37:23.634546+0000","last_undegraded":"2023-10-01T09:37:23.634546+0000","last_fullsized":"2023-10-01T09:37:23.634546+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"448'9648","reported_seq":10680,"reported_epoch":448,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:37:34.100357+0000","last_change":"2023-10-01T09:36:23.771443+0000","last_active":"2023-10-01T09:37:34.100357+0000","last_peered":"2023-10-01T09:37:34.100357+0000","last_clean":"2023-10-01T09:35:09.934020+0000","last_became_active":"2023-10-01T09:36:23.749995+0000","last_became_peered":"2023-10-01T09:36:23.749995+0000","last_unstale":"2023-10-01T09:37:34.100357+0000","last_undegraded":"2023-10-01T09:36:23.566811+0000","last_fullsized":"2023-10-01T09:36:23.220185+0000","mapping_epoch":431,"log_start":"407'2792","ondisk_log_start":"407'2792","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6856,"log_dups_size":0,"ondisk_log_size":6856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:23:09.158739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213909504,"num_objects":3264,"num_object_clones":0,"num_object_copies":9792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9648,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":476,"num_bytes_recovered":30998528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2","objects":-1623},{"shards":"1,2,3","objects":2346},{"shards":"1,3,7","objects":918}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":299,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630981+0000","last_change":"2023-10-01T09:36:22.223384+0000","last_active":"2023-10-01T09:37:23.630981+0000","last_peered":"2023-10-01T09:37:23.630981+0000","last_clean":"2023-10-01T09:37:23.630981+0000","last_became_active":"2023-10-01T09:36:22.223098+0000","last_became_peered":"2023-10-01T09:36:22.223098+0000","last_unstale":"2023-10-01T09:37:23.630981+0000","last_undegraded":"2023-10-01T09:37:23.630981+0000","last_fullsized":"2023-10-01T09:37:23.630981+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:15:40.489055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":355,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.631533+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:37:23.631533+0000","last_peered":"2023-10-01T09:37:23.631533+0000","last_clean":"2023-10-01T09:37:23.631533+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:37:23.631533+0000","last_undegraded":"2023-10-01T09:37:23.631533+0000","last_fullsized":"2023-10-01T09:37:23.631533+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"448'9545","reported_seq":9614,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-01T09:37:34.888823+0000","last_change":"2023-10-01T09:37:22.715127+0000","last_active":"2023-10-01T09:37:34.888823+0000","last_peered":"2023-10-01T09:37:34.888823+0000","last_clean":"2023-10-01T09:37:34.888823+0000","last_became_active":"2023-10-01T09:37:22.680102+0000","last_became_peered":"2023-10-01T09:37:22.680102+0000","last_unstale":"2023-10-01T09:37:34.888823+0000","last_undegraded":"2023-10-01T09:37:34.888823+0000","last_fullsized":"2023-10-01T09:37:34.888823+0000","mapping_epoch":445,"log_start":"407'2662","ondisk_log_start":"407'2662","created":400,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6883,"log_dups_size":0,"ondisk_log_size":6883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215941120,"num_objects":3295,"num_object_clones":0,"num_object_copies":9885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9545,"num_write_kb":410880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1272,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630633+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:37:23.630633+0000","last_peered":"2023-10-01T09:37:23.630633+0000","last_clean":"2023-10-01T09:37:23.630633+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:37:23.630633+0000","last_undegraded":"2023-10-01T09:37:23.630633+0000","last_fullsized":"2023-10-01T09:37:23.630633+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":169,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.632114+0000","last_change":"2023-10-01T09:37:22.667746+0000","last_active":"2023-10-01T09:37:23.632114+0000","last_peered":"2023-10-01T09:37:23.632114+0000","last_clean":"2023-10-01T09:37:23.632114+0000","last_became_active":"2023-10-01T09:37:22.667475+0000","last_became_peered":"2023-10-01T09:37:22.667475+0000","last_unstale":"2023-10-01T09:37:23.632114+0000","last_undegraded":"2023-10-01T09:37:23.632114+0000","last_fullsized":"2023-10-01T09:37:23.632114+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":403,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.632364+0000","last_change":"2023-10-01T09:36:22.236067+0000","last_active":"2023-10-01T09:37:23.632364+0000","last_peered":"2023-10-01T09:37:23.632364+0000","last_clean":"2023-10-01T09:37:23.632364+0000","last_became_active":"2023-10-01T09:36:22.235732+0000","last_became_peered":"2023-10-01T09:36:22.235732+0000","last_unstale":"2023-10-01T09:37:23.632364+0000","last_undegraded":"2023-10-01T09:37:23.632364+0000","last_fullsized":"2023-10-01T09:37:23.632364+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"448'9616","reported_seq":9674,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-01T09:37:34.409208+0000","last_change":"2023-10-01T09:34:06.140052+0000","last_active":"2023-10-01T09:37:34.409208+0000","last_peered":"2023-10-01T09:37:34.409208+0000","last_clean":"2023-10-01T09:37:34.409208+0000","last_became_active":"2023-10-01T09:34:06.139672+0000","last_became_peered":"2023-10-01T09:34:06.139672+0000","last_unstale":"2023-10-01T09:37:34.409208+0000","last_undegraded":"2023-10-01T09:37:34.409208+0000","last_fullsized":"2023-10-01T09:37:34.409208+0000","mapping_epoch":400,"log_start":"406'2078","ondisk_log_start":"406'2078","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7538,"log_dups_size":0,"ondisk_log_size":7538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:39:55.000830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212467712,"num_objects":3242,"num_object_clones":0,"num_object_copies":9726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9616,"num_write_kb":411456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1113,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.632211+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:37:23.632211+0000","last_peered":"2023-10-01T09:37:23.632211+0000","last_clean":"2023-10-01T09:37:23.632211+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:37:23.632211+0000","last_undegraded":"2023-10-01T09:37:23.632211+0000","last_fullsized":"2023-10-01T09:37:23.632211+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":423,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.639024+0000","last_change":"2023-10-01T09:32:27.451746+0000","last_active":"2023-10-01T09:37:23.639024+0000","last_peered":"2023-10-01T09:37:23.639024+0000","last_clean":"2023-10-01T09:37:23.639024+0000","last_became_active":"2023-10-01T09:32:27.451069+0000","last_became_peered":"2023-10-01T09:32:27.451069+0000","last_unstale":"2023-10-01T09:37:23.639024+0000","last_undegraded":"2023-10-01T09:37:23.639024+0000","last_fullsized":"2023-10-01T09:37:23.639024+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:50.586397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":127,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.632734+0000","last_change":"2023-10-01T09:37:22.680903+0000","last_active":"2023-10-01T09:37:23.632734+0000","last_peered":"2023-10-01T09:37:23.632734+0000","last_clean":"2023-10-01T09:37:23.632734+0000","last_became_active":"2023-10-01T09:37:22.680103+0000","last_became_peered":"2023-10-01T09:37:22.680103+0000","last_unstale":"2023-10-01T09:37:23.632734+0000","last_undegraded":"2023-10-01T09:37:23.632734+0000","last_fullsized":"2023-10-01T09:37:23.632734+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"448'9369","reported_seq":10738,"reported_epoch":448,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:37:34.105505+0000","last_change":"2023-10-01T09:37:22.805677+0000","last_active":"2023-10-01T09:37:34.105505+0000","last_peered":"2023-10-01T09:37:34.105505+0000","last_clean":"2023-10-01T09:36:21.153294+0000","last_became_active":"2023-10-01T09:37:22.773254+0000","last_became_peered":"2023-10-01T09:37:22.773254+0000","last_unstale":"2023-10-01T09:37:34.105505+0000","last_undegraded":"2023-10-01T09:37:22.629142+0000","last_fullsized":"2023-10-01T09:37:34.105505+0000","mapping_epoch":445,"log_start":"407'2507","ondisk_log_start":"407'2507","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6862,"log_dups_size":0,"ondisk_log_size":6862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:29:55.127437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208863232,"num_objects":3187,"num_object_clones":0,"num_object_copies":9561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5634,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9369,"num_write_kb":401792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":644,"num_bytes_recovered":42205184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":["1","5","6","7"],"object_location_counts":[{"shards":"1,5,7","objects":3187}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":720,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.638565+0000","last_change":"2023-10-01T09:37:22.668411+0000","last_active":"2023-10-01T09:37:23.638565+0000","last_peered":"2023-10-01T09:37:23.638565+0000","last_clean":"2023-10-01T09:37:23.638565+0000","last_became_active":"2023-10-01T09:37:22.667691+0000","last_became_peered":"2023-10-01T09:37:22.667691+0000","last_unstale":"2023-10-01T09:37:23.638565+0000","last_undegraded":"2023-10-01T09:37:23.638565+0000","last_fullsized":"2023-10-01T09:37:23.638565+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:53:57.859137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":498,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.631047+0000","last_change":"2023-10-01T09:35:34.562777+0000","last_active":"2023-10-01T09:37:23.631047+0000","last_peered":"2023-10-01T09:37:23.631047+0000","last_clean":"2023-10-01T09:37:23.631047+0000","last_became_active":"2023-10-01T09:35:34.562492+0000","last_became_peered":"2023-10-01T09:35:34.562492+0000","last_unstale":"2023-10-01T09:37:23.631047+0000","last_undegraded":"2023-10-01T09:37:23.631047+0000","last_fullsized":"2023-10-01T09:37:23.631047+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:26:15.751778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":317,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.633059+0000","last_change":"2023-10-01T09:37:22.673957+0000","last_active":"2023-10-01T09:37:23.633059+0000","last_peered":"2023-10-01T09:37:23.633059+0000","last_clean":"2023-10-01T09:37:23.633059+0000","last_became_active":"2023-10-01T09:37:22.673685+0000","last_became_peered":"2023-10-01T09:37:22.673685+0000","last_unstale":"2023-10-01T09:37:23.633059+0000","last_undegraded":"2023-10-01T09:37:23.633059+0000","last_fullsized":"2023-10-01T09:37:23.633059+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"448'9502","reported_seq":14360,"reported_epoch":448,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:37:34.437932+0000","last_change":"2023-10-01T09:37:22.682829+0000","last_active":"2023-10-01T09:37:34.437932+0000","last_peered":"2023-10-01T09:37:34.437932+0000","last_clean":"2023-10-01T09:36:21.170373+0000","last_became_active":"2023-10-01T09:36:23.812646+0000","last_became_peered":"2023-10-01T09:36:23.812646+0000","last_unstale":"2023-10-01T09:37:34.437932+0000","last_undegraded":"2023-10-01T09:37:22.679747+0000","last_fullsized":"2023-10-01T09:36:23.211397+0000","mapping_epoch":431,"log_start":"406'1957","ondisk_log_start":"406'1957","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7545,"log_dups_size":0,"ondisk_log_size":7545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:34:58.841399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212992000,"num_objects":3250,"num_object_clones":0,"num_object_copies":9750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9558,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9502,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1597,"num_bytes_recovered":104529920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":3250}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1444,"reported_epoch":447,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:37:23.636269+0000","last_change":"2023-10-01T09:37:23.636269+0000","last_active":"2023-10-01T09:37:23.636269+0000","last_peered":"2023-10-01T09:37:23.636269+0000","last_clean":"2023-10-01T09:35:27.946455+0000","last_became_active":"2023-10-01T09:37:23.635623+0000","last_became_peered":"2023-10-01T09:37:23.635623+0000","last_unstale":"2023-10-01T09:37:23.636269+0000","last_undegraded":"2023-10-01T09:37:23.631750+0000","last_fullsized":"2023-10-01T09:37:23.636269+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":467,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.638549+0000","last_change":"2023-10-01T09:36:29.654983+0000","last_active":"2023-10-01T09:37:23.638549+0000","last_peered":"2023-10-01T09:37:23.638549+0000","last_clean":"2023-10-01T09:37:23.638549+0000","last_became_active":"2023-10-01T09:36:29.650683+0000","last_became_peered":"2023-10-01T09:36:29.650683+0000","last_unstale":"2023-10-01T09:37:23.638549+0000","last_undegraded":"2023-10-01T09:37:23.638549+0000","last_fullsized":"2023-10-01T09:37:23.638549+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:23.591743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":238,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.632063+0000","last_change":"2023-10-01T09:37:22.669175+0000","last_active":"2023-10-01T09:37:23.632063+0000","last_peered":"2023-10-01T09:37:23.632063+0000","last_clean":"2023-10-01T09:37:23.632063+0000","last_became_active":"2023-10-01T09:37:22.668788+0000","last_became_peered":"2023-10-01T09:37:22.668788+0000","last_unstale":"2023-10-01T09:37:23.632063+0000","last_undegraded":"2023-10-01T09:37:23.632063+0000","last_fullsized":"2023-10-01T09:37:23.632063+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":362,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630936+0000","last_change":"2023-10-01T09:32:27.453291+0000","last_active":"2023-10-01T09:37:23.630936+0000","last_peered":"2023-10-01T09:37:23.630936+0000","last_clean":"2023-10-01T09:37:23.630936+0000","last_became_active":"2023-10-01T09:32:27.453026+0000","last_became_peered":"2023-10-01T09:32:27.453026+0000","last_unstale":"2023-10-01T09:37:23.630936+0000","last_undegraded":"2023-10-01T09:37:23.630936+0000","last_fullsized":"2023-10-01T09:37:23.630936+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"448'9491","reported_seq":9549,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-01T09:37:34.380720+0000","last_change":"2023-10-01T09:34:06.140844+0000","last_active":"2023-10-01T09:37:34.380720+0000","last_peered":"2023-10-01T09:37:34.380720+0000","last_clean":"2023-10-01T09:37:34.380720+0000","last_became_active":"2023-10-01T09:34:06.139812+0000","last_became_peered":"2023-10-01T09:34:06.139812+0000","last_unstale":"2023-10-01T09:37:34.380720+0000","last_undegraded":"2023-10-01T09:37:34.380720+0000","last_fullsized":"2023-10-01T09:37:34.380720+0000","mapping_epoch":400,"log_start":"406'1967","ondisk_log_start":"406'1967","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7524,"log_dups_size":0,"ondisk_log_size":7524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:31:05.105491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214106112,"num_objects":3267,"num_object_clones":0,"num_object_copies":9801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9491,"num_write_kb":408256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":628,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630356+0000","last_change":"2023-10-01T09:37:22.770678+0000","last_active":"2023-10-01T09:37:23.630356+0000","last_peered":"2023-10-01T09:37:23.630356+0000","last_clean":"2023-10-01T09:37:23.630356+0000","last_became_active":"2023-10-01T09:37:22.674516+0000","last_became_peered":"2023-10-01T09:37:22.674516+0000","last_unstale":"2023-10-01T09:37:23.630356+0000","last_undegraded":"2023-10-01T09:37:23.630356+0000","last_fullsized":"2023-10-01T09:37:23.630356+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":475,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.638651+0000","last_change":"2023-10-01T09:36:22.225663+0000","last_active":"2023-10-01T09:37:23.638651+0000","last_peered":"2023-10-01T09:37:23.638651+0000","last_clean":"2023-10-01T09:37:23.638651+0000","last_became_active":"2023-10-01T09:36:22.225382+0000","last_became_peered":"2023-10-01T09:36:22.225382+0000","last_unstale":"2023-10-01T09:37:23.638651+0000","last_undegraded":"2023-10-01T09:37:23.638651+0000","last_fullsized":"2023-10-01T09:37:23.638651+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:19:41.386221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":475,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.639018+0000","last_change":"2023-10-01T09:35:34.551948+0000","last_active":"2023-10-01T09:37:23.639018+0000","last_peered":"2023-10-01T09:37:23.639018+0000","last_clean":"2023-10-01T09:37:23.639018+0000","last_became_active":"2023-10-01T09:35:34.551665+0000","last_became_peered":"2023-10-01T09:35:34.551665+0000","last_unstale":"2023-10-01T09:37:23.639018+0000","last_undegraded":"2023-10-01T09:37:23.639018+0000","last_fullsized":"2023-10-01T09:37:23.639018+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:31:30.655381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"448'9703","reported_seq":9805,"reported_epoch":448,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:37:33.970246+0000","last_change":"2023-10-01T09:36:23.705972+0000","last_active":"2023-10-01T09:37:33.970246+0000","last_peered":"2023-10-01T09:37:33.970246+0000","last_clean":"2023-10-01T09:36:21.165407+0000","last_became_active":"2023-10-01T09:36:23.529481+0000","last_became_peered":"2023-10-01T09:36:23.529481+0000","last_unstale":"2023-10-01T09:37:33.970246+0000","last_undegraded":"2023-10-01T09:37:33.970246+0000","last_fullsized":"2023-10-01T09:37:33.970246+0000","mapping_epoch":431,"log_start":"407'2844","ondisk_log_start":"407'2844","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6859,"log_dups_size":0,"ondisk_log_size":6859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:01:56.211865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217645056,"num_objects":3321,"num_object_clones":0,"num_object_copies":9963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3312,"num_objects_unfound":0,"num_objects_dirty":3321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9703,"num_write_kb":416768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":3321}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":886,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630856+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:37:23.630856+0000","last_peered":"2023-10-01T09:37:23.630856+0000","last_clean":"2023-10-01T09:37:23.630856+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:37:23.630856+0000","last_undegraded":"2023-10-01T09:37:23.630856+0000","last_fullsized":"2023-10-01T09:37:23.630856+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":504,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630709+0000","last_change":"2023-10-01T09:35:34.559426+0000","last_active":"2023-10-01T09:37:23.630709+0000","last_peered":"2023-10-01T09:37:23.630709+0000","last_clean":"2023-10-01T09:37:23.630709+0000","last_became_active":"2023-10-01T09:35:34.558890+0000","last_became_peered":"2023-10-01T09:35:34.558890+0000","last_unstale":"2023-10-01T09:37:23.630709+0000","last_undegraded":"2023-10-01T09:37:23.630709+0000","last_fullsized":"2023-10-01T09:37:23.630709+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:55:37.464032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":439,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.639168+0000","last_change":"2023-10-01T09:33:11.656981+0000","last_active":"2023-10-01T09:37:23.639168+0000","last_peered":"2023-10-01T09:37:23.639168+0000","last_clean":"2023-10-01T09:37:23.639168+0000","last_became_active":"2023-10-01T09:33:11.656334+0000","last_became_peered":"2023-10-01T09:33:11.656334+0000","last_unstale":"2023-10-01T09:37:23.639168+0000","last_undegraded":"2023-10-01T09:37:23.639168+0000","last_fullsized":"2023-10-01T09:37:23.639168+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:50:15.562113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"448'9458","reported_seq":9522,"reported_epoch":448,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:37:34.846977+0000","last_change":"2023-10-01T09:36:23.622968+0000","last_active":"2023-10-01T09:37:34.846977+0000","last_peered":"2023-10-01T09:37:34.846977+0000","last_clean":"2023-10-01T09:36:21.149981+0000","last_became_active":"2023-10-01T09:36:23.603244+0000","last_became_peered":"2023-10-01T09:36:23.603244+0000","last_unstale":"2023-10-01T09:37:34.846977+0000","last_undegraded":"2023-10-01T09:37:34.846977+0000","last_fullsized":"2023-10-01T09:37:34.846977+0000","mapping_epoch":431,"log_start":"407'2605","ondisk_log_start":"407'2605","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6853,"log_dups_size":0,"ondisk_log_size":6853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218693656,"num_objects":3338,"num_object_clones":0,"num_object_copies":10014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6676,"num_objects_unfound":0,"num_objects_dirty":3338,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9458,"num_write_kb":409409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,1,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":3338}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":387,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630635+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:37:23.630635+0000","last_peered":"2023-10-01T09:37:23.630635+0000","last_clean":"2023-10-01T09:37:23.630635+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:37:23.630635+0000","last_undegraded":"2023-10-01T09:37:23.630635+0000","last_fullsized":"2023-10-01T09:37:23.630635+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1077,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.631148+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:37:23.631148+0000","last_peered":"2023-10-01T09:37:23.631148+0000","last_clean":"2023-10-01T09:37:23.631148+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:37:23.631148+0000","last_undegraded":"2023-10-01T09:37:23.631148+0000","last_fullsized":"2023-10-01T09:37:23.631148+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":169,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.638445+0000","last_change":"2023-10-01T09:36:22.236397+0000","last_active":"2023-10-01T09:37:23.638445+0000","last_peered":"2023-10-01T09:37:23.638445+0000","last_clean":"2023-10-01T09:37:23.638445+0000","last_became_active":"2023-10-01T09:36:22.236066+0000","last_became_peered":"2023-10-01T09:36:22.236066+0000","last_unstale":"2023-10-01T09:37:23.638445+0000","last_undegraded":"2023-10-01T09:37:23.638445+0000","last_fullsized":"2023-10-01T09:37:23.638445+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:04:06.876501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"448'9606","reported_seq":9680,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-01T09:37:31.861276+0000","last_change":"2023-10-01T09:37:23.665454+0000","last_active":"2023-10-01T09:37:31.861276+0000","last_peered":"2023-10-01T09:37:31.861276+0000","last_clean":"2023-10-01T09:37:31.861276+0000","last_became_active":"2023-10-01T09:37:23.635951+0000","last_became_peered":"2023-10-01T09:37:23.635951+0000","last_unstale":"2023-10-01T09:37:31.861276+0000","last_undegraded":"2023-10-01T09:37:31.861276+0000","last_fullsized":"2023-10-01T09:37:31.861276+0000","mapping_epoch":446,"log_start":"405'600","ondisk_log_start":"405'600","created":400,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":9006,"log_dups_size":0,"ondisk_log_size":9006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223215616,"num_objects":3406,"num_object_clones":0,"num_object_copies":10218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9606,"num_write_kb":416384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":208,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.629527+0000","last_change":"2023-10-01T09:37:22.671608+0000","last_active":"2023-10-01T09:37:23.629527+0000","last_peered":"2023-10-01T09:37:23.629527+0000","last_clean":"2023-10-01T09:37:23.629527+0000","last_became_active":"2023-10-01T09:37:22.671097+0000","last_became_peered":"2023-10-01T09:37:22.671097+0000","last_unstale":"2023-10-01T09:37:23.629527+0000","last_undegraded":"2023-10-01T09:37:23.629527+0000","last_fullsized":"2023-10-01T09:37:23.629527+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":462,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.639098+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:37:23.639098+0000","last_peered":"2023-10-01T09:37:23.639098+0000","last_clean":"2023-10-01T09:37:23.639098+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:37:23.639098+0000","last_undegraded":"2023-10-01T09:37:23.639098+0000","last_fullsized":"2023-10-01T09:37:23.639098+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":171,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.632016+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:37:23.632016+0000","last_peered":"2023-10-01T09:37:23.632016+0000","last_clean":"2023-10-01T09:37:23.632016+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:37:23.632016+0000","last_undegraded":"2023-10-01T09:37:23.632016+0000","last_fullsized":"2023-10-01T09:37:23.632016+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":411,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.638943+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:37:23.638943+0000","last_peered":"2023-10-01T09:37:23.638943+0000","last_clean":"2023-10-01T09:37:23.638943+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:37:23.638943+0000","last_undegraded":"2023-10-01T09:37:23.638943+0000","last_fullsized":"2023-10-01T09:37:23.638943+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":516,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630786+0000","last_change":"2023-10-01T09:37:22.674112+0000","last_active":"2023-10-01T09:37:23.630786+0000","last_peered":"2023-10-01T09:37:23.630786+0000","last_clean":"2023-10-01T09:37:23.630786+0000","last_became_active":"2023-10-01T09:37:22.673785+0000","last_became_peered":"2023-10-01T09:37:22.673785+0000","last_unstale":"2023-10-01T09:37:23.630786+0000","last_undegraded":"2023-10-01T09:37:23.630786+0000","last_fullsized":"2023-10-01T09:37:23.630786+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:05.627976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":183,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.632526+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:37:23.632526+0000","last_peered":"2023-10-01T09:37:23.632526+0000","last_clean":"2023-10-01T09:37:23.632526+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:37:23.632526+0000","last_undegraded":"2023-10-01T09:37:23.632526+0000","last_fullsized":"2023-10-01T09:37:23.632526+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":246,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.634057+0000","last_change":"2023-10-01T09:37:22.671305+0000","last_active":"2023-10-01T09:37:23.634057+0000","last_peered":"2023-10-01T09:37:23.634057+0000","last_clean":"2023-10-01T09:37:23.634057+0000","last_became_active":"2023-10-01T09:37:22.670844+0000","last_became_peered":"2023-10-01T09:37:22.670844+0000","last_unstale":"2023-10-01T09:37:23.634057+0000","last_undegraded":"2023-10-01T09:37:23.634057+0000","last_fullsized":"2023-10-01T09:37:23.634057+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":183,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.632203+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:37:23.632203+0000","last_peered":"2023-10-01T09:37:23.632203+0000","last_clean":"2023-10-01T09:37:23.632203+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:37:23.632203+0000","last_undegraded":"2023-10-01T09:37:23.632203+0000","last_fullsized":"2023-10-01T09:37:23.632203+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":357,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.638683+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:37:23.638683+0000","last_peered":"2023-10-01T09:37:23.638683+0000","last_clean":"2023-10-01T09:37:23.638683+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:37:23.638683+0000","last_undegraded":"2023-10-01T09:37:23.638683+0000","last_fullsized":"2023-10-01T09:37:23.638683+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":13,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-10-01T09:37:26.733032+0000","last_change":"2023-10-01T09:37:22.673935+0000","last_active":"2023-10-01T09:37:26.733032+0000","last_peered":"2023-10-01T09:37:26.733032+0000","last_clean":"2023-10-01T09:37:26.733032+0000","last_became_active":"2023-10-01T09:37:22.673609+0000","last_became_peered":"2023-10-01T09:37:22.673609+0000","last_unstale":"2023-10-01T09:37:26.733032+0000","last_undegraded":"2023-10-01T09:37:26.733032+0000","last_fullsized":"2023-10-01T09:37:26.733032+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:02:43.715678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":393,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.638610+0000","last_change":"2023-10-01T09:37:22.668216+0000","last_active":"2023-10-01T09:37:23.638610+0000","last_peered":"2023-10-01T09:37:23.638610+0000","last_clean":"2023-10-01T09:37:23.638610+0000","last_became_active":"2023-10-01T09:37:22.667243+0000","last_became_peered":"2023-10-01T09:37:22.667243+0000","last_unstale":"2023-10-01T09:37:23.638610+0000","last_undegraded":"2023-10-01T09:37:23.638610+0000","last_fullsized":"2023-10-01T09:37:23.638610+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:45:16.527071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":586,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.639249+0000","last_change":"2023-10-01T09:35:34.658812+0000","last_active":"2023-10-01T09:37:23.639249+0000","last_peered":"2023-10-01T09:37:23.639249+0000","last_clean":"2023-10-01T09:37:23.639249+0000","last_became_active":"2023-10-01T09:35:34.550952+0000","last_became_peered":"2023-10-01T09:35:34.550952+0000","last_unstale":"2023-10-01T09:37:23.639249+0000","last_undegraded":"2023-10-01T09:37:23.639249+0000","last_fullsized":"2023-10-01T09:37:23.639249+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:24:17.304117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":417,"reported_epoch":447,"state":"active+clean","last_fresh":"2023-10-01T09:37:23.630912+0000","last_change":"2023-10-01T09:32:27.445299+0000","last_active":"2023-10-01T09:37:23.630912+0000","last_peered":"2023-10-01T09:37:23.630912+0000","last_clean":"2023-10-01T09:37:23.630912+0000","last_became_active":"2023-10-01T09:32:27.444956+0000","last_became_peered":"2023-10-01T09:32:27.444956+0000","last_unstale":"2023-10-01T09:37:23.630912+0000","last_undegraded":"2023-10-01T09:37:23.630912+0000","last_fullsized":"2023-10-01T09:37:23.630912+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:49:07.862356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":123,"num_bytes_recovered":34520,"num_keys_recovered":90,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3441164312,"num_objects":52509,"num_object_clones":0,"num_object_copies":157527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22222,"num_objects_misplaced":24645,"num_objects_unfound":0,"num_objects_dirty":52509,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152257,"num_write_kb":6552449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12941,"num_bytes_recovered":847249408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11413303296,"data_stored":11413291080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117515,"ondisk_log_size":117515,"up":48,"acting":44,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":297,"num_bytes_recovered":12582969,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430643,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3129604,"kb_used_data":2226492,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":90615548,"statfs":{"total":95995035648,"available":92790321152,"internally_reserved":0,"allocated":2279927808,"data_stored":2279025678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:36:38 2023","interfaces":[{"interface":"back","average":{"1min":6.1459999999999999,"5min":5.2000000000000002,"15min":3.3919999999999999},"min":{"1min":0.23400000000000001,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":81.162999999999997,"5min":108.90000000000001,"15min":108.90000000000001},"last":0.29099999999999998},{"interface":"front","average":{"1min":5.4770000000000003,"5min":5.1820000000000004,"15min":3.6030000000000002},"min":{"1min":0.26500000000000001,"5min":0.23300000000000001,"15min":0.22500000000000001},"max":{"1min":81.323999999999998,"5min":109.187,"15min":109.187},"last":0.38100000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:36:55 2023","interfaces":[{"interface":"back","average":{"1min":8.3919999999999995,"5min":9.4329999999999998,"15min":4.4790000000000001},"min":{"1min":0.249,"5min":0.23300000000000001,"15min":0.222},"max":{"1min":82.063000000000002,"5min":205.34200000000001,"15min":205.34200000000001},"last":3.3500000000000001},{"interface":"front","average":{"1min":9.0250000000000004,"5min":9.6099999999999994,"15min":4.5549999999999997},"min":{"1min":0.33600000000000002,"5min":0.24399999999999999,"15min":0.23599999999999999},"max":{"1min":82.331000000000003,"5min":205.44499999999999,"15min":205.44499999999999},"last":1.5620000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:37:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6539999999999999,"5min":2.2210000000000001,"15min":2.2389999999999999},"min":{"1min":0.372,"5min":0.27100000000000002,"15min":0.249},"max":{"1min":12.994,"5min":14.901999999999999,"15min":59.878},"last":12.994},{"interface":"front","average":{"1min":2.415,"5min":1.8149999999999999,"15min":2.1669999999999998},"min":{"1min":0.35799999999999998,"5min":0.27700000000000002,"15min":0.247},"max":{"1min":8.0050000000000008,"5min":8.0050000000000008,"15min":60.186},"last":2.6899999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:36:56 2023","interfaces":[{"interface":"back","average":{"1min":4.5960000000000001,"5min":3.7829999999999999,"15min":3.589},"min":{"1min":0.37,"5min":0.32900000000000001,"15min":0.28199999999999997},"max":{"1min":32.936999999999998,"5min":76.555000000000007,"15min":164.595},"last":1.21},{"interface":"front","average":{"1min":4.6900000000000004,"5min":3.7519999999999998,"15min":3.5089999999999999},"min":{"1min":0.435,"5min":0.33200000000000002,"15min":0.308},"max":{"1min":33.021999999999998,"5min":76.534000000000006,"15min":164.60499999999999},"last":1.2210000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:36:38 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":1.351,"15min":1.0289999999999999},"min":{"1min":0.29399999999999998,"5min":0.26700000000000002,"15min":0.23499999999999999},"max":{"1min":9.6020000000000003,"5min":9.6020000000000003,"15min":9.6020000000000003},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.284,"15min":1.044},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":3.0979999999999999,"5min":8.0009999999999994,"15min":9.4339999999999993},"last":0.54300000000000004}]},{"osd":5,"last update":"Sun Oct 1 09:37:18 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.0569999999999999,"15min":1.0069999999999999},"min":{"1min":0.29999999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.7090000000000001,"5min":6.1159999999999997,"15min":6.1159999999999997},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.71399999999999997,"5min":1.1859999999999999,"15min":0.92400000000000004},"min":{"1min":0.34499999999999997,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.0019999999999998,"5min":13.932,"15min":13.932},"last":0.59799999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:37:13 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.371,"15min":0.95999999999999996},"min":{"1min":0.32100000000000001,"5min":0.307,"15min":0.30299999999999999},"max":{"1min":6.069,"5min":11.853999999999999,"15min":11.853999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.343,"15min":0.91200000000000003},"min":{"1min":0.38200000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.2389999999999999,"5min":7.266,"15min":7.266},"last":0.46700000000000003}]}]},{"osd":6,"up_from":294,"seq":1262720385154,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":980348,"kb_used_data":638644,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":92764804,"statfs":{"total":95995035648,"available":94991159296,"internally_reserved":0,"allocated":653971456,"data_stored":653077393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":2.927,"15min":3.6000000000000001},"min":{"1min":0.249,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":6.2779999999999996,"5min":40.426000000000002,"15min":130.02699999999999},"last":1.962},{"interface":"front","average":{"1min":1.5840000000000001,"5min":2.5329999999999999,"15min":3.3199999999999998},"min":{"1min":0.28499999999999998,"5min":0.221,"15min":0.221},"max":{"1min":10.295,"5min":40.447000000000003,"15min":133.72900000000001},"last":0.31}]},{"osd":1,"last update":"Sun Oct 1 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":9.8670000000000009,"5min":5.5590000000000002,"15min":3.8719999999999999},"min":{"1min":0.26700000000000002,"5min":0.24199999999999999,"15min":0.23899999999999999},"max":{"1min":121.95,"5min":121.95,"15min":121.95},"last":5.7249999999999996},{"interface":"front","average":{"1min":9.5500000000000007,"5min":5.5190000000000001,"15min":3.7909999999999999},"min":{"1min":0.30599999999999999,"5min":0.25600000000000001,"15min":0.24399999999999999},"max":{"1min":121.98099999999999,"5min":121.98099999999999,"15min":121.98099999999999},"last":5.3019999999999996}]},{"osd":2,"last update":"Sun Oct 1 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":1.738,"15min":2.238},"min":{"1min":0.34200000000000003,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":4.9420000000000002,"5min":10.92,"15min":26.289999999999999},"last":3.9980000000000002},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.712,"15min":2.302},"min":{"1min":0.33500000000000002,"5min":0.309,"15min":0.27100000000000002},"max":{"1min":2.8610000000000002,"5min":12.804,"15min":26.388999999999999},"last":0.47599999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":2.7949999999999999,"15min":2.6920000000000002},"min":{"1min":0.35999999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.5609999999999999,"5min":89.753,"15min":89.753},"last":2.8780000000000001},{"interface":"front","average":{"1min":1.343,"5min":2.5640000000000001,"15min":2.0539999999999998},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.29299999999999998},"max":{"1min":4.4939999999999998,"5min":89.849999999999994,"15min":89.849999999999994},"last":0.372}]},{"osd":4,"last update":"Sun Oct 1 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.012,"15min":0.64800000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":1.823,"5min":6.0640000000000001,"15min":6.0640000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":1.0669999999999999,"15min":0.76700000000000002},"min":{"1min":0.315,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":1.762,"5min":9.4090000000000007,"15min":9.4090000000000007},"last":0.40799999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:37:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.0800000000000001,"15min":1.0580000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.6280000000000001,"5min":12.023999999999999,"15min":12.023999999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.95899999999999996,"5min":1.1240000000000001,"15min":1.1279999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.234,"5min":4.5309999999999997,"15min":4.5309999999999997},"last":0.44800000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":2.3919999999999999,"15min":3.4020000000000001},"min":{"1min":0.31900000000000001,"5min":0.30499999999999999,"15min":0.27900000000000003},"max":{"1min":3.105,"5min":94.245999999999995,"15min":115.11},"last":0.38},{"interface":"front","average":{"1min":0.67800000000000005,"5min":2.2149999999999999,"15min":3.323},"min":{"1min":0.35499999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.2890000000000001,"5min":94.715000000000003,"15min":114.666},"last":0.39600000000000002}]}]},{"osd":1,"up_from":94,"seq":403726926131,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2614800,"kb_used_data":1905864,"kb_used_omap":7,"kb_used_meta":708856,"kb_avail":91130352,"statfs":{"total":95995035648,"available":93317480448,"internally_reserved":0,"allocated":1951604736,"data_stored":1949759531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":725868967},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:36:46 2023","interfaces":[{"interface":"back","average":{"1min":17.527999999999999,"5min":6.2039999999999997,"15min":4.2619999999999996},"min":{"1min":0.25600000000000001,"5min":0.183,"15min":0.183},"max":{"1min":154.21299999999999,"5min":154.21299999999999,"15min":154.21299999999999},"last":22.736999999999998},{"interface":"front","average":{"1min":7.9710000000000001,"5min":4.1900000000000004,"15min":3.6629999999999998},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.17599999999999999},"max":{"1min":95.236999999999995,"5min":95.236999999999995,"15min":95.236999999999995},"last":25.047999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":4.6210000000000004,"5min":3.218,"15min":3.9140000000000001},"min":{"1min":0.23100000000000001,"5min":0.20000000000000001,"15min":0.182},"max":{"1min":13.898999999999999,"5min":51.640000000000001,"15min":77.325000000000003},"last":1.9410000000000001},{"interface":"front","average":{"1min":3.9950000000000001,"5min":4.6509999999999998,"15min":4.7430000000000003},"min":{"1min":0.27500000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":15.845000000000001,"5min":154.197,"15min":154.197},"last":0.63400000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:37:01 2023","interfaces":[{"interface":"back","average":{"1min":5.4889999999999999,"5min":3.758,"15min":4.5949999999999998},"min":{"1min":0.26700000000000002,"5min":0.219,"15min":0.219},"max":{"1min":27.486000000000001,"5min":52.417999999999999,"15min":200.56800000000001},"last":1.577},{"interface":"front","average":{"1min":12.455,"5min":5.1319999999999997,"15min":5.1210000000000004},"min":{"1min":0.318,"5min":0.216,"15min":0.21199999999999999},"max":{"1min":154.483,"5min":154.483,"15min":200.63800000000001},"last":5.1470000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:36:46 2023","interfaces":[{"interface":"back","average":{"1min":10.246,"5min":3.3660000000000001,"15min":2.0910000000000002},"min":{"1min":0.27600000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":154.101,"5min":154.101,"15min":154.101},"last":0.58799999999999997},{"interface":"front","average":{"1min":10.698,"5min":3.6669999999999998,"15min":2.5739999999999998},"min":{"1min":0.35299999999999998,"5min":0.32800000000000001,"15min":0.27500000000000002},"max":{"1min":154.56999999999999,"5min":154.56999999999999,"15min":154.56999999999999},"last":1.165}]},{"osd":5,"last update":"Sun Oct 1 09:37:25 2023","interfaces":[{"interface":"back","average":{"1min":3.7160000000000002,"5min":3.6230000000000002,"15min":1.8109999999999999},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":18.193999999999999,"5min":154.22999999999999,"15min":154.22999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":3.9609999999999999,"5min":4.024,"15min":2.6899999999999999},"min":{"1min":0.41999999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":13.848000000000001,"5min":154.54400000000001,"15min":154.54400000000001},"last":1.7849999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:37:01 2023","interfaces":[{"interface":"back","average":{"1min":10.959,"5min":4.0060000000000002,"15min":3.1880000000000002},"min":{"1min":0.442,"5min":0.313,"15min":0.29899999999999999},"max":{"1min":154.12200000000001,"5min":154.12200000000001,"15min":154.12200000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":10.606,"5min":3.5449999999999999,"15min":2.4399999999999999},"min":{"1min":0.35499999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":154.16999999999999,"5min":154.16999999999999,"15min":154.16999999999999},"last":0.66600000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:36:46 2023","interfaces":[{"interface":"back","average":{"1min":10.465999999999999,"5min":3.976,"15min":2.5169999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.316},"max":{"1min":154.20699999999999,"5min":154.20699999999999,"15min":154.20699999999999},"last":0.621},{"interface":"front","average":{"1min":10.913,"5min":4.0780000000000003,"15min":2.77},"min":{"1min":0.437,"5min":0.34300000000000003,"15min":0.30199999999999999},"max":{"1min":154.51599999999999,"5min":154.51599999999999,"15min":154.51599999999999},"last":1.198}]}]},{"osd":0,"up_from":89,"seq":382252089665,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1664548,"kb_used_data":906780,"kb_used_omap":8,"kb_used_meta":757687,"kb_avail":92080604,"statfs":{"total":95995035648,"available":94290538496,"internally_reserved":0,"allocated":928542720,"data_stored":926684118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":775872450},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":18.449000000000002,"5min":6.5510000000000002,"15min":3.8399999999999999},"min":{"1min":0.182,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":216.821,"5min":216.821,"15min":216.821},"last":0.51300000000000001},{"interface":"front","average":{"1min":16.364000000000001,"5min":6.3550000000000004,"15min":4.5570000000000004},"min":{"1min":0.26400000000000001,"5min":0.246,"15min":0.20200000000000001},"max":{"1min":217.02699999999999,"5min":217.02699999999999,"15min":217.02699999999999},"last":25.888999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:37:26 2023","interfaces":[{"interface":"back","average":{"1min":4.7539999999999996,"5min":2.778,"15min":2.25},"min":{"1min":0.28599999999999998,"5min":0.21299999999999999,"15min":0.20100000000000001},"max":{"1min":17.462,"5min":17.462,"15min":17.731000000000002},"last":0.25900000000000001},{"interface":"front","average":{"1min":3.536,"5min":2.4359999999999999,"15min":2.887},"min":{"1min":0.34499999999999997,"5min":0.222,"15min":0.189},"max":{"1min":9.4550000000000001,"5min":15.472,"15min":22.07},"last":25.844999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:36:53 2023","interfaces":[{"interface":"back","average":{"1min":16.757000000000001,"5min":5.6799999999999997,"15min":4.7279999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":222.16499999999999,"5min":222.16499999999999,"15min":222.16499999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":14.465999999999999,"5min":4.7130000000000001,"15min":4.976},"min":{"1min":0.28199999999999997,"5min":0.24399999999999999,"15min":0.219},"max":{"1min":222.227,"5min":222.227,"15min":222.227},"last":4.9400000000000004}]},{"osd":4,"last update":"Sun Oct 1 09:36:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4540000000000002,"5min":2.621,"15min":2.2149999999999999},"min":{"1min":0.34799999999999998,"5min":0.27600000000000002,"15min":0.27300000000000002},"max":{"1min":15.449999999999999,"5min":62.206000000000003,"15min":62.206000000000003},"last":5.0529999999999999},{"interface":"front","average":{"1min":2.8769999999999998,"5min":2.585,"15min":2.2269999999999999},"min":{"1min":0.34499999999999997,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":9.1850000000000005,"5min":62.073999999999998,"15min":62.073999999999998},"last":4.984}]},{"osd":5,"last update":"Sun Oct 1 09:37:17 2023","interfaces":[{"interface":"back","average":{"1min":3.1699999999999999,"5min":2.3210000000000002,"15min":1.9470000000000001},"min":{"1min":0.48699999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":12.268000000000001,"5min":16.808,"15min":16.808},"last":5.0199999999999996},{"interface":"front","average":{"1min":3.911,"5min":2.3140000000000001,"15min":1.859},"min":{"1min":0.35299999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":17.507000000000001,"5min":17.507000000000001,"15min":17.507000000000001},"last":0.46400000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:37:12 2023","interfaces":[{"interface":"back","average":{"1min":3.7320000000000002,"5min":2.2679999999999998,"15min":1.7889999999999999},"min":{"1min":0.35699999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":17.535,"5min":17.535,"15min":17.535},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.0249999999999999,"5min":2.1659999999999999,"15min":2.3460000000000001},"min":{"1min":0.41799999999999998,"5min":0.34200000000000003,"15min":0.32100000000000001},"max":{"1min":5.5670000000000002,"5min":15.651999999999999,"15min":18.521000000000001},"last":5.1200000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:36:44 2023","interfaces":[{"interface":"back","average":{"1min":3.956,"5min":2.919,"15min":1.9470000000000001},"min":{"1min":0.41299999999999998,"5min":0.39400000000000002,"15min":0.34499999999999997},"max":{"1min":17.559999999999999,"5min":35.130000000000003,"15min":35.130000000000003},"last":0.54700000000000004},{"interface":"front","average":{"1min":2.407,"5min":2.6429999999999998,"15min":2.3380000000000001},"min":{"1min":0.40600000000000003,"5min":0.38200000000000001,"15min":0.35099999999999998},"max":{"1min":16.088999999999999,"5min":38.590000000000003,"15min":38.590000000000003},"last":5.1470000000000002}]}]},{"osd":2,"up_from":99,"seq":425201762605,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1275812,"kb_used_data":564956,"kb_used_omap":11,"kb_used_meta":710836,"kb_avail":92469340,"statfs":{"total":95995035648,"available":94688604160,"internally_reserved":0,"allocated":578514944,"data_stored":576712980,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":727896951},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:36:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":7.7619999999999996,"15min":6.8010000000000002},"min":{"1min":0.19400000000000001,"5min":0.191,"15min":0.17599999999999999},"max":{"1min":11.518000000000001,"5min":201.892,"15min":224.56},"last":0.37},{"interface":"front","average":{"1min":2.6040000000000001,"5min":7.8579999999999997,"15min":7.2679999999999998},"min":{"1min":0.52100000000000002,"5min":0.23100000000000001,"15min":0.20699999999999999},"max":{"1min":7.3319999999999999,"5min":200.089,"15min":225.66900000000001},"last":0.378}]},{"osd":1,"last update":"Sun Oct 1 09:37:32 2023","interfaces":[{"interface":"back","average":{"1min":6.9980000000000002,"5min":12.176,"15min":8.5950000000000006},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.191},"max":{"1min":65.566999999999993,"5min":196.34399999999999,"15min":224.917},"last":0.34200000000000003},{"interface":"front","average":{"1min":8.5939999999999994,"5min":12.744999999999999,"15min":9.532},"min":{"1min":0.30099999999999999,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":65.733999999999995,"5min":196.542,"15min":225.625},"last":3.782}]},{"osd":3,"last update":"Sun Oct 1 09:36:59 2023","interfaces":[{"interface":"back","average":{"1min":6.2629999999999999,"5min":2.6160000000000001,"15min":4.9290000000000003},"min":{"1min":0.246,"5min":0.216,"15min":0.21099999999999999},"max":{"1min":50.289000000000001,"5min":50.289000000000001,"15min":226.124},"last":9.8970000000000002},{"interface":"front","average":{"1min":6.5030000000000001,"5min":2.665,"15min":4.7759999999999998},"min":{"1min":0.23200000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":54.115000000000002,"5min":54.115000000000002,"15min":226.32900000000001},"last":0.39600000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:37:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.5800000000000001,"15min":2.5920000000000001},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.27200000000000002},"max":{"1min":4.9089999999999998,"5min":22.084,"15min":224.90199999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.4209999999999998,"5min":2.0550000000000002,"15min":2.8700000000000001},"min":{"1min":0.33200000000000002,"5min":0.27800000000000002,"15min":0.27500000000000002},"max":{"1min":11.475,"5min":22.266999999999999,"15min":225.386},"last":6.2610000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:37:16 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.508,"15min":1.6100000000000001},"min":{"1min":0.309,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":11.305999999999999,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":2.0459999999999998,"5min":1.3779999999999999,"15min":1.339},"min":{"1min":0.441,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.8380000000000001,"5min":10.194000000000001,"15min":10.194000000000001},"last":0.45000000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:37:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7130000000000001,"5min":1.887,"15min":2.8559999999999999},"min":{"1min":0.35399999999999998,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":11.526999999999999,"5min":16.184999999999999,"15min":225.41499999999999},"last":6.3019999999999996},{"interface":"front","average":{"1min":1.913,"5min":1.6499999999999999,"15min":2.3969999999999998},"min":{"1min":0.33700000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":11.239000000000001,"5min":13.614000000000001,"15min":225.02000000000001},"last":0.62}]},{"osd":7,"last update":"Sun Oct 1 09:37:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":4.7560000000000002,"15min":3.851},"min":{"1min":0.372,"5min":0.31,"15min":0.308},"max":{"1min":11.337999999999999,"5min":139.03299999999999,"15min":224.94999999999999},"last":0.64700000000000002},{"interface":"front","average":{"1min":1.8540000000000001,"5min":4.524,"15min":3.9249999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.34499999999999997},"max":{"1min":5.8150000000000004,"5min":139.065,"15min":224.92599999999999},"last":0.47799999999999998}]}]},{"osd":3,"up_from":104,"seq":446676599078,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2744140,"kb_used_data":2014916,"kb_used_omap":3,"kb_used_meta":729212,"kb_avail":91001012,"statfs":{"total":95995035648,"available":93185036288,"internally_reserved":0,"allocated":2063273984,"data_stored":2061374509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":746713831},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:36:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0649999999999999,"5min":4.5730000000000004,"15min":5.6029999999999998},"min":{"1min":0.22600000000000001,"5min":0.186,"15min":0.18099999999999999},"max":{"1min":15.612,"5min":137.34399999999999,"15min":137.34399999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":4.0309999999999997,"5min":4.4960000000000004,"15min":5.4909999999999997},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.184},"max":{"1min":15.635,"5min":137.73400000000001,"15min":137.73400000000001},"last":6.6159999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:36:59 2023","interfaces":[{"interface":"back","average":{"1min":17.129000000000001,"5min":8.6790000000000003,"15min":5.4669999999999996},"min":{"1min":0.34499999999999997,"5min":0.22700000000000001,"15min":0.20399999999999999},"max":{"1min":153.77000000000001,"5min":162.22999999999999,"15min":162.22999999999999},"last":3.1840000000000002},{"interface":"front","average":{"1min":16.012,"5min":8.4529999999999994,"15min":5.8120000000000003},"min":{"1min":0.29499999999999998,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":151.02199999999999,"5min":162.70699999999999,"15min":162.70699999999999},"last":14.141999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:36:59 2023","interfaces":[{"interface":"back","average":{"1min":3.5,"5min":1.708,"15min":3.1760000000000002},"min":{"1min":0.22800000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":14.194000000000001,"5min":14.194000000000001,"15min":62.909999999999997},"last":3.302},{"interface":"front","average":{"1min":2.1269999999999998,"5min":2.3700000000000001,"15min":3.8839999999999999},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":15.523,"5min":15.523,"15min":78.594999999999999},"last":0.65800000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:36:59 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":2.0030000000000001,"15min":2.2149999999999999},"min":{"1min":0.29099999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.407,"5min":29.419,"15min":108.896},"last":0.59499999999999997},{"interface":"front","average":{"1min":2.8780000000000001,"5min":2.363,"15min":2.4529999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.23899999999999999},"max":{"1min":9.0609999999999999,"5min":33.002000000000002,"15min":108.916},"last":0.46100000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:37:16 2023","interfaces":[{"interface":"back","average":{"1min":2.589,"5min":2.7069999999999999,"15min":3.4910000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":9.0839999999999996,"5min":28.117999999999999,"15min":28.117999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":3.6640000000000001,"5min":2.9249999999999998,"15min":3.4580000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":14.537000000000001,"5min":28.34,"15min":28.34},"last":3.234}]},{"osd":6,"last update":"Sun Oct 1 09:37:13 2023","interfaces":[{"interface":"back","average":{"1min":3.6019999999999999,"5min":1.9330000000000001,"15min":2.4340000000000002},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":12.521000000000001,"5min":12.521000000000001,"15min":18.361000000000001},"last":3.3330000000000002},{"interface":"front","average":{"1min":3.746,"5min":1.9890000000000001,"15min":2.3839999999999999},"min":{"1min":0.38600000000000001,"5min":0.33600000000000002,"15min":0.32800000000000001},"max":{"1min":13.031000000000001,"5min":13.031000000000001,"15min":18.332000000000001},"last":3.355}]},{"osd":7,"last update":"Sun Oct 1 09:36:59 2023","interfaces":[{"interface":"back","average":{"1min":3.6280000000000001,"5min":2.0750000000000002,"15min":2.5459999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.298,"5min":11.298,"15min":52.006},"last":3.2759999999999998},{"interface":"front","average":{"1min":2.492,"5min":2.081,"15min":2.4500000000000002},"min":{"1min":0.34699999999999998,"5min":0.311,"15min":0.311},"max":{"1min":8.2599999999999998,"5min":13.191000000000001,"15min":51.890999999999998},"last":0.628}]}]},{"osd":4,"up_from":30,"seq":128849019328,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1765956,"kb_used_data":871676,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":91979196,"statfs":{"total":95995035648,"available":94186696704,"internally_reserved":0,"allocated":892596224,"data_stored":891694112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:36:47 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":6.3319999999999999,"15min":4.4699999999999998},"min":{"1min":0.26900000000000002,"5min":0.22600000000000001,"15min":0.223},"max":{"1min":16.763999999999999,"5min":155.315,"15min":155.315},"last":4.6639999999999997},{"interface":"front","average":{"1min":3.2959999999999998,"5min":6.9160000000000004,"15min":4.6909999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":17.798999999999999,"5min":155.334,"15min":155.334},"last":6.8360000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:36:51 2023","interfaces":[{"interface":"back","average":{"1min":13.585000000000001,"5min":6.7750000000000004,"15min":6.1509999999999998},"min":{"1min":0.33700000000000002,"5min":0.25,"15min":0.25},"max":{"1min":165.666,"5min":190.61199999999999,"15min":190.61199999999999},"last":7.141},{"interface":"front","average":{"1min":12.738,"5min":8.9930000000000003,"15min":6.8300000000000001},"min":{"1min":0.26500000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":165.70500000000001,"5min":253.273,"15min":253.273},"last":1.7410000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:36:31 2023","interfaces":[{"interface":"back","average":{"1min":3.0379999999999998,"5min":3.8999999999999999,"15min":3.3900000000000001},"min":{"1min":0.35199999999999998,"5min":0.28299999999999997,"15min":0.26700000000000002},"max":{"1min":7.6230000000000002,"5min":170.45400000000001,"15min":170.45400000000001},"last":8.6379999999999999},{"interface":"front","average":{"1min":3.1800000000000002,"5min":2.8170000000000002,"15min":3.0089999999999999},"min":{"1min":0.433,"5min":0.33300000000000002,"15min":0.25800000000000001},"max":{"1min":6.6680000000000001,"5min":36.515999999999998,"15min":91.510999999999996},"last":8.9540000000000006}]},{"osd":3,"last update":"Sun Oct 1 09:36:51 2023","interfaces":[{"interface":"back","average":{"1min":11.609,"5min":3.7839999999999998,"15min":3.3889999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":140.881,"5min":140.881,"15min":140.881},"last":3.218},{"interface":"front","average":{"1min":11.242000000000001,"5min":3.7360000000000002,"15min":3.629},"min":{"1min":0.32400000000000001,"5min":0.317,"15min":0.30399999999999999},"max":{"1min":140.911,"5min":140.911,"15min":140.911},"last":0.39000000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:37:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":1.526,"15min":1.232},"min":{"1min":0.32100000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.8929999999999998,"5min":20.457000000000001,"15min":20.457000000000001},"last":6.8810000000000002},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.397,"15min":0.96499999999999997},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.899,"5min":20.475000000000001,"15min":20.475000000000001},"last":0.34999999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:37:04 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":1.2869999999999999,"15min":0.86399999999999999},"min":{"1min":0.29199999999999998,"5min":0.28000000000000003,"15min":0.27200000000000002},"max":{"1min":7.7649999999999997,"5min":11.358000000000001,"15min":11.358000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.6180000000000001,"5min":1.4399999999999999,"15min":0.90800000000000003},"min":{"1min":0.35299999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.7060000000000004,"5min":10.076000000000001,"15min":10.076000000000001},"last":6.9139999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:36:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":1.5900000000000001,"15min":1.034},"min":{"1min":0.316,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":7.7960000000000003,"5min":22.256,"15min":22.256},"last":1.9159999999999999},{"interface":"front","average":{"1min":2.129,"5min":1.7529999999999999,"15min":1.268},"min":{"1min":0.34200000000000003,"5min":0.308,"15min":0.28699999999999998},"max":{"1min":5.7190000000000003,"5min":22.635999999999999,"15min":22.635999999999999},"last":9}]}]},{"osd":5,"up_from":379,"seq":1627792605238,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2472124,"kb_used_data":2111668,"kb_used_omap":13,"kb_used_meta":360434,"kb_avail":91273028,"statfs":{"total":95995035648,"available":93463580672,"internally_reserved":0,"allocated":2162348032,"data_stored":2161449606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":369084914},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:37:17 2023","interfaces":[{"interface":"back","average":{"1min":3.4159999999999999,"5min":3.3740000000000001,"15min":2.1280000000000001},"min":{"1min":0.249,"5min":0.221,"15min":0.221},"max":{"1min":13.385999999999999,"5min":84.340999999999994,"15min":84.340999999999994},"last":1.1000000000000001},{"interface":"front","average":{"1min":2.7679999999999998,"5min":3.6469999999999998,"15min":2.036},"min":{"1min":0.317,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":12.445,"5min":84.356999999999999,"15min":84.356999999999999},"last":5.6109999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:37:17 2023","interfaces":[{"interface":"back","average":{"1min":15.673999999999999,"5min":9.0890000000000004,"15min":4.1609999999999996},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":236.488,"5min":236.488,"15min":236.488},"last":0.754},{"interface":"front","average":{"1min":14.218999999999999,"5min":8.6229999999999993,"15min":3.8039999999999998},"min":{"1min":0.28100000000000003,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":236.29900000000001,"5min":236.29900000000001,"15min":236.29900000000001},"last":1.847}]},{"osd":2,"last update":"Sun Oct 1 09:37:17 2023","interfaces":[{"interface":"back","average":{"1min":3.3090000000000002,"5min":2.0499999999999998,"15min":1.552},"min":{"1min":0.34300000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":14.199999999999999,"5min":14.199999999999999,"15min":14.199999999999999},"last":5.5620000000000003},{"interface":"front","average":{"1min":2.702,"5min":1.74,"15min":1.661},"min":{"1min":0.32700000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":14.430999999999999,"5min":14.430999999999999,"15min":14.430999999999999},"last":0.79800000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:37:17 2023","interfaces":[{"interface":"back","average":{"1min":2.9140000000000001,"5min":1.625,"15min":1.3160000000000001},"min":{"1min":0.43099999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":14.004,"5min":14.004,"15min":14.004},"last":1.4530000000000001},{"interface":"front","average":{"1min":3.0680000000000001,"5min":1.984,"15min":1.446},"min":{"1min":0.40999999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":13.991,"5min":13.991,"15min":13.991},"last":5.6429999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:37:17 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.242,"15min":1.04},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.7729999999999997,"5min":12.808999999999999,"15min":12.808999999999999},"last":0.376},{"interface":"front","average":{"1min":0.81499999999999995,"5min":1.4390000000000001,"15min":1.5900000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.7970000000000002,"5min":14.68,"15min":14.68},"last":0.53500000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:37:17 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.91600000000000004,"15min":0.995},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.0270000000000001,"5min":4.3810000000000002,"15min":4.3810000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.5980000000000001,"15min":1.53},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.1779999999999999,"5min":14.193,"15min":14.193},"last":0.40699999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:37:17 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.4510000000000001,"15min":1.3959999999999999},"min":{"1min":0.39300000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":6.7130000000000001,"5min":10.813000000000001,"15min":10.813000000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.131,"15min":0.97299999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":10.786,"5min":12.045999999999999,"15min":12.045999999999999},"last":0.55300000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":914948096,"data_stored":914948096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1936527360,"data_stored":1936523288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":570228736,"data_stored":570228736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2052456448,"data_stored":2052456448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":881328128,"data_stored":881328128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2145325056,"data_stored":2145320984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":645201920,"data_stored":645201920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2267287552,"data_stored":2267283480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:37:38.447 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-01T09:37:38.447 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 11 2023-10-01T09:37:38.507 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:38 smithi190 ceph-mon[160435]: from='client.36264 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:37:38.508 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:38 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 22235/157611 objects degraded (14.108%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:38.508 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:38 smithi190 ceph-mon[160435]: pgmap v1257: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 22235/156501 objects degraded (14.208%); 24645/156501 objects misplaced (15.748%); 5.6 MiB/s, 84 objects/s recovering 2023-10-01T09:37:38.524 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:37:38.602 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:38.697 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:37:38.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:38.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:37:38.788 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:37:38.788 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:37:38.788 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:37:38.788 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:37:38.788 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:37:38.801 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:38.864 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:38.993 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:38.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:38.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:39.098 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:39.127 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:39.178 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:39.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:39.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:39.285 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:39.318 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:39.362 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:39.455 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:39.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:39.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:39.559 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:39.595 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:37:39.598 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:39.658 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:39.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:39.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:39.764 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:37:39.776 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:37:39.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:39 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3860573517' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:39 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:39.920 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:39 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3860573517' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:39.920 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:39 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:39.920 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:39 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3860573517' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:39.920 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:39 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:39.934 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:39.974 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:37:40.196 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:37:40.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:40.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:37:40.309 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:37:40.406 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:40.454 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:37:40.912 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:40 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:37:40.912 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:40 smithi190 ceph-mon[160435]: failed to encode map e449 with expected crc 2023-10-01T09:37:40.912 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:40 smithi190 ceph-mon[160435]: failed to encode map e449 with expected crc 2023-10-01T09:37:40.912 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:40 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:37:40.913 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:40 smithi190 ceph-mon[160435]: osdmap e449: 8 total, 8 up, 8 in 2023-10-01T09:37:40.913 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:40 smithi190 ceph-mon[160435]: pgmap v1259: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 21885/151500 objects degraded (14.446%); 23927/151500 objects misplaced (15.793%); 6.1 MiB/s, 96 objects/s recovering 2023-10-01T09:37:40.913 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:37:40.913 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:37:41.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:37:41.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[192513]: failed to encode map e449 with expected crc 2023-10-01T09:37:41.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[192513]: failed to encode map e449 with expected crc 2023-10-01T09:37:41.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:37:41.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[192513]: osdmap e449: 8 total, 8 up, 8 in 2023-10-01T09:37:41.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[192513]: pgmap v1259: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 21885/151500 objects degraded (14.446%); 23927/151500 objects misplaced (15.793%); 6.1 MiB/s, 96 objects/s recovering 2023-10-01T09:37:41.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:37:41.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:37:41.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:37:41.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[185886]: failed to encode map e449 with expected crc 2023-10-01T09:37:41.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[185886]: failed to encode map e449 with expected crc 2023-10-01T09:37:41.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:37:41.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[185886]: osdmap e449: 8 total, 8 up, 8 in 2023-10-01T09:37:41.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[185886]: pgmap v1259: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 21885/151500 objects degraded (14.446%); 23927/151500 objects misplaced (15.793%); 6.1 MiB/s, 96 objects/s recovering 2023-10-01T09:37:41.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:37:41.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:37:41.998 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:37:42.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:42.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:37:42.117 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:42.152 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:42.287 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:42.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:42.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:42.357 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:42.399 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:42.448 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:42.514 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:42.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:42.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:42.548 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:37:42.548 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:37:42.548 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:37:42.548 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:37:42.548 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.548 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:132665 9.3 9:c75edcf0:::benchmark_data_smithi138_497829_object30280:head [delete] snapc 0=[] ondisk+write+known_if_redirected e436)", 2023-10-01T09:37:42.549 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:36:38.303327+0000", 2023-10-01T09:37:42.549 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 64.241070875000005, 2023-10-01T09:37:42.549 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24726221000000001, 2023-10-01T09:37:42.549 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.549 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.549 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.549 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.549 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.549 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 132665 2023-10-01T09:37:42.549 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.549 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.549 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.550 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.550 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:36:38.303327+0000", 2023-10-01T09:37:42.550 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.550 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.550 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.550 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.550 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:36:38.303327+0000", 2023-10-01T09:37:42.550 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:42.550 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.550 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.550 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.551 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:36:38.303325+0000", 2023-10-01T09:37:42.551 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0616e-05 2023-10-01T09:37:42.551 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.551 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.551 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.551 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:36:38.303336+0000", 2023-10-01T09:37:42.551 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6653e-05 2023-10-01T09:37:42.551 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.551 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.551 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.551 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:36:38.303362+0000", 2023-10-01T09:37:42.551 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3870000000000001e-06 2023-10-01T09:37:42.552 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.552 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:36:38.303372+0000", 2023-10-01T09:37:42.552 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9988000000000002e-05 2023-10-01T09:37:42.552 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.552 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:36:38.303432+0000", 2023-10-01T09:37:42.552 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013836 2023-10-01T09:37:42.552 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.553 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:36:38.303570+0000", 2023-10-01T09:37:42.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013862499999999999 2023-10-01T09:37:42.553 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.553 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.553 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:37:42.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:36:38.303709+0000", 2023-10-01T09:37:42.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0042711219999999996 2023-10-01T09:37:42.553 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.553 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.553 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:36:38.307980+0000", 2023-10-01T09:37:42.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.186662103 2023-10-01T09:37:42.554 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.554 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.554 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.554 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:36:38.494642+0000", 2023-10-01T09:37:42.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.055782142999999999 2023-10-01T09:37:42.554 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.554 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.554 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.554 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:36:38.550424+0000", 2023-10-01T09:37:42.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6589999999999995e-05 2023-10-01T09:37:42.554 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.555 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.555 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.555 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:36:38.550521+0000", 2023-10-01T09:37:42.555 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8532e-05 2023-10-01T09:37:42.555 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.555 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.555 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.555 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:36:38.550589+0000", 2023-10-01T09:37:42.555 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24726221000000001 2023-10-01T09:37:42.555 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.555 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.555 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.556 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.556 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:141461 9.3 9:cb830d82:::benchmark_data_smithi138_497829_object39076:head [delete] snapc 0=[] ondisk+write+known_if_redirected e440)", 2023-10-01T09:37:42.556 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:03.239315+0000", 2023-10-01T09:37:42.556 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 39.305083265, 2023-10-01T09:37:42.556 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.25312904899999999, 2023-10-01T09:37:42.556 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.556 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.556 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.556 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.556 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.557 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 141461 2023-10-01T09:37:42.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.557 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239315+0000", 2023-10-01T09:37:42.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239315+0000", 2023-10-01T09:37:42.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:42.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.558 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239313+0000", 2023-10-01T09:37:42.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2360000000000004e-06 2023-10-01T09:37:42.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.558 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239321+0000", 2023-10-01T09:37:42.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6582999999999999e-05 2023-10-01T09:37:42.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.559 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239348+0000", 2023-10-01T09:37:42.559 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6290000000000001e-06 2023-10-01T09:37:42.559 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.559 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.559 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.559 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239355+0000", 2023-10-01T09:37:42.559 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2180000000000004e-05 2023-10-01T09:37:42.559 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.561 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.561 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.561 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239418+0000", 2023-10-01T09:37:42.561 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013170099999999999 2023-10-01T09:37:42.561 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.561 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.562 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239549+0000", 2023-10-01T09:37:42.562 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014726299999999999 2023-10-01T09:37:42.562 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.562 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:37:42.562 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239697+0000", 2023-10-01T09:37:42.562 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0072165490000000001 2023-10-01T09:37:42.562 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.562 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.246913+0000", 2023-10-01T09:37:42.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050893930000000002 2023-10-01T09:37:42.563 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.563 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.563 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.252003+0000", 2023-10-01T09:37:42.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24036835300000001 2023-10-01T09:37:42.563 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.563 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.563 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.492371+0000", 2023-10-01T09:37:42.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7339000000000003e-05 2023-10-01T09:37:42.564 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.564 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.564 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.564 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.492408+0000", 2023-10-01T09:37:42.564 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5488000000000003e-05 2023-10-01T09:37:42.564 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.564 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.564 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.564 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.492444+0000", 2023-10-01T09:37:42.564 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.25312904899999999 2023-10-01T09:37:42.564 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.564 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.565 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.565 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.565 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.565 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:141463 9.3 9:c1aa7253:::benchmark_data_smithi138_497829_object39078:head [delete] snapc 0=[] ondisk+write+known_if_redirected e440)", 2023-10-01T09:37:42.565 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:03.239376+0000", 2023-10-01T09:37:42.565 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 39.305021703999998, 2023-10-01T09:37:42.565 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.25405740700000001, 2023-10-01T09:37:42.565 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.565 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.565 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.565 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.566 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.566 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 141463 2023-10-01T09:37:42.566 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.566 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.566 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.566 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.566 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239376+0000", 2023-10-01T09:37:42.566 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.566 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.566 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.566 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.566 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239376+0000", 2023-10-01T09:37:42.567 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:37:42.567 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.567 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.567 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.567 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239376+0000", 2023-10-01T09:37:42.567 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2710000000000001e-06 2023-10-01T09:37:42.567 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.567 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.567 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.567 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239379+0000", 2023-10-01T09:37:42.567 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2795e-05 2023-10-01T09:37:42.567 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.568 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.568 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.568 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239392+0000", 2023-10-01T09:37:42.568 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1609999999999996e-06 2023-10-01T09:37:42.568 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.568 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.568 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.568 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.239396+0000", 2023-10-01T09:37:42.568 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038622109999999999 2023-10-01T09:37:42.568 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.568 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.568 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.569 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.243258+0000", 2023-10-01T09:37:42.569 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000160765 2023-10-01T09:37:42.569 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.569 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.569 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.569 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.243419+0000", 2023-10-01T09:37:42.569 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000118683 2023-10-01T09:37:42.569 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.569 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.569 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:37:42.569 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.243538+0000", 2023-10-01T09:37:42.569 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003500506 2023-10-01T09:37:42.570 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.570 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.570 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.570 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.247038+0000", 2023-10-01T09:37:42.570 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050020610000000004 2023-10-01T09:37:42.570 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.570 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.570 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.570 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.252040+0000", 2023-10-01T09:37:42.570 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24131724099999999 2023-10-01T09:37:42.570 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.570 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.571 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.571 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.493357+0000", 2023-10-01T09:37:42.571 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8696999999999999e-05 2023-10-01T09:37:42.571 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.571 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.571 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.571 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.493396+0000", 2023-10-01T09:37:42.571 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7453999999999997e-05 2023-10-01T09:37:42.571 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.571 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.571 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.571 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:03.493434+0000", 2023-10-01T09:37:42.572 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.25405740700000001 2023-10-01T09:37:42.572 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.572 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.572 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.572 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.572 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.572 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:145796 9.c 9:3a5a6dfb:::benchmark_data_smithi138_497829_object43411:head [delete] snapc 0=[] ondisk+write+known_if_redirected e444)", 2023-10-01T09:37:42.572 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:16.533300+0000", 2023-10-01T09:37:42.572 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 26.011097452000001, 2023-10-01T09:37:42.572 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.29276776999999998, 2023-10-01T09:37:42.572 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.572 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.573 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.573 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.573 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.573 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 145796 2023-10-01T09:37:42.573 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.573 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.573 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.573 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.573 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533300+0000", 2023-10-01T09:37:42.573 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.573 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.573 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.574 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.574 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533300+0000", 2023-10-01T09:37:42.574 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:37:42.574 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.574 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.574 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.574 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533298+0000", 2023-10-01T09:37:42.574 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0709999999999995e-06 2023-10-01T09:37:42.574 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.574 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.574 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.574 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533306+0000", 2023-10-01T09:37:42.575 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5869999999999999e-05 2023-10-01T09:37:42.575 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.575 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.575 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.575 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533322+0000", 2023-10-01T09:37:42.575 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.815e-06 2023-10-01T09:37:42.575 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.575 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.575 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.575 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533329+0000", 2023-10-01T09:37:42.575 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00163053 2023-10-01T09:37:42.575 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.576 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.576 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.576 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.534959+0000", 2023-10-01T09:37:42.576 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014893600000000001 2023-10-01T09:37:42.576 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.576 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.576 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.576 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.535108+0000", 2023-10-01T09:37:42.576 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000164644 2023-10-01T09:37:42.576 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.576 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.576 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,3,4", 2023-10-01T09:37:42.577 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.535273+0000", 2023-10-01T09:37:42.577 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0048375980000000002 2023-10-01T09:37:42.577 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.577 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.577 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.577 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.540110+0000", 2023-10-01T09:37:42.577 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001286875 2023-10-01T09:37:42.577 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.577 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.577 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.577 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.541397+0000", 2023-10-01T09:37:42.578 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001817658 2023-10-01T09:37:42.578 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.578 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.578 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.578 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.543215+0000", 2023-10-01T09:37:42.578 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28271621800000002 2023-10-01T09:37:42.578 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.578 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.578 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.578 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.825931+0000", 2023-10-01T09:37:42.578 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1245999999999998e-05 2023-10-01T09:37:42.578 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.579 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.579 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.579 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.825972+0000", 2023-10-01T09:37:42.579 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5760000000000005e-05 2023-10-01T09:37:42.579 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.579 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.579 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.579 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.826068+0000", 2023-10-01T09:37:42.579 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.29276776999999998 2023-10-01T09:37:42.579 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.579 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.579 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.580 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.580 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.580 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147931 9.3 9:cc030d19:::benchmark_data_smithi138_497829_object45546:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:37:42.580 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.304355+0000", 2023-10-01T09:37:42.580 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 20.240042902999999, 2023-10-01T09:37:42.580 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3960295309999999, 2023-10-01T09:37:42.580 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.580 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.580 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.580 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.580 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.580 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147931 2023-10-01T09:37:42.581 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.581 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.581 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.581 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.581 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304355+0000", 2023-10-01T09:37:42.581 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.581 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.581 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.581 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.581 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304355+0000", 2023-10-01T09:37:42.581 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:42.581 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.582 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.582 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.582 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304354+0000", 2023-10-01T09:37:42.582 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1670000000000004e-06 2023-10-01T09:37:42.582 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.582 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.582 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.582 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304360+0000", 2023-10-01T09:37:42.582 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.27e-05 2023-10-01T09:37:42.582 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.582 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.583 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.583 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304372+0000", 2023-10-01T09:37:42.583 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.815e-06 2023-10-01T09:37:42.583 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.583 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.583 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.583 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304379+0000", 2023-10-01T09:37:42.583 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000106932 2023-10-01T09:37:42.583 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.583 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.583 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.583 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304486+0000", 2023-10-01T09:37:42.584 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3089999999999996e-06 2023-10-01T09:37:42.584 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.584 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.584 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.584 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304495+0000", 2023-10-01T09:37:42.584 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.322764053 2023-10-01T09:37:42.584 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.584 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.584 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.584 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.627259+0000", 2023-10-01T09:37:42.584 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4950000000000008e-06 2023-10-01T09:37:42.584 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.584 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.585 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.585 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.627267+0000", 2023-10-01T09:37:42.585 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.028340244 2023-10-01T09:37:42.585 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.585 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.585 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.585 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.655607+0000", 2023-10-01T09:37:42.585 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000144229 2023-10-01T09:37:42.585 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.585 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.585 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.586 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.655752+0000", 2023-10-01T09:37:42.586 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00018547299999999999 2023-10-01T09:37:42.586 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.586 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.586 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:37:42.586 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.655937+0000", 2023-10-01T09:37:42.586 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021581132999999999 2023-10-01T09:37:42.586 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.586 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.586 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.586 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677518+0000", 2023-10-01T09:37:42.586 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.022700635 2023-10-01T09:37:42.587 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.587 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.587 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.587 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700219+0000", 2023-10-01T09:37:42.587 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2954999999999998e-05 2023-10-01T09:37:42.587 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.587 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.587 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.587 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700262+0000", 2023-10-01T09:37:42.587 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4472000000000002e-05 2023-10-01T09:37:42.587 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.587 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.588 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.588 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700326+0000", 2023-10-01T09:37:42.588 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8291000000000001e-05 2023-10-01T09:37:42.588 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.588 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.588 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.588 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700385+0000", 2023-10-01T09:37:42.588 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3960295309999999 2023-10-01T09:37:42.588 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.588 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.588 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.588 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.589 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.589 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147969 9.c 9:344c11ae:::benchmark_data_smithi138_497829_object45584:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:37:42.589 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.490163+0000", 2023-10-01T09:37:42.589 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 20.054235090999999, 2023-10-01T09:37:42.589 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.18335328, 2023-10-01T09:37:42.589 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.589 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.589 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.589 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.589 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.589 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147969 2023-10-01T09:37:42.589 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.590 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.590 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.590 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.590 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490163+0000", 2023-10-01T09:37:42.590 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.590 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.590 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.590 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.590 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490163+0000", 2023-10-01T09:37:42.590 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:42.590 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.590 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.591 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.591 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490162+0000", 2023-10-01T09:37:42.591 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7320000000000003e-06 2023-10-01T09:37:42.591 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.591 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.591 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.591 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490167+0000", 2023-10-01T09:37:42.591 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1588e-05 2023-10-01T09:37:42.591 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.591 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.591 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.591 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490179+0000", 2023-10-01T09:37:42.592 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1599999999999997e-06 2023-10-01T09:37:42.592 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.592 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.592 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.592 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490184+0000", 2023-10-01T09:37:42.592 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7620999999999998e-05 2023-10-01T09:37:42.592 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.592 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.592 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.592 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490222+0000", 2023-10-01T09:37:42.592 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5396e-05 2023-10-01T09:37:42.592 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.593 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.593 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.593 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490237+0000", 2023-10-01T09:37:42.593 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14082081399999999 2023-10-01T09:37:42.593 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.593 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.593 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.593 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.631058+0000", 2023-10-01T09:37:42.593 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.152e-06 2023-10-01T09:37:42.593 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.593 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.593 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.593 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.631063+0000", 2023-10-01T09:37:42.594 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0296697379999999 2023-10-01T09:37:42.594 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.594 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.594 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.594 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660733+0000", 2023-10-01T09:37:42.594 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011841399999999999 2023-10-01T09:37:42.594 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.594 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.594 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.594 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660851+0000", 2023-10-01T09:37:42.594 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013647799999999999 2023-10-01T09:37:42.594 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.595 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.595 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:37:42.595 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660988+0000", 2023-10-01T09:37:42.595 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045588740000000001 2023-10-01T09:37:42.595 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.595 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.595 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.595 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665546+0000", 2023-10-01T09:37:42.595 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078629470000000003 2023-10-01T09:37:42.595 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.595 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.595 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.596 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673409+0000", 2023-10-01T09:37:42.596 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4159999999999997e-05 2023-10-01T09:37:42.596 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.596 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.596 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.596 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673454+0000", 2023-10-01T09:37:42.596 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2487000000000003e-05 2023-10-01T09:37:42.596 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.596 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.596 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.596 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673486+0000", 2023-10-01T09:37:42.596 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0018e-05 2023-10-01T09:37:42.597 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.597 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.597 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.597 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673516+0000", 2023-10-01T09:37:42.597 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.18335328 2023-10-01T09:37:42.597 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.597 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.597 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.597 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.597 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.597 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147981 9.3 9:c3aeeda4:::benchmark_data_smithi138_497829_object45596:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:37:42.597 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.563343+0000", 2023-10-01T09:37:42.597 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 19.981054879999999, 2023-10-01T09:37:42.598 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1372912850000001, 2023-10-01T09:37:42.598 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.598 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.598 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.598 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.598 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.598 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147981 2023-10-01T09:37:42.598 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.598 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.598 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.598 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.599 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563343+0000", 2023-10-01T09:37:42.599 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.599 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.599 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.599 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.599 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563343+0000", 2023-10-01T09:37:42.599 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999933 2023-10-01T09:37:42.599 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.599 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.599 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.599 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563336+0000", 2023-10-01T09:37:42.599 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1314e-05 2023-10-01T09:37:42.600 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.600 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.600 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.600 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563347+0000", 2023-10-01T09:37:42.600 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0358e-05 2023-10-01T09:37:42.600 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.600 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.600 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.600 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563358+0000", 2023-10-01T09:37:42.600 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3170000000000003e-06 2023-10-01T09:37:42.600 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.600 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.601 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.601 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563362+0000", 2023-10-01T09:37:42.601 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1795999999999997e-05 2023-10-01T09:37:42.601 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.601 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.601 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.601 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563414+0000", 2023-10-01T09:37:42.601 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1110999999999999e-05 2023-10-01T09:37:42.601 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.601 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.601 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.601 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563425+0000", 2023-10-01T09:37:42.602 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.063862261000000003 2023-10-01T09:37:42.602 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.602 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.602 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.602 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.627287+0000", 2023-10-01T09:37:42.602 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4970000000000001e-06 2023-10-01T09:37:42.602 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.602 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.602 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.602 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.627293+0000", 2023-10-01T09:37:42.602 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0332602529999999 2023-10-01T09:37:42.603 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.603 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.603 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.603 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660553+0000", 2023-10-01T09:37:42.603 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3721999999999996e-05 2023-10-01T09:37:42.603 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.603 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.603 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.603 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660637+0000", 2023-10-01T09:37:42.603 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012805999999999999 2023-10-01T09:37:42.603 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.603 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.604 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:37:42.604 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660765+0000", 2023-10-01T09:37:42.604 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.025676641 2023-10-01T09:37:42.604 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.604 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.604 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.604 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686441+0000", 2023-10-01T09:37:42.604 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013987271000000001 2023-10-01T09:37:42.604 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.604 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.604 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.604 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700429+0000", 2023-10-01T09:37:42.604 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000107107 2023-10-01T09:37:42.605 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.605 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.605 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.605 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700536+0000", 2023-10-01T09:37:42.605 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0742000000000002e-05 2023-10-01T09:37:42.605 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.605 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.605 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.605 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700597+0000", 2023-10-01T09:37:42.605 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7715000000000001e-05 2023-10-01T09:37:42.605 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.605 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.606 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.606 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700634+0000", 2023-10-01T09:37:42.606 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1372912850000001 2023-10-01T09:37:42.606 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.606 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.606 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.606 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.606 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.606 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147984 9.3 9:c9142d5f:::benchmark_data_smithi138_497829_object45599:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:37:42.606 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.707974+0000", 2023-10-01T09:37:42.606 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 19.836424375, 2023-10-01T09:37:42.607 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99286916199999997, 2023-10-01T09:37:42.607 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.607 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.607 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.607 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.607 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.607 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147984 2023-10-01T09:37:42.607 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.607 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.607 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.607 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.608 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.707974+0000", 2023-10-01T09:37:42.608 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.608 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.608 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.608 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.608 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.707974+0000", 2023-10-01T09:37:42.608 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:37:42.608 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.608 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.608 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.608 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.707971+0000", 2023-10-01T09:37:42.608 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1004e-05 2023-10-01T09:37:42.609 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.609 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.609 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.609 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.707982+0000", 2023-10-01T09:37:42.609 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8074e-05 2023-10-01T09:37:42.609 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.609 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.609 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.609 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.708000+0000", 2023-10-01T09:37:42.609 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.503e-06 2023-10-01T09:37:42.609 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.610 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.610 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.610 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.708008+0000", 2023-10-01T09:37:42.610 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0042882299999999996 2023-10-01T09:37:42.610 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.610 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.610 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.610 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712296+0000", 2023-10-01T09:37:42.610 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0655e-05 2023-10-01T09:37:42.610 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.610 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.610 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.611 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712307+0000", 2023-10-01T09:37:42.611 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.95292995599999997 2023-10-01T09:37:42.611 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.611 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.611 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.611 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665237+0000", 2023-10-01T09:37:42.611 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011695400000000001 2023-10-01T09:37:42.611 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.611 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.611 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.611 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665354+0000", 2023-10-01T09:37:42.612 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000102116 2023-10-01T09:37:42.612 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.612 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.612 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:37:42.612 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665456+0000", 2023-10-01T09:37:42.612 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020992165 2023-10-01T09:37:42.612 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.612 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.612 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.612 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686448+0000", 2023-10-01T09:37:42.612 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014224582 2023-10-01T09:37:42.612 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.613 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.613 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.613 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700672+0000", 2023-10-01T09:37:42.613 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2769000000000006e-05 2023-10-01T09:37:42.613 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.613 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.613 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.613 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700765+0000", 2023-10-01T09:37:42.613 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4345000000000003e-05 2023-10-01T09:37:42.613 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.613 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.613 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.614 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700810+0000", 2023-10-01T09:37:42.614 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3080000000000002e-05 2023-10-01T09:37:42.614 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.614 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.614 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.614 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700843+0000", 2023-10-01T09:37:42.614 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99286916199999997 2023-10-01T09:37:42.614 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.614 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.614 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.614 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.615 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.615 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147985 9.3 9:c26d6b78:::benchmark_data_smithi138_497829_object45600:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:37:42.615 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.712409+0000", 2023-10-01T09:37:42.615 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 19.831989128, 2023-10-01T09:37:42.615 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98851199499999998, 2023-10-01T09:37:42.615 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.615 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.615 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.615 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.615 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.615 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147985 2023-10-01T09:37:42.616 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.616 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.616 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.616 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.616 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712409+0000", 2023-10-01T09:37:42.616 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.616 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.616 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.616 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.616 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712409+0000", 2023-10-01T09:37:42.616 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:42.616 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.617 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.617 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.617 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712407+0000", 2023-10-01T09:37:42.617 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8050000000000007e-06 2023-10-01T09:37:42.617 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.617 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.617 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.617 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712415+0000", 2023-10-01T09:37:42.617 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6009999999999995e-06 2023-10-01T09:37:42.617 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.617 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.618 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.618 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712424+0000", 2023-10-01T09:37:42.618 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8139999999999997e-06 2023-10-01T09:37:42.618 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.618 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.618 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.618 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712429+0000", 2023-10-01T09:37:42.618 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4980999999999993e-05 2023-10-01T09:37:42.618 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.618 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.618 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.618 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712494+0000", 2023-10-01T09:37:42.619 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5420000000000002e-06 2023-10-01T09:37:42.619 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.619 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.619 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.619 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712501+0000", 2023-10-01T09:37:42.619 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.95723478900000003 2023-10-01T09:37:42.619 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.619 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.619 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.619 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.669736+0000", 2023-10-01T09:37:42.619 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000116869 2023-10-01T09:37:42.619 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.620 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.620 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.620 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.669853+0000", 2023-10-01T09:37:42.620 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016995500000000001 2023-10-01T09:37:42.620 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.620 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.620 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:37:42.620 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.670023+0000", 2023-10-01T09:37:42.620 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016431211000000001 2023-10-01T09:37:42.620 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.620 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.620 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.621 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686454+0000", 2023-10-01T09:37:42.621 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014249539 2023-10-01T09:37:42.621 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.621 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.621 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.621 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700703+0000", 2023-10-01T09:37:42.621 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000168577 2023-10-01T09:37:42.621 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.621 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.621 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.621 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700872+0000", 2023-10-01T09:37:42.621 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.245e-05 2023-10-01T09:37:42.622 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.622 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.622 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.622 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700894+0000", 2023-10-01T09:37:42.622 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.639e-05 2023-10-01T09:37:42.622 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.622 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.622 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.622 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700921+0000", 2023-10-01T09:37:42.622 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98851199499999998 2023-10-01T09:37:42.622 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.623 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.623 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.623 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.623 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.623 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148000 9.c 9:3eba568d:::benchmark_data_smithi138_497829_object45615:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:37:42.623 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.786332+0000", 2023-10-01T09:37:42.623 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 19.758065412000001, 2023-10-01T09:37:42.623 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.89635619300000002, 2023-10-01T09:37:42.623 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.623 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.623 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.623 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.624 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.624 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148000 2023-10-01T09:37:42.624 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.624 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.624 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.624 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.624 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786332+0000", 2023-10-01T09:37:42.624 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.624 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.624 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.624 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.625 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786332+0000", 2023-10-01T09:37:42.625 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:37:42.625 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.625 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.625 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.625 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786330+0000", 2023-10-01T09:37:42.625 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6572e-05 2023-10-01T09:37:42.625 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.625 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.625 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.625 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786347+0000", 2023-10-01T09:37:42.625 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7059e-05 2023-10-01T09:37:42.626 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.626 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.626 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.626 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786364+0000", 2023-10-01T09:37:42.626 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.357e-06 2023-10-01T09:37:42.626 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.626 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.626 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.626 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786371+0000", 2023-10-01T09:37:42.626 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7531999999999998e-05 2023-10-01T09:37:42.626 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.627 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.627 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.627 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786419+0000", 2023-10-01T09:37:42.627 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0134e-05 2023-10-01T09:37:42.627 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.627 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.627 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.627 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786429+0000", 2023-10-01T09:37:42.627 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.879158459 2023-10-01T09:37:42.627 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.627 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.627 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.628 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665587+0000", 2023-10-01T09:37:42.628 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000112847 2023-10-01T09:37:42.628 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.628 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.628 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.628 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665700+0000", 2023-10-01T09:37:42.628 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011616399999999999 2023-10-01T09:37:42.628 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.628 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.628 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:37:42.628 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665816+0000", 2023-10-01T09:37:42.628 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016651933000000001 2023-10-01T09:37:42.629 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.629 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.629 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.629 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682468+0000", 2023-10-01T09:37:42.629 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3857000000000006e-05 2023-10-01T09:37:42.629 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.629 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.629 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.629 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682562+0000", 2023-10-01T09:37:42.629 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4292999999999998e-05 2023-10-01T09:37:42.629 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.629 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.630 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.630 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682596+0000", 2023-10-01T09:37:42.630 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7022999999999999e-05 2023-10-01T09:37:42.630 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.630 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.630 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.630 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682653+0000", 2023-10-01T09:37:42.630 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5318999999999998e-05 2023-10-01T09:37:42.630 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.630 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.630 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.631 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682689+0000", 2023-10-01T09:37:42.631 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.89635619300000002 2023-10-01T09:37:42.631 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.631 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.631 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.631 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.631 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.631 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148010 9.c 9:365ddc65:::benchmark_data_smithi138_497829_object45625:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:37:42.631 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.803374+0000", 2023-10-01T09:37:42.631 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 19.741024395, 2023-10-01T09:37:42.631 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.87947385199999994, 2023-10-01T09:37:42.632 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.632 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.632 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.632 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.632 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.632 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148010 2023-10-01T09:37:42.632 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.632 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.632 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.632 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.632 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803374+0000", 2023-10-01T09:37:42.632 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.633 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.633 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.633 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.633 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803374+0000", 2023-10-01T09:37:42.633 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:42.633 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.633 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.633 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.633 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803372+0000", 2023-10-01T09:37:42.633 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6549999999999999e-06 2023-10-01T09:37:42.635 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.635 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.635 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.635 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803381+0000", 2023-10-01T09:37:42.635 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7244999999999998e-05 2023-10-01T09:37:42.635 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.635 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.635 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.635 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803398+0000", 2023-10-01T09:37:42.635 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6320000000000008e-06 2023-10-01T09:37:42.636 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.636 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.636 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.636 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803406+0000", 2023-10-01T09:37:42.636 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9969000000000002e-05 2023-10-01T09:37:42.636 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.636 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.636 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.636 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803456+0000", 2023-10-01T09:37:42.636 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4809999999999995e-06 2023-10-01T09:37:42.637 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.637 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.637 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.637 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803466+0000", 2023-10-01T09:37:42.637 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.86655919599999998 2023-10-01T09:37:42.637 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.637 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.637 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.637 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.670025+0000", 2023-10-01T09:37:42.637 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9349000000000003e-05 2023-10-01T09:37:42.637 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.638 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.638 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.638 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.670094+0000", 2023-10-01T09:37:42.638 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7009000000000007e-05 2023-10-01T09:37:42.638 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.638 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.638 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:37:42.638 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.670181+0000", 2023-10-01T09:37:42.638 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012516879999999999 2023-10-01T09:37:42.638 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.638 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.638 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.639 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682698+0000", 2023-10-01T09:37:42.639 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3449000000000003e-05 2023-10-01T09:37:42.639 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.639 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.639 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.639 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682751+0000", 2023-10-01T09:37:42.639 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4245999999999997e-05 2023-10-01T09:37:42.639 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.639 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.639 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.639 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682786+0000", 2023-10-01T09:37:42.639 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9830999999999999e-05 2023-10-01T09:37:42.640 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.640 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.640 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.640 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682816+0000", 2023-10-01T09:37:42.640 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1853999999999997e-05 2023-10-01T09:37:42.640 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.640 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.640 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.640 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682847+0000", 2023-10-01T09:37:42.640 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.87947385199999994 2023-10-01T09:37:42.640 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.640 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.641 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.641 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.641 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.641 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148037 9.3 9:c1ff9704:::benchmark_data_smithi138_497829_object45652:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:37:42.641 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.881445+0000", 2023-10-01T09:37:42.641 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 19.662953335000001, 2023-10-01T09:37:42.641 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81953648800000001, 2023-10-01T09:37:42.641 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.641 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.641 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.641 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.641 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.642 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148037 2023-10-01T09:37:42.642 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.642 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.642 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.642 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.642 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881445+0000", 2023-10-01T09:37:42.642 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.642 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.642 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.642 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.642 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881445+0000", 2023-10-01T09:37:42.642 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:42.643 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.643 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.643 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.643 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881443+0000", 2023-10-01T09:37:42.643 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2189999999999993e-06 2023-10-01T09:37:42.643 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.643 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.643 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.643 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881452+0000", 2023-10-01T09:37:42.643 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2901000000000001e-05 2023-10-01T09:37:42.643 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.643 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.644 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.644 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881465+0000", 2023-10-01T09:37:42.644 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1500000000000004e-06 2023-10-01T09:37:42.644 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.644 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.644 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.644 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881471+0000", 2023-10-01T09:37:42.644 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9982000000000003e-05 2023-10-01T09:37:42.644 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.644 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.644 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.644 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881531+0000", 2023-10-01T09:37:42.645 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1070000000000001e-05 2023-10-01T09:37:42.645 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.645 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.645 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.645 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881542+0000", 2023-10-01T09:37:42.645 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.79148766599999998 2023-10-01T09:37:42.645 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.645 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.645 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.645 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673030+0000", 2023-10-01T09:37:42.645 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000120554 2023-10-01T09:37:42.645 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.646 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.646 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.646 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673151+0000", 2023-10-01T09:37:42.646 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000136756 2023-10-01T09:37:42.646 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.646 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.646 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:37:42.646 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673287+0000", 2023-10-01T09:37:42.646 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017487816999999999 2023-10-01T09:37:42.646 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.646 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.646 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.647 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.690775+0000", 2023-10-01T09:37:42.647 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0099592180000000006 2023-10-01T09:37:42.647 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.647 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.647 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.647 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700734+0000", 2023-10-01T09:37:42.647 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00020710800000000001 2023-10-01T09:37:42.647 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.647 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.647 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.647 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700942+0000", 2023-10-01T09:37:42.647 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0435999999999999e-05 2023-10-01T09:37:42.648 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.648 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.648 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.648 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700962+0000", 2023-10-01T09:37:42.648 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9011000000000001e-05 2023-10-01T09:37:42.648 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.648 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.648 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.648 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700981+0000", 2023-10-01T09:37:42.648 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81953648800000001 2023-10-01T09:37:42.648 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.648 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.649 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.649 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.649 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.649 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148041 9.3 9:c062535c:::benchmark_data_smithi138_497829_object45656:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:37:42.649 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.886259+0000", 2023-10-01T09:37:42.649 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 19.658139207000001, 2023-10-01T09:37:42.649 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81483497999999999, 2023-10-01T09:37:42.649 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.649 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.649 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.649 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.649 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.650 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148041 2023-10-01T09:37:42.650 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.650 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.650 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.650 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.650 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886259+0000", 2023-10-01T09:37:42.650 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.650 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.650 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.650 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.650 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886259+0000", 2023-10-01T09:37:42.651 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:42.651 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.651 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.651 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.651 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886257+0000", 2023-10-01T09:37:42.651 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1755000000000001e-05 2023-10-01T09:37:42.651 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.651 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.651 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.651 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886278+0000", 2023-10-01T09:37:42.651 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8885e-05 2023-10-01T09:37:42.651 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.652 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.652 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.652 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886297+0000", 2023-10-01T09:37:42.652 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7020000000000001e-06 2023-10-01T09:37:42.652 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.652 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.652 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.652 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886305+0000", 2023-10-01T09:37:42.652 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6515999999999999e-05 2023-10-01T09:37:42.652 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.652 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.652 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.652 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886361+0000", 2023-10-01T09:37:42.653 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2311e-05 2023-10-01T09:37:42.653 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.653 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.653 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.653 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886374+0000", 2023-10-01T09:37:42.653 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.79119420699999998 2023-10-01T09:37:42.653 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.653 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.653 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.653 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677568+0000", 2023-10-01T09:37:42.653 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3848e-05 2023-10-01T09:37:42.654 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.654 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.654 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.654 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677662+0000", 2023-10-01T09:37:42.654 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7428000000000003e-05 2023-10-01T09:37:42.654 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.654 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.654 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:37:42.654 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677759+0000", 2023-10-01T09:37:42.654 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013022529 2023-10-01T09:37:42.654 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.655 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.655 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.655 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.690782+0000", 2023-10-01T09:37:42.655 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010232722 2023-10-01T09:37:42.655 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.655 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.655 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.655 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701015+0000", 2023-10-01T09:37:42.655 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1299e-05 2023-10-01T09:37:42.655 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.655 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.655 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.656 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701046+0000", 2023-10-01T09:37:42.656 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7508000000000002e-05 2023-10-01T09:37:42.656 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.656 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.656 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.656 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701073+0000", 2023-10-01T09:37:42.656 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0361e-05 2023-10-01T09:37:42.656 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.656 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.656 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.656 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701094+0000", 2023-10-01T09:37:42.656 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81483497999999999 2023-10-01T09:37:42.657 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.657 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.657 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.657 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.657 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.657 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148060 9.3 9:cf7f7f25:::benchmark_data_smithi138_497829_object45675:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:37:42.657 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.972375+0000", 2023-10-01T09:37:42.657 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 19.572022517000001, 2023-10-01T09:37:42.657 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.72881600400000002, 2023-10-01T09:37:42.657 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.657 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.657 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.658 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.658 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.658 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148060 2023-10-01T09:37:42.658 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.658 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.658 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.658 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.658 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972375+0000", 2023-10-01T09:37:42.658 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.658 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.658 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.658 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.659 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972375+0000", 2023-10-01T09:37:42.659 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:42.659 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.659 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.659 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.659 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972374+0000", 2023-10-01T09:37:42.659 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1189000000000001e-05 2023-10-01T09:37:42.659 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.659 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.659 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.659 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972385+0000", 2023-10-01T09:37:42.660 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7291000000000001e-05 2023-10-01T09:37:42.660 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.660 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.660 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.660 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972402+0000", 2023-10-01T09:37:42.660 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9349999999999997e-06 2023-10-01T09:37:42.660 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.660 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.660 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.660 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972409+0000", 2023-10-01T09:37:42.660 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8267000000000003e-05 2023-10-01T09:37:42.660 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.661 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.661 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.661 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972467+0000", 2023-10-01T09:37:42.661 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1688999999999999e-05 2023-10-01T09:37:42.661 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.661 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.661 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.661 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972479+0000", 2023-10-01T09:37:42.661 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.70958122800000001 2023-10-01T09:37:42.661 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.661 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.661 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.661 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682060+0000", 2023-10-01T09:37:42.662 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3533000000000001e-05 2023-10-01T09:37:42.662 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.662 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.662 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.662 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682144+0000", 2023-10-01T09:37:42.662 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000118854 2023-10-01T09:37:42.662 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.662 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.662 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:37:42.662 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682263+0000", 2023-10-01T09:37:42.662 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012972465000000001 2023-10-01T09:37:42.662 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.663 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.663 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.663 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.695235+0000", 2023-10-01T09:37:42.663 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0058802189999999999 2023-10-01T09:37:42.663 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.663 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.663 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.663 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701115+0000", 2023-10-01T09:37:42.663 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5998999999999999e-05 2023-10-01T09:37:42.663 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.663 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.664 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.664 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701151+0000", 2023-10-01T09:37:42.664 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0852999999999998e-05 2023-10-01T09:37:42.664 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.664 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.664 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.664 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701172+0000", 2023-10-01T09:37:42.664 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9335999999999999e-05 2023-10-01T09:37:42.664 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.664 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.664 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.665 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701191+0000", 2023-10-01T09:37:42.665 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.72881600400000002 2023-10-01T09:37:42.665 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.665 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.665 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.665 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.665 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.665 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148074 9.c 9:3b5a7b44:::benchmark_data_smithi138_497829_object45689:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:37:42.665 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.007110+0000", 2023-10-01T09:37:42.665 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 19.537288191999998, 2023-10-01T09:37:42.665 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.68042775499999997, 2023-10-01T09:37:42.665 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.666 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.666 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.666 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.666 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.666 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148074 2023-10-01T09:37:42.666 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.666 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.666 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.666 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.666 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007110+0000", 2023-10-01T09:37:42.666 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.666 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.667 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.667 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.667 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007110+0000", 2023-10-01T09:37:42.667 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:42.667 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.667 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.667 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.667 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007108+0000", 2023-10-01T09:37:42.667 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4759999999999998e-06 2023-10-01T09:37:42.667 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.667 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.668 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.668 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007116+0000", 2023-10-01T09:37:42.668 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5668999999999999e-05 2023-10-01T09:37:42.668 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.668 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.668 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.668 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007132+0000", 2023-10-01T09:37:42.668 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6610000000000003e-06 2023-10-01T09:37:42.668 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.668 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.668 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.668 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007139+0000", 2023-10-01T09:37:42.669 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1598000000000003e-05 2023-10-01T09:37:42.669 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.669 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.669 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.669 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007190+0000", 2023-10-01T09:37:42.669 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0829999999999998e-06 2023-10-01T09:37:42.669 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.669 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.669 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.669 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007197+0000", 2023-10-01T09:37:42.669 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.66635359100000002 2023-10-01T09:37:42.669 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.669 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.670 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.670 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673551+0000", 2023-10-01T09:37:42.670 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0988000000000004e-05 2023-10-01T09:37:42.670 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.670 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.670 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.670 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673642+0000", 2023-10-01T09:37:42.670 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010931599999999999 2023-10-01T09:37:42.670 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.670 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.670 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:37:42.670 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673751+0000", 2023-10-01T09:37:42.671 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0089520429999999998 2023-10-01T09:37:42.671 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.671 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.671 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.671 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682703+0000", 2023-10-01T09:37:42.671 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00020024499999999999 2023-10-01T09:37:42.671 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.671 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.671 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.671 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682903+0000", 2023-10-01T09:37:42.671 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045564280000000004 2023-10-01T09:37:42.671 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.672 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.672 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.672 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.687460+0000", 2023-10-01T09:37:42.672 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6650999999999999e-05 2023-10-01T09:37:42.672 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.672 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.672 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.672 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.687496+0000", 2023-10-01T09:37:42.672 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0963000000000001e-05 2023-10-01T09:37:42.672 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.672 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.672 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.673 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.687537+0000", 2023-10-01T09:37:42.673 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.68042775499999997 2023-10-01T09:37:42.673 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.673 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.673 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.673 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.673 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.673 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148078 9.3 9:c8b0a7d5:::benchmark_data_smithi138_497829_object45693:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:37:42.673 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.015312+0000", 2023-10-01T09:37:42.673 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 19.529086264, 2023-10-01T09:37:42.673 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.70011192700000002, 2023-10-01T09:37:42.673 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.674 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.674 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.674 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.674 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.674 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148078 2023-10-01T09:37:42.674 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.674 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.674 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.674 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.674 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015312+0000", 2023-10-01T09:37:42.674 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.674 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.675 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.675 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.675 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015312+0000", 2023-10-01T09:37:42.675 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:42.675 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.675 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.675 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.675 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015310+0000", 2023-10-01T09:37:42.675 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7510000000000008e-06 2023-10-01T09:37:42.675 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.675 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.675 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.676 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015317+0000", 2023-10-01T09:37:42.676 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2976000000000001e-05 2023-10-01T09:37:42.676 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.676 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.676 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.676 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015330+0000", 2023-10-01T09:37:42.676 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5339999999999999e-06 2023-10-01T09:37:42.676 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.676 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.676 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.676 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015336+0000", 2023-10-01T09:37:42.676 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5003999999999997e-05 2023-10-01T09:37:42.677 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.677 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.677 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.677 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015391+0000", 2023-10-01T09:37:42.677 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5939999999999999e-06 2023-10-01T09:37:42.677 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.677 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.677 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.677 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015398+0000", 2023-10-01T09:37:42.677 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.67108258300000001 2023-10-01T09:37:42.677 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.677 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.677 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.678 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686480+0000", 2023-10-01T09:37:42.678 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000109298 2023-10-01T09:37:42.678 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.678 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.678 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.678 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686589+0000", 2023-10-01T09:37:42.678 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013103600000000001 2023-10-01T09:37:42.678 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.678 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.678 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:37:42.678 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686720+0000", 2023-10-01T09:37:42.678 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013092273999999999 2023-10-01T09:37:42.679 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.679 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.679 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.679 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.699813+0000", 2023-10-01T09:37:42.679 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001319221 2023-10-01T09:37:42.679 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.679 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.679 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.679 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701132+0000", 2023-10-01T09:37:42.679 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014181050000000001 2023-10-01T09:37:42.679 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.679 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.680 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.680 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715313+0000", 2023-10-01T09:37:42.680 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0659000000000002e-05 2023-10-01T09:37:42.680 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.680 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.680 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.680 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715384+0000", 2023-10-01T09:37:42.680 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9857999999999997e-05 2023-10-01T09:37:42.680 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.680 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.680 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.680 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715424+0000", 2023-10-01T09:37:42.681 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.70011192700000002 2023-10-01T09:37:42.681 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.681 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.681 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.681 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.681 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.681 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148091 9.3 9:c92a3cf4:::benchmark_data_smithi138_497829_object45706:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:37:42.681 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.098648+0000", 2023-10-01T09:37:42.681 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 19.445750313000001, 2023-10-01T09:37:42.681 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.61693706400000003, 2023-10-01T09:37:42.681 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.681 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.682 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.682 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.682 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.682 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148091 2023-10-01T09:37:42.682 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.682 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.682 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.682 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.682 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098648+0000", 2023-10-01T09:37:42.682 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.682 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.682 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.683 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.683 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098648+0000", 2023-10-01T09:37:42.683 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:37:42.683 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.683 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.683 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.683 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098645+0000", 2023-10-01T09:37:42.683 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3414000000000001e-05 2023-10-01T09:37:42.683 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.683 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.683 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.683 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098658+0000", 2023-10-01T09:37:42.684 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5611e-05 2023-10-01T09:37:42.684 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.684 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.684 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.684 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098674+0000", 2023-10-01T09:37:42.684 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0550000000000002e-06 2023-10-01T09:37:42.684 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.684 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.684 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.684 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098681+0000", 2023-10-01T09:37:42.684 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8112999999999998e-05 2023-10-01T09:37:42.684 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.684 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.685 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.685 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098729+0000", 2023-10-01T09:37:42.685 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7700000000000001e-06 2023-10-01T09:37:42.685 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.685 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.685 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.685 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098737+0000", 2023-10-01T09:37:42.685 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.59207254099999995 2023-10-01T09:37:42.685 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.685 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.685 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.685 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.690810+0000", 2023-10-01T09:37:42.686 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011513 2023-10-01T09:37:42.686 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.686 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.686 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.686 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.690925+0000", 2023-10-01T09:37:42.686 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013793200000000001 2023-10-01T09:37:42.686 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.686 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.686 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:37:42.686 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.691063+0000", 2023-10-01T09:37:42.686 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0087986049999999993 2023-10-01T09:37:42.687 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.687 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.687 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.687 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.699861+0000", 2023-10-01T09:37:42.687 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.01539976 2023-10-01T09:37:42.687 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.687 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.687 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.687 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715261+0000", 2023-10-01T09:37:42.687 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00022492599999999999 2023-10-01T09:37:42.687 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.687 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.688 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.688 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715486+0000", 2023-10-01T09:37:42.688 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1239000000000002e-05 2023-10-01T09:37:42.688 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.688 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.688 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.688 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715537+0000", 2023-10-01T09:37:42.688 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7519999999999999e-05 2023-10-01T09:37:42.688 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.688 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.688 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.688 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715585+0000", 2023-10-01T09:37:42.689 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.61693706400000003 2023-10-01T09:37:42.689 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.689 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.689 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.689 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.689 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.689 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148104 9.c 9:3af9d94a:::benchmark_data_smithi138_497829_object45719:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:37:42.689 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.131588+0000", 2023-10-01T09:37:42.689 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 19.412810176000001, 2023-10-01T09:37:42.689 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55710424700000005, 2023-10-01T09:37:42.689 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.689 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.690 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.690 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.690 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.690 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148104 2023-10-01T09:37:42.690 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.690 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.690 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.690 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.690 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131588+0000", 2023-10-01T09:37:42.690 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.690 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.691 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.691 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.691 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131588+0000", 2023-10-01T09:37:42.691 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:42.691 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.691 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.691 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.691 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131586+0000", 2023-10-01T09:37:42.691 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3512e-05 2023-10-01T09:37:42.691 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.691 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.691 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.691 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131599+0000", 2023-10-01T09:37:42.692 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7265000000000002e-05 2023-10-01T09:37:42.692 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.692 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.692 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.692 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131616+0000", 2023-10-01T09:37:42.692 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5329999999999997e-06 2023-10-01T09:37:42.692 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.692 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.692 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.692 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131624+0000", 2023-10-01T09:37:42.692 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5498999999999999e-05 2023-10-01T09:37:42.692 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.693 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.693 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.693 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131670+0000", 2023-10-01T09:37:42.693 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1920000000000005e-06 2023-10-01T09:37:42.693 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.693 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.693 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.693 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131678+0000", 2023-10-01T09:37:42.693 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.54630871999999997 2023-10-01T09:37:42.693 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.693 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.693 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.694 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677986+0000", 2023-10-01T09:37:42.694 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000109057 2023-10-01T09:37:42.694 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.694 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.694 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.694 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.678095+0000", 2023-10-01T09:37:42.694 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013558 2023-10-01T09:37:42.694 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.694 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.694 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:37:42.694 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.678231+0000", 2023-10-01T09:37:42.694 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0046248979999999997 2023-10-01T09:37:42.695 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.695 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.695 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.695 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682856+0000", 2023-10-01T09:37:42.695 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7060999999999999e-05 2023-10-01T09:37:42.695 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.695 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.695 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.695 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682943+0000", 2023-10-01T09:37:42.695 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.005695922 2023-10-01T09:37:42.695 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.695 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.696 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.696 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688639+0000", 2023-10-01T09:37:42.696 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9950999999999998e-05 2023-10-01T09:37:42.696 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.696 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.696 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.696 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688669+0000", 2023-10-01T09:37:42.696 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3079e-05 2023-10-01T09:37:42.696 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.696 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.696 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.696 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688692+0000", 2023-10-01T09:37:42.697 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55710424700000005 2023-10-01T09:37:42.697 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.697 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.697 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.697 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.697 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.697 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148105 9.c 9:3f26355b:::benchmark_data_smithi138_497829_object45720:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:37:42.697 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.132784+0000", 2023-10-01T09:37:42.697 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 19.411613490000001, 2023-10-01T09:37:42.697 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55601706699999998, 2023-10-01T09:37:42.697 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.697 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.698 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.698 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.698 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.698 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148105 2023-10-01T09:37:42.698 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.698 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.698 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.698 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.698 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132784+0000", 2023-10-01T09:37:42.698 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.698 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.699 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.699 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.699 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132784+0000", 2023-10-01T09:37:42.699 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:37:42.699 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.699 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.699 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.699 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132784+0000", 2023-10-01T09:37:42.699 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2379999999999997e-06 2023-10-01T09:37:42.699 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.699 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.699 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.700 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132792+0000", 2023-10-01T09:37:42.700 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0162e-05 2023-10-01T09:37:42.700 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.700 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.700 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.700 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132802+0000", 2023-10-01T09:37:42.700 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5860000000000004e-06 2023-10-01T09:37:42.700 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.700 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.700 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.700 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132808+0000", 2023-10-01T09:37:42.700 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5326e-05 2023-10-01T09:37:42.701 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.701 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.701 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.701 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132843+0000", 2023-10-01T09:37:42.701 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6760000000000004e-06 2023-10-01T09:37:42.701 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.701 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.701 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.701 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132849+0000", 2023-10-01T09:37:42.701 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55011066399999997 2023-10-01T09:37:42.701 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.701 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.702 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.702 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682959+0000", 2023-10-01T09:37:42.702 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9478000000000001e-05 2023-10-01T09:37:42.702 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.702 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.702 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.702 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.683029+0000", 2023-10-01T09:37:42.702 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2601999999999994e-05 2023-10-01T09:37:42.702 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.702 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.702 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:37:42.703 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.683101+0000", 2023-10-01T09:37:42.703 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0044813810000000004 2023-10-01T09:37:42.703 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.703 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.703 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.703 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.687583+0000", 2023-10-01T09:37:42.703 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000419534 2023-10-01T09:37:42.703 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.703 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.703 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.703 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688002+0000", 2023-10-01T09:37:42.703 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00075196499999999995 2023-10-01T09:37:42.703 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.704 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.704 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.704 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688754+0000", 2023-10-01T09:37:42.704 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6500999999999999e-05 2023-10-01T09:37:42.704 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.704 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.704 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.704 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688781+0000", 2023-10-01T09:37:42.704 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0809999999999999e-05 2023-10-01T09:37:42.704 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.704 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.704 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.705 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688801+0000", 2023-10-01T09:37:42.705 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55601706699999998 2023-10-01T09:37:42.705 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.705 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.705 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.705 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.705 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.705 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148110 9.3 9:caeedb58:::benchmark_data_smithi138_497829_object45725:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:37:42.705 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.149195+0000", 2023-10-01T09:37:42.705 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 19.395202992000002, 2023-10-01T09:37:42.705 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.57476550699999995, 2023-10-01T09:37:42.705 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:42.706 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:42.706 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:42.706 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:37:42.706 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:37:42.706 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148110 2023-10-01T09:37:42.706 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.706 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:42.706 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.706 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:42.706 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149195+0000", 2023-10-01T09:37:42.706 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:42.706 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.707 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.707 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:42.707 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149195+0000", 2023-10-01T09:37:42.707 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:42.707 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.707 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.707 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:42.707 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149193+0000", 2023-10-01T09:37:42.707 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1443999999999999e-05 2023-10-01T09:37:42.707 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.707 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.707 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:42.708 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149205+0000", 2023-10-01T09:37:42.708 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0498e-05 2023-10-01T09:37:42.708 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.708 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.708 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:42.708 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149215+0000", 2023-10-01T09:37:42.708 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2490000000000001e-06 2023-10-01T09:37:42.708 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.708 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.708 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:42.708 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149220+0000", 2023-10-01T09:37:42.708 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5479999999999998e-05 2023-10-01T09:37:42.708 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.709 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.709 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.709 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149266+0000", 2023-10-01T09:37:42.709 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.516e-06 2023-10-01T09:37:42.709 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.709 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.709 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:42.709 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149272+0000", 2023-10-01T09:37:42.709 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.54601357399999995 2023-10-01T09:37:42.709 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.709 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.709 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:42.710 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.695286+0000", 2023-10-01T09:37:42.710 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000100777 2023-10-01T09:37:42.710 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.710 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.710 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:42.710 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.695387+0000", 2023-10-01T09:37:42.710 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5045999999999999e-05 2023-10-01T09:37:42.710 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.710 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.710 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:37:42.710 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.695472+0000", 2023-10-01T09:37:42.710 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0047012119999999998 2023-10-01T09:37:42.711 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.711 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.711 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:42.711 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700173+0000", 2023-10-01T09:37:42.711 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015477148 2023-10-01T09:37:42.711 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.711 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.711 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.711 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715650+0000", 2023-10-01T09:37:42.711 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0082495999999999993 2023-10-01T09:37:42.711 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.711 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.712 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:42.712 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.723900+0000", 2023-10-01T09:37:42.712 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4461000000000001e-05 2023-10-01T09:37:42.712 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.712 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.712 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:42.712 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.723934+0000", 2023-10-01T09:37:42.712 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6350999999999999e-05 2023-10-01T09:37:42.712 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:42.712 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:42.712 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:42.712 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.723960+0000", 2023-10-01T09:37:42.713 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.57476550699999995 2023-10-01T09:37:42.713 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.713 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.713 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.713 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:42.713 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:42.713 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:37:42.713 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:42.869 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:43.154 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T09:37:43.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:43 smithi138 ceph-mon[185886]: pgmap v1260: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 21885/150699 objects degraded (14.522%); 23927/150699 objects misplaced (15.877%); 6.1 MiB/s, 96 objects/s recovering 2023-10-01T09:37:43.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:43 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 21885/150699 objects degraded (14.522%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:43.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:43 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3294201068' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-01T09:37:43.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:43 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-01T09:37:43.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:43 smithi138 ceph-mon[192513]: pgmap v1260: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 21885/150699 objects degraded (14.522%); 23927/150699 objects misplaced (15.877%); 6.1 MiB/s, 96 objects/s recovering 2023-10-01T09:37:43.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:43 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 21885/150699 objects degraded (14.522%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:43.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:43 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3294201068' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-01T09:37:43.323 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:43 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-01T09:37:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:43 smithi190 ceph-mon[160435]: pgmap v1260: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 21885/150699 objects degraded (14.522%); 23927/150699 objects misplaced (15.877%); 6.1 MiB/s, 96 objects/s recovering 2023-10-01T09:37:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:43 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 21885/150699 objects degraded (14.522%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:43 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3294201068' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-01T09:37:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:43 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-01T09:37:43.635 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:37:44.046 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:37:44.074 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:44.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:44.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:44.172 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:37:44.186 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:37:44.322 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:44.731 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:37:44.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:44.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:37:44.841 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:37:44.934 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:45.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:37:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:37:44] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.33.4" 2023-10-01T09:37:45.169 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T09:37:45.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:45.170 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:37:45.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:37:45.271 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:45.336 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:45.408 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:45 smithi138 ceph-mon[192513]: pgmap v1261: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 21885/150294 objects degraded (14.561%); 23927/150294 objects misplaced (15.920%); 6.1 MiB/s, 96 objects/s recovering 2023-10-01T09:37:45.408 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:45 smithi138 ceph-mon[185886]: pgmap v1261: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 21885/150294 objects degraded (14.561%); 23927/150294 objects misplaced (15.920%); 6.1 MiB/s, 96 objects/s recovering 2023-10-01T09:37:45.512 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:45.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:45 smithi190 ceph-mon[160435]: pgmap v1261: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 21885/150294 objects degraded (14.561%); 23927/150294 objects misplaced (15.920%); 6.1 MiB/s, 96 objects/s recovering 2023-10-01T09:37:45.563 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:45.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:45.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:45.692 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:45.722 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:37:45.745 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:45.754 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:37:45.800 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:37:45.859 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:45.863 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:37:45.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:45.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:45.909 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:37:45.956 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:37:45.969 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:37:45.970 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:37:46.018 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:46.033 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:37:46.043 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:37:46.043 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:37:46.043 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:37:46.043 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:37:46.043 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:37:46.044 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:37:46.044 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:37:46.044 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:37:46.081 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:37:46.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:46.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:37:46.186 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:37:46.241 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:46.307 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:37:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:46 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2776882318' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:46.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:46 smithi190 ceph-mon[160435]: pgmap v1262: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 21473/145083 objects degraded (14.800%); 23117/145083 objects misplaced (15.934%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:37:46.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:46.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:37:46.325 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:37:46.325 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:37:46.412 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:37:46.508 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:46 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2776882318' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:46.509 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:46 smithi138 ceph-mon[192513]: pgmap v1262: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 21473/145083 objects degraded (14.800%); 23117/145083 objects misplaced (15.934%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:37:46.509 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:46 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2776882318' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:46.509 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:46 smithi138 ceph-mon[185886]: pgmap v1262: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 21473/145083 objects degraded (14.800%); 23117/145083 objects misplaced (15.934%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:37:46.516 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:47.361 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:37:47.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:47 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 21473/145083 objects degraded (14.800%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:47.838 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:37:47.955 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:47 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 21473/145083 objects degraded (14.800%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:47.955 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:47 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 21473/145083 objects degraded (14.800%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:48.192 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:37:48.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:48.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:37:48.310 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:37:48.427 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:48.624 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:37:48.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:48.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:37:48.738 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:37:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:48 smithi190 ceph-mon[160435]: pgmap v1263: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 21473/143904 objects degraded (14.922%); 23117/143904 objects misplaced (16.064%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:37:48.837 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:48 smithi138 ceph-mon[185886]: pgmap v1263: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 21473/143904 objects degraded (14.922%); 23117/143904 objects misplaced (16.064%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:37:48.837 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:48 smithi138 ceph-mon[192513]: pgmap v1263: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 21473/143904 objects degraded (14.922%); 23117/143904 objects misplaced (16.064%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:37:48.841 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:48.952 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:37:48.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:48.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:37:49.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:37:49.150 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:49.314 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:37:49.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:49.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:37:49.444 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:37:49.531 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:49.669 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:37:49.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:49.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:37:49.734 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:37:49.774 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:37:49.780 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:37:49.781 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1086912846' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:37:49.781 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:49 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:37:49.781 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:37:49.781 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1086912846' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:37:49.781 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:49 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:37:49.869 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:37:49.869 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:37:49.878 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:49.894 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:37:49.894 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1086912846' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:37:49.894 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:49 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:37:49.980 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:37:50.038 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:37:50.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:50.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:37:50.145 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:37:50.171 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 2, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:37:50.171 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:37:50.172 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-01T09:37:50.172 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:37:50.172 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:37:50.172 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-01T09:37:50.214 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:51.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:50 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:37:51.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:50 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:37:51.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:50 smithi190 ceph-mon[160435]: osdmap e450: 8 total, 8 up, 8 in 2023-10-01T09:37:51.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:50 smithi190 ceph-mon[160435]: failed to encode map e450 with expected crc 2023-10-01T09:37:51.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:50 smithi190 ceph-mon[160435]: failed to encode map e450 with expected crc 2023-10-01T09:37:51.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:50 smithi190 ceph-mon[160435]: pgmap v1265: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 21097/139212 objects degraded (15.155%); 22410/139212 objects misplaced (16.098%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:37:51.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:50 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:37:51.077 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:50 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:37:51.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:50 smithi138 ceph-mon[185886]: osdmap e450: 8 total, 8 up, 8 in 2023-10-01T09:37:51.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:50 smithi138 ceph-mon[185886]: failed to encode map e450 with expected crc 2023-10-01T09:37:51.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:50 smithi138 ceph-mon[185886]: failed to encode map e450 with expected crc 2023-10-01T09:37:51.078 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:50 smithi138 ceph-mon[185886]: pgmap v1265: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 21097/139212 objects degraded (15.155%); 22410/139212 objects misplaced (16.098%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:37:51.078 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:50 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:37:51.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:50 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:37:51.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:50 smithi138 ceph-mon[192513]: osdmap e450: 8 total, 8 up, 8 in 2023-10-01T09:37:51.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:50 smithi138 ceph-mon[192513]: failed to encode map e450 with expected crc 2023-10-01T09:37:51.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:50 smithi138 ceph-mon[192513]: failed to encode map e450 with expected crc 2023-10-01T09:37:51.079 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:50 smithi138 ceph-mon[192513]: pgmap v1265: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 21097/139212 objects degraded (15.155%); 22410/139212 objects misplaced (16.098%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:37:51.080 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:37:51.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:51.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:37:51.200 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:37:51.254 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:51.285 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:37:51.335 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:37:51.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:51.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:37:51.415 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:51.443 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:51.474 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:37:51.548 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:51.658 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:51.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:51.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:51.765 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:51.813 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:52.142 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:52.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:52.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:52.246 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:52.352 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:53.254 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:53.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:53.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:53.285 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:37:53.366 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:37:53.508 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:53 smithi138 ceph-mon[185886]: pgmap v1266: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 21097/138408 objects degraded (15.243%); 22410/138408 objects misplaced (16.191%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:37:53.508 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:53 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 21097/138408 objects degraded (15.243%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:53.508 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:53 smithi138 ceph-mon[192513]: pgmap v1266: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 21097/138408 objects degraded (15.243%); 22410/138408 objects misplaced (16.191%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:37:53.508 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:53 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 21097/138408 objects degraded (15.243%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:53.519 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:37:53.523 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:53.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:53 smithi190 ceph-mon[160435]: pgmap v1266: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 21097/138408 objects degraded (15.243%); 22410/138408 objects misplaced (16.191%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:37:53.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:53 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 21097/138408 objects degraded (15.243%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:53.758 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:37:53.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:53.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:37:53.868 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:37:53.931 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:53.990 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:37:53.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:53.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:37:54.080 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:37:54.080 INFO:teuthology.orchestra.run.smithi190.stdout: "success": "" 2023-10-01T09:37:54.080 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:37:54.096 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:54.148 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:37:54.161 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:54.189 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:37:54.220 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:37:54.254 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:37:54.269 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:37:54.323 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:37:54.364 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:37:54.406 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:37:54.417 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:54 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2545926364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:54.462 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:37:54.491 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:37:54.491 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:37:54.491 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:37:54.491 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:37:54.491 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:37:54.491 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:37:54.492 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:37:54.492 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:37:54.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:54 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2545926364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:54.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:54 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2545926364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:37:54.640 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:54.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:54.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:54.744 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:54.797 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:55.314 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:37:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:37:54] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.33.4" 2023-10-01T09:37:55.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:55 smithi138 ceph-mon[185886]: pgmap v1267: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 21097/137982 objects degraded (15.290%); 22410/137982 objects misplaced (16.241%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:37:55.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:55 smithi138 ceph-mon[192513]: pgmap v1267: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 21097/137982 objects degraded (15.290%); 22410/137982 objects misplaced (16.241%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:37:55.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:55 smithi190 ceph-mon[160435]: pgmap v1267: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 21097/137982 objects degraded (15.290%); 22410/137982 objects misplaced (16.241%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:37:56.096 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:56.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:56.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:56.206 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:37:56.308 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:37:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:37:56.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:56 smithi190 ceph-mon[160435]: pgmap v1268: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 20756/132753 objects degraded (15.635%); 21621/132753 objects misplaced (16.287%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:37:56.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:37:56.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:37:56.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:56 smithi138 ceph-mon[185886]: pgmap v1268: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 20756/132753 objects degraded (15.635%); 21621/132753 objects misplaced (16.287%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:37:56.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:37:56.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:37:56.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:56 smithi138 ceph-mon[192513]: pgmap v1268: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 20756/132753 objects degraded (15.635%); 21621/132753 objects misplaced (16.287%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:37:56.807 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:37:56.807 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:37:56.807 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:37:56.807 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:37:56.807 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.808 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:61933 8.5 8:ae340eae:::benchmark_data_smithi138_431617_object61932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e340)", 2023-10-01T09:37:56.808 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:13.546596+0000", 2023-10-01T09:37:56.808 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 463.25773807000002, 2023-10-01T09:37:56.808 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.170064412, 2023-10-01T09:37:56.808 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.808 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.808 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.808 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.808 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.808 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 61933 2023-10-01T09:37:56.809 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.809 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.809 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.809 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.809 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546596+0000", 2023-10-01T09:37:56.809 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.809 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.809 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.809 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.809 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546596+0000", 2023-10-01T09:37:56.810 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:37:56.810 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.810 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.810 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.810 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546595+0000", 2023-10-01T09:37:56.810 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0971999999999999e-05 2023-10-01T09:37:56.810 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.810 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.810 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.811 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546646+0000", 2023-10-01T09:37:56.811 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7750000000000004e-06 2023-10-01T09:37:56.811 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.811 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.811 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.811 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546656+0000", 2023-10-01T09:37:56.811 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8340000000000001e-06 2023-10-01T09:37:56.811 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.811 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.812 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.812 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.546663+0000", 2023-10-01T09:37:56.812 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019821679999999999 2023-10-01T09:37:56.812 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.812 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.812 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.812 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.548645+0000", 2023-10-01T09:37:56.812 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010491400000000001 2023-10-01T09:37:56.812 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.812 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.813 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.813 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.548750+0000", 2023-10-01T09:37:56.813 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011701099999999999 2023-10-01T09:37:56.813 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.813 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.813 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.813 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.548867+0000", 2023-10-01T09:37:56.813 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0020976710000000002 2023-10-01T09:37:56.813 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.813 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.814 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.814 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.550964+0000", 2023-10-01T09:37:56.814 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0018446389999999999 2023-10-01T09:37:56.814 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.814 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.814 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.814 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.552809+0000", 2023-10-01T09:37:56.814 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16371334600000001 2023-10-01T09:37:56.814 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.814 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.815 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.815 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716522+0000", 2023-10-01T09:37:56.815 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3269999999999994e-05 2023-10-01T09:37:56.815 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.815 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.815 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.815 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716606+0000", 2023-10-01T09:37:56.815 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4738999999999999e-05 2023-10-01T09:37:56.815 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.815 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.816 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.816 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:13.716660+0000", 2023-10-01T09:37:56.816 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.170064412 2023-10-01T09:37:56.816 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.816 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.816 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.816 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.816 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.816 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:75567 8.5 8:aef5727f:::benchmark_data_smithi138_431617_object75566:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-10-01T09:37:56.816 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:28.647427+0000", 2023-10-01T09:37:56.817 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 448.15690702000001, 2023-10-01T09:37:56.817 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17919886700000001, 2023-10-01T09:37:56.817 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.817 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.817 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.817 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.817 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.817 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 75567 2023-10-01T09:37:56.817 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.817 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.818 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.818 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.818 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647427+0000", 2023-10-01T09:37:56.818 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.818 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.818 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.818 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.818 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647427+0000", 2023-10-01T09:37:56.818 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:56.819 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.819 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.819 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.819 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647425+0000", 2023-10-01T09:37:56.819 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8278e-05 2023-10-01T09:37:56.819 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.819 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.819 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.819 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647483+0000", 2023-10-01T09:37:56.819 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4720999999999999e-05 2023-10-01T09:37:56.820 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.820 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.820 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.820 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647508+0000", 2023-10-01T09:37:56.820 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8720000000000002e-06 2023-10-01T09:37:56.820 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.820 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.820 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.820 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647516+0000", 2023-10-01T09:37:56.820 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00034716700000000001 2023-10-01T09:37:56.821 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.821 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.821 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.821 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647863+0000", 2023-10-01T09:37:56.821 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5581999999999994e-05 2023-10-01T09:37:56.821 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.821 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.821 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.821 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647959+0000", 2023-10-01T09:37:56.821 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000119572 2023-10-01T09:37:56.822 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.822 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.822 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.822 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.648078+0000", 2023-10-01T09:37:56.822 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078048329999999997 2023-10-01T09:37:56.822 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.822 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.822 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.822 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.655883+0000", 2023-10-01T09:37:56.822 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013701100000000001 2023-10-01T09:37:56.823 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.823 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.823 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.823 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.656020+0000", 2023-10-01T09:37:56.823 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17050426699999999 2023-10-01T09:37:56.823 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.823 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.823 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.823 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.826524+0000", 2023-10-01T09:37:56.824 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8119999999999999e-05 2023-10-01T09:37:56.824 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.824 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.824 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.824 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.826582+0000", 2023-10-01T09:37:56.824 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3550999999999997e-05 2023-10-01T09:37:56.824 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.824 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.824 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.824 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.826626+0000", 2023-10-01T09:37:56.825 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17919886700000001 2023-10-01T09:37:56.825 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.825 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.825 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.825 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.825 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.825 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:75568 8.5 8:ad74c15b:::benchmark_data_smithi138_431617_object75567:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-10-01T09:37:56.825 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:28.647524+0000", 2023-10-01T09:37:56.825 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 448.15681063300002, 2023-10-01T09:37:56.825 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18103444699999999, 2023-10-01T09:37:56.826 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.826 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.826 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.826 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.826 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.826 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 75568 2023-10-01T09:37:56.826 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.826 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.827 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.827 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.827 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647524+0000", 2023-10-01T09:37:56.827 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.827 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.827 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.827 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.827 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647524+0000", 2023-10-01T09:37:56.827 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:37:56.827 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.828 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.828 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.828 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647523+0000", 2023-10-01T09:37:56.828 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4457999999999998e-05 2023-10-01T09:37:56.828 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.828 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.828 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.828 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647597+0000", 2023-10-01T09:37:56.828 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6279999999999994e-06 2023-10-01T09:37:56.828 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.828 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.829 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.829 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647606+0000", 2023-10-01T09:37:56.829 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0679999999999996e-06 2023-10-01T09:37:56.829 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.829 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.829 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.829 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647611+0000", 2023-10-01T09:37:56.829 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0030153350000000001 2023-10-01T09:37:56.829 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.829 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.830 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.830 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.650626+0000", 2023-10-01T09:37:56.830 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013843899999999999 2023-10-01T09:37:56.830 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.830 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.830 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.830 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.650765+0000", 2023-10-01T09:37:56.830 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017521299999999999 2023-10-01T09:37:56.830 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.830 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.831 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.831 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.650940+0000", 2023-10-01T09:37:56.831 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049486560000000001 2023-10-01T09:37:56.831 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.831 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.831 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.831 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.655889+0000", 2023-10-01T09:37:56.831 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00030954499999999999 2023-10-01T09:37:56.831 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.831 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.832 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.832 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.656198+0000", 2023-10-01T09:37:56.832 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.172268646 2023-10-01T09:37:56.832 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.832 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.832 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.832 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828467+0000", 2023-10-01T09:37:56.832 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0330000000000001e-05 2023-10-01T09:37:56.832 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.832 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.833 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.833 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828517+0000", 2023-10-01T09:37:56.833 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0725000000000001e-05 2023-10-01T09:37:56.833 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.833 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.833 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.833 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828558+0000", 2023-10-01T09:37:56.833 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18103444699999999 2023-10-01T09:37:56.833 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.833 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.834 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.834 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.834 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.834 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:75571 8.5 8:a2fb1dba:::benchmark_data_smithi138_431617_object75570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-10-01T09:37:56.834 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:28.647622+0000", 2023-10-01T09:37:56.834 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 448.15671200499997, 2023-10-01T09:37:56.834 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18107846399999999, 2023-10-01T09:37:56.834 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.834 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.835 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.835 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.835 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.835 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 75571 2023-10-01T09:37:56.835 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.835 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.835 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.835 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.835 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647622+0000", 2023-10-01T09:37:56.835 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.836 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.836 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.836 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.836 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647622+0000", 2023-10-01T09:37:56.836 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:37:56.836 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.836 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.836 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.836 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647621+0000", 2023-10-01T09:37:56.836 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8969e-05 2023-10-01T09:37:56.837 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.837 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.837 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.837 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647650+0000", 2023-10-01T09:37:56.837 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0261000000000001e-05 2023-10-01T09:37:56.837 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.837 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.837 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.837 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647661+0000", 2023-10-01T09:37:56.837 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8960000000000002e-06 2023-10-01T09:37:56.838 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.838 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.838 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.838 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.647666+0000", 2023-10-01T09:37:56.838 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053841219999999999 2023-10-01T09:37:56.838 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.838 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.838 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.838 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.653050+0000", 2023-10-01T09:37:56.838 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015679600000000001 2023-10-01T09:37:56.839 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.839 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.839 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.839 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.653206+0000", 2023-10-01T09:37:56.839 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00019690199999999999 2023-10-01T09:37:56.839 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.839 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.839 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.839 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.653403+0000", 2023-10-01T09:37:56.839 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002801993 2023-10-01T09:37:56.840 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.840 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.840 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.840 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.656205+0000", 2023-10-01T09:37:56.840 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00025078999999999998 2023-10-01T09:37:56.840 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.840 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.840 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.840 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.656456+0000", 2023-10-01T09:37:56.841 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.172176575 2023-10-01T09:37:56.841 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.841 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.841 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.841 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828633+0000", 2023-10-01T09:37:56.841 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1740999999999999e-05 2023-10-01T09:37:56.841 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.841 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.841 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.841 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828674+0000", 2023-10-01T09:37:56.841 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6143999999999999e-05 2023-10-01T09:37:56.842 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.842 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.842 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.842 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:28.828701+0000", 2023-10-01T09:37:56.842 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18107846399999999 2023-10-01T09:37:56.842 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.842 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.842 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.842 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.842 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.843 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:89387 8.5 8:ac2bd163:::benchmark_data_smithi138_431617_object89386:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-01T09:37:56.843 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:46.701591+0000", 2023-10-01T09:37:56.843 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 430.10274292600002, 2023-10-01T09:37:56.843 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.242286687, 2023-10-01T09:37:56.843 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.843 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.843 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.843 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.843 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.843 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 89387 2023-10-01T09:37:56.844 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.844 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.844 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.844 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.844 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701591+0000", 2023-10-01T09:37:56.844 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.844 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.844 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.844 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.844 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701591+0000", 2023-10-01T09:37:56.845 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:56.845 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.845 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.845 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.845 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701590+0000", 2023-10-01T09:37:56.845 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2069e-05 2023-10-01T09:37:56.845 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.845 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.845 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.846 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701632+0000", 2023-10-01T09:37:56.846 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.005e-05 2023-10-01T09:37:56.846 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.846 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.846 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.846 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701642+0000", 2023-10-01T09:37:56.846 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2100000000000001e-06 2023-10-01T09:37:56.846 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.846 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.846 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.847 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701647+0000", 2023-10-01T09:37:56.847 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012216000000000001 2023-10-01T09:37:56.847 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.847 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.847 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.847 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701769+0000", 2023-10-01T09:37:56.847 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015872299999999999 2023-10-01T09:37:56.847 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.847 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.848 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.848 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.701928+0000", 2023-10-01T09:37:56.848 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017488099999999999 2023-10-01T09:37:56.848 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.848 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.848 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.702103+0000", 2023-10-01T09:37:56.848 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0026736849999999999 2023-10-01T09:37:56.848 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.849 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.849 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.704776+0000", 2023-10-01T09:37:56.849 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00095142499999999999 2023-10-01T09:37:56.849 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.849 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.849 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.849 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.705728+0000", 2023-10-01T09:37:56.849 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.238030466 2023-10-01T09:37:56.849 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.849 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.849 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.850 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.943758+0000", 2023-10-01T09:37:56.850 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1157999999999999e-05 2023-10-01T09:37:56.850 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.850 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.850 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.850 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.943829+0000", 2023-10-01T09:37:56.850 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8421000000000003e-05 2023-10-01T09:37:56.850 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.850 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.851 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.851 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.943878+0000", 2023-10-01T09:37:56.851 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.242286687 2023-10-01T09:37:56.851 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.851 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.851 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.851 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.851 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.851 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:89417 8.5 8:ac07dffe:::benchmark_data_smithi138_431617_object89416:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-10-01T09:37:56.851 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:46.731169+0000", 2023-10-01T09:37:56.852 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 430.07316481399999, 2023-10-01T09:37:56.852 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21775186899999999, 2023-10-01T09:37:56.852 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.852 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.852 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.852 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.852 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.852 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 89417 2023-10-01T09:37:56.852 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.852 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.853 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.853 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.853 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731169+0000", 2023-10-01T09:37:56.853 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.853 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.853 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.853 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.853 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731169+0000", 2023-10-01T09:37:56.853 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:56.853 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.854 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.854 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.854 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731168+0000", 2023-10-01T09:37:56.854 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0485000000000003e-05 2023-10-01T09:37:56.854 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.854 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.854 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.854 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731208+0000", 2023-10-01T09:37:56.854 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5139e-05 2023-10-01T09:37:56.854 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.855 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.855 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.855 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731223+0000", 2023-10-01T09:37:56.855 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4389999999999997e-06 2023-10-01T09:37:56.855 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.855 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.855 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.855 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731230+0000", 2023-10-01T09:37:56.855 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8512e-05 2023-10-01T09:37:56.855 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.856 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.856 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.856 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731268+0000", 2023-10-01T09:37:56.856 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012213799999999999 2023-10-01T09:37:56.856 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.856 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.856 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.856 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731390+0000", 2023-10-01T09:37:56.856 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000120797 2023-10-01T09:37:56.856 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.857 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.857 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.857 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.731511+0000", 2023-10-01T09:37:56.857 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002402623 2023-10-01T09:37:56.857 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.857 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.857 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.857 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.733914+0000", 2023-10-01T09:37:56.857 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00025578500000000002 2023-10-01T09:37:56.857 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.858 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.858 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.858 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.734170+0000", 2023-10-01T09:37:56.858 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21464570199999999 2023-10-01T09:37:56.858 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.858 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.858 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.858 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.948815+0000", 2023-10-01T09:37:56.858 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2032999999999994e-05 2023-10-01T09:37:56.858 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.859 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.859 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.859 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.948877+0000", 2023-10-01T09:37:56.859 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3804999999999998e-05 2023-10-01T09:37:56.859 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.859 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.859 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.859 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:46.948921+0000", 2023-10-01T09:37:56.859 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21775186899999999 2023-10-01T09:37:56.859 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.860 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.860 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.860 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.860 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.860 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:98811 8.5 8:aea6ced9:::benchmark_data_smithi138_431617_object98810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-01T09:37:56.860 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:59.492826+0000", 2023-10-01T09:37:56.860 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 417.31150828, 2023-10-01T09:37:56.860 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28100355799999999, 2023-10-01T09:37:56.860 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.861 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.861 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.861 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.861 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.861 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 98811 2023-10-01T09:37:56.861 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.861 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.861 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.861 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.861 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.492826+0000", 2023-10-01T09:37:56.862 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.862 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.862 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.862 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.862 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.492826+0000", 2023-10-01T09:37:56.862 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:56.862 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.862 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.862 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.862 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.492825+0000", 2023-10-01T09:37:56.863 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9044000000000001e-05 2023-10-01T09:37:56.863 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.863 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.863 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.863 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.492864+0000", 2023-10-01T09:37:56.863 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4671000000000001e-05 2023-10-01T09:37:56.863 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.863 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.863 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.863 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.492878+0000", 2023-10-01T09:37:56.864 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2079999999999997e-06 2023-10-01T09:37:56.864 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.864 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.864 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.864 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.492885+0000", 2023-10-01T09:37:56.864 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0015081719999999999 2023-10-01T09:37:56.864 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.864 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.864 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.864 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.494393+0000", 2023-10-01T09:37:56.865 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000132967 2023-10-01T09:37:56.865 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.865 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.865 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.865 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.494526+0000", 2023-10-01T09:37:56.865 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014614700000000001 2023-10-01T09:37:56.865 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.865 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.866 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.866 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.494672+0000", 2023-10-01T09:37:56.866 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0034588140000000002 2023-10-01T09:37:56.866 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.866 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.866 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.866 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.498131+0000", 2023-10-01T09:37:56.866 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001235756 2023-10-01T09:37:56.867 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.867 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.867 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.867 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.499366+0000", 2023-10-01T09:37:56.867 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27437294899999998 2023-10-01T09:37:56.867 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.867 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.867 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.867 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.773739+0000", 2023-10-01T09:37:56.868 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0510000000000003e-05 2023-10-01T09:37:56.868 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.868 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.868 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.868 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.773790+0000", 2023-10-01T09:37:56.868 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9598999999999998e-05 2023-10-01T09:37:56.868 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.868 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.869 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.869 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.773829+0000", 2023-10-01T09:37:56.869 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28100355799999999 2023-10-01T09:37:56.869 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.869 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.869 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.869 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.869 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.869 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:98822 8.5 8:a281756c:::benchmark_data_smithi138_431617_object98821:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-01T09:37:56.870 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:30:59.506300+0000", 2023-10-01T09:37:56.870 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 417.29803403099999, 2023-10-01T09:37:56.870 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.267638088, 2023-10-01T09:37:56.870 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.870 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.870 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.870 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.870 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.871 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 98822 2023-10-01T09:37:56.871 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.871 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.871 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.871 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.871 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506300+0000", 2023-10-01T09:37:56.871 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.872 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.872 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.872 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.872 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506300+0000", 2023-10-01T09:37:56.872 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:56.872 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.872 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.872 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.872 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506298+0000", 2023-10-01T09:37:56.872 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1570999999999998e-05 2023-10-01T09:37:56.873 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.873 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.873 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.873 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506350+0000", 2023-10-01T09:37:56.873 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8213e-05 2023-10-01T09:37:56.873 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.873 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.873 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.873 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506368+0000", 2023-10-01T09:37:56.874 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4329999999999998e-06 2023-10-01T09:37:56.874 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.874 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.874 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.874 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506376+0000", 2023-10-01T09:37:56.874 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2095999999999997e-05 2023-10-01T09:37:56.874 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.874 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.874 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.875 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506428+0000", 2023-10-01T09:37:56.875 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016378599999999999 2023-10-01T09:37:56.875 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.875 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.875 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.875 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506591+0000", 2023-10-01T09:37:56.875 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000195434 2023-10-01T09:37:56.875 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.875 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.876 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.876 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.506787+0000", 2023-10-01T09:37:56.876 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0027607909999999999 2023-10-01T09:37:56.876 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.876 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.876 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.876 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.509548+0000", 2023-10-01T09:37:56.876 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001199079 2023-10-01T09:37:56.876 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.876 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.877 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.877 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.510747+0000", 2023-10-01T09:37:56.877 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26312249300000001 2023-10-01T09:37:56.877 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.877 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.877 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.877 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.773869+0000", 2023-10-01T09:37:56.877 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8377999999999997e-05 2023-10-01T09:37:56.877 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.878 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.878 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.878 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.773908+0000", 2023-10-01T09:37:56.878 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0613e-05 2023-10-01T09:37:56.878 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.878 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.878 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.878 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:30:59.773938+0000", 2023-10-01T09:37:56.878 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.267638088 2023-10-01T09:37:56.878 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.879 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.879 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.879 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.879 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.879 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:102764 8.5 8:ab6d8cb6:::benchmark_data_smithi138_431617_object1353:head [delete] snapc 0=[] ondisk+write+known_if_redirected e348)", 2023-10-01T09:37:56.879 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:31:06.313439+0000", 2023-10-01T09:37:56.879 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 410.49089536299999, 2023-10-01T09:37:56.879 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18351769100000001, 2023-10-01T09:37:56.880 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.880 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.880 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.880 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.880 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.880 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 102764 2023-10-01T09:37:56.880 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.880 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.880 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.880 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.881 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313439+0000", 2023-10-01T09:37:56.881 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.881 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.881 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.881 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.881 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313439+0000", 2023-10-01T09:37:56.881 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:56.881 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.881 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.882 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.882 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313437+0000", 2023-10-01T09:37:56.882 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0399999999999993e-06 2023-10-01T09:37:56.882 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.882 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.882 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.882 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313445+0000", 2023-10-01T09:37:56.882 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.738e-05 2023-10-01T09:37:56.882 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.882 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.883 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.883 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313463+0000", 2023-10-01T09:37:56.883 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7819999999999996e-06 2023-10-01T09:37:56.883 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.883 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.883 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.883 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313469+0000", 2023-10-01T09:37:56.883 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4065000000000004e-05 2023-10-01T09:37:56.883 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.883 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.884 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.884 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313533+0000", 2023-10-01T09:37:56.884 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014046600000000001 2023-10-01T09:37:56.884 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.884 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.884 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.884 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313674+0000", 2023-10-01T09:37:56.884 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013492699999999999 2023-10-01T09:37:56.884 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.884 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.885 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.885 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313809+0000", 2023-10-01T09:37:56.885 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0067771769999999997 2023-10-01T09:37:56.885 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.885 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.885 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.885 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.320586+0000", 2023-10-01T09:37:56.885 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0048310360000000004 2023-10-01T09:37:56.885 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.886 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.886 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.886 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.325417+0000", 2023-10-01T09:37:56.886 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17142922099999999 2023-10-01T09:37:56.886 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.886 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.886 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.886 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.496846+0000", 2023-10-01T09:37:56.886 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0718999999999999e-05 2023-10-01T09:37:56.886 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.887 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.887 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.887 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.496907+0000", 2023-10-01T09:37:56.887 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9557e-05 2023-10-01T09:37:56.887 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.887 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.887 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.887 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.496956+0000", 2023-10-01T09:37:56.887 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18351769100000001 2023-10-01T09:37:56.887 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.888 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.888 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.888 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.888 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.888 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:102767 8.5 8:aad588e2:::benchmark_data_smithi138_431617_object1356:head [delete] snapc 0=[] ondisk+write+known_if_redirected e348)", 2023-10-01T09:37:56.888 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:31:06.313484+0000", 2023-10-01T09:37:56.888 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 410.49085038499999, 2023-10-01T09:37:56.888 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18457310299999999, 2023-10-01T09:37:56.888 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.889 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.889 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.889 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.889 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.889 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 102767 2023-10-01T09:37:56.889 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.889 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.889 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.889 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.889 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313484+0000", 2023-10-01T09:37:56.889 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.890 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.890 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.890 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.890 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313484+0000", 2023-10-01T09:37:56.890 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:37:56.890 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.890 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.890 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.890 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313483+0000", 2023-10-01T09:37:56.891 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9440000000000001e-06 2023-10-01T09:37:56.891 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.891 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.891 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.891 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313486+0000", 2023-10-01T09:37:56.891 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5590000000000006e-06 2023-10-01T09:37:56.891 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.891 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.891 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.891 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313495+0000", 2023-10-01T09:37:56.892 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0080000000000004e-06 2023-10-01T09:37:56.892 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.892 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.892 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.892 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.313499+0000", 2023-10-01T09:37:56.892 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0036149279999999999 2023-10-01T09:37:56.892 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.892 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.892 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.892 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.317114+0000", 2023-10-01T09:37:56.893 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000124485 2023-10-01T09:37:56.893 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.893 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.893 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.893 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.317238+0000", 2023-10-01T09:37:56.893 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013227399999999999 2023-10-01T09:37:56.893 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.893 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.893 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.893 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.317370+0000", 2023-10-01T09:37:56.894 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035185339999999998 2023-10-01T09:37:56.894 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.894 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.894 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.894 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.320889+0000", 2023-10-01T09:37:56.894 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045892579999999997 2023-10-01T09:37:56.894 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.894 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.894 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.894 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.325478+0000", 2023-10-01T09:37:56.895 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.172485942 2023-10-01T09:37:56.895 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.895 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.895 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.895 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.497964+0000", 2023-10-01T09:37:56.895 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8517999999999999e-05 2023-10-01T09:37:56.895 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.895 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.895 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.895 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.498013+0000", 2023-10-01T09:37:56.896 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4134000000000001e-05 2023-10-01T09:37:56.896 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.896 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.896 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.896 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:06.498057+0000", 2023-10-01T09:37:56.896 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18457310299999999 2023-10-01T09:37:56.896 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.896 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.896 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.897 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.897 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.897 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:115052 8.5 8:a3e965a9:::benchmark_data_smithi138_431617_object13641:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-10-01T09:37:56.897 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:31:22.838052+0000", 2023-10-01T09:37:56.897 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 393.96628174, 2023-10-01T09:37:56.897 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26344810600000002, 2023-10-01T09:37:56.897 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.897 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.897 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.897 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.898 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.898 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 115052 2023-10-01T09:37:56.898 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.898 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.898 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.898 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.898 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838052+0000", 2023-10-01T09:37:56.898 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.898 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.898 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.899 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.899 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838052+0000", 2023-10-01T09:37:56.899 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:56.899 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.899 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.899 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.899 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838051+0000", 2023-10-01T09:37:56.901 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6899999999999997e-06 2023-10-01T09:37:56.901 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.901 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.901 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.901 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838057+0000", 2023-10-01T09:37:56.901 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3438000000000001e-05 2023-10-01T09:37:56.901 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.901 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.901 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.901 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838070+0000", 2023-10-01T09:37:56.902 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2419999999999997e-06 2023-10-01T09:37:56.902 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.902 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.902 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.902 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838076+0000", 2023-10-01T09:37:56.902 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2471e-05 2023-10-01T09:37:56.902 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.902 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.902 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.902 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838119+0000", 2023-10-01T09:37:56.903 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012397600000000001 2023-10-01T09:37:56.903 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.903 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.903 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.903 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838243+0000", 2023-10-01T09:37:56.903 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000152304 2023-10-01T09:37:56.903 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.903 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.903 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.903 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838395+0000", 2023-10-01T09:37:56.904 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0071079480000000002 2023-10-01T09:37:56.904 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.904 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.904 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.904 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.845503+0000", 2023-10-01T09:37:56.904 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0060857350000000001 2023-10-01T09:37:56.904 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.904 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.904 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.904 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.851589+0000", 2023-10-01T09:37:56.905 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.249801365 2023-10-01T09:37:56.905 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.905 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.905 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.905 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101390+0000", 2023-10-01T09:37:56.905 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2990000000000005e-05 2023-10-01T09:37:56.905 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.905 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.905 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.905 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101453+0000", 2023-10-01T09:37:56.906 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7327000000000003e-05 2023-10-01T09:37:56.906 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.906 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.906 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.906 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101501+0000", 2023-10-01T09:37:56.906 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26344810600000002 2023-10-01T09:37:56.906 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.906 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.906 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.906 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.907 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.907 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:115054 8.5 8:ac73bb40:::benchmark_data_smithi138_431617_object13643:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-10-01T09:37:56.907 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:31:22.838091+0000", 2023-10-01T09:37:56.907 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 393.96624358000003, 2023-10-01T09:37:56.907 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26356659199999999, 2023-10-01T09:37:56.907 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.907 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.907 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.907 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.907 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.908 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 115054 2023-10-01T09:37:56.908 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.908 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.908 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.908 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.908 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838091+0000", 2023-10-01T09:37:56.908 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.908 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.908 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.908 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.909 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838091+0000", 2023-10-01T09:37:56.909 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:37:56.909 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.909 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.909 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.909 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838090+0000", 2023-10-01T09:37:56.909 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7269999999999999e-06 2023-10-01T09:37:56.909 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.909 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.909 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.910 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838093+0000", 2023-10-01T09:37:56.910 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7969999999999997e-06 2023-10-01T09:37:56.910 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.910 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.910 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.910 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838099+0000", 2023-10-01T09:37:56.910 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7689999999999998e-06 2023-10-01T09:37:56.910 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.910 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.910 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.911 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.838102+0000", 2023-10-01T09:37:56.911 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0041852180000000001 2023-10-01T09:37:56.911 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.911 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.911 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.911 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.842288+0000", 2023-10-01T09:37:56.911 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012376100000000001 2023-10-01T09:37:56.911 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.911 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.911 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.912 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.842411+0000", 2023-10-01T09:37:56.912 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000119407 2023-10-01T09:37:56.912 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.912 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.912 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.912 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.842531+0000", 2023-10-01T09:37:56.912 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0031142069999999999 2023-10-01T09:37:56.912 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.912 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.912 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.913 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.845645+0000", 2023-10-01T09:37:56.913 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0059780909999999996 2023-10-01T09:37:56.913 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.913 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.913 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.913 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.851623+0000", 2023-10-01T09:37:56.913 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24993852699999999 2023-10-01T09:37:56.913 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.913 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.913 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.914 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101562+0000", 2023-10-01T09:37:56.914 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2067000000000001e-05 2023-10-01T09:37:56.914 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.914 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.914 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.914 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101614+0000", 2023-10-01T09:37:56.914 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3449999999999999e-05 2023-10-01T09:37:56.914 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.914 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.914 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.915 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101657+0000", 2023-10-01T09:37:56.915 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26356659199999999 2023-10-01T09:37:56.915 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.915 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.915 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.915 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.915 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.915 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:115065 8.5 8:a4841ad2:::benchmark_data_smithi138_431617_object13654:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-10-01T09:37:56.915 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:31:22.847762+0000", 2023-10-01T09:37:56.915 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 393.95657254999998, 2023-10-01T09:37:56.916 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.25401542300000002, 2023-10-01T09:37:56.916 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.916 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.916 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.916 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.916 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.916 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 115065 2023-10-01T09:37:56.916 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.916 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.916 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.917 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.917 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847762+0000", 2023-10-01T09:37:56.917 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.917 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.917 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.917 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.917 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847762+0000", 2023-10-01T09:37:56.917 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:56.917 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.917 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.918 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.918 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847760+0000", 2023-10-01T09:37:56.918 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9060000000000002e-06 2023-10-01T09:37:56.918 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.918 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.918 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.918 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847768+0000", 2023-10-01T09:37:56.918 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6707000000000001e-05 2023-10-01T09:37:56.918 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.918 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.919 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.919 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847784+0000", 2023-10-01T09:37:56.919 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0369999999999997e-06 2023-10-01T09:37:56.919 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.919 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.919 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.919 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847791+0000", 2023-10-01T09:37:56.919 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5365999999999999e-05 2023-10-01T09:37:56.919 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.920 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.920 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.920 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847837+0000", 2023-10-01T09:37:56.920 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012898299999999999 2023-10-01T09:37:56.920 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.920 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.920 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.920 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.847966+0000", 2023-10-01T09:37:56.920 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014580100000000001 2023-10-01T09:37:56.920 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.921 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.921 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.921 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.848112+0000", 2023-10-01T09:37:56.921 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0036017150000000001 2023-10-01T09:37:56.921 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.921 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.921 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.921 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.851713+0000", 2023-10-01T09:37:56.921 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4385000000000003e-05 2023-10-01T09:37:56.921 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.922 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.922 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.922 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:22.851758+0000", 2023-10-01T09:37:56.922 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24993739000000001 2023-10-01T09:37:56.922 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.922 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.922 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.922 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101695+0000", 2023-10-01T09:37:56.922 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4341000000000001e-05 2023-10-01T09:37:56.922 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.923 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.923 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.923 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101739+0000", 2023-10-01T09:37:56.923 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7678000000000001e-05 2023-10-01T09:37:56.923 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.923 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.923 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.923 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:31:23.101777+0000", 2023-10-01T09:37:56.923 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.25401542300000002 2023-10-01T09:37:56.923 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.924 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.924 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.924 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.924 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.924 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:149332 8.5 8:ad5fd0df:::benchmark_data_smithi138_431617_object47921:head [delete] snapc 0=[] ondisk+write+known_if_redirected e358)", 2023-10-01T09:37:56.924 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:11.735369+0000", 2023-10-01T09:37:56.924 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 345.068964725, 2023-10-01T09:37:56.924 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.19642474700000001, 2023-10-01T09:37:56.924 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.924 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.925 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.925 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.925 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.925 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 149332 2023-10-01T09:37:56.925 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.925 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.925 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.925 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.925 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735369+0000", 2023-10-01T09:37:56.925 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.926 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.926 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.926 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.926 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735369+0000", 2023-10-01T09:37:56.926 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:56.926 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.926 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.926 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.926 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735367+0000", 2023-10-01T09:37:56.927 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7149999999999993e-06 2023-10-01T09:37:56.927 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.927 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.927 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.927 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735377+0000", 2023-10-01T09:37:56.927 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8277000000000001e-05 2023-10-01T09:37:56.927 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.927 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.927 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.927 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735395+0000", 2023-10-01T09:37:56.928 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8809999999999996e-06 2023-10-01T09:37:56.928 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.928 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.928 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.928 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735403+0000", 2023-10-01T09:37:56.928 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3931000000000003e-05 2023-10-01T09:37:56.928 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.928 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.929 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.929 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735457+0000", 2023-10-01T09:37:56.929 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000136833 2023-10-01T09:37:56.929 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.929 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.929 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.929 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735594+0000", 2023-10-01T09:37:56.929 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016526299999999999 2023-10-01T09:37:56.929 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.929 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.930 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.930 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.735759+0000", 2023-10-01T09:37:56.930 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0076574440000000002 2023-10-01T09:37:56.930 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.930 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.930 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.930 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.743417+0000", 2023-10-01T09:37:56.930 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053083100000000001 2023-10-01T09:37:56.930 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.930 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.931 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.931 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.748725+0000", 2023-10-01T09:37:56.931 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18300225000000001 2023-10-01T09:37:56.931 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.931 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.931 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.931 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.931727+0000", 2023-10-01T09:37:56.931 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6539999999999999e-05 2023-10-01T09:37:56.931 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.932 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.932 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.932 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.931764+0000", 2023-10-01T09:37:56.932 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0328000000000002e-05 2023-10-01T09:37:56.932 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.932 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.932 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.932 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.931794+0000", 2023-10-01T09:37:56.932 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.19642474700000001 2023-10-01T09:37:56.933 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.933 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.933 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.933 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.933 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.933 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:149342 8.5 8:a6fdf006:::benchmark_data_smithi138_431617_object47931:head [delete] snapc 0=[] ondisk+write+known_if_redirected e358)", 2023-10-01T09:37:56.933 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:11.736386+0000", 2023-10-01T09:37:56.933 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 345.067947656, 2023-10-01T09:37:56.933 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.19553300500000001, 2023-10-01T09:37:56.934 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.934 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.934 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.934 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.934 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.934 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 149342 2023-10-01T09:37:56.934 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.934 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.935 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.935 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.935 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.736386+0000", 2023-10-01T09:37:56.935 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.935 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.935 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.935 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.935 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.736386+0000", 2023-10-01T09:37:56.935 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:56.936 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.936 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.936 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.936 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.736385+0000", 2023-10-01T09:37:56.936 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1870000000000002e-06 2023-10-01T09:37:56.936 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.936 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.936 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.937 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.736391+0000", 2023-10-01T09:37:56.937 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5701000000000001e-05 2023-10-01T09:37:56.937 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.937 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.937 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.937 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.736407+0000", 2023-10-01T09:37:56.937 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5139999999999998e-06 2023-10-01T09:37:56.937 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.938 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.938 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.938 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.736414+0000", 2023-10-01T09:37:56.938 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0031694340000000001 2023-10-01T09:37:56.938 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.938 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.938 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.938 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.739583+0000", 2023-10-01T09:37:56.939 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000117611 2023-10-01T09:37:56.939 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.939 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.939 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.939 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.739701+0000", 2023-10-01T09:37:56.939 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5253000000000004e-05 2023-10-01T09:37:56.939 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.939 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.939 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.940 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.739796+0000", 2023-10-01T09:37:56.940 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0040039790000000004 2023-10-01T09:37:56.940 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.940 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.940 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.940 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.743800+0000", 2023-10-01T09:37:56.940 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050418490000000002 2023-10-01T09:37:56.940 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.940 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.941 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.941 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.748842+0000", 2023-10-01T09:37:56.941 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.182993609 2023-10-01T09:37:56.941 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.941 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.941 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.941 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.931835+0000", 2023-10-01T09:37:56.941 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7147000000000001e-05 2023-10-01T09:37:56.941 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.942 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.942 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.942 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.931883+0000", 2023-10-01T09:37:56.942 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6977000000000002e-05 2023-10-01T09:37:56.942 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.942 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.942 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.942 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:11.931919+0000", 2023-10-01T09:37:56.942 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.19553300500000001 2023-10-01T09:37:56.942 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.943 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.943 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.943 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.943 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.943 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:159796 8.5 8:ae0f494c:::benchmark_data_smithi138_431617_object58385:head [delete] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-10-01T09:37:56.943 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:27.321194+0000", 2023-10-01T09:37:56.943 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 329.483139865, 2023-10-01T09:37:56.943 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2966308610000001, 2023-10-01T09:37:56.943 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.944 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:37:56.944 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.944 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.944 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.944 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 159796 2023-10-01T09:37:56.944 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.944 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.944 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.944 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.945 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321194+0000", 2023-10-01T09:37:56.945 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.945 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.945 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.945 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.945 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321194+0000", 2023-10-01T09:37:56.945 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:37:56.945 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.945 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.945 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.946 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321192+0000", 2023-10-01T09:37:56.946 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3389999999999998e-06 2023-10-01T09:37:56.946 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.946 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.946 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.946 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321199+0000", 2023-10-01T09:37:56.946 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2383e-05 2023-10-01T09:37:56.946 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.946 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.946 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.947 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321211+0000", 2023-10-01T09:37:56.947 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0760000000000002e-06 2023-10-01T09:37:56.947 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.947 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.947 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.947 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321216+0000", 2023-10-01T09:37:56.947 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3895e-05 2023-10-01T09:37:56.947 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.947 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.947 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.948 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321250+0000", 2023-10-01T09:37:56.948 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000108306 2023-10-01T09:37:56.948 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.948 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.948 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.948 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321358+0000", 2023-10-01T09:37:56.948 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7575e-05 2023-10-01T09:37:56.948 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.948 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.948 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,7", 2023-10-01T09:37:56.949 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.321456+0000", 2023-10-01T09:37:56.949 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0041519549999999997 2023-10-01T09:37:56.949 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.949 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.949 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.949 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.325608+0000", 2023-10-01T09:37:56.949 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00030447800000000001 2023-10-01T09:37:56.949 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.949 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.949 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.950 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.325912+0000", 2023-10-01T09:37:56.950 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017190989 2023-10-01T09:37:56.950 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.950 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.950 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.950 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.343103+0000", 2023-10-01T09:37:56.950 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3421000000000001e-05 2023-10-01T09:37:56.950 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.950 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.950 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:56.951 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.343117+0000", 2023-10-01T09:37:56.951 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.100368394 2023-10-01T09:37:56.951 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.951 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.951 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.951 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.443485+0000", 2023-10-01T09:37:56.951 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1826e-05 2023-10-01T09:37:56.951 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.951 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.951 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:56.951 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.443497+0000", 2023-10-01T09:37:56.952 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.174244764 2023-10-01T09:37:56.952 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.952 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.952 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.952 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.617742+0000", 2023-10-01T09:37:56.952 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3264999999999996e-05 2023-10-01T09:37:56.952 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.952 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.952 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.952 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.617825+0000", 2023-10-01T09:37:56.953 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2966308610000001 2023-10-01T09:37:56.953 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.953 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.953 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.953 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.953 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.953 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:159825 8.5 8:ac107179:::benchmark_data_smithi138_431617_object58414:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-10-01T09:37:56.953 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:27.380723+0000", 2023-10-01T09:37:56.953 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 329.42361080900002, 2023-10-01T09:37:56.953 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2513447710000001, 2023-10-01T09:37:56.954 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.954 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.954 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.954 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.954 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.954 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 159825 2023-10-01T09:37:56.954 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.954 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.954 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.954 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.955 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380723+0000", 2023-10-01T09:37:56.955 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.955 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.955 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.955 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.955 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380723+0000", 2023-10-01T09:37:56.955 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:56.955 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.955 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.955 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.956 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380722+0000", 2023-10-01T09:37:56.956 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9830000000000001e-06 2023-10-01T09:37:56.956 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.956 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.956 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.956 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380728+0000", 2023-10-01T09:37:56.956 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.329e-05 2023-10-01T09:37:56.956 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.956 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.956 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.956 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380741+0000", 2023-10-01T09:37:56.957 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3140000000000003e-06 2023-10-01T09:37:56.957 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.957 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.957 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.957 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380747+0000", 2023-10-01T09:37:56.957 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2110000000000002e-05 2023-10-01T09:37:56.957 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.957 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.957 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.957 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380789+0000", 2023-10-01T09:37:56.958 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0670000000000003e-06 2023-10-01T09:37:56.958 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.958 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.958 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:56.958 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.380796+0000", 2023-10-01T09:37:56.958 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.062726183000000005 2023-10-01T09:37:56.958 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.958 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.958 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.958 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.443522+0000", 2023-10-01T09:37:56.959 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5750000000000001e-06 2023-10-01T09:37:56.959 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.959 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.959 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:56.959 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.443531+0000", 2023-10-01T09:37:56.959 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1746131310000001 2023-10-01T09:37:56.959 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.959 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.960 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.960 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.618144+0000", 2023-10-01T09:37:56.960 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00018594600000000001 2023-10-01T09:37:56.960 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.960 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.960 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.960 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.618330+0000", 2023-10-01T09:37:56.960 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000157885 2023-10-01T09:37:56.960 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.960 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.961 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,4,7", 2023-10-01T09:37:56.961 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.618488+0000", 2023-10-01T09:37:56.961 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0091334640000000009 2023-10-01T09:37:56.961 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.961 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.961 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.961 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.627621+0000", 2023-10-01T09:37:56.961 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000203346 2023-10-01T09:37:56.961 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.961 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.962 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.962 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.627824+0000", 2023-10-01T09:37:56.962 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024199270000000001 2023-10-01T09:37:56.962 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.962 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.962 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.962 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.630244+0000", 2023-10-01T09:37:56.962 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0016233860000000001 2023-10-01T09:37:56.962 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.963 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.963 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.963 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.631868+0000", 2023-10-01T09:37:56.963 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6780999999999996e-05 2023-10-01T09:37:56.963 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.963 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.963 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.963 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.631934+0000", 2023-10-01T09:37:56.963 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013365799999999999 2023-10-01T09:37:56.963 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.964 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.964 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.964 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.632068+0000", 2023-10-01T09:37:56.964 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2513447710000001 2023-10-01T09:37:56.964 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.964 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.964 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.964 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.964 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.964 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:159837 8.5 8:a337ee5f:::benchmark_data_smithi138_431617_object58426:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-10-01T09:37:56.965 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:27.399363+0000", 2023-10-01T09:37:56.965 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 329.40497072599999, 2023-10-01T09:37:56.965 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2329211799999999, 2023-10-01T09:37:56.965 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.965 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.965 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.965 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.965 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.965 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 159837 2023-10-01T09:37:56.966 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.966 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.966 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.966 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.966 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399363+0000", 2023-10-01T09:37:56.966 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.966 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.966 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.966 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.966 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399363+0000", 2023-10-01T09:37:56.967 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:37:56.967 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.967 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.967 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.967 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399362+0000", 2023-10-01T09:37:56.967 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1049999999999995e-06 2023-10-01T09:37:56.967 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.967 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.967 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.967 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399370+0000", 2023-10-01T09:37:56.968 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3217999999999999e-05 2023-10-01T09:37:56.968 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.968 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.968 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.968 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399383+0000", 2023-10-01T09:37:56.968 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.575e-06 2023-10-01T09:37:56.968 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.968 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.968 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.969 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399389+0000", 2023-10-01T09:37:56.969 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6829000000000002e-05 2023-10-01T09:37:56.969 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.969 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.969 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.969 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399436+0000", 2023-10-01T09:37:56.969 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4390000000000003e-06 2023-10-01T09:37:56.969 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.969 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.969 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:56.969 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.399443+0000", 2023-10-01T09:37:56.970 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.044108631000000002 2023-10-01T09:37:56.970 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.970 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.970 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.970 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.443552+0000", 2023-10-01T09:37:56.970 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6140000000000003e-06 2023-10-01T09:37:56.970 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.970 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.970 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:37:56.971 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:27.443560+0000", 2023-10-01T09:37:56.971 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1789550630000001 2023-10-01T09:37:56.971 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.971 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.971 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.971 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.622515+0000", 2023-10-01T09:37:56.971 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012748499999999999 2023-10-01T09:37:56.971 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.971 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.971 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.972 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.622642+0000", 2023-10-01T09:37:56.972 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015045100000000001 2023-10-01T09:37:56.972 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.972 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.972 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,4,7", 2023-10-01T09:37:56.972 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.622793+0000", 2023-10-01T09:37:56.972 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050188949999999998 2023-10-01T09:37:56.972 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.972 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.972 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.973 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.627811+0000", 2023-10-01T09:37:56.973 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002583288 2023-10-01T09:37:56.973 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.973 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.973 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.973 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.630395+0000", 2023-10-01T09:37:56.973 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00032070599999999998 2023-10-01T09:37:56.973 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.973 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.973 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.974 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.630715+0000", 2023-10-01T09:37:56.974 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0013941660000000001 2023-10-01T09:37:56.974 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.974 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.974 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.974 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.632110+0000", 2023-10-01T09:37:56.974 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2788999999999999e-05 2023-10-01T09:37:56.974 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.974 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.974 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.975 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.632162+0000", 2023-10-01T09:37:56.975 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000122216 2023-10-01T09:37:56.975 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.975 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.975 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.975 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:28.632285+0000", 2023-10-01T09:37:56.975 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2329211799999999 2023-10-01T09:37:56.975 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.975 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.975 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.976 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.976 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.976 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:161648 8.5 8:a4b0b645:::benchmark_data_smithi138_431617_object60237:head [delete] snapc 0=[] ondisk+write+known_if_redirected e364)", 2023-10-01T09:37:56.976 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:31.211170+0000", 2023-10-01T09:37:56.976 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 325.59316381600001, 2023-10-01T09:37:56.976 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17941784, 2023-10-01T09:37:56.976 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.976 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.976 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.977 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.977 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.977 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 161648 2023-10-01T09:37:56.977 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.977 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.977 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.977 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.977 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211170+0000", 2023-10-01T09:37:56.978 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.978 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.978 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.978 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.978 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211170+0000", 2023-10-01T09:37:56.978 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:37:56.978 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.978 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.978 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.978 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211168+0000", 2023-10-01T09:37:56.979 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4460000000000001e-06 2023-10-01T09:37:56.979 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.979 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.979 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.979 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211176+0000", 2023-10-01T09:37:56.979 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1656e-05 2023-10-01T09:37:56.979 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.979 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.979 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.979 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211188+0000", 2023-10-01T09:37:56.980 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6570000000000002e-06 2023-10-01T09:37:56.980 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.980 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.980 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.980 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211194+0000", 2023-10-01T09:37:56.980 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00048738600000000003 2023-10-01T09:37:56.980 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.980 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.980 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.980 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211681+0000", 2023-10-01T09:37:56.981 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000131089 2023-10-01T09:37:56.981 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.981 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.981 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.981 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211812+0000", 2023-10-01T09:37:56.981 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000138503 2023-10-01T09:37:56.981 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.981 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.981 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,4,7", 2023-10-01T09:37:56.981 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.211951+0000", 2023-10-01T09:37:56.982 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013556732 2023-10-01T09:37:56.982 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.982 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.982 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.982 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.225507+0000", 2023-10-01T09:37:56.982 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0031414619999999998 2023-10-01T09:37:56.982 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.982 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.982 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.982 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.228649+0000", 2023-10-01T09:37:56.983 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.009401487 2023-10-01T09:37:56.983 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.983 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.983 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.983 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.238050+0000", 2023-10-01T09:37:56.983 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15239425700000001 2023-10-01T09:37:56.983 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.983 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.983 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.983 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.390445+0000", 2023-10-01T09:37:56.983 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9906e-05 2023-10-01T09:37:56.984 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.984 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.984 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.984 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.390495+0000", 2023-10-01T09:37:56.984 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3560999999999994e-05 2023-10-01T09:37:56.984 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.984 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.984 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.984 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.390588+0000", 2023-10-01T09:37:56.984 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17941784 2023-10-01T09:37:56.985 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.985 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.985 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.985 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.985 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.985 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:161661 8.5 8:aec7fe18:::benchmark_data_smithi138_431617_object60250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e364)", 2023-10-01T09:37:56.985 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:31.232803+0000", 2023-10-01T09:37:56.985 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 325.57153144599999, 2023-10-01T09:37:56.985 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16878474500000001, 2023-10-01T09:37:56.985 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:37:56.986 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:37:56.986 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:37:56.986 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:37:56.986 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:37:56.986 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 161661 2023-10-01T09:37:56.986 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.986 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:37:56.986 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.986 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:37:56.987 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.232803+0000", 2023-10-01T09:37:56.987 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:37:56.987 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.987 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.987 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:37:56.987 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.232803+0000", 2023-10-01T09:37:56.987 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:37:56.987 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.987 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.988 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:37:56.988 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.232802+0000", 2023-10-01T09:37:56.988 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9800000000000003e-06 2023-10-01T09:37:56.988 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.988 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.988 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:37:56.988 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.232808+0000", 2023-10-01T09:37:56.988 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0929999999999992e-06 2023-10-01T09:37:56.988 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.988 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.989 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:37:56.989 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.232817+0000", 2023-10-01T09:37:56.989 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5669999999999996e-06 2023-10-01T09:37:56.989 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.989 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.989 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:37:56.989 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.232821+0000", 2023-10-01T09:37:56.989 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00095037599999999996 2023-10-01T09:37:56.989 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.989 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.990 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:37:56.990 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.233772+0000", 2023-10-01T09:37:56.990 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014877 2023-10-01T09:37:56.990 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.990 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.990 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:37:56.990 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.233920+0000", 2023-10-01T09:37:56.990 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00018008600000000001 2023-10-01T09:37:56.990 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.990 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.991 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,4,7", 2023-10-01T09:37:56.991 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.234100+0000", 2023-10-01T09:37:56.991 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0072926019999999996 2023-10-01T09:37:56.991 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.991 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.991 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.991 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.241393+0000", 2023-10-01T09:37:56.991 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2293000000000001e-05 2023-10-01T09:37:56.991 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.991 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.992 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:37:56.992 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.241405+0000", 2023-10-01T09:37:56.992 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001707376 2023-10-01T09:37:56.992 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.992 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.992 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.992 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.243113+0000", 2023-10-01T09:37:56.992 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15829549900000001 2023-10-01T09:37:56.992 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.992 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.993 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:37:56.993 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.401408+0000", 2023-10-01T09:37:56.993 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8941000000000003e-05 2023-10-01T09:37:56.993 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.993 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.993 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:37:56.993 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.401477+0000", 2023-10-01T09:37:56.993 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000110337 2023-10-01T09:37:56.993 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:37:56.993 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:37:56.994 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:37:56.994 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:31.401587+0000", 2023-10-01T09:37:56.994 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16878474500000001 2023-10-01T09:37:56.994 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.994 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.994 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.994 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:37:56.994 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:37:56.994 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:37:57.137 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:37:57.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:57.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:37:57.245 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:37:57.297 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:57.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:57 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 20756/132753 objects degraded (15.635%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:57.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:57 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 20756/132753 objects degraded (15.635%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:58.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:57 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 20756/132753 objects degraded (15.635%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:37:58.255 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:37:58.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:58.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:37:58.286 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:37:58.371 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:37:58.409 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:58.579 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:37:58.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:58.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:37:58.685 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:37:58.767 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:37:58.784 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:37:58.990 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:58 smithi138 ceph-mon[185886]: pgmap v1269: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 20756/131670 objects degraded (15.764%); 21621/131670 objects misplaced (16.421%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:37:58.990 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:58 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4110135238' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:58.990 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:58 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:58.990 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:58 smithi138 ceph-mon[192513]: pgmap v1269: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 20756/131670 objects degraded (15.764%); 21621/131670 objects misplaced (16.421%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:37:58.991 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:58 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4110135238' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:58.991 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:58 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:58.996 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:37:59.046 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:37:59.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:37:59.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:37:59.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:58 smithi190 ceph-mon[160435]: pgmap v1269: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 407 op/s; 20756/131670 objects degraded (15.764%); 21621/131670 objects misplaced (16.421%); 6.2 MiB/s, 99 objects/s recovering 2023-10-01T09:37:59.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:58 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4110135238' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:59.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:58 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:37:59.103 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:37:59.151 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:37:59.292 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:59 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:59 smithi190 ceph-mon[160435]: failed to encode map e451 with expected crc 2023-10-01T09:38:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:59 smithi190 ceph-mon[160435]: failed to encode map e451 with expected crc 2023-10-01T09:38:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:59 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:38:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:59 smithi190 ceph-mon[160435]: osdmap e451: 8 total, 8 up, 8 in 2023-10-01T09:38:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:37:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:00.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:00.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[185886]: failed to encode map e451 with expected crc 2023-10-01T09:38:00.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[185886]: failed to encode map e451 with expected crc 2023-10-01T09:38:00.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:38:00.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[185886]: osdmap e451: 8 total, 8 up, 8 in 2023-10-01T09:38:00.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:00.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:00.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:00.074 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:00.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:00.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[192513]: failed to encode map e451 with expected crc 2023-10-01T09:38:00.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[192513]: failed to encode map e451 with expected crc 2023-10-01T09:38:00.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:38:00.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[192513]: osdmap e451: 8 total, 8 up, 8 in 2023-10-01T09:38:00.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:00.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:00.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:00.075 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:37:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:00.091 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:00.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:00.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:00.199 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:00.235 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:00.294 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:00.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:00.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:00.360 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:00.387 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 2, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:38:00.387 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:38:00.387 INFO:tasks.thrashosds.thrasher:Reweighting osd 3 to 0.636452580824203 2023-10-01T09:38:00.387 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 0.636452580824203 2023-10-01T09:38:00.398 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:00.517 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:00.725 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:38:00.726 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:38:00.726 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:00.726 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:00 smithi138 ceph-mon[185886]: pgmap v1271: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 20388/126252 objects degraded (16.149%); 20796/126252 objects misplaced (16.472%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:00.740 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:00.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:00.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:00.848 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:00.972 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:01.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:38:01.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:38:01.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:01.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:00 smithi190 ceph-mon[160435]: pgmap v1271: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 20388/126252 objects degraded (16.149%); 20796/126252 objects misplaced (16.472%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:01.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:38:01.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:38:01.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:01.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:00 smithi138 ceph-mon[192513]: pgmap v1271: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 20388/126252 objects degraded (16.149%); 20796/126252 objects misplaced (16.472%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:01.377 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:38:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:38:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.46067.0:163705 9.9 e451/400 9:95d3a2a7:::benchmark_data_smithi138_497829_object61320:head v 451'10305, mlcod=451'10302)", 2023-10-01T09:38:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:38:01.368372+0000", 2023-10-01T09:38:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0078629790000000008, 2023-10-01T09:38:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078846780000000009, 2023-10-01T09:38:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:38:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:38:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:38:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:38:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.368372+0000", 2023-10-01T09:38:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:38:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:38:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.368372+0000", 2023-10-01T09:38:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:38:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:38:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.368372+0000", 2023-10-01T09:38:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3049999999999998e-06 2023-10-01T09:38:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:38:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.368378+0000", 2023-10-01T09:38:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0036e-05 2023-10-01T09:38:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:38:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.368388+0000", 2023-10-01T09:38:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9240000000000004e-06 2023-10-01T09:38:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:38:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.368394+0000", 2023-10-01T09:38:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2940000000000003e-05 2023-10-01T09:38:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:38:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.368467+0000", 2023-10-01T09:38:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8263000000000001e-05 2023-10-01T09:38:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:38:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.368485+0000", 2023-10-01T09:38:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011287 2023-10-01T09:38:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:38:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:38:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:38:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:163693 9.7 9:e68bf290:::benchmark_data_smithi138_497829_object61308:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-10-01T09:38:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:38:01.353990+0000", 2023-10-01T09:38:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.022244758, 2023-10-01T09:38:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.022312818000000002, 2023-10-01T09:38:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:38:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:38:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:38:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:38:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:38:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 163693 2023-10-01T09:38:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:38:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:38:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.353990+0000", 2023-10-01T09:38:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:38:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:38:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.353990+0000", 2023-10-01T09:38:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:38:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:38:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.353989+0000", 2023-10-01T09:38:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8100000000000003e-06 2023-10-01T09:38:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:38:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.353995+0000", 2023-10-01T09:38:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5944000000000001e-05 2023-10-01T09:38:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:38:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.354021+0000", 2023-10-01T09:38:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9609999999999996e-06 2023-10-01T09:38:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:38:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.354026+0000", 2023-10-01T09:38:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3483999999999999e-05 2023-10-01T09:38:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:38:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.354069+0000", 2023-10-01T09:38:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3252999999999999e-05 2023-10-01T09:38:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for degraded object", 2023-10-01T09:38:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.354133+0000", 2023-10-01T09:38:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015547444000000001 2023-10-01T09:38:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:38:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.369680+0000", 2023-10-01T09:38:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9599999999999995e-05 2023-10-01T09:38:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:38:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.369780+0000", 2023-10-01T09:38:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000175961 2023-10-01T09:38:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,2", 2023-10-01T09:38:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.369956+0000", 2023-10-01T09:38:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015965348000000001 2023-10-01T09:38:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:38:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:38:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:38:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:163697 9.d 9:bcfacb01:::benchmark_data_smithi138_497829_object61312:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-10-01T09:38:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:38:01.359574+0000", 2023-10-01T09:38:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.016661189, 2023-10-01T09:38:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016808035999999998, 2023-10-01T09:38:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:38:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:38:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:38:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:38:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:38:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 163697 2023-10-01T09:38:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:38:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:38:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359574+0000", 2023-10-01T09:38:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:38:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:38:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359574+0000", 2023-10-01T09:38:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:38:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:38:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359572+0000", 2023-10-01T09:38:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7010000000000001e-06 2023-10-01T09:38:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:38:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359578+0000", 2023-10-01T09:38:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2391e-05 2023-10-01T09:38:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:38:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359591+0000", 2023-10-01T09:38:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4879999999999998e-06 2023-10-01T09:38:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:38:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359596+0000", 2023-10-01T09:38:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5670000000000002e-05 2023-10-01T09:38:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:38:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359642+0000", 2023-10-01T09:38:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016055900000000001 2023-10-01T09:38:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:38:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359802+0000", 2023-10-01T09:38:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015457399999999999 2023-10-01T09:38:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,4,6,7", 2023-10-01T09:38:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359957+0000", 2023-10-01T09:38:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038793970000000001 2023-10-01T09:38:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:38:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.363836+0000", 2023-10-01T09:38:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0044270819999999997 2023-10-01T09:38:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.368263+0000", 2023-10-01T09:38:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0086893869999999998 2023-10-01T09:38:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:38:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:38:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:38:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:163698 9.6 9:6b9542fc:::benchmark_data_smithi138_497829_object61313:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-10-01T09:38:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:38:01.359610+0000", 2023-10-01T09:38:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.016625282000000002, 2023-10-01T09:38:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016844616999999999, 2023-10-01T09:38:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:38:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:38:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:38:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:38:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:38:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 163698 2023-10-01T09:38:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:38:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:38:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359610+0000", 2023-10-01T09:38:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:38:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:38:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359610+0000", 2023-10-01T09:38:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:38:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:38:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359609+0000", 2023-10-01T09:38:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7549999999999999e-06 2023-10-01T09:38:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:38:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359612+0000", 2023-10-01T09:38:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7540000000000003e-06 2023-10-01T09:38:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:38:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359618+0000", 2023-10-01T09:38:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9149999999999998e-06 2023-10-01T09:38:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:38:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359622+0000", 2023-10-01T09:38:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7725999999999997e-05 2023-10-01T09:38:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:38:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359670+0000", 2023-10-01T09:38:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012005100000000001 2023-10-01T09:38:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:38:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359790+0000", 2023-10-01T09:38:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016705600000000001 2023-10-01T09:38:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,2", 2023-10-01T09:38:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.359957+0000", 2023-10-01T09:38:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0040075149999999997 2023-10-01T09:38:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:38:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.363964+0000", 2023-10-01T09:38:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011971304 2023-10-01T09:38:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.375936+0000", 2023-10-01T09:38:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016325682000000001 2023-10-01T09:38:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:38:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:38:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:38:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:163704 9.f 9:fddf4ea3:::benchmark_data_smithi138_497829_object61319:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-10-01T09:38:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:38:01.366517+0000", 2023-10-01T09:38:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0097177029999999994, 2023-10-01T09:38:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0099875450000000005, 2023-10-01T09:38:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:38:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:38:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:38:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:38:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:38:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 163704 2023-10-01T09:38:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:38:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:38:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.366517+0000", 2023-10-01T09:38:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:38:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:38:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.366517+0000", 2023-10-01T09:38:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:38:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:38:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.366516+0000", 2023-10-01T09:38:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.559e-06 2023-10-01T09:38:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:38:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.366523+0000", 2023-10-01T09:38:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1419e-05 2023-10-01T09:38:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:38:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.366535+0000", 2023-10-01T09:38:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5019999999999998e-06 2023-10-01T09:38:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:38:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.366541+0000", 2023-10-01T09:38:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2483e-05 2023-10-01T09:38:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:38:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.366584+0000", 2023-10-01T09:38:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000133654 2023-10-01T09:38:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:38:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.366718+0000", 2023-10-01T09:38:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000133843 2023-10-01T09:38:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:38:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.366851+0000", 2023-10-01T09:38:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003714831 2023-10-01T09:38:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:38:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:38:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:38:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:38:01.370566+0000", 2023-10-01T09:38:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0040488019999999998 2023-10-01T09:38:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:38:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:38:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:38:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:38:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:38:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 5 2023-10-01T09:38:01.407 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:38:01.423 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:01.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:01.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:01.537 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:01.593 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:01.669 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:01.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:01.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:01.774 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:01.818 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:01.877 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:01.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:01.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:01.981 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:02.104 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:02.204 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:38:02.247 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:02.304 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:02.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:02.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:02.413 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:02.450 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:02.533 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:02.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:02.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:02.544 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T09:38:02.645 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:02.727 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:03.112 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:38:03.244 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:03 smithi138 ceph-mon[192513]: pgmap v1272: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 430 op/s; 20388/125481 objects degraded (16.248%); 20796/125481 objects misplaced (16.573%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:03.244 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:03 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 20388/125481 objects degraded (16.248%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:03.507 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:03 smithi190 ceph-mon[160435]: pgmap v1272: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 430 op/s; 20388/125481 objects degraded (16.248%); 20796/125481 objects misplaced (16.573%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:03.507 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:03 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 20388/125481 objects degraded (16.248%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:03.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:03 smithi138 ceph-mon[185886]: pgmap v1272: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 430 op/s; 20388/125481 objects degraded (16.248%); 20796/125481 objects misplaced (16.573%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:03.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:03 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 20388/125481 objects degraded (16.248%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:03.557 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:03.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:03.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:03.580 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:38:03.618 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:38:03.666 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:03.754 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:03.838 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:38:03.839 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:38:03.839 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:38:03.839 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:38:03.839 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:38:03.849 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:38:03.854 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:03.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:03.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:03.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:38:03.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:38:03.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:38:03.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:38:03.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:38:03.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:38:03.865 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:38:03.866 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:38:03.960 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:04.000 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:04.080 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:04.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:04.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:04.186 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:04.257 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2272944404' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:04.262 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2272944404' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:04.267 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:04.443 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:04.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:04.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:04.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2272944404' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:04.558 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:04.635 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:05.156 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:05.160 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:38:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:38:04] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.33.4" 2023-10-01T09:38:05.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:05.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:05.179 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:38:05.179 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:38:05.267 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:05.395 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:05.449 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:05 smithi138 ceph-mon[185886]: pgmap v1273: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 20388/125034 objects degraded (16.306%); 20796/125034 objects misplaced (16.632%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:05.449 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:05 smithi138 ceph-mon[192513]: pgmap v1273: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 20388/125034 objects degraded (16.306%); 20796/125034 objects misplaced (16.632%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:05.534 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:05.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:05.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:05.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:05 smithi190 ceph-mon[160435]: pgmap v1273: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 20388/125034 objects degraded (16.306%); 20796/125034 objects misplaced (16.632%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:05.642 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:38:05.658 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:05.772 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:05.985 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:38:06.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:06.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:38:06.110 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:38:06.228 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:06.262 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T09:38:06.298 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:06 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3587073362' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.636452580824203}]: dispatch 2023-10-01T09:38:06.358 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.3 to 0.636453 (a2ee) 2023-10-01T09:38:06.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:06 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3587073362' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.636452580824203}]: dispatch 2023-10-01T09:38:06.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:06 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3587073362' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.636452580824203}]: dispatch 2023-10-01T09:38:06.736 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:38:06.821 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:38:06.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:06.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:38:06.924 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:06.955 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:07.001 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:07.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:07.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:07.104 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:07.132 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:07.177 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:07.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:07.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:07.213 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:38:07.281 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:07.286 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:07 smithi190 ceph-mon[160435]: pgmap v1274: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 19981/119499 objects degraded (16.721%); 19915/119499 objects misplaced (16.665%); 6.6 MiB/s, 105 objects/s recovering 2023-10-01T09:38:07.313 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:07.389 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:07.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:07.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:07.493 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:07.522 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:07.539 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:07 smithi138 ceph-mon[185886]: pgmap v1274: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 19981/119499 objects degraded (16.721%); 19915/119499 objects misplaced (16.665%); 6.6 MiB/s, 105 objects/s recovering 2023-10-01T09:38:07.539 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:07 smithi138 ceph-mon[185886]: failed to encode map e452 with expected crc 2023-10-01T09:38:07.539 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:07 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3587073362' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.636452580824203}]': finished 2023-10-01T09:38:07.539 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:07 smithi138 ceph-mon[185886]: osdmap e452: 8 total, 8 up, 8 in 2023-10-01T09:38:07.539 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:07 smithi138 ceph-mon[185886]: failed to encode map e452 with expected crc 2023-10-01T09:38:07.539 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:07 smithi138 ceph-mon[192513]: pgmap v1274: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 441 op/s; 19981/119499 objects degraded (16.721%); 19915/119499 objects misplaced (16.665%); 6.6 MiB/s, 105 objects/s recovering 2023-10-01T09:38:07.540 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:07 smithi138 ceph-mon[192513]: failed to encode map e452 with expected crc 2023-10-01T09:38:07.540 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:07 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3587073362' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.636452580824203}]': finished 2023-10-01T09:38:07.540 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:07 smithi138 ceph-mon[192513]: osdmap e452: 8 total, 8 up, 8 in 2023-10-01T09:38:07.540 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:07 smithi138 ceph-mon[192513]: failed to encode map e452 with expected crc 2023-10-01T09:38:07.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:07 smithi190 ceph-mon[160435]: failed to encode map e452 with expected crc 2023-10-01T09:38:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:07 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3587073362' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.636452580824203}]': finished 2023-10-01T09:38:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:07 smithi190 ceph-mon[160435]: osdmap e452: 8 total, 8 up, 8 in 2023-10-01T09:38:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:07 smithi190 ceph-mon[160435]: failed to encode map e452 with expected crc 2023-10-01T09:38:07.566 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:07.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:07.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:07.673 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:07.700 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:08.202 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:08.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:08.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:08.306 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:08.333 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:08 smithi190 ceph-mon[160435]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:08 smithi190 ceph-mon[160435]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:08 smithi190 ceph-mon[160435]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:08 smithi190 ceph-mon[160435]: osdmap e453: 8 total, 8 up, 8 in 2023-10-01T09:38:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:08 smithi190 ceph-mon[160435]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:08 smithi190 ceph-mon[160435]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:08 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 19981/119499 objects degraded (16.721%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:08 smithi190 ceph-mon[160435]: pgmap v1277: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 19981/118230 objects degraded (16.900%); 19915/118230 objects misplaced (16.844%); 4.2 MiB/s, 67 objects/s recovering 2023-10-01T09:38:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[185886]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[185886]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[185886]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[185886]: osdmap e453: 8 total, 8 up, 8 in 2023-10-01T09:38:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[185886]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[185886]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 19981/119499 objects degraded (16.721%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[185886]: pgmap v1277: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 19981/118230 objects degraded (16.900%); 19915/118230 objects misplaced (16.844%); 4.2 MiB/s, 67 objects/s recovering 2023-10-01T09:38:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[192513]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[192513]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[192513]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[192513]: osdmap e453: 8 total, 8 up, 8 in 2023-10-01T09:38:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[192513]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[192513]: failed to encode map e453 with expected crc 2023-10-01T09:38:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 19981/119499 objects degraded (16.721%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:08.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:08 smithi138 ceph-mon[192513]: pgmap v1277: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 19981/118230 objects degraded (16.900%); 19915/118230 objects misplaced (16.844%); 4.2 MiB/s, 67 objects/s recovering 2023-10-01T09:38:08.680 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:08.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:08.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:08.757 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:38:08.757 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:38:08.784 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:08.815 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:08.852 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:08.994 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:08.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:08.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:09.099 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:09.136 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:09.193 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:09.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:09.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:09.300 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:09.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:09 smithi138 ceph-mon[185886]: failed to encode map e454 with expected crc 2023-10-01T09:38:09.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:09 smithi138 ceph-mon[185886]: failed to encode map e454 with expected crc 2023-10-01T09:38:09.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:09 smithi138 ceph-mon[185886]: osdmap e454: 8 total, 8 up, 8 in 2023-10-01T09:38:09.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:09 smithi138 ceph-mon[185886]: 9.a starting backfill to osd.6 from (407'2207,444'9105] MIN to 451'10219 2023-10-01T09:38:09.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:09 smithi138 ceph-mon[185886]: failed to encode map e454 with expected crc 2023-10-01T09:38:09.370 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:09.554 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:09.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:09 smithi190 ceph-mon[160435]: failed to encode map e454 with expected crc 2023-10-01T09:38:09.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:09 smithi190 ceph-mon[160435]: failed to encode map e454 with expected crc 2023-10-01T09:38:09.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:09 smithi190 ceph-mon[160435]: osdmap e454: 8 total, 8 up, 8 in 2023-10-01T09:38:09.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:09 smithi190 ceph-mon[160435]: 9.a starting backfill to osd.6 from (407'2207,444'9105] MIN to 451'10219 2023-10-01T09:38:09.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:09 smithi190 ceph-mon[160435]: failed to encode map e454 with expected crc 2023-10-01T09:38:09.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:09 smithi138 ceph-mon[192513]: failed to encode map e454 with expected crc 2023-10-01T09:38:09.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:09 smithi138 ceph-mon[192513]: failed to encode map e454 with expected crc 2023-10-01T09:38:09.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:09 smithi138 ceph-mon[192513]: osdmap e454: 8 total, 8 up, 8 in 2023-10-01T09:38:09.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:09 smithi138 ceph-mon[192513]: 9.a starting backfill to osd.6 from (407'2207,444'9105] MIN to 451'10219 2023-10-01T09:38:09.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:09 smithi138 ceph-mon[192513]: failed to encode map e454 with expected crc 2023-10-01T09:38:09.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:09.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:09.668 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:09.697 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:38:09.765 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:09.950 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:09.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:09.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:10.070 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:10.115 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:10.183 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:10.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:10.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:10.298 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:38:10.339 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:10.380 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:38:10.397 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:38:10.404 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:10 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1502344190' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:38:10.404 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:10 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:38:10.405 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:10 smithi190 ceph-mon[160435]: pgmap v1279: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 17305/115278 objects degraded (15.012%); 19229/115278 objects misplaced (16.681%); 11 MiB/s, 175 objects/s recovering 2023-10-01T09:38:10.405 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:38:10.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:10.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:38:10.506 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:10.542 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:10.582 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:10.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:10.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:10.689 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:10.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:10 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1502344190' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:38:10.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:10 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:38:10.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:10 smithi138 ceph-mon[192513]: pgmap v1279: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 17305/115278 objects degraded (15.012%); 19229/115278 objects misplaced (16.681%); 11 MiB/s, 175 objects/s recovering 2023-10-01T09:38:10.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:38:10.714 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:10 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1502344190' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:38:10.714 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:10 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:38:10.714 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:10 smithi138 ceph-mon[185886]: pgmap v1279: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 17305/115278 objects degraded (15.012%); 19229/115278 objects misplaced (16.681%); 11 MiB/s, 175 objects/s recovering 2023-10-01T09:38:10.715 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:38:10.716 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:11.199 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:11.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:11.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:11.307 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:11.353 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:11.576 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:11 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:11.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:11 smithi138 ceph-mon[185886]: failed to encode map e455 with expected crc 2023-10-01T09:38:11.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:11 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:38:11.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:11 smithi138 ceph-mon[185886]: osdmap e455: 8 total, 8 up, 8 in 2023-10-01T09:38:11.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:11 smithi138 ceph-mon[185886]: failed to encode map e455 with expected crc 2023-10-01T09:38:11.577 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:11 smithi138 ceph-mon[185886]: failed to encode map e455 with expected crc 2023-10-01T09:38:11.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:11 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:11.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:11 smithi138 ceph-mon[192513]: failed to encode map e455 with expected crc 2023-10-01T09:38:11.577 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:11 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:38:11.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:11 smithi138 ceph-mon[192513]: osdmap e455: 8 total, 8 up, 8 in 2023-10-01T09:38:11.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:11 smithi138 ceph-mon[192513]: failed to encode map e455 with expected crc 2023-10-01T09:38:11.578 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:11 smithi138 ceph-mon[192513]: failed to encode map e455 with expected crc 2023-10-01T09:38:11.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:11 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:11 smithi190 ceph-mon[160435]: failed to encode map e455 with expected crc 2023-10-01T09:38:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:11 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:38:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:11 smithi190 ceph-mon[160435]: osdmap e455: 8 total, 8 up, 8 in 2023-10-01T09:38:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:11 smithi190 ceph-mon[160435]: failed to encode map e455 with expected crc 2023-10-01T09:38:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:11 smithi190 ceph-mon[160435]: failed to encode map e455 with expected crc 2023-10-01T09:38:11.938 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:11.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:11.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:12.052 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:38:12.128 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:12.138 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:12.323 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:38:12.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:12.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:38:12.435 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:12.468 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:12.534 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:12 smithi138 ceph-mon[185886]: pgmap v1281: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 216 op/s; 17305/114780 objects degraded (15.077%); 19229/114780 objects misplaced (16.753%); 5.5 MiB/s, 88 objects/s recovering 2023-10-01T09:38:12.536 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:12.538 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:12 smithi138 ceph-mon[192513]: pgmap v1281: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 216 op/s; 17305/114780 objects degraded (15.077%); 19229/114780 objects misplaced (16.753%); 5.5 MiB/s, 88 objects/s recovering 2023-10-01T09:38:12.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:12.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:12.641 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:12.674 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:12 smithi190 ceph-mon[160435]: pgmap v1281: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 216 op/s; 17305/114780 objects degraded (15.077%); 19229/114780 objects misplaced (16.753%); 5.5 MiB/s, 88 objects/s recovering 2023-10-01T09:38:12.677 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:38:12.677 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:12.724 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:38:12.747 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:12.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:12.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:12.763 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:38:12.804 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:38:12.846 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:38:12.855 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:12.886 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:38:12.936 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:38:12.951 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:12.976 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:38:12.996 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:38:12.997 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:38:12.997 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:38:12.997 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:38:12.997 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:38:12.997 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:38:12.997 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:38:12.997 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:38:13.070 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:13.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:13.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:13.177 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:13.292 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:13.627 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 2, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:38:13.628 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:38:13.628 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-01T09:38:13.628 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:38:13.652 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:13 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3863301346' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:13.652 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:13 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 17305/114780 objects degraded (15.077%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:13.652 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:13 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3863301346' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:13.652 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:13 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 17305/114780 objects degraded (15.077%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:13.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:13 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3863301346' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:13.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:13 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 17305/114780 objects degraded (15.077%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:13.949 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:38:14.588 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:14.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:14.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:14.694 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:14.728 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:14 smithi190 ceph-mon[160435]: pgmap v1282: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 17305/114546 objects degraded (15.107%); 19229/114546 objects misplaced (16.787%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T09:38:14.730 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:14.789 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:14.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:14.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:14 smithi138 ceph-mon[192513]: pgmap v1282: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 17305/114546 objects degraded (15.107%); 19229/114546 objects misplaced (16.787%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T09:38:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:14 smithi138 ceph-mon[185886]: pgmap v1282: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 17305/114546 objects degraded (15.107%); 19229/114546 objects misplaced (16.787%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T09:38:14.897 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:38:15.004 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:15.229 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:38:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:38:14] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.33.4" 2023-10-01T09:38:15.229 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:38:15.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:15.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:38:15.336 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:15.459 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:15.493 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:15.502 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:15.613 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:38:15.613 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:38:15.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.613 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:168840 9.8 9:1ed142be:::benchmark_data_smithi138_497829_object66455:head [delete] snapc 0=[] ondisk+write+known_if_redirected e455)", 2023-10-01T09:38:15.614 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:38:15.567412+0000", 2023-10-01T09:38:15.614 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.042540486000000002, 2023-10-01T09:38:15.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.042574855000000002, 2023-10-01T09:38:15.614 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:15.614 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:38:15.614 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:38:15.614 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:38:15.614 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:38:15.614 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 168840 2023-10-01T09:38:15.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.614 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:15.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:15.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.567412+0000", 2023-10-01T09:38:15.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:15.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:15.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.567412+0000", 2023-10-01T09:38:15.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:15.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:15.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.567409+0000", 2023-10-01T09:38:15.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:38:15.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:15.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.567423+0000", 2023-10-01T09:38:15.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3774000000000001e-05 2023-10-01T09:38:15.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:15.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.567444+0000", 2023-10-01T09:38:15.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1089000000000001e-05 2023-10-01T09:38:15.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:15.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.567456+0000", 2023-10-01T09:38:15.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2343e-05 2023-10-01T09:38:15.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:15.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.572406+0000", 2023-10-01T09:38:15.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049493599999999999 2023-10-01T09:38:15.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:38:15.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.574467+0000", 2023-10-01T09:38:15.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020607289999999999 2023-10-01T09:38:15.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:15.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.605244+0000", 2023-10-01T09:38:15.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0307779 2023-10-01T09:38:15.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:38:15.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.605348+0000", 2023-10-01T09:38:15.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010309 2023-10-01T09:38:15.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,7", 2023-10-01T09:38:15.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.605505+0000", 2023-10-01T09:38:15.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000157398 2023-10-01T09:38:15.619 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:15.619 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:15.619 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:15.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.619 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:168844 9.0 e455/431 9:0cd10089:::benchmark_data_smithi138_497829_object66459:head v 455'10490, mlcod=407'2555)", 2023-10-01T09:38:15.619 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:38:15.577714+0000", 2023-10-01T09:38:15.619 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.032238271999999998, 2023-10-01T09:38:15.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.032364914000000002, 2023-10-01T09:38:15.620 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:15.620 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:38:15.620 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:15.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:15.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.577714+0000", 2023-10-01T09:38:15.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:15.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:15.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.577714+0000", 2023-10-01T09:38:15.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:15.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:15.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.577714+0000", 2023-10-01T09:38:15.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967296 2023-10-01T09:38:15.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:15.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.577724+0000", 2023-10-01T09:38:15.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0071000000000001e-05 2023-10-01T09:38:15.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:15.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.577731+0000", 2023-10-01T09:38:15.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1779999999999997e-06 2023-10-01T09:38:15.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:15.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.577735+0000", 2023-10-01T09:38:15.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1819999999999999e-06 2023-10-01T09:38:15.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:15.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.584985+0000", 2023-10-01T09:38:15.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0072503910000000001 2023-10-01T09:38:15.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:38:15.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.585014+0000", 2023-10-01T09:38:15.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8603e-05 2023-10-01T09:38:15.623 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:15.623 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:15.623 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:15.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.623 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:168849 9.9 9:902018ab:::benchmark_data_smithi138_497829_object66464:head [delete] snapc 0=[] ondisk+write+known_if_redirected e455)", 2023-10-01T09:38:15.623 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:38:15.596419+0000", 2023-10-01T09:38:15.624 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.013533442, 2023-10-01T09:38:15.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013720853, 2023-10-01T09:38:15.624 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:15.624 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:38:15.624 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:38:15.624 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:38:15.624 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:38:15.624 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 168849 2023-10-01T09:38:15.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.624 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:15.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:15.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.596419+0000", 2023-10-01T09:38:15.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:15.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:15.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.596419+0000", 2023-10-01T09:38:15.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:15.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:15.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.596416+0000", 2023-10-01T09:38:15.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:38:15.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:15.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.596427+0000", 2023-10-01T09:38:15.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0197000000000001e-05 2023-10-01T09:38:15.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:15.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.596444+0000", 2023-10-01T09:38:15.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7178000000000001e-05 2023-10-01T09:38:15.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:15.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.596454+0000", 2023-10-01T09:38:15.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0233e-05 2023-10-01T09:38:15.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:15.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.596524+0000", 2023-10-01T09:38:15.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0117999999999995e-05 2023-10-01T09:38:15.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:38:15.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.596680+0000", 2023-10-01T09:38:15.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000155499 2023-10-01T09:38:15.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:38:15.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.596809+0000", 2023-10-01T09:38:15.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000129787 2023-10-01T09:38:15.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:38:15.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.604448+0000", 2023-10-01T09:38:15.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0076386079999999999 2023-10-01T09:38:15.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:15.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.605136+0000", 2023-10-01T09:38:15.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00068824900000000004 2023-10-01T09:38:15.628 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:15.628 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:15.629 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:15.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.629 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:168851 9.5 9.bf52f435 (undecoded) ondisk+write+known_if_redirected e455)", 2023-10-01T09:38:15.629 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:38:15.607682+0000", 2023-10-01T09:38:15.629 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.002269592, 2023-10-01T09:38:15.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025437849999999998, 2023-10-01T09:38:15.629 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:15.629 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:38:15.629 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:38:15.629 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:38:15.629 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:38:15.629 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 168851 2023-10-01T09:38:15.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.630 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:15.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:15.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.607682+0000", 2023-10-01T09:38:15.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:15.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:15.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.607682+0000", 2023-10-01T09:38:15.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:15.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:15.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.607680+0000", 2023-10-01T09:38:15.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:38:15.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:15.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.607690+0000", 2023-10-01T09:38:15.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6900000000000004e-06 2023-10-01T09:38:15.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:15.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.607707+0000", 2023-10-01T09:38:15.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7220000000000001e-05 2023-10-01T09:38:15.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:15.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.607717+0000", 2023-10-01T09:38:15.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4020000000000008e-06 2023-10-01T09:38:15.632 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:15.632 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:15.632 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:15.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.632 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:168826 9.8 9:1dc2babe:::benchmark_data_smithi138_497829_object66441:head [delete] snapc 0=[] ondisk+write+known_if_redirected e455)", 2023-10-01T09:38:15.633 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:38:15.526822+0000", 2023-10-01T09:38:15.633 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.083129935000000002, 2023-10-01T09:38:15.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.083459456000000001, 2023-10-01T09:38:15.633 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:15.633 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:38:15.633 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:38:15.633 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:38:15.633 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:38:15.633 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 168826 2023-10-01T09:38:15.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.633 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:15.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:15.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.526822+0000", 2023-10-01T09:38:15.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:15.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:15.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.526822+0000", 2023-10-01T09:38:15.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:15.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:15.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.526820+0000", 2023-10-01T09:38:15.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:38:15.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:15.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.526829+0000", 2023-10-01T09:38:15.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3590000000000008e-06 2023-10-01T09:38:15.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:15.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.526841+0000", 2023-10-01T09:38:15.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.26e-05 2023-10-01T09:38:15.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:15.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.526849+0000", 2023-10-01T09:38:15.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9189999999999999e-06 2023-10-01T09:38:15.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:15.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.530363+0000", 2023-10-01T09:38:15.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003513207 2023-10-01T09:38:15.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:38:15.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.531228+0000", 2023-10-01T09:38:15.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00086530199999999998 2023-10-01T09:38:15.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:15.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.609086+0000", 2023-10-01T09:38:15.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.077858004999999994 2023-10-01T09:38:15.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:38:15.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.609182+0000", 2023-10-01T09:38:15.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5976000000000004e-05 2023-10-01T09:38:15.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:15.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:15.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 1,7", 2023-10-01T09:38:15.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:15.609440+0000", 2023-10-01T09:38:15.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000258552 2023-10-01T09:38:15.638 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:15.638 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:15.638 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:15.638 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:15.638 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:38:15.638 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 5 2023-10-01T09:38:15.638 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:38:15.639 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:15.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:15.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:15.756 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:15.839 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:16.177 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:16.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:16.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:16.284 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:16.331 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:16.422 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:16.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:16.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:16.527 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:16.554 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:17.060 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:38:17.249 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:38:17.381 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:17 smithi138 ceph-mon[185886]: pgmap v1283: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 16950/109671 objects degraded (15.455%); 18568/109671 objects misplaced (16.931%); 7.5 MiB/s, 119 objects/s recovering 2023-10-01T09:38:17.383 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:17 smithi138 ceph-mon[192513]: pgmap v1283: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 16950/109671 objects degraded (15.455%); 18568/109671 objects misplaced (16.931%); 7.5 MiB/s, 119 objects/s recovering 2023-10-01T09:38:17.497 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:17.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:17.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:17.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:17 smithi190 ceph-mon[160435]: pgmap v1283: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 16950/109671 objects degraded (15.455%); 18568/109671 objects misplaced (16.931%); 7.5 MiB/s, 119 objects/s recovering 2023-10-01T09:38:17.606 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:38:17.635 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:17.684 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:38:17.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:17.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:38:17.789 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:17.864 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:18.196 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:18.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:18.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:18.302 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:18.410 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:18.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:18 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 16950/109671 objects degraded (15.455%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:18.574 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:18 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 16950/109671 objects degraded (15.455%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:18.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:18 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 16950/109671 objects degraded (15.455%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:18.649 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:18.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:18.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:18.774 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:38:18.887 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:18.935 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:38:18.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:18.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:38:19.039 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:38:19.073 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:19.142 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:38:19.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:19.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:38:19.248 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:19.274 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:19.327 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:19.337 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:19 smithi190 ceph-mon[160435]: pgmap v1284: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 328 op/s; 16950/108504 objects degraded (15.622%); 18568/108504 objects misplaced (17.113%); 6.0 MiB/s, 96 objects/s recovering 2023-10-01T09:38:19.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:19.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:19.438 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:19.527 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:19.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:19 smithi138 ceph-mon[192513]: pgmap v1284: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 328 op/s; 16950/108504 objects degraded (15.622%); 18568/108504 objects misplaced (17.113%); 6.0 MiB/s, 96 objects/s recovering 2023-10-01T09:38:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:19 smithi138 ceph-mon[185886]: pgmap v1284: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 328 op/s; 16950/108504 objects degraded (15.622%); 18568/108504 objects misplaced (17.113%); 6.0 MiB/s, 96 objects/s recovering 2023-10-01T09:38:19.598 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:19.641 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:19.734 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:19.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:19.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:19.845 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:19.880 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:19.946 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:19.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:19.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:20.057 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:20.095 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:20.156 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:20.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:20.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:20.262 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:38:20.364 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:20.481 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:38:20.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:20.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:38:20.598 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:38:20.746 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:21.285 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:38:21.292 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:38:21.370 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:38:21.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:21 smithi138 ceph-mon[192513]: pgmap v1285: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 16575/103521 objects degraded (16.011%); 17816/103521 objects misplaced (17.210%); 5.8 MiB/s, 92 objects/s recovering 2023-10-01T09:38:21.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:21 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1047822162' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:38:21.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:21 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:38:21.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:21 smithi138 ceph-mon[185886]: pgmap v1285: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 16575/103521 objects degraded (16.011%); 17816/103521 objects misplaced (17.210%); 5.8 MiB/s, 92 objects/s recovering 2023-10-01T09:38:21.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:21 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1047822162' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:38:21.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:21 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:38:21.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:21 smithi190 ceph-mon[160435]: pgmap v1285: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 391 op/s; 16575/103521 objects degraded (16.011%); 17816/103521 objects misplaced (17.210%); 5.8 MiB/s, 92 objects/s recovering 2023-10-01T09:38:21.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:21 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1047822162' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:38:21.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:21 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:38:22.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:22 smithi138 ceph-mon[185886]: from='client.46337 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:38:22.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:22 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:38:22.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:22 smithi138 ceph-mon[185886]: failed to encode map e456 with expected crc 2023-10-01T09:38:22.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:22 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:38:22.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:22 smithi138 ceph-mon[185886]: osdmap e456: 8 total, 8 up, 8 in 2023-10-01T09:38:22.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:22 smithi138 ceph-mon[185886]: pgmap v1287: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 16575/102816 objects degraded (16.121%); 17816/102816 objects misplaced (17.328%); 5.8 MiB/s, 92 objects/s recovering 2023-10-01T09:38:22.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:22 smithi138 ceph-mon[192513]: from='client.46337 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:38:22.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:22 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:38:22.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:22 smithi138 ceph-mon[192513]: failed to encode map e456 with expected crc 2023-10-01T09:38:22.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:22 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:38:22.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:22 smithi138 ceph-mon[192513]: osdmap e456: 8 total, 8 up, 8 in 2023-10-01T09:38:22.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:22 smithi138 ceph-mon[192513]: pgmap v1287: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 16575/102816 objects degraded (16.121%); 17816/102816 objects misplaced (17.328%); 5.8 MiB/s, 92 objects/s recovering 2023-10-01T09:38:22.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:22 smithi190 ceph-mon[160435]: from='client.46337 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:38:22.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:22 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:38:22.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:22 smithi190 ceph-mon[160435]: failed to encode map e456 with expected crc 2023-10-01T09:38:22.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:22 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:38:22.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:22 smithi190 ceph-mon[160435]: osdmap e456: 8 total, 8 up, 8 in 2023-10-01T09:38:22.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:22 smithi190 ceph-mon[160435]: pgmap v1287: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 16575/102816 objects degraded (16.121%); 17816/102816 objects misplaced (17.328%); 5.8 MiB/s, 92 objects/s recovering 2023-10-01T09:38:22.931 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:38:22.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:22.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:38:23.049 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:23.244 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:23.500 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:23.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:23.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:23.523 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:23 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 16575/102816 objects degraded (16.121%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:23.523 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:23 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 16575/102816 objects degraded (16.121%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:23.624 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:23.688 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:23.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:23 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 16575/102816 objects degraded (16.121%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:23.996 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:24.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:24.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:24.039 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1285,"stamp":"2023-10-01T09:38:20.188257+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2264340427,"num_objects":34507,"num_object_clones":0,"num_object_copies":103521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16575,"num_objects_misplaced":17816,"num_objects_unfound":0,"num_objects_dirty":34507,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":171393,"num_write_kb":6567590,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17861,"num_bytes_recovered":1157075377,"num_keys_recovered":90,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121115,"ondisk_log_size":121115,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13673428,"kb_used_data":8230932,"kb_used_omap":50,"kb_used_meta":5442189,"kb_avail":736287788,"statfs":{"total":767960285184,"available":753958694912,"internally_reserved":0,"allocated":8428474368,"data_stored":8417418207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51738,"internal_metadata":5572802022},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,6,0,8,6,5,6],"upper_bound":128},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-256835584,"num_objects":-3919,"num_object_clones":0,"num_object_copies":-11757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-730,"num_objects_misplaced":-1413,"num_objects_unfound":0,"num_objects_dirty":-3919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3919,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":926,"num_bytes_recovered":60686336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.011062"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":367,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.458123+0000","last_change":"2023-10-01T09:35:34.561455+0000","last_active":"2023-10-01T09:38:08.458123+0000","last_peered":"2023-10-01T09:38:08.458123+0000","last_clean":"2023-10-01T09:38:08.458123+0000","last_became_active":"2023-10-01T09:35:34.560978+0000","last_became_peered":"2023-10-01T09:35:34.560978+0000","last_unstale":"2023-10-01T09:38:08.458123+0000","last_undegraded":"2023-10-01T09:38:08.458123+0000","last_fullsized":"2023-10-01T09:38:08.458123+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:02:58.965198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.454502+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:38:08.454502+0000","last_peered":"2023-10-01T09:38:08.454502+0000","last_clean":"2023-10-01T09:38:08.454502+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:38:08.454502+0000","last_undegraded":"2023-10-01T09:38:08.454502+0000","last_fullsized":"2023-10-01T09:38:08.454502+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":452,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.315733+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:38:08.315733+0000","last_peered":"2023-10-01T09:38:08.315733+0000","last_clean":"2023-10-01T09:38:08.315733+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:38:08.315733+0000","last_undegraded":"2023-10-01T09:38:08.315733+0000","last_fullsized":"2023-10-01T09:38:08.315733+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":492,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.455121+0000","last_change":"2023-10-01T09:36:22.227558+0000","last_active":"2023-10-01T09:38:08.455121+0000","last_peered":"2023-10-01T09:38:08.455121+0000","last_clean":"2023-10-01T09:38:08.455121+0000","last_became_active":"2023-10-01T09:36:22.226550+0000","last_became_peered":"2023-10-01T09:36:22.226550+0000","last_unstale":"2023-10-01T09:38:08.455121+0000","last_undegraded":"2023-10-01T09:38:08.455121+0000","last_fullsized":"2023-10-01T09:38:08.455121+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:35:45.988649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":374,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313437+0000","last_change":"2023-10-01T09:37:22.672668+0000","last_active":"2023-10-01T09:38:08.313437+0000","last_peered":"2023-10-01T09:38:08.313437+0000","last_clean":"2023-10-01T09:38:08.313437+0000","last_became_active":"2023-10-01T09:37:22.671882+0000","last_became_peered":"2023-10-01T09:37:22.671882+0000","last_unstale":"2023-10-01T09:38:08.313437+0000","last_undegraded":"2023-10-01T09:38:08.313437+0000","last_fullsized":"2023-10-01T09:38:08.313437+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:42:45.864800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":138,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.318591+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:38:08.318591+0000","last_peered":"2023-10-01T09:38:08.318591+0000","last_clean":"2023-10-01T09:38:08.318591+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:38:08.318591+0000","last_undegraded":"2023-10-01T09:38:08.318591+0000","last_fullsized":"2023-10-01T09:38:08.318591+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":528,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.455250+0000","last_change":"2023-10-01T09:37:22.671362+0000","last_active":"2023-10-01T09:38:08.455250+0000","last_peered":"2023-10-01T09:38:08.455250+0000","last_clean":"2023-10-01T09:38:08.455250+0000","last_became_active":"2023-10-01T09:37:22.671053+0000","last_became_peered":"2023-10-01T09:37:22.671053+0000","last_unstale":"2023-10-01T09:38:08.455250+0000","last_undegraded":"2023-10-01T09:38:08.455250+0000","last_fullsized":"2023-10-01T09:38:08.455250+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:42.476593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":476,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.315906+0000","last_change":"2023-10-01T09:37:22.667841+0000","last_active":"2023-10-01T09:38:08.315906+0000","last_peered":"2023-10-01T09:38:08.315906+0000","last_clean":"2023-10-01T09:38:08.315906+0000","last_became_active":"2023-10-01T09:37:22.667525+0000","last_became_peered":"2023-10-01T09:37:22.667525+0000","last_unstale":"2023-10-01T09:38:08.315906+0000","last_undegraded":"2023-10-01T09:38:08.315906+0000","last_fullsized":"2023-10-01T09:38:08.315906+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:03:37.695850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":149,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.318538+0000","last_change":"2023-10-01T09:36:22.254817+0000","last_active":"2023-10-01T09:38:08.318538+0000","last_peered":"2023-10-01T09:38:08.318538+0000","last_clean":"2023-10-01T09:38:08.318538+0000","last_became_active":"2023-10-01T09:36:22.254460+0000","last_became_peered":"2023-10-01T09:36:22.254460+0000","last_unstale":"2023-10-01T09:38:08.318538+0000","last_undegraded":"2023-10-01T09:38:08.318538+0000","last_fullsized":"2023-10-01T09:38:08.318538+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":52,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.317799+0000","last_change":"2023-10-01T09:37:22.680561+0000","last_active":"2023-10-01T09:38:08.317799+0000","last_peered":"2023-10-01T09:38:08.317799+0000","last_clean":"2023-10-01T09:38:08.317799+0000","last_became_active":"2023-10-01T09:37:22.680228+0000","last_became_peered":"2023-10-01T09:37:22.680228+0000","last_unstale":"2023-10-01T09:38:08.317799+0000","last_undegraded":"2023-10-01T09:38:08.317799+0000","last_fullsized":"2023-10-01T09:38:08.317799+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":71,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313977+0000","last_change":"2023-10-01T09:37:22.697476+0000","last_active":"2023-10-01T09:38:08.313977+0000","last_peered":"2023-10-01T09:38:08.313977+0000","last_clean":"2023-10-01T09:38:08.313977+0000","last_became_active":"2023-10-01T09:37:22.671750+0000","last_became_peered":"2023-10-01T09:37:22.671750+0000","last_unstale":"2023-10-01T09:38:08.313977+0000","last_undegraded":"2023-10-01T09:38:08.313977+0000","last_fullsized":"2023-10-01T09:38:08.313977+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:36:00.363327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":35,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313636+0000","last_change":"2023-10-01T09:36:22.223469+0000","last_active":"2023-10-01T09:38:08.313636+0000","last_peered":"2023-10-01T09:38:08.313636+0000","last_clean":"2023-10-01T09:38:08.313636+0000","last_became_active":"2023-10-01T09:36:22.222681+0000","last_became_peered":"2023-10-01T09:36:22.222681+0000","last_unstale":"2023-10-01T09:38:08.313636+0000","last_undegraded":"2023-10-01T09:38:08.313636+0000","last_fullsized":"2023-10-01T09:38:08.313636+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:19:28.367922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":482,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.461361+0000","last_change":"2023-10-01T09:32:27.498041+0000","last_active":"2023-10-01T09:38:08.461361+0000","last_peered":"2023-10-01T09:38:08.461361+0000","last_clean":"2023-10-01T09:38:08.461361+0000","last_became_active":"2023-10-01T09:32:27.454394+0000","last_became_peered":"2023-10-01T09:32:27.454394+0000","last_unstale":"2023-10-01T09:38:08.461361+0000","last_undegraded":"2023-10-01T09:38:08.461361+0000","last_fullsized":"2023-10-01T09:38:08.461361+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":297,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.315974+0000","last_change":"2023-10-01T09:35:23.610660+0000","last_active":"2023-10-01T09:38:08.315974+0000","last_peered":"2023-10-01T09:38:08.315974+0000","last_clean":"2023-10-01T09:38:08.315974+0000","last_became_active":"2023-10-01T09:35:23.610273+0000","last_became_peered":"2023-10-01T09:35:23.610273+0000","last_unstale":"2023-10-01T09:38:08.315974+0000","last_undegraded":"2023-10-01T09:38:08.315974+0000","last_fullsized":"2023-10-01T09:38:08.315974+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:43.996644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":558,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.461259+0000","last_change":"2023-10-01T09:35:34.640669+0000","last_active":"2023-10-01T09:38:08.461259+0000","last_peered":"2023-10-01T09:38:08.461259+0000","last_clean":"2023-10-01T09:38:08.461259+0000","last_became_active":"2023-10-01T09:35:34.552977+0000","last_became_peered":"2023-10-01T09:35:34.552977+0000","last_unstale":"2023-10-01T09:38:08.461259+0000","last_undegraded":"2023-10-01T09:38:08.461259+0000","last_fullsized":"2023-10-01T09:38:08.461259+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":306,"num_keys_recovered":90,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":292,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.461416+0000","last_change":"2023-10-01T09:37:22.681548+0000","last_active":"2023-10-01T09:38:08.461416+0000","last_peered":"2023-10-01T09:38:08.461416+0000","last_clean":"2023-10-01T09:38:08.461416+0000","last_became_active":"2023-10-01T09:37:22.681195+0000","last_became_peered":"2023-10-01T09:37:22.681195+0000","last_unstale":"2023-10-01T09:38:08.461416+0000","last_undegraded":"2023-10-01T09:38:08.461416+0000","last_fullsized":"2023-10-01T09:38:08.461416+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":457,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313603+0000","last_change":"2023-10-01T09:36:22.223650+0000","last_active":"2023-10-01T09:38:08.313603+0000","last_peered":"2023-10-01T09:38:08.313603+0000","last_clean":"2023-10-01T09:38:08.313603+0000","last_became_active":"2023-10-01T09:36:22.223087+0000","last_became_peered":"2023-10-01T09:36:22.223087+0000","last_unstale":"2023-10-01T09:38:08.313603+0000","last_undegraded":"2023-10-01T09:38:08.313603+0000","last_fullsized":"2023-10-01T09:38:08.313603+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:52:21.962744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":273,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.461574+0000","last_change":"2023-10-01T09:37:22.670521+0000","last_active":"2023-10-01T09:38:08.461574+0000","last_peered":"2023-10-01T09:38:08.461574+0000","last_clean":"2023-10-01T09:38:08.461574+0000","last_became_active":"2023-10-01T09:37:22.670154+0000","last_became_peered":"2023-10-01T09:37:22.670154+0000","last_unstale":"2023-10-01T09:38:08.461574+0000","last_undegraded":"2023-10-01T09:38:08.461574+0000","last_fullsized":"2023-10-01T09:38:08.461574+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":466,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.461116+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:38:08.461116+0000","last_peered":"2023-10-01T09:38:08.461116+0000","last_clean":"2023-10-01T09:38:08.461116+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:38:08.461116+0000","last_undegraded":"2023-10-01T09:38:08.461116+0000","last_fullsized":"2023-10-01T09:38:08.461116+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":457,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.455180+0000","last_change":"2023-10-01T09:37:22.681587+0000","last_active":"2023-10-01T09:38:08.455180+0000","last_peered":"2023-10-01T09:38:08.455180+0000","last_clean":"2023-10-01T09:38:08.455180+0000","last_became_active":"2023-10-01T09:37:22.681290+0000","last_became_peered":"2023-10-01T09:37:22.681290+0000","last_unstale":"2023-10-01T09:38:08.455180+0000","last_undegraded":"2023-10-01T09:38:08.455180+0000","last_fullsized":"2023-10-01T09:38:08.455180+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:13:27.218691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":312,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313312+0000","last_change":"2023-10-01T09:37:22.717074+0000","last_active":"2023-10-01T09:38:08.313312+0000","last_peered":"2023-10-01T09:38:08.313312+0000","last_clean":"2023-10-01T09:38:08.313312+0000","last_became_active":"2023-10-01T09:37:22.672350+0000","last_became_peered":"2023-10-01T09:37:22.672350+0000","last_unstale":"2023-10-01T09:38:08.313312+0000","last_undegraded":"2023-10-01T09:38:08.313312+0000","last_fullsized":"2023-10-01T09:38:08.313312+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:44:43.921813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":255,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.456753+0000","last_change":"2023-10-01T09:36:22.223054+0000","last_active":"2023-10-01T09:38:08.456753+0000","last_peered":"2023-10-01T09:38:08.456753+0000","last_clean":"2023-10-01T09:38:08.456753+0000","last_became_active":"2023-10-01T09:36:22.222776+0000","last_became_peered":"2023-10-01T09:36:22.222776+0000","last_unstale":"2023-10-01T09:38:08.456753+0000","last_undegraded":"2023-10-01T09:38:08.456753+0000","last_fullsized":"2023-10-01T09:38:08.456753+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:09:23.536588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"455'10748","reported_seq":15215,"reported_epoch":455,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:38:19.576059+0000","last_change":"2023-10-01T09:37:22.729877+0000","last_active":"2023-10-01T09:38:19.576059+0000","last_peered":"2023-10-01T09:38:19.576059+0000","last_clean":"2023-10-01T09:36:21.212087+0000","last_became_active":"2023-10-01T09:36:23.799500+0000","last_became_peered":"2023-10-01T09:36:23.799500+0000","last_unstale":"2023-10-01T09:38:19.576059+0000","last_undegraded":"2023-10-01T09:36:23.263244+0000","last_fullsized":"2023-10-01T09:36:23.212896+0000","mapping_epoch":431,"log_start":"409'3198","ondisk_log_start":"409'3198","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7550,"log_dups_size":0,"ondisk_log_size":7550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:09:34.039273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141688832,"num_objects":2162,"num_object_clones":0,"num_object_copies":6486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":672,"num_objects_misplaced":1995,"num_objects_unfound":0,"num_objects_dirty":2162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10748,"num_write_kb":413120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2195,"num_bytes_recovered":143785984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":-505},{"shards":"0,5,7","objects":1995}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":128,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313983+0000","last_change":"2023-10-01T09:36:22.387631+0000","last_active":"2023-10-01T09:38:08.313983+0000","last_peered":"2023-10-01T09:38:08.313983+0000","last_clean":"2023-10-01T09:38:08.313983+0000","last_became_active":"2023-10-01T09:36:22.237417+0000","last_became_peered":"2023-10-01T09:36:22.237417+0000","last_unstale":"2023-10-01T09:38:08.313983+0000","last_undegraded":"2023-10-01T09:38:08.313983+0000","last_fullsized":"2023-10-01T09:38:08.313983+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:06:04.026880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":166,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.317644+0000","last_change":"2023-10-01T09:36:22.236445+0000","last_active":"2023-10-01T09:38:08.317644+0000","last_peered":"2023-10-01T09:38:08.317644+0000","last_clean":"2023-10-01T09:38:08.317644+0000","last_became_active":"2023-10-01T09:36:22.236081+0000","last_became_peered":"2023-10-01T09:36:22.236081+0000","last_unstale":"2023-10-01T09:38:08.317644+0000","last_undegraded":"2023-10-01T09:38:08.317644+0000","last_fullsized":"2023-10-01T09:38:08.317644+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"455'10809","reported_seq":10900,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-01T09:38:19.597828+0000","last_change":"2023-10-01T09:37:22.701634+0000","last_active":"2023-10-01T09:38:19.597828+0000","last_peered":"2023-10-01T09:38:19.597828+0000","last_clean":"2023-10-01T09:38:19.597828+0000","last_became_active":"2023-10-01T09:37:22.671424+0000","last_became_peered":"2023-10-01T09:37:22.671424+0000","last_unstale":"2023-10-01T09:38:19.597828+0000","last_undegraded":"2023-10-01T09:38:19.597828+0000","last_fullsized":"2023-10-01T09:38:19.597828+0000","mapping_epoch":445,"log_start":"410'3934","ondisk_log_start":"410'3934","created":400,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6875,"log_dups_size":0,"ondisk_log_size":6875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:07:45.393494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141361152,"num_objects":2157,"num_object_clones":0,"num_object_copies":6471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10809,"num_write_kb":414912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":449,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.461567+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:38:08.461567+0000","last_peered":"2023-10-01T09:38:08.461567+0000","last_clean":"2023-10-01T09:38:08.461567+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:38:08.461567+0000","last_undegraded":"2023-10-01T09:38:08.461567+0000","last_fullsized":"2023-10-01T09:38:08.461567+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":498,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.315671+0000","last_change":"2023-10-01T09:36:22.223799+0000","last_active":"2023-10-01T09:38:08.315671+0000","last_peered":"2023-10-01T09:38:08.315671+0000","last_clean":"2023-10-01T09:38:08.315671+0000","last_became_active":"2023-10-01T09:36:22.223546+0000","last_became_peered":"2023-10-01T09:36:22.223546+0000","last_unstale":"2023-10-01T09:38:08.315671+0000","last_undegraded":"2023-10-01T09:38:08.315671+0000","last_fullsized":"2023-10-01T09:38:08.315671+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:54:07.013864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"455'10608","reported_seq":18614,"reported_epoch":455,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-01T09:38:19.938240+0000","last_change":"2023-10-01T09:37:06.460981+0000","last_active":"2023-10-01T09:38:19.938240+0000","last_peered":"2023-10-01T09:38:19.938240+0000","last_clean":"2023-10-01T09:36:21.192430+0000","last_became_active":"2023-10-01T09:36:23.791409+0000","last_became_peered":"2023-10-01T09:36:23.791409+0000","last_unstale":"2023-10-01T09:38:19.938240+0000","last_undegraded":"2023-10-01T09:38:19.938240+0000","last_fullsized":"2023-10-01T09:36:23.217545+0000","mapping_epoch":431,"log_start":"409'3707","ondisk_log_start":"409'3707","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6901,"log_dups_size":0,"ondisk_log_size":6901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141295616,"num_objects":2156,"num_object_clones":0,"num_object_copies":6468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4306,"num_objects_unfound":0,"num_objects_dirty":2156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10608,"num_write_kb":408448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2645,"num_bytes_recovered":173277184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,1],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"1,5","objects":3},{"shards":"3,5,7","objects":2153}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":535,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.461407+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:38:08.461407+0000","last_peered":"2023-10-01T09:38:08.461407+0000","last_clean":"2023-10-01T09:38:08.461407+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:38:08.461407+0000","last_undegraded":"2023-10-01T09:38:08.461407+0000","last_fullsized":"2023-10-01T09:38:08.461407+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":319,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.457990+0000","last_change":"2023-10-01T09:36:22.247103+0000","last_active":"2023-10-01T09:38:08.457990+0000","last_peered":"2023-10-01T09:38:08.457990+0000","last_clean":"2023-10-01T09:38:08.457990+0000","last_became_active":"2023-10-01T09:36:22.246793+0000","last_became_peered":"2023-10-01T09:36:22.246793+0000","last_unstale":"2023-10-01T09:38:08.457990+0000","last_undegraded":"2023-10-01T09:38:08.457990+0000","last_fullsized":"2023-10-01T09:38:08.457990+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:25:05.754730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"455'10544","reported_seq":13389,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-01T09:38:19.866600+0000","last_change":"2023-10-01T09:36:28.672838+0000","last_active":"2023-10-01T09:38:19.866600+0000","last_peered":"2023-10-01T09:38:19.866600+0000","last_clean":"2023-10-01T09:38:19.866600+0000","last_became_active":"2023-10-01T09:36:28.606609+0000","last_became_peered":"2023-10-01T09:36:28.606609+0000","last_unstale":"2023-10-01T09:38:19.866600+0000","last_undegraded":"2023-10-01T09:38:19.866600+0000","last_fullsized":"2023-10-01T09:38:19.866600+0000","mapping_epoch":433,"log_start":"409'3698","ondisk_log_start":"409'3698","created":400,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6846,"log_dups_size":0,"ondisk_log_size":6846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137232384,"num_objects":2094,"num_object_clones":0,"num_object_copies":6282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10544,"num_write_kb":404416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1377,"num_bytes_recovered":90243072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":175,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.318313+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:38:08.318313+0000","last_peered":"2023-10-01T09:38:08.318313+0000","last_clean":"2023-10-01T09:38:08.318313+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:38:08.318313+0000","last_undegraded":"2023-10-01T09:38:08.318313+0000","last_fullsized":"2023-10-01T09:38:08.318313+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":431,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313714+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:38:08.313714+0000","last_peered":"2023-10-01T09:38:08.313714+0000","last_clean":"2023-10-01T09:38:08.313714+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:38:08.313714+0000","last_undegraded":"2023-10-01T09:38:08.313714+0000","last_fullsized":"2023-10-01T09:38:08.313714+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"455'10374","reported_seq":19427,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-01T09:38:17.276898+0000","last_change":"2023-10-01T09:37:22.694633+0000","last_active":"2023-10-01T09:38:17.276898+0000","last_peered":"2023-10-01T09:38:17.276898+0000","last_clean":"2023-10-01T09:38:17.276898+0000","last_became_active":"2023-10-01T09:37:22.668427+0000","last_became_peered":"2023-10-01T09:37:22.668427+0000","last_unstale":"2023-10-01T09:38:17.276898+0000","last_undegraded":"2023-10-01T09:38:17.276898+0000","last_fullsized":"2023-10-01T09:38:17.276898+0000","mapping_epoch":445,"log_start":"407'2579","ondisk_log_start":"407'2579","created":400,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7795,"log_dups_size":0,"ondisk_log_size":7795,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:50:23.248095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140378112,"num_objects":2142,"num_object_clones":0,"num_object_copies":6426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10374,"num_write_kb":400512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4478,"num_bytes_recovered":293339136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":184,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313747+0000","last_change":"2023-10-01T09:37:22.723456+0000","last_active":"2023-10-01T09:38:08.313747+0000","last_peered":"2023-10-01T09:38:08.313747+0000","last_clean":"2023-10-01T09:38:08.313747+0000","last_became_active":"2023-10-01T09:37:22.677109+0000","last_became_peered":"2023-10-01T09:37:22.677109+0000","last_unstale":"2023-10-01T09:38:08.313747+0000","last_undegraded":"2023-10-01T09:38:08.313747+0000","last_fullsized":"2023-10-01T09:38:08.313747+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:04:02.812669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":109,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.317200+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:38:08.317200+0000","last_peered":"2023-10-01T09:38:08.317200+0000","last_clean":"2023-10-01T09:38:08.317200+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:38:08.317200+0000","last_undegraded":"2023-10-01T09:38:08.317200+0000","last_fullsized":"2023-10-01T09:38:08.317200+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"455'10581","reported_seq":13550,"reported_epoch":455,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-01T09:38:19.569377+0000","last_change":"2023-10-01T09:37:06.423304+0000","last_active":"2023-10-01T09:38:19.569377+0000","last_peered":"2023-10-01T09:38:19.569377+0000","last_clean":"2023-10-01T09:35:09.959309+0000","last_became_active":"2023-10-01T09:36:23.678618+0000","last_became_peered":"2023-10-01T09:36:23.678618+0000","last_unstale":"2023-10-01T09:38:19.569377+0000","last_undegraded":"2023-10-01T09:38:19.569377+0000","last_fullsized":"2023-10-01T09:38:19.569377+0000","mapping_epoch":431,"log_start":"409'3714","ondisk_log_start":"409'3714","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6867,"log_dups_size":0,"ondisk_log_size":6867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:32:05.325065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138346496,"num_objects":2111,"num_object_clones":0,"num_object_copies":6333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4912,"num_objects_unfound":0,"num_objects_dirty":2111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10581,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1435,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[1,6,2],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":-1157}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":375,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.315836+0000","last_change":"2023-10-01T09:37:22.684537+0000","last_active":"2023-10-01T09:38:08.315836+0000","last_peered":"2023-10-01T09:38:08.315836+0000","last_clean":"2023-10-01T09:38:08.315836+0000","last_became_active":"2023-10-01T09:37:22.683692+0000","last_became_peered":"2023-10-01T09:37:22.683692+0000","last_unstale":"2023-10-01T09:38:08.315836+0000","last_undegraded":"2023-10-01T09:38:08.315836+0000","last_fullsized":"2023-10-01T09:38:08.315836+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":502,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.454676+0000","last_change":"2023-10-01T09:35:34.558802+0000","last_active":"2023-10-01T09:38:08.454676+0000","last_peered":"2023-10-01T09:38:08.454676+0000","last_clean":"2023-10-01T09:38:08.454676+0000","last_became_active":"2023-10-01T09:35:34.558511+0000","last_became_peered":"2023-10-01T09:35:34.558511+0000","last_unstale":"2023-10-01T09:38:08.454676+0000","last_undegraded":"2023-10-01T09:38:08.454676+0000","last_fullsized":"2023-10-01T09:38:08.454676+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:40:52.864568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"455'10543","reported_seq":11313,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-01T09:38:16.101678+0000","last_change":"2023-10-01T09:37:23.700088+0000","last_active":"2023-10-01T09:38:16.101678+0000","last_peered":"2023-10-01T09:38:16.101678+0000","last_clean":"2023-10-01T09:38:16.101678+0000","last_became_active":"2023-10-01T09:37:23.635822+0000","last_became_peered":"2023-10-01T09:37:23.635822+0000","last_unstale":"2023-10-01T09:38:16.101678+0000","last_undegraded":"2023-10-01T09:38:16.101678+0000","last_fullsized":"2023-10-01T09:38:16.101678+0000","mapping_epoch":446,"log_start":"405'500","ondisk_log_start":"405'500","created":400,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":10043,"log_dups_size":0,"ondisk_log_size":10043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146079744,"num_objects":2229,"num_object_clones":0,"num_object_copies":6687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10543,"num_write_kb":408704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":343,"num_bytes_recovered":22478848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":345,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.461122+0000","last_change":"2023-10-01T09:35:34.551196+0000","last_active":"2023-10-01T09:38:08.461122+0000","last_peered":"2023-10-01T09:38:08.461122+0000","last_clean":"2023-10-01T09:38:08.461122+0000","last_became_active":"2023-10-01T09:35:34.550803+0000","last_became_peered":"2023-10-01T09:35:34.550803+0000","last_unstale":"2023-10-01T09:38:08.461122+0000","last_undegraded":"2023-10-01T09:38:08.461122+0000","last_fullsized":"2023-10-01T09:38:08.461122+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":181,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.316771+0000","last_change":"2023-10-01T09:37:22.681270+0000","last_active":"2023-10-01T09:38:08.316771+0000","last_peered":"2023-10-01T09:38:08.316771+0000","last_clean":"2023-10-01T09:38:08.316771+0000","last_became_active":"2023-10-01T09:37:22.680951+0000","last_became_peered":"2023-10-01T09:37:22.680951+0000","last_unstale":"2023-10-01T09:38:08.316771+0000","last_undegraded":"2023-10-01T09:38:08.316771+0000","last_fullsized":"2023-10-01T09:38:08.316771+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"455'10773","reported_seq":11812,"reported_epoch":455,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:38:19.618555+0000","last_change":"2023-10-01T09:36:23.771443+0000","last_active":"2023-10-01T09:38:19.618555+0000","last_peered":"2023-10-01T09:38:19.618555+0000","last_clean":"2023-10-01T09:35:09.934020+0000","last_became_active":"2023-10-01T09:36:23.749995+0000","last_became_peered":"2023-10-01T09:36:23.749995+0000","last_unstale":"2023-10-01T09:38:19.618555+0000","last_undegraded":"2023-10-01T09:36:23.566811+0000","last_fullsized":"2023-10-01T09:36:23.220185+0000","mapping_epoch":431,"log_start":"410'3892","ondisk_log_start":"410'3892","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6881,"log_dups_size":0,"ondisk_log_size":6881,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:23:09.158739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140181504,"num_objects":2139,"num_object_clones":0,"num_object_copies":6417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10773,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":476,"num_bytes_recovered":30998528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2","objects":-2748},{"shards":"1,2,3","objects":1221},{"shards":"1,3,7","objects":918}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":306,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.315800+0000","last_change":"2023-10-01T09:36:22.223384+0000","last_active":"2023-10-01T09:38:08.315800+0000","last_peered":"2023-10-01T09:38:08.315800+0000","last_clean":"2023-10-01T09:38:08.315800+0000","last_became_active":"2023-10-01T09:36:22.223098+0000","last_became_peered":"2023-10-01T09:36:22.223098+0000","last_unstale":"2023-10-01T09:38:08.315800+0000","last_undegraded":"2023-10-01T09:38:08.315800+0000","last_fullsized":"2023-10-01T09:38:08.315800+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:15:40.489055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":362,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.456821+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:38:08.456821+0000","last_peered":"2023-10-01T09:38:08.456821+0000","last_clean":"2023-10-01T09:38:08.456821+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:38:08.456821+0000","last_undegraded":"2023-10-01T09:38:08.456821+0000","last_fullsized":"2023-10-01T09:38:08.456821+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"455'10706","reported_seq":10782,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-01T09:38:19.951431+0000","last_change":"2023-10-01T09:37:22.715127+0000","last_active":"2023-10-01T09:38:19.951431+0000","last_peered":"2023-10-01T09:38:19.951431+0000","last_clean":"2023-10-01T09:38:19.951431+0000","last_became_active":"2023-10-01T09:37:22.680102+0000","last_became_peered":"2023-10-01T09:37:22.680102+0000","last_unstale":"2023-10-01T09:38:19.951431+0000","last_undegraded":"2023-10-01T09:38:19.951431+0000","last_fullsized":"2023-10-01T09:38:19.951431+0000","mapping_epoch":445,"log_start":"410'3862","ondisk_log_start":"410'3862","created":400,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6844,"log_dups_size":0,"ondisk_log_size":6844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139853824,"num_objects":2134,"num_object_clones":0,"num_object_copies":6402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10706,"num_write_kb":410880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1279,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.316175+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:38:08.316175+0000","last_peered":"2023-10-01T09:38:08.316175+0000","last_clean":"2023-10-01T09:38:08.316175+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:38:08.316175+0000","last_undegraded":"2023-10-01T09:38:08.316175+0000","last_fullsized":"2023-10-01T09:38:08.316175+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":176,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.318601+0000","last_change":"2023-10-01T09:37:22.667746+0000","last_active":"2023-10-01T09:38:08.318601+0000","last_peered":"2023-10-01T09:38:08.318601+0000","last_clean":"2023-10-01T09:38:08.318601+0000","last_became_active":"2023-10-01T09:37:22.667475+0000","last_became_peered":"2023-10-01T09:37:22.667475+0000","last_unstale":"2023-10-01T09:38:08.318601+0000","last_undegraded":"2023-10-01T09:38:08.318601+0000","last_fullsized":"2023-10-01T09:38:08.318601+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":410,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.318106+0000","last_change":"2023-10-01T09:36:22.236067+0000","last_active":"2023-10-01T09:38:08.318106+0000","last_peered":"2023-10-01T09:38:08.318106+0000","last_clean":"2023-10-01T09:38:08.318106+0000","last_became_active":"2023-10-01T09:36:22.235732+0000","last_became_peered":"2023-10-01T09:36:22.235732+0000","last_unstale":"2023-10-01T09:38:08.318106+0000","last_undegraded":"2023-10-01T09:38:08.318106+0000","last_fullsized":"2023-10-01T09:38:08.318106+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"455'10722","reported_seq":10787,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-01T09:38:19.526942+0000","last_change":"2023-10-01T09:34:06.140052+0000","last_active":"2023-10-01T09:38:19.526942+0000","last_peered":"2023-10-01T09:38:19.526942+0000","last_clean":"2023-10-01T09:38:19.526942+0000","last_became_active":"2023-10-01T09:34:06.139672+0000","last_became_peered":"2023-10-01T09:34:06.139672+0000","last_unstale":"2023-10-01T09:38:19.526942+0000","last_undegraded":"2023-10-01T09:38:19.526942+0000","last_fullsized":"2023-10-01T09:38:19.526942+0000","mapping_epoch":400,"log_start":"409'3178","ondisk_log_start":"409'3178","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7544,"log_dups_size":0,"ondisk_log_size":7544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:39:55.000830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139984896,"num_objects":2136,"num_object_clones":0,"num_object_copies":6408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10722,"num_write_kb":411456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1120,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.317989+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:38:08.317989+0000","last_peered":"2023-10-01T09:38:08.317989+0000","last_clean":"2023-10-01T09:38:08.317989+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:38:08.317989+0000","last_undegraded":"2023-10-01T09:38:08.317989+0000","last_fullsized":"2023-10-01T09:38:08.317989+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":431,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313899+0000","last_change":"2023-10-01T09:32:27.451746+0000","last_active":"2023-10-01T09:38:08.313899+0000","last_peered":"2023-10-01T09:38:08.313899+0000","last_clean":"2023-10-01T09:38:08.313899+0000","last_became_active":"2023-10-01T09:32:27.451069+0000","last_became_peered":"2023-10-01T09:32:27.451069+0000","last_unstale":"2023-10-01T09:38:08.313899+0000","last_undegraded":"2023-10-01T09:38:08.313899+0000","last_fullsized":"2023-10-01T09:38:08.313899+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:50.586397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":143,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.318402+0000","last_change":"2023-10-01T09:38:07.318624+0000","last_active":"2023-10-01T09:38:08.318402+0000","last_peered":"2023-10-01T09:38:08.318402+0000","last_clean":"2023-10-01T09:38:08.318402+0000","last_became_active":"2023-10-01T09:38:07.318266+0000","last_became_peered":"2023-10-01T09:38:07.318266+0000","last_unstale":"2023-10-01T09:38:08.318402+0000","last_undegraded":"2023-10-01T09:38:08.318402+0000","last_fullsized":"2023-10-01T09:38:08.318402+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"455'10501","reported_seq":14149,"reported_epoch":455,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-01T09:38:19.620973+0000","last_change":"2023-10-01T09:38:08.682352+0000","last_active":"2023-10-01T09:38:19.620973+0000","last_peered":"2023-10-01T09:38:19.620973+0000","last_clean":"2023-10-01T09:36:21.153294+0000","last_became_active":"2023-10-01T09:38:08.590431+0000","last_became_peered":"2023-10-01T09:38:08.590431+0000","last_unstale":"2023-10-01T09:38:19.620973+0000","last_undegraded":"2023-10-01T09:38:08.584718+0000","last_fullsized":"2023-10-01T09:38:19.620973+0000","mapping_epoch":453,"log_start":"409'3607","ondisk_log_start":"409'3607","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6894,"log_dups_size":0,"ondisk_log_size":6894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:06:34.257665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134676480,"num_objects":2055,"num_object_clones":0,"num_object_copies":6165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2055,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10501,"num_write_kb":401792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1775,"num_bytes_recovered":116326400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,3],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,3,7","objects":1},{"shards":"1,5,7","objects":2054}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":735,"reported_epoch":454,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:38:08.313322+0000","last_change":"2023-10-01T09:38:07.327355+0000","last_active":"2023-10-01T09:38:08.313322+0000","last_peered":"2023-10-01T09:38:08.313322+0000","last_clean":"2023-10-01T09:37:58.763801+0000","last_became_active":"2023-10-01T09:38:07.326767+0000","last_became_peered":"2023-10-01T09:38:07.326767+0000","last_unstale":"2023-10-01T09:38:08.313322+0000","last_undegraded":"2023-10-01T09:38:07.315924+0000","last_fullsized":"2023-10-01T09:38:08.313322+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:45:16.157432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":["0","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":506,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-01T09:38:10.363122+0000","last_change":"2023-10-01T09:35:34.562777+0000","last_active":"2023-10-01T09:38:10.363122+0000","last_peered":"2023-10-01T09:38:10.363122+0000","last_clean":"2023-10-01T09:38:10.363122+0000","last_became_active":"2023-10-01T09:35:34.562492+0000","last_became_peered":"2023-10-01T09:35:34.562492+0000","last_unstale":"2023-10-01T09:38:10.363122+0000","last_undegraded":"2023-10-01T09:38:10.363122+0000","last_fullsized":"2023-10-01T09:38:10.363122+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:26:15.751778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":324,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.318820+0000","last_change":"2023-10-01T09:37:22.673957+0000","last_active":"2023-10-01T09:38:08.318820+0000","last_peered":"2023-10-01T09:38:08.318820+0000","last_clean":"2023-10-01T09:38:08.318820+0000","last_became_active":"2023-10-01T09:37:22.673685+0000","last_became_peered":"2023-10-01T09:37:22.673685+0000","last_unstale":"2023-10-01T09:38:08.318820+0000","last_undegraded":"2023-10-01T09:38:08.318820+0000","last_fullsized":"2023-10-01T09:38:08.318820+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"455'10595","reported_seq":18739,"reported_epoch":455,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:38:19.575914+0000","last_change":"2023-10-01T09:37:22.682829+0000","last_active":"2023-10-01T09:38:19.575914+0000","last_peered":"2023-10-01T09:38:19.575914+0000","last_clean":"2023-10-01T09:36:21.170373+0000","last_became_active":"2023-10-01T09:36:23.812646+0000","last_became_peered":"2023-10-01T09:36:23.812646+0000","last_unstale":"2023-10-01T09:38:19.575914+0000","last_undegraded":"2023-10-01T09:37:22.679747+0000","last_fullsized":"2023-10-01T09:36:23.211397+0000","mapping_epoch":431,"log_start":"408'3057","ondisk_log_start":"408'3057","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7538,"log_dups_size":0,"ondisk_log_size":7538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:34:58.841399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141361152,"num_objects":2157,"num_object_clones":0,"num_object_copies":6471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7372,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10595,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2690,"num_bytes_recovered":176160768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":2157}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1451,"reported_epoch":454,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:38:08.317558+0000","last_change":"2023-10-01T09:37:23.636269+0000","last_active":"2023-10-01T09:38:08.317558+0000","last_peered":"2023-10-01T09:38:08.317558+0000","last_clean":"2023-10-01T09:35:27.946455+0000","last_became_active":"2023-10-01T09:37:23.635623+0000","last_became_peered":"2023-10-01T09:37:23.635623+0000","last_unstale":"2023-10-01T09:38:08.317558+0000","last_undegraded":"2023-10-01T09:37:23.631750+0000","last_fullsized":"2023-10-01T09:38:08.317558+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":474,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313216+0000","last_change":"2023-10-01T09:36:29.654983+0000","last_active":"2023-10-01T09:38:08.313216+0000","last_peered":"2023-10-01T09:38:08.313216+0000","last_clean":"2023-10-01T09:38:08.313216+0000","last_became_active":"2023-10-01T09:36:29.650683+0000","last_became_peered":"2023-10-01T09:36:29.650683+0000","last_unstale":"2023-10-01T09:38:08.313216+0000","last_undegraded":"2023-10-01T09:38:08.313216+0000","last_fullsized":"2023-10-01T09:38:08.313216+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:23.591743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":254,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.317843+0000","last_change":"2023-10-01T09:38:07.494808+0000","last_active":"2023-10-01T09:38:08.317843+0000","last_peered":"2023-10-01T09:38:08.317843+0000","last_clean":"2023-10-01T09:38:08.317843+0000","last_became_active":"2023-10-01T09:38:07.494419+0000","last_became_peered":"2023-10-01T09:38:07.494419+0000","last_unstale":"2023-10-01T09:38:08.317843+0000","last_undegraded":"2023-10-01T09:38:08.317843+0000","last_fullsized":"2023-10-01T09:38:08.317843+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":369,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.461489+0000","last_change":"2023-10-01T09:32:27.453291+0000","last_active":"2023-10-01T09:38:08.461489+0000","last_peered":"2023-10-01T09:38:08.461489+0000","last_clean":"2023-10-01T09:38:08.461489+0000","last_became_active":"2023-10-01T09:32:27.453026+0000","last_became_peered":"2023-10-01T09:32:27.453026+0000","last_unstale":"2023-10-01T09:38:08.461489+0000","last_undegraded":"2023-10-01T09:38:08.461489+0000","last_fullsized":"2023-10-01T09:38:08.461489+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"455'10607","reported_seq":10672,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-01T09:38:19.531549+0000","last_change":"2023-10-01T09:34:06.140844+0000","last_active":"2023-10-01T09:38:19.531549+0000","last_peered":"2023-10-01T09:38:19.531549+0000","last_clean":"2023-10-01T09:38:19.531549+0000","last_became_active":"2023-10-01T09:34:06.139812+0000","last_became_peered":"2023-10-01T09:34:06.139812+0000","last_unstale":"2023-10-01T09:38:19.531549+0000","last_undegraded":"2023-10-01T09:38:19.531549+0000","last_fullsized":"2023-10-01T09:38:19.531549+0000","mapping_epoch":400,"log_start":"409'3067","ondisk_log_start":"409'3067","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7540,"log_dups_size":0,"ondisk_log_size":7540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:31:05.105491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140967936,"num_objects":2151,"num_object_clones":0,"num_object_copies":6453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10607,"num_write_kb":408256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":635,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.461008+0000","last_change":"2023-10-01T09:37:22.770678+0000","last_active":"2023-10-01T09:38:08.461008+0000","last_peered":"2023-10-01T09:38:08.461008+0000","last_clean":"2023-10-01T09:38:08.461008+0000","last_became_active":"2023-10-01T09:37:22.674516+0000","last_became_peered":"2023-10-01T09:37:22.674516+0000","last_unstale":"2023-10-01T09:38:08.461008+0000","last_undegraded":"2023-10-01T09:38:08.461008+0000","last_fullsized":"2023-10-01T09:38:08.461008+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":483,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313441+0000","last_change":"2023-10-01T09:36:22.225663+0000","last_active":"2023-10-01T09:38:08.313441+0000","last_peered":"2023-10-01T09:38:08.313441+0000","last_clean":"2023-10-01T09:38:08.313441+0000","last_became_active":"2023-10-01T09:36:22.225382+0000","last_became_peered":"2023-10-01T09:36:22.225382+0000","last_unstale":"2023-10-01T09:38:08.313441+0000","last_undegraded":"2023-10-01T09:38:08.313441+0000","last_fullsized":"2023-10-01T09:38:08.313441+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:19:41.386221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":482,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313975+0000","last_change":"2023-10-01T09:35:34.551948+0000","last_active":"2023-10-01T09:38:08.313975+0000","last_peered":"2023-10-01T09:38:08.313975+0000","last_clean":"2023-10-01T09:38:08.313975+0000","last_became_active":"2023-10-01T09:35:34.551665+0000","last_became_peered":"2023-10-01T09:35:34.551665+0000","last_unstale":"2023-10-01T09:38:08.313975+0000","last_undegraded":"2023-10-01T09:38:08.313975+0000","last_fullsized":"2023-10-01T09:38:08.313975+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:31:30.655381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"455'10837","reported_seq":10946,"reported_epoch":455,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:38:19.532541+0000","last_change":"2023-10-01T09:36:23.705972+0000","last_active":"2023-10-01T09:38:19.532541+0000","last_peered":"2023-10-01T09:38:19.532541+0000","last_clean":"2023-10-01T09:36:21.165407+0000","last_became_active":"2023-10-01T09:36:23.529481+0000","last_became_peered":"2023-10-01T09:36:23.529481+0000","last_unstale":"2023-10-01T09:38:19.532541+0000","last_undegraded":"2023-10-01T09:38:19.532541+0000","last_fullsized":"2023-10-01T09:38:19.532541+0000","mapping_epoch":431,"log_start":"410'3944","ondisk_log_start":"410'3944","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6893,"log_dups_size":0,"ondisk_log_size":6893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:01:56.211865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143327232,"num_objects":2187,"num_object_clones":0,"num_object_copies":6561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2181,"num_objects_unfound":0,"num_objects_dirty":2187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10837,"num_write_kb":416768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":2187}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":893,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.461472+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:38:08.461472+0000","last_peered":"2023-10-01T09:38:08.461472+0000","last_clean":"2023-10-01T09:38:08.461472+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:38:08.461472+0000","last_undegraded":"2023-10-01T09:38:08.461472+0000","last_fullsized":"2023-10-01T09:38:08.461472+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":511,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.455576+0000","last_change":"2023-10-01T09:35:34.559426+0000","last_active":"2023-10-01T09:38:08.455576+0000","last_peered":"2023-10-01T09:38:08.455576+0000","last_clean":"2023-10-01T09:38:08.455576+0000","last_became_active":"2023-10-01T09:35:34.558890+0000","last_became_peered":"2023-10-01T09:35:34.558890+0000","last_unstale":"2023-10-01T09:38:08.455576+0000","last_undegraded":"2023-10-01T09:38:08.455576+0000","last_fullsized":"2023-10-01T09:38:08.455576+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:55:37.464032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":447,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313824+0000","last_change":"2023-10-01T09:33:11.656981+0000","last_active":"2023-10-01T09:38:08.313824+0000","last_peered":"2023-10-01T09:38:08.313824+0000","last_clean":"2023-10-01T09:38:08.313824+0000","last_became_active":"2023-10-01T09:33:11.656334+0000","last_became_peered":"2023-10-01T09:33:11.656334+0000","last_unstale":"2023-10-01T09:38:08.313824+0000","last_undegraded":"2023-10-01T09:38:08.313824+0000","last_fullsized":"2023-10-01T09:38:08.313824+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:50:15.562113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"455'10585","reported_seq":10657,"reported_epoch":455,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:38:19.841834+0000","last_change":"2023-10-01T09:36:23.622968+0000","last_active":"2023-10-01T09:38:19.841834+0000","last_peered":"2023-10-01T09:38:19.841834+0000","last_clean":"2023-10-01T09:36:21.149981+0000","last_became_active":"2023-10-01T09:36:23.603244+0000","last_became_peered":"2023-10-01T09:36:23.603244+0000","last_unstale":"2023-10-01T09:38:19.841834+0000","last_undegraded":"2023-10-01T09:38:19.841834+0000","last_fullsized":"2023-10-01T09:38:19.841834+0000","mapping_epoch":431,"log_start":"410'3705","ondisk_log_start":"410'3705","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6880,"log_dups_size":0,"ondisk_log_size":6880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144834584,"num_objects":2211,"num_object_clones":0,"num_object_copies":6633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4422,"num_objects_unfound":0,"num_objects_dirty":2211,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10585,"num_write_kb":409409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,1,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":2211}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":394,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.461547+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:38:08.461547+0000","last_peered":"2023-10-01T09:38:08.461547+0000","last_clean":"2023-10-01T09:38:08.461547+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:38:08.461547+0000","last_undegraded":"2023-10-01T09:38:08.461547+0000","last_fullsized":"2023-10-01T09:38:08.461547+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1085,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:09.592807+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:38:09.592807+0000","last_peered":"2023-10-01T09:38:09.592807+0000","last_clean":"2023-10-01T09:38:09.592807+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:38:09.592807+0000","last_undegraded":"2023-10-01T09:38:09.592807+0000","last_fullsized":"2023-10-01T09:38:09.592807+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":176,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313121+0000","last_change":"2023-10-01T09:36:22.236397+0000","last_active":"2023-10-01T09:38:08.313121+0000","last_peered":"2023-10-01T09:38:08.313121+0000","last_clean":"2023-10-01T09:38:08.313121+0000","last_became_active":"2023-10-01T09:36:22.236066+0000","last_became_peered":"2023-10-01T09:36:22.236066+0000","last_unstale":"2023-10-01T09:38:08.313121+0000","last_undegraded":"2023-10-01T09:38:08.313121+0000","last_fullsized":"2023-10-01T09:38:08.313121+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:04:06.876501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"455'10754","reported_seq":10835,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-01T09:38:16.139687+0000","last_change":"2023-10-01T09:37:23.665454+0000","last_active":"2023-10-01T09:38:16.139687+0000","last_peered":"2023-10-01T09:38:16.139687+0000","last_clean":"2023-10-01T09:38:16.139687+0000","last_became_active":"2023-10-01T09:37:23.635951+0000","last_became_peered":"2023-10-01T09:37:23.635951+0000","last_unstale":"2023-10-01T09:38:16.139687+0000","last_undegraded":"2023-10-01T09:38:16.139687+0000","last_fullsized":"2023-10-01T09:38:16.139687+0000","mapping_epoch":446,"log_start":"405'700","ondisk_log_start":"405'700","created":400,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":10054,"log_dups_size":0,"ondisk_log_size":10054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147980288,"num_objects":2258,"num_object_clones":0,"num_object_copies":6774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10754,"num_write_kb":416384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":215,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.315542+0000","last_change":"2023-10-01T09:37:22.671608+0000","last_active":"2023-10-01T09:38:08.315542+0000","last_peered":"2023-10-01T09:38:08.315542+0000","last_clean":"2023-10-01T09:38:08.315542+0000","last_became_active":"2023-10-01T09:37:22.671097+0000","last_became_peered":"2023-10-01T09:37:22.671097+0000","last_unstale":"2023-10-01T09:38:08.315542+0000","last_undegraded":"2023-10-01T09:38:08.315542+0000","last_fullsized":"2023-10-01T09:38:08.315542+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":469,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.455052+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:38:08.455052+0000","last_peered":"2023-10-01T09:38:08.455052+0000","last_clean":"2023-10-01T09:38:08.455052+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:38:08.455052+0000","last_undegraded":"2023-10-01T09:38:08.455052+0000","last_fullsized":"2023-10-01T09:38:08.455052+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":178,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.317930+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:38:08.317930+0000","last_peered":"2023-10-01T09:38:08.317930+0000","last_clean":"2023-10-01T09:38:08.317930+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:38:08.317930+0000","last_undegraded":"2023-10-01T09:38:08.317930+0000","last_fullsized":"2023-10-01T09:38:08.317930+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":418,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.455219+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:38:08.455219+0000","last_peered":"2023-10-01T09:38:08.455219+0000","last_clean":"2023-10-01T09:38:08.455219+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:38:08.455219+0000","last_undegraded":"2023-10-01T09:38:08.455219+0000","last_fullsized":"2023-10-01T09:38:08.455219+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":523,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.454846+0000","last_change":"2023-10-01T09:37:22.674112+0000","last_active":"2023-10-01T09:38:08.454846+0000","last_peered":"2023-10-01T09:38:08.454846+0000","last_clean":"2023-10-01T09:38:08.454846+0000","last_became_active":"2023-10-01T09:37:22.673785+0000","last_became_peered":"2023-10-01T09:37:22.673785+0000","last_unstale":"2023-10-01T09:38:08.454846+0000","last_undegraded":"2023-10-01T09:38:08.454846+0000","last_fullsized":"2023-10-01T09:38:08.454846+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:05.627976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":190,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.318269+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:38:08.318269+0000","last_peered":"2023-10-01T09:38:08.318269+0000","last_clean":"2023-10-01T09:38:08.318269+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:38:08.318269+0000","last_undegraded":"2023-10-01T09:38:08.318269+0000","last_fullsized":"2023-10-01T09:38:08.318269+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":253,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.316425+0000","last_change":"2023-10-01T09:37:22.671305+0000","last_active":"2023-10-01T09:38:08.316425+0000","last_peered":"2023-10-01T09:38:08.316425+0000","last_clean":"2023-10-01T09:38:08.316425+0000","last_became_active":"2023-10-01T09:37:22.670844+0000","last_became_peered":"2023-10-01T09:37:22.670844+0000","last_unstale":"2023-10-01T09:38:08.316425+0000","last_undegraded":"2023-10-01T09:38:08.316425+0000","last_fullsized":"2023-10-01T09:38:08.316425+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":190,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.318659+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:38:08.318659+0000","last_peered":"2023-10-01T09:38:08.318659+0000","last_clean":"2023-10-01T09:38:08.318659+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:38:08.318659+0000","last_undegraded":"2023-10-01T09:38:08.318659+0000","last_fullsized":"2023-10-01T09:38:08.318659+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":365,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-01T09:38:10.366130+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:38:10.366130+0000","last_peered":"2023-10-01T09:38:10.366130+0000","last_clean":"2023-10-01T09:38:10.366130+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:38:10.366130+0000","last_undegraded":"2023-10-01T09:38:10.366130+0000","last_fullsized":"2023-10-01T09:38:10.366130+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":20,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.461478+0000","last_change":"2023-10-01T09:37:22.673935+0000","last_active":"2023-10-01T09:38:08.461478+0000","last_peered":"2023-10-01T09:38:08.461478+0000","last_clean":"2023-10-01T09:38:08.461478+0000","last_became_active":"2023-10-01T09:37:22.673609+0000","last_became_peered":"2023-10-01T09:37:22.673609+0000","last_unstale":"2023-10-01T09:38:08.461478+0000","last_undegraded":"2023-10-01T09:38:08.461478+0000","last_fullsized":"2023-10-01T09:38:08.461478+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:02:43.715678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":401,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.313359+0000","last_change":"2023-10-01T09:37:22.668216+0000","last_active":"2023-10-01T09:38:08.313359+0000","last_peered":"2023-10-01T09:38:08.313359+0000","last_clean":"2023-10-01T09:38:08.313359+0000","last_became_active":"2023-10-01T09:37:22.667243+0000","last_became_peered":"2023-10-01T09:37:22.667243+0000","last_unstale":"2023-10-01T09:38:08.313359+0000","last_undegraded":"2023-10-01T09:38:08.313359+0000","last_fullsized":"2023-10-01T09:38:08.313359+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:45:16.527071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":594,"reported_epoch":454,"state":"active+clean","last_fresh":"2023-10-01T09:38:08.455319+0000","last_change":"2023-10-01T09:35:34.658812+0000","last_active":"2023-10-01T09:38:08.455319+0000","last_peered":"2023-10-01T09:38:08.455319+0000","last_clean":"2023-10-01T09:38:08.455319+0000","last_became_active":"2023-10-01T09:35:34.550952+0000","last_became_peered":"2023-10-01T09:35:34.550952+0000","last_unstale":"2023-10-01T09:38:08.455319+0000","last_undegraded":"2023-10-01T09:38:08.455319+0000","last_fullsized":"2023-10-01T09:38:08.455319+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:24:17.304117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":425,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-10-01T09:38:10.361358+0000","last_change":"2023-10-01T09:32:27.445299+0000","last_active":"2023-10-01T09:38:10.361358+0000","last_peered":"2023-10-01T09:38:10.361358+0000","last_clean":"2023-10-01T09:38:10.361358+0000","last_became_active":"2023-10-01T09:32:27.444956+0000","last_became_peered":"2023-10-01T09:32:27.444956+0000","last_unstale":"2023-10-01T09:38:10.361358+0000","last_undegraded":"2023-10-01T09:38:10.361358+0000","last_fullsized":"2023-10-01T09:38:10.361358+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:49:07.862356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":123,"num_bytes_recovered":34520,"num_keys_recovered":90,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2259550232,"num_objects":34479,"num_object_clones":0,"num_object_copies":103437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16552,"num_objects_misplaced":17816,"num_objects_unfound":0,"num_objects_dirty":34479,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170287,"num_write_kb":6552449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17431,"num_bytes_recovered":1141506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8334618624,"data_stored":8334606408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119945,"ondisk_log_size":119945,"up":48,"acting":44,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":297,"num_bytes_recovered":12582969,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430652,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2628156,"kb_used_data":1720948,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":91116996,"statfs":{"total":95995035648,"available":93303803904,"internally_reserved":0,"allocated":1762250752,"data_stored":1761347552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:37:43 2023","interfaces":[{"interface":"back","average":{"1min":1.853,"5min":5.2190000000000003,"15min":3.427},"min":{"1min":0.27100000000000002,"5min":0.218,"15min":0.19500000000000001},"max":{"1min":11.525,"5min":108.90000000000001,"15min":108.90000000000001},"last":4.0069999999999997},{"interface":"front","average":{"1min":2.4929999999999999,"5min":5.2830000000000004,"15min":3.637},"min":{"1min":0.26400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.451000000000001,"5min":109.187,"15min":109.187},"last":0.34799999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:37:59 2023","interfaces":[{"interface":"back","average":{"1min":2.3340000000000001,"5min":7.1070000000000002,"15min":4.4500000000000002},"min":{"1min":0.315,"5min":0.23300000000000001,"15min":0.222},"max":{"1min":8.3610000000000007,"5min":133.54499999999999,"15min":205.34200000000001},"last":4.1059999999999999},{"interface":"front","average":{"1min":2.206,"5min":7.3970000000000002,"15min":4.5490000000000004},"min":{"1min":0.27500000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.0209999999999999,"5min":133.39400000000001,"15min":205.44499999999999},"last":1.321}]},{"osd":2,"last update":"Sun Oct 1 09:37:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6539999999999999,"5min":2.2210000000000001,"15min":2.2389999999999999},"min":{"1min":0.372,"5min":0.27100000000000002,"15min":0.249},"max":{"1min":12.994,"5min":14.901999999999999,"15min":59.878},"last":4.0730000000000004},{"interface":"front","average":{"1min":2.415,"5min":1.8149999999999999,"15min":2.1669999999999998},"min":{"1min":0.35799999999999998,"5min":0.27700000000000002,"15min":0.247},"max":{"1min":8.0050000000000008,"5min":8.0050000000000008,"15min":60.186},"last":0.435}]},{"osd":3,"last update":"Sun Oct 1 09:37:59 2023","interfaces":[{"interface":"back","average":{"1min":2.6659999999999999,"5min":3.052,"15min":3.5449999999999999},"min":{"1min":0.54900000000000004,"5min":0.32900000000000001,"15min":0.28199999999999997},"max":{"1min":9.1959999999999997,"5min":76.555000000000007,"15min":164.595},"last":2.3769999999999998},{"interface":"front","average":{"1min":2.0449999999999999,"5min":3.1360000000000001,"15min":3.444},"min":{"1min":0.49099999999999999,"5min":0.33200000000000002,"15min":0.313},"max":{"1min":4.6799999999999997,"5min":76.534000000000006,"15min":164.60499999999999},"last":2.3679999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:37:43 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":1.1080000000000001,"15min":0.97199999999999998},"min":{"1min":0.29999999999999999,"5min":0.26700000000000002,"15min":0.23499999999999999},"max":{"1min":0.81899999999999995,"5min":9.6020000000000003,"15min":9.6020000000000003},"last":4.0430000000000001},{"interface":"front","average":{"1min":0.82199999999999995,"5min":1.198,"15min":0.996},"min":{"1min":0.28599999999999998,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":5.1870000000000003,"5min":8.0009999999999994,"15min":8.0009999999999994},"last":0.40899999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:37:18 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.0569999999999999,"15min":1.0069999999999999},"min":{"1min":0.29999999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.7090000000000001,"5min":6.1159999999999997,"15min":6.1159999999999997},"last":1.4139999999999999},{"interface":"front","average":{"1min":0.71399999999999997,"5min":1.1859999999999999,"15min":0.92400000000000004},"min":{"1min":0.34499999999999997,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.0019999999999998,"5min":13.932,"15min":13.932},"last":3.8359999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:37:13 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.371,"15min":0.95999999999999996},"min":{"1min":0.32100000000000001,"5min":0.307,"15min":0.30299999999999999},"max":{"1min":6.069,"5min":11.853999999999999,"15min":11.853999999999999},"last":0.503},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.343,"15min":0.91200000000000003},"min":{"1min":0.38200000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.2389999999999999,"5min":7.266,"15min":7.266},"last":3.9609999999999999}]}]},{"osd":6,"up_from":294,"seq":1262720385163,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":754164,"kb_used_data":424684,"kb_used_omap":6,"kb_used_meta":329401,"kb_avail":92990988,"statfs":{"total":95995035648,"available":95222771712,"internally_reserved":0,"allocated":434876416,"data_stored":433981283,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":337306669},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:38:07 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":2.4279999999999999,"15min":3.5760000000000001},"min":{"1min":0.246,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.7480000000000002,"5min":40.426000000000002,"15min":130.02699999999999},"last":1.044},{"interface":"front","average":{"1min":2.0019999999999998,"5min":2.2290000000000001,"15min":3.3570000000000002},"min":{"1min":0.25700000000000001,"5min":0.221,"15min":0.221},"max":{"1min":7.2030000000000003,"5min":40.447000000000003,"15min":133.72900000000001},"last":24.765999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:38:07 2023","interfaces":[{"interface":"back","average":{"1min":3.7250000000000001,"5min":5.6050000000000004,"15min":3.9980000000000002},"min":{"1min":0.30599999999999999,"5min":0.26700000000000002,"15min":0.23899999999999999},"max":{"1min":18.497,"5min":121.95,"15min":121.95},"last":2.9550000000000001},{"interface":"front","average":{"1min":3.419,"5min":5.4130000000000003,"15min":3.8959999999999999},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.24399999999999999},"max":{"1min":18.445,"5min":121.98099999999999,"15min":121.98099999999999},"last":2.9620000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:38:07 2023","interfaces":[{"interface":"back","average":{"1min":3.5649999999999999,"5min":1.853,"15min":2.2759999999999998},"min":{"1min":0.30399999999999999,"5min":0.27600000000000002,"15min":0.254},"max":{"1min":9.657,"5min":9.657,"15min":26.289999999999999},"last":6.0049999999999999},{"interface":"front","average":{"1min":2.9710000000000001,"5min":1.7,"15min":2.286},"min":{"1min":0.33200000000000002,"5min":0.309,"15min":0.27100000000000002},"max":{"1min":21.704999999999998,"5min":21.704999999999998,"15min":26.388999999999999},"last":3.5950000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:38:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0779999999999998,"5min":2.7229999999999999,"15min":2.6659999999999999},"min":{"1min":0.33500000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.9980000000000002,"5min":89.753,"15min":89.753},"last":0.54900000000000004},{"interface":"front","average":{"1min":2.984,"5min":2.6099999999999999,"15min":2.157},"min":{"1min":0.36399999999999999,"5min":0.318,"15min":0.29299999999999998},"max":{"1min":9.1850000000000005,"5min":89.849999999999994,"15min":89.849999999999994},"last":0.505}]},{"osd":4,"last update":"Sun Oct 1 09:38:07 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.89300000000000002,"15min":0.67400000000000004},"min":{"1min":0.27100000000000002,"5min":0.246,"15min":0.246},"max":{"1min":2.036,"5min":6.0640000000000001,"15min":6.0640000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.83899999999999997,"15min":0.77400000000000002},"min":{"1min":0.35799999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":4.048,"5min":8.8249999999999993,"15min":9.4090000000000007},"last":0.38500000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:37:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.0800000000000001,"15min":1.0580000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.6280000000000001,"5min":12.023999999999999,"15min":12.023999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.95899999999999996,"5min":1.1240000000000001,"15min":1.1279999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.234,"5min":4.5309999999999997,"15min":4.5309999999999997},"last":0.45400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:38:07 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":2.1440000000000001,"15min":3.0720000000000001},"min":{"1min":0.35299999999999998,"5min":0.30499999999999999,"15min":0.27900000000000003},"max":{"1min":4.484,"5min":94.245999999999995,"15min":115.11},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.3380000000000001,"5min":2.3119999999999998,"15min":3.0409999999999999},"min":{"1min":0.39000000000000001,"5min":0.308,"15min":0.28799999999999998},"max":{"1min":6.9699999999999998,"5min":94.715000000000003,"15min":114.666},"last":0.46700000000000003}]}]},{"osd":1,"up_from":94,"seq":403726926140,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1980948,"kb_used_data":1254028,"kb_used_omap":7,"kb_used_meta":726904,"kb_avail":91764204,"statfs":{"total":95995035648,"available":93966544896,"internally_reserved":0,"allocated":1284124672,"data_stored":1282266993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":744350119},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,6,2,4,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:37:47 2023","interfaces":[{"interface":"back","average":{"1min":5.0449999999999999,"5min":6.4269999999999996,"15min":4.4240000000000004},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":22.736999999999998,"5min":154.21299999999999,"15min":154.21299999999999},"last":2.452},{"interface":"front","average":{"1min":5.4329999999999998,"5min":4.6159999999999997,"15min":3.8239999999999998},"min":{"1min":0.27300000000000002,"5min":0.19500000000000001,"15min":0.188},"max":{"1min":25.047999999999998,"5min":95.236999999999995,"15min":95.236999999999995},"last":2.2069999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:37:30 2023","interfaces":[{"interface":"back","average":{"1min":4.6210000000000004,"5min":3.218,"15min":3.9140000000000001},"min":{"1min":0.23100000000000001,"5min":0.20000000000000001,"15min":0.182},"max":{"1min":13.898999999999999,"5min":51.640000000000001,"15min":77.325000000000003},"last":3.0470000000000002},{"interface":"front","average":{"1min":3.9950000000000001,"5min":4.6509999999999998,"15min":4.7430000000000003},"min":{"1min":0.27500000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":15.845000000000001,"5min":154.197,"15min":154.197},"last":3.8290000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:38:06 2023","interfaces":[{"interface":"back","average":{"1min":9.141,"5min":4.7759999999999998,"15min":4.7919999999999998},"min":{"1min":0.40899999999999997,"5min":0.219,"15min":0.219},"max":{"1min":95.305000000000007,"5min":95.305000000000007,"15min":200.56800000000001},"last":5.5110000000000001},{"interface":"front","average":{"1min":10.339,"5min":6.46,"15min":5.3890000000000002},"min":{"1min":0.42299999999999999,"5min":0.23000000000000001,"15min":0.21199999999999999},"max":{"1min":98.033000000000001,"5min":154.483,"15min":200.63800000000001},"last":4.3869999999999996}]},{"osd":4,"last update":"Sun Oct 1 09:37:47 2023","interfaces":[{"interface":"back","average":{"1min":3.9319999999999999,"5min":3.6459999999999999,"15min":2.2120000000000002},"min":{"1min":0.42899999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":18.251999999999999,"5min":154.101,"15min":154.101},"last":0.373},{"interface":"front","average":{"1min":4.069,"5min":3.8959999999999999,"15min":2.6789999999999998},"min":{"1min":0.46999999999999997,"5min":0.32800000000000001,"15min":0.27500000000000002},"max":{"1min":13.773999999999999,"5min":154.56999999999999,"15min":154.56999999999999},"last":2.6560000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:37:25 2023","interfaces":[{"interface":"back","average":{"1min":3.7160000000000002,"5min":3.6230000000000002,"15min":1.8109999999999999},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":18.193999999999999,"5min":154.22999999999999,"15min":154.22999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":3.9609999999999999,"5min":4.024,"15min":2.6899999999999999},"min":{"1min":0.41999999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":13.848000000000001,"5min":154.54400000000001,"15min":154.54400000000001},"last":2.6219999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:38:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2000000000000002,"5min":3.8439999999999999,"15min":3.1200000000000001},"min":{"1min":0.375,"5min":0.313,"15min":0.29899999999999999},"max":{"1min":7.2089999999999996,"5min":154.12200000000001,"15min":154.12200000000001},"last":2.169},{"interface":"front","average":{"1min":4.3710000000000004,"5min":3.9239999999999999,"15min":2.5960000000000001},"min":{"1min":0.46999999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":21.096,"5min":154.16999999999999,"15min":154.16999999999999},"last":0.33200000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:37:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0019999999999998,"5min":4.0170000000000003,"15min":2.5600000000000001},"min":{"1min":0.46100000000000002,"5min":0.33900000000000002,"15min":0.316},"max":{"1min":18.286000000000001,"5min":154.20699999999999,"15min":154.20699999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":4.2619999999999996,"5min":4.3810000000000002,"15min":2.8170000000000002},"min":{"1min":0.43099999999999999,"5min":0.34300000000000003,"15min":0.30199999999999999},"max":{"1min":13.819000000000001,"5min":154.51599999999999,"15min":154.51599999999999},"last":2.9569999999999999}]}]},{"osd":0,"up_from":89,"seq":382252089674,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1386276,"kb_used_data":622812,"kb_used_omap":8,"kb_used_meta":763447,"kb_avail":92358876,"statfs":{"total":95995035648,"available":94575489024,"internally_reserved":0,"allocated":637759488,"data_stored":635892235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":781770690},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,5,0,1,1,1,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:37:55 2023","interfaces":[{"interface":"back","average":{"1min":11.526999999999999,"5min":8.2189999999999994,"15min":4.5229999999999997},"min":{"1min":0.26000000000000001,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":185.78399999999999,"5min":216.821,"15min":216.821},"last":5.0869999999999997},{"interface":"front","average":{"1min":12.972,"5min":8.3070000000000004,"15min":5.3360000000000003},"min":{"1min":0.28199999999999997,"5min":0.246,"15min":0.21199999999999999},"max":{"1min":186.077,"5min":217.02699999999999,"15min":217.02699999999999},"last":1.216}]},{"osd":2,"last update":"Sun Oct 1 09:37:26 2023","interfaces":[{"interface":"back","average":{"1min":4.7539999999999996,"5min":2.778,"15min":2.25},"min":{"1min":0.28599999999999998,"5min":0.21299999999999999,"15min":0.20100000000000001},"max":{"1min":17.462,"5min":17.462,"15min":17.731000000000002},"last":4.7130000000000001},{"interface":"front","average":{"1min":3.536,"5min":2.4359999999999999,"15min":2.887},"min":{"1min":0.34499999999999997,"5min":0.222,"15min":0.189},"max":{"1min":9.4550000000000001,"5min":15.472,"15min":22.07},"last":1.1950000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:37:55 2023","interfaces":[{"interface":"back","average":{"1min":3.919,"5min":5.5250000000000004,"15min":4.7619999999999996},"min":{"1min":0.29399999999999998,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":14.224,"5min":222.16499999999999,"15min":222.16499999999999},"last":4.7380000000000004},{"interface":"front","average":{"1min":4.4539999999999997,"5min":5.1539999999999999,"15min":5.0030000000000001},"min":{"1min":0.32000000000000001,"5min":0.24399999999999999,"15min":0.219},"max":{"1min":11.367000000000001,"5min":222.227,"15min":222.227},"last":1.165}]},{"osd":4,"last update":"Sun Oct 1 09:37:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9369999999999998,"5min":2.7429999999999999,"15min":2.327},"min":{"1min":0.35199999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.1319999999999997,"5min":62.206000000000003,"15min":62.206000000000003},"last":3.3439999999999999},{"interface":"front","average":{"1min":4.0119999999999996,"5min":2.9409999999999998,"15min":2.3530000000000002},"min":{"1min":0.33700000000000002,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":12.222,"5min":62.073999999999998,"15min":62.073999999999998},"last":1.135}]},{"osd":5,"last update":"Sun Oct 1 09:37:17 2023","interfaces":[{"interface":"back","average":{"1min":3.1699999999999999,"5min":2.3210000000000002,"15min":1.9470000000000001},"min":{"1min":0.48699999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":12.268000000000001,"5min":16.808,"15min":16.808},"last":4.5679999999999996},{"interface":"front","average":{"1min":3.911,"5min":2.3140000000000001,"15min":1.859},"min":{"1min":0.35299999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":17.507000000000001,"5min":17.507000000000001,"15min":17.507000000000001},"last":4.6459999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:38:15 2023","interfaces":[{"interface":"back","average":{"1min":2.7130000000000001,"5min":2.3730000000000002,"15min":1.8720000000000001},"min":{"1min":0.39200000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":8.9920000000000009,"5min":17.535,"15min":17.535},"last":4.7619999999999996},{"interface":"front","average":{"1min":3.2989999999999999,"5min":2.3730000000000002,"15min":2.399},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":8.8819999999999997,"5min":15.651999999999999,"15min":18.521000000000001},"last":5.1120000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:37:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1659999999999999,"5min":2.9630000000000001,"15min":2.0409999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.34499999999999997},"max":{"1min":8.923,"5min":35.130000000000003,"15min":35.130000000000003},"last":4.6829999999999998},{"interface":"front","average":{"1min":3.3559999999999999,"5min":2.8610000000000002,"15min":2.4740000000000002},"min":{"1min":0.39700000000000002,"5min":0.38200000000000001,"15min":0.35099999999999998},"max":{"1min":8.8339999999999996,"5min":38.590000000000003,"15min":38.590000000000003},"last":3.423}]}]},{"osd":2,"up_from":99,"seq":425201762613,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1086372,"kb_used_data":375388,"kb_used_omap":11,"kb_used_meta":710836,"kb_avail":92658780,"statfs":{"total":95995035648,"available":94882590720,"internally_reserved":0,"allocated":384397312,"data_stored":382586689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":727896951},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:37:40 2023","interfaces":[{"interface":"back","average":{"1min":7.923,"5min":7.4219999999999997,"15min":7.2469999999999999},"min":{"1min":0.24399999999999999,"5min":0.191,"15min":0.191},"max":{"1min":75.989000000000004,"5min":201.892,"15min":224.56},"last":0.70199999999999996},{"interface":"front","average":{"1min":8.4109999999999996,"5min":7.6319999999999997,"15min":7.6950000000000003},"min":{"1min":0.221,"5min":0.221,"15min":0.20699999999999999},"max":{"1min":73.721999999999994,"5min":200.089,"15min":225.66900000000001},"last":8.6460000000000008}]},{"osd":1,"last update":"Sun Oct 1 09:37:32 2023","interfaces":[{"interface":"back","average":{"1min":6.9980000000000002,"5min":12.176,"15min":8.5950000000000006},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.191},"max":{"1min":65.566999999999993,"5min":196.34399999999999,"15min":224.917},"last":0.38600000000000001},{"interface":"front","average":{"1min":8.5939999999999994,"5min":12.744999999999999,"15min":9.532},"min":{"1min":0.30099999999999999,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":65.733999999999995,"5min":196.542,"15min":225.625},"last":1.0740000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:38:00 2023","interfaces":[{"interface":"back","average":{"1min":4.8470000000000004,"5min":3.0350000000000001,"15min":5.0949999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":30.039000000000001,"5min":50.289000000000001,"15min":226.124},"last":0.81200000000000006},{"interface":"front","average":{"1min":2.931,"5min":2.734,"15min":4.8319999999999999},"min":{"1min":0.245,"5min":0.215,"15min":0.20300000000000001},"max":{"1min":17.515000000000001,"5min":54.115000000000002,"15min":226.32900000000001},"last":0.89700000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:37:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":1.5800000000000001,"15min":2.5920000000000001},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.27200000000000002},"max":{"1min":4.9089999999999998,"5min":22.084,"15min":224.90199999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.4209999999999998,"5min":2.0550000000000002,"15min":2.8700000000000001},"min":{"1min":0.33200000000000002,"5min":0.27800000000000002,"15min":0.27500000000000002},"max":{"1min":11.475,"5min":22.266999999999999,"15min":225.386},"last":0.78400000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:37:16 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.508,"15min":1.6100000000000001},"min":{"1min":0.309,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":11.305999999999999,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.0459999999999998,"5min":1.3779999999999999,"15min":1.339},"min":{"1min":0.441,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.8380000000000001,"5min":10.194000000000001,"15min":10.194000000000001},"last":0.623}]},{"osd":6,"last update":"Sun Oct 1 09:38:06 2023","interfaces":[{"interface":"back","average":{"1min":3.9409999999999998,"5min":2.1040000000000001,"15min":2.98},"min":{"1min":0.36799999999999999,"5min":0.34100000000000003,"15min":0.317},"max":{"1min":11.429,"5min":11.526999999999999,"15min":225.41499999999999},"last":0.746},{"interface":"front","average":{"1min":1.8460000000000001,"5min":1.5529999999999999,"15min":2.4279999999999999},"min":{"1min":0.36499999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.3309999999999995,"5min":13.614000000000001,"15min":225.02000000000001},"last":0.45200000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:37:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":4.7560000000000002,"15min":3.851},"min":{"1min":0.372,"5min":0.31,"15min":0.308},"max":{"1min":11.337999999999999,"5min":139.03299999999999,"15min":224.94999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.8540000000000001,"5min":4.524,"15min":3.9249999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.34499999999999997},"max":{"1min":5.8150000000000004,"5min":139.065,"15min":224.92599999999999},"last":0.58799999999999997}]}]},{"osd":3,"up_from":104,"seq":446676599087,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2383384,"kb_used_data":1641872,"kb_used_omap":3,"kb_used_meta":741500,"kb_avail":91361768,"statfs":{"total":95995035648,"available":93554450432,"internally_reserved":0,"allocated":1681276928,"data_stored":1679357333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":759296743},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:38:00 2023","interfaces":[{"interface":"back","average":{"1min":3.5419999999999998,"5min":4.2489999999999997,"15min":5.7400000000000002},"min":{"1min":0.20599999999999999,"5min":0.186,"15min":0.18099999999999999},"max":{"1min":11.224,"5min":137.34399999999999,"15min":137.34399999999999},"last":0.89100000000000001},{"interface":"front","average":{"1min":4.4539999999999997,"5min":4.3710000000000004,"15min":5.6840000000000002},"min":{"1min":0.36699999999999999,"5min":0.20899999999999999,"15min":0.184},"max":{"1min":17.628,"5min":137.73400000000001,"15min":137.73400000000001},"last":1.3540000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:38:00 2023","interfaces":[{"interface":"back","average":{"1min":9.0719999999999992,"5min":8.7940000000000005,"15min":5.8570000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.20399999999999999},"max":{"1min":108.84999999999999,"5min":162.22999999999999,"15min":162.22999999999999},"last":4.3179999999999996},{"interface":"front","average":{"1min":11.337999999999999,"5min":9.1829999999999998,"15min":6.3460000000000001},"min":{"1min":0.40500000000000003,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":109.13200000000001,"5min":162.70699999999999,"15min":162.70699999999999},"last":1.391}]},{"osd":2,"last update":"Sun Oct 1 09:38:00 2023","interfaces":[{"interface":"back","average":{"1min":3.5350000000000001,"5min":2.1480000000000001,"15min":3.1850000000000001},"min":{"1min":0.23000000000000001,"5min":0.22800000000000001,"15min":0.20799999999999999},"max":{"1min":12.361000000000001,"5min":14.194000000000001,"15min":62.909999999999997},"last":13.108000000000001},{"interface":"front","average":{"1min":3.0249999999999999,"5min":2.4009999999999998,"15min":3.8300000000000001},"min":{"1min":0.29299999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":11.202,"5min":15.523,"15min":78.594999999999999},"last":0.59299999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:38:00 2023","interfaces":[{"interface":"back","average":{"1min":3.113,"5min":2.2629999999999999,"15min":2.004},"min":{"1min":0.32700000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":11.153,"5min":29.419,"15min":29.419},"last":0.54100000000000004},{"interface":"front","average":{"1min":2.2290000000000001,"5min":2.4710000000000001,"15min":2.1720000000000002},"min":{"1min":0.28699999999999998,"5min":0.27900000000000003,"15min":0.23899999999999999},"max":{"1min":7.617,"5min":33.002000000000002,"15min":36.947000000000003},"last":1.325}]},{"osd":5,"last update":"Sun Oct 1 09:37:16 2023","interfaces":[{"interface":"back","average":{"1min":2.589,"5min":2.7069999999999999,"15min":3.4910000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":9.0839999999999996,"5min":28.117999999999999,"15min":28.117999999999999},"last":1.294},{"interface":"front","average":{"1min":3.6640000000000001,"5min":2.9249999999999998,"15min":3.4580000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":14.537000000000001,"5min":28.34,"15min":28.34},"last":12.989000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:38:13 2023","interfaces":[{"interface":"back","average":{"1min":3.7360000000000002,"5min":2.323,"15min":2.4729999999999999},"min":{"1min":0.40100000000000002,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":13.032999999999999,"5min":13.032999999999999,"15min":18.361000000000001},"last":13.032999999999999},{"interface":"front","average":{"1min":3.3849999999999998,"5min":2.3370000000000002,"15min":2.3969999999999998},"min":{"1min":0.42599999999999999,"5min":0.33600000000000002,"15min":0.32800000000000001},"max":{"1min":13.057,"5min":13.057,"15min":18.332000000000001},"last":13.057}]},{"osd":7,"last update":"Sun Oct 1 09:38:00 2023","interfaces":[{"interface":"back","average":{"1min":3.677,"5min":2.5070000000000001,"15min":2.6930000000000001},"min":{"1min":0.37,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":12.429,"5min":12.429,"15min":52.006},"last":13.085000000000001},{"interface":"front","average":{"1min":3.0579999999999998,"5min":2.3260000000000001,"15min":2.5510000000000002},"min":{"1min":0.38600000000000001,"5min":0.311,"15min":0.311},"max":{"1min":11.109999999999999,"5min":13.191000000000001,"15min":51.890999999999998},"last":0.57099999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019337,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1468412,"kb_used_data":582324,"kb_used_omap":0,"kb_used_meta":886080,"kb_avail":92276740,"statfs":{"total":95995035648,"available":94491381760,"internally_reserved":0,"allocated":596299776,"data_stored":595396594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":907345920},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:37:48 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":6.0030000000000001,"15min":4.5339999999999998},"min":{"1min":0.28699999999999998,"5min":0.22600000000000001,"15min":0.223},"max":{"1min":9.2690000000000001,"5min":155.315,"15min":155.315},"last":0.45300000000000001},{"interface":"front","average":{"1min":3.6850000000000001,"5min":6.6390000000000002,"15min":4.806},"min":{"1min":0.253,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":12.119999999999999,"5min":155.334,"15min":155.334},"last":0.66700000000000004}]},{"osd":1,"last update":"Sun Oct 1 09:37:52 2023","interfaces":[{"interface":"back","average":{"1min":6.734,"5min":7.6390000000000002,"15min":6.3170000000000002},"min":{"1min":0.28299999999999997,"5min":0.25,"15min":0.25},"max":{"1min":88.885999999999996,"5min":190.61199999999999,"15min":190.61199999999999},"last":6.0960000000000001},{"interface":"front","average":{"1min":6.5570000000000004,"5min":9.8239999999999998,"15min":7.0449999999999999},"min":{"1min":0.30399999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":88.829999999999998,"5min":253.273,"15min":253.273},"last":6.1029999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:37:33 2023","interfaces":[{"interface":"back","average":{"1min":2.4340000000000002,"5min":4.0709999999999997,"15min":3.032},"min":{"1min":0.29099999999999998,"5min":0.28299999999999997,"15min":0.26700000000000002},"max":{"1min":8.6379999999999999,"5min":170.45400000000001,"15min":170.45400000000001},"last":1.1100000000000001},{"interface":"front","average":{"1min":3.327,"5min":2.8399999999999999,"15min":2.6720000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.25800000000000001},"max":{"1min":8.9540000000000006,"5min":36.515999999999998,"15min":58.771000000000001},"last":6.7030000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:37:52 2023","interfaces":[{"interface":"back","average":{"1min":2.3519999999999999,"5min":3.8500000000000001,"15min":3.4660000000000002},"min":{"1min":0.33800000000000002,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":15.989000000000001,"5min":140.881,"15min":140.881},"last":5.7569999999999997},{"interface":"front","average":{"1min":3.556,"5min":4.0709999999999997,"15min":3.7229999999999999},"min":{"1min":0.36799999999999999,"5min":0.317,"15min":0.30399999999999999},"max":{"1min":15.978999999999999,"5min":140.911,"15min":140.911},"last":0.61399999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:37:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":1.526,"15min":1.232},"min":{"1min":0.32100000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.8929999999999998,"5min":20.457000000000001,"15min":20.457000000000001},"last":0.377},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.397,"15min":0.96499999999999997},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.899,"5min":20.475000000000001,"15min":20.475000000000001},"last":0.437}]},{"osd":6,"last update":"Sun Oct 1 09:38:04 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.123,"15min":0.89600000000000002},"min":{"1min":0.34300000000000003,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":3.653,"5min":7.9710000000000001,"15min":11.358000000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.272,"5min":1.4039999999999999,"15min":0.95499999999999996},"min":{"1min":0.32700000000000001,"5min":0.32500000000000001,"15min":0.28999999999999998},"max":{"1min":6.9139999999999997,"5min":7.5229999999999997,"15min":10.076000000000001},"last":0.503}]},{"osd":7,"last update":"Sun Oct 1 09:37:33 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":1.54,"15min":1.034},"min":{"1min":0.33100000000000002,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":3.5899999999999999,"5min":22.256,"15min":22.256},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.7190000000000001,"5min":1.927,"15min":1.3129999999999999},"min":{"1min":0.39100000000000001,"5min":0.308,"15min":0.28699999999999998},"max":{"1min":9,"5min":22.635999999999999,"15min":22.635999999999999},"last":0.53100000000000003}]}]},{"osd":5,"up_from":379,"seq":1627792605247,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1985716,"kb_used_data":1608876,"kb_used_omap":13,"kb_used_meta":376818,"kb_avail":91759436,"statfs":{"total":95995035648,"available":93961662464,"internally_reserved":0,"allocated":1647489024,"data_stored":1646589528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":385862130},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":3.4140000000000001,"15min":2.141},"min":{"1min":0.23499999999999999,"5min":0.221,"15min":0.221},"max":{"1min":6.4870000000000001,"5min":84.340999999999994,"15min":84.340999999999994},"last":4.5739999999999998},{"interface":"front","average":{"1min":1.7729999999999999,"5min":3.7559999999999998,"15min":2.0720000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":5.6109999999999998,"5min":84.356999999999999,"15min":84.356999999999999},"last":4.5110000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":9.0960000000000001,"15min":4.1639999999999997},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":4.7670000000000003,"5min":236.488,"15min":236.488},"last":4.3460000000000001},{"interface":"front","average":{"1min":3.234,"5min":8.9900000000000002,"15min":3.9260000000000002},"min":{"1min":0.27300000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":12.132999999999999,"5min":236.29900000000001,"15min":236.29900000000001},"last":4.226}]},{"osd":2,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7090000000000001,"5min":2.1309999999999998,"15min":1.579},"min":{"1min":0.32500000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.5620000000000003,"5min":14.199999999999999,"15min":14.199999999999999},"last":3.9700000000000002},{"interface":"front","average":{"1min":2.028,"5min":1.821,"15min":1.6879999999999999},"min":{"1min":0.31,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.298,"5min":14.430999999999999,"15min":14.430999999999999},"last":3.8650000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":2.012,"5min":1.7949999999999999,"15min":1.373},"min":{"1min":0.31,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.4800000000000004,"5min":14.004,"15min":14.004},"last":3.1179999999999999},{"interface":"front","average":{"1min":1.9910000000000001,"5min":2.1469999999999998,"15min":1.5009999999999999},"min":{"1min":0.38100000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":8.4930000000000003,"5min":13.991,"15min":13.991},"last":3.137}]},{"osd":4,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":1.157,"15min":1.0109999999999999},"min":{"1min":0.27900000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.2120000000000002,"5min":12.808999999999999,"15min":12.808999999999999},"last":0.37},{"interface":"front","average":{"1min":0.754,"5min":1.256,"15min":1.5289999999999999},"min":{"1min":0.32500000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.843,"5min":14.68,"15min":14.68},"last":0.48799999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":0.93200000000000005,"15min":1},"min":{"1min":0.29899999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.5690000000000008,"5min":8.5690000000000008,"15min":8.5690000000000008},"last":0.376},{"interface":"front","average":{"1min":0.59699999999999998,"5min":1.4179999999999999,"15min":1.47},"min":{"1min":0.316,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.968,"5min":14.193,"15min":14.193},"last":0.39100000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.3049999999999999,"15min":1.347},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":2.1309999999999998,"5min":10.813000000000001,"15min":10.813000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.80500000000000005,"5min":1.113,"15min":0.96699999999999997},"min":{"1min":0.34899999999999998,"5min":0.318,"15min":0.318},"max":{"1min":3.4769999999999999,"5min":12.045999999999999,"15min":12.045999999999999},"last":0.40100000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":623706112,"data_stored":623706112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1271664640,"data_stored":1271660568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":375914496,"data_stored":375914496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1670643712,"data_stored":1670643712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":587595776,"data_stored":587595776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1631391744,"data_stored":1631387672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":426704896,"data_stored":426704896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1746997248,"data_stored":1746993176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:38:24.046 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-01T09:38:24.046 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 21 2023-10-01T09:38:24.103 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:24.230 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:24.393 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:24.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:24.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:24.500 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:24.541 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:38:24.541 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:38:24.541 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:38:24.542 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:38:24.542 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:38:24.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:24 smithi138 ceph-mon[185886]: pgmap v1288: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 16575/102414 objects degraded (16.184%); 17816/102414 objects misplaced (17.396%); 5.8 MiB/s, 92 objects/s recovering 2023-10-01T09:38:24.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:24 smithi138 ceph-mon[192513]: pgmap v1288: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 16575/102414 objects degraded (16.184%); 17816/102414 objects misplaced (17.396%); 5.8 MiB/s, 92 objects/s recovering 2023-10-01T09:38:24.576 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:24.753 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:38:24.784 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:24.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:24.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:24.796 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:38:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:24 smithi190 ceph-mon[160435]: pgmap v1288: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 16575/102414 objects degraded (16.184%); 17816/102414 objects misplaced (17.396%); 5.8 MiB/s, 92 objects/s recovering 2023-10-01T09:38:24.832 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:38:24.879 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:38:24.897 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:24.918 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:38:24.975 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:38:25.009 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:25.028 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:38:25.070 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:38:25.090 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:38:25.090 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:38:25.090 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:38:25.090 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:38:25.090 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:38:25.090 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:38:25.091 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:38:25.091 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:38:25.112 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:25.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:25.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:25.218 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:25.241 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:38:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:38:24] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.33.4" 2023-10-01T09:38:25.244 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:25.360 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:25.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:25.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:25.470 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:38:25.554 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:25 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4172580901' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:25.554 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:25.554 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:38:25.554 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:25 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4172580901' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:25.554 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:25.555 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:38:25.562 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:25 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4172580901' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:38:25.964 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:38:26.134 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:38:26.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:26.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:38:26.243 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:26.281 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:26.339 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:26.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:26.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:26.445 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:26.484 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:26.518 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:38:26.538 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:26.542 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:26 smithi190 ceph-mon[160435]: pgmap v1289: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 16187/97254 objects degraded (16.644%); 17081/97254 objects misplaced (17.563%); 6.1 MiB/s, 97 objects/s recovering 2023-10-01T09:38:26.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:26.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:26.645 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:26.664 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:26.697 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:26.756 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:26.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:26.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:26.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:26 smithi138 ceph-mon[185886]: pgmap v1289: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 16187/97254 objects degraded (16.644%); 17081/97254 objects misplaced (17.563%); 6.1 MiB/s, 97 objects/s recovering 2023-10-01T09:38:26.824 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:26 smithi138 ceph-mon[192513]: pgmap v1289: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 413 op/s; 16187/97254 objects degraded (16.644%); 17081/97254 objects misplaced (17.563%); 6.1 MiB/s, 97 objects/s recovering 2023-10-01T09:38:26.863 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:26.901 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:26.904 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:26.965 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:26.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:26.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:27.071 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:27.156 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:27.322 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:27.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:27.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:27.440 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:27.527 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:27.714 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:27 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 16187/97254 objects degraded (16.644%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:28.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:27 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 16187/97254 objects degraded (16.644%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:28.068 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:27 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 16187/97254 objects degraded (16.644%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:28.183 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:28.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:28.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:28.302 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:28.356 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:28.428 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:28.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:28.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:28.534 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:28.563 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:38:28.564 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:28.611 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:28.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:28.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:28.715 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:28.744 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:28 smithi190 ceph-mon[160435]: pgmap v1290: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 16187/96066 objects degraded (16.850%); 17081/96066 objects misplaced (17.780%); 6.1 MiB/s, 97 objects/s recovering 2023-10-01T09:38:28.810 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:28.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:28 smithi138 ceph-mon[192513]: pgmap v1290: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 16187/96066 objects degraded (16.850%); 17081/96066 objects misplaced (17.780%); 6.1 MiB/s, 97 objects/s recovering 2023-10-01T09:38:28.829 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:28 smithi138 ceph-mon[185886]: pgmap v1290: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 16187/96066 objects degraded (16.850%); 17081/96066 objects misplaced (17.780%); 6.1 MiB/s, 97 objects/s recovering 2023-10-01T09:38:29.557 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:29.558 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:29.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:29.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:29.677 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:29.709 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:29.768 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:29.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:29.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:29.877 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:29.925 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:29.997 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:30.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:30.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:30.105 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:30.146 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:30.210 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:30.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:30.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:30.316 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:30.345 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:30.415 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:30.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:30.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:30.514 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:30.522 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:30.674 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:30.972 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T09:38:31.254 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:38:31.255 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:31 smithi138 ceph-mon[192513]: pgmap v1291: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 421 op/s; 15765/90858 objects degraded (17.351%); 16339/90858 objects misplaced (17.983%); 6.3 MiB/s, 100 objects/s recovering 2023-10-01T09:38:31.255 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3994606689' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:38:31.255 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3603171579' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-01T09:38:31.255 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:31 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-01T09:38:31.256 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:31 smithi138 ceph-mon[185886]: pgmap v1291: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 421 op/s; 15765/90858 objects degraded (17.351%); 16339/90858 objects misplaced (17.983%); 6.3 MiB/s, 100 objects/s recovering 2023-10-01T09:38:31.256 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3994606689' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:38:31.256 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3603171579' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-01T09:38:31.256 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:31 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-01T09:38:31.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:31 smithi190 ceph-mon[160435]: pgmap v1291: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 421 op/s; 15765/90858 objects degraded (17.351%); 16339/90858 objects misplaced (17.983%); 6.3 MiB/s, 100 objects/s recovering 2023-10-01T09:38:31.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3994606689' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:38:31.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3603171579' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-01T09:38:31.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:31 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-01T09:38:32.258 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:32 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:32.258 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:32 smithi138 ceph-mon[185886]: failed to encode map e457 with expected crc 2023-10-01T09:38:32.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:32 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:32 smithi190 ceph-mon[160435]: failed to encode map e457 with expected crc 2023-10-01T09:38:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:32 smithi190 ceph-mon[160435]: failed to encode map e457 with expected crc 2023-10-01T09:38:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:32 smithi190 ceph-mon[160435]: failed to encode map e457 with expected crc 2023-10-01T09:38:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:32 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3994606689' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:38:32.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:32 smithi190 ceph-mon[160435]: osdmap e457: 8 total, 8 up, 8 in 2023-10-01T09:38:32.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:32 smithi138 ceph-mon[185886]: failed to encode map e457 with expected crc 2023-10-01T09:38:32.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:32 smithi138 ceph-mon[185886]: failed to encode map e457 with expected crc 2023-10-01T09:38:32.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:32 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3994606689' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:38:32.575 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:32 smithi138 ceph-mon[185886]: osdmap e457: 8 total, 8 up, 8 in 2023-10-01T09:38:32.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:32 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:32.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:32 smithi138 ceph-mon[192513]: failed to encode map e457 with expected crc 2023-10-01T09:38:32.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:32 smithi138 ceph-mon[192513]: failed to encode map e457 with expected crc 2023-10-01T09:38:32.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:32 smithi138 ceph-mon[192513]: failed to encode map e457 with expected crc 2023-10-01T09:38:32.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:32 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3994606689' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:38:32.576 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:32 smithi138 ceph-mon[192513]: osdmap e457: 8 total, 8 up, 8 in 2023-10-01T09:38:32.661 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:32.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:32.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:32.771 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:32.804 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:32.881 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:32.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:32.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:32.995 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:33.022 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:33.284 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:33 smithi138 ceph-mon[185886]: pgmap v1293: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 15765/89955 objects degraded (17.525%); 16339/89955 objects misplaced (18.164%); 6.3 MiB/s, 100 objects/s recovering 2023-10-01T09:38:33.285 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:33 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 15765/89955 objects degraded (17.525%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:33.504 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:33.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:33.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:33.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:33 smithi190 ceph-mon[160435]: pgmap v1293: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 15765/89955 objects degraded (17.525%); 16339/89955 objects misplaced (18.164%); 6.3 MiB/s, 100 objects/s recovering 2023-10-01T09:38:33.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:33 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 15765/89955 objects degraded (17.525%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:33 smithi138 ceph-mon[192513]: pgmap v1293: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 15765/89955 objects degraded (17.525%); 16339/89955 objects misplaced (18.164%); 6.3 MiB/s, 100 objects/s recovering 2023-10-01T09:38:33.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:33 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 15765/89955 objects degraded (17.525%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:33.622 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:38:33.691 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T09:38:33.735 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:34.751 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:38:34.761 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:38:34.761 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:38:35.051 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:38:35.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:35.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:38:35.173 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:35.214 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:38:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:38:34] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.33.4" 2023-10-01T09:38:35.251 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:35.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:35 smithi190 ceph-mon[160435]: pgmap v1294: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 15765/89589 objects degraded (17.597%); 16339/89589 objects misplaced (18.238%); 6.3 MiB/s, 100 objects/s recovering 2023-10-01T09:38:35.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:35 smithi138 ceph-mon[185886]: pgmap v1294: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 15765/89589 objects degraded (17.597%); 16339/89589 objects misplaced (18.238%); 6.3 MiB/s, 100 objects/s recovering 2023-10-01T09:38:35.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:35 smithi138 ceph-mon[192513]: pgmap v1294: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 15765/89589 objects degraded (17.597%); 16339/89589 objects misplaced (18.238%); 6.3 MiB/s, 100 objects/s recovering 2023-10-01T09:38:35.868 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:35.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:35.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:35.976 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:38:35.977 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:36.089 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:36.124 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:36.287 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:36.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:36.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:36.405 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:36.443 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:36.503 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:36.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:36.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:36.610 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:38:36.650 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:36.715 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:38:36.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:36.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:38:36.728 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:38:36.769 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:38:36.812 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:38:36.828 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:36.852 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:38:36.860 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:36.892 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:38:36.927 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:38:36.928 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:36.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:36.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:36.963 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:38:37.006 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:38:37.026 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:38:37.026 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:38:37.026 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:38:37.026 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:38:37.026 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:38:37.027 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:38:37.027 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:38:37.027 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:38:37.040 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:37.146 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:37.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:37 smithi138 ceph-mon[185886]: pgmap v1295: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 15400/84549 objects degraded (18.214%); 15589/84549 objects misplaced (18.438%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:37.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:37 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/70427326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:37.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:37 smithi190 ceph-mon[160435]: pgmap v1295: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 15400/84549 objects degraded (18.214%); 15589/84549 objects misplaced (18.438%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:37 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/70427326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:37.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:37 smithi138 ceph-mon[192513]: pgmap v1295: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 15400/84549 objects degraded (18.214%); 15589/84549 objects misplaced (18.438%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:37.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:37 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/70427326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:38.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:38 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 15400/84549 objects degraded (18.214%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:38.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:38 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 15400/84549 objects degraded (18.214%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:38.594 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:38.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:38.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:38.693 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 2, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:38:38.693 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:38:38.694 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-01T09:38:38.694 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:38:38.702 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:38 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 15400/84549 objects degraded (18.214%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:38.861 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:38.866 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:39.077 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:39.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:39.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:39.185 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:39.221 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:39.287 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:39.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:39.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:39.334 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:39 smithi138 ceph-mon[192513]: pgmap v1296: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 15400/83340 objects degraded (18.479%); 15589/83340 objects misplaced (18.705%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:39.334 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:39 smithi138 ceph-mon[185886]: pgmap v1296: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 15400/83340 objects degraded (18.479%); 15589/83340 objects misplaced (18.705%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:39.395 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:39.493 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:39.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:39 smithi190 ceph-mon[160435]: pgmap v1296: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 15400/83340 objects degraded (18.479%); 15589/83340 objects misplaced (18.705%); 6.3 MiB/s, 101 objects/s recovering 2023-10-01T09:38:39.759 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:39.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:39.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:39.878 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:39.911 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:39.976 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:39.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:39.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:40.081 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:38:40.117 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:40.199 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:38:40.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:40.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:38:40.307 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:40.331 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:38:40.339 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:40.381 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:40.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:40.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:40.485 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:40.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:40 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/880501653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:38:40.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:40.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:38:40.570 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:40.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:40 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/880501653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:38:40.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:40.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:38:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:40 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/880501653' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:38:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:38:40.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:38:40.867 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:38:41.710 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:41.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:41.715 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:41 smithi138 ceph-mon[185886]: pgmap v1297: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 14990/77901 objects degraded (19.242%); 14744/77901 objects misplaced (18.927%); 6.5 MiB/s, 104 objects/s recovering 2023-10-01T09:38:41.715 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:41 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:41.715 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:41 smithi138 ceph-mon[185886]: failed to encode map e458 with expected crc 2023-10-01T09:38:41.715 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:41 smithi138 ceph-mon[185886]: failed to encode map e458 with expected crc 2023-10-01T09:38:41.715 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:41 smithi138 ceph-mon[185886]: failed to encode map e458 with expected crc 2023-10-01T09:38:41.715 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:41 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/880501653' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:38:41.715 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:41 smithi138 ceph-mon[185886]: osdmap e458: 8 total, 8 up, 8 in 2023-10-01T09:38:41.716 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:41 smithi138 ceph-mon[192513]: pgmap v1297: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 14990/77901 objects degraded (19.242%); 14744/77901 objects misplaced (18.927%); 6.5 MiB/s, 104 objects/s recovering 2023-10-01T09:38:41.716 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:41 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:41.716 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:41 smithi138 ceph-mon[192513]: failed to encode map e458 with expected crc 2023-10-01T09:38:41.716 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:41 smithi138 ceph-mon[192513]: failed to encode map e458 with expected crc 2023-10-01T09:38:41.716 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:41 smithi138 ceph-mon[192513]: failed to encode map e458 with expected crc 2023-10-01T09:38:41.716 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:41 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/880501653' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:38:41.716 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:41 smithi138 ceph-mon[192513]: osdmap e458: 8 total, 8 up, 8 in 2023-10-01T09:38:41.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:41.773 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:41 smithi190 ceph-mon[160435]: pgmap v1297: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 14990/77901 objects degraded (19.242%); 14744/77901 objects misplaced (18.927%); 6.5 MiB/s, 104 objects/s recovering 2023-10-01T09:38:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:41 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:41 smithi190 ceph-mon[160435]: failed to encode map e458 with expected crc 2023-10-01T09:38:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:41 smithi190 ceph-mon[160435]: failed to encode map e458 with expected crc 2023-10-01T09:38:41.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:41 smithi190 ceph-mon[160435]: failed to encode map e458 with expected crc 2023-10-01T09:38:41.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:41 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/880501653' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:38:41.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:41 smithi190 ceph-mon[160435]: osdmap e458: 8 total, 8 up, 8 in 2023-10-01T09:38:41.817 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:41.867 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:41.921 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:41.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:41.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:42.031 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:42.140 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:42.322 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:42.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:42.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:42.327 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:42.429 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:42.479 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:42.495 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:38:42.495 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:38:42.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.495 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:180244 9.2 9:404d4039:::benchmark_data_smithi138_497829_object77859:head [delete] snapc 0=[] ondisk+write+known_if_redirected e458)", 2023-10-01T09:38:42.495 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:38:42.462428+0000", 2023-10-01T09:38:42.495 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.031522302000000002, 2023-10-01T09:38:42.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.031553378999999999, 2023-10-01T09:38:42.495 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:42.495 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:38:42.495 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:38:42.496 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:38:42.496 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:38:42.496 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 180244 2023-10-01T09:38:42.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.496 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:42.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:42.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.462428+0000", 2023-10-01T09:38:42.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:42.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.462428+0000", 2023-10-01T09:38:42.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:42.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.462428+0000", 2023-10-01T09:38:42.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:38:42.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:42.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.462430+0000", 2023-10-01T09:38:42.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4439999999999998e-06 2023-10-01T09:38:42.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:42.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.462436+0000", 2023-10-01T09:38:42.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4399999999999996e-06 2023-10-01T09:38:42.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:42.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.462440+0000", 2023-10-01T09:38:42.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3680000000000004e-06 2023-10-01T09:38:42.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:42.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.475112+0000", 2023-10-01T09:38:42.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012672022 2023-10-01T09:38:42.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:38:42.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.475293+0000", 2023-10-01T09:38:42.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000180874 2023-10-01T09:38:42.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,7", 2023-10-01T09:38:42.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.475460+0000", 2023-10-01T09:38:42.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016637299999999999 2023-10-01T09:38:42.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:42.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.492705+0000", 2023-10-01T09:38:42.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017245000999999999 2023-10-01T09:38:42.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:42.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.492778+0000", 2023-10-01T09:38:42.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3266e-05 2023-10-01T09:38:42.500 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.500 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:42.500 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.500 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:180249 9.4 9:262d7f77:::benchmark_data_smithi138_497829_object77864:head [delete] snapc 0=[] ondisk+write+known_if_redirected e458)", 2023-10-01T09:38:42.501 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:38:42.477362+0000", 2023-10-01T09:38:42.501 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.016588960999999999, 2023-10-01T09:38:42.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016713193000000001, 2023-10-01T09:38:42.501 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:42.501 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:38:42.501 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:38:42.501 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:38:42.501 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:38:42.501 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 180249 2023-10-01T09:38:42.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.501 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:42.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:42.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477362+0000", 2023-10-01T09:38:42.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:42.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477362+0000", 2023-10-01T09:38:42.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:42.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477360+0000", 2023-10-01T09:38:42.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:38:42.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:42.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477367+0000", 2023-10-01T09:38:42.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.392e-06 2023-10-01T09:38:42.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:42.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477380+0000", 2023-10-01T09:38:42.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3332999999999999e-05 2023-10-01T09:38:42.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:42.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477394+0000", 2023-10-01T09:38:42.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4253000000000001e-05 2023-10-01T09:38:42.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:42.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477618+0000", 2023-10-01T09:38:42.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000223878 2023-10-01T09:38:42.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.504 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:38:42.504 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477748+0000", 2023-10-01T09:38:42.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012952300000000001 2023-10-01T09:38:42.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:38:42.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477870+0000", 2023-10-01T09:38:42.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000122483 2023-10-01T09:38:42.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.505 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:38:42.505 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.488140+0000", 2023-10-01T09:38:42.505 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010270227 2023-10-01T09:38:42.505 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.505 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:42.505 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.505 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.505 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.506 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:180250 9.a 9:575e3b29:::benchmark_data_smithi138_497829_object77865:head [delete] snapc 0=[] ondisk+write+known_if_redirected e458)", 2023-10-01T09:38:42.506 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:38:42.477409+0000", 2023-10-01T09:38:42.506 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.016541381000000001, 2023-10-01T09:38:42.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016743811000000001, 2023-10-01T09:38:42.506 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:42.506 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "delayed", 2023-10-01T09:38:42.506 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:38:42.506 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:38:42.506 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:38:42.506 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 180250 2023-10-01T09:38:42.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.506 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:42.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:42.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477409+0000", 2023-10-01T09:38:42.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:42.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477409+0000", 2023-10-01T09:38:42.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:42.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477409+0000", 2023-10-01T09:38:42.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:38:42.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:42.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477412+0000", 2023-10-01T09:38:42.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5489999999999998e-06 2023-10-01T09:38:42.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:42.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477420+0000", 2023-10-01T09:38:42.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5050000000000001e-06 2023-10-01T09:38:42.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:42.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477426+0000", 2023-10-01T09:38:42.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8749999999999997e-06 2023-10-01T09:38:42.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:42.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.486924+0000", 2023-10-01T09:38:42.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094980350000000002 2023-10-01T09:38:42.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:38:42.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.491676+0000", 2023-10-01T09:38:42.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047518150000000004 2023-10-01T09:38:42.509 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.509 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:42.510 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.510 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:180252 9.e 9:7fffb5e3:::benchmark_data_smithi138_497829_object77867:head [delete] snapc 0=[] ondisk+write+known_if_redirected e458)", 2023-10-01T09:38:42.510 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:38:42.477434+0000", 2023-10-01T09:38:42.510 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.016517164000000001, 2023-10-01T09:38:42.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016794782000000001, 2023-10-01T09:38:42.510 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:42.510 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:38:42.510 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:38:42.510 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:38:42.510 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:38:42.510 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 180252 2023-10-01T09:38:42.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.511 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:42.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:42.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477434+0000", 2023-10-01T09:38:42.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:42.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477434+0000", 2023-10-01T09:38:42.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:42.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477433+0000", 2023-10-01T09:38:42.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:38:42.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:42.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477436+0000", 2023-10-01T09:38:42.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8310000000000001e-06 2023-10-01T09:38:42.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:42.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477443+0000", 2023-10-01T09:38:42.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.635e-06 2023-10-01T09:38:42.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:42.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477448+0000", 2023-10-01T09:38:42.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4539999999999998e-06 2023-10-01T09:38:42.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:42.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477496+0000", 2023-10-01T09:38:42.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7919e-05 2023-10-01T09:38:42.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:38:42.514 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477654+0000", 2023-10-01T09:38:42.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000157594 2023-10-01T09:38:42.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4,7", 2023-10-01T09:38:42.514 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.477798+0000", 2023-10-01T09:38:42.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014455699999999999 2023-10-01T09:38:42.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.514 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:38:42.515 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.488158+0000", 2023-10-01T09:38:42.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010359831999999999 2023-10-01T09:38:42.515 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.515 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:42.515 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.515 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:180251 9.d e458/431 9:bfd11e44:::benchmark_data_smithi138_497829_object77866:head v 458'11173, mlcod=458'11171)", 2023-10-01T09:38:42.515 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:38:42.480305+0000", 2023-10-01T09:38:42.515 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.013645226, 2023-10-01T09:38:42.515 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014001726000000001, 2023-10-01T09:38:42.515 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:42.515 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:38:42.516 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:42.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:42.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.480305+0000", 2023-10-01T09:38:42.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:42.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.480305+0000", 2023-10-01T09:38:42.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:42.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.480305+0000", 2023-10-01T09:38:42.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:38:42.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:42.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.480309+0000", 2023-10-01T09:38:42.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8279999999999996e-06 2023-10-01T09:38:42.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:42.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.480314+0000", 2023-10-01T09:38:42.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1660000000000002e-06 2023-10-01T09:38:42.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:42.518 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.480318+0000", 2023-10-01T09:38:42.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9369999999999997e-06 2023-10-01T09:38:42.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:42.518 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.488127+0000", 2023-10-01T09:38:42.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0078084280000000001 2023-10-01T09:38:42.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:38:42.519 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.488154+0000", 2023-10-01T09:38:42.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7078999999999999e-05 2023-10-01T09:38:42.519 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.519 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:42.519 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.519 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:180232 9.a 9:5248657f:::benchmark_data_smithi138_497829_object77847:head [delete] snapc 0=[] ondisk+write+known_if_redirected e458)", 2023-10-01T09:38:42.519 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:38:42.445159+0000", 2023-10-01T09:38:42.519 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.048792009999999997, 2023-10-01T09:38:42.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.049208973000000003, 2023-10-01T09:38:42.519 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:42.519 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:38:42.520 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:38:42.520 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:38:42.520 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:38:42.520 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 180232 2023-10-01T09:38:42.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.520 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:42.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:42.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.445159+0000", 2023-10-01T09:38:42.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:42.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.445159+0000", 2023-10-01T09:38:42.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.521 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:42.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.445157+0000", 2023-10-01T09:38:42.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:38:42.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.521 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:42.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.445162+0000", 2023-10-01T09:38:42.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5709999999999999e-06 2023-10-01T09:38:42.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:42.522 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.445171+0000", 2023-10-01T09:38:42.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7290000000000006e-06 2023-10-01T09:38:42.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.522 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:42.522 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.445176+0000", 2023-10-01T09:38:42.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7100000000000004e-06 2023-10-01T09:38:42.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.522 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:42.522 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.449646+0000", 2023-10-01T09:38:42.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044697779999999998 2023-10-01T09:38:42.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:38:42.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.451218+0000", 2023-10-01T09:38:42.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001572321 2023-10-01T09:38:42.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:42.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.473878+0000", 2023-10-01T09:38:42.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.022659489000000001 2023-10-01T09:38:42.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:38:42.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.473971+0000", 2023-10-01T09:38:42.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3319999999999994e-05 2023-10-01T09:38:42.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,6,7", 2023-10-01T09:38:42.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.474105+0000", 2023-10-01T09:38:42.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013342599999999999 2023-10-01T09:38:42.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:42.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.486888+0000", 2023-10-01T09:38:42.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01278293 2023-10-01T09:38:42.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:42.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.492744+0000", 2023-10-01T09:38:42.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058564580000000001 2023-10-01T09:38:42.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:42.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.494271+0000", 2023-10-01T09:38:42.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001527392 2023-10-01T09:38:42.525 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.525 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:42.526 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.526 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:180257 9.d e458/431 9:b9582955:::benchmark_data_smithi138_497829_object77872:head v 458'11174, mlcod=458'11171)", 2023-10-01T09:38:42.526 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:38:42.488227+0000", 2023-10-01T09:38:42.526 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.005723517, 2023-10-01T09:38:42.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0062527759999999998, 2023-10-01T09:38:42.526 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:42.526 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:38:42.526 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:42.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:42.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.488227+0000", 2023-10-01T09:38:42.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.527 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:42.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.488227+0000", 2023-10-01T09:38:42.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.527 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:42.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.488226+0000", 2023-10-01T09:38:42.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:38:42.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.527 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:42.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.488235+0000", 2023-10-01T09:38:42.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7450000000000001e-06 2023-10-01T09:38:42.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:42.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.488246+0000", 2023-10-01T09:38:42.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0654e-05 2023-10-01T09:38:42.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.528 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:42.528 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.488252+0000", 2023-10-01T09:38:42.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6590000000000001e-06 2023-10-01T09:38:42.528 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.529 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:42.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.492851+0000", 2023-10-01T09:38:42.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045985219999999999 2023-10-01T09:38:42.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.529 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.529 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:38:42.529 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.492880+0000", 2023-10-01T09:38:42.529 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8770000000000001e-05 2023-10-01T09:38:42.529 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.529 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:42.529 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.529 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.530 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.530 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:180259 9.e 9:7e2608cf:::benchmark_data_smithi138_497829_object77874:head [delete] snapc 0=[] ondisk+write+known_if_redirected e458)", 2023-10-01T09:38:42.530 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:38:42.489594+0000", 2023-10-01T09:38:42.530 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0043563509999999996, 2023-10-01T09:38:42.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0049573000000000004, 2023-10-01T09:38:42.530 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:42.530 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:38:42.530 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:38:42.530 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:38:42.530 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:38:42.530 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 180259 2023-10-01T09:38:42.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.531 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:42.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:42.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.489594+0000", 2023-10-01T09:38:42.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:42.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.489594+0000", 2023-10-01T09:38:42.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:42.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.489593+0000", 2023-10-01T09:38:42.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:38:42.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:42.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.489597+0000", 2023-10-01T09:38:42.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.027e-06 2023-10-01T09:38:42.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:42.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.489603+0000", 2023-10-01T09:38:42.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3929999999999996e-06 2023-10-01T09:38:42.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:42.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.489607+0000", 2023-10-01T09:38:42.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.878e-06 2023-10-01T09:38:42.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:42.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.491279+0000", 2023-10-01T09:38:42.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001672053 2023-10-01T09:38:42.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:38:42.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.491427+0000", 2023-10-01T09:38:42.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014774499999999999 2023-10-01T09:38:42.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4,7", 2023-10-01T09:38:42.534 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.491584+0000", 2023-10-01T09:38:42.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015694299999999999 2023-10-01T09:38:42.534 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.534 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:42.534 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.534 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.46067.0:180252 9.e e458/431)", 2023-10-01T09:38:42.534 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:38:42.492239+0000", 2023-10-01T09:38:42.535 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0017117740000000001, 2023-10-01T09:38:42.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002380839, 2023-10-01T09:38:42.535 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:42.535 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "queued for pg", 2023-10-01T09:38:42.535 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:42.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:42.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.492239+0000", 2023-10-01T09:38:42.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:42.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.492239+0000", 2023-10-01T09:38:42.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:42.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:42.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.492238+0000", 2023-10-01T09:38:42.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:38:42.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:42.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.492245+0000", 2023-10-01T09:38:42.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5130000000000002e-06 2023-10-01T09:38:42.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:42.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.492255+0000", 2023-10-01T09:38:42.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0180000000000001e-05 2023-10-01T09:38:42.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:42.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:42.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:42.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:38:42.492260+0000", 2023-10-01T09:38:42.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8720000000000001e-06 2023-10-01T09:38:42.537 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.537 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:42.537 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.537 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:42.537 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:38:42.537 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 9 2023-10-01T09:38:42.537 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:38:42.538 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:42 smithi138 ceph-mon[192513]: pgmap v1299: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 14990/77016 objects degraded (19.463%); 14744/77016 objects misplaced (19.144%); 6.5 MiB/s, 104 objects/s recovering 2023-10-01T09:38:42.538 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:42 smithi138 ceph-mon[185886]: pgmap v1299: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 14990/77016 objects degraded (19.463%); 14744/77016 objects misplaced (19.144%); 6.5 MiB/s, 104 objects/s recovering 2023-10-01T09:38:42.548 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:42.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:42.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:42.656 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:42.687 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:42 smithi190 ceph-mon[160435]: pgmap v1299: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 14990/77016 objects degraded (19.463%); 14744/77016 objects misplaced (19.144%); 6.5 MiB/s, 104 objects/s recovering 2023-10-01T09:38:42.693 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:42.754 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:42.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:42.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:42.866 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:42.918 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:42.985 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:42.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:42.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:43.093 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:38:43.140 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:43.219 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:38:43.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:43.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:38:43.325 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:43.351 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:43.392 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:43.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:43.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:43.498 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:43.544 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:43.621 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:43.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:43.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:43.737 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:43.822 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:43.861 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:38:43.916 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:43 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 14990/77016 objects degraded (19.463%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:43.918 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:43 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 14990/77016 objects degraded (19.463%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:44.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:43 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 14990/77016 objects degraded (19.463%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:44.477 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:44.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:44.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:44.582 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:44.641 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:38:44.719 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:44.912 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:44 smithi138 ceph-mon[192513]: pgmap v1300: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 14990/76620 objects degraded (19.564%); 14744/76620 objects misplaced (19.243%); 6.5 MiB/s, 104 objects/s recovering 2023-10-01T09:38:44.913 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:44 smithi138 ceph-mon[185886]: pgmap v1300: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 14990/76620 objects degraded (19.564%); 14744/76620 objects misplaced (19.243%); 6.5 MiB/s, 104 objects/s recovering 2023-10-01T09:38:44.922 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:44.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:44.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:45.031 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:45.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:44 smithi190 ceph-mon[160435]: pgmap v1300: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 14990/76620 objects degraded (19.564%); 14744/76620 objects misplaced (19.243%); 6.5 MiB/s, 104 objects/s recovering 2023-10-01T09:38:45.131 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:45.325 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:38:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:38:44] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.33.4" 2023-10-01T09:38:45.617 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:45.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:45.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:45.655 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:38:45.663 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:38:45.729 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:45.758 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:45.825 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:45.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:45.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:45.938 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:45.996 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:46.053 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:46.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:46.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:46.181 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:46.223 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:46.274 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:46.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:46.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:46.379 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:46.491 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:46.721 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:46.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:46.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:46.829 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1300,"stamp":"2023-10-01T09:38:44.202261+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1676679115,"num_objects":25540,"num_object_clones":0,"num_object_copies":76620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14990,"num_objects_misplaced":14744,"num_objects_unfound":0,"num_objects_dirty":25540,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":180360,"num_write_kb":6567590,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19909,"num_bytes_recovered":1291293105,"num_keys_recovered":90,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121282,"ondisk_log_size":121282,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12114304,"kb_used_data":6610560,"kb_used_omap":50,"kb_used_meta":5502861,"kb_avail":737846912,"statfs":{"total":767960285184,"available":755555237888,"internally_reserved":0,"allocated":6769213440,"data_stored":6758143130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51738,"internal_metadata":5634930150},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,5,7,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-283312128,"num_objects":-4323,"num_object_clones":0,"num_object_copies":-12969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-775,"num_objects_misplaced":-1595,"num_objects_unfound":0,"num_objects_dirty":-4323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4323,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1045,"num_bytes_recovered":68485120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008873"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":369,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.370097+0000","last_change":"2023-10-01T09:35:34.561455+0000","last_active":"2023-10-01T09:38:21.370097+0000","last_peered":"2023-10-01T09:38:21.370097+0000","last_clean":"2023-10-01T09:38:21.370097+0000","last_became_active":"2023-10-01T09:35:34.560978+0000","last_became_peered":"2023-10-01T09:35:34.560978+0000","last_unstale":"2023-10-01T09:38:21.370097+0000","last_undegraded":"2023-10-01T09:38:21.370097+0000","last_fullsized":"2023-10-01T09:38:21.370097+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:02:58.965198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":74,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.366057+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:38:21.366057+0000","last_peered":"2023-10-01T09:38:21.366057+0000","last_clean":"2023-10-01T09:38:21.366057+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:38:21.366057+0000","last_undegraded":"2023-10-01T09:38:21.366057+0000","last_fullsized":"2023-10-01T09:38:21.366057+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":454,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.368799+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:38:21.368799+0000","last_peered":"2023-10-01T09:38:21.368799+0000","last_clean":"2023-10-01T09:38:21.368799+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:38:21.368799+0000","last_undegraded":"2023-10-01T09:38:21.368799+0000","last_fullsized":"2023-10-01T09:38:21.368799+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":494,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.404186+0000","last_change":"2023-10-01T09:36:22.227558+0000","last_active":"2023-10-01T09:38:21.404186+0000","last_peered":"2023-10-01T09:38:21.404186+0000","last_clean":"2023-10-01T09:38:21.404186+0000","last_became_active":"2023-10-01T09:36:22.226550+0000","last_became_peered":"2023-10-01T09:36:22.226550+0000","last_unstale":"2023-10-01T09:38:21.404186+0000","last_undegraded":"2023-10-01T09:38:21.404186+0000","last_fullsized":"2023-10-01T09:38:21.404186+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:35:45.988649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":376,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.421424+0000","last_change":"2023-10-01T09:37:22.672668+0000","last_active":"2023-10-01T09:38:21.421424+0000","last_peered":"2023-10-01T09:38:21.421424+0000","last_clean":"2023-10-01T09:38:21.421424+0000","last_became_active":"2023-10-01T09:37:22.671882+0000","last_became_peered":"2023-10-01T09:37:22.671882+0000","last_unstale":"2023-10-01T09:38:21.421424+0000","last_undegraded":"2023-10-01T09:38:21.421424+0000","last_fullsized":"2023-10-01T09:38:21.421424+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:42:45.864800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":140,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.358143+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:38:21.358143+0000","last_peered":"2023-10-01T09:38:21.358143+0000","last_clean":"2023-10-01T09:38:21.358143+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:38:21.358143+0000","last_undegraded":"2023-10-01T09:38:21.358143+0000","last_fullsized":"2023-10-01T09:38:21.358143+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":530,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.401032+0000","last_change":"2023-10-01T09:37:22.671362+0000","last_active":"2023-10-01T09:38:21.401032+0000","last_peered":"2023-10-01T09:38:21.401032+0000","last_clean":"2023-10-01T09:38:21.401032+0000","last_became_active":"2023-10-01T09:37:22.671053+0000","last_became_peered":"2023-10-01T09:37:22.671053+0000","last_unstale":"2023-10-01T09:38:21.401032+0000","last_undegraded":"2023-10-01T09:38:21.401032+0000","last_fullsized":"2023-10-01T09:38:21.401032+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:42.476593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":478,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.365899+0000","last_change":"2023-10-01T09:37:22.667841+0000","last_active":"2023-10-01T09:38:21.365899+0000","last_peered":"2023-10-01T09:38:21.365899+0000","last_clean":"2023-10-01T09:38:21.365899+0000","last_became_active":"2023-10-01T09:37:22.667525+0000","last_became_peered":"2023-10-01T09:37:22.667525+0000","last_unstale":"2023-10-01T09:38:21.365899+0000","last_undegraded":"2023-10-01T09:38:21.365899+0000","last_fullsized":"2023-10-01T09:38:21.365899+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:03:37.695850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":151,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.358106+0000","last_change":"2023-10-01T09:36:22.254817+0000","last_active":"2023-10-01T09:38:21.358106+0000","last_peered":"2023-10-01T09:38:21.358106+0000","last_clean":"2023-10-01T09:38:21.358106+0000","last_became_active":"2023-10-01T09:36:22.254460+0000","last_became_peered":"2023-10-01T09:36:22.254460+0000","last_unstale":"2023-10-01T09:38:21.358106+0000","last_undegraded":"2023-10-01T09:38:21.358106+0000","last_fullsized":"2023-10-01T09:38:21.358106+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":54,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.357758+0000","last_change":"2023-10-01T09:37:22.680561+0000","last_active":"2023-10-01T09:38:21.357758+0000","last_peered":"2023-10-01T09:38:21.357758+0000","last_clean":"2023-10-01T09:38:21.357758+0000","last_became_active":"2023-10-01T09:37:22.680228+0000","last_became_peered":"2023-10-01T09:37:22.680228+0000","last_unstale":"2023-10-01T09:38:21.357758+0000","last_undegraded":"2023-10-01T09:38:21.357758+0000","last_fullsized":"2023-10-01T09:38:21.357758+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":73,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.403335+0000","last_change":"2023-10-01T09:37:22.697476+0000","last_active":"2023-10-01T09:38:21.403335+0000","last_peered":"2023-10-01T09:38:21.403335+0000","last_clean":"2023-10-01T09:38:21.403335+0000","last_became_active":"2023-10-01T09:37:22.671750+0000","last_became_peered":"2023-10-01T09:37:22.671750+0000","last_unstale":"2023-10-01T09:38:21.403335+0000","last_undegraded":"2023-10-01T09:38:21.403335+0000","last_fullsized":"2023-10-01T09:38:21.403335+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:36:00.363327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":37,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.400904+0000","last_change":"2023-10-01T09:36:22.223469+0000","last_active":"2023-10-01T09:38:21.400904+0000","last_peered":"2023-10-01T09:38:21.400904+0000","last_clean":"2023-10-01T09:38:21.400904+0000","last_became_active":"2023-10-01T09:36:22.222681+0000","last_became_peered":"2023-10-01T09:36:22.222681+0000","last_unstale":"2023-10-01T09:38:21.400904+0000","last_undegraded":"2023-10-01T09:38:21.400904+0000","last_fullsized":"2023-10-01T09:38:21.400904+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:19:28.367922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":485,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.300517+0000","last_change":"2023-10-01T09:32:27.498041+0000","last_active":"2023-10-01T09:38:31.300517+0000","last_peered":"2023-10-01T09:38:31.300517+0000","last_clean":"2023-10-01T09:38:31.300517+0000","last_became_active":"2023-10-01T09:32:27.454394+0000","last_became_peered":"2023-10-01T09:32:27.454394+0000","last_unstale":"2023-10-01T09:38:31.300517+0000","last_undegraded":"2023-10-01T09:38:31.300517+0000","last_fullsized":"2023-10-01T09:38:31.300517+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":299,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.366441+0000","last_change":"2023-10-01T09:35:23.610660+0000","last_active":"2023-10-01T09:38:21.366441+0000","last_peered":"2023-10-01T09:38:21.366441+0000","last_clean":"2023-10-01T09:38:21.366441+0000","last_became_active":"2023-10-01T09:35:23.610273+0000","last_became_peered":"2023-10-01T09:35:23.610273+0000","last_unstale":"2023-10-01T09:38:21.366441+0000","last_undegraded":"2023-10-01T09:38:21.366441+0000","last_fullsized":"2023-10-01T09:38:21.366441+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:43.996644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":561,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.300540+0000","last_change":"2023-10-01T09:35:34.640669+0000","last_active":"2023-10-01T09:38:31.300540+0000","last_peered":"2023-10-01T09:38:31.300540+0000","last_clean":"2023-10-01T09:38:31.300540+0000","last_became_active":"2023-10-01T09:35:34.552977+0000","last_became_peered":"2023-10-01T09:35:34.552977+0000","last_unstale":"2023-10-01T09:38:31.300540+0000","last_undegraded":"2023-10-01T09:38:31.300540+0000","last_fullsized":"2023-10-01T09:38:31.300540+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":306,"num_keys_recovered":90,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":295,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.300531+0000","last_change":"2023-10-01T09:37:22.681548+0000","last_active":"2023-10-01T09:38:31.300531+0000","last_peered":"2023-10-01T09:38:31.300531+0000","last_clean":"2023-10-01T09:38:31.300531+0000","last_became_active":"2023-10-01T09:37:22.681195+0000","last_became_peered":"2023-10-01T09:37:22.681195+0000","last_unstale":"2023-10-01T09:38:31.300531+0000","last_undegraded":"2023-10-01T09:38:31.300531+0000","last_fullsized":"2023-10-01T09:38:31.300531+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":459,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.401068+0000","last_change":"2023-10-01T09:36:22.223650+0000","last_active":"2023-10-01T09:38:21.401068+0000","last_peered":"2023-10-01T09:38:21.401068+0000","last_clean":"2023-10-01T09:38:21.401068+0000","last_became_active":"2023-10-01T09:36:22.223087+0000","last_became_peered":"2023-10-01T09:36:22.223087+0000","last_unstale":"2023-10-01T09:38:21.401068+0000","last_undegraded":"2023-10-01T09:38:21.401068+0000","last_fullsized":"2023-10-01T09:38:21.401068+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:52:21.962744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":276,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.300907+0000","last_change":"2023-10-01T09:37:22.670521+0000","last_active":"2023-10-01T09:38:31.300907+0000","last_peered":"2023-10-01T09:38:31.300907+0000","last_clean":"2023-10-01T09:38:31.300907+0000","last_became_active":"2023-10-01T09:37:22.670154+0000","last_became_peered":"2023-10-01T09:37:22.670154+0000","last_unstale":"2023-10-01T09:38:31.300907+0000","last_undegraded":"2023-10-01T09:38:31.300907+0000","last_fullsized":"2023-10-01T09:38:31.300907+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":469,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.300448+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:38:31.300448+0000","last_peered":"2023-10-01T09:38:31.300448+0000","last_clean":"2023-10-01T09:38:31.300448+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:38:31.300448+0000","last_undegraded":"2023-10-01T09:38:31.300448+0000","last_fullsized":"2023-10-01T09:38:31.300448+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":459,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.404241+0000","last_change":"2023-10-01T09:37:22.681587+0000","last_active":"2023-10-01T09:38:21.404241+0000","last_peered":"2023-10-01T09:38:21.404241+0000","last_clean":"2023-10-01T09:38:21.404241+0000","last_became_active":"2023-10-01T09:37:22.681290+0000","last_became_peered":"2023-10-01T09:37:22.681290+0000","last_unstale":"2023-10-01T09:38:21.404241+0000","last_undegraded":"2023-10-01T09:38:21.404241+0000","last_fullsized":"2023-10-01T09:38:21.404241+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:13:27.218691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":314,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.421270+0000","last_change":"2023-10-01T09:37:22.717074+0000","last_active":"2023-10-01T09:38:21.421270+0000","last_peered":"2023-10-01T09:38:21.421270+0000","last_clean":"2023-10-01T09:38:21.421270+0000","last_became_active":"2023-10-01T09:37:22.672350+0000","last_became_peered":"2023-10-01T09:37:22.672350+0000","last_unstale":"2023-10-01T09:38:21.421270+0000","last_undegraded":"2023-10-01T09:38:21.421270+0000","last_fullsized":"2023-10-01T09:38:21.421270+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:44:43.921813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":257,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.366676+0000","last_change":"2023-10-01T09:36:22.223054+0000","last_active":"2023-10-01T09:38:21.366676+0000","last_peered":"2023-10-01T09:38:21.366676+0000","last_clean":"2023-10-01T09:38:21.366676+0000","last_became_active":"2023-10-01T09:36:22.222776+0000","last_became_peered":"2023-10-01T09:36:22.222776+0000","last_unstale":"2023-10-01T09:38:21.366676+0000","last_undegraded":"2023-10-01T09:38:21.366676+0000","last_fullsized":"2023-10-01T09:38:21.366676+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:09:23.536588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"457'11322","reported_seq":16723,"reported_epoch":457,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:38:39.640567+0000","last_change":"2023-10-01T09:37:22.729877+0000","last_active":"2023-10-01T09:38:39.640567+0000","last_peered":"2023-10-01T09:38:39.640567+0000","last_clean":"2023-10-01T09:36:21.212087+0000","last_became_active":"2023-10-01T09:36:23.799500+0000","last_became_peered":"2023-10-01T09:36:23.799500+0000","last_unstale":"2023-10-01T09:38:39.640567+0000","last_undegraded":"2023-10-01T09:36:23.263244+0000","last_fullsized":"2023-10-01T09:36:23.212896+0000","mapping_epoch":431,"log_start":"409'3798","ondisk_log_start":"409'3798","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7524,"log_dups_size":0,"ondisk_log_size":7524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:09:34.039273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104071168,"num_objects":1588,"num_object_clones":0,"num_object_copies":4764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":672,"num_objects_misplaced":1529,"num_objects_unfound":0,"num_objects_dirty":1588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11322,"num_write_kb":413120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2661,"num_bytes_recovered":174325760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5],"avail_no_missing":["0","5","7"],"object_location_counts":[{"shards":"0,5","objects":-613},{"shards":"0,5,7","objects":1529}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":131,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.406763+0000","last_change":"2023-10-01T09:36:22.387631+0000","last_active":"2023-10-01T09:38:31.406763+0000","last_peered":"2023-10-01T09:38:31.406763+0000","last_clean":"2023-10-01T09:38:31.406763+0000","last_became_active":"2023-10-01T09:36:22.237417+0000","last_became_peered":"2023-10-01T09:36:22.237417+0000","last_unstale":"2023-10-01T09:38:31.406763+0000","last_undegraded":"2023-10-01T09:38:31.406763+0000","last_fullsized":"2023-10-01T09:38:31.406763+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:06:04.026880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":168,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.357616+0000","last_change":"2023-10-01T09:36:22.236445+0000","last_active":"2023-10-01T09:38:21.357616+0000","last_peered":"2023-10-01T09:38:21.357616+0000","last_clean":"2023-10-01T09:38:21.357616+0000","last_became_active":"2023-10-01T09:36:22.236081+0000","last_became_peered":"2023-10-01T09:36:22.236081+0000","last_unstale":"2023-10-01T09:38:21.357616+0000","last_undegraded":"2023-10-01T09:38:21.357616+0000","last_fullsized":"2023-10-01T09:38:21.357616+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"457'11319","reported_seq":11412,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:39.700759+0000","last_change":"2023-10-01T09:37:22.701634+0000","last_active":"2023-10-01T09:38:39.700759+0000","last_peered":"2023-10-01T09:38:39.700759+0000","last_clean":"2023-10-01T09:38:39.700759+0000","last_became_active":"2023-10-01T09:37:22.671424+0000","last_became_peered":"2023-10-01T09:37:22.671424+0000","last_unstale":"2023-10-01T09:38:39.700759+0000","last_undegraded":"2023-10-01T09:38:39.700759+0000","last_fullsized":"2023-10-01T09:38:39.700759+0000","mapping_epoch":445,"log_start":"410'4434","ondisk_log_start":"410'4434","created":400,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6885,"log_dups_size":0,"ondisk_log_size":6885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:07:45.393494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107937792,"num_objects":1647,"num_object_clones":0,"num_object_copies":4941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11319,"num_write_kb":414912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":452,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.302644+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:38:31.302644+0000","last_peered":"2023-10-01T09:38:31.302644+0000","last_clean":"2023-10-01T09:38:31.302644+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:38:31.302644+0000","last_undegraded":"2023-10-01T09:38:31.302644+0000","last_fullsized":"2023-10-01T09:38:31.302644+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":501,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:29.686389+0000","last_change":"2023-10-01T09:36:22.223799+0000","last_active":"2023-10-01T09:38:29.686389+0000","last_peered":"2023-10-01T09:38:29.686389+0000","last_clean":"2023-10-01T09:38:29.686389+0000","last_became_active":"2023-10-01T09:36:22.223546+0000","last_became_peered":"2023-10-01T09:36:22.223546+0000","last_unstale":"2023-10-01T09:38:29.686389+0000","last_undegraded":"2023-10-01T09:38:29.686389+0000","last_fullsized":"2023-10-01T09:38:29.686389+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:54:07.013864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"457'11137","reported_seq":20725,"reported_epoch":457,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-01T09:38:39.988727+0000","last_change":"2023-10-01T09:37:06.460981+0000","last_active":"2023-10-01T09:38:39.988727+0000","last_peered":"2023-10-01T09:38:39.988727+0000","last_clean":"2023-10-01T09:36:21.192430+0000","last_became_active":"2023-10-01T09:36:23.791409+0000","last_became_peered":"2023-10-01T09:36:23.791409+0000","last_unstale":"2023-10-01T09:38:39.988727+0000","last_undegraded":"2023-10-01T09:38:39.988727+0000","last_fullsized":"2023-10-01T09:36:23.217545+0000","mapping_epoch":431,"log_start":"410'4307","ondisk_log_start":"410'4307","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6830,"log_dups_size":0,"ondisk_log_size":6830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106627072,"num_objects":1627,"num_object_clones":0,"num_object_copies":4881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3252,"num_objects_unfound":0,"num_objects_dirty":1627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11137,"num_write_kb":408448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3172,"num_bytes_recovered":207814656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,1],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"1,5","objects":1},{"shards":"3,5,7","objects":1626}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":538,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.300780+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:38:31.300780+0000","last_peered":"2023-10-01T09:38:31.300780+0000","last_clean":"2023-10-01T09:38:31.300780+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:38:31.300780+0000","last_undegraded":"2023-10-01T09:38:31.300780+0000","last_fullsized":"2023-10-01T09:38:31.300780+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":321,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.370016+0000","last_change":"2023-10-01T09:36:22.247103+0000","last_active":"2023-10-01T09:38:21.370016+0000","last_peered":"2023-10-01T09:38:21.370016+0000","last_clean":"2023-10-01T09:38:21.370016+0000","last_became_active":"2023-10-01T09:36:22.246793+0000","last_became_peered":"2023-10-01T09:36:22.246793+0000","last_unstale":"2023-10-01T09:38:21.370016+0000","last_undegraded":"2023-10-01T09:38:21.370016+0000","last_fullsized":"2023-10-01T09:38:21.370016+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:25:05.754730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"457'11072","reported_seq":13919,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:39.878419+0000","last_change":"2023-10-01T09:36:28.672838+0000","last_active":"2023-10-01T09:38:39.878419+0000","last_peered":"2023-10-01T09:38:39.878419+0000","last_clean":"2023-10-01T09:38:39.878419+0000","last_became_active":"2023-10-01T09:36:28.606609+0000","last_became_peered":"2023-10-01T09:36:28.606609+0000","last_unstale":"2023-10-01T09:38:39.878419+0000","last_undegraded":"2023-10-01T09:38:39.878419+0000","last_fullsized":"2023-10-01T09:38:39.878419+0000","mapping_epoch":433,"log_start":"410'4198","ondisk_log_start":"410'4198","created":400,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6874,"log_dups_size":0,"ondisk_log_size":6874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102629376,"num_objects":1566,"num_object_clones":0,"num_object_copies":4698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11072,"num_write_kb":404416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1377,"num_bytes_recovered":90243072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":177,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.358048+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:38:21.358048+0000","last_peered":"2023-10-01T09:38:21.358048+0000","last_clean":"2023-10-01T09:38:21.358048+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:38:21.358048+0000","last_undegraded":"2023-10-01T09:38:21.358048+0000","last_fullsized":"2023-10-01T09:38:21.358048+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":433,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.411266+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:38:21.411266+0000","last_peered":"2023-10-01T09:38:21.411266+0000","last_clean":"2023-10-01T09:38:21.411266+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:38:21.411266+0000","last_undegraded":"2023-10-01T09:38:21.411266+0000","last_fullsized":"2023-10-01T09:38:21.411266+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"458'11044","reported_seq":20101,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-10-01T09:38:42.150444+0000","last_change":"2023-10-01T09:37:22.694633+0000","last_active":"2023-10-01T09:38:42.150444+0000","last_peered":"2023-10-01T09:38:42.150444+0000","last_clean":"2023-10-01T09:38:42.150444+0000","last_became_active":"2023-10-01T09:37:22.668427+0000","last_became_peered":"2023-10-01T09:37:22.668427+0000","last_unstale":"2023-10-01T09:38:42.150444+0000","last_undegraded":"2023-10-01T09:38:42.150444+0000","last_fullsized":"2023-10-01T09:38:42.150444+0000","mapping_epoch":445,"log_start":"409'3079","ondisk_log_start":"409'3079","created":400,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7965,"log_dups_size":0,"ondisk_log_size":7965,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:50:23.248095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96468992,"num_objects":1472,"num_object_clones":0,"num_object_copies":4416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11044,"num_write_kb":400512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4478,"num_bytes_recovered":293339136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":186,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.401017+0000","last_change":"2023-10-01T09:37:22.723456+0000","last_active":"2023-10-01T09:38:21.401017+0000","last_peered":"2023-10-01T09:38:21.401017+0000","last_clean":"2023-10-01T09:38:21.401017+0000","last_became_active":"2023-10-01T09:37:22.677109+0000","last_became_peered":"2023-10-01T09:37:22.677109+0000","last_unstale":"2023-10-01T09:38:21.401017+0000","last_undegraded":"2023-10-01T09:38:21.401017+0000","last_fullsized":"2023-10-01T09:38:21.401017+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:04:02.812669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":112,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.254900+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:38:31.254900+0000","last_peered":"2023-10-01T09:38:31.254900+0000","last_clean":"2023-10-01T09:38:31.254900+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:38:31.254900+0000","last_undegraded":"2023-10-01T09:38:31.254900+0000","last_fullsized":"2023-10-01T09:38:31.254900+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"457'11147","reported_seq":14117,"reported_epoch":457,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-01T09:38:39.650314+0000","last_change":"2023-10-01T09:37:06.423304+0000","last_active":"2023-10-01T09:38:39.650314+0000","last_peered":"2023-10-01T09:38:39.650314+0000","last_clean":"2023-10-01T09:35:09.959309+0000","last_became_active":"2023-10-01T09:36:23.678618+0000","last_became_peered":"2023-10-01T09:36:23.678618+0000","last_unstale":"2023-10-01T09:38:39.650314+0000","last_undegraded":"2023-10-01T09:38:39.650314+0000","last_fullsized":"2023-10-01T09:38:39.650314+0000","mapping_epoch":431,"log_start":"410'4314","ondisk_log_start":"410'4314","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6833,"log_dups_size":0,"ondisk_log_size":6833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:32:05.325065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101253120,"num_objects":1545,"num_object_clones":0,"num_object_copies":4635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4912,"num_objects_unfound":0,"num_objects_dirty":1545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11147,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1435,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[1,6,2],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":-1723}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":378,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.251986+0000","last_change":"2023-10-01T09:37:22.684537+0000","last_active":"2023-10-01T09:38:31.251986+0000","last_peered":"2023-10-01T09:38:31.251986+0000","last_clean":"2023-10-01T09:38:31.251986+0000","last_became_active":"2023-10-01T09:37:22.683692+0000","last_became_peered":"2023-10-01T09:37:22.683692+0000","last_unstale":"2023-10-01T09:38:31.251986+0000","last_undegraded":"2023-10-01T09:38:31.251986+0000","last_fullsized":"2023-10-01T09:38:31.251986+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":504,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.366231+0000","last_change":"2023-10-01T09:35:34.558802+0000","last_active":"2023-10-01T09:38:21.366231+0000","last_peered":"2023-10-01T09:38:21.366231+0000","last_clean":"2023-10-01T09:38:21.366231+0000","last_became_active":"2023-10-01T09:35:34.558511+0000","last_became_peered":"2023-10-01T09:35:34.558511+0000","last_unstale":"2023-10-01T09:38:21.366231+0000","last_undegraded":"2023-10-01T09:38:21.366231+0000","last_fullsized":"2023-10-01T09:38:21.366231+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:40:52.864568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"458'11211","reported_seq":11984,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-10-01T09:38:41.086441+0000","last_change":"2023-10-01T09:37:23.700088+0000","last_active":"2023-10-01T09:38:41.086441+0000","last_peered":"2023-10-01T09:38:41.086441+0000","last_clean":"2023-10-01T09:38:41.086441+0000","last_became_active":"2023-10-01T09:37:23.635822+0000","last_became_peered":"2023-10-01T09:37:23.635822+0000","last_unstale":"2023-10-01T09:38:41.086441+0000","last_undegraded":"2023-10-01T09:38:41.086441+0000","last_fullsized":"2023-10-01T09:38:41.086441+0000","mapping_epoch":446,"log_start":"405'1200","ondisk_log_start":"405'1200","created":400,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":10011,"log_dups_size":0,"ondisk_log_size":10011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102301696,"num_objects":1561,"num_object_clones":0,"num_object_copies":4683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11211,"num_write_kb":408704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":343,"num_bytes_recovered":22478848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":348,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.300531+0000","last_change":"2023-10-01T09:35:34.551196+0000","last_active":"2023-10-01T09:38:31.300531+0000","last_peered":"2023-10-01T09:38:31.300531+0000","last_clean":"2023-10-01T09:38:31.300531+0000","last_became_active":"2023-10-01T09:35:34.550803+0000","last_became_peered":"2023-10-01T09:35:34.550803+0000","last_unstale":"2023-10-01T09:38:31.300531+0000","last_undegraded":"2023-10-01T09:38:31.300531+0000","last_fullsized":"2023-10-01T09:38:31.300531+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":184,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.255194+0000","last_change":"2023-10-01T09:37:22.681270+0000","last_active":"2023-10-01T09:38:31.255194+0000","last_peered":"2023-10-01T09:38:31.255194+0000","last_clean":"2023-10-01T09:38:31.255194+0000","last_became_active":"2023-10-01T09:37:22.680951+0000","last_became_peered":"2023-10-01T09:37:22.680951+0000","last_unstale":"2023-10-01T09:38:31.255194+0000","last_undegraded":"2023-10-01T09:38:31.255194+0000","last_fullsized":"2023-10-01T09:38:31.255194+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"457'11302","reported_seq":12344,"reported_epoch":457,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:38:39.684318+0000","last_change":"2023-10-01T09:36:23.771443+0000","last_active":"2023-10-01T09:38:39.684318+0000","last_peered":"2023-10-01T09:38:39.684318+0000","last_clean":"2023-10-01T09:35:09.934020+0000","last_became_active":"2023-10-01T09:36:23.749995+0000","last_became_peered":"2023-10-01T09:36:23.749995+0000","last_unstale":"2023-10-01T09:38:39.684318+0000","last_undegraded":"2023-10-01T09:36:23.566811+0000","last_fullsized":"2023-10-01T09:36:23.220185+0000","mapping_epoch":431,"log_start":"410'4392","ondisk_log_start":"410'4392","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6910,"log_dups_size":0,"ondisk_log_size":6910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:23:09.158739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105512960,"num_objects":1610,"num_object_clones":0,"num_object_copies":4830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11302,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":476,"num_bytes_recovered":30998528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2","objects":-3277},{"shards":"1,2,3","objects":692},{"shards":"1,3,7","objects":918}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":308,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.365748+0000","last_change":"2023-10-01T09:36:22.223384+0000","last_active":"2023-10-01T09:38:21.365748+0000","last_peered":"2023-10-01T09:38:21.365748+0000","last_clean":"2023-10-01T09:38:21.365748+0000","last_became_active":"2023-10-01T09:36:22.223098+0000","last_became_peered":"2023-10-01T09:36:22.223098+0000","last_unstale":"2023-10-01T09:38:21.365748+0000","last_undegraded":"2023-10-01T09:38:21.365748+0000","last_fullsized":"2023-10-01T09:38:21.365748+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:15:40.489055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":364,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.366749+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:38:21.366749+0000","last_peered":"2023-10-01T09:38:21.366749+0000","last_clean":"2023-10-01T09:38:21.366749+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:38:21.366749+0000","last_undegraded":"2023-10-01T09:38:21.366749+0000","last_fullsized":"2023-10-01T09:38:21.366749+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"457'11249","reported_seq":11327,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:39.771288+0000","last_change":"2023-10-01T09:37:22.715127+0000","last_active":"2023-10-01T09:38:39.771288+0000","last_peered":"2023-10-01T09:38:39.771288+0000","last_clean":"2023-10-01T09:38:39.771288+0000","last_became_active":"2023-10-01T09:37:22.680102+0000","last_became_peered":"2023-10-01T09:37:22.680102+0000","last_unstale":"2023-10-01T09:38:39.771288+0000","last_undegraded":"2023-10-01T09:38:39.771288+0000","last_fullsized":"2023-10-01T09:38:39.771288+0000","mapping_epoch":445,"log_start":"410'4362","ondisk_log_start":"410'4362","created":400,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6887,"log_dups_size":0,"ondisk_log_size":6887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104267776,"num_objects":1591,"num_object_clones":0,"num_object_copies":4773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11249,"num_write_kb":410880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1281,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.365992+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:38:21.365992+0000","last_peered":"2023-10-01T09:38:21.365992+0000","last_clean":"2023-10-01T09:38:21.365992+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:38:21.365992+0000","last_undegraded":"2023-10-01T09:38:21.365992+0000","last_fullsized":"2023-10-01T09:38:21.365992+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":178,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.358708+0000","last_change":"2023-10-01T09:37:22.667746+0000","last_active":"2023-10-01T09:38:21.358708+0000","last_peered":"2023-10-01T09:38:21.358708+0000","last_clean":"2023-10-01T09:38:21.358708+0000","last_became_active":"2023-10-01T09:37:22.667475+0000","last_became_peered":"2023-10-01T09:37:22.667475+0000","last_unstale":"2023-10-01T09:38:21.358708+0000","last_undegraded":"2023-10-01T09:38:21.358708+0000","last_fullsized":"2023-10-01T09:38:21.358708+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":412,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.358113+0000","last_change":"2023-10-01T09:36:22.236067+0000","last_active":"2023-10-01T09:38:21.358113+0000","last_peered":"2023-10-01T09:38:21.358113+0000","last_clean":"2023-10-01T09:38:21.358113+0000","last_became_active":"2023-10-01T09:36:22.235732+0000","last_became_peered":"2023-10-01T09:36:22.235732+0000","last_unstale":"2023-10-01T09:38:21.358113+0000","last_undegraded":"2023-10-01T09:38:21.358113+0000","last_fullsized":"2023-10-01T09:38:21.358113+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"457'11258","reported_seq":11326,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:39.655345+0000","last_change":"2023-10-01T09:34:06.140052+0000","last_active":"2023-10-01T09:38:39.655345+0000","last_peered":"2023-10-01T09:38:39.655345+0000","last_clean":"2023-10-01T09:38:39.655345+0000","last_became_active":"2023-10-01T09:34:06.139672+0000","last_became_peered":"2023-10-01T09:34:06.139672+0000","last_unstale":"2023-10-01T09:38:39.655345+0000","last_undegraded":"2023-10-01T09:38:39.655345+0000","last_fullsized":"2023-10-01T09:38:39.655345+0000","mapping_epoch":400,"log_start":"409'3678","ondisk_log_start":"409'3678","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7580,"log_dups_size":0,"ondisk_log_size":7580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:39:55.000830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104857600,"num_objects":1600,"num_object_clones":0,"num_object_copies":4800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11258,"num_write_kb":411456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1122,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.358394+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:38:21.358394+0000","last_peered":"2023-10-01T09:38:21.358394+0000","last_clean":"2023-10-01T09:38:21.358394+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:38:21.358394+0000","last_undegraded":"2023-10-01T09:38:21.358394+0000","last_fullsized":"2023-10-01T09:38:21.358394+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":433,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.401255+0000","last_change":"2023-10-01T09:32:27.451746+0000","last_active":"2023-10-01T09:38:21.401255+0000","last_peered":"2023-10-01T09:38:21.401255+0000","last_clean":"2023-10-01T09:38:21.401255+0000","last_became_active":"2023-10-01T09:32:27.451069+0000","last_became_peered":"2023-10-01T09:32:27.451069+0000","last_unstale":"2023-10-01T09:38:21.401255+0000","last_undegraded":"2023-10-01T09:38:21.401255+0000","last_fullsized":"2023-10-01T09:38:21.401255+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:50.586397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":145,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.358120+0000","last_change":"2023-10-01T09:38:07.318624+0000","last_active":"2023-10-01T09:38:21.358120+0000","last_peered":"2023-10-01T09:38:21.358120+0000","last_clean":"2023-10-01T09:38:21.358120+0000","last_became_active":"2023-10-01T09:38:07.318266+0000","last_became_peered":"2023-10-01T09:38:07.318266+0000","last_unstale":"2023-10-01T09:38:21.358120+0000","last_undegraded":"2023-10-01T09:38:21.358120+0000","last_fullsized":"2023-10-01T09:38:21.358120+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"457'11027","reported_seq":15730,"reported_epoch":457,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-01T09:38:39.742569+0000","last_change":"2023-10-01T09:38:08.682352+0000","last_active":"2023-10-01T09:38:39.742569+0000","last_peered":"2023-10-01T09:38:39.742569+0000","last_clean":"2023-10-01T09:36:21.153294+0000","last_became_active":"2023-10-01T09:38:08.590431+0000","last_became_peered":"2023-10-01T09:38:08.590431+0000","last_unstale":"2023-10-01T09:38:39.742569+0000","last_undegraded":"2023-10-01T09:38:08.584718+0000","last_fullsized":"2023-10-01T09:38:39.742569+0000","mapping_epoch":453,"log_start":"410'4207","ondisk_log_start":"410'4207","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6820,"log_dups_size":0,"ondisk_log_size":6820,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:06:34.257665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100204544,"num_objects":1529,"num_object_clones":0,"num_object_copies":4587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1529,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11027,"num_write_kb":401792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2301,"num_bytes_recovered":150798336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,3],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,3,7","objects":1},{"shards":"1,5,7","objects":1528}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":737,"reported_epoch":456,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:38:21.400842+0000","last_change":"2023-10-01T09:38:07.327355+0000","last_active":"2023-10-01T09:38:21.400842+0000","last_peered":"2023-10-01T09:38:21.400842+0000","last_clean":"2023-10-01T09:37:58.763801+0000","last_became_active":"2023-10-01T09:38:07.326767+0000","last_became_peered":"2023-10-01T09:38:07.326767+0000","last_unstale":"2023-10-01T09:38:21.400842+0000","last_undegraded":"2023-10-01T09:38:07.315924+0000","last_fullsized":"2023-10-01T09:38:21.400842+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:45:16.157432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":["0","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":508,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.365824+0000","last_change":"2023-10-01T09:35:34.562777+0000","last_active":"2023-10-01T09:38:21.365824+0000","last_peered":"2023-10-01T09:38:21.365824+0000","last_clean":"2023-10-01T09:38:21.365824+0000","last_became_active":"2023-10-01T09:35:34.562492+0000","last_became_peered":"2023-10-01T09:35:34.562492+0000","last_unstale":"2023-10-01T09:38:21.365824+0000","last_undegraded":"2023-10-01T09:38:21.365824+0000","last_fullsized":"2023-10-01T09:38:21.365824+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:26:15.751778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":326,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.358305+0000","last_change":"2023-10-01T09:37:22.673957+0000","last_active":"2023-10-01T09:38:21.358305+0000","last_peered":"2023-10-01T09:38:21.358305+0000","last_clean":"2023-10-01T09:38:21.358305+0000","last_became_active":"2023-10-01T09:37:22.673685+0000","last_became_peered":"2023-10-01T09:37:22.673685+0000","last_unstale":"2023-10-01T09:38:21.358305+0000","last_undegraded":"2023-10-01T09:38:21.358305+0000","last_fullsized":"2023-10-01T09:38:21.358305+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"457'11124","reported_seq":20859,"reported_epoch":457,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:38:39.660084+0000","last_change":"2023-10-01T09:37:22.682829+0000","last_active":"2023-10-01T09:38:39.660084+0000","last_peered":"2023-10-01T09:38:39.660084+0000","last_clean":"2023-10-01T09:36:21.170373+0000","last_became_active":"2023-10-01T09:36:23.812646+0000","last_became_peered":"2023-10-01T09:36:23.812646+0000","last_unstale":"2023-10-01T09:38:39.660084+0000","last_undegraded":"2023-10-01T09:37:22.679747+0000","last_fullsized":"2023-10-01T09:36:23.211397+0000","mapping_epoch":431,"log_start":"409'3557","ondisk_log_start":"409'3557","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7567,"log_dups_size":0,"ondisk_log_size":7567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:34:58.841399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106692608,"num_objects":1628,"num_object_clones":0,"num_object_copies":4884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6313,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11124,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3219,"num_bytes_recovered":210829312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1627},{"shards":"0,3,5,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1453,"reported_epoch":456,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:38:21.357464+0000","last_change":"2023-10-01T09:37:23.636269+0000","last_active":"2023-10-01T09:38:21.357464+0000","last_peered":"2023-10-01T09:38:21.357464+0000","last_clean":"2023-10-01T09:35:27.946455+0000","last_became_active":"2023-10-01T09:37:23.635623+0000","last_became_peered":"2023-10-01T09:37:23.635623+0000","last_unstale":"2023-10-01T09:38:21.357464+0000","last_undegraded":"2023-10-01T09:37:23.631750+0000","last_fullsized":"2023-10-01T09:38:21.357464+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":477,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:24.627919+0000","last_change":"2023-10-01T09:36:29.654983+0000","last_active":"2023-10-01T09:38:24.627919+0000","last_peered":"2023-10-01T09:38:24.627919+0000","last_clean":"2023-10-01T09:38:24.627919+0000","last_became_active":"2023-10-01T09:36:29.650683+0000","last_became_peered":"2023-10-01T09:36:29.650683+0000","last_unstale":"2023-10-01T09:38:24.627919+0000","last_undegraded":"2023-10-01T09:38:24.627919+0000","last_fullsized":"2023-10-01T09:38:24.627919+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:23.591743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":256,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.357864+0000","last_change":"2023-10-01T09:38:07.494808+0000","last_active":"2023-10-01T09:38:21.357864+0000","last_peered":"2023-10-01T09:38:21.357864+0000","last_clean":"2023-10-01T09:38:21.357864+0000","last_became_active":"2023-10-01T09:38:07.494419+0000","last_became_peered":"2023-10-01T09:38:07.494419+0000","last_unstale":"2023-10-01T09:38:21.357864+0000","last_undegraded":"2023-10-01T09:38:21.357864+0000","last_fullsized":"2023-10-01T09:38:21.357864+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":372,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.300867+0000","last_change":"2023-10-01T09:32:27.453291+0000","last_active":"2023-10-01T09:38:31.300867+0000","last_peered":"2023-10-01T09:38:31.300867+0000","last_clean":"2023-10-01T09:38:31.300867+0000","last_became_active":"2023-10-01T09:32:27.453026+0000","last_became_peered":"2023-10-01T09:32:27.453026+0000","last_unstale":"2023-10-01T09:38:31.300867+0000","last_undegraded":"2023-10-01T09:38:31.300867+0000","last_fullsized":"2023-10-01T09:38:31.300867+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"457'11155","reported_seq":11223,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:39.623395+0000","last_change":"2023-10-01T09:34:06.140844+0000","last_active":"2023-10-01T09:38:39.623395+0000","last_peered":"2023-10-01T09:38:39.623395+0000","last_clean":"2023-10-01T09:38:39.623395+0000","last_became_active":"2023-10-01T09:34:06.139812+0000","last_became_peered":"2023-10-01T09:34:06.139812+0000","last_unstale":"2023-10-01T09:38:39.623395+0000","last_undegraded":"2023-10-01T09:38:39.623395+0000","last_fullsized":"2023-10-01T09:38:39.623395+0000","mapping_epoch":400,"log_start":"409'3567","ondisk_log_start":"409'3567","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7588,"log_dups_size":0,"ondisk_log_size":7588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:31:05.105491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105054208,"num_objects":1603,"num_object_clones":0,"num_object_copies":4809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11155,"num_write_kb":408256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":638,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.300270+0000","last_change":"2023-10-01T09:37:22.770678+0000","last_active":"2023-10-01T09:38:31.300270+0000","last_peered":"2023-10-01T09:38:31.300270+0000","last_clean":"2023-10-01T09:38:31.300270+0000","last_became_active":"2023-10-01T09:37:22.674516+0000","last_became_peered":"2023-10-01T09:37:22.674516+0000","last_unstale":"2023-10-01T09:38:31.300270+0000","last_undegraded":"2023-10-01T09:38:31.300270+0000","last_fullsized":"2023-10-01T09:38:31.300270+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":485,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.401624+0000","last_change":"2023-10-01T09:36:22.225663+0000","last_active":"2023-10-01T09:38:21.401624+0000","last_peered":"2023-10-01T09:38:21.401624+0000","last_clean":"2023-10-01T09:38:21.401624+0000","last_became_active":"2023-10-01T09:36:22.225382+0000","last_became_peered":"2023-10-01T09:36:22.225382+0000","last_unstale":"2023-10-01T09:38:21.401624+0000","last_undegraded":"2023-10-01T09:38:21.401624+0000","last_fullsized":"2023-10-01T09:38:21.401624+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:19:41.386221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":484,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.404023+0000","last_change":"2023-10-01T09:35:34.551948+0000","last_active":"2023-10-01T09:38:21.404023+0000","last_peered":"2023-10-01T09:38:21.404023+0000","last_clean":"2023-10-01T09:38:21.404023+0000","last_became_active":"2023-10-01T09:35:34.551665+0000","last_became_peered":"2023-10-01T09:35:34.551665+0000","last_unstale":"2023-10-01T09:38:21.404023+0000","last_undegraded":"2023-10-01T09:38:21.404023+0000","last_fullsized":"2023-10-01T09:38:21.404023+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:31:30.655381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"457'11346","reported_seq":11458,"reported_epoch":457,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:38:39.739497+0000","last_change":"2023-10-01T09:36:23.705972+0000","last_active":"2023-10-01T09:38:39.739497+0000","last_peered":"2023-10-01T09:38:39.739497+0000","last_clean":"2023-10-01T09:36:21.165407+0000","last_became_active":"2023-10-01T09:36:23.529481+0000","last_became_peered":"2023-10-01T09:36:23.529481+0000","last_unstale":"2023-10-01T09:38:39.739497+0000","last_undegraded":"2023-10-01T09:38:39.739497+0000","last_fullsized":"2023-10-01T09:38:39.739497+0000","mapping_epoch":431,"log_start":"410'4444","ondisk_log_start":"410'4444","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6902,"log_dups_size":0,"ondisk_log_size":6902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:01:56.211865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109969408,"num_objects":1678,"num_object_clones":0,"num_object_copies":5034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1673,"num_objects_unfound":0,"num_objects_dirty":1678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11346,"num_write_kb":416768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":1678}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":896,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.300826+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:38:31.300826+0000","last_peered":"2023-10-01T09:38:31.300826+0000","last_clean":"2023-10-01T09:38:31.300826+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:38:31.300826+0000","last_undegraded":"2023-10-01T09:38:31.300826+0000","last_fullsized":"2023-10-01T09:38:31.300826+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":513,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.366065+0000","last_change":"2023-10-01T09:35:34.559426+0000","last_active":"2023-10-01T09:38:21.366065+0000","last_peered":"2023-10-01T09:38:21.366065+0000","last_clean":"2023-10-01T09:38:21.366065+0000","last_became_active":"2023-10-01T09:35:34.558890+0000","last_became_peered":"2023-10-01T09:35:34.558890+0000","last_unstale":"2023-10-01T09:38:21.366065+0000","last_undegraded":"2023-10-01T09:38:21.366065+0000","last_fullsized":"2023-10-01T09:38:21.366065+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:55:37.464032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":449,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.401159+0000","last_change":"2023-10-01T09:33:11.656981+0000","last_active":"2023-10-01T09:38:21.401159+0000","last_peered":"2023-10-01T09:38:21.401159+0000","last_clean":"2023-10-01T09:38:21.401159+0000","last_became_active":"2023-10-01T09:33:11.656334+0000","last_became_peered":"2023-10-01T09:33:11.656334+0000","last_unstale":"2023-10-01T09:38:21.401159+0000","last_undegraded":"2023-10-01T09:38:21.401159+0000","last_fullsized":"2023-10-01T09:38:21.401159+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:50:15.562113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"457'11107","reported_seq":11181,"reported_epoch":457,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:38:39.921715+0000","last_change":"2023-10-01T09:36:23.622968+0000","last_active":"2023-10-01T09:38:39.921715+0000","last_peered":"2023-10-01T09:38:39.921715+0000","last_clean":"2023-10-01T09:36:21.149981+0000","last_became_active":"2023-10-01T09:36:23.603244+0000","last_became_peered":"2023-10-01T09:36:23.603244+0000","last_unstale":"2023-10-01T09:38:39.921715+0000","last_undegraded":"2023-10-01T09:38:39.921715+0000","last_fullsized":"2023-10-01T09:38:39.921715+0000","mapping_epoch":431,"log_start":"410'4205","ondisk_log_start":"410'4205","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6902,"log_dups_size":0,"ondisk_log_size":6902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110624792,"num_objects":1689,"num_object_clones":0,"num_object_copies":5067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3378,"num_objects_unfound":0,"num_objects_dirty":1689,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11107,"num_write_kb":409409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,1,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":1689}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":397,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.301140+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:38:31.301140+0000","last_peered":"2023-10-01T09:38:31.301140+0000","last_clean":"2023-10-01T09:38:31.301140+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:38:31.301140+0000","last_undegraded":"2023-10-01T09:38:31.301140+0000","last_fullsized":"2023-10-01T09:38:31.301140+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1087,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.369241+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:38:21.369241+0000","last_peered":"2023-10-01T09:38:21.369241+0000","last_clean":"2023-10-01T09:38:21.369241+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:38:21.369241+0000","last_undegraded":"2023-10-01T09:38:21.369241+0000","last_fullsized":"2023-10-01T09:38:21.369241+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":179,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:24.627881+0000","last_change":"2023-10-01T09:36:22.236397+0000","last_active":"2023-10-01T09:38:24.627881+0000","last_peered":"2023-10-01T09:38:24.627881+0000","last_clean":"2023-10-01T09:38:24.627881+0000","last_became_active":"2023-10-01T09:36:22.236066+0000","last_became_peered":"2023-10-01T09:36:22.236066+0000","last_unstale":"2023-10-01T09:38:24.627881+0000","last_undegraded":"2023-10-01T09:38:24.627881+0000","last_fullsized":"2023-10-01T09:38:24.627881+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:04:06.876501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.c","version":"458'11434","reported_seq":11518,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-10-01T09:38:41.076026+0000","last_change":"2023-10-01T09:37:23.665454+0000","last_active":"2023-10-01T09:38:41.076026+0000","last_peered":"2023-10-01T09:38:41.076026+0000","last_clean":"2023-10-01T09:38:41.076026+0000","last_became_active":"2023-10-01T09:37:23.635951+0000","last_became_peered":"2023-10-01T09:37:23.635951+0000","last_unstale":"2023-10-01T09:38:41.076026+0000","last_undegraded":"2023-10-01T09:38:41.076026+0000","last_fullsized":"2023-10-01T09:38:41.076026+0000","mapping_epoch":446,"log_start":"405'1400","ondisk_log_start":"405'1400","created":400,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":10034,"log_dups_size":0,"ondisk_log_size":10034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103415808,"num_objects":1578,"num_object_clones":0,"num_object_copies":4734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11434,"num_write_kb":416384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":218,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.252417+0000","last_change":"2023-10-01T09:37:22.671608+0000","last_active":"2023-10-01T09:38:31.252417+0000","last_peered":"2023-10-01T09:38:31.252417+0000","last_clean":"2023-10-01T09:38:31.252417+0000","last_became_active":"2023-10-01T09:37:22.671097+0000","last_became_peered":"2023-10-01T09:37:22.671097+0000","last_unstale":"2023-10-01T09:38:31.252417+0000","last_undegraded":"2023-10-01T09:38:31.252417+0000","last_fullsized":"2023-10-01T09:38:31.252417+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":471,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.404111+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:38:21.404111+0000","last_peered":"2023-10-01T09:38:21.404111+0000","last_clean":"2023-10-01T09:38:21.404111+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:38:21.404111+0000","last_undegraded":"2023-10-01T09:38:21.404111+0000","last_fullsized":"2023-10-01T09:38:21.404111+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":180,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.357865+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:38:21.357865+0000","last_peered":"2023-10-01T09:38:21.357865+0000","last_clean":"2023-10-01T09:38:21.357865+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:38:21.357865+0000","last_undegraded":"2023-10-01T09:38:21.357865+0000","last_fullsized":"2023-10-01T09:38:21.357865+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":421,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:29.645403+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:38:29.645403+0000","last_peered":"2023-10-01T09:38:29.645403+0000","last_clean":"2023-10-01T09:38:29.645403+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:38:29.645403+0000","last_undegraded":"2023-10-01T09:38:29.645403+0000","last_fullsized":"2023-10-01T09:38:29.645403+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":525,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.366130+0000","last_change":"2023-10-01T09:37:22.674112+0000","last_active":"2023-10-01T09:38:21.366130+0000","last_peered":"2023-10-01T09:38:21.366130+0000","last_clean":"2023-10-01T09:38:21.366130+0000","last_became_active":"2023-10-01T09:37:22.673785+0000","last_became_peered":"2023-10-01T09:37:22.673785+0000","last_unstale":"2023-10-01T09:38:21.366130+0000","last_undegraded":"2023-10-01T09:38:21.366130+0000","last_fullsized":"2023-10-01T09:38:21.366130+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:05.627976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":192,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.358002+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:38:21.358002+0000","last_peered":"2023-10-01T09:38:21.358002+0000","last_clean":"2023-10-01T09:38:21.358002+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:38:21.358002+0000","last_undegraded":"2023-10-01T09:38:21.358002+0000","last_fullsized":"2023-10-01T09:38:21.358002+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":256,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.254471+0000","last_change":"2023-10-01T09:37:22.671305+0000","last_active":"2023-10-01T09:38:31.254471+0000","last_peered":"2023-10-01T09:38:31.254471+0000","last_clean":"2023-10-01T09:38:31.254471+0000","last_became_active":"2023-10-01T09:37:22.670844+0000","last_became_peered":"2023-10-01T09:37:22.670844+0000","last_unstale":"2023-10-01T09:38:31.254471+0000","last_undegraded":"2023-10-01T09:38:31.254471+0000","last_fullsized":"2023-10-01T09:38:31.254471+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":192,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.358754+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:38:21.358754+0000","last_peered":"2023-10-01T09:38:21.358754+0000","last_clean":"2023-10-01T09:38:21.358754+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:38:21.358754+0000","last_undegraded":"2023-10-01T09:38:21.358754+0000","last_fullsized":"2023-10-01T09:38:21.358754+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":367,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.400982+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:38:21.400982+0000","last_peered":"2023-10-01T09:38:21.400982+0000","last_clean":"2023-10-01T09:38:21.400982+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:38:21.400982+0000","last_undegraded":"2023-10-01T09:38:21.400982+0000","last_fullsized":"2023-10-01T09:38:21.400982+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":23,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-10-01T09:38:31.306814+0000","last_change":"2023-10-01T09:37:22.673935+0000","last_active":"2023-10-01T09:38:31.306814+0000","last_peered":"2023-10-01T09:38:31.306814+0000","last_clean":"2023-10-01T09:38:31.306814+0000","last_became_active":"2023-10-01T09:37:22.673609+0000","last_became_peered":"2023-10-01T09:37:22.673609+0000","last_unstale":"2023-10-01T09:38:31.306814+0000","last_undegraded":"2023-10-01T09:38:31.306814+0000","last_fullsized":"2023-10-01T09:38:31.306814+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:02:43.715678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":403,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.401542+0000","last_change":"2023-10-01T09:37:22.668216+0000","last_active":"2023-10-01T09:38:21.401542+0000","last_peered":"2023-10-01T09:38:21.401542+0000","last_clean":"2023-10-01T09:38:21.401542+0000","last_became_active":"2023-10-01T09:37:22.667243+0000","last_became_peered":"2023-10-01T09:37:22.667243+0000","last_unstale":"2023-10-01T09:38:21.401542+0000","last_undegraded":"2023-10-01T09:38:21.401542+0000","last_fullsized":"2023-10-01T09:38:21.401542+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:45:16.527071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":596,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.401105+0000","last_change":"2023-10-01T09:35:34.658812+0000","last_active":"2023-10-01T09:38:21.401105+0000","last_peered":"2023-10-01T09:38:21.401105+0000","last_clean":"2023-10-01T09:38:21.401105+0000","last_became_active":"2023-10-01T09:35:34.550952+0000","last_became_peered":"2023-10-01T09:35:34.550952+0000","last_unstale":"2023-10-01T09:38:21.401105+0000","last_undegraded":"2023-10-01T09:38:21.401105+0000","last_fullsized":"2023-10-01T09:38:21.401105+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:24:17.304117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":427,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-10-01T09:38:21.365652+0000","last_change":"2023-10-01T09:32:27.445299+0000","last_active":"2023-10-01T09:38:21.365652+0000","last_peered":"2023-10-01T09:38:21.365652+0000","last_clean":"2023-10-01T09:38:21.365652+0000","last_became_active":"2023-10-01T09:32:27.444956+0000","last_became_peered":"2023-10-01T09:32:27.444956+0000","last_unstale":"2023-10-01T09:38:21.365652+0000","last_undegraded":"2023-10-01T09:38:21.365652+0000","last_fullsized":"2023-10-01T09:38:21.365652+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:49:07.862356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":123,"num_bytes_recovered":34520,"num_keys_recovered":90,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1671888920,"num_objects":25512,"num_object_clones":0,"num_object_copies":76536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14967,"num_objects_misplaced":14744,"num_objects_unfound":0,"num_objects_dirty":25512,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":179254,"num_write_kb":6552449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19479,"num_bytes_recovered":1275723776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6664499200,"data_stored":6664486984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120112,"ondisk_log_size":120112,"up":48,"acting":44,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":297,"num_bytes_recovered":12582969,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430657,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2315604,"kb_used_data":1420684,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":91429548,"statfs":{"total":95995035648,"available":93623857152,"internally_reserved":0,"allocated":1454780416,"data_stored":1453876814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:37:43 2023","interfaces":[{"interface":"back","average":{"1min":1.853,"5min":5.2190000000000003,"15min":3.427},"min":{"1min":0.27100000000000002,"5min":0.218,"15min":0.19500000000000001},"max":{"1min":11.525,"5min":108.90000000000001,"15min":108.90000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":2.4929999999999999,"5min":5.2830000000000004,"15min":3.637},"min":{"1min":0.26400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.451000000000001,"5min":109.187,"15min":109.187},"last":0.59099999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:37:59 2023","interfaces":[{"interface":"back","average":{"1min":2.3340000000000001,"5min":7.1070000000000002,"15min":4.4500000000000002},"min":{"1min":0.315,"5min":0.23300000000000001,"15min":0.222},"max":{"1min":8.3610000000000007,"5min":133.54499999999999,"15min":205.34200000000001},"last":1.863},{"interface":"front","average":{"1min":2.206,"5min":7.3970000000000002,"15min":4.5490000000000004},"min":{"1min":0.27500000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.0209999999999999,"5min":133.39400000000001,"15min":205.44499999999999},"last":1.97}]},{"osd":2,"last update":"Sun Oct 1 09:38:36 2023","interfaces":[{"interface":"back","average":{"1min":2.3969999999999998,"5min":2.145,"15min":2.0680000000000001},"min":{"1min":0.29099999999999998,"5min":0.28299999999999997,"15min":0.249},"max":{"1min":11.602,"5min":14.901999999999999,"15min":14.901999999999999},"last":1.5009999999999999},{"interface":"front","average":{"1min":2.3010000000000002,"5min":1.8520000000000001,"15min":1.903},"min":{"1min":0.30599999999999999,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":10.627000000000001,"5min":10.627000000000001,"15min":12.388999999999999},"last":2.0329999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:37:59 2023","interfaces":[{"interface":"back","average":{"1min":2.6659999999999999,"5min":3.052,"15min":3.5449999999999999},"min":{"1min":0.54900000000000004,"5min":0.32900000000000001,"15min":0.28199999999999997},"max":{"1min":9.1959999999999997,"5min":76.555000000000007,"15min":164.595},"last":3.7050000000000001},{"interface":"front","average":{"1min":2.0449999999999999,"5min":3.1360000000000001,"15min":3.444},"min":{"1min":0.49099999999999999,"5min":0.33200000000000002,"15min":0.313},"max":{"1min":4.6799999999999997,"5min":76.534000000000006,"15min":164.60499999999999},"last":5.577}]},{"osd":4,"last update":"Sun Oct 1 09:37:43 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":1.1080000000000001,"15min":0.97199999999999998},"min":{"1min":0.29999999999999999,"5min":0.26700000000000002,"15min":0.23499999999999999},"max":{"1min":0.81899999999999995,"5min":9.6020000000000003,"15min":9.6020000000000003},"last":2.4969999999999999},{"interface":"front","average":{"1min":0.82199999999999995,"5min":1.198,"15min":0.996},"min":{"1min":0.28599999999999998,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":5.1870000000000003,"5min":8.0009999999999994,"15min":8.0009999999999994},"last":0.83699999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":3.165,"5min":1.494,"15min":1.1519999999999999},"min":{"1min":0.30299999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":45.222000000000001,"5min":45.222000000000001,"15min":45.222000000000001},"last":2.2210000000000001},{"interface":"front","average":{"1min":3.3140000000000001,"5min":1.6899999999999999,"15min":1.0920000000000001},"min":{"1min":0.36499999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":44.905000000000001,"5min":44.905000000000001,"15min":44.905000000000001},"last":5.6180000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:38:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":1.24,"15min":0.98599999999999999},"min":{"1min":0.372,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":5.1580000000000004,"5min":11.853999999999999,"15min":11.853999999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":1.034,"5min":1.27,"15min":0.93999999999999995},"min":{"1min":0.39900000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.9609999999999999,"5min":7.266,"15min":7.266},"last":5.6440000000000001}]}]},{"osd":6,"up_from":294,"seq":1262720385168,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":637260,"kb_used_data":295556,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":93107892,"statfs":{"total":95995035648,"available":95342481408,"internally_reserved":0,"allocated":302649344,"data_stored":301753809,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:38:07 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":2.4279999999999999,"15min":3.5760000000000001},"min":{"1min":0.246,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.7480000000000002,"5min":40.426000000000002,"15min":130.02699999999999},"last":2.6909999999999998},{"interface":"front","average":{"1min":2.0019999999999998,"5min":2.2290000000000001,"15min":3.3570000000000002},"min":{"1min":0.25700000000000001,"5min":0.221,"15min":0.221},"max":{"1min":7.2030000000000003,"5min":40.447000000000003,"15min":133.72900000000001},"last":6.4740000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:38:07 2023","interfaces":[{"interface":"back","average":{"1min":3.7250000000000001,"5min":5.6050000000000004,"15min":3.9980000000000002},"min":{"1min":0.30599999999999999,"5min":0.26700000000000002,"15min":0.23899999999999999},"max":{"1min":18.497,"5min":121.95,"15min":121.95},"last":5.0259999999999998},{"interface":"front","average":{"1min":3.419,"5min":5.4130000000000003,"15min":3.8959999999999999},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.24399999999999999},"max":{"1min":18.445,"5min":121.98099999999999,"15min":121.98099999999999},"last":6.3159999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:38:07 2023","interfaces":[{"interface":"back","average":{"1min":3.5649999999999999,"5min":1.853,"15min":2.2759999999999998},"min":{"1min":0.30399999999999999,"5min":0.27600000000000002,"15min":0.254},"max":{"1min":9.657,"5min":9.657,"15min":26.289999999999999},"last":7.4630000000000001},{"interface":"front","average":{"1min":2.9710000000000001,"5min":1.7,"15min":2.286},"min":{"1min":0.33200000000000002,"5min":0.309,"15min":0.27100000000000002},"max":{"1min":21.704999999999998,"5min":21.704999999999998,"15min":26.388999999999999},"last":2.5590000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:38:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0779999999999998,"5min":2.7229999999999999,"15min":2.6659999999999999},"min":{"1min":0.33500000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.9980000000000002,"5min":89.753,"15min":89.753},"last":3.726},{"interface":"front","average":{"1min":2.984,"5min":2.6099999999999999,"15min":2.157},"min":{"1min":0.36399999999999999,"5min":0.318,"15min":0.29299999999999998},"max":{"1min":9.1850000000000005,"5min":89.849999999999994,"15min":89.849999999999994},"last":7.2670000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:38:07 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.89300000000000002,"15min":0.67400000000000004},"min":{"1min":0.27100000000000002,"5min":0.246,"15min":0.246},"max":{"1min":2.036,"5min":6.0640000000000001,"15min":6.0640000000000001},"last":0.439},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.83899999999999997,"15min":0.77400000000000002},"min":{"1min":0.35799999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":4.048,"5min":8.8249999999999993,"15min":9.4090000000000007},"last":0.45600000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:38:18 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":1.0109999999999999,"15min":1.0349999999999999},"min":{"1min":0.29699999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.0359999999999996,"5min":12.023999999999999,"15min":12.023999999999999},"last":1.579},{"interface":"front","average":{"1min":1.115,"5min":1.121,"15min":1.127},"min":{"1min":0.35699999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.6360000000000001,"5min":6.6360000000000001,"15min":6.6360000000000001},"last":0.92800000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:38:07 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":2.1440000000000001,"15min":3.0720000000000001},"min":{"1min":0.35299999999999998,"5min":0.30499999999999999,"15min":0.27900000000000003},"max":{"1min":4.484,"5min":94.245999999999995,"15min":115.11},"last":1.3979999999999999},{"interface":"front","average":{"1min":1.3380000000000001,"5min":2.3119999999999998,"15min":3.0409999999999999},"min":{"1min":0.39000000000000001,"5min":0.308,"15min":0.28799999999999998},"max":{"1min":6.9699999999999998,"5min":94.715000000000003,"15min":114.666},"last":4.9859999999999998}]}]},{"osd":1,"up_from":94,"seq":403726926144,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1703588,"kb_used_data":950108,"kb_used_omap":7,"kb_used_meta":753144,"kb_avail":92041564,"statfs":{"total":95995035648,"available":94250561536,"internally_reserved":0,"allocated":972910592,"data_stored":971052645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":771219879},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:37:47 2023","interfaces":[{"interface":"back","average":{"1min":5.0449999999999999,"5min":6.4269999999999996,"15min":4.4240000000000004},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":22.736999999999998,"5min":154.21299999999999,"15min":154.21299999999999},"last":22.835999999999999},{"interface":"front","average":{"1min":5.4329999999999998,"5min":4.6159999999999997,"15min":3.8239999999999998},"min":{"1min":0.27300000000000002,"5min":0.19500000000000001,"15min":0.188},"max":{"1min":25.047999999999998,"5min":95.236999999999995,"15min":95.236999999999995},"last":15.617000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:38:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1760000000000002,"5min":2.4399999999999999,"15min":3.5009999999999999},"min":{"1min":0.22900000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":11.358000000000001,"5min":13.898999999999999,"15min":56.573},"last":22.891999999999999},{"interface":"front","average":{"1min":4.9669999999999996,"5min":4.4909999999999997,"15min":4.4699999999999998},"min":{"1min":0.40200000000000002,"5min":0.254,"15min":0.20499999999999999},"max":{"1min":11.27,"5min":154.197,"15min":154.197},"last":10.385999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:38:06 2023","interfaces":[{"interface":"back","average":{"1min":9.141,"5min":4.7759999999999998,"15min":4.7919999999999998},"min":{"1min":0.40899999999999997,"5min":0.219,"15min":0.219},"max":{"1min":95.305000000000007,"5min":95.305000000000007,"15min":200.56800000000001},"last":15.664999999999999},{"interface":"front","average":{"1min":10.339,"5min":6.46,"15min":5.3890000000000002},"min":{"1min":0.42299999999999999,"5min":0.23000000000000001,"15min":0.21199999999999999},"max":{"1min":98.033000000000001,"5min":154.483,"15min":200.63800000000001},"last":10.273999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:37:47 2023","interfaces":[{"interface":"back","average":{"1min":3.9319999999999999,"5min":3.6459999999999999,"15min":2.2120000000000002},"min":{"1min":0.42899999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":18.251999999999999,"5min":154.101,"15min":154.101},"last":2.4180000000000001},{"interface":"front","average":{"1min":4.069,"5min":3.8959999999999999,"15min":2.6789999999999998},"min":{"1min":0.46999999999999997,"5min":0.32800000000000001,"15min":0.27500000000000002},"max":{"1min":13.773999999999999,"5min":154.56999999999999,"15min":154.56999999999999},"last":10.222}]},{"osd":5,"last update":"Sun Oct 1 09:38:26 2023","interfaces":[{"interface":"back","average":{"1min":3.125,"5min":4.0670000000000002,"15min":1.9590000000000001},"min":{"1min":0.37,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":21.219999999999999,"5min":154.22999999999999,"15min":154.22999999999999},"last":2.3719999999999999},{"interface":"front","average":{"1min":2.8769999999999998,"5min":4.1950000000000003,"15min":2.7469999999999999},"min":{"1min":0.49399999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":7.9909999999999997,"5min":154.54400000000001,"15min":154.54400000000001},"last":10.305}]},{"osd":6,"last update":"Sun Oct 1 09:38:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2000000000000002,"5min":3.8439999999999999,"15min":3.1200000000000001},"min":{"1min":0.375,"5min":0.313,"15min":0.29899999999999999},"max":{"1min":7.2089999999999996,"5min":154.12200000000001,"15min":154.12200000000001},"last":10.353},{"interface":"front","average":{"1min":4.3710000000000004,"5min":3.9239999999999999,"15min":2.5960000000000001},"min":{"1min":0.46999999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":21.096,"5min":154.16999999999999,"15min":154.16999999999999},"last":2.4009999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:37:47 2023","interfaces":[{"interface":"back","average":{"1min":4.0019999999999998,"5min":4.0170000000000003,"15min":2.5600000000000001},"min":{"1min":0.46100000000000002,"5min":0.33900000000000002,"15min":0.316},"max":{"1min":18.286000000000001,"5min":154.20699999999999,"15min":154.20699999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":4.2619999999999996,"5min":4.3810000000000002,"15min":2.8170000000000002},"min":{"1min":0.43099999999999999,"5min":0.34300000000000003,"15min":0.30199999999999999},"max":{"1min":13.819000000000001,"5min":154.51599999999999,"15min":154.51599999999999},"last":10.34}]}]},{"osd":0,"up_from":89,"seq":382252089678,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1259960,"kb_used_data":484208,"kb_used_omap":8,"kb_used_meta":775735,"kb_avail":92485192,"statfs":{"total":95995035648,"available":94704836608,"internally_reserved":0,"allocated":495828992,"data_stored":493957648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":794353602},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:37:55 2023","interfaces":[{"interface":"back","average":{"1min":11.526999999999999,"5min":8.2189999999999994,"15min":4.5229999999999997},"min":{"1min":0.26000000000000001,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":185.78399999999999,"5min":216.821,"15min":216.821},"last":4.9470000000000001},{"interface":"front","average":{"1min":12.972,"5min":8.3070000000000004,"15min":5.3360000000000003},"min":{"1min":0.28199999999999997,"5min":0.246,"15min":0.21199999999999999},"max":{"1min":186.077,"5min":217.02699999999999,"15min":217.02699999999999},"last":0.752}]},{"osd":2,"last update":"Sun Oct 1 09:38:27 2023","interfaces":[{"interface":"back","average":{"1min":3.7669999999999999,"5min":3.1600000000000001,"15min":2.4159999999999999},"min":{"1min":0.25900000000000001,"5min":0.21299999999999999,"15min":0.20100000000000001},"max":{"1min":10.297000000000001,"5min":17.462,"15min":17.731000000000002},"last":0.56299999999999994},{"interface":"front","average":{"1min":4.1269999999999998,"5min":2.8420000000000001,"15min":3.0609999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.189},"max":{"1min":25.844999999999999,"5min":25.844999999999999,"15min":25.844999999999999},"last":0.68300000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:37:55 2023","interfaces":[{"interface":"back","average":{"1min":3.919,"5min":5.5250000000000004,"15min":4.7619999999999996},"min":{"1min":0.29399999999999998,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":14.224,"5min":222.16499999999999,"15min":222.16499999999999},"last":0.65400000000000003},{"interface":"front","average":{"1min":4.4539999999999997,"5min":5.1539999999999999,"15min":5.0030000000000001},"min":{"1min":0.32000000000000001,"5min":0.24399999999999999,"15min":0.219},"max":{"1min":11.367000000000001,"5min":222.227,"15min":222.227},"last":5.6139999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:37:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9369999999999998,"5min":2.7429999999999999,"15min":2.327},"min":{"1min":0.35199999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.1319999999999997,"5min":62.206000000000003,"15min":62.206000000000003},"last":2.137},{"interface":"front","average":{"1min":4.0119999999999996,"5min":2.9409999999999998,"15min":2.3530000000000002},"min":{"1min":0.33700000000000002,"5min":0.28299999999999997,"15min":0.26900000000000002},"max":{"1min":12.222,"5min":62.073999999999998,"15min":62.073999999999998},"last":1.72}]},{"osd":5,"last update":"Sun Oct 1 09:38:20 2023","interfaces":[{"interface":"back","average":{"1min":3.5939999999999999,"5min":2.6869999999999998,"15min":2.069},"min":{"1min":0.40100000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":9.532,"5min":16.808,"15min":16.808},"last":1.6850000000000001},{"interface":"front","average":{"1min":3.012,"5min":2.5899999999999999,"15min":1.9510000000000001},"min":{"1min":0.36599999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.9740000000000002,"5min":17.507000000000001,"15min":17.507000000000001},"last":0.59299999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:38:15 2023","interfaces":[{"interface":"back","average":{"1min":2.7130000000000001,"5min":2.3730000000000002,"15min":1.8720000000000001},"min":{"1min":0.39200000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":8.9920000000000009,"5min":17.535,"15min":17.535},"last":0.61399999999999999},{"interface":"front","average":{"1min":3.2989999999999999,"5min":2.3730000000000002,"15min":2.399},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":8.8819999999999997,"5min":15.651999999999999,"15min":18.521000000000001},"last":0.70999999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:37:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1659999999999999,"5min":2.9630000000000001,"15min":2.0409999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.34499999999999997},"max":{"1min":8.923,"5min":35.130000000000003,"15min":35.130000000000003},"last":0.63400000000000001},{"interface":"front","average":{"1min":3.3559999999999999,"5min":2.8610000000000002,"15min":2.4740000000000002},"min":{"1min":0.39700000000000002,"5min":0.38200000000000001,"15min":0.35099999999999998},"max":{"1min":8.8339999999999996,"5min":38.590000000000003,"15min":38.590000000000003},"last":0.73099999999999998}]}]},{"osd":2,"up_from":99,"seq":425201762618,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":971768,"kb_used_data":246960,"kb_used_omap":11,"kb_used_meta":724788,"kb_avail":92773384,"statfs":{"total":95995035648,"available":94999945216,"internally_reserved":0,"allocated":252887040,"data_stored":251072326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":742183799},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:37:40 2023","interfaces":[{"interface":"back","average":{"1min":7.923,"5min":7.4219999999999997,"15min":7.2469999999999999},"min":{"1min":0.24399999999999999,"5min":0.191,"15min":0.191},"max":{"1min":75.989000000000004,"5min":201.892,"15min":224.56},"last":10.737},{"interface":"front","average":{"1min":8.4109999999999996,"5min":7.6319999999999997,"15min":7.6950000000000003},"min":{"1min":0.221,"5min":0.221,"15min":0.20699999999999999},"max":{"1min":73.721999999999994,"5min":200.089,"15min":225.66900000000001},"last":10.813000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:38:35 2023","interfaces":[{"interface":"back","average":{"1min":5.4980000000000002,"5min":8.9640000000000004,"15min":8.8260000000000005},"min":{"1min":0.23899999999999999,"5min":0.20300000000000001,"15min":0.191},"max":{"1min":51.863,"5min":176.52000000000001,"15min":224.917},"last":0.21299999999999999},{"interface":"front","average":{"1min":8.3680000000000003,"5min":10.141999999999999,"15min":9.9930000000000003},"min":{"1min":0.40100000000000002,"5min":0.22900000000000001,"15min":0.22},"max":{"1min":58.590000000000003,"5min":179.85300000000001,"15min":225.625},"last":0.24199999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:38:00 2023","interfaces":[{"interface":"back","average":{"1min":4.8470000000000004,"5min":3.0350000000000001,"15min":5.0949999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":30.039000000000001,"5min":50.289000000000001,"15min":226.124},"last":0.253},{"interface":"front","average":{"1min":2.931,"5min":2.734,"15min":4.8319999999999999},"min":{"1min":0.245,"5min":0.215,"15min":0.20300000000000001},"max":{"1min":17.515000000000001,"5min":54.115000000000002,"15min":226.32900000000001},"last":0.26300000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:38:35 2023","interfaces":[{"interface":"back","average":{"1min":2.895,"5min":1.5880000000000001,"15min":2.6400000000000001},"min":{"1min":0.34899999999999998,"5min":0.29999999999999999,"15min":0.28599999999999998},"max":{"1min":16.048999999999999,"5min":16.048999999999999,"15min":224.90199999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":4.0780000000000003,"5min":2.2650000000000001,"15min":2.9710000000000001},"min":{"1min":0.34799999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.457000000000001,"5min":11.475,"15min":225.386},"last":0.45900000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:38:20 2023","interfaces":[{"interface":"back","average":{"1min":4.9409999999999998,"5min":2.1640000000000001,"15min":1.8280000000000001},"min":{"1min":0.34799999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":60.432000000000002,"5min":60.432000000000002,"15min":60.432000000000002},"last":0.29799999999999999},{"interface":"front","average":{"1min":6.5039999999999996,"5min":2.4140000000000001,"15min":1.6839999999999999},"min":{"1min":0.34000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":60.470999999999997,"5min":60.470999999999997,"15min":60.470999999999997},"last":0.35699999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:38:06 2023","interfaces":[{"interface":"back","average":{"1min":3.9409999999999998,"5min":2.1040000000000001,"15min":2.98},"min":{"1min":0.36799999999999999,"5min":0.34100000000000003,"15min":0.317},"max":{"1min":11.429,"5min":11.526999999999999,"15min":225.41499999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.8460000000000001,"5min":1.5529999999999999,"15min":2.4279999999999999},"min":{"1min":0.36499999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.3309999999999995,"5min":13.614000000000001,"15min":225.02000000000001},"last":0.44800000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:38:35 2023","interfaces":[{"interface":"back","average":{"1min":2.665,"5min":4.8250000000000002,"15min":3.7839999999999998},"min":{"1min":0.36199999999999999,"5min":0.33000000000000002,"15min":0.308},"max":{"1min":15.615,"5min":139.03299999999999,"15min":224.94999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":4.1779999999999999,"5min":5.0419999999999998,"15min":3.9929999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":16.018000000000001,"5min":139.065,"15min":224.92599999999999},"last":0.38300000000000001}]}]},{"osd":3,"up_from":104,"seq":446676599092,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2173236,"kb_used_data":1431468,"kb_used_omap":3,"kb_used_meta":741500,"kb_avail":91571916,"statfs":{"total":95995035648,"available":93769641984,"internally_reserved":0,"allocated":1465823232,"data_stored":1463899412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":759296743},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:38:00 2023","interfaces":[{"interface":"back","average":{"1min":3.5419999999999998,"5min":4.2489999999999997,"15min":5.7400000000000002},"min":{"1min":0.20599999999999999,"5min":0.186,"15min":0.18099999999999999},"max":{"1min":11.224,"5min":137.34399999999999,"15min":137.34399999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":4.4539999999999997,"5min":4.3710000000000004,"15min":5.6840000000000002},"min":{"1min":0.36699999999999999,"5min":0.20899999999999999,"15min":0.184},"max":{"1min":17.628,"5min":137.73400000000001,"15min":137.73400000000001},"last":0.621}]},{"osd":1,"last update":"Sun Oct 1 09:38:00 2023","interfaces":[{"interface":"back","average":{"1min":9.0719999999999992,"5min":8.7940000000000005,"15min":5.8570000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.20399999999999999},"max":{"1min":108.84999999999999,"5min":162.22999999999999,"15min":162.22999999999999},"last":7.1829999999999998},{"interface":"front","average":{"1min":11.337999999999999,"5min":9.1829999999999998,"15min":6.3460000000000001},"min":{"1min":0.40500000000000003,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":109.13200000000001,"5min":162.70699999999999,"15min":162.70699999999999},"last":5.2750000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:38:00 2023","interfaces":[{"interface":"back","average":{"1min":3.5350000000000001,"5min":2.1480000000000001,"15min":3.1850000000000001},"min":{"1min":0.23000000000000001,"5min":0.22800000000000001,"15min":0.20799999999999999},"max":{"1min":12.361000000000001,"5min":14.194000000000001,"15min":62.909999999999997},"last":4.1020000000000003},{"interface":"front","average":{"1min":3.0249999999999999,"5min":2.4009999999999998,"15min":3.8300000000000001},"min":{"1min":0.29299999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":11.202,"5min":15.523,"15min":78.594999999999999},"last":0.52800000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:38:00 2023","interfaces":[{"interface":"back","average":{"1min":3.113,"5min":2.2629999999999999,"15min":2.004},"min":{"1min":0.32700000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":11.153,"5min":29.419,"15min":29.419},"last":0.45200000000000001},{"interface":"front","average":{"1min":2.2290000000000001,"5min":2.4710000000000001,"15min":2.1720000000000002},"min":{"1min":0.28699999999999998,"5min":0.27900000000000003,"15min":0.23899999999999999},"max":{"1min":7.617,"5min":33.002000000000002,"15min":36.947000000000003},"last":0.54700000000000004}]},{"osd":5,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8480000000000001,"5min":2.2999999999999998,"15min":3.355},"min":{"1min":0.35699999999999998,"5min":0.309,"15min":0.309},"max":{"1min":7.6529999999999996,"5min":28.117999999999999,"15min":28.117999999999999},"last":7.2320000000000002},{"interface":"front","average":{"1min":3.5310000000000001,"5min":2.887,"15min":3.4449999999999998},"min":{"1min":0.34100000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":12.989000000000001,"5min":28.34,"15min":28.34},"last":4.0499999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:38:13 2023","interfaces":[{"interface":"back","average":{"1min":3.7360000000000002,"5min":2.323,"15min":2.4729999999999999},"min":{"1min":0.40100000000000002,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":13.032999999999999,"5min":13.032999999999999,"15min":18.361000000000001},"last":4.2149999999999999},{"interface":"front","average":{"1min":3.3849999999999998,"5min":2.3370000000000002,"15min":2.3969999999999998},"min":{"1min":0.42599999999999999,"5min":0.33600000000000002,"15min":0.32800000000000001},"max":{"1min":13.057,"5min":13.057,"15min":18.332000000000001},"last":4.4210000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:38:00 2023","interfaces":[{"interface":"back","average":{"1min":3.677,"5min":2.5070000000000001,"15min":2.6930000000000001},"min":{"1min":0.37,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":12.429,"5min":12.429,"15min":52.006},"last":4.1779999999999999},{"interface":"front","average":{"1min":3.0579999999999998,"5min":2.3260000000000001,"15min":2.5510000000000002},"min":{"1min":0.38600000000000001,"5min":0.311,"15min":0.311},"max":{"1min":11.109999999999999,"5min":13.191000000000001,"15min":51.890999999999998},"last":5.2300000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019342,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1311700,"kb_used_data":413324,"kb_used_omap":0,"kb_used_meta":898368,"kb_avail":92433452,"statfs":{"total":95995035648,"available":94651854848,"internally_reserved":0,"allocated":423243776,"data_stored":422340192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919928832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:37:48 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":6.0030000000000001,"15min":4.5339999999999998},"min":{"1min":0.28699999999999998,"5min":0.22600000000000001,"15min":0.223},"max":{"1min":9.2690000000000001,"5min":155.315,"15min":155.315},"last":0.246},{"interface":"front","average":{"1min":3.6850000000000001,"5min":6.6390000000000002,"15min":4.806},"min":{"1min":0.253,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":12.119999999999999,"5min":155.334,"15min":155.334},"last":4.258}]},{"osd":1,"last update":"Sun Oct 1 09:37:52 2023","interfaces":[{"interface":"back","average":{"1min":6.734,"5min":7.6390000000000002,"15min":6.3170000000000002},"min":{"1min":0.28299999999999997,"5min":0.25,"15min":0.25},"max":{"1min":88.885999999999996,"5min":190.61199999999999,"15min":190.61199999999999},"last":4.4219999999999997},{"interface":"front","average":{"1min":6.5570000000000004,"5min":9.8239999999999998,"15min":7.0449999999999999},"min":{"1min":0.30399999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":88.829999999999998,"5min":253.273,"15min":253.273},"last":3.1200000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:38:34 2023","interfaces":[{"interface":"back","average":{"1min":1.756,"5min":1.948,"15min":2.8580000000000001},"min":{"1min":0.30399999999999999,"5min":0.29099999999999998,"15min":0.26700000000000002},"max":{"1min":5.8019999999999996,"5min":8.6379999999999999,"15min":170.45400000000001},"last":4.2960000000000003},{"interface":"front","average":{"1min":2.7650000000000001,"5min":2.4940000000000002,"15min":2.5640000000000001},"min":{"1min":0.32000000000000001,"5min":0.316,"15min":0.25800000000000001},"max":{"1min":10.625,"5min":12.045,"15min":58.771000000000001},"last":5.9740000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:37:52 2023","interfaces":[{"interface":"back","average":{"1min":2.3519999999999999,"5min":3.8500000000000001,"15min":3.4660000000000002},"min":{"1min":0.33800000000000002,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":15.989000000000001,"5min":140.881,"15min":140.881},"last":1.216},{"interface":"front","average":{"1min":3.556,"5min":4.0709999999999997,"15min":3.7229999999999999},"min":{"1min":0.36799999999999999,"5min":0.317,"15min":0.30399999999999999},"max":{"1min":15.978999999999999,"5min":140.911,"15min":140.911},"last":0.39300000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:38:26 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.532,"15min":1.234},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.8810000000000002,"5min":20.457000000000001,"15min":20.457000000000001},"last":4.4569999999999999},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.4350000000000001,"15min":0.97799999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":3.9009999999999998,"5min":20.475000000000001,"15min":20.475000000000001},"last":4.4489999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:38:04 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.123,"15min":0.89600000000000002},"min":{"1min":0.34300000000000003,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":3.653,"5min":7.9710000000000001,"15min":11.358000000000001},"last":1.2509999999999999},{"interface":"front","average":{"1min":1.272,"5min":1.4039999999999999,"15min":0.95499999999999996},"min":{"1min":0.32700000000000001,"5min":0.32500000000000001,"15min":0.28999999999999998},"max":{"1min":6.9139999999999997,"5min":7.5229999999999997,"15min":10.076000000000001},"last":4.3220000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:38:34 2023","interfaces":[{"interface":"back","average":{"1min":2.673,"5min":1.6910000000000001,"15min":1.1240000000000001},"min":{"1min":0.371,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":38.034999999999997,"5min":38.034999999999997,"15min":38.034999999999997},"last":1.7010000000000001},{"interface":"front","average":{"1min":2.8620000000000001,"5min":2.125,"15min":1.331},"min":{"1min":0.39000000000000001,"5min":0.34200000000000003,"15min":0.28699999999999998},"max":{"1min":37.723999999999997,"5min":37.723999999999997,"15min":37.723999999999997},"last":4.3490000000000002}]}]},{"osd":5,"up_from":379,"seq":1627792605251,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1741188,"kb_used_data":1368252,"kb_used_omap":13,"kb_used_meta":372722,"kb_avail":92003964,"statfs":{"total":95995035648,"available":94212059136,"internally_reserved":0,"allocated":1401090048,"data_stored":1400190284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":381667826},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":3.4140000000000001,"15min":2.141},"min":{"1min":0.23499999999999999,"5min":0.221,"15min":0.221},"max":{"1min":6.4870000000000001,"5min":84.340999999999994,"15min":84.340999999999994},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.7729999999999999,"5min":3.7559999999999998,"15min":2.0720000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":5.6109999999999998,"5min":84.356999999999999,"15min":84.356999999999999},"last":0.40100000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":9.0960000000000001,"15min":4.1639999999999997},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":4.7670000000000003,"5min":236.488,"15min":236.488},"last":25.414000000000001},{"interface":"front","average":{"1min":3.234,"5min":8.9900000000000002,"15min":3.9260000000000002},"min":{"1min":0.27300000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":12.132999999999999,"5min":236.29900000000001,"15min":236.29900000000001},"last":25.484000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7090000000000001,"5min":2.1309999999999998,"15min":1.579},"min":{"1min":0.32500000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.5620000000000003,"5min":14.199999999999999,"15min":14.199999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.028,"5min":1.821,"15min":1.6879999999999999},"min":{"1min":0.31,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.298,"5min":14.430999999999999,"15min":14.430999999999999},"last":0.38500000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":2.012,"5min":1.7949999999999999,"15min":1.373},"min":{"1min":0.31,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.4800000000000004,"5min":14.004,"15min":14.004},"last":0.442},{"interface":"front","average":{"1min":1.9910000000000001,"5min":2.1469999999999998,"15min":1.5009999999999999},"min":{"1min":0.38100000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":8.4930000000000003,"5min":13.991,"15min":13.991},"last":0.45100000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":1.157,"15min":1.0109999999999999},"min":{"1min":0.27900000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.2120000000000002,"5min":12.808999999999999,"15min":12.808999999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.754,"5min":1.256,"15min":1.5289999999999999},"min":{"1min":0.32500000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.843,"5min":14.68,"15min":14.68},"last":0.33300000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":0.93200000000000005,"15min":1},"min":{"1min":0.29899999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.5690000000000008,"5min":8.5690000000000008,"15min":8.5690000000000008},"last":0.311},{"interface":"front","average":{"1min":0.59699999999999998,"5min":1.4179999999999999,"15min":1.47},"min":{"1min":0.316,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.968,"5min":14.193,"15min":14.193},"last":0.47499999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:38:19 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.3049999999999999,"15min":1.347},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":2.1309999999999998,"5min":10.813000000000001,"15min":10.813000000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.80500000000000005,"5min":1.113,"15min":0.96699999999999997},"min":{"1min":0.34899999999999998,"5min":0.318,"15min":0.318},"max":{"1min":3.4769999999999999,"5min":12.045999999999999,"15min":12.045999999999999},"last":0.46600000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":480313344,"data_stored":480313344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":952504320,"data_stored":952500248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":245301248,"data_stored":245301248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1454768128,"data_stored":1454768128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":414121984,"data_stored":414121984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1384124416,"data_stored":1384120344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":294256640,"data_stored":294256640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1439109120,"data_stored":1439105048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:38:46.835 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 21 2023-10-01T09:38:46.835 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:46.836 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:46.939 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:47.177 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:47.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:47.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:47.211 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:47.292 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:38:47.318 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:47.373 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:38:47.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:47.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:38:47.480 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:47.510 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:47.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:47 smithi190 ceph-mon[160435]: from='client.36312 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:38:47.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:47 smithi190 ceph-mon[160435]: pgmap v1301: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 14516/70899 objects degraded (20.474%); 13927/70899 objects misplaced (19.643%); 7.1 MiB/s, 113 objects/s recovering 2023-10-01T09:38:47.557 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:47.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:47.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:47.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:47 smithi138 ceph-mon[185886]: from='client.36312 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:38:47.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:47 smithi138 ceph-mon[185886]: pgmap v1301: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 14516/70899 objects degraded (20.474%); 13927/70899 objects misplaced (19.643%); 7.1 MiB/s, 113 objects/s recovering 2023-10-01T09:38:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:47 smithi138 ceph-mon[192513]: from='client.36312 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:38:47.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:47 smithi138 ceph-mon[192513]: pgmap v1301: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 14516/70899 objects degraded (20.474%); 13927/70899 objects misplaced (19.643%); 7.1 MiB/s, 113 objects/s recovering 2023-10-01T09:38:47.614 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:38:47.663 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:47.670 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:38:47.701 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:47.720 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:38:47.766 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:47.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:47.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:47.792 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:38:47.831 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:38:47.860 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:38:47.874 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:47.902 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:38:47.914 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:47.944 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:38:47.984 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:47.985 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:38:47.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:38:47.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:38:47.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:38:47.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:38:47.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:38:47.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:38:47.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:38:47.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:47.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:48.091 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:48.134 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:48.195 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:48.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:48.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:48.302 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:48.341 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:48.400 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:48.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:48.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:48.510 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:48.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:48 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2042473492' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:48.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:48 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 14516/70899 objects degraded (20.474%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:48 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2042473492' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:48 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 14516/70899 objects degraded (20.474%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:48.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:48 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2042473492' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:48.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:48 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 14516/70899 objects degraded (20.474%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:48.607 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:49.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:49 smithi190 ceph-mon[160435]: pgmap v1302: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 14516/70503 objects degraded (20.589%); 13927/70503 objects misplaced (19.754%); 7.1 MiB/s, 113 objects/s recovering 2023-10-01T09:38:49.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:49 smithi138 ceph-mon[185886]: pgmap v1302: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 14516/70503 objects degraded (20.589%); 13927/70503 objects misplaced (19.754%); 7.1 MiB/s, 113 objects/s recovering 2023-10-01T09:38:49.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:49 smithi138 ceph-mon[192513]: pgmap v1302: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 14516/70503 objects degraded (20.589%); 13927/70503 objects misplaced (19.754%); 7.1 MiB/s, 113 objects/s recovering 2023-10-01T09:38:50.183 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:50.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:50.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:50.294 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:50.323 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:50.386 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:50.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:50.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:50.492 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:50.601 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:50.741 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:50.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:50.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:50.846 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:50.910 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:50.984 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:51.250 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:51.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:51.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:51.362 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:51.396 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:51.411 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:38:51.412 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:38:51.451 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:51.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:51.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:51.556 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:51.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:51 smithi138 ceph-mon[192513]: pgmap v1303: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 13983/65199 objects degraded (21.447%); 13065/65199 objects misplaced (20.039%); 7.9 MiB/s, 126 objects/s recovering 2023-10-01T09:38:51.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:51 smithi138 ceph-mon[185886]: pgmap v1303: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 13983/65199 objects degraded (21.447%); 13065/65199 objects misplaced (20.039%); 7.9 MiB/s, 126 objects/s recovering 2023-10-01T09:38:51.651 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:51.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:51 smithi190 ceph-mon[160435]: pgmap v1303: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 422 op/s; 13983/65199 objects degraded (21.447%); 13065/65199 objects misplaced (20.039%); 7.9 MiB/s, 126 objects/s recovering 2023-10-01T09:38:52.370 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:38:52.635 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:38:52.692 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:52 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2398408014' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:38:52.692 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:52 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:38:52.692 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:52 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2398408014' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:38:52.692 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:52 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:38:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:52 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2398408014' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:38:52.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:52 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:38:52.890 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:52.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:52.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:52.996 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:53.084 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:53.362 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:53 smithi138 ceph-mon[185886]: pgmap v1304: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 13983/64395 objects degraded (21.714%); 13065/64395 objects misplaced (20.289%); 6.6 MiB/s, 106 objects/s recovering 2023-10-01T09:38:53.362 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:53 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:53.362 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:53 smithi138 ceph-mon[185886]: failed to encode map e459 with expected crc 2023-10-01T09:38:53.362 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:53 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:38:53.362 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:53 smithi138 ceph-mon[185886]: osdmap e459: 8 total, 8 up, 8 in 2023-10-01T09:38:53.362 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:53 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 13983/64395 objects degraded (21.714%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:53.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:53 smithi190 ceph-mon[160435]: pgmap v1304: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 13983/64395 objects degraded (21.714%); 13065/64395 objects misplaced (20.289%); 6.6 MiB/s, 106 objects/s recovering 2023-10-01T09:38:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:53 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:53 smithi190 ceph-mon[160435]: failed to encode map e459 with expected crc 2023-10-01T09:38:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:53 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:38:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:53 smithi190 ceph-mon[160435]: osdmap e459: 8 total, 8 up, 8 in 2023-10-01T09:38:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:53 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 13983/64395 objects degraded (21.714%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:53.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:53 smithi138 ceph-mon[192513]: pgmap v1304: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 13983/64395 objects degraded (21.714%); 13065/64395 objects misplaced (20.289%); 6.6 MiB/s, 106 objects/s recovering 2023-10-01T09:38:53.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:53 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:38:53.826 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:53 smithi138 ceph-mon[192513]: failed to encode map e459 with expected crc 2023-10-01T09:38:53.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:53 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:38:53.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:53 smithi138 ceph-mon[192513]: osdmap e459: 8 total, 8 up, 8 in 2023-10-01T09:38:53.827 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:53 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 13983/64395 objects degraded (21.714%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:54.303 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:54.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:54.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:54.320 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:38:54.417 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:54.445 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:54.497 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:54.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:54.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:54.546 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:54 smithi138 ceph-mon[192513]: pgmap v1306: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 13983/64020 objects degraded (21.842%); 13065/64020 objects misplaced (20.408%); 7.9 MiB/s, 126 objects/s recovering 2023-10-01T09:38:54.551 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:54 smithi138 ceph-mon[185886]: pgmap v1306: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 13983/64020 objects degraded (21.842%); 13065/64020 objects misplaced (20.408%); 7.9 MiB/s, 126 objects/s recovering 2023-10-01T09:38:54.602 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:54.632 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:54.696 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:54.700 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:54 smithi190 ceph-mon[160435]: pgmap v1306: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 13983/64020 objects degraded (21.842%); 13065/64020 objects misplaced (20.408%); 7.9 MiB/s, 126 objects/s recovering 2023-10-01T09:38:54.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:54.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:54.802 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:54.835 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:54.904 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:54.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:54.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:55.010 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:55.044 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:55.101 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:55.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:55.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:55.147 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:55.206 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:55.282 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:38:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:38:54] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.33.4" 2023-10-01T09:38:55.288 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:55.358 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:55.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:38:55.579 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:38:55.592 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:55.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:55.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:55.610 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T09:38:55.698 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:38:55.728 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:38:55.730 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:55.788 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:38:55.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:55.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:38:55.893 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:55.926 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:56.010 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:56.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:56.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:56.122 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:56.210 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:56.547 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:38:56.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:56 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1377308537' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-01T09:38:56.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:56 smithi138 ceph-mon[185886]: pgmap v1307: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 13426/58863 objects degraded (22.809%); 12252/58863 objects misplaced (20.814%); 8.5 MiB/s, 135 objects/s recovering 2023-10-01T09:38:56.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:56 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1377308537' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-01T09:38:56.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:56 smithi138 ceph-mon[192513]: pgmap v1307: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 13426/58863 objects degraded (22.809%); 12252/58863 objects misplaced (20.814%); 8.5 MiB/s, 135 objects/s recovering 2023-10-01T09:38:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:56 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1377308537' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-01T09:38:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:56 smithi190 ceph-mon[160435]: pgmap v1307: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 13426/58863 objects degraded (22.809%); 12252/58863 objects misplaced (20.814%); 8.5 MiB/s, 135 objects/s recovering 2023-10-01T09:38:57.636 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:57.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:57.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:57.748 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:38:57.805 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:57.977 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:57 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 13426/58863 objects degraded (22.809%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:57.978 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:57 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 13426/58863 objects degraded (22.809%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:58.033 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T09:38:58.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:57 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 13426/58863 objects degraded (22.809%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:38:58.083 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:38:58.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:58.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:38:58.192 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:38:58.224 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:58.288 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:38:58.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:58.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:38:58.395 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:38:58.454 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:58.577 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:38:58.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:58.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:38:58.694 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:58.756 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:58 smithi138 ceph-mon[185886]: pgmap v1308: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 13426/57660 objects degraded (23.285%); 12252/57660 objects misplaced (21.249%); 8.5 MiB/s, 135 objects/s recovering 2023-10-01T09:38:58.766 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:58.941 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:38:58.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:58.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:38:59.048 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:59.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:58 smithi190 ceph-mon[160435]: pgmap v1308: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 13426/57660 objects degraded (23.285%); 12252/57660 objects misplaced (21.249%); 8.5 MiB/s, 135 objects/s recovering 2023-10-01T09:38:59.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:58 smithi138 ceph-mon[192513]: pgmap v1308: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 13426/57660 objects degraded (23.285%); 12252/57660 objects misplaced (21.249%); 8.5 MiB/s, 135 objects/s recovering 2023-10-01T09:38:59.078 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:59.157 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:59.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:59.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:59.266 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:38:59.301 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:59.358 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:38:59.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:59.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:38:59.389 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:38:59.466 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:38:59.499 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:59.551 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:38:59.556 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:38:59.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:59.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:38:59.602 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:38:59.648 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:38:59.663 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:38:59.691 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:38:59.701 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:38:59.748 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:38:59.767 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:38:59.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:38:59.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:38:59.785 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:38:59.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:38:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/18410904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:59.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:38:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/18410904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:38:59.842 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:38:59.872 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:38:59.881 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:38:59.910 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:38:59.910 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:38:59.910 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:38:59.910 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:38:59.910 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:38:59.910 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:38:59.910 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:38:59.910 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:38:59.965 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:38:59.965 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:38:59.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:38:59.965 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:38:59.965 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.966 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:159815 8.b 8:ddfb8718:::benchmark_data_smithi138_431617_object58404:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-10-01T09:38:59.966 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:27.367095+0000", 2023-10-01T09:38:59.966 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 392.58249608699998, 2023-10-01T09:38:59.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2605747329999999, 2023-10-01T09:38:59.966 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:59.966 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:38:59.966 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:38:59.966 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:38:59.966 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:38:59.966 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 159815 2023-10-01T09:38:59.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.967 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:59.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:59.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.367095+0000", 2023-10-01T09:38:59.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:59.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:59.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.367095+0000", 2023-10-01T09:38:59.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:59.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:59.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.367094+0000", 2023-10-01T09:38:59.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:38:59.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:59.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.367099+0000", 2023-10-01T09:38:59.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0889999999999999e-06 2023-10-01T09:38:59.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:59.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.367108+0000", 2023-10-01T09:38:59.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7190000000000007e-06 2023-10-01T09:38:59.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:59.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.367115+0000", 2023-10-01T09:38:59.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8769999999999993e-06 2023-10-01T09:38:59.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:59.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.367369+0000", 2023-10-01T09:38:59.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00025321499999999999 2023-10-01T09:38:59.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:38:59.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.367379+0000", 2023-10-01T09:38:59.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0869e-05 2023-10-01T09:38:59.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:59.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.439489+0000", 2023-10-01T09:38:59.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.072109638000000004 2023-10-01T09:38:59.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:38:59.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.439502+0000", 2023-10-01T09:38:59.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2627e-05 2023-10-01T09:38:59.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:59.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.615480+0000", 2023-10-01T09:38:59.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.175978143 2023-10-01T09:38:59.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:38:59.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.615616+0000", 2023-10-01T09:38:59.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013615599999999999 2023-10-01T09:38:59.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,4", 2023-10-01T09:38:59.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.615759+0000", 2023-10-01T09:38:59.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014247899999999999 2023-10-01T09:38:59.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:38:59.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.627393+0000", 2023-10-01T09:38:59.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011634396 2023-10-01T09:38:59.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:59.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.627439+0000", 2023-10-01T09:38:59.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6372000000000003e-05 2023-10-01T09:38:59.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:59.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.627482+0000", 2023-10-01T09:38:59.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2376000000000002e-05 2023-10-01T09:38:59.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:59.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.627516+0000", 2023-10-01T09:38:59.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4400000000000003e-05 2023-10-01T09:38:59.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:38:59.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.627564+0000", 2023-10-01T09:38:59.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8352000000000001e-05 2023-10-01T09:38:59.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.975 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:38:59.975 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.627669+0000", 2023-10-01T09:38:59.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105033 2023-10-01T09:38:59.975 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:59.975 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:59.975 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:59.975 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.975 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:159816 8.b 8:d3c01fa3:::benchmark_data_smithi138_431617_object58405:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-10-01T09:38:59.975 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:27.368744+0000", 2023-10-01T09:38:59.976 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 392.580846764, 2023-10-01T09:38:59.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2666213499999999, 2023-10-01T09:38:59.976 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:59.976 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:38:59.976 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:38:59.976 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:38:59.976 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:38:59.976 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 159816 2023-10-01T09:38:59.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.976 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:59.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:59.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.368744+0000", 2023-10-01T09:38:59.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:59.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:59.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.368744+0000", 2023-10-01T09:38:59.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:59.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:59.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.368743+0000", 2023-10-01T09:38:59.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:38:59.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:59.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.368749+0000", 2023-10-01T09:38:59.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8499999999999999e-06 2023-10-01T09:38:59.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:59.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.368760+0000", 2023-10-01T09:38:59.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1161999999999999e-05 2023-10-01T09:38:59.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:59.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.368768+0000", 2023-10-01T09:38:59.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8189999999999992e-06 2023-10-01T09:38:59.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:59.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.368817+0000", 2023-10-01T09:38:59.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8980999999999997e-05 2023-10-01T09:38:59.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:38:59.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.368828+0000", 2023-10-01T09:38:59.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1823e-05 2023-10-01T09:38:59.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:59.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.440262+0000", 2023-10-01T09:38:59.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.071433808000000001 2023-10-01T09:38:59.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:38:59.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.440271+0000", 2023-10-01T09:38:59.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6389999999999998e-06 2023-10-01T09:38:59.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:59.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.619519+0000", 2023-10-01T09:38:59.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1792484000000001 2023-10-01T09:38:59.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:38:59.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.619664+0000", 2023-10-01T09:38:59.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014433600000000001 2023-10-01T09:38:59.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.982 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,4", 2023-10-01T09:38:59.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.619780+0000", 2023-10-01T09:38:59.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011660899999999999 2023-10-01T09:38:59.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.982 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:38:59.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.627447+0000", 2023-10-01T09:38:59.983 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0076668700000000001 2023-10-01T09:38:59.983 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.983 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.983 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:59.983 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.634737+0000", 2023-10-01T09:38:59.983 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0072898429999999998 2023-10-01T09:38:59.983 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.983 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.983 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:59.983 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.634786+0000", 2023-10-01T09:38:59.983 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9472999999999999e-05 2023-10-01T09:38:59.984 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:59.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.635240+0000", 2023-10-01T09:38:59.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00045322100000000002 2023-10-01T09:38:59.984 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:38:59.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.635279+0000", 2023-10-01T09:38:59.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9067999999999999e-05 2023-10-01T09:38:59.984 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:38:59.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.635365+0000", 2023-10-01T09:38:59.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6737999999999995e-05 2023-10-01T09:38:59.985 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:59.985 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:59.985 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:59.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.985 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:159832 8.b 8:de465a56:::benchmark_data_smithi138_431617_object58421:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-10-01T09:38:59.985 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:27.392148+0000", 2023-10-01T09:38:59.985 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 392.55744273699997, 2023-10-01T09:38:59.986 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2472873710000001, 2023-10-01T09:38:59.986 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:59.986 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:38:59.986 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:38:59.986 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:38:59.986 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:38:59.986 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 159832 2023-10-01T09:38:59.986 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.986 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:59.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.986 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:59.986 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.392148+0000", 2023-10-01T09:38:59.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:59.987 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.987 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:59.987 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.392148+0000", 2023-10-01T09:38:59.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:59.987 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.987 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:59.987 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.392147+0000", 2023-10-01T09:38:59.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:38:59.987 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:59.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.392153+0000", 2023-10-01T09:38:59.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7689999999999999e-06 2023-10-01T09:38:59.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:59.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.392170+0000", 2023-10-01T09:38:59.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6311999999999999e-05 2023-10-01T09:38:59.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:59.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.392176+0000", 2023-10-01T09:38:59.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4579999999999998e-06 2023-10-01T09:38:59.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:59.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.392501+0000", 2023-10-01T09:38:59.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00032431799999999998 2023-10-01T09:38:59.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:38:59.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.392510+0000", 2023-10-01T09:38:59.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.115e-06 2023-10-01T09:38:59.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:59.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.440336+0000", 2023-10-01T09:38:59.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.047826494999999997 2023-10-01T09:38:59.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:38:59.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.440343+0000", 2023-10-01T09:38:59.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3139999999999998e-06 2023-10-01T09:38:59.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.991 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:59.991 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.623460+0000", 2023-10-01T09:38:59.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.183116241 2023-10-01T09:38:59.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.991 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:38:59.991 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.623602+0000", 2023-10-01T09:38:59.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014209300000000001 2023-10-01T09:38:59.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.991 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,4", 2023-10-01T09:38:59.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.623741+0000", 2023-10-01T09:38:59.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013870000000000001 2023-10-01T09:38:59.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:59.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.634767+0000", 2023-10-01T09:38:59.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011026963000000001 2023-10-01T09:38:59.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:59.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.635390+0000", 2023-10-01T09:38:59.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00062297000000000003 2023-10-01T09:38:59.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:38:59.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.638500+0000", 2023-10-01T09:38:59.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0031096140000000001 2023-10-01T09:38:59.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:38:59.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.639294+0000", 2023-10-01T09:38:59.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00079407999999999996 2023-10-01T09:38:59.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:38:59.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.639338+0000", 2023-10-01T09:38:59.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3498999999999998e-05 2023-10-01T09:38:59.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:38:59.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.639435+0000", 2023-10-01T09:38:59.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7796999999999999e-05 2023-10-01T09:38:59.994 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:59.994 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:38:59.995 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:38:59.995 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.995 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:159842 8.b 8:d61bcfa5:::benchmark_data_smithi138_431617_object58431:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-10-01T09:38:59.995 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:27.416912+0000", 2023-10-01T09:38:59.995 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 392.53267862799999, 2023-10-01T09:38:59.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.222719713, 2023-10-01T09:38:59.995 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:38:59.995 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:38:59.995 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:38:59.995 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:38:59.996 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:38:59.996 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 159842 2023-10-01T09:38:59.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.996 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:38:59.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:38:59.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.416912+0000", 2023-10-01T09:38:59.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:59.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:38:59.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.416912+0000", 2023-10-01T09:38:59.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:38:59.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:38:59.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.416911+0000", 2023-10-01T09:38:59.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:38:59.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:38:59.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.416917+0000", 2023-10-01T09:38:59.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3049999999999998e-06 2023-10-01T09:38:59.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:38:59.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.416930+0000", 2023-10-01T09:38:59.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2346e-05 2023-10-01T09:38:59.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:38:59.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.416936+0000", 2023-10-01T09:38:59.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6200000000000001e-06 2023-10-01T09:38:59.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.999 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:38:59.999 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.416984+0000", 2023-10-01T09:38:59.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8210000000000001e-05 2023-10-01T09:38:59.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:38:59.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:38:59.999 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:38:59.999 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.416994+0000", 2023-10-01T09:38:59.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2029999999999998e-06 2023-10-01T09:39:00.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.000 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.000 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.440361+0000", 2023-10-01T09:39:00.000 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.023366947999999998 2023-10-01T09:39:00.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.000 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.000 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:27.440368+0000", 2023-10-01T09:39:00.000 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0199999999999997e-06 2023-10-01T09:39:00.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.631360+0000", 2023-10-01T09:39:00.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.190992611 2023-10-01T09:39:00.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.631467+0000", 2023-10-01T09:39:00.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106407 2023-10-01T09:39:00.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3,4", 2023-10-01T09:39:00.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.631550+0000", 2023-10-01T09:39:00.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3557e-05 2023-10-01T09:39:00.002 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.636043+0000", 2023-10-01T09:39:00.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044925470000000004 2023-10-01T09:39:00.002 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.003 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.638490+0000", 2023-10-01T09:39:00.003 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024473989999999998 2023-10-01T09:39:00.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.638601+0000", 2023-10-01T09:39:00.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011127400000000001 2023-10-01T09:39:00.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.639483+0000", 2023-10-01T09:39:00.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00088130599999999999 2023-10-01T09:39:00.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.639530+0000", 2023-10-01T09:39:00.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7536e-05 2023-10-01T09:39:00.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:28.639632+0000", 2023-10-01T09:39:00.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000101597 2023-10-01T09:39:00.006 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.006 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.006 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.007 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169545 8.a 8:520deb63:::benchmark_data_smithi138_431617_object68134:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-01T09:39:00.007 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.034378+0000", 2023-10-01T09:39:00.007 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 374.91521279099999, 2023-10-01T09:39:00.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3076351800000001, 2023-10-01T09:39:00.007 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.007 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:39:00.007 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.007 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:39:00.007 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:39:00.008 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169545 2023-10-01T09:39:00.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.008 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.034378+0000", 2023-10-01T09:39:00.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.034378+0000", 2023-10-01T09:39:00.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.034378+0000", 2023-10-01T09:39:00.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:39:00.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.034380+0000", 2023-10-01T09:39:00.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3099999999999999e-06 2023-10-01T09:39:00.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.034386+0000", 2023-10-01T09:39:00.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7599999999999999e-06 2023-10-01T09:39:00.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.034390+0000", 2023-10-01T09:39:00.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6240000000000001e-06 2023-10-01T09:39:00.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.034441+0000", 2023-10-01T09:39:00.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0778000000000001e-05 2023-10-01T09:39:00.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.282643+0000", 2023-10-01T09:39:00.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.248202327 2023-10-01T09:39:00.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:39:00.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.282752+0000", 2023-10-01T09:39:00.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108059 2023-10-01T09:39:00.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.012 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.295840+0000", 2023-10-01T09:39:00.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013088246 2023-10-01T09:39:00.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.012 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.295852+0000", 2023-10-01T09:39:00.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1948e-05 2023-10-01T09:39:00.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.013 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.329638+0000", 2023-10-01T09:39:00.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0337860720000001 2023-10-01T09:39:00.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.013 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for missing object", 2023-10-01T09:39:00.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.329751+0000", 2023-10-01T09:39:00.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000112711 2023-10-01T09:39:00.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.341964+0000", 2023-10-01T09:39:00.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012213089999999999 2023-10-01T09:39:00.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.342013+0000", 2023-10-01T09:39:00.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9583000000000003e-05 2023-10-01T09:39:00.014 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.014 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.015 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.015 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169547 8.3 8:c393b927:::benchmark_data_smithi138_431617_object68136:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-10-01T09:39:00.015 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.299562+0000", 2023-10-01T09:39:00.015 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 374.650028792, 2023-10-01T09:39:00.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0346791870000001, 2023-10-01T09:39:00.015 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.015 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:00.015 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.015 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:39:00.016 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:39:00.016 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169547 2023-10-01T09:39:00.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.016 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299562+0000", 2023-10-01T09:39:00.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299562+0000", 2023-10-01T09:39:00.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299561+0000", 2023-10-01T09:39:00.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:39:00.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.017 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.017 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299566+0000", 2023-10-01T09:39:00.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4480000000000002e-06 2023-10-01T09:39:00.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.018 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299577+0000", 2023-10-01T09:39:00.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1161999999999999e-05 2023-10-01T09:39:00.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.018 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299585+0000", 2023-10-01T09:39:00.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6090000000000003e-06 2023-10-01T09:39:00.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299631+0000", 2023-10-01T09:39:00.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6668000000000001e-05 2023-10-01T09:39:00.019 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299638+0000", 2023-10-01T09:39:00.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9890000000000003e-06 2023-10-01T09:39:00.019 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.317710+0000", 2023-10-01T09:39:00.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0180717269999999 2023-10-01T09:39:00.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.317841+0000", 2023-10-01T09:39:00.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000130653 2023-10-01T09:39:00.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 6", 2023-10-01T09:39:00.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.317966+0000", 2023-10-01T09:39:00.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012496100000000001 2023-10-01T09:39:00.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.330425+0000", 2023-10-01T09:39:00.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012458683999999999 2023-10-01T09:39:00.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.334079+0000", 2023-10-01T09:39:00.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036542430000000002 2023-10-01T09:39:00.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.334129+0000", 2023-10-01T09:39:00.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0127999999999998e-05 2023-10-01T09:39:00.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.022 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.334241+0000", 2023-10-01T09:39:00.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011231299999999999 2023-10-01T09:39:00.022 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.022 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.022 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.023 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169549 8.a 8:5eaf9f3f:::benchmark_data_smithi138_431617_object68138:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-10-01T09:39:00.023 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.299597+0000", 2023-10-01T09:39:00.023 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 374.64999339399998, 2023-10-01T09:39:00.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.042072745, 2023-10-01T09:39:00.023 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.023 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:00.023 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.023 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:39:00.023 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:39:00.023 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169549 2023-10-01T09:39:00.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.024 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299597+0000", 2023-10-01T09:39:00.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.024 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.024 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299597+0000", 2023-10-01T09:39:00.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.025 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.025 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299597+0000", 2023-10-01T09:39:00.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:39:00.025 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.025 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.025 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.025 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299600+0000", 2023-10-01T09:39:00.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1099999999999999e-06 2023-10-01T09:39:00.025 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.025 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.025 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299608+0000", 2023-10-01T09:39:00.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6960000000000005e-06 2023-10-01T09:39:00.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299613+0000", 2023-10-01T09:39:00.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6130000000000003e-06 2023-10-01T09:39:00.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299676+0000", 2023-10-01T09:39:00.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2646999999999998e-05 2023-10-01T09:39:00.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299685+0000", 2023-10-01T09:39:00.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5320000000000006e-06 2023-10-01T09:39:00.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.329932+0000", 2023-10-01T09:39:00.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0302476169999999 2023-10-01T09:39:00.028 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.028 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.028 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.028 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.330008+0000", 2023-10-01T09:39:00.028 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6239e-05 2023-10-01T09:39:00.028 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.028 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.028 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2", 2023-10-01T09:39:00.028 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.330108+0000", 2023-10-01T09:39:00.028 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9154000000000006e-05 2023-10-01T09:39:00.028 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.028 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.029 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.029 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.333692+0000", 2023-10-01T09:39:00.029 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035844940000000001 2023-10-01T09:39:00.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.029 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.029 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.341552+0000", 2023-10-01T09:39:00.029 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0078601750000000005 2023-10-01T09:39:00.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.029 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.029 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.341591+0000", 2023-10-01T09:39:00.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8510999999999998e-05 2023-10-01T09:39:00.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.341670+0000", 2023-10-01T09:39:00.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9436000000000003e-05 2023-10-01T09:39:00.030 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.030 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.030 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.031 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169552 8.3 8:c77972c9:::benchmark_data_smithi138_431617_object68141:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-10-01T09:39:00.031 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.299625+0000", 2023-10-01T09:39:00.031 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 374.64996615199999, 2023-10-01T09:39:00.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0347296130000001, 2023-10-01T09:39:00.031 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.031 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:00.031 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.031 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:39:00.031 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:39:00.031 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169552 2023-10-01T09:39:00.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.032 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.032 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.032 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299625+0000", 2023-10-01T09:39:00.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.032 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.032 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.032 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299625+0000", 2023-10-01T09:39:00.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.032 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.033 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299624+0000", 2023-10-01T09:39:00.033 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:39:00.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.033 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299627+0000", 2023-10-01T09:39:00.033 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9500000000000001e-06 2023-10-01T09:39:00.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.034 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299634+0000", 2023-10-01T09:39:00.034 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6089999999999997e-06 2023-10-01T09:39:00.034 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.034 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.034 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.034 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299639+0000", 2023-10-01T09:39:00.034 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4859999999999997e-06 2023-10-01T09:39:00.034 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.034 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.034 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.034 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299776+0000", 2023-10-01T09:39:00.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000136859 2023-10-01T09:39:00.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.035 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299784+0000", 2023-10-01T09:39:00.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6920000000000002e-06 2023-10-01T09:39:00.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.035 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.322116+0000", 2023-10-01T09:39:00.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0223319740000001 2023-10-01T09:39:00.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.036 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.322260+0000", 2023-10-01T09:39:00.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014420700000000001 2023-10-01T09:39:00.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.036 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 6", 2023-10-01T09:39:00.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.322383+0000", 2023-10-01T09:39:00.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012283500000000001 2023-10-01T09:39:00.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.037 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.330454+0000", 2023-10-01T09:39:00.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0080711829999999991 2023-10-01T09:39:00.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.037 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.334262+0000", 2023-10-01T09:39:00.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0038080969999999999 2023-10-01T09:39:00.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.038 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.038 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.334295+0000", 2023-10-01T09:39:00.038 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3164999999999998e-05 2023-10-01T09:39:00.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.038 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.038 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.038 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.334354+0000", 2023-10-01T09:39:00.038 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9126999999999998e-05 2023-10-01T09:39:00.038 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.038 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.038 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.039 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.039 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169555 8.3 8:cc867d22:::benchmark_data_smithi138_431617_object68144:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-10-01T09:39:00.039 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.299725+0000", 2023-10-01T09:39:00.039 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 374.64986612899997, 2023-10-01T09:39:00.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.148607846, 2023-10-01T09:39:00.039 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.039 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:00.039 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.039 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:39:00.039 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:39:00.039 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169555 2023-10-01T09:39:00.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.040 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.040 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.040 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299725+0000", 2023-10-01T09:39:00.040 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.040 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.040 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299725+0000", 2023-10-01T09:39:00.040 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.041 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.041 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.041 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299724+0000", 2023-10-01T09:39:00.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:39:00.041 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.041 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.041 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.041 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299729+0000", 2023-10-01T09:39:00.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.685e-06 2023-10-01T09:39:00.041 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.041 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.041 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.042 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299737+0000", 2023-10-01T09:39:00.042 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6400000000000003e-06 2023-10-01T09:39:00.042 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.042 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.042 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.042 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299743+0000", 2023-10-01T09:39:00.042 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8969999999999996e-06 2023-10-01T09:39:00.042 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.042 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.042 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.042 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299815+0000", 2023-10-01T09:39:00.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1476000000000005e-05 2023-10-01T09:39:00.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.043 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.043 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.043 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299820+0000", 2023-10-01T09:39:00.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3020000000000002e-06 2023-10-01T09:39:00.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.043 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.043 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.043 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.326311+0000", 2023-10-01T09:39:00.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.02649078 2023-10-01T09:39:00.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.044 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.326426+0000", 2023-10-01T09:39:00.044 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011513 2023-10-01T09:39:00.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 6", 2023-10-01T09:39:00.044 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.326550+0000", 2023-10-01T09:39:00.044 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012398899999999999 2023-10-01T09:39:00.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.045 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.334358+0000", 2023-10-01T09:39:00.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0078087019999999998 2023-10-01T09:39:00.045 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.045 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.045 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.448191+0000", 2023-10-01T09:39:00.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.113832608 2023-10-01T09:39:00.045 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.045 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.045 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.448240+0000", 2023-10-01T09:39:00.046 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9373999999999999e-05 2023-10-01T09:39:00.046 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.046 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.046 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.046 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.448333+0000", 2023-10-01T09:39:00.046 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2048999999999999e-05 2023-10-01T09:39:00.046 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.046 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.046 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.046 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.046 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.047 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:169556 8.3 8:ca376d3f:::benchmark_data_smithi138_431617_object68145:head [delete] snapc 0=[] ondisk+write+known_if_redirected e369)", 2023-10-01T09:39:00.047 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:32:45.299781+0000", 2023-10-01T09:39:00.047 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 374.64980972699999, 2023-10-01T09:39:00.047 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1487514860000001, 2023-10-01T09:39:00.047 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.047 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:00.047 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.047 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:39:00.047 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:39:00.047 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 169556 2023-10-01T09:39:00.047 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.048 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.048 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.048 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.048 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299781+0000", 2023-10-01T09:39:00.048 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.048 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.048 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.048 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.048 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299781+0000", 2023-10-01T09:39:00.048 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.048 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.048 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299781+0000", 2023-10-01T09:39:00.049 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:39:00.049 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.049 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299784+0000", 2023-10-01T09:39:00.049 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9799999999999998e-06 2023-10-01T09:39:00.049 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.049 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299789+0000", 2023-10-01T09:39:00.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7740000000000001e-06 2023-10-01T09:39:00.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299794+0000", 2023-10-01T09:39:00.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5600000000000004e-06 2023-10-01T09:39:00.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299853+0000", 2023-10-01T09:39:00.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8597999999999997e-05 2023-10-01T09:39:00.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.051 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.051 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:45.299859+0000", 2023-10-01T09:39:00.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4579999999999998e-06 2023-10-01T09:39:00.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.051 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.051 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.330554+0000", 2023-10-01T09:39:00.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.030695025 2023-10-01T09:39:00.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.052 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.330661+0000", 2023-10-01T09:39:00.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010734 2023-10-01T09:39:00.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.052 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 6", 2023-10-01T09:39:00.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.330759+0000", 2023-10-01T09:39:00.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.789e-05 2023-10-01T09:39:00.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.053 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.334363+0000", 2023-10-01T09:39:00.053 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036032740000000001 2023-10-01T09:39:00.053 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.053 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.448417+0000", 2023-10-01T09:39:00.053 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.114054928 2023-10-01T09:39:00.053 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.054 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.448462+0000", 2023-10-01T09:39:00.054 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4620000000000003e-05 2023-10-01T09:39:00.054 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.054 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.054 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.054 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:32:46.448533+0000", 2023-10-01T09:39:00.054 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0510000000000001e-05 2023-10-01T09:39:00.054 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.054 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.054 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.054 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.055 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.055 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92726 9.2 9:4513d176:::benchmark_data_smithi138_497829_object92725:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-10-01T09:39:00.055 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.528465+0000", 2023-10-01T09:39:00.055 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 206.42112554299999, 2023-10-01T09:39:00.055 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1036305710000001, 2023-10-01T09:39:00.055 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.055 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:39:00.055 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.055 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:00.055 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:00.055 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92726 2023-10-01T09:39:00.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.056 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.528465+0000", 2023-10-01T09:39:00.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.528465+0000", 2023-10-01T09:39:00.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.528464+0000", 2023-10-01T09:39:00.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:39:00.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.528491+0000", 2023-10-01T09:39:00.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7314999999999999e-05 2023-10-01T09:39:00.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.528501+0000", 2023-10-01T09:39:00.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0254999999999999e-05 2023-10-01T09:39:00.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.528510+0000", 2023-10-01T09:39:00.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2849999999999995e-06 2023-10-01T09:39:00.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.533736+0000", 2023-10-01T09:39:00.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0052263479999999996 2023-10-01T09:39:00.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.533933+0000", 2023-10-01T09:39:00.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000197474 2023-10-01T09:39:00.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:39:00.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.534132+0000", 2023-10-01T09:39:00.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019866999999999999 2023-10-01T09:39:00.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.548469+0000", 2023-10-01T09:39:00.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014337216 2023-10-01T09:39:00.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.549134+0000", 2023-10-01T09:39:00.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00066513599999999998 2023-10-01T09:39:00.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.556602+0000", 2023-10-01T09:39:00.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0074679619999999999 2023-10-01T09:39:00.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.556622+0000", 2023-10-01T09:39:00.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9607e-05 2023-10-01T09:39:00.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.062 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.062 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.062 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.631986+0000", 2023-10-01T09:39:00.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.075363702 2023-10-01T09:39:00.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.062 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.062 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.062 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.632096+0000", 2023-10-01T09:39:00.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110096 2023-10-01T09:39:00.062 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.063 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.063 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.063 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.063 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92775 9.0 9:0798a061:::benchmark_data_smithi138_497829_object92774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:39:00.063 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.607380+0000", 2023-10-01T09:39:00.063 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 206.342210679, 2023-10-01T09:39:00.063 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0222879359999999, 2023-10-01T09:39:00.063 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.063 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:00.063 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.063 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:00.064 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:00.064 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92775 2023-10-01T09:39:00.064 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.064 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607380+0000", 2023-10-01T09:39:00.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.064 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607380+0000", 2023-10-01T09:39:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607378+0000", 2023-10-01T09:39:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:39:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607417+0000", 2023-10-01T09:39:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8266000000000003e-05 2023-10-01T09:39:00.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607432+0000", 2023-10-01T09:39:00.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4946000000000001e-05 2023-10-01T09:39:00.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607439+0000", 2023-10-01T09:39:00.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3719999999999999e-06 2023-10-01T09:39:00.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607543+0000", 2023-10-01T09:39:00.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010367299999999999 2023-10-01T09:39:00.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607553+0000", 2023-10-01T09:39:00.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0441999999999999e-05 2023-10-01T09:39:00.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.621571+0000", 2023-10-01T09:39:00.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0140181539999999 2023-10-01T09:39:00.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.621738+0000", 2023-10-01T09:39:00.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016643699999999999 2023-10-01T09:39:00.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:39:00.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.621891+0000", 2023-10-01T09:39:00.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015302299999999999 2023-10-01T09:39:00.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.069 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.628749+0000", 2023-10-01T09:39:00.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0068583519999999999 2023-10-01T09:39:00.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.069 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.069 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.628828+0000", 2023-10-01T09:39:00.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8592999999999998e-05 2023-10-01T09:39:00.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.628884+0000", 2023-10-01T09:39:00.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5968999999999999e-05 2023-10-01T09:39:00.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.628943+0000", 2023-10-01T09:39:00.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9694000000000001e-05 2023-10-01T09:39:00.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.071 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.071 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.629668+0000", 2023-10-01T09:39:00.071 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00072471999999999999 2023-10-01T09:39:00.071 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.071 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.071 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.071 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.071 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.071 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92776 9.2 9:41354763:::benchmark_data_smithi138_497829_object92775:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:39:00.071 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.607457+0000", 2023-10-01T09:39:00.071 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 206.34213393300001, 2023-10-01T09:39:00.072 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0409147700000001, 2023-10-01T09:39:00.072 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.072 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:00.072 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.072 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:00.072 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:00.072 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92776 2023-10-01T09:39:00.072 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.072 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.072 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607457+0000", 2023-10-01T09:39:00.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607457+0000", 2023-10-01T09:39:00.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607456+0000", 2023-10-01T09:39:00.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:39:00.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607476+0000", 2023-10-01T09:39:00.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9949000000000002e-05 2023-10-01T09:39:00.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607485+0000", 2023-10-01T09:39:00.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0680000000000003e-06 2023-10-01T09:39:00.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607490+0000", 2023-10-01T09:39:00.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2220000000000001e-06 2023-10-01T09:39:00.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607550+0000", 2023-10-01T09:39:00.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9349999999999999e-05 2023-10-01T09:39:00.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607561+0000", 2023-10-01T09:39:00.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0869e-05 2023-10-01T09:39:00.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.076 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.632780+0000", 2023-10-01T09:39:00.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.025219221 2023-10-01T09:39:00.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.076 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.632953+0000", 2023-10-01T09:39:00.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000173294 2023-10-01T09:39:00.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.077 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:39:00.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.633148+0000", 2023-10-01T09:39:00.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019489400000000001 2023-10-01T09:39:00.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.077 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.647265+0000", 2023-10-01T09:39:00.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014117054 2023-10-01T09:39:00.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.647300+0000", 2023-10-01T09:39:00.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5153e-05 2023-10-01T09:39:00.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.647360+0000", 2023-10-01T09:39:00.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9716999999999998e-05 2023-10-01T09:39:00.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.647417+0000", 2023-10-01T09:39:00.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7259999999999997e-05 2023-10-01T09:39:00.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.648372+0000", 2023-10-01T09:39:00.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000954359 2023-10-01T09:39:00.080 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.080 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.080 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.080 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92779 9.0 9:0c623f1e:::benchmark_data_smithi138_497829_object92778:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:39:00.080 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.608374+0000", 2023-10-01T09:39:00.080 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 206.34121659799999, 2023-10-01T09:39:00.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0311556449999999, 2023-10-01T09:39:00.080 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.080 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:00.081 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.081 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:00.081 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:00.081 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92779 2023-10-01T09:39:00.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.081 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608374+0000", 2023-10-01T09:39:00.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608374+0000", 2023-10-01T09:39:00.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608374+0000", 2023-10-01T09:39:00.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:39:00.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.083 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.083 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608394+0000", 2023-10-01T09:39:00.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0027e-05 2023-10-01T09:39:00.083 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.083 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.083 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608402+0000", 2023-10-01T09:39:00.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8519999999999993e-06 2023-10-01T09:39:00.083 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.084 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.084 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608410+0000", 2023-10-01T09:39:00.084 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6420000000000007e-06 2023-10-01T09:39:00.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.084 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.084 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.084 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608465+0000", 2023-10-01T09:39:00.084 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4969999999999997e-05 2023-10-01T09:39:00.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.084 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.084 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608475+0000", 2023-10-01T09:39:00.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0304e-05 2023-10-01T09:39:00.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.085 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.625720+0000", 2023-10-01T09:39:00.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0172442719999999 2023-10-01T09:39:00.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.085 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.625884+0000", 2023-10-01T09:39:00.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016460699999999999 2023-10-01T09:39:00.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:39:00.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.626061+0000", 2023-10-01T09:39:00.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000176381 2023-10-01T09:39:00.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.634818+0000", 2023-10-01T09:39:00.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0087576919999999992 2023-10-01T09:39:00.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.634863+0000", 2023-10-01T09:39:00.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5006999999999998e-05 2023-10-01T09:39:00.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.638629+0000", 2023-10-01T09:39:00.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037660789999999999 2023-10-01T09:39:00.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.638690+0000", 2023-10-01T09:39:00.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0847999999999997e-05 2023-10-01T09:39:00.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.639530+0000", 2023-10-01T09:39:00.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00083966399999999995 2023-10-01T09:39:00.088 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.088 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.088 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.089 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92794 9.0 9:01172f72:::benchmark_data_smithi138_497829_object92793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:39:00.089 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.623408+0000", 2023-10-01T09:39:00.089 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 206.32618305099999, 2023-10-01T09:39:00.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.019196808, 2023-10-01T09:39:00.089 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.089 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:00.089 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.089 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:00.089 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:00.089 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92794 2023-10-01T09:39:00.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.090 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623408+0000", 2023-10-01T09:39:00.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.090 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623408+0000", 2023-10-01T09:39:00.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.090 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.091 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.091 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.091 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623406+0000", 2023-10-01T09:39:00.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:39:00.091 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.091 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.091 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.091 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623432+0000", 2023-10-01T09:39:00.091 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5511999999999999e-05 2023-10-01T09:39:00.091 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.091 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.091 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.092 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623446+0000", 2023-10-01T09:39:00.092 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4956e-05 2023-10-01T09:39:00.092 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.092 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.092 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.092 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623456+0000", 2023-10-01T09:39:00.092 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1910000000000006e-06 2023-10-01T09:39:00.092 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.092 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.092 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.093 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623523+0000", 2023-10-01T09:39:00.093 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7577999999999996e-05 2023-10-01T09:39:00.093 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.093 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.093 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.093 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623540+0000", 2023-10-01T09:39:00.093 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6483000000000001e-05 2023-10-01T09:39:00.093 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.093 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.093 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.093 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.629728+0000", 2023-10-01T09:39:00.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.006188071 2023-10-01T09:39:00.094 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.094 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.094 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.630863+0000", 2023-10-01T09:39:00.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001135287 2023-10-01T09:39:00.094 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.094 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.094 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:39:00.094 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.631095+0000", 2023-10-01T09:39:00.094 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00023191100000000001 2023-10-01T09:39:00.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.639546+0000", 2023-10-01T09:39:00.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0084506949999999994 2023-10-01T09:39:00.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.095 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.095 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.639589+0000", 2023-10-01T09:39:00.095 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3439000000000002e-05 2023-10-01T09:39:00.095 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.095 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.641642+0000", 2023-10-01T09:39:00.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020529680000000001 2023-10-01T09:39:00.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.641693+0000", 2023-10-01T09:39:00.096 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1221000000000003e-05 2023-10-01T09:39:00.096 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.096 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.096 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.096 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.642605+0000", 2023-10-01T09:39:00.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00091121699999999997 2023-10-01T09:39:00.097 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.097 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.097 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.097 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.097 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.097 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92802 9.0 9:053736bf:::benchmark_data_smithi138_497829_object92801:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:39:00.097 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.630809+0000", 2023-10-01T09:39:00.097 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 206.318781511, 2023-10-01T09:39:00.097 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.01467168, 2023-10-01T09:39:00.097 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.098 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:00.098 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.098 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:00.098 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:00.098 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92802 2023-10-01T09:39:00.098 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.098 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.098 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.098 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.098 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630809+0000", 2023-10-01T09:39:00.098 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.099 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.099 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.099 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630809+0000", 2023-10-01T09:39:00.099 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.099 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.099 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.099 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630808+0000", 2023-10-01T09:39:00.099 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:39:00.099 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.099 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.100 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.100 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630830+0000", 2023-10-01T09:39:00.100 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.137e-05 2023-10-01T09:39:00.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.100 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.100 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.100 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630841+0000", 2023-10-01T09:39:00.100 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1817999999999999e-05 2023-10-01T09:39:00.100 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.100 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.100 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.101 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630849+0000", 2023-10-01T09:39:00.101 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6650000000000003e-06 2023-10-01T09:39:00.101 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.101 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.101 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.101 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630903+0000", 2023-10-01T09:39:00.101 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4423999999999999e-05 2023-10-01T09:39:00.101 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.101 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.101 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.101 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630912+0000", 2023-10-01T09:39:00.102 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5750000000000001e-06 2023-10-01T09:39:00.102 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.102 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.102 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.102 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.634989+0000", 2023-10-01T09:39:00.102 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0040773009999999 2023-10-01T09:39:00.102 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.102 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.102 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.635152+0000", 2023-10-01T09:39:00.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000162886 2023-10-01T09:39:00.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:39:00.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.635337+0000", 2023-10-01T09:39:00.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018489099999999999 2023-10-01T09:39:00.103 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.103 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.103 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.103 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.640106+0000", 2023-10-01T09:39:00.103 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047690930000000003 2023-10-01T09:39:00.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.642662+0000", 2023-10-01T09:39:00.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025556910000000001 2023-10-01T09:39:00.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.104 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.104 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.104 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.644600+0000", 2023-10-01T09:39:00.104 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001938005 2023-10-01T09:39:00.104 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.644661+0000", 2023-10-01T09:39:00.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1155000000000007e-05 2023-10-01T09:39:00.105 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.105 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.105 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.105 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.645481+0000", 2023-10-01T09:39:00.105 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00081991500000000005 2023-10-01T09:39:00.105 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.105 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.105 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.106 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.106 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.106 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92810 9.2 9:41e5de28:::benchmark_data_smithi138_497829_object92809:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:39:00.106 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.641159+0000", 2023-10-01T09:39:00.106 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 206.30843138500001, 2023-10-01T09:39:00.106 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0105064420000001, 2023-10-01T09:39:00.106 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.106 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:00.106 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.106 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:00.107 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:00.107 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92810 2023-10-01T09:39:00.107 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.107 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.107 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641159+0000", 2023-10-01T09:39:00.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.107 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.107 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.107 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641159+0000", 2023-10-01T09:39:00.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.108 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.108 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.108 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641158+0000", 2023-10-01T09:39:00.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:39:00.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.108 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.108 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.108 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641178+0000", 2023-10-01T09:39:00.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9389e-05 2023-10-01T09:39:00.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641186+0000", 2023-10-01T09:39:00.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9270000000000005e-06 2023-10-01T09:39:00.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641190+0000", 2023-10-01T09:39:00.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7940000000000002e-06 2023-10-01T09:39:00.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.110 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.110 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.110 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641257+0000", 2023-10-01T09:39:00.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6339000000000003e-05 2023-10-01T09:39:00.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.110 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.110 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.110 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641270+0000", 2023-10-01T09:39:00.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2748999999999999e-05 2023-10-01T09:39:00.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.636539+0000", 2023-10-01T09:39:00.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99526912099999998 2023-10-01T09:39:00.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.636704+0000", 2023-10-01T09:39:00.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016581500000000001 2023-10-01T09:39:00.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:39:00.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.636861+0000", 2023-10-01T09:39:00.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015662599999999999 2023-10-01T09:39:00.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.647350+0000", 2023-10-01T09:39:00.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010488509 2023-10-01T09:39:00.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.648383+0000", 2023-10-01T09:39:00.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001033462 2023-10-01T09:39:00.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.650664+0000", 2023-10-01T09:39:00.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022806929999999999 2023-10-01T09:39:00.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.650721+0000", 2023-10-01T09:39:00.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7343000000000003e-05 2023-10-01T09:39:00.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.651666+0000", 2023-10-01T09:39:00.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00094476399999999996 2023-10-01T09:39:00.114 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.114 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.114 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.114 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92814 9.2 9:4a803fc0:::benchmark_data_smithi138_497829_object92813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:39:00.114 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.644861+0000", 2023-10-01T09:39:00.115 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 206.304730025, 2023-10-01T09:39:00.115 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.022649412, 2023-10-01T09:39:00.115 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.115 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:00.115 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.115 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:00.115 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:00.115 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92814 2023-10-01T09:39:00.115 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.115 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.115 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.115 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.116 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644861+0000", 2023-10-01T09:39:00.116 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.116 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.116 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.116 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.116 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644861+0000", 2023-10-01T09:39:00.116 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.116 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.116 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.116 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.116 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644860+0000", 2023-10-01T09:39:00.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:39:00.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.117 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.117 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644879+0000", 2023-10-01T09:39:00.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9857999999999999e-05 2023-10-01T09:39:00.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.117 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.117 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644892+0000", 2023-10-01T09:39:00.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2292e-05 2023-10-01T09:39:00.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.118 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.118 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.118 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644900+0000", 2023-10-01T09:39:00.118 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.259e-06 2023-10-01T09:39:00.118 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.118 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.118 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.118 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644960+0000", 2023-10-01T09:39:00.118 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0247000000000001e-05 2023-10-01T09:39:00.118 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.118 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.119 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644968+0000", 2023-10-01T09:39:00.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8099999999999998e-06 2023-10-01T09:39:00.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.119 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.640030+0000", 2023-10-01T09:39:00.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99506214699999995 2023-10-01T09:39:00.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.119 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.640250+0000", 2023-10-01T09:39:00.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021938700000000001 2023-10-01T09:39:00.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:39:00.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.640445+0000", 2023-10-01T09:39:00.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019577400000000001 2023-10-01T09:39:00.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.648390+0000", 2023-10-01T09:39:00.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0079443840000000005 2023-10-01T09:39:00.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.652516+0000", 2023-10-01T09:39:00.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041264450000000003 2023-10-01T09:39:00.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.666515+0000", 2023-10-01T09:39:00.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013999161 2023-10-01T09:39:00.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.666577+0000", 2023-10-01T09:39:00.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1958000000000005e-05 2023-10-01T09:39:00.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.667510+0000", 2023-10-01T09:39:00.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00093287700000000003 2023-10-01T09:39:00.122 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.122 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.123 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.123 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.123 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92832 9.2 9:46d6656f:::benchmark_data_smithi138_497829_object92831:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:39:00.123 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.698836+0000", 2023-10-01T09:39:00.123 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 206.25075479, 2023-10-01T09:39:00.123 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96914843500000003, 2023-10-01T09:39:00.123 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.123 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:00.123 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.123 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:00.124 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:00.124 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92832 2023-10-01T09:39:00.124 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.124 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.124 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.124 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698836+0000", 2023-10-01T09:39:00.124 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.124 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.124 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.125 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698836+0000", 2023-10-01T09:39:00.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.125 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.125 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.125 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.125 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698835+0000", 2023-10-01T09:39:00.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:39:00.125 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.125 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.125 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.125 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698854+0000", 2023-10-01T09:39:00.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9089e-05 2023-10-01T09:39:00.126 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.126 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.126 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698865+0000", 2023-10-01T09:39:00.126 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0429e-05 2023-10-01T09:39:00.126 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.126 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.126 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698870+0000", 2023-10-01T09:39:00.126 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3229999999999997e-06 2023-10-01T09:39:00.126 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.127 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698930+0000", 2023-10-01T09:39:00.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0136999999999997e-05 2023-10-01T09:39:00.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.127 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698943+0000", 2023-10-01T09:39:00.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3081999999999999e-05 2023-10-01T09:39:00.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.643687+0000", 2023-10-01T09:39:00.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94474369199999997 2023-10-01T09:39:00.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.643851+0000", 2023-10-01T09:39:00.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000164239 2023-10-01T09:39:00.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:39:00.129 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.643968+0000", 2023-10-01T09:39:00.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000116741 2023-10-01T09:39:00.129 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.129 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.129 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.129 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.649439+0000", 2023-10-01T09:39:00.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005470801 2023-10-01T09:39:00.129 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.129 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.129 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.129 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.653893+0000", 2023-10-01T09:39:00.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044539159999999996 2023-10-01T09:39:00.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.130 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.130 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.667566+0000", 2023-10-01T09:39:00.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013673284000000001 2023-10-01T09:39:00.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.130 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.130 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.667617+0000", 2023-10-01T09:39:00.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0624000000000002e-05 2023-10-01T09:39:00.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.131 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.131 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.131 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.667984+0000", 2023-10-01T09:39:00.131 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00036788899999999999 2023-10-01T09:39:00.131 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.131 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.131 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.131 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.131 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.131 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:147923 9.a 9:5afed6f2:::benchmark_data_smithi138_497829_object45538:head [delete] snapc 0=[] ondisk+write+known_if_redirected e444)", 2023-10-01T09:39:00.131 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:22.282541+0000", 2023-10-01T09:39:00.131 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 97.667049539999994, 2023-10-01T09:39:00.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.55728740300000001, 2023-10-01T09:39:00.132 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:00.132 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:00.132 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:00.132 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:00.132 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:00.132 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 147923 2023-10-01T09:39:00.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.132 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:00.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:00.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282541+0000", 2023-10-01T09:39:00.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:00.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282541+0000", 2023-10-01T09:39:00.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:00.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:00.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282541+0000", 2023-10-01T09:39:00.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:39:00.134 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.134 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.134 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:00.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282546+0000", 2023-10-01T09:39:00.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.972e-06 2023-10-01T09:39:00.134 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.134 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.134 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:00.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282560+0000", 2023-10-01T09:39:00.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4630999999999999e-05 2023-10-01T09:39:00.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:00.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282567+0000", 2023-10-01T09:39:00.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3010000000000001e-06 2023-10-01T09:39:00.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.285815+0000", 2023-10-01T09:39:00.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003247963 2023-10-01T09:39:00.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:39:00.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289067+0000", 2023-10-01T09:39:00.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032514169999999999 2023-10-01T09:39:00.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.307845+0000", 2023-10-01T09:39:00.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018777933 2023-10-01T09:39:00.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.137 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:00.137 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.307857+0000", 2023-10-01T09:39:00.137 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.254e-05 2023-10-01T09:39:00.137 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.137 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.137 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.137 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.805080+0000", 2023-10-01T09:39:00.137 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.49722238000000002 2023-10-01T09:39:00.137 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.137 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.138 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:39:00.138 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.805434+0000", 2023-10-01T09:39:00.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00035419299999999999 2023-10-01T09:39:00.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.138 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:00.138 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.815559+0000", 2023-10-01T09:39:00.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010125265 2023-10-01T09:39:00.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.138 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:00.139 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.815641+0000", 2023-10-01T09:39:00.139 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1499e-05 2023-10-01T09:39:00.139 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.139 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.139 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:39:00.139 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.815740+0000", 2023-10-01T09:39:00.139 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9504000000000001e-05 2023-10-01T09:39:00.139 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.139 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.139 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:00.139 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.827989+0000", 2023-10-01T09:39:00.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01224926 2023-10-01T09:39:00.140 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.140 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.140 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.828043+0000", 2023-10-01T09:39:00.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3876999999999999e-05 2023-10-01T09:39:00.140 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.140 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.140 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:00.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.836681+0000", 2023-10-01T09:39:00.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086381300000000008 2023-10-01T09:39:00.141 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.141 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:00.141 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.836733+0000", 2023-10-01T09:39:00.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1399e-05 2023-10-01T09:39:00.141 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:00.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:00.141 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:00.141 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.839829+0000", 2023-10-01T09:39:00.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030958539999999999 2023-10-01T09:39:00.142 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.142 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.142 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.142 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:00.142 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:00.142 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:39:00.143 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:38:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/18410904' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:00.143 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:01.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:00 smithi190 ceph-mon[160435]: pgmap v1309: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 12868/52188 objects degraded (24.657%); 11363/52188 objects misplaced (21.773%); 8.9 MiB/s, 141 objects/s recovering 2023-10-01T09:39:01.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:39:01.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:00 smithi138 ceph-mon[185886]: pgmap v1309: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 12868/52188 objects degraded (24.657%); 11363/52188 objects misplaced (21.773%); 8.9 MiB/s, 141 objects/s recovering 2023-10-01T09:39:01.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:39:01.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:00 smithi138 ceph-mon[192513]: pgmap v1309: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 433 op/s; 12868/52188 objects degraded (24.657%); 11363/52188 objects misplaced (21.773%); 8.9 MiB/s, 141 objects/s recovering 2023-10-01T09:39:01.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:39:02.039 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:02 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:02.039 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:02 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:02.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:02 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:02.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:02 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:02.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:02 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:02.323 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:02 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:02.371 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:02.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:02.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:02.458 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:39:02.481 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:02.594 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:02.789 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:02.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:02.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:02.914 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:03.013 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:03.034 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 2, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:39:03.034 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:39:03.034 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T09:39:03.034 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.680166 2023-10-01T09:39:03.035 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 0.6801663609867138 2023-10-01T09:39:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:03 smithi190 ceph-mon[160435]: pgmap v1310: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 12868/51411 objects degraded (25.030%); 11363/51411 objects misplaced (22.102%); 8.9 MiB/s, 141 objects/s recovering 2023-10-01T09:39:03.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:03 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:03.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:03 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:03.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:03 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 12868/52188 objects degraded (24.657%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:03.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:03 smithi138 ceph-mon[192513]: pgmap v1310: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 12868/51411 objects degraded (25.030%); 11363/51411 objects misplaced (22.102%); 8.9 MiB/s, 141 objects/s recovering 2023-10-01T09:39:03.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:03 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:03.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:03 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:03.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:03 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 12868/52188 objects degraded (24.657%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:03.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:03 smithi138 ceph-mon[185886]: pgmap v1310: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 12868/51411 objects degraded (25.030%); 11363/51411 objects misplaced (22.102%); 8.9 MiB/s, 141 objects/s recovering 2023-10-01T09:39:03.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:03 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:03.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:03 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:03.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:03 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 12868/52188 objects degraded (24.657%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:03.397 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:03.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:03.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:03.508 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:03.536 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:03.562 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:03.627 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:03.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:03.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:03.735 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:39:03.821 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:04.002 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:39:04.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:04.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:39:04.111 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:04.155 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:04.236 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:04.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:04.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:04.333 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:04.343 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:39:04.372 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:04.406 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:39:04.480 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:39:04.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:04.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:39:04.584 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:39:04.703 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:05.296 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:39:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:39:04] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.33.4" 2023-10-01T09:39:05.297 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:05 smithi138 ceph-mon[185886]: pgmap v1311: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 376 op/s; 12868/50976 objects degraded (25.243%); 11363/50976 objects misplaced (22.291%); 7.5 MiB/s, 119 objects/s recovering 2023-10-01T09:39:05.297 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1506869755' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:05.297 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:05 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:05.297 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:05 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:05.297 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:05 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:05.356 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:39:05.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:05 smithi190 ceph-mon[160435]: pgmap v1311: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 376 op/s; 12868/50976 objects degraded (25.243%); 11363/50976 objects misplaced (22.291%); 7.5 MiB/s, 119 objects/s recovering 2023-10-01T09:39:05.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1506869755' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:05.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:05 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:05.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:05 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:05.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:05 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:05.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:05 smithi138 ceph-mon[192513]: pgmap v1311: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 376 op/s; 12868/50976 objects degraded (25.243%); 11363/50976 objects misplaced (22.291%); 7.5 MiB/s, 119 objects/s recovering 2023-10-01T09:39:05.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1506869755' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:05.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:05 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:05.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:05 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:05.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:05 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:05.611 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:39:05.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:05.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:39:05.723 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:39:05.762 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:05.823 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:39:05.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:05.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:39:05.933 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:05.960 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:06.304 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:06 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:39:06.304 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:06 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:39:06.304 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:06 smithi138 ceph-mon[192513]: osdmap e460: 8 total, 8 up, 8 in 2023-10-01T09:39:06.304 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:06 smithi138 ceph-mon[192513]: failed to encode map e460 with expected crc 2023-10-01T09:39:06.305 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:06 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:39:06.305 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:06 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:39:06.305 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:06 smithi138 ceph-mon[185886]: osdmap e460: 8 total, 8 up, 8 in 2023-10-01T09:39:06.305 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:06 smithi138 ceph-mon[185886]: failed to encode map e460 with expected crc 2023-10-01T09:39:06.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:06 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:39:06.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:06 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:39:06.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:06 smithi190 ceph-mon[160435]: osdmap e460: 8 total, 8 up, 8 in 2023-10-01T09:39:06.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:06 smithi190 ceph-mon[160435]: failed to encode map e460 with expected crc 2023-10-01T09:39:06.557 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:06.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:06.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:06.664 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:39:06.772 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:07.090 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:39:07.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:07.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:39:07.202 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:07.240 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:07.246 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:07.318 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:07.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:07.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:07.410 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:07 smithi138 ceph-mon[192513]: pgmap v1313: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 939 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 12341/44928 objects degraded (27.468%); 10562/44928 objects misplaced (23.509%); 8.8 MiB/s, 141 objects/s recovering 2023-10-01T09:39:07.410 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:07 smithi138 ceph-mon[185886]: pgmap v1313: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 939 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 12341/44928 objects degraded (27.468%); 10562/44928 objects misplaced (23.509%); 8.8 MiB/s, 141 objects/s recovering 2023-10-01T09:39:07.426 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:07.456 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:07.518 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:07.523 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:07 smithi190 ceph-mon[160435]: pgmap v1313: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 939 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 12341/44928 objects degraded (27.468%); 10562/44928 objects misplaced (23.509%); 8.8 MiB/s, 141 objects/s recovering 2023-10-01T09:39:07.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:07.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:07.625 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:07.727 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:08.375 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:08.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:08.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:08.481 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:08.509 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:08.571 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:08.576 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:08 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 12341/44928 objects degraded (27.468%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:08.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:08.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:08.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:08 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 12341/44928 objects degraded (27.468%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:08.586 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:08 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 12341/44928 objects degraded (27.468%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:08.604 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:39:08.604 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:39:08.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:08.604 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:191380 9.7 e460/431 9:e9275cee:::benchmark_data_smithi138_497829_object88995:head v 460'11931, mlcod=419'5548)", 2023-10-01T09:39:08.604 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:39:08.598263+0000", 2023-10-01T09:39:08.604 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0054960790000000001, 2023-10-01T09:39:08.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0055221669999999997, 2023-10-01T09:39:08.605 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:08.605 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:39:08.605 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:08.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:08.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:08.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:08.598263+0000", 2023-10-01T09:39:08.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:08.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:08.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:08.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:08.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:08.598263+0000", 2023-10-01T09:39:08.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:08.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:08.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:08.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:08.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:08.598262+0000", 2023-10-01T09:39:08.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:39:08.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:08.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:08.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:08.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:08.598274+0000", 2023-10-01T09:39:08.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1598e-05 2023-10-01T09:39:08.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:08.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:08.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:08.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:08.598295+0000", 2023-10-01T09:39:08.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1733999999999999e-05 2023-10-01T09:39:08.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:08.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:08.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:08.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:08.598307+0000", 2023-10-01T09:39:08.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1209000000000001e-05 2023-10-01T09:39:08.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:08.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:08.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:08.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:08.598399+0000", 2023-10-01T09:39:08.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2207999999999995e-05 2023-10-01T09:39:08.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:08.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:08.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:08.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:08.598430+0000", 2023-10-01T09:39:08.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0682000000000002e-05 2023-10-01T09:39:08.608 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:08.608 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:08.608 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:08.608 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:08.608 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:39:08.608 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 1 2023-10-01T09:39:08.609 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:39:08.678 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:08.706 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:08.778 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:08.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:08.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:08.886 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:08.893 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:39:08.925 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:08.994 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:09.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:09.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:09.105 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:09.175 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:09.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:09 smithi138 ceph-mon[192513]: pgmap v1314: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 931 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 12341/44574 objects degraded (27.687%); 10562/44574 objects misplaced (23.695%); 8.8 MiB/s, 141 objects/s recovering 2023-10-01T09:39:09.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:09 smithi138 ceph-mon[185886]: pgmap v1314: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 931 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 12341/44574 objects degraded (27.687%); 10562/44574 objects misplaced (23.695%); 8.8 MiB/s, 141 objects/s recovering 2023-10-01T09:39:09.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:09 smithi190 ceph-mon[160435]: pgmap v1314: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 931 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 12341/44574 objects degraded (27.687%); 10562/44574 objects misplaced (23.695%); 8.8 MiB/s, 141 objects/s recovering 2023-10-01T09:39:09.812 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:09.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:09.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:09.918 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:09.951 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:09.963 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:39:10.019 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:10.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:10.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:10.126 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:39:10.218 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:11.060 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:39:11.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:11.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:39:11.180 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:11.305 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:11.317 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:11.335 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:11 smithi138 ceph-mon[185886]: pgmap v1315: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 825 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 11840/39471 objects degraded (29.997%); 9759/39471 objects misplaced (24.724%); 8.3 MiB/s, 133 objects/s recovering 2023-10-01T09:39:11.336 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:11.336 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:39:11.443 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:11.574 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:11.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:11.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:11.593 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:11 smithi138 ceph-mon[192513]: pgmap v1315: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 825 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 11840/39471 objects degraded (29.997%); 9759/39471 objects misplaced (24.724%); 8.3 MiB/s, 133 objects/s recovering 2023-10-01T09:39:11.593 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:11.593 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:39:11.679 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:39:11.779 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:11.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:11 smithi190 ceph-mon[160435]: pgmap v1315: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 825 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 423 op/s; 11840/39471 objects degraded (29.997%); 9759/39471 objects misplaced (24.724%); 8.3 MiB/s, 133 objects/s recovering 2023-10-01T09:39:11.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:11.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:39:11.977 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:39:11.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:11.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:39:12.087 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:12.093 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:39:12.123 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:12.141 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:39:12.182 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:12.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:12.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:12.197 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:39:12.246 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:39:12.290 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:39:12.291 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:39:12.338 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:39:12.379 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:39:12.391 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:12.421 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.0 primary-affinity to 0.680166 (8445752) 2023-10-01T09:39:12.421 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:39:12.455 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:39:12.455 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:39:12.455 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:39:12.455 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:39:12.455 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:39:12.455 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:39:12.455 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:39:12.455 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:39:12.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:12 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2056264919' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6801663609867138}]: dispatch 2023-10-01T09:39:12.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:12 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/939089534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:12.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:12 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2056264919' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6801663609867138}]: dispatch 2023-10-01T09:39:12.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:12 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/939089534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:12.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:12 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2056264919' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6801663609867138}]: dispatch 2023-10-01T09:39:12.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:12 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/939089534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:13.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: pgmap v1316: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 808 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 11840/38646 objects degraded (30.637%); 9759/38646 objects misplaced (25.252%); 8.3 MiB/s, 133 objects/s recovering 2023-10-01T09:39:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2056264919' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6801663609867138}]': finished 2023-10-01T09:39:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: failed to encode map e461 with expected crc 2023-10-01T09:39:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: osdmap e461: 8 total, 8 up, 8 in 2023-10-01T09:39:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 11840/38646 objects degraded (30.637%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: osdmap e462: 8 total, 8 up, 8 in 2023-10-01T09:39:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:13 smithi190 ceph-mon[160435]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: pgmap v1316: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 808 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 11840/38646 objects degraded (30.637%); 9759/38646 objects misplaced (25.252%); 8.3 MiB/s, 133 objects/s recovering 2023-10-01T09:39:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2056264919' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6801663609867138}]': finished 2023-10-01T09:39:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: failed to encode map e461 with expected crc 2023-10-01T09:39:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: osdmap e461: 8 total, 8 up, 8 in 2023-10-01T09:39:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 11840/38646 objects degraded (30.637%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:13.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: osdmap e462: 8 total, 8 up, 8 in 2023-10-01T09:39:13.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[185886]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: pgmap v1316: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 808 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 425 op/s; 11840/38646 objects degraded (30.637%); 9759/38646 objects misplaced (25.252%); 8.3 MiB/s, 133 objects/s recovering 2023-10-01T09:39:13.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2056264919' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6801663609867138}]': finished 2023-10-01T09:39:13.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: failed to encode map e461 with expected crc 2023-10-01T09:39:13.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: osdmap e461: 8 total, 8 up, 8 in 2023-10-01T09:39:13.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 11840/38646 objects degraded (30.637%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:13.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: osdmap e462: 8 total, 8 up, 8 in 2023-10-01T09:39:13.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: failed to encode map e462 with expected crc 2023-10-01T09:39:13.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:13 smithi138 ceph-mon[192513]: failed to encode map e462 with expected crc 2023-10-01T09:39:14.756 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:39:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:14.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:39:14.865 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:14.902 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:14.957 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:14.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:14.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:14.970 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:14 smithi190 ceph-mon[160435]: failed to encode map e463 with expected crc 2023-10-01T09:39:14.970 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:14 smithi190 ceph-mon[160435]: failed to encode map e463 with expected crc 2023-10-01T09:39:14.970 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:14 smithi190 ceph-mon[160435]: failed to encode map e463 with expected crc 2023-10-01T09:39:14.970 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:14 smithi190 ceph-mon[160435]: osdmap e463: 8 total, 8 up, 8 in 2023-10-01T09:39:14.970 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:14 smithi190 ceph-mon[160435]: failed to encode map e463 with expected crc 2023-10-01T09:39:14.970 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:14 smithi190 ceph-mon[160435]: pgmap v1320: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 799 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 11840/38238 objects degraded (30.964%); 9759/38238 objects misplaced (25.522%); 6.8 MiB/s, 108 objects/s recovering 2023-10-01T09:39:15.050 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:14 smithi138 ceph-mon[192513]: failed to encode map e463 with expected crc 2023-10-01T09:39:15.051 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:14 smithi138 ceph-mon[192513]: failed to encode map e463 with expected crc 2023-10-01T09:39:15.051 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:14 smithi138 ceph-mon[192513]: failed to encode map e463 with expected crc 2023-10-01T09:39:15.051 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:14 smithi138 ceph-mon[192513]: osdmap e463: 8 total, 8 up, 8 in 2023-10-01T09:39:15.051 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:14 smithi138 ceph-mon[192513]: failed to encode map e463 with expected crc 2023-10-01T09:39:15.051 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:14 smithi138 ceph-mon[192513]: pgmap v1320: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 799 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 11840/38238 objects degraded (30.964%); 9759/38238 objects misplaced (25.522%); 6.8 MiB/s, 108 objects/s recovering 2023-10-01T09:39:15.051 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:14 smithi138 ceph-mon[185886]: failed to encode map e463 with expected crc 2023-10-01T09:39:15.052 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:14 smithi138 ceph-mon[185886]: failed to encode map e463 with expected crc 2023-10-01T09:39:15.052 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:14 smithi138 ceph-mon[185886]: failed to encode map e463 with expected crc 2023-10-01T09:39:15.052 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:14 smithi138 ceph-mon[185886]: osdmap e463: 8 total, 8 up, 8 in 2023-10-01T09:39:15.052 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:14 smithi138 ceph-mon[185886]: failed to encode map e463 with expected crc 2023-10-01T09:39:15.052 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:14 smithi138 ceph-mon[185886]: pgmap v1320: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 799 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 11840/38238 objects degraded (30.964%); 9759/38238 objects misplaced (25.522%); 6.8 MiB/s, 108 objects/s recovering 2023-10-01T09:39:15.052 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:39:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:39:14] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.33.4" 2023-10-01T09:39:15.069 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:39:15.109 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:16.658 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:39:16.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:16.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:39:16.778 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:16.815 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:16.902 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:16.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:16.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:17.014 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:17.045 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:17.109 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:17.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:17.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:17.170 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:39:17.170 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:39:17.220 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:39:17.259 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:17.305 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:17 smithi138 ceph-mon[185886]: pgmap v1321: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 699 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 336 op/s; 9154/33411 objects degraded (27.398%); 9595/33411 objects misplaced (28.718%); 6.7 MiB/s, 107 objects/s recovering 2023-10-01T09:39:17.319 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:39:17.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:17.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:39:17.436 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:39:17.548 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:17.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:17 smithi190 ceph-mon[160435]: pgmap v1321: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 699 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 336 op/s; 9154/33411 objects degraded (27.398%); 9595/33411 objects misplaced (28.718%); 6.7 MiB/s, 107 objects/s recovering 2023-10-01T09:39:17.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:17 smithi138 ceph-mon[192513]: pgmap v1321: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 699 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 336 op/s; 9154/33411 objects degraded (27.398%); 9595/33411 objects misplaced (28.718%); 6.7 MiB/s, 107 objects/s recovering 2023-10-01T09:39:18.121 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:39:18.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:18.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:39:18.228 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:18.253 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:18.296 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:18.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:18.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:18.403 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:18.479 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:18 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 9154/33411 objects degraded (27.398%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:18.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:18 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1464445800' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:39:18.480 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:18 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:39:18.483 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:18 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 9154/33411 objects degraded (27.398%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:18.484 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:18 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1464445800' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:39:18.484 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:18 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:39:18.491 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:18.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:18 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 9154/33411 objects degraded (27.398%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:18.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:18 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1464445800' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:39:18.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:18 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:39:18.746 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:39:18.749 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:39:18.975 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:39:19.073 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:19.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:19.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:19.179 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:19.275 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:19.317 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:19 smithi138 ceph-mon[185886]: pgmap v1322: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 679 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 14066/32448 objects degraded (43.349%); 4683/32448 objects misplaced (14.432%); 6.7 MiB/s, 107 objects/s recovering 2023-10-01T09:39:19.317 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:19 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:39:19.317 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:19 smithi138 ceph-mon[185886]: failed to encode map e464 with expected crc 2023-10-01T09:39:19.317 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:19 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:39:19.317 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:19 smithi138 ceph-mon[185886]: osdmap e464: 8 total, 8 up, 8 in 2023-10-01T09:39:19.317 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:19 smithi138 ceph-mon[185886]: failed to encode map e464 with expected crc 2023-10-01T09:39:19.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:19 smithi138 ceph-mon[192513]: pgmap v1322: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 679 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 14066/32448 objects degraded (43.349%); 4683/32448 objects misplaced (14.432%); 6.7 MiB/s, 107 objects/s recovering 2023-10-01T09:39:19.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:19 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:39:19.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:19 smithi138 ceph-mon[192513]: failed to encode map e464 with expected crc 2023-10-01T09:39:19.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:19 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:39:19.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:19 smithi138 ceph-mon[192513]: osdmap e464: 8 total, 8 up, 8 in 2023-10-01T09:39:19.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:19 smithi138 ceph-mon[192513]: failed to encode map e464 with expected crc 2023-10-01T09:39:19.796 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:19.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:19.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:19.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:19 smithi190 ceph-mon[160435]: pgmap v1322: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 679 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 343 op/s; 14066/32448 objects degraded (43.349%); 4683/32448 objects misplaced (14.432%); 6.7 MiB/s, 107 objects/s recovering 2023-10-01T09:39:19.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:19 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:39:19.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:19 smithi190 ceph-mon[160435]: failed to encode map e464 with expected crc 2023-10-01T09:39:19.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:19 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:39:19.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:19 smithi190 ceph-mon[160435]: osdmap e464: 8 total, 8 up, 8 in 2023-10-01T09:39:19.812 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:19 smithi190 ceph-mon[160435]: failed to encode map e464 with expected crc 2023-10-01T09:39:19.907 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:39:19.937 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:19.991 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:39:20.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:20.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:39:20.103 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:20.124 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:20.196 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:20.421 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:20.509 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:20.523 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:20 smithi138 ceph-mon[185886]: failed to encode map e464 with expected crc 2023-10-01T09:39:20.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:20.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:20.532 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:20 smithi138 ceph-mon[192513]: failed to encode map e464 with expected crc 2023-10-01T09:39:20.624 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:20.715 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:20.815 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:20 smithi190 ceph-mon[160435]: failed to encode map e464 with expected crc 2023-10-01T09:39:21.066 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:21.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:21.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:21.183 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:21.225 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:21.314 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:21.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:21.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:21.346 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:21 smithi138 ceph-mon[185886]: pgmap v1324: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 571 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 13817/27261 objects degraded (50.684%); 3624/27261 objects misplaced (13.294%); 9.7 MiB/s, 155 objects/s recovering 2023-10-01T09:39:21.346 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:21 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:21.347 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:21 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:21.425 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:21.478 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:21 smithi190 ceph-mon[160435]: pgmap v1324: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 571 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 13817/27261 objects degraded (50.684%); 3624/27261 objects misplaced (13.294%); 9.7 MiB/s, 155 objects/s recovering 2023-10-01T09:39:21.479 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:21 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:21.479 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:21 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:21.481 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:21.561 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:21.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:21.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:21.680 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:21.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:21 smithi138 ceph-mon[192513]: pgmap v1324: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 571 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 484 op/s; 13817/27261 objects degraded (50.684%); 3624/27261 objects misplaced (13.294%); 9.7 MiB/s, 155 objects/s recovering 2023-10-01T09:39:21.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:21 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:21.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:21 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:21.760 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:21.938 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:39:22.043 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 2, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:39:22.044 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:39:22.044 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-01T09:39:22.044 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:39:22.044 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:39:22.044 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-01T09:39:22.612 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:39:23.263 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:23.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:23.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:23.371 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:23.374 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:23 smithi138 ceph-mon[185886]: pgmap v1325: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 531 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 13688/25338 objects degraded (54.022%); 3624/25338 objects misplaced (14.303%); 9.5 MiB/s, 152 objects/s recovering 2023-10-01T09:39:23.374 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:23 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 13688/25338 objects degraded (54.022%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:23.374 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:23 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:39:23.374 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:23 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:39:23.374 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:23 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:23.413 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:23.506 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:23.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:23.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:23.611 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:39:23.653 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:23 smithi138 ceph-mon[192513]: pgmap v1325: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 531 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 13688/25338 objects degraded (54.022%); 3624/25338 objects misplaced (14.303%); 9.5 MiB/s, 152 objects/s recovering 2023-10-01T09:39:23.654 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:23 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 13688/25338 objects degraded (54.022%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:23.654 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:23 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:39:23.654 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:23 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:39:23.654 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:23 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:23.727 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:23.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:23 smithi190 ceph-mon[160435]: pgmap v1325: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 531 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 13688/25338 objects degraded (54.022%); 3624/25338 objects misplaced (14.303%); 9.5 MiB/s, 152 objects/s recovering 2023-10-01T09:39:23.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:23 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 13688/25338 objects degraded (54.022%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:23 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:39:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:23 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:39:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:23 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:23.918 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:23.942 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:39:23.942 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:39:23.951 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:39:23.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:23.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:39:24.062 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:24.106 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:24.196 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:24.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:24.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:24.289 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:39:24.302 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:24.332 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:39:24.357 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:24.386 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:39:24.419 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:24.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:24.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:24.429 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:39:24.466 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:39:24.492 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:39:24.521 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:39:24.527 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:39:24.555 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:39:24.564 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:24.590 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:39:24.590 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:39:24.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:39:24.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:39:24.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:39:24.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:39:24.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:39:24.591 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:39:24.621 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1974034366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:24.621 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1974034366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:24.646 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:39:24.649 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1974034366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:24.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:24.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:39:24.751 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:24.846 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:24.911 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:24.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:24.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:25.015 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:25.117 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:25.199 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:39:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:39:24] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.33.4" 2023-10-01T09:39:25.563 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:25 smithi138 ceph-mon[185886]: pgmap v1326: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 521 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 13688/24870 objects degraded (55.038%); 3624/24870 objects misplaced (14.572%); 8.1 MiB/s, 129 objects/s recovering 2023-10-01T09:39:25.563 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:25.563 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:39:25.563 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:25 smithi138 ceph-mon[192513]: pgmap v1326: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 521 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 13688/24870 objects degraded (55.038%); 3624/24870 objects misplaced (14.572%); 8.1 MiB/s, 129 objects/s recovering 2023-10-01T09:39:25.563 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:25.564 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:39:25.655 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:39:25.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:25 smithi190 ceph-mon[160435]: pgmap v1326: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 521 MiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 13688/24870 objects degraded (55.038%); 3624/24870 objects misplaced (14.572%); 8.1 MiB/s, 129 objects/s recovering 2023-10-01T09:39:25.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:25.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:39:25.811 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:25.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:25.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:25.927 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:25.957 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:26.055 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:26.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:26.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:26.105 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:39:26.105 INFO:teuthology.orchestra.run.smithi190.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T09:39:26.105 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:39:26.158 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:39:26.198 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:26.399 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:39:26.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:26.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:39:26.508 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:26.556 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:26.637 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:26.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:26.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:26.742 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:39:26.776 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:26 smithi190 ceph-mon[160435]: pgmap v1327: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 412 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 13545/19623 objects degraded (69.026%); 2547/19623 objects misplaced (12.980%); 6.9 MiB/s, 110 objects/s recovering 2023-10-01T09:39:26.783 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:26.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:26 smithi138 ceph-mon[192513]: pgmap v1327: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 412 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 13545/19623 objects degraded (69.026%); 2547/19623 objects misplaced (12.980%); 6.9 MiB/s, 110 objects/s recovering 2023-10-01T09:39:26.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:26 smithi138 ceph-mon[185886]: pgmap v1327: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 412 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 13545/19623 objects degraded (69.026%); 2547/19623 objects misplaced (12.980%); 6.9 MiB/s, 110 objects/s recovering 2023-10-01T09:39:26.830 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:39:26.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:26.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:39:26.934 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:39:26.986 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:27.216 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:39:27.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:27.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:39:27.327 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:27.361 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:27.368 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:27.482 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:27.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:27.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:27.594 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:27.681 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:27.766 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:27 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 13545/19623 objects degraded (69.026%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:27.902 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:27 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 13545/19623 objects degraded (69.026%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:27.903 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:27 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 13545/19623 objects degraded (69.026%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:27.970 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:27.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:27.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:28.075 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:39:28.194 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:28.385 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:39:28.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:28.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:39:28.503 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:28.585 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:28.697 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:28.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:28.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:28.775 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:39:28.806 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:28.832 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:28.872 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:28.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:28.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:28.979 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:29.022 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:28 smithi190 ceph-mon[160435]: pgmap v1328: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 361 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 13401/17217 objects degraded (77.836%); 2547/17217 objects misplaced (14.794%); 7.8 MiB/s, 124 objects/s recovering 2023-10-01T09:39:29.022 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3330482921' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:29.022 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:28 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:29.024 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:29.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:28 smithi138 ceph-mon[185886]: pgmap v1328: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 361 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 13401/17217 objects degraded (77.836%); 2547/17217 objects misplaced (14.794%); 7.8 MiB/s, 124 objects/s recovering 2023-10-01T09:39:29.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3330482921' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:29.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:28 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:29.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:28 smithi138 ceph-mon[192513]: pgmap v1328: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 361 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 13401/17217 objects degraded (77.836%); 2547/17217 objects misplaced (14.794%); 7.8 MiB/s, 124 objects/s recovering 2023-10-01T09:39:29.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3330482921' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:29.074 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:28 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:29.113 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:29.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:29.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:29.167 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:39:29.224 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:29.271 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:29.339 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:29.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:29.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:29.448 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:29.533 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:29 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:39:29.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:29 smithi138 ceph-mon[192513]: failed to encode map e465 with expected crc 2023-10-01T09:39:29.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:29 smithi138 ceph-mon[192513]: failed to encode map e465 with expected crc 2023-10-01T09:39:29.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:29 smithi138 ceph-mon[192513]: failed to encode map e465 with expected crc 2023-10-01T09:39:29.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:29 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:39:29.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:29 smithi138 ceph-mon[192513]: osdmap e465: 8 total, 8 up, 8 in 2023-10-01T09:39:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:29 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:39:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:29 smithi138 ceph-mon[185886]: failed to encode map e465 with expected crc 2023-10-01T09:39:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:29 smithi138 ceph-mon[185886]: failed to encode map e465 with expected crc 2023-10-01T09:39:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:29 smithi138 ceph-mon[185886]: failed to encode map e465 with expected crc 2023-10-01T09:39:29.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:29 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:39:29.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:29 smithi138 ceph-mon[185886]: osdmap e465: 8 total, 8 up, 8 in 2023-10-01T09:39:30.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:29 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:39:30.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:29 smithi190 ceph-mon[160435]: failed to encode map e465 with expected crc 2023-10-01T09:39:30.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:29 smithi190 ceph-mon[160435]: failed to encode map e465 with expected crc 2023-10-01T09:39:30.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:29 smithi190 ceph-mon[160435]: failed to encode map e465 with expected crc 2023-10-01T09:39:30.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:29 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:39:30.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:29 smithi190 ceph-mon[160435]: osdmap e465: 8 total, 8 up, 8 in 2023-10-01T09:39:30.127 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:30.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:30.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:30.255 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:39:30.352 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:30.556 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:39:30.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:30.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:39:30.610 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:30.680 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:39:30.762 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:30.766 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:39:30.766 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:39:30.766 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.766 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:201627 9.1 9:8e851fa2:::benchmark_data_smithi138_497829_object99242:head [delete] snapc 0=[] ondisk+write+known_if_redirected e465)", 2023-10-01T09:39:30.766 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:39:30.743372+0000", 2023-10-01T09:39:30.767 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.014806837, 2023-10-01T09:39:30.767 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014862492, 2023-10-01T09:39:30.767 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:30.767 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:39:30.767 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:30.767 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:30.767 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:30.767 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 201627 2023-10-01T09:39:30.767 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.767 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:30.767 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:30.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743372+0000", 2023-10-01T09:39:30.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:30.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.768 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:30.768 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743372+0000", 2023-10-01T09:39:30.768 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:30.768 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.768 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.768 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:30.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743371+0000", 2023-10-01T09:39:30.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:39:30.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:30.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743377+0000", 2023-10-01T09:39:30.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.657e-06 2023-10-01T09:39:30.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.769 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.769 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:30.769 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743395+0000", 2023-10-01T09:39:30.769 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7995999999999998e-05 2023-10-01T09:39:30.769 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.770 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:30.770 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743404+0000", 2023-10-01T09:39:30.770 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3829999999999992e-06 2023-10-01T09:39:30.770 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.770 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:30.770 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743445+0000", 2023-10-01T09:39:30.770 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1096000000000001e-05 2023-10-01T09:39:30.770 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.770 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.771 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:30.771 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743567+0000", 2023-10-01T09:39:30.771 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000121836 2023-10-01T09:39:30.771 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.771 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.771 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:39:30.771 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743685+0000", 2023-10-01T09:39:30.771 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011807900000000001 2023-10-01T09:39:30.771 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.771 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.771 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:30.772 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.747461+0000", 2023-10-01T09:39:30.772 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003776499 2023-10-01T09:39:30.772 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.772 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.772 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:30.772 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.747870+0000", 2023-10-01T09:39:30.772 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00040892000000000001 2023-10-01T09:39:30.772 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:30.772 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:30.772 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:30.772 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.772 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.773 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:201621 9.f e465/445 9:f9561221:::benchmark_data_smithi138_497829_object99236:head v 465'12653, mlcod=465'12650)", 2023-10-01T09:39:30.773 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:39:30.743835+0000", 2023-10-01T09:39:30.773 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0143435, 2023-10-01T09:39:30.773 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014490064, 2023-10-01T09:39:30.773 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:30.773 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:39:30.773 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:30.773 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.773 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:30.773 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743835+0000", 2023-10-01T09:39:30.773 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:30.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:30.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743835+0000", 2023-10-01T09:39:30.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:30.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.774 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:30.774 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743835+0000", 2023-10-01T09:39:30.774 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:39:30.774 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.774 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:30.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743847+0000", 2023-10-01T09:39:30.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1782e-05 2023-10-01T09:39:30.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:30.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743858+0000", 2023-10-01T09:39:30.775 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1036000000000001e-05 2023-10-01T09:39:30.775 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.775 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.775 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:30.775 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.743864+0000", 2023-10-01T09:39:30.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8360000000000003e-06 2023-10-01T09:39:30.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.776 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:30.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.748309+0000", 2023-10-01T09:39:30.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.004444262 2023-10-01T09:39:30.776 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:30.776 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:30.776 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:30.776 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:30.748330+0000", 2023-10-01T09:39:30.776 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1501e-05 2023-10-01T09:39:30.776 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:30.777 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:30.777 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:30.777 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:30.777 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:39:30.777 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 2 2023-10-01T09:39:30.777 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:39:31.027 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:30 smithi138 ceph-mon[185886]: pgmap v1330: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 259 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 13266/12294 objects degraded (107.906%); 1457/12294 objects misplaced (11.851%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:39:31.027 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:30 smithi138 ceph-mon[192513]: pgmap v1330: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 259 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 13266/12294 objects degraded (107.906%); 1457/12294 objects misplaced (11.851%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:39:31.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:30 smithi190 ceph-mon[160435]: pgmap v1330: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 259 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 13266/12294 objects degraded (107.906%); 1457/12294 objects misplaced (11.851%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:39:31.098 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:39:31.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:31.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:39:31.210 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:31.255 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:31.400 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:31.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:31.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:31.506 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:31.549 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:31.600 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:31.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:31.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:31.706 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:31.768 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:32.223 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:39:32.235 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 2, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:39:32.236 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:39:32.236 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-01T09:39:32.236 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:39:32.805 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:32.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:32.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:32.878 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:39:32.914 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:39:32.954 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:33.013 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:39:33.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:33.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:39:33.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:39:33.251 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:33.426 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:33.463 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:33.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:33 smithi138 ceph-mon[185886]: pgmap v1331: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 220 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 13119/10446 objects degraded (125.589%); 1457/10446 objects misplaced (13.948%); 7.3 MiB/s, 117 objects/s recovering 2023-10-01T09:39:33.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:33 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 13119/10446 objects degraded (125.589%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:33.572 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:39:33.574 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:33 smithi138 ceph-mon[192513]: pgmap v1331: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 220 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 13119/10446 objects degraded (125.589%); 1457/10446 objects misplaced (13.948%); 7.3 MiB/s, 117 objects/s recovering 2023-10-01T09:39:33.575 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:33 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 13119/10446 objects degraded (125.589%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:33.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:33.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:39:33.681 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:33.741 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:33 smithi190 ceph-mon[160435]: pgmap v1331: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 220 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 13119/10446 objects degraded (125.589%); 1457/10446 objects misplaced (13.948%); 7.3 MiB/s, 117 objects/s recovering 2023-10-01T09:39:33.741 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:33 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 13119/10446 objects degraded (125.589%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:33.745 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:33.836 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:33.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:33.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:33.947 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:34.009 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:39:34.040 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:34.058 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:39:34.148 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:39:34.174 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:34.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:34.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:34.208 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:39:34.229 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:39:34.275 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:39:34.291 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:34.331 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:39:34.336 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:34.386 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:39:34.401 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:39:34.401 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:39:34.401 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:39:34.401 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:39:34.401 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:39:34.401 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:39:34.401 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:39:34.401 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:39:34.404 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:34.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:34.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:34.512 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:34.554 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:34.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:34 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/959945247' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:34.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:34 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/959945247' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:34.637 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:34.653 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:34 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/959945247' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:34.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:34.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:34.755 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:34.825 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:34.943 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:34.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:34.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:35.058 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:35.071 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:39:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:39:34] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.33.4" 2023-10-01T09:39:35.126 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:35.515 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:35.517 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:35 smithi138 ceph-mon[192513]: pgmap v1332: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 211 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 13119/9996 objects degraded (131.242%); 1457/9996 objects misplaced (14.576%); 7.3 MiB/s, 117 objects/s recovering 2023-10-01T09:39:35.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:35 smithi138 ceph-mon[185886]: pgmap v1332: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 211 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 13119/9996 objects degraded (131.242%); 1457/9996 objects misplaced (14.576%); 7.3 MiB/s, 117 objects/s recovering 2023-10-01T09:39:35.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:35.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:35.619 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:35.662 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:35.714 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:35.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:35.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:35 smithi190 ceph-mon[160435]: pgmap v1332: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 211 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 13119/9996 objects degraded (131.242%); 1457/9996 objects misplaced (14.576%); 7.3 MiB/s, 117 objects/s recovering 2023-10-01T09:39:35.818 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:35.858 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:35.904 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:35.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:35.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:36.011 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:36.062 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:37.006 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:37.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:37.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:37.114 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Removed 102382 objects 2023-10-01T09:39:37.114 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Clean up completed and total clean up time :230.324 2023-10-01T09:39:37.119 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:37.151 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:37.158 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:37.253 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:37.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:37.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:37.357 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:37.390 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:37.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:37 smithi138 ceph-mon[185886]: pgmap v1333: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 94 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 12960/4377 objects degraded (296.093%); 324/4377 objects misplaced (7.402%); 7.3 MiB/s, 118 objects/s recovering 2023-10-01T09:39:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:37 smithi138 ceph-mon[192513]: pgmap v1333: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 94 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 12960/4377 objects degraded (296.093%); 324/4377 objects misplaced (7.402%); 7.3 MiB/s, 118 objects/s recovering 2023-10-01T09:39:37.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:37 smithi190 ceph-mon[160435]: pgmap v1333: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 94 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 12960/4377 objects degraded (296.093%); 324/4377 objects misplaced (7.402%); 7.3 MiB/s, 118 objects/s recovering 2023-10-01T09:39:37.841 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:37.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:37.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:37.946 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:39:37.978 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:38.130 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:39:38.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:38.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:39:38.163 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:39:38.165 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:39:38.234 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:38.263 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:38.312 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:38.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:38.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:38.416 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:38.445 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:38.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:38 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 12960/4377 objects degraded (296.093%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:38.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3647026175' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:39:38.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:38 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 12960/4377 objects degraded (296.093%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:38.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3647026175' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:39:38.723 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:39:38.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:38 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 12960/4377 objects degraded (296.093%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3647026175' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:39:39.144 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:39:39.498 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[185886]: from='client.36375 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:39:39.498 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[185886]: pgmap v1334: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 51 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 12824/2301 objects degraded (557.323%); 324/2301 objects misplaced (14.081%); 7.2 MiB/s, 118 objects/s recovering 2023-10-01T09:39:39.498 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:39:39.498 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[185886]: failed to encode map e466 with expected crc 2023-10-01T09:39:39.498 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[185886]: failed to encode map e466 with expected crc 2023-10-01T09:39:39.499 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[185886]: failed to encode map e466 with expected crc 2023-10-01T09:39:39.499 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3647026175' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:39:39.499 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[185886]: osdmap e466: 8 total, 8 up, 8 in 2023-10-01T09:39:39.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[192513]: from='client.36375 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:39:39.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[192513]: pgmap v1334: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 51 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 12824/2301 objects degraded (557.323%); 324/2301 objects misplaced (14.081%); 7.2 MiB/s, 118 objects/s recovering 2023-10-01T09:39:39.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:39:39.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[192513]: failed to encode map e466 with expected crc 2023-10-01T09:39:39.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[192513]: failed to encode map e466 with expected crc 2023-10-01T09:39:39.499 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[192513]: failed to encode map e466 with expected crc 2023-10-01T09:39:39.500 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3647026175' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:39:39.500 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:39 smithi138 ceph-mon[192513]: osdmap e466: 8 total, 8 up, 8 in 2023-10-01T09:39:39.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:39 smithi190 ceph-mon[160435]: from='client.36375 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:39:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:39 smithi190 ceph-mon[160435]: pgmap v1334: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 51 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 12824/2301 objects degraded (557.323%); 324/2301 objects misplaced (14.081%); 7.2 MiB/s, 118 objects/s recovering 2023-10-01T09:39:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:39 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:39:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:39 smithi190 ceph-mon[160435]: failed to encode map e466 with expected crc 2023-10-01T09:39:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:39 smithi190 ceph-mon[160435]: failed to encode map e466 with expected crc 2023-10-01T09:39:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:39 smithi190 ceph-mon[160435]: failed to encode map e466 with expected crc 2023-10-01T09:39:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:39 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3647026175' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:39:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:39 smithi190 ceph-mon[160435]: osdmap e466: 8 total, 8 up, 8 in 2023-10-01T09:39:40.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:39:40.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:39:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:39:41.467 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1333,"stamp":"2023-10-01T09:39:36.247945+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":98506699,"num_objects":1459,"num_object_clones":0,"num_object_copies":4377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12960,"num_objects_misplaced":324,"num_objects_unfound":0,"num_objects_dirty":1459,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":204441,"num_write_kb":6567590,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26864,"num_bytes_recovered":1745719639,"num_keys_recovered":90,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121707,"ondisk_log_size":121707,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8144972,"kb_used_data":2601292,"kb_used_omap":50,"kb_used_meta":5543309,"kb_avail":741816244,"statfs":{"total":767960285184,"available":759619833856,"internally_reserved":0,"allocated":2663723008,"data_stored":2652601989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51738,"internal_metadata":5676348902},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,4,1,6,4,0,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-333053952,"num_objects":-5082,"num_object_clones":0,"num_object_copies":-15246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-585,"num_objects_misplaced":-2223,"num_objects_unfound":0,"num_objects_dirty":-5082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5082,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1190,"num_bytes_recovered":76414976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008671"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":378,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.746063+0000","last_change":"2023-10-01T09:35:34.561455+0000","last_active":"2023-10-01T09:39:18.746063+0000","last_peered":"2023-10-01T09:39:18.746063+0000","last_clean":"2023-10-01T09:39:18.746063+0000","last_became_active":"2023-10-01T09:35:34.560978+0000","last_became_peered":"2023-10-01T09:35:34.560978+0000","last_unstale":"2023-10-01T09:39:18.746063+0000","last_undegraded":"2023-10-01T09:39:18.746063+0000","last_fullsized":"2023-10-01T09:39:18.746063+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:02:58.965198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":82,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.742278+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:39:18.742278+0000","last_peered":"2023-10-01T09:39:18.742278+0000","last_clean":"2023-10-01T09:39:18.742278+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:39:18.742278+0000","last_undegraded":"2023-10-01T09:39:18.742278+0000","last_fullsized":"2023-10-01T09:39:18.742278+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":463,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.742145+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:39:18.742145+0000","last_peered":"2023-10-01T09:39:18.742145+0000","last_clean":"2023-10-01T09:39:18.742145+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:39:18.742145+0000","last_undegraded":"2023-10-01T09:39:18.742145+0000","last_fullsized":"2023-10-01T09:39:18.742145+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":503,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.729831+0000","last_change":"2023-10-01T09:36:22.227558+0000","last_active":"2023-10-01T09:39:18.729831+0000","last_peered":"2023-10-01T09:39:18.729831+0000","last_clean":"2023-10-01T09:39:18.729831+0000","last_became_active":"2023-10-01T09:36:22.226550+0000","last_became_peered":"2023-10-01T09:36:22.226550+0000","last_unstale":"2023-10-01T09:39:18.729831+0000","last_undegraded":"2023-10-01T09:39:18.729831+0000","last_fullsized":"2023-10-01T09:39:18.729831+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:35:45.988649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":384,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.740546+0000","last_change":"2023-10-01T09:37:22.672668+0000","last_active":"2023-10-01T09:39:18.740546+0000","last_peered":"2023-10-01T09:39:18.740546+0000","last_clean":"2023-10-01T09:39:18.740546+0000","last_became_active":"2023-10-01T09:37:22.671882+0000","last_became_peered":"2023-10-01T09:37:22.671882+0000","last_unstale":"2023-10-01T09:39:18.740546+0000","last_undegraded":"2023-10-01T09:39:18.740546+0000","last_fullsized":"2023-10-01T09:39:18.740546+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:42:45.864800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":148,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.725540+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:39:18.725540+0000","last_peered":"2023-10-01T09:39:18.725540+0000","last_clean":"2023-10-01T09:39:18.725540+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:39:18.725540+0000","last_undegraded":"2023-10-01T09:39:18.725540+0000","last_fullsized":"2023-10-01T09:39:18.725540+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":524,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.728646+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:39:18.728646+0000","last_peered":"2023-10-01T09:39:18.728646+0000","last_clean":"2023-10-01T09:39:18.728646+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:39:18.728646+0000","last_undegraded":"2023-10-01T09:39:18.728646+0000","last_fullsized":"2023-10-01T09:39:18.728646+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":486,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.742161+0000","last_change":"2023-10-01T09:37:22.667841+0000","last_active":"2023-10-01T09:39:18.742161+0000","last_peered":"2023-10-01T09:39:18.742161+0000","last_clean":"2023-10-01T09:39:18.742161+0000","last_became_active":"2023-10-01T09:37:22.667525+0000","last_became_peered":"2023-10-01T09:37:22.667525+0000","last_unstale":"2023-10-01T09:39:18.742161+0000","last_undegraded":"2023-10-01T09:39:18.742161+0000","last_fullsized":"2023-10-01T09:39:18.742161+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:03:37.695850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":159,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.725384+0000","last_change":"2023-10-01T09:36:22.254817+0000","last_active":"2023-10-01T09:39:18.725384+0000","last_peered":"2023-10-01T09:39:18.725384+0000","last_clean":"2023-10-01T09:39:18.725384+0000","last_became_active":"2023-10-01T09:36:22.254460+0000","last_became_peered":"2023-10-01T09:36:22.254460+0000","last_unstale":"2023-10-01T09:39:18.725384+0000","last_undegraded":"2023-10-01T09:39:18.725384+0000","last_fullsized":"2023-10-01T09:39:18.725384+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":62,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.724504+0000","last_change":"2023-10-01T09:37:22.680561+0000","last_active":"2023-10-01T09:39:18.724504+0000","last_peered":"2023-10-01T09:39:18.724504+0000","last_clean":"2023-10-01T09:39:18.724504+0000","last_became_active":"2023-10-01T09:37:22.680228+0000","last_became_peered":"2023-10-01T09:37:22.680228+0000","last_unstale":"2023-10-01T09:39:18.724504+0000","last_undegraded":"2023-10-01T09:39:18.724504+0000","last_fullsized":"2023-10-01T09:39:18.724504+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":62,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.727411+0000","last_change":"2023-10-01T09:39:12.700441+0000","last_active":"2023-10-01T09:39:18.727411+0000","last_peered":"2023-10-01T09:39:18.727411+0000","last_clean":"2023-10-01T09:39:18.727411+0000","last_became_active":"2023-10-01T09:39:12.700081+0000","last_became_peered":"2023-10-01T09:39:12.700081+0000","last_unstale":"2023-10-01T09:39:18.727411+0000","last_undegraded":"2023-10-01T09:39:18.727411+0000","last_fullsized":"2023-10-01T09:39:18.727411+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:38:34.987240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.729569+0000","last_change":"2023-10-01T09:36:22.223469+0000","last_active":"2023-10-01T09:39:18.729569+0000","last_peered":"2023-10-01T09:39:18.729569+0000","last_clean":"2023-10-01T09:39:18.729569+0000","last_became_active":"2023-10-01T09:36:22.222681+0000","last_became_peered":"2023-10-01T09:36:22.222681+0000","last_unstale":"2023-10-01T09:39:18.729569+0000","last_undegraded":"2023-10-01T09:39:18.729569+0000","last_fullsized":"2023-10-01T09:39:18.729569+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:19:28.367922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":492,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.737816+0000","last_change":"2023-10-01T09:32:27.498041+0000","last_active":"2023-10-01T09:39:18.737816+0000","last_peered":"2023-10-01T09:39:18.737816+0000","last_clean":"2023-10-01T09:39:18.737816+0000","last_became_active":"2023-10-01T09:32:27.454394+0000","last_became_peered":"2023-10-01T09:32:27.454394+0000","last_unstale":"2023-10-01T09:39:18.737816+0000","last_undegraded":"2023-10-01T09:39:18.737816+0000","last_fullsized":"2023-10-01T09:39:18.737816+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":307,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.742390+0000","last_change":"2023-10-01T09:35:23.610660+0000","last_active":"2023-10-01T09:39:18.742390+0000","last_peered":"2023-10-01T09:39:18.742390+0000","last_clean":"2023-10-01T09:39:18.742390+0000","last_became_active":"2023-10-01T09:35:23.610273+0000","last_became_peered":"2023-10-01T09:35:23.610273+0000","last_unstale":"2023-10-01T09:39:18.742390+0000","last_undegraded":"2023-10-01T09:39:18.742390+0000","last_fullsized":"2023-10-01T09:39:18.742390+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:43.996644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":568,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.737892+0000","last_change":"2023-10-01T09:35:34.640669+0000","last_active":"2023-10-01T09:39:18.737892+0000","last_peered":"2023-10-01T09:39:18.737892+0000","last_clean":"2023-10-01T09:39:18.737892+0000","last_became_active":"2023-10-01T09:35:34.552977+0000","last_became_peered":"2023-10-01T09:35:34.552977+0000","last_unstale":"2023-10-01T09:39:18.737892+0000","last_undegraded":"2023-10-01T09:39:18.737892+0000","last_fullsized":"2023-10-01T09:39:18.737892+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":306,"num_keys_recovered":90,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":302,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.737892+0000","last_change":"2023-10-01T09:37:22.681548+0000","last_active":"2023-10-01T09:39:18.737892+0000","last_peered":"2023-10-01T09:39:18.737892+0000","last_clean":"2023-10-01T09:39:18.737892+0000","last_became_active":"2023-10-01T09:37:22.681195+0000","last_became_peered":"2023-10-01T09:37:22.681195+0000","last_unstale":"2023-10-01T09:39:18.737892+0000","last_undegraded":"2023-10-01T09:39:18.737892+0000","last_fullsized":"2023-10-01T09:39:18.737892+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":468,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.729716+0000","last_change":"2023-10-01T09:36:22.223650+0000","last_active":"2023-10-01T09:39:18.729716+0000","last_peered":"2023-10-01T09:39:18.729716+0000","last_clean":"2023-10-01T09:39:18.729716+0000","last_became_active":"2023-10-01T09:36:22.223087+0000","last_became_peered":"2023-10-01T09:36:22.223087+0000","last_unstale":"2023-10-01T09:39:18.729716+0000","last_undegraded":"2023-10-01T09:39:18.729716+0000","last_fullsized":"2023-10-01T09:39:18.729716+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:52:21.962744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":283,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.738618+0000","last_change":"2023-10-01T09:37:22.670521+0000","last_active":"2023-10-01T09:39:18.738618+0000","last_peered":"2023-10-01T09:39:18.738618+0000","last_clean":"2023-10-01T09:39:18.738618+0000","last_became_active":"2023-10-01T09:37:22.670154+0000","last_became_peered":"2023-10-01T09:37:22.670154+0000","last_unstale":"2023-10-01T09:39:18.738618+0000","last_undegraded":"2023-10-01T09:39:18.738618+0000","last_fullsized":"2023-10-01T09:39:18.738618+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":476,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.737657+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:39:18.737657+0000","last_peered":"2023-10-01T09:39:18.737657+0000","last_clean":"2023-10-01T09:39:18.737657+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:39:18.737657+0000","last_undegraded":"2023-10-01T09:39:18.737657+0000","last_fullsized":"2023-10-01T09:39:18.737657+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":467,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.729899+0000","last_change":"2023-10-01T09:37:22.681587+0000","last_active":"2023-10-01T09:39:18.729899+0000","last_peered":"2023-10-01T09:39:18.729899+0000","last_clean":"2023-10-01T09:39:18.729899+0000","last_became_active":"2023-10-01T09:37:22.681290+0000","last_became_peered":"2023-10-01T09:37:22.681290+0000","last_unstale":"2023-10-01T09:39:18.729899+0000","last_undegraded":"2023-10-01T09:39:18.729899+0000","last_fullsized":"2023-10-01T09:39:18.729899+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:13:27.218691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":322,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.740418+0000","last_change":"2023-10-01T09:37:22.717074+0000","last_active":"2023-10-01T09:39:18.740418+0000","last_peered":"2023-10-01T09:39:18.740418+0000","last_clean":"2023-10-01T09:39:18.740418+0000","last_became_active":"2023-10-01T09:37:22.672350+0000","last_became_peered":"2023-10-01T09:37:22.672350+0000","last_unstale":"2023-10-01T09:39:18.740418+0000","last_undegraded":"2023-10-01T09:39:18.740418+0000","last_fullsized":"2023-10-01T09:39:18.740418+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:44:43.921813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":266,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.742689+0000","last_change":"2023-10-01T09:36:22.223054+0000","last_active":"2023-10-01T09:39:18.742689+0000","last_peered":"2023-10-01T09:39:18.742689+0000","last_clean":"2023-10-01T09:39:18.742689+0000","last_became_active":"2023-10-01T09:36:22.222776+0000","last_became_peered":"2023-10-01T09:36:22.222776+0000","last_unstale":"2023-10-01T09:39:18.742689+0000","last_undegraded":"2023-10-01T09:39:18.742689+0000","last_fullsized":"2023-10-01T09:39:18.742689+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:09:23.536588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"465'12857","reported_seq":20140,"reported_epoch":465,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:39:34.909363+0000","last_change":"2023-10-01T09:39:13.841369+0000","last_active":"2023-10-01T09:39:34.909363+0000","last_peered":"2023-10-01T09:39:34.909363+0000","last_clean":"2023-10-01T09:36:21.212087+0000","last_became_active":"2023-10-01T09:39:13.709334+0000","last_became_peered":"2023-10-01T09:39:13.709334+0000","last_unstale":"2023-10-01T09:39:34.909363+0000","last_undegraded":"2023-10-01T09:39:13.707150+0000","last_fullsized":"2023-10-01T09:39:13.704104+0000","mapping_epoch":462,"log_start":"423'5954","ondisk_log_start":"423'5954","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6903,"log_dups_size":0,"ondisk_log_size":6903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3473408,"num_objects":53,"num_object_clones":0,"num_object_copies":159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2858,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12857,"num_write_kb":413120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3595,"num_bytes_recovered":235536384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0],"avail_no_missing":["5","0","7"],"object_location_counts":[{"shards":"0,5","objects":-540},{"shards":"0,5,7","objects":53}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":139,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.731513+0000","last_change":"2023-10-01T09:36:22.387631+0000","last_active":"2023-10-01T09:39:18.731513+0000","last_peered":"2023-10-01T09:39:18.731513+0000","last_clean":"2023-10-01T09:39:18.731513+0000","last_became_active":"2023-10-01T09:36:22.237417+0000","last_became_peered":"2023-10-01T09:36:22.237417+0000","last_unstale":"2023-10-01T09:39:18.731513+0000","last_undegraded":"2023-10-01T09:39:18.731513+0000","last_fullsized":"2023-10-01T09:39:18.731513+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:06:04.026880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":176,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.724394+0000","last_change":"2023-10-01T09:36:22.236445+0000","last_active":"2023-10-01T09:39:18.724394+0000","last_peered":"2023-10-01T09:39:18.724394+0000","last_clean":"2023-10-01T09:39:18.724394+0000","last_became_active":"2023-10-01T09:36:22.236081+0000","last_became_peered":"2023-10-01T09:36:22.236081+0000","last_unstale":"2023-10-01T09:39:18.724394+0000","last_undegraded":"2023-10-01T09:39:18.724394+0000","last_fullsized":"2023-10-01T09:39:18.724394+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"465'12908","reported_seq":13009,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-01T09:39:34.982858+0000","last_change":"2023-10-01T09:37:22.701634+0000","last_active":"2023-10-01T09:39:34.982858+0000","last_peered":"2023-10-01T09:39:34.982858+0000","last_clean":"2023-10-01T09:39:34.982858+0000","last_became_active":"2023-10-01T09:37:22.671424+0000","last_became_peered":"2023-10-01T09:37:22.671424+0000","last_unstale":"2023-10-01T09:39:34.982858+0000","last_undegraded":"2023-10-01T09:39:34.982858+0000","last_fullsized":"2023-10-01T09:39:34.982858+0000","mapping_epoch":445,"log_start":"423'6034","ondisk_log_start":"423'6034","created":400,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6874,"log_dups_size":0,"ondisk_log_size":6874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:07:45.393494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3801088,"num_objects":58,"num_object_clones":0,"num_object_copies":174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12908,"num_write_kb":414912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":459,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.738632+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:39:18.738632+0000","last_peered":"2023-10-01T09:39:18.738632+0000","last_clean":"2023-10-01T09:39:18.738632+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:39:18.738632+0000","last_undegraded":"2023-10-01T09:39:18.738632+0000","last_fullsized":"2023-10-01T09:39:18.738632+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":509,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.742065+0000","last_change":"2023-10-01T09:36:22.223799+0000","last_active":"2023-10-01T09:39:18.742065+0000","last_peered":"2023-10-01T09:39:18.742065+0000","last_clean":"2023-10-01T09:39:18.742065+0000","last_became_active":"2023-10-01T09:36:22.223546+0000","last_became_peered":"2023-10-01T09:36:22.223546+0000","last_unstale":"2023-10-01T09:39:18.742065+0000","last_undegraded":"2023-10-01T09:39:18.742065+0000","last_fullsized":"2023-10-01T09:39:18.742065+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:54:07.013864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"465'12694","reported_seq":26965,"reported_epoch":465,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-01T09:39:34.979412+0000","last_change":"2023-10-01T09:37:06.460981+0000","last_active":"2023-10-01T09:39:34.979412+0000","last_peered":"2023-10-01T09:39:34.979412+0000","last_clean":"2023-10-01T09:36:21.192430+0000","last_became_active":"2023-10-01T09:36:23.791409+0000","last_became_peered":"2023-10-01T09:36:23.791409+0000","last_unstale":"2023-10-01T09:39:34.979412+0000","last_undegraded":"2023-10-01T09:39:34.979412+0000","last_fullsized":"2023-10-01T09:36:23.217545+0000","mapping_epoch":431,"log_start":"422'5807","ondisk_log_start":"422'5807","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6887,"log_dups_size":0,"ondisk_log_size":6887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4587520,"num_objects":70,"num_object_clones":0,"num_object_copies":210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":136,"num_objects_unfound":0,"num_objects_dirty":70,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12694,"num_write_kb":408448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4730,"num_bytes_recovered":309919744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,1],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"1,5","objects":2},{"shards":"3,5,7","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":545,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.738476+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:39:18.738476+0000","last_peered":"2023-10-01T09:39:18.738476+0000","last_clean":"2023-10-01T09:39:18.738476+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:39:18.738476+0000","last_undegraded":"2023-10-01T09:39:18.738476+0000","last_fullsized":"2023-10-01T09:39:18.738476+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":329,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.746003+0000","last_change":"2023-10-01T09:36:22.247103+0000","last_active":"2023-10-01T09:39:18.746003+0000","last_peered":"2023-10-01T09:39:18.746003+0000","last_clean":"2023-10-01T09:39:18.746003+0000","last_became_active":"2023-10-01T09:36:22.246793+0000","last_became_peered":"2023-10-01T09:36:22.246793+0000","last_unstale":"2023-10-01T09:39:18.746003+0000","last_undegraded":"2023-10-01T09:39:18.746003+0000","last_fullsized":"2023-10-01T09:39:18.746003+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:25:05.754730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.6","version":"465'12583","reported_seq":15438,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-01T09:39:34.970972+0000","last_change":"2023-10-01T09:36:28.672838+0000","last_active":"2023-10-01T09:39:34.970972+0000","last_peered":"2023-10-01T09:39:34.970972+0000","last_clean":"2023-10-01T09:39:34.970972+0000","last_became_active":"2023-10-01T09:36:28.606609+0000","last_became_peered":"2023-10-01T09:36:28.606609+0000","last_unstale":"2023-10-01T09:39:34.970972+0000","last_undegraded":"2023-10-01T09:39:34.970972+0000","last_fullsized":"2023-10-01T09:39:34.970972+0000","mapping_epoch":433,"log_start":"420'5698","ondisk_log_start":"420'5698","created":400,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6885,"log_dups_size":0,"ondisk_log_size":6885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3604480,"num_objects":55,"num_object_clones":0,"num_object_copies":165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":55,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12583,"num_write_kb":404416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1377,"num_bytes_recovered":90243072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":185,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.725427+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:39:18.725427+0000","last_peered":"2023-10-01T09:39:18.725427+0000","last_clean":"2023-10-01T09:39:18.725427+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:39:18.725427+0000","last_undegraded":"2023-10-01T09:39:18.725427+0000","last_fullsized":"2023-10-01T09:39:18.725427+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":441,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.729623+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:39:18.729623+0000","last_peered":"2023-10-01T09:39:18.729623+0000","last_clean":"2023-10-01T09:39:18.729623+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:39:18.729623+0000","last_undegraded":"2023-10-01T09:39:18.729623+0000","last_fullsized":"2023-10-01T09:39:18.729623+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"465'12386","reported_seq":21450,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-01T09:39:32.226571+0000","last_change":"2023-10-01T09:37:22.694633+0000","last_active":"2023-10-01T09:39:32.226571+0000","last_peered":"2023-10-01T09:39:32.226571+0000","last_clean":"2023-10-01T09:39:32.226571+0000","last_became_active":"2023-10-01T09:37:22.668427+0000","last_became_peered":"2023-10-01T09:37:22.668427+0000","last_unstale":"2023-10-01T09:39:32.226571+0000","last_undegraded":"2023-10-01T09:39:32.226571+0000","last_fullsized":"2023-10-01T09:39:32.226571+0000","mapping_epoch":445,"log_start":"411'4479","ondisk_log_start":"411'4479","created":400,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7907,"log_dups_size":0,"ondisk_log_size":7907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:50:23.248095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8519680,"num_objects":130,"num_object_clones":0,"num_object_copies":390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12386,"num_write_kb":400512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4478,"num_bytes_recovered":293339136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":176,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.727632+0000","last_change":"2023-10-01T09:39:12.697497+0000","last_active":"2023-10-01T09:39:18.727632+0000","last_peered":"2023-10-01T09:39:18.727632+0000","last_clean":"2023-10-01T09:39:18.727632+0000","last_became_active":"2023-10-01T09:39:12.697156+0000","last_became_peered":"2023-10-01T09:39:12.697156+0000","last_unstale":"2023-10-01T09:39:18.727632+0000","last_undegraded":"2023-10-01T09:39:18.727632+0000","last_fullsized":"2023-10-01T09:39:18.727632+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:35.794627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":119,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.722634+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:39:18.722634+0000","last_peered":"2023-10-01T09:39:18.722634+0000","last_clean":"2023-10-01T09:39:18.722634+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:39:18.722634+0000","last_undegraded":"2023-10-01T09:39:18.722634+0000","last_fullsized":"2023-10-01T09:39:18.722634+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"465'12556","reported_seq":16380,"reported_epoch":465,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:39:32.156119+0000","last_change":"2023-10-01T09:39:13.842949+0000","last_active":"2023-10-01T09:39:32.156119+0000","last_peered":"2023-10-01T09:39:32.156119+0000","last_clean":"2023-10-01T09:35:09.959309+0000","last_became_active":"2023-10-01T09:39:13.711903+0000","last_became_peered":"2023-10-01T09:39:13.711903+0000","last_unstale":"2023-10-01T09:39:32.156119+0000","last_undegraded":"2023-10-01T09:39:13.703337+0000","last_fullsized":"2023-10-01T09:39:32.156119+0000","mapping_epoch":462,"log_start":"416'5114","ondisk_log_start":"416'5114","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7442,"log_dups_size":0,"ondisk_log_size":7442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8912896,"num_objects":136,"num_object_clones":0,"num_object_copies":408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4492,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12556,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1855,"num_bytes_recovered":121503744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":136}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":385,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.723301+0000","last_change":"2023-10-01T09:37:22.684537+0000","last_active":"2023-10-01T09:39:18.723301+0000","last_peered":"2023-10-01T09:39:18.723301+0000","last_clean":"2023-10-01T09:39:18.723301+0000","last_became_active":"2023-10-01T09:37:22.683692+0000","last_became_peered":"2023-10-01T09:37:22.683692+0000","last_unstale":"2023-10-01T09:39:18.723301+0000","last_undegraded":"2023-10-01T09:39:18.723301+0000","last_fullsized":"2023-10-01T09:39:18.723301+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":512,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.742384+0000","last_change":"2023-10-01T09:35:34.558802+0000","last_active":"2023-10-01T09:39:18.742384+0000","last_peered":"2023-10-01T09:39:18.742384+0000","last_clean":"2023-10-01T09:39:18.742384+0000","last_became_active":"2023-10-01T09:35:34.558511+0000","last_became_peered":"2023-10-01T09:35:34.558511+0000","last_unstale":"2023-10-01T09:39:18.742384+0000","last_undegraded":"2023-10-01T09:39:18.742384+0000","last_fullsized":"2023-10-01T09:39:18.742384+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:40:52.864568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"465'12617","reported_seq":13397,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-01T09:39:31.242313+0000","last_change":"2023-10-01T09:37:23.700088+0000","last_active":"2023-10-01T09:39:31.242313+0000","last_peered":"2023-10-01T09:39:31.242313+0000","last_clean":"2023-10-01T09:39:31.242313+0000","last_became_active":"2023-10-01T09:37:23.635822+0000","last_became_peered":"2023-10-01T09:37:23.635822+0000","last_unstale":"2023-10-01T09:39:31.242313+0000","last_undegraded":"2023-10-01T09:39:31.242313+0000","last_fullsized":"2023-10-01T09:39:31.242313+0000","mapping_epoch":446,"log_start":"407'2600","ondisk_log_start":"407'2600","created":400,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":10017,"log_dups_size":0,"ondisk_log_size":10017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10158080,"num_objects":155,"num_object_clones":0,"num_object_copies":465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12617,"num_write_kb":408704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":343,"num_bytes_recovered":22478848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":355,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.737750+0000","last_change":"2023-10-01T09:35:34.551196+0000","last_active":"2023-10-01T09:39:18.737750+0000","last_peered":"2023-10-01T09:39:18.737750+0000","last_clean":"2023-10-01T09:39:18.737750+0000","last_became_active":"2023-10-01T09:35:34.550803+0000","last_became_peered":"2023-10-01T09:35:34.550803+0000","last_unstale":"2023-10-01T09:39:18.737750+0000","last_undegraded":"2023-10-01T09:39:18.737750+0000","last_fullsized":"2023-10-01T09:39:18.737750+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":191,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.721751+0000","last_change":"2023-10-01T09:37:22.681270+0000","last_active":"2023-10-01T09:39:18.721751+0000","last_peered":"2023-10-01T09:39:18.721751+0000","last_clean":"2023-10-01T09:39:18.721751+0000","last_became_active":"2023-10-01T09:37:22.680951+0000","last_became_peered":"2023-10-01T09:37:22.680951+0000","last_unstale":"2023-10-01T09:39:18.721751+0000","last_undegraded":"2023-10-01T09:39:18.721751+0000","last_fullsized":"2023-10-01T09:39:18.721751+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"465'12862","reported_seq":15748,"reported_epoch":465,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:39:34.982176+0000","last_change":"2023-10-01T09:36:23.771443+0000","last_active":"2023-10-01T09:39:34.982176+0000","last_peered":"2023-10-01T09:39:34.982176+0000","last_clean":"2023-10-01T09:35:09.934020+0000","last_became_active":"2023-10-01T09:36:23.749995+0000","last_became_peered":"2023-10-01T09:36:23.749995+0000","last_unstale":"2023-10-01T09:39:34.982176+0000","last_undegraded":"2023-10-01T09:36:23.566811+0000","last_fullsized":"2023-10-01T09:36:23.220185+0000","mapping_epoch":431,"log_start":"423'5992","ondisk_log_start":"423'5992","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6870,"log_dups_size":0,"ondisk_log_size":6870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:23:09.158739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3276800,"num_objects":50,"num_object_clones":0,"num_object_copies":150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5535,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":50,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12862,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1394,"num_bytes_recovered":91160576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2","objects":-3919},{"shards":"1,2,3","objects":50}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":317,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:25.003553+0000","last_change":"2023-10-01T09:36:22.223384+0000","last_active":"2023-10-01T09:39:25.003553+0000","last_peered":"2023-10-01T09:39:25.003553+0000","last_clean":"2023-10-01T09:39:25.003553+0000","last_became_active":"2023-10-01T09:36:22.223098+0000","last_became_peered":"2023-10-01T09:36:22.223098+0000","last_unstale":"2023-10-01T09:39:25.003553+0000","last_undegraded":"2023-10-01T09:39:25.003553+0000","last_fullsized":"2023-10-01T09:39:25.003553+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:15:40.489055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":373,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.742723+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:39:18.742723+0000","last_peered":"2023-10-01T09:39:18.742723+0000","last_clean":"2023-10-01T09:39:18.742723+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:39:18.742723+0000","last_undegraded":"2023-10-01T09:39:18.742723+0000","last_fullsized":"2023-10-01T09:39:18.742723+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"465'12779","reported_seq":12865,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-01T09:39:35.005246+0000","last_change":"2023-10-01T09:37:22.715127+0000","last_active":"2023-10-01T09:39:35.005246+0000","last_peered":"2023-10-01T09:39:35.005246+0000","last_clean":"2023-10-01T09:39:35.005246+0000","last_became_active":"2023-10-01T09:37:22.680102+0000","last_became_peered":"2023-10-01T09:37:22.680102+0000","last_unstale":"2023-10-01T09:39:35.005246+0000","last_undegraded":"2023-10-01T09:39:35.005246+0000","last_fullsized":"2023-10-01T09:39:35.005246+0000","mapping_epoch":445,"log_start":"422'5862","ondisk_log_start":"422'5862","created":400,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6917,"log_dups_size":0,"ondisk_log_size":6917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3997696,"num_objects":61,"num_object_clones":0,"num_object_copies":183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":61,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12779,"num_write_kb":410880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":1290,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:19.892377+0000","last_change":"2023-10-01T09:12:09.997929+0000","last_active":"2023-10-01T09:39:19.892377+0000","last_peered":"2023-10-01T09:39:19.892377+0000","last_clean":"2023-10-01T09:39:19.892377+0000","last_became_active":"2023-10-01T09:12:09.997287+0000","last_became_peered":"2023-10-01T09:12:09.997287+0000","last_unstale":"2023-10-01T09:39:19.892377+0000","last_undegraded":"2023-10-01T09:39:19.892377+0000","last_fullsized":"2023-10-01T09:39:19.892377+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:07.551911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":186,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.726284+0000","last_change":"2023-10-01T09:37:22.667746+0000","last_active":"2023-10-01T09:39:18.726284+0000","last_peered":"2023-10-01T09:39:18.726284+0000","last_clean":"2023-10-01T09:39:18.726284+0000","last_became_active":"2023-10-01T09:37:22.667475+0000","last_became_peered":"2023-10-01T09:37:22.667475+0000","last_unstale":"2023-10-01T09:39:18.726284+0000","last_undegraded":"2023-10-01T09:39:18.726284+0000","last_fullsized":"2023-10-01T09:39:18.726284+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":420,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.724922+0000","last_change":"2023-10-01T09:36:22.236067+0000","last_active":"2023-10-01T09:39:18.724922+0000","last_peered":"2023-10-01T09:39:18.724922+0000","last_clean":"2023-10-01T09:39:18.724922+0000","last_became_active":"2023-10-01T09:36:22.235732+0000","last_became_peered":"2023-10-01T09:36:22.235732+0000","last_unstale":"2023-10-01T09:39:18.724922+0000","last_undegraded":"2023-10-01T09:39:18.724922+0000","last_fullsized":"2023-10-01T09:39:18.724922+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"465'12684","reported_seq":12768,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-01T09:39:31.197463+0000","last_change":"2023-10-01T09:39:12.758430+0000","last_active":"2023-10-01T09:39:31.197463+0000","last_peered":"2023-10-01T09:39:31.197463+0000","last_clean":"2023-10-01T09:39:31.197463+0000","last_became_active":"2023-10-01T09:39:12.701821+0000","last_became_peered":"2023-10-01T09:39:12.701821+0000","last_unstale":"2023-10-01T09:39:31.197463+0000","last_undegraded":"2023-10-01T09:39:31.197463+0000","last_fullsized":"2023-10-01T09:39:31.197463+0000","mapping_epoch":461,"log_start":"411'4578","ondisk_log_start":"411'4578","created":400,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":8106,"log_dups_size":0,"ondisk_log_size":8106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11403264,"num_objects":174,"num_object_clones":0,"num_object_copies":522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12684,"num_write_kb":411456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1130,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.724766+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:39:18.724766+0000","last_peered":"2023-10-01T09:39:18.724766+0000","last_clean":"2023-10-01T09:39:18.724766+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:39:18.724766+0000","last_undegraded":"2023-10-01T09:39:18.724766+0000","last_fullsized":"2023-10-01T09:39:18.724766+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":340,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.721360+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:39:18.721360+0000","last_peered":"2023-10-01T09:39:18.721360+0000","last_clean":"2023-10-01T09:39:18.721360+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:39:18.721360+0000","last_undegraded":"2023-10-01T09:39:18.721360+0000","last_fullsized":"2023-10-01T09:39:18.721360+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":153,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.725362+0000","last_change":"2023-10-01T09:38:07.318624+0000","last_active":"2023-10-01T09:39:18.725362+0000","last_peered":"2023-10-01T09:39:18.725362+0000","last_clean":"2023-10-01T09:39:18.725362+0000","last_became_active":"2023-10-01T09:38:07.318266+0000","last_became_peered":"2023-10-01T09:38:07.318266+0000","last_unstale":"2023-10-01T09:39:18.725362+0000","last_undegraded":"2023-10-01T09:39:18.725362+0000","last_fullsized":"2023-10-01T09:39:18.725362+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"465'12504","reported_seq":20166,"reported_epoch":465,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-10-01T09:39:35.003539+0000","last_change":"2023-10-01T09:38:08.682352+0000","last_active":"2023-10-01T09:39:35.003539+0000","last_peered":"2023-10-01T09:39:35.003539+0000","last_clean":"2023-10-01T09:36:21.153294+0000","last_became_active":"2023-10-01T09:38:08.590431+0000","last_became_peered":"2023-10-01T09:38:08.590431+0000","last_unstale":"2023-10-01T09:39:35.003539+0000","last_undegraded":"2023-10-01T09:38:08.584718+0000","last_fullsized":"2023-10-01T09:39:35.003539+0000","mapping_epoch":453,"log_start":"420'5607","ondisk_log_start":"420'5607","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6897,"log_dups_size":0,"ondisk_log_size":6897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:06:34.257665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3407872,"num_objects":52,"num_object_clones":0,"num_object_copies":156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":52,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":52,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12504,"num_write_kb":401792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3777,"num_bytes_recovered":247529472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,3],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":52}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":746,"reported_epoch":464,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:39:18.732589+0000","last_change":"2023-10-01T09:38:07.327355+0000","last_active":"2023-10-01T09:39:18.732589+0000","last_peered":"2023-10-01T09:39:18.732589+0000","last_clean":"2023-10-01T09:37:58.763801+0000","last_became_active":"2023-10-01T09:38:07.326767+0000","last_became_peered":"2023-10-01T09:38:07.326767+0000","last_unstale":"2023-10-01T09:39:18.732589+0000","last_undegraded":"2023-10-01T09:38:07.315924+0000","last_fullsized":"2023-10-01T09:39:18.732589+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:45:16.157432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":["0","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":516,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.742102+0000","last_change":"2023-10-01T09:35:34.562777+0000","last_active":"2023-10-01T09:39:18.742102+0000","last_peered":"2023-10-01T09:39:18.742102+0000","last_clean":"2023-10-01T09:39:18.742102+0000","last_became_active":"2023-10-01T09:35:34.562492+0000","last_became_peered":"2023-10-01T09:35:34.562492+0000","last_unstale":"2023-10-01T09:39:18.742102+0000","last_undegraded":"2023-10-01T09:39:18.742102+0000","last_fullsized":"2023-10-01T09:39:18.742102+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:26:15.751778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":334,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.728733+0000","last_change":"2023-10-01T09:37:22.673957+0000","last_active":"2023-10-01T09:39:18.728733+0000","last_peered":"2023-10-01T09:39:18.728733+0000","last_clean":"2023-10-01T09:39:18.728733+0000","last_became_active":"2023-10-01T09:37:22.673685+0000","last_became_peered":"2023-10-01T09:37:22.673685+0000","last_unstale":"2023-10-01T09:39:18.728733+0000","last_undegraded":"2023-10-01T09:39:18.728733+0000","last_fullsized":"2023-10-01T09:39:18.728733+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"465'12680","reported_seq":27380,"reported_epoch":465,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-01T09:39:34.708888+0000","last_change":"2023-10-01T09:39:34.718779+0000","last_active":"2023-10-01T09:39:34.708888+0000","last_peered":"2023-10-01T09:39:34.708888+0000","last_clean":"2023-10-01T09:36:21.170373+0000","last_became_active":"2023-10-01T09:36:23.812646+0000","last_became_peered":"2023-10-01T09:36:23.812646+0000","last_unstale":"2023-10-01T09:39:34.708888+0000","last_undegraded":"2023-10-01T09:39:34.708628+0000","last_fullsized":"2023-10-01T09:36:23.211397+0000","mapping_epoch":431,"log_start":"416'5157","ondisk_log_start":"416'5157","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7523,"log_dups_size":0,"ondisk_log_size":7523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:34:58.841399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4718592,"num_objects":72,"num_object_clones":0,"num_object_copies":216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":72,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12680,"num_write_kb":408064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4871,"num_bytes_recovered":317521920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,1],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,1","objects":72}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1461,"reported_epoch":464,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:39:18.724289+0000","last_change":"2023-10-01T09:37:23.636269+0000","last_active":"2023-10-01T09:39:18.724289+0000","last_peered":"2023-10-01T09:39:18.724289+0000","last_clean":"2023-10-01T09:35:27.946455+0000","last_became_active":"2023-10-01T09:37:23.635623+0000","last_became_peered":"2023-10-01T09:37:23.635623+0000","last_unstale":"2023-10-01T09:39:18.724289+0000","last_undegraded":"2023-10-01T09:37:23.631750+0000","last_fullsized":"2023-10-01T09:39:18.724289+0000","mapping_epoch":446,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":485,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.740304+0000","last_change":"2023-10-01T09:36:29.654983+0000","last_active":"2023-10-01T09:39:18.740304+0000","last_peered":"2023-10-01T09:39:18.740304+0000","last_clean":"2023-10-01T09:39:18.740304+0000","last_became_active":"2023-10-01T09:36:29.650683+0000","last_became_peered":"2023-10-01T09:36:29.650683+0000","last_unstale":"2023-10-01T09:39:18.740304+0000","last_undegraded":"2023-10-01T09:39:18.740304+0000","last_fullsized":"2023-10-01T09:39:18.740304+0000","mapping_epoch":434,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:23.591743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":264,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.724677+0000","last_change":"2023-10-01T09:38:07.494808+0000","last_active":"2023-10-01T09:39:18.724677+0000","last_peered":"2023-10-01T09:39:18.724677+0000","last_clean":"2023-10-01T09:39:18.724677+0000","last_became_active":"2023-10-01T09:38:07.494419+0000","last_became_peered":"2023-10-01T09:38:07.494419+0000","last_unstale":"2023-10-01T09:39:18.724677+0000","last_undegraded":"2023-10-01T09:39:18.724677+0000","last_fullsized":"2023-10-01T09:39:18.724677+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":379,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.738563+0000","last_change":"2023-10-01T09:32:27.453291+0000","last_active":"2023-10-01T09:39:18.738563+0000","last_peered":"2023-10-01T09:39:18.738563+0000","last_clean":"2023-10-01T09:39:18.738563+0000","last_became_active":"2023-10-01T09:32:27.453026+0000","last_became_peered":"2023-10-01T09:32:27.453026+0000","last_unstale":"2023-10-01T09:39:18.738563+0000","last_undegraded":"2023-10-01T09:39:18.738563+0000","last_fullsized":"2023-10-01T09:39:18.738563+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.b","version":"465'12678","reported_seq":12753,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-01T09:39:34.709266+0000","last_change":"2023-10-01T09:34:06.140844+0000","last_active":"2023-10-01T09:39:34.709266+0000","last_peered":"2023-10-01T09:39:34.709266+0000","last_clean":"2023-10-01T09:39:34.709266+0000","last_became_active":"2023-10-01T09:34:06.139812+0000","last_became_peered":"2023-10-01T09:34:06.139812+0000","last_unstale":"2023-10-01T09:39:34.709266+0000","last_undegraded":"2023-10-01T09:39:34.709266+0000","last_fullsized":"2023-10-01T09:39:34.709266+0000","mapping_epoch":400,"log_start":"417'5167","ondisk_log_start":"417'5167","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":7511,"log_dups_size":0,"ondisk_log_size":7511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:31:05.105491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5242880,"num_objects":80,"num_object_clones":0,"num_object_copies":240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12678,"num_write_kb":408256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":645,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.737506+0000","last_change":"2023-10-01T09:37:22.770678+0000","last_active":"2023-10-01T09:39:18.737506+0000","last_peered":"2023-10-01T09:39:18.737506+0000","last_clean":"2023-10-01T09:39:18.737506+0000","last_became_active":"2023-10-01T09:37:22.674516+0000","last_became_peered":"2023-10-01T09:37:22.674516+0000","last_unstale":"2023-10-01T09:39:18.737506+0000","last_undegraded":"2023-10-01T09:39:18.737506+0000","last_fullsized":"2023-10-01T09:39:18.737506+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":493,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.732779+0000","last_change":"2023-10-01T09:36:22.225663+0000","last_active":"2023-10-01T09:39:18.732779+0000","last_peered":"2023-10-01T09:39:18.732779+0000","last_clean":"2023-10-01T09:39:18.732779+0000","last_became_active":"2023-10-01T09:36:22.225382+0000","last_became_peered":"2023-10-01T09:36:22.225382+0000","last_unstale":"2023-10-01T09:39:18.732779+0000","last_undegraded":"2023-10-01T09:39:18.732779+0000","last_fullsized":"2023-10-01T09:39:18.732779+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:19:41.386221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":493,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.729669+0000","last_change":"2023-10-01T09:35:34.551948+0000","last_active":"2023-10-01T09:39:18.729669+0000","last_peered":"2023-10-01T09:39:18.729669+0000","last_clean":"2023-10-01T09:39:18.729669+0000","last_became_active":"2023-10-01T09:35:34.551665+0000","last_became_peered":"2023-10-01T09:35:34.551665+0000","last_unstale":"2023-10-01T09:39:18.729669+0000","last_undegraded":"2023-10-01T09:39:18.729669+0000","last_fullsized":"2023-10-01T09:39:18.729669+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:31:30.655381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"465'12958","reported_seq":13078,"reported_epoch":465,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:39:35.011664+0000","last_change":"2023-10-01T09:36:23.705972+0000","last_active":"2023-10-01T09:39:35.011664+0000","last_peered":"2023-10-01T09:39:35.011664+0000","last_clean":"2023-10-01T09:36:21.165407+0000","last_became_active":"2023-10-01T09:36:23.529481+0000","last_became_peered":"2023-10-01T09:36:23.529481+0000","last_unstale":"2023-10-01T09:39:35.011664+0000","last_undegraded":"2023-10-01T09:39:35.011664+0000","last_fullsized":"2023-10-01T09:39:35.011664+0000","mapping_epoch":431,"log_start":"423'6044","ondisk_log_start":"423'6044","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6914,"log_dups_size":0,"ondisk_log_size":6914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:01:56.211865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4325376,"num_objects":66,"num_object_clones":0,"num_object_copies":198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":66,"num_objects_unfound":0,"num_objects_dirty":66,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12958,"num_write_kb":416768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":66}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":903,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.738529+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:39:18.738529+0000","last_peered":"2023-10-01T09:39:18.738529+0000","last_clean":"2023-10-01T09:39:18.738529+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:39:18.738529+0000","last_undegraded":"2023-10-01T09:39:18.738529+0000","last_fullsized":"2023-10-01T09:39:18.738529+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":522,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:19.892398+0000","last_change":"2023-10-01T09:35:34.559426+0000","last_active":"2023-10-01T09:39:19.892398+0000","last_peered":"2023-10-01T09:39:19.892398+0000","last_clean":"2023-10-01T09:39:19.892398+0000","last_became_active":"2023-10-01T09:35:34.558890+0000","last_became_peered":"2023-10-01T09:35:34.558890+0000","last_unstale":"2023-10-01T09:39:19.892398+0000","last_undegraded":"2023-10-01T09:39:19.892398+0000","last_fullsized":"2023-10-01T09:39:19.892398+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:55:37.464032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":375,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.724877+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:39:18.724877+0000","last_peered":"2023-10-01T09:39:18.724877+0000","last_clean":"2023-10-01T09:39:18.724877+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:39:18.724877+0000","last_undegraded":"2023-10-01T09:39:18.724877+0000","last_fullsized":"2023-10-01T09:39:18.724877+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"465'12735","reported_seq":12815,"reported_epoch":465,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T09:39:35.003648+0000","last_change":"2023-10-01T09:36:23.622968+0000","last_active":"2023-10-01T09:39:35.003648+0000","last_peered":"2023-10-01T09:39:35.003648+0000","last_clean":"2023-10-01T09:36:21.149981+0000","last_became_active":"2023-10-01T09:36:23.603244+0000","last_became_peered":"2023-10-01T09:36:23.603244+0000","last_unstale":"2023-10-01T09:39:35.003648+0000","last_undegraded":"2023-10-01T09:39:35.003648+0000","last_fullsized":"2023-10-01T09:39:35.003648+0000","mapping_epoch":431,"log_start":"423'5905","ondisk_log_start":"423'5905","created":400,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":6830,"log_dups_size":0,"ondisk_log_size":6830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3932184,"num_objects":61,"num_object_clones":0,"num_object_copies":183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":122,"num_objects_unfound":0,"num_objects_dirty":61,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12735,"num_write_kb":409409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,1,7],"avail_no_missing":["5","1","7"],"object_location_counts":[{"shards":"1,5,7","objects":61}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":404,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.738593+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:39:18.738593+0000","last_peered":"2023-10-01T09:39:18.738593+0000","last_clean":"2023-10-01T09:39:18.738593+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:39:18.738593+0000","last_undegraded":"2023-10-01T09:39:18.738593+0000","last_fullsized":"2023-10-01T09:39:18.738593+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1095,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.742633+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:39:18.742633+0000","last_peered":"2023-10-01T09:39:18.742633+0000","last_clean":"2023-10-01T09:39:18.742633+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:39:18.742633+0000","last_undegraded":"2023-10-01T09:39:18.742633+0000","last_fullsized":"2023-10-01T09:39:18.742633+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":157,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.741554+0000","last_change":"2023-10-01T09:39:12.722198+0000","last_active":"2023-10-01T09:39:18.741554+0000","last_peered":"2023-10-01T09:39:18.741554+0000","last_clean":"2023-10-01T09:39:18.741554+0000","last_became_active":"2023-10-01T09:39:12.721845+0000","last_became_peered":"2023-10-01T09:39:12.721845+0000","last_unstale":"2023-10-01T09:39:18.741554+0000","last_undegraded":"2023-10-01T09:39:18.741554+0000","last_fullsized":"2023-10-01T09:39:18.741554+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:33:18.055981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.c","version":"465'12854","reported_seq":12945,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-10-01T09:39:31.205472+0000","last_change":"2023-10-01T09:37:23.665454+0000","last_active":"2023-10-01T09:39:31.205472+0000","last_peered":"2023-10-01T09:39:31.205472+0000","last_clean":"2023-10-01T09:39:31.205472+0000","last_became_active":"2023-10-01T09:37:23.635951+0000","last_became_peered":"2023-10-01T09:37:23.635951+0000","last_unstale":"2023-10-01T09:39:31.205472+0000","last_undegraded":"2023-10-01T09:39:31.205472+0000","last_fullsized":"2023-10-01T09:39:31.205472+0000","mapping_epoch":446,"log_start":"408'2800","ondisk_log_start":"408'2800","created":400,"last_epoch_clean":447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:34:05.106756+0000","last_clean_scrub_stamp":"2023-10-01T09:34:05.106756+0000","objects_scrubbed":0,"log_size":10054,"log_dups_size":0,"ondisk_log_size":10054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10354688,"num_objects":158,"num_object_clones":0,"num_object_copies":474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12854,"num_write_kb":416384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":225,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.723411+0000","last_change":"2023-10-01T09:37:22.671608+0000","last_active":"2023-10-01T09:39:18.723411+0000","last_peered":"2023-10-01T09:39:18.723411+0000","last_clean":"2023-10-01T09:39:18.723411+0000","last_became_active":"2023-10-01T09:37:22.671097+0000","last_became_peered":"2023-10-01T09:37:22.671097+0000","last_unstale":"2023-10-01T09:39:18.723411+0000","last_undegraded":"2023-10-01T09:39:18.723411+0000","last_fullsized":"2023-10-01T09:39:18.723411+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":480,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.729755+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:39:18.729755+0000","last_peered":"2023-10-01T09:39:18.729755+0000","last_clean":"2023-10-01T09:39:18.729755+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:39:18.729755+0000","last_undegraded":"2023-10-01T09:39:18.729755+0000","last_fullsized":"2023-10-01T09:39:18.729755+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":188,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.724578+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:39:18.724578+0000","last_peered":"2023-10-01T09:39:18.724578+0000","last_clean":"2023-10-01T09:39:18.724578+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:39:18.724578+0000","last_undegraded":"2023-10-01T09:39:18.724578+0000","last_fullsized":"2023-10-01T09:39:18.724578+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":429,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.729769+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:39:18.729769+0000","last_peered":"2023-10-01T09:39:18.729769+0000","last_clean":"2023-10-01T09:39:18.729769+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:39:18.729769+0000","last_undegraded":"2023-10-01T09:39:18.729769+0000","last_fullsized":"2023-10-01T09:39:18.729769+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":534,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.743903+0000","last_change":"2023-10-01T09:37:22.674112+0000","last_active":"2023-10-01T09:39:18.743903+0000","last_peered":"2023-10-01T09:39:18.743903+0000","last_clean":"2023-10-01T09:39:18.743903+0000","last_became_active":"2023-10-01T09:37:22.673785+0000","last_became_peered":"2023-10-01T09:37:22.673785+0000","last_unstale":"2023-10-01T09:39:18.743903+0000","last_undegraded":"2023-10-01T09:39:18.743903+0000","last_fullsized":"2023-10-01T09:39:18.743903+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:05.627976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":200,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.725162+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:39:18.725162+0000","last_peered":"2023-10-01T09:39:18.725162+0000","last_clean":"2023-10-01T09:39:18.725162+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:39:18.725162+0000","last_undegraded":"2023-10-01T09:39:18.725162+0000","last_fullsized":"2023-10-01T09:39:18.725162+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":263,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.721264+0000","last_change":"2023-10-01T09:37:22.671305+0000","last_active":"2023-10-01T09:39:18.721264+0000","last_peered":"2023-10-01T09:39:18.721264+0000","last_clean":"2023-10-01T09:39:18.721264+0000","last_became_active":"2023-10-01T09:37:22.670844+0000","last_became_peered":"2023-10-01T09:37:22.670844+0000","last_unstale":"2023-10-01T09:39:18.721264+0000","last_undegraded":"2023-10-01T09:39:18.721264+0000","last_fullsized":"2023-10-01T09:39:18.721264+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":200,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.726380+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:39:18.726380+0000","last_peered":"2023-10-01T09:39:18.726380+0000","last_clean":"2023-10-01T09:39:18.726380+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:39:18.726380+0000","last_undegraded":"2023-10-01T09:39:18.726380+0000","last_fullsized":"2023-10-01T09:39:18.726380+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":375,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.729637+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:39:18.729637+0000","last_peered":"2023-10-01T09:39:18.729637+0000","last_clean":"2023-10-01T09:39:18.729637+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:39:18.729637+0000","last_undegraded":"2023-10-01T09:39:18.729637+0000","last_fullsized":"2023-10-01T09:39:18.729637+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":31,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:27.390812+0000","last_change":"2023-10-01T09:37:22.673935+0000","last_active":"2023-10-01T09:39:27.390812+0000","last_peered":"2023-10-01T09:39:27.390812+0000","last_clean":"2023-10-01T09:39:27.390812+0000","last_became_active":"2023-10-01T09:37:22.673609+0000","last_became_peered":"2023-10-01T09:37:22.673609+0000","last_unstale":"2023-10-01T09:39:27.390812+0000","last_undegraded":"2023-10-01T09:39:27.390812+0000","last_fullsized":"2023-10-01T09:39:27.390812+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:02:43.715678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":411,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.732761+0000","last_change":"2023-10-01T09:37:22.668216+0000","last_active":"2023-10-01T09:39:18.732761+0000","last_peered":"2023-10-01T09:39:18.732761+0000","last_clean":"2023-10-01T09:39:18.732761+0000","last_became_active":"2023-10-01T09:37:22.667243+0000","last_became_peered":"2023-10-01T09:37:22.667243+0000","last_unstale":"2023-10-01T09:39:18.732761+0000","last_undegraded":"2023-10-01T09:39:18.732761+0000","last_fullsized":"2023-10-01T09:39:18.732761+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:45:16.527071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":604,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.731696+0000","last_change":"2023-10-01T09:35:34.658812+0000","last_active":"2023-10-01T09:39:18.731696+0000","last_peered":"2023-10-01T09:39:18.731696+0000","last_clean":"2023-10-01T09:39:18.731696+0000","last_became_active":"2023-10-01T09:35:34.550952+0000","last_became_peered":"2023-10-01T09:35:34.550952+0000","last_unstale":"2023-10-01T09:39:18.731696+0000","last_undegraded":"2023-10-01T09:39:18.731696+0000","last_fullsized":"2023-10-01T09:39:18.731696+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:24:17.304117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":435,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-10-01T09:39:18.742015+0000","last_change":"2023-10-01T09:32:27.445299+0000","last_active":"2023-10-01T09:39:18.742015+0000","last_peered":"2023-10-01T09:39:18.742015+0000","last_clean":"2023-10-01T09:39:18.742015+0000","last_became_active":"2023-10-01T09:32:27.444956+0000","last_became_peered":"2023-10-01T09:32:27.444956+0000","last_unstale":"2023-10-01T09:39:18.742015+0000","last_undegraded":"2023-10-01T09:39:18.742015+0000","last_fullsized":"2023-10-01T09:39:18.742015+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:49:07.862356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":34430,"num_keys_recovered":90,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":93716504,"num_objects":1431,"num_object_clones":0,"num_object_copies":4293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12937,"num_objects_misplaced":324,"num_objects_unfound":0,"num_objects_dirty":1431,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":203335,"num_write_kb":6552449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26437,"num_bytes_recovered":1730150400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2551525376,"data_stored":2551513160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120537,"ondisk_log_size":120537,"up":48,"acting":44,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":297,"num_bytes_recovered":12582969,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430667,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1632012,"kb_used_data":737092,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":92113140,"statfs":{"total":95995035648,"available":94323855360,"internally_reserved":0,"allocated":754782208,"data_stored":753877536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:38:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":5.2119999999999997,"15min":3.407},"min":{"1min":0.24199999999999999,"5min":0.218,"15min":0.19500000000000001},"max":{"1min":7.5289999999999999,"5min":108.90000000000001,"15min":108.90000000000001},"last":0.23899999999999999},{"interface":"front","average":{"1min":3.1619999999999999,"5min":5.4050000000000002,"15min":3.7210000000000001},"min":{"1min":0.26700000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":29.501000000000001,"5min":109.187,"15min":109.187},"last":12.026}]},{"osd":1,"last update":"Sun Oct 1 09:39:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4249999999999998,"5min":6.516,"15min":4.4880000000000004},"min":{"1min":0.28100000000000003,"5min":0.23300000000000001,"15min":0.222},"max":{"1min":10.176,"5min":133.54499999999999,"15min":205.34200000000001},"last":9.7100000000000009},{"interface":"front","average":{"1min":2.29,"5min":6.6600000000000001,"15min":4.5170000000000003},"min":{"1min":0.27600000000000002,"5min":0.27500000000000002,"15min":0.24399999999999999},"max":{"1min":6.7510000000000003,"5min":133.39400000000001,"15min":205.44499999999999},"last":5.7060000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:38:36 2023","interfaces":[{"interface":"back","average":{"1min":2.3969999999999998,"5min":2.145,"15min":2.0680000000000001},"min":{"1min":0.29099999999999998,"5min":0.28299999999999997,"15min":0.249},"max":{"1min":11.602,"5min":14.901999999999999,"15min":14.901999999999999},"last":4.4560000000000004},{"interface":"front","average":{"1min":2.3010000000000002,"5min":1.8520000000000001,"15min":1.903},"min":{"1min":0.30599999999999999,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":10.627000000000001,"5min":10.627000000000001,"15min":12.388999999999999},"last":3.7109999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:39:01 2023","interfaces":[{"interface":"back","average":{"1min":3.2229999999999999,"5min":3.415,"15min":3.5489999999999999},"min":{"1min":0.34399999999999997,"5min":0.33300000000000002,"15min":0.28199999999999997},"max":{"1min":35.104999999999997,"5min":76.555000000000007,"15min":164.595},"last":4.5750000000000002},{"interface":"front","average":{"1min":3.2959999999999998,"5min":3.5139999999999998,"15min":3.4710000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.313},"max":{"1min":35.118000000000002,"5min":76.534000000000006,"15min":164.60499999999999},"last":4.5640000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:38:44 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":1.129,"15min":0.89000000000000001},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.23499999999999999},"max":{"1min":6.6219999999999999,"5min":9.6020000000000003,"15min":9.6020000000000003},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.999,"15min":0.90900000000000003},"min":{"1min":0.33800000000000002,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":3.8130000000000002,"5min":5.1870000000000003,"15min":8.0009999999999994},"last":3.641}]},{"osd":5,"last update":"Sun Oct 1 09:39:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.581,"15min":1.181},"min":{"1min":0.38600000000000001,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":5.2949999999999999,"5min":45.222000000000001,"15min":45.222000000000001},"last":3.6819999999999999},{"interface":"front","average":{"1min":2.4119999999999999,"5min":2.0139999999999998,"15min":1.2},"min":{"1min":0.33400000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":17.84,"5min":44.905000000000001,"15min":44.905000000000001},"last":0.37}]},{"osd":6,"last update":"Sun Oct 1 09:39:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":1.331,"15min":1.02},"min":{"1min":0.42799999999999999,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":5.2610000000000001,"5min":11.853999999999999,"15min":11.853999999999999},"last":3.7349999999999999},{"interface":"front","average":{"1min":1.8140000000000001,"5min":1.4670000000000001,"15min":1.0209999999999999},"min":{"1min":0.376,"5min":0.32400000000000001,"15min":0.32300000000000001},"max":{"1min":14.584,"5min":14.584,"15min":14.584},"last":2.633}]}]},{"osd":6,"up_from":294,"seq":1262720385178,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":372612,"kb_used_data":39100,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":93372540,"statfs":{"total":95995035648,"available":95613480960,"internally_reserved":0,"allocated":40038400,"data_stored":39141795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,0,0,1,0,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:39:10 2023","interfaces":[{"interface":"back","average":{"1min":8.0800000000000001,"5min":3.1070000000000002,"15min":3.98},"min":{"1min":0.23400000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":64.239999999999995,"5min":64.239999999999995,"15min":130.02699999999999},"last":4.1790000000000003},{"interface":"front","average":{"1min":9.7490000000000006,"5min":3.5009999999999999,"15min":3.9100000000000001},"min":{"1min":0.28999999999999998,"5min":0.221,"15min":0.221},"max":{"1min":64.182000000000002,"5min":64.182000000000002,"15min":133.72900000000001},"last":3.3260000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:39:10 2023","interfaces":[{"interface":"back","average":{"1min":8.4900000000000002,"5min":6.9969999999999999,"15min":4.4429999999999996},"min":{"1min":0.39900000000000002,"5min":0.26700000000000002,"15min":0.23899999999999999},"max":{"1min":65.954999999999998,"5min":121.95,"15min":121.95},"last":9.952},{"interface":"front","average":{"1min":8.8399999999999999,"5min":6.9240000000000004,"15min":4.3630000000000004},"min":{"1min":0.44400000000000001,"5min":0.28100000000000003,"15min":0.24399999999999999},"max":{"1min":65.960999999999999,"5min":121.98099999999999,"15min":121.98099999999999},"last":0.373}]},{"osd":2,"last update":"Sun Oct 1 09:39:10 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":2.1030000000000002,"15min":2.242},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":8.3759999999999994,"5min":9.657,"15min":26.289999999999999},"last":2.7149999999999999},{"interface":"front","average":{"1min":3.036,"5min":1.978,"15min":2.274},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.6109999999999998,"5min":21.704999999999998,"15min":26.388999999999999},"last":2.5819999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:39:10 2023","interfaces":[{"interface":"back","average":{"1min":2.3239999999999998,"5min":2.774,"15min":2.6560000000000001},"min":{"1min":0.32200000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":8.4329999999999998,"5min":89.753,"15min":89.753},"last":6.1109999999999998},{"interface":"front","average":{"1min":2.1339999999999999,"5min":2.677,"15min":2.2040000000000002},"min":{"1min":0.36699999999999999,"5min":0.318,"15min":0.29299999999999998},"max":{"1min":7.2670000000000003,"5min":89.849999999999994,"15min":89.849999999999994},"last":10.191000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.78000000000000003,"15min":0.71199999999999997},"min":{"1min":0.253,"5min":0.246,"15min":0.246},"max":{"1min":6.5830000000000002,"5min":6.5830000000000002,"15min":6.5830000000000002},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.66900000000000004,"15min":0.78800000000000003},"min":{"1min":0.33100000000000002,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":5.0679999999999996,"5min":5.0679999999999996,"15min":9.4090000000000007},"last":3.762}]},{"osd":5,"last update":"Sun Oct 1 09:39:20 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.92700000000000005,"15min":1.0069999999999999},"min":{"1min":0.30199999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.04,"5min":12.023999999999999,"15min":12.023999999999999},"last":3.7999999999999998},{"interface":"front","average":{"1min":1.042,"5min":1.103,"15min":1.121},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.4509999999999996,"5min":6.6360000000000001,"15min":6.6360000000000001},"last":2.6230000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":2.085,"15min":2.758},"min":{"1min":0.311,"5min":0.30499999999999999,"15min":0.27900000000000003},"max":{"1min":6.4119999999999999,"5min":94.245999999999995,"15min":115.11},"last":3.8279999999999998},{"interface":"front","average":{"1min":1.208,"5min":2.3399999999999999,"15min":2.7509999999999999},"min":{"1min":0.34899999999999998,"5min":0.32100000000000001,"15min":0.28799999999999998},"max":{"1min":4.9859999999999998,"5min":94.715000000000003,"15min":114.666},"last":2.8580000000000001}]}]},{"osd":1,"up_from":94,"seq":403726926155,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":876396,"kb_used_data":102436,"kb_used_omap":7,"kb_used_meta":773880,"kb_avail":92868756,"statfs":{"total":95995035648,"available":95097606144,"internally_reserved":0,"allocated":104894464,"data_stored":103027781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":792453543},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:38:50 2023","interfaces":[{"interface":"back","average":{"1min":4.2690000000000001,"5min":6.1200000000000001,"15min":4.5810000000000004},"min":{"1min":0.20899999999999999,"5min":0.182,"15min":0.182},"max":{"1min":22.835999999999999,"5min":154.21299999999999,"15min":154.21299999999999},"last":0.28299999999999997},{"interface":"front","average":{"1min":5.5439999999999996,"5min":4.5039999999999996,"15min":3.9969999999999999},"min":{"1min":0.252,"5min":0.19500000000000001,"15min":0.188},"max":{"1min":15.617000000000001,"5min":95.236999999999995,"15min":95.236999999999995},"last":14.853999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:38:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1760000000000002,"5min":2.4399999999999999,"15min":3.5009999999999999},"min":{"1min":0.22900000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":11.358000000000001,"5min":13.898999999999999,"15min":56.573},"last":7.9470000000000001},{"interface":"front","average":{"1min":4.9669999999999996,"5min":4.4909999999999997,"15min":4.4699999999999998},"min":{"1min":0.40200000000000002,"5min":0.254,"15min":0.20499999999999999},"max":{"1min":11.27,"5min":154.197,"15min":154.197},"last":6.181}]},{"osd":3,"last update":"Sun Oct 1 09:39:08 2023","interfaces":[{"interface":"back","average":{"1min":4.931,"5min":5.0609999999999999,"15min":4.9729999999999999},"min":{"1min":0.24299999999999999,"5min":0.219,"15min":0.219},"max":{"1min":15.664999999999999,"5min":95.305000000000007,"15min":200.56800000000001},"last":14.811999999999999},{"interface":"front","average":{"1min":6.2359999999999998,"5min":7.032,"15min":5.7030000000000003},"min":{"1min":0.29399999999999998,"5min":0.23000000000000001,"15min":0.21199999999999999},"max":{"1min":20.181000000000001,"5min":154.483,"15min":200.63800000000001},"last":7.9180000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:38:50 2023","interfaces":[{"interface":"back","average":{"1min":3.8889999999999998,"5min":4.2430000000000003,"15min":2.3809999999999998},"min":{"1min":0.33400000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":21.015000000000001,"5min":154.101,"15min":154.101},"last":5.7460000000000004},{"interface":"front","average":{"1min":3.3799999999999999,"5min":4.141,"15min":2.79},"min":{"1min":0.40899999999999997,"5min":0.32800000000000001,"15min":0.27500000000000002},"max":{"1min":10.222,"5min":154.56999999999999,"15min":154.56999999999999},"last":0.311}]},{"osd":5,"last update":"Sun Oct 1 09:39:28 2023","interfaces":[{"interface":"back","average":{"1min":3.831,"5min":4.6520000000000001,"15min":2.1539999999999999},"min":{"1min":0.31900000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":19.513000000000002,"5min":154.22999999999999,"15min":154.22999999999999},"last":1.6799999999999999},{"interface":"front","average":{"1min":3.254,"5min":4.4409999999999998,"15min":2.8290000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":10.305,"5min":154.54400000000001,"15min":154.54400000000001},"last":0.47799999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:39:08 2023","interfaces":[{"interface":"back","average":{"1min":3.75,"5min":4.1470000000000002,"15min":3.1549999999999998},"min":{"1min":0.32900000000000001,"5min":0.313,"15min":0.29899999999999999},"max":{"1min":10.353,"5min":154.12200000000001,"15min":154.12200000000001},"last":6.21},{"interface":"front","average":{"1min":2.645,"5min":4.1859999999999999,"15min":2.637},"min":{"1min":0.33200000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":10.259,"5min":154.16999999999999,"15min":154.16999999999999},"last":0.44700000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:38:50 2023","interfaces":[{"interface":"back","average":{"1min":3.8759999999999999,"5min":4.4980000000000002,"15min":2.7069999999999999},"min":{"1min":0.39800000000000002,"5min":0.33900000000000002,"15min":0.316},"max":{"1min":21.062999999999999,"5min":154.20699999999999,"15min":154.20699999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":3.6349999999999998,"5min":4.6310000000000002,"15min":2.9359999999999999},"min":{"1min":0.47499999999999998,"5min":0.34300000000000003,"15min":0.30199999999999999},"max":{"1min":13.161,"5min":154.51599999999999,"15min":154.51599999999999},"last":0.54800000000000004}]}]},{"osd":0,"up_from":89,"seq":382252089689,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":808768,"kb_used_data":28856,"kb_used_omap":8,"kb_used_meta":779831,"kb_avail":92936384,"statfs":{"total":95995035648,"available":95166857216,"internally_reserved":0,"allocated":29548544,"data_stored":27668464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":798547906},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:38:57 2023","interfaces":[{"interface":"back","average":{"1min":3.1970000000000001,"5min":8.093,"15min":4.5789999999999997},"min":{"1min":0.26600000000000001,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":10.699999999999999,"5min":216.821,"15min":216.821},"last":0.26300000000000001},{"interface":"front","average":{"1min":3.4209999999999998,"5min":8.0009999999999994,"15min":5.2999999999999998},"min":{"1min":0.376,"5min":0.246,"15min":0.21199999999999999},"max":{"1min":11.558,"5min":217.02699999999999,"15min":217.02699999999999},"last":4.6769999999999996}]},{"osd":2,"last update":"Sun Oct 1 09:39:28 2023","interfaces":[{"interface":"back","average":{"1min":3.077,"5min":3.2410000000000001,"15min":2.4849999999999999},"min":{"1min":0.253,"5min":0.21299999999999999,"15min":0.20100000000000001},"max":{"1min":19.788,"5min":19.788,"15min":19.788},"last":2.5579999999999998},{"interface":"front","average":{"1min":4.0670000000000002,"5min":3.2709999999999999,"15min":3.202},"min":{"1min":0.32600000000000001,"5min":0.215,"15min":0.189},"max":{"1min":19.375,"5min":25.844999999999999,"15min":25.844999999999999},"last":2.5870000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:38:57 2023","interfaces":[{"interface":"back","average":{"1min":3.2410000000000001,"5min":5.5430000000000001,"15min":4.8220000000000001},"min":{"1min":0.34100000000000003,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":15.35,"5min":222.16499999999999,"15min":222.16499999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":4.2679999999999998,"5min":5.2409999999999997,"15min":5.0519999999999996},"min":{"1min":0.34499999999999997,"5min":0.24399999999999999,"15min":0.219},"max":{"1min":14.545999999999999,"5min":222.227,"15min":222.227},"last":0.40100000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:38:48 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":2.1219999999999999,"15min":2.3319999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.27600000000000002},"max":{"1min":6.9509999999999996,"5min":15.449999999999999,"15min":62.206000000000003},"last":0.45000000000000001},{"interface":"front","average":{"1min":3.234,"5min":2.5699999999999998,"15min":2.4119999999999999},"min":{"1min":0.35399999999999998,"5min":0.33700000000000002,"15min":0.26900000000000002},"max":{"1min":14.461,"5min":14.461,"15min":62.073999999999998},"last":0.436}]},{"osd":5,"last update":"Sun Oct 1 09:39:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8149999999999999,"5min":2.8980000000000001,"15min":2.1400000000000001},"min":{"1min":0.38800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":14.515000000000001,"5min":16.808,"15min":16.808},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.629,"5min":2.589,"15min":1.9510000000000001},"min":{"1min":0.39900000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.9580000000000002,"5min":17.507000000000001,"15min":17.507000000000001},"last":5.0629999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:39:15 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":2.3559999999999999,"15min":1.883},"min":{"1min":0.434,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":4.8719999999999999,"5min":17.535,"15min":17.535},"last":6.5},{"interface":"front","average":{"1min":2.0209999999999999,"5min":2.411,"15min":2.367},"min":{"1min":0.45900000000000002,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":8.5429999999999993,"5min":15.651999999999999,"15min":18.521000000000001},"last":10.446}]},{"osd":7,"last update":"Sun Oct 1 09:38:48 2023","interfaces":[{"interface":"back","average":{"1min":4.109,"5min":3.1739999999999999,"15min":2.1789999999999998},"min":{"1min":0.44700000000000001,"5min":0.377,"15min":0.34499999999999997},"max":{"1min":48.895000000000003,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.51700000000000002},{"interface":"front","average":{"1min":5.0750000000000002,"5min":3.21,"15min":2.6579999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":48.274000000000001,"5min":48.274000000000001,"15min":48.274000000000001},"last":1.353}]}]},{"osd":2,"up_from":99,"seq":425201762629,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":739912,"kb_used_data":15104,"kb_used_omap":11,"kb_used_meta":724788,"kb_avail":93005240,"statfs":{"total":95995035648,"available":95237365760,"internally_reserved":0,"allocated":15466496,"data_stored":13635392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":742183799},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:38:42 2023","interfaces":[{"interface":"back","average":{"1min":3.4740000000000002,"5min":5.6749999999999998,"15min":6.9770000000000003},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":10.737,"5min":95.597999999999999,"15min":224.56},"last":4.3710000000000004},{"interface":"front","average":{"1min":4.3970000000000002,"5min":5.9660000000000002,"15min":7.5199999999999996},"min":{"1min":0.23999999999999999,"5min":0.221,"15min":0.20699999999999999},"max":{"1min":16.561,"5min":95.558000000000007,"15min":225.66900000000001},"last":5.3029999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:38:35 2023","interfaces":[{"interface":"back","average":{"1min":5.4980000000000002,"5min":8.9640000000000004,"15min":8.8260000000000005},"min":{"1min":0.23899999999999999,"5min":0.20300000000000001,"15min":0.191},"max":{"1min":51.863,"5min":176.52000000000001,"15min":224.917},"last":0.83499999999999996},{"interface":"front","average":{"1min":8.3680000000000003,"5min":10.141999999999999,"15min":9.9930000000000003},"min":{"1min":0.40100000000000002,"5min":0.22900000000000001,"15min":0.22},"max":{"1min":58.590000000000003,"5min":179.85300000000001,"15min":225.625},"last":4.7009999999999996}]},{"osd":3,"last update":"Sun Oct 1 09:39:01 2023","interfaces":[{"interface":"back","average":{"1min":4.048,"5min":3.4199999999999999,"15min":5.1239999999999997},"min":{"1min":0.22900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":19.486000000000001,"5min":50.289000000000001,"15min":226.124},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.6539999999999999,"5min":2.8450000000000002,"15min":4.8159999999999998},"min":{"1min":0.22,"5min":0.215,"15min":0.20300000000000001},"max":{"1min":10.569000000000001,"5min":54.115000000000002,"15min":226.32900000000001},"last":0.47999999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:38:35 2023","interfaces":[{"interface":"back","average":{"1min":2.895,"5min":1.5880000000000001,"15min":2.6400000000000001},"min":{"1min":0.34899999999999998,"5min":0.29999999999999999,"15min":0.28599999999999998},"max":{"1min":16.048999999999999,"5min":16.048999999999999,"15min":224.90199999999999},"last":4.7930000000000001},{"interface":"front","average":{"1min":4.0780000000000003,"5min":2.2650000000000001,"15min":2.9710000000000001},"min":{"1min":0.34799999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.457000000000001,"5min":11.475,"15min":225.386},"last":0.69599999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:39:20 2023","interfaces":[{"interface":"back","average":{"1min":1.548,"5min":2.141,"15min":1.821},"min":{"1min":0.29799999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.1040000000000001,"5min":60.432000000000002,"15min":60.432000000000002},"last":0.57199999999999995},{"interface":"front","average":{"1min":2.0179999999999998,"5min":2.5539999999999998,"15min":1.7310000000000001},"min":{"1min":0.35699999999999998,"5min":0.34000000000000002,"15min":0.33000000000000002},"max":{"1min":7.9379999999999997,"5min":60.470999999999997,"15min":60.470999999999997},"last":4.7409999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:39:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3780000000000001,"5min":2.2829999999999999,"15min":2.9990000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":10.314,"5min":11.526999999999999,"15min":225.41499999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":2.5179999999999998,"5min":1.6200000000000001,"15min":2.5030000000000001},"min":{"1min":0.36699999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":10.539999999999999,"5min":11.239000000000001,"15min":225.02000000000001},"last":0.621}]},{"osd":7,"last update":"Sun Oct 1 09:38:35 2023","interfaces":[{"interface":"back","average":{"1min":2.665,"5min":4.8250000000000002,"15min":3.7839999999999998},"min":{"1min":0.36199999999999999,"5min":0.33000000000000002,"15min":0.308},"max":{"1min":15.615,"5min":139.03299999999999,"15min":224.94999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":4.1779999999999999,"5min":5.0419999999999998,"15min":3.9929999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":16.018000000000001,"5min":139.065,"15min":224.92599999999999},"last":4.7670000000000003}]}]},{"osd":3,"up_from":104,"seq":446676599102,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1759480,"kb_used_data":981744,"kb_used_omap":3,"kb_used_meta":777596,"kb_avail":91985672,"statfs":{"total":95995035648,"available":94193328128,"internally_reserved":0,"allocated":1005305856,"data_stored":1003369603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":796259047},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:39:00 2023","interfaces":[{"interface":"back","average":{"1min":9.2040000000000006,"5min":5.1520000000000001,"15min":6.1589999999999998},"min":{"1min":0.223,"5min":0.19900000000000001,"15min":0.18099999999999999},"max":{"1min":118.748,"5min":137.34399999999999,"15min":137.34399999999999},"last":4.798},{"interface":"front","average":{"1min":9.1430000000000007,"5min":5.7210000000000001,"15min":6.0949999999999998},"min":{"1min":0.38700000000000001,"5min":0.20899999999999999,"15min":0.184},"max":{"1min":118.86199999999999,"5min":137.73400000000001,"15min":137.73400000000001},"last":10.529}]},{"osd":1,"last update":"Sun Oct 1 09:39:00 2023","interfaces":[{"interface":"back","average":{"1min":8.1280000000000001,"5min":9.7859999999999996,"15min":6.0369999999999999},"min":{"1min":0.29899999999999999,"5min":0.222,"15min":0.20399999999999999},"max":{"1min":90.944000000000003,"5min":162.22999999999999,"15min":162.22999999999999},"last":5.7220000000000004},{"interface":"front","average":{"1min":7.3810000000000002,"5min":9.5809999999999995,"15min":6.548},"min":{"1min":0.27400000000000002,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":83.341999999999999,"5min":162.70699999999999,"15min":162.70699999999999},"last":0.36299999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:39:00 2023","interfaces":[{"interface":"back","average":{"1min":4.5,"5min":2.6960000000000002,"15min":3.2999999999999998},"min":{"1min":0.308,"5min":0.22800000000000001,"15min":0.20799999999999999},"max":{"1min":13.108000000000001,"5min":14.194000000000001,"15min":62.909999999999997},"last":11.622},{"interface":"front","average":{"1min":1.827,"5min":1.9590000000000001,"15min":3.819},"min":{"1min":0.26100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":7.6749999999999998,"5min":15.523,"15min":78.594999999999999},"last":0.39700000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:39:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3280000000000001,"5min":1.6120000000000001,"15min":1.9259999999999999},"min":{"1min":0.314,"5min":0.29099999999999998,"15min":0.26200000000000001},"max":{"1min":7.6159999999999997,"5min":11.153,"15min":29.419},"last":0.44700000000000001},{"interface":"front","average":{"1min":2.137,"5min":2.0800000000000001,"15min":2.0579999999999998},"min":{"1min":0.32700000000000001,"5min":0.27900000000000003,"15min":0.23899999999999999},"max":{"1min":10.833,"5min":10.833,"15min":33.002000000000002},"last":3.9510000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:39:22 2023","interfaces":[{"interface":"back","average":{"1min":2.8759999999999999,"5min":2.0990000000000002,"15min":3.2879999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.919,"5min":10.919,"15min":28.117999999999999},"last":3.9990000000000001},{"interface":"front","average":{"1min":2.2210000000000001,"5min":2.5859999999999999,"15min":3.3450000000000002},"min":{"1min":0.35499999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.6029999999999998,"5min":14.537000000000001,"15min":28.34},"last":6.3520000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:39:16 2023","interfaces":[{"interface":"back","average":{"1min":2.6789999999999998,"5min":2.5470000000000002,"15min":2.4409999999999998},"min":{"1min":0.38300000000000001,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":7.6660000000000004,"5min":13.032999999999999,"15min":18.361000000000001},"last":6.3949999999999996},{"interface":"front","average":{"1min":2.9249999999999998,"5min":2.5880000000000001,"15min":2.379},"min":{"1min":0.42899999999999999,"5min":0.33600000000000002,"15min":0.32800000000000001},"max":{"1min":7.6399999999999997,"5min":13.057,"15min":18.332000000000001},"last":6.4210000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:39:00 2023","interfaces":[{"interface":"back","average":{"1min":3.3580000000000001,"5min":2.7200000000000002,"15min":2.597},"min":{"1min":0.40699999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":13.085000000000001,"5min":13.085000000000001,"15min":52.006},"last":11.659000000000001},{"interface":"front","average":{"1min":1.7110000000000001,"5min":1.9930000000000001,"15min":2.3410000000000002},"min":{"1min":0.45500000000000002,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":7.6550000000000002,"5min":11.109999999999999,"15min":51.890999999999998},"last":3.8889999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019353,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":909132,"kb_used_data":14852,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":92836020,"statfs":{"total":95995035648,"available":95064084480,"internally_reserved":0,"allocated":15208448,"data_stored":14303794,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:38:48 2023","interfaces":[{"interface":"back","average":{"1min":4.25,"5min":2.9969999999999999,"15min":4.742},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":72.269999999999996,"5min":72.269999999999996,"15min":155.315},"last":0.254},{"interface":"front","average":{"1min":4.774,"5min":3.5289999999999999,"15min":4.952},"min":{"1min":0.253,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":72.132999999999996,"5min":72.132999999999996,"15min":155.334},"last":0.68200000000000005}]},{"osd":1,"last update":"Sun Oct 1 09:38:52 2023","interfaces":[{"interface":"back","average":{"1min":12.032999999999999,"5min":7.173,"15min":6.9450000000000003},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":146.929,"5min":165.666,"15min":190.61199999999999},"last":0.60799999999999998},{"interface":"front","average":{"1min":11.866,"5min":6.9900000000000002,"15min":7.6980000000000004},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":146.91,"5min":165.70500000000001,"15min":253.273},"last":0.60199999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:38:34 2023","interfaces":[{"interface":"back","average":{"1min":1.756,"5min":1.948,"15min":2.8580000000000001},"min":{"1min":0.30399999999999999,"5min":0.29099999999999998,"15min":0.26700000000000002},"max":{"1min":5.8019999999999996,"5min":8.6379999999999999,"15min":170.45400000000001},"last":2.4129999999999998},{"interface":"front","average":{"1min":2.7650000000000001,"5min":2.4940000000000002,"15min":2.5640000000000001},"min":{"1min":0.32000000000000001,"5min":0.316,"15min":0.25800000000000001},"max":{"1min":10.625,"5min":12.045,"15min":58.771000000000001},"last":0.47699999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:38:52 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":3.6520000000000001,"15min":3.4769999999999999},"min":{"1min":0.29099999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":5.7569999999999997,"5min":140.881,"15min":140.881},"last":5.8319999999999999},{"interface":"front","average":{"1min":2.0779999999999998,"5min":3.9649999999999999,"15min":3.665},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":9.3140000000000001,"5min":140.911,"15min":140.911},"last":4.6360000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:39:26 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":1.546,"15min":1.238},"min":{"1min":0.28399999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.702,"5min":20.457000000000001,"15min":20.457000000000001},"last":0.44},{"interface":"front","average":{"1min":1.28,"5min":1.5409999999999999,"15min":1.0129999999999999},"min":{"1min":0.35899999999999999,"5min":0.307,"15min":0.307},"max":{"1min":5.6740000000000004,"5min":20.475000000000001,"15min":20.475000000000001},"last":0.36899999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:39:05 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.96899999999999997,"15min":0.89500000000000002},"min":{"1min":0.311,"5min":0.29199999999999998,"15min":0.27200000000000002},"max":{"1min":1.2509999999999999,"5min":7.7649999999999997,"15min":11.358000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.999,"5min":1.2949999999999999,"15min":0.98399999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":4.3220000000000001,"5min":6.9139999999999997,"15min":10.076000000000001},"last":0.503}]},{"osd":7,"last update":"Sun Oct 1 09:38:34 2023","interfaces":[{"interface":"back","average":{"1min":2.673,"5min":1.6910000000000001,"15min":1.1240000000000001},"min":{"1min":0.371,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":38.034999999999997,"5min":38.034999999999997,"15min":38.034999999999997},"last":1.3680000000000001},{"interface":"front","average":{"1min":2.8620000000000001,"5min":2.125,"15min":1.331},"min":{"1min":0.39000000000000001,"5min":0.34200000000000003,"15min":0.28699999999999998},"max":{"1min":37.723999999999997,"5min":37.723999999999997,"15min":37.723999999999997},"last":0.64300000000000002}]}]},{"osd":5,"up_from":379,"seq":1627792605262,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1046660,"kb_used_data":682108,"kb_used_omap":13,"kb_used_meta":364530,"kb_avail":92698492,"statfs":{"total":95995035648,"available":94923255808,"internally_reserved":0,"allocated":698478592,"data_stored":697577624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":373279218},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:39:24 2023","interfaces":[{"interface":"back","average":{"1min":4.04,"5min":3.9209999999999998,"15min":2.3100000000000001},"min":{"1min":0.32400000000000001,"5min":0.221,"15min":0.221},"max":{"1min":12.571999999999999,"5min":84.340999999999994,"15min":84.340999999999994},"last":10.000999999999999},{"interface":"front","average":{"1min":2.6190000000000002,"5min":4.0330000000000004,"15min":2.165},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.6460000000000008,"5min":84.356999999999999,"15min":84.356999999999999},"last":0.38400000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:39:24 2023","interfaces":[{"interface":"back","average":{"1min":15.868,"5min":11.93,"15min":5.1079999999999997},"min":{"1min":0.25900000000000001,"5min":0.255,"15min":0.255},"max":{"1min":295.71300000000002,"5min":295.71300000000002,"15min":295.71300000000002},"last":1.0629999999999999},{"interface":"front","average":{"1min":16.186,"5min":11.949,"15min":4.9130000000000003},"min":{"1min":0.25900000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":299.59399999999999,"5min":299.59399999999999,"15min":299.59399999999999},"last":9.9710000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:39:24 2023","interfaces":[{"interface":"back","average":{"1min":3.222,"5min":2.5139999999999998,"15min":1.7070000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.7040000000000006,"5min":14.199999999999999,"15min":14.199999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":2.0960000000000001,"5min":1.9159999999999999,"15min":1.72},"min":{"1min":0.32100000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.2370000000000001,"5min":14.430999999999999,"15min":14.430999999999999},"last":6.984}]},{"osd":3,"last update":"Sun Oct 1 09:39:24 2023","interfaces":[{"interface":"back","average":{"1min":3.9009999999999998,"5min":2.343,"15min":1.5549999999999999},"min":{"1min":0.442,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":12.734,"5min":14.004,"15min":14.004},"last":10.022},{"interface":"front","average":{"1min":3.9079999999999999,"5min":2.6930000000000001,"15min":1.6830000000000001},"min":{"1min":0.36299999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":11.766,"5min":13.991,"15min":13.991},"last":0.432}]},{"osd":4,"last update":"Sun Oct 1 09:39:24 2023","interfaces":[{"interface":"back","average":{"1min":1.288,"5min":1.2270000000000001,"15min":1.0349999999999999},"min":{"1min":0.28100000000000003,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.6790000000000003,"5min":12.808999999999999,"15min":12.808999999999999},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.151,"15min":1.494},"min":{"1min":0.307,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.4700000000000006,"5min":9.4700000000000006,"15min":14.68},"last":7.0780000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:39:24 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":1.0309999999999999,"15min":1.0329999999999999},"min":{"1min":0.311,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.545,"5min":12.545,"15min":12.545},"last":10.500999999999999},{"interface":"front","average":{"1min":1.034,"5min":1.325,"15min":1.4390000000000001},"min":{"1min":0.33100000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.5039999999999996,"5min":14.193,"15min":14.193},"last":0.45300000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:39:24 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.274,"15min":1.337},"min":{"1min":0.34200000000000003,"5min":0.307,"15min":0.307},"max":{"1min":8.2010000000000005,"5min":9.984,"15min":10.813000000000001},"last":1.085},{"interface":"front","average":{"1min":1.4570000000000001,"5min":1.226,"15min":1.004},"min":{"1min":0.35299999999999998,"5min":0.318,"15min":0.318},"max":{"1min":12.500999999999999,"5min":12.500999999999999,"15min":12.500999999999999},"last":10.039}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":18546688,"data_stored":18546688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":79826944,"data_stored":79822872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":5505024,"data_stored":5505024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":995491840,"data_stored":995491840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":7274496,"data_stored":7274496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":675287040,"data_stored":675282968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":27590656,"data_stored":27590656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":742002688,"data_stored":741998616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:39:41.473 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 21 2023-10-01T09:39:41.522 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:41.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:41.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:41.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:41 smithi138 ceph-mon[185886]: pgmap v1336: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 8.4 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 12772/267 objects degraded (4783.521%); 5.3 MiB/s, 143 objects/s recovering 2023-10-01T09:39:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:41 smithi138 ceph-mon[192513]: pgmap v1336: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 8.4 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 12772/267 objects degraded (4783.521%); 5.3 MiB/s, 143 objects/s recovering 2023-10-01T09:39:41.627 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:41.661 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:41.744 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:41.766 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:41.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:41.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:41.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:41 smithi190 ceph-mon[160435]: pgmap v1336: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 8.4 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 12772/267 objects degraded (4783.521%); 5.3 MiB/s, 143 objects/s recovering 2023-10-01T09:39:41.871 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:41.872 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:39:41.872 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:39:41.905 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:43.428 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:39:43.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:43 smithi138 ceph-mon[185886]: pgmap v1337: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 12772/84 objects degraded (15204.762%); 4.4 MiB/s, 129 objects/s recovering 2023-10-01T09:39:43.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:43 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 12772/84 objects degraded (15204.762%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:43.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:43 smithi138 ceph-mon[192513]: pgmap v1337: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 12772/84 objects degraded (15204.762%); 4.4 MiB/s, 129 objects/s recovering 2023-10-01T09:39:43.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:43 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 12772/84 objects degraded (15204.762%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:43.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:43 smithi190 ceph-mon[160435]: pgmap v1337: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 345 op/s; 12772/84 objects degraded (15204.762%); 4.4 MiB/s, 129 objects/s recovering 2023-10-01T09:39:43.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:43 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 12772/84 objects degraded (15204.762%), 6 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:43.991 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:39:44.009 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:44.087 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:44.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:44.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:44.199 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:44.233 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:44.288 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:44.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:44.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:44.393 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:44.427 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:44.551 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:44.767 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:39:44.789 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:44.790 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:39:44.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:44.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:44.814 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:39:44.836 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:39:44.856 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:39:44.875 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:39:44.894 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:39:44.896 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:39:44.913 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:39:44.914 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:39:44.915 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:39:44.915 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:39:44.915 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:39:44.915 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:39:44.915 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:39:44.915 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:39:44.915 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:39:44.934 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:44.978 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:39:44.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:44.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:39:45.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:39:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:39:44] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.33.4" 2023-10-01T09:39:45.085 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:45.112 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:45.154 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:45.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:45.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:45.259 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:45.293 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:45.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:45 smithi138 ceph-mon[192513]: pgmap v1338: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 330 op/s; 12772/84 objects degraded (15204.762%); 4.4 MiB/s, 129 objects/s recovering 2023-10-01T09:39:45.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:45 smithi138 ceph-mon[192513]: failed to encode map e467 with expected crc 2023-10-01T09:39:45.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:45 smithi138 ceph-mon[192513]: failed to encode map e467 with expected crc 2023-10-01T09:39:45.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:45 smithi138 ceph-mon[192513]: osdmap e467: 8 total, 8 up, 8 in 2023-10-01T09:39:45.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:45 smithi138 ceph-mon[192513]: failed to encode map e467 with expected crc 2023-10-01T09:39:45.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:45 smithi138 ceph-mon[192513]: failed to encode map e467 with expected crc 2023-10-01T09:39:45.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:45 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4006936402' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:45.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:45 smithi138 ceph-mon[185886]: pgmap v1338: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 330 op/s; 12772/84 objects degraded (15204.762%); 4.4 MiB/s, 129 objects/s recovering 2023-10-01T09:39:45.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:45 smithi138 ceph-mon[185886]: failed to encode map e467 with expected crc 2023-10-01T09:39:45.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:45 smithi138 ceph-mon[185886]: failed to encode map e467 with expected crc 2023-10-01T09:39:45.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:45 smithi138 ceph-mon[185886]: osdmap e467: 8 total, 8 up, 8 in 2023-10-01T09:39:45.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:45 smithi138 ceph-mon[185886]: failed to encode map e467 with expected crc 2023-10-01T09:39:45.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:45 smithi138 ceph-mon[185886]: failed to encode map e467 with expected crc 2023-10-01T09:39:45.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:45 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4006936402' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:45.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:45 smithi190 ceph-mon[160435]: pgmap v1338: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 330 op/s; 12772/84 objects degraded (15204.762%); 4.4 MiB/s, 129 objects/s recovering 2023-10-01T09:39:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:45 smithi190 ceph-mon[160435]: failed to encode map e467 with expected crc 2023-10-01T09:39:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:45 smithi190 ceph-mon[160435]: failed to encode map e467 with expected crc 2023-10-01T09:39:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:45 smithi190 ceph-mon[160435]: osdmap e467: 8 total, 8 up, 8 in 2023-10-01T09:39:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:45 smithi190 ceph-mon[160435]: failed to encode map e467 with expected crc 2023-10-01T09:39:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:45 smithi190 ceph-mon[160435]: failed to encode map e467 with expected crc 2023-10-01T09:39:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:45 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4006936402' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:46.247 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:46.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:46.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:46.351 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:46.379 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:46 smithi138 ceph-mon[192513]: failed to encode map e468 with expected crc 2023-10-01T09:39:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:46 smithi138 ceph-mon[192513]: osdmap e468: 8 total, 8 up, 8 in 2023-10-01T09:39:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:46 smithi138 ceph-mon[192513]: failed to encode map e468 with expected crc 2023-10-01T09:39:46.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:46 smithi138 ceph-mon[185886]: failed to encode map e468 with expected crc 2023-10-01T09:39:46.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:46 smithi138 ceph-mon[185886]: osdmap e468: 8 total, 8 up, 8 in 2023-10-01T09:39:46.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:46 smithi138 ceph-mon[185886]: failed to encode map e468 with expected crc 2023-10-01T09:39:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:46 smithi190 ceph-mon[160435]: failed to encode map e468 with expected crc 2023-10-01T09:39:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:46 smithi190 ceph-mon[160435]: osdmap e468: 8 total, 8 up, 8 in 2023-10-01T09:39:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:46 smithi190 ceph-mon[160435]: failed to encode map e468 with expected crc 2023-10-01T09:39:47.734 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:47.737 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:47 smithi138 ceph-mon[192513]: pgmap v1341: 89 pgs: 1 peering, 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 12772/84 objects degraded (15204.762%); 1019 KiB/s, 273 objects/s recovering 2023-10-01T09:39:47.737 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:47 smithi138 ceph-mon[185886]: pgmap v1341: 89 pgs: 1 peering, 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 12772/84 objects degraded (15204.762%); 1019 KiB/s, 273 objects/s recovering 2023-10-01T09:39:47.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:47.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:47.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:47 smithi190 ceph-mon[160435]: pgmap v1341: 89 pgs: 1 peering, 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 12772/84 objects degraded (15204.762%); 1019 KiB/s, 273 objects/s recovering 2023-10-01T09:39:47.839 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:47.865 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:47.907 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:47.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:47.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:48.012 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:39:48.037 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:48.080 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:39:48.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:48.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:39:48.109 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:48.187 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:48.212 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:48.254 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:48.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:48.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:48.363 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:48.398 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:48.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:48 smithi190 ceph-mon[160435]: failed to encode map e468 with expected crc 2023-10-01T09:39:48.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:48 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 12772/84 objects degraded (15204.762%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:48.655 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:48.657 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:48 smithi138 ceph-mon[192513]: failed to encode map e468 with expected crc 2023-10-01T09:39:48.657 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:48 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 12772/84 objects degraded (15204.762%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:48.657 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:48 smithi138 ceph-mon[185886]: failed to encode map e468 with expected crc 2023-10-01T09:39:48.657 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:48 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 12772/84 objects degraded (15204.762%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:39:48.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:48.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:48.739 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T09:39:48.760 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:48.789 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:49.556 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:49.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:49.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:49.659 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:39:49.685 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:49 smithi138 ceph-mon[192513]: pgmap v1342: 89 pgs: 1 peering, 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 7 op/s; 12772/84 objects degraded (15204.762%); 0 B/s, 171 objects/s recovering 2023-10-01T09:39:49.685 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3101815901' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-01T09:39:49.685 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:49 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-01T09:39:49.685 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:49 smithi138 ceph-mon[185886]: pgmap v1342: 89 pgs: 1 peering, 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 7 op/s; 12772/84 objects degraded (15204.762%); 0 B/s, 171 objects/s recovering 2023-10-01T09:39:49.686 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3101815901' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-01T09:39:49.686 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:49 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-01T09:39:49.688 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:49.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:49 smithi190 ceph-mon[160435]: pgmap v1342: 89 pgs: 1 peering, 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 7 op/s; 12772/84 objects degraded (15204.762%); 0 B/s, 171 objects/s recovering 2023-10-01T09:39:49.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3101815901' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-01T09:39:49.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:49 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-10-01T09:39:49.812 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:39:50.037 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:39:50.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:50.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:39:50.141 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:39:50.167 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:50.671 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:39:50.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:50.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:39:50.696 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T09:39:50.696 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-10-01T09:39:50.697 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-10-01T09:39:50.778 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:39:50.820 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:51.001 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:39:51.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:51.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:39:51.105 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:51.138 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:51.185 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:51.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:51.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:51.261 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:51.289 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:51.314 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:51.357 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:51.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:51.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:51.465 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:51.497 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:51.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:51 smithi190 ceph-mon[160435]: pgmap v1343: 89 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12772/84 objects degraded (15204.762%); 0 B/s, 364 objects/s recovering 2023-10-01T09:39:51.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:51 smithi138 ceph-mon[192513]: pgmap v1343: 89 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12772/84 objects degraded (15204.762%); 0 B/s, 364 objects/s recovering 2023-10-01T09:39:51.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:51 smithi138 ceph-mon[185886]: pgmap v1343: 89 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12772/84 objects degraded (15204.762%); 0 B/s, 364 objects/s recovering 2023-10-01T09:39:52.417 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:39:52.575 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:52.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:52.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:52.680 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:52.703 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:52 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/902302100' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:52.703 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:52 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:52.705 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:52.719 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:52 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/902302100' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:52.719 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:52 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:52.719 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:52 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/902302100' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:52.719 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:52 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:39:52.748 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:52.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:52.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:52.853 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:52.881 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:53.377 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:53.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:53.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:53.481 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:53.507 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:53.555 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:53.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:53.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:53.663 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:53.688 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:53 smithi190 ceph-mon[160435]: pgmap v1344: 89 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12772/84 objects degraded (15204.762%); 0 B/s, 364 objects/s recovering 2023-10-01T09:39:53.688 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:53 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:39:53.688 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:53 smithi190 ceph-mon[160435]: failed to encode map e469 with expected crc 2023-10-01T09:39:53.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:53 smithi190 ceph-mon[160435]: failed to encode map e469 with expected crc 2023-10-01T09:39:53.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:53 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:39:53.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:53 smithi190 ceph-mon[160435]: osdmap e469: 8 total, 8 up, 8 in 2023-10-01T09:39:53.690 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:53.735 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:53.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:53.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:53 smithi138 ceph-mon[192513]: pgmap v1344: 89 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12772/84 objects degraded (15204.762%); 0 B/s, 364 objects/s recovering 2023-10-01T09:39:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:53 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:39:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:53 smithi138 ceph-mon[192513]: failed to encode map e469 with expected crc 2023-10-01T09:39:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:53 smithi138 ceph-mon[192513]: failed to encode map e469 with expected crc 2023-10-01T09:39:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:53 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:39:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:53 smithi138 ceph-mon[192513]: osdmap e469: 8 total, 8 up, 8 in 2023-10-01T09:39:53.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:53 smithi138 ceph-mon[185886]: pgmap v1344: 89 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12772/84 objects degraded (15204.762%); 0 B/s, 364 objects/s recovering 2023-10-01T09:39:53.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:53 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:39:53.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:53 smithi138 ceph-mon[185886]: failed to encode map e469 with expected crc 2023-10-01T09:39:53.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:53 smithi138 ceph-mon[185886]: failed to encode map e469 with expected crc 2023-10-01T09:39:53.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:53 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:39:53.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:53 smithi138 ceph-mon[185886]: osdmap e469: 8 total, 8 up, 8 in 2023-10-01T09:39:53.841 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:39:53.870 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:54.026 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:54.074 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:39:54.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:54.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:39:54.107 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:39:54.107 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:39:54.107 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:39:54.107 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:39:54.107 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.108 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:187123 8.8 8:1531d290:::benchmark_data_smithi138_431617_object85712:head [delete] snapc 0=[] ondisk+write+known_if_redirected e384)", 2023-10-01T09:39:54.108 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:18.925346+0000", 2023-10-01T09:39:54.108 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 395.17955603299998, 2023-10-01T09:39:54.108 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.18660890799999999, 2023-10-01T09:39:54.108 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.108 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.108 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:54.108 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:39:54.108 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:39:54.108 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 187123 2023-10-01T09:39:54.108 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.108 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925346+0000", 2023-10-01T09:39:54.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.109 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.109 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925346+0000", 2023-10-01T09:39:54.109 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.109 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.109 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.110 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.110 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925344+0000", 2023-10-01T09:39:54.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:39:54.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.110 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.110 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.110 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925353+0000", 2023-10-01T09:39:54.110 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3270000000000001e-06 2023-10-01T09:39:54.110 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.110 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925373+0000", 2023-10-01T09:39:54.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9842000000000001e-05 2023-10-01T09:39:54.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.111 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925381+0000", 2023-10-01T09:39:54.111 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5310000000000001e-06 2023-10-01T09:39:54.111 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.111 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.111 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925432+0000", 2023-10-01T09:39:54.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1103999999999997e-05 2023-10-01T09:39:54.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925571+0000", 2023-10-01T09:39:54.112 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013912700000000001 2023-10-01T09:39:54.112 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.112 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.112 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:39:54.112 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.925723+0000", 2023-10-01T09:39:54.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000151879 2023-10-01T09:39:54.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.931279+0000", 2023-10-01T09:39:54.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0055557790000000003 2023-10-01T09:39:54.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.113 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.113 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:54.113 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.934123+0000", 2023-10-01T09:39:54.113 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028444490000000002 2023-10-01T09:39:54.113 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111698+0000", 2023-10-01T09:39:54.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.177574809 2023-10-01T09:39:54.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.114 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.114 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111751+0000", 2023-10-01T09:39:54.114 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2756000000000001e-05 2023-10-01T09:39:54.114 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.114 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.115 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.115 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111955+0000", 2023-10-01T09:39:54.115 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020406600000000001 2023-10-01T09:39:54.115 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.115 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.115 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.115 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.115 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.115 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.45674.0:187129 8.8 8:155c408a:::benchmark_data_smithi138_431617_object85718:head [delete] snapc 0=[] ondisk+write+known_if_redirected e384)", 2023-10-01T09:39:54.115 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:33:18.936343+0000", 2023-10-01T09:39:54.116 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 395.16855907299998, 2023-10-01T09:39:54.116 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17579768900000001, 2023-10-01T09:39:54.116 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.116 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.116 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:54.116 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.45674", 2023-10-01T09:39:54.116 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:39:54.116 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 187129 2023-10-01T09:39:54.116 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.116 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.116 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.116 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.117 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936343+0000", 2023-10-01T09:39:54.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.117 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.117 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936343+0000", 2023-10-01T09:39:54.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.117 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.117 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.117 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.117 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936341+0000", 2023-10-01T09:39:54.117 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:39:54.118 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.118 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.118 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.118 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936350+0000", 2023-10-01T09:39:54.118 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6170000000000007e-06 2023-10-01T09:39:54.118 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.118 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.118 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.118 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936369+0000", 2023-10-01T09:39:54.118 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9264999999999999e-05 2023-10-01T09:39:54.118 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.119 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936379+0000", 2023-10-01T09:39:54.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7550000000000006e-06 2023-10-01T09:39:54.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.119 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.119 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.119 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936435+0000", 2023-10-01T09:39:54.119 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5951e-05 2023-10-01T09:39:54.119 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936602+0000", 2023-10-01T09:39:54.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000167446 2023-10-01T09:39:54.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.120 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,7", 2023-10-01T09:39:54.120 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.936694+0000", 2023-10-01T09:39:54.120 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2208999999999998e-05 2023-10-01T09:39:54.120 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.120 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:54.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.940157+0000", 2023-10-01T09:39:54.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034625760000000002 2023-10-01T09:39:54.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:18.941169+0000", 2023-10-01T09:39:54.121 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001012352 2023-10-01T09:39:54.121 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.121 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.121 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.121 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.111993+0000", 2023-10-01T09:39:54.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17082314400000001 2023-10-01T09:39:54.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.112030+0000", 2023-10-01T09:39:54.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7453000000000002e-05 2023-10-01T09:39:54.122 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.122 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.122 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.122 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:33:19.112141+0000", 2023-10-01T09:39:54.122 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110803 2023-10-01T09:39:54.123 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.123 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.123 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.123 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.123 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.123 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:50898 9.7 e409/400 9:e780c3ef:::benchmark_data_smithi138_497829_object50897:head v 409'3182, mlcod=409'3180)", 2023-10-01T09:39:54.123 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:34:44.818857+0000", 2023-10-01T09:39:54.123 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 309.28604532600002, 2023-10-01T09:39:54.123 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.175987492, 2023-10-01T09:39:54.123 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.124 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.124 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.124 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.124 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818857+0000", 2023-10-01T09:39:54.124 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.124 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.124 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.124 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.125 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818857+0000", 2023-10-01T09:39:54.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.125 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.125 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.125 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.125 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818856+0000", 2023-10-01T09:39:54.125 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:39:54.125 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.125 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.126 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818901+0000", 2023-10-01T09:39:54.126 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4372000000000001e-05 2023-10-01T09:39:54.126 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.126 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.126 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818909+0000", 2023-10-01T09:39:54.126 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6240000000000008e-06 2023-10-01T09:39:54.126 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.126 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.126 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818915+0000", 2023-10-01T09:39:54.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0249999999999999e-06 2023-10-01T09:39:54.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.127 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818976+0000", 2023-10-01T09:39:54.127 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1261999999999997e-05 2023-10-01T09:39:54.127 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.127 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.127 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.127 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818992+0000", 2023-10-01T09:39:54.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5268e-05 2023-10-01T09:39:54.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.994803+0000", 2023-10-01T09:39:54.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17581116399999999 2023-10-01T09:39:54.128 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.128 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.128 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.128 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.994844+0000", 2023-10-01T09:39:54.128 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1482e-05 2023-10-01T09:39:54.128 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.129 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.129 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.129 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.129 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.129 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:50900 9.1 e409/400 9:8a8ebdf9:::benchmark_data_smithi138_497829_object50899:head v 409'3075, mlcod=409'3073)", 2023-10-01T09:39:54.129 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:34:44.818933+0000", 2023-10-01T09:39:54.129 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 309.28596881599998, 2023-10-01T09:39:54.129 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.176232109, 2023-10-01T09:39:54.129 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.129 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.129 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.129 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.130 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.130 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818933+0000", 2023-10-01T09:39:54.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.130 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.130 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818933+0000", 2023-10-01T09:39:54.130 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.130 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.130 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.130 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.131 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818933+0000", 2023-10-01T09:39:54.131 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:39:54.131 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.131 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.131 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.131 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818985+0000", 2023-10-01T09:39:54.131 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2595e-05 2023-10-01T09:39:54.131 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.131 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.131 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.131 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.818996+0000", 2023-10-01T09:39:54.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0823e-05 2023-10-01T09:39:54.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.819003+0000", 2023-10-01T09:39:54.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9539999999999998e-06 2023-10-01T09:39:54.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.132 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.132 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.132 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.992729+0000", 2023-10-01T09:39:54.132 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.173726357 2023-10-01T09:39:54.132 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.992748+0000", 2023-10-01T09:39:54.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8726e-05 2023-10-01T09:39:54.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.133 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.995125+0000", 2023-10-01T09:39:54.133 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002376534 2023-10-01T09:39:54.133 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.133 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.133 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.134 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.995165+0000", 2023-10-01T09:39:54.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0775000000000002e-05 2023-10-01T09:39:54.134 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.134 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.134 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.134 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.134 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.134 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:50903 9.4 e409/400 9:2f73ff33:::benchmark_data_smithi138_497829_object50902:head v 409'3191, mlcod=409'3189)", 2023-10-01T09:39:54.134 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:34:44.823126+0000", 2023-10-01T09:39:54.134 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 309.28177595800003, 2023-10-01T09:39:54.134 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.17174679700000001, 2023-10-01T09:39:54.134 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.135 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.135 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.823126+0000", 2023-10-01T09:39:54.135 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.135 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.135 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.135 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.135 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.823126+0000", 2023-10-01T09:39:54.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.823125+0000", 2023-10-01T09:39:54.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:39:54.136 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.136 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.136 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.136 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.823150+0000", 2023-10-01T09:39:54.136 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4986000000000001e-05 2023-10-01T09:39:54.137 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.137 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.137 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.137 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.992731+0000", 2023-10-01T09:39:54.137 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16958084700000001 2023-10-01T09:39:54.137 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.137 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.137 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.137 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.992741+0000", 2023-10-01T09:39:54.137 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7769999999999997e-06 2023-10-01T09:39:54.137 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.138 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.138 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.992783+0000", 2023-10-01T09:39:54.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2511e-05 2023-10-01T09:39:54.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.138 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.138 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.992804+0000", 2023-10-01T09:39:54.138 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0214999999999999e-05 2023-10-01T09:39:54.138 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.138 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.139 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.139 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.994840+0000", 2023-10-01T09:39:54.139 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020361759999999998 2023-10-01T09:39:54.139 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.139 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.139 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.139 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:34:44.994873+0000", 2023-10-01T09:39:54.139 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3142000000000002e-05 2023-10-01T09:39:54.139 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.139 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.139 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.139 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.140 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.140 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:108564 9.1 e427/400 9:895d5137:::benchmark_data_smithi138_497829_object6179:head v 427'6627, mlcod=427'6625)", 2023-10-01T09:39:54.140 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:55.637242+0000", 2023-10-01T09:39:54.140 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 238.467659669, 2023-10-01T09:39:54.140 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.167424715, 2023-10-01T09:39:54.140 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.140 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.140 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.140 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.140 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.140 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.637242+0000", 2023-10-01T09:39:54.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.141 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.141 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.141 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.637242+0000", 2023-10-01T09:39:54.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.141 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.141 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.141 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.141 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.637242+0000", 2023-10-01T09:39:54.141 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:39:54.141 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.142 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.142 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.142 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.637250+0000", 2023-10-01T09:39:54.142 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3389999999999993e-06 2023-10-01T09:39:54.142 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.142 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.142 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.142 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.637260+0000", 2023-10-01T09:39:54.142 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6840000000000008e-06 2023-10-01T09:39:54.142 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.142 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.143 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.143 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.637266+0000", 2023-10-01T09:39:54.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7059999999999998e-06 2023-10-01T09:39:54.143 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.143 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.143 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.143 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.638253+0000", 2023-10-01T09:39:54.143 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00098667299999999998 2023-10-01T09:39:54.143 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.143 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.143 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.143 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.638273+0000", 2023-10-01T09:39:54.144 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0242999999999999e-05 2023-10-01T09:39:54.144 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.144 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.144 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.144 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.804633+0000", 2023-10-01T09:39:54.144 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.16636009900000001 2023-10-01T09:39:54.144 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.144 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.144 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.144 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.804667+0000", 2023-10-01T09:39:54.145 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3862999999999997e-05 2023-10-01T09:39:54.145 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.145 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.145 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.145 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.145 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.145 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:108569 9.1 e427/400 9:8f07d951:::benchmark_data_smithi138_497829_object6184:head v 427'6628, mlcod=427'6625)", 2023-10-01T09:39:54.145 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:55.638919+0000", 2023-10-01T09:39:54.145 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 238.46598336, 2023-10-01T09:39:54.145 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.165768103, 2023-10-01T09:39:54.145 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.146 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.146 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.146 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.146 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.146 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.638919+0000", 2023-10-01T09:39:54.146 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.146 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.146 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.146 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.146 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.638919+0000", 2023-10-01T09:39:54.146 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.146 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.147 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.147 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.147 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.638918+0000", 2023-10-01T09:39:54.147 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:39:54.147 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.147 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.147 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.147 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.638928+0000", 2023-10-01T09:39:54.147 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7720000000000006e-06 2023-10-01T09:39:54.147 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.147 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.148 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.148 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.791286+0000", 2023-10-01T09:39:54.148 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.15235870800000001 2023-10-01T09:39:54.148 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.148 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.148 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.148 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.791299+0000", 2023-10-01T09:39:54.148 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2269999999999999e-05 2023-10-01T09:39:54.148 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.148 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.148 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.148 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.794740+0000", 2023-10-01T09:39:54.149 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034412140000000002 2023-10-01T09:39:54.149 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.149 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.149 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.149 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.794759+0000", 2023-10-01T09:39:54.149 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8938000000000001e-05 2023-10-01T09:39:54.149 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.149 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.149 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.149 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.804670+0000", 2023-10-01T09:39:54.149 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0099107410000000007 2023-10-01T09:39:54.150 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.150 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.150 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.150 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:55.804687+0000", 2023-10-01T09:39:54.150 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7297999999999999e-05 2023-10-01T09:39:54.150 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.150 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.150 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.150 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.150 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.150 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:132658 9.a e436/431 9:549aa041:::benchmark_data_smithi138_497829_object30273:head v 436'8159, mlcod=405'1569)", 2023-10-01T09:39:54.150 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:36:38.289459+0000", 2023-10-01T09:39:54.151 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 195.81544349199999, 2023-10-01T09:39:54.151 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.247702749, 2023-10-01T09:39:54.151 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.151 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.151 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.151 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.151 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.151 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289459+0000", 2023-10-01T09:39:54.151 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.151 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.151 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.151 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.152 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289459+0000", 2023-10-01T09:39:54.152 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.152 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.152 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.152 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.152 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289458+0000", 2023-10-01T09:39:54.152 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:39:54.152 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.152 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.152 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.152 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289466+0000", 2023-10-01T09:39:54.153 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5660000000000007e-06 2023-10-01T09:39:54.153 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.153 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.153 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.153 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289479+0000", 2023-10-01T09:39:54.153 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2714000000000001e-05 2023-10-01T09:39:54.153 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.153 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.153 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.153 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289486+0000", 2023-10-01T09:39:54.153 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.391e-06 2023-10-01T09:39:54.154 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.154 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.154 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.154 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289548+0000", 2023-10-01T09:39:54.154 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2012000000000002e-05 2023-10-01T09:39:54.154 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.154 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.154 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.154 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.289575+0000", 2023-10-01T09:39:54.154 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6438e-05 2023-10-01T09:39:54.154 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.155 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.155 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.155 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537129+0000", 2023-10-01T09:39:54.155 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.247554408 2023-10-01T09:39:54.155 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.155 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.155 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.155 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537161+0000", 2023-10-01T09:39:54.155 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2141000000000003e-05 2023-10-01T09:39:54.155 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.155 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.155 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.156 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.156 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.156 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:132659 9.a e436/431 9:5345b610:::benchmark_data_smithi138_497829_object30274:head v 436'8160, mlcod=405'1569)", 2023-10-01T09:39:54.156 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:36:38.291551+0000", 2023-10-01T09:39:54.156 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 195.81335129799999, 2023-10-01T09:39:54.156 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24563221299999999, 2023-10-01T09:39:54.156 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.156 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.156 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.156 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.156 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.157 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.291551+0000", 2023-10-01T09:39:54.157 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.157 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.157 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.157 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.157 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.291551+0000", 2023-10-01T09:39:54.157 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.157 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.157 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.157 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.157 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.291551+0000", 2023-10-01T09:39:54.158 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:39:54.158 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.158 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.158 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.158 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.291554+0000", 2023-10-01T09:39:54.158 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7299999999999999e-06 2023-10-01T09:39:54.158 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.158 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.158 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.158 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.291560+0000", 2023-10-01T09:39:54.158 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4029999999999998e-06 2023-10-01T09:39:54.159 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.159 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.159 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.159 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.291564+0000", 2023-10-01T09:39:54.159 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8999999999999999e-06 2023-10-01T09:39:54.159 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.159 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.159 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.159 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.298221+0000", 2023-10-01T09:39:54.159 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0066577329999999999 2023-10-01T09:39:54.159 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.160 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.160 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.160 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.298245+0000", 2023-10-01T09:39:54.160 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3175000000000001e-05 2023-10-01T09:39:54.160 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.160 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.160 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.160 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537164+0000", 2023-10-01T09:39:54.160 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.238919627 2023-10-01T09:39:54.160 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.160 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.160 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.161 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537183+0000", 2023-10-01T09:39:54.161 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8912000000000001e-05 2023-10-01T09:39:54.161 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.161 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.161 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.161 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.161 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.161 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:132664 9.4 e436/400 9:24a98379:::benchmark_data_smithi138_497829_object30279:head v 436'8340, mlcod=436'8338)", 2023-10-01T09:39:54.161 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:36:38.303519+0000", 2023-10-01T09:39:54.161 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 195.80138270500001, 2023-10-01T09:39:54.162 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23341430599999999, 2023-10-01T09:39:54.162 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.162 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.162 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.162 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.162 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.162 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.303519+0000", 2023-10-01T09:39:54.162 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.162 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.162 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.162 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.163 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.303519+0000", 2023-10-01T09:39:54.163 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.163 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.163 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.163 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.163 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.303518+0000", 2023-10-01T09:39:54.163 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:39:54.163 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.163 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.163 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.163 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.303528+0000", 2023-10-01T09:39:54.163 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6739999999999992e-06 2023-10-01T09:39:54.164 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.164 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.164 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.164 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532367+0000", 2023-10-01T09:39:54.164 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22883935999999999 2023-10-01T09:39:54.164 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.164 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.164 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.164 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532376+0000", 2023-10-01T09:39:54.164 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5339999999999999e-06 2023-10-01T09:39:54.164 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.164 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.165 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.165 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532538+0000", 2023-10-01T09:39:54.165 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016260299999999999 2023-10-01T09:39:54.165 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.165 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.165 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.165 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532561+0000", 2023-10-01T09:39:54.165 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2904e-05 2023-10-01T09:39:54.165 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.165 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.165 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.165 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.536896+0000", 2023-10-01T09:39:54.166 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043344639999999997 2023-10-01T09:39:54.166 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.166 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.166 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.166 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.536934+0000", 2023-10-01T09:39:54.166 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8022000000000003e-05 2023-10-01T09:39:54.166 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.166 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.166 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.166 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.166 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.166 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:132665 9.3 e436/400 9:c75edcf0:::benchmark_data_smithi138_497829_object30280:head v 436'8280, mlcod=436'8278)", 2023-10-01T09:39:54.167 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:36:38.306765+0000", 2023-10-01T09:39:54.167 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 195.79813742600001, 2023-10-01T09:39:54.167 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.24329358100000001, 2023-10-01T09:39:54.167 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.167 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.167 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.167 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.167 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.167 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.306765+0000", 2023-10-01T09:39:54.167 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.167 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.168 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.168 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.168 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.306765+0000", 2023-10-01T09:39:54.168 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.168 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.168 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.168 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.168 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.306763+0000", 2023-10-01T09:39:54.168 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:39:54.168 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.168 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.169 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.169 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.306775+0000", 2023-10-01T09:39:54.169 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1369000000000001e-05 2023-10-01T09:39:54.169 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.169 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.169 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.169 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537194+0000", 2023-10-01T09:39:54.169 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23041979100000001 2023-10-01T09:39:54.169 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.169 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.169 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.169 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537208+0000", 2023-10-01T09:39:54.169 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3708e-05 2023-10-01T09:39:54.170 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.170 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.170 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.170 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537351+0000", 2023-10-01T09:39:54.170 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014312999999999999 2023-10-01T09:39:54.170 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.170 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.170 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.170 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.537373+0000", 2023-10-01T09:39:54.170 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2087e-05 2023-10-01T09:39:54.170 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.171 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.171 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.171 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.549826+0000", 2023-10-01T09:39:54.171 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012452420000000001 2023-10-01T09:39:54.171 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.171 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.171 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.171 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.550058+0000", 2023-10-01T09:39:54.171 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000232527 2023-10-01T09:39:54.171 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.171 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.171 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.172 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.172 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.172 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:132647 9.f e436/400 9:f2582533:::benchmark_data_smithi138_497829_object30262:head v 436'8314, mlcod=436'8312)", 2023-10-01T09:39:54.172 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:36:38.318653+0000", 2023-10-01T09:39:54.172 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 195.786249298, 2023-10-01T09:39:54.172 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.22333981, 2023-10-01T09:39:54.172 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.172 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.172 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.172 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.172 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.173 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.318653+0000", 2023-10-01T09:39:54.173 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.173 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.173 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.173 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.173 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.318653+0000", 2023-10-01T09:39:54.173 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.173 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.173 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.173 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.173 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.318652+0000", 2023-10-01T09:39:54.174 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:39:54.174 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.174 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.174 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.174 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.318661+0000", 2023-10-01T09:39:54.174 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8020000000000003e-06 2023-10-01T09:39:54.174 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.174 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.174 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.174 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532259+0000", 2023-10-01T09:39:54.174 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.213598017 2023-10-01T09:39:54.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.175 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.175 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.175 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532275+0000", 2023-10-01T09:39:54.175 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5883e-05 2023-10-01T09:39:54.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.175 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.175 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.175 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532434+0000", 2023-10-01T09:39:54.175 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000158286 2023-10-01T09:39:54.175 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.176 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.176 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.176 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.532462+0000", 2023-10-01T09:39:54.176 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8693000000000001e-05 2023-10-01T09:39:54.176 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.176 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.176 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.176 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.541945+0000", 2023-10-01T09:39:54.176 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094831150000000003 2023-10-01T09:39:54.177 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.177 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.177 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.177 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:36:38.541993+0000", 2023-10-01T09:39:54.177 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7264e-05 2023-10-01T09:39:54.177 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.177 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.177 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.177 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.177 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.177 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:147927 9.1 9:88c25810:::benchmark_data_smithi138_497829_object45542:head [delete] snapc 0=[] ondisk+write+known_if_redirected e444)", 2023-10-01T09:39:54.177 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:22.269510+0000", 2023-10-01T09:39:54.178 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.835392017, 2023-10-01T09:39:54.178 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.42430222200000001, 2023-10-01T09:39:54.178 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.178 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:39:54.178 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:54.178 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:54.178 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:54.178 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 147927 2023-10-01T09:39:54.178 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.178 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.178 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.179 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.179 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269510+0000", 2023-10-01T09:39:54.179 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.179 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.179 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.179 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.179 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269510+0000", 2023-10-01T09:39:54.179 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.179 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.179 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.180 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.180 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269508+0000", 2023-10-01T09:39:54.180 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:39:54.180 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.180 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.180 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.180 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269517+0000", 2023-10-01T09:39:54.180 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8389999999999996e-06 2023-10-01T09:39:54.180 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.180 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.180 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.180 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269531+0000", 2023-10-01T09:39:54.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3882e-05 2023-10-01T09:39:54.181 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.181 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.181 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.181 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269540+0000", 2023-10-01T09:39:54.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4409999999999993e-06 2023-10-01T09:39:54.181 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.181 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.181 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.181 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269579+0000", 2023-10-01T09:39:54.181 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9886999999999999e-05 2023-10-01T09:39:54.181 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269697+0000", 2023-10-01T09:39:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011801100000000001 2023-10-01T09:39:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,4,6", 2023-10-01T09:39:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269781+0000", 2023-10-01T09:39:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3877999999999999e-05 2023-10-01T09:39:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.182 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.183 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.273704+0000", 2023-10-01T09:39:54.183 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039223030000000002 2023-10-01T09:39:54.183 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.183 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.183 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:54.183 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.279487+0000", 2023-10-01T09:39:54.183 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057834059999999996 2023-10-01T09:39:54.183 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.183 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.183 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.183 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.286463+0000", 2023-10-01T09:39:54.183 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069760109999999998 2023-10-01T09:39:54.184 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.184 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:54.184 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.286474+0000", 2023-10-01T09:39:54.184 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0564000000000001e-05 2023-10-01T09:39:54.184 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.184 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.184 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.693718+0000", 2023-10-01T09:39:54.184 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.40724410300000002 2023-10-01T09:39:54.184 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.184 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.185 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.185 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.693812+0000", 2023-10-01T09:39:54.185 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4624999999999996e-05 2023-10-01T09:39:54.185 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.185 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.185 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.185 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.185 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.185 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:147929 9.1 9:84c1980f:::benchmark_data_smithi138_497829_object45544:head [delete] snapc 0=[] ondisk+write+known_if_redirected e444)", 2023-10-01T09:39:54.185 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:22.289771+0000", 2023-10-01T09:39:54.185 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.815130823, 2023-10-01T09:39:54.185 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.44011142599999997, 2023-10-01T09:39:54.186 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.186 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.186 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:54.186 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:54.186 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:54.186 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 147929 2023-10-01T09:39:54.186 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.186 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.186 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.186 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.186 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289771+0000", 2023-10-01T09:39:54.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289771+0000", 2023-10-01T09:39:54.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.187 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.187 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.187 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289770+0000", 2023-10-01T09:39:54.187 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:39:54.187 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289776+0000", 2023-10-01T09:39:54.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8130000000000001e-06 2023-10-01T09:39:54.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.188 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289786+0000", 2023-10-01T09:39:54.188 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0533999999999999e-05 2023-10-01T09:39:54.188 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.188 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.188 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.189 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289793+0000", 2023-10-01T09:39:54.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9110000000000004e-06 2023-10-01T09:39:54.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.189 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.189 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.290885+0000", 2023-10-01T09:39:54.189 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010919930000000001 2023-10-01T09:39:54.189 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.189 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.189 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:54.189 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.290895+0000", 2023-10-01T09:39:54.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9359999999999994e-06 2023-10-01T09:39:54.190 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.190 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.190 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.694724+0000", 2023-10-01T09:39:54.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.40382902199999998 2023-10-01T09:39:54.190 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.190 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.190 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.694835+0000", 2023-10-01T09:39:54.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111291 2023-10-01T09:39:54.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:39:54.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.695529+0000", 2023-10-01T09:39:54.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00069338699999999995 2023-10-01T09:39:54.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:54.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.711318+0000", 2023-10-01T09:39:54.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015789381000000002 2023-10-01T09:39:54.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.720238+0000", 2023-10-01T09:39:54.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0089196480000000005 2023-10-01T09:39:54.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.729790+0000", 2023-10-01T09:39:54.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0095524689999999992 2023-10-01T09:39:54.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.729838+0000", 2023-10-01T09:39:54.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7947e-05 2023-10-01T09:39:54.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.729883+0000", 2023-10-01T09:39:54.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4607000000000002e-05 2023-10-01T09:39:54.193 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.193 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.193 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.194 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:147933 9.1 9:89b639aa:::benchmark_data_smithi138_497829_object45548:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:39:54.194 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:22.305679+0000", 2023-10-01T09:39:54.194 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.79922355299999, 2023-10-01T09:39:54.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.424334777, 2023-10-01T09:39:54.194 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.194 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.194 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:54.194 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:54.194 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:54.194 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 147933 2023-10-01T09:39:54.194 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.195 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305679+0000", 2023-10-01T09:39:54.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305679+0000", 2023-10-01T09:39:54.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305677+0000", 2023-10-01T09:39:54.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:39:54.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305685+0000", 2023-10-01T09:39:54.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5050000000000001e-06 2023-10-01T09:39:54.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305702+0000", 2023-10-01T09:39:54.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7244999999999998e-05 2023-10-01T09:39:54.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305712+0000", 2023-10-01T09:39:54.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9410000000000002e-06 2023-10-01T09:39:54.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305776+0000", 2023-10-01T09:39:54.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4111999999999999e-05 2023-10-01T09:39:54.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.198 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:54.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305787+0000", 2023-10-01T09:39:54.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1102e-05 2023-10-01T09:39:54.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.198 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.703627+0000", 2023-10-01T09:39:54.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.397840311 2023-10-01T09:39:54.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.199 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.703767+0000", 2023-10-01T09:39:54.199 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013993999999999999 2023-10-01T09:39:54.199 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:39:54.199 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.703898+0000", 2023-10-01T09:39:54.199 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013054399999999999 2023-10-01T09:39:54.199 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.200 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:54.200 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.715395+0000", 2023-10-01T09:39:54.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011496908 2023-10-01T09:39:54.200 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.200 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.200 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.200 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.729920+0000", 2023-10-01T09:39:54.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014524789999999999 2023-10-01T09:39:54.200 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.200 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.200 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.201 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.729948+0000", 2023-10-01T09:39:54.201 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7948000000000001e-05 2023-10-01T09:39:54.201 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.201 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.201 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.201 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.729982+0000", 2023-10-01T09:39:54.201 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4576999999999998e-05 2023-10-01T09:39:54.201 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.201 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.201 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.201 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.730013+0000", 2023-10-01T09:39:54.201 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1300999999999997e-05 2023-10-01T09:39:54.201 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:147961 9.1 9:8c87c16e:::benchmark_data_smithi138_497829_object45576:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:39:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:22.469797+0000", 2023-10-01T09:39:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.63510535899999, 2023-10-01T09:39:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.260341398, 2023-10-01T09:39:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.202 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 147961 2023-10-01T09:39:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469797+0000", 2023-10-01T09:39:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.203 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469797+0000", 2023-10-01T09:39:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469795+0000", 2023-10-01T09:39:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:39:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469806+0000", 2023-10-01T09:39:54.204 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1085e-05 2023-10-01T09:39:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469828+0000", 2023-10-01T09:39:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2636000000000001e-05 2023-10-01T09:39:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469840+0000", 2023-10-01T09:39:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.136e-05 2023-10-01T09:39:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.205 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469896+0000", 2023-10-01T09:39:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6166999999999999e-05 2023-10-01T09:39:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469909+0000", 2023-10-01T09:39:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2568999999999999e-05 2023-10-01T09:39:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.206 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.707773+0000", 2023-10-01T09:39:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23786492100000001 2023-10-01T09:39:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.707891+0000", 2023-10-01T09:39:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117571 2023-10-01T09:39:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:39:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.707971+0000", 2023-10-01T09:39:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9968000000000004e-05 2023-10-01T09:39:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.207 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.715402+0000", 2023-10-01T09:39:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0074309370000000003 2023-10-01T09:39:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.730048+0000", 2023-10-01T09:39:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014646144 2023-10-01T09:39:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.208 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.730077+0000", 2023-10-01T09:39:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8615e-05 2023-10-01T09:39:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.730109+0000", 2023-10-01T09:39:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2013e-05 2023-10-01T09:39:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.730138+0000", 2023-10-01T09:39:54.209 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9527e-05 2023-10-01T09:39:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:147973 9.1 9:85c71572:::benchmark_data_smithi138_497829_object45588:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:39:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:22.511296+0000", 2023-10-01T09:39:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 151.59360584199999, 2023-10-01T09:39:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23218191699999999, 2023-10-01T09:39:54.210 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 147973 2023-10-01T09:39:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511296+0000", 2023-10-01T09:39:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.211 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511296+0000", 2023-10-01T09:39:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511294+0000", 2023-10-01T09:39:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:39:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.212 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511305+0000", 2023-10-01T09:39:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0614999999999999e-05 2023-10-01T09:39:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511328+0000", 2023-10-01T09:39:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3229000000000001e-05 2023-10-01T09:39:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.213 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511338+0000", 2023-10-01T09:39:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.012e-05 2023-10-01T09:39:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511512+0000", 2023-10-01T09:39:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017422800000000001 2023-10-01T09:39:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:39:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511524+0000", 2023-10-01T09:39:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2063e-05 2023-10-01T09:39:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.711357+0000", 2023-10-01T09:39:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19983310900000001 2023-10-01T09:39:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.711498+0000", 2023-10-01T09:39:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014025499999999999 2023-10-01T09:39:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:39:54.215 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.711635+0000", 2023-10-01T09:39:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013784999999999999 2023-10-01T09:39:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.715508+0000", 2023-10-01T09:39:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0038724050000000002 2023-10-01T09:39:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.730227+0000", 2023-10-01T09:39:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014719603 2023-10-01T09:39:54.216 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.743379+0000", 2023-10-01T09:39:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013151949 2023-10-01T09:39:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.743431+0000", 2023-10-01T09:39:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2051e-05 2023-10-01T09:39:54.217 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.743478+0000", 2023-10-01T09:39:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6755999999999998e-05 2023-10-01T09:39:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:188161 9.1 9:8412981f:::benchmark_data_smithi138_497829_object85776:head [delete] snapc 0=[] ondisk+write+known_if_redirected e459)", 2023-10-01T09:39:54.218 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:39:00.913551+0000", 2023-10-01T09:39:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 53.191350729, 2023-10-01T09:39:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.13231204199999999, 2023-10-01T09:39:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:39:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:39:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:39:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 188161 2023-10-01T09:39:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.219 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.913551+0000", 2023-10-01T09:39:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.913551+0000", 2023-10-01T09:39:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.913549+0000", 2023-10-01T09:39:54.220 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:39:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.913560+0000", 2023-10-01T09:39:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0995000000000001e-05 2023-10-01T09:39:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.913577+0000", 2023-10-01T09:39:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7611000000000001e-05 2023-10-01T09:39:54.221 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.913589+0000", 2023-10-01T09:39:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1552000000000001e-05 2023-10-01T09:39:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.913644+0000", 2023-10-01T09:39:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5095e-05 2023-10-01T09:39:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.913778+0000", 2023-10-01T09:39:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013408699999999999 2023-10-01T09:39:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:39:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.913885+0000", 2023-10-01T09:39:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106741 2023-10-01T09:39:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.920368+0000", 2023-10-01T09:39:54.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064826730000000004 2023-10-01T09:39:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:39:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.920396+0000", 2023-10-01T09:39:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8265000000000001e-05 2023-10-01T09:39:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:39:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:01.045761+0000", 2023-10-01T09:39:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.125365481 2023-10-01T09:39:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:01.045820+0000", 2023-10-01T09:39:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8928999999999998e-05 2023-10-01T09:39:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:01.045863+0000", 2023-10-01T09:39:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.32e-05 2023-10-01T09:39:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.225 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:188157 9.c e459/446 9:36fa1930:::benchmark_data_smithi138_497829_object85772:head v 459'11967, mlcod=459'11965)", 2023-10-01T09:39:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:39:00.914382+0000", 2023-10-01T09:39:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 53.190520178, 2023-10-01T09:39:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.130316037, 2023-10-01T09:39:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.226 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.914382+0000", 2023-10-01T09:39:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.914382+0000", 2023-10-01T09:39:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.914381+0000", 2023-10-01T09:39:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:39:54.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.914392+0000", 2023-10-01T09:39:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0594000000000001e-05 2023-10-01T09:39:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.914401+0000", 2023-10-01T09:39:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0429999999999996e-06 2023-10-01T09:39:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.228 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.914408+0000", 2023-10-01T09:39:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8199999999999999e-06 2023-10-01T09:39:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.914737+0000", 2023-10-01T09:39:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00032927400000000001 2023-10-01T09:39:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.229 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.914758+0000", 2023-10-01T09:39:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1421999999999999e-05 2023-10-01T09:39:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:01.044640+0000", 2023-10-01T09:39:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.12988180899999999 2023-10-01T09:39:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:01.044698+0000", 2023-10-01T09:39:54.230 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7948999999999997e-05 2023-10-01T09:39:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.46067.0:188169 9.c e459/446 9:3feb9b36:::benchmark_data_smithi138_497829_object85784:head v 459'11968, mlcod=459'11965)", 2023-10-01T09:39:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:39:00.937095+0000", 2023-10-01T09:39:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 53.167807388999996, 2023-10-01T09:39:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.14418440599999999, 2023-10-01T09:39:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:39:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:39:54.231 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:39:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:39:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.937095+0000", 2023-10-01T09:39:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:39:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.937095+0000", 2023-10-01T09:39:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:39:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:39:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.937094+0000", 2023-10-01T09:39:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:39:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:39:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:00.937106+0000", 2023-10-01T09:39:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2247e-05 2023-10-01T09:39:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:39:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:01.045415+0000", 2023-10-01T09:39:54.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.108309218 2023-10-01T09:39:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:39:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:01.045425+0000", 2023-10-01T09:39:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0027000000000001e-05 2023-10-01T09:39:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:39:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:01.058353+0000", 2023-10-01T09:39:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012927624 2023-10-01T09:39:54.234 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:39:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:01.058374+0000", 2023-10-01T09:39:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1379999999999999e-05 2023-10-01T09:39:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:39:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:01.081234+0000", 2023-10-01T09:39:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.022860195 2023-10-01T09:39:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:39:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:39:54.235 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:39:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:39:01.081279+0000", 2023-10-01T09:39:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4935999999999998e-05 2023-10-01T09:39:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:39:54.236 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:39:54.236 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:39:54.236 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:54.268 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:54.313 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:54.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:54.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:54.416 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:39:54.442 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:54.484 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:39:54.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:54.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:39:54.589 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:54.615 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:54.965 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:54.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:54.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:55.072 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:55.102 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:55.210 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:39:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:39:54] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.33.4" 2023-10-01T09:39:55.225 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:39:55.227 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:39:55.244 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:39:55.266 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:39:55.286 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:39:55.304 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:39:55.354 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:55.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:55.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:55.431 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:39:55.431 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:39:55.431 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:39:55.432 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:39:55.432 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:39:55.432 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:39:55.432 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:39:55.432 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:39:55.432 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:39:55.432 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:39:55.432 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:39:55.459 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:55.490 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:55.536 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:55.539 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:55 smithi190 ceph-mon[160435]: pgmap v1346: 89 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12772/84 objects degraded (15204.762%); 0 B/s, 174 objects/s recovering 2023-10-01T09:39:55.539 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:55 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1895770973' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:55.539 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:55.539 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:39:55.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:55.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:55.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:55 smithi138 ceph-mon[192513]: pgmap v1346: 89 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12772/84 objects degraded (15204.762%); 0 B/s, 174 objects/s recovering 2023-10-01T09:39:55.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:55 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1895770973' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:55.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:55.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:39:55.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:55 smithi138 ceph-mon[185886]: pgmap v1346: 89 pgs: 1 active+recovering+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12772/84 objects degraded (15204.762%); 0 B/s, 174 objects/s recovering 2023-10-01T09:39:55.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:55 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1895770973' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:39:55.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:39:55.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:39:55.623 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:39:55.641 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:55.668 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:55.697 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 2, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:39:55.697 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:39:55.697 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 1, 7, 5, 6, 4, 2, 3] 2023-10-01T09:39:55.698 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd out 2 2023-10-01T09:39:55.739 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:55.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:55.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:55.843 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:55.872 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:55.916 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:55.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:55.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:55.952 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:39:56.019 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:39:56.046 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:56.242 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:39:56.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:56.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:39:56.347 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:56.373 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:56.416 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:56.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:56.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:56.520 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:39:56.552 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:56.684 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:56.706 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:39:56.708 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:56 smithi138 ceph-mon[192513]: pgmap v1347: 89 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12772/84 objects degraded (15204.762%); 0 B/s, 242 objects/s recovering 2023-10-01T09:39:56.709 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:56 smithi138 ceph-mon[185886]: pgmap v1347: 89 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12772/84 objects degraded (15204.762%); 0 B/s, 242 objects/s recovering 2023-10-01T09:39:56.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:56.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:39:56.735 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:56 smithi190 ceph-mon[160435]: pgmap v1347: 89 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 12772/84 objects degraded (15204.762%); 0 B/s, 242 objects/s recovering 2023-10-01T09:39:56.811 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:56.838 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:56.885 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:56.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:56.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:56.988 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:39:57.015 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:57.057 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:39:57.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:57.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:39:57.160 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:39:57.192 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:57.337 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:39:57.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:57.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:39:57.440 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:57.467 INFO:teuthology.orchestra.run.smithi138.stderr:pool 'unique_pool_4' removed 2023-10-01T09:39:57.469 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:57.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4245580477' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:39:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:57 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:39:57.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4245580477' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:39:57.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:57 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:39:57.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4245580477' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:39:57.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:57 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:39:58.292 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:58.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:58.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:58.395 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:39:58.415 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:39:58.430 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:58.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:58 smithi138 ceph-mon[192513]: failed to encode map e470 with expected crc 2023-10-01T09:39:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:58 smithi138 ceph-mon[192513]: failed to encode map e470 with expected crc 2023-10-01T09:39:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:58 smithi138 ceph-mon[192513]: failed to encode map e470 with expected crc 2023-10-01T09:39:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:58 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:39:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:58 smithi138 ceph-mon[192513]: osdmap e470: 8 total, 8 up, 8 in 2023-10-01T09:39:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:58 smithi138 ceph-mon[192513]: pgmap v1349: 73 pgs: 2 active+recovery_wait+degraded, 71 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 23/84 objects degraded (27.381%) 2023-10-01T09:39:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:58 smithi138 ceph-mon[185886]: failed to encode map e470 with expected crc 2023-10-01T09:39:58.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:58 smithi138 ceph-mon[185886]: failed to encode map e470 with expected crc 2023-10-01T09:39:58.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:58 smithi138 ceph-mon[185886]: failed to encode map e470 with expected crc 2023-10-01T09:39:58.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:58 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:39:58.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:58 smithi138 ceph-mon[185886]: osdmap e470: 8 total, 8 up, 8 in 2023-10-01T09:39:58.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:58 smithi138 ceph-mon[185886]: pgmap v1349: 73 pgs: 2 active+recovery_wait+degraded, 71 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 23/84 objects degraded (27.381%) 2023-10-01T09:39:58.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:58 smithi190 ceph-mon[160435]: failed to encode map e470 with expected crc 2023-10-01T09:39:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:58 smithi190 ceph-mon[160435]: failed to encode map e470 with expected crc 2023-10-01T09:39:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:58 smithi190 ceph-mon[160435]: failed to encode map e470 with expected crc 2023-10-01T09:39:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:58 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:39:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:58 smithi190 ceph-mon[160435]: osdmap e470: 8 total, 8 up, 8 in 2023-10-01T09:39:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:58 smithi190 ceph-mon[160435]: pgmap v1349: 73 pgs: 2 active+recovery_wait+degraded, 71 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 23/84 objects degraded (27.381%) 2023-10-01T09:39:58.851 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:39:58.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:58.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:39:58.955 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:39:58.990 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:59.088 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:39:59.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:59.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:39:59.128 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T09:39:59.128 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T09:39:59.128 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-10-01T09:39:59.129 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-10-01T09:39:59.192 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:39:59.223 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:59.424 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:59.452 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:39:59.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:59.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:39:59.469 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:39:59 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 23/84 objects degraded (27.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:39:59.469 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:59 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 23/84 objects degraded (27.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:39:59.492 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:39:59 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 23/84 objects degraded (27.381%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T09:39:59.557 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:39:59.586 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:59.602 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:39:59.689 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:39:59.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:59.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:39:59.794 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:39:59.822 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:39:59.918 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:39:59.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:39:59.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:39:59.965 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:39:59.966 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:40:00.011 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:39:59 smithi138 conmon[185862]: 2023-10-01T09:39:59.760+0000 7f70d64ec700 -1 mon.a@0(leader).osd e470 definitely_dead 0 2023-10-01T09:40:00.022 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:00.052 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:00.498 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:40:00.500 INFO:teuthology.orchestra.run.smithi138.stderr:marked out osd.2. 2023-10-01T09:40:00.711 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2634960162' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:40:00.711 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:40:00.711 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/179967956' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:40:00.711 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[192513]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-01T09:40:00.711 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[192513]: Health detail: HEALTH_WARN Degraded data redundancy: 23/84 objects degraded (27.381%), 2 pgs degraded 2023-10-01T09:40:00.711 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[192513]: [WRN] PG_DEGRADED: Degraded data redundancy: 23/84 objects degraded (27.381%), 2 pgs degraded 2023-10-01T09:40:00.711 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[192513]: pg 2.1 is active+recovery_wait+degraded, acting [0,2,1] 2023-10-01T09:40:00.712 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[192513]: pg 2.3 is active+recovery_wait+degraded, acting [5,2,3] 2023-10-01T09:40:00.712 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[192513]: pgmap v1350: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-01T09:40:00.712 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2634960162' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:40:00.712 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:40:00.712 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/179967956' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:40:00.712 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[185886]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-01T09:40:00.712 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[185886]: Health detail: HEALTH_WARN Degraded data redundancy: 23/84 objects degraded (27.381%), 2 pgs degraded 2023-10-01T09:40:00.712 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[185886]: [WRN] PG_DEGRADED: Degraded data redundancy: 23/84 objects degraded (27.381%), 2 pgs degraded 2023-10-01T09:40:00.712 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[185886]: pg 2.1 is active+recovery_wait+degraded, acting [0,2,1] 2023-10-01T09:40:00.712 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[185886]: pg 2.3 is active+recovery_wait+degraded, acting [5,2,3] 2023-10-01T09:40:00.713 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:00 smithi138 ceph-mon[185886]: pgmap v1350: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-01T09:40:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:00 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2634960162' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:40:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:00 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:40:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:00 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/179967956' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-10-01T09:40:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:00 smithi190 ceph-mon[160435]: Client client.admin marked osd.2 out, while it was still marked up 2023-10-01T09:40:00.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:00 smithi190 ceph-mon[160435]: Health detail: HEALTH_WARN Degraded data redundancy: 23/84 objects degraded (27.381%), 2 pgs degraded 2023-10-01T09:40:00.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:00 smithi190 ceph-mon[160435]: [WRN] PG_DEGRADED: Degraded data redundancy: 23/84 objects degraded (27.381%), 2 pgs degraded 2023-10-01T09:40:00.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:00 smithi190 ceph-mon[160435]: pg 2.1 is active+recovery_wait+degraded, acting [0,2,1] 2023-10-01T09:40:00.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:00 smithi190 ceph-mon[160435]: pg 2.3 is active+recovery_wait+degraded, acting [5,2,3] 2023-10-01T09:40:00.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:00 smithi190 ceph-mon[160435]: pgmap v1350: 73 pgs: 73 active+clean; 4.6 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2023-10-01T09:40:00.847 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:40:01.197 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:40:01.558 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:01.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:01.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:01.663 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:40:01.688 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:01 smithi190 ceph-mon[160435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/84 objects degraded (27.381%), 2 pgs degraded) 2023-10-01T09:40:01.688 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:01 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:40:01.688 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:01 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:01.688 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:01 smithi190 ceph-mon[160435]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:01 smithi190 ceph-mon[160435]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:01 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:40:01.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/179967956' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-01T09:40:01.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:01 smithi190 ceph-mon[160435]: osdmap e471: 8 total, 8 up, 7 in 2023-10-01T09:40:01.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:01 smithi190 ceph-mon[160435]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:01 smithi190 ceph-mon[160435]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:01 smithi190 ceph-mon[160435]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.691 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:01.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[185886]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/84 objects degraded (27.381%), 2 pgs degraded) 2023-10-01T09:40:01.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:40:01.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:01.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[185886]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[185886]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:40:01.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/179967956' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-01T09:40:01.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[185886]: osdmap e471: 8 total, 8 up, 7 in 2023-10-01T09:40:01.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[185886]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[185886]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[185886]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[192513]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/84 objects degraded (27.381%), 2 pgs degraded) 2023-10-01T09:40:01.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:40:01.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:01.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[192513]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[192513]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:40:01.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/179967956' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-10-01T09:40:01.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[192513]: osdmap e471: 8 total, 8 up, 7 in 2023-10-01T09:40:01.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[192513]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[192513]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:01 smithi138 ceph-mon[192513]: failed to encode map e471 with expected crc 2023-10-01T09:40:01.836 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:40:01.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:01.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:40:01.940 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:01.974 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:02.115 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:40:02.134 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:40:02.154 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:40:02.171 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:40:02.172 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:02.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:02.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:02.189 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:40:02.208 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:40:02.225 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:40:02.243 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:40:02.244 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:40:02.244 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:40:02.244 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:40:02.244 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:40:02.244 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:40:02.244 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:40:02.244 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:40:02.244 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:40:02.277 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:02.304 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:02.348 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:02.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:02.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:02.452 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:02.478 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:02 smithi190 ceph-mon[160435]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:02 smithi190 ceph-mon[160435]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:02 smithi190 ceph-mon[160435]: osdmap e472: 8 total, 8 up, 7 in 2023-10-01T09:40:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:02 smithi190 ceph-mon[160435]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:02 smithi190 ceph-mon[160435]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:02 smithi190 ceph-mon[160435]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:02 smithi190 ceph-mon[160435]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:02 smithi190 ceph-mon[160435]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:02 smithi190 ceph-mon[160435]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:02 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3710016642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:02.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:02 smithi190 ceph-mon[160435]: pgmap v1353: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-01T09:40:02.727 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:40:02.727 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:40:02.727 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:40:02.727 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:40:02.728 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:40:02.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[192513]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[192513]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[192513]: osdmap e472: 8 total, 8 up, 7 in 2023-10-01T09:40:02.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[192513]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.777 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[192513]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.778 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[192513]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.778 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[192513]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.778 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[192513]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.778 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[192513]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.778 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3710016642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:02.778 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[192513]: pgmap v1353: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-01T09:40:02.778 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[185886]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.778 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[185886]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.778 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[185886]: osdmap e472: 8 total, 8 up, 7 in 2023-10-01T09:40:02.779 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[185886]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.779 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[185886]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.779 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[185886]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.779 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[185886]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.779 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[185886]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.779 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[185886]: failed to encode map e472 with expected crc 2023-10-01T09:40:02.779 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3710016642' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:02.779 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:02 smithi138 ceph-mon[185886]: pgmap v1353: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail 2023-10-01T09:40:03.228 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:03.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:03.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:03.333 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:03.360 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:03.403 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:03.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:03.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:03.506 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:03.537 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:03.567 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:40:03.585 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:03.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:03.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:03.625 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:40:03.691 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:03.718 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:03.775 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:03.777 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:03 smithi138 ceph-mon[185886]: failed to encode map e473 with expected crc 2023-10-01T09:40:03.777 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:03 smithi138 ceph-mon[185886]: failed to encode map e473 with expected crc 2023-10-01T09:40:03.777 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:03 smithi138 ceph-mon[185886]: osdmap e473: 8 total, 8 up, 7 in 2023-10-01T09:40:03.777 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:03 smithi138 ceph-mon[185886]: failed to encode map e473 with expected crc 2023-10-01T09:40:03.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:03.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:03.779 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:03 smithi138 ceph-mon[192513]: failed to encode map e473 with expected crc 2023-10-01T09:40:03.779 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:03 smithi138 ceph-mon[192513]: failed to encode map e473 with expected crc 2023-10-01T09:40:03.779 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:03 smithi138 ceph-mon[192513]: osdmap e473: 8 total, 8 up, 7 in 2023-10-01T09:40:03.779 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:03 smithi138 ceph-mon[192513]: failed to encode map e473 with expected crc 2023-10-01T09:40:03.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:03 smithi190 ceph-mon[160435]: failed to encode map e473 with expected crc 2023-10-01T09:40:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:03 smithi190 ceph-mon[160435]: failed to encode map e473 with expected crc 2023-10-01T09:40:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:03 smithi190 ceph-mon[160435]: osdmap e473: 8 total, 8 up, 7 in 2023-10-01T09:40:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:03 smithi190 ceph-mon[160435]: failed to encode map e473 with expected crc 2023-10-01T09:40:03.879 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:03.906 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:03.932 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:40:03.950 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:03.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:03.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:03.988 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:04.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:04.083 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:04.125 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:04.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:04.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:04.229 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:04.257 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:04.300 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:04.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:04.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:04.404 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:04.432 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:04.632 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:04.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:04.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:04.724 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:04 smithi138 ceph-mon[185886]: failed to encode map e474 with expected crc 2023-10-01T09:40:04.733 INFO:teuthology.orchestra.run.smithi138.stderr:pool 'unique_pool_5' created 2023-10-01T09:40:04.736 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:04.763 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:04.808 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:04.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:04.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:04.912 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:04.940 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:05.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:04 smithi190 ceph-mon[160435]: failed to encode map e474 with expected crc 2023-10-01T09:40:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:04 smithi190 ceph-mon[160435]: osdmap e474: 8 total, 8 up, 7 in 2023-10-01T09:40:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:04 smithi190 ceph-mon[160435]: failed to encode map e474 with expected crc 2023-10-01T09:40:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/35109059' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-01T09:40:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:04 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-01T09:40:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:04 smithi190 ceph-mon[160435]: pgmap v1356: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:40:05.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:40:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:40:04] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-01T09:40:05.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:04 smithi138 ceph-mon[185886]: osdmap e474: 8 total, 8 up, 7 in 2023-10-01T09:40:05.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:04 smithi138 ceph-mon[185886]: failed to encode map e474 with expected crc 2023-10-01T09:40:05.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/35109059' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-01T09:40:05.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:04 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-01T09:40:05.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:04 smithi138 ceph-mon[185886]: pgmap v1356: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:40:05.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:04 smithi138 ceph-mon[192513]: failed to encode map e474 with expected crc 2023-10-01T09:40:05.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:04 smithi138 ceph-mon[192513]: osdmap e474: 8 total, 8 up, 7 in 2023-10-01T09:40:05.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:04 smithi138 ceph-mon[192513]: failed to encode map e474 with expected crc 2023-10-01T09:40:05.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/35109059' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-01T09:40:05.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:04 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-01T09:40:05.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:04 smithi138 ceph-mon[192513]: pgmap v1356: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:40:05.187 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:05.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:05.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:05.291 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:05.318 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:05.325 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:40:05.624 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:40:05.624 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:40:05.624 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:40:05.624 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:40:05.624 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.625 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "MOSDPGPush(8.e 369/361 [PushOp(8:71da5c83:::benchmark_data_smithi138_431617_object81342:head, version: 342'5187, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:71da5c83:::benchmark_data_smithi138_431617_object81342:head@342'5187, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-01T09:40:05.625 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:45.778621+0000", 2023-10-01T09:40:05.625 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 439.84339378599998, 2023-10-01T09:40:05.625 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.101742287, 2023-10-01T09:40:05.625 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.625 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:40:05.625 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.625 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.625 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.625 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778621+0000", 2023-10-01T09:40:05.625 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.625 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.625 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.626 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.626 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778621+0000", 2023-10-01T09:40:05.626 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:40:05.626 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.626 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.626 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.626 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778620+0000", 2023-10-01T09:40:05.626 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9847999999999999e-05 2023-10-01T09:40:05.626 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.627 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.627 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.627 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778690+0000", 2023-10-01T09:40:05.627 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.265e-05 2023-10-01T09:40:05.627 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.627 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.627 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.627 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778733+0000", 2023-10-01T09:40:05.627 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9190000000000004e-06 2023-10-01T09:40:05.627 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.627 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.627 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.627 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778739+0000", 2023-10-01T09:40:05.628 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.101418614 2023-10-01T09:40:05.628 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.628 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.628 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.628 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.880157+0000", 2023-10-01T09:40:05.628 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0553e-05 2023-10-01T09:40:05.628 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.628 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.628 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.628 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.880168+0000", 2023-10-01T09:40:05.628 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00019595600000000001 2023-10-01T09:40:05.628 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.629 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.629 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.629 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.880364+0000", 2023-10-01T09:40:05.629 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.101742287 2023-10-01T09:40:05.629 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.629 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.629 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.629 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.629 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.629 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "pg_backfill(progress 8.e e 369/369 lb 8:71d9e500:::benchmark_data_smithi138_431617_object97540:head)", 2023-10-01T09:40:05.629 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:45.778749+0000", 2023-10-01T09:40:05.630 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 439.84326583299998, 2023-10-01T09:40:05.630 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.10167633199999999, 2023-10-01T09:40:05.630 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.630 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:40:05.630 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.630 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.630 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.630 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778749+0000", 2023-10-01T09:40:05.630 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.630 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.630 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.631 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.631 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778749+0000", 2023-10-01T09:40:05.631 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:40:05.631 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.631 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.631 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.631 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778749+0000", 2023-10-01T09:40:05.631 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3109999999999996e-06 2023-10-01T09:40:05.631 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.631 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.631 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.631 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778754+0000", 2023-10-01T09:40:05.632 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2940000000000002e-06 2023-10-01T09:40:05.632 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.632 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.632 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.632 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778761+0000", 2023-10-01T09:40:05.632 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9700000000000001e-06 2023-10-01T09:40:05.632 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.632 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.632 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.632 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.778765+0000", 2023-10-01T09:40:05.632 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.10161822099999999 2023-10-01T09:40:05.632 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.633 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.633 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.633 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.880383+0000", 2023-10-01T09:40:05.633 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9059999999999997e-06 2023-10-01T09:40:05.633 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.633 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.633 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.633 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.880393+0000", 2023-10-01T09:40:05.633 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2981000000000002e-05 2023-10-01T09:40:05.633 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.633 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.634 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.634 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:45.880426+0000", 2023-10-01T09:40:05.634 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.10167633199999999 2023-10-01T09:40:05.634 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.634 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.634 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.634 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.634 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.634 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:177431 8.4 8:24757737:::benchmark_data_smithi138_431617_object76020:head [delete] snapc 0=[] ondisk+write+known_if_redirected e374)", 2023-10-01T09:40:05.634 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:59.731362+0000", 2023-10-01T09:40:05.634 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 425.89065292700002, 2023-10-01T09:40:05.634 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.091568272000000006, 2023-10-01T09:40:05.635 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.635 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.635 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:40:05.635 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:40:05.635 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:40:05.635 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 177431 2023-10-01T09:40:05.635 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.635 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.635 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.635 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.635 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731362+0000", 2023-10-01T09:40:05.636 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.636 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.636 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.636 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.636 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731362+0000", 2023-10-01T09:40:05.636 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:40:05.636 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.636 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.636 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.636 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731361+0000", 2023-10-01T09:40:05.636 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0410000000000003e-06 2023-10-01T09:40:05.637 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.637 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.637 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.637 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731367+0000", 2023-10-01T09:40:05.637 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2029999999999992e-06 2023-10-01T09:40:05.637 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.637 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.637 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.637 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731375+0000", 2023-10-01T09:40:05.637 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4909999999999999e-06 2023-10-01T09:40:05.637 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.638 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.638 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.638 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731380+0000", 2023-10-01T09:40:05.638 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7421e-05 2023-10-01T09:40:05.638 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.638 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.638 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.638 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731417+0000", 2023-10-01T09:40:05.638 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3317000000000003e-05 2023-10-01T09:40:05.638 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.638 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.638 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.639 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.731491+0000", 2023-10-01T09:40:05.639 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.073553882000000001 2023-10-01T09:40:05.639 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.639 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.639 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:40:05.639 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.805044+0000", 2023-10-01T09:40:05.639 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0070079729999999998 2023-10-01T09:40:05.639 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.639 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.639 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.639 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.812052+0000", 2023-10-01T09:40:05.639 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0210999999999999e-05 2023-10-01T09:40:05.640 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.640 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.640 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:40:05.640 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.812063+0000", 2023-10-01T09:40:05.640 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010705756 2023-10-01T09:40:05.640 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.640 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.640 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.640 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.822768+0000", 2023-10-01T09:40:05.640 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3562e-05 2023-10-01T09:40:05.640 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.640 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.641 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.641 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.822822+0000", 2023-10-01T09:40:05.641 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010863500000000001 2023-10-01T09:40:05.641 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.641 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.641 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.641 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.822931+0000", 2023-10-01T09:40:05.641 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.091568272000000006 2023-10-01T09:40:05.641 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.641 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.641 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.641 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.642 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.642 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:177437 8.4 8:218da7f3:::benchmark_data_smithi138_431617_object76026:head [delete] snapc 0=[] ondisk+write+known_if_redirected e374)", 2023-10-01T09:40:05.642 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:32:59.736172+0000", 2023-10-01T09:40:05.642 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 425.88584363400003, 2023-10-01T09:40:05.642 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.087005369999999999, 2023-10-01T09:40:05.642 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.642 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.642 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:40:05.642 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:40:05.642 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:40:05.642 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 177437 2023-10-01T09:40:05.642 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.643 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.643 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.643 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.643 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.736172+0000", 2023-10-01T09:40:05.643 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.643 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.643 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.643 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.643 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.736172+0000", 2023-10-01T09:40:05.643 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:40:05.643 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.643 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.644 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.644 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.736170+0000", 2023-10-01T09:40:05.644 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9699999999999999e-06 2023-10-01T09:40:05.644 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.644 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.644 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.644 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.736178+0000", 2023-10-01T09:40:05.644 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.068732363000000005 2023-10-01T09:40:05.644 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.644 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.644 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.645 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.804910+0000", 2023-10-01T09:40:05.645 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2921999999999998e-05 2023-10-01T09:40:05.645 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.645 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.645 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.645 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.804933+0000", 2023-10-01T09:40:05.645 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003639083 2023-10-01T09:40:05.645 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.645 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.645 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.645 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.808572+0000", 2023-10-01T09:40:05.646 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000132459 2023-10-01T09:40:05.646 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.646 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.646 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.646 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.808705+0000", 2023-10-01T09:40:05.646 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000144124 2023-10-01T09:40:05.646 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.646 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.646 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:40:05.646 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.808849+0000", 2023-10-01T09:40:05.646 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014097115 2023-10-01T09:40:05.647 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.647 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.647 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:40:05.647 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.822946+0000", 2023-10-01T09:40:05.647 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8383000000000002e-05 2023-10-01T09:40:05.647 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.647 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.647 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.647 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.822984+0000", 2023-10-01T09:40:05.647 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8476000000000001e-05 2023-10-01T09:40:05.647 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.647 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.648 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.648 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.823033+0000", 2023-10-01T09:40:05.648 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3470000000000002e-05 2023-10-01T09:40:05.648 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.648 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.648 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.648 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.823076+0000", 2023-10-01T09:40:05.648 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000100905 2023-10-01T09:40:05.648 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.648 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.648 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.648 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:32:59.823177+0000", 2023-10-01T09:40:05.649 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.087005369999999999 2023-10-01T09:40:05.649 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.649 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.649 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.649 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.649 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.649 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:178396 8.4 8:230afef1:::benchmark_data_smithi138_431617_object76985:head [delete] snapc 0=[] ondisk+write+known_if_redirected e374)", 2023-10-01T09:40:05.649 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:01.128189+0000", 2023-10-01T09:40:05.649 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 424.49382661999999, 2023-10-01T09:40:05.649 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17065782800000001, 2023-10-01T09:40:05.649 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.649 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.650 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:40:05.650 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:40:05.650 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:40:05.650 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 178396 2023-10-01T09:40:05.650 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.650 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.650 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.650 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.650 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128189+0000", 2023-10-01T09:40:05.650 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.650 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.651 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.651 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.651 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128189+0000", 2023-10-01T09:40:05.651 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:40:05.651 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.651 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.651 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.651 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128187+0000", 2023-10-01T09:40:05.651 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3019999999999995e-06 2023-10-01T09:40:05.651 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.652 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.652 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.652 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128195+0000", 2023-10-01T09:40:05.652 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4068e-05 2023-10-01T09:40:05.652 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.652 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.652 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.652 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128209+0000", 2023-10-01T09:40:05.652 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0709999999999995e-06 2023-10-01T09:40:05.652 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.652 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.653 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.653 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128217+0000", 2023-10-01T09:40:05.653 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3971999999999998e-05 2023-10-01T09:40:05.653 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.653 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.653 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.653 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128271+0000", 2023-10-01T09:40:05.653 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000141423 2023-10-01T09:40:05.653 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.653 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.654 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.654 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128413+0000", 2023-10-01T09:40:05.654 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000146489 2023-10-01T09:40:05.654 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.654 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.654 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:40:05.654 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128559+0000", 2023-10-01T09:40:05.654 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16657522299999999 2023-10-01T09:40:05.654 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.654 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.654 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:40:05.655 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295134+0000", 2023-10-01T09:40:05.655 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0034864190000000001 2023-10-01T09:40:05.655 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.655 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.655 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.655 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.298621+0000", 2023-10-01T09:40:05.655 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1519999999999994e-05 2023-10-01T09:40:05.655 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.655 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.655 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.655 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.298682+0000", 2023-10-01T09:40:05.655 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2095999999999997e-05 2023-10-01T09:40:05.655 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.656 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.656 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.656 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.298734+0000", 2023-10-01T09:40:05.656 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000111989 2023-10-01T09:40:05.656 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.656 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.656 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.656 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.298846+0000", 2023-10-01T09:40:05.656 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17065782800000001 2023-10-01T09:40:05.656 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.656 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.656 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.657 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.657 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.657 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:178400 8.4 8:2fa69b17:::benchmark_data_smithi138_431617_object76989:head [delete] snapc 0=[] ondisk+write+known_if_redirected e374)", 2023-10-01T09:40:05.657 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:01.128235+0000", 2023-10-01T09:40:05.657 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 424.49378023999998, 2023-10-01T09:40:05.657 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.170802483, 2023-10-01T09:40:05.657 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.657 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.657 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:40:05.657 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:40:05.657 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:40:05.658 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 178400 2023-10-01T09:40:05.658 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.658 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.658 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.658 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.658 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128235+0000", 2023-10-01T09:40:05.658 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.658 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.658 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.658 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.658 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128235+0000", 2023-10-01T09:40:05.658 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:40:05.659 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.659 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.659 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.659 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128234+0000", 2023-10-01T09:40:05.659 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9519999999999999e-06 2023-10-01T09:40:05.659 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.659 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.659 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.659 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128237+0000", 2023-10-01T09:40:05.659 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0060000000000003e-06 2023-10-01T09:40:05.659 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.659 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.660 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.660 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128244+0000", 2023-10-01T09:40:05.660 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4630000000000003e-06 2023-10-01T09:40:05.660 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.660 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.660 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.660 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.128249+0000", 2023-10-01T09:40:05.660 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024535020000000002 2023-10-01T09:40:05.660 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.660 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.660 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.660 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.130702+0000", 2023-10-01T09:40:05.661 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4159999999999994e-05 2023-10-01T09:40:05.661 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.661 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.661 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.661 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.130787+0000", 2023-10-01T09:40:05.661 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000105208 2023-10-01T09:40:05.661 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.661 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.661 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:40:05.661 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.130892+0000", 2023-10-01T09:40:05.661 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16425838400000001 2023-10-01T09:40:05.661 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.662 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.662 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:40:05.662 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295150+0000", 2023-10-01T09:40:05.662 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035032010000000001 2023-10-01T09:40:05.662 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.662 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.662 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.662 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.298653+0000", 2023-10-01T09:40:05.662 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00023980900000000001 2023-10-01T09:40:05.662 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.662 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.663 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.663 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.298893+0000", 2023-10-01T09:40:05.663 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.269e-05 2023-10-01T09:40:05.663 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.663 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.663 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.663 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.298936+0000", 2023-10-01T09:40:05.663 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010165 2023-10-01T09:40:05.663 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.663 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.663 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.663 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.299038+0000", 2023-10-01T09:40:05.664 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.170802483 2023-10-01T09:40:05.664 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.664 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.664 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.664 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.664 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.664 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:178405 8.e e374/361 8:7705199a:::benchmark_data_smithi138_431617_object76994:head v 374'11414, mlcod=374'11412)", 2023-10-01T09:40:05.664 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:01.133009+0000", 2023-10-01T09:40:05.664 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 424.48900638399999, 2023-10-01T09:40:05.664 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17295598600000001, 2023-10-01T09:40:05.664 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.665 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.665 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.665 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.665 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.665 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.133009+0000", 2023-10-01T09:40:05.665 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.665 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.665 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.665 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.665 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.133009+0000", 2023-10-01T09:40:05.665 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:40:05.665 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.666 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.666 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.666 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.133008+0000", 2023-10-01T09:40:05.666 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5589999999999999e-06 2023-10-01T09:40:05.666 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.666 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.666 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.666 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.133013+0000", 2023-10-01T09:40:05.666 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2519999999999996e-06 2023-10-01T09:40:05.666 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.666 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.667 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.667 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.133019+0000", 2023-10-01T09:40:05.667 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1579999999999998e-06 2023-10-01T09:40:05.667 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.667 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.667 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.667 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.133023+0000", 2023-10-01T09:40:05.667 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.162030965 2023-10-01T09:40:05.667 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.667 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.667 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.668 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295054+0000", 2023-10-01T09:40:05.668 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7665999999999999e-05 2023-10-01T09:40:05.668 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.668 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.668 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.668 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295122+0000", 2023-10-01T09:40:05.668 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010807071 2023-10-01T09:40:05.668 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.668 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.668 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.668 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.305929+0000", 2023-10-01T09:40:05.669 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5861000000000001e-05 2023-10-01T09:40:05.669 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.669 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.669 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.669 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.305965+0000", 2023-10-01T09:40:05.669 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17295598600000001 2023-10-01T09:40:05.669 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.669 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.669 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.670 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.670 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.670 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "MOSDPGPush(8.2 374/369 [PushOp(8:4bc097d3:::benchmark_data_smithi138_431617_object83895:head, version: 344'5152, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:4bc097d3:::benchmark_data_smithi138_431617_object83895:head@344'5152, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-01T09:40:05.670 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:01.134252+0000", 2023-10-01T09:40:05.670 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 424.48776299600001, 2023-10-01T09:40:05.670 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16124813499999999, 2023-10-01T09:40:05.670 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.670 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:40:05.670 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.670 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.670 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.671 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.134252+0000", 2023-10-01T09:40:05.671 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.671 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.671 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.671 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.671 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.134252+0000", 2023-10-01T09:40:05.671 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:40:05.671 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.671 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.671 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.671 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.134252+0000", 2023-10-01T09:40:05.672 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9670000000000001e-05 2023-10-01T09:40:05.672 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.672 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.672 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.672 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.134331+0000", 2023-10-01T09:40:05.672 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.160729709 2023-10-01T09:40:05.672 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.672 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.672 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.672 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295061+0000", 2023-10-01T09:40:05.673 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.515e-05 2023-10-01T09:40:05.673 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.673 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.673 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.673 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295106+0000", 2023-10-01T09:40:05.673 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011285099999999999 2023-10-01T09:40:05.673 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.673 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.673 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.673 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295219+0000", 2023-10-01T09:40:05.673 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4350000000000003e-06 2023-10-01T09:40:05.674 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.674 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.674 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.674 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295228+0000", 2023-10-01T09:40:05.674 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00027193200000000001 2023-10-01T09:40:05.674 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.674 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.674 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.674 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295500+0000", 2023-10-01T09:40:05.675 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16124813499999999 2023-10-01T09:40:05.675 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.675 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.675 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.675 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.675 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.675 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:178408 8.4 8:22b10d42:::benchmark_data_smithi138_431617_object76997:head [delete] snapc 0=[] ondisk+write+known_if_redirected e374)", 2023-10-01T09:40:05.675 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:01.136744+0000", 2023-10-01T09:40:05.675 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 424.48527120599999, 2023-10-01T09:40:05.675 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17711885099999999, 2023-10-01T09:40:05.675 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.675 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.676 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:40:05.676 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:40:05.676 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:40:05.676 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 178408 2023-10-01T09:40:05.676 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.676 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.676 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.676 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.677 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.136744+0000", 2023-10-01T09:40:05.677 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.677 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.677 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.677 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.677 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.136744+0000", 2023-10-01T09:40:05.677 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:40:05.677 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.677 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.677 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.678 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.136743+0000", 2023-10-01T09:40:05.678 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7950000000000002e-06 2023-10-01T09:40:05.678 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.678 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.678 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.678 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.136753+0000", 2023-10-01T09:40:05.678 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.158290036 2023-10-01T09:40:05.678 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.678 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.678 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.678 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295043+0000", 2023-10-01T09:40:05.679 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8039e-05 2023-10-01T09:40:05.679 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.679 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.679 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.679 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295081+0000", 2023-10-01T09:40:05.679 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0966999999999998e-05 2023-10-01T09:40:05.679 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.679 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.679 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.679 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295172+0000", 2023-10-01T09:40:05.680 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014039400000000001 2023-10-01T09:40:05.680 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.680 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.680 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.680 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295312+0000", 2023-10-01T09:40:05.680 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000162378 2023-10-01T09:40:05.680 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.680 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.680 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:40:05.680 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295474+0000", 2023-10-01T09:40:05.680 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010599005 2023-10-01T09:40:05.681 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.681 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.681 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.681 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.306073+0000", 2023-10-01T09:40:05.681 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1469e-05 2023-10-01T09:40:05.681 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.681 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.681 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:40:05.681 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.306085+0000", 2023-10-01T09:40:05.681 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0076060090000000004 2023-10-01T09:40:05.681 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.682 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.682 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.682 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.313691+0000", 2023-10-01T09:40:05.682 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7948999999999997e-05 2023-10-01T09:40:05.682 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.682 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.682 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.682 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.313749+0000", 2023-10-01T09:40:05.682 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011403100000000001 2023-10-01T09:40:05.682 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.682 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.683 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.683 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.313863+0000", 2023-10-01T09:40:05.683 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.17711885099999999 2023-10-01T09:40:05.683 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.683 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.683 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.683 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.683 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.683 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:178401 8.b e374/361 8:de46d23f:::benchmark_data_smithi138_431617_object76990:head v 374'11065, mlcod=374'11063)", 2023-10-01T09:40:05.683 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:01.137716+0000", 2023-10-01T09:40:05.683 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 424.48429931499999, 2023-10-01T09:40:05.684 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.166376951, 2023-10-01T09:40:05.684 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.684 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.684 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.684 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.684 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.684 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.137716+0000", 2023-10-01T09:40:05.684 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.684 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.684 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.684 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.684 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.137716+0000", 2023-10-01T09:40:05.685 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:40:05.685 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.685 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.685 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.685 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.137715+0000", 2023-10-01T09:40:05.685 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.8770000000000005e-06 2023-10-01T09:40:05.685 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.685 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.685 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.685 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.137725+0000", 2023-10-01T09:40:05.685 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15732311399999999 2023-10-01T09:40:05.685 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.686 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.686 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.686 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295048+0000", 2023-10-01T09:40:05.686 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3589000000000001e-05 2023-10-01T09:40:05.686 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.686 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.686 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.686 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295072+0000", 2023-10-01T09:40:05.686 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1915000000000005e-05 2023-10-01T09:40:05.687 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.687 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.687 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.687 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295144+0000", 2023-10-01T09:40:05.687 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2493e-05 2023-10-01T09:40:05.687 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.687 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.687 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.687 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.295166+0000", 2023-10-01T09:40:05.687 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0088856999999999998 2023-10-01T09:40:05.687 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.687 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.688 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.688 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.304052+0000", 2023-10-01T09:40:05.688 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1014999999999999e-05 2023-10-01T09:40:05.688 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.688 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.688 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.688 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:01.304093+0000", 2023-10-01T09:40:05.688 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.166376951 2023-10-01T09:40:05.688 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.688 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.688 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.689 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.689 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.689 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:178948 8.4 8:206e4418:::benchmark_data_smithi138_431617_object77537:head [delete] snapc 0=[] ondisk+write+known_if_redirected e375)", 2023-10-01T09:40:05.689 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:02.137484+0000", 2023-10-01T09:40:05.689 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 423.48453108699999, 2023-10-01T09:40:05.689 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.117356637, 2023-10-01T09:40:05.689 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.689 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.689 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:40:05.689 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:40:05.689 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:40:05.689 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 178948 2023-10-01T09:40:05.690 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.690 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.690 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.690 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.690 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.137484+0000", 2023-10-01T09:40:05.690 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.690 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.690 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.690 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.690 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.137484+0000", 2023-10-01T09:40:05.691 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:40:05.691 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.691 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.691 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.691 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.137483+0000", 2023-10-01T09:40:05.691 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4009999999999996e-06 2023-10-01T09:40:05.691 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.691 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.691 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.691 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.137488+0000", 2023-10-01T09:40:05.691 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.102937139 2023-10-01T09:40:05.692 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.692 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.692 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.692 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240426+0000", 2023-10-01T09:40:05.692 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7660000000000002e-05 2023-10-01T09:40:05.692 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.692 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.692 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.692 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240463+0000", 2023-10-01T09:40:05.692 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6388999999999998e-05 2023-10-01T09:40:05.693 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.693 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.693 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.693 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240520+0000", 2023-10-01T09:40:05.693 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000149646 2023-10-01T09:40:05.693 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.693 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.693 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.693 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240669+0000", 2023-10-01T09:40:05.693 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016274799999999999 2023-10-01T09:40:05.693 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.694 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.694 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:40:05.694 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240832+0000", 2023-10-01T09:40:05.694 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.01014233 2023-10-01T09:40:05.694 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.694 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.694 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:40:05.694 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.250974+0000", 2023-10-01T09:40:05.694 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9999000000000001e-05 2023-10-01T09:40:05.694 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.694 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.695 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.695 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.251014+0000", 2023-10-01T09:40:05.695 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00366593 2023-10-01T09:40:05.695 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.695 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.695 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.695 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.254680+0000", 2023-10-01T09:40:05.695 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3010999999999998e-05 2023-10-01T09:40:05.695 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.695 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.695 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.695 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.254733+0000", 2023-10-01T09:40:05.696 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010754700000000001 2023-10-01T09:40:05.696 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.696 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.696 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.696 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.254841+0000", 2023-10-01T09:40:05.696 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.117356637 2023-10-01T09:40:05.696 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.696 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.696 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.696 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.696 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.697 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:178947 8.b e375/361 8:d41f59f4:::benchmark_data_smithi138_431617_object77536:head v 375'11098, mlcod=375'11095)", 2023-10-01T09:40:05.697 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:02.138023+0000", 2023-10-01T09:40:05.697 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 423.48399219800001, 2023-10-01T09:40:05.697 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.115564793, 2023-10-01T09:40:05.697 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.697 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.697 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.697 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.697 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.697 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.138023+0000", 2023-10-01T09:40:05.697 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.698 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.698 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.698 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.698 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.138023+0000", 2023-10-01T09:40:05.698 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:40:05.698 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.698 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.698 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.698 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.138022+0000", 2023-10-01T09:40:05.698 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5499999999999995e-06 2023-10-01T09:40:05.698 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.699 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.699 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.699 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.138031+0000", 2023-10-01T09:40:05.699 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.102400555 2023-10-01T09:40:05.699 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.699 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.699 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.699 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240431+0000", 2023-10-01T09:40:05.699 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6390000000000004e-06 2023-10-01T09:40:05.699 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.699 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.700 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.700 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240441+0000", 2023-10-01T09:40:05.700 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4252999999999997e-05 2023-10-01T09:40:05.700 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.700 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.700 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.700 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240495+0000", 2023-10-01T09:40:05.700 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6033999999999998e-05 2023-10-01T09:40:05.700 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.700 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.701 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.701 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240521+0000", 2023-10-01T09:40:05.701 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013027031 2023-10-01T09:40:05.701 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.701 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.701 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.701 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.253548+0000", 2023-10-01T09:40:05.701 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9552999999999999e-05 2023-10-01T09:40:05.701 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.701 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.701 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.702 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.253588+0000", 2023-10-01T09:40:05.702 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.115564793 2023-10-01T09:40:05.702 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.702 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.702 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.702 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.702 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.702 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:178951 8.e e375/361 8:76df1f5b:::benchmark_data_smithi138_431617_object77540:head v 375'11448, mlcod=375'11446)", 2023-10-01T09:40:05.702 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:02.148362+0000", 2023-10-01T09:40:05.702 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 423.47365352100002, 2023-10-01T09:40:05.702 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.099382195000000007, 2023-10-01T09:40:05.703 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.703 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.703 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.703 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.703 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.703 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.148362+0000", 2023-10-01T09:40:05.703 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.703 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.703 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.703 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.703 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.148362+0000", 2023-10-01T09:40:05.703 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:40:05.704 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.704 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.704 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.704 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.148361+0000", 2023-10-01T09:40:05.704 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8310000000000003e-06 2023-10-01T09:40:05.704 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.704 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.704 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.704 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.148368+0000", 2023-10-01T09:40:05.704 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.092058944000000004 2023-10-01T09:40:05.704 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.705 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.705 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.705 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240427+0000", 2023-10-01T09:40:05.705 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4754e-05 2023-10-01T09:40:05.705 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.705 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.705 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.705 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240441+0000", 2023-10-01T09:40:05.705 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3739000000000002e-05 2023-10-01T09:40:05.705 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.705 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.706 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.706 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240495+0000", 2023-10-01T09:40:05.706 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5982999999999998e-05 2023-10-01T09:40:05.706 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.706 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.706 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.706 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.240521+0000", 2023-10-01T09:40:05.706 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.007186011 2023-10-01T09:40:05.706 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.706 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.707 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.707 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.247707+0000", 2023-10-01T09:40:05.707 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6711999999999997e-05 2023-10-01T09:40:05.707 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.707 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.707 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.707 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:02.247744+0000", 2023-10-01T09:40:05.707 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.099382195000000007 2023-10-01T09:40:05.707 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.707 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.707 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.708 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.708 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.708 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:187125 8.4 8:24541909:::benchmark_data_smithi138_431617_object85714:head [delete] snapc 0=[] ondisk+write+known_if_redirected e384)", 2023-10-01T09:40:05.708 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:18.925404+0000", 2023-10-01T09:40:05.708 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 406.69661096300001, 2023-10-01T09:40:05.708 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18668991800000001, 2023-10-01T09:40:05.708 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.708 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.708 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:40:05.708 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:40:05.708 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:40:05.708 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 187125 2023-10-01T09:40:05.709 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.709 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.709 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.709 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.709 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925404+0000", 2023-10-01T09:40:05.709 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.709 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.709 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.709 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.709 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925404+0000", 2023-10-01T09:40:05.709 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:40:05.710 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.710 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.710 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.710 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925402+0000", 2023-10-01T09:40:05.710 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.568e-06 2023-10-01T09:40:05.710 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.710 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.710 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.710 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925411+0000", 2023-10-01T09:40:05.710 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3375e-05 2023-10-01T09:40:05.711 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.711 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.711 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.711 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925424+0000", 2023-10-01T09:40:05.711 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4150000000000004e-06 2023-10-01T09:40:05.711 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.711 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.711 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.711 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925431+0000", 2023-10-01T09:40:05.711 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9450000000000002e-05 2023-10-01T09:40:05.711 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.711 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.712 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.712 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925490+0000", 2023-10-01T09:40:05.712 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000144986 2023-10-01T09:40:05.712 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.712 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.712 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.712 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925635+0000", 2023-10-01T09:40:05.712 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016538100000000001 2023-10-01T09:40:05.712 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.712 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.712 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:40:05.713 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.925801+0000", 2023-10-01T09:40:05.713 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033733119999999998 2023-10-01T09:40:05.713 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.713 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.713 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:40:05.713 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.929174+0000", 2023-10-01T09:40:05.713 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019933059999999998 2023-10-01T09:40:05.713 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.713 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.714 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.714 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.931167+0000", 2023-10-01T09:40:05.714 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18060246999999999 2023-10-01T09:40:05.714 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.714 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.714 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.714 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.111770+0000", 2023-10-01T09:40:05.714 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3547000000000006e-05 2023-10-01T09:40:05.714 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.714 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.715 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.715 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.111843+0000", 2023-10-01T09:40:05.715 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00025091799999999999 2023-10-01T09:40:05.715 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.715 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.715 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.715 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.112094+0000", 2023-10-01T09:40:05.715 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.18668991800000001 2023-10-01T09:40:05.715 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.715 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.715 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.716 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.716 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.716 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.45674.0:187134 8.4 8:2f1ba86e:::benchmark_data_smithi138_431617_object85723:head [delete] snapc 0=[] ondisk+write+known_if_redirected e384)", 2023-10-01T09:40:05.716 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:18.945154+0000", 2023-10-01T09:40:05.716 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 406.67686127299999, 2023-10-01T09:40:05.716 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.167213626, 2023-10-01T09:40:05.716 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.716 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.716 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:40:05.716 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.45674", 2023-10-01T09:40:05.716 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3294095605", 2023-10-01T09:40:05.717 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 187134 2023-10-01T09:40:05.717 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.717 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.717 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.717 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.717 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945154+0000", 2023-10-01T09:40:05.717 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.717 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.717 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.718 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.718 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945154+0000", 2023-10-01T09:40:05.718 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:40:05.718 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.718 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.718 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.718 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945152+0000", 2023-10-01T09:40:05.718 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2859e-05 2023-10-01T09:40:05.718 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.718 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.718 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.719 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945165+0000", 2023-10-01T09:40:05.719 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3360000000000001e-05 2023-10-01T09:40:05.719 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.719 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.719 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.719 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945178+0000", 2023-10-01T09:40:05.719 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2300000000000002e-06 2023-10-01T09:40:05.719 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.719 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.719 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.719 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945186+0000", 2023-10-01T09:40:05.720 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9648000000000003e-05 2023-10-01T09:40:05.720 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.720 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.720 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.720 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945235+0000", 2023-10-01T09:40:05.720 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9283000000000005e-05 2023-10-01T09:40:05.720 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.720 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.720 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.720 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945334+0000", 2023-10-01T09:40:05.720 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000114163 2023-10-01T09:40:05.720 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.721 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.721 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 2,5", 2023-10-01T09:40:05.721 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.945449+0000", 2023-10-01T09:40:05.721 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0023364940000000002 2023-10-01T09:40:05.721 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.721 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.721 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:40:05.721 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.947785+0000", 2023-10-01T09:40:05.721 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0018238289999999999 2023-10-01T09:40:05.721 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.721 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.722 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.722 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.949609+0000", 2023-10-01T09:40:05.722 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.1625383 2023-10-01T09:40:05.722 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.722 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.722 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:40:05.722 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.112147+0000", 2023-10-01T09:40:05.722 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7358000000000002e-05 2023-10-01T09:40:05.722 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.722 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.722 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.723 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.112205+0000", 2023-10-01T09:40:05.723 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016299000000000001 2023-10-01T09:40:05.723 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.723 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.723 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.723 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.112368+0000", 2023-10-01T09:40:05.723 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.167213626 2023-10-01T09:40:05.723 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.723 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.723 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.723 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.724 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.724 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:187138 8.b e384/380 8:d4b7dda1:::benchmark_data_smithi138_431617_object85727:head v 384'11643, mlcod=384'11641)", 2023-10-01T09:40:05.724 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:18.957268+0000", 2023-10-01T09:40:05.724 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 406.66474772200002, 2023-10-01T09:40:05.724 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14524411300000001, 2023-10-01T09:40:05.724 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.724 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.724 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.724 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.724 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.724 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957268+0000", 2023-10-01T09:40:05.725 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.725 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.725 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.725 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.725 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957268+0000", 2023-10-01T09:40:05.725 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:40:05.725 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.725 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.725 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.725 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957266+0000", 2023-10-01T09:40:05.725 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4919000000000001e-05 2023-10-01T09:40:05.726 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.726 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.726 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.726 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957281+0000", 2023-10-01T09:40:05.726 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2416e-05 2023-10-01T09:40:05.726 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.726 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.726 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.726 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957303+0000", 2023-10-01T09:40:05.726 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2563e-05 2023-10-01T09:40:05.726 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.727 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.727 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.727 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957316+0000", 2023-10-01T09:40:05.727 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000103028 2023-10-01T09:40:05.727 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.727 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.727 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.727 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957419+0000", 2023-10-01T09:40:05.727 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4715999999999998e-05 2023-10-01T09:40:05.727 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.727 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.728 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.728 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957453+0000", 2023-10-01T09:40:05.728 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.145028781 2023-10-01T09:40:05.728 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.728 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.728 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.728 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.102482+0000", 2023-10-01T09:40:05.728 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9589000000000001e-05 2023-10-01T09:40:05.728 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.728 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.728 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.729 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.102512+0000", 2023-10-01T09:40:05.729 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14524411300000001 2023-10-01T09:40:05.729 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.729 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.729 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.729 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.729 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.729 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:187139 8.2 e384/380 8:45fc0a90:::benchmark_data_smithi138_431617_object85728:head v 384'11540, mlcod=368'10456)", 2023-10-01T09:40:05.729 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:18.957444+0000", 2023-10-01T09:40:05.729 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 406.66457124599998, 2023-10-01T09:40:05.729 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14461728400000001, 2023-10-01T09:40:05.730 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.730 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.730 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.730 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.730 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.730 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957444+0000", 2023-10-01T09:40:05.730 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.730 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.730 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.730 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.731 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957444+0000", 2023-10-01T09:40:05.731 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:40:05.731 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.731 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.731 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.731 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957443+0000", 2023-10-01T09:40:05.731 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8043e-05 2023-10-01T09:40:05.731 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.731 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.731 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.731 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:18.957461+0000", 2023-10-01T09:40:05.731 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14196397799999999 2023-10-01T09:40:05.732 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.732 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.732 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.732 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.099425+0000", 2023-10-01T09:40:05.732 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4381e-05 2023-10-01T09:40:05.732 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.732 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.732 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.732 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.099439+0000", 2023-10-01T09:40:05.732 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3649999999999997e-05 2023-10-01T09:40:05.732 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.733 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.733 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.733 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.099483+0000", 2023-10-01T09:40:05.733 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2201999999999999e-05 2023-10-01T09:40:05.733 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.733 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.733 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.733 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.099505+0000", 2023-10-01T09:40:05.733 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002511629 2023-10-01T09:40:05.733 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.733 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.734 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.734 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.102016+0000", 2023-10-01T09:40:05.734 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4898000000000003e-05 2023-10-01T09:40:05.734 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.734 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.734 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.734 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:19.102061+0000", 2023-10-01T09:40:05.734 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14461728400000001 2023-10-01T09:40:05.734 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.734 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.734 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.735 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.735 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.735 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:187962 8.2 e384/380 8:4458b8ad:::benchmark_data_smithi138_431617_object86551:head v 384'11600, mlcod=368'10456)", 2023-10-01T09:40:05.735 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:20.442352+0000", 2023-10-01T09:40:05.735 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 405.17966367399998, 2023-10-01T09:40:05.735 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14790603499999999, 2023-10-01T09:40:05.735 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.735 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.735 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.735 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.735 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.735 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442352+0000", 2023-10-01T09:40:05.736 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.736 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.736 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.736 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.736 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442352+0000", 2023-10-01T09:40:05.736 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:40:05.736 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.736 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.736 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.737 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442350+0000", 2023-10-01T09:40:05.737 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5109000000000001e-05 2023-10-01T09:40:05.737 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.737 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.737 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.737 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442365+0000", 2023-10-01T09:40:05.737 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.383e-05 2023-10-01T09:40:05.737 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.737 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.737 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.737 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442379+0000", 2023-10-01T09:40:05.738 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8100000000000003e-06 2023-10-01T09:40:05.738 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.738 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.738 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.738 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442385+0000", 2023-10-01T09:40:05.738 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3603000000000002e-05 2023-10-01T09:40:05.738 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.738 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.738 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.738 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442429+0000", 2023-10-01T09:40:05.738 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7490999999999999e-05 2023-10-01T09:40:05.739 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.739 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.739 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.739 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.442446+0000", 2023-10-01T09:40:05.739 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.147771334 2023-10-01T09:40:05.739 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.739 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.739 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.739 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.590217+0000", 2023-10-01T09:40:05.739 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0160000000000002e-05 2023-10-01T09:40:05.739 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.740 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.740 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.740 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.590258+0000", 2023-10-01T09:40:05.740 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14790603499999999 2023-10-01T09:40:05.740 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.740 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.740 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.740 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.740 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.740 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:187966 8.5 e384/367 8:a88a80c6:::benchmark_data_smithi138_431617_object86555:head v 384'11853, mlcod=384'11851)", 2023-10-01T09:40:05.740 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:20.448350+0000", 2023-10-01T09:40:05.741 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 405.17366522700001, 2023-10-01T09:40:05.741 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14374200000000001, 2023-10-01T09:40:05.741 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.741 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.741 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.741 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.741 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.741 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448350+0000", 2023-10-01T09:40:05.741 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.741 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.741 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.742 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.742 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448350+0000", 2023-10-01T09:40:05.742 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:40:05.742 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.742 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.742 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.742 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448349+0000", 2023-10-01T09:40:05.742 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5560000000000008e-06 2023-10-01T09:40:05.742 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.742 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.742 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.743 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448358+0000", 2023-10-01T09:40:05.743 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13449048599999999 2023-10-01T09:40:05.743 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.743 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.743 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.743 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.582848+0000", 2023-10-01T09:40:05.743 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0599e-05 2023-10-01T09:40:05.743 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.743 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.743 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.743 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.582859+0000", 2023-10-01T09:40:05.744 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4080999999999999e-05 2023-10-01T09:40:05.744 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.744 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.744 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.744 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.582913+0000", 2023-10-01T09:40:05.744 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4927999999999999e-05 2023-10-01T09:40:05.744 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.744 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.744 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.744 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.582938+0000", 2023-10-01T09:40:05.744 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0091225230000000004 2023-10-01T09:40:05.745 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.745 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.745 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.745 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.592060+0000", 2023-10-01T09:40:05.745 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1761000000000003e-05 2023-10-01T09:40:05.745 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.745 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.745 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.745 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.592092+0000", 2023-10-01T09:40:05.745 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14374200000000001 2023-10-01T09:40:05.745 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.746 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.746 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.746 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.746 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.746 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.45674.0:187967 8.b e384/380 8:d518ca2e:::benchmark_data_smithi138_431617_object86556:head v 384'11699, mlcod=384'11697)", 2023-10-01T09:40:05.746 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:33:20.448508+0000", 2023-10-01T09:40:05.746 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 405.17350678100001, 2023-10-01T09:40:05.746 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13899778400000001, 2023-10-01T09:40:05.746 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:05.746 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:40:05.746 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:05.747 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.747 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:05.747 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448508+0000", 2023-10-01T09:40:05.747 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:05.747 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.747 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.747 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:05.747 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448508+0000", 2023-10-01T09:40:05.747 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:40:05.747 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.748 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.748 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:05.748 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448508+0000", 2023-10-01T09:40:05.748 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1119e-05 2023-10-01T09:40:05.748 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.748 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.748 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:05.748 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.448519+0000", 2023-10-01T09:40:05.748 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13449881799999999 2023-10-01T09:40:05.748 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.748 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.749 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:05.749 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.583018+0000", 2023-10-01T09:40:05.749 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8060000000000002e-06 2023-10-01T09:40:05.749 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.749 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.749 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:05.749 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.583022+0000", 2023-10-01T09:40:05.749 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010691099999999999 2023-10-01T09:40:05.749 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.749 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.749 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:05.749 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.583129+0000", 2023-10-01T09:40:05.750 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9769999999999999e-05 2023-10-01T09:40:05.750 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.750 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.750 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:05.750 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.583149+0000", 2023-10-01T09:40:05.750 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0043129889999999997 2023-10-01T09:40:05.750 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.750 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.750 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:40:05.750 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.587462+0000", 2023-10-01T09:40:05.750 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4264000000000002e-05 2023-10-01T09:40:05.751 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:05.751 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:05.751 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:40:05.751 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:33:20.587506+0000", 2023-10-01T09:40:05.751 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13899778400000001 2023-10-01T09:40:05.751 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.751 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.751 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.751 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:05.751 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:05.751 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:40:05.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:05.753 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:05.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:05.772 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-10-01T09:40:05.853 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:05.887 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:05 smithi138 ceph-mon[192513]: failed to encode map e475 with expected crc 2023-10-01T09:40:05.888 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:05 smithi138 ceph-mon[192513]: failed to encode map e475 with expected crc 2023-10-01T09:40:05.888 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:05 smithi138 ceph-mon[192513]: failed to encode map e475 with expected crc 2023-10-01T09:40:05.888 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:05 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-01T09:40:05.888 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:05 smithi138 ceph-mon[192513]: osdmap e475: 8 total, 8 up, 7 in 2023-10-01T09:40:05.888 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:05 smithi138 ceph-mon[192513]: failed to encode map e475 with expected crc 2023-10-01T09:40:05.888 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:05 smithi138 ceph-mon[185886]: failed to encode map e475 with expected crc 2023-10-01T09:40:05.888 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:05 smithi138 ceph-mon[185886]: failed to encode map e475 with expected crc 2023-10-01T09:40:05.888 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:05 smithi138 ceph-mon[185886]: failed to encode map e475 with expected crc 2023-10-01T09:40:05.888 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:05 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-01T09:40:05.889 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:05 smithi138 ceph-mon[185886]: osdmap e475: 8 total, 8 up, 7 in 2023-10-01T09:40:05.889 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:05 smithi138 ceph-mon[185886]: failed to encode map e475 with expected crc 2023-10-01T09:40:05.889 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:05.989 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:05.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:05.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:06.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:05 smithi190 ceph-mon[160435]: failed to encode map e475 with expected crc 2023-10-01T09:40:06.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:05 smithi190 ceph-mon[160435]: failed to encode map e475 with expected crc 2023-10-01T09:40:06.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:05 smithi190 ceph-mon[160435]: failed to encode map e475 with expected crc 2023-10-01T09:40:06.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:05 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-01T09:40:06.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:05 smithi190 ceph-mon[160435]: osdmap e475: 8 total, 8 up, 7 in 2023-10-01T09:40:06.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:05 smithi190 ceph-mon[160435]: failed to encode map e475 with expected crc 2023-10-01T09:40:06.077 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:06.093 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:06.129 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:06.183 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:06.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:06.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:06.288 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:06.289 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:06.317 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:06.328 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:40:06.411 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:06.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:06.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:06.515 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:40:06.543 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:06.590 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:40:06.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:06.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:40:06.684 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:40:06.697 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:06.728 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:06.750 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:40:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[192513]: failed to encode map e476 with expected crc 2023-10-01T09:40:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[192513]: failed to encode map e476 with expected crc 2023-10-01T09:40:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[192513]: osdmap e476: 8 total, 8 up, 7 in 2023-10-01T09:40:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[192513]: failed to encode map e476 with expected crc 2023-10-01T09:40:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[192513]: failed to encode map e476 with expected crc 2023-10-01T09:40:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[192513]: pgmap v1359: 89 pgs: 3 creating+peering, 13 unknown, 4 peering, 69 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 138 B/s, 6 objects/s recovering 2023-10-01T09:40:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3918920473' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:06.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:06.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[185886]: failed to encode map e476 with expected crc 2023-10-01T09:40:06.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[185886]: failed to encode map e476 with expected crc 2023-10-01T09:40:06.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[185886]: osdmap e476: 8 total, 8 up, 7 in 2023-10-01T09:40:06.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[185886]: failed to encode map e476 with expected crc 2023-10-01T09:40:06.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[185886]: failed to encode map e476 with expected crc 2023-10-01T09:40:06.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:06.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[185886]: pgmap v1359: 89 pgs: 3 creating+peering, 13 unknown, 4 peering, 69 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 138 B/s, 6 objects/s recovering 2023-10-01T09:40:06.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3918920473' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:06.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:06 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:06.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:06 smithi190 ceph-mon[160435]: failed to encode map e476 with expected crc 2023-10-01T09:40:06.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:06 smithi190 ceph-mon[160435]: failed to encode map e476 with expected crc 2023-10-01T09:40:06.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:06 smithi190 ceph-mon[160435]: osdmap e476: 8 total, 8 up, 7 in 2023-10-01T09:40:06.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:06 smithi190 ceph-mon[160435]: failed to encode map e476 with expected crc 2023-10-01T09:40:06.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:06 smithi190 ceph-mon[160435]: failed to encode map e476 with expected crc 2023-10-01T09:40:06.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:06 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:06.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:06 smithi190 ceph-mon[160435]: pgmap v1359: 89 pgs: 3 creating+peering, 13 unknown, 4 peering, 69 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 138 B/s, 6 objects/s recovering 2023-10-01T09:40:06.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:06 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3918920473' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:06.889 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:06 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:07.044 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 3] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:40:07.044 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:40:07.044 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T09:40:07.044 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.676709 2023-10-01T09:40:07.044 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 0.6767090125341452 2023-10-01T09:40:07.810 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:07.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:07.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:07.915 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:07.943 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:08.048 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:07 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:08.048 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:07 smithi190 ceph-mon[160435]: failed to encode map e477 with expected crc 2023-10-01T09:40:08.048 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:07 smithi190 ceph-mon[160435]: failed to encode map e477 with expected crc 2023-10-01T09:40:08.048 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:07 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:40:08.048 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:07 smithi190 ceph-mon[160435]: osdmap e477: 8 total, 8 up, 7 in 2023-10-01T09:40:08.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:07 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:08.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:07 smithi138 ceph-mon[185886]: failed to encode map e477 with expected crc 2023-10-01T09:40:08.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:07 smithi138 ceph-mon[185886]: failed to encode map e477 with expected crc 2023-10-01T09:40:08.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:07 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:40:08.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:07 smithi138 ceph-mon[185886]: osdmap e477: 8 total, 8 up, 7 in 2023-10-01T09:40:08.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:07 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:08.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:07 smithi138 ceph-mon[192513]: failed to encode map e477 with expected crc 2023-10-01T09:40:08.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:07 smithi138 ceph-mon[192513]: failed to encode map e477 with expected crc 2023-10-01T09:40:08.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:07 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:40:08.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:07 smithi138 ceph-mon[192513]: osdmap e477: 8 total, 8 up, 7 in 2023-10-01T09:40:08.089 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:08.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:08.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:08.194 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:08.221 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:08.265 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:40:08.265 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:40:08.266 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 0 2023-10-01T09:40:08.266 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:40:08.267 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:08.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:08.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:08.372 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:08.401 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:08.504 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:08.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:08.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:08.608 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:08.635 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:08.707 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:40:08.725 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:40:08.737 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:08 smithi138 ceph-mon[185886]: failed to encode map e477 with expected crc 2023-10-01T09:40:08.737 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:08 smithi138 ceph-mon[185886]: failed to encode map e477 with expected crc 2023-10-01T09:40:08.737 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:08 smithi138 ceph-mon[185886]: pgmap v1361: 89 pgs: 3 creating+peering, 8 unknown, 78 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 129 B/s, 2 keys/s, 5 objects/s recovering 2023-10-01T09:40:08.737 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:08 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3150661171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:08.748 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:40:08.762 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:08.769 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:40:08.794 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:40:08.812 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:40:08.830 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:40:08.848 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:40:08.849 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:40:08.850 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:40:08.850 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:40:08.850 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:40:08.850 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:40:08.850 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:40:08.850 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:40:08.850 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:40:08.883 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:08.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:08.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:08.966 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:40:08.986 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:08.990 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:08 smithi190 ceph-mon[160435]: failed to encode map e477 with expected crc 2023-10-01T09:40:08.990 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:08 smithi190 ceph-mon[160435]: failed to encode map e477 with expected crc 2023-10-01T09:40:08.990 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:08 smithi190 ceph-mon[160435]: pgmap v1361: 89 pgs: 3 creating+peering, 8 unknown, 78 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 129 B/s, 2 keys/s, 5 objects/s recovering 2023-10-01T09:40:08.991 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:08 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3150661171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:09.015 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:09.026 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:08 smithi138 ceph-mon[192513]: failed to encode map e477 with expected crc 2023-10-01T09:40:09.026 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:08 smithi138 ceph-mon[192513]: failed to encode map e477 with expected crc 2023-10-01T09:40:09.026 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:08 smithi138 ceph-mon[192513]: pgmap v1361: 89 pgs: 3 creating+peering, 8 unknown, 78 active+clean; 4.6 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 129 B/s, 2 keys/s, 5 objects/s recovering 2023-10-01T09:40:09.027 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:08 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3150661171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:09.058 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:09.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:09.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:09.164 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:09.194 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:09.321 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:40:09.487 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:09.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:09.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:09.590 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:09.619 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:09.715 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:09.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:09.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:09.744 INFO:teuthology.orchestra.run.smithi138.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-10-01T09:40:09.745 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:09 smithi138 ceph-mon[185886]: failed to encode map e477 with expected crc 2023-10-01T09:40:09.745 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:09 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1176275193' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:40:09.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:09 smithi190 ceph-mon[160435]: failed to encode map e477 with expected crc 2023-10-01T09:40:09.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:09 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1176275193' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:40:09.819 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:09.848 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:09.999 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:10.001 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:09 smithi138 ceph-mon[192513]: failed to encode map e477 with expected crc 2023-10-01T09:40:10.001 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:09 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1176275193' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:40:10.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:10.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:10.103 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:40:10.131 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:10.238 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:40:10.303 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:40:10.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:10.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:40:10.407 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:10.435 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:10.505 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:10.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:10.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:10.608 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:10.636 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:10.680 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:10.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:10.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:10.784 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:10.808 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:10 smithi138 ceph-mon[192513]: failed to encode map e478 with expected crc 2023-10-01T09:40:10.808 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:10 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1176275193' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:40:10.808 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:10 smithi138 ceph-mon[192513]: osdmap e478: 8 total, 8 up, 7 in 2023-10-01T09:40:10.808 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:10 smithi138 ceph-mon[192513]: failed to encode map e478 with expected crc 2023-10-01T09:40:10.808 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:10 smithi138 ceph-mon[192513]: pgmap v1363: 89 pgs: 89 active+clean; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 106 B/s, 1 keys/s, 4 objects/s recovering 2023-10-01T09:40:10.808 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:10.808 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:40:10.809 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:10 smithi138 ceph-mon[185886]: failed to encode map e478 with expected crc 2023-10-01T09:40:10.809 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:10 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1176275193' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:40:10.809 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:10 smithi138 ceph-mon[185886]: osdmap e478: 8 total, 8 up, 7 in 2023-10-01T09:40:10.809 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:10 smithi138 ceph-mon[185886]: failed to encode map e478 with expected crc 2023-10-01T09:40:10.809 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:10 smithi138 ceph-mon[185886]: pgmap v1363: 89 pgs: 89 active+clean; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 106 B/s, 1 keys/s, 4 objects/s recovering 2023-10-01T09:40:10.809 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:10.809 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:40:10.810 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:10.827 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:40:10.827 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:40:11.005 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:10 smithi190 ceph-mon[160435]: failed to encode map e478 with expected crc 2023-10-01T09:40:11.005 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:10 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1176275193' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T09:40:11.005 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:10 smithi190 ceph-mon[160435]: osdmap e478: 8 total, 8 up, 7 in 2023-10-01T09:40:11.005 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:10 smithi190 ceph-mon[160435]: failed to encode map e478 with expected crc 2023-10-01T09:40:11.005 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:10 smithi190 ceph-mon[160435]: pgmap v1363: 89 pgs: 89 active+clean; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 106 B/s, 1 keys/s, 4 objects/s recovering 2023-10-01T09:40:11.005 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:11.005 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:40:11.016 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:11.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:11.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:11.108 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:11.119 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:11.146 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:11.250 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:11.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:11.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:11.357 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:11.386 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:11.483 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:11.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:11.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:11.536 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:40:11.550 DEBUG:teuthology.orchestra.run.smithi138:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-10-01T09:40:11.553 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T09:40:11.553 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T09:40:11.587 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:11.601 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:hints = 1 2023-10-01T09:40:11.601 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T09:40:11.601 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Object prefix: benchmark_data_smithi138_573643 2023-10-01T09:40:11.602 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:40:11.602 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 0 0 0 0 0 0 - 0 2023-10-01T09:40:11.632 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:11.683 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:40:11.935 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:11.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:11.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:12.045 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:12.049 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:40:12.071 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:12.184 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.4 primary-affinity to 0.676709 (8443482) 2023-10-01T09:40:12.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:12 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/165577203' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6767090125341452}]: dispatch 2023-10-01T09:40:12.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:12 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/165577203' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6767090125341452}]: dispatch 2023-10-01T09:40:12.445 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:12 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/165577203' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6767090125341452}]: dispatch 2023-10-01T09:40:12.602 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 1 16 1322 1306 81.6205 81.625 0.00101103 0.00732216 2023-10-01T09:40:12.647 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:12.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:12.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:12.750 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:12.777 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:13.169 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:13.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:13.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:13.273 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:13.276 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:13.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:13 smithi138 ceph-mon[185886]: failed to encode map e479 with expected crc 2023-10-01T09:40:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:13 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/165577203' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6767090125341452}]': finished 2023-10-01T09:40:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:13 smithi138 ceph-mon[185886]: osdmap e479: 8 total, 8 up, 7 in 2023-10-01T09:40:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:13 smithi138 ceph-mon[185886]: failed to encode map e479 with expected crc 2023-10-01T09:40:13.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:13 smithi138 ceph-mon[185886]: pgmap v1365: 89 pgs: 89 active+clean; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-01T09:40:13.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:13 smithi138 ceph-mon[192513]: failed to encode map e479 with expected crc 2023-10-01T09:40:13.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:13 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/165577203' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6767090125341452}]': finished 2023-10-01T09:40:13.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:13 smithi138 ceph-mon[192513]: osdmap e479: 8 total, 8 up, 7 in 2023-10-01T09:40:13.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:13 smithi138 ceph-mon[192513]: failed to encode map e479 with expected crc 2023-10-01T09:40:13.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:13 smithi138 ceph-mon[192513]: pgmap v1365: 89 pgs: 89 active+clean; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-01T09:40:13.338 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:13.491 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:13.495 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:13 smithi190 ceph-mon[160435]: failed to encode map e479 with expected crc 2023-10-01T09:40:13.495 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:13 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/165577203' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.6767090125341452}]': finished 2023-10-01T09:40:13.495 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:13 smithi190 ceph-mon[160435]: osdmap e479: 8 total, 8 up, 7 in 2023-10-01T09:40:13.495 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:13 smithi190 ceph-mon[160435]: failed to encode map e479 with expected crc 2023-10-01T09:40:13.495 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:13 smithi190 ceph-mon[160435]: pgmap v1365: 89 pgs: 89 active+clean; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 5 B/s, 1 keys/s, 0 objects/s recovering 2023-10-01T09:40:13.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:13.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:13.597 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:13.602 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 2 16 2624 2608 81.486 81.375 0.00509145 0.0122456 2023-10-01T09:40:13.646 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:13.732 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:13.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:13.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:13.839 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:13.884 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:13.979 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:40:13.979 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:40:14.079 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:14.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:14.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:14.184 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:14.205 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:40:14.244 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:14.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:14 smithi190 ceph-mon[160435]: failed to encode map e480 with expected crc 2023-10-01T09:40:14.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:14 smithi190 ceph-mon[160435]: failed to encode map e480 with expected crc 2023-10-01T09:40:14.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:14 smithi190 ceph-mon[160435]: osdmap e480: 8 total, 8 up, 7 in 2023-10-01T09:40:14.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:14 smithi190 ceph-mon[160435]: failed to encode map e480 with expected crc 2023-10-01T09:40:14.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/996713534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:40:14.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:14 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:40:14.352 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:14.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:14.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:14.458 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:14.486 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:14.527 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:14.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:14.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:14.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:14 smithi138 ceph-mon[192513]: failed to encode map e480 with expected crc 2023-10-01T09:40:14.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:14 smithi138 ceph-mon[192513]: failed to encode map e480 with expected crc 2023-10-01T09:40:14.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:14 smithi138 ceph-mon[192513]: osdmap e480: 8 total, 8 up, 7 in 2023-10-01T09:40:14.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:14 smithi138 ceph-mon[192513]: failed to encode map e480 with expected crc 2023-10-01T09:40:14.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/996713534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:40:14.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:14 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:40:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:14 smithi138 ceph-mon[185886]: failed to encode map e480 with expected crc 2023-10-01T09:40:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:14 smithi138 ceph-mon[185886]: failed to encode map e480 with expected crc 2023-10-01T09:40:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:14 smithi138 ceph-mon[185886]: osdmap e480: 8 total, 8 up, 7 in 2023-10-01T09:40:14.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:14 smithi138 ceph-mon[185886]: failed to encode map e480 with expected crc 2023-10-01T09:40:14.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/996713534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:40:14.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:14 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:40:14.602 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 3 16 5228 5212 108.567 162.75 0.00430358 0.00919888 2023-10-01T09:40:14.630 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:14.687 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:14.737 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:14.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:14.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:14.840 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:40:14.866 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:14.907 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:40:14.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:14.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:40:15.010 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:15.036 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:15 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:15 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:40:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:15 smithi190 ceph-mon[160435]: osdmap e481: 8 total, 8 up, 7 in 2023-10-01T09:40:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:15 smithi190 ceph-mon[160435]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:15 smithi190 ceph-mon[160435]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:15 smithi190 ceph-mon[160435]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:15 smithi190 ceph-mon[160435]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:15 smithi190 ceph-mon[160435]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:15 smithi190 ceph-mon[160435]: pgmap v1368: 89 pgs: 89 active+clean; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail 2023-10-01T09:40:15.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:40:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:40:14] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.33.4" 2023-10-01T09:40:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:40:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[185886]: osdmap e481: 8 total, 8 up, 7 in 2023-10-01T09:40:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[185886]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[185886]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[185886]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[185886]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[185886]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[185886]: pgmap v1368: 89 pgs: 89 active+clean; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail 2023-10-01T09:40:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:40:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[192513]: osdmap e481: 8 total, 8 up, 7 in 2023-10-01T09:40:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[192513]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[192513]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[192513]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[192513]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[192513]: failed to encode map e481 with expected crc 2023-10-01T09:40:15.322 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:15 smithi138 ceph-mon[192513]: pgmap v1368: 89 pgs: 89 active+clean; 4.6 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail 2023-10-01T09:40:15.329 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:15.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:15.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:15.427 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:40:15.437 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:15.539 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:15.602 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 4 16 7369 7353 114.876 133.812 0.00598538 0.00869672 2023-10-01T09:40:15.665 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:15.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:15.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:15.773 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:15.824 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:15.992 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:15.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:15.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:16.099 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:16.121 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:40:16.149 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:16.253 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:16.255 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:40:16.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:16.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:16.285 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:40:16.318 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:40:16.348 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:40:16.357 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:16.379 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:40:16.403 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:16.415 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:40:16.449 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:40:16.483 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:40:16.486 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:40:16.486 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:40:16.486 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:40:16.487 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:40:16.487 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:40:16.487 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:40:16.487 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:40:16.487 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:40:16.489 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:16.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:16.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:16.595 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:16.602 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 5 16 10209 10193 127.397 177.5 0.00633121 0.00784179 2023-10-01T09:40:16.648 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:16.831 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:16.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:16.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:16.939 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:16.985 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:17.062 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:17.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:17.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:17.168 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:40:17.231 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:17.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:17 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:17.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:17 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/523361020' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:17.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:17 smithi190 ceph-mon[160435]: pgmap v1369: 89 pgs: 89 active+clean; 301 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s 2023-10-01T09:40:17.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:17 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:17.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:17 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/523361020' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:17.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:17 smithi138 ceph-mon[192513]: pgmap v1369: 89 pgs: 89 active+clean; 301 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s 2023-10-01T09:40:17.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:17 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:17.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:17 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/523361020' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:17.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:17 smithi138 ceph-mon[185886]: pgmap v1369: 89 pgs: 89 active+clean; 301 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s 2023-10-01T09:40:17.361 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:40:17.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:17.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:40:17.467 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:17.501 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:17.601 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:17.602 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 6 16 12960 12944 134.818 171.938 0.00376924 0.0074094 2023-10-01T09:40:17.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:17.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:17.708 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:17.736 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:17.767 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:40:17.777 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:17.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:17.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:17.880 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:17.906 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:17.998 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:18.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:18.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:18.102 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:18.152 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:18.172 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 3] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:40:18.238 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:40:18.238 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T09:40:18.238 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.716576 2023-10-01T09:40:18.238 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0.7165760796939897 2023-10-01T09:40:18.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:18.240 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:18.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:18.293 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:18.341 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:40:18.395 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:18.602 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 7 16 14677 14661 130.885 107.312 0.00420467 0.007635 2023-10-01T09:40:18.706 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:40:18.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:18.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:40:18.818 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:18.861 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:18 smithi138 ceph-mon[192513]: pgmap v1370: 89 pgs: 89 active+clean; 448 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T09:40:18.861 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:18 smithi138 ceph-mon[185886]: pgmap v1370: 89 pgs: 89 active+clean; 448 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T09:40:18.865 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:18.998 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:18 smithi190 ceph-mon[160435]: pgmap v1370: 89 pgs: 89 active+clean; 448 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T09:40:19.054 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:19.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:19.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:19.160 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:19.194 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:19.260 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:19.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:19.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:19.305 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:40:19.305 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:40:19.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.306 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.36456.0:16340 10.e e481/479 10:777b09c9:::benchmark_data_smithi138_573643_object16339:head v 481'1067, mlcod=481'1065)", 2023-10-01T09:40:19.306 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:19.303532+0000", 2023-10-01T09:40:19.306 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.001059965, 2023-10-01T09:40:19.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001087979, 2023-10-01T09:40:19.306 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:19.306 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:40:19.306 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:19.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:19.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.303532+0000", 2023-10-01T09:40:19.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:19.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:19.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.303532+0000", 2023-10-01T09:40:19.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:40:19.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:19.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.303532+0000", 2023-10-01T09:40:19.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3315e-05 2023-10-01T09:40:19.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:19.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.303555+0000", 2023-10-01T09:40:19.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3393999999999999e-05 2023-10-01T09:40:19.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:19.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.303568+0000", 2023-10-01T09:40:19.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0530000000000003e-06 2023-10-01T09:40:19.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:19.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.303574+0000", 2023-10-01T09:40:19.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5018000000000005e-05 2023-10-01T09:40:19.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:19.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.303659+0000", 2023-10-01T09:40:19.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4351000000000003e-05 2023-10-01T09:40:19.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:19.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.303704+0000", 2023-10-01T09:40:19.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017137200000000001 2023-10-01T09:40:19.310 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:19.310 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:19.311 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:19.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.311 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.36456.0:16345 10.0 e481/475 10:00a1bdbf:::benchmark_data_smithi138_573643_object16344:head v 481'990, mlcod=481'988)", 2023-10-01T09:40:19.311 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:19.304388+0000", 2023-10-01T09:40:19.311 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.00020402500000000001, 2023-10-01T09:40:19.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00030151000000000001, 2023-10-01T09:40:19.311 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:40:19.311 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:40:19.311 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:40:19.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:40:19.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.304388+0000", 2023-10-01T09:40:19.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:40:19.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:40:19.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.304388+0000", 2023-10-01T09:40:19.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:40:19.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:40:19.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.304388+0000", 2023-10-01T09:40:19.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013852600000000001 2023-10-01T09:40:19.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:40:19.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.304526+0000", 2023-10-01T09:40:19.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4051e-05 2023-10-01T09:40:19.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:40:19.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.304540+0000", 2023-10-01T09:40:19.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1970000000000001e-06 2023-10-01T09:40:19.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:40:19.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.304546+0000", 2023-10-01T09:40:19.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7697e-05 2023-10-01T09:40:19.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:40:19.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.304604+0000", 2023-10-01T09:40:19.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3428e-05 2023-10-01T09:40:19.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:40:19.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:40:19.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:40:19.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:19.304627+0000", 2023-10-01T09:40:19.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000239213 2023-10-01T09:40:19.315 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:19.315 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:40:19.315 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:19.315 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:40:19.315 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:40:19.315 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 2 2023-10-01T09:40:19.315 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:40:19.365 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:19.409 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:19.481 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:19.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:19.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:19.586 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:19.603 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 8 16 16952 16936 132.296 142.188 0.00176897 0.0074896 2023-10-01T09:40:19.617 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:19.663 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:19.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:19.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:19.767 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:19.809 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:19.905 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:19.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:19.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:20.011 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:40:20.063 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:20.076 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:40:20.105 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:40:20.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:20.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:40:20.209 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:20.235 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:20.312 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:20.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:20.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:20.351 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:20.419 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:20.484 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:20.563 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:20.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:20.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:20.603 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 9 16 18767 18751 130.199 113.438 0.00405074 0.00767443 2023-10-01T09:40:20.668 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:20.721 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:20.780 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:40:20.897 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:20.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:20.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:21.002 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:21.075 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:21.114 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:21.174 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:21.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:21.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:21.283 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:21.331 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:40:21.338 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:21.461 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:40:21.603 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 10 16 20887 20871 130.428 132.5 0.00563066 0.00766062 2023-10-01T09:40:21.753 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:21 smithi190 ceph-mon[160435]: pgmap v1371: 89 pgs: 89 active+clean; 867 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-01T09:40:21.753 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:21 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3687567131' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:21.753 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:21 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:21.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:21 smithi138 ceph-mon[192513]: pgmap v1371: 89 pgs: 89 active+clean; 867 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-01T09:40:21.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:21 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3687567131' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:21 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:21.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:21 smithi138 ceph-mon[185886]: pgmap v1371: 89 pgs: 89 active+clean; 867 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 108 MiB/s wr, 1.72k op/s 2023-10-01T09:40:21.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:21 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3687567131' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:21.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:21 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:22.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:22 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:40:22.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:22 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:40:22.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:22 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:40:22.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:22 smithi190 ceph-mon[160435]: osdmap e482: 8 total, 8 up, 7 in 2023-10-01T09:40:22.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:22 smithi190 ceph-mon[160435]: failed to encode map e482 with expected crc 2023-10-01T09:40:22.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:22 smithi190 ceph-mon[160435]: failed to encode map e482 with expected crc 2023-10-01T09:40:22.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:22 smithi190 ceph-mon[160435]: failed to encode map e482 with expected crc 2023-10-01T09:40:22.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:22 smithi190 ceph-mon[160435]: failed to encode map e482 with expected crc 2023-10-01T09:40:22.603 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 11 16 22969 22953 130.4 130.125 0.00831714 0.00766403 2023-10-01T09:40:22.747 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:22.750 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:40:22.750 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:40:22.751 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:40:22.751 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[192513]: osdmap e482: 8 total, 8 up, 7 in 2023-10-01T09:40:22.751 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[192513]: failed to encode map e482 with expected crc 2023-10-01T09:40:22.751 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[192513]: failed to encode map e482 with expected crc 2023-10-01T09:40:22.751 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[192513]: failed to encode map e482 with expected crc 2023-10-01T09:40:22.751 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[192513]: failed to encode map e482 with expected crc 2023-10-01T09:40:22.751 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:40:22.752 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:40:22.752 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:40:22.752 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[185886]: osdmap e482: 8 total, 8 up, 7 in 2023-10-01T09:40:22.752 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[185886]: failed to encode map e482 with expected crc 2023-10-01T09:40:22.752 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[185886]: failed to encode map e482 with expected crc 2023-10-01T09:40:22.752 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[185886]: failed to encode map e482 with expected crc 2023-10-01T09:40:22.752 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:22 smithi138 ceph-mon[185886]: failed to encode map e482 with expected crc 2023-10-01T09:40:22.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:22.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:22.854 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:22.898 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:23.250 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:23.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:23.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:23.358 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:23.429 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:23.560 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:23 smithi138 ceph-mon[185886]: pgmap v1373: 89 pgs: 89 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 145 MiB/s wr, 2.31k op/s 2023-10-01T09:40:23.560 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:23 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:40:23.560 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:23 smithi138 ceph-mon[192513]: pgmap v1373: 89 pgs: 89 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 145 MiB/s wr, 2.31k op/s 2023-10-01T09:40:23.561 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:23 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:40:23.603 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 12 16 24702 24686 128.558 108.312 0.00405951 0.00777302 2023-10-01T09:40:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:23 smithi190 ceph-mon[160435]: pgmap v1373: 89 pgs: 89 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 145 MiB/s wr, 2.31k op/s 2023-10-01T09:40:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:23 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:40:23.850 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:23.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:23.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:23.959 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:24.020 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:24.151 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:24.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:24.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:24.222 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:40:24.222 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:40:24.222 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:40:24.222 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:40:24.223 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:40:24.257 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:24.296 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:24.345 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:24.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:24.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:24.354 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.1 primary-affinity to 0.716576 (8469612) 2023-10-01T09:40:24.449 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:24.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3917383020' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7165760796939897}]: dispatch 2023-10-01T09:40:24.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:24 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7165760796939897}]: dispatch 2023-10-01T09:40:24.474 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3917383020' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7165760796939897}]: dispatch 2023-10-01T09:40:24.474 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:24 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7165760796939897}]: dispatch 2023-10-01T09:40:24.475 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:24.534 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:24.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:24.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:24.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3917383020' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7165760796939897}]: dispatch 2023-10-01T09:40:24.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:24 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7165760796939897}]: dispatch 2023-10-01T09:40:24.603 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 13 16 26150 26134 125.63 90.5 0.00176593 0.00791619 2023-10-01T09:40:24.637 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:24.677 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:24.872 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:24.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:24.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:24.976 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:25.003 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:25.045 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:25.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:25.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:25.148 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:25.174 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:25.215 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:25.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:25.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:25.319 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:25.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:40:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:40:24] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.33.4" 2023-10-01T09:40:25.351 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:25.397 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:40:25.439 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:25.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:25.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:25.549 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:25.604 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 14 16 26526 26510 118.324 23.5 0.010884 0.00844368 2023-10-01T09:40:25.614 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[185886]: pgmap v1374: 89 pgs: 89 active+clean; 1.1 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 117 MiB/s wr, 1.86k op/s 2023-10-01T09:40:25.614 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[185886]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.615 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[185886]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.615 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[185886]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.615 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[185886]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.615 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[185886]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.615 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7165760796939897}]': finished 2023-10-01T09:40:25.615 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[185886]: osdmap e483: 8 total, 8 up, 7 in 2023-10-01T09:40:25.615 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:25.615 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:40:25.618 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[192513]: pgmap v1374: 89 pgs: 89 active+clean; 1.1 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 117 MiB/s wr, 1.86k op/s 2023-10-01T09:40:25.618 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[192513]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.618 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[192513]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.618 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[192513]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.618 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[192513]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.618 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[192513]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.619 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7165760796939897}]': finished 2023-10-01T09:40:25.619 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[192513]: osdmap e483: 8 total, 8 up, 7 in 2023-10-01T09:40:25.619 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:25.619 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:40:25.619 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:25.705 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:25 smithi190 ceph-mon[160435]: pgmap v1374: 89 pgs: 89 active+clean; 1.1 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 117 MiB/s wr, 1.86k op/s 2023-10-01T09:40:25.705 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:25 smithi190 ceph-mon[160435]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.705 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:25 smithi190 ceph-mon[160435]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:25 smithi190 ceph-mon[160435]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:25 smithi190 ceph-mon[160435]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:25 smithi190 ceph-mon[160435]: failed to encode map e483 with expected crc 2023-10-01T09:40:25.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:25 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7165760796939897}]': finished 2023-10-01T09:40:25.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:25 smithi190 ceph-mon[160435]: osdmap e483: 8 total, 8 up, 7 in 2023-10-01T09:40:25.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:25.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:40:25.736 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:40:25.811 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:25.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:25.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:25.920 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:25.975 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:26.053 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:26.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:26.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:26.146 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:40:26.159 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:26.179 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:40:26.210 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:26.215 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:40:26.226 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:26.248 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:40:26.282 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:40:26.313 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:40:26.339 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:40:26.369 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:40:26.371 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:40:26.372 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:40:26.372 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:40:26.372 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:40:26.372 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:40:26.372 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:40:26.372 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:40:26.372 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:40:26.400 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:26.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:26.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:26.506 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:26.514 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:40:26.596 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:26.604 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 15 16 28167 28151 117.273 102.562 0.00826482 0.00852071 2023-10-01T09:40:26.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[185886]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[185886]: osdmap e484: 8 total, 8 up, 7 in 2023-10-01T09:40:26.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[185886]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[185886]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.668 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[185886]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.668 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[185886]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.668 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[185886]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.668 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3444692275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:26.668 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[192513]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.668 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[192513]: osdmap e484: 8 total, 8 up, 7 in 2023-10-01T09:40:26.668 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[192513]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.668 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[192513]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.668 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[192513]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.668 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[192513]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.669 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[192513]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.669 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:26 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3444692275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:26.715 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:26.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:26.724 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:26 smithi190 ceph-mon[160435]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.724 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:26 smithi190 ceph-mon[160435]: osdmap e484: 8 total, 8 up, 7 in 2023-10-01T09:40:26.725 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:26 smithi190 ceph-mon[160435]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.725 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:26 smithi190 ceph-mon[160435]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.725 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:26 smithi190 ceph-mon[160435]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.725 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:26 smithi190 ceph-mon[160435]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.725 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:26 smithi190 ceph-mon[160435]: failed to encode map e484 with expected crc 2023-10-01T09:40:26.725 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:26 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3444692275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:26.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:26.827 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:26.907 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:27.024 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:27.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:27.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:27.136 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:27.173 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:27.269 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:27.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:27.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:27.373 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:27.430 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:27.605 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 16 16 29679 29663 115.849 94.5 0.00483879 0.00862593 2023-10-01T09:40:27.641 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:27 smithi138 ceph-mon[185886]: pgmap v1377: 89 pgs: 4 peering, 85 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-01T09:40:27.641 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:27 smithi138 ceph-mon[185886]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T09:40:27.642 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:27 smithi138 ceph-mon[192513]: pgmap v1377: 89 pgs: 4 peering, 85 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-01T09:40:27.642 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:27 smithi138 ceph-mon[192513]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T09:40:27.758 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:27.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:27.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:27.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:27 smithi190 ceph-mon[160435]: pgmap v1377: 89 pgs: 4 peering, 85 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-01T09:40:27.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:27 smithi190 ceph-mon[160435]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T09:40:27.861 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:27.889 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:27.985 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:27.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:27.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:28.089 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:28.115 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:28.509 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:28.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:28.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:28.605 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 17 16 30577 30561 112.336 56.125 0.0127987 0.00889531 2023-10-01T09:40:28.612 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:28.675 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:28.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:28 smithi138 ceph-mon[185886]: pgmap v1378: 89 pgs: 4 peering, 85 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-01T09:40:28.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:28 smithi138 ceph-mon[192513]: pgmap v1378: 89 pgs: 4 peering, 85 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-01T09:40:28.896 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:28.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:28.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:29.016 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:29.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:28 smithi190 ceph-mon[160435]: pgmap v1378: 89 pgs: 4 peering, 85 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-10-01T09:40:29.066 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:29.142 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:29.142 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:40:29.143 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:40:29.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:29.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:29.249 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:29.302 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:29.416 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:29.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:29.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:29.524 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:29.567 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:29.605 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 18 16 32290 32274 112.043 107.062 0.00679025 0.00891958 2023-10-01T09:40:29.643 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:40:29.646 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:29.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:29.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:29.731 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:29 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2121452186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:40:29.744 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:40:29.751 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:29.986 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:29 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2121452186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:40:29.990 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:30.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:29 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2121452186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:40:30.345 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:30.349 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:40:30.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:30.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:30.460 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:30.493 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 3] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:40:30.494 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-01T09:40:30.565 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:30.605 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 19 15 33233 33218 109.248 59 0.0117651 0.00914844 2023-10-01T09:40:30.727 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:30.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:30.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:30.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:30 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:30.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:30 smithi138 ceph-mon[185886]: failed to encode map e485 with expected crc 2023-10-01T09:40:30.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:30 smithi138 ceph-mon[185886]: failed to encode map e485 with expected crc 2023-10-01T09:40:30.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:30 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2121452186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:40:30.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:30 smithi138 ceph-mon[185886]: osdmap e485: 8 total, 8 up, 7 in 2023-10-01T09:40:30.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:30 smithi138 ceph-mon[185886]: pgmap v1380: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-01T09:40:30.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:30 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:30.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:30 smithi138 ceph-mon[192513]: failed to encode map e485 with expected crc 2023-10-01T09:40:30.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:30 smithi138 ceph-mon[192513]: failed to encode map e485 with expected crc 2023-10-01T09:40:30.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:30 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2121452186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:40:30.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:30 smithi138 ceph-mon[192513]: osdmap e485: 8 total, 8 up, 7 in 2023-10-01T09:40:30.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:30 smithi138 ceph-mon[192513]: pgmap v1380: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-01T09:40:30.837 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:30.906 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:31.035 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:31.038 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:30 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:31.038 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:30 smithi190 ceph-mon[160435]: failed to encode map e485 with expected crc 2023-10-01T09:40:31.038 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:30 smithi190 ceph-mon[160435]: failed to encode map e485 with expected crc 2023-10-01T09:40:31.039 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:30 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2121452186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:40:31.039 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:30 smithi190 ceph-mon[160435]: osdmap e485: 8 total, 8 up, 7 in 2023-10-01T09:40:31.039 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:30 smithi190 ceph-mon[160435]: pgmap v1380: 89 pgs: 89 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-10-01T09:40:31.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:31.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:31.140 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:31.180 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:40:31.182 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:31.524 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:31.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:31.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:31.607 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:40:31.607668+0000 min lat: 0.00101103 max lat: 1.02643 avg lat: 0.00924259 2023-10-01T09:40:31.607 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:40:31.607 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 20 16 34623 34607 108.118 86.8125 0.0088557 0.00924259 2023-10-01T09:40:31.619 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:31.638 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:31.655 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:31.694 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:31.780 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:31.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:31.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:31.787 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:31 smithi138 ceph-mon[185886]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-01T09:40:31.787 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:31 smithi138 ceph-mon[192513]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-01T09:40:31.888 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:31.941 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:31 smithi190 ceph-mon[160435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-01T09:40:31.949 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:32.607 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 21 16 36152 36136 107.52 95.5625 0.00383613 0.00924999 2023-10-01T09:40:33.231 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:33.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:33.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:33.275 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:33 smithi138 ceph-mon[192513]: pgmap v1381: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-01T09:40:33.275 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:33 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:33.275 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:33 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:33.275 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:33 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:33.276 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:33 smithi138 ceph-mon[185886]: pgmap v1381: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-01T09:40:33.276 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:33 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:33.276 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:33 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:33.276 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:33 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:33.341 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:33.383 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:33.525 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:40:33.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:33 smithi190 ceph-mon[160435]: pgmap v1381: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-10-01T09:40:33.571 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:33 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:33.572 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:33 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:33.572 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:33 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:33.595 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:33.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:33.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:33.611 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 22 16 37048 37032 105.159 56 0.00907847 0.00950294 2023-10-01T09:40:33.700 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:33.736 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:33.833 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:40:33.834 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:40:34.214 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:34.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:34.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:34.320 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:34.475 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:34 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:34.475 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:34 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:34.508 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:34 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:34.611 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 23 16 38215 38199 103.758 72.9375 0.00940407 0.0095845 2023-10-01T09:40:34.758 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:40:34.793 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:40:34.807 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:34.824 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:40:34.860 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:40:34.897 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:40:34.936 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:40:34.937 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:40:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:40:34] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.33.4" 2023-10-01T09:40:34.969 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:40:35.000 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:35.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:35.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:35.011 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:40:35.013 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:40:35.014 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:40:35.014 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:40:35.014 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:40:35.014 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:40:35.014 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:40:35.014 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:40:35.014 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:40:35.111 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T09:40:35.111 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:35.138 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:35.196 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:35.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:35.297 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:35 smithi138 ceph-mon[185886]: pgmap v1382: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-01T09:40:35.297 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1016798310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:35.297 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:35 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:35.297 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:35 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:35.297 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2044945127' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:40:35.297 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:35 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:40:35.300 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:35 smithi190 ceph-mon[160435]: pgmap v1382: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-01T09:40:35.300 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1016798310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:35.300 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:35 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:35.300 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:35 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:35.300 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2044945127' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:40:35.300 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:35 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:40:35.311 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:35.351 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:35.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:35 smithi138 ceph-mon[192513]: pgmap v1382: 89 pgs: 89 active+clean; 2.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-01T09:40:35.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1016798310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:35.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:35 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:35.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:35 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:35.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2044945127' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:40:35.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:35 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:40:35.612 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 24 16 39385 39369 102.482 73.125 0.013656 0.00975176 2023-10-01T09:40:35.798 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:40:36.107 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.0 to 1 (10000) 2023-10-01T09:40:36.612 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 25 16 40687 40671 101.638 81.375 0.00759751 0.00983387 2023-10-01T09:40:36.914 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:36.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:36.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:37.022 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:37.051 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:37 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:37 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:40:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:37 smithi190 ceph-mon[160435]: osdmap e486: 8 total, 8 up, 7 in 2023-10-01T09:40:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:37 smithi190 ceph-mon[160435]: failed to encode map e486 with expected crc 2023-10-01T09:40:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:37 smithi190 ceph-mon[160435]: failed to encode map e486 with expected crc 2023-10-01T09:40:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:37 smithi190 ceph-mon[160435]: failed to encode map e486 with expected crc 2023-10-01T09:40:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:37 smithi190 ceph-mon[160435]: failed to encode map e486 with expected crc 2023-10-01T09:40:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:37 smithi190 ceph-mon[160435]: pgmap v1384: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T09:40:37.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:37.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:40:37.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[185886]: osdmap e486: 8 total, 8 up, 7 in 2023-10-01T09:40:37.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[185886]: failed to encode map e486 with expected crc 2023-10-01T09:40:37.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[185886]: failed to encode map e486 with expected crc 2023-10-01T09:40:37.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[185886]: failed to encode map e486 with expected crc 2023-10-01T09:40:37.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[185886]: failed to encode map e486 with expected crc 2023-10-01T09:40:37.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[185886]: pgmap v1384: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T09:40:37.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:37.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:40:37.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[192513]: osdmap e486: 8 total, 8 up, 7 in 2023-10-01T09:40:37.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[192513]: failed to encode map e486 with expected crc 2023-10-01T09:40:37.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[192513]: failed to encode map e486 with expected crc 2023-10-01T09:40:37.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[192513]: failed to encode map e486 with expected crc 2023-10-01T09:40:37.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[192513]: failed to encode map e486 with expected crc 2023-10-01T09:40:37.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:37 smithi138 ceph-mon[192513]: pgmap v1384: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T09:40:37.465 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-01T09:40:37.612 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 26 16 41444 41428 99.5487 47.3125 0.00537265 0.00999635 2023-10-01T09:40:37.970 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:38.030 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:38.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:38.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:38.055 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:38 smithi138 ceph-mon[185886]: failed to encode map e486 with expected crc 2023-10-01T09:40:38.055 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:38 smithi138 ceph-mon[185886]: failed to encode map e486 with expected crc 2023-10-01T09:40:38.055 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:38 smithi138 ceph-mon[185886]: failed to encode map e486 with expected crc 2023-10-01T09:40:38.056 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:38 smithi138 ceph-mon[192513]: failed to encode map e486 with expected crc 2023-10-01T09:40:38.056 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:38 smithi138 ceph-mon[192513]: failed to encode map e486 with expected crc 2023-10-01T09:40:38.056 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:38 smithi138 ceph-mon[192513]: failed to encode map e486 with expected crc 2023-10-01T09:40:38.136 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:40:38.226 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:38.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:38 smithi190 ceph-mon[160435]: failed to encode map e486 with expected crc 2023-10-01T09:40:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:38 smithi190 ceph-mon[160435]: failed to encode map e486 with expected crc 2023-10-01T09:40:38.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:38 smithi190 ceph-mon[160435]: failed to encode map e486 with expected crc 2023-10-01T09:40:38.328 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:40:38.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:38.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:40:38.434 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:38.485 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:38.610 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:38.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:38.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:38.615 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 27 16 42673 42657 98.6968 76.8125 0.00893777 0.0101258 2023-10-01T09:40:38.716 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:38.754 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:38.810 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:38.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:38.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:38.869 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:40:38.869 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:40:38.870 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:40:38.870 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:40:38.870 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:40:38.913 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:38.951 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:38.971 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:40:39.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:39 smithi190 ceph-mon[160435]: pgmap v1385: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-01T09:40:39.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:39 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3728152840' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:39.085 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:40:39.226 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:39.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:39.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:39.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:39 smithi138 ceph-mon[185886]: pgmap v1385: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-01T09:40:39.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:39 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3728152840' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:39.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:39 smithi138 ceph-mon[192513]: pgmap v1385: 89 pgs: 89 active+clean; 2.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-01T09:40:39.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:39 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3728152840' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:40:39.331 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:39.405 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:39.499 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:39.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:39.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:39.602 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:39.614 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 28 16 43796 43780 97.6788 70.1875 0.00224341 0.0101851 2023-10-01T09:40:39.629 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:39.945 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:39.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:39.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:40.051 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:40.106 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:40.213 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:40 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:40.213 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:40 smithi138 ceph-mon[192513]: failed to encode map e487 with expected crc 2023-10-01T09:40:40.213 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:40 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3728152840' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:40:40.214 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:40 smithi138 ceph-mon[192513]: failed to encode map e487 with expected crc 2023-10-01T09:40:40.214 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:40 smithi138 ceph-mon[192513]: osdmap e487: 8 total, 8 up, 7 in 2023-10-01T09:40:40.214 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:40 smithi138 ceph-mon[192513]: failed to encode map e487 with expected crc 2023-10-01T09:40:40.214 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:40 smithi138 ceph-mon[192513]: failed to encode map e487 with expected crc 2023-10-01T09:40:40.214 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:40 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:40.214 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:40 smithi138 ceph-mon[185886]: failed to encode map e487 with expected crc 2023-10-01T09:40:40.214 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:40 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3728152840' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:40:40.215 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:40 smithi138 ceph-mon[185886]: failed to encode map e487 with expected crc 2023-10-01T09:40:40.215 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:40 smithi138 ceph-mon[185886]: osdmap e487: 8 total, 8 up, 7 in 2023-10-01T09:40:40.215 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:40 smithi138 ceph-mon[185886]: failed to encode map e487 with expected crc 2023-10-01T09:40:40.215 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:40 smithi138 ceph-mon[185886]: failed to encode map e487 with expected crc 2023-10-01T09:40:40.216 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T09:40:40.243 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:40.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:40.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:40.350 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:40.363 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:40.414 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:40 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:40.414 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:40 smithi190 ceph-mon[160435]: failed to encode map e487 with expected crc 2023-10-01T09:40:40.414 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:40 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3728152840' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:40:40.414 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:40 smithi190 ceph-mon[160435]: failed to encode map e487 with expected crc 2023-10-01T09:40:40.415 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:40 smithi190 ceph-mon[160435]: osdmap e487: 8 total, 8 up, 7 in 2023-10-01T09:40:40.415 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:40 smithi190 ceph-mon[160435]: failed to encode map e487 with expected crc 2023-10-01T09:40:40.415 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:40 smithi190 ceph-mon[160435]: failed to encode map e487 with expected crc 2023-10-01T09:40:40.427 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:40.566 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:40.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:40.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:40.614 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 29 16 44961 44945 96.8214 72.8125 0.00289736 0.010319 2023-10-01T09:40:40.671 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:40.699 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:40.853 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:40.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:40.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:40.959 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:41.003 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:41.082 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:41.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:41.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:41.192 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:41.220 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:41.267 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:41.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:41.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:41.293 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:40:41.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:41 smithi138 ceph-mon[192513]: failed to encode map e487 with expected crc 2023-10-01T09:40:41.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:41 smithi138 ceph-mon[192513]: failed to encode map e487 with expected crc 2023-10-01T09:40:41.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:41 smithi138 ceph-mon[192513]: pgmap v1387: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T09:40:41.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:41.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:40:41.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:41 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1988842628' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:40:41.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:41 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:40:41.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:41 smithi138 ceph-mon[185886]: failed to encode map e487 with expected crc 2023-10-01T09:40:41.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:41 smithi138 ceph-mon[185886]: failed to encode map e487 with expected crc 2023-10-01T09:40:41.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:41 smithi138 ceph-mon[185886]: pgmap v1387: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T09:40:41.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:41.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:40:41.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:41 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1988842628' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:40:41.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:41 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:40:41.323 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.1 to 1 (10000) 2023-10-01T09:40:41.373 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:41.445 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:41 smithi190 ceph-mon[160435]: failed to encode map e487 with expected crc 2023-10-01T09:40:41.445 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:41 smithi190 ceph-mon[160435]: failed to encode map e487 with expected crc 2023-10-01T09:40:41.445 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:41 smithi190 ceph-mon[160435]: pgmap v1387: 89 pgs: 89 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T09:40:41.446 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:41.446 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:40:41.446 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:41 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1988842628' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:40:41.446 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:41 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:40:41.451 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:41.614 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 30 16 45872 45856 95.4923 56.9375 0.0154157 0.0104658 2023-10-01T09:40:41.624 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:41.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:41.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:41.731 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:42.039 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:42.072 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:40:42.287 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:42.290 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:42.312 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:42.312 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:40:42.312 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:40:42.312 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:42.312 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:40:42.312 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[192513]: osdmap e488: 8 total, 8 up, 7 in 2023-10-01T09:40:42.312 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[192513]: failed to encode map e488 with expected crc 2023-10-01T09:40:42.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:42.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:42.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:40:42.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:40:42.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:42.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:40:42.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[185886]: osdmap e488: 8 total, 8 up, 7 in 2023-10-01T09:40:42.313 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:42 smithi138 ceph-mon[185886]: failed to encode map e488 with expected crc 2023-10-01T09:40:42.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:42.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:42.395 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-01T09:40:42.415 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:42.466 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:42.515 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:42.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:42.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:40:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:40:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:42 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:40:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:42 smithi190 ceph-mon[160435]: osdmap e488: 8 total, 8 up, 7 in 2023-10-01T09:40:42.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:42 smithi190 ceph-mon[160435]: failed to encode map e488 with expected crc 2023-10-01T09:40:42.617 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 31 16 46677 46661 94.0265 50.3125 0.00363095 0.0106204 2023-10-01T09:40:42.619 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:42.657 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:42.788 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:42.856 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:40:42.899 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:40:42.922 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:42.928 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:40:42.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:42.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:42.958 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:40:42.991 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:40:42.993 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:43.021 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:40:43.037 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:43.053 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:40:43.098 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:40:43.102 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:40:43.103 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:40:43.103 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:40:43.103 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:40:43.103 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:40:43.103 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:40:43.103 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:40:43.103 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:40:43.111 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:43.188 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:43.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:43.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:43.298 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:43.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:43 smithi138 ceph-mon[192513]: failed to encode map e488 with expected crc 2023-10-01T09:40:43.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:43 smithi138 ceph-mon[192513]: pgmap v1389: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-01T09:40:43.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:43 smithi138 ceph-mon[192513]: failed to encode map e489 with expected crc 2023-10-01T09:40:43.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:43 smithi138 ceph-mon[192513]: osdmap e489: 8 total, 8 up, 7 in 2023-10-01T09:40:43.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:43 smithi138 ceph-mon[192513]: failed to encode map e489 with expected crc 2023-10-01T09:40:43.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:43 smithi138 ceph-mon[192513]: failed to encode map e489 with expected crc 2023-10-01T09:40:43.326 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:43 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2706677167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:43.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:43 smithi138 ceph-mon[185886]: failed to encode map e488 with expected crc 2023-10-01T09:40:43.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:43 smithi138 ceph-mon[185886]: pgmap v1389: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-01T09:40:43.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:43 smithi138 ceph-mon[185886]: failed to encode map e489 with expected crc 2023-10-01T09:40:43.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:43 smithi138 ceph-mon[185886]: osdmap e489: 8 total, 8 up, 7 in 2023-10-01T09:40:43.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:43 smithi138 ceph-mon[185886]: failed to encode map e489 with expected crc 2023-10-01T09:40:43.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:43 smithi138 ceph-mon[185886]: failed to encode map e489 with expected crc 2023-10-01T09:40:43.327 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:43 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2706677167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:43.349 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:43.475 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:43.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:43.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:43.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:43 smithi190 ceph-mon[160435]: failed to encode map e488 with expected crc 2023-10-01T09:40:43.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:43 smithi190 ceph-mon[160435]: pgmap v1389: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-01T09:40:43.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:43 smithi190 ceph-mon[160435]: failed to encode map e489 with expected crc 2023-10-01T09:40:43.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:43 smithi190 ceph-mon[160435]: osdmap e489: 8 total, 8 up, 7 in 2023-10-01T09:40:43.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:43 smithi190 ceph-mon[160435]: failed to encode map e489 with expected crc 2023-10-01T09:40:43.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:43 smithi190 ceph-mon[160435]: failed to encode map e489 with expected crc 2023-10-01T09:40:43.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:43 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2706677167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:43.581 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:43.617 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 32 16 47960 47944 93.594 80.1875 0.00846074 0.0106788 2023-10-01T09:40:43.631 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:40:43.631 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:40:43.634 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:44.161 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:44.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:44.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:44.265 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:44.294 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:44.361 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:44.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:44.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:44.473 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:44.500 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:44.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:44 smithi190 ceph-mon[160435]: failed to encode map e489 with expected crc 2023-10-01T09:40:44.618 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 33 16 48800 48784 92.3491 52.5 0.00499464 0.0108173 2023-10-01T09:40:44.807 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:44.815 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:40:44.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:44.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:44.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:44 smithi138 ceph-mon[192513]: failed to encode map e489 with expected crc 2023-10-01T09:40:44.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:44 smithi138 ceph-mon[185886]: failed to encode map e489 with expected crc 2023-10-01T09:40:44.920 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:44.990 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:45.104 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:40:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:40:44] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.33.4" 2023-10-01T09:40:45.229 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:45.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:45.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:45.335 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:45.362 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:45.419 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:45.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:45.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:45.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:45 smithi138 ceph-mon[192513]: pgmap v1391: 89 pgs: 1 activating, 88 active+clean; 2.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 56 MiB/s wr, 902 op/s; 4/134607 objects degraded (0.003%); 0 B/s, 0 objects/s recovering 2023-10-01T09:40:45.471 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:45 smithi138 ceph-mon[185886]: pgmap v1391: 89 pgs: 1 activating, 88 active+clean; 2.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 56 MiB/s wr, 902 op/s; 4/134607 objects degraded (0.003%); 0 B/s, 0 objects/s recovering 2023-10-01T09:40:45.527 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:45.576 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:45.618 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 34 16 49981 49965 91.8039 73.8125 0.0104689 0.0108874 2023-10-01T09:40:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:45 smithi190 ceph-mon[160435]: pgmap v1391: 89 pgs: 1 activating, 88 active+clean; 2.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 56 MiB/s wr, 902 op/s; 4/134607 objects degraded (0.003%); 0 B/s, 0 objects/s recovering 2023-10-01T09:40:45.816 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:45.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:45.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:45.909 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:45.926 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:45.976 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:46.209 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:46.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:46.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:46.317 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:46.436 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:46.615 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:46.618 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 35 16 51393 51377 91.7022 88.25 0.00630251 0.0108987 2023-10-01T09:40:46.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:46.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:46.725 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:46.776 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:46.962 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:40:46.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:46.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:40:47.070 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:47.096 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:40:47.128 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:47.195 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:47.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:47.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:47.303 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:47.329 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:47.343 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.7 to 1 (10000) 2023-10-01T09:40:47.345 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:40:47.366 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:47 smithi138 ceph-mon[192513]: pgmap v1392: 89 pgs: 1 activating, 88 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 4/145305 objects degraded (0.003%); 3 B/s, 0 objects/s recovering 2023-10-01T09:40:47.366 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:47 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2297192028' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:40:47.366 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:47 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2245053049' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:40:47.366 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:47 smithi138 ceph-mon[185886]: pgmap v1392: 89 pgs: 1 activating, 88 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 4/145305 objects degraded (0.003%); 3 B/s, 0 objects/s recovering 2023-10-01T09:40:47.366 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:47 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2297192028' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:40:47.367 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:47 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2245053049' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:40:47.389 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:47.404 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:47 smithi190 ceph-mon[160435]: pgmap v1392: 89 pgs: 1 activating, 88 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 4/145305 objects degraded (0.003%); 3 B/s, 0 objects/s recovering 2023-10-01T09:40:47.404 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:47 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2297192028' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:40:47.404 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:47 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2245053049' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:40:47.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:47.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:47.509 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:47.537 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:47.579 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:47.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:47.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:47.618 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 36 16 52366 52350 90.8442 60.8125 0.00273304 0.0109379 2023-10-01T09:40:47.685 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:47.712 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:48.224 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:48.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:48.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:48.328 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:40:48.356 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-01T09:40:48.359 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:48.410 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:40:48.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:48.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:40:48.517 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:48.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:48.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[185886]: failed to encode map e490 with expected crc 2023-10-01T09:40:48.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2297192028' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:40:48.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2245053049' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:40:48.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[185886]: failed to encode map e490 with expected crc 2023-10-01T09:40:48.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[185886]: osdmap e490: 8 total, 8 up, 7 in 2023-10-01T09:40:48.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[185886]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[185886]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[185886]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[185886]: osdmap e491: 8 total, 8 up, 7 in 2023-10-01T09:40:48.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[185886]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[185886]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:48.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[192513]: failed to encode map e490 with expected crc 2023-10-01T09:40:48.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2297192028' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:40:48.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2245053049' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:40:48.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[192513]: failed to encode map e490 with expected crc 2023-10-01T09:40:48.543 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[192513]: osdmap e490: 8 total, 8 up, 7 in 2023-10-01T09:40:48.543 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[192513]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.543 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[192513]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.543 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[192513]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.543 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[192513]: osdmap e491: 8 total, 8 up, 7 in 2023-10-01T09:40:48.543 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[192513]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.543 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:48 smithi138 ceph-mon[192513]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.543 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:48.618 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 37 16 52366 52350 88.3897 0 - 0.0109379 2023-10-01T09:40:48.772 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:48.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:48.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:48.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:48 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:40:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:48 smithi190 ceph-mon[160435]: failed to encode map e490 with expected crc 2023-10-01T09:40:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:48 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2297192028' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:40:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:48 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2245053049' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:40:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:48 smithi190 ceph-mon[160435]: failed to encode map e490 with expected crc 2023-10-01T09:40:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:48 smithi190 ceph-mon[160435]: osdmap e490: 8 total, 8 up, 7 in 2023-10-01T09:40:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:48 smithi190 ceph-mon[160435]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:48 smithi190 ceph-mon[160435]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:48 smithi190 ceph-mon[160435]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:48 smithi190 ceph-mon[160435]: osdmap e491: 8 total, 8 up, 7 in 2023-10-01T09:40:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:48 smithi190 ceph-mon[160435]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:48 smithi190 ceph-mon[160435]: failed to encode map e491 with expected crc 2023-10-01T09:40:48.876 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:48.934 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:49.018 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:49.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:49.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:49.122 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:49.161 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:49.193 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:49.228 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:49.289 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:40:49.290 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:40:49.295 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:49.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:49.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:49.405 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:49.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:49 smithi138 ceph-mon[185886]: failed to encode map e491 with expected crc 2023-10-01T09:40:49.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:49 smithi138 ceph-mon[185886]: failed to encode map e491 with expected crc 2023-10-01T09:40:49.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:49 smithi138 ceph-mon[185886]: failed to encode map e491 with expected crc 2023-10-01T09:40:49.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:49 smithi138 ceph-mon[185886]: pgmap v1395: 89 pgs: 1 activating, 88 active+clean; 3.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 59 MiB/s wr, 938 op/s; 4/150483 objects degraded (0.003%); 682 KiB/s, 2 objects/s recovering 2023-10-01T09:40:49.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:49 smithi138 ceph-mon[185886]: failed to encode map e492 with expected crc 2023-10-01T09:40:49.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:49 smithi138 ceph-mon[185886]: failed to encode map e492 with expected crc 2023-10-01T09:40:49.456 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:49 smithi138 ceph-mon[185886]: osdmap e492: 8 total, 8 up, 7 in 2023-10-01T09:40:49.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:49 smithi138 ceph-mon[192513]: failed to encode map e491 with expected crc 2023-10-01T09:40:49.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:49 smithi138 ceph-mon[192513]: failed to encode map e491 with expected crc 2023-10-01T09:40:49.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:49 smithi138 ceph-mon[192513]: failed to encode map e491 with expected crc 2023-10-01T09:40:49.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:49 smithi138 ceph-mon[192513]: pgmap v1395: 89 pgs: 1 activating, 88 active+clean; 3.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 59 MiB/s wr, 938 op/s; 4/150483 objects degraded (0.003%); 682 KiB/s, 2 objects/s recovering 2023-10-01T09:40:49.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:49 smithi138 ceph-mon[192513]: failed to encode map e492 with expected crc 2023-10-01T09:40:49.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:49 smithi138 ceph-mon[192513]: failed to encode map e492 with expected crc 2023-10-01T09:40:49.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:49 smithi138 ceph-mon[192513]: osdmap e492: 8 total, 8 up, 7 in 2023-10-01T09:40:49.460 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:49.618 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 38 16 53255 53239 87.526 27.7812 0.0133711 0.0114202 2023-10-01T09:40:49.645 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:49.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:49.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:49.752 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:49.778 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:49.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:49 smithi190 ceph-mon[160435]: failed to encode map e491 with expected crc 2023-10-01T09:40:49.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:49 smithi190 ceph-mon[160435]: failed to encode map e491 with expected crc 2023-10-01T09:40:49.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:49 smithi190 ceph-mon[160435]: failed to encode map e491 with expected crc 2023-10-01T09:40:49.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:49 smithi190 ceph-mon[160435]: pgmap v1395: 89 pgs: 1 activating, 88 active+clean; 3.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 59 MiB/s wr, 938 op/s; 4/150483 objects degraded (0.003%); 682 KiB/s, 2 objects/s recovering 2023-10-01T09:40:49.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:49 smithi190 ceph-mon[160435]: failed to encode map e492 with expected crc 2023-10-01T09:40:49.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:49 smithi190 ceph-mon[160435]: failed to encode map e492 with expected crc 2023-10-01T09:40:49.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:49 smithi190 ceph-mon[160435]: osdmap e492: 8 total, 8 up, 7 in 2023-10-01T09:40:49.819 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:49.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:49.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:49.923 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:49.966 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:50.302 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:50.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:50.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:50.410 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:50.471 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:50.531 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:40:50.565 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:50 smithi138 ceph-mon[185886]: failed to encode map e492 with expected crc 2023-10-01T09:40:50.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:50 smithi138 ceph-mon[185886]: failed to encode map e493 with expected crc 2023-10-01T09:40:50.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:50 smithi138 ceph-mon[185886]: osdmap e493: 8 total, 8 up, 7 in 2023-10-01T09:40:50.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:50 smithi138 ceph-mon[192513]: failed to encode map e492 with expected crc 2023-10-01T09:40:50.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:50 smithi138 ceph-mon[192513]: failed to encode map e493 with expected crc 2023-10-01T09:40:50.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:50 smithi138 ceph-mon[192513]: osdmap e493: 8 total, 8 up, 7 in 2023-10-01T09:40:50.580 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:50.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:50.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:50.618 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 39 16 54365 54349 87.0606 69.375 0.00633829 0.0114812 2023-10-01T09:40:50.693 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:40:50.759 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:50.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:50 smithi190 ceph-mon[160435]: failed to encode map e492 with expected crc 2023-10-01T09:40:50.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:50 smithi190 ceph-mon[160435]: failed to encode map e493 with expected crc 2023-10-01T09:40:50.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:50 smithi190 ceph-mon[160435]: osdmap e493: 8 total, 8 up, 7 in 2023-10-01T09:40:50.874 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:40:51.619 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:40:51.619109+0000 min lat: 0.00101103 max lat: 1.68936 avg lat: 0.0115319 2023-10-01T09:40:51.619 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:40:51.619 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 40 16 55509 55493 86.6716 71.5 0.00445888 0.0115319 2023-10-01T09:40:51.648 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:51 smithi138 ceph-mon[192513]: failed to encode map e493 with expected crc 2023-10-01T09:40:51.649 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:51 smithi138 ceph-mon[192513]: pgmap v1398: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 activating, 86 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s; 4/157104 objects degraded (0.003%); 3277/157104 objects misplaced (2.086%); 1.2 MiB/s, 8 objects/s recovering 2023-10-01T09:40:51.649 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:51 smithi138 ceph-mon[192513]: failed to encode map e494 with expected crc 2023-10-01T09:40:51.649 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:51 smithi138 ceph-mon[192513]: osdmap e494: 8 total, 8 up, 7 in 2023-10-01T09:40:51.649 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:51 smithi138 ceph-mon[192513]: failed to encode map e494 with expected crc 2023-10-01T09:40:51.649 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:51 smithi138 ceph-mon[185886]: failed to encode map e493 with expected crc 2023-10-01T09:40:51.649 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:51 smithi138 ceph-mon[185886]: pgmap v1398: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 activating, 86 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s; 4/157104 objects degraded (0.003%); 3277/157104 objects misplaced (2.086%); 1.2 MiB/s, 8 objects/s recovering 2023-10-01T09:40:51.650 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:51 smithi138 ceph-mon[185886]: failed to encode map e494 with expected crc 2023-10-01T09:40:51.650 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:51 smithi138 ceph-mon[185886]: osdmap e494: 8 total, 8 up, 7 in 2023-10-01T09:40:51.650 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:51 smithi138 ceph-mon[185886]: failed to encode map e494 with expected crc 2023-10-01T09:40:51.671 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:40:51.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:51.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:40:51.776 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:51 smithi190 ceph-mon[160435]: failed to encode map e493 with expected crc 2023-10-01T09:40:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:51 smithi190 ceph-mon[160435]: pgmap v1398: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 activating, 86 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s; 4/157104 objects degraded (0.003%); 3277/157104 objects misplaced (2.086%); 1.2 MiB/s, 8 objects/s recovering 2023-10-01T09:40:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:51 smithi190 ceph-mon[160435]: failed to encode map e494 with expected crc 2023-10-01T09:40:51.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:51 smithi190 ceph-mon[160435]: osdmap e494: 8 total, 8 up, 7 in 2023-10-01T09:40:51.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:51 smithi190 ceph-mon[160435]: failed to encode map e494 with expected crc 2023-10-01T09:40:51.850 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:51.943 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:51.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:51.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:52.048 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:52.075 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:52.477 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:52.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:52.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:52.586 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:52.618 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 41 16 56577 56561 86.1857 66.75 0.0172146 0.0115978 2023-10-01T09:40:52.639 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:52.750 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:52.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:52.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:52.787 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:52.867 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:52.915 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:40:52.933 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:52.945 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:40:52.980 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:40:53.016 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:40:53.031 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:53.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:53.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:53.051 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:40:53.090 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:40:53.123 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:40:53.140 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:40:53.157 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:40:53.163 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:40:53.163 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:40:53.163 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:40:53.163 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:40:53.163 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:40:53.163 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:40:53.163 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:40:53.164 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:40:53.195 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:53.291 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:40:53.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:53.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:40:53.398 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:53.464 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:53 smithi190 ceph-mon[160435]: pgmap v1400: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 87 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s; 3277/162390 objects misplaced (2.018%); 324 KiB/s, 8 objects/s recovering 2023-10-01T09:40:53.464 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2102674925' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:53.469 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:53.548 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:53.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:53.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:53.619 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 42 16 57680 57664 85.775 68.9375 0.01606 0.0116532 2023-10-01T09:40:53.662 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:53.729 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:53.790 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:53.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:53 smithi138 ceph-mon[192513]: pgmap v1400: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 87 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s; 3277/162390 objects misplaced (2.018%); 324 KiB/s, 8 objects/s recovering 2023-10-01T09:40:53.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2102674925' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:53.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:53 smithi138 ceph-mon[185886]: pgmap v1400: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 87 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s; 3277/162390 objects misplaced (2.018%); 324 KiB/s, 8 objects/s recovering 2023-10-01T09:40:53.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2102674925' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:40:53.836 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:53.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:53.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:53.945 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:54.006 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:54.115 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:54.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:54.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:54.222 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:40:54.290 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:54.338 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:40:54.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:54.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:40:54.398 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.5 to 1 (10000) 2023-10-01T09:40:54.445 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:40:54.496 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:54.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:54 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2781331678' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:40:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:54 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:40:54.619 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 43 16 58625 58609 85.1538 59.0625 0.00296242 0.0116935 2023-10-01T09:40:54.640 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:54 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2781331678' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:40:54.640 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:54 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:40:54.640 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:54 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2781331678' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:40:54.640 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:54 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:40:55.003 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:40:55.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:55.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:40:55.107 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:55.137 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:55.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:40:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:40:54] "GET /metrics HTTP/1.1" 200 36507 "" "Prometheus/2.33.4" 2023-10-01T09:40:55.619 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 44 16 58625 58609 83.2191 0 - 0.0116935 2023-10-01T09:40:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:55 smithi190 ceph-mon[160435]: pgmap v1401: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 87 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 44 MiB/s wr, 706 op/s; 3277/163194 objects misplaced (2.008%); 245 KiB/s, 6 objects/s recovering 2023-10-01T09:40:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:55 smithi190 ceph-mon[160435]: failed to encode map e495 with expected crc 2023-10-01T09:40:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:55 smithi190 ceph-mon[160435]: failed to encode map e495 with expected crc 2023-10-01T09:40:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:55 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:40:55.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:55 smithi190 ceph-mon[160435]: osdmap e495: 8 total, 8 up, 7 in 2023-10-01T09:40:55.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:55 smithi190 ceph-mon[160435]: failed to encode map e495 with expected crc 2023-10-01T09:40:55.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:40:55.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:55 smithi138 ceph-mon[185886]: pgmap v1401: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 87 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 44 MiB/s wr, 706 op/s; 3277/163194 objects misplaced (2.008%); 245 KiB/s, 6 objects/s recovering 2023-10-01T09:40:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:55 smithi138 ceph-mon[185886]: failed to encode map e495 with expected crc 2023-10-01T09:40:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:55 smithi138 ceph-mon[185886]: failed to encode map e495 with expected crc 2023-10-01T09:40:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:55 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:40:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:55 smithi138 ceph-mon[185886]: osdmap e495: 8 total, 8 up, 7 in 2023-10-01T09:40:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:55 smithi138 ceph-mon[185886]: failed to encode map e495 with expected crc 2023-10-01T09:40:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:40:55.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:55 smithi138 ceph-mon[192513]: pgmap v1401: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 87 active+clean; 3.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 44 MiB/s wr, 706 op/s; 3277/163194 objects misplaced (2.008%); 245 KiB/s, 6 objects/s recovering 2023-10-01T09:40:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:55 smithi138 ceph-mon[192513]: failed to encode map e495 with expected crc 2023-10-01T09:40:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:55 smithi138 ceph-mon[192513]: failed to encode map e495 with expected crc 2023-10-01T09:40:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:55 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:40:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:55 smithi138 ceph-mon[192513]: osdmap e495: 8 total, 8 up, 7 in 2023-10-01T09:40:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:55 smithi138 ceph-mon[192513]: failed to encode map e495 with expected crc 2023-10-01T09:40:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:40:56.619 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 45 16 58659 58643 81.4175 1.0625 0.0148159 0.0122772 2023-10-01T09:40:56.752 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[192513]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.752 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[192513]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.752 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[192513]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.752 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[192513]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.752 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[192513]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.753 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[192513]: osdmap e496: 8 total, 8 up, 7 in 2023-10-01T09:40:56.753 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[192513]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.753 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:56.753 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[192513]: osdmap e497: 8 total, 8 up, 7 in 2023-10-01T09:40:56.753 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[185886]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.753 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[185886]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.753 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[185886]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.754 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[185886]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.754 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[185886]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.754 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[185886]: osdmap e496: 8 total, 8 up, 7 in 2023-10-01T09:40:56.754 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[185886]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.754 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:56.754 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:56 smithi138 ceph-mon[185886]: osdmap e497: 8 total, 8 up, 7 in 2023-10-01T09:40:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:56 smithi190 ceph-mon[160435]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:56 smithi190 ceph-mon[160435]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:56 smithi190 ceph-mon[160435]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:56 smithi190 ceph-mon[160435]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:56 smithi190 ceph-mon[160435]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:56 smithi190 ceph-mon[160435]: osdmap e496: 8 total, 8 up, 7 in 2023-10-01T09:40:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:56 smithi190 ceph-mon[160435]: failed to encode map e496 with expected crc 2023-10-01T09:40:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:40:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:56 smithi190 ceph-mon[160435]: osdmap e497: 8 total, 8 up, 7 in 2023-10-01T09:40:56.848 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-01T09:40:56.885 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:56.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:56.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:56.913 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:40:56.990 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:57.065 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:57.619 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 46 16 59738 59722 81.1136 67.4375 0.00936507 0.0123224 2023-10-01T09:40:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:57 smithi190 ceph-mon[160435]: pgmap v1404: 89 pgs: 1 active+recovering+undersized+remapped, 2 peering, 86 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 836 op/s; 3042/172170 objects misplaced (1.767%); 2.5 MiB/s, 40 objects/s recovering 2023-10-01T09:40:57.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:57 smithi190 ceph-mon[160435]: failed to encode map e497 with expected crc 2023-10-01T09:40:57.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:57 smithi190 ceph-mon[160435]: failed to encode map e497 with expected crc 2023-10-01T09:40:57.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:57 smithi138 ceph-mon[192513]: pgmap v1404: 89 pgs: 1 active+recovering+undersized+remapped, 2 peering, 86 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 836 op/s; 3042/172170 objects misplaced (1.767%); 2.5 MiB/s, 40 objects/s recovering 2023-10-01T09:40:57.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:57 smithi138 ceph-mon[192513]: failed to encode map e497 with expected crc 2023-10-01T09:40:57.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:57 smithi138 ceph-mon[192513]: failed to encode map e497 with expected crc 2023-10-01T09:40:57.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:57 smithi138 ceph-mon[185886]: pgmap v1404: 89 pgs: 1 active+recovering+undersized+remapped, 2 peering, 86 active+clean; 3.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 52 MiB/s wr, 836 op/s; 3042/172170 objects misplaced (1.767%); 2.5 MiB/s, 40 objects/s recovering 2023-10-01T09:40:57.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:57 smithi138 ceph-mon[185886]: failed to encode map e497 with expected crc 2023-10-01T09:40:57.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:57 smithi138 ceph-mon[185886]: failed to encode map e497 with expected crc 2023-10-01T09:40:58.189 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:58.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:58.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:58.299 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:40:58.359 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:58.463 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:40:58.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:58.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:40:58.572 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:40:58.619 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 47 16 60728 60712 80.7042 61.875 0.00783586 0.0123839 2023-10-01T09:40:58.663 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:58.686 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:58.770 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:58 smithi190 ceph-mon[160435]: pgmap v1406: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 peering, 85 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 784 op/s; 10101/176505 objects misplaced (5.723%); 5.0 MiB/s, 79 objects/s recovering 2023-10-01T09:40:58.799 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:40:58.886 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:40:58.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:40:58.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:40:58.904 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:58 smithi138 ceph-mon[192513]: pgmap v1406: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 peering, 85 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 784 op/s; 10101/176505 objects misplaced (5.723%); 5.0 MiB/s, 79 objects/s recovering 2023-10-01T09:40:58.905 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:58 smithi138 ceph-mon[185886]: pgmap v1406: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 peering, 85 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 784 op/s; 10101/176505 objects misplaced (5.723%); 5.0 MiB/s, 79 objects/s recovering 2023-10-01T09:40:58.966 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:40:58.966 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:40:58.966 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:40:58.966 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:40:58.966 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:40:59.006 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:40:59.068 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:40:59.619 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 48 16 61738 61722 80.3372 63.125 0.015156 0.012441 2023-10-01T09:40:59.788 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:41:00.045 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:41:00.052 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3253645766' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:00.052 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:40:59 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:00.052 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3253645766' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:00.052 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:40:59 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:00.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:00.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:41:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3253645766' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:00.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:40:59 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:00.154 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:00.202 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:00.620 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 49 16 62752 62736 79.991 63.375 0.00705341 0.0124963 2023-10-01T09:41:01.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:00 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:41:01.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:00 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:41:01.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:00 smithi190 ceph-mon[160435]: failed to encode map e498 with expected crc 2023-10-01T09:41:01.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:00 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:41:01.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:00 smithi190 ceph-mon[160435]: osdmap e498: 8 total, 8 up, 7 in 2023-10-01T09:41:01.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:00 smithi190 ceph-mon[160435]: failed to encode map e498 with expected crc 2023-10-01T09:41:01.065 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:00 smithi190 ceph-mon[160435]: pgmap v1408: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s; 10101/179256 objects misplaced (5.635%); 5.0 MiB/s, 81 objects/s recovering 2023-10-01T09:41:01.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:00 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:41:01.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:00 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:41:01.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:00 smithi138 ceph-mon[192513]: failed to encode map e498 with expected crc 2023-10-01T09:41:01.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:00 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:41:01.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:00 smithi138 ceph-mon[192513]: osdmap e498: 8 total, 8 up, 7 in 2023-10-01T09:41:01.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:00 smithi138 ceph-mon[192513]: failed to encode map e498 with expected crc 2023-10-01T09:41:01.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:00 smithi138 ceph-mon[192513]: pgmap v1408: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s; 10101/179256 objects misplaced (5.635%); 5.0 MiB/s, 81 objects/s recovering 2023-10-01T09:41:01.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:00 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:41:01.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:00 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:41:01.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:00 smithi138 ceph-mon[185886]: failed to encode map e498 with expected crc 2023-10-01T09:41:01.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:00 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:41:01.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:00 smithi138 ceph-mon[185886]: osdmap e498: 8 total, 8 up, 7 in 2023-10-01T09:41:01.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:00 smithi138 ceph-mon[185886]: failed to encode map e498 with expected crc 2023-10-01T09:41:01.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:00 smithi138 ceph-mon[185886]: pgmap v1408: 89 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 3.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s; 10101/179256 objects misplaced (5.635%); 5.0 MiB/s, 81 objects/s recovering 2023-10-01T09:41:01.620 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 50 16 63732 63716 79.6162 61.25 0.00497556 0.0125546 2023-10-01T09:41:01.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:01.701 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:01.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:01.802 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:01.879 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:01.978 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:01.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:01.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:02.086 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:41:02.154 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:02.174 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:41:02.253 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:41:02.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:02.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:41:02.368 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:41:02.433 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:02.556 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:41:02.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:02.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:41:02.620 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 51 16 64706 64690 79.2487 60.875 0.052713 0.0126132 2023-10-01T09:41:02.669 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:02.712 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:02.965 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:02.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:02.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:03.071 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:03.136 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:03.236 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:03.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:03.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:03.354 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:41:03.417 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:03.435 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:03.530 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:41:03.531 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:03 smithi138 ceph-mon[192513]: pgmap v1409: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 855 op/s; 9889/188475 objects misplaced (5.247%); 4.1 MiB/s, 66 objects/s recovering 2023-10-01T09:41:03.534 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:03 smithi138 ceph-mon[185886]: pgmap v1409: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 855 op/s; 9889/188475 objects misplaced (5.247%); 4.1 MiB/s, 66 objects/s recovering 2023-10-01T09:41:03.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:03.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:41:03.620 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 52 16 65711 65695 78.9326 62.8125 0.0183119 0.0126633 2023-10-01T09:41:03.637 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:03.692 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:03 smithi190 ceph-mon[160435]: pgmap v1409: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 855 op/s; 9889/188475 objects misplaced (5.247%); 4.1 MiB/s, 66 objects/s recovering 2023-10-01T09:41:03.697 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:03.809 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:03.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:03.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:03.916 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:41:03.920 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:03.954 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:41:03.989 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:03.996 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:41:04.030 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:41:04.062 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:41:04.097 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:41:04.130 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:41:04.135 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:04.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:04.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:04.165 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:41:04.168 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:41:04.168 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:41:04.168 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:41:04.168 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:41:04.168 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:41:04.168 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:41:04.168 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:41:04.168 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:41:04.243 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:04.291 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:04.364 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.6 to 1 (10000) 2023-10-01T09:41:04.392 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1222394688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:04.392 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2156010664' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:41:04.393 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1222394688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:04.393 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2156010664' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:41:04.620 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 53 16 66601 66585 78.4929 55.625 0.0139491 0.0127351 2023-10-01T09:41:04.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1222394688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:04.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2156010664' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:41:04.816 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:41:05.321 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:41:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:41:04] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.33.4" 2023-10-01T09:41:05.623 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 54 16 67485 67469 78.059 55.25 0.0130608 0.0128053 2023-10-01T09:41:05.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:05 smithi190 ceph-mon[160435]: pgmap v1410: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 757 op/s; 9889/190356 objects misplaced (5.195%); 3.5 MiB/s, 57 objects/s recovering 2023-10-01T09:41:05.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:05 smithi190 ceph-mon[160435]: failed to encode map e499 with expected crc 2023-10-01T09:41:05.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2156010664' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:41:05.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:05 smithi190 ceph-mon[160435]: osdmap e499: 8 total, 8 up, 7 in 2023-10-01T09:41:05.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:05 smithi138 ceph-mon[185886]: pgmap v1410: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 757 op/s; 9889/190356 objects misplaced (5.195%); 3.5 MiB/s, 57 objects/s recovering 2023-10-01T09:41:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:05 smithi138 ceph-mon[185886]: failed to encode map e499 with expected crc 2023-10-01T09:41:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2156010664' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:41:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:05 smithi138 ceph-mon[185886]: osdmap e499: 8 total, 8 up, 7 in 2023-10-01T09:41:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:05 smithi138 ceph-mon[192513]: pgmap v1410: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 3.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 757 op/s; 9889/190356 objects misplaced (5.195%); 3.5 MiB/s, 57 objects/s recovering 2023-10-01T09:41:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:05 smithi138 ceph-mon[192513]: failed to encode map e499 with expected crc 2023-10-01T09:41:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2156010664' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:41:05.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:05 smithi138 ceph-mon[192513]: osdmap e499: 8 total, 8 up, 7 in 2023-10-01T09:41:05.934 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:05.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:05.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:06.041 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:06.094 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:06.178 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:06.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:06.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:06.284 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:41:06.327 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:06.519 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:41:06.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:06.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:41:06.622 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 55 16 68383 68367 77.6602 56.125 0.0326606 0.0128717 2023-10-01T09:41:06.625 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:06.723 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:06.792 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-01T09:41:06.830 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:06.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:06.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:06.941 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:41:06.995 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:07.020 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:07.106 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:41:07.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:07.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:41:07.217 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:07.278 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:07.407 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:07 smithi138 ceph-mon[192513]: pgmap v1412: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 50 MiB/s wr, 802 op/s; 9889/195777 objects misplaced (5.051%); 1.7 MiB/s, 26 objects/s recovering 2023-10-01T09:41:07.412 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:07 smithi138 ceph-mon[185886]: pgmap v1412: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 50 MiB/s wr, 802 op/s; 9889/195777 objects misplaced (5.051%); 1.7 MiB/s, 26 objects/s recovering 2023-10-01T09:41:07.426 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:07.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:07.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:07.464 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:41:07.464 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:41:07.536 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:41:07.594 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:07.623 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 56 16 69317 69301 77.3158 58.375 0.0265736 0.0129294 2023-10-01T09:41:07.697 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:41:07.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:07.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:41:07.721 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:07 smithi190 ceph-mon[160435]: pgmap v1412: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 50 MiB/s wr, 802 op/s; 9889/195777 objects misplaced (5.051%); 1.7 MiB/s, 26 objects/s recovering 2023-10-01T09:41:07.821 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:41:07.869 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:08.003 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:41:08.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:08.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:41:08.114 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:08.196 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:08.263 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:08.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:08.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:08.369 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:08.402 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:08.623 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 57 16 70130 70114 76.8509 50.8125 0.00909066 0.0130068 2023-10-01T09:41:08.631 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:41:09.063 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:08 smithi190 ceph-mon[160435]: pgmap v1413: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 9681/204423 objects misplaced (4.736%); 3.1 MiB/s, 49 objects/s recovering 2023-10-01T09:41:09.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:08 smithi138 ceph-mon[192513]: pgmap v1413: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 9681/204423 objects misplaced (4.736%); 3.1 MiB/s, 49 objects/s recovering 2023-10-01T09:41:09.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:08 smithi138 ceph-mon[185886]: pgmap v1413: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 9681/204423 objects misplaced (4.736%); 3.1 MiB/s, 49 objects/s recovering 2023-10-01T09:41:09.081 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:09.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:09.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:09.186 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:09.212 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:09.550 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:09.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:09.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:09.616 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:09.623 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 58 16 71010 70994 76.4742 55 0.0110499 0.0130712 2023-10-01T09:41:09.657 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:09.658 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:41:09.799 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:10.120 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:10.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:10.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:10.226 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:10.256 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:10.330 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:10.623 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 59 16 71907 71891 76.1282 56.0625 0.00715599 0.0131304 2023-10-01T09:41:11.257 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:11.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:11.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:11.359 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:11.379 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:41:11.431 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:11.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:11 smithi138 ceph-mon[185886]: pgmap v1414: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 946 op/s; 9681/207663 objects misplaced (4.662%); 2.6 MiB/s, 41 objects/s recovering 2023-10-01T09:41:11.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:11.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:41:11.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1114966399' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:41:11.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:11 smithi138 ceph-mon[192513]: pgmap v1414: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 946 op/s; 9681/207663 objects misplaced (4.662%); 2.6 MiB/s, 41 objects/s recovering 2023-10-01T09:41:11.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:11.543 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:41:11.543 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1114966399' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:41:11.625 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:41:11.625334+0000 min lat: 0.00101103 max lat: 2.19532 avg lat: 0.0131727 2023-10-01T09:41:11.625 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:41:11.625 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 60 16 72894 72878 75.8856 61.6875 0.0124968 0.0131727 2023-10-01T09:41:11.677 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:11.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:11.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:11.782 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:11.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:11 smithi190 ceph-mon[160435]: pgmap v1414: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 946 op/s; 9681/207663 objects misplaced (4.662%); 2.6 MiB/s, 41 objects/s recovering 2023-10-01T09:41:11.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:11.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:41:11.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1114966399' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:41:11.826 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:12.617 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:12 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:41:12.617 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:12 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1114966399' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:41:12.617 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:12 smithi138 ceph-mon[192513]: osdmap e500: 8 total, 8 up, 7 in 2023-10-01T09:41:12.617 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:12 smithi138 ceph-mon[192513]: failed to encode map e500 with expected crc 2023-10-01T09:41:12.617 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:12 smithi138 ceph-mon[192513]: failed to encode map e500 with expected crc 2023-10-01T09:41:12.617 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:12 smithi138 ceph-mon[192513]: failed to encode map e500 with expected crc 2023-10-01T09:41:12.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:12 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:41:12.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:12 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1114966399' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:41:12.619 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:12 smithi138 ceph-mon[185886]: osdmap e500: 8 total, 8 up, 7 in 2023-10-01T09:41:12.619 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:12 smithi138 ceph-mon[185886]: failed to encode map e500 with expected crc 2023-10-01T09:41:12.619 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:12 smithi138 ceph-mon[185886]: failed to encode map e500 with expected crc 2023-10-01T09:41:12.619 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:12 smithi138 ceph-mon[185886]: failed to encode map e500 with expected crc 2023-10-01T09:41:12.624 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 61 16 73767 73751 75.536 54.5625 0.0107233 0.0132332 2023-10-01T09:41:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:12 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:41:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:12 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1114966399' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:41:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:12 smithi190 ceph-mon[160435]: osdmap e500: 8 total, 8 up, 7 in 2023-10-01T09:41:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:12 smithi190 ceph-mon[160435]: failed to encode map e500 with expected crc 2023-10-01T09:41:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:12 smithi190 ceph-mon[160435]: failed to encode map e500 with expected crc 2023-10-01T09:41:12.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:12 smithi190 ceph-mon[160435]: failed to encode map e500 with expected crc 2023-10-01T09:41:13.303 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:13.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:13.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:13.407 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:41:13.445 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:13 smithi138 ceph-mon[185886]: failed to encode map e500 with expected crc 2023-10-01T09:41:13.446 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:13 smithi138 ceph-mon[185886]: pgmap v1416: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 9453/216123 objects misplaced (4.374%); 3.4 MiB/s, 54 objects/s recovering 2023-10-01T09:41:13.446 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:13 smithi138 ceph-mon[192513]: failed to encode map e500 with expected crc 2023-10-01T09:41:13.446 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:13 smithi138 ceph-mon[192513]: pgmap v1416: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 9453/216123 objects misplaced (4.374%); 3.4 MiB/s, 54 objects/s recovering 2023-10-01T09:41:13.449 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:13.539 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:13.626 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 62 16 74555 74539 75.1105 49.25 0.00533137 0.0133088 2023-10-01T09:41:13.664 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:41:13.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:13.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:41:13.782 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:13.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:13 smithi190 ceph-mon[160435]: failed to encode map e500 with expected crc 2023-10-01T09:41:13.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:13 smithi190 ceph-mon[160435]: pgmap v1416: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 9453/216123 objects misplaced (4.374%); 3.4 MiB/s, 54 objects/s recovering 2023-10-01T09:41:13.846 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:13.977 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:13.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:13.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:14.085 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:14.149 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:14.244 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:14.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:14.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:14.312 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:41:14.342 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:41:14.355 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:14.369 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:41:14.401 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:41:14.402 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.4 to 1 (10000) 2023-10-01T09:41:14.428 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:14.438 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1093085644' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:41:14.438 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:14 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:41:14.438 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3098053446' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:14.438 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1093085644' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:41:14.438 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:14 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:41:14.438 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3098053446' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:14.446 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:41:14.473 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:41:14.498 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:41:14.529 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:41:14.534 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:41:14.535 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:41:14.535 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:41:14.535 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:41:14.535 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:41:14.535 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:41:14.535 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:41:14.535 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:41:14.542 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:14.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:14.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:14.626 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 63 16 75457 75441 74.8132 56.375 0.00412245 0.0133607 2023-10-01T09:41:14.653 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:14.687 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1093085644' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:41:14.687 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:14 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:41:14.687 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3098053446' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:14.690 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:14.857 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:14.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:14.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:14.971 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:15.008 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:15.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:41:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:41:14] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.33.4" 2023-10-01T09:41:15.200 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:41:15.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:15 smithi138 ceph-mon[185886]: pgmap v1417: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 9453/217923 objects misplaced (4.338%); 2.7 MiB/s, 43 objects/s recovering 2023-10-01T09:41:15.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:15 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:41:15.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:15 smithi138 ceph-mon[185886]: osdmap e501: 8 total, 8 up, 7 in 2023-10-01T09:41:15.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:15 smithi138 ceph-mon[185886]: failed to encode map e501 with expected crc 2023-10-01T09:41:15.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:15 smithi138 ceph-mon[185886]: failed to encode map e501 with expected crc 2023-10-01T09:41:15.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:15 smithi138 ceph-mon[185886]: failed to encode map e501 with expected crc 2023-10-01T09:41:15.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:15 smithi138 ceph-mon[185886]: failed to encode map e501 with expected crc 2023-10-01T09:41:15.518 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:15 smithi138 ceph-mon[192513]: pgmap v1417: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 9453/217923 objects misplaced (4.338%); 2.7 MiB/s, 43 objects/s recovering 2023-10-01T09:41:15.518 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:15 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:41:15.518 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:15 smithi138 ceph-mon[192513]: osdmap e501: 8 total, 8 up, 7 in 2023-10-01T09:41:15.518 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:15 smithi138 ceph-mon[192513]: failed to encode map e501 with expected crc 2023-10-01T09:41:15.518 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:15 smithi138 ceph-mon[192513]: failed to encode map e501 with expected crc 2023-10-01T09:41:15.518 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:15 smithi138 ceph-mon[192513]: failed to encode map e501 with expected crc 2023-10-01T09:41:15.518 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:15 smithi138 ceph-mon[192513]: failed to encode map e501 with expected crc 2023-10-01T09:41:15.626 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 64 16 76289 76273 74.4567 52 0.0182851 0.0134252 2023-10-01T09:41:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:15 smithi190 ceph-mon[160435]: pgmap v1417: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 9453/217923 objects misplaced (4.338%); 2.7 MiB/s, 43 objects/s recovering 2023-10-01T09:41:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:15 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:41:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:15 smithi190 ceph-mon[160435]: osdmap e501: 8 total, 8 up, 7 in 2023-10-01T09:41:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:15 smithi190 ceph-mon[160435]: failed to encode map e501 with expected crc 2023-10-01T09:41:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:15 smithi190 ceph-mon[160435]: failed to encode map e501 with expected crc 2023-10-01T09:41:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:15 smithi190 ceph-mon[160435]: failed to encode map e501 with expected crc 2023-10-01T09:41:15.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:15 smithi190 ceph-mon[160435]: failed to encode map e501 with expected crc 2023-10-01T09:41:16.289 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:16.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:16.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:16.316 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-01T09:41:16.404 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:16.478 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:16.627 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 65 16 77176 77160 74.1635 55.4375 0.0177282 0.0134794 2023-10-01T09:41:16.630 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:16.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:16.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:16.749 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:41:16.786 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:16.980 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:41:16.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:16.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:41:17.089 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:17.131 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:17.263 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:17.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:17.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:17.345 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:17.376 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:17.402 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:17 smithi190 ceph-mon[160435]: pgmap v1419: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 766 op/s; 9453/222831 objects misplaced (4.242%); 1.8 MiB/s, 28 objects/s recovering 2023-10-01T09:41:17.415 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:17.442 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:41:17.442 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:41:17.442 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.442 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:77897 10.2 10:4f1de193:::benchmark_data_smithi138_573643_object77896:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e501)", 2023-10-01T09:41:17.442 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:41:17.438225+0000", 2023-10-01T09:41:17.442 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0036819840000000001, 2023-10-01T09:41:17.443 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037178799999999998, 2023-10-01T09:41:17.443 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:41:17.443 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:41:17.443 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:41:17.443 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:41:17.443 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:41:17.443 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77897 2023-10-01T09:41:17.443 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.443 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:41:17.443 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.443 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:41:17.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438225+0000", 2023-10-01T09:41:17.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:41:17.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:41:17.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438225+0000", 2023-10-01T09:41:17.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:41:17.444 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.444 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.444 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:41:17.444 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438223+0000", 2023-10-01T09:41:17.444 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:41:17.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:41:17.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438265+0000", 2023-10-01T09:41:17.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1817999999999998e-05 2023-10-01T09:41:17.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.445 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.445 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:41:17.445 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438285+0000", 2023-10-01T09:41:17.445 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0313e-05 2023-10-01T09:41:17.445 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:41:17.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438310+0000", 2023-10-01T09:41:17.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4511999999999998e-05 2023-10-01T09:41:17.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.446 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:41:17.446 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438409+0000", 2023-10-01T09:41:17.446 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9684000000000003e-05 2023-10-01T09:41:17.446 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.446 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:41:17.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438589+0000", 2023-10-01T09:41:17.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017981599999999999 2023-10-01T09:41:17.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.447 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.447 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:41:17.447 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438748+0000", 2023-10-01T09:41:17.447 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000159171 2023-10-01T09:41:17.447 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:41:17.447 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:41:17.447 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:41:17.447 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.448 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.448 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:77898 10.2 10:4bb1e7d8:::benchmark_data_smithi138_573643_object77897:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e501)", 2023-10-01T09:41:17.448 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:41:17.438329+0000", 2023-10-01T09:41:17.448 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.003577687, 2023-10-01T09:41:17.448 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00367674, 2023-10-01T09:41:17.448 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:41:17.448 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:41:17.448 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:41:17.448 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:41:17.448 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:41:17.448 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 77898 2023-10-01T09:41:17.448 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.449 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:41:17.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:41:17.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438329+0000", 2023-10-01T09:41:17.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:41:17.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.449 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:41:17.449 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438329+0000", 2023-10-01T09:41:17.449 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:41:17.449 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.449 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:41:17.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438328+0000", 2023-10-01T09:41:17.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:41:17.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:41:17.450 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438352+0000", 2023-10-01T09:41:17.450 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3774e-05 2023-10-01T09:41:17.450 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.450 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.450 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:41:17.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438363+0000", 2023-10-01T09:41:17.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0597e-05 2023-10-01T09:41:17.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:41:17.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.438368+0000", 2023-10-01T09:41:17.451 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7379999999999999e-06 2023-10-01T09:41:17.451 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.451 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.451 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:41:17.451 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.441687+0000", 2023-10-01T09:41:17.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003318143 2023-10-01T09:41:17.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:41:17.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.441834+0000", 2023-10-01T09:41:17.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014776599999999999 2023-10-01T09:41:17.452 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.452 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.452 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:41:17.452 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.441973+0000", 2023-10-01T09:41:17.452 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000138543 2023-10-01T09:41:17.453 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:41:17.453 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:41:17.453 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:41:17.453 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.453 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.453 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:77902 10.0 e501/483 10:0c894d7e:::benchmark_data_smithi138_573643_object77901:head v 501'4879, mlcod=501'4877)", 2023-10-01T09:41:17.453 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:41:17.440482+0000", 2023-10-01T09:41:17.453 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0014242720000000001, 2023-10-01T09:41:17.453 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015881490000000001, 2023-10-01T09:41:17.453 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:41:17.453 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:41:17.454 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:41:17.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:41:17.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.440482+0000", 2023-10-01T09:41:17.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:41:17.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.454 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:41:17.454 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.440482+0000", 2023-10-01T09:41:17.454 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:41:17.454 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.454 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:41:17.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.440482+0000", 2023-10-01T09:41:17.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:41:17.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:41:17.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.440514+0000", 2023-10-01T09:41:17.455 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2978999999999997e-05 2023-10-01T09:41:17.455 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.455 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.455 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:41:17.455 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.440528+0000", 2023-10-01T09:41:17.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3791999999999999e-05 2023-10-01T09:41:17.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:41:17.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.440536+0000", 2023-10-01T09:41:17.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1550000000000007e-06 2023-10-01T09:41:17.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.456 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.456 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:41:17.456 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.441347+0000", 2023-10-01T09:41:17.456 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00081065599999999999 2023-10-01T09:41:17.456 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:41:17.457 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:41:17.457 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:41:17.457 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:41:17.441367+0000", 2023-10-01T09:41:17.457 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0322999999999998e-05 2023-10-01T09:41:17.457 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:41:17.457 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:41:17.457 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:41:17.457 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:41:17.457 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:41:17.457 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 3 2023-10-01T09:41:17.457 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:41:17.458 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:17.544 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:17.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:17.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:17.627 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 66 16 78077 78061 73.893 56.3125 0.0243154 0.0135267 2023-10-01T09:41:17.657 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:17.704 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:17 smithi138 ceph-mon[185886]: pgmap v1419: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 766 op/s; 9453/222831 objects misplaced (4.242%); 1.8 MiB/s, 28 objects/s recovering 2023-10-01T09:41:17.704 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:17 smithi138 ceph-mon[192513]: pgmap v1419: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 766 op/s; 9453/222831 objects misplaced (4.242%); 1.8 MiB/s, 28 objects/s recovering 2023-10-01T09:41:17.711 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:18.627 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 67 16 78923 78907 73.5794 52.875 0.0266726 0.013586 2023-10-01T09:41:18.645 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:18.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:18.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:18.755 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:18.809 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:18.862 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:41:18.907 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:18 smithi138 ceph-mon[192513]: pgmap v1420: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 9234/230739 objects misplaced (4.002%); 3.5 MiB/s, 55 objects/s recovering 2023-10-01T09:41:18.909 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:18 smithi138 ceph-mon[185886]: pgmap v1420: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 9234/230739 objects misplaced (4.002%); 3.5 MiB/s, 55 objects/s recovering 2023-10-01T09:41:19.041 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:41:19.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:18 smithi190 ceph-mon[160435]: pgmap v1420: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 9234/230739 objects misplaced (4.002%); 3.5 MiB/s, 55 objects/s recovering 2023-10-01T09:41:19.114 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:19.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:19.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:19.225 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:19.305 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:19.627 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 68 16 79833 79817 73.3337 56.875 0.0109802 0.0136307 2023-10-01T09:41:20.627 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 69 16 80576 80560 72.944 46.4375 0.0152698 0.0137037 2023-10-01T09:41:20.765 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:20.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:20.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:20.822 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:20.878 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:20.932 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:21.044 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:21.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:21.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:21.122 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:21.160 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:21.233 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:21.455 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:21 smithi138 ceph-mon[185886]: pgmap v1421: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 833 op/s; 9234/234114 objects misplaced (3.944%); 3.1 MiB/s, 50 objects/s recovering 2023-10-01T09:41:21.457 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:21 smithi138 ceph-mon[192513]: pgmap v1421: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 833 op/s; 9234/234114 objects misplaced (3.944%); 3.1 MiB/s, 50 objects/s recovering 2023-10-01T09:41:21.563 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:21.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:21.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:21.627 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 70 16 81405 81389 72.6421 51.8125 0.0148241 0.0137604 2023-10-01T09:41:21.669 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:21.745 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:21.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:21 smithi190 ceph-mon[160435]: pgmap v1421: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 833 op/s; 9234/234114 objects misplaced (3.944%); 3.1 MiB/s, 50 objects/s recovering 2023-10-01T09:41:22.086 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:22.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:22.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:22.192 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:22.239 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:22.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:22 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3268277119' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:22.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:22 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:22.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:22 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4060702513' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:41:22.342 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:22 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:41:22.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:22 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3268277119' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:22.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:22 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:22.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:22 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4060702513' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:41:22.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:22 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:41:22.373 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.3 to 1 (10000) 2023-10-01T09:41:22.373 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:41:22.627 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 71 16 82202 82186 72.3206 49.8125 0.00388174 0.0137879 2023-10-01T09:41:22.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:22 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3268277119' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:22 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:22 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4060702513' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:41:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:22 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:41:23.628 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 72 16 82202 82186 71.3164 0 - 0.0137879 2023-10-01T09:41:23.772 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[192513]: pgmap v1422: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s; 9005/241962 objects misplaced (3.722%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T09:41:23.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:41:23.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[192513]: failed to encode map e502 with expected crc 2023-10-01T09:41:23.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:41:23.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:41:23.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[192513]: osdmap e502: 8 total, 8 up, 7 in 2023-10-01T09:41:23.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[192513]: failed to encode map e502 with expected crc 2023-10-01T09:41:23.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[192513]: failed to encode map e503 with expected crc 2023-10-01T09:41:23.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[192513]: osdmap e503: 8 total, 8 up, 7 in 2023-10-01T09:41:23.773 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[192513]: failed to encode map e503 with expected crc 2023-10-01T09:41:23.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[185886]: pgmap v1422: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s; 9005/241962 objects misplaced (3.722%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T09:41:23.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:41:23.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[185886]: failed to encode map e502 with expected crc 2023-10-01T09:41:23.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:41:23.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:41:23.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[185886]: osdmap e502: 8 total, 8 up, 7 in 2023-10-01T09:41:23.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[185886]: failed to encode map e502 with expected crc 2023-10-01T09:41:23.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[185886]: failed to encode map e503 with expected crc 2023-10-01T09:41:23.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[185886]: osdmap e503: 8 total, 8 up, 7 in 2023-10-01T09:41:23.774 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:23 smithi138 ceph-mon[185886]: failed to encode map e503 with expected crc 2023-10-01T09:41:23.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:23 smithi190 ceph-mon[160435]: pgmap v1422: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s; 9005/241962 objects misplaced (3.722%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T09:41:23.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:23 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:41:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:23 smithi190 ceph-mon[160435]: failed to encode map e502 with expected crc 2023-10-01T09:41:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:23 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:41:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:23 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:41:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:23 smithi190 ceph-mon[160435]: osdmap e502: 8 total, 8 up, 7 in 2023-10-01T09:41:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:23 smithi190 ceph-mon[160435]: failed to encode map e502 with expected crc 2023-10-01T09:41:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:23 smithi190 ceph-mon[160435]: failed to encode map e503 with expected crc 2023-10-01T09:41:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:23 smithi190 ceph-mon[160435]: osdmap e503: 8 total, 8 up, 7 in 2023-10-01T09:41:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:23 smithi190 ceph-mon[160435]: failed to encode map e503 with expected crc 2023-10-01T09:41:24.566 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:24.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:24 smithi138 ceph-mon[185886]: failed to encode map e502 with expected crc 2023-10-01T09:41:24.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:24 smithi138 ceph-mon[185886]: failed to encode map e503 with expected crc 2023-10-01T09:41:24.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:24 smithi138 ceph-mon[185886]: failed to encode map e503 with expected crc 2023-10-01T09:41:24.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:24 smithi138 ceph-mon[185886]: failed to encode map e503 with expected crc 2023-10-01T09:41:24.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:24 smithi138 ceph-mon[185886]: osdmap e504: 8 total, 8 up, 7 in 2023-10-01T09:41:24.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:24 smithi138 ceph-mon[192513]: failed to encode map e502 with expected crc 2023-10-01T09:41:24.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:24 smithi138 ceph-mon[192513]: failed to encode map e503 with expected crc 2023-10-01T09:41:24.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:24 smithi138 ceph-mon[192513]: failed to encode map e503 with expected crc 2023-10-01T09:41:24.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:24 smithi138 ceph-mon[192513]: failed to encode map e503 with expected crc 2023-10-01T09:41:24.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:24 smithi138 ceph-mon[192513]: osdmap e504: 8 total, 8 up, 7 in 2023-10-01T09:41:24.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:24.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:24.628 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 73 16 82346 82330 70.4629 4.5 0.00349114 0.0139471 2023-10-01T09:41:24.670 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:24.701 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:24.799 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:24.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:24.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:24 smithi190 ceph-mon[160435]: failed to encode map e502 with expected crc 2023-10-01T09:41:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:24 smithi190 ceph-mon[160435]: failed to encode map e503 with expected crc 2023-10-01T09:41:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:24 smithi190 ceph-mon[160435]: failed to encode map e503 with expected crc 2023-10-01T09:41:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:24 smithi190 ceph-mon[160435]: failed to encode map e503 with expected crc 2023-10-01T09:41:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:24 smithi190 ceph-mon[160435]: osdmap e504: 8 total, 8 up, 7 in 2023-10-01T09:41:24.903 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:24.930 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:25.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:41:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:41:24] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.33.4" 2023-10-01T09:41:25.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:25 smithi138 ceph-mon[192513]: failed to encode map e504 with expected crc 2023-10-01T09:41:25.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:25 smithi138 ceph-mon[192513]: failed to encode map e504 with expected crc 2023-10-01T09:41:25.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:25 smithi138 ceph-mon[192513]: pgmap v1426: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 45 MiB/s wr, 713 op/s; 9005/243582 objects misplaced (3.697%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T09:41:25.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:25.599 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:41:25.600 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:25 smithi138 ceph-mon[185886]: failed to encode map e504 with expected crc 2023-10-01T09:41:25.600 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:25 smithi138 ceph-mon[185886]: failed to encode map e504 with expected crc 2023-10-01T09:41:25.600 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:25 smithi138 ceph-mon[185886]: pgmap v1426: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 45 MiB/s wr, 713 op/s; 9005/243582 objects misplaced (3.697%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T09:41:25.600 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:25.600 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:41:25.628 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 74 16 82346 82330 69.511 0 - 0.0139471 2023-10-01T09:41:25.699 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:25.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:25.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:25.804 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:25 smithi190 ceph-mon[160435]: failed to encode map e504 with expected crc 2023-10-01T09:41:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:25 smithi190 ceph-mon[160435]: failed to encode map e504 with expected crc 2023-10-01T09:41:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:25 smithi190 ceph-mon[160435]: pgmap v1426: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 45 MiB/s wr, 713 op/s; 9005/243582 objects misplaced (3.697%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T09:41:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:41:25.818 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-01T09:41:25.818 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:41:25.833 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:25.870 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:41:25.870 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:41:25.870 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:41:25.871 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:41:25.871 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:41:25.879 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:25.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:25.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:25.961 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:41:25.983 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:41:25.986 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:41:26.004 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:41:26.013 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:26.026 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:41:26.046 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:41:26.058 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:41:26.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:26.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:41:26.165 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:26.194 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:26.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:26 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2632851881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:26 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2632851881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:26.628 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 75 16 82346 82330 68.5844 0 - 0.0139471 2023-10-01T09:41:26.685 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:41:26.771 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:26.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:26.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:26.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:26 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2632851881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:26.875 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:26.891 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:41:26.905 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:26.976 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:26.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:26.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:26.999 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:27.080 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:27.108 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:27.135 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:27.160 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:27.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:27.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:27.267 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:27.299 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:27.501 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:27.503 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:27 smithi138 ceph-mon[192513]: pgmap v1427: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 37 MiB/s wr, 595 op/s; 9005/244839 objects misplaced (3.678%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T09:41:27.504 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:27 smithi138 ceph-mon[185886]: pgmap v1427: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 37 MiB/s wr, 595 op/s; 9005/244839 objects misplaced (3.678%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T09:41:27.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:27.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:27.605 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:27.628 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 76 16 82346 82330 67.6822 0 - 0.0139471 2023-10-01T09:41:27.630 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:27 smithi190 ceph-mon[160435]: pgmap v1427: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+remapped, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 37 MiB/s wr, 595 op/s; 9005/244839 objects misplaced (3.678%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T09:41:27.633 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:27.687 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:27.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:27.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:27.794 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:41:27.823 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:28.628 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 77 16 82346 82330 66.8035 0 - 0.0139471 2023-10-01T09:41:29.024 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:41:29.025 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:28 smithi138 ceph-mon[192513]: pgmap v1428: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 15 MiB/s wr, 233 op/s; 10/246168 objects degraded (0.004%); 8105/246168 objects misplaced (3.292%); 9.4 MiB/s, 149 objects/s recovering 2023-10-01T09:41:29.026 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:28 smithi138 ceph-mon[185886]: pgmap v1428: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 15 MiB/s wr, 233 op/s; 10/246168 objects degraded (0.004%); 8105/246168 objects misplaced (3.292%); 9.4 MiB/s, 149 objects/s recovering 2023-10-01T09:41:29.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:29.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:41:29.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:28 smithi190 ceph-mon[160435]: pgmap v1428: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 15 MiB/s wr, 233 op/s; 10/246168 objects degraded (0.004%); 8105/246168 objects misplaced (3.292%); 9.4 MiB/s, 149 objects/s recovering 2023-10-01T09:41:29.128 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:29.158 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:29.274 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:29.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:29.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:29.378 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:29.409 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:29.429 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:29.628 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 78 16 82346 82330 65.9472 0 - 0.0139471 2023-10-01T09:41:29.762 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:29.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:29.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:29.870 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:41:29.901 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:29.992 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:41:29.994 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:41:29.995 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:29 smithi138 ceph-mon[192513]: Health check failed: Degraded data redundancy: 10/246168 objects degraded (0.004%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:41:29.995 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:29 smithi138 ceph-mon[185886]: Health check failed: Degraded data redundancy: 10/246168 objects degraded (0.004%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:41:30.056 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:41:30.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:29 smithi190 ceph-mon[160435]: Health check failed: Degraded data redundancy: 10/246168 objects degraded (0.004%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:41:30.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:30.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:41:30.161 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:30.187 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:30.238 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:30.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:30.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:30.343 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:30.374 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:30.520 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:30.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:30.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:30.625 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:30.628 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 79 16 82346 82330 65.1126 0 - 0.0139471 2023-10-01T09:41:30.651 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:30.772 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:41:31.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:30 smithi190 ceph-mon[160435]: failed to encode map e505 with expected crc 2023-10-01T09:41:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:30 smithi190 ceph-mon[160435]: osdmap e505: 8 total, 8 up, 7 in 2023-10-01T09:41:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:30 smithi190 ceph-mon[160435]: failed to encode map e505 with expected crc 2023-10-01T09:41:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:30 smithi190 ceph-mon[160435]: failed to encode map e505 with expected crc 2023-10-01T09:41:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:30 smithi190 ceph-mon[160435]: from='client.46601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:41:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:30 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3639731035' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:41:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:30 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:41:31.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:30 smithi190 ceph-mon[160435]: pgmap v1430: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 7.1 MiB/s wr, 114 op/s; 10/246168 objects degraded (0.004%); 8105/246168 objects misplaced (3.292%); 7.5 MiB/s, 119 objects/s recovering 2023-10-01T09:41:31.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[192513]: failed to encode map e505 with expected crc 2023-10-01T09:41:31.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[192513]: osdmap e505: 8 total, 8 up, 7 in 2023-10-01T09:41:31.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[192513]: failed to encode map e505 with expected crc 2023-10-01T09:41:31.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[192513]: failed to encode map e505 with expected crc 2023-10-01T09:41:31.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[192513]: from='client.46601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:41:31.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3639731035' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:41:31.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:41:31.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[192513]: pgmap v1430: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 7.1 MiB/s wr, 114 op/s; 10/246168 objects degraded (0.004%); 8105/246168 objects misplaced (3.292%); 7.5 MiB/s, 119 objects/s recovering 2023-10-01T09:41:31.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[185886]: failed to encode map e505 with expected crc 2023-10-01T09:41:31.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[185886]: osdmap e505: 8 total, 8 up, 7 in 2023-10-01T09:41:31.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[185886]: failed to encode map e505 with expected crc 2023-10-01T09:41:31.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[185886]: failed to encode map e505 with expected crc 2023-10-01T09:41:31.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[185886]: from='client.46601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:41:31.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3639731035' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:41:31.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:41:31.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:30 smithi138 ceph-mon[185886]: pgmap v1430: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 7.1 MiB/s wr, 114 op/s; 10/246168 objects degraded (0.004%); 8105/246168 objects misplaced (3.292%); 7.5 MiB/s, 119 objects/s recovering 2023-10-01T09:41:31.198 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:31.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:31.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:31.302 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:41:31.329 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:31.377 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:41:31.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:31.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:41:31.482 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:41:31.510 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:31.628 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:2023-10-01T09:41:31.628994+0000 min lat: 0.00101103 max lat: 2.19532 avg lat: 0.0139471 2023-10-01T09:41:31.628 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T09:41:31.629 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 80 16 82346 82330 64.2989 0 - 0.0139471 2023-10-01T09:41:31.760 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:31 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:41:31.760 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:31 smithi138 ceph-mon[192513]: failed to encode map e506 with expected crc 2023-10-01T09:41:31.760 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:31 smithi138 ceph-mon[192513]: failed to encode map e506 with expected crc 2023-10-01T09:41:31.761 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:31 smithi138 ceph-mon[192513]: failed to encode map e506 with expected crc 2023-10-01T09:41:31.761 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:31 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:41:31.761 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:31 smithi138 ceph-mon[192513]: osdmap e506: 8 total, 8 up, 7 in 2023-10-01T09:41:31.761 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:31 smithi138 ceph-mon[192513]: failed to encode map e506 with expected crc 2023-10-01T09:41:32.011 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:41:32.014 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:31 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:41:32.014 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:31 smithi138 ceph-mon[185886]: failed to encode map e506 with expected crc 2023-10-01T09:41:32.014 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:31 smithi138 ceph-mon[185886]: failed to encode map e506 with expected crc 2023-10-01T09:41:32.014 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:31 smithi138 ceph-mon[185886]: failed to encode map e506 with expected crc 2023-10-01T09:41:32.014 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:31 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:41:32.014 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:31 smithi138 ceph-mon[185886]: osdmap e506: 8 total, 8 up, 7 in 2023-10-01T09:41:32.014 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:31 smithi138 ceph-mon[185886]: failed to encode map e506 with expected crc 2023-10-01T09:41:32.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:32.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:41:32.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:31 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:41:32.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:31 smithi190 ceph-mon[160435]: failed to encode map e506 with expected crc 2023-10-01T09:41:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:31 smithi190 ceph-mon[160435]: failed to encode map e506 with expected crc 2023-10-01T09:41:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:31 smithi190 ceph-mon[160435]: failed to encode map e506 with expected crc 2023-10-01T09:41:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:31 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:41:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:31 smithi190 ceph-mon[160435]: osdmap e506: 8 total, 8 up, 7 in 2023-10-01T09:41:32.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:31 smithi190 ceph-mon[160435]: failed to encode map e506 with expected crc 2023-10-01T09:41:32.117 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:41:32.146 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:32.245 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:41:32.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:32.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:41:32.349 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:32.377 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:32.628 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 81 16 82346 82330 63.5053 0 - 0.0139471 2023-10-01T09:41:33.015 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:32 smithi138 ceph-mon[185886]: pgmap v1432: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 6.7 MiB/s wr, 107 op/s; 10/246168 objects degraded (0.004%); 7300/246168 objects misplaced (2.965%); 13 MiB/s, 213 objects/s recovering 2023-10-01T09:41:33.015 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:32 smithi138 ceph-mon[192513]: pgmap v1432: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 6.7 MiB/s wr, 107 op/s; 10/246168 objects degraded (0.004%); 7300/246168 objects misplaced (2.965%); 13 MiB/s, 213 objects/s recovering 2023-10-01T09:41:33.016 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1429,"stamp":"2023-10-01T09:41:29.754848+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5380577203,"num_objects":82056,"num_object_clones":0,"num_object_copies":246168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":8105,"num_objects_unfound":0,"num_objects_dirty":82056,"num_whiteouts":0,"num_read":6259,"num_read_kb":12659,"num_write":83134,"num_write_kb":5264933,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2997,"num_bytes_recovered":183272463,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83198,"ondisk_log_size":83198,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":281,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":22081572,"kb_used_data":17075372,"kb_used_omap":39,"kb_used_meta":5006104,"kb_avail":634134492,"statfs":{"total":671965249536,"available":649353719808,"internally_reserved":0,"allocated":17485180928,"data_stored":17475504861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5126251119},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":91881472,"num_objects":1402,"num_object_clones":0,"num_object_copies":4206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":-900,"num_objects_unfound":0,"num_objects_dirty":1402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1402,"num_write_kb":89728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":900,"num_bytes_recovered":58982400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.401815"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":280,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.751109+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:41:22.751109+0000","last_peered":"2023-10-01T09:41:22.751109+0000","last_clean":"2023-10-01T09:41:22.751109+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:41:22.751109+0000","last_undegraded":"2023-10-01T09:41:22.751109+0000","last_fullsized":"2023-10-01T09:41:22.751109+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":122,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.746879+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:41:22.746879+0000","last_peered":"2023-10-01T09:41:22.746879+0000","last_clean":"2023-10-01T09:41:22.746879+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:41:22.746879+0000","last_undegraded":"2023-10-01T09:41:22.746879+0000","last_fullsized":"2023-10-01T09:41:22.746879+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":503,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.747334+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:41:22.747334+0000","last_peered":"2023-10-01T09:41:22.747334+0000","last_clean":"2023-10-01T09:41:22.747334+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:41:22.747334+0000","last_undegraded":"2023-10-01T09:41:22.747334+0000","last_fullsized":"2023-10-01T09:41:22.747334+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":561,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.760090+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:41:22.760090+0000","last_peered":"2023-10-01T09:41:22.760090+0000","last_clean":"2023-10-01T09:41:22.760090+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:41:22.760090+0000","last_undegraded":"2023-10-01T09:41:22.760090+0000","last_fullsized":"2023-10-01T09:41:22.760090+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":442,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.744520+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:41:22.744520+0000","last_peered":"2023-10-01T09:41:22.744520+0000","last_clean":"2023-10-01T09:41:22.744520+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:41:22.744520+0000","last_undegraded":"2023-10-01T09:41:22.744520+0000","last_fullsized":"2023-10-01T09:41:22.744520+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":184,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.351000+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:41:11.351000+0000","last_peered":"2023-10-01T09:41:11.351000+0000","last_clean":"2023-10-01T09:41:11.351000+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:41:11.351000+0000","last_undegraded":"2023-10-01T09:41:11.351000+0000","last_fullsized":"2023-10-01T09:41:11.351000+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":560,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.351087+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:41:11.351087+0000","last_peered":"2023-10-01T09:41:11.351087+0000","last_clean":"2023-10-01T09:41:11.351087+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:41:11.351087+0000","last_undegraded":"2023-10-01T09:41:11.351087+0000","last_fullsized":"2023-10-01T09:41:11.351087+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":489,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.755033+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:41:22.755033+0000","last_peered":"2023-10-01T09:41:22.755033+0000","last_clean":"2023-10-01T09:41:22.755033+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:41:22.755033+0000","last_undegraded":"2023-10-01T09:41:22.755033+0000","last_fullsized":"2023-10-01T09:41:22.755033+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":213,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.351629+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:41:11.351629+0000","last_peered":"2023-10-01T09:41:11.351629+0000","last_clean":"2023-10-01T09:41:11.351629+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:41:11.351629+0000","last_undegraded":"2023-10-01T09:41:11.351629+0000","last_fullsized":"2023-10-01T09:41:11.351629+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.748681+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:41:22.748681+0000","last_peered":"2023-10-01T09:41:22.748681+0000","last_clean":"2023-10-01T09:41:22.748681+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:41:22.748681+0000","last_undegraded":"2023-10-01T09:41:22.748681+0000","last_fullsized":"2023-10-01T09:41:22.748681+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":29,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.748310+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:41:22.748310+0000","last_peered":"2023-10-01T09:41:22.748310+0000","last_clean":"2023-10-01T09:41:22.748310+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:41:22.748310+0000","last_undegraded":"2023-10-01T09:41:22.748310+0000","last_fullsized":"2023-10-01T09:41:22.748310+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":26,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.746760+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:41:22.746760+0000","last_peered":"2023-10-01T09:41:22.746760+0000","last_clean":"2023-10-01T09:41:22.746760+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:41:22.746760+0000","last_undegraded":"2023-10-01T09:41:22.746760+0000","last_fullsized":"2023-10-01T09:41:22.746760+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":405,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.752106+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:41:22.752106+0000","last_peered":"2023-10-01T09:41:22.752106+0000","last_clean":"2023-10-01T09:41:22.752106+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:41:22.752106+0000","last_undegraded":"2023-10-01T09:41:22.752106+0000","last_fullsized":"2023-10-01T09:41:22.752106+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":286,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.750178+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:41:22.750178+0000","last_peered":"2023-10-01T09:41:22.750178+0000","last_clean":"2023-10-01T09:41:22.750178+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:41:22.750178+0000","last_undegraded":"2023-10-01T09:41:22.750178+0000","last_fullsized":"2023-10-01T09:41:22.750178+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":622,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.748295+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:41:22.748295+0000","last_peered":"2023-10-01T09:41:22.748295+0000","last_clean":"2023-10-01T09:41:22.748295+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:41:22.748295+0000","last_undegraded":"2023-10-01T09:41:22.748295+0000","last_fullsized":"2023-10-01T09:41:22.748295+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":359,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.748975+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:41:22.748975+0000","last_peered":"2023-10-01T09:41:22.748975+0000","last_clean":"2023-10-01T09:41:22.748975+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:41:22.748975+0000","last_undegraded":"2023-10-01T09:41:22.748975+0000","last_fullsized":"2023-10-01T09:41:22.748975+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":32,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.757407+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:41:22.757407+0000","last_peered":"2023-10-01T09:41:22.757407+0000","last_clean":"2023-10-01T09:41:22.757407+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:41:22.757407+0000","last_undegraded":"2023-10-01T09:41:22.757407+0000","last_fullsized":"2023-10-01T09:41:22.757407+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":15,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.351271+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:41:11.351271+0000","last_peered":"2023-10-01T09:41:11.351271+0000","last_clean":"2023-10-01T09:41:11.351271+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:41:11.351271+0000","last_undegraded":"2023-10-01T09:41:11.351271+0000","last_fullsized":"2023-10-01T09:41:11.351271+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":515,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.748882+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:41:22.748882+0000","last_peered":"2023-10-01T09:41:22.748882+0000","last_clean":"2023-10-01T09:41:22.748882+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:41:22.748882+0000","last_undegraded":"2023-10-01T09:41:22.748882+0000","last_fullsized":"2023-10-01T09:41:22.748882+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":525,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.745209+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:41:22.745209+0000","last_peered":"2023-10-01T09:41:22.745209+0000","last_clean":"2023-10-01T09:41:22.745209+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:41:22.745209+0000","last_undegraded":"2023-10-01T09:41:22.745209+0000","last_fullsized":"2023-10-01T09:41:22.745209+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":381,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.744359+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:41:22.744359+0000","last_peered":"2023-10-01T09:41:22.744359+0000","last_clean":"2023-10-01T09:41:22.744359+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:41:22.744359+0000","last_undegraded":"2023-10-01T09:41:22.744359+0000","last_fullsized":"2023-10-01T09:41:22.744359+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":32,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.756808+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:41:22.756808+0000","last_peered":"2023-10-01T09:41:22.756808+0000","last_clean":"2023-10-01T09:41:22.756808+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:41:22.756808+0000","last_undegraded":"2023-10-01T09:41:22.756808+0000","last_fullsized":"2023-10-01T09:41:22.756808+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"501'5006","reported_seq":5040,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-01T09:41:20.023220+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:41:20.023220+0000","last_peered":"2023-10-01T09:41:20.023220+0000","last_clean":"2023-10-01T09:41:20.023220+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:41:20.023220+0000","last_undegraded":"2023-10-01T09:41:20.023220+0000","last_fullsized":"2023-10-01T09:41:20.023220+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5006,"log_dups_size":0,"ondisk_log_size":5006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328073216,"num_objects":5006,"num_object_clones":0,"num_object_copies":15018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5006,"num_write_kb":320384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":27,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.751821+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:41:22.751821+0000","last_peered":"2023-10-01T09:41:22.751821+0000","last_clean":"2023-10-01T09:41:22.751821+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:41:22.751821+0000","last_undegraded":"2023-10-01T09:41:22.751821+0000","last_fullsized":"2023-10-01T09:41:22.751821+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":221,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.350877+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:41:11.350877+0000","last_peered":"2023-10-01T09:41:11.350877+0000","last_clean":"2023-10-01T09:41:11.350877+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:41:11.350877+0000","last_undegraded":"2023-10-01T09:41:11.350877+0000","last_fullsized":"2023-10-01T09:41:11.350877+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"501'5109","reported_seq":5156,"reported_epoch":501,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-01T09:41:21.776239+0000","last_change":"2023-10-01T09:40:56.563622+0000","last_active":"2023-10-01T09:41:21.776239+0000","last_peered":"2023-10-01T09:41:21.776239+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:40:56.557114+0000","last_became_peered":"2023-10-01T09:40:56.557114+0000","last_unstale":"2023-10-01T09:41:21.776239+0000","last_undegraded":"2023-10-01T09:41:21.776239+0000","last_fullsized":"2023-10-01T09:41:21.776239+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5109,"log_dups_size":0,"ondisk_log_size":5109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:34:28.457154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334823424,"num_objects":5109,"num_object_clones":0,"num_object_copies":15327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3698,"num_objects_unfound":0,"num_objects_dirty":5109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5109,"num_write_kb":326976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[0,6,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,6","objects":1411}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":498,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.749615+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:41:22.749615+0000","last_peered":"2023-10-01T09:41:22.749615+0000","last_clean":"2023-10-01T09:41:22.749615+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:41:22.749615+0000","last_undegraded":"2023-10-01T09:41:22.749615+0000","last_fullsized":"2023-10-01T09:41:22.749615+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":558,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.763300+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:41:22.763300+0000","last_peered":"2023-10-01T09:41:22.763300+0000","last_clean":"2023-10-01T09:41:22.763300+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:41:22.763300+0000","last_undegraded":"2023-10-01T09:41:22.763300+0000","last_fullsized":"2023-10-01T09:41:22.763300+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"504'5147","reported_seq":5185,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-01T09:41:24.182673+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:41:24.182673+0000","last_peered":"2023-10-01T09:41:24.182673+0000","last_clean":"2023-10-01T09:41:24.182673+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:41:24.182673+0000","last_undegraded":"2023-10-01T09:41:24.182673+0000","last_fullsized":"2023-10-01T09:41:24.182673+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5147,"log_dups_size":0,"ondisk_log_size":5147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337313792,"num_objects":5147,"num_object_clones":0,"num_object_copies":15441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5147,"num_write_kb":329408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":584,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.750152+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:41:22.750152+0000","last_peered":"2023-10-01T09:41:22.750152+0000","last_clean":"2023-10-01T09:41:22.750152+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:41:22.750152+0000","last_undegraded":"2023-10-01T09:41:22.750152+0000","last_fullsized":"2023-10-01T09:41:22.750152+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":378,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.746940+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:41:22.746940+0000","last_peered":"2023-10-01T09:41:22.746940+0000","last_clean":"2023-10-01T09:41:22.746940+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:41:22.746940+0000","last_undegraded":"2023-10-01T09:41:22.746940+0000","last_fullsized":"2023-10-01T09:41:22.746940+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"504'5147","reported_seq":10170,"reported_epoch":504,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-01T09:41:26.841616+0000","last_change":"2023-10-01T09:40:55.404296+0000","last_active":"2023-10-01T09:41:26.841616+0000","last_peered":"2023-10-01T09:41:26.841616+0000","last_clean":"2023-10-01T09:40:47.170737+0000","last_became_active":"2023-10-01T09:40:48.834770+0000","last_became_peered":"2023-10-01T09:40:48.834770+0000","last_unstale":"2023-10-01T09:41:26.841616+0000","last_undegraded":"2023-10-01T09:41:26.841616+0000","last_fullsized":"2023-10-01T09:40:48.728822+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5147,"log_dups_size":0,"ondisk_log_size":5147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:52:35.816227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337313792,"num_objects":5147,"num_object_clones":0,"num_object_copies":15441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":805,"num_objects_unfound":0,"num_objects_dirty":5147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5147,"num_write_kb":329408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2486,"num_bytes_recovered":162922496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3","objects":4342},{"shards":"0,3,4","objects":805}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":221,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.351011+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:41:11.351011+0000","last_peered":"2023-10-01T09:41:11.351011+0000","last_clean":"2023-10-01T09:41:11.351011+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:41:11.351011+0000","last_undegraded":"2023-10-01T09:41:11.351011+0000","last_fullsized":"2023-10-01T09:41:11.351011+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":481,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.745320+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:41:22.745320+0000","last_peered":"2023-10-01T09:41:22.745320+0000","last_clean":"2023-10-01T09:41:22.745320+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:41:22.745320+0000","last_undegraded":"2023-10-01T09:41:22.745320+0000","last_fullsized":"2023-10-01T09:41:22.745320+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"504'5189","reported_seq":5227,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-01T09:41:24.227036+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:41:24.227036+0000","last_peered":"2023-10-01T09:41:24.227036+0000","last_clean":"2023-10-01T09:41:24.227036+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:41:24.227036+0000","last_undegraded":"2023-10-01T09:41:24.227036+0000","last_fullsized":"2023-10-01T09:41:24.227036+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5189,"log_dups_size":0,"ondisk_log_size":5189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340066304,"num_objects":5189,"num_object_clones":0,"num_object_copies":15567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5189,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":84,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.750580+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:41:22.750580+0000","last_peered":"2023-10-01T09:41:22.750580+0000","last_clean":"2023-10-01T09:41:22.750580+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:41:22.750580+0000","last_undegraded":"2023-10-01T09:41:22.750580+0000","last_fullsized":"2023-10-01T09:41:22.750580+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":158,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.752276+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:41:22.752276+0000","last_peered":"2023-10-01T09:41:22.752276+0000","last_clean":"2023-10-01T09:41:22.752276+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:41:22.752276+0000","last_undegraded":"2023-10-01T09:41:22.752276+0000","last_fullsized":"2023-10-01T09:41:22.752276+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"501'4992","reported_seq":5026,"reported_epoch":501,"state":"active+clean","last_fresh":"2023-10-01T09:41:20.020521+0000","last_change":"2023-10-01T09:40:06.128963+0000","last_active":"2023-10-01T09:41:20.020521+0000","last_peered":"2023-10-01T09:41:20.020521+0000","last_clean":"2023-10-01T09:41:20.020521+0000","last_became_active":"2023-10-01T09:40:06.128645+0000","last_became_peered":"2023-10-01T09:40:06.128645+0000","last_unstale":"2023-10-01T09:41:20.020521+0000","last_undegraded":"2023-10-01T09:41:20.020521+0000","last_fullsized":"2023-10-01T09:41:20.020521+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":4992,"log_dups_size":0,"ondisk_log_size":4992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327155712,"num_objects":4992,"num_object_clones":0,"num_object_copies":14976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4992,"num_write_kb":319488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":402,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.751251+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:41:22.751251+0000","last_peered":"2023-10-01T09:41:22.751251+0000","last_clean":"2023-10-01T09:41:22.751251+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:41:22.751251+0000","last_undegraded":"2023-10-01T09:41:22.751251+0000","last_fullsized":"2023-10-01T09:41:22.751251+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":561,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.746973+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:41:22.746973+0000","last_peered":"2023-10-01T09:41:22.746973+0000","last_clean":"2023-10-01T09:41:22.746973+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:41:22.746973+0000","last_undegraded":"2023-10-01T09:41:22.746973+0000","last_fullsized":"2023-10-01T09:41:22.746973+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"504'5137","reported_seq":5183,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-01T09:41:24.246042+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:41:24.246042+0000","last_peered":"2023-10-01T09:41:24.246042+0000","last_clean":"2023-10-01T09:41:24.246042+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:41:24.246042+0000","last_undegraded":"2023-10-01T09:41:24.246042+0000","last_fullsized":"2023-10-01T09:41:24.246042+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5137,"log_dups_size":0,"ondisk_log_size":5137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336658432,"num_objects":5137,"num_object_clones":0,"num_object_copies":15411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5137,"num_write_kb":328768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":398,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.750164+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:41:22.750164+0000","last_peered":"2023-10-01T09:41:22.750164+0000","last_clean":"2023-10-01T09:41:22.750164+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:41:22.750164+0000","last_undegraded":"2023-10-01T09:41:22.750164+0000","last_fullsized":"2023-10-01T09:41:22.750164+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":239,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.752725+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:41:22.752725+0000","last_peered":"2023-10-01T09:41:22.752725+0000","last_clean":"2023-10-01T09:41:22.752725+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:41:22.752725+0000","last_undegraded":"2023-10-01T09:41:22.752725+0000","last_fullsized":"2023-10-01T09:41:22.752725+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"504'5196","reported_seq":5233,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-01T09:41:24.190289+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:41:24.190289+0000","last_peered":"2023-10-01T09:41:24.190289+0000","last_clean":"2023-10-01T09:41:24.190289+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:41:24.190289+0000","last_undegraded":"2023-10-01T09:41:24.190289+0000","last_fullsized":"2023-10-01T09:41:24.190289+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5196,"log_dups_size":0,"ondisk_log_size":5196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340525056,"num_objects":5196,"num_object_clones":0,"num_object_copies":15588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5196,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":365,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.760771+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:41:22.760771+0000","last_peered":"2023-10-01T09:41:22.760771+0000","last_clean":"2023-10-01T09:41:22.760771+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:41:22.760771+0000","last_undegraded":"2023-10-01T09:41:22.760771+0000","last_fullsized":"2023-10-01T09:41:22.760771+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":413,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.747658+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:41:22.747658+0000","last_peered":"2023-10-01T09:41:22.747658+0000","last_clean":"2023-10-01T09:41:22.747658+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:41:22.747658+0000","last_undegraded":"2023-10-01T09:41:22.747658+0000","last_fullsized":"2023-10-01T09:41:22.747658+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"504'5044","reported_seq":5081,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-01T09:41:24.200764+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:41:24.200764+0000","last_peered":"2023-10-01T09:41:24.200764+0000","last_clean":"2023-10-01T09:41:24.200764+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:41:24.200764+0000","last_undegraded":"2023-10-01T09:41:24.200764+0000","last_fullsized":"2023-10-01T09:41:24.200764+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5044,"log_dups_size":0,"ondisk_log_size":5044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330563584,"num_objects":5044,"num_object_clones":0,"num_object_copies":15132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5044,"num_write_kb":322816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":897,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.748819+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:41:22.748819+0000","last_peered":"2023-10-01T09:41:22.748819+0000","last_clean":"2023-10-01T09:41:22.748819+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:41:22.748819+0000","last_undegraded":"2023-10-01T09:41:22.748819+0000","last_fullsized":"2023-10-01T09:41:22.748819+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":231,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.351428+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:41:11.351428+0000","last_peered":"2023-10-01T09:41:11.351428+0000","last_clean":"2023-10-01T09:41:11.351428+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:41:11.351428+0000","last_undegraded":"2023-10-01T09:41:11.351428+0000","last_fullsized":"2023-10-01T09:41:11.351428+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":26,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.746770+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:41:22.746770+0000","last_peered":"2023-10-01T09:41:22.746770+0000","last_clean":"2023-10-01T09:41:22.746770+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:41:22.746770+0000","last_undegraded":"2023-10-01T09:41:22.746770+0000","last_fullsized":"2023-10-01T09:41:22.746770+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"504'5122","reported_seq":5160,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-01T09:41:24.233063+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:41:24.233063+0000","last_peered":"2023-10-01T09:41:24.233063+0000","last_clean":"2023-10-01T09:41:24.233063+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:41:24.233063+0000","last_undegraded":"2023-10-01T09:41:24.233063+0000","last_fullsized":"2023-10-01T09:41:24.233063+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5122,"log_dups_size":0,"ondisk_log_size":5122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335675392,"num_objects":5122,"num_object_clones":0,"num_object_copies":15366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5122,"num_write_kb":327808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1166,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.351474+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:41:11.351474+0000","last_peered":"2023-10-01T09:41:11.351474+0000","last_clean":"2023-10-01T09:41:11.351474+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:41:11.351474+0000","last_undegraded":"2023-10-01T09:41:11.351474+0000","last_fullsized":"2023-10-01T09:41:11.351474+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":379,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.750396+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:41:22.750396+0000","last_peered":"2023-10-01T09:41:22.750396+0000","last_clean":"2023-10-01T09:41:22.750396+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:41:22.750396+0000","last_undegraded":"2023-10-01T09:41:22.750396+0000","last_fullsized":"2023-10-01T09:41:22.750396+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":195,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.351179+0000","last_change":"2023-10-01T09:40:47.743038+0000","last_active":"2023-10-01T09:41:11.351179+0000","last_peered":"2023-10-01T09:41:11.351179+0000","last_clean":"2023-10-01T09:41:11.351179+0000","last_became_active":"2023-10-01T09:40:47.742668+0000","last_became_peered":"2023-10-01T09:40:47.742668+0000","last_unstale":"2023-10-01T09:41:11.351179+0000","last_undegraded":"2023-10-01T09:41:11.351179+0000","last_fullsized":"2023-10-01T09:41:11.351179+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"504'5124","reported_seq":5166,"reported_epoch":504,"state":"active+recovery_wait+remapped","last_fresh":"2023-10-01T09:41:24.223567+0000","last_change":"2023-10-01T09:40:56.579219+0000","last_active":"2023-10-01T09:41:24.223567+0000","last_peered":"2023-10-01T09:41:24.223567+0000","last_clean":"2023-10-01T09:40:54.363445+0000","last_became_active":"2023-10-01T09:40:56.569721+0000","last_became_peered":"2023-10-01T09:40:56.569721+0000","last_unstale":"2023-10-01T09:41:24.223567+0000","last_undegraded":"2023-10-01T09:41:24.223567+0000","last_fullsized":"2023-10-01T09:41:24.223567+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5124,"log_dups_size":0,"ondisk_log_size":5124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:17:29.742329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335806464,"num_objects":5124,"num_object_clones":0,"num_object_copies":15372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3602,"num_objects_unfound":0,"num_objects_dirty":5124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5124,"num_write_kb":327936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":1522}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":848,"reported_epoch":503,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:41:22.763216+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:41:22.763216+0000","last_peered":"2023-10-01T09:41:22.763216+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:41:22.763216+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:41:22.763216+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":565,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.747857+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:41:22.747857+0000","last_peered":"2023-10-01T09:41:22.747857+0000","last_clean":"2023-10-01T09:41:22.747857+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:41:22.747857+0000","last_undegraded":"2023-10-01T09:41:22.747857+0000","last_fullsized":"2023-10-01T09:41:22.747857+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":26,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.747556+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:41:22.747556+0000","last_peered":"2023-10-01T09:41:22.747556+0000","last_clean":"2023-10-01T09:41:22.747556+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:41:22.747556+0000","last_undegraded":"2023-10-01T09:41:22.747556+0000","last_fullsized":"2023-10-01T09:41:22.747556+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"504'5090","reported_seq":5128,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-01T09:41:24.217402+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:41:24.217402+0000","last_peered":"2023-10-01T09:41:24.217402+0000","last_clean":"2023-10-01T09:41:24.217402+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:41:24.217402+0000","last_undegraded":"2023-10-01T09:41:24.217402+0000","last_fullsized":"2023-10-01T09:41:24.217402+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5090,"log_dups_size":0,"ondisk_log_size":5090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333578240,"num_objects":5090,"num_object_clones":0,"num_object_copies":15270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5090,"num_write_kb":325760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1617,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.351073+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:41:11.351073+0000","last_peered":"2023-10-01T09:41:11.351073+0000","last_clean":"2023-10-01T09:41:11.351073+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:41:11.351073+0000","last_undegraded":"2023-10-01T09:41:11.351073+0000","last_fullsized":"2023-10-01T09:41:11.351073+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":534,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.748928+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:41:22.748928+0000","last_peered":"2023-10-01T09:41:22.748928+0000","last_clean":"2023-10-01T09:41:22.748928+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:41:22.748928+0000","last_undegraded":"2023-10-01T09:41:22.748928+0000","last_fullsized":"2023-10-01T09:41:22.748928+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":300,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.351562+0000","last_change":"2023-10-01T09:38:07.494808+0000","last_active":"2023-10-01T09:41:11.351562+0000","last_peered":"2023-10-01T09:41:11.351562+0000","last_clean":"2023-10-01T09:41:11.351562+0000","last_became_active":"2023-10-01T09:38:07.494419+0000","last_became_peered":"2023-10-01T09:38:07.494419+0000","last_unstale":"2023-10-01T09:41:11.351562+0000","last_undegraded":"2023-10-01T09:41:11.351562+0000","last_fullsized":"2023-10-01T09:41:11.351562+0000","mapping_epoch":452,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":453,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":302,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.751851+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:41:22.751851+0000","last_peered":"2023-10-01T09:41:22.751851+0000","last_clean":"2023-10-01T09:41:22.751851+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:41:22.751851+0000","last_undegraded":"2023-10-01T09:41:22.751851+0000","last_fullsized":"2023-10-01T09:41:22.751851+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"504'5217","reported_seq":5254,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-01T09:41:24.221610+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:41:24.221610+0000","last_peered":"2023-10-01T09:41:24.221610+0000","last_clean":"2023-10-01T09:41:24.221610+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:41:24.221610+0000","last_undegraded":"2023-10-01T09:41:24.221610+0000","last_fullsized":"2023-10-01T09:41:24.221610+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5217,"log_dups_size":0,"ondisk_log_size":5217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341901312,"num_objects":5217,"num_object_clones":0,"num_object_copies":15651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5217,"num_write_kb":333888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":711,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.749095+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:41:22.749095+0000","last_peered":"2023-10-01T09:41:22.749095+0000","last_clean":"2023-10-01T09:41:22.749095+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:41:22.749095+0000","last_undegraded":"2023-10-01T09:41:22.749095+0000","last_fullsized":"2023-10-01T09:41:22.749095+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":542,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.764382+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:41:22.764382+0000","last_peered":"2023-10-01T09:41:22.764382+0000","last_clean":"2023-10-01T09:41:22.764382+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:41:22.764382+0000","last_undegraded":"2023-10-01T09:41:22.764382+0000","last_fullsized":"2023-10-01T09:41:22.764382+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":542,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.744683+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:41:22.744683+0000","last_peered":"2023-10-01T09:41:22.744683+0000","last_clean":"2023-10-01T09:41:22.744683+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:41:22.744683+0000","last_undegraded":"2023-10-01T09:41:22.744683+0000","last_fullsized":"2023-10-01T09:41:22.744683+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"504'5172","reported_seq":5210,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-01T09:41:24.178636+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:41:24.178636+0000","last_peered":"2023-10-01T09:41:24.178636+0000","last_clean":"2023-10-01T09:41:24.178636+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:41:24.178636+0000","last_undegraded":"2023-10-01T09:41:24.178636+0000","last_fullsized":"2023-10-01T09:41:24.178636+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5172,"log_dups_size":0,"ondisk_log_size":5172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338952192,"num_objects":5172,"num_object_clones":0,"num_object_copies":15516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5172,"num_write_kb":331008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":942,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.750291+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:41:22.750291+0000","last_peered":"2023-10-01T09:41:22.750291+0000","last_clean":"2023-10-01T09:41:22.750291+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:41:22.750291+0000","last_undegraded":"2023-10-01T09:41:22.750291+0000","last_fullsized":"2023-10-01T09:41:22.750291+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":558,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.351098+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:41:11.351098+0000","last_peered":"2023-10-01T09:41:11.351098+0000","last_clean":"2023-10-01T09:41:11.351098+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:41:11.351098+0000","last_undegraded":"2023-10-01T09:41:11.351098+0000","last_fullsized":"2023-10-01T09:41:11.351098+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":411,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.351152+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:41:11.351152+0000","last_peered":"2023-10-01T09:41:11.351152+0000","last_clean":"2023-10-01T09:41:11.351152+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:41:11.351152+0000","last_undegraded":"2023-10-01T09:41:11.351152+0000","last_fullsized":"2023-10-01T09:41:11.351152+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"504'5137","reported_seq":5182,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-01T09:41:24.193948+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:41:24.193948+0000","last_peered":"2023-10-01T09:41:24.193948+0000","last_clean":"2023-10-01T09:41:24.193948+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:41:24.193948+0000","last_undegraded":"2023-10-01T09:41:24.193948+0000","last_fullsized":"2023-10-01T09:41:24.193948+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5137,"log_dups_size":0,"ondisk_log_size":5137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336658432,"num_objects":5137,"num_object_clones":0,"num_object_copies":15411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5137,"num_write_kb":328768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":443,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.752042+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:41:22.752042+0000","last_peered":"2023-10-01T09:41:22.752042+0000","last_clean":"2023-10-01T09:41:22.752042+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:41:22.752042+0000","last_undegraded":"2023-10-01T09:41:22.752042+0000","last_fullsized":"2023-10-01T09:41:22.752042+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1135,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.747415+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:41:22.747415+0000","last_peered":"2023-10-01T09:41:22.747415+0000","last_clean":"2023-10-01T09:41:22.747415+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:41:22.747415+0000","last_undegraded":"2023-10-01T09:41:22.747415+0000","last_fullsized":"2023-10-01T09:41:22.747415+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":32,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.756458+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:41:22.756458+0000","last_peered":"2023-10-01T09:41:22.756458+0000","last_clean":"2023-10-01T09:41:22.756458+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:41:22.756458+0000","last_undegraded":"2023-10-01T09:41:22.756458+0000","last_fullsized":"2023-10-01T09:41:22.756458+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"504'5199","reported_seq":5244,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-01T09:41:24.217045+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:41:24.217045+0000","last_peered":"2023-10-01T09:41:24.217045+0000","last_clean":"2023-10-01T09:41:24.217045+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:41:24.217045+0000","last_undegraded":"2023-10-01T09:41:24.217045+0000","last_fullsized":"2023-10-01T09:41:24.217045+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5199,"log_dups_size":0,"ondisk_log_size":5199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340721664,"num_objects":5199,"num_object_clones":0,"num_object_copies":15597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5199,"num_write_kb":332736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":36,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.751900+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:41:22.751900+0000","last_peered":"2023-10-01T09:41:22.751900+0000","last_clean":"2023-10-01T09:41:22.751900+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:41:22.751900+0000","last_undegraded":"2023-10-01T09:41:22.751900+0000","last_fullsized":"2023-10-01T09:41:22.751900+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":520,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.744777+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:41:22.744777+0000","last_peered":"2023-10-01T09:41:22.744777+0000","last_clean":"2023-10-01T09:41:22.744777+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:41:22.744777+0000","last_undegraded":"2023-10-01T09:41:22.744777+0000","last_fullsized":"2023-10-01T09:41:22.744777+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":224,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.350850+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:41:11.350850+0000","last_peered":"2023-10-01T09:41:11.350850+0000","last_clean":"2023-10-01T09:41:11.350850+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:41:11.350850+0000","last_undegraded":"2023-10-01T09:41:11.350850+0000","last_fullsized":"2023-10-01T09:41:11.350850+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":469,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.745475+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:41:22.745475+0000","last_peered":"2023-10-01T09:41:22.745475+0000","last_clean":"2023-10-01T09:41:22.745475+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:41:22.745475+0000","last_undegraded":"2023-10-01T09:41:22.745475+0000","last_fullsized":"2023-10-01T09:41:22.745475+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":583,"reported_epoch":504,"state":"active+clean","last_fresh":"2023-10-01T09:41:23.934022+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:41:23.934022+0000","last_peered":"2023-10-01T09:41:23.934022+0000","last_clean":"2023-10-01T09:41:23.934022+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:41:23.934022+0000","last_undegraded":"2023-10-01T09:41:23.934022+0000","last_fullsized":"2023-10-01T09:41:23.934022+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":236,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.350918+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:41:11.350918+0000","last_peered":"2023-10-01T09:41:11.350918+0000","last_clean":"2023-10-01T09:41:11.350918+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:41:11.350918+0000","last_undegraded":"2023-10-01T09:41:11.350918+0000","last_fullsized":"2023-10-01T09:41:11.350918+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":26,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.747092+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:41:22.747092+0000","last_peered":"2023-10-01T09:41:22.747092+0000","last_clean":"2023-10-01T09:41:22.747092+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:41:22.747092+0000","last_undegraded":"2023-10-01T09:41:22.747092+0000","last_fullsized":"2023-10-01T09:41:22.747092+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":236,"reported_epoch":500,"state":"active+clean","last_fresh":"2023-10-01T09:41:11.351513+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:41:11.351513+0000","last_peered":"2023-10-01T09:41:11.351513+0000","last_clean":"2023-10-01T09:41:11.351513+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:41:11.351513+0000","last_undegraded":"2023-10-01T09:41:11.351513+0000","last_fullsized":"2023-10-01T09:41:11.351513+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":415,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.746032+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:41:22.746032+0000","last_peered":"2023-10-01T09:41:22.746032+0000","last_clean":"2023-10-01T09:41:22.746032+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:41:22.746032+0000","last_undegraded":"2023-10-01T09:41:22.746032+0000","last_fullsized":"2023-10-01T09:41:22.746032+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":43,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.747128+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:41:22.747128+0000","last_peered":"2023-10-01T09:41:22.747128+0000","last_clean":"2023-10-01T09:41:22.747128+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:41:22.747128+0000","last_undegraded":"2023-10-01T09:41:22.747128+0000","last_fullsized":"2023-10-01T09:41:22.747128+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":460,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.744283+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:41:22.744283+0000","last_peered":"2023-10-01T09:41:22.744283+0000","last_clean":"2023-10-01T09:41:22.744283+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:41:22.744283+0000","last_undegraded":"2023-10-01T09:41:22.744283+0000","last_fullsized":"2023-10-01T09:41:22.744283+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":659,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.745736+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:41:22.745736+0000","last_peered":"2023-10-01T09:41:22.745736+0000","last_clean":"2023-10-01T09:41:22.745736+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:41:22.745736+0000","last_undegraded":"2023-10-01T09:41:22.745736+0000","last_fullsized":"2023-10-01T09:41:22.745736+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":355,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.750829+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:41:22.750829+0000","last_peered":"2023-10-01T09:41:22.750829+0000","last_clean":"2023-10-01T09:41:22.750829+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:41:22.750829+0000","last_undegraded":"2023-10-01T09:41:22.750829+0000","last_fullsized":"2023-10-01T09:41:22.750829+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5375787008,"num_objects":82028,"num_object_clones":0,"num_object_copies":246084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8105,"num_objects_unfound":0,"num_objects_dirty":82028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":82028,"num_write_kb":5249792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2486,"num_bytes_recovered":162922496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16472145920,"data_stored":16472145920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82028,"ondisk_log_size":82028,"up":48,"acting":47,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430690,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2766864,"kb_used_data":1867848,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":90978288,"statfs":{"total":95995035648,"available":93161766912,"internally_reserved":0,"allocated":1912676352,"data_stored":1911745644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":2.6640000000000001,"15min":3.4079999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":2.718,"5min":81.162999999999997,"15min":108.90000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.95999999999999996,"5min":3.1440000000000001,"15min":3.8069999999999999},"min":{"1min":0.27900000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.3659999999999997,"5min":81.323999999999998,"15min":109.187},"last":0.39100000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:41:05 2023","interfaces":[{"interface":"back","average":{"1min":11.349,"5min":6.492,"15min":5.5579999999999998},"min":{"1min":0.248,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":229.161,"5min":229.161,"15min":229.161},"last":0.433},{"interface":"front","average":{"1min":11.241,"5min":6.6150000000000002,"15min":5.6059999999999999},"min":{"1min":0.309,"5min":0.27500000000000002,"15min":0.24399999999999999},"max":{"1min":230.126,"5min":230.126,"15min":230.126},"last":0.32400000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":2.367,"15min":2.081},"min":{"1min":0.30299999999999999,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":2.774,"5min":14.901999999999999,"15min":14.901999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.93700000000000006,"5min":2.3050000000000002,"15min":1.996},"min":{"1min":0.375,"5min":0.29699999999999999,"15min":0.247},"max":{"1min":2.907,"5min":14.682,"15min":14.682},"last":0.28699999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:41:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":2.6699999999999999,"15min":3.528},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28199999999999997},"max":{"1min":4.3120000000000003,"5min":35.104999999999997,"15min":164.595},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.3240000000000001,"5min":2.738,"15min":3.5},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.1440000000000001,"5min":35.118000000000002,"15min":164.60499999999999},"last":0.41599999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.1970000000000001,"15min":0.98399999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.23499999999999999},"max":{"1min":4.21,"5min":9.6020000000000003,"15min":9.6020000000000003},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.234,"5min":1.0960000000000001,"15min":0.96599999999999997},"min":{"1min":0.34699999999999998,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":8.4949999999999992,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.42299999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":8.516,"5min":2.984,"15min":1.679},"min":{"1min":0.311,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":135.40000000000001,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":8.1989999999999998,"5min":3.1360000000000001,"15min":1.71},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":135.339,"5min":135.339,"15min":135.339},"last":0.312}]},{"osd":6,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0699999999999998,"5min":1.498,"15min":1.1819999999999999},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":11.58,"5min":11.58,"15min":11.853999999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.988,"5min":1.3939999999999999,"15min":1.147},"min":{"1min":0.373,"5min":0.373,"15min":0.32300000000000001},"max":{"1min":13.021000000000001,"5min":14.584,"15min":14.584},"last":0.373}]}]},{"osd":6,"up_from":294,"seq":1262720385201,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2646280,"kb_used_data":2308672,"kb_used_omap":6,"kb_used_meta":337593,"kb_avail":91098872,"statfs":{"total":95995035648,"available":93285244928,"internally_reserved":0,"allocated":2364080128,"data_stored":2363157443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":345695277},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":7.4020000000000001,"5min":3.9239999999999999,"15min":4.2949999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":158.48599999999999,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":8.1300000000000008,"5min":4.5199999999999996,"15min":4.335},"min":{"1min":0.34399999999999997,"5min":0.25700000000000001,"15min":0.221},"max":{"1min":158.37700000000001,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.373}]},{"osd":1,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":12.536,"5min":7.2130000000000001,"15min":5.1319999999999997},"min":{"1min":0.38200000000000001,"5min":0.25800000000000001,"15min":0.23899999999999999},"max":{"1min":168.947,"5min":168.947,"15min":168.947},"last":0.32700000000000001},{"interface":"front","average":{"1min":12.667999999999999,"5min":7.1269999999999998,"15min":5.0389999999999997},"min":{"1min":0.32400000000000001,"5min":0.30599999999999999,"15min":0.24399999999999999},"max":{"1min":169.07900000000001,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.35599999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.919,"15min":1.9850000000000001},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":8.657,"5min":9.657,"15min":26.289999999999999},"last":0.436},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.944,"15min":2.0179999999999998},"min":{"1min":0.29399999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.247999999999999,"5min":21.704999999999998,"15min":26.388999999999999},"last":0.40500000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.903,"15min":2.5209999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":4.149,"5min":9.0180000000000007,"15min":89.753},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.7110000000000001,"5min":1.9870000000000001,"15min":2.2429999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":8.5470000000000006,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.48799999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":4.9210000000000003,"5min":1.5780000000000001,"15min":1.038},"min":{"1min":0.26300000000000001,"5min":0.246,"15min":0.246},"max":{"1min":63.134,"5min":63.134,"15min":63.134},"last":0.44900000000000001},{"interface":"front","average":{"1min":4.3339999999999996,"5min":1.3959999999999999,"15min":1.056},"min":{"1min":0.35299999999999998,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":62.973999999999997,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.47199999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":1.0089999999999999,"15min":1.0529999999999999},"min":{"1min":0.312,"5min":0.251,"15min":0.251},"max":{"1min":9.9939999999999998,"5min":9.9939999999999998,"15min":12.023999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.1880000000000002,"5min":1.234,"15min":1.1739999999999999},"min":{"1min":0.314,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":15.457000000000001,"5min":15.457000000000001,"15min":15.457000000000001},"last":0.499}]},{"osd":7,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":0.86099999999999999,"15min":2.1339999999999999},"min":{"1min":0.34100000000000003,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":3.492,"5min":6.4119999999999999,"15min":115.11},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.2310000000000001,"15min":2.2040000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28799999999999998},"max":{"1min":8.6899999999999995,"5min":8.6899999999999995,"15min":114.666},"last":0.54600000000000004}]}]},{"osd":1,"up_from":94,"seq":403726926177,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1802376,"kb_used_data":996480,"kb_used_omap":7,"kb_used_meta":805880,"kb_avail":91942776,"statfs":{"total":95995035648,"available":94149402624,"internally_reserved":0,"allocated":1020395520,"data_stored":1018420401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":825221543},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:40:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":6.8710000000000004,"15min":4.7690000000000001},"min":{"1min":0.20699999999999999,"5min":0.182,"15min":0.182},"max":{"1min":1.772,"5min":154.21299999999999,"15min":154.21299999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.65800000000000003,"5min":5.3220000000000001,"15min":4.1139999999999999},"min":{"1min":0.24399999999999999,"5min":0.19500000000000001,"15min":0.188},"max":{"1min":2,"5min":95.236999999999995,"15min":95.236999999999995},"last":0.29699999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:40:36 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":2.891,"15min":3.7029999999999998},"min":{"1min":0.23000000000000001,"5min":0.22900000000000001,"15min":0.20000000000000001},"max":{"1min":1.98,"5min":22.891999999999999,"15min":56.573},"last":0.318},{"interface":"front","average":{"1min":0.52500000000000002,"5min":4.6989999999999998,"15min":4.4800000000000004},"min":{"1min":0.28399999999999997,"5min":0.254,"15min":0.20499999999999999},"max":{"1min":1.851,"5min":154.197,"15min":154.197},"last":0.32800000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:41:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":4.9089999999999998,"15min":4.8239999999999998},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":18.503,"5min":95.305000000000007,"15min":200.56800000000001},"last":0.38},{"interface":"front","average":{"1min":1.4119999999999999,"5min":6.7089999999999996,"15min":5.54},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":16.515000000000001,"5min":154.483,"15min":200.63800000000001},"last":0.34699999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:40:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":4.4109999999999996,"15min":2.5230000000000001},"min":{"1min":0.314,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.5789999999999997,"5min":154.101,"15min":154.101},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.92800000000000005,"5min":4.3239999999999998,"15min":2.8679999999999999},"min":{"1min":0.40799999999999997,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":2.7290000000000001,"5min":154.56999999999999,"15min":154.56999999999999},"last":0.42699999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:40:31 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":4.343,"15min":2.1400000000000001},"min":{"1min":0.38300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.6799999999999999,"5min":154.22999999999999,"15min":154.22999999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.69499999999999995,"5min":4.1799999999999997,"15min":2.7400000000000002},"min":{"1min":0.309,"5min":0.307,"15min":0.307},"max":{"1min":2.9089999999999998,"5min":154.54400000000001,"15min":154.54400000000001},"last":0.39400000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:41:08 2023","interfaces":[{"interface":"back","average":{"1min":2.363,"5min":4.1890000000000001,"15min":2.9950000000000001},"min":{"1min":0.34300000000000003,"5min":0.32900000000000001,"15min":0.29899999999999999},"max":{"1min":23.672999999999998,"5min":154.12200000000001,"15min":154.12200000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.27,"5min":4.4729999999999999,"15min":2.6829999999999998},"min":{"1min":0.34599999999999997,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":23.382000000000001,"5min":154.16999999999999,"15min":154.16999999999999},"last":0.435}]},{"osd":7,"last update":"Sun Oct 1 09:40:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":4.5149999999999997,"15min":2.8319999999999999},"min":{"1min":0.45500000000000002,"5min":0.33900000000000002,"15min":0.316},"max":{"1min":9.2460000000000004,"5min":154.20699999999999,"15min":154.20699999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.3149999999999999,"5min":4.5449999999999999,"15min":3.0430000000000001},"min":{"1min":0.44,"5min":0.43099999999999999,"15min":0.30199999999999999},"max":{"1min":5.0110000000000001,"5min":154.51599999999999,"15min":154.51599999999999},"last":0.46700000000000003}]}]},{"osd":0,"up_from":89,"seq":382252089712,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4851412,"kb_used_data":3960268,"kb_used_omap":8,"kb_used_meta":891127,"kb_avail":88893740,"statfs":{"total":95995035648,"available":91027189760,"internally_reserved":0,"allocated":4055314432,"data_stored":4053333738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":912515010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:41:06 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":7.5830000000000002,"15min":4.54},"min":{"1min":0.187,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":8.4939999999999998,"5min":216.821,"15min":216.821},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.095,"5min":9.0879999999999992,"15min":5.7130000000000001},"min":{"1min":0.22,"5min":0.215,"15min":0.21199999999999999},"max":{"1min":8.6229999999999993,"5min":217.02699999999999,"15min":217.02699999999999},"last":0.438}]},{"osd":2,"last update":"Sun Oct 1 09:40:33 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":2.8980000000000001,"15min":2.4340000000000002},"min":{"1min":0.27100000000000002,"5min":0.253,"15min":0.20100000000000001},"max":{"1min":3.9700000000000002,"5min":19.788,"15min":19.788},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.83299999999999996,"5min":2.915,"15min":3.0979999999999999},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.189},"max":{"1min":4.5359999999999996,"5min":25.844999999999999,"15min":25.844999999999999},"last":0.46800000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:41:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":5.4249999999999998,"15min":4.7450000000000001},"min":{"1min":0.24199999999999999,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":5.0869999999999997,"5min":222.16499999999999,"15min":222.16499999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.80200000000000005,"5min":5.2400000000000002,"15min":4.9770000000000003},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.219},"max":{"1min":5.8419999999999996,"5min":222.227,"15min":222.227},"last":0.42799999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:40:52 2023","interfaces":[{"interface":"back","average":{"1min":4.2089999999999996,"5min":2.7109999999999999,"15min":2.5169999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":48.116999999999997,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.61599999999999999},{"interface":"front","average":{"1min":3.8650000000000002,"5min":3.0670000000000002,"15min":2.609},"min":{"1min":0.36299999999999999,"5min":0.33700000000000002,"15min":0.28299999999999997},"max":{"1min":47.854999999999997,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.47699999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.494,"5min":2.371,"15min":2.0569999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":5.8769999999999998,"5min":14.515000000000001,"15min":16.808},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.2879999999999998,"15min":1.9259999999999999},"min":{"1min":0.49099999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.8689999999999998,"5min":17.507000000000001,"15min":17.507000000000001},"last":0.52800000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:41:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":2.2509999999999999,"15min":1.897},"min":{"1min":0.38600000000000001,"5min":0.35699999999999998,"15min":0.35499999999999998},"max":{"1min":5.2110000000000003,"5min":17.535,"15min":17.535},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.649,"5min":2.1739999999999999,"15min":2.2679999999999998},"min":{"1min":0.44800000000000001,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":4.2640000000000002,"5min":10.446,"15min":18.521000000000001},"last":0.64800000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":2.7149999999999999,"15min":2.1619999999999999},"min":{"1min":0.45100000000000001,"5min":0.377,"15min":0.34499999999999997},"max":{"1min":3.097,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.88600000000000001,"5min":2.742,"15min":2.6269999999999998},"min":{"1min":0.371,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":2.4199999999999999,"5min":48.274000000000001,"15min":48.274000000000001},"last":0.66900000000000004}]}]},{"osd":2,"up_from":0,"seq":425201762651,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599125,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2516424,"kb_used_data":1715008,"kb_used_omap":3,"kb_used_meta":801404,"kb_avail":91228728,"statfs":{"total":95995035648,"available":93418217472,"internally_reserved":0,"allocated":1756168192,"data_stored":1754134948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":820638439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":5.0030000000000001,"15min":5.9409999999999998},"min":{"1min":0.183,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":12.999000000000001,"5min":118.748,"15min":137.34399999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.97699999999999998,"5min":5.3849999999999998,"15min":5.8490000000000002},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.184},"max":{"1min":10.377000000000001,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.252}]},{"osd":1,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":8.3849999999999998,"15min":5.7290000000000001},"min":{"1min":0.252,"5min":0.216,"15min":0.20399999999999999},"max":{"1min":14.917999999999999,"5min":153.77000000000001,"15min":162.22999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.359,"5min":8.468,"15min":6.2320000000000002},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":15.074,"5min":151.02199999999999,"15min":162.70699999999999},"last":0.48899999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":3.0110000000000001,"15min":2.8839999999999999},"min":{"1min":0.26000000000000001,"5min":0.22800000000000001,"15min":0.20799999999999999},"max":{"1min":2.7970000000000002,"5min":14.254,"15min":43.222999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.7070000000000001,"15min":3.206},"min":{"1min":0.247,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":1.4239999999999999,"5min":15.523,"15min":78.594999999999999},"last":0.315}]},{"osd":4,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":4.617,"5min":2.5249999999999999,"15min":2.1760000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":77.793000000000006,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.442},{"interface":"front","average":{"1min":4.8399999999999999,"5min":2.9620000000000002,"15min":2.306},"min":{"1min":0.378,"5min":0.27900000000000003,"15min":0.23899999999999999},"max":{"1min":76.646000000000001,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.45000000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":2.1869999999999998,"15min":3.012},"min":{"1min":0.29399999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.0659999999999998,"5min":10.919,"15min":28.117999999999999},"last":0.5},{"interface":"front","average":{"1min":1.768,"5min":2.6379999999999999,"15min":3.1000000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.1559999999999997,"5min":14.537000000000001,"15min":28.34},"last":0.47699999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:41:19 2023","interfaces":[{"interface":"back","average":{"1min":5.2430000000000003,"5min":3.4009999999999998,"15min":2.4870000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":73.887,"5min":73.887,"15min":73.887},"last":0.51400000000000001},{"interface":"front","average":{"1min":5.8470000000000004,"5min":3.5840000000000001,"15min":2.4780000000000002},"min":{"1min":0.44700000000000001,"5min":0.38600000000000001,"15min":0.32800000000000001},"max":{"1min":74.016999999999996,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.53300000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":2.8690000000000002,"15min":2.6429999999999998},"min":{"1min":0.46400000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.5570000000000004,"5min":13.085000000000001,"15min":52.006},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.228,"5min":2.0640000000000001,"15min":2.3359999999999999},"min":{"1min":0.371,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":4.5810000000000004,"5min":11.109999999999999,"15min":51.890999999999998},"last":0.52000000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019375,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5065736,"kb_used_data":4167360,"kb_used_omap":0,"kb_used_meta":898368,"kb_avail":88679416,"statfs":{"total":95995035648,"available":90807721984,"internally_reserved":0,"allocated":4267376640,"data_stored":4266453510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919928832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:40:50 2023","interfaces":[{"interface":"back","average":{"1min":4.6970000000000001,"5min":3.8300000000000001,"15min":5.2060000000000004},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":79.837000000000003,"5min":79.837000000000003,"15min":155.315},"last":0.39800000000000002},{"interface":"front","average":{"1min":5.0030000000000001,"5min":4.2080000000000002,"15min":5.3689999999999998},"min":{"1min":0.28399999999999997,"5min":0.251,"15min":0.23899999999999999},"max":{"1min":78.983000000000004,"5min":78.983000000000004,"15min":155.334},"last":0.28100000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:40:54 2023","interfaces":[{"interface":"back","average":{"1min":4.5149999999999997,"5min":10.486000000000001,"15min":7.7160000000000002},"min":{"1min":0.32200000000000001,"5min":0.224,"15min":0.224},"max":{"1min":77.953000000000003,"5min":165.666,"15min":190.61199999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":4.6669999999999998,"5min":10.384,"15min":8.5039999999999996},"min":{"1min":0.26700000000000002,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":77.962999999999994,"5min":165.70500000000001,"15min":253.273},"last":0.48399999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.9710000000000001,"15min":2.8220000000000001},"min":{"1min":0.31,"5min":0.29099999999999998,"15min":0.26700000000000002},"max":{"1min":1.8029999999999999,"5min":10.161,"15min":170.45400000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.63100000000000001,"5min":2.323,"15min":2.4950000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":1.8680000000000001,"5min":10.625,"15min":58.771000000000001},"last":0.52900000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:40:54 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":3.6360000000000001,"15min":3.419},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.2719999999999998,"5min":140.881,"15min":140.881},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.84599999999999997,"5min":3.8860000000000001,"15min":3.633},"min":{"1min":0.32800000000000001,"5min":0.311,"15min":0.311},"max":{"1min":2.8250000000000002,"5min":140.911,"15min":140.911},"last":0.433}]},{"osd":5,"last update":"Sun Oct 1 09:40:28 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":1.3280000000000001,"15min":1.206},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.948,"5min":6.8810000000000002,"15min":20.457000000000001},"last":0.497},{"interface":"front","average":{"1min":0.85399999999999998,"5min":1.3720000000000001,"15min":1.02},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.0819999999999999,"5min":5.7210000000000001,"15min":20.475000000000001},"last":0.379}]},{"osd":6,"last update":"Sun Oct 1 09:41:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":0.97999999999999998,"15min":0.95399999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":5.5709999999999997,"5min":7.7649999999999997,"15min":11.358000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.381,"5min":1.22,"15min":1.0569999999999999},"min":{"1min":0.317,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":6.1660000000000004,"5min":6.9139999999999997,"15min":10.076000000000001},"last":0.55700000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":1.341,"15min":1.135},"min":{"1min":0.32000000000000001,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":2.355,"5min":38.034999999999997,"15min":38.034999999999997},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.7090000000000001,"15min":1.3460000000000001},"min":{"1min":0.372,"5min":0.34200000000000003,"15min":0.28699999999999998},"max":{"1min":1.8939999999999999,"5min":37.723999999999997,"15min":37.723999999999997},"last":0.58599999999999997}]}]},{"osd":5,"up_from":379,"seq":1627792605283,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2432480,"kb_used_data":2059736,"kb_used_omap":13,"kb_used_meta":372722,"kb_avail":91312672,"statfs":{"total":95995035648,"available":93504176128,"internally_reserved":0,"allocated":2109169664,"data_stored":2108259177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":381667826},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:40:24 2023","interfaces":[{"interface":"back","average":{"1min":1.7090000000000001,"5min":3.0640000000000001,"15min":2.3239999999999998},"min":{"1min":0.30299999999999999,"5min":0.221,"15min":0.221},"max":{"1min":10.172000000000001,"5min":60.168999999999997,"15min":84.340999999999994},"last":0.29599999999999999},{"interface":"front","average":{"1min":1.395,"5min":2.9830000000000001,"15min":2.1760000000000002},"min":{"1min":0.32100000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.8609999999999998,"5min":60.521000000000001,"15min":84.356999999999999},"last":0.28299999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:40:24 2023","interfaces":[{"interface":"back","average":{"1min":7.6429999999999998,"5min":11.151,"15min":5.5049999999999999},"min":{"1min":0.315,"5min":0.255,"15min":0.255},"max":{"1min":149.94900000000001,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.38300000000000001},{"interface":"front","average":{"1min":8.7569999999999997,"5min":11.331,"15min":5.4029999999999996},"min":{"1min":0.28000000000000003,"5min":0.25900000000000001,"15min":0.24199999999999999},"max":{"1min":149.41999999999999,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.33000000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:40:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":2.3420000000000001,"15min":1.6870000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.1500000000000004,"5min":14.199999999999999,"15min":14.199999999999999},"last":0.318},{"interface":"front","average":{"1min":1.4710000000000001,"5min":2.0289999999999999,"15min":1.71},"min":{"1min":0.33500000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.984,"5min":14.430999999999999,"15min":14.430999999999999},"last":0.40300000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:40:24 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":2.3330000000000002,"15min":1.5649999999999999},"min":{"1min":0.34000000000000002,"5min":0.308,"15min":0.30299999999999999},"max":{"1min":10.022,"5min":14.004,"15min":14.004},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.0600000000000001,"5min":2.5910000000000002,"15min":1.675},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.1760000000000002,"5min":13.991,"15min":13.991},"last":3.6960000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:40:24 2023","interfaces":[{"interface":"back","average":{"1min":6.7300000000000004,"5min":2.2989999999999999,"15min":1.421},"min":{"1min":0.33400000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":122.971,"5min":122.971,"15min":122.971},"last":3.8250000000000002},{"interface":"front","average":{"1min":6.5620000000000003,"5min":2.1739999999999999,"15min":1.821},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":122.89100000000001,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.753}]},{"osd":6,"last update":"Sun Oct 1 09:40:24 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":1.0940000000000001,"15min":1.0489999999999999},"min":{"1min":0.32700000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":10.500999999999999,"5min":12.545,"15min":12.545},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.274,"15min":1.407},"min":{"1min":0.316,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.4729999999999999,"5min":14.193,"15min":14.193},"last":3.726}]},{"osd":7,"last update":"Sun Oct 1 09:40:24 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.113,"15min":1.3},"min":{"1min":0.31900000000000001,"5min":0.307,"15min":0.307},"max":{"1min":2.6840000000000002,"5min":9.984,"15min":10.813000000000001},"last":0.496},{"interface":"front","average":{"1min":1.3140000000000001,"5min":1.2729999999999999,"15min":1.032},"min":{"1min":0.373,"5min":0.318,"15min":0.318},"max":{"1min":10.039,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.47099999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4044619776,"data_stored":4044619776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1010302976,"data_stored":1010302976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1352204288,"data_stored":1352204288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4262658048,"data_stored":4262658048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1819541504,"data_stored":1819541504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2358771712,"data_stored":2358771712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1624047616,"data_stored":1624047616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:41:33.023 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:41:33.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:32 smithi190 ceph-mon[160435]: pgmap v1432: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 6.7 MiB/s wr, 107 op/s; 10/246168 objects degraded (0.004%); 7300/246168 objects misplaced (2.965%); 13 MiB/s, 213 objects/s recovering 2023-10-01T09:41:33.251 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:33.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:33.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:33.354 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:33.382 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:33.425 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:33.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:33.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:33.531 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:41:33.558 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:33.580 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:33.601 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:41:33.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:33.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:41:33.615 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:41:33.616 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:41:33.628 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 82 16 82346 82330 62.731 0 - 0.0139471 2023-10-01T09:41:33.707 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:33.738 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:33.986 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:33.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:33.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:34.090 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:41:34.117 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:34.161 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:41:34.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:34.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:41:34.268 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:34.298 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:34.494 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:34.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:34.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:34.597 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:34.625 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:34.629 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 83 16 82346 82330 61.9754 0 - 0.0139471 2023-10-01T09:41:34.688 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:41:34.707 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:41:34.725 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:41:34.726 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:41:34.727 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:41:34.727 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:41:34.727 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:41:34.727 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:41:34.727 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:41:34.727 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:41:34.727 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:41:35.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:41:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:41:34] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.33.4" 2023-10-01T09:41:35.453 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:41:35.629 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 84 16 82346 82330 61.2378 0 - 0.0139471 2023-10-01T09:41:35.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:35 smithi190 ceph-mon[160435]: pgmap v1433: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.5 MiB/s wr, 55 op/s; 10/246168 objects degraded (0.004%); 6991/246168 objects misplaced (2.840%); 16 MiB/s, 251 objects/s recovering 2023-10-01T09:41:35.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:35 smithi138 ceph-mon[185886]: pgmap v1433: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.5 MiB/s wr, 55 op/s; 10/246168 objects degraded (0.004%); 6991/246168 objects misplaced (2.840%); 16 MiB/s, 251 objects/s recovering 2023-10-01T09:41:35.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:35 smithi138 ceph-mon[192513]: pgmap v1433: 89 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 3.5 MiB/s wr, 55 op/s; 10/246168 objects degraded (0.004%); 6991/246168 objects misplaced (2.840%); 16 MiB/s, 251 objects/s recovering 2023-10-01T09:41:35.976 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:35.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:35.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:36.081 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:41:36.108 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:36.629 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 85 16 82346 82330 60.5175 0 - 0.0139471 2023-10-01T09:41:36.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:36 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 13964/247074 objects degraded (5.652%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:36.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:36 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 13964/247074 objects degraded (5.652%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:36.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:36 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 13964/247074 objects degraded (5.652%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:37.208 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:41:37.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:37.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:41:37.226 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:41:37.312 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:41:37.371 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:37.510 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:37.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:37 smithi138 ceph-mon[192513]: pgmap v1434: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 2.4 MiB/s wr, 37 op/s; 13964/247074 objects degraded (5.652%); 3293/247074 objects misplaced (1.333%); 8.7 MiB/s, 139 objects/s recovering 2023-10-01T09:41:37.565 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:37 smithi138 ceph-mon[185886]: pgmap v1434: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 2.4 MiB/s wr, 37 op/s; 13964/247074 objects degraded (5.652%); 3293/247074 objects misplaced (1.333%); 8.7 MiB/s, 139 objects/s recovering 2023-10-01T09:41:37.629 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 86 16 82346 82330 59.814 0 - 0.0139471 2023-10-01T09:41:37.790 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:41:37.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:37.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:41:37.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:37 smithi190 ceph-mon[160435]: pgmap v1434: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 2.4 MiB/s wr, 37 op/s; 13964/247074 objects degraded (5.652%); 3293/247074 objects misplaced (1.333%); 8.7 MiB/s, 139 objects/s recovering 2023-10-01T09:41:37.894 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:37.922 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:37.966 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:37.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:37.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:38.073 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:41:38.074 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:41:38.075 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:41:38.101 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:38.149 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:41:38.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:38.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:41:38.258 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:38.289 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:38.338 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:38.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:38.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:38.442 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:41:38.472 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:38.629 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 87 16 82404 82388 59.1683 0.258929 0.00363052 0.0153041 2023-10-01T09:41:39.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:38 smithi190 ceph-mon[160435]: from='client.36585 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:41:39.060 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:38 smithi190 ceph-mon[160435]: pgmap v1435: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 2.2 MiB/s wr, 35 op/s; 13964/247074 objects degraded (5.652%); 2688/247074 objects misplaced (1.088%); 13 MiB/s, 201 objects/s recovering 2023-10-01T09:41:39.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:38 smithi138 ceph-mon[192513]: from='client.36585 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:41:39.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:38 smithi138 ceph-mon[192513]: pgmap v1435: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 2.2 MiB/s wr, 35 op/s; 13964/247074 objects degraded (5.652%); 2688/247074 objects misplaced (1.088%); 13 MiB/s, 201 objects/s recovering 2023-10-01T09:41:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:38 smithi138 ceph-mon[185886]: from='client.36585 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:41:39.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:38 smithi138 ceph-mon[185886]: pgmap v1435: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 2.2 MiB/s wr, 35 op/s; 13964/247074 objects degraded (5.652%); 2688/247074 objects misplaced (1.088%); 13 MiB/s, 201 objects/s recovering 2023-10-01T09:41:39.177 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:41:39.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:39.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:41:39.282 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:39.299 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:41:39.313 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:39.629 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 88 16 82698 82682 58.7048 18.375 0.00375804 0.0169528 2023-10-01T09:41:40.029 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:40.242 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:40.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:40.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:40.340 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1434,"stamp":"2023-10-01T09:41:36.279062+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5400369075,"num_objects":82358,"num_object_clones":0,"num_object_copies":247074,"num_objects_missing_on_primary":3698,"num_objects_missing":3698,"num_objects_degraded":13964,"num_objects_misplaced":3293,"num_objects_unfound":0,"num_objects_dirty":82358,"num_whiteouts":0,"num_read":6259,"num_read_kb":12659,"num_write":83436,"num_write_kb":5284261,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4111,"num_bytes_recovered":256279567,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83500,"ondisk_log_size":83500,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":22150132,"kb_used_data":17148028,"kb_used_omap":39,"kb_used_meta":5002008,"kb_avail":634065932,"statfs":{"total":671965249536,"available":649283514368,"internally_reserved":0,"allocated":17559580672,"data_stored":17549877475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5122056815},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,16],"upper_bound":32768},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19791872,"num_objects":302,"num_object_clones":0,"num_object_copies":906,"num_objects_missing_on_primary":3698,"num_objects_missing":3698,"num_objects_degraded":13954,"num_objects_misplaced":-4812,"num_objects_unfound":0,"num_objects_dirty":302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":302,"num_write_kb":19328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1114,"num_bytes_recovered":73007104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001903"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":282,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.765131+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:41:29.765131+0000","last_peered":"2023-10-01T09:41:29.765131+0000","last_clean":"2023-10-01T09:41:29.765131+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:41:29.765131+0000","last_undegraded":"2023-10-01T09:41:29.765131+0000","last_fullsized":"2023-10-01T09:41:29.765131+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":124,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.236020+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:41:30.236020+0000","last_peered":"2023-10-01T09:41:30.236020+0000","last_clean":"2023-10-01T09:41:30.236020+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:41:30.236020+0000","last_undegraded":"2023-10-01T09:41:30.236020+0000","last_fullsized":"2023-10-01T09:41:30.236020+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":505,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.234956+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:41:30.234956+0000","last_peered":"2023-10-01T09:41:30.234956+0000","last_clean":"2023-10-01T09:41:30.234956+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:41:30.234956+0000","last_undegraded":"2023-10-01T09:41:30.234956+0000","last_fullsized":"2023-10-01T09:41:30.234956+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":563,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.761066+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:41:29.761066+0000","last_peered":"2023-10-01T09:41:29.761066+0000","last_clean":"2023-10-01T09:41:29.761066+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:41:29.761066+0000","last_undegraded":"2023-10-01T09:41:29.761066+0000","last_fullsized":"2023-10-01T09:41:29.761066+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":444,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.759522+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:41:29.759522+0000","last_peered":"2023-10-01T09:41:29.759522+0000","last_clean":"2023-10-01T09:41:29.759522+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:41:29.759522+0000","last_undegraded":"2023-10-01T09:41:29.759522+0000","last_fullsized":"2023-10-01T09:41:29.759522+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":187,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.753326+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:41:22.753326+0000","last_peered":"2023-10-01T09:41:22.753326+0000","last_clean":"2023-10-01T09:41:22.753326+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:41:22.753326+0000","last_undegraded":"2023-10-01T09:41:22.753326+0000","last_fullsized":"2023-10-01T09:41:22.753326+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":563,"reported_epoch":503,"state":"active+clean","last_fresh":"2023-10-01T09:41:22.753396+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:41:22.753396+0000","last_peered":"2023-10-01T09:41:22.753396+0000","last_clean":"2023-10-01T09:41:22.753396+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:41:22.753396+0000","last_undegraded":"2023-10-01T09:41:22.753396+0000","last_fullsized":"2023-10-01T09:41:22.753396+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":491,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.758994+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:41:29.758994+0000","last_peered":"2023-10-01T09:41:29.758994+0000","last_clean":"2023-10-01T09:41:29.758994+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:41:29.758994+0000","last_undegraded":"2023-10-01T09:41:29.758994+0000","last_fullsized":"2023-10-01T09:41:29.758994+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":218,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760615+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:41:29.760615+0000","last_peered":"2023-10-01T09:41:29.760615+0000","last_clean":"2023-10-01T09:41:29.760615+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:41:29.760615+0000","last_undegraded":"2023-10-01T09:41:29.760615+0000","last_fullsized":"2023-10-01T09:41:29.760615+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":25,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.764414+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:41:29.764414+0000","last_peered":"2023-10-01T09:41:29.764414+0000","last_clean":"2023-10-01T09:41:29.764414+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:41:29.764414+0000","last_undegraded":"2023-10-01T09:41:29.764414+0000","last_fullsized":"2023-10-01T09:41:29.764414+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":31,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.765328+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:41:29.765328+0000","last_peered":"2023-10-01T09:41:29.765328+0000","last_clean":"2023-10-01T09:41:29.765328+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:41:29.765328+0000","last_undegraded":"2023-10-01T09:41:29.765328+0000","last_fullsized":"2023-10-01T09:41:29.765328+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":28,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.234974+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:41:30.234974+0000","last_peered":"2023-10-01T09:41:30.234974+0000","last_clean":"2023-10-01T09:41:30.234974+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:41:30.234974+0000","last_undegraded":"2023-10-01T09:41:30.234974+0000","last_fullsized":"2023-10-01T09:41:30.234974+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":407,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.766518+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:41:29.766518+0000","last_peered":"2023-10-01T09:41:29.766518+0000","last_clean":"2023-10-01T09:41:29.766518+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:41:29.766518+0000","last_undegraded":"2023-10-01T09:41:29.766518+0000","last_fullsized":"2023-10-01T09:41:29.766518+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":288,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.765463+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:41:29.765463+0000","last_peered":"2023-10-01T09:41:29.765463+0000","last_clean":"2023-10-01T09:41:29.765463+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:41:29.765463+0000","last_undegraded":"2023-10-01T09:41:29.765463+0000","last_fullsized":"2023-10-01T09:41:29.765463+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":624,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.764453+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:41:29.764453+0000","last_peered":"2023-10-01T09:41:29.764453+0000","last_clean":"2023-10-01T09:41:29.764453+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:41:29.764453+0000","last_undegraded":"2023-10-01T09:41:29.764453+0000","last_fullsized":"2023-10-01T09:41:29.764453+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":361,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.764832+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:41:29.764832+0000","last_peered":"2023-10-01T09:41:29.764832+0000","last_clean":"2023-10-01T09:41:29.764832+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:41:29.764832+0000","last_undegraded":"2023-10-01T09:41:29.764832+0000","last_fullsized":"2023-10-01T09:41:29.764832+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":34,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.764359+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:41:29.764359+0000","last_peered":"2023-10-01T09:41:29.764359+0000","last_clean":"2023-10-01T09:41:29.764359+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:41:29.764359+0000","last_undegraded":"2023-10-01T09:41:29.764359+0000","last_fullsized":"2023-10-01T09:41:29.764359+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":20,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760515+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:41:29.760515+0000","last_peered":"2023-10-01T09:41:29.760515+0000","last_clean":"2023-10-01T09:41:29.760515+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:41:29.760515+0000","last_undegraded":"2023-10-01T09:41:29.760515+0000","last_fullsized":"2023-10-01T09:41:29.760515+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":517,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.764688+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:41:29.764688+0000","last_peered":"2023-10-01T09:41:29.764688+0000","last_clean":"2023-10-01T09:41:29.764688+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:41:29.764688+0000","last_undegraded":"2023-10-01T09:41:29.764688+0000","last_fullsized":"2023-10-01T09:41:29.764688+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":527,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.761130+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:41:29.761130+0000","last_peered":"2023-10-01T09:41:29.761130+0000","last_clean":"2023-10-01T09:41:29.761130+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:41:29.761130+0000","last_undegraded":"2023-10-01T09:41:29.761130+0000","last_fullsized":"2023-10-01T09:41:29.761130+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":383,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.759393+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:41:29.759393+0000","last_peered":"2023-10-01T09:41:29.759393+0000","last_clean":"2023-10-01T09:41:29.759393+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:41:29.759393+0000","last_undegraded":"2023-10-01T09:41:29.759393+0000","last_fullsized":"2023-10-01T09:41:29.759393+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":34,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.766621+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:41:29.766621+0000","last_peered":"2023-10-01T09:41:29.766621+0000","last_clean":"2023-10-01T09:41:29.766621+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:41:29.766621+0000","last_undegraded":"2023-10-01T09:41:29.766621+0000","last_fullsized":"2023-10-01T09:41:29.766621+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"504'5137","reported_seq":5176,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760794+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:41:29.760794+0000","last_peered":"2023-10-01T09:41:29.760794+0000","last_clean":"2023-10-01T09:41:29.760794+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:41:29.760794+0000","last_undegraded":"2023-10-01T09:41:29.760794+0000","last_fullsized":"2023-10-01T09:41:29.760794+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5137,"log_dups_size":0,"ondisk_log_size":5137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336658432,"num_objects":5137,"num_object_clones":0,"num_object_copies":15411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5137,"num_write_kb":328768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":29,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.766524+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:41:29.766524+0000","last_peered":"2023-10-01T09:41:29.766524+0000","last_clean":"2023-10-01T09:41:29.766524+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:41:29.766524+0000","last_undegraded":"2023-10-01T09:41:29.766524+0000","last_fullsized":"2023-10-01T09:41:29.766524+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":226,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.761385+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:41:29.761385+0000","last_peered":"2023-10-01T09:41:29.761385+0000","last_clean":"2023-10-01T09:41:29.761385+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:41:29.761385+0000","last_undegraded":"2023-10-01T09:41:29.761385+0000","last_fullsized":"2023-10-01T09:41:29.761385+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"501'5135","reported_seq":5185,"reported_epoch":504,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-10-01T09:41:34.629037+0000","last_change":"2023-10-01T09:41:34.656563+0000","last_active":"2023-10-01T09:41:22.367037+0000","last_peered":"2023-10-01T09:41:22.345686+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:40:56.557114+0000","last_became_peered":"2023-10-01T09:40:56.557114+0000","last_unstale":"2023-10-01T09:41:34.629037+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5135,"log_dups_size":0,"ondisk_log_size":5135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336527360,"num_objects":5135,"num_object_clones":0,"num_object_copies":15405,"num_objects_missing_on_primary":3698,"num_objects_missing":3698,"num_objects_degraded":8833,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5135,"num_write_kb":328640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,5,6","objects":1437},{"shards":"0,6","objects":3698}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":500,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.766304+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:41:29.766304+0000","last_peered":"2023-10-01T09:41:29.766304+0000","last_clean":"2023-10-01T09:41:29.766304+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:41:29.766304+0000","last_undegraded":"2023-10-01T09:41:29.766304+0000","last_fullsized":"2023-10-01T09:41:29.766304+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":560,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.234921+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:41:30.234921+0000","last_peered":"2023-10-01T09:41:30.234921+0000","last_clean":"2023-10-01T09:41:30.234921+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:41:30.234921+0000","last_undegraded":"2023-10-01T09:41:30.234921+0000","last_fullsized":"2023-10-01T09:41:30.234921+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"504'5147","reported_seq":5187,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760254+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:41:29.760254+0000","last_peered":"2023-10-01T09:41:29.760254+0000","last_clean":"2023-10-01T09:41:29.760254+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:41:29.760254+0000","last_undegraded":"2023-10-01T09:41:29.760254+0000","last_fullsized":"2023-10-01T09:41:29.760254+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5147,"log_dups_size":0,"ondisk_log_size":5147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337313792,"num_objects":5147,"num_object_clones":0,"num_object_copies":15441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5147,"num_write_kb":329408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":586,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.765467+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:41:29.765467+0000","last_peered":"2023-10-01T09:41:29.765467+0000","last_clean":"2023-10-01T09:41:29.765467+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:41:29.765467+0000","last_undegraded":"2023-10-01T09:41:29.765467+0000","last_fullsized":"2023-10-01T09:41:29.765467+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":380,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.236349+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:41:30.236349+0000","last_peered":"2023-10-01T09:41:30.236349+0000","last_clean":"2023-10-01T09:41:30.236349+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:41:30.236349+0000","last_undegraded":"2023-10-01T09:41:30.236349+0000","last_fullsized":"2023-10-01T09:41:30.236349+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"504'5147","reported_seq":11792,"reported_epoch":506,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.777755+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:41:30.777755+0000","last_peered":"2023-10-01T09:41:30.777755+0000","last_clean":"2023-10-01T09:41:30.777755+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:41:30.777755+0000","last_undegraded":"2023-10-01T09:41:30.777755+0000","last_fullsized":"2023-10-01T09:41:30.777755+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5147,"log_dups_size":0,"ondisk_log_size":5147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337313792,"num_objects":5147,"num_object_clones":0,"num_object_copies":15441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5147,"num_write_kb":329408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":226,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760626+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:41:29.760626+0000","last_peered":"2023-10-01T09:41:29.760626+0000","last_clean":"2023-10-01T09:41:29.760626+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:41:29.760626+0000","last_undegraded":"2023-10-01T09:41:29.760626+0000","last_fullsized":"2023-10-01T09:41:29.760626+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":483,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760534+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:41:29.760534+0000","last_peered":"2023-10-01T09:41:29.760534+0000","last_clean":"2023-10-01T09:41:29.760534+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:41:29.760534+0000","last_undegraded":"2023-10-01T09:41:29.760534+0000","last_fullsized":"2023-10-01T09:41:29.760534+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"504'5189","reported_seq":5229,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.236432+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:41:30.236432+0000","last_peered":"2023-10-01T09:41:30.236432+0000","last_clean":"2023-10-01T09:41:30.236432+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:41:30.236432+0000","last_undegraded":"2023-10-01T09:41:30.236432+0000","last_fullsized":"2023-10-01T09:41:30.236432+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5189,"log_dups_size":0,"ondisk_log_size":5189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340066304,"num_objects":5189,"num_object_clones":0,"num_object_copies":15567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5189,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":86,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.765806+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:41:29.765806+0000","last_peered":"2023-10-01T09:41:29.765806+0000","last_clean":"2023-10-01T09:41:29.765806+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:41:29.765806+0000","last_undegraded":"2023-10-01T09:41:29.765806+0000","last_fullsized":"2023-10-01T09:41:29.765806+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":160,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.766603+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:41:29.766603+0000","last_peered":"2023-10-01T09:41:29.766603+0000","last_clean":"2023-10-01T09:41:29.766603+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:41:29.766603+0000","last_undegraded":"2023-10-01T09:41:29.766603+0000","last_fullsized":"2023-10-01T09:41:29.766603+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"504'5137","reported_seq":5183,"reported_epoch":505,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:41:29.763356+0000","last_change":"2023-10-01T09:41:23.954209+0000","last_active":"2023-10-01T09:41:29.763356+0000","last_peered":"2023-10-01T09:41:29.763356+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:41:29.763356+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5137,"log_dups_size":0,"ondisk_log_size":5137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336658432,"num_objects":5137,"num_object_clones":0,"num_object_copies":15411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5121,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5137,"num_write_kb":328768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":5120},{"shards":"4,5","objects":16}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":404,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.765219+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:41:29.765219+0000","last_peered":"2023-10-01T09:41:29.765219+0000","last_clean":"2023-10-01T09:41:29.765219+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:41:29.765219+0000","last_undegraded":"2023-10-01T09:41:29.765219+0000","last_fullsized":"2023-10-01T09:41:29.765219+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":563,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.236114+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:41:30.236114+0000","last_peered":"2023-10-01T09:41:30.236114+0000","last_clean":"2023-10-01T09:41:30.236114+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:41:30.236114+0000","last_undegraded":"2023-10-01T09:41:30.236114+0000","last_fullsized":"2023-10-01T09:41:30.236114+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"504'5137","reported_seq":5185,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.758277+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:41:29.758277+0000","last_peered":"2023-10-01T09:41:29.758277+0000","last_clean":"2023-10-01T09:41:29.758277+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:41:29.758277+0000","last_undegraded":"2023-10-01T09:41:29.758277+0000","last_fullsized":"2023-10-01T09:41:29.758277+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5137,"log_dups_size":0,"ondisk_log_size":5137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336658432,"num_objects":5137,"num_object_clones":0,"num_object_copies":15411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5137,"num_write_kb":328768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":400,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.765016+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:41:29.765016+0000","last_peered":"2023-10-01T09:41:29.765016+0000","last_clean":"2023-10-01T09:41:29.765016+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:41:29.765016+0000","last_undegraded":"2023-10-01T09:41:29.765016+0000","last_fullsized":"2023-10-01T09:41:29.765016+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":241,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.765495+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:41:29.765495+0000","last_peered":"2023-10-01T09:41:29.765495+0000","last_clean":"2023-10-01T09:41:29.765495+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:41:29.765495+0000","last_undegraded":"2023-10-01T09:41:29.765495+0000","last_fullsized":"2023-10-01T09:41:29.765495+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"504'5196","reported_seq":5235,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.764584+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:41:29.764584+0000","last_peered":"2023-10-01T09:41:29.764584+0000","last_clean":"2023-10-01T09:41:29.764584+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:41:29.764584+0000","last_undegraded":"2023-10-01T09:41:29.764584+0000","last_fullsized":"2023-10-01T09:41:29.764584+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5196,"log_dups_size":0,"ondisk_log_size":5196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340525056,"num_objects":5196,"num_object_clones":0,"num_object_copies":15588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5196,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":368,"reported_epoch":506,"state":"active+clean","last_fresh":"2023-10-01T09:41:31.178473+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:41:31.178473+0000","last_peered":"2023-10-01T09:41:31.178473+0000","last_clean":"2023-10-01T09:41:31.178473+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:41:31.178473+0000","last_undegraded":"2023-10-01T09:41:31.178473+0000","last_fullsized":"2023-10-01T09:41:31.178473+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":415,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.235146+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:41:30.235146+0000","last_peered":"2023-10-01T09:41:30.235146+0000","last_clean":"2023-10-01T09:41:30.235146+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:41:30.235146+0000","last_undegraded":"2023-10-01T09:41:30.235146+0000","last_fullsized":"2023-10-01T09:41:30.235146+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"504'5044","reported_seq":5083,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.764597+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:41:29.764597+0000","last_peered":"2023-10-01T09:41:29.764597+0000","last_clean":"2023-10-01T09:41:29.764597+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:41:29.764597+0000","last_undegraded":"2023-10-01T09:41:29.764597+0000","last_fullsized":"2023-10-01T09:41:29.764597+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5044,"log_dups_size":0,"ondisk_log_size":5044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330563584,"num_objects":5044,"num_object_clones":0,"num_object_copies":15132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5044,"num_write_kb":322816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":899,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.764683+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:41:29.764683+0000","last_peered":"2023-10-01T09:41:29.764683+0000","last_clean":"2023-10-01T09:41:29.764683+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:41:29.764683+0000","last_undegraded":"2023-10-01T09:41:29.764683+0000","last_fullsized":"2023-10-01T09:41:29.764683+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":236,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760627+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:41:29.760627+0000","last_peered":"2023-10-01T09:41:29.760627+0000","last_clean":"2023-10-01T09:41:29.760627+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:41:29.760627+0000","last_undegraded":"2023-10-01T09:41:29.760627+0000","last_fullsized":"2023-10-01T09:41:29.760627+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.236200+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:41:30.236200+0000","last_peered":"2023-10-01T09:41:30.236200+0000","last_clean":"2023-10-01T09:41:30.236200+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:41:30.236200+0000","last_undegraded":"2023-10-01T09:41:30.236200+0000","last_fullsized":"2023-10-01T09:41:30.236200+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"504'5122","reported_seq":5162,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.759522+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:41:29.759522+0000","last_peered":"2023-10-01T09:41:29.759522+0000","last_clean":"2023-10-01T09:41:29.759522+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:41:29.759522+0000","last_undegraded":"2023-10-01T09:41:29.759522+0000","last_fullsized":"2023-10-01T09:41:29.759522+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5122,"log_dups_size":0,"ondisk_log_size":5122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335675392,"num_objects":5122,"num_object_clones":0,"num_object_copies":15366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5122,"num_write_kb":327808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1171,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760497+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:41:29.760497+0000","last_peered":"2023-10-01T09:41:29.760497+0000","last_clean":"2023-10-01T09:41:29.760497+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:41:29.760497+0000","last_undegraded":"2023-10-01T09:41:29.760497+0000","last_fullsized":"2023-10-01T09:41:29.760497+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":381,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.765664+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:41:29.765664+0000","last_peered":"2023-10-01T09:41:29.765664+0000","last_clean":"2023-10-01T09:41:29.765664+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:41:29.765664+0000","last_undegraded":"2023-10-01T09:41:29.765664+0000","last_fullsized":"2023-10-01T09:41:29.765664+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":209,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760806+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:41:29.760806+0000","last_peered":"2023-10-01T09:41:29.760806+0000","last_clean":"2023-10-01T09:41:29.760806+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:41:29.760806+0000","last_undegraded":"2023-10-01T09:41:29.760806+0000","last_fullsized":"2023-10-01T09:41:29.760806+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"504'5124","reported_seq":5787,"reported_epoch":506,"state":"active+recovering+remapped","last_fresh":"2023-10-01T09:41:32.429335+0000","last_change":"2023-10-01T09:41:29.602605+0000","last_active":"2023-10-01T09:41:32.429335+0000","last_peered":"2023-10-01T09:41:32.429335+0000","last_clean":"2023-10-01T09:40:54.363445+0000","last_became_active":"2023-10-01T09:40:56.569721+0000","last_became_peered":"2023-10-01T09:40:56.569721+0000","last_unstale":"2023-10-01T09:41:32.429335+0000","last_undegraded":"2023-10-01T09:41:32.429335+0000","last_fullsized":"2023-10-01T09:41:32.429335+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5124,"log_dups_size":0,"ondisk_log_size":5124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:17:29.742329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335806464,"num_objects":5124,"num_object_clones":0,"num_object_copies":15372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3293,"num_objects_unfound":0,"num_objects_dirty":5124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5124,"num_write_kb":327936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":309,"num_bytes_recovered":20250624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":1831}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":850,"reported_epoch":505,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:41:29.759653+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:41:29.759653+0000","last_peered":"2023-10-01T09:41:29.759653+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:41:29.759653+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:41:29.759653+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":567,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.234964+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:41:30.234964+0000","last_peered":"2023-10-01T09:41:30.234964+0000","last_clean":"2023-10-01T09:41:30.234964+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:41:30.234964+0000","last_undegraded":"2023-10-01T09:41:30.234964+0000","last_fullsized":"2023-10-01T09:41:30.234964+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":28,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.235059+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:41:30.235059+0000","last_peered":"2023-10-01T09:41:30.235059+0000","last_clean":"2023-10-01T09:41:30.235059+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:41:30.235059+0000","last_undegraded":"2023-10-01T09:41:30.235059+0000","last_fullsized":"2023-10-01T09:41:30.235059+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"504'5090","reported_seq":5130,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.236675+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:41:30.236675+0000","last_peered":"2023-10-01T09:41:30.236675+0000","last_clean":"2023-10-01T09:41:30.236675+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:41:30.236675+0000","last_undegraded":"2023-10-01T09:41:30.236675+0000","last_fullsized":"2023-10-01T09:41:30.236675+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5090,"log_dups_size":0,"ondisk_log_size":5090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333578240,"num_objects":5090,"num_object_clones":0,"num_object_copies":15270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5090,"num_write_kb":325760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1622,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760710+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:41:29.760710+0000","last_peered":"2023-10-01T09:41:29.760710+0000","last_clean":"2023-10-01T09:41:29.760710+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:41:29.760710+0000","last_undegraded":"2023-10-01T09:41:29.760710+0000","last_fullsized":"2023-10-01T09:41:29.760710+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":536,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.764913+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:41:29.764913+0000","last_peered":"2023-10-01T09:41:29.764913+0000","last_clean":"2023-10-01T09:41:29.764913+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:41:29.764913+0000","last_undegraded":"2023-10-01T09:41:29.764913+0000","last_fullsized":"2023-10-01T09:41:29.764913+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":314,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760848+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:41:29.760848+0000","last_peered":"2023-10-01T09:41:29.760848+0000","last_clean":"2023-10-01T09:41:29.760848+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:41:29.760848+0000","last_undegraded":"2023-10-01T09:41:29.760848+0000","last_fullsized":"2023-10-01T09:41:29.760848+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":304,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.766132+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:41:29.766132+0000","last_peered":"2023-10-01T09:41:29.766132+0000","last_clean":"2023-10-01T09:41:29.766132+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:41:29.766132+0000","last_undegraded":"2023-10-01T09:41:29.766132+0000","last_fullsized":"2023-10-01T09:41:29.766132+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"504'5217","reported_seq":5256,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.765008+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:41:29.765008+0000","last_peered":"2023-10-01T09:41:29.765008+0000","last_clean":"2023-10-01T09:41:29.765008+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:41:29.765008+0000","last_undegraded":"2023-10-01T09:41:29.765008+0000","last_fullsized":"2023-10-01T09:41:29.765008+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5217,"log_dups_size":0,"ondisk_log_size":5217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341901312,"num_objects":5217,"num_object_clones":0,"num_object_copies":15651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5217,"num_write_kb":333888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":713,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.765094+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:41:29.765094+0000","last_peered":"2023-10-01T09:41:29.765094+0000","last_clean":"2023-10-01T09:41:29.765094+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:41:29.765094+0000","last_undegraded":"2023-10-01T09:41:29.765094+0000","last_fullsized":"2023-10-01T09:41:29.765094+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":544,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760030+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:41:29.760030+0000","last_peered":"2023-10-01T09:41:29.760030+0000","last_clean":"2023-10-01T09:41:29.760030+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:41:29.760030+0000","last_undegraded":"2023-10-01T09:41:29.760030+0000","last_fullsized":"2023-10-01T09:41:29.760030+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":544,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760913+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:41:29.760913+0000","last_peered":"2023-10-01T09:41:29.760913+0000","last_clean":"2023-10-01T09:41:29.760913+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:41:29.760913+0000","last_undegraded":"2023-10-01T09:41:29.760913+0000","last_fullsized":"2023-10-01T09:41:29.760913+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"504'5172","reported_seq":5212,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.762085+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:41:29.762085+0000","last_peered":"2023-10-01T09:41:29.762085+0000","last_clean":"2023-10-01T09:41:29.762085+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:41:29.762085+0000","last_undegraded":"2023-10-01T09:41:29.762085+0000","last_fullsized":"2023-10-01T09:41:29.762085+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5172,"log_dups_size":0,"ondisk_log_size":5172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338952192,"num_objects":5172,"num_object_clones":0,"num_object_copies":15516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5172,"num_write_kb":331008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":944,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.765540+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:41:29.765540+0000","last_peered":"2023-10-01T09:41:29.765540+0000","last_clean":"2023-10-01T09:41:29.765540+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:41:29.765540+0000","last_undegraded":"2023-10-01T09:41:29.765540+0000","last_fullsized":"2023-10-01T09:41:29.765540+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":563,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760718+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:41:29.760718+0000","last_peered":"2023-10-01T09:41:29.760718+0000","last_clean":"2023-10-01T09:41:29.760718+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:41:29.760718+0000","last_undegraded":"2023-10-01T09:41:29.760718+0000","last_fullsized":"2023-10-01T09:41:29.760718+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":416,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760770+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:41:29.760770+0000","last_peered":"2023-10-01T09:41:29.760770+0000","last_clean":"2023-10-01T09:41:29.760770+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:41:29.760770+0000","last_undegraded":"2023-10-01T09:41:29.760770+0000","last_fullsized":"2023-10-01T09:41:29.760770+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"504'5137","reported_seq":5184,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.765289+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:41:29.765289+0000","last_peered":"2023-10-01T09:41:29.765289+0000","last_clean":"2023-10-01T09:41:29.765289+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:41:29.765289+0000","last_undegraded":"2023-10-01T09:41:29.765289+0000","last_fullsized":"2023-10-01T09:41:29.765289+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5137,"log_dups_size":0,"ondisk_log_size":5137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336658432,"num_objects":5137,"num_object_clones":0,"num_object_copies":15411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5137,"num_write_kb":328768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":445,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.766713+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:41:29.766713+0000","last_peered":"2023-10-01T09:41:29.766713+0000","last_clean":"2023-10-01T09:41:29.766713+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:41:29.766713+0000","last_undegraded":"2023-10-01T09:41:29.766713+0000","last_fullsized":"2023-10-01T09:41:29.766713+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1137,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.235732+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:41:30.235732+0000","last_peered":"2023-10-01T09:41:30.235732+0000","last_clean":"2023-10-01T09:41:30.235732+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:41:30.235732+0000","last_undegraded":"2023-10-01T09:41:30.235732+0000","last_fullsized":"2023-10-01T09:41:30.235732+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":34,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.764335+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:41:29.764335+0000","last_peered":"2023-10-01T09:41:29.764335+0000","last_clean":"2023-10-01T09:41:29.764335+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:41:29.764335+0000","last_undegraded":"2023-10-01T09:41:29.764335+0000","last_fullsized":"2023-10-01T09:41:29.764335+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"504'5199","reported_seq":5246,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.766413+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:41:29.766413+0000","last_peered":"2023-10-01T09:41:29.766413+0000","last_clean":"2023-10-01T09:41:29.766413+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:41:29.766413+0000","last_undegraded":"2023-10-01T09:41:29.766413+0000","last_fullsized":"2023-10-01T09:41:29.766413+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5199,"log_dups_size":0,"ondisk_log_size":5199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340721664,"num_objects":5199,"num_object_clones":0,"num_object_copies":15597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5199,"num_write_kb":332736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":38,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.766578+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:41:29.766578+0000","last_peered":"2023-10-01T09:41:29.766578+0000","last_clean":"2023-10-01T09:41:29.766578+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:41:29.766578+0000","last_undegraded":"2023-10-01T09:41:29.766578+0000","last_fullsized":"2023-10-01T09:41:29.766578+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":522,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760998+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:41:29.760998+0000","last_peered":"2023-10-01T09:41:29.760998+0000","last_clean":"2023-10-01T09:41:29.760998+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:41:29.760998+0000","last_undegraded":"2023-10-01T09:41:29.760998+0000","last_fullsized":"2023-10-01T09:41:29.760998+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":229,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760701+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:41:29.760701+0000","last_peered":"2023-10-01T09:41:29.760701+0000","last_clean":"2023-10-01T09:41:29.760701+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:41:29.760701+0000","last_undegraded":"2023-10-01T09:41:29.760701+0000","last_fullsized":"2023-10-01T09:41:29.760701+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":471,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760685+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:41:29.760685+0000","last_peered":"2023-10-01T09:41:29.760685+0000","last_clean":"2023-10-01T09:41:29.760685+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:41:29.760685+0000","last_undegraded":"2023-10-01T09:41:29.760685+0000","last_fullsized":"2023-10-01T09:41:29.760685+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":585,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.235268+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:41:30.235268+0000","last_peered":"2023-10-01T09:41:30.235268+0000","last_clean":"2023-10-01T09:41:30.235268+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:41:30.235268+0000","last_undegraded":"2023-10-01T09:41:30.235268+0000","last_fullsized":"2023-10-01T09:41:30.235268+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":241,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760712+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:41:29.760712+0000","last_peered":"2023-10-01T09:41:29.760712+0000","last_clean":"2023-10-01T09:41:29.760712+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:41:29.760712+0000","last_undegraded":"2023-10-01T09:41:29.760712+0000","last_fullsized":"2023-10-01T09:41:29.760712+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":28,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.236591+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:41:30.236591+0000","last_peered":"2023-10-01T09:41:30.236591+0000","last_clean":"2023-10-01T09:41:30.236591+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:41:30.236591+0000","last_undegraded":"2023-10-01T09:41:30.236591+0000","last_fullsized":"2023-10-01T09:41:30.236591+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":241,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.760656+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:41:29.760656+0000","last_peered":"2023-10-01T09:41:29.760656+0000","last_clean":"2023-10-01T09:41:29.760656+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:41:29.760656+0000","last_undegraded":"2023-10-01T09:41:29.760656+0000","last_fullsized":"2023-10-01T09:41:29.760656+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":417,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.759445+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:41:29.759445+0000","last_peered":"2023-10-01T09:41:29.759445+0000","last_clean":"2023-10-01T09:41:29.759445+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:41:29.759445+0000","last_undegraded":"2023-10-01T09:41:29.759445+0000","last_fullsized":"2023-10-01T09:41:29.759445+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":45,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:30.235496+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:41:30.235496+0000","last_peered":"2023-10-01T09:41:30.235496+0000","last_clean":"2023-10-01T09:41:30.235496+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:41:30.235496+0000","last_undegraded":"2023-10-01T09:41:30.235496+0000","last_fullsized":"2023-10-01T09:41:30.235496+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":462,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.759945+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:41:29.759945+0000","last_peered":"2023-10-01T09:41:29.759945+0000","last_clean":"2023-10-01T09:41:29.759945+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:41:29.759945+0000","last_undegraded":"2023-10-01T09:41:29.759945+0000","last_fullsized":"2023-10-01T09:41:29.759945+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":661,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.761338+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:41:29.761338+0000","last_peered":"2023-10-01T09:41:29.761338+0000","last_clean":"2023-10-01T09:41:29.761338+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:41:29.761338+0000","last_undegraded":"2023-10-01T09:41:29.761338+0000","last_fullsized":"2023-10-01T09:41:29.761338+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":357,"reported_epoch":505,"state":"active+clean","last_fresh":"2023-10-01T09:41:29.764835+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:41:29.764835+0000","last_peered":"2023-10-01T09:41:29.764835+0000","last_clean":"2023-10-01T09:41:29.764835+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:41:29.764835+0000","last_undegraded":"2023-10-01T09:41:29.764835+0000","last_fullsized":"2023-10-01T09:41:29.764835+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5395578880,"num_objects":82330,"num_object_clones":0,"num_object_copies":246990,"num_objects_missing_on_primary":3698,"num_objects_missing":3698,"num_objects_degraded":13954,"num_objects_misplaced":3293,"num_objects_unfound":0,"num_objects_dirty":82330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":82330,"num_write_kb":5269120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3600,"num_bytes_recovered":235929600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16621895680,"data_stored":16621895680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82330,"ondisk_log_size":82330,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430691,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2816288,"kb_used_data":1913176,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":90928864,"statfs":{"total":95995035648,"available":93111156736,"internally_reserved":0,"allocated":1959092224,"data_stored":1958160934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":2.6640000000000001,"15min":3.4079999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":2.718,"5min":81.162999999999997,"15min":108.90000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.95999999999999996,"5min":3.1440000000000001,"15min":3.8069999999999999},"min":{"1min":0.27900000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.3659999999999997,"5min":81.323999999999998,"15min":109.187},"last":0.39100000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:41:05 2023","interfaces":[{"interface":"back","average":{"1min":11.349,"5min":6.492,"15min":5.5579999999999998},"min":{"1min":0.248,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":229.161,"5min":229.161,"15min":229.161},"last":0.433},{"interface":"front","average":{"1min":11.241,"5min":6.6150000000000002,"15min":5.6059999999999999},"min":{"1min":0.309,"5min":0.27500000000000002,"15min":0.24399999999999999},"max":{"1min":230.126,"5min":230.126,"15min":230.126},"last":0.32400000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:40:44 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":2.367,"15min":2.081},"min":{"1min":0.30299999999999999,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":2.774,"5min":14.901999999999999,"15min":14.901999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.93700000000000006,"5min":2.3050000000000002,"15min":1.996},"min":{"1min":0.375,"5min":0.29699999999999999,"15min":0.247},"max":{"1min":2.907,"5min":14.682,"15min":14.682},"last":0.28699999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:41:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":2.6699999999999999,"15min":3.528},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28199999999999997},"max":{"1min":4.3120000000000003,"5min":35.104999999999997,"15min":164.595},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.3240000000000001,"5min":2.738,"15min":3.5},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.1440000000000001,"5min":35.118000000000002,"15min":164.60499999999999},"last":0.41599999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:40:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.1970000000000001,"15min":0.98399999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.23499999999999999},"max":{"1min":4.21,"5min":9.6020000000000003,"15min":9.6020000000000003},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.234,"5min":1.0960000000000001,"15min":0.96599999999999997},"min":{"1min":0.34699999999999998,"5min":0.28000000000000003,"15min":0.26400000000000001},"max":{"1min":8.4949999999999992,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.42299999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":8.516,"5min":2.984,"15min":1.679},"min":{"1min":0.311,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":135.40000000000001,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":8.1989999999999998,"5min":3.1360000000000001,"15min":1.71},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":135.339,"5min":135.339,"15min":135.339},"last":0.312}]},{"osd":6,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0699999999999998,"5min":1.498,"15min":1.1819999999999999},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":11.58,"5min":11.58,"15min":11.853999999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.988,"5min":1.3939999999999999,"15min":1.147},"min":{"1min":0.373,"5min":0.373,"15min":0.32300000000000001},"max":{"1min":13.021000000000001,"5min":14.584,"15min":14.584},"last":0.373}]}]},{"osd":6,"up_from":294,"seq":1262720385202,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2646296,"kb_used_data":2308688,"kb_used_omap":6,"kb_used_meta":337593,"kb_avail":91098856,"statfs":{"total":95995035648,"available":93285228544,"internally_reserved":0,"allocated":2364096512,"data_stored":2363173245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":345695277},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":7.4020000000000001,"5min":3.9239999999999999,"15min":4.2949999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":158.48599999999999,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":8.1300000000000008,"5min":4.5199999999999996,"15min":4.335},"min":{"1min":0.34399999999999997,"5min":0.25700000000000001,"15min":0.221},"max":{"1min":158.37700000000001,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.28699999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":12.536,"5min":7.2130000000000001,"15min":5.1319999999999997},"min":{"1min":0.38200000000000001,"5min":0.25800000000000001,"15min":0.23899999999999999},"max":{"1min":168.947,"5min":168.947,"15min":168.947},"last":0.41799999999999998},{"interface":"front","average":{"1min":12.667999999999999,"5min":7.1269999999999998,"15min":5.0389999999999997},"min":{"1min":0.32400000000000001,"5min":0.30599999999999999,"15min":0.24399999999999999},"max":{"1min":169.07900000000001,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.44500000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.919,"15min":1.9850000000000001},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":8.657,"5min":9.657,"15min":26.289999999999999},"last":0.373},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.944,"15min":2.0179999999999998},"min":{"1min":0.29399999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.247999999999999,"5min":21.704999999999998,"15min":26.388999999999999},"last":0.40500000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.903,"15min":2.5209999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":4.149,"5min":9.0180000000000007,"15min":89.753},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.7110000000000001,"5min":1.9870000000000001,"15min":2.2429999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":8.5470000000000006,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.45500000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":4.9210000000000003,"5min":1.5780000000000001,"15min":1.038},"min":{"1min":0.26300000000000001,"5min":0.246,"15min":0.246},"max":{"1min":63.134,"5min":63.134,"15min":63.134},"last":0.33600000000000002},{"interface":"front","average":{"1min":4.3339999999999996,"5min":1.3959999999999999,"15min":1.056},"min":{"1min":0.35299999999999998,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":62.973999999999997,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.52000000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":1.0089999999999999,"15min":1.0529999999999999},"min":{"1min":0.312,"5min":0.251,"15min":0.251},"max":{"1min":9.9939999999999998,"5min":9.9939999999999998,"15min":12.023999999999999},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.1880000000000002,"5min":1.234,"15min":1.1739999999999999},"min":{"1min":0.314,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":15.457000000000001,"5min":15.457000000000001,"15min":15.457000000000001},"last":0.47399999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":0.86099999999999999,"15min":2.1339999999999999},"min":{"1min":0.34100000000000003,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":3.492,"5min":6.4119999999999999,"15min":115.11},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.2310000000000001,"15min":2.2040000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28799999999999998},"max":{"1min":8.6899999999999995,"5min":8.6899999999999995,"15min":114.666},"last":0.501}]}]},{"osd":1,"up_from":94,"seq":403726926179,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1802400,"kb_used_data":996504,"kb_used_omap":7,"kb_used_meta":805880,"kb_avail":91942752,"statfs":{"total":95995035648,"available":94149378048,"internally_reserved":0,"allocated":1020420096,"data_stored":1018436786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":825221543},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:40:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":6.8710000000000004,"15min":4.7690000000000001},"min":{"1min":0.20699999999999999,"5min":0.182,"15min":0.182},"max":{"1min":1.772,"5min":154.21299999999999,"15min":154.21299999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.65800000000000003,"5min":5.3220000000000001,"15min":4.1139999999999999},"min":{"1min":0.24399999999999999,"5min":0.19500000000000001,"15min":0.188},"max":{"1min":2,"5min":95.236999999999995,"15min":95.236999999999995},"last":0.27700000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:40:36 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":2.891,"15min":3.7029999999999998},"min":{"1min":0.23000000000000001,"5min":0.22900000000000001,"15min":0.20000000000000001},"max":{"1min":1.98,"5min":22.891999999999999,"15min":56.573},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.52500000000000002,"5min":4.6989999999999998,"15min":4.4800000000000004},"min":{"1min":0.28399999999999997,"5min":0.254,"15min":0.20499999999999999},"max":{"1min":1.851,"5min":154.197,"15min":154.197},"last":0.35299999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:41:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":4.9089999999999998,"15min":4.8239999999999998},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":18.503,"5min":95.305000000000007,"15min":200.56800000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.4119999999999999,"5min":6.7089999999999996,"15min":5.54},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":16.515000000000001,"5min":154.483,"15min":200.63800000000001},"last":0.317}]},{"osd":4,"last update":"Sun Oct 1 09:40:51 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":4.4109999999999996,"15min":2.5230000000000001},"min":{"1min":0.314,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.5789999999999997,"5min":154.101,"15min":154.101},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.92800000000000005,"5min":4.3239999999999998,"15min":2.8679999999999999},"min":{"1min":0.40799999999999997,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":2.7290000000000001,"5min":154.56999999999999,"15min":154.56999999999999},"last":0.53300000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":2.4870000000000001,"15min":2.1509999999999998},"min":{"1min":0.32900000000000001,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":5.0410000000000004,"5min":21.219999999999999,"15min":154.22999999999999},"last":0.372},{"interface":"front","average":{"1min":0.83399999999999996,"5min":2.3239999999999998,"15min":2.661},"min":{"1min":0.311,"5min":0.307,"15min":0.307},"max":{"1min":5.1050000000000004,"5min":13.848000000000001,"15min":154.54400000000001},"last":0.56100000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:41:08 2023","interfaces":[{"interface":"back","average":{"1min":2.363,"5min":4.1890000000000001,"15min":2.9950000000000001},"min":{"1min":0.34300000000000003,"5min":0.32900000000000001,"15min":0.29899999999999999},"max":{"1min":23.672999999999998,"5min":154.12200000000001,"15min":154.12200000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.27,"5min":4.4729999999999999,"15min":2.6829999999999998},"min":{"1min":0.34599999999999997,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":23.382000000000001,"5min":154.16999999999999,"15min":154.16999999999999},"last":0.44900000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:40:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":4.5149999999999997,"15min":2.8319999999999999},"min":{"1min":0.45500000000000002,"5min":0.33900000000000002,"15min":0.316},"max":{"1min":9.2460000000000004,"5min":154.20699999999999,"15min":154.20699999999999},"last":0.433},{"interface":"front","average":{"1min":1.3149999999999999,"5min":4.5449999999999999,"15min":3.0430000000000001},"min":{"1min":0.44,"5min":0.43099999999999999,"15min":0.30199999999999999},"max":{"1min":5.0110000000000001,"5min":154.51599999999999,"15min":154.51599999999999},"last":0.497}]}]},{"osd":0,"up_from":89,"seq":382252089713,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4851436,"kb_used_data":3960292,"kb_used_omap":8,"kb_used_meta":891127,"kb_avail":88893716,"statfs":{"total":95995035648,"available":91027165184,"internally_reserved":0,"allocated":4055339008,"data_stored":4053350123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":912515010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:41:06 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":7.5830000000000002,"15min":4.54},"min":{"1min":0.187,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":8.4939999999999998,"5min":216.821,"15min":216.821},"last":0.24299999999999999},{"interface":"front","average":{"1min":1.095,"5min":9.0879999999999992,"15min":5.7130000000000001},"min":{"1min":0.22,"5min":0.215,"15min":0.21199999999999999},"max":{"1min":8.6229999999999993,"5min":217.02699999999999,"15min":217.02699999999999},"last":0.38}]},{"osd":2,"last update":"Sun Oct 1 09:40:33 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":2.8980000000000001,"15min":2.4340000000000002},"min":{"1min":0.27100000000000002,"5min":0.253,"15min":0.20100000000000001},"max":{"1min":3.9700000000000002,"5min":19.788,"15min":19.788},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.83299999999999996,"5min":2.915,"15min":3.0979999999999999},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.189},"max":{"1min":4.5359999999999996,"5min":25.844999999999999,"15min":25.844999999999999},"last":0.42399999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:41:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":5.4249999999999998,"15min":4.7450000000000001},"min":{"1min":0.24199999999999999,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":5.0869999999999997,"5min":222.16499999999999,"15min":222.16499999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.80200000000000005,"5min":5.2400000000000002,"15min":4.9770000000000003},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.219},"max":{"1min":5.8419999999999996,"5min":222.227,"15min":222.227},"last":0.36199999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:40:52 2023","interfaces":[{"interface":"back","average":{"1min":4.2089999999999996,"5min":2.7109999999999999,"15min":2.5169999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":48.116999999999997,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.46000000000000002},{"interface":"front","average":{"1min":3.8650000000000002,"5min":3.0670000000000002,"15min":2.609},"min":{"1min":0.36299999999999999,"5min":0.33700000000000002,"15min":0.28299999999999997},"max":{"1min":47.854999999999997,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.48099999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.494,"5min":2.371,"15min":2.0569999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":5.8769999999999998,"5min":14.515000000000001,"15min":16.808},"last":0.437},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.2879999999999998,"15min":1.9259999999999999},"min":{"1min":0.49099999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.8689999999999998,"5min":17.507000000000001,"15min":17.507000000000001},"last":0.39800000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:41:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":2.2509999999999999,"15min":1.897},"min":{"1min":0.38600000000000001,"5min":0.35699999999999998,"15min":0.35499999999999998},"max":{"1min":5.2110000000000003,"5min":17.535,"15min":17.535},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.649,"5min":2.1739999999999999,"15min":2.2679999999999998},"min":{"1min":0.44800000000000001,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":4.2640000000000002,"5min":10.446,"15min":18.521000000000001},"last":0.53000000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:40:52 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":2.7149999999999999,"15min":2.1619999999999999},"min":{"1min":0.45100000000000001,"5min":0.377,"15min":0.34499999999999997},"max":{"1min":3.097,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.88600000000000001,"5min":2.742,"15min":2.6269999999999998},"min":{"1min":0.371,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":2.4199999999999999,"5min":48.274000000000001,"15min":48.274000000000001},"last":0.48999999999999999}]}]},{"osd":2,"up_from":0,"seq":425201762653,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599126,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2493408,"kb_used_data":1691992,"kb_used_omap":3,"kb_used_meta":801404,"kb_avail":91251744,"statfs":{"total":95995035648,"available":93441785856,"internally_reserved":0,"allocated":1732599808,"data_stored":1730558373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":820638439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":5.0030000000000001,"15min":5.9409999999999998},"min":{"1min":0.183,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":12.999000000000001,"5min":118.748,"15min":137.34399999999999},"last":0.314},{"interface":"front","average":{"1min":0.97699999999999998,"5min":5.3849999999999998,"15min":5.8490000000000002},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.184},"max":{"1min":10.377000000000001,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.246}]},{"osd":1,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":8.3849999999999998,"15min":5.7290000000000001},"min":{"1min":0.252,"5min":0.216,"15min":0.20399999999999999},"max":{"1min":14.917999999999999,"5min":153.77000000000001,"15min":162.22999999999999},"last":0.214},{"interface":"front","average":{"1min":1.359,"5min":8.468,"15min":6.2320000000000002},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":15.074,"5min":151.02199999999999,"15min":162.70699999999999},"last":0.39300000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":3.0110000000000001,"15min":2.8839999999999999},"min":{"1min":0.26000000000000001,"5min":0.22800000000000001,"15min":0.20799999999999999},"max":{"1min":2.7970000000000002,"5min":14.254,"15min":43.222999999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.7070000000000001,"15min":3.206},"min":{"1min":0.247,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":1.4239999999999999,"5min":15.523,"15min":78.594999999999999},"last":0.45000000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":4.617,"5min":2.5249999999999999,"15min":2.1760000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":77.793000000000006,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.52300000000000002},{"interface":"front","average":{"1min":4.8399999999999999,"5min":2.9620000000000002,"15min":2.306},"min":{"1min":0.378,"5min":0.27900000000000003,"15min":0.23899999999999999},"max":{"1min":76.646000000000001,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.40500000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":2.1869999999999998,"15min":3.012},"min":{"1min":0.29399999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.0659999999999998,"5min":10.919,"15min":28.117999999999999},"last":0.379},{"interface":"front","average":{"1min":1.768,"5min":2.6379999999999999,"15min":3.1000000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.1559999999999997,"5min":14.537000000000001,"15min":28.34},"last":0.57599999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:41:19 2023","interfaces":[{"interface":"back","average":{"1min":5.2430000000000003,"5min":3.4009999999999998,"15min":2.4870000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":73.887,"5min":73.887,"15min":73.887},"last":0.53900000000000003},{"interface":"front","average":{"1min":5.8470000000000004,"5min":3.5840000000000001,"15min":2.4780000000000002},"min":{"1min":0.44700000000000001,"5min":0.38600000000000001,"15min":0.32800000000000001},"max":{"1min":74.016999999999996,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.63400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":2.8690000000000002,"15min":2.6429999999999998},"min":{"1min":0.46400000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.5570000000000004,"5min":13.085000000000001,"15min":52.006},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.228,"5min":2.0640000000000001,"15min":2.3359999999999999},"min":{"1min":0.371,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":4.5810000000000004,"5min":11.109999999999999,"15min":51.890999999999998},"last":0.55900000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019377,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5056216,"kb_used_data":4157840,"kb_used_omap":0,"kb_used_meta":898368,"kb_avail":88688936,"statfs":{"total":95995035648,"available":90817470464,"internally_reserved":0,"allocated":4257628160,"data_stored":4256704448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919928832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:40:50 2023","interfaces":[{"interface":"back","average":{"1min":4.6970000000000001,"5min":3.8300000000000001,"15min":5.2060000000000004},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":79.837000000000003,"5min":79.837000000000003,"15min":155.315},"last":0.47499999999999998},{"interface":"front","average":{"1min":5.0030000000000001,"5min":4.2080000000000002,"15min":5.3689999999999998},"min":{"1min":0.28399999999999997,"5min":0.251,"15min":0.23899999999999999},"max":{"1min":78.983000000000004,"5min":78.983000000000004,"15min":155.334},"last":0.33900000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:40:54 2023","interfaces":[{"interface":"back","average":{"1min":4.5149999999999997,"5min":10.486000000000001,"15min":7.7160000000000002},"min":{"1min":0.32200000000000001,"5min":0.224,"15min":0.224},"max":{"1min":77.953000000000003,"5min":165.666,"15min":190.61199999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":4.6669999999999998,"5min":10.384,"15min":8.5039999999999996},"min":{"1min":0.26700000000000002,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":77.962999999999994,"5min":165.70500000000001,"15min":253.273},"last":0.438}]},{"osd":2,"last update":"Sun Oct 1 09:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.9710000000000001,"15min":2.8220000000000001},"min":{"1min":0.31,"5min":0.29099999999999998,"15min":0.26700000000000002},"max":{"1min":1.8029999999999999,"5min":10.161,"15min":170.45400000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":2.323,"15min":2.4950000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":1.8680000000000001,"5min":10.625,"15min":58.771000000000001},"last":0.51900000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:40:54 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":3.6360000000000001,"15min":3.419},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.2719999999999998,"5min":140.881,"15min":140.881},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.84599999999999997,"5min":3.8860000000000001,"15min":3.633},"min":{"1min":0.32800000000000001,"5min":0.311,"15min":0.311},"max":{"1min":2.8250000000000002,"5min":140.911,"15min":140.911},"last":0.46600000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:41:30 2023","interfaces":[{"interface":"back","average":{"1min":7.3849999999999998,"5min":2.387,"15min":1.6259999999999999},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":110.003,"5min":110.003,"15min":110.003},"last":0.49099999999999999},{"interface":"front","average":{"1min":6.8159999999999998,"5min":2.294,"15min":1.425},"min":{"1min":0.29899999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":108.56399999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.501}]},{"osd":6,"last update":"Sun Oct 1 09:41:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":0.97999999999999998,"15min":0.95399999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":5.5709999999999997,"5min":7.7649999999999997,"15min":11.358000000000001},"last":0.378},{"interface":"front","average":{"1min":1.381,"5min":1.22,"15min":1.0569999999999999},"min":{"1min":0.317,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":6.1660000000000004,"5min":6.9139999999999997,"15min":10.076000000000001},"last":0.54600000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:40:39 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":1.341,"15min":1.135},"min":{"1min":0.32000000000000001,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":2.355,"5min":38.034999999999997,"15min":38.034999999999997},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.7090000000000001,"15min":1.3460000000000001},"min":{"1min":0.372,"5min":0.34200000000000003,"15min":0.28699999999999998},"max":{"1min":1.8939999999999999,"5min":37.723999999999997,"15min":37.723999999999997},"last":0.57599999999999996}]}]},{"osd":5,"up_from":379,"seq":1627792605284,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2484088,"kb_used_data":2119536,"kb_used_omap":13,"kb_used_meta":364530,"kb_avail":91261064,"statfs":{"total":95995035648,"available":93451329536,"internally_reserved":0,"allocated":2170404864,"data_stored":2169493566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":373279218},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,16],"upper_bound":32768},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":2.508,"15min":2.335},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.6379999999999999,"5min":13.385999999999999,"15min":84.340999999999994},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.0030000000000001,"15min":2.1909999999999998},"min":{"1min":0.28299999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.2279999999999998,"5min":12.445,"15min":84.356999999999999},"last":0.33000000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":8.1319999999999997,"5min":9.8100000000000005,"15min":5.9340000000000002},"min":{"1min":0.29299999999999998,"5min":0.255,"15min":0.255},"max":{"1min":106.56999999999999,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.35299999999999998},{"interface":"front","average":{"1min":8.1590000000000007,"5min":10.111000000000001,"15min":5.8540000000000001},"min":{"1min":0.27500000000000002,"5min":0.25900000000000001,"15min":0.24199999999999999},"max":{"1min":106.596,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.50700000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.653,"5min":2.1789999999999998,"15min":1.71},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.3470000000000004,"5min":14.199999999999999,"15min":14.199999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":2.0379999999999998,"5min":2.0670000000000002,"15min":1.7370000000000001},"min":{"1min":0.373,"5min":0.31,"15min":0.27100000000000002},"max":{"1min":5.782,"5min":14.430999999999999,"15min":14.430999999999999},"last":0.39000000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":2.2730000000000001,"15min":1.5700000000000001},"min":{"1min":0.35399999999999998,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":3.4409999999999998,"5min":14.004,"15min":14.004},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.782,"5min":2.3610000000000002,"15min":1.7150000000000001},"min":{"1min":0.35299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.4400000000000004,"5min":13.991,"15min":13.991},"last":0.40400000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1320000000000001,"5min":2.3300000000000001,"15min":1.5},"min":{"1min":0.36699999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.9459999999999997,"5min":122.971,"15min":122.971},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.6520000000000001,"5min":2.3849999999999998,"15min":1.8859999999999999},"min":{"1min":0.32100000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.445,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.46000000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":1.335,"15min":1.1160000000000001},"min":{"1min":0.33900000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.7750000000000004,"5min":12.545,"15min":12.545},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.8560000000000001,"5min":1.0529999999999999,"15min":1.431},"min":{"1min":0.36899999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.4160000000000004,"5min":6.4729999999999999,"15min":14.193},"last":0.44}]},{"osd":7,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5459999999999998,"5min":1.238,"15min":1.379},"min":{"1min":0.46800000000000003,"5min":0.307,"15min":0.307},"max":{"1min":5.758,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.2210000000000001,"5min":1.3740000000000001,"15min":1.121},"min":{"1min":0.41199999999999998,"5min":0.318,"15min":0.318},"max":{"1min":5.2300000000000004,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.53800000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4044619776,"data_stored":4044619776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1010302976,"data_stored":1010302976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1352204288,"data_stored":1352204288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4250927104,"data_stored":4250927104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1918959616,"data_stored":1918959616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2358771712,"data_stored":2358771712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1686110208,"data_stored":1686110208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:41:40.346 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:41:40.347 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:40.381 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:40.629 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 89 16 82960 82944 58.2293 16.375 0.00411901 0.0170932 2023-10-01T09:41:40.848 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:40.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:40.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:40.892 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:40.953 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:41:40.985 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:41.044 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:41:41.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:41.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:41:41.148 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:41.191 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:41.248 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:41.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:41.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:41.344 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:41:41.353 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:41.389 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:41.462 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:41.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:41.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:41.569 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:41 smithi138 ceph-mon[192513]: pgmap v1436: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 2.8 MiB/s wr, 44 op/s; 13964/247506 objects degraded (5.642%); 2688/247506 objects misplaced (1.086%); 11 MiB/s, 171 objects/s recovering 2023-10-01T09:41:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:41:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:41 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1344522154' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:41 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:41.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:41 smithi138 ceph-mon[185886]: pgmap v1436: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 2.8 MiB/s wr, 44 op/s; 13964/247506 objects degraded (5.642%); 2688/247506 objects misplaced (1.086%); 11 MiB/s, 171 objects/s recovering 2023-10-01T09:41:41.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:41.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:41:41.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:41 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1344522154' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:41.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:41 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:41.607 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:41.629 INFO:tasks.radosbench.radosbench.0.smithi138.stdout: 90 14 83219 83205 57.7636 16.3125 1.00163 0.0172235 2023-10-01T09:41:41.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:41 smithi190 ceph-mon[160435]: pgmap v1436: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 2.8 MiB/s wr, 44 op/s; 13964/247506 objects degraded (5.642%); 2688/247506 objects misplaced (1.086%); 11 MiB/s, 171 objects/s recovering 2023-10-01T09:41:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:41:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:41 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1344522154' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:41 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:41:41.810 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:41.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:41.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:41.914 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:41.944 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:42.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:41:42.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[185886]: failed to encode map e507 with expected crc 2023-10-01T09:41:42.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[185886]: failed to encode map e507 with expected crc 2023-10-01T09:41:42.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:41:42.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[185886]: osdmap e507: 8 total, 8 up, 7 in 2023-10-01T09:41:42.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[185886]: failed to encode map e507 with expected crc 2023-10-01T09:41:42.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[185886]: failed to encode map e507 with expected crc 2023-10-01T09:41:42.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:41:42.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:41:42.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[192513]: failed to encode map e507 with expected crc 2023-10-01T09:41:42.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[192513]: failed to encode map e507 with expected crc 2023-10-01T09:41:42.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:41:42.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[192513]: osdmap e507: 8 total, 8 up, 7 in 2023-10-01T09:41:42.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[192513]: failed to encode map e507 with expected crc 2023-10-01T09:41:42.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[192513]: failed to encode map e507 with expected crc 2023-10-01T09:41:42.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:41:42.630 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Total time run: 90.8666 2023-10-01T09:41:42.630 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Total writes made: 83219 2023-10-01T09:41:42.630 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Write size: 65536 2023-10-01T09:41:42.630 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Object size: 65536 2023-10-01T09:41:42.630 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Bandwidth (MB/sec): 57.2398 2023-10-01T09:41:42.630 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev Bandwidth: 41.7541 2023-10-01T09:41:42.631 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max bandwidth (MB/sec): 177.5 2023-10-01T09:41:42.631 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min bandwidth (MB/sec): 0 2023-10-01T09:41:42.631 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Average IOPS: 915 2023-10-01T09:41:42.631 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev IOPS: 668.072 2023-10-01T09:41:42.631 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max IOPS: 2840 2023-10-01T09:41:42.631 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min IOPS: 0 2023-10-01T09:41:42.631 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Average Latency(s): 0.0173862 2023-10-01T09:41:42.631 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Stddev Latency(s): 0.219239 2023-10-01T09:41:42.631 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Max latency(s): 16.0782 2023-10-01T09:41:42.631 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Min latency(s): 0.00101103 2023-10-01T09:41:42.634 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T09:41:42.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:42 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:41:42.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:42 smithi190 ceph-mon[160435]: failed to encode map e507 with expected crc 2023-10-01T09:41:42.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:42 smithi190 ceph-mon[160435]: failed to encode map e507 with expected crc 2023-10-01T09:41:42.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:42 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:41:42.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:42 smithi190 ceph-mon[160435]: osdmap e507: 8 total, 8 up, 7 in 2023-10-01T09:41:42.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:42 smithi190 ceph-mon[160435]: failed to encode map e507 with expected crc 2023-10-01T09:41:42.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:42 smithi190 ceph-mon[160435]: failed to encode map e507 with expected crc 2023-10-01T09:41:42.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:41:42.847 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:42.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:42.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:42.952 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:42.986 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:43.139 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:43.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:43.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:43.244 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:41:43.302 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:43.349 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:41:43.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:43.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:41:43.453 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:43.480 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:43.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:43 smithi190 ceph-mon[160435]: pgmap v1438: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 85 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5.6 MiB/s wr, 90 op/s; 13964/248877 objects degraded (5.611%); 2688/248877 objects misplaced (1.080%); 5.7 MiB/s, 91 objects/s recovering 2023-10-01T09:41:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:43 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 13964/248877 objects degraded (5.611%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:43.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:43 smithi138 ceph-mon[185886]: pgmap v1438: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 85 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5.6 MiB/s wr, 90 op/s; 13964/248877 objects degraded (5.611%); 2688/248877 objects misplaced (1.080%); 5.7 MiB/s, 91 objects/s recovering 2023-10-01T09:41:43.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:43 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 13964/248877 objects degraded (5.611%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:43.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:43 smithi138 ceph-mon[192513]: pgmap v1438: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 85 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5.6 MiB/s wr, 90 op/s; 13964/248877 objects degraded (5.611%); 2688/248877 objects misplaced (1.080%); 5.7 MiB/s, 91 objects/s recovering 2023-10-01T09:41:43.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:43 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 13964/248877 objects degraded (5.611%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:43.580 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:43.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:43.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:43.686 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:43.719 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:43.770 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:43.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:43.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:43.877 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:43.916 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:43.979 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:43.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:43.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:44.083 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:44.111 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:44.324 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:44.346 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:44.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:44.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:44.451 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:44.482 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:44.536 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:44.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:44.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:44.643 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:44.678 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:44.927 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:44.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:44.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:44.945 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:41:44.965 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:41:44.988 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:41:45.001 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:41:45.001 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:41:45.001 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-10-01T09:41:45.001 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:41:45.015 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:41:45.032 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:45.036 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:41:45.060 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:45.060 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:41:45.088 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:41:45.109 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:45.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:45.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:45.115 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:41:45.116 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:41:45.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:41:45.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:41:45.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:41:45.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:41:45.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:41:45.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:41:45.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:41:45.215 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:45.243 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:45.320 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:45 smithi190 ceph-mon[160435]: pgmap v1439: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 6.4 MiB/s wr, 101 op/s; 13964/249225 objects degraded (5.603%); 2147/249225 objects misplaced (0.861%); 7.2 MiB/s, 114 objects/s recovering 2023-10-01T09:41:45.320 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:45 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4215908144' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:45.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:41:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:41:44] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-01T09:41:45.346 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:45.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:45.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:45.454 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:41:45.485 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:45.590 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:41:45.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:45.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:41:45.693 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:41:45.723 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:45.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:45 smithi138 ceph-mon[185886]: pgmap v1439: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 6.4 MiB/s wr, 101 op/s; 13964/249225 objects degraded (5.603%); 2147/249225 objects misplaced (0.861%); 7.2 MiB/s, 114 objects/s recovering 2023-10-01T09:41:45.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:45 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4215908144' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:45.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:45 smithi138 ceph-mon[192513]: pgmap v1439: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 6.4 MiB/s wr, 101 op/s; 13964/249225 objects degraded (5.603%); 2147/249225 objects misplaced (0.861%); 7.2 MiB/s, 114 objects/s recovering 2023-10-01T09:41:45.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:45 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4215908144' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:45.877 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:41:45.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:45.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:41:45.983 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:41:46.013 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:46.061 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:41:46.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:46.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:41:46.167 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:46.195 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:46.250 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:46.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:46.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:46.356 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:46.361 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:41:46.390 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:47.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:46 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:47.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:46 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:47.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:46 smithi190 ceph-mon[160435]: pgmap v1440: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5.4 MiB/s wr, 103 op/s; 13936/249195 objects degraded (5.592%); 2147/249195 objects misplaced (0.862%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:41:47.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:46 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:47.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:46 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:47.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:46 smithi138 ceph-mon[192513]: pgmap v1440: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5.4 MiB/s wr, 103 op/s; 13936/249195 objects degraded (5.592%); 2147/249195 objects misplaced (0.862%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:41:47.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:46 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:47.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:46 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:47.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:46 smithi138 ceph-mon[185886]: pgmap v1440: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5.4 MiB/s wr, 103 op/s; 13936/249195 objects degraded (5.592%); 2147/249195 objects misplaced (0.862%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:41:47.979 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:41:48.110 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:48.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:48.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:48.214 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:48.243 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:48.291 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:48.296 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:47 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 13936/249195 objects degraded (5.592%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:48.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:48.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:48.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:48 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 13936/249195 objects degraded (5.592%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:48.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:47 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 13936/249195 objects degraded (5.592%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:48.397 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:48.434 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:48.533 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:48.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:48.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:48.638 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:41:48.667 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:48.716 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:41:48.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:48.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:41:48.822 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:48.856 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:48.887 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:48.918 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:48.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:48.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:48.937 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:49.023 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:49.073 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:49.134 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:49.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:49.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:49.238 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:49.279 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:49 smithi138 ceph-mon[192513]: pgmap v1441: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 5.6 MiB/s wr, 133 op/s; 13936/248397 objects degraded (5.610%); 1461/248397 objects misplaced (0.588%); 7.7 MiB/s, 123 objects/s recovering 2023-10-01T09:41:49.280 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:49 smithi138 ceph-mon[185886]: pgmap v1441: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 5.6 MiB/s wr, 133 op/s; 13936/248397 objects degraded (5.610%); 1461/248397 objects misplaced (0.588%); 7.7 MiB/s, 123 objects/s recovering 2023-10-01T09:41:49.282 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:49.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:49 smithi190 ceph-mon[160435]: pgmap v1441: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 5.6 MiB/s wr, 133 op/s; 13936/248397 objects degraded (5.610%); 1461/248397 objects misplaced (0.588%); 7.7 MiB/s, 123 objects/s recovering 2023-10-01T09:41:49.335 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:41:49.335 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 13936/248397 objects degraded (5.610%), 3 pgs degraded","count":3},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1930,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":507,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":85},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":82799,"data_bytes":5429204939,"bytes_used":22683406336,"bytes_avail":649281843200,"bytes_total":671965249536,"degraded_objects":13936,"degraded_total":248397,"degraded_ratio":0.056103737162687149,"misplaced_objects":1461,"misplaced_total":248397,"misplaced_ratio":0.0058817135472650642,"recovering_objects_per_sec":123,"recovering_bytes_per_sec":8085183,"recovering_keys_per_sec":0,"num_objects_recovered":1234,"num_bytes_recovered":80871424,"num_keys_recovered":0,"read_bytes_sec":102,"write_bytes_sec":5824841,"read_op_per_sec":0,"write_op_per_sec":133},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":32,"modified":"2023-10-01T09:40:14.257304+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (50s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:41:49.980 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:41:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:50 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2781009753' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:41:50.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:50 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2781009753' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:41:50.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:50 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2781009753' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:41:50.905 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:50.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:50.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:51.010 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:51.041 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:51.085 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:51.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:51.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:51.190 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:51.217 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:51 smithi138 ceph-mon[192513]: pgmap v1442: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 4.7 MiB/s wr, 137 op/s; 13867/247872 objects degraded (5.594%); 1461/247872 objects misplaced (0.589%); 7.9 MiB/s, 126 objects/s recovering 2023-10-01T09:41:51.217 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:51 smithi138 ceph-mon[185886]: pgmap v1442: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 4.7 MiB/s wr, 137 op/s; 13867/247872 objects degraded (5.594%); 1461/247872 objects misplaced (0.589%); 7.9 MiB/s, 126 objects/s recovering 2023-10-01T09:41:51.221 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:51.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:51 smithi190 ceph-mon[160435]: pgmap v1442: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 4.7 MiB/s wr, 137 op/s; 13867/247872 objects degraded (5.594%); 1461/247872 objects misplaced (0.589%); 7.9 MiB/s, 126 objects/s recovering 2023-10-01T09:41:52.032 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:52.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:52.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:52.086 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:41:52.086 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:41:52.139 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:41:52.166 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:52.211 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:41:52.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:52.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:41:52.231 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:52.316 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:41:52.343 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:52.388 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:41:52.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:52.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:41:52.493 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:41:52.529 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:52.578 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:41:52.578 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:41:52.578 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:41:52.578 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:41:52.578 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:41:53.448 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:41:53.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:53.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:41:53.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:53 smithi138 ceph-mon[192513]: pgmap v1443: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 93 B/s rd, 3.0 MiB/s wr, 137 op/s; 13867/246831 objects degraded (5.618%); 1461/246831 objects misplaced (0.592%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:41:53.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:53 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 13867/246831 objects degraded (5.618%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:53.452 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/330356852' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:41:53.453 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:53 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:41:53.453 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:53 smithi138 ceph-mon[185886]: pgmap v1443: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 93 B/s rd, 3.0 MiB/s wr, 137 op/s; 13867/246831 objects degraded (5.618%); 1461/246831 objects misplaced (0.592%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:41:53.453 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:53 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 13867/246831 objects degraded (5.618%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:53.453 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/330356852' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:41:53.453 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:53 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:41:53.553 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:53.586 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:53.758 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:41:53.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:53 smithi190 ceph-mon[160435]: pgmap v1443: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 93 B/s rd, 3.0 MiB/s wr, 137 op/s; 13867/246831 objects degraded (5.618%); 1461/246831 objects misplaced (0.592%); 7.2 MiB/s, 115 objects/s recovering 2023-10-01T09:41:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:53 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 13867/246831 objects degraded (5.618%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/330356852' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:41:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:53 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:41:54.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:54 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:41:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:54 smithi190 ceph-mon[160435]: failed to encode map e508 with expected crc 2023-10-01T09:41:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:54 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:41:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:54 smithi190 ceph-mon[160435]: osdmap e508: 8 total, 8 up, 7 in 2023-10-01T09:41:54.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:54 smithi190 ceph-mon[160435]: failed to encode map e508 with expected crc 2023-10-01T09:41:54.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:54 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:41:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:54 smithi138 ceph-mon[192513]: failed to encode map e508 with expected crc 2023-10-01T09:41:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:54 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:41:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:54 smithi138 ceph-mon[192513]: osdmap e508: 8 total, 8 up, 7 in 2023-10-01T09:41:54.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:54 smithi138 ceph-mon[192513]: failed to encode map e508 with expected crc 2023-10-01T09:41:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:54 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:41:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:54 smithi138 ceph-mon[185886]: failed to encode map e508 with expected crc 2023-10-01T09:41:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:54 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:41:54.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:54 smithi138 ceph-mon[185886]: osdmap e508: 8 total, 8 up, 7 in 2023-10-01T09:41:54.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:54 smithi138 ceph-mon[185886]: failed to encode map e508 with expected crc 2023-10-01T09:41:54.938 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:54.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:54.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:41:55.020 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:41:55.024 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:55.042 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:41:55.069 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:55.117 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:41:55.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:55.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:41:55.178 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:41:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:41:54] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-01T09:41:55.226 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:55.262 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:55.515 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:55.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:55 smithi138 ceph-mon[185886]: failed to encode map e508 with expected crc 2023-10-01T09:41:55.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:55 smithi138 ceph-mon[185886]: pgmap v1445: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 1.1 MiB/s wr, 121 op/s; 13867/246630 objects degraded (5.623%); 544/246630 objects misplaced (0.221%); 10 MiB/s, 164 objects/s recovering 2023-10-01T09:41:55.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:41:55.517 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:55 smithi138 ceph-mon[192513]: failed to encode map e508 with expected crc 2023-10-01T09:41:55.517 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:55 smithi138 ceph-mon[192513]: pgmap v1445: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 1.1 MiB/s wr, 121 op/s; 13867/246630 objects degraded (5.623%); 544/246630 objects misplaced (0.221%); 10 MiB/s, 164 objects/s recovering 2023-10-01T09:41:55.518 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:41:55.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:55.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:55.570 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:41:55.594 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:41:55.617 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:41:55.620 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:55.638 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:41:55.645 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:55 smithi190 ceph-mon[160435]: failed to encode map e508 with expected crc 2023-10-01T09:41:55.645 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:55 smithi190 ceph-mon[160435]: pgmap v1445: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 1.1 MiB/s wr, 121 op/s; 13867/246630 objects degraded (5.623%); 544/246630 objects misplaced (0.221%); 10 MiB/s, 164 objects/s recovering 2023-10-01T09:41:55.645 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:41:55.647 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:55.657 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:41:55.675 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:41:55.692 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:55.693 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:41:55.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:55.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:55.716 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:41:55.718 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:41:55.718 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:41:55.718 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:41:55.718 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:41:55.718 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:41:55.718 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:41:55.718 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:41:55.718 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:41:55.798 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:55.831 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:56.431 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:56.433 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:56 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3773198313' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:56.433 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:56 smithi138 ceph-mon[192513]: failed to encode map e509 with expected crc 2023-10-01T09:41:56.433 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:56 smithi138 ceph-mon[192513]: osdmap e509: 8 total, 8 up, 7 in 2023-10-01T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:56 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3773198313' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:56 smithi138 ceph-mon[185886]: failed to encode map e509 with expected crc 2023-10-01T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:56 smithi138 ceph-mon[185886]: osdmap e509: 8 total, 8 up, 7 in 2023-10-01T09:41:56.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:56.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:56.536 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:56.565 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:56.714 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:56.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:56.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:56 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3773198313' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:41:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:56 smithi190 ceph-mon[160435]: failed to encode map e509 with expected crc 2023-10-01T09:41:56.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:56 smithi190 ceph-mon[160435]: osdmap e509: 8 total, 8 up, 7 in 2023-10-01T09:41:56.818 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:41:56.845 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:56.893 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:41:56.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:56.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:41:56.956 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:41:57.000 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:41:57.026 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:57.074 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:41:57.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:57.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:41:57.177 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:41:57.205 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:57.250 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:41:57.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:57.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:41:57.355 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:41:57.382 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:57.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:57 smithi190 ceph-mon[160435]: failed to encode map e509 with expected crc 2023-10-01T09:41:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:57 smithi190 ceph-mon[160435]: failed to encode map e509 with expected crc 2023-10-01T09:41:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:57 smithi190 ceph-mon[160435]: pgmap v1447: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 114 op/s; 8782/245646 objects degraded (3.575%); 5556/245646 objects misplaced (2.262%); 7.7 MiB/s, 123 objects/s recovering 2023-10-01T09:41:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:57 smithi190 ceph-mon[160435]: osdmap e510: 8 total, 8 up, 7 in 2023-10-01T09:41:57.572 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:57.595 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:41:57.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:57.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:41:57.600 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:57 smithi138 ceph-mon[192513]: failed to encode map e509 with expected crc 2023-10-01T09:41:57.600 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:57 smithi138 ceph-mon[192513]: failed to encode map e509 with expected crc 2023-10-01T09:41:57.600 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:57 smithi138 ceph-mon[192513]: pgmap v1447: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 114 op/s; 8782/245646 objects degraded (3.575%); 5556/245646 objects misplaced (2.262%); 7.7 MiB/s, 123 objects/s recovering 2023-10-01T09:41:57.600 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:57 smithi138 ceph-mon[192513]: osdmap e510: 8 total, 8 up, 7 in 2023-10-01T09:41:57.600 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:57 smithi138 ceph-mon[185886]: failed to encode map e509 with expected crc 2023-10-01T09:41:57.600 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:57 smithi138 ceph-mon[185886]: failed to encode map e509 with expected crc 2023-10-01T09:41:57.600 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:57 smithi138 ceph-mon[185886]: pgmap v1447: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 114 op/s; 8782/245646 objects degraded (3.575%); 5556/245646 objects misplaced (2.262%); 7.7 MiB/s, 123 objects/s recovering 2023-10-01T09:41:57.600 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:57 smithi138 ceph-mon[185886]: osdmap e510: 8 total, 8 up, 7 in 2023-10-01T09:41:57.700 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:57.732 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:57.790 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:57.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:57.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:57.898 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:41:57.924 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:57.932 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:41:57.968 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:41:57.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:57.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:41:58.080 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:41:58.106 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:58.152 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:41:58.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:58.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:41:58.258 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:41:58.298 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:41:58.496 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:41:58.497 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:41:58.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:58 smithi190 ceph-mon[160435]: failed to encode map e510 with expected crc 2023-10-01T09:41:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:58 smithi190 ceph-mon[160435]: failed to encode map e510 with expected crc 2023-10-01T09:41:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:58 smithi190 ceph-mon[160435]: failed to encode map e510 with expected crc 2023-10-01T09:41:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:58 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8782/245646 objects degraded (3.575%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:58.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:58.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:58 smithi138 ceph-mon[192513]: failed to encode map e510 with expected crc 2023-10-01T09:41:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:58 smithi138 ceph-mon[192513]: failed to encode map e510 with expected crc 2023-10-01T09:41:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:58 smithi138 ceph-mon[192513]: failed to encode map e510 with expected crc 2023-10-01T09:41:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:58 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8782/245646 objects degraded (3.575%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:58.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:58 smithi138 ceph-mon[185886]: failed to encode map e510 with expected crc 2023-10-01T09:41:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:58 smithi138 ceph-mon[185886]: failed to encode map e510 with expected crc 2023-10-01T09:41:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:58 smithi138 ceph-mon[185886]: failed to encode map e510 with expected crc 2023-10-01T09:41:58.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:58 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8782/245646 objects degraded (3.575%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:41:58.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:58.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:41:58.680 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:41:59.345 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:41:59 smithi138 ceph-mon[185886]: pgmap v1449: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 8782/244656 objects degraded (3.590%); 5556/244656 objects misplaced (2.271%); 10 MiB/s, 159 objects/s recovering 2023-10-01T09:41:59.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:41:59 smithi138 ceph-mon[192513]: pgmap v1449: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 8782/244656 objects degraded (3.590%); 5556/244656 objects misplaced (2.271%); 10 MiB/s, 159 objects/s recovering 2023-10-01T09:41:59.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:41:59 smithi190 ceph-mon[160435]: pgmap v1449: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 85 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 8782/244656 objects degraded (3.590%); 5556/244656 objects misplaced (2.271%); 10 MiB/s, 159 objects/s recovering 2023-10-01T09:41:59.932 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:41:59.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:41:59.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:00.037 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:00.065 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:00.144 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1449,"stamp":"2023-10-01T09:41:58.284482+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5347481547,"num_objects":81552,"num_object_clones":0,"num_object_copies":244656,"num_objects_missing_on_primary":3608,"num_objects_missing":3608,"num_objects_degraded":8782,"num_objects_misplaced":5556,"num_objects_unfound":0,"num_objects_dirty":81552,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":86022,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6940,"num_bytes_recovered":441680911,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86086,"ondisk_log_size":86086,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":21877440,"kb_used_data":16840904,"kb_used_omap":39,"kb_used_meta":5036440,"kb_avail":634338624,"statfs":{"total":671965249536,"available":649562750976,"internally_reserved":0,"allocated":17245085696,"data_stored":17235341098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5157315183},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,0,0,1,0,1,1,1,4,7,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-47513600,"num_objects":-725,"num_object_clones":0,"num_object_copies":-2175,"num_objects_missing_on_primary":-34,"num_objects_missing":-34,"num_objects_degraded":-5085,"num_objects_misplaced":4095,"num_objects_unfound":0,"num_objects_dirty":-725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":725,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":956,"num_bytes_recovered":62652416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001491"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":285,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.754455+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:41:53.754455+0000","last_peered":"2023-10-01T09:41:53.754455+0000","last_clean":"2023-10-01T09:41:53.754455+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:41:53.754455+0000","last_undegraded":"2023-10-01T09:41:53.754455+0000","last_fullsized":"2023-10-01T09:41:53.754455+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":126,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.359425+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:41:41.359425+0000","last_peered":"2023-10-01T09:41:41.359425+0000","last_clean":"2023-10-01T09:41:41.359425+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:41:41.359425+0000","last_undegraded":"2023-10-01T09:41:41.359425+0000","last_fullsized":"2023-10-01T09:41:41.359425+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":507,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.359569+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:41:41.359569+0000","last_peered":"2023-10-01T09:41:41.359569+0000","last_clean":"2023-10-01T09:41:41.359569+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:41:41.359569+0000","last_undegraded":"2023-10-01T09:41:41.359569+0000","last_fullsized":"2023-10-01T09:41:41.359569+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":567,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.763156+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:41:55.763156+0000","last_peered":"2023-10-01T09:41:55.763156+0000","last_clean":"2023-10-01T09:41:55.763156+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:41:55.763156+0000","last_undegraded":"2023-10-01T09:41:55.763156+0000","last_fullsized":"2023-10-01T09:41:55.763156+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":448,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.762943+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:41:55.762943+0000","last_peered":"2023-10-01T09:41:55.762943+0000","last_clean":"2023-10-01T09:41:55.762943+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:41:55.762943+0000","last_undegraded":"2023-10-01T09:41:55.762943+0000","last_fullsized":"2023-10-01T09:41:55.762943+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":189,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.442814+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:41:41.442814+0000","last_peered":"2023-10-01T09:41:41.442814+0000","last_clean":"2023-10-01T09:41:41.442814+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:41:41.442814+0000","last_undegraded":"2023-10-01T09:41:41.442814+0000","last_fullsized":"2023-10-01T09:41:41.442814+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":565,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.442921+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:41:41.442921+0000","last_peered":"2023-10-01T09:41:41.442921+0000","last_clean":"2023-10-01T09:41:41.442921+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:41:41.442921+0000","last_undegraded":"2023-10-01T09:41:41.442921+0000","last_fullsized":"2023-10-01T09:41:41.442921+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":495,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.758173+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:41:55.758173+0000","last_peered":"2023-10-01T09:41:55.758173+0000","last_clean":"2023-10-01T09:41:55.758173+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:41:55.758173+0000","last_undegraded":"2023-10-01T09:41:55.758173+0000","last_fullsized":"2023-10-01T09:41:55.758173+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":220,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.347507+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:41:41.347507+0000","last_peered":"2023-10-01T09:41:41.347507+0000","last_clean":"2023-10-01T09:41:41.347507+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:41:41.347507+0000","last_undegraded":"2023-10-01T09:41:41.347507+0000","last_fullsized":"2023-10-01T09:41:41.347507+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":28,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.785689+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:41:53.785689+0000","last_peered":"2023-10-01T09:41:53.785689+0000","last_clean":"2023-10-01T09:41:53.785689+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:41:53.785689+0000","last_undegraded":"2023-10-01T09:41:53.785689+0000","last_fullsized":"2023-10-01T09:41:53.785689+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":34,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.786028+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:41:53.786028+0000","last_peered":"2023-10-01T09:41:53.786028+0000","last_clean":"2023-10-01T09:41:53.786028+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:41:53.786028+0000","last_undegraded":"2023-10-01T09:41:53.786028+0000","last_fullsized":"2023-10-01T09:41:53.786028+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":30,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.360264+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:41:41.360264+0000","last_peered":"2023-10-01T09:41:41.360264+0000","last_clean":"2023-10-01T09:41:41.360264+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:41:41.360264+0000","last_undegraded":"2023-10-01T09:41:41.360264+0000","last_fullsized":"2023-10-01T09:41:41.360264+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":410,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.755187+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:41:53.755187+0000","last_peered":"2023-10-01T09:41:53.755187+0000","last_clean":"2023-10-01T09:41:53.755187+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:41:53.755187+0000","last_undegraded":"2023-10-01T09:41:53.755187+0000","last_fullsized":"2023-10-01T09:41:53.755187+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":291,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.754627+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:41:53.754627+0000","last_peered":"2023-10-01T09:41:53.754627+0000","last_clean":"2023-10-01T09:41:53.754627+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:41:53.754627+0000","last_undegraded":"2023-10-01T09:41:53.754627+0000","last_fullsized":"2023-10-01T09:41:53.754627+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":627,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.785956+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:41:53.785956+0000","last_peered":"2023-10-01T09:41:53.785956+0000","last_clean":"2023-10-01T09:41:53.785956+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:41:53.785956+0000","last_undegraded":"2023-10-01T09:41:53.785956+0000","last_fullsized":"2023-10-01T09:41:53.785956+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":364,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.786126+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:41:53.786126+0000","last_peered":"2023-10-01T09:41:53.786126+0000","last_clean":"2023-10-01T09:41:53.786126+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:41:53.786126+0000","last_undegraded":"2023-10-01T09:41:53.786126+0000","last_fullsized":"2023-10-01T09:41:53.786126+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":37,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.785878+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:41:53.785878+0000","last_peered":"2023-10-01T09:41:53.785878+0000","last_clean":"2023-10-01T09:41:53.785878+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:41:53.785878+0000","last_undegraded":"2023-10-01T09:41:53.785878+0000","last_fullsized":"2023-10-01T09:41:53.785878+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":22,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.346000+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:41:41.346000+0000","last_peered":"2023-10-01T09:41:41.346000+0000","last_clean":"2023-10-01T09:41:41.346000+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:41:41.346000+0000","last_undegraded":"2023-10-01T09:41:41.346000+0000","last_fullsized":"2023-10-01T09:41:41.346000+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":520,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.785810+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:41:53.785810+0000","last_peered":"2023-10-01T09:41:53.785810+0000","last_clean":"2023-10-01T09:41:53.785810+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:41:53.785810+0000","last_undegraded":"2023-10-01T09:41:53.785810+0000","last_fullsized":"2023-10-01T09:41:53.785810+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":531,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.763242+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:41:55.763242+0000","last_peered":"2023-10-01T09:41:55.763242+0000","last_clean":"2023-10-01T09:41:55.763242+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:41:55.763242+0000","last_undegraded":"2023-10-01T09:41:55.763242+0000","last_fullsized":"2023-10-01T09:41:55.763242+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":387,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.762851+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:41:55.762851+0000","last_peered":"2023-10-01T09:41:55.762851+0000","last_clean":"2023-10-01T09:41:55.762851+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:41:55.762851+0000","last_undegraded":"2023-10-01T09:41:55.762851+0000","last_fullsized":"2023-10-01T09:41:55.762851+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":37,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.786530+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:41:53.786530+0000","last_peered":"2023-10-01T09:41:53.786530+0000","last_clean":"2023-10-01T09:41:53.786530+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:41:53.786530+0000","last_undegraded":"2023-10-01T09:41:53.786530+0000","last_fullsized":"2023-10-01T09:41:53.786530+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"508'5319","reported_seq":5360,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:54.724974+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:41:54.724974+0000","last_peered":"2023-10-01T09:41:54.724974+0000","last_clean":"2023-10-01T09:41:54.724974+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:41:54.724974+0000","last_undegraded":"2023-10-01T09:41:54.724974+0000","last_fullsized":"2023-10-01T09:41:54.724974+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5319,"log_dups_size":0,"ondisk_log_size":5319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332464128,"num_objects":5073,"num_object_clones":0,"num_object_copies":15219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5319,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":32,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.786457+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:41:53.786457+0000","last_peered":"2023-10-01T09:41:53.786457+0000","last_clean":"2023-10-01T09:41:53.786457+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:41:53.786457+0000","last_undegraded":"2023-10-01T09:41:53.786457+0000","last_fullsized":"2023-10-01T09:41:53.786457+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":228,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.348127+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:41:41.348127+0000","last_peered":"2023-10-01T09:41:41.348127+0000","last_clean":"2023-10-01T09:41:41.348127+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:41:41.348127+0000","last_undegraded":"2023-10-01T09:41:41.348127+0000","last_fullsized":"2023-10-01T09:41:41.348127+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"508'5283","reported_seq":5588,"reported_epoch":508,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:41:54.875946+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:41:54.875946+0000","last_peered":"2023-10-01T09:41:54.875946+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:41:54.875946+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5283,"log_dups_size":0,"ondisk_log_size":5283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335740928,"num_objects":5123,"num_object_clones":0,"num_object_copies":15369,"num_objects_missing_on_primary":3608,"num_objects_missing":3608,"num_objects_degraded":8663,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5283,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":5242880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1447},{"shards":"0,4,6","objects":3608},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":503,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.786307+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:41:53.786307+0000","last_peered":"2023-10-01T09:41:53.786307+0000","last_clean":"2023-10-01T09:41:53.786307+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:41:53.786307+0000","last_undegraded":"2023-10-01T09:41:53.786307+0000","last_fullsized":"2023-10-01T09:41:53.786307+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":562,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.359492+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:41:41.359492+0000","last_peered":"2023-10-01T09:41:41.359492+0000","last_clean":"2023-10-01T09:41:41.359492+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:41:41.359492+0000","last_undegraded":"2023-10-01T09:41:41.359492+0000","last_fullsized":"2023-10-01T09:41:41.359492+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"510'5318","reported_seq":5362,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766740+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:41:56.766740+0000","last_peered":"2023-10-01T09:41:56.766740+0000","last_clean":"2023-10-01T09:41:56.766740+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:41:56.766740+0000","last_undegraded":"2023-10-01T09:41:56.766740+0000","last_fullsized":"2023-10-01T09:41:56.766740+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5318,"log_dups_size":0,"ondisk_log_size":5318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335151104,"num_objects":5114,"num_object_clones":0,"num_object_copies":15342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5318,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":589,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.786168+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:41:53.786168+0000","last_peered":"2023-10-01T09:41:53.786168+0000","last_clean":"2023-10-01T09:41:53.786168+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:41:53.786168+0000","last_undegraded":"2023-10-01T09:41:53.786168+0000","last_fullsized":"2023-10-01T09:41:53.786168+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":383,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.760722+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:41:53.760722+0000","last_peered":"2023-10-01T09:41:53.760722+0000","last_clean":"2023-10-01T09:41:53.760722+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:41:53.760722+0000","last_undegraded":"2023-10-01T09:41:53.760722+0000","last_fullsized":"2023-10-01T09:41:53.760722+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"510'5290","reported_seq":11939,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766905+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:41:56.766905+0000","last_peered":"2023-10-01T09:41:56.766905+0000","last_clean":"2023-10-01T09:41:56.766905+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:41:56.766905+0000","last_undegraded":"2023-10-01T09:41:56.766905+0000","last_fullsized":"2023-10-01T09:41:56.766905+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5290,"log_dups_size":0,"ondisk_log_size":5290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335020032,"num_objects":5112,"num_object_clones":0,"num_object_copies":15336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5290,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":228,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.345777+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:41:41.345777+0000","last_peered":"2023-10-01T09:41:41.345777+0000","last_clean":"2023-10-01T09:41:41.345777+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:41:41.345777+0000","last_undegraded":"2023-10-01T09:41:41.345777+0000","last_fullsized":"2023-10-01T09:41:41.345777+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":487,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.763039+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:41:55.763039+0000","last_peered":"2023-10-01T09:41:55.763039+0000","last_clean":"2023-10-01T09:41:55.763039+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:41:55.763039+0000","last_undegraded":"2023-10-01T09:41:55.763039+0000","last_fullsized":"2023-10-01T09:41:55.763039+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"508'5358","reported_seq":5400,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:54.742882+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:41:54.742882+0000","last_peered":"2023-10-01T09:41:54.742882+0000","last_clean":"2023-10-01T09:41:54.742882+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:41:54.742882+0000","last_undegraded":"2023-10-01T09:41:54.742882+0000","last_fullsized":"2023-10-01T09:41:54.742882+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5358,"log_dups_size":0,"ondisk_log_size":5358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337510400,"num_objects":5150,"num_object_clones":0,"num_object_copies":15450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5358,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":89,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.754911+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:41:53.754911+0000","last_peered":"2023-10-01T09:41:53.754911+0000","last_clean":"2023-10-01T09:41:53.754911+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:41:53.754911+0000","last_undegraded":"2023-10-01T09:41:53.754911+0000","last_fullsized":"2023-10-01T09:41:53.754911+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":163,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.755275+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:41:53.755275+0000","last_peered":"2023-10-01T09:41:53.755275+0000","last_clean":"2023-10-01T09:41:53.755275+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:41:53.755275+0000","last_undegraded":"2023-10-01T09:41:53.755275+0000","last_fullsized":"2023-10-01T09:41:53.755275+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"508'5292","reported_seq":5341,"reported_epoch":508,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:41:54.738190+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:41:54.738190+0000","last_peered":"2023-10-01T09:41:54.738190+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:41:54.738190+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5292,"log_dups_size":0,"ondisk_log_size":5292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332660736,"num_objects":5076,"num_object_clones":0,"num_object_copies":15228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":109,"num_objects_misplaced":5012,"num_objects_unfound":0,"num_objects_dirty":5076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5292,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":5012},{"shards":"4,5","objects":-45}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":407,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.754602+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:41:53.754602+0000","last_peered":"2023-10-01T09:41:53.754602+0000","last_clean":"2023-10-01T09:41:53.754602+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:41:53.754602+0000","last_undegraded":"2023-10-01T09:41:53.754602+0000","last_fullsized":"2023-10-01T09:41:53.754602+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":565,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.359958+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:41:41.359958+0000","last_peered":"2023-10-01T09:41:41.359958+0000","last_clean":"2023-10-01T09:41:41.359958+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:41:41.359958+0000","last_undegraded":"2023-10-01T09:41:41.359958+0000","last_fullsized":"2023-10-01T09:41:41.359958+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"510'5305","reported_seq":5357,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:57.498200+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:41:57.498200+0000","last_peered":"2023-10-01T09:41:57.498200+0000","last_clean":"2023-10-01T09:41:57.498200+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:41:57.498200+0000","last_undegraded":"2023-10-01T09:41:57.498200+0000","last_fullsized":"2023-10-01T09:41:57.498200+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5305,"log_dups_size":0,"ondisk_log_size":5305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332464128,"num_objects":5073,"num_object_clones":0,"num_object_copies":15219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5305,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":403,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.754438+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:41:53.754438+0000","last_peered":"2023-10-01T09:41:53.754438+0000","last_clean":"2023-10-01T09:41:53.754438+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:41:53.754438+0000","last_undegraded":"2023-10-01T09:41:53.754438+0000","last_fullsized":"2023-10-01T09:41:53.754438+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":244,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.755486+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:41:53.755486+0000","last_peered":"2023-10-01T09:41:53.755486+0000","last_clean":"2023-10-01T09:41:53.755486+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:41:53.755486+0000","last_undegraded":"2023-10-01T09:41:53.755486+0000","last_fullsized":"2023-10-01T09:41:53.755486+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"509'5381","reported_seq":5423,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.250406+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:41:56.250406+0000","last_peered":"2023-10-01T09:41:56.250406+0000","last_clean":"2023-10-01T09:41:56.250406+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:41:56.250406+0000","last_undegraded":"2023-10-01T09:41:56.250406+0000","last_fullsized":"2023-10-01T09:41:56.250406+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5381,"log_dups_size":0,"ondisk_log_size":5381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335347712,"num_objects":5117,"num_object_clones":0,"num_object_copies":15351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5381,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":370,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.359400+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:41:41.359400+0000","last_peered":"2023-10-01T09:41:41.359400+0000","last_clean":"2023-10-01T09:41:41.359400+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:41:41.359400+0000","last_undegraded":"2023-10-01T09:41:41.359400+0000","last_fullsized":"2023-10-01T09:41:41.359400+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":417,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.360411+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:41:41.360411+0000","last_peered":"2023-10-01T09:41:41.360411+0000","last_clean":"2023-10-01T09:41:41.360411+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:41:41.360411+0000","last_undegraded":"2023-10-01T09:41:41.360411+0000","last_fullsized":"2023-10-01T09:41:41.360411+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"508'5188","reported_seq":5230,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.730204+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:41:55.730204+0000","last_peered":"2023-10-01T09:41:55.730204+0000","last_clean":"2023-10-01T09:41:55.730204+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:41:55.730204+0000","last_undegraded":"2023-10-01T09:41:55.730204+0000","last_fullsized":"2023-10-01T09:41:55.730204+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5188,"log_dups_size":0,"ondisk_log_size":5188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327024640,"num_objects":4990,"num_object_clones":0,"num_object_copies":14970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5188,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":902,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.786043+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:41:53.786043+0000","last_peered":"2023-10-01T09:41:53.786043+0000","last_clean":"2023-10-01T09:41:53.786043+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:41:53.786043+0000","last_undegraded":"2023-10-01T09:41:53.786043+0000","last_fullsized":"2023-10-01T09:41:53.786043+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":238,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.346133+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:41:41.346133+0000","last_peered":"2023-10-01T09:41:41.346133+0000","last_clean":"2023-10-01T09:41:41.346133+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:41:41.346133+0000","last_undegraded":"2023-10-01T09:41:41.346133+0000","last_fullsized":"2023-10-01T09:41:41.346133+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":30,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.360262+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:41:41.360262+0000","last_peered":"2023-10-01T09:41:41.360262+0000","last_clean":"2023-10-01T09:41:41.360262+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:41:41.360262+0000","last_undegraded":"2023-10-01T09:41:41.360262+0000","last_fullsized":"2023-10-01T09:41:41.360262+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"509'5293","reported_seq":5337,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.447709+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:41:56.447709+0000","last_peered":"2023-10-01T09:41:56.447709+0000","last_clean":"2023-10-01T09:41:56.447709+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:41:56.447709+0000","last_undegraded":"2023-10-01T09:41:56.447709+0000","last_fullsized":"2023-10-01T09:41:56.447709+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5293,"log_dups_size":0,"ondisk_log_size":5293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332726272,"num_objects":5077,"num_object_clones":0,"num_object_copies":15231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5293,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1173,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.346483+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:41:41.346483+0000","last_peered":"2023-10-01T09:41:41.346483+0000","last_clean":"2023-10-01T09:41:41.346483+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:41:41.346483+0000","last_undegraded":"2023-10-01T09:41:41.346483+0000","last_fullsized":"2023-10-01T09:41:41.346483+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":384,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.754806+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:41:53.754806+0000","last_peered":"2023-10-01T09:41:53.754806+0000","last_clean":"2023-10-01T09:41:53.754806+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:41:53.754806+0000","last_undegraded":"2023-10-01T09:41:53.754806+0000","last_fullsized":"2023-10-01T09:41:53.754806+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":211,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.345912+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:41:41.345912+0000","last_peered":"2023-10-01T09:41:41.345912+0000","last_clean":"2023-10-01T09:41:41.345912+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:41:41.345912+0000","last_undegraded":"2023-10-01T09:41:41.345912+0000","last_fullsized":"2023-10-01T09:41:41.345912+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"507'5264","reported_seq":11426,"reported_epoch":507,"state":"active+recovering+remapped","last_fresh":"2023-10-01T09:41:52.438267+0000","last_change":"2023-10-01T09:41:29.602605+0000","last_active":"2023-10-01T09:41:52.438267+0000","last_peered":"2023-10-01T09:41:52.438267+0000","last_clean":"2023-10-01T09:40:54.363445+0000","last_became_active":"2023-10-01T09:40:56.569721+0000","last_became_peered":"2023-10-01T09:40:56.569721+0000","last_unstale":"2023-10-01T09:41:52.438267+0000","last_undegraded":"2023-10-01T09:41:52.438267+0000","last_fullsized":"2023-10-01T09:41:52.438267+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5264,"log_dups_size":0,"ondisk_log_size":5264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:17:29.742329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335020032,"num_objects":5112,"num_object_clones":0,"num_object_copies":15336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":544,"num_objects_unfound":0,"num_objects_dirty":5112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5264,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3058,"num_bytes_recovered":200409088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":4568}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":854,"reported_epoch":509,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:41:55.762718+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:41:55.762718+0000","last_peered":"2023-10-01T09:41:55.762718+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:41:55.762718+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:41:55.762718+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":569,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.359405+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:41:41.359405+0000","last_peered":"2023-10-01T09:41:41.359405+0000","last_clean":"2023-10-01T09:41:41.359405+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:41:41.359405+0000","last_undegraded":"2023-10-01T09:41:41.359405+0000","last_fullsized":"2023-10-01T09:41:41.359405+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":30,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.360150+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:41:41.360150+0000","last_peered":"2023-10-01T09:41:41.360150+0000","last_clean":"2023-10-01T09:41:41.360150+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:41:41.360150+0000","last_undegraded":"2023-10-01T09:41:41.360150+0000","last_fullsized":"2023-10-01T09:41:41.360150+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"508'5241","reported_seq":5283,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:54.993437+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:41:54.993437+0000","last_peered":"2023-10-01T09:41:54.993437+0000","last_clean":"2023-10-01T09:41:54.993437+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:41:54.993437+0000","last_undegraded":"2023-10-01T09:41:54.993437+0000","last_fullsized":"2023-10-01T09:41:54.993437+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5241,"log_dups_size":0,"ondisk_log_size":5241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329711616,"num_objects":5031,"num_object_clones":0,"num_object_copies":15093,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5031,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5241,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1624,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.348433+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:41:41.348433+0000","last_peered":"2023-10-01T09:41:41.348433+0000","last_clean":"2023-10-01T09:41:41.348433+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:41:41.348433+0000","last_undegraded":"2023-10-01T09:41:41.348433+0000","last_fullsized":"2023-10-01T09:41:41.348433+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":539,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.785691+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:41:53.785691+0000","last_peered":"2023-10-01T09:41:53.785691+0000","last_clean":"2023-10-01T09:41:53.785691+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:41:53.785691+0000","last_undegraded":"2023-10-01T09:41:53.785691+0000","last_fullsized":"2023-10-01T09:41:53.785691+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":316,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.346556+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:41:41.346556+0000","last_peered":"2023-10-01T09:41:41.346556+0000","last_clean":"2023-10-01T09:41:41.346556+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:41:41.346556+0000","last_undegraded":"2023-10-01T09:41:41.346556+0000","last_fullsized":"2023-10-01T09:41:41.346556+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":307,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.754965+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:41:53.754965+0000","last_peered":"2023-10-01T09:41:53.754965+0000","last_clean":"2023-10-01T09:41:53.754965+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:41:53.754965+0000","last_undegraded":"2023-10-01T09:41:53.754965+0000","last_fullsized":"2023-10-01T09:41:53.754965+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"509'5371","reported_seq":5413,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.758558+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:41:55.758558+0000","last_peered":"2023-10-01T09:41:55.758558+0000","last_clean":"2023-10-01T09:41:55.758558+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:41:55.758558+0000","last_undegraded":"2023-10-01T09:41:55.758558+0000","last_fullsized":"2023-10-01T09:41:55.758558+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5371,"log_dups_size":0,"ondisk_log_size":5371,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338231296,"num_objects":5161,"num_object_clones":0,"num_object_copies":15483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5371,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":716,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.785786+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:41:53.785786+0000","last_peered":"2023-10-01T09:41:53.785786+0000","last_clean":"2023-10-01T09:41:53.785786+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:41:53.785786+0000","last_undegraded":"2023-10-01T09:41:53.785786+0000","last_fullsized":"2023-10-01T09:41:53.785786+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":548,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.763262+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:41:55.763262+0000","last_peered":"2023-10-01T09:41:55.763262+0000","last_clean":"2023-10-01T09:41:55.763262+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:41:55.763262+0000","last_undegraded":"2023-10-01T09:41:55.763262+0000","last_fullsized":"2023-10-01T09:41:55.763262+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":548,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.763073+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:41:55.763073+0000","last_peered":"2023-10-01T09:41:55.763073+0000","last_clean":"2023-10-01T09:41:55.763073+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:41:55.763073+0000","last_undegraded":"2023-10-01T09:41:55.763073+0000","last_fullsized":"2023-10-01T09:41:55.763073+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"510'5353","reported_seq":5398,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.767028+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:41:56.767028+0000","last_peered":"2023-10-01T09:41:56.767028+0000","last_clean":"2023-10-01T09:41:56.767028+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:41:56.767028+0000","last_undegraded":"2023-10-01T09:41:56.767028+0000","last_fullsized":"2023-10-01T09:41:56.767028+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5353,"log_dups_size":0,"ondisk_log_size":5353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334626840,"num_objects":5107,"num_object_clones":0,"num_object_copies":15321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5107,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5353,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":947,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.786252+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:41:53.786252+0000","last_peered":"2023-10-01T09:41:53.786252+0000","last_clean":"2023-10-01T09:41:53.786252+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:41:53.786252+0000","last_undegraded":"2023-10-01T09:41:53.786252+0000","last_fullsized":"2023-10-01T09:41:53.786252+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":565,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.345847+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:41:41.345847+0000","last_peered":"2023-10-01T09:41:41.345847+0000","last_clean":"2023-10-01T09:41:41.345847+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:41:41.345847+0000","last_undegraded":"2023-10-01T09:41:41.345847+0000","last_fullsized":"2023-10-01T09:41:41.345847+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":418,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.348507+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:41:41.348507+0000","last_peered":"2023-10-01T09:41:41.348507+0000","last_clean":"2023-10-01T09:41:41.348507+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:41:41.348507+0000","last_undegraded":"2023-10-01T09:41:41.348507+0000","last_fullsized":"2023-10-01T09:41:41.348507+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"509'5297","reported_seq":5347,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.763566+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:41:55.763566+0000","last_peered":"2023-10-01T09:41:55.763566+0000","last_clean":"2023-10-01T09:41:55.763566+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:41:55.763566+0000","last_undegraded":"2023-10-01T09:41:55.763566+0000","last_fullsized":"2023-10-01T09:41:55.763566+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5297,"log_dups_size":0,"ondisk_log_size":5297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331546624,"num_objects":5059,"num_object_clones":0,"num_object_copies":15177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5297,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":448,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.786619+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:41:53.786619+0000","last_peered":"2023-10-01T09:41:53.786619+0000","last_clean":"2023-10-01T09:41:53.786619+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:41:53.786619+0000","last_undegraded":"2023-10-01T09:41:53.786619+0000","last_fullsized":"2023-10-01T09:41:53.786619+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1139,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.360159+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:41:41.360159+0000","last_peered":"2023-10-01T09:41:41.360159+0000","last_clean":"2023-10-01T09:41:41.360159+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:41:41.360159+0000","last_undegraded":"2023-10-01T09:41:41.360159+0000","last_fullsized":"2023-10-01T09:41:41.360159+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":37,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.785637+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:41:53.785637+0000","last_peered":"2023-10-01T09:41:53.785637+0000","last_clean":"2023-10-01T09:41:53.785637+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:41:53.785637+0000","last_undegraded":"2023-10-01T09:41:53.785637+0000","last_fullsized":"2023-10-01T09:41:53.785637+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"509'5363","reported_seq":5413,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.794219+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:41:55.794219+0000","last_peered":"2023-10-01T09:41:55.794219+0000","last_clean":"2023-10-01T09:41:55.794219+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:41:55.794219+0000","last_undegraded":"2023-10-01T09:41:55.794219+0000","last_fullsized":"2023-10-01T09:41:55.794219+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5363,"log_dups_size":0,"ondisk_log_size":5363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337444864,"num_objects":5149,"num_object_clones":0,"num_object_copies":15447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5363,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":41,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.786452+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:41:53.786452+0000","last_peered":"2023-10-01T09:41:53.786452+0000","last_clean":"2023-10-01T09:41:53.786452+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:41:53.786452+0000","last_undegraded":"2023-10-01T09:41:53.786452+0000","last_fullsized":"2023-10-01T09:41:53.786452+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":526,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.763313+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:41:55.763313+0000","last_peered":"2023-10-01T09:41:55.763313+0000","last_clean":"2023-10-01T09:41:55.763313+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:41:55.763313+0000","last_undegraded":"2023-10-01T09:41:55.763313+0000","last_fullsized":"2023-10-01T09:41:55.763313+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":231,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.348918+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:41:41.348918+0000","last_peered":"2023-10-01T09:41:41.348918+0000","last_clean":"2023-10-01T09:41:41.348918+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:41:41.348918+0000","last_undegraded":"2023-10-01T09:41:41.348918+0000","last_fullsized":"2023-10-01T09:41:41.348918+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":475,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.763051+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:41:55.763051+0000","last_peered":"2023-10-01T09:41:55.763051+0000","last_clean":"2023-10-01T09:41:55.763051+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:41:55.763051+0000","last_undegraded":"2023-10-01T09:41:55.763051+0000","last_fullsized":"2023-10-01T09:41:55.763051+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":587,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.359794+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:41:41.359794+0000","last_peered":"2023-10-01T09:41:41.359794+0000","last_clean":"2023-10-01T09:41:41.359794+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:41:41.359794+0000","last_undegraded":"2023-10-01T09:41:41.359794+0000","last_fullsized":"2023-10-01T09:41:41.359794+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":243,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.345704+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:41:41.345704+0000","last_peered":"2023-10-01T09:41:41.345704+0000","last_clean":"2023-10-01T09:41:41.345704+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:41:41.345704+0000","last_undegraded":"2023-10-01T09:41:41.345704+0000","last_fullsized":"2023-10-01T09:41:41.345704+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":30,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.359941+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:41:41.359941+0000","last_peered":"2023-10-01T09:41:41.359941+0000","last_clean":"2023-10-01T09:41:41.359941+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:41:41.359941+0000","last_undegraded":"2023-10-01T09:41:41.359941+0000","last_fullsized":"2023-10-01T09:41:41.359941+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":243,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.346213+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:41:41.346213+0000","last_peered":"2023-10-01T09:41:41.346213+0000","last_clean":"2023-10-01T09:41:41.346213+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:41:41.346213+0000","last_undegraded":"2023-10-01T09:41:41.346213+0000","last_fullsized":"2023-10-01T09:41:41.346213+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":421,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.762924+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:41:55.762924+0000","last_peered":"2023-10-01T09:41:55.762924+0000","last_clean":"2023-10-01T09:41:55.762924+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:41:55.762924+0000","last_undegraded":"2023-10-01T09:41:55.762924+0000","last_fullsized":"2023-10-01T09:41:55.762924+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":47,"reported_epoch":507,"state":"active+clean","last_fresh":"2023-10-01T09:41:41.360020+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:41:41.360020+0000","last_peered":"2023-10-01T09:41:41.360020+0000","last_clean":"2023-10-01T09:41:41.360020+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:41:41.360020+0000","last_undegraded":"2023-10-01T09:41:41.360020+0000","last_fullsized":"2023-10-01T09:41:41.360020+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":466,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.763151+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:41:55.763151+0000","last_peered":"2023-10-01T09:41:55.763151+0000","last_clean":"2023-10-01T09:41:55.763151+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:41:55.763151+0000","last_undegraded":"2023-10-01T09:41:55.763151+0000","last_fullsized":"2023-10-01T09:41:55.763151+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":665,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.763331+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:41:55.763331+0000","last_peered":"2023-10-01T09:41:55.763331+0000","last_clean":"2023-10-01T09:41:55.763331+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:41:55.763331+0000","last_undegraded":"2023-10-01T09:41:55.763331+0000","last_fullsized":"2023-10-01T09:41:55.763331+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":360,"reported_epoch":508,"state":"active+clean","last_fresh":"2023-10-01T09:41:53.754321+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:41:53.754321+0000","last_peered":"2023-10-01T09:41:53.754321+0000","last_clean":"2023-10-01T09:41:53.754321+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:41:53.754321+0000","last_undegraded":"2023-10-01T09:41:53.754321+0000","last_fullsized":"2023-10-01T09:41:53.754321+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5342691352,"num_objects":81524,"num_object_clones":0,"num_object_copies":244572,"num_objects_missing_on_primary":3608,"num_objects_missing":3608,"num_objects_degraded":8772,"num_objects_misplaced":5556,"num_objects_unfound":0,"num_objects_dirty":81524,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":84916,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6429,"num_bytes_recovered":421330944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16619483136,"data_stored":16619470920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84916,"ondisk_log_size":84916,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430696,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2710832,"kb_used_data":1803624,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":91034320,"statfs":{"total":95995035648,"available":93219143680,"internally_reserved":0,"allocated":1846910976,"data_stored":1845979104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.766,"15min":3.0510000000000002},"min":{"1min":0.30099999999999999,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":6.8550000000000004,"5min":11.525,"15min":108.90000000000001},"last":0.246},{"interface":"front","average":{"1min":0.82099999999999995,"5min":2.2130000000000001,"15min":3.3359999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.6639999999999997,"5min":29.501000000000001,"15min":109.187},"last":0.28899999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:41:05 2023","interfaces":[{"interface":"back","average":{"1min":11.349,"5min":6.492,"15min":5.5579999999999998},"min":{"1min":0.248,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":229.161,"5min":229.161,"15min":229.161},"last":0.28000000000000003},{"interface":"front","average":{"1min":11.241,"5min":6.6150000000000002,"15min":5.6059999999999999},"min":{"1min":0.309,"5min":0.27500000000000002,"15min":0.24399999999999999},"max":{"1min":230.126,"5min":230.126,"15min":230.126},"last":0.32600000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:41:47 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":2.1680000000000001,"15min":2.0169999999999999},"min":{"1min":0.33200000000000002,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":5.3090000000000002,"5min":14.866,"15min":14.901999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.2829999999999999,"5min":2.1749999999999998,"15min":1.9079999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.6929999999999996,"5min":14.682,"15min":14.682},"last":0.35299999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:41:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":2.6699999999999999,"15min":3.528},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28199999999999997},"max":{"1min":4.3120000000000003,"5min":35.104999999999997,"15min":164.595},"last":0.433},{"interface":"front","average":{"1min":1.3240000000000001,"5min":2.738,"15min":3.5},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.1440000000000001,"5min":35.118000000000002,"15min":164.60499999999999},"last":0.41999999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.202,"15min":1.0529999999999999},"min":{"1min":0.30499999999999999,"5min":0.27600000000000002,"15min":0.254},"max":{"1min":6.8769999999999998,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.4710000000000001,"5min":1.1739999999999999,"15min":1.0369999999999999},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":5.7439999999999998,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.38200000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":8.516,"5min":2.984,"15min":1.679},"min":{"1min":0.311,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":135.40000000000001,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":8.1989999999999998,"5min":3.1360000000000001,"15min":1.71},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":135.339,"5min":135.339,"15min":135.339},"last":0.48899999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0699999999999998,"5min":1.498,"15min":1.1819999999999999},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":11.58,"5min":11.58,"15min":11.853999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.988,"5min":1.3939999999999999,"15min":1.147},"min":{"1min":0.373,"5min":0.373,"15min":0.32300000000000001},"max":{"1min":13.021000000000001,"5min":14.584,"15min":14.584},"last":0.56699999999999995}]}]},{"osd":6,"up_from":294,"seq":1262720385207,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2622076,"kb_used_data":2280372,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":91123076,"statfs":{"total":95995035648,"available":93310029824,"internally_reserved":0,"allocated":2335100928,"data_stored":2334172359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":7.4020000000000001,"5min":3.9239999999999999,"15min":4.2949999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":158.48599999999999,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":8.1300000000000008,"5min":4.5199999999999996,"15min":4.335},"min":{"1min":0.34399999999999997,"5min":0.25700000000000001,"15min":0.221},"max":{"1min":158.37700000000001,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.442}]},{"osd":1,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":12.536,"5min":7.2130000000000001,"15min":5.1319999999999997},"min":{"1min":0.38200000000000001,"5min":0.25800000000000001,"15min":0.23899999999999999},"max":{"1min":168.947,"5min":168.947,"15min":168.947},"last":0.41699999999999998},{"interface":"front","average":{"1min":12.667999999999999,"5min":7.1269999999999998,"15min":5.0389999999999997},"min":{"1min":0.32400000000000001,"5min":0.30599999999999999,"15min":0.24399999999999999},"max":{"1min":169.07900000000001,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.45400000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.919,"15min":1.9850000000000001},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":8.657,"5min":9.657,"15min":26.289999999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.944,"15min":2.0179999999999998},"min":{"1min":0.29399999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.247999999999999,"5min":21.704999999999998,"15min":26.388999999999999},"last":1.0649999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.903,"15min":2.5209999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":4.149,"5min":9.0180000000000007,"15min":89.753},"last":1.9450000000000001},{"interface":"front","average":{"1min":1.7110000000000001,"5min":1.9870000000000001,"15min":2.2429999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":8.5470000000000006,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.39300000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":4.9210000000000003,"5min":1.5780000000000001,"15min":1.038},"min":{"1min":0.26300000000000001,"5min":0.246,"15min":0.246},"max":{"1min":63.134,"5min":63.134,"15min":63.134},"last":0.42499999999999999},{"interface":"front","average":{"1min":4.3339999999999996,"5min":1.3959999999999999,"15min":1.056},"min":{"1min":0.35299999999999998,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":62.973999999999997,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.433}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":1.0089999999999999,"15min":1.0529999999999999},"min":{"1min":0.312,"5min":0.251,"15min":0.251},"max":{"1min":9.9939999999999998,"5min":9.9939999999999998,"15min":12.023999999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.1880000000000002,"5min":1.234,"15min":1.1739999999999999},"min":{"1min":0.314,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":15.457000000000001,"5min":15.457000000000001,"15min":15.457000000000001},"last":0.48099999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":0.86099999999999999,"15min":2.1339999999999999},"min":{"1min":0.34100000000000003,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":3.492,"5min":6.4119999999999999,"15min":115.11},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.2310000000000001,"15min":2.2040000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28799999999999998},"max":{"1min":8.6899999999999995,"5min":8.6899999999999995,"15min":114.666},"last":0.47099999999999997}]}]},{"osd":1,"up_from":94,"seq":403726926183,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1797112,"kb_used_data":987120,"kb_used_omap":7,"kb_used_meta":809976,"kb_avail":91948040,"statfs":{"total":95995035648,"available":94154792960,"internally_reserved":0,"allocated":1010810880,"data_stored":1008819342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":829415847},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":3.5089999999999999,"15min":4.5229999999999997},"min":{"1min":0.19400000000000001,"5min":0.182,"15min":0.182},"max":{"1min":4.423,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.247},{"interface":"front","average":{"1min":1.528,"5min":4.0339999999999998,"15min":3.859},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":13.872,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.44800000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:41:37 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":2.6459999999999999,"15min":3.4620000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":2.8740000000000001,"5min":22.891999999999999,"15min":56.573},"last":0.83099999999999996},{"interface":"front","average":{"1min":1.2769999999999999,"5min":3.0059999999999998,"15min":4.2670000000000003},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.20499999999999999},"max":{"1min":6.9710000000000001,"5min":15.845000000000001,"15min":154.197},"last":0.35499999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:41:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":4.9089999999999998,"15min":4.8239999999999998},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":18.503,"5min":95.305000000000007,"15min":200.56800000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.4119999999999999,"5min":6.7089999999999996,"15min":5.54},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":16.515000000000001,"5min":154.483,"15min":200.63800000000001},"last":0.33000000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":2.4870000000000001,"15min":2.3940000000000001},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.5510000000000002,"5min":21.015000000000001,"15min":154.101},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.60799999999999998,"5min":2.306,"15min":2.7410000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.4209999999999998,"5min":13.773999999999999,"15min":154.56999999999999},"last":0.39800000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":2.4870000000000001,"15min":2.1509999999999998},"min":{"1min":0.32900000000000001,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":5.0410000000000004,"5min":21.219999999999999,"15min":154.22999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.83399999999999996,"5min":2.3239999999999998,"15min":2.661},"min":{"1min":0.311,"5min":0.307,"15min":0.307},"max":{"1min":5.1050000000000004,"5min":13.848000000000001,"15min":154.54400000000001},"last":0.372}]},{"osd":6,"last update":"Sun Oct 1 09:41:08 2023","interfaces":[{"interface":"back","average":{"1min":2.363,"5min":4.1890000000000001,"15min":2.9950000000000001},"min":{"1min":0.34300000000000003,"5min":0.32900000000000001,"15min":0.29899999999999999},"max":{"1min":23.672999999999998,"5min":154.12200000000001,"15min":154.12200000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.27,"5min":4.4729999999999999,"15min":2.6829999999999998},"min":{"1min":0.34599999999999997,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":23.382000000000001,"5min":154.16999999999999,"15min":154.16999999999999},"last":0.501}]},{"osd":7,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.617,"15min":2.7250000000000001},"min":{"1min":0.40500000000000003,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":5.2169999999999996,"5min":21.062999999999999,"15min":154.20699999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.88300000000000001,"5min":2.5390000000000001,"15min":2.9399999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":3.8260000000000001,"5min":13.819000000000001,"15min":154.51599999999999},"last":0.47999999999999998}]}]},{"osd":0,"up_from":89,"seq":382252089718,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4820176,"kb_used_data":3924936,"kb_used_omap":8,"kb_used_meta":895223,"kb_avail":88924976,"statfs":{"total":95995035648,"available":91059175424,"internally_reserved":0,"allocated":4019134464,"data_stored":4017132955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":916709314},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:41:06 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":7.5830000000000002,"15min":4.54},"min":{"1min":0.187,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":8.4939999999999998,"5min":216.821,"15min":216.821},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.095,"5min":9.0879999999999992,"15min":5.7130000000000001},"min":{"1min":0.22,"5min":0.215,"15min":0.21199999999999999},"max":{"1min":8.6229999999999993,"5min":217.02699999999999,"15min":217.02699999999999},"last":0.34899999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:41:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":2.7530000000000001,"15min":2.2509999999999999},"min":{"1min":0.30599999999999999,"5min":0.253,"15min":0.20100000000000001},"max":{"1min":4.9459999999999997,"5min":19.788,"15min":19.788},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.085,"5min":2.7290000000000001,"15min":2.7719999999999998},"min":{"1min":0.29599999999999999,"5min":0.215,"15min":0.189},"max":{"1min":3.4470000000000001,"5min":25.844999999999999,"15min":25.844999999999999},"last":0.42099999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:41:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":5.4249999999999998,"15min":4.7450000000000001},"min":{"1min":0.24199999999999999,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":5.0869999999999997,"5min":222.16499999999999,"15min":222.16499999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.80200000000000005,"5min":5.2400000000000002,"15min":4.9770000000000003},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.219},"max":{"1min":5.8419999999999996,"5min":222.227,"15min":222.227},"last":0.33400000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":2.3559999999999999,"15min":2.4260000000000002},"min":{"1min":0.29899999999999999,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":2.7480000000000002,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.034,"5min":2.698,"15min":2.536},"min":{"1min":0.376,"5min":0.33700000000000002,"15min":0.28299999999999997},"max":{"1min":3.1619999999999999,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.376}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.494,"5min":2.371,"15min":2.0569999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":5.8769999999999998,"5min":14.515000000000001,"15min":16.808},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.2879999999999998,"15min":1.9259999999999999},"min":{"1min":0.49099999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.8689999999999998,"5min":17.507000000000001,"15min":17.507000000000001},"last":0.55600000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:41:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":2.2509999999999999,"15min":1.897},"min":{"1min":0.38600000000000001,"5min":0.35699999999999998,"15min":0.35499999999999998},"max":{"1min":5.2110000000000003,"5min":17.535,"15min":17.535},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.649,"5min":2.1739999999999999,"15min":2.2679999999999998},"min":{"1min":0.44800000000000001,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":4.2640000000000002,"5min":10.446,"15min":18.521000000000001},"last":0.628}]},{"osd":7,"last update":"Sun Oct 1 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.198,"15min":2.1909999999999998},"min":{"1min":0.41799999999999998,"5min":0.377,"15min":0.34499999999999997},"max":{"1min":4.976,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.069,"5min":2.4740000000000002,"15min":2.5459999999999998},"min":{"1min":0.41199999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":4.226,"5min":48.274000000000001,"15min":48.274000000000001},"last":0.69599999999999995}]}]},{"osd":2,"up_from":0,"seq":425201762657,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599131,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2369096,"kb_used_data":1541440,"kb_used_omap":3,"kb_used_meta":827644,"kb_avail":91376056,"statfs":{"total":95995035648,"available":93569081344,"internally_reserved":0,"allocated":1578434560,"data_stored":1576384278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":847508199},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":5.0030000000000001,"15min":5.9409999999999998},"min":{"1min":0.183,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":12.999000000000001,"5min":118.748,"15min":137.34399999999999},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.97699999999999998,"5min":5.3849999999999998,"15min":5.8490000000000002},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.184},"max":{"1min":10.377000000000001,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.24299999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":8.3849999999999998,"15min":5.7290000000000001},"min":{"1min":0.252,"5min":0.216,"15min":0.20399999999999999},"max":{"1min":14.917999999999999,"5min":153.77000000000001,"15min":162.22999999999999},"last":0.29899999999999999},{"interface":"front","average":{"1min":1.359,"5min":8.468,"15min":6.2320000000000002},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":15.074,"5min":151.02199999999999,"15min":162.70699999999999},"last":0.33100000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":3.0110000000000001,"15min":2.8839999999999999},"min":{"1min":0.26000000000000001,"5min":0.22800000000000001,"15min":0.20799999999999999},"max":{"1min":2.7970000000000002,"5min":14.254,"15min":43.222999999999999},"last":0.38},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.7070000000000001,"15min":3.206},"min":{"1min":0.247,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":1.4239999999999999,"5min":15.523,"15min":78.594999999999999},"last":0.37}]},{"osd":4,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":4.617,"5min":2.5249999999999999,"15min":2.1760000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":77.793000000000006,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.45000000000000001},{"interface":"front","average":{"1min":4.8399999999999999,"5min":2.9620000000000002,"15min":2.306},"min":{"1min":0.378,"5min":0.27900000000000003,"15min":0.23899999999999999},"max":{"1min":76.646000000000001,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.34799999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":2.1869999999999998,"15min":3.012},"min":{"1min":0.29399999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.0659999999999998,"5min":10.919,"15min":28.117999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.768,"5min":2.6379999999999999,"15min":3.1000000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.1559999999999997,"5min":14.537000000000001,"15min":28.34},"last":0.437}]},{"osd":6,"last update":"Sun Oct 1 09:41:19 2023","interfaces":[{"interface":"back","average":{"1min":5.2430000000000003,"5min":3.4009999999999998,"15min":2.4870000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":73.887,"5min":73.887,"15min":73.887},"last":0.47599999999999998},{"interface":"front","average":{"1min":5.8470000000000004,"5min":3.5840000000000001,"15min":2.4780000000000002},"min":{"1min":0.44700000000000001,"5min":0.38600000000000001,"15min":0.32800000000000001},"max":{"1min":74.016999999999996,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.53100000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":2.8690000000000002,"15min":2.6429999999999998},"min":{"1min":0.46400000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.5570000000000004,"5min":13.085000000000001,"15min":52.006},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.228,"5min":2.0640000000000001,"15min":2.3359999999999999},"min":{"1min":0.371,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":4.5810000000000004,"5min":11.109999999999999,"15min":51.890999999999998},"last":0.499}]}]},{"osd":4,"up_from":30,"seq":128849019381,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4971372,"kb_used_data":4081188,"kb_used_omap":0,"kb_used_meta":890176,"kb_avail":88773780,"statfs":{"total":95995035648,"available":90904350720,"internally_reserved":0,"allocated":4179136512,"data_stored":4178208146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911540224},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:53 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":3.5209999999999999,"15min":4.9050000000000002},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":9.6690000000000005,"5min":79.837000000000003,"15min":155.315},"last":0.308},{"interface":"front","average":{"1min":1.875,"5min":3.9239999999999999,"15min":5.1550000000000002},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.23899999999999999},"max":{"1min":6.1459999999999999,"5min":78.983000000000004,"15min":155.334},"last":2.0819999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:40:54 2023","interfaces":[{"interface":"back","average":{"1min":4.5149999999999997,"5min":10.486000000000001,"15min":7.7160000000000002},"min":{"1min":0.32200000000000001,"5min":0.224,"15min":0.224},"max":{"1min":77.953000000000003,"5min":165.666,"15min":190.61199999999999},"last":2.1760000000000002},{"interface":"front","average":{"1min":4.6669999999999998,"5min":10.384,"15min":8.5039999999999996},"min":{"1min":0.26700000000000002,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":77.962999999999994,"5min":165.70500000000001,"15min":253.273},"last":0.48099999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:41:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.6279999999999999,"15min":2.6560000000000001},"min":{"1min":0.38300000000000001,"5min":0.29099999999999998,"15min":0.28299999999999997},"max":{"1min":5.702,"5min":10.161,"15min":170.45400000000001},"last":2.242},{"interface":"front","average":{"1min":1.5089999999999999,"5min":1.988,"15min":2.371},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":5.6150000000000002,"5min":10.625,"15min":58.771000000000001},"last":2.1930000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:40:54 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":3.6360000000000001,"15min":3.419},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.2719999999999998,"5min":140.881,"15min":140.881},"last":0.377},{"interface":"front","average":{"1min":0.84599999999999997,"5min":3.8860000000000001,"15min":3.633},"min":{"1min":0.32800000000000001,"5min":0.311,"15min":0.311},"max":{"1min":2.8250000000000002,"5min":140.911,"15min":140.911},"last":0.51300000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:41:30 2023","interfaces":[{"interface":"back","average":{"1min":7.3849999999999998,"5min":2.387,"15min":1.6259999999999999},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":110.003,"5min":110.003,"15min":110.003},"last":2.1190000000000002},{"interface":"front","average":{"1min":6.8159999999999998,"5min":2.294,"15min":1.425},"min":{"1min":0.29899999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":108.56399999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.57399999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:41:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":0.97999999999999998,"15min":0.95399999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":5.5709999999999997,"5min":7.7649999999999997,"15min":11.358000000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.381,"5min":1.22,"15min":1.0569999999999999},"min":{"1min":0.317,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":6.1660000000000004,"5min":6.9139999999999997,"15min":10.076000000000001},"last":2.1549999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.2250000000000001,"15min":1.171},"min":{"1min":0.34599999999999997,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":3.8580000000000001,"5min":38.034999999999997,"15min":38.034999999999997},"last":1.825},{"interface":"front","average":{"1min":2.0299999999999998,"5min":1.6899999999999999,"15min":1.4530000000000001},"min":{"1min":0.377,"5min":0.35899999999999999,"15min":0.28699999999999998},"max":{"1min":12.092000000000001,"5min":37.723999999999997,"15min":37.723999999999997},"last":2.2149999999999999}]}]},{"osd":5,"up_from":379,"seq":1627792605288,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2586776,"kb_used_data":2222224,"kb_used_omap":13,"kb_used_meta":364530,"kb_avail":91158376,"statfs":{"total":95995035648,"available":93346177024,"internally_reserved":0,"allocated":2275557376,"data_stored":2274644914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":373279218},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,1,0,1,1,1,4,7,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":2.508,"15min":2.335},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.6379999999999999,"5min":13.385999999999999,"15min":84.340999999999994},"last":0.23200000000000001},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.0030000000000001,"15min":2.1909999999999998},"min":{"1min":0.28299999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.2279999999999998,"5min":12.445,"15min":84.356999999999999},"last":0.34200000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":8.1319999999999997,"5min":9.8100000000000005,"15min":5.9340000000000002},"min":{"1min":0.29299999999999998,"5min":0.255,"15min":0.255},"max":{"1min":106.56999999999999,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.28999999999999998},{"interface":"front","average":{"1min":8.1590000000000007,"5min":10.111000000000001,"15min":5.8540000000000001},"min":{"1min":0.27500000000000002,"5min":0.25900000000000001,"15min":0.24199999999999999},"max":{"1min":106.596,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.36399999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.653,"5min":2.1789999999999998,"15min":1.71},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.3470000000000004,"5min":14.199999999999999,"15min":14.199999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.0379999999999998,"5min":2.0670000000000002,"15min":1.7370000000000001},"min":{"1min":0.373,"5min":0.31,"15min":0.27100000000000002},"max":{"1min":5.782,"5min":14.430999999999999,"15min":14.430999999999999},"last":0.32000000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":2.2730000000000001,"15min":1.5700000000000001},"min":{"1min":0.35399999999999998,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":3.4409999999999998,"5min":14.004,"15min":14.004},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.782,"5min":2.3610000000000002,"15min":1.7150000000000001},"min":{"1min":0.35299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.4400000000000004,"5min":13.991,"15min":13.991},"last":0.48599999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1320000000000001,"5min":2.3300000000000001,"15min":1.5},"min":{"1min":0.36699999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.9459999999999997,"5min":122.971,"15min":122.971},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.6520000000000001,"5min":2.3849999999999998,"15min":1.8859999999999999},"min":{"1min":0.32100000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.445,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.33300000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":1.335,"15min":1.1160000000000001},"min":{"1min":0.33900000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.7750000000000004,"5min":12.545,"15min":12.545},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.8560000000000001,"5min":1.0529999999999999,"15min":1.431},"min":{"1min":0.36899999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.4160000000000004,"5min":6.4729999999999999,"15min":14.193},"last":0.51200000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5459999999999998,"5min":1.238,"15min":1.379},"min":{"1min":0.46800000000000003,"5min":0.307,"15min":0.307},"max":{"1min":5.758,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.2210000000000001,"5min":1.3740000000000001,"15min":1.121},"min":{"1min":0.41199999999999998,"5min":0.318,"15min":0.318},"max":{"1min":5.2300000000000004,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.58599999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4007661568,"data_stored":4007657496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1000669184,"data_stored":1000669184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1345257472,"data_stored":1345257472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4168814592,"data_stored":4168810520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2100166656,"data_stored":2100166656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2329415680,"data_stored":2329411608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1667497984,"data_stored":1667497984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:42:00.212 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:00.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:00.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:00.316 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:00.350 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:00.446 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:00.455 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:00.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:00.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:00.553 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:42:00.554 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:42:00.559 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:00.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:00 smithi138 ceph-mon[185886]: from='client.46646 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:00.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:42:00.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:42:00.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:42:00.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:00 smithi138 ceph-mon[192513]: from='client.46646 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:00.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:42:00.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:42:00.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:42:00.588 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:00.785 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:00.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:00.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:00.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:00 smithi190 ceph-mon[160435]: from='client.46646 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:00.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:42:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:42:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:42:00.889 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:00.918 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:01.354 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:42:01.541 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:01 smithi138 ceph-mon[185886]: pgmap v1450: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 8739/243816 objects degraded (3.584%); 4977/243816 objects misplaced (2.041%); 5.9 MiB/s, 94 objects/s recovering 2023-10-01T09:42:01.542 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2039490185' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:01.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:01 smithi138 ceph-mon[192513]: pgmap v1450: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 8739/243816 objects degraded (3.584%); 4977/243816 objects misplaced (2.041%); 5.9 MiB/s, 94 objects/s recovering 2023-10-01T09:42:01.542 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2039490185' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:01.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:01 smithi190 ceph-mon[160435]: pgmap v1450: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 86 active+clean; 5.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 8739/243816 objects degraded (3.584%); 4977/243816 objects misplaced (2.041%); 5.9 MiB/s, 94 objects/s recovering 2023-10-01T09:42:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2039490185' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:01.961 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:42:02.218 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:02.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:02.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:02.322 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:02.351 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:02.397 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:02.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:02.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:02.501 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:02.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:02 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:02.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:02 smithi138 ceph-mon[192513]: failed to encode map e511 with expected crc 2023-10-01T09:42:02.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:02 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2039490185' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:42:02.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:02 smithi138 ceph-mon[192513]: osdmap e511: 8 total, 8 up, 7 in 2023-10-01T09:42:02.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:02 smithi138 ceph-mon[192513]: failed to encode map e511 with expected crc 2023-10-01T09:42:02.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:02 smithi138 ceph-mon[192513]: failed to encode map e511 with expected crc 2023-10-01T09:42:02.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:02 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:02.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:02 smithi138 ceph-mon[185886]: failed to encode map e511 with expected crc 2023-10-01T09:42:02.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:02 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2039490185' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:42:02.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:02 smithi138 ceph-mon[185886]: osdmap e511: 8 total, 8 up, 7 in 2023-10-01T09:42:02.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:02 smithi138 ceph-mon[185886]: failed to encode map e511 with expected crc 2023-10-01T09:42:02.526 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:02 smithi138 ceph-mon[185886]: failed to encode map e511 with expected crc 2023-10-01T09:42:02.527 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:02.796 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:02.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:02.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:02.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:02 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:02.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:02 smithi190 ceph-mon[160435]: failed to encode map e511 with expected crc 2023-10-01T09:42:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:02 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2039490185' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:42:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:02 smithi190 ceph-mon[160435]: osdmap e511: 8 total, 8 up, 7 in 2023-10-01T09:42:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:02 smithi190 ceph-mon[160435]: failed to encode map e511 with expected crc 2023-10-01T09:42:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:02 smithi190 ceph-mon[160435]: failed to encode map e511 with expected crc 2023-10-01T09:42:02.839 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:02.903 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:02.930 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:02.974 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:02.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:02.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:03.081 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:03.108 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:03.152 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:42:03.155 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:03.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:03.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:03.262 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:03.289 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:03.333 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:03.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:03.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:03.362 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:03 smithi138 ceph-mon[185886]: pgmap v1452: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 8739/242748 objects degraded (3.600%); 4977/242748 objects misplaced (2.050%); 5.6 MiB/s, 88 objects/s recovering 2023-10-01T09:42:03.441 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:03.482 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:03.498 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:42:03.520 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:42:03.539 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:42:03.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:03 smithi190 ceph-mon[160435]: pgmap v1452: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 8739/242748 objects degraded (3.600%); 4977/242748 objects misplaced (2.050%); 5.6 MiB/s, 88 objects/s recovering 2023-10-01T09:42:03.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:03 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8739/242748 objects degraded (3.600%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:03.558 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:42:03.579 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:42:03.579 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:03.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:03.598 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:42:03.617 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:42:03.634 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:42:03.635 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:42:03.635 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:42:03.635 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:42:03.635 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:42:03.636 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:42:03.636 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:42:03.636 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:42:03.636 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:42:03.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:03 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8739/242748 objects degraded (3.600%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:03.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:03 smithi138 ceph-mon[192513]: pgmap v1452: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 8739/242748 objects degraded (3.600%); 4977/242748 objects misplaced (2.050%); 5.6 MiB/s, 88 objects/s recovering 2023-10-01T09:42:03.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:03 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8739/242748 objects degraded (3.600%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:03.684 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:03.711 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:03.756 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:03.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:03.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:03.862 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:03.889 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:04.280 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:42:04.384 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:04.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:04.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:04.488 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:04.515 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:04.516 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:04.566 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:04.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2591139565' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:04.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2591139565' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:04.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:04.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:04.672 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:04.700 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:04.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2591139565' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:04.950 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:42:04.950 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:42:04.950 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-10-01T09:42:04.951 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:42:04.952 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:04.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:04.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:04.956 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:42:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:42:04] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-10-01T09:42:05.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:05.088 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:05.740 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:05 smithi138 ceph-mon[192513]: pgmap v1453: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 8739/242631 objects degraded (3.602%); 4977/242631 objects misplaced (2.051%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T09:42:05.741 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:05 smithi138 ceph-mon[185886]: pgmap v1453: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 8739/242631 objects degraded (3.602%); 4977/242631 objects misplaced (2.051%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T09:42:05.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:05 smithi190 ceph-mon[160435]: pgmap v1453: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 8739/242631 objects degraded (3.602%); 4977/242631 objects misplaced (2.051%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T09:42:05.957 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:05.985 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:42:06.137 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:06.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:06.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:06.241 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:06.271 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:06.486 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:06.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:06.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:06.498 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:42:06.589 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:06.625 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:07.293 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:07.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:07.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:07.397 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:07.433 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:07.459 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:07.565 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:07 smithi138 ceph-mon[185886]: pgmap v1454: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 116 op/s; 8686/241899 objects degraded (3.591%); 4950/241899 objects misplaced (2.046%); 4.1 MiB/s, 65 objects/s recovering 2023-10-01T09:42:07.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:07 smithi138 ceph-mon[192513]: pgmap v1454: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 116 op/s; 8686/241899 objects degraded (3.591%); 4950/241899 objects misplaced (2.046%); 4.1 MiB/s, 65 objects/s recovering 2023-10-01T09:42:07.587 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:07.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:07.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:07.692 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:07.716 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:07 smithi190 ceph-mon[160435]: pgmap v1454: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 116 op/s; 8686/241899 objects degraded (3.591%); 4950/241899 objects misplaced (2.046%); 4.1 MiB/s, 65 objects/s recovering 2023-10-01T09:42:07.718 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:07.734 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:42:07.736 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:42:07.760 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:07.776 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:07.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:07.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:07.881 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:07.916 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:08.480 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:08.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:08.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:08.584 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:08.613 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:08.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:08 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8686/241899 objects degraded (3.591%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:08.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:08 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2265595300' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:42:08.714 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:08 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:42:08.714 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:08 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8686/241899 objects degraded (3.591%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:08.714 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:08 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2265595300' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:42:08.714 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:08 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:42:08.762 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1454,"stamp":"2023-10-01T09:42:06.286494+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5287253963,"num_objects":80633,"num_object_clones":0,"num_object_copies":241899,"num_objects_missing_on_primary":3528,"num_objects_missing":3528,"num_objects_degraded":8686,"num_objects_misplaced":4950,"num_objects_unfound":0,"num_objects_dirty":80633,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":86941,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7562,"num_bytes_recovered":482444303,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87005,"ondisk_log_size":87005,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":21625920,"kb_used_data":16577096,"kb_used_omap":39,"kb_used_meta":5048728,"kb_avail":634590144,"statfs":{"total":671965249536,"available":649820307456,"internally_reserved":0,"allocated":16974946304,"data_stored":16965189229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5169898095},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-72613888,"num_objects":-1108,"num_object_clones":0,"num_object_copies":-3324,"num_objects_missing_on_primary":-80,"num_objects_missing":-80,"num_objects_degraded":-96,"num_objects_misplaced":-606,"num_objects_unfound":0,"num_objects_dirty":-1108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1108,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":622,"num_bytes_recovered":40763392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.524220"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":287,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.771495+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:41:56.771495+0000","last_peered":"2023-10-01T09:41:56.771495+0000","last_clean":"2023-10-01T09:41:56.771495+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:41:56.771495+0000","last_undegraded":"2023-10-01T09:41:56.771495+0000","last_fullsized":"2023-10-01T09:41:56.771495+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":129,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.786049+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:41:56.786049+0000","last_peered":"2023-10-01T09:41:56.786049+0000","last_clean":"2023-10-01T09:41:56.786049+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:41:56.786049+0000","last_undegraded":"2023-10-01T09:41:56.786049+0000","last_fullsized":"2023-10-01T09:41:56.786049+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":510,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.785834+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:41:56.785834+0000","last_peered":"2023-10-01T09:41:56.785834+0000","last_clean":"2023-10-01T09:41:56.785834+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:41:56.785834+0000","last_undegraded":"2023-10-01T09:41:56.785834+0000","last_fullsized":"2023-10-01T09:41:56.785834+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":568,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766939+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:41:56.766939+0000","last_peered":"2023-10-01T09:41:56.766939+0000","last_clean":"2023-10-01T09:41:56.766939+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:41:56.766939+0000","last_undegraded":"2023-10-01T09:41:56.766939+0000","last_fullsized":"2023-10-01T09:41:56.766939+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":449,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766665+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:41:56.766665+0000","last_peered":"2023-10-01T09:41:56.766665+0000","last_clean":"2023-10-01T09:41:56.766665+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:41:56.766665+0000","last_undegraded":"2023-10-01T09:41:56.766665+0000","last_fullsized":"2023-10-01T09:41:56.766665+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":192,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.859280+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:41:56.859280+0000","last_peered":"2023-10-01T09:41:56.859280+0000","last_clean":"2023-10-01T09:41:56.859280+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:41:56.859280+0000","last_undegraded":"2023-10-01T09:41:56.859280+0000","last_fullsized":"2023-10-01T09:41:56.859280+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":568,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.859394+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:41:56.859394+0000","last_peered":"2023-10-01T09:41:56.859394+0000","last_clean":"2023-10-01T09:41:56.859394+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:41:56.859394+0000","last_undegraded":"2023-10-01T09:41:56.859394+0000","last_fullsized":"2023-10-01T09:41:56.859394+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":496,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.785161+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:41:56.785161+0000","last_peered":"2023-10-01T09:41:56.785161+0000","last_clean":"2023-10-01T09:41:56.785161+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:41:56.785161+0000","last_undegraded":"2023-10-01T09:41:56.785161+0000","last_fullsized":"2023-10-01T09:41:56.785161+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":223,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.767056+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:41:56.767056+0000","last_peered":"2023-10-01T09:41:56.767056+0000","last_clean":"2023-10-01T09:41:56.767056+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:41:56.767056+0000","last_undegraded":"2023-10-01T09:41:56.767056+0000","last_fullsized":"2023-10-01T09:41:56.767056+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":29,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.758434+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:41:55.758434+0000","last_peered":"2023-10-01T09:41:55.758434+0000","last_clean":"2023-10-01T09:41:55.758434+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:41:55.758434+0000","last_undegraded":"2023-10-01T09:41:55.758434+0000","last_fullsized":"2023-10-01T09:41:55.758434+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":35,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.757615+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:41:55.757615+0000","last_peered":"2023-10-01T09:41:55.757615+0000","last_clean":"2023-10-01T09:41:55.757615+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:41:55.757615+0000","last_undegraded":"2023-10-01T09:41:55.757615+0000","last_fullsized":"2023-10-01T09:41:55.757615+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":33,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.786279+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:41:56.786279+0000","last_peered":"2023-10-01T09:41:56.786279+0000","last_clean":"2023-10-01T09:41:56.786279+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:41:56.786279+0000","last_undegraded":"2023-10-01T09:41:56.786279+0000","last_fullsized":"2023-10-01T09:41:56.786279+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":412,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.771758+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:41:56.771758+0000","last_peered":"2023-10-01T09:41:56.771758+0000","last_clean":"2023-10-01T09:41:56.771758+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:41:56.771758+0000","last_undegraded":"2023-10-01T09:41:56.771758+0000","last_fullsized":"2023-10-01T09:41:56.771758+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":293,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.771609+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:41:56.771609+0000","last_peered":"2023-10-01T09:41:56.771609+0000","last_clean":"2023-10-01T09:41:56.771609+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:41:56.771609+0000","last_undegraded":"2023-10-01T09:41:56.771609+0000","last_fullsized":"2023-10-01T09:41:56.771609+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":628,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.757366+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:41:55.757366+0000","last_peered":"2023-10-01T09:41:55.757366+0000","last_clean":"2023-10-01T09:41:55.757366+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:41:55.757366+0000","last_undegraded":"2023-10-01T09:41:55.757366+0000","last_fullsized":"2023-10-01T09:41:55.757366+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":365,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.758264+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:41:55.758264+0000","last_peered":"2023-10-01T09:41:55.758264+0000","last_clean":"2023-10-01T09:41:55.758264+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:41:55.758264+0000","last_undegraded":"2023-10-01T09:41:55.758264+0000","last_fullsized":"2023-10-01T09:41:55.758264+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":38,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.757204+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:41:55.757204+0000","last_peered":"2023-10-01T09:41:55.757204+0000","last_clean":"2023-10-01T09:41:55.757204+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:41:55.757204+0000","last_undegraded":"2023-10-01T09:41:55.757204+0000","last_fullsized":"2023-10-01T09:41:55.757204+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":25,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766874+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:41:56.766874+0000","last_peered":"2023-10-01T09:41:56.766874+0000","last_clean":"2023-10-01T09:41:56.766874+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:41:56.766874+0000","last_undegraded":"2023-10-01T09:41:56.766874+0000","last_fullsized":"2023-10-01T09:41:56.766874+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":521,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.758923+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:41:55.758923+0000","last_peered":"2023-10-01T09:41:55.758923+0000","last_clean":"2023-10-01T09:41:55.758923+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:41:55.758923+0000","last_undegraded":"2023-10-01T09:41:55.758923+0000","last_fullsized":"2023-10-01T09:41:55.758923+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":532,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766998+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:41:56.766998+0000","last_peered":"2023-10-01T09:41:56.766998+0000","last_clean":"2023-10-01T09:41:56.766998+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:41:56.766998+0000","last_undegraded":"2023-10-01T09:41:56.766998+0000","last_fullsized":"2023-10-01T09:41:56.766998+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":388,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766584+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:41:56.766584+0000","last_peered":"2023-10-01T09:41:56.766584+0000","last_clean":"2023-10-01T09:41:56.766584+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:41:56.766584+0000","last_undegraded":"2023-10-01T09:41:56.766584+0000","last_fullsized":"2023-10-01T09:41:56.766584+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":38,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.758440+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:41:55.758440+0000","last_peered":"2023-10-01T09:41:55.758440+0000","last_clean":"2023-10-01T09:41:55.758440+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:41:55.758440+0000","last_undegraded":"2023-10-01T09:41:55.758440+0000","last_fullsized":"2023-10-01T09:41:55.758440+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"511'5379","reported_seq":5423,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:04.975189+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:42:04.975189+0000","last_peered":"2023-10-01T09:42:04.975189+0000","last_clean":"2023-10-01T09:42:04.975189+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:42:04.975189+0000","last_undegraded":"2023-10-01T09:42:04.975189+0000","last_fullsized":"2023-10-01T09:42:04.975189+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5379,"log_dups_size":0,"ondisk_log_size":5379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328531968,"num_objects":5013,"num_object_clones":0,"num_object_copies":15039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5379,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":33,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.758286+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:41:55.758286+0000","last_peered":"2023-10-01T09:41:55.758286+0000","last_clean":"2023-10-01T09:41:55.758286+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:41:55.758286+0000","last_undegraded":"2023-10-01T09:41:55.758286+0000","last_fullsized":"2023-10-01T09:41:55.758286+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":231,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766607+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:41:56.766607+0000","last_peered":"2023-10-01T09:41:56.766607+0000","last_clean":"2023-10-01T09:41:56.766607+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:41:56.766607+0000","last_undegraded":"2023-10-01T09:41:56.766607+0000","last_fullsized":"2023-10-01T09:41:56.766607+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"511'5360","reported_seq":5906,"reported_epoch":511,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:42:04.934618+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:42:04.934618+0000","last_peered":"2023-10-01T09:42:04.934618+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:42:04.934618+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330694656,"num_objects":5046,"num_object_clones":0,"num_object_copies":15138,"num_objects_missing_on_primary":3528,"num_objects_missing":3528,"num_objects_degraded":8505,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5360,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":158,"num_bytes_recovered":10354688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1449},{"shards":"0,4,6","objects":3528},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":504,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.758186+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:41:55.758186+0000","last_peered":"2023-10-01T09:41:55.758186+0000","last_clean":"2023-10-01T09:41:55.758186+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:41:55.758186+0000","last_undegraded":"2023-10-01T09:41:55.758186+0000","last_fullsized":"2023-10-01T09:41:55.758186+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":565,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.785759+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:41:56.785759+0000","last_peered":"2023-10-01T09:41:56.785759+0000","last_clean":"2023-10-01T09:41:56.785759+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:41:56.785759+0000","last_undegraded":"2023-10-01T09:41:56.785759+0000","last_fullsized":"2023-10-01T09:41:56.785759+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"511'5351","reported_seq":5396,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.358888+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:42:01.358888+0000","last_peered":"2023-10-01T09:42:01.358888+0000","last_clean":"2023-10-01T09:42:01.358888+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:42:01.358888+0000","last_undegraded":"2023-10-01T09:42:01.358888+0000","last_fullsized":"2023-10-01T09:42:01.358888+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5351,"log_dups_size":0,"ondisk_log_size":5351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332988416,"num_objects":5081,"num_object_clones":0,"num_object_copies":15243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5351,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":590,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.757741+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:41:55.757741+0000","last_peered":"2023-10-01T09:41:55.757741+0000","last_clean":"2023-10-01T09:41:55.757741+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:41:55.757741+0000","last_undegraded":"2023-10-01T09:41:55.757741+0000","last_fullsized":"2023-10-01T09:41:55.757741+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":386,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.786461+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:41:56.786461+0000","last_peered":"2023-10-01T09:41:56.786461+0000","last_clean":"2023-10-01T09:41:56.786461+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:41:56.786461+0000","last_undegraded":"2023-10-01T09:41:56.786461+0000","last_fullsized":"2023-10-01T09:41:56.786461+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"511'5325","reported_seq":11975,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.544697+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:42:01.544697+0000","last_peered":"2023-10-01T09:42:01.544697+0000","last_clean":"2023-10-01T09:42:01.544697+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:42:01.544697+0000","last_undegraded":"2023-10-01T09:42:01.544697+0000","last_fullsized":"2023-10-01T09:42:01.544697+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5325,"log_dups_size":0,"ondisk_log_size":5325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332726272,"num_objects":5077,"num_object_clones":0,"num_object_copies":15231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5325,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":231,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766738+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:41:56.766738+0000","last_peered":"2023-10-01T09:41:56.766738+0000","last_clean":"2023-10-01T09:41:56.766738+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:41:56.766738+0000","last_undegraded":"2023-10-01T09:41:56.766738+0000","last_fullsized":"2023-10-01T09:41:56.766738+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":488,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766907+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:41:56.766907+0000","last_peered":"2023-10-01T09:41:56.766907+0000","last_clean":"2023-10-01T09:41:56.766907+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:41:56.766907+0000","last_undegraded":"2023-10-01T09:41:56.766907+0000","last_fullsized":"2023-10-01T09:41:56.766907+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"511'5428","reported_seq":5473,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:05.025804+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:42:05.025804+0000","last_peered":"2023-10-01T09:42:05.025804+0000","last_clean":"2023-10-01T09:42:05.025804+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:42:05.025804+0000","last_undegraded":"2023-10-01T09:42:05.025804+0000","last_fullsized":"2023-10-01T09:42:05.025804+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5428,"log_dups_size":0,"ondisk_log_size":5428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332922880,"num_objects":5080,"num_object_clones":0,"num_object_copies":15240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5428,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":91,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.771985+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:41:56.771985+0000","last_peered":"2023-10-01T09:41:56.771985+0000","last_clean":"2023-10-01T09:41:56.771985+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:41:56.771985+0000","last_undegraded":"2023-10-01T09:41:56.771985+0000","last_fullsized":"2023-10-01T09:41:56.771985+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":165,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.771896+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:41:56.771896+0000","last_peered":"2023-10-01T09:41:56.771896+0000","last_clean":"2023-10-01T09:41:56.771896+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:41:56.771896+0000","last_undegraded":"2023-10-01T09:41:56.771896+0000","last_fullsized":"2023-10-01T09:41:56.771896+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"511'5354","reported_seq":5406,"reported_epoch":511,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:42:04.948510+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:42:04.948510+0000","last_peered":"2023-10-01T09:42:04.948510+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:42:04.948510+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5354,"log_dups_size":0,"ondisk_log_size":5354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328597504,"num_objects":5014,"num_object_clones":0,"num_object_copies":15042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":171,"num_objects_misplaced":4950,"num_objects_unfound":0,"num_objects_dirty":5014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5354,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4950},{"shards":"4,5","objects":-107}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":409,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.772235+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:41:56.772235+0000","last_peered":"2023-10-01T09:41:56.772235+0000","last_clean":"2023-10-01T09:41:56.772235+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:41:56.772235+0000","last_undegraded":"2023-10-01T09:41:56.772235+0000","last_fullsized":"2023-10-01T09:41:56.772235+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":568,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.785690+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:41:56.785690+0000","last_peered":"2023-10-01T09:41:56.785690+0000","last_clean":"2023-10-01T09:41:56.785690+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:41:56.785690+0000","last_undegraded":"2023-10-01T09:41:56.785690+0000","last_fullsized":"2023-10-01T09:41:56.785690+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"511'5344","reported_seq":5397,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:02.481807+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:42:02.481807+0000","last_peered":"2023-10-01T09:42:02.481807+0000","last_clean":"2023-10-01T09:42:02.481807+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:42:02.481807+0000","last_undegraded":"2023-10-01T09:42:02.481807+0000","last_fullsized":"2023-10-01T09:42:02.481807+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5344,"log_dups_size":0,"ondisk_log_size":5344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329908224,"num_objects":5034,"num_object_clones":0,"num_object_copies":15102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5344,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":405,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.771822+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:41:56.771822+0000","last_peered":"2023-10-01T09:41:56.771822+0000","last_clean":"2023-10-01T09:41:56.771822+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:41:56.771822+0000","last_undegraded":"2023-10-01T09:41:56.771822+0000","last_fullsized":"2023-10-01T09:41:56.771822+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":246,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.772253+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:41:56.772253+0000","last_peered":"2023-10-01T09:41:56.772253+0000","last_clean":"2023-10-01T09:41:56.772253+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:41:56.772253+0000","last_undegraded":"2023-10-01T09:41:56.772253+0000","last_fullsized":"2023-10-01T09:41:56.772253+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"510'5430","reported_seq":5473,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:42:00.550947+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:42:00.550947+0000","last_peered":"2023-10-01T09:42:00.550947+0000","last_clean":"2023-10-01T09:42:00.550947+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:42:00.550947+0000","last_undegraded":"2023-10-01T09:42:00.550947+0000","last_fullsized":"2023-10-01T09:42:00.550947+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5430,"log_dups_size":0,"ondisk_log_size":5430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332136448,"num_objects":5068,"num_object_clones":0,"num_object_copies":15204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5430,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":373,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.787022+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:41:56.787022+0000","last_peered":"2023-10-01T09:41:56.787022+0000","last_clean":"2023-10-01T09:41:56.787022+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:41:56.787022+0000","last_undegraded":"2023-10-01T09:41:56.787022+0000","last_fullsized":"2023-10-01T09:41:56.787022+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":420,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.786200+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:41:56.786200+0000","last_peered":"2023-10-01T09:41:56.786200+0000","last_clean":"2023-10-01T09:41:56.786200+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:41:56.786200+0000","last_undegraded":"2023-10-01T09:41:56.786200+0000","last_fullsized":"2023-10-01T09:41:56.786200+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"510'5226","reported_seq":5269,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.228615+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:42:01.228615+0000","last_peered":"2023-10-01T09:42:01.228615+0000","last_clean":"2023-10-01T09:42:01.228615+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:42:01.228615+0000","last_undegraded":"2023-10-01T09:42:01.228615+0000","last_fullsized":"2023-10-01T09:42:01.228615+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5226,"log_dups_size":0,"ondisk_log_size":5226,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324534272,"num_objects":4952,"num_object_clones":0,"num_object_copies":14856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5226,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":903,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.758164+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:41:55.758164+0000","last_peered":"2023-10-01T09:41:55.758164+0000","last_clean":"2023-10-01T09:41:55.758164+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:41:55.758164+0000","last_undegraded":"2023-10-01T09:41:55.758164+0000","last_fullsized":"2023-10-01T09:41:55.758164+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":241,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766957+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:41:56.766957+0000","last_peered":"2023-10-01T09:41:56.766957+0000","last_clean":"2023-10-01T09:41:56.766957+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:41:56.766957+0000","last_undegraded":"2023-10-01T09:41:56.766957+0000","last_fullsized":"2023-10-01T09:41:56.766957+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.786344+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:41:56.786344+0000","last_peered":"2023-10-01T09:41:56.786344+0000","last_clean":"2023-10-01T09:41:56.786344+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:41:56.786344+0000","last_undegraded":"2023-10-01T09:41:56.786344+0000","last_fullsized":"2023-10-01T09:41:56.786344+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"511'5332","reported_seq":5377,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.811127+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:42:01.811127+0000","last_peered":"2023-10-01T09:42:01.811127+0000","last_clean":"2023-10-01T09:42:01.811127+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:42:01.811127+0000","last_undegraded":"2023-10-01T09:42:01.811127+0000","last_fullsized":"2023-10-01T09:42:01.811127+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5332,"log_dups_size":0,"ondisk_log_size":5332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330170368,"num_objects":5038,"num_object_clones":0,"num_object_copies":15114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5332,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1176,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766743+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:41:56.766743+0000","last_peered":"2023-10-01T09:41:56.766743+0000","last_clean":"2023-10-01T09:41:56.766743+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:41:56.766743+0000","last_undegraded":"2023-10-01T09:41:56.766743+0000","last_fullsized":"2023-10-01T09:41:56.766743+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":386,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.771830+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:41:56.771830+0000","last_peered":"2023-10-01T09:41:56.771830+0000","last_clean":"2023-10-01T09:41:56.771830+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:41:56.771830+0000","last_undegraded":"2023-10-01T09:41:56.771830+0000","last_fullsized":"2023-10-01T09:41:56.771830+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":214,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766838+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:41:56.766838+0000","last_peered":"2023-10-01T09:41:56.766838+0000","last_clean":"2023-10-01T09:41:56.766838+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:41:56.766838+0000","last_undegraded":"2023-10-01T09:41:56.766838+0000","last_fullsized":"2023-10-01T09:41:56.766838+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"511'5375","reported_seq":12640,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:04.942073+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:42:04.942073+0000","last_peered":"2023-10-01T09:42:04.942073+0000","last_clean":"2023-10-01T09:42:04.942073+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:42:04.942073+0000","last_undegraded":"2023-10-01T09:42:04.942073+0000","last_fullsized":"2023-10-01T09:42:04.942073+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5375,"log_dups_size":0,"ondisk_log_size":5375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327745536,"num_objects":5001,"num_object_clones":0,"num_object_copies":15003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5375,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":855,"reported_epoch":510,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:41:56.766433+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:41:56.766433+0000","last_peered":"2023-10-01T09:41:56.766433+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:41:56.766433+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:41:56.766433+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":572,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.787081+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:41:56.787081+0000","last_peered":"2023-10-01T09:41:56.787081+0000","last_clean":"2023-10-01T09:41:56.787081+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:41:56.787081+0000","last_undegraded":"2023-10-01T09:41:56.787081+0000","last_fullsized":"2023-10-01T09:41:56.787081+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":33,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.786073+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:41:56.786073+0000","last_peered":"2023-10-01T09:41:56.786073+0000","last_clean":"2023-10-01T09:41:56.786073+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:41:56.786073+0000","last_undegraded":"2023-10-01T09:41:56.786073+0000","last_fullsized":"2023-10-01T09:41:56.786073+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"511'5313","reported_seq":5358,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:05.048078+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:42:05.048078+0000","last_peered":"2023-10-01T09:42:05.048078+0000","last_clean":"2023-10-01T09:42:05.048078+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:42:05.048078+0000","last_undegraded":"2023-10-01T09:42:05.048078+0000","last_fullsized":"2023-10-01T09:42:05.048078+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5313,"log_dups_size":0,"ondisk_log_size":5313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324993024,"num_objects":4959,"num_object_clones":0,"num_object_copies":14877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5313,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1627,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766781+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:41:56.766781+0000","last_peered":"2023-10-01T09:41:56.766781+0000","last_clean":"2023-10-01T09:41:56.766781+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:41:56.766781+0000","last_undegraded":"2023-10-01T09:41:56.766781+0000","last_fullsized":"2023-10-01T09:41:56.766781+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":540,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.758459+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:41:55.758459+0000","last_peered":"2023-10-01T09:41:55.758459+0000","last_clean":"2023-10-01T09:41:55.758459+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:41:55.758459+0000","last_undegraded":"2023-10-01T09:41:55.758459+0000","last_fullsized":"2023-10-01T09:41:55.758459+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":319,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766790+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:41:56.766790+0000","last_peered":"2023-10-01T09:41:56.766790+0000","last_clean":"2023-10-01T09:41:56.766790+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:41:56.766790+0000","last_undegraded":"2023-10-01T09:41:56.766790+0000","last_fullsized":"2023-10-01T09:41:56.766790+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":309,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.771993+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:41:56.771993+0000","last_peered":"2023-10-01T09:41:56.771993+0000","last_clean":"2023-10-01T09:41:56.771993+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:41:56.771993+0000","last_undegraded":"2023-10-01T09:41:56.771993+0000","last_fullsized":"2023-10-01T09:41:56.771993+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"510'5411","reported_seq":5454,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.263850+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:42:01.263850+0000","last_peered":"2023-10-01T09:42:01.263850+0000","last_clean":"2023-10-01T09:42:01.263850+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:42:01.263850+0000","last_undegraded":"2023-10-01T09:42:01.263850+0000","last_fullsized":"2023-10-01T09:42:01.263850+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5411,"log_dups_size":0,"ondisk_log_size":5411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335609856,"num_objects":5121,"num_object_clones":0,"num_object_copies":15363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5411,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":717,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.758644+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:41:55.758644+0000","last_peered":"2023-10-01T09:41:55.758644+0000","last_clean":"2023-10-01T09:41:55.758644+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:41:55.758644+0000","last_undegraded":"2023-10-01T09:41:55.758644+0000","last_fullsized":"2023-10-01T09:41:55.758644+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":549,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766626+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:41:56.766626+0000","last_peered":"2023-10-01T09:41:56.766626+0000","last_clean":"2023-10-01T09:41:56.766626+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:41:56.766626+0000","last_undegraded":"2023-10-01T09:41:56.766626+0000","last_fullsized":"2023-10-01T09:41:56.766626+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":549,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766811+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:41:56.766811+0000","last_peered":"2023-10-01T09:41:56.766811+0000","last_clean":"2023-10-01T09:41:56.766811+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:41:56.766811+0000","last_undegraded":"2023-10-01T09:41:56.766811+0000","last_fullsized":"2023-10-01T09:41:56.766811+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"511'5388","reported_seq":5434,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.359310+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:42:01.359310+0000","last_peered":"2023-10-01T09:42:01.359310+0000","last_clean":"2023-10-01T09:42:01.359310+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:42:01.359310+0000","last_undegraded":"2023-10-01T09:42:01.359310+0000","last_fullsized":"2023-10-01T09:42:01.359310+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5388,"log_dups_size":0,"ondisk_log_size":5388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332333080,"num_objects":5072,"num_object_clones":0,"num_object_copies":15216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5072,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5388,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":948,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.757816+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:41:55.757816+0000","last_peered":"2023-10-01T09:41:55.757816+0000","last_clean":"2023-10-01T09:41:55.757816+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:41:55.757816+0000","last_undegraded":"2023-10-01T09:41:55.757816+0000","last_fullsized":"2023-10-01T09:41:55.757816+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":568,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766921+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:41:56.766921+0000","last_peered":"2023-10-01T09:41:56.766921+0000","last_clean":"2023-10-01T09:41:56.766921+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:41:56.766921+0000","last_undegraded":"2023-10-01T09:41:56.766921+0000","last_fullsized":"2023-10-01T09:41:56.766921+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":421,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766877+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:41:56.766877+0000","last_peered":"2023-10-01T09:41:56.766877+0000","last_clean":"2023-10-01T09:41:56.766877+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:41:56.766877+0000","last_undegraded":"2023-10-01T09:41:56.766877+0000","last_fullsized":"2023-10-01T09:41:56.766877+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"511'5379","reported_seq":5431,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:06.178309+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:42:06.178309+0000","last_peered":"2023-10-01T09:42:06.178309+0000","last_clean":"2023-10-01T09:42:06.178309+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:42:06.178309+0000","last_undegraded":"2023-10-01T09:42:06.178309+0000","last_fullsized":"2023-10-01T09:42:06.178309+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5379,"log_dups_size":0,"ondisk_log_size":5379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326172672,"num_objects":4977,"num_object_clones":0,"num_object_copies":14931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5379,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":449,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.758582+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:41:55.758582+0000","last_peered":"2023-10-01T09:41:55.758582+0000","last_clean":"2023-10-01T09:41:55.758582+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:41:55.758582+0000","last_undegraded":"2023-10-01T09:41:55.758582+0000","last_fullsized":"2023-10-01T09:41:55.758582+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1142,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.786168+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:41:56.786168+0000","last_peered":"2023-10-01T09:41:56.786168+0000","last_clean":"2023-10-01T09:41:56.786168+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:41:56.786168+0000","last_undegraded":"2023-10-01T09:41:56.786168+0000","last_fullsized":"2023-10-01T09:41:56.786168+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":38,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.758358+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:41:55.758358+0000","last_peered":"2023-10-01T09:41:55.758358+0000","last_clean":"2023-10-01T09:41:55.758358+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:41:55.758358+0000","last_undegraded":"2023-10-01T09:41:55.758358+0000","last_fullsized":"2023-10-01T09:41:55.758358+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"511'5440","reported_seq":5492,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:05.760107+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:42:05.760107+0000","last_peered":"2023-10-01T09:42:05.760107+0000","last_clean":"2023-10-01T09:42:05.760107+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:42:05.760107+0000","last_undegraded":"2023-10-01T09:42:05.760107+0000","last_fullsized":"2023-10-01T09:42:05.760107+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5440,"log_dups_size":0,"ondisk_log_size":5440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332398592,"num_objects":5072,"num_object_clones":0,"num_object_copies":15216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5440,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":42,"reported_epoch":509,"state":"active+clean","last_fresh":"2023-10-01T09:41:55.758368+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:41:55.758368+0000","last_peered":"2023-10-01T09:41:55.758368+0000","last_clean":"2023-10-01T09:41:55.758368+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:41:55.758368+0000","last_undegraded":"2023-10-01T09:41:55.758368+0000","last_fullsized":"2023-10-01T09:41:55.758368+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":527,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766890+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:41:56.766890+0000","last_peered":"2023-10-01T09:41:56.766890+0000","last_clean":"2023-10-01T09:41:56.766890+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:41:56.766890+0000","last_undegraded":"2023-10-01T09:41:56.766890+0000","last_fullsized":"2023-10-01T09:41:56.766890+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":234,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766672+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:41:56.766672+0000","last_peered":"2023-10-01T09:41:56.766672+0000","last_clean":"2023-10-01T09:41:56.766672+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:41:56.766672+0000","last_undegraded":"2023-10-01T09:41:56.766672+0000","last_fullsized":"2023-10-01T09:41:56.766672+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":476,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.767001+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:41:56.767001+0000","last_peered":"2023-10-01T09:41:56.767001+0000","last_clean":"2023-10-01T09:41:56.767001+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:41:56.767001+0000","last_undegraded":"2023-10-01T09:41:56.767001+0000","last_fullsized":"2023-10-01T09:41:56.767001+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":590,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.785922+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:41:56.785922+0000","last_peered":"2023-10-01T09:41:56.785922+0000","last_clean":"2023-10-01T09:41:56.785922+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:41:56.785922+0000","last_undegraded":"2023-10-01T09:41:56.785922+0000","last_fullsized":"2023-10-01T09:41:56.785922+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":246,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766730+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:41:56.766730+0000","last_peered":"2023-10-01T09:41:56.766730+0000","last_clean":"2023-10-01T09:41:56.766730+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:41:56.766730+0000","last_undegraded":"2023-10-01T09:41:56.766730+0000","last_fullsized":"2023-10-01T09:41:56.766730+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":33,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.785629+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:41:56.785629+0000","last_peered":"2023-10-01T09:41:56.785629+0000","last_clean":"2023-10-01T09:41:56.785629+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:41:56.785629+0000","last_undegraded":"2023-10-01T09:41:56.785629+0000","last_fullsized":"2023-10-01T09:41:56.785629+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":246,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766988+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:41:56.766988+0000","last_peered":"2023-10-01T09:41:56.766988+0000","last_clean":"2023-10-01T09:41:56.766988+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:41:56.766988+0000","last_undegraded":"2023-10-01T09:41:56.766988+0000","last_fullsized":"2023-10-01T09:41:56.766988+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":422,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766720+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:41:56.766720+0000","last_peered":"2023-10-01T09:41:56.766720+0000","last_clean":"2023-10-01T09:41:56.766720+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:41:56.766720+0000","last_undegraded":"2023-10-01T09:41:56.766720+0000","last_fullsized":"2023-10-01T09:41:56.766720+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":51,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.364351+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:42:01.364351+0000","last_peered":"2023-10-01T09:42:01.364351+0000","last_clean":"2023-10-01T09:42:01.364351+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:42:01.364351+0000","last_undegraded":"2023-10-01T09:42:01.364351+0000","last_fullsized":"2023-10-01T09:42:01.364351+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":467,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.766616+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:41:56.766616+0000","last_peered":"2023-10-01T09:41:56.766616+0000","last_clean":"2023-10-01T09:41:56.766616+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:41:56.766616+0000","last_undegraded":"2023-10-01T09:41:56.766616+0000","last_fullsized":"2023-10-01T09:41:56.766616+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":666,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.767045+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:41:56.767045+0000","last_peered":"2023-10-01T09:41:56.767045+0000","last_clean":"2023-10-01T09:41:56.767045+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:41:56.767045+0000","last_undegraded":"2023-10-01T09:41:56.767045+0000","last_fullsized":"2023-10-01T09:41:56.767045+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":362,"reported_epoch":510,"state":"active+clean","last_fresh":"2023-10-01T09:41:56.771403+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:41:56.771403+0000","last_peered":"2023-10-01T09:41:56.771403+0000","last_clean":"2023-10-01T09:41:56.771403+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:41:56.771403+0000","last_undegraded":"2023-10-01T09:41:56.771403+0000","last_fullsized":"2023-10-01T09:41:56.771403+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5282463768,"num_objects":80605,"num_object_clones":0,"num_object_copies":241815,"num_objects_missing_on_primary":3528,"num_objects_missing":3528,"num_objects_degraded":8676,"num_objects_misplaced":4950,"num_objects_unfound":0,"num_objects_dirty":80605,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":85835,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7051,"num_bytes_recovered":462094336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16433688576,"data_stored":16433676360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85835,"ondisk_log_size":85835,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430697,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2678912,"kb_used_data":1771704,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":91066240,"statfs":{"total":95995035648,"available":93251829760,"internally_reserved":0,"allocated":1814224896,"data_stored":1813292376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.766,"15min":3.0510000000000002},"min":{"1min":0.30099999999999999,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":6.8550000000000004,"5min":11.525,"15min":108.90000000000001},"last":0.439},{"interface":"front","average":{"1min":0.82099999999999995,"5min":2.2130000000000001,"15min":3.3359999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.6639999999999997,"5min":29.501000000000001,"15min":109.187},"last":0.58399999999999996}]},{"osd":1,"last update":"Sun Oct 1 09:41:05 2023","interfaces":[{"interface":"back","average":{"1min":11.349,"5min":6.492,"15min":5.5579999999999998},"min":{"1min":0.248,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":229.161,"5min":229.161,"15min":229.161},"last":0.47999999999999998},{"interface":"front","average":{"1min":11.241,"5min":6.6150000000000002,"15min":5.6059999999999999},"min":{"1min":0.309,"5min":0.27500000000000002,"15min":0.24399999999999999},"max":{"1min":230.126,"5min":230.126,"15min":230.126},"last":0.46800000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:41:47 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":2.1680000000000001,"15min":2.0169999999999999},"min":{"1min":0.33200000000000002,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":5.3090000000000002,"5min":14.866,"15min":14.901999999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.2829999999999999,"5min":2.1749999999999998,"15min":1.9079999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.6929999999999996,"5min":14.682,"15min":14.682},"last":0.55200000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:41:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":2.6699999999999999,"15min":3.528},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.28199999999999997},"max":{"1min":4.3120000000000003,"5min":35.104999999999997,"15min":164.595},"last":0.5},{"interface":"front","average":{"1min":1.3240000000000001,"5min":2.738,"15min":3.5},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.1440000000000001,"5min":35.118000000000002,"15min":164.60499999999999},"last":0.41199999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.202,"15min":1.0529999999999999},"min":{"1min":0.30499999999999999,"5min":0.27600000000000002,"15min":0.254},"max":{"1min":6.8769999999999998,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.4710000000000001,"5min":1.1739999999999999,"15min":1.0369999999999999},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":5.7439999999999998,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.42599999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":8.516,"5min":2.984,"15min":1.679},"min":{"1min":0.311,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":135.40000000000001,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":8.1989999999999998,"5min":3.1360000000000001,"15min":1.71},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":135.339,"5min":135.339,"15min":135.339},"last":0.34699999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0699999999999998,"5min":1.498,"15min":1.1819999999999999},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":11.58,"5min":11.58,"15min":11.853999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.988,"5min":1.3939999999999999,"15min":1.147},"min":{"1min":0.373,"5min":0.373,"15min":0.32300000000000001},"max":{"1min":13.021000000000001,"5min":14.584,"15min":14.584},"last":0.44700000000000001}]}]},{"osd":6,"up_from":294,"seq":1262720385208,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2606596,"kb_used_data":2264892,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":91138556,"statfs":{"total":95995035648,"available":93325881344,"internally_reserved":0,"allocated":2319249408,"data_stored":2318320515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":7.4020000000000001,"5min":3.9239999999999999,"15min":4.2949999999999999},"min":{"1min":0.219,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":158.48599999999999,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":8.1300000000000008,"5min":4.5199999999999996,"15min":4.335},"min":{"1min":0.34399999999999997,"5min":0.25700000000000001,"15min":0.221},"max":{"1min":158.37700000000001,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.44}]},{"osd":1,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":12.536,"5min":7.2130000000000001,"15min":5.1319999999999997},"min":{"1min":0.38200000000000001,"5min":0.25800000000000001,"15min":0.23899999999999999},"max":{"1min":168.947,"5min":168.947,"15min":168.947},"last":0.51800000000000002},{"interface":"front","average":{"1min":12.667999999999999,"5min":7.1269999999999998,"15min":5.0389999999999997},"min":{"1min":0.32400000000000001,"5min":0.30599999999999999,"15min":0.24399999999999999},"max":{"1min":169.07900000000001,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.53500000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.919,"15min":1.9850000000000001},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":8.657,"5min":9.657,"15min":26.289999999999999},"last":0.498},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.944,"15min":2.0179999999999998},"min":{"1min":0.29399999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.247999999999999,"5min":21.704999999999998,"15min":26.388999999999999},"last":0.376}]},{"osd":3,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.903,"15min":2.5209999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":4.149,"5min":9.0180000000000007,"15min":89.753},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.7110000000000001,"5min":1.9870000000000001,"15min":2.2429999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":8.5470000000000006,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.42399999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":4.9210000000000003,"5min":1.5780000000000001,"15min":1.038},"min":{"1min":0.26300000000000001,"5min":0.246,"15min":0.246},"max":{"1min":63.134,"5min":63.134,"15min":63.134},"last":0.28599999999999998},{"interface":"front","average":{"1min":4.3339999999999996,"5min":1.3959999999999999,"15min":1.056},"min":{"1min":0.35299999999999998,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":62.973999999999997,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.36699999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":1.0089999999999999,"15min":1.0529999999999999},"min":{"1min":0.312,"5min":0.251,"15min":0.251},"max":{"1min":9.9939999999999998,"5min":9.9939999999999998,"15min":12.023999999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":2.1880000000000002,"5min":1.234,"15min":1.1739999999999999},"min":{"1min":0.314,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":15.457000000000001,"5min":15.457000000000001,"15min":15.457000000000001},"last":0.34599999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:41:11 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":0.86099999999999999,"15min":2.1339999999999999},"min":{"1min":0.34100000000000003,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":3.492,"5min":6.4119999999999999,"15min":115.11},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.2310000000000001,"15min":2.2040000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28799999999999998},"max":{"1min":8.6899999999999995,"5min":8.6899999999999995,"15min":114.666},"last":0.38400000000000001}]}]},{"osd":1,"up_from":94,"seq":403726926185,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1784280,"kb_used_data":974288,"kb_used_omap":7,"kb_used_meta":809976,"kb_avail":91960872,"statfs":{"total":95995035648,"available":94167932928,"internally_reserved":0,"allocated":997670912,"data_stored":995670785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":829415847},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":3.5089999999999999,"15min":4.5229999999999997},"min":{"1min":0.19400000000000001,"5min":0.182,"15min":0.182},"max":{"1min":4.423,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.528,"5min":4.0339999999999998,"15min":3.859},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":13.872,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.33800000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:41:37 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":2.6459999999999999,"15min":3.4620000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":2.8740000000000001,"5min":22.891999999999999,"15min":56.573},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.2769999999999999,"5min":3.0059999999999998,"15min":4.2670000000000003},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.20499999999999999},"max":{"1min":6.9710000000000001,"5min":15.845000000000001,"15min":154.197},"last":0.438}]},{"osd":3,"last update":"Sun Oct 1 09:41:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":4.9089999999999998,"15min":4.8239999999999998},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":18.503,"5min":95.305000000000007,"15min":200.56800000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.4119999999999999,"5min":6.7089999999999996,"15min":5.54},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":16.515000000000001,"5min":154.483,"15min":200.63800000000001},"last":0.501}]},{"osd":4,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":2.4870000000000001,"15min":2.3940000000000001},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.5510000000000002,"5min":21.015000000000001,"15min":154.101},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":2.306,"15min":2.7410000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.4209999999999998,"5min":13.773999999999999,"15min":154.56999999999999},"last":0.57099999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":2.4870000000000001,"15min":2.1509999999999998},"min":{"1min":0.32900000000000001,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":5.0410000000000004,"5min":21.219999999999999,"15min":154.22999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.83399999999999996,"5min":2.3239999999999998,"15min":2.661},"min":{"1min":0.311,"5min":0.307,"15min":0.307},"max":{"1min":5.1050000000000004,"5min":13.848000000000001,"15min":154.54400000000001},"last":0.46000000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:41:08 2023","interfaces":[{"interface":"back","average":{"1min":2.363,"5min":4.1890000000000001,"15min":2.9950000000000001},"min":{"1min":0.34300000000000003,"5min":0.32900000000000001,"15min":0.29899999999999999},"max":{"1min":23.672999999999998,"5min":154.12200000000001,"15min":154.12200000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.27,"5min":4.4729999999999999,"15min":2.6829999999999998},"min":{"1min":0.34599999999999997,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":23.382000000000001,"5min":154.16999999999999,"15min":154.16999999999999},"last":0.44800000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.617,"15min":2.7250000000000001},"min":{"1min":0.40500000000000003,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":5.2169999999999996,"5min":21.062999999999999,"15min":154.20699999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.88300000000000001,"5min":2.5390000000000001,"15min":2.9399999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":3.8260000000000001,"5min":13.819000000000001,"15min":154.51599999999999},"last":0.59999999999999998}]}]},{"osd":0,"up_from":89,"seq":382252089719,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4798304,"kb_used_data":3898968,"kb_used_omap":8,"kb_used_meta":899319,"kb_avail":88946848,"statfs":{"total":95995035648,"available":91081572352,"internally_reserved":0,"allocated":3992543232,"data_stored":3990541075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":920903618},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:41:06 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":7.5830000000000002,"15min":4.54},"min":{"1min":0.187,"5min":0.182,"15min":0.17399999999999999},"max":{"1min":8.4939999999999998,"5min":216.821,"15min":216.821},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.095,"5min":9.0879999999999992,"15min":5.7130000000000001},"min":{"1min":0.22,"5min":0.215,"15min":0.21199999999999999},"max":{"1min":8.6229999999999993,"5min":217.02699999999999,"15min":217.02699999999999},"last":0.26700000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:41:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":2.7530000000000001,"15min":2.2509999999999999},"min":{"1min":0.30599999999999999,"5min":0.253,"15min":0.20100000000000001},"max":{"1min":4.9459999999999997,"5min":19.788,"15min":19.788},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.085,"5min":2.7290000000000001,"15min":2.7719999999999998},"min":{"1min":0.29599999999999999,"5min":0.215,"15min":0.189},"max":{"1min":3.4470000000000001,"5min":25.844999999999999,"15min":25.844999999999999},"last":0.34899999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:41:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":5.4249999999999998,"15min":4.7450000000000001},"min":{"1min":0.24199999999999999,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":5.0869999999999997,"5min":222.16499999999999,"15min":222.16499999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.80200000000000005,"5min":5.2400000000000002,"15min":4.9770000000000003},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.219},"max":{"1min":5.8419999999999996,"5min":222.227,"15min":222.227},"last":0.41299999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":2.3559999999999999,"15min":2.4260000000000002},"min":{"1min":0.29899999999999999,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":2.7480000000000002,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.034,"5min":2.698,"15min":2.536},"min":{"1min":0.376,"5min":0.33700000000000002,"15min":0.28299999999999997},"max":{"1min":3.1619999999999999,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.38400000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.494,"5min":2.371,"15min":2.0569999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":5.8769999999999998,"5min":14.515000000000001,"15min":16.808},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.2879999999999998,"15min":1.9259999999999999},"min":{"1min":0.49099999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.8689999999999998,"5min":17.507000000000001,"15min":17.507000000000001},"last":0.48499999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:41:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":2.2509999999999999,"15min":1.897},"min":{"1min":0.38600000000000001,"5min":0.35699999999999998,"15min":0.35499999999999998},"max":{"1min":5.2110000000000003,"5min":17.535,"15min":17.535},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.649,"5min":2.1739999999999999,"15min":2.2679999999999998},"min":{"1min":0.44800000000000001,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":4.2640000000000002,"5min":10.446,"15min":18.521000000000001},"last":0.47099999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.198,"15min":2.1909999999999998},"min":{"1min":0.41799999999999998,"5min":0.377,"15min":0.34499999999999997},"max":{"1min":4.976,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.069,"5min":2.4740000000000002,"15min":2.5459999999999998},"min":{"1min":0.41199999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":4.226,"5min":48.274000000000001,"15min":48.274000000000001},"last":0.51100000000000001}]}]},{"osd":2,"up_from":0,"seq":425201762659,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599132,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2323280,"kb_used_data":1495624,"kb_used_omap":3,"kb_used_meta":827644,"kb_avail":91421872,"statfs":{"total":95995035648,"available":93615996928,"internally_reserved":0,"allocated":1531518976,"data_stored":1529468370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":847508199},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":5.0030000000000001,"15min":5.9409999999999998},"min":{"1min":0.183,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":12.999000000000001,"5min":118.748,"15min":137.34399999999999},"last":0.255},{"interface":"front","average":{"1min":0.97699999999999998,"5min":5.3849999999999998,"15min":5.8490000000000002},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.184},"max":{"1min":10.377000000000001,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.27100000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":8.3849999999999998,"15min":5.7290000000000001},"min":{"1min":0.252,"5min":0.216,"15min":0.20399999999999999},"max":{"1min":14.917999999999999,"5min":153.77000000000001,"15min":162.22999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.359,"5min":8.468,"15min":6.2320000000000002},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":15.074,"5min":151.02199999999999,"15min":162.70699999999999},"last":0.309}]},{"osd":2,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":3.0110000000000001,"15min":2.8839999999999999},"min":{"1min":0.26000000000000001,"5min":0.22800000000000001,"15min":0.20799999999999999},"max":{"1min":2.7970000000000002,"5min":14.254,"15min":43.222999999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.7070000000000001,"15min":3.206},"min":{"1min":0.247,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":1.4239999999999999,"5min":15.523,"15min":78.594999999999999},"last":0.33000000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":4.617,"5min":2.5249999999999999,"15min":2.1760000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":77.793000000000006,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.373},{"interface":"front","average":{"1min":4.8399999999999999,"5min":2.9620000000000002,"15min":2.306},"min":{"1min":0.378,"5min":0.27900000000000003,"15min":0.23899999999999999},"max":{"1min":76.646000000000001,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.49299999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":2.1869999999999998,"15min":3.012},"min":{"1min":0.29399999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.0659999999999998,"5min":10.919,"15min":28.117999999999999},"last":0.443},{"interface":"front","average":{"1min":1.768,"5min":2.6379999999999999,"15min":3.1000000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.1559999999999997,"5min":14.537000000000001,"15min":28.34},"last":0.57199999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:41:19 2023","interfaces":[{"interface":"back","average":{"1min":5.2430000000000003,"5min":3.4009999999999998,"15min":2.4870000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":73.887,"5min":73.887,"15min":73.887},"last":0.60099999999999998},{"interface":"front","average":{"1min":5.8470000000000004,"5min":3.5840000000000001,"15min":2.4780000000000002},"min":{"1min":0.44700000000000001,"5min":0.38600000000000001,"15min":0.32800000000000001},"max":{"1min":74.016999999999996,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.627}]},{"osd":7,"last update":"Sun Oct 1 09:41:01 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":2.8690000000000002,"15min":2.6429999999999998},"min":{"1min":0.46400000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.5570000000000004,"5min":13.085000000000001,"15min":52.006},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.228,"5min":2.0640000000000001,"15min":2.3359999999999999},"min":{"1min":0.371,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":4.5810000000000004,"5min":11.109999999999999,"15min":51.890999999999998},"last":0.51100000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019383,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4906116,"kb_used_data":4011836,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":88839036,"statfs":{"total":95995035648,"available":90971172864,"internally_reserved":0,"allocated":4108120064,"data_stored":4107190726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:53 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":3.5209999999999999,"15min":4.9050000000000002},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":9.6690000000000005,"5min":79.837000000000003,"15min":155.315},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.875,"5min":3.9239999999999999,"15min":5.1550000000000002},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.23899999999999999},"max":{"1min":6.1459999999999999,"5min":78.983000000000004,"15min":155.334},"last":0.30299999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":8.1189999999999998,"15min":7.0599999999999996},"min":{"1min":0.35399999999999998,"5min":0.224,"15min":0.224},"max":{"1min":5.944,"5min":146.929,"15min":190.61199999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.0660000000000001,"5min":8.0500000000000007,"15min":7.8029999999999999},"min":{"1min":0.38500000000000001,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":3.7469999999999999,"5min":146.91,"15min":253.273},"last":0.39400000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:41:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.6279999999999999,"15min":2.6560000000000001},"min":{"1min":0.38300000000000001,"5min":0.29099999999999998,"15min":0.28299999999999997},"max":{"1min":5.702,"5min":10.161,"15min":170.45400000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.5089999999999999,"5min":1.988,"15min":2.371},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":5.6150000000000002,"5min":10.625,"15min":58.771000000000001},"last":0.49199999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.5600000000000001,"15min":2.931},"min":{"1min":0.33600000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.5279999999999996,"5min":15.989000000000001,"15min":140.881},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.825,"15min":3.145},"min":{"1min":0.313,"5min":0.311,"15min":0.311},"max":{"1min":3.786,"5min":15.978999999999999,"15min":140.911},"last":0.42699999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:41:30 2023","interfaces":[{"interface":"back","average":{"1min":7.3849999999999998,"5min":2.387,"15min":1.6259999999999999},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":110.003,"5min":110.003,"15min":110.003},"last":0.443},{"interface":"front","average":{"1min":6.8159999999999998,"5min":2.294,"15min":1.425},"min":{"1min":0.29899999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":108.56399999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.46100000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:41:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":0.97999999999999998,"15min":0.95399999999999996},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":5.5709999999999997,"5min":7.7649999999999997,"15min":11.358000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.381,"5min":1.22,"15min":1.0569999999999999},"min":{"1min":0.317,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":6.1660000000000004,"5min":6.9139999999999997,"15min":10.076000000000001},"last":0.51300000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.2250000000000001,"15min":1.171},"min":{"1min":0.34599999999999997,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":3.8580000000000001,"5min":38.034999999999997,"15min":38.034999999999997},"last":0.38100000000000001},{"interface":"front","average":{"1min":2.0299999999999998,"5min":1.6899999999999999,"15min":1.4530000000000001},"min":{"1min":0.377,"5min":0.35899999999999999,"15min":0.28699999999999998},"max":{"1min":12.092000000000001,"5min":37.723999999999997,"15min":37.723999999999997},"last":0.56599999999999995}]}]},{"osd":5,"up_from":379,"seq":1627792605290,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2528432,"kb_used_data":2159784,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":91216720,"statfs":{"total":95995035648,"available":93405921280,"internally_reserved":0,"allocated":2211618816,"data_stored":2210705382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":2.508,"15min":2.335},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.6379999999999999,"5min":13.385999999999999,"15min":84.340999999999994},"last":0.378},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.0030000000000001,"15min":2.1909999999999998},"min":{"1min":0.28299999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.2279999999999998,"5min":12.445,"15min":84.356999999999999},"last":0.33100000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":8.1319999999999997,"5min":9.8100000000000005,"15min":5.9340000000000002},"min":{"1min":0.29299999999999998,"5min":0.255,"15min":0.255},"max":{"1min":106.56999999999999,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.49399999999999999},{"interface":"front","average":{"1min":8.1590000000000007,"5min":10.111000000000001,"15min":5.8540000000000001},"min":{"1min":0.27500000000000002,"5min":0.25900000000000001,"15min":0.24199999999999999},"max":{"1min":106.596,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.52300000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.653,"5min":2.1789999999999998,"15min":1.71},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.3470000000000004,"5min":14.199999999999999,"15min":14.199999999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":2.0379999999999998,"5min":2.0670000000000002,"15min":1.7370000000000001},"min":{"1min":0.373,"5min":0.31,"15min":0.27100000000000002},"max":{"1min":5.782,"5min":14.430999999999999,"15min":14.430999999999999},"last":0.47399999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":2.2730000000000001,"15min":1.5700000000000001},"min":{"1min":0.35399999999999998,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":3.4409999999999998,"5min":14.004,"15min":14.004},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.782,"5min":2.3610000000000002,"15min":1.7150000000000001},"min":{"1min":0.35299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.4400000000000004,"5min":13.991,"15min":13.991},"last":0.64500000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1320000000000001,"5min":2.3300000000000001,"15min":1.5},"min":{"1min":0.36699999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.9459999999999997,"5min":122.971,"15min":122.971},"last":0.501},{"interface":"front","average":{"1min":2.6520000000000001,"5min":2.3849999999999998,"15min":1.8859999999999999},"min":{"1min":0.32100000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.445,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.45200000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":1.335,"15min":1.1160000000000001},"min":{"1min":0.33900000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.7750000000000004,"5min":12.545,"15min":12.545},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.8560000000000001,"5min":1.0529999999999999,"15min":1.431},"min":{"1min":0.36899999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.4160000000000004,"5min":6.4729999999999999,"15min":14.193},"last":0.41399999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5459999999999998,"5min":1.238,"15min":1.379},"min":{"1min":0.46800000000000003,"5min":0.307,"15min":0.307},"max":{"1min":5.758,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.63},{"interface":"front","average":{"1min":2.2210000000000001,"5min":1.3740000000000001,"15min":1.121},"min":{"1min":0.41199999999999998,"5min":0.318,"15min":0.318},"max":{"1min":5.2300000000000004,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.70399999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3981185024,"data_stored":3981180952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":986972160,"data_stored":986972160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1327693824,"data_stored":1327693824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4098166784,"data_stored":4098162712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2074017792,"data_stored":2074017792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2311458816,"data_stored":2311454744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1654194176,"data_stored":1654194176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:42:08.769 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:42:08.770 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:42:08.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:08 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8686/241899 objects degraded (3.591%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:08.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:08 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2265595300' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:42:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:08 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:42:09.414 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:09.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:09.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:09.518 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:09.547 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:09.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:09 smithi138 ceph-mon[192513]: from='client.36636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:09.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:09 smithi138 ceph-mon[192513]: pgmap v1455: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 8686/241104 objects degraded (3.603%); 4950/241104 objects misplaced (2.053%); 3.9 MiB/s, 62 objects/s recovering 2023-10-01T09:42:09.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:09 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:09.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:09 smithi138 ceph-mon[192513]: failed to encode map e512 with expected crc 2023-10-01T09:42:09.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:09 smithi138 ceph-mon[192513]: failed to encode map e512 with expected crc 2023-10-01T09:42:09.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:09 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:42:09.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:09 smithi138 ceph-mon[192513]: osdmap e512: 8 total, 8 up, 7 in 2023-10-01T09:42:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:09 smithi138 ceph-mon[185886]: from='client.36636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:09.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:09 smithi138 ceph-mon[185886]: pgmap v1455: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 8686/241104 objects degraded (3.603%); 4950/241104 objects misplaced (2.053%); 3.9 MiB/s, 62 objects/s recovering 2023-10-01T09:42:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:09 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:09 smithi138 ceph-mon[185886]: failed to encode map e512 with expected crc 2023-10-01T09:42:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:09 smithi138 ceph-mon[185886]: failed to encode map e512 with expected crc 2023-10-01T09:42:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:09 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:42:09.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:09 smithi138 ceph-mon[185886]: osdmap e512: 8 total, 8 up, 7 in 2023-10-01T09:42:09.603 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:09.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:09.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:09.710 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:09.735 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:09 smithi190 ceph-mon[160435]: from='client.36636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:09.735 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:09 smithi190 ceph-mon[160435]: pgmap v1455: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 8686/241104 objects degraded (3.603%); 4950/241104 objects misplaced (2.053%); 3.9 MiB/s, 62 objects/s recovering 2023-10-01T09:42:09.735 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:09 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:09.735 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:09 smithi190 ceph-mon[160435]: failed to encode map e512 with expected crc 2023-10-01T09:42:09.735 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:09 smithi190 ceph-mon[160435]: failed to encode map e512 with expected crc 2023-10-01T09:42:09.735 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:09 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:42:09.735 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:09 smithi190 ceph-mon[160435]: osdmap e512: 8 total, 8 up, 7 in 2023-10-01T09:42:09.737 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:09.786 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:09.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:09.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:09.895 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:09.922 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:09.975 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:09.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:09.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:10.082 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:10.110 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:10.457 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:10.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:10.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:10.561 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:10.591 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:10.629 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:42:10.630 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:42:10.637 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:10.641 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:42:10.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:10.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:10.731 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:10.743 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:10.781 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:10.843 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:10.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:10.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:10.853 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:42:10.854 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:42:10.854 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:42:10.854 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:42:10.854 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:42:10.949 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:10.976 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:11.020 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:11.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:11.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:11.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:11.156 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:11.446 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:42:11.447 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:42:11.447 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:42:11.452 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:42:11.471 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:42:11.503 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:42:11.511 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:42:11.531 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:42:11.532 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:42:11.532 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:42:11.533 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:42:11.533 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:42:11.533 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:42:11.533 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:42:11.533 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:42:11.533 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:42:11.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:11 smithi138 ceph-mon[185886]: pgmap v1457: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 8659/240357 objects degraded (3.603%); 4899/240357 objects misplaced (2.038%); 573 KiB/s, 8 objects/s recovering 2023-10-01T09:42:11.674 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/740093793' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:11.674 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:11 smithi138 ceph-mon[192513]: pgmap v1457: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 8659/240357 objects degraded (3.603%); 4899/240357 objects misplaced (2.038%); 573 KiB/s, 8 objects/s recovering 2023-10-01T09:42:11.674 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/740093793' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:11.797 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:42:11.802 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:11.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:11.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:11 smithi190 ceph-mon[160435]: pgmap v1457: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 105 op/s; 8659/240357 objects degraded (3.603%); 4899/240357 objects misplaced (2.038%); 573 KiB/s, 8 objects/s recovering 2023-10-01T09:42:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/740093793' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:11.907 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:11.934 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:11.978 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:11.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:11.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:12.086 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:12.112 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:12.162 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:12.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:12.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:12.267 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:12.269 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:42:12.298 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:12.347 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:12.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:12.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:12.453 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:12.482 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:12.935 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:12.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:12.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:12.998 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:13.041 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:13.070 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:13.123 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:13.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:13.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:13.229 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:13.255 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:13.267 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:13.300 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:13.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:13.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:13.405 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:13.436 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:13.489 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:13.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:13.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:13.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:13 smithi190 ceph-mon[160435]: pgmap v1458: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 119 op/s; 8659/239169 objects degraded (3.620%); 4899/239169 objects misplaced (2.048%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:42:13.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:13 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8659/239169 objects degraded (3.620%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:13.594 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:13.618 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:42:13.618 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 8659/239169 objects degraded (3.620%), 3 pgs degraded","count":3},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1954,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":512,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":79723,"data_bytes":5227616203,"bytes_used":21884162048,"bytes_avail":650081087488,"bytes_total":671965249536,"degraded_objects":8659,"degraded_total":239169,"degraded_ratio":0.036204524833904059,"misplaced_objects":4899,"misplaced_total":239169,"misplaced_ratio":0.020483423855098281,"recovering_objects_per_sec":7,"recovering_bytes_per_sec":524162,"recovering_keys_per_sec":0,"num_objects_recovered":80,"num_bytes_recovered":5242880,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":119},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":33,"modified":"2023-10-01T09:41:58.284615+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:42:13.620 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:13.648 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:13 smithi138 ceph-mon[185886]: pgmap v1458: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 119 op/s; 8659/239169 objects degraded (3.620%); 4899/239169 objects misplaced (2.048%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:42:13.648 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:13 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8659/239169 objects degraded (3.620%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:13.648 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:13 smithi138 ceph-mon[192513]: pgmap v1458: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 119 op/s; 8659/239169 objects degraded (3.620%); 4899/239169 objects misplaced (2.048%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:42:13.648 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:13 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8659/239169 objects degraded (3.620%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:13.667 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:13.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:13.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:13.772 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:13.799 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:13.854 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:13.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:13.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:13.959 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:13.990 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:14.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1983338183' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:42:14.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1983338183' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:42:14.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1983338183' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:42:14.843 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:42:15.008 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:42:15.008 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:42:15.143 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:15.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:15.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:15.248 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:15.276 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:15.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:42:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:42:14] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-01T09:42:15.327 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:15.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:15.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:15.433 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:15.459 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:15.508 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:15.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:15.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:15.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:15 smithi190 ceph-mon[160435]: pgmap v1459: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 8659/239076 objects degraded (3.622%); 4899/239076 objects misplaced (2.049%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:42:15.615 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:15.642 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:15 smithi138 ceph-mon[192513]: pgmap v1459: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 8659/239076 objects degraded (3.622%); 4899/239076 objects misplaced (2.049%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:42:15.642 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:15 smithi138 ceph-mon[185886]: pgmap v1459: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 8659/239076 objects degraded (3.622%); 4899/239076 objects misplaced (2.049%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:42:15.644 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:42:15.644 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:42:15.645 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:16.245 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:16.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:16.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:16.335 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:42:16.349 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:16.362 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:16.381 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:16.419 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:16.627 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:16.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:16.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:16.732 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:16.748 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:42:16.760 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:16.812 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:16.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:16.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:16.917 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:16.944 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:16.994 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:16.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:16.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:17.098 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:17.127 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:17.419 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:42:17.423 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:17.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:17.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:17.527 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:17.555 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:17.598 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:17.599 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:17 smithi190 ceph-mon[160435]: pgmap v1460: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 8612/238140 objects degraded (3.616%); 4864/238140 objects misplaced (2.042%); 505 KiB/s, 7 objects/s recovering 2023-10-01T09:42:17.599 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:17 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3813725291' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:17.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:17.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:17.701 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:17.729 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:17.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:17 smithi138 ceph-mon[185886]: pgmap v1460: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 8612/238140 objects degraded (3.616%); 4864/238140 objects misplaced (2.042%); 505 KiB/s, 7 objects/s recovering 2023-10-01T09:42:17.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:17 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3813725291' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:17 smithi138 ceph-mon[192513]: pgmap v1460: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 8612/238140 objects degraded (3.616%); 4864/238140 objects misplaced (2.042%); 505 KiB/s, 7 objects/s recovering 2023-10-01T09:42:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:17 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3813725291' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:18.021 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:18.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:18.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:18.130 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:18.163 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:18.260 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:18.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:18.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:18.364 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:18.393 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:18.451 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:42:18.451 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:42:18.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:18 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:42:18.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:18 smithi190 ceph-mon[160435]: failed to encode map e513 with expected crc 2023-10-01T09:42:18.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:18 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3813725291' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:42:18.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:18 smithi190 ceph-mon[160435]: osdmap e513: 8 total, 8 up, 7 in 2023-10-01T09:42:18.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:18 smithi190 ceph-mon[160435]: failed to encode map e513 with expected crc 2023-10-01T09:42:18.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:18 smithi190 ceph-mon[160435]: failed to encode map e513 with expected crc 2023-10-01T09:42:18.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:18 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8612/238140 objects degraded (3.616%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:18.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:18 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:42:18.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:18 smithi138 ceph-mon[185886]: failed to encode map e513 with expected crc 2023-10-01T09:42:18.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:18 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3813725291' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:42:18.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:18 smithi138 ceph-mon[185886]: osdmap e513: 8 total, 8 up, 7 in 2023-10-01T09:42:18.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:18 smithi138 ceph-mon[185886]: failed to encode map e513 with expected crc 2023-10-01T09:42:18.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:18 smithi138 ceph-mon[185886]: failed to encode map e513 with expected crc 2023-10-01T09:42:18.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:18 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8612/238140 objects degraded (3.616%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:18.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:18 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:42:18.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:18 smithi138 ceph-mon[192513]: failed to encode map e513 with expected crc 2023-10-01T09:42:18.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:18 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3813725291' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:42:18.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:18 smithi138 ceph-mon[192513]: osdmap e513: 8 total, 8 up, 7 in 2023-10-01T09:42:18.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:18 smithi138 ceph-mon[192513]: failed to encode map e513 with expected crc 2023-10-01T09:42:18.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:18 smithi138 ceph-mon[192513]: failed to encode map e513 with expected crc 2023-10-01T09:42:18.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:18 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8612/238140 objects degraded (3.616%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:18.695 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:18.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:18.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:18.803 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:18.835 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:18.881 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:18.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:18.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:18.986 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:19.016 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:19.118 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:19.119 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:42:19.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:19.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:19.138 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:42:19.156 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:42:19.176 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:42:19.194 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:42:19.212 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:42:19.223 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:19.231 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:42:19.251 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:42:19.251 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:19.252 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:42:19.254 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:42:19.254 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:42:19.254 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:42:19.254 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:42:19.254 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:42:19.254 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:42:19.254 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:42:19.255 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:42:19.300 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:19.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:19.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:19.398 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:42:19.400 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:42:19.404 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:19.432 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:19.477 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:19.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:19.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:19.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:19 smithi190 ceph-mon[160435]: pgmap v1462: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 8612/237234 objects degraded (3.630%); 4864/237234 objects misplaced (2.050%); 530 KiB/s, 8 objects/s recovering 2023-10-01T09:42:19.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:19 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1269175343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:19.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:19 smithi138 ceph-mon[192513]: pgmap v1462: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 8612/237234 objects degraded (3.630%); 4864/237234 objects misplaced (2.050%); 530 KiB/s, 8 objects/s recovering 2023-10-01T09:42:19.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:19 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1269175343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:19 smithi138 ceph-mon[185886]: pgmap v1462: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 8612/237234 objects degraded (3.630%); 4864/237234 objects misplaced (2.050%); 530 KiB/s, 8 objects/s recovering 2023-10-01T09:42:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1269175343' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:19.582 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:19.612 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:19.660 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:42:19.955 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:19.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:19.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:20.059 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:20.085 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:20.131 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:20.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:20.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:20.235 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:20.267 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:20.312 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:20.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:20.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:20.416 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:20.443 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:20.488 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:20.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:20.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:20.502 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1462,"stamp":"2023-10-01T09:42:18.289545+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5185345483,"num_objects":79078,"num_object_clones":0,"num_object_copies":237234,"num_objects_missing_on_primary":3447,"num_objects_missing":3447,"num_objects_degraded":8612,"num_objects_misplaced":4864,"num_objects_unfound":0,"num_objects_dirty":79078,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":88496,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7641,"num_bytes_recovered":487621647,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88560,"ondisk_log_size":88560,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":21095172,"kb_used_data":16058572,"kb_used_omap":39,"kb_used_meta":5036440,"kb_avail":635120892,"statfs":{"total":671965249536,"available":650363793408,"internally_reserved":0,"allocated":16443977728,"data_stored":16434214238,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5157315183},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-84541440,"num_objects":-1290,"num_object_clones":0,"num_object_copies":-3870,"num_objects_missing_on_primary":-81,"num_objects_missing":-81,"num_objects_degraded":-74,"num_objects_misplaced":-86,"num_objects_unfound":0,"num_objects_dirty":-1290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1290,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":79,"num_bytes_recovered":5177344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.532608"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":288,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.359772+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:42:01.359772+0000","last_peered":"2023-10-01T09:42:01.359772+0000","last_clean":"2023-10-01T09:42:01.359772+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:42:01.359772+0000","last_undegraded":"2023-10-01T09:42:01.359772+0000","last_fullsized":"2023-10-01T09:42:01.359772+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":130,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.364194+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:42:01.364194+0000","last_peered":"2023-10-01T09:42:01.364194+0000","last_clean":"2023-10-01T09:42:01.364194+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:42:01.364194+0000","last_undegraded":"2023-10-01T09:42:01.364194+0000","last_fullsized":"2023-10-01T09:42:01.364194+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":511,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.363834+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:42:01.363834+0000","last_peered":"2023-10-01T09:42:01.363834+0000","last_clean":"2023-10-01T09:42:01.363834+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:42:01.363834+0000","last_undegraded":"2023-10-01T09:42:01.363834+0000","last_fullsized":"2023-10-01T09:42:01.363834+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":569,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.359193+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:42:01.359193+0000","last_peered":"2023-10-01T09:42:01.359193+0000","last_clean":"2023-10-01T09:42:01.359193+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:42:01.359193+0000","last_undegraded":"2023-10-01T09:42:01.359193+0000","last_fullsized":"2023-10-01T09:42:01.359193+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":450,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.358857+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:42:01.358857+0000","last_peered":"2023-10-01T09:42:01.358857+0000","last_clean":"2023-10-01T09:42:01.358857+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:42:01.358857+0000","last_undegraded":"2023-10-01T09:42:01.358857+0000","last_fullsized":"2023-10-01T09:42:01.358857+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":193,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.474063+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:42:01.474063+0000","last_peered":"2023-10-01T09:42:01.474063+0000","last_clean":"2023-10-01T09:42:01.474063+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:42:01.474063+0000","last_undegraded":"2023-10-01T09:42:01.474063+0000","last_fullsized":"2023-10-01T09:42:01.474063+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":569,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.474135+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:42:01.474135+0000","last_peered":"2023-10-01T09:42:01.474135+0000","last_clean":"2023-10-01T09:42:01.474135+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:42:01.474135+0000","last_undegraded":"2023-10-01T09:42:01.474135+0000","last_fullsized":"2023-10-01T09:42:01.474135+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":497,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.372569+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:42:01.372569+0000","last_peered":"2023-10-01T09:42:01.372569+0000","last_clean":"2023-10-01T09:42:01.372569+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:42:01.372569+0000","last_undegraded":"2023-10-01T09:42:01.372569+0000","last_fullsized":"2023-10-01T09:42:01.372569+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":224,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.367535+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:42:01.367535+0000","last_peered":"2023-10-01T09:42:01.367535+0000","last_clean":"2023-10-01T09:42:01.367535+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:42:01.367535+0000","last_undegraded":"2023-10-01T09:42:01.367535+0000","last_fullsized":"2023-10-01T09:42:01.367535+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":31,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.353309+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:42:01.353309+0000","last_peered":"2023-10-01T09:42:01.353309+0000","last_clean":"2023-10-01T09:42:01.353309+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:42:01.353309+0000","last_undegraded":"2023-10-01T09:42:01.353309+0000","last_fullsized":"2023-10-01T09:42:01.353309+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":37,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.353736+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:42:01.353736+0000","last_peered":"2023-10-01T09:42:01.353736+0000","last_clean":"2023-10-01T09:42:01.353736+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:42:01.353736+0000","last_undegraded":"2023-10-01T09:42:01.353736+0000","last_fullsized":"2023-10-01T09:42:01.353736+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":34,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.363932+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:42:01.363932+0000","last_peered":"2023-10-01T09:42:01.363932+0000","last_clean":"2023-10-01T09:42:01.363932+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:42:01.363932+0000","last_undegraded":"2023-10-01T09:42:01.363932+0000","last_fullsized":"2023-10-01T09:42:01.363932+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":413,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.360447+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:42:01.360447+0000","last_peered":"2023-10-01T09:42:01.360447+0000","last_clean":"2023-10-01T09:42:01.360447+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:42:01.360447+0000","last_undegraded":"2023-10-01T09:42:01.360447+0000","last_fullsized":"2023-10-01T09:42:01.360447+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":294,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.359924+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:42:01.359924+0000","last_peered":"2023-10-01T09:42:01.359924+0000","last_clean":"2023-10-01T09:42:01.359924+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:42:01.359924+0000","last_undegraded":"2023-10-01T09:42:01.359924+0000","last_fullsized":"2023-10-01T09:42:01.359924+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":630,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.352904+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:42:01.352904+0000","last_peered":"2023-10-01T09:42:01.352904+0000","last_clean":"2023-10-01T09:42:01.352904+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:42:01.352904+0000","last_undegraded":"2023-10-01T09:42:01.352904+0000","last_fullsized":"2023-10-01T09:42:01.352904+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":367,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.353219+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:42:01.353219+0000","last_peered":"2023-10-01T09:42:01.353219+0000","last_clean":"2023-10-01T09:42:01.353219+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:42:01.353219+0000","last_undegraded":"2023-10-01T09:42:01.353219+0000","last_fullsized":"2023-10-01T09:42:01.353219+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":40,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.352752+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:42:01.352752+0000","last_peered":"2023-10-01T09:42:01.352752+0000","last_clean":"2023-10-01T09:42:01.352752+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:42:01.352752+0000","last_undegraded":"2023-10-01T09:42:01.352752+0000","last_fullsized":"2023-10-01T09:42:01.352752+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":26,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.362382+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:42:01.362382+0000","last_peered":"2023-10-01T09:42:01.362382+0000","last_clean":"2023-10-01T09:42:01.362382+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:42:01.362382+0000","last_undegraded":"2023-10-01T09:42:01.362382+0000","last_fullsized":"2023-10-01T09:42:01.362382+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":523,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.353538+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:42:01.353538+0000","last_peered":"2023-10-01T09:42:01.353538+0000","last_clean":"2023-10-01T09:42:01.353538+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:42:01.353538+0000","last_undegraded":"2023-10-01T09:42:01.353538+0000","last_fullsized":"2023-10-01T09:42:01.353538+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":533,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.359273+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:42:01.359273+0000","last_peered":"2023-10-01T09:42:01.359273+0000","last_clean":"2023-10-01T09:42:01.359273+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:42:01.359273+0000","last_undegraded":"2023-10-01T09:42:01.359273+0000","last_fullsized":"2023-10-01T09:42:01.359273+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":389,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.358788+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:42:01.358788+0000","last_peered":"2023-10-01T09:42:01.358788+0000","last_clean":"2023-10-01T09:42:01.358788+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:42:01.358788+0000","last_undegraded":"2023-10-01T09:42:01.358788+0000","last_fullsized":"2023-10-01T09:42:01.358788+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":40,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.353250+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:42:01.353250+0000","last_peered":"2023-10-01T09:42:01.353250+0000","last_clean":"2023-10-01T09:42:01.353250+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:42:01.353250+0000","last_undegraded":"2023-10-01T09:42:01.353250+0000","last_fullsized":"2023-10-01T09:42:01.353250+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"512'5461","reported_seq":5506,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:14.564842+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:42:14.564842+0000","last_peered":"2023-10-01T09:42:14.564842+0000","last_clean":"2023-10-01T09:42:14.564842+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:42:14.564842+0000","last_undegraded":"2023-10-01T09:42:14.564842+0000","last_fullsized":"2023-10-01T09:42:14.564842+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5461,"log_dups_size":0,"ondisk_log_size":5461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323158016,"num_objects":4931,"num_object_clones":0,"num_object_copies":14793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5461,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":35,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.353089+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:42:01.353089+0000","last_peered":"2023-10-01T09:42:01.353089+0000","last_clean":"2023-10-01T09:42:01.353089+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:42:01.353089+0000","last_undegraded":"2023-10-01T09:42:01.353089+0000","last_fullsized":"2023-10-01T09:42:01.353089+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":232,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.362368+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:42:01.362368+0000","last_peered":"2023-10-01T09:42:01.362368+0000","last_clean":"2023-10-01T09:42:01.362368+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:42:01.362368+0000","last_undegraded":"2023-10-01T09:42:01.362368+0000","last_fullsized":"2023-10-01T09:42:01.362368+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"512'5440","reported_seq":6226,"reported_epoch":512,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:42:15.123662+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:42:15.123662+0000","last_peered":"2023-10-01T09:42:15.123662+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:42:15.123662+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5440,"log_dups_size":0,"ondisk_log_size":5440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325451776,"num_objects":4966,"num_object_clones":0,"num_object_copies":14898,"num_objects_missing_on_primary":3447,"num_objects_missing":3447,"num_objects_degraded":8345,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5440,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":237,"num_bytes_recovered":15532032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1451},{"shards":"0,4,6","objects":3447},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":506,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.353788+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:42:01.353788+0000","last_peered":"2023-10-01T09:42:01.353788+0000","last_clean":"2023-10-01T09:42:01.353788+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:42:01.353788+0000","last_undegraded":"2023-10-01T09:42:01.353788+0000","last_fullsized":"2023-10-01T09:42:01.353788+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":566,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.363786+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:42:01.363786+0000","last_peered":"2023-10-01T09:42:01.363786+0000","last_clean":"2023-10-01T09:42:01.363786+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:42:01.363786+0000","last_undegraded":"2023-10-01T09:42:01.363786+0000","last_fullsized":"2023-10-01T09:42:01.363786+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"512'5472","reported_seq":5519,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:16.677598+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:42:16.677598+0000","last_peered":"2023-10-01T09:42:16.677598+0000","last_clean":"2023-10-01T09:42:16.677598+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:42:16.677598+0000","last_undegraded":"2023-10-01T09:42:16.677598+0000","last_fullsized":"2023-10-01T09:42:16.677598+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5472,"log_dups_size":0,"ondisk_log_size":5472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325058560,"num_objects":4960,"num_object_clones":0,"num_object_copies":14880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5472,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":592,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.353917+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:42:01.353917+0000","last_peered":"2023-10-01T09:42:01.353917+0000","last_clean":"2023-10-01T09:42:01.353917+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:42:01.353917+0000","last_undegraded":"2023-10-01T09:42:01.353917+0000","last_fullsized":"2023-10-01T09:42:01.353917+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":387,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.363913+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:42:01.363913+0000","last_peered":"2023-10-01T09:42:01.363913+0000","last_clean":"2023-10-01T09:42:01.363913+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:42:01.363913+0000","last_undegraded":"2023-10-01T09:42:01.363913+0000","last_fullsized":"2023-10-01T09:42:01.363913+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"512'5444","reported_seq":12096,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:16.625590+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:42:16.625590+0000","last_peered":"2023-10-01T09:42:16.625590+0000","last_clean":"2023-10-01T09:42:16.625590+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:42:16.625590+0000","last_undegraded":"2023-10-01T09:42:16.625590+0000","last_fullsized":"2023-10-01T09:42:16.625590+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5444,"log_dups_size":0,"ondisk_log_size":5444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324927488,"num_objects":4958,"num_object_clones":0,"num_object_copies":14874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5444,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":232,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.362853+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:42:01.362853+0000","last_peered":"2023-10-01T09:42:01.362853+0000","last_clean":"2023-10-01T09:42:01.362853+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:42:01.362853+0000","last_undegraded":"2023-10-01T09:42:01.362853+0000","last_fullsized":"2023-10-01T09:42:01.362853+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":489,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.359414+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:42:01.359414+0000","last_peered":"2023-10-01T09:42:01.359414+0000","last_clean":"2023-10-01T09:42:01.359414+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:42:01.359414+0000","last_undegraded":"2023-10-01T09:42:01.359414+0000","last_fullsized":"2023-10-01T09:42:01.359414+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"512'5495","reported_seq":5542,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:15.161676+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:42:15.161676+0000","last_peered":"2023-10-01T09:42:15.161676+0000","last_clean":"2023-10-01T09:42:15.161676+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:42:15.161676+0000","last_undegraded":"2023-10-01T09:42:15.161676+0000","last_fullsized":"2023-10-01T09:42:15.161676+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5495,"log_dups_size":0,"ondisk_log_size":5495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328531968,"num_objects":5013,"num_object_clones":0,"num_object_copies":15039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5495,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":92,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.360247+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:42:01.360247+0000","last_peered":"2023-10-01T09:42:01.360247+0000","last_clean":"2023-10-01T09:42:01.360247+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:42:01.360247+0000","last_undegraded":"2023-10-01T09:42:01.360247+0000","last_fullsized":"2023-10-01T09:42:01.360247+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":166,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.360508+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:42:01.360508+0000","last_peered":"2023-10-01T09:42:01.360508+0000","last_clean":"2023-10-01T09:42:01.360508+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:42:01.360508+0000","last_undegraded":"2023-10-01T09:42:01.360508+0000","last_fullsized":"2023-10-01T09:42:01.360508+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"512'5440","reported_seq":5493,"reported_epoch":512,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:42:14.984588+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:42:14.984588+0000","last_peered":"2023-10-01T09:42:14.984588+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:42:14.984588+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5440,"log_dups_size":0,"ondisk_log_size":5440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322961408,"num_objects":4928,"num_object_clones":0,"num_object_copies":14784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":257,"num_objects_misplaced":4864,"num_objects_unfound":0,"num_objects_dirty":4928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5440,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4864},{"shards":"4,5","objects":-193}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":410,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.359896+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:42:01.359896+0000","last_peered":"2023-10-01T09:42:01.359896+0000","last_clean":"2023-10-01T09:42:01.359896+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:42:01.359896+0000","last_undegraded":"2023-10-01T09:42:01.359896+0000","last_fullsized":"2023-10-01T09:42:01.359896+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":570,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.758932+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:42:08.758932+0000","last_peered":"2023-10-01T09:42:08.758932+0000","last_clean":"2023-10-01T09:42:08.758932+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:42:08.758932+0000","last_undegraded":"2023-10-01T09:42:08.758932+0000","last_fullsized":"2023-10-01T09:42:08.758932+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"512'5443","reported_seq":5498,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:16.868209+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:42:16.868209+0000","last_peered":"2023-10-01T09:42:16.868209+0000","last_clean":"2023-10-01T09:42:16.868209+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:42:16.868209+0000","last_undegraded":"2023-10-01T09:42:16.868209+0000","last_fullsized":"2023-10-01T09:42:16.868209+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5443,"log_dups_size":0,"ondisk_log_size":5443,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323420160,"num_objects":4935,"num_object_clones":0,"num_object_copies":14805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5443,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":406,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.360013+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:42:01.360013+0000","last_peered":"2023-10-01T09:42:01.360013+0000","last_clean":"2023-10-01T09:42:01.360013+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:42:01.360013+0000","last_undegraded":"2023-10-01T09:42:01.360013+0000","last_fullsized":"2023-10-01T09:42:01.360013+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":247,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.360814+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:42:01.360814+0000","last_peered":"2023-10-01T09:42:01.360814+0000","last_clean":"2023-10-01T09:42:01.360814+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:42:01.360814+0000","last_undegraded":"2023-10-01T09:42:01.360814+0000","last_fullsized":"2023-10-01T09:42:01.360814+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"512'5563","reported_seq":5608,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:16.275255+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:42:16.275255+0000","last_peered":"2023-10-01T09:42:16.275255+0000","last_clean":"2023-10-01T09:42:16.275255+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:42:16.275255+0000","last_undegraded":"2023-10-01T09:42:16.275255+0000","last_fullsized":"2023-10-01T09:42:16.275255+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5563,"log_dups_size":0,"ondisk_log_size":5563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323420160,"num_objects":4935,"num_object_clones":0,"num_object_copies":14805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5563,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":374,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.363713+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:42:01.363713+0000","last_peered":"2023-10-01T09:42:01.363713+0000","last_clean":"2023-10-01T09:42:01.363713+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:42:01.363713+0000","last_undegraded":"2023-10-01T09:42:01.363713+0000","last_fullsized":"2023-10-01T09:42:01.363713+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":421,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.364370+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:42:01.364370+0000","last_peered":"2023-10-01T09:42:01.364370+0000","last_clean":"2023-10-01T09:42:01.364370+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:42:01.364370+0000","last_undegraded":"2023-10-01T09:42:01.364370+0000","last_fullsized":"2023-10-01T09:42:01.364370+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"512'5335","reported_seq":5380,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:15.986525+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:42:15.986525+0000","last_peered":"2023-10-01T09:42:15.986525+0000","last_clean":"2023-10-01T09:42:15.986525+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:42:15.986525+0000","last_undegraded":"2023-10-01T09:42:15.986525+0000","last_fullsized":"2023-10-01T09:42:15.986525+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5335,"log_dups_size":0,"ondisk_log_size":5335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317390848,"num_objects":4843,"num_object_clones":0,"num_object_copies":14529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5335,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":905,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.353087+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:42:01.353087+0000","last_peered":"2023-10-01T09:42:01.353087+0000","last_clean":"2023-10-01T09:42:01.353087+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:42:01.353087+0000","last_undegraded":"2023-10-01T09:42:01.353087+0000","last_fullsized":"2023-10-01T09:42:01.353087+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":242,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.362549+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:42:01.362549+0000","last_peered":"2023-10-01T09:42:01.362549+0000","last_clean":"2023-10-01T09:42:01.362549+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:42:01.362549+0000","last_undegraded":"2023-10-01T09:42:01.362549+0000","last_fullsized":"2023-10-01T09:42:01.362549+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":34,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.363795+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:42:01.363795+0000","last_peered":"2023-10-01T09:42:01.363795+0000","last_clean":"2023-10-01T09:42:01.363795+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:42:01.363795+0000","last_undegraded":"2023-10-01T09:42:01.363795+0000","last_fullsized":"2023-10-01T09:42:01.363795+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"512'5434","reported_seq":5480,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:16.685599+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:42:16.685599+0000","last_peered":"2023-10-01T09:42:16.685599+0000","last_clean":"2023-10-01T09:42:16.685599+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:42:16.685599+0000","last_undegraded":"2023-10-01T09:42:16.685599+0000","last_fullsized":"2023-10-01T09:42:16.685599+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5434,"log_dups_size":0,"ondisk_log_size":5434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323485696,"num_objects":4936,"num_object_clones":0,"num_object_copies":14808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5434,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1177,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.362548+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:42:01.362548+0000","last_peered":"2023-10-01T09:42:01.362548+0000","last_clean":"2023-10-01T09:42:01.362548+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:42:01.362548+0000","last_undegraded":"2023-10-01T09:42:01.362548+0000","last_fullsized":"2023-10-01T09:42:01.362548+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":387,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.360124+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:42:01.360124+0000","last_peered":"2023-10-01T09:42:01.360124+0000","last_clean":"2023-10-01T09:42:01.360124+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:42:01.360124+0000","last_undegraded":"2023-10-01T09:42:01.360124+0000","last_fullsized":"2023-10-01T09:42:01.360124+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":215,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.363014+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:42:01.363014+0000","last_peered":"2023-10-01T09:42:01.363014+0000","last_clean":"2023-10-01T09:42:01.363014+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:42:01.363014+0000","last_undegraded":"2023-10-01T09:42:01.363014+0000","last_fullsized":"2023-10-01T09:42:01.363014+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"512'5451","reported_seq":12717,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:14.621268+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:42:14.621268+0000","last_peered":"2023-10-01T09:42:14.621268+0000","last_clean":"2023-10-01T09:42:14.621268+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:42:14.621268+0000","last_undegraded":"2023-10-01T09:42:14.621268+0000","last_fullsized":"2023-10-01T09:42:14.621268+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5451,"log_dups_size":0,"ondisk_log_size":5451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322764800,"num_objects":4925,"num_object_clones":0,"num_object_copies":14775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5451,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":856,"reported_epoch":511,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:42:01.360627+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:42:01.360627+0000","last_peered":"2023-10-01T09:42:01.360627+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:42:01.360627+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:42:01.360627+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":574,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.758719+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:42:08.758719+0000","last_peered":"2023-10-01T09:42:08.758719+0000","last_clean":"2023-10-01T09:42:08.758719+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:42:08.758719+0000","last_undegraded":"2023-10-01T09:42:08.758719+0000","last_fullsized":"2023-10-01T09:42:08.758719+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":34,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.364311+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:42:01.364311+0000","last_peered":"2023-10-01T09:42:01.364311+0000","last_clean":"2023-10-01T09:42:01.364311+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:42:01.364311+0000","last_undegraded":"2023-10-01T09:42:01.364311+0000","last_fullsized":"2023-10-01T09:42:01.364311+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"512'5383","reported_seq":5429,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:14.992625+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:42:14.992625+0000","last_peered":"2023-10-01T09:42:14.992625+0000","last_clean":"2023-10-01T09:42:14.992625+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:42:14.992625+0000","last_undegraded":"2023-10-01T09:42:14.992625+0000","last_fullsized":"2023-10-01T09:42:14.992625+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5383,"log_dups_size":0,"ondisk_log_size":5383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320405504,"num_objects":4889,"num_object_clones":0,"num_object_copies":14667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5383,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1628,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.365468+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:42:01.365468+0000","last_peered":"2023-10-01T09:42:01.365468+0000","last_clean":"2023-10-01T09:42:01.365468+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:42:01.365468+0000","last_undegraded":"2023-10-01T09:42:01.365468+0000","last_fullsized":"2023-10-01T09:42:01.365468+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":542,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.352620+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:42:01.352620+0000","last_peered":"2023-10-01T09:42:01.352620+0000","last_clean":"2023-10-01T09:42:01.352620+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:42:01.352620+0000","last_undegraded":"2023-10-01T09:42:01.352620+0000","last_fullsized":"2023-10-01T09:42:01.352620+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":320,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.362553+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:42:01.362553+0000","last_peered":"2023-10-01T09:42:01.362553+0000","last_clean":"2023-10-01T09:42:01.362553+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:42:01.362553+0000","last_undegraded":"2023-10-01T09:42:01.362553+0000","last_fullsized":"2023-10-01T09:42:01.362553+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":310,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.360679+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:42:01.360679+0000","last_peered":"2023-10-01T09:42:01.360679+0000","last_clean":"2023-10-01T09:42:01.360679+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:42:01.360679+0000","last_undegraded":"2023-10-01T09:42:01.360679+0000","last_fullsized":"2023-10-01T09:42:01.360679+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"512'5520","reported_seq":5565,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:16.264079+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:42:16.264079+0000","last_peered":"2023-10-01T09:42:16.264079+0000","last_clean":"2023-10-01T09:42:16.264079+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:42:16.264079+0000","last_undegraded":"2023-10-01T09:42:16.264079+0000","last_fullsized":"2023-10-01T09:42:16.264079+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5520,"log_dups_size":0,"ondisk_log_size":5520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328466432,"num_objects":5012,"num_object_clones":0,"num_object_copies":15036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5520,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":719,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.352779+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:42:01.352779+0000","last_peered":"2023-10-01T09:42:01.352779+0000","last_clean":"2023-10-01T09:42:01.352779+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:42:01.352779+0000","last_undegraded":"2023-10-01T09:42:01.352779+0000","last_fullsized":"2023-10-01T09:42:01.352779+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":551,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.762212+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:42:08.762212+0000","last_peered":"2023-10-01T09:42:08.762212+0000","last_clean":"2023-10-01T09:42:08.762212+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:42:08.762212+0000","last_undegraded":"2023-10-01T09:42:08.762212+0000","last_fullsized":"2023-10-01T09:42:08.762212+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":550,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.359012+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:42:01.359012+0000","last_peered":"2023-10-01T09:42:01.359012+0000","last_clean":"2023-10-01T09:42:01.359012+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:42:01.359012+0000","last_undegraded":"2023-10-01T09:42:01.359012+0000","last_fullsized":"2023-10-01T09:42:01.359012+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"512'5501","reported_seq":5548,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:16.583349+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:42:16.583349+0000","last_peered":"2023-10-01T09:42:16.583349+0000","last_clean":"2023-10-01T09:42:16.583349+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:42:16.583349+0000","last_undegraded":"2023-10-01T09:42:16.583349+0000","last_fullsized":"2023-10-01T09:42:16.583349+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5501,"log_dups_size":0,"ondisk_log_size":5501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324927512,"num_objects":4959,"num_object_clones":0,"num_object_copies":14877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4959,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5501,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":950,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.354002+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:42:01.354002+0000","last_peered":"2023-10-01T09:42:01.354002+0000","last_clean":"2023-10-01T09:42:01.354002+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:42:01.354002+0000","last_undegraded":"2023-10-01T09:42:01.354002+0000","last_fullsized":"2023-10-01T09:42:01.354002+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":569,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.362936+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:42:01.362936+0000","last_peered":"2023-10-01T09:42:01.362936+0000","last_clean":"2023-10-01T09:42:01.362936+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:42:01.362936+0000","last_undegraded":"2023-10-01T09:42:01.362936+0000","last_fullsized":"2023-10-01T09:42:01.362936+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":422,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.365562+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:42:01.365562+0000","last_peered":"2023-10-01T09:42:01.365562+0000","last_clean":"2023-10-01T09:42:01.365562+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:42:01.365562+0000","last_undegraded":"2023-10-01T09:42:01.365562+0000","last_fullsized":"2023-10-01T09:42:01.365562+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"512'5477","reported_seq":5530,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:16.082894+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:42:16.082894+0000","last_peered":"2023-10-01T09:42:16.082894+0000","last_clean":"2023-10-01T09:42:16.082894+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:42:16.082894+0000","last_undegraded":"2023-10-01T09:42:16.082894+0000","last_fullsized":"2023-10-01T09:42:16.082894+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5477,"log_dups_size":0,"ondisk_log_size":5477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319750144,"num_objects":4879,"num_object_clones":0,"num_object_copies":14637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5477,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":451,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.353455+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:42:01.353455+0000","last_peered":"2023-10-01T09:42:01.353455+0000","last_clean":"2023-10-01T09:42:01.353455+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:42:01.353455+0000","last_undegraded":"2023-10-01T09:42:01.353455+0000","last_fullsized":"2023-10-01T09:42:01.353455+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1143,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.364437+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:42:01.364437+0000","last_peered":"2023-10-01T09:42:01.364437+0000","last_clean":"2023-10-01T09:42:01.364437+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:42:01.364437+0000","last_undegraded":"2023-10-01T09:42:01.364437+0000","last_fullsized":"2023-10-01T09:42:01.364437+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":40,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.352532+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:42:01.352532+0000","last_peered":"2023-10-01T09:42:01.352532+0000","last_clean":"2023-10-01T09:42:01.352532+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:42:01.352532+0000","last_undegraded":"2023-10-01T09:42:01.352532+0000","last_fullsized":"2023-10-01T09:42:01.352532+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"512'5531","reported_seq":5584,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:16.046861+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:42:16.046861+0000","last_peered":"2023-10-01T09:42:16.046861+0000","last_clean":"2023-10-01T09:42:16.046861+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:42:16.046861+0000","last_undegraded":"2023-10-01T09:42:16.046861+0000","last_fullsized":"2023-10-01T09:42:16.046861+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5531,"log_dups_size":0,"ondisk_log_size":5531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326434816,"num_objects":4981,"num_object_clones":0,"num_object_copies":14943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5531,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":44,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.353174+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:42:01.353174+0000","last_peered":"2023-10-01T09:42:01.353174+0000","last_clean":"2023-10-01T09:42:01.353174+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:42:01.353174+0000","last_undegraded":"2023-10-01T09:42:01.353174+0000","last_fullsized":"2023-10-01T09:42:01.353174+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":528,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.359111+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:42:01.359111+0000","last_peered":"2023-10-01T09:42:01.359111+0000","last_clean":"2023-10-01T09:42:01.359111+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:42:01.359111+0000","last_undegraded":"2023-10-01T09:42:01.359111+0000","last_fullsized":"2023-10-01T09:42:01.359111+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":235,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.362531+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:42:01.362531+0000","last_peered":"2023-10-01T09:42:01.362531+0000","last_clean":"2023-10-01T09:42:01.362531+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:42:01.362531+0000","last_undegraded":"2023-10-01T09:42:01.362531+0000","last_fullsized":"2023-10-01T09:42:01.362531+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":477,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.359527+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:42:01.359527+0000","last_peered":"2023-10-01T09:42:01.359527+0000","last_clean":"2023-10-01T09:42:01.359527+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:42:01.359527+0000","last_undegraded":"2023-10-01T09:42:01.359527+0000","last_fullsized":"2023-10-01T09:42:01.359527+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":591,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.364189+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:42:01.364189+0000","last_peered":"2023-10-01T09:42:01.364189+0000","last_clean":"2023-10-01T09:42:01.364189+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:42:01.364189+0000","last_undegraded":"2023-10-01T09:42:01.364189+0000","last_fullsized":"2023-10-01T09:42:01.364189+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":247,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.362791+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:42:01.362791+0000","last_peered":"2023-10-01T09:42:01.362791+0000","last_clean":"2023-10-01T09:42:01.362791+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:42:01.362791+0000","last_undegraded":"2023-10-01T09:42:01.362791+0000","last_fullsized":"2023-10-01T09:42:01.362791+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":34,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.363892+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:42:01.363892+0000","last_peered":"2023-10-01T09:42:01.363892+0000","last_clean":"2023-10-01T09:42:01.363892+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:42:01.363892+0000","last_undegraded":"2023-10-01T09:42:01.363892+0000","last_fullsized":"2023-10-01T09:42:01.363892+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":247,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.362640+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:42:01.362640+0000","last_peered":"2023-10-01T09:42:01.362640+0000","last_clean":"2023-10-01T09:42:01.362640+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:42:01.362640+0000","last_undegraded":"2023-10-01T09:42:01.362640+0000","last_fullsized":"2023-10-01T09:42:01.362640+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":423,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.359767+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:42:01.359767+0000","last_peered":"2023-10-01T09:42:01.359767+0000","last_clean":"2023-10-01T09:42:01.359767+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:42:01.359767+0000","last_undegraded":"2023-10-01T09:42:01.359767+0000","last_fullsized":"2023-10-01T09:42:01.359767+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":52,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:05.241337+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:42:05.241337+0000","last_peered":"2023-10-01T09:42:05.241337+0000","last_clean":"2023-10-01T09:42:05.241337+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:42:05.241337+0000","last_undegraded":"2023-10-01T09:42:05.241337+0000","last_fullsized":"2023-10-01T09:42:05.241337+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":468,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.360710+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:42:01.360710+0000","last_peered":"2023-10-01T09:42:01.360710+0000","last_clean":"2023-10-01T09:42:01.360710+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:42:01.360710+0000","last_undegraded":"2023-10-01T09:42:01.360710+0000","last_fullsized":"2023-10-01T09:42:01.360710+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":668,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.762703+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:42:08.762703+0000","last_peered":"2023-10-01T09:42:08.762703+0000","last_clean":"2023-10-01T09:42:08.762703+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:42:08.762703+0000","last_undegraded":"2023-10-01T09:42:08.762703+0000","last_fullsized":"2023-10-01T09:42:08.762703+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":363,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-10-01T09:42:01.359664+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:42:01.359664+0000","last_peered":"2023-10-01T09:42:01.359664+0000","last_clean":"2023-10-01T09:42:01.359664+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:42:01.359664+0000","last_undegraded":"2023-10-01T09:42:01.359664+0000","last_fullsized":"2023-10-01T09:42:01.359664+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5180555288,"num_objects":79050,"num_object_clones":0,"num_object_copies":237150,"num_objects_missing_on_primary":3447,"num_objects_missing":3447,"num_objects_degraded":8602,"num_objects_misplaced":4864,"num_objects_unfound":0,"num_objects_dirty":79050,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":87390,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7130,"num_bytes_recovered":467271680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16093491200,"data_stored":16093478984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87390,"ondisk_log_size":87390,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430700,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2584912,"kb_used_data":1677704,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":91160240,"statfs":{"total":95995035648,"available":93348085760,"internally_reserved":0,"allocated":1717968896,"data_stored":1717035728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.766,"15min":3.0510000000000002},"min":{"1min":0.30099999999999999,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":6.8550000000000004,"5min":11.525,"15min":108.90000000000001},"last":0.38},{"interface":"front","average":{"1min":0.82099999999999995,"5min":2.2130000000000001,"15min":3.3359999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.6639999999999997,"5min":29.501000000000001,"15min":109.187},"last":0.44800000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:42:06 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":5.0410000000000004,"15min":5.492},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":6.8289999999999997,"5min":229.161,"15min":229.161},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.92100000000000004,"5min":4.9939999999999998,"15min":5.5490000000000004},"min":{"1min":0.28899999999999998,"5min":0.27500000000000002,"15min":0.24399999999999999},"max":{"1min":5.7240000000000002,"5min":230.126,"15min":230.126},"last":0.41899999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:41:47 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":2.1680000000000001,"15min":2.0169999999999999},"min":{"1min":0.33200000000000002,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":5.3090000000000002,"5min":14.866,"15min":14.901999999999999},"last":0.439},{"interface":"front","average":{"1min":1.2829999999999999,"5min":2.1749999999999998,"15min":1.9079999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.6929999999999996,"5min":14.682,"15min":14.682},"last":0.51800000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:42:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.9410000000000001,"15min":2.8610000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":5.7750000000000004,"5min":35.104999999999997,"15min":80.352999999999994},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.004,"5min":2,"15min":2.8359999999999999},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.548,"5min":35.118000000000002,"15min":80.364000000000004},"last":0.40300000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.202,"15min":1.0529999999999999},"min":{"1min":0.30499999999999999,"5min":0.27600000000000002,"15min":0.254},"max":{"1min":6.8769999999999998,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.4710000000000001,"5min":1.1739999999999999,"15min":1.0369999999999999},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":5.7439999999999998,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.47799999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":8.516,"5min":2.984,"15min":1.679},"min":{"1min":0.311,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":135.40000000000001,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":8.1989999999999998,"5min":3.1360000000000001,"15min":1.71},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":135.339,"5min":135.339,"15min":135.339},"last":0.53900000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0699999999999998,"5min":1.498,"15min":1.1819999999999999},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":11.58,"5min":11.58,"15min":11.853999999999999},"last":0.496},{"interface":"front","average":{"1min":1.988,"5min":1.3939999999999999,"15min":1.147},"min":{"1min":0.373,"5min":0.373,"15min":0.32300000000000001},"max":{"1min":13.021000000000001,"5min":14.584,"15min":14.584},"last":0.501}]}]},{"osd":6,"up_from":294,"seq":1262720385211,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2541900,"kb_used_data":2212484,"kb_used_omap":6,"kb_used_meta":329401,"kb_avail":91203252,"statfs":{"total":95995035648,"available":93392130048,"internally_reserved":0,"allocated":2265583616,"data_stored":2264654399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":337306669},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":3.786,"15min":4.1879999999999997},"min":{"1min":0.249,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":1.8400000000000001,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.374},{"interface":"front","average":{"1min":0.76300000000000001,"5min":4.3559999999999999,"15min":4.2889999999999997},"min":{"1min":0.249,"5min":0.249,"15min":0.221},"max":{"1min":2.9969999999999999,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.36699999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":5.3860000000000001,"15min":5.0590000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":3.0289999999999999,"5min":168.947,"15min":168.947},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.79100000000000004,"5min":5.375,"15min":4.9690000000000003},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":2.6030000000000002,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.41599999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.8109999999999999,"15min":1.8420000000000001},"min":{"1min":0.34499999999999997,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":3.0489999999999999,"5min":9.657,"15min":26.289999999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.885,"15min":1.859},"min":{"1min":0.32600000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.6739999999999999,"5min":21.704999999999998,"15min":26.388999999999999},"last":0.44900000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":1.597,"15min":2.4020000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.9450000000000001,"5min":9.0180000000000007,"15min":89.753},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.073,"5min":1.9330000000000001,"15min":2.2189999999999999},"min":{"1min":0.318,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":3.7160000000000002,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.501}]},{"osd":4,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.665,"15min":1.0800000000000001},"min":{"1min":0.28599999999999998,"5min":0.253,"15min":0.246},"max":{"1min":4.0250000000000004,"5min":63.134,"15min":63.134},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.518,"15min":1.0960000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.28299999999999997},"max":{"1min":7.4039999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.45700000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":1.0089999999999999,"15min":1.0529999999999999},"min":{"1min":0.312,"5min":0.251,"15min":0.251},"max":{"1min":9.9939999999999998,"5min":9.9939999999999998,"15min":12.023999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.1880000000000002,"5min":1.234,"15min":1.1739999999999999},"min":{"1min":0.314,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":15.457000000000001,"5min":15.457000000000001,"15min":15.457000000000001},"last":0.38800000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.82999999999999996,"15min":1.802},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":2.0419999999999998,"5min":6.4119999999999999,"15min":115.11},"last":0.437},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.282,"15min":1.8959999999999999},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.28799999999999998},"max":{"1min":3.2679999999999998,"5min":8.6899999999999995,"15min":114.666},"last":0.42799999999999999}]}]},{"osd":1,"up_from":94,"seq":403726926187,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1770596,"kb_used_data":960604,"kb_used_omap":7,"kb_used_meta":809976,"kb_avail":91974556,"statfs":{"total":95995035648,"available":94181945344,"internally_reserved":0,"allocated":983658496,"data_stored":981654222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":829415847},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":3.5089999999999999,"15min":4.5229999999999997},"min":{"1min":0.19400000000000001,"5min":0.182,"15min":0.182},"max":{"1min":4.423,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.528,"5min":4.0339999999999998,"15min":3.859},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":13.872,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.32100000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:41:37 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":2.6459999999999999,"15min":3.4620000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":2.8740000000000001,"5min":22.891999999999999,"15min":56.573},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.2769999999999999,"5min":3.0059999999999998,"15min":4.2670000000000003},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.20499999999999999},"max":{"1min":6.9710000000000001,"5min":15.845000000000001,"15min":154.197},"last":0.34999999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":3.9089999999999998,"15min":4.4429999999999996},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":3.1949999999999998,"5min":95.305000000000007,"15min":200.56800000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.68799999999999994,"5min":4.3550000000000004,"15min":5.2569999999999997},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":4.7839999999999998,"5min":98.033000000000001,"15min":200.63800000000001},"last":0.40200000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":2.4870000000000001,"15min":2.3940000000000001},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.5510000000000002,"5min":21.015000000000001,"15min":154.101},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":2.306,"15min":2.7410000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.4209999999999998,"5min":13.773999999999999,"15min":154.56999999999999},"last":0.375}]},{"osd":5,"last update":"Sun Oct 1 09:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":2.4870000000000001,"15min":2.1509999999999998},"min":{"1min":0.32900000000000001,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":5.0410000000000004,"5min":21.219999999999999,"15min":154.22999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.83399999999999996,"5min":2.3239999999999998,"15min":2.661},"min":{"1min":0.311,"5min":0.307,"15min":0.307},"max":{"1min":5.1050000000000004,"5min":13.848000000000001,"15min":154.54400000000001},"last":0.496}]},{"osd":6,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.1669999999999998,"15min":2.8370000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.3660000000000001,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.626,"5min":2.4769999999999999,"15min":2.5899999999999999},"min":{"1min":0.36699999999999999,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":1.575,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.52600000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.617,"15min":2.7250000000000001},"min":{"1min":0.40500000000000003,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":5.2169999999999996,"5min":21.062999999999999,"15min":154.20699999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.88300000000000001,"5min":2.5390000000000001,"15min":2.9399999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":3.8260000000000001,"5min":13.819000000000001,"15min":154.51599999999999},"last":0.51800000000000002}]}]},{"osd":0,"up_from":89,"seq":382252089722,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4713768,"kb_used_data":3826720,"kb_used_omap":8,"kb_used_meta":887031,"kb_avail":89031384,"statfs":{"total":95995035648,"available":91168137216,"internally_reserved":0,"allocated":3918561280,"data_stored":3916558799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":908320706},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:42:08 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":3.9860000000000002,"15min":4.4039999999999999},"min":{"1min":0.20399999999999999,"5min":0.187,"15min":0.17399999999999999},"max":{"1min":1.661,"5min":185.78399999999999,"15min":216.821},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.437,"5min":5.9020000000000001,"15min":5.4400000000000004},"min":{"1min":0.249,"5min":0.215,"15min":0.215},"max":{"1min":1.5589999999999999,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.29199999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:41:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":2.7530000000000001,"15min":2.2509999999999999},"min":{"1min":0.30599999999999999,"5min":0.253,"15min":0.20100000000000001},"max":{"1min":4.9459999999999997,"5min":19.788,"15min":19.788},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.085,"5min":2.7290000000000001,"15min":2.7719999999999998},"min":{"1min":0.29599999999999999,"5min":0.215,"15min":0.189},"max":{"1min":3.4470000000000001,"5min":25.844999999999999,"15min":25.844999999999999},"last":0.34799999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:42:08 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":2.1749999999999998,"15min":4.524},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.20899999999999999},"max":{"1min":1.706,"5min":15.35,"15min":222.16499999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.873,"5min":2.5219999999999998,"15min":4.7089999999999996},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":2.859,"5min":14.545999999999999,"15min":222.227},"last":0.31}]},{"osd":4,"last update":"Sun Oct 1 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":2.3559999999999999,"15min":2.4260000000000002},"min":{"1min":0.29899999999999999,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":2.7480000000000002,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.034,"5min":2.698,"15min":2.536},"min":{"1min":0.376,"5min":0.33700000000000002,"15min":0.28299999999999997},"max":{"1min":3.1619999999999999,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.54100000000000004}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.494,"5min":2.371,"15min":2.0569999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":5.8769999999999998,"5min":14.515000000000001,"15min":16.808},"last":0.377},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.2879999999999998,"15min":1.9259999999999999},"min":{"1min":0.49099999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.8689999999999998,"5min":17.507000000000001,"15min":17.507000000000001},"last":0.48299999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:41:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":2.2509999999999999,"15min":1.897},"min":{"1min":0.38600000000000001,"5min":0.35699999999999998,"15min":0.35499999999999998},"max":{"1min":5.2110000000000003,"5min":17.535,"15min":17.535},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.649,"5min":2.1739999999999999,"15min":2.2679999999999998},"min":{"1min":0.44800000000000001,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":4.2640000000000002,"5min":10.446,"15min":18.521000000000001},"last":0.51700000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.198,"15min":2.1909999999999998},"min":{"1min":0.41799999999999998,"5min":0.377,"15min":0.34499999999999997},"max":{"1min":4.976,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.069,"5min":2.4740000000000002,"15min":2.5459999999999998},"min":{"1min":0.41199999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":4.226,"5min":48.274000000000001,"15min":48.274000000000001},"last":0.496}]}]},{"osd":2,"up_from":0,"seq":425201762661,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599135,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2189464,"kb_used_data":1357712,"kb_used_omap":3,"kb_used_meta":831740,"kb_avail":91555688,"statfs":{"total":95995035648,"available":93753024512,"internally_reserved":0,"allocated":1390297088,"data_stored":1388246158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":851702503},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":5.9850000000000003,"5min":5.5869999999999997,"15min":6.0170000000000003},"min":{"1min":0.216,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":64.504000000000005,"5min":118.748,"15min":137.34399999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":6.024,"5min":5.7830000000000004,"15min":6.0090000000000003},"min":{"1min":0.24299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":64.448999999999998,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.27000000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":5.149,"15min":5.4130000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":6.0250000000000004,"5min":108.84999999999999,"15min":162.22999999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.60199999999999998,"5min":5.3860000000000001,"15min":5.718},"min":{"1min":0.23100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.931,"5min":109.13200000000001,"15min":162.70699999999999},"last":0.23100000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":2.5070000000000001,"15min":2.5},"min":{"1min":0.27000000000000002,"5min":0.23000000000000001,"15min":0.20799999999999999},"max":{"1min":7.141,"5min":14.254,"15min":43.222999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.4359999999999999,"15min":2.823},"min":{"1min":0.28299999999999997,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":3.4550000000000001,"5min":11.202,"15min":78.594999999999999},"last":0.32200000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":2.423,"15min":1.972},"min":{"1min":0.312,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":6.8890000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.96399999999999997,"5min":2.5800000000000001,"15min":2.202},"min":{"1min":0.33900000000000002,"5min":0.28699999999999998,"15min":0.27900000000000003},"max":{"1min":6.0940000000000003,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.45800000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":2.1869999999999998,"15min":3.012},"min":{"1min":0.29399999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.0659999999999998,"5min":10.919,"15min":28.117999999999999},"last":0.374},{"interface":"front","average":{"1min":1.768,"5min":2.6379999999999999,"15min":3.1000000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.1559999999999997,"5min":14.537000000000001,"15min":28.34},"last":0.36299999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:41:19 2023","interfaces":[{"interface":"back","average":{"1min":5.2430000000000003,"5min":3.4009999999999998,"15min":2.4870000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":73.887,"5min":73.887,"15min":73.887},"last":0.40300000000000002},{"interface":"front","average":{"1min":5.8470000000000004,"5min":3.5840000000000001,"15min":2.4780000000000002},"min":{"1min":0.44700000000000001,"5min":0.38600000000000001,"15min":0.32800000000000001},"max":{"1min":74.016999999999996,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.42499999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":2.3839999999999999,"15min":2.4249999999999998},"min":{"1min":0.41499999999999998,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":7.0620000000000003,"5min":13.085000000000001,"15min":52.006},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.774,"15min":2.1360000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.2690000000000001,"5min":11.109999999999999,"15min":51.890999999999998},"last":0.46800000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019385,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4832268,"kb_used_data":3933892,"kb_used_omap":0,"kb_used_meta":898368,"kb_avail":88912884,"statfs":{"total":95995035648,"available":91046793216,"internally_reserved":0,"allocated":4028305408,"data_stored":4027375746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919928832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:53 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":3.5209999999999999,"15min":4.9050000000000002},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":9.6690000000000005,"5min":79.837000000000003,"15min":155.315},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.875,"5min":3.9239999999999999,"15min":5.1550000000000002},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.23899999999999999},"max":{"1min":6.1459999999999999,"5min":78.983000000000004,"15min":155.334},"last":0.498}]},{"osd":1,"last update":"Sun Oct 1 09:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":8.1189999999999998,"15min":7.0599999999999996},"min":{"1min":0.35399999999999998,"5min":0.224,"15min":0.224},"max":{"1min":5.944,"5min":146.929,"15min":190.61199999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.0660000000000001,"5min":8.0500000000000007,"15min":7.8029999999999999},"min":{"1min":0.38500000000000001,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":3.7469999999999999,"5min":146.91,"15min":253.273},"last":0.51900000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:41:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.6279999999999999,"15min":2.6560000000000001},"min":{"1min":0.38300000000000001,"5min":0.29099999999999998,"15min":0.28299999999999997},"max":{"1min":5.702,"5min":10.161,"15min":170.45400000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.5089999999999999,"5min":1.988,"15min":2.371},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":5.6150000000000002,"5min":10.625,"15min":58.771000000000001},"last":0.51600000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.5600000000000001,"15min":2.931},"min":{"1min":0.33600000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.5279999999999996,"5min":15.989000000000001,"15min":140.881},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.825,"15min":3.145},"min":{"1min":0.313,"5min":0.311,"15min":0.311},"max":{"1min":3.786,"5min":15.978999999999999,"15min":140.911},"last":0.49199999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:41:30 2023","interfaces":[{"interface":"back","average":{"1min":7.3849999999999998,"5min":2.387,"15min":1.6259999999999999},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":110.003,"5min":110.003,"15min":110.003},"last":0.32600000000000001},{"interface":"front","average":{"1min":6.8159999999999998,"5min":2.294,"15min":1.425},"min":{"1min":0.29899999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":108.56399999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.40100000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":0.90000000000000002,"15min":0.996},"min":{"1min":0.34599999999999997,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":9.9849999999999994,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.087,"15min":1.083},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":4.0220000000000002,"5min":6.9139999999999997,"15min":10.076000000000001},"last":0.379}]},{"osd":7,"last update":"Sun Oct 1 09:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.2250000000000001,"15min":1.171},"min":{"1min":0.34599999999999997,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":3.8580000000000001,"5min":38.034999999999997,"15min":38.034999999999997},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.0299999999999998,"5min":1.6899999999999999,"15min":1.4530000000000001},"min":{"1min":0.377,"5min":0.35899999999999999,"15min":0.28699999999999998},"max":{"1min":12.092000000000001,"5min":37.723999999999997,"15min":37.723999999999997},"last":0.47699999999999998}]}]},{"osd":5,"up_from":379,"seq":1627792605292,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2462264,"kb_used_data":2089456,"kb_used_omap":13,"kb_used_meta":372722,"kb_avail":91282888,"statfs":{"total":95995035648,"available":93473677312,"internally_reserved":0,"allocated":2139602944,"data_stored":2138689186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":381667826},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":2.508,"15min":2.335},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.6379999999999999,"5min":13.385999999999999,"15min":84.340999999999994},"last":0.438},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.0030000000000001,"15min":2.1909999999999998},"min":{"1min":0.28299999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.2279999999999998,"5min":12.445,"15min":84.356999999999999},"last":0.35599999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":8.1319999999999997,"5min":9.8100000000000005,"15min":5.9340000000000002},"min":{"1min":0.29299999999999998,"5min":0.255,"15min":0.255},"max":{"1min":106.56999999999999,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.313},{"interface":"front","average":{"1min":8.1590000000000007,"5min":10.111000000000001,"15min":5.8540000000000001},"min":{"1min":0.27500000000000002,"5min":0.25900000000000001,"15min":0.24199999999999999},"max":{"1min":106.596,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.499}]},{"osd":2,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.653,"5min":2.1789999999999998,"15min":1.71},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.3470000000000004,"5min":14.199999999999999,"15min":14.199999999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":2.0379999999999998,"5min":2.0670000000000002,"15min":1.7370000000000001},"min":{"1min":0.373,"5min":0.31,"15min":0.27100000000000002},"max":{"1min":5.782,"5min":14.430999999999999,"15min":14.430999999999999},"last":0.374}]},{"osd":3,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":2.2730000000000001,"15min":1.5700000000000001},"min":{"1min":0.35399999999999998,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":3.4409999999999998,"5min":14.004,"15min":14.004},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.782,"5min":2.3610000000000002,"15min":1.7150000000000001},"min":{"1min":0.35299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.4400000000000004,"5min":13.991,"15min":13.991},"last":0.42299999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1320000000000001,"5min":2.3300000000000001,"15min":1.5},"min":{"1min":0.36699999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.9459999999999997,"5min":122.971,"15min":122.971},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.6520000000000001,"5min":2.3849999999999998,"15min":1.8859999999999999},"min":{"1min":0.32100000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.445,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.34499999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":1.335,"15min":1.1160000000000001},"min":{"1min":0.33900000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.7750000000000004,"5min":12.545,"15min":12.545},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.8560000000000001,"5min":1.0529999999999999,"15min":1.431},"min":{"1min":0.36899999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.4160000000000004,"5min":6.4729999999999999,"15min":14.193},"last":0.45200000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5459999999999998,"5min":1.238,"15min":1.379},"min":{"1min":0.46800000000000003,"5min":0.307,"15min":0.307},"max":{"1min":5.758,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":2.2210000000000001,"5min":1.3740000000000001,"15min":1.121},"min":{"1min":0.41199999999999998,"5min":0.318,"15min":0.318},"max":{"1min":5.2300000000000004,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.67400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3907719168,"data_stored":3907715096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":973209600,"data_stored":973209600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1274937344,"data_stored":1274937344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4018802688,"data_stored":4018798616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2042232832,"data_stored":2042232832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2260144128,"data_stored":2260140056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1616445440,"data_stored":1616445440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:42:20.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:20 smithi190 ceph-mon[160435]: from='client.36672 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:20.592 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:20.619 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:20.717 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:42:20.742 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:20 smithi138 ceph-mon[185886]: from='client.36672 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:20.742 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:20 smithi138 ceph-mon[192513]: from='client.36672 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:20.806 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:20.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:20.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:20.910 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:20.937 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:20.983 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:20.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:20.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:21.074 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:21.093 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:21.122 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:21.205 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:21.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:21.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:21.309 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:21.331 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:42:21.331 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:42:21.331 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:42:21.331 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:42:21.332 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:42:21.344 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:21.495 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:21.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:21.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:21.599 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:21.630 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:21.677 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:21.678 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:21 smithi138 ceph-mon[192513]: pgmap v1463: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8571/236673 objects degraded (3.621%); 4833/236673 objects misplaced (2.042%); 486 KiB/s, 7 objects/s recovering 2023-10-01T09:42:21.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:21 smithi138 ceph-mon[185886]: pgmap v1463: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8571/236673 objects degraded (3.621%); 4833/236673 objects misplaced (2.042%); 486 KiB/s, 7 objects/s recovering 2023-10-01T09:42:21.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:21.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:21.781 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:21.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:21 smithi190 ceph-mon[160435]: pgmap v1463: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8571/236673 objects degraded (3.621%); 4833/236673 objects misplaced (2.042%); 486 KiB/s, 7 objects/s recovering 2023-10-01T09:42:21.810 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:21.962 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:42:22.107 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:22.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:22.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:22.166 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:42:22.212 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:22.240 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:22.345 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:22.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:22.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:22.449 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:22.476 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:22.529 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:22.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:22.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:22.557 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:22.582 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:42:22.637 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:22.664 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:22.764 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:22.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:22.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:22.869 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:22.896 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:23.147 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:23.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:23.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:23.251 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:23.281 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:23.326 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:23.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:23.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:23.430 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:23.459 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:23.509 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:42:23.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:23 smithi190 ceph-mon[160435]: pgmap v1464: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 104 op/s; 8571/236031 objects degraded (3.631%); 4833/236031 objects misplaced (2.048%); 486 KiB/s, 7 objects/s recovering 2023-10-01T09:42:23.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:23 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8571/236031 objects degraded (3.631%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:23.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:23 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/554050967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:42:23.647 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:23.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:23.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:23.753 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:23.766 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:42:23.782 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:23 smithi138 ceph-mon[192513]: pgmap v1464: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 104 op/s; 8571/236031 objects degraded (3.631%); 4833/236031 objects misplaced (2.048%); 486 KiB/s, 7 objects/s recovering 2023-10-01T09:42:23.782 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:23 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8571/236031 objects degraded (3.631%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:23.782 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:23 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/554050967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:42:23.782 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:23 smithi138 ceph-mon[185886]: pgmap v1464: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 104 op/s; 8571/236031 objects degraded (3.631%); 4833/236031 objects misplaced (2.048%); 486 KiB/s, 7 objects/s recovering 2023-10-01T09:42:23.783 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:23 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8571/236031 objects degraded (3.631%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T09:42:23.783 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:23 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/554050967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:42:23.785 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:23.886 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:23.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:23.991 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:24.022 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:24.073 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:24.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:24.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:24.149 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:42:24.169 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:42:24.177 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:24.189 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:42:24.209 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:24.210 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:42:24.228 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:42:24.246 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:42:24.256 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:24.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:24.264 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:42:24.283 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:42:24.284 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:42:24.284 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:42:24.285 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:42:24.285 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:42:24.285 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:42:24.285 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:42:24.285 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:42:24.285 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:42:24.302 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:42:24.302 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:42:24.360 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:24.388 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:24.434 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:24.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:24.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:24.538 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:24.563 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:24 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:24.563 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/554050967' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:42:24.563 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:24 smithi138 ceph-mon[185886]: osdmap e514: 8 total, 8 up, 7 in 2023-10-01T09:42:24.564 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:24 smithi138 ceph-mon[185886]: failed to encode map e514 with expected crc 2023-10-01T09:42:24.564 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:24 smithi138 ceph-mon[185886]: failed to encode map e514 with expected crc 2023-10-01T09:42:24.564 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/726116509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:24.564 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:24 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:24.564 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/554050967' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:42:24.564 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:24 smithi138 ceph-mon[192513]: osdmap e514: 8 total, 8 up, 7 in 2023-10-01T09:42:24.564 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:24 smithi138 ceph-mon[192513]: failed to encode map e514 with expected crc 2023-10-01T09:42:24.564 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:24 smithi138 ceph-mon[192513]: failed to encode map e514 with expected crc 2023-10-01T09:42:24.565 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/726116509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:24.565 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:24 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/554050967' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:42:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:24 smithi190 ceph-mon[160435]: osdmap e514: 8 total, 8 up, 7 in 2023-10-01T09:42:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:24 smithi190 ceph-mon[160435]: failed to encode map e514 with expected crc 2023-10-01T09:42:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:24 smithi190 ceph-mon[160435]: failed to encode map e514 with expected crc 2023-10-01T09:42:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/726116509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:24.809 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:24.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:24.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:24.913 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:24.942 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:25.064 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:42:25.199 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:25.200 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:42:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:42:24] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-01T09:42:25.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:25.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:25.288 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:25.302 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:25.338 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:25.442 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:25.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:25.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:25.466 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:42:25.505 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:25 smithi138 ceph-mon[185886]: failed to encode map e514 with expected crc 2023-10-01T09:42:25.505 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:25 smithi138 ceph-mon[185886]: pgmap v1466: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 8571/235950 objects degraded (3.633%); 4833/235950 objects misplaced (2.048%); 288 KiB/s, 4 objects/s recovering 2023-10-01T09:42:25.505 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:42:25.505 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:25 smithi138 ceph-mon[192513]: failed to encode map e514 with expected crc 2023-10-01T09:42:25.505 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:25 smithi138 ceph-mon[192513]: pgmap v1466: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 8571/235950 objects degraded (3.633%); 4833/235950 objects misplaced (2.048%); 288 KiB/s, 4 objects/s recovering 2023-10-01T09:42:25.506 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:42:25.549 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:25.579 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:25.679 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:25 smithi190 ceph-mon[160435]: failed to encode map e514 with expected crc 2023-10-01T09:42:25.679 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:25 smithi190 ceph-mon[160435]: pgmap v1466: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 8571/235950 objects degraded (3.633%); 4833/235950 objects misplaced (2.048%); 288 KiB/s, 4 objects/s recovering 2023-10-01T09:42:25.679 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:42:25.688 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:25.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:25.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:25.792 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:25.825 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:25.875 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:25.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:25.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:25.980 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:26.007 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:26.057 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:26.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:26.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:26.160 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:26.191 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:26.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:26.290 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:26.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:26.391 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:26.422 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:26.600 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:26.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:26.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:26.708 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:26.738 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:26.837 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:26.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:26.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:26.940 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:26.963 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:42:26.964 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:42:26.969 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:27.023 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:27.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:27.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:27.045 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:42:27.045 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:42:27.126 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:27.161 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:27.223 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:27.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:27.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:27.273 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:42:27.314 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:42:27.330 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:27.358 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:27.403 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:27.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:27.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:27.507 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:27.534 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:27.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:27 smithi138 ceph-mon[192513]: pgmap v1467: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 87 op/s; 8538/235014 objects degraded (3.633%); 4794/235014 objects misplaced (2.040%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:27.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:27 smithi138 ceph-mon[185886]: pgmap v1467: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 87 op/s; 8538/235014 objects degraded (3.633%); 4794/235014 objects misplaced (2.040%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:27.632 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:27.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:27.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:27.696 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1467,"stamp":"2023-10-01T09:42:26.291488+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5136848843,"num_objects":78338,"num_object_clones":0,"num_object_copies":235014,"num_objects_missing_on_primary":3376,"num_objects_missing":3376,"num_objects_degraded":8538,"num_objects_misplaced":4794,"num_objects_unfound":0,"num_objects_dirty":78338,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":89236,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7714,"num_bytes_recovered":492405775,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89300,"ondisk_log_size":89300,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20863960,"kb_used_data":15819232,"kb_used_omap":39,"kb_used_meta":5044632,"kb_avail":635352104,"statfs":{"total":671965249536,"available":650600554496,"internally_reserved":0,"allocated":16198893568,"data_stored":16189123015,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5165703791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1,0,0,3,1,0,6,6,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-50921472,"num_objects":-777,"num_object_clones":0,"num_object_copies":-2331,"num_objects_missing_on_primary":-71,"num_objects_missing":-71,"num_objects_degraded":-74,"num_objects_misplaced":-70,"num_objects_unfound":0,"num_objects_dirty":-777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":777,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.895872"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":290,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.497066+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:42:17.497066+0000","last_peered":"2023-10-01T09:42:17.497066+0000","last_clean":"2023-10-01T09:42:17.497066+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:42:17.497066+0000","last_undegraded":"2023-10-01T09:42:17.497066+0000","last_fullsized":"2023-10-01T09:42:17.497066+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":132,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.397175+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:42:17.397175+0000","last_peered":"2023-10-01T09:42:17.397175+0000","last_clean":"2023-10-01T09:42:17.397175+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:42:17.397175+0000","last_undegraded":"2023-10-01T09:42:17.397175+0000","last_fullsized":"2023-10-01T09:42:17.397175+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":513,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.396944+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:42:17.396944+0000","last_peered":"2023-10-01T09:42:17.396944+0000","last_clean":"2023-10-01T09:42:17.396944+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:42:17.396944+0000","last_undegraded":"2023-10-01T09:42:17.396944+0000","last_fullsized":"2023-10-01T09:42:17.396944+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":570,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.762807+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:42:08.762807+0000","last_peered":"2023-10-01T09:42:08.762807+0000","last_clean":"2023-10-01T09:42:08.762807+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:42:08.762807+0000","last_undegraded":"2023-10-01T09:42:08.762807+0000","last_fullsized":"2023-10-01T09:42:08.762807+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":451,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.762275+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:42:08.762275+0000","last_peered":"2023-10-01T09:42:08.762275+0000","last_clean":"2023-10-01T09:42:08.762275+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:42:08.762275+0000","last_undegraded":"2023-10-01T09:42:08.762275+0000","last_fullsized":"2023-10-01T09:42:08.762275+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":195,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.463925+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:42:17.463925+0000","last_peered":"2023-10-01T09:42:17.463925+0000","last_clean":"2023-10-01T09:42:17.463925+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:42:17.463925+0000","last_undegraded":"2023-10-01T09:42:17.463925+0000","last_fullsized":"2023-10-01T09:42:17.463925+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":571,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.463980+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:42:17.463980+0000","last_peered":"2023-10-01T09:42:17.463980+0000","last_clean":"2023-10-01T09:42:17.463980+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:42:17.463980+0000","last_undegraded":"2023-10-01T09:42:17.463980+0000","last_fullsized":"2023-10-01T09:42:17.463980+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":498,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.770830+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:42:08.770830+0000","last_peered":"2023-10-01T09:42:08.770830+0000","last_clean":"2023-10-01T09:42:08.770830+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:42:08.770830+0000","last_undegraded":"2023-10-01T09:42:08.770830+0000","last_fullsized":"2023-10-01T09:42:08.770830+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":226,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.450600+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:42:17.450600+0000","last_peered":"2023-10-01T09:42:17.450600+0000","last_clean":"2023-10-01T09:42:17.450600+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:42:17.450600+0000","last_undegraded":"2023-10-01T09:42:17.450600+0000","last_fullsized":"2023-10-01T09:42:17.450600+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":32,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.761204+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:42:08.761204+0000","last_peered":"2023-10-01T09:42:08.761204+0000","last_clean":"2023-10-01T09:42:08.761204+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:42:08.761204+0000","last_undegraded":"2023-10-01T09:42:08.761204+0000","last_fullsized":"2023-10-01T09:42:08.761204+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":38,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.761989+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:42:08.761989+0000","last_peered":"2023-10-01T09:42:08.761989+0000","last_clean":"2023-10-01T09:42:08.761989+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:42:08.761989+0000","last_undegraded":"2023-10-01T09:42:08.761989+0000","last_fullsized":"2023-10-01T09:42:08.761989+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":36,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.396916+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:42:17.396916+0000","last_peered":"2023-10-01T09:42:17.396916+0000","last_clean":"2023-10-01T09:42:17.396916+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:42:17.396916+0000","last_undegraded":"2023-10-01T09:42:17.396916+0000","last_fullsized":"2023-10-01T09:42:17.396916+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":415,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.497287+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:42:17.497287+0000","last_peered":"2023-10-01T09:42:17.497287+0000","last_clean":"2023-10-01T09:42:17.497287+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:42:17.497287+0000","last_undegraded":"2023-10-01T09:42:17.497287+0000","last_fullsized":"2023-10-01T09:42:17.497287+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":296,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.497126+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:42:17.497126+0000","last_peered":"2023-10-01T09:42:17.497126+0000","last_clean":"2023-10-01T09:42:17.497126+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:42:17.497126+0000","last_undegraded":"2023-10-01T09:42:17.497126+0000","last_fullsized":"2023-10-01T09:42:17.497126+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":631,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.761438+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:42:08.761438+0000","last_peered":"2023-10-01T09:42:08.761438+0000","last_clean":"2023-10-01T09:42:08.761438+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:42:08.761438+0000","last_undegraded":"2023-10-01T09:42:08.761438+0000","last_fullsized":"2023-10-01T09:42:08.761438+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":368,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.761814+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:42:08.761814+0000","last_peered":"2023-10-01T09:42:08.761814+0000","last_clean":"2023-10-01T09:42:08.761814+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:42:08.761814+0000","last_undegraded":"2023-10-01T09:42:08.761814+0000","last_fullsized":"2023-10-01T09:42:08.761814+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":41,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.761246+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:42:08.761246+0000","last_peered":"2023-10-01T09:42:08.761246+0000","last_clean":"2023-10-01T09:42:08.761246+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:42:08.761246+0000","last_undegraded":"2023-10-01T09:42:08.761246+0000","last_fullsized":"2023-10-01T09:42:08.761246+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":28,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.450428+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:42:17.450428+0000","last_peered":"2023-10-01T09:42:17.450428+0000","last_clean":"2023-10-01T09:42:17.450428+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:42:17.450428+0000","last_undegraded":"2023-10-01T09:42:17.450428+0000","last_fullsized":"2023-10-01T09:42:17.450428+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":524,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.761360+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:42:08.761360+0000","last_peered":"2023-10-01T09:42:08.761360+0000","last_clean":"2023-10-01T09:42:08.761360+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:42:08.761360+0000","last_undegraded":"2023-10-01T09:42:08.761360+0000","last_fullsized":"2023-10-01T09:42:08.761360+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":535,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.421055+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:42:17.421055+0000","last_peered":"2023-10-01T09:42:17.421055+0000","last_clean":"2023-10-01T09:42:17.421055+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:42:17.421055+0000","last_undegraded":"2023-10-01T09:42:17.421055+0000","last_fullsized":"2023-10-01T09:42:17.421055+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":391,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.420092+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:42:17.420092+0000","last_peered":"2023-10-01T09:42:17.420092+0000","last_clean":"2023-10-01T09:42:17.420092+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:42:17.420092+0000","last_undegraded":"2023-10-01T09:42:17.420092+0000","last_fullsized":"2023-10-01T09:42:17.420092+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":41,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.761994+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:42:08.761994+0000","last_peered":"2023-10-01T09:42:08.761994+0000","last_clean":"2023-10-01T09:42:08.761994+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:42:08.761994+0000","last_undegraded":"2023-10-01T09:42:08.761994+0000","last_fullsized":"2023-10-01T09:42:08.761994+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"514'5527","reported_seq":5574,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:25.008098+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:42:25.008098+0000","last_peered":"2023-10-01T09:42:25.008098+0000","last_clean":"2023-10-01T09:42:25.008098+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:42:25.008098+0000","last_undegraded":"2023-10-01T09:42:25.008098+0000","last_fullsized":"2023-10-01T09:42:25.008098+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5527,"log_dups_size":0,"ondisk_log_size":5527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318832640,"num_objects":4865,"num_object_clones":0,"num_object_copies":14595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5527,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":36,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.761915+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:42:08.761915+0000","last_peered":"2023-10-01T09:42:08.761915+0000","last_clean":"2023-10-01T09:42:08.761915+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:42:08.761915+0000","last_undegraded":"2023-10-01T09:42:08.761915+0000","last_fullsized":"2023-10-01T09:42:08.761915+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":234,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.450095+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:42:17.450095+0000","last_peered":"2023-10-01T09:42:17.450095+0000","last_clean":"2023-10-01T09:42:17.450095+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:42:17.450095+0000","last_undegraded":"2023-10-01T09:42:17.450095+0000","last_fullsized":"2023-10-01T09:42:17.450095+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"514'5513","reported_seq":6518,"reported_epoch":514,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:42:25.208531+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:42:25.208531+0000","last_peered":"2023-10-01T09:42:25.208531+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:42:25.208531+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5513,"log_dups_size":0,"ondisk_log_size":5513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320667648,"num_objects":4893,"num_object_clones":0,"num_object_copies":14679,"num_objects_missing_on_primary":3376,"num_objects_missing":3376,"num_objects_degraded":8201,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5513,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":310,"num_bytes_recovered":20316160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1449},{"shards":"0,4,6","objects":3376},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":507,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.762007+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:42:08.762007+0000","last_peered":"2023-10-01T09:42:08.762007+0000","last_clean":"2023-10-01T09:42:08.762007+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:42:08.762007+0000","last_undegraded":"2023-10-01T09:42:08.762007+0000","last_fullsized":"2023-10-01T09:42:08.762007+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":568,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.396930+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:42:17.396930+0000","last_peered":"2023-10-01T09:42:17.396930+0000","last_clean":"2023-10-01T09:42:17.396930+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:42:17.396930+0000","last_undegraded":"2023-10-01T09:42:17.396930+0000","last_fullsized":"2023-10-01T09:42:17.396930+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"513'5495","reported_seq":5543,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:21.500124+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:42:21.500124+0000","last_peered":"2023-10-01T09:42:21.500124+0000","last_clean":"2023-10-01T09:42:21.500124+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:42:21.500124+0000","last_undegraded":"2023-10-01T09:42:21.500124+0000","last_fullsized":"2023-10-01T09:42:21.500124+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5495,"log_dups_size":0,"ondisk_log_size":5495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323551232,"num_objects":4937,"num_object_clones":0,"num_object_copies":14811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5495,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":593,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.762085+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:42:08.762085+0000","last_peered":"2023-10-01T09:42:08.762085+0000","last_clean":"2023-10-01T09:42:08.762085+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:42:08.762085+0000","last_undegraded":"2023-10-01T09:42:08.762085+0000","last_fullsized":"2023-10-01T09:42:08.762085+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":389,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.397231+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:42:17.397231+0000","last_peered":"2023-10-01T09:42:17.397231+0000","last_clean":"2023-10-01T09:42:17.397231+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:42:17.397231+0000","last_undegraded":"2023-10-01T09:42:17.397231+0000","last_fullsized":"2023-10-01T09:42:17.397231+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"513'5467","reported_seq":12120,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:21.265892+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:42:21.265892+0000","last_peered":"2023-10-01T09:42:21.265892+0000","last_clean":"2023-10-01T09:42:21.265892+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:42:21.265892+0000","last_undegraded":"2023-10-01T09:42:21.265892+0000","last_fullsized":"2023-10-01T09:42:21.265892+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5467,"log_dups_size":0,"ondisk_log_size":5467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323420160,"num_objects":4935,"num_object_clones":0,"num_object_copies":14805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5467,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":234,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.450383+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:42:17.450383+0000","last_peered":"2023-10-01T09:42:17.450383+0000","last_clean":"2023-10-01T09:42:17.450383+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:42:17.450383+0000","last_undegraded":"2023-10-01T09:42:17.450383+0000","last_fullsized":"2023-10-01T09:42:17.450383+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":490,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.762361+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:42:08.762361+0000","last_peered":"2023-10-01T09:42:08.762361+0000","last_clean":"2023-10-01T09:42:08.762361+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:42:08.762361+0000","last_undegraded":"2023-10-01T09:42:08.762361+0000","last_fullsized":"2023-10-01T09:42:08.762361+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"514'5563","reported_seq":5612,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:25.004957+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:42:25.004957+0000","last_peered":"2023-10-01T09:42:25.004957+0000","last_clean":"2023-10-01T09:42:25.004957+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:42:25.004957+0000","last_undegraded":"2023-10-01T09:42:25.004957+0000","last_fullsized":"2023-10-01T09:42:25.004957+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5563,"log_dups_size":0,"ondisk_log_size":5563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324075520,"num_objects":4945,"num_object_clones":0,"num_object_copies":14835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5563,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":94,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.497408+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:42:17.497408+0000","last_peered":"2023-10-01T09:42:17.497408+0000","last_clean":"2023-10-01T09:42:17.497408+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:42:17.497408+0000","last_undegraded":"2023-10-01T09:42:17.497408+0000","last_fullsized":"2023-10-01T09:42:17.497408+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":168,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.497379+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:42:17.497379+0000","last_peered":"2023-10-01T09:42:17.497379+0000","last_clean":"2023-10-01T09:42:17.497379+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:42:17.497379+0000","last_undegraded":"2023-10-01T09:42:17.497379+0000","last_fullsized":"2023-10-01T09:42:17.497379+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"514'5510","reported_seq":5565,"reported_epoch":514,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:42:25.040543+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:42:25.040543+0000","last_peered":"2023-10-01T09:42:25.040543+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:42:25.040543+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5510,"log_dups_size":0,"ondisk_log_size":5510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318373888,"num_objects":4858,"num_object_clones":0,"num_object_copies":14574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":327,"num_objects_misplaced":4794,"num_objects_unfound":0,"num_objects_dirty":4858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5510,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4794},{"shards":"4,5","objects":-263}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":412,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.497212+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:42:17.497212+0000","last_peered":"2023-10-01T09:42:17.497212+0000","last_clean":"2023-10-01T09:42:17.497212+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:42:17.497212+0000","last_undegraded":"2023-10-01T09:42:17.497212+0000","last_fullsized":"2023-10-01T09:42:17.497212+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":572,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.397269+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:42:17.397269+0000","last_peered":"2023-10-01T09:42:17.397269+0000","last_clean":"2023-10-01T09:42:17.397269+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:42:17.397269+0000","last_undegraded":"2023-10-01T09:42:17.397269+0000","last_fullsized":"2023-10-01T09:42:17.397269+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"513'5470","reported_seq":5525,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:22.194368+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:42:22.194368+0000","last_peered":"2023-10-01T09:42:22.194368+0000","last_clean":"2023-10-01T09:42:22.194368+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:42:22.194368+0000","last_undegraded":"2023-10-01T09:42:22.194368+0000","last_fullsized":"2023-10-01T09:42:22.194368+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5470,"log_dups_size":0,"ondisk_log_size":5470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321650688,"num_objects":4908,"num_object_clones":0,"num_object_copies":14724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5470,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":408,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.496998+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:42:17.496998+0000","last_peered":"2023-10-01T09:42:17.496998+0000","last_clean":"2023-10-01T09:42:17.496998+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:42:17.496998+0000","last_undegraded":"2023-10-01T09:42:17.496998+0000","last_fullsized":"2023-10-01T09:42:17.496998+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":249,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.648583+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:42:17.648583+0000","last_peered":"2023-10-01T09:42:17.648583+0000","last_clean":"2023-10-01T09:42:17.648583+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:42:17.648583+0000","last_undegraded":"2023-10-01T09:42:17.648583+0000","last_fullsized":"2023-10-01T09:42:17.648583+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"513'5585","reported_seq":5631,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:21.193317+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:42:21.193317+0000","last_peered":"2023-10-01T09:42:21.193317+0000","last_clean":"2023-10-01T09:42:21.193317+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:42:21.193317+0000","last_undegraded":"2023-10-01T09:42:21.193317+0000","last_fullsized":"2023-10-01T09:42:21.193317+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5585,"log_dups_size":0,"ondisk_log_size":5585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321978368,"num_objects":4913,"num_object_clones":0,"num_object_copies":14739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5585,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":376,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.396934+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:42:17.396934+0000","last_peered":"2023-10-01T09:42:17.396934+0000","last_clean":"2023-10-01T09:42:17.396934+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:42:17.396934+0000","last_undegraded":"2023-10-01T09:42:17.396934+0000","last_fullsized":"2023-10-01T09:42:17.396934+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":423,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.420618+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:42:17.420618+0000","last_peered":"2023-10-01T09:42:17.420618+0000","last_clean":"2023-10-01T09:42:17.420618+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:42:17.420618+0000","last_undegraded":"2023-10-01T09:42:17.420618+0000","last_fullsized":"2023-10-01T09:42:17.420618+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"513'5367","reported_seq":5413,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:21.252810+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:42:21.252810+0000","last_peered":"2023-10-01T09:42:21.252810+0000","last_clean":"2023-10-01T09:42:21.252810+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:42:21.252810+0000","last_undegraded":"2023-10-01T09:42:21.252810+0000","last_fullsized":"2023-10-01T09:42:21.252810+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5367,"log_dups_size":0,"ondisk_log_size":5367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315293696,"num_objects":4811,"num_object_clones":0,"num_object_copies":14433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5367,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":906,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.761741+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:42:08.761741+0000","last_peered":"2023-10-01T09:42:08.761741+0000","last_clean":"2023-10-01T09:42:08.761741+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:42:08.761741+0000","last_undegraded":"2023-10-01T09:42:08.761741+0000","last_fullsized":"2023-10-01T09:42:08.761741+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":244,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.450519+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:42:17.450519+0000","last_peered":"2023-10-01T09:42:17.450519+0000","last_clean":"2023-10-01T09:42:17.450519+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:42:17.450519+0000","last_undegraded":"2023-10-01T09:42:17.450519+0000","last_fullsized":"2023-10-01T09:42:17.450519+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":36,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.397004+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:42:17.397004+0000","last_peered":"2023-10-01T09:42:17.397004+0000","last_clean":"2023-10-01T09:42:17.397004+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:42:17.397004+0000","last_undegraded":"2023-10-01T09:42:17.397004+0000","last_fullsized":"2023-10-01T09:42:17.397004+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"513'5457","reported_seq":5505,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:21.262308+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:42:21.262308+0000","last_peered":"2023-10-01T09:42:21.262308+0000","last_clean":"2023-10-01T09:42:21.262308+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:42:21.262308+0000","last_undegraded":"2023-10-01T09:42:21.262308+0000","last_fullsized":"2023-10-01T09:42:21.262308+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5457,"log_dups_size":0,"ondisk_log_size":5457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321978368,"num_objects":4913,"num_object_clones":0,"num_object_copies":14739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5457,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1179,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.450191+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:42:17.450191+0000","last_peered":"2023-10-01T09:42:17.450191+0000","last_clean":"2023-10-01T09:42:17.450191+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:42:17.450191+0000","last_undegraded":"2023-10-01T09:42:17.450191+0000","last_fullsized":"2023-10-01T09:42:17.450191+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":389,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.497268+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:42:17.497268+0000","last_peered":"2023-10-01T09:42:17.497268+0000","last_clean":"2023-10-01T09:42:17.497268+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:42:17.497268+0000","last_undegraded":"2023-10-01T09:42:17.497268+0000","last_fullsized":"2023-10-01T09:42:17.497268+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":217,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.450474+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:42:17.450474+0000","last_peered":"2023-10-01T09:42:17.450474+0000","last_clean":"2023-10-01T09:42:17.450474+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:42:17.450474+0000","last_undegraded":"2023-10-01T09:42:17.450474+0000","last_fullsized":"2023-10-01T09:42:17.450474+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"514'5521","reported_seq":12789,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:25.030401+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:42:25.030401+0000","last_peered":"2023-10-01T09:42:25.030401+0000","last_clean":"2023-10-01T09:42:25.030401+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:42:25.030401+0000","last_undegraded":"2023-10-01T09:42:25.030401+0000","last_fullsized":"2023-10-01T09:42:25.030401+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318177280,"num_objects":4855,"num_object_clones":0,"num_object_copies":14565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5521,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":857,"reported_epoch":512,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:42:08.762004+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:42:08.762004+0000","last_peered":"2023-10-01T09:42:08.762004+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:42:08.762004+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:42:08.762004+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":576,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.396950+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:42:17.396950+0000","last_peered":"2023-10-01T09:42:17.396950+0000","last_clean":"2023-10-01T09:42:17.396950+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:42:17.396950+0000","last_undegraded":"2023-10-01T09:42:17.396950+0000","last_fullsized":"2023-10-01T09:42:17.396950+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":36,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.420477+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:42:17.420477+0000","last_peered":"2023-10-01T09:42:17.420477+0000","last_clean":"2023-10-01T09:42:17.420477+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:42:17.420477+0000","last_undegraded":"2023-10-01T09:42:17.420477+0000","last_fullsized":"2023-10-01T09:42:17.420477+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"514'5455","reported_seq":5504,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:25.007610+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:42:25.007610+0000","last_peered":"2023-10-01T09:42:25.007610+0000","last_clean":"2023-10-01T09:42:25.007610+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:42:25.007610+0000","last_undegraded":"2023-10-01T09:42:25.007610+0000","last_fullsized":"2023-10-01T09:42:25.007610+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5455,"log_dups_size":0,"ondisk_log_size":5455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315686912,"num_objects":4817,"num_object_clones":0,"num_object_copies":14451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5455,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1630,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.450334+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:42:17.450334+0000","last_peered":"2023-10-01T09:42:17.450334+0000","last_clean":"2023-10-01T09:42:17.450334+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:42:17.450334+0000","last_undegraded":"2023-10-01T09:42:17.450334+0000","last_fullsized":"2023-10-01T09:42:17.450334+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":543,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.761546+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:42:08.761546+0000","last_peered":"2023-10-01T09:42:08.761546+0000","last_clean":"2023-10-01T09:42:08.761546+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:42:08.761546+0000","last_undegraded":"2023-10-01T09:42:08.761546+0000","last_fullsized":"2023-10-01T09:42:08.761546+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":322,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.450244+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:42:17.450244+0000","last_peered":"2023-10-01T09:42:17.450244+0000","last_clean":"2023-10-01T09:42:17.450244+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:42:17.450244+0000","last_undegraded":"2023-10-01T09:42:17.450244+0000","last_fullsized":"2023-10-01T09:42:17.450244+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":312,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.497533+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:42:17.497533+0000","last_peered":"2023-10-01T09:42:17.497533+0000","last_clean":"2023-10-01T09:42:17.497533+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:42:17.497533+0000","last_undegraded":"2023-10-01T09:42:17.497533+0000","last_fullsized":"2023-10-01T09:42:17.497533+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"513'5536","reported_seq":5582,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:21.210165+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:42:21.210165+0000","last_peered":"2023-10-01T09:42:21.210165+0000","last_clean":"2023-10-01T09:42:21.210165+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:42:21.210165+0000","last_undegraded":"2023-10-01T09:42:21.210165+0000","last_fullsized":"2023-10-01T09:42:21.210165+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5536,"log_dups_size":0,"ondisk_log_size":5536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327417856,"num_objects":4996,"num_object_clones":0,"num_object_copies":14988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5536,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":720,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.761594+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:42:08.761594+0000","last_peered":"2023-10-01T09:42:08.761594+0000","last_clean":"2023-10-01T09:42:08.761594+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:42:08.761594+0000","last_undegraded":"2023-10-01T09:42:08.761594+0000","last_fullsized":"2023-10-01T09:42:08.761594+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":552,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:11.850547+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:42:11.850547+0000","last_peered":"2023-10-01T09:42:11.850547+0000","last_clean":"2023-10-01T09:42:11.850547+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:42:11.850547+0000","last_undegraded":"2023-10-01T09:42:11.850547+0000","last_fullsized":"2023-10-01T09:42:11.850547+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":551,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.762642+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:42:08.762642+0000","last_peered":"2023-10-01T09:42:08.762642+0000","last_clean":"2023-10-01T09:42:08.762642+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:42:08.762642+0000","last_undegraded":"2023-10-01T09:42:08.762642+0000","last_fullsized":"2023-10-01T09:42:08.762642+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"513'5526","reported_seq":5574,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:21.784900+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:42:21.784900+0000","last_peered":"2023-10-01T09:42:21.784900+0000","last_clean":"2023-10-01T09:42:21.784900+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:42:21.784900+0000","last_undegraded":"2023-10-01T09:42:21.784900+0000","last_fullsized":"2023-10-01T09:42:21.784900+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5526,"log_dups_size":0,"ondisk_log_size":5526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323289112,"num_objects":4934,"num_object_clones":0,"num_object_copies":14802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4934,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5526,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":951,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.762101+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:42:08.762101+0000","last_peered":"2023-10-01T09:42:08.762101+0000","last_clean":"2023-10-01T09:42:08.762101+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:42:08.762101+0000","last_undegraded":"2023-10-01T09:42:08.762101+0000","last_fullsized":"2023-10-01T09:42:08.762101+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":571,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.450472+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:42:17.450472+0000","last_peered":"2023-10-01T09:42:17.450472+0000","last_clean":"2023-10-01T09:42:17.450472+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:42:17.450472+0000","last_undegraded":"2023-10-01T09:42:17.450472+0000","last_fullsized":"2023-10-01T09:42:17.450472+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":424,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.450421+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:42:17.450421+0000","last_peered":"2023-10-01T09:42:17.450421+0000","last_clean":"2023-10-01T09:42:17.450421+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:42:17.450421+0000","last_undegraded":"2023-10-01T09:42:17.450421+0000","last_fullsized":"2023-10-01T09:42:17.450421+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"514'5544","reported_seq":5599,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:25.889506+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:42:25.889506+0000","last_peered":"2023-10-01T09:42:25.889506+0000","last_clean":"2023-10-01T09:42:25.889506+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:42:25.889506+0000","last_undegraded":"2023-10-01T09:42:25.889506+0000","last_fullsized":"2023-10-01T09:42:25.889506+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5544,"log_dups_size":0,"ondisk_log_size":5544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315359232,"num_objects":4812,"num_object_clones":0,"num_object_copies":14436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5544,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":452,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.762088+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:42:08.762088+0000","last_peered":"2023-10-01T09:42:08.762088+0000","last_clean":"2023-10-01T09:42:08.762088+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:42:08.762088+0000","last_undegraded":"2023-10-01T09:42:08.762088+0000","last_fullsized":"2023-10-01T09:42:08.762088+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1145,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.420606+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:42:17.420606+0000","last_peered":"2023-10-01T09:42:17.420606+0000","last_clean":"2023-10-01T09:42:17.420606+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:42:17.420606+0000","last_undegraded":"2023-10-01T09:42:17.420606+0000","last_fullsized":"2023-10-01T09:42:17.420606+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":41,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.761500+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:42:08.761500+0000","last_peered":"2023-10-01T09:42:08.761500+0000","last_clean":"2023-10-01T09:42:08.761500+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:42:08.761500+0000","last_undegraded":"2023-10-01T09:42:08.761500+0000","last_fullsized":"2023-10-01T09:42:08.761500+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"514'5594","reported_seq":5649,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:25.693271+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:42:25.693271+0000","last_peered":"2023-10-01T09:42:25.693271+0000","last_clean":"2023-10-01T09:42:25.693271+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:42:25.693271+0000","last_undegraded":"2023-10-01T09:42:25.693271+0000","last_fullsized":"2023-10-01T09:42:25.693271+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5594,"log_dups_size":0,"ondisk_log_size":5594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322306048,"num_objects":4918,"num_object_clones":0,"num_object_copies":14754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5594,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":45,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.761973+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:42:08.761973+0000","last_peered":"2023-10-01T09:42:08.761973+0000","last_clean":"2023-10-01T09:42:08.761973+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:42:08.761973+0000","last_undegraded":"2023-10-01T09:42:08.761973+0000","last_fullsized":"2023-10-01T09:42:08.761973+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":529,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.762733+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:42:08.762733+0000","last_peered":"2023-10-01T09:42:08.762733+0000","last_clean":"2023-10-01T09:42:08.762733+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:42:08.762733+0000","last_undegraded":"2023-10-01T09:42:08.762733+0000","last_fullsized":"2023-10-01T09:42:08.762733+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":237,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.450127+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:42:17.450127+0000","last_peered":"2023-10-01T09:42:17.450127+0000","last_clean":"2023-10-01T09:42:17.450127+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:42:17.450127+0000","last_undegraded":"2023-10-01T09:42:17.450127+0000","last_fullsized":"2023-10-01T09:42:17.450127+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":478,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.762451+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:42:08.762451+0000","last_peered":"2023-10-01T09:42:08.762451+0000","last_clean":"2023-10-01T09:42:08.762451+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:42:08.762451+0000","last_undegraded":"2023-10-01T09:42:08.762451+0000","last_fullsized":"2023-10-01T09:42:08.762451+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":593,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.397199+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:42:17.397199+0000","last_peered":"2023-10-01T09:42:17.397199+0000","last_clean":"2023-10-01T09:42:17.397199+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:42:17.397199+0000","last_undegraded":"2023-10-01T09:42:17.397199+0000","last_fullsized":"2023-10-01T09:42:17.397199+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":249,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.450378+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:42:17.450378+0000","last_peered":"2023-10-01T09:42:17.450378+0000","last_clean":"2023-10-01T09:42:17.450378+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:42:17.450378+0000","last_undegraded":"2023-10-01T09:42:17.450378+0000","last_fullsized":"2023-10-01T09:42:17.450378+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":36,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.397040+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:42:17.397040+0000","last_peered":"2023-10-01T09:42:17.397040+0000","last_clean":"2023-10-01T09:42:17.397040+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:42:17.397040+0000","last_undegraded":"2023-10-01T09:42:17.397040+0000","last_fullsized":"2023-10-01T09:42:17.397040+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":249,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.450648+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:42:17.450648+0000","last_peered":"2023-10-01T09:42:17.450648+0000","last_clean":"2023-10-01T09:42:17.450648+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:42:17.450648+0000","last_undegraded":"2023-10-01T09:42:17.450648+0000","last_fullsized":"2023-10-01T09:42:17.450648+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":424,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.762358+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:42:08.762358+0000","last_peered":"2023-10-01T09:42:08.762358+0000","last_clean":"2023-10-01T09:42:08.762358+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:42:08.762358+0000","last_undegraded":"2023-10-01T09:42:08.762358+0000","last_fullsized":"2023-10-01T09:42:08.762358+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":54,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.420319+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:42:17.420319+0000","last_peered":"2023-10-01T09:42:17.420319+0000","last_clean":"2023-10-01T09:42:17.420319+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:42:17.420319+0000","last_undegraded":"2023-10-01T09:42:17.420319+0000","last_fullsized":"2023-10-01T09:42:17.420319+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":469,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:08.762197+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:42:08.762197+0000","last_peered":"2023-10-01T09:42:08.762197+0000","last_clean":"2023-10-01T09:42:08.762197+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:42:08.762197+0000","last_undegraded":"2023-10-01T09:42:08.762197+0000","last_fullsized":"2023-10-01T09:42:08.762197+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":669,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-10-01T09:42:11.850647+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:42:11.850647+0000","last_peered":"2023-10-01T09:42:11.850647+0000","last_clean":"2023-10-01T09:42:11.850647+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:42:11.850647+0000","last_undegraded":"2023-10-01T09:42:11.850647+0000","last_fullsized":"2023-10-01T09:42:11.850647+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":365,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-10-01T09:42:17.496933+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:42:17.496933+0000","last_peered":"2023-10-01T09:42:17.496933+0000","last_clean":"2023-10-01T09:42:17.496933+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:42:17.496933+0000","last_undegraded":"2023-10-01T09:42:17.496933+0000","last_fullsized":"2023-10-01T09:42:17.496933+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5132058648,"num_objects":78310,"num_object_clones":0,"num_object_copies":234930,"num_objects_missing_on_primary":3376,"num_objects_missing":3376,"num_objects_degraded":8528,"num_objects_misplaced":4794,"num_objects_unfound":0,"num_objects_dirty":78310,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":88130,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7203,"num_bytes_recovered":472055808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15931748352,"data_stored":15931736136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88130,"ondisk_log_size":88130,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430701,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2562008,"kb_used_data":1650704,"kb_used_omap":0,"kb_used_meta":911296,"kb_avail":91183144,"statfs":{"total":95995035648,"available":93371539456,"internally_reserved":0,"allocated":1690320896,"data_stored":1689387404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":933167104},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.766,"15min":3.0510000000000002},"min":{"1min":0.30099999999999999,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":6.8550000000000004,"5min":11.525,"15min":108.90000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.82099999999999995,"5min":2.2130000000000001,"15min":3.3359999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.6639999999999997,"5min":29.501000000000001,"15min":109.187},"last":0.432}]},{"osd":1,"last update":"Sun Oct 1 09:42:06 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":5.0410000000000004,"15min":5.492},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":6.8289999999999997,"5min":229.161,"15min":229.161},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.92100000000000004,"5min":4.9939999999999998,"15min":5.5490000000000004},"min":{"1min":0.28899999999999998,"5min":0.27500000000000002,"15min":0.24399999999999999},"max":{"1min":5.7240000000000002,"5min":230.126,"15min":230.126},"last":0.496}]},{"osd":2,"last update":"Sun Oct 1 09:41:47 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":2.1680000000000001,"15min":2.0169999999999999},"min":{"1min":0.33200000000000002,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":5.3090000000000002,"5min":14.866,"15min":14.901999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.2829999999999999,"5min":2.1749999999999998,"15min":1.9079999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.6929999999999996,"5min":14.682,"15min":14.682},"last":0.57099999999999995}]},{"osd":3,"last update":"Sun Oct 1 09:42:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.9410000000000001,"15min":2.8610000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":5.7750000000000004,"5min":35.104999999999997,"15min":80.352999999999994},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.004,"5min":2,"15min":2.8359999999999999},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.548,"5min":35.118000000000002,"15min":80.364000000000004},"last":0.56200000000000006}]},{"osd":4,"last update":"Sun Oct 1 09:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.202,"15min":1.0529999999999999},"min":{"1min":0.30499999999999999,"5min":0.27600000000000002,"15min":0.254},"max":{"1min":6.8769999999999998,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.4710000000000001,"5min":1.1739999999999999,"15min":1.0369999999999999},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":5.7439999999999998,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.314}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":8.516,"5min":2.984,"15min":1.679},"min":{"1min":0.311,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":135.40000000000001,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":8.1989999999999998,"5min":3.1360000000000001,"15min":1.71},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":135.339,"5min":135.339,"15min":135.339},"last":0.38700000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0699999999999998,"5min":1.498,"15min":1.1819999999999999},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":11.58,"5min":11.58,"15min":11.853999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.988,"5min":1.3939999999999999,"15min":1.147},"min":{"1min":0.373,"5min":0.373,"15min":0.32300000000000001},"max":{"1min":13.021000000000001,"5min":14.584,"15min":14.584},"last":0.442}]}]},{"osd":6,"up_from":294,"seq":1262720385212,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2529044,"kb_used_data":2199628,"kb_used_omap":6,"kb_used_meta":329401,"kb_avail":91216108,"statfs":{"total":95995035648,"available":93405294592,"internally_reserved":0,"allocated":2252419072,"data_stored":2251489531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":337306669},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":3.786,"15min":4.1879999999999997},"min":{"1min":0.249,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":1.8400000000000001,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.76300000000000001,"5min":4.3559999999999999,"15min":4.2889999999999997},"min":{"1min":0.249,"5min":0.249,"15min":0.221},"max":{"1min":2.9969999999999999,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.44800000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":5.3860000000000001,"15min":5.0590000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":3.0289999999999999,"5min":168.947,"15min":168.947},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.79100000000000004,"5min":5.375,"15min":4.9690000000000003},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":2.6030000000000002,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.34699999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.8109999999999999,"15min":1.8420000000000001},"min":{"1min":0.34499999999999997,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":3.0489999999999999,"5min":9.657,"15min":26.289999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.885,"15min":1.859},"min":{"1min":0.32600000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.6739999999999999,"5min":21.704999999999998,"15min":26.388999999999999},"last":0.314}]},{"osd":3,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":1.597,"15min":2.4020000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.9450000000000001,"5min":9.0180000000000007,"15min":89.753},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.073,"5min":1.9330000000000001,"15min":2.2189999999999999},"min":{"1min":0.318,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":3.7160000000000002,"5min":10.191000000000001,"15min":89.849999999999994},"last":1.381}]},{"osd":4,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.665,"15min":1.0800000000000001},"min":{"1min":0.28599999999999998,"5min":0.253,"15min":0.246},"max":{"1min":4.0250000000000004,"5min":63.134,"15min":63.134},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.518,"15min":1.0960000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.28299999999999997},"max":{"1min":7.4039999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.55100000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":1.0089999999999999,"15min":1.0529999999999999},"min":{"1min":0.312,"5min":0.251,"15min":0.251},"max":{"1min":9.9939999999999998,"5min":9.9939999999999998,"15min":12.023999999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.1880000000000002,"5min":1.234,"15min":1.1739999999999999},"min":{"1min":0.314,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":15.457000000000001,"5min":15.457000000000001,"15min":15.457000000000001},"last":0.63100000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.82999999999999996,"15min":1.802},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":2.0419999999999998,"5min":6.4119999999999999,"15min":115.11},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.282,"15min":1.8959999999999999},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.28799999999999998},"max":{"1min":3.2679999999999998,"5min":8.6899999999999995,"15min":114.666},"last":0.51800000000000002}]}]},{"osd":1,"up_from":94,"seq":403726926189,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1757496,"kb_used_data":947504,"kb_used_omap":7,"kb_used_meta":809976,"kb_avail":91987656,"statfs":{"total":95995035648,"available":94195359744,"internally_reserved":0,"allocated":970244096,"data_stored":968235351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":829415847},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":3.5089999999999999,"15min":4.5229999999999997},"min":{"1min":0.19400000000000001,"5min":0.182,"15min":0.182},"max":{"1min":4.423,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.20699999999999999},{"interface":"front","average":{"1min":1.528,"5min":4.0339999999999998,"15min":3.859},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":13.872,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.29499999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:41:37 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":2.6459999999999999,"15min":3.4620000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":2.8740000000000001,"5min":22.891999999999999,"15min":56.573},"last":0.25700000000000001},{"interface":"front","average":{"1min":1.2769999999999999,"5min":3.0059999999999998,"15min":4.2670000000000003},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.20499999999999999},"max":{"1min":6.9710000000000001,"5min":15.845000000000001,"15min":154.197},"last":0.33900000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":3.9089999999999998,"15min":4.4429999999999996},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":3.1949999999999998,"5min":95.305000000000007,"15min":200.56800000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.68799999999999994,"5min":4.3550000000000004,"15min":5.2569999999999997},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":4.7839999999999998,"5min":98.033000000000001,"15min":200.63800000000001},"last":0.38}]},{"osd":4,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":2.4870000000000001,"15min":2.3940000000000001},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.5510000000000002,"5min":21.015000000000001,"15min":154.101},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.60799999999999998,"5min":2.306,"15min":2.7410000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.4209999999999998,"5min":13.773999999999999,"15min":154.56999999999999},"last":0.35099999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":2.4870000000000001,"15min":2.1509999999999998},"min":{"1min":0.32900000000000001,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":5.0410000000000004,"5min":21.219999999999999,"15min":154.22999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.83399999999999996,"5min":2.3239999999999998,"15min":2.661},"min":{"1min":0.311,"5min":0.307,"15min":0.307},"max":{"1min":5.1050000000000004,"5min":13.848000000000001,"15min":154.54400000000001},"last":0.316}]},{"osd":6,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.1669999999999998,"15min":2.8370000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.3660000000000001,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.626,"5min":2.4769999999999999,"15min":2.5899999999999999},"min":{"1min":0.36699999999999999,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":1.575,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.48199999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.617,"15min":2.7250000000000001},"min":{"1min":0.40500000000000003,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":5.2169999999999996,"5min":21.062999999999999,"15min":154.20699999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.88300000000000001,"5min":2.5390000000000001,"15min":2.9399999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":3.8260000000000001,"5min":13.819000000000001,"15min":154.51599999999999},"last":0.46200000000000002}]}]},{"osd":0,"up_from":89,"seq":382252089723,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4699696,"kb_used_data":3812648,"kb_used_omap":8,"kb_used_meta":887031,"kb_avail":89045456,"statfs":{"total":95995035648,"available":91182546944,"internally_reserved":0,"allocated":3904151552,"data_stored":3902148747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":908320706},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:42:08 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":3.9860000000000002,"15min":4.4039999999999999},"min":{"1min":0.20399999999999999,"5min":0.187,"15min":0.17399999999999999},"max":{"1min":1.661,"5min":185.78399999999999,"15min":216.821},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.437,"5min":5.9020000000000001,"15min":5.4400000000000004},"min":{"1min":0.249,"5min":0.215,"15min":0.215},"max":{"1min":1.5589999999999999,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.28000000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:41:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":2.7530000000000001,"15min":2.2509999999999999},"min":{"1min":0.30599999999999999,"5min":0.253,"15min":0.20100000000000001},"max":{"1min":4.9459999999999997,"5min":19.788,"15min":19.788},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.085,"5min":2.7290000000000001,"15min":2.7719999999999998},"min":{"1min":0.29599999999999999,"5min":0.215,"15min":0.189},"max":{"1min":3.4470000000000001,"5min":25.844999999999999,"15min":25.844999999999999},"last":0.34200000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:42:08 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":2.1749999999999998,"15min":4.524},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.20899999999999999},"max":{"1min":1.706,"5min":15.35,"15min":222.16499999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.873,"5min":2.5219999999999998,"15min":4.7089999999999996},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":2.859,"5min":14.545999999999999,"15min":222.227},"last":0.31}]},{"osd":4,"last update":"Sun Oct 1 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":2.3559999999999999,"15min":2.4260000000000002},"min":{"1min":0.29899999999999999,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":2.7480000000000002,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.034,"5min":2.698,"15min":2.536},"min":{"1min":0.376,"5min":0.33700000000000002,"15min":0.28299999999999997},"max":{"1min":3.1619999999999999,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.40100000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.494,"5min":2.371,"15min":2.0569999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":5.8769999999999998,"5min":14.515000000000001,"15min":16.808},"last":0.433},{"interface":"front","average":{"1min":1.8340000000000001,"5min":2.2879999999999998,"15min":1.9259999999999999},"min":{"1min":0.49099999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.8689999999999998,"5min":17.507000000000001,"15min":17.507000000000001},"last":0.44800000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:41:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":2.2509999999999999,"15min":1.897},"min":{"1min":0.38600000000000001,"5min":0.35699999999999998,"15min":0.35499999999999998},"max":{"1min":5.2110000000000003,"5min":17.535,"15min":17.535},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.649,"5min":2.1739999999999999,"15min":2.2679999999999998},"min":{"1min":0.44800000000000001,"5min":0.33200000000000002,"15min":0.32100000000000001},"max":{"1min":4.2640000000000002,"5min":10.446,"15min":18.521000000000001},"last":0.46000000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.198,"15min":2.1909999999999998},"min":{"1min":0.41799999999999998,"5min":0.377,"15min":0.34499999999999997},"max":{"1min":4.976,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.069,"5min":2.4740000000000002,"15min":2.5459999999999998},"min":{"1min":0.41199999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":4.226,"5min":48.274000000000001,"15min":48.274000000000001},"last":0.48799999999999999}]}]},{"osd":2,"up_from":0,"seq":425201762663,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599136,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2145888,"kb_used_data":1314136,"kb_used_omap":3,"kb_used_meta":831740,"kb_avail":91599264,"statfs":{"total":95995035648,"available":93797646336,"internally_reserved":0,"allocated":1345675264,"data_stored":1343624010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":851702503},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":5.9850000000000003,"5min":5.5869999999999997,"15min":6.0170000000000003},"min":{"1min":0.216,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":64.504000000000005,"5min":118.748,"15min":137.34399999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":6.024,"5min":5.7830000000000004,"15min":6.0090000000000003},"min":{"1min":0.24299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":64.448999999999998,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.246}]},{"osd":1,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":5.149,"15min":5.4130000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":6.0250000000000004,"5min":108.84999999999999,"15min":162.22999999999999},"last":1.2849999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":5.3860000000000001,"15min":5.718},"min":{"1min":0.23100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.931,"5min":109.13200000000001,"15min":162.70699999999999},"last":0.40999999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":2.5070000000000001,"15min":2.5},"min":{"1min":0.27000000000000002,"5min":0.23000000000000001,"15min":0.20799999999999999},"max":{"1min":7.141,"5min":14.254,"15min":43.222999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.4359999999999999,"15min":2.823},"min":{"1min":0.28299999999999997,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":3.4550000000000001,"5min":11.202,"15min":78.594999999999999},"last":0.59999999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":2.423,"15min":1.972},"min":{"1min":0.312,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":6.8890000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.96399999999999997,"5min":2.5800000000000001,"15min":2.202},"min":{"1min":0.33900000000000002,"5min":0.28699999999999998,"15min":0.27900000000000003},"max":{"1min":6.0940000000000003,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.376}]},{"osd":5,"last update":"Sun Oct 1 09:41:25 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":2.1869999999999998,"15min":3.012},"min":{"1min":0.29399999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.0659999999999998,"5min":10.919,"15min":28.117999999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.768,"5min":2.6379999999999999,"15min":3.1000000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.1559999999999997,"5min":14.537000000000001,"15min":28.34},"last":0.435}]},{"osd":6,"last update":"Sun Oct 1 09:41:19 2023","interfaces":[{"interface":"back","average":{"1min":5.2430000000000003,"5min":3.4009999999999998,"15min":2.4870000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":73.887,"5min":73.887,"15min":73.887},"last":0.496},{"interface":"front","average":{"1min":5.8470000000000004,"5min":3.5840000000000001,"15min":2.4780000000000002},"min":{"1min":0.44700000000000001,"5min":0.38600000000000001,"15min":0.32800000000000001},"max":{"1min":74.016999999999996,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.55700000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":2.3839999999999999,"15min":2.4249999999999998},"min":{"1min":0.41499999999999998,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":7.0620000000000003,"5min":13.085000000000001,"15min":52.006},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.774,"15min":2.1360000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.2690000000000001,"5min":11.109999999999999,"15min":51.890999999999998},"last":0.629}]}]},{"osd":4,"up_from":30,"seq":128849019387,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4770652,"kb_used_data":3868180,"kb_used_omap":0,"kb_used_meta":902464,"kb_avail":88974500,"statfs":{"total":95995035648,"available":91109888000,"internally_reserved":0,"allocated":3961016320,"data_stored":3960086010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924123136},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:53 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":3.5209999999999999,"15min":4.9050000000000002},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":9.6690000000000005,"5min":79.837000000000003,"15min":155.315},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.875,"5min":3.9239999999999999,"15min":5.1550000000000002},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.23899999999999999},"max":{"1min":6.1459999999999999,"5min":78.983000000000004,"15min":155.334},"last":0.45000000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":8.1189999999999998,"15min":7.0599999999999996},"min":{"1min":0.35399999999999998,"5min":0.224,"15min":0.224},"max":{"1min":5.944,"5min":146.929,"15min":190.61199999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":1.0660000000000001,"5min":8.0500000000000007,"15min":7.8029999999999999},"min":{"1min":0.38500000000000001,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":3.7469999999999999,"5min":146.91,"15min":253.273},"last":0.36699999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:41:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.6279999999999999,"15min":2.6560000000000001},"min":{"1min":0.38300000000000001,"5min":0.29099999999999998,"15min":0.28299999999999997},"max":{"1min":5.702,"5min":10.161,"15min":170.45400000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.5089999999999999,"5min":1.988,"15min":2.371},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":5.6150000000000002,"5min":10.625,"15min":58.771000000000001},"last":0.46999999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.5600000000000001,"15min":2.931},"min":{"1min":0.33600000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.5279999999999996,"5min":15.989000000000001,"15min":140.881},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.825,"15min":3.145},"min":{"1min":0.313,"5min":0.311,"15min":0.311},"max":{"1min":3.786,"5min":15.978999999999999,"15min":140.911},"last":0.40999999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:41:30 2023","interfaces":[{"interface":"back","average":{"1min":7.3849999999999998,"5min":2.387,"15min":1.6259999999999999},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":110.003,"5min":110.003,"15min":110.003},"last":0.48999999999999999},{"interface":"front","average":{"1min":6.8159999999999998,"5min":2.294,"15min":1.425},"min":{"1min":0.29899999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":108.56399999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.317}]},{"osd":6,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":0.90000000000000002,"15min":0.996},"min":{"1min":0.34599999999999997,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":9.9849999999999994,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.087,"15min":1.083},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":4.0220000000000002,"5min":6.9139999999999997,"15min":10.076000000000001},"last":0.51000000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.2250000000000001,"15min":1.171},"min":{"1min":0.34599999999999997,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":3.8580000000000001,"5min":38.034999999999997,"15min":38.034999999999997},"last":0.502},{"interface":"front","average":{"1min":2.0299999999999998,"5min":1.6899999999999999,"15min":1.4530000000000001},"min":{"1min":0.377,"5min":0.35899999999999999,"15min":0.28699999999999998},"max":{"1min":12.092000000000001,"5min":37.723999999999997,"15min":37.723999999999997},"last":0.63400000000000001}]}]},{"osd":5,"up_from":379,"seq":1627792605294,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2399176,"kb_used_data":2026432,"kb_used_omap":13,"kb_used_meta":372722,"kb_avail":91345976,"statfs":{"total":95995035648,"available":93538279424,"internally_reserved":0,"allocated":2075066368,"data_stored":2074151962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":381667826},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,0,0,3,1,0,6,6,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":2.508,"15min":2.335},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.6379999999999999,"5min":13.385999999999999,"15min":84.340999999999994},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.0030000000000001,"15min":2.1909999999999998},"min":{"1min":0.28299999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.2279999999999998,"5min":12.445,"15min":84.356999999999999},"last":0.5}]},{"osd":1,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":8.1319999999999997,"5min":9.8100000000000005,"15min":5.9340000000000002},"min":{"1min":0.29299999999999998,"5min":0.255,"15min":0.255},"max":{"1min":106.56999999999999,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.47599999999999998},{"interface":"front","average":{"1min":8.1590000000000007,"5min":10.111000000000001,"15min":5.8540000000000001},"min":{"1min":0.27500000000000002,"5min":0.25900000000000001,"15min":0.24199999999999999},"max":{"1min":106.596,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.63200000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.653,"5min":2.1789999999999998,"15min":1.71},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.3470000000000004,"5min":14.199999999999999,"15min":14.199999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.0379999999999998,"5min":2.0670000000000002,"15min":1.7370000000000001},"min":{"1min":0.373,"5min":0.31,"15min":0.27100000000000002},"max":{"1min":5.782,"5min":14.430999999999999,"15min":14.430999999999999},"last":0.55400000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":2.2730000000000001,"15min":1.5700000000000001},"min":{"1min":0.35399999999999998,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":3.4409999999999998,"5min":14.004,"15min":14.004},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.782,"5min":2.3610000000000002,"15min":1.7150000000000001},"min":{"1min":0.35299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.4400000000000004,"5min":13.991,"15min":13.991},"last":0.53800000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1320000000000001,"5min":2.3300000000000001,"15min":1.5},"min":{"1min":0.36699999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.9459999999999997,"5min":122.971,"15min":122.971},"last":0.61799999999999999},{"interface":"front","average":{"1min":2.6520000000000001,"5min":2.3849999999999998,"15min":1.8859999999999999},"min":{"1min":0.32100000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":12.445,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.52400000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":1.335,"15min":1.1160000000000001},"min":{"1min":0.33900000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":6.7750000000000004,"5min":12.545,"15min":12.545},"last":0.68200000000000005},{"interface":"front","average":{"1min":1.8560000000000001,"5min":1.0529999999999999,"15min":1.431},"min":{"1min":0.36899999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.4160000000000004,"5min":6.4729999999999999,"15min":14.193},"last":0.58899999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:41:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5459999999999998,"5min":1.238,"15min":1.379},"min":{"1min":0.46800000000000003,"5min":0.307,"15min":0.307},"max":{"1min":5.758,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":2.2210000000000001,"5min":1.3740000000000001,"15min":1.121},"min":{"1min":0.41199999999999998,"5min":0.318,"15min":0.318},"max":{"1min":5.2300000000000004,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.66100000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3893104640,"data_stored":3893100568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":959971328,"data_stored":959971328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1259470848,"data_stored":1259470848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3949006848,"data_stored":3949002776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2015952896,"data_stored":2015952896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2245857280,"data_stored":2245853208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1608384512,"data_stored":1608384512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:42:27.703 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:42:27.719 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:27 smithi190 ceph-mon[160435]: pgmap v1467: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 87 op/s; 8538/235014 objects degraded (3.633%); 4794/235014 objects misplaced (2.040%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:27.736 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:42:27.737 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:27.773 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:27.821 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:27.828 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:27.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:27.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:27.876 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:27.934 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:27.970 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:28.064 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:28.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:28.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:28.153 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:42:28.167 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:28.195 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:28.442 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:28.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:28.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:28.548 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:28.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:28 smithi138 ceph-mon[192513]: from='client.36690 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:28.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:28 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8538/235014 objects degraded (3.633%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:28.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:28 smithi138 ceph-mon[185886]: from='client.36690 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:28.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:28 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8538/235014 objects degraded (3.633%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:28.577 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:28.626 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:28.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:28.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:28.715 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:28 smithi190 ceph-mon[160435]: from='client.36690 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:28.716 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:28 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8538/235014 objects degraded (3.633%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:28.729 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:28.759 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:28.805 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:28.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:28.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:28.910 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:28.937 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:28.983 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:28.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:28.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:29.087 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:29.119 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:29.166 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:29.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:29.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:29.270 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:29.300 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:29.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:29 smithi190 ceph-mon[160435]: pgmap v1468: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 8538/233946 objects degraded (3.650%); 4794/233946 objects misplaced (2.049%); 467 KiB/s, 7 objects/s recovering 2023-10-01T09:42:29.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:29 smithi138 ceph-mon[192513]: pgmap v1468: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 8538/233946 objects degraded (3.650%); 4794/233946 objects misplaced (2.049%); 467 KiB/s, 7 objects/s recovering 2023-10-01T09:42:29.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:29 smithi138 ceph-mon[185886]: pgmap v1468: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 8538/233946 objects degraded (3.650%); 4794/233946 objects misplaced (2.049%); 467 KiB/s, 7 objects/s recovering 2023-10-01T09:42:29.755 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:42:29.755 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:42:29.879 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:29.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:29.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:29.959 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:29.983 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:30.015 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:30.067 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:30.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:30.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:30.169 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:30.201 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:30.308 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:30.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:30.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:30.334 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:42:30.354 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:42:30.374 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:42:30.393 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:42:30.415 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:30 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1770769718' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:30.415 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:30 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:30.415 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:30 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3644813531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:30.416 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:30.423 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:42:30.423 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:42:30.439 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:42:30.446 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:42:30.449 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:30.477 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:42:30.490 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:42:30.491 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:42:30.491 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:42:30.491 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:42:30.491 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:42:30.492 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:42:30.492 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:42:30.492 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:42:30.492 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:42:30.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:30 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1770769718' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:30.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:30 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:30.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:30 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3644813531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:30.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:30 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1770769718' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:30.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:30 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:30.663 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:30 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3644813531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:30.832 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:42:31.552 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:31 smithi138 ceph-mon[192513]: pgmap v1469: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 115 op/s; 8511/233217 objects degraded (3.649%); 4749/233217 objects misplaced (2.036%); 461 KiB/s, 7 objects/s recovering 2023-10-01T09:42:31.552 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:31 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:31.552 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:31 smithi138 ceph-mon[192513]: failed to encode map e515 with expected crc 2023-10-01T09:42:31.552 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:31 smithi138 ceph-mon[192513]: failed to encode map e515 with expected crc 2023-10-01T09:42:31.552 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:31 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:42:31.553 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:31 smithi138 ceph-mon[192513]: osdmap e515: 8 total, 8 up, 7 in 2023-10-01T09:42:31.553 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:31 smithi138 ceph-mon[185886]: pgmap v1469: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 115 op/s; 8511/233217 objects degraded (3.649%); 4749/233217 objects misplaced (2.036%); 461 KiB/s, 7 objects/s recovering 2023-10-01T09:42:31.554 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:31 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:31.554 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:31 smithi138 ceph-mon[185886]: failed to encode map e515 with expected crc 2023-10-01T09:42:31.554 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:31 smithi138 ceph-mon[185886]: failed to encode map e515 with expected crc 2023-10-01T09:42:31.554 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:31 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:42:31.554 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:31 smithi138 ceph-mon[185886]: osdmap e515: 8 total, 8 up, 7 in 2023-10-01T09:42:31.678 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:31.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:31.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:31.782 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:31.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:31 smithi190 ceph-mon[160435]: pgmap v1469: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 115 op/s; 8511/233217 objects degraded (3.649%); 4749/233217 objects misplaced (2.036%); 461 KiB/s, 7 objects/s recovering 2023-10-01T09:42:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:31 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:31 smithi190 ceph-mon[160435]: failed to encode map e515 with expected crc 2023-10-01T09:42:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:31 smithi190 ceph-mon[160435]: failed to encode map e515 with expected crc 2023-10-01T09:42:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:31 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:42:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:31 smithi190 ceph-mon[160435]: osdmap e515: 8 total, 8 up, 7 in 2023-10-01T09:42:31.817 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:31.888 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:31.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:31.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:31.995 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:32.024 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:32.187 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:32.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:32.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:32.290 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:32.319 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:32.379 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:32.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:32.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:32.410 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:42:32.410 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:42:32.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:42:32.411 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:42:32.411 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.412 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:145796 9.c 9:3a5a6dfb:::benchmark_data_smithi138_497829_object43411:head [delete] snapc 0=[] ondisk+write+known_if_redirected e444)", 2023-10-01T09:42:32.412 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:16.533300+0000", 2023-10-01T09:42:32.412 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 315.87230783500002, 2023-10-01T09:42:32.412 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.29276776999999998, 2023-10-01T09:42:32.412 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.412 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.412 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.412 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.412 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.413 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 145796 2023-10-01T09:42:32.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.413 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.413 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.413 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533300+0000", 2023-10-01T09:42:32.413 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.413 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.413 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.414 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.414 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533300+0000", 2023-10-01T09:42:32.414 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:42:32.414 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.414 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.414 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.414 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533298+0000", 2023-10-01T09:42:32.414 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0709999999999995e-06 2023-10-01T09:42:32.414 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.414 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.415 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.415 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533306+0000", 2023-10-01T09:42:32.415 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5869999999999999e-05 2023-10-01T09:42:32.415 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.415 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.415 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.415 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533322+0000", 2023-10-01T09:42:32.415 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.815e-06 2023-10-01T09:42:32.415 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.416 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.416 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.416 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533329+0000", 2023-10-01T09:42:32.416 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00163053 2023-10-01T09:42:32.416 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.416 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.416 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.416 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.534959+0000", 2023-10-01T09:42:32.416 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014893600000000001 2023-10-01T09:42:32.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.417 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.417 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.535108+0000", 2023-10-01T09:42:32.417 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000164644 2023-10-01T09:42:32.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.417 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,3,4", 2023-10-01T09:42:32.417 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.535273+0000", 2023-10-01T09:42:32.417 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0048375980000000002 2023-10-01T09:42:32.417 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.417 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.418 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.418 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.540110+0000", 2023-10-01T09:42:32.418 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001286875 2023-10-01T09:42:32.418 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.418 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.418 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.418 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.541397+0000", 2023-10-01T09:42:32.419 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001817658 2023-10-01T09:42:32.419 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.419 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.419 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.419 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.543215+0000", 2023-10-01T09:42:32.419 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28271621800000002 2023-10-01T09:42:32.419 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.419 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.419 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.420 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.825931+0000", 2023-10-01T09:42:32.420 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1245999999999998e-05 2023-10-01T09:42:32.420 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.420 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.420 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.420 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.825972+0000", 2023-10-01T09:42:32.420 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5760000000000005e-05 2023-10-01T09:42:32.420 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.420 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.420 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.421 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.826068+0000", 2023-10-01T09:42:32.421 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.29276776999999998 2023-10-01T09:42:32.421 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.421 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.421 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.421 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.421 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.421 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147931 9.3 9:cc030d19:::benchmark_data_smithi138_497829_object45546:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:42:32.421 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.304355+0000", 2023-10-01T09:42:32.421 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 310.10125328599997, 2023-10-01T09:42:32.422 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3960295309999999, 2023-10-01T09:42:32.422 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.422 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.422 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.422 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.422 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.422 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147931 2023-10-01T09:42:32.422 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.422 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.423 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.423 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.423 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304355+0000", 2023-10-01T09:42:32.423 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.423 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.423 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.423 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.423 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304355+0000", 2023-10-01T09:42:32.423 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:32.423 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.424 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.424 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.424 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304354+0000", 2023-10-01T09:42:32.424 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1670000000000004e-06 2023-10-01T09:42:32.424 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.424 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.424 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.424 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304360+0000", 2023-10-01T09:42:32.424 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.27e-05 2023-10-01T09:42:32.424 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.425 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.425 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.425 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304372+0000", 2023-10-01T09:42:32.425 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.815e-06 2023-10-01T09:42:32.425 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.425 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.425 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.425 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304379+0000", 2023-10-01T09:42:32.425 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000106932 2023-10-01T09:42:32.425 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.425 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.425 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.426 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304486+0000", 2023-10-01T09:42:32.426 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3089999999999996e-06 2023-10-01T09:42:32.426 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.426 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.426 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.426 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304495+0000", 2023-10-01T09:42:32.426 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.322764053 2023-10-01T09:42:32.426 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.426 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.426 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.426 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.627259+0000", 2023-10-01T09:42:32.426 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4950000000000008e-06 2023-10-01T09:42:32.427 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.427 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.427 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.427 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.627267+0000", 2023-10-01T09:42:32.427 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.028340244 2023-10-01T09:42:32.427 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.427 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.427 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.427 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.655607+0000", 2023-10-01T09:42:32.427 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000144229 2023-10-01T09:42:32.427 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.428 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.428 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.428 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.655752+0000", 2023-10-01T09:42:32.428 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00018547299999999999 2023-10-01T09:42:32.428 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.428 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.428 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:42:32.428 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.655937+0000", 2023-10-01T09:42:32.428 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021581132999999999 2023-10-01T09:42:32.428 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.428 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.429 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.429 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677518+0000", 2023-10-01T09:42:32.429 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.022700635 2023-10-01T09:42:32.429 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.429 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.429 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.429 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700219+0000", 2023-10-01T09:42:32.429 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2954999999999998e-05 2023-10-01T09:42:32.429 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.429 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.429 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.429 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700262+0000", 2023-10-01T09:42:32.430 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4472000000000002e-05 2023-10-01T09:42:32.430 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.430 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.430 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.430 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700326+0000", 2023-10-01T09:42:32.430 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8291000000000001e-05 2023-10-01T09:42:32.430 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.430 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.430 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.430 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700385+0000", 2023-10-01T09:42:32.430 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3960295309999999 2023-10-01T09:42:32.431 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.431 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.431 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.431 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.431 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.431 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147969 9.c 9:344c11ae:::benchmark_data_smithi138_497829_object45584:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:42:32.431 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.490163+0000", 2023-10-01T09:42:32.431 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.91544547400002, 2023-10-01T09:42:32.431 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.18335328, 2023-10-01T09:42:32.431 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.431 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.431 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.432 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.432 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.432 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147969 2023-10-01T09:42:32.432 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.432 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.432 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.432 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.432 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490163+0000", 2023-10-01T09:42:32.432 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.432 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.432 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.433 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.433 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490163+0000", 2023-10-01T09:42:32.433 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:32.433 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.433 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.433 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.433 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490162+0000", 2023-10-01T09:42:32.433 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7320000000000003e-06 2023-10-01T09:42:32.433 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.433 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.433 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.433 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490167+0000", 2023-10-01T09:42:32.434 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1588e-05 2023-10-01T09:42:32.434 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.434 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.434 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.434 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490179+0000", 2023-10-01T09:42:32.434 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1599999999999997e-06 2023-10-01T09:42:32.434 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.434 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.434 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.434 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490184+0000", 2023-10-01T09:42:32.434 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7620999999999998e-05 2023-10-01T09:42:32.434 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.435 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.435 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.435 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490222+0000", 2023-10-01T09:42:32.435 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5396e-05 2023-10-01T09:42:32.435 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.435 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.435 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.435 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490237+0000", 2023-10-01T09:42:32.435 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14082081399999999 2023-10-01T09:42:32.435 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.435 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.436 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.436 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.631058+0000", 2023-10-01T09:42:32.436 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.152e-06 2023-10-01T09:42:32.436 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.436 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.436 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.436 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.631063+0000", 2023-10-01T09:42:32.436 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0296697379999999 2023-10-01T09:42:32.436 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.436 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.437 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.437 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660733+0000", 2023-10-01T09:42:32.437 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011841399999999999 2023-10-01T09:42:32.437 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.437 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.438 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.438 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660851+0000", 2023-10-01T09:42:32.438 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013647799999999999 2023-10-01T09:42:32.438 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.438 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.438 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:42:32.438 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660988+0000", 2023-10-01T09:42:32.438 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045588740000000001 2023-10-01T09:42:32.438 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.438 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.438 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.439 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665546+0000", 2023-10-01T09:42:32.439 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078629470000000003 2023-10-01T09:42:32.439 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.439 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.439 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.439 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673409+0000", 2023-10-01T09:42:32.439 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4159999999999997e-05 2023-10-01T09:42:32.439 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.439 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.439 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.439 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673454+0000", 2023-10-01T09:42:32.439 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2487000000000003e-05 2023-10-01T09:42:32.440 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.440 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.440 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.440 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673486+0000", 2023-10-01T09:42:32.440 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0018e-05 2023-10-01T09:42:32.440 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.440 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.440 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.440 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673516+0000", 2023-10-01T09:42:32.440 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.18335328 2023-10-01T09:42:32.440 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.440 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.441 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.441 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.441 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.441 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147981 9.3 9:c3aeeda4:::benchmark_data_smithi138_497829_object45596:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:42:32.441 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.563343+0000", 2023-10-01T09:42:32.441 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.842265263, 2023-10-01T09:42:32.441 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1372912850000001, 2023-10-01T09:42:32.441 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.441 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.441 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.441 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.442 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.442 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147981 2023-10-01T09:42:32.442 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.442 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.442 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.442 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.442 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563343+0000", 2023-10-01T09:42:32.442 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.442 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.442 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.442 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.442 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563343+0000", 2023-10-01T09:42:32.443 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999933 2023-10-01T09:42:32.443 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.443 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.443 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.443 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563336+0000", 2023-10-01T09:42:32.443 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1314e-05 2023-10-01T09:42:32.443 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.443 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.443 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.443 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563347+0000", 2023-10-01T09:42:32.443 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0358e-05 2023-10-01T09:42:32.443 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.444 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.444 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.444 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563358+0000", 2023-10-01T09:42:32.444 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3170000000000003e-06 2023-10-01T09:42:32.444 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.444 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.444 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.444 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563362+0000", 2023-10-01T09:42:32.444 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1795999999999997e-05 2023-10-01T09:42:32.444 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.444 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.445 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.445 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563414+0000", 2023-10-01T09:42:32.445 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1110999999999999e-05 2023-10-01T09:42:32.445 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.445 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.445 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.445 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563425+0000", 2023-10-01T09:42:32.445 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.063862261000000003 2023-10-01T09:42:32.445 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.445 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.446 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.446 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.627287+0000", 2023-10-01T09:42:32.446 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4970000000000001e-06 2023-10-01T09:42:32.446 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.446 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.446 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.446 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.627293+0000", 2023-10-01T09:42:32.446 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0332602529999999 2023-10-01T09:42:32.446 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.446 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.446 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.447 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660553+0000", 2023-10-01T09:42:32.447 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3721999999999996e-05 2023-10-01T09:42:32.447 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.447 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.447 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.447 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660637+0000", 2023-10-01T09:42:32.447 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012805999999999999 2023-10-01T09:42:32.447 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.447 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.447 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:42:32.447 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660765+0000", 2023-10-01T09:42:32.448 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.025676641 2023-10-01T09:42:32.448 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.448 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.448 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.448 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686441+0000", 2023-10-01T09:42:32.448 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013987271000000001 2023-10-01T09:42:32.448 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.448 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.448 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.448 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700429+0000", 2023-10-01T09:42:32.448 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000107107 2023-10-01T09:42:32.448 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.449 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.449 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.449 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700536+0000", 2023-10-01T09:42:32.449 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0742000000000002e-05 2023-10-01T09:42:32.449 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.449 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.449 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.449 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700597+0000", 2023-10-01T09:42:32.449 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7715000000000001e-05 2023-10-01T09:42:32.449 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.449 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.450 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.450 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700634+0000", 2023-10-01T09:42:32.450 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1372912850000001 2023-10-01T09:42:32.450 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.450 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.450 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.450 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.450 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.450 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147984 9.3 9:c9142d5f:::benchmark_data_smithi138_497829_object45599:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:42:32.450 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.707974+0000", 2023-10-01T09:42:32.450 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.69763475799999, 2023-10-01T09:42:32.450 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99286916199999997, 2023-10-01T09:42:32.451 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.451 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.451 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.451 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.451 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.451 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147984 2023-10-01T09:42:32.451 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.451 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.451 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.451 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.451 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.707974+0000", 2023-10-01T09:42:32.451 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.452 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.452 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.452 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.452 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.707974+0000", 2023-10-01T09:42:32.452 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:42:32.452 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.452 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.452 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.452 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.707971+0000", 2023-10-01T09:42:32.452 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1004e-05 2023-10-01T09:42:32.452 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.453 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.453 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.453 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.707982+0000", 2023-10-01T09:42:32.453 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8074e-05 2023-10-01T09:42:32.453 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.453 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.453 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.453 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.708000+0000", 2023-10-01T09:42:32.453 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.503e-06 2023-10-01T09:42:32.453 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.453 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.453 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.454 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.708008+0000", 2023-10-01T09:42:32.454 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0042882299999999996 2023-10-01T09:42:32.454 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.454 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.454 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.454 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712296+0000", 2023-10-01T09:42:32.454 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0655e-05 2023-10-01T09:42:32.454 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.454 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.455 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.455 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712307+0000", 2023-10-01T09:42:32.455 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.95292995599999997 2023-10-01T09:42:32.455 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.455 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.455 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.455 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665237+0000", 2023-10-01T09:42:32.455 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011695400000000001 2023-10-01T09:42:32.455 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.455 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.455 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.456 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665354+0000", 2023-10-01T09:42:32.456 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000102116 2023-10-01T09:42:32.456 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.456 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.456 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:42:32.456 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665456+0000", 2023-10-01T09:42:32.456 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020992165 2023-10-01T09:42:32.456 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.456 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.456 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.456 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686448+0000", 2023-10-01T09:42:32.456 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014224582 2023-10-01T09:42:32.457 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.457 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.457 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.457 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700672+0000", 2023-10-01T09:42:32.457 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2769000000000006e-05 2023-10-01T09:42:32.457 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.457 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.457 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.457 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700765+0000", 2023-10-01T09:42:32.457 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4345000000000003e-05 2023-10-01T09:42:32.457 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.457 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.458 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.458 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700810+0000", 2023-10-01T09:42:32.458 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3080000000000002e-05 2023-10-01T09:42:32.458 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.458 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.458 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.458 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700843+0000", 2023-10-01T09:42:32.458 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99286916199999997 2023-10-01T09:42:32.458 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.458 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.458 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.458 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.459 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.459 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147985 9.3 9:c26d6b78:::benchmark_data_smithi138_497829_object45600:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:42:32.459 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.712409+0000", 2023-10-01T09:42:32.459 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.69319951099999, 2023-10-01T09:42:32.459 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98851199499999998, 2023-10-01T09:42:32.459 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.459 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.459 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.459 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.459 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.459 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147985 2023-10-01T09:42:32.459 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.460 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.460 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.460 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.460 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712409+0000", 2023-10-01T09:42:32.460 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.460 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.460 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.460 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.460 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712409+0000", 2023-10-01T09:42:32.460 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:32.460 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.460 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.461 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.461 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712407+0000", 2023-10-01T09:42:32.461 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8050000000000007e-06 2023-10-01T09:42:32.461 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.461 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.461 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.461 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712415+0000", 2023-10-01T09:42:32.461 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6009999999999995e-06 2023-10-01T09:42:32.461 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.461 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.461 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.461 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712424+0000", 2023-10-01T09:42:32.461 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8139999999999997e-06 2023-10-01T09:42:32.462 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.462 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.462 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.462 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712429+0000", 2023-10-01T09:42:32.462 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4980999999999993e-05 2023-10-01T09:42:32.462 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.462 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.462 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.462 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712494+0000", 2023-10-01T09:42:32.462 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5420000000000002e-06 2023-10-01T09:42:32.462 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.462 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.463 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.463 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712501+0000", 2023-10-01T09:42:32.463 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.95723478900000003 2023-10-01T09:42:32.463 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.463 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.463 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.463 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.669736+0000", 2023-10-01T09:42:32.463 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000116869 2023-10-01T09:42:32.463 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.463 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.464 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.464 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.669853+0000", 2023-10-01T09:42:32.464 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016995500000000001 2023-10-01T09:42:32.464 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.464 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.464 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:42:32.464 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.670023+0000", 2023-10-01T09:42:32.464 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016431211000000001 2023-10-01T09:42:32.464 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.464 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.465 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.465 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686454+0000", 2023-10-01T09:42:32.465 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014249539 2023-10-01T09:42:32.465 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.465 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.465 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.465 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700703+0000", 2023-10-01T09:42:32.465 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000168577 2023-10-01T09:42:32.465 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.465 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.465 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.466 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700872+0000", 2023-10-01T09:42:32.466 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.245e-05 2023-10-01T09:42:32.466 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.466 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.466 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.466 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700894+0000", 2023-10-01T09:42:32.466 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.639e-05 2023-10-01T09:42:32.466 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.466 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.466 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.466 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700921+0000", 2023-10-01T09:42:32.467 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98851199499999998 2023-10-01T09:42:32.467 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.467 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.467 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.467 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.467 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.467 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148000 9.c 9:3eba568d:::benchmark_data_smithi138_497829_object45615:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:42:32.467 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.786332+0000", 2023-10-01T09:42:32.467 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.61927579500002, 2023-10-01T09:42:32.467 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.89635619300000002, 2023-10-01T09:42:32.467 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.468 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.468 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.468 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.468 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.468 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148000 2023-10-01T09:42:32.468 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.468 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.468 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.468 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.468 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786332+0000", 2023-10-01T09:42:32.468 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.468 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.469 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.469 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.469 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786332+0000", 2023-10-01T09:42:32.469 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:42:32.469 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.469 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.469 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.469 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786330+0000", 2023-10-01T09:42:32.469 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6572e-05 2023-10-01T09:42:32.469 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.469 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.469 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.470 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786347+0000", 2023-10-01T09:42:32.470 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7059e-05 2023-10-01T09:42:32.470 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.470 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.470 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.470 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786364+0000", 2023-10-01T09:42:32.470 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.357e-06 2023-10-01T09:42:32.470 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.470 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.470 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.470 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786371+0000", 2023-10-01T09:42:32.471 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7531999999999998e-05 2023-10-01T09:42:32.471 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.471 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.471 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.471 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786419+0000", 2023-10-01T09:42:32.471 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0134e-05 2023-10-01T09:42:32.471 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.471 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.471 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.471 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786429+0000", 2023-10-01T09:42:32.471 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.879158459 2023-10-01T09:42:32.471 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.472 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.472 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.472 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665587+0000", 2023-10-01T09:42:32.472 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000112847 2023-10-01T09:42:32.472 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.472 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.472 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.472 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665700+0000", 2023-10-01T09:42:32.472 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011616399999999999 2023-10-01T09:42:32.472 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.472 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.473 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:42:32.473 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665816+0000", 2023-10-01T09:42:32.473 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016651933000000001 2023-10-01T09:42:32.473 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.473 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.473 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.473 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682468+0000", 2023-10-01T09:42:32.473 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3857000000000006e-05 2023-10-01T09:42:32.473 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.473 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.473 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.473 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682562+0000", 2023-10-01T09:42:32.474 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4292999999999998e-05 2023-10-01T09:42:32.474 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.474 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.474 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.474 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682596+0000", 2023-10-01T09:42:32.474 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7022999999999999e-05 2023-10-01T09:42:32.474 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.474 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.474 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.474 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682653+0000", 2023-10-01T09:42:32.474 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5318999999999998e-05 2023-10-01T09:42:32.474 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.475 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.475 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.475 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682689+0000", 2023-10-01T09:42:32.475 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.89635619300000002 2023-10-01T09:42:32.475 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.475 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.475 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.475 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.475 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.475 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148010 9.c 9:365ddc65:::benchmark_data_smithi138_497829_object45625:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:42:32.475 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.803374+0000", 2023-10-01T09:42:32.475 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.60223477800002, 2023-10-01T09:42:32.476 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.87947385199999994, 2023-10-01T09:42:32.476 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.476 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.476 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.476 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.476 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.476 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148010 2023-10-01T09:42:32.476 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.476 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.476 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.476 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.476 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803374+0000", 2023-10-01T09:42:32.477 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.477 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.477 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.477 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.477 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803374+0000", 2023-10-01T09:42:32.477 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:42:32.477 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.477 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.477 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.477 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803372+0000", 2023-10-01T09:42:32.477 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6549999999999999e-06 2023-10-01T09:42:32.478 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.478 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.478 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.478 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803381+0000", 2023-10-01T09:42:32.478 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7244999999999998e-05 2023-10-01T09:42:32.478 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.478 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.478 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.478 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803398+0000", 2023-10-01T09:42:32.478 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6320000000000008e-06 2023-10-01T09:42:32.478 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.479 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.479 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.479 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803406+0000", 2023-10-01T09:42:32.479 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9969000000000002e-05 2023-10-01T09:42:32.479 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.479 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.479 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.479 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803456+0000", 2023-10-01T09:42:32.479 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4809999999999995e-06 2023-10-01T09:42:32.479 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.479 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.479 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.479 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803466+0000", 2023-10-01T09:42:32.480 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.86655919599999998 2023-10-01T09:42:32.480 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.480 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.480 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.480 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.670025+0000", 2023-10-01T09:42:32.480 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9349000000000003e-05 2023-10-01T09:42:32.480 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.480 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.480 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.480 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.670094+0000", 2023-10-01T09:42:32.480 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7009000000000007e-05 2023-10-01T09:42:32.481 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.481 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.481 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:42:32.481 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.670181+0000", 2023-10-01T09:42:32.481 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012516879999999999 2023-10-01T09:42:32.481 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.481 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.481 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.481 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682698+0000", 2023-10-01T09:42:32.481 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3449000000000003e-05 2023-10-01T09:42:32.481 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.481 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.482 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.482 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682751+0000", 2023-10-01T09:42:32.482 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4245999999999997e-05 2023-10-01T09:42:32.482 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.482 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.482 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.482 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682786+0000", 2023-10-01T09:42:32.482 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9830999999999999e-05 2023-10-01T09:42:32.482 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.482 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.482 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.483 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682816+0000", 2023-10-01T09:42:32.483 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1853999999999997e-05 2023-10-01T09:42:32.483 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.483 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.483 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.483 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682847+0000", 2023-10-01T09:42:32.483 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.87947385199999994 2023-10-01T09:42:32.483 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.483 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.483 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.483 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.484 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.484 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148037 9.3 9:c1ff9704:::benchmark_data_smithi138_497829_object45652:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:42:32.484 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.881445+0000", 2023-10-01T09:42:32.484 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.52416371800001, 2023-10-01T09:42:32.484 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81953648800000001, 2023-10-01T09:42:32.484 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.484 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.484 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.484 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.484 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.484 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148037 2023-10-01T09:42:32.484 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.485 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.485 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.485 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.485 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881445+0000", 2023-10-01T09:42:32.485 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.485 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.485 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.485 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.485 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881445+0000", 2023-10-01T09:42:32.485 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:32.485 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.485 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.486 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.486 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881443+0000", 2023-10-01T09:42:32.486 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2189999999999993e-06 2023-10-01T09:42:32.486 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.486 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.486 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.486 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881452+0000", 2023-10-01T09:42:32.486 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2901000000000001e-05 2023-10-01T09:42:32.486 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.486 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.486 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.487 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881465+0000", 2023-10-01T09:42:32.487 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1500000000000004e-06 2023-10-01T09:42:32.487 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.487 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.487 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.487 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881471+0000", 2023-10-01T09:42:32.487 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9982000000000003e-05 2023-10-01T09:42:32.487 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.487 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.487 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.487 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881531+0000", 2023-10-01T09:42:32.487 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1070000000000001e-05 2023-10-01T09:42:32.488 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.488 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.488 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.488 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881542+0000", 2023-10-01T09:42:32.488 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.79148766599999998 2023-10-01T09:42:32.488 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.488 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.488 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.488 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673030+0000", 2023-10-01T09:42:32.488 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000120554 2023-10-01T09:42:32.488 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.489 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.489 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.489 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673151+0000", 2023-10-01T09:42:32.489 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000136756 2023-10-01T09:42:32.489 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.489 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.489 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:42:32.489 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673287+0000", 2023-10-01T09:42:32.489 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017487816999999999 2023-10-01T09:42:32.489 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.489 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.490 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.490 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.690775+0000", 2023-10-01T09:42:32.490 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0099592180000000006 2023-10-01T09:42:32.490 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.490 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.490 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.490 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700734+0000", 2023-10-01T09:42:32.490 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00020710800000000001 2023-10-01T09:42:32.490 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.490 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.490 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.491 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700942+0000", 2023-10-01T09:42:32.491 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0435999999999999e-05 2023-10-01T09:42:32.491 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.491 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.491 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.491 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700962+0000", 2023-10-01T09:42:32.491 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9011000000000001e-05 2023-10-01T09:42:32.491 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.491 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.491 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.491 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700981+0000", 2023-10-01T09:42:32.491 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81953648800000001 2023-10-01T09:42:32.492 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.492 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.492 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.492 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.492 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.492 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148041 9.3 9:c062535c:::benchmark_data_smithi138_497829_object45656:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:42:32.492 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.886259+0000", 2023-10-01T09:42:32.492 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.51934958999999, 2023-10-01T09:42:32.492 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81483497999999999, 2023-10-01T09:42:32.492 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.492 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.493 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.493 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.493 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.493 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148041 2023-10-01T09:42:32.493 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.493 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.493 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.493 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.493 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886259+0000", 2023-10-01T09:42:32.493 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.493 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.494 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.494 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.494 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886259+0000", 2023-10-01T09:42:32.494 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:42:32.494 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.494 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.494 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.494 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886257+0000", 2023-10-01T09:42:32.494 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1755000000000001e-05 2023-10-01T09:42:32.494 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.494 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.495 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.495 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886278+0000", 2023-10-01T09:42:32.495 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8885e-05 2023-10-01T09:42:32.495 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.495 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.495 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.495 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886297+0000", 2023-10-01T09:42:32.495 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7020000000000001e-06 2023-10-01T09:42:32.495 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.495 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.495 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.495 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886305+0000", 2023-10-01T09:42:32.496 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6515999999999999e-05 2023-10-01T09:42:32.496 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.496 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.496 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.496 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886361+0000", 2023-10-01T09:42:32.496 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2311e-05 2023-10-01T09:42:32.496 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.496 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.496 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.496 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886374+0000", 2023-10-01T09:42:32.496 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.79119420699999998 2023-10-01T09:42:32.496 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.497 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.497 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.497 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677568+0000", 2023-10-01T09:42:32.497 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3848e-05 2023-10-01T09:42:32.497 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.497 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.497 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.497 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677662+0000", 2023-10-01T09:42:32.497 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7428000000000003e-05 2023-10-01T09:42:32.498 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.498 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.498 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:42:32.498 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677759+0000", 2023-10-01T09:42:32.498 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013022529 2023-10-01T09:42:32.498 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.498 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.498 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.498 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.690782+0000", 2023-10-01T09:42:32.499 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010232722 2023-10-01T09:42:32.499 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.499 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.499 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.499 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701015+0000", 2023-10-01T09:42:32.499 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1299e-05 2023-10-01T09:42:32.499 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.499 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.499 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.499 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701046+0000", 2023-10-01T09:42:32.499 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7508000000000002e-05 2023-10-01T09:42:32.500 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.500 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.500 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.500 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701073+0000", 2023-10-01T09:42:32.500 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0361e-05 2023-10-01T09:42:32.500 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.500 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.500 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.501 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701094+0000", 2023-10-01T09:42:32.501 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81483497999999999 2023-10-01T09:42:32.501 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.501 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.501 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.501 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.501 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.501 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148060 9.3 9:cf7f7f25:::benchmark_data_smithi138_497829_object45675:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:42:32.501 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.972375+0000", 2023-10-01T09:42:32.501 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.43323290000001, 2023-10-01T09:42:32.501 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.72881600400000002, 2023-10-01T09:42:32.501 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.502 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.502 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.502 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.502 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.502 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148060 2023-10-01T09:42:32.502 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.502 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.502 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.502 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.502 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972375+0000", 2023-10-01T09:42:32.503 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.503 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.503 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.503 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.503 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972375+0000", 2023-10-01T09:42:32.503 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:42:32.503 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.503 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.503 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.503 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972374+0000", 2023-10-01T09:42:32.503 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1189000000000001e-05 2023-10-01T09:42:32.503 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.504 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.504 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.504 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972385+0000", 2023-10-01T09:42:32.504 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7291000000000001e-05 2023-10-01T09:42:32.504 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.504 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.504 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.504 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972402+0000", 2023-10-01T09:42:32.504 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9349999999999997e-06 2023-10-01T09:42:32.504 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.504 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.505 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.505 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972409+0000", 2023-10-01T09:42:32.505 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8267000000000003e-05 2023-10-01T09:42:32.505 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.505 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.505 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.505 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972467+0000", 2023-10-01T09:42:32.505 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1688999999999999e-05 2023-10-01T09:42:32.505 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.505 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.505 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.505 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972479+0000", 2023-10-01T09:42:32.506 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.70958122800000001 2023-10-01T09:42:32.506 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.506 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.506 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.506 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682060+0000", 2023-10-01T09:42:32.506 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3533000000000001e-05 2023-10-01T09:42:32.506 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.506 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.506 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.506 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682144+0000", 2023-10-01T09:42:32.506 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000118854 2023-10-01T09:42:32.507 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.507 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.507 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:42:32.507 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682263+0000", 2023-10-01T09:42:32.507 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012972465000000001 2023-10-01T09:42:32.507 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.507 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.507 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.507 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.695235+0000", 2023-10-01T09:42:32.507 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0058802189999999999 2023-10-01T09:42:32.507 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.507 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.508 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.508 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701115+0000", 2023-10-01T09:42:32.508 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5998999999999999e-05 2023-10-01T09:42:32.508 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.508 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.508 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.508 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701151+0000", 2023-10-01T09:42:32.508 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0852999999999998e-05 2023-10-01T09:42:32.508 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.509 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.509 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.509 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701172+0000", 2023-10-01T09:42:32.510 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9335999999999999e-05 2023-10-01T09:42:32.510 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.510 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.510 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.510 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701191+0000", 2023-10-01T09:42:32.510 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.72881600400000002 2023-10-01T09:42:32.510 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.510 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.510 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.510 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.511 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.511 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148074 9.c 9:3b5a7b44:::benchmark_data_smithi138_497829_object45689:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:42:32.511 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.007110+0000", 2023-10-01T09:42:32.511 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.39849857500002, 2023-10-01T09:42:32.511 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.68042775499999997, 2023-10-01T09:42:32.511 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.511 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.511 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.511 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.511 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.511 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148074 2023-10-01T09:42:32.512 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.512 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.512 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.512 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.512 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007110+0000", 2023-10-01T09:42:32.512 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.512 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.512 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.512 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.512 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007110+0000", 2023-10-01T09:42:32.512 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:42:32.512 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.513 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.513 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.513 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007108+0000", 2023-10-01T09:42:32.513 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4759999999999998e-06 2023-10-01T09:42:32.513 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.513 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.513 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.513 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007116+0000", 2023-10-01T09:42:32.513 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5668999999999999e-05 2023-10-01T09:42:32.513 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.513 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.513 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.514 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007132+0000", 2023-10-01T09:42:32.514 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6610000000000003e-06 2023-10-01T09:42:32.514 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.514 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.514 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.514 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007139+0000", 2023-10-01T09:42:32.514 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1598000000000003e-05 2023-10-01T09:42:32.514 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.514 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.514 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.514 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007190+0000", 2023-10-01T09:42:32.515 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0829999999999998e-06 2023-10-01T09:42:32.515 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.515 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.515 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.515 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007197+0000", 2023-10-01T09:42:32.515 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.66635359100000002 2023-10-01T09:42:32.515 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.515 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.515 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.515 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673551+0000", 2023-10-01T09:42:32.515 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0988000000000004e-05 2023-10-01T09:42:32.516 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.516 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.516 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.516 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673642+0000", 2023-10-01T09:42:32.516 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010931599999999999 2023-10-01T09:42:32.516 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.516 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.516 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:42:32.516 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673751+0000", 2023-10-01T09:42:32.516 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0089520429999999998 2023-10-01T09:42:32.516 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.517 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.517 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.517 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682703+0000", 2023-10-01T09:42:32.517 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00020024499999999999 2023-10-01T09:42:32.517 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.517 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.517 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.517 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682903+0000", 2023-10-01T09:42:32.517 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045564280000000004 2023-10-01T09:42:32.517 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.517 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.517 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.518 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.687460+0000", 2023-10-01T09:42:32.518 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6650999999999999e-05 2023-10-01T09:42:32.518 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.518 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.518 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.518 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.687496+0000", 2023-10-01T09:42:32.518 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0963000000000001e-05 2023-10-01T09:42:32.518 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.518 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.518 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.519 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.687537+0000", 2023-10-01T09:42:32.519 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.68042775499999997 2023-10-01T09:42:32.519 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.519 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.519 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.519 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.519 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.519 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148078 9.3 9:c8b0a7d5:::benchmark_data_smithi138_497829_object45693:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:42:32.519 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.015312+0000", 2023-10-01T09:42:32.519 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.39029664700001, 2023-10-01T09:42:32.519 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.70011192700000002, 2023-10-01T09:42:32.520 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.520 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.520 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.520 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.520 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.520 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148078 2023-10-01T09:42:32.520 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.520 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.520 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.520 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.521 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015312+0000", 2023-10-01T09:42:32.521 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.521 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.521 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.521 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.521 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015312+0000", 2023-10-01T09:42:32.521 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:42:32.521 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.521 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.521 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.521 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015310+0000", 2023-10-01T09:42:32.521 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7510000000000008e-06 2023-10-01T09:42:32.522 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.522 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.522 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.522 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015317+0000", 2023-10-01T09:42:32.522 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2976000000000001e-05 2023-10-01T09:42:32.522 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.522 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.522 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.522 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015330+0000", 2023-10-01T09:42:32.522 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5339999999999999e-06 2023-10-01T09:42:32.522 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.523 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.523 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.523 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015336+0000", 2023-10-01T09:42:32.523 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5003999999999997e-05 2023-10-01T09:42:32.523 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.523 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.523 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.523 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015391+0000", 2023-10-01T09:42:32.523 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5939999999999999e-06 2023-10-01T09:42:32.523 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.523 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.524 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.524 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015398+0000", 2023-10-01T09:42:32.524 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.67108258300000001 2023-10-01T09:42:32.524 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.524 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.524 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.524 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686480+0000", 2023-10-01T09:42:32.524 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000109298 2023-10-01T09:42:32.524 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.524 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.524 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.524 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686589+0000", 2023-10-01T09:42:32.525 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013103600000000001 2023-10-01T09:42:32.525 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.525 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.525 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:42:32.525 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686720+0000", 2023-10-01T09:42:32.525 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013092273999999999 2023-10-01T09:42:32.525 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.525 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.525 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.525 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.699813+0000", 2023-10-01T09:42:32.525 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001319221 2023-10-01T09:42:32.526 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.526 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.526 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.526 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701132+0000", 2023-10-01T09:42:32.526 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014181050000000001 2023-10-01T09:42:32.526 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.526 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.526 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.526 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715313+0000", 2023-10-01T09:42:32.526 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0659000000000002e-05 2023-10-01T09:42:32.526 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.527 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.527 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.527 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715384+0000", 2023-10-01T09:42:32.527 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9857999999999997e-05 2023-10-01T09:42:32.527 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.527 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.527 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.527 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715424+0000", 2023-10-01T09:42:32.527 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.70011192700000002 2023-10-01T09:42:32.527 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.527 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.528 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.528 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.528 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.528 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148091 9.3 9:c92a3cf4:::benchmark_data_smithi138_497829_object45706:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:42:32.528 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.098648+0000", 2023-10-01T09:42:32.528 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.30696069599998, 2023-10-01T09:42:32.528 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.61693706400000003, 2023-10-01T09:42:32.528 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.528 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.528 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.528 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.528 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.529 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148091 2023-10-01T09:42:32.529 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.529 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.529 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.529 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.529 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098648+0000", 2023-10-01T09:42:32.529 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.529 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.529 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.529 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.529 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098648+0000", 2023-10-01T09:42:32.529 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:42:32.530 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.530 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.530 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.530 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098645+0000", 2023-10-01T09:42:32.530 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3414000000000001e-05 2023-10-01T09:42:32.530 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.530 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.530 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.530 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098658+0000", 2023-10-01T09:42:32.530 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5611e-05 2023-10-01T09:42:32.530 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.530 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.531 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.531 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098674+0000", 2023-10-01T09:42:32.531 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0550000000000002e-06 2023-10-01T09:42:32.531 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.531 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.531 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.531 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098681+0000", 2023-10-01T09:42:32.531 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8112999999999998e-05 2023-10-01T09:42:32.531 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.531 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.531 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.531 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098729+0000", 2023-10-01T09:42:32.532 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7700000000000001e-06 2023-10-01T09:42:32.532 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.532 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.532 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.532 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098737+0000", 2023-10-01T09:42:32.532 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.59207254099999995 2023-10-01T09:42:32.532 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.532 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.532 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.532 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.690810+0000", 2023-10-01T09:42:32.532 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011513 2023-10-01T09:42:32.533 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.533 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.533 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.533 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.690925+0000", 2023-10-01T09:42:32.533 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013793200000000001 2023-10-01T09:42:32.533 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.533 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.533 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:42:32.533 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.691063+0000", 2023-10-01T09:42:32.533 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0087986049999999993 2023-10-01T09:42:32.533 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.534 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.534 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.534 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.699861+0000", 2023-10-01T09:42:32.534 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.01539976 2023-10-01T09:42:32.534 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.534 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.534 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.534 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715261+0000", 2023-10-01T09:42:32.534 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00022492599999999999 2023-10-01T09:42:32.534 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.534 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.535 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.535 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715486+0000", 2023-10-01T09:42:32.535 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1239000000000002e-05 2023-10-01T09:42:32.535 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.535 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.535 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.535 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715537+0000", 2023-10-01T09:42:32.535 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7519999999999999e-05 2023-10-01T09:42:32.535 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.535 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.535 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.535 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715585+0000", 2023-10-01T09:42:32.536 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.61693706400000003 2023-10-01T09:42:32.536 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.536 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.536 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.536 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.536 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.536 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148104 9.c 9:3af9d94a:::benchmark_data_smithi138_497829_object45719:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:42:32.536 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.131588+0000", 2023-10-01T09:42:32.536 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.27402055900001, 2023-10-01T09:42:32.536 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55710424700000005, 2023-10-01T09:42:32.536 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.536 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.537 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.537 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.537 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.537 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148104 2023-10-01T09:42:32.537 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.537 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.537 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.537 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.537 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131588+0000", 2023-10-01T09:42:32.537 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.537 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.537 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.538 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.538 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131588+0000", 2023-10-01T09:42:32.538 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:42:32.538 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.538 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.538 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.538 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131586+0000", 2023-10-01T09:42:32.538 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3512e-05 2023-10-01T09:42:32.538 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.538 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.541 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.541 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131599+0000", 2023-10-01T09:42:32.541 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7265000000000002e-05 2023-10-01T09:42:32.541 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.541 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.541 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.541 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131616+0000", 2023-10-01T09:42:32.541 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5329999999999997e-06 2023-10-01T09:42:32.542 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.542 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.542 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.542 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131624+0000", 2023-10-01T09:42:32.542 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5498999999999999e-05 2023-10-01T09:42:32.542 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.542 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.542 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.542 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131670+0000", 2023-10-01T09:42:32.542 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1920000000000005e-06 2023-10-01T09:42:32.542 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.543 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.543 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.543 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131678+0000", 2023-10-01T09:42:32.543 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.54630871999999997 2023-10-01T09:42:32.543 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.543 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.543 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.543 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677986+0000", 2023-10-01T09:42:32.543 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000109057 2023-10-01T09:42:32.543 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.543 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.543 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.544 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.678095+0000", 2023-10-01T09:42:32.544 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013558 2023-10-01T09:42:32.544 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.544 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.544 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:42:32.544 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.678231+0000", 2023-10-01T09:42:32.544 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0046248979999999997 2023-10-01T09:42:32.544 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.544 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.544 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.544 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682856+0000", 2023-10-01T09:42:32.545 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7060999999999999e-05 2023-10-01T09:42:32.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.545 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.545 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682943+0000", 2023-10-01T09:42:32.545 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.005695922 2023-10-01T09:42:32.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.545 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.545 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688639+0000", 2023-10-01T09:42:32.545 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9950999999999998e-05 2023-10-01T09:42:32.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.546 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.546 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688669+0000", 2023-10-01T09:42:32.546 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3079e-05 2023-10-01T09:42:32.546 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.546 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.546 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688692+0000", 2023-10-01T09:42:32.546 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55710424700000005 2023-10-01T09:42:32.546 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.547 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.547 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.547 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.547 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.547 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148105 9.c 9:3f26355b:::benchmark_data_smithi138_497829_object45720:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:42:32.547 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.132784+0000", 2023-10-01T09:42:32.547 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.27282387299999, 2023-10-01T09:42:32.547 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55601706699999998, 2023-10-01T09:42:32.547 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.547 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.547 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.547 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.548 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.548 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148105 2023-10-01T09:42:32.548 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.548 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.548 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.548 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.548 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132784+0000", 2023-10-01T09:42:32.548 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.548 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.548 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.548 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.549 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132784+0000", 2023-10-01T09:42:32.549 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:42:32.549 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.549 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.549 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.549 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132784+0000", 2023-10-01T09:42:32.549 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2379999999999997e-06 2023-10-01T09:42:32.549 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.549 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.549 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.549 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132792+0000", 2023-10-01T09:42:32.550 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0162e-05 2023-10-01T09:42:32.550 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.550 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.550 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.550 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132802+0000", 2023-10-01T09:42:32.550 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5860000000000004e-06 2023-10-01T09:42:32.550 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.550 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.550 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.550 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132808+0000", 2023-10-01T09:42:32.550 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5326e-05 2023-10-01T09:42:32.551 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.551 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.551 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.551 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132843+0000", 2023-10-01T09:42:32.551 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6760000000000004e-06 2023-10-01T09:42:32.551 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.551 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.551 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.551 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132849+0000", 2023-10-01T09:42:32.551 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55011066399999997 2023-10-01T09:42:32.551 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.552 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682959+0000", 2023-10-01T09:42:32.552 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9478000000000001e-05 2023-10-01T09:42:32.552 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.552 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.683029+0000", 2023-10-01T09:42:32.552 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2601999999999994e-05 2023-10-01T09:42:32.552 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.553 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:42:32.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.683101+0000", 2023-10-01T09:42:32.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0044813810000000004 2023-10-01T09:42:32.553 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.553 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.553 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.687583+0000", 2023-10-01T09:42:32.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000419534 2023-10-01T09:42:32.553 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.553 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.553 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688002+0000", 2023-10-01T09:42:32.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00075196499999999995 2023-10-01T09:42:32.554 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.554 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.554 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.554 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688754+0000", 2023-10-01T09:42:32.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6500999999999999e-05 2023-10-01T09:42:32.554 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.554 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.554 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.554 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688781+0000", 2023-10-01T09:42:32.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0809999999999999e-05 2023-10-01T09:42:32.555 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.555 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.555 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.555 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688801+0000", 2023-10-01T09:42:32.555 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55601706699999998 2023-10-01T09:42:32.555 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.555 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.555 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.555 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.555 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.555 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148110 9.3 9:caeedb58:::benchmark_data_smithi138_497829_object45725:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:42:32.556 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.149195+0000", 2023-10-01T09:42:32.556 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 309.25641337500002, 2023-10-01T09:42:32.556 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.57476550699999995, 2023-10-01T09:42:32.556 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.556 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.556 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.556 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:42:32.556 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:42:32.556 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148110 2023-10-01T09:42:32.556 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.556 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149195+0000", 2023-10-01T09:42:32.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149195+0000", 2023-10-01T09:42:32.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:42:32.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.558 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149193+0000", 2023-10-01T09:42:32.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1443999999999999e-05 2023-10-01T09:42:32.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.558 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149205+0000", 2023-10-01T09:42:32.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0498e-05 2023-10-01T09:42:32.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.558 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149215+0000", 2023-10-01T09:42:32.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2490000000000001e-06 2023-10-01T09:42:32.559 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.559 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.559 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.559 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149220+0000", 2023-10-01T09:42:32.559 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5479999999999998e-05 2023-10-01T09:42:32.559 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.559 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.559 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.559 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149266+0000", 2023-10-01T09:42:32.559 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.516e-06 2023-10-01T09:42:32.559 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.559 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.560 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:42:32.560 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149272+0000", 2023-10-01T09:42:32.560 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.54601357399999995 2023-10-01T09:42:32.560 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.560 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.560 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.560 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.695286+0000", 2023-10-01T09:42:32.560 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000100777 2023-10-01T09:42:32.560 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.560 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.560 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.560 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.695387+0000", 2023-10-01T09:42:32.561 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5045999999999999e-05 2023-10-01T09:42:32.561 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.561 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.561 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:42:32.561 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.695472+0000", 2023-10-01T09:42:32.561 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0047012119999999998 2023-10-01T09:42:32.561 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.561 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.561 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.561 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700173+0000", 2023-10-01T09:42:32.561 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015477148 2023-10-01T09:42:32.561 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.562 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.562 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715650+0000", 2023-10-01T09:42:32.562 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0082495999999999993 2023-10-01T09:42:32.562 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.562 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.562 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.723900+0000", 2023-10-01T09:42:32.562 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4461000000000001e-05 2023-10-01T09:42:32.562 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.563 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.563 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.723934+0000", 2023-10-01T09:42:32.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6350999999999999e-05 2023-10-01T09:42:32.563 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.563 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.563 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.723960+0000", 2023-10-01T09:42:32.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.57476550699999995 2023-10-01T09:42:32.563 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.563 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.563 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.564 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.564 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.564 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:43770 10.8 10:128672d5:::benchmark_data_smithi138_573643_object43769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-10-01T09:42:32.564 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:39.471235+0000", 2023-10-01T09:42:32.564 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 112.934373194, 2023-10-01T09:42:32.564 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.30291565999999998, 2023-10-01T09:42:32.564 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.564 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.564 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.564 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:32.565 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:32.565 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 43770 2023-10-01T09:42:32.565 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.565 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.565 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.565 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.565 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471235+0000", 2023-10-01T09:42:32.565 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.565 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.565 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.565 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.565 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471235+0000", 2023-10-01T09:42:32.566 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:32.566 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.566 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.566 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.566 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471233+0000", 2023-10-01T09:42:32.566 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5883000000000006e-05 2023-10-01T09:42:32.566 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.566 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.566 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.566 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471299+0000", 2023-10-01T09:42:32.566 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2113000000000001e-05 2023-10-01T09:42:32.567 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.567 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.567 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.567 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471311+0000", 2023-10-01T09:42:32.567 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.045e-06 2023-10-01T09:42:32.567 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.567 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.567 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.567 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471317+0000", 2023-10-01T09:42:32.567 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6418999999999997e-05 2023-10-01T09:42:32.568 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.568 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.568 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.568 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471363+0000", 2023-10-01T09:42:32.568 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013875700000000001 2023-10-01T09:42:32.568 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.568 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.568 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.568 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471502+0000", 2023-10-01T09:42:32.568 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000163809 2023-10-01T09:42:32.568 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.569 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.569 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:42:32.569 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471666+0000", 2023-10-01T09:42:32.569 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00176851 2023-10-01T09:42:32.569 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.569 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.569 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.569 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.473434+0000", 2023-10-01T09:42:32.569 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00060446899999999995 2023-10-01T09:42:32.569 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.569 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.570 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.570 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.474038+0000", 2023-10-01T09:42:32.570 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.30000729500000001 2023-10-01T09:42:32.570 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.570 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.570 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.570 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.774046+0000", 2023-10-01T09:42:32.570 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6264000000000002e-05 2023-10-01T09:42:32.570 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.570 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.571 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.571 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.774102+0000", 2023-10-01T09:42:32.571 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8707e-05 2023-10-01T09:42:32.571 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.571 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.571 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.571 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.774151+0000", 2023-10-01T09:42:32.571 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.30291565999999998 2023-10-01T09:42:32.571 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.571 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.571 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.572 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.572 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.572 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:43786 10.8 10:176af140:::benchmark_data_smithi138_573643_object43785:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-10-01T09:42:32.572 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:39.487738+0000", 2023-10-01T09:42:32.572 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 112.917870647, 2023-10-01T09:42:32.572 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28799713799999999, 2023-10-01T09:42:32.572 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.572 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.572 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.572 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:32.572 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:32.572 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 43786 2023-10-01T09:42:32.573 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.573 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.573 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.573 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.573 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487738+0000", 2023-10-01T09:42:32.573 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.573 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.573 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.573 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.573 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487738+0000", 2023-10-01T09:42:32.573 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:32.574 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.574 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.574 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.574 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487736+0000", 2023-10-01T09:42:32.574 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3594e-05 2023-10-01T09:42:32.574 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.574 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.574 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.574 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487820+0000", 2023-10-01T09:42:32.574 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2801e-05 2023-10-01T09:42:32.574 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.575 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.575 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.575 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487833+0000", 2023-10-01T09:42:32.575 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3660000000000003e-06 2023-10-01T09:42:32.575 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.575 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.575 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.575 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487840+0000", 2023-10-01T09:42:32.575 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0319000000000003e-05 2023-10-01T09:42:32.575 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.575 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.576 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.576 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487900+0000", 2023-10-01T09:42:32.576 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014719200000000001 2023-10-01T09:42:32.576 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.576 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.576 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.576 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.488048+0000", 2023-10-01T09:42:32.576 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000160569 2023-10-01T09:42:32.576 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.576 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.576 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:42:32.577 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.488208+0000", 2023-10-01T09:42:32.577 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001939374 2023-10-01T09:42:32.577 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.577 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.577 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.577 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.490148+0000", 2023-10-01T09:42:32.577 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0023000109999999998 2023-10-01T09:42:32.577 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.577 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.577 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.577 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.492448+0000", 2023-10-01T09:42:32.577 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28318099499999999 2023-10-01T09:42:32.578 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.578 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.578 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.578 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.775629+0000", 2023-10-01T09:42:32.578 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4634999999999994e-05 2023-10-01T09:42:32.578 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.578 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.578 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.578 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.775693+0000", 2023-10-01T09:42:32.579 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1610000000000003e-05 2023-10-01T09:42:32.579 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.579 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.579 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.579 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.775735+0000", 2023-10-01T09:42:32.579 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28799713799999999 2023-10-01T09:42:32.579 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.579 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.579 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.579 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.579 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.580 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:46603 10.1 10:852eafe3:::benchmark_data_smithi138_573643_object46602:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e488)", 2023-10-01T09:42:32.580 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.281765+0000", 2023-10-01T09:42:32.580 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 110.12384343799999, 2023-10-01T09:42:32.580 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28067689299999998, 2023-10-01T09:42:32.580 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:32.580 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:42:32.580 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:32.580 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:32.580 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:32.580 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 46603 2023-10-01T09:42:32.580 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.580 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:32.581 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.581 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:32.581 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.281765+0000", 2023-10-01T09:42:32.581 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:32.581 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.581 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.581 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:32.581 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.281765+0000", 2023-10-01T09:42:32.581 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:42:32.581 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.581 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.581 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:32.582 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.281764+0000", 2023-10-01T09:42:32.582 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8473000000000002e-05 2023-10-01T09:42:32.582 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.582 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.582 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:32.582 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.281803+0000", 2023-10-01T09:42:32.582 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7389999999999999e-06 2023-10-01T09:42:32.582 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.582 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.582 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:32.582 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.281808+0000", 2023-10-01T09:42:32.582 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.886e-06 2023-10-01T09:42:32.583 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.583 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.583 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:32.583 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.281810+0000", 2023-10-01T09:42:32.583 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001121457 2023-10-01T09:42:32.583 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.583 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.583 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:32.583 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.282932+0000", 2023-10-01T09:42:32.583 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1827000000000002e-05 2023-10-01T09:42:32.583 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.583 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.584 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:32.584 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.283004+0000", 2023-10-01T09:42:32.584 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6164999999999999e-05 2023-10-01T09:42:32.584 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.584 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.584 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:42:32.584 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.283090+0000", 2023-10-01T09:42:32.584 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00082320799999999997 2023-10-01T09:42:32.584 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.584 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.584 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:42:32.584 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.283913+0000", 2023-10-01T09:42:32.585 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002203132 2023-10-01T09:42:32.585 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.585 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.585 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.585 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.286116+0000", 2023-10-01T09:42:32.585 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.276216884 2023-10-01T09:42:32.585 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.585 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.585 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:42:32.585 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.562333+0000", 2023-10-01T09:42:32.585 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1144999999999997e-05 2023-10-01T09:42:32.585 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.586 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.586 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:42:32.586 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.562404+0000", 2023-10-01T09:42:32.586 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7435999999999999e-05 2023-10-01T09:42:32.586 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:32.586 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:32.586 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:42:32.586 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.562442+0000", 2023-10-01T09:42:32.586 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28067689299999998 2023-10-01T09:42:32.586 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.586 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.586 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.587 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:32.587 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:32.587 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:42:32.587 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:32.616 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:32.670 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:32.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:32.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:32.775 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:32.805 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:32.851 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:32.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:32.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:32.954 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:32.967 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:42:32.968 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 8511/232221 objects degraded (3.665%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1973,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":515,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":77407,"data_bytes":5075834827,"bytes_used":21063872512,"bytes_avail":650901377024,"bytes_total":671965249536,"degraded_objects":8511,"degraded_total":232221,"degraded_ratio":0.036650432131460978,"misplaced_objects":4749,"misplaced_total":232221,"misplaced_ratio":0.020450346867854328,"recovering_objects_per_sec":8,"recovering_bytes_per_sec":552909,"recovering_keys_per_sec":0,"num_objects_recovered":72,"num_bytes_recovered":4718592,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":145},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":33,"modified":"2023-10-01T09:41:58.284615+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:42:32.986 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:33.240 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:42:33.243 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:33.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:33.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:33.350 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:33.377 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:33.430 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:33.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:33.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:33.537 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:33.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:33 smithi190 ceph-mon[160435]: pgmap v1471: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 145 op/s; 8511/232221 objects degraded (3.665%); 4749/232221 objects misplaced (2.045%); 540 KiB/s, 8 objects/s recovering 2023-10-01T09:42:33.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:33 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8511/232221 objects degraded (3.665%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:33.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:33 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2760437674' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:42:33.571 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:33.572 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:42:33.629 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:33.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:33.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:33.675 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:42:33.678 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:42:33.704 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:33 smithi138 ceph-mon[192513]: pgmap v1471: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 145 op/s; 8511/232221 objects degraded (3.665%); 4749/232221 objects misplaced (2.045%); 540 KiB/s, 8 objects/s recovering 2023-10-01T09:42:33.704 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:33 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8511/232221 objects degraded (3.665%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:33.704 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:33 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2760437674' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:42:33.704 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:33 smithi138 ceph-mon[185886]: pgmap v1471: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 145 op/s; 8511/232221 objects degraded (3.665%); 4749/232221 objects misplaced (2.045%); 540 KiB/s, 8 objects/s recovering 2023-10-01T09:42:33.704 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:33 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8511/232221 objects degraded (3.665%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:33.704 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:33 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2760437674' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:42:33.732 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:42:33.732 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:42:33.734 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:33.806 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:33.876 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:33.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:33.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:33.982 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:33.983 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:34.017 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:34.158 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:34.216 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:34.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:34.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:34.323 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:34.351 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:34.496 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:34.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:34.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:34.600 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:34.630 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:35.187 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:42:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:42:34] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.33.4" 2023-10-01T09:42:35.251 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:42:35.251 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:42:35.251 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 0 2023-10-01T09:42:35.251 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:42:35.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:35 smithi138 ceph-mon[185886]: pgmap v1472: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 8511/232107 objects degraded (3.667%); 4749/232107 objects misplaced (2.046%); 461 KiB/s, 7 objects/s recovering 2023-10-01T09:42:35.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:35 smithi138 ceph-mon[192513]: pgmap v1472: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 8511/232107 objects degraded (3.667%); 4749/232107 objects misplaced (2.046%); 461 KiB/s, 7 objects/s recovering 2023-10-01T09:42:35.686 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:35.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:35.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:35.769 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:35.791 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:35.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:35 smithi190 ceph-mon[160435]: pgmap v1472: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 8511/232107 objects degraded (3.667%); 4749/232107 objects misplaced (2.046%); 461 KiB/s, 7 objects/s recovering 2023-10-01T09:42:35.822 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:35.958 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:35.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:35.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:36.022 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:42:36.063 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:36.097 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:36.147 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:36.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:36.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:36.154 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:42:36.178 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:42:36.203 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:42:36.231 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:42:36.252 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:42:36.253 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:36.276 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:42:36.286 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:36.297 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:42:36.316 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:42:36.317 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:42:36.317 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:42:36.317 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:42:36.317 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:42:36.317 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:42:36.318 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:42:36.318 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:42:36.318 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:42:36.341 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:36.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:36.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:36.448 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:36.450 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:42:36.481 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:36 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/144942292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:36.482 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:36 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/144942292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:36.486 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:36.666 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:36 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/144942292' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:36.934 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:36.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:36.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:37.038 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:37.065 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:37.432 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:42:37.618 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:37.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:37.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:37.721 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:37.749 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:37 smithi190 ceph-mon[160435]: pgmap v1473: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 133 op/s; 8488/231015 objects degraded (3.674%); 4701/231015 objects misplaced (2.035%); 454 KiB/s, 7 objects/s recovering 2023-10-01T09:42:37.749 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:37 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3324829178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:42:37.753 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:37.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:37 smithi138 ceph-mon[185886]: pgmap v1473: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 133 op/s; 8488/231015 objects degraded (3.674%); 4701/231015 objects misplaced (2.035%); 454 KiB/s, 7 objects/s recovering 2023-10-01T09:42:37.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:37 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3324829178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:42:37.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:37 smithi138 ceph-mon[192513]: pgmap v1473: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 133 op/s; 8488/231015 objects degraded (3.674%); 4701/231015 objects misplaced (2.035%); 454 KiB/s, 7 objects/s recovering 2023-10-01T09:42:37.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:37 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3324829178' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:42:37.890 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:37.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:37.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:37.996 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:38.063 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:38.140 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:38.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:38.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:38.172 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:42:38.172 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:42:38.247 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:38.275 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:38.377 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:38.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:38.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:38.481 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:38.508 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:38.557 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:38.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:38.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:38.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:38 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:38.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:38 smithi138 ceph-mon[192513]: failed to encode map e516 with expected crc 2023-10-01T09:42:38.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:38 smithi138 ceph-mon[192513]: failed to encode map e516 with expected crc 2023-10-01T09:42:38.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:38 smithi138 ceph-mon[192513]: failed to encode map e516 with expected crc 2023-10-01T09:42:38.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3324829178' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:42:38.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:38 smithi138 ceph-mon[192513]: osdmap e516: 8 total, 8 up, 7 in 2023-10-01T09:42:38.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:38 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8488/231015 objects degraded (3.674%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:38.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:38 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:38.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:38 smithi138 ceph-mon[185886]: failed to encode map e516 with expected crc 2023-10-01T09:42:38.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:38 smithi138 ceph-mon[185886]: failed to encode map e516 with expected crc 2023-10-01T09:42:38.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:38 smithi138 ceph-mon[185886]: failed to encode map e516 with expected crc 2023-10-01T09:42:38.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3324829178' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:42:38.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:38 smithi138 ceph-mon[185886]: osdmap e516: 8 total, 8 up, 7 in 2023-10-01T09:42:38.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:38 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8488/231015 objects degraded (3.674%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:38.645 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:42:38.647 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:42:38.660 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:38.688 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:38 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:38.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:38 smithi190 ceph-mon[160435]: failed to encode map e516 with expected crc 2023-10-01T09:42:38.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:38 smithi190 ceph-mon[160435]: failed to encode map e516 with expected crc 2023-10-01T09:42:38.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:38 smithi190 ceph-mon[160435]: failed to encode map e516 with expected crc 2023-10-01T09:42:38.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3324829178' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:42:38.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:38 smithi190 ceph-mon[160435]: osdmap e516: 8 total, 8 up, 7 in 2023-10-01T09:42:38.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:38 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8488/231015 objects degraded (3.674%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:38.690 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:38.751 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:38.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:38.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:38.857 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:38.887 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:38.933 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:38.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:38.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:39.037 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:39.066 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:42:39.066 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:39.109 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:39.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:39.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:39.215 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:39.242 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:39.287 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:39.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:39.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:39.396 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:39.424 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:39.438 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1475,"stamp":"2023-10-01T09:42:38.294556+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5023799243,"num_objects":76613,"num_object_clones":0,"num_object_copies":229839,"num_objects_missing_on_primary":3304,"num_objects_missing":3304,"num_objects_degraded":8488,"num_objects_misplaced":4701,"num_objects_unfound":0,"num_objects_dirty":76613,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":90961,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7785,"num_bytes_recovered":497058831,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91025,"ondisk_log_size":91025,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20307964,"kb_used_data":15287812,"kb_used_omap":39,"kb_used_meta":5020056,"kb_avail":635908100,"statfs":{"total":671965249536,"available":651169894400,"internally_reserved":0,"allocated":15654719488,"data_stored":15644941224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5140537967},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1,2,0,0,1,1,4,7,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-73793536,"num_objects":-1126,"num_object_clones":0,"num_object_copies":-3378,"num_objects_missing_on_primary":-35,"num_objects_missing":-35,"num_objects_degraded":-23,"num_objects_misplaced":-48,"num_objects_unfound":0,"num_objects_dirty":-1126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1126,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001986"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":291,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.764513+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:42:23.764513+0000","last_peered":"2023-10-01T09:42:23.764513+0000","last_clean":"2023-10-01T09:42:23.764513+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:42:23.764513+0000","last_undegraded":"2023-10-01T09:42:23.764513+0000","last_fullsized":"2023-10-01T09:42:23.764513+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":133,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.008537+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:42:24.008537+0000","last_peered":"2023-10-01T09:42:24.008537+0000","last_clean":"2023-10-01T09:42:24.008537+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:42:24.008537+0000","last_undegraded":"2023-10-01T09:42:24.008537+0000","last_fullsized":"2023-10-01T09:42:24.008537+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":514,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.008415+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:42:24.008415+0000","last_peered":"2023-10-01T09:42:24.008415+0000","last_clean":"2023-10-01T09:42:24.008415+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:42:24.008415+0000","last_undegraded":"2023-10-01T09:42:24.008415+0000","last_fullsized":"2023-10-01T09:42:24.008415+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":572,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.769020+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:42:23.769020+0000","last_peered":"2023-10-01T09:42:23.769020+0000","last_clean":"2023-10-01T09:42:23.769020+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:42:23.769020+0000","last_undegraded":"2023-10-01T09:42:23.769020+0000","last_fullsized":"2023-10-01T09:42:23.769020+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":453,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.768238+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:42:23.768238+0000","last_peered":"2023-10-01T09:42:23.768238+0000","last_clean":"2023-10-01T09:42:23.768238+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:42:23.768238+0000","last_undegraded":"2023-10-01T09:42:23.768238+0000","last_fullsized":"2023-10-01T09:42:23.768238+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":196,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.885079+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:42:23.885079+0000","last_peered":"2023-10-01T09:42:23.885079+0000","last_clean":"2023-10-01T09:42:23.885079+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:42:23.885079+0000","last_undegraded":"2023-10-01T09:42:23.885079+0000","last_fullsized":"2023-10-01T09:42:23.885079+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":572,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.885203+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:42:23.885203+0000","last_peered":"2023-10-01T09:42:23.885203+0000","last_clean":"2023-10-01T09:42:23.885203+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:42:23.885203+0000","last_undegraded":"2023-10-01T09:42:23.885203+0000","last_fullsized":"2023-10-01T09:42:23.885203+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":501,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.443838+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:42:30.443838+0000","last_peered":"2023-10-01T09:42:30.443838+0000","last_clean":"2023-10-01T09:42:30.443838+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:42:30.443838+0000","last_undegraded":"2023-10-01T09:42:30.443838+0000","last_fullsized":"2023-10-01T09:42:30.443838+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":227,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.830698+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:42:23.830698+0000","last_peered":"2023-10-01T09:42:23.830698+0000","last_clean":"2023-10-01T09:42:23.830698+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:42:23.830698+0000","last_undegraded":"2023-10-01T09:42:23.830698+0000","last_fullsized":"2023-10-01T09:42:23.830698+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":34,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113207+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:42:24.113207+0000","last_peered":"2023-10-01T09:42:24.113207+0000","last_clean":"2023-10-01T09:42:24.113207+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:42:24.113207+0000","last_undegraded":"2023-10-01T09:42:24.113207+0000","last_fullsized":"2023-10-01T09:42:24.113207+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":40,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113812+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:42:24.113812+0000","last_peered":"2023-10-01T09:42:24.113812+0000","last_clean":"2023-10-01T09:42:24.113812+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:42:24.113812+0000","last_undegraded":"2023-10-01T09:42:24.113812+0000","last_fullsized":"2023-10-01T09:42:24.113812+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":37,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.008472+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:42:24.008472+0000","last_peered":"2023-10-01T09:42:24.008472+0000","last_clean":"2023-10-01T09:42:24.008472+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:42:24.008472+0000","last_undegraded":"2023-10-01T09:42:24.008472+0000","last_fullsized":"2023-10-01T09:42:24.008472+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":416,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.764767+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:42:23.764767+0000","last_peered":"2023-10-01T09:42:23.764767+0000","last_clean":"2023-10-01T09:42:23.764767+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:42:23.764767+0000","last_undegraded":"2023-10-01T09:42:23.764767+0000","last_fullsized":"2023-10-01T09:42:23.764767+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":297,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.764643+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:42:23.764643+0000","last_peered":"2023-10-01T09:42:23.764643+0000","last_clean":"2023-10-01T09:42:23.764643+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:42:23.764643+0000","last_undegraded":"2023-10-01T09:42:23.764643+0000","last_fullsized":"2023-10-01T09:42:23.764643+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":633,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113430+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:42:24.113430+0000","last_peered":"2023-10-01T09:42:24.113430+0000","last_clean":"2023-10-01T09:42:24.113430+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:42:24.113430+0000","last_undegraded":"2023-10-01T09:42:24.113430+0000","last_fullsized":"2023-10-01T09:42:24.113430+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":370,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113568+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:42:24.113568+0000","last_peered":"2023-10-01T09:42:24.113568+0000","last_clean":"2023-10-01T09:42:24.113568+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:42:24.113568+0000","last_undegraded":"2023-10-01T09:42:24.113568+0000","last_fullsized":"2023-10-01T09:42:24.113568+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":43,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113325+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:42:24.113325+0000","last_peered":"2023-10-01T09:42:24.113325+0000","last_clean":"2023-10-01T09:42:24.113325+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:42:24.113325+0000","last_undegraded":"2023-10-01T09:42:24.113325+0000","last_fullsized":"2023-10-01T09:42:24.113325+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":29,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.830744+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:42:23.830744+0000","last_peered":"2023-10-01T09:42:23.830744+0000","last_clean":"2023-10-01T09:42:23.830744+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:42:23.830744+0000","last_undegraded":"2023-10-01T09:42:23.830744+0000","last_fullsized":"2023-10-01T09:42:23.830744+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":526,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113394+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:42:24.113394+0000","last_peered":"2023-10-01T09:42:24.113394+0000","last_clean":"2023-10-01T09:42:24.113394+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:42:24.113394+0000","last_undegraded":"2023-10-01T09:42:24.113394+0000","last_fullsized":"2023-10-01T09:42:24.113394+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":537,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.769100+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:42:23.769100+0000","last_peered":"2023-10-01T09:42:23.769100+0000","last_clean":"2023-10-01T09:42:23.769100+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:42:23.769100+0000","last_undegraded":"2023-10-01T09:42:23.769100+0000","last_fullsized":"2023-10-01T09:42:23.769100+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":393,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.768175+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:42:23.768175+0000","last_peered":"2023-10-01T09:42:23.768175+0000","last_clean":"2023-10-01T09:42:23.768175+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:42:23.768175+0000","last_undegraded":"2023-10-01T09:42:23.768175+0000","last_fullsized":"2023-10-01T09:42:23.768175+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":43,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113780+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:42:24.113780+0000","last_peered":"2023-10-01T09:42:24.113780+0000","last_clean":"2023-10-01T09:42:24.113780+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:42:24.113780+0000","last_undegraded":"2023-10-01T09:42:24.113780+0000","last_fullsized":"2023-10-01T09:42:24.113780+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"515'5617","reported_seq":5665,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:35.296434+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:42:35.296434+0000","last_peered":"2023-10-01T09:42:35.296434+0000","last_clean":"2023-10-01T09:42:35.296434+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:42:35.296434+0000","last_undegraded":"2023-10-01T09:42:35.296434+0000","last_fullsized":"2023-10-01T09:42:35.296434+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5617,"log_dups_size":0,"ondisk_log_size":5617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312934400,"num_objects":4775,"num_object_clones":0,"num_object_copies":14325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5617,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":38,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113692+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:42:24.113692+0000","last_peered":"2023-10-01T09:42:24.113692+0000","last_clean":"2023-10-01T09:42:24.113692+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:42:24.113692+0000","last_undegraded":"2023-10-01T09:42:24.113692+0000","last_fullsized":"2023-10-01T09:42:24.113692+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":235,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.830594+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:42:23.830594+0000","last_peered":"2023-10-01T09:42:23.830594+0000","last_clean":"2023-10-01T09:42:23.830594+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:42:23.830594+0000","last_undegraded":"2023-10-01T09:42:23.830594+0000","last_fullsized":"2023-10-01T09:42:23.830594+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"515'5583","reported_seq":6803,"reported_epoch":515,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:42:35.285074+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:42:35.285074+0000","last_peered":"2023-10-01T09:42:35.285074+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:42:35.285074+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5583,"log_dups_size":0,"ondisk_log_size":5583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316080128,"num_objects":4823,"num_object_clones":0,"num_object_copies":14469,"num_objects_missing_on_primary":3304,"num_objects_missing":3304,"num_objects_degraded":8058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5583,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":381,"num_bytes_recovered":24969216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1450},{"shards":"0,4,6","objects":3304},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":509,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113742+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:42:24.113742+0000","last_peered":"2023-10-01T09:42:24.113742+0000","last_clean":"2023-10-01T09:42:24.113742+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:42:24.113742+0000","last_undegraded":"2023-10-01T09:42:24.113742+0000","last_fullsized":"2023-10-01T09:42:24.113742+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":569,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.008420+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:42:24.008420+0000","last_peered":"2023-10-01T09:42:24.008420+0000","last_clean":"2023-10-01T09:42:24.008420+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:42:24.008420+0000","last_undegraded":"2023-10-01T09:42:24.008420+0000","last_fullsized":"2023-10-01T09:42:24.008420+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"515'5635","reported_seq":5685,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:36.466280+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:42:36.466280+0000","last_peered":"2023-10-01T09:42:36.466280+0000","last_clean":"2023-10-01T09:42:36.466280+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:42:36.466280+0000","last_undegraded":"2023-10-01T09:42:36.466280+0000","last_fullsized":"2023-10-01T09:42:36.466280+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5635,"log_dups_size":0,"ondisk_log_size":5635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314376192,"num_objects":4797,"num_object_clones":0,"num_object_copies":14391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5635,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":595,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113106+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:42:24.113106+0000","last_peered":"2023-10-01T09:42:24.113106+0000","last_clean":"2023-10-01T09:42:24.113106+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:42:24.113106+0000","last_undegraded":"2023-10-01T09:42:24.113106+0000","last_fullsized":"2023-10-01T09:42:24.113106+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":390,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.008591+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:42:24.008591+0000","last_peered":"2023-10-01T09:42:24.008591+0000","last_clean":"2023-10-01T09:42:24.008591+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:42:24.008591+0000","last_undegraded":"2023-10-01T09:42:24.008591+0000","last_fullsized":"2023-10-01T09:42:24.008591+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"515'5589","reported_seq":12244,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:36.449814+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:42:36.449814+0000","last_peered":"2023-10-01T09:42:36.449814+0000","last_clean":"2023-10-01T09:42:36.449814+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:42:36.449814+0000","last_undegraded":"2023-10-01T09:42:36.449814+0000","last_fullsized":"2023-10-01T09:42:36.449814+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5589,"log_dups_size":0,"ondisk_log_size":5589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315424768,"num_objects":4813,"num_object_clones":0,"num_object_copies":14439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5589,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":235,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.830472+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:42:23.830472+0000","last_peered":"2023-10-01T09:42:23.830472+0000","last_clean":"2023-10-01T09:42:23.830472+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:42:23.830472+0000","last_undegraded":"2023-10-01T09:42:23.830472+0000","last_fullsized":"2023-10-01T09:42:23.830472+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":493,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:26.852478+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:42:26.852478+0000","last_peered":"2023-10-01T09:42:26.852478+0000","last_clean":"2023-10-01T09:42:26.852478+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:42:26.852478+0000","last_undegraded":"2023-10-01T09:42:26.852478+0000","last_fullsized":"2023-10-01T09:42:26.852478+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"515'5643","reported_seq":5693,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:35.174307+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:42:35.174307+0000","last_peered":"2023-10-01T09:42:35.174307+0000","last_clean":"2023-10-01T09:42:35.174307+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:42:35.174307+0000","last_undegraded":"2023-10-01T09:42:35.174307+0000","last_fullsized":"2023-10-01T09:42:35.174307+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5643,"log_dups_size":0,"ondisk_log_size":5643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318832640,"num_objects":4865,"num_object_clones":0,"num_object_copies":14595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5643,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":95,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.764829+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:42:23.764829+0000","last_peered":"2023-10-01T09:42:23.764829+0000","last_clean":"2023-10-01T09:42:23.764829+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:42:23.764829+0000","last_undegraded":"2023-10-01T09:42:23.764829+0000","last_fullsized":"2023-10-01T09:42:23.764829+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":169,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.764851+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:42:23.764851+0000","last_peered":"2023-10-01T09:42:23.764851+0000","last_clean":"2023-10-01T09:42:23.764851+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:42:23.764851+0000","last_undegraded":"2023-10-01T09:42:23.764851+0000","last_fullsized":"2023-10-01T09:42:23.764851+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"515'5603","reported_seq":5659,"reported_epoch":515,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:42:35.202965+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:42:35.202965+0000","last_peered":"2023-10-01T09:42:35.202965+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:42:35.202965+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5603,"log_dups_size":0,"ondisk_log_size":5603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312279040,"num_objects":4765,"num_object_clones":0,"num_object_copies":14295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":420,"num_objects_misplaced":4701,"num_objects_unfound":0,"num_objects_dirty":4765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5603,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4701},{"shards":"4,5","objects":-356}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":413,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.764587+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:42:23.764587+0000","last_peered":"2023-10-01T09:42:23.764587+0000","last_clean":"2023-10-01T09:42:23.764587+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:42:23.764587+0000","last_undegraded":"2023-10-01T09:42:23.764587+0000","last_fullsized":"2023-10-01T09:42:23.764587+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":573,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.008616+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:42:24.008616+0000","last_peered":"2023-10-01T09:42:24.008616+0000","last_clean":"2023-10-01T09:42:24.008616+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:42:24.008616+0000","last_undegraded":"2023-10-01T09:42:24.008616+0000","last_fullsized":"2023-10-01T09:42:24.008616+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"516'5586","reported_seq":5644,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.549602+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:42:37.549602+0000","last_peered":"2023-10-01T09:42:37.549602+0000","last_clean":"2023-10-01T09:42:37.549602+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:42:37.549602+0000","last_undegraded":"2023-10-01T09:42:37.549602+0000","last_fullsized":"2023-10-01T09:42:37.549602+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5586,"log_dups_size":0,"ondisk_log_size":5586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314048512,"num_objects":4792,"num_object_clones":0,"num_object_copies":14376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5586,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":409,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.764471+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:42:23.764471+0000","last_peered":"2023-10-01T09:42:23.764471+0000","last_clean":"2023-10-01T09:42:23.764471+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:42:23.764471+0000","last_undegraded":"2023-10-01T09:42:23.764471+0000","last_fullsized":"2023-10-01T09:42:23.764471+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":250,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.765034+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:42:23.765034+0000","last_peered":"2023-10-01T09:42:23.765034+0000","last_clean":"2023-10-01T09:42:23.765034+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:42:23.765034+0000","last_undegraded":"2023-10-01T09:42:23.765034+0000","last_fullsized":"2023-10-01T09:42:23.765034+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"515'5709","reported_seq":5757,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:36.016958+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:42:36.016958+0000","last_peered":"2023-10-01T09:42:36.016958+0000","last_clean":"2023-10-01T09:42:36.016958+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:42:36.016958+0000","last_undegraded":"2023-10-01T09:42:36.016958+0000","last_fullsized":"2023-10-01T09:42:36.016958+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5709,"log_dups_size":0,"ondisk_log_size":5709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313851904,"num_objects":4789,"num_object_clones":0,"num_object_copies":14367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5709,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":377,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.010293+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:42:24.010293+0000","last_peered":"2023-10-01T09:42:24.010293+0000","last_clean":"2023-10-01T09:42:24.010293+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:42:24.010293+0000","last_undegraded":"2023-10-01T09:42:24.010293+0000","last_fullsized":"2023-10-01T09:42:24.010293+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":424,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.008780+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:42:24.008780+0000","last_peered":"2023-10-01T09:42:24.008780+0000","last_clean":"2023-10-01T09:42:24.008780+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:42:24.008780+0000","last_undegraded":"2023-10-01T09:42:24.008780+0000","last_fullsized":"2023-10-01T09:42:24.008780+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"515'5519","reported_seq":5567,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:36.207689+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:42:36.207689+0000","last_peered":"2023-10-01T09:42:36.207689+0000","last_clean":"2023-10-01T09:42:36.207689+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:42:36.207689+0000","last_undegraded":"2023-10-01T09:42:36.207689+0000","last_fullsized":"2023-10-01T09:42:36.207689+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5519,"log_dups_size":0,"ondisk_log_size":5519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305332224,"num_objects":4659,"num_object_clones":0,"num_object_copies":13977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5519,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":908,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113501+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:42:24.113501+0000","last_peered":"2023-10-01T09:42:24.113501+0000","last_clean":"2023-10-01T09:42:24.113501+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:42:24.113501+0000","last_undegraded":"2023-10-01T09:42:24.113501+0000","last_fullsized":"2023-10-01T09:42:24.113501+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":245,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.830848+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:42:23.830848+0000","last_peered":"2023-10-01T09:42:23.830848+0000","last_clean":"2023-10-01T09:42:23.830848+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:42:23.830848+0000","last_undegraded":"2023-10-01T09:42:23.830848+0000","last_fullsized":"2023-10-01T09:42:23.830848+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":37,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.008510+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:42:24.008510+0000","last_peered":"2023-10-01T09:42:24.008510+0000","last_clean":"2023-10-01T09:42:24.008510+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:42:24.008510+0000","last_undegraded":"2023-10-01T09:42:24.008510+0000","last_fullsized":"2023-10-01T09:42:24.008510+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"515'5582","reported_seq":5632,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:36.295333+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:42:36.295333+0000","last_peered":"2023-10-01T09:42:36.295333+0000","last_clean":"2023-10-01T09:42:36.295333+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:42:36.295333+0000","last_undegraded":"2023-10-01T09:42:36.295333+0000","last_fullsized":"2023-10-01T09:42:36.295333+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5582,"log_dups_size":0,"ondisk_log_size":5582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313786368,"num_objects":4788,"num_object_clones":0,"num_object_copies":14364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5582,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1180,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.830271+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:42:23.830271+0000","last_peered":"2023-10-01T09:42:23.830271+0000","last_clean":"2023-10-01T09:42:23.830271+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:42:23.830271+0000","last_undegraded":"2023-10-01T09:42:23.830271+0000","last_fullsized":"2023-10-01T09:42:23.830271+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":390,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.764773+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:42:23.764773+0000","last_peered":"2023-10-01T09:42:23.764773+0000","last_clean":"2023-10-01T09:42:23.764773+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:42:23.764773+0000","last_undegraded":"2023-10-01T09:42:23.764773+0000","last_fullsized":"2023-10-01T09:42:23.764773+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":218,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.830925+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:42:23.830925+0000","last_peered":"2023-10-01T09:42:23.830925+0000","last_clean":"2023-10-01T09:42:23.830925+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:42:23.830925+0000","last_undegraded":"2023-10-01T09:42:23.830925+0000","last_fullsized":"2023-10-01T09:42:23.830925+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"515'5617","reported_seq":12886,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:35.262537+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:42:35.262537+0000","last_peered":"2023-10-01T09:42:35.262537+0000","last_clean":"2023-10-01T09:42:35.262537+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:42:35.262537+0000","last_undegraded":"2023-10-01T09:42:35.262537+0000","last_fullsized":"2023-10-01T09:42:35.262537+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5617,"log_dups_size":0,"ondisk_log_size":5617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311885824,"num_objects":4759,"num_object_clones":0,"num_object_copies":14277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5617,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":859,"reported_epoch":514,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:42:23.768034+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:42:23.768034+0000","last_peered":"2023-10-01T09:42:23.768034+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:42:23.768034+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:42:23.768034+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":577,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.010310+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:42:24.010310+0000","last_peered":"2023-10-01T09:42:24.010310+0000","last_clean":"2023-10-01T09:42:24.010310+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:42:24.010310+0000","last_undegraded":"2023-10-01T09:42:24.010310+0000","last_fullsized":"2023-10-01T09:42:24.010310+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":37,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.008673+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:42:24.008673+0000","last_peered":"2023-10-01T09:42:24.008673+0000","last_clean":"2023-10-01T09:42:24.008673+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:42:24.008673+0000","last_undegraded":"2023-10-01T09:42:24.008673+0000","last_fullsized":"2023-10-01T09:42:24.008673+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"515'5539","reported_seq":5589,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:34.872683+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:42:34.872683+0000","last_peered":"2023-10-01T09:42:34.872683+0000","last_clean":"2023-10-01T09:42:34.872683+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:42:34.872683+0000","last_undegraded":"2023-10-01T09:42:34.872683+0000","last_fullsized":"2023-10-01T09:42:34.872683+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5539,"log_dups_size":0,"ondisk_log_size":5539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310181888,"num_objects":4733,"num_object_clones":0,"num_object_copies":14199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5539,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1631,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.830671+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:42:23.830671+0000","last_peered":"2023-10-01T09:42:23.830671+0000","last_clean":"2023-10-01T09:42:23.830671+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:42:23.830671+0000","last_undegraded":"2023-10-01T09:42:23.830671+0000","last_fullsized":"2023-10-01T09:42:23.830671+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":545,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113217+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:42:24.113217+0000","last_peered":"2023-10-01T09:42:24.113217+0000","last_clean":"2023-10-01T09:42:24.113217+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:42:24.113217+0000","last_undegraded":"2023-10-01T09:42:24.113217+0000","last_fullsized":"2023-10-01T09:42:24.113217+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":323,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.830575+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:42:23.830575+0000","last_peered":"2023-10-01T09:42:23.830575+0000","last_clean":"2023-10-01T09:42:23.830575+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:42:23.830575+0000","last_undegraded":"2023-10-01T09:42:23.830575+0000","last_fullsized":"2023-10-01T09:42:23.830575+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":313,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.764947+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:42:23.764947+0000","last_peered":"2023-10-01T09:42:23.764947+0000","last_clean":"2023-10-01T09:42:23.764947+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:42:23.764947+0000","last_undegraded":"2023-10-01T09:42:23.764947+0000","last_fullsized":"2023-10-01T09:42:23.764947+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"515'5672","reported_seq":5720,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:36.252773+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:42:36.252773+0000","last_peered":"2023-10-01T09:42:36.252773+0000","last_clean":"2023-10-01T09:42:36.252773+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:42:36.252773+0000","last_undegraded":"2023-10-01T09:42:36.252773+0000","last_fullsized":"2023-10-01T09:42:36.252773+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5672,"log_dups_size":0,"ondisk_log_size":5672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318504960,"num_objects":4860,"num_object_clones":0,"num_object_copies":14580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5672,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":722,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113328+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:42:24.113328+0000","last_peered":"2023-10-01T09:42:24.113328+0000","last_clean":"2023-10-01T09:42:24.113328+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:42:24.113328+0000","last_undegraded":"2023-10-01T09:42:24.113328+0000","last_fullsized":"2023-10-01T09:42:24.113328+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":554,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.768388+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:42:23.768388+0000","last_peered":"2023-10-01T09:42:23.768388+0000","last_clean":"2023-10-01T09:42:23.768388+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:42:23.768388+0000","last_undegraded":"2023-10-01T09:42:23.768388+0000","last_fullsized":"2023-10-01T09:42:23.768388+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":553,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.768895+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:42:23.768895+0000","last_peered":"2023-10-01T09:42:23.768895+0000","last_clean":"2023-10-01T09:42:23.768895+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:42:23.768895+0000","last_undegraded":"2023-10-01T09:42:23.768895+0000","last_fullsized":"2023-10-01T09:42:23.768895+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"515'5650","reported_seq":5701,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:36.248777+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:42:36.248777+0000","last_peered":"2023-10-01T09:42:36.248777+0000","last_clean":"2023-10-01T09:42:36.248777+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:42:36.248777+0000","last_undegraded":"2023-10-01T09:42:36.248777+0000","last_fullsized":"2023-10-01T09:42:36.248777+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5650,"log_dups_size":0,"ondisk_log_size":5650,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315162648,"num_objects":4810,"num_object_clones":0,"num_object_copies":14430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4810,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5650,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":953,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113676+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:42:24.113676+0000","last_peered":"2023-10-01T09:42:24.113676+0000","last_clean":"2023-10-01T09:42:24.113676+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:42:24.113676+0000","last_undegraded":"2023-10-01T09:42:24.113676+0000","last_fullsized":"2023-10-01T09:42:24.113676+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":572,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.830569+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:42:23.830569+0000","last_peered":"2023-10-01T09:42:23.830569+0000","last_clean":"2023-10-01T09:42:23.830569+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:42:23.830569+0000","last_undegraded":"2023-10-01T09:42:23.830569+0000","last_fullsized":"2023-10-01T09:42:23.830569+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":425,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.830756+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:42:23.830756+0000","last_peered":"2023-10-01T09:42:23.830756+0000","last_clean":"2023-10-01T09:42:23.830756+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:42:23.830756+0000","last_undegraded":"2023-10-01T09:42:23.830756+0000","last_fullsized":"2023-10-01T09:42:23.830756+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"515'5638","reported_seq":5694,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:36.089156+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:42:36.089156+0000","last_peered":"2023-10-01T09:42:36.089156+0000","last_clean":"2023-10-01T09:42:36.089156+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:42:36.089156+0000","last_undegraded":"2023-10-01T09:42:36.089156+0000","last_fullsized":"2023-10-01T09:42:36.089156+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5638,"log_dups_size":0,"ondisk_log_size":5638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309198848,"num_objects":4718,"num_object_clones":0,"num_object_copies":14154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5638,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":454,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113871+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:42:24.113871+0000","last_peered":"2023-10-01T09:42:24.113871+0000","last_clean":"2023-10-01T09:42:24.113871+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:42:24.113871+0000","last_undegraded":"2023-10-01T09:42:24.113871+0000","last_fullsized":"2023-10-01T09:42:24.113871+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1147,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.244355+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:42:30.244355+0000","last_peered":"2023-10-01T09:42:30.244355+0000","last_clean":"2023-10-01T09:42:30.244355+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:42:30.244355+0000","last_undegraded":"2023-10-01T09:42:30.244355+0000","last_fullsized":"2023-10-01T09:42:30.244355+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":43,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113150+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:42:24.113150+0000","last_peered":"2023-10-01T09:42:24.113150+0000","last_clean":"2023-10-01T09:42:24.113150+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:42:24.113150+0000","last_undegraded":"2023-10-01T09:42:24.113150+0000","last_fullsized":"2023-10-01T09:42:24.113150+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"515'5673","reported_seq":5729,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:36.177037+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:42:36.177037+0000","last_peered":"2023-10-01T09:42:36.177037+0000","last_clean":"2023-10-01T09:42:36.177037+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:42:36.177037+0000","last_undegraded":"2023-10-01T09:42:36.177037+0000","last_fullsized":"2023-10-01T09:42:36.177037+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5673,"log_dups_size":0,"ondisk_log_size":5673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317128704,"num_objects":4839,"num_object_clones":0,"num_object_copies":14517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5673,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":47,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.113732+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:42:24.113732+0000","last_peered":"2023-10-01T09:42:24.113732+0000","last_clean":"2023-10-01T09:42:24.113732+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:42:24.113732+0000","last_undegraded":"2023-10-01T09:42:24.113732+0000","last_fullsized":"2023-10-01T09:42:24.113732+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":531,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.769017+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:42:23.769017+0000","last_peered":"2023-10-01T09:42:23.769017+0000","last_clean":"2023-10-01T09:42:23.769017+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:42:23.769017+0000","last_undegraded":"2023-10-01T09:42:23.769017+0000","last_fullsized":"2023-10-01T09:42:23.769017+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":238,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.830271+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:42:23.830271+0000","last_peered":"2023-10-01T09:42:23.830271+0000","last_clean":"2023-10-01T09:42:23.830271+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:42:23.830271+0000","last_undegraded":"2023-10-01T09:42:23.830271+0000","last_fullsized":"2023-10-01T09:42:23.830271+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":480,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.768734+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:42:23.768734+0000","last_peered":"2023-10-01T09:42:23.768734+0000","last_clean":"2023-10-01T09:42:23.768734+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:42:23.768734+0000","last_undegraded":"2023-10-01T09:42:23.768734+0000","last_fullsized":"2023-10-01T09:42:23.768734+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":594,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.008716+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:42:24.008716+0000","last_peered":"2023-10-01T09:42:24.008716+0000","last_clean":"2023-10-01T09:42:24.008716+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:42:24.008716+0000","last_undegraded":"2023-10-01T09:42:24.008716+0000","last_fullsized":"2023-10-01T09:42:24.008716+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":250,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.830305+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:42:23.830305+0000","last_peered":"2023-10-01T09:42:23.830305+0000","last_clean":"2023-10-01T09:42:23.830305+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:42:23.830305+0000","last_undegraded":"2023-10-01T09:42:23.830305+0000","last_fullsized":"2023-10-01T09:42:23.830305+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":37,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.008602+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:42:24.008602+0000","last_peered":"2023-10-01T09:42:24.008602+0000","last_clean":"2023-10-01T09:42:24.008602+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:42:24.008602+0000","last_undegraded":"2023-10-01T09:42:24.008602+0000","last_fullsized":"2023-10-01T09:42:24.008602+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":250,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.830610+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:42:23.830610+0000","last_peered":"2023-10-01T09:42:23.830610+0000","last_clean":"2023-10-01T09:42:23.830610+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:42:23.830610+0000","last_undegraded":"2023-10-01T09:42:23.830610+0000","last_fullsized":"2023-10-01T09:42:23.830610+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":426,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.768540+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:42:23.768540+0000","last_peered":"2023-10-01T09:42:23.768540+0000","last_clean":"2023-10-01T09:42:23.768540+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:42:23.768540+0000","last_undegraded":"2023-10-01T09:42:23.768540+0000","last_fullsized":"2023-10-01T09:42:23.768540+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":55,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:24.008830+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:42:24.008830+0000","last_peered":"2023-10-01T09:42:24.008830+0000","last_clean":"2023-10-01T09:42:24.008830+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:42:24.008830+0000","last_undegraded":"2023-10-01T09:42:24.008830+0000","last_fullsized":"2023-10-01T09:42:24.008830+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":471,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.768383+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:42:23.768383+0000","last_peered":"2023-10-01T09:42:23.768383+0000","last_clean":"2023-10-01T09:42:23.768383+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:42:23.768383+0000","last_undegraded":"2023-10-01T09:42:23.768383+0000","last_fullsized":"2023-10-01T09:42:23.768383+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":671,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.769233+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:42:23.769233+0000","last_peered":"2023-10-01T09:42:23.769233+0000","last_clean":"2023-10-01T09:42:23.769233+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:42:23.769233+0000","last_undegraded":"2023-10-01T09:42:23.769233+0000","last_fullsized":"2023-10-01T09:42:23.769233+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":366,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-10-01T09:42:23.764351+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:42:23.764351+0000","last_peered":"2023-10-01T09:42:23.764351+0000","last_clean":"2023-10-01T09:42:23.764351+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:42:23.764351+0000","last_undegraded":"2023-10-01T09:42:23.764351+0000","last_fullsized":"2023-10-01T09:42:23.764351+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5019009048,"num_objects":76585,"num_object_clones":0,"num_object_copies":229755,"num_objects_missing_on_primary":3304,"num_objects_missing":3304,"num_objects_degraded":8478,"num_objects_misplaced":4701,"num_objects_unfound":0,"num_objects_dirty":76585,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":89855,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7274,"num_bytes_recovered":476708864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15561207808,"data_stored":15561195592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89855,"ondisk_log_size":89855,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430704,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2446312,"kb_used_data":1551392,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":91298840,"statfs":{"total":95995035648,"available":93490012160,"internally_reserved":0,"allocated":1588625408,"data_stored":1587691268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.766,"15min":3.0510000000000002},"min":{"1min":0.30099999999999999,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":6.8550000000000004,"5min":11.525,"15min":108.90000000000001},"last":0.379},{"interface":"front","average":{"1min":0.82099999999999995,"5min":2.2130000000000001,"15min":3.3359999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.6639999999999997,"5min":29.501000000000001,"15min":109.187},"last":0.44900000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:42:06 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":5.0410000000000004,"15min":5.492},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":6.8289999999999997,"5min":229.161,"15min":229.161},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.92100000000000004,"5min":4.9939999999999998,"15min":5.5490000000000004},"min":{"1min":0.28899999999999998,"5min":0.27500000000000002,"15min":0.24399999999999999},"max":{"1min":5.7240000000000002,"5min":230.126,"15min":230.126},"last":0.41899999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:41:47 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":2.1680000000000001,"15min":2.0169999999999999},"min":{"1min":0.33200000000000002,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":5.3090000000000002,"5min":14.866,"15min":14.901999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.2829999999999999,"5min":2.1749999999999998,"15min":1.9079999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.6929999999999996,"5min":14.682,"15min":14.682},"last":0.34000000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:42:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.9410000000000001,"15min":2.8610000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":5.7750000000000004,"5min":35.104999999999997,"15min":80.352999999999994},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.004,"5min":2,"15min":2.8359999999999999},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.548,"5min":35.118000000000002,"15min":80.364000000000004},"last":0.35399999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.202,"15min":1.0529999999999999},"min":{"1min":0.30499999999999999,"5min":0.27600000000000002,"15min":0.254},"max":{"1min":6.8769999999999998,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.4710000000000001,"5min":1.1739999999999999,"15min":1.0369999999999999},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":5.7439999999999998,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.36799999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":2.8879999999999999,"15min":1.6419999999999999},"min":{"1min":0.318,"5min":0.30299999999999999,"15min":0.29599999999999999},"max":{"1min":0.64600000000000002,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.41999999999999998,"5min":3.0779999999999998,"15min":1.6850000000000001},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54500000000000004,"5min":135.339,"15min":135.339},"last":0.40600000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.4379999999999999,"15min":1.173},"min":{"1min":0.313,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":2.448,"5min":11.58,"15min":11.853999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":1.2789999999999999,"15min":1.1359999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":0.56699999999999995,"5min":14.584,"15min":14.584},"last":0.437}]}]},{"osd":6,"up_from":294,"seq":1262720385215,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2480172,"kb_used_data":2146660,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":91264980,"statfs":{"total":95995035648,"available":93455339520,"internally_reserved":0,"allocated":2198179840,"data_stored":2197249327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":3.786,"15min":4.1879999999999997},"min":{"1min":0.249,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":1.8400000000000001,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.76300000000000001,"5min":4.3559999999999999,"15min":4.2889999999999997},"min":{"1min":0.249,"5min":0.249,"15min":0.221},"max":{"1min":2.9969999999999999,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.39400000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":5.3860000000000001,"15min":5.0590000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":3.0289999999999999,"5min":168.947,"15min":168.947},"last":0.496},{"interface":"front","average":{"1min":0.79100000000000004,"5min":5.375,"15min":4.9690000000000003},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":2.6030000000000002,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.40500000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.8109999999999999,"15min":1.8420000000000001},"min":{"1min":0.34499999999999997,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":3.0489999999999999,"5min":9.657,"15min":26.289999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.885,"15min":1.859},"min":{"1min":0.32600000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.6739999999999999,"5min":21.704999999999998,"15min":26.388999999999999},"last":0.435}]},{"osd":3,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":1.597,"15min":2.4020000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.9450000000000001,"5min":9.0180000000000007,"15min":89.753},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.073,"5min":1.9330000000000001,"15min":2.2189999999999999},"min":{"1min":0.318,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":3.7160000000000002,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.51000000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.665,"15min":1.0800000000000001},"min":{"1min":0.28599999999999998,"5min":0.253,"15min":0.246},"max":{"1min":4.0250000000000004,"5min":63.134,"15min":63.134},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.518,"15min":1.0960000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.28299999999999997},"max":{"1min":7.4039999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.505}]},{"osd":5,"last update":"Sun Oct 1 09:42:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.93200000000000005,"15min":1.02},"min":{"1min":0.26700000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.8620000000000001,"5min":9.9939999999999998,"15min":12.023999999999999},"last":34.094999999999999},{"interface":"front","average":{"1min":0.66500000000000004,"5min":1.175,"15min":1.143},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.726,"5min":15.457000000000001,"15min":15.457000000000001},"last":34.072000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.82999999999999996,"15min":1.802},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":2.0419999999999998,"5min":6.4119999999999999,"15min":115.11},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.282,"15min":1.8959999999999999},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.28799999999999998},"max":{"1min":3.2679999999999998,"5min":8.6899999999999995,"15min":114.666},"last":0.44500000000000001}]}]},{"osd":1,"up_from":94,"seq":403726926191,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1746112,"kb_used_data":932024,"kb_used_omap":7,"kb_used_meta":814072,"kb_avail":91999040,"statfs":{"total":95995035648,"available":94207016960,"internally_reserved":0,"allocated":954392576,"data_stored":952383507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":833610151},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":3.5089999999999999,"15min":4.5229999999999997},"min":{"1min":0.19400000000000001,"5min":0.182,"15min":0.182},"max":{"1min":4.423,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.318},{"interface":"front","average":{"1min":1.528,"5min":4.0339999999999998,"15min":3.859},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":13.872,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.26700000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:41:37 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":2.6459999999999999,"15min":3.4620000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":2.8740000000000001,"5min":22.891999999999999,"15min":56.573},"last":0.24099999999999999},{"interface":"front","average":{"1min":1.2769999999999999,"5min":3.0059999999999998,"15min":4.2670000000000003},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.20499999999999999},"max":{"1min":6.9710000000000001,"5min":15.845000000000001,"15min":154.197},"last":0.30599999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":3.9089999999999998,"15min":4.4429999999999996},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":3.1949999999999998,"5min":95.305000000000007,"15min":200.56800000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.68799999999999994,"5min":4.3550000000000004,"15min":5.2569999999999997},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":4.7839999999999998,"5min":98.033000000000001,"15min":200.63800000000001},"last":0.29799999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":2.4870000000000001,"15min":2.3940000000000001},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.5510000000000002,"5min":21.015000000000001,"15min":154.101},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.60799999999999998,"5min":2.306,"15min":2.7410000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.4209999999999998,"5min":13.773999999999999,"15min":154.56999999999999},"last":0.438}]},{"osd":5,"last update":"Sun Oct 1 09:41:33 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":2.4870000000000001,"15min":2.1509999999999998},"min":{"1min":0.32900000000000001,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":5.0410000000000004,"5min":21.219999999999999,"15min":154.22999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.83399999999999996,"5min":2.3239999999999998,"15min":2.661},"min":{"1min":0.311,"5min":0.307,"15min":0.307},"max":{"1min":5.1050000000000004,"5min":13.848000000000001,"15min":154.54400000000001},"last":0.40600000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.1669999999999998,"15min":2.8370000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.3660000000000001,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.626,"5min":2.4769999999999999,"15min":2.5899999999999999},"min":{"1min":0.36699999999999999,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":1.575,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.42699999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.617,"15min":2.7250000000000001},"min":{"1min":0.40500000000000003,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":5.2169999999999996,"5min":21.062999999999999,"15min":154.20699999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.88300000000000001,"5min":2.5390000000000001,"15min":2.9399999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":3.8260000000000001,"5min":13.819000000000001,"15min":154.51599999999999},"last":0.46100000000000002}]}]},{"osd":0,"up_from":89,"seq":382252089726,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4620288,"kb_used_data":3729144,"kb_used_omap":8,"kb_used_meta":891127,"kb_avail":89124864,"statfs":{"total":95995035648,"available":91263860736,"internally_reserved":0,"allocated":3818643456,"data_stored":3816640003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":912515010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:42:08 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":3.9860000000000002,"15min":4.4039999999999999},"min":{"1min":0.20399999999999999,"5min":0.187,"15min":0.17399999999999999},"max":{"1min":1.661,"5min":185.78399999999999,"15min":216.821},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.437,"5min":5.9020000000000001,"15min":5.4400000000000004},"min":{"1min":0.249,"5min":0.215,"15min":0.215},"max":{"1min":1.5589999999999999,"5min":206.60599999999999,"15min":217.02699999999999},"last":1.3380000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:42:36 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":1.8779999999999999,"15min":2.1579999999999999},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.20100000000000001},"max":{"1min":0.51200000000000001,"5min":19.788,"15min":19.788},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.36899999999999999,"5min":2.0960000000000001,"15min":2.5230000000000001},"min":{"1min":0.315,"5min":0.215,"15min":0.189},"max":{"1min":0.439,"5min":25.844999999999999,"15min":25.844999999999999},"last":0.315}]},{"osd":3,"last update":"Sun Oct 1 09:42:08 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":2.1749999999999998,"15min":4.524},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.20899999999999999},"max":{"1min":1.706,"5min":15.35,"15min":222.16499999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.873,"5min":2.5219999999999998,"15min":4.7089999999999996},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":2.859,"5min":14.545999999999999,"15min":222.227},"last":0.63200000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":2.3559999999999999,"15min":2.4260000000000002},"min":{"1min":0.29899999999999999,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":2.7480000000000002,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.37},{"interface":"front","average":{"1min":1.034,"5min":2.698,"15min":2.536},"min":{"1min":0.376,"5min":0.33700000000000002,"15min":0.28299999999999997},"max":{"1min":3.1619999999999999,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.59799999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:42:30 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.837,"15min":1.9730000000000001},"min":{"1min":0.33700000000000002,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":1.9730000000000001,"5min":14.515000000000001,"15min":16.808},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.47199999999999998,"5min":1.6000000000000001,"15min":1.8480000000000001},"min":{"1min":0.34799999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64000000000000001,"5min":8.9740000000000002,"15min":17.507000000000001},"last":0.73499999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.617,"15min":1.8360000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.35499999999999998},"max":{"1min":1.5029999999999999,"5min":8.9920000000000009,"15min":17.535},"last":1.5900000000000001},{"interface":"front","average":{"1min":0.54900000000000004,"5min":1.879,"15min":2.137},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":1.5760000000000001,"5min":10.446,"15min":18.521000000000001},"last":1.6080000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.198,"15min":2.1909999999999998},"min":{"1min":0.41799999999999998,"5min":0.377,"15min":0.34499999999999997},"max":{"1min":4.976,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.069,"5min":2.4740000000000002,"15min":2.5459999999999998},"min":{"1min":0.41199999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":4.226,"5min":48.274000000000001,"15min":48.274000000000001},"last":1.5620000000000001}]}]},{"osd":2,"up_from":0,"seq":425201762665,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599139,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2029364,"kb_used_data":1193516,"kb_used_omap":3,"kb_used_meta":835836,"kb_avail":91715788,"statfs":{"total":95995035648,"available":93916966912,"internally_reserved":0,"allocated":1222160384,"data_stored":1220104659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":855896807},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":5.9850000000000003,"5min":5.5869999999999997,"15min":6.0170000000000003},"min":{"1min":0.216,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":64.504000000000005,"5min":118.748,"15min":137.34399999999999},"last":0.28299999999999997},{"interface":"front","average":{"1min":6.024,"5min":5.7830000000000004,"15min":6.0090000000000003},"min":{"1min":0.24299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":64.448999999999998,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.35799999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":5.149,"15min":5.4130000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":6.0250000000000004,"5min":108.84999999999999,"15min":162.22999999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":5.3860000000000001,"15min":5.718},"min":{"1min":0.23100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.931,"5min":109.13200000000001,"15min":162.70699999999999},"last":0.32100000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":2.5070000000000001,"15min":2.5},"min":{"1min":0.27000000000000002,"5min":0.23000000000000001,"15min":0.20799999999999999},"max":{"1min":7.141,"5min":14.254,"15min":43.222999999999999},"last":0.377},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.4359999999999999,"15min":2.823},"min":{"1min":0.28299999999999997,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":3.4550000000000001,"5min":11.202,"15min":78.594999999999999},"last":0.39000000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":2.423,"15min":1.972},"min":{"1min":0.312,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":6.8890000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.434},{"interface":"front","average":{"1min":0.96399999999999997,"5min":2.5800000000000001,"15min":2.202},"min":{"1min":0.33900000000000002,"5min":0.28699999999999998,"15min":0.27900000000000003},"max":{"1min":6.0940000000000003,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.44500000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.766,"15min":2.7850000000000001},"min":{"1min":0.32500000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.70499999999999996,"5min":10.919,"15min":28.117999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":2.0139999999999998,"15min":2.8879999999999999},"min":{"1min":0.35399999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.5580000000000001,"5min":12.989000000000001,"15min":28.34},"last":0.51800000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":2.7970000000000002,"15min":2.3159999999999998},"min":{"1min":0.34399999999999997,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":1.621,"5min":73.887,"15min":73.887},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.66200000000000003,"5min":2.9670000000000001,"15min":2.3090000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.32800000000000001},"max":{"1min":1.8440000000000001,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.59699999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":2.3839999999999999,"15min":2.4249999999999998},"min":{"1min":0.41499999999999998,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":7.0620000000000003,"5min":13.085000000000001,"15min":52.006},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.774,"15min":2.1360000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.2690000000000001,"5min":11.109999999999999,"15min":51.890999999999998},"last":0.46899999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019389,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4675684,"kb_used_data":3785500,"kb_used_omap":0,"kb_used_meta":890176,"kb_avail":89069468,"statfs":{"total":95995035648,"available":91207135232,"internally_reserved":0,"allocated":3876352000,"data_stored":3875421366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911540224},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:53 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":3.5209999999999999,"15min":4.9050000000000002},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":9.6690000000000005,"5min":79.837000000000003,"15min":155.315},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.875,"5min":3.9239999999999999,"15min":5.1550000000000002},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.23899999999999999},"max":{"1min":6.1459999999999999,"5min":78.983000000000004,"15min":155.334},"last":0.499}]},{"osd":1,"last update":"Sun Oct 1 09:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":8.1189999999999998,"15min":7.0599999999999996},"min":{"1min":0.35399999999999998,"5min":0.224,"15min":0.224},"max":{"1min":5.944,"5min":146.929,"15min":190.61199999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.0660000000000001,"5min":8.0500000000000007,"15min":7.8029999999999999},"min":{"1min":0.38500000000000001,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":3.7469999999999999,"5min":146.91,"15min":253.273},"last":0.45000000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:41:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.6279999999999999,"15min":2.6560000000000001},"min":{"1min":0.38300000000000001,"5min":0.29099999999999998,"15min":0.28299999999999997},"max":{"1min":5.702,"5min":10.161,"15min":170.45400000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.5089999999999999,"5min":1.988,"15min":2.371},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":5.6150000000000002,"5min":10.625,"15min":58.771000000000001},"last":0.41799999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.5600000000000001,"15min":2.931},"min":{"1min":0.33600000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.5279999999999996,"5min":15.989000000000001,"15min":140.881},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.825,"15min":3.145},"min":{"1min":0.313,"5min":0.311,"15min":0.311},"max":{"1min":3.786,"5min":15.978999999999999,"15min":140.911},"last":0.40200000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:42:31 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":2.1579999999999999,"15min":1.5900000000000001},"min":{"1min":0.26800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.1190000000000002,"5min":110.003,"15min":110.003},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.46000000000000002,"5min":2.069,"15min":1.405},"min":{"1min":0.317,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.0780000000000001,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.48699999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":0.90000000000000002,"15min":0.996},"min":{"1min":0.34599999999999997,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":9.9849999999999994,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.087,"15min":1.083},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":4.0220000000000002,"5min":6.9139999999999997,"15min":10.076000000000001},"last":0.435}]},{"osd":7,"last update":"Sun Oct 1 09:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.2250000000000001,"15min":1.171},"min":{"1min":0.34599999999999997,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":3.8580000000000001,"5min":38.034999999999997,"15min":38.034999999999997},"last":0.50600000000000001},{"interface":"front","average":{"1min":2.0299999999999998,"5min":1.6899999999999999,"15min":1.4530000000000001},"min":{"1min":0.377,"5min":0.35899999999999999,"15min":0.28699999999999998},"max":{"1min":12.092000000000001,"5min":37.723999999999997,"15min":37.723999999999997},"last":0.53300000000000003}]}]},{"osd":5,"up_from":379,"seq":1627792605296,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2310032,"kb_used_data":1949576,"kb_used_omap":13,"kb_used_meta":360434,"kb_avail":91435120,"statfs":{"total":95995035648,"available":93629562880,"internally_reserved":0,"allocated":1996365824,"data_stored":1995451094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":369084914},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,2,0,0,1,1,4,7,1],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":1.897,"15min":2.258},"min":{"1min":0.23200000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.50700000000000001,"5min":12.571999999999999,"15min":84.340999999999994},"last":0.371},{"interface":"front","average":{"1min":0.39500000000000002,"5min":1.5289999999999999,"15min":2.1349999999999998},"min":{"1min":0.27300000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60299999999999998,"5min":8.6460000000000008,"15min":84.356999999999999},"last":0.27500000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":6.7510000000000003,"15min":5.8460000000000001},"min":{"1min":0.26800000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.55400000000000005,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.41199999999999998,"5min":7.3490000000000002,"15min":5.7889999999999997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":0.63200000000000001,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.32200000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.6020000000000001,"15min":1.6519999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.56599999999999995,"5min":8.7040000000000006,"15min":14.199999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.47799999999999998,"5min":1.6220000000000001,"15min":1.661},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":1.931,"5min":8.2370000000000001,"15min":14.430999999999999},"last":0.38400000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":1.796,"15min":1.5269999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.8720000000000001,"5min":12.734,"15min":14.004},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":1.8580000000000001,"15min":1.673},"min":{"1min":0.39800000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.8819999999999999,"5min":11.766,"15min":13.991},"last":0.34000000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":2.2200000000000002,"15min":1.4670000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.873,"5min":122.971,"15min":122.971},"last":0.315},{"interface":"front","average":{"1min":0.46000000000000002,"5min":2.3140000000000001,"15min":1.806},"min":{"1min":0.307,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.78400000000000003,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.41299999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":1.2849999999999999,"15min":1.0780000000000001},"min":{"1min":0.34499999999999997,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":0.68200000000000005,"5min":12.545,"15min":12.545},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.998,"15min":1.3640000000000001},"min":{"1min":0.34100000000000003,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":0.69999999999999996,"5min":6.4729999999999999,"15min":14.193},"last":2.629}]},{"osd":7,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":1.141,"15min":1.3200000000000001},"min":{"1min":0.32300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.63,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.441},{"interface":"front","average":{"1min":0.53200000000000003,"5min":1.2649999999999999,"15min":1.097},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.318},"max":{"1min":0.70399999999999996,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.59799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3807842304,"data_stored":3807838232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":943783936,"data_stored":943783936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1206648832,"data_stored":1206648832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3865776128,"data_stored":3865772056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1980170240,"data_stored":1980170240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2192248832,"data_stored":2192244760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1564737536,"data_stored":1564737536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:42:39.463 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:42:39.468 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:39.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:39.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:39.494 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:42:39.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:39 smithi190 ceph-mon[160435]: pgmap v1475: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 8488/229839 objects degraded (3.693%); 4701/229839 objects misplaced (2.045%); 288 KiB/s, 4 objects/s recovering 2023-10-01T09:42:39.575 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:39.602 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:39.647 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:39.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:39.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:39.755 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:39.785 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:39 smithi138 ceph-mon[185886]: pgmap v1475: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 8488/229839 objects degraded (3.693%); 4701/229839 objects misplaced (2.045%); 288 KiB/s, 4 objects/s recovering 2023-10-01T09:42:39.786 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:39 smithi138 ceph-mon[192513]: pgmap v1475: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 8488/229839 objects degraded (3.693%); 4701/229839 objects misplaced (2.045%); 288 KiB/s, 4 objects/s recovering 2023-10-01T09:42:39.787 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:39.851 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:39.892 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:39.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:39.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:39.998 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:40.025 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:40.121 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:40.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:40.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:40.226 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:40.253 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:40.326 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:42:40.351 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:40.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:40.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:40.455 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:40.483 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:40.540 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:40.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:40.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:40.643 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:40.673 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:40.720 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:40.721 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:40 smithi190 ceph-mon[160435]: from='client.36726 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:40.721 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:42:40.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:40.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:40 smithi138 ceph-mon[192513]: from='client.36726 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:40.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:42:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:40 smithi138 ceph-mon[185886]: from='client.36726 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:40.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:42:40.823 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:40.854 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:40.904 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:40.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:40.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:41.011 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:41.044 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:41.055 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:42:41.056 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:42:41.085 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:41.110 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:41.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:41.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:41.214 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:41.248 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:41.344 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:41.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:41.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:41.449 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:41.473 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:42:41.477 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:41.497 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:42:41.520 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:41 smithi138 ceph-mon[192513]: pgmap v1476: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8488/229590 objects degraded (3.697%); 4701/229590 objects misplaced (2.048%); 233 KiB/s, 3 objects/s recovering 2023-10-01T09:42:41.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:41 smithi138 ceph-mon[185886]: pgmap v1476: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8488/229590 objects degraded (3.697%); 4701/229590 objects misplaced (2.048%); 233 KiB/s, 3 objects/s recovering 2023-10-01T09:42:41.523 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:42:41.532 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:41.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:41.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:41.546 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:42:41.565 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:42:41.584 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:42:41.603 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:42:41.623 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:42:41.625 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:42:41.625 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:42:41.625 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:42:41.625 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:42:41.625 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:42:41.625 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:42:41.625 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:42:41.626 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:42:41.636 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:41.664 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:41.734 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:41 smithi190 ceph-mon[160435]: pgmap v1476: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8488/229590 objects degraded (3.697%); 4701/229590 objects misplaced (2.048%); 233 KiB/s, 3 objects/s recovering 2023-10-01T09:42:41.758 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:42:41.764 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:41.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:41.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:41.868 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:41.900 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:42.178 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:42:42.446 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:42:42.471 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:42 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2458366263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:42.472 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:42 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2458366263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:42.693 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:42.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:42.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:42.748 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:42 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2458366263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:42.797 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:42.825 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:42.877 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:42.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:42.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:42.983 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:43.025 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:43.080 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:43.177 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:43.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:43.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:43.281 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:43.330 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:43.421 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:43.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:43.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:43.526 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:43.544 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:43 smithi190 ceph-mon[160435]: pgmap v1477: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 8448/227874 objects degraded (3.707%); 4660/227874 objects misplaced (2.045%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:43.545 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:43 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8448/227874 objects degraded (3.707%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:43.555 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:43.658 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:43.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:43.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:43.733 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:42:43.733 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:42:43.733 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.733 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90808 10.7 10:e318de92:::benchmark_data_smithi138_573643_object7586:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.734 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:42.492018+0000", 2023-10-01T09:42:43.734 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.238348802, 2023-10-01T09:42:43.734 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2383790399999999, 2023-10-01T09:42:43.734 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.734 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.734 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.734 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.734 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.734 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90808 2023-10-01T09:42:43.734 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.734 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.735 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.735 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.735 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.492018+0000", 2023-10-01T09:42:43.735 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.735 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.735 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.735 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.735 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.492018+0000", 2023-10-01T09:42:43.735 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:43.735 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.735 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.736 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.736 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.492016+0000", 2023-10-01T09:42:43.736 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1758e-05 2023-10-01T09:42:43.736 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.736 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.736 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.736 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.492028+0000", 2023-10-01T09:42:43.736 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.341e-05 2023-10-01T09:42:43.736 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.736 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.736 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.737 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.492041+0000", 2023-10-01T09:42:43.737 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4619999999999999e-06 2023-10-01T09:42:43.737 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.737 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.737 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.737 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.492049+0000", 2023-10-01T09:42:43.737 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16250604099999999 2023-10-01T09:42:43.737 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.737 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.738 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.738 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.654555+0000", 2023-10-01T09:42:43.738 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0068476600000000002 2023-10-01T09:42:43.738 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.738 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.738 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.738 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.661402+0000", 2023-10-01T09:42:43.738 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.041528078000000003 2023-10-01T09:42:43.738 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.739 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.739 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.739 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.702931+0000", 2023-10-01T09:42:43.739 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0072749939999999999 2023-10-01T09:42:43.739 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.739 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.739 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.739 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.710206+0000", 2023-10-01T09:42:43.739 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21818797600000001 2023-10-01T09:42:43.739 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.739 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.740 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.740 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.740 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.740 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90709 10.7 10:e29e3b9a:::benchmark_data_smithi138_573643_object7487:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.740 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:41.872678+0000", 2023-10-01T09:42:43.740 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.8576879260000001, 2023-10-01T09:42:43.740 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8578185700000001, 2023-10-01T09:42:43.741 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.741 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.741 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.741 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.741 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.741 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90709 2023-10-01T09:42:43.741 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.741 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.741 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.741 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.741 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.872678+0000", 2023-10-01T09:42:43.742 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.742 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.742 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.742 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.742 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.872678+0000", 2023-10-01T09:42:43.742 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:42:43.742 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.742 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.743 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.743 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.872678+0000", 2023-10-01T09:42:43.743 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7140000000000002e-06 2023-10-01T09:42:43.743 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.743 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.743 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.743 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.872681+0000", 2023-10-01T09:42:43.743 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0798999999999999e-05 2023-10-01T09:42:43.743 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.743 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.743 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.743 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.872692+0000", 2023-10-01T09:42:43.744 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9200000000000003e-06 2023-10-01T09:42:43.744 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.744 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.744 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.744 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.872696+0000", 2023-10-01T09:42:43.744 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.107060265 2023-10-01T09:42:43.744 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.745 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.745 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.745 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.979757+0000", 2023-10-01T09:42:43.745 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0099213040000000006 2023-10-01T09:42:43.745 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.745 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.745 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.745 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.989678+0000", 2023-10-01T09:42:43.745 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.067723030000000004 2023-10-01T09:42:43.746 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.746 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.746 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.746 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.057401+0000", 2023-10-01T09:42:43.746 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0080726740000000002 2023-10-01T09:42:43.746 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.746 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.747 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.747 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.065474+0000", 2023-10-01T09:42:43.747 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.19279528000000001 2023-10-01T09:42:43.747 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.747 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.747 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.747 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.747 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.747 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90836 10.7 10:e641c44b:::benchmark_data_smithi138_573643_object7614:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.748 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:42.692007+0000", 2023-10-01T09:42:43.748 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.0383592559999999, 2023-10-01T09:42:43.748 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.03858154, 2023-10-01T09:42:43.748 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.748 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.748 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.749 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.749 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.749 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90836 2023-10-01T09:42:43.749 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.749 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.749 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.749 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.749 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.692007+0000", 2023-10-01T09:42:43.750 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.750 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.750 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.750 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.750 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.692007+0000", 2023-10-01T09:42:43.750 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:42:43.751 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.751 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.751 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.751 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.692005+0000", 2023-10-01T09:42:43.751 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2469999999999997e-06 2023-10-01T09:42:43.751 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.751 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.751 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.751 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.692014+0000", 2023-10-01T09:42:43.751 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3654e-05 2023-10-01T09:42:43.751 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.752 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.752 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.752 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.692028+0000", 2023-10-01T09:42:43.752 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7880000000000001e-06 2023-10-01T09:42:43.752 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.752 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.752 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.752 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.692035+0000", 2023-10-01T09:42:43.752 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.085737852000000003 2023-10-01T09:42:43.753 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.753 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.753 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.753 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.777773+0000", 2023-10-01T09:42:43.753 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0063672210000000002 2023-10-01T09:42:43.753 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.753 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.753 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.753 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.784140+0000", 2023-10-01T09:42:43.754 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.043043277999999997 2023-10-01T09:42:43.754 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.754 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.754 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.754 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.827183+0000", 2023-10-01T09:42:43.754 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0080286460000000004 2023-10-01T09:42:43.754 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.754 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.754 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.755 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.835212+0000", 2023-10-01T09:42:43.755 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14320475599999999 2023-10-01T09:42:43.755 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.755 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.755 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.755 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.755 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.755 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90877 10.7 10:ec7284af:::benchmark_data_smithi138_573643_object7655:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.755 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:43.065546+0000", 2023-10-01T09:42:43.756 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.66482000200000002, 2023-10-01T09:42:43.756 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.66514501199999998, 2023-10-01T09:42:43.756 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.756 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.756 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.756 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.756 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.756 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90877 2023-10-01T09:42:43.757 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.757 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.757 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.757 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.757 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.065546+0000", 2023-10-01T09:42:43.757 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.757 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.757 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.757 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.757 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.065546+0000", 2023-10-01T09:42:43.758 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:42:43.758 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.758 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.758 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.758 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.065545+0000", 2023-10-01T09:42:43.758 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4340000000000003e-06 2023-10-01T09:42:43.758 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.758 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.758 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.759 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.065552+0000", 2023-10-01T09:42:43.759 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3747000000000001e-05 2023-10-01T09:42:43.759 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.759 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.759 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.759 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.065566+0000", 2023-10-01T09:42:43.759 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0400000000000004e-06 2023-10-01T09:42:43.759 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.759 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.759 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.759 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.065573+0000", 2023-10-01T09:42:43.759 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21328731100000001 2023-10-01T09:42:43.760 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.760 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.760 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.760 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.278860+0000", 2023-10-01T09:42:43.760 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018570514 2023-10-01T09:42:43.760 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.760 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.760 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.760 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.297431+0000", 2023-10-01T09:42:43.760 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.073548684000000003 2023-10-01T09:42:43.760 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.760 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.760 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.761 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.370979+0000", 2023-10-01T09:42:43.761 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0064009679999999999 2023-10-01T09:42:43.761 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.761 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.761 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.761 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.377380+0000", 2023-10-01T09:42:43.761 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.31183397800000001 2023-10-01T09:42:43.761 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.761 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.761 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.761 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.762 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.762 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90911 10.7 10:e3ab68e7:::benchmark_data_smithi138_573643_object7689:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.762 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:43.201811+0000", 2023-10-01T09:42:43.762 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.52855512900000001, 2023-10-01T09:42:43.762 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.52896469700000004, 2023-10-01T09:42:43.762 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.762 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.762 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.762 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.762 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.762 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90911 2023-10-01T09:42:43.763 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.763 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.763 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.763 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.763 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.201811+0000", 2023-10-01T09:42:43.763 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.763 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.763 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.763 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.763 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.201811+0000", 2023-10-01T09:42:43.763 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:43.763 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.764 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.764 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.764 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.201810+0000", 2023-10-01T09:42:43.764 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9209999999999992e-06 2023-10-01T09:42:43.764 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.764 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.764 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.764 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.201818+0000", 2023-10-01T09:42:43.764 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3196999999999999e-05 2023-10-01T09:42:43.764 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.764 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.765 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.765 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.201831+0000", 2023-10-01T09:42:43.765 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6110000000000001e-06 2023-10-01T09:42:43.765 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.765 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.765 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.765 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.201836+0000", 2023-10-01T09:42:43.765 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.097011848999999997 2023-10-01T09:42:43.765 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.765 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.765 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.765 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.298848+0000", 2023-10-01T09:42:43.766 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011941332000000001 2023-10-01T09:42:43.766 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.766 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.766 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.766 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.310790+0000", 2023-10-01T09:42:43.766 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.091462875999999999 2023-10-01T09:42:43.766 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.766 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.766 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.766 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.402253+0000", 2023-10-01T09:42:43.767 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0064216100000000003 2023-10-01T09:42:43.767 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.767 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.767 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.767 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.408674+0000", 2023-10-01T09:42:43.767 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.20686284699999999 2023-10-01T09:42:43.767 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.767 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.767 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.767 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.768 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.768 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90912 10.7 10:e7aed270:::benchmark_data_smithi138_573643_object7690:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.768 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:43.209378+0000", 2023-10-01T09:42:43.768 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.52098840899999999, 2023-10-01T09:42:43.768 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.52148222399999999, 2023-10-01T09:42:43.768 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.768 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.768 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.768 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.768 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.768 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90912 2023-10-01T09:42:43.769 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.769 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.769 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.769 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.769 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.209378+0000", 2023-10-01T09:42:43.769 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.769 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.769 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.769 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.769 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.209378+0000", 2023-10-01T09:42:43.769 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:42:43.769 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.770 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.770 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.770 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.209377+0000", 2023-10-01T09:42:43.770 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0499999999999997e-06 2023-10-01T09:42:43.770 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.770 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.770 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.770 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.209383+0000", 2023-10-01T09:42:43.770 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2807e-05 2023-10-01T09:42:43.770 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.771 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.771 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.771 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.209396+0000", 2023-10-01T09:42:43.771 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2530000000000004e-06 2023-10-01T09:42:43.771 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.771 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.771 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.771 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.209401+0000", 2023-10-01T09:42:43.771 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.102953348 2023-10-01T09:42:43.772 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.772 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.772 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.772 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.312354+0000", 2023-10-01T09:42:43.772 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017561756000000001 2023-10-01T09:42:43.772 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.772 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.772 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.772 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.329916+0000", 2023-10-01T09:42:43.772 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.11234161199999999 2023-10-01T09:42:43.772 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.773 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.773 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.773 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.442258+0000", 2023-10-01T09:42:43.773 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0076315439999999997 2023-10-01T09:42:43.773 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.773 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.773 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.773 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.449889+0000", 2023-10-01T09:42:43.773 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.24051140800000001 2023-10-01T09:42:43.774 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.774 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.774 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.774 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.774 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.774 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90957 10.7 10:e775ae4c:::benchmark_data_smithi138_573643_object7735:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.774 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:43.634638+0000", 2023-10-01T09:42:43.774 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.095728665000000004, 2023-10-01T09:42:43.774 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.096307053000000004, 2023-10-01T09:42:43.774 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.774 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.775 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.775 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.775 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.775 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90957 2023-10-01T09:42:43.775 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.775 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.775 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.775 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.775 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.634638+0000", 2023-10-01T09:42:43.776 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.776 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.776 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.776 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.776 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.634638+0000", 2023-10-01T09:42:43.776 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:43.776 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.776 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.776 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.776 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.634636+0000", 2023-10-01T09:42:43.776 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.8400000000000001e-06 2023-10-01T09:42:43.776 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.777 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.777 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.777 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.634645+0000", 2023-10-01T09:42:43.777 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5141e-05 2023-10-01T09:42:43.777 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.777 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.777 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.777 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.634660+0000", 2023-10-01T09:42:43.777 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6220000000000003e-06 2023-10-01T09:42:43.778 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.778 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.778 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.778 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.634667+0000", 2023-10-01T09:42:43.778 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.074906397999999999 2023-10-01T09:42:43.778 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.778 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.778 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.779 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.709573+0000", 2023-10-01T09:42:43.779 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0064518930000000002 2023-10-01T09:42:43.779 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.779 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.779 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.779 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.716025+0000", 2023-10-01T09:42:43.780 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.081387283000000005 2023-10-01T09:42:43.780 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.780 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.780 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.780 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.780 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.780 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "MOSDPGPushReply(10.7 516/503 [PushReplyOp(10:e7685cb2:::benchmark_data_smithi138_573643_object7442:head)])", 2023-10-01T09:42:43.780 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:43.653398+0000", 2023-10-01T09:42:43.780 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.076968711999999995, 2023-10-01T09:42:43.781 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.077616286000000007, 2023-10-01T09:42:43.781 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.781 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "reached pg", 2023-10-01T09:42:43.781 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.781 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.781 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.782 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.653398+0000", 2023-10-01T09:42:43.782 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.782 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.782 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.782 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.782 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.653398+0000", 2023-10-01T09:42:43.782 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999938 2023-10-01T09:42:43.783 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.783 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.783 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.783 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.653391+0000", 2023-10-01T09:42:43.783 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8416000000000001e-05 2023-10-01T09:42:43.783 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.783 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.784 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.784 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.653410+0000", 2023-10-01T09:42:43.784 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9292000000000001e-05 2023-10-01T09:42:43.784 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.784 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.784 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.784 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.653429+0000", 2023-10-01T09:42:43.784 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1564e-05 2023-10-01T09:42:43.784 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.784 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.784 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.785 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.653441+0000", 2023-10-01T09:42:43.785 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.063356267999999993 2023-10-01T09:42:43.785 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.785 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.785 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.785 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.716797+0000", 2023-10-01T09:42:43.785 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.063399160999999996 2023-10-01T09:42:43.785 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.785 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.785 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.786 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.786 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.786 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop_reply(client.36456.0:90641 10.7 e516/503)", 2023-10-01T09:42:43.786 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:43.703618+0000", 2023-10-01T09:42:43.786 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.026747956999999999, 2023-10-01T09:42:43.786 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.027450777999999999, 2023-10-01T09:42:43.786 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.786 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:42:43.786 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.786 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.787 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.787 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.703618+0000", 2023-10-01T09:42:43.787 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.787 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.787 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.787 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.787 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.703618+0000", 2023-10-01T09:42:43.787 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:42:43.787 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.788 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.788 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.788 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.703617+0000", 2023-10-01T09:42:43.788 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.8779999999999993e-06 2023-10-01T09:42:43.788 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.788 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.788 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.788 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.703627+0000", 2023-10-01T09:42:43.788 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3815000000000001e-05 2023-10-01T09:42:43.788 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.788 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.789 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.789 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.703640+0000", 2023-10-01T09:42:43.789 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1080000000000004e-06 2023-10-01T09:42:43.789 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.789 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.789 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.789 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.703648+0000", 2023-10-01T09:42:43.789 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9082e-05 2023-10-01T09:42:43.790 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.790 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.790 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.790 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.790 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.790 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop_reply(client.36456.0:90641 10.7 e516/503)", 2023-10-01T09:42:43.790 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:43.706639+0000", 2023-10-01T09:42:43.790 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.02372722, 2023-10-01T09:42:43.790 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.024479516999999999, 2023-10-01T09:42:43.791 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.791 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:42:43.791 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.791 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.791 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.791 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.706639+0000", 2023-10-01T09:42:43.791 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.791 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.791 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.791 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.791 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.706639+0000", 2023-10-01T09:42:43.792 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:43.792 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.792 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.792 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.792 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.706638+0000", 2023-10-01T09:42:43.792 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7329999999999998e-06 2023-10-01T09:42:43.792 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.792 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.792 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.792 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.706648+0000", 2023-10-01T09:42:43.792 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6220999999999999e-05 2023-10-01T09:42:43.792 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.792 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.793 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.793 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.706664+0000", 2023-10-01T09:42:43.793 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7689999999999996e-06 2023-10-01T09:42:43.793 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.793 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.793 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.793 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.706672+0000", 2023-10-01T09:42:43.793 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2360000000000002e-05 2023-10-01T09:42:43.793 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.793 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.793 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.793 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.794 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.794 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "MOSDPGPush(10.7 516/503 [PushOp(10:e775ae4c:::benchmark_data_smithi138_573643_object7735:head, version: 481'454, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:e775ae4c:::benchmark_data_smithi138_573643_object7735:head@481'454, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-01T09:42:43.794 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:43.716916+0000", 2023-10-01T09:42:43.794 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.013450444000000001, 2023-10-01T09:42:43.794 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014295131000000001, 2023-10-01T09:42:43.794 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.794 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:42:43.794 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.794 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.794 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.795 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.716916+0000", 2023-10-01T09:42:43.795 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.795 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.795 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.795 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.795 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.716916+0000", 2023-10-01T09:42:43.795 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999943 2023-10-01T09:42:43.795 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.795 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.795 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.795 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.716910+0000", 2023-10-01T09:42:43.795 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9259999999999999e-05 2023-10-01T09:42:43.795 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.796 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.796 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.796 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.716960+0000", 2023-10-01T09:42:43.796 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0309000000000002e-05 2023-10-01T09:42:43.796 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.796 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.796 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.796 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.716980+0000", 2023-10-01T09:42:43.796 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5990000000000008e-06 2023-10-01T09:42:43.796 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.796 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.796 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.797 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.716990+0000", 2023-10-01T09:42:43.797 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3676000000000004e-05 2023-10-01T09:42:43.797 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.797 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.797 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.797 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.797 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.797 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90818 10.7 10:ea68a37d:::benchmark_data_smithi138_573643_object7596:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.797 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:42.626868+0000", 2023-10-01T09:42:43.797 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.103498471, 2023-10-01T09:42:43.797 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.104399927, 2023-10-01T09:42:43.797 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.798 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.798 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.798 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.798 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.798 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90818 2023-10-01T09:42:43.798 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.798 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.798 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.798 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.798 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.626868+0000", 2023-10-01T09:42:43.798 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.798 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.799 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.799 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.799 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.626868+0000", 2023-10-01T09:42:43.799 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:43.799 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.799 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.799 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.799 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.626866+0000", 2023-10-01T09:42:43.799 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7549999999999994e-06 2023-10-01T09:42:43.799 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.799 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.799 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.799 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.626874+0000", 2023-10-01T09:42:43.800 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2177999999999999e-05 2023-10-01T09:42:43.800 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.800 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.800 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.800 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.626886+0000", 2023-10-01T09:42:43.800 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1309999999999997e-06 2023-10-01T09:42:43.800 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.800 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.800 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.800 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.626891+0000", 2023-10-01T09:42:43.800 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.035585897999999998 2023-10-01T09:42:43.800 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.801 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.801 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.801 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.662477+0000", 2023-10-01T09:42:43.801 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0071721190000000002 2023-10-01T09:42:43.801 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.801 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.801 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.801 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.669649+0000", 2023-10-01T09:42:43.801 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.066647715999999996 2023-10-01T09:42:43.801 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.801 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.801 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.801 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.736297+0000", 2023-10-01T09:42:43.802 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0064270170000000001 2023-10-01T09:42:43.802 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.802 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.802 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.802 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.742724+0000", 2023-10-01T09:42:43.802 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.11585617500000001 2023-10-01T09:42:43.802 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.802 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.802 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.802 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.802 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.802 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90745 10.7 10:eaf18a26:::benchmark_data_smithi138_573643_object7523:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.803 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:42.012307+0000", 2023-10-01T09:42:43.803 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.7180594230000001, 2023-10-01T09:42:43.803 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7190561259999999, 2023-10-01T09:42:43.803 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.803 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.803 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.803 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.803 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.803 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90745 2023-10-01T09:42:43.803 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.803 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.803 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.804 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.804 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.012307+0000", 2023-10-01T09:42:43.804 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.804 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.804 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.804 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.804 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.012307+0000", 2023-10-01T09:42:43.804 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:43.804 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.804 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.804 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.804 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.012306+0000", 2023-10-01T09:42:43.805 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7979999999999996e-06 2023-10-01T09:42:43.805 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.805 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.805 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.805 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.012311+0000", 2023-10-01T09:42:43.805 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6570000000000003e-06 2023-10-01T09:42:43.805 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.805 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.805 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.805 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.012319+0000", 2023-10-01T09:42:43.805 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2300000000000002e-06 2023-10-01T09:42:43.806 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.806 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.806 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.806 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.012323+0000", 2023-10-01T09:42:43.806 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.054344014000000003 2023-10-01T09:42:43.806 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.806 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.806 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.806 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.066667+0000", 2023-10-01T09:42:43.806 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0081895059999999992 2023-10-01T09:42:43.806 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.806 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.807 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.807 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.074857+0000", 2023-10-01T09:42:43.807 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.044817418999999997 2023-10-01T09:42:43.807 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.807 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.807 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.807 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.119674+0000", 2023-10-01T09:42:43.807 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0088180020000000001 2023-10-01T09:42:43.807 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.807 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.807 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.807 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.128492+0000", 2023-10-01T09:42:43.808 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.116185396 2023-10-01T09:42:43.808 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.808 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.808 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.808 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.808 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.808 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90822 10.7 10:e2448415:::benchmark_data_smithi138_573643_object7600:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.808 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:42.640152+0000", 2023-10-01T09:42:43.808 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.090214322, 2023-10-01T09:42:43.809 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.091303924, 2023-10-01T09:42:43.809 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.809 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.809 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.809 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.809 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.809 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90822 2023-10-01T09:42:43.809 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.809 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.809 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.809 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.810 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.640152+0000", 2023-10-01T09:42:43.810 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.810 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.810 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.810 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.810 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.640152+0000", 2023-10-01T09:42:43.810 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:43.810 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.810 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.810 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.811 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.640150+0000", 2023-10-01T09:42:43.811 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9270000000000005e-06 2023-10-01T09:42:43.811 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.811 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.811 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.811 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.640158+0000", 2023-10-01T09:42:43.811 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1819e-05 2023-10-01T09:42:43.812 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.812 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.812 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.812 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.640170+0000", 2023-10-01T09:42:43.812 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9859999999999997e-06 2023-10-01T09:42:43.812 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.812 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.812 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.812 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.640177+0000", 2023-10-01T09:42:43.813 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.030313257 2023-10-01T09:42:43.813 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.813 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.813 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.813 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.670490+0000", 2023-10-01T09:42:43.813 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0069108640000000001 2023-10-01T09:42:43.813 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.813 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.813 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.813 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.677401+0000", 2023-10-01T09:42:43.814 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.093434950000000003 2023-10-01T09:42:43.814 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.814 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.814 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.814 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.770836+0000", 2023-10-01T09:42:43.814 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0062808730000000002 2023-10-01T09:42:43.814 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.814 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.814 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.815 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.777117+0000", 2023-10-01T09:42:43.815 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13696503400000001 2023-10-01T09:42:43.815 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.815 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.815 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.815 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.815 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.815 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90945 10.7 10:ec76612a:::benchmark_data_smithi138_573643_object7723:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.815 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:43.311102+0000", 2023-10-01T09:42:43.815 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.41926459599999999, 2023-10-01T09:42:43.816 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.42044247600000001, 2023-10-01T09:42:43.816 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.816 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.816 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.816 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.816 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.817 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90945 2023-10-01T09:42:43.817 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.817 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.817 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.817 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.818 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.311102+0000", 2023-10-01T09:42:43.818 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.818 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.818 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.818 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.819 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.311102+0000", 2023-10-01T09:42:43.819 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:42:43.819 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.819 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.819 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.819 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.311100+0000", 2023-10-01T09:42:43.819 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5979999999999999e-06 2023-10-01T09:42:43.819 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.820 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.820 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.820 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.311107+0000", 2023-10-01T09:42:43.820 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5043e-05 2023-10-01T09:42:43.820 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.820 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.820 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.820 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.311122+0000", 2023-10-01T09:42:43.821 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0990000000000001e-06 2023-10-01T09:42:43.821 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.821 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.821 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.821 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.311129+0000", 2023-10-01T09:42:43.821 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13960212 2023-10-01T09:42:43.821 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.821 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.821 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.821 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.450732+0000", 2023-10-01T09:42:43.822 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00623676 2023-10-01T09:42:43.822 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.822 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.822 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.822 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.456968+0000", 2023-10-01T09:42:43.822 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.041887572999999997 2023-10-01T09:42:43.822 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.823 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.823 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.823 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.498856+0000", 2023-10-01T09:42:43.823 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010720786 2023-10-01T09:42:43.823 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.823 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.823 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.824 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.509577+0000", 2023-10-01T09:42:43.824 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.198474873 2023-10-01T09:42:43.824 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.824 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.824 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.824 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.824 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.824 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90747 10.7 10:e813022c:::benchmark_data_smithi138_573643_object7525:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.825 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:42.017599+0000", 2023-10-01T09:42:43.825 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.7127676040000002, 2023-10-01T09:42:43.825 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.714047358, 2023-10-01T09:42:43.825 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.825 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.825 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.825 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.825 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.826 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90747 2023-10-01T09:42:43.826 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.826 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.826 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.826 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.826 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.017599+0000", 2023-10-01T09:42:43.826 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.826 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.826 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.827 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.827 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.017599+0000", 2023-10-01T09:42:43.827 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:43.827 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.827 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.827 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.827 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.017598+0000", 2023-10-01T09:42:43.827 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9020000000000004e-06 2023-10-01T09:42:43.827 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.827 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.827 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.827 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.017603+0000", 2023-10-01T09:42:43.828 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3377000000000001e-05 2023-10-01T09:42:43.828 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.828 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.828 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.828 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.017617+0000", 2023-10-01T09:42:43.828 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3740000000000003e-06 2023-10-01T09:42:43.828 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.828 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.828 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.828 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.017623+0000", 2023-10-01T09:42:43.828 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.058251325 2023-10-01T09:42:43.829 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.829 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.829 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.829 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.075874+0000", 2023-10-01T09:42:43.829 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.007801166 2023-10-01T09:42:43.829 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.829 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.829 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.829 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.083676+0000", 2023-10-01T09:42:43.829 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.071429468999999995 2023-10-01T09:42:43.829 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.829 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.830 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.830 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.155105+0000", 2023-10-01T09:42:43.830 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0091262259999999994 2023-10-01T09:42:43.830 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.830 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.830 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.830 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.164231+0000", 2023-10-01T09:42:43.830 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14663256499999999 2023-10-01T09:42:43.830 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.830 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.830 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.830 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.831 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.831 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90664 10.7 10:e7685cb2:::benchmark_data_smithi138_573643_object7442:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.831 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:41.756060+0000", 2023-10-01T09:42:43.831 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.97430624, 2023-10-01T09:42:43.831 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.975670255, 2023-10-01T09:42:43.831 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.831 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.831 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.831 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.831 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.831 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90664 2023-10-01T09:42:43.831 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.832 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.832 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.832 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.832 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.756060+0000", 2023-10-01T09:42:43.832 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.832 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.832 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.832 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.832 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.756060+0000", 2023-10-01T09:42:43.832 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:42:43.832 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.832 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.833 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.833 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.756058+0000", 2023-10-01T09:42:43.833 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2258e-05 2023-10-01T09:42:43.833 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.833 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.833 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.833 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.756071+0000", 2023-10-01T09:42:43.833 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5318999999999999e-05 2023-10-01T09:42:43.833 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.833 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.833 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.833 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.756096+0000", 2023-10-01T09:42:43.834 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0380000000000002e-06 2023-10-01T09:42:43.834 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.834 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.834 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.834 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.756103+0000", 2023-10-01T09:42:43.834 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.041765869999999997 2023-10-01T09:42:43.834 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.834 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.834 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.834 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.797869+0000", 2023-10-01T09:42:43.834 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011755167 2023-10-01T09:42:43.834 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.835 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.835 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.835 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.809624+0000", 2023-10-01T09:42:43.835 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.097582284000000005 2023-10-01T09:42:43.835 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.835 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.835 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.835 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.907206+0000", 2023-10-01T09:42:43.835 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078116000000000001 2023-10-01T09:42:43.835 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.835 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.835 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.836 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.915018+0000", 2023-10-01T09:42:43.836 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15895780500000001 2023-10-01T09:42:43.836 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.836 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.836 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.836 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.836 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.836 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90641 10.7 10:e70ed100:::benchmark_data_smithi138_573643_object7419:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.836 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:41.644918+0000", 2023-10-01T09:42:43.837 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 2.0854484599999998, 2023-10-01T09:42:43.837 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.086896512, 2023-10-01T09:42:43.837 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.837 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:42:43.837 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.837 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.837 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.837 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90641 2023-10-01T09:42:43.837 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.837 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.837 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.838 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.838 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.644918+0000", 2023-10-01T09:42:43.838 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.838 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.838 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.838 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.838 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.644918+0000", 2023-10-01T09:42:43.838 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:42:43.838 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.838 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.838 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.839 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.644916+0000", 2023-10-01T09:42:43.839 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5380000000000002e-06 2023-10-01T09:42:43.839 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.839 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.839 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.839 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.644925+0000", 2023-10-01T09:42:43.839 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4134000000000001e-05 2023-10-01T09:42:43.839 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.840 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.840 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.840 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.644939+0000", 2023-10-01T09:42:43.840 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.858e-06 2023-10-01T09:42:43.840 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.840 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.840 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.840 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.644947+0000", 2023-10-01T09:42:43.840 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14100929700000001 2023-10-01T09:42:43.840 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.840 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.841 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.841 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.785956+0000", 2023-10-01T09:42:43.841 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010614157000000001 2023-10-01T09:42:43.841 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.841 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.841 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.841 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.796570+0000", 2023-10-01T09:42:43.841 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.065651086999999997 2023-10-01T09:42:43.841 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.841 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.842 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.842 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.862221+0000", 2023-10-01T09:42:43.842 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011043025999999999 2023-10-01T09:42:43.842 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.842 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.842 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.842 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:41.873264+0000", 2023-10-01T09:42:43.842 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.810745498 2023-10-01T09:42:43.842 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.843 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.843 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.843 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.684010+0000", 2023-10-01T09:42:43.843 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050803209999999996 2023-10-01T09:42:43.843 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.843 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.843 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:42:43.843 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.689090+0000", 2023-10-01T09:42:43.843 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010804351 2023-10-01T09:42:43.843 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.843 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.844 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:42:43.844 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.699894+0000", 2023-10-01T09:42:43.844 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0549765390000001 2023-10-01T09:42:43.844 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.844 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.844 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.844 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.844 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.844 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90955 10.7 10:e7819c14:::benchmark_data_smithi138_573643_object7733:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.845 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:43.352044+0000", 2023-10-01T09:42:43.845 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.37832268499999999, 2023-10-01T09:42:43.845 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.37989106500000003, 2023-10-01T09:42:43.845 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.845 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.845 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.845 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.845 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.845 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90955 2023-10-01T09:42:43.845 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.846 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.846 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.846 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.846 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.352044+0000", 2023-10-01T09:42:43.846 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.846 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.846 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.846 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.846 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.352044+0000", 2023-10-01T09:42:43.847 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:42:43.847 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.847 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.847 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.847 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.352042+0000", 2023-10-01T09:42:43.847 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0652999999999999e-05 2023-10-01T09:42:43.848 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.848 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.848 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.352052+0000", 2023-10-01T09:42:43.848 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8768000000000001e-05 2023-10-01T09:42:43.848 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.848 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.848 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.352071+0000", 2023-10-01T09:42:43.849 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3570000000000004e-06 2023-10-01T09:42:43.849 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.849 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.849 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.849 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.352080+0000", 2023-10-01T09:42:43.849 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.105624262 2023-10-01T09:42:43.850 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.850 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.850 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.850 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.457705+0000", 2023-10-01T09:42:43.850 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0062699369999999997 2023-10-01T09:42:43.850 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.850 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.851 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.851 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.463975+0000", 2023-10-01T09:42:43.851 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.079443898999999998 2023-10-01T09:42:43.851 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.851 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.851 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.852 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.543418+0000", 2023-10-01T09:42:43.852 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0074844400000000002 2023-10-01T09:42:43.852 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.852 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.852 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.852 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:43.550903+0000", 2023-10-01T09:42:43.852 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.19885915300000001 2023-10-01T09:42:43.852 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.853 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.853 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.853 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.853 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.853 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90760 10.7 10:e122826c:::benchmark_data_smithi138_573643_object7538:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:42:43.853 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:42.286483+0000", 2023-10-01T09:42:43.853 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.443883676, 2023-10-01T09:42:43.853 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.445543375, 2023-10-01T09:42:43.853 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:42:43.854 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:42:43.854 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:42:43.854 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:42:43.854 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:42:43.854 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90760 2023-10-01T09:42:43.854 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.854 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:42:43.854 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.854 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:42:43.854 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.286483+0000", 2023-10-01T09:42:43.855 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:42:43.855 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.855 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.855 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:42:43.855 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.286483+0000", 2023-10-01T09:42:43.855 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:42:43.855 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.855 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.855 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:42:43.855 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.286481+0000", 2023-10-01T09:42:43.856 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5120000000000002e-06 2023-10-01T09:42:43.856 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.856 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.856 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:42:43.856 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.286489+0000", 2023-10-01T09:42:43.856 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2741999999999999e-05 2023-10-01T09:42:43.856 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.856 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.856 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:42:43.857 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.286502+0000", 2023-10-01T09:42:43.857 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.4460000000000004e-06 2023-10-01T09:42:43.857 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.857 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.857 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:42:43.857 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.286509+0000", 2023-10-01T09:42:43.857 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.055014236000000001 2023-10-01T09:42:43.857 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.857 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.858 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.858 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.341523+0000", 2023-10-01T09:42:43.858 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0072722669999999998 2023-10-01T09:42:43.858 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.858 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.858 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:42:43.858 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.348795+0000", 2023-10-01T09:42:43.858 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.108009428 2023-10-01T09:42:43.858 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.858 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.859 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:42:43.859 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.456805+0000", 2023-10-01T09:42:43.859 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.006520128 2023-10-01T09:42:43.859 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:42:43.859 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:42:43.859 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:42:43.859 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.463325+0000", 2023-10-01T09:42:43.859 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.176842311 2023-10-01T09:42:43.859 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.860 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:42:43.860 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.860 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:42:43.860 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:42:43.860 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 20 2023-10-01T09:42:43.860 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:42:43.860 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:43.861 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:43 smithi138 ceph-mon[192513]: pgmap v1477: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 8448/227874 objects degraded (3.707%); 4660/227874 objects misplaced (2.045%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:43.861 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:43 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8448/227874 objects degraded (3.707%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:43.862 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:43 smithi138 ceph-mon[185886]: pgmap v1477: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 8448/227874 objects degraded (3.707%); 4660/227874 objects misplaced (2.045%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:43.862 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:43 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8448/227874 objects degraded (3.707%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:43.894 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:43.994 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:43.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:43.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:44.098 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:44.126 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:44.418 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:44.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:44.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:44.458 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:42:44.484 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T09:42:44.505 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:44 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3919730465' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:44.505 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:44 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:44.505 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:44 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3919730465' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:44.505 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:44 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:44.523 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:44.558 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:44.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:44 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3919730465' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:44.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:44 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:44.931 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:42:44.997 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:42:45.143 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:42:45.145 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:42:45.200 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:45.202 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:42:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:42:44] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.33.4" 2023-10-01T09:42:45.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:45.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:45.305 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:45.337 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:45.400 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:45.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:45.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:45.452 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[185886]: pgmap v1478: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 146 op/s; 8448/227706 objects degraded (3.710%); 4660/227706 objects misplaced (2.046%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:45.452 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:42:45.452 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[185886]: failed to encode map e517 with expected crc 2023-10-01T09:42:45.452 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[185886]: failed to encode map e517 with expected crc 2023-10-01T09:42:45.452 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:42:45.452 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[185886]: osdmap e517: 8 total, 8 up, 7 in 2023-10-01T09:42:45.452 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[185886]: failed to encode map e517 with expected crc 2023-10-01T09:42:45.452 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[185886]: from='client.46769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:45.505 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:45.534 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:45.579 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:45.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:45.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:45.683 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:45.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:45 smithi190 ceph-mon[160435]: pgmap v1478: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 146 op/s; 8448/227706 objects degraded (3.710%); 4660/227706 objects misplaced (2.046%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:45.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:45 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:42:45.706 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:45 smithi190 ceph-mon[160435]: failed to encode map e517 with expected crc 2023-10-01T09:42:45.707 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:45 smithi190 ceph-mon[160435]: failed to encode map e517 with expected crc 2023-10-01T09:42:45.707 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:45 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:42:45.707 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:45 smithi190 ceph-mon[160435]: osdmap e517: 8 total, 8 up, 7 in 2023-10-01T09:42:45.707 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:45 smithi190 ceph-mon[160435]: failed to encode map e517 with expected crc 2023-10-01T09:42:45.707 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:45 smithi190 ceph-mon[160435]: from='client.46769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:45.708 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:45.763 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:45.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:45.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:45.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[192513]: pgmap v1478: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 146 op/s; 8448/227706 objects degraded (3.710%); 4660/227706 objects misplaced (2.046%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:45.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:42:45.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[192513]: failed to encode map e517 with expected crc 2023-10-01T09:42:45.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[192513]: failed to encode map e517 with expected crc 2023-10-01T09:42:45.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:42:45.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[192513]: osdmap e517: 8 total, 8 up, 7 in 2023-10-01T09:42:45.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[192513]: failed to encode map e517 with expected crc 2023-10-01T09:42:45.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:45 smithi138 ceph-mon[192513]: from='client.46769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:45.852 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1479,"stamp":"2023-10-01T09:42:44.451206+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4977203147,"num_objects":75902,"num_object_clones":0,"num_object_copies":227706,"num_objects_missing_on_primary":3269,"num_objects_missing":3269,"num_objects_degraded":8448,"num_objects_misplaced":4660,"num_objects_unfound":0,"num_objects_dirty":75902,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":91672,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7831,"num_bytes_recovered":500073487,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91736,"ondisk_log_size":91736,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":276,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20127984,"kb_used_data":15107448,"kb_used_omap":39,"kb_used_meta":5020440,"kb_avail":636088080,"statfs":{"total":671965249536,"available":651354193920,"internally_reserved":0,"allocated":15470026752,"data_stored":15460242721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5140931183},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-72286208,"num_objects":-1103,"num_object_clones":0,"num_object_copies":-3309,"num_objects_missing_on_primary":-35,"num_objects_missing":-35,"num_objects_degraded":-40,"num_objects_misplaced":-41,"num_objects_unfound":0,"num_objects_dirty":-1103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1103,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":3014656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.157164"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":292,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.465635+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:42:30.465635+0000","last_peered":"2023-10-01T09:42:30.465635+0000","last_clean":"2023-10-01T09:42:30.465635+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:42:30.465635+0000","last_undegraded":"2023-10-01T09:42:30.465635+0000","last_fullsized":"2023-10-01T09:42:30.465635+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":134,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.462289+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:42:30.462289+0000","last_peered":"2023-10-01T09:42:30.462289+0000","last_clean":"2023-10-01T09:42:30.462289+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:42:30.462289+0000","last_undegraded":"2023-10-01T09:42:30.462289+0000","last_fullsized":"2023-10-01T09:42:30.462289+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":515,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.461933+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:42:30.461933+0000","last_peered":"2023-10-01T09:42:30.461933+0000","last_clean":"2023-10-01T09:42:30.461933+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:42:30.461933+0000","last_undegraded":"2023-10-01T09:42:30.461933+0000","last_fullsized":"2023-10-01T09:42:30.461933+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":573,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.421560+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:42:30.421560+0000","last_peered":"2023-10-01T09:42:30.421560+0000","last_clean":"2023-10-01T09:42:30.421560+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:42:30.421560+0000","last_undegraded":"2023-10-01T09:42:30.421560+0000","last_fullsized":"2023-10-01T09:42:30.421560+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":454,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.421400+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:42:30.421400+0000","last_peered":"2023-10-01T09:42:30.421400+0000","last_clean":"2023-10-01T09:42:30.421400+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:42:30.421400+0000","last_undegraded":"2023-10-01T09:42:30.421400+0000","last_fullsized":"2023-10-01T09:42:30.421400+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":197,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.476249+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:42:30.476249+0000","last_peered":"2023-10-01T09:42:30.476249+0000","last_clean":"2023-10-01T09:42:30.476249+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:42:30.476249+0000","last_undegraded":"2023-10-01T09:42:30.476249+0000","last_fullsized":"2023-10-01T09:42:30.476249+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":573,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.476316+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:42:30.476316+0000","last_peered":"2023-10-01T09:42:30.476316+0000","last_clean":"2023-10-01T09:42:30.476316+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:42:30.476316+0000","last_undegraded":"2023-10-01T09:42:30.476316+0000","last_fullsized":"2023-10-01T09:42:30.476316+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":501,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.443838+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:42:30.443838+0000","last_peered":"2023-10-01T09:42:30.443838+0000","last_clean":"2023-10-01T09:42:30.443838+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:42:30.443838+0000","last_undegraded":"2023-10-01T09:42:30.443838+0000","last_fullsized":"2023-10-01T09:42:30.443838+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":228,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.421116+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:42:30.421116+0000","last_peered":"2023-10-01T09:42:30.421116+0000","last_clean":"2023-10-01T09:42:30.421116+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:42:30.421116+0000","last_undegraded":"2023-10-01T09:42:30.421116+0000","last_fullsized":"2023-10-01T09:42:30.421116+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":35,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.511825+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:42:30.511825+0000","last_peered":"2023-10-01T09:42:30.511825+0000","last_clean":"2023-10-01T09:42:30.511825+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:42:30.511825+0000","last_undegraded":"2023-10-01T09:42:30.511825+0000","last_fullsized":"2023-10-01T09:42:30.511825+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":41,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.512080+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:42:30.512080+0000","last_peered":"2023-10-01T09:42:30.512080+0000","last_clean":"2023-10-01T09:42:30.512080+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:42:30.512080+0000","last_undegraded":"2023-10-01T09:42:30.512080+0000","last_fullsized":"2023-10-01T09:42:30.512080+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":38,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.462046+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:42:30.462046+0000","last_peered":"2023-10-01T09:42:30.462046+0000","last_clean":"2023-10-01T09:42:30.462046+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:42:30.462046+0000","last_undegraded":"2023-10-01T09:42:30.462046+0000","last_fullsized":"2023-10-01T09:42:30.462046+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":417,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.466486+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:42:30.466486+0000","last_peered":"2023-10-01T09:42:30.466486+0000","last_clean":"2023-10-01T09:42:30.466486+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:42:30.466486+0000","last_undegraded":"2023-10-01T09:42:30.466486+0000","last_fullsized":"2023-10-01T09:42:30.466486+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":298,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.465938+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:42:30.465938+0000","last_peered":"2023-10-01T09:42:30.465938+0000","last_clean":"2023-10-01T09:42:30.465938+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:42:30.465938+0000","last_undegraded":"2023-10-01T09:42:30.465938+0000","last_fullsized":"2023-10-01T09:42:30.465938+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":634,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.512223+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:42:30.512223+0000","last_peered":"2023-10-01T09:42:30.512223+0000","last_clean":"2023-10-01T09:42:30.512223+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:42:30.512223+0000","last_undegraded":"2023-10-01T09:42:30.512223+0000","last_fullsized":"2023-10-01T09:42:30.512223+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":371,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.512670+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:42:30.512670+0000","last_peered":"2023-10-01T09:42:30.512670+0000","last_clean":"2023-10-01T09:42:30.512670+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:42:30.512670+0000","last_undegraded":"2023-10-01T09:42:30.512670+0000","last_fullsized":"2023-10-01T09:42:30.512670+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":44,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.512067+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:42:30.512067+0000","last_peered":"2023-10-01T09:42:30.512067+0000","last_clean":"2023-10-01T09:42:30.512067+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:42:30.512067+0000","last_undegraded":"2023-10-01T09:42:30.512067+0000","last_fullsized":"2023-10-01T09:42:30.512067+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":30,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.420369+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:42:30.420369+0000","last_peered":"2023-10-01T09:42:30.420369+0000","last_clean":"2023-10-01T09:42:30.420369+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:42:30.420369+0000","last_undegraded":"2023-10-01T09:42:30.420369+0000","last_fullsized":"2023-10-01T09:42:30.420369+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":527,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.513839+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:42:30.513839+0000","last_peered":"2023-10-01T09:42:30.513839+0000","last_clean":"2023-10-01T09:42:30.513839+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:42:30.513839+0000","last_undegraded":"2023-10-01T09:42:30.513839+0000","last_fullsized":"2023-10-01T09:42:30.513839+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":538,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.421604+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:42:30.421604+0000","last_peered":"2023-10-01T09:42:30.421604+0000","last_clean":"2023-10-01T09:42:30.421604+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:42:30.421604+0000","last_undegraded":"2023-10-01T09:42:30.421604+0000","last_fullsized":"2023-10-01T09:42:30.421604+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":394,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.421335+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:42:30.421335+0000","last_peered":"2023-10-01T09:42:30.421335+0000","last_clean":"2023-10-01T09:42:30.421335+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:42:30.421335+0000","last_undegraded":"2023-10-01T09:42:30.421335+0000","last_fullsized":"2023-10-01T09:42:30.421335+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":44,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.513059+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:42:30.513059+0000","last_peered":"2023-10-01T09:42:30.513059+0000","last_clean":"2023-10-01T09:42:30.513059+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:42:30.513059+0000","last_undegraded":"2023-10-01T09:42:30.513059+0000","last_fullsized":"2023-10-01T09:42:30.513059+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"516'5666","reported_seq":5715,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:40.321741+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:42:40.321741+0000","last_peered":"2023-10-01T09:42:40.321741+0000","last_clean":"2023-10-01T09:42:40.321741+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:42:40.321741+0000","last_undegraded":"2023-10-01T09:42:40.321741+0000","last_fullsized":"2023-10-01T09:42:40.321741+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5666,"log_dups_size":0,"ondisk_log_size":5666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309723136,"num_objects":4726,"num_object_clones":0,"num_object_copies":14178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5666,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":39,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.512900+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:42:30.512900+0000","last_peered":"2023-10-01T09:42:30.512900+0000","last_clean":"2023-10-01T09:42:30.512900+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:42:30.512900+0000","last_undegraded":"2023-10-01T09:42:30.512900+0000","last_fullsized":"2023-10-01T09:42:30.512900+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":236,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.419691+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:42:30.419691+0000","last_peered":"2023-10-01T09:42:30.419691+0000","last_clean":"2023-10-01T09:42:30.419691+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:42:30.419691+0000","last_undegraded":"2023-10-01T09:42:30.419691+0000","last_fullsized":"2023-10-01T09:42:30.419691+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"516'5619","reported_seq":6945,"reported_epoch":516,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:42:40.416230+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:42:40.416230+0000","last_peered":"2023-10-01T09:42:40.416230+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:42:40.416230+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5619,"log_dups_size":0,"ondisk_log_size":5619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313720832,"num_objects":4787,"num_object_clones":0,"num_object_copies":14361,"num_objects_missing_on_primary":3269,"num_objects_missing":3269,"num_objects_degraded":7987,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5619,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":417,"num_bytes_recovered":27328512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1449},{"shards":"0,4,6","objects":3269},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":510,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.512793+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:42:30.512793+0000","last_peered":"2023-10-01T09:42:30.512793+0000","last_clean":"2023-10-01T09:42:30.512793+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:42:30.512793+0000","last_undegraded":"2023-10-01T09:42:30.512793+0000","last_fullsized":"2023-10-01T09:42:30.512793+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":570,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.461906+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:42:30.461906+0000","last_peered":"2023-10-01T09:42:30.461906+0000","last_clean":"2023-10-01T09:42:30.461906+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:42:30.461906+0000","last_undegraded":"2023-10-01T09:42:30.461906+0000","last_fullsized":"2023-10-01T09:42:30.461906+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"516'5678","reported_seq":5729,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:41.743862+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:42:41.743862+0000","last_peered":"2023-10-01T09:42:41.743862+0000","last_clean":"2023-10-01T09:42:41.743862+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:42:41.743862+0000","last_undegraded":"2023-10-01T09:42:41.743862+0000","last_fullsized":"2023-10-01T09:42:41.743862+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5678,"log_dups_size":0,"ondisk_log_size":5678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311558144,"num_objects":4754,"num_object_clones":0,"num_object_copies":14262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5678,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":596,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.512215+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:42:30.512215+0000","last_peered":"2023-10-01T09:42:30.512215+0000","last_clean":"2023-10-01T09:42:30.512215+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:42:30.512215+0000","last_undegraded":"2023-10-01T09:42:30.512215+0000","last_fullsized":"2023-10-01T09:42:30.512215+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":391,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.462266+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:42:30.462266+0000","last_peered":"2023-10-01T09:42:30.462266+0000","last_clean":"2023-10-01T09:42:30.462266+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:42:30.462266+0000","last_undegraded":"2023-10-01T09:42:30.462266+0000","last_fullsized":"2023-10-01T09:42:30.462266+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"516'5632","reported_seq":12288,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:41.736405+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:42:41.736405+0000","last_peered":"2023-10-01T09:42:41.736405+0000","last_clean":"2023-10-01T09:42:41.736405+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:42:41.736405+0000","last_undegraded":"2023-10-01T09:42:41.736405+0000","last_fullsized":"2023-10-01T09:42:41.736405+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5632,"log_dups_size":0,"ondisk_log_size":5632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312606720,"num_objects":4770,"num_object_clones":0,"num_object_copies":14310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5632,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":236,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.420696+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:42:30.420696+0000","last_peered":"2023-10-01T09:42:30.420696+0000","last_clean":"2023-10-01T09:42:30.420696+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:42:30.420696+0000","last_undegraded":"2023-10-01T09:42:30.420696+0000","last_fullsized":"2023-10-01T09:42:30.420696+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":494,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.421495+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:42:30.421495+0000","last_peered":"2023-10-01T09:42:30.421495+0000","last_clean":"2023-10-01T09:42:30.421495+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:42:30.421495+0000","last_undegraded":"2023-10-01T09:42:30.421495+0000","last_fullsized":"2023-10-01T09:42:30.421495+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"516'5689","reported_seq":5740,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:40.055363+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:42:40.055363+0000","last_peered":"2023-10-01T09:42:40.055363+0000","last_clean":"2023-10-01T09:42:40.055363+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:42:40.055363+0000","last_undegraded":"2023-10-01T09:42:40.055363+0000","last_fullsized":"2023-10-01T09:42:40.055363+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315817984,"num_objects":4819,"num_object_clones":0,"num_object_copies":14457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5689,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":96,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.466240+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:42:30.466240+0000","last_peered":"2023-10-01T09:42:30.466240+0000","last_clean":"2023-10-01T09:42:30.466240+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:42:30.466240+0000","last_undegraded":"2023-10-01T09:42:30.466240+0000","last_fullsized":"2023-10-01T09:42:30.466240+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.466535+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:42:30.466535+0000","last_peered":"2023-10-01T09:42:30.466535+0000","last_clean":"2023-10-01T09:42:30.466535+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:42:30.466535+0000","last_undegraded":"2023-10-01T09:42:30.466535+0000","last_fullsized":"2023-10-01T09:42:30.466535+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"516'5644","reported_seq":5721,"reported_epoch":516,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:42:39.805470+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:42:39.805470+0000","last_peered":"2023-10-01T09:42:39.805470+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:42:39.805470+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5644,"log_dups_size":0,"ondisk_log_size":5644,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309592064,"num_objects":4724,"num_object_clones":0,"num_object_copies":14172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":451,"num_objects_misplaced":4660,"num_objects_unfound":0,"num_objects_dirty":4724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5644,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4660},{"shards":"4,5","objects":-387}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":414,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.465776+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:42:30.465776+0000","last_peered":"2023-10-01T09:42:30.465776+0000","last_clean":"2023-10-01T09:42:30.465776+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:42:30.465776+0000","last_undegraded":"2023-10-01T09:42:30.465776+0000","last_fullsized":"2023-10-01T09:42:30.465776+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":574,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.462358+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:42:30.462358+0000","last_peered":"2023-10-01T09:42:30.462358+0000","last_clean":"2023-10-01T09:42:30.462358+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:42:30.462358+0000","last_undegraded":"2023-10-01T09:42:30.462358+0000","last_fullsized":"2023-10-01T09:42:30.462358+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"516'5642","reported_seq":5700,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:42.452366+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:42:42.452366+0000","last_peered":"2023-10-01T09:42:42.452366+0000","last_clean":"2023-10-01T09:42:42.452366+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:42:42.452366+0000","last_undegraded":"2023-10-01T09:42:42.452366+0000","last_fullsized":"2023-10-01T09:42:42.452366+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5642,"log_dups_size":0,"ondisk_log_size":5642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310378496,"num_objects":4736,"num_object_clones":0,"num_object_copies":14208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5642,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":410,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.465839+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:42:30.465839+0000","last_peered":"2023-10-01T09:42:30.465839+0000","last_clean":"2023-10-01T09:42:30.465839+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:42:30.465839+0000","last_undegraded":"2023-10-01T09:42:30.465839+0000","last_fullsized":"2023-10-01T09:42:30.465839+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":251,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.467888+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:42:30.467888+0000","last_peered":"2023-10-01T09:42:30.467888+0000","last_clean":"2023-10-01T09:42:30.467888+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:42:30.467888+0000","last_undegraded":"2023-10-01T09:42:30.467888+0000","last_fullsized":"2023-10-01T09:42:30.467888+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"516'5740","reported_seq":5789,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:41.082936+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:42:41.082936+0000","last_peered":"2023-10-01T09:42:41.082936+0000","last_clean":"2023-10-01T09:42:41.082936+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:42:41.082936+0000","last_undegraded":"2023-10-01T09:42:41.082936+0000","last_fullsized":"2023-10-01T09:42:41.082936+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5740,"log_dups_size":0,"ondisk_log_size":5740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311820288,"num_objects":4758,"num_object_clones":0,"num_object_copies":14274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5740,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":378,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.461801+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:42:30.461801+0000","last_peered":"2023-10-01T09:42:30.461801+0000","last_clean":"2023-10-01T09:42:30.461801+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:42:30.461801+0000","last_undegraded":"2023-10-01T09:42:30.461801+0000","last_fullsized":"2023-10-01T09:42:30.461801+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":425,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.462100+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:42:30.462100+0000","last_peered":"2023-10-01T09:42:30.462100+0000","last_clean":"2023-10-01T09:42:30.462100+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:42:30.462100+0000","last_undegraded":"2023-10-01T09:42:30.462100+0000","last_fullsized":"2023-10-01T09:42:30.462100+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"516'5561","reported_seq":5610,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:40.982044+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:42:40.982044+0000","last_peered":"2023-10-01T09:42:40.982044+0000","last_clean":"2023-10-01T09:42:40.982044+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:42:40.982044+0000","last_undegraded":"2023-10-01T09:42:40.982044+0000","last_fullsized":"2023-10-01T09:42:40.982044+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5561,"log_dups_size":0,"ondisk_log_size":5561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302579712,"num_objects":4617,"num_object_clones":0,"num_object_copies":13851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5561,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":909,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.512535+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:42:30.512535+0000","last_peered":"2023-10-01T09:42:30.512535+0000","last_clean":"2023-10-01T09:42:30.512535+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:42:30.512535+0000","last_undegraded":"2023-10-01T09:42:30.512535+0000","last_fullsized":"2023-10-01T09:42:30.512535+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":246,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.420505+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:42:30.420505+0000","last_peered":"2023-10-01T09:42:30.420505+0000","last_clean":"2023-10-01T09:42:30.420505+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:42:30.420505+0000","last_undegraded":"2023-10-01T09:42:30.420505+0000","last_fullsized":"2023-10-01T09:42:30.420505+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":38,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.462115+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:42:30.462115+0000","last_peered":"2023-10-01T09:42:30.462115+0000","last_clean":"2023-10-01T09:42:30.462115+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:42:30.462115+0000","last_undegraded":"2023-10-01T09:42:30.462115+0000","last_fullsized":"2023-10-01T09:42:30.462115+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"516'5637","reported_seq":5688,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:41.785798+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:42:41.785798+0000","last_peered":"2023-10-01T09:42:41.785798+0000","last_clean":"2023-10-01T09:42:41.785798+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:42:41.785798+0000","last_undegraded":"2023-10-01T09:42:41.785798+0000","last_fullsized":"2023-10-01T09:42:41.785798+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5637,"log_dups_size":0,"ondisk_log_size":5637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310181888,"num_objects":4733,"num_object_clones":0,"num_object_copies":14199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5637,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1181,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.419966+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:42:30.419966+0000","last_peered":"2023-10-01T09:42:30.419966+0000","last_clean":"2023-10-01T09:42:30.419966+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:42:30.419966+0000","last_undegraded":"2023-10-01T09:42:30.419966+0000","last_fullsized":"2023-10-01T09:42:30.419966+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":391,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.466088+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:42:30.466088+0000","last_peered":"2023-10-01T09:42:30.466088+0000","last_clean":"2023-10-01T09:42:30.466088+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:42:30.466088+0000","last_undegraded":"2023-10-01T09:42:30.466088+0000","last_fullsized":"2023-10-01T09:42:30.466088+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":219,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.420899+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:42:30.420899+0000","last_peered":"2023-10-01T09:42:30.420899+0000","last_clean":"2023-10-01T09:42:30.420899+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:42:30.420899+0000","last_undegraded":"2023-10-01T09:42:30.420899+0000","last_fullsized":"2023-10-01T09:42:30.420899+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"516'5662","reported_seq":12932,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:39.967205+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:42:39.967205+0000","last_peered":"2023-10-01T09:42:39.967205+0000","last_clean":"2023-10-01T09:42:39.967205+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:42:39.967205+0000","last_undegraded":"2023-10-01T09:42:39.967205+0000","last_fullsized":"2023-10-01T09:42:39.967205+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5662,"log_dups_size":0,"ondisk_log_size":5662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308936704,"num_objects":4714,"num_object_clones":0,"num_object_copies":14142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5662,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":860,"reported_epoch":515,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:42:30.421160+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:42:30.421160+0000","last_peered":"2023-10-01T09:42:30.421160+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:42:30.421160+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:42:30.421160+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":578,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.461857+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:42:30.461857+0000","last_peered":"2023-10-01T09:42:30.461857+0000","last_clean":"2023-10-01T09:42:30.461857+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:42:30.461857+0000","last_undegraded":"2023-10-01T09:42:30.461857+0000","last_fullsized":"2023-10-01T09:42:30.461857+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":38,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.461963+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:42:30.461963+0000","last_peered":"2023-10-01T09:42:30.461963+0000","last_clean":"2023-10-01T09:42:30.461963+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:42:30.461963+0000","last_undegraded":"2023-10-01T09:42:30.461963+0000","last_fullsized":"2023-10-01T09:42:30.461963+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"516'5576","reported_seq":5627,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:39.753654+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:42:39.753654+0000","last_peered":"2023-10-01T09:42:39.753654+0000","last_clean":"2023-10-01T09:42:39.753654+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:42:39.753654+0000","last_undegraded":"2023-10-01T09:42:39.753654+0000","last_fullsized":"2023-10-01T09:42:39.753654+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5576,"log_dups_size":0,"ondisk_log_size":5576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307757056,"num_objects":4696,"num_object_clones":0,"num_object_copies":14088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5576,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1632,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.420173+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:42:30.420173+0000","last_peered":"2023-10-01T09:42:30.420173+0000","last_clean":"2023-10-01T09:42:30.420173+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:42:30.420173+0000","last_undegraded":"2023-10-01T09:42:30.420173+0000","last_fullsized":"2023-10-01T09:42:30.420173+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":546,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.511826+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:42:30.511826+0000","last_peered":"2023-10-01T09:42:30.511826+0000","last_clean":"2023-10-01T09:42:30.511826+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:42:30.511826+0000","last_undegraded":"2023-10-01T09:42:30.511826+0000","last_fullsized":"2023-10-01T09:42:30.511826+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":324,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.420050+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:42:30.420050+0000","last_peered":"2023-10-01T09:42:30.420050+0000","last_clean":"2023-10-01T09:42:30.420050+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:42:30.420050+0000","last_undegraded":"2023-10-01T09:42:30.420050+0000","last_fullsized":"2023-10-01T09:42:30.420050+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":314,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.466309+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:42:30.466309+0000","last_peered":"2023-10-01T09:42:30.466309+0000","last_clean":"2023-10-01T09:42:30.466309+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:42:30.466309+0000","last_undegraded":"2023-10-01T09:42:30.466309+0000","last_fullsized":"2023-10-01T09:42:30.466309+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"516'5714","reported_seq":5763,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:41.014648+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:42:41.014648+0000","last_peered":"2023-10-01T09:42:41.014648+0000","last_clean":"2023-10-01T09:42:41.014648+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:42:41.014648+0000","last_undegraded":"2023-10-01T09:42:41.014648+0000","last_fullsized":"2023-10-01T09:42:41.014648+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5714,"log_dups_size":0,"ondisk_log_size":5714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315752448,"num_objects":4818,"num_object_clones":0,"num_object_copies":14454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5714,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":723,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.511914+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:42:30.511914+0000","last_peered":"2023-10-01T09:42:30.511914+0000","last_clean":"2023-10-01T09:42:30.511914+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:42:30.511914+0000","last_undegraded":"2023-10-01T09:42:30.511914+0000","last_fullsized":"2023-10-01T09:42:30.511914+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":555,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.421859+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:42:30.421859+0000","last_peered":"2023-10-01T09:42:30.421859+0000","last_clean":"2023-10-01T09:42:30.421859+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:42:30.421859+0000","last_undegraded":"2023-10-01T09:42:30.421859+0000","last_fullsized":"2023-10-01T09:42:30.421859+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":554,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.421444+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:42:30.421444+0000","last_peered":"2023-10-01T09:42:30.421444+0000","last_clean":"2023-10-01T09:42:30.421444+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:42:30.421444+0000","last_undegraded":"2023-10-01T09:42:30.421444+0000","last_fullsized":"2023-10-01T09:42:30.421444+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"516'5707","reported_seq":5759,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:41.763297+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:42:41.763297+0000","last_peered":"2023-10-01T09:42:41.763297+0000","last_clean":"2023-10-01T09:42:41.763297+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:42:41.763297+0000","last_undegraded":"2023-10-01T09:42:41.763297+0000","last_fullsized":"2023-10-01T09:42:41.763297+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5707,"log_dups_size":0,"ondisk_log_size":5707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311427096,"num_objects":4753,"num_object_clones":0,"num_object_copies":14259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4753,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5707,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":954,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.512294+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:42:30.512294+0000","last_peered":"2023-10-01T09:42:30.512294+0000","last_clean":"2023-10-01T09:42:30.512294+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:42:30.512294+0000","last_undegraded":"2023-10-01T09:42:30.512294+0000","last_fullsized":"2023-10-01T09:42:30.512294+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":573,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.420806+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:42:30.420806+0000","last_peered":"2023-10-01T09:42:30.420806+0000","last_clean":"2023-10-01T09:42:30.420806+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:42:30.420806+0000","last_undegraded":"2023-10-01T09:42:30.420806+0000","last_fullsized":"2023-10-01T09:42:30.420806+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":426,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.420248+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:42:30.420248+0000","last_peered":"2023-10-01T09:42:30.420248+0000","last_clean":"2023-10-01T09:42:30.420248+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:42:30.420248+0000","last_undegraded":"2023-10-01T09:42:30.420248+0000","last_fullsized":"2023-10-01T09:42:30.420248+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"516'5681","reported_seq":5738,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:40.914178+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:42:40.914178+0000","last_peered":"2023-10-01T09:42:40.914178+0000","last_clean":"2023-10-01T09:42:40.914178+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:42:40.914178+0000","last_undegraded":"2023-10-01T09:42:40.914178+0000","last_fullsized":"2023-10-01T09:42:40.914178+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5681,"log_dups_size":0,"ondisk_log_size":5681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306380800,"num_objects":4675,"num_object_clones":0,"num_object_copies":14025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5681,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":455,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.513203+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:42:30.513203+0000","last_peered":"2023-10-01T09:42:30.513203+0000","last_clean":"2023-10-01T09:42:30.513203+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:42:30.513203+0000","last_undegraded":"2023-10-01T09:42:30.513203+0000","last_fullsized":"2023-10-01T09:42:30.513203+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1148,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.462647+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:42:30.462647+0000","last_peered":"2023-10-01T09:42:30.462647+0000","last_clean":"2023-10-01T09:42:30.462647+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:42:30.462647+0000","last_undegraded":"2023-10-01T09:42:30.462647+0000","last_fullsized":"2023-10-01T09:42:30.462647+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":44,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.511775+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:42:30.511775+0000","last_peered":"2023-10-01T09:42:30.511775+0000","last_clean":"2023-10-01T09:42:30.511775+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:42:30.511775+0000","last_undegraded":"2023-10-01T09:42:30.511775+0000","last_fullsized":"2023-10-01T09:42:30.511775+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"516'5718","reported_seq":5775,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:41.226737+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:42:41.226737+0000","last_peered":"2023-10-01T09:42:41.226737+0000","last_clean":"2023-10-01T09:42:41.226737+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:42:41.226737+0000","last_undegraded":"2023-10-01T09:42:41.226737+0000","last_fullsized":"2023-10-01T09:42:41.226737+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5718,"log_dups_size":0,"ondisk_log_size":5718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314179584,"num_objects":4794,"num_object_clones":0,"num_object_copies":14382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5718,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":48,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.512984+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:42:30.512984+0000","last_peered":"2023-10-01T09:42:30.512984+0000","last_clean":"2023-10-01T09:42:30.512984+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:42:30.512984+0000","last_undegraded":"2023-10-01T09:42:30.512984+0000","last_fullsized":"2023-10-01T09:42:30.512984+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":532,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.421510+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:42:30.421510+0000","last_peered":"2023-10-01T09:42:30.421510+0000","last_clean":"2023-10-01T09:42:30.421510+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:42:30.421510+0000","last_undegraded":"2023-10-01T09:42:30.421510+0000","last_fullsized":"2023-10-01T09:42:30.421510+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":239,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.419718+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:42:30.419718+0000","last_peered":"2023-10-01T09:42:30.419718+0000","last_clean":"2023-10-01T09:42:30.419718+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:42:30.419718+0000","last_undegraded":"2023-10-01T09:42:30.419718+0000","last_fullsized":"2023-10-01T09:42:30.419718+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":481,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.421662+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:42:30.421662+0000","last_peered":"2023-10-01T09:42:30.421662+0000","last_clean":"2023-10-01T09:42:30.421662+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:42:30.421662+0000","last_undegraded":"2023-10-01T09:42:30.421662+0000","last_fullsized":"2023-10-01T09:42:30.421662+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":595,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.462127+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:42:30.462127+0000","last_peered":"2023-10-01T09:42:30.462127+0000","last_clean":"2023-10-01T09:42:30.462127+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:42:30.462127+0000","last_undegraded":"2023-10-01T09:42:30.462127+0000","last_fullsized":"2023-10-01T09:42:30.462127+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":251,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.420202+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:42:30.420202+0000","last_peered":"2023-10-01T09:42:30.420202+0000","last_clean":"2023-10-01T09:42:30.420202+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:42:30.420202+0000","last_undegraded":"2023-10-01T09:42:30.420202+0000","last_fullsized":"2023-10-01T09:42:30.420202+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":38,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.461802+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:42:30.461802+0000","last_peered":"2023-10-01T09:42:30.461802+0000","last_clean":"2023-10-01T09:42:30.461802+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:42:30.461802+0000","last_undegraded":"2023-10-01T09:42:30.461802+0000","last_fullsized":"2023-10-01T09:42:30.461802+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":251,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.420587+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:42:30.420587+0000","last_peered":"2023-10-01T09:42:30.420587+0000","last_clean":"2023-10-01T09:42:30.420587+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:42:30.420587+0000","last_undegraded":"2023-10-01T09:42:30.420587+0000","last_fullsized":"2023-10-01T09:42:30.420587+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":427,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.421935+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:42:30.421935+0000","last_peered":"2023-10-01T09:42:30.421935+0000","last_clean":"2023-10-01T09:42:30.421935+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:42:30.421935+0000","last_undegraded":"2023-10-01T09:42:30.421935+0000","last_fullsized":"2023-10-01T09:42:30.421935+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":56,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.462487+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:42:30.462487+0000","last_peered":"2023-10-01T09:42:30.462487+0000","last_clean":"2023-10-01T09:42:30.462487+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:42:30.462487+0000","last_undegraded":"2023-10-01T09:42:30.462487+0000","last_fullsized":"2023-10-01T09:42:30.462487+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":472,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.421775+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:42:30.421775+0000","last_peered":"2023-10-01T09:42:30.421775+0000","last_clean":"2023-10-01T09:42:30.421775+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:42:30.421775+0000","last_undegraded":"2023-10-01T09:42:30.421775+0000","last_fullsized":"2023-10-01T09:42:30.421775+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":672,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.422223+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:42:30.422223+0000","last_peered":"2023-10-01T09:42:30.422223+0000","last_clean":"2023-10-01T09:42:30.422223+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:42:30.422223+0000","last_undegraded":"2023-10-01T09:42:30.422223+0000","last_fullsized":"2023-10-01T09:42:30.422223+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":367,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-10-01T09:42:30.465451+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:42:30.465451+0000","last_peered":"2023-10-01T09:42:30.465451+0000","last_clean":"2023-10-01T09:42:30.465451+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:42:30.465451+0000","last_undegraded":"2023-10-01T09:42:30.465451+0000","last_fullsized":"2023-10-01T09:42:30.465451+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4972412952,"num_objects":75874,"num_object_clones":0,"num_object_copies":227622,"num_objects_missing_on_primary":3269,"num_objects_missing":3269,"num_objects_degraded":8438,"num_objects_misplaced":4660,"num_objects_unfound":0,"num_objects_dirty":75874,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":90566,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7320,"num_bytes_recovered":479723520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15405559808,"data_stored":15405547592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90566,"ondisk_log_size":90566,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430705,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2421296,"kb_used_data":1526376,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":91323856,"statfs":{"total":95995035648,"available":93515628544,"internally_reserved":0,"allocated":1563009024,"data_stored":1562074560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.766,"15min":3.0510000000000002},"min":{"1min":0.30099999999999999,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":6.8550000000000004,"5min":11.525,"15min":108.90000000000001},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.82099999999999995,"5min":2.2130000000000001,"15min":3.3359999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.6639999999999997,"5min":29.501000000000001,"15min":109.187},"last":0.76600000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:42:06 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":5.0410000000000004,"15min":5.492},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":6.8289999999999997,"5min":229.161,"15min":229.161},"last":0.94499999999999995},{"interface":"front","average":{"1min":0.92100000000000004,"5min":4.9939999999999998,"15min":5.5490000000000004},"min":{"1min":0.28899999999999998,"5min":0.27500000000000002,"15min":0.24399999999999999},"max":{"1min":5.7240000000000002,"5min":230.126,"15min":230.126},"last":0.79600000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:41:47 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":2.1680000000000001,"15min":2.0169999999999999},"min":{"1min":0.33200000000000002,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":5.3090000000000002,"5min":14.866,"15min":14.901999999999999},"last":0.77500000000000002},{"interface":"front","average":{"1min":1.2829999999999999,"5min":2.1749999999999998,"15min":1.9079999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.6929999999999996,"5min":14.682,"15min":14.682},"last":0.96599999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:42:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.9410000000000001,"15min":2.8610000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":5.7750000000000004,"5min":35.104999999999997,"15min":80.352999999999994},"last":0.90700000000000003},{"interface":"front","average":{"1min":1.004,"5min":2,"15min":2.8359999999999999},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.548,"5min":35.118000000000002,"15min":80.364000000000004},"last":0.80500000000000005}]},{"osd":4,"last update":"Sun Oct 1 09:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.202,"15min":1.0529999999999999},"min":{"1min":0.30499999999999999,"5min":0.27600000000000002,"15min":0.254},"max":{"1min":6.8769999999999998,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.69399999999999995},{"interface":"front","average":{"1min":1.4710000000000001,"5min":1.1739999999999999,"15min":1.0369999999999999},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":5.7439999999999998,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.70999999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":2.8879999999999999,"15min":1.6419999999999999},"min":{"1min":0.318,"5min":0.30299999999999999,"15min":0.29599999999999999},"max":{"1min":0.64600000000000002,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.41999999999999998,"5min":3.0779999999999998,"15min":1.6850000000000001},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54500000000000004,"5min":135.339,"15min":135.339},"last":0.621}]},{"osd":6,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.4379999999999999,"15min":1.173},"min":{"1min":0.313,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":2.448,"5min":11.58,"15min":11.853999999999999},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.44400000000000001,"5min":1.2789999999999999,"15min":1.1359999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":0.56699999999999995,"5min":14.584,"15min":14.584},"last":0.72199999999999998}]}]},{"osd":6,"up_from":294,"seq":1262720385216,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2461932,"kb_used_data":2124324,"kb_used_omap":6,"kb_used_meta":337593,"kb_avail":91283220,"statfs":{"total":95995035648,"available":93474017280,"internally_reserved":0,"allocated":2175307776,"data_stored":2174377263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":345695277},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":3.786,"15min":4.1879999999999997},"min":{"1min":0.249,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":1.8400000000000001,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.76300000000000001,"5min":4.3559999999999999,"15min":4.2889999999999997},"min":{"1min":0.249,"5min":0.249,"15min":0.221},"max":{"1min":2.9969999999999999,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.52800000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":5.3860000000000001,"15min":5.0590000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":3.0289999999999999,"5min":168.947,"15min":168.947},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.79100000000000004,"5min":5.375,"15min":4.9690000000000003},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":2.6030000000000002,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.8109999999999999,"15min":1.8420000000000001},"min":{"1min":0.34499999999999997,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":3.0489999999999999,"5min":9.657,"15min":26.289999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.885,"15min":1.859},"min":{"1min":0.32600000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.6739999999999999,"5min":21.704999999999998,"15min":26.388999999999999},"last":0.47399999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":1.597,"15min":2.4020000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.9450000000000001,"5min":9.0180000000000007,"15min":89.753},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.073,"5min":1.9330000000000001,"15min":2.2189999999999999},"min":{"1min":0.318,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":3.7160000000000002,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.46000000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.665,"15min":1.0800000000000001},"min":{"1min":0.28599999999999998,"5min":0.253,"15min":0.246},"max":{"1min":4.0250000000000004,"5min":63.134,"15min":63.134},"last":0.317},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.518,"15min":1.0960000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.28299999999999997},"max":{"1min":7.4039999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.38900000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:42:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.93200000000000005,"15min":1.02},"min":{"1min":0.26700000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.8620000000000001,"5min":9.9939999999999998,"15min":12.023999999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.66500000000000004,"5min":1.175,"15min":1.143},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.726,"5min":15.457000000000001,"15min":15.457000000000001},"last":0.34999999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.82999999999999996,"15min":1.802},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":2.0419999999999998,"5min":6.4119999999999999,"15min":115.11},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.282,"15min":1.8959999999999999},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.28799999999999998},"max":{"1min":3.2679999999999998,"5min":8.6899999999999995,"15min":114.666},"last":0.40300000000000002}]}]},{"osd":1,"up_from":94,"seq":403726926192,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1737612,"kb_used_data":923524,"kb_used_omap":7,"kb_used_meta":814072,"kb_avail":92007540,"statfs":{"total":95995035648,"available":94215720960,"internally_reserved":0,"allocated":945688576,"data_stored":943675360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":833610151},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":3.5089999999999999,"15min":4.5229999999999997},"min":{"1min":0.19400000000000001,"5min":0.182,"15min":0.182},"max":{"1min":4.423,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.254},{"interface":"front","average":{"1min":1.528,"5min":4.0339999999999998,"15min":3.859},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":13.872,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.33700000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:42:38 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":1.7929999999999999,"15min":2.7719999999999998},"min":{"1min":0.24099999999999999,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.83099999999999996,"5min":22.891999999999999,"15min":56.573},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.36699999999999999,"5min":2.2799999999999998,"15min":3.593},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":0.54400000000000004,"5min":15.189,"15min":154.197},"last":0.38800000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":3.9089999999999998,"15min":4.4429999999999996},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":3.1949999999999998,"5min":95.305000000000007,"15min":200.56800000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.68799999999999994,"5min":4.3550000000000004,"15min":5.2569999999999997},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":4.7839999999999998,"5min":98.033000000000001,"15min":200.63800000000001},"last":0.315}]},{"osd":4,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":2.4870000000000001,"15min":2.3940000000000001},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.5510000000000002,"5min":21.015000000000001,"15min":154.101},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":2.306,"15min":2.7410000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.4209999999999998,"5min":13.773999999999999,"15min":154.56999999999999},"last":0.45200000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:42:35 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.843,"15min":2.1240000000000001},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.2450000000000001,"5min":21.219999999999999,"15min":154.22999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.6499999999999999,"15min":2.5649999999999999},"min":{"1min":0.316,"5min":0.307,"15min":0.307},"max":{"1min":2.407,"5min":10.305,"15min":154.54400000000001},"last":0.40300000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.1669999999999998,"15min":2.8370000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.3660000000000001,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.44},{"interface":"front","average":{"1min":0.626,"5min":2.4769999999999999,"15min":2.5899999999999999},"min":{"1min":0.36699999999999999,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":1.575,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.49199999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.617,"15min":2.7250000000000001},"min":{"1min":0.40500000000000003,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":5.2169999999999996,"5min":21.062999999999999,"15min":154.20699999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.88300000000000001,"5min":2.5390000000000001,"15min":2.9399999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":3.8260000000000001,"5min":13.819000000000001,"15min":154.51599999999999},"last":0.50900000000000001}]}]},{"osd":0,"up_from":89,"seq":382252089727,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4595592,"kb_used_data":3700352,"kb_used_omap":8,"kb_used_meta":895223,"kb_avail":89149560,"statfs":{"total":95995035648,"available":91289149440,"internally_reserved":0,"allocated":3789160448,"data_stored":3787156671,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":916709314},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:42:08 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":3.9860000000000002,"15min":4.4039999999999999},"min":{"1min":0.20399999999999999,"5min":0.187,"15min":0.17399999999999999},"max":{"1min":1.661,"5min":185.78399999999999,"15min":216.821},"last":0.224},{"interface":"front","average":{"1min":0.437,"5min":5.9020000000000001,"15min":5.4400000000000004},"min":{"1min":0.249,"5min":0.215,"15min":0.215},"max":{"1min":1.5589999999999999,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.36599999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:42:36 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":1.8779999999999999,"15min":2.1579999999999999},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.20100000000000001},"max":{"1min":0.51200000000000001,"5min":19.788,"15min":19.788},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.36899999999999999,"5min":2.0960000000000001,"15min":2.5230000000000001},"min":{"1min":0.315,"5min":0.215,"15min":0.189},"max":{"1min":0.439,"5min":25.844999999999999,"15min":25.844999999999999},"last":0.311}]},{"osd":3,"last update":"Sun Oct 1 09:42:08 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":2.1749999999999998,"15min":4.524},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.20899999999999999},"max":{"1min":1.706,"5min":15.35,"15min":222.16499999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.873,"5min":2.5219999999999998,"15min":4.7089999999999996},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":2.859,"5min":14.545999999999999,"15min":222.227},"last":0.251}]},{"osd":4,"last update":"Sun Oct 1 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":2.3559999999999999,"15min":2.4260000000000002},"min":{"1min":0.29899999999999999,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":2.7480000000000002,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.034,"5min":2.698,"15min":2.536},"min":{"1min":0.376,"5min":0.33700000000000002,"15min":0.28299999999999997},"max":{"1min":3.1619999999999999,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.41199999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:42:30 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.837,"15min":1.9730000000000001},"min":{"1min":0.33700000000000002,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":1.9730000000000001,"5min":14.515000000000001,"15min":16.808},"last":0.378},{"interface":"front","average":{"1min":0.47199999999999998,"5min":1.6000000000000001,"15min":1.8480000000000001},"min":{"1min":0.34799999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64000000000000001,"5min":8.9740000000000002,"15min":17.507000000000001},"last":0.5}]},{"osd":6,"last update":"Sun Oct 1 09:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.617,"15min":1.8360000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.35499999999999998},"max":{"1min":1.5029999999999999,"5min":8.9920000000000009,"15min":17.535},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.54900000000000004,"5min":1.879,"15min":2.137},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":1.5760000000000001,"5min":10.446,"15min":18.521000000000001},"last":0.48199999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.198,"15min":2.1909999999999998},"min":{"1min":0.41799999999999998,"5min":0.377,"15min":0.34499999999999997},"max":{"1min":4.976,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.069,"5min":2.4740000000000002,"15min":2.5459999999999998},"min":{"1min":0.41199999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":4.226,"5min":48.274000000000001,"15min":48.274000000000001},"last":0.42499999999999999}]}]},{"osd":2,"up_from":0,"seq":425201762667,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599140,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1993660,"kb_used_data":1165620,"kb_used_omap":3,"kb_used_meta":828028,"kb_avail":91751492,"statfs":{"total":95995035648,"available":93953527808,"internally_reserved":0,"allocated":1193594880,"data_stored":1191538831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":847901415},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":5.9850000000000003,"5min":5.5869999999999997,"15min":6.0170000000000003},"min":{"1min":0.216,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":64.504000000000005,"5min":118.748,"15min":137.34399999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":6.024,"5min":5.7830000000000004,"15min":6.0090000000000003},"min":{"1min":0.24299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":64.448999999999998,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.42199999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":5.149,"15min":5.4130000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":6.0250000000000004,"5min":108.84999999999999,"15min":162.22999999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.60199999999999998,"5min":5.3860000000000001,"15min":5.718},"min":{"1min":0.23100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.931,"5min":109.13200000000001,"15min":162.70699999999999},"last":0.44800000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":2.5070000000000001,"15min":2.5},"min":{"1min":0.27000000000000002,"5min":0.23000000000000001,"15min":0.20799999999999999},"max":{"1min":7.141,"5min":14.254,"15min":43.222999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.4359999999999999,"15min":2.823},"min":{"1min":0.28299999999999997,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":3.4550000000000001,"5min":11.202,"15min":78.594999999999999},"last":0.48699999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":2.423,"15min":1.972},"min":{"1min":0.312,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":6.8890000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.96399999999999997,"5min":2.5800000000000001,"15min":2.202},"min":{"1min":0.33900000000000002,"5min":0.28699999999999998,"15min":0.27900000000000003},"max":{"1min":6.0940000000000003,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.58499999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.766,"15min":2.7850000000000001},"min":{"1min":0.32500000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.70499999999999996,"5min":10.919,"15min":28.117999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.54600000000000004,"5min":2.0139999999999998,"15min":2.8879999999999999},"min":{"1min":0.35399999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.5580000000000001,"5min":12.989000000000001,"15min":28.34},"last":0.64500000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":2.7970000000000002,"15min":2.3159999999999998},"min":{"1min":0.34399999999999997,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":1.621,"5min":73.887,"15min":73.887},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.66200000000000003,"5min":2.9670000000000001,"15min":2.3090000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.32800000000000001},"max":{"1min":1.8440000000000001,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.57499999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":2.3839999999999999,"15min":2.4249999999999998},"min":{"1min":0.41499999999999998,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":7.0620000000000003,"5min":13.085000000000001,"15min":52.006},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.774,"15min":2.1360000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.2690000000000001,"5min":11.109999999999999,"15min":51.890999999999998},"last":0.71599999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019390,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4635628,"kb_used_data":3745444,"kb_used_omap":0,"kb_used_meta":890176,"kb_avail":89109524,"statfs":{"total":95995035648,"available":91248152576,"internally_reserved":0,"allocated":3835334656,"data_stored":3834403698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911540224},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:53 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":3.5209999999999999,"15min":4.9050000000000002},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":9.6690000000000005,"5min":79.837000000000003,"15min":155.315},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.875,"5min":3.9239999999999999,"15min":5.1550000000000002},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.23899999999999999},"max":{"1min":6.1459999999999999,"5min":78.983000000000004,"15min":155.334},"last":0.52500000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":8.1189999999999998,"15min":7.0599999999999996},"min":{"1min":0.35399999999999998,"5min":0.224,"15min":0.224},"max":{"1min":5.944,"5min":146.929,"15min":190.61199999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.0660000000000001,"5min":8.0500000000000007,"15min":7.8029999999999999},"min":{"1min":0.38500000000000001,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":3.7469999999999999,"5min":146.91,"15min":253.273},"last":0.503}]},{"osd":2,"last update":"Sun Oct 1 09:41:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.6279999999999999,"15min":2.6560000000000001},"min":{"1min":0.38300000000000001,"5min":0.29099999999999998,"15min":0.28299999999999997},"max":{"1min":5.702,"5min":10.161,"15min":170.45400000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.5089999999999999,"5min":1.988,"15min":2.371},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":5.6150000000000002,"5min":10.625,"15min":58.771000000000001},"last":0.67200000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.5600000000000001,"15min":2.931},"min":{"1min":0.33600000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.5279999999999996,"5min":15.989000000000001,"15min":140.881},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.825,"15min":3.145},"min":{"1min":0.313,"5min":0.311,"15min":0.311},"max":{"1min":3.786,"5min":15.978999999999999,"15min":140.911},"last":0.57499999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:42:31 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":2.1579999999999999,"15min":1.5900000000000001},"min":{"1min":0.26800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.1190000000000002,"5min":110.003,"15min":110.003},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.46000000000000002,"5min":2.069,"15min":1.405},"min":{"1min":0.317,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.0780000000000001,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.61399999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":0.90000000000000002,"15min":0.996},"min":{"1min":0.34599999999999997,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":9.9849999999999994,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.087,"15min":1.083},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":4.0220000000000002,"5min":6.9139999999999997,"15min":10.076000000000001},"last":0.44600000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:41:43 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.2250000000000001,"15min":1.171},"min":{"1min":0.34599999999999997,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":3.8580000000000001,"5min":38.034999999999997,"15min":38.034999999999997},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.0299999999999998,"5min":1.6899999999999999,"15min":1.4530000000000001},"min":{"1min":0.377,"5min":0.35899999999999999,"15min":0.28699999999999998},"max":{"1min":12.092000000000001,"5min":37.723999999999997,"15min":37.723999999999997},"last":0.56200000000000006}]}]},{"osd":5,"up_from":379,"seq":1627792605297,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2282264,"kb_used_data":1921808,"kb_used_omap":13,"kb_used_meta":360434,"kb_avail":91462888,"statfs":{"total":95995035648,"available":93657997312,"internally_reserved":0,"allocated":1967931392,"data_stored":1967016338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":369084914},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":1.897,"15min":2.258},"min":{"1min":0.23200000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.50700000000000001,"5min":12.571999999999999,"15min":84.340999999999994},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.39500000000000002,"5min":1.5289999999999999,"15min":2.1349999999999998},"min":{"1min":0.27300000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60299999999999998,"5min":8.6460000000000008,"15min":84.356999999999999},"last":0.34499999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":6.7510000000000003,"15min":5.8460000000000001},"min":{"1min":0.26800000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.55400000000000005,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.41199999999999998,"5min":7.3490000000000002,"15min":5.7889999999999997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":0.63200000000000001,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.36699999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.6020000000000001,"15min":1.6519999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.56599999999999995,"5min":8.7040000000000006,"15min":14.199999999999999},"last":0.377},{"interface":"front","average":{"1min":0.47799999999999998,"5min":1.6220000000000001,"15min":1.661},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":1.931,"5min":8.2370000000000001,"15min":14.430999999999999},"last":0.42699999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":1.796,"15min":1.5269999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.8720000000000001,"5min":12.734,"15min":14.004},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":1.8580000000000001,"15min":1.673},"min":{"1min":0.39800000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.8819999999999999,"5min":11.766,"15min":13.991},"last":0.41599999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":2.2200000000000002,"15min":1.4670000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.873,"5min":122.971,"15min":122.971},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.46000000000000002,"5min":2.3140000000000001,"15min":1.806},"min":{"1min":0.307,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.78400000000000003,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.56100000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":1.2849999999999999,"15min":1.0780000000000001},"min":{"1min":0.34499999999999997,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":0.68200000000000005,"5min":12.545,"15min":12.545},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.998,"15min":1.3640000000000001},"min":{"1min":0.34100000000000003,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":0.69999999999999996,"5min":6.4729999999999999,"15min":14.193},"last":0.55000000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":1.141,"15min":1.3200000000000001},"min":{"1min":0.32300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.63,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":1.2649999999999999,"15min":1.097},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.318},"max":{"1min":0.70399999999999996,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.501}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3777236992,"data_stored":3777232920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":935460864,"data_stored":935460864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1186332672,"data_stored":1186332672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3823702016,"data_stored":3823697944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1962082304,"data_stored":1962082304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2169507840,"data_stored":2169503768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1551237120,"data_stored":1551237120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:42:45.858 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:42:45.866 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:45.930 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:45.933 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:45.982 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:45.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:45.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:46.088 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:46.118 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:46.224 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:46.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:46.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:46.329 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:46.359 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:46.459 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:46 smithi138 ceph-mon[192513]: pgmap v1480: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8427/226677 objects degraded (3.718%); 4611/226677 objects misplaced (2.034%); 577 KiB/s, 9 objects/s recovering 2023-10-01T09:42:46.459 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:46 smithi138 ceph-mon[185886]: pgmap v1480: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8427/226677 objects degraded (3.718%); 4611/226677 objects misplaced (2.034%); 577 KiB/s, 9 objects/s recovering 2023-10-01T09:42:46.511 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:46.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:46.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:46.555 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:42:46.555 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:42:46.556 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:42:46.556 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:42:46.556 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:42:46.582 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:46 smithi190 ceph-mon[160435]: pgmap v1480: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8427/226677 objects degraded (3.718%); 4611/226677 objects misplaced (2.034%); 577 KiB/s, 9 objects/s recovering 2023-10-01T09:42:46.616 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:46.643 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:46.685 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:46.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:46.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:46.789 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:46.816 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:47.008 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:47.028 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:47.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:47.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:47.132 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:47.164 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:47.229 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:42:47.277 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T09:42:47.277 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:47.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:47.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:47.382 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:47.413 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:47.463 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:47.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:47.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:47.567 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:47.597 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:47.615 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:42:47.634 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:42:47.654 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:42:47.657 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:42:47.674 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:42:47.696 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:42:47.698 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:47.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:47.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:47.719 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:42:47.737 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:42:47.755 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:42:47.756 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:42:47.756 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:42:47.756 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:42:47.756 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:42:47.756 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:42:47.757 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:42:47.757 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:42:47.757 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:42:47.808 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:47.838 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:47.908 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:47 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/287549936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:48.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:47 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/287549936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:48.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:47 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/287549936' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:48.245 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:48.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:48.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:48.349 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:48.376 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:48.697 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:48.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:48.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:48.802 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:48.825 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:48.832 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:48.903 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:48 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8427/226677 objects degraded (3.718%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:48.903 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:48 smithi138 ceph-mon[185886]: pgmap v1481: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 8427/225615 objects degraded (3.735%); 4611/225615 objects misplaced (2.044%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:42:48.903 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:48 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8427/226677 objects degraded (3.718%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:48.903 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:48 smithi138 ceph-mon[192513]: pgmap v1481: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 8427/225615 objects degraded (3.735%); 4611/225615 objects misplaced (2.044%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:42:48.936 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:48.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:48.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:48.965 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:48 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8427/226677 objects degraded (3.718%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:48.966 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:48 smithi190 ceph-mon[160435]: pgmap v1481: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 8427/225615 objects degraded (3.735%); 4611/225615 objects misplaced (2.044%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:42:49.040 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:49.072 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:49.172 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:49.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:49.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:49.185 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:42:49.185 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:42:49.276 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:49.305 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:49.441 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:42:49.441 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 8427/225615 objects degraded (3.735%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1990,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":517,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":75205,"data_bytes":4931524555,"bytes_used":20466671616,"bytes_avail":651498577920,"bytes_total":671965249536,"degraded_objects":8427,"degraded_total":225615,"degraded_ratio":0.037351239944152646,"misplaced_objects":4611,"misplaced_total":225615,"misplaced_ratio":0.020437470912838242,"recovering_objects_per_sec":7,"recovering_bytes_per_sec":524141,"recovering_keys_per_sec":0,"num_objects_recovered":80,"num_bytes_recovered":5242880,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":140},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":33,"modified":"2023-10-01T09:41:58.284615+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:42:49.945 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:42:50.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/329454134' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:42:50.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/329454134' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:42:50.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/329454134' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:42:50.258 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:50.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:50.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:50.362 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:50.391 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:50.689 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:42:50.690 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:42:50.699 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:50.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:50.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:50.756 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:42:50.803 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:50.811 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:50.835 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:50.957 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:50.958 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:50 smithi138 ceph-mon[185886]: pgmap v1482: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 8427/225423 objects degraded (3.738%); 4611/225423 objects misplaced (2.045%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:42:50.959 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:50 smithi138 ceph-mon[192513]: pgmap v1482: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 8427/225423 objects degraded (3.738%); 4611/225423 objects misplaced (2.045%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:42:50.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:50.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:51.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:50 smithi190 ceph-mon[160435]: pgmap v1482: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 8427/225423 objects degraded (3.738%); 4611/225423 objects misplaced (2.045%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:42:51.061 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:51.092 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:51.140 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:51.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:51.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:51.248 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:51.276 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:51.322 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:51.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:51.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:51.431 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:51.469 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:51.525 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:51.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:51.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:51.628 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:51.657 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:51.702 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:51.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:51.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:51.786 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:42:51.806 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:51.833 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:52.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:51 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2528738818' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:42:52.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:51 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:42:52.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:51 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2528738818' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:42:52.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:51 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:42:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:51 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2528738818' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:42:52.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:51 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:42:52.178 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:52.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:52.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:52.282 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:52.311 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:52.365 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:52.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:52.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:52.470 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:52.497 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:52.566 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:52.595 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:52.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:52.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:52.701 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:52.720 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:52.733 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:52.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:52 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:52.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:52 smithi138 ceph-mon[185886]: failed to encode map e518 with expected crc 2023-10-01T09:42:52.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:52 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:42:52.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:52 smithi138 ceph-mon[185886]: osdmap e518: 8 total, 8 up, 7 in 2023-10-01T09:42:52.777 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:52 smithi138 ceph-mon[185886]: failed to encode map e518 with expected crc 2023-10-01T09:42:52.777 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:52 smithi138 ceph-mon[185886]: pgmap v1484: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 8381/224013 objects degraded (3.741%); 4579/224013 objects misplaced (2.044%); 584 KiB/s, 9 objects/s recovering 2023-10-01T09:42:52.777 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:52 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8427/225423 objects degraded (3.738%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:52.795 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:52.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:52.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:52.899 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:52.925 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:52.973 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:52.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:52.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:53.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:52 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:52 smithi190 ceph-mon[160435]: failed to encode map e518 with expected crc 2023-10-01T09:42:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:52 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:42:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:52 smithi190 ceph-mon[160435]: osdmap e518: 8 total, 8 up, 7 in 2023-10-01T09:42:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:52 smithi190 ceph-mon[160435]: failed to encode map e518 with expected crc 2023-10-01T09:42:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:52 smithi190 ceph-mon[160435]: pgmap v1484: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 8381/224013 objects degraded (3.741%); 4579/224013 objects misplaced (2.044%); 584 KiB/s, 9 objects/s recovering 2023-10-01T09:42:53.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:52 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8427/225423 objects degraded (3.738%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:53.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:52 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:42:53.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:52 smithi138 ceph-mon[192513]: failed to encode map e518 with expected crc 2023-10-01T09:42:53.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:52 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:42:53.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:52 smithi138 ceph-mon[192513]: osdmap e518: 8 total, 8 up, 7 in 2023-10-01T09:42:53.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:52 smithi138 ceph-mon[192513]: failed to encode map e518 with expected crc 2023-10-01T09:42:53.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:52 smithi138 ceph-mon[192513]: pgmap v1484: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 8381/224013 objects degraded (3.741%); 4579/224013 objects misplaced (2.044%); 584 KiB/s, 9 objects/s recovering 2023-10-01T09:42:53.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:52 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8427/225423 objects degraded (3.738%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:53.081 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:53.109 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:53.156 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:53.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:53.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:53.265 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:53.291 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:53.338 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:53.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:53.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:53.446 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:53.477 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:53.523 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:53.530 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:42:53.530 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:42:53.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:53.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:53.632 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:53.662 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:53.706 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:53.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:53.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:53.813 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:42:53.839 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:53.885 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:42:53.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:53.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:42:53.990 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:54.016 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:54.064 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:54.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:54.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:54.171 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:54.198 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:54.252 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:54.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:54.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:54.360 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:54.388 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:54.441 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:54.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:54.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:54.547 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:54.574 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:54.621 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:54.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:54.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:54.727 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:42:54.755 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:54.806 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:42:54.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:54.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:42:54.831 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:42:54.867 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:42:54.913 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:54.987 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:55.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:42:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:42:54] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.33.4" 2023-10-01T09:42:55.087 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:55.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:55.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:55.192 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:55.218 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:55.262 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:55.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:55.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:55.353 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:55 smithi138 ceph-mon[192513]: pgmap v1485: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 129 op/s; 8381/223875 objects degraded (3.744%); 4579/223875 objects misplaced (2.045%); 474 KiB/s, 7 objects/s recovering 2023-10-01T09:42:55.353 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:42:55.368 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:55.404 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:55.469 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:55.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:55.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:55.489 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:42:55.508 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:42:55.527 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:42:55.546 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:42:55.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:55 smithi190 ceph-mon[160435]: pgmap v1485: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 129 op/s; 8381/223875 objects degraded (3.744%); 4579/223875 objects misplaced (2.045%); 474 KiB/s, 7 objects/s recovering 2023-10-01T09:42:55.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:42:55.567 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:42:55.575 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:55.586 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:55.613 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:55 smithi138 ceph-mon[185886]: pgmap v1485: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 129 op/s; 8381/223875 objects degraded (3.744%); 4579/223875 objects misplaced (2.045%); 474 KiB/s, 7 objects/s recovering 2023-10-01T09:42:55.613 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:42:55.617 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:55.670 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:42:55.692 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:42:55.718 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:42:55.719 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:42:55.719 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:42:55.719 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:42:55.719 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:42:55.719 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:42:55.720 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:42:55.720 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:42:55.720 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:42:55.724 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:42:55.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:55.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:42:55.828 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:55.855 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:55.873 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:55.900 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:55.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:55.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:56.011 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:42:56.042 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:56.160 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:42:56.162 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:42:56.624 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:56 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/98915885' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:56.626 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:56 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/98915885' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:56.685 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:42:56.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:56.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:42:56.788 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:56 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/98915885' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:42:56.817 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:57.213 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:57.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:57.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:57.317 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:57.344 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:57.390 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:57.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:57.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:57.496 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:57.522 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:57.566 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:57.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:57.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:57.618 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:57 smithi138 ceph-mon[192513]: from='client.46799 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:57.618 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:57 smithi138 ceph-mon[192513]: pgmap v1486: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 8328/223032 objects degraded (3.734%); 4549/223032 objects misplaced (2.040%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:57.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:57 smithi138 ceph-mon[185886]: from='client.46799 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:57.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:57 smithi138 ceph-mon[185886]: pgmap v1486: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 8328/223032 objects degraded (3.734%); 4549/223032 objects misplaced (2.040%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:57.672 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:42:57.702 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:57.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:57 smithi190 ceph-mon[160435]: from='client.46799 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:42:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:57 smithi190 ceph-mon[160435]: pgmap v1486: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 8328/223032 objects degraded (3.734%); 4549/223032 objects misplaced (2.040%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:58.122 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:42:58.123 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1485,"stamp":"2023-10-01T09:42:54.298838+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4893513675,"num_objects":74625,"num_object_clones":0,"num_object_copies":223875,"num_objects_missing_on_primary":3194,"num_objects_missing":3194,"num_objects_degraded":8381,"num_objects_misplaced":4579,"num_objects_unfound":0,"num_objects_dirty":74625,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":92949,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7904,"num_bytes_recovered":504857615,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93013,"ondisk_log_size":93013,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19876024,"kb_used_data":14839104,"kb_used_omap":39,"kb_used_meta":5036824,"kb_avail":636340040,"statfs":{"total":671965249536,"available":651612200960,"internally_reserved":0,"allocated":15195242496,"data_stored":15185455549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5157708399},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-83689472,"num_objects":-1277,"num_object_clones":0,"num_object_copies":-3831,"num_objects_missing_on_primary":-75,"num_objects_missing":-75,"num_objects_degraded":-67,"num_objects_misplaced":-81,"num_objects_unfound":0,"num_objects_dirty":-1277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1277,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.847632"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":293,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.459577+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:42:37.459577+0000","last_peered":"2023-10-01T09:42:37.459577+0000","last_clean":"2023-10-01T09:42:37.459577+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:42:37.459577+0000","last_undegraded":"2023-10-01T09:42:37.459577+0000","last_fullsized":"2023-10-01T09:42:37.459577+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":136,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:44.497929+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:42:44.497929+0000","last_peered":"2023-10-01T09:42:44.497929+0000","last_clean":"2023-10-01T09:42:44.497929+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:42:44.497929+0000","last_undegraded":"2023-10-01T09:42:44.497929+0000","last_fullsized":"2023-10-01T09:42:44.497929+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":516,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.464050+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:42:37.464050+0000","last_peered":"2023-10-01T09:42:37.464050+0000","last_clean":"2023-10-01T09:42:37.464050+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:42:37.464050+0000","last_undegraded":"2023-10-01T09:42:37.464050+0000","last_fullsized":"2023-10-01T09:42:37.464050+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":575,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:44.458529+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:42:44.458529+0000","last_peered":"2023-10-01T09:42:44.458529+0000","last_clean":"2023-10-01T09:42:44.458529+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:42:44.458529+0000","last_undegraded":"2023-10-01T09:42:44.458529+0000","last_fullsized":"2023-10-01T09:42:44.458529+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":457,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.816828+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:42:51.816828+0000","last_peered":"2023-10-01T09:42:51.816828+0000","last_clean":"2023-10-01T09:42:51.816828+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:42:51.816828+0000","last_undegraded":"2023-10-01T09:42:51.816828+0000","last_fullsized":"2023-10-01T09:42:51.816828+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":198,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.729943+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:42:37.729943+0000","last_peered":"2023-10-01T09:42:37.729943+0000","last_clean":"2023-10-01T09:42:37.729943+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:42:37.729943+0000","last_undegraded":"2023-10-01T09:42:37.729943+0000","last_fullsized":"2023-10-01T09:42:37.729943+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":574,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.730005+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:42:37.730005+0000","last_peered":"2023-10-01T09:42:37.730005+0000","last_clean":"2023-10-01T09:42:37.730005+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:42:37.730005+0000","last_undegraded":"2023-10-01T09:42:37.730005+0000","last_fullsized":"2023-10-01T09:42:37.730005+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":504,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:47.455853+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:42:47.455853+0000","last_peered":"2023-10-01T09:42:47.455853+0000","last_clean":"2023-10-01T09:42:47.455853+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:42:47.455853+0000","last_undegraded":"2023-10-01T09:42:47.455853+0000","last_fullsized":"2023-10-01T09:42:47.455853+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":229,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.430551+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:42:37.430551+0000","last_peered":"2023-10-01T09:42:37.430551+0000","last_clean":"2023-10-01T09:42:37.430551+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:42:37.430551+0000","last_undegraded":"2023-10-01T09:42:37.430551+0000","last_fullsized":"2023-10-01T09:42:37.430551+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":36,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.430627+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:42:37.430627+0000","last_peered":"2023-10-01T09:42:37.430627+0000","last_clean":"2023-10-01T09:42:37.430627+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:42:37.430627+0000","last_undegraded":"2023-10-01T09:42:37.430627+0000","last_fullsized":"2023-10-01T09:42:37.430627+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":42,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.431131+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:42:37.431131+0000","last_peered":"2023-10-01T09:42:37.431131+0000","last_clean":"2023-10-01T09:42:37.431131+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:42:37.431131+0000","last_undegraded":"2023-10-01T09:42:37.431131+0000","last_fullsized":"2023-10-01T09:42:37.431131+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":39,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.464606+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:42:37.464606+0000","last_peered":"2023-10-01T09:42:37.464606+0000","last_clean":"2023-10-01T09:42:37.464606+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:42:37.464606+0000","last_undegraded":"2023-10-01T09:42:37.464606+0000","last_fullsized":"2023-10-01T09:42:37.464606+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":418,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.459764+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:42:37.459764+0000","last_peered":"2023-10-01T09:42:37.459764+0000","last_clean":"2023-10-01T09:42:37.459764+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:42:37.459764+0000","last_undegraded":"2023-10-01T09:42:37.459764+0000","last_fullsized":"2023-10-01T09:42:37.459764+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":299,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.459728+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:42:37.459728+0000","last_peered":"2023-10-01T09:42:37.459728+0000","last_clean":"2023-10-01T09:42:37.459728+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:42:37.459728+0000","last_undegraded":"2023-10-01T09:42:37.459728+0000","last_fullsized":"2023-10-01T09:42:37.459728+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":635,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.430988+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:42:37.430988+0000","last_peered":"2023-10-01T09:42:37.430988+0000","last_clean":"2023-10-01T09:42:37.430988+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:42:37.430988+0000","last_undegraded":"2023-10-01T09:42:37.430988+0000","last_fullsized":"2023-10-01T09:42:37.430988+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":372,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.431286+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:42:37.431286+0000","last_peered":"2023-10-01T09:42:37.431286+0000","last_clean":"2023-10-01T09:42:37.431286+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:42:37.431286+0000","last_undegraded":"2023-10-01T09:42:37.431286+0000","last_fullsized":"2023-10-01T09:42:37.431286+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":45,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.430929+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:42:37.430929+0000","last_peered":"2023-10-01T09:42:37.430929+0000","last_clean":"2023-10-01T09:42:37.430929+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:42:37.430929+0000","last_undegraded":"2023-10-01T09:42:37.430929+0000","last_fullsized":"2023-10-01T09:42:37.430929+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":31,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.431630+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:42:37.431630+0000","last_peered":"2023-10-01T09:42:37.431630+0000","last_clean":"2023-10-01T09:42:37.431630+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:42:37.431630+0000","last_undegraded":"2023-10-01T09:42:37.431630+0000","last_fullsized":"2023-10-01T09:42:37.431630+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":528,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.430846+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:42:37.430846+0000","last_peered":"2023-10-01T09:42:37.430846+0000","last_clean":"2023-10-01T09:42:37.430846+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:42:37.430846+0000","last_undegraded":"2023-10-01T09:42:37.430846+0000","last_fullsized":"2023-10-01T09:42:37.430846+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":540,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:44.458649+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:42:44.458649+0000","last_peered":"2023-10-01T09:42:44.458649+0000","last_clean":"2023-10-01T09:42:44.458649+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:42:44.458649+0000","last_undegraded":"2023-10-01T09:42:44.458649+0000","last_fullsized":"2023-10-01T09:42:44.458649+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":396,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:44.458744+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:42:44.458744+0000","last_peered":"2023-10-01T09:42:44.458744+0000","last_clean":"2023-10-01T09:42:44.458744+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:42:44.458744+0000","last_undegraded":"2023-10-01T09:42:44.458744+0000","last_fullsized":"2023-10-01T09:42:44.458744+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":45,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.436680+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:42:37.436680+0000","last_peered":"2023-10-01T09:42:37.436680+0000","last_clean":"2023-10-01T09:42:37.436680+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:42:37.436680+0000","last_undegraded":"2023-10-01T09:42:37.436680+0000","last_fullsized":"2023-10-01T09:42:37.436680+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"517'5750","reported_seq":5800,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:50.464951+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:42:50.464951+0000","last_peered":"2023-10-01T09:42:50.464951+0000","last_clean":"2023-10-01T09:42:50.464951+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:42:50.464951+0000","last_undegraded":"2023-10-01T09:42:50.464951+0000","last_fullsized":"2023-10-01T09:42:50.464951+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5750,"log_dups_size":0,"ondisk_log_size":5750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304218112,"num_objects":4642,"num_object_clones":0,"num_object_copies":13926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5750,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":40,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.436580+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:42:37.436580+0000","last_peered":"2023-10-01T09:42:37.436580+0000","last_clean":"2023-10-01T09:42:37.436580+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:42:37.436580+0000","last_undegraded":"2023-10-01T09:42:37.436580+0000","last_fullsized":"2023-10-01T09:42:37.436580+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":237,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.428897+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:42:37.428897+0000","last_peered":"2023-10-01T09:42:37.428897+0000","last_clean":"2023-10-01T09:42:37.428897+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:42:37.428897+0000","last_undegraded":"2023-10-01T09:42:37.428897+0000","last_fullsized":"2023-10-01T09:42:37.428897+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"517'5692","reported_seq":7242,"reported_epoch":517,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:42:50.464850+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:42:50.464850+0000","last_peered":"2023-10-01T09:42:50.464850+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:42:50.464850+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5692,"log_dups_size":0,"ondisk_log_size":5692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308936704,"num_objects":4714,"num_object_clones":0,"num_object_copies":14142,"num_objects_missing_on_primary":3194,"num_objects_missing":3194,"num_objects_degraded":7839,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5692,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":490,"num_bytes_recovered":32112640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1451},{"shards":"0,4,6","objects":3194},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":511,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.436569+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:42:37.436569+0000","last_peered":"2023-10-01T09:42:37.436569+0000","last_clean":"2023-10-01T09:42:37.436569+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:42:37.436569+0000","last_undegraded":"2023-10-01T09:42:37.436569+0000","last_fullsized":"2023-10-01T09:42:37.436569+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":572,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:44.497628+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:42:44.497628+0000","last_peered":"2023-10-01T09:42:44.497628+0000","last_clean":"2023-10-01T09:42:44.497628+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:42:44.497628+0000","last_undegraded":"2023-10-01T09:42:44.497628+0000","last_fullsized":"2023-10-01T09:42:44.497628+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"517'5760","reported_seq":5813,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.669151+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:42:51.669151+0000","last_peered":"2023-10-01T09:42:51.669151+0000","last_clean":"2023-10-01T09:42:51.669151+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:42:51.669151+0000","last_undegraded":"2023-10-01T09:42:51.669151+0000","last_fullsized":"2023-10-01T09:42:51.669151+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5760,"log_dups_size":0,"ondisk_log_size":5760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306184192,"num_objects":4672,"num_object_clones":0,"num_object_copies":14016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5760,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":597,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.431270+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:42:37.431270+0000","last_peered":"2023-10-01T09:42:37.431270+0000","last_clean":"2023-10-01T09:42:37.431270+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:42:37.431270+0000","last_undegraded":"2023-10-01T09:42:37.431270+0000","last_fullsized":"2023-10-01T09:42:37.431270+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":392,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.464799+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:42:37.464799+0000","last_peered":"2023-10-01T09:42:37.464799+0000","last_clean":"2023-10-01T09:42:37.464799+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:42:37.464799+0000","last_undegraded":"2023-10-01T09:42:37.464799+0000","last_fullsized":"2023-10-01T09:42:37.464799+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"517'5709","reported_seq":12367,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.698084+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:42:51.698084+0000","last_peered":"2023-10-01T09:42:51.698084+0000","last_clean":"2023-10-01T09:42:51.698084+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:42:51.698084+0000","last_undegraded":"2023-10-01T09:42:51.698084+0000","last_fullsized":"2023-10-01T09:42:51.698084+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5709,"log_dups_size":0,"ondisk_log_size":5709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307560448,"num_objects":4693,"num_object_clones":0,"num_object_copies":14079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5709,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":237,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.431486+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:42:37.431486+0000","last_peered":"2023-10-01T09:42:37.431486+0000","last_clean":"2023-10-01T09:42:37.431486+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:42:37.431486+0000","last_undegraded":"2023-10-01T09:42:37.431486+0000","last_fullsized":"2023-10-01T09:42:37.431486+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":496,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:44.458547+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:42:44.458547+0000","last_peered":"2023-10-01T09:42:44.458547+0000","last_clean":"2023-10-01T09:42:44.458547+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:42:44.458547+0000","last_undegraded":"2023-10-01T09:42:44.458547+0000","last_fullsized":"2023-10-01T09:42:44.458547+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"517'5769","reported_seq":5821,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:50.134036+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:42:50.134036+0000","last_peered":"2023-10-01T09:42:50.134036+0000","last_clean":"2023-10-01T09:42:50.134036+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:42:50.134036+0000","last_undegraded":"2023-10-01T09:42:50.134036+0000","last_fullsized":"2023-10-01T09:42:50.134036+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5769,"log_dups_size":0,"ondisk_log_size":5769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310575104,"num_objects":4739,"num_object_clones":0,"num_object_copies":14217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5769,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":97,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.459911+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:42:37.459911+0000","last_peered":"2023-10-01T09:42:37.459911+0000","last_clean":"2023-10-01T09:42:37.459911+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:42:37.459911+0000","last_undegraded":"2023-10-01T09:42:37.459911+0000","last_fullsized":"2023-10-01T09:42:37.459911+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":171,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.459897+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:42:37.459897+0000","last_peered":"2023-10-01T09:42:37.459897+0000","last_clean":"2023-10-01T09:42:37.459897+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:42:37.459897+0000","last_undegraded":"2023-10-01T09:42:37.459897+0000","last_fullsized":"2023-10-01T09:42:37.459897+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"517'5725","reported_seq":5803,"reported_epoch":517,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:42:50.231419+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:42:50.231419+0000","last_peered":"2023-10-01T09:42:50.231419+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:42:50.231419+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5725,"log_dups_size":0,"ondisk_log_size":5725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304283648,"num_objects":4643,"num_object_clones":0,"num_object_copies":13929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":532,"num_objects_misplaced":4579,"num_objects_unfound":0,"num_objects_dirty":4643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5725,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4579},{"shards":"4,5","objects":-468}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":415,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.459729+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:42:37.459729+0000","last_peered":"2023-10-01T09:42:37.459729+0000","last_clean":"2023-10-01T09:42:37.459729+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:42:37.459729+0000","last_undegraded":"2023-10-01T09:42:37.459729+0000","last_fullsized":"2023-10-01T09:42:37.459729+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":575,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.464260+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:42:37.464260+0000","last_peered":"2023-10-01T09:42:37.464260+0000","last_clean":"2023-10-01T09:42:37.464260+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:42:37.464260+0000","last_undegraded":"2023-10-01T09:42:37.464260+0000","last_fullsized":"2023-10-01T09:42:37.464260+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"518'5736","reported_seq":5796,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:52.489794+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:42:52.489794+0000","last_peered":"2023-10-01T09:42:52.489794+0000","last_clean":"2023-10-01T09:42:52.489794+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:42:52.489794+0000","last_undegraded":"2023-10-01T09:42:52.489794+0000","last_fullsized":"2023-10-01T09:42:52.489794+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5736,"log_dups_size":0,"ondisk_log_size":5736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304218112,"num_objects":4642,"num_object_clones":0,"num_object_copies":13926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5736,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":411,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.459446+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:42:37.459446+0000","last_peered":"2023-10-01T09:42:37.459446+0000","last_clean":"2023-10-01T09:42:37.459446+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:42:37.459446+0000","last_undegraded":"2023-10-01T09:42:37.459446+0000","last_fullsized":"2023-10-01T09:42:37.459446+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":252,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.460516+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:42:37.460516+0000","last_peered":"2023-10-01T09:42:37.460516+0000","last_clean":"2023-10-01T09:42:37.460516+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:42:37.460516+0000","last_undegraded":"2023-10-01T09:42:37.460516+0000","last_fullsized":"2023-10-01T09:42:37.460516+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"517'5820","reported_seq":5870,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:50.489810+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:42:50.489810+0000","last_peered":"2023-10-01T09:42:50.489810+0000","last_clean":"2023-10-01T09:42:50.489810+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:42:50.489810+0000","last_undegraded":"2023-10-01T09:42:50.489810+0000","last_fullsized":"2023-10-01T09:42:50.489810+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5820,"log_dups_size":0,"ondisk_log_size":5820,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306577408,"num_objects":4678,"num_object_clones":0,"num_object_copies":14034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5820,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":379,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.464035+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:42:37.464035+0000","last_peered":"2023-10-01T09:42:37.464035+0000","last_clean":"2023-10-01T09:42:37.464035+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:42:37.464035+0000","last_undegraded":"2023-10-01T09:42:37.464035+0000","last_fullsized":"2023-10-01T09:42:37.464035+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":426,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.464264+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:42:37.464264+0000","last_peered":"2023-10-01T09:42:37.464264+0000","last_clean":"2023-10-01T09:42:37.464264+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:42:37.464264+0000","last_undegraded":"2023-10-01T09:42:37.464264+0000","last_fullsized":"2023-10-01T09:42:37.464264+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"517'5640","reported_seq":5690,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:50.600272+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:42:50.600272+0000","last_peered":"2023-10-01T09:42:50.600272+0000","last_clean":"2023-10-01T09:42:50.600272+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:42:50.600272+0000","last_undegraded":"2023-10-01T09:42:50.600272+0000","last_fullsized":"2023-10-01T09:42:50.600272+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5640,"log_dups_size":0,"ondisk_log_size":5640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297402368,"num_objects":4538,"num_object_clones":0,"num_object_copies":13614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5640,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":910,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.431145+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:42:37.431145+0000","last_peered":"2023-10-01T09:42:37.431145+0000","last_clean":"2023-10-01T09:42:37.431145+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:42:37.431145+0000","last_undegraded":"2023-10-01T09:42:37.431145+0000","last_fullsized":"2023-10-01T09:42:37.431145+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":247,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.431708+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:42:37.431708+0000","last_peered":"2023-10-01T09:42:37.431708+0000","last_clean":"2023-10-01T09:42:37.431708+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:42:37.431708+0000","last_undegraded":"2023-10-01T09:42:37.431708+0000","last_fullsized":"2023-10-01T09:42:37.431708+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":40,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:44.497646+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:42:44.497646+0000","last_peered":"2023-10-01T09:42:44.497646+0000","last_clean":"2023-10-01T09:42:44.497646+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:42:44.497646+0000","last_undegraded":"2023-10-01T09:42:44.497646+0000","last_fullsized":"2023-10-01T09:42:44.497646+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"517'5717","reported_seq":5770,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.753758+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:42:51.753758+0000","last_peered":"2023-10-01T09:42:51.753758+0000","last_clean":"2023-10-01T09:42:51.753758+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:42:51.753758+0000","last_undegraded":"2023-10-01T09:42:51.753758+0000","last_fullsized":"2023-10-01T09:42:51.753758+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5717,"log_dups_size":0,"ondisk_log_size":5717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304939008,"num_objects":4653,"num_object_clones":0,"num_object_copies":13959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5717,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1182,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.429265+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:42:37.429265+0000","last_peered":"2023-10-01T09:42:37.429265+0000","last_clean":"2023-10-01T09:42:37.429265+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:42:37.429265+0000","last_undegraded":"2023-10-01T09:42:37.429265+0000","last_fullsized":"2023-10-01T09:42:37.429265+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":392,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.459829+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:42:37.459829+0000","last_peered":"2023-10-01T09:42:37.459829+0000","last_clean":"2023-10-01T09:42:37.459829+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:42:37.459829+0000","last_undegraded":"2023-10-01T09:42:37.459829+0000","last_fullsized":"2023-10-01T09:42:37.459829+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":220,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.431575+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:42:37.431575+0000","last_peered":"2023-10-01T09:42:37.431575+0000","last_clean":"2023-10-01T09:42:37.431575+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:42:37.431575+0000","last_undegraded":"2023-10-01T09:42:37.431575+0000","last_fullsized":"2023-10-01T09:42:37.431575+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"517'5734","reported_seq":13005,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:50.415326+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:42:50.415326+0000","last_peered":"2023-10-01T09:42:50.415326+0000","last_clean":"2023-10-01T09:42:50.415326+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:42:50.415326+0000","last_undegraded":"2023-10-01T09:42:50.415326+0000","last_fullsized":"2023-10-01T09:42:50.415326+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5734,"log_dups_size":0,"ondisk_log_size":5734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304218112,"num_objects":4642,"num_object_clones":0,"num_object_copies":13926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5734,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":862,"reported_epoch":517,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:42:44.458027+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:42:44.458027+0000","last_peered":"2023-10-01T09:42:44.458027+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:42:44.458027+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:42:44.458027+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":579,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.465017+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:42:37.465017+0000","last_peered":"2023-10-01T09:42:37.465017+0000","last_clean":"2023-10-01T09:42:37.465017+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:42:37.465017+0000","last_undegraded":"2023-10-01T09:42:37.465017+0000","last_fullsized":"2023-10-01T09:42:37.465017+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":39,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.464158+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:42:37.464158+0000","last_peered":"2023-10-01T09:42:37.464158+0000","last_clean":"2023-10-01T09:42:37.464158+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:42:37.464158+0000","last_undegraded":"2023-10-01T09:42:37.464158+0000","last_fullsized":"2023-10-01T09:42:37.464158+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"517'5643","reported_seq":5695,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:50.244704+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:42:50.244704+0000","last_peered":"2023-10-01T09:42:50.244704+0000","last_clean":"2023-10-01T09:42:50.244704+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:42:50.244704+0000","last_undegraded":"2023-10-01T09:42:50.244704+0000","last_fullsized":"2023-10-01T09:42:50.244704+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5643,"log_dups_size":0,"ondisk_log_size":5643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303366144,"num_objects":4629,"num_object_clones":0,"num_object_copies":13887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5643,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1633,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.431878+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:42:37.431878+0000","last_peered":"2023-10-01T09:42:37.431878+0000","last_clean":"2023-10-01T09:42:37.431878+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:42:37.431878+0000","last_undegraded":"2023-10-01T09:42:37.431878+0000","last_fullsized":"2023-10-01T09:42:37.431878+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":547,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.430634+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:42:37.430634+0000","last_peered":"2023-10-01T09:42:37.430634+0000","last_clean":"2023-10-01T09:42:37.430634+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:42:37.430634+0000","last_undegraded":"2023-10-01T09:42:37.430634+0000","last_fullsized":"2023-10-01T09:42:37.430634+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":325,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.429360+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:42:37.429360+0000","last_peered":"2023-10-01T09:42:37.429360+0000","last_clean":"2023-10-01T09:42:37.429360+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:42:37.429360+0000","last_undegraded":"2023-10-01T09:42:37.429360+0000","last_fullsized":"2023-10-01T09:42:37.429360+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":315,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.461613+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:42:37.461613+0000","last_peered":"2023-10-01T09:42:37.461613+0000","last_clean":"2023-10-01T09:42:37.461613+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:42:37.461613+0000","last_undegraded":"2023-10-01T09:42:37.461613+0000","last_fullsized":"2023-10-01T09:42:37.461613+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"517'5802","reported_seq":5852,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:50.441223+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:42:50.441223+0000","last_peered":"2023-10-01T09:42:50.441223+0000","last_clean":"2023-10-01T09:42:50.441223+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:42:50.441223+0000","last_undegraded":"2023-10-01T09:42:50.441223+0000","last_fullsized":"2023-10-01T09:42:50.441223+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5802,"log_dups_size":0,"ondisk_log_size":5802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309985280,"num_objects":4730,"num_object_clones":0,"num_object_copies":14190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5802,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":724,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.430722+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:42:37.430722+0000","last_peered":"2023-10-01T09:42:37.430722+0000","last_clean":"2023-10-01T09:42:37.430722+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:42:37.430722+0000","last_undegraded":"2023-10-01T09:42:37.430722+0000","last_fullsized":"2023-10-01T09:42:37.430722+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":557,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:44.458322+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:42:44.458322+0000","last_peered":"2023-10-01T09:42:44.458322+0000","last_clean":"2023-10-01T09:42:44.458322+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:42:44.458322+0000","last_undegraded":"2023-10-01T09:42:44.458322+0000","last_fullsized":"2023-10-01T09:42:44.458322+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":556,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:44.458427+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:42:44.458427+0000","last_peered":"2023-10-01T09:42:44.458427+0000","last_clean":"2023-10-01T09:42:44.458427+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:42:44.458427+0000","last_undegraded":"2023-10-01T09:42:44.458427+0000","last_fullsized":"2023-10-01T09:42:44.458427+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"517'5791","reported_seq":5845,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.700118+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:42:51.700118+0000","last_peered":"2023-10-01T09:42:51.700118+0000","last_clean":"2023-10-01T09:42:51.700118+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:42:51.700118+0000","last_undegraded":"2023-10-01T09:42:51.700118+0000","last_fullsized":"2023-10-01T09:42:51.700118+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5791,"log_dups_size":0,"ondisk_log_size":5791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305922072,"num_objects":4669,"num_object_clones":0,"num_object_copies":14007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4669,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5791,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":955,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.431222+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:42:37.431222+0000","last_peered":"2023-10-01T09:42:37.431222+0000","last_clean":"2023-10-01T09:42:37.431222+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:42:37.431222+0000","last_undegraded":"2023-10-01T09:42:37.431222+0000","last_fullsized":"2023-10-01T09:42:37.431222+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":574,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.431531+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:42:37.431531+0000","last_peered":"2023-10-01T09:42:37.431531+0000","last_clean":"2023-10-01T09:42:37.431531+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:42:37.431531+0000","last_undegraded":"2023-10-01T09:42:37.431531+0000","last_fullsized":"2023-10-01T09:42:37.431531+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":427,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.431956+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:42:37.431956+0000","last_peered":"2023-10-01T09:42:37.431956+0000","last_clean":"2023-10-01T09:42:37.431956+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:42:37.431956+0000","last_undegraded":"2023-10-01T09:42:37.431956+0000","last_fullsized":"2023-10-01T09:42:37.431956+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"517'5756","reported_seq":5814,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:50.607714+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:42:50.607714+0000","last_peered":"2023-10-01T09:42:50.607714+0000","last_clean":"2023-10-01T09:42:50.607714+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:42:50.607714+0000","last_undegraded":"2023-10-01T09:42:50.607714+0000","last_fullsized":"2023-10-01T09:42:50.607714+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5756,"log_dups_size":0,"ondisk_log_size":5756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301465600,"num_objects":4600,"num_object_clones":0,"num_object_copies":13800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5756,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":456,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.436790+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:42:37.436790+0000","last_peered":"2023-10-01T09:42:37.436790+0000","last_clean":"2023-10-01T09:42:37.436790+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:42:37.436790+0000","last_undegraded":"2023-10-01T09:42:37.436790+0000","last_fullsized":"2023-10-01T09:42:37.436790+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1149,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.464528+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:42:37.464528+0000","last_peered":"2023-10-01T09:42:37.464528+0000","last_clean":"2023-10-01T09:42:37.464528+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:42:37.464528+0000","last_undegraded":"2023-10-01T09:42:37.464528+0000","last_fullsized":"2023-10-01T09:42:37.464528+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":45,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.430595+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:42:37.430595+0000","last_peered":"2023-10-01T09:42:37.430595+0000","last_clean":"2023-10-01T09:42:37.430595+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:42:37.430595+0000","last_undegraded":"2023-10-01T09:42:37.430595+0000","last_fullsized":"2023-10-01T09:42:37.430595+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"517'5799","reported_seq":5857,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:50.010805+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:42:50.010805+0000","last_peered":"2023-10-01T09:42:50.010805+0000","last_clean":"2023-10-01T09:42:50.010805+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:42:50.010805+0000","last_undegraded":"2023-10-01T09:42:50.010805+0000","last_fullsized":"2023-10-01T09:42:50.010805+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5799,"log_dups_size":0,"ondisk_log_size":5799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308871168,"num_objects":4713,"num_object_clones":0,"num_object_copies":14139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5799,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":49,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.436695+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:42:37.436695+0000","last_peered":"2023-10-01T09:42:37.436695+0000","last_clean":"2023-10-01T09:42:37.436695+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:42:37.436695+0000","last_undegraded":"2023-10-01T09:42:37.436695+0000","last_fullsized":"2023-10-01T09:42:37.436695+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":534,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:44.458483+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:42:44.458483+0000","last_peered":"2023-10-01T09:42:44.458483+0000","last_clean":"2023-10-01T09:42:44.458483+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:42:44.458483+0000","last_undegraded":"2023-10-01T09:42:44.458483+0000","last_fullsized":"2023-10-01T09:42:44.458483+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":240,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.429045+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:42:37.429045+0000","last_peered":"2023-10-01T09:42:37.429045+0000","last_clean":"2023-10-01T09:42:37.429045+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:42:37.429045+0000","last_undegraded":"2023-10-01T09:42:37.429045+0000","last_fullsized":"2023-10-01T09:42:37.429045+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":483,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:44.458378+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:42:44.458378+0000","last_peered":"2023-10-01T09:42:44.458378+0000","last_clean":"2023-10-01T09:42:44.458378+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:42:44.458378+0000","last_undegraded":"2023-10-01T09:42:44.458378+0000","last_fullsized":"2023-10-01T09:42:44.458378+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":596,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.464300+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:42:37.464300+0000","last_peered":"2023-10-01T09:42:37.464300+0000","last_clean":"2023-10-01T09:42:37.464300+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:42:37.464300+0000","last_undegraded":"2023-10-01T09:42:37.464300+0000","last_fullsized":"2023-10-01T09:42:37.464300+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":252,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.431432+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:42:37.431432+0000","last_peered":"2023-10-01T09:42:37.431432+0000","last_clean":"2023-10-01T09:42:37.431432+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:42:37.431432+0000","last_undegraded":"2023-10-01T09:42:37.431432+0000","last_fullsized":"2023-10-01T09:42:37.431432+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":39,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.465244+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:42:37.465244+0000","last_peered":"2023-10-01T09:42:37.465244+0000","last_clean":"2023-10-01T09:42:37.465244+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:42:37.465244+0000","last_undegraded":"2023-10-01T09:42:37.465244+0000","last_fullsized":"2023-10-01T09:42:37.465244+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":252,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.431753+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:42:37.431753+0000","last_peered":"2023-10-01T09:42:37.431753+0000","last_clean":"2023-10-01T09:42:37.431753+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:42:37.431753+0000","last_undegraded":"2023-10-01T09:42:37.431753+0000","last_fullsized":"2023-10-01T09:42:37.431753+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":429,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:44.458403+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:42:44.458403+0000","last_peered":"2023-10-01T09:42:44.458403+0000","last_clean":"2023-10-01T09:42:44.458403+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:42:44.458403+0000","last_undegraded":"2023-10-01T09:42:44.458403+0000","last_fullsized":"2023-10-01T09:42:44.458403+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":57,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.464371+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:42:37.464371+0000","last_peered":"2023-10-01T09:42:37.464371+0000","last_clean":"2023-10-01T09:42:37.464371+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:42:37.464371+0000","last_undegraded":"2023-10-01T09:42:37.464371+0000","last_fullsized":"2023-10-01T09:42:37.464371+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":475,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.816882+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:42:51.816882+0000","last_peered":"2023-10-01T09:42:51.816882+0000","last_clean":"2023-10-01T09:42:51.816882+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:42:51.816882+0000","last_undegraded":"2023-10-01T09:42:51.816882+0000","last_fullsized":"2023-10-01T09:42:51.816882+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":674,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-10-01T09:42:44.458813+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:42:44.458813+0000","last_peered":"2023-10-01T09:42:44.458813+0000","last_clean":"2023-10-01T09:42:44.458813+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:42:44.458813+0000","last_undegraded":"2023-10-01T09:42:44.458813+0000","last_fullsized":"2023-10-01T09:42:44.458813+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":368,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-10-01T09:42:37.459313+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:42:37.459313+0000","last_peered":"2023-10-01T09:42:37.459313+0000","last_clean":"2023-10-01T09:42:37.459313+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:42:37.459313+0000","last_undegraded":"2023-10-01T09:42:37.459313+0000","last_fullsized":"2023-10-01T09:42:37.459313+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4888723480,"num_objects":74597,"num_object_clones":0,"num_object_copies":223791,"num_objects_missing_on_primary":3194,"num_objects_missing":3194,"num_objects_degraded":8371,"num_objects_misplaced":4579,"num_objects_unfound":0,"num_objects_dirty":74597,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":91843,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7393,"num_bytes_recovered":484507648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15133913088,"data_stored":15133900872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91843,"ondisk_log_size":91843,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430707,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2399096,"kb_used_data":1500080,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":91346056,"statfs":{"total":95995035648,"available":93538361344,"internally_reserved":0,"allocated":1536081920,"data_stored":1535147132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.766,"15min":3.0510000000000002},"min":{"1min":0.30099999999999999,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":6.8550000000000004,"5min":11.525,"15min":108.90000000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.82099999999999995,"5min":2.2130000000000001,"15min":3.3359999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.6639999999999997,"5min":29.501000000000001,"15min":109.187},"last":0.46999999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:42:06 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":5.0410000000000004,"15min":5.492},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":6.8289999999999997,"5min":229.161,"15min":229.161},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.92100000000000004,"5min":4.9939999999999998,"15min":5.5490000000000004},"min":{"1min":0.28899999999999998,"5min":0.27500000000000002,"15min":0.24399999999999999},"max":{"1min":5.7240000000000002,"5min":230.126,"15min":230.126},"last":0.50800000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:42:50 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":1.5369999999999999,"15min":1.891},"min":{"1min":0.33900000000000002,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":1.3560000000000001,"5min":14.866,"15min":14.901999999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.49299999999999999,"5min":1.7909999999999999,"15min":1.821},"min":{"1min":0.33500000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.96599999999999997,"5min":14.682,"15min":14.682},"last":0.67600000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:42:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.9410000000000001,"15min":2.8610000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":5.7750000000000004,"5min":35.104999999999997,"15min":80.352999999999994},"last":0.64700000000000002},{"interface":"front","average":{"1min":1.004,"5min":2,"15min":2.8359999999999999},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.548,"5min":35.118000000000002,"15min":80.364000000000004},"last":0.45500000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.202,"15min":1.0529999999999999},"min":{"1min":0.30499999999999999,"5min":0.27600000000000002,"15min":0.254},"max":{"1min":6.8769999999999998,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.4710000000000001,"5min":1.1739999999999999,"15min":1.0369999999999999},"min":{"1min":0.33900000000000002,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":5.7439999999999998,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.59299999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":2.8879999999999999,"15min":1.6419999999999999},"min":{"1min":0.318,"5min":0.30299999999999999,"15min":0.29599999999999999},"max":{"1min":0.64600000000000002,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.41999999999999998,"5min":3.0779999999999998,"15min":1.6850000000000001},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54500000000000004,"5min":135.339,"15min":135.339},"last":0.53300000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.4379999999999999,"15min":1.173},"min":{"1min":0.313,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":2.448,"5min":11.58,"15min":11.853999999999999},"last":0.622},{"interface":"front","average":{"1min":0.44400000000000001,"5min":1.2789999999999999,"15min":1.1359999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":0.56699999999999995,"5min":14.584,"15min":14.584},"last":0.495}]}]},{"osd":6,"up_from":294,"seq":1262720385218,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2427068,"kb_used_data":2089460,"kb_used_omap":6,"kb_used_meta":337593,"kb_avail":91318084,"statfs":{"total":95995035648,"available":93509718016,"internally_reserved":0,"allocated":2139607040,"data_stored":2138675879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":345695277},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":3.786,"15min":4.1879999999999997},"min":{"1min":0.249,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":1.8400000000000001,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.76300000000000001,"5min":4.3559999999999999,"15min":4.2889999999999997},"min":{"1min":0.249,"5min":0.249,"15min":0.221},"max":{"1min":2.9969999999999999,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.52100000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":5.3860000000000001,"15min":5.0590000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":3.0289999999999999,"5min":168.947,"15min":168.947},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.79100000000000004,"5min":5.375,"15min":4.9690000000000003},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":2.6030000000000002,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.64100000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.8109999999999999,"15min":1.8420000000000001},"min":{"1min":0.34499999999999997,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":3.0489999999999999,"5min":9.657,"15min":26.289999999999999},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.885,"15min":1.859},"min":{"1min":0.32600000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.6739999999999999,"5min":21.704999999999998,"15min":26.388999999999999},"last":0.54200000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":1.597,"15min":2.4020000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.9450000000000001,"5min":9.0180000000000007,"15min":89.753},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.073,"5min":1.9330000000000001,"15min":2.2189999999999999},"min":{"1min":0.318,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":3.7160000000000002,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.46100000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.665,"15min":1.0800000000000001},"min":{"1min":0.28599999999999998,"5min":0.253,"15min":0.246},"max":{"1min":4.0250000000000004,"5min":63.134,"15min":63.134},"last":0.376},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.518,"15min":1.0960000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.28299999999999997},"max":{"1min":7.4039999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.66300000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:42:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.93200000000000005,"15min":1.02},"min":{"1min":0.26700000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.8620000000000001,"5min":9.9939999999999998,"15min":12.023999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.66500000000000004,"5min":1.175,"15min":1.143},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.726,"5min":15.457000000000001,"15min":15.457000000000001},"last":0.50600000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.82999999999999996,"15min":1.802},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":2.0419999999999998,"5min":6.4119999999999999,"15min":115.11},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.282,"15min":1.8959999999999999},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.28799999999999998},"max":{"1min":3.2679999999999998,"5min":8.6899999999999995,"15min":114.666},"last":0.68300000000000005}]}]},{"osd":1,"up_from":94,"seq":403726926194,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1722516,"kb_used_data":908428,"kb_used_omap":7,"kb_used_meta":814072,"kb_avail":92022636,"statfs":{"total":95995035648,"available":94231179264,"internally_reserved":0,"allocated":930230272,"data_stored":928216732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":833610151},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":3.5089999999999999,"15min":4.5229999999999997},"min":{"1min":0.19400000000000001,"5min":0.182,"15min":0.182},"max":{"1min":4.423,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.224},{"interface":"front","average":{"1min":1.528,"5min":4.0339999999999998,"15min":3.859},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":13.872,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.26700000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:42:38 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":1.7929999999999999,"15min":2.7719999999999998},"min":{"1min":0.24099999999999999,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.83099999999999996,"5min":22.891999999999999,"15min":56.573},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.36699999999999999,"5min":2.2799999999999998,"15min":3.593},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":0.54400000000000004,"5min":15.189,"15min":154.197},"last":0.35099999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":3.9089999999999998,"15min":4.4429999999999996},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":3.1949999999999998,"5min":95.305000000000007,"15min":200.56800000000001},"last":0.317},{"interface":"front","average":{"1min":0.68799999999999994,"5min":4.3550000000000004,"15min":5.2569999999999997},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":4.7839999999999998,"5min":98.033000000000001,"15min":200.63800000000001},"last":0.32300000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":2.4870000000000001,"15min":2.3940000000000001},"min":{"1min":0.30299999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.5510000000000002,"5min":21.015000000000001,"15min":154.101},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.60799999999999998,"5min":2.306,"15min":2.7410000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.4209999999999998,"5min":13.773999999999999,"15min":154.56999999999999},"last":0.35999999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:42:35 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.843,"15min":2.1240000000000001},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.2450000000000001,"5min":21.219999999999999,"15min":154.22999999999999},"last":0.505},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.6499999999999999,"15min":2.5649999999999999},"min":{"1min":0.316,"5min":0.307,"15min":0.307},"max":{"1min":2.407,"5min":10.305,"15min":154.54400000000001},"last":0.41499999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.1669999999999998,"15min":2.8370000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.3660000000000001,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.372},{"interface":"front","average":{"1min":0.626,"5min":2.4769999999999999,"15min":2.5899999999999999},"min":{"1min":0.36699999999999999,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":1.575,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.45100000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.617,"15min":2.7250000000000001},"min":{"1min":0.40500000000000003,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":5.2169999999999996,"5min":21.062999999999999,"15min":154.20699999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.88300000000000001,"5min":2.5390000000000001,"15min":2.9399999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":3.8260000000000001,"5min":13.819000000000001,"15min":154.51599999999999},"last":0.52300000000000002}]}]},{"osd":0,"up_from":89,"seq":382252089729,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4549072,"kb_used_data":3649736,"kb_used_omap":8,"kb_used_meta":899319,"kb_avail":89196080,"statfs":{"total":95995035648,"available":91336785920,"internally_reserved":0,"allocated":3737329664,"data_stored":3735325563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":920903618},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:42:08 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":3.9860000000000002,"15min":4.4039999999999999},"min":{"1min":0.20399999999999999,"5min":0.187,"15min":0.17399999999999999},"max":{"1min":1.661,"5min":185.78399999999999,"15min":216.821},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.437,"5min":5.9020000000000001,"15min":5.4400000000000004},"min":{"1min":0.249,"5min":0.215,"15min":0.215},"max":{"1min":1.5589999999999999,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.29399999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:42:36 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":1.8779999999999999,"15min":2.1579999999999999},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.20100000000000001},"max":{"1min":0.51200000000000001,"5min":19.788,"15min":19.788},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.36899999999999999,"5min":2.0960000000000001,"15min":2.5230000000000001},"min":{"1min":0.315,"5min":0.215,"15min":0.189},"max":{"1min":0.439,"5min":25.844999999999999,"15min":25.844999999999999},"last":0.38800000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:42:08 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":2.1749999999999998,"15min":4.524},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.20899999999999999},"max":{"1min":1.706,"5min":15.35,"15min":222.16499999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.873,"5min":2.5219999999999998,"15min":4.7089999999999996},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":2.859,"5min":14.545999999999999,"15min":222.227},"last":0.28199999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":2.3559999999999999,"15min":2.4260000000000002},"min":{"1min":0.29899999999999999,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":2.7480000000000002,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.034,"5min":2.698,"15min":2.536},"min":{"1min":0.376,"5min":0.33700000000000002,"15min":0.28299999999999997},"max":{"1min":3.1619999999999999,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.34799999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:42:30 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.837,"15min":1.9730000000000001},"min":{"1min":0.33700000000000002,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":1.9730000000000001,"5min":14.515000000000001,"15min":16.808},"last":0.313},{"interface":"front","average":{"1min":0.47199999999999998,"5min":1.6000000000000001,"15min":1.8480000000000001},"min":{"1min":0.34799999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64000000000000001,"5min":8.9740000000000002,"15min":17.507000000000001},"last":0.45800000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.617,"15min":1.8360000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.35499999999999998},"max":{"1min":1.5029999999999999,"5min":8.9920000000000009,"15min":17.535},"last":0.437},{"interface":"front","average":{"1min":0.54900000000000004,"5min":1.879,"15min":2.137},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":1.5760000000000001,"5min":10.446,"15min":18.521000000000001},"last":0.40699999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:41:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":2.198,"15min":2.1909999999999998},"min":{"1min":0.41799999999999998,"5min":0.377,"15min":0.34499999999999997},"max":{"1min":4.976,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.069,"5min":2.4740000000000002,"15min":2.5459999999999998},"min":{"1min":0.41199999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":4.226,"5min":48.274000000000001,"15min":48.274000000000001},"last":0.48499999999999999}]}]},{"osd":2,"up_from":0,"seq":425201762669,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599142,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1959628,"kb_used_data":1131588,"kb_used_omap":3,"kb_used_meta":828028,"kb_avail":91785524,"statfs":{"total":95995035648,"available":93988376576,"internally_reserved":0,"allocated":1158746112,"data_stored":1156689415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":847901415},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":5.9850000000000003,"5min":5.5869999999999997,"15min":6.0170000000000003},"min":{"1min":0.216,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":64.504000000000005,"5min":118.748,"15min":137.34399999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":6.024,"5min":5.7830000000000004,"15min":6.0090000000000003},"min":{"1min":0.24299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":64.448999999999998,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.23400000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":5.149,"15min":5.4130000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":6.0250000000000004,"5min":108.84999999999999,"15min":162.22999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.60199999999999998,"5min":5.3860000000000001,"15min":5.718},"min":{"1min":0.23100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.931,"5min":109.13200000000001,"15min":162.70699999999999},"last":0.28599999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":2.5070000000000001,"15min":2.5},"min":{"1min":0.27000000000000002,"5min":0.23000000000000001,"15min":0.20799999999999999},"max":{"1min":7.141,"5min":14.254,"15min":43.222999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.4359999999999999,"15min":2.823},"min":{"1min":0.28299999999999997,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":3.4550000000000001,"5min":11.202,"15min":78.594999999999999},"last":0.375}]},{"osd":4,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":2.423,"15min":1.972},"min":{"1min":0.312,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":6.8890000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.442},{"interface":"front","average":{"1min":0.96399999999999997,"5min":2.5800000000000001,"15min":2.202},"min":{"1min":0.33900000000000002,"5min":0.28699999999999998,"15min":0.27900000000000003},"max":{"1min":6.0940000000000003,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.40799999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.766,"15min":2.7850000000000001},"min":{"1min":0.32500000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.70499999999999996,"5min":10.919,"15min":28.117999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":2.0139999999999998,"15min":2.8879999999999999},"min":{"1min":0.35399999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.5580000000000001,"5min":12.989000000000001,"15min":28.34},"last":0.46700000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":2.7970000000000002,"15min":2.3159999999999998},"min":{"1min":0.34399999999999997,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":1.621,"5min":73.887,"15min":73.887},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.66200000000000003,"5min":2.9670000000000001,"15min":2.3090000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.32800000000000001},"max":{"1min":1.8440000000000001,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.58599999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":2.3839999999999999,"15min":2.4249999999999998},"min":{"1min":0.41499999999999998,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":7.0620000000000003,"5min":13.085000000000001,"15min":52.006},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.774,"15min":2.1360000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.2690000000000001,"5min":11.109999999999999,"15min":51.890999999999998},"last":0.47799999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019392,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4562164,"kb_used_data":3667884,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":89182988,"statfs":{"total":95995035648,"available":91323379712,"internally_reserved":0,"allocated":3755913216,"data_stored":3754981934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:41:53 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":3.5209999999999999,"15min":4.9050000000000002},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":9.6690000000000005,"5min":79.837000000000003,"15min":155.315},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.875,"5min":3.9239999999999999,"15min":5.1550000000000002},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.23899999999999999},"max":{"1min":6.1459999999999999,"5min":78.983000000000004,"15min":155.334},"last":0.41399999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":8.1189999999999998,"15min":7.0599999999999996},"min":{"1min":0.35399999999999998,"5min":0.224,"15min":0.224},"max":{"1min":5.944,"5min":146.929,"15min":190.61199999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.0660000000000001,"5min":8.0500000000000007,"15min":7.8029999999999999},"min":{"1min":0.38500000000000001,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":3.7469999999999999,"5min":146.91,"15min":253.273},"last":0.42599999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:42:43 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":1.2809999999999999,"15min":2.6019999999999999},"min":{"1min":0.34799999999999998,"5min":0.30399999999999999,"15min":0.28299999999999997},"max":{"1min":2.8210000000000002,"5min":10.161,"15min":170.45400000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.4410000000000001,"15min":2.2730000000000001},"min":{"1min":0.38400000000000001,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":2.1930000000000001,"5min":10.625,"15min":58.771000000000001},"last":0.374}]},{"osd":3,"last update":"Sun Oct 1 09:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.5600000000000001,"15min":2.931},"min":{"1min":0.33600000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.5279999999999996,"5min":15.989000000000001,"15min":140.881},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.825,"15min":3.145},"min":{"1min":0.313,"5min":0.311,"15min":0.311},"max":{"1min":3.786,"5min":15.978999999999999,"15min":140.911},"last":0.38200000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:42:31 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":2.1579999999999999,"15min":1.5900000000000001},"min":{"1min":0.26800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.1190000000000002,"5min":110.003,"15min":110.003},"last":0.318},{"interface":"front","average":{"1min":0.46000000000000002,"5min":2.069,"15min":1.405},"min":{"1min":0.317,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.0780000000000001,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.46700000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":0.90000000000000002,"15min":0.996},"min":{"1min":0.34599999999999997,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":9.9849999999999994,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.432},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.087,"15min":1.083},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":4.0220000000000002,"5min":6.9139999999999997,"15min":10.076000000000001},"last":0.44400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:42:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":1.159,"15min":1.1799999999999999},"min":{"1min":0.38100000000000001,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":1.825,"5min":38.034999999999997,"15min":38.034999999999997},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.623,"5min":1.47,"15min":1.462},"min":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":2.2149999999999999,"5min":37.723999999999997,"15min":37.723999999999997},"last":0.48199999999999998}]}]},{"osd":5,"up_from":379,"seq":1627792605299,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2256480,"kb_used_data":1891928,"kb_used_omap":13,"kb_used_meta":364530,"kb_avail":91488672,"statfs":{"total":95995035648,"available":93684400128,"internally_reserved":0,"allocated":1937334272,"data_stored":1936418894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":373279218},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":1.897,"15min":2.258},"min":{"1min":0.23200000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.50700000000000001,"5min":12.571999999999999,"15min":84.340999999999994},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.39500000000000002,"5min":1.5289999999999999,"15min":2.1349999999999998},"min":{"1min":0.27300000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60299999999999998,"5min":8.6460000000000008,"15min":84.356999999999999},"last":0.32900000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":6.7510000000000003,"15min":5.8460000000000001},"min":{"1min":0.26800000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.55400000000000005,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.41199999999999998,"5min":7.3490000000000002,"15min":5.7889999999999997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":0.63200000000000001,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.42799999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.6020000000000001,"15min":1.6519999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.56599999999999995,"5min":8.7040000000000006,"15min":14.199999999999999},"last":0.377},{"interface":"front","average":{"1min":0.47799999999999998,"5min":1.6220000000000001,"15min":1.661},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":1.931,"5min":8.2370000000000001,"15min":14.430999999999999},"last":0.443}]},{"osd":3,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":1.796,"15min":1.5269999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.8720000000000001,"5min":12.734,"15min":14.004},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":1.8580000000000001,"15min":1.673},"min":{"1min":0.39800000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.8819999999999999,"5min":11.766,"15min":13.991},"last":0.46400000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":2.2200000000000002,"15min":1.4670000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.873,"5min":122.971,"15min":122.971},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":2.3140000000000001,"15min":1.806},"min":{"1min":0.307,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.78400000000000003,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.48299999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":1.2849999999999999,"15min":1.0780000000000001},"min":{"1min":0.34499999999999997,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":0.68200000000000005,"5min":12.545,"15min":12.545},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.998,"15min":1.3640000000000001},"min":{"1min":0.34100000000000003,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":0.69999999999999996,"5min":6.4729999999999999,"15min":14.193},"last":0.39300000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":1.141,"15min":1.3200000000000001},"min":{"1min":0.32300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.63,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":1.2649999999999999,"15min":1.097},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.318},"max":{"1min":0.70399999999999996,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.49399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3726315520,"data_stored":3726311448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":919470080,"data_stored":919470080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1151860736,"data_stored":1151860736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3747221504,"data_stored":3747217432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1929379840,"data_stored":1929379840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2133921792,"data_stored":2133917720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1525743616,"data_stored":1525743616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:42:58.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:58.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:42:58.231 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:58.261 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:58.679 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:42:58.707 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:58 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8328/223032 objects degraded (3.734%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:58.708 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:58 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8328/223032 objects degraded (3.734%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:58.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:58 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8328/223032 objects degraded (3.734%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:42:58.966 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:58.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:58.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:59.071 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:59.092 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:42:59.099 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:59.148 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:59.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:59.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:59.252 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:42:59.282 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:59.327 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:42:59.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:59.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:42:59.433 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:42:59.463 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:59.511 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:42:59.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:59.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:42:59.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:59 smithi190 ceph-mon[160435]: pgmap v1487: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 8328/222072 objects degraded (3.750%); 4549/222072 objects misplaced (2.048%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:59.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:42:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3712290087' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:59.615 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:42:59.635 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:59 smithi138 ceph-mon[192513]: pgmap v1487: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 8328/222072 objects degraded (3.750%); 4549/222072 objects misplaced (2.048%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:59.635 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:42:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3712290087' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:59.635 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:59 smithi138 ceph-mon[185886]: pgmap v1487: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 8328/222072 objects degraded (3.750%); 4549/222072 objects misplaced (2.048%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:42:59.635 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:42:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3712290087' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:42:59.638 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:42:59.638 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:42:59.638 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:42:59.638 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:42:59.638 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:42:59.649 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:42:59.694 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:42:59.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:42:59.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:42:59.773 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:42:59.801 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:42:59.830 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:00.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:00 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:00 smithi190 ceph-mon[160435]: failed to encode map e519 with expected crc 2023-10-01T09:43:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:00 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3712290087' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:43:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:00 smithi190 ceph-mon[160435]: osdmap e519: 8 total, 8 up, 7 in 2023-10-01T09:43:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:00 smithi190 ceph-mon[160435]: failed to encode map e519 with expected crc 2023-10-01T09:43:00.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:43:00.815 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:00 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:00.816 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:00 smithi138 ceph-mon[185886]: failed to encode map e519 with expected crc 2023-10-01T09:43:00.816 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:00 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3712290087' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:43:00.816 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:00 smithi138 ceph-mon[185886]: osdmap e519: 8 total, 8 up, 7 in 2023-10-01T09:43:00.816 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:00 smithi138 ceph-mon[185886]: failed to encode map e519 with expected crc 2023-10-01T09:43:00.816 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:43:00.816 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:00 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:00.817 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:00 smithi138 ceph-mon[192513]: failed to encode map e519 with expected crc 2023-10-01T09:43:00.817 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:00 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3712290087' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:43:00.817 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:00 smithi138 ceph-mon[192513]: osdmap e519: 8 total, 8 up, 7 in 2023-10-01T09:43:00.817 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:00 smithi138 ceph-mon[192513]: failed to encode map e519 with expected crc 2023-10-01T09:43:00.817 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:43:01.131 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:43:01.239 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:01.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:01.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:01.258 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:43:01.348 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:01.381 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:01.450 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:01.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:01.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:01.554 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:01.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:01 smithi138 ceph-mon[192513]: failed to encode map e519 with expected crc 2023-10-01T09:43:01.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:01 smithi138 ceph-mon[192513]: failed to encode map e519 with expected crc 2023-10-01T09:43:01.581 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:01 smithi138 ceph-mon[192513]: pgmap v1489: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 103 op/s; 8328/221859 objects degraded (3.754%); 4549/221859 objects misplaced (2.050%); 323 KiB/s, 5 objects/s recovering 2023-10-01T09:43:01.582 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:01 smithi138 ceph-mon[185886]: failed to encode map e519 with expected crc 2023-10-01T09:43:01.582 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:01 smithi138 ceph-mon[185886]: failed to encode map e519 with expected crc 2023-10-01T09:43:01.582 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:01 smithi138 ceph-mon[185886]: pgmap v1489: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 103 op/s; 8328/221859 objects degraded (3.754%); 4549/221859 objects misplaced (2.050%); 323 KiB/s, 5 objects/s recovering 2023-10-01T09:43:01.589 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:01.688 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:01.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:01.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:01.799 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:01.802 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:43:01.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:01 smithi190 ceph-mon[160435]: failed to encode map e519 with expected crc 2023-10-01T09:43:01.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:01 smithi190 ceph-mon[160435]: failed to encode map e519 with expected crc 2023-10-01T09:43:01.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:01 smithi190 ceph-mon[160435]: pgmap v1489: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 103 op/s; 8328/221859 objects degraded (3.754%); 4549/221859 objects misplaced (2.050%); 323 KiB/s, 5 objects/s recovering 2023-10-01T09:43:01.821 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:01.832 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:43:01.838 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:01.866 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:43:01.877 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:01.896 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:43:01.921 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:43:01.943 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:43:01.962 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:43:01.983 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:43:01.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:43:01.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:43:01.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:43:01.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:43:01.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:43:01.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:43:01.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:43:01.986 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:43:02.268 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:02.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:02.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:02.373 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:43:02.405 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:02.509 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:43:02.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:02.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:43:02.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:02 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4120357385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:02.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:02 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4120357385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:02.614 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:02.647 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:02.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:02 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4120357385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:03.442 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:03.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:03.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:03.478 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:43:03.503 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:03 smithi138 ceph-mon[185886]: pgmap v1490: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 8293/220470 objects degraded (3.762%); 4500/220470 objects misplaced (2.041%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:43:03.503 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:03 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8293/220470 objects degraded (3.762%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:03.503 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:03 smithi138 ceph-mon[192513]: pgmap v1490: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 8293/220470 objects degraded (3.762%); 4500/220470 objects misplaced (2.041%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:43:03.503 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:03 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8293/220470 objects degraded (3.762%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:03.546 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:43:03.581 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:03.637 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:43:03.643 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:03 smithi190 ceph-mon[160435]: pgmap v1490: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 118 op/s; 8293/220470 objects degraded (3.762%); 4500/220470 objects misplaced (2.041%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:43:03.643 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:03 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8293/220470 objects degraded (3.762%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:03.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:03.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:43:03.746 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:03.773 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:03.834 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:03.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:03.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:03.940 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:03.967 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:04.412 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:04.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:04.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:04.515 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:43:04.546 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:04.592 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:43:04.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:04.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:43:04.698 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:04.704 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:04.731 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:04.781 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:04.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:04.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:04.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:04 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:04 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:04 smithi190 ceph-mon[160435]: pgmap v1491: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 115 op/s; 8293/220398 objects degraded (3.763%); 4500/220398 objects misplaced (2.042%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:43:04.811 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:43:04.811 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:43:04.832 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:04 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:04.832 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:04 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:04.832 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:04 smithi138 ceph-mon[192513]: pgmap v1491: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 115 op/s; 8293/220398 objects degraded (3.763%); 4500/220398 objects misplaced (2.042%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:43:04.832 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:04 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:04.832 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:04 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:04.833 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:04 smithi138 ceph-mon[185886]: pgmap v1491: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 115 op/s; 8293/220398 objects degraded (3.763%); 4500/220398 objects misplaced (2.042%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:43:04.885 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:04.917 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:05.208 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:43:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:43:04] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-01T09:43:05.242 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:05.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:05.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:05.323 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:43:05.341 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:43:05.346 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:05.373 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:05.433 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:43:05.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:05 smithi138 ceph-mon[192513]: from='client.46817 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:05.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:05 smithi138 ceph-mon[185886]: from='client.46817 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:05.917 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:05.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:05.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:05.922 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:43:06.022 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:06.055 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:06.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:05 smithi190 ceph-mon[160435]: from='client.46817 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:06.616 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:06.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:06.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:06.720 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:43:06.756 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:06.818 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:43:06.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:06.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:43:06.914 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:06 smithi138 ceph-mon[185886]: pgmap v1492: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8249/219342 objects degraded (3.761%); 4468/219342 objects misplaced (2.037%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:43:06.914 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:06 smithi138 ceph-mon[192513]: pgmap v1492: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8249/219342 objects degraded (3.761%); 4468/219342 objects misplaced (2.037%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:43:06.925 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:06.954 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:07.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:06 smithi190 ceph-mon[160435]: pgmap v1492: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8249/219342 objects degraded (3.761%); 4468/219342 objects misplaced (2.037%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:43:07.432 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:07.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:07.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:07.537 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:07.568 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:07.864 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1491,"stamp":"2023-10-01T09:43:04.301550+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4817557451,"num_objects":73466,"num_object_clones":0,"num_object_copies":220398,"num_objects_missing_on_primary":3111,"num_objects_missing":3111,"num_objects_degraded":8293,"num_objects_misplaced":4500,"num_objects_unfound":0,"num_objects_dirty":73466,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":94108,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7988,"num_bytes_recovered":510362639,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94172,"ondisk_log_size":94172,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19641892,"kb_used_data":14592684,"kb_used_omap":39,"kb_used_meta":5049112,"kb_avail":636574172,"statfs":{"total":671965249536,"available":651851952128,"internally_reserved":0,"allocated":14942908416,"data_stored":14933106112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5170291311},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,2,2,2,0,3,4,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-75956224,"num_objects":-1159,"num_object_clones":0,"num_object_copies":-3477,"num_objects_missing_on_primary":-83,"num_objects_missing":-83,"num_objects_degraded":-88,"num_objects_misplaced":-79,"num_objects_unfound":0,"num_objects_dirty":-1159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1159,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":5505024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002711"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":295,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.974560+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:42:51.974560+0000","last_peered":"2023-10-01T09:42:51.974560+0000","last_clean":"2023-10-01T09:42:51.974560+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:42:51.974560+0000","last_undegraded":"2023-10-01T09:42:51.974560+0000","last_fullsized":"2023-10-01T09:42:51.974560+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":138,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975757+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:42:51.975757+0000","last_peered":"2023-10-01T09:42:51.975757+0000","last_clean":"2023-10-01T09:42:51.975757+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:42:51.975757+0000","last_undegraded":"2023-10-01T09:42:51.975757+0000","last_fullsized":"2023-10-01T09:42:51.975757+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":518,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975301+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:42:51.975301+0000","last_peered":"2023-10-01T09:42:51.975301+0000","last_clean":"2023-10-01T09:42:51.975301+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:42:51.975301+0000","last_undegraded":"2023-10-01T09:42:51.975301+0000","last_fullsized":"2023-10-01T09:42:51.975301+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":577,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.777273+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:42:59.777273+0000","last_peered":"2023-10-01T09:42:59.777273+0000","last_clean":"2023-10-01T09:42:59.777273+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:42:59.777273+0000","last_undegraded":"2023-10-01T09:42:59.777273+0000","last_fullsized":"2023-10-01T09:42:59.777273+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":458,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.855240+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:42:51.855240+0000","last_peered":"2023-10-01T09:42:51.855240+0000","last_clean":"2023-10-01T09:42:51.855240+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:42:51.855240+0000","last_undegraded":"2023-10-01T09:42:51.855240+0000","last_fullsized":"2023-10-01T09:42:51.855240+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":200,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:52.012000+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:42:52.012000+0000","last_peered":"2023-10-01T09:42:52.012000+0000","last_clean":"2023-10-01T09:42:52.012000+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:42:52.012000+0000","last_undegraded":"2023-10-01T09:42:52.012000+0000","last_fullsized":"2023-10-01T09:42:52.012000+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":576,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:52.012059+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:42:52.012059+0000","last_peered":"2023-10-01T09:42:52.012059+0000","last_clean":"2023-10-01T09:42:52.012059+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:42:52.012059+0000","last_undegraded":"2023-10-01T09:42:52.012059+0000","last_fullsized":"2023-10-01T09:42:52.012059+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":505,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.974556+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:42:51.974556+0000","last_peered":"2023-10-01T09:42:51.974556+0000","last_clean":"2023-10-01T09:42:51.974556+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:42:51.974556+0000","last_undegraded":"2023-10-01T09:42:51.974556+0000","last_fullsized":"2023-10-01T09:42:51.974556+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":231,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.951138+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:42:51.951138+0000","last_peered":"2023-10-01T09:42:51.951138+0000","last_clean":"2023-10-01T09:42:51.951138+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:42:51.951138+0000","last_undegraded":"2023-10-01T09:42:51.951138+0000","last_fullsized":"2023-10-01T09:42:51.951138+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":38,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.784852+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:42:51.784852+0000","last_peered":"2023-10-01T09:42:51.784852+0000","last_clean":"2023-10-01T09:42:51.784852+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:42:51.784852+0000","last_undegraded":"2023-10-01T09:42:51.784852+0000","last_fullsized":"2023-10-01T09:42:51.784852+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":44,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.785039+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:42:51.785039+0000","last_peered":"2023-10-01T09:42:51.785039+0000","last_clean":"2023-10-01T09:42:51.785039+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:42:51.785039+0000","last_undegraded":"2023-10-01T09:42:51.785039+0000","last_fullsized":"2023-10-01T09:42:51.785039+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":41,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975331+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:42:51.975331+0000","last_peered":"2023-10-01T09:42:51.975331+0000","last_clean":"2023-10-01T09:42:51.975331+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:42:51.975331+0000","last_undegraded":"2023-10-01T09:42:51.975331+0000","last_fullsized":"2023-10-01T09:42:51.975331+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":420,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.974781+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:42:51.974781+0000","last_peered":"2023-10-01T09:42:51.974781+0000","last_clean":"2023-10-01T09:42:51.974781+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:42:51.974781+0000","last_undegraded":"2023-10-01T09:42:51.974781+0000","last_fullsized":"2023-10-01T09:42:51.974781+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":301,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.974569+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:42:51.974569+0000","last_peered":"2023-10-01T09:42:51.974569+0000","last_clean":"2023-10-01T09:42:51.974569+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:42:51.974569+0000","last_undegraded":"2023-10-01T09:42:51.974569+0000","last_fullsized":"2023-10-01T09:42:51.974569+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":637,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.785210+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:42:51.785210+0000","last_peered":"2023-10-01T09:42:51.785210+0000","last_clean":"2023-10-01T09:42:51.785210+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:42:51.785210+0000","last_undegraded":"2023-10-01T09:42:51.785210+0000","last_fullsized":"2023-10-01T09:42:51.785210+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":374,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.785670+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:42:51.785670+0000","last_peered":"2023-10-01T09:42:51.785670+0000","last_clean":"2023-10-01T09:42:51.785670+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:42:51.785670+0000","last_undegraded":"2023-10-01T09:42:51.785670+0000","last_fullsized":"2023-10-01T09:42:51.785670+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":47,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.785031+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:42:51.785031+0000","last_peered":"2023-10-01T09:42:51.785031+0000","last_clean":"2023-10-01T09:42:51.785031+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:42:51.785031+0000","last_undegraded":"2023-10-01T09:42:51.785031+0000","last_fullsized":"2023-10-01T09:42:51.785031+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":33,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.950813+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:42:51.950813+0000","last_peered":"2023-10-01T09:42:51.950813+0000","last_clean":"2023-10-01T09:42:51.950813+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:42:51.950813+0000","last_undegraded":"2023-10-01T09:42:51.950813+0000","last_fullsized":"2023-10-01T09:42:51.950813+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":530,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.785284+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:42:51.785284+0000","last_peered":"2023-10-01T09:42:51.785284+0000","last_clean":"2023-10-01T09:42:51.785284+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:42:51.785284+0000","last_undegraded":"2023-10-01T09:42:51.785284+0000","last_fullsized":"2023-10-01T09:42:51.785284+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":541,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.817970+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:42:51.817970+0000","last_peered":"2023-10-01T09:42:51.817970+0000","last_clean":"2023-10-01T09:42:51.817970+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:42:51.817970+0000","last_undegraded":"2023-10-01T09:42:51.817970+0000","last_fullsized":"2023-10-01T09:42:51.817970+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":397,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.816679+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:42:51.816679+0000","last_peered":"2023-10-01T09:42:51.816679+0000","last_clean":"2023-10-01T09:42:51.816679+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:42:51.816679+0000","last_undegraded":"2023-10-01T09:42:51.816679+0000","last_fullsized":"2023-10-01T09:42:51.816679+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":47,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.786036+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:42:51.786036+0000","last_peered":"2023-10-01T09:42:51.786036+0000","last_clean":"2023-10-01T09:42:51.786036+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:42:51.786036+0000","last_undegraded":"2023-10-01T09:42:51.786036+0000","last_fullsized":"2023-10-01T09:42:51.786036+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"519'5818","reported_seq":5870,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.271005+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:43:00.271005+0000","last_peered":"2023-10-01T09:43:00.271005+0000","last_clean":"2023-10-01T09:43:00.271005+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:43:00.271005+0000","last_undegraded":"2023-10-01T09:43:00.271005+0000","last_fullsized":"2023-10-01T09:43:00.271005+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5818,"log_dups_size":0,"ondisk_log_size":5818,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299761664,"num_objects":4574,"num_object_clones":0,"num_object_copies":13722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5818,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":42,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.785883+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:42:51.785883+0000","last_peered":"2023-10-01T09:42:51.785883+0000","last_clean":"2023-10-01T09:42:51.785883+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:42:51.785883+0000","last_undegraded":"2023-10-01T09:42:51.785883+0000","last_fullsized":"2023-10-01T09:42:51.785883+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":239,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.950742+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:42:51.950742+0000","last_peered":"2023-10-01T09:42:51.950742+0000","last_clean":"2023-10-01T09:42:51.950742+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:42:51.950742+0000","last_undegraded":"2023-10-01T09:42:51.950742+0000","last_fullsized":"2023-10-01T09:42:51.950742+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"519'5776","reported_seq":7578,"reported_epoch":519,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:43:00.617152+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:43:00.617152+0000","last_peered":"2023-10-01T09:43:00.617152+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:43:00.617152+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5776,"log_dups_size":0,"ondisk_log_size":5776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303431680,"num_objects":4630,"num_object_clones":0,"num_object_copies":13890,"num_objects_missing_on_primary":3111,"num_objects_missing":3111,"num_objects_degraded":7672,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5776,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":574,"num_bytes_recovered":37617664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1450},{"shards":"0,4,6","objects":3111},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":513,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.785787+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:42:51.785787+0000","last_peered":"2023-10-01T09:42:51.785787+0000","last_clean":"2023-10-01T09:42:51.785787+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:42:51.785787+0000","last_undegraded":"2023-10-01T09:42:51.785787+0000","last_fullsized":"2023-10-01T09:42:51.785787+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":574,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975221+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:42:51.975221+0000","last_peered":"2023-10-01T09:42:51.975221+0000","last_clean":"2023-10-01T09:42:51.975221+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:42:51.975221+0000","last_undegraded":"2023-10-01T09:42:51.975221+0000","last_fullsized":"2023-10-01T09:42:51.975221+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"519'5846","reported_seq":5900,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:01.261447+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:43:01.261447+0000","last_peered":"2023-10-01T09:43:01.261447+0000","last_clean":"2023-10-01T09:43:01.261447+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:43:01.261447+0000","last_undegraded":"2023-10-01T09:43:01.261447+0000","last_fullsized":"2023-10-01T09:43:01.261447+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5846,"log_dups_size":0,"ondisk_log_size":5846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300548096,"num_objects":4586,"num_object_clones":0,"num_object_copies":13758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5846,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":599,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.785485+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:42:51.785485+0000","last_peered":"2023-10-01T09:42:51.785485+0000","last_clean":"2023-10-01T09:42:51.785485+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:42:51.785485+0000","last_undegraded":"2023-10-01T09:42:51.785485+0000","last_fullsized":"2023-10-01T09:42:51.785485+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":394,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975443+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:42:51.975443+0000","last_peered":"2023-10-01T09:42:51.975443+0000","last_clean":"2023-10-01T09:42:51.975443+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:42:51.975443+0000","last_undegraded":"2023-10-01T09:42:51.975443+0000","last_fullsized":"2023-10-01T09:42:51.975443+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"519'5776","reported_seq":12435,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:01.806283+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:43:01.806283+0000","last_peered":"2023-10-01T09:43:01.806283+0000","last_clean":"2023-10-01T09:43:01.806283+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:43:01.806283+0000","last_undegraded":"2023-10-01T09:43:01.806283+0000","last_fullsized":"2023-10-01T09:43:01.806283+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5776,"log_dups_size":0,"ondisk_log_size":5776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303169536,"num_objects":4626,"num_object_clones":0,"num_object_copies":13878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5776,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":239,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.950918+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:42:51.950918+0000","last_peered":"2023-10-01T09:42:51.950918+0000","last_clean":"2023-10-01T09:42:51.950918+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:42:51.950918+0000","last_undegraded":"2023-10-01T09:42:51.950918+0000","last_fullsized":"2023-10-01T09:42:51.950918+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":497,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.816851+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:42:51.816851+0000","last_peered":"2023-10-01T09:42:51.816851+0000","last_clean":"2023-10-01T09:42:51.816851+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:42:51.816851+0000","last_undegraded":"2023-10-01T09:42:51.816851+0000","last_fullsized":"2023-10-01T09:42:51.816851+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"519'5850","reported_seq":5904,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.047157+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:43:00.047157+0000","last_peered":"2023-10-01T09:43:00.047157+0000","last_clean":"2023-10-01T09:43:00.047157+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:43:00.047157+0000","last_undegraded":"2023-10-01T09:43:00.047157+0000","last_fullsized":"2023-10-01T09:43:00.047157+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5850,"log_dups_size":0,"ondisk_log_size":5850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305266688,"num_objects":4658,"num_object_clones":0,"num_object_copies":13974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5850,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":99,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.974743+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:42:51.974743+0000","last_peered":"2023-10-01T09:42:51.974743+0000","last_clean":"2023-10-01T09:42:51.974743+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:42:51.974743+0000","last_undegraded":"2023-10-01T09:42:51.974743+0000","last_fullsized":"2023-10-01T09:42:51.974743+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":173,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.974861+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:42:51.974861+0000","last_peered":"2023-10-01T09:42:51.974861+0000","last_clean":"2023-10-01T09:42:51.974861+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:42:51.974861+0000","last_undegraded":"2023-10-01T09:42:51.974861+0000","last_fullsized":"2023-10-01T09:42:51.974861+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"519'5804","reported_seq":5884,"reported_epoch":519,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:43:00.277641+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:43:00.277641+0000","last_peered":"2023-10-01T09:43:00.277641+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:43:00.277641+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5804,"log_dups_size":0,"ondisk_log_size":5804,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299106304,"num_objects":4564,"num_object_clones":0,"num_object_copies":13692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":611,"num_objects_misplaced":4500,"num_objects_unfound":0,"num_objects_dirty":4564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5804,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4500},{"shards":"4,5","objects":-547}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":417,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.974668+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:42:51.974668+0000","last_peered":"2023-10-01T09:42:51.974668+0000","last_clean":"2023-10-01T09:42:51.974668+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:42:51.974668+0000","last_undegraded":"2023-10-01T09:42:51.974668+0000","last_fullsized":"2023-10-01T09:42:51.974668+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":577,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975801+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:42:51.975801+0000","last_peered":"2023-10-01T09:42:51.975801+0000","last_clean":"2023-10-01T09:42:51.975801+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:42:51.975801+0000","last_undegraded":"2023-10-01T09:42:51.975801+0000","last_fullsized":"2023-10-01T09:42:51.975801+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"519'5792","reported_seq":5853,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:02.349506+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:43:02.349506+0000","last_peered":"2023-10-01T09:43:02.349506+0000","last_clean":"2023-10-01T09:43:02.349506+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:43:02.349506+0000","last_undegraded":"2023-10-01T09:43:02.349506+0000","last_fullsized":"2023-10-01T09:43:02.349506+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5792,"log_dups_size":0,"ondisk_log_size":5792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300548096,"num_objects":4586,"num_object_clones":0,"num_object_copies":13758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5792,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":413,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.974777+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:42:51.974777+0000","last_peered":"2023-10-01T09:42:51.974777+0000","last_clean":"2023-10-01T09:42:51.974777+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:42:51.974777+0000","last_undegraded":"2023-10-01T09:42:51.974777+0000","last_fullsized":"2023-10-01T09:42:51.974777+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":254,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975005+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:42:51.975005+0000","last_peered":"2023-10-01T09:42:51.975005+0000","last_clean":"2023-10-01T09:42:51.975005+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:42:51.975005+0000","last_undegraded":"2023-10-01T09:42:51.975005+0000","last_fullsized":"2023-10-01T09:42:51.975005+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"519'5895","reported_seq":5947,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:01.155855+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:43:01.155855+0000","last_peered":"2023-10-01T09:43:01.155855+0000","last_clean":"2023-10-01T09:43:01.155855+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:43:01.155855+0000","last_undegraded":"2023-10-01T09:43:01.155855+0000","last_fullsized":"2023-10-01T09:43:01.155855+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5895,"log_dups_size":0,"ondisk_log_size":5895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301662208,"num_objects":4603,"num_object_clones":0,"num_object_copies":13809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5895,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":381,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.974948+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:42:51.974948+0000","last_peered":"2023-10-01T09:42:51.974948+0000","last_clean":"2023-10-01T09:42:51.974948+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:42:51.974948+0000","last_undegraded":"2023-10-01T09:42:51.974948+0000","last_fullsized":"2023-10-01T09:42:51.974948+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":428,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975865+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:42:51.975865+0000","last_peered":"2023-10-01T09:42:51.975865+0000","last_clean":"2023-10-01T09:42:51.975865+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:42:51.975865+0000","last_undegraded":"2023-10-01T09:42:51.975865+0000","last_fullsized":"2023-10-01T09:42:51.975865+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"519'5727","reported_seq":5779,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:01.200456+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:43:01.200456+0000","last_peered":"2023-10-01T09:43:01.200456+0000","last_clean":"2023-10-01T09:43:01.200456+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:43:01.200456+0000","last_undegraded":"2023-10-01T09:43:01.200456+0000","last_fullsized":"2023-10-01T09:43:01.200456+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5727,"log_dups_size":0,"ondisk_log_size":5727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291700736,"num_objects":4451,"num_object_clones":0,"num_object_copies":13353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5727,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":912,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.785534+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:42:51.785534+0000","last_peered":"2023-10-01T09:42:51.785534+0000","last_clean":"2023-10-01T09:42:51.785534+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:42:51.785534+0000","last_undegraded":"2023-10-01T09:42:51.785534+0000","last_fullsized":"2023-10-01T09:42:51.785534+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":249,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.950952+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:42:51.950952+0000","last_peered":"2023-10-01T09:42:51.950952+0000","last_clean":"2023-10-01T09:42:51.950952+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:42:51.950952+0000","last_undegraded":"2023-10-01T09:42:51.950952+0000","last_fullsized":"2023-10-01T09:42:51.950952+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":42,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975372+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:42:51.975372+0000","last_peered":"2023-10-01T09:42:51.975372+0000","last_clean":"2023-10-01T09:42:51.975372+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:42:51.975372+0000","last_undegraded":"2023-10-01T09:42:51.975372+0000","last_fullsized":"2023-10-01T09:42:51.975372+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"519'5778","reported_seq":5832,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:01.135216+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:43:01.135216+0000","last_peered":"2023-10-01T09:43:01.135216+0000","last_clean":"2023-10-01T09:43:01.135216+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:43:01.135216+0000","last_undegraded":"2023-10-01T09:43:01.135216+0000","last_fullsized":"2023-10-01T09:43:01.135216+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5778,"log_dups_size":0,"ondisk_log_size":5778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300941312,"num_objects":4592,"num_object_clones":0,"num_object_copies":13776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5778,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1184,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.950894+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:42:51.950894+0000","last_peered":"2023-10-01T09:42:51.950894+0000","last_clean":"2023-10-01T09:42:51.950894+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:42:51.950894+0000","last_undegraded":"2023-10-01T09:42:51.950894+0000","last_fullsized":"2023-10-01T09:42:51.950894+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":394,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.974692+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:42:51.974692+0000","last_peered":"2023-10-01T09:42:51.974692+0000","last_clean":"2023-10-01T09:42:51.974692+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:42:51.974692+0000","last_undegraded":"2023-10-01T09:42:51.974692+0000","last_fullsized":"2023-10-01T09:42:51.974692+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":222,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.951073+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:42:51.951073+0000","last_peered":"2023-10-01T09:42:51.951073+0000","last_clean":"2023-10-01T09:42:51.951073+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:42:51.951073+0000","last_undegraded":"2023-10-01T09:42:51.951073+0000","last_fullsized":"2023-10-01T09:42:51.951073+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"519'5802","reported_seq":13075,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.029776+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:43:00.029776+0000","last_peered":"2023-10-01T09:43:00.029776+0000","last_clean":"2023-10-01T09:43:00.029776+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:43:00.029776+0000","last_undegraded":"2023-10-01T09:43:00.029776+0000","last_fullsized":"2023-10-01T09:43:00.029776+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5802,"log_dups_size":0,"ondisk_log_size":5802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299761664,"num_objects":4574,"num_object_clones":0,"num_object_copies":13722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5802,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":863,"reported_epoch":518,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:42:51.816542+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:42:51.816542+0000","last_peered":"2023-10-01T09:42:51.816542+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:42:51.816542+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:42:51.816542+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":581,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975035+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:42:51.975035+0000","last_peered":"2023-10-01T09:42:51.975035+0000","last_clean":"2023-10-01T09:42:51.975035+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:42:51.975035+0000","last_undegraded":"2023-10-01T09:42:51.975035+0000","last_fullsized":"2023-10-01T09:42:51.975035+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":42,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:55.249722+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:42:55.249722+0000","last_peered":"2023-10-01T09:42:55.249722+0000","last_clean":"2023-10-01T09:42:55.249722+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:42:55.249722+0000","last_undegraded":"2023-10-01T09:42:55.249722+0000","last_fullsized":"2023-10-01T09:42:55.249722+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"519'5704","reported_seq":5758,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.021554+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:43:00.021554+0000","last_peered":"2023-10-01T09:43:00.021554+0000","last_clean":"2023-10-01T09:43:00.021554+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:43:00.021554+0000","last_undegraded":"2023-10-01T09:43:00.021554+0000","last_fullsized":"2023-10-01T09:43:00.021554+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299368448,"num_objects":4568,"num_object_clones":0,"num_object_copies":13704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5704,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1635,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.951086+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:42:51.951086+0000","last_peered":"2023-10-01T09:42:51.951086+0000","last_clean":"2023-10-01T09:42:51.951086+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:42:51.951086+0000","last_undegraded":"2023-10-01T09:42:51.951086+0000","last_fullsized":"2023-10-01T09:42:51.951086+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":549,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.784845+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:42:51.784845+0000","last_peered":"2023-10-01T09:42:51.784845+0000","last_clean":"2023-10-01T09:42:51.784845+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:42:51.784845+0000","last_undegraded":"2023-10-01T09:42:51.784845+0000","last_fullsized":"2023-10-01T09:42:51.784845+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":327,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.950977+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:42:51.950977+0000","last_peered":"2023-10-01T09:42:51.950977+0000","last_clean":"2023-10-01T09:42:51.950977+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:42:51.950977+0000","last_undegraded":"2023-10-01T09:42:51.950977+0000","last_fullsized":"2023-10-01T09:42:51.950977+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":317,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975062+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:42:51.975062+0000","last_peered":"2023-10-01T09:42:51.975062+0000","last_clean":"2023-10-01T09:42:51.975062+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:42:51.975062+0000","last_undegraded":"2023-10-01T09:42:51.975062+0000","last_fullsized":"2023-10-01T09:42:51.975062+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"519'5869","reported_seq":5921,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:01.246250+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:43:01.246250+0000","last_peered":"2023-10-01T09:43:01.246250+0000","last_clean":"2023-10-01T09:43:01.246250+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:43:01.246250+0000","last_undegraded":"2023-10-01T09:43:01.246250+0000","last_fullsized":"2023-10-01T09:43:01.246250+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5869,"log_dups_size":0,"ondisk_log_size":5869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305594368,"num_objects":4663,"num_object_clones":0,"num_object_copies":13989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5869,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":726,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.784934+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:42:51.784934+0000","last_peered":"2023-10-01T09:42:51.784934+0000","last_clean":"2023-10-01T09:42:51.784934+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:42:51.784934+0000","last_undegraded":"2023-10-01T09:42:51.784934+0000","last_fullsized":"2023-10-01T09:42:51.784934+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":558,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.816928+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:42:51.816928+0000","last_peered":"2023-10-01T09:42:51.816928+0000","last_clean":"2023-10-01T09:42:51.816928+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:42:51.816928+0000","last_undegraded":"2023-10-01T09:42:51.816928+0000","last_fullsized":"2023-10-01T09:42:51.816928+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":557,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.817890+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:42:51.817890+0000","last_peered":"2023-10-01T09:42:51.817890+0000","last_clean":"2023-10-01T09:42:51.817890+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:42:51.817890+0000","last_undegraded":"2023-10-01T09:42:51.817890+0000","last_fullsized":"2023-10-01T09:42:51.817890+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"519'5865","reported_seq":5920,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:01.256278+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:43:01.256278+0000","last_peered":"2023-10-01T09:43:01.256278+0000","last_clean":"2023-10-01T09:43:01.256278+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:43:01.256278+0000","last_undegraded":"2023-10-01T09:43:01.256278+0000","last_fullsized":"2023-10-01T09:43:01.256278+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5865,"log_dups_size":0,"ondisk_log_size":5865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301072408,"num_objects":4595,"num_object_clones":0,"num_object_copies":13785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4595,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5865,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":957,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.785482+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:42:51.785482+0000","last_peered":"2023-10-01T09:42:51.785482+0000","last_clean":"2023-10-01T09:42:51.785482+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:42:51.785482+0000","last_undegraded":"2023-10-01T09:42:51.785482+0000","last_fullsized":"2023-10-01T09:42:51.785482+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":576,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.950999+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:42:51.950999+0000","last_peered":"2023-10-01T09:42:51.950999+0000","last_clean":"2023-10-01T09:42:51.950999+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:42:51.950999+0000","last_undegraded":"2023-10-01T09:42:51.950999+0000","last_fullsized":"2023-10-01T09:42:51.950999+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":429,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.951167+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:42:51.951167+0000","last_peered":"2023-10-01T09:42:51.951167+0000","last_clean":"2023-10-01T09:42:51.951167+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:42:51.951167+0000","last_undegraded":"2023-10-01T09:42:51.951167+0000","last_fullsized":"2023-10-01T09:42:51.951167+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"519'5828","reported_seq":5888,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:01.182214+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:43:01.182214+0000","last_peered":"2023-10-01T09:43:01.182214+0000","last_clean":"2023-10-01T09:43:01.182214+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:43:01.182214+0000","last_undegraded":"2023-10-01T09:43:01.182214+0000","last_fullsized":"2023-10-01T09:43:01.182214+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5828,"log_dups_size":0,"ondisk_log_size":5828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296747008,"num_objects":4528,"num_object_clones":0,"num_object_copies":13584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5828,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":458,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.786119+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:42:51.786119+0000","last_peered":"2023-10-01T09:42:51.786119+0000","last_clean":"2023-10-01T09:42:51.786119+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:42:51.786119+0000","last_undegraded":"2023-10-01T09:42:51.786119+0000","last_fullsized":"2023-10-01T09:42:51.786119+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1151,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975919+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:42:51.975919+0000","last_peered":"2023-10-01T09:42:51.975919+0000","last_clean":"2023-10-01T09:42:51.975919+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:42:51.975919+0000","last_undegraded":"2023-10-01T09:42:51.975919+0000","last_fullsized":"2023-10-01T09:42:51.975919+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":47,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.784779+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:42:51.784779+0000","last_peered":"2023-10-01T09:42:51.784779+0000","last_clean":"2023-10-01T09:42:51.784779+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:42:51.784779+0000","last_undegraded":"2023-10-01T09:42:51.784779+0000","last_fullsized":"2023-10-01T09:42:51.784779+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"519'5872","reported_seq":5932,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:01.123788+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:43:01.123788+0000","last_peered":"2023-10-01T09:43:01.123788+0000","last_clean":"2023-10-01T09:43:01.123788+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:43:01.123788+0000","last_undegraded":"2023-10-01T09:43:01.123788+0000","last_fullsized":"2023-10-01T09:43:01.123788+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5872,"log_dups_size":0,"ondisk_log_size":5872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304087040,"num_objects":4640,"num_object_clones":0,"num_object_copies":13920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5872,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":51,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.785963+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:42:51.785963+0000","last_peered":"2023-10-01T09:42:51.785963+0000","last_clean":"2023-10-01T09:42:51.785963+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:42:51.785963+0000","last_undegraded":"2023-10-01T09:42:51.785963+0000","last_fullsized":"2023-10-01T09:42:51.785963+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":535,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.817916+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:42:51.817916+0000","last_peered":"2023-10-01T09:42:51.817916+0000","last_clean":"2023-10-01T09:42:51.817916+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:42:51.817916+0000","last_undegraded":"2023-10-01T09:42:51.817916+0000","last_fullsized":"2023-10-01T09:42:51.817916+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":242,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.951033+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:42:51.951033+0000","last_peered":"2023-10-01T09:42:51.951033+0000","last_clean":"2023-10-01T09:42:51.951033+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:42:51.951033+0000","last_undegraded":"2023-10-01T09:42:51.951033+0000","last_fullsized":"2023-10-01T09:42:51.951033+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":484,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.816948+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:42:51.816948+0000","last_peered":"2023-10-01T09:42:51.816948+0000","last_clean":"2023-10-01T09:42:51.816948+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:42:51.816948+0000","last_undegraded":"2023-10-01T09:42:51.816948+0000","last_fullsized":"2023-10-01T09:42:51.816948+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":598,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975768+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:42:51.975768+0000","last_peered":"2023-10-01T09:42:51.975768+0000","last_clean":"2023-10-01T09:42:51.975768+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:42:51.975768+0000","last_undegraded":"2023-10-01T09:42:51.975768+0000","last_fullsized":"2023-10-01T09:42:51.975768+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":254,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.950838+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:42:51.950838+0000","last_peered":"2023-10-01T09:42:51.950838+0000","last_clean":"2023-10-01T09:42:51.950838+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:42:51.950838+0000","last_undegraded":"2023-10-01T09:42:51.950838+0000","last_fullsized":"2023-10-01T09:42:51.950838+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":41,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975610+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:42:51.975610+0000","last_peered":"2023-10-01T09:42:51.975610+0000","last_clean":"2023-10-01T09:42:51.975610+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:42:51.975610+0000","last_undegraded":"2023-10-01T09:42:51.975610+0000","last_fullsized":"2023-10-01T09:42:51.975610+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":254,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.950977+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:42:51.950977+0000","last_peered":"2023-10-01T09:42:51.950977+0000","last_clean":"2023-10-01T09:42:51.950977+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:42:51.950977+0000","last_undegraded":"2023-10-01T09:42:51.950977+0000","last_fullsized":"2023-10-01T09:42:51.950977+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":430,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.816801+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:42:51.816801+0000","last_peered":"2023-10-01T09:42:51.816801+0000","last_clean":"2023-10-01T09:42:51.816801+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:42:51.816801+0000","last_undegraded":"2023-10-01T09:42:51.816801+0000","last_fullsized":"2023-10-01T09:42:51.816801+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":59,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.975819+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:42:51.975819+0000","last_peered":"2023-10-01T09:42:51.975819+0000","last_clean":"2023-10-01T09:42:51.975819+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:42:51.975819+0000","last_undegraded":"2023-10-01T09:42:51.975819+0000","last_fullsized":"2023-10-01T09:42:51.975819+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":476,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.855279+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:42:51.855279+0000","last_peered":"2023-10-01T09:42:51.855279+0000","last_clean":"2023-10-01T09:42:51.855279+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:42:51.855279+0000","last_undegraded":"2023-10-01T09:42:51.855279+0000","last_fullsized":"2023-10-01T09:42:51.855279+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":675,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.817891+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:42:51.817891+0000","last_peered":"2023-10-01T09:42:51.817891+0000","last_clean":"2023-10-01T09:42:51.817891+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:42:51.817891+0000","last_undegraded":"2023-10-01T09:42:51.817891+0000","last_fullsized":"2023-10-01T09:42:51.817891+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":370,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-10-01T09:42:51.974404+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:42:51.974404+0000","last_peered":"2023-10-01T09:42:51.974404+0000","last_clean":"2023-10-01T09:42:51.974404+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:42:51.974404+0000","last_undegraded":"2023-10-01T09:42:51.974404+0000","last_fullsized":"2023-10-01T09:42:51.974404+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4812767256,"num_objects":73438,"num_object_clones":0,"num_object_copies":220314,"num_objects_missing_on_primary":3111,"num_objects_missing":3111,"num_objects_degraded":8283,"num_objects_misplaced":4500,"num_objects_unfound":0,"num_objects_dirty":73438,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":93002,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7477,"num_bytes_recovered":490012672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14882779136,"data_stored":14882766920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93002,"ondisk_log_size":93002,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430709,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2376200,"kb_used_data":1477184,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":91368952,"statfs":{"total":95995035648,"available":93561806848,"internally_reserved":0,"allocated":1512636416,"data_stored":1511700980,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:56 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.4950000000000001,"15min":2.9569999999999999},"min":{"1min":0.246,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":1.3280000000000001,"5min":10.391999999999999,"15min":108.90000000000001},"last":0.253},{"interface":"front","average":{"1min":0.49199999999999999,"5min":1.8129999999999999,"15min":3.129},"min":{"1min":0.246,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.018,"5min":29.501000000000001,"15min":109.187},"last":0.29399999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:42:06 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":5.0410000000000004,"15min":5.492},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":6.8289999999999997,"5min":229.161,"15min":229.161},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.92100000000000004,"5min":4.9939999999999998,"15min":5.5490000000000004},"min":{"1min":0.28899999999999998,"5min":0.27500000000000002,"15min":0.24399999999999999},"max":{"1min":5.7240000000000002,"5min":230.126,"15min":230.126},"last":0.26400000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:42:50 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":1.5369999999999999,"15min":1.891},"min":{"1min":0.33900000000000002,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":1.3560000000000001,"5min":14.866,"15min":14.901999999999999},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":1.7909999999999999,"15min":1.821},"min":{"1min":0.33500000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.96599999999999997,"5min":14.682,"15min":14.682},"last":0.32000000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:42:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.9410000000000001,"15min":2.8610000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":5.7750000000000004,"5min":35.104999999999997,"15min":80.352999999999994},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.004,"5min":2,"15min":2.8359999999999999},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.548,"5min":35.118000000000002,"15min":80.364000000000004},"last":0.41799999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:42:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":1.2150000000000001,"15min":1.052},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.381,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.309},{"interface":"front","average":{"1min":0.49099999999999999,"5min":1.1080000000000001,"15min":1.03},"min":{"1min":0.314,"5min":0.30299999999999999,"15min":0.28000000000000003},"max":{"1min":0.97899999999999998,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.39900000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":2.8879999999999999,"15min":1.6419999999999999},"min":{"1min":0.318,"5min":0.30299999999999999,"15min":0.29599999999999999},"max":{"1min":0.64600000000000002,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.41999999999999998,"5min":3.0779999999999998,"15min":1.6850000000000001},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54500000000000004,"5min":135.339,"15min":135.339},"last":2.2599999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.4379999999999999,"15min":1.173},"min":{"1min":0.313,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":2.448,"5min":11.58,"15min":11.853999999999999},"last":0.377},{"interface":"front","average":{"1min":0.44400000000000001,"5min":1.2789999999999999,"15min":1.1359999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":0.56699999999999995,"5min":14.584,"15min":14.584},"last":0.38600000000000001}]}]},{"osd":6,"up_from":294,"seq":1262720385220,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2400708,"kb_used_data":2059004,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":91344444,"statfs":{"total":95995035648,"available":93536710656,"internally_reserved":0,"allocated":2108420096,"data_stored":2107488611,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":3.786,"15min":4.1879999999999997},"min":{"1min":0.249,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":1.8400000000000001,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.76300000000000001,"5min":4.3559999999999999,"15min":4.2889999999999997},"min":{"1min":0.249,"5min":0.249,"15min":0.221},"max":{"1min":2.9969999999999999,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.316}]},{"osd":1,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":5.3860000000000001,"15min":5.0590000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":3.0289999999999999,"5min":168.947,"15min":168.947},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.79100000000000004,"5min":5.375,"15min":4.9690000000000003},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":2.6030000000000002,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.378}]},{"osd":2,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.8109999999999999,"15min":1.8420000000000001},"min":{"1min":0.34499999999999997,"5min":0.26900000000000002,"15min":0.254},"max":{"1min":3.0489999999999999,"5min":9.657,"15min":26.289999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.885,"15min":1.859},"min":{"1min":0.32600000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.6739999999999999,"5min":21.704999999999998,"15min":26.388999999999999},"last":0.41499999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":1.597,"15min":2.4020000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.9450000000000001,"5min":9.0180000000000007,"15min":89.753},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.073,"5min":1.9330000000000001,"15min":2.2189999999999999},"min":{"1min":0.318,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":3.7160000000000002,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.56200000000000006}]},{"osd":4,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.665,"15min":1.0800000000000001},"min":{"1min":0.28599999999999998,"5min":0.253,"15min":0.246},"max":{"1min":4.0250000000000004,"5min":63.134,"15min":63.134},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.518,"15min":1.0960000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.28299999999999997},"max":{"1min":7.4039999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.629}]},{"osd":5,"last update":"Sun Oct 1 09:42:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.93200000000000005,"15min":1.02},"min":{"1min":0.26700000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.8620000000000001,"5min":9.9939999999999998,"15min":12.023999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.66500000000000004,"5min":1.175,"15min":1.143},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.726,"5min":15.457000000000001,"15min":15.457000000000001},"last":0.57499999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:42:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.82999999999999996,"15min":1.802},"min":{"1min":0.32700000000000001,"5min":0.311,"15min":0.27900000000000003},"max":{"1min":2.0419999999999998,"5min":6.4119999999999999,"15min":115.11},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.282,"15min":1.8959999999999999},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.28799999999999998},"max":{"1min":3.2679999999999998,"5min":8.6899999999999995,"15min":114.666},"last":0.60599999999999998}]}]},{"osd":1,"up_from":94,"seq":403726926196,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1708712,"kb_used_data":894624,"kb_used_omap":7,"kb_used_meta":814072,"kb_avail":92036440,"statfs":{"total":95995035648,"available":94245314560,"internally_reserved":0,"allocated":916094976,"data_stored":914076965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":833610151},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.30299999999999999,"5min":2.5600000000000001,"15min":4.2009999999999996},"min":{"1min":0.20699999999999999,"5min":0.19400000000000001,"15min":0.182},"max":{"1min":0.57399999999999995,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.308},{"interface":"front","average":{"1min":0.33600000000000002,"5min":3.0139999999999998,"15min":3.7090000000000001},"min":{"1min":0.26300000000000001,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":0.53400000000000003,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.38300000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:42:38 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":1.7929999999999999,"15min":2.7719999999999998},"min":{"1min":0.24099999999999999,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.83099999999999996,"5min":22.891999999999999,"15min":56.573},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.36699999999999999,"5min":2.2799999999999998,"15min":3.593},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":0.54400000000000004,"5min":15.189,"15min":154.197},"last":0.45400000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":3.9089999999999998,"15min":4.4429999999999996},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":3.1949999999999998,"5min":95.305000000000007,"15min":200.56800000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.68799999999999994,"5min":4.3550000000000004,"15min":5.2569999999999997},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":4.7839999999999998,"5min":98.033000000000001,"15min":200.63800000000001},"last":0.47199999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.8029999999999999,"15min":2.3109999999999999},"min":{"1min":0.32900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.8109999999999999,"5min":21.015000000000001,"15min":154.101},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.46400000000000002,"5min":1.585,"15min":2.5249999999999999},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.83199999999999996,"5min":10.222,"15min":154.56999999999999},"last":0.59799999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:42:35 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.843,"15min":2.1240000000000001},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.2450000000000001,"5min":21.219999999999999,"15min":154.22999999999999},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.6499999999999999,"15min":2.5649999999999999},"min":{"1min":0.316,"5min":0.307,"15min":0.307},"max":{"1min":2.407,"5min":10.305,"15min":154.54400000000001},"last":0.56999999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.1669999999999998,"15min":2.8370000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.3660000000000001,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.626,"5min":2.4769999999999999,"15min":2.5899999999999999},"min":{"1min":0.36699999999999999,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":1.575,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.61199999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.9319999999999999,"15min":2.6440000000000001},"min":{"1min":0.40100000000000002,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":1.3029999999999999,"5min":21.062999999999999,"15min":154.20699999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.804,"15min":2.718},"min":{"1min":0.41699999999999998,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":1.3220000000000001,"5min":13.161,"15min":154.51599999999999},"last":0.63800000000000001}]}]},{"osd":0,"up_from":89,"seq":382252089731,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4503844,"kb_used_data":3604508,"kb_used_omap":8,"kb_used_meta":899319,"kb_avail":89241308,"statfs":{"total":95995035648,"available":91383099392,"internally_reserved":0,"allocated":3691016192,"data_stored":3689007620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":920903618},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:42:08 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":3.9860000000000002,"15min":4.4039999999999999},"min":{"1min":0.20399999999999999,"5min":0.187,"15min":0.17399999999999999},"max":{"1min":1.661,"5min":185.78399999999999,"15min":216.821},"last":0.20300000000000001},{"interface":"front","average":{"1min":0.437,"5min":5.9020000000000001,"15min":5.4400000000000004},"min":{"1min":0.249,"5min":0.215,"15min":0.215},"max":{"1min":1.5589999999999999,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.22900000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:42:36 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":1.8779999999999999,"15min":2.1579999999999999},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.20100000000000001},"max":{"1min":0.51200000000000001,"5min":19.788,"15min":19.788},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.36899999999999999,"5min":2.0960000000000001,"15min":2.5230000000000001},"min":{"1min":0.315,"5min":0.215,"15min":0.189},"max":{"1min":0.439,"5min":25.844999999999999,"15min":25.844999999999999},"last":0.316}]},{"osd":3,"last update":"Sun Oct 1 09:42:08 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":2.1749999999999998,"15min":4.524},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.20899999999999999},"max":{"1min":1.706,"5min":15.35,"15min":222.16499999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.873,"5min":2.5219999999999998,"15min":4.7089999999999996},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":2.859,"5min":14.545999999999999,"15min":222.227},"last":0.38300000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:42:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.8859999999999999,"15min":2.266},"min":{"1min":0.29299999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":3.694,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.443,"5min":1.9850000000000001,"15min":2.2639999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28299999999999997},"max":{"1min":0.59799999999999998,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.32500000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:42:30 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.837,"15min":1.9730000000000001},"min":{"1min":0.33700000000000002,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":1.9730000000000001,"5min":14.515000000000001,"15min":16.808},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.47199999999999998,"5min":1.6000000000000001,"15min":1.8480000000000001},"min":{"1min":0.34799999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64000000000000001,"5min":8.9740000000000002,"15min":17.507000000000001},"last":0.40200000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.617,"15min":1.8360000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.35499999999999998},"max":{"1min":1.5029999999999999,"5min":8.9920000000000009,"15min":17.535},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.54900000000000004,"5min":1.879,"15min":2.137},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":1.5760000000000001,"5min":10.446,"15min":18.521000000000001},"last":0.433}]},{"osd":7,"last update":"Sun Oct 1 09:42:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":1.671,"15min":2.1139999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":0.77100000000000002,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.83599999999999997,"5min":1.97,"15min":2.4020000000000001},"min":{"1min":0.41899999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":3.7320000000000002,"5min":48.274000000000001,"15min":48.274000000000001},"last":0.46700000000000003}]}]},{"osd":2,"up_from":0,"seq":425201762671,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599144,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1929944,"kb_used_data":1097808,"kb_used_omap":3,"kb_used_meta":832124,"kb_avail":91815208,"statfs":{"total":95995035648,"available":94018772992,"internally_reserved":0,"allocated":1124155392,"data_stored":1122094548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":852095719},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":5.9850000000000003,"5min":5.5869999999999997,"15min":6.0170000000000003},"min":{"1min":0.216,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":64.504000000000005,"5min":118.748,"15min":137.34399999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":6.024,"5min":5.7830000000000004,"15min":6.0090000000000003},"min":{"1min":0.24299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":64.448999999999998,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.32300000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":5.149,"15min":5.4130000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":6.0250000000000004,"5min":108.84999999999999,"15min":162.22999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":5.3860000000000001,"15min":5.718},"min":{"1min":0.23100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.931,"5min":109.13200000000001,"15min":162.70699999999999},"last":0.40699999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":2.5070000000000001,"15min":2.5},"min":{"1min":0.27000000000000002,"5min":0.23000000000000001,"15min":0.20799999999999999},"max":{"1min":7.141,"5min":14.254,"15min":43.222999999999999},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.77200000000000002,"5min":1.4359999999999999,"15min":2.823},"min":{"1min":0.28299999999999997,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":3.4550000000000001,"5min":11.202,"15min":78.594999999999999},"last":0.34300000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":2.423,"15min":1.972},"min":{"1min":0.312,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":6.8890000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.96399999999999997,"5min":2.5800000000000001,"15min":2.202},"min":{"1min":0.33900000000000002,"5min":0.28699999999999998,"15min":0.27900000000000003},"max":{"1min":6.0940000000000003,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.39800000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.766,"15min":2.7850000000000001},"min":{"1min":0.32500000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.70499999999999996,"5min":10.919,"15min":28.117999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.54600000000000004,"5min":2.0139999999999998,"15min":2.8879999999999999},"min":{"1min":0.35399999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.5580000000000001,"5min":12.989000000000001,"15min":28.34},"last":0.38600000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":2.7970000000000002,"15min":2.3159999999999998},"min":{"1min":0.34399999999999997,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":1.621,"5min":73.887,"15min":73.887},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.66200000000000003,"5min":2.9670000000000001,"15min":2.3090000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.32800000000000001},"max":{"1min":1.8440000000000001,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.46000000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:42:02 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":2.3839999999999999,"15min":2.4249999999999998},"min":{"1min":0.41499999999999998,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":7.0620000000000003,"5min":13.085000000000001,"15min":52.006},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.774,"15min":2.1360000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.2690000000000001,"5min":11.109999999999999,"15min":51.890999999999998},"last":0.56899999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019394,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4494276,"kb_used_data":3595900,"kb_used_omap":0,"kb_used_meta":898368,"kb_avail":89250876,"statfs":{"total":95995035648,"available":91392897024,"internally_reserved":0,"allocated":3682201600,"data_stored":3681269670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919928832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:53 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":3.129,"15min":4.7549999999999999},"min":{"1min":0.26100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.66400000000000003,"5min":79.837000000000003,"15min":155.315},"last":0.439},{"interface":"front","average":{"1min":0.53900000000000003,"5min":3.2949999999999999,"15min":5.1200000000000001},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.251},"max":{"1min":2.843,"5min":78.983000000000004,"15min":155.334},"last":0.45400000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:42:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":6.8879999999999999,"15min":6.556},"min":{"1min":0.252,"5min":0.224,"15min":0.224},"max":{"1min":2.8889999999999998,"5min":146.929,"15min":190.61199999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.45500000000000002,"5min":6.8300000000000001,"15min":7.266},"min":{"1min":0.27500000000000002,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":0.624,"5min":146.91,"15min":253.273},"last":0.35699999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:42:43 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":1.2809999999999999,"15min":2.6019999999999999},"min":{"1min":0.34799999999999998,"5min":0.30399999999999999,"15min":0.28299999999999997},"max":{"1min":2.8210000000000002,"5min":10.161,"15min":170.45400000000001},"last":0.376},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.4410000000000001,"15min":2.2730000000000001},"min":{"1min":0.38400000000000001,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":2.1930000000000001,"5min":10.625,"15min":58.771000000000001},"last":0.48799999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:42:58 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":1.1799999999999999,"15min":2.7519999999999998},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.67500000000000004,"5min":7.5830000000000002,"15min":140.881},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.47099999999999997,"5min":1.208,"15min":2.976},"min":{"1min":0.33300000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.69599999999999995,"5min":9.3140000000000001,"15min":140.911},"last":0.46300000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:42:31 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":2.1579999999999999,"15min":1.5900000000000001},"min":{"1min":0.26800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.1190000000000002,"5min":110.003,"15min":110.003},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":2.069,"15min":1.405},"min":{"1min":0.317,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.0780000000000001,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.42699999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:42:09 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":0.90000000000000002,"15min":0.996},"min":{"1min":0.34599999999999997,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":9.9849999999999994,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.504},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.087,"15min":1.083},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":4.0220000000000002,"5min":6.9139999999999997,"15min":10.076000000000001},"last":0.51000000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:42:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":1.159,"15min":1.1799999999999999},"min":{"1min":0.38100000000000001,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":1.825,"5min":38.034999999999997,"15min":38.034999999999997},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.623,"5min":1.47,"15min":1.462},"min":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":2.2149999999999999,"5min":37.723999999999997,"15min":37.723999999999997},"last":0.53200000000000003}]}]},{"osd":5,"up_from":379,"seq":1627792605301,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2228208,"kb_used_data":1863656,"kb_used_omap":13,"kb_used_meta":364530,"kb_avail":91516944,"statfs":{"total":95995035648,"available":93713350656,"internally_reserved":0,"allocated":1908383744,"data_stored":1907467718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":373279218},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2,2,2,0,3,4,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":1.897,"15min":2.258},"min":{"1min":0.23200000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.50700000000000001,"5min":12.571999999999999,"15min":84.340999999999994},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":1.5289999999999999,"15min":2.1349999999999998},"min":{"1min":0.27300000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60299999999999998,"5min":8.6460000000000008,"15min":84.356999999999999},"last":0.504}]},{"osd":1,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":6.7510000000000003,"15min":5.8460000000000001},"min":{"1min":0.26800000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.55400000000000005,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.41199999999999998,"5min":7.3490000000000002,"15min":5.7889999999999997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":0.63200000000000001,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.432}]},{"osd":2,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.6020000000000001,"15min":1.6519999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.56599999999999995,"5min":8.7040000000000006,"15min":14.199999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.47799999999999998,"5min":1.6220000000000001,"15min":1.661},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":1.931,"5min":8.2370000000000001,"15min":14.430999999999999},"last":0.36799999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":1.796,"15min":1.5269999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.8720000000000001,"5min":12.734,"15min":14.004},"last":0.373},{"interface":"front","average":{"1min":0.55300000000000005,"5min":1.8580000000000001,"15min":1.673},"min":{"1min":0.39800000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.8819999999999999,"5min":11.766,"15min":13.991},"last":0.48499999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":2.2200000000000002,"15min":1.4670000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.873,"5min":122.971,"15min":122.971},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.46000000000000002,"5min":2.3140000000000001,"15min":1.806},"min":{"1min":0.307,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.78400000000000003,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.28999999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":1.2849999999999999,"15min":1.0780000000000001},"min":{"1min":0.34499999999999997,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":0.68200000000000005,"5min":12.545,"15min":12.545},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.998,"15min":1.3640000000000001},"min":{"1min":0.34100000000000003,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":0.69999999999999996,"5min":6.4729999999999999,"15min":14.193},"last":0.45100000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":1.141,"15min":1.3200000000000001},"min":{"1min":0.32300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.63,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":1.2649999999999999,"15min":1.097},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.318},"max":{"1min":0.70399999999999996,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.47299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3679326208,"data_stored":3679322136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":905707520,"data_stored":905707520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1117847552,"data_stored":1117847552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3672969216,"data_stored":3672965144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1902051328,"data_stored":1902051328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2102988800,"data_stored":2102984728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1501888512,"data_stored":1501888512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:43:07.871 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:43:08.030 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:08.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:08.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:08.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:07 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8249/219342 objects degraded (3.761%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:08.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:07 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8249/219342 objects degraded (3.761%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:08.071 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:08.135 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:43:08.161 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:07 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8249/219342 objects degraded (3.761%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:08.164 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:08.165 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:08.212 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:43:08.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:08.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:43:08.320 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:08.351 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:08.547 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:08.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:08.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:08.650 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:08.682 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:08.726 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:08.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:08.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:08.830 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:08.838 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:43:08.860 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:08.862 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:08 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2153769080' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:43:08.862 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:08 smithi138 ceph-mon[192513]: pgmap v1493: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 8249/218793 objects degraded (3.770%); 4468/218793 objects misplaced (2.042%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:43:08.862 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:08 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2153769080' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:43:08.862 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:08 smithi138 ceph-mon[185886]: pgmap v1493: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 8249/218793 objects degraded (3.770%); 4468/218793 objects misplaced (2.042%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:43:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:08 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2153769080' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:43:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:08 smithi190 ceph-mon[160435]: pgmap v1493: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 8249/218793 objects degraded (3.770%); 4468/218793 objects misplaced (2.042%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:43:10.094 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:09 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:10.095 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:09 smithi138 ceph-mon[192513]: failed to encode map e520 with expected crc 2023-10-01T09:43:10.095 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:09 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2153769080' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:43:10.095 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:09 smithi138 ceph-mon[192513]: osdmap e520: 8 total, 8 up, 7 in 2023-10-01T09:43:10.095 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:09 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:10.095 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:09 smithi138 ceph-mon[185886]: failed to encode map e520 with expected crc 2023-10-01T09:43:10.095 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:09 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2153769080' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:43:10.095 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:09 smithi138 ceph-mon[185886]: osdmap e520: 8 total, 8 up, 7 in 2023-10-01T09:43:10.213 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:10.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:10.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:10.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:09 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:09 smithi190 ceph-mon[160435]: failed to encode map e520 with expected crc 2023-10-01T09:43:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:09 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2153769080' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:43:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:09 smithi190 ceph-mon[160435]: osdmap e520: 8 total, 8 up, 7 in 2023-10-01T09:43:10.318 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:10.344 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:10.628 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:10.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:10.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:10.732 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:10.743 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:10.765 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:10.811 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:10.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:10.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:10.914 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:10.945 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:10.993 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:10.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:10.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:11.098 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:11.116 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:43:11.116 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:43:11.130 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:11.147 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:11.147 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:11.147 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:11 smithi138 ceph-mon[192513]: pgmap v1495: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 8249/218586 objects degraded (3.774%); 4468/218586 objects misplaced (2.044%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:43:11.147 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:43:11.147 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:43:11.148 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:43:11.148 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:11.148 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:11.148 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:11.148 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:11 smithi138 ceph-mon[185886]: pgmap v1495: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 8249/218586 objects degraded (3.774%); 4468/218586 objects misplaced (2.044%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:43:11.148 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:43:11.148 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:43:11.148 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:43:11.148 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:11.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:11 smithi190 ceph-mon[160435]: pgmap v1495: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 109 op/s; 8249/218586 objects degraded (3.774%); 4468/218586 objects misplaced (2.044%); 512 KiB/s, 7 objects/s recovering 2023-10-01T09:43:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:43:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:43:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:43:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:11.478 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:11.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:11.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:11.582 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:11.609 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:11.659 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:11.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:11.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:11.765 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:11.794 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:11.845 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:11.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:11.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:11.912 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:43:11.949 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:11.982 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:12.121 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:43:12.324 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:43:12.344 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:43:12.371 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:43:12.390 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:43:12.407 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:43:12.426 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:43:12.443 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:43:12.461 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:43:12.462 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:43:12.462 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:43:12.462 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:43:12.462 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:43:12.462 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:43:12.462 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:43:12.462 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:43:12.462 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:43:12.487 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:12 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2380547751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:12.488 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:12 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2380547751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:12.537 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:12.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:12.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:12.642 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:12.670 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:12.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:12 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2380547751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:13.241 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:13.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:13.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:13.348 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:13.379 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:13.388 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:13.413 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:13.440 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:13.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:13.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:13.524 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:13 smithi138 ceph-mon[185886]: pgmap v1496: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 8188/217398 objects degraded (3.766%); 4439/217398 objects misplaced (2.042%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:43:13.524 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:13 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8188/217398 objects degraded (3.766%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:13.524 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:13 smithi138 ceph-mon[192513]: pgmap v1496: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 8188/217398 objects degraded (3.766%); 4439/217398 objects misplaced (2.042%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:43:13.524 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:13 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8188/217398 objects degraded (3.766%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:13.545 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:13.572 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:13.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:13 smithi190 ceph-mon[160435]: pgmap v1496: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 102 op/s; 8188/217398 objects degraded (3.766%); 4439/217398 objects misplaced (2.042%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:43:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:13 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8188/217398 objects degraded (3.766%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:13.873 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:13.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:13.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:13.979 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:43:14.006 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:14.052 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:43:14.052 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 8188/217398 objects degraded (3.766%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2014,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":520,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":72466,"data_bytes":4752021451,"bytes_used":19910049792,"bytes_avail":652055199744,"bytes_total":671965249536,"degraded_objects":8188,"degraded_total":217398,"degraded_ratio":0.037663639959889236,"misplaced_objects":4439,"misplaced_total":217398,"misplaced_ratio":0.020418771101850065,"recovering_objects_per_sec":8,"recovering_bytes_per_sec":537248,"recovering_keys_per_sec":0,"num_objects_recovered":82,"num_bytes_recovered":5373952,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":102},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":33,"modified":"2023-10-01T09:41:58.284615+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:43:14.055 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:43:14.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:14.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:43:14.163 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:14.191 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:14.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/928347251' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:43:14.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/928347251' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:43:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/928347251' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:43:15.237 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:15.239 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:43:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:43:14] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-01T09:43:15.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:15.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:15.258 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:43:15.342 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:15.371 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:15.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:15 smithi138 ceph-mon[185886]: pgmap v1497: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 103 op/s; 8188/217299 objects degraded (3.768%); 4439/217299 objects misplaced (2.043%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:43:15.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:15 smithi138 ceph-mon[192513]: pgmap v1497: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 103 op/s; 8188/217299 objects degraded (3.768%); 4439/217299 objects misplaced (2.043%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:43:15.668 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:43:15.668 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:43:15.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:15 smithi190 ceph-mon[160435]: pgmap v1497: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 103 op/s; 8188/217299 objects degraded (3.768%); 4439/217299 objects misplaced (2.043%); 525 KiB/s, 8 objects/s recovering 2023-10-01T09:43:15.907 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:15.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:15.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:16.012 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:16.041 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:16.101 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:16.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:16.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:16.183 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:43:16.183 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:43:16.183 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:43:16.183 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:43:16.183 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:43:16.206 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:16.232 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:16.277 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:16.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:16.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:16.324 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:16.382 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:16.410 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:16.463 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:16.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:16.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:16.571 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:16.600 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:16.744 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:16.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:16.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:16.848 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:16.879 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:17.329 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:17.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:17.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:17.378 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:43:17.432 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:17.459 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:17.503 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:17.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:17.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:17.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:17 smithi138 ceph-mon[185886]: pgmap v1498: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 113 op/s; 8137/215931 objects degraded (3.768%); 4402/215931 objects misplaced (2.039%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:43:17.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:17 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1176761652' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:43:17.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:17 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:43:17.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:17 smithi138 ceph-mon[192513]: pgmap v1498: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 113 op/s; 8137/215931 objects degraded (3.768%); 4402/215931 objects misplaced (2.039%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:43:17.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:17 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1176761652' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:43:17.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:17 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:43:17.610 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:43:17.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:17 smithi190 ceph-mon[160435]: pgmap v1498: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 113 op/s; 8137/215931 objects degraded (3.768%); 4402/215931 objects misplaced (2.039%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:43:17.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:17 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1176761652' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:43:17.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:17 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:43:17.636 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:17.681 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:43:17.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:17.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:43:17.786 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:17.816 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:17.857 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:43:17.868 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:17.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:17.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:17.925 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:17.974 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:18.006 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:18.156 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:18.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:18.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:18.261 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:18.288 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:18.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:18 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:43:18.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:18 smithi138 ceph-mon[192513]: failed to encode map e521 with expected crc 2023-10-01T09:43:18.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:18 smithi138 ceph-mon[192513]: failed to encode map e521 with expected crc 2023-10-01T09:43:18.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:18 smithi138 ceph-mon[192513]: failed to encode map e521 with expected crc 2023-10-01T09:43:18.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:18 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:43:18.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:18 smithi138 ceph-mon[192513]: osdmap e521: 8 total, 8 up, 7 in 2023-10-01T09:43:18.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:18 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8137/215931 objects degraded (3.768%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:18.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:18 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:43:18.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:18 smithi138 ceph-mon[185886]: failed to encode map e521 with expected crc 2023-10-01T09:43:18.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:18 smithi138 ceph-mon[185886]: failed to encode map e521 with expected crc 2023-10-01T09:43:18.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:18 smithi138 ceph-mon[185886]: failed to encode map e521 with expected crc 2023-10-01T09:43:18.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:18 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:43:18.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:18 smithi138 ceph-mon[185886]: osdmap e521: 8 total, 8 up, 7 in 2023-10-01T09:43:18.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:18 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8137/215931 objects degraded (3.768%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:18.645 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:18.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:18.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:18.748 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:18.779 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:18 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:43:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:18 smithi190 ceph-mon[160435]: failed to encode map e521 with expected crc 2023-10-01T09:43:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:18 smithi190 ceph-mon[160435]: failed to encode map e521 with expected crc 2023-10-01T09:43:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:18 smithi190 ceph-mon[160435]: failed to encode map e521 with expected crc 2023-10-01T09:43:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:18 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:43:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:18 smithi190 ceph-mon[160435]: osdmap e521: 8 total, 8 up, 7 in 2023-10-01T09:43:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:18 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8137/215931 objects degraded (3.768%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:18.930 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:18.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:18.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:19.008 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:43:19.028 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:43:19.035 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:19.050 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:43:19.065 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:19.073 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:43:19.099 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:43:19.125 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:43:19.144 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:43:19.166 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:43:19.167 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:43:19.167 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:43:19.167 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:43:19.167 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:43:19.167 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:43:19.167 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:43:19.167 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:43:19.168 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:43:19.223 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:19.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:19.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:19.256 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:19.328 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:43:19.357 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:19.409 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:43:19.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:19.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:43:19.517 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:43:19.545 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:19.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:19 smithi138 ceph-mon[192513]: pgmap v1500: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 123 op/s; 8137/215292 objects degraded (3.780%); 4402/215292 objects misplaced (2.045%); 594 KiB/s, 9 objects/s recovering 2023-10-01T09:43:19.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:19 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/919629216' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:19 smithi138 ceph-mon[185886]: pgmap v1500: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 123 op/s; 8137/215292 objects degraded (3.780%); 4402/215292 objects misplaced (2.045%); 594 KiB/s, 9 objects/s recovering 2023-10-01T09:43:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/919629216' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:19.591 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:43:19.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:19.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:43:19.696 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:19.724 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:19.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:19 smithi190 ceph-mon[160435]: pgmap v1500: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 123 op/s; 8137/215292 objects degraded (3.780%); 4402/215292 objects misplaced (2.045%); 594 KiB/s, 9 objects/s recovering 2023-10-01T09:43:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:19 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/919629216' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:20.287 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:20.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:20.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:20.390 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:20.418 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:20.475 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:20.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:20.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:20.581 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:43:20.610 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:20.657 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:43:20.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:20.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:43:20.765 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:20.791 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:20.836 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:20.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:20.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:20.857 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:43:20.942 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:20.969 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:21.233 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:21.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:21.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:21.338 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:43:21.340 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:21.340 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:43:21.370 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:21.468 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:21 smithi138 ceph-mon[185886]: pgmap v1501: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 117 op/s; 8137/215058 objects degraded (3.784%); 4402/215058 objects misplaced (2.047%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:43:21.469 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:21 smithi138 ceph-mon[192513]: pgmap v1501: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 117 op/s; 8137/215058 objects degraded (3.784%); 4402/215058 objects misplaced (2.047%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:43:21.618 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:21.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:21.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:21.673 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:43:21.673 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:43:21.722 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:21.723 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:21.758 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:21.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:21 smithi190 ceph-mon[160435]: pgmap v1501: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 117 op/s; 8137/215058 objects degraded (3.784%); 4402/215058 objects misplaced (2.047%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:43:22.349 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:43:22.522 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:22.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:22.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:22.577 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:43:22.628 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:22.652 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:22 smithi138 ceph-mon[185886]: from='client.36822 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:22.652 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:22 smithi138 ceph-mon[192513]: from='client.36822 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:22.655 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:22.751 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:22.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:22.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:22.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:22 smithi190 ceph-mon[160435]: from='client.36822 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:22.855 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:22.890 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:22.998 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1501,"stamp":"2023-10-01T09:43:20.305700+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4700903371,"num_objects":71686,"num_object_clones":0,"num_object_copies":215058,"num_objects_missing_on_primary":2984,"num_objects_missing":2984,"num_objects_degraded":8137,"num_objects_misplaced":4402,"num_objects_unfound":0,"num_objects_dirty":71686,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":95888,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8115,"num_bytes_recovered":518685711,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95952,"ondisk_log_size":95952,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19272492,"kb_used_data":14235572,"kb_used_omap":39,"kb_used_meta":5036824,"kb_avail":636943572,"statfs":{"total":671965249536,"available":652230217728,"internally_reserved":0,"allocated":14577225728,"data_stored":14567413186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5157708399},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1,0,1,3,1,0,2,5,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-77070336,"num_objects":-1176,"num_object_clones":0,"num_object_copies":-3528,"num_objects_missing_on_primary":-90,"num_objects_missing":-90,"num_objects_degraded":-112,"num_objects_misplaced":-66,"num_objects_unfound":0,"num_objects_dirty":-1176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1176,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":88,"num_bytes_recovered":5767168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002521"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":296,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.015423+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:43:00.015423+0000","last_peered":"2023-10-01T09:43:00.015423+0000","last_clean":"2023-10-01T09:43:00.015423+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:43:00.015423+0000","last_undegraded":"2023-10-01T09:43:00.015423+0000","last_fullsized":"2023-10-01T09:43:00.015423+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":140,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050426+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:43:09.050426+0000","last_peered":"2023-10-01T09:43:09.050426+0000","last_clean":"2023-10-01T09:43:09.050426+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:43:09.050426+0000","last_undegraded":"2023-10-01T09:43:09.050426+0000","last_fullsized":"2023-10-01T09:43:09.050426+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":520,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050213+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:43:09.050213+0000","last_peered":"2023-10-01T09:43:09.050213+0000","last_clean":"2023-10-01T09:43:09.050213+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:43:09.050213+0000","last_undegraded":"2023-10-01T09:43:09.050213+0000","last_fullsized":"2023-10-01T09:43:09.050213+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":578,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:01.856936+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:43:01.856936+0000","last_peered":"2023-10-01T09:43:01.856936+0000","last_clean":"2023-10-01T09:43:01.856936+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:43:01.856936+0000","last_undegraded":"2023-10-01T09:43:01.856936+0000","last_fullsized":"2023-10-01T09:43:01.856936+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":459,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.776955+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:42:59.776955+0000","last_peered":"2023-10-01T09:42:59.776955+0000","last_clean":"2023-10-01T09:42:59.776955+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:42:59.776955+0000","last_undegraded":"2023-10-01T09:42:59.776955+0000","last_fullsized":"2023-10-01T09:42:59.776955+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":201,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.842655+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:42:59.842655+0000","last_peered":"2023-10-01T09:42:59.842655+0000","last_clean":"2023-10-01T09:42:59.842655+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:42:59.842655+0000","last_undegraded":"2023-10-01T09:42:59.842655+0000","last_fullsized":"2023-10-01T09:42:59.842655+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":577,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.842731+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:42:59.842731+0000","last_peered":"2023-10-01T09:42:59.842731+0000","last_clean":"2023-10-01T09:42:59.842731+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:42:59.842731+0000","last_undegraded":"2023-10-01T09:42:59.842731+0000","last_fullsized":"2023-10-01T09:42:59.842731+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":507,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:08.830193+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:43:08.830193+0000","last_peered":"2023-10-01T09:43:08.830193+0000","last_clean":"2023-10-01T09:43:08.830193+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:43:08.830193+0000","last_undegraded":"2023-10-01T09:43:08.830193+0000","last_fullsized":"2023-10-01T09:43:08.830193+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":232,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.833982+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:42:59.833982+0000","last_peered":"2023-10-01T09:42:59.833982+0000","last_clean":"2023-10-01T09:42:59.833982+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:42:59.833982+0000","last_undegraded":"2023-10-01T09:42:59.833982+0000","last_fullsized":"2023-10-01T09:42:59.833982+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":39,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.771782+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:42:59.771782+0000","last_peered":"2023-10-01T09:42:59.771782+0000","last_clean":"2023-10-01T09:42:59.771782+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:42:59.771782+0000","last_undegraded":"2023-10-01T09:42:59.771782+0000","last_fullsized":"2023-10-01T09:42:59.771782+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":45,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.772464+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:42:59.772464+0000","last_peered":"2023-10-01T09:42:59.772464+0000","last_clean":"2023-10-01T09:42:59.772464+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:42:59.772464+0000","last_undegraded":"2023-10-01T09:42:59.772464+0000","last_fullsized":"2023-10-01T09:42:59.772464+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":44,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050200+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:43:09.050200+0000","last_peered":"2023-10-01T09:43:09.050200+0000","last_clean":"2023-10-01T09:43:09.050200+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:43:09.050200+0000","last_undegraded":"2023-10-01T09:43:09.050200+0000","last_fullsized":"2023-10-01T09:43:09.050200+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":421,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.015622+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:43:00.015622+0000","last_peered":"2023-10-01T09:43:00.015622+0000","last_clean":"2023-10-01T09:43:00.015622+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:43:00.015622+0000","last_undegraded":"2023-10-01T09:43:00.015622+0000","last_fullsized":"2023-10-01T09:43:00.015622+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":302,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.015817+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:43:00.015817+0000","last_peered":"2023-10-01T09:43:00.015817+0000","last_clean":"2023-10-01T09:43:00.015817+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:43:00.015817+0000","last_undegraded":"2023-10-01T09:43:00.015817+0000","last_fullsized":"2023-10-01T09:43:00.015817+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":638,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.771868+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:42:59.771868+0000","last_peered":"2023-10-01T09:42:59.771868+0000","last_clean":"2023-10-01T09:42:59.771868+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:42:59.771868+0000","last_undegraded":"2023-10-01T09:42:59.771868+0000","last_fullsized":"2023-10-01T09:42:59.771868+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":375,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.772392+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:42:59.772392+0000","last_peered":"2023-10-01T09:42:59.772392+0000","last_clean":"2023-10-01T09:42:59.772392+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:42:59.772392+0000","last_undegraded":"2023-10-01T09:42:59.772392+0000","last_fullsized":"2023-10-01T09:42:59.772392+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":48,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.771937+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:42:59.771937+0000","last_peered":"2023-10-01T09:42:59.771937+0000","last_clean":"2023-10-01T09:42:59.771937+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:42:59.771937+0000","last_undegraded":"2023-10-01T09:42:59.771937+0000","last_fullsized":"2023-10-01T09:42:59.771937+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":34,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.833826+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:42:59.833826+0000","last_peered":"2023-10-01T09:42:59.833826+0000","last_clean":"2023-10-01T09:42:59.833826+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:42:59.833826+0000","last_undegraded":"2023-10-01T09:42:59.833826+0000","last_fullsized":"2023-10-01T09:42:59.833826+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":531,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.772040+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:42:59.772040+0000","last_peered":"2023-10-01T09:42:59.772040+0000","last_clean":"2023-10-01T09:42:59.772040+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:42:59.772040+0000","last_undegraded":"2023-10-01T09:42:59.772040+0000","last_fullsized":"2023-10-01T09:42:59.772040+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":542,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.777342+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:42:59.777342+0000","last_peered":"2023-10-01T09:42:59.777342+0000","last_clean":"2023-10-01T09:42:59.777342+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:42:59.777342+0000","last_undegraded":"2023-10-01T09:42:59.777342+0000","last_fullsized":"2023-10-01T09:42:59.777342+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":398,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.776681+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:42:59.776681+0000","last_peered":"2023-10-01T09:42:59.776681+0000","last_clean":"2023-10-01T09:42:59.776681+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:42:59.776681+0000","last_undegraded":"2023-10-01T09:42:59.776681+0000","last_fullsized":"2023-10-01T09:42:59.776681+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":48,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.772661+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:42:59.772661+0000","last_peered":"2023-10-01T09:42:59.772661+0000","last_clean":"2023-10-01T09:42:59.772661+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:42:59.772661+0000","last_undegraded":"2023-10-01T09:42:59.772661+0000","last_fullsized":"2023-10-01T09:42:59.772661+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"520'5935","reported_seq":5988,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:15.244348+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:43:15.244348+0000","last_peered":"2023-10-01T09:43:15.244348+0000","last_clean":"2023-10-01T09:43:15.244348+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:43:15.244348+0000","last_undegraded":"2023-10-01T09:43:15.244348+0000","last_fullsized":"2023-10-01T09:43:15.244348+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5935,"log_dups_size":0,"ondisk_log_size":5935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292093952,"num_objects":4457,"num_object_clones":0,"num_object_copies":13371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5935,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":43,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.772559+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:42:59.772559+0000","last_peered":"2023-10-01T09:42:59.772559+0000","last_clean":"2023-10-01T09:42:59.772559+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:42:59.772559+0000","last_undegraded":"2023-10-01T09:42:59.772559+0000","last_fullsized":"2023-10-01T09:42:59.772559+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":240,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.833591+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:42:59.833591+0000","last_peered":"2023-10-01T09:42:59.833591+0000","last_clean":"2023-10-01T09:42:59.833591+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:42:59.833591+0000","last_undegraded":"2023-10-01T09:42:59.833591+0000","last_fullsized":"2023-10-01T09:42:59.833591+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"520'5903","reported_seq":8086,"reported_epoch":520,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:43:15.748395+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:43:15.748395+0000","last_peered":"2023-10-01T09:43:15.748395+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:43:15.748395+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5903,"log_dups_size":0,"ondisk_log_size":5903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295108608,"num_objects":4503,"num_object_clones":0,"num_object_copies":13509,"num_objects_missing_on_primary":2984,"num_objects_missing":2984,"num_objects_degraded":7419,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5903,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":700,"num_bytes_recovered":45875200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1451},{"shards":"0,4,6","objects":2984},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":514,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.772732+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:42:59.772732+0000","last_peered":"2023-10-01T09:42:59.772732+0000","last_clean":"2023-10-01T09:42:59.772732+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:42:59.772732+0000","last_undegraded":"2023-10-01T09:42:59.772732+0000","last_fullsized":"2023-10-01T09:42:59.772732+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":576,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050187+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:43:09.050187+0000","last_peered":"2023-10-01T09:43:09.050187+0000","last_clean":"2023-10-01T09:43:09.050187+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:43:09.050187+0000","last_undegraded":"2023-10-01T09:43:09.050187+0000","last_fullsized":"2023-10-01T09:43:09.050187+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"520'5947","reported_seq":6002,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:16.827717+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:43:16.827717+0000","last_peered":"2023-10-01T09:43:16.827717+0000","last_clean":"2023-10-01T09:43:16.827717+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:43:16.827717+0000","last_undegraded":"2023-10-01T09:43:16.827717+0000","last_fullsized":"2023-10-01T09:43:16.827717+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5947,"log_dups_size":0,"ondisk_log_size":5947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293928960,"num_objects":4485,"num_object_clones":0,"num_object_copies":13455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5947,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":600,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.772472+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:42:59.772472+0000","last_peered":"2023-10-01T09:42:59.772472+0000","last_clean":"2023-10-01T09:42:59.772472+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:42:59.772472+0000","last_undegraded":"2023-10-01T09:42:59.772472+0000","last_fullsized":"2023-10-01T09:42:59.772472+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":396,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050329+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:43:09.050329+0000","last_peered":"2023-10-01T09:43:09.050329+0000","last_clean":"2023-10-01T09:43:09.050329+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:43:09.050329+0000","last_undegraded":"2023-10-01T09:43:09.050329+0000","last_fullsized":"2023-10-01T09:43:09.050329+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"520'5895","reported_seq":12555,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:16.834955+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:43:16.834955+0000","last_peered":"2023-10-01T09:43:16.834955+0000","last_clean":"2023-10-01T09:43:16.834955+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:43:16.834955+0000","last_undegraded":"2023-10-01T09:43:16.834955+0000","last_fullsized":"2023-10-01T09:43:16.834955+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5895,"log_dups_size":0,"ondisk_log_size":5895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295370752,"num_objects":4507,"num_object_clones":0,"num_object_copies":13521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5895,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":240,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.833816+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:42:59.833816+0000","last_peered":"2023-10-01T09:42:59.833816+0000","last_clean":"2023-10-01T09:42:59.833816+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:42:59.833816+0000","last_undegraded":"2023-10-01T09:42:59.833816+0000","last_fullsized":"2023-10-01T09:42:59.833816+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":498,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.776932+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:42:59.776932+0000","last_peered":"2023-10-01T09:42:59.776932+0000","last_clean":"2023-10-01T09:42:59.776932+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:42:59.776932+0000","last_undegraded":"2023-10-01T09:42:59.776932+0000","last_fullsized":"2023-10-01T09:42:59.776932+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"521'5996","reported_seq":6052,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:20.072195+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:43:20.072195+0000","last_peered":"2023-10-01T09:43:20.072195+0000","last_clean":"2023-10-01T09:43:20.072195+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:43:20.072195+0000","last_undegraded":"2023-10-01T09:43:20.072195+0000","last_fullsized":"2023-10-01T09:43:20.072195+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5996,"log_dups_size":0,"ondisk_log_size":5996,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295698432,"num_objects":4512,"num_object_clones":0,"num_object_copies":13536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5996,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":100,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.016125+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:43:00.016125+0000","last_peered":"2023-10-01T09:43:00.016125+0000","last_clean":"2023-10-01T09:43:00.016125+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:43:00.016125+0000","last_undegraded":"2023-10-01T09:43:00.016125+0000","last_fullsized":"2023-10-01T09:43:00.016125+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":174,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.015679+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:43:00.015679+0000","last_peered":"2023-10-01T09:43:00.015679+0000","last_clean":"2023-10-01T09:43:00.015679+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:43:00.015679+0000","last_undegraded":"2023-10-01T09:43:00.015679+0000","last_fullsized":"2023-10-01T09:43:00.015679+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"520'5902","reported_seq":5985,"reported_epoch":520,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:43:15.254368+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:43:15.254368+0000","last_peered":"2023-10-01T09:43:15.254368+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:43:15.254368+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5902,"log_dups_size":0,"ondisk_log_size":5902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292683776,"num_objects":4466,"num_object_clones":0,"num_object_copies":13398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":708,"num_objects_misplaced":4402,"num_objects_unfound":0,"num_objects_dirty":4466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5902,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4402},{"shards":"4,5","objects":-644}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":418,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.015498+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:43:00.015498+0000","last_peered":"2023-10-01T09:43:00.015498+0000","last_clean":"2023-10-01T09:43:00.015498+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:43:00.015498+0000","last_undegraded":"2023-10-01T09:43:00.015498+0000","last_fullsized":"2023-10-01T09:43:00.015498+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":579,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050427+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:43:09.050427+0000","last_peered":"2023-10-01T09:43:09.050427+0000","last_clean":"2023-10-01T09:43:09.050427+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:43:09.050427+0000","last_undegraded":"2023-10-01T09:43:09.050427+0000","last_fullsized":"2023-10-01T09:43:09.050427+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"521'5903","reported_seq":5966,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.533618+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:43:17.533618+0000","last_peered":"2023-10-01T09:43:17.533618+0000","last_clean":"2023-10-01T09:43:17.533618+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:43:17.533618+0000","last_undegraded":"2023-10-01T09:43:17.533618+0000","last_fullsized":"2023-10-01T09:43:17.533618+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5903,"log_dups_size":0,"ondisk_log_size":5903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293273600,"num_objects":4475,"num_object_clones":0,"num_object_copies":13425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5903,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":414,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.015326+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:43:00.015326+0000","last_peered":"2023-10-01T09:43:00.015326+0000","last_clean":"2023-10-01T09:43:00.015326+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:43:00.015326+0000","last_undegraded":"2023-10-01T09:43:00.015326+0000","last_fullsized":"2023-10-01T09:43:00.015326+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":255,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.015978+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:43:00.015978+0000","last_peered":"2023-10-01T09:43:00.015978+0000","last_clean":"2023-10-01T09:43:00.015978+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:43:00.015978+0000","last_undegraded":"2023-10-01T09:43:00.015978+0000","last_fullsized":"2023-10-01T09:43:00.015978+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"520'6006","reported_seq":6059,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:16.174333+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:43:16.174333+0000","last_peered":"2023-10-01T09:43:16.174333+0000","last_clean":"2023-10-01T09:43:16.174333+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:43:16.174333+0000","last_undegraded":"2023-10-01T09:43:16.174333+0000","last_fullsized":"2023-10-01T09:43:16.174333+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6006,"log_dups_size":0,"ondisk_log_size":6006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294387712,"num_objects":4492,"num_object_clones":0,"num_object_copies":13476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6006,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":383,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050130+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:43:09.050130+0000","last_peered":"2023-10-01T09:43:09.050130+0000","last_clean":"2023-10-01T09:43:09.050130+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:43:09.050130+0000","last_undegraded":"2023-10-01T09:43:09.050130+0000","last_fullsized":"2023-10-01T09:43:09.050130+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":430,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.127199+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:43:09.127199+0000","last_peered":"2023-10-01T09:43:09.127199+0000","last_clean":"2023-10-01T09:43:09.127199+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:43:09.127199+0000","last_undegraded":"2023-10-01T09:43:09.127199+0000","last_fullsized":"2023-10-01T09:43:09.127199+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"520'5819","reported_seq":5872,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:15.966708+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:43:15.966708+0000","last_peered":"2023-10-01T09:43:15.966708+0000","last_clean":"2023-10-01T09:43:15.966708+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:43:15.966708+0000","last_undegraded":"2023-10-01T09:43:15.966708+0000","last_fullsized":"2023-10-01T09:43:15.966708+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5819,"log_dups_size":0,"ondisk_log_size":5819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285671424,"num_objects":4359,"num_object_clones":0,"num_object_copies":13077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5819,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":913,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.772303+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:42:59.772303+0000","last_peered":"2023-10-01T09:42:59.772303+0000","last_clean":"2023-10-01T09:42:59.772303+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:42:59.772303+0000","last_undegraded":"2023-10-01T09:42:59.772303+0000","last_fullsized":"2023-10-01T09:42:59.772303+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":250,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.833913+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:42:59.833913+0000","last_peered":"2023-10-01T09:42:59.833913+0000","last_clean":"2023-10-01T09:42:59.833913+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:42:59.833913+0000","last_undegraded":"2023-10-01T09:42:59.833913+0000","last_fullsized":"2023-10-01T09:42:59.833913+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050247+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:43:09.050247+0000","last_peered":"2023-10-01T09:43:09.050247+0000","last_clean":"2023-10-01T09:43:09.050247+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:43:09.050247+0000","last_undegraded":"2023-10-01T09:43:09.050247+0000","last_fullsized":"2023-10-01T09:43:09.050247+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"520'5869","reported_seq":5924,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:16.406132+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:43:16.406132+0000","last_peered":"2023-10-01T09:43:16.406132+0000","last_clean":"2023-10-01T09:43:16.406132+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:43:16.406132+0000","last_undegraded":"2023-10-01T09:43:16.406132+0000","last_fullsized":"2023-10-01T09:43:16.406132+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5869,"log_dups_size":0,"ondisk_log_size":5869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294977536,"num_objects":4501,"num_object_clones":0,"num_object_copies":13503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5869,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1185,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.833700+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:42:59.833700+0000","last_peered":"2023-10-01T09:42:59.833700+0000","last_clean":"2023-10-01T09:42:59.833700+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:42:59.833700+0000","last_undegraded":"2023-10-01T09:42:59.833700+0000","last_fullsized":"2023-10-01T09:42:59.833700+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":395,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.016038+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:43:00.016038+0000","last_peered":"2023-10-01T09:43:00.016038+0000","last_clean":"2023-10-01T09:43:00.016038+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:43:00.016038+0000","last_undegraded":"2023-10-01T09:43:00.016038+0000","last_fullsized":"2023-10-01T09:43:00.016038+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":223,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.833903+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:42:59.833903+0000","last_peered":"2023-10-01T09:42:59.833903+0000","last_clean":"2023-10-01T09:42:59.833903+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:42:59.833903+0000","last_undegraded":"2023-10-01T09:42:59.833903+0000","last_fullsized":"2023-10-01T09:42:59.833903+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"520'5905","reported_seq":13179,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:15.436478+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:43:15.436478+0000","last_peered":"2023-10-01T09:43:15.436478+0000","last_clean":"2023-10-01T09:43:15.436478+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:43:15.436478+0000","last_undegraded":"2023-10-01T09:43:15.436478+0000","last_fullsized":"2023-10-01T09:43:15.436478+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5905,"log_dups_size":0,"ondisk_log_size":5905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293011456,"num_objects":4471,"num_object_clones":0,"num_object_copies":13413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5905,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":865,"reported_epoch":520,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:43:08.834240+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:43:08.834240+0000","last_peered":"2023-10-01T09:43:08.834240+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:43:08.834240+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:43:08.834240+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":583,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050174+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:43:09.050174+0000","last_peered":"2023-10-01T09:43:09.050174+0000","last_clean":"2023-10-01T09:43:09.050174+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:43:09.050174+0000","last_undegraded":"2023-10-01T09:43:09.050174+0000","last_fullsized":"2023-10-01T09:43:09.050174+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":44,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.127142+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:43:09.127142+0000","last_peered":"2023-10-01T09:43:09.127142+0000","last_clean":"2023-10-01T09:43:09.127142+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:43:09.127142+0000","last_undegraded":"2023-10-01T09:43:09.127142+0000","last_fullsized":"2023-10-01T09:43:09.127142+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"521'5848","reported_seq":5904,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:19.347534+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:43:19.347534+0000","last_peered":"2023-10-01T09:43:19.347534+0000","last_clean":"2023-10-01T09:43:19.347534+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:43:19.347534+0000","last_undegraded":"2023-10-01T09:43:19.347534+0000","last_fullsized":"2023-10-01T09:43:19.347534+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5848,"log_dups_size":0,"ondisk_log_size":5848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289931264,"num_objects":4424,"num_object_clones":0,"num_object_copies":13272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5848,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1636,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.833680+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:42:59.833680+0000","last_peered":"2023-10-01T09:42:59.833680+0000","last_clean":"2023-10-01T09:42:59.833680+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:42:59.833680+0000","last_undegraded":"2023-10-01T09:42:59.833680+0000","last_fullsized":"2023-10-01T09:42:59.833680+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":550,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.771785+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:42:59.771785+0000","last_peered":"2023-10-01T09:42:59.771785+0000","last_clean":"2023-10-01T09:42:59.771785+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:42:59.771785+0000","last_undegraded":"2023-10-01T09:42:59.771785+0000","last_fullsized":"2023-10-01T09:42:59.771785+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":328,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.833708+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:42:59.833708+0000","last_peered":"2023-10-01T09:42:59.833708+0000","last_clean":"2023-10-01T09:42:59.833708+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:42:59.833708+0000","last_undegraded":"2023-10-01T09:42:59.833708+0000","last_fullsized":"2023-10-01T09:42:59.833708+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":318,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.015705+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:43:00.015705+0000","last_peered":"2023-10-01T09:43:00.015705+0000","last_clean":"2023-10-01T09:43:00.015705+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:43:00.015705+0000","last_undegraded":"2023-10-01T09:43:00.015705+0000","last_fullsized":"2023-10-01T09:43:00.015705+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"520'5975","reported_seq":6028,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:16.008785+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:43:16.008785+0000","last_peered":"2023-10-01T09:43:16.008785+0000","last_clean":"2023-10-01T09:43:16.008785+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:43:16.008785+0000","last_undegraded":"2023-10-01T09:43:16.008785+0000","last_fullsized":"2023-10-01T09:43:16.008785+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5975,"log_dups_size":0,"ondisk_log_size":5975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298647552,"num_objects":4557,"num_object_clones":0,"num_object_copies":13671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5975,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":727,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.772006+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:42:59.772006+0000","last_peered":"2023-10-01T09:42:59.772006+0000","last_clean":"2023-10-01T09:42:59.772006+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:42:59.772006+0000","last_undegraded":"2023-10-01T09:42:59.772006+0000","last_fullsized":"2023-10-01T09:42:59.772006+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":559,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.776738+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:42:59.776738+0000","last_peered":"2023-10-01T09:42:59.776738+0000","last_clean":"2023-10-01T09:42:59.776738+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:42:59.776738+0000","last_undegraded":"2023-10-01T09:42:59.776738+0000","last_fullsized":"2023-10-01T09:42:59.776738+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":558,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.777108+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:42:59.777108+0000","last_peered":"2023-10-01T09:42:59.777108+0000","last_clean":"2023-10-01T09:42:59.777108+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:42:59.777108+0000","last_undegraded":"2023-10-01T09:42:59.777108+0000","last_fullsized":"2023-10-01T09:42:59.777108+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"520'5979","reported_seq":6035,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:16.831890+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:43:16.831890+0000","last_peered":"2023-10-01T09:43:16.831890+0000","last_clean":"2023-10-01T09:43:16.831890+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:43:16.831890+0000","last_undegraded":"2023-10-01T09:43:16.831890+0000","last_fullsized":"2023-10-01T09:43:16.831890+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5979,"log_dups_size":0,"ondisk_log_size":5979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293601304,"num_objects":4481,"num_object_clones":0,"num_object_copies":13443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4481,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5979,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":958,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.772554+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:42:59.772554+0000","last_peered":"2023-10-01T09:42:59.772554+0000","last_clean":"2023-10-01T09:42:59.772554+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:42:59.772554+0000","last_undegraded":"2023-10-01T09:42:59.772554+0000","last_fullsized":"2023-10-01T09:42:59.772554+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":577,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.833820+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:42:59.833820+0000","last_peered":"2023-10-01T09:42:59.833820+0000","last_clean":"2023-10-01T09:42:59.833820+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:42:59.833820+0000","last_undegraded":"2023-10-01T09:42:59.833820+0000","last_fullsized":"2023-10-01T09:42:59.833820+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":430,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.833760+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:42:59.833760+0000","last_peered":"2023-10-01T09:42:59.833760+0000","last_clean":"2023-10-01T09:42:59.833760+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:42:59.833760+0000","last_undegraded":"2023-10-01T09:42:59.833760+0000","last_fullsized":"2023-10-01T09:42:59.833760+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"520'5930","reported_seq":5991,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:16.112254+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:43:16.112254+0000","last_peered":"2023-10-01T09:43:16.112254+0000","last_clean":"2023-10-01T09:43:16.112254+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:43:16.112254+0000","last_undegraded":"2023-10-01T09:43:16.112254+0000","last_fullsized":"2023-10-01T09:43:16.112254+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5930,"log_dups_size":0,"ondisk_log_size":5930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290062336,"num_objects":4426,"num_object_clones":0,"num_object_copies":13278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5930,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":459,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.772738+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:42:59.772738+0000","last_peered":"2023-10-01T09:42:59.772738+0000","last_clean":"2023-10-01T09:42:59.772738+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:42:59.772738+0000","last_undegraded":"2023-10-01T09:42:59.772738+0000","last_fullsized":"2023-10-01T09:42:59.772738+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1153,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.127184+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:43:09.127184+0000","last_peered":"2023-10-01T09:43:09.127184+0000","last_clean":"2023-10-01T09:43:09.127184+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:43:09.127184+0000","last_undegraded":"2023-10-01T09:43:09.127184+0000","last_fullsized":"2023-10-01T09:43:09.127184+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":48,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.771769+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:42:59.771769+0000","last_peered":"2023-10-01T09:42:59.771769+0000","last_clean":"2023-10-01T09:42:59.771769+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:42:59.771769+0000","last_undegraded":"2023-10-01T09:42:59.771769+0000","last_fullsized":"2023-10-01T09:42:59.771769+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"520'5970","reported_seq":6031,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:16.035114+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:43:16.035114+0000","last_peered":"2023-10-01T09:43:16.035114+0000","last_clean":"2023-10-01T09:43:16.035114+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:43:16.035114+0000","last_undegraded":"2023-10-01T09:43:16.035114+0000","last_fullsized":"2023-10-01T09:43:16.035114+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5970,"log_dups_size":0,"ondisk_log_size":5970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297664512,"num_objects":4542,"num_object_clones":0,"num_object_copies":13626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5970,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":52,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.772653+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:42:59.772653+0000","last_peered":"2023-10-01T09:42:59.772653+0000","last_clean":"2023-10-01T09:42:59.772653+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:42:59.772653+0000","last_undegraded":"2023-10-01T09:42:59.772653+0000","last_fullsized":"2023-10-01T09:42:59.772653+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":536,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.777195+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:42:59.777195+0000","last_peered":"2023-10-01T09:42:59.777195+0000","last_clean":"2023-10-01T09:42:59.777195+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:42:59.777195+0000","last_undegraded":"2023-10-01T09:42:59.777195+0000","last_fullsized":"2023-10-01T09:42:59.777195+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":243,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.833605+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:42:59.833605+0000","last_peered":"2023-10-01T09:42:59.833605+0000","last_clean":"2023-10-01T09:42:59.833605+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:42:59.833605+0000","last_undegraded":"2023-10-01T09:42:59.833605+0000","last_fullsized":"2023-10-01T09:42:59.833605+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":485,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.777029+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:42:59.777029+0000","last_peered":"2023-10-01T09:42:59.777029+0000","last_clean":"2023-10-01T09:42:59.777029+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:42:59.777029+0000","last_undegraded":"2023-10-01T09:42:59.777029+0000","last_fullsized":"2023-10-01T09:42:59.777029+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":600,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050425+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:43:09.050425+0000","last_peered":"2023-10-01T09:43:09.050425+0000","last_clean":"2023-10-01T09:43:09.050425+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:43:09.050425+0000","last_undegraded":"2023-10-01T09:43:09.050425+0000","last_fullsized":"2023-10-01T09:43:09.050425+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":255,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.833698+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:42:59.833698+0000","last_peered":"2023-10-01T09:42:59.833698+0000","last_clean":"2023-10-01T09:42:59.833698+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:42:59.833698+0000","last_undegraded":"2023-10-01T09:42:59.833698+0000","last_fullsized":"2023-10-01T09:42:59.833698+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":44,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:15.251918+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:43:15.251918+0000","last_peered":"2023-10-01T09:43:15.251918+0000","last_clean":"2023-10-01T09:43:15.251918+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:43:15.251918+0000","last_undegraded":"2023-10-01T09:43:15.251918+0000","last_fullsized":"2023-10-01T09:43:15.251918+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":255,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.834009+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:42:59.834009+0000","last_peered":"2023-10-01T09:42:59.834009+0000","last_clean":"2023-10-01T09:42:59.834009+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:42:59.834009+0000","last_undegraded":"2023-10-01T09:42:59.834009+0000","last_fullsized":"2023-10-01T09:42:59.834009+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":432,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:08.835184+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:43:08.835184+0000","last_peered":"2023-10-01T09:43:08.835184+0000","last_clean":"2023-10-01T09:43:08.835184+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:43:08.835184+0000","last_undegraded":"2023-10-01T09:43:08.835184+0000","last_fullsized":"2023-10-01T09:43:08.835184+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":61,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.127072+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:43:09.127072+0000","last_peered":"2023-10-01T09:43:09.127072+0000","last_clean":"2023-10-01T09:43:09.127072+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:43:09.127072+0000","last_undegraded":"2023-10-01T09:43:09.127072+0000","last_fullsized":"2023-10-01T09:43:09.127072+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":477,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.776700+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:42:59.776700+0000","last_peered":"2023-10-01T09:42:59.776700+0000","last_clean":"2023-10-01T09:42:59.776700+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:42:59.776700+0000","last_undegraded":"2023-10-01T09:42:59.776700+0000","last_fullsized":"2023-10-01T09:42:59.776700+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":676,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:42:59.777413+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:42:59.777413+0000","last_peered":"2023-10-01T09:42:59.777413+0000","last_clean":"2023-10-01T09:42:59.777413+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:42:59.777413+0000","last_undegraded":"2023-10-01T09:42:59.777413+0000","last_fullsized":"2023-10-01T09:42:59.777413+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":371,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-10-01T09:43:00.015240+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:43:00.015240+0000","last_peered":"2023-10-01T09:43:00.015240+0000","last_clean":"2023-10-01T09:43:00.015240+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:43:00.015240+0000","last_undegraded":"2023-10-01T09:43:00.015240+0000","last_fullsized":"2023-10-01T09:43:00.015240+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4696113176,"num_objects":71658,"num_object_clones":0,"num_object_copies":214974,"num_objects_missing_on_primary":2984,"num_objects_missing":2984,"num_objects_degraded":8127,"num_objects_misplaced":4402,"num_objects_unfound":0,"num_objects_dirty":71658,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":94782,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7604,"num_bytes_recovered":498335744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14502342656,"data_stored":14502330440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94782,"ondisk_log_size":94782,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430712,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2347536,"kb_used_data":1444424,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":91397616,"statfs":{"total":95995035648,"available":93591158784,"internally_reserved":0,"allocated":1479090176,"data_stored":1478154416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:56 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.4950000000000001,"15min":2.9569999999999999},"min":{"1min":0.246,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":1.3280000000000001,"5min":10.391999999999999,"15min":108.90000000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":1.8129999999999999,"15min":3.129},"min":{"1min":0.246,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.018,"5min":29.501000000000001,"15min":109.187},"last":0.219}]},{"osd":1,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":4.6749999999999998,"15min":5.3520000000000003},"min":{"1min":0.31900000000000001,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":1.2949999999999999,"5min":229.161,"15min":229.161},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.47399999999999998,"5min":4.6479999999999997,"15min":5.3689999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":0.90900000000000003,"5min":230.126,"15min":230.126},"last":0.29599999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:42:50 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":1.5369999999999999,"15min":1.891},"min":{"1min":0.33900000000000002,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":1.3560000000000001,"5min":14.866,"15min":14.901999999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.49299999999999999,"5min":1.7909999999999999,"15min":1.821},"min":{"1min":0.33500000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.96599999999999997,"5min":14.682,"15min":14.682},"last":0.40100000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":1.514,"15min":2.7330000000000001},"min":{"1min":0.30199999999999999,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":0.90700000000000003,"5min":35.104999999999997,"15min":80.352999999999994},"last":0.496},{"interface":"front","average":{"1min":0.50600000000000001,"5min":1.6930000000000001,"15min":2.7130000000000001},"min":{"1min":0.35399999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.153,"5min":35.118000000000002,"15min":80.364000000000004},"last":0.48899999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:42:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":1.2150000000000001,"15min":1.052},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.381,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":1.1080000000000001,"15min":1.03},"min":{"1min":0.314,"5min":0.30299999999999999,"15min":0.28000000000000003},"max":{"1min":0.97899999999999998,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.376}]},{"osd":5,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":2.8879999999999999,"15min":1.6419999999999999},"min":{"1min":0.318,"5min":0.30299999999999999,"15min":0.29599999999999999},"max":{"1min":0.64600000000000002,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.441},{"interface":"front","average":{"1min":0.41999999999999998,"5min":3.0779999999999998,"15min":1.6850000000000001},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54500000000000004,"5min":135.339,"15min":135.339},"last":0.42899999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.4379999999999999,"15min":1.173},"min":{"1min":0.313,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":2.448,"5min":11.58,"15min":11.853999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.44400000000000001,"5min":1.2789999999999999,"15min":1.1359999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":0.56699999999999995,"5min":14.584,"15min":14.584},"last":0.38300000000000001}]}]},{"osd":6,"up_from":294,"seq":1262720385223,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2354636,"kb_used_data":2012932,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":91390516,"statfs":{"total":95995035648,"available":93583888384,"internally_reserved":0,"allocated":2061242368,"data_stored":2060310559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":3.5720000000000001,"15min":4.0919999999999996},"min":{"1min":0.25,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":3.3340000000000001,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":4.0410000000000004,"15min":4.2210000000000001},"min":{"1min":0.29899999999999999,"5min":0.249,"15min":0.221},"max":{"1min":0.623,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.41899999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":4.7489999999999997,"15min":4.9740000000000002},"min":{"1min":0.30599999999999999,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":1.9410000000000001,"5min":168.947,"15min":168.947},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.53000000000000003,"5min":4.7969999999999997,"15min":4.8810000000000002},"min":{"1min":0.34699999999999998,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":1.962,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.34699999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":1.194,"15min":1.673},"min":{"1min":0.308,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.075,"5min":8.657,"15min":10.92},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.3759999999999999,"15min":1.673},"min":{"1min":0.314,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.58499999999999996,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.39200000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.3069999999999999,"15min":2.2789999999999999},"min":{"1min":0.35499999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.3759999999999999,"5min":9.0180000000000007,"15min":89.753},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.502,"5min":1.4359999999999999,"15min":2.157},"min":{"1min":0.34399999999999997,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":1.381,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.48899999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":1.5900000000000001,"15min":1.0800000000000001},"min":{"1min":0.30099999999999999,"5min":0.253,"15min":0.246},"max":{"1min":0.53400000000000003,"5min":63.134,"15min":63.134},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.5009999999999999,"15min":1.097},"min":{"1min":0.33700000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":1.5669999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.41099999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:42:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.93200000000000005,"15min":1.02},"min":{"1min":0.26700000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.8620000000000001,"5min":9.9939999999999998,"15min":12.023999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.66500000000000004,"5min":1.175,"15min":1.143},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.726,"5min":15.457000000000001,"15min":15.457000000000001},"last":0.45600000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.79500000000000004,"15min":1.46},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":1.4750000000000001,"5min":6.4119999999999999,"15min":94.245999999999995},"last":0.438},{"interface":"front","average":{"1min":0.48999999999999999,"5min":1.113,"15min":1.5569999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":0.68300000000000005,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.38400000000000001}]}]},{"osd":1,"up_from":94,"seq":403726926200,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1686844,"kb_used_data":868660,"kb_used_omap":7,"kb_used_meta":818168,"kb_avail":92058308,"statfs":{"total":95995035648,"available":94267707392,"internally_reserved":0,"allocated":889507840,"data_stored":887485358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":837804455},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.30299999999999999,"5min":2.5600000000000001,"15min":4.2009999999999996},"min":{"1min":0.20699999999999999,"5min":0.19400000000000001,"15min":0.182},"max":{"1min":0.57399999999999995,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.33600000000000002,"5min":3.0139999999999998,"15min":3.7090000000000001},"min":{"1min":0.26300000000000001,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":0.53400000000000003,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.315}]},{"osd":2,"last update":"Sun Oct 1 09:42:38 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":1.7929999999999999,"15min":2.7719999999999998},"min":{"1min":0.24099999999999999,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.83099999999999996,"5min":22.891999999999999,"15min":56.573},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.36699999999999999,"5min":2.2799999999999998,"15min":3.593},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":0.54400000000000004,"5min":15.189,"15min":154.197},"last":0.28999999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":2.1680000000000001,"15min":3.9969999999999999},"min":{"1min":0.27400000000000002,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":1.1990000000000001,"5min":18.503,"15min":200.56800000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":2.3660000000000001,"15min":4.6989999999999998},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":0.51900000000000002,"5min":20.181000000000001,"15min":200.63800000000001},"last":0.44600000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.8029999999999999,"15min":2.3109999999999999},"min":{"1min":0.32900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.8109999999999999,"5min":21.015000000000001,"15min":154.101},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.46400000000000002,"5min":1.585,"15min":2.5249999999999999},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.83199999999999996,"5min":10.222,"15min":154.56999999999999},"last":0.48099999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:42:35 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.843,"15min":2.1240000000000001},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.2450000000000001,"5min":21.219999999999999,"15min":154.22999999999999},"last":0.438},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.6499999999999999,"15min":2.5649999999999999},"min":{"1min":0.316,"5min":0.307,"15min":0.307},"max":{"1min":2.407,"5min":10.305,"15min":154.54400000000001},"last":0.51800000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.849,"15min":2.6640000000000001},"min":{"1min":0.372,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.2789999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.505},{"interface":"front","average":{"1min":0.495,"5min":1.702,"15min":2.4870000000000001},"min":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":0.67300000000000004,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.46000000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.9319999999999999,"15min":2.6440000000000001},"min":{"1min":0.40100000000000002,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":1.3029999999999999,"5min":21.062999999999999,"15min":154.20699999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.804,"15min":2.718},"min":{"1min":0.41699999999999998,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":1.3220000000000001,"5min":13.161,"15min":154.51599999999999},"last":0.54700000000000004}]}]},{"osd":0,"up_from":89,"seq":382252089734,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4428268,"kb_used_data":3541220,"kb_used_omap":8,"kb_used_meta":887031,"kb_avail":89316884,"statfs":{"total":95995035648,"available":91460489216,"internally_reserved":0,"allocated":3626209280,"data_stored":3624200384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":908320706},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":1.76,"15min":4.2480000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17399999999999999},"max":{"1min":1.891,"5min":52.316000000000003,"15min":216.821},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":3.4119999999999999,"15min":5.2130000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":3.552,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.20499999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:42:36 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":1.8779999999999999,"15min":2.1579999999999999},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.20100000000000001},"max":{"1min":0.51200000000000001,"5min":19.788,"15min":19.788},"last":0.372},{"interface":"front","average":{"1min":0.36899999999999999,"5min":2.0960000000000001,"15min":2.5230000000000001},"min":{"1min":0.315,"5min":0.215,"15min":0.189},"max":{"1min":0.439,"5min":25.844999999999999,"15min":25.844999999999999},"last":0.45000000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":1.474,"15min":3.907},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":1.1499999999999999,"5min":15.35,"15min":222.16499999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.748,"15min":3.9460000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":2.9550000000000001,"5min":14.545999999999999,"15min":222.227},"last":0.42799999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:42:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.8859999999999999,"15min":2.266},"min":{"1min":0.29299999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":3.694,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.443,"5min":1.9850000000000001,"15min":2.2639999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28299999999999997},"max":{"1min":0.59799999999999998,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.47299999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:42:30 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.837,"15min":1.9730000000000001},"min":{"1min":0.33700000000000002,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":1.9730000000000001,"5min":14.515000000000001,"15min":16.808},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.47199999999999998,"5min":1.6000000000000001,"15min":1.8480000000000001},"min":{"1min":0.34799999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64000000000000001,"5min":8.9740000000000002,"15min":17.507000000000001},"last":0.48699999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.617,"15min":1.8360000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.35499999999999998},"max":{"1min":1.5029999999999999,"5min":8.9920000000000009,"15min":17.535},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":1.879,"15min":2.137},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":1.5760000000000001,"5min":10.446,"15min":18.521000000000001},"last":0.496}]},{"osd":7,"last update":"Sun Oct 1 09:42:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":1.671,"15min":2.1139999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":0.77100000000000002,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.83599999999999997,"5min":1.97,"15min":2.4020000000000001},"min":{"1min":0.41899999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":3.7320000000000002,"5min":48.274000000000001,"15min":48.274000000000001},"last":0.55500000000000005}]}]},{"osd":2,"up_from":0,"seq":425201762674,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599147,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1879908,"kb_used_data":1047772,"kb_used_omap":3,"kb_used_meta":832124,"kb_avail":91865244,"statfs":{"total":95995035648,"available":94070009856,"internally_reserved":0,"allocated":1072918528,"data_stored":1070853537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":852095719},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":4.9660000000000002,"15min":5.0880000000000001},"min":{"1min":0.20300000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":2.7989999999999999,"5min":118.748,"15min":137.34399999999999},"last":0.621},{"interface":"front","average":{"1min":0.28899999999999998,"5min":4.9500000000000002,"15min":5.1379999999999999},"min":{"1min":0.20699999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.42199999999999999,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.249}]},{"osd":1,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":3.4329999999999998,"15min":5.2869999999999999},"min":{"1min":0.23000000000000001,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":2.7250000000000001,"5min":90.944000000000003,"15min":162.22999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.33300000000000002,"5min":3.1850000000000001,"15min":5.4160000000000004},"min":{"1min":0.23100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.45300000000000001,"5min":83.341999999999999,"15min":162.70699999999999},"last":0.58199999999999996}]},{"osd":2,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":1.8740000000000001,"15min":2.2029999999999998},"min":{"1min":0.251,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":0.54900000000000004,"5min":14.254,"15min":36.472999999999999},"last":0.505},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.90700000000000003,"15min":2.4910000000000001},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":0.59999999999999998,"5min":7.6749999999999998,"15min":78.594999999999999},"last":0.28699999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":1.8859999999999999,"15min":1.875},"min":{"1min":0.32600000000000001,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":0.65300000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.42799999999999999,"5min":2.2189999999999999,"15min":2.1469999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":0.64300000000000002,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.42099999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.766,"15min":2.7850000000000001},"min":{"1min":0.32500000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.70499999999999996,"5min":10.919,"15min":28.117999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":2.0139999999999998,"15min":2.8879999999999999},"min":{"1min":0.35399999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.5580000000000001,"5min":12.989000000000001,"15min":28.34},"last":0.36699999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:42:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":2.7970000000000002,"15min":2.3159999999999998},"min":{"1min":0.34399999999999997,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":1.621,"5min":73.887,"15min":73.887},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.66200000000000003,"5min":2.9670000000000001,"15min":2.3090000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.32800000000000001},"max":{"1min":1.8440000000000001,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.40600000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":1.7889999999999999,"15min":2.3090000000000002},"min":{"1min":0.39400000000000002,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":4.1580000000000004,"5min":13.085000000000001,"15min":52.006},"last":0.433},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.306,"15min":2.032},"min":{"1min":0.41899999999999998,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":4.1779999999999999,"5min":8.3460000000000001,"15min":51.890999999999998},"last":0.56200000000000006}]}]},{"osd":4,"up_from":30,"seq":128849019397,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4381708,"kb_used_data":3495620,"kb_used_omap":0,"kb_used_meta":886080,"kb_avail":89363444,"statfs":{"total":95995035648,"available":91508166656,"internally_reserved":0,"allocated":3579514880,"data_stored":3578582626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":907345920},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:53 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":3.129,"15min":4.7549999999999999},"min":{"1min":0.26100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.66400000000000003,"5min":79.837000000000003,"15min":155.315},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":3.2949999999999999,"15min":5.1200000000000001},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.251},"max":{"1min":2.843,"5min":78.983000000000004,"15min":155.334},"last":0.58999999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:42:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":6.8879999999999999,"15min":6.556},"min":{"1min":0.252,"5min":0.224,"15min":0.224},"max":{"1min":2.8889999999999998,"5min":146.929,"15min":190.61199999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.45500000000000002,"5min":6.8300000000000001,"15min":7.266},"min":{"1min":0.27500000000000002,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":0.624,"5min":146.91,"15min":253.273},"last":0.59999999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:42:43 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":1.2809999999999999,"15min":2.6019999999999999},"min":{"1min":0.34799999999999998,"5min":0.30399999999999999,"15min":0.28299999999999997},"max":{"1min":2.8210000000000002,"5min":10.161,"15min":170.45400000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.4410000000000001,"15min":2.2730000000000001},"min":{"1min":0.38400000000000001,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":2.1930000000000001,"5min":10.625,"15min":58.771000000000001},"last":0.63100000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:42:58 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":1.1799999999999999,"15min":2.7519999999999998},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.67500000000000004,"5min":7.5830000000000002,"15min":140.881},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.47099999999999997,"5min":1.208,"15min":2.976},"min":{"1min":0.33300000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.69599999999999995,"5min":9.3140000000000001,"15min":140.911},"last":0.51500000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:42:31 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":2.1579999999999999,"15min":1.5900000000000001},"min":{"1min":0.26800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.1190000000000002,"5min":110.003,"15min":110.003},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.46000000000000002,"5min":2.069,"15min":1.405},"min":{"1min":0.317,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.0780000000000001,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.41599999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:43:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.79200000000000004,"15min":0.99299999999999999},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":0.80900000000000005,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.95199999999999996,"15min":1.085},"min":{"1min":0.379,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":2.9239999999999999,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.438}]},{"osd":7,"last update":"Sun Oct 1 09:42:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":1.159,"15min":1.1799999999999999},"min":{"1min":0.38100000000000001,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":1.825,"5min":38.034999999999997,"15min":38.034999999999997},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.623,"5min":1.47,"15min":1.462},"min":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":2.2149999999999999,"5min":37.723999999999997,"15min":37.723999999999997},"last":0.46999999999999997}]}]},{"osd":5,"up_from":379,"seq":1627792605304,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2193592,"kb_used_data":1824944,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":91551560,"statfs":{"total":95995035648,"available":93748797440,"internally_reserved":0,"allocated":1868742656,"data_stored":1867826306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,3,1,0,2,5,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":1.897,"15min":2.258},"min":{"1min":0.23200000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.50700000000000001,"5min":12.571999999999999,"15min":84.340999999999994},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":1.5289999999999999,"15min":2.1349999999999998},"min":{"1min":0.27300000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60299999999999998,"5min":8.6460000000000008,"15min":84.356999999999999},"last":0.375}]},{"osd":1,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":6.7510000000000003,"15min":5.8460000000000001},"min":{"1min":0.26800000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.55400000000000005,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.37},{"interface":"front","average":{"1min":0.41199999999999998,"5min":7.3490000000000002,"15min":5.7889999999999997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":0.63200000000000001,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.42099999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.6020000000000001,"15min":1.6519999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.56599999999999995,"5min":8.7040000000000006,"15min":14.199999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.47799999999999998,"5min":1.6220000000000001,"15min":1.661},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":1.931,"5min":8.2370000000000001,"15min":14.430999999999999},"last":0.433}]},{"osd":3,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":1.796,"15min":1.5269999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.8720000000000001,"5min":12.734,"15min":14.004},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":1.8580000000000001,"15min":1.673},"min":{"1min":0.39800000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.8819999999999999,"5min":11.766,"15min":13.991},"last":0.46600000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":2.2200000000000002,"15min":1.4670000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.873,"5min":122.971,"15min":122.971},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.46000000000000002,"5min":2.3140000000000001,"15min":1.806},"min":{"1min":0.307,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.78400000000000003,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.33500000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":1.2849999999999999,"15min":1.0780000000000001},"min":{"1min":0.34499999999999997,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":0.68200000000000005,"5min":12.545,"15min":12.545},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.998,"15min":1.3640000000000001},"min":{"1min":0.34100000000000003,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":0.69999999999999996,"5min":6.4729999999999999,"15min":14.193},"last":0.443}]},{"osd":7,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":1.141,"15min":1.3200000000000001},"min":{"1min":0.32300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.63,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":1.2649999999999999,"15min":1.097},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.318},"max":{"1min":0.70399999999999996,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.47699999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3608481792,"data_stored":3608477720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":877985792,"data_stored":877985792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1066270720,"data_stored":1066270720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3568177152,"data_stored":3568173080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1862270976,"data_stored":1862270976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2051936256,"data_stored":2051932184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1467219968,"data_stored":1467219968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:43:23.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:23.006 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:23.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:23.107 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:23.140 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:23.192 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:23.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:23.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:23.300 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:23.329 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:23.376 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:23.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:23.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:23.480 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:23.505 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:23 smithi138 ceph-mon[192513]: pgmap v1502: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 8089/213624 objects degraded (3.787%); 4363/213624 objects misplaced (2.042%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:43:23.505 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:23 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8089/213624 objects degraded (3.787%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:23.506 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:23 smithi138 ceph-mon[185886]: pgmap v1502: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 8089/213624 objects degraded (3.787%); 4363/213624 objects misplaced (2.042%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:43:23.506 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:23 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8089/213624 objects degraded (3.787%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:23.508 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:23.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:23 smithi190 ceph-mon[160435]: pgmap v1502: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 8089/213624 objects degraded (3.787%); 4363/213624 objects misplaced (2.042%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:43:23.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:23 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8089/213624 objects degraded (3.787%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:23.784 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:23.802 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:23.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:23.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:23.905 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:23.906 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:23.937 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:24.058 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:24.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:24.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:24.162 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:24.188 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:24.235 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:24.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:24.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:24.340 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:24.382 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:24.553 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:24.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:24.658 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:43:24.684 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:24.729 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:43:24.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:24.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:43:24.836 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:43:24.863 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:24.908 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:43:24.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:24.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:43:25.015 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:25.043 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:25.283 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:43:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:43:24] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-01T09:43:25.294 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:25.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:25.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:25.399 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:25.400 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:43:25.426 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:25.473 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:25.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:25.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:25.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:25 smithi138 ceph-mon[192513]: pgmap v1503: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 8089/213540 objects degraded (3.788%); 4363/213540 objects misplaced (2.043%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:43:25.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:25 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/795125340' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:43:25.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:25 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:43:25.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:25.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:43:25.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:25 smithi138 ceph-mon[185886]: pgmap v1503: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 8089/213540 objects degraded (3.788%); 4363/213540 objects misplaced (2.043%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:43:25.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:25 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/795125340' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:43:25.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:25 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:43:25.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:25.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:43:25.580 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:25.615 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:25 smithi190 ceph-mon[160435]: pgmap v1503: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 8089/213540 objects degraded (3.788%); 4363/213540 objects misplaced (2.043%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:43:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:25 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/795125340' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:43:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:25 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:43:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:43:26.006 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:43:26.645 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:26.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:26.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:26.652 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:26 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:26.652 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:26 smithi138 ceph-mon[185886]: failed to encode map e522 with expected crc 2023-10-01T09:43:26.652 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:26 smithi138 ceph-mon[185886]: failed to encode map e522 with expected crc 2023-10-01T09:43:26.652 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:26 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:43:26.652 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:26 smithi138 ceph-mon[185886]: osdmap e522: 8 total, 8 up, 7 in 2023-10-01T09:43:26.652 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:26 smithi138 ceph-mon[185886]: failed to encode map e522 with expected crc 2023-10-01T09:43:26.653 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:26 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:26.653 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:26 smithi138 ceph-mon[192513]: failed to encode map e522 with expected crc 2023-10-01T09:43:26.653 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:26 smithi138 ceph-mon[192513]: failed to encode map e522 with expected crc 2023-10-01T09:43:26.653 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:26 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:43:26.653 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:26 smithi138 ceph-mon[192513]: osdmap e522: 8 total, 8 up, 7 in 2023-10-01T09:43:26.653 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:26 smithi138 ceph-mon[192513]: failed to encode map e522 with expected crc 2023-10-01T09:43:26.753 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:26.782 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:26 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:26.782 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:26 smithi190 ceph-mon[160435]: failed to encode map e522 with expected crc 2023-10-01T09:43:26.782 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:26 smithi190 ceph-mon[160435]: failed to encode map e522 with expected crc 2023-10-01T09:43:26.783 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:26 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:43:26.783 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:26 smithi190 ceph-mon[160435]: osdmap e522: 8 total, 8 up, 7 in 2023-10-01T09:43:26.783 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:26 smithi190 ceph-mon[160435]: failed to encode map e522 with expected crc 2023-10-01T09:43:26.786 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:26.801 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:43:26.801 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:43:26.848 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:26.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:26.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:26.951 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:43:26.955 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:26.976 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:43:26.992 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:26.998 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:43:27.026 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:43:27.046 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:43:27.047 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:27.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:27.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:27.065 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:43:27.085 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:43:27.104 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:43:27.105 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:43:27.105 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:43:27.106 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:43:27.106 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:43:27.106 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:43:27.106 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:43:27.106 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:43:27.106 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:43:27.152 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:27.179 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:27.223 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:27.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:27.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:27.332 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:27.364 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:27.417 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:27.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:27.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:27.523 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:27.546 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:27 smithi190 ceph-mon[160435]: pgmap v1505: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 112 op/s; 8039/212397 objects degraded (3.785%); 4326/212397 objects misplaced (2.037%); 623 KiB/s, 9 objects/s recovering 2023-10-01T09:43:27.547 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:27 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1742580750' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:27.549 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:27.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:27 smithi138 ceph-mon[185886]: pgmap v1505: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 112 op/s; 8039/212397 objects degraded (3.785%); 4326/212397 objects misplaced (2.037%); 623 KiB/s, 9 objects/s recovering 2023-10-01T09:43:27.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:27 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1742580750' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:27.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:27 smithi138 ceph-mon[192513]: pgmap v1505: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 112 op/s; 8039/212397 objects degraded (3.785%); 4326/212397 objects misplaced (2.037%); 623 KiB/s, 9 objects/s recovering 2023-10-01T09:43:27.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:27 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1742580750' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:27.597 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:27.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:27.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:27.705 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:43:27.741 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:27.792 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:43:27.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:27.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:43:27.889 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:27.904 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:27.916 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:43:27.935 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:28.103 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:28.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:28.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:28.208 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:28.237 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:28.286 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:28.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:28.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:28.393 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:28.424 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:29.201 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:43:29.202 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:28 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8039/212397 objects degraded (3.785%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:29.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:29.204 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:29.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:29.204 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:28 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8039/212397 objects degraded (3.785%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:29.205 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:28 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8039/212397 objects degraded (3.785%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:29.205 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:29.305 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:43:29.332 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:29.383 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:43:29.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:29.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:43:29.489 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:29.512 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:29 smithi138 ceph-mon[192513]: pgmap v1506: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 8039/211677 objects degraded (3.798%); 4326/211677 objects misplaced (2.044%); 557 KiB/s, 8 objects/s recovering 2023-10-01T09:43:29.512 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:29 smithi138 ceph-mon[185886]: pgmap v1506: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 8039/211677 objects degraded (3.798%); 4326/211677 objects misplaced (2.044%); 557 KiB/s, 8 objects/s recovering 2023-10-01T09:43:29.520 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:29.576 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:29.586 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:29 smithi190 ceph-mon[160435]: pgmap v1506: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 8039/211677 objects degraded (3.798%); 4326/211677 objects misplaced (2.044%); 557 KiB/s, 8 objects/s recovering 2023-10-01T09:43:29.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:29.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:29.688 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:29.714 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:29.760 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:29.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:29.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:29.868 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:43:29.894 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:29.939 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:43:29.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:29.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:43:29.991 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:43:29.993 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:43:30.045 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:30.073 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:30.126 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:30.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:30.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:30.164 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:43:30.233 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:30.260 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:30.307 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:30.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:30.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:30.414 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:30.440 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:30.484 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:30.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:30.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:30.592 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:30.618 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:30.661 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:30.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:30.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:30.771 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:30.797 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:30.842 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:30.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:30.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:30.948 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:30.981 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:31.047 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:31.050 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1506,"stamp":"2023-10-01T09:43:28.307784+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4627044299,"num_objects":70559,"num_object_clones":0,"num_object_copies":211677,"num_objects_missing_on_primary":2897,"num_objects_missing":2897,"num_objects_degraded":8039,"num_objects_misplaced":4326,"num_objects_unfound":0,"num_objects_dirty":70559,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":97015,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8202,"num_bytes_recovered":524387343,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97079,"ondisk_log_size":97079,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19029764,"kb_used_data":13988684,"kb_used_omap":39,"kb_used_meta":5040920,"kb_avail":637186300,"statfs":{"total":671965249536,"available":652478771200,"internally_reserved":0,"allocated":14324412416,"data_stored":14314596310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5161902703},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-78970880,"num_objects":-1205,"num_object_clones":0,"num_object_copies":-3615,"num_objects_missing_on_primary":-87,"num_objects_missing":-87,"num_objects_degraded":-98,"num_objects_misplaced":-76,"num_objects_unfound":0,"num_objects_dirty":-1205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1205,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":5701632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002470"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":298,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.374516+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:43:17.374516+0000","last_peered":"2023-10-01T09:43:17.374516+0000","last_clean":"2023-10-01T09:43:17.374516+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:43:17.374516+0000","last_undegraded":"2023-10-01T09:43:17.374516+0000","last_fullsized":"2023-10-01T09:43:17.374516+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":140,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050426+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:43:09.050426+0000","last_peered":"2023-10-01T09:43:09.050426+0000","last_clean":"2023-10-01T09:43:09.050426+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:43:09.050426+0000","last_undegraded":"2023-10-01T09:43:09.050426+0000","last_fullsized":"2023-10-01T09:43:09.050426+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":520,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050213+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:43:09.050213+0000","last_peered":"2023-10-01T09:43:09.050213+0000","last_clean":"2023-10-01T09:43:09.050213+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:43:09.050213+0000","last_undegraded":"2023-10-01T09:43:09.050213+0000","last_fullsized":"2023-10-01T09:43:09.050213+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":580,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.392610+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:43:17.392610+0000","last_peered":"2023-10-01T09:43:17.392610+0000","last_clean":"2023-10-01T09:43:17.392610+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:43:17.392610+0000","last_undegraded":"2023-10-01T09:43:17.392610+0000","last_fullsized":"2023-10-01T09:43:17.392610+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":461,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.383085+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:43:17.383085+0000","last_peered":"2023-10-01T09:43:17.383085+0000","last_clean":"2023-10-01T09:43:17.383085+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:43:17.383085+0000","last_undegraded":"2023-10-01T09:43:17.383085+0000","last_fullsized":"2023-10-01T09:43:17.383085+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":203,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.400109+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:43:17.400109+0000","last_peered":"2023-10-01T09:43:17.400109+0000","last_clean":"2023-10-01T09:43:17.400109+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:43:17.400109+0000","last_undegraded":"2023-10-01T09:43:17.400109+0000","last_fullsized":"2023-10-01T09:43:17.400109+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":579,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.400211+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:43:17.400211+0000","last_peered":"2023-10-01T09:43:17.400211+0000","last_clean":"2023-10-01T09:43:17.400211+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:43:17.400211+0000","last_undegraded":"2023-10-01T09:43:17.400211+0000","last_fullsized":"2023-10-01T09:43:17.400211+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":508,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.384110+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:43:17.384110+0000","last_peered":"2023-10-01T09:43:17.384110+0000","last_clean":"2023-10-01T09:43:17.384110+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:43:17.384110+0000","last_undegraded":"2023-10-01T09:43:17.384110+0000","last_fullsized":"2023-10-01T09:43:17.384110+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":234,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.375086+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:43:17.375086+0000","last_peered":"2023-10-01T09:43:17.375086+0000","last_clean":"2023-10-01T09:43:17.375086+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:43:17.375086+0000","last_undegraded":"2023-10-01T09:43:17.375086+0000","last_fullsized":"2023-10-01T09:43:17.375086+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":41,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.401024+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:43:17.401024+0000","last_peered":"2023-10-01T09:43:17.401024+0000","last_clean":"2023-10-01T09:43:17.401024+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:43:17.401024+0000","last_undegraded":"2023-10-01T09:43:17.401024+0000","last_fullsized":"2023-10-01T09:43:17.401024+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":47,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.402692+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:43:17.402692+0000","last_peered":"2023-10-01T09:43:17.402692+0000","last_clean":"2023-10-01T09:43:17.402692+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:43:17.402692+0000","last_undegraded":"2023-10-01T09:43:17.402692+0000","last_fullsized":"2023-10-01T09:43:17.402692+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":44,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050200+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:43:09.050200+0000","last_peered":"2023-10-01T09:43:09.050200+0000","last_clean":"2023-10-01T09:43:09.050200+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:43:09.050200+0000","last_undegraded":"2023-10-01T09:43:09.050200+0000","last_fullsized":"2023-10-01T09:43:09.050200+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":423,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.376246+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:43:17.376246+0000","last_peered":"2023-10-01T09:43:17.376246+0000","last_clean":"2023-10-01T09:43:17.376246+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:43:17.376246+0000","last_undegraded":"2023-10-01T09:43:17.376246+0000","last_fullsized":"2023-10-01T09:43:17.376246+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":304,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.375957+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:43:17.375957+0000","last_peered":"2023-10-01T09:43:17.375957+0000","last_clean":"2023-10-01T09:43:17.375957+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:43:17.375957+0000","last_undegraded":"2023-10-01T09:43:17.375957+0000","last_fullsized":"2023-10-01T09:43:17.375957+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":640,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.401274+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:43:17.401274+0000","last_peered":"2023-10-01T09:43:17.401274+0000","last_clean":"2023-10-01T09:43:17.401274+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:43:17.401274+0000","last_undegraded":"2023-10-01T09:43:17.401274+0000","last_fullsized":"2023-10-01T09:43:17.401274+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":377,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.401488+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:43:17.401488+0000","last_peered":"2023-10-01T09:43:17.401488+0000","last_clean":"2023-10-01T09:43:17.401488+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:43:17.401488+0000","last_undegraded":"2023-10-01T09:43:17.401488+0000","last_fullsized":"2023-10-01T09:43:17.401488+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":50,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.402908+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:43:17.402908+0000","last_peered":"2023-10-01T09:43:17.402908+0000","last_clean":"2023-10-01T09:43:17.402908+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:43:17.402908+0000","last_undegraded":"2023-10-01T09:43:17.402908+0000","last_fullsized":"2023-10-01T09:43:17.402908+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":36,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.374671+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:43:17.374671+0000","last_peered":"2023-10-01T09:43:17.374671+0000","last_clean":"2023-10-01T09:43:17.374671+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:43:17.374671+0000","last_undegraded":"2023-10-01T09:43:17.374671+0000","last_fullsized":"2023-10-01T09:43:17.374671+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":533,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.401280+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:43:17.401280+0000","last_peered":"2023-10-01T09:43:17.401280+0000","last_clean":"2023-10-01T09:43:17.401280+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:43:17.401280+0000","last_undegraded":"2023-10-01T09:43:17.401280+0000","last_fullsized":"2023-10-01T09:43:17.401280+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":544,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.392580+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:43:17.392580+0000","last_peered":"2023-10-01T09:43:17.392580+0000","last_clean":"2023-10-01T09:43:17.392580+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:43:17.392580+0000","last_undegraded":"2023-10-01T09:43:17.392580+0000","last_fullsized":"2023-10-01T09:43:17.392580+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":400,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.383015+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:43:17.383015+0000","last_peered":"2023-10-01T09:43:17.383015+0000","last_clean":"2023-10-01T09:43:17.383015+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:43:17.383015+0000","last_undegraded":"2023-10-01T09:43:17.383015+0000","last_fullsized":"2023-10-01T09:43:17.383015+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":50,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.401481+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:43:17.401481+0000","last_peered":"2023-10-01T09:43:17.401481+0000","last_clean":"2023-10-01T09:43:17.401481+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:43:17.401481+0000","last_undegraded":"2023-10-01T09:43:17.401481+0000","last_fullsized":"2023-10-01T09:43:17.401481+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"522'5997","reported_seq":6052,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.613633+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:43:25.613633+0000","last_peered":"2023-10-01T09:43:25.613633+0000","last_clean":"2023-10-01T09:43:25.613633+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:43:25.613633+0000","last_undegraded":"2023-10-01T09:43:25.613633+0000","last_fullsized":"2023-10-01T09:43:25.613633+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5997,"log_dups_size":0,"ondisk_log_size":5997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288030720,"num_objects":4395,"num_object_clones":0,"num_object_copies":13185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5997,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":45,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.401202+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:43:17.401202+0000","last_peered":"2023-10-01T09:43:17.401202+0000","last_clean":"2023-10-01T09:43:17.401202+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:43:17.401202+0000","last_undegraded":"2023-10-01T09:43:17.401202+0000","last_fullsized":"2023-10-01T09:43:17.401202+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":242,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.374946+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:43:17.374946+0000","last_peered":"2023-10-01T09:43:17.374946+0000","last_clean":"2023-10-01T09:43:17.374946+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:43:17.374946+0000","last_undegraded":"2023-10-01T09:43:17.374946+0000","last_fullsized":"2023-10-01T09:43:17.374946+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"521'5990","reported_seq":8436,"reported_epoch":522,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:43:25.757681+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:43:25.757681+0000","last_peered":"2023-10-01T09:43:25.757681+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:43:25.757681+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5990,"log_dups_size":0,"ondisk_log_size":5990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289406976,"num_objects":4416,"num_object_clones":0,"num_object_copies":13248,"num_objects_missing_on_primary":2897,"num_objects_missing":2897,"num_objects_degraded":7245,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5990,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":787,"num_bytes_recovered":51576832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1451},{"shards":"0,4,6","objects":2897},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":516,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.403048+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:43:17.403048+0000","last_peered":"2023-10-01T09:43:17.403048+0000","last_clean":"2023-10-01T09:43:17.403048+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:43:17.403048+0000","last_undegraded":"2023-10-01T09:43:17.403048+0000","last_fullsized":"2023-10-01T09:43:17.403048+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":576,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050187+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:43:09.050187+0000","last_peered":"2023-10-01T09:43:09.050187+0000","last_clean":"2023-10-01T09:43:09.050187+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:43:09.050187+0000","last_undegraded":"2023-10-01T09:43:09.050187+0000","last_fullsized":"2023-10-01T09:43:09.050187+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"522'6030","reported_seq":6087,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:26.779601+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:43:26.779601+0000","last_peered":"2023-10-01T09:43:26.779601+0000","last_clean":"2023-10-01T09:43:26.779601+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:43:26.779601+0000","last_undegraded":"2023-10-01T09:43:26.779601+0000","last_fullsized":"2023-10-01T09:43:26.779601+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6030,"log_dups_size":0,"ondisk_log_size":6030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288489472,"num_objects":4402,"num_object_clones":0,"num_object_copies":13206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6030,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":602,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.402746+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:43:17.402746+0000","last_peered":"2023-10-01T09:43:17.402746+0000","last_clean":"2023-10-01T09:43:17.402746+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:43:17.402746+0000","last_undegraded":"2023-10-01T09:43:17.402746+0000","last_fullsized":"2023-10-01T09:43:17.402746+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":396,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050329+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:43:09.050329+0000","last_peered":"2023-10-01T09:43:09.050329+0000","last_clean":"2023-10-01T09:43:09.050329+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:43:09.050329+0000","last_undegraded":"2023-10-01T09:43:09.050329+0000","last_fullsized":"2023-10-01T09:43:09.050329+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"522'5972","reported_seq":12634,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:26.825388+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:43:26.825388+0000","last_peered":"2023-10-01T09:43:26.825388+0000","last_clean":"2023-10-01T09:43:26.825388+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:43:26.825388+0000","last_undegraded":"2023-10-01T09:43:26.825388+0000","last_fullsized":"2023-10-01T09:43:26.825388+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5972,"log_dups_size":0,"ondisk_log_size":5972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290324480,"num_objects":4430,"num_object_clones":0,"num_object_copies":13290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5972,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":242,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.375030+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:43:17.375030+0000","last_peered":"2023-10-01T09:43:17.375030+0000","last_clean":"2023-10-01T09:43:17.375030+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:43:17.375030+0000","last_undegraded":"2023-10-01T09:43:17.375030+0000","last_fullsized":"2023-10-01T09:43:17.375030+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":500,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.386708+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:43:17.386708+0000","last_peered":"2023-10-01T09:43:17.386708+0000","last_clean":"2023-10-01T09:43:17.386708+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:43:17.386708+0000","last_undegraded":"2023-10-01T09:43:17.386708+0000","last_fullsized":"2023-10-01T09:43:17.386708+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"521'6027","reported_seq":6083,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:24.992726+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:43:24.992726+0000","last_peered":"2023-10-01T09:43:24.992726+0000","last_clean":"2023-10-01T09:43:24.992726+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:43:24.992726+0000","last_undegraded":"2023-10-01T09:43:24.992726+0000","last_fullsized":"2023-10-01T09:43:24.992726+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6027,"log_dups_size":0,"ondisk_log_size":6027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293666816,"num_objects":4481,"num_object_clones":0,"num_object_copies":13443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6027,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":102,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.376193+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:43:17.376193+0000","last_peered":"2023-10-01T09:43:17.376193+0000","last_clean":"2023-10-01T09:43:17.376193+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:43:17.376193+0000","last_undegraded":"2023-10-01T09:43:17.376193+0000","last_fullsized":"2023-10-01T09:43:17.376193+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":176,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.376414+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:43:17.376414+0000","last_peered":"2023-10-01T09:43:17.376414+0000","last_clean":"2023-10-01T09:43:17.376414+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:43:17.376414+0000","last_undegraded":"2023-10-01T09:43:17.376414+0000","last_fullsized":"2023-10-01T09:43:17.376414+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"522'5978","reported_seq":6063,"reported_epoch":522,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:43:25.701284+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:43:25.701284+0000","last_peered":"2023-10-01T09:43:25.701284+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:43:25.701284+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5978,"log_dups_size":0,"ondisk_log_size":5978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287703040,"num_objects":4390,"num_object_clones":0,"num_object_copies":13170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":784,"num_objects_misplaced":4326,"num_objects_unfound":0,"num_objects_dirty":4390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5978,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4326},{"shards":"4,5","objects":-720}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":420,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.375894+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:43:17.375894+0000","last_peered":"2023-10-01T09:43:17.375894+0000","last_clean":"2023-10-01T09:43:17.375894+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:43:17.375894+0000","last_undegraded":"2023-10-01T09:43:17.375894+0000","last_fullsized":"2023-10-01T09:43:17.375894+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":579,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050427+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:43:09.050427+0000","last_peered":"2023-10-01T09:43:09.050427+0000","last_clean":"2023-10-01T09:43:09.050427+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:43:09.050427+0000","last_undegraded":"2023-10-01T09:43:09.050427+0000","last_fullsized":"2023-10-01T09:43:09.050427+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"522'5980","reported_seq":6044,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:27.376791+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:43:27.376791+0000","last_peered":"2023-10-01T09:43:27.376791+0000","last_clean":"2023-10-01T09:43:27.376791+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:43:27.376791+0000","last_undegraded":"2023-10-01T09:43:27.376791+0000","last_fullsized":"2023-10-01T09:43:27.376791+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5980,"log_dups_size":0,"ondisk_log_size":5980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288227328,"num_objects":4398,"num_object_clones":0,"num_object_copies":13194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5980,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":416,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.374570+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:43:17.374570+0000","last_peered":"2023-10-01T09:43:17.374570+0000","last_clean":"2023-10-01T09:43:17.374570+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:43:17.374570+0000","last_undegraded":"2023-10-01T09:43:17.374570+0000","last_fullsized":"2023-10-01T09:43:17.374570+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":257,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.376888+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:43:17.376888+0000","last_peered":"2023-10-01T09:43:17.376888+0000","last_clean":"2023-10-01T09:43:17.376888+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:43:17.376888+0000","last_undegraded":"2023-10-01T09:43:17.376888+0000","last_fullsized":"2023-10-01T09:43:17.376888+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"522'6081","reported_seq":6136,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:26.140947+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:43:26.140947+0000","last_peered":"2023-10-01T09:43:26.140947+0000","last_clean":"2023-10-01T09:43:26.140947+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:43:26.140947+0000","last_undegraded":"2023-10-01T09:43:26.140947+0000","last_fullsized":"2023-10-01T09:43:26.140947+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6081,"log_dups_size":0,"ondisk_log_size":6081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289472512,"num_objects":4417,"num_object_clones":0,"num_object_copies":13251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6081,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":383,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050130+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:43:09.050130+0000","last_peered":"2023-10-01T09:43:09.050130+0000","last_clean":"2023-10-01T09:43:09.050130+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:43:09.050130+0000","last_undegraded":"2023-10-01T09:43:09.050130+0000","last_fullsized":"2023-10-01T09:43:09.050130+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":431,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.385968+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:43:17.385968+0000","last_peered":"2023-10-01T09:43:17.385968+0000","last_clean":"2023-10-01T09:43:17.385968+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:43:17.385968+0000","last_undegraded":"2023-10-01T09:43:17.385968+0000","last_fullsized":"2023-10-01T09:43:17.385968+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"522'5880","reported_seq":5935,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.687764+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:43:25.687764+0000","last_peered":"2023-10-01T09:43:25.687764+0000","last_clean":"2023-10-01T09:43:25.687764+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:43:25.687764+0000","last_undegraded":"2023-10-01T09:43:25.687764+0000","last_fullsized":"2023-10-01T09:43:25.687764+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5880,"log_dups_size":0,"ondisk_log_size":5880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281673728,"num_objects":4298,"num_object_clones":0,"num_object_copies":12894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5880,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":915,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.401376+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:43:17.401376+0000","last_peered":"2023-10-01T09:43:17.401376+0000","last_clean":"2023-10-01T09:43:17.401376+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:43:17.401376+0000","last_undegraded":"2023-10-01T09:43:17.401376+0000","last_fullsized":"2023-10-01T09:43:17.401376+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":252,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.374831+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:43:17.374831+0000","last_peered":"2023-10-01T09:43:17.374831+0000","last_clean":"2023-10-01T09:43:17.374831+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:43:17.374831+0000","last_undegraded":"2023-10-01T09:43:17.374831+0000","last_fullsized":"2023-10-01T09:43:17.374831+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050247+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:43:09.050247+0000","last_peered":"2023-10-01T09:43:09.050247+0000","last_clean":"2023-10-01T09:43:09.050247+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:43:09.050247+0000","last_undegraded":"2023-10-01T09:43:09.050247+0000","last_fullsized":"2023-10-01T09:43:09.050247+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"522'5945","reported_seq":6002,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:26.811272+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:43:26.811272+0000","last_peered":"2023-10-01T09:43:26.811272+0000","last_clean":"2023-10-01T09:43:26.811272+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:43:26.811272+0000","last_undegraded":"2023-10-01T09:43:26.811272+0000","last_fullsized":"2023-10-01T09:43:26.811272+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5945,"log_dups_size":0,"ondisk_log_size":5945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289996800,"num_objects":4425,"num_object_clones":0,"num_object_copies":13275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5945,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1187,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.374936+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:43:17.374936+0000","last_peered":"2023-10-01T09:43:17.374936+0000","last_clean":"2023-10-01T09:43:17.374936+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:43:17.374936+0000","last_undegraded":"2023-10-01T09:43:17.374936+0000","last_fullsized":"2023-10-01T09:43:17.374936+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":397,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.376097+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:43:17.376097+0000","last_peered":"2023-10-01T09:43:17.376097+0000","last_clean":"2023-10-01T09:43:17.376097+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:43:17.376097+0000","last_undegraded":"2023-10-01T09:43:17.376097+0000","last_fullsized":"2023-10-01T09:43:17.376097+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":225,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.375184+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:43:17.375184+0000","last_peered":"2023-10-01T09:43:17.375184+0000","last_clean":"2023-10-01T09:43:17.375184+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:43:17.375184+0000","last_undegraded":"2023-10-01T09:43:17.375184+0000","last_fullsized":"2023-10-01T09:43:17.375184+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"522'5977","reported_seq":13253,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.533067+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:43:25.533067+0000","last_peered":"2023-10-01T09:43:25.533067+0000","last_clean":"2023-10-01T09:43:25.533067+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:43:25.533067+0000","last_undegraded":"2023-10-01T09:43:25.533067+0000","last_fullsized":"2023-10-01T09:43:25.533067+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5977,"log_dups_size":0,"ondisk_log_size":5977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288292864,"num_objects":4399,"num_object_clones":0,"num_object_copies":13197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5977,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":867,"reported_epoch":521,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:43:17.382870+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:43:17.382870+0000","last_peered":"2023-10-01T09:43:17.382870+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:43:17.382870+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:43:17.382870+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":583,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050174+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:43:09.050174+0000","last_peered":"2023-10-01T09:43:09.050174+0000","last_clean":"2023-10-01T09:43:09.050174+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:43:09.050174+0000","last_undegraded":"2023-10-01T09:43:09.050174+0000","last_fullsized":"2023-10-01T09:43:09.050174+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":44,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.127142+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:43:09.127142+0000","last_peered":"2023-10-01T09:43:09.127142+0000","last_clean":"2023-10-01T09:43:09.127142+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:43:09.127142+0000","last_undegraded":"2023-10-01T09:43:09.127142+0000","last_fullsized":"2023-10-01T09:43:09.127142+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"521'5877","reported_seq":5933,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:24.641787+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:43:24.641787+0000","last_peered":"2023-10-01T09:43:24.641787+0000","last_clean":"2023-10-01T09:43:24.641787+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:43:24.641787+0000","last_undegraded":"2023-10-01T09:43:24.641787+0000","last_fullsized":"2023-10-01T09:43:24.641787+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5877,"log_dups_size":0,"ondisk_log_size":5877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288030720,"num_objects":4395,"num_object_clones":0,"num_object_copies":13185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5877,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1638,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.375286+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:43:17.375286+0000","last_peered":"2023-10-01T09:43:17.375286+0000","last_clean":"2023-10-01T09:43:17.375286+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:43:17.375286+0000","last_undegraded":"2023-10-01T09:43:17.375286+0000","last_fullsized":"2023-10-01T09:43:17.375286+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":552,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.401026+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:43:17.401026+0000","last_peered":"2023-10-01T09:43:17.401026+0000","last_clean":"2023-10-01T09:43:17.401026+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:43:17.401026+0000","last_undegraded":"2023-10-01T09:43:17.401026+0000","last_fullsized":"2023-10-01T09:43:17.401026+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":330,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.374981+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:43:17.374981+0000","last_peered":"2023-10-01T09:43:17.374981+0000","last_clean":"2023-10-01T09:43:17.374981+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:43:17.374981+0000","last_undegraded":"2023-10-01T09:43:17.374981+0000","last_fullsized":"2023-10-01T09:43:17.374981+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":320,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.376231+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:43:17.376231+0000","last_peered":"2023-10-01T09:43:17.376231+0000","last_clean":"2023-10-01T09:43:17.376231+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:43:17.376231+0000","last_undegraded":"2023-10-01T09:43:17.376231+0000","last_fullsized":"2023-10-01T09:43:17.376231+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"522'6058","reported_seq":6113,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:26.168135+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:43:26.168135+0000","last_peered":"2023-10-01T09:43:26.168135+0000","last_clean":"2023-10-01T09:43:26.168135+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:43:26.168135+0000","last_undegraded":"2023-10-01T09:43:26.168135+0000","last_fullsized":"2023-10-01T09:43:26.168135+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6058,"log_dups_size":0,"ondisk_log_size":6058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293208064,"num_objects":4474,"num_object_clones":0,"num_object_copies":13422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6058,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":729,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.401114+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:43:17.401114+0000","last_peered":"2023-10-01T09:43:17.401114+0000","last_clean":"2023-10-01T09:43:17.401114+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:43:17.401114+0000","last_undegraded":"2023-10-01T09:43:17.401114+0000","last_fullsized":"2023-10-01T09:43:17.401114+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":561,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.383159+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:43:17.383159+0000","last_peered":"2023-10-01T09:43:17.383159+0000","last_clean":"2023-10-01T09:43:17.383159+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:43:17.383159+0000","last_undegraded":"2023-10-01T09:43:17.383159+0000","last_fullsized":"2023-10-01T09:43:17.383159+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":561,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.398041+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:43:25.398041+0000","last_peered":"2023-10-01T09:43:25.398041+0000","last_clean":"2023-10-01T09:43:25.398041+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:43:25.398041+0000","last_undegraded":"2023-10-01T09:43:25.398041+0000","last_fullsized":"2023-10-01T09:43:25.398041+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"522'6061","reported_seq":6119,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:26.814868+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:43:26.814868+0000","last_peered":"2023-10-01T09:43:26.814868+0000","last_clean":"2023-10-01T09:43:26.814868+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:43:26.814868+0000","last_undegraded":"2023-10-01T09:43:26.814868+0000","last_fullsized":"2023-10-01T09:43:26.814868+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6061,"log_dups_size":0,"ondisk_log_size":6061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288227352,"num_objects":4399,"num_object_clones":0,"num_object_copies":13197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4399,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6061,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":960,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.402792+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:43:17.402792+0000","last_peered":"2023-10-01T09:43:17.402792+0000","last_clean":"2023-10-01T09:43:17.402792+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:43:17.402792+0000","last_undegraded":"2023-10-01T09:43:17.402792+0000","last_fullsized":"2023-10-01T09:43:17.402792+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":579,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.375110+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:43:17.375110+0000","last_peered":"2023-10-01T09:43:17.375110+0000","last_clean":"2023-10-01T09:43:17.375110+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:43:17.375110+0000","last_undegraded":"2023-10-01T09:43:17.375110+0000","last_fullsized":"2023-10-01T09:43:17.375110+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":432,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.375367+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:43:17.375367+0000","last_peered":"2023-10-01T09:43:17.375367+0000","last_clean":"2023-10-01T09:43:17.375367+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:43:17.375367+0000","last_undegraded":"2023-10-01T09:43:17.375367+0000","last_fullsized":"2023-10-01T09:43:17.375367+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"522'6013","reported_seq":6076,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:26.110522+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:43:26.110522+0000","last_peered":"2023-10-01T09:43:26.110522+0000","last_clean":"2023-10-01T09:43:26.110522+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:43:26.110522+0000","last_undegraded":"2023-10-01T09:43:26.110522+0000","last_fullsized":"2023-10-01T09:43:26.110522+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6013,"log_dups_size":0,"ondisk_log_size":6013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284622848,"num_objects":4343,"num_object_clones":0,"num_object_copies":13029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6013,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":461,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.401568+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:43:17.401568+0000","last_peered":"2023-10-01T09:43:17.401568+0000","last_clean":"2023-10-01T09:43:17.401568+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:43:17.401568+0000","last_undegraded":"2023-10-01T09:43:17.401568+0000","last_fullsized":"2023-10-01T09:43:17.401568+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1153,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.127184+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:43:09.127184+0000","last_peered":"2023-10-01T09:43:09.127184+0000","last_clean":"2023-10-01T09:43:09.127184+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:43:09.127184+0000","last_undegraded":"2023-10-01T09:43:09.127184+0000","last_fullsized":"2023-10-01T09:43:09.127184+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":50,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.400969+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:43:17.400969+0000","last_peered":"2023-10-01T09:43:17.400969+0000","last_clean":"2023-10-01T09:43:17.400969+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:43:17.400969+0000","last_undegraded":"2023-10-01T09:43:17.400969+0000","last_fullsized":"2023-10-01T09:43:17.400969+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"522'6043","reported_seq":6106,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:26.081699+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:43:26.081699+0000","last_peered":"2023-10-01T09:43:26.081699+0000","last_clean":"2023-10-01T09:43:26.081699+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:43:26.081699+0000","last_undegraded":"2023-10-01T09:43:26.081699+0000","last_fullsized":"2023-10-01T09:43:26.081699+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6043,"log_dups_size":0,"ondisk_log_size":6043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292880384,"num_objects":4469,"num_object_clones":0,"num_object_copies":13407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6043,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":54,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.401403+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:43:17.401403+0000","last_peered":"2023-10-01T09:43:17.401403+0000","last_clean":"2023-10-01T09:43:17.401403+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:43:17.401403+0000","last_undegraded":"2023-10-01T09:43:17.401403+0000","last_fullsized":"2023-10-01T09:43:17.401403+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":539,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.398083+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:43:25.398083+0000","last_peered":"2023-10-01T09:43:25.398083+0000","last_clean":"2023-10-01T09:43:25.398083+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:43:25.398083+0000","last_undegraded":"2023-10-01T09:43:25.398083+0000","last_fullsized":"2023-10-01T09:43:25.398083+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":245,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.374831+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:43:17.374831+0000","last_peered":"2023-10-01T09:43:17.374831+0000","last_clean":"2023-10-01T09:43:17.374831+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:43:17.374831+0000","last_undegraded":"2023-10-01T09:43:17.374831+0000","last_fullsized":"2023-10-01T09:43:17.374831+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":488,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.519386+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:43:25.519386+0000","last_peered":"2023-10-01T09:43:25.519386+0000","last_clean":"2023-10-01T09:43:25.519386+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:43:25.519386+0000","last_undegraded":"2023-10-01T09:43:25.519386+0000","last_fullsized":"2023-10-01T09:43:25.519386+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":600,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.050425+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:43:09.050425+0000","last_peered":"2023-10-01T09:43:09.050425+0000","last_clean":"2023-10-01T09:43:09.050425+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:43:09.050425+0000","last_undegraded":"2023-10-01T09:43:09.050425+0000","last_fullsized":"2023-10-01T09:43:09.050425+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":257,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.374942+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:43:17.374942+0000","last_peered":"2023-10-01T09:43:17.374942+0000","last_clean":"2023-10-01T09:43:17.374942+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:43:17.374942+0000","last_undegraded":"2023-10-01T09:43:17.374942+0000","last_fullsized":"2023-10-01T09:43:17.374942+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":44,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:15.251918+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:43:15.251918+0000","last_peered":"2023-10-01T09:43:15.251918+0000","last_clean":"2023-10-01T09:43:15.251918+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:43:15.251918+0000","last_undegraded":"2023-10-01T09:43:15.251918+0000","last_fullsized":"2023-10-01T09:43:15.251918+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":257,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.374834+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:43:17.374834+0000","last_peered":"2023-10-01T09:43:17.374834+0000","last_clean":"2023-10-01T09:43:17.374834+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:43:17.374834+0000","last_undegraded":"2023-10-01T09:43:17.374834+0000","last_fullsized":"2023-10-01T09:43:17.374834+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":434,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.383291+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:43:17.383291+0000","last_peered":"2023-10-01T09:43:17.383291+0000","last_clean":"2023-10-01T09:43:17.383291+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:43:17.383291+0000","last_undegraded":"2023-10-01T09:43:17.383291+0000","last_fullsized":"2023-10-01T09:43:17.383291+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":61,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-10-01T09:43:09.127072+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:43:09.127072+0000","last_peered":"2023-10-01T09:43:09.127072+0000","last_clean":"2023-10-01T09:43:09.127072+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:43:09.127072+0000","last_undegraded":"2023-10-01T09:43:09.127072+0000","last_fullsized":"2023-10-01T09:43:09.127072+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":479,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.383052+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:43:17.383052+0000","last_peered":"2023-10-01T09:43:17.383052+0000","last_clean":"2023-10-01T09:43:17.383052+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:43:17.383052+0000","last_undegraded":"2023-10-01T09:43:17.383052+0000","last_fullsized":"2023-10-01T09:43:17.383052+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":678,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.392888+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:43:17.392888+0000","last_peered":"2023-10-01T09:43:17.392888+0000","last_clean":"2023-10-01T09:43:17.392888+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:43:17.392888+0000","last_undegraded":"2023-10-01T09:43:17.392888+0000","last_fullsized":"2023-10-01T09:43:17.392888+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":373,"reported_epoch":521,"state":"active+clean","last_fresh":"2023-10-01T09:43:17.374433+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:43:17.374433+0000","last_peered":"2023-10-01T09:43:17.374433+0000","last_clean":"2023-10-01T09:43:17.374433+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:43:17.374433+0000","last_undegraded":"2023-10-01T09:43:17.374433+0000","last_fullsized":"2023-10-01T09:43:17.374433+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4622254104,"num_objects":70531,"num_object_clones":0,"num_object_copies":211593,"num_objects_missing_on_primary":2897,"num_objects_missing":2897,"num_objects_degraded":8029,"num_objects_misplaced":4326,"num_objects_unfound":0,"num_objects_dirty":70531,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":95909,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7691,"num_bytes_recovered":504037376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14259204096,"data_stored":14259191880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95909,"ondisk_log_size":95909,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430714,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2323296,"kb_used_data":1420184,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":91421856,"statfs":{"total":95995035648,"available":93615980544,"internally_reserved":0,"allocated":1454268416,"data_stored":1453332008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:56 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.4950000000000001,"15min":2.9569999999999999},"min":{"1min":0.246,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":1.3280000000000001,"5min":10.391999999999999,"15min":108.90000000000001},"last":1.484},{"interface":"front","average":{"1min":0.49199999999999999,"5min":1.8129999999999999,"15min":3.129},"min":{"1min":0.246,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.018,"5min":29.501000000000001,"15min":109.187},"last":0.26900000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":4.6749999999999998,"15min":5.3520000000000003},"min":{"1min":0.31900000000000001,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":1.2949999999999999,"5min":229.161,"15min":229.161},"last":1.4470000000000001},{"interface":"front","average":{"1min":0.47399999999999998,"5min":4.6479999999999997,"15min":5.3689999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":0.90900000000000003,"5min":230.126,"15min":230.126},"last":0.29699999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:42:50 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":1.5369999999999999,"15min":1.891},"min":{"1min":0.33900000000000002,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":1.3560000000000001,"5min":14.866,"15min":14.901999999999999},"last":1.992},{"interface":"front","average":{"1min":0.49299999999999999,"5min":1.7909999999999999,"15min":1.821},"min":{"1min":0.33500000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.96599999999999997,"5min":14.682,"15min":14.682},"last":1.262}]},{"osd":3,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":1.514,"15min":2.7330000000000001},"min":{"1min":0.30199999999999999,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":0.90700000000000003,"5min":35.104999999999997,"15min":80.352999999999994},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.50600000000000001,"5min":1.6930000000000001,"15min":2.7130000000000001},"min":{"1min":0.35399999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.153,"5min":35.118000000000002,"15min":80.364000000000004},"last":0.41799999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:42:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":1.2150000000000001,"15min":1.052},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.381,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":1.508},{"interface":"front","average":{"1min":0.49099999999999999,"5min":1.1080000000000001,"15min":1.03},"min":{"1min":0.314,"5min":0.30299999999999999,"15min":0.28000000000000003},"max":{"1min":0.97899999999999998,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.53600000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":2.8879999999999999,"15min":1.6419999999999999},"min":{"1min":0.318,"5min":0.30299999999999999,"15min":0.29599999999999999},"max":{"1min":0.64600000000000002,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.41999999999999998,"5min":3.0779999999999998,"15min":1.6850000000000001},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54500000000000004,"5min":135.339,"15min":135.339},"last":0.38300000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.4379999999999999,"15min":1.173},"min":{"1min":0.313,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":2.448,"5min":11.58,"15min":11.853999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.44400000000000001,"5min":1.2789999999999999,"15min":1.1359999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":0.56699999999999995,"5min":14.584,"15min":14.584},"last":0.442}]}]},{"osd":6,"up_from":294,"seq":1262720385225,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2307100,"kb_used_data":1977620,"kb_used_omap":6,"kb_used_meta":329401,"kb_avail":91438052,"statfs":{"total":95995035648,"available":93632565248,"internally_reserved":0,"allocated":2025082880,"data_stored":2024150423,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":337306669},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":3.5720000000000001,"15min":4.0919999999999996},"min":{"1min":0.25,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":3.3340000000000001,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.374},{"interface":"front","average":{"1min":0.42599999999999999,"5min":4.0410000000000004,"15min":4.2210000000000001},"min":{"1min":0.29899999999999999,"5min":0.249,"15min":0.221},"max":{"1min":0.623,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.38600000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":4.7489999999999997,"15min":4.9740000000000002},"min":{"1min":0.30599999999999999,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":1.9410000000000001,"5min":168.947,"15min":168.947},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.53000000000000003,"5min":4.7969999999999997,"15min":4.8810000000000002},"min":{"1min":0.34699999999999998,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":1.962,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.39800000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":1.194,"15min":1.673},"min":{"1min":0.308,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.075,"5min":8.657,"15min":10.92},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.3759999999999999,"15min":1.673},"min":{"1min":0.314,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.58499999999999996,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.42299999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.3069999999999999,"15min":2.2789999999999999},"min":{"1min":0.35499999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.3759999999999999,"5min":9.0180000000000007,"15min":89.753},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.502,"5min":1.4359999999999999,"15min":2.157},"min":{"1min":0.34399999999999997,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":1.381,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.41699999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":1.5900000000000001,"15min":1.0800000000000001},"min":{"1min":0.30099999999999999,"5min":0.253,"15min":0.246},"max":{"1min":0.53400000000000003,"5min":63.134,"15min":63.134},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.5009999999999999,"15min":1.097},"min":{"1min":0.33700000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":1.5669999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.438}]},{"osd":5,"last update":"Sun Oct 1 09:42:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.93200000000000005,"15min":1.02},"min":{"1min":0.26700000000000002,"5min":0.251,"15min":0.251},"max":{"1min":1.8620000000000001,"5min":9.9939999999999998,"15min":12.023999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.66500000000000004,"5min":1.175,"15min":1.143},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.726,"5min":15.457000000000001,"15min":15.457000000000001},"last":0.35799999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.79500000000000004,"15min":1.46},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":1.4750000000000001,"5min":6.4119999999999999,"15min":94.245999999999995},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.48999999999999999,"5min":1.113,"15min":1.5569999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":0.68300000000000005,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.47999999999999998}]}]},{"osd":1,"up_from":94,"seq":403726926201,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1680508,"kb_used_data":862324,"kb_used_omap":7,"kb_used_meta":818168,"kb_avail":92064644,"statfs":{"total":95995035648,"available":94274195456,"internally_reserved":0,"allocated":883019776,"data_stored":880997294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":837804455},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.30299999999999999,"5min":2.5600000000000001,"15min":4.2009999999999996},"min":{"1min":0.20699999999999999,"5min":0.19400000000000001,"15min":0.182},"max":{"1min":0.57399999999999995,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.33600000000000002,"5min":3.0139999999999998,"15min":3.7090000000000001},"min":{"1min":0.26300000000000001,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":0.53400000000000003,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.40899999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:42:38 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":1.7929999999999999,"15min":2.7719999999999998},"min":{"1min":0.24099999999999999,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.83099999999999996,"5min":22.891999999999999,"15min":56.573},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.36699999999999999,"5min":2.2799999999999998,"15min":3.593},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":0.54400000000000004,"5min":15.189,"15min":154.197},"last":0.44600000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":2.1680000000000001,"15min":3.9969999999999999},"min":{"1min":0.27400000000000002,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":1.1990000000000001,"5min":18.503,"15min":200.56800000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":2.3660000000000001,"15min":4.6989999999999998},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":0.51900000000000002,"5min":20.181000000000001,"15min":200.63800000000001},"last":0.42699999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.8029999999999999,"15min":2.3109999999999999},"min":{"1min":0.32900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.8109999999999999,"5min":21.015000000000001,"15min":154.101},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.46400000000000002,"5min":1.585,"15min":2.5249999999999999},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.83199999999999996,"5min":10.222,"15min":154.56999999999999},"last":0.56399999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:42:35 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.843,"15min":2.1240000000000001},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.2450000000000001,"5min":21.219999999999999,"15min":154.22999999999999},"last":0.497},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.6499999999999999,"15min":2.5649999999999999},"min":{"1min":0.316,"5min":0.307,"15min":0.307},"max":{"1min":2.407,"5min":10.305,"15min":154.54400000000001},"last":0.54200000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.849,"15min":2.6640000000000001},"min":{"1min":0.372,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.2789999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.495,"5min":1.702,"15min":2.4870000000000001},"min":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":0.67300000000000004,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.51500000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.9319999999999999,"15min":2.6440000000000001},"min":{"1min":0.40100000000000002,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":1.3029999999999999,"5min":21.062999999999999,"15min":154.20699999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.804,"15min":2.718},"min":{"1min":0.41699999999999998,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":1.3220000000000001,"5min":13.161,"15min":154.51599999999999},"last":0.59899999999999998}]}]},{"osd":0,"up_from":89,"seq":382252089736,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4376124,"kb_used_data":3484980,"kb_used_omap":8,"kb_used_meta":891127,"kb_avail":89369028,"statfs":{"total":95995035648,"available":91513884672,"internally_reserved":0,"allocated":3568619520,"data_stored":3566609976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":912515010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":1.76,"15min":4.2480000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17399999999999999},"max":{"1min":1.891,"5min":52.316000000000003,"15min":216.821},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":3.4119999999999999,"15min":5.2130000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":3.552,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.30099999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:42:36 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":1.8779999999999999,"15min":2.1579999999999999},"min":{"1min":0.27700000000000002,"5min":0.253,"15min":0.20100000000000001},"max":{"1min":0.51200000000000001,"5min":19.788,"15min":19.788},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.36899999999999999,"5min":2.0960000000000001,"15min":2.5230000000000001},"min":{"1min":0.315,"5min":0.215,"15min":0.189},"max":{"1min":0.439,"5min":25.844999999999999,"15min":25.844999999999999},"last":0.36699999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":1.474,"15min":3.907},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":1.1499999999999999,"5min":15.35,"15min":222.16499999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.748,"15min":3.9460000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":2.9550000000000001,"5min":14.545999999999999,"15min":222.227},"last":0.38100000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:42:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.8859999999999999,"15min":2.266},"min":{"1min":0.29299999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":3.694,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.443,"5min":1.9850000000000001,"15min":2.2639999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28299999999999997},"max":{"1min":0.59799999999999998,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.40999999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:42:30 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.837,"15min":1.9730000000000001},"min":{"1min":0.33700000000000002,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":1.9730000000000001,"5min":14.515000000000001,"15min":16.808},"last":0.439},{"interface":"front","average":{"1min":0.47199999999999998,"5min":1.6000000000000001,"15min":1.8480000000000001},"min":{"1min":0.34799999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64000000000000001,"5min":8.9740000000000002,"15min":17.507000000000001},"last":0.503}]},{"osd":6,"last update":"Sun Oct 1 09:43:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.1850000000000001,"15min":1.774},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.35499999999999998},"max":{"1min":1.5900000000000001,"5min":6.5,"15min":17.535},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.88,"5min":1.395,"15min":2.0289999999999999},"min":{"1min":0.32600000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":3.6539999999999999,"5min":10.446,"15min":18.521000000000001},"last":0.48299999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:42:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":1.671,"15min":2.1139999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":0.77100000000000002,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.83599999999999997,"5min":1.97,"15min":2.4020000000000001},"min":{"1min":0.41899999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":3.7320000000000002,"5min":48.274000000000001,"15min":48.274000000000001},"last":0.51700000000000002}]}]},{"osd":2,"up_from":0,"seq":425201762675,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599149,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1849588,"kb_used_data":1013356,"kb_used_omap":3,"kb_used_meta":836220,"kb_avail":91895564,"statfs":{"total":95995035648,"available":94101057536,"internally_reserved":0,"allocated":1037676544,"data_stored":1035610905,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":856290023},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":4.9660000000000002,"15min":5.0880000000000001},"min":{"1min":0.20300000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":2.7989999999999999,"5min":118.748,"15min":137.34399999999999},"last":0.495},{"interface":"front","average":{"1min":0.28899999999999998,"5min":4.9500000000000002,"15min":5.1379999999999999},"min":{"1min":0.20699999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.42199999999999999,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.28199999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":3.4329999999999998,"15min":5.2869999999999999},"min":{"1min":0.23000000000000001,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":2.7250000000000001,"5min":90.944000000000003,"15min":162.22999999999999},"last":0.308},{"interface":"front","average":{"1min":0.33300000000000002,"5min":3.1850000000000001,"15min":5.4160000000000004},"min":{"1min":0.23100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.45300000000000001,"5min":83.341999999999999,"15min":162.70699999999999},"last":0.62}]},{"osd":2,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":1.8740000000000001,"15min":2.2029999999999998},"min":{"1min":0.251,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":0.54900000000000004,"5min":14.254,"15min":36.472999999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.90700000000000003,"15min":2.4910000000000001},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":0.59999999999999998,"5min":7.6749999999999998,"15min":78.594999999999999},"last":0.57099999999999995}]},{"osd":4,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":1.8859999999999999,"15min":1.875},"min":{"1min":0.32600000000000001,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":0.65300000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.42799999999999999,"5min":2.2189999999999999,"15min":2.1469999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":0.64300000000000002,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.47399999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.766,"15min":2.7850000000000001},"min":{"1min":0.32500000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.70499999999999996,"5min":10.919,"15min":28.117999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":2.0139999999999998,"15min":2.8879999999999999},"min":{"1min":0.35399999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.5580000000000001,"5min":12.989000000000001,"15min":28.34},"last":1.038}]},{"osd":6,"last update":"Sun Oct 1 09:43:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":2.165,"15min":2.145},"min":{"1min":0.34200000000000003,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":2.5430000000000001,"5min":73.887,"15min":73.887},"last":1.1000000000000001},{"interface":"front","average":{"1min":0.63,"5min":2.4159999999999999,"15min":2.1379999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":2.6000000000000001,"5min":74.016999999999996,"15min":74.016999999999996},"last":1.0780000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":1.7889999999999999,"15min":2.3090000000000002},"min":{"1min":0.39400000000000002,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":4.1580000000000004,"5min":13.085000000000001,"15min":52.006},"last":1.119},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.306,"15min":2.032},"min":{"1min":0.41899999999999998,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":4.1779999999999999,"5min":8.3460000000000001,"15min":51.890999999999998},"last":0.753}]}]},{"osd":4,"up_from":30,"seq":128849019399,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4323420,"kb_used_data":3433236,"kb_used_omap":0,"kb_used_meta":890176,"kb_avail":89421732,"statfs":{"total":95995035648,"available":91567853568,"internally_reserved":0,"allocated":3515633664,"data_stored":3514700762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911540224},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:53 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":3.129,"15min":4.7549999999999999},"min":{"1min":0.26100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.66400000000000003,"5min":79.837000000000003,"15min":155.315},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.53900000000000003,"5min":3.2949999999999999,"15min":5.1200000000000001},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.251},"max":{"1min":2.843,"5min":78.983000000000004,"15min":155.334},"last":0.39900000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:42:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":6.8879999999999999,"15min":6.556},"min":{"1min":0.252,"5min":0.224,"15min":0.224},"max":{"1min":2.8889999999999998,"5min":146.929,"15min":190.61199999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.45500000000000002,"5min":6.8300000000000001,"15min":7.266},"min":{"1min":0.27500000000000002,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":0.624,"5min":146.91,"15min":253.273},"last":0.495}]},{"osd":2,"last update":"Sun Oct 1 09:42:43 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":1.2809999999999999,"15min":2.6019999999999999},"min":{"1min":0.34799999999999998,"5min":0.30399999999999999,"15min":0.28299999999999997},"max":{"1min":2.8210000000000002,"5min":10.161,"15min":170.45400000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.4410000000000001,"15min":2.2730000000000001},"min":{"1min":0.38400000000000001,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":2.1930000000000001,"5min":10.625,"15min":58.771000000000001},"last":0.437}]},{"osd":3,"last update":"Sun Oct 1 09:42:58 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":1.1799999999999999,"15min":2.7519999999999998},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.67500000000000004,"5min":7.5830000000000002,"15min":140.881},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.47099999999999997,"5min":1.208,"15min":2.976},"min":{"1min":0.33300000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.69599999999999995,"5min":9.3140000000000001,"15min":140.911},"last":0.46700000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:42:31 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":2.1579999999999999,"15min":1.5900000000000001},"min":{"1min":0.26800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.1190000000000002,"5min":110.003,"15min":110.003},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.46000000000000002,"5min":2.069,"15min":1.405},"min":{"1min":0.317,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.0780000000000001,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.41999999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:43:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.79200000000000004,"15min":0.99299999999999999},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":0.80900000000000005,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.95199999999999996,"15min":1.085},"min":{"1min":0.379,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":2.9239999999999999,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.47999999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:42:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":1.159,"15min":1.1799999999999999},"min":{"1min":0.38100000000000001,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":1.825,"5min":38.034999999999997,"15min":38.034999999999997},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.623,"5min":1.47,"15min":1.462},"min":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":2.2149999999999999,"5min":37.723999999999997,"15min":37.723999999999997},"last":0.504}]}]},{"osd":5,"up_from":379,"seq":1627792605306,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2169728,"kb_used_data":1796984,"kb_used_omap":13,"kb_used_meta":372722,"kb_avail":91575424,"statfs":{"total":95995035648,"available":93773234176,"internally_reserved":0,"allocated":1840111616,"data_stored":1839194942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":381667826},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":1.897,"15min":2.258},"min":{"1min":0.23200000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.50700000000000001,"5min":12.571999999999999,"15min":84.340999999999994},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.39500000000000002,"5min":1.5289999999999999,"15min":2.1349999999999998},"min":{"1min":0.27300000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60299999999999998,"5min":8.6460000000000008,"15min":84.356999999999999},"last":0.47199999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":6.7510000000000003,"15min":5.8460000000000001},"min":{"1min":0.26800000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.55400000000000005,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.41199999999999998,"5min":7.3490000000000002,"15min":5.7889999999999997},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":0.63200000000000001,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.36399999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.6020000000000001,"15min":1.6519999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.56599999999999995,"5min":8.7040000000000006,"15min":14.199999999999999},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.47799999999999998,"5min":1.6220000000000001,"15min":1.661},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":1.931,"5min":8.2370000000000001,"15min":14.430999999999999},"last":0.5}]},{"osd":3,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":1.796,"15min":1.5269999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.8720000000000001,"5min":12.734,"15min":14.004},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":1.8580000000000001,"15min":1.673},"min":{"1min":0.39800000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.8819999999999999,"5min":11.766,"15min":13.991},"last":0.63400000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":2.2200000000000002,"15min":1.4670000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.873,"5min":122.971,"15min":122.971},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.46000000000000002,"5min":2.3140000000000001,"15min":1.806},"min":{"1min":0.307,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.78400000000000003,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.53600000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":1.2849999999999999,"15min":1.0780000000000001},"min":{"1min":0.34499999999999997,"5min":0.29899999999999999,"15min":0.28299999999999997},"max":{"1min":0.68200000000000005,"5min":12.545,"15min":12.545},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.998,"15min":1.3640000000000001},"min":{"1min":0.34100000000000003,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":0.69999999999999996,"5min":6.4729999999999999,"15min":14.193},"last":0.52400000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":1.141,"15min":1.3200000000000001},"min":{"1min":0.32300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.63,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.53200000000000003,"5min":1.2649999999999999,"15min":1.097},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.318},"max":{"1min":0.70399999999999996,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.51600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3557429248,"data_stored":3557425176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":871432192,"data_stored":871432192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1030684672,"data_stored":1030684672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3508998144,"data_stored":3508994072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1833172992,"data_stored":1833172992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2013925376,"data_stored":2013921304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1443561472,"data_stored":1443561472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:43:31.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:43:31.181 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:31.262 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:31.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:31.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:31.366 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:31.393 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:31.500 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:43:31.501 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:43:31.501 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:43:31.501 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:43:31.501 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:43:31.505 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:31 smithi138 ceph-mon[192513]: from='client.36840 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:31.505 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:31 smithi138 ceph-mon[192513]: pgmap v1507: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8039/211380 objects degraded (3.803%); 4326/211380 objects misplaced (2.047%); 557 KiB/s, 8 objects/s recovering 2023-10-01T09:43:31.506 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:31 smithi138 ceph-mon[185886]: from='client.36840 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:31.506 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:31 smithi138 ceph-mon[185886]: pgmap v1507: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8039/211380 objects degraded (3.803%); 4326/211380 objects misplaced (2.047%); 557 KiB/s, 8 objects/s recovering 2023-10-01T09:43:31.619 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:31.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:31.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:31.722 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:31.755 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:31.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:31 smithi190 ceph-mon[160435]: from='client.36840 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:31 smithi190 ceph-mon[160435]: pgmap v1507: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 8039/211380 objects degraded (3.803%); 4326/211380 objects misplaced (2.047%); 557 KiB/s, 8 objects/s recovering 2023-10-01T09:43:33.034 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:33.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:33.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:33.141 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:43:33.168 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:33.213 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:43:33.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:33.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:43:33.320 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:43:33.349 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:33.394 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:43:33.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:33.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:43:33.499 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:33.527 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:33.548 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:33.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:33 smithi190 ceph-mon[160435]: pgmap v1508: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 131 op/s; 8000/209667 objects degraded (3.816%); 4280/209667 objects misplaced (2.041%); 550 KiB/s, 8 objects/s recovering 2023-10-01T09:43:33.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:33 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 8000/209667 objects degraded (3.816%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:33.580 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:43:33.619 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:33.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:33.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:33.688 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:33 smithi138 ceph-mon[192513]: pgmap v1508: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 131 op/s; 8000/209667 objects degraded (3.816%); 4280/209667 objects misplaced (2.041%); 550 KiB/s, 8 objects/s recovering 2023-10-01T09:43:33.688 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:33 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 8000/209667 objects degraded (3.816%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:33.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:33 smithi138 ceph-mon[185886]: pgmap v1508: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 131 op/s; 8000/209667 objects degraded (3.816%); 4280/209667 objects misplaced (2.041%); 550 KiB/s, 8 objects/s recovering 2023-10-01T09:43:33.689 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:33 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 8000/209667 objects degraded (3.816%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:33.723 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:33.751 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:33.794 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:33.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:33.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:33.900 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:43:33.927 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:33.970 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:43:33.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:33.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:43:34.078 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:34.084 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:43:34.118 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:43:34.143 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:34.149 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:43:34.182 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:43:34.207 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:34.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:34.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:34.215 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:43:34.235 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:43:34.252 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:43:34.271 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:43:34.272 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:43:34.272 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:43:34.272 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:43:34.272 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:43:34.273 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:43:34.273 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:43:34.273 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:43:34.273 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:43:34.312 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:43:34.339 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:34.386 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:43:34.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:34.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:43:34.418 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:43:34.491 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:34.522 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:34.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:34 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/931549053' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:34.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:34 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3890879486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:43:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:34 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/931549053' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:34 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3890879486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:43:34.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:34 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/931549053' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:34.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:34 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3890879486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:43:35.309 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:35.311 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:43:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:43:34] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-01T09:43:35.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:35.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:35.413 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:35.444 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:35.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:35 smithi138 ceph-mon[192513]: pgmap v1509: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 132 op/s; 8000/209562 objects degraded (3.817%); 4280/209562 objects misplaced (2.042%); 550 KiB/s, 8 objects/s recovering 2023-10-01T09:43:35.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:35 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:35.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:35 smithi138 ceph-mon[192513]: failed to encode map e523 with expected crc 2023-10-01T09:43:35.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:35 smithi138 ceph-mon[192513]: failed to encode map e523 with expected crc 2023-10-01T09:43:35.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3890879486' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:43:35.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:35 smithi138 ceph-mon[192513]: osdmap e523: 8 total, 8 up, 7 in 2023-10-01T09:43:35.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:35 smithi138 ceph-mon[192513]: failed to encode map e523 with expected crc 2023-10-01T09:43:35.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:35 smithi138 ceph-mon[185886]: pgmap v1509: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 132 op/s; 8000/209562 objects degraded (3.817%); 4280/209562 objects misplaced (2.042%); 550 KiB/s, 8 objects/s recovering 2023-10-01T09:43:35.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:35 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:35.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:35 smithi138 ceph-mon[185886]: failed to encode map e523 with expected crc 2023-10-01T09:43:35.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:35 smithi138 ceph-mon[185886]: failed to encode map e523 with expected crc 2023-10-01T09:43:35.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3890879486' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:43:35.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:35 smithi138 ceph-mon[185886]: osdmap e523: 8 total, 8 up, 7 in 2023-10-01T09:43:35.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:35 smithi138 ceph-mon[185886]: failed to encode map e523 with expected crc 2023-10-01T09:43:35.591 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:35.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:35.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:35.695 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:35.722 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:35.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:35 smithi190 ceph-mon[160435]: pgmap v1509: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 132 op/s; 8000/209562 objects degraded (3.817%); 4280/209562 objects misplaced (2.042%); 550 KiB/s, 8 objects/s recovering 2023-10-01T09:43:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:35 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:35 smithi190 ceph-mon[160435]: failed to encode map e523 with expected crc 2023-10-01T09:43:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:35 smithi190 ceph-mon[160435]: failed to encode map e523 with expected crc 2023-10-01T09:43:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3890879486' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:43:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:35 smithi190 ceph-mon[160435]: osdmap e523: 8 total, 8 up, 7 in 2023-10-01T09:43:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:35 smithi190 ceph-mon[160435]: failed to encode map e523 with expected crc 2023-10-01T09:43:36.055 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:36.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:36.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:36.116 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:36.159 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:43:36.185 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:36.235 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:43:36.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:36.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:43:36.343 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:36.372 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:36.475 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:36.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:36.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:36.580 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:36.609 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:36.655 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:36.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:36.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:36.734 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:43:36.735 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 7947/208341 objects degraded (3.814%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2037,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":523,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":69447,"data_bytes":4554168267,"bytes_used":19296153600,"bytes_avail":652669095936,"bytes_total":671965249536,"degraded_objects":7947,"degraded_total":208341,"degraded_ratio":0.038144196293576398,"misplaced_objects":4241,"misplaced_total":208341,"misplaced_ratio":0.020356050897326977,"recovering_objects_per_sec":8,"recovering_bytes_per_sec":583123,"recovering_keys_per_sec":0,"num_objects_recovered":89,"num_bytes_recovered":5832704,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":135},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":33,"modified":"2023-10-01T09:41:58.284615+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:43:36.762 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:36.789 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:36.840 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:36.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:36.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:36.944 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:36.970 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:37.331 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:43:37.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:37 smithi138 ceph-mon[192513]: pgmap v1511: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 7947/208341 objects degraded (3.814%); 4241/208341 objects misplaced (2.036%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:43:37.717 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:37 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4136724810' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:43:37.717 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:37 smithi138 ceph-mon[185886]: pgmap v1511: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 7947/208341 objects degraded (3.814%); 4241/208341 objects misplaced (2.036%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:43:37.717 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:37 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4136724810' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:43:37.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:37.718 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:37.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:37.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:37 smithi190 ceph-mon[160435]: pgmap v1511: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 7947/208341 objects degraded (3.814%); 4241/208341 objects misplaced (2.036%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:43:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:37 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4136724810' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:43:37.819 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:43:37.847 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:43:37.847 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:43:37.848 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:37.885 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:37.891 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:43:37.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:37.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:43:37.998 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:38.029 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:38.110 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:43:38.142 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:38.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:38.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:38.247 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:43:38.276 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:38.324 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:43:38.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:38.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:43:38.430 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:38.433 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:43:38.433 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:43:38.461 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:38.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:38 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7947/208341 objects degraded (3.814%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:38.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:38 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7947/208341 objects degraded (3.814%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:38.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:38 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7947/208341 objects degraded (3.814%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:38.715 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:38.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:38.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:38.819 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:38.846 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:39.338 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:39.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:39.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:39.441 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:43:39.476 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:39.531 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:43:39.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:39.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:43:39.641 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:39.665 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:39 smithi138 ceph-mon[192513]: pgmap v1512: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 7947/207825 objects degraded (3.824%); 4241/207825 objects misplaced (2.041%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:43:39.665 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:39 smithi138 ceph-mon[185886]: pgmap v1512: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 7947/207825 objects degraded (3.824%); 4241/207825 objects misplaced (2.041%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:43:39.668 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:39.713 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:39.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:39.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:39.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:39 smithi190 ceph-mon[160435]: pgmap v1512: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 7947/207825 objects degraded (3.824%); 4241/207825 objects misplaced (2.041%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:43:39.816 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:39.848 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:39.896 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:39.927 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:43:40.004 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:40.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:40.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:40.077 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:43:40.098 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:43:40.110 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:40.112 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:43:40.118 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:43:40.139 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:40.141 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:43:40.162 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:43:40.183 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:43:40.185 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:43:40.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:40.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:43:40.206 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:43:40.231 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:43:40.232 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:43:40.232 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:43:40.232 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:43:40.232 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:43:40.232 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:43:40.233 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:43:40.233 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:43:40.233 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:43:40.290 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:43:40.318 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:40.365 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:43:40.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:40.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:43:40.424 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:40 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/879953358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:40.424 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:40.425 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:43:40.470 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:40.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:40 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/879953358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:40.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:40.476 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:43:40.476 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:40 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/879953358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:40.476 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:40.476 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:43:40.502 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:40.955 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:40.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:40.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:41.059 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:43:41.090 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:41.162 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:43:41.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:41.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:43:41.270 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:41.298 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:41.554 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:41 smithi138 ceph-mon[192513]: pgmap v1513: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 124 op/s; 7947/207657 objects degraded (3.827%); 4241/207657 objects misplaced (2.042%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:43:41.554 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:41 smithi138 ceph-mon[185886]: pgmap v1513: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 124 op/s; 7947/207657 objects degraded (3.827%); 4241/207657 objects misplaced (2.042%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:43:41.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:41 smithi190 ceph-mon[160435]: pgmap v1513: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 124 op/s; 7947/207657 objects degraded (3.827%); 4241/207657 objects misplaced (2.042%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:43:41.945 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:41.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:41.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:42.050 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:42.078 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:42.129 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:42.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:42.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:42.240 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:43:42.267 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:42.312 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:43:42.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:42.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:43:42.418 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:42.447 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:42.597 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:42.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:42.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:42.703 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:42.739 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:42.792 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:42.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:42.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:42.833 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:42.897 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:42.925 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:42.973 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:42.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:42.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:43.040 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:43:43.041 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:43:43.078 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:43.114 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:43.572 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:43:43.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:43 smithi190 ceph-mon[160435]: pgmap v1514: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 7898/206034 objects degraded (3.833%); 4199/206034 objects misplaced (2.038%); 601 KiB/s, 9 objects/s recovering 2023-10-01T09:43:43.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:43 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7898/206034 objects degraded (3.833%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:43.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:43 smithi138 ceph-mon[192513]: pgmap v1514: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 7898/206034 objects degraded (3.833%); 4199/206034 objects misplaced (2.038%); 601 KiB/s, 9 objects/s recovering 2023-10-01T09:43:43.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:43 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7898/206034 objects degraded (3.833%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:43.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:43 smithi138 ceph-mon[185886]: pgmap v1514: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 7898/206034 objects degraded (3.833%); 4199/206034 objects misplaced (2.038%); 601 KiB/s, 9 objects/s recovering 2023-10-01T09:43:43.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:43 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7898/206034 objects degraded (3.833%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:43.869 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:43.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:43.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:43.973 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:44.002 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:44.071 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1514,"stamp":"2023-10-01T09:43:42.311594+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4503771083,"num_objects":68678,"num_object_clones":0,"num_object_copies":206034,"num_objects_missing_on_primary":2765,"num_objects_missing":2765,"num_objects_degraded":7898,"num_objects_misplaced":4199,"num_objects_unfound":0,"num_objects_dirty":68678,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":98896,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8338,"num_bytes_recovered":533300239,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98960,"ondisk_log_size":98960,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18705688,"kb_used_data":13664672,"kb_used_omap":39,"kb_used_meta":5040920,"kb_avail":637510376,"statfs":{"total":671965249536,"available":652810625024,"internally_reserved":0,"allocated":13992624128,"data_stored":13982793637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5161902703},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,0,0,0,0,1,2,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-79364096,"num_objects":-1211,"num_object_clones":0,"num_object_copies":-3633,"num_objects_missing_on_primary":-89,"num_objects_missing":-89,"num_objects_degraded":-102,"num_objects_misplaced":-81,"num_objects_unfound":0,"num_objects_dirty":-1211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1211,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":94,"num_bytes_recovered":6160384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002881"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":299,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.523162+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:43:25.523162+0000","last_peered":"2023-10-01T09:43:25.523162+0000","last_clean":"2023-10-01T09:43:25.523162+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:43:25.523162+0000","last_undegraded":"2023-10-01T09:43:25.523162+0000","last_fullsized":"2023-10-01T09:43:25.523162+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":142,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.393229+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:43:25.393229+0000","last_peered":"2023-10-01T09:43:25.393229+0000","last_clean":"2023-10-01T09:43:25.393229+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:43:25.393229+0000","last_undegraded":"2023-10-01T09:43:25.393229+0000","last_fullsized":"2023-10-01T09:43:25.393229+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":523,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.695438+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:43:34.695438+0000","last_peered":"2023-10-01T09:43:34.695438+0000","last_clean":"2023-10-01T09:43:34.695438+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:43:34.695438+0000","last_undegraded":"2023-10-01T09:43:34.695438+0000","last_fullsized":"2023-10-01T09:43:34.695438+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":581,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.519223+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:43:25.519223+0000","last_peered":"2023-10-01T09:43:25.519223+0000","last_clean":"2023-10-01T09:43:25.519223+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:43:25.519223+0000","last_undegraded":"2023-10-01T09:43:25.519223+0000","last_fullsized":"2023-10-01T09:43:25.519223+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":462,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.397488+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:43:25.397488+0000","last_peered":"2023-10-01T09:43:25.397488+0000","last_clean":"2023-10-01T09:43:25.397488+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:43:25.397488+0000","last_undegraded":"2023-10-01T09:43:25.397488+0000","last_fullsized":"2023-10-01T09:43:25.397488+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":204,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.580107+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:43:25.580107+0000","last_peered":"2023-10-01T09:43:25.580107+0000","last_clean":"2023-10-01T09:43:25.580107+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:43:25.580107+0000","last_undegraded":"2023-10-01T09:43:25.580107+0000","last_fullsized":"2023-10-01T09:43:25.580107+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":580,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.580216+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:43:25.580216+0000","last_peered":"2023-10-01T09:43:25.580216+0000","last_clean":"2023-10-01T09:43:25.580216+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:43:25.580216+0000","last_undegraded":"2023-10-01T09:43:25.580216+0000","last_fullsized":"2023-10-01T09:43:25.580216+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":509,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.512518+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:43:25.512518+0000","last_peered":"2023-10-01T09:43:25.512518+0000","last_clean":"2023-10-01T09:43:25.512518+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:43:25.512518+0000","last_undegraded":"2023-10-01T09:43:25.512518+0000","last_fullsized":"2023-10-01T09:43:25.512518+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":235,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.461581+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:43:25.461581+0000","last_peered":"2023-10-01T09:43:25.461581+0000","last_clean":"2023-10-01T09:43:25.461581+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:43:25.461581+0000","last_undegraded":"2023-10-01T09:43:25.461581+0000","last_fullsized":"2023-10-01T09:43:25.461581+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":42,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.396378+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:43:25.396378+0000","last_peered":"2023-10-01T09:43:25.396378+0000","last_clean":"2023-10-01T09:43:25.396378+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:43:25.396378+0000","last_undegraded":"2023-10-01T09:43:25.396378+0000","last_fullsized":"2023-10-01T09:43:25.396378+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":48,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.396650+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:43:25.396650+0000","last_peered":"2023-10-01T09:43:25.396650+0000","last_clean":"2023-10-01T09:43:25.396650+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:43:25.396650+0000","last_undegraded":"2023-10-01T09:43:25.396650+0000","last_fullsized":"2023-10-01T09:43:25.396650+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.393549+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:43:25.393549+0000","last_peered":"2023-10-01T09:43:25.393549+0000","last_clean":"2023-10-01T09:43:25.393549+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:43:25.393549+0000","last_undegraded":"2023-10-01T09:43:25.393549+0000","last_fullsized":"2023-10-01T09:43:25.393549+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":424,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.524388+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:43:25.524388+0000","last_peered":"2023-10-01T09:43:25.524388+0000","last_clean":"2023-10-01T09:43:25.524388+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:43:25.524388+0000","last_undegraded":"2023-10-01T09:43:25.524388+0000","last_fullsized":"2023-10-01T09:43:25.524388+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":305,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.522816+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:43:25.522816+0000","last_peered":"2023-10-01T09:43:25.522816+0000","last_clean":"2023-10-01T09:43:25.522816+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:43:25.522816+0000","last_undegraded":"2023-10-01T09:43:25.522816+0000","last_fullsized":"2023-10-01T09:43:25.522816+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":641,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.396801+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:43:25.396801+0000","last_peered":"2023-10-01T09:43:25.396801+0000","last_clean":"2023-10-01T09:43:25.396801+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:43:25.396801+0000","last_undegraded":"2023-10-01T09:43:25.396801+0000","last_fullsized":"2023-10-01T09:43:25.396801+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":378,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.397138+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:43:25.397138+0000","last_peered":"2023-10-01T09:43:25.397138+0000","last_clean":"2023-10-01T09:43:25.397138+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:43:25.397138+0000","last_undegraded":"2023-10-01T09:43:25.397138+0000","last_fullsized":"2023-10-01T09:43:25.397138+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":51,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.396640+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:43:25.396640+0000","last_peered":"2023-10-01T09:43:25.396640+0000","last_clean":"2023-10-01T09:43:25.396640+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:43:25.396640+0000","last_undegraded":"2023-10-01T09:43:25.396640+0000","last_fullsized":"2023-10-01T09:43:25.396640+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":37,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.461302+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:43:25.461302+0000","last_peered":"2023-10-01T09:43:25.461302+0000","last_clean":"2023-10-01T09:43:25.461302+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:43:25.461302+0000","last_undegraded":"2023-10-01T09:43:25.461302+0000","last_fullsized":"2023-10-01T09:43:25.461302+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":534,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.397279+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:43:25.397279+0000","last_peered":"2023-10-01T09:43:25.397279+0000","last_clean":"2023-10-01T09:43:25.397279+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:43:25.397279+0000","last_undegraded":"2023-10-01T09:43:25.397279+0000","last_fullsized":"2023-10-01T09:43:25.397279+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":545,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.519340+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:43:25.519340+0000","last_peered":"2023-10-01T09:43:25.519340+0000","last_clean":"2023-10-01T09:43:25.519340+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:43:25.519340+0000","last_undegraded":"2023-10-01T09:43:25.519340+0000","last_fullsized":"2023-10-01T09:43:25.519340+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":401,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.397417+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:43:25.397417+0000","last_peered":"2023-10-01T09:43:25.397417+0000","last_clean":"2023-10-01T09:43:25.397417+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:43:25.397417+0000","last_undegraded":"2023-10-01T09:43:25.397417+0000","last_fullsized":"2023-10-01T09:43:25.397417+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":51,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.397151+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:43:25.397151+0000","last_peered":"2023-10-01T09:43:25.397151+0000","last_clean":"2023-10-01T09:43:25.397151+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:43:25.397151+0000","last_undegraded":"2023-10-01T09:43:25.397151+0000","last_fullsized":"2023-10-01T09:43:25.397151+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"523'6130","reported_seq":6186,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:41.061374+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:43:41.061374+0000","last_peered":"2023-10-01T09:43:41.061374+0000","last_clean":"2023-10-01T09:43:41.061374+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:43:41.061374+0000","last_undegraded":"2023-10-01T09:43:41.061374+0000","last_fullsized":"2023-10-01T09:43:41.061374+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6130,"log_dups_size":0,"ondisk_log_size":6130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279314432,"num_objects":4262,"num_object_clones":0,"num_object_copies":12786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6130,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":46,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.396994+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:43:25.396994+0000","last_peered":"2023-10-01T09:43:25.396994+0000","last_clean":"2023-10-01T09:43:25.396994+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:43:25.396994+0000","last_undegraded":"2023-10-01T09:43:25.396994+0000","last_fullsized":"2023-10-01T09:43:25.396994+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":243,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.461069+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:43:25.461069+0000","last_peered":"2023-10-01T09:43:25.461069+0000","last_clean":"2023-10-01T09:43:25.461069+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:43:25.461069+0000","last_undegraded":"2023-10-01T09:43:25.461069+0000","last_fullsized":"2023-10-01T09:43:25.461069+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"523'6126","reported_seq":8975,"reported_epoch":523,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:43:41.045322+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:43:41.045322+0000","last_peered":"2023-10-01T09:43:41.045322+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:43:41.045322+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6126,"log_dups_size":0,"ondisk_log_size":6126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280494080,"num_objects":4280,"num_object_clones":0,"num_object_copies":12840,"num_objects_missing_on_primary":2765,"num_objects_missing":2765,"num_objects_degraded":6977,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6126,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":923,"num_bytes_recovered":60489728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1447},{"shards":"0,4,6","objects":2765},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":517,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.396898+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:43:25.396898+0000","last_peered":"2023-10-01T09:43:25.396898+0000","last_clean":"2023-10-01T09:43:25.396898+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:43:25.396898+0000","last_undegraded":"2023-10-01T09:43:25.396898+0000","last_fullsized":"2023-10-01T09:43:25.396898+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":578,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.393186+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:43:25.393186+0000","last_peered":"2023-10-01T09:43:25.393186+0000","last_clean":"2023-10-01T09:43:25.393186+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:43:25.393186+0000","last_undegraded":"2023-10-01T09:43:25.393186+0000","last_fullsized":"2023-10-01T09:43:25.393186+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"523'6142","reported_seq":6200,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:41.667308+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:43:41.667308+0000","last_peered":"2023-10-01T09:43:41.667308+0000","last_clean":"2023-10-01T09:43:41.667308+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:43:41.667308+0000","last_undegraded":"2023-10-01T09:43:41.667308+0000","last_fullsized":"2023-10-01T09:43:41.667308+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6142,"log_dups_size":0,"ondisk_log_size":6142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281149440,"num_objects":4290,"num_object_clones":0,"num_object_copies":12870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6142,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":603,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.396740+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:43:25.396740+0000","last_peered":"2023-10-01T09:43:25.396740+0000","last_clean":"2023-10-01T09:43:25.396740+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:43:25.396740+0000","last_undegraded":"2023-10-01T09:43:25.396740+0000","last_fullsized":"2023-10-01T09:43:25.396740+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":398,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.393385+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:43:25.393385+0000","last_peered":"2023-10-01T09:43:25.393385+0000","last_clean":"2023-10-01T09:43:25.393385+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:43:25.393385+0000","last_undegraded":"2023-10-01T09:43:25.393385+0000","last_fullsized":"2023-10-01T09:43:25.393385+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"523'6101","reported_seq":12764,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:41.670564+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:43:41.670564+0000","last_peered":"2023-10-01T09:43:41.670564+0000","last_clean":"2023-10-01T09:43:41.670564+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:43:41.670564+0000","last_undegraded":"2023-10-01T09:43:41.670564+0000","last_fullsized":"2023-10-01T09:43:41.670564+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6101,"log_dups_size":0,"ondisk_log_size":6101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281870336,"num_objects":4301,"num_object_clones":0,"num_object_copies":12903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6101,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":243,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.461312+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:43:25.461312+0000","last_peered":"2023-10-01T09:43:25.461312+0000","last_clean":"2023-10-01T09:43:25.461312+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:43:25.461312+0000","last_undegraded":"2023-10-01T09:43:25.461312+0000","last_fullsized":"2023-10-01T09:43:25.461312+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":501,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.397859+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:43:25.397859+0000","last_peered":"2023-10-01T09:43:25.397859+0000","last_clean":"2023-10-01T09:43:25.397859+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:43:25.397859+0000","last_undegraded":"2023-10-01T09:43:25.397859+0000","last_fullsized":"2023-10-01T09:43:25.397859+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"523'6124","reported_seq":6182,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:39.844421+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:43:39.844421+0000","last_peered":"2023-10-01T09:43:39.844421+0000","last_clean":"2023-10-01T09:43:39.844421+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:43:39.844421+0000","last_undegraded":"2023-10-01T09:43:39.844421+0000","last_fullsized":"2023-10-01T09:43:39.844421+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6124,"log_dups_size":0,"ondisk_log_size":6124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287309824,"num_objects":4384,"num_object_clones":0,"num_object_copies":13152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6124,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":103,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.523088+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:43:25.523088+0000","last_peered":"2023-10-01T09:43:25.523088+0000","last_clean":"2023-10-01T09:43:25.523088+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:43:25.523088+0000","last_undegraded":"2023-10-01T09:43:25.523088+0000","last_fullsized":"2023-10-01T09:43:25.523088+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":177,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.524437+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:43:25.524437+0000","last_peered":"2023-10-01T09:43:25.524437+0000","last_clean":"2023-10-01T09:43:25.524437+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:43:25.524437+0000","last_undegraded":"2023-10-01T09:43:25.524437+0000","last_fullsized":"2023-10-01T09:43:25.524437+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"523'6105","reported_seq":6191,"reported_epoch":523,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:43:41.061336+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:43:41.061336+0000","last_peered":"2023-10-01T09:43:41.061336+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:43:41.061336+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6105,"log_dups_size":0,"ondisk_log_size":6105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279379968,"num_objects":4263,"num_object_clones":0,"num_object_copies":12789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":911,"num_objects_misplaced":4199,"num_objects_unfound":0,"num_objects_dirty":4263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6105,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4199},{"shards":"4,5","objects":-847}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":421,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.523291+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:43:25.523291+0000","last_peered":"2023-10-01T09:43:25.523291+0000","last_clean":"2023-10-01T09:43:25.523291+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:43:25.523291+0000","last_undegraded":"2023-10-01T09:43:25.523291+0000","last_fullsized":"2023-10-01T09:43:25.523291+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":581,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.393286+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:43:25.393286+0000","last_peered":"2023-10-01T09:43:25.393286+0000","last_clean":"2023-10-01T09:43:25.393286+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:43:25.393286+0000","last_undegraded":"2023-10-01T09:43:25.393286+0000","last_fullsized":"2023-10-01T09:43:25.393286+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"523'6045","reported_seq":6110,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:37.179484+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:43:37.179484+0000","last_peered":"2023-10-01T09:43:37.179484+0000","last_clean":"2023-10-01T09:43:37.179484+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:43:37.179484+0000","last_undegraded":"2023-10-01T09:43:37.179484+0000","last_fullsized":"2023-10-01T09:43:37.179484+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6045,"log_dups_size":0,"ondisk_log_size":6045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283967488,"num_objects":4333,"num_object_clones":0,"num_object_copies":12999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6045,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":417,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.522648+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:43:25.522648+0000","last_peered":"2023-10-01T09:43:25.522648+0000","last_clean":"2023-10-01T09:43:25.522648+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:43:25.522648+0000","last_undegraded":"2023-10-01T09:43:25.522648+0000","last_fullsized":"2023-10-01T09:43:25.522648+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":258,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.523819+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:43:25.523819+0000","last_peered":"2023-10-01T09:43:25.523819+0000","last_clean":"2023-10-01T09:43:25.523819+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:43:25.523819+0000","last_undegraded":"2023-10-01T09:43:25.523819+0000","last_fullsized":"2023-10-01T09:43:25.523819+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"523'6198","reported_seq":6254,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:41.183485+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:43:41.183485+0000","last_peered":"2023-10-01T09:43:41.183485+0000","last_clean":"2023-10-01T09:43:41.183485+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:43:41.183485+0000","last_undegraded":"2023-10-01T09:43:41.183485+0000","last_fullsized":"2023-10-01T09:43:41.183485+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6198,"log_dups_size":0,"ondisk_log_size":6198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281804800,"num_objects":4300,"num_object_clones":0,"num_object_copies":12900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6198,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":385,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.393094+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:43:25.393094+0000","last_peered":"2023-10-01T09:43:25.393094+0000","last_clean":"2023-10-01T09:43:25.393094+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:43:25.393094+0000","last_undegraded":"2023-10-01T09:43:25.393094+0000","last_fullsized":"2023-10-01T09:43:25.393094+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":433,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.519172+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:43:25.519172+0000","last_peered":"2023-10-01T09:43:25.519172+0000","last_clean":"2023-10-01T09:43:25.519172+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:43:25.519172+0000","last_undegraded":"2023-10-01T09:43:25.519172+0000","last_fullsized":"2023-10-01T09:43:25.519172+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"523'5998","reported_seq":6054,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:41.148787+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:43:41.148787+0000","last_peered":"2023-10-01T09:43:41.148787+0000","last_clean":"2023-10-01T09:43:41.148787+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:43:41.148787+0000","last_undegraded":"2023-10-01T09:43:41.148787+0000","last_fullsized":"2023-10-01T09:43:41.148787+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":5998,"log_dups_size":0,"ondisk_log_size":5998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273940480,"num_objects":4180,"num_object_clones":0,"num_object_copies":12540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5998,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":916,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.397039+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:43:25.397039+0000","last_peered":"2023-10-01T09:43:25.397039+0000","last_clean":"2023-10-01T09:43:25.397039+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:43:25.397039+0000","last_undegraded":"2023-10-01T09:43:25.397039+0000","last_fullsized":"2023-10-01T09:43:25.397039+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":253,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.461456+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:43:25.461456+0000","last_peered":"2023-10-01T09:43:25.461456+0000","last_clean":"2023-10-01T09:43:25.461456+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:43:25.461456+0000","last_undegraded":"2023-10-01T09:43:25.461456+0000","last_fullsized":"2023-10-01T09:43:25.461456+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":46,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.393248+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:43:25.393248+0000","last_peered":"2023-10-01T09:43:25.393248+0000","last_clean":"2023-10-01T09:43:25.393248+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:43:25.393248+0000","last_undegraded":"2023-10-01T09:43:25.393248+0000","last_fullsized":"2023-10-01T09:43:25.393248+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"523'6050","reported_seq":6108,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:41.187148+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:43:41.187148+0000","last_peered":"2023-10-01T09:43:41.187148+0000","last_clean":"2023-10-01T09:43:41.187148+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:43:41.187148+0000","last_undegraded":"2023-10-01T09:43:41.187148+0000","last_fullsized":"2023-10-01T09:43:41.187148+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6050,"log_dups_size":0,"ondisk_log_size":6050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283115520,"num_objects":4320,"num_object_clones":0,"num_object_copies":12960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6050,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1188,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.461282+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:43:25.461282+0000","last_peered":"2023-10-01T09:43:25.461282+0000","last_clean":"2023-10-01T09:43:25.461282+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:43:25.461282+0000","last_undegraded":"2023-10-01T09:43:25.461282+0000","last_fullsized":"2023-10-01T09:43:25.461282+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":398,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.522998+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:43:25.522998+0000","last_peered":"2023-10-01T09:43:25.522998+0000","last_clean":"2023-10-01T09:43:25.522998+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:43:25.522998+0000","last_undegraded":"2023-10-01T09:43:25.522998+0000","last_fullsized":"2023-10-01T09:43:25.522998+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":226,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.461462+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:43:25.461462+0000","last_peered":"2023-10-01T09:43:25.461462+0000","last_clean":"2023-10-01T09:43:25.461462+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:43:25.461462+0000","last_undegraded":"2023-10-01T09:43:25.461462+0000","last_fullsized":"2023-10-01T09:43:25.461462+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"523'6113","reported_seq":13390,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:41.086270+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:43:41.086270+0000","last_peered":"2023-10-01T09:43:41.086270+0000","last_clean":"2023-10-01T09:43:41.086270+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:43:41.086270+0000","last_undegraded":"2023-10-01T09:43:41.086270+0000","last_fullsized":"2023-10-01T09:43:41.086270+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6113,"log_dups_size":0,"ondisk_log_size":6113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279379968,"num_objects":4263,"num_object_clones":0,"num_object_copies":12789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6113,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":868,"reported_epoch":522,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:43:25.397259+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:43:25.397259+0000","last_peered":"2023-10-01T09:43:25.397259+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:43:25.397259+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:43:25.397259+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":585,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.393097+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:43:25.393097+0000","last_peered":"2023-10-01T09:43:25.393097+0000","last_clean":"2023-10-01T09:43:25.393097+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:43:25.393097+0000","last_undegraded":"2023-10-01T09:43:25.393097+0000","last_fullsized":"2023-10-01T09:43:25.393097+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":46,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.393755+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:43:25.393755+0000","last_peered":"2023-10-01T09:43:25.393755+0000","last_clean":"2023-10-01T09:43:25.393755+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:43:25.393755+0000","last_undegraded":"2023-10-01T09:43:25.393755+0000","last_fullsized":"2023-10-01T09:43:25.393755+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"523'6002","reported_seq":6060,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:40.007115+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:43:40.007115+0000","last_peered":"2023-10-01T09:43:40.007115+0000","last_clean":"2023-10-01T09:43:40.007115+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:43:40.007115+0000","last_undegraded":"2023-10-01T09:43:40.007115+0000","last_fullsized":"2023-10-01T09:43:40.007115+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6002,"log_dups_size":0,"ondisk_log_size":6002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279838720,"num_objects":4270,"num_object_clones":0,"num_object_copies":12810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6002,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1639,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.461420+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:43:25.461420+0000","last_peered":"2023-10-01T09:43:25.461420+0000","last_clean":"2023-10-01T09:43:25.461420+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:43:25.461420+0000","last_undegraded":"2023-10-01T09:43:25.461420+0000","last_fullsized":"2023-10-01T09:43:25.461420+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":553,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.396377+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:43:25.396377+0000","last_peered":"2023-10-01T09:43:25.396377+0000","last_clean":"2023-10-01T09:43:25.396377+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:43:25.396377+0000","last_undegraded":"2023-10-01T09:43:25.396377+0000","last_fullsized":"2023-10-01T09:43:25.396377+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":331,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.461379+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:43:25.461379+0000","last_peered":"2023-10-01T09:43:25.461379+0000","last_clean":"2023-10-01T09:43:25.461379+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:43:25.461379+0000","last_undegraded":"2023-10-01T09:43:25.461379+0000","last_fullsized":"2023-10-01T09:43:25.461379+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":321,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.523357+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:43:25.523357+0000","last_peered":"2023-10-01T09:43:25.523357+0000","last_clean":"2023-10-01T09:43:25.523357+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:43:25.523357+0000","last_undegraded":"2023-10-01T09:43:25.523357+0000","last_fullsized":"2023-10-01T09:43:25.523357+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"523'6202","reported_seq":6258,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:41.168234+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:43:41.168234+0000","last_peered":"2023-10-01T09:43:41.168234+0000","last_clean":"2023-10-01T09:43:41.168234+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:43:41.168234+0000","last_undegraded":"2023-10-01T09:43:41.168234+0000","last_fullsized":"2023-10-01T09:43:41.168234+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6202,"log_dups_size":0,"ondisk_log_size":6202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283770880,"num_objects":4330,"num_object_clones":0,"num_object_copies":12990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6202,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":730,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.396471+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:43:25.396471+0000","last_peered":"2023-10-01T09:43:25.396471+0000","last_clean":"2023-10-01T09:43:25.396471+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:43:25.396471+0000","last_undegraded":"2023-10-01T09:43:25.396471+0000","last_fullsized":"2023-10-01T09:43:25.396471+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":562,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.397637+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:43:25.397637+0000","last_peered":"2023-10-01T09:43:25.397637+0000","last_clean":"2023-10-01T09:43:25.397637+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:43:25.397637+0000","last_undegraded":"2023-10-01T09:43:25.397637+0000","last_fullsized":"2023-10-01T09:43:25.397637+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":562,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:26.860777+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:43:26.860777+0000","last_peered":"2023-10-01T09:43:26.860777+0000","last_clean":"2023-10-01T09:43:26.860777+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:43:26.860777+0000","last_undegraded":"2023-10-01T09:43:26.860777+0000","last_fullsized":"2023-10-01T09:43:26.860777+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"523'6163","reported_seq":6222,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:41.663296+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:43:41.663296+0000","last_peered":"2023-10-01T09:43:41.663296+0000","last_clean":"2023-10-01T09:43:41.663296+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:43:41.663296+0000","last_undegraded":"2023-10-01T09:43:41.663296+0000","last_fullsized":"2023-10-01T09:43:41.663296+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6163,"log_dups_size":0,"ondisk_log_size":6163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281542680,"num_objects":4297,"num_object_clones":0,"num_object_copies":12891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4297,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6163,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":961,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.396820+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:43:25.396820+0000","last_peered":"2023-10-01T09:43:25.396820+0000","last_clean":"2023-10-01T09:43:25.396820+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:43:25.396820+0000","last_undegraded":"2023-10-01T09:43:25.396820+0000","last_fullsized":"2023-10-01T09:43:25.396820+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":580,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.461410+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:43:25.461410+0000","last_peered":"2023-10-01T09:43:25.461410+0000","last_clean":"2023-10-01T09:43:25.461410+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:43:25.461410+0000","last_undegraded":"2023-10-01T09:43:25.461410+0000","last_fullsized":"2023-10-01T09:43:25.461410+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":433,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.461498+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:43:25.461498+0000","last_peered":"2023-10-01T09:43:25.461498+0000","last_clean":"2023-10-01T09:43:25.461498+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:43:25.461498+0000","last_undegraded":"2023-10-01T09:43:25.461498+0000","last_fullsized":"2023-10-01T09:43:25.461498+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"523'6140","reported_seq":6204,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:41.128852+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:43:41.128852+0000","last_peered":"2023-10-01T09:43:41.128852+0000","last_clean":"2023-10-01T09:43:41.128852+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:43:41.128852+0000","last_undegraded":"2023-10-01T09:43:41.128852+0000","last_fullsized":"2023-10-01T09:43:41.128852+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6140,"log_dups_size":0,"ondisk_log_size":6140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276299776,"num_objects":4216,"num_object_clones":0,"num_object_copies":12648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6140,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":462,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.397230+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:43:25.397230+0000","last_peered":"2023-10-01T09:43:25.397230+0000","last_clean":"2023-10-01T09:43:25.397230+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:43:25.397230+0000","last_undegraded":"2023-10-01T09:43:25.397230+0000","last_fullsized":"2023-10-01T09:43:25.397230+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1155,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.393548+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:43:25.393548+0000","last_peered":"2023-10-01T09:43:25.393548+0000","last_clean":"2023-10-01T09:43:25.393548+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:43:25.393548+0000","last_undegraded":"2023-10-01T09:43:25.393548+0000","last_fullsized":"2023-10-01T09:43:25.393548+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":51,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.396315+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:43:25.396315+0000","last_peered":"2023-10-01T09:43:25.396315+0000","last_clean":"2023-10-01T09:43:25.396315+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:43:25.396315+0000","last_undegraded":"2023-10-01T09:43:25.396315+0000","last_fullsized":"2023-10-01T09:43:25.396315+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"523'6151","reported_seq":6215,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:41.199052+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:43:41.199052+0000","last_peered":"2023-10-01T09:43:41.199052+0000","last_clean":"2023-10-01T09:43:41.199052+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:43:41.199052+0000","last_undegraded":"2023-10-01T09:43:41.199052+0000","last_fullsized":"2023-10-01T09:43:41.199052+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6151,"log_dups_size":0,"ondisk_log_size":6151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285802496,"num_objects":4361,"num_object_clones":0,"num_object_copies":13083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6151,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":55,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.397079+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:43:25.397079+0000","last_peered":"2023-10-01T09:43:25.397079+0000","last_clean":"2023-10-01T09:43:25.397079+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:43:25.397079+0000","last_undegraded":"2023-10-01T09:43:25.397079+0000","last_fullsized":"2023-10-01T09:43:25.397079+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":540,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:26.860794+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:43:26.860794+0000","last_peered":"2023-10-01T09:43:26.860794+0000","last_clean":"2023-10-01T09:43:26.860794+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:43:26.860794+0000","last_undegraded":"2023-10-01T09:43:26.860794+0000","last_fullsized":"2023-10-01T09:43:26.860794+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":246,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.461181+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:43:25.461181+0000","last_peered":"2023-10-01T09:43:25.461181+0000","last_clean":"2023-10-01T09:43:25.461181+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:43:25.461181+0000","last_undegraded":"2023-10-01T09:43:25.461181+0000","last_fullsized":"2023-10-01T09:43:25.461181+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":489,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:26.860726+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:43:26.860726+0000","last_peered":"2023-10-01T09:43:26.860726+0000","last_clean":"2023-10-01T09:43:26.860726+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:43:26.860726+0000","last_undegraded":"2023-10-01T09:43:26.860726+0000","last_fullsized":"2023-10-01T09:43:26.860726+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":602,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.393594+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:43:25.393594+0000","last_peered":"2023-10-01T09:43:25.393594+0000","last_clean":"2023-10-01T09:43:25.393594+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:43:25.393594+0000","last_undegraded":"2023-10-01T09:43:25.393594+0000","last_fullsized":"2023-10-01T09:43:25.393594+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":258,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.461225+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:43:25.461225+0000","last_peered":"2023-10-01T09:43:25.461225+0000","last_clean":"2023-10-01T09:43:25.461225+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:43:25.461225+0000","last_undegraded":"2023-10-01T09:43:25.461225+0000","last_fullsized":"2023-10-01T09:43:25.461225+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":46,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.393150+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:43:25.393150+0000","last_peered":"2023-10-01T09:43:25.393150+0000","last_clean":"2023-10-01T09:43:25.393150+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:43:25.393150+0000","last_undegraded":"2023-10-01T09:43:25.393150+0000","last_fullsized":"2023-10-01T09:43:25.393150+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":258,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.461538+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:43:25.461538+0000","last_peered":"2023-10-01T09:43:25.461538+0000","last_clean":"2023-10-01T09:43:25.461538+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:43:25.461538+0000","last_undegraded":"2023-10-01T09:43:25.461538+0000","last_fullsized":"2023-10-01T09:43:25.461538+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":435,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.397786+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:43:25.397786+0000","last_peered":"2023-10-01T09:43:25.397786+0000","last_clean":"2023-10-01T09:43:25.397786+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:43:25.397786+0000","last_undegraded":"2023-10-01T09:43:25.397786+0000","last_fullsized":"2023-10-01T09:43:25.397786+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":63,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.393406+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:43:25.393406+0000","last_peered":"2023-10-01T09:43:25.393406+0000","last_clean":"2023-10-01T09:43:25.393406+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:43:25.393406+0000","last_undegraded":"2023-10-01T09:43:25.393406+0000","last_fullsized":"2023-10-01T09:43:25.393406+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":480,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.397610+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:43:25.397610+0000","last_peered":"2023-10-01T09:43:25.397610+0000","last_clean":"2023-10-01T09:43:25.397610+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:43:25.397610+0000","last_undegraded":"2023-10-01T09:43:25.397610+0000","last_fullsized":"2023-10-01T09:43:25.397610+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":679,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.519237+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:43:25.519237+0000","last_peered":"2023-10-01T09:43:25.519237+0000","last_clean":"2023-10-01T09:43:25.519237+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:43:25.519237+0000","last_undegraded":"2023-10-01T09:43:25.519237+0000","last_fullsized":"2023-10-01T09:43:25.519237+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":374,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-10-01T09:43:25.522532+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:43:25.522532+0000","last_peered":"2023-10-01T09:43:25.522532+0000","last_clean":"2023-10-01T09:43:25.522532+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:43:25.522532+0000","last_undegraded":"2023-10-01T09:43:25.522532+0000","last_fullsized":"2023-10-01T09:43:25.522532+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4498980888,"num_objects":68650,"num_object_clones":0,"num_object_copies":205950,"num_objects_missing_on_primary":2765,"num_objects_missing":2765,"num_objects_degraded":7888,"num_objects_misplaced":4199,"num_objects_unfound":0,"num_objects_dirty":68650,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":97790,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7827,"num_bytes_recovered":512950272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13920579584,"data_stored":13920567368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97790,"ondisk_log_size":97790,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430717,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2286824,"kb_used_data":1379616,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":91458328,"statfs":{"total":95995035648,"available":93653327872,"internally_reserved":0,"allocated":1412726784,"data_stored":1411790052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:56 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.4950000000000001,"15min":2.9569999999999999},"min":{"1min":0.246,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":1.3280000000000001,"5min":10.391999999999999,"15min":108.90000000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":1.8129999999999999,"15min":3.129},"min":{"1min":0.246,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.018,"5min":29.501000000000001,"15min":109.187},"last":0.34999999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":4.6749999999999998,"15min":5.3520000000000003},"min":{"1min":0.31900000000000001,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":1.2949999999999999,"5min":229.161,"15min":229.161},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":4.6479999999999997,"15min":5.3689999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":0.90900000000000003,"5min":230.126,"15min":230.126},"last":0.45100000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:42:50 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":1.5369999999999999,"15min":1.891},"min":{"1min":0.33900000000000002,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":1.3560000000000001,"5min":14.866,"15min":14.901999999999999},"last":0.442},{"interface":"front","average":{"1min":0.49299999999999999,"5min":1.7909999999999999,"15min":1.821},"min":{"1min":0.33500000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.96599999999999997,"5min":14.682,"15min":14.682},"last":0.51700000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":1.514,"15min":2.7330000000000001},"min":{"1min":0.30199999999999999,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":0.90700000000000003,"5min":35.104999999999997,"15min":80.352999999999994},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.50600000000000001,"5min":1.6930000000000001,"15min":2.7130000000000001},"min":{"1min":0.35399999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.153,"5min":35.118000000000002,"15min":80.364000000000004},"last":0.46200000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:42:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":1.2150000000000001,"15min":1.052},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.381,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.504},{"interface":"front","average":{"1min":0.49099999999999999,"5min":1.1080000000000001,"15min":1.03},"min":{"1min":0.314,"5min":0.30299999999999999,"15min":0.28000000000000003},"max":{"1min":0.97899999999999998,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.54700000000000004}]},{"osd":5,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":2.3620000000000001,"15min":1.6120000000000001},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":1.2749999999999999,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.62,"5min":2.5390000000000001,"15min":1.6739999999999999},"min":{"1min":0.35999999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.2599999999999998,"5min":135.339,"15min":135.339},"last":0.56599999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":1.3200000000000001,"15min":1.1599999999999999},"min":{"1min":0.377,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":1.409,"5min":11.58,"15min":11.853999999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":1.175,"15min":1.1299999999999999},"min":{"1min":0.378,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":1.23,"5min":14.584,"15min":14.584},"last":0.60999999999999999}]}]},{"osd":6,"up_from":294,"seq":1262720385227,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2273508,"kb_used_data":1944092,"kb_used_omap":6,"kb_used_meta":329401,"kb_avail":91471644,"statfs":{"total":95995035648,"available":93666963456,"internally_reserved":0,"allocated":1990750208,"data_stored":1989817427,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":337306669},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":3.5720000000000001,"15min":4.0919999999999996},"min":{"1min":0.25,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":3.3340000000000001,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.247},{"interface":"front","average":{"1min":0.42599999999999999,"5min":4.0410000000000004,"15min":4.2210000000000001},"min":{"1min":0.29899999999999999,"5min":0.249,"15min":0.221},"max":{"1min":0.623,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.26300000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":4.7489999999999997,"15min":4.9740000000000002},"min":{"1min":0.30599999999999999,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":1.9410000000000001,"5min":168.947,"15min":168.947},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":4.7969999999999997,"15min":4.8810000000000002},"min":{"1min":0.34699999999999998,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":1.962,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.376}]},{"osd":2,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":1.194,"15min":1.673},"min":{"1min":0.308,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.075,"5min":8.657,"15min":10.92},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.3759999999999999,"15min":1.673},"min":{"1min":0.314,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.58499999999999996,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.35399999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.3069999999999999,"15min":2.2789999999999999},"min":{"1min":0.35499999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.3759999999999999,"5min":9.0180000000000007,"15min":89.753},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.502,"5min":1.4359999999999999,"15min":2.157},"min":{"1min":0.34399999999999997,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":1.381,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.308}]},{"osd":4,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":1.5900000000000001,"15min":1.0800000000000001},"min":{"1min":0.30099999999999999,"5min":0.253,"15min":0.246},"max":{"1min":0.53400000000000003,"5min":63.134,"15min":63.134},"last":0.438},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.5009999999999999,"15min":1.097},"min":{"1min":0.33700000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":1.5669999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.36899999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:43:29 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":1.2450000000000001,"15min":1.101},"min":{"1min":0.309,"5min":0.251,"15min":0.251},"max":{"1min":34.094999999999999,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.409,"15min":1.22},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":34.072000000000003,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.41799999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.79500000000000004,"15min":1.46},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":1.4750000000000001,"5min":6.4119999999999999,"15min":94.245999999999995},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.48999999999999999,"5min":1.113,"15min":1.5569999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":0.68300000000000005,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.38600000000000001}]}]},{"osd":1,"up_from":94,"seq":403726926204,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1659796,"kb_used_data":841612,"kb_used_omap":7,"kb_used_meta":818168,"kb_avail":92085356,"statfs":{"total":95995035648,"available":94295404544,"internally_reserved":0,"allocated":861810688,"data_stored":859779912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":837804455},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.30299999999999999,"5min":2.5600000000000001,"15min":4.2009999999999996},"min":{"1min":0.20699999999999999,"5min":0.19400000000000001,"15min":0.182},"max":{"1min":0.57399999999999995,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.38},{"interface":"front","average":{"1min":0.33600000000000002,"5min":3.0139999999999998,"15min":3.7090000000000001},"min":{"1min":0.26300000000000001,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":0.53400000000000003,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.24299999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":1.4279999999999999,"15min":2.589},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.61399999999999999,"5min":22.891999999999999,"15min":56.573},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.379,"15min":3.254},"min":{"1min":0.28999999999999998,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":1.752,"5min":15.189,"15min":154.197},"last":0.30599999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":2.1680000000000001,"15min":3.9969999999999999},"min":{"1min":0.27400000000000002,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":1.1990000000000001,"5min":18.503,"15min":200.56800000000001},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.39300000000000002,"5min":2.3660000000000001,"15min":4.6989999999999998},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":0.51900000000000002,"5min":20.181000000000001,"15min":200.63800000000001},"last":1.5740000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.8029999999999999,"15min":2.3109999999999999},"min":{"1min":0.32900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.8109999999999999,"5min":21.015000000000001,"15min":154.101},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.46400000000000002,"5min":1.585,"15min":2.5249999999999999},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.83199999999999996,"5min":10.222,"15min":154.56999999999999},"last":0.45600000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:43:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.3200000000000001,"15min":2.097},"min":{"1min":0.372,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.1850000000000001,"5min":19.513000000000002,"15min":154.22999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.1719999999999999,"15min":2.4630000000000001},"min":{"1min":0.31,"5min":0.307,"15min":0.307},"max":{"1min":0.82699999999999996,"5min":10.305,"15min":154.54400000000001},"last":0.49099999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.849,"15min":2.6640000000000001},"min":{"1min":0.372,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.2789999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.495,"5min":1.702,"15min":2.4870000000000001},"min":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":0.67300000000000004,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.5}]},{"osd":7,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.9319999999999999,"15min":2.6440000000000001},"min":{"1min":0.40100000000000002,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":1.3029999999999999,"5min":21.062999999999999,"15min":154.20699999999999},"last":2.4700000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.804,"15min":2.718},"min":{"1min":0.41699999999999998,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":1.3220000000000001,"5min":13.161,"15min":154.51599999999999},"last":2.48}]}]},{"osd":0,"up_from":89,"seq":382252089739,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4313476,"kb_used_data":3418236,"kb_used_omap":8,"kb_used_meta":895223,"kb_avail":89431676,"statfs":{"total":95995035648,"available":91578036224,"internally_reserved":0,"allocated":3500273664,"data_stored":3498263796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":916709314},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":1.76,"15min":4.2480000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17399999999999999},"max":{"1min":1.891,"5min":52.316000000000003,"15min":216.821},"last":0.438},{"interface":"front","average":{"1min":0.52200000000000002,"5min":3.4119999999999999,"15min":5.2130000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":3.552,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.34200000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:43:36 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":1.204,"15min":2.085},"min":{"1min":0.246,"5min":0.246,"15min":0.20100000000000001},"max":{"1min":1.1100000000000001,"5min":19.788,"15min":19.788},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.4139999999999999,"15min":2.3380000000000001},"min":{"1min":0.27200000000000002,"5min":0.23000000000000001,"15min":0.189},"max":{"1min":3.5939999999999999,"5min":19.375,"15min":25.844999999999999},"last":0.379}]},{"osd":3,"last update":"Sun Oct 1 09:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":1.474,"15min":3.907},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":1.1499999999999999,"5min":15.35,"15min":222.16499999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.748,"15min":3.9460000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":2.9550000000000001,"5min":14.545999999999999,"15min":222.227},"last":3.4119999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:42:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.8859999999999999,"15min":2.266},"min":{"1min":0.29299999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":3.694,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.443,"5min":1.9850000000000001,"15min":2.2639999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28299999999999997},"max":{"1min":0.59799999999999998,"5min":47.854999999999997,"15min":62.073999999999998},"last":3.5049999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:43:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.2030000000000001,"15min":1.883},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66500000000000004,"5min":14.515000000000001,"15min":16.808},"last":3.476},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.093,"15min":1.7709999999999999},"min":{"1min":0.33500000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.73499999999999999,"5min":5.9580000000000002,"15min":17.507000000000001},"last":0.63900000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:43:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.1850000000000001,"15min":1.774},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.35499999999999998},"max":{"1min":1.5900000000000001,"5min":6.5,"15min":17.535},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.88,"5min":1.395,"15min":2.0289999999999999},"min":{"1min":0.32600000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":3.6539999999999999,"5min":10.446,"15min":18.521000000000001},"last":0.60699999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:42:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":1.671,"15min":2.1139999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":0.77100000000000002,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.83599999999999997,"5min":1.97,"15min":2.4020000000000001},"min":{"1min":0.41899999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":3.7320000000000002,"5min":48.274000000000001,"15min":48.274000000000001},"last":0.57599999999999996}]}]},{"osd":2,"up_from":0,"seq":425201762678,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599151,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1817024,"kb_used_data":980792,"kb_used_omap":3,"kb_used_meta":836220,"kb_avail":91928128,"statfs":{"total":95995035648,"available":94134403072,"internally_reserved":0,"allocated":1004331008,"data_stored":1002261222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":856290023},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":4.9660000000000002,"15min":5.0880000000000001},"min":{"1min":0.20300000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":2.7989999999999999,"5min":118.748,"15min":137.34399999999999},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.28899999999999998,"5min":4.9500000000000002,"15min":5.1379999999999999},"min":{"1min":0.20699999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.42199999999999999,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.23300000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":3.4329999999999998,"15min":5.2869999999999999},"min":{"1min":0.23000000000000001,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":2.7250000000000001,"5min":90.944000000000003,"15min":162.22999999999999},"last":0.313},{"interface":"front","average":{"1min":0.33300000000000002,"5min":3.1850000000000001,"15min":5.4160000000000004},"min":{"1min":0.23100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.45300000000000001,"5min":83.341999999999999,"15min":162.70699999999999},"last":0.32300000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":1.8740000000000001,"15min":2.2029999999999998},"min":{"1min":0.251,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":0.54900000000000004,"5min":14.254,"15min":36.472999999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.90700000000000003,"15min":2.4910000000000001},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":0.59999999999999998,"5min":7.6749999999999998,"15min":78.594999999999999},"last":0.35399999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":1.8859999999999999,"15min":1.875},"min":{"1min":0.32600000000000001,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":0.65300000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.42799999999999999,"5min":2.2189999999999999,"15min":2.1469999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":0.64300000000000002,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.46700000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":1.5589999999999999,"15min":2.581},"min":{"1min":0.317,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3099999999999996,"5min":10.919,"15min":28.117999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.4279999999999999,"15min":2.6800000000000002},"min":{"1min":0.36699999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.512,"5min":10.798999999999999,"15min":28.34},"last":0.379}]},{"osd":6,"last update":"Sun Oct 1 09:43:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":2.165,"15min":2.145},"min":{"1min":0.34200000000000003,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":2.5430000000000001,"5min":73.887,"15min":73.887},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.63,"5min":2.4159999999999999,"15min":2.1379999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":2.6000000000000001,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.45300000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":1.7889999999999999,"15min":2.3090000000000002},"min":{"1min":0.39400000000000002,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":4.1580000000000004,"5min":13.085000000000001,"15min":52.006},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.306,"15min":2.032},"min":{"1min":0.41899999999999998,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":4.1779999999999999,"5min":8.3460000000000001,"15min":51.890999999999998},"last":0.495}]}]},{"osd":4,"up_from":30,"seq":128849019402,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4247140,"kb_used_data":3352860,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":89498012,"statfs":{"total":95995035648,"available":91645964288,"internally_reserved":0,"allocated":3433328640,"data_stored":3432395414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:53 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":3.129,"15min":4.7549999999999999},"min":{"1min":0.26100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.66400000000000003,"5min":79.837000000000003,"15min":155.315},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.53900000000000003,"5min":3.2949999999999999,"15min":5.1200000000000001},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.251},"max":{"1min":2.843,"5min":78.983000000000004,"15min":155.334},"last":0.52500000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:42:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":6.8879999999999999,"15min":6.556},"min":{"1min":0.252,"5min":0.224,"15min":0.224},"max":{"1min":2.8889999999999998,"5min":146.929,"15min":190.61199999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":6.8300000000000001,"15min":7.266},"min":{"1min":0.27500000000000002,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":0.624,"5min":146.91,"15min":253.273},"last":0.41199999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:42:43 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":1.2809999999999999,"15min":2.6019999999999999},"min":{"1min":0.34799999999999998,"5min":0.30399999999999999,"15min":0.28299999999999997},"max":{"1min":2.8210000000000002,"5min":10.161,"15min":170.45400000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.4410000000000001,"15min":2.2730000000000001},"min":{"1min":0.38400000000000001,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":2.1930000000000001,"5min":10.625,"15min":58.771000000000001},"last":0.42299999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:42:58 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":1.1799999999999999,"15min":2.7519999999999998},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.67500000000000004,"5min":7.5830000000000002,"15min":140.881},"last":0.313},{"interface":"front","average":{"1min":0.47099999999999997,"5min":1.208,"15min":2.976},"min":{"1min":0.33300000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.69599999999999995,"5min":9.3140000000000001,"15min":140.911},"last":0.38300000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:43:33 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":2.0569999999999999,"15min":1.5580000000000001},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":3.052,"5min":110.003,"15min":110.003},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.64900000000000002,"5min":2.0110000000000001,"15min":1.399},"min":{"1min":0.33500000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.0049999999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.434}]},{"osd":6,"last update":"Sun Oct 1 09:43:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.79200000000000004,"15min":0.99299999999999999},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":0.80900000000000005,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.37},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.95199999999999996,"15min":1.085},"min":{"1min":0.379,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":2.9239999999999999,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.58399999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:42:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":1.159,"15min":1.1799999999999999},"min":{"1min":0.38100000000000001,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":1.825,"5min":38.034999999999997,"15min":38.034999999999997},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.623,"5min":1.47,"15min":1.462},"min":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":2.2149999999999999,"5min":37.723999999999997,"15min":37.723999999999997},"last":0.61299999999999999}]}]},{"osd":5,"up_from":379,"seq":1627792605309,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2107920,"kb_used_data":1747464,"kb_used_omap":13,"kb_used_meta":360434,"kb_avail":91637232,"statfs":{"total":95995035648,"available":93836525568,"internally_reserved":0,"allocated":1789403136,"data_stored":1788485814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":369084914},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,0,0,0,1,2,2],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":1.6479999999999999,"15min":2.1890000000000001},"min":{"1min":0.22900000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.1880000000000002,"5min":12.571999999999999,"15min":84.340999999999994},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.40400000000000003,"5min":1.2549999999999999,"15min":2.0800000000000001},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.53900000000000003,"5min":8.6460000000000008,"15min":84.356999999999999},"last":0.41099999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":6.4870000000000001,"15min":5.7599999999999998},"min":{"1min":0.26000000000000001,"5min":0.25900000000000001,"15min":0.255},"max":{"1min":0.76900000000000002,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.443},{"interface":"front","average":{"1min":0.53800000000000003,"5min":6.8099999999999996,"15min":5.7320000000000002},"min":{"1min":0.32200000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":2.258,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.436}]},{"osd":2,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":1.3600000000000001,"15min":1.5980000000000001},"min":{"1min":0.315,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.752,"5min":8.7040000000000006,"15min":14.199999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.44800000000000001,"5min":1.306,"15min":1.583},"min":{"1min":0.32900000000000001,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":0.85999999999999999,"5min":8.2370000000000001,"15min":14.430999999999999},"last":0.42599999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.5069999999999999,"15min":1.488},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.2280000000000002,"5min":12.734,"15min":14.004},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.5640000000000001,"15min":1.629},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.3979999999999999,"5min":11.766,"15min":13.991},"last":0.46200000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":2.2029999999999998,"15min":1.4330000000000001},"min":{"1min":0.30099999999999999,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.66800000000000004,"5min":122.971,"15min":122.971},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.438,"5min":2.25,"15min":1.724},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.73799999999999999,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.33500000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":1.165,"15min":1.044},"min":{"1min":0.312,"5min":0.311,"15min":0.28299999999999997},"max":{"1min":2.2839999999999998,"5min":12.545,"15min":12.545},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.99099999999999999,"15min":1.302},"min":{"1min":0.378,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":2.629,"5min":6.4729999999999999,"15min":14.193},"last":0.496}]},{"osd":7,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.1140000000000001,"15min":1.262},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.81200000000000006,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.2270000000000001,"15min":1.0780000000000001},"min":{"1min":0.36299999999999999,"5min":0.35299999999999998,"15min":0.318},"max":{"1min":2.3079999999999998,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.45200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3483897856,"data_stored":3483893784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":850001920,"data_stored":850001920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":998178816,"data_stored":998178816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3423866880,"data_stored":3423862808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1781530624,"data_stored":1781530624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1982402560,"data_stored":1982398488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1400700928,"data_stored":1400700928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:43:44.100 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:44.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:44.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:44.204 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:44.238 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:44.290 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:44.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:44.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:44.394 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:44.421 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:44.440 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:43:44.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:44 smithi138 ceph-mon[192513]: from='client.36870 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:44.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:44 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2680695045' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:43:44.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:44 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:43:44.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:44 smithi138 ceph-mon[185886]: from='client.36870 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:44.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:44 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2680695045' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:43:44.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:44 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:43:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:44 smithi190 ceph-mon[160435]: from='client.36870 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:44 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2680695045' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:43:44.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:44 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:43:45.083 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:43:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:43:44] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-01T09:43:45.223 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:45.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:45.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:45.327 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:45.358 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:45.416 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:45.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:45.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:45.522 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:45.561 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:45.652 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:45.662 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:45.670 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:45 smithi138 ceph-mon[192513]: pgmap v1515: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 7898/205944 objects degraded (3.835%); 4199/205944 objects misplaced (2.039%); 601 KiB/s, 9 objects/s recovering 2023-10-01T09:43:45.670 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:45 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:45.670 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:45 smithi138 ceph-mon[192513]: failed to encode map e524 with expected crc 2023-10-01T09:43:45.671 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:45 smithi138 ceph-mon[192513]: failed to encode map e524 with expected crc 2023-10-01T09:43:45.671 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:45 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:43:45.671 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:45 smithi138 ceph-mon[192513]: osdmap e524: 8 total, 8 up, 7 in 2023-10-01T09:43:45.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:45.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:45.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:45 smithi138 ceph-mon[185886]: pgmap v1515: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 7898/205944 objects degraded (3.835%); 4199/205944 objects misplaced (2.039%); 601 KiB/s, 9 objects/s recovering 2023-10-01T09:43:45.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:45 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:45.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:45 smithi138 ceph-mon[185886]: failed to encode map e524 with expected crc 2023-10-01T09:43:45.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:45 smithi138 ceph-mon[185886]: failed to encode map e524 with expected crc 2023-10-01T09:43:45.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:45 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:43:45.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:45 smithi138 ceph-mon[185886]: osdmap e524: 8 total, 8 up, 7 in 2023-10-01T09:43:45.753 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:43:45.753 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:43:45.753 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-10-01T09:43:45.753 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:43:45.773 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:45.801 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:45.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:45 smithi190 ceph-mon[160435]: pgmap v1515: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 7898/205944 objects degraded (3.835%); 4199/205944 objects misplaced (2.039%); 601 KiB/s, 9 objects/s recovering 2023-10-01T09:43:45.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:45 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:43:45.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:45 smithi190 ceph-mon[160435]: failed to encode map e524 with expected crc 2023-10-01T09:43:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:45 smithi190 ceph-mon[160435]: failed to encode map e524 with expected crc 2023-10-01T09:43:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:45 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:43:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:45 smithi190 ceph-mon[160435]: osdmap e524: 8 total, 8 up, 7 in 2023-10-01T09:43:45.903 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:45.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:45.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:46.007 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:46.035 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:46.080 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:46.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:46.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:46.184 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:46.211 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:47.080 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:43:47.147 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:47.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:47.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:47.241 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:43:47.252 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:47.280 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:47.334 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:47.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:47.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:47.342 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:43:47.440 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:47.467 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:47.510 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:47.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:47.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:47.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:47 smithi190 ceph-mon[160435]: pgmap v1517: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 107 op/s; 7898/205119 objects degraded (3.850%); 4199/205119 objects misplaced (2.047%); 301 KiB/s, 4 objects/s recovering 2023-10-01T09:43:47.615 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:47.643 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:47.650 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:47.656 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:43:47.684 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:43:47.687 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:47.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:47.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:47.699 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:47.717 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:43:47.741 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:43:47.763 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:43:47.785 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:43:47.792 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:47.806 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:43:47.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:47 smithi138 ceph-mon[192513]: pgmap v1517: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 107 op/s; 7898/205119 objects degraded (3.850%); 4199/205119 objects misplaced (2.047%); 301 KiB/s, 4 objects/s recovering 2023-10-01T09:43:47.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:47 smithi138 ceph-mon[185886]: pgmap v1517: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 107 op/s; 7898/205119 objects degraded (3.850%); 4199/205119 objects misplaced (2.047%); 301 KiB/s, 4 objects/s recovering 2023-10-01T09:43:47.823 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:47.826 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:47.830 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:43:47.834 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:43:47.834 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:43:47.834 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:43:47.834 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:43:47.834 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:43:47.834 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:43:47.835 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:43:47.835 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:43:48.085 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:48.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:48.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:48.188 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:48.215 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:48.730 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:48 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3191251975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:48.731 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:48 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7898/205119 objects degraded (3.850%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:48.731 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:48 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3191251975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:48.731 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:48 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7898/205119 objects degraded (3.850%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:48 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3191251975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:48 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7898/205119 objects degraded (3.850%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:48.878 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:48.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:48.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:48.982 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:49.012 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:49.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:49 smithi138 ceph-mon[192513]: pgmap v1518: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 7840/203769 objects degraded (3.847%); 4158/203769 objects misplaced (2.041%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:43:49.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:49 smithi138 ceph-mon[185886]: pgmap v1518: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 7840/203769 objects degraded (3.847%); 4158/203769 objects misplaced (2.041%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:43:49.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:49 smithi190 ceph-mon[160435]: pgmap v1518: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 135 op/s; 7840/203769 objects degraded (3.847%); 4158/203769 objects misplaced (2.041%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:43:49.865 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:49.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:49.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:49.970 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:49.998 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:50.146 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:50.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:50.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:50.251 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:50.278 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:50.323 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:50.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:50.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:50.428 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:50.455 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:50.501 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:50.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:50.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:50.606 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:50.634 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:50.750 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:43:51.135 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:51.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:51.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:51.197 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:43:51.197 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:43:51.240 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:51.277 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:51.431 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:51.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:51.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:51.535 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:51.561 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:51.568 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:43:51.570 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:43:51.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:51 smithi138 ceph-mon[192513]: pgmap v1519: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 139 op/s; 7840/203457 objects degraded (3.853%); 4158/203457 objects misplaced (2.044%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:43:51.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:51 smithi138 ceph-mon[185886]: pgmap v1519: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 139 op/s; 7840/203457 objects degraded (3.853%); 4158/203457 objects misplaced (2.044%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:43:51.608 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:51.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:51.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:51.713 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:51.745 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:51.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:51 smithi190 ceph-mon[160435]: pgmap v1519: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 139 op/s; 7840/203457 objects degraded (3.853%); 4158/203457 objects misplaced (2.044%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:43:52.043 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:52.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:52.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:52.147 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:52.173 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:52.219 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:52.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:52.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:52.324 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:52.353 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:52.434 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:43:52.439 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:52 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/482269711' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:43:52.457 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:43:52.740 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:52 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/482269711' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:43:52.741 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:52 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/482269711' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:43:52.769 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:52.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:52.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:52.873 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:52.902 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:52.983 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1519,"stamp":"2023-10-01T09:43:50.313713+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4447475659,"num_objects":67819,"num_object_clones":0,"num_object_copies":203457,"num_objects_missing_on_primary":2719,"num_objects_missing":2719,"num_objects_degraded":7840,"num_objects_misplaced":4158,"num_objects_unfound":0,"num_objects_dirty":67819,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":99755,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8391,"num_bytes_recovered":536773647,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99819,"ondisk_log_size":99819,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18529300,"kb_used_data":13475996,"kb_used_omap":39,"kb_used_meta":5053208,"kb_avail":637686764,"statfs":{"total":671965249536,"available":652991246336,"internally_reserved":0,"allocated":13799419904,"data_stored":13789583322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5174485615},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,1,1,1,1,6,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-91750400,"num_objects":-1400,"num_object_clones":0,"num_object_copies":-4200,"num_objects_missing_on_primary":-90,"num_objects_missing":-90,"num_objects_degraded":-107,"num_objects_misplaced":-83,"num_objects_unfound":0,"num_objects_dirty":-1400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1400,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":6553600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002725"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":300,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.415826+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:43:34.415826+0000","last_peered":"2023-10-01T09:43:34.415826+0000","last_clean":"2023-10-01T09:43:34.415826+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:43:34.415826+0000","last_undegraded":"2023-10-01T09:43:34.415826+0000","last_fullsized":"2023-10-01T09:43:34.415826+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":143,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.695572+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:43:34.695572+0000","last_peered":"2023-10-01T09:43:34.695572+0000","last_clean":"2023-10-01T09:43:34.695572+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:43:34.695572+0000","last_undegraded":"2023-10-01T09:43:34.695572+0000","last_fullsized":"2023-10-01T09:43:34.695572+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":524,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:40.255101+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:43:40.255101+0000","last_peered":"2023-10-01T09:43:40.255101+0000","last_clean":"2023-10-01T09:43:40.255101+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:43:40.255101+0000","last_undegraded":"2023-10-01T09:43:40.255101+0000","last_fullsized":"2023-10-01T09:43:40.255101+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":582,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.419504+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:43:34.419504+0000","last_peered":"2023-10-01T09:43:34.419504+0000","last_clean":"2023-10-01T09:43:34.419504+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:43:34.419504+0000","last_undegraded":"2023-10-01T09:43:34.419504+0000","last_fullsized":"2023-10-01T09:43:34.419504+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":463,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.418934+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:43:34.418934+0000","last_peered":"2023-10-01T09:43:34.418934+0000","last_clean":"2023-10-01T09:43:34.418934+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:43:34.418934+0000","last_undegraded":"2023-10-01T09:43:34.418934+0000","last_fullsized":"2023-10-01T09:43:34.418934+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":205,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.488441+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:43:34.488441+0000","last_peered":"2023-10-01T09:43:34.488441+0000","last_clean":"2023-10-01T09:43:34.488441+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:43:34.488441+0000","last_undegraded":"2023-10-01T09:43:34.488441+0000","last_fullsized":"2023-10-01T09:43:34.488441+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":581,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.488505+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:43:34.488505+0000","last_peered":"2023-10-01T09:43:34.488505+0000","last_clean":"2023-10-01T09:43:34.488505+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:43:34.488505+0000","last_undegraded":"2023-10-01T09:43:34.488505+0000","last_fullsized":"2023-10-01T09:43:34.488505+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":510,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.412253+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:43:34.412253+0000","last_peered":"2023-10-01T09:43:34.412253+0000","last_clean":"2023-10-01T09:43:34.412253+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:43:34.412253+0000","last_undegraded":"2023-10-01T09:43:34.412253+0000","last_fullsized":"2023-10-01T09:43:34.412253+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":236,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.449726+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:43:34.449726+0000","last_peered":"2023-10-01T09:43:34.449726+0000","last_clean":"2023-10-01T09:43:34.449726+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:43:34.449726+0000","last_undegraded":"2023-10-01T09:43:34.449726+0000","last_fullsized":"2023-10-01T09:43:34.449726+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":43,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.572568+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:43:34.572568+0000","last_peered":"2023-10-01T09:43:34.572568+0000","last_clean":"2023-10-01T09:43:34.572568+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:43:34.572568+0000","last_undegraded":"2023-10-01T09:43:34.572568+0000","last_fullsized":"2023-10-01T09:43:34.572568+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":49,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.572995+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:43:34.572995+0000","last_peered":"2023-10-01T09:43:34.572995+0000","last_clean":"2023-10-01T09:43:34.572995+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:43:34.572995+0000","last_undegraded":"2023-10-01T09:43:34.572995+0000","last_fullsized":"2023-10-01T09:43:34.572995+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.695829+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:43:34.695829+0000","last_peered":"2023-10-01T09:43:34.695829+0000","last_clean":"2023-10-01T09:43:34.695829+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:43:34.695829+0000","last_undegraded":"2023-10-01T09:43:34.695829+0000","last_fullsized":"2023-10-01T09:43:34.695829+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":425,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.416238+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:43:34.416238+0000","last_peered":"2023-10-01T09:43:34.416238+0000","last_clean":"2023-10-01T09:43:34.416238+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:43:34.416238+0000","last_undegraded":"2023-10-01T09:43:34.416238+0000","last_fullsized":"2023-10-01T09:43:34.416238+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":306,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.415920+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:43:34.415920+0000","last_peered":"2023-10-01T09:43:34.415920+0000","last_clean":"2023-10-01T09:43:34.415920+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:43:34.415920+0000","last_undegraded":"2023-10-01T09:43:34.415920+0000","last_fullsized":"2023-10-01T09:43:34.415920+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":642,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.572799+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:43:34.572799+0000","last_peered":"2023-10-01T09:43:34.572799+0000","last_clean":"2023-10-01T09:43:34.572799+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:43:34.572799+0000","last_undegraded":"2023-10-01T09:43:34.572799+0000","last_fullsized":"2023-10-01T09:43:34.572799+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":379,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.572948+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:43:34.572948+0000","last_peered":"2023-10-01T09:43:34.572948+0000","last_clean":"2023-10-01T09:43:34.572948+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:43:34.572948+0000","last_undegraded":"2023-10-01T09:43:34.572948+0000","last_fullsized":"2023-10-01T09:43:34.572948+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.572724+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:43:34.572724+0000","last_peered":"2023-10-01T09:43:34.572724+0000","last_clean":"2023-10-01T09:43:34.572724+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:43:34.572724+0000","last_undegraded":"2023-10-01T09:43:34.572724+0000","last_fullsized":"2023-10-01T09:43:34.572724+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":38,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.449434+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:43:34.449434+0000","last_peered":"2023-10-01T09:43:34.449434+0000","last_clean":"2023-10-01T09:43:34.449434+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:43:34.449434+0000","last_undegraded":"2023-10-01T09:43:34.449434+0000","last_fullsized":"2023-10-01T09:43:34.449434+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":535,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.572693+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:43:34.572693+0000","last_peered":"2023-10-01T09:43:34.572693+0000","last_clean":"2023-10-01T09:43:34.572693+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:43:34.572693+0000","last_undegraded":"2023-10-01T09:43:34.572693+0000","last_fullsized":"2023-10-01T09:43:34.572693+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":546,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.419575+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:43:34.419575+0000","last_peered":"2023-10-01T09:43:34.419575+0000","last_clean":"2023-10-01T09:43:34.419575+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:43:34.419575+0000","last_undegraded":"2023-10-01T09:43:34.419575+0000","last_fullsized":"2023-10-01T09:43:34.419575+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":402,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.419012+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:43:34.419012+0000","last_peered":"2023-10-01T09:43:34.419012+0000","last_clean":"2023-10-01T09:43:34.419012+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:43:34.419012+0000","last_undegraded":"2023-10-01T09:43:34.419012+0000","last_fullsized":"2023-10-01T09:43:34.419012+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":52,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.573171+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:43:34.573171+0000","last_peered":"2023-10-01T09:43:34.573171+0000","last_clean":"2023-10-01T09:43:34.573171+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:43:34.573171+0000","last_undegraded":"2023-10-01T09:43:34.573171+0000","last_fullsized":"2023-10-01T09:43:34.573171+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"524'6181","reported_seq":6238,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:46.301210+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:43:46.301210+0000","last_peered":"2023-10-01T09:43:46.301210+0000","last_clean":"2023-10-01T09:43:46.301210+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:43:46.301210+0000","last_undegraded":"2023-10-01T09:43:46.301210+0000","last_fullsized":"2023-10-01T09:43:46.301210+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6181,"log_dups_size":0,"ondisk_log_size":6181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275972096,"num_objects":4211,"num_object_clones":0,"num_object_copies":12633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6181,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":47,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.573085+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:43:34.573085+0000","last_peered":"2023-10-01T09:43:34.573085+0000","last_clean":"2023-10-01T09:43:34.573085+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:43:34.573085+0000","last_undegraded":"2023-10-01T09:43:34.573085+0000","last_fullsized":"2023-10-01T09:43:34.573085+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":244,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.449387+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:43:34.449387+0000","last_peered":"2023-10-01T09:43:34.449387+0000","last_clean":"2023-10-01T09:43:34.449387+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:43:34.449387+0000","last_undegraded":"2023-10-01T09:43:34.449387+0000","last_fullsized":"2023-10-01T09:43:34.449387+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"524'6174","reported_seq":9167,"reported_epoch":524,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:43:46.324840+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:43:46.324840+0000","last_peered":"2023-10-01T09:43:46.324840+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:43:46.324840+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6174,"log_dups_size":0,"ondisk_log_size":6174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277348352,"num_objects":4232,"num_object_clones":0,"num_object_copies":12696,"num_objects_missing_on_primary":2719,"num_objects_missing":2719,"num_objects_degraded":6883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6174,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":971,"num_bytes_recovered":63635456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1445},{"shards":"0,4,6","objects":2719},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":518,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.573108+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:43:34.573108+0000","last_peered":"2023-10-01T09:43:34.573108+0000","last_clean":"2023-10-01T09:43:34.573108+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:43:34.573108+0000","last_undegraded":"2023-10-01T09:43:34.573108+0000","last_fullsized":"2023-10-01T09:43:34.573108+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":579,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.695439+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:43:34.695439+0000","last_peered":"2023-10-01T09:43:34.695439+0000","last_clean":"2023-10-01T09:43:34.695439+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:43:34.695439+0000","last_undegraded":"2023-10-01T09:43:34.695439+0000","last_fullsized":"2023-10-01T09:43:34.695439+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"524'6194","reported_seq":6253,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:46.641191+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:43:46.641191+0000","last_peered":"2023-10-01T09:43:46.641191+0000","last_clean":"2023-10-01T09:43:46.641191+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:43:46.641191+0000","last_undegraded":"2023-10-01T09:43:46.641191+0000","last_fullsized":"2023-10-01T09:43:46.641191+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6194,"log_dups_size":0,"ondisk_log_size":6194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277741568,"num_objects":4238,"num_object_clones":0,"num_object_copies":12714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6194,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":604,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.573150+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:43:34.573150+0000","last_peered":"2023-10-01T09:43:34.573150+0000","last_clean":"2023-10-01T09:43:34.573150+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:43:34.573150+0000","last_undegraded":"2023-10-01T09:43:34.573150+0000","last_fullsized":"2023-10-01T09:43:34.573150+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":399,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.696020+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:43:34.696020+0000","last_peered":"2023-10-01T09:43:34.696020+0000","last_clean":"2023-10-01T09:43:34.696020+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:43:34.696020+0000","last_undegraded":"2023-10-01T09:43:34.696020+0000","last_fullsized":"2023-10-01T09:43:34.696020+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"524'6149","reported_seq":12813,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:46.518056+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:43:46.518056+0000","last_peered":"2023-10-01T09:43:46.518056+0000","last_clean":"2023-10-01T09:43:46.518056+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:43:46.518056+0000","last_undegraded":"2023-10-01T09:43:46.518056+0000","last_fullsized":"2023-10-01T09:43:46.518056+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6149,"log_dups_size":0,"ondisk_log_size":6149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278724608,"num_objects":4253,"num_object_clones":0,"num_object_copies":12759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6149,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":244,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.449575+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:43:34.449575+0000","last_peered":"2023-10-01T09:43:34.449575+0000","last_clean":"2023-10-01T09:43:34.449575+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:43:34.449575+0000","last_undegraded":"2023-10-01T09:43:34.449575+0000","last_fullsized":"2023-10-01T09:43:34.449575+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":502,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.418976+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:43:34.418976+0000","last_peered":"2023-10-01T09:43:34.418976+0000","last_clean":"2023-10-01T09:43:34.418976+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:43:34.418976+0000","last_undegraded":"2023-10-01T09:43:34.418976+0000","last_fullsized":"2023-10-01T09:43:34.418976+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"524'6215","reported_seq":6274,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:50.228041+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:43:50.228041+0000","last_peered":"2023-10-01T09:43:50.228041+0000","last_clean":"2023-10-01T09:43:50.228041+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:43:50.228041+0000","last_undegraded":"2023-10-01T09:43:50.228041+0000","last_fullsized":"2023-10-01T09:43:50.228041+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6215,"log_dups_size":0,"ondisk_log_size":6215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281346048,"num_objects":4293,"num_object_clones":0,"num_object_copies":12879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6215,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":104,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.416085+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:43:34.416085+0000","last_peered":"2023-10-01T09:43:34.416085+0000","last_clean":"2023-10-01T09:43:34.416085+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:43:34.416085+0000","last_undegraded":"2023-10-01T09:43:34.416085+0000","last_fullsized":"2023-10-01T09:43:34.416085+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":178,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.416349+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:43:34.416349+0000","last_peered":"2023-10-01T09:43:34.416349+0000","last_clean":"2023-10-01T09:43:34.416349+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:43:34.416349+0000","last_undegraded":"2023-10-01T09:43:34.416349+0000","last_fullsized":"2023-10-01T09:43:34.416349+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"524'6143","reported_seq":6240,"reported_epoch":524,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:43:46.354647+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:43:46.354647+0000","last_peered":"2023-10-01T09:43:46.354647+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:43:46.354647+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6143,"log_dups_size":0,"ondisk_log_size":6143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276889600,"num_objects":4225,"num_object_clones":0,"num_object_copies":12675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":947,"num_objects_misplaced":4158,"num_objects_unfound":0,"num_objects_dirty":4225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6143,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1048576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4158},{"shards":"4,5","objects":-880}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":422,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.415920+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:43:34.415920+0000","last_peered":"2023-10-01T09:43:34.415920+0000","last_clean":"2023-10-01T09:43:34.415920+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:43:34.415920+0000","last_undegraded":"2023-10-01T09:43:34.415920+0000","last_fullsized":"2023-10-01T09:43:34.415920+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":582,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.695649+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:43:34.695649+0000","last_peered":"2023-10-01T09:43:34.695649+0000","last_clean":"2023-10-01T09:43:34.695649+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:43:34.695649+0000","last_undegraded":"2023-10-01T09:43:34.695649+0000","last_fullsized":"2023-10-01T09:43:34.695649+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"524'6130","reported_seq":6196,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:47.405823+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:43:47.405823+0000","last_peered":"2023-10-01T09:43:47.405823+0000","last_clean":"2023-10-01T09:43:47.405823+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:43:47.405823+0000","last_undegraded":"2023-10-01T09:43:47.405823+0000","last_fullsized":"2023-10-01T09:43:47.405823+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6130,"log_dups_size":0,"ondisk_log_size":6130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278396928,"num_objects":4248,"num_object_clones":0,"num_object_copies":12744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6130,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":418,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.415816+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:43:34.415816+0000","last_peered":"2023-10-01T09:43:34.415816+0000","last_clean":"2023-10-01T09:43:34.415816+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:43:34.415816+0000","last_undegraded":"2023-10-01T09:43:34.415816+0000","last_fullsized":"2023-10-01T09:43:34.415816+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":259,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.416443+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:43:34.416443+0000","last_peered":"2023-10-01T09:43:34.416443+0000","last_clean":"2023-10-01T09:43:34.416443+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:43:34.416443+0000","last_undegraded":"2023-10-01T09:43:34.416443+0000","last_fullsized":"2023-10-01T09:43:34.416443+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"524'6240","reported_seq":6297,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:46.272948+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:43:46.272948+0000","last_peered":"2023-10-01T09:43:46.272948+0000","last_clean":"2023-10-01T09:43:46.272948+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:43:46.272948+0000","last_undegraded":"2023-10-01T09:43:46.272948+0000","last_fullsized":"2023-10-01T09:43:46.272948+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6240,"log_dups_size":0,"ondisk_log_size":6240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279052288,"num_objects":4258,"num_object_clones":0,"num_object_copies":12774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6240,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":387,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.511281+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:43:44.511281+0000","last_peered":"2023-10-01T09:43:44.511281+0000","last_clean":"2023-10-01T09:43:44.511281+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:43:44.511281+0000","last_undegraded":"2023-10-01T09:43:44.511281+0000","last_fullsized":"2023-10-01T09:43:44.511281+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":434,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.695985+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:43:34.695985+0000","last_peered":"2023-10-01T09:43:34.695985+0000","last_clean":"2023-10-01T09:43:34.695985+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:43:34.695985+0000","last_undegraded":"2023-10-01T09:43:34.695985+0000","last_fullsized":"2023-10-01T09:43:34.695985+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"524'6044","reported_seq":6101,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:45.606023+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:43:45.606023+0000","last_peered":"2023-10-01T09:43:45.606023+0000","last_clean":"2023-10-01T09:43:45.606023+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:43:45.606023+0000","last_undegraded":"2023-10-01T09:43:45.606023+0000","last_fullsized":"2023-10-01T09:43:45.606023+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6044,"log_dups_size":0,"ondisk_log_size":6044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270925824,"num_objects":4134,"num_object_clones":0,"num_object_copies":12402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6044,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":917,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.572857+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:43:34.572857+0000","last_peered":"2023-10-01T09:43:34.572857+0000","last_clean":"2023-10-01T09:43:34.572857+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:43:34.572857+0000","last_undegraded":"2023-10-01T09:43:34.572857+0000","last_fullsized":"2023-10-01T09:43:34.572857+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":254,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.449522+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:43:34.449522+0000","last_peered":"2023-10-01T09:43:34.449522+0000","last_clean":"2023-10-01T09:43:34.449522+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:43:34.449522+0000","last_undegraded":"2023-10-01T09:43:34.449522+0000","last_fullsized":"2023-10-01T09:43:34.449522+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":47,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.695765+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:43:34.695765+0000","last_peered":"2023-10-01T09:43:34.695765+0000","last_clean":"2023-10-01T09:43:34.695765+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:43:34.695765+0000","last_undegraded":"2023-10-01T09:43:34.695765+0000","last_fullsized":"2023-10-01T09:43:34.695765+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"524'6113","reported_seq":6172,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:46.686551+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:43:46.686551+0000","last_peered":"2023-10-01T09:43:46.686551+0000","last_clean":"2023-10-01T09:43:46.686551+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:43:46.686551+0000","last_undegraded":"2023-10-01T09:43:46.686551+0000","last_fullsized":"2023-10-01T09:43:46.686551+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6113,"log_dups_size":0,"ondisk_log_size":6113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278986752,"num_objects":4257,"num_object_clones":0,"num_object_copies":12771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6113,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1189,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.449427+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:43:34.449427+0000","last_peered":"2023-10-01T09:43:34.449427+0000","last_clean":"2023-10-01T09:43:34.449427+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:43:34.449427+0000","last_undegraded":"2023-10-01T09:43:34.449427+0000","last_fullsized":"2023-10-01T09:43:34.449427+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":399,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.416036+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:43:34.416036+0000","last_peered":"2023-10-01T09:43:34.416036+0000","last_clean":"2023-10-01T09:43:34.416036+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:43:34.416036+0000","last_undegraded":"2023-10-01T09:43:34.416036+0000","last_fullsized":"2023-10-01T09:43:34.416036+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":227,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.449704+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:43:34.449704+0000","last_peered":"2023-10-01T09:43:34.449704+0000","last_clean":"2023-10-01T09:43:34.449704+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:43:34.449704+0000","last_undegraded":"2023-10-01T09:43:34.449704+0000","last_fullsized":"2023-10-01T09:43:34.449704+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"524'6164","reported_seq":13442,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:45.693860+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:43:45.693860+0000","last_peered":"2023-10-01T09:43:45.693860+0000","last_clean":"2023-10-01T09:43:45.693860+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:43:45.693860+0000","last_undegraded":"2023-10-01T09:43:45.693860+0000","last_fullsized":"2023-10-01T09:43:45.693860+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6164,"log_dups_size":0,"ondisk_log_size":6164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276037632,"num_objects":4212,"num_object_clones":0,"num_object_copies":12636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6164,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":869,"reported_epoch":523,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:43:34.418733+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:43:34.418733+0000","last_peered":"2023-10-01T09:43:34.418733+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:43:34.418733+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:43:34.418733+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":586,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.695461+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:43:34.695461+0000","last_peered":"2023-10-01T09:43:34.695461+0000","last_clean":"2023-10-01T09:43:34.695461+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:43:34.695461+0000","last_undegraded":"2023-10-01T09:43:34.695461+0000","last_fullsized":"2023-10-01T09:43:34.695461+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":47,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.695913+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:43:34.695913+0000","last_peered":"2023-10-01T09:43:34.695913+0000","last_clean":"2023-10-01T09:43:34.695913+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:43:34.695913+0000","last_undegraded":"2023-10-01T09:43:34.695913+0000","last_fullsized":"2023-10-01T09:43:34.695913+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"524'6090","reported_seq":6149,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:50.188552+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:43:50.188552+0000","last_peered":"2023-10-01T09:43:50.188552+0000","last_clean":"2023-10-01T09:43:50.188552+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:43:50.188552+0000","last_undegraded":"2023-10-01T09:43:50.188552+0000","last_fullsized":"2023-10-01T09:43:50.188552+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6090,"log_dups_size":0,"ondisk_log_size":6090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274071552,"num_objects":4182,"num_object_clones":0,"num_object_copies":12546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6090,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1640,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.449521+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:43:34.449521+0000","last_peered":"2023-10-01T09:43:34.449521+0000","last_clean":"2023-10-01T09:43:34.449521+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:43:34.449521+0000","last_undegraded":"2023-10-01T09:43:34.449521+0000","last_fullsized":"2023-10-01T09:43:34.449521+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":554,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.572589+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:43:34.572589+0000","last_peered":"2023-10-01T09:43:34.572589+0000","last_clean":"2023-10-01T09:43:34.572589+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:43:34.572589+0000","last_undegraded":"2023-10-01T09:43:34.572589+0000","last_fullsized":"2023-10-01T09:43:34.572589+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":332,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.449522+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:43:34.449522+0000","last_peered":"2023-10-01T09:43:34.449522+0000","last_clean":"2023-10-01T09:43:34.449522+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:43:34.449522+0000","last_undegraded":"2023-10-01T09:43:34.449522+0000","last_fullsized":"2023-10-01T09:43:34.449522+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":322,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.416253+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:43:34.416253+0000","last_peered":"2023-10-01T09:43:34.416253+0000","last_clean":"2023-10-01T09:43:34.416253+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:43:34.416253+0000","last_undegraded":"2023-10-01T09:43:34.416253+0000","last_fullsized":"2023-10-01T09:43:34.416253+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"524'6235","reported_seq":6292,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:45.580280+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:43:45.580280+0000","last_peered":"2023-10-01T09:43:45.580280+0000","last_clean":"2023-10-01T09:43:45.580280+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:43:45.580280+0000","last_undegraded":"2023-10-01T09:43:45.580280+0000","last_fullsized":"2023-10-01T09:43:45.580280+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6235,"log_dups_size":0,"ondisk_log_size":6235,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281608192,"num_objects":4297,"num_object_clones":0,"num_object_copies":12891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6235,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":731,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.572682+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:43:34.572682+0000","last_peered":"2023-10-01T09:43:34.572682+0000","last_clean":"2023-10-01T09:43:34.572682+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:43:34.572682+0000","last_undegraded":"2023-10-01T09:43:34.572682+0000","last_fullsized":"2023-10-01T09:43:34.572682+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":563,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.419006+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:43:34.419006+0000","last_peered":"2023-10-01T09:43:34.419006+0000","last_clean":"2023-10-01T09:43:34.419006+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:43:34.419006+0000","last_undegraded":"2023-10-01T09:43:34.419006+0000","last_fullsized":"2023-10-01T09:43:34.419006+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":563,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.419371+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:43:34.419371+0000","last_peered":"2023-10-01T09:43:34.419371+0000","last_clean":"2023-10-01T09:43:34.419371+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:43:34.419371+0000","last_undegraded":"2023-10-01T09:43:34.419371+0000","last_fullsized":"2023-10-01T09:43:34.419371+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"524'6207","reported_seq":6267,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:46.525291+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:43:46.525291+0000","last_peered":"2023-10-01T09:43:46.525291+0000","last_clean":"2023-10-01T09:43:46.525291+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:43:46.525291+0000","last_undegraded":"2023-10-01T09:43:46.525291+0000","last_fullsized":"2023-10-01T09:43:46.525291+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6207,"log_dups_size":0,"ondisk_log_size":6207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278659096,"num_objects":4253,"num_object_clones":0,"num_object_copies":12759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4253,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6207,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":962,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.573164+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:43:34.573164+0000","last_peered":"2023-10-01T09:43:34.573164+0000","last_clean":"2023-10-01T09:43:34.573164+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:43:34.573164+0000","last_undegraded":"2023-10-01T09:43:34.573164+0000","last_fullsized":"2023-10-01T09:43:34.573164+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":581,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.449655+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:43:34.449655+0000","last_peered":"2023-10-01T09:43:34.449655+0000","last_clean":"2023-10-01T09:43:34.449655+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:43:34.449655+0000","last_undegraded":"2023-10-01T09:43:34.449655+0000","last_fullsized":"2023-10-01T09:43:34.449655+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":434,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.449604+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:43:34.449604+0000","last_peered":"2023-10-01T09:43:34.449604+0000","last_clean":"2023-10-01T09:43:34.449604+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:43:34.449604+0000","last_undegraded":"2023-10-01T09:43:34.449604+0000","last_fullsized":"2023-10-01T09:43:34.449604+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"524'6182","reported_seq":6247,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:45.677332+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:43:45.677332+0000","last_peered":"2023-10-01T09:43:45.677332+0000","last_clean":"2023-10-01T09:43:45.677332+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:43:45.677332+0000","last_undegraded":"2023-10-01T09:43:45.677332+0000","last_fullsized":"2023-10-01T09:43:45.677332+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6182,"log_dups_size":0,"ondisk_log_size":6182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273547264,"num_objects":4174,"num_object_clones":0,"num_object_copies":12522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6182,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":463,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.573222+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:43:34.573222+0000","last_peered":"2023-10-01T09:43:34.573222+0000","last_clean":"2023-10-01T09:43:34.573222+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:43:34.573222+0000","last_undegraded":"2023-10-01T09:43:34.573222+0000","last_fullsized":"2023-10-01T09:43:34.573222+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1156,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.695867+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:43:34.695867+0000","last_peered":"2023-10-01T09:43:34.695867+0000","last_clean":"2023-10-01T09:43:34.695867+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:43:34.695867+0000","last_undegraded":"2023-10-01T09:43:34.695867+0000","last_fullsized":"2023-10-01T09:43:34.695867+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":52,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.572533+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:43:34.572533+0000","last_peered":"2023-10-01T09:43:34.572533+0000","last_clean":"2023-10-01T09:43:34.572533+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:43:34.572533+0000","last_undegraded":"2023-10-01T09:43:34.572533+0000","last_fullsized":"2023-10-01T09:43:34.572533+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"524'6188","reported_seq":6253,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:45.631385+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:43:45.631385+0000","last_peered":"2023-10-01T09:43:45.631385+0000","last_clean":"2023-10-01T09:43:45.631385+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:43:45.631385+0000","last_undegraded":"2023-10-01T09:43:45.631385+0000","last_fullsized":"2023-10-01T09:43:45.631385+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6188,"log_dups_size":0,"ondisk_log_size":6188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283377664,"num_objects":4324,"num_object_clones":0,"num_object_copies":12972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6188,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":56,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.573113+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:43:34.573113+0000","last_peered":"2023-10-01T09:43:34.573113+0000","last_clean":"2023-10-01T09:43:34.573113+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:43:34.573113+0000","last_undegraded":"2023-10-01T09:43:34.573113+0000","last_fullsized":"2023-10-01T09:43:34.573113+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":541,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.419438+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:43:34.419438+0000","last_peered":"2023-10-01T09:43:34.419438+0000","last_clean":"2023-10-01T09:43:34.419438+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:43:34.419438+0000","last_undegraded":"2023-10-01T09:43:34.419438+0000","last_fullsized":"2023-10-01T09:43:34.419438+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":247,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.449415+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:43:34.449415+0000","last_peered":"2023-10-01T09:43:34.449415+0000","last_clean":"2023-10-01T09:43:34.449415+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:43:34.449415+0000","last_undegraded":"2023-10-01T09:43:34.449415+0000","last_fullsized":"2023-10-01T09:43:34.449415+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":490,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.419189+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:43:34.419189+0000","last_peered":"2023-10-01T09:43:34.419189+0000","last_clean":"2023-10-01T09:43:34.419189+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:43:34.419189+0000","last_undegraded":"2023-10-01T09:43:34.419189+0000","last_fullsized":"2023-10-01T09:43:34.419189+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":603,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.695647+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:43:34.695647+0000","last_peered":"2023-10-01T09:43:34.695647+0000","last_clean":"2023-10-01T09:43:34.695647+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:43:34.695647+0000","last_undegraded":"2023-10-01T09:43:34.695647+0000","last_fullsized":"2023-10-01T09:43:34.695647+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":259,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.449491+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:43:34.449491+0000","last_peered":"2023-10-01T09:43:34.449491+0000","last_clean":"2023-10-01T09:43:34.449491+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:43:34.449491+0000","last_undegraded":"2023-10-01T09:43:34.449491+0000","last_fullsized":"2023-10-01T09:43:34.449491+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":47,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.695561+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:43:34.695561+0000","last_peered":"2023-10-01T09:43:34.695561+0000","last_clean":"2023-10-01T09:43:34.695561+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:43:34.695561+0000","last_undegraded":"2023-10-01T09:43:34.695561+0000","last_fullsized":"2023-10-01T09:43:34.695561+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":259,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.449554+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:43:34.449554+0000","last_peered":"2023-10-01T09:43:34.449554+0000","last_clean":"2023-10-01T09:43:34.449554+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:43:34.449554+0000","last_undegraded":"2023-10-01T09:43:34.449554+0000","last_fullsized":"2023-10-01T09:43:34.449554+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":436,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.419125+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:43:34.419125+0000","last_peered":"2023-10-01T09:43:34.419125+0000","last_clean":"2023-10-01T09:43:34.419125+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:43:34.419125+0000","last_undegraded":"2023-10-01T09:43:34.419125+0000","last_fullsized":"2023-10-01T09:43:34.419125+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":64,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.695771+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:43:34.695771+0000","last_peered":"2023-10-01T09:43:34.695771+0000","last_clean":"2023-10-01T09:43:34.695771+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:43:34.695771+0000","last_undegraded":"2023-10-01T09:43:34.695771+0000","last_fullsized":"2023-10-01T09:43:34.695771+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":481,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.418925+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:43:34.418925+0000","last_peered":"2023-10-01T09:43:34.418925+0000","last_clean":"2023-10-01T09:43:34.418925+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:43:34.418925+0000","last_undegraded":"2023-10-01T09:43:34.418925+0000","last_fullsized":"2023-10-01T09:43:34.418925+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":680,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.419489+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:43:34.419489+0000","last_peered":"2023-10-01T09:43:34.419489+0000","last_clean":"2023-10-01T09:43:34.419489+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:43:34.419489+0000","last_undegraded":"2023-10-01T09:43:34.419489+0000","last_fullsized":"2023-10-01T09:43:34.419489+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":375,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-10-01T09:43:34.415686+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:43:34.415686+0000","last_peered":"2023-10-01T09:43:34.415686+0000","last_clean":"2023-10-01T09:43:34.415686+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:43:34.415686+0000","last_undegraded":"2023-10-01T09:43:34.415686+0000","last_fullsized":"2023-10-01T09:43:34.415686+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4442685464,"num_objects":67791,"num_object_clones":0,"num_object_copies":203373,"num_objects_missing_on_primary":2719,"num_objects_missing":2719,"num_objects_degraded":7830,"num_objects_misplaced":4158,"num_objects_unfound":0,"num_objects_dirty":67791,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":98649,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7880,"num_bytes_recovered":516423680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13727641600,"data_stored":13727629384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98649,"ondisk_log_size":98649,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430718,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2271408,"kb_used_data":1364200,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":91473744,"statfs":{"total":95995035648,"available":93669113856,"internally_reserved":0,"allocated":1396940800,"data_stored":1396003744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:56 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.4950000000000001,"15min":2.9569999999999999},"min":{"1min":0.246,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":1.3280000000000001,"5min":10.391999999999999,"15min":108.90000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.49199999999999999,"5min":1.8129999999999999,"15min":3.129},"min":{"1min":0.246,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.018,"5min":29.501000000000001,"15min":109.187},"last":0.371}]},{"osd":1,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":4.6749999999999998,"15min":5.3520000000000003},"min":{"1min":0.31900000000000001,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":1.2949999999999999,"5min":229.161,"15min":229.161},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.47399999999999998,"5min":4.6479999999999997,"15min":5.3689999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":0.90900000000000003,"5min":230.126,"15min":230.126},"last":0.28999999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:42:50 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":1.5369999999999999,"15min":1.891},"min":{"1min":0.33900000000000002,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":1.3560000000000001,"5min":14.866,"15min":14.901999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":1.7909999999999999,"15min":1.821},"min":{"1min":0.33500000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.96599999999999997,"5min":14.682,"15min":14.682},"last":0.52200000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":1.514,"15min":2.7330000000000001},"min":{"1min":0.30199999999999999,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":0.90700000000000003,"5min":35.104999999999997,"15min":80.352999999999994},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.50600000000000001,"5min":1.6930000000000001,"15min":2.7130000000000001},"min":{"1min":0.35399999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.153,"5min":35.118000000000002,"15min":80.364000000000004},"last":0.46999999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:42:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":1.2150000000000001,"15min":1.052},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.381,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.49099999999999999,"5min":1.1080000000000001,"15min":1.03},"min":{"1min":0.314,"5min":0.30299999999999999,"15min":0.28000000000000003},"max":{"1min":0.97899999999999998,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.48899999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":2.3620000000000001,"15min":1.6120000000000001},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":1.2749999999999999,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.62,"5min":2.5390000000000001,"15min":1.6739999999999999},"min":{"1min":0.35999999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.2599999999999998,"5min":135.339,"15min":135.339},"last":0.51300000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":1.3200000000000001,"15min":1.1599999999999999},"min":{"1min":0.377,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":1.409,"5min":11.58,"15min":11.853999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.51400000000000001,"5min":1.175,"15min":1.1299999999999999},"min":{"1min":0.378,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":1.23,"5min":14.584,"15min":14.584},"last":0.624}]}]},{"osd":6,"up_from":294,"seq":1262720385229,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2238828,"kb_used_data":1905316,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":91506324,"statfs":{"total":95995035648,"available":93702475776,"internally_reserved":0,"allocated":1951043584,"data_stored":1950110479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":3.5720000000000001,"15min":4.0919999999999996},"min":{"1min":0.25,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":3.3340000000000001,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":4.0410000000000004,"15min":4.2210000000000001},"min":{"1min":0.29899999999999999,"5min":0.249,"15min":0.221},"max":{"1min":0.623,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.46999999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":4.7489999999999997,"15min":4.9740000000000002},"min":{"1min":0.30599999999999999,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":1.9410000000000001,"5min":168.947,"15min":168.947},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":4.7969999999999997,"15min":4.8810000000000002},"min":{"1min":0.34699999999999998,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":1.962,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.37}]},{"osd":2,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":1.194,"15min":1.673},"min":{"1min":0.308,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.075,"5min":8.657,"15min":10.92},"last":0.436},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.3759999999999999,"15min":1.673},"min":{"1min":0.314,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.58499999999999996,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.41599999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.3069999999999999,"15min":2.2789999999999999},"min":{"1min":0.35499999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.3759999999999999,"5min":9.0180000000000007,"15min":89.753},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.502,"5min":1.4359999999999999,"15min":2.157},"min":{"1min":0.34399999999999997,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":1.381,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.30399999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":1.5900000000000001,"15min":1.0800000000000001},"min":{"1min":0.30099999999999999,"5min":0.253,"15min":0.246},"max":{"1min":0.53400000000000003,"5min":63.134,"15min":63.134},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.5009999999999999,"15min":1.097},"min":{"1min":0.33700000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":1.5669999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.44500000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:43:29 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":1.2450000000000001,"15min":1.101},"min":{"1min":0.309,"5min":0.251,"15min":0.251},"max":{"1min":34.094999999999999,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.409,"15min":1.22},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":34.072000000000003,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.45600000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.79500000000000004,"15min":1.46},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":1.4750000000000001,"5min":6.4119999999999999,"15min":94.245999999999995},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.48999999999999999,"5min":1.113,"15min":1.5569999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":0.68300000000000005,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.39200000000000002}]}]},{"osd":1,"up_from":94,"seq":403726926206,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1643228,"kb_used_data":825044,"kb_used_omap":7,"kb_used_meta":818168,"kb_avail":92101924,"statfs":{"total":95995035648,"available":94312370176,"internally_reserved":0,"allocated":844845056,"data_stored":842813956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":837804455},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.30299999999999999,"5min":2.5600000000000001,"15min":4.2009999999999996},"min":{"1min":0.20699999999999999,"5min":0.19400000000000001,"15min":0.182},"max":{"1min":0.57399999999999995,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.186},{"interface":"front","average":{"1min":0.33600000000000002,"5min":3.0139999999999998,"15min":3.7090000000000001},"min":{"1min":0.26300000000000001,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":0.53400000000000003,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.223}]},{"osd":2,"last update":"Sun Oct 1 09:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":1.4279999999999999,"15min":2.589},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.61399999999999999,"5min":22.891999999999999,"15min":56.573},"last":0.246},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.379,"15min":3.254},"min":{"1min":0.28999999999999998,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":1.752,"5min":15.189,"15min":154.197},"last":0.30399999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":2.1680000000000001,"15min":3.9969999999999999},"min":{"1min":0.27400000000000002,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":1.1990000000000001,"5min":18.503,"15min":200.56800000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.39300000000000002,"5min":2.3660000000000001,"15min":4.6989999999999998},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":0.51900000000000002,"5min":20.181000000000001,"15min":200.63800000000001},"last":0.27800000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.8029999999999999,"15min":2.3109999999999999},"min":{"1min":0.32900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.8109999999999999,"5min":21.015000000000001,"15min":154.101},"last":0.378},{"interface":"front","average":{"1min":0.46400000000000002,"5min":1.585,"15min":2.5249999999999999},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.83199999999999996,"5min":10.222,"15min":154.56999999999999},"last":0.32100000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:43:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.3200000000000001,"15min":2.097},"min":{"1min":0.372,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.1850000000000001,"5min":19.513000000000002,"15min":154.22999999999999},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.1719999999999999,"15min":2.4630000000000001},"min":{"1min":0.31,"5min":0.307,"15min":0.307},"max":{"1min":0.82699999999999996,"5min":10.305,"15min":154.54400000000001},"last":0.35699999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.849,"15min":2.6640000000000001},"min":{"1min":0.372,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.2789999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.495,"5min":1.702,"15min":2.4870000000000001},"min":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":0.67300000000000004,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.39900000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.9319999999999999,"15min":2.6440000000000001},"min":{"1min":0.40100000000000002,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":1.3029999999999999,"5min":21.062999999999999,"15min":154.20699999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.804,"15min":2.718},"min":{"1min":0.41699999999999998,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":1.3220000000000001,"5min":13.161,"15min":154.51599999999999},"last":0.45600000000000002}]}]},{"osd":0,"up_from":89,"seq":382252089740,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4286668,"kb_used_data":3387332,"kb_used_omap":8,"kb_used_meta":899319,"kb_avail":89458484,"statfs":{"total":95995035648,"available":91605487616,"internally_reserved":0,"allocated":3468627968,"data_stored":3466617776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":920903618},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":1.76,"15min":4.2480000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17399999999999999},"max":{"1min":1.891,"5min":52.316000000000003,"15min":216.821},"last":0.315},{"interface":"front","average":{"1min":0.52200000000000002,"5min":3.4119999999999999,"15min":5.2130000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":3.552,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.34899999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:43:36 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":1.204,"15min":2.085},"min":{"1min":0.246,"5min":0.246,"15min":0.20100000000000001},"max":{"1min":1.1100000000000001,"5min":19.788,"15min":19.788},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.4139999999999999,"15min":2.3380000000000001},"min":{"1min":0.27200000000000002,"5min":0.23000000000000001,"15min":0.189},"max":{"1min":3.5939999999999999,"5min":19.375,"15min":25.844999999999999},"last":0.39600000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":1.474,"15min":3.907},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":1.1499999999999999,"5min":15.35,"15min":222.16499999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.748,"15min":3.9460000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":2.9550000000000001,"5min":14.545999999999999,"15min":222.227},"last":0.27700000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:42:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.8859999999999999,"15min":2.266},"min":{"1min":0.29299999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":3.694,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.434},{"interface":"front","average":{"1min":0.443,"5min":1.9850000000000001,"15min":2.2639999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28299999999999997},"max":{"1min":0.59799999999999998,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.47799999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:43:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.2030000000000001,"15min":1.883},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66500000000000004,"5min":14.515000000000001,"15min":16.808},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.093,"15min":1.7709999999999999},"min":{"1min":0.33500000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.73499999999999999,"5min":5.9580000000000002,"15min":17.507000000000001},"last":0.44600000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:43:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.1850000000000001,"15min":1.774},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.35499999999999998},"max":{"1min":1.5900000000000001,"5min":6.5,"15min":17.535},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.88,"5min":1.395,"15min":2.0289999999999999},"min":{"1min":0.32600000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":3.6539999999999999,"5min":10.446,"15min":18.521000000000001},"last":0.45600000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:42:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":1.671,"15min":2.1139999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":0.77100000000000002,"5min":48.895000000000003,"15min":48.895000000000003},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.83599999999999997,"5min":1.97,"15min":2.4020000000000001},"min":{"1min":0.41899999999999998,"5min":0.35299999999999998,"15min":0.35099999999999998},"max":{"1min":3.7320000000000002,"5min":48.274000000000001,"15min":48.274000000000001},"last":0.50700000000000001}]}]},{"osd":2,"up_from":0,"seq":425201762680,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599153,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1785036,"kb_used_data":944708,"kb_used_omap":3,"kb_used_meta":840316,"kb_avail":91960116,"statfs":{"total":95995035648,"available":94167158784,"internally_reserved":0,"allocated":967380992,"data_stored":965307059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":860484327},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":4.9660000000000002,"15min":5.0880000000000001},"min":{"1min":0.20300000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":2.7989999999999999,"5min":118.748,"15min":137.34399999999999},"last":0.316},{"interface":"front","average":{"1min":0.28899999999999998,"5min":4.9500000000000002,"15min":5.1379999999999999},"min":{"1min":0.20699999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.42199999999999999,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.36699999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":3.4329999999999998,"15min":5.2869999999999999},"min":{"1min":0.23000000000000001,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":2.7250000000000001,"5min":90.944000000000003,"15min":162.22999999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.33300000000000002,"5min":3.1850000000000001,"15min":5.4160000000000004},"min":{"1min":0.23100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.45300000000000001,"5min":83.341999999999999,"15min":162.70699999999999},"last":0.34699999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":1.8740000000000001,"15min":2.2029999999999998},"min":{"1min":0.251,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":0.54900000000000004,"5min":14.254,"15min":36.472999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.90700000000000003,"15min":2.4910000000000001},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":0.59999999999999998,"5min":7.6749999999999998,"15min":78.594999999999999},"last":0.62}]},{"osd":4,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":1.8859999999999999,"15min":1.875},"min":{"1min":0.32600000000000001,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":0.65300000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.42799999999999999,"5min":2.2189999999999999,"15min":2.1469999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":0.64300000000000002,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.40600000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":1.5589999999999999,"15min":2.581},"min":{"1min":0.317,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3099999999999996,"5min":10.919,"15min":28.117999999999999},"last":0.432},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.4279999999999999,"15min":2.6800000000000002},"min":{"1min":0.36699999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.512,"5min":10.798999999999999,"15min":28.34},"last":0.55100000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:43:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":2.165,"15min":2.145},"min":{"1min":0.34200000000000003,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":2.5430000000000001,"5min":73.887,"15min":73.887},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.63,"5min":2.4159999999999999,"15min":2.1379999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":2.6000000000000001,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.57899999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":1.7889999999999999,"15min":2.3090000000000002},"min":{"1min":0.39400000000000002,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":4.1580000000000004,"5min":13.085000000000001,"15min":52.006},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.306,"15min":2.032},"min":{"1min":0.41899999999999998,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":4.1779999999999999,"5min":8.3460000000000001,"15min":51.890999999999998},"last":0.56899999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019403,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4215020,"kb_used_data":3320740,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":89530132,"statfs":{"total":95995035648,"available":91678855168,"internally_reserved":0,"allocated":3400437760,"data_stored":3399504210,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:53 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":3.129,"15min":4.7549999999999999},"min":{"1min":0.26100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.66400000000000003,"5min":79.837000000000003,"15min":155.315},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":3.2949999999999999,"15min":5.1200000000000001},"min":{"1min":0.29399999999999998,"5min":0.253,"15min":0.251},"max":{"1min":2.843,"5min":78.983000000000004,"15min":155.334},"last":0.42999999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:42:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":6.8879999999999999,"15min":6.556},"min":{"1min":0.252,"5min":0.224,"15min":0.224},"max":{"1min":2.8889999999999998,"5min":146.929,"15min":190.61199999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.45500000000000002,"5min":6.8300000000000001,"15min":7.266},"min":{"1min":0.27500000000000002,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":0.624,"5min":146.91,"15min":253.273},"last":0.41899999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:42:43 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":1.2809999999999999,"15min":2.6019999999999999},"min":{"1min":0.34799999999999998,"5min":0.30399999999999999,"15min":0.28299999999999997},"max":{"1min":2.8210000000000002,"5min":10.161,"15min":170.45400000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.4410000000000001,"15min":2.2730000000000001},"min":{"1min":0.38400000000000001,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":2.1930000000000001,"5min":10.625,"15min":58.771000000000001},"last":0.45800000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:42:58 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":1.1799999999999999,"15min":2.7519999999999998},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.67500000000000004,"5min":7.5830000000000002,"15min":140.881},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.47099999999999997,"5min":1.208,"15min":2.976},"min":{"1min":0.33300000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.69599999999999995,"5min":9.3140000000000001,"15min":140.911},"last":0.378}]},{"osd":5,"last update":"Sun Oct 1 09:43:33 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":2.0569999999999999,"15min":1.5580000000000001},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":3.052,"5min":110.003,"15min":110.003},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.64900000000000002,"5min":2.0110000000000001,"15min":1.399},"min":{"1min":0.33500000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.0049999999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.49199999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:43:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.79200000000000004,"15min":0.99299999999999999},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":0.80900000000000005,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.95199999999999996,"15min":1.085},"min":{"1min":0.379,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":2.9239999999999999,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.40600000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:42:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":1.159,"15min":1.1799999999999999},"min":{"1min":0.38100000000000001,"5min":0.32000000000000001,"15min":0.28899999999999998},"max":{"1min":1.825,"5min":38.034999999999997,"15min":38.034999999999997},"last":0.313},{"interface":"front","average":{"1min":0.623,"5min":1.47,"15min":1.462},"min":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":2.2149999999999999,"5min":37.723999999999997,"15min":37.723999999999997},"last":0.36699999999999999}]}]},{"osd":5,"up_from":379,"seq":1627792605310,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2089112,"kb_used_data":1728656,"kb_used_omap":13,"kb_used_meta":360434,"kb_avail":91656040,"statfs":{"total":95995035648,"available":93855784960,"internally_reserved":0,"allocated":1770143744,"data_stored":1769226098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":369084914},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,1,1,1,1,6,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":1.6479999999999999,"15min":2.1890000000000001},"min":{"1min":0.22900000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.1880000000000002,"5min":12.571999999999999,"15min":84.340999999999994},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.40400000000000003,"5min":1.2549999999999999,"15min":2.0800000000000001},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.53900000000000003,"5min":8.6460000000000008,"15min":84.356999999999999},"last":0.41299999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":6.4870000000000001,"15min":5.7599999999999998},"min":{"1min":0.26000000000000001,"5min":0.25900000000000001,"15min":0.255},"max":{"1min":0.76900000000000002,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":6.8099999999999996,"15min":5.7320000000000002},"min":{"1min":0.32200000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":2.258,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.434}]},{"osd":2,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":1.3600000000000001,"15min":1.5980000000000001},"min":{"1min":0.315,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.752,"5min":8.7040000000000006,"15min":14.199999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.44800000000000001,"5min":1.306,"15min":1.583},"min":{"1min":0.32900000000000001,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":0.85999999999999999,"5min":8.2370000000000001,"15min":14.430999999999999},"last":0.35799999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.5069999999999999,"15min":1.488},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.2280000000000002,"5min":12.734,"15min":14.004},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.5640000000000001,"15min":1.629},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.3979999999999999,"5min":11.766,"15min":13.991},"last":0.46899999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":2.2029999999999998,"15min":1.4330000000000001},"min":{"1min":0.30099999999999999,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.66800000000000004,"5min":122.971,"15min":122.971},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.438,"5min":2.25,"15min":1.724},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.73799999999999999,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.45500000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":1.165,"15min":1.044},"min":{"1min":0.312,"5min":0.311,"15min":0.28299999999999997},"max":{"1min":2.2839999999999998,"5min":12.545,"15min":12.545},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.99099999999999999,"15min":1.302},"min":{"1min":0.378,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":2.629,"5min":6.4729999999999999,"15min":14.193},"last":0.38400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.1140000000000001,"15min":1.262},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.81200000000000006,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.2270000000000001,"15min":1.0780000000000001},"min":{"1min":0.36299999999999999,"5min":0.35299999999999998,"15min":0.318},"max":{"1min":2.3079999999999998,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.502}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3450867712,"data_stored":3450863640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":832634880,"data_stored":832634880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":959184896,"data_stored":959184896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3394244608,"data_stored":3394240536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1762459648,"data_stored":1762459648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1941377024,"data_stored":1941372952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1386872832,"data_stored":1386872832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:43:52.990 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:43:53.166 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:53.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:53.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:53.270 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:53.299 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:53.395 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:53.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:53.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:53.406 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:53.419 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:53.462 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:53.500 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:53.527 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:53.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[185886]: from='client.36888 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:53.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[185886]: pgmap v1520: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7796/201717 objects degraded (3.865%); 4106/201717 objects misplaced (2.036%); 627 KiB/s, 9 objects/s recovering 2023-10-01T09:43:53.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:43:53.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[185886]: failed to encode map e525 with expected crc 2023-10-01T09:43:53.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/482269711' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:43:53.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[185886]: osdmap e525: 8 total, 8 up, 7 in 2023-10-01T09:43:53.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[185886]: failed to encode map e525 with expected crc 2023-10-01T09:43:53.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7796/201717 objects degraded (3.865%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:53.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[192513]: from='client.36888 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:53.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[192513]: pgmap v1520: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7796/201717 objects degraded (3.865%); 4106/201717 objects misplaced (2.036%); 627 KiB/s, 9 objects/s recovering 2023-10-01T09:43:53.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:43:53.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[192513]: failed to encode map e525 with expected crc 2023-10-01T09:43:53.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/482269711' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:43:53.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[192513]: osdmap e525: 8 total, 8 up, 7 in 2023-10-01T09:43:53.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[192513]: failed to encode map e525 with expected crc 2023-10-01T09:43:53.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:53 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7796/201717 objects degraded (3.865%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:53.638 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:53.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:53.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:53.744 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:43:53.767 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:53 smithi190 ceph-mon[160435]: from='client.36888 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:43:53.768 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:53 smithi190 ceph-mon[160435]: pgmap v1520: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7796/201717 objects degraded (3.865%); 4106/201717 objects misplaced (2.036%); 627 KiB/s, 9 objects/s recovering 2023-10-01T09:43:53.768 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:53 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:43:53.768 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:53 smithi190 ceph-mon[160435]: failed to encode map e525 with expected crc 2023-10-01T09:43:53.768 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/482269711' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:43:53.768 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:53 smithi190 ceph-mon[160435]: osdmap e525: 8 total, 8 up, 7 in 2023-10-01T09:43:53.768 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:53 smithi190 ceph-mon[160435]: failed to encode map e525 with expected crc 2023-10-01T09:43:53.768 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:53 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7796/201717 objects degraded (3.865%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:53.770 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:53.823 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:43:53.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:53.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:43:53.931 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:53.960 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:55.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:43:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:43:54] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-01T09:43:55.081 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:55.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:55.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:55.185 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:55.214 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:55.382 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:43:55.534 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:55.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:55.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:55.640 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:55.649 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:55 smithi138 ceph-mon[185886]: pgmap v1522: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 146 op/s; 7796/201594 objects degraded (3.867%); 4106/201594 objects misplaced (2.037%); 635 KiB/s, 9 objects/s recovering 2023-10-01T09:43:55.649 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:55.649 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:43:55.649 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:55 smithi138 ceph-mon[192513]: pgmap v1522: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 146 op/s; 7796/201594 objects degraded (3.867%); 4106/201594 objects misplaced (2.037%); 635 KiB/s, 9 objects/s recovering 2023-10-01T09:43:55.649 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:55.650 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:43:55.666 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:55.710 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:55.713 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:55 smithi190 ceph-mon[160435]: pgmap v1522: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 146 op/s; 7796/201594 objects degraded (3.867%); 4106/201594 objects misplaced (2.037%); 635 KiB/s, 9 objects/s recovering 2023-10-01T09:43:55.713 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:43:55.713 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:43:55.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:55.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:55.815 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:43:55.841 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:55.887 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:43:55.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:55.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:43:55.997 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:56.026 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:56.077 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:56.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:56.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:56.182 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:56.212 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:56.264 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:43:56.264 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:43:56.538 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:56.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:56.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:56.642 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:43:56.672 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:56.769 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:43:56.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:56.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:43:56.875 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:56.881 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:43:56.881 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 7796/200787 objects degraded (3.883%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2057,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":525,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":66929,"data_bytes":4389148619,"bytes_used":18804092928,"bytes_avail":653161156608,"bytes_total":671965249536,"degraded_objects":7796,"degraded_total":200787,"degraded_ratio":0.038827214909331782,"misplaced_objects":4106,"misplaced_total":200787,"misplaced_ratio":0.020449531095140625,"recovering_objects_per_sec":9,"recovering_bytes_per_sec":642085,"recovering_keys_per_sec":0,"num_objects_recovered":98,"num_bytes_recovered":6422528,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":144},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":34,"modified":"2023-10-01T09:43:40.311181+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:43:56.903 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:56.951 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:56.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:56.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:57.056 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:57.083 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:57.379 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:57.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:57.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:57.427 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:57.453 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:43:57.482 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:57.522 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:57.575 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:57.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:57.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:57.680 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:43:57.707 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:57 smithi138 ceph-mon[192513]: pgmap v1523: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 7796/200787 objects degraded (3.883%); 4106/200787 objects misplaced (2.045%); 627 KiB/s, 9 objects/s recovering 2023-10-01T09:43:57.707 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2133257108' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:43:57.707 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:57 smithi138 ceph-mon[185886]: pgmap v1523: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 7796/200787 objects degraded (3.883%); 4106/200787 objects misplaced (2.045%); 627 KiB/s, 9 objects/s recovering 2023-10-01T09:43:57.707 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2133257108' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:43:57.710 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:57.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:57 smithi190 ceph-mon[160435]: pgmap v1523: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 144 op/s; 7796/200787 objects degraded (3.883%); 4106/200787 objects misplaced (2.045%); 627 KiB/s, 9 objects/s recovering 2023-10-01T09:43:57.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2133257108' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:43:57.808 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:43:57.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:57.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:43:57.821 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:43:57.841 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:43:57.850 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:43:57.860 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:43:57.878 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:43:57.896 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:43:57.912 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:57.914 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:43:57.934 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:43:57.939 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:57.952 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:43:57.953 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:43:57.954 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:43:57.954 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:43:57.954 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:43:57.954 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:43:57.954 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:43:57.954 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:43:57.954 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:43:58.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:58 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7796/200787 objects degraded (3.883%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:58.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:58 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1172688966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:58 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7796/200787 objects degraded (3.883%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:58 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1172688966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:58.679 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:43:58.679 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:43:58.737 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:58.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:58.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:58 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7796/200787 objects degraded (3.883%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:43:58.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:58 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1172688966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:43:58.841 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:58.869 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:59.074 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:43:59.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:59.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:43:59.178 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:43:59.213 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:59.265 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:43:59.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:59.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:43:59.368 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:59.396 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:59.573 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:59.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:59.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:59.653 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:43:59.653 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:43:59.653 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 0 2023-10-01T09:43:59.653 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:43:59.677 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:43:59.704 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:43:59 smithi138 ceph-mon[192513]: pgmap v1524: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 7734/199722 objects degraded (3.872%); 4075/199722 objects misplaced (2.040%); 589 KiB/s, 9 objects/s recovering 2023-10-01T09:43:59.704 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:43:59 smithi138 ceph-mon[185886]: pgmap v1524: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 7734/199722 objects degraded (3.872%); 4075/199722 objects misplaced (2.040%); 589 KiB/s, 9 objects/s recovering 2023-10-01T09:43:59.707 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:43:59.804 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:43:59.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:43:59 smithi190 ceph-mon[160435]: pgmap v1524: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 7734/199722 objects degraded (3.872%); 4075/199722 objects misplaced (2.040%); 589 KiB/s, 9 objects/s recovering 2023-10-01T09:43:59.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:43:59.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:43:59.908 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:43:59.911 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:43:59.938 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:00.040 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:00.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:00.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:00.144 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:00.175 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:00.332 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:00.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:00.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:00.400 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:44:00.436 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:00.465 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:44:00.474 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:00.522 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:00.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:00.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:00.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:00 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/200259846' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:44:00.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:00 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:44:00.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:00 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/200259846' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:44:00.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:00 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:44:00.631 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:00.662 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:00.804 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:44:00.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:00 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/200259846' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:44:00.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:00 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:44:01.014 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:44:01.122 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:01.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:01.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:01.226 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:01.269 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:01.373 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:01.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:01.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:01.477 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:01.504 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:01.555 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:01.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:01 smithi190 ceph-mon[160435]: pgmap v1525: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 132 op/s; 7734/199494 objects degraded (3.877%); 4075/199494 objects misplaced (2.043%); 589 KiB/s, 9 objects/s recovering 2023-10-01T09:44:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:01 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:01 smithi190 ceph-mon[160435]: failed to encode map e526 with expected crc 2023-10-01T09:44:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:01 smithi190 ceph-mon[160435]: failed to encode map e526 with expected crc 2023-10-01T09:44:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:01 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:44:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:01 smithi190 ceph-mon[160435]: osdmap e526: 8 total, 8 up, 7 in 2023-10-01T09:44:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:01 smithi190 ceph-mon[160435]: failed to encode map e526 with expected crc 2023-10-01T09:44:01.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:01.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:01.658 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:01.690 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:01.728 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:01 smithi138 ceph-mon[192513]: pgmap v1525: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 132 op/s; 7734/199494 objects degraded (3.877%); 4075/199494 objects misplaced (2.043%); 589 KiB/s, 9 objects/s recovering 2023-10-01T09:44:01.728 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:01 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:01.729 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:01 smithi138 ceph-mon[192513]: failed to encode map e526 with expected crc 2023-10-01T09:44:01.729 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:01 smithi138 ceph-mon[192513]: failed to encode map e526 with expected crc 2023-10-01T09:44:01.729 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:01 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:44:01.729 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:01 smithi138 ceph-mon[192513]: osdmap e526: 8 total, 8 up, 7 in 2023-10-01T09:44:01.729 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:01 smithi138 ceph-mon[192513]: failed to encode map e526 with expected crc 2023-10-01T09:44:01.729 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:01 smithi138 ceph-mon[185886]: pgmap v1525: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 132 op/s; 7734/199494 objects degraded (3.877%); 4075/199494 objects misplaced (2.043%); 589 KiB/s, 9 objects/s recovering 2023-10-01T09:44:01.729 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:01 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:01.729 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:01 smithi138 ceph-mon[185886]: failed to encode map e526 with expected crc 2023-10-01T09:44:01.729 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:01 smithi138 ceph-mon[185886]: failed to encode map e526 with expected crc 2023-10-01T09:44:01.730 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:01 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:44:01.730 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:01 smithi138 ceph-mon[185886]: osdmap e526: 8 total, 8 up, 7 in 2023-10-01T09:44:01.730 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:01 smithi138 ceph-mon[185886]: failed to encode map e526 with expected crc 2023-10-01T09:44:01.818 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:01.837 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:01.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:01.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:01.942 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:01.971 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:02.037 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:02.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:02.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:02.137 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:44:02.138 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:44:02.144 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:02.179 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:02.237 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:02.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:02.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:02.297 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:44:02.298 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:44:02.298 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:44:02.298 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:44:02.298 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:44:02.343 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:02.371 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:03.013 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:44:03.176 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1526,"stamp":"2023-10-01T09:44:00.457288+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4360902603,"num_objects":66498,"num_object_clones":0,"num_object_copies":199494,"num_objects_missing_on_primary":2622,"num_objects_missing":2622,"num_objects_degraded":7734,"num_objects_misplaced":4075,"num_objects_unfound":0,"num_objects_dirty":66498,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":101076,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8483,"num_bytes_recovered":542802959,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101140,"ondisk_log_size":101140,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18287056,"kb_used_data":13217368,"kb_used_omap":39,"kb_used_meta":5069592,"kb_avail":637929008,"statfs":{"total":671965249536,"available":653239304192,"internally_reserved":0,"allocated":13534584832,"data_stored":13524742159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5191262831},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-48562176,"num_objects":-741,"num_object_clones":0,"num_object_copies":-2223,"num_objects_missing_on_primary":-46,"num_objects_missing":-46,"num_objects_degraded":-62,"num_objects_misplaced":-31,"num_objects_unfound":0,"num_objects_dirty":-741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":741,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":3080192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.143134"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":301,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.497405+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:43:44.497405+0000","last_peered":"2023-10-01T09:43:44.497405+0000","last_clean":"2023-10-01T09:43:44.497405+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:43:44.497405+0000","last_undegraded":"2023-10-01T09:43:44.497405+0000","last_fullsized":"2023-10-01T09:43:44.497405+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":144,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.509434+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:43:44.509434+0000","last_peered":"2023-10-01T09:43:44.509434+0000","last_clean":"2023-10-01T09:43:44.509434+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:43:44.509434+0000","last_undegraded":"2023-10-01T09:43:44.509434+0000","last_fullsized":"2023-10-01T09:43:44.509434+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":525,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.509181+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:43:44.509181+0000","last_peered":"2023-10-01T09:43:44.509181+0000","last_clean":"2023-10-01T09:43:44.509181+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:43:44.509181+0000","last_undegraded":"2023-10-01T09:43:44.509181+0000","last_fullsized":"2023-10-01T09:43:44.509181+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":583,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.438802+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:43:44.438802+0000","last_peered":"2023-10-01T09:43:44.438802+0000","last_clean":"2023-10-01T09:43:44.438802+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:43:44.438802+0000","last_undegraded":"2023-10-01T09:43:44.438802+0000","last_fullsized":"2023-10-01T09:43:44.438802+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":464,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.438401+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:43:44.438401+0000","last_peered":"2023-10-01T09:43:44.438401+0000","last_clean":"2023-10-01T09:43:44.438401+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:43:44.438401+0000","last_undegraded":"2023-10-01T09:43:44.438401+0000","last_fullsized":"2023-10-01T09:43:44.438401+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":206,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.471933+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:43:44.471933+0000","last_peered":"2023-10-01T09:43:44.471933+0000","last_clean":"2023-10-01T09:43:44.471933+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:43:44.471933+0000","last_undegraded":"2023-10-01T09:43:44.471933+0000","last_fullsized":"2023-10-01T09:43:44.471933+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":582,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.471994+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:43:44.471994+0000","last_peered":"2023-10-01T09:43:44.471994+0000","last_clean":"2023-10-01T09:43:44.471994+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:43:44.471994+0000","last_undegraded":"2023-10-01T09:43:44.471994+0000","last_fullsized":"2023-10-01T09:43:44.471994+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":511,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.434599+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:43:44.434599+0000","last_peered":"2023-10-01T09:43:44.434599+0000","last_clean":"2023-10-01T09:43:44.434599+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:43:44.434599+0000","last_undegraded":"2023-10-01T09:43:44.434599+0000","last_fullsized":"2023-10-01T09:43:44.434599+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":237,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.457870+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:43:44.457870+0000","last_peered":"2023-10-01T09:43:44.457870+0000","last_clean":"2023-10-01T09:43:44.457870+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:43:44.457870+0000","last_undegraded":"2023-10-01T09:43:44.457870+0000","last_fullsized":"2023-10-01T09:43:44.457870+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":44,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.437277+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:43:44.437277+0000","last_peered":"2023-10-01T09:43:44.437277+0000","last_clean":"2023-10-01T09:43:44.437277+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:43:44.437277+0000","last_undegraded":"2023-10-01T09:43:44.437277+0000","last_fullsized":"2023-10-01T09:43:44.437277+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":50,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.437053+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:43:44.437053+0000","last_peered":"2023-10-01T09:43:44.437053+0000","last_clean":"2023-10-01T09:43:44.437053+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:43:44.437053+0000","last_undegraded":"2023-10-01T09:43:44.437053+0000","last_fullsized":"2023-10-01T09:43:44.437053+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.509132+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:43:44.509132+0000","last_peered":"2023-10-01T09:43:44.509132+0000","last_clean":"2023-10-01T09:43:44.509132+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:43:44.509132+0000","last_undegraded":"2023-10-01T09:43:44.509132+0000","last_fullsized":"2023-10-01T09:43:44.509132+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":426,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.498320+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:43:44.498320+0000","last_peered":"2023-10-01T09:43:44.498320+0000","last_clean":"2023-10-01T09:43:44.498320+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:43:44.498320+0000","last_undegraded":"2023-10-01T09:43:44.498320+0000","last_fullsized":"2023-10-01T09:43:44.498320+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":307,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.497979+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:43:44.497979+0000","last_peered":"2023-10-01T09:43:44.497979+0000","last_clean":"2023-10-01T09:43:44.497979+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:43:44.497979+0000","last_undegraded":"2023-10-01T09:43:44.497979+0000","last_fullsized":"2023-10-01T09:43:44.497979+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":643,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.437180+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:43:44.437180+0000","last_peered":"2023-10-01T09:43:44.437180+0000","last_clean":"2023-10-01T09:43:44.437180+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:43:44.437180+0000","last_undegraded":"2023-10-01T09:43:44.437180+0000","last_fullsized":"2023-10-01T09:43:44.437180+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":380,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.437422+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:43:44.437422+0000","last_peered":"2023-10-01T09:43:44.437422+0000","last_clean":"2023-10-01T09:43:44.437422+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:43:44.437422+0000","last_undegraded":"2023-10-01T09:43:44.437422+0000","last_fullsized":"2023-10-01T09:43:44.437422+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.437019+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:43:44.437019+0000","last_peered":"2023-10-01T09:43:44.437019+0000","last_clean":"2023-10-01T09:43:44.437019+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:43:44.437019+0000","last_undegraded":"2023-10-01T09:43:44.437019+0000","last_fullsized":"2023-10-01T09:43:44.437019+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":39,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.457777+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:43:44.457777+0000","last_peered":"2023-10-01T09:43:44.457777+0000","last_clean":"2023-10-01T09:43:44.457777+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:43:44.457777+0000","last_undegraded":"2023-10-01T09:43:44.457777+0000","last_fullsized":"2023-10-01T09:43:44.457777+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":536,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.437443+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:43:44.437443+0000","last_peered":"2023-10-01T09:43:44.437443+0000","last_clean":"2023-10-01T09:43:44.437443+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:43:44.437443+0000","last_undegraded":"2023-10-01T09:43:44.437443+0000","last_fullsized":"2023-10-01T09:43:44.437443+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":547,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.438878+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:43:44.438878+0000","last_peered":"2023-10-01T09:43:44.438878+0000","last_clean":"2023-10-01T09:43:44.438878+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:43:44.438878+0000","last_undegraded":"2023-10-01T09:43:44.438878+0000","last_fullsized":"2023-10-01T09:43:44.438878+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":403,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.438311+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:43:44.438311+0000","last_peered":"2023-10-01T09:43:44.438311+0000","last_clean":"2023-10-01T09:43:44.438311+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:43:44.438311+0000","last_undegraded":"2023-10-01T09:43:44.438311+0000","last_fullsized":"2023-10-01T09:43:44.438311+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":53,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.437432+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:43:44.437432+0000","last_peered":"2023-10-01T09:43:44.437432+0000","last_clean":"2023-10-01T09:43:44.437432+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:43:44.437432+0000","last_undegraded":"2023-10-01T09:43:44.437432+0000","last_fullsized":"2023-10-01T09:43:44.437432+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"525'6269","reported_seq":6327,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:56.344063+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:43:56.344063+0000","last_peered":"2023-10-01T09:43:56.344063+0000","last_clean":"2023-10-01T09:43:56.344063+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:43:56.344063+0000","last_undegraded":"2023-10-01T09:43:56.344063+0000","last_fullsized":"2023-10-01T09:43:56.344063+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6269,"log_dups_size":0,"ondisk_log_size":6269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270204928,"num_objects":4123,"num_object_clones":0,"num_object_copies":12369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6269,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":48,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.437328+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:43:44.437328+0000","last_peered":"2023-10-01T09:43:44.437328+0000","last_clean":"2023-10-01T09:43:44.437328+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:43:44.437328+0000","last_undegraded":"2023-10-01T09:43:44.437328+0000","last_fullsized":"2023-10-01T09:43:44.437328+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":245,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.457381+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:43:44.457381+0000","last_peered":"2023-10-01T09:43:44.457381+0000","last_clean":"2023-10-01T09:43:44.457381+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:43:44.457381+0000","last_undegraded":"2023-10-01T09:43:44.457381+0000","last_fullsized":"2023-10-01T09:43:44.457381+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"525'6264","reported_seq":9537,"reported_epoch":525,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:43:56.481062+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:43:56.481062+0000","last_peered":"2023-10-01T09:43:56.481062+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:43:56.481062+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6264,"log_dups_size":0,"ondisk_log_size":6264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271450112,"num_objects":4142,"num_object_clones":0,"num_object_copies":12426,"num_objects_missing_on_primary":2622,"num_objects_missing":2622,"num_objects_degraded":6695,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6264,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1062,"num_bytes_recovered":69599232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1451},{"shards":"0,4,6","objects":2622},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":519,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.437187+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:43:44.437187+0000","last_peered":"2023-10-01T09:43:44.437187+0000","last_clean":"2023-10-01T09:43:44.437187+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:43:44.437187+0000","last_undegraded":"2023-10-01T09:43:44.437187+0000","last_fullsized":"2023-10-01T09:43:44.437187+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":580,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.509132+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:43:44.509132+0000","last_peered":"2023-10-01T09:43:44.509132+0000","last_clean":"2023-10-01T09:43:44.509132+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:43:44.509132+0000","last_undegraded":"2023-10-01T09:43:44.509132+0000","last_fullsized":"2023-10-01T09:43:44.509132+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"525'6279","reported_seq":6339,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:56.558143+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:43:56.558143+0000","last_peered":"2023-10-01T09:43:56.558143+0000","last_clean":"2023-10-01T09:43:56.558143+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:43:56.558143+0000","last_undegraded":"2023-10-01T09:43:56.558143+0000","last_fullsized":"2023-10-01T09:43:56.558143+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6279,"log_dups_size":0,"ondisk_log_size":6279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272171008,"num_objects":4153,"num_object_clones":0,"num_object_copies":12459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6279,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":605,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.437115+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:43:44.437115+0000","last_peered":"2023-10-01T09:43:44.437115+0000","last_clean":"2023-10-01T09:43:44.437115+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:43:44.437115+0000","last_undegraded":"2023-10-01T09:43:44.437115+0000","last_fullsized":"2023-10-01T09:43:44.437115+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":400,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.509250+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:43:44.509250+0000","last_peered":"2023-10-01T09:43:44.509250+0000","last_clean":"2023-10-01T09:43:44.509250+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:43:44.509250+0000","last_undegraded":"2023-10-01T09:43:44.509250+0000","last_fullsized":"2023-10-01T09:43:44.509250+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"525'6247","reported_seq":12912,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:56.809955+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:43:56.809955+0000","last_peered":"2023-10-01T09:43:56.809955+0000","last_clean":"2023-10-01T09:43:56.809955+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:43:56.809955+0000","last_undegraded":"2023-10-01T09:43:56.809955+0000","last_fullsized":"2023-10-01T09:43:56.809955+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6247,"log_dups_size":0,"ondisk_log_size":6247,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272302080,"num_objects":4155,"num_object_clones":0,"num_object_copies":12465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6247,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":245,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.457556+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:43:44.457556+0000","last_peered":"2023-10-01T09:43:44.457556+0000","last_clean":"2023-10-01T09:43:44.457556+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:43:44.457556+0000","last_undegraded":"2023-10-01T09:43:44.457556+0000","last_fullsized":"2023-10-01T09:43:44.457556+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":503,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.438406+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:43:44.438406+0000","last_peered":"2023-10-01T09:43:44.438406+0000","last_clean":"2023-10-01T09:43:44.438406+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:43:44.438406+0000","last_undegraded":"2023-10-01T09:43:44.438406+0000","last_fullsized":"2023-10-01T09:43:44.438406+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"525'6281","reported_seq":6341,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:59.843390+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:43:59.843390+0000","last_peered":"2023-10-01T09:43:59.843390+0000","last_clean":"2023-10-01T09:43:59.843390+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:43:59.843390+0000","last_undegraded":"2023-10-01T09:43:59.843390+0000","last_fullsized":"2023-10-01T09:43:59.843390+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6281,"log_dups_size":0,"ondisk_log_size":6281,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277020672,"num_objects":4227,"num_object_clones":0,"num_object_copies":12681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6281,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":105,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.498305+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:43:44.498305+0000","last_peered":"2023-10-01T09:43:44.498305+0000","last_clean":"2023-10-01T09:43:44.498305+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:43:44.498305+0000","last_undegraded":"2023-10-01T09:43:44.498305+0000","last_fullsized":"2023-10-01T09:43:44.498305+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":179,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.498443+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:43:44.498443+0000","last_peered":"2023-10-01T09:43:44.498443+0000","last_clean":"2023-10-01T09:43:44.498443+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:43:44.498443+0000","last_undegraded":"2023-10-01T09:43:44.498443+0000","last_fullsized":"2023-10-01T09:43:44.498443+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"525'6229","reported_seq":6329,"reported_epoch":525,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:43:56.110027+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:43:56.110027+0000","last_peered":"2023-10-01T09:43:56.110027+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:43:56.110027+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6229,"log_dups_size":0,"ondisk_log_size":6229,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271253504,"num_objects":4139,"num_object_clones":0,"num_object_copies":12417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":4075,"num_objects_unfound":0,"num_objects_dirty":4139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6229,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":1114112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4075},{"shards":"4,5","objects":-965}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":423,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.497513+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:43:44.497513+0000","last_peered":"2023-10-01T09:43:44.497513+0000","last_clean":"2023-10-01T09:43:44.497513+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:43:44.497513+0000","last_undegraded":"2023-10-01T09:43:44.497513+0000","last_fullsized":"2023-10-01T09:43:44.497513+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":583,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.509444+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:43:44.509444+0000","last_peered":"2023-10-01T09:43:44.509444+0000","last_clean":"2023-10-01T09:43:44.509444+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:43:44.509444+0000","last_undegraded":"2023-10-01T09:43:44.509444+0000","last_fullsized":"2023-10-01T09:43:44.509444+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"525'6204","reported_seq":6271,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:57.240382+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:43:57.240382+0000","last_peered":"2023-10-01T09:43:57.240382+0000","last_clean":"2023-10-01T09:43:57.240382+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:43:57.240382+0000","last_undegraded":"2023-10-01T09:43:57.240382+0000","last_fullsized":"2023-10-01T09:43:57.240382+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6204,"log_dups_size":0,"ondisk_log_size":6204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273547264,"num_objects":4174,"num_object_clones":0,"num_object_copies":12522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6204,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":419,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.499496+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:43:44.499496+0000","last_peered":"2023-10-01T09:43:44.499496+0000","last_clean":"2023-10-01T09:43:44.499496+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:43:44.499496+0000","last_undegraded":"2023-10-01T09:43:44.499496+0000","last_fullsized":"2023-10-01T09:43:44.499496+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":260,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.498607+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:43:44.498607+0000","last_peered":"2023-10-01T09:43:44.498607+0000","last_clean":"2023-10-01T09:43:44.498607+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:43:44.498607+0000","last_undegraded":"2023-10-01T09:43:44.498607+0000","last_fullsized":"2023-10-01T09:43:44.498607+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"525'6320","reported_seq":6378,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:55.943109+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:43:55.943109+0000","last_peered":"2023-10-01T09:43:55.943109+0000","last_clean":"2023-10-01T09:43:55.943109+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:43:55.943109+0000","last_undegraded":"2023-10-01T09:43:55.943109+0000","last_fullsized":"2023-10-01T09:43:55.943109+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6320,"log_dups_size":0,"ondisk_log_size":6320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273809408,"num_objects":4178,"num_object_clones":0,"num_object_copies":12534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6320,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":388,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:45.255560+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:43:45.255560+0000","last_peered":"2023-10-01T09:43:45.255560+0000","last_clean":"2023-10-01T09:43:45.255560+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:43:45.255560+0000","last_undegraded":"2023-10-01T09:43:45.255560+0000","last_fullsized":"2023-10-01T09:43:45.255560+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":435,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.509650+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:43:44.509650+0000","last_peered":"2023-10-01T09:43:44.509650+0000","last_clean":"2023-10-01T09:43:44.509650+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:43:44.509650+0000","last_undegraded":"2023-10-01T09:43:44.509650+0000","last_fullsized":"2023-10-01T09:43:44.509650+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"525'6139","reported_seq":6197,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:56.125947+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:43:56.125947+0000","last_peered":"2023-10-01T09:43:56.125947+0000","last_clean":"2023-10-01T09:43:56.125947+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:43:56.125947+0000","last_undegraded":"2023-10-01T09:43:56.125947+0000","last_fullsized":"2023-10-01T09:43:56.125947+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6139,"log_dups_size":0,"ondisk_log_size":6139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264699904,"num_objects":4039,"num_object_clones":0,"num_object_copies":12117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6139,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":918,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.437296+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:43:44.437296+0000","last_peered":"2023-10-01T09:43:44.437296+0000","last_clean":"2023-10-01T09:43:44.437296+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:43:44.437296+0000","last_undegraded":"2023-10-01T09:43:44.437296+0000","last_fullsized":"2023-10-01T09:43:44.437296+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":255,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.457772+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:43:44.457772+0000","last_peered":"2023-10-01T09:43:44.457772+0000","last_clean":"2023-10-01T09:43:44.457772+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:43:44.457772+0000","last_undegraded":"2023-10-01T09:43:44.457772+0000","last_fullsized":"2023-10-01T09:43:44.457772+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":48,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.509182+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:43:44.509182+0000","last_peered":"2023-10-01T09:43:44.509182+0000","last_clean":"2023-10-01T09:43:44.509182+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:43:44.509182+0000","last_undegraded":"2023-10-01T09:43:44.509182+0000","last_fullsized":"2023-10-01T09:43:44.509182+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"525'6188","reported_seq":6248,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:56.715537+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:43:56.715537+0000","last_peered":"2023-10-01T09:43:56.715537+0000","last_clean":"2023-10-01T09:43:56.715537+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:43:56.715537+0000","last_undegraded":"2023-10-01T09:43:56.715537+0000","last_fullsized":"2023-10-01T09:43:56.715537+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6188,"log_dups_size":0,"ondisk_log_size":6188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274071552,"num_objects":4182,"num_object_clones":0,"num_object_copies":12546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6188,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1190,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.457513+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:43:44.457513+0000","last_peered":"2023-10-01T09:43:44.457513+0000","last_clean":"2023-10-01T09:43:44.457513+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:43:44.457513+0000","last_undegraded":"2023-10-01T09:43:44.457513+0000","last_fullsized":"2023-10-01T09:43:44.457513+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":400,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.498170+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:43:44.498170+0000","last_peered":"2023-10-01T09:43:44.498170+0000","last_clean":"2023-10-01T09:43:44.498170+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:43:44.498170+0000","last_undegraded":"2023-10-01T09:43:44.498170+0000","last_fullsized":"2023-10-01T09:43:44.498170+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":228,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.457709+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:43:44.457709+0000","last_peered":"2023-10-01T09:43:44.457709+0000","last_clean":"2023-10-01T09:43:44.457709+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:43:44.457709+0000","last_undegraded":"2023-10-01T09:43:44.457709+0000","last_fullsized":"2023-10-01T09:43:44.457709+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"525'6244","reported_seq":13523,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:56.147859+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:43:56.147859+0000","last_peered":"2023-10-01T09:43:56.147859+0000","last_clean":"2023-10-01T09:43:56.147859+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:43:56.147859+0000","last_undegraded":"2023-10-01T09:43:56.147859+0000","last_fullsized":"2023-10-01T09:43:56.147859+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6244,"log_dups_size":0,"ondisk_log_size":6244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270794752,"num_objects":4132,"num_object_clones":0,"num_object_copies":12396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6244,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":870,"reported_epoch":524,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:43:44.438078+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:43:44.438078+0000","last_peered":"2023-10-01T09:43:44.438078+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:43:44.438078+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:43:44.438078+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":587,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.511339+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:43:44.511339+0000","last_peered":"2023-10-01T09:43:44.511339+0000","last_clean":"2023-10-01T09:43:44.511339+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:43:44.511339+0000","last_undegraded":"2023-10-01T09:43:44.511339+0000","last_fullsized":"2023-10-01T09:43:44.511339+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":48,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.509598+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:43:44.509598+0000","last_peered":"2023-10-01T09:43:44.509598+0000","last_clean":"2023-10-01T09:43:44.509598+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:43:44.509598+0000","last_undegraded":"2023-10-01T09:43:44.509598+0000","last_fullsized":"2023-10-01T09:43:44.509598+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"525'6163","reported_seq":6223,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:59.561476+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:43:59.561476+0000","last_peered":"2023-10-01T09:43:59.561476+0000","last_clean":"2023-10-01T09:43:59.561476+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:43:59.561476+0000","last_undegraded":"2023-10-01T09:43:59.561476+0000","last_fullsized":"2023-10-01T09:43:59.561476+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6163,"log_dups_size":0,"ondisk_log_size":6163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269287424,"num_objects":4109,"num_object_clones":0,"num_object_copies":12327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6163,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1641,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.457496+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:43:44.457496+0000","last_peered":"2023-10-01T09:43:44.457496+0000","last_clean":"2023-10-01T09:43:44.457496+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:43:44.457496+0000","last_undegraded":"2023-10-01T09:43:44.457496+0000","last_fullsized":"2023-10-01T09:43:44.457496+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":555,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.436837+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:43:44.436837+0000","last_peered":"2023-10-01T09:43:44.436837+0000","last_clean":"2023-10-01T09:43:44.436837+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:43:44.436837+0000","last_undegraded":"2023-10-01T09:43:44.436837+0000","last_fullsized":"2023-10-01T09:43:44.436837+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":333,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.457569+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:43:44.457569+0000","last_peered":"2023-10-01T09:43:44.457569+0000","last_clean":"2023-10-01T09:43:44.457569+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:43:44.457569+0000","last_undegraded":"2023-10-01T09:43:44.457569+0000","last_fullsized":"2023-10-01T09:43:44.457569+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":323,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.497998+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:43:44.497998+0000","last_peered":"2023-10-01T09:43:44.497998+0000","last_clean":"2023-10-01T09:43:44.497998+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:43:44.497998+0000","last_undegraded":"2023-10-01T09:43:44.497998+0000","last_fullsized":"2023-10-01T09:43:44.497998+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"525'6313","reported_seq":6371,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:56.087414+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:43:56.087414+0000","last_peered":"2023-10-01T09:43:56.087414+0000","last_clean":"2023-10-01T09:43:56.087414+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:43:56.087414+0000","last_undegraded":"2023-10-01T09:43:56.087414+0000","last_fullsized":"2023-10-01T09:43:56.087414+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6313,"log_dups_size":0,"ondisk_log_size":6313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276496384,"num_objects":4219,"num_object_clones":0,"num_object_copies":12657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6313,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":732,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.436930+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:43:44.436930+0000","last_peered":"2023-10-01T09:43:44.436930+0000","last_clean":"2023-10-01T09:43:44.436930+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:43:44.436930+0000","last_undegraded":"2023-10-01T09:43:44.436930+0000","last_fullsized":"2023-10-01T09:43:44.436930+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":564,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.438716+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:43:44.438716+0000","last_peered":"2023-10-01T09:43:44.438716+0000","last_clean":"2023-10-01T09:43:44.438716+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:43:44.438716+0000","last_undegraded":"2023-10-01T09:43:44.438716+0000","last_fullsized":"2023-10-01T09:43:44.438716+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":564,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.438951+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:43:44.438951+0000","last_peered":"2023-10-01T09:43:44.438951+0000","last_clean":"2023-10-01T09:43:44.438951+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:43:44.438951+0000","last_undegraded":"2023-10-01T09:43:44.438951+0000","last_fullsized":"2023-10-01T09:43:44.438951+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"525'6284","reported_seq":6345,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:56.749327+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:43:56.749327+0000","last_peered":"2023-10-01T09:43:56.749327+0000","last_clean":"2023-10-01T09:43:56.749327+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:43:56.749327+0000","last_undegraded":"2023-10-01T09:43:56.749327+0000","last_fullsized":"2023-10-01T09:43:56.749327+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6284,"log_dups_size":0,"ondisk_log_size":6284,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273612824,"num_objects":4176,"num_object_clones":0,"num_object_copies":12528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4176,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6284,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":963,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.437149+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:43:44.437149+0000","last_peered":"2023-10-01T09:43:44.437149+0000","last_clean":"2023-10-01T09:43:44.437149+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:43:44.437149+0000","last_undegraded":"2023-10-01T09:43:44.437149+0000","last_fullsized":"2023-10-01T09:43:44.437149+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":582,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.457666+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:43:44.457666+0000","last_peered":"2023-10-01T09:43:44.457666+0000","last_clean":"2023-10-01T09:43:44.457666+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:43:44.457666+0000","last_undegraded":"2023-10-01T09:43:44.457666+0000","last_fullsized":"2023-10-01T09:43:44.457666+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":435,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.457598+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:43:44.457598+0000","last_peered":"2023-10-01T09:43:44.457598+0000","last_clean":"2023-10-01T09:43:44.457598+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:43:44.457598+0000","last_undegraded":"2023-10-01T09:43:44.457598+0000","last_fullsized":"2023-10-01T09:43:44.457598+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"525'6263","reported_seq":6329,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:55.994759+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:43:55.994759+0000","last_peered":"2023-10-01T09:43:55.994759+0000","last_clean":"2023-10-01T09:43:55.994759+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:43:55.994759+0000","last_undegraded":"2023-10-01T09:43:55.994759+0000","last_fullsized":"2023-10-01T09:43:55.994759+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6263,"log_dups_size":0,"ondisk_log_size":6263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268238848,"num_objects":4093,"num_object_clones":0,"num_object_copies":12279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6263,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":464,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.437502+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:43:44.437502+0000","last_peered":"2023-10-01T09:43:44.437502+0000","last_clean":"2023-10-01T09:43:44.437502+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:43:44.437502+0000","last_undegraded":"2023-10-01T09:43:44.437502+0000","last_fullsized":"2023-10-01T09:43:44.437502+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1157,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.509574+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:43:44.509574+0000","last_peered":"2023-10-01T09:43:44.509574+0000","last_clean":"2023-10-01T09:43:44.509574+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:43:44.509574+0000","last_undegraded":"2023-10-01T09:43:44.509574+0000","last_fullsized":"2023-10-01T09:43:44.509574+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":53,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.436773+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:43:44.436773+0000","last_peered":"2023-10-01T09:43:44.436773+0000","last_clean":"2023-10-01T09:43:44.436773+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:43:44.436773+0000","last_undegraded":"2023-10-01T09:43:44.436773+0000","last_fullsized":"2023-10-01T09:43:44.436773+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"525'6283","reported_seq":6349,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:56.047111+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:43:56.047111+0000","last_peered":"2023-10-01T09:43:56.047111+0000","last_clean":"2023-10-01T09:43:56.047111+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:43:56.047111+0000","last_undegraded":"2023-10-01T09:43:56.047111+0000","last_fullsized":"2023-10-01T09:43:56.047111+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6283,"log_dups_size":0,"ondisk_log_size":6283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277151744,"num_objects":4229,"num_object_clones":0,"num_object_copies":12687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6283,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":57,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.437386+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:43:44.437386+0000","last_peered":"2023-10-01T09:43:44.437386+0000","last_clean":"2023-10-01T09:43:44.437386+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:43:44.437386+0000","last_undegraded":"2023-10-01T09:43:44.437386+0000","last_fullsized":"2023-10-01T09:43:44.437386+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":542,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.438724+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:43:44.438724+0000","last_peered":"2023-10-01T09:43:44.438724+0000","last_clean":"2023-10-01T09:43:44.438724+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:43:44.438724+0000","last_undegraded":"2023-10-01T09:43:44.438724+0000","last_fullsized":"2023-10-01T09:43:44.438724+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":248,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.457449+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:43:44.457449+0000","last_peered":"2023-10-01T09:43:44.457449+0000","last_clean":"2023-10-01T09:43:44.457449+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:43:44.457449+0000","last_undegraded":"2023-10-01T09:43:44.457449+0000","last_fullsized":"2023-10-01T09:43:44.457449+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":491,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.438523+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:43:44.438523+0000","last_peered":"2023-10-01T09:43:44.438523+0000","last_clean":"2023-10-01T09:43:44.438523+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:43:44.438523+0000","last_undegraded":"2023-10-01T09:43:44.438523+0000","last_fullsized":"2023-10-01T09:43:44.438523+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":605,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.457134+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:43:52.457134+0000","last_peered":"2023-10-01T09:43:52.457134+0000","last_clean":"2023-10-01T09:43:52.457134+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:43:52.457134+0000","last_undegraded":"2023-10-01T09:43:52.457134+0000","last_fullsized":"2023-10-01T09:43:52.457134+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":260,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.457422+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:43:44.457422+0000","last_peered":"2023-10-01T09:43:44.457422+0000","last_clean":"2023-10-01T09:43:44.457422+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:43:44.457422+0000","last_undegraded":"2023-10-01T09:43:44.457422+0000","last_fullsized":"2023-10-01T09:43:44.457422+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":48,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.509193+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:43:44.509193+0000","last_peered":"2023-10-01T09:43:44.509193+0000","last_clean":"2023-10-01T09:43:44.509193+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:43:44.509193+0000","last_undegraded":"2023-10-01T09:43:44.509193+0000","last_fullsized":"2023-10-01T09:43:44.509193+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":260,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.457851+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:43:44.457851+0000","last_peered":"2023-10-01T09:43:44.457851+0000","last_clean":"2023-10-01T09:43:44.457851+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:43:44.457851+0000","last_undegraded":"2023-10-01T09:43:44.457851+0000","last_fullsized":"2023-10-01T09:43:44.457851+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":437,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.438370+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:43:44.438370+0000","last_peered":"2023-10-01T09:43:44.438370+0000","last_clean":"2023-10-01T09:43:44.438370+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:43:44.438370+0000","last_undegraded":"2023-10-01T09:43:44.438370+0000","last_fullsized":"2023-10-01T09:43:44.438370+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":65,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.509471+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:43:44.509471+0000","last_peered":"2023-10-01T09:43:44.509471+0000","last_clean":"2023-10-01T09:43:44.509471+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:43:44.509471+0000","last_undegraded":"2023-10-01T09:43:44.509471+0000","last_fullsized":"2023-10-01T09:43:44.509471+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":482,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.438556+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:43:44.438556+0000","last_peered":"2023-10-01T09:43:44.438556+0000","last_clean":"2023-10-01T09:43:44.438556+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:43:44.438556+0000","last_undegraded":"2023-10-01T09:43:44.438556+0000","last_fullsized":"2023-10-01T09:43:44.438556+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":681,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.438868+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:43:44.438868+0000","last_peered":"2023-10-01T09:43:44.438868+0000","last_clean":"2023-10-01T09:43:44.438868+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:43:44.438868+0000","last_undegraded":"2023-10-01T09:43:44.438868+0000","last_fullsized":"2023-10-01T09:43:44.438868+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":376,"reported_epoch":524,"state":"active+clean","last_fresh":"2023-10-01T09:43:44.497267+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:43:44.497267+0000","last_peered":"2023-10-01T09:43:44.497267+0000","last_clean":"2023-10-01T09:43:44.497267+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:43:44.497267+0000","last_undegraded":"2023-10-01T09:43:44.497267+0000","last_fullsized":"2023-10-01T09:43:44.497267+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4356112408,"num_objects":66470,"num_object_clones":0,"num_object_copies":199410,"num_objects_missing_on_primary":2622,"num_objects_missing":2622,"num_objects_degraded":7724,"num_objects_misplaced":4075,"num_objects_unfound":0,"num_objects_dirty":66470,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":99970,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7972,"num_bytes_recovered":522452992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13466939392,"data_stored":13466927176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99970,"ondisk_log_size":99970,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430720,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2243704,"kb_used_data":1336496,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":91501448,"statfs":{"total":95995035648,"available":93697482752,"internally_reserved":0,"allocated":1368571904,"data_stored":1367634524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:56 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.4950000000000001,"15min":2.9569999999999999},"min":{"1min":0.246,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":1.3280000000000001,"5min":10.391999999999999,"15min":108.90000000000001},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":1.8129999999999999,"15min":3.129},"min":{"1min":0.246,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.018,"5min":29.501000000000001,"15min":109.187},"last":0.29999999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":4.6749999999999998,"15min":5.3520000000000003},"min":{"1min":0.31900000000000001,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":1.2949999999999999,"5min":229.161,"15min":229.161},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":4.6479999999999997,"15min":5.3689999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":0.90900000000000003,"5min":230.126,"15min":230.126},"last":0.44600000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:43:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.169,"15min":1.76},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.992,"5min":14.866,"15min":14.901999999999999},"last":0.501},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.4450000000000001,"15min":1.7170000000000001},"min":{"1min":0.32000000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4490000000000001,"5min":14.682,"15min":14.682},"last":1.1539999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":1.514,"15min":2.7330000000000001},"min":{"1min":0.30199999999999999,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":0.90700000000000003,"5min":35.104999999999997,"15min":80.352999999999994},"last":1.018},{"interface":"front","average":{"1min":0.50600000000000001,"5min":1.6930000000000001,"15min":2.7130000000000001},"min":{"1min":0.35399999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.153,"5min":35.118000000000002,"15min":80.364000000000004},"last":1.026}]},{"osd":4,"last update":"Sun Oct 1 09:42:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":1.2150000000000001,"15min":1.052},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.381,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.49099999999999999,"5min":1.1080000000000001,"15min":1.03},"min":{"1min":0.314,"5min":0.30299999999999999,"15min":0.28000000000000003},"max":{"1min":0.97899999999999998,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.34100000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":2.3620000000000001,"15min":1.6120000000000001},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":1.2749999999999999,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.62,"5min":2.5390000000000001,"15min":1.6739999999999999},"min":{"1min":0.35999999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.2599999999999998,"5min":135.339,"15min":135.339},"last":0.52400000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":1.3200000000000001,"15min":1.1599999999999999},"min":{"1min":0.377,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":1.409,"5min":11.58,"15min":11.853999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":1.175,"15min":1.1299999999999999},"min":{"1min":0.378,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":1.23,"5min":14.584,"15min":14.584},"last":0.48299999999999998}]}]},{"osd":6,"up_from":294,"seq":1262720385231,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2207092,"kb_used_data":1869484,"kb_used_omap":6,"kb_used_meta":337593,"kb_avail":91538060,"statfs":{"total":95995035648,"available":93734973440,"internally_reserved":0,"allocated":1914351616,"data_stored":1913418187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":345695277},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":3.5720000000000001,"15min":4.0919999999999996},"min":{"1min":0.25,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":3.3340000000000001,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.42599999999999999,"5min":4.0410000000000004,"15min":4.2210000000000001},"min":{"1min":0.29899999999999999,"5min":0.249,"15min":0.221},"max":{"1min":0.623,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.35699999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":4.7489999999999997,"15min":4.9740000000000002},"min":{"1min":0.30599999999999999,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":1.9410000000000001,"5min":168.947,"15min":168.947},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":4.7969999999999997,"15min":4.8810000000000002},"min":{"1min":0.34699999999999998,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":1.962,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.40300000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":1.194,"15min":1.673},"min":{"1min":0.308,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.075,"5min":8.657,"15min":10.92},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.3759999999999999,"15min":1.673},"min":{"1min":0.314,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.58499999999999996,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.36799999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.3069999999999999,"15min":2.2789999999999999},"min":{"1min":0.35499999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.3759999999999999,"5min":9.0180000000000007,"15min":89.753},"last":0.442},{"interface":"front","average":{"1min":0.502,"5min":1.4359999999999999,"15min":2.157},"min":{"1min":0.34399999999999997,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":1.381,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.40999999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":1.5900000000000001,"15min":1.0800000000000001},"min":{"1min":0.30099999999999999,"5min":0.253,"15min":0.246},"max":{"1min":0.53400000000000003,"5min":63.134,"15min":63.134},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.5009999999999999,"15min":1.097},"min":{"1min":0.33700000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":1.5669999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.33900000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:43:29 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":1.2450000000000001,"15min":1.101},"min":{"1min":0.309,"5min":0.251,"15min":0.251},"max":{"1min":34.094999999999999,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.409,"15min":1.22},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":34.072000000000003,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.434}]},{"osd":7,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.79500000000000004,"15min":1.46},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":1.4750000000000001,"5min":6.4119999999999999,"15min":94.245999999999995},"last":0.38},{"interface":"front","average":{"1min":0.48999999999999999,"5min":1.113,"15min":1.5569999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":0.68300000000000005,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.50600000000000001}]}]},{"osd":1,"up_from":94,"seq":403726926208,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1628452,"kb_used_data":810268,"kb_used_omap":7,"kb_used_meta":818168,"kb_avail":92116700,"statfs":{"total":95995035648,"available":94327500800,"internally_reserved":0,"allocated":829714432,"data_stored":827683008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":837804455},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.30299999999999999,"5min":2.5600000000000001,"15min":4.2009999999999996},"min":{"1min":0.20699999999999999,"5min":0.19400000000000001,"15min":0.182},"max":{"1min":0.57399999999999995,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.252},{"interface":"front","average":{"1min":0.33600000000000002,"5min":3.0139999999999998,"15min":3.7090000000000001},"min":{"1min":0.26300000000000001,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":0.53400000000000003,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.31900000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":1.4279999999999999,"15min":2.589},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.61399999999999999,"5min":22.891999999999999,"15min":56.573},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.379,"15min":3.254},"min":{"1min":0.28999999999999998,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":1.752,"5min":15.189,"15min":154.197},"last":0.26600000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":2.1680000000000001,"15min":3.9969999999999999},"min":{"1min":0.27400000000000002,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":1.1990000000000001,"5min":18.503,"15min":200.56800000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":2.3660000000000001,"15min":4.6989999999999998},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":0.51900000000000002,"5min":20.181000000000001,"15min":200.63800000000001},"last":0.43099999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":1.8029999999999999,"15min":2.3109999999999999},"min":{"1min":0.32900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.8109999999999999,"5min":21.015000000000001,"15min":154.101},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.46400000000000002,"5min":1.585,"15min":2.5249999999999999},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.83199999999999996,"5min":10.222,"15min":154.56999999999999},"last":0.49299999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:43:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.3200000000000001,"15min":2.097},"min":{"1min":0.372,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.1850000000000001,"5min":19.513000000000002,"15min":154.22999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.1719999999999999,"15min":2.4630000000000001},"min":{"1min":0.31,"5min":0.307,"15min":0.307},"max":{"1min":0.82699999999999996,"5min":10.305,"15min":154.54400000000001},"last":0.53200000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.849,"15min":2.6640000000000001},"min":{"1min":0.372,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.2789999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.495,"5min":1.702,"15min":2.4870000000000001},"min":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":0.67300000000000004,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.44700000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:42:57 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.9319999999999999,"15min":2.6440000000000001},"min":{"1min":0.40100000000000002,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":1.3029999999999999,"5min":21.062999999999999,"15min":154.20699999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.804,"15min":2.718},"min":{"1min":0.41699999999999998,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":1.3220000000000001,"5min":13.161,"15min":154.51599999999999},"last":0.56599999999999995}]}]},{"osd":0,"up_from":89,"seq":382252089742,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4238356,"kb_used_data":3334924,"kb_used_omap":8,"kb_used_meta":903415,"kb_avail":89506796,"statfs":{"total":95995035648,"available":91654959104,"internally_reserved":0,"allocated":3414962176,"data_stored":3412951660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":925097922},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":1.76,"15min":4.2480000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17399999999999999},"max":{"1min":1.891,"5min":52.316000000000003,"15min":216.821},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":3.4119999999999999,"15min":5.2130000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":3.552,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.26600000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:43:36 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":1.204,"15min":2.085},"min":{"1min":0.246,"5min":0.246,"15min":0.20100000000000001},"max":{"1min":1.1100000000000001,"5min":19.788,"15min":19.788},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.4139999999999999,"15min":2.3380000000000001},"min":{"1min":0.27200000000000002,"5min":0.23000000000000001,"15min":0.189},"max":{"1min":3.5939999999999999,"5min":19.375,"15min":25.844999999999999},"last":0.38500000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":1.474,"15min":3.907},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":1.1499999999999999,"5min":15.35,"15min":222.16499999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.748,"15min":3.9460000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":2.9550000000000001,"5min":14.545999999999999,"15min":222.227},"last":0.29099999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.5449999999999999,"15min":2.1339999999999999},"min":{"1min":0.28799999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":3.375,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.58399999999999996,"5min":1.4550000000000001,"15min":2.1309999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":3.5049999999999999,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.36899999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:43:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.2030000000000001,"15min":1.883},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66500000000000004,"5min":14.515000000000001,"15min":16.808},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.093,"15min":1.7709999999999999},"min":{"1min":0.33500000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.73499999999999999,"5min":5.9580000000000002,"15min":17.507000000000001},"last":0.55400000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:43:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.1850000000000001,"15min":1.774},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.35499999999999998},"max":{"1min":1.5900000000000001,"5min":6.5,"15min":17.535},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.88,"5min":1.395,"15min":2.0289999999999999},"min":{"1min":0.32600000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":3.6539999999999999,"5min":10.446,"15min":18.521000000000001},"last":0.45900000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.95499999999999996,"15min":2.048},"min":{"1min":0.379,"5min":0.374,"15min":0.35699999999999998},"max":{"1min":0.78600000000000003,"5min":4.976,"15min":48.895000000000003},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.093,"15min":2.206},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.35099999999999998},"max":{"1min":3.4319999999999999,"5min":9.4969999999999999,"15min":48.274000000000001},"last":0.49399999999999999}]}]},{"osd":2,"up_from":0,"seq":425201762682,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599155,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1748120,"kb_used_data":907792,"kb_used_omap":3,"kb_used_meta":840316,"kb_avail":91997032,"statfs":{"total":95995035648,"available":94204960768,"internally_reserved":0,"allocated":929579008,"data_stored":927500928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":860484327},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":4.9660000000000002,"15min":5.0880000000000001},"min":{"1min":0.20300000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":2.7989999999999999,"5min":118.748,"15min":137.34399999999999},"last":0.23200000000000001},{"interface":"front","average":{"1min":0.28899999999999998,"5min":4.9500000000000002,"15min":5.1379999999999999},"min":{"1min":0.20699999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.42199999999999999,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.40500000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":3.4329999999999998,"15min":5.2869999999999999},"min":{"1min":0.23000000000000001,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":2.7250000000000001,"5min":90.944000000000003,"15min":162.22999999999999},"last":0.25},{"interface":"front","average":{"1min":0.33300000000000002,"5min":3.1850000000000001,"15min":5.4160000000000004},"min":{"1min":0.23100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.45300000000000001,"5min":83.341999999999999,"15min":162.70699999999999},"last":0.41299999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":1.8740000000000001,"15min":2.2029999999999998},"min":{"1min":0.251,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":0.54900000000000004,"5min":14.254,"15min":36.472999999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.90700000000000003,"15min":2.4910000000000001},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":0.59999999999999998,"5min":7.6749999999999998,"15min":78.594999999999999},"last":0.36799999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":1.8859999999999999,"15min":1.875},"min":{"1min":0.32600000000000001,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":0.65300000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.42799999999999999,"5min":2.2189999999999999,"15min":2.1469999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":0.64300000000000002,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.35499999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":1.5589999999999999,"15min":2.581},"min":{"1min":0.317,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3099999999999996,"5min":10.919,"15min":28.117999999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.4279999999999999,"15min":2.6800000000000002},"min":{"1min":0.36699999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.512,"5min":10.798999999999999,"15min":28.34},"last":0.33600000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:43:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":2.165,"15min":2.145},"min":{"1min":0.34200000000000003,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":2.5430000000000001,"5min":73.887,"15min":73.887},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.63,"5min":2.4159999999999999,"15min":2.1379999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":2.6000000000000001,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.42999999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":1.7889999999999999,"15min":2.3090000000000002},"min":{"1min":0.39400000000000002,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":4.1580000000000004,"5min":13.085000000000001,"15min":52.006},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.306,"15min":2.032},"min":{"1min":0.41899999999999998,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":4.1779999999999999,"5min":8.3460000000000001,"15min":51.890999999999998},"last":0.47699999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019405,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4161716,"kb_used_data":3263340,"kb_used_omap":0,"kb_used_meta":898368,"kb_avail":89583436,"statfs":{"total":95995035648,"available":91733438464,"internally_reserved":0,"allocated":3341660160,"data_stored":3340726286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919928832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":2.3570000000000002,"15min":4.25},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.47299999999999998,"5min":79.837000000000003,"15min":155.315},"last":0.312},{"interface":"front","average":{"1min":0.44900000000000001,"5min":2.4300000000000002,"15min":4.7039999999999997},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":0.63700000000000001,"5min":78.983000000000004,"15min":155.334},"last":0.28499999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:42:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":6.8879999999999999,"15min":6.556},"min":{"1min":0.252,"5min":0.224,"15min":0.224},"max":{"1min":2.8889999999999998,"5min":146.929,"15min":190.61199999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.45500000000000002,"5min":6.8300000000000001,"15min":7.266},"min":{"1min":0.27500000000000002,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":0.624,"5min":146.91,"15min":253.273},"last":0.29499999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.05,"15min":2.444},"min":{"1min":0.34200000000000003,"5min":0.31,"15min":0.28299999999999997},"max":{"1min":2.9510000000000001,"5min":10.161,"15min":170.45400000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.0029999999999999,"15min":2.1739999999999999},"min":{"1min":0.314,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":3.0219999999999998,"5min":5.9740000000000002,"15min":58.771000000000001},"last":0.45900000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:42:58 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":1.1799999999999999,"15min":2.7519999999999998},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.67500000000000004,"5min":7.5830000000000002,"15min":140.881},"last":1.4610000000000001},{"interface":"front","average":{"1min":0.47099999999999997,"5min":1.208,"15min":2.976},"min":{"1min":0.33300000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.69599999999999995,"5min":9.3140000000000001,"15min":140.911},"last":0.439}]},{"osd":5,"last update":"Sun Oct 1 09:43:33 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":2.0569999999999999,"15min":1.5580000000000001},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":3.052,"5min":110.003,"15min":110.003},"last":0.378},{"interface":"front","average":{"1min":0.64900000000000002,"5min":2.0110000000000001,"15min":1.399},"min":{"1min":0.33500000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.0049999999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.39200000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:43:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.79200000000000004,"15min":0.99299999999999999},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":0.80900000000000005,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.95199999999999996,"15min":1.085},"min":{"1min":0.379,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":2.9239999999999999,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.48099999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.71599999999999997,"15min":1.161},"min":{"1min":0.313,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":0.69399999999999995,"5min":4.7030000000000003,"15min":38.034999999999997},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.026,"15min":1.4410000000000001},"min":{"1min":0.36699999999999999,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":2.9940000000000002,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.496}]}]},{"osd":5,"up_from":379,"seq":1627792605312,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2059616,"kb_used_data":1695064,"kb_used_omap":13,"kb_used_meta":364530,"kb_avail":91685536,"statfs":{"total":95995035648,"available":93885988864,"internally_reserved":0,"allocated":1735745536,"data_stored":1734827566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":373279218},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":1.6479999999999999,"15min":2.1890000000000001},"min":{"1min":0.22900000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.1880000000000002,"5min":12.571999999999999,"15min":84.340999999999994},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.40400000000000003,"5min":1.2549999999999999,"15min":2.0800000000000001},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.53900000000000003,"5min":8.6460000000000008,"15min":84.356999999999999},"last":0.42499999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":6.4870000000000001,"15min":5.7599999999999998},"min":{"1min":0.26000000000000001,"5min":0.25900000000000001,"15min":0.255},"max":{"1min":0.76900000000000002,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":6.8099999999999996,"15min":5.7320000000000002},"min":{"1min":0.32200000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":2.258,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.39800000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":1.3600000000000001,"15min":1.5980000000000001},"min":{"1min":0.315,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.752,"5min":8.7040000000000006,"15min":14.199999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.44800000000000001,"5min":1.306,"15min":1.583},"min":{"1min":0.32900000000000001,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":0.85999999999999999,"5min":8.2370000000000001,"15min":14.430999999999999},"last":0.52000000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.5069999999999999,"15min":1.488},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.2280000000000002,"5min":12.734,"15min":14.004},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.5640000000000001,"15min":1.629},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.3979999999999999,"5min":11.766,"15min":13.991},"last":0.47099999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":2.2029999999999998,"15min":1.4330000000000001},"min":{"1min":0.30099999999999999,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.66800000000000004,"5min":122.971,"15min":122.971},"last":0.5},{"interface":"front","average":{"1min":0.438,"5min":2.25,"15min":1.724},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.73799999999999999,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.41299999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":1.165,"15min":1.044},"min":{"1min":0.312,"5min":0.311,"15min":0.28299999999999997},"max":{"1min":2.2839999999999998,"5min":12.545,"15min":12.545},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.99099999999999999,"15min":1.302},"min":{"1min":0.378,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":2.629,"5min":6.4729999999999999,"15min":14.193},"last":0.54500000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.1140000000000001,"15min":1.262},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.81200000000000006,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.2270000000000001,"15min":1.0780000000000001},"min":{"1min":0.36299999999999999,"5min":0.35299999999999998,"15min":0.318},"max":{"1min":2.3079999999999998,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.53600000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3397718016,"data_stored":3397713944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":818348032,"data_stored":818348032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":923336704,"data_stored":923336704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3331985408,"data_stored":3331981336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1730805760,"data_stored":1730805760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1906380800,"data_stored":1906376728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1358364672,"data_stored":1358364672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:44:03.359 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:03.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:03.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:03.425 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:44:03.463 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:03.497 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:03.567 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:03.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:03.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:03.581 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:44:03.612 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:03 smithi138 ceph-mon[192513]: from='client.46943 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:03.612 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:03 smithi138 ceph-mon[192513]: pgmap v1527: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 7684/198156 objects degraded (3.878%); 4041/198156 objects misplaced (2.039%); 577 KiB/s, 9 objects/s recovering 2023-10-01T09:44:03.612 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:03 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7684/198156 objects degraded (3.878%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:03.613 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:03 smithi138 ceph-mon[185886]: from='client.46943 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:03.613 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:03 smithi138 ceph-mon[185886]: pgmap v1527: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 7684/198156 objects degraded (3.878%); 4041/198156 objects misplaced (2.039%); 577 KiB/s, 9 objects/s recovering 2023-10-01T09:44:03.613 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:03 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7684/198156 objects degraded (3.878%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:03.671 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:03.701 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:03.704 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:03 smithi190 ceph-mon[160435]: from='client.46943 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:03.704 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:03 smithi190 ceph-mon[160435]: pgmap v1527: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 7684/198156 objects degraded (3.878%); 4041/198156 objects misplaced (2.039%); 577 KiB/s, 9 objects/s recovering 2023-10-01T09:44:03.704 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:03 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7684/198156 objects degraded (3.878%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:03.759 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:03.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:03.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:03.843 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:44:03.863 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:44:03.866 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:03.890 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:44:03.898 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:03.911 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:44:03.935 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:44:03.946 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:03.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:03.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:03.961 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:44:03.983 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:44:03.993 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:04.001 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:44:04.003 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:44:04.003 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:44:04.003 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:44:04.003 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:44:04.003 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:44:04.003 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:44:04.003 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:44:04.004 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:44:04.050 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:04.084 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:04.131 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:04.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:04.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:04.236 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:04.267 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:04.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3279148942' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:04.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3279148942' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:04.805 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3279148942' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:04.917 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:04.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:04.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:05.021 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:05.046 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:44:05.047 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:44:05.050 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:05.197 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:44:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:44:04] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.33.4" 2023-10-01T09:44:05.201 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:05.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:05.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:05.305 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:05.337 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:05.383 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:05.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:05.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:05.486 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:05.516 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:05.722 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:05 smithi138 ceph-mon[192513]: pgmap v1528: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 119 op/s; 7684/198021 objects degraded (3.880%); 4041/198021 objects misplaced (2.041%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:44:05.722 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:05 smithi138 ceph-mon[185886]: pgmap v1528: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 119 op/s; 7684/198021 objects degraded (3.880%); 4041/198021 objects misplaced (2.041%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:44:05.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:05 smithi190 ceph-mon[160435]: pgmap v1528: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 119 op/s; 7684/198021 objects degraded (3.880%); 4041/198021 objects misplaced (2.041%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:44:05.841 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:44:05.842 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:05.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:05.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:05.946 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:05.976 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:06.078 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:06.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:06.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:06.182 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:06.183 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:44:06.220 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:06.269 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:06.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:06.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:06.304 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:44:06.376 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:06.405 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:06.452 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:06.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:06.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:06.478 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:44:06.511 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:06 smithi190 ceph-mon[160435]: pgmap v1529: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 7684/197016 objects degraded (3.900%); 4041/197016 objects misplaced (2.051%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:44:06.511 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:06 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1201273447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:06.511 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:06 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:06.555 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:06.582 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:06.687 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:06.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:06.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:06.790 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:06.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:06 smithi138 ceph-mon[185886]: pgmap v1529: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 7684/197016 objects degraded (3.900%); 4041/197016 objects misplaced (2.051%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:44:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:06 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1201273447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:06 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:06 smithi138 ceph-mon[192513]: pgmap v1529: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 7684/197016 objects degraded (3.900%); 4041/197016 objects misplaced (2.051%); 569 KiB/s, 8 objects/s recovering 2023-10-01T09:44:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:06 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1201273447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:06 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:06.824 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:06.892 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:44:06.932 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:06.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:06.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:07.036 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:07.062 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:07.193 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:07.202 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:07.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:07.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:07.310 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:07.328 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:07.349 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:07.467 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:07.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:07.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:07.575 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:07.601 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:07 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:07.601 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:07 smithi190 ceph-mon[160435]: failed to encode map e527 with expected crc 2023-10-01T09:44:07.601 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:07 smithi190 ceph-mon[160435]: failed to encode map e527 with expected crc 2023-10-01T09:44:07.601 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:07 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:44:07.602 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:07 smithi190 ceph-mon[160435]: osdmap e527: 8 total, 8 up, 7 in 2023-10-01T09:44:07.604 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:07.706 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:07.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:07.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:07.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:07 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:07.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:07 smithi138 ceph-mon[192513]: failed to encode map e527 with expected crc 2023-10-01T09:44:07.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:07 smithi138 ceph-mon[192513]: failed to encode map e527 with expected crc 2023-10-01T09:44:07.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:07 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:44:07.713 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:07 smithi138 ceph-mon[192513]: osdmap e527: 8 total, 8 up, 7 in 2023-10-01T09:44:07.713 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:07 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:07.713 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:07 smithi138 ceph-mon[185886]: failed to encode map e527 with expected crc 2023-10-01T09:44:07.713 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:07 smithi138 ceph-mon[185886]: failed to encode map e527 with expected crc 2023-10-01T09:44:07.714 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:07 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:44:07.714 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:07 smithi138 ceph-mon[185886]: osdmap e527: 8 total, 8 up, 7 in 2023-10-01T09:44:07.813 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:07.847 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:07.897 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:07.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:07.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:07.936 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:44:07.936 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:44:08.001 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:08.034 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:08.693 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:44:08.765 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:08 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7684/197016 objects degraded (3.900%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:08.765 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:08 smithi190 ceph-mon[160435]: pgmap v1531: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 7637/195801 objects degraded (3.900%); 4005/195801 objects misplaced (2.045%); 664 KiB/s, 10 objects/s recovering 2023-10-01T09:44:08.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:08 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7684/197016 objects degraded (3.900%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:08.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:08 smithi138 ceph-mon[192513]: pgmap v1531: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 7637/195801 objects degraded (3.900%); 4005/195801 objects misplaced (2.045%); 664 KiB/s, 10 objects/s recovering 2023-10-01T09:44:08.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:08 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7684/197016 objects degraded (3.900%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:08.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:08 smithi138 ceph-mon[185886]: pgmap v1531: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 7637/195801 objects degraded (3.900%); 4005/195801 objects misplaced (2.045%); 664 KiB/s, 10 objects/s recovering 2023-10-01T09:44:08.993 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:09.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:09.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:09.108 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:09.120 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:44:09.144 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:09.195 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:09.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:09.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:09.220 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:44:09.298 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:09.334 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:09.443 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:09.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:09.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:09.548 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:09.586 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:09.694 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:09.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:09.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:09.731 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:44:09.738 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:44:09.765 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:44:09.773 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:09.782 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:09 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1576971980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:09.787 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:44:09.799 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:09.812 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:44:09.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:09 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1576971980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:09.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:09 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1576971980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:09.829 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:09.836 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:44:09.857 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:44:09.877 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:44:09.883 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:09.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:09.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:09.897 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:44:09.916 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:44:09.917 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:44:09.917 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:44:09.917 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:44:09.917 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:44:09.917 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:44:09.917 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:44:09.918 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:44:09.918 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:44:09.988 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:10.019 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:10.215 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:10.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:10.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:10.318 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:10.347 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:10.688 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:44:10.688 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:44:10.688 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:44:10.688 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:44:10.689 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:44:10.707 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1531,"stamp":"2023-10-01T09:44:08.318239+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4280227787,"num_objects":65267,"num_object_clones":0,"num_object_copies":195801,"num_objects_missing_on_primary":2538,"num_objects_missing":2538,"num_objects_degraded":7637,"num_objects_misplaced":4005,"num_objects_unfound":0,"num_objects_dirty":65267,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":102307,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8566,"num_bytes_recovered":548242447,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102371,"ondisk_log_size":102371,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17990436,"kb_used_data":12965804,"kb_used_omap":39,"kb_used_meta":5024536,"kb_avail":638225628,"statfs":{"total":671965249536,"available":653543043072,"internally_reserved":0,"allocated":13276983296,"data_stored":13267133560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5145125487},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,1,0,0,5,2,8],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-80674816,"num_objects":-1231,"num_object_clones":0,"num_object_copies":-3693,"num_objects_missing_on_primary":-84,"num_objects_missing":-84,"num_objects_degraded":-97,"num_objects_misplaced":-70,"num_objects_unfound":0,"num_objects_dirty":-1231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1231,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":83,"num_bytes_recovered":5439488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002029"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":302,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.454170+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:43:52.454170+0000","last_peered":"2023-10-01T09:43:52.454170+0000","last_clean":"2023-10-01T09:43:52.454170+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:43:52.454170+0000","last_undegraded":"2023-10-01T09:43:52.454170+0000","last_fullsized":"2023-10-01T09:43:52.454170+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":145,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.456835+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:43:52.456835+0000","last_peered":"2023-10-01T09:43:52.456835+0000","last_clean":"2023-10-01T09:43:52.456835+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:43:52.456835+0000","last_undegraded":"2023-10-01T09:43:52.456835+0000","last_fullsized":"2023-10-01T09:43:52.456835+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":526,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.456868+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:43:52.456868+0000","last_peered":"2023-10-01T09:43:52.456868+0000","last_clean":"2023-10-01T09:43:52.456868+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:43:52.456868+0000","last_undegraded":"2023-10-01T09:43:52.456868+0000","last_fullsized":"2023-10-01T09:43:52.456868+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":585,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.461987+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:44:00.461987+0000","last_peered":"2023-10-01T09:44:00.461987+0000","last_clean":"2023-10-01T09:44:00.461987+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:44:00.461987+0000","last_undegraded":"2023-10-01T09:44:00.461987+0000","last_fullsized":"2023-10-01T09:44:00.461987+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":466,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.461528+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:44:00.461528+0000","last_peered":"2023-10-01T09:44:00.461528+0000","last_clean":"2023-10-01T09:44:00.461528+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:44:00.461528+0000","last_undegraded":"2023-10-01T09:44:00.461528+0000","last_fullsized":"2023-10-01T09:44:00.461528+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":208,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.562955+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:44:00.562955+0000","last_peered":"2023-10-01T09:44:00.562955+0000","last_clean":"2023-10-01T09:44:00.562955+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:44:00.562955+0000","last_undegraded":"2023-10-01T09:44:00.562955+0000","last_fullsized":"2023-10-01T09:44:00.562955+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":583,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.545164+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:43:52.545164+0000","last_peered":"2023-10-01T09:43:52.545164+0000","last_clean":"2023-10-01T09:43:52.545164+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:43:52.545164+0000","last_undegraded":"2023-10-01T09:43:52.545164+0000","last_fullsized":"2023-10-01T09:43:52.545164+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":513,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.478430+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:44:00.478430+0000","last_peered":"2023-10-01T09:44:00.478430+0000","last_clean":"2023-10-01T09:44:00.478430+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:44:00.478430+0000","last_undegraded":"2023-10-01T09:44:00.478430+0000","last_fullsized":"2023-10-01T09:44:00.478430+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":239,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.466987+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:44:00.466987+0000","last_peered":"2023-10-01T09:44:00.466987+0000","last_clean":"2023-10-01T09:44:00.466987+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:44:00.466987+0000","last_undegraded":"2023-10-01T09:44:00.466987+0000","last_fullsized":"2023-10-01T09:44:00.466987+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":45,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.469523+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:43:52.469523+0000","last_peered":"2023-10-01T09:43:52.469523+0000","last_clean":"2023-10-01T09:43:52.469523+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:43:52.469523+0000","last_undegraded":"2023-10-01T09:43:52.469523+0000","last_fullsized":"2023-10-01T09:43:52.469523+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":51,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.468710+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:43:52.468710+0000","last_peered":"2023-10-01T09:43:52.468710+0000","last_clean":"2023-10-01T09:43:52.468710+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:43:52.468710+0000","last_undegraded":"2023-10-01T09:43:52.468710+0000","last_fullsized":"2023-10-01T09:43:52.468710+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":49,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.457591+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:43:52.457591+0000","last_peered":"2023-10-01T09:43:52.457591+0000","last_clean":"2023-10-01T09:43:52.457591+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:43:52.457591+0000","last_undegraded":"2023-10-01T09:43:52.457591+0000","last_fullsized":"2023-10-01T09:43:52.457591+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":427,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.455016+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:43:52.455016+0000","last_peered":"2023-10-01T09:43:52.455016+0000","last_clean":"2023-10-01T09:43:52.455016+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:43:52.455016+0000","last_undegraded":"2023-10-01T09:43:52.455016+0000","last_fullsized":"2023-10-01T09:43:52.455016+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":308,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.454406+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:43:52.454406+0000","last_peered":"2023-10-01T09:43:52.454406+0000","last_clean":"2023-10-01T09:43:52.454406+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:43:52.454406+0000","last_undegraded":"2023-10-01T09:43:52.454406+0000","last_fullsized":"2023-10-01T09:43:52.454406+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":644,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.467929+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:43:52.467929+0000","last_peered":"2023-10-01T09:43:52.467929+0000","last_clean":"2023-10-01T09:43:52.467929+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:43:52.467929+0000","last_undegraded":"2023-10-01T09:43:52.467929+0000","last_fullsized":"2023-10-01T09:43:52.467929+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":381,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.468589+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:43:52.468589+0000","last_peered":"2023-10-01T09:43:52.468589+0000","last_clean":"2023-10-01T09:43:52.468589+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:43:52.468589+0000","last_undegraded":"2023-10-01T09:43:52.468589+0000","last_fullsized":"2023-10-01T09:43:52.468589+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":54,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.468643+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:43:52.468643+0000","last_peered":"2023-10-01T09:43:52.468643+0000","last_clean":"2023-10-01T09:43:52.468643+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:43:52.468643+0000","last_undegraded":"2023-10-01T09:43:52.468643+0000","last_fullsized":"2023-10-01T09:43:52.468643+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":41,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.466580+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:44:00.466580+0000","last_peered":"2023-10-01T09:44:00.466580+0000","last_clean":"2023-10-01T09:44:00.466580+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:44:00.466580+0000","last_undegraded":"2023-10-01T09:44:00.466580+0000","last_fullsized":"2023-10-01T09:44:00.466580+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":537,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.470227+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:43:52.470227+0000","last_peered":"2023-10-01T09:43:52.470227+0000","last_clean":"2023-10-01T09:43:52.470227+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:43:52.470227+0000","last_undegraded":"2023-10-01T09:43:52.470227+0000","last_fullsized":"2023-10-01T09:43:52.470227+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":549,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.462068+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:44:00.462068+0000","last_peered":"2023-10-01T09:44:00.462068+0000","last_clean":"2023-10-01T09:44:00.462068+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:44:00.462068+0000","last_undegraded":"2023-10-01T09:44:00.462068+0000","last_fullsized":"2023-10-01T09:44:00.462068+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":405,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.461485+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:44:00.461485+0000","last_peered":"2023-10-01T09:44:00.461485+0000","last_clean":"2023-10-01T09:44:00.461485+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:44:00.461485+0000","last_undegraded":"2023-10-01T09:44:00.461485+0000","last_fullsized":"2023-10-01T09:44:00.461485+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":54,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.468378+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:43:52.468378+0000","last_peered":"2023-10-01T09:43:52.468378+0000","last_clean":"2023-10-01T09:43:52.468378+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:43:52.468378+0000","last_undegraded":"2023-10-01T09:43:52.468378+0000","last_fullsized":"2023-10-01T09:43:52.468378+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"526'6351","reported_seq":6411,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.315233+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:44:06.315233+0000","last_peered":"2023-10-01T09:44:06.315233+0000","last_clean":"2023-10-01T09:44:06.315233+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:44:06.315233+0000","last_undegraded":"2023-10-01T09:44:06.315233+0000","last_fullsized":"2023-10-01T09:44:06.315233+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6351,"log_dups_size":0,"ondisk_log_size":6351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264830976,"num_objects":4041,"num_object_clones":0,"num_object_copies":12123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6351,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":49,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.468228+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:43:52.468228+0000","last_peered":"2023-10-01T09:43:52.468228+0000","last_clean":"2023-10-01T09:43:52.468228+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:43:52.468228+0000","last_undegraded":"2023-10-01T09:43:52.468228+0000","last_fullsized":"2023-10-01T09:43:52.468228+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":247,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.462928+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:44:00.462928+0000","last_peered":"2023-10-01T09:44:00.462928+0000","last_clean":"2023-10-01T09:44:00.462928+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:44:00.462928+0000","last_undegraded":"2023-10-01T09:44:00.462928+0000","last_fullsized":"2023-10-01T09:44:00.462928+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"526'6347","reported_seq":9872,"reported_epoch":526,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:44:06.525071+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:44:06.525071+0000","last_peered":"2023-10-01T09:44:06.525071+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:44:06.525071+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6347,"log_dups_size":0,"ondisk_log_size":6347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266010624,"num_objects":4059,"num_object_clones":0,"num_object_copies":12177,"num_objects_missing_on_primary":2538,"num_objects_missing":2538,"num_objects_degraded":6528,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6347,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1145,"num_bytes_recovered":75038720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1452},{"shards":"0,4,6","objects":2538},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":520,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.468193+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:43:52.468193+0000","last_peered":"2023-10-01T09:43:52.468193+0000","last_clean":"2023-10-01T09:43:52.468193+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:43:52.468193+0000","last_undegraded":"2023-10-01T09:43:52.468193+0000","last_fullsized":"2023-10-01T09:43:52.468193+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":581,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.456783+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:43:52.456783+0000","last_peered":"2023-10-01T09:43:52.456783+0000","last_clean":"2023-10-01T09:43:52.456783+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:43:52.456783+0000","last_undegraded":"2023-10-01T09:43:52.456783+0000","last_fullsized":"2023-10-01T09:43:52.456783+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"527'6370","reported_seq":6432,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.780790+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:44:06.780790+0000","last_peered":"2023-10-01T09:44:06.780790+0000","last_clean":"2023-10-01T09:44:06.780790+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:44:06.780790+0000","last_undegraded":"2023-10-01T09:44:06.780790+0000","last_fullsized":"2023-10-01T09:44:06.780790+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6370,"log_dups_size":0,"ondisk_log_size":6370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266207232,"num_objects":4062,"num_object_clones":0,"num_object_copies":12186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6370,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":606,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.468745+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:43:52.468745+0000","last_peered":"2023-10-01T09:43:52.468745+0000","last_clean":"2023-10-01T09:43:52.468745+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:43:52.468745+0000","last_undegraded":"2023-10-01T09:43:52.468745+0000","last_fullsized":"2023-10-01T09:43:52.468745+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":402,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.485836+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:44:00.485836+0000","last_peered":"2023-10-01T09:44:00.485836+0000","last_clean":"2023-10-01T09:44:00.485836+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:44:00.485836+0000","last_undegraded":"2023-10-01T09:44:00.485836+0000","last_fullsized":"2023-10-01T09:44:00.485836+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"527'6318","reported_seq":12985,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.788604+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:44:06.788604+0000","last_peered":"2023-10-01T09:44:06.788604+0000","last_clean":"2023-10-01T09:44:06.788604+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:44:06.788604+0000","last_undegraded":"2023-10-01T09:44:06.788604+0000","last_fullsized":"2023-10-01T09:44:06.788604+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6318,"log_dups_size":0,"ondisk_log_size":6318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267649024,"num_objects":4084,"num_object_clones":0,"num_object_copies":12252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6318,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":247,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.466617+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:44:00.466617+0000","last_peered":"2023-10-01T09:44:00.466617+0000","last_clean":"2023-10-01T09:44:00.466617+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:44:00.466617+0000","last_undegraded":"2023-10-01T09:44:00.466617+0000","last_fullsized":"2023-10-01T09:44:00.466617+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":505,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.461758+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:44:00.461758+0000","last_peered":"2023-10-01T09:44:00.461758+0000","last_clean":"2023-10-01T09:44:00.461758+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:44:00.461758+0000","last_undegraded":"2023-10-01T09:44:00.461758+0000","last_fullsized":"2023-10-01T09:44:00.461758+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"526'6335","reported_seq":6396,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:05.081438+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:44:05.081438+0000","last_peered":"2023-10-01T09:44:05.081438+0000","last_clean":"2023-10-01T09:44:05.081438+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:44:05.081438+0000","last_undegraded":"2023-10-01T09:44:05.081438+0000","last_fullsized":"2023-10-01T09:44:05.081438+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6335,"log_dups_size":0,"ondisk_log_size":6335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273481728,"num_objects":4173,"num_object_clones":0,"num_object_copies":12519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6335,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":106,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.454710+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:43:52.454710+0000","last_peered":"2023-10-01T09:43:52.454710+0000","last_clean":"2023-10-01T09:43:52.454710+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:43:52.454710+0000","last_undegraded":"2023-10-01T09:43:52.454710+0000","last_fullsized":"2023-10-01T09:43:52.454710+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":180,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.455148+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:43:52.455148+0000","last_peered":"2023-10-01T09:43:52.455148+0000","last_clean":"2023-10-01T09:43:52.455148+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:43:52.455148+0000","last_undegraded":"2023-10-01T09:43:52.455148+0000","last_fullsized":"2023-10-01T09:43:52.455148+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"526'6299","reported_seq":6401,"reported_epoch":526,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:44:06.150557+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:44:06.150557+0000","last_peered":"2023-10-01T09:44:06.150557+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:44:06.150557+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6299,"log_dups_size":0,"ondisk_log_size":6299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266665984,"num_objects":4069,"num_object_clones":0,"num_object_copies":12207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1099,"num_objects_misplaced":4005,"num_objects_unfound":0,"num_objects_dirty":4069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6299,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":1114112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4005},{"shards":"4,5","objects":-1035}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":424,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.454302+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:43:52.454302+0000","last_peered":"2023-10-01T09:43:52.454302+0000","last_clean":"2023-10-01T09:43:52.454302+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:43:52.454302+0000","last_undegraded":"2023-10-01T09:43:52.454302+0000","last_fullsized":"2023-10-01T09:43:52.454302+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":584,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.456923+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:43:52.456923+0000","last_peered":"2023-10-01T09:43:52.456923+0000","last_clean":"2023-10-01T09:43:52.456923+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:43:52.456923+0000","last_undegraded":"2023-10-01T09:43:52.456923+0000","last_fullsized":"2023-10-01T09:43:52.456923+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"527'6301","reported_seq":6370,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:07.429583+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:44:07.429583+0000","last_peered":"2023-10-01T09:44:07.429583+0000","last_clean":"2023-10-01T09:44:07.429583+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:44:07.429583+0000","last_undegraded":"2023-10-01T09:44:07.429583+0000","last_fullsized":"2023-10-01T09:44:07.429583+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6301,"log_dups_size":0,"ondisk_log_size":6301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267190272,"num_objects":4077,"num_object_clones":0,"num_object_copies":12231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6301,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":420,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.454118+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:43:52.454118+0000","last_peered":"2023-10-01T09:43:52.454118+0000","last_clean":"2023-10-01T09:43:52.454118+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:43:52.454118+0000","last_undegraded":"2023-10-01T09:43:52.454118+0000","last_fullsized":"2023-10-01T09:43:52.454118+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":261,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.455561+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:43:52.455561+0000","last_peered":"2023-10-01T09:43:52.455561+0000","last_clean":"2023-10-01T09:43:52.455561+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:43:52.455561+0000","last_undegraded":"2023-10-01T09:43:52.455561+0000","last_fullsized":"2023-10-01T09:43:52.455561+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"526'6398","reported_seq":6457,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:05.482784+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:44:05.482784+0000","last_peered":"2023-10-01T09:44:05.482784+0000","last_clean":"2023-10-01T09:44:05.482784+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:44:05.482784+0000","last_undegraded":"2023-10-01T09:44:05.482784+0000","last_fullsized":"2023-10-01T09:44:05.482784+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6398,"log_dups_size":0,"ondisk_log_size":6398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268697600,"num_objects":4100,"num_object_clones":0,"num_object_copies":12300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6398,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":389,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.456772+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:43:52.456772+0000","last_peered":"2023-10-01T09:43:52.456772+0000","last_clean":"2023-10-01T09:43:52.456772+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:43:52.456772+0000","last_undegraded":"2023-10-01T09:43:52.456772+0000","last_fullsized":"2023-10-01T09:43:52.456772+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":436,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.457480+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:43:52.457480+0000","last_peered":"2023-10-01T09:43:52.457480+0000","last_clean":"2023-10-01T09:43:52.457480+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:43:52.457480+0000","last_undegraded":"2023-10-01T09:43:52.457480+0000","last_fullsized":"2023-10-01T09:43:52.457480+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"526'6224","reported_seq":6283,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.155095+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:44:06.155095+0000","last_peered":"2023-10-01T09:44:06.155095+0000","last_clean":"2023-10-01T09:44:06.155095+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:44:06.155095+0000","last_undegraded":"2023-10-01T09:44:06.155095+0000","last_fullsized":"2023-10-01T09:44:06.155095+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6224,"log_dups_size":0,"ondisk_log_size":6224,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259129344,"num_objects":3954,"num_object_clones":0,"num_object_copies":11862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6224,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":919,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.468406+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:43:52.468406+0000","last_peered":"2023-10-01T09:43:52.468406+0000","last_clean":"2023-10-01T09:43:52.468406+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:43:52.468406+0000","last_undegraded":"2023-10-01T09:43:52.468406+0000","last_fullsized":"2023-10-01T09:43:52.468406+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":257,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.466713+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:44:00.466713+0000","last_peered":"2023-10-01T09:44:00.466713+0000","last_clean":"2023-10-01T09:44:00.466713+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:44:00.466713+0000","last_undegraded":"2023-10-01T09:44:00.466713+0000","last_fullsized":"2023-10-01T09:44:00.466713+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":49,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.457628+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:43:52.457628+0000","last_peered":"2023-10-01T09:43:52.457628+0000","last_clean":"2023-10-01T09:43:52.457628+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:43:52.457628+0000","last_undegraded":"2023-10-01T09:43:52.457628+0000","last_fullsized":"2023-10-01T09:43:52.457628+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"527'6260","reported_seq":6322,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.473886+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:44:06.473886+0000","last_peered":"2023-10-01T09:44:06.473886+0000","last_clean":"2023-10-01T09:44:06.473886+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:44:06.473886+0000","last_undegraded":"2023-10-01T09:44:06.473886+0000","last_fullsized":"2023-10-01T09:44:06.473886+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6260,"log_dups_size":0,"ondisk_log_size":6260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269352960,"num_objects":4110,"num_object_clones":0,"num_object_copies":12330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6260,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1192,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.463079+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:44:00.463079+0000","last_peered":"2023-10-01T09:44:00.463079+0000","last_clean":"2023-10-01T09:44:00.463079+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:44:00.463079+0000","last_undegraded":"2023-10-01T09:44:00.463079+0000","last_fullsized":"2023-10-01T09:44:00.463079+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":401,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.454617+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:43:52.454617+0000","last_peered":"2023-10-01T09:43:52.454617+0000","last_clean":"2023-10-01T09:43:52.454617+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:43:52.454617+0000","last_undegraded":"2023-10-01T09:43:52.454617+0000","last_fullsized":"2023-10-01T09:43:52.454617+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":230,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.466737+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:44:00.466737+0000","last_peered":"2023-10-01T09:44:00.466737+0000","last_clean":"2023-10-01T09:44:00.466737+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:44:00.466737+0000","last_undegraded":"2023-10-01T09:44:00.466737+0000","last_fullsized":"2023-10-01T09:44:00.466737+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"526'6337","reported_seq":13618,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.415045+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:44:06.415045+0000","last_peered":"2023-10-01T09:44:06.415045+0000","last_clean":"2023-10-01T09:44:06.415045+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:44:06.415045+0000","last_undegraded":"2023-10-01T09:44:06.415045+0000","last_fullsized":"2023-10-01T09:44:06.415045+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6337,"log_dups_size":0,"ondisk_log_size":6337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264699904,"num_objects":4039,"num_object_clones":0,"num_object_copies":12117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6337,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":872,"reported_epoch":526,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:44:00.461355+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:44:00.461355+0000","last_peered":"2023-10-01T09:44:00.461355+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:44:00.461355+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:44:00.461355+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":588,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.456724+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:43:52.456724+0000","last_peered":"2023-10-01T09:43:52.456724+0000","last_clean":"2023-10-01T09:43:52.456724+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:43:52.456724+0000","last_undegraded":"2023-10-01T09:43:52.456724+0000","last_fullsized":"2023-10-01T09:43:52.456724+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":49,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.457404+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:43:52.457404+0000","last_peered":"2023-10-01T09:43:52.457404+0000","last_clean":"2023-10-01T09:43:52.457404+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:43:52.457404+0000","last_undegraded":"2023-10-01T09:43:52.457404+0000","last_fullsized":"2023-10-01T09:43:52.457404+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"526'6210","reported_seq":6271,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:05.053546+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:44:05.053546+0000","last_peered":"2023-10-01T09:44:05.053546+0000","last_clean":"2023-10-01T09:44:05.053546+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:44:05.053546+0000","last_undegraded":"2023-10-01T09:44:05.053546+0000","last_fullsized":"2023-10-01T09:44:05.053546+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6210,"log_dups_size":0,"ondisk_log_size":6210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266207232,"num_objects":4062,"num_object_clones":0,"num_object_copies":12186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6210,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1643,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.466482+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:44:00.466482+0000","last_peered":"2023-10-01T09:44:00.466482+0000","last_clean":"2023-10-01T09:44:00.466482+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:44:00.466482+0000","last_undegraded":"2023-10-01T09:44:00.466482+0000","last_fullsized":"2023-10-01T09:44:00.466482+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":556,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.467761+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:43:52.467761+0000","last_peered":"2023-10-01T09:43:52.467761+0000","last_clean":"2023-10-01T09:43:52.467761+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:43:52.467761+0000","last_undegraded":"2023-10-01T09:43:52.467761+0000","last_fullsized":"2023-10-01T09:43:52.467761+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":335,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.463156+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:44:00.463156+0000","last_peered":"2023-10-01T09:44:00.463156+0000","last_clean":"2023-10-01T09:44:00.463156+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:44:00.463156+0000","last_undegraded":"2023-10-01T09:44:00.463156+0000","last_fullsized":"2023-10-01T09:44:00.463156+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":324,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.454498+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:43:52.454498+0000","last_peered":"2023-10-01T09:43:52.454498+0000","last_clean":"2023-10-01T09:43:52.454498+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:43:52.454498+0000","last_undegraded":"2023-10-01T09:43:52.454498+0000","last_fullsized":"2023-10-01T09:43:52.454498+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"526'6395","reported_seq":6454,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.182458+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:44:06.182458+0000","last_peered":"2023-10-01T09:44:06.182458+0000","last_clean":"2023-10-01T09:44:06.182458+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:44:06.182458+0000","last_undegraded":"2023-10-01T09:44:06.182458+0000","last_fullsized":"2023-10-01T09:44:06.182458+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6395,"log_dups_size":0,"ondisk_log_size":6395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271122432,"num_objects":4137,"num_object_clones":0,"num_object_copies":12411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6395,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":733,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.467915+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:43:52.467915+0000","last_peered":"2023-10-01T09:43:52.467915+0000","last_clean":"2023-10-01T09:43:52.467915+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:43:52.467915+0000","last_undegraded":"2023-10-01T09:43:52.467915+0000","last_fullsized":"2023-10-01T09:43:52.467915+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":566,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.461598+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:44:00.461598+0000","last_peered":"2023-10-01T09:44:00.461598+0000","last_clean":"2023-10-01T09:44:00.461598+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:44:00.461598+0000","last_undegraded":"2023-10-01T09:44:00.461598+0000","last_fullsized":"2023-10-01T09:44:00.461598+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":566,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.461781+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:44:00.461781+0000","last_peered":"2023-10-01T09:44:00.461781+0000","last_clean":"2023-10-01T09:44:00.461781+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:44:00.461781+0000","last_undegraded":"2023-10-01T09:44:00.461781+0000","last_fullsized":"2023-10-01T09:44:00.461781+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"527'6343","reported_seq":6406,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.474649+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:44:06.474649+0000","last_peered":"2023-10-01T09:44:06.474649+0000","last_clean":"2023-10-01T09:44:06.474649+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:44:06.474649+0000","last_undegraded":"2023-10-01T09:44:06.474649+0000","last_fullsized":"2023-10-01T09:44:06.474649+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6343,"log_dups_size":0,"ondisk_log_size":6343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269746200,"num_objects":4117,"num_object_clones":0,"num_object_copies":12351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4117,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6343,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":964,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.468791+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:43:52.468791+0000","last_peered":"2023-10-01T09:43:52.468791+0000","last_clean":"2023-10-01T09:43:52.468791+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:43:52.468791+0000","last_undegraded":"2023-10-01T09:43:52.468791+0000","last_fullsized":"2023-10-01T09:43:52.468791+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":584,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.466682+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:44:00.466682+0000","last_peered":"2023-10-01T09:44:00.466682+0000","last_clean":"2023-10-01T09:44:00.466682+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:44:00.466682+0000","last_undegraded":"2023-10-01T09:44:00.466682+0000","last_fullsized":"2023-10-01T09:44:00.466682+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":437,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.466569+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:44:00.466569+0000","last_peered":"2023-10-01T09:44:00.466569+0000","last_clean":"2023-10-01T09:44:00.466569+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:44:00.466569+0000","last_undegraded":"2023-10-01T09:44:00.466569+0000","last_fullsized":"2023-10-01T09:44:00.466569+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"526'6337","reported_seq":6404,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.035450+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:44:06.035450+0000","last_peered":"2023-10-01T09:44:06.035450+0000","last_clean":"2023-10-01T09:44:06.035450+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:44:06.035450+0000","last_undegraded":"2023-10-01T09:44:06.035450+0000","last_fullsized":"2023-10-01T09:44:06.035450+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6337,"log_dups_size":0,"ondisk_log_size":6337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263389184,"num_objects":4019,"num_object_clones":0,"num_object_copies":12057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6337,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":465,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.468457+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:43:52.468457+0000","last_peered":"2023-10-01T09:43:52.468457+0000","last_clean":"2023-10-01T09:43:52.468457+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:43:52.468457+0000","last_undegraded":"2023-10-01T09:43:52.468457+0000","last_fullsized":"2023-10-01T09:43:52.468457+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1158,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.457187+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:43:52.457187+0000","last_peered":"2023-10-01T09:43:52.457187+0000","last_clean":"2023-10-01T09:43:52.457187+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:43:52.457187+0000","last_undegraded":"2023-10-01T09:43:52.457187+0000","last_fullsized":"2023-10-01T09:43:52.457187+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":54,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.467679+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:43:52.467679+0000","last_peered":"2023-10-01T09:43:52.467679+0000","last_clean":"2023-10-01T09:43:52.467679+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:43:52.467679+0000","last_undegraded":"2023-10-01T09:43:52.467679+0000","last_fullsized":"2023-10-01T09:43:52.467679+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"526'6376","reported_seq":6443,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.173502+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:44:06.173502+0000","last_peered":"2023-10-01T09:44:06.173502+0000","last_clean":"2023-10-01T09:44:06.173502+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:44:06.173502+0000","last_undegraded":"2023-10-01T09:44:06.173502+0000","last_fullsized":"2023-10-01T09:44:06.173502+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6376,"log_dups_size":0,"ondisk_log_size":6376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271056896,"num_objects":4136,"num_object_clones":0,"num_object_copies":12408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6376,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":58,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.468303+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:43:52.468303+0000","last_peered":"2023-10-01T09:43:52.468303+0000","last_clean":"2023-10-01T09:43:52.468303+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:43:52.468303+0000","last_undegraded":"2023-10-01T09:43:52.468303+0000","last_fullsized":"2023-10-01T09:43:52.468303+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":544,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.461884+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:44:00.461884+0000","last_peered":"2023-10-01T09:44:00.461884+0000","last_clean":"2023-10-01T09:44:00.461884+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:44:00.461884+0000","last_undegraded":"2023-10-01T09:44:00.461884+0000","last_fullsized":"2023-10-01T09:44:00.461884+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":250,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.463056+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:44:00.463056+0000","last_peered":"2023-10-01T09:44:00.463056+0000","last_clean":"2023-10-01T09:44:00.463056+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:44:00.463056+0000","last_undegraded":"2023-10-01T09:44:00.463056+0000","last_fullsized":"2023-10-01T09:44:00.463056+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":493,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.461912+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:44:00.461912+0000","last_peered":"2023-10-01T09:44:00.461912+0000","last_clean":"2023-10-01T09:44:00.461912+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:44:00.461912+0000","last_undegraded":"2023-10-01T09:44:00.461912+0000","last_fullsized":"2023-10-01T09:44:00.461912+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":606,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:55.256643+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:43:55.256643+0000","last_peered":"2023-10-01T09:43:55.256643+0000","last_clean":"2023-10-01T09:43:55.256643+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:43:55.256643+0000","last_undegraded":"2023-10-01T09:43:55.256643+0000","last_fullsized":"2023-10-01T09:43:55.256643+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":262,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.466524+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:44:00.466524+0000","last_peered":"2023-10-01T09:44:00.466524+0000","last_clean":"2023-10-01T09:44:00.466524+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:44:00.466524+0000","last_undegraded":"2023-10-01T09:44:00.466524+0000","last_fullsized":"2023-10-01T09:44:00.466524+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":49,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.456909+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:43:52.456909+0000","last_peered":"2023-10-01T09:43:52.456909+0000","last_clean":"2023-10-01T09:43:52.456909+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:43:52.456909+0000","last_undegraded":"2023-10-01T09:43:52.456909+0000","last_fullsized":"2023-10-01T09:43:52.456909+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":262,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.466786+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:44:00.466786+0000","last_peered":"2023-10-01T09:44:00.466786+0000","last_clean":"2023-10-01T09:44:00.466786+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:44:00.466786+0000","last_undegraded":"2023-10-01T09:44:00.466786+0000","last_fullsized":"2023-10-01T09:44:00.466786+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":439,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.461586+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:44:00.461586+0000","last_peered":"2023-10-01T09:44:00.461586+0000","last_clean":"2023-10-01T09:44:00.461586+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:44:00.461586+0000","last_undegraded":"2023-10-01T09:44:00.461586+0000","last_fullsized":"2023-10-01T09:44:00.461586+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":67,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.485985+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:44:00.485985+0000","last_peered":"2023-10-01T09:44:00.485985+0000","last_clean":"2023-10-01T09:44:00.485985+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:44:00.485985+0000","last_undegraded":"2023-10-01T09:44:00.485985+0000","last_fullsized":"2023-10-01T09:44:00.485985+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":484,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.461608+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:44:00.461608+0000","last_peered":"2023-10-01T09:44:00.461608+0000","last_clean":"2023-10-01T09:44:00.461608+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:44:00.461608+0000","last_undegraded":"2023-10-01T09:44:00.461608+0000","last_fullsized":"2023-10-01T09:44:00.461608+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":683,"reported_epoch":526,"state":"active+clean","last_fresh":"2023-10-01T09:44:00.462097+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:44:00.462097+0000","last_peered":"2023-10-01T09:44:00.462097+0000","last_clean":"2023-10-01T09:44:00.462097+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:44:00.462097+0000","last_undegraded":"2023-10-01T09:44:00.462097+0000","last_fullsized":"2023-10-01T09:44:00.462097+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":377,"reported_epoch":525,"state":"active+clean","last_fresh":"2023-10-01T09:43:52.456178+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:43:52.456178+0000","last_peered":"2023-10-01T09:43:52.456178+0000","last_clean":"2023-10-01T09:43:52.456178+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:43:52.456178+0000","last_undegraded":"2023-10-01T09:43:52.456178+0000","last_fullsized":"2023-10-01T09:43:52.456178+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4275437592,"num_objects":65239,"num_object_clones":0,"num_object_copies":195717,"num_objects_missing_on_primary":2538,"num_objects_missing":2538,"num_objects_degraded":7627,"num_objects_misplaced":4005,"num_objects_unfound":0,"num_objects_dirty":65239,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":101201,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8055,"num_bytes_recovered":527892480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13214232576,"data_stored":13214220360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101201,"ondisk_log_size":101201,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430722,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2205056,"kb_used_data":1310136,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":91540096,"statfs":{"total":95995035648,"available":93737058304,"internally_reserved":0,"allocated":1341579264,"data_stored":1340641560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":1.1359999999999999,"15min":2.657},"min":{"1min":0.27300000000000002,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":1.484,"5min":10.391999999999999,"15min":108.90000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.39900000000000002,"5min":1.26,"15min":2.8090000000000002},"min":{"1min":0.219,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.67200000000000004,"5min":12.026,"15min":109.187},"last":0.27800000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":4.6749999999999998,"15min":5.3520000000000003},"min":{"1min":0.31900000000000001,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":1.2949999999999999,"5min":229.161,"15min":229.161},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.47399999999999998,"5min":4.6479999999999997,"15min":5.3689999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":0.90900000000000003,"5min":230.126,"15min":230.126},"last":0.35399999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:43:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.169,"15min":1.76},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.992,"5min":14.866,"15min":14.901999999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.4450000000000001,"15min":1.7170000000000001},"min":{"1min":0.32000000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4490000000000001,"5min":14.682,"15min":14.682},"last":0.45200000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":1.514,"15min":2.7330000000000001},"min":{"1min":0.30199999999999999,"5min":0.29199999999999998,"15min":0.28799999999999998},"max":{"1min":0.90700000000000003,"5min":35.104999999999997,"15min":80.352999999999994},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.50600000000000001,"5min":1.6930000000000001,"15min":2.7130000000000001},"min":{"1min":0.35399999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.153,"5min":35.118000000000002,"15min":80.364000000000004},"last":0.46300000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":1.046,"15min":1.0429999999999999},"min":{"1min":0.30199999999999999,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.508,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.436},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.0289999999999999,"15min":1.0189999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.28000000000000003},"max":{"1min":0.84399999999999997,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.39100000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":2.3620000000000001,"15min":1.6120000000000001},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":1.2749999999999999,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.62,"5min":2.5390000000000001,"15min":1.6739999999999999},"min":{"1min":0.35999999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.2599999999999998,"5min":135.339,"15min":135.339},"last":0.501}]},{"osd":6,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":1.3200000000000001,"15min":1.1599999999999999},"min":{"1min":0.377,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":1.409,"5min":11.58,"15min":11.853999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.51400000000000001,"5min":1.175,"15min":1.1299999999999999},"min":{"1min":0.378,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":1.23,"5min":14.584,"15min":14.584},"last":0.51900000000000002}]}]},{"osd":6,"up_from":294,"seq":1262720385233,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2171268,"kb_used_data":1833660,"kb_used_omap":6,"kb_used_meta":337593,"kb_avail":91573884,"statfs":{"total":95995035648,"available":93771657216,"internally_reserved":0,"allocated":1877667840,"data_stored":1876733763,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":345695277},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":3.5720000000000001,"15min":4.0919999999999996},"min":{"1min":0.25,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":3.3340000000000001,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.317},{"interface":"front","average":{"1min":0.42599999999999999,"5min":4.0410000000000004,"15min":4.2210000000000001},"min":{"1min":0.29899999999999999,"5min":0.249,"15min":0.221},"max":{"1min":0.623,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.63900000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":4.7489999999999997,"15min":4.9740000000000002},"min":{"1min":0.30599999999999999,"5min":0.254,"15min":0.23899999999999999},"max":{"1min":1.9410000000000001,"5min":168.947,"15min":168.947},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.53000000000000003,"5min":4.7969999999999997,"15min":4.8810000000000002},"min":{"1min":0.34699999999999998,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":1.962,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.58699999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":1.194,"15min":1.673},"min":{"1min":0.308,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.075,"5min":8.657,"15min":10.92},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":1.3759999999999999,"15min":1.673},"min":{"1min":0.314,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.58499999999999996,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.35599999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.3069999999999999,"15min":2.2789999999999999},"min":{"1min":0.35499999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.3759999999999999,"5min":9.0180000000000007,"15min":89.753},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.502,"5min":1.4359999999999999,"15min":2.157},"min":{"1min":0.34399999999999997,"5min":0.31,"15min":0.29299999999999998},"max":{"1min":1.381,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.45500000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":1.5900000000000001,"15min":1.0800000000000001},"min":{"1min":0.30099999999999999,"5min":0.253,"15min":0.246},"max":{"1min":0.53400000000000003,"5min":63.134,"15min":63.134},"last":0.497},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.5009999999999999,"15min":1.097},"min":{"1min":0.33700000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":1.5669999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.626}]},{"osd":5,"last update":"Sun Oct 1 09:43:29 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":1.2450000000000001,"15min":1.101},"min":{"1min":0.309,"5min":0.251,"15min":0.251},"max":{"1min":34.094999999999999,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.409,"15min":1.22},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":34.072000000000003,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.61799999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:43:15 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.79500000000000004,"15min":1.46},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":1.4750000000000001,"5min":6.4119999999999999,"15min":94.245999999999995},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.48999999999999999,"5min":1.113,"15min":1.5569999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":0.68300000000000005,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.68500000000000005}]}]},{"osd":1,"up_from":94,"seq":403726926209,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1623980,"kb_used_data":801700,"kb_used_omap":7,"kb_used_meta":822264,"kb_avail":92121172,"statfs":{"total":95995035648,"available":94332080128,"internally_reserved":0,"allocated":820940800,"data_stored":818909052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":841998759},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":1.7789999999999999,"15min":4.0039999999999996},"min":{"1min":0.186,"5min":0.186,"15min":0.182},"max":{"1min":1.2889999999999999,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.23599999999999999},{"interface":"front","average":{"1min":0.42099999999999999,"5min":1.99,"15min":3.5329999999999999},"min":{"1min":0.223,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":1.702,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.25700000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":1.4279999999999999,"15min":2.589},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.61399999999999999,"5min":22.891999999999999,"15min":56.573},"last":0.307},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.379,"15min":3.254},"min":{"1min":0.28999999999999998,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":1.752,"5min":15.189,"15min":154.197},"last":2.2200000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":2.1680000000000001,"15min":3.9969999999999999},"min":{"1min":0.27400000000000002,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":1.1990000000000001,"5min":18.503,"15min":200.56800000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":2.3660000000000001,"15min":4.6989999999999998},"min":{"1min":0.245,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":0.51900000000000002,"5min":20.181000000000001,"15min":200.63800000000001},"last":0.435}]},{"osd":4,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.147,"15min":2.2530000000000001},"min":{"1min":0.35499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.0550000000000002,"5min":19.539000000000001,"15min":154.101},"last":2.9209999999999998},{"interface":"front","average":{"1min":0.46899999999999997,"5min":1.0029999999999999,"15min":2.399},"min":{"1min":0.29499999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.59799999999999998,"5min":10.084,"15min":154.56999999999999},"last":0.33200000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:43:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.3200000000000001,"15min":2.097},"min":{"1min":0.372,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.1850000000000001,"5min":19.513000000000002,"15min":154.22999999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.1719999999999999,"15min":2.4630000000000001},"min":{"1min":0.31,"5min":0.307,"15min":0.307},"max":{"1min":0.82699999999999996,"5min":10.305,"15min":154.54400000000001},"last":0.40600000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.849,"15min":2.6640000000000001},"min":{"1min":0.372,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.2789999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.495,"5min":1.702,"15min":2.4870000000000001},"min":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":0.67300000000000004,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.34699999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.3109999999999999,"15min":2.5840000000000001},"min":{"1min":0.38400000000000001,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":2.8170000000000002,"5min":19.431999999999999,"15min":154.20699999999999},"last":1.948},{"interface":"front","average":{"1min":0.77800000000000002,"5min":1.2330000000000001,"15min":2.6379999999999999},"min":{"1min":0.45200000000000001,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":2.8290000000000002,"5min":10.111000000000001,"15min":154.51599999999999},"last":1.9750000000000001}]}]},{"osd":0,"up_from":89,"seq":382252089744,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4166312,"kb_used_data":3279264,"kb_used_omap":8,"kb_used_meta":887031,"kb_avail":89578840,"statfs":{"total":95995035648,"available":91728732160,"internally_reserved":0,"allocated":3357966336,"data_stored":3355951349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":908320706},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":1.76,"15min":4.2480000000000002},"min":{"1min":0.17799999999999999,"5min":0.17799999999999999,"15min":0.17399999999999999},"max":{"1min":1.891,"5min":52.316000000000003,"15min":216.821},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":3.4119999999999999,"15min":5.2130000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":3.552,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.24399999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:43:36 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":1.204,"15min":2.085},"min":{"1min":0.246,"5min":0.246,"15min":0.20100000000000001},"max":{"1min":1.1100000000000001,"5min":19.788,"15min":19.788},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.4139999999999999,"15min":2.3380000000000001},"min":{"1min":0.27200000000000002,"5min":0.23000000000000001,"15min":0.189},"max":{"1min":3.5939999999999999,"5min":19.375,"15min":25.844999999999999},"last":0.313}]},{"osd":3,"last update":"Sun Oct 1 09:43:13 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":1.474,"15min":3.907},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":1.1499999999999999,"5min":15.35,"15min":222.16499999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.58699999999999997,"5min":1.748,"15min":3.9460000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":2.9550000000000001,"5min":14.545999999999999,"15min":222.227},"last":0.33600000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.5449999999999999,"15min":2.1339999999999999},"min":{"1min":0.28799999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":3.375,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.438},{"interface":"front","average":{"1min":0.58399999999999996,"5min":1.4550000000000001,"15min":2.1309999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":3.5049999999999999,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.39200000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:43:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.2030000000000001,"15min":1.883},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66500000000000004,"5min":14.515000000000001,"15min":16.808},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.093,"15min":1.7709999999999999},"min":{"1min":0.33500000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.73499999999999999,"5min":5.9580000000000002,"15min":17.507000000000001},"last":0.52600000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:43:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.1850000000000001,"15min":1.774},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.35499999999999998},"max":{"1min":1.5900000000000001,"5min":6.5,"15min":17.535},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.88,"5min":1.395,"15min":2.0289999999999999},"min":{"1min":0.32600000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":3.6539999999999999,"5min":10.446,"15min":18.521000000000001},"last":0.38700000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.95499999999999996,"15min":2.048},"min":{"1min":0.379,"5min":0.374,"15min":0.35699999999999998},"max":{"1min":0.78600000000000003,"5min":4.976,"15min":48.895000000000003},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.093,"15min":2.206},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.35099999999999998},"max":{"1min":3.4319999999999999,"5min":9.4969999999999999,"15min":48.274000000000001},"last":0.46500000000000002}]}]},{"osd":2,"up_from":0,"seq":425201762683,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599157,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1700136,"kb_used_data":872096,"kb_used_omap":3,"kb_used_meta":828028,"kb_avail":92045016,"statfs":{"total":95995035648,"available":94254096384,"internally_reserved":0,"allocated":893026304,"data_stored":890947576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":847901415},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":4.9660000000000002,"15min":5.0880000000000001},"min":{"1min":0.20300000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":2.7989999999999999,"5min":118.748,"15min":137.34399999999999},"last":0.24299999999999999},{"interface":"front","average":{"1min":0.28899999999999998,"5min":4.9500000000000002,"15min":5.1379999999999999},"min":{"1min":0.20699999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.42199999999999999,"5min":118.86199999999999,"15min":137.73400000000001},"last":0.44500000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":3.4329999999999998,"15min":5.2869999999999999},"min":{"1min":0.23000000000000001,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":2.7250000000000001,"5min":90.944000000000003,"15min":162.22999999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.33300000000000002,"5min":3.1850000000000001,"15min":5.4160000000000004},"min":{"1min":0.23100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.45300000000000001,"5min":83.341999999999999,"15min":162.70699999999999},"last":0.28899999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":1.8740000000000001,"15min":2.2029999999999998},"min":{"1min":0.251,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":0.54900000000000004,"5min":14.254,"15min":36.472999999999999},"last":0.877},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.90700000000000003,"15min":2.4910000000000001},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":0.59999999999999998,"5min":7.6749999999999998,"15min":78.594999999999999},"last":0.377}]},{"osd":4,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":1.8859999999999999,"15min":1.875},"min":{"1min":0.32600000000000001,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":0.65300000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.42799999999999999,"5min":2.2189999999999999,"15min":2.1469999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":0.64300000000000002,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.52300000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":1.5589999999999999,"15min":2.581},"min":{"1min":0.317,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3099999999999996,"5min":10.919,"15min":28.117999999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.4279999999999999,"15min":2.6800000000000002},"min":{"1min":0.36699999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.512,"5min":10.798999999999999,"15min":28.34},"last":0.92300000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:43:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":2.165,"15min":2.145},"min":{"1min":0.34200000000000003,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":2.5430000000000001,"5min":73.887,"15min":73.887},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.63,"5min":2.4159999999999999,"15min":2.1379999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":2.6000000000000001,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.98399999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:43:07 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":1.7889999999999999,"15min":2.3090000000000002},"min":{"1min":0.39400000000000002,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":4.1580000000000004,"5min":13.085000000000001,"15min":52.006},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.306,"15min":2.032},"min":{"1min":0.41899999999999998,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":4.1779999999999999,"5min":8.3460000000000001,"15min":51.890999999999998},"last":0.435}]}]},{"osd":4,"up_from":30,"seq":128849019407,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4090172,"kb_used_data":3204084,"kb_used_omap":0,"kb_used_meta":886080,"kb_avail":89654980,"statfs":{"total":95995035648,"available":91806699520,"internally_reserved":0,"allocated":3280982016,"data_stored":3280047818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":907345920},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":2.3570000000000002,"15min":4.25},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.47299999999999998,"5min":79.837000000000003,"15min":155.315},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.44900000000000001,"5min":2.4300000000000002,"15min":4.7039999999999997},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":0.63700000000000001,"5min":78.983000000000004,"15min":155.334},"last":0.434}]},{"osd":1,"last update":"Sun Oct 1 09:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":4.5709999999999997,"15min":6.4699999999999998},"min":{"1min":0.28899999999999998,"5min":0.252,"15min":0.224},"max":{"1min":0.52900000000000003,"5min":126.742,"15min":190.61199999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.40200000000000002,"5min":4.5369999999999999,"15min":7.1600000000000001},"min":{"1min":0.27600000000000002,"5min":0.26700000000000002,"15min":0.25600000000000001},"max":{"1min":0.59999999999999998,"5min":126.791,"15min":253.273},"last":0.41599999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.05,"15min":2.444},"min":{"1min":0.34200000000000003,"5min":0.31,"15min":0.28299999999999997},"max":{"1min":2.9510000000000001,"5min":10.161,"15min":170.45400000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.0029999999999999,"15min":2.1739999999999999},"min":{"1min":0.314,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":3.0219999999999998,"5min":5.9740000000000002,"15min":58.771000000000001},"last":0.40300000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.96499999999999997,"15min":2.3079999999999998},"min":{"1min":0.28100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.1299999999999999,"5min":7.5830000000000002,"15min":140.881},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.876,"15min":2.4830000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.311},"max":{"1min":0.51500000000000001,"5min":6.8719999999999999,"15min":140.911},"last":0.49099999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:43:33 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":2.0569999999999999,"15min":1.5580000000000001},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":3.052,"5min":110.003,"15min":110.003},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.64900000000000002,"5min":2.0110000000000001,"15min":1.399},"min":{"1min":0.33500000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.0049999999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.441}]},{"osd":6,"last update":"Sun Oct 1 09:43:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.79200000000000004,"15min":0.99299999999999999},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":0.80900000000000005,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.95199999999999996,"15min":1.085},"min":{"1min":0.379,"5min":0.314,"15min":0.28999999999999998},"max":{"1min":2.9239999999999999,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.46999999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.71599999999999997,"15min":1.161},"min":{"1min":0.313,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":0.69399999999999995,"5min":4.7030000000000003,"15min":38.034999999999997},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.026,"15min":1.4410000000000001},"min":{"1min":0.36699999999999999,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":2.9940000000000002,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.502}]}]},{"osd":5,"up_from":379,"seq":1627792605314,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2033512,"kb_used_data":1664864,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":91711640,"statfs":{"total":95995035648,"available":93912719360,"internally_reserved":0,"allocated":1704820736,"data_stored":1703902442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,1,0,0,5,2,8],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":1.6479999999999999,"15min":2.1890000000000001},"min":{"1min":0.22900000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.1880000000000002,"5min":12.571999999999999,"15min":84.340999999999994},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.40400000000000003,"5min":1.2549999999999999,"15min":2.0800000000000001},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.53900000000000003,"5min":8.6460000000000008,"15min":84.356999999999999},"last":0.28100000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":6.4870000000000001,"15min":5.7599999999999998},"min":{"1min":0.26000000000000001,"5min":0.25900000000000001,"15min":0.255},"max":{"1min":0.76900000000000002,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.875},{"interface":"front","average":{"1min":0.53800000000000003,"5min":6.8099999999999996,"15min":5.7320000000000002},"min":{"1min":0.32200000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":2.258,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.41699999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":1.3600000000000001,"15min":1.5980000000000001},"min":{"1min":0.315,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.752,"5min":8.7040000000000006,"15min":14.199999999999999},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.44800000000000001,"5min":1.306,"15min":1.583},"min":{"1min":0.32900000000000001,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":0.85999999999999999,"5min":8.2370000000000001,"15min":14.430999999999999},"last":0.93500000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.5069999999999999,"15min":1.488},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.2280000000000002,"5min":12.734,"15min":14.004},"last":0.499},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.5640000000000001,"15min":1.629},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.3979999999999999,"5min":11.766,"15min":13.991},"last":0.44400000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":2.2029999999999998,"15min":1.4330000000000001},"min":{"1min":0.30099999999999999,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.66800000000000004,"5min":122.971,"15min":122.971},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.438,"5min":2.25,"15min":1.724},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.73799999999999999,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.95699999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":1.165,"15min":1.044},"min":{"1min":0.312,"5min":0.311,"15min":0.28299999999999997},"max":{"1min":2.2839999999999998,"5min":12.545,"15min":12.545},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.99099999999999999,"15min":1.302},"min":{"1min":0.378,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":2.629,"5min":6.4729999999999999,"15min":14.193},"last":0.39500000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.1140000000000001,"15min":1.262},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.81200000000000006,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.91000000000000003},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.2270000000000001,"15min":1.0780000000000001},"min":{"1min":0.36299999999999999,"5min":0.35299999999999998,"15min":0.318},"max":{"1min":2.3079999999999998,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.61599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3346403328,"data_stored":3346399256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":808779776,"data_stored":808779776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":886636544,"data_stored":886636544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3273265152,"data_stored":3273261080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1699545088,"data_stored":1699545088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1867976704,"data_stored":1867972632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1331625984,"data_stored":1331625984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:44:10.714 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:44:10.962 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:10.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:10.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:11.069 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:11.097 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:11.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:11 smithi138 ceph-mon[192513]: from='client.46961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:11.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:11.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:44:11.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:11 smithi138 ceph-mon[192513]: pgmap v1532: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 7637/195513 objects degraded (3.906%); 4005/195513 objects misplaced (2.048%); 539 KiB/s, 8 objects/s recovering 2023-10-01T09:44:11.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:11 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:44:11.386 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:11.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:11.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:11.455 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:44:11.489 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:11.517 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:11.563 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:11.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:11.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:11.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:11 smithi138 ceph-mon[185886]: from='client.46961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:11.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:11.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:44:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:11 smithi138 ceph-mon[185886]: pgmap v1532: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 7637/195513 objects degraded (3.906%); 4005/195513 objects misplaced (2.048%); 539 KiB/s, 8 objects/s recovering 2023-10-01T09:44:11.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:11 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:44:11.667 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:11.695 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:11 smithi190 ceph-mon[160435]: from='client.46961 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:44:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:11 smithi190 ceph-mon[160435]: pgmap v1532: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 134 op/s; 7637/195513 objects degraded (3.906%); 4005/195513 objects misplaced (2.048%); 539 KiB/s, 8 objects/s recovering 2023-10-01T09:44:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:11 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:44:11.834 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:11.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:11.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:11.876 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:11.938 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:11.965 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:12.010 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:12.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:12.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:12.057 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:44:12.113 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:12.141 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:12.193 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:12.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:12.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:12.301 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:12.329 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:12.385 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:12.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:12.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:12.489 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:12.517 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:12.718 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:12.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:12.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:12.827 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:12.860 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:13.009 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:44:13.026 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:13.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:13.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:13.133 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:13.161 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:13.676 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:13.679 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:13 smithi190 ceph-mon[160435]: pgmap v1533: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 136 op/s; 7591/194052 objects degraded (3.912%); 3965/194052 objects misplaced (2.043%); 544 KiB/s, 8 objects/s recovering 2023-10-01T09:44:13.679 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:13 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7591/194052 objects degraded (3.912%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:13.679 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:13 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3076995607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:44:13.679 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:13 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:44:13.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:13.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:13.723 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:13.781 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:13.797 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:44:13.816 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:13.817 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:13 smithi138 ceph-mon[192513]: pgmap v1533: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 136 op/s; 7591/194052 objects degraded (3.912%); 3965/194052 objects misplaced (2.043%); 544 KiB/s, 8 objects/s recovering 2023-10-01T09:44:13.817 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:13 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7591/194052 objects degraded (3.912%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:13.818 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:13 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3076995607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:44:13.818 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:13 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:44:13.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:13 smithi138 ceph-mon[185886]: pgmap v1533: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 136 op/s; 7591/194052 objects degraded (3.912%); 3965/194052 objects misplaced (2.043%); 544 KiB/s, 8 objects/s recovering 2023-10-01T09:44:13.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:13 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7591/194052 objects degraded (3.912%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:13.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:13 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3076995607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:44:13.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:13 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:44:13.962 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:44:13.962 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 7591/194052 objects degraded (3.912%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2074,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":528,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":64684,"data_bytes":4242020299,"bytes_used":18326032384,"bytes_avail":653639217152,"bytes_total":671965249536,"degraded_objects":7591,"degraded_total":194052,"degraded_ratio":0.039118380640240757,"misplaced_objects":3965,"misplaced_total":194052,"misplaced_ratio":0.020432667532413996,"recovering_objects_per_sec":8,"recovering_bytes_per_sec":556831,"recovering_keys_per_sec":0,"num_objects_recovered":85,"num_bytes_recovered":5570560,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":136},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":34,"modified":"2023-10-01T09:43:40.311181+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:44:14.007 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:14.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:14.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:14.110 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:14.142 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:14.201 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:14.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:14.252 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:44:14.252 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:44:14.253 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:44:14.253 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:44:14.253 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.253 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:145796 9.c 9:3a5a6dfb:::benchmark_data_smithi138_497829_object43411:head [delete] snapc 0=[] ondisk+write+known_if_redirected e444)", 2023-10-01T09:44:14.253 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:16.533300+0000", 2023-10-01T09:44:14.253 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 417.71589845199998, 2023-10-01T09:44:14.253 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.29276776999999998, 2023-10-01T09:44:14.253 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.253 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.253 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.253 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.253 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.254 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 145796 2023-10-01T09:44:14.254 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.254 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.254 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.254 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.254 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533300+0000", 2023-10-01T09:44:14.254 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.254 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.254 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.254 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.254 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533300+0000", 2023-10-01T09:44:14.254 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:44:14.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.255 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.255 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533298+0000", 2023-10-01T09:44:14.255 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0709999999999995e-06 2023-10-01T09:44:14.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.255 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.255 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533306+0000", 2023-10-01T09:44:14.255 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5869999999999999e-05 2023-10-01T09:44:14.255 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.255 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.256 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.256 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533322+0000", 2023-10-01T09:44:14.256 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.815e-06 2023-10-01T09:44:14.256 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.256 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.256 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.256 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.533329+0000", 2023-10-01T09:44:14.256 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00163053 2023-10-01T09:44:14.256 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.256 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.256 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.256 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.534959+0000", 2023-10-01T09:44:14.257 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014893600000000001 2023-10-01T09:44:14.257 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.257 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.257 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.257 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.535108+0000", 2023-10-01T09:44:14.257 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000164644 2023-10-01T09:44:14.257 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.257 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.257 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,3,4", 2023-10-01T09:44:14.257 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.535273+0000", 2023-10-01T09:44:14.257 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0048375980000000002 2023-10-01T09:44:14.257 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.257 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.258 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.258 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.540110+0000", 2023-10-01T09:44:14.258 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001286875 2023-10-01T09:44:14.258 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.258 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.258 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.258 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.541397+0000", 2023-10-01T09:44:14.258 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001817658 2023-10-01T09:44:14.258 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.258 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.258 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.258 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.543215+0000", 2023-10-01T09:44:14.259 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28271621800000002 2023-10-01T09:44:14.259 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.259 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.259 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.259 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.825931+0000", 2023-10-01T09:44:14.259 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1245999999999998e-05 2023-10-01T09:44:14.259 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.259 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.259 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.259 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.825972+0000", 2023-10-01T09:44:14.259 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5760000000000005e-05 2023-10-01T09:44:14.259 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.260 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.260 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.260 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:16.826068+0000", 2023-10-01T09:44:14.260 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.29276776999999998 2023-10-01T09:44:14.260 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.260 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.260 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.260 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.260 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.260 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147931 9.3 9:cc030d19:::benchmark_data_smithi138_497829_object45546:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:44:14.260 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.304355+0000", 2023-10-01T09:44:14.260 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.94484390299999, 2023-10-01T09:44:14.261 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3960295309999999, 2023-10-01T09:44:14.261 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.261 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.261 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.261 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.261 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.261 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147931 2023-10-01T09:44:14.261 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.261 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.261 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.261 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.261 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304355+0000", 2023-10-01T09:44:14.261 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.262 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.262 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.262 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.262 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304355+0000", 2023-10-01T09:44:14.262 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:44:14.262 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.262 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.262 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.262 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304354+0000", 2023-10-01T09:44:14.262 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1670000000000004e-06 2023-10-01T09:44:14.262 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.262 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.263 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.263 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304360+0000", 2023-10-01T09:44:14.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.27e-05 2023-10-01T09:44:14.263 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.263 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.263 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.263 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304372+0000", 2023-10-01T09:44:14.263 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.815e-06 2023-10-01T09:44:14.263 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.263 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.263 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.263 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304379+0000", 2023-10-01T09:44:14.264 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000106932 2023-10-01T09:44:14.264 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.264 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.264 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.264 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304486+0000", 2023-10-01T09:44:14.264 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3089999999999996e-06 2023-10-01T09:44:14.264 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.264 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.264 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.264 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.304495+0000", 2023-10-01T09:44:14.264 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.322764053 2023-10-01T09:44:14.264 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.265 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.265 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.265 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.627259+0000", 2023-10-01T09:44:14.265 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4950000000000008e-06 2023-10-01T09:44:14.265 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.265 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.265 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.265 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.627267+0000", 2023-10-01T09:44:14.265 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.028340244 2023-10-01T09:44:14.265 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.265 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.265 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.265 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.655607+0000", 2023-10-01T09:44:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000144229 2023-10-01T09:44:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.655752+0000", 2023-10-01T09:44:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00018547299999999999 2023-10-01T09:44:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:44:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.655937+0000", 2023-10-01T09:44:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021581132999999999 2023-10-01T09:44:14.266 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677518+0000", 2023-10-01T09:44:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.022700635 2023-10-01T09:44:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700219+0000", 2023-10-01T09:44:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2954999999999998e-05 2023-10-01T09:44:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.267 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700262+0000", 2023-10-01T09:44:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4472000000000002e-05 2023-10-01T09:44:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700326+0000", 2023-10-01T09:44:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8291000000000001e-05 2023-10-01T09:44:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700385+0000", 2023-10-01T09:44:14.268 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3960295309999999 2023-10-01T09:44:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147969 9.c 9:344c11ae:::benchmark_data_smithi138_497829_object45584:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:44:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.490163+0000", 2023-10-01T09:44:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.75903609099998, 2023-10-01T09:44:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.18335328, 2023-10-01T09:44:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.269 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147969 2023-10-01T09:44:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490163+0000", 2023-10-01T09:44:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.270 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490163+0000", 2023-10-01T09:44:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:44:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490162+0000", 2023-10-01T09:44:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7320000000000003e-06 2023-10-01T09:44:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490167+0000", 2023-10-01T09:44:14.271 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1588e-05 2023-10-01T09:44:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490179+0000", 2023-10-01T09:44:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1599999999999997e-06 2023-10-01T09:44:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490184+0000", 2023-10-01T09:44:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7620999999999998e-05 2023-10-01T09:44:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.272 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490222+0000", 2023-10-01T09:44:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5396e-05 2023-10-01T09:44:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.490237+0000", 2023-10-01T09:44:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14082081399999999 2023-10-01T09:44:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.631058+0000", 2023-10-01T09:44:14.273 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.152e-06 2023-10-01T09:44:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.631063+0000", 2023-10-01T09:44:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0296697379999999 2023-10-01T09:44:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660733+0000", 2023-10-01T09:44:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011841399999999999 2023-10-01T09:44:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.274 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660851+0000", 2023-10-01T09:44:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013647799999999999 2023-10-01T09:44:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:44:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660988+0000", 2023-10-01T09:44:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045588740000000001 2023-10-01T09:44:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665546+0000", 2023-10-01T09:44:14.275 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078629470000000003 2023-10-01T09:44:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673409+0000", 2023-10-01T09:44:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4159999999999997e-05 2023-10-01T09:44:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673454+0000", 2023-10-01T09:44:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2487000000000003e-05 2023-10-01T09:44:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.276 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673486+0000", 2023-10-01T09:44:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0018e-05 2023-10-01T09:44:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673516+0000", 2023-10-01T09:44:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.18335328 2023-10-01T09:44:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.277 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147981 9.3 9:c3aeeda4:::benchmark_data_smithi138_497829_object45596:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:44:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.563343+0000", 2023-10-01T09:44:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.68585588000002, 2023-10-01T09:44:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1372912850000001, 2023-10-01T09:44:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147981 2023-10-01T09:44:14.278 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563343+0000", 2023-10-01T09:44:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563343+0000", 2023-10-01T09:44:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999933 2023-10-01T09:44:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.279 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563336+0000", 2023-10-01T09:44:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1314e-05 2023-10-01T09:44:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563347+0000", 2023-10-01T09:44:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0358e-05 2023-10-01T09:44:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563358+0000", 2023-10-01T09:44:14.280 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3170000000000003e-06 2023-10-01T09:44:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563362+0000", 2023-10-01T09:44:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1795999999999997e-05 2023-10-01T09:44:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563414+0000", 2023-10-01T09:44:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1110999999999999e-05 2023-10-01T09:44:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.281 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.563425+0000", 2023-10-01T09:44:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.063862261000000003 2023-10-01T09:44:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.627287+0000", 2023-10-01T09:44:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4970000000000001e-06 2023-10-01T09:44:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.282 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.627293+0000", 2023-10-01T09:44:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0332602529999999 2023-10-01T09:44:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660553+0000", 2023-10-01T09:44:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3721999999999996e-05 2023-10-01T09:44:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660637+0000", 2023-10-01T09:44:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012805999999999999 2023-10-01T09:44:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.283 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:44:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.660765+0000", 2023-10-01T09:44:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.025676641 2023-10-01T09:44:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686441+0000", 2023-10-01T09:44:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013987271000000001 2023-10-01T09:44:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.284 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700429+0000", 2023-10-01T09:44:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000107107 2023-10-01T09:44:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700536+0000", 2023-10-01T09:44:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0742000000000002e-05 2023-10-01T09:44:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700597+0000", 2023-10-01T09:44:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7715000000000001e-05 2023-10-01T09:44:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.285 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700634+0000", 2023-10-01T09:44:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1372912850000001 2023-10-01T09:44:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147984 9.3 9:c9142d5f:::benchmark_data_smithi138_497829_object45599:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:44:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.707974+0000", 2023-10-01T09:44:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.54122537500001, 2023-10-01T09:44:14.286 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99286916199999997, 2023-10-01T09:44:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147984 2023-10-01T09:44:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.707974+0000", 2023-10-01T09:44:14.287 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.707974+0000", 2023-10-01T09:44:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:44:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.707971+0000", 2023-10-01T09:44:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1004e-05 2023-10-01T09:44:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.288 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.707982+0000", 2023-10-01T09:44:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8074e-05 2023-10-01T09:44:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.708000+0000", 2023-10-01T09:44:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.503e-06 2023-10-01T09:44:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.289 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.708008+0000", 2023-10-01T09:44:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0042882299999999996 2023-10-01T09:44:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712296+0000", 2023-10-01T09:44:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0655e-05 2023-10-01T09:44:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712307+0000", 2023-10-01T09:44:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.95292995599999997 2023-10-01T09:44:14.290 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665237+0000", 2023-10-01T09:44:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011695400000000001 2023-10-01T09:44:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665354+0000", 2023-10-01T09:44:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000102116 2023-10-01T09:44:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.291 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:44:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665456+0000", 2023-10-01T09:44:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020992165 2023-10-01T09:44:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686448+0000", 2023-10-01T09:44:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014224582 2023-10-01T09:44:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700672+0000", 2023-10-01T09:44:14.292 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2769000000000006e-05 2023-10-01T09:44:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700765+0000", 2023-10-01T09:44:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4345000000000003e-05 2023-10-01T09:44:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700810+0000", 2023-10-01T09:44:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3080000000000002e-05 2023-10-01T09:44:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.293 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700843+0000", 2023-10-01T09:44:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.99286916199999997 2023-10-01T09:44:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:147985 9.3 9:c26d6b78:::benchmark_data_smithi138_497829_object45600:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:44:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.712409+0000", 2023-10-01T09:44:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.53679012800001, 2023-10-01T09:44:14.294 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98851199499999998, 2023-10-01T09:44:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 147985 2023-10-01T09:44:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712409+0000", 2023-10-01T09:44:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.295 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712409+0000", 2023-10-01T09:44:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:44:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712407+0000", 2023-10-01T09:44:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8050000000000007e-06 2023-10-01T09:44:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.296 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712415+0000", 2023-10-01T09:44:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6009999999999995e-06 2023-10-01T09:44:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712424+0000", 2023-10-01T09:44:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8139999999999997e-06 2023-10-01T09:44:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712429+0000", 2023-10-01T09:44:14.297 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4980999999999993e-05 2023-10-01T09:44:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712494+0000", 2023-10-01T09:44:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5420000000000002e-06 2023-10-01T09:44:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.712501+0000", 2023-10-01T09:44:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.95723478900000003 2023-10-01T09:44:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.298 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.669736+0000", 2023-10-01T09:44:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000116869 2023-10-01T09:44:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.669853+0000", 2023-10-01T09:44:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016995500000000001 2023-10-01T09:44:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:44:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.670023+0000", 2023-10-01T09:44:14.299 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016431211000000001 2023-10-01T09:44:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686454+0000", 2023-10-01T09:44:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014249539 2023-10-01T09:44:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700703+0000", 2023-10-01T09:44:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000168577 2023-10-01T09:44:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.300 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700872+0000", 2023-10-01T09:44:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.245e-05 2023-10-01T09:44:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700894+0000", 2023-10-01T09:44:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.639e-05 2023-10-01T09:44:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.301 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700921+0000", 2023-10-01T09:44:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.98851199499999998 2023-10-01T09:44:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148000 9.c 9:3eba568d:::benchmark_data_smithi138_497829_object45615:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:44:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.786332+0000", 2023-10-01T09:44:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.46286641199998, 2023-10-01T09:44:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.89635619300000002, 2023-10-01T09:44:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.302 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148000 2023-10-01T09:44:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786332+0000", 2023-10-01T09:44:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.303 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786332+0000", 2023-10-01T09:44:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:44:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786330+0000", 2023-10-01T09:44:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6572e-05 2023-10-01T09:44:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786347+0000", 2023-10-01T09:44:14.304 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7059e-05 2023-10-01T09:44:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786364+0000", 2023-10-01T09:44:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.357e-06 2023-10-01T09:44:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786371+0000", 2023-10-01T09:44:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7531999999999998e-05 2023-10-01T09:44:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.305 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786419+0000", 2023-10-01T09:44:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0134e-05 2023-10-01T09:44:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.786429+0000", 2023-10-01T09:44:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.879158459 2023-10-01T09:44:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.306 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665587+0000", 2023-10-01T09:44:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000112847 2023-10-01T09:44:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665700+0000", 2023-10-01T09:44:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011616399999999999 2023-10-01T09:44:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:44:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.665816+0000", 2023-10-01T09:44:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016651933000000001 2023-10-01T09:44:14.307 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682468+0000", 2023-10-01T09:44:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3857000000000006e-05 2023-10-01T09:44:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682562+0000", 2023-10-01T09:44:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4292999999999998e-05 2023-10-01T09:44:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.308 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682596+0000", 2023-10-01T09:44:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7022999999999999e-05 2023-10-01T09:44:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682653+0000", 2023-10-01T09:44:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5318999999999998e-05 2023-10-01T09:44:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682689+0000", 2023-10-01T09:44:14.309 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.89635619300000002 2023-10-01T09:44:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148010 9.c 9:365ddc65:::benchmark_data_smithi138_497829_object45625:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:44:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.803374+0000", 2023-10-01T09:44:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.44582539499999, 2023-10-01T09:44:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.87947385199999994, 2023-10-01T09:44:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.310 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148010 2023-10-01T09:44:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803374+0000", 2023-10-01T09:44:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.311 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803374+0000", 2023-10-01T09:44:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:44:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803372+0000", 2023-10-01T09:44:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6549999999999999e-06 2023-10-01T09:44:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803381+0000", 2023-10-01T09:44:14.312 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7244999999999998e-05 2023-10-01T09:44:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803398+0000", 2023-10-01T09:44:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6320000000000008e-06 2023-10-01T09:44:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803406+0000", 2023-10-01T09:44:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9969000000000002e-05 2023-10-01T09:44:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.313 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803456+0000", 2023-10-01T09:44:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.4809999999999995e-06 2023-10-01T09:44:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.803466+0000", 2023-10-01T09:44:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.86655919599999998 2023-10-01T09:44:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.314 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.670025+0000", 2023-10-01T09:44:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9349000000000003e-05 2023-10-01T09:44:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.670094+0000", 2023-10-01T09:44:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7009000000000007e-05 2023-10-01T09:44:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:44:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.670181+0000", 2023-10-01T09:44:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012516879999999999 2023-10-01T09:44:14.315 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682698+0000", 2023-10-01T09:44:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3449000000000003e-05 2023-10-01T09:44:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682751+0000", 2023-10-01T09:44:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4245999999999997e-05 2023-10-01T09:44:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.316 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682786+0000", 2023-10-01T09:44:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9830999999999999e-05 2023-10-01T09:44:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682816+0000", 2023-10-01T09:44:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1853999999999997e-05 2023-10-01T09:44:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682847+0000", 2023-10-01T09:44:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.87947385199999994 2023-10-01T09:44:14.317 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148037 9.3 9:c1ff9704:::benchmark_data_smithi138_497829_object45652:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:44:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.881445+0000", 2023-10-01T09:44:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.36775433499997, 2023-10-01T09:44:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81953648800000001, 2023-10-01T09:44:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.318 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148037 2023-10-01T09:44:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881445+0000", 2023-10-01T09:44:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.319 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881445+0000", 2023-10-01T09:44:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:44:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881443+0000", 2023-10-01T09:44:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2189999999999993e-06 2023-10-01T09:44:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881452+0000", 2023-10-01T09:44:14.320 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2901000000000001e-05 2023-10-01T09:44:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881465+0000", 2023-10-01T09:44:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1500000000000004e-06 2023-10-01T09:44:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881471+0000", 2023-10-01T09:44:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9982000000000003e-05 2023-10-01T09:44:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.321 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881531+0000", 2023-10-01T09:44:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1070000000000001e-05 2023-10-01T09:44:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.881542+0000", 2023-10-01T09:44:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.79148766599999998 2023-10-01T09:44:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673030+0000", 2023-10-01T09:44:14.322 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000120554 2023-10-01T09:44:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673151+0000", 2023-10-01T09:44:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000136756 2023-10-01T09:44:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:44:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673287+0000", 2023-10-01T09:44:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017487816999999999 2023-10-01T09:44:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.323 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.690775+0000", 2023-10-01T09:44:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0099592180000000006 2023-10-01T09:44:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700734+0000", 2023-10-01T09:44:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00020710800000000001 2023-10-01T09:44:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.324 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700942+0000", 2023-10-01T09:44:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0435999999999999e-05 2023-10-01T09:44:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700962+0000", 2023-10-01T09:44:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9011000000000001e-05 2023-10-01T09:44:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700981+0000", 2023-10-01T09:44:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81953648800000001 2023-10-01T09:44:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.325 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148041 9.3 9:c062535c:::benchmark_data_smithi138_497829_object45656:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:44:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.886259+0000", 2023-10-01T09:44:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.36294020700001, 2023-10-01T09:44:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81483497999999999, 2023-10-01T09:44:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.326 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148041 2023-10-01T09:44:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886259+0000", 2023-10-01T09:44:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.327 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886259+0000", 2023-10-01T09:44:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:44:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886257+0000", 2023-10-01T09:44:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1755000000000001e-05 2023-10-01T09:44:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886278+0000", 2023-10-01T09:44:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8885e-05 2023-10-01T09:44:14.328 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886297+0000", 2023-10-01T09:44:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7020000000000001e-06 2023-10-01T09:44:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886305+0000", 2023-10-01T09:44:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6515999999999999e-05 2023-10-01T09:44:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.329 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886361+0000", 2023-10-01T09:44:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2311e-05 2023-10-01T09:44:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.886374+0000", 2023-10-01T09:44:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.79119420699999998 2023-10-01T09:44:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677568+0000", 2023-10-01T09:44:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3848e-05 2023-10-01T09:44:14.330 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677662+0000", 2023-10-01T09:44:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7428000000000003e-05 2023-10-01T09:44:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:44:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677759+0000", 2023-10-01T09:44:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013022529 2023-10-01T09:44:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.331 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.690782+0000", 2023-10-01T09:44:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010232722 2023-10-01T09:44:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701015+0000", 2023-10-01T09:44:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1299e-05 2023-10-01T09:44:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701046+0000", 2023-10-01T09:44:14.332 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7508000000000002e-05 2023-10-01T09:44:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701073+0000", 2023-10-01T09:44:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0361e-05 2023-10-01T09:44:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701094+0000", 2023-10-01T09:44:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.81483497999999999 2023-10-01T09:44:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.333 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148060 9.3 9:cf7f7f25:::benchmark_data_smithi138_497829_object45675:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:44:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:22.972375+0000", 2023-10-01T09:44:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.27682351700003, 2023-10-01T09:44:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.72881600400000002, 2023-10-01T09:44:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.334 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148060 2023-10-01T09:44:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972375+0000", 2023-10-01T09:44:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972375+0000", 2023-10-01T09:44:14.335 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:44:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972374+0000", 2023-10-01T09:44:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1189000000000001e-05 2023-10-01T09:44:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972385+0000", 2023-10-01T09:44:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7291000000000001e-05 2023-10-01T09:44:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.336 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972402+0000", 2023-10-01T09:44:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9349999999999997e-06 2023-10-01T09:44:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972409+0000", 2023-10-01T09:44:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.8267000000000003e-05 2023-10-01T09:44:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.337 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972467+0000", 2023-10-01T09:44:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1688999999999999e-05 2023-10-01T09:44:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:22.972479+0000", 2023-10-01T09:44:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.70958122800000001 2023-10-01T09:44:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682060+0000", 2023-10-01T09:44:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3533000000000001e-05 2023-10-01T09:44:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.338 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682144+0000", 2023-10-01T09:44:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000118854 2023-10-01T09:44:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:44:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682263+0000", 2023-10-01T09:44:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012972465000000001 2023-10-01T09:44:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.339 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.695235+0000", 2023-10-01T09:44:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0058802189999999999 2023-10-01T09:44:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701115+0000", 2023-10-01T09:44:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5998999999999999e-05 2023-10-01T09:44:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701151+0000", 2023-10-01T09:44:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0852999999999998e-05 2023-10-01T09:44:14.340 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701172+0000", 2023-10-01T09:44:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9335999999999999e-05 2023-10-01T09:44:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701191+0000", 2023-10-01T09:44:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.72881600400000002 2023-10-01T09:44:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.341 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148074 9.c 9:3b5a7b44:::benchmark_data_smithi138_497829_object45689:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:44:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.007110+0000", 2023-10-01T09:44:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.24208919199998, 2023-10-01T09:44:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.68042775499999997, 2023-10-01T09:44:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148074 2023-10-01T09:44:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.342 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007110+0000", 2023-10-01T09:44:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007110+0000", 2023-10-01T09:44:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:44:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007108+0000", 2023-10-01T09:44:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4759999999999998e-06 2023-10-01T09:44:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007116+0000", 2023-10-01T09:44:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5668999999999999e-05 2023-10-01T09:44:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007132+0000", 2023-10-01T09:44:14.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6610000000000003e-06 2023-10-01T09:44:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007139+0000", 2023-10-01T09:44:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1598000000000003e-05 2023-10-01T09:44:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007190+0000", 2023-10-01T09:44:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0829999999999998e-06 2023-10-01T09:44:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.345 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.007197+0000", 2023-10-01T09:44:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.66635359100000002 2023-10-01T09:44:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673551+0000", 2023-10-01T09:44:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.0988000000000004e-05 2023-10-01T09:44:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673642+0000", 2023-10-01T09:44:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010931599999999999 2023-10-01T09:44:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:44:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.673751+0000", 2023-10-01T09:44:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0089520429999999998 2023-10-01T09:44:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682703+0000", 2023-10-01T09:44:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00020024499999999999 2023-10-01T09:44:14.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682903+0000", 2023-10-01T09:44:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0045564280000000004 2023-10-01T09:44:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.687460+0000", 2023-10-01T09:44:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6650999999999999e-05 2023-10-01T09:44:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.348 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.687496+0000", 2023-10-01T09:44:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0963000000000001e-05 2023-10-01T09:44:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.687537+0000", 2023-10-01T09:44:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.68042775499999997 2023-10-01T09:44:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148078 9.3 9:c8b0a7d5:::benchmark_data_smithi138_497829_object45693:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:44:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.015312+0000", 2023-10-01T09:44:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.23388726399997, 2023-10-01T09:44:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.70011192700000002, 2023-10-01T09:44:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148078 2023-10-01T09:44:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015312+0000", 2023-10-01T09:44:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015312+0000", 2023-10-01T09:44:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:44:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015310+0000", 2023-10-01T09:44:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7510000000000008e-06 2023-10-01T09:44:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015317+0000", 2023-10-01T09:44:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2976000000000001e-05 2023-10-01T09:44:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015330+0000", 2023-10-01T09:44:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5339999999999999e-06 2023-10-01T09:44:14.352 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015336+0000", 2023-10-01T09:44:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5003999999999997e-05 2023-10-01T09:44:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015391+0000", 2023-10-01T09:44:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5939999999999999e-06 2023-10-01T09:44:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.353 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.015398+0000", 2023-10-01T09:44:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.67108258300000001 2023-10-01T09:44:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686480+0000", 2023-10-01T09:44:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000109298 2023-10-01T09:44:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686589+0000", 2023-10-01T09:44:14.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013103600000000001 2023-10-01T09:44:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:44:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.686720+0000", 2023-10-01T09:44:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013092273999999999 2023-10-01T09:44:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.699813+0000", 2023-10-01T09:44:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001319221 2023-10-01T09:44:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.701132+0000", 2023-10-01T09:44:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014181050000000001 2023-10-01T09:44:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715313+0000", 2023-10-01T09:44:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0659000000000002e-05 2023-10-01T09:44:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715384+0000", 2023-10-01T09:44:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9857999999999997e-05 2023-10-01T09:44:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715424+0000", 2023-10-01T09:44:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.70011192700000002 2023-10-01T09:44:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.357 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148091 9.3 9:c92a3cf4:::benchmark_data_smithi138_497829_object45706:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:44:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.098648+0000", 2023-10-01T09:44:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.15055131299999, 2023-10-01T09:44:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.61693706400000003, 2023-10-01T09:44:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148091 2023-10-01T09:44:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.358 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098648+0000", 2023-10-01T09:44:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098648+0000", 2023-10-01T09:44:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:44:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.359 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098645+0000", 2023-10-01T09:44:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3414000000000001e-05 2023-10-01T09:44:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098658+0000", 2023-10-01T09:44:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5611e-05 2023-10-01T09:44:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098674+0000", 2023-10-01T09:44:14.360 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0550000000000002e-06 2023-10-01T09:44:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098681+0000", 2023-10-01T09:44:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8112999999999998e-05 2023-10-01T09:44:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098729+0000", 2023-10-01T09:44:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.7700000000000001e-06 2023-10-01T09:44:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.098737+0000", 2023-10-01T09:44:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.59207254099999995 2023-10-01T09:44:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.690810+0000", 2023-10-01T09:44:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011513 2023-10-01T09:44:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.690925+0000", 2023-10-01T09:44:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013793200000000001 2023-10-01T09:44:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:44:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.691063+0000", 2023-10-01T09:44:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0087986049999999993 2023-10-01T09:44:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.699861+0000", 2023-10-01T09:44:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.01539976 2023-10-01T09:44:14.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715261+0000", 2023-10-01T09:44:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00022492599999999999 2023-10-01T09:44:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715486+0000", 2023-10-01T09:44:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1239000000000002e-05 2023-10-01T09:44:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715537+0000", 2023-10-01T09:44:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7519999999999999e-05 2023-10-01T09:44:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715585+0000", 2023-10-01T09:44:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.61693706400000003 2023-10-01T09:44:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148104 9.c 9:3af9d94a:::benchmark_data_smithi138_497829_object45719:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:44:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.131588+0000", 2023-10-01T09:44:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.11761117600003, 2023-10-01T09:44:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55710424700000005, 2023-10-01T09:44:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148104 2023-10-01T09:44:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.366 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131588+0000", 2023-10-01T09:44:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131588+0000", 2023-10-01T09:44:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:44:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.367 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131586+0000", 2023-10-01T09:44:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3512e-05 2023-10-01T09:44:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131599+0000", 2023-10-01T09:44:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7265000000000002e-05 2023-10-01T09:44:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131616+0000", 2023-10-01T09:44:14.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5329999999999997e-06 2023-10-01T09:44:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131624+0000", 2023-10-01T09:44:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5498999999999999e-05 2023-10-01T09:44:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131670+0000", 2023-10-01T09:44:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1920000000000005e-06 2023-10-01T09:44:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.131678+0000", 2023-10-01T09:44:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.54630871999999997 2023-10-01T09:44:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.677986+0000", 2023-10-01T09:44:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000109057 2023-10-01T09:44:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.370 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.678095+0000", 2023-10-01T09:44:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013558 2023-10-01T09:44:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:44:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.678231+0000", 2023-10-01T09:44:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0046248979999999997 2023-10-01T09:44:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682856+0000", 2023-10-01T09:44:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7060999999999999e-05 2023-10-01T09:44:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682943+0000", 2023-10-01T09:44:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.005695922 2023-10-01T09:44:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688639+0000", 2023-10-01T09:44:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9950999999999998e-05 2023-10-01T09:44:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688669+0000", 2023-10-01T09:44:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3079e-05 2023-10-01T09:44:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688692+0000", 2023-10-01T09:44:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55710424700000005 2023-10-01T09:44:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.373 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148105 9.c 9:3f26355b:::benchmark_data_smithi138_497829_object45720:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:44:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.132784+0000", 2023-10-01T09:44:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.11641449000001, 2023-10-01T09:44:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55601706699999998, 2023-10-01T09:44:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148105 2023-10-01T09:44:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132784+0000", 2023-10-01T09:44:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132784+0000", 2023-10-01T09:44:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:44:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132784+0000", 2023-10-01T09:44:14.375 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2379999999999997e-06 2023-10-01T09:44:14.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.376 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132792+0000", 2023-10-01T09:44:14.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0162e-05 2023-10-01T09:44:14.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.376 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132802+0000", 2023-10-01T09:44:14.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5860000000000004e-06 2023-10-01T09:44:14.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132808+0000", 2023-10-01T09:44:14.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5326e-05 2023-10-01T09:44:14.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132843+0000", 2023-10-01T09:44:14.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6760000000000004e-06 2023-10-01T09:44:14.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.132849+0000", 2023-10-01T09:44:14.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55011066399999997 2023-10-01T09:44:14.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.682959+0000", 2023-10-01T09:44:14.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9478000000000001e-05 2023-10-01T09:44:14.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.683029+0000", 2023-10-01T09:44:14.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.2601999999999994e-05 2023-10-01T09:44:14.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:44:14.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.683101+0000", 2023-10-01T09:44:14.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0044813810000000004 2023-10-01T09:44:14.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.687583+0000", 2023-10-01T09:44:14.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000419534 2023-10-01T09:44:14.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688002+0000", 2023-10-01T09:44:14.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00075196499999999995 2023-10-01T09:44:14.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.380 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688754+0000", 2023-10-01T09:44:14.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6500999999999999e-05 2023-10-01T09:44:14.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.380 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.380 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688781+0000", 2023-10-01T09:44:14.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0809999999999999e-05 2023-10-01T09:44:14.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.688801+0000", 2023-10-01T09:44:14.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.55601706699999998 2023-10-01T09:44:14.381 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.381 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.381 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.381 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.46067.0:148110 9.3 9:caeedb58:::benchmark_data_smithi138_497829_object45725:head [delete] snapc 0=[] ondisk+write+known_if_redirected e446)", 2023-10-01T09:44:14.381 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:37:23.149195+0000", 2023-10-01T09:44:14.381 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 411.10000399199998, 2023-10-01T09:44:14.382 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.57476550699999995, 2023-10-01T09:44:14.382 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.382 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.382 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.382 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.46067", 2023-10-01T09:44:14.382 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:44:14.382 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 148110 2023-10-01T09:44:14.382 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.382 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.382 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.382 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.382 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149195+0000", 2023-10-01T09:44:14.383 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.383 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.383 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.383 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.383 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149195+0000", 2023-10-01T09:44:14.383 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:44:14.383 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.383 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.383 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.383 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149193+0000", 2023-10-01T09:44:14.383 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1443999999999999e-05 2023-10-01T09:44:14.383 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149205+0000", 2023-10-01T09:44:14.384 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0498e-05 2023-10-01T09:44:14.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149215+0000", 2023-10-01T09:44:14.384 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2490000000000001e-06 2023-10-01T09:44:14.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149220+0000", 2023-10-01T09:44:14.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5479999999999998e-05 2023-10-01T09:44:14.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149266+0000", 2023-10-01T09:44:14.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.516e-06 2023-10-01T09:44:14.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for peered", 2023-10-01T09:44:14.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.149272+0000", 2023-10-01T09:44:14.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.54601357399999995 2023-10-01T09:44:14.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.386 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.695286+0000", 2023-10-01T09:44:14.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000100777 2023-10-01T09:44:14.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.386 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.695387+0000", 2023-10-01T09:44:14.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.5045999999999999e-05 2023-10-01T09:44:14.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.386 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 1,3", 2023-10-01T09:44:14.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.695472+0000", 2023-10-01T09:44:14.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0047012119999999998 2023-10-01T09:44:14.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.700173+0000", 2023-10-01T09:44:14.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015477148 2023-10-01T09:44:14.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.715650+0000", 2023-10-01T09:44:14.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0082495999999999993 2023-10-01T09:44:14.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.388 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.723900+0000", 2023-10-01T09:44:14.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4461000000000001e-05 2023-10-01T09:44:14.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.388 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.723934+0000", 2023-10-01T09:44:14.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6350999999999999e-05 2023-10-01T09:44:14.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.389 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.389 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.389 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:37:23.723960+0000", 2023-10-01T09:44:14.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.57476550699999995 2023-10-01T09:44:14.389 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.389 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.389 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.389 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.389 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.389 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:43770 10.8 10:128672d5:::benchmark_data_smithi138_573643_object43769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-10-01T09:44:14.389 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:39.471235+0000", 2023-10-01T09:44:14.389 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 214.77796381100001, 2023-10-01T09:44:14.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.30291565999999998, 2023-10-01T09:44:14.390 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.390 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.390 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.390 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:14.390 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:14.390 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 43770 2023-10-01T09:44:14.390 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.390 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.390 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.390 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.390 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471235+0000", 2023-10-01T09:44:14.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.391 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.391 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471235+0000", 2023-10-01T09:44:14.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:44:14.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.391 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.391 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471233+0000", 2023-10-01T09:44:14.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5883000000000006e-05 2023-10-01T09:44:14.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471299+0000", 2023-10-01T09:44:14.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2113000000000001e-05 2023-10-01T09:44:14.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471311+0000", 2023-10-01T09:44:14.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.045e-06 2023-10-01T09:44:14.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471317+0000", 2023-10-01T09:44:14.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6418999999999997e-05 2023-10-01T09:44:14.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471363+0000", 2023-10-01T09:44:14.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013875700000000001 2023-10-01T09:44:14.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471502+0000", 2023-10-01T09:44:14.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000163809 2023-10-01T09:44:14.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.394 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:44:14.394 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.471666+0000", 2023-10-01T09:44:14.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00176851 2023-10-01T09:44:14.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.394 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.394 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.473434+0000", 2023-10-01T09:44:14.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00060446899999999995 2023-10-01T09:44:14.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.474038+0000", 2023-10-01T09:44:14.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.30000729500000001 2023-10-01T09:44:14.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.774046+0000", 2023-10-01T09:44:14.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6264000000000002e-05 2023-10-01T09:44:14.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.774102+0000", 2023-10-01T09:44:14.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8707e-05 2023-10-01T09:44:14.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.774151+0000", 2023-10-01T09:44:14.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.30291565999999998 2023-10-01T09:44:14.396 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.396 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.396 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.397 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:43786 10.8 10:176af140:::benchmark_data_smithi138_573643_object43785:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-10-01T09:44:14.397 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:39.487738+0000", 2023-10-01T09:44:14.397 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 214.76146126399999, 2023-10-01T09:44:14.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28799713799999999, 2023-10-01T09:44:14.397 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.397 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.397 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.397 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:14.397 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:14.397 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 43786 2023-10-01T09:44:14.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.398 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487738+0000", 2023-10-01T09:44:14.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487738+0000", 2023-10-01T09:44:14.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:44:14.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487736+0000", 2023-10-01T09:44:14.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3594e-05 2023-10-01T09:44:14.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487820+0000", 2023-10-01T09:44:14.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2801e-05 2023-10-01T09:44:14.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487833+0000", 2023-10-01T09:44:14.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3660000000000003e-06 2023-10-01T09:44:14.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.400 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487840+0000", 2023-10-01T09:44:14.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0319000000000003e-05 2023-10-01T09:44:14.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.400 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.400 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487900+0000", 2023-10-01T09:44:14.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014719200000000001 2023-10-01T09:44:14.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.488048+0000", 2023-10-01T09:44:14.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000160569 2023-10-01T09:44:14.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,5", 2023-10-01T09:44:14.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.488208+0000", 2023-10-01T09:44:14.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001939374 2023-10-01T09:44:14.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.402 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.490148+0000", 2023-10-01T09:44:14.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0023000109999999998 2023-10-01T09:44:14.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.402 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.492448+0000", 2023-10-01T09:44:14.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28318099499999999 2023-10-01T09:44:14.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.402 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.775629+0000", 2023-10-01T09:44:14.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.4634999999999994e-05 2023-10-01T09:44:14.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.775693+0000", 2023-10-01T09:44:14.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1610000000000003e-05 2023-10-01T09:44:14.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.775735+0000", 2023-10-01T09:44:14.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28799713799999999 2023-10-01T09:44:14.403 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.404 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.404 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.404 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:46603 10.1 10:852eafe3:::benchmark_data_smithi138_573643_object46602:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e488)", 2023-10-01T09:44:14.404 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.281765+0000", 2023-10-01T09:44:14.404 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 211.96743405500001, 2023-10-01T09:44:14.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28067689299999998, 2023-10-01T09:44:14.404 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:14.404 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:14.404 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:14.404 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:14.405 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:14.405 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 46603 2023-10-01T09:44:14.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.405 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:14.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:14.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.281765+0000", 2023-10-01T09:44:14.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:14.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:14.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.281765+0000", 2023-10-01T09:44:14.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:44:14.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:14.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.281764+0000", 2023-10-01T09:44:14.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8473000000000002e-05 2023-10-01T09:44:14.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:14.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.281803+0000", 2023-10-01T09:44:14.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7389999999999999e-06 2023-10-01T09:44:14.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:14.407 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.281808+0000", 2023-10-01T09:44:14.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.886e-06 2023-10-01T09:44:14.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.407 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:14.407 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.281810+0000", 2023-10-01T09:44:14.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001121457 2023-10-01T09:44:14.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.407 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:14.407 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.282932+0000", 2023-10-01T09:44:14.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1827000000000002e-05 2023-10-01T09:44:14.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:14.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.283004+0000", 2023-10-01T09:44:14.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.6164999999999999e-05 2023-10-01T09:44:14.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:44:14.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.283090+0000", 2023-10-01T09:44:14.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00082320799999999997 2023-10-01T09:44:14.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:44:14.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.283913+0000", 2023-10-01T09:44:14.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002203132 2023-10-01T09:44:14.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.286116+0000", 2023-10-01T09:44:14.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.276216884 2023-10-01T09:44:14.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:14.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.562333+0000", 2023-10-01T09:44:14.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1144999999999997e-05 2023-10-01T09:44:14.410 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.410 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.410 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:44:14.410 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.562404+0000", 2023-10-01T09:44:14.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7435999999999999e-05 2023-10-01T09:44:14.410 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:14.410 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:14.410 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:44:14.410 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.562442+0000", 2023-10-01T09:44:14.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28067689299999998 2023-10-01T09:44:14.410 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.411 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.411 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.411 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:14.411 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:14.411 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:44:14.411 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:14.441 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:14 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:14 smithi190 ceph-mon[160435]: failed to encode map e528 with expected crc 2023-10-01T09:44:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:14 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:44:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:14 smithi190 ceph-mon[160435]: osdmap e528: 8 total, 8 up, 7 in 2023-10-01T09:44:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:14 smithi190 ceph-mon[160435]: failed to encode map e528 with expected crc 2023-10-01T09:44:14.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/61968915' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:44:14.686 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:14.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:14.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:14.791 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:14 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:14 smithi138 ceph-mon[192513]: failed to encode map e528 with expected crc 2023-10-01T09:44:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:14 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:44:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:14 smithi138 ceph-mon[192513]: osdmap e528: 8 total, 8 up, 7 in 2023-10-01T09:44:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:14 smithi138 ceph-mon[192513]: failed to encode map e528 with expected crc 2023-10-01T09:44:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/61968915' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:44:14.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:14 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:14 smithi138 ceph-mon[185886]: failed to encode map e528 with expected crc 2023-10-01T09:44:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:14 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:44:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:14 smithi138 ceph-mon[185886]: osdmap e528: 8 total, 8 up, 7 in 2023-10-01T09:44:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:14 smithi138 ceph-mon[185886]: failed to encode map e528 with expected crc 2023-10-01T09:44:14.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/61968915' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:44:14.822 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:14.982 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:44:14.982 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:44:15.264 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:44:15.295 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:44:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:44:14] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.33.4" 2023-10-01T09:44:15.314 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:15.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:15.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:15.422 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:15.450 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:15.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:15 smithi190 ceph-mon[160435]: pgmap v1535: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 127 op/s; 7591/193947 objects degraded (3.914%); 3965/193947 objects misplaced (2.044%); 680 KiB/s, 10 objects/s recovering 2023-10-01T09:44:15.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:15 smithi138 ceph-mon[185886]: pgmap v1535: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 127 op/s; 7591/193947 objects degraded (3.914%); 3965/193947 objects misplaced (2.044%); 680 KiB/s, 10 objects/s recovering 2023-10-01T09:44:15.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:15 smithi138 ceph-mon[192513]: pgmap v1535: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 127 op/s; 7591/193947 objects degraded (3.914%); 3965/193947 objects misplaced (2.044%); 680 KiB/s, 10 objects/s recovering 2023-10-01T09:44:15.680 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:15.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:15.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:15.785 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:15.837 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:15.908 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:15.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:15.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:15.934 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:44:16.015 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:16.045 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:16.075 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:16.146 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:16.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:16.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:16.250 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:16.279 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:16.472 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:44:16.480 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:16.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:16.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:16.504 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:44:16.528 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:44:16.551 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:44:16.580 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:44:16.587 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:16.603 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:44:16.622 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:44:16.623 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:16.640 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:44:16.641 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:44:16.642 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:44:16.642 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:44:16.642 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:44:16.642 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:44:16.642 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:44:16.642 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:44:16.642 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:44:16.676 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:16.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:16.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:16.783 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:16.810 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:17.102 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:44:17.178 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:17.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:17.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:17.282 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:17.314 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:17.395 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:17.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:17.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:17.499 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:17.499 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:17 smithi190 ceph-mon[160435]: pgmap v1536: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 132 op/s; 7591/193113 objects degraded (3.931%); 3965/193113 objects misplaced (2.053%); 552 KiB/s, 8 objects/s recovering 2023-10-01T09:44:17.500 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:17 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2252922927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:17.500 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:17 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:17.500 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:17 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:17.539 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:17.678 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:17 smithi138 ceph-mon[192513]: pgmap v1536: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 132 op/s; 7591/193113 objects degraded (3.931%); 3965/193113 objects misplaced (2.053%); 552 KiB/s, 8 objects/s recovering 2023-10-01T09:44:17.678 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:17 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2252922927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:17.678 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:17 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:17.678 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:17 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:17.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:17 smithi138 ceph-mon[185886]: pgmap v1536: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 132 op/s; 7591/193113 objects degraded (3.931%); 3965/193113 objects misplaced (2.053%); 552 KiB/s, 8 objects/s recovering 2023-10-01T09:44:17.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:17 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2252922927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:17.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:17 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:17.679 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:17 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:17.692 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:17.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:17.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:17.778 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:44:17.779 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:44:17.796 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:17.825 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:18.086 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:18.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:18.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:18.190 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:18.221 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:18.379 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:18 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7591/193113 objects degraded (3.931%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:18.478 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:18.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:18.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:18.547 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:44:18.583 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:18.614 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:18.633 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:18.668 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:18.672 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:18 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7591/193113 objects degraded (3.931%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:18.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:18.776 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:18.802 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:18 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7591/193113 objects degraded (3.931%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:18.805 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:18.853 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:44:18.855 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:44:18.859 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:18.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:18.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:18.951 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:44:18.963 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:18.992 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:19.084 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:19.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:19.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:19.188 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:19.221 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:19.278 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:19.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:19.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:19.385 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:19.415 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:19.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:19 smithi138 ceph-mon[192513]: pgmap v1537: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 7557/192000 objects degraded (3.936%); 3920/192000 objects misplaced (2.042%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:44:19.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:19 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:19.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:19 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:19 smithi138 ceph-mon[185886]: pgmap v1537: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 7557/192000 objects degraded (3.936%); 3920/192000 objects misplaced (2.042%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:44:19.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:19 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:19.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:19 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:19.786 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1537,"stamp":"2023-10-01T09:44:18.322245+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4197193675,"num_objects":64000,"num_object_clones":0,"num_object_copies":192000,"num_objects_missing_on_primary":2457,"num_objects_missing":2457,"num_objects_degraded":7557,"num_objects_misplaced":3920,"num_objects_unfound":0,"num_objects_dirty":64000,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":103574,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8650,"num_bytes_recovered":553747471,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103638,"ondisk_log_size":103638,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17737732,"kb_used_data":12717196,"kb_used_omap":39,"kb_used_meta":5020440,"kb_avail":638478332,"statfs":{"total":671965249536,"available":653801811968,"internally_reserved":0,"allocated":13022408704,"data_stored":13012547758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5140931183},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,0,0,0,1,0,0,3,5,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-83034112,"num_objects":-1267,"num_object_clones":0,"num_object_copies":-3801,"num_objects_missing_on_primary":-81,"num_objects_missing":-81,"num_objects_degraded":-80,"num_objects_misplaced":-85,"num_objects_unfound":0,"num_objects_dirty":-1267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1267,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":5505024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004006"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":304,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.475703+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:44:06.475703+0000","last_peered":"2023-10-01T09:44:06.475703+0000","last_clean":"2023-10-01T09:44:06.475703+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:44:06.475703+0000","last_undegraded":"2023-10-01T09:44:06.475703+0000","last_fullsized":"2023-10-01T09:44:06.475703+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":147,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486928+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:44:06.486928+0000","last_peered":"2023-10-01T09:44:06.486928+0000","last_clean":"2023-10-01T09:44:06.486928+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:44:06.486928+0000","last_undegraded":"2023-10-01T09:44:06.486928+0000","last_fullsized":"2023-10-01T09:44:06.486928+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":528,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486545+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:44:06.486545+0000","last_peered":"2023-10-01T09:44:06.486545+0000","last_clean":"2023-10-01T09:44:06.486545+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:44:06.486545+0000","last_undegraded":"2023-10-01T09:44:06.486545+0000","last_fullsized":"2023-10-01T09:44:06.486545+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":586,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.474963+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:44:06.474963+0000","last_peered":"2023-10-01T09:44:06.474963+0000","last_clean":"2023-10-01T09:44:06.474963+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:44:06.474963+0000","last_undegraded":"2023-10-01T09:44:06.474963+0000","last_fullsized":"2023-10-01T09:44:06.474963+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":467,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.473701+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:44:06.473701+0000","last_peered":"2023-10-01T09:44:06.473701+0000","last_clean":"2023-10-01T09:44:06.473701+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:44:06.473701+0000","last_undegraded":"2023-10-01T09:44:06.473701+0000","last_fullsized":"2023-10-01T09:44:06.473701+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":209,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.573025+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:44:06.573025+0000","last_peered":"2023-10-01T09:44:06.573025+0000","last_clean":"2023-10-01T09:44:06.573025+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:44:06.573025+0000","last_undegraded":"2023-10-01T09:44:06.573025+0000","last_fullsized":"2023-10-01T09:44:06.573025+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":585,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.573144+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:44:06.573144+0000","last_peered":"2023-10-01T09:44:06.573144+0000","last_clean":"2023-10-01T09:44:06.573144+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:44:06.573144+0000","last_undegraded":"2023-10-01T09:44:06.573144+0000","last_fullsized":"2023-10-01T09:44:06.573144+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":514,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486299+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:44:06.486299+0000","last_peered":"2023-10-01T09:44:06.486299+0000","last_clean":"2023-10-01T09:44:06.486299+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:44:06.486299+0000","last_undegraded":"2023-10-01T09:44:06.486299+0000","last_fullsized":"2023-10-01T09:44:06.486299+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":240,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.488170+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:44:06.488170+0000","last_peered":"2023-10-01T09:44:06.488170+0000","last_clean":"2023-10-01T09:44:06.488170+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:44:06.488170+0000","last_undegraded":"2023-10-01T09:44:06.488170+0000","last_fullsized":"2023-10-01T09:44:06.488170+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":47,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.487982+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:44:06.487982+0000","last_peered":"2023-10-01T09:44:06.487982+0000","last_clean":"2023-10-01T09:44:06.487982+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:44:06.487982+0000","last_undegraded":"2023-10-01T09:44:06.487982+0000","last_fullsized":"2023-10-01T09:44:06.487982+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":53,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486851+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:44:06.486851+0000","last_peered":"2023-10-01T09:44:06.486851+0000","last_clean":"2023-10-01T09:44:06.486851+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:44:06.486851+0000","last_undegraded":"2023-10-01T09:44:06.486851+0000","last_fullsized":"2023-10-01T09:44:06.486851+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":51,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.487841+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:44:06.487841+0000","last_peered":"2023-10-01T09:44:06.487841+0000","last_clean":"2023-10-01T09:44:06.487841+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:44:06.487841+0000","last_undegraded":"2023-10-01T09:44:06.487841+0000","last_fullsized":"2023-10-01T09:44:06.487841+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":429,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.475959+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:44:06.475959+0000","last_peered":"2023-10-01T09:44:06.475959+0000","last_clean":"2023-10-01T09:44:06.475959+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:44:06.475959+0000","last_undegraded":"2023-10-01T09:44:06.475959+0000","last_fullsized":"2023-10-01T09:44:06.475959+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":310,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.475697+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:44:06.475697+0000","last_peered":"2023-10-01T09:44:06.475697+0000","last_clean":"2023-10-01T09:44:06.475697+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:44:06.475697+0000","last_undegraded":"2023-10-01T09:44:06.475697+0000","last_fullsized":"2023-10-01T09:44:06.475697+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":646,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486237+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:44:06.486237+0000","last_peered":"2023-10-01T09:44:06.486237+0000","last_clean":"2023-10-01T09:44:06.486237+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:44:06.486237+0000","last_undegraded":"2023-10-01T09:44:06.486237+0000","last_fullsized":"2023-10-01T09:44:06.486237+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":383,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486876+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:44:06.486876+0000","last_peered":"2023-10-01T09:44:06.486876+0000","last_clean":"2023-10-01T09:44:06.486876+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:44:06.486876+0000","last_undegraded":"2023-10-01T09:44:06.486876+0000","last_fullsized":"2023-10-01T09:44:06.486876+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":56,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486078+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:44:06.486078+0000","last_peered":"2023-10-01T09:44:06.486078+0000","last_clean":"2023-10-01T09:44:06.486078+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:44:06.486078+0000","last_undegraded":"2023-10-01T09:44:06.486078+0000","last_fullsized":"2023-10-01T09:44:06.486078+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":42,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486396+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:44:06.486396+0000","last_peered":"2023-10-01T09:44:06.486396+0000","last_clean":"2023-10-01T09:44:06.486396+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:44:06.486396+0000","last_undegraded":"2023-10-01T09:44:06.486396+0000","last_fullsized":"2023-10-01T09:44:06.486396+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":539,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486596+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:44:06.486596+0000","last_peered":"2023-10-01T09:44:06.486596+0000","last_clean":"2023-10-01T09:44:06.486596+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:44:06.486596+0000","last_undegraded":"2023-10-01T09:44:06.486596+0000","last_fullsized":"2023-10-01T09:44:06.486596+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":550,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.475030+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:44:06.475030+0000","last_peered":"2023-10-01T09:44:06.475030+0000","last_clean":"2023-10-01T09:44:06.475030+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:44:06.475030+0000","last_undegraded":"2023-10-01T09:44:06.475030+0000","last_fullsized":"2023-10-01T09:44:06.475030+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":406,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.473621+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:44:06.473621+0000","last_peered":"2023-10-01T09:44:06.473621+0000","last_clean":"2023-10-01T09:44:06.473621+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:44:06.473621+0000","last_undegraded":"2023-10-01T09:44:06.473621+0000","last_fullsized":"2023-10-01T09:44:06.473621+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":56,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.487004+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:44:06.487004+0000","last_peered":"2023-10-01T09:44:06.487004+0000","last_clean":"2023-10-01T09:44:06.487004+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:44:06.487004+0000","last_undegraded":"2023-10-01T09:44:06.487004+0000","last_fullsized":"2023-10-01T09:44:06.487004+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"528'6441","reported_seq":6502,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:16.465104+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:44:16.465104+0000","last_peered":"2023-10-01T09:44:16.465104+0000","last_clean":"2023-10-01T09:44:16.465104+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:44:16.465104+0000","last_undegraded":"2023-10-01T09:44:16.465104+0000","last_fullsized":"2023-10-01T09:44:16.465104+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6441,"log_dups_size":0,"ondisk_log_size":6441,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258932736,"num_objects":3951,"num_object_clones":0,"num_object_copies":11853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6441,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":51,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486843+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:44:06.486843+0000","last_peered":"2023-10-01T09:44:06.486843+0000","last_clean":"2023-10-01T09:44:06.486843+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:44:06.486843+0000","last_undegraded":"2023-10-01T09:44:06.486843+0000","last_fullsized":"2023-10-01T09:44:06.486843+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":248,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.485987+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:44:06.485987+0000","last_peered":"2023-10-01T09:44:06.485987+0000","last_clean":"2023-10-01T09:44:06.485987+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:44:06.485987+0000","last_undegraded":"2023-10-01T09:44:06.485987+0000","last_fullsized":"2023-10-01T09:44:06.485987+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"528'6432","reported_seq":10208,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:44:16.640382+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:44:16.640382+0000","last_peered":"2023-10-01T09:44:16.640382+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:44:16.640382+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6432,"log_dups_size":0,"ondisk_log_size":6432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260440064,"num_objects":3974,"num_object_clones":0,"num_object_copies":11922,"num_objects_missing_on_primary":2457,"num_objects_missing":2457,"num_objects_degraded":6363,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6432,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1229,"num_bytes_recovered":80543744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1449},{"shards":"0,4,6","objects":2457},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":522,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486749+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:44:06.486749+0000","last_peered":"2023-10-01T09:44:06.486749+0000","last_clean":"2023-10-01T09:44:06.486749+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:44:06.486749+0000","last_undegraded":"2023-10-01T09:44:06.486749+0000","last_fullsized":"2023-10-01T09:44:06.486749+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":583,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486467+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:44:06.486467+0000","last_peered":"2023-10-01T09:44:06.486467+0000","last_clean":"2023-10-01T09:44:06.486467+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:44:06.486467+0000","last_undegraded":"2023-10-01T09:44:06.486467+0000","last_fullsized":"2023-10-01T09:44:06.486467+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"528'6446","reported_seq":6509,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:15.891129+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:44:15.891129+0000","last_peered":"2023-10-01T09:44:15.891129+0000","last_clean":"2023-10-01T09:44:15.891129+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:44:15.891129+0000","last_undegraded":"2023-10-01T09:44:15.891129+0000","last_fullsized":"2023-10-01T09:44:15.891129+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6446,"log_dups_size":0,"ondisk_log_size":6446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261226496,"num_objects":3986,"num_object_clones":0,"num_object_copies":11958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6446,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":608,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486930+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:44:06.486930+0000","last_peered":"2023-10-01T09:44:06.486930+0000","last_clean":"2023-10-01T09:44:06.486930+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:44:06.486930+0000","last_undegraded":"2023-10-01T09:44:06.486930+0000","last_fullsized":"2023-10-01T09:44:06.486930+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":404,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.488064+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:44:06.488064+0000","last_peered":"2023-10-01T09:44:06.488064+0000","last_clean":"2023-10-01T09:44:06.488064+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:44:06.488064+0000","last_undegraded":"2023-10-01T09:44:06.488064+0000","last_fullsized":"2023-10-01T09:44:06.488064+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"528'6384","reported_seq":13052,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:16.509473+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:44:16.509473+0000","last_peered":"2023-10-01T09:44:16.509473+0000","last_clean":"2023-10-01T09:44:16.509473+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:44:16.509473+0000","last_undegraded":"2023-10-01T09:44:16.509473+0000","last_fullsized":"2023-10-01T09:44:16.509473+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6384,"log_dups_size":0,"ondisk_log_size":6384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263323648,"num_objects":4018,"num_object_clones":0,"num_object_copies":12054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6384,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":248,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486541+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:44:06.486541+0000","last_peered":"2023-10-01T09:44:06.486541+0000","last_clean":"2023-10-01T09:44:06.486541+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:44:06.486541+0000","last_undegraded":"2023-10-01T09:44:06.486541+0000","last_fullsized":"2023-10-01T09:44:06.486541+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":506,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.474139+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:44:06.474139+0000","last_peered":"2023-10-01T09:44:06.474139+0000","last_clean":"2023-10-01T09:44:06.474139+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:44:06.474139+0000","last_undegraded":"2023-10-01T09:44:06.474139+0000","last_fullsized":"2023-10-01T09:44:06.474139+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"528'6405","reported_seq":6468,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:15.124101+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:44:15.124101+0000","last_peered":"2023-10-01T09:44:15.124101+0000","last_clean":"2023-10-01T09:44:15.124101+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:44:15.124101+0000","last_undegraded":"2023-10-01T09:44:15.124101+0000","last_fullsized":"2023-10-01T09:44:15.124101+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6405,"log_dups_size":0,"ondisk_log_size":6405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268894208,"num_objects":4103,"num_object_clones":0,"num_object_copies":12309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6405,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":108,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.475961+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:44:06.475961+0000","last_peered":"2023-10-01T09:44:06.475961+0000","last_clean":"2023-10-01T09:44:06.475961+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:44:06.475961+0000","last_undegraded":"2023-10-01T09:44:06.475961+0000","last_fullsized":"2023-10-01T09:44:06.475961+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":182,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.476112+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:44:06.476112+0000","last_peered":"2023-10-01T09:44:06.476112+0000","last_clean":"2023-10-01T09:44:06.476112+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:44:06.476112+0000","last_undegraded":"2023-10-01T09:44:06.476112+0000","last_fullsized":"2023-10-01T09:44:06.476112+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"528'6384","reported_seq":6487,"reported_epoch":528,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:44:16.671036+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:44:16.671036+0000","last_peered":"2023-10-01T09:44:16.671036+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:44:16.671036+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6384,"log_dups_size":0,"ondisk_log_size":6384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261095424,"num_objects":3984,"num_object_clones":0,"num_object_copies":11952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1184,"num_objects_misplaced":3920,"num_objects_unfound":0,"num_objects_dirty":3984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6384,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":1114112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":3920},{"shards":"4,5","objects":-1120}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":426,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.475762+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:44:06.475762+0000","last_peered":"2023-10-01T09:44:06.475762+0000","last_clean":"2023-10-01T09:44:06.475762+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:44:06.475762+0000","last_undegraded":"2023-10-01T09:44:06.475762+0000","last_fullsized":"2023-10-01T09:44:06.475762+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":586,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.487002+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:44:06.487002+0000","last_peered":"2023-10-01T09:44:06.487002+0000","last_clean":"2023-10-01T09:44:06.487002+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:44:06.487002+0000","last_undegraded":"2023-10-01T09:44:06.487002+0000","last_fullsized":"2023-10-01T09:44:06.487002+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"528'6386","reported_seq":6456,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:17.192986+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:44:17.192986+0000","last_peered":"2023-10-01T09:44:17.192986+0000","last_clean":"2023-10-01T09:44:17.192986+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:44:17.192986+0000","last_undegraded":"2023-10-01T09:44:17.192986+0000","last_fullsized":"2023-10-01T09:44:17.192986+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6386,"log_dups_size":0,"ondisk_log_size":6386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261619712,"num_objects":3992,"num_object_clones":0,"num_object_copies":11976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6386,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":422,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.475569+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:44:06.475569+0000","last_peered":"2023-10-01T09:44:06.475569+0000","last_clean":"2023-10-01T09:44:06.475569+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:44:06.475569+0000","last_undegraded":"2023-10-01T09:44:06.475569+0000","last_fullsized":"2023-10-01T09:44:06.475569+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":263,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.476373+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:44:06.476373+0000","last_peered":"2023-10-01T09:44:06.476373+0000","last_clean":"2023-10-01T09:44:06.476373+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:44:06.476373+0000","last_undegraded":"2023-10-01T09:44:06.476373+0000","last_fullsized":"2023-10-01T09:44:06.476373+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"528'6467","reported_seq":6528,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:16.031162+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:44:16.031162+0000","last_peered":"2023-10-01T09:44:16.031162+0000","last_clean":"2023-10-01T09:44:16.031162+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:44:16.031162+0000","last_undegraded":"2023-10-01T09:44:16.031162+0000","last_fullsized":"2023-10-01T09:44:16.031162+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6467,"log_dups_size":0,"ondisk_log_size":6467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264175616,"num_objects":4031,"num_object_clones":0,"num_object_copies":12093,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4031,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6467,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":391,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486349+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:44:06.486349+0000","last_peered":"2023-10-01T09:44:06.486349+0000","last_clean":"2023-10-01T09:44:06.486349+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:44:06.486349+0000","last_undegraded":"2023-10-01T09:44:06.486349+0000","last_fullsized":"2023-10-01T09:44:06.486349+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":438,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.487617+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:44:06.487617+0000","last_peered":"2023-10-01T09:44:06.487617+0000","last_clean":"2023-10-01T09:44:06.487617+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:44:06.487617+0000","last_undegraded":"2023-10-01T09:44:06.487617+0000","last_fullsized":"2023-10-01T09:44:06.487617+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"528'6301","reported_seq":6362,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:16.038078+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:44:16.038078+0000","last_peered":"2023-10-01T09:44:16.038078+0000","last_clean":"2023-10-01T09:44:16.038078+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:44:16.038078+0000","last_undegraded":"2023-10-01T09:44:16.038078+0000","last_fullsized":"2023-10-01T09:44:16.038078+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6301,"log_dups_size":0,"ondisk_log_size":6301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254083072,"num_objects":3877,"num_object_clones":0,"num_object_copies":11631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6301,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":921,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486736+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:44:06.486736+0000","last_peered":"2023-10-01T09:44:06.486736+0000","last_clean":"2023-10-01T09:44:06.486736+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:44:06.486736+0000","last_undegraded":"2023-10-01T09:44:06.486736+0000","last_fullsized":"2023-10-01T09:44:06.486736+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":258,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486542+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:44:06.486542+0000","last_peered":"2023-10-01T09:44:06.486542+0000","last_clean":"2023-10-01T09:44:06.486542+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:44:06.486542+0000","last_undegraded":"2023-10-01T09:44:06.486542+0000","last_fullsized":"2023-10-01T09:44:06.486542+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":51,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.487922+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:44:06.487922+0000","last_peered":"2023-10-01T09:44:06.487922+0000","last_clean":"2023-10-01T09:44:06.487922+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:44:06.487922+0000","last_undegraded":"2023-10-01T09:44:06.487922+0000","last_fullsized":"2023-10-01T09:44:06.487922+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"528'6338","reported_seq":6401,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:16.493358+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:44:16.493358+0000","last_peered":"2023-10-01T09:44:16.493358+0000","last_clean":"2023-10-01T09:44:16.493358+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:44:16.493358+0000","last_undegraded":"2023-10-01T09:44:16.493358+0000","last_fullsized":"2023-10-01T09:44:16.493358+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6338,"log_dups_size":0,"ondisk_log_size":6338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264241152,"num_objects":4032,"num_object_clones":0,"num_object_copies":12096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6338,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1193,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486093+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:44:06.486093+0000","last_peered":"2023-10-01T09:44:06.486093+0000","last_clean":"2023-10-01T09:44:06.486093+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:44:06.486093+0000","last_undegraded":"2023-10-01T09:44:06.486093+0000","last_fullsized":"2023-10-01T09:44:06.486093+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":403,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.475878+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:44:06.475878+0000","last_peered":"2023-10-01T09:44:06.475878+0000","last_clean":"2023-10-01T09:44:06.475878+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:44:06.475878+0000","last_undegraded":"2023-10-01T09:44:06.475878+0000","last_fullsized":"2023-10-01T09:44:06.475878+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":231,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486678+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:44:06.486678+0000","last_peered":"2023-10-01T09:44:06.486678+0000","last_clean":"2023-10-01T09:44:06.486678+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:44:06.486678+0000","last_undegraded":"2023-10-01T09:44:06.486678+0000","last_fullsized":"2023-10-01T09:44:06.486678+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"528'6425","reported_seq":13707,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:16.439455+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:44:16.439455+0000","last_peered":"2023-10-01T09:44:16.439455+0000","last_clean":"2023-10-01T09:44:16.439455+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:44:16.439455+0000","last_undegraded":"2023-10-01T09:44:16.439455+0000","last_fullsized":"2023-10-01T09:44:16.439455+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6425,"log_dups_size":0,"ondisk_log_size":6425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258932736,"num_objects":3951,"num_object_clones":0,"num_object_copies":11853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6425,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":873,"reported_epoch":527,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:44:06.473647+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:44:06.473647+0000","last_peered":"2023-10-01T09:44:06.473647+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:44:06.473647+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:44:06.473647+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":591,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.951764+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:44:13.951764+0000","last_peered":"2023-10-01T09:44:13.951764+0000","last_clean":"2023-10-01T09:44:13.951764+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:44:13.951764+0000","last_undegraded":"2023-10-01T09:44:13.951764+0000","last_fullsized":"2023-10-01T09:44:13.951764+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":51,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.487510+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:44:06.487510+0000","last_peered":"2023-10-01T09:44:06.487510+0000","last_clean":"2023-10-01T09:44:06.487510+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:44:06.487510+0000","last_undegraded":"2023-10-01T09:44:06.487510+0000","last_fullsized":"2023-10-01T09:44:06.487510+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"528'6306","reported_seq":6369,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:15.109041+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:44:15.109041+0000","last_peered":"2023-10-01T09:44:15.109041+0000","last_clean":"2023-10-01T09:44:15.109041+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:44:15.109041+0000","last_undegraded":"2023-10-01T09:44:15.109041+0000","last_fullsized":"2023-10-01T09:44:15.109041+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6306,"log_dups_size":0,"ondisk_log_size":6306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259915776,"num_objects":3966,"num_object_clones":0,"num_object_copies":11898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6306,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1644,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486225+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:44:06.486225+0000","last_peered":"2023-10-01T09:44:06.486225+0000","last_clean":"2023-10-01T09:44:06.486225+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:44:06.486225+0000","last_undegraded":"2023-10-01T09:44:06.486225+0000","last_fullsized":"2023-10-01T09:44:06.486225+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":558,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486058+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:44:06.486058+0000","last_peered":"2023-10-01T09:44:06.486058+0000","last_clean":"2023-10-01T09:44:06.486058+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:44:06.486058+0000","last_undegraded":"2023-10-01T09:44:06.486058+0000","last_fullsized":"2023-10-01T09:44:06.486058+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":336,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486131+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:44:06.486131+0000","last_peered":"2023-10-01T09:44:06.486131+0000","last_clean":"2023-10-01T09:44:06.486131+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:44:06.486131+0000","last_undegraded":"2023-10-01T09:44:06.486131+0000","last_fullsized":"2023-10-01T09:44:06.486131+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":326,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.476445+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:44:06.476445+0000","last_peered":"2023-10-01T09:44:06.476445+0000","last_clean":"2023-10-01T09:44:06.476445+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:44:06.476445+0000","last_undegraded":"2023-10-01T09:44:06.476445+0000","last_fullsized":"2023-10-01T09:44:06.476445+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"528'6473","reported_seq":6534,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:15.786825+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:44:15.786825+0000","last_peered":"2023-10-01T09:44:15.786825+0000","last_clean":"2023-10-01T09:44:15.786825+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:44:15.786825+0000","last_undegraded":"2023-10-01T09:44:15.786825+0000","last_fullsized":"2023-10-01T09:44:15.786825+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6473,"log_dups_size":0,"ondisk_log_size":6473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266010624,"num_objects":4059,"num_object_clones":0,"num_object_copies":12177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6473,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":735,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486232+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:44:06.486232+0000","last_peered":"2023-10-01T09:44:06.486232+0000","last_clean":"2023-10-01T09:44:06.486232+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:44:06.486232+0000","last_undegraded":"2023-10-01T09:44:06.486232+0000","last_fullsized":"2023-10-01T09:44:06.486232+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":568,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.793556+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:44:13.793556+0000","last_peered":"2023-10-01T09:44:13.793556+0000","last_clean":"2023-10-01T09:44:13.793556+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:44:13.793556+0000","last_undegraded":"2023-10-01T09:44:13.793556+0000","last_fullsized":"2023-10-01T09:44:13.793556+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":567,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.474841+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:44:06.474841+0000","last_peered":"2023-10-01T09:44:06.474841+0000","last_clean":"2023-10-01T09:44:06.474841+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:44:06.474841+0000","last_undegraded":"2023-10-01T09:44:06.474841+0000","last_fullsized":"2023-10-01T09:44:06.474841+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"528'6417","reported_seq":6481,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:16.502159+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:44:16.502159+0000","last_peered":"2023-10-01T09:44:16.502159+0000","last_clean":"2023-10-01T09:44:16.502159+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:44:16.502159+0000","last_undegraded":"2023-10-01T09:44:16.502159+0000","last_fullsized":"2023-10-01T09:44:16.502159+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6417,"log_dups_size":0,"ondisk_log_size":6417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264896536,"num_objects":4043,"num_object_clones":0,"num_object_copies":12129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4043,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6417,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":966,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.487005+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:44:06.487005+0000","last_peered":"2023-10-01T09:44:06.487005+0000","last_clean":"2023-10-01T09:44:06.487005+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:44:06.487005+0000","last_undegraded":"2023-10-01T09:44:06.487005+0000","last_fullsized":"2023-10-01T09:44:06.487005+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":585,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486611+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:44:06.486611+0000","last_peered":"2023-10-01T09:44:06.486611+0000","last_clean":"2023-10-01T09:44:06.486611+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:44:06.486611+0000","last_undegraded":"2023-10-01T09:44:06.486611+0000","last_fullsized":"2023-10-01T09:44:06.486611+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":438,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486307+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:44:06.486307+0000","last_peered":"2023-10-01T09:44:06.486307+0000","last_clean":"2023-10-01T09:44:06.486307+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:44:06.486307+0000","last_undegraded":"2023-10-01T09:44:06.486307+0000","last_fullsized":"2023-10-01T09:44:06.486307+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"528'6415","reported_seq":6484,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:16.122131+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:44:16.122131+0000","last_peered":"2023-10-01T09:44:16.122131+0000","last_clean":"2023-10-01T09:44:16.122131+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:44:16.122131+0000","last_undegraded":"2023-10-01T09:44:16.122131+0000","last_fullsized":"2023-10-01T09:44:16.122131+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6415,"log_dups_size":0,"ondisk_log_size":6415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258277376,"num_objects":3941,"num_object_clones":0,"num_object_copies":11823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6415,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":467,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.487092+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:44:06.487092+0000","last_peered":"2023-10-01T09:44:06.487092+0000","last_clean":"2023-10-01T09:44:06.487092+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:44:06.487092+0000","last_undegraded":"2023-10-01T09:44:06.487092+0000","last_fullsized":"2023-10-01T09:44:06.487092+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1160,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.487312+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:44:06.487312+0000","last_peered":"2023-10-01T09:44:06.487312+0000","last_clean":"2023-10-01T09:44:06.487312+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:44:06.487312+0000","last_undegraded":"2023-10-01T09:44:06.487312+0000","last_fullsized":"2023-10-01T09:44:06.487312+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":56,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.485972+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:44:06.485972+0000","last_peered":"2023-10-01T09:44:06.485972+0000","last_clean":"2023-10-01T09:44:06.485972+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:44:06.485972+0000","last_undegraded":"2023-10-01T09:44:06.485972+0000","last_fullsized":"2023-10-01T09:44:06.485972+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"528'6448","reported_seq":6517,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:16.042362+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:44:16.042362+0000","last_peered":"2023-10-01T09:44:16.042362+0000","last_clean":"2023-10-01T09:44:16.042362+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:44:16.042362+0000","last_undegraded":"2023-10-01T09:44:16.042362+0000","last_fullsized":"2023-10-01T09:44:16.042362+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6448,"log_dups_size":0,"ondisk_log_size":6448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266338304,"num_objects":4064,"num_object_clones":0,"num_object_copies":12192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6448,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":60,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486925+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:44:06.486925+0000","last_peered":"2023-10-01T09:44:06.486925+0000","last_clean":"2023-10-01T09:44:06.486925+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:44:06.486925+0000","last_undegraded":"2023-10-01T09:44:06.486925+0000","last_fullsized":"2023-10-01T09:44:06.486925+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":545,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.474909+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:44:06.474909+0000","last_peered":"2023-10-01T09:44:06.474909+0000","last_clean":"2023-10-01T09:44:06.474909+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:44:06.474909+0000","last_undegraded":"2023-10-01T09:44:06.474909+0000","last_fullsized":"2023-10-01T09:44:06.474909+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":251,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486457+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:44:06.486457+0000","last_peered":"2023-10-01T09:44:06.486457+0000","last_clean":"2023-10-01T09:44:06.486457+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:44:06.486457+0000","last_undegraded":"2023-10-01T09:44:06.486457+0000","last_fullsized":"2023-10-01T09:44:06.486457+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":494,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.474290+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:44:06.474290+0000","last_peered":"2023-10-01T09:44:06.474290+0000","last_clean":"2023-10-01T09:44:06.474290+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:44:06.474290+0000","last_undegraded":"2023-10-01T09:44:06.474290+0000","last_fullsized":"2023-10-01T09:44:06.474290+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":608,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486788+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:44:06.486788+0000","last_peered":"2023-10-01T09:44:06.486788+0000","last_clean":"2023-10-01T09:44:06.486788+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:44:06.486788+0000","last_undegraded":"2023-10-01T09:44:06.486788+0000","last_fullsized":"2023-10-01T09:44:06.486788+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":263,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486466+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:44:06.486466+0000","last_peered":"2023-10-01T09:44:06.486466+0000","last_clean":"2023-10-01T09:44:06.486466+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:44:06.486466+0000","last_undegraded":"2023-10-01T09:44:06.486466+0000","last_fullsized":"2023-10-01T09:44:06.486466+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":51,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.488299+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:44:06.488299+0000","last_peered":"2023-10-01T09:44:06.488299+0000","last_clean":"2023-10-01T09:44:06.488299+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:44:06.488299+0000","last_undegraded":"2023-10-01T09:44:06.488299+0000","last_fullsized":"2023-10-01T09:44:06.488299+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":263,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486626+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:44:06.486626+0000","last_peered":"2023-10-01T09:44:06.486626+0000","last_clean":"2023-10-01T09:44:06.486626+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:44:06.486626+0000","last_undegraded":"2023-10-01T09:44:06.486626+0000","last_fullsized":"2023-10-01T09:44:06.486626+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":440,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.473807+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:44:06.473807+0000","last_peered":"2023-10-01T09:44:06.473807+0000","last_clean":"2023-10-01T09:44:06.473807+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:44:06.473807+0000","last_undegraded":"2023-10-01T09:44:06.473807+0000","last_fullsized":"2023-10-01T09:44:06.473807+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":69,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.487132+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:44:06.487132+0000","last_peered":"2023-10-01T09:44:06.487132+0000","last_clean":"2023-10-01T09:44:06.487132+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:44:06.487132+0000","last_undegraded":"2023-10-01T09:44:06.487132+0000","last_fullsized":"2023-10-01T09:44:06.487132+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":485,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.473799+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:44:06.473799+0000","last_peered":"2023-10-01T09:44:06.473799+0000","last_clean":"2023-10-01T09:44:06.473799+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:44:06.473799+0000","last_undegraded":"2023-10-01T09:44:06.473799+0000","last_fullsized":"2023-10-01T09:44:06.473799+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":685,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.794451+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:44:13.794451+0000","last_peered":"2023-10-01T09:44:13.794451+0000","last_clean":"2023-10-01T09:44:13.794451+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:44:13.794451+0000","last_undegraded":"2023-10-01T09:44:13.794451+0000","last_fullsized":"2023-10-01T09:44:13.794451+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":379,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.475464+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:44:06.475464+0000","last_peered":"2023-10-01T09:44:06.475464+0000","last_clean":"2023-10-01T09:44:06.475464+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:44:06.475464+0000","last_undegraded":"2023-10-01T09:44:06.475464+0000","last_fullsized":"2023-10-01T09:44:06.475464+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4192403480,"num_objects":63972,"num_object_clones":0,"num_object_copies":191916,"num_objects_missing_on_primary":2457,"num_objects_missing":2457,"num_objects_degraded":7547,"num_objects_misplaced":3920,"num_objects_unfound":0,"num_objects_dirty":63972,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":102468,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8139,"num_bytes_recovered":533397504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12961853440,"data_stored":12961841224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102468,"ondisk_log_size":102468,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430724,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2180688,"kb_used_data":1285768,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":91564464,"statfs":{"total":95995035648,"available":93762011136,"internally_reserved":0,"allocated":1316626432,"data_stored":1315688080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":1.1359999999999999,"15min":2.657},"min":{"1min":0.27300000000000002,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":1.484,"5min":10.391999999999999,"15min":108.90000000000001},"last":2.3050000000000002},{"interface":"front","average":{"1min":0.39900000000000002,"5min":1.26,"15min":2.8090000000000002},"min":{"1min":0.219,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.67200000000000004,"5min":12.026,"15min":109.187},"last":0.25600000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":4.2910000000000004,"15min":5.2409999999999997},"min":{"1min":0.28999999999999998,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":1.4470000000000001,"5min":229.161,"15min":229.161},"last":2.387},{"interface":"front","average":{"1min":0.42999999999999999,"5min":4.2759999999999998,"15min":5.2510000000000003},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":0.96499999999999997,"5min":230.126,"15min":230.126},"last":0.29599999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:43:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.169,"15min":1.76},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.992,"5min":14.866,"15min":14.901999999999999},"last":2.339},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.4450000000000001,"15min":1.7170000000000001},"min":{"1min":0.32000000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4490000000000001,"5min":14.682,"15min":14.682},"last":0.34999999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.96899999999999997,"15min":2.569},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.018,"5min":6.0069999999999997,"15min":80.352999999999994},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":1.1359999999999999,"15min":2.5510000000000002},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.026,"5min":14.528,"15min":80.364000000000004},"last":0.35599999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":1.046,"15min":1.0429999999999999},"min":{"1min":0.30199999999999999,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.508,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":2.3639999999999999},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.0289999999999999,"15min":1.0189999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.28000000000000003},"max":{"1min":0.84399999999999997,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.55100000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":2.3620000000000001,"15min":1.6120000000000001},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":1.2749999999999999,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.432},{"interface":"front","average":{"1min":0.62,"5min":2.5390000000000001,"15min":1.6739999999999999},"min":{"1min":0.35999999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.2599999999999998,"5min":135.339,"15min":135.339},"last":0.48399999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":1.3200000000000001,"15min":1.1599999999999999},"min":{"1min":0.377,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":1.409,"5min":11.58,"15min":11.853999999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.51400000000000001,"5min":1.175,"15min":1.1299999999999999},"min":{"1min":0.378,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":1.23,"5min":14.584,"15min":14.584},"last":0.41999999999999998}]}]},{"osd":6,"up_from":294,"seq":1262720385235,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2138572,"kb_used_data":1796868,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":91606580,"statfs":{"total":95995035648,"available":93805137920,"internally_reserved":0,"allocated":1839992832,"data_stored":1839058431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":2.0630000000000002,"15min":4.0090000000000003},"min":{"1min":0.247,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":3.2029999999999998,"5min":158.48599999999999,"15min":158.48599999999999},"last":3.2029999999999998},{"interface":"front","average":{"1min":0.41399999999999998,"5min":2.1739999999999999,"15min":4.0730000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.221},"max":{"1min":0.63900000000000001,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.33100000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":3.1379999999999999,"15min":4.8600000000000003},"min":{"1min":0.29199999999999998,"5min":0.254,"15min":0.24199999999999999},"max":{"1min":0.58899999999999997,"5min":168.947,"15min":168.947},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.56000000000000005,"5min":3.141,"15min":4.8200000000000003},"min":{"1min":0.32500000000000001,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":2.8290000000000002,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.45000000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.78400000000000003,"15min":1.5620000000000001},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":0.65700000000000003,"5min":8.657,"15min":10.92},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.88600000000000001,"15min":1.5629999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":2.863,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.48999999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.96499999999999997,"15min":2.101},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.3149999999999999,"5min":9.0180000000000007,"15min":89.753},"last":3.3149999999999999},{"interface":"front","average":{"1min":0.439,"5min":1.097,"15min":2.0179999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.61199999999999999,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.38200000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.5049999999999999,"15min":1.0740000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.7749999999999999,"5min":63.134,"15min":63.134},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.478,"15min":1.075},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":3.2519999999999998,"5min":62.973999999999997,"15min":62.973999999999997},"last":3.2519999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:43:29 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":1.2450000000000001,"15min":1.101},"min":{"1min":0.309,"5min":0.251,"15min":0.251},"max":{"1min":34.094999999999999,"5min":34.094999999999999,"15min":34.094999999999999},"last":3.2890000000000001},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.409,"15min":1.22},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":34.072000000000003,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.54500000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.73999999999999999,"15min":1.2729999999999999},"min":{"1min":0.32700000000000001,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":3.339,"5min":3.8279999999999998,"15min":94.245999999999995},"last":3.339},{"interface":"front","average":{"1min":0.496,"5min":0.96999999999999997,"15min":1.3640000000000001},"min":{"1min":0.374,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":0.68600000000000005,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.40500000000000003}]}]},{"osd":1,"up_from":94,"seq":403726926211,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1589760,"kb_used_data":783864,"kb_used_omap":7,"kb_used_meta":805880,"kb_avail":92155392,"statfs":{"total":95995035648,"available":94367121408,"internally_reserved":0,"allocated":802676736,"data_stored":800640517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":825221543},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":1.7789999999999999,"15min":4.0039999999999996},"min":{"1min":0.186,"5min":0.186,"15min":0.182},"max":{"1min":1.2889999999999999,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.42099999999999999,"5min":1.99,"15min":3.5329999999999999},"min":{"1min":0.223,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":1.702,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.371}]},{"osd":2,"last update":"Sun Oct 1 09:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":1.4279999999999999,"15min":2.589},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.61399999999999999,"5min":22.891999999999999,"15min":56.573},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.379,"15min":3.254},"min":{"1min":0.28999999999999998,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":1.752,"5min":15.189,"15min":154.197},"last":0.28799999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":1.2769999999999999,"15min":3.7909999999999999},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":1.7889999999999999,"5min":18.503,"15min":200.56800000000001},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.2330000000000001,"15min":4.5039999999999996},"min":{"1min":0.253,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":1.9370000000000001,"5min":16.515000000000001,"15min":200.63800000000001},"last":0.38500000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.147,"15min":2.2530000000000001},"min":{"1min":0.35499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.0550000000000002,"5min":19.539000000000001,"15min":154.101},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.46899999999999997,"5min":1.0029999999999999,"15min":2.399},"min":{"1min":0.29499999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.59799999999999998,"5min":10.084,"15min":154.56999999999999},"last":0.48999999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:43:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.3200000000000001,"15min":2.097},"min":{"1min":0.372,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.1850000000000001,"5min":19.513000000000002,"15min":154.22999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.1719999999999999,"15min":2.4630000000000001},"min":{"1min":0.31,"5min":0.307,"15min":0.307},"max":{"1min":0.82699999999999996,"5min":10.305,"15min":154.54400000000001},"last":0.47299999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.236,"15min":2.5539999999999998},"min":{"1min":0.34300000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.4399999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.81399999999999995,"5min":1.335,"15min":2.4119999999999999},"min":{"1min":0.34699999999999998,"5min":0.34599999999999997,"15min":0.32700000000000001},"max":{"1min":2.5630000000000002,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.45800000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.3109999999999999,"15min":2.5840000000000001},"min":{"1min":0.38400000000000001,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":2.8170000000000002,"5min":19.431999999999999,"15min":154.20699999999999},"last":0.433},{"interface":"front","average":{"1min":0.77800000000000002,"5min":1.2330000000000001,"15min":2.6379999999999999},"min":{"1min":0.45200000000000001,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":2.8290000000000002,"5min":10.111000000000001,"15min":154.51599999999999},"last":0.52400000000000002}]}]},{"osd":0,"up_from":89,"seq":382252089746,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4119728,"kb_used_data":3228584,"kb_used_omap":8,"kb_used_meta":891127,"kb_avail":89625424,"statfs":{"total":95995035648,"available":91776434176,"internally_reserved":0,"allocated":3306070016,"data_stored":3304054705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":912515010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:44:16 2023","interfaces":[{"interface":"back","average":{"1min":0.317,"5min":1.1839999999999999,"15min":4.1120000000000001},"min":{"1min":0.21199999999999999,"5min":0.17799999999999999,"15min":0.17399999999999999},"max":{"1min":0.46300000000000002,"5min":52.316000000000003,"15min":216.821},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.317,"5min":2.7919999999999998,"15min":4.9809999999999999},"min":{"1min":0.24399999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.435,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.35299999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:43:36 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":1.204,"15min":2.085},"min":{"1min":0.246,"5min":0.246,"15min":0.20100000000000001},"max":{"1min":1.1100000000000001,"5min":19.788,"15min":19.788},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.4139999999999999,"15min":2.3380000000000001},"min":{"1min":0.27200000000000002,"5min":0.23000000000000001,"15min":0.189},"max":{"1min":3.5939999999999999,"5min":19.375,"15min":25.844999999999999},"last":0.316}]},{"osd":3,"last update":"Sun Oct 1 09:44:16 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.90200000000000002,"15min":3.46},"min":{"1min":0.26900000000000002,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.59199999999999997,"5min":8.2889999999999997,"15min":222.16499999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.0369999999999999,"15min":3.375},"min":{"1min":0.26900000000000002,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":3.4119999999999999,"5min":9.7539999999999996,"15min":222.227},"last":1.343}]},{"osd":4,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.5449999999999999,"15min":2.1339999999999999},"min":{"1min":0.28799999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":3.375,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.58399999999999996,"5min":1.4550000000000001,"15min":2.1309999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":3.5049999999999999,"5min":47.854999999999997,"15min":62.073999999999998},"last":1.3859999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:43:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.2030000000000001,"15min":1.883},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66500000000000004,"5min":14.515000000000001,"15min":16.808},"last":1.4179999999999999},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.093,"15min":1.7709999999999999},"min":{"1min":0.33500000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.73499999999999999,"5min":5.9580000000000002,"15min":17.507000000000001},"last":0.39100000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:43:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.1850000000000001,"15min":1.774},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.35499999999999998},"max":{"1min":1.5900000000000001,"5min":6.5,"15min":17.535},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.88,"5min":1.395,"15min":2.0289999999999999},"min":{"1min":0.32600000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":3.6539999999999999,"5min":10.446,"15min":18.521000000000001},"last":0.501}]},{"osd":7,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.95499999999999996,"15min":2.048},"min":{"1min":0.379,"5min":0.374,"15min":0.35699999999999998},"max":{"1min":0.78600000000000003,"5min":4.976,"15min":48.895000000000003},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.093,"15min":2.206},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.35099999999999998},"max":{"1min":3.4319999999999999,"5min":9.4969999999999999,"15min":48.274000000000001},"last":0.439}]}]},{"osd":2,"up_from":0,"seq":425201762685,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599159,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1667508,"kb_used_data":839468,"kb_used_omap":3,"kb_used_meta":828028,"kb_avail":92077644,"statfs":{"total":95995035648,"available":94287507456,"internally_reserved":0,"allocated":859615232,"data_stored":857532357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":847901415},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":3.2000000000000002,"15min":4.8920000000000003},"min":{"1min":0.20100000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":1.3580000000000001,"5min":92.649000000000001,"15min":137.34399999999999},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.36399999999999999,"5min":3.194,"15min":4.9740000000000002},"min":{"1min":0.21199999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":1.2549999999999999,"5min":92.284000000000006,"15min":137.73400000000001},"last":0.38200000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":1.8819999999999999,"15min":5.1500000000000004},"min":{"1min":0.25,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":1.1599999999999999,"5min":61.439999999999998,"15min":162.22999999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.39600000000000002,"5min":1.788,"15min":5.181},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.4119999999999999,"5min":62.723999999999997,"15min":162.70699999999999},"last":0.36099999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.0780000000000001,"15min":1.748},"min":{"1min":0.27800000000000002,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":2.4710000000000001,"5min":14.254,"15min":14.254},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.66300000000000003,"15min":1.798},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":4.5049999999999999,"5min":4.5049999999999999,"15min":15.523},"last":0.318}]},{"osd":4,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.72,"15min":1.8080000000000001},"min":{"1min":0.35199999999999998,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":1.444,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.887,"15min":2.0630000000000002},"min":{"1min":0.29499999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.2250000000000001,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.53200000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":1.5589999999999999,"15min":2.581},"min":{"1min":0.317,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3099999999999996,"5min":10.919,"15min":28.117999999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.4279999999999999,"15min":2.6800000000000002},"min":{"1min":0.36699999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.512,"5min":10.798999999999999,"15min":28.34},"last":0.51300000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:43:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":2.165,"15min":2.145},"min":{"1min":0.34200000000000003,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":2.5430000000000001,"5min":73.887,"15min":73.887},"last":0.622},{"interface":"front","average":{"1min":0.63,"5min":2.4159999999999999,"15min":2.1379999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":2.6000000000000001,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.64800000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":1.268,"15min":1.845},"min":{"1min":0.42299999999999999,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":2.573,"5min":11.659000000000001,"15min":13.085000000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.0740000000000001,"15min":1.726},"min":{"1min":0.39700000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":1.472,"5min":8.3460000000000001,"15min":13.191000000000001},"last":0.46999999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019409,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4039116,"kb_used_data":3148932,"kb_used_omap":0,"kb_used_meta":890176,"kb_avail":89706036,"statfs":{"total":95995035648,"available":91858980864,"internally_reserved":0,"allocated":3224506368,"data_stored":3223571522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911540224},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":2.3570000000000002,"15min":4.25},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.47299999999999998,"5min":79.837000000000003,"15min":155.315},"last":0.22500000000000001},{"interface":"front","average":{"1min":0.44900000000000001,"5min":2.4300000000000002,"15min":4.7039999999999997},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":0.63700000000000001,"5min":78.983000000000004,"15min":155.334},"last":0.38500000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":4.5709999999999997,"15min":6.4699999999999998},"min":{"1min":0.28899999999999998,"5min":0.252,"15min":0.224},"max":{"1min":0.52900000000000003,"5min":126.742,"15min":190.61199999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.40200000000000002,"5min":4.5369999999999999,"15min":7.1600000000000001},"min":{"1min":0.27600000000000002,"5min":0.26700000000000002,"15min":0.25600000000000001},"max":{"1min":0.59999999999999998,"5min":126.791,"15min":253.273},"last":2.2810000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.05,"15min":2.444},"min":{"1min":0.34200000000000003,"5min":0.31,"15min":0.28299999999999997},"max":{"1min":2.9510000000000001,"5min":10.161,"15min":170.45400000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.0029999999999999,"15min":2.1739999999999999},"min":{"1min":0.314,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":3.0219999999999998,"5min":5.9740000000000002,"15min":58.771000000000001},"last":0.49399999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.96499999999999997,"15min":2.3079999999999998},"min":{"1min":0.28100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.1299999999999999,"5min":7.5830000000000002,"15min":140.881},"last":0.31},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.876,"15min":2.4830000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.311},"max":{"1min":0.51500000000000001,"5min":6.8719999999999999,"15min":140.911},"last":2.3250000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:43:33 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":2.0569999999999999,"15min":1.5580000000000001},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":3.052,"5min":110.003,"15min":110.003},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.64900000000000002,"5min":2.0110000000000001,"15min":1.399},"min":{"1min":0.33500000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.0049999999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":2.3519999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:44:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.78200000000000003,"15min":0.93000000000000005},"min":{"1min":0.33500000000000002,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":0.73599999999999999,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.85099999999999998,"15min":1.0589999999999999},"min":{"1min":0.40600000000000003,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":0.73599999999999999,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.44500000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.71599999999999997,"15min":1.161},"min":{"1min":0.313,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":0.69399999999999995,"5min":4.7030000000000003,"15min":38.034999999999997},"last":0.37},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.026,"15min":1.4410000000000001},"min":{"1min":0.36699999999999999,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":2.9940000000000002,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.51500000000000001}]}]},{"osd":5,"up_from":379,"seq":1627792605316,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2002360,"kb_used_data":1633712,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":91742792,"statfs":{"total":95995035648,"available":93944619008,"internally_reserved":0,"allocated":1672921088,"data_stored":1672002146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,1,0,0,3,5,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":1.6479999999999999,"15min":2.1890000000000001},"min":{"1min":0.22900000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.1880000000000002,"5min":12.571999999999999,"15min":84.340999999999994},"last":3.677},{"interface":"front","average":{"1min":0.40400000000000003,"5min":1.2549999999999999,"15min":2.0800000000000001},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.53900000000000003,"5min":8.6460000000000008,"15min":84.356999999999999},"last":3.8250000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":6.4870000000000001,"15min":5.7599999999999998},"min":{"1min":0.26000000000000001,"5min":0.25900000000000001,"15min":0.255},"max":{"1min":0.76900000000000002,"5min":295.71300000000002,"15min":295.71300000000002},"last":3.7799999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":6.8099999999999996,"15min":5.7320000000000002},"min":{"1min":0.32200000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":2.258,"5min":299.59399999999999,"15min":299.59399999999999},"last":3.9580000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":1.3600000000000001,"15min":1.5980000000000001},"min":{"1min":0.315,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.752,"5min":8.7040000000000006,"15min":14.199999999999999},"last":3.7029999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":1.306,"15min":1.583},"min":{"1min":0.32900000000000001,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":0.85999999999999999,"5min":8.2370000000000001,"15min":14.430999999999999},"last":3.8780000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.5069999999999999,"15min":1.488},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.2280000000000002,"5min":12.734,"15min":14.004},"last":4.0209999999999999},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.5640000000000001,"15min":1.629},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.3979999999999999,"5min":11.766,"15min":13.991},"last":3.8610000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":2.2029999999999998,"15min":1.4330000000000001},"min":{"1min":0.30099999999999999,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.66800000000000004,"5min":122.971,"15min":122.971},"last":4.0839999999999996},{"interface":"front","average":{"1min":0.438,"5min":2.25,"15min":1.724},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.73799999999999999,"5min":122.89100000000001,"15min":122.89100000000001},"last":4.0570000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":1.165,"15min":1.044},"min":{"1min":0.312,"5min":0.311,"15min":0.28299999999999997},"max":{"1min":2.2839999999999998,"5min":12.545,"15min":12.545},"last":4.3300000000000001},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.99099999999999999,"15min":1.302},"min":{"1min":0.378,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":2.629,"5min":6.4729999999999999,"15min":14.193},"last":4.0449999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.1140000000000001,"15min":1.262},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.81200000000000006,"5min":8.2010000000000005,"15min":10.813000000000001},"last":4.0999999999999996},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.2270000000000001,"15min":1.0780000000000001},"min":{"1min":0.36299999999999999,"5min":0.35299999999999998,"15min":0.318},"max":{"1min":2.3079999999999998,"5min":12.500999999999999,"15min":12.500999999999999},"last":4.2000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3294695424,"data_stored":3294691352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":792330240,"data_stored":792330240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":853278720,"data_stored":853278720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3215790080,"data_stored":3215786008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1666777088,"data_stored":1666777088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1832128512,"data_stored":1832124440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1306853376,"data_stored":1306853376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:44:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:19 smithi190 ceph-mon[160435]: pgmap v1537: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 7557/192000 objects degraded (3.936%); 3920/192000 objects misplaced (2.042%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:44:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:19 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:19 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:19.820 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:19.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:19.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:19.924 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:19.954 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:20.113 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:20.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:20.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:20.151 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:44:20.216 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:20.245 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:20.305 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:20.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:20.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:20.409 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:20.441 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:20.506 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:20 smithi138 ceph-mon[192513]: from='client.56872 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:20.507 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:20 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:20.507 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:20 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:20.507 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:20 smithi138 ceph-mon[185886]: from='client.56872 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:20.507 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:20 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:20.507 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:20 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:20.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:20 smithi190 ceph-mon[160435]: from='client.56872 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:20.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:20 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:20.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:20 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:20.599 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:20.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:20.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:20.654 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:44:20.655 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:44:20.703 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:20.738 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:20.889 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:20.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:20.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:20.992 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:21.023 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:21.078 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:21.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:21.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:21.184 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:21.213 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:21.258 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:21.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:21.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:21.361 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:21.391 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:21.410 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:44:21.437 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:21.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:21.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:21.543 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:21.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:21 smithi138 ceph-mon[185886]: pgmap v1538: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 7557/191721 objects degraded (3.942%); 3920/191721 objects misplaced (2.045%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:44:21.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:21 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/444614423' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:21.549 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:21 smithi138 ceph-mon[192513]: pgmap v1538: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 7557/191721 objects degraded (3.942%); 3920/191721 objects misplaced (2.045%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:44:21.549 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:21 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/444614423' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:21.569 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:21.599 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:21.617 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:21.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:21.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:21.731 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:21.757 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:21 smithi190 ceph-mon[160435]: pgmap v1538: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 126 op/s; 7557/191721 objects degraded (3.942%); 3920/191721 objects misplaced (2.045%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:44:21.757 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:21 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/444614423' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:21.760 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:21.824 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:21.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:21.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:21.859 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:44:21.874 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:44:21.930 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:21.959 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:22.060 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:22.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:22.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:22.165 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:22.199 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:22.547 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:22.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:22.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:22.651 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:22.676 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:22 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:44:22.676 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:22 smithi190 ceph-mon[160435]: failed to encode map e529 with expected crc 2023-10-01T09:44:22.676 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:22 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/444614423' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:44:22.676 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:22 smithi190 ceph-mon[160435]: osdmap e529: 8 total, 8 up, 7 in 2023-10-01T09:44:22.681 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:22.731 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:22.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:22.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:22.770 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:22 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:44:22.770 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:22 smithi138 ceph-mon[192513]: failed to encode map e529 with expected crc 2023-10-01T09:44:22.770 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:22 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/444614423' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:44:22.770 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:22 smithi138 ceph-mon[192513]: osdmap e529: 8 total, 8 up, 7 in 2023-10-01T09:44:22.770 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:22 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:44:22.771 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:22 smithi138 ceph-mon[185886]: failed to encode map e529 with expected crc 2023-10-01T09:44:22.771 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:22 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/444614423' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:44:22.771 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:22 smithi138 ceph-mon[185886]: osdmap e529: 8 total, 8 up, 7 in 2023-10-01T09:44:22.794 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:44:22.834 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:22.865 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:23.096 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:23.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:23.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:23.205 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:23.234 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:23.400 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:23.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:23.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:23.503 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:23.542 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:23.574 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:23.579 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:44:23.579 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:44:23.598 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:23.600 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:23 smithi190 ceph-mon[160435]: failed to encode map e529 with expected crc 2023-10-01T09:44:23.600 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:23 smithi190 ceph-mon[160435]: failed to encode map e529 with expected crc 2023-10-01T09:44:23.601 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:23 smithi190 ceph-mon[160435]: pgmap v1540: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 7507/190389 objects degraded (3.943%); 3881/190389 objects misplaced (2.038%); 629 KiB/s, 9 objects/s recovering 2023-10-01T09:44:23.601 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:23 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7557/191721 objects degraded (3.942%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:23.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:23.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:23.703 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:23.734 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:23 smithi138 ceph-mon[185886]: failed to encode map e529 with expected crc 2023-10-01T09:44:23.734 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:23 smithi138 ceph-mon[185886]: failed to encode map e529 with expected crc 2023-10-01T09:44:23.734 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:23 smithi138 ceph-mon[185886]: pgmap v1540: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 7507/190389 objects degraded (3.943%); 3881/190389 objects misplaced (2.038%); 629 KiB/s, 9 objects/s recovering 2023-10-01T09:44:23.735 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:23 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7557/191721 objects degraded (3.942%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:23.735 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:23 smithi138 ceph-mon[192513]: failed to encode map e529 with expected crc 2023-10-01T09:44:23.735 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:23 smithi138 ceph-mon[192513]: failed to encode map e529 with expected crc 2023-10-01T09:44:23.735 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:23 smithi138 ceph-mon[192513]: pgmap v1540: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 7507/190389 objects degraded (3.943%); 3881/190389 objects misplaced (2.038%); 629 KiB/s, 9 objects/s recovering 2023-10-01T09:44:23.736 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:23 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7557/191721 objects degraded (3.942%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:23.737 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:23.887 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:23.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:23.991 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:24.028 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:24.100 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:44:24.120 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:44:24.142 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:44:24.165 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:44:24.188 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:44:24.206 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:44:24.226 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:44:24.244 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:44:24.245 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:44:24.245 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:44:24.245 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:44:24.246 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:44:24.246 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:44:24.246 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:44:24.246 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:44:24.246 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:44:24.358 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:44:24.633 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:24.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:24.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:24.741 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:24.766 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/763430465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:24.767 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/763430465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:24.769 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:44:24.770 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/763430465' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:25.016 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:25.017 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:44:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:44:24] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.33.4" 2023-10-01T09:44:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:25.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:25.088 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:44:25.119 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:25.153 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:25.412 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:25.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:25.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:25.518 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:25.549 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:25.687 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:25 smithi138 ceph-mon[192513]: pgmap v1541: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 7507/190290 objects degraded (3.945%); 3881/190290 objects misplaced (2.040%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:44:25.687 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:25.688 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:44:25.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:25 smithi138 ceph-mon[185886]: pgmap v1541: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 7507/190290 objects degraded (3.945%); 3881/190290 objects misplaced (2.040%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:44:25.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:25.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:44:25.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:25 smithi190 ceph-mon[160435]: pgmap v1541: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 7507/190290 objects degraded (3.945%); 3881/190290 objects misplaced (2.040%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:44:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:44:26.049 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:26.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:26.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:26.153 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:26.183 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:26.229 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:26.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:26.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:26.333 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:26.348 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:26.370 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:26.420 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:26.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:26.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:26.464 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:44:26.465 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:44:26.465 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.465 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103551 10.7 10:e47a1008:::benchmark_data_smithi138_573643_object20329:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.465 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:25.205649+0000", 2023-10-01T09:44:26.465 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.2564572809999999, 2023-10-01T09:44:26.465 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.256477463, 2023-10-01T09:44:26.465 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.465 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:44:26.465 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.465 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.465 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.466 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103551 2023-10-01T09:44:26.466 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.466 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.466 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.466 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.466 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.205649+0000", 2023-10-01T09:44:26.466 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.466 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.466 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.466 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.466 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.205649+0000", 2023-10-01T09:44:26.466 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:44:26.467 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.467 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.467 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.467 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.205648+0000", 2023-10-01T09:44:26.467 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7239999999999998e-06 2023-10-01T09:44:26.467 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.467 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.467 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.467 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.205656+0000", 2023-10-01T09:44:26.467 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6436e-05 2023-10-01T09:44:26.467 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.467 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.468 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.468 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.205673+0000", 2023-10-01T09:44:26.468 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3429999999999998e-06 2023-10-01T09:44:26.468 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.468 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.468 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.468 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.205680+0000", 2023-10-01T09:44:26.468 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.070048502999999998 2023-10-01T09:44:26.468 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.468 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.468 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.468 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.275729+0000", 2023-10-01T09:44:26.469 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0055389100000000002 2023-10-01T09:44:26.469 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.469 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.469 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.469 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.281267+0000", 2023-10-01T09:44:26.469 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.035851274000000002 2023-10-01T09:44:26.469 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.469 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.469 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.469 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.317119+0000", 2023-10-01T09:44:26.469 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053554550000000003 2023-10-01T09:44:26.469 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.470 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.470 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:44:26.470 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.322474+0000", 2023-10-01T09:44:26.470 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.11682498500000001 2023-10-01T09:44:26.470 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.470 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.470 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.470 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.470 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.470 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103563 10.7 10:e6a3b2cf:::benchmark_data_smithi138_573643_object20341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.470 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:25.252442+0000", 2023-10-01T09:44:26.470 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.209664471, 2023-10-01T09:44:26.470 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.209739465, 2023-10-01T09:44:26.471 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.471 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:44:26.471 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.471 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.471 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.471 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103563 2023-10-01T09:44:26.471 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.471 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.471 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.471 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.471 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.252442+0000", 2023-10-01T09:44:26.471 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.471 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.472 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.472 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.472 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.252442+0000", 2023-10-01T09:44:26.472 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:44:26.472 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.472 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.472 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.472 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.252440+0000", 2023-10-01T09:44:26.472 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0251999999999999e-05 2023-10-01T09:44:26.472 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.472 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.472 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.473 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.252450+0000", 2023-10-01T09:44:26.473 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7577000000000001e-05 2023-10-01T09:44:26.473 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.473 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.473 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.473 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.252468+0000", 2023-10-01T09:44:26.473 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.49e-06 2023-10-01T09:44:26.473 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.473 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.473 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.473 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.252476+0000", 2023-10-01T09:44:26.473 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.029667265000000002 2023-10-01T09:44:26.473 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.473 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.474 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.474 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.282144+0000", 2023-10-01T09:44:26.474 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0055582510000000002 2023-10-01T09:44:26.474 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.474 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.474 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.474 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.287702+0000", 2023-10-01T09:44:26.474 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.055428612000000002 2023-10-01T09:44:26.474 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.474 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.474 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.475 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.343131+0000", 2023-10-01T09:44:26.475 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049364420000000001 2023-10-01T09:44:26.475 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.475 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.475 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:44:26.475 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.348067+0000", 2023-10-01T09:44:26.475 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.095625012999999995 2023-10-01T09:44:26.475 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.475 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.475 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.475 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.476 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.476 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103592 10.7 10:ebc548ac:::benchmark_data_smithi138_573643_object20370:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.476 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:25.544650+0000", 2023-10-01T09:44:26.476 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.91745668700000005, 2023-10-01T09:44:26.476 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.91757908499999996, 2023-10-01T09:44:26.476 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.476 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:44:26.476 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.476 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.476 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.476 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103592 2023-10-01T09:44:26.476 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.477 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.477 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.477 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.477 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.544650+0000", 2023-10-01T09:44:26.477 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.477 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.477 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.477 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.477 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.544650+0000", 2023-10-01T09:44:26.477 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:44:26.477 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.477 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.478 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.478 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.544649+0000", 2023-10-01T09:44:26.478 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6559999999999997e-06 2023-10-01T09:44:26.478 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.478 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.478 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.478 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.544654+0000", 2023-10-01T09:44:26.478 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0343000000000001e-05 2023-10-01T09:44:26.478 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.478 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.478 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.479 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.544665+0000", 2023-10-01T09:44:26.479 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.164e-06 2023-10-01T09:44:26.479 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.479 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.479 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.479 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.544670+0000", 2023-10-01T09:44:26.479 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21716595999999999 2023-10-01T09:44:26.479 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.479 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.479 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.479 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.761836+0000", 2023-10-01T09:44:26.479 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053903400000000004 2023-10-01T09:44:26.480 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.480 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.480 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.480 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.767226+0000", 2023-10-01T09:44:26.480 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.036101451999999999 2023-10-01T09:44:26.480 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.480 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.480 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.480 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.803328+0000", 2023-10-01T09:44:26.480 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053882160000000004 2023-10-01T09:44:26.480 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.480 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.481 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:44:26.481 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.808716+0000", 2023-10-01T09:44:26.481 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.264065934 2023-10-01T09:44:26.481 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.481 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.481 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.481 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.481 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.481 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103676 10.7 10:e22edab0:::benchmark_data_smithi138_573643_object20454:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.481 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:25.824730+0000", 2023-10-01T09:44:26.481 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.63737694300000003, 2023-10-01T09:44:26.481 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.637546681, 2023-10-01T09:44:26.482 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.482 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:44:26.482 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.482 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.482 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.482 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103676 2023-10-01T09:44:26.482 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.482 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.482 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.482 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.482 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.824730+0000", 2023-10-01T09:44:26.483 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.483 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.483 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.483 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.483 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.824730+0000", 2023-10-01T09:44:26.483 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:44:26.483 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.483 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.483 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.483 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.824728+0000", 2023-10-01T09:44:26.483 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.541e-06 2023-10-01T09:44:26.483 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.484 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.484 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.484 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.824736+0000", 2023-10-01T09:44:26.484 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3937000000000001e-05 2023-10-01T09:44:26.484 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.484 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.484 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.484 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.824750+0000", 2023-10-01T09:44:26.484 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.336e-06 2023-10-01T09:44:26.484 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.484 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.484 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.485 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.824757+0000", 2023-10-01T09:44:26.485 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.051248604000000003 2023-10-01T09:44:26.485 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.485 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.485 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.485 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.876005+0000", 2023-10-01T09:44:26.485 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0051198720000000001 2023-10-01T09:44:26.485 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.485 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.485 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.485 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.881125+0000", 2023-10-01T09:44:26.486 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.053190276000000002 2023-10-01T09:44:26.486 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.486 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.486 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.486 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.934316+0000", 2023-10-01T09:44:26.486 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053877960000000003 2023-10-01T09:44:26.486 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.486 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.486 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:44:26.486 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.939703+0000", 2023-10-01T09:44:26.486 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.11497377 2023-10-01T09:44:26.486 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.487 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.487 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.487 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.487 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.487 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103522 10.7 10:e2c36216:::benchmark_data_smithi138_573643_object20300:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.487 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:25.125880+0000", 2023-10-01T09:44:26.487 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.3362266250000001, 2023-10-01T09:44:26.487 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.336442009, 2023-10-01T09:44:26.487 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.487 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:44:26.487 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.488 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.488 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.488 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103522 2023-10-01T09:44:26.488 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.488 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.488 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.488 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.488 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.125880+0000", 2023-10-01T09:44:26.488 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.488 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.488 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.488 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.489 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.125880+0000", 2023-10-01T09:44:26.489 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:44:26.489 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.489 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.489 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.489 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.125878+0000", 2023-10-01T09:44:26.489 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2229999999999996e-06 2023-10-01T09:44:26.489 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.489 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.489 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.489 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.125887+0000", 2023-10-01T09:44:26.489 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6460000000000002e-05 2023-10-01T09:44:26.490 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.490 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.490 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.490 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.125903+0000", 2023-10-01T09:44:26.490 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6790000000000005e-06 2023-10-01T09:44:26.490 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.490 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.490 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.490 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.125911+0000", 2023-10-01T09:44:26.490 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.027129019000000001 2023-10-01T09:44:26.490 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.491 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.491 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.491 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.153040+0000", 2023-10-01T09:44:26.491 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014493453999999999 2023-10-01T09:44:26.491 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.491 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.491 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.491 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.167534+0000", 2023-10-01T09:44:26.491 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.097475034000000002 2023-10-01T09:44:26.491 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.491 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.491 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.492 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.265009+0000", 2023-10-01T09:44:26.492 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0085656970000000006 2023-10-01T09:44:26.492 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.492 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.492 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:44:26.492 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.273574+0000", 2023-10-01T09:44:26.492 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.147694456 2023-10-01T09:44:26.492 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.492 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.492 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.492 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.493 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.493 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103699 10.7 10:e837f3d3:::benchmark_data_smithi138_573643_object20477:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.493 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:26.162058+0000", 2023-10-01T09:44:26.493 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.30004802600000002, 2023-10-01T09:44:26.493 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.30031525799999997, 2023-10-01T09:44:26.493 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.493 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:44:26.493 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.493 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.493 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.493 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103699 2023-10-01T09:44:26.494 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.494 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.494 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.494 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.494 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.162058+0000", 2023-10-01T09:44:26.494 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.494 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.494 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.494 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.494 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.162058+0000", 2023-10-01T09:44:26.494 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:44:26.494 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.495 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.495 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.495 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.162057+0000", 2023-10-01T09:44:26.495 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4689999999999999e-06 2023-10-01T09:44:26.495 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.495 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.495 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.495 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.162062+0000", 2023-10-01T09:44:26.495 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0060000000000003e-06 2023-10-01T09:44:26.495 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.495 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.496 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.496 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.162069+0000", 2023-10-01T09:44:26.496 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0249999999999996e-06 2023-10-01T09:44:26.496 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.496 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.496 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.496 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.162073+0000", 2023-10-01T09:44:26.496 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16883171999999999 2023-10-01T09:44:26.496 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.496 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.496 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.496 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.330905+0000", 2023-10-01T09:44:26.496 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0069266939999999997 2023-10-01T09:44:26.497 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.497 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.497 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.497 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.337831+0000", 2023-10-01T09:44:26.497 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.038120479999999998 2023-10-01T09:44:26.497 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.497 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.497 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.497 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.375952+0000", 2023-10-01T09:44:26.497 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0056842799999999999 2023-10-01T09:44:26.497 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.498 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.498 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:44:26.498 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.381636+0000", 2023-10-01T09:44:26.498 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.219577578 2023-10-01T09:44:26.498 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.498 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.498 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.498 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.498 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.498 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103702 10.7 10:e1578e4c:::benchmark_data_smithi138_573643_object20480:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.498 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:26.172000+0000", 2023-10-01T09:44:26.499 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.29010639599999999, 2023-10-01T09:44:26.499 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.29042330599999999, 2023-10-01T09:44:26.499 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.499 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:44:26.499 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.499 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.499 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.499 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103702 2023-10-01T09:44:26.499 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.499 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.499 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.500 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.500 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.172000+0000", 2023-10-01T09:44:26.500 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.500 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.500 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.500 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.500 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.172000+0000", 2023-10-01T09:44:26.500 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999967 2023-10-01T09:44:26.500 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.500 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.500 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.501 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.171997+0000", 2023-10-01T09:44:26.501 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7049999999999994e-06 2023-10-01T09:44:26.501 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.501 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.501 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.501 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.172006+0000", 2023-10-01T09:44:26.501 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3261000000000001e-05 2023-10-01T09:44:26.501 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.501 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.501 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.502 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.172019+0000", 2023-10-01T09:44:26.502 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6140000000000003e-06 2023-10-01T09:44:26.502 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.502 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.502 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.502 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.172027+0000", 2023-10-01T09:44:26.502 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.166855376 2023-10-01T09:44:26.502 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.502 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.502 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.502 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.338882+0000", 2023-10-01T09:44:26.503 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0072503799999999998 2023-10-01T09:44:26.503 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.503 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.503 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.503 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.346133+0000", 2023-10-01T09:44:26.503 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.057261526 2023-10-01T09:44:26.503 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.503 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.503 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.503 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.403394+0000", 2023-10-01T09:44:26.503 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.005330942 2023-10-01T09:44:26.503 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.504 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.504 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:44:26.504 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.408725+0000", 2023-10-01T09:44:26.504 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.236725249 2023-10-01T09:44:26.504 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.504 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.504 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.504 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.504 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.504 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103473 10.7 10:edc15838:::benchmark_data_smithi138_573643_object20251:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.504 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:24.908210+0000", 2023-10-01T09:44:26.504 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.553896282, 2023-10-01T09:44:26.505 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.554259034, 2023-10-01T09:44:26.505 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.505 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:44:26.505 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.505 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.505 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.505 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103473 2023-10-01T09:44:26.505 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.505 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.505 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.505 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.505 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.908210+0000", 2023-10-01T09:44:26.505 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.506 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.506 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.506 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.506 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.908210+0000", 2023-10-01T09:44:26.506 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:44:26.506 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.506 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.506 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.506 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.908209+0000", 2023-10-01T09:44:26.506 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0599999999999996e-06 2023-10-01T09:44:26.506 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.506 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.507 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.507 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.908215+0000", 2023-10-01T09:44:26.507 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2157999999999999e-05 2023-10-01T09:44:26.507 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.507 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.507 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.507 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.908227+0000", 2023-10-01T09:44:26.507 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2190000000000003e-06 2023-10-01T09:44:26.507 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.507 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.507 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.507 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.908232+0000", 2023-10-01T09:44:26.508 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.22113702599999999 2023-10-01T09:44:26.508 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.508 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.508 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.508 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.129369+0000", 2023-10-01T09:44:26.508 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0083733369999999998 2023-10-01T09:44:26.508 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.508 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.508 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.508 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.137743+0000", 2023-10-01T09:44:26.508 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.067193242 2023-10-01T09:44:26.508 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.508 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.509 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.509 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.204936+0000", 2023-10-01T09:44:26.509 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053613020000000001 2023-10-01T09:44:26.509 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.509 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.509 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:44:26.509 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.210297+0000", 2023-10-01T09:44:26.509 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.30208717099999999 2023-10-01T09:44:26.509 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.509 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.509 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.510 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.510 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.510 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103737 10.7 10:e5a676ff:::benchmark_data_smithi138_573643_object20515:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.510 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:26.378888+0000", 2023-10-01T09:44:26.510 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.083218362000000004, 2023-10-01T09:44:26.510 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.083626117999999999, 2023-10-01T09:44:26.510 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.510 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:44:26.510 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.510 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.510 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.510 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103737 2023-10-01T09:44:26.511 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.511 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.511 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.511 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.511 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.378888+0000", 2023-10-01T09:44:26.511 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.511 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.511 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.511 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.511 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.378888+0000", 2023-10-01T09:44:26.511 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:44:26.511 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.512 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.512 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.512 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.378887+0000", 2023-10-01T09:44:26.512 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.195e-06 2023-10-01T09:44:26.512 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.512 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.512 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.512 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.378893+0000", 2023-10-01T09:44:26.512 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2201999999999999e-05 2023-10-01T09:44:26.512 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.512 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.512 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.513 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.378905+0000", 2023-10-01T09:44:26.513 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0330000000000003e-06 2023-10-01T09:44:26.513 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.513 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.513 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.513 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.378912+0000", 2023-10-01T09:44:26.513 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.074812423000000003 2023-10-01T09:44:26.513 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.513 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.513 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.513 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.453725+0000", 2023-10-01T09:44:26.513 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050930309999999996 2023-10-01T09:44:26.513 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.514 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.514 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.514 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.458818+0000", 2023-10-01T09:44:26.514 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.079929696999999994 2023-10-01T09:44:26.514 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.514 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.514 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.514 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.514 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.514 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103739 10.7 10.94687b47 (undecoded) ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.514 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:26.388427+0000", 2023-10-01T09:44:26.515 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.073679085000000005, 2023-10-01T09:44:26.515 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.074124224000000002, 2023-10-01T09:44:26.515 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.515 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:44:26.515 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.515 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.515 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.515 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103739 2023-10-01T09:44:26.515 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.515 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.515 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.516 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.516 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.388427+0000", 2023-10-01T09:44:26.516 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.516 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.516 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.516 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.516 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.388427+0000", 2023-10-01T09:44:26.516 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:44:26.516 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.516 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.516 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.516 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.388426+0000", 2023-10-01T09:44:26.516 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0756e-05 2023-10-01T09:44:26.517 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.517 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.517 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.517 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.388437+0000", 2023-10-01T09:44:26.517 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3304e-05 2023-10-01T09:44:26.517 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.517 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.517 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.517 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.388450+0000", 2023-10-01T09:44:26.517 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1259999999999998e-06 2023-10-01T09:44:26.517 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.517 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.518 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.518 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.388458+0000", 2023-10-01T09:44:26.518 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0636000000000003e-05 2023-10-01T09:44:26.518 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.518 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.518 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.518 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.518 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.518 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop_reply(client.36456.0:103457 10.7 e529/503 ondisk, result = 0)", 2023-10-01T09:44:26.518 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:26.449610+0000", 2023-10-01T09:44:26.518 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.012496802, 2023-10-01T09:44:26.518 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012972378999999999, 2023-10-01T09:44:26.519 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.519 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:44:26.519 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.519 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.519 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.519 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.449610+0000", 2023-10-01T09:44:26.519 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.519 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.519 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.519 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.519 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.449610+0000", 2023-10-01T09:44:26.519 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:44:26.520 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.520 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.520 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.520 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.449608+0000", 2023-10-01T09:44:26.520 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4360000000000002e-06 2023-10-01T09:44:26.520 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.520 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.520 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.520 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.449617+0000", 2023-10-01T09:44:26.520 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3123e-05 2023-10-01T09:44:26.520 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.520 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.521 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.521 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.449630+0000", 2023-10-01T09:44:26.521 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9870000000000002e-06 2023-10-01T09:44:26.521 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.521 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.521 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.521 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.449637+0000", 2023-10-01T09:44:26.521 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0099333949999999994 2023-10-01T09:44:26.521 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.521 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.521 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.521 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.459570+0000", 2023-10-01T09:44:26.522 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00059986900000000005 2023-10-01T09:44:26.522 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.522 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.522 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:26.522 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.460170+0000", 2023-10-01T09:44:26.522 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010560412 2023-10-01T09:44:26.522 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.522 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.522 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.522 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.522 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.522 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103650 10.7 10:e0ea32e2:::benchmark_data_smithi138_573643_object20428:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.523 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:25.739715+0000", 2023-10-01T09:44:26.523 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.72239167500000001, 2023-10-01T09:44:26.523 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.72290263899999996, 2023-10-01T09:44:26.523 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.523 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:44:26.523 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.523 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.523 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.523 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103650 2023-10-01T09:44:26.523 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.523 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.523 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.523 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.524 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.739715+0000", 2023-10-01T09:44:26.524 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.524 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.524 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.524 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.524 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.739715+0000", 2023-10-01T09:44:26.524 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:44:26.524 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.524 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.524 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.524 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.739713+0000", 2023-10-01T09:44:26.525 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.303e-06 2023-10-01T09:44:26.525 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.525 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.525 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.525 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.739722+0000", 2023-10-01T09:44:26.525 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7275e-05 2023-10-01T09:44:26.525 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.525 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.525 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.525 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.739739+0000", 2023-10-01T09:44:26.525 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9600000000000003e-06 2023-10-01T09:44:26.525 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.526 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.526 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.526 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.739746+0000", 2023-10-01T09:44:26.526 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.028059995000000001 2023-10-01T09:44:26.526 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.526 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.526 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.526 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.767806+0000", 2023-10-01T09:44:26.526 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053916600000000004 2023-10-01T09:44:26.526 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.526 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.526 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.527 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.773197+0000", 2023-10-01T09:44:26.527 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.058846426 2023-10-01T09:44:26.527 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.527 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.527 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.527 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.832044+0000", 2023-10-01T09:44:26.527 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0052840049999999996 2023-10-01T09:44:26.527 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.527 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.527 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:44:26.527 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.837328+0000", 2023-10-01T09:44:26.527 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.097613070999999996 2023-10-01T09:44:26.528 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.528 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.528 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.528 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.528 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.528 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103686 10.7 10:e4bd2d8a:::benchmark_data_smithi138_573643_object20464:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.528 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:26.126537+0000", 2023-10-01T09:44:26.528 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.33556994499999998, 2023-10-01T09:44:26.528 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.33612611799999997, 2023-10-01T09:44:26.528 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.528 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:44:26.528 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.529 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.529 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.529 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103686 2023-10-01T09:44:26.529 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.529 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.529 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.529 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.529 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.126537+0000", 2023-10-01T09:44:26.529 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.529 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.529 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.530 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.530 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.126537+0000", 2023-10-01T09:44:26.530 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:44:26.530 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.530 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.530 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.530 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.126535+0000", 2023-10-01T09:44:26.530 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.5880000000000004e-06 2023-10-01T09:44:26.530 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.530 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.530 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.531 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.126544+0000", 2023-10-01T09:44:26.531 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5645999999999999e-05 2023-10-01T09:44:26.531 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.531 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.531 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.531 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.126560+0000", 2023-10-01T09:44:26.531 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1249999999999993e-06 2023-10-01T09:44:26.531 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.531 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.531 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.531 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.126568+0000", 2023-10-01T09:44:26.533 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.079488260000000005 2023-10-01T09:44:26.533 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.533 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.533 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.533 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.206056+0000", 2023-10-01T09:44:26.533 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0058656330000000003 2023-10-01T09:44:26.534 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.534 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.534 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.534 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.211922+0000", 2023-10-01T09:44:26.534 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.088608105000000006 2023-10-01T09:44:26.534 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.534 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.534 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.534 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.300530+0000", 2023-10-01T09:44:26.534 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.006372412 2023-10-01T09:44:26.534 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.534 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.535 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:44:26.535 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.306902+0000", 2023-10-01T09:44:26.535 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.180365893 2023-10-01T09:44:26.535 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.535 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.535 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.535 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.535 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.535 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop_reply(client.36456.0:103457 10.7 e529/503)", 2023-10-01T09:44:26.535 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:26.450878+0000", 2023-10-01T09:44:26.535 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.011228831, 2023-10-01T09:44:26.535 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011831012, 2023-10-01T09:44:26.536 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.536 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:44:26.536 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.536 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.536 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.536 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.450878+0000", 2023-10-01T09:44:26.536 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.536 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.536 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.536 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.536 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.450878+0000", 2023-10-01T09:44:26.536 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:44:26.537 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.537 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.537 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.537 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.450877+0000", 2023-10-01T09:44:26.537 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7329999999999992e-06 2023-10-01T09:44:26.537 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.537 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.537 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.537 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.450885+0000", 2023-10-01T09:44:26.537 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1999e-05 2023-10-01T09:44:26.537 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.537 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.538 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.538 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.450897+0000", 2023-10-01T09:44:26.538 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1109999999999999e-06 2023-10-01T09:44:26.538 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.538 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.538 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.538 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.450903+0000", 2023-10-01T09:44:26.538 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4907999999999999e-05 2023-10-01T09:44:26.538 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.538 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.538 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.538 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.539 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.539 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "MOSDPGPush(10.7 529/503 [PushOp(10:e5a676ff:::benchmark_data_smithi138_573643_object20515:head, version: 482'1328, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:e5a676ff:::benchmark_data_smithi138_573643_object20515:head@482'1328, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-01T09:44:26.539 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:26.459501+0000", 2023-10-01T09:44:26.539 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.002605343, 2023-10-01T09:44:26.539 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0032487739999999998, 2023-10-01T09:44:26.539 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.539 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:44:26.539 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.539 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.539 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.539 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.459501+0000", 2023-10-01T09:44:26.539 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.540 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.540 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.540 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.540 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.459501+0000", 2023-10-01T09:44:26.540 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:44:26.540 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.540 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.540 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.540 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.459500+0000", 2023-10-01T09:44:26.540 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4845000000000003e-05 2023-10-01T09:44:26.540 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.540 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.541 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.541 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.459535+0000", 2023-10-01T09:44:26.541 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5962e-05 2023-10-01T09:44:26.541 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.541 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.541 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.541 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.459561+0000", 2023-10-01T09:44:26.541 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.341e-05 2023-10-01T09:44:26.541 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.541 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.541 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.541 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.459574+0000", 2023-10-01T09:44:26.542 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3324000000000005e-05 2023-10-01T09:44:26.542 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.542 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.542 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.542 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.542 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.542 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103457 10.7 10:ee11e46b:::benchmark_data_smithi138_573643_object20235:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.542 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:24.706125+0000", 2023-10-01T09:44:26.542 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.755981783, 2023-10-01T09:44:26.542 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.756654457, 2023-10-01T09:44:26.542 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.542 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:44:26.543 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.543 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.543 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.543 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103457 2023-10-01T09:44:26.543 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.543 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.543 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.543 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.543 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.706125+0000", 2023-10-01T09:44:26.543 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.543 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.543 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.544 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.544 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.706125+0000", 2023-10-01T09:44:26.544 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:44:26.544 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.544 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.544 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.544 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.706123+0000", 2023-10-01T09:44:26.544 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3090000000000001e-06 2023-10-01T09:44:26.544 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.544 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.544 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.544 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.706130+0000", 2023-10-01T09:44:26.545 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2388000000000001e-05 2023-10-01T09:44:26.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.545 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.545 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.706142+0000", 2023-10-01T09:44:26.545 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9869999999999996e-06 2023-10-01T09:44:26.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.545 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.545 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.706148+0000", 2023-10-01T09:44:26.545 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016779889999999999 2023-10-01T09:44:26.545 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.545 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.546 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.722928+0000", 2023-10-01T09:44:26.546 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0054314460000000004 2023-10-01T09:44:26.546 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.546 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.546 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.728359+0000", 2023-10-01T09:44:26.546 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.080916096000000007 2023-10-01T09:44:26.546 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.546 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.546 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.547 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.809275+0000", 2023-10-01T09:44:26.547 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0057216339999999997 2023-10-01T09:44:26.547 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.547 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.547 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:44:26.547 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:24.814997+0000", 2023-10-01T09:44:26.547 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6173688739999998 2023-10-01T09:44:26.547 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.547 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.547 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.547 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.432366+0000", 2023-10-01T09:44:26.547 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0042923170000000004 2023-10-01T09:44:26.548 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.548 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.548 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:44:26.548 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.436658+0000", 2023-10-01T09:44:26.548 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0089436510000000004 2023-10-01T09:44:26.548 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.548 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.548 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:44:26.548 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.445602+0000", 2023-10-01T09:44:26.548 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.01511183 2023-10-01T09:44:26.548 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.548 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.549 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:26.549 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:26.460714+0000", 2023-10-01T09:44:26.549 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7545890339999999 2023-10-01T09:44:26.549 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.549 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.549 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.549 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.549 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.549 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103654 10.7 10:ecd34ab6:::benchmark_data_smithi138_573643_object20432:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.549 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:25.745916+0000", 2023-10-01T09:44:26.549 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.71619061500000003, 2023-10-01T09:44:26.549 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.71692226400000003, 2023-10-01T09:44:26.550 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.550 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:44:26.550 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.550 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.550 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.550 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103654 2023-10-01T09:44:26.550 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.550 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.550 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.550 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.550 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.745916+0000", 2023-10-01T09:44:26.550 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.551 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.551 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.551 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.551 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.745916+0000", 2023-10-01T09:44:26.551 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:44:26.551 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.551 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.551 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.551 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.745915+0000", 2023-10-01T09:44:26.551 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9529999999999999e-06 2023-10-01T09:44:26.551 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.551 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.552 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.745922+0000", 2023-10-01T09:44:26.552 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3311e-05 2023-10-01T09:44:26.552 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.552 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.745936+0000", 2023-10-01T09:44:26.552 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9540000000000001e-06 2023-10-01T09:44:26.552 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.552 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.552 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.552 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.745942+0000", 2023-10-01T09:44:26.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.028000316000000001 2023-10-01T09:44:26.553 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.553 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.553 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.773942+0000", 2023-10-01T09:44:26.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053366739999999996 2023-10-01T09:44:26.553 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.553 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.553 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.553 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.779279+0000", 2023-10-01T09:44:26.553 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.082029332999999996 2023-10-01T09:44:26.553 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.554 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.554 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.554 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.861308+0000", 2023-10-01T09:44:26.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0051957130000000002 2023-10-01T09:44:26.554 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.554 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.554 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:44:26.554 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.866504+0000", 2023-10-01T09:44:26.554 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.12058793 2023-10-01T09:44:26.554 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.554 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.554 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.555 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.555 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.555 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103501 10.7 10:e45fae71:::benchmark_data_smithi138_573643_object20279:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.555 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:25.030055+0000", 2023-10-01T09:44:26.555 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 1.4320514449999999, 2023-10-01T09:44:26.555 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4328287419999999, 2023-10-01T09:44:26.555 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.555 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:44:26.555 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.555 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.555 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.555 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103501 2023-10-01T09:44:26.556 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.556 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.556 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.556 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.030055+0000", 2023-10-01T09:44:26.556 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.556 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.556 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.556 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.556 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.030055+0000", 2023-10-01T09:44:26.556 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:44:26.556 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.030053+0000", 2023-10-01T09:44:26.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.3870000000000001e-06 2023-10-01T09:44:26.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.030062+0000", 2023-10-01T09:44:26.557 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.49e-05 2023-10-01T09:44:26.557 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.557 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.557 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.557 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.030077+0000", 2023-10-01T09:44:26.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0730000000000005e-06 2023-10-01T09:44:26.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.558 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.030085+0000", 2023-10-01T09:44:26.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.108764794 2023-10-01T09:44:26.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.558 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.558 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.558 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.138850+0000", 2023-10-01T09:44:26.558 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012980478 2023-10-01T09:44:26.558 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.559 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.559 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.559 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.151831+0000", 2023-10-01T09:44:26.559 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.083241763999999996 2023-10-01T09:44:26.559 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.559 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.559 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.559 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.235073+0000", 2023-10-01T09:44:26.559 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0058020509999999999 2023-10-01T09:44:26.559 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.559 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.560 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:44:26.560 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.240875+0000", 2023-10-01T09:44:26.560 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.21081952200000001 2023-10-01T09:44:26.560 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.560 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.560 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.560 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.560 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.560 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:103655 10.7 10:e343e3c9:::benchmark_data_smithi138_573643_object20433:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:44:26.560 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:44:25.747578+0000", 2023-10-01T09:44:26.560 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.71452838699999999, 2023-10-01T09:44:26.560 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.71535169799999998, 2023-10-01T09:44:26.561 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:44:26.561 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "delayed", 2023-10-01T09:44:26.561 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:44:26.561 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:44:26.561 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:26.561 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 103655 2023-10-01T09:44:26.561 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.561 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:44:26.561 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.561 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:44:26.561 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.747578+0000", 2023-10-01T09:44:26.561 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:44:26.562 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.562 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:44:26.562 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.747578+0000", 2023-10-01T09:44:26.562 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:44:26.562 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.562 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.562 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:44:26.562 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.747577+0000", 2023-10-01T09:44:26.562 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5199999999999999e-06 2023-10-01T09:44:26.562 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.562 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.563 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:44:26.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.747582+0000", 2023-10-01T09:44:26.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.1260000000000004e-06 2023-10-01T09:44:26.563 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.563 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.563 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:44:26.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.747591+0000", 2023-10-01T09:44:26.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1359999999999999e-06 2023-10-01T09:44:26.563 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.563 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.563 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:44:26.563 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.747595+0000", 2023-10-01T09:44:26.563 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.121018985 2023-10-01T09:44:26.564 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.564 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.564 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.564 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.868614+0000", 2023-10-01T09:44:26.564 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0051776230000000001 2023-10-01T09:44:26.564 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.564 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.564 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:44:26.564 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.873792+0000", 2023-10-01T09:44:26.564 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.027952505999999998 2023-10-01T09:44:26.564 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.565 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.565 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:44:26.565 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.901744+0000", 2023-10-01T09:44:26.565 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0055483299999999998 2023-10-01T09:44:26.565 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:44:26.565 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:44:26.565 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:44:26.565 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:44:25.907292+0000", 2023-10-01T09:44:26.565 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15971433099999999 2023-10-01T09:44:26.565 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.565 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:44:26.565 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.566 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:44:26.566 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:44:26.566 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 19 2023-10-01T09:44:26.566 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:44:26.566 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:26.603 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:26.689 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:44:26.691 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:44:26.745 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:26.745 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:26.745 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:44:26.745 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:44:26.745 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:26.745 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:26 smithi190 ceph-mon[160435]: pgmap v1542: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 7507/189432 objects degraded (3.963%); 3881/189432 objects misplaced (2.049%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:44:26.756 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:26.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:26.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:26.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:44:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:44:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:26.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:26 smithi138 ceph-mon[192513]: pgmap v1542: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 7507/189432 objects degraded (3.963%); 3881/189432 objects misplaced (2.049%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:44:26.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:26.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:26.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:44:26.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:44:26.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:26.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:26 smithi138 ceph-mon[185886]: pgmap v1542: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 122 op/s; 7507/189432 objects degraded (3.963%); 3881/189432 objects misplaced (2.049%); 537 KiB/s, 8 objects/s recovering 2023-10-01T09:44:26.859 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:26.888 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:26.933 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:26.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:26.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:27.037 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:27.067 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:27.377 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:27.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:27.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:27.383 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T09:44:27.483 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:27.514 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:27.576 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:27.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:27.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:27.683 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:27.712 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:27.761 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:27 smithi190 ceph-mon[160435]: from='client.47009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:27.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:27 smithi138 ceph-mon[192513]: from='client.47009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:27.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:27 smithi138 ceph-mon[185886]: from='client.47009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:27.827 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1542,"stamp":"2023-10-01T09:44:26.326970+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4141094859,"num_objects":63144,"num_object_clones":0,"num_object_copies":189432,"num_objects_missing_on_primary":2412,"num_objects_missing":2412,"num_objects_degraded":7507,"num_objects_misplaced":3881,"num_objects_unfound":0,"num_objects_dirty":63144,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":104430,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8694,"num_bytes_recovered":556631055,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104494,"ondisk_log_size":104494,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17590960,"kb_used_data":12553976,"kb_used_omap":39,"kb_used_meta":5036824,"kb_avail":638625104,"statfs":{"total":671965249536,"available":653952106496,"internally_reserved":0,"allocated":12855271424,"data_stored":12845405035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5157708399},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-80412672,"num_objects":-1227,"num_object_clones":0,"num_object_copies":-3681,"num_objects_missing_on_primary":-84,"num_objects_missing":-84,"num_objects_degraded":-84,"num_objects_misplaced":-84,"num_objects_unfound":0,"num_objects_dirty":-1227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1227,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":5505024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005214"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":305,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.937954+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:44:13.937954+0000","last_peered":"2023-10-01T09:44:13.937954+0000","last_clean":"2023-10-01T09:44:13.937954+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:44:13.937954+0000","last_undegraded":"2023-10-01T09:44:13.937954+0000","last_fullsized":"2023-10-01T09:44:13.937954+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":148,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.952125+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:44:13.952125+0000","last_peered":"2023-10-01T09:44:13.952125+0000","last_clean":"2023-10-01T09:44:13.952125+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:44:13.952125+0000","last_undegraded":"2023-10-01T09:44:13.952125+0000","last_fullsized":"2023-10-01T09:44:13.952125+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":529,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.951696+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:44:13.951696+0000","last_peered":"2023-10-01T09:44:13.951696+0000","last_clean":"2023-10-01T09:44:13.951696+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:44:13.951696+0000","last_undegraded":"2023-10-01T09:44:13.951696+0000","last_fullsized":"2023-10-01T09:44:13.951696+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":586,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.474963+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:44:06.474963+0000","last_peered":"2023-10-01T09:44:06.474963+0000","last_clean":"2023-10-01T09:44:06.474963+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:44:06.474963+0000","last_undegraded":"2023-10-01T09:44:06.474963+0000","last_fullsized":"2023-10-01T09:44:06.474963+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":467,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.473701+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:44:06.473701+0000","last_peered":"2023-10-01T09:44:06.473701+0000","last_clean":"2023-10-01T09:44:06.473701+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:44:06.473701+0000","last_undegraded":"2023-10-01T09:44:06.473701+0000","last_fullsized":"2023-10-01T09:44:06.473701+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":209,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.573025+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:44:06.573025+0000","last_peered":"2023-10-01T09:44:06.573025+0000","last_clean":"2023-10-01T09:44:06.573025+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:44:06.573025+0000","last_undegraded":"2023-10-01T09:44:06.573025+0000","last_fullsized":"2023-10-01T09:44:06.573025+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":585,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.573144+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:44:06.573144+0000","last_peered":"2023-10-01T09:44:06.573144+0000","last_clean":"2023-10-01T09:44:06.573144+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:44:06.573144+0000","last_undegraded":"2023-10-01T09:44:06.573144+0000","last_fullsized":"2023-10-01T09:44:06.573144+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":515,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.786738+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:44:13.786738+0000","last_peered":"2023-10-01T09:44:13.786738+0000","last_clean":"2023-10-01T09:44:13.786738+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:44:13.786738+0000","last_undegraded":"2023-10-01T09:44:13.786738+0000","last_fullsized":"2023-10-01T09:44:13.786738+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":240,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.488170+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:44:06.488170+0000","last_peered":"2023-10-01T09:44:06.488170+0000","last_clean":"2023-10-01T09:44:06.488170+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:44:06.488170+0000","last_undegraded":"2023-10-01T09:44:06.488170+0000","last_fullsized":"2023-10-01T09:44:06.488170+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":48,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.792707+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:44:13.792707+0000","last_peered":"2023-10-01T09:44:13.792707+0000","last_clean":"2023-10-01T09:44:13.792707+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:44:13.792707+0000","last_undegraded":"2023-10-01T09:44:13.792707+0000","last_fullsized":"2023-10-01T09:44:13.792707+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":54,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.792792+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:44:13.792792+0000","last_peered":"2023-10-01T09:44:13.792792+0000","last_clean":"2023-10-01T09:44:13.792792+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:44:13.792792+0000","last_undegraded":"2023-10-01T09:44:13.792792+0000","last_fullsized":"2023-10-01T09:44:13.792792+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":52,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.951733+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:44:13.951733+0000","last_peered":"2023-10-01T09:44:13.951733+0000","last_clean":"2023-10-01T09:44:13.951733+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:44:13.951733+0000","last_undegraded":"2023-10-01T09:44:13.951733+0000","last_fullsized":"2023-10-01T09:44:13.951733+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":430,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.938910+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:44:13.938910+0000","last_peered":"2023-10-01T09:44:13.938910+0000","last_clean":"2023-10-01T09:44:13.938910+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:44:13.938910+0000","last_undegraded":"2023-10-01T09:44:13.938910+0000","last_fullsized":"2023-10-01T09:44:13.938910+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":311,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.939244+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:44:13.939244+0000","last_peered":"2023-10-01T09:44:13.939244+0000","last_clean":"2023-10-01T09:44:13.939244+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:44:13.939244+0000","last_undegraded":"2023-10-01T09:44:13.939244+0000","last_fullsized":"2023-10-01T09:44:13.939244+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":647,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.792592+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:44:13.792592+0000","last_peered":"2023-10-01T09:44:13.792592+0000","last_clean":"2023-10-01T09:44:13.792592+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:44:13.792592+0000","last_undegraded":"2023-10-01T09:44:13.792592+0000","last_fullsized":"2023-10-01T09:44:13.792592+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":384,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.792915+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:44:13.792915+0000","last_peered":"2023-10-01T09:44:13.792915+0000","last_clean":"2023-10-01T09:44:13.792915+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:44:13.792915+0000","last_undegraded":"2023-10-01T09:44:13.792915+0000","last_fullsized":"2023-10-01T09:44:13.792915+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":57,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.792589+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:44:13.792589+0000","last_peered":"2023-10-01T09:44:13.792589+0000","last_clean":"2023-10-01T09:44:13.792589+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:44:13.792589+0000","last_undegraded":"2023-10-01T09:44:13.792589+0000","last_fullsized":"2023-10-01T09:44:13.792589+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":42,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486396+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:44:06.486396+0000","last_peered":"2023-10-01T09:44:06.486396+0000","last_clean":"2023-10-01T09:44:06.486396+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:44:06.486396+0000","last_undegraded":"2023-10-01T09:44:06.486396+0000","last_fullsized":"2023-10-01T09:44:06.486396+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":540,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.792942+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:44:13.792942+0000","last_peered":"2023-10-01T09:44:13.792942+0000","last_clean":"2023-10-01T09:44:13.792942+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:44:13.792942+0000","last_undegraded":"2023-10-01T09:44:13.792942+0000","last_fullsized":"2023-10-01T09:44:13.792942+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":550,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.475030+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:44:06.475030+0000","last_peered":"2023-10-01T09:44:06.475030+0000","last_clean":"2023-10-01T09:44:06.475030+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:44:06.475030+0000","last_undegraded":"2023-10-01T09:44:06.475030+0000","last_fullsized":"2023-10-01T09:44:06.475030+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":406,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.473621+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:44:06.473621+0000","last_peered":"2023-10-01T09:44:06.473621+0000","last_clean":"2023-10-01T09:44:06.473621+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:44:06.473621+0000","last_undegraded":"2023-10-01T09:44:06.473621+0000","last_fullsized":"2023-10-01T09:44:06.473621+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":57,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.793564+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:44:13.793564+0000","last_peered":"2023-10-01T09:44:13.793564+0000","last_clean":"2023-10-01T09:44:13.793564+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:44:13.793564+0000","last_undegraded":"2023-10-01T09:44:13.793564+0000","last_fullsized":"2023-10-01T09:44:13.793564+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"528'6466","reported_seq":6527,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.336642+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:44:21.336642+0000","last_peered":"2023-10-01T09:44:21.336642+0000","last_clean":"2023-10-01T09:44:21.336642+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:44:21.336642+0000","last_undegraded":"2023-10-01T09:44:21.336642+0000","last_fullsized":"2023-10-01T09:44:21.336642+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6466,"log_dups_size":0,"ondisk_log_size":6466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257294336,"num_objects":3926,"num_object_clones":0,"num_object_copies":11778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6466,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":52,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.793233+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:44:13.793233+0000","last_peered":"2023-10-01T09:44:13.793233+0000","last_clean":"2023-10-01T09:44:13.793233+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:44:13.793233+0000","last_undegraded":"2023-10-01T09:44:13.793233+0000","last_fullsized":"2023-10-01T09:44:13.793233+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":248,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.485987+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:44:06.485987+0000","last_peered":"2023-10-01T09:44:06.485987+0000","last_clean":"2023-10-01T09:44:06.485987+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:44:06.485987+0000","last_undegraded":"2023-10-01T09:44:06.485987+0000","last_fullsized":"2023-10-01T09:44:06.485987+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"528'6475","reported_seq":10383,"reported_epoch":528,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:44:21.695194+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:44:21.695194+0000","last_peered":"2023-10-01T09:44:21.695194+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:44:21.695194+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6475,"log_dups_size":0,"ondisk_log_size":6475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257622016,"num_objects":3931,"num_object_clones":0,"num_object_copies":11793,"num_objects_missing_on_primary":2412,"num_objects_missing":2412,"num_objects_degraded":6274,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6475,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1273,"num_bytes_recovered":83427328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1450},{"shards":"0,4,6","objects":2412},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":523,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.793133+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:44:13.793133+0000","last_peered":"2023-10-01T09:44:13.793133+0000","last_clean":"2023-10-01T09:44:13.793133+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:44:13.793133+0000","last_undegraded":"2023-10-01T09:44:13.793133+0000","last_fullsized":"2023-10-01T09:44:13.793133+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":584,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.951687+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:44:13.951687+0000","last_peered":"2023-10-01T09:44:13.951687+0000","last_clean":"2023-10-01T09:44:13.951687+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:44:13.951687+0000","last_undegraded":"2023-10-01T09:44:13.951687+0000","last_fullsized":"2023-10-01T09:44:13.951687+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"528'6474","reported_seq":6538,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.770525+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:44:21.770525+0000","last_peered":"2023-10-01T09:44:21.770525+0000","last_clean":"2023-10-01T09:44:21.770525+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:44:21.770525+0000","last_undegraded":"2023-10-01T09:44:21.770525+0000","last_fullsized":"2023-10-01T09:44:21.770525+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6474,"log_dups_size":0,"ondisk_log_size":6474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259391488,"num_objects":3958,"num_object_clones":0,"num_object_copies":11874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6474,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":609,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.792897+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:44:13.792897+0000","last_peered":"2023-10-01T09:44:13.792897+0000","last_clean":"2023-10-01T09:44:13.792897+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:44:13.792897+0000","last_undegraded":"2023-10-01T09:44:13.792897+0000","last_fullsized":"2023-10-01T09:44:13.792897+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":405,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.951941+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:44:13.951941+0000","last_peered":"2023-10-01T09:44:13.951941+0000","last_clean":"2023-10-01T09:44:13.951941+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:44:13.951941+0000","last_undegraded":"2023-10-01T09:44:13.951941+0000","last_fullsized":"2023-10-01T09:44:13.951941+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"528'6416","reported_seq":13085,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.797764+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:44:21.797764+0000","last_peered":"2023-10-01T09:44:21.797764+0000","last_clean":"2023-10-01T09:44:21.797764+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:44:21.797764+0000","last_undegraded":"2023-10-01T09:44:21.797764+0000","last_fullsized":"2023-10-01T09:44:21.797764+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6416,"log_dups_size":0,"ondisk_log_size":6416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261226496,"num_objects":3986,"num_object_clones":0,"num_object_copies":11958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6416,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":248,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486541+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:44:06.486541+0000","last_peered":"2023-10-01T09:44:06.486541+0000","last_clean":"2023-10-01T09:44:06.486541+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:44:06.486541+0000","last_undegraded":"2023-10-01T09:44:06.486541+0000","last_fullsized":"2023-10-01T09:44:06.486541+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":506,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.474139+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:44:06.474139+0000","last_peered":"2023-10-01T09:44:06.474139+0000","last_clean":"2023-10-01T09:44:06.474139+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:44:06.474139+0000","last_undegraded":"2023-10-01T09:44:06.474139+0000","last_fullsized":"2023-10-01T09:44:06.474139+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"529'6495","reported_seq":6560,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:25.196319+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:44:25.196319+0000","last_peered":"2023-10-01T09:44:25.196319+0000","last_clean":"2023-10-01T09:44:25.196319+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:44:25.196319+0000","last_undegraded":"2023-10-01T09:44:25.196319+0000","last_fullsized":"2023-10-01T09:44:25.196319+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6495,"log_dups_size":0,"ondisk_log_size":6495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262995968,"num_objects":4013,"num_object_clones":0,"num_object_copies":12039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6495,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":109,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.939597+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:44:13.939597+0000","last_peered":"2023-10-01T09:44:13.939597+0000","last_clean":"2023-10-01T09:44:13.939597+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:44:13.939597+0000","last_undegraded":"2023-10-01T09:44:13.939597+0000","last_fullsized":"2023-10-01T09:44:13.939597+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":183,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.938984+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:44:13.938984+0000","last_peered":"2023-10-01T09:44:13.938984+0000","last_clean":"2023-10-01T09:44:13.938984+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:44:13.938984+0000","last_undegraded":"2023-10-01T09:44:13.938984+0000","last_fullsized":"2023-10-01T09:44:13.938984+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"528'6423","reported_seq":6526,"reported_epoch":528,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:44:21.062934+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:44:21.062934+0000","last_peered":"2023-10-01T09:44:21.062934+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:44:21.062934+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6423,"log_dups_size":0,"ondisk_log_size":6423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258539520,"num_objects":3945,"num_object_clones":0,"num_object_copies":11835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1223,"num_objects_misplaced":3881,"num_objects_unfound":0,"num_objects_dirty":3945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6423,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":1114112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":3881},{"shards":"4,5","objects":-1159}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":427,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.938086+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:44:13.938086+0000","last_peered":"2023-10-01T09:44:13.938086+0000","last_clean":"2023-10-01T09:44:13.938086+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:44:13.938086+0000","last_undegraded":"2023-10-01T09:44:13.938086+0000","last_fullsized":"2023-10-01T09:44:13.938086+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":587,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.952261+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:44:13.952261+0000","last_peered":"2023-10-01T09:44:13.952261+0000","last_clean":"2023-10-01T09:44:13.952261+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:44:13.952261+0000","last_undegraded":"2023-10-01T09:44:13.952261+0000","last_fullsized":"2023-10-01T09:44:13.952261+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"529'6419","reported_seq":6490,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.179564+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:44:22.179564+0000","last_peered":"2023-10-01T09:44:22.179564+0000","last_clean":"2023-10-01T09:44:22.179564+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:44:22.179564+0000","last_undegraded":"2023-10-01T09:44:22.179564+0000","last_fullsized":"2023-10-01T09:44:22.179564+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6419,"log_dups_size":0,"ondisk_log_size":6419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259457024,"num_objects":3959,"num_object_clones":0,"num_object_copies":11877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6419,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":423,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.939806+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:44:13.939806+0000","last_peered":"2023-10-01T09:44:13.939806+0000","last_clean":"2023-10-01T09:44:13.939806+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:44:13.939806+0000","last_undegraded":"2023-10-01T09:44:13.939806+0000","last_fullsized":"2023-10-01T09:44:13.939806+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":264,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.938608+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:44:13.938608+0000","last_peered":"2023-10-01T09:44:13.938608+0000","last_clean":"2023-10-01T09:44:13.938608+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:44:13.938608+0000","last_undegraded":"2023-10-01T09:44:13.938608+0000","last_fullsized":"2023-10-01T09:44:13.938608+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"529'6561","reported_seq":6623,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:25.806001+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:44:25.806001+0000","last_peered":"2023-10-01T09:44:25.806001+0000","last_clean":"2023-10-01T09:44:25.806001+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:44:25.806001+0000","last_undegraded":"2023-10-01T09:44:25.806001+0000","last_fullsized":"2023-10-01T09:44:25.806001+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6561,"log_dups_size":0,"ondisk_log_size":6561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258015232,"num_objects":3937,"num_object_clones":0,"num_object_copies":11811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6561,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":392,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.951907+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:44:13.951907+0000","last_peered":"2023-10-01T09:44:13.951907+0000","last_clean":"2023-10-01T09:44:13.951907+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:44:13.951907+0000","last_undegraded":"2023-10-01T09:44:13.951907+0000","last_fullsized":"2023-10-01T09:44:13.951907+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":439,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.952461+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:44:13.952461+0000","last_peered":"2023-10-01T09:44:13.952461+0000","last_clean":"2023-10-01T09:44:13.952461+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:44:13.952461+0000","last_undegraded":"2023-10-01T09:44:13.952461+0000","last_fullsized":"2023-10-01T09:44:13.952461+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"529'6394","reported_seq":6456,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:25.776186+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:44:25.776186+0000","last_peered":"2023-10-01T09:44:25.776186+0000","last_clean":"2023-10-01T09:44:25.776186+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:44:25.776186+0000","last_undegraded":"2023-10-01T09:44:25.776186+0000","last_fullsized":"2023-10-01T09:44:25.776186+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6394,"log_dups_size":0,"ondisk_log_size":6394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247988224,"num_objects":3784,"num_object_clones":0,"num_object_copies":11352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6394,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":922,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.792770+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:44:13.792770+0000","last_peered":"2023-10-01T09:44:13.792770+0000","last_clean":"2023-10-01T09:44:13.792770+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:44:13.792770+0000","last_undegraded":"2023-10-01T09:44:13.792770+0000","last_fullsized":"2023-10-01T09:44:13.792770+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":258,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486542+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:44:06.486542+0000","last_peered":"2023-10-01T09:44:06.486542+0000","last_clean":"2023-10-01T09:44:06.486542+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:44:06.486542+0000","last_undegraded":"2023-10-01T09:44:06.486542+0000","last_fullsized":"2023-10-01T09:44:06.486542+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":52,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.951811+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:44:13.951811+0000","last_peered":"2023-10-01T09:44:13.951811+0000","last_clean":"2023-10-01T09:44:13.951811+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:44:13.951811+0000","last_undegraded":"2023-10-01T09:44:13.951811+0000","last_fullsized":"2023-10-01T09:44:13.951811+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"528'6372","reported_seq":6435,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.288764+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:44:21.288764+0000","last_peered":"2023-10-01T09:44:21.288764+0000","last_clean":"2023-10-01T09:44:21.288764+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:44:21.288764+0000","last_undegraded":"2023-10-01T09:44:21.288764+0000","last_fullsized":"2023-10-01T09:44:21.288764+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6372,"log_dups_size":0,"ondisk_log_size":6372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262012928,"num_objects":3998,"num_object_clones":0,"num_object_copies":11994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6372,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1193,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486093+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:44:06.486093+0000","last_peered":"2023-10-01T09:44:06.486093+0000","last_clean":"2023-10-01T09:44:06.486093+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:44:06.486093+0000","last_undegraded":"2023-10-01T09:44:06.486093+0000","last_fullsized":"2023-10-01T09:44:06.486093+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":404,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.939447+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:44:13.939447+0000","last_peered":"2023-10-01T09:44:13.939447+0000","last_clean":"2023-10-01T09:44:13.939447+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:44:13.939447+0000","last_undegraded":"2023-10-01T09:44:13.939447+0000","last_fullsized":"2023-10-01T09:44:13.939447+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":231,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486678+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:44:06.486678+0000","last_peered":"2023-10-01T09:44:06.486678+0000","last_clean":"2023-10-01T09:44:06.486678+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:44:06.486678+0000","last_undegraded":"2023-10-01T09:44:06.486678+0000","last_fullsized":"2023-10-01T09:44:06.486678+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"528'6453","reported_seq":13735,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.300795+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:44:21.300795+0000","last_peered":"2023-10-01T09:44:21.300795+0000","last_clean":"2023-10-01T09:44:21.300795+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:44:21.300795+0000","last_undegraded":"2023-10-01T09:44:21.300795+0000","last_fullsized":"2023-10-01T09:44:21.300795+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6453,"log_dups_size":0,"ondisk_log_size":6453,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257097728,"num_objects":3923,"num_object_clones":0,"num_object_copies":11769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6453,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":873,"reported_epoch":527,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:44:06.473647+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:44:06.473647+0000","last_peered":"2023-10-01T09:44:06.473647+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:44:06.473647+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:44:06.473647+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":592,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:15.258894+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:44:15.258894+0000","last_peered":"2023-10-01T09:44:15.258894+0000","last_clean":"2023-10-01T09:44:15.258894+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:44:15.258894+0000","last_undegraded":"2023-10-01T09:44:15.258894+0000","last_fullsized":"2023-10-01T09:44:15.258894+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":52,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.952376+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:44:13.952376+0000","last_peered":"2023-10-01T09:44:13.952376+0000","last_clean":"2023-10-01T09:44:13.952376+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:44:13.952376+0000","last_undegraded":"2023-10-01T09:44:13.952376+0000","last_fullsized":"2023-10-01T09:44:13.952376+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"529'6393","reported_seq":6457,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:25.241212+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:44:25.241212+0000","last_peered":"2023-10-01T09:44:25.241212+0000","last_clean":"2023-10-01T09:44:25.241212+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:44:25.241212+0000","last_undegraded":"2023-10-01T09:44:25.241212+0000","last_fullsized":"2023-10-01T09:44:25.241212+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6393,"log_dups_size":0,"ondisk_log_size":6393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254214144,"num_objects":3879,"num_object_clones":0,"num_object_copies":11637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6393,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1644,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486225+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:44:06.486225+0000","last_peered":"2023-10-01T09:44:06.486225+0000","last_clean":"2023-10-01T09:44:06.486225+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:44:06.486225+0000","last_undegraded":"2023-10-01T09:44:06.486225+0000","last_fullsized":"2023-10-01T09:44:06.486225+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":559,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.792400+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:44:13.792400+0000","last_peered":"2023-10-01T09:44:13.792400+0000","last_clean":"2023-10-01T09:44:13.792400+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:44:13.792400+0000","last_undegraded":"2023-10-01T09:44:13.792400+0000","last_fullsized":"2023-10-01T09:44:13.792400+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":336,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486131+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:44:06.486131+0000","last_peered":"2023-10-01T09:44:06.486131+0000","last_clean":"2023-10-01T09:44:06.486131+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:44:06.486131+0000","last_undegraded":"2023-10-01T09:44:06.486131+0000","last_fullsized":"2023-10-01T09:44:06.486131+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":327,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.938502+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:44:13.938502+0000","last_peered":"2023-10-01T09:44:13.938502+0000","last_clean":"2023-10-01T09:44:13.938502+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:44:13.938502+0000","last_undegraded":"2023-10-01T09:44:13.938502+0000","last_fullsized":"2023-10-01T09:44:13.938502+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"529'6542","reported_seq":6604,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:25.798927+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:44:25.798927+0000","last_peered":"2023-10-01T09:44:25.798927+0000","last_clean":"2023-10-01T09:44:25.798927+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:44:25.798927+0000","last_undegraded":"2023-10-01T09:44:25.798927+0000","last_fullsized":"2023-10-01T09:44:25.798927+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6542,"log_dups_size":0,"ondisk_log_size":6542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261488640,"num_objects":3990,"num_object_clones":0,"num_object_copies":11970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6542,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":736,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.792965+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:44:13.792965+0000","last_peered":"2023-10-01T09:44:13.792965+0000","last_clean":"2023-10-01T09:44:13.792965+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:44:13.792965+0000","last_undegraded":"2023-10-01T09:44:13.792965+0000","last_fullsized":"2023-10-01T09:44:13.792965+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":568,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.793556+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:44:13.793556+0000","last_peered":"2023-10-01T09:44:13.793556+0000","last_clean":"2023-10-01T09:44:13.793556+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:44:13.793556+0000","last_undegraded":"2023-10-01T09:44:13.793556+0000","last_fullsized":"2023-10-01T09:44:13.793556+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":567,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.474841+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:44:06.474841+0000","last_peered":"2023-10-01T09:44:06.474841+0000","last_clean":"2023-10-01T09:44:06.474841+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:44:06.474841+0000","last_undegraded":"2023-10-01T09:44:06.474841+0000","last_fullsized":"2023-10-01T09:44:06.474841+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"528'6447","reported_seq":6511,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.305136+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:44:21.305136+0000","last_peered":"2023-10-01T09:44:21.305136+0000","last_clean":"2023-10-01T09:44:21.305136+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:44:21.305136+0000","last_undegraded":"2023-10-01T09:44:21.305136+0000","last_fullsized":"2023-10-01T09:44:21.305136+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6447,"log_dups_size":0,"ondisk_log_size":6447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262930456,"num_objects":4013,"num_object_clones":0,"num_object_copies":12039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4013,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6447,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":967,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.792990+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:44:13.792990+0000","last_peered":"2023-10-01T09:44:13.792990+0000","last_clean":"2023-10-01T09:44:13.792990+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:44:13.792990+0000","last_undegraded":"2023-10-01T09:44:13.792990+0000","last_fullsized":"2023-10-01T09:44:13.792990+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":585,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486611+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:44:06.486611+0000","last_peered":"2023-10-01T09:44:06.486611+0000","last_clean":"2023-10-01T09:44:06.486611+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:44:06.486611+0000","last_undegraded":"2023-10-01T09:44:06.486611+0000","last_fullsized":"2023-10-01T09:44:06.486611+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":438,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486307+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:44:06.486307+0000","last_peered":"2023-10-01T09:44:06.486307+0000","last_clean":"2023-10-01T09:44:06.486307+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:44:06.486307+0000","last_undegraded":"2023-10-01T09:44:06.486307+0000","last_fullsized":"2023-10-01T09:44:06.486307+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"529'6477","reported_seq":6547,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:25.739833+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:44:25.739833+0000","last_peered":"2023-10-01T09:44:25.739833+0000","last_clean":"2023-10-01T09:44:25.739833+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:44:25.739833+0000","last_undegraded":"2023-10-01T09:44:25.739833+0000","last_fullsized":"2023-10-01T09:44:25.739833+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6477,"log_dups_size":0,"ondisk_log_size":6477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254214144,"num_objects":3879,"num_object_clones":0,"num_object_copies":11637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6477,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":468,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.793426+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:44:13.793426+0000","last_peered":"2023-10-01T09:44:13.793426+0000","last_clean":"2023-10-01T09:44:13.793426+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:44:13.793426+0000","last_undegraded":"2023-10-01T09:44:13.793426+0000","last_fullsized":"2023-10-01T09:44:13.793426+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1161,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.952590+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:44:13.952590+0000","last_peered":"2023-10-01T09:44:13.952590+0000","last_clean":"2023-10-01T09:44:13.952590+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:44:13.952590+0000","last_undegraded":"2023-10-01T09:44:13.952590+0000","last_fullsized":"2023-10-01T09:44:13.952590+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":57,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.792371+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:44:13.792371+0000","last_peered":"2023-10-01T09:44:13.792371+0000","last_clean":"2023-10-01T09:44:13.792371+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:44:13.792371+0000","last_undegraded":"2023-10-01T09:44:13.792371+0000","last_fullsized":"2023-10-01T09:44:13.792371+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"529'6517","reported_seq":6587,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:25.824565+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:44:25.824565+0000","last_peered":"2023-10-01T09:44:25.824565+0000","last_clean":"2023-10-01T09:44:25.824565+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:44:25.824565+0000","last_undegraded":"2023-10-01T09:44:25.824565+0000","last_fullsized":"2023-10-01T09:44:25.824565+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6517,"log_dups_size":0,"ondisk_log_size":6517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261816320,"num_objects":3995,"num_object_clones":0,"num_object_copies":11985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6517,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":61,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.793308+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:44:13.793308+0000","last_peered":"2023-10-01T09:44:13.793308+0000","last_clean":"2023-10-01T09:44:13.793308+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:44:13.793308+0000","last_undegraded":"2023-10-01T09:44:13.793308+0000","last_fullsized":"2023-10-01T09:44:13.793308+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":545,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.474909+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:44:06.474909+0000","last_peered":"2023-10-01T09:44:06.474909+0000","last_clean":"2023-10-01T09:44:06.474909+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:44:06.474909+0000","last_undegraded":"2023-10-01T09:44:06.474909+0000","last_fullsized":"2023-10-01T09:44:06.474909+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":251,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486457+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:44:06.486457+0000","last_peered":"2023-10-01T09:44:06.486457+0000","last_clean":"2023-10-01T09:44:06.486457+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:44:06.486457+0000","last_undegraded":"2023-10-01T09:44:06.486457+0000","last_fullsized":"2023-10-01T09:44:06.486457+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":494,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.474290+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:44:06.474290+0000","last_peered":"2023-10-01T09:44:06.474290+0000","last_clean":"2023-10-01T09:44:06.474290+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:44:06.474290+0000","last_undegraded":"2023-10-01T09:44:06.474290+0000","last_fullsized":"2023-10-01T09:44:06.474290+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":609,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.951950+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:44:13.951950+0000","last_peered":"2023-10-01T09:44:13.951950+0000","last_clean":"2023-10-01T09:44:13.951950+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:44:13.951950+0000","last_undegraded":"2023-10-01T09:44:13.951950+0000","last_fullsized":"2023-10-01T09:44:13.951950+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":263,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486466+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:44:06.486466+0000","last_peered":"2023-10-01T09:44:06.486466+0000","last_clean":"2023-10-01T09:44:06.486466+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:44:06.486466+0000","last_undegraded":"2023-10-01T09:44:06.486466+0000","last_fullsized":"2023-10-01T09:44:06.486466+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":52,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.951732+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:44:13.951732+0000","last_peered":"2023-10-01T09:44:13.951732+0000","last_clean":"2023-10-01T09:44:13.951732+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:44:13.951732+0000","last_undegraded":"2023-10-01T09:44:13.951732+0000","last_fullsized":"2023-10-01T09:44:13.951732+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":263,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.486626+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:44:06.486626+0000","last_peered":"2023-10-01T09:44:06.486626+0000","last_clean":"2023-10-01T09:44:06.486626+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:44:06.486626+0000","last_undegraded":"2023-10-01T09:44:06.486626+0000","last_fullsized":"2023-10-01T09:44:06.486626+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":440,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.473807+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:44:06.473807+0000","last_peered":"2023-10-01T09:44:06.473807+0000","last_clean":"2023-10-01T09:44:06.473807+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:44:06.473807+0000","last_undegraded":"2023-10-01T09:44:06.473807+0000","last_fullsized":"2023-10-01T09:44:06.473807+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":70,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.952333+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:44:13.952333+0000","last_peered":"2023-10-01T09:44:13.952333+0000","last_clean":"2023-10-01T09:44:13.952333+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:44:13.952333+0000","last_undegraded":"2023-10-01T09:44:13.952333+0000","last_fullsized":"2023-10-01T09:44:13.952333+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":485,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-10-01T09:44:06.473799+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:44:06.473799+0000","last_peered":"2023-10-01T09:44:06.473799+0000","last_clean":"2023-10-01T09:44:06.473799+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:44:06.473799+0000","last_undegraded":"2023-10-01T09:44:06.473799+0000","last_fullsized":"2023-10-01T09:44:06.473799+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":685,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.794451+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:44:13.794451+0000","last_peered":"2023-10-01T09:44:13.794451+0000","last_clean":"2023-10-01T09:44:13.794451+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:44:13.794451+0000","last_undegraded":"2023-10-01T09:44:13.794451+0000","last_fullsized":"2023-10-01T09:44:13.794451+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":380,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-10-01T09:44:13.937775+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:44:13.937775+0000","last_peered":"2023-10-01T09:44:13.937775+0000","last_clean":"2023-10-01T09:44:13.937775+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:44:13.937775+0000","last_undegraded":"2023-10-01T09:44:13.937775+0000","last_fullsized":"2023-10-01T09:44:13.937775+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4136304664,"num_objects":63116,"num_object_clones":0,"num_object_copies":189348,"num_objects_missing_on_primary":2412,"num_objects_missing":2412,"num_objects_degraded":7497,"num_objects_misplaced":3881,"num_objects_unfound":0,"num_objects_dirty":63116,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":103324,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8183,"num_bytes_recovered":536281088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12797423616,"data_stored":12797411400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103324,"ondisk_log_size":103324,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430726,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2159384,"kb_used_data":1260368,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":91585768,"statfs":{"total":95995035648,"available":93783826432,"internally_reserved":0,"allocated":1290616832,"data_stored":1289678156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":1.1359999999999999,"15min":2.657},"min":{"1min":0.27300000000000002,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":1.484,"5min":10.391999999999999,"15min":108.90000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.39900000000000002,"5min":1.26,"15min":2.8090000000000002},"min":{"1min":0.219,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.67200000000000004,"5min":12.026,"15min":109.187},"last":0.434}]},{"osd":1,"last update":"Sun Oct 1 09:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":4.2910000000000004,"15min":5.2409999999999997},"min":{"1min":0.28999999999999998,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":1.4470000000000001,"5min":229.161,"15min":229.161},"last":0.314},{"interface":"front","average":{"1min":0.42999999999999999,"5min":4.2759999999999998,"15min":5.2510000000000003},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":0.96499999999999997,"5min":230.126,"15min":230.126},"last":0.48199999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:43:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.169,"15min":1.76},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.992,"5min":14.866,"15min":14.901999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.4450000000000001,"15min":1.7170000000000001},"min":{"1min":0.32000000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4490000000000001,"5min":14.682,"15min":14.682},"last":0.39900000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.96899999999999997,"15min":2.569},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.018,"5min":6.0069999999999997,"15min":80.352999999999994},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.51300000000000001,"5min":1.1359999999999999,"15min":2.5510000000000002},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.026,"5min":14.528,"15min":80.364000000000004},"last":0.55500000000000005}]},{"osd":4,"last update":"Sun Oct 1 09:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":1.046,"15min":1.0429999999999999},"min":{"1min":0.30199999999999999,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.508,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.0289999999999999,"15min":1.0189999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.28000000000000003},"max":{"1min":0.84399999999999997,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.46300000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":2.3620000000000001,"15min":1.6120000000000001},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":1.2749999999999999,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.62,"5min":2.5390000000000001,"15min":1.6739999999999999},"min":{"1min":0.35999999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.2599999999999998,"5min":135.339,"15min":135.339},"last":0.626}]},{"osd":6,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":1.3200000000000001,"15min":1.1599999999999999},"min":{"1min":0.377,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":1.409,"5min":11.58,"15min":11.853999999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":1.175,"15min":1.1299999999999999},"min":{"1min":0.378,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":1.23,"5min":14.584,"15min":14.584},"last":0.59899999999999998}]}]},{"osd":6,"up_from":294,"seq":1262720385236,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2123284,"kb_used_data":1781580,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":91621868,"statfs":{"total":95995035648,"available":93820792832,"internally_reserved":0,"allocated":1824337920,"data_stored":1823403195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":2.0630000000000002,"15min":4.0090000000000003},"min":{"1min":0.247,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":3.2029999999999998,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.41399999999999998,"5min":2.1739999999999999,"15min":4.0730000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.221},"max":{"1min":0.63900000000000001,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.45600000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":3.1379999999999999,"15min":4.8600000000000003},"min":{"1min":0.29199999999999998,"5min":0.254,"15min":0.24199999999999999},"max":{"1min":0.58899999999999997,"5min":168.947,"15min":168.947},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.56000000000000005,"5min":3.141,"15min":4.8200000000000003},"min":{"1min":0.32500000000000001,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":2.8290000000000002,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.46400000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.78400000000000003,"15min":1.5620000000000001},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":0.65700000000000003,"5min":8.657,"15min":10.92},"last":0.315},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.88600000000000001,"15min":1.5629999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":2.863,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.60599999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.96499999999999997,"15min":2.101},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.3149999999999999,"5min":9.0180000000000007,"15min":89.753},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.439,"5min":1.097,"15min":2.0179999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.61199999999999999,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.39500000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.5049999999999999,"15min":1.0740000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.7749999999999999,"5min":63.134,"15min":63.134},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.478,"15min":1.075},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":3.2519999999999998,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.48099999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:43:29 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":1.2450000000000001,"15min":1.101},"min":{"1min":0.309,"5min":0.251,"15min":0.251},"max":{"1min":34.094999999999999,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.409,"15min":1.22},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":34.072000000000003,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.56699999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.73999999999999999,"15min":1.2729999999999999},"min":{"1min":0.32700000000000001,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":3.339,"5min":3.8279999999999998,"15min":94.245999999999995},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.496,"5min":0.96999999999999997,"15min":1.3640000000000001},"min":{"1min":0.374,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":0.68600000000000005,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.42999999999999999}]}]},{"osd":1,"up_from":94,"seq":403726926213,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1572936,"kb_used_data":766976,"kb_used_omap":7,"kb_used_meta":805880,"kb_avail":92172216,"statfs":{"total":95995035648,"available":94384349184,"internally_reserved":0,"allocated":785383424,"data_stored":783346881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":825221543},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":1.7789999999999999,"15min":4.0039999999999996},"min":{"1min":0.186,"5min":0.186,"15min":0.182},"max":{"1min":1.2889999999999999,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":1.99,"15min":3.5329999999999999},"min":{"1min":0.223,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":1.702,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.249}]},{"osd":2,"last update":"Sun Oct 1 09:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":1.4279999999999999,"15min":2.589},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.61399999999999999,"5min":22.891999999999999,"15min":56.573},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.379,"15min":3.254},"min":{"1min":0.28999999999999998,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":1.752,"5min":15.189,"15min":154.197},"last":0.28299999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":1.2769999999999999,"15min":3.7909999999999999},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":1.7889999999999999,"5min":18.503,"15min":200.56800000000001},"last":0.31},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.2330000000000001,"15min":4.5039999999999996},"min":{"1min":0.253,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":1.9370000000000001,"5min":16.515000000000001,"15min":200.63800000000001},"last":0.38100000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.147,"15min":2.2530000000000001},"min":{"1min":0.35499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.0550000000000002,"5min":19.539000000000001,"15min":154.101},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.46899999999999997,"5min":1.0029999999999999,"15min":2.399},"min":{"1min":0.29499999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.59799999999999998,"5min":10.084,"15min":154.56999999999999},"last":0.42599999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:43:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.3200000000000001,"15min":2.097},"min":{"1min":0.372,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.1850000000000001,"5min":19.513000000000002,"15min":154.22999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.1719999999999999,"15min":2.4630000000000001},"min":{"1min":0.31,"5min":0.307,"15min":0.307},"max":{"1min":0.82699999999999996,"5min":10.305,"15min":154.54400000000001},"last":0.40799999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.236,"15min":2.5539999999999998},"min":{"1min":0.34300000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.4399999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.81399999999999995,"5min":1.335,"15min":2.4119999999999999},"min":{"1min":0.34699999999999998,"5min":0.34599999999999997,"15min":0.32700000000000001},"max":{"1min":2.5630000000000002,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.47199999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.3109999999999999,"15min":2.5840000000000001},"min":{"1min":0.38400000000000001,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":2.8170000000000002,"5min":19.431999999999999,"15min":154.20699999999999},"last":0.504},{"interface":"front","average":{"1min":0.77800000000000002,"5min":1.2330000000000001,"15min":2.6379999999999999},"min":{"1min":0.45200000000000001,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":2.8290000000000002,"5min":10.111000000000001,"15min":154.51599999999999},"last":0.45400000000000001}]}]},{"osd":0,"up_from":89,"seq":382252089747,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4101872,"kb_used_data":3206632,"kb_used_omap":8,"kb_used_meta":895223,"kb_avail":89643280,"statfs":{"total":95995035648,"available":91794718720,"internally_reserved":0,"allocated":3283591168,"data_stored":3281575857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":916709314},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:44:16 2023","interfaces":[{"interface":"back","average":{"1min":0.317,"5min":1.1839999999999999,"15min":4.1120000000000001},"min":{"1min":0.21199999999999999,"5min":0.17799999999999999,"15min":0.17399999999999999},"max":{"1min":0.46300000000000002,"5min":52.316000000000003,"15min":216.821},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.317,"5min":2.7919999999999998,"15min":4.9809999999999999},"min":{"1min":0.24399999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.435,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.39100000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:43:36 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":1.204,"15min":2.085},"min":{"1min":0.246,"5min":0.246,"15min":0.20100000000000001},"max":{"1min":1.1100000000000001,"5min":19.788,"15min":19.788},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.4139999999999999,"15min":2.3380000000000001},"min":{"1min":0.27200000000000002,"5min":0.23000000000000001,"15min":0.189},"max":{"1min":3.5939999999999999,"5min":19.375,"15min":25.844999999999999},"last":0.42399999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:44:16 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.90200000000000002,"15min":3.46},"min":{"1min":0.26900000000000002,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.59199999999999997,"5min":8.2889999999999997,"15min":222.16499999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.0369999999999999,"15min":3.375},"min":{"1min":0.26900000000000002,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":3.4119999999999999,"5min":9.7539999999999996,"15min":222.227},"last":0.54000000000000004}]},{"osd":4,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.5449999999999999,"15min":2.1339999999999999},"min":{"1min":0.28799999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":3.375,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":1.4550000000000001,"15min":2.1309999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":3.5049999999999999,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.436}]},{"osd":5,"last update":"Sun Oct 1 09:43:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.2030000000000001,"15min":1.883},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66500000000000004,"5min":14.515000000000001,"15min":16.808},"last":0.376},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.093,"15min":1.7709999999999999},"min":{"1min":0.33500000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.73499999999999999,"5min":5.9580000000000002,"15min":17.507000000000001},"last":0.40600000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:43:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.1850000000000001,"15min":1.774},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.35499999999999998},"max":{"1min":1.5900000000000001,"5min":6.5,"15min":17.535},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.88,"5min":1.395,"15min":2.0289999999999999},"min":{"1min":0.32600000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":3.6539999999999999,"5min":10.446,"15min":18.521000000000001},"last":0.63}]},{"osd":7,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.95499999999999996,"15min":2.048},"min":{"1min":0.379,"5min":0.374,"15min":0.35699999999999998},"max":{"1min":0.78600000000000003,"5min":4.976,"15min":48.895000000000003},"last":0.503},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.093,"15min":2.206},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.35099999999999998},"max":{"1min":3.4319999999999999,"5min":9.4969999999999999,"15min":48.274000000000001},"last":0.65700000000000003}]}]},{"osd":2,"up_from":0,"seq":425201762687,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599160,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1656576,"kb_used_data":824440,"kb_used_omap":3,"kb_used_meta":832124,"kb_avail":92088576,"statfs":{"total":95995035648,"available":94298701824,"internally_reserved":0,"allocated":844226560,"data_stored":842139538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":852095719},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":3.2000000000000002,"15min":4.8920000000000003},"min":{"1min":0.20100000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":1.3580000000000001,"5min":92.649000000000001,"15min":137.34399999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.36399999999999999,"5min":3.194,"15min":4.9740000000000002},"min":{"1min":0.21199999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":1.2549999999999999,"5min":92.284000000000006,"15min":137.73400000000001},"last":0.313}]},{"osd":1,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":1.8819999999999999,"15min":5.1500000000000004},"min":{"1min":0.25,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":1.1599999999999999,"5min":61.439999999999998,"15min":162.22999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.39600000000000002,"5min":1.788,"15min":5.181},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.4119999999999999,"5min":62.723999999999997,"15min":162.70699999999999},"last":0.32700000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.0780000000000001,"15min":1.748},"min":{"1min":0.27800000000000002,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":2.4710000000000001,"5min":14.254,"15min":14.254},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.66300000000000003,"15min":1.798},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":4.5049999999999999,"5min":4.5049999999999999,"15min":15.523},"last":0.34399999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.72,"15min":1.8080000000000001},"min":{"1min":0.35199999999999998,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":1.444,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.371},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.887,"15min":2.0630000000000002},"min":{"1min":0.29499999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.2250000000000001,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.41599999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:43:29 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":1.5589999999999999,"15min":2.581},"min":{"1min":0.317,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3099999999999996,"5min":10.919,"15min":28.117999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.4279999999999999,"15min":2.6800000000000002},"min":{"1min":0.36699999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.512,"5min":10.798999999999999,"15min":28.34},"last":0.38100000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:43:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":2.165,"15min":2.145},"min":{"1min":0.34200000000000003,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":2.5430000000000001,"5min":73.887,"15min":73.887},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.63,"5min":2.4159999999999999,"15min":2.1379999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":2.6000000000000001,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.40500000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":1.268,"15min":1.845},"min":{"1min":0.42299999999999999,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":2.573,"5min":11.659000000000001,"15min":13.085000000000001},"last":1.7030000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.0740000000000001,"15min":1.726},"min":{"1min":0.39700000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":1.472,"5min":8.3460000000000001,"15min":13.191000000000001},"last":1.6919999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019411,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3987412,"kb_used_data":3093132,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":89757740,"statfs":{"total":95995035648,"available":91911925760,"internally_reserved":0,"allocated":3167367168,"data_stored":3166431998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":2.3570000000000002,"15min":4.25},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.47299999999999998,"5min":79.837000000000003,"15min":155.315},"last":2.9889999999999999},{"interface":"front","average":{"1min":0.44900000000000001,"5min":2.4300000000000002,"15min":4.7039999999999997},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":0.63700000000000001,"5min":78.983000000000004,"15min":155.334},"last":0.26100000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":4.5709999999999997,"15min":6.4699999999999998},"min":{"1min":0.28899999999999998,"5min":0.252,"15min":0.224},"max":{"1min":0.52900000000000003,"5min":126.742,"15min":190.61199999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.40200000000000002,"5min":4.5369999999999999,"15min":7.1600000000000001},"min":{"1min":0.27600000000000002,"5min":0.26700000000000002,"15min":0.25600000000000001},"max":{"1min":0.59999999999999998,"5min":126.791,"15min":253.273},"last":3.1339999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.05,"15min":2.444},"min":{"1min":0.34200000000000003,"5min":0.31,"15min":0.28299999999999997},"max":{"1min":2.9510000000000001,"5min":10.161,"15min":170.45400000000001},"last":0.38},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.0029999999999999,"15min":2.1739999999999999},"min":{"1min":0.314,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":3.0219999999999998,"5min":5.9740000000000002,"15min":58.771000000000001},"last":0.40100000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.96499999999999997,"15min":2.3079999999999998},"min":{"1min":0.28100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.1299999999999999,"5min":7.5830000000000002,"15min":140.881},"last":3.0299999999999998},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.876,"15min":2.4830000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.311},"max":{"1min":0.51500000000000001,"5min":6.8719999999999999,"15min":140.911},"last":3.161}]},{"osd":5,"last update":"Sun Oct 1 09:43:33 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":2.0569999999999999,"15min":1.5580000000000001},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":3.052,"5min":110.003,"15min":110.003},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.64900000000000002,"5min":2.0110000000000001,"15min":1.399},"min":{"1min":0.33500000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.0049999999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":3.2909999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:44:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.78200000000000003,"15min":0.93000000000000005},"min":{"1min":0.33500000000000002,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":0.73599999999999999,"5min":9.9849999999999994,"15min":11.358000000000001},"last":3.0539999999999998},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.85099999999999998,"15min":1.0589999999999999},"min":{"1min":0.40600000000000003,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":0.73599999999999999,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.44700000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.71599999999999997,"15min":1.161},"min":{"1min":0.313,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":0.69399999999999995,"5min":4.7030000000000003,"15min":38.034999999999997},"last":3.0830000000000002},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.026,"15min":1.4410000000000001},"min":{"1min":0.36699999999999999,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":2.9940000000000002,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.79400000000000004}]}]},{"osd":5,"up_from":379,"seq":1627792605317,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1989496,"kb_used_data":1620848,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":91755656,"statfs":{"total":95995035648,"available":93957791744,"internally_reserved":0,"allocated":1659748352,"data_stored":1658829410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":1.6479999999999999,"15min":2.1890000000000001},"min":{"1min":0.22900000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.1880000000000002,"5min":12.571999999999999,"15min":84.340999999999994},"last":27.584},{"interface":"front","average":{"1min":0.40400000000000003,"5min":1.2549999999999999,"15min":2.0800000000000001},"min":{"1min":0.26600000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.53900000000000003,"5min":8.6460000000000008,"15min":84.356999999999999},"last":27.669}]},{"osd":1,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":6.4870000000000001,"15min":5.7599999999999998},"min":{"1min":0.26000000000000001,"5min":0.25900000000000001,"15min":0.255},"max":{"1min":0.76900000000000002,"5min":295.71300000000002,"15min":295.71300000000002},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":6.8099999999999996,"15min":5.7320000000000002},"min":{"1min":0.32200000000000001,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":2.258,"5min":299.59399999999999,"15min":299.59399999999999},"last":0.39100000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":1.3600000000000001,"15min":1.5980000000000001},"min":{"1min":0.315,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.752,"5min":8.7040000000000006,"15min":14.199999999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.44800000000000001,"5min":1.306,"15min":1.583},"min":{"1min":0.32900000000000001,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":0.85999999999999999,"5min":8.2370000000000001,"15min":14.430999999999999},"last":0.34499999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.5069999999999999,"15min":1.488},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.2280000000000002,"5min":12.734,"15min":14.004},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.5640000000000001,"15min":1.629},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.3979999999999999,"5min":11.766,"15min":13.991},"last":0.47499999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":2.2029999999999998,"15min":1.4330000000000001},"min":{"1min":0.30099999999999999,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.66800000000000004,"5min":122.971,"15min":122.971},"last":0.442},{"interface":"front","average":{"1min":0.438,"5min":2.25,"15min":1.724},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.73799999999999999,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.35999999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":1.165,"15min":1.044},"min":{"1min":0.312,"5min":0.311,"15min":0.28299999999999997},"max":{"1min":2.2839999999999998,"5min":12.545,"15min":12.545},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.99099999999999999,"15min":1.302},"min":{"1min":0.378,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":2.629,"5min":6.4729999999999999,"15min":14.193},"last":0.50700000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.1140000000000001,"15min":1.262},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.81200000000000006,"5min":8.2010000000000005,"15min":10.813000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.2270000000000001,"15min":1.0780000000000001},"min":{"1min":0.36299999999999999,"5min":0.35299999999999998,"15min":0.318},"max":{"1min":2.3079999999999998,"5min":12.500999999999999,"15min":12.500999999999999},"last":0.54800000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3271626752,"data_stored":3271622680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":774963200,"data_stored":774963200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":837877760,"data_stored":837877760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3160870912,"data_stored":3160866840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1654063104,"data_stored":1654063104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1817382912,"data_stored":1817378840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1280638976,"data_stored":1280638976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:44:27.834 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:44:27.880 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:44:28.012 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:28.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:28.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:28.028 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:44:28.120 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:28.155 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:28.308 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:28.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:28.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:28.415 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:28.447 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:28.486 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:28.495 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:28.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:28.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:28.530 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:28.599 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:28.628 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:28.683 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:28.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:28.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:28.787 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:28.822 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:29.018 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:29.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:28 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7507/189432 objects degraded (3.963%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:29.020 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:28 smithi138 ceph-mon[192513]: pgmap v1543: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 7458/188223 objects degraded (3.962%); 3843/188223 objects misplaced (2.042%); 557 KiB/s, 8 objects/s recovering 2023-10-01T09:44:29.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:28 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7507/189432 objects degraded (3.963%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:29.020 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:28 smithi138 ceph-mon[185886]: pgmap v1543: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 7458/188223 objects degraded (3.962%); 3843/188223 objects misplaced (2.042%); 557 KiB/s, 8 objects/s recovering 2023-10-01T09:44:29.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:29.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:28 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7507/189432 objects degraded (3.963%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:29.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:28 smithi190 ceph-mon[160435]: pgmap v1543: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 7458/188223 objects degraded (3.962%); 3843/188223 objects misplaced (2.042%); 557 KiB/s, 8 objects/s recovering 2023-10-01T09:44:29.122 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:29.156 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:29.291 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:29.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:29.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:29.396 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:29.424 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:29.624 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:44:29.625 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:44:29.625 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:44:29.625 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:44:29.625 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:44:29.771 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:44:29.821 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:29.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:29.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:29.925 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:29.952 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:30.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:29 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/587408580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:44:30.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:29 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:44:30.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:29 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/587408580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:44:30.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:29 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:44:30.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:29 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/587408580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:44:30.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:29 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:44:30.384 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T09:44:30.775 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:44:30.825 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:30.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:30.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:30.929 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:30.965 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:31.025 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:30 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:31.026 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:30 smithi190 ceph-mon[160435]: failed to encode map e530 with expected crc 2023-10-01T09:44:31.026 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:30 smithi190 ceph-mon[160435]: failed to encode map e530 with expected crc 2023-10-01T09:44:31.026 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:30 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:44:31.026 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:30 smithi190 ceph-mon[160435]: osdmap e530: 8 total, 8 up, 7 in 2023-10-01T09:44:31.026 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:30 smithi190 ceph-mon[160435]: failed to encode map e530 with expected crc 2023-10-01T09:44:31.026 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:30 smithi190 ceph-mon[160435]: pgmap v1545: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 95 op/s; 7458/187965 objects degraded (3.968%); 3843/187965 objects misplaced (2.045%); 325 KiB/s, 5 objects/s recovering 2023-10-01T09:44:31.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:30 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:31.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:30 smithi138 ceph-mon[185886]: failed to encode map e530 with expected crc 2023-10-01T09:44:31.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:30 smithi138 ceph-mon[185886]: failed to encode map e530 with expected crc 2023-10-01T09:44:31.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:30 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:44:31.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:30 smithi138 ceph-mon[185886]: osdmap e530: 8 total, 8 up, 7 in 2023-10-01T09:44:31.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:30 smithi138 ceph-mon[185886]: failed to encode map e530 with expected crc 2023-10-01T09:44:31.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:30 smithi138 ceph-mon[185886]: pgmap v1545: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 95 op/s; 7458/187965 objects degraded (3.968%); 3843/187965 objects misplaced (2.045%); 325 KiB/s, 5 objects/s recovering 2023-10-01T09:44:31.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:30 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:31.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:30 smithi138 ceph-mon[192513]: failed to encode map e530 with expected crc 2023-10-01T09:44:31.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:30 smithi138 ceph-mon[192513]: failed to encode map e530 with expected crc 2023-10-01T09:44:31.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:30 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:44:31.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:30 smithi138 ceph-mon[192513]: osdmap e530: 8 total, 8 up, 7 in 2023-10-01T09:44:31.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:30 smithi138 ceph-mon[192513]: failed to encode map e530 with expected crc 2023-10-01T09:44:31.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:30 smithi138 ceph-mon[192513]: pgmap v1545: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 95 op/s; 7458/187965 objects degraded (3.968%); 3843/187965 objects misplaced (2.045%); 325 KiB/s, 5 objects/s recovering 2023-10-01T09:44:31.161 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:31.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:31.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:31.265 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:31.295 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:31.329 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:44:31.346 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:31.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:31.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:31.350 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:44:31.381 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:44:31.404 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:44:31.425 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:44:31.446 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:44:31.450 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:31.452 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:44:31.453 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 7458/187965 objects degraded (3.968%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2092,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":530,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":62655,"data_bytes":4109047755,"bytes_used":17942982656,"bytes_avail":654022266880,"bytes_total":671965249536,"degraded_objects":7458,"degraded_total":187965,"degraded_ratio":0.039677599553108289,"misplaced_objects":3843,"misplaced_total":187965,"misplaced_ratio":0.020445295666746469,"recovering_objects_per_sec":5,"recovering_bytes_per_sec":333074,"recovering_keys_per_sec":0,"num_objects_recovered":43,"num_bytes_recovered":2818048,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":95},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":34,"modified":"2023-10-01T09:43:40.311181+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:44:31.466 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:44:31.485 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:44:31.486 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:44:31.486 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:44:31.486 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:44:31.487 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:44:31.487 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:44:31.487 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:44:31.487 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:44:31.487 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:44:31.487 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:31.530 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:31.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:31.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:31.633 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:31.662 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:31.775 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1668449101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:31.775 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1534761236' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:44:31.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1668449101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:31.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1534761236' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:44:31.776 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1668449101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:31.776 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1534761236' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:44:32.392 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:44:32.393 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:44:32.619 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:44:32.619 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:44:32.663 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:32.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:32.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:32.770 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:32.798 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:32.846 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:44:32.849 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:32.853 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:32 smithi138 ceph-mon[192513]: pgmap v1546: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7412/186075 objects degraded (3.983%); 3803/186075 objects misplaced (2.044%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:44:32.853 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:32 smithi138 ceph-mon[185886]: pgmap v1546: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7412/186075 objects degraded (3.983%); 3803/186075 objects misplaced (2.044%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:44:32.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:32.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:32.955 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:32.983 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:33.032 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:33.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:33.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:33.048 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:33.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:32 smithi190 ceph-mon[160435]: pgmap v1546: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7412/186075 objects degraded (3.983%); 3803/186075 objects misplaced (2.044%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:44:33.137 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:33.145 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:44:33.172 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:33.231 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:33.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:33.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:33.340 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:33.369 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:33.418 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:33.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:33.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:33.522 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:33.551 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:33.604 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:33.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:33.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:33.709 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:33.737 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:33.780 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:33.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:33.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:33.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:33 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7412/186075 objects degraded (3.983%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:33.885 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:33.913 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:34.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:33 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7412/186075 objects degraded (3.983%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:34.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:33 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7412/186075 objects degraded (3.983%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:34.120 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:34.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:34.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:34.141 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:34.166 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:34.225 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:34.253 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:34.297 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:34.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:34.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:34.405 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:34.431 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:34.476 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:34.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:34.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:34.581 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:34.615 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:34.660 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:44:34.677 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:34.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:34.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:34.783 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:34.817 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:44:34.819 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:34.819 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:44:34.881 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:34.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:34.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:34.910 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:34 smithi138 ceph-mon[185886]: pgmap v1547: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7412/185973 objects degraded (3.986%); 3803/185973 objects misplaced (2.045%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:44:34.910 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:34 smithi138 ceph-mon[192513]: pgmap v1547: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7412/185973 objects degraded (3.986%); 3803/185973 objects misplaced (2.045%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:44:34.988 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:35.021 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:35.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:34 smithi190 ceph-mon[160435]: pgmap v1547: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7412/185973 objects degraded (3.986%); 3803/185973 objects misplaced (2.045%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:44:35.293 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:44:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:44:34] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.33.4" 2023-10-01T09:44:36.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:35 smithi190 ceph-mon[160435]: from='client.37008 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:36.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:35 smithi138 ceph-mon[192513]: from='client.37008 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:36.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:35 smithi138 ceph-mon[185886]: from='client.37008 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:36.580 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:36.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:36.685 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:36.715 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:36.767 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:36.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:36.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:36.783 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1547,"stamp":"2023-10-01T09:44:34.329266+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4065531851,"num_objects":61991,"num_object_clones":0,"num_object_copies":185973,"num_objects_missing_on_primary":2327,"num_objects_missing":2327,"num_objects_degraded":7412,"num_objects_misplaced":3803,"num_objects_unfound":0,"num_objects_dirty":61991,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":105583,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8782,"num_bytes_recovered":562398223,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105647,"ondisk_log_size":105647,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17383808,"kb_used_data":12324744,"kb_used_omap":39,"kb_used_meta":5058968,"kb_avail":638832256,"statfs":{"total":671965249536,"available":654164230144,"internally_reserved":0,"allocated":12620537856,"data_stored":12610660905,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5180383855},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,1,3,0,0,0,6,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-94306304,"num_objects":-1439,"num_object_clones":0,"num_object_copies":-4317,"num_objects_missing_on_primary":-85,"num_objects_missing":-85,"num_objects_degraded":-95,"num_objects_misplaced":-78,"num_objects_unfound":0,"num_objects_dirty":-1439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1439,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":88,"num_bytes_recovered":5767168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002754"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":306,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124136+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:44:22.124136+0000","last_peered":"2023-10-01T09:44:22.124136+0000","last_clean":"2023-10-01T09:44:22.124136+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:44:22.124136+0000","last_undegraded":"2023-10-01T09:44:22.124136+0000","last_fullsized":"2023-10-01T09:44:22.124136+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":149,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124499+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:44:22.124499+0000","last_peered":"2023-10-01T09:44:22.124499+0000","last_clean":"2023-10-01T09:44:22.124499+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:44:22.124499+0000","last_undegraded":"2023-10-01T09:44:22.124499+0000","last_fullsized":"2023-10-01T09:44:22.124499+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":530,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124216+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:44:22.124216+0000","last_peered":"2023-10-01T09:44:22.124216+0000","last_clean":"2023-10-01T09:44:22.124216+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:44:22.124216+0000","last_undegraded":"2023-10-01T09:44:22.124216+0000","last_fullsized":"2023-10-01T09:44:22.124216+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":588,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.873031+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:44:21.873031+0000","last_peered":"2023-10-01T09:44:21.873031+0000","last_clean":"2023-10-01T09:44:21.873031+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:44:21.873031+0000","last_undegraded":"2023-10-01T09:44:21.873031+0000","last_fullsized":"2023-10-01T09:44:21.873031+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":469,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.872448+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:44:21.872448+0000","last_peered":"2023-10-01T09:44:21.872448+0000","last_clean":"2023-10-01T09:44:21.872448+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:44:21.872448+0000","last_undegraded":"2023-10-01T09:44:21.872448+0000","last_fullsized":"2023-10-01T09:44:21.872448+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":211,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.950605+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:44:21.950605+0000","last_peered":"2023-10-01T09:44:21.950605+0000","last_clean":"2023-10-01T09:44:21.950605+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:44:21.950605+0000","last_undegraded":"2023-10-01T09:44:21.950605+0000","last_fullsized":"2023-10-01T09:44:21.950605+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":587,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.950705+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:44:21.950705+0000","last_peered":"2023-10-01T09:44:21.950705+0000","last_clean":"2023-10-01T09:44:21.950705+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:44:21.950705+0000","last_undegraded":"2023-10-01T09:44:21.950705+0000","last_fullsized":"2023-10-01T09:44:21.950705+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":516,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.868516+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:44:21.868516+0000","last_peered":"2023-10-01T09:44:21.868516+0000","last_clean":"2023-10-01T09:44:21.868516+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:44:21.868516+0000","last_undegraded":"2023-10-01T09:44:21.868516+0000","last_fullsized":"2023-10-01T09:44:21.868516+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":242,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.941478+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:44:21.941478+0000","last_peered":"2023-10-01T09:44:21.941478+0000","last_clean":"2023-10-01T09:44:21.941478+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:44:21.941478+0000","last_undegraded":"2023-10-01T09:44:21.941478+0000","last_fullsized":"2023-10-01T09:44:21.941478+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":49,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.149016+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:44:22.149016+0000","last_peered":"2023-10-01T09:44:22.149016+0000","last_clean":"2023-10-01T09:44:22.149016+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:44:22.149016+0000","last_undegraded":"2023-10-01T09:44:22.149016+0000","last_fullsized":"2023-10-01T09:44:22.149016+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":55,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.149569+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:44:22.149569+0000","last_peered":"2023-10-01T09:44:22.149569+0000","last_clean":"2023-10-01T09:44:22.149569+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:44:22.149569+0000","last_undegraded":"2023-10-01T09:44:22.149569+0000","last_fullsized":"2023-10-01T09:44:22.149569+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":53,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124214+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:44:22.124214+0000","last_peered":"2023-10-01T09:44:22.124214+0000","last_clean":"2023-10-01T09:44:22.124214+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:44:22.124214+0000","last_undegraded":"2023-10-01T09:44:22.124214+0000","last_fullsized":"2023-10-01T09:44:22.124214+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":431,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124506+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:44:22.124506+0000","last_peered":"2023-10-01T09:44:22.124506+0000","last_clean":"2023-10-01T09:44:22.124506+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:44:22.124506+0000","last_undegraded":"2023-10-01T09:44:22.124506+0000","last_fullsized":"2023-10-01T09:44:22.124506+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":312,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124346+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:44:22.124346+0000","last_peered":"2023-10-01T09:44:22.124346+0000","last_clean":"2023-10-01T09:44:22.124346+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:44:22.124346+0000","last_undegraded":"2023-10-01T09:44:22.124346+0000","last_fullsized":"2023-10-01T09:44:22.124346+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":648,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.149152+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:44:22.149152+0000","last_peered":"2023-10-01T09:44:22.149152+0000","last_clean":"2023-10-01T09:44:22.149152+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:44:22.149152+0000","last_undegraded":"2023-10-01T09:44:22.149152+0000","last_fullsized":"2023-10-01T09:44:22.149152+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":385,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.149521+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:44:22.149521+0000","last_peered":"2023-10-01T09:44:22.149521+0000","last_clean":"2023-10-01T09:44:22.149521+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:44:22.149521+0000","last_undegraded":"2023-10-01T09:44:22.149521+0000","last_fullsized":"2023-10-01T09:44:22.149521+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":58,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.149064+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:44:22.149064+0000","last_peered":"2023-10-01T09:44:22.149064+0000","last_clean":"2023-10-01T09:44:22.149064+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:44:22.149064+0000","last_undegraded":"2023-10-01T09:44:22.149064+0000","last_fullsized":"2023-10-01T09:44:22.149064+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":44,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.941059+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:44:21.941059+0000","last_peered":"2023-10-01T09:44:21.941059+0000","last_clean":"2023-10-01T09:44:21.941059+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:44:21.941059+0000","last_undegraded":"2023-10-01T09:44:21.941059+0000","last_fullsized":"2023-10-01T09:44:21.941059+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":541,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.149192+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:44:22.149192+0000","last_peered":"2023-10-01T09:44:22.149192+0000","last_clean":"2023-10-01T09:44:22.149192+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:44:22.149192+0000","last_undegraded":"2023-10-01T09:44:22.149192+0000","last_fullsized":"2023-10-01T09:44:22.149192+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":552,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.873089+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:44:21.873089+0000","last_peered":"2023-10-01T09:44:21.873089+0000","last_clean":"2023-10-01T09:44:21.873089+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:44:21.873089+0000","last_undegraded":"2023-10-01T09:44:21.873089+0000","last_fullsized":"2023-10-01T09:44:21.873089+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":408,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.872401+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:44:21.872401+0000","last_peered":"2023-10-01T09:44:21.872401+0000","last_clean":"2023-10-01T09:44:21.872401+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:44:21.872401+0000","last_undegraded":"2023-10-01T09:44:21.872401+0000","last_fullsized":"2023-10-01T09:44:21.872401+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":58,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.150016+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:44:22.150016+0000","last_peered":"2023-10-01T09:44:22.150016+0000","last_clean":"2023-10-01T09:44:22.150016+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:44:22.150016+0000","last_undegraded":"2023-10-01T09:44:22.150016+0000","last_fullsized":"2023-10-01T09:44:22.150016+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"530'6565","reported_seq":6628,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:31.101930+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:44:31.101930+0000","last_peered":"2023-10-01T09:44:31.101930+0000","last_clean":"2023-10-01T09:44:31.101930+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:44:31.101930+0000","last_undegraded":"2023-10-01T09:44:31.101930+0000","last_fullsized":"2023-10-01T09:44:31.101930+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6565,"log_dups_size":0,"ondisk_log_size":6565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250806272,"num_objects":3827,"num_object_clones":0,"num_object_copies":11481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6565,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":53,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.149876+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:44:22.149876+0000","last_peered":"2023-10-01T09:44:22.149876+0000","last_clean":"2023-10-01T09:44:22.149876+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:44:22.149876+0000","last_undegraded":"2023-10-01T09:44:22.149876+0000","last_fullsized":"2023-10-01T09:44:22.149876+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":250,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.940706+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:44:21.940706+0000","last_peered":"2023-10-01T09:44:21.940706+0000","last_clean":"2023-10-01T09:44:21.940706+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:44:21.940706+0000","last_undegraded":"2023-10-01T09:44:21.940706+0000","last_fullsized":"2023-10-01T09:44:21.940706+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"530'6562","reported_seq":10731,"reported_epoch":530,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:44:31.701168+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:44:31.701168+0000","last_peered":"2023-10-01T09:44:31.701168+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:44:31.701168+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6562,"log_dups_size":0,"ondisk_log_size":6562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251920384,"num_objects":3844,"num_object_clones":0,"num_object_copies":11532,"num_objects_missing_on_primary":2327,"num_objects_missing":2327,"num_objects_degraded":6102,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6562,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1360,"num_bytes_recovered":89128960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1448},{"shards":"0,4,6","objects":2327},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":524,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.149878+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:44:22.149878+0000","last_peered":"2023-10-01T09:44:22.149878+0000","last_clean":"2023-10-01T09:44:22.149878+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:44:22.149878+0000","last_undegraded":"2023-10-01T09:44:22.149878+0000","last_fullsized":"2023-10-01T09:44:22.149878+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":585,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124215+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:44:22.124215+0000","last_peered":"2023-10-01T09:44:22.124215+0000","last_clean":"2023-10-01T09:44:22.124215+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:44:22.124215+0000","last_undegraded":"2023-10-01T09:44:22.124215+0000","last_fullsized":"2023-10-01T09:44:22.124215+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"530'6569","reported_seq":6635,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:31.582134+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:44:31.582134+0000","last_peered":"2023-10-01T09:44:31.582134+0000","last_clean":"2023-10-01T09:44:31.582134+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:44:31.582134+0000","last_undegraded":"2023-10-01T09:44:31.582134+0000","last_fullsized":"2023-10-01T09:44:31.582134+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6569,"log_dups_size":0,"ondisk_log_size":6569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253165568,"num_objects":3863,"num_object_clones":0,"num_object_copies":11589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6569,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":610,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.149558+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:44:22.149558+0000","last_peered":"2023-10-01T09:44:22.149558+0000","last_clean":"2023-10-01T09:44:22.149558+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:44:22.149558+0000","last_undegraded":"2023-10-01T09:44:22.149558+0000","last_fullsized":"2023-10-01T09:44:22.149558+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":406,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124363+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:44:22.124363+0000","last_peered":"2023-10-01T09:44:22.124363+0000","last_clean":"2023-10-01T09:44:22.124363+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:44:22.124363+0000","last_undegraded":"2023-10-01T09:44:22.124363+0000","last_fullsized":"2023-10-01T09:44:22.124363+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"530'6511","reported_seq":13182,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:31.783796+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:44:31.783796+0000","last_peered":"2023-10-01T09:44:31.783796+0000","last_clean":"2023-10-01T09:44:31.783796+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:44:31.783796+0000","last_undegraded":"2023-10-01T09:44:31.783796+0000","last_fullsized":"2023-10-01T09:44:31.783796+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6511,"log_dups_size":0,"ondisk_log_size":6511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255000576,"num_objects":3891,"num_object_clones":0,"num_object_copies":11673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6511,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":250,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.940970+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:44:21.940970+0000","last_peered":"2023-10-01T09:44:21.940970+0000","last_clean":"2023-10-01T09:44:21.940970+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:44:21.940970+0000","last_undegraded":"2023-10-01T09:44:21.940970+0000","last_fullsized":"2023-10-01T09:44:21.940970+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":508,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.872678+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:44:21.872678+0000","last_peered":"2023-10-01T09:44:21.872678+0000","last_clean":"2023-10-01T09:44:21.872678+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:44:21.872678+0000","last_undegraded":"2023-10-01T09:44:21.872678+0000","last_fullsized":"2023-10-01T09:44:21.872678+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"530'6540","reported_seq":6606,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:30.229186+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:44:30.229186+0000","last_peered":"2023-10-01T09:44:30.229186+0000","last_clean":"2023-10-01T09:44:30.229186+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:44:30.229186+0000","last_undegraded":"2023-10-01T09:44:30.229186+0000","last_fullsized":"2023-10-01T09:44:30.229186+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6540,"log_dups_size":0,"ondisk_log_size":6540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260046848,"num_objects":3968,"num_object_clones":0,"num_object_copies":11904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6540,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":110,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124508+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:44:22.124508+0000","last_peered":"2023-10-01T09:44:22.124508+0000","last_clean":"2023-10-01T09:44:22.124508+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:44:22.124508+0000","last_undegraded":"2023-10-01T09:44:22.124508+0000","last_fullsized":"2023-10-01T09:44:22.124508+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":184,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124639+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:44:22.124639+0000","last_peered":"2023-10-01T09:44:22.124639+0000","last_clean":"2023-10-01T09:44:22.124639+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:44:22.124639+0000","last_undegraded":"2023-10-01T09:44:22.124639+0000","last_fullsized":"2023-10-01T09:44:22.124639+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"530'6501","reported_seq":6608,"reported_epoch":530,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:44:31.452699+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:44:31.452699+0000","last_peered":"2023-10-01T09:44:31.452699+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:44:31.452699+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6501,"log_dups_size":0,"ondisk_log_size":6501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253427712,"num_objects":3867,"num_object_clones":0,"num_object_copies":11601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1300,"num_objects_misplaced":3803,"num_objects_unfound":0,"num_objects_dirty":3867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6501,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1179648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":3803},{"shards":"4,5","objects":-1236}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":428,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124231+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:44:22.124231+0000","last_peered":"2023-10-01T09:44:22.124231+0000","last_clean":"2023-10-01T09:44:22.124231+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:44:22.124231+0000","last_undegraded":"2023-10-01T09:44:22.124231+0000","last_fullsized":"2023-10-01T09:44:22.124231+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":589,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.881404+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:44:29.881404+0000","last_peered":"2023-10-01T09:44:29.881404+0000","last_clean":"2023-10-01T09:44:29.881404+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:44:29.881404+0000","last_undegraded":"2023-10-01T09:44:29.881404+0000","last_fullsized":"2023-10-01T09:44:29.881404+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"530'6504","reported_seq":6576,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:32.251916+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:44:32.251916+0000","last_peered":"2023-10-01T09:44:32.251916+0000","last_clean":"2023-10-01T09:44:32.251916+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:44:32.251916+0000","last_undegraded":"2023-10-01T09:44:32.251916+0000","last_fullsized":"2023-10-01T09:44:32.251916+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6504,"log_dups_size":0,"ondisk_log_size":6504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253886464,"num_objects":3874,"num_object_clones":0,"num_object_copies":11622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6504,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":424,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124097+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:44:22.124097+0000","last_peered":"2023-10-01T09:44:22.124097+0000","last_clean":"2023-10-01T09:44:22.124097+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:44:22.124097+0000","last_undegraded":"2023-10-01T09:44:22.124097+0000","last_fullsized":"2023-10-01T09:44:22.124097+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":265,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124964+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:44:22.124964+0000","last_peered":"2023-10-01T09:44:22.124964+0000","last_clean":"2023-10-01T09:44:22.124964+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:44:22.124964+0000","last_undegraded":"2023-10-01T09:44:22.124964+0000","last_fullsized":"2023-10-01T09:44:22.124964+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"530'6608","reported_seq":6671,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:30.961920+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:44:30.961920+0000","last_peered":"2023-10-01T09:44:30.961920+0000","last_clean":"2023-10-01T09:44:30.961920+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:44:30.961920+0000","last_undegraded":"2023-10-01T09:44:30.961920+0000","last_fullsized":"2023-10-01T09:44:30.961920+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6608,"log_dups_size":0,"ondisk_log_size":6608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254935040,"num_objects":3890,"num_object_clones":0,"num_object_copies":11670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6608,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":393,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.126089+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:44:22.126089+0000","last_peered":"2023-10-01T09:44:22.126089+0000","last_clean":"2023-10-01T09:44:22.126089+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:44:22.126089+0000","last_undegraded":"2023-10-01T09:44:22.126089+0000","last_fullsized":"2023-10-01T09:44:22.126089+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":440,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124665+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:44:22.124665+0000","last_peered":"2023-10-01T09:44:22.124665+0000","last_clean":"2023-10-01T09:44:22.124665+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:44:22.124665+0000","last_undegraded":"2023-10-01T09:44:22.124665+0000","last_fullsized":"2023-10-01T09:44:22.124665+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"530'6431","reported_seq":6494,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:30.439283+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:44:30.439283+0000","last_peered":"2023-10-01T09:44:30.439283+0000","last_clean":"2023-10-01T09:44:30.439283+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:44:30.439283+0000","last_undegraded":"2023-10-01T09:44:30.439283+0000","last_fullsized":"2023-10-01T09:44:30.439283+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6431,"log_dups_size":0,"ondisk_log_size":6431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245563392,"num_objects":3747,"num_object_clones":0,"num_object_copies":11241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6431,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":923,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.149436+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:44:22.149436+0000","last_peered":"2023-10-01T09:44:22.149436+0000","last_clean":"2023-10-01T09:44:22.149436+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:44:22.149436+0000","last_undegraded":"2023-10-01T09:44:22.149436+0000","last_fullsized":"2023-10-01T09:44:22.149436+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":260,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.941180+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:44:21.941180+0000","last_peered":"2023-10-01T09:44:21.941180+0000","last_clean":"2023-10-01T09:44:21.941180+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:44:21.941180+0000","last_undegraded":"2023-10-01T09:44:21.941180+0000","last_fullsized":"2023-10-01T09:44:21.941180+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":53,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124274+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:44:22.124274+0000","last_peered":"2023-10-01T09:44:22.124274+0000","last_clean":"2023-10-01T09:44:22.124274+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:44:22.124274+0000","last_undegraded":"2023-10-01T09:44:22.124274+0000","last_fullsized":"2023-10-01T09:44:22.124274+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"530'6475","reported_seq":6541,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:31.831171+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:44:31.831171+0000","last_peered":"2023-10-01T09:44:31.831171+0000","last_clean":"2023-10-01T09:44:31.831171+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:44:31.831171+0000","last_undegraded":"2023-10-01T09:44:31.831171+0000","last_fullsized":"2023-10-01T09:44:31.831171+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6475,"log_dups_size":0,"ondisk_log_size":6475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255262720,"num_objects":3895,"num_object_clones":0,"num_object_copies":11685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6475,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1195,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.940820+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:44:21.940820+0000","last_peered":"2023-10-01T09:44:21.940820+0000","last_clean":"2023-10-01T09:44:21.940820+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:44:21.940820+0000","last_undegraded":"2023-10-01T09:44:21.940820+0000","last_fullsized":"2023-10-01T09:44:21.940820+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":405,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124428+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:44:22.124428+0000","last_peered":"2023-10-01T09:44:22.124428+0000","last_clean":"2023-10-01T09:44:22.124428+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:44:22.124428+0000","last_undegraded":"2023-10-01T09:44:22.124428+0000","last_fullsized":"2023-10-01T09:44:22.124428+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":233,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.941058+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:44:21.941058+0000","last_peered":"2023-10-01T09:44:21.941058+0000","last_clean":"2023-10-01T09:44:21.941058+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:44:21.941058+0000","last_undegraded":"2023-10-01T09:44:21.941058+0000","last_fullsized":"2023-10-01T09:44:21.941058+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"530'6548","reported_seq":13832,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:31.479149+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:44:31.479149+0000","last_peered":"2023-10-01T09:44:31.479149+0000","last_clean":"2023-10-01T09:44:31.479149+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:44:31.479149+0000","last_undegraded":"2023-10-01T09:44:31.479149+0000","last_fullsized":"2023-10-01T09:44:31.479149+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6548,"log_dups_size":0,"ondisk_log_size":6548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250871808,"num_objects":3828,"num_object_clones":0,"num_object_copies":11484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6548,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":875,"reported_epoch":529,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:44:21.872242+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:44:21.872242+0000","last_peered":"2023-10-01T09:44:21.872242+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:44:21.872242+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:44:21.872242+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":593,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.126088+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:44:22.126088+0000","last_peered":"2023-10-01T09:44:22.126088+0000","last_clean":"2023-10-01T09:44:22.126088+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:44:22.126088+0000","last_undegraded":"2023-10-01T09:44:22.126088+0000","last_fullsized":"2023-10-01T09:44:22.126088+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":53,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124585+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:44:22.124585+0000","last_peered":"2023-10-01T09:44:22.124585+0000","last_clean":"2023-10-01T09:44:22.124585+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:44:22.124585+0000","last_undegraded":"2023-10-01T09:44:22.124585+0000","last_fullsized":"2023-10-01T09:44:22.124585+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"530'6434","reported_seq":6499,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:30.183863+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:44:30.183863+0000","last_peered":"2023-10-01T09:44:30.183863+0000","last_clean":"2023-10-01T09:44:30.183863+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:44:30.183863+0000","last_undegraded":"2023-10-01T09:44:30.183863+0000","last_fullsized":"2023-10-01T09:44:30.183863+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6434,"log_dups_size":0,"ondisk_log_size":6434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251527168,"num_objects":3838,"num_object_clones":0,"num_object_copies":11514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6434,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1646,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.940915+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:44:21.940915+0000","last_peered":"2023-10-01T09:44:21.940915+0000","last_clean":"2023-10-01T09:44:21.940915+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:44:21.940915+0000","last_undegraded":"2023-10-01T09:44:21.940915+0000","last_fullsized":"2023-10-01T09:44:21.940915+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":560,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.148988+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:44:22.148988+0000","last_peered":"2023-10-01T09:44:22.148988+0000","last_clean":"2023-10-01T09:44:22.148988+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:44:22.148988+0000","last_undegraded":"2023-10-01T09:44:22.148988+0000","last_fullsized":"2023-10-01T09:44:22.148988+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":338,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.940851+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:44:21.940851+0000","last_peered":"2023-10-01T09:44:21.940851+0000","last_clean":"2023-10-01T09:44:21.940851+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:44:21.940851+0000","last_undegraded":"2023-10-01T09:44:21.940851+0000","last_fullsized":"2023-10-01T09:44:21.940851+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":328,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124640+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:44:22.124640+0000","last_peered":"2023-10-01T09:44:22.124640+0000","last_clean":"2023-10-01T09:44:22.124640+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:44:22.124640+0000","last_undegraded":"2023-10-01T09:44:22.124640+0000","last_fullsized":"2023-10-01T09:44:22.124640+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"530'6593","reported_seq":6656,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:31.093754+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:44:31.093754+0000","last_peered":"2023-10-01T09:44:31.093754+0000","last_clean":"2023-10-01T09:44:31.093754+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:44:31.093754+0000","last_undegraded":"2023-10-01T09:44:31.093754+0000","last_fullsized":"2023-10-01T09:44:31.093754+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6593,"log_dups_size":0,"ondisk_log_size":6593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258146304,"num_objects":3939,"num_object_clones":0,"num_object_copies":11817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6593,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":737,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.149077+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:44:22.149077+0000","last_peered":"2023-10-01T09:44:22.149077+0000","last_clean":"2023-10-01T09:44:22.149077+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:44:22.149077+0000","last_undegraded":"2023-10-01T09:44:22.149077+0000","last_fullsized":"2023-10-01T09:44:22.149077+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":570,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.872442+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:44:21.872442+0000","last_peered":"2023-10-01T09:44:21.872442+0000","last_clean":"2023-10-01T09:44:21.872442+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:44:21.872442+0000","last_undegraded":"2023-10-01T09:44:21.872442+0000","last_fullsized":"2023-10-01T09:44:21.872442+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":569,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.872919+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:44:21.872919+0000","last_peered":"2023-10-01T09:44:21.872919+0000","last_clean":"2023-10-01T09:44:21.872919+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:44:21.872919+0000","last_undegraded":"2023-10-01T09:44:21.872919+0000","last_fullsized":"2023-10-01T09:44:21.872919+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"530'6544","reported_seq":6610,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:31.801797+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:44:31.801797+0000","last_peered":"2023-10-01T09:44:31.801797+0000","last_clean":"2023-10-01T09:44:31.801797+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:44:31.801797+0000","last_undegraded":"2023-10-01T09:44:31.801797+0000","last_fullsized":"2023-10-01T09:44:31.801797+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6544,"log_dups_size":0,"ondisk_log_size":6544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256573464,"num_objects":3916,"num_object_clones":0,"num_object_copies":11748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3916,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6544,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":968,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.149647+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:44:22.149647+0000","last_peered":"2023-10-01T09:44:22.149647+0000","last_clean":"2023-10-01T09:44:22.149647+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:44:22.149647+0000","last_undegraded":"2023-10-01T09:44:22.149647+0000","last_fullsized":"2023-10-01T09:44:22.149647+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":587,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.941181+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:44:21.941181+0000","last_peered":"2023-10-01T09:44:21.941181+0000","last_clean":"2023-10-01T09:44:21.941181+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:44:21.941181+0000","last_undegraded":"2023-10-01T09:44:21.941181+0000","last_fullsized":"2023-10-01T09:44:21.941181+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":440,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.941000+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:44:21.941000+0000","last_peered":"2023-10-01T09:44:21.941000+0000","last_clean":"2023-10-01T09:44:21.941000+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:44:21.941000+0000","last_undegraded":"2023-10-01T09:44:21.941000+0000","last_fullsized":"2023-10-01T09:44:21.941000+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"530'6519","reported_seq":6590,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:30.225632+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:44:30.225632+0000","last_peered":"2023-10-01T09:44:30.225632+0000","last_clean":"2023-10-01T09:44:30.225632+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:44:30.225632+0000","last_undegraded":"2023-10-01T09:44:30.225632+0000","last_fullsized":"2023-10-01T09:44:30.225632+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6519,"log_dups_size":0,"ondisk_log_size":6519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251461632,"num_objects":3837,"num_object_clones":0,"num_object_copies":11511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6519,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":469,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.150089+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:44:22.150089+0000","last_peered":"2023-10-01T09:44:22.150089+0000","last_clean":"2023-10-01T09:44:22.150089+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:44:22.150089+0000","last_undegraded":"2023-10-01T09:44:22.150089+0000","last_fullsized":"2023-10-01T09:44:22.150089+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1162,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124632+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:44:22.124632+0000","last_peered":"2023-10-01T09:44:22.124632+0000","last_clean":"2023-10-01T09:44:22.124632+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:44:22.124632+0000","last_undegraded":"2023-10-01T09:44:22.124632+0000","last_fullsized":"2023-10-01T09:44:22.124632+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":58,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.148976+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:44:22.148976+0000","last_peered":"2023-10-01T09:44:22.148976+0000","last_clean":"2023-10-01T09:44:22.148976+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:44:22.148976+0000","last_undegraded":"2023-10-01T09:44:22.148976+0000","last_fullsized":"2023-10-01T09:44:22.148976+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"530'6573","reported_seq":6644,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:30.935587+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:44:30.935587+0000","last_peered":"2023-10-01T09:44:30.935587+0000","last_clean":"2023-10-01T09:44:30.935587+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:44:30.935587+0000","last_undegraded":"2023-10-01T09:44:30.935587+0000","last_fullsized":"2023-10-01T09:44:30.935587+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258146304,"num_objects":3939,"num_object_clones":0,"num_object_copies":11817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6573,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":62,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.149950+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:44:22.149950+0000","last_peered":"2023-10-01T09:44:22.149950+0000","last_clean":"2023-10-01T09:44:22.149950+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:44:22.149950+0000","last_undegraded":"2023-10-01T09:44:22.149950+0000","last_fullsized":"2023-10-01T09:44:22.149950+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":547,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.872990+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:44:21.872990+0000","last_peered":"2023-10-01T09:44:21.872990+0000","last_clean":"2023-10-01T09:44:21.872990+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:44:21.872990+0000","last_undegraded":"2023-10-01T09:44:21.872990+0000","last_fullsized":"2023-10-01T09:44:21.872990+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":253,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.940743+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:44:21.940743+0000","last_peered":"2023-10-01T09:44:21.940743+0000","last_clean":"2023-10-01T09:44:21.940743+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:44:21.940743+0000","last_undegraded":"2023-10-01T09:44:21.940743+0000","last_fullsized":"2023-10-01T09:44:21.940743+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":496,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.872789+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:44:21.872789+0000","last_peered":"2023-10-01T09:44:21.872789+0000","last_clean":"2023-10-01T09:44:21.872789+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:44:21.872789+0000","last_undegraded":"2023-10-01T09:44:21.872789+0000","last_fullsized":"2023-10-01T09:44:21.872789+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":610,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124519+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:44:22.124519+0000","last_peered":"2023-10-01T09:44:22.124519+0000","last_clean":"2023-10-01T09:44:22.124519+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:44:22.124519+0000","last_undegraded":"2023-10-01T09:44:22.124519+0000","last_fullsized":"2023-10-01T09:44:22.124519+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":265,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.940975+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:44:21.940975+0000","last_peered":"2023-10-01T09:44:21.940975+0000","last_clean":"2023-10-01T09:44:21.940975+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:44:21.940975+0000","last_undegraded":"2023-10-01T09:44:21.940975+0000","last_fullsized":"2023-10-01T09:44:21.940975+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":53,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124273+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:44:22.124273+0000","last_peered":"2023-10-01T09:44:22.124273+0000","last_clean":"2023-10-01T09:44:22.124273+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:44:22.124273+0000","last_undegraded":"2023-10-01T09:44:22.124273+0000","last_fullsized":"2023-10-01T09:44:22.124273+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":265,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.941252+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:44:21.941252+0000","last_peered":"2023-10-01T09:44:21.941252+0000","last_clean":"2023-10-01T09:44:21.941252+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:44:21.941252+0000","last_undegraded":"2023-10-01T09:44:21.941252+0000","last_fullsized":"2023-10-01T09:44:21.941252+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":442,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.872579+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:44:21.872579+0000","last_peered":"2023-10-01T09:44:21.872579+0000","last_clean":"2023-10-01T09:44:21.872579+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:44:21.872579+0000","last_undegraded":"2023-10-01T09:44:21.872579+0000","last_fullsized":"2023-10-01T09:44:21.872579+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":71,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.124456+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:44:22.124456+0000","last_peered":"2023-10-01T09:44:22.124456+0000","last_clean":"2023-10-01T09:44:22.124456+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:44:22.124456+0000","last_undegraded":"2023-10-01T09:44:22.124456+0000","last_fullsized":"2023-10-01T09:44:22.124456+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":487,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.872432+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:44:21.872432+0000","last_peered":"2023-10-01T09:44:21.872432+0000","last_clean":"2023-10-01T09:44:21.872432+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:44:21.872432+0000","last_undegraded":"2023-10-01T09:44:21.872432+0000","last_fullsized":"2023-10-01T09:44:21.872432+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":687,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.873359+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:44:21.873359+0000","last_peered":"2023-10-01T09:44:21.873359+0000","last_clean":"2023-10-01T09:44:21.873359+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:44:21.873359+0000","last_undegraded":"2023-10-01T09:44:21.873359+0000","last_fullsized":"2023-10-01T09:44:21.873359+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":381,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:22.125880+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:44:22.125880+0000","last_peered":"2023-10-01T09:44:22.125880+0000","last_clean":"2023-10-01T09:44:22.125880+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:44:22.125880+0000","last_undegraded":"2023-10-01T09:44:22.125880+0000","last_fullsized":"2023-10-01T09:44:22.125880+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4060741656,"num_objects":61963,"num_object_clones":0,"num_object_copies":185889,"num_objects_missing_on_primary":2327,"num_objects_missing":2327,"num_objects_degraded":7402,"num_objects_misplaced":3803,"num_objects_unfound":0,"num_objects_dirty":61963,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":104477,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8271,"num_bytes_recovered":542048256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12562018304,"data_stored":12562006088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104477,"ondisk_log_size":104477,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430727,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2143840,"kb_used_data":1244824,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":91601312,"statfs":{"total":95995035648,"available":93799743488,"internally_reserved":0,"allocated":1274699776,"data_stored":1273760776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":1.1359999999999999,"15min":2.657},"min":{"1min":0.27300000000000002,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":1.484,"5min":10.391999999999999,"15min":108.90000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.39900000000000002,"5min":1.26,"15min":2.8090000000000002},"min":{"1min":0.219,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.67200000000000004,"5min":12.026,"15min":109.187},"last":0.53500000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":4.2910000000000004,"15min":5.2409999999999997},"min":{"1min":0.28999999999999998,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":1.4470000000000001,"5min":229.161,"15min":229.161},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.42999999999999999,"5min":4.2759999999999998,"15min":5.2510000000000003},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":0.96499999999999997,"5min":230.126,"15min":230.126},"last":0.55000000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:43:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.169,"15min":1.76},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.992,"5min":14.866,"15min":14.901999999999999},"last":0.371},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.4450000000000001,"15min":1.7170000000000001},"min":{"1min":0.32000000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4490000000000001,"5min":14.682,"15min":14.682},"last":0.60199999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.96899999999999997,"15min":2.569},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.018,"5min":6.0069999999999997,"15min":80.352999999999994},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.51300000000000001,"5min":1.1359999999999999,"15min":2.5510000000000002},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.026,"5min":14.528,"15min":80.364000000000004},"last":0.499}]},{"osd":4,"last update":"Sun Oct 1 09:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":1.046,"15min":1.0429999999999999},"min":{"1min":0.30199999999999999,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.508,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.0289999999999999,"15min":1.0189999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.28000000000000003},"max":{"1min":0.84399999999999997,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.45100000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":2.177,"15min":1.5800000000000001},"min":{"1min":0.34399999999999997,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":0.871,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.61299999999999999,"5min":2.1789999999999998,"15min":1.6619999999999999},"min":{"1min":0.307,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.3319999999999999,"5min":135.339,"15min":135.339},"last":0.40300000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":1.1639999999999999,"15min":1.1359999999999999},"min":{"1min":0.38,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":0.998,"5min":11.58,"15min":11.853999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.94199999999999995,"15min":1.097},"min":{"1min":0.38300000000000001,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":2.298,"5min":13.021000000000001,"15min":14.584},"last":0.46100000000000002}]}]},{"osd":6,"up_from":294,"seq":1262720385238,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2082716,"kb_used_data":1741012,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":91662436,"statfs":{"total":95995035648,"available":93862334464,"internally_reserved":0,"allocated":1782796288,"data_stored":1781861239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":2.0630000000000002,"15min":4.0090000000000003},"min":{"1min":0.247,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":3.2029999999999998,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":2.1739999999999999,"15min":4.0730000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.221},"max":{"1min":0.63900000000000001,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.35999999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":3.1379999999999999,"15min":4.8600000000000003},"min":{"1min":0.29199999999999998,"5min":0.254,"15min":0.24199999999999999},"max":{"1min":0.58899999999999997,"5min":168.947,"15min":168.947},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":3.141,"15min":4.8200000000000003},"min":{"1min":0.32500000000000001,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":2.8290000000000002,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.51900000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.78400000000000003,"15min":1.5620000000000001},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":0.65700000000000003,"5min":8.657,"15min":10.92},"last":0.309},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.88600000000000001,"15min":1.5629999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":2.863,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.53600000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.96499999999999997,"15min":2.101},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.3149999999999999,"5min":9.0180000000000007,"15min":89.753},"last":0.316},{"interface":"front","average":{"1min":0.439,"5min":1.097,"15min":2.0179999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.61199999999999999,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.38600000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.5049999999999999,"15min":1.0740000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.7749999999999999,"5min":63.134,"15min":63.134},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.478,"15min":1.075},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":3.2519999999999998,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.40600000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:44:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.246,"15min":1.073},"min":{"1min":0.28799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":3.2890000000000001,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.434},{"interface":"front","average":{"1min":0.624,"5min":1.3260000000000001,"15min":1.1859999999999999},"min":{"1min":0.29899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.887,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.55200000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.73999999999999999,"15min":1.2729999999999999},"min":{"1min":0.32700000000000001,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":3.339,"5min":3.8279999999999998,"15min":94.245999999999995},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.496,"5min":0.96999999999999997,"15min":1.3640000000000001},"min":{"1min":0.374,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":0.68600000000000005,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.42299999999999999}]}]},{"osd":1,"up_from":94,"seq":403726926214,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1564628,"kb_used_data":758732,"kb_used_omap":7,"kb_used_meta":805880,"kb_avail":92180524,"statfs":{"total":95995035648,"available":94392856576,"internally_reserved":0,"allocated":776941568,"data_stored":774900878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":825221543},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":1.7789999999999999,"15min":4.0039999999999996},"min":{"1min":0.186,"5min":0.186,"15min":0.182},"max":{"1min":1.2889999999999999,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":1.99,"15min":3.5329999999999999},"min":{"1min":0.223,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":1.702,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.41599999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:43:39 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":1.4279999999999999,"15min":2.589},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.61399999999999999,"5min":22.891999999999999,"15min":56.573},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.379,"15min":3.254},"min":{"1min":0.28999999999999998,"5min":0.23699999999999999,"15min":0.20499999999999999},"max":{"1min":1.752,"5min":15.189,"15min":154.197},"last":0.38100000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":1.2769999999999999,"15min":3.7909999999999999},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":1.7889999999999999,"5min":18.503,"15min":200.56800000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.2330000000000001,"15min":4.5039999999999996},"min":{"1min":0.253,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":1.9370000000000001,"5min":16.515000000000001,"15min":200.63800000000001},"last":0.42999999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.147,"15min":2.2530000000000001},"min":{"1min":0.35499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.0550000000000002,"5min":19.539000000000001,"15min":154.101},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.46899999999999997,"5min":1.0029999999999999,"15min":2.399},"min":{"1min":0.29499999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.59799999999999998,"5min":10.084,"15min":154.56999999999999},"last":0.55600000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:43:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.3200000000000001,"15min":2.097},"min":{"1min":0.372,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.1850000000000001,"5min":19.513000000000002,"15min":154.22999999999999},"last":59.561},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.1719999999999999,"15min":2.4630000000000001},"min":{"1min":0.31,"5min":0.307,"15min":0.307},"max":{"1min":0.82699999999999996,"5min":10.305,"15min":154.54400000000001},"last":59.536000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.236,"15min":2.5539999999999998},"min":{"1min":0.34300000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.4399999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.501},{"interface":"front","average":{"1min":0.81399999999999995,"5min":1.335,"15min":2.4119999999999999},"min":{"1min":0.34699999999999998,"5min":0.34599999999999997,"15min":0.32700000000000001},"max":{"1min":2.5630000000000002,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.57999999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.3109999999999999,"15min":2.5840000000000001},"min":{"1min":0.38400000000000001,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":2.8170000000000002,"5min":19.431999999999999,"15min":154.20699999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.77800000000000002,"5min":1.2330000000000001,"15min":2.6379999999999999},"min":{"1min":0.45200000000000001,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":2.8290000000000002,"5min":10.111000000000001,"15min":154.51599999999999},"last":0.60099999999999998}]}]},{"osd":0,"up_from":89,"seq":382252089749,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4059588,"kb_used_data":3146300,"kb_used_omap":8,"kb_used_meta":913271,"kb_avail":89685564,"statfs":{"total":95995035648,"available":91838017536,"internally_reserved":0,"allocated":3221811200,"data_stored":3219791418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":935190466},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:44:16 2023","interfaces":[{"interface":"back","average":{"1min":0.317,"5min":1.1839999999999999,"15min":4.1120000000000001},"min":{"1min":0.21199999999999999,"5min":0.17799999999999999,"15min":0.17399999999999999},"max":{"1min":0.46300000000000002,"5min":52.316000000000003,"15min":216.821},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.317,"5min":2.7919999999999998,"15min":4.9809999999999999},"min":{"1min":0.24399999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.435,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.36099999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:43:36 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":1.204,"15min":2.085},"min":{"1min":0.246,"5min":0.246,"15min":0.20100000000000001},"max":{"1min":1.1100000000000001,"5min":19.788,"15min":19.788},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.71599999999999997,"5min":1.4139999999999999,"15min":2.3380000000000001},"min":{"1min":0.27200000000000002,"5min":0.23000000000000001,"15min":0.189},"max":{"1min":3.5939999999999999,"5min":19.375,"15min":25.844999999999999},"last":0.42399999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:44:16 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.90200000000000002,"15min":3.46},"min":{"1min":0.26900000000000002,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.59199999999999997,"5min":8.2889999999999997,"15min":222.16499999999999},"last":0.378},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.0369999999999999,"15min":3.375},"min":{"1min":0.26900000000000002,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":3.4119999999999999,"5min":9.7539999999999996,"15min":222.227},"last":0.40000000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.5449999999999999,"15min":2.1339999999999999},"min":{"1min":0.28799999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":3.375,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.58399999999999996,"5min":1.4550000000000001,"15min":2.1309999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":3.5049999999999999,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.438}]},{"osd":5,"last update":"Sun Oct 1 09:43:34 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":1.2030000000000001,"15min":1.883},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.66500000000000004,"5min":14.515000000000001,"15min":16.808},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.093,"15min":1.7709999999999999},"min":{"1min":0.33500000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.73499999999999999,"5min":5.9580000000000002,"15min":17.507000000000001},"last":0.58999999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:44:27 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.95599999999999996,"15min":1.732},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.81399999999999995,"5min":6.5,"15min":17.535},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.504,"5min":1.0920000000000001,"15min":1.8300000000000001},"min":{"1min":0.38700000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":0.86199999999999999,"5min":10.446,"15min":15.651999999999999},"last":0.66800000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.95499999999999996,"15min":2.048},"min":{"1min":0.379,"5min":0.374,"15min":0.35699999999999998},"max":{"1min":0.78600000000000003,"5min":4.976,"15min":48.895000000000003},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.093,"15min":2.206},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.35099999999999998},"max":{"1min":3.4319999999999999,"5min":9.4969999999999999,"15min":48.274000000000001},"last":0.69899999999999995}]}]},{"osd":2,"up_from":0,"seq":425201762689,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599162,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1618760,"kb_used_data":786624,"kb_used_omap":3,"kb_used_meta":832124,"kb_avail":92126392,"statfs":{"total":95995035648,"available":94337425408,"internally_reserved":0,"allocated":805502976,"data_stored":803415630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":852095719},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":3.2000000000000002,"15min":4.8920000000000003},"min":{"1min":0.20100000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":1.3580000000000001,"5min":92.649000000000001,"15min":137.34399999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.36399999999999999,"5min":3.194,"15min":4.9740000000000002},"min":{"1min":0.21199999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":1.2549999999999999,"5min":92.284000000000006,"15min":137.73400000000001},"last":0.35499999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":1.8819999999999999,"15min":5.1500000000000004},"min":{"1min":0.25,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":1.1599999999999999,"5min":61.439999999999998,"15min":162.22999999999999},"last":0.443},{"interface":"front","average":{"1min":0.39600000000000002,"5min":1.788,"15min":5.181},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.4119999999999999,"5min":62.723999999999997,"15min":162.70699999999999},"last":0.33900000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.0780000000000001,"15min":1.748},"min":{"1min":0.27800000000000002,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":2.4710000000000001,"5min":14.254,"15min":14.254},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.66300000000000003,"15min":1.798},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":4.5049999999999999,"5min":4.5049999999999999,"15min":15.523},"last":0.43099999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.72,"15min":1.8080000000000001},"min":{"1min":0.35199999999999998,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":1.444,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.887,"15min":2.0630000000000002},"min":{"1min":0.29499999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.2250000000000001,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.42299999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:44:32 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":1.0780000000000001,"15min":2.3530000000000002},"min":{"1min":0.32000000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.196,"5min":9.4320000000000004,"15min":28.117999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.073,"15min":2.4609999999999999},"min":{"1min":0.33600000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.92300000000000004,"5min":10.798999999999999,"15min":28.34},"last":0.374}]},{"osd":6,"last update":"Sun Oct 1 09:44:28 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.7350000000000001,"15min":1.8999999999999999},"min":{"1min":0.37,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":1.1000000000000001,"5min":73.887,"15min":73.887},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.952,"15min":1.974},"min":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":1.294,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.45400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":1.268,"15min":1.845},"min":{"1min":0.42299999999999999,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":2.573,"5min":11.659000000000001,"15min":13.085000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.0740000000000001,"15min":1.726},"min":{"1min":0.39700000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":1.472,"5min":8.3460000000000001,"15min":13.191000000000001},"last":0.51500000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019412,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3954332,"kb_used_data":3060052,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":89790820,"statfs":{"total":95995035648,"available":91945799680,"internally_reserved":0,"allocated":3133493248,"data_stored":3132557754,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":2.3570000000000002,"15min":4.25},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.47299999999999998,"5min":79.837000000000003,"15min":155.315},"last":0.374},{"interface":"front","average":{"1min":0.44900000000000001,"5min":2.4300000000000002,"15min":4.7039999999999997},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":0.63700000000000001,"5min":78.983000000000004,"15min":155.334},"last":0.32600000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":4.5709999999999997,"15min":6.4699999999999998},"min":{"1min":0.28899999999999998,"5min":0.252,"15min":0.224},"max":{"1min":0.52900000000000003,"5min":126.742,"15min":190.61199999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.40200000000000002,"5min":4.5369999999999999,"15min":7.1600000000000001},"min":{"1min":0.27600000000000002,"5min":0.26700000000000002,"15min":0.25600000000000001},"max":{"1min":0.59999999999999998,"5min":126.791,"15min":253.273},"last":0.55200000000000005}]},{"osd":2,"last update":"Sun Oct 1 09:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.05,"15min":2.444},"min":{"1min":0.34200000000000003,"5min":0.31,"15min":0.28299999999999997},"max":{"1min":2.9510000000000001,"5min":10.161,"15min":170.45400000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.0029999999999999,"15min":2.1739999999999999},"min":{"1min":0.314,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":3.0219999999999998,"5min":5.9740000000000002,"15min":58.771000000000001},"last":0.48299999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.96499999999999997,"15min":2.3079999999999998},"min":{"1min":0.28100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.1299999999999999,"5min":7.5830000000000002,"15min":140.881},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.876,"15min":2.4830000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.311},"max":{"1min":0.51500000000000001,"5min":6.8719999999999999,"15min":140.911},"last":0.52500000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:43:33 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":2.0569999999999999,"15min":1.5580000000000001},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":3.052,"5min":110.003,"15min":110.003},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.64900000000000002,"5min":2.0110000000000001,"15min":1.399},"min":{"1min":0.33500000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.0049999999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.58099999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:44:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.78200000000000003,"15min":0.93000000000000005},"min":{"1min":0.33500000000000002,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":0.73599999999999999,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.85099999999999998,"15min":1.0589999999999999},"min":{"1min":0.40600000000000003,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":0.73599999999999999,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.70099999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.71599999999999997,"15min":1.161},"min":{"1min":0.313,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":0.69399999999999995,"5min":4.7030000000000003,"15min":38.034999999999997},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.026,"15min":1.4410000000000001},"min":{"1min":0.36699999999999999,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":2.9940000000000002,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.80000000000000004}]}]},{"osd":5,"up_from":379,"seq":1627792605319,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1959944,"kb_used_data":1587200,"kb_used_omap":13,"kb_used_meta":372722,"kb_avail":91785208,"statfs":{"total":95995035648,"available":93988052992,"internally_reserved":0,"allocated":1625292800,"data_stored":1624373210,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":381667826},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,1,3,0,0,0,6,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":1.244,"15min":2.2229999999999999},"min":{"1min":0.24299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":27.584,"5min":27.584,"15min":84.340999999999994},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.9430000000000001,"5min":1.1200000000000001,"15min":2.1280000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23300000000000001},"max":{"1min":27.669,"5min":27.669,"15min":84.356999999999999},"last":0.38100000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":3.4430000000000001,"15min":5.6900000000000004},"min":{"1min":0.28699999999999998,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":3.7799999999999998,"5min":149.94900000000001,"15min":295.71300000000002},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.71199999999999997,"5min":3.7149999999999999,"15min":5.6859999999999999},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":3.9580000000000002,"5min":149.41999999999999,"15min":299.59399999999999},"last":0.42199999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.83799999999999997,"15min":1.552},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.7029999999999998,"5min":5.3470000000000004,"15min":14.199999999999999},"last":0.439},{"interface":"front","average":{"1min":0.84599999999999997,"5min":1.056,"15min":1.5309999999999999},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":3.944,"5min":6.984,"15min":14.430999999999999},"last":0.45300000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.876,"15min":1.46},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.0209999999999999,"5min":10.022,"15min":14.004},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.91300000000000003,"15min":1.5940000000000001},"min":{"1min":0.34300000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.8610000000000002,"5min":5.4400000000000004,"15min":13.991},"last":0.47299999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":2.085,"15min":1.417},"min":{"1min":0.28599999999999998,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":4.0839999999999996,"5min":122.971,"15min":122.971},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.86299999999999999,"5min":2.1949999999999998,"15min":1.6699999999999999},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.0810000000000004,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.41199999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":1.0169999999999999,"15min":1.0269999999999999},"min":{"1min":0.32500000000000001,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":4.3300000000000001,"5min":10.500999999999999,"15min":12.545},"last":0.37},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.97199999999999998,"15min":1.264},"min":{"1min":0.314,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":4.0599999999999996,"5min":6.4729999999999999,"15min":14.193},"last":0.498}]},{"osd":7,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.0529999999999999,"15min":1.232},"min":{"1min":0.378,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.0999999999999996,"5min":5.758,"15min":10.813000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.996,"5min":1.1339999999999999,"15min":1.0840000000000001},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":4.2000000000000002,"5min":10.039,"15min":12.500999999999999},"last":0.46600000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3210219520,"data_stored":3210215448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":766443520,"data_stored":766443520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":799277056,"data_stored":799277056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3127119872,"data_stored":3127115800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1618673664,"data_stored":1618673664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1775308800,"data_stored":1775304728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1264975872,"data_stored":1264975872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:44:36.870 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:36.891 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:36.898 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:36.942 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:36.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:36.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:37.051 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:37.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:36 smithi190 ceph-mon[160435]: pgmap v1548: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 149 op/s; 7412/184932 objects degraded (4.008%); 3803/184932 objects misplaced (2.056%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:44:37.060 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:36 smithi138 ceph-mon[185886]: pgmap v1548: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 149 op/s; 7412/184932 objects degraded (4.008%); 3803/184932 objects misplaced (2.056%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:44:37.061 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:36 smithi138 ceph-mon[192513]: pgmap v1548: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 149 op/s; 7412/184932 objects degraded (4.008%); 3803/184932 objects misplaced (2.056%); 563 KiB/s, 8 objects/s recovering 2023-10-01T09:44:37.078 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:37.135 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:37.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:37.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:37.241 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:37.267 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:37.291 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:44:37.292 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:44:37.312 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:37.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:37.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:37.417 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:37.447 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:37.818 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:44:38.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:37 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2133707473' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:38.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:37 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:38.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:37 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7412/184932 objects degraded (4.008%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:38.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:37 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2133707473' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:38.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:37 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:38.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:37 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7412/184932 objects degraded (4.008%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:37 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2133707473' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:37 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:38.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:37 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7412/184932 objects degraded (4.008%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:38.576 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:44:38.621 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:38.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:38.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:38.725 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:38.752 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:38.990 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:38.993 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:38 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:38.993 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:38 smithi138 ceph-mon[192513]: failed to encode map e531 with expected crc 2023-10-01T09:44:38.993 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:38 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:44:38.993 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:38 smithi138 ceph-mon[192513]: osdmap e531: 8 total, 8 up, 7 in 2023-10-01T09:44:38.993 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:38 smithi138 ceph-mon[192513]: pgmap v1550: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 181 op/s; 7373/183549 objects degraded (4.017%); 3746/183549 objects misplaced (2.041%); 687 KiB/s, 10 objects/s recovering 2023-10-01T09:44:38.994 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:38 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:38.994 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:38 smithi138 ceph-mon[185886]: failed to encode map e531 with expected crc 2023-10-01T09:44:38.994 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:38 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:44:38.994 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:38 smithi138 ceph-mon[185886]: osdmap e531: 8 total, 8 up, 7 in 2023-10-01T09:44:38.994 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:38 smithi138 ceph-mon[185886]: pgmap v1550: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 181 op/s; 7373/183549 objects degraded (4.017%); 3746/183549 objects misplaced (2.041%); 687 KiB/s, 10 objects/s recovering 2023-10-01T09:44:38.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:38.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:39.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:38 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:39.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:38 smithi190 ceph-mon[160435]: failed to encode map e531 with expected crc 2023-10-01T09:44:39.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:38 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:44:39.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:38 smithi190 ceph-mon[160435]: osdmap e531: 8 total, 8 up, 7 in 2023-10-01T09:44:39.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:38 smithi190 ceph-mon[160435]: pgmap v1550: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 181 op/s; 7373/183549 objects degraded (4.017%); 3746/183549 objects misplaced (2.041%); 687 KiB/s, 10 objects/s recovering 2023-10-01T09:44:39.095 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:39.131 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:39.202 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:39.209 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:39.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:39.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:39.317 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:39.347 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:39.532 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:44:39.542 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:39.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:39.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:39.556 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:44:39.577 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:44:39.599 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:44:39.621 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:44:39.639 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:44:39.647 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:39.657 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:44:39.675 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:39.677 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:44:39.678 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:44:39.678 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:44:39.678 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:44:39.678 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:44:39.678 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:44:39.679 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:44:39.679 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:44:39.679 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:44:39.733 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:39.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:39.791 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:44:39.812 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:39 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2744093797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:39.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:39 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2744093797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:39.839 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:39.869 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:39.927 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:39.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:39.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:39.934 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:39 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2744093797' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:40.035 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:40.062 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:40.115 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:40.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:40.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:40.222 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:40.255 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:40.304 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:40.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:40.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:40.410 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:40.441 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:40.493 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:40.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:40.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:40.600 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:40.628 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:40.675 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:40.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:40.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:40.785 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:40.812 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:40.858 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:40.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:40.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:40.968 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:41.000 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:41.019 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:41.059 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:41.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:41.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:41.091 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:44:41.163 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:41.195 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:41.351 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:41.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:41.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:41.444 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:44:41.444 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:44:41.444 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:44:41.445 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:44:41.445 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:44:41.455 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:41.482 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:41.482 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:44:41.482 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:41 smithi138 ceph-mon[185886]: pgmap v1551: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 154 op/s; 7373/183315 objects degraded (4.022%); 3746/183315 objects misplaced (2.043%); 589 KiB/s, 9 objects/s recovering 2023-10-01T09:44:41.483 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:41.483 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:44:41.483 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:41 smithi138 ceph-mon[192513]: pgmap v1551: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 154 op/s; 7373/183315 objects degraded (4.022%); 3746/183315 objects misplaced (2.043%); 589 KiB/s, 9 objects/s recovering 2023-10-01T09:44:41.492 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:41.646 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:41.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:41.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:41.750 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:41.780 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:41.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:44:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:44:41.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:41 smithi190 ceph-mon[160435]: pgmap v1551: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 154 op/s; 7373/183315 objects degraded (4.022%); 3746/183315 objects misplaced (2.043%); 589 KiB/s, 9 objects/s recovering 2023-10-01T09:44:42.221 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:42.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:42.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:42.270 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:44:42.324 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:42.352 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:42.397 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:42.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:42.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:42.472 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:44:42.504 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:42.537 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:42.588 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:42.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:42.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:42.694 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:42.733 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:42.783 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:42.810 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:42.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:42.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:42.920 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:42.950 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:42.978 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:43.001 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:43.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:43.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:43.110 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:43.144 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:43.246 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:43.505 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:44:43.509 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:44:43.604 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:43.609 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:43 smithi138 ceph-mon[192513]: pgmap v1552: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 7313/181839 objects degraded (4.022%); 3703/181839 objects misplaced (2.036%); 633 KiB/s, 9 objects/s recovering 2023-10-01T09:44:43.609 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:43 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7313/181839 objects degraded (4.022%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:43.609 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:43 smithi138 ceph-mon[185886]: pgmap v1552: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 7313/181839 objects degraded (4.022%); 3703/181839 objects misplaced (2.036%); 633 KiB/s, 9 objects/s recovering 2023-10-01T09:44:43.609 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:43 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7313/181839 objects degraded (4.022%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:43.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:43.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:43.712 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:43.746 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:43 smithi190 ceph-mon[160435]: pgmap v1552: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 7313/181839 objects degraded (4.022%); 3703/181839 objects misplaced (2.036%); 633 KiB/s, 9 objects/s recovering 2023-10-01T09:44:43.746 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:43 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7313/181839 objects degraded (4.022%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:43.748 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:43.801 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:43.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:43.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:43.907 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:43.934 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:43.979 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:43.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:43.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:44.087 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:44.116 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:44.164 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:44.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:44.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:44.270 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:44.302 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:44.358 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:44.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:44.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:44.468 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:44.505 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:45.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:44:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:44:44] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.33.4" 2023-10-01T09:44:45.555 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:45.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:45.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:45.660 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:45.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:45 smithi138 ceph-mon[185886]: from='client.56932 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:45.688 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:45 smithi138 ceph-mon[185886]: pgmap v1553: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 7313/181704 objects degraded (4.025%); 3703/181704 objects misplaced (2.038%); 633 KiB/s, 9 objects/s recovering 2023-10-01T09:44:45.688 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:45 smithi138 ceph-mon[192513]: from='client.56932 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:45.689 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:45 smithi138 ceph-mon[192513]: pgmap v1553: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 7313/181704 objects degraded (4.025%); 3703/181704 objects misplaced (2.038%); 633 KiB/s, 9 objects/s recovering 2023-10-01T09:44:45.691 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:45.712 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1552,"stamp":"2023-10-01T09:44:42.331940+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3975223243,"num_objects":60613,"num_object_clones":0,"num_object_copies":181839,"num_objects_missing_on_primary":2227,"num_objects_missing":2227,"num_objects_degraded":7313,"num_objects_misplaced":3703,"num_objects_unfound":0,"num_objects_dirty":60613,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":106961,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8881,"num_bytes_recovered":568886287,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107025,"ondisk_log_size":107025,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17136552,"kb_used_data":12089776,"kb_used_omap":39,"kb_used_meta":5046680,"kb_avail":639079512,"statfs":{"total":671965249536,"available":654417420288,"internally_reserved":0,"allocated":12379930624,"data_stored":12370052053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5167800943},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-92536832,"num_objects":-1412,"num_object_clones":0,"num_object_copies":-4236,"num_objects_missing_on_primary":-100,"num_objects_missing":-100,"num_objects_degraded":-99,"num_objects_misplaced":-100,"num_objects_unfound":0,"num_objects_dirty":-1412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1412,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":6488064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003197"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":307,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.767226+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:44:29.767226+0000","last_peered":"2023-10-01T09:44:29.767226+0000","last_clean":"2023-10-01T09:44:29.767226+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:44:29.767226+0000","last_undegraded":"2023-10-01T09:44:29.767226+0000","last_fullsized":"2023-10-01T09:44:29.767226+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":150,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.881412+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:44:29.881412+0000","last_peered":"2023-10-01T09:44:29.881412+0000","last_clean":"2023-10-01T09:44:29.881412+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:44:29.881412+0000","last_undegraded":"2023-10-01T09:44:29.881412+0000","last_fullsized":"2023-10-01T09:44:29.881412+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":531,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.881052+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:44:29.881052+0000","last_peered":"2023-10-01T09:44:29.881052+0000","last_clean":"2023-10-01T09:44:29.881052+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:44:29.881052+0000","last_undegraded":"2023-10-01T09:44:29.881052+0000","last_fullsized":"2023-10-01T09:44:29.881052+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":589,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.765954+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:44:29.765954+0000","last_peered":"2023-10-01T09:44:29.765954+0000","last_clean":"2023-10-01T09:44:29.765954+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:44:29.765954+0000","last_undegraded":"2023-10-01T09:44:29.765954+0000","last_fullsized":"2023-10-01T09:44:29.765954+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":470,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.764755+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:44:29.764755+0000","last_peered":"2023-10-01T09:44:29.764755+0000","last_clean":"2023-10-01T09:44:29.764755+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:44:29.764755+0000","last_undegraded":"2023-10-01T09:44:29.764755+0000","last_fullsized":"2023-10-01T09:44:29.764755+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":212,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.836993+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:44:29.836993+0000","last_peered":"2023-10-01T09:44:29.836993+0000","last_clean":"2023-10-01T09:44:29.836993+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:44:29.836993+0000","last_undegraded":"2023-10-01T09:44:29.836993+0000","last_fullsized":"2023-10-01T09:44:29.836993+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":588,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.837088+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:44:29.837088+0000","last_peered":"2023-10-01T09:44:29.837088+0000","last_clean":"2023-10-01T09:44:29.837088+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:44:29.837088+0000","last_undegraded":"2023-10-01T09:44:29.837088+0000","last_fullsized":"2023-10-01T09:44:29.837088+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":516,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-10-01T09:44:21.868516+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:44:21.868516+0000","last_peered":"2023-10-01T09:44:21.868516+0000","last_clean":"2023-10-01T09:44:21.868516+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:44:21.868516+0000","last_undegraded":"2023-10-01T09:44:21.868516+0000","last_fullsized":"2023-10-01T09:44:21.868516+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":243,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.823925+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:44:29.823925+0000","last_peered":"2023-10-01T09:44:29.823925+0000","last_clean":"2023-10-01T09:44:29.823925+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:44:29.823925+0000","last_undegraded":"2023-10-01T09:44:29.823925+0000","last_fullsized":"2023-10-01T09:44:29.823925+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":50,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.879910+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:44:29.879910+0000","last_peered":"2023-10-01T09:44:29.879910+0000","last_clean":"2023-10-01T09:44:29.879910+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:44:29.879910+0000","last_undegraded":"2023-10-01T09:44:29.879910+0000","last_fullsized":"2023-10-01T09:44:29.879910+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":56,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.880356+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:44:29.880356+0000","last_peered":"2023-10-01T09:44:29.880356+0000","last_clean":"2023-10-01T09:44:29.880356+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:44:29.880356+0000","last_undegraded":"2023-10-01T09:44:29.880356+0000","last_fullsized":"2023-10-01T09:44:29.880356+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.881034+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:44:29.881034+0000","last_peered":"2023-10-01T09:44:29.881034+0000","last_clean":"2023-10-01T09:44:29.881034+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:44:29.881034+0000","last_undegraded":"2023-10-01T09:44:29.881034+0000","last_fullsized":"2023-10-01T09:44:29.881034+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":432,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.767859+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:44:29.767859+0000","last_peered":"2023-10-01T09:44:29.767859+0000","last_clean":"2023-10-01T09:44:29.767859+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:44:29.767859+0000","last_undegraded":"2023-10-01T09:44:29.767859+0000","last_fullsized":"2023-10-01T09:44:29.767859+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":313,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.767349+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:44:29.767349+0000","last_peered":"2023-10-01T09:44:29.767349+0000","last_clean":"2023-10-01T09:44:29.767349+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:44:29.767349+0000","last_undegraded":"2023-10-01T09:44:29.767349+0000","last_fullsized":"2023-10-01T09:44:29.767349+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":649,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.880302+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:44:29.880302+0000","last_peered":"2023-10-01T09:44:29.880302+0000","last_clean":"2023-10-01T09:44:29.880302+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:44:29.880302+0000","last_undegraded":"2023-10-01T09:44:29.880302+0000","last_fullsized":"2023-10-01T09:44:29.880302+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":386,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.880242+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:44:29.880242+0000","last_peered":"2023-10-01T09:44:29.880242+0000","last_clean":"2023-10-01T09:44:29.880242+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:44:29.880242+0000","last_undegraded":"2023-10-01T09:44:29.880242+0000","last_fullsized":"2023-10-01T09:44:29.880242+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":59,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.880060+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:44:29.880060+0000","last_peered":"2023-10-01T09:44:29.880060+0000","last_clean":"2023-10-01T09:44:29.880060+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:44:29.880060+0000","last_undegraded":"2023-10-01T09:44:29.880060+0000","last_fullsized":"2023-10-01T09:44:29.880060+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":45,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.823506+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:44:29.823506+0000","last_peered":"2023-10-01T09:44:29.823506+0000","last_clean":"2023-10-01T09:44:29.823506+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:44:29.823506+0000","last_undegraded":"2023-10-01T09:44:29.823506+0000","last_fullsized":"2023-10-01T09:44:29.823506+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":542,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.880045+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:44:29.880045+0000","last_peered":"2023-10-01T09:44:29.880045+0000","last_clean":"2023-10-01T09:44:29.880045+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:44:29.880045+0000","last_undegraded":"2023-10-01T09:44:29.880045+0000","last_fullsized":"2023-10-01T09:44:29.880045+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":553,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.766027+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:44:29.766027+0000","last_peered":"2023-10-01T09:44:29.766027+0000","last_clean":"2023-10-01T09:44:29.766027+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:44:29.766027+0000","last_undegraded":"2023-10-01T09:44:29.766027+0000","last_fullsized":"2023-10-01T09:44:29.766027+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":410,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.815479+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:44:37.815479+0000","last_peered":"2023-10-01T09:44:37.815479+0000","last_clean":"2023-10-01T09:44:37.815479+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:44:37.815479+0000","last_undegraded":"2023-10-01T09:44:37.815479+0000","last_fullsized":"2023-10-01T09:44:37.815479+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":59,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.880501+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:44:29.880501+0000","last_peered":"2023-10-01T09:44:29.880501+0000","last_clean":"2023-10-01T09:44:29.880501+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:44:29.880501+0000","last_undegraded":"2023-10-01T09:44:29.880501+0000","last_fullsized":"2023-10-01T09:44:29.880501+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"531'6676","reported_seq":6740,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:41.568328+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:44:41.568328+0000","last_peered":"2023-10-01T09:44:41.568328+0000","last_clean":"2023-10-01T09:44:41.568328+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:44:41.568328+0000","last_undegraded":"2023-10-01T09:44:41.568328+0000","last_fullsized":"2023-10-01T09:44:41.568328+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6676,"log_dups_size":0,"ondisk_log_size":6676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243531776,"num_objects":3716,"num_object_clones":0,"num_object_copies":11148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6676,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":54,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.880301+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:44:29.880301+0000","last_peered":"2023-10-01T09:44:29.880301+0000","last_clean":"2023-10-01T09:44:29.880301+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:44:29.880301+0000","last_undegraded":"2023-10-01T09:44:29.880301+0000","last_fullsized":"2023-10-01T09:44:29.880301+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":251,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.823205+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:44:29.823205+0000","last_peered":"2023-10-01T09:44:29.823205+0000","last_clean":"2023-10-01T09:44:29.823205+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:44:29.823205+0000","last_undegraded":"2023-10-01T09:44:29.823205+0000","last_fullsized":"2023-10-01T09:44:29.823205+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"531'6662","reported_seq":11128,"reported_epoch":531,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:44:41.806606+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:44:41.806606+0000","last_peered":"2023-10-01T09:44:41.806606+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:44:41.806606+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6662,"log_dups_size":0,"ondisk_log_size":6662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245366784,"num_objects":3744,"num_object_clones":0,"num_object_copies":11232,"num_objects_missing_on_primary":2227,"num_objects_missing":2227,"num_objects_degraded":5903,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6662,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1459,"num_bytes_recovered":95617024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1449},{"shards":"0,4,6","objects":2227},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":525,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.880371+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:44:29.880371+0000","last_peered":"2023-10-01T09:44:29.880371+0000","last_clean":"2023-10-01T09:44:29.880371+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:44:29.880371+0000","last_undegraded":"2023-10-01T09:44:29.880371+0000","last_fullsized":"2023-10-01T09:44:29.880371+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":586,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.881008+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:44:29.881008+0000","last_peered":"2023-10-01T09:44:29.881008+0000","last_clean":"2023-10-01T09:44:29.881008+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:44:29.881008+0000","last_undegraded":"2023-10-01T09:44:29.881008+0000","last_fullsized":"2023-10-01T09:44:29.881008+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"531'6650","reported_seq":6717,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:41.140584+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:44:41.140584+0000","last_peered":"2023-10-01T09:44:41.140584+0000","last_clean":"2023-10-01T09:44:41.140584+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:44:41.140584+0000","last_undegraded":"2023-10-01T09:44:41.140584+0000","last_fullsized":"2023-10-01T09:44:41.140584+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6650,"log_dups_size":0,"ondisk_log_size":6650,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247857152,"num_objects":3782,"num_object_clones":0,"num_object_copies":11346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6650,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":611,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.880262+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:44:29.880262+0000","last_peered":"2023-10-01T09:44:29.880262+0000","last_clean":"2023-10-01T09:44:29.880262+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:44:29.880262+0000","last_undegraded":"2023-10-01T09:44:29.880262+0000","last_fullsized":"2023-10-01T09:44:29.880262+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":407,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.881221+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:44:29.881221+0000","last_peered":"2023-10-01T09:44:29.881221+0000","last_clean":"2023-10-01T09:44:29.881221+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:44:29.881221+0000","last_undegraded":"2023-10-01T09:44:29.881221+0000","last_fullsized":"2023-10-01T09:44:29.881221+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"531'6609","reported_seq":13281,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:41.050230+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:44:41.050230+0000","last_peered":"2023-10-01T09:44:41.050230+0000","last_clean":"2023-10-01T09:44:41.050230+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:44:41.050230+0000","last_undegraded":"2023-10-01T09:44:41.050230+0000","last_fullsized":"2023-10-01T09:44:41.050230+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6609,"log_dups_size":0,"ondisk_log_size":6609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248578048,"num_objects":3793,"num_object_clones":0,"num_object_copies":11379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6609,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":251,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.823167+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:44:29.823167+0000","last_peered":"2023-10-01T09:44:29.823167+0000","last_clean":"2023-10-01T09:44:29.823167+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:44:29.823167+0000","last_undegraded":"2023-10-01T09:44:29.823167+0000","last_fullsized":"2023-10-01T09:44:29.823167+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":510,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.815715+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:44:37.815715+0000","last_peered":"2023-10-01T09:44:37.815715+0000","last_clean":"2023-10-01T09:44:37.815715+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:44:37.815715+0000","last_undegraded":"2023-10-01T09:44:37.815715+0000","last_fullsized":"2023-10-01T09:44:37.815715+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"531'6633","reported_seq":6700,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:40.060993+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:44:40.060993+0000","last_peered":"2023-10-01T09:44:40.060993+0000","last_clean":"2023-10-01T09:44:40.060993+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:44:40.060993+0000","last_undegraded":"2023-10-01T09:44:40.060993+0000","last_fullsized":"2023-10-01T09:44:40.060993+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6633,"log_dups_size":0,"ondisk_log_size":6633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253952000,"num_objects":3875,"num_object_clones":0,"num_object_copies":11625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6633,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":111,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.767601+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:44:29.767601+0000","last_peered":"2023-10-01T09:44:29.767601+0000","last_clean":"2023-10-01T09:44:29.767601+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:44:29.767601+0000","last_undegraded":"2023-10-01T09:44:29.767601+0000","last_fullsized":"2023-10-01T09:44:29.767601+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":185,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.767938+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:44:29.767938+0000","last_peered":"2023-10-01T09:44:29.767938+0000","last_clean":"2023-10-01T09:44:29.767938+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:44:29.767938+0000","last_undegraded":"2023-10-01T09:44:29.767938+0000","last_fullsized":"2023-10-01T09:44:29.767938+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"531'6601","reported_seq":6709,"reported_epoch":531,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:44:41.225904+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:44:41.225904+0000","last_peered":"2023-10-01T09:44:41.225904+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:44:41.225904+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6601,"log_dups_size":0,"ondisk_log_size":6601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246874112,"num_objects":3767,"num_object_clones":0,"num_object_copies":11301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1400,"num_objects_misplaced":3703,"num_objects_unfound":0,"num_objects_dirty":3767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6601,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":1179648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":3703},{"shards":"4,5","objects":-1336}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":429,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.767418+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:44:29.767418+0000","last_peered":"2023-10-01T09:44:29.767418+0000","last_clean":"2023-10-01T09:44:29.767418+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:44:29.767418+0000","last_undegraded":"2023-10-01T09:44:29.767418+0000","last_fullsized":"2023-10-01T09:44:29.767418+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":590,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:30.260529+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:44:30.260529+0000","last_peered":"2023-10-01T09:44:30.260529+0000","last_clean":"2023-10-01T09:44:30.260529+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:44:30.260529+0000","last_undegraded":"2023-10-01T09:44:30.260529+0000","last_fullsized":"2023-10-01T09:44:30.260529+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"530'6543","reported_seq":6615,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.348082+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:44:37.348082+0000","last_peered":"2023-10-01T09:44:37.348082+0000","last_clean":"2023-10-01T09:44:37.348082+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:44:37.348082+0000","last_undegraded":"2023-10-01T09:44:37.348082+0000","last_fullsized":"2023-10-01T09:44:37.348082+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6543,"log_dups_size":0,"ondisk_log_size":6543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251330560,"num_objects":3835,"num_object_clones":0,"num_object_copies":11505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6543,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":425,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.767167+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:44:29.767167+0000","last_peered":"2023-10-01T09:44:29.767167+0000","last_clean":"2023-10-01T09:44:29.767167+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:44:29.767167+0000","last_undegraded":"2023-10-01T09:44:29.767167+0000","last_fullsized":"2023-10-01T09:44:29.767167+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":266,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.767789+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:44:29.767789+0000","last_peered":"2023-10-01T09:44:29.767789+0000","last_clean":"2023-10-01T09:44:29.767789+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:44:29.767789+0000","last_undegraded":"2023-10-01T09:44:29.767789+0000","last_fullsized":"2023-10-01T09:44:29.767789+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"531'6698","reported_seq":6762,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:41.102301+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:44:41.102301+0000","last_peered":"2023-10-01T09:44:41.102301+0000","last_clean":"2023-10-01T09:44:41.102301+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:44:41.102301+0000","last_undegraded":"2023-10-01T09:44:41.102301+0000","last_fullsized":"2023-10-01T09:44:41.102301+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6698,"log_dups_size":0,"ondisk_log_size":6698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249036800,"num_objects":3800,"num_object_clones":0,"num_object_copies":11400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6698,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":394,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.881081+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:44:29.881081+0000","last_peered":"2023-10-01T09:44:29.881081+0000","last_clean":"2023-10-01T09:44:29.881081+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:44:29.881081+0000","last_undegraded":"2023-10-01T09:44:29.881081+0000","last_fullsized":"2023-10-01T09:44:29.881081+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":441,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.881667+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:44:29.881667+0000","last_peered":"2023-10-01T09:44:29.881667+0000","last_clean":"2023-10-01T09:44:29.881667+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:44:29.881667+0000","last_undegraded":"2023-10-01T09:44:29.881667+0000","last_fullsized":"2023-10-01T09:44:29.881667+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"531'6503","reported_seq":6567,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:40.641463+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:44:40.641463+0000","last_peered":"2023-10-01T09:44:40.641463+0000","last_clean":"2023-10-01T09:44:40.641463+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:44:40.641463+0000","last_undegraded":"2023-10-01T09:44:40.641463+0000","last_fullsized":"2023-10-01T09:44:40.641463+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6503,"log_dups_size":0,"ondisk_log_size":6503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240844800,"num_objects":3675,"num_object_clones":0,"num_object_copies":11025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6503,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":924,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.880164+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:44:29.880164+0000","last_peered":"2023-10-01T09:44:29.880164+0000","last_clean":"2023-10-01T09:44:29.880164+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:44:29.880164+0000","last_undegraded":"2023-10-01T09:44:29.880164+0000","last_fullsized":"2023-10-01T09:44:29.880164+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":261,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.823632+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:44:29.823632+0000","last_peered":"2023-10-01T09:44:29.823632+0000","last_clean":"2023-10-01T09:44:29.823632+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:44:29.823632+0000","last_undegraded":"2023-10-01T09:44:29.823632+0000","last_fullsized":"2023-10-01T09:44:29.823632+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":54,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.881084+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:44:29.881084+0000","last_peered":"2023-10-01T09:44:29.881084+0000","last_clean":"2023-10-01T09:44:29.881084+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:44:29.881084+0000","last_undegraded":"2023-10-01T09:44:29.881084+0000","last_fullsized":"2023-10-01T09:44:29.881084+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"531'6553","reported_seq":6620,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:41.081561+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:44:41.081561+0000","last_peered":"2023-10-01T09:44:41.081561+0000","last_clean":"2023-10-01T09:44:41.081561+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:44:41.081561+0000","last_undegraded":"2023-10-01T09:44:41.081561+0000","last_fullsized":"2023-10-01T09:44:41.081561+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6553,"log_dups_size":0,"ondisk_log_size":6553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250150912,"num_objects":3817,"num_object_clones":0,"num_object_copies":11451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6553,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1196,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.823169+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:44:29.823169+0000","last_peered":"2023-10-01T09:44:29.823169+0000","last_clean":"2023-10-01T09:44:29.823169+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:44:29.823169+0000","last_undegraded":"2023-10-01T09:44:29.823169+0000","last_fullsized":"2023-10-01T09:44:29.823169+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":406,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.767511+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:44:29.767511+0000","last_peered":"2023-10-01T09:44:29.767511+0000","last_clean":"2023-10-01T09:44:29.767511+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:44:29.767511+0000","last_undegraded":"2023-10-01T09:44:29.767511+0000","last_fullsized":"2023-10-01T09:44:29.767511+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":234,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.823328+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:44:29.823328+0000","last_peered":"2023-10-01T09:44:29.823328+0000","last_clean":"2023-10-01T09:44:29.823328+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:44:29.823328+0000","last_undegraded":"2023-10-01T09:44:29.823328+0000","last_fullsized":"2023-10-01T09:44:29.823328+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"531'6628","reported_seq":13913,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:41.247966+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:44:41.247966+0000","last_peered":"2023-10-01T09:44:41.247966+0000","last_clean":"2023-10-01T09:44:41.247966+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:44:41.247966+0000","last_undegraded":"2023-10-01T09:44:41.247966+0000","last_fullsized":"2023-10-01T09:44:41.247966+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6628,"log_dups_size":0,"ondisk_log_size":6628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245628928,"num_objects":3748,"num_object_clones":0,"num_object_copies":11244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6628,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":876,"reported_epoch":530,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:44:29.764737+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:44:29.764737+0000","last_peered":"2023-10-01T09:44:29.764737+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:44:29.764737+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:44:29.764737+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":594,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.881081+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:44:29.881081+0000","last_peered":"2023-10-01T09:44:29.881081+0000","last_clean":"2023-10-01T09:44:29.881081+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:44:29.881081+0000","last_undegraded":"2023-10-01T09:44:29.881081+0000","last_fullsized":"2023-10-01T09:44:29.881081+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":54,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.881539+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:44:29.881539+0000","last_peered":"2023-10-01T09:44:29.881539+0000","last_clean":"2023-10-01T09:44:29.881539+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:44:29.881539+0000","last_undegraded":"2023-10-01T09:44:29.881539+0000","last_fullsized":"2023-10-01T09:44:29.881539+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"531'6529","reported_seq":6596,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:39.903647+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:44:39.903647+0000","last_peered":"2023-10-01T09:44:39.903647+0000","last_clean":"2023-10-01T09:44:39.903647+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:44:39.903647+0000","last_undegraded":"2023-10-01T09:44:39.903647+0000","last_fullsized":"2023-10-01T09:44:39.903647+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6529,"log_dups_size":0,"ondisk_log_size":6529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245301248,"num_objects":3743,"num_object_clones":0,"num_object_copies":11229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6529,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1647,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.823970+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:44:29.823970+0000","last_peered":"2023-10-01T09:44:29.823970+0000","last_clean":"2023-10-01T09:44:29.823970+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:44:29.823970+0000","last_undegraded":"2023-10-01T09:44:29.823970+0000","last_fullsized":"2023-10-01T09:44:29.823970+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":561,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.879871+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:44:29.879871+0000","last_peered":"2023-10-01T09:44:29.879871+0000","last_clean":"2023-10-01T09:44:29.879871+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:44:29.879871+0000","last_undegraded":"2023-10-01T09:44:29.879871+0000","last_fullsized":"2023-10-01T09:44:29.879871+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":339,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.823251+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:44:29.823251+0000","last_peered":"2023-10-01T09:44:29.823251+0000","last_clean":"2023-10-01T09:44:29.823251+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:44:29.823251+0000","last_undegraded":"2023-10-01T09:44:29.823251+0000","last_fullsized":"2023-10-01T09:44:29.823251+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":329,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.767472+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:44:29.767472+0000","last_peered":"2023-10-01T09:44:29.767472+0000","last_clean":"2023-10-01T09:44:29.767472+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:44:29.767472+0000","last_undegraded":"2023-10-01T09:44:29.767472+0000","last_fullsized":"2023-10-01T09:44:29.767472+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"531'6686","reported_seq":6750,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:41.239093+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:44:41.239093+0000","last_peered":"2023-10-01T09:44:41.239093+0000","last_clean":"2023-10-01T09:44:41.239093+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:44:41.239093+0000","last_undegraded":"2023-10-01T09:44:41.239093+0000","last_fullsized":"2023-10-01T09:44:41.239093+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6686,"log_dups_size":0,"ondisk_log_size":6686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252051456,"num_objects":3846,"num_object_clones":0,"num_object_copies":11538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6686,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":738,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.879961+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:44:29.879961+0000","last_peered":"2023-10-01T09:44:29.879961+0000","last_clean":"2023-10-01T09:44:29.879961+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:44:29.879961+0000","last_undegraded":"2023-10-01T09:44:29.879961+0000","last_fullsized":"2023-10-01T09:44:29.879961+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":571,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.764955+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:44:29.764955+0000","last_peered":"2023-10-01T09:44:29.764955+0000","last_clean":"2023-10-01T09:44:29.764955+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:44:29.764955+0000","last_undegraded":"2023-10-01T09:44:29.764955+0000","last_fullsized":"2023-10-01T09:44:29.764955+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":570,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.765795+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:44:29.765795+0000","last_peered":"2023-10-01T09:44:29.765795+0000","last_clean":"2023-10-01T09:44:29.765795+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:44:29.765795+0000","last_undegraded":"2023-10-01T09:44:29.765795+0000","last_fullsized":"2023-10-01T09:44:29.765795+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"531'6616","reported_seq":6684,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:41.588870+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:44:41.588870+0000","last_peered":"2023-10-01T09:44:41.588870+0000","last_clean":"2023-10-01T09:44:41.588870+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:44:41.588870+0000","last_undegraded":"2023-10-01T09:44:41.588870+0000","last_fullsized":"2023-10-01T09:44:41.588870+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6616,"log_dups_size":0,"ondisk_log_size":6616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251854872,"num_objects":3844,"num_object_clones":0,"num_object_copies":11532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3844,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6616,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":969,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.880369+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:44:29.880369+0000","last_peered":"2023-10-01T09:44:29.880369+0000","last_clean":"2023-10-01T09:44:29.880369+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:44:29.880369+0000","last_undegraded":"2023-10-01T09:44:29.880369+0000","last_fullsized":"2023-10-01T09:44:29.880369+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":588,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.823250+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:44:29.823250+0000","last_peered":"2023-10-01T09:44:29.823250+0000","last_clean":"2023-10-01T09:44:29.823250+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:44:29.823250+0000","last_undegraded":"2023-10-01T09:44:29.823250+0000","last_fullsized":"2023-10-01T09:44:29.823250+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":441,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.824049+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:44:29.824049+0000","last_peered":"2023-10-01T09:44:29.824049+0000","last_clean":"2023-10-01T09:44:29.824049+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:44:29.824049+0000","last_undegraded":"2023-10-01T09:44:29.824049+0000","last_fullsized":"2023-10-01T09:44:29.824049+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"531'6607","reported_seq":6679,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:41.089542+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:44:41.089542+0000","last_peered":"2023-10-01T09:44:41.089542+0000","last_clean":"2023-10-01T09:44:41.089542+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:44:41.089542+0000","last_undegraded":"2023-10-01T09:44:41.089542+0000","last_fullsized":"2023-10-01T09:44:41.089542+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6607,"log_dups_size":0,"ondisk_log_size":6607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245694464,"num_objects":3749,"num_object_clones":0,"num_object_copies":11247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6607,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":470,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.880560+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:44:29.880560+0000","last_peered":"2023-10-01T09:44:29.880560+0000","last_clean":"2023-10-01T09:44:29.880560+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:44:29.880560+0000","last_undegraded":"2023-10-01T09:44:29.880560+0000","last_fullsized":"2023-10-01T09:44:29.880560+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1164,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.825984+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:44:37.825984+0000","last_peered":"2023-10-01T09:44:37.825984+0000","last_clean":"2023-10-01T09:44:37.825984+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:44:37.825984+0000","last_undegraded":"2023-10-01T09:44:37.825984+0000","last_fullsized":"2023-10-01T09:44:37.825984+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":59,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.879857+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:44:29.879857+0000","last_peered":"2023-10-01T09:44:29.879857+0000","last_clean":"2023-10-01T09:44:29.879857+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:44:29.879857+0000","last_undegraded":"2023-10-01T09:44:29.879857+0000","last_fullsized":"2023-10-01T09:44:29.879857+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"531'6661","reported_seq":6733,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:41.113148+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:44:41.113148+0000","last_peered":"2023-10-01T09:44:41.113148+0000","last_clean":"2023-10-01T09:44:41.113148+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:44:41.113148+0000","last_undegraded":"2023-10-01T09:44:41.113148+0000","last_fullsized":"2023-10-01T09:44:41.113148+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6661,"log_dups_size":0,"ondisk_log_size":6661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252379136,"num_objects":3851,"num_object_clones":0,"num_object_copies":11553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6661,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":63,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.880497+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:44:29.880497+0000","last_peered":"2023-10-01T09:44:29.880497+0000","last_clean":"2023-10-01T09:44:29.880497+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:44:29.880497+0000","last_undegraded":"2023-10-01T09:44:29.880497+0000","last_fullsized":"2023-10-01T09:44:29.880497+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":548,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.765882+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:44:29.765882+0000","last_peered":"2023-10-01T09:44:29.765882+0000","last_clean":"2023-10-01T09:44:29.765882+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:44:29.765882+0000","last_undegraded":"2023-10-01T09:44:29.765882+0000","last_fullsized":"2023-10-01T09:44:29.765882+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":254,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.823431+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:44:29.823431+0000","last_peered":"2023-10-01T09:44:29.823431+0000","last_clean":"2023-10-01T09:44:29.823431+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:44:29.823431+0000","last_undegraded":"2023-10-01T09:44:29.823431+0000","last_fullsized":"2023-10-01T09:44:29.823431+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":497,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.765402+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:44:29.765402+0000","last_peered":"2023-10-01T09:44:29.765402+0000","last_clean":"2023-10-01T09:44:29.765402+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:44:29.765402+0000","last_undegraded":"2023-10-01T09:44:29.765402+0000","last_fullsized":"2023-10-01T09:44:29.765402+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":611,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.881533+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:44:29.881533+0000","last_peered":"2023-10-01T09:44:29.881533+0000","last_clean":"2023-10-01T09:44:29.881533+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:44:29.881533+0000","last_undegraded":"2023-10-01T09:44:29.881533+0000","last_fullsized":"2023-10-01T09:44:29.881533+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":266,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.823078+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:44:29.823078+0000","last_peered":"2023-10-01T09:44:29.823078+0000","last_clean":"2023-10-01T09:44:29.823078+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:44:29.823078+0000","last_undegraded":"2023-10-01T09:44:29.823078+0000","last_fullsized":"2023-10-01T09:44:29.823078+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":54,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.881270+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:44:29.881270+0000","last_peered":"2023-10-01T09:44:29.881270+0000","last_clean":"2023-10-01T09:44:29.881270+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:44:29.881270+0000","last_undegraded":"2023-10-01T09:44:29.881270+0000","last_fullsized":"2023-10-01T09:44:29.881270+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":266,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.823705+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:44:29.823705+0000","last_peered":"2023-10-01T09:44:29.823705+0000","last_clean":"2023-10-01T09:44:29.823705+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:44:29.823705+0000","last_undegraded":"2023-10-01T09:44:29.823705+0000","last_fullsized":"2023-10-01T09:44:29.823705+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":443,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.765302+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:44:29.765302+0000","last_peered":"2023-10-01T09:44:29.765302+0000","last_clean":"2023-10-01T09:44:29.765302+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:44:29.765302+0000","last_undegraded":"2023-10-01T09:44:29.765302+0000","last_fullsized":"2023-10-01T09:44:29.765302+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":72,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.881454+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:44:29.881454+0000","last_peered":"2023-10-01T09:44:29.881454+0000","last_clean":"2023-10-01T09:44:29.881454+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:44:29.881454+0000","last_undegraded":"2023-10-01T09:44:29.881454+0000","last_fullsized":"2023-10-01T09:44:29.881454+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":488,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.764874+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:44:29.764874+0000","last_peered":"2023-10-01T09:44:29.764874+0000","last_clean":"2023-10-01T09:44:29.764874+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:44:29.764874+0000","last_undegraded":"2023-10-01T09:44:29.764874+0000","last_fullsized":"2023-10-01T09:44:29.764874+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":688,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.766031+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:44:29.766031+0000","last_peered":"2023-10-01T09:44:29.766031+0000","last_clean":"2023-10-01T09:44:29.766031+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:44:29.766031+0000","last_undegraded":"2023-10-01T09:44:29.766031+0000","last_fullsized":"2023-10-01T09:44:29.766031+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":382,"reported_epoch":530,"state":"active+clean","last_fresh":"2023-10-01T09:44:29.767041+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:44:29.767041+0000","last_peered":"2023-10-01T09:44:29.767041+0000","last_clean":"2023-10-01T09:44:29.767041+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:44:29.767041+0000","last_undegraded":"2023-10-01T09:44:29.767041+0000","last_fullsized":"2023-10-01T09:44:29.767041+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3970433048,"num_objects":60585,"num_object_clones":0,"num_object_copies":181755,"num_objects_missing_on_primary":2227,"num_objects_missing":2227,"num_objects_degraded":7303,"num_objects_misplaced":3703,"num_objects_unfound":0,"num_objects_dirty":60585,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":105855,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8370,"num_bytes_recovered":548536320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12317110272,"data_stored":12317098056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105855,"ondisk_log_size":105855,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430729,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2116648,"kb_used_data":1217632,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":91628504,"statfs":{"total":95995035648,"available":93827588096,"internally_reserved":0,"allocated":1246855168,"data_stored":1245915844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":1.1359999999999999,"15min":2.657},"min":{"1min":0.27300000000000002,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":1.484,"5min":10.391999999999999,"15min":108.90000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.39900000000000002,"5min":1.26,"15min":2.8090000000000002},"min":{"1min":0.219,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.67200000000000004,"5min":12.026,"15min":109.187},"last":0.47399999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":4.2910000000000004,"15min":5.2409999999999997},"min":{"1min":0.28999999999999998,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":1.4470000000000001,"5min":229.161,"15min":229.161},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.42999999999999999,"5min":4.2759999999999998,"15min":5.2510000000000003},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":0.96499999999999997,"5min":230.126,"15min":230.126},"last":0.36799999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:43:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.169,"15min":1.76},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.992,"5min":14.866,"15min":14.901999999999999},"last":0.435},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.4450000000000001,"15min":1.7170000000000001},"min":{"1min":0.32000000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4490000000000001,"5min":14.682,"15min":14.682},"last":0.495}]},{"osd":3,"last update":"Sun Oct 1 09:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.96899999999999997,"15min":2.569},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.018,"5min":6.0069999999999997,"15min":80.352999999999994},"last":1.474},{"interface":"front","average":{"1min":0.51300000000000001,"5min":1.1359999999999999,"15min":2.5510000000000002},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.026,"5min":14.528,"15min":80.364000000000004},"last":1.498}]},{"osd":4,"last update":"Sun Oct 1 09:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":1.046,"15min":1.0429999999999999},"min":{"1min":0.30199999999999999,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.508,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.0289999999999999,"15min":1.0189999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.28000000000000003},"max":{"1min":0.84399999999999997,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.51200000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":2.177,"15min":1.5800000000000001},"min":{"1min":0.34399999999999997,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":0.871,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.61299999999999999,"5min":2.1789999999999998,"15min":1.6619999999999999},"min":{"1min":0.307,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.3319999999999999,"5min":135.339,"15min":135.339},"last":0.39700000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":1.1639999999999999,"15min":1.1359999999999999},"min":{"1min":0.38,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":0.998,"5min":11.58,"15min":11.853999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.94199999999999995,"15min":1.097},"min":{"1min":0.38300000000000001,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":2.298,"5min":13.021000000000001,"15min":14.584},"last":0.42499999999999999}]}]},{"osd":6,"up_from":294,"seq":1262720385239,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2048092,"kb_used_data":1718676,"kb_used_omap":6,"kb_used_meta":329401,"kb_avail":91697060,"statfs":{"total":95995035648,"available":93897789440,"internally_reserved":0,"allocated":1759924224,"data_stored":1758989175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":337306669},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":2.0630000000000002,"15min":4.0090000000000003},"min":{"1min":0.247,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":3.2029999999999998,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.41399999999999998,"5min":2.1739999999999999,"15min":4.0730000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.221},"max":{"1min":0.63900000000000001,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.45500000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":3.1379999999999999,"15min":4.8600000000000003},"min":{"1min":0.29199999999999998,"5min":0.254,"15min":0.24199999999999999},"max":{"1min":0.58899999999999997,"5min":168.947,"15min":168.947},"last":0.497},{"interface":"front","average":{"1min":0.56000000000000005,"5min":3.141,"15min":4.8200000000000003},"min":{"1min":0.32500000000000001,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":2.8290000000000002,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.377}]},{"osd":2,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.78400000000000003,"15min":1.5620000000000001},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":0.65700000000000003,"5min":8.657,"15min":10.92},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.88600000000000001,"15min":1.5629999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":2.863,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.32700000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.96499999999999997,"15min":2.101},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.3149999999999999,"5min":9.0180000000000007,"15min":89.753},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.439,"5min":1.097,"15min":2.0179999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.61199999999999999,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.56699999999999995}]},{"osd":4,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.5049999999999999,"15min":1.0740000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.7749999999999999,"5min":63.134,"15min":63.134},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.478,"15min":1.075},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":3.2519999999999998,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.504}]},{"osd":5,"last update":"Sun Oct 1 09:44:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.246,"15min":1.073},"min":{"1min":0.28799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":3.2890000000000001,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.624,"5min":1.3260000000000001,"15min":1.1859999999999999},"min":{"1min":0.29899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.887,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.41899999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.73999999999999999,"15min":1.2729999999999999},"min":{"1min":0.32700000000000001,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":3.339,"5min":3.8279999999999998,"15min":94.245999999999995},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.496,"5min":0.96999999999999997,"15min":1.3640000000000001},"min":{"1min":0.374,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":0.68600000000000005,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.65800000000000003}]}]},{"osd":1,"up_from":94,"seq":403726926216,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1553244,"kb_used_data":743252,"kb_used_omap":7,"kb_used_meta":809976,"kb_avail":92191908,"statfs":{"total":95995035648,"available":94404513792,"internally_reserved":0,"allocated":761090048,"data_stored":759049034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":829415847},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":1.7789999999999999,"15min":4.0039999999999996},"min":{"1min":0.186,"5min":0.186,"15min":0.182},"max":{"1min":1.2889999999999999,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.42099999999999999,"5min":1.99,"15min":3.5329999999999999},"min":{"1min":0.223,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":1.702,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.373}]},{"osd":2,"last update":"Sun Oct 1 09:44:39 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.42399999999999999,"15min":2.4889999999999999},"min":{"1min":0.23300000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":1.3240000000000001,"5min":2.8740000000000001,"15min":56.573},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.61199999999999999,"15min":3.1829999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":2.2200000000000002,"5min":6.9710000000000001,"15min":154.197},"last":0.33200000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":1.2769999999999999,"15min":3.7909999999999999},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":1.7889999999999999,"5min":18.503,"15min":200.56800000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.2330000000000001,"15min":4.5039999999999996},"min":{"1min":0.253,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":1.9370000000000001,"5min":16.515000000000001,"15min":200.63800000000001},"last":0.38300000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.147,"15min":2.2530000000000001},"min":{"1min":0.35499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.0550000000000002,"5min":19.539000000000001,"15min":154.101},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.46899999999999997,"5min":1.0029999999999999,"15min":2.399},"min":{"1min":0.29499999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.59799999999999998,"5min":10.084,"15min":154.56999999999999},"last":0.56200000000000006}]},{"osd":5,"last update":"Sun Oct 1 09:44:39 2023","interfaces":[{"interface":"back","average":{"1min":3.593,"5min":1.272,"15min":2.2759999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":59.561,"5min":59.561,"15min":154.22999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":3.5979999999999999,"5min":1.24,"15min":2.5680000000000001},"min":{"1min":0.35699999999999998,"5min":0.309,"15min":0.307},"max":{"1min":59.536000000000001,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.58099999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.236,"15min":2.5539999999999998},"min":{"1min":0.34300000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.4399999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.503},{"interface":"front","average":{"1min":0.81399999999999995,"5min":1.335,"15min":2.4119999999999999},"min":{"1min":0.34699999999999998,"5min":0.34599999999999997,"15min":0.32700000000000001},"max":{"1min":2.5630000000000002,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.44700000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.3109999999999999,"15min":2.5840000000000001},"min":{"1min":0.38400000000000001,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":2.8170000000000002,"5min":19.431999999999999,"15min":154.20699999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.77800000000000002,"5min":1.2330000000000001,"15min":2.6379999999999999},"min":{"1min":0.45200000000000001,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":2.8290000000000002,"5min":10.111000000000001,"15min":154.51599999999999},"last":0.59799999999999998}]}]},{"osd":0,"up_from":89,"seq":382252089751,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4008524,"kb_used_data":3091140,"kb_used_omap":8,"kb_used_meta":917367,"kb_avail":89736628,"statfs":{"total":95995035648,"available":91890307072,"internally_reserved":0,"allocated":3165327360,"data_stored":3163307254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":939384770},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:44:16 2023","interfaces":[{"interface":"back","average":{"1min":0.317,"5min":1.1839999999999999,"15min":4.1120000000000001},"min":{"1min":0.21199999999999999,"5min":0.17799999999999999,"15min":0.17399999999999999},"max":{"1min":0.46300000000000002,"5min":52.316000000000003,"15min":216.821},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.317,"5min":2.7919999999999998,"15min":4.9809999999999999},"min":{"1min":0.24399999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.435,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.32400000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.69999999999999996,"15min":2.0339999999999998},"min":{"1min":0.29299999999999998,"5min":0.246,"15min":0.20100000000000001},"max":{"1min":4.2430000000000003,"5min":4.9459999999999997,"15min":19.788},"last":0.309},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.70799999999999996,"15min":2.1200000000000001},"min":{"1min":0.30399999999999999,"5min":0.23000000000000001,"15min":0.189},"max":{"1min":3.4940000000000002,"5min":4.5359999999999996,"15min":25.844999999999999},"last":0.36399999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:44:16 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.90200000000000002,"15min":3.46},"min":{"1min":0.26900000000000002,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.59199999999999997,"5min":8.2889999999999997,"15min":222.16499999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.0369999999999999,"15min":3.375},"min":{"1min":0.26900000000000002,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":3.4119999999999999,"5min":9.7539999999999996,"15min":222.227},"last":0.27200000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.5449999999999999,"15min":2.1339999999999999},"min":{"1min":0.28799999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":3.375,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.441},{"interface":"front","average":{"1min":0.58399999999999996,"5min":1.4550000000000001,"15min":2.1309999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":3.5049999999999999,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.35599999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.76300000000000001,"15min":1.8069999999999999},"min":{"1min":0.33200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.476,"5min":5.8769999999999998,"15min":16.808},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.90300000000000002,"15min":1.708},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":4.2729999999999997,"5min":5.0629999999999997,"15min":17.507000000000001},"last":0.378}]},{"osd":6,"last update":"Sun Oct 1 09:44:27 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.95599999999999996,"15min":1.732},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.81399999999999995,"5min":6.5,"15min":17.535},"last":0.432},{"interface":"front","average":{"1min":0.504,"5min":1.0920000000000001,"15min":1.8300000000000001},"min":{"1min":0.38700000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":0.86199999999999999,"5min":10.446,"15min":15.651999999999999},"last":0.46899999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.95499999999999996,"15min":2.048},"min":{"1min":0.379,"5min":0.374,"15min":0.35699999999999998},"max":{"1min":0.78600000000000003,"5min":4.976,"15min":48.895000000000003},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.093,"15min":2.206},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.35099999999999998},"max":{"1min":3.4319999999999999,"5min":9.4969999999999999,"15min":48.274000000000001},"last":0.497}]}]},{"osd":2,"up_from":0,"seq":425201762690,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599163,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1601416,"kb_used_data":769280,"kb_used_omap":3,"kb_used_meta":832124,"kb_avail":92143736,"statfs":{"total":95995035648,"available":94355185664,"internally_reserved":0,"allocated":787742720,"data_stored":785655374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":852095719},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":3.2000000000000002,"15min":4.8920000000000003},"min":{"1min":0.20100000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":1.3580000000000001,"5min":92.649000000000001,"15min":137.34399999999999},"last":0.318},{"interface":"front","average":{"1min":0.36399999999999999,"5min":3.194,"15min":4.9740000000000002},"min":{"1min":0.21199999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":1.2549999999999999,"5min":92.284000000000006,"15min":137.73400000000001},"last":0.23599999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":1.8819999999999999,"15min":5.1500000000000004},"min":{"1min":0.25,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":1.1599999999999999,"5min":61.439999999999998,"15min":162.22999999999999},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":1.788,"15min":5.181},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.4119999999999999,"5min":62.723999999999997,"15min":162.70699999999999},"last":0.35799999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.0780000000000001,"15min":1.748},"min":{"1min":0.27800000000000002,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":2.4710000000000001,"5min":14.254,"15min":14.254},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.66300000000000003,"15min":1.798},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":4.5049999999999999,"5min":4.5049999999999999,"15min":15.523},"last":0.52500000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.72,"15min":1.8080000000000001},"min":{"1min":0.35199999999999998,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":1.444,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.887,"15min":2.0630000000000002},"min":{"1min":0.29499999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.2250000000000001,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.46100000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:44:32 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":1.0780000000000001,"15min":2.3530000000000002},"min":{"1min":0.32000000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.196,"5min":9.4320000000000004,"15min":28.117999999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.073,"15min":2.4609999999999999},"min":{"1min":0.33600000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.92300000000000004,"5min":10.798999999999999,"15min":28.34},"last":0.53600000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:44:28 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.7350000000000001,"15min":1.8999999999999999},"min":{"1min":0.37,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":1.1000000000000001,"5min":73.887,"15min":73.887},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.952,"15min":1.974},"min":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":1.294,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.44900000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":1.268,"15min":1.845},"min":{"1min":0.42299999999999999,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":2.573,"5min":11.659000000000001,"15min":13.085000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.0740000000000001,"15min":1.726},"min":{"1min":0.39700000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":1.472,"5min":8.3460000000000001,"15min":13.191000000000001},"last":0.55000000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019414,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3896484,"kb_used_data":2998108,"kb_used_omap":0,"kb_used_meta":898368,"kb_avail":89848668,"statfs":{"total":95995035648,"available":92005036032,"internally_reserved":0,"allocated":3070062592,"data_stored":3069126774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919928832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":2.3570000000000002,"15min":4.25},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.47299999999999998,"5min":79.837000000000003,"15min":155.315},"last":0.379},{"interface":"front","average":{"1min":0.44900000000000001,"5min":2.4300000000000002,"15min":4.7039999999999997},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":0.63700000000000001,"5min":78.983000000000004,"15min":155.334},"last":0.35199999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":4.5709999999999997,"15min":6.4699999999999998},"min":{"1min":0.28899999999999998,"5min":0.252,"15min":0.224},"max":{"1min":0.52900000000000003,"5min":126.742,"15min":190.61199999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.40200000000000002,"5min":4.5369999999999999,"15min":7.1600000000000001},"min":{"1min":0.27600000000000002,"5min":0.26700000000000002,"15min":0.25600000000000001},"max":{"1min":0.59999999999999998,"5min":126.791,"15min":253.273},"last":0.72499999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":1.05,"15min":2.444},"min":{"1min":0.34200000000000003,"5min":0.31,"15min":0.28299999999999997},"max":{"1min":2.9510000000000001,"5min":10.161,"15min":170.45400000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.0029999999999999,"15min":2.1739999999999999},"min":{"1min":0.314,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":3.0219999999999998,"5min":5.9740000000000002,"15min":58.771000000000001},"last":0.56200000000000006}]},{"osd":3,"last update":"Sun Oct 1 09:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.96499999999999997,"15min":2.3079999999999998},"min":{"1min":0.28100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.1299999999999999,"5min":7.5830000000000002,"15min":140.881},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.876,"15min":2.4830000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.311},"max":{"1min":0.51500000000000001,"5min":6.8719999999999999,"15min":140.911},"last":0.75700000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":1.929,"15min":1.52},"min":{"1min":0.33800000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":1.7509999999999999,"5min":110.003,"15min":110.003},"last":0.505},{"interface":"front","average":{"1min":0.84499999999999997,"5min":1.9239999999999999,"15min":1.405},"min":{"1min":0.38400000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.2909999999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.78100000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:44:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.78200000000000003,"15min":0.93000000000000005},"min":{"1min":0.33500000000000002,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":0.73599999999999999,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.85099999999999998,"15min":1.0589999999999999},"min":{"1min":0.40600000000000003,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":0.73599999999999999,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.53200000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:43:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.71599999999999997,"15min":1.161},"min":{"1min":0.313,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":0.69399999999999995,"5min":4.7030000000000003,"15min":38.034999999999997},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.026,"15min":1.4410000000000001},"min":{"1min":0.36699999999999999,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":2.9940000000000002,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.58899999999999997}]}]},{"osd":5,"up_from":379,"seq":1627792605321,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1912144,"kb_used_data":1551688,"kb_used_omap":13,"kb_used_meta":360434,"kb_avail":91833008,"statfs":{"total":95995035648,"available":94037000192,"internally_reserved":0,"allocated":1588928512,"data_stored":1588008598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":369084914},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":1.244,"15min":2.2229999999999999},"min":{"1min":0.24299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":27.584,"5min":27.584,"15min":84.340999999999994},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.9430000000000001,"5min":1.1200000000000001,"15min":2.1280000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23300000000000001},"max":{"1min":27.669,"5min":27.669,"15min":84.356999999999999},"last":0.38200000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":3.4430000000000001,"15min":5.6900000000000004},"min":{"1min":0.28699999999999998,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":3.7799999999999998,"5min":149.94900000000001,"15min":295.71300000000002},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.71199999999999997,"5min":3.7149999999999999,"15min":5.6859999999999999},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":3.9580000000000002,"5min":149.41999999999999,"15min":299.59399999999999},"last":0.41599999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.83799999999999997,"15min":1.552},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.7029999999999998,"5min":5.3470000000000004,"15min":14.199999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.84599999999999997,"5min":1.056,"15min":1.5309999999999999},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":3.944,"5min":6.984,"15min":14.430999999999999},"last":0.34200000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.876,"15min":1.46},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.0209999999999999,"5min":10.022,"15min":14.004},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.91300000000000003,"15min":1.5940000000000001},"min":{"1min":0.34300000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.8610000000000002,"5min":5.4400000000000004,"15min":13.991},"last":0.42999999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":2.085,"15min":1.417},"min":{"1min":0.28599999999999998,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":4.0839999999999996,"5min":122.971,"15min":122.971},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.86299999999999999,"5min":2.1949999999999998,"15min":1.6699999999999999},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.0810000000000004,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.439}]},{"osd":6,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":1.0169999999999999,"15min":1.0269999999999999},"min":{"1min":0.32500000000000001,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":4.3300000000000001,"5min":10.500999999999999,"15min":12.545},"last":0.375},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.97199999999999998,"15min":1.264},"min":{"1min":0.314,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":4.0599999999999996,"5min":6.4729999999999999,"15min":14.193},"last":0.45600000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.0529999999999999,"15min":1.232},"min":{"1min":0.378,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.0999999999999996,"5min":5.758,"15min":10.813000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.996,"5min":1.1339999999999999,"15min":1.0840000000000001},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":4.2000000000000002,"5min":10.039,"15min":12.500999999999999},"last":0.49399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3154186240,"data_stored":3154182168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":749010944,"data_stored":749010944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":778108928,"data_stored":778108928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3063484416,"data_stored":3063480344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1583153152,"data_stored":1583153152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1752961024,"data_stored":1752956952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1236205568,"data_stored":1236205568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:44:45.719 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:44:45.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:45 smithi190 ceph-mon[160435]: from='client.56932 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:45.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:45 smithi190 ceph-mon[160435]: pgmap v1553: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 7313/181704 objects degraded (4.025%); 3703/181704 objects misplaced (2.038%); 633 KiB/s, 9 objects/s recovering 2023-10-01T09:44:45.919 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:45.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:45.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:46.026 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:46.056 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:46.161 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:46.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:46.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:46.265 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:46.296 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:46.539 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:44:46.539 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:44:46.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:44:46.539 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:44:46.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.539 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58507 10.7 10:e26da5e8:::benchmark_data_smithi138_573643_object58506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:44:46.539 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.384942+0000", 2023-10-01T09:44:46.539 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.15103469900001, 2023-10-01T09:44:46.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1937820219999999, 2023-10-01T09:44:46.540 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.540 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:44:46.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:44:46.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:46.540 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58507 2023-10-01T09:44:46.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.540 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.384942+0000", 2023-10-01T09:44:46.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.384942+0000", 2023-10-01T09:44:46.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.384940+0000", 2023-10-01T09:44:46.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:44:46.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.384966+0000", 2023-10-01T09:44:46.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6027999999999999e-05 2023-10-01T09:44:46.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.384977+0000", 2023-10-01T09:44:46.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.132e-05 2023-10-01T09:44:46.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.384987+0000", 2023-10-01T09:44:46.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9210000000000004e-06 2023-10-01T09:44:46.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385047+0000", 2023-10-01T09:44:46.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0436000000000002e-05 2023-10-01T09:44:46.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:44:46.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385059+0000", 2023-10-01T09:44:46.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1185e-05 2023-10-01T09:44:46.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.561399+0000", 2023-10-01T09:44:46.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.17634086 2023-10-01T09:44:46.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.561597+0000", 2023-10-01T09:44:46.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000197326 2023-10-01T09:44:46.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:44:46.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.561770+0000", 2023-10-01T09:44:46.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000173506 2023-10-01T09:44:46.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.567454+0000", 2023-10-01T09:44:46.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056839639999999997 2023-10-01T09:44:46.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:44:46.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.567469+0000", 2023-10-01T09:44:46.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4489e-05 2023-10-01T09:44:46.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.567521+0000", 2023-10-01T09:44:46.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2615999999999999e-05 2023-10-01T09:44:46.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.576645+0000", 2023-10-01T09:44:46.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0091240330000000001 2023-10-01T09:44:46.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.576711+0000", 2023-10-01T09:44:46.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5662999999999998e-05 2023-10-01T09:44:46.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.578724+0000", 2023-10-01T09:44:46.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020126079999999999 2023-10-01T09:44:46.548 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.548 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.548 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.548 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58516 10.7 10:ead56eb9:::benchmark_data_smithi138_573643_object58515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:44:46.548 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385002+0000", 2023-10-01T09:44:46.548 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.15097427399999, 2023-10-01T09:44:46.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2021478640000001, 2023-10-01T09:44:46.548 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.548 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.549 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:44:46.549 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:44:46.549 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:46.549 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58516 2023-10-01T09:44:46.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.549 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385002+0000", 2023-10-01T09:44:46.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385002+0000", 2023-10-01T09:44:46.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.550 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.550 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.550 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385002+0000", 2023-10-01T09:44:46.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:44:46.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385030+0000", 2023-10-01T09:44:46.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8259000000000002e-05 2023-10-01T09:44:46.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385037+0000", 2023-10-01T09:44:46.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9589999999999998e-06 2023-10-01T09:44:46.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385042+0000", 2023-10-01T09:44:46.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1919999999999996e-06 2023-10-01T09:44:46.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385080+0000", 2023-10-01T09:44:46.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8325000000000003e-05 2023-10-01T09:44:46.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:44:46.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385089+0000", 2023-10-01T09:44:46.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4880000000000007e-06 2023-10-01T09:44:46.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.565074+0000", 2023-10-01T09:44:46.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1799852770000001 2023-10-01T09:44:46.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.565200+0000", 2023-10-01T09:44:46.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012553799999999999 2023-10-01T09:44:46.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:44:46.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.565406+0000", 2023-10-01T09:44:46.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020614099999999999 2023-10-01T09:44:46.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:44:46.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.571877+0000", 2023-10-01T09:44:46.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064711639999999997 2023-10-01T09:44:46.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572124+0000", 2023-10-01T09:44:46.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00024668299999999998 2023-10-01T09:44:46.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572181+0000", 2023-10-01T09:44:46.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7689999999999998e-05 2023-10-01T09:44:46.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.585282+0000", 2023-10-01T09:44:46.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013100799 2023-10-01T09:44:46.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.585327+0000", 2023-10-01T09:44:46.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4759999999999998e-05 2023-10-01T09:44:46.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.557 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.557 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.587150+0000", 2023-10-01T09:44:46.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001823046 2023-10-01T09:44:46.557 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.558 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.558 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.558 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.558 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.558 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58518 10.7 10:e3c213cc:::benchmark_data_smithi138_573643_object58517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:44:46.558 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385051+0000", 2023-10-01T09:44:46.558 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.150925048, 2023-10-01T09:44:46.558 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.204210961, 2023-10-01T09:44:46.558 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.558 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.559 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:44:46.559 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:44:46.559 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:46.559 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58518 2023-10-01T09:44:46.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.559 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.559 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385051+0000", 2023-10-01T09:44:46.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385051+0000", 2023-10-01T09:44:46.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385051+0000", 2023-10-01T09:44:46.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:44:46.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385072+0000", 2023-10-01T09:44:46.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1389000000000001e-05 2023-10-01T09:44:46.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385078+0000", 2023-10-01T09:44:46.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0240000000000003e-06 2023-10-01T09:44:46.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385083+0000", 2023-10-01T09:44:46.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5140000000000003e-06 2023-10-01T09:44:46.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385111+0000", 2023-10-01T09:44:46.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8581999999999998e-05 2023-10-01T09:44:46.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:44:46.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385118+0000", 2023-10-01T09:44:46.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8340000000000001e-06 2023-10-01T09:44:46.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.567540+0000", 2023-10-01T09:44:46.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1824218449999999 2023-10-01T09:44:46.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.567733+0000", 2023-10-01T09:44:46.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000192898 2023-10-01T09:44:46.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:44:46.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.567867+0000", 2023-10-01T09:44:46.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013439499999999999 2023-10-01T09:44:46.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:44:46.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572194+0000", 2023-10-01T09:44:46.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043261869999999996 2023-10-01T09:44:46.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572238+0000", 2023-10-01T09:44:46.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4169999999999999e-05 2023-10-01T09:44:46.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572290+0000", 2023-10-01T09:44:46.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2352000000000003e-05 2023-10-01T09:44:46.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.587219+0000", 2023-10-01T09:44:46.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014928784 2023-10-01T09:44:46.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.587486+0000", 2023-10-01T09:44:46.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00026707599999999999 2023-10-01T09:44:46.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589262+0000", 2023-10-01T09:44:46.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017762780000000001 2023-10-01T09:44:46.567 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.568 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.568 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.568 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58527 10.7 10:ea9f70c7:::benchmark_data_smithi138_573643_object58526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:44:46.568 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385089+0000", 2023-10-01T09:44:46.568 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.15088757300001, 2023-10-01T09:44:46.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.209989129, 2023-10-01T09:44:46.568 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.569 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.569 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:44:46.569 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:44:46.569 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:46.569 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58527 2023-10-01T09:44:46.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.569 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385089+0000", 2023-10-01T09:44:46.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.570 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385089+0000", 2023-10-01T09:44:46.570 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.570 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385088+0000", 2023-10-01T09:44:46.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:44:46.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.571 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.571 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385113+0000", 2023-10-01T09:44:46.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4671000000000001e-05 2023-10-01T09:44:46.571 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.571 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385119+0000", 2023-10-01T09:44:46.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8320000000000002e-06 2023-10-01T09:44:46.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385123+0000", 2023-10-01T09:44:46.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6179999999999997e-06 2023-10-01T09:44:46.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385165+0000", 2023-10-01T09:44:46.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1640000000000001e-05 2023-10-01T09:44:46.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:44:46.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385173+0000", 2023-10-01T09:44:46.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3189999999999995e-06 2023-10-01T09:44:46.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.569904+0000", 2023-10-01T09:44:46.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1847305889999999 2023-10-01T09:44:46.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.570077+0000", 2023-10-01T09:44:46.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017342599999999999 2023-10-01T09:44:46.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:44:46.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.570269+0000", 2023-10-01T09:44:46.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000191447 2023-10-01T09:44:46.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:44:46.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572301+0000", 2023-10-01T09:44:46.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002032467 2023-10-01T09:44:46.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.576561+0000", 2023-10-01T09:44:46.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042592810000000002 2023-10-01T09:44:46.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.576591+0000", 2023-10-01T09:44:46.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9870999999999999e-05 2023-10-01T09:44:46.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.592534+0000", 2023-10-01T09:44:46.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015943704 2023-10-01T09:44:46.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.592603+0000", 2023-10-01T09:44:46.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9125e-05 2023-10-01T09:44:46.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.595078+0000", 2023-10-01T09:44:46.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024745240000000001 2023-10-01T09:44:46.578 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.578 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.578 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.579 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58532 10.7 10:ecf17a99:::benchmark_data_smithi138_573643_object58531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:44:46.579 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385131+0000", 2023-10-01T09:44:46.579 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.15084528599999, 2023-10-01T09:44:46.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2138979649999999, 2023-10-01T09:44:46.579 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.579 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.579 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:44:46.579 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:44:46.579 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:46.579 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58532 2023-10-01T09:44:46.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.580 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385131+0000", 2023-10-01T09:44:46.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385131+0000", 2023-10-01T09:44:46.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385131+0000", 2023-10-01T09:44:46.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:44:46.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385150+0000", 2023-10-01T09:44:46.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9346000000000001e-05 2023-10-01T09:44:46.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.582 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.582 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385155+0000", 2023-10-01T09:44:46.582 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3550000000000004e-06 2023-10-01T09:44:46.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385160+0000", 2023-10-01T09:44:46.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1500000000000001e-06 2023-10-01T09:44:46.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.583 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.583 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.583 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385194+0000", 2023-10-01T09:44:46.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4592999999999999e-05 2023-10-01T09:44:46.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:44:46.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385201+0000", 2023-10-01T09:44:46.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8809999999999998e-06 2023-10-01T09:44:46.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572324+0000", 2023-10-01T09:44:46.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.187123296 2023-10-01T09:44:46.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572429+0000", 2023-10-01T09:44:46.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010501 2023-10-01T09:44:46.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:44:46.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572536+0000", 2023-10-01T09:44:46.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106325 2023-10-01T09:44:46.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:44:46.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.576600+0000", 2023-10-01T09:44:46.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040639450000000002 2023-10-01T09:44:46.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.578792+0000", 2023-10-01T09:44:46.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021927990000000001 2023-10-01T09:44:46.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.578829+0000", 2023-10-01T09:44:46.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6913999999999999e-05 2023-10-01T09:44:46.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.587 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.596616+0000", 2023-10-01T09:44:46.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017786626999999999 2023-10-01T09:44:46.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.596688+0000", 2023-10-01T09:44:46.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2444e-05 2023-10-01T09:44:46.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.599029+0000", 2023-10-01T09:44:46.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002340623 2023-10-01T09:44:46.589 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.589 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.589 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.589 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58541 10.7 10:ec507a14:::benchmark_data_smithi138_573643_object58540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:44:46.589 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385164+0000", 2023-10-01T09:44:46.589 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.15081251800001, 2023-10-01T09:44:46.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.216211062, 2023-10-01T09:44:46.590 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.590 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.590 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:44:46.590 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:44:46.590 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:46.591 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58541 2023-10-01T09:44:46.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.591 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385164+0000", 2023-10-01T09:44:46.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385164+0000", 2023-10-01T09:44:46.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385163+0000", 2023-10-01T09:44:46.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:44:46.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385182+0000", 2023-10-01T09:44:46.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8678999999999999e-05 2023-10-01T09:44:46.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385188+0000", 2023-10-01T09:44:46.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4550000000000003e-06 2023-10-01T09:44:46.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385192+0000", 2023-10-01T09:44:46.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4549999999999997e-06 2023-10-01T09:44:46.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385238+0000", 2023-10-01T09:44:46.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5664000000000002e-05 2023-10-01T09:44:46.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:44:46.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385245+0000", 2023-10-01T09:44:46.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1860000000000003e-06 2023-10-01T09:44:46.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.574514+0000", 2023-10-01T09:44:46.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.189269406 2023-10-01T09:44:46.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.574676+0000", 2023-10-01T09:44:46.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000161853 2023-10-01T09:44:46.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:44:46.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.574876+0000", 2023-10-01T09:44:46.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019950799999999999 2023-10-01T09:44:46.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.578875+0000", 2023-10-01T09:44:46.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003999598 2023-10-01T09:44:46.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:44:46.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.578887+0000", 2023-10-01T09:44:46.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1678000000000001e-05 2023-10-01T09:44:46.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.578923+0000", 2023-10-01T09:44:46.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6443999999999999e-05 2023-10-01T09:44:46.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.599150+0000", 2023-10-01T09:44:46.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020226289000000001 2023-10-01T09:44:46.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.599226+0000", 2023-10-01T09:44:46.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6650000000000006e-05 2023-10-01T09:44:46.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.601375+0000", 2023-10-01T09:44:46.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021485469999999998 2023-10-01T09:44:46.600 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.600 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.600 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.600 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58579 10.7 10:e45d4eec:::benchmark_data_smithi138_573643_object58578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:44:46.600 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385196+0000", 2023-10-01T09:44:46.600 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.150780145, 2023-10-01T09:44:46.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.218245164, 2023-10-01T09:44:46.601 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.601 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.601 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:44:46.601 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:44:46.601 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:46.601 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58579 2023-10-01T09:44:46.601 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.601 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385196+0000", 2023-10-01T09:44:46.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385196+0000", 2023-10-01T09:44:46.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385196+0000", 2023-10-01T09:44:46.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:44:46.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385233+0000", 2023-10-01T09:44:46.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6854999999999999e-05 2023-10-01T09:44:46.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385239+0000", 2023-10-01T09:44:46.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9950000000000002e-06 2023-10-01T09:44:46.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385243+0000", 2023-10-01T09:44:46.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4959999999999998e-06 2023-10-01T09:44:46.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385266+0000", 2023-10-01T09:44:46.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2819999999999998e-05 2023-10-01T09:44:46.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:44:46.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385273+0000", 2023-10-01T09:44:46.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1550000000000001e-06 2023-10-01T09:44:46.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.578945+0000", 2023-10-01T09:44:46.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.193672235 2023-10-01T09:44:46.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.579103+0000", 2023-10-01T09:44:46.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015765000000000001 2023-10-01T09:44:46.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:44:46.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.579233+0000", 2023-10-01T09:44:46.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013000800000000001 2023-10-01T09:44:46.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:44:46.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589288+0000", 2023-10-01T09:44:46.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010055364000000001 2023-10-01T09:44:46.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589351+0000", 2023-10-01T09:44:46.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2792000000000005e-05 2023-10-01T09:44:46.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.610 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.610 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589444+0000", 2023-10-01T09:44:46.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3288000000000005e-05 2023-10-01T09:44:46.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.601431+0000", 2023-10-01T09:44:46.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011986939 2023-10-01T09:44:46.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.601491+0000", 2023-10-01T09:44:46.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9592000000000001e-05 2023-10-01T09:44:46.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.603441+0000", 2023-10-01T09:44:46.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001950301 2023-10-01T09:44:46.612 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.612 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.612 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.613 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58611 10.7 10:e059d00b:::benchmark_data_smithi138_573643_object58610:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:44:46.613 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385248+0000", 2023-10-01T09:44:46.613 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.150728638, 2023-10-01T09:44:46.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.219992653, 2023-10-01T09:44:46.613 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.613 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.613 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:44:46.613 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:44:46.613 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:46.614 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58611 2023-10-01T09:44:46.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.614 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385248+0000", 2023-10-01T09:44:46.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385248+0000", 2023-10-01T09:44:46.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385247+0000", 2023-10-01T09:44:46.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:44:46.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385268+0000", 2023-10-01T09:44:46.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1151000000000001e-05 2023-10-01T09:44:46.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385274+0000", 2023-10-01T09:44:46.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4670000000000003e-06 2023-10-01T09:44:46.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385278+0000", 2023-10-01T09:44:46.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.34e-06 2023-10-01T09:44:46.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385308+0000", 2023-10-01T09:44:46.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9634000000000001e-05 2023-10-01T09:44:46.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:44:46.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385315+0000", 2023-10-01T09:44:46.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4250000000000001e-06 2023-10-01T09:44:46.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.618 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.618 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.580981+0000", 2023-10-01T09:44:46.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.195665966 2023-10-01T09:44:46.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.581114+0000", 2023-10-01T09:44:46.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013232900000000001 2023-10-01T09:44:46.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:44:46.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.581238+0000", 2023-10-01T09:44:46.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000123955 2023-10-01T09:44:46.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:44:46.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589297+0000", 2023-10-01T09:44:46.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0080591729999999993 2023-10-01T09:44:46.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589402+0000", 2023-10-01T09:44:46.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010554 2023-10-01T09:44:46.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589470+0000", 2023-10-01T09:44:46.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7218000000000006e-05 2023-10-01T09:44:46.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.603490+0000", 2023-10-01T09:44:46.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014020566 2023-10-01T09:44:46.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.603542+0000", 2023-10-01T09:44:46.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2351000000000001e-05 2023-10-01T09:44:46.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.605240+0000", 2023-10-01T09:44:46.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0016979 2023-10-01T09:44:46.623 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.623 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.623 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.623 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58625 10.7 10:eb5c5627:::benchmark_data_smithi138_573643_object58624:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:44:46.623 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385285+0000", 2023-10-01T09:44:46.623 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 231.150691002, 2023-10-01T09:44:46.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.221588224, 2023-10-01T09:44:46.624 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.624 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.624 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:44:46.624 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:44:46.624 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:46.624 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58625 2023-10-01T09:44:46.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.624 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385285+0000", 2023-10-01T09:44:46.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385285+0000", 2023-10-01T09:44:46.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385285+0000", 2023-10-01T09:44:46.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:44:46.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385327+0000", 2023-10-01T09:44:46.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1712999999999999e-05 2023-10-01T09:44:46.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385343+0000", 2023-10-01T09:44:46.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6050999999999999e-05 2023-10-01T09:44:46.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385350+0000", 2023-10-01T09:44:46.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3950000000000003e-06 2023-10-01T09:44:46.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385383+0000", 2023-10-01T09:44:46.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2758e-05 2023-10-01T09:44:46.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.628 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:44:46.628 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385391+0000", 2023-10-01T09:44:46.628 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6960000000000005e-06 2023-10-01T09:44:46.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.583192+0000", 2023-10-01T09:44:46.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.197801355 2023-10-01T09:44:46.629 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.629 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.629 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.629 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.583333+0000", 2023-10-01T09:44:46.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000141426 2023-10-01T09:44:46.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:44:46.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.583441+0000", 2023-10-01T09:44:46.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107808 2023-10-01T09:44:46.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:44:46.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589360+0000", 2023-10-01T09:44:46.630 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059190650000000003 2023-10-01T09:44:46.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589489+0000", 2023-10-01T09:44:46.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000128369 2023-10-01T09:44:46.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589511+0000", 2023-10-01T09:44:46.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2337999999999999e-05 2023-10-01T09:44:46.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.605294+0000", 2023-10-01T09:44:46.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015783177999999998 2023-10-01T09:44:46.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.605345+0000", 2023-10-01T09:44:46.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0424999999999999e-05 2023-10-01T09:44:46.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.606874+0000", 2023-10-01T09:44:46.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015289909999999999 2023-10-01T09:44:46.634 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.634 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.635 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.635 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:89631 10.a 10:5f97a957:::benchmark_data_smithi138_573643_object6409:head [delete] snapc 0=[] ondisk+write+known_if_redirected e515)", 2023-10-01T09:44:46.635 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:42:34.324321+0000", 2023-10-01T09:44:46.635 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 132.21165505600001, 2023-10-01T09:44:46.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.083869808000000004, 2023-10-01T09:44:46.636 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.636 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.636 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:44:46.636 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:44:46.636 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:46.636 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 89631 2023-10-01T09:44:46.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.637 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:34.324321+0000", 2023-10-01T09:44:46.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:34.324321+0000", 2023-10-01T09:44:46.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.639 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:34.324320+0000", 2023-10-01T09:44:46.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:44:46.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.639 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.639 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:34.324326+0000", 2023-10-01T09:44:46.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8000000000000004e-06 2023-10-01T09:44:46.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:34.324337+0000", 2023-10-01T09:44:46.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1090999999999999e-05 2023-10-01T09:44:46.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.641 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:34.324343+0000", 2023-10-01T09:44:46.641 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.908e-06 2023-10-01T09:44:46.641 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:34.324382+0000", 2023-10-01T09:44:46.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9033999999999999e-05 2023-10-01T09:44:46.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:34.324479+0000", 2023-10-01T09:44:46.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7014000000000003e-05 2023-10-01T09:44:46.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,6", 2023-10-01T09:44:46.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:34.324555+0000", 2023-10-01T09:44:46.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6228000000000003e-05 2023-10-01T09:44:46.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:44:46.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:34.326914+0000", 2023-10-01T09:44:46.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023586200000000001 2023-10-01T09:44:46.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:34.327114+0000", 2023-10-01T09:44:46.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000200323 2023-10-01T09:44:46.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:34.408089+0000", 2023-10-01T09:44:46.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.080975121999999997 2023-10-01T09:44:46.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:34.408146+0000", 2023-10-01T09:44:46.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6793999999999999e-05 2023-10-01T09:44:46.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:34.408191+0000", 2023-10-01T09:44:46.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4913999999999997e-05 2023-10-01T09:44:46.646 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.646 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.646 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.646 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:90813 10.4 10:200bcc97:::benchmark_data_smithi138_573643_object7591:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:44:46.646 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:42:42.508517+0000", 2023-10-01T09:44:46.647 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 124.027459279, 2023-10-01T09:44:46.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11063846099999999, 2023-10-01T09:44:46.647 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.647 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.647 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:44:46.647 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:44:46.647 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:46.647 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 90813 2023-10-01T09:44:46.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.648 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508517+0000", 2023-10-01T09:44:46.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508517+0000", 2023-10-01T09:44:46.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508516+0000", 2023-10-01T09:44:46.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:44:46.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508521+0000", 2023-10-01T09:44:46.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9660000000000004e-06 2023-10-01T09:44:46.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508528+0000", 2023-10-01T09:44:46.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0400000000000004e-06 2023-10-01T09:44:46.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508533+0000", 2023-10-01T09:44:46.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6110000000000004e-06 2023-10-01T09:44:46.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508566+0000", 2023-10-01T09:44:46.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3018000000000001e-05 2023-10-01T09:44:46.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508642+0000", 2023-10-01T09:44:46.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6259000000000003e-05 2023-10-01T09:44:46.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:44:46.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508706+0000", 2023-10-01T09:44:46.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3503000000000005e-05 2023-10-01T09:44:46.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:44:46.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511112+0000", 2023-10-01T09:44:46.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00240596 2023-10-01T09:44:46.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511312+0000", 2023-10-01T09:44:46.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020054400000000001 2023-10-01T09:44:46.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.654 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.654 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511439+0000", 2023-10-01T09:44:46.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000127346 2023-10-01T09:44:46.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511468+0000", 2023-10-01T09:44:46.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8252e-05 2023-10-01T09:44:46.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.619155+0000", 2023-10-01T09:44:46.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.10768778599999999 2023-10-01T09:44:46.655 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.655 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.656 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.656 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.36456.0:90813 10.4 e516/475 ondisk, result = 0)", 2023-10-01T09:44:46.656 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:42:42.511386+0000", 2023-10-01T09:44:46.656 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 124.024590121, 2023-10-01T09:44:46.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.107776915, 2023-10-01T09:44:46.656 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.656 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:44:46.657 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511386+0000", 2023-10-01T09:44:46.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511386+0000", 2023-10-01T09:44:46.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511386+0000", 2023-10-01T09:44:46.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:44:46.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511393+0000", 2023-10-01T09:44:46.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9700000000000002e-06 2023-10-01T09:44:46.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511401+0000", 2023-10-01T09:44:46.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1969999999999996e-06 2023-10-01T09:44:46.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511405+0000", 2023-10-01T09:44:46.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2409999999999997e-06 2023-10-01T09:44:46.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511431+0000", 2023-10-01T09:44:46.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.588e-05 2023-10-01T09:44:46.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511436+0000", 2023-10-01T09:44:46.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1719999999999998e-06 2023-10-01T09:44:46.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.619163+0000", 2023-10-01T09:44:46.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.107727068 2023-10-01T09:44:46.661 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.661 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.661 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.662 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:90814 10.5 10:a1f50e2f:::benchmark_data_smithi138_573643_object7592:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:44:46.662 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:42:42.511584+0000", 2023-10-01T09:44:46.662 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 124.02439209000001, 2023-10-01T09:44:46.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.111412285, 2023-10-01T09:44:46.662 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.662 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.662 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:44:46.662 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:44:46.662 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:46.663 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 90814 2023-10-01T09:44:46.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.663 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511584+0000", 2023-10-01T09:44:46.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511584+0000", 2023-10-01T09:44:46.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511583+0000", 2023-10-01T09:44:46.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:44:46.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511588+0000", 2023-10-01T09:44:46.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8770000000000001e-06 2023-10-01T09:44:46.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.619145+0000", 2023-10-01T09:44:46.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.107557183 2023-10-01T09:44:46.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.619157+0000", 2023-10-01T09:44:46.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2109e-05 2023-10-01T09:44:46.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.619213+0000", 2023-10-01T09:44:46.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5448000000000002e-05 2023-10-01T09:44:46.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.619352+0000", 2023-10-01T09:44:46.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013952999999999999 2023-10-01T09:44:46.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:44:46.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.619468+0000", 2023-10-01T09:44:46.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011534199999999999 2023-10-01T09:44:46.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:44:46.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.622489+0000", 2023-10-01T09:44:46.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030212780000000001 2023-10-01T09:44:46.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.622708+0000", 2023-10-01T09:44:46.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021842899999999999 2023-10-01T09:44:46.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.622889+0000", 2023-10-01T09:44:46.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018121500000000001 2023-10-01T09:44:46.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.669 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.669 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.622948+0000", 2023-10-01T09:44:46.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9582e-05 2023-10-01T09:44:46.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.622996+0000", 2023-10-01T09:44:46.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8189000000000002e-05 2023-10-01T09:44:46.670 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.670 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.670 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.670 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:96170 10.d 10:bb11c78e:::benchmark_data_smithi138_573643_object12948:head [delete] snapc 0=[] ondisk+write+known_if_redirected e522)", 2023-10-01T09:44:46.671 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:43:27.154361+0000", 2023-10-01T09:44:46.671 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 79.381614859999999, 2023-10-01T09:44:46.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.083002505000000004, 2023-10-01T09:44:46.671 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.671 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.671 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:44:46.671 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:44:46.671 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:46.671 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 96170 2023-10-01T09:44:46.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.672 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.154361+0000", 2023-10-01T09:44:46.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.154361+0000", 2023-10-01T09:44:46.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.154358+0000", 2023-10-01T09:44:46.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999967 2023-10-01T09:44:46.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.154374+0000", 2023-10-01T09:44:46.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5247e-05 2023-10-01T09:44:46.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.154402+0000", 2023-10-01T09:44:46.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8062e-05 2023-10-01T09:44:46.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.154432+0000", 2023-10-01T09:44:46.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0198000000000001e-05 2023-10-01T09:44:46.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.154475+0000", 2023-10-01T09:44:46.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3325999999999998e-05 2023-10-01T09:44:46.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.154568+0000", 2023-10-01T09:44:46.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2668000000000001e-05 2023-10-01T09:44:46.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:44:46.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.154641+0000", 2023-10-01T09:44:46.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2721999999999999e-05 2023-10-01T09:44:46.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:44:46.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.157220+0000", 2023-10-01T09:44:46.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025796080000000002 2023-10-01T09:44:46.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.157619+0000", 2023-10-01T09:44:46.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00039917799999999998 2023-10-01T09:44:46.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.237276+0000", 2023-10-01T09:44:46.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.079657147999999997 2023-10-01T09:44:46.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.237323+0000", 2023-10-01T09:44:46.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6901999999999999e-05 2023-10-01T09:44:46.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.237364+0000", 2023-10-01T09:44:46.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0559999999999998e-05 2023-10-01T09:44:46.679 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.679 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.679 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.679 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.36456.0:96170 10.d e522/475 ondisk, result = 0)", 2023-10-01T09:44:46.679 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:43:27.158314+0000", 2023-10-01T09:44:46.679 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 79.377661982999996, 2023-10-01T09:44:46.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.079054685999999999, 2023-10-01T09:44:46.680 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.680 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:44:46.680 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.158314+0000", 2023-10-01T09:44:46.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.158314+0000", 2023-10-01T09:44:46.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.158314+0000", 2023-10-01T09:44:46.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:44:46.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.158318+0000", 2023-10-01T09:44:46.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3309999999999997e-06 2023-10-01T09:44:46.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.158327+0000", 2023-10-01T09:44:46.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7099999999999996e-06 2023-10-01T09:44:46.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.158332+0000", 2023-10-01T09:44:46.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7859999999999996e-06 2023-10-01T09:44:46.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.684 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.684 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.237262+0000", 2023-10-01T09:44:46.684 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.078930537999999995 2023-10-01T09:44:46.684 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.684 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.237270+0000", 2023-10-01T09:44:46.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3729999999999993e-06 2023-10-01T09:44:46.685 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.685 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.685 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.685 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:27.237369+0000", 2023-10-01T09:44:46.685 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8523999999999994e-05 2023-10-01T09:44:46.685 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.685 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.686 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.686 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.686 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.686 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:98551 10.5 10:af68cebb:::benchmark_data_smithi138_573643_object15329:head [delete] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-01T09:44:46.686 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:43:46.353721+0000", 2023-10-01T09:44:46.686 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 60.182255011999999, 2023-10-01T09:44:46.686 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.079319018000000005, 2023-10-01T09:44:46.686 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.686 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.687 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:44:46.687 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:44:46.687 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:44:46.687 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 98551 2023-10-01T09:44:46.687 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.687 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.687 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.687 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.687 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.353721+0000", 2023-10-01T09:44:46.687 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.353721+0000", 2023-10-01T09:44:46.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.688 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.688 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.688 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.688 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.353720+0000", 2023-10-01T09:44:46.688 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:44:46.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.353725+0000", 2023-10-01T09:44:46.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1719999999999998e-06 2023-10-01T09:44:46.689 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.689 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.689 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.689 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.353734+0000", 2023-10-01T09:44:46.689 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3580000000000003e-06 2023-10-01T09:44:46.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.353739+0000", 2023-10-01T09:44:46.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6110000000000001e-06 2023-10-01T09:44:46.690 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.690 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.690 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.690 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.353771+0000", 2023-10-01T09:44:46.690 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1646999999999997e-05 2023-10-01T09:44:46.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.353854+0000", 2023-10-01T09:44:46.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2979000000000006e-05 2023-10-01T09:44:46.691 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.691 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.691 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:44:46.691 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.353925+0000", 2023-10-01T09:44:46.691 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0698999999999995e-05 2023-10-01T09:44:46.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:44:46.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.356191+0000", 2023-10-01T09:44:46.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002266658 2023-10-01T09:44:46.692 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.692 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.692 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.692 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.432908+0000", 2023-10-01T09:44:46.692 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.076716610000000005 2023-10-01T09:44:46.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:44:46.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.432964+0000", 2023-10-01T09:44:46.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5572000000000003e-05 2023-10-01T09:44:46.693 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.693 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.693 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.693 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.433001+0000", 2023-10-01T09:44:46.693 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7036999999999998e-05 2023-10-01T09:44:46.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.694 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.694 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.433040+0000", 2023-10-01T09:44:46.694 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9691000000000003e-05 2023-10-01T09:44:46.694 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.694 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.694 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.694 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.694 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.695 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:98553 10.8 e524/475 10:1e92f894:::benchmark_data_smithi138_573643_object15331:head v 524'6237, mlcod=524'6235)", 2023-10-01T09:44:46.695 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:43:46.356680+0000", 2023-10-01T09:44:46.695 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 60.179296391000001, 2023-10-01T09:44:46.695 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.080019546999999996, 2023-10-01T09:44:46.695 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.695 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.695 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.695 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.695 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.695 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.356680+0000", 2023-10-01T09:44:46.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.356680+0000", 2023-10-01T09:44:46.696 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.696 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.696 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.696 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.696 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.356679+0000", 2023-10-01T09:44:46.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:44:46.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.356689+0000", 2023-10-01T09:44:46.697 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0159999999999999e-05 2023-10-01T09:44:46.697 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.697 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.697 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.697 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.356700+0000", 2023-10-01T09:44:46.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0441999999999999e-05 2023-10-01T09:44:46.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.698 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.356707+0000", 2023-10-01T09:44:46.698 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7909999999999999e-06 2023-10-01T09:44:46.698 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.698 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.698 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.432847+0000", 2023-10-01T09:44:46.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.076140342999999999 2023-10-01T09:44:46.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.699 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.432880+0000", 2023-10-01T09:44:46.699 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3284000000000002e-05 2023-10-01T09:44:46.699 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.699 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.699 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.436654+0000", 2023-10-01T09:44:46.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037742439999999999 2023-10-01T09:44:46.700 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.700 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.700 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.700 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.436699+0000", 2023-10-01T09:44:46.700 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4968e-05 2023-10-01T09:44:46.700 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.700 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.700 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.701 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.701 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.701 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:103952 10.8 e529/475 10:1c93a04e:::benchmark_data_smithi138_573643_object20730:head v 529'6554, mlcod=529'6552)", 2023-10-01T09:44:46.701 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:44:27.616960+0000", 2023-10-01T09:44:46.701 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.919016300999999, 2023-10-01T09:44:46.701 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11918105399999999, 2023-10-01T09:44:46.701 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.701 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.701 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.616960+0000", 2023-10-01T09:44:46.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.702 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.702 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.702 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.702 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.616960+0000", 2023-10-01T09:44:46.702 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.616953+0000", 2023-10-01T09:44:46.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999928 2023-10-01T09:44:46.703 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.703 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.703 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.703 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.616973+0000", 2023-10-01T09:44:46.703 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0029000000000001e-05 2023-10-01T09:44:46.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.616995+0000", 2023-10-01T09:44:46.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1849999999999999e-05 2023-10-01T09:44:46.704 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.704 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.704 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.704 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617008+0000", 2023-10-01T09:44:46.704 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3519999999999999e-05 2023-10-01T09:44:46.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617091+0000", 2023-10-01T09:44:46.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2974999999999998e-05 2023-10-01T09:44:46.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.705 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.705 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.705 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617127+0000", 2023-10-01T09:44:46.705 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5456000000000001e-05 2023-10-01T09:44:46.705 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.736078+0000", 2023-10-01T09:44:46.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11895101399999999 2023-10-01T09:44:46.706 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.706 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.706 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.706 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.736141+0000", 2023-10-01T09:44:46.706 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3173e-05 2023-10-01T09:44:46.707 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.707 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.707 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.707 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.707 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.707 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:103953 10.2 e529/475 10:414c62ad:::benchmark_data_smithi138_573643_object20731:head v 529'6517, mlcod=529'6515)", 2023-10-01T09:44:46.707 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:44:27.617965+0000", 2023-10-01T09:44:46.707 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.918011578000002, 2023-10-01T09:44:46.707 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.121350888, 2023-10-01T09:44:46.707 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.708 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.708 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617965+0000", 2023-10-01T09:44:46.708 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.708 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.708 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.708 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.708 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617965+0000", 2023-10-01T09:44:46.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.709 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617964+0000", 2023-10-01T09:44:46.709 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:44:46.709 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.709 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.709 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617970+0000", 2023-10-01T09:44:46.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.494e-06 2023-10-01T09:44:46.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.710 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617980+0000", 2023-10-01T09:44:46.710 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8959999999999998e-06 2023-10-01T09:44:46.710 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.710 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.710 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617987+0000", 2023-10-01T09:44:46.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.263e-06 2023-10-01T09:44:46.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.711 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.711 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.618028+0000", 2023-10-01T09:44:46.711 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1325999999999997e-05 2023-10-01T09:44:46.711 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.711 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.618042+0000", 2023-10-01T09:44:46.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3721e-05 2023-10-01T09:44:46.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.712 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.712 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.739290+0000", 2023-10-01T09:44:46.712 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.121248288 2023-10-01T09:44:46.712 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.712 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.713 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.713 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.739316+0000", 2023-10-01T09:44:46.713 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5629000000000001e-05 2023-10-01T09:44:46.713 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.713 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.713 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.713 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.713 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.713 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:103956 10.6 e529/475 10:6e8702de:::benchmark_data_smithi138_573643_object20734:head v 529'6521, mlcod=529'6519)", 2023-10-01T09:44:46.713 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:44:27.630692+0000", 2023-10-01T09:44:46.714 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 18.905284499, 2023-10-01T09:44:46.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.108410692, 2023-10-01T09:44:46.714 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:44:46.714 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:44:46.714 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:44:46.714 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.714 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:44:46.714 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.630692+0000", 2023-10-01T09:44:46.714 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.714 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:44:46.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.630692+0000", 2023-10-01T09:44:46.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:44:46.715 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.715 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.715 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:44:46.715 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.630685+0000", 2023-10-01T09:44:46.715 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999928 2023-10-01T09:44:46.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:44:46.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.630706+0000", 2023-10-01T09:44:46.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1769000000000001e-05 2023-10-01T09:44:46.716 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.716 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.716 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:44:46.716 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.736062+0000", 2023-10-01T09:44:46.716 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.105355559 2023-10-01T09:44:46.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:44:46.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.736081+0000", 2023-10-01T09:44:46.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8751999999999999e-05 2023-10-01T09:44:46.717 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.717 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.717 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:44:46.717 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.736124+0000", 2023-10-01T09:44:46.717 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.295e-05 2023-10-01T09:44:46.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:44:46.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.736150+0000", 2023-10-01T09:44:46.718 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6084999999999998e-05 2023-10-01T09:44:46.718 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.718 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.718 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:44:46.718 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.739067+0000", 2023-10-01T09:44:46.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002917546 2023-10-01T09:44:46.719 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:44:46.719 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:44:46.719 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:44:46.719 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.739102+0000", 2023-10-01T09:44:46.719 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5247000000000003e-05 2023-10-01T09:44:46.719 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.719 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.719 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.719 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:44:46.720 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:44:46.720 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:44:46.808 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:46.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:46.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:46.873 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:44:46.919 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:46.966 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:47.015 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:44:47.015 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:44:47.015 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:44:47.015 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:44:47.015 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:44:47.016 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:44:47.016 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:44:47.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:44:47.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:44:47.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:44:47.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:44:47.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:44:47.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:44:47.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:44:47.016 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:44:47.022 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:47.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:47.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:47.135 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:47.164 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:47.399 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:44:47.415 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:47.421 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:47 smithi138 ceph-mon[192513]: pgmap v1554: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7313/180612 objects degraded (4.049%); 3703/180612 objects misplaced (2.050%); 633 KiB/s, 9 objects/s recovering 2023-10-01T09:44:47.421 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:47 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2390628409' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:47.421 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:47 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/584867447' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:44:47.421 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:47 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:44:47.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:47.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:47.422 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:47 smithi138 ceph-mon[185886]: pgmap v1554: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7313/180612 objects degraded (4.049%); 3703/180612 objects misplaced (2.050%); 633 KiB/s, 9 objects/s recovering 2023-10-01T09:44:47.423 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:47 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2390628409' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:47.423 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:47 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/584867447' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:44:47.423 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:47 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:44:47.523 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:47.551 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:47.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:47 smithi190 ceph-mon[160435]: pgmap v1554: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7313/180612 objects degraded (4.049%); 3703/180612 objects misplaced (2.050%); 633 KiB/s, 9 objects/s recovering 2023-10-01T09:44:47.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:47 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2390628409' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:47.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:47 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/584867447' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:44:47.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:47 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:44:48.365 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:48.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:48.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:48.421 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:44:48.469 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:48.477 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:48.495 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:48.550 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:48.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:48.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:48.656 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:48.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:48 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:48.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:48 smithi138 ceph-mon[185886]: failed to encode map e532 with expected crc 2023-10-01T09:44:48.683 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:48 smithi138 ceph-mon[185886]: failed to encode map e532 with expected crc 2023-10-01T09:44:48.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:48 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:44:48.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:48 smithi138 ceph-mon[185886]: osdmap e532: 8 total, 8 up, 7 in 2023-10-01T09:44:48.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:48 smithi138 ceph-mon[185886]: failed to encode map e532 with expected crc 2023-10-01T09:44:48.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:48 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7313/180612 objects degraded (4.049%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:48.684 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:48 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:48.684 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:48 smithi138 ceph-mon[192513]: failed to encode map e532 with expected crc 2023-10-01T09:44:48.684 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:48 smithi138 ceph-mon[192513]: failed to encode map e532 with expected crc 2023-10-01T09:44:48.684 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:48 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:44:48.684 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:48 smithi138 ceph-mon[192513]: osdmap e532: 8 total, 8 up, 7 in 2023-10-01T09:44:48.685 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:48 smithi138 ceph-mon[192513]: failed to encode map e532 with expected crc 2023-10-01T09:44:48.685 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:48 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7313/180612 objects degraded (4.049%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:48.686 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:48.755 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:48.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:48.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:48.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:48 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:44:48.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:48 smithi190 ceph-mon[160435]: failed to encode map e532 with expected crc 2023-10-01T09:44:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:48 smithi190 ceph-mon[160435]: failed to encode map e532 with expected crc 2023-10-01T09:44:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:48 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:44:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:48 smithi190 ceph-mon[160435]: osdmap e532: 8 total, 8 up, 7 in 2023-10-01T09:44:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:48 smithi190 ceph-mon[160435]: failed to encode map e532 with expected crc 2023-10-01T09:44:48.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:48 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7313/180612 objects degraded (4.049%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:48.864 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:48.895 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:48.956 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:48.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:48.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:49.062 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:49.091 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:49.141 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:49.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:49.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:49.245 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:49.275 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:49.322 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:49.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:49.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:49.430 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:49.459 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:49.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:49 smithi190 ceph-mon[160435]: pgmap v1556: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 149 op/s; 7282/179061 objects degraded (4.067%); 3635/179061 objects misplaced (2.030%); 653 KiB/s, 10 objects/s recovering 2023-10-01T09:44:49.572 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:49.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:49.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:49.631 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:49.676 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:49.716 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:49 smithi138 ceph-mon[192513]: pgmap v1556: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 149 op/s; 7282/179061 objects degraded (4.067%); 3635/179061 objects misplaced (2.030%); 653 KiB/s, 10 objects/s recovering 2023-10-01T09:44:49.717 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:49 smithi138 ceph-mon[185886]: pgmap v1556: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 149 op/s; 7282/179061 objects degraded (4.067%); 3635/179061 objects misplaced (2.030%); 653 KiB/s, 10 objects/s recovering 2023-10-01T09:44:49.718 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:49.782 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:49.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:49.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:49.885 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:49.914 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:49.960 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:49.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:49.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:50.065 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:50.091 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:50.136 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:50.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:50.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:50.244 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:50.280 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:50.335 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:50.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:50.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:50.363 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:44:50.439 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:50.468 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:50.505 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:44:50.513 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:50.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:50.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:50.614 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:44:50.614 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 7282/178716 objects degraded (4.075%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2111,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":532,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":59572,"data_bytes":3907000267,"bytes_used":17314947072,"bytes_avail":654650302464,"bytes_total":671965249536,"degraded_objects":7282,"degraded_total":178716,"degraded_ratio":0.040746211866872578,"misplaced_objects":3635,"misplaced_total":178716,"misplaced_ratio":0.020339533113990914,"recovering_objects_per_sec":10,"recovering_bytes_per_sec":668307,"recovering_keys_per_sec":0,"num_objects_recovered":102,"num_bytes_recovered":6684672,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":153},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":34,"modified":"2023-10-01T09:43:40.311181+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:44:50.619 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:50.649 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:50.950 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:50.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:50.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:51.054 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:51.083 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:51.527 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:51.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:51.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:51.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:51 smithi138 ceph-mon[192513]: pgmap v1557: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 7282/178716 objects degraded (4.075%); 3635/178716 objects misplaced (2.034%); 653 KiB/s, 10 objects/s recovering 2023-10-01T09:44:51.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:51 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1047099606' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:44:51.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:51 smithi138 ceph-mon[185886]: pgmap v1557: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 7282/178716 objects degraded (4.075%); 3635/178716 objects misplaced (2.034%); 653 KiB/s, 10 objects/s recovering 2023-10-01T09:44:51.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:51 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1047099606' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:44:51.632 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:51.656 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:51 smithi190 ceph-mon[160435]: pgmap v1557: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 153 op/s; 7282/178716 objects degraded (4.075%); 3635/178716 objects misplaced (2.034%); 653 KiB/s, 10 objects/s recovering 2023-10-01T09:44:51.656 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:51 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1047099606' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:44:51.658 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:51.690 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:44:51.690 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:44:51.711 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:51.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:51.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:51.817 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:44:51.849 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:51.866 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:51.974 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:44:51.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:51.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:44:51.992 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:44:51.993 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:44:52.015 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:52.078 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:52.106 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:52.165 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:52.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:52.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:52.270 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:52.298 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:52.796 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:52.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:52.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:52.902 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:52.932 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:52.990 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:44:53.626 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:53.694 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:53.695 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:53 smithi138 ceph-mon[185886]: pgmap v1558: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 7228/176571 objects degraded (4.094%); 3589/176571 objects misplaced (2.033%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:44:53.696 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:53 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7228/176571 objects degraded (4.094%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:53.696 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:53 smithi138 ceph-mon[192513]: pgmap v1558: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 7228/176571 objects degraded (4.094%); 3589/176571 objects misplaced (2.033%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:44:53.696 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:53 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7228/176571 objects degraded (4.094%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:53.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:53.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:53.798 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:53.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:53 smithi190 ceph-mon[160435]: pgmap v1558: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 7228/176571 objects degraded (4.094%); 3589/176571 objects misplaced (2.033%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:44:53.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:53 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7228/176571 objects degraded (4.094%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:53.826 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:53.875 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:53.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:53.980 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:54.006 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:54.051 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:54.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:54.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:54.156 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:54.188 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:54.718 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:54.739 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:54.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:54.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:54.844 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:54.875 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:54.924 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:54.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:54.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:55.030 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:55.056 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:55.102 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:55.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:55.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:55.208 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:55.231 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:44:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:44:54] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.33.4" 2023-10-01T09:44:55.234 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:55.413 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:44:55.559 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:55.562 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:55 smithi138 ceph-mon[185886]: pgmap v1559: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 7228/176406 objects degraded (4.097%); 3589/176406 objects misplaced (2.035%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:44:55.562 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:55 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2354060893' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:55.562 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:55 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:55.562 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:44:55.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:55.563 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:55 smithi138 ceph-mon[192513]: pgmap v1559: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 7228/176406 objects degraded (4.097%); 3589/176406 objects misplaced (2.035%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:44:55.563 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:55 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2354060893' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:55.563 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:55 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:55.563 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:44:55.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:55.665 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:55.699 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:55.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:55 smithi190 ceph-mon[160435]: pgmap v1559: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 7228/176406 objects degraded (4.097%); 3589/176406 objects misplaced (2.035%); 640 KiB/s, 9 objects/s recovering 2023-10-01T09:44:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:55 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2354060893' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:55 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:44:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:44:56.564 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:56.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:56.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:56.668 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:56.701 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:56 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:44:56.701 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:56 smithi190 ceph-mon[160435]: failed to encode map e533 with expected crc 2023-10-01T09:44:56.701 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:56 smithi190 ceph-mon[160435]: failed to encode map e533 with expected crc 2023-10-01T09:44:56.701 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:56 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:44:56.701 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:56 smithi190 ceph-mon[160435]: osdmap e533: 8 total, 8 up, 7 in 2023-10-01T09:44:56.702 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:56.756 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:56.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:56.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:56.770 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:44:56.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:56 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:44:56.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:56 smithi138 ceph-mon[192513]: failed to encode map e533 with expected crc 2023-10-01T09:44:56.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:56 smithi138 ceph-mon[192513]: failed to encode map e533 with expected crc 2023-10-01T09:44:56.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:56 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:44:56.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:56 smithi138 ceph-mon[192513]: osdmap e533: 8 total, 8 up, 7 in 2023-10-01T09:44:56.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:56 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:44:56.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:56 smithi138 ceph-mon[185886]: failed to encode map e533 with expected crc 2023-10-01T09:44:56.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:56 smithi138 ceph-mon[185886]: failed to encode map e533 with expected crc 2023-10-01T09:44:56.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:56 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:44:56.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:56 smithi138 ceph-mon[185886]: osdmap e533: 8 total, 8 up, 7 in 2023-10-01T09:44:56.793 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:44:56.813 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:44:56.833 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:44:56.852 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:44:56.862 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:56.874 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:44:56.897 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:44:56.900 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:56.916 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:44:56.918 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:44:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:44:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:44:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:44:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:44:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:44:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:44:56.919 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:44:56.960 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:56.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:56.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:57.066 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:57.104 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:57.652 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:57.653 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:57 smithi138 ceph-mon[192513]: pgmap v1561: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 7228/175383 objects degraded (4.121%); 3589/175383 objects misplaced (2.046%); 358 KiB/s, 5 objects/s recovering 2023-10-01T09:44:57.654 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2808518819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:57.654 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:57 smithi138 ceph-mon[185886]: pgmap v1561: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 7228/175383 objects degraded (4.121%); 3589/175383 objects misplaced (2.046%); 358 KiB/s, 5 objects/s recovering 2023-10-01T09:44:57.654 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2808518819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:57.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:57.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:57.755 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:57.780 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:57 smithi190 ceph-mon[160435]: pgmap v1561: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 7228/175383 objects degraded (4.121%); 3589/175383 objects misplaced (2.046%); 358 KiB/s, 5 objects/s recovering 2023-10-01T09:44:57.780 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2808518819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:44:57.783 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:57.835 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:57.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:57.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:57.907 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:44:57.909 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:44:57.942 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:57.984 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:58.094 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:44:58.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:58.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:44:58.200 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:58.225 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:58.271 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:58.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:58.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:58.377 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:58.406 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:58.456 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:58.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:58.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:58.546 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:44:58.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:58 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7228/175383 objects degraded (4.121%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:58.561 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:44:58.587 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:58.633 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:44:58.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:58.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:44:58.692 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:58 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7228/175383 objects degraded (4.121%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:58.692 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:58 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7228/175383 objects degraded (4.121%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:44:58.738 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:44:58.771 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:58.791 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1561,"stamp":"2023-10-01T09:44:56.335658+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3834189771,"num_objects":58461,"num_object_clones":0,"num_object_copies":175383,"num_objects_missing_on_primary":2128,"num_objects_missing":2128,"num_objects_degraded":7228,"num_objects_misplaced":3589,"num_objects_unfound":0,"num_objects_dirty":58461,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":109113,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8981,"num_bytes_recovered":575439887,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109077,"ondisk_log_size":109077,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16723276,"kb_used_data":11680596,"kb_used_omap":39,"kb_used_meta":5042584,"kb_avail":639492788,"statfs":{"total":671965249536,"available":654840614912,"internally_reserved":0,"allocated":11960930304,"data_stored":11951036700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5163606639},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-83296256,"num_objects":-1271,"num_object_clones":0,"num_object_copies":-3813,"num_objects_missing_on_primary":-50,"num_objects_missing":-50,"num_objects_degraded":-54,"num_objects_misplaced":-46,"num_objects_unfound":0,"num_objects_dirty":-1271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1271,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":3276800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.944951"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":309,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.397111+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:44:47.397111+0000","last_peered":"2023-10-01T09:44:47.397111+0000","last_clean":"2023-10-01T09:44:47.397111+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:44:47.397111+0000","last_undegraded":"2023-10-01T09:44:47.397111+0000","last_fullsized":"2023-10-01T09:44:47.397111+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":151,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.825686+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:44:37.825686+0000","last_peered":"2023-10-01T09:44:37.825686+0000","last_clean":"2023-10-01T09:44:37.825686+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:44:37.825686+0000","last_undegraded":"2023-10-01T09:44:37.825686+0000","last_fullsized":"2023-10-01T09:44:37.825686+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":532,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.825282+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:44:37.825282+0000","last_peered":"2023-10-01T09:44:37.825282+0000","last_clean":"2023-10-01T09:44:37.825282+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:44:37.825282+0000","last_undegraded":"2023-10-01T09:44:37.825282+0000","last_fullsized":"2023-10-01T09:44:37.825282+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":590,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.815813+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:44:37.815813+0000","last_peered":"2023-10-01T09:44:37.815813+0000","last_clean":"2023-10-01T09:44:37.815813+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:44:37.815813+0000","last_undegraded":"2023-10-01T09:44:37.815813+0000","last_fullsized":"2023-10-01T09:44:37.815813+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":471,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.815562+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:44:37.815562+0000","last_peered":"2023-10-01T09:44:37.815562+0000","last_clean":"2023-10-01T09:44:37.815562+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:44:37.815562+0000","last_undegraded":"2023-10-01T09:44:37.815562+0000","last_fullsized":"2023-10-01T09:44:37.815562+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":213,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.912981+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:44:37.912981+0000","last_peered":"2023-10-01T09:44:37.912981+0000","last_clean":"2023-10-01T09:44:37.912981+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:44:37.912981+0000","last_undegraded":"2023-10-01T09:44:37.912981+0000","last_fullsized":"2023-10-01T09:44:37.912981+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":589,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.913045+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:44:37.913045+0000","last_peered":"2023-10-01T09:44:37.913045+0000","last_clean":"2023-10-01T09:44:37.913045+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:44:37.913045+0000","last_undegraded":"2023-10-01T09:44:37.913045+0000","last_fullsized":"2023-10-01T09:44:37.913045+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":518,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.812215+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:44:37.812215+0000","last_peered":"2023-10-01T09:44:37.812215+0000","last_clean":"2023-10-01T09:44:37.812215+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:44:37.812215+0000","last_undegraded":"2023-10-01T09:44:37.812215+0000","last_fullsized":"2023-10-01T09:44:37.812215+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":244,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.816823+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:44:37.816823+0000","last_peered":"2023-10-01T09:44:37.816823+0000","last_clean":"2023-10-01T09:44:37.816823+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:44:37.816823+0000","last_undegraded":"2023-10-01T09:44:37.816823+0000","last_fullsized":"2023-10-01T09:44:37.816823+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":52,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.400646+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:44:47.400646+0000","last_peered":"2023-10-01T09:44:47.400646+0000","last_clean":"2023-10-01T09:44:47.400646+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:44:47.400646+0000","last_undegraded":"2023-10-01T09:44:47.400646+0000","last_fullsized":"2023-10-01T09:44:47.400646+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":58,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.401449+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:44:47.401449+0000","last_peered":"2023-10-01T09:44:47.401449+0000","last_clean":"2023-10-01T09:44:47.401449+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:44:47.401449+0000","last_undegraded":"2023-10-01T09:44:47.401449+0000","last_fullsized":"2023-10-01T09:44:47.401449+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":55,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.825309+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:44:37.825309+0000","last_peered":"2023-10-01T09:44:37.825309+0000","last_clean":"2023-10-01T09:44:37.825309+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:44:37.825309+0000","last_undegraded":"2023-10-01T09:44:37.825309+0000","last_fullsized":"2023-10-01T09:44:37.825309+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":434,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.400273+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:44:47.400273+0000","last_peered":"2023-10-01T09:44:47.400273+0000","last_clean":"2023-10-01T09:44:47.400273+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:44:47.400273+0000","last_undegraded":"2023-10-01T09:44:47.400273+0000","last_fullsized":"2023-10-01T09:44:47.400273+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":315,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.397347+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:44:47.397347+0000","last_peered":"2023-10-01T09:44:47.397347+0000","last_clean":"2023-10-01T09:44:47.397347+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:44:47.397347+0000","last_undegraded":"2023-10-01T09:44:47.397347+0000","last_fullsized":"2023-10-01T09:44:47.397347+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":651,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.401171+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:44:47.401171+0000","last_peered":"2023-10-01T09:44:47.401171+0000","last_clean":"2023-10-01T09:44:47.401171+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:44:47.401171+0000","last_undegraded":"2023-10-01T09:44:47.401171+0000","last_fullsized":"2023-10-01T09:44:47.401171+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":388,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.401504+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:44:47.401504+0000","last_peered":"2023-10-01T09:44:47.401504+0000","last_clean":"2023-10-01T09:44:47.401504+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:44:47.401504+0000","last_undegraded":"2023-10-01T09:44:47.401504+0000","last_fullsized":"2023-10-01T09:44:47.401504+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":61,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.401149+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:44:47.401149+0000","last_peered":"2023-10-01T09:44:47.401149+0000","last_clean":"2023-10-01T09:44:47.401149+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:44:47.401149+0000","last_undegraded":"2023-10-01T09:44:47.401149+0000","last_fullsized":"2023-10-01T09:44:47.401149+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":46,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.816554+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:44:37.816554+0000","last_peered":"2023-10-01T09:44:37.816554+0000","last_clean":"2023-10-01T09:44:37.816554+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:44:37.816554+0000","last_undegraded":"2023-10-01T09:44:37.816554+0000","last_fullsized":"2023-10-01T09:44:37.816554+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":544,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.401022+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:44:47.401022+0000","last_peered":"2023-10-01T09:44:47.401022+0000","last_clean":"2023-10-01T09:44:47.401022+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:44:47.401022+0000","last_undegraded":"2023-10-01T09:44:47.401022+0000","last_fullsized":"2023-10-01T09:44:47.401022+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":555,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:46.871222+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:44:46.871222+0000","last_peered":"2023-10-01T09:44:46.871222+0000","last_clean":"2023-10-01T09:44:46.871222+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:44:46.871222+0000","last_undegraded":"2023-10-01T09:44:46.871222+0000","last_fullsized":"2023-10-01T09:44:46.871222+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":411,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:41.870616+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:44:41.870616+0000","last_peered":"2023-10-01T09:44:41.870616+0000","last_clean":"2023-10-01T09:44:41.870616+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:44:41.870616+0000","last_undegraded":"2023-10-01T09:44:41.870616+0000","last_fullsized":"2023-10-01T09:44:41.870616+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":61,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.401835+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:44:47.401835+0000","last_peered":"2023-10-01T09:44:47.401835+0000","last_clean":"2023-10-01T09:44:47.401835+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:44:47.401835+0000","last_undegraded":"2023-10-01T09:44:47.401835+0000","last_fullsized":"2023-10-01T09:44:47.401835+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"532'6790","reported_seq":6855,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:51.853663+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:44:51.853663+0000","last_peered":"2023-10-01T09:44:51.853663+0000","last_clean":"2023-10-01T09:44:51.853663+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:44:51.853663+0000","last_undegraded":"2023-10-01T09:44:51.853663+0000","last_fullsized":"2023-10-01T09:44:51.853663+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6790,"log_dups_size":0,"ondisk_log_size":6790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236060672,"num_objects":3602,"num_object_clones":0,"num_object_copies":10806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6790,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":56,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.401608+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:44:47.401608+0000","last_peered":"2023-10-01T09:44:47.401608+0000","last_clean":"2023-10-01T09:44:47.401608+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:44:47.401608+0000","last_undegraded":"2023-10-01T09:44:47.401608+0000","last_fullsized":"2023-10-01T09:44:47.401608+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":252,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.816489+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:44:37.816489+0000","last_peered":"2023-10-01T09:44:37.816489+0000","last_clean":"2023-10-01T09:44:37.816489+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:44:37.816489+0000","last_undegraded":"2023-10-01T09:44:37.816489+0000","last_fullsized":"2023-10-01T09:44:37.816489+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"532'6760","reported_seq":11526,"reported_epoch":532,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:44:51.861569+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:44:51.861569+0000","last_peered":"2023-10-01T09:44:51.861569+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:44:51.861569+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6760,"log_dups_size":0,"ondisk_log_size":6760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238944256,"num_objects":3646,"num_object_clones":0,"num_object_copies":10938,"num_objects_missing_on_primary":2128,"num_objects_missing":2128,"num_objects_degraded":5705,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6760,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1558,"num_bytes_recovered":102105088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1449},{"shards":"0,4,6","objects":2128},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":527,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.401513+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:44:47.401513+0000","last_peered":"2023-10-01T09:44:47.401513+0000","last_clean":"2023-10-01T09:44:47.401513+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:44:47.401513+0000","last_undegraded":"2023-10-01T09:44:47.401513+0000","last_fullsized":"2023-10-01T09:44:47.401513+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":587,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.825236+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:44:37.825236+0000","last_peered":"2023-10-01T09:44:37.825236+0000","last_clean":"2023-10-01T09:44:37.825236+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:44:37.825236+0000","last_undegraded":"2023-10-01T09:44:37.825236+0000","last_fullsized":"2023-10-01T09:44:37.825236+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"532'6771","reported_seq":6839,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:51.606873+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:44:51.606873+0000","last_peered":"2023-10-01T09:44:51.606873+0000","last_clean":"2023-10-01T09:44:51.606873+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:44:51.606873+0000","last_undegraded":"2023-10-01T09:44:51.606873+0000","last_fullsized":"2023-10-01T09:44:51.606873+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6771,"log_dups_size":0,"ondisk_log_size":6771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239927296,"num_objects":3661,"num_object_clones":0,"num_object_copies":10983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6771,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":613,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.401570+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:44:47.401570+0000","last_peered":"2023-10-01T09:44:47.401570+0000","last_clean":"2023-10-01T09:44:47.401570+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:44:47.401570+0000","last_undegraded":"2023-10-01T09:44:47.401570+0000","last_fullsized":"2023-10-01T09:44:47.401570+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":408,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.825397+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:44:37.825397+0000","last_peered":"2023-10-01T09:44:37.825397+0000","last_clean":"2023-10-01T09:44:37.825397+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:44:37.825397+0000","last_undegraded":"2023-10-01T09:44:37.825397+0000","last_fullsized":"2023-10-01T09:44:37.825397+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"532'6728","reported_seq":13401,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:51.611102+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:44:51.611102+0000","last_peered":"2023-10-01T09:44:51.611102+0000","last_clean":"2023-10-01T09:44:51.611102+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:44:51.611102+0000","last_undegraded":"2023-10-01T09:44:51.611102+0000","last_fullsized":"2023-10-01T09:44:51.611102+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6728,"log_dups_size":0,"ondisk_log_size":6728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240779264,"num_objects":3674,"num_object_clones":0,"num_object_copies":11022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6728,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":252,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.816636+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:44:37.816636+0000","last_peered":"2023-10-01T09:44:37.816636+0000","last_clean":"2023-10-01T09:44:37.816636+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:44:37.816636+0000","last_undegraded":"2023-10-01T09:44:37.816636+0000","last_fullsized":"2023-10-01T09:44:37.816636+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":511,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:41.870700+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:44:41.870700+0000","last_peered":"2023-10-01T09:44:41.870700+0000","last_clean":"2023-10-01T09:44:41.870700+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:44:41.870700+0000","last_undegraded":"2023-10-01T09:44:41.870700+0000","last_fullsized":"2023-10-01T09:44:41.870700+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"532'6792","reported_seq":6860,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.001951+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:44:55.001951+0000","last_peered":"2023-10-01T09:44:55.001951+0000","last_clean":"2023-10-01T09:44:55.001951+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:44:55.001951+0000","last_undegraded":"2023-10-01T09:44:55.001951+0000","last_fullsized":"2023-10-01T09:44:55.001951+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6792,"log_dups_size":0,"ondisk_log_size":6792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243531776,"num_objects":3716,"num_object_clones":0,"num_object_copies":11148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6792,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":113,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.397484+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:44:47.397484+0000","last_peered":"2023-10-01T09:44:47.397484+0000","last_clean":"2023-10-01T09:44:47.397484+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:44:47.397484+0000","last_undegraded":"2023-10-01T09:44:47.397484+0000","last_fullsized":"2023-10-01T09:44:47.397484+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":187,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.400412+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:44:47.400412+0000","last_peered":"2023-10-01T09:44:47.400412+0000","last_clean":"2023-10-01T09:44:47.400412+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:44:47.400412+0000","last_undegraded":"2023-10-01T09:44:47.400412+0000","last_fullsized":"2023-10-01T09:44:47.400412+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"532'6715","reported_seq":6826,"reported_epoch":532,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:44:51.804859+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:44:51.804859+0000","last_peered":"2023-10-01T09:44:51.804859+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:44:51.804859+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6715,"log_dups_size":0,"ondisk_log_size":6715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239403008,"num_objects":3653,"num_object_clones":0,"num_object_copies":10959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1513,"num_objects_misplaced":3589,"num_objects_unfound":0,"num_objects_dirty":3653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6715,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":1245184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":3589},{"shards":"4,5","objects":-1449}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":431,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.397189+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:44:47.397189+0000","last_peered":"2023-10-01T09:44:47.397189+0000","last_clean":"2023-10-01T09:44:47.397189+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:44:47.397189+0000","last_undegraded":"2023-10-01T09:44:47.397189+0000","last_fullsized":"2023-10-01T09:44:47.397189+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":591,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.825729+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:44:37.825729+0000","last_peered":"2023-10-01T09:44:37.825729+0000","last_clean":"2023-10-01T09:44:37.825729+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:44:37.825729+0000","last_undegraded":"2023-10-01T09:44:37.825729+0000","last_fullsized":"2023-10-01T09:44:37.825729+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"532'6688","reported_seq":6762,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:52.149155+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:44:52.149155+0000","last_peered":"2023-10-01T09:44:52.149155+0000","last_clean":"2023-10-01T09:44:52.149155+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:44:52.149155+0000","last_undegraded":"2023-10-01T09:44:52.149155+0000","last_fullsized":"2023-10-01T09:44:52.149155+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6688,"log_dups_size":0,"ondisk_log_size":6688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241827840,"num_objects":3690,"num_object_clones":0,"num_object_copies":11070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6688,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":427,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.397029+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:44:47.397029+0000","last_peered":"2023-10-01T09:44:47.397029+0000","last_clean":"2023-10-01T09:44:47.397029+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:44:47.397029+0000","last_undegraded":"2023-10-01T09:44:47.397029+0000","last_fullsized":"2023-10-01T09:44:47.397029+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":268,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.400457+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:44:47.400457+0000","last_peered":"2023-10-01T09:44:47.400457+0000","last_clean":"2023-10-01T09:44:47.400457+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:44:47.400457+0000","last_undegraded":"2023-10-01T09:44:47.400457+0000","last_fullsized":"2023-10-01T09:44:47.400457+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"533'6859","reported_seq":6925,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:56.096673+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:44:56.096673+0000","last_peered":"2023-10-01T09:44:56.096673+0000","last_clean":"2023-10-01T09:44:56.096673+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:44:56.096673+0000","last_undegraded":"2023-10-01T09:44:56.096673+0000","last_fullsized":"2023-10-01T09:44:56.096673+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6859,"log_dups_size":0,"ondisk_log_size":6859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238485504,"num_objects":3639,"num_object_clones":0,"num_object_copies":10917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6859,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":395,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.827933+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:44:37.827933+0000","last_peered":"2023-10-01T09:44:37.827933+0000","last_clean":"2023-10-01T09:44:37.827933+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:44:37.827933+0000","last_undegraded":"2023-10-01T09:44:37.827933+0000","last_fullsized":"2023-10-01T09:44:37.827933+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":442,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.825945+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:44:37.825945+0000","last_peered":"2023-10-01T09:44:37.825945+0000","last_clean":"2023-10-01T09:44:37.825945+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:44:37.825945+0000","last_undegraded":"2023-10-01T09:44:37.825945+0000","last_fullsized":"2023-10-01T09:44:37.825945+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"533'6668","reported_seq":6734,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:56.209968+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:44:56.209968+0000","last_peered":"2023-10-01T09:44:56.209968+0000","last_clean":"2023-10-01T09:44:56.209968+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:44:56.209968+0000","last_undegraded":"2023-10-01T09:44:56.209968+0000","last_fullsized":"2023-10-01T09:44:56.209968+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6668,"log_dups_size":0,"ondisk_log_size":6668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230031360,"num_objects":3510,"num_object_clones":0,"num_object_copies":10530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6668,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":926,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.401331+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:44:47.401331+0000","last_peered":"2023-10-01T09:44:47.401331+0000","last_clean":"2023-10-01T09:44:47.401331+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:44:47.401331+0000","last_undegraded":"2023-10-01T09:44:47.401331+0000","last_fullsized":"2023-10-01T09:44:47.401331+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":262,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.816698+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:44:37.816698+0000","last_peered":"2023-10-01T09:44:37.816698+0000","last_clean":"2023-10-01T09:44:37.816698+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:44:37.816698+0000","last_undegraded":"2023-10-01T09:44:37.816698+0000","last_fullsized":"2023-10-01T09:44:37.816698+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":56,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.410025+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:44:47.410025+0000","last_peered":"2023-10-01T09:44:47.410025+0000","last_clean":"2023-10-01T09:44:47.410025+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:44:47.410025+0000","last_undegraded":"2023-10-01T09:44:47.410025+0000","last_fullsized":"2023-10-01T09:44:47.410025+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"532'6674","reported_seq":6742,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:51.758315+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:44:51.758315+0000","last_peered":"2023-10-01T09:44:51.758315+0000","last_clean":"2023-10-01T09:44:51.758315+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:44:51.758315+0000","last_undegraded":"2023-10-01T09:44:51.758315+0000","last_fullsized":"2023-10-01T09:44:51.758315+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6674,"log_dups_size":0,"ondisk_log_size":6674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242221056,"num_objects":3696,"num_object_clones":0,"num_object_copies":11088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6674,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1197,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.816972+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:44:37.816972+0000","last_peered":"2023-10-01T09:44:37.816972+0000","last_clean":"2023-10-01T09:44:37.816972+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:44:37.816972+0000","last_undegraded":"2023-10-01T09:44:37.816972+0000","last_fullsized":"2023-10-01T09:44:37.816972+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":408,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.397435+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:44:47.397435+0000","last_peered":"2023-10-01T09:44:47.397435+0000","last_clean":"2023-10-01T09:44:47.397435+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:44:47.397435+0000","last_undegraded":"2023-10-01T09:44:47.397435+0000","last_fullsized":"2023-10-01T09:44:47.397435+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":235,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.816771+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:44:37.816771+0000","last_peered":"2023-10-01T09:44:37.816771+0000","last_clean":"2023-10-01T09:44:37.816771+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:44:37.816771+0000","last_undegraded":"2023-10-01T09:44:37.816771+0000","last_fullsized":"2023-10-01T09:44:37.816771+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"532'6748","reported_seq":14034,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:51.506481+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:44:51.506481+0000","last_peered":"2023-10-01T09:44:51.506481+0000","last_clean":"2023-10-01T09:44:51.506481+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:44:51.506481+0000","last_undegraded":"2023-10-01T09:44:51.506481+0000","last_fullsized":"2023-10-01T09:44:51.506481+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6748,"log_dups_size":0,"ondisk_log_size":6748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237764608,"num_objects":3628,"num_object_clones":0,"num_object_copies":10884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6748,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":877,"reported_epoch":531,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:44:37.815328+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:44:37.815328+0000","last_peered":"2023-10-01T09:44:37.815328+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:44:37.815328+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:44:37.815328+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":595,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.827928+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:44:37.827928+0000","last_peered":"2023-10-01T09:44:37.827928+0000","last_clean":"2023-10-01T09:44:37.827928+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:44:37.827928+0000","last_undegraded":"2023-10-01T09:44:37.827928+0000","last_fullsized":"2023-10-01T09:44:37.827928+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":55,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.825618+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:44:37.825618+0000","last_peered":"2023-10-01T09:44:37.825618+0000","last_clean":"2023-10-01T09:44:37.825618+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:44:37.825618+0000","last_undegraded":"2023-10-01T09:44:37.825618+0000","last_fullsized":"2023-10-01T09:44:37.825618+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"532'6668","reported_seq":6736,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.040616+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:44:55.040616+0000","last_peered":"2023-10-01T09:44:55.040616+0000","last_clean":"2023-10-01T09:44:55.040616+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:44:55.040616+0000","last_undegraded":"2023-10-01T09:44:55.040616+0000","last_fullsized":"2023-10-01T09:44:55.040616+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6668,"log_dups_size":0,"ondisk_log_size":6668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236191744,"num_objects":3604,"num_object_clones":0,"num_object_copies":10812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6668,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1648,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.820010+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:44:37.820010+0000","last_peered":"2023-10-01T09:44:37.820010+0000","last_clean":"2023-10-01T09:44:37.820010+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:44:37.820010+0000","last_undegraded":"2023-10-01T09:44:37.820010+0000","last_fullsized":"2023-10-01T09:44:37.820010+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":563,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.400721+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:44:47.400721+0000","last_peered":"2023-10-01T09:44:47.400721+0000","last_clean":"2023-10-01T09:44:47.400721+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:44:47.400721+0000","last_undegraded":"2023-10-01T09:44:47.400721+0000","last_fullsized":"2023-10-01T09:44:47.400721+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":340,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.817038+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:44:37.817038+0000","last_peered":"2023-10-01T09:44:37.817038+0000","last_clean":"2023-10-01T09:44:37.817038+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:44:37.817038+0000","last_undegraded":"2023-10-01T09:44:37.817038+0000","last_fullsized":"2023-10-01T09:44:37.817038+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":331,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.399895+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:44:47.399895+0000","last_peered":"2023-10-01T09:44:47.399895+0000","last_clean":"2023-10-01T09:44:47.399895+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:44:47.399895+0000","last_undegraded":"2023-10-01T09:44:47.399895+0000","last_fullsized":"2023-10-01T09:44:47.399895+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"533'6839","reported_seq":6905,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:56.175144+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:44:56.175144+0000","last_peered":"2023-10-01T09:44:56.175144+0000","last_clean":"2023-10-01T09:44:56.175144+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:44:56.175144+0000","last_undegraded":"2023-10-01T09:44:56.175144+0000","last_fullsized":"2023-10-01T09:44:56.175144+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6839,"log_dups_size":0,"ondisk_log_size":6839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242024448,"num_objects":3693,"num_object_clones":0,"num_object_copies":11079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6839,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":740,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.400817+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:44:47.400817+0000","last_peered":"2023-10-01T09:44:47.400817+0000","last_clean":"2023-10-01T09:44:47.400817+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:44:47.400817+0000","last_undegraded":"2023-10-01T09:44:47.400817+0000","last_fullsized":"2023-10-01T09:44:47.400817+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":572,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.815579+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:44:37.815579+0000","last_peered":"2023-10-01T09:44:37.815579+0000","last_clean":"2023-10-01T09:44:37.815579+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:44:37.815579+0000","last_undegraded":"2023-10-01T09:44:37.815579+0000","last_fullsized":"2023-10-01T09:44:37.815579+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":571,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.815706+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:44:37.815706+0000","last_peered":"2023-10-01T09:44:37.815706+0000","last_clean":"2023-10-01T09:44:37.815706+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:44:37.815706+0000","last_undegraded":"2023-10-01T09:44:37.815706+0000","last_fullsized":"2023-10-01T09:44:37.815706+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"532'6736","reported_seq":6805,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:51.718841+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:44:51.718841+0000","last_peered":"2023-10-01T09:44:51.718841+0000","last_clean":"2023-10-01T09:44:51.718841+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:44:51.718841+0000","last_undegraded":"2023-10-01T09:44:51.718841+0000","last_fullsized":"2023-10-01T09:44:51.718841+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6736,"log_dups_size":0,"ondisk_log_size":6736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243990552,"num_objects":3724,"num_object_clones":0,"num_object_copies":11172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3724,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6736,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":971,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.401657+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:44:47.401657+0000","last_peered":"2023-10-01T09:44:47.401657+0000","last_clean":"2023-10-01T09:44:47.401657+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:44:47.401657+0000","last_undegraded":"2023-10-01T09:44:47.401657+0000","last_fullsized":"2023-10-01T09:44:47.401657+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":589,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.816706+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:44:37.816706+0000","last_peered":"2023-10-01T09:44:37.816706+0000","last_clean":"2023-10-01T09:44:37.816706+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:44:37.816706+0000","last_undegraded":"2023-10-01T09:44:37.816706+0000","last_fullsized":"2023-10-01T09:44:37.816706+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":442,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.820087+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:44:37.820087+0000","last_peered":"2023-10-01T09:44:37.820087+0000","last_clean":"2023-10-01T09:44:37.820087+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:44:37.820087+0000","last_undegraded":"2023-10-01T09:44:37.820087+0000","last_fullsized":"2023-10-01T09:44:37.820087+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"533'6756","reported_seq":6830,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:56.137565+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:44:56.137565+0000","last_peered":"2023-10-01T09:44:56.137565+0000","last_clean":"2023-10-01T09:44:56.137565+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:44:56.137565+0000","last_undegraded":"2023-10-01T09:44:56.137565+0000","last_fullsized":"2023-10-01T09:44:56.137565+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6756,"log_dups_size":0,"ondisk_log_size":6756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235929600,"num_objects":3600,"num_object_clones":0,"num_object_copies":10800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6756,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":472,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.401923+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:44:47.401923+0000","last_peered":"2023-10-01T09:44:47.401923+0000","last_clean":"2023-10-01T09:44:47.401923+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:44:47.401923+0000","last_undegraded":"2023-10-01T09:44:47.401923+0000","last_fullsized":"2023-10-01T09:44:47.401923+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1165,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:40.261692+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:44:40.261692+0000","last_peered":"2023-10-01T09:44:40.261692+0000","last_clean":"2023-10-01T09:44:40.261692+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:44:40.261692+0000","last_undegraded":"2023-10-01T09:44:40.261692+0000","last_fullsized":"2023-10-01T09:44:40.261692+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":61,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.400646+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:44:47.400646+0000","last_peered":"2023-10-01T09:44:47.400646+0000","last_clean":"2023-10-01T09:44:47.400646+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:44:47.400646+0000","last_undegraded":"2023-10-01T09:44:47.400646+0000","last_fullsized":"2023-10-01T09:44:47.400646+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"533'6815","reported_seq":6889,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:56.195102+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:44:56.195102+0000","last_peered":"2023-10-01T09:44:56.195102+0000","last_clean":"2023-10-01T09:44:56.195102+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:44:56.195102+0000","last_undegraded":"2023-10-01T09:44:56.195102+0000","last_fullsized":"2023-10-01T09:44:56.195102+0000","mapping_epoch":479,"log_start":"480'100","ondisk_log_start":"480'100","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6715,"log_dups_size":0,"ondisk_log_size":6715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242286592,"num_objects":3697,"num_object_clones":0,"num_object_copies":11091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6815,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":65,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.401729+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:44:47.401729+0000","last_peered":"2023-10-01T09:44:47.401729+0000","last_clean":"2023-10-01T09:44:47.401729+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:44:47.401729+0000","last_undegraded":"2023-10-01T09:44:47.401729+0000","last_fullsized":"2023-10-01T09:44:47.401729+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":549,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.815937+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:44:37.815937+0000","last_peered":"2023-10-01T09:44:37.815937+0000","last_clean":"2023-10-01T09:44:37.815937+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:44:37.815937+0000","last_undegraded":"2023-10-01T09:44:37.815937+0000","last_fullsized":"2023-10-01T09:44:37.815937+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":255,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.816589+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:44:37.816589+0000","last_peered":"2023-10-01T09:44:37.816589+0000","last_clean":"2023-10-01T09:44:37.816589+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:44:37.816589+0000","last_undegraded":"2023-10-01T09:44:37.816589+0000","last_fullsized":"2023-10-01T09:44:37.816589+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":498,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.815903+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:44:37.815903+0000","last_peered":"2023-10-01T09:44:37.815903+0000","last_clean":"2023-10-01T09:44:37.815903+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:44:37.815903+0000","last_undegraded":"2023-10-01T09:44:37.815903+0000","last_fullsized":"2023-10-01T09:44:37.815903+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":612,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.825657+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:44:37.825657+0000","last_peered":"2023-10-01T09:44:37.825657+0000","last_clean":"2023-10-01T09:44:37.825657+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:44:37.825657+0000","last_undegraded":"2023-10-01T09:44:37.825657+0000","last_fullsized":"2023-10-01T09:44:37.825657+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":267,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.816554+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:44:37.816554+0000","last_peered":"2023-10-01T09:44:37.816554+0000","last_clean":"2023-10-01T09:44:37.816554+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:44:37.816554+0000","last_undegraded":"2023-10-01T09:44:37.816554+0000","last_fullsized":"2023-10-01T09:44:37.816554+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":55,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.825280+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:44:37.825280+0000","last_peered":"2023-10-01T09:44:37.825280+0000","last_clean":"2023-10-01T09:44:37.825280+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:44:37.825280+0000","last_undegraded":"2023-10-01T09:44:37.825280+0000","last_fullsized":"2023-10-01T09:44:37.825280+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":267,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.816789+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:44:37.816789+0000","last_peered":"2023-10-01T09:44:37.816789+0000","last_clean":"2023-10-01T09:44:37.816789+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:44:37.816789+0000","last_undegraded":"2023-10-01T09:44:37.816789+0000","last_fullsized":"2023-10-01T09:44:37.816789+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":444,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.815952+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:44:37.815952+0000","last_peered":"2023-10-01T09:44:37.815952+0000","last_clean":"2023-10-01T09:44:37.815952+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:44:37.815952+0000","last_undegraded":"2023-10-01T09:44:37.815952+0000","last_fullsized":"2023-10-01T09:44:37.815952+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":73,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.825837+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:44:37.825837+0000","last_peered":"2023-10-01T09:44:37.825837+0000","last_clean":"2023-10-01T09:44:37.825837+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:44:37.825837+0000","last_undegraded":"2023-10-01T09:44:37.825837+0000","last_fullsized":"2023-10-01T09:44:37.825837+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":489,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.815567+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:44:37.815567+0000","last_peered":"2023-10-01T09:44:37.815567+0000","last_clean":"2023-10-01T09:44:37.815567+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:44:37.815567+0000","last_undegraded":"2023-10-01T09:44:37.815567+0000","last_fullsized":"2023-10-01T09:44:37.815567+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":689,"reported_epoch":531,"state":"active+clean","last_fresh":"2023-10-01T09:44:37.816046+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:44:37.816046+0000","last_peered":"2023-10-01T09:44:37.816046+0000","last_clean":"2023-10-01T09:44:37.816046+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:44:37.816046+0000","last_undegraded":"2023-10-01T09:44:37.816046+0000","last_fullsized":"2023-10-01T09:44:37.816046+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":384,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.396919+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:44:47.396919+0000","last_peered":"2023-10-01T09:44:47.396919+0000","last_clean":"2023-10-01T09:44:47.396919+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:44:47.396919+0000","last_undegraded":"2023-10-01T09:44:47.396919+0000","last_fullsized":"2023-10-01T09:44:47.396919+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3829399576,"num_objects":58433,"num_object_clones":0,"num_object_copies":175299,"num_objects_missing_on_primary":2128,"num_objects_missing":2128,"num_objects_degraded":7218,"num_objects_misplaced":3589,"num_objects_unfound":0,"num_objects_dirty":58433,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":108007,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8470,"num_bytes_recovered":555089920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11887783936,"data_stored":11887771720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107907,"ondisk_log_size":107907,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430732,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2071096,"kb_used_data":1167984,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":91674056,"statfs":{"total":95995035648,"available":93874233344,"internally_reserved":0,"allocated":1196015616,"data_stored":1195075644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":1.1359999999999999,"15min":2.657},"min":{"1min":0.27300000000000002,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":1.484,"5min":10.391999999999999,"15min":108.90000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.39900000000000002,"5min":1.26,"15min":2.8090000000000002},"min":{"1min":0.219,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.67200000000000004,"5min":12.026,"15min":109.187},"last":0.53600000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":4.2910000000000004,"15min":5.2409999999999997},"min":{"1min":0.28999999999999998,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":1.4470000000000001,"5min":229.161,"15min":229.161},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.42999999999999999,"5min":4.2759999999999998,"15min":5.2510000000000003},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":0.96499999999999997,"5min":230.126,"15min":230.126},"last":0.42899999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:44:54 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.81399999999999995,"15min":1.6519999999999999},"min":{"1min":0.30399999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.4300000000000002,"5min":5.3090000000000002,"15min":14.901999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.76100000000000001,"15min":1.6299999999999999},"min":{"1min":0.33200000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4630000000000001,"5min":5.6929999999999996,"15min":14.682},"last":0.45600000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.96899999999999997,"15min":2.569},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.018,"5min":6.0069999999999997,"15min":80.352999999999994},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":1.1359999999999999,"15min":2.5510000000000002},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.026,"5min":14.528,"15min":80.364000000000004},"last":0.443}]},{"osd":4,"last update":"Sun Oct 1 09:43:56 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":1.046,"15min":1.0429999999999999},"min":{"1min":0.30199999999999999,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.508,"5min":7.3399999999999999,"15min":9.6020000000000003},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.0289999999999999,"15min":1.0189999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.28000000000000003},"max":{"1min":0.84399999999999997,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.51900000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":2.177,"15min":1.5800000000000001},"min":{"1min":0.34399999999999997,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":0.871,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.61299999999999999,"5min":2.1789999999999998,"15min":1.6619999999999999},"min":{"1min":0.307,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.3319999999999999,"5min":135.339,"15min":135.339},"last":0.40000000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":1.1639999999999999,"15min":1.1359999999999999},"min":{"1min":0.38,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":0.998,"5min":11.58,"15min":11.853999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.94199999999999995,"15min":1.097},"min":{"1min":0.38300000000000001,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":2.298,"5min":13.021000000000001,"15min":14.584},"last":0.503}]}]},{"osd":6,"up_from":294,"seq":1262720385242,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1989548,"kb_used_data":1656036,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":91755604,"statfs":{"total":95995035648,"available":93957738496,"internally_reserved":0,"allocated":1695780864,"data_stored":1694845167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":2.0630000000000002,"15min":4.0090000000000003},"min":{"1min":0.247,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":3.2029999999999998,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.41399999999999998,"5min":2.1739999999999999,"15min":4.0730000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.221},"max":{"1min":0.63900000000000001,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.47999999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":3.1379999999999999,"15min":4.8600000000000003},"min":{"1min":0.29199999999999998,"5min":0.254,"15min":0.24199999999999999},"max":{"1min":0.58899999999999997,"5min":168.947,"15min":168.947},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.56000000000000005,"5min":3.141,"15min":4.8200000000000003},"min":{"1min":0.32500000000000001,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":2.8290000000000002,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.441}]},{"osd":2,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.78400000000000003,"15min":1.5620000000000001},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":0.65700000000000003,"5min":8.657,"15min":10.92},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.88600000000000001,"15min":1.5629999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":2.863,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.46200000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.96499999999999997,"15min":2.101},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.3149999999999999,"5min":9.0180000000000007,"15min":89.753},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.439,"5min":1.097,"15min":2.0179999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.61199999999999999,"5min":10.191000000000001,"15min":89.849999999999994},"last":0.42199999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.5049999999999999,"15min":1.0740000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.7749999999999999,"5min":63.134,"15min":63.134},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.478,"15min":1.075},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":3.2519999999999998,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.33600000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:44:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.246,"15min":1.073},"min":{"1min":0.28799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":3.2890000000000001,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.624,"5min":1.3260000000000001,"15min":1.1859999999999999},"min":{"1min":0.29899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.887,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.32500000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.73999999999999999,"15min":1.2729999999999999},"min":{"1min":0.32700000000000001,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":3.339,"5min":3.8279999999999998,"15min":94.245999999999995},"last":0.434},{"interface":"front","average":{"1min":0.496,"5min":0.96999999999999997,"15min":1.3640000000000001},"min":{"1min":0.374,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":0.68600000000000005,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.51000000000000001}]}]},{"osd":1,"up_from":94,"seq":403726926219,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1524072,"kb_used_data":714080,"kb_used_omap":7,"kb_used_meta":809976,"kb_avail":92221080,"statfs":{"total":95995035648,"available":94434385920,"internally_reserved":0,"allocated":731217920,"data_stored":729172759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":829415847},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":1.7789999999999999,"15min":4.0039999999999996},"min":{"1min":0.186,"5min":0.186,"15min":0.182},"max":{"1min":1.2889999999999999,"5min":46.692999999999998,"15min":154.21299999999999},"last":0.318},{"interface":"front","average":{"1min":0.42099999999999999,"5min":1.99,"15min":3.5329999999999999},"min":{"1min":0.223,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":1.702,"5min":46.701000000000001,"15min":95.236999999999995},"last":0.26600000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:44:39 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.42399999999999999,"15min":2.4889999999999999},"min":{"1min":0.23300000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":1.3240000000000001,"5min":2.8740000000000001,"15min":56.573},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.61199999999999999,"15min":3.1829999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":2.2200000000000002,"5min":6.9710000000000001,"15min":154.197},"last":0.307}]},{"osd":3,"last update":"Sun Oct 1 09:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":1.2769999999999999,"15min":3.7909999999999999},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":1.7889999999999999,"5min":18.503,"15min":200.56800000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.2330000000000001,"15min":4.5039999999999996},"min":{"1min":0.253,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":1.9370000000000001,"5min":16.515000000000001,"15min":200.63800000000001},"last":0.41199999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.147,"15min":2.2530000000000001},"min":{"1min":0.35499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.0550000000000002,"5min":19.539000000000001,"15min":154.101},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.46899999999999997,"5min":1.0029999999999999,"15min":2.399},"min":{"1min":0.29499999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.59799999999999998,"5min":10.084,"15min":154.56999999999999},"last":0.53000000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:44:39 2023","interfaces":[{"interface":"back","average":{"1min":3.593,"5min":1.272,"15min":2.2759999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":59.561,"5min":59.561,"15min":154.22999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":3.5979999999999999,"5min":1.24,"15min":2.5680000000000001},"min":{"1min":0.35699999999999998,"5min":0.309,"15min":0.307},"max":{"1min":59.536000000000001,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.59899999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.236,"15min":2.5539999999999998},"min":{"1min":0.34300000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.4399999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.378},{"interface":"front","average":{"1min":0.81399999999999995,"5min":1.335,"15min":2.4119999999999999},"min":{"1min":0.34699999999999998,"5min":0.34599999999999997,"15min":0.32700000000000001},"max":{"1min":2.5630000000000002,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.55600000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.3109999999999999,"15min":2.5840000000000001},"min":{"1min":0.38400000000000001,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":2.8170000000000002,"5min":19.431999999999999,"15min":154.20699999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.77800000000000002,"5min":1.2330000000000001,"15min":2.6379999999999999},"min":{"1min":0.45200000000000001,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":2.8290000000000002,"5min":10.111000000000001,"15min":154.51599999999999},"last":0.57499999999999996}]}]},{"osd":0,"up_from":89,"seq":382252089753,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3928340,"kb_used_data":3023244,"kb_used_omap":8,"kb_used_meta":905079,"kb_avail":89816812,"statfs":{"total":95995035648,"available":91972415488,"internally_reserved":0,"allocated":3095801856,"data_stored":3093781426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":926801858},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:44:16 2023","interfaces":[{"interface":"back","average":{"1min":0.317,"5min":1.1839999999999999,"15min":4.1120000000000001},"min":{"1min":0.21199999999999999,"5min":0.17799999999999999,"15min":0.17399999999999999},"max":{"1min":0.46300000000000002,"5min":52.316000000000003,"15min":216.821},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.317,"5min":2.7919999999999998,"15min":4.9809999999999999},"min":{"1min":0.24399999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.435,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.26800000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.69999999999999996,"15min":2.0339999999999998},"min":{"1min":0.29299999999999998,"5min":0.246,"15min":0.20100000000000001},"max":{"1min":4.2430000000000003,"5min":4.9459999999999997,"15min":19.788},"last":0.313},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.70799999999999996,"15min":2.1200000000000001},"min":{"1min":0.30399999999999999,"5min":0.23000000000000001,"15min":0.189},"max":{"1min":3.4940000000000002,"5min":4.5359999999999996,"15min":25.844999999999999},"last":0.32600000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:44:16 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.90200000000000002,"15min":3.46},"min":{"1min":0.26900000000000002,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.59199999999999997,"5min":8.2889999999999997,"15min":222.16499999999999},"last":0.438},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.0369999999999999,"15min":3.375},"min":{"1min":0.26900000000000002,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":3.4119999999999999,"5min":9.7539999999999996,"15min":222.227},"last":0.29999999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.5449999999999999,"15min":2.1339999999999999},"min":{"1min":0.28799999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":3.375,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.378},{"interface":"front","average":{"1min":0.58399999999999996,"5min":1.4550000000000001,"15min":2.1309999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":3.5049999999999999,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.33700000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.76300000000000001,"15min":1.8069999999999999},"min":{"1min":0.33200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.476,"5min":5.8769999999999998,"15min":16.808},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.90300000000000002,"15min":1.708},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":4.2729999999999997,"5min":5.0629999999999997,"15min":17.507000000000001},"last":0.52200000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:44:27 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.95599999999999996,"15min":1.732},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.81399999999999995,"5min":6.5,"15min":17.535},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.504,"5min":1.0920000000000001,"15min":1.8300000000000001},"min":{"1min":0.38700000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":0.86199999999999999,"5min":10.446,"15min":15.651999999999999},"last":0.48199999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.95499999999999996,"15min":2.048},"min":{"1min":0.379,"5min":0.374,"15min":0.35699999999999998},"max":{"1min":0.78600000000000003,"5min":4.976,"15min":48.895000000000003},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.093,"15min":2.206},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.35099999999999998},"max":{"1min":3.4319999999999999,"5min":9.4969999999999999,"15min":48.274000000000001},"last":0.53700000000000003}]}]},{"osd":2,"up_from":0,"seq":425201762693,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599166,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1558176,"kb_used_data":721944,"kb_used_omap":3,"kb_used_meta":836220,"kb_avail":92186976,"statfs":{"total":95995035648,"available":94399463424,"internally_reserved":0,"allocated":739270656,"data_stored":737175016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":856290023},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":3.2000000000000002,"15min":4.8920000000000003},"min":{"1min":0.20100000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":1.3580000000000001,"5min":92.649000000000001,"15min":137.34399999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.36399999999999999,"5min":3.194,"15min":4.9740000000000002},"min":{"1min":0.21199999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":1.2549999999999999,"5min":92.284000000000006,"15min":137.73400000000001},"last":0.28399999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":1.8819999999999999,"15min":5.1500000000000004},"min":{"1min":0.25,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":1.1599999999999999,"5min":61.439999999999998,"15min":162.22999999999999},"last":0.252},{"interface":"front","average":{"1min":0.39600000000000002,"5min":1.788,"15min":5.181},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.4119999999999999,"5min":62.723999999999997,"15min":162.70699999999999},"last":0.41099999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.0780000000000001,"15min":1.748},"min":{"1min":0.27800000000000002,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":2.4710000000000001,"5min":14.254,"15min":14.254},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.66300000000000003,"15min":1.798},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":4.5049999999999999,"5min":4.5049999999999999,"15min":15.523},"last":0.38200000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.72,"15min":1.8080000000000001},"min":{"1min":0.35199999999999998,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":1.444,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.887,"15min":2.0630000000000002},"min":{"1min":0.29499999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.2250000000000001,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.40300000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:44:32 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":1.0780000000000001,"15min":2.3530000000000002},"min":{"1min":0.32000000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.196,"5min":9.4320000000000004,"15min":28.117999999999999},"last":0.314},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.073,"15min":2.4609999999999999},"min":{"1min":0.33600000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.92300000000000004,"5min":10.798999999999999,"15min":28.34},"last":0.39200000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:44:28 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.7350000000000001,"15min":1.8999999999999999},"min":{"1min":0.37,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":1.1000000000000001,"5min":73.887,"15min":73.887},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.952,"15min":1.974},"min":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":1.294,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.46899999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":1.268,"15min":1.845},"min":{"1min":0.42299999999999999,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":2.573,"5min":11.659000000000001,"15min":13.085000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.0740000000000001,"15min":1.726},"min":{"1min":0.39700000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":1.472,"5min":8.3460000000000001,"15min":13.191000000000001},"last":0.57199999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019417,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3776628,"kb_used_data":2886444,"kb_used_omap":0,"kb_used_meta":890176,"kb_avail":89968524,"statfs":{"total":95995035648,"available":92127768576,"internally_reserved":0,"allocated":2955718656,"data_stored":2954782190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911540224},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:43:55 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":2.3570000000000002,"15min":4.25},"min":{"1min":0.251,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.47299999999999998,"5min":79.837000000000003,"15min":155.315},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.44900000000000001,"5min":2.4300000000000002,"15min":4.7039999999999997},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.251},"max":{"1min":0.63700000000000001,"5min":78.983000000000004,"15min":155.334},"last":0.29399999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":4.5709999999999997,"15min":6.4699999999999998},"min":{"1min":0.28899999999999998,"5min":0.252,"15min":0.224},"max":{"1min":0.52900000000000003,"5min":126.742,"15min":190.61199999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.40200000000000002,"5min":4.5369999999999999,"15min":7.1600000000000001},"min":{"1min":0.27600000000000002,"5min":0.26700000000000002,"15min":0.25600000000000001},"max":{"1min":0.59999999999999998,"5min":126.791,"15min":253.273},"last":0.44500000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:44:50 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.77200000000000002,"15min":2.375},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.2329999999999997,"5min":5.702,"15min":170.45400000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.755,"15min":2.133},"min":{"1min":0.33100000000000002,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":0.71099999999999997,"5min":5.6150000000000002,"15min":58.771000000000001},"last":0.40899999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:43:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.96499999999999997,"15min":2.3079999999999998},"min":{"1min":0.28100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.1299999999999999,"5min":7.5830000000000002,"15min":140.881},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.876,"15min":2.4830000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.311},"max":{"1min":0.51500000000000001,"5min":6.8719999999999999,"15min":140.911},"last":0.48099999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":1.929,"15min":1.52},"min":{"1min":0.33800000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":1.7509999999999999,"5min":110.003,"15min":110.003},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.84499999999999997,"5min":1.9239999999999999,"15min":1.405},"min":{"1min":0.38400000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.2909999999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.50600000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:44:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.78200000000000003,"15min":0.93000000000000005},"min":{"1min":0.33500000000000002,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":0.73599999999999999,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.85099999999999998,"15min":1.0589999999999999},"min":{"1min":0.40600000000000003,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":0.73599999999999999,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.438}]},{"osd":7,"last update":"Sun Oct 1 09:44:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.65000000000000002,"15min":1.1399999999999999},"min":{"1min":0.36399999999999999,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":3.0830000000000002,"5min":3.8580000000000001,"15min":38.034999999999997},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.91600000000000004,"15min":1.4350000000000001},"min":{"1min":0.39900000000000002,"5min":0.36699999999999999,"15min":0.308},"max":{"1min":0.80000000000000004,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.58099999999999996}]}]},{"osd":5,"up_from":379,"seq":1627792605323,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1875416,"kb_used_data":1510864,"kb_used_omap":13,"kb_used_meta":364530,"kb_avail":91869736,"statfs":{"total":95995035648,"available":94074609664,"internally_reserved":0,"allocated":1547124736,"data_stored":1546204498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":373279218},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":1.244,"15min":2.2229999999999999},"min":{"1min":0.24299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":27.584,"5min":27.584,"15min":84.340999999999994},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.9430000000000001,"5min":1.1200000000000001,"15min":2.1280000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23300000000000001},"max":{"1min":27.669,"5min":27.669,"15min":84.356999999999999},"last":0.47999999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":3.4430000000000001,"15min":5.6900000000000004},"min":{"1min":0.28699999999999998,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":3.7799999999999998,"5min":149.94900000000001,"15min":295.71300000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.71199999999999997,"5min":3.7149999999999999,"15min":5.6859999999999999},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":3.9580000000000002,"5min":149.41999999999999,"15min":299.59399999999999},"last":0.42299999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.83799999999999997,"15min":1.552},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.7029999999999998,"5min":5.3470000000000004,"15min":14.199999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.84599999999999997,"5min":1.056,"15min":1.5309999999999999},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":3.944,"5min":6.984,"15min":14.430999999999999},"last":0.504}]},{"osd":3,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.876,"15min":1.46},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.0209999999999999,"5min":10.022,"15min":14.004},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.91300000000000003,"15min":1.5940000000000001},"min":{"1min":0.34300000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.8610000000000002,"5min":5.4400000000000004,"15min":13.991},"last":0.45700000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":2.085,"15min":1.417},"min":{"1min":0.28599999999999998,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":4.0839999999999996,"5min":122.971,"15min":122.971},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.86299999999999999,"5min":2.1949999999999998,"15min":1.6699999999999999},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.0810000000000004,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.376}]},{"osd":6,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":1.0169999999999999,"15min":1.0269999999999999},"min":{"1min":0.32500000000000001,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":4.3300000000000001,"5min":10.500999999999999,"15min":12.545},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.97199999999999998,"15min":1.264},"min":{"1min":0.314,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":4.0599999999999996,"5min":6.4729999999999999,"15min":14.193},"last":0.434}]},{"osd":7,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.0529999999999999,"15min":1.232},"min":{"1min":0.378,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.0999999999999996,"5min":5.758,"15min":10.813000000000001},"last":0.443},{"interface":"front","average":{"1min":0.996,"5min":1.1339999999999999,"15min":1.0840000000000001},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":4.2000000000000002,"5min":10.039,"15min":12.500999999999999},"last":0.495}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3079344128,"data_stored":3079340056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":719847424,"data_stored":719847424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":733085696,"data_stored":733085696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2947682304,"data_stored":2947678232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1537867776,"data_stored":1537867776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1685786624,"data_stored":1685782552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1184169984,"data_stored":1184169984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:44:58.929 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:44:58.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:58.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:44:58.958 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:44:58.999 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:44:58.999 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:44:58.999 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:44:59.000 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:44:59.000 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:44:59.033 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:44:59.059 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:59.102 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:44:59.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:59.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:44:59.209 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:44:59.235 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:59.280 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:44:59.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:59.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:44:59.390 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:44:59.431 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:59.480 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:44:59.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:59.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:44:59.586 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:44:59.614 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:44:59.743 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:59 smithi138 ceph-mon[192513]: from='client.37062 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:59.744 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:44:59 smithi138 ceph-mon[192513]: pgmap v1562: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 168 op/s; 7171/173994 objects degraded (4.121%); 3540/173994 objects misplaced (2.035%); 678 KiB/s, 10 objects/s recovering 2023-10-01T09:44:59.744 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:59 smithi138 ceph-mon[185886]: from='client.37062 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:59.744 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:44:59 smithi138 ceph-mon[185886]: pgmap v1562: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 168 op/s; 7171/173994 objects degraded (4.121%); 3540/173994 objects misplaced (2.035%); 678 KiB/s, 10 objects/s recovering 2023-10-01T09:44:59.763 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:44:59.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:59 smithi190 ceph-mon[160435]: from='client.37062 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:44:59.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:44:59 smithi190 ceph-mon[160435]: pgmap v1562: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 168 op/s; 7171/173994 objects degraded (4.121%); 3540/173994 objects misplaced (2.035%); 678 KiB/s, 10 objects/s recovering 2023-10-01T09:44:59.813 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:44:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:44:59.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:44:59.920 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:44:59.949 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:00.002 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:45:00.589 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:00.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:00.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:00.693 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:00.728 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:00.741 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:00.765 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:00.781 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:00.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:00.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:00.889 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:00.918 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:00.994 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:00.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:00.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:01.097 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:01.130 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:01.800 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:45:01.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:01 smithi190 ceph-mon[160435]: pgmap v1563: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 169 op/s; 7171/173643 objects degraded (4.130%); 3540/173643 objects misplaced (2.039%); 678 KiB/s, 10 objects/s recovering 2023-10-01T09:45:01.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:01 smithi138 ceph-mon[192513]: pgmap v1563: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 169 op/s; 7171/173643 objects degraded (4.130%); 3540/173643 objects misplaced (2.039%); 678 KiB/s, 10 objects/s recovering 2023-10-01T09:45:01.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:01 smithi138 ceph-mon[185886]: pgmap v1563: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 169 op/s; 7171/173643 objects degraded (4.130%); 3540/173643 objects misplaced (2.039%); 678 KiB/s, 10 objects/s recovering 2023-10-01T09:45:02.026 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:02.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:02.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:02.133 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:02.161 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:02.205 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:02.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:02.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:02.310 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:02.337 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:02.380 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:02.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:02.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:02.437 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:45:02.485 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:02.514 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:02.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:02 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3377100294' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:45:02.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:02 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3377100294' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:45:02.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:02 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3377100294' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:45:02.934 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:45:02.934 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:45:02.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:45:02.934 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:45:02.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.934 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92726 9.2 9:4513d176:::benchmark_data_smithi138_497829_object92725:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e420)", 2023-10-01T09:45:02.934 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.528465+0000", 2023-10-01T09:45:02.934 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.34262338300005, 2023-10-01T09:45:02.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1036305710000001, 2023-10-01T09:45:02.935 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:02.935 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:45:02.935 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:02.935 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:45:02.935 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:45:02.935 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92726 2023-10-01T09:45:02.935 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.935 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:02.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.935 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:02.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.528465+0000", 2023-10-01T09:45:02.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:02.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.528465+0000", 2023-10-01T09:45:02.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:02.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.528464+0000", 2023-10-01T09:45:02.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:45:02.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:02.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.528491+0000", 2023-10-01T09:45:02.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7314999999999999e-05 2023-10-01T09:45:02.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:02.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.528501+0000", 2023-10-01T09:45:02.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0254999999999999e-05 2023-10-01T09:45:02.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:02.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.528510+0000", 2023-10-01T09:45:02.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2849999999999995e-06 2023-10-01T09:45:02.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.533736+0000", 2023-10-01T09:45:02.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0052263479999999996 2023-10-01T09:45:02.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.938 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:02.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.533933+0000", 2023-10-01T09:45:02.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000197474 2023-10-01T09:45:02.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:45:02.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.534132+0000", 2023-10-01T09:45:02.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019866999999999999 2023-10-01T09:45:02.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.548469+0000", 2023-10-01T09:45:02.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014337216 2023-10-01T09:45:02.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:02.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.549134+0000", 2023-10-01T09:45:02.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00066513599999999998 2023-10-01T09:45:02.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.556602+0000", 2023-10-01T09:45:02.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0074679619999999999 2023-10-01T09:45:02.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:45:02.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.556622+0000", 2023-10-01T09:45:02.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9607e-05 2023-10-01T09:45:02.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.631986+0000", 2023-10-01T09:45:02.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.075363702 2023-10-01T09:45:02.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:02.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.632096+0000", 2023-10-01T09:45:02.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000110096 2023-10-01T09:45:02.942 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.942 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:02.942 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.942 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92775 9.0 9:0798a061:::benchmark_data_smithi138_497829_object92774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:45:02.942 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.607380+0000", 2023-10-01T09:45:02.942 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.26370851900003, 2023-10-01T09:45:02.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0222879359999999, 2023-10-01T09:45:02.942 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:02.942 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:02.943 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:02.943 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:45:02.943 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:45:02.943 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92775 2023-10-01T09:45:02.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.943 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:02.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:02.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607380+0000", 2023-10-01T09:45:02.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.944 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:02.944 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607380+0000", 2023-10-01T09:45:02.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.944 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:02.944 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607378+0000", 2023-10-01T09:45:02.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:45:02.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.944 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:02.944 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607417+0000", 2023-10-01T09:45:02.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8266000000000003e-05 2023-10-01T09:45:02.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:02.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607432+0000", 2023-10-01T09:45:02.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4946000000000001e-05 2023-10-01T09:45:02.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:02.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607439+0000", 2023-10-01T09:45:02.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3719999999999999e-06 2023-10-01T09:45:02.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607543+0000", 2023-10-01T09:45:02.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010367299999999999 2023-10-01T09:45:02.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:45:02.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607553+0000", 2023-10-01T09:45:02.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0441999999999999e-05 2023-10-01T09:45:02.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.621571+0000", 2023-10-01T09:45:02.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0140181539999999 2023-10-01T09:45:02.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:02.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.621738+0000", 2023-10-01T09:45:02.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016643699999999999 2023-10-01T09:45:02.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.947 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:45:02.947 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.621891+0000", 2023-10-01T09:45:02.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015302299999999999 2023-10-01T09:45:02.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:02.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.628749+0000", 2023-10-01T09:45:02.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0068583519999999999 2023-10-01T09:45:02.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.628828+0000", 2023-10-01T09:45:02.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8592999999999998e-05 2023-10-01T09:45:02.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.628884+0000", 2023-10-01T09:45:02.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5968999999999999e-05 2023-10-01T09:45:02.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:02.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.628943+0000", 2023-10-01T09:45:02.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9694000000000001e-05 2023-10-01T09:45:02.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:02.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.629668+0000", 2023-10-01T09:45:02.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00072471999999999999 2023-10-01T09:45:02.950 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.950 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:02.950 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.950 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.950 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92776 9.2 9:41354763:::benchmark_data_smithi138_497829_object92775:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:45:02.950 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.607457+0000", 2023-10-01T09:45:02.950 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.26363177300004, 2023-10-01T09:45:02.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0409147700000001, 2023-10-01T09:45:02.951 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:02.951 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:02.951 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:02.951 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:45:02.951 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:45:02.951 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92776 2023-10-01T09:45:02.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.951 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:02.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:02.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607457+0000", 2023-10-01T09:45:02.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:02.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607457+0000", 2023-10-01T09:45:02.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:02.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607456+0000", 2023-10-01T09:45:02.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:45:02.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:02.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607476+0000", 2023-10-01T09:45:02.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9949000000000002e-05 2023-10-01T09:45:02.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:02.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607485+0000", 2023-10-01T09:45:02.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0680000000000003e-06 2023-10-01T09:45:02.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:02.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607490+0000", 2023-10-01T09:45:02.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2220000000000001e-06 2023-10-01T09:45:02.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607550+0000", 2023-10-01T09:45:02.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9349999999999999e-05 2023-10-01T09:45:02.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:45:02.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.607561+0000", 2023-10-01T09:45:02.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0869e-05 2023-10-01T09:45:02.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.632780+0000", 2023-10-01T09:45:02.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.025219221 2023-10-01T09:45:02.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:02.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.632953+0000", 2023-10-01T09:45:02.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000173294 2023-10-01T09:45:02.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:45:02.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.633148+0000", 2023-10-01T09:45:02.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019489400000000001 2023-10-01T09:45:02.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.647265+0000", 2023-10-01T09:45:02.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014117054 2023-10-01T09:45:02.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.956 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.956 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.647300+0000", 2023-10-01T09:45:02.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5153e-05 2023-10-01T09:45:02.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:02.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.647360+0000", 2023-10-01T09:45:02.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9716999999999998e-05 2023-10-01T09:45:02.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:02.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.647417+0000", 2023-10-01T09:45:02.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7259999999999997e-05 2023-10-01T09:45:02.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:02.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.648372+0000", 2023-10-01T09:45:02.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000954359 2023-10-01T09:45:02.958 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.958 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:02.958 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.958 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92779 9.0 9:0c623f1e:::benchmark_data_smithi138_497829_object92778:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:45:02.958 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.608374+0000", 2023-10-01T09:45:02.958 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.26271443799999, 2023-10-01T09:45:02.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0311556449999999, 2023-10-01T09:45:02.959 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:02.959 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:02.959 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:02.959 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:45:02.959 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:45:02.959 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92779 2023-10-01T09:45:02.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.959 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:02.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.959 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:02.959 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608374+0000", 2023-10-01T09:45:02.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:02.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608374+0000", 2023-10-01T09:45:02.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:02.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608374+0000", 2023-10-01T09:45:02.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:45:02.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:02.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608394+0000", 2023-10-01T09:45:02.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0027e-05 2023-10-01T09:45:02.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:02.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608402+0000", 2023-10-01T09:45:02.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8519999999999993e-06 2023-10-01T09:45:02.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:02.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608410+0000", 2023-10-01T09:45:02.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6420000000000007e-06 2023-10-01T09:45:02.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608465+0000", 2023-10-01T09:45:02.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4969999999999997e-05 2023-10-01T09:45:02.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:45:02.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.608475+0000", 2023-10-01T09:45:02.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0304e-05 2023-10-01T09:45:02.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.625720+0000", 2023-10-01T09:45:02.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0172442719999999 2023-10-01T09:45:02.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:02.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.625884+0000", 2023-10-01T09:45:02.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016460699999999999 2023-10-01T09:45:02.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:45:02.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.626061+0000", 2023-10-01T09:45:02.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000176381 2023-10-01T09:45:02.964 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.964 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.964 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.634818+0000", 2023-10-01T09:45:02.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0087576919999999992 2023-10-01T09:45:02.964 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.964 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.964 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.634863+0000", 2023-10-01T09:45:02.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5006999999999998e-05 2023-10-01T09:45:02.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.965 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:02.965 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.638629+0000", 2023-10-01T09:45:02.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037660789999999999 2023-10-01T09:45:02.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.965 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:02.965 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.638690+0000", 2023-10-01T09:45:02.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0847999999999997e-05 2023-10-01T09:45:02.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.965 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:02.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.639530+0000", 2023-10-01T09:45:02.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00083966399999999995 2023-10-01T09:45:02.966 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.966 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:02.966 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.966 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92794 9.0 9:01172f72:::benchmark_data_smithi138_497829_object92793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:45:02.966 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.623408+0000", 2023-10-01T09:45:02.966 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.24768089099996, 2023-10-01T09:45:02.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.019196808, 2023-10-01T09:45:02.967 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:02.967 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:02.967 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:02.967 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:45:02.967 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:45:02.967 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92794 2023-10-01T09:45:02.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.967 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:02.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:02.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623408+0000", 2023-10-01T09:45:02.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:02.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623408+0000", 2023-10-01T09:45:02.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:02.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623406+0000", 2023-10-01T09:45:02.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:45:02.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:02.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623432+0000", 2023-10-01T09:45:02.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5511999999999999e-05 2023-10-01T09:45:02.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:02.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623446+0000", 2023-10-01T09:45:02.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4956e-05 2023-10-01T09:45:02.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:02.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623456+0000", 2023-10-01T09:45:02.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1910000000000006e-06 2023-10-01T09:45:02.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623523+0000", 2023-10-01T09:45:02.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7577999999999996e-05 2023-10-01T09:45:02.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:45:02.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.623540+0000", 2023-10-01T09:45:02.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6483000000000001e-05 2023-10-01T09:45:02.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.629728+0000", 2023-10-01T09:45:02.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.006188071 2023-10-01T09:45:02.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:02.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.630863+0000", 2023-10-01T09:45:02.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001135287 2023-10-01T09:45:02.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:45:02.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.631095+0000", 2023-10-01T09:45:02.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00023191100000000001 2023-10-01T09:45:02.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:02.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.639546+0000", 2023-10-01T09:45:02.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0084506949999999994 2023-10-01T09:45:02.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.639589+0000", 2023-10-01T09:45:02.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3439000000000002e-05 2023-10-01T09:45:02.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.641642+0000", 2023-10-01T09:45:02.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020529680000000001 2023-10-01T09:45:02.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.973 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:02.973 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.641693+0000", 2023-10-01T09:45:02.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1221000000000003e-05 2023-10-01T09:45:02.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:02.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.642605+0000", 2023-10-01T09:45:02.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00091121699999999997 2023-10-01T09:45:02.974 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.974 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:02.974 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.974 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92802 9.0 9:053736bf:::benchmark_data_smithi138_497829_object92801:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:45:02.975 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.630809+0000", 2023-10-01T09:45:02.975 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.24027935100003, 2023-10-01T09:45:02.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.01467168, 2023-10-01T09:45:02.975 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:02.975 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:02.975 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:02.975 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:45:02.975 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:45:02.975 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92802 2023-10-01T09:45:02.975 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.975 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:02.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:02.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630809+0000", 2023-10-01T09:45:02.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:02.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630809+0000", 2023-10-01T09:45:02.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:02.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630808+0000", 2023-10-01T09:45:02.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:45:02.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:02.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630830+0000", 2023-10-01T09:45:02.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.137e-05 2023-10-01T09:45:02.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:02.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630841+0000", 2023-10-01T09:45:02.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1817999999999999e-05 2023-10-01T09:45:02.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:02.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630849+0000", 2023-10-01T09:45:02.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6650000000000003e-06 2023-10-01T09:45:02.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630903+0000", 2023-10-01T09:45:02.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4423999999999999e-05 2023-10-01T09:45:02.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.978 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:45:02.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.630912+0000", 2023-10-01T09:45:02.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5750000000000001e-06 2023-10-01T09:45:02.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.634989+0000", 2023-10-01T09:45:02.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0040773009999999 2023-10-01T09:45:02.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:02.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.635152+0000", 2023-10-01T09:45:02.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000162886 2023-10-01T09:45:02.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,6", 2023-10-01T09:45:02.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.635337+0000", 2023-10-01T09:45:02.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018489099999999999 2023-10-01T09:45:02.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.640106+0000", 2023-10-01T09:45:02.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0047690930000000003 2023-10-01T09:45:02.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.642662+0000", 2023-10-01T09:45:02.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025556910000000001 2023-10-01T09:45:02.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:02.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.644600+0000", 2023-10-01T09:45:02.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001938005 2023-10-01T09:45:02.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:02.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.644661+0000", 2023-10-01T09:45:02.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1155000000000007e-05 2023-10-01T09:45:02.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.982 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:02.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.645481+0000", 2023-10-01T09:45:02.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00081991500000000005 2023-10-01T09:45:02.982 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.982 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:02.982 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.982 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92810 9.2 9:41e5de28:::benchmark_data_smithi138_497829_object92809:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:45:02.983 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.641159+0000", 2023-10-01T09:45:02.983 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.22992922499998, 2023-10-01T09:45:02.983 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0105064420000001, 2023-10-01T09:45:02.983 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:02.983 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:02.983 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:02.983 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:45:02.983 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:45:02.983 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92810 2023-10-01T09:45:02.983 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.983 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:02.983 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:02.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641159+0000", 2023-10-01T09:45:02.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.984 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:02.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641159+0000", 2023-10-01T09:45:02.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.984 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:02.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641158+0000", 2023-10-01T09:45:02.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:45:02.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:02.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641178+0000", 2023-10-01T09:45:02.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9389e-05 2023-10-01T09:45:02.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:02.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641186+0000", 2023-10-01T09:45:02.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9270000000000005e-06 2023-10-01T09:45:02.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.986 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:02.986 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641190+0000", 2023-10-01T09:45:02.986 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7940000000000002e-06 2023-10-01T09:45:02.986 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.986 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.986 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641257+0000", 2023-10-01T09:45:02.986 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6339000000000003e-05 2023-10-01T09:45:02.986 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:45:02.987 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.641270+0000", 2023-10-01T09:45:02.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2748999999999999e-05 2023-10-01T09:45:02.987 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.987 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.987 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.636539+0000", 2023-10-01T09:45:02.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99526912099999998 2023-10-01T09:45:02.987 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.987 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:02.987 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.636704+0000", 2023-10-01T09:45:02.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016581500000000001 2023-10-01T09:45:02.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:45:02.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.636861+0000", 2023-10-01T09:45:02.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015662599999999999 2023-10-01T09:45:02.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.647350+0000", 2023-10-01T09:45:02.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010488509 2023-10-01T09:45:02.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:02.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.648383+0000", 2023-10-01T09:45:02.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001033462 2023-10-01T09:45:02.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.650664+0000", 2023-10-01T09:45:02.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022806929999999999 2023-10-01T09:45:02.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:02.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.650721+0000", 2023-10-01T09:45:02.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7343000000000003e-05 2023-10-01T09:45:02.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:02.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.651666+0000", 2023-10-01T09:45:02.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00094476399999999996 2023-10-01T09:45:02.990 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.991 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:02.991 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.991 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92814 9.2 9:4a803fc0:::benchmark_data_smithi138_497829_object92813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:45:02.991 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.644861+0000", 2023-10-01T09:45:02.991 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.22622786500006, 2023-10-01T09:45:02.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.022649412, 2023-10-01T09:45:02.991 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:02.991 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:02.991 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:02.991 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:45:02.992 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:45:02.992 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92814 2023-10-01T09:45:02.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.992 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:02.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:02.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644861+0000", 2023-10-01T09:45:02.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:02.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644861+0000", 2023-10-01T09:45:02.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:02.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:02.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644860+0000", 2023-10-01T09:45:02.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:45:02.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:02.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644879+0000", 2023-10-01T09:45:02.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9857999999999999e-05 2023-10-01T09:45:02.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:02.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644892+0000", 2023-10-01T09:45:02.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2292e-05 2023-10-01T09:45:02.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.994 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:02.994 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644900+0000", 2023-10-01T09:45:02.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.259e-06 2023-10-01T09:45:02.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.995 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644960+0000", 2023-10-01T09:45:02.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0247000000000001e-05 2023-10-01T09:45:02.995 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:45:02.995 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.644968+0000", 2023-10-01T09:45:02.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8099999999999998e-06 2023-10-01T09:45:02.995 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:02.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.640030+0000", 2023-10-01T09:45:02.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.99506214699999995 2023-10-01T09:45:02.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:02.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.640250+0000", 2023-10-01T09:45:02.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021938700000000001 2023-10-01T09:45:02.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:45:02.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.640445+0000", 2023-10-01T09:45:02.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019577400000000001 2023-10-01T09:45:02.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:02.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.648390+0000", 2023-10-01T09:45:02.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0079443840000000005 2023-10-01T09:45:02.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.652516+0000", 2023-10-01T09:45:02.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041264450000000003 2023-10-01T09:45:02.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:02.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.666515+0000", 2023-10-01T09:45:02.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013999161 2023-10-01T09:45:02.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:02.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.666577+0000", 2023-10-01T09:45:02.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1958000000000005e-05 2023-10-01T09:45:02.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.999 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:02.999 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.667510+0000", 2023-10-01T09:45:02.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00093287700000000003 2023-10-01T09:45:02.999 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.999 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:02.999 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:02.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:02.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:02.999 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:92832 9.2 9:46d6656f:::benchmark_data_smithi138_497829_object92831:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e421)", 2023-10-01T09:45:02.999 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:35:33.698836+0000", 2023-10-01T09:45:02.999 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 569.17225263, 2023-10-01T09:45:02.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.96914843500000003, 2023-10-01T09:45:03.000 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:03.000 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:03.000 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:03.000 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:45:03.000 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:45:03.000 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 92832 2023-10-01T09:45:03.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.000 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:03.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.000 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:03.000 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698836+0000", 2023-10-01T09:45:03.000 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:03.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698836+0000", 2023-10-01T09:45:03.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:03.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698835+0000", 2023-10-01T09:45:03.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:45:03.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:03.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698854+0000", 2023-10-01T09:45:03.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9089e-05 2023-10-01T09:45:03.002 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:03.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698865+0000", 2023-10-01T09:45:03.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0429e-05 2023-10-01T09:45:03.002 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:03.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698870+0000", 2023-10-01T09:45:03.003 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3229999999999997e-06 2023-10-01T09:45:03.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.003 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.003 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698930+0000", 2023-10-01T09:45:03.003 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0136999999999997e-05 2023-10-01T09:45:03.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.003 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:45:03.003 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:33.698943+0000", 2023-10-01T09:45:03.003 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3081999999999999e-05 2023-10-01T09:45:03.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.004 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.004 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.004 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.643687+0000", 2023-10-01T09:45:03.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.94474369199999997 2023-10-01T09:45:03.004 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.004 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.004 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:03.004 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.643851+0000", 2023-10-01T09:45:03.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000164239 2023-10-01T09:45:03.004 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.004 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.004 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,3", 2023-10-01T09:45:03.004 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.643968+0000", 2023-10-01T09:45:03.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000116741 2023-10-01T09:45:03.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:03.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.649439+0000", 2023-10-01T09:45:03.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005470801 2023-10-01T09:45:03.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.653893+0000", 2023-10-01T09:45:03.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044539159999999996 2023-10-01T09:45:03.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.667566+0000", 2023-10-01T09:45:03.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013673284000000001 2023-10-01T09:45:03.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:03.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.667617+0000", 2023-10-01T09:45:03.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0624000000000002e-05 2023-10-01T09:45:03.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:03.007 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:35:34.667984+0000", 2023-10-01T09:45:03.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00036788899999999999 2023-10-01T09:45:03.007 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.007 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:03.007 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.007 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:147923 9.a 9:5afed6f2:::benchmark_data_smithi138_497829_object45538:head [delete] snapc 0=[] ondisk+write+known_if_redirected e444)", 2023-10-01T09:45:03.007 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:22.282541+0000", 2023-10-01T09:45:03.007 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 460.58854738000002, 2023-10-01T09:45:03.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.55728740300000001, 2023-10-01T09:45:03.007 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:03.008 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:03.008 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:03.008 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:45:03.008 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:45:03.008 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 147923 2023-10-01T09:45:03.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.008 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:03.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:03.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282541+0000", 2023-10-01T09:45:03.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:03.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282541+0000", 2023-10-01T09:45:03.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:03.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282541+0000", 2023-10-01T09:45:03.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:45:03.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:03.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282546+0000", 2023-10-01T09:45:03.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.972e-06 2023-10-01T09:45:03.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:03.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282560+0000", 2023-10-01T09:45:03.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4630999999999999e-05 2023-10-01T09:45:03.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:03.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282567+0000", 2023-10-01T09:45:03.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3010000000000001e-06 2023-10-01T09:45:03.010 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.285815+0000", 2023-10-01T09:45:03.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003247963 2023-10-01T09:45:03.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:45:03.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289067+0000", 2023-10-01T09:45:03.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032514169999999999 2023-10-01T09:45:03.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.307845+0000", 2023-10-01T09:45:03.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018777933 2023-10-01T09:45:03.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.012 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:45:03.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.307857+0000", 2023-10-01T09:45:03.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.254e-05 2023-10-01T09:45:03.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.012 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.805080+0000", 2023-10-01T09:45:03.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.49722238000000002 2023-10-01T09:45:03.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.013 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:45:03.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.805434+0000", 2023-10-01T09:45:03.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00035419299999999999 2023-10-01T09:45:03.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.013 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.815559+0000", 2023-10-01T09:45:03.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010125265 2023-10-01T09:45:03.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:03.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.815641+0000", 2023-10-01T09:45:03.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1499e-05 2023-10-01T09:45:03.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:45:03.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.815740+0000", 2023-10-01T09:45:03.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9504000000000001e-05 2023-10-01T09:45:03.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:03.015 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.827989+0000", 2023-10-01T09:45:03.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01224926 2023-10-01T09:45:03.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.015 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.828043+0000", 2023-10-01T09:45:03.015 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3876999999999999e-05 2023-10-01T09:45:03.015 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.015 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.015 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.836681+0000", 2023-10-01T09:45:03.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086381300000000008 2023-10-01T09:45:03.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:03.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.836733+0000", 2023-10-01T09:45:03.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1399e-05 2023-10-01T09:45:03.016 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.016 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.016 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:03.016 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.839829+0000", 2023-10-01T09:45:03.016 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030958539999999999 2023-10-01T09:45:03.017 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.017 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:03.017 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.017 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.017 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.017 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:94709 10.b 10:d32b1004:::benchmark_data_smithi138_573643_object11487:head [delete] snapc 0=[] ondisk+write+known_if_redirected e520)", 2023-10-01T09:45:03.017 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:43:16.167925+0000", 2023-10-01T09:45:03.017 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 106.703163796, 2023-10-01T09:45:03.017 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010144777000000001, 2023-10-01T09:45:03.017 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:03.017 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:03.017 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:03.018 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:45:03.018 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:03.018 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 94709 2023-10-01T09:45:03.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.018 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:03.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:03.018 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:16.167925+0000", 2023-10-01T09:45:03.018 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.018 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.018 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.018 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:03.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:16.167925+0000", 2023-10-01T09:45:03.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.019 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:03.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:16.167922+0000", 2023-10-01T09:45:03.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:45:03.019 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.019 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.019 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:03.019 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:16.167935+0000", 2023-10-01T09:45:03.019 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2478e-05 2023-10-01T09:45:03.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:03.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:16.167966+0000", 2023-10-01T09:45:03.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1352000000000001e-05 2023-10-01T09:45:03.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:03.020 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:16.167980+0000", 2023-10-01T09:45:03.020 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3608000000000001e-05 2023-10-01T09:45:03.020 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.020 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.020 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:16.168045+0000", 2023-10-01T09:45:03.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4800000000000003e-05 2023-10-01T09:45:03.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:03.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:16.168188+0000", 2023-10-01T09:45:03.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014371699999999999 2023-10-01T09:45:03.021 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.021 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.021 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:45:03.021 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:16.168312+0000", 2023-10-01T09:45:03.021 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000124017 2023-10-01T09:45:03.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.022 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:16.171585+0000", 2023-10-01T09:45:03.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032728259999999999 2023-10-01T09:45:03.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.022 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:03.022 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:16.172289+0000", 2023-10-01T09:45:03.022 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00070351599999999995 2023-10-01T09:45:03.022 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.022 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:16.177974+0000", 2023-10-01T09:45:03.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056856939999999998 2023-10-01T09:45:03.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:03.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:16.178018+0000", 2023-10-01T09:45:03.023 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3319999999999999e-05 2023-10-01T09:45:03.023 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.023 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.023 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:03.023 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:16.178070+0000", 2023-10-01T09:45:03.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1898999999999999e-05 2023-10-01T09:45:03.024 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.024 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:03.024 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.024 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.024 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.024 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:94877 10.b 10:d6a0de7d:::benchmark_data_smithi138_573643_object11655:head [delete] snapc 0=[] ondisk+write+known_if_redirected e520)", 2023-10-01T09:45:03.024 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:43:17.382480+0000", 2023-10-01T09:45:03.024 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 105.488608837, 2023-10-01T09:45:03.024 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0102287, 2023-10-01T09:45:03.024 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:03.024 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:03.025 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:03.025 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:45:03.025 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:03.025 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 94877 2023-10-01T09:45:03.025 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.025 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:03.025 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.025 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:03.025 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:17.382480+0000", 2023-10-01T09:45:03.025 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.025 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.025 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:03.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:17.382480+0000", 2023-10-01T09:45:03.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:03.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:17.382478+0000", 2023-10-01T09:45:03.026 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:45:03.026 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.026 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.026 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:03.026 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:17.382486+0000", 2023-10-01T09:45:03.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9049999999999997e-06 2023-10-01T09:45:03.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:03.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:17.382506+0000", 2023-10-01T09:45:03.027 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9381e-05 2023-10-01T09:45:03.027 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.027 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.027 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:03.027 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:17.382518+0000", 2023-10-01T09:45:03.029 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2065000000000001e-05 2023-10-01T09:45:03.029 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.029 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:17.382579+0000", 2023-10-01T09:45:03.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1789000000000001e-05 2023-10-01T09:45:03.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:03.030 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:17.382736+0000", 2023-10-01T09:45:03.030 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015679499999999999 2023-10-01T09:45:03.030 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.030 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.030 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:45:03.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:17.382853+0000", 2023-10-01T09:45:03.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000116471 2023-10-01T09:45:03.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:17.386276+0000", 2023-10-01T09:45:03.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034237790000000001 2023-10-01T09:45:03.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.031 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.031 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:03.031 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:17.386411+0000", 2023-10-01T09:45:03.031 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013471199999999999 2023-10-01T09:45:03.031 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.032 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.032 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:17.392607+0000", 2023-10-01T09:45:03.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0061954009999999997 2023-10-01T09:45:03.032 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.032 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:03.032 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:17.392664+0000", 2023-10-01T09:45:03.032 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7006000000000003e-05 2023-10-01T09:45:03.032 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.032 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.033 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:03.033 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:17.392709+0000", 2023-10-01T09:45:03.033 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4926000000000003e-05 2023-10-01T09:45:03.033 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.033 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:03.033 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.033 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.033 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.033 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:97791 10.b 10:d970adc6:::benchmark_data_smithi138_573643_object14569:head [delete] snapc 0=[] ondisk+write+known_if_redirected e523)", 2023-10-01T09:45:03.033 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:43:41.091369+0000", 2023-10-01T09:45:03.033 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 81.779720029000003, 2023-10-01T09:45:03.033 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010267528999999999, 2023-10-01T09:45:03.034 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:03.034 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:03.034 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:03.034 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:45:03.034 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:03.034 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 97791 2023-10-01T09:45:03.034 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.034 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:03.034 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.034 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:03.034 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:41.091369+0000", 2023-10-01T09:45:03.034 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:03.035 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:41.091369+0000", 2023-10-01T09:45:03.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.035 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:03.035 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:41.091367+0000", 2023-10-01T09:45:03.035 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:45:03.035 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.035 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:03.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:41.091374+0000", 2023-10-01T09:45:03.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8399999999999997e-06 2023-10-01T09:45:03.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.036 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:03.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:41.091390+0000", 2023-10-01T09:45:03.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5512e-05 2023-10-01T09:45:03.036 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.036 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.036 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:03.036 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:41.091397+0000", 2023-10-01T09:45:03.036 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.294e-06 2023-10-01T09:45:03.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.037 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:41.091444+0000", 2023-10-01T09:45:03.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7398000000000003e-05 2023-10-01T09:45:03.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.037 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.037 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:03.037 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:41.091540+0000", 2023-10-01T09:45:03.037 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6199999999999994e-05 2023-10-01T09:45:03.037 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.038 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.038 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:45:03.038 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:41.091618+0000", 2023-10-01T09:45:03.038 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7898999999999994e-05 2023-10-01T09:45:03.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.038 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.038 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:03.038 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:41.094330+0000", 2023-10-01T09:45:03.038 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0027114579999999999 2023-10-01T09:45:03.038 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.038 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.038 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.039 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:41.094548+0000", 2023-10-01T09:45:03.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021787999999999999 2023-10-01T09:45:03.039 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.039 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.039 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.039 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:41.101542+0000", 2023-10-01T09:45:03.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069942720000000002 2023-10-01T09:45:03.039 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.039 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.039 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:03.039 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:41.101595+0000", 2023-10-01T09:45:03.039 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2667000000000002e-05 2023-10-01T09:45:03.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.040 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:03.040 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:41.101636+0000", 2023-10-01T09:45:03.040 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1505000000000003e-05 2023-10-01T09:45:03.040 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.040 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:03.040 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.040 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.040 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.040 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:98428 10.2 10:465b81c4:::benchmark_data_smithi138_573643_object15206:head [delete] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-01T09:45:03.040 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:43:45.525522+0000", 2023-10-01T09:45:03.041 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 77.345566392999999, 2023-10-01T09:45:03.041 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010810827, 2023-10-01T09:45:03.041 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:03.041 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:03.041 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:03.041 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:45:03.041 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:03.041 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 98428 2023-10-01T09:45:03.041 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.041 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:03.041 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.042 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:03.042 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525522+0000", 2023-10-01T09:45:03.042 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.042 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.042 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.042 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:03.042 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525522+0000", 2023-10-01T09:45:03.042 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.042 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.042 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.042 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:03.042 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525521+0000", 2023-10-01T09:45:03.042 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:45:03.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.043 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.043 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:03.043 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525527+0000", 2023-10-01T09:45:03.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9970000000000004e-06 2023-10-01T09:45:03.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.043 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.043 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:03.043 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525540+0000", 2023-10-01T09:45:03.043 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.344e-05 2023-10-01T09:45:03.043 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.043 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:03.044 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525550+0000", 2023-10-01T09:45:03.044 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6409999999999997e-06 2023-10-01T09:45:03.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.044 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525599+0000", 2023-10-01T09:45:03.044 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9360999999999997e-05 2023-10-01T09:45:03.044 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.044 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.044 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:03.044 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525771+0000", 2023-10-01T09:45:03.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000171807 2023-10-01T09:45:03.045 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.045 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.045 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:45:03.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525859+0000", 2023-10-01T09:45:03.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7646999999999996e-05 2023-10-01T09:45:03.045 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.045 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.045 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:03.045 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.529423+0000", 2023-10-01T09:45:03.045 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035644040000000002 2023-10-01T09:45:03.045 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.046 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.046 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.046 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.533158+0000", 2023-10-01T09:45:03.046 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037345740000000001 2023-10-01T09:45:03.046 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.046 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.046 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.046 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.536237+0000", 2023-10-01T09:45:03.046 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003079217 2023-10-01T09:45:03.046 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.046 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.046 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:03.046 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.536292+0000", 2023-10-01T09:45:03.047 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4551e-05 2023-10-01T09:45:03.047 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.047 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.047 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:03.047 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.536333+0000", 2023-10-01T09:45:03.047 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1378000000000003e-05 2023-10-01T09:45:03.047 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.047 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:03.047 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.047 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.047 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.047 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:98554 10.2 10:449bd2ed:::benchmark_data_smithi138_573643_object15332:head [delete] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-01T09:45:03.048 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:43:46.357532+0000", 2023-10-01T09:45:03.048 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 76.513556803, 2023-10-01T09:45:03.048 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.079442952999999997, 2023-10-01T09:45:03.048 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:03.048 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:03.048 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:03.048 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:45:03.048 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:03.048 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 98554 2023-10-01T09:45:03.048 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.048 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:03.048 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:03.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357532+0000", 2023-10-01T09:45:03.049 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.049 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.049 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:03.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357532+0000", 2023-10-01T09:45:03.049 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.049 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.049 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.049 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:03.049 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357530+0000", 2023-10-01T09:45:03.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:45:03.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:03.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357537+0000", 2023-10-01T09:45:03.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2639999999999997e-06 2023-10-01T09:45:03.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.050 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:03.050 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357548+0000", 2023-10-01T09:45:03.050 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1671000000000001e-05 2023-10-01T09:45:03.050 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.050 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:03.051 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357556+0000", 2023-10-01T09:45:03.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3769999999999998e-06 2023-10-01T09:45:03.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.051 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.051 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357594+0000", 2023-10-01T09:45:03.051 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8191000000000001e-05 2023-10-01T09:45:03.051 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.051 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.051 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:03.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357675+0000", 2023-10-01T09:45:03.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0648000000000004e-05 2023-10-01T09:45:03.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.052 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:45:03.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357738+0000", 2023-10-01T09:45:03.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3931999999999997e-05 2023-10-01T09:45:03.052 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.052 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.052 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:03.052 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.360298+0000", 2023-10-01T09:45:03.052 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025591089999999999 2023-10-01T09:45:03.053 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.053 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.361416+0000", 2023-10-01T09:45:03.053 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001118747 2023-10-01T09:45:03.053 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.053 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.436873+0000", 2023-10-01T09:45:03.053 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.075457065000000004 2023-10-01T09:45:03.053 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.053 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.053 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:03.054 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.436928+0000", 2023-10-01T09:45:03.054 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4527999999999997e-05 2023-10-01T09:45:03.054 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.054 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.054 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:03.054 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.436975+0000", 2023-10-01T09:45:03.054 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6888000000000002e-05 2023-10-01T09:45:03.054 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.054 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:03.054 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.054 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.054 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.055 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:100505 10.b 10:d87c0769:::benchmark_data_smithi138_573643_object17283:head [delete] snapc 0=[] ondisk+write+known_if_redirected e526)", 2023-10-01T09:45:03.055 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:44:02.008873+0000", 2023-10-01T09:45:03.055 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 60.862215536000001, 2023-10-01T09:45:03.055 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094751469999999997, 2023-10-01T09:45:03.055 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:03.055 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:03.055 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:03.055 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:45:03.055 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:03.055 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 100505 2023-10-01T09:45:03.055 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.055 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:03.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:03.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:02.008873+0000", 2023-10-01T09:45:03.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:03.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:02.008873+0000", 2023-10-01T09:45:03.056 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.056 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.056 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.056 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:03.056 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:02.008873+0000", 2023-10-01T09:45:03.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:45:03.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:03.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:02.008876+0000", 2023-10-01T09:45:03.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4750000000000002e-06 2023-10-01T09:45:03.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.057 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.057 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:03.057 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:02.008883+0000", 2023-10-01T09:45:03.057 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.601e-06 2023-10-01T09:45:03.057 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:03.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:02.008888+0000", 2023-10-01T09:45:03.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3589999999999999e-06 2023-10-01T09:45:03.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.058 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:02.012607+0000", 2023-10-01T09:45:03.058 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037185790000000001 2023-10-01T09:45:03.058 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.058 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.058 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:03.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:02.012738+0000", 2023-10-01T09:45:03.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000131619 2023-10-01T09:45:03.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:45:03.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:02.012819+0000", 2023-10-01T09:45:03.059 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1209000000000001e-05 2023-10-01T09:45:03.059 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.059 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.059 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:03.059 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:02.015606+0000", 2023-10-01T09:45:03.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0027863919999999999 2023-10-01T09:45:03.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:02.016428+0000", 2023-10-01T09:45:03.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000822012 2023-10-01T09:45:03.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.060 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.060 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.060 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:02.018281+0000", 2023-10-01T09:45:03.060 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0018534090000000001 2023-10-01T09:45:03.060 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:03.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:02.018319+0000", 2023-10-01T09:45:03.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7462999999999997e-05 2023-10-01T09:45:03.061 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.061 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.061 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:03.061 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:02.018348+0000", 2023-10-01T09:45:03.061 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9522999999999998e-05 2023-10-01T09:45:03.061 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.061 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:03.061 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.062 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.062 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.062 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:102430 10.2 10:4315b427:::benchmark_data_smithi138_573643_object19208:head [delete] snapc 0=[] ondisk+write+known_if_redirected e528)", 2023-10-01T09:45:03.062 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:44:15.860023+0000", 2023-10-01T09:45:03.062 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 47.011065700000003, 2023-10-01T09:45:03.062 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0090449689999999999, 2023-10-01T09:45:03.062 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:03.062 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:03.062 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:03.062 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:45:03.062 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:03.062 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 102430 2023-10-01T09:45:03.063 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.063 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:03.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.063 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:03.063 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:15.860023+0000", 2023-10-01T09:45:03.063 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.063 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.063 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.063 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:03.063 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:15.860023+0000", 2023-10-01T09:45:03.063 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.063 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:03.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:15.860022+0000", 2023-10-01T09:45:03.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:45:03.064 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:03.064 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:15.860028+0000", 2023-10-01T09:45:03.064 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.015e-06 2023-10-01T09:45:03.064 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.064 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.064 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:03.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:15.860039+0000", 2023-10-01T09:45:03.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1085e-05 2023-10-01T09:45:03.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:03.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:15.860044+0000", 2023-10-01T09:45:03.065 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4020000000000001e-06 2023-10-01T09:45:03.065 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.065 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.065 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.065 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:15.860086+0000", 2023-10-01T09:45:03.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1365000000000001e-05 2023-10-01T09:45:03.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:03.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:15.860190+0000", 2023-10-01T09:45:03.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010458099999999999 2023-10-01T09:45:03.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.066 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.066 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:45:03.066 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:15.860279+0000", 2023-10-01T09:45:03.066 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8503000000000003e-05 2023-10-01T09:45:03.066 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:03.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:15.862918+0000", 2023-10-01T09:45:03.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0026395680000000001 2023-10-01T09:45:03.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.067 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:15.863392+0000", 2023-10-01T09:45:03.067 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00047377500000000001 2023-10-01T09:45:03.067 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.067 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.067 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:15.868884+0000", 2023-10-01T09:45:03.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0054920480000000002 2023-10-01T09:45:03.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:03.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:15.869025+0000", 2023-10-01T09:45:03.068 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014129100000000001 2023-10-01T09:45:03.068 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.068 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.068 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:03.068 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:15.869068+0000", 2023-10-01T09:45:03.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2417999999999999e-05 2023-10-01T09:45:03.069 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.069 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:03.069 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.069 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.069 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.069 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:103729 10.b 10:db22441b:::benchmark_data_smithi138_573643_object20507:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:45:03.069 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:44:26.352243+0000", 2023-10-01T09:45:03.069 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 36.518845448, 2023-10-01T09:45:03.069 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011887605000000001, 2023-10-01T09:45:03.069 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:03.069 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:03.070 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:03.070 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:45:03.070 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:03.070 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 103729 2023-10-01T09:45:03.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.070 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:03.070 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.070 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:03.070 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352243+0000", 2023-10-01T09:45:03.070 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.070 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.071 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.071 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:03.071 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352243+0000", 2023-10-01T09:45:03.071 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.071 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.071 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.071 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:03.071 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352241+0000", 2023-10-01T09:45:03.071 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:45:03.071 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.071 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.071 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:03.072 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352252+0000", 2023-10-01T09:45:03.072 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.096e-05 2023-10-01T09:45:03.072 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.072 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:03.072 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352275+0000", 2023-10-01T09:45:03.072 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2866000000000001e-05 2023-10-01T09:45:03.072 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.072 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.072 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:03.072 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352286+0000", 2023-10-01T09:45:03.072 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.111e-05 2023-10-01T09:45:03.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352352+0000", 2023-10-01T09:45:03.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6421999999999995e-05 2023-10-01T09:45:03.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.073 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:03.073 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352529+0000", 2023-10-01T09:45:03.073 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000176742 2023-10-01T09:45:03.073 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.073 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:45:03.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352672+0000", 2023-10-01T09:45:03.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014320399999999999 2023-10-01T09:45:03.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:03.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.356654+0000", 2023-10-01T09:45:03.074 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039821409999999998 2023-10-01T09:45:03.074 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.074 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.074 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.074 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.361739+0000", 2023-10-01T09:45:03.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050842980000000001 2023-10-01T09:45:03.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.364028+0000", 2023-10-01T09:45:03.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00228911 2023-10-01T09:45:03.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.075 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.075 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:03.075 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.364086+0000", 2023-10-01T09:45:03.075 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8260000000000001e-05 2023-10-01T09:45:03.075 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.076 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:03.076 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.364131+0000", 2023-10-01T09:45:03.076 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4733e-05 2023-10-01T09:45:03.076 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.076 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:03.076 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.076 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.076 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.076 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:103953 10.2 10:414c62ad:::benchmark_data_smithi138_573643_object20731:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:45:03.076 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:44:27.617662+0000", 2023-10-01T09:45:03.076 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 35.253426218000001, 2023-10-01T09:45:03.077 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.121890762, 2023-10-01T09:45:03.077 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:03.077 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:03.077 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:03.077 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:45:03.077 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:03.077 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 103953 2023-10-01T09:45:03.077 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.077 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:03.077 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.077 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:03.077 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617662+0000", 2023-10-01T09:45:03.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:03.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617662+0000", 2023-10-01T09:45:03.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.078 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:03.078 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617661+0000", 2023-10-01T09:45:03.078 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:45:03.078 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.078 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:03.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617669+0000", 2023-10-01T09:45:03.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1100000000000003e-06 2023-10-01T09:45:03.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:03.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617684+0000", 2023-10-01T09:45:03.079 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4935e-05 2023-10-01T09:45:03.079 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.079 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.079 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:03.079 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617691+0000", 2023-10-01T09:45:03.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0480000000000001e-06 2023-10-01T09:45:03.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.080 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.080 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617736+0000", 2023-10-01T09:45:03.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4866e-05 2023-10-01T09:45:03.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.080 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.080 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:03.080 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617830+0000", 2023-10-01T09:45:03.080 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4351999999999994e-05 2023-10-01T09:45:03.080 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:45:03.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617902+0000", 2023-10-01T09:45:03.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2039000000000006e-05 2023-10-01T09:45:03.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:03.081 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.620668+0000", 2023-10-01T09:45:03.081 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0027652670000000001 2023-10-01T09:45:03.081 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.081 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.081 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.620958+0000", 2023-10-01T09:45:03.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00029073499999999997 2023-10-01T09:45:03.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.739468+0000", 2023-10-01T09:45:03.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11850937 2023-10-01T09:45:03.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.082 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.082 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:03.082 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.739506+0000", 2023-10-01T09:45:03.082 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7892999999999998e-05 2023-10-01T09:45:03.082 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.083 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:03.083 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.739553+0000", 2023-10-01T09:45:03.083 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7629000000000001e-05 2023-10-01T09:45:03.083 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.083 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:03.083 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.083 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.083 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.083 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:106708 10.2 10:4998bc04:::benchmark_data_smithi138_573643_object23486:head [delete] snapc 0=[] ondisk+write+known_if_redirected e531)", 2023-10-01T09:45:03.083 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:44:45.947260+0000", 2023-10-01T09:45:03.084 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 16.923828388, 2023-10-01T09:45:03.084 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0089259449999999994, 2023-10-01T09:45:03.084 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:45:03.084 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:03.084 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:45:03.084 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:45:03.084 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:03.084 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 106708 2023-10-01T09:45:03.084 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.084 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:45:03.084 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.084 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:45:03.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:45.947260+0000", 2023-10-01T09:45:03.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.085 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:45:03.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:45.947260+0000", 2023-10-01T09:45:03.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:45:03.085 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.085 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.085 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:45:03.085 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:45.947258+0000", 2023-10-01T09:45:03.085 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:45:03.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:45:03.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:45.947268+0000", 2023-10-01T09:45:03.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0431999999999999e-05 2023-10-01T09:45:03.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.086 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:45:03.086 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:45.947286+0000", 2023-10-01T09:45:03.086 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7946999999999999e-05 2023-10-01T09:45:03.086 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.086 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:45:03.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:45.947296+0000", 2023-10-01T09:45:03.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0037999999999999e-05 2023-10-01T09:45:03.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:45:03.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:45.947404+0000", 2023-10-01T09:45:03.087 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010785099999999999 2023-10-01T09:45:03.087 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.087 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.087 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:45:03.087 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:45.947563+0000", 2023-10-01T09:45:03.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015905699999999999 2023-10-01T09:45:03.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:45:03.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:45.947670+0000", 2023-10-01T09:45:03.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106926 2023-10-01T09:45:03.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.088 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.088 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:45.951242+0000", 2023-10-01T09:45:03.088 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035718149999999999 2023-10-01T09:45:03.088 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.088 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:45:03.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:45.952620+0000", 2023-10-01T09:45:03.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001378256 2023-10-01T09:45:03.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:03.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:45.956107+0000", 2023-10-01T09:45:03.089 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0034865059999999999 2023-10-01T09:45:03.089 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.089 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.089 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:45:03.089 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:45.956155+0000", 2023-10-01T09:45:03.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8494e-05 2023-10-01T09:45:03.090 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:45:03.090 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:45:03.090 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:45:03.090 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:45.956186+0000", 2023-10-01T09:45:03.090 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1171999999999999e-05 2023-10-01T09:45:03.090 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.090 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:03.090 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.090 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:45:03.090 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:45:03.090 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:45:03.310 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:03.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:03.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:03.417 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:03.445 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:03.490 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:03.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:03.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:03 smithi138 ceph-mon[192513]: pgmap v1564: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 7093/172002 objects degraded (4.124%); 3504/172002 objects misplaced (2.037%); 749 KiB/s, 11 objects/s recovering 2023-10-01T09:45:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:03 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:03 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3377100294' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:45:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:03 smithi138 ceph-mon[192513]: osdmap e534: 8 total, 8 up, 7 in 2023-10-01T09:45:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:03 smithi138 ceph-mon[192513]: failed to encode map e534 with expected crc 2023-10-01T09:45:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:03 smithi138 ceph-mon[192513]: failed to encode map e534 with expected crc 2023-10-01T09:45:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:03 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7093/172002 objects degraded (4.124%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:03.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:03 smithi138 ceph-mon[185886]: pgmap v1564: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 7093/172002 objects degraded (4.124%); 3504/172002 objects misplaced (2.037%); 749 KiB/s, 11 objects/s recovering 2023-10-01T09:45:03.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:03 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:03.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:03 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3377100294' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:45:03.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:03 smithi138 ceph-mon[185886]: osdmap e534: 8 total, 8 up, 7 in 2023-10-01T09:45:03.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:03 smithi138 ceph-mon[185886]: failed to encode map e534 with expected crc 2023-10-01T09:45:03.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:03 smithi138 ceph-mon[185886]: failed to encode map e534 with expected crc 2023-10-01T09:45:03.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:03 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7093/172002 objects degraded (4.124%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:03.595 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:03.622 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:03.722 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:03.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:03.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:03.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:03 smithi190 ceph-mon[160435]: pgmap v1564: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 7093/172002 objects degraded (4.124%); 3504/172002 objects misplaced (2.037%); 749 KiB/s, 11 objects/s recovering 2023-10-01T09:45:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:03 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:03 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3377100294' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:45:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:03 smithi190 ceph-mon[160435]: osdmap e534: 8 total, 8 up, 7 in 2023-10-01T09:45:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:03 smithi190 ceph-mon[160435]: failed to encode map e534 with expected crc 2023-10-01T09:45:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:03 smithi190 ceph-mon[160435]: failed to encode map e534 with expected crc 2023-10-01T09:45:03.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:03 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7093/172002 objects degraded (4.124%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:03.826 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:03.852 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:03.907 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:03.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:03.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:04.014 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:04.041 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:04.100 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:04.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:04.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:04.205 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:04.236 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:04.284 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:04.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:04.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:04.389 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:04.421 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:04.485 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:45:04.584 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:04.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:04.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:04.628 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:04.681 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:45:04.688 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:04.709 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:45:04.719 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:04.730 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:45:04.752 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:45:04.767 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:04.774 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:45:04.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:04.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:04.801 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:45:04.823 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:45:04.844 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:45:04.846 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:45:04.846 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:45:04.846 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:45:04.846 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:45:04.846 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:45:04.846 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:45:04.846 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:45:04.846 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:45:04.877 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:04.907 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:04.963 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:04.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:04.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:05.069 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:05.098 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:05.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:45:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:45:04] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-01T09:45:05.695 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:45:05.698 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:05.699 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:05 smithi138 ceph-mon[185886]: pgmap v1566: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 159 op/s; 7093/171858 objects degraded (4.127%); 3504/171858 objects misplaced (2.039%); 838 KiB/s, 13 objects/s recovering 2023-10-01T09:45:05.699 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/700370040' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:05.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:05 smithi138 ceph-mon[192513]: pgmap v1566: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 159 op/s; 7093/171858 objects degraded (4.127%); 3504/171858 objects misplaced (2.039%); 838 KiB/s, 13 objects/s recovering 2023-10-01T09:45:05.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/700370040' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:05.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:05.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:05.802 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:05.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:05 smithi190 ceph-mon[160435]: pgmap v1566: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 159 op/s; 7093/171858 objects degraded (4.127%); 3504/171858 objects misplaced (2.039%); 838 KiB/s, 13 objects/s recovering 2023-10-01T09:45:05.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/700370040' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:05.827 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:05.872 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:05.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:05.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:05.977 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:06.003 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:06.052 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:06.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:06.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:06.157 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:06.183 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:06.227 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:06.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:06.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:06.332 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:06.360 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:06.406 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:06.413 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:06.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:06.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:06.479 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:06.520 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:06.547 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:06.592 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:06.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:06.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:06.700 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:06.728 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:06.783 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:06.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:06.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:06.888 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:06.914 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:06.976 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:06.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:06.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:07.048 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:45:07.050 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:45:07.081 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:07.114 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:07.168 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:07.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:07.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:07.274 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:07.303 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:07.630 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:45:07.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:07 smithi190 ceph-mon[160435]: pgmap v1567: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 7093/170949 objects degraded (4.149%); 3504/170949 objects misplaced (2.050%); 749 KiB/s, 11 objects/s recovering 2023-10-01T09:45:07.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:07 smithi138 ceph-mon[185886]: pgmap v1567: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 7093/170949 objects degraded (4.149%); 3504/170949 objects misplaced (2.050%); 749 KiB/s, 11 objects/s recovering 2023-10-01T09:45:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:07 smithi138 ceph-mon[192513]: pgmap v1567: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 7093/170949 objects degraded (4.149%); 3504/170949 objects misplaced (2.050%); 749 KiB/s, 11 objects/s recovering 2023-10-01T09:45:08.356 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:08.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:08.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:08.459 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:08.486 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:08.530 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:08.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:08.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:08.639 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:08.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:08 smithi138 ceph-mon[185886]: from='client.47105 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:08.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:08 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 7093/170949 objects degraded (4.149%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:08.667 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:08 smithi138 ceph-mon[192513]: from='client.47105 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:08.667 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:08 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 7093/170949 objects degraded (4.149%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:08.669 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:08 smithi190 ceph-mon[160435]: from='client.47105 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:08.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:08 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 7093/170949 objects degraded (4.149%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:09.210 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1567,"stamp":"2023-10-01T09:45:06.338242+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3737327563,"num_objects":56983,"num_object_clones":0,"num_object_copies":170949,"num_objects_missing_on_primary":2025,"num_objects_missing":2025,"num_objects_degraded":7093,"num_objects_misplaced":3504,"num_objects_unfound":0,"num_objects_dirty":56983,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":110591,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9098,"num_bytes_recovered":583107599,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110355,"ondisk_log_size":110355,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16465488,"kb_used_data":11410520,"kb_used_omap":39,"kb_used_meta":5054872,"kb_avail":639750576,"statfs":{"total":671965249536,"available":655104589824,"internally_reserved":0,"allocated":11684372480,"data_stored":11674472461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5176189551},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,2,0,2,2,5,0,7],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-96862208,"num_objects":-1478,"num_object_clones":0,"num_object_copies":-4434,"num_objects_missing_on_primary":-103,"num_objects_missing":-103,"num_objects_degraded":-135,"num_objects_misplaced":-85,"num_objects_unfound":0,"num_objects_dirty":-1478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1478,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":7667712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002584"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":310,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.411904+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:44:55.411904+0000","last_peered":"2023-10-01T09:44:55.411904+0000","last_clean":"2023-10-01T09:44:55.411904+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:44:55.411904+0000","last_undegraded":"2023-10-01T09:44:55.411904+0000","last_fullsized":"2023-10-01T09:44:55.411904+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":153,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.407132+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:44:55.407132+0000","last_peered":"2023-10-01T09:44:55.407132+0000","last_clean":"2023-10-01T09:44:55.407132+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:44:55.407132+0000","last_undegraded":"2023-10-01T09:44:55.407132+0000","last_fullsized":"2023-10-01T09:44:55.407132+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":534,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.407070+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:44:55.407070+0000","last_peered":"2023-10-01T09:44:55.407070+0000","last_clean":"2023-10-01T09:44:55.407070+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:44:55.407070+0000","last_undegraded":"2023-10-01T09:44:55.407070+0000","last_fullsized":"2023-10-01T09:44:55.407070+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":592,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.413564+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:44:55.413564+0000","last_peered":"2023-10-01T09:44:55.413564+0000","last_clean":"2023-10-01T09:44:55.413564+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:44:55.413564+0000","last_undegraded":"2023-10-01T09:44:55.413564+0000","last_fullsized":"2023-10-01T09:44:55.413564+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":472,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.396599+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:44:47.396599+0000","last_peered":"2023-10-01T09:44:47.396599+0000","last_clean":"2023-10-01T09:44:47.396599+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:44:47.396599+0000","last_undegraded":"2023-10-01T09:44:47.396599+0000","last_fullsized":"2023-10-01T09:44:47.396599+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":214,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.468116+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:44:47.468116+0000","last_peered":"2023-10-01T09:44:47.468116+0000","last_clean":"2023-10-01T09:44:47.468116+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:44:47.468116+0000","last_undegraded":"2023-10-01T09:44:47.468116+0000","last_fullsized":"2023-10-01T09:44:47.468116+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":590,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.447373+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:44:47.447373+0000","last_peered":"2023-10-01T09:44:47.447373+0000","last_clean":"2023-10-01T09:44:47.447373+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:44:47.447373+0000","last_undegraded":"2023-10-01T09:44:47.447373+0000","last_fullsized":"2023-10-01T09:44:47.447373+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":520,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.423924+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:44:55.423924+0000","last_peered":"2023-10-01T09:44:55.423924+0000","last_clean":"2023-10-01T09:44:55.423924+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:44:55.423924+0000","last_undegraded":"2023-10-01T09:44:55.423924+0000","last_fullsized":"2023-10-01T09:44:55.423924+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":245,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.415252+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:44:47.415252+0000","last_peered":"2023-10-01T09:44:47.415252+0000","last_clean":"2023-10-01T09:44:47.415252+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:44:47.415252+0000","last_undegraded":"2023-10-01T09:44:47.415252+0000","last_fullsized":"2023-10-01T09:44:47.415252+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":53,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.414688+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:44:55.414688+0000","last_peered":"2023-10-01T09:44:55.414688+0000","last_clean":"2023-10-01T09:44:55.414688+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:44:55.414688+0000","last_undegraded":"2023-10-01T09:44:55.414688+0000","last_fullsized":"2023-10-01T09:44:55.414688+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":59,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.415611+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:44:55.415611+0000","last_peered":"2023-10-01T09:44:55.415611+0000","last_clean":"2023-10-01T09:44:55.415611+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:44:55.415611+0000","last_undegraded":"2023-10-01T09:44:55.415611+0000","last_fullsized":"2023-10-01T09:44:55.415611+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":57,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.407320+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:44:55.407320+0000","last_peered":"2023-10-01T09:44:55.407320+0000","last_clean":"2023-10-01T09:44:55.407320+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:44:55.407320+0000","last_undegraded":"2023-10-01T09:44:55.407320+0000","last_fullsized":"2023-10-01T09:44:55.407320+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":435,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.412897+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:44:55.412897+0000","last_peered":"2023-10-01T09:44:55.412897+0000","last_clean":"2023-10-01T09:44:55.412897+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:44:55.412897+0000","last_undegraded":"2023-10-01T09:44:55.412897+0000","last_fullsized":"2023-10-01T09:44:55.412897+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":316,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.412208+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:44:55.412208+0000","last_peered":"2023-10-01T09:44:55.412208+0000","last_clean":"2023-10-01T09:44:55.412208+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:44:55.412208+0000","last_undegraded":"2023-10-01T09:44:55.412208+0000","last_fullsized":"2023-10-01T09:44:55.412208+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":652,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.414905+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:44:55.414905+0000","last_peered":"2023-10-01T09:44:55.414905+0000","last_clean":"2023-10-01T09:44:55.414905+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:44:55.414905+0000","last_undegraded":"2023-10-01T09:44:55.414905+0000","last_fullsized":"2023-10-01T09:44:55.414905+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":389,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.415411+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:44:55.415411+0000","last_peered":"2023-10-01T09:44:55.415411+0000","last_clean":"2023-10-01T09:44:55.415411+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:44:55.415411+0000","last_undegraded":"2023-10-01T09:44:55.415411+0000","last_fullsized":"2023-10-01T09:44:55.415411+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":62,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.414729+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:44:55.414729+0000","last_peered":"2023-10-01T09:44:55.414729+0000","last_clean":"2023-10-01T09:44:55.414729+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:44:55.414729+0000","last_undegraded":"2023-10-01T09:44:55.414729+0000","last_fullsized":"2023-10-01T09:44:55.414729+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":47,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.414660+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:44:47.414660+0000","last_peered":"2023-10-01T09:44:47.414660+0000","last_clean":"2023-10-01T09:44:47.414660+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:44:47.414660+0000","last_undegraded":"2023-10-01T09:44:47.414660+0000","last_fullsized":"2023-10-01T09:44:47.414660+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":545,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.416019+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:44:55.416019+0000","last_peered":"2023-10-01T09:44:55.416019+0000","last_clean":"2023-10-01T09:44:55.416019+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:44:55.416019+0000","last_undegraded":"2023-10-01T09:44:55.416019+0000","last_fullsized":"2023-10-01T09:44:55.416019+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":556,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.397386+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:44:47.397386+0000","last_peered":"2023-10-01T09:44:47.397386+0000","last_clean":"2023-10-01T09:44:47.397386+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:44:47.397386+0000","last_undegraded":"2023-10-01T09:44:47.397386+0000","last_fullsized":"2023-10-01T09:44:47.397386+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":412,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.396458+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:44:47.396458+0000","last_peered":"2023-10-01T09:44:47.396458+0000","last_clean":"2023-10-01T09:44:47.396458+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:44:47.396458+0000","last_undegraded":"2023-10-01T09:44:47.396458+0000","last_fullsized":"2023-10-01T09:44:47.396458+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":62,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.415127+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:44:55.415127+0000","last_peered":"2023-10-01T09:44:55.415127+0000","last_clean":"2023-10-01T09:44:55.415127+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:44:55.415127+0000","last_undegraded":"2023-10-01T09:44:55.415127+0000","last_fullsized":"2023-10-01T09:44:55.415127+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"533'6878","reported_seq":6944,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:45:01.838490+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:45:01.838490+0000","last_peered":"2023-10-01T09:45:01.838490+0000","last_clean":"2023-10-01T09:45:01.838490+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:45:01.838490+0000","last_undegraded":"2023-10-01T09:45:01.838490+0000","last_fullsized":"2023-10-01T09:45:01.838490+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6878,"log_dups_size":0,"ondisk_log_size":6878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230293504,"num_objects":3514,"num_object_clones":0,"num_object_copies":10542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6878,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":57,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.414962+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:44:55.414962+0000","last_peered":"2023-10-01T09:44:55.414962+0000","last_clean":"2023-10-01T09:44:55.414962+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:44:55.414962+0000","last_undegraded":"2023-10-01T09:44:55.414962+0000","last_fullsized":"2023-10-01T09:44:55.414962+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":253,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.414800+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:44:47.414800+0000","last_peered":"2023-10-01T09:44:47.414800+0000","last_clean":"2023-10-01T09:44:47.414800+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:44:47.414800+0000","last_undegraded":"2023-10-01T09:44:47.414800+0000","last_fullsized":"2023-10-01T09:44:47.414800+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"533'6866","reported_seq":11946,"reported_epoch":533,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:45:01.986690+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:45:01.986690+0000","last_peered":"2023-10-01T09:45:01.986690+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:45:01.986690+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6866,"log_dups_size":0,"ondisk_log_size":6866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231997440,"num_objects":3540,"num_object_clones":0,"num_object_copies":10620,"num_objects_missing_on_primary":2025,"num_objects_missing":2025,"num_objects_degraded":5497,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6866,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1663,"num_bytes_recovered":108986368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1447},{"shards":"0,4,6","objects":2025},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":528,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.414869+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:44:55.414869+0000","last_peered":"2023-10-01T09:44:55.414869+0000","last_clean":"2023-10-01T09:44:55.414869+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:44:55.414869+0000","last_undegraded":"2023-10-01T09:44:55.414869+0000","last_fullsized":"2023-10-01T09:44:55.414869+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":590,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:45:00.265225+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:45:00.265225+0000","last_peered":"2023-10-01T09:45:00.265225+0000","last_clean":"2023-10-01T09:45:00.265225+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:45:00.265225+0000","last_undegraded":"2023-10-01T09:45:00.265225+0000","last_fullsized":"2023-10-01T09:45:00.265225+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"533'6863","reported_seq":6932,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:45:01.017408+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:45:01.017408+0000","last_peered":"2023-10-01T09:45:01.017408+0000","last_clean":"2023-10-01T09:45:01.017408+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:45:01.017408+0000","last_undegraded":"2023-10-01T09:45:01.017408+0000","last_fullsized":"2023-10-01T09:45:01.017408+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6863,"log_dups_size":0,"ondisk_log_size":6863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233897984,"num_objects":3569,"num_object_clones":0,"num_object_copies":10707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6863,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":614,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.415718+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:44:55.415718+0000","last_peered":"2023-10-01T09:44:55.415718+0000","last_clean":"2023-10-01T09:44:55.415718+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:44:55.415718+0000","last_undegraded":"2023-10-01T09:44:55.415718+0000","last_fullsized":"2023-10-01T09:44:55.415718+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":410,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.407516+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:44:55.407516+0000","last_peered":"2023-10-01T09:44:55.407516+0000","last_clean":"2023-10-01T09:44:55.407516+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:44:55.407516+0000","last_undegraded":"2023-10-01T09:44:55.407516+0000","last_fullsized":"2023-10-01T09:44:55.407516+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"533'6818","reported_seq":13492,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:45:01.116588+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:45:01.116588+0000","last_peered":"2023-10-01T09:45:01.116588+0000","last_clean":"2023-10-01T09:45:01.116588+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:45:01.116588+0000","last_undegraded":"2023-10-01T09:45:01.116588+0000","last_fullsized":"2023-10-01T09:45:01.116588+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6818,"log_dups_size":0,"ondisk_log_size":6818,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234881024,"num_objects":3584,"num_object_clones":0,"num_object_copies":10752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6818,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":253,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.414917+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:44:47.414917+0000","last_peered":"2023-10-01T09:44:47.414917+0000","last_clean":"2023-10-01T09:44:47.414917+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:44:47.414917+0000","last_undegraded":"2023-10-01T09:44:47.414917+0000","last_fullsized":"2023-10-01T09:44:47.414917+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":512,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.396758+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:44:47.396758+0000","last_peered":"2023-10-01T09:44:47.396758+0000","last_clean":"2023-10-01T09:44:47.396758+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:44:47.396758+0000","last_undegraded":"2023-10-01T09:44:47.396758+0000","last_fullsized":"2023-10-01T09:44:47.396758+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"534'6902","reported_seq":6972,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:05.025310+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:45:05.025310+0000","last_peered":"2023-10-01T09:45:05.025310+0000","last_clean":"2023-10-01T09:45:05.025310+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:45:05.025310+0000","last_undegraded":"2023-10-01T09:45:05.025310+0000","last_fullsized":"2023-10-01T09:45:05.025310+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6902,"log_dups_size":0,"ondisk_log_size":6902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236322816,"num_objects":3606,"num_object_clones":0,"num_object_copies":10818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6902,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":114,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.412558+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:44:55.412558+0000","last_peered":"2023-10-01T09:44:55.412558+0000","last_clean":"2023-10-01T09:44:55.412558+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:44:55.412558+0000","last_undegraded":"2023-10-01T09:44:55.412558+0000","last_fullsized":"2023-10-01T09:44:55.412558+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":188,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.413042+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:44:55.413042+0000","last_peered":"2023-10-01T09:44:55.413042+0000","last_clean":"2023-10-01T09:44:55.413042+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:44:55.413042+0000","last_undegraded":"2023-10-01T09:44:55.413042+0000","last_fullsized":"2023-10-01T09:44:55.413042+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"533'6800","reported_seq":6936,"reported_epoch":533,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:45:01.899308+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:45:01.899308+0000","last_peered":"2023-10-01T09:45:01.899308+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:45:01.899308+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6800,"log_dups_size":0,"ondisk_log_size":6800,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233832448,"num_objects":3568,"num_object_clones":0,"num_object_copies":10704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1586,"num_objects_misplaced":3504,"num_objects_unfound":0,"num_objects_dirty":3568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6800,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":2031616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":3504},{"shards":"4,5","objects":-1522}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":432,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.412045+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:44:55.412045+0000","last_peered":"2023-10-01T09:44:55.412045+0000","last_clean":"2023-10-01T09:44:55.412045+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:44:55.412045+0000","last_undegraded":"2023-10-01T09:44:55.412045+0000","last_fullsized":"2023-10-01T09:44:55.412045+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":593,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.407467+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:44:55.407467+0000","last_peered":"2023-10-01T09:44:55.407467+0000","last_clean":"2023-10-01T09:44:55.407467+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:44:55.407467+0000","last_undegraded":"2023-10-01T09:44:55.407467+0000","last_fullsized":"2023-10-01T09:44:55.407467+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"534'6785","reported_seq":6861,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.436332+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:45:02.436332+0000","last_peered":"2023-10-01T09:45:02.436332+0000","last_clean":"2023-10-01T09:45:02.436332+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:45:02.436332+0000","last_undegraded":"2023-10-01T09:45:02.436332+0000","last_fullsized":"2023-10-01T09:45:02.436332+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6785,"log_dups_size":0,"ondisk_log_size":6785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235470848,"num_objects":3593,"num_object_clones":0,"num_object_copies":10779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6785,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":428,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.411883+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:44:55.411883+0000","last_peered":"2023-10-01T09:44:55.411883+0000","last_clean":"2023-10-01T09:44:55.411883+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:44:55.411883+0000","last_undegraded":"2023-10-01T09:44:55.411883+0000","last_fullsized":"2023-10-01T09:44:55.411883+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":269,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.413231+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:44:55.413231+0000","last_peered":"2023-10-01T09:44:55.413231+0000","last_clean":"2023-10-01T09:44:55.413231+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:44:55.413231+0000","last_undegraded":"2023-10-01T09:44:55.413231+0000","last_fullsized":"2023-10-01T09:44:55.413231+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"534'6950","reported_seq":7017,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:05.729374+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:45:05.729374+0000","last_peered":"2023-10-01T09:45:05.729374+0000","last_clean":"2023-10-01T09:45:05.729374+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:45:05.729374+0000","last_undegraded":"2023-10-01T09:45:05.729374+0000","last_fullsized":"2023-10-01T09:45:05.729374+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6950,"log_dups_size":0,"ondisk_log_size":6950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232521728,"num_objects":3548,"num_object_clones":0,"num_object_copies":10644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6950,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":397,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.406966+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:44:55.406966+0000","last_peered":"2023-10-01T09:44:55.406966+0000","last_clean":"2023-10-01T09:44:55.406966+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:44:55.406966+0000","last_undegraded":"2023-10-01T09:44:55.406966+0000","last_fullsized":"2023-10-01T09:44:55.406966+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":444,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.407797+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:44:55.407797+0000","last_peered":"2023-10-01T09:44:55.407797+0000","last_clean":"2023-10-01T09:44:55.407797+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:44:55.407797+0000","last_undegraded":"2023-10-01T09:44:55.407797+0000","last_fullsized":"2023-10-01T09:44:55.407797+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"534'6737","reported_seq":6804,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:05.724657+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:45:05.724657+0000","last_peered":"2023-10-01T09:45:05.724657+0000","last_clean":"2023-10-01T09:45:05.724657+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:45:05.724657+0000","last_undegraded":"2023-10-01T09:45:05.724657+0000","last_fullsized":"2023-10-01T09:45:05.724657+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6737,"log_dups_size":0,"ondisk_log_size":6737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225509376,"num_objects":3441,"num_object_clones":0,"num_object_copies":10323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6737,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":927,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.415225+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:44:55.415225+0000","last_peered":"2023-10-01T09:44:55.415225+0000","last_clean":"2023-10-01T09:44:55.415225+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:44:55.415225+0000","last_undegraded":"2023-10-01T09:44:55.415225+0000","last_fullsized":"2023-10-01T09:44:55.415225+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":263,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.415152+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:44:47.415152+0000","last_peered":"2023-10-01T09:44:47.415152+0000","last_clean":"2023-10-01T09:44:47.415152+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:44:47.415152+0000","last_undegraded":"2023-10-01T09:44:47.415152+0000","last_fullsized":"2023-10-01T09:44:47.415152+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":58,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.407388+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:44:55.407388+0000","last_peered":"2023-10-01T09:44:55.407388+0000","last_clean":"2023-10-01T09:44:55.407388+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:44:55.407388+0000","last_undegraded":"2023-10-01T09:44:55.407388+0000","last_fullsized":"2023-10-01T09:44:55.407388+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"533'6761","reported_seq":6830,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:45:00.993315+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:45:00.993315+0000","last_peered":"2023-10-01T09:45:00.993315+0000","last_clean":"2023-10-01T09:45:00.993315+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:45:00.993315+0000","last_undegraded":"2023-10-01T09:45:00.993315+0000","last_fullsized":"2023-10-01T09:45:00.993315+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6761,"log_dups_size":0,"ondisk_log_size":6761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236519424,"num_objects":3609,"num_object_clones":0,"num_object_copies":10827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6761,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1198,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.414897+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:44:47.414897+0000","last_peered":"2023-10-01T09:44:47.414897+0000","last_clean":"2023-10-01T09:44:47.414897+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:44:47.414897+0000","last_undegraded":"2023-10-01T09:44:47.414897+0000","last_fullsized":"2023-10-01T09:44:47.414897+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":409,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.412414+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:44:55.412414+0000","last_peered":"2023-10-01T09:44:55.412414+0000","last_clean":"2023-10-01T09:44:55.412414+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:44:55.412414+0000","last_undegraded":"2023-10-01T09:44:55.412414+0000","last_fullsized":"2023-10-01T09:44:55.412414+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":236,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.415076+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:44:47.415076+0000","last_peered":"2023-10-01T09:44:47.415076+0000","last_clean":"2023-10-01T09:44:47.415076+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:44:47.415076+0000","last_undegraded":"2023-10-01T09:44:47.415076+0000","last_fullsized":"2023-10-01T09:44:47.415076+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"533'6844","reported_seq":14131,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:45:01.879022+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:45:01.879022+0000","last_peered":"2023-10-01T09:45:01.879022+0000","last_clean":"2023-10-01T09:45:01.879022+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:45:01.879022+0000","last_undegraded":"2023-10-01T09:45:01.879022+0000","last_fullsized":"2023-10-01T09:45:01.879022+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6844,"log_dups_size":0,"ondisk_log_size":6844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231473152,"num_objects":3532,"num_object_clones":0,"num_object_copies":10596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6844,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":878,"reported_epoch":532,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:44:47.396157+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:44:47.396157+0000","last_peered":"2023-10-01T09:44:47.396157+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:44:47.396157+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:44:47.396157+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":597,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.406980+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:44:55.406980+0000","last_peered":"2023-10-01T09:44:55.406980+0000","last_clean":"2023-10-01T09:44:55.406980+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:44:55.406980+0000","last_undegraded":"2023-10-01T09:44:55.406980+0000","last_fullsized":"2023-10-01T09:44:55.406980+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":58,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.577986+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:45:02.577986+0000","last_peered":"2023-10-01T09:45:02.577986+0000","last_clean":"2023-10-01T09:45:02.577986+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:45:02.577986+0000","last_undegraded":"2023-10-01T09:45:02.577986+0000","last_fullsized":"2023-10-01T09:45:02.577986+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"534'6761","reported_seq":6831,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:04.995731+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:45:04.995731+0000","last_peered":"2023-10-01T09:45:04.995731+0000","last_clean":"2023-10-01T09:45:04.995731+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:45:04.995731+0000","last_undegraded":"2023-10-01T09:45:04.995731+0000","last_fullsized":"2023-10-01T09:45:04.995731+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6761,"log_dups_size":0,"ondisk_log_size":6761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230096896,"num_objects":3511,"num_object_clones":0,"num_object_copies":10533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6761,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1649,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.414949+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:44:47.414949+0000","last_peered":"2023-10-01T09:44:47.414949+0000","last_clean":"2023-10-01T09:44:47.414949+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:44:47.414949+0000","last_undegraded":"2023-10-01T09:44:47.414949+0000","last_fullsized":"2023-10-01T09:44:47.414949+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":564,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.414483+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:44:55.414483+0000","last_peered":"2023-10-01T09:44:55.414483+0000","last_clean":"2023-10-01T09:44:55.414483+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:44:55.414483+0000","last_undegraded":"2023-10-01T09:44:55.414483+0000","last_fullsized":"2023-10-01T09:44:55.414483+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":341,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.415017+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:44:47.415017+0000","last_peered":"2023-10-01T09:44:47.415017+0000","last_clean":"2023-10-01T09:44:47.415017+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:44:47.415017+0000","last_undegraded":"2023-10-01T09:44:47.415017+0000","last_fullsized":"2023-10-01T09:44:47.415017+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":332,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.413056+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:44:55.413056+0000","last_peered":"2023-10-01T09:44:55.413056+0000","last_clean":"2023-10-01T09:44:55.413056+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:44:55.413056+0000","last_undegraded":"2023-10-01T09:44:55.413056+0000","last_fullsized":"2023-10-01T09:44:55.413056+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"534'6946","reported_seq":7013,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:05.380826+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:45:05.380826+0000","last_peered":"2023-10-01T09:45:05.380826+0000","last_clean":"2023-10-01T09:45:05.380826+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:45:05.380826+0000","last_undegraded":"2023-10-01T09:45:05.380826+0000","last_fullsized":"2023-10-01T09:45:05.380826+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6946,"log_dups_size":0,"ondisk_log_size":6946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235012096,"num_objects":3586,"num_object_clones":0,"num_object_copies":10758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6946,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":741,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.414572+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:44:55.414572+0000","last_peered":"2023-10-01T09:44:55.414572+0000","last_clean":"2023-10-01T09:44:55.414572+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:44:55.414572+0000","last_undegraded":"2023-10-01T09:44:55.414572+0000","last_fullsized":"2023-10-01T09:44:55.414572+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":573,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.397514+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:44:47.397514+0000","last_peered":"2023-10-01T09:44:47.397514+0000","last_clean":"2023-10-01T09:44:47.397514+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:44:47.397514+0000","last_undegraded":"2023-10-01T09:44:47.397514+0000","last_fullsized":"2023-10-01T09:44:47.397514+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":572,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.397149+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:44:47.397149+0000","last_peered":"2023-10-01T09:44:47.397149+0000","last_clean":"2023-10-01T09:44:47.397149+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:44:47.397149+0000","last_undegraded":"2023-10-01T09:44:47.397149+0000","last_fullsized":"2023-10-01T09:44:47.397149+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"533'6832","reported_seq":6902,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:45:01.793095+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:45:01.793095+0000","last_peered":"2023-10-01T09:45:01.793095+0000","last_clean":"2023-10-01T09:45:01.793095+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:45:01.793095+0000","last_undegraded":"2023-10-01T09:45:01.793095+0000","last_fullsized":"2023-10-01T09:45:01.793095+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6832,"log_dups_size":0,"ondisk_log_size":6832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237699096,"num_objects":3628,"num_object_clones":0,"num_object_copies":10884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3628,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6832,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":972,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.415821+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:44:55.415821+0000","last_peered":"2023-10-01T09:44:55.415821+0000","last_clean":"2023-10-01T09:44:55.415821+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:44:55.415821+0000","last_undegraded":"2023-10-01T09:44:55.415821+0000","last_fullsized":"2023-10-01T09:44:55.415821+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":590,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.415003+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:44:47.415003+0000","last_peered":"2023-10-01T09:44:47.415003+0000","last_clean":"2023-10-01T09:44:47.415003+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:44:47.415003+0000","last_undegraded":"2023-10-01T09:44:47.415003+0000","last_fullsized":"2023-10-01T09:44:47.415003+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":443,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.415021+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:44:47.415021+0000","last_peered":"2023-10-01T09:44:47.415021+0000","last_clean":"2023-10-01T09:44:47.415021+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:44:47.415021+0000","last_undegraded":"2023-10-01T09:44:47.415021+0000","last_fullsized":"2023-10-01T09:44:47.415021+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"534'6840","reported_seq":6915,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:05.434516+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:45:05.434516+0000","last_peered":"2023-10-01T09:45:05.434516+0000","last_clean":"2023-10-01T09:45:05.434516+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:45:05.434516+0000","last_undegraded":"2023-10-01T09:45:05.434516+0000","last_fullsized":"2023-10-01T09:45:05.434516+0000","mapping_epoch":479,"log_start":"480'100","ondisk_log_start":"480'100","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6740,"log_dups_size":0,"ondisk_log_size":6740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230424576,"num_objects":3516,"num_object_clones":0,"num_object_copies":10548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6840,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":473,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.415168+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:44:55.415168+0000","last_peered":"2023-10-01T09:44:55.415168+0000","last_clean":"2023-10-01T09:44:55.415168+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:44:55.415168+0000","last_undegraded":"2023-10-01T09:44:55.415168+0000","last_fullsized":"2023-10-01T09:44:55.415168+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1167,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.407718+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:44:55.407718+0000","last_peered":"2023-10-01T09:44:55.407718+0000","last_clean":"2023-10-01T09:44:55.407718+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:44:55.407718+0000","last_undegraded":"2023-10-01T09:44:55.407718+0000","last_fullsized":"2023-10-01T09:44:55.407718+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":62,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.414450+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:44:55.414450+0000","last_peered":"2023-10-01T09:44:55.414450+0000","last_clean":"2023-10-01T09:44:55.414450+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:44:55.414450+0000","last_undegraded":"2023-10-01T09:44:55.414450+0000","last_fullsized":"2023-10-01T09:44:55.414450+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"534'6902","reported_seq":6977,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:05.410871+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:45:05.410871+0000","last_peered":"2023-10-01T09:45:05.410871+0000","last_clean":"2023-10-01T09:45:05.410871+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:45:05.410871+0000","last_undegraded":"2023-10-01T09:45:05.410871+0000","last_fullsized":"2023-10-01T09:45:05.410871+0000","mapping_epoch":479,"log_start":"480'200","ondisk_log_start":"480'200","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6702,"log_dups_size":0,"ondisk_log_size":6702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236584960,"num_objects":3610,"num_object_clones":0,"num_object_copies":10830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6902,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":66,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.415039+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:44:55.415039+0000","last_peered":"2023-10-01T09:44:55.415039+0000","last_clean":"2023-10-01T09:44:55.415039+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:44:55.415039+0000","last_undegraded":"2023-10-01T09:44:55.415039+0000","last_fullsized":"2023-10-01T09:44:55.415039+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":550,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.397241+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:44:47.397241+0000","last_peered":"2023-10-01T09:44:47.397241+0000","last_clean":"2023-10-01T09:44:47.397241+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:44:47.397241+0000","last_undegraded":"2023-10-01T09:44:47.397241+0000","last_fullsized":"2023-10-01T09:44:47.397241+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":256,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.414866+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:44:47.414866+0000","last_peered":"2023-10-01T09:44:47.414866+0000","last_clean":"2023-10-01T09:44:47.414866+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:44:47.414866+0000","last_undegraded":"2023-10-01T09:44:47.414866+0000","last_fullsized":"2023-10-01T09:44:47.414866+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":499,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.396906+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:44:47.396906+0000","last_peered":"2023-10-01T09:44:47.396906+0000","last_clean":"2023-10-01T09:44:47.396906+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:44:47.396906+0000","last_undegraded":"2023-10-01T09:44:47.396906+0000","last_fullsized":"2023-10-01T09:44:47.396906+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":614,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.407306+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:44:55.407306+0000","last_peered":"2023-10-01T09:44:55.407306+0000","last_clean":"2023-10-01T09:44:55.407306+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:44:55.407306+0000","last_undegraded":"2023-10-01T09:44:55.407306+0000","last_fullsized":"2023-10-01T09:44:55.407306+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":268,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.414837+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:44:47.414837+0000","last_peered":"2023-10-01T09:44:47.414837+0000","last_clean":"2023-10-01T09:44:47.414837+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:44:47.414837+0000","last_undegraded":"2023-10-01T09:44:47.414837+0000","last_fullsized":"2023-10-01T09:44:47.414837+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":57,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.407505+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:44:55.407505+0000","last_peered":"2023-10-01T09:44:55.407505+0000","last_clean":"2023-10-01T09:44:55.407505+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:44:55.407505+0000","last_undegraded":"2023-10-01T09:44:55.407505+0000","last_fullsized":"2023-10-01T09:44:55.407505+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":268,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.415178+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:44:47.415178+0000","last_peered":"2023-10-01T09:44:47.415178+0000","last_clean":"2023-10-01T09:44:47.415178+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:44:47.415178+0000","last_undegraded":"2023-10-01T09:44:47.415178+0000","last_fullsized":"2023-10-01T09:44:47.415178+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":445,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.397649+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:44:47.397649+0000","last_peered":"2023-10-01T09:44:47.397649+0000","last_clean":"2023-10-01T09:44:47.397649+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:44:47.397649+0000","last_undegraded":"2023-10-01T09:44:47.397649+0000","last_fullsized":"2023-10-01T09:44:47.397649+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":75,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.407567+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:44:55.407567+0000","last_peered":"2023-10-01T09:44:55.407567+0000","last_clean":"2023-10-01T09:44:55.407567+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:44:55.407567+0000","last_undegraded":"2023-10-01T09:44:55.407567+0000","last_fullsized":"2023-10-01T09:44:55.407567+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":490,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.397439+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:44:47.397439+0000","last_peered":"2023-10-01T09:44:47.397439+0000","last_clean":"2023-10-01T09:44:47.397439+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:44:47.397439+0000","last_undegraded":"2023-10-01T09:44:47.397439+0000","last_fullsized":"2023-10-01T09:44:47.397439+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":690,"reported_epoch":532,"state":"active+clean","last_fresh":"2023-10-01T09:44:47.397226+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:44:47.397226+0000","last_peered":"2023-10-01T09:44:47.397226+0000","last_clean":"2023-10-01T09:44:47.397226+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:44:47.397226+0000","last_undegraded":"2023-10-01T09:44:47.397226+0000","last_fullsized":"2023-10-01T09:44:47.397226+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":385,"reported_epoch":533,"state":"active+clean","last_fresh":"2023-10-01T09:44:55.411777+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:44:55.411777+0000","last_peered":"2023-10-01T09:44:55.411777+0000","last_clean":"2023-10-01T09:44:55.411777+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:44:55.411777+0000","last_undegraded":"2023-10-01T09:44:55.411777+0000","last_fullsized":"2023-10-01T09:44:55.411777+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3732537368,"num_objects":56955,"num_object_clones":0,"num_object_copies":170865,"num_objects_missing_on_primary":2025,"num_objects_missing":2025,"num_objects_degraded":7083,"num_objects_misplaced":3504,"num_objects_unfound":0,"num_objects_dirty":56955,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":109485,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8587,"num_bytes_recovered":562757632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11619676160,"data_stored":11619663944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109185,"ondisk_log_size":109185,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430734,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2040768,"kb_used_data":1137656,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":91704384,"statfs":{"total":95995035648,"available":93905289216,"internally_reserved":0,"allocated":1164959744,"data_stored":1164019448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.79300000000000004,"15min":2.6280000000000001},"min":{"1min":0.254,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":3.3849999999999998,"5min":6.8550000000000004,"15min":108.90000000000001},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.626,"15min":2.7559999999999998},"min":{"1min":0.25600000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.98399999999999999,"5min":5.6639999999999997,"15min":109.187},"last":0.46899999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":4.2910000000000004,"15min":5.2409999999999997},"min":{"1min":0.28999999999999998,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":1.4470000000000001,"5min":229.161,"15min":229.161},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.42999999999999999,"5min":4.2759999999999998,"15min":5.2510000000000003},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":0.96499999999999997,"5min":230.126,"15min":230.126},"last":0.61499999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:44:54 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.81399999999999995,"15min":1.6519999999999999},"min":{"1min":0.30399999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.4300000000000002,"5min":5.3090000000000002,"15min":14.901999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.76100000000000001,"15min":1.6299999999999999},"min":{"1min":0.33200000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4630000000000001,"5min":5.6929999999999996,"15min":14.682},"last":0.55000000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:44:09 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.96899999999999997,"15min":2.569},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.018,"5min":6.0069999999999997,"15min":80.352999999999994},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.51300000000000001,"5min":1.1359999999999999,"15min":2.5510000000000002},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.026,"5min":14.528,"15min":80.364000000000004},"last":0.64700000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.80200000000000005,"15min":1.0409999999999999},"min":{"1min":0.31,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":3.4609999999999999,"5min":6.8769999999999998,"15min":9.6020000000000003},"last":0.313},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.82599999999999996,"15min":0.999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28000000000000003},"max":{"1min":0.63900000000000001,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.38500000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":2.177,"15min":1.5800000000000001},"min":{"1min":0.34399999999999997,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":0.871,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.502},{"interface":"front","average":{"1min":0.61299999999999999,"5min":2.1789999999999998,"15min":1.6619999999999999},"min":{"1min":0.307,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.3319999999999999,"5min":135.339,"15min":135.339},"last":0.36899999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":1.1639999999999999,"15min":1.1359999999999999},"min":{"1min":0.38,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":0.998,"5min":11.58,"15min":11.853999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.94199999999999995,"15min":1.097},"min":{"1min":0.38300000000000001,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":2.298,"5min":13.021000000000001,"15min":14.584},"last":0.41099999999999998}]}]},{"osd":6,"up_from":294,"seq":1262720385244,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1948020,"kb_used_data":1614508,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":91797132,"statfs":{"total":95995035648,"available":94000263168,"internally_reserved":0,"allocated":1653256192,"data_stored":1652320171,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":2.0630000000000002,"15min":4.0090000000000003},"min":{"1min":0.247,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":3.2029999999999998,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.41399999999999998,"5min":2.1739999999999999,"15min":4.0730000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.221},"max":{"1min":0.63900000000000001,"5min":158.37700000000001,"15min":158.37700000000001},"last":2.1760000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":3.1379999999999999,"15min":4.8600000000000003},"min":{"1min":0.29199999999999998,"5min":0.254,"15min":0.24199999999999999},"max":{"1min":0.58899999999999997,"5min":168.947,"15min":168.947},"last":0.252},{"interface":"front","average":{"1min":0.56000000000000005,"5min":3.141,"15min":4.8200000000000003},"min":{"1min":0.32500000000000001,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":2.8290000000000002,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.59899999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.78400000000000003,"15min":1.5620000000000001},"min":{"1min":0.32500000000000001,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":0.65700000000000003,"5min":8.657,"15min":10.92},"last":0.311},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.88600000000000001,"15min":1.5629999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":2.863,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.57299999999999995}]},{"osd":3,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.96499999999999997,"15min":2.101},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.3149999999999999,"5min":9.0180000000000007,"15min":89.753},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.439,"5min":1.097,"15min":2.0179999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.61199999999999999,"5min":10.191000000000001,"15min":89.849999999999994},"last":2.1379999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":1.5049999999999999,"15min":1.0740000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.7749999999999999,"5min":63.134,"15min":63.134},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.478,"15min":1.075},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":3.2519999999999998,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.48199999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:44:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.246,"15min":1.073},"min":{"1min":0.28799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":3.2890000000000001,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.624,"5min":1.3260000000000001,"15min":1.1859999999999999},"min":{"1min":0.29899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.887,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.72799999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:44:15 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.73999999999999999,"15min":1.2729999999999999},"min":{"1min":0.32700000000000001,"5min":0.29899999999999999,"15min":0.27900000000000003},"max":{"1min":3.339,"5min":3.8279999999999998,"15min":94.245999999999995},"last":0.628},{"interface":"front","average":{"1min":0.496,"5min":0.96999999999999997,"15min":1.3640000000000001},"min":{"1min":0.374,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":0.68600000000000005,"5min":8.6899999999999995,"15min":94.715000000000003},"last":2.202}]}]},{"osd":1,"up_from":94,"seq":403726926221,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1505084,"kb_used_data":695092,"kb_used_omap":7,"kb_used_meta":809976,"kb_avail":92240068,"statfs":{"total":95995035648,"available":94453829632,"internally_reserved":0,"allocated":711774208,"data_stored":709724576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":829415847},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":1.7789999999999999,"15min":4.0039999999999996},"min":{"1min":0.186,"5min":0.186,"15min":0.182},"max":{"1min":1.2889999999999999,"5min":46.692999999999998,"15min":154.21299999999999},"last":32.735999999999997},{"interface":"front","average":{"1min":0.42099999999999999,"5min":1.99,"15min":3.5329999999999999},"min":{"1min":0.223,"5min":0.20899999999999999,"15min":0.188},"max":{"1min":1.702,"5min":46.701000000000001,"15min":95.236999999999995},"last":32.591999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:44:39 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.42399999999999999,"15min":2.4889999999999999},"min":{"1min":0.23300000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":1.3240000000000001,"5min":2.8740000000000001,"15min":56.573},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.61199999999999999,"15min":3.1829999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":2.2200000000000002,"5min":6.9710000000000001,"15min":154.197},"last":0.28899999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":1.2769999999999999,"15min":3.7909999999999999},"min":{"1min":0.26500000000000001,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":1.7889999999999999,"5min":18.503,"15min":200.56800000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.2330000000000001,"15min":4.5039999999999996},"min":{"1min":0.253,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":1.9370000000000001,"5min":16.515000000000001,"15min":200.63800000000001},"last":0.309}]},{"osd":4,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.147,"15min":2.2530000000000001},"min":{"1min":0.35499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.0550000000000002,"5min":19.539000000000001,"15min":154.101},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.46899999999999997,"5min":1.0029999999999999,"15min":2.399},"min":{"1min":0.29499999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.59799999999999998,"5min":10.084,"15min":154.56999999999999},"last":0.40799999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:44:39 2023","interfaces":[{"interface":"back","average":{"1min":3.593,"5min":1.272,"15min":2.2759999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":59.561,"5min":59.561,"15min":154.22999999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":3.5979999999999999,"5min":1.24,"15min":2.5680000000000001},"min":{"1min":0.35699999999999998,"5min":0.309,"15min":0.307},"max":{"1min":59.536000000000001,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.438}]},{"osd":6,"last update":"Sun Oct 1 09:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.236,"15min":2.5539999999999998},"min":{"1min":0.34300000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.4399999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.81399999999999995,"5min":1.335,"15min":2.4119999999999999},"min":{"1min":0.34699999999999998,"5min":0.34599999999999997,"15min":0.32700000000000001},"max":{"1min":2.5630000000000002,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.41899999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:44:02 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.3109999999999999,"15min":2.5840000000000001},"min":{"1min":0.38400000000000001,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":2.8170000000000002,"5min":19.431999999999999,"15min":154.20699999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.77800000000000002,"5min":1.2330000000000001,"15min":2.6379999999999999},"min":{"1min":0.45200000000000001,"5min":0.40000000000000002,"15min":0.34300000000000003},"max":{"1min":2.8290000000000002,"5min":10.111000000000001,"15min":154.51599999999999},"last":0.48299999999999998}]}]},{"osd":0,"up_from":89,"seq":382252089755,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3871068,"kb_used_data":2961876,"kb_used_omap":8,"kb_used_meta":909175,"kb_avail":89874084,"statfs":{"total":95995035648,"available":92031062016,"internally_reserved":0,"allocated":3032961024,"data_stored":3030940270,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":930996162},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:44:16 2023","interfaces":[{"interface":"back","average":{"1min":0.317,"5min":1.1839999999999999,"15min":4.1120000000000001},"min":{"1min":0.21199999999999999,"5min":0.17799999999999999,"15min":0.17399999999999999},"max":{"1min":0.46300000000000002,"5min":52.316000000000003,"15min":216.821},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.317,"5min":2.7919999999999998,"15min":4.9809999999999999},"min":{"1min":0.24399999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.435,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.71599999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.69999999999999996,"15min":2.0339999999999998},"min":{"1min":0.29299999999999998,"5min":0.246,"15min":0.20100000000000001},"max":{"1min":4.2430000000000003,"5min":4.9459999999999997,"15min":19.788},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.70799999999999996,"15min":2.1200000000000001},"min":{"1min":0.30399999999999999,"5min":0.23000000000000001,"15min":0.189},"max":{"1min":3.4940000000000002,"5min":4.5359999999999996,"15min":25.844999999999999},"last":0.77700000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:44:16 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.90200000000000002,"15min":3.46},"min":{"1min":0.26900000000000002,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.59199999999999997,"5min":8.2889999999999997,"15min":222.16499999999999},"last":0.435},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.0369999999999999,"15min":3.375},"min":{"1min":0.26900000000000002,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":3.4119999999999999,"5min":9.7539999999999996,"15min":222.227},"last":1.573}]},{"osd":4,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.341,"15min":2.028},"min":{"1min":0.34000000000000002,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":4.048,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.64700000000000002,"5min":1.3140000000000001,"15min":2.0880000000000001},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":4,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.45400000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.76300000000000001,"15min":1.8069999999999999},"min":{"1min":0.33200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.476,"5min":5.8769999999999998,"15min":16.808},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.90300000000000002,"15min":1.708},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":4.2729999999999997,"5min":5.0629999999999997,"15min":17.507000000000001},"last":0.64500000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:44:27 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.95599999999999996,"15min":1.732},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.81399999999999995,"5min":6.5,"15min":17.535},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.504,"5min":1.0920000000000001,"15min":1.8300000000000001},"min":{"1min":0.38700000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":0.86199999999999999,"5min":10.446,"15min":15.651999999999999},"last":0.81299999999999994}]},{"osd":7,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.86199999999999999,"15min":2.0579999999999998},"min":{"1min":0.375,"5min":0.374,"15min":0.374},"max":{"1min":4.3360000000000003,"5min":4.976,"15min":48.895000000000003},"last":0.754},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.88900000000000001,"15min":2.1179999999999999},"min":{"1min":0.439,"5min":0.371,"15min":0.35099999999999998},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.86799999999999999}]}]},{"osd":2,"up_from":0,"seq":425201762695,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599168,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1545768,"kb_used_data":705440,"kb_used_omap":3,"kb_used_meta":840316,"kb_avail":92199384,"statfs":{"total":95995035648,"available":94412169216,"internally_reserved":0,"allocated":722370560,"data_stored":720274596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":860484327},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":3.2000000000000002,"15min":4.8920000000000003},"min":{"1min":0.20100000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":1.3580000000000001,"5min":92.649000000000001,"15min":137.34399999999999},"last":0.214},{"interface":"front","average":{"1min":0.36399999999999999,"5min":3.194,"15min":4.9740000000000002},"min":{"1min":0.21199999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":1.2549999999999999,"5min":92.284000000000006,"15min":137.73400000000001},"last":0.23899999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":1.8819999999999999,"15min":5.1500000000000004},"min":{"1min":0.25,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":1.1599999999999999,"5min":61.439999999999998,"15min":162.22999999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.39600000000000002,"5min":1.788,"15min":5.181},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.4119999999999999,"5min":62.723999999999997,"15min":162.70699999999999},"last":0.30099999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.0780000000000001,"15min":1.748},"min":{"1min":0.27800000000000002,"5min":0.23899999999999999,"15min":0.20799999999999999},"max":{"1min":2.4710000000000001,"5min":14.254,"15min":14.254},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.66300000000000003,"15min":1.798},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":4.5049999999999999,"5min":4.5049999999999999,"15min":15.523},"last":0.27000000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":1.72,"15min":1.8080000000000001},"min":{"1min":0.35199999999999998,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":1.444,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.47399999999999998,"5min":1.887,"15min":2.0630000000000002},"min":{"1min":0.29499999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.2250000000000001,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.46600000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:44:32 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":1.0780000000000001,"15min":2.3530000000000002},"min":{"1min":0.32000000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.196,"5min":9.4320000000000004,"15min":28.117999999999999},"last":0.503},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.073,"15min":2.4609999999999999},"min":{"1min":0.33600000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.92300000000000004,"5min":10.798999999999999,"15min":28.34},"last":0.38600000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:44:28 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.7350000000000001,"15min":1.8999999999999999},"min":{"1min":0.37,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":1.1000000000000001,"5min":73.887,"15min":73.887},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.952,"15min":1.974},"min":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":1.294,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.41899999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:44:07 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":1.268,"15min":1.845},"min":{"1min":0.42299999999999999,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":2.573,"5min":11.659000000000001,"15min":13.085000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.0740000000000001,"15min":1.726},"min":{"1min":0.39700000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":1.472,"5min":8.3460000000000001,"15min":13.191000000000001},"last":0.48699999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019419,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3717820,"kb_used_data":2823540,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":90027332,"statfs":{"total":95995035648,"available":92187987968,"internally_reserved":0,"allocated":2891304960,"data_stored":2890368170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.4950000000000001,"15min":4.1369999999999996},"min":{"1min":0.22500000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.9889999999999999,"5min":79.837000000000003,"15min":155.315},"last":0.255},{"interface":"front","average":{"1min":0.39500000000000002,"5min":1.6519999999999999,"15min":4.5110000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.77600000000000002,"5min":78.983000000000004,"15min":155.334},"last":0.26700000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":1.5489999999999999,"15min":6.298},"min":{"1min":0.33000000000000002,"5min":0.252,"15min":0.224},"max":{"1min":0.67700000000000005,"5min":77.953000000000003,"15min":190.61199999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.754,"5min":1.468,"15min":7.0140000000000002},"min":{"1min":0.35299999999999998,"5min":0.26700000000000002,"15min":0.25600000000000001},"max":{"1min":3.1339999999999999,"5min":77.962999999999994,"15min":253.273},"last":0.31900000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:44:50 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.77200000000000002,"15min":2.375},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.2329999999999997,"5min":5.702,"15min":170.45400000000001},"last":0.442},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.755,"15min":2.133},"min":{"1min":0.33100000000000002,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":0.71099999999999997,"5min":5.6150000000000002,"15min":58.771000000000001},"last":0.40799999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.77200000000000002,"15min":2.1379999999999999},"min":{"1min":0.31,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.0299999999999998,"5min":5.5279999999999996,"15min":140.881},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.749,"5min":0.68300000000000005,"15min":2.327},"min":{"1min":0.312,"5min":0.312,"15min":0.311},"max":{"1min":3.161,"5min":3.786,"15min":140.911},"last":0.35499999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":1.929,"15min":1.52},"min":{"1min":0.33800000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":1.7509999999999999,"5min":110.003,"15min":110.003},"last":0.375},{"interface":"front","average":{"1min":0.84499999999999997,"5min":1.9239999999999999,"15min":1.405},"min":{"1min":0.38400000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.2909999999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.38500000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:44:12 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.78200000000000003,"15min":0.93000000000000005},"min":{"1min":0.33500000000000002,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":0.73599999999999999,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.432},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.85099999999999998,"15min":1.0589999999999999},"min":{"1min":0.40600000000000003,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":0.73599999999999999,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.47599999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:44:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.65000000000000002,"15min":1.1399999999999999},"min":{"1min":0.36399999999999999,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":3.0830000000000002,"5min":3.8580000000000001,"15min":38.034999999999997},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.91600000000000004,"15min":1.4350000000000001},"min":{"1min":0.39900000000000002,"5min":0.36699999999999999,"15min":0.308},"max":{"1min":0.80000000000000004,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.50700000000000001}]}]},{"osd":5,"up_from":379,"seq":1627792605325,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1836960,"kb_used_data":1472408,"kb_used_omap":13,"kb_used_meta":364530,"kb_avail":91908192,"statfs":{"total":95995035648,"available":94113988608,"internally_reserved":0,"allocated":1507745792,"data_stored":1506825230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":373279218},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,0,2,2,5,0,7],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":1.244,"15min":2.2229999999999999},"min":{"1min":0.24299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":27.584,"5min":27.584,"15min":84.340999999999994},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.9430000000000001,"5min":1.1200000000000001,"15min":2.1280000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23300000000000001},"max":{"1min":27.669,"5min":27.669,"15min":84.356999999999999},"last":0.33400000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":3.4430000000000001,"15min":5.6900000000000004},"min":{"1min":0.28699999999999998,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":3.7799999999999998,"5min":149.94900000000001,"15min":295.71300000000002},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.71199999999999997,"5min":3.7149999999999999,"15min":5.6859999999999999},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":3.9580000000000002,"5min":149.41999999999999,"15min":299.59399999999999},"last":0.27200000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.83799999999999997,"15min":1.552},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.7029999999999998,"5min":5.3470000000000004,"15min":14.199999999999999},"last":0.376},{"interface":"front","average":{"1min":0.84599999999999997,"5min":1.056,"15min":1.5309999999999999},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":3.944,"5min":6.984,"15min":14.430999999999999},"last":0.315}]},{"osd":3,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.876,"15min":1.46},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.0209999999999999,"5min":10.022,"15min":14.004},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.91300000000000003,"15min":1.5940000000000001},"min":{"1min":0.34300000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.8610000000000002,"5min":5.4400000000000004,"15min":13.991},"last":0.40200000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":2.085,"15min":1.417},"min":{"1min":0.28599999999999998,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":4.0839999999999996,"5min":122.971,"15min":122.971},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.86299999999999999,"5min":2.1949999999999998,"15min":1.6699999999999999},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.0810000000000004,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.35499999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":1.0169999999999999,"15min":1.0269999999999999},"min":{"1min":0.32500000000000001,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":4.3300000000000001,"5min":10.500999999999999,"15min":12.545},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.97199999999999998,"15min":1.264},"min":{"1min":0.314,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":4.0599999999999996,"5min":6.4729999999999999,"15min":14.193},"last":0.42999999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.0529999999999999,"15min":1.232},"min":{"1min":0.378,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.0999999999999996,"5min":5.758,"15min":10.813000000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.996,"5min":1.1339999999999999,"15min":1.0840000000000001},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":4.2000000000000002,"5min":10.039,"15min":12.500999999999999},"last":0.443}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3017478144,"data_stored":3017474072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":699596800,"data_stored":699596800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":716177408,"data_stored":716177408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2885488640,"data_stored":2885484568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1502216192,"data_stored":1502216192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1643909120,"data_stored":1643905048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1154809856,"data_stored":1154809856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:45:09.216 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:45:09.267 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:09.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:09.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:09.339 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:09.372 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:09.427 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:09.489 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:09.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:09.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:09.593 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:09.628 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:09.736 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:09.738 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:09 smithi138 ceph-mon[192513]: pgmap v1568: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7041/169692 objects degraded (4.149%); 3452/169692 objects misplaced (2.034%); 710 KiB/s, 11 objects/s recovering 2023-10-01T09:45:09.738 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:09 smithi138 ceph-mon[185886]: pgmap v1568: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7041/169692 objects degraded (4.149%); 3452/169692 objects misplaced (2.034%); 710 KiB/s, 11 objects/s recovering 2023-10-01T09:45:09.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:09.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:09.748 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:45:09.748 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:45:09.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:09 smithi190 ceph-mon[160435]: pgmap v1568: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 143 op/s; 7041/169692 objects degraded (4.149%); 3452/169692 objects misplaced (2.034%); 710 KiB/s, 11 objects/s recovering 2023-10-01T09:45:09.841 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:09.869 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:10.068 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:10.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:10.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:10.172 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:10.200 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:10.248 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:10.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:10.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:10.353 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:10.381 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:10.449 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:45:10.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:10 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1216317025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:10 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:10.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:45:10.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:10 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1216317025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:10.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:10 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:10.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:45:10.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:10 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1216317025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:10.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:10 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:10.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:45:11.342 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:11.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:11.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:11.500 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:45:11.600 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:11.629 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:11 smithi138 ceph-mon[185886]: pgmap v1569: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 7041/169407 objects degraded (4.156%); 3452/169407 objects misplaced (2.038%); 710 KiB/s, 11 objects/s recovering 2023-10-01T09:45:11.629 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:11 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:11.629 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:11 smithi138 ceph-mon[185886]: failed to encode map e535 with expected crc 2023-10-01T09:45:11.629 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:11 smithi138 ceph-mon[185886]: failed to encode map e535 with expected crc 2023-10-01T09:45:11.630 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:11 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:45:11.630 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:11 smithi138 ceph-mon[185886]: osdmap e535: 8 total, 8 up, 7 in 2023-10-01T09:45:11.630 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:11 smithi138 ceph-mon[185886]: failed to encode map e535 with expected crc 2023-10-01T09:45:11.630 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:11 smithi138 ceph-mon[192513]: pgmap v1569: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 7041/169407 objects degraded (4.156%); 3452/169407 objects misplaced (2.038%); 710 KiB/s, 11 objects/s recovering 2023-10-01T09:45:11.630 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:11 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:11.630 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:11 smithi138 ceph-mon[192513]: failed to encode map e535 with expected crc 2023-10-01T09:45:11.631 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:11 smithi138 ceph-mon[192513]: failed to encode map e535 with expected crc 2023-10-01T09:45:11.631 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:11 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:45:11.631 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:11 smithi138 ceph-mon[192513]: osdmap e535: 8 total, 8 up, 7 in 2023-10-01T09:45:11.631 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:11 smithi138 ceph-mon[192513]: failed to encode map e535 with expected crc 2023-10-01T09:45:11.632 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:11.670 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:11.702 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:11.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:11.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:11 smithi190 ceph-mon[160435]: pgmap v1569: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 7041/169407 objects degraded (4.156%); 3452/169407 objects misplaced (2.038%); 710 KiB/s, 11 objects/s recovering 2023-10-01T09:45:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:11 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:11 smithi190 ceph-mon[160435]: failed to encode map e535 with expected crc 2023-10-01T09:45:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:11 smithi190 ceph-mon[160435]: failed to encode map e535 with expected crc 2023-10-01T09:45:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:11 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:45:11.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:11 smithi190 ceph-mon[160435]: osdmap e535: 8 total, 8 up, 7 in 2023-10-01T09:45:11.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:11 smithi190 ceph-mon[160435]: failed to encode map e535 with expected crc 2023-10-01T09:45:11.811 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:11.840 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:11.892 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:11.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:11.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:11.999 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:12.025 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:12.072 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:12.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:12.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:12.116 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:45:12.135 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:45:12.156 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:45:12.174 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:45:12.180 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:12.193 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:45:12.210 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:12.213 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:45:12.233 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:45:12.255 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:45:12.257 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:45:12.257 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:45:12.257 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:45:12.257 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:45:12.257 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:45:12.257 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:45:12.257 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:45:12.258 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:45:12.308 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:12.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:12.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:12.413 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:12.446 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:12.549 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:12.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:12.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:12.653 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:12.680 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:12.776 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:12 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2935429911' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:12.776 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:12 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2935429911' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:12.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:12 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2935429911' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:13.325 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:13.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:13.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:13.396 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:13.432 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:13.461 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:13.512 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:13.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:13.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:13.617 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:13.643 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:13 smithi138 ceph-mon[185886]: pgmap v1571: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 157 op/s; 6992/167310 objects degraded (4.179%); 3394/167310 objects misplaced (2.029%); 672 KiB/s, 10 objects/s recovering 2023-10-01T09:45:13.643 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:13 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6992/167310 objects degraded (4.179%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:13.643 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:13 smithi138 ceph-mon[192513]: pgmap v1571: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 157 op/s; 6992/167310 objects degraded (4.179%); 3394/167310 objects misplaced (2.029%); 672 KiB/s, 10 objects/s recovering 2023-10-01T09:45:13.643 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:13 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6992/167310 objects degraded (4.179%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:13.646 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:13.695 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:45:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:13 smithi190 ceph-mon[160435]: pgmap v1571: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 157 op/s; 6992/167310 objects degraded (4.179%); 3394/167310 objects misplaced (2.029%); 672 KiB/s, 10 objects/s recovering 2023-10-01T09:45:13.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:13 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6992/167310 objects degraded (4.179%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:13.826 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:13.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:13.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:13.932 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:13.966 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:14.178 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:14.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:14.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:14.282 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:14.311 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:14.355 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:45:14.355 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 6992/167310 objects degraded (4.179%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2135,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":535,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":55770,"data_bytes":3657832395,"bytes_used":16682274816,"bytes_avail":655282974720,"bytes_total":671965249536,"degraded_objects":6992,"degraded_total":167310,"degraded_ratio":0.041790687944534097,"misplaced_objects":3394,"misplaced_total":167310,"misplaced_ratio":0.020285697208774135,"recovering_objects_per_sec":10,"recovering_bytes_per_sec":687820,"recovering_keys_per_sec":0,"num_objects_recovered":104,"num_bytes_recovered":6815744,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":157},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":34,"modified":"2023-10-01T09:43:40.311181+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:45:14.355 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:14.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:14.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:14.460 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:14.491 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:14.794 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1430046522' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:45:14.794 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1430046522' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:45:14.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1430046522' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:45:15.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:45:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:45:14] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-01T09:45:15.304 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:15.315 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:15.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:15.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:15.327 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:45:15.327 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:45:15.344 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:45:15.421 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:15.448 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:15.503 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:15.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:15.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:15.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:15 smithi138 ceph-mon[185886]: pgmap v1572: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 154 op/s; 6992/167211 objects degraded (4.182%); 3394/167211 objects misplaced (2.030%); 665 KiB/s, 10 objects/s recovering 2023-10-01T09:45:15.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:15 smithi138 ceph-mon[192513]: pgmap v1572: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 154 op/s; 6992/167211 objects degraded (4.182%); 3394/167211 objects misplaced (2.030%); 665 KiB/s, 10 objects/s recovering 2023-10-01T09:45:15.611 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:15.637 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:15.681 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:15.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:15.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:15.744 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:45:15.745 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:45:15.745 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:45:15.745 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:45:15.745 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:45:15.793 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:15.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:15 smithi190 ceph-mon[160435]: pgmap v1572: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 154 op/s; 6992/167211 objects degraded (4.182%); 3394/167211 objects misplaced (2.030%); 665 KiB/s, 10 objects/s recovering 2023-10-01T09:45:15.824 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:15.876 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:15.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:15.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:15.982 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:16.013 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:16.462 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:16.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:16.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:16.558 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:45:16.566 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:16.593 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:16.640 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:16.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:16.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:16.750 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:16.777 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:17.132 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:17.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:17.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:17.235 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:17.263 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:17.308 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:17.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:17.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:17.414 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:17.440 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:17.487 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:17.492 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:17.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:17.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:17.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:17 smithi190 ceph-mon[160435]: pgmap v1573: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 160 op/s; 6992/166122 objects degraded (4.209%); 3394/166122 objects misplaced (2.043%); 665 KiB/s, 10 objects/s recovering 2023-10-01T09:45:17.594 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:17.597 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:17.620 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:17.678 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:17.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:17.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:17.781 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:17.812 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:17 smithi138 ceph-mon[192513]: pgmap v1573: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 160 op/s; 6992/166122 objects degraded (4.209%); 3394/166122 objects misplaced (2.043%); 665 KiB/s, 10 objects/s recovering 2023-10-01T09:45:17.812 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:17 smithi138 ceph-mon[185886]: pgmap v1573: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 160 op/s; 6992/166122 objects degraded (4.209%); 3394/166122 objects misplaced (2.043%); 665 KiB/s, 10 objects/s recovering 2023-10-01T09:45:17.816 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:17.878 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:17.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:17.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:17.981 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:18.007 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:18.051 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:18.055 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:18.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:18.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:18.158 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:18.189 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:18.803 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:18 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6992/166122 objects degraded (4.209%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:18.804 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:18 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2415630790' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:45:18.804 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:18 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:45:18.804 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:18 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6992/166122 objects degraded (4.209%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:18.804 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:18 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2415630790' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:45:18.804 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:18 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:45:18.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:18 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6992/166122 objects degraded (4.209%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:18 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2415630790' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:45:18.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:18 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:45:18.810 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:45:19.483 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:19.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:19.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:19.588 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:19.614 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:19.666 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:19.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:19.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:19.772 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:19.799 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:19 smithi138 ceph-mon[192513]: pgmap v1574: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 6942/164715 objects degraded (4.215%); 3336/164715 objects misplaced (2.025%); 685 KiB/s, 10 objects/s recovering 2023-10-01T09:45:19.799 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:19 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:19.799 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:19 smithi138 ceph-mon[192513]: failed to encode map e536 with expected crc 2023-10-01T09:45:19.799 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:19 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:45:19.799 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:19 smithi138 ceph-mon[192513]: osdmap e536: 8 total, 8 up, 7 in 2023-10-01T09:45:19.799 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:19 smithi138 ceph-mon[192513]: failed to encode map e536 with expected crc 2023-10-01T09:45:19.800 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:19 smithi138 ceph-mon[185886]: pgmap v1574: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 6942/164715 objects degraded (4.215%); 3336/164715 objects misplaced (2.025%); 685 KiB/s, 10 objects/s recovering 2023-10-01T09:45:19.800 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:19 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:19.800 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:19 smithi138 ceph-mon[185886]: failed to encode map e536 with expected crc 2023-10-01T09:45:19.800 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:19 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:45:19.800 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:19 smithi138 ceph-mon[185886]: osdmap e536: 8 total, 8 up, 7 in 2023-10-01T09:45:19.800 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:19 smithi138 ceph-mon[185886]: failed to encode map e536 with expected crc 2023-10-01T09:45:19.803 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:19 smithi190 ceph-mon[160435]: pgmap v1574: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 6942/164715 objects degraded (4.215%); 3336/164715 objects misplaced (2.025%); 685 KiB/s, 10 objects/s recovering 2023-10-01T09:45:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:19 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:19 smithi190 ceph-mon[160435]: failed to encode map e536 with expected crc 2023-10-01T09:45:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:19 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:45:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:19 smithi190 ceph-mon[160435]: osdmap e536: 8 total, 8 up, 7 in 2023-10-01T09:45:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:19 smithi190 ceph-mon[160435]: failed to encode map e536 with expected crc 2023-10-01T09:45:19.905 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:19.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:19.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:20.010 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:20.040 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:20.290 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:20.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:20.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:20.394 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:20.423 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:20.480 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:20.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:20.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:20.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:20 smithi138 ceph-mon[192513]: failed to encode map e536 with expected crc 2023-10-01T09:45:20.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:20 smithi138 ceph-mon[192513]: failed to encode map e536 with expected crc 2023-10-01T09:45:20.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:20 smithi138 ceph-mon[185886]: failed to encode map e536 with expected crc 2023-10-01T09:45:20.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:20 smithi138 ceph-mon[185886]: failed to encode map e536 with expected crc 2023-10-01T09:45:20.588 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:20.614 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:20.682 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:20.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:20.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:20.701 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:20 smithi190 ceph-mon[160435]: failed to encode map e536 with expected crc 2023-10-01T09:45:20.701 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:20 smithi190 ceph-mon[160435]: failed to encode map e536 with expected crc 2023-10-01T09:45:20.802 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:20.835 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:20.890 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:20.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:20.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:20.999 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:21.008 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:45:21.008 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:45:21.008 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:45:21.008 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:45:21.008 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:45:21.008 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:45:21.008 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:45:21.008 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:45:21.009 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:45:21.009 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:45:21.009 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:45:21.009 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:45:21.009 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:45:21.009 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:45:21.009 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:45:21.009 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:45:21.012 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:45:21.014 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:45:21.031 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:21.085 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:21.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:21.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:21.193 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:21.222 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:21.270 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:21.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:21.379 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:21.405 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:21.453 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:21.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:21.559 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:21.587 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:21.638 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:21.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:21.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:21.744 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:21.768 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:21 smithi190 ceph-mon[160435]: pgmap v1576: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 6942/164565 objects degraded (4.218%); 3336/164565 objects misplaced (2.027%); 692 KiB/s, 10 objects/s recovering 2023-10-01T09:45:21.768 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:21 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1599543768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:21.771 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:21.819 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:21.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:21 smithi138 ceph-mon[185886]: pgmap v1576: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 6942/164565 objects degraded (4.218%); 3336/164565 objects misplaced (2.027%); 692 KiB/s, 10 objects/s recovering 2023-10-01T09:45:21.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:21 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1599543768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:21.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:21 smithi138 ceph-mon[192513]: pgmap v1576: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 6942/164565 objects degraded (4.218%); 3336/164565 objects misplaced (2.027%); 692 KiB/s, 10 objects/s recovering 2023-10-01T09:45:21.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:21 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1599543768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:21.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:21.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:21.925 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:21.939 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1576,"stamp":"2023-10-01T09:45:20.341959+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3597866955,"num_objects":54855,"num_object_clones":0,"num_object_copies":164565,"num_objects_missing_on_primary":1863,"num_objects_missing":1863,"num_objects_degraded":6942,"num_objects_misplaced":3336,"num_objects_unfound":0,"num_objects_dirty":54855,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":112719,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9255,"num_bytes_recovered":593396751,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112283,"ondisk_log_size":112283,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16107260,"kb_used_data":11031812,"kb_used_omap":39,"kb_used_meta":5075352,"kb_avail":640108804,"statfs":{"total":671965249536,"available":655471415296,"internally_reserved":0,"allocated":11296575488,"data_stored":11286660112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5197161071},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,0,0,0,0,1,2,4,3,4],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-105775104,"num_objects":-1614,"num_object_clones":0,"num_object_copies":-4842,"num_objects_missing_on_primary":-108,"num_objects_missing":-108,"num_objects_degraded":-99,"num_objects_misplaced":-116,"num_objects_unfound":0,"num_objects_dirty":-1614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1614,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":107,"num_bytes_recovered":7012352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.899562"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":311,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.444994+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:45:02.444994+0000","last_peered":"2023-10-01T09:45:02.444994+0000","last_clean":"2023-10-01T09:45:02.444994+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:45:02.444994+0000","last_undegraded":"2023-10-01T09:45:02.444994+0000","last_fullsized":"2023-10-01T09:45:02.444994+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":156,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:15.268741+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:45:15.268741+0000","last_peered":"2023-10-01T09:45:15.268741+0000","last_clean":"2023-10-01T09:45:15.268741+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:45:15.268741+0000","last_undegraded":"2023-10-01T09:45:15.268741+0000","last_fullsized":"2023-10-01T09:45:15.268741+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":536,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.449317+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:45:10.449317+0000","last_peered":"2023-10-01T09:45:10.449317+0000","last_clean":"2023-10-01T09:45:10.449317+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:45:10.449317+0000","last_undegraded":"2023-10-01T09:45:10.449317+0000","last_fullsized":"2023-10-01T09:45:10.449317+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":594,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.438251+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:45:02.438251+0000","last_peered":"2023-10-01T09:45:02.438251+0000","last_clean":"2023-10-01T09:45:02.438251+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:45:02.438251+0000","last_undegraded":"2023-10-01T09:45:02.438251+0000","last_fullsized":"2023-10-01T09:45:02.438251+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":475,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.447394+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:45:10.447394+0000","last_peered":"2023-10-01T09:45:10.447394+0000","last_clean":"2023-10-01T09:45:10.447394+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:45:10.447394+0000","last_undegraded":"2023-10-01T09:45:10.447394+0000","last_fullsized":"2023-10-01T09:45:10.447394+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":216,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.505722+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:45:02.505722+0000","last_peered":"2023-10-01T09:45:02.505722+0000","last_clean":"2023-10-01T09:45:02.505722+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:45:02.505722+0000","last_undegraded":"2023-10-01T09:45:02.505722+0000","last_fullsized":"2023-10-01T09:45:02.505722+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":592,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.505787+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:45:02.505787+0000","last_peered":"2023-10-01T09:45:02.505787+0000","last_clean":"2023-10-01T09:45:02.505787+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:45:02.505787+0000","last_undegraded":"2023-10-01T09:45:02.505787+0000","last_fullsized":"2023-10-01T09:45:02.505787+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":522,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:07.476742+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:45:07.476742+0000","last_peered":"2023-10-01T09:45:07.476742+0000","last_clean":"2023-10-01T09:45:07.476742+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:45:07.476742+0000","last_undegraded":"2023-10-01T09:45:07.476742+0000","last_fullsized":"2023-10-01T09:45:07.476742+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":247,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437747+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:45:02.437747+0000","last_peered":"2023-10-01T09:45:02.437747+0000","last_clean":"2023-10-01T09:45:02.437747+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:45:02.437747+0000","last_undegraded":"2023-10-01T09:45:02.437747+0000","last_fullsized":"2023-10-01T09:45:02.437747+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":54,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.446454+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:45:02.446454+0000","last_peered":"2023-10-01T09:45:02.446454+0000","last_clean":"2023-10-01T09:45:02.446454+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:45:02.446454+0000","last_undegraded":"2023-10-01T09:45:02.446454+0000","last_fullsized":"2023-10-01T09:45:02.446454+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":60,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.444218+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:45:02.444218+0000","last_peered":"2023-10-01T09:45:02.444218+0000","last_clean":"2023-10-01T09:45:02.444218+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:45:02.444218+0000","last_undegraded":"2023-10-01T09:45:02.444218+0000","last_fullsized":"2023-10-01T09:45:02.444218+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":59,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.449776+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:45:10.449776+0000","last_peered":"2023-10-01T09:45:10.449776+0000","last_clean":"2023-10-01T09:45:10.449776+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:45:10.449776+0000","last_undegraded":"2023-10-01T09:45:10.449776+0000","last_fullsized":"2023-10-01T09:45:10.449776+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":436,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.444485+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:45:02.444485+0000","last_peered":"2023-10-01T09:45:02.444485+0000","last_clean":"2023-10-01T09:45:02.444485+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:45:02.444485+0000","last_undegraded":"2023-10-01T09:45:02.444485+0000","last_fullsized":"2023-10-01T09:45:02.444485+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":317,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.443935+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:45:02.443935+0000","last_peered":"2023-10-01T09:45:02.443935+0000","last_clean":"2023-10-01T09:45:02.443935+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:45:02.443935+0000","last_undegraded":"2023-10-01T09:45:02.443935+0000","last_fullsized":"2023-10-01T09:45:02.443935+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":653,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.446108+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:45:02.446108+0000","last_peered":"2023-10-01T09:45:02.446108+0000","last_clean":"2023-10-01T09:45:02.446108+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:45:02.446108+0000","last_undegraded":"2023-10-01T09:45:02.446108+0000","last_fullsized":"2023-10-01T09:45:02.446108+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":390,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.444739+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:45:02.444739+0000","last_peered":"2023-10-01T09:45:02.444739+0000","last_clean":"2023-10-01T09:45:02.444739+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:45:02.444739+0000","last_undegraded":"2023-10-01T09:45:02.444739+0000","last_fullsized":"2023-10-01T09:45:02.444739+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":63,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.443941+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:45:02.443941+0000","last_peered":"2023-10-01T09:45:02.443941+0000","last_clean":"2023-10-01T09:45:02.443941+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:45:02.443941+0000","last_undegraded":"2023-10-01T09:45:02.443941+0000","last_fullsized":"2023-10-01T09:45:02.443941+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":49,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437628+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:45:02.437628+0000","last_peered":"2023-10-01T09:45:02.437628+0000","last_clean":"2023-10-01T09:45:02.437628+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:45:02.437628+0000","last_undegraded":"2023-10-01T09:45:02.437628+0000","last_fullsized":"2023-10-01T09:45:02.437628+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":546,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.446671+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:45:02.446671+0000","last_peered":"2023-10-01T09:45:02.446671+0000","last_clean":"2023-10-01T09:45:02.446671+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:45:02.446671+0000","last_undegraded":"2023-10-01T09:45:02.446671+0000","last_fullsized":"2023-10-01T09:45:02.446671+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":558,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.438291+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:45:02.438291+0000","last_peered":"2023-10-01T09:45:02.438291+0000","last_clean":"2023-10-01T09:45:02.438291+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:45:02.438291+0000","last_undegraded":"2023-10-01T09:45:02.438291+0000","last_fullsized":"2023-10-01T09:45:02.438291+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":414,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437925+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:45:02.437925+0000","last_peered":"2023-10-01T09:45:02.437925+0000","last_clean":"2023-10-01T09:45:02.437925+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:45:02.437925+0000","last_undegraded":"2023-10-01T09:45:02.437925+0000","last_fullsized":"2023-10-01T09:45:02.437925+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":63,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.444228+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:45:02.444228+0000","last_peered":"2023-10-01T09:45:02.444228+0000","last_clean":"2023-10-01T09:45:02.444228+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:45:02.444228+0000","last_undegraded":"2023-10-01T09:45:02.444228+0000","last_fullsized":"2023-10-01T09:45:02.444228+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"535'7021","reported_seq":7089,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:16.891294+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:45:16.891294+0000","last_peered":"2023-10-01T09:45:16.891294+0000","last_clean":"2023-10-01T09:45:16.891294+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:45:16.891294+0000","last_undegraded":"2023-10-01T09:45:16.891294+0000","last_fullsized":"2023-10-01T09:45:16.891294+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7021,"log_dups_size":0,"ondisk_log_size":7021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220921856,"num_objects":3371,"num_object_clones":0,"num_object_copies":10113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7021,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":58,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.444052+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:45:02.444052+0000","last_peered":"2023-10-01T09:45:02.444052+0000","last_clean":"2023-10-01T09:45:02.444052+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:45:02.444052+0000","last_undegraded":"2023-10-01T09:45:02.444052+0000","last_fullsized":"2023-10-01T09:45:02.444052+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":255,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437075+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:45:02.437075+0000","last_peered":"2023-10-01T09:45:02.437075+0000","last_clean":"2023-10-01T09:45:02.437075+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:45:02.437075+0000","last_undegraded":"2023-10-01T09:45:02.437075+0000","last_fullsized":"2023-10-01T09:45:02.437075+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"535'7023","reported_seq":12582,"reported_epoch":535,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:45:17.009698+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:45:17.009698+0000","last_peered":"2023-10-01T09:45:17.009698+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:45:17.009698+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7023,"log_dups_size":0,"ondisk_log_size":7023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221708288,"num_objects":3383,"num_object_clones":0,"num_object_copies":10149,"num_objects_missing_on_primary":1863,"num_objects_missing":1863,"num_objects_degraded":5178,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7023,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1820,"num_bytes_recovered":119275520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1452},{"shards":"0,4,6","objects":1863},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":529,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.445997+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:45:02.445997+0000","last_peered":"2023-10-01T09:45:02.445997+0000","last_clean":"2023-10-01T09:45:02.445997+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:45:02.445997+0000","last_undegraded":"2023-10-01T09:45:02.445997+0000","last_fullsized":"2023-10-01T09:45:02.445997+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":592,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.449275+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:45:10.449275+0000","last_peered":"2023-10-01T09:45:10.449275+0000","last_clean":"2023-10-01T09:45:10.449275+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:45:10.449275+0000","last_undegraded":"2023-10-01T09:45:10.449275+0000","last_fullsized":"2023-10-01T09:45:10.449275+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"535'7019","reported_seq":7090,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:16.709219+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:45:16.709219+0000","last_peered":"2023-10-01T09:45:16.709219+0000","last_clean":"2023-10-01T09:45:16.709219+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:45:16.709219+0000","last_undegraded":"2023-10-01T09:45:16.709219+0000","last_fullsized":"2023-10-01T09:45:16.709219+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7019,"log_dups_size":0,"ondisk_log_size":7019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223674368,"num_objects":3413,"num_object_clones":0,"num_object_copies":10239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7019,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":615,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.444306+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:45:02.444306+0000","last_peered":"2023-10-01T09:45:02.444306+0000","last_clean":"2023-10-01T09:45:02.444306+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:45:02.444306+0000","last_undegraded":"2023-10-01T09:45:02.444306+0000","last_fullsized":"2023-10-01T09:45:02.444306+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":412,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.449861+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:45:10.449861+0000","last_peered":"2023-10-01T09:45:10.449861+0000","last_clean":"2023-10-01T09:45:10.449861+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:45:10.449861+0000","last_undegraded":"2023-10-01T09:45:10.449861+0000","last_fullsized":"2023-10-01T09:45:10.449861+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"535'6982","reported_seq":13658,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:16.432458+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:45:16.432458+0000","last_peered":"2023-10-01T09:45:16.432458+0000","last_clean":"2023-10-01T09:45:16.432458+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:45:16.432458+0000","last_undegraded":"2023-10-01T09:45:16.432458+0000","last_fullsized":"2023-10-01T09:45:16.432458+0000","mapping_epoch":505,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6982,"log_dups_size":0,"ondisk_log_size":6982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224133120,"num_objects":3420,"num_object_clones":0,"num_object_copies":10260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6982,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":255,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437494+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:45:02.437494+0000","last_peered":"2023-10-01T09:45:02.437494+0000","last_clean":"2023-10-01T09:45:02.437494+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:45:02.437494+0000","last_undegraded":"2023-10-01T09:45:02.437494+0000","last_fullsized":"2023-10-01T09:45:02.437494+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":514,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437988+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:45:02.437988+0000","last_peered":"2023-10-01T09:45:02.437988+0000","last_clean":"2023-10-01T09:45:02.437988+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:45:02.437988+0000","last_undegraded":"2023-10-01T09:45:02.437988+0000","last_fullsized":"2023-10-01T09:45:02.437988+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"536'7030","reported_seq":7102,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.972512+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:45:19.972512+0000","last_peered":"2023-10-01T09:45:19.972512+0000","last_clean":"2023-10-01T09:45:19.972512+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:45:19.972512+0000","last_undegraded":"2023-10-01T09:45:19.972512+0000","last_fullsized":"2023-10-01T09:45:19.972512+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7030,"log_dups_size":0,"ondisk_log_size":7030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227934208,"num_objects":3478,"num_object_clones":0,"num_object_copies":10434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7030,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":115,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.444239+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:45:02.444239+0000","last_peered":"2023-10-01T09:45:02.444239+0000","last_clean":"2023-10-01T09:45:02.444239+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:45:02.444239+0000","last_undegraded":"2023-10-01T09:45:02.444239+0000","last_fullsized":"2023-10-01T09:45:02.444239+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":189,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.444605+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:45:02.444605+0000","last_peered":"2023-10-01T09:45:02.444605+0000","last_clean":"2023-10-01T09:45:02.444605+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:45:02.444605+0000","last_undegraded":"2023-10-01T09:45:02.444605+0000","last_fullsized":"2023-10-01T09:45:02.444605+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"535'6968","reported_seq":7106,"reported_epoch":535,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:45:16.928659+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:45:16.928659+0000","last_peered":"2023-10-01T09:45:16.928659+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:45:16.928659+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6968,"log_dups_size":0,"ondisk_log_size":6968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222822400,"num_objects":3400,"num_object_clones":0,"num_object_copies":10200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1754,"num_objects_misplaced":3336,"num_objects_unfound":0,"num_objects_dirty":3400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6968,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":2031616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":3336},{"shards":"4,5","objects":-1690}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":433,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.445109+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:45:02.445109+0000","last_peered":"2023-10-01T09:45:02.445109+0000","last_clean":"2023-10-01T09:45:02.445109+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:45:02.445109+0000","last_undegraded":"2023-10-01T09:45:02.445109+0000","last_fullsized":"2023-10-01T09:45:02.445109+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":595,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.449440+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:45:10.449440+0000","last_peered":"2023-10-01T09:45:10.449440+0000","last_clean":"2023-10-01T09:45:10.449440+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:45:10.449440+0000","last_undegraded":"2023-10-01T09:45:10.449440+0000","last_fullsized":"2023-10-01T09:45:10.449440+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"535'6920","reported_seq":6997,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:17.232026+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:45:17.232026+0000","last_peered":"2023-10-01T09:45:17.232026+0000","last_clean":"2023-10-01T09:45:17.232026+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:45:17.232026+0000","last_undegraded":"2023-10-01T09:45:17.232026+0000","last_fullsized":"2023-10-01T09:45:17.232026+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6920,"log_dups_size":0,"ondisk_log_size":6920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226623488,"num_objects":3458,"num_object_clones":0,"num_object_copies":10374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6920,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":429,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.447122+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:45:02.447122+0000","last_peered":"2023-10-01T09:45:02.447122+0000","last_clean":"2023-10-01T09:45:02.447122+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:45:02.447122+0000","last_undegraded":"2023-10-01T09:45:02.447122+0000","last_fullsized":"2023-10-01T09:45:02.447122+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":270,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.445602+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:45:02.445602+0000","last_peered":"2023-10-01T09:45:02.445602+0000","last_clean":"2023-10-01T09:45:02.445602+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:45:02.445602+0000","last_undegraded":"2023-10-01T09:45:02.445602+0000","last_fullsized":"2023-10-01T09:45:02.445602+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"535'7063","reported_seq":7131,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:16.239817+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:45:16.239817+0000","last_peered":"2023-10-01T09:45:16.239817+0000","last_clean":"2023-10-01T09:45:16.239817+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:45:16.239817+0000","last_undegraded":"2023-10-01T09:45:16.239817+0000","last_fullsized":"2023-10-01T09:45:16.239817+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7063,"log_dups_size":0,"ondisk_log_size":7063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225116160,"num_objects":3435,"num_object_clones":0,"num_object_copies":10305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7063,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":399,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.449283+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:45:10.449283+0000","last_peered":"2023-10-01T09:45:10.449283+0000","last_clean":"2023-10-01T09:45:10.449283+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:45:10.449283+0000","last_undegraded":"2023-10-01T09:45:10.449283+0000","last_fullsized":"2023-10-01T09:45:10.449283+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":446,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.449568+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:45:10.449568+0000","last_peered":"2023-10-01T09:45:10.449568+0000","last_clean":"2023-10-01T09:45:10.449568+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:45:10.449568+0000","last_undegraded":"2023-10-01T09:45:10.449568+0000","last_fullsized":"2023-10-01T09:45:10.449568+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"535'6839","reported_seq":6907,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:16.218413+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:45:16.218413+0000","last_peered":"2023-10-01T09:45:16.218413+0000","last_clean":"2023-10-01T09:45:16.218413+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:45:16.218413+0000","last_undegraded":"2023-10-01T09:45:16.218413+0000","last_fullsized":"2023-10-01T09:45:16.218413+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6839,"log_dups_size":0,"ondisk_log_size":6839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218824704,"num_objects":3339,"num_object_clones":0,"num_object_copies":10017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6839,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":928,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.444587+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:45:02.444587+0000","last_peered":"2023-10-01T09:45:02.444587+0000","last_clean":"2023-10-01T09:45:02.444587+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:45:02.444587+0000","last_undegraded":"2023-10-01T09:45:02.444587+0000","last_fullsized":"2023-10-01T09:45:02.444587+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":265,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437718+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:45:02.437718+0000","last_peered":"2023-10-01T09:45:02.437718+0000","last_clean":"2023-10-01T09:45:02.437718+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:45:02.437718+0000","last_undegraded":"2023-10-01T09:45:02.437718+0000","last_fullsized":"2023-10-01T09:45:02.437718+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":60,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.449785+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:45:10.449785+0000","last_peered":"2023-10-01T09:45:10.449785+0000","last_clean":"2023-10-01T09:45:10.449785+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:45:10.449785+0000","last_undegraded":"2023-10-01T09:45:10.449785+0000","last_fullsized":"2023-10-01T09:45:10.449785+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"535'6908","reported_seq":6979,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:16.748249+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:45:16.748249+0000","last_peered":"2023-10-01T09:45:16.748249+0000","last_clean":"2023-10-01T09:45:16.748249+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:45:16.748249+0000","last_undegraded":"2023-10-01T09:45:16.748249+0000","last_fullsized":"2023-10-01T09:45:16.748249+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6908,"log_dups_size":0,"ondisk_log_size":6908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226885632,"num_objects":3462,"num_object_clones":0,"num_object_copies":10386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6908,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1200,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437242+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:45:02.437242+0000","last_peered":"2023-10-01T09:45:02.437242+0000","last_clean":"2023-10-01T09:45:02.437242+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:45:02.437242+0000","last_undegraded":"2023-10-01T09:45:02.437242+0000","last_fullsized":"2023-10-01T09:45:02.437242+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":410,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.444112+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:45:02.444112+0000","last_peered":"2023-10-01T09:45:02.444112+0000","last_clean":"2023-10-01T09:45:02.444112+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:45:02.444112+0000","last_undegraded":"2023-10-01T09:45:02.444112+0000","last_fullsized":"2023-10-01T09:45:02.444112+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":238,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437599+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:45:02.437599+0000","last_peered":"2023-10-01T09:45:02.437599+0000","last_clean":"2023-10-01T09:45:02.437599+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:45:02.437599+0000","last_undegraded":"2023-10-01T09:45:02.437599+0000","last_fullsized":"2023-10-01T09:45:02.437599+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"535'6983","reported_seq":14272,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:16.909259+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:45:16.909259+0000","last_peered":"2023-10-01T09:45:16.909259+0000","last_clean":"2023-10-01T09:45:16.909259+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:45:16.909259+0000","last_undegraded":"2023-10-01T09:45:16.909259+0000","last_fullsized":"2023-10-01T09:45:16.909259+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6983,"log_dups_size":0,"ondisk_log_size":6983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222363648,"num_objects":3393,"num_object_clones":0,"num_object_copies":10179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6983,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":881,"reported_epoch":535,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:45:10.447048+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:45:10.447048+0000","last_peered":"2023-10-01T09:45:10.447048+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:45:10.447048+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:45:10.447048+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":599,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.449369+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:45:10.449369+0000","last_peered":"2023-10-01T09:45:10.449369+0000","last_clean":"2023-10-01T09:45:10.449369+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:45:10.449369+0000","last_undegraded":"2023-10-01T09:45:10.449369+0000","last_fullsized":"2023-10-01T09:45:10.449369+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":60,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.449433+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:45:10.449433+0000","last_peered":"2023-10-01T09:45:10.449433+0000","last_clean":"2023-10-01T09:45:10.449433+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:45:10.449433+0000","last_undegraded":"2023-10-01T09:45:10.449433+0000","last_fullsized":"2023-10-01T09:45:10.449433+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"536'6878","reported_seq":6950,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.919716+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:45:19.919716+0000","last_peered":"2023-10-01T09:45:19.919716+0000","last_clean":"2023-10-01T09:45:19.919716+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:45:19.919716+0000","last_undegraded":"2023-10-01T09:45:19.919716+0000","last_fullsized":"2023-10-01T09:45:19.919716+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6878,"log_dups_size":0,"ondisk_log_size":6878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222429184,"num_objects":3394,"num_object_clones":0,"num_object_copies":10182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6878,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1651,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437441+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:45:02.437441+0000","last_peered":"2023-10-01T09:45:02.437441+0000","last_clean":"2023-10-01T09:45:02.437441+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:45:02.437441+0000","last_undegraded":"2023-10-01T09:45:02.437441+0000","last_fullsized":"2023-10-01T09:45:02.437441+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":565,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.443814+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:45:02.443814+0000","last_peered":"2023-10-01T09:45:02.443814+0000","last_clean":"2023-10-01T09:45:02.443814+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:45:02.443814+0000","last_undegraded":"2023-10-01T09:45:02.443814+0000","last_fullsized":"2023-10-01T09:45:02.443814+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":343,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437290+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:45:02.437290+0000","last_peered":"2023-10-01T09:45:02.437290+0000","last_clean":"2023-10-01T09:45:02.437290+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:45:02.437290+0000","last_undegraded":"2023-10-01T09:45:02.437290+0000","last_fullsized":"2023-10-01T09:45:02.437290+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":333,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.446749+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:45:02.446749+0000","last_peered":"2023-10-01T09:45:02.446749+0000","last_clean":"2023-10-01T09:45:02.446749+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:45:02.446749+0000","last_undegraded":"2023-10-01T09:45:02.446749+0000","last_fullsized":"2023-10-01T09:45:02.446749+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"535'7054","reported_seq":7122,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:15.780492+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:45:15.780492+0000","last_peered":"2023-10-01T09:45:15.780492+0000","last_clean":"2023-10-01T09:45:15.780492+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:45:15.780492+0000","last_undegraded":"2023-10-01T09:45:15.780492+0000","last_fullsized":"2023-10-01T09:45:15.780492+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7054,"log_dups_size":0,"ondisk_log_size":7054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227934208,"num_objects":3478,"num_object_clones":0,"num_object_copies":10434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7054,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":742,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.443989+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:45:02.443989+0000","last_peered":"2023-10-01T09:45:02.443989+0000","last_clean":"2023-10-01T09:45:02.443989+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:45:02.443989+0000","last_undegraded":"2023-10-01T09:45:02.443989+0000","last_fullsized":"2023-10-01T09:45:02.443989+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":575,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.439253+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:45:02.439253+0000","last_peered":"2023-10-01T09:45:02.439253+0000","last_clean":"2023-10-01T09:45:02.439253+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:45:02.439253+0000","last_undegraded":"2023-10-01T09:45:02.439253+0000","last_fullsized":"2023-10-01T09:45:02.439253+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":574,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.438172+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:45:02.438172+0000","last_peered":"2023-10-01T09:45:02.438172+0000","last_clean":"2023-10-01T09:45:02.438172+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:45:02.438172+0000","last_undegraded":"2023-10-01T09:45:02.438172+0000","last_fullsized":"2023-10-01T09:45:02.438172+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"535'6976","reported_seq":7048,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:16.756592+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:45:16.756592+0000","last_peered":"2023-10-01T09:45:16.756592+0000","last_clean":"2023-10-01T09:45:16.756592+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:45:16.756592+0000","last_undegraded":"2023-10-01T09:45:16.756592+0000","last_fullsized":"2023-10-01T09:45:16.756592+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6976,"log_dups_size":0,"ondisk_log_size":6976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228261912,"num_objects":3484,"num_object_clones":0,"num_object_copies":10452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3484,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6976,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":973,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.444404+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:45:02.444404+0000","last_peered":"2023-10-01T09:45:02.444404+0000","last_clean":"2023-10-01T09:45:02.444404+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:45:02.444404+0000","last_undegraded":"2023-10-01T09:45:02.444404+0000","last_fullsized":"2023-10-01T09:45:02.444404+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":592,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437589+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:45:02.437589+0000","last_peered":"2023-10-01T09:45:02.437589+0000","last_clean":"2023-10-01T09:45:02.437589+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:45:02.437589+0000","last_undegraded":"2023-10-01T09:45:02.437589+0000","last_fullsized":"2023-10-01T09:45:02.437589+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":445,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437491+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:45:02.437491+0000","last_peered":"2023-10-01T09:45:02.437491+0000","last_clean":"2023-10-01T09:45:02.437491+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:45:02.437491+0000","last_undegraded":"2023-10-01T09:45:02.437491+0000","last_fullsized":"2023-10-01T09:45:02.437491+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"535'6945","reported_seq":7021,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:16.217705+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:45:16.217705+0000","last_peered":"2023-10-01T09:45:16.217705+0000","last_clean":"2023-10-01T09:45:16.217705+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:45:16.217705+0000","last_undegraded":"2023-10-01T09:45:16.217705+0000","last_fullsized":"2023-10-01T09:45:16.217705+0000","mapping_epoch":479,"log_start":"480'200","ondisk_log_start":"480'200","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6745,"log_dups_size":0,"ondisk_log_size":6745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223543296,"num_objects":3411,"num_object_clones":0,"num_object_copies":10233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6945,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":474,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.444324+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:45:02.444324+0000","last_peered":"2023-10-01T09:45:02.444324+0000","last_clean":"2023-10-01T09:45:02.444324+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:45:02.444324+0000","last_undegraded":"2023-10-01T09:45:02.444324+0000","last_fullsized":"2023-10-01T09:45:02.444324+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1169,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.449789+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:45:10.449789+0000","last_peered":"2023-10-01T09:45:10.449789+0000","last_clean":"2023-10-01T09:45:10.449789+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:45:10.449789+0000","last_undegraded":"2023-10-01T09:45:10.449789+0000","last_fullsized":"2023-10-01T09:45:10.449789+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":63,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.443727+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:45:02.443727+0000","last_peered":"2023-10-01T09:45:02.443727+0000","last_clean":"2023-10-01T09:45:02.443727+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:45:02.443727+0000","last_undegraded":"2023-10-01T09:45:02.443727+0000","last_fullsized":"2023-10-01T09:45:02.443727+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"535'7004","reported_seq":7080,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:15.829963+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:45:15.829963+0000","last_peered":"2023-10-01T09:45:15.829963+0000","last_clean":"2023-10-01T09:45:15.829963+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:45:15.829963+0000","last_undegraded":"2023-10-01T09:45:15.829963+0000","last_fullsized":"2023-10-01T09:45:15.829963+0000","mapping_epoch":479,"log_start":"481'300","ondisk_log_start":"481'300","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6704,"log_dups_size":0,"ondisk_log_size":6704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229900288,"num_objects":3508,"num_object_clones":0,"num_object_copies":10524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7004,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":67,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.444144+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:45:02.444144+0000","last_peered":"2023-10-01T09:45:02.444144+0000","last_clean":"2023-10-01T09:45:02.444144+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:45:02.444144+0000","last_undegraded":"2023-10-01T09:45:02.444144+0000","last_fullsized":"2023-10-01T09:45:02.444144+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":552,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.438196+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:45:02.438196+0000","last_peered":"2023-10-01T09:45:02.438196+0000","last_clean":"2023-10-01T09:45:02.438196+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:45:02.438196+0000","last_undegraded":"2023-10-01T09:45:02.438196+0000","last_fullsized":"2023-10-01T09:45:02.438196+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":258,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437162+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:45:02.437162+0000","last_peered":"2023-10-01T09:45:02.437162+0000","last_clean":"2023-10-01T09:45:02.437162+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:45:02.437162+0000","last_undegraded":"2023-10-01T09:45:02.437162+0000","last_fullsized":"2023-10-01T09:45:02.437162+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":501,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.438299+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:45:02.438299+0000","last_peered":"2023-10-01T09:45:02.438299+0000","last_clean":"2023-10-01T09:45:02.438299+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:45:02.438299+0000","last_undegraded":"2023-10-01T09:45:02.438299+0000","last_fullsized":"2023-10-01T09:45:02.438299+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":616,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.449437+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:45:10.449437+0000","last_peered":"2023-10-01T09:45:10.449437+0000","last_clean":"2023-10-01T09:45:10.449437+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:45:10.449437+0000","last_undegraded":"2023-10-01T09:45:10.449437+0000","last_fullsized":"2023-10-01T09:45:10.449437+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":270,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437469+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:45:02.437469+0000","last_peered":"2023-10-01T09:45:02.437469+0000","last_clean":"2023-10-01T09:45:02.437469+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:45:02.437469+0000","last_undegraded":"2023-10-01T09:45:02.437469+0000","last_fullsized":"2023-10-01T09:45:02.437469+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":59,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.449542+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:45:10.449542+0000","last_peered":"2023-10-01T09:45:10.449542+0000","last_clean":"2023-10-01T09:45:10.449542+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:45:10.449542+0000","last_undegraded":"2023-10-01T09:45:10.449542+0000","last_fullsized":"2023-10-01T09:45:10.449542+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":270,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437736+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:45:02.437736+0000","last_peered":"2023-10-01T09:45:02.437736+0000","last_clean":"2023-10-01T09:45:02.437736+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:45:02.437736+0000","last_undegraded":"2023-10-01T09:45:02.437736+0000","last_fullsized":"2023-10-01T09:45:02.437736+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":447,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.437926+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:45:02.437926+0000","last_peered":"2023-10-01T09:45:02.437926+0000","last_clean":"2023-10-01T09:45:02.437926+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:45:02.437926+0000","last_undegraded":"2023-10-01T09:45:02.437926+0000","last_fullsized":"2023-10-01T09:45:02.437926+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":77,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.449600+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:45:10.449600+0000","last_peered":"2023-10-01T09:45:10.449600+0000","last_clean":"2023-10-01T09:45:10.449600+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:45:10.449600+0000","last_undegraded":"2023-10-01T09:45:10.449600+0000","last_fullsized":"2023-10-01T09:45:10.449600+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":493,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-10-01T09:45:10.447278+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:45:10.447278+0000","last_peered":"2023-10-01T09:45:10.447278+0000","last_clean":"2023-10-01T09:45:10.447278+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:45:10.447278+0000","last_undegraded":"2023-10-01T09:45:10.447278+0000","last_fullsized":"2023-10-01T09:45:10.447278+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":692,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.440605+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:45:02.440605+0000","last_peered":"2023-10-01T09:45:02.440605+0000","last_clean":"2023-10-01T09:45:02.440605+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:45:02.440605+0000","last_undegraded":"2023-10-01T09:45:02.440605+0000","last_fullsized":"2023-10-01T09:45:02.440605+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":386,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-10-01T09:45:02.445280+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:45:02.445280+0000","last_peered":"2023-10-01T09:45:02.445280+0000","last_clean":"2023-10-01T09:45:02.445280+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:45:02.445280+0000","last_undegraded":"2023-10-01T09:45:02.445280+0000","last_fullsized":"2023-10-01T09:45:02.445280+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3593076760,"num_objects":54827,"num_object_clones":0,"num_object_copies":164481,"num_objects_missing_on_primary":1863,"num_objects_missing":1863,"num_objects_degraded":6932,"num_objects_misplaced":3336,"num_objects_unfound":0,"num_objects_dirty":54827,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":111613,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8744,"num_bytes_recovered":573046784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11231506432,"data_stored":11231494216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111113,"ondisk_log_size":111113,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430736,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2010440,"kb_used_data":1103232,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":91734712,"statfs":{"total":95995035648,"available":93936345088,"internally_reserved":0,"allocated":1129709568,"data_stored":1128768948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.79300000000000004,"15min":2.6280000000000001},"min":{"1min":0.254,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":3.3849999999999998,"5min":6.8550000000000004,"15min":108.90000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.626,"15min":2.7559999999999998},"min":{"1min":0.25600000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.98399999999999999,"5min":5.6639999999999997,"15min":109.187},"last":0.307}]},{"osd":1,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":2.843,"15min":5.2329999999999997},"min":{"1min":0.314,"5min":0.248,"15min":0.222},"max":{"1min":3.4089999999999998,"5min":229.161,"15min":229.161},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.49099999999999999,"5min":2.7109999999999999,"15min":5.2290000000000001},"min":{"1min":0.29599999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":1.427,"5min":230.126,"15min":230.126},"last":0.47799999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:44:54 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.81399999999999995,"15min":1.6519999999999999},"min":{"1min":0.30399999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.4300000000000002,"5min":5.3090000000000002,"15min":14.901999999999999},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.76100000000000001,"15min":1.6299999999999999},"min":{"1min":0.33200000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4630000000000001,"5min":5.6929999999999996,"15min":14.682},"last":0.34399999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.79200000000000004,"15min":2.2810000000000001},"min":{"1min":0.32600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.3149999999999999,"5min":5.7750000000000004,"15min":76.555000000000007},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.82499999999999996,"15min":2.2709999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.8479999999999999,"5min":6.548,"15min":76.534000000000006},"last":0.39900000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.80200000000000005,"15min":1.0409999999999999},"min":{"1min":0.31,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":3.4609999999999999,"5min":6.8769999999999998,"15min":9.6020000000000003},"last":0.751},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.82599999999999996,"15min":0.999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28000000000000003},"max":{"1min":0.63900000000000001,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.60099999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":2.177,"15min":1.5800000000000001},"min":{"1min":0.34399999999999997,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":0.871,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.61299999999999999,"5min":2.1789999999999998,"15min":1.6619999999999999},"min":{"1min":0.307,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.3319999999999999,"5min":135.339,"15min":135.339},"last":0.46899999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":1.1639999999999999,"15min":1.1359999999999999},"min":{"1min":0.38,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":0.998,"5min":11.58,"15min":11.853999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.94199999999999995,"15min":1.097},"min":{"1min":0.38300000000000001,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":2.298,"5min":13.021000000000001,"15min":14.584},"last":0.42799999999999999}]}]},{"osd":6,"up_from":294,"seq":1262720385247,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1886788,"kb_used_data":1549180,"kb_used_omap":6,"kb_used_meta":337593,"kb_avail":91858364,"statfs":{"total":95995035648,"available":94062964736,"internally_reserved":0,"allocated":1586360320,"data_stored":1585423651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":345695277},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":1.865,"15min":3.9809999999999999},"min":{"1min":0.27100000000000002,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":0.80200000000000005,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.495},{"interface":"front","average":{"1min":0.58999999999999997,"5min":2.0640000000000001,"15min":4.0209999999999999},"min":{"1min":0.311,"5min":0.245,"15min":0.221},"max":{"1min":2.1760000000000002,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.36699999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":2.9470000000000001,"15min":4.8600000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.24199999999999999},"max":{"1min":1.9059999999999999,"5min":168.947,"15min":168.947},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":2.9980000000000002,"15min":4.8109999999999999},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":0.59899999999999998,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.376}]},{"osd":2,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.75600000000000001,"15min":1.5249999999999999},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":1.9350000000000001,"5min":8.657,"15min":10.92},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.76000000000000001,"15min":1.542},"min":{"1min":0.32300000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.2649999999999999,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.39400000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.70599999999999996,"15min":2.0699999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.1909999999999998,"5min":4.149,"15min":89.753},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.875,"15min":2.0179999999999998},"min":{"1min":0.38500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.1379999999999999,"5min":8.5470000000000006,"15min":89.849999999999994},"last":0.40200000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":1.4510000000000001,"15min":1.0680000000000001},"min":{"1min":0.27200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":0.623,"5min":63.134,"15min":63.134},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.4219999999999999,"15min":1.0669999999999999},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":0.73499999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.47599999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:44:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.246,"15min":1.073},"min":{"1min":0.28799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":3.2890000000000001,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.307},{"interface":"front","average":{"1min":0.624,"5min":1.3260000000000001,"15min":1.1859999999999999},"min":{"1min":0.29899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.887,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.33900000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.67800000000000005,"15min":1.272},"min":{"1min":0.40200000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.83699999999999997,"5min":3.492,"15min":94.245999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.87,"15min":1.379},"min":{"1min":0.42299999999999999,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":2.202,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.44}]}]},{"osd":1,"up_from":94,"seq":403726926224,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1485456,"kb_used_data":671368,"kb_used_omap":7,"kb_used_meta":814072,"kb_avail":92259696,"statfs":{"total":95995035648,"available":94473928704,"internally_reserved":0,"allocated":687480832,"data_stored":685426729,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":833610151},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7469999999999999,"5min":0.93400000000000005,"15min":4.101},"min":{"1min":0.189,"5min":0.186,"15min":0.182},"max":{"1min":32.735999999999997,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.222},{"interface":"front","average":{"1min":2.7959999999999998,"5min":1.147,"15min":3.6240000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":32.591999999999999,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.23899999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:44:39 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.42399999999999999,"15min":2.4889999999999999},"min":{"1min":0.23300000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":1.3240000000000001,"5min":2.8740000000000001,"15min":56.573},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.61199999999999999,"15min":3.1829999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":2.2200000000000002,"5min":6.9710000000000001,"15min":154.197},"last":0.27200000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.752,"15min":3.7629999999999999},"min":{"1min":0.254,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":0.48899999999999999,"5min":18.503,"15min":200.56800000000001},"last":0.31},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.70799999999999996,"15min":4.484},"min":{"1min":0.26300000000000001,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":2.7360000000000002,"5min":16.515000000000001,"15min":200.63800000000001},"last":0.28299999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.73299999999999998,"15min":2.2160000000000002},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":2.9209999999999998,"5min":4.5789999999999997,"15min":154.101},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.58199999999999996,"15min":2.3420000000000001},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.56200000000000006,"5min":2.7290000000000001,"15min":154.56999999999999},"last":0.379}]},{"osd":5,"last update":"Sun Oct 1 09:44:39 2023","interfaces":[{"interface":"back","average":{"1min":3.593,"5min":1.272,"15min":2.2759999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":59.561,"5min":59.561,"15min":154.22999999999999},"last":0.316},{"interface":"front","average":{"1min":3.5979999999999999,"5min":1.24,"15min":2.5680000000000001},"min":{"1min":0.35699999999999998,"5min":0.309,"15min":0.307},"max":{"1min":59.536000000000001,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.42399999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.99399999999999999,"15min":2.5409999999999999},"min":{"1min":0.33500000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.86299999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.96999999999999997,"15min":2.419},"min":{"1min":0.38200000000000001,"5min":0.34599999999999997,"15min":0.32700000000000001},"max":{"1min":2.194,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.41299999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.94999999999999996,"15min":2.5529999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.33900000000000002},"max":{"1min":2.331,"5min":9.2460000000000004,"15min":154.20699999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.82999999999999996,"15min":2.609},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":1.9750000000000001,"5min":5.0110000000000001,"15min":154.51599999999999},"last":0.40699999999999997}]}]},{"osd":0,"up_from":89,"seq":382252089758,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3783084,"kb_used_data":2865700,"kb_used_omap":8,"kb_used_meta":917367,"kb_avail":89962068,"statfs":{"total":95995035648,"available":92121157632,"internally_reserved":0,"allocated":2934476800,"data_stored":2932455398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":939384770},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:44:16 2023","interfaces":[{"interface":"back","average":{"1min":0.317,"5min":1.1839999999999999,"15min":4.1120000000000001},"min":{"1min":0.21199999999999999,"5min":0.17799999999999999,"15min":0.17399999999999999},"max":{"1min":0.46300000000000002,"5min":52.316000000000003,"15min":216.821},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.317,"5min":2.7919999999999998,"15min":4.9809999999999999},"min":{"1min":0.24399999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.435,"5min":206.60599999999999,"15min":217.02699999999999},"last":0.36199999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.69999999999999996,"15min":2.0339999999999998},"min":{"1min":0.29299999999999998,"5min":0.246,"15min":0.20100000000000001},"max":{"1min":4.2430000000000003,"5min":4.9459999999999997,"15min":19.788},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.70799999999999996,"15min":2.1200000000000001},"min":{"1min":0.30399999999999999,"5min":0.23000000000000001,"15min":0.189},"max":{"1min":3.4940000000000002,"5min":4.5359999999999996,"15min":25.844999999999999},"last":0.49299999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:44:16 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.90200000000000002,"15min":3.46},"min":{"1min":0.26900000000000002,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.59199999999999997,"5min":8.2889999999999997,"15min":222.16499999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.71099999999999997,"5min":1.0369999999999999,"15min":3.375},"min":{"1min":0.26900000000000002,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":3.4119999999999999,"5min":9.7539999999999996,"15min":222.227},"last":0.38600000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.341,"15min":2.028},"min":{"1min":0.34000000000000002,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":4.048,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.64700000000000002,"5min":1.3140000000000001,"15min":2.0880000000000001},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":4,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.41699999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.76300000000000001,"15min":1.8069999999999999},"min":{"1min":0.33200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.476,"5min":5.8769999999999998,"15min":16.808},"last":0.436},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.90300000000000002,"15min":1.708},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":4.2729999999999997,"5min":5.0629999999999997,"15min":17.507000000000001},"last":0.55100000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:44:27 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.95599999999999996,"15min":1.732},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.81399999999999995,"5min":6.5,"15min":17.535},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.504,"5min":1.0920000000000001,"15min":1.8300000000000001},"min":{"1min":0.38700000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":0.86199999999999999,"5min":10.446,"15min":15.651999999999999},"last":0.52400000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.86199999999999999,"15min":2.0579999999999998},"min":{"1min":0.375,"5min":0.374,"15min":0.374},"max":{"1min":4.3360000000000003,"5min":4.976,"15min":48.895000000000003},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.88900000000000001,"15min":2.1179999999999999},"min":{"1min":0.439,"5min":0.371,"15min":0.35099999999999998},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.56100000000000005}]}]},{"osd":2,"up_from":0,"seq":425201762698,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599171,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1502848,"kb_used_data":674808,"kb_used_omap":3,"kb_used_meta":828028,"kb_avail":92242304,"statfs":{"total":95995035648,"available":94456119296,"internally_reserved":0,"allocated":691003392,"data_stored":688899134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":847901415},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":1.6839999999999999,"15min":4.843},"min":{"1min":0.20399999999999999,"5min":0.183,"15min":0.17899999999999999},"max":{"1min":0.877,"5min":64.504000000000005,"15min":137.34399999999999},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.34599999999999997,"5min":1.6000000000000001,"15min":4.9329999999999998},"min":{"1min":0.20599999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.80200000000000005,"5min":64.448999999999998,"15min":137.73400000000001},"last":0.27600000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.67900000000000005,"15min":5.0720000000000001},"min":{"1min":0.252,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":0.60499999999999998,"5min":14.917999999999999,"15min":162.22999999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.60699999999999998,"15min":5.125},"min":{"1min":0.26400000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.57399999999999995,"5min":15.074,"15min":162.70699999999999},"last":0.25700000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.60099999999999998,"15min":1.6859999999999999},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.20799999999999999},"max":{"1min":0.99199999999999999,"5min":7.141,"15min":14.254},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.432,"5min":0.55900000000000005,"15min":1.77},"min":{"1min":0.27000000000000002,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":1.111,"5min":4.5049999999999999,"15min":15.523},"last":0.33700000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":1.371,"15min":1.7769999999999999},"min":{"1min":0.33200000000000002,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":0.64200000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.4570000000000001,"15min":2.0409999999999999},"min":{"1min":0.35799999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.5609999999999999,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.36799999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:44:32 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":1.0780000000000001,"15min":2.3530000000000002},"min":{"1min":0.32000000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.196,"5min":9.4320000000000004,"15min":28.117999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.073,"15min":2.4609999999999999},"min":{"1min":0.33600000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.92300000000000004,"5min":10.798999999999999,"15min":28.34},"last":0.34899999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:44:28 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.7350000000000001,"15min":1.8999999999999999},"min":{"1min":0.37,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":1.1000000000000001,"5min":73.887,"15min":73.887},"last":0.377},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.952,"15min":1.974},"min":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":1.294,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.41399999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.96299999999999997,"15min":1.8360000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":3.0299999999999998,"5min":7.0620000000000003,"15min":13.085000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.86899999999999999,"15min":1.716},"min":{"1min":0.40000000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.0150000000000001,"5min":4.5810000000000004,"15min":13.191000000000001},"last":0.48899999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019421,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3650436,"kb_used_data":2752060,"kb_used_omap":0,"kb_used_meta":898368,"kb_avail":90094716,"statfs":{"total":95995035648,"available":92256989184,"internally_reserved":0,"allocated":2818109440,"data_stored":2817172326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919928832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.4950000000000001,"15min":4.1369999999999996},"min":{"1min":0.22500000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.9889999999999999,"5min":79.837000000000003,"15min":155.315},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":1.6519999999999999,"15min":4.5110000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.77600000000000002,"5min":78.983000000000004,"15min":155.334},"last":0.29599999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":1.5489999999999999,"15min":6.298},"min":{"1min":0.33000000000000002,"5min":0.252,"15min":0.224},"max":{"1min":0.67700000000000005,"5min":77.953000000000003,"15min":190.61199999999999},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.754,"5min":1.468,"15min":7.0140000000000002},"min":{"1min":0.35299999999999998,"5min":0.26700000000000002,"15min":0.25600000000000001},"max":{"1min":3.1339999999999999,"5min":77.962999999999994,"15min":253.273},"last":0.57199999999999995}]},{"osd":2,"last update":"Sun Oct 1 09:44:50 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.77200000000000002,"15min":2.375},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.2329999999999997,"5min":5.702,"15min":170.45400000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.755,"15min":2.133},"min":{"1min":0.33100000000000002,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":0.71099999999999997,"5min":5.6150000000000002,"15min":58.771000000000001},"last":0.45800000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.77200000000000002,"15min":2.1379999999999999},"min":{"1min":0.31,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.0299999999999998,"5min":5.5279999999999996,"15min":140.881},"last":2.0379999999999998},{"interface":"front","average":{"1min":0.749,"5min":0.68300000000000005,"15min":2.327},"min":{"1min":0.312,"5min":0.312,"15min":0.311},"max":{"1min":3.161,"5min":3.786,"15min":140.911},"last":6.2969999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":1.929,"15min":1.52},"min":{"1min":0.33800000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":1.7509999999999999,"5min":110.003,"15min":110.003},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.84499999999999997,"5min":1.9239999999999999,"15min":1.405},"min":{"1min":0.38400000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.2909999999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.59599999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:45:14 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.76000000000000001,"15min":0.94199999999999995},"min":{"1min":0.32800000000000001,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":3.0539999999999998,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.79000000000000004,"15min":1.0569999999999999},"min":{"1min":0.35899999999999999,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":1.028,"5min":6.1660000000000004,"15min":10.076000000000001},"last":1.028}]},{"osd":7,"last update":"Sun Oct 1 09:44:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.65000000000000002,"15min":1.1399999999999999},"min":{"1min":0.36399999999999999,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":3.0830000000000002,"5min":3.8580000000000001,"15min":38.034999999999997},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.91600000000000004,"15min":1.4350000000000001},"min":{"1min":0.39900000000000002,"5min":0.36699999999999999,"15min":0.308},"max":{"1min":0.80000000000000004,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.61599999999999999}]}]},{"osd":5,"up_from":379,"seq":1627792605328,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1788208,"kb_used_data":1415464,"kb_used_omap":13,"kb_used_meta":372722,"kb_avail":91956944,"statfs":{"total":95995035648,"available":94163910656,"internally_reserved":0,"allocated":1449435136,"data_stored":1448513926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":381667826},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,0,0,0,1,2,4,3,4],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":1.244,"15min":2.2229999999999999},"min":{"1min":0.24299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":27.584,"5min":27.584,"15min":84.340999999999994},"last":0.54700000000000004},{"interface":"front","average":{"1min":1.9430000000000001,"5min":1.1200000000000001,"15min":2.1280000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23300000000000001},"max":{"1min":27.669,"5min":27.669,"15min":84.356999999999999},"last":0.755}]},{"osd":1,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":3.4430000000000001,"15min":5.6900000000000004},"min":{"1min":0.28699999999999998,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":3.7799999999999998,"5min":149.94900000000001,"15min":295.71300000000002},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.71199999999999997,"5min":3.7149999999999999,"15min":5.6859999999999999},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":3.9580000000000002,"5min":149.41999999999999,"15min":299.59399999999999},"last":0.505}]},{"osd":2,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.83799999999999997,"15min":1.552},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.7029999999999998,"5min":5.3470000000000004,"15min":14.199999999999999},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.84599999999999997,"5min":1.056,"15min":1.5309999999999999},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":3.944,"5min":6.984,"15min":14.430999999999999},"last":0.54000000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.876,"15min":1.46},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.0209999999999999,"5min":10.022,"15min":14.004},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.91300000000000003,"15min":1.5940000000000001},"min":{"1min":0.34300000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.8610000000000002,"5min":5.4400000000000004,"15min":13.991},"last":0.78600000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":2.085,"15min":1.417},"min":{"1min":0.28599999999999998,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":4.0839999999999996,"5min":122.971,"15min":122.971},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.86299999999999999,"5min":2.1949999999999998,"15min":1.6699999999999999},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.0810000000000004,"5min":122.89100000000001,"15min":122.89100000000001},"last":0.51300000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":1.0169999999999999,"15min":1.0269999999999999},"min":{"1min":0.32500000000000001,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":4.3300000000000001,"5min":10.500999999999999,"15min":12.545},"last":0.434},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.97199999999999998,"15min":1.264},"min":{"1min":0.314,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":4.0599999999999996,"5min":6.4729999999999999,"15min":14.193},"last":0.35199999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.0529999999999999,"15min":1.232},"min":{"1min":0.378,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.0999999999999996,"5min":5.758,"15min":10.813000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.996,"5min":1.1339999999999999,"15min":1.0840000000000001},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":4.2000000000000002,"5min":10.039,"15min":12.500999999999999},"last":0.47999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2919895040,"data_stored":2919890968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":675479552,"data_stored":675479552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":684851200,"data_stored":684851200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2811105280,"data_stored":2811101208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1443233792,"data_stored":1443233792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1577979904,"data_stored":1577975832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1118961664,"data_stored":1118961664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:45:21.964 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:22.044 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:22.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:22.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:22.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:45:22.148 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:22.180 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:22.332 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:22.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:22.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:22.411 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:45:22.411 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:45:22.439 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:22.474 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:22.607 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:22.625 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:22.628 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:22 smithi138 ceph-mon[192513]: from='client.37113 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:22.628 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:22 smithi138 ceph-mon[185886]: from='client.37113 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:22.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:22.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:22.730 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:22.757 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:22 smithi190 ceph-mon[160435]: from='client.37113 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:22.823 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:22.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:22.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:22.928 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:22.957 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:23.429 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:23.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:23.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:23.492 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:45:23.533 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:23.560 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:23.607 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:23.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:23.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:23.716 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:23.745 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:23 smithi190 ceph-mon[160435]: pgmap v1577: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 6868/163065 objects degraded (4.212%); 3296/163065 objects misplaced (2.021%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:45:23.746 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:23 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6868/163065 objects degraded (4.212%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:23.747 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:23.767 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:23 smithi138 ceph-mon[192513]: pgmap v1577: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 6868/163065 objects degraded (4.212%); 3296/163065 objects misplaced (2.021%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:45:23.768 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:23 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6868/163065 objects degraded (4.212%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:23.768 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:23 smithi138 ceph-mon[185886]: pgmap v1577: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 141 op/s; 6868/163065 objects degraded (4.212%); 3296/163065 objects misplaced (2.021%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:45:23.768 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:23 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6868/163065 objects degraded (4.212%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:23.801 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:23.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:23.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:23.908 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:23.939 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:23.981 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:45:23.983 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:23.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:23.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:24.090 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:24.120 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:24.241 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:24.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:24.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:24.345 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:24.377 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:24.431 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:24.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:24.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:24.461 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:24.486 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:24 smithi138 ceph-mon[185886]: pgmap v1578: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 6868/162924 objects degraded (4.215%); 3296/162924 objects misplaced (2.023%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:45:24.541 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:24.571 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:24.575 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:24.617 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:24.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:24.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:24.722 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:24.752 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:24 smithi138 ceph-mon[192513]: pgmap v1578: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 6868/162924 objects degraded (4.215%); 3296/162924 objects misplaced (2.023%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:45:24.755 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:24 smithi190 ceph-mon[160435]: pgmap v1578: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 142 op/s; 6868/162924 objects degraded (4.215%); 3296/162924 objects misplaced (2.023%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:45:24.950 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:45:25.017 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:25.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:25.021 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:45:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:45:24] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-01T09:45:25.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:25.122 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:25.157 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:25.219 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:25.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:25.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:25.328 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:25.356 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:25.487 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:25 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/131389267' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:25.487 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:45:25.502 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:45:25.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:25 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/131389267' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:25.792 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:45:25.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:25 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/131389267' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:25.792 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:45:26.505 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:26.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:26.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:26 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:45:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:26 smithi138 ceph-mon[185886]: failed to encode map e537 with expected crc 2023-10-01T09:45:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:26 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/131389267' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:45:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:26 smithi138 ceph-mon[185886]: osdmap e537: 8 total, 8 up, 7 in 2023-10-01T09:45:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:26 smithi138 ceph-mon[185886]: failed to encode map e537 with expected crc 2023-10-01T09:45:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:26 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:45:26.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:26 smithi138 ceph-mon[185886]: pgmap v1580: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 6868/161628 objects degraded (4.249%); 3296/161628 objects misplaced (2.039%); 480 KiB/s, 7 objects/s recovering 2023-10-01T09:45:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:26 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:45:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:26 smithi138 ceph-mon[192513]: failed to encode map e537 with expected crc 2023-10-01T09:45:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:26 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/131389267' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:45:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:26 smithi138 ceph-mon[192513]: osdmap e537: 8 total, 8 up, 7 in 2023-10-01T09:45:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:26 smithi138 ceph-mon[192513]: failed to encode map e537 with expected crc 2023-10-01T09:45:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:26 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:45:26.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:26 smithi138 ceph-mon[192513]: pgmap v1580: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 6868/161628 objects degraded (4.249%); 3296/161628 objects misplaced (2.039%); 480 KiB/s, 7 objects/s recovering 2023-10-01T09:45:26.609 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:26.638 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:26 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:45:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:26 smithi190 ceph-mon[160435]: failed to encode map e537 with expected crc 2023-10-01T09:45:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:26 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/131389267' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:45:26.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:26 smithi190 ceph-mon[160435]: osdmap e537: 8 total, 8 up, 7 in 2023-10-01T09:45:26.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:26 smithi190 ceph-mon[160435]: failed to encode map e537 with expected crc 2023-10-01T09:45:26.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:26 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:45:26.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:26 smithi190 ceph-mon[160435]: pgmap v1580: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 6868/161628 objects degraded (4.249%); 3296/161628 objects misplaced (2.039%); 480 KiB/s, 7 objects/s recovering 2023-10-01T09:45:26.939 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:26.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:26.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:27.043 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:27.081 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:27.134 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:27.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:27.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:27.244 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:27.275 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:27.320 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:27.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:27.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:27.375 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:27.424 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:27.427 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:45:27.427 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:45:27.454 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:27.859 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:45:27.859 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:45:27.859 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:45:27.859 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:45:27.861 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:27.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:27.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:27.874 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:45:27.966 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:27.996 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:28.042 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:28.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:28.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:28.054 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:45:28.054 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:45:28.055 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:45:28.055 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:45:28.055 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:45:28.055 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:45:28.055 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:45:28.055 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:45:28.055 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:45:28.055 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:45:28.055 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:45:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:27 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3618826620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:28.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:27 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6868/161628 objects degraded (4.249%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:28.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:27 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3618826620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:28.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:27 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6868/161628 objects degraded (4.249%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:28.146 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:28.179 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:28.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:27 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3618826620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:28.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:27 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6868/161628 objects degraded (4.249%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:28.329 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:28.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:28.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:28.433 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:28.465 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:28.594 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:28.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:28.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:28.700 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:28.730 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:28.757 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:45:28.826 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:28.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:28.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:28.869 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:45:28.938 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:28.970 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:29.000 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:28 smithi138 ceph-mon[185886]: pgmap v1581: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 6818/159972 objects degraded (4.262%); 3235/159972 objects misplaced (2.022%); 818 KiB/s, 12 objects/s recovering 2023-10-01T09:45:29.000 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:28 smithi138 ceph-mon[192513]: pgmap v1581: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 6818/159972 objects degraded (4.262%); 3235/159972 objects misplaced (2.022%); 818 KiB/s, 12 objects/s recovering 2023-10-01T09:45:29.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:28 smithi190 ceph-mon[160435]: pgmap v1581: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 165 op/s; 6818/159972 objects degraded (4.262%); 3235/159972 objects misplaced (2.022%); 818 KiB/s, 12 objects/s recovering 2023-10-01T09:45:29.120 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:29.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:29.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:29.233 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:29.259 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:29.359 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:29.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:29.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:29.464 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:29.494 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:29.754 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:29.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:29.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:29.858 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:29.890 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:29.945 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:29.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:29.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:30.052 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:30.082 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:30.137 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:30.148 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:30.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:30.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:30.254 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:30.276 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:30.282 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:30.327 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:30.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:30.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:30.443 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:30.481 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:30.505 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:45:30.506 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:45:30.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:30 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:30 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:30.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:30 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:30.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:30 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:30.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:30 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:30.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:30 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:31.002 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:45:31.397 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:31.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:31.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:31.502 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:31.535 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:31.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:31 smithi190 ceph-mon[160435]: pgmap v1582: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 6818/159621 objects degraded (4.271%); 3235/159621 objects misplaced (2.027%); 781 KiB/s, 12 objects/s recovering 2023-10-01T09:45:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:31 smithi190 ceph-mon[160435]: from='client.47156 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:31.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:31 smithi138 ceph-mon[185886]: pgmap v1582: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 6818/159621 objects degraded (4.271%); 3235/159621 objects misplaced (2.027%); 781 KiB/s, 12 objects/s recovering 2023-10-01T09:45:31.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:31 smithi138 ceph-mon[185886]: from='client.47156 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:31 smithi138 ceph-mon[192513]: pgmap v1582: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 6818/159621 objects degraded (4.271%); 3235/159621 objects misplaced (2.027%); 781 KiB/s, 12 objects/s recovering 2023-10-01T09:45:31.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:31 smithi138 ceph-mon[192513]: from='client.47156 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:32.143 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:32.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:32.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:32.250 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:32.277 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:32.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:32 smithi190 ceph-mon[160435]: pgmap v1583: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 184 op/s; 6769/157542 objects degraded (4.297%); 3176/157542 objects misplaced (2.016%); 742 KiB/s, 11 objects/s recovering 2023-10-01T09:45:32.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:32 smithi138 ceph-mon[192513]: pgmap v1583: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 184 op/s; 6769/157542 objects degraded (4.297%); 3176/157542 objects misplaced (2.016%); 742 KiB/s, 11 objects/s recovering 2023-10-01T09:45:32.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:32 smithi138 ceph-mon[185886]: pgmap v1583: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 184 op/s; 6769/157542 objects degraded (4.297%); 3176/157542 objects misplaced (2.016%); 742 KiB/s, 11 objects/s recovering 2023-10-01T09:45:33.079 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:33.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:33.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:33.183 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:33.212 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:33.260 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:33.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:33.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:33.338 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1582,"stamp":"2023-10-01T09:45:30.344683+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3489863627,"num_objects":53207,"num_object_clones":0,"num_object_copies":159621,"num_objects_missing_on_primary":1760,"num_objects_missing":1760,"num_objects_degraded":6818,"num_objects_misplaced":3235,"num_objects_unfound":0,"num_objects_dirty":53207,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":114367,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9377,"num_bytes_recovered":601392143,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":113531,"ondisk_log_size":113531,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15814640,"kb_used_data":10739192,"kb_used_omap":39,"kb_used_meta":5075352,"kb_avail":640401424,"statfs":{"total":671965249536,"available":655771058176,"internally_reserved":0,"allocated":10996932608,"data_stored":10987001551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5197161071},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-108003328,"num_objects":-1648,"num_object_clones":0,"num_object_copies":-4944,"num_objects_missing_on_primary":-103,"num_objects_missing":-103,"num_objects_degraded":-124,"num_objects_misplaced":-101,"num_objects_unfound":0,"num_objects_dirty":-1648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1648,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":7995392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002724"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":313,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.806048+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:45:18.806048+0000","last_peered":"2023-10-01T09:45:18.806048+0000","last_clean":"2023-10-01T09:45:18.806048+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:45:18.806048+0000","last_undegraded":"2023-10-01T09:45:18.806048+0000","last_fullsized":"2023-10-01T09:45:18.806048+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":157,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.077867+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:45:19.077867+0000","last_peered":"2023-10-01T09:45:19.077867+0000","last_clean":"2023-10-01T09:45:19.077867+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:45:19.077867+0000","last_undegraded":"2023-10-01T09:45:19.077867+0000","last_fullsized":"2023-10-01T09:45:19.077867+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":537,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.077568+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:45:19.077568+0000","last_peered":"2023-10-01T09:45:19.077568+0000","last_clean":"2023-10-01T09:45:19.077568+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:45:19.077568+0000","last_undegraded":"2023-10-01T09:45:19.077568+0000","last_fullsized":"2023-10-01T09:45:19.077568+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":596,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.811123+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:45:18.811123+0000","last_peered":"2023-10-01T09:45:18.811123+0000","last_clean":"2023-10-01T09:45:18.811123+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:45:18.811123+0000","last_undegraded":"2023-10-01T09:45:18.811123+0000","last_fullsized":"2023-10-01T09:45:18.811123+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":477,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.810714+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:45:18.810714+0000","last_peered":"2023-10-01T09:45:18.810714+0000","last_clean":"2023-10-01T09:45:18.810714+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:45:18.810714+0000","last_undegraded":"2023-10-01T09:45:18.810714+0000","last_fullsized":"2023-10-01T09:45:18.810714+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":218,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.858392+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:45:18.858392+0000","last_peered":"2023-10-01T09:45:18.858392+0000","last_clean":"2023-10-01T09:45:18.858392+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:45:18.858392+0000","last_undegraded":"2023-10-01T09:45:18.858392+0000","last_fullsized":"2023-10-01T09:45:18.858392+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":594,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.858448+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:45:18.858448+0000","last_peered":"2023-10-01T09:45:18.858448+0000","last_clean":"2023-10-01T09:45:18.858448+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:45:18.858448+0000","last_undegraded":"2023-10-01T09:45:18.858448+0000","last_fullsized":"2023-10-01T09:45:18.858448+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":524,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.069002+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:45:19.069002+0000","last_peered":"2023-10-01T09:45:19.069002+0000","last_clean":"2023-10-01T09:45:19.069002+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:45:19.069002+0000","last_undegraded":"2023-10-01T09:45:19.069002+0000","last_fullsized":"2023-10-01T09:45:19.069002+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":249,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.808237+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:45:18.808237+0000","last_peered":"2023-10-01T09:45:18.808237+0000","last_clean":"2023-10-01T09:45:18.808237+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:45:18.808237+0000","last_undegraded":"2023-10-01T09:45:18.808237+0000","last_fullsized":"2023-10-01T09:45:18.808237+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":56,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.050573+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:45:19.050573+0000","last_peered":"2023-10-01T09:45:19.050573+0000","last_clean":"2023-10-01T09:45:19.050573+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:45:19.050573+0000","last_undegraded":"2023-10-01T09:45:19.050573+0000","last_fullsized":"2023-10-01T09:45:19.050573+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":62,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.051249+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:45:19.051249+0000","last_peered":"2023-10-01T09:45:19.051249+0000","last_clean":"2023-10-01T09:45:19.051249+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:45:19.051249+0000","last_undegraded":"2023-10-01T09:45:19.051249+0000","last_fullsized":"2023-10-01T09:45:19.051249+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":61,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.269773+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:45:25.269773+0000","last_peered":"2023-10-01T09:45:25.269773+0000","last_clean":"2023-10-01T09:45:25.269773+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:45:25.269773+0000","last_undegraded":"2023-10-01T09:45:25.269773+0000","last_fullsized":"2023-10-01T09:45:25.269773+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":438,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.806466+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:45:18.806466+0000","last_peered":"2023-10-01T09:45:18.806466+0000","last_clean":"2023-10-01T09:45:18.806466+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:45:18.806466+0000","last_undegraded":"2023-10-01T09:45:18.806466+0000","last_fullsized":"2023-10-01T09:45:18.806466+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":319,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.806426+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:45:18.806426+0000","last_peered":"2023-10-01T09:45:18.806426+0000","last_clean":"2023-10-01T09:45:18.806426+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:45:18.806426+0000","last_undegraded":"2023-10-01T09:45:18.806426+0000","last_fullsized":"2023-10-01T09:45:18.806426+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":655,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.050802+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:45:19.050802+0000","last_peered":"2023-10-01T09:45:19.050802+0000","last_clean":"2023-10-01T09:45:19.050802+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:45:19.050802+0000","last_undegraded":"2023-10-01T09:45:19.050802+0000","last_fullsized":"2023-10-01T09:45:19.050802+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":392,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.050992+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:45:19.050992+0000","last_peered":"2023-10-01T09:45:19.050992+0000","last_clean":"2023-10-01T09:45:19.050992+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:45:19.050992+0000","last_undegraded":"2023-10-01T09:45:19.050992+0000","last_fullsized":"2023-10-01T09:45:19.050992+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":65,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.050737+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:45:19.050737+0000","last_peered":"2023-10-01T09:45:19.050737+0000","last_clean":"2023-10-01T09:45:19.050737+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:45:19.050737+0000","last_undegraded":"2023-10-01T09:45:19.050737+0000","last_fullsized":"2023-10-01T09:45:19.050737+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":51,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.808068+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:45:18.808068+0000","last_peered":"2023-10-01T09:45:18.808068+0000","last_clean":"2023-10-01T09:45:18.808068+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:45:18.808068+0000","last_undegraded":"2023-10-01T09:45:18.808068+0000","last_fullsized":"2023-10-01T09:45:18.808068+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":548,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.050810+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:45:19.050810+0000","last_peered":"2023-10-01T09:45:19.050810+0000","last_clean":"2023-10-01T09:45:19.050810+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:45:19.050810+0000","last_undegraded":"2023-10-01T09:45:19.050810+0000","last_fullsized":"2023-10-01T09:45:19.050810+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":560,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.817731+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:45:18.817731+0000","last_peered":"2023-10-01T09:45:18.817731+0000","last_clean":"2023-10-01T09:45:18.817731+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:45:18.817731+0000","last_undegraded":"2023-10-01T09:45:18.817731+0000","last_fullsized":"2023-10-01T09:45:18.817731+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":416,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.810612+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:45:18.810612+0000","last_peered":"2023-10-01T09:45:18.810612+0000","last_clean":"2023-10-01T09:45:18.810612+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:45:18.810612+0000","last_undegraded":"2023-10-01T09:45:18.810612+0000","last_fullsized":"2023-10-01T09:45:18.810612+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":65,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.050988+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:45:19.050988+0000","last_peered":"2023-10-01T09:45:19.050988+0000","last_clean":"2023-10-01T09:45:19.050988+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:45:19.050988+0000","last_undegraded":"2023-10-01T09:45:19.050988+0000","last_fullsized":"2023-10-01T09:45:19.050988+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"537'7116","reported_seq":7186,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:26.892829+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:45:26.892829+0000","last_peered":"2023-10-01T09:45:26.892829+0000","last_clean":"2023-10-01T09:45:26.892829+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:45:26.892829+0000","last_undegraded":"2023-10-01T09:45:26.892829+0000","last_fullsized":"2023-10-01T09:45:26.892829+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7116,"log_dups_size":0,"ondisk_log_size":7116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214695936,"num_objects":3276,"num_object_clones":0,"num_object_copies":9828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7116,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":60,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.050911+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:45:19.050911+0000","last_peered":"2023-10-01T09:45:19.050911+0000","last_clean":"2023-10-01T09:45:19.050911+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:45:19.050911+0000","last_undegraded":"2023-10-01T09:45:19.050911+0000","last_fullsized":"2023-10-01T09:45:19.050911+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":257,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.807708+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:45:18.807708+0000","last_peered":"2023-10-01T09:45:18.807708+0000","last_clean":"2023-10-01T09:45:18.807708+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:45:18.807708+0000","last_undegraded":"2023-10-01T09:45:18.807708+0000","last_fullsized":"2023-10-01T09:45:18.807708+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"537'7132","reported_seq":13014,"reported_epoch":537,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:45:27.048904+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:45:27.048904+0000","last_peered":"2023-10-01T09:45:27.048904+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:45:27.048904+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7132,"log_dups_size":0,"ondisk_log_size":7132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214564864,"num_objects":3274,"num_object_clones":0,"num_object_copies":9822,"num_objects_missing_on_primary":1760,"num_objects_missing":1760,"num_objects_degraded":4966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7132,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1929,"num_bytes_recovered":126418944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1446},{"shards":"0,4,6","objects":1760},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":531,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.051180+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:45:19.051180+0000","last_peered":"2023-10-01T09:45:19.051180+0000","last_clean":"2023-10-01T09:45:19.051180+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:45:19.051180+0000","last_undegraded":"2023-10-01T09:45:19.051180+0000","last_fullsized":"2023-10-01T09:45:19.051180+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":593,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.077557+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:45:19.077557+0000","last_peered":"2023-10-01T09:45:19.077557+0000","last_clean":"2023-10-01T09:45:19.077557+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:45:19.077557+0000","last_undegraded":"2023-10-01T09:45:19.077557+0000","last_fullsized":"2023-10-01T09:45:19.077557+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"537'7106","reported_seq":7179,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:26.638734+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:45:26.638734+0000","last_peered":"2023-10-01T09:45:26.638734+0000","last_clean":"2023-10-01T09:45:26.638734+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:45:26.638734+0000","last_undegraded":"2023-10-01T09:45:26.638734+0000","last_fullsized":"2023-10-01T09:45:26.638734+0000","mapping_epoch":475,"log_start":"480'100","ondisk_log_start":"480'100","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7006,"log_dups_size":0,"ondisk_log_size":7006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217972736,"num_objects":3326,"num_object_clones":0,"num_object_copies":9978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7106,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":617,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.051138+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:45:19.051138+0000","last_peered":"2023-10-01T09:45:19.051138+0000","last_clean":"2023-10-01T09:45:19.051138+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:45:19.051138+0000","last_undegraded":"2023-10-01T09:45:19.051138+0000","last_fullsized":"2023-10-01T09:45:19.051138+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":413,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.078792+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:45:19.078792+0000","last_peered":"2023-10-01T09:45:19.078792+0000","last_clean":"2023-10-01T09:45:19.078792+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:45:19.078792+0000","last_undegraded":"2023-10-01T09:45:19.078792+0000","last_fullsized":"2023-10-01T09:45:19.078792+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"537'7089","reported_seq":13767,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:26.825913+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:45:26.825913+0000","last_peered":"2023-10-01T09:45:26.825913+0000","last_clean":"2023-10-01T09:45:26.825913+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:45:26.825913+0000","last_undegraded":"2023-10-01T09:45:26.825913+0000","last_fullsized":"2023-10-01T09:45:26.825913+0000","mapping_epoch":505,"log_start":"480'100","ondisk_log_start":"480'100","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6989,"log_dups_size":0,"ondisk_log_size":6989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217120768,"num_objects":3313,"num_object_clones":0,"num_object_copies":9939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7089,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":257,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.807991+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:45:18.807991+0000","last_peered":"2023-10-01T09:45:18.807991+0000","last_clean":"2023-10-01T09:45:18.807991+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:45:18.807991+0000","last_undegraded":"2023-10-01T09:45:18.807991+0000","last_fullsized":"2023-10-01T09:45:18.807991+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":516,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.810868+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:45:18.810868+0000","last_peered":"2023-10-01T09:45:18.810868+0000","last_clean":"2023-10-01T09:45:18.810868+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:45:18.810868+0000","last_undegraded":"2023-10-01T09:45:18.810868+0000","last_fullsized":"2023-10-01T09:45:18.810868+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"537'7155","reported_seq":7228,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:30.077399+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:45:30.077399+0000","last_peered":"2023-10-01T09:45:30.077399+0000","last_clean":"2023-10-01T09:45:30.077399+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:45:30.077399+0000","last_undegraded":"2023-10-01T09:45:30.077399+0000","last_fullsized":"2023-10-01T09:45:30.077399+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7155,"log_dups_size":0,"ondisk_log_size":7155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219742208,"num_objects":3353,"num_object_clones":0,"num_object_copies":10059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7155,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":117,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.806715+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:45:18.806715+0000","last_peered":"2023-10-01T09:45:18.806715+0000","last_clean":"2023-10-01T09:45:18.806715+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:45:18.806715+0000","last_undegraded":"2023-10-01T09:45:18.806715+0000","last_fullsized":"2023-10-01T09:45:18.806715+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":191,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.806602+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:45:18.806602+0000","last_peered":"2023-10-01T09:45:18.806602+0000","last_clean":"2023-10-01T09:45:18.806602+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:45:18.806602+0000","last_undegraded":"2023-10-01T09:45:18.806602+0000","last_fullsized":"2023-10-01T09:45:18.806602+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"537'7067","reported_seq":7233,"reported_epoch":537,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:45:27.106732+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:45:27.106732+0000","last_peered":"2023-10-01T09:45:27.106732+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:45:27.106732+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7067,"log_dups_size":0,"ondisk_log_size":7067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216334336,"num_objects":3301,"num_object_clones":0,"num_object_copies":9903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1842,"num_objects_misplaced":3235,"num_objects_unfound":0,"num_objects_dirty":3301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7067,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":2883584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":3235},{"shards":"4,5","objects":-1776}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":435,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.806198+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:45:18.806198+0000","last_peered":"2023-10-01T09:45:18.806198+0000","last_clean":"2023-10-01T09:45:18.806198+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:45:18.806198+0000","last_undegraded":"2023-10-01T09:45:18.806198+0000","last_fullsized":"2023-10-01T09:45:18.806198+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":596,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.077950+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:45:19.077950+0000","last_peered":"2023-10-01T09:45:19.077950+0000","last_clean":"2023-10-01T09:45:19.077950+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:45:19.077950+0000","last_undegraded":"2023-10-01T09:45:19.077950+0000","last_fullsized":"2023-10-01T09:45:19.077950+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"537'7040","reported_seq":7119,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:27.548590+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:45:27.548590+0000","last_peered":"2023-10-01T09:45:27.548590+0000","last_clean":"2023-10-01T09:45:27.548590+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:45:27.548590+0000","last_undegraded":"2023-10-01T09:45:27.548590+0000","last_fullsized":"2023-10-01T09:45:27.548590+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7040,"log_dups_size":0,"ondisk_log_size":7040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218759168,"num_objects":3338,"num_object_clones":0,"num_object_copies":10014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7040,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":431,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.805963+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:45:18.805963+0000","last_peered":"2023-10-01T09:45:18.805963+0000","last_clean":"2023-10-01T09:45:18.805963+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:45:18.805963+0000","last_undegraded":"2023-10-01T09:45:18.805963+0000","last_fullsized":"2023-10-01T09:45:18.805963+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":272,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.806570+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:45:18.806570+0000","last_peered":"2023-10-01T09:45:18.806570+0000","last_clean":"2023-10-01T09:45:18.806570+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:45:18.806570+0000","last_undegraded":"2023-10-01T09:45:18.806570+0000","last_fullsized":"2023-10-01T09:45:18.806570+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"537'7179","reported_seq":7249,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:26.047142+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:45:26.047142+0000","last_peered":"2023-10-01T09:45:26.047142+0000","last_clean":"2023-10-01T09:45:26.047142+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:45:26.047142+0000","last_undegraded":"2023-10-01T09:45:26.047142+0000","last_fullsized":"2023-10-01T09:45:26.047142+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7179,"log_dups_size":0,"ondisk_log_size":7179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217513984,"num_objects":3319,"num_object_clones":0,"num_object_copies":9957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7179,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":400,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.077537+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:45:19.077537+0000","last_peered":"2023-10-01T09:45:19.077537+0000","last_clean":"2023-10-01T09:45:19.077537+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:45:19.077537+0000","last_undegraded":"2023-10-01T09:45:19.077537+0000","last_fullsized":"2023-10-01T09:45:19.077537+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":447,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.077775+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:45:19.077775+0000","last_peered":"2023-10-01T09:45:19.077775+0000","last_clean":"2023-10-01T09:45:19.077775+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:45:19.077775+0000","last_undegraded":"2023-10-01T09:45:19.077775+0000","last_fullsized":"2023-10-01T09:45:19.077775+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"537'6939","reported_seq":7009,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.858141+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:45:25.858141+0000","last_peered":"2023-10-01T09:45:25.858141+0000","last_clean":"2023-10-01T09:45:25.858141+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:45:25.858141+0000","last_undegraded":"2023-10-01T09:45:25.858141+0000","last_fullsized":"2023-10-01T09:45:25.858141+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6939,"log_dups_size":0,"ondisk_log_size":6939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212271104,"num_objects":3239,"num_object_clones":0,"num_object_copies":9717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6939,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":930,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.050902+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:45:19.050902+0000","last_peered":"2023-10-01T09:45:19.050902+0000","last_clean":"2023-10-01T09:45:19.050902+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:45:19.050902+0000","last_undegraded":"2023-10-01T09:45:19.050902+0000","last_fullsized":"2023-10-01T09:45:19.050902+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":267,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.808206+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:45:18.808206+0000","last_peered":"2023-10-01T09:45:18.808206+0000","last_clean":"2023-10-01T09:45:18.808206+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:45:18.808206+0000","last_undegraded":"2023-10-01T09:45:18.808206+0000","last_fullsized":"2023-10-01T09:45:18.808206+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":61,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.078656+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:45:19.078656+0000","last_peered":"2023-10-01T09:45:19.078656+0000","last_clean":"2023-10-01T09:45:19.078656+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:45:19.078656+0000","last_undegraded":"2023-10-01T09:45:19.078656+0000","last_fullsized":"2023-10-01T09:45:19.078656+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"537'7004","reported_seq":7077,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:26.786181+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:45:26.786181+0000","last_peered":"2023-10-01T09:45:26.786181+0000","last_clean":"2023-10-01T09:45:26.786181+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:45:26.786181+0000","last_undegraded":"2023-10-01T09:45:26.786181+0000","last_fullsized":"2023-10-01T09:45:26.786181+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7004,"log_dups_size":0,"ondisk_log_size":7004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220594176,"num_objects":3366,"num_object_clones":0,"num_object_copies":10098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7004,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1202,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.807816+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:45:18.807816+0000","last_peered":"2023-10-01T09:45:18.807816+0000","last_clean":"2023-10-01T09:45:18.807816+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:45:18.807816+0000","last_undegraded":"2023-10-01T09:45:18.807816+0000","last_fullsized":"2023-10-01T09:45:18.807816+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":412,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.806631+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:45:18.806631+0000","last_peered":"2023-10-01T09:45:18.806631+0000","last_clean":"2023-10-01T09:45:18.806631+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:45:18.806631+0000","last_undegraded":"2023-10-01T09:45:18.806631+0000","last_fullsized":"2023-10-01T09:45:18.806631+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":240,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.808066+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:45:18.808066+0000","last_peered":"2023-10-01T09:45:18.808066+0000","last_clean":"2023-10-01T09:45:18.808066+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:45:18.808066+0000","last_undegraded":"2023-10-01T09:45:18.808066+0000","last_fullsized":"2023-10-01T09:45:18.808066+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"537'7083","reported_seq":14374,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:27.018823+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:45:27.018823+0000","last_peered":"2023-10-01T09:45:27.018823+0000","last_clean":"2023-10-01T09:45:27.018823+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:45:27.018823+0000","last_undegraded":"2023-10-01T09:45:27.018823+0000","last_fullsized":"2023-10-01T09:45:27.018823+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7083,"log_dups_size":0,"ondisk_log_size":7083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215810048,"num_objects":3293,"num_object_clones":0,"num_object_copies":9879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7083,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":883,"reported_epoch":536,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:45:18.810475+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:45:18.810475+0000","last_peered":"2023-10-01T09:45:18.810475+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:45:18.810475+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:45:18.810475+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":600,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.077548+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:45:19.077548+0000","last_peered":"2023-10-01T09:45:19.077548+0000","last_clean":"2023-10-01T09:45:19.077548+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:45:19.077548+0000","last_undegraded":"2023-10-01T09:45:19.077548+0000","last_fullsized":"2023-10-01T09:45:19.077548+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":61,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.077699+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:45:19.077699+0000","last_peered":"2023-10-01T09:45:19.077699+0000","last_clean":"2023-10-01T09:45:19.077699+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:45:19.077699+0000","last_undegraded":"2023-10-01T09:45:19.077699+0000","last_fullsized":"2023-10-01T09:45:19.077699+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"537'7005","reported_seq":7078,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:30.194096+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:45:30.194096+0000","last_peered":"2023-10-01T09:45:30.194096+0000","last_clean":"2023-10-01T09:45:30.194096+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:45:30.194096+0000","last_undegraded":"2023-10-01T09:45:30.194096+0000","last_fullsized":"2023-10-01T09:45:30.194096+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7005,"log_dups_size":0,"ondisk_log_size":7005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214106112,"num_objects":3267,"num_object_clones":0,"num_object_copies":9801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7005,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1653,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.807968+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:45:18.807968+0000","last_peered":"2023-10-01T09:45:18.807968+0000","last_clean":"2023-10-01T09:45:18.807968+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:45:18.807968+0000","last_undegraded":"2023-10-01T09:45:18.807968+0000","last_fullsized":"2023-10-01T09:45:18.807968+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":567,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.050626+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:45:19.050626+0000","last_peered":"2023-10-01T09:45:19.050626+0000","last_clean":"2023-10-01T09:45:19.050626+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:45:19.050626+0000","last_undegraded":"2023-10-01T09:45:19.050626+0000","last_fullsized":"2023-10-01T09:45:19.050626+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":345,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.807874+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:45:18.807874+0000","last_peered":"2023-10-01T09:45:18.807874+0000","last_clean":"2023-10-01T09:45:18.807874+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:45:18.807874+0000","last_undegraded":"2023-10-01T09:45:18.807874+0000","last_fullsized":"2023-10-01T09:45:18.807874+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":335,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.806739+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:45:18.806739+0000","last_peered":"2023-10-01T09:45:18.806739+0000","last_clean":"2023-10-01T09:45:18.806739+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:45:18.806739+0000","last_undegraded":"2023-10-01T09:45:18.806739+0000","last_fullsized":"2023-10-01T09:45:18.806739+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"537'7151","reported_seq":7221,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:26.037918+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:45:26.037918+0000","last_peered":"2023-10-01T09:45:26.037918+0000","last_clean":"2023-10-01T09:45:26.037918+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:45:26.037918+0000","last_undegraded":"2023-10-01T09:45:26.037918+0000","last_fullsized":"2023-10-01T09:45:26.037918+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7151,"log_dups_size":0,"ondisk_log_size":7151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221577216,"num_objects":3381,"num_object_clones":0,"num_object_copies":10143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7151,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":744,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.050732+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:45:19.050732+0000","last_peered":"2023-10-01T09:45:19.050732+0000","last_clean":"2023-10-01T09:45:19.050732+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:45:19.050732+0000","last_undegraded":"2023-10-01T09:45:19.050732+0000","last_fullsized":"2023-10-01T09:45:19.050732+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":577,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.810670+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:45:18.810670+0000","last_peered":"2023-10-01T09:45:18.810670+0000","last_clean":"2023-10-01T09:45:18.810670+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:45:18.810670+0000","last_undegraded":"2023-10-01T09:45:18.810670+0000","last_fullsized":"2023-10-01T09:45:18.810670+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":576,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.811030+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:45:18.811030+0000","last_peered":"2023-10-01T09:45:18.811030+0000","last_clean":"2023-10-01T09:45:18.811030+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:45:18.811030+0000","last_undegraded":"2023-10-01T09:45:18.811030+0000","last_fullsized":"2023-10-01T09:45:18.811030+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"537'7067","reported_seq":7141,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:26.800253+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:45:26.800253+0000","last_peered":"2023-10-01T09:45:26.800253+0000","last_clean":"2023-10-01T09:45:26.800253+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:45:26.800253+0000","last_undegraded":"2023-10-01T09:45:26.800253+0000","last_fullsized":"2023-10-01T09:45:26.800253+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7067,"log_dups_size":0,"ondisk_log_size":7067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222298136,"num_objects":3393,"num_object_clones":0,"num_object_copies":10179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3393,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7067,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":975,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.051225+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:45:19.051225+0000","last_peered":"2023-10-01T09:45:19.051225+0000","last_clean":"2023-10-01T09:45:19.051225+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:45:19.051225+0000","last_undegraded":"2023-10-01T09:45:19.051225+0000","last_fullsized":"2023-10-01T09:45:19.051225+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":594,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.808007+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:45:18.808007+0000","last_peered":"2023-10-01T09:45:18.808007+0000","last_clean":"2023-10-01T09:45:18.808007+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:45:18.808007+0000","last_undegraded":"2023-10-01T09:45:18.808007+0000","last_fullsized":"2023-10-01T09:45:18.808007+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":447,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.808037+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:45:18.808037+0000","last_peered":"2023-10-01T09:45:18.808037+0000","last_clean":"2023-10-01T09:45:18.808037+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:45:18.808037+0000","last_undegraded":"2023-10-01T09:45:18.808037+0000","last_fullsized":"2023-10-01T09:45:18.808037+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"537'7024","reported_seq":7102,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.801034+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:45:25.801034+0000","last_peered":"2023-10-01T09:45:25.801034+0000","last_clean":"2023-10-01T09:45:25.801034+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:45:25.801034+0000","last_undegraded":"2023-10-01T09:45:25.801034+0000","last_fullsized":"2023-10-01T09:45:25.801034+0000","mapping_epoch":479,"log_start":"480'300","ondisk_log_start":"480'300","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6724,"log_dups_size":0,"ondisk_log_size":6724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218365952,"num_objects":3332,"num_object_clones":0,"num_object_copies":9996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7024,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":476,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.051062+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:45:19.051062+0000","last_peered":"2023-10-01T09:45:19.051062+0000","last_clean":"2023-10-01T09:45:19.051062+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:45:19.051062+0000","last_undegraded":"2023-10-01T09:45:19.051062+0000","last_fullsized":"2023-10-01T09:45:19.051062+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1170,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.078314+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:45:19.078314+0000","last_peered":"2023-10-01T09:45:19.078314+0000","last_clean":"2023-10-01T09:45:19.078314+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:45:19.078314+0000","last_undegraded":"2023-10-01T09:45:19.078314+0000","last_fullsized":"2023-10-01T09:45:19.078314+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":65,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.050534+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:45:19.050534+0000","last_peered":"2023-10-01T09:45:19.050534+0000","last_clean":"2023-10-01T09:45:19.050534+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:45:19.050534+0000","last_undegraded":"2023-10-01T09:45:19.050534+0000","last_fullsized":"2023-10-01T09:45:19.050534+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"537'7104","reported_seq":7182,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.980989+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:45:25.980989+0000","last_peered":"2023-10-01T09:45:25.980989+0000","last_clean":"2023-10-01T09:45:25.980989+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:45:25.980989+0000","last_undegraded":"2023-10-01T09:45:25.980989+0000","last_fullsized":"2023-10-01T09:45:25.980989+0000","mapping_epoch":479,"log_start":"481'400","ondisk_log_start":"481'400","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6704,"log_dups_size":0,"ondisk_log_size":6704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223346688,"num_objects":3408,"num_object_clones":0,"num_object_copies":10224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7104,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":69,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.050965+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:45:19.050965+0000","last_peered":"2023-10-01T09:45:19.050965+0000","last_clean":"2023-10-01T09:45:19.050965+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:45:19.050965+0000","last_undegraded":"2023-10-01T09:45:19.050965+0000","last_fullsized":"2023-10-01T09:45:19.050965+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":554,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.811119+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:45:18.811119+0000","last_peered":"2023-10-01T09:45:18.811119+0000","last_clean":"2023-10-01T09:45:18.811119+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:45:18.811119+0000","last_undegraded":"2023-10-01T09:45:18.811119+0000","last_fullsized":"2023-10-01T09:45:18.811119+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":260,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.807776+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:45:18.807776+0000","last_peered":"2023-10-01T09:45:18.807776+0000","last_clean":"2023-10-01T09:45:18.807776+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:45:18.807776+0000","last_undegraded":"2023-10-01T09:45:18.807776+0000","last_fullsized":"2023-10-01T09:45:18.807776+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":503,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.810978+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:45:18.810978+0000","last_peered":"2023-10-01T09:45:18.810978+0000","last_clean":"2023-10-01T09:45:18.810978+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:45:18.810978+0000","last_undegraded":"2023-10-01T09:45:18.810978+0000","last_fullsized":"2023-10-01T09:45:18.810978+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":617,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.077845+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:45:19.077845+0000","last_peered":"2023-10-01T09:45:19.077845+0000","last_clean":"2023-10-01T09:45:19.077845+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:45:19.077845+0000","last_undegraded":"2023-10-01T09:45:19.077845+0000","last_fullsized":"2023-10-01T09:45:19.077845+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":272,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.807786+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:45:18.807786+0000","last_peered":"2023-10-01T09:45:18.807786+0000","last_clean":"2023-10-01T09:45:18.807786+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:45:18.807786+0000","last_undegraded":"2023-10-01T09:45:18.807786+0000","last_fullsized":"2023-10-01T09:45:18.807786+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":60,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.077656+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:45:19.077656+0000","last_peered":"2023-10-01T09:45:19.077656+0000","last_clean":"2023-10-01T09:45:19.077656+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:45:19.077656+0000","last_undegraded":"2023-10-01T09:45:19.077656+0000","last_fullsized":"2023-10-01T09:45:19.077656+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":272,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.808276+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:45:18.808276+0000","last_peered":"2023-10-01T09:45:18.808276+0000","last_clean":"2023-10-01T09:45:18.808276+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:45:18.808276+0000","last_undegraded":"2023-10-01T09:45:18.808276+0000","last_fullsized":"2023-10-01T09:45:18.808276+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":449,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.810725+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:45:18.810725+0000","last_peered":"2023-10-01T09:45:18.810725+0000","last_clean":"2023-10-01T09:45:18.810725+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:45:18.810725+0000","last_undegraded":"2023-10-01T09:45:18.810725+0000","last_fullsized":"2023-10-01T09:45:18.810725+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":78,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:19.078111+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:45:19.078111+0000","last_peered":"2023-10-01T09:45:19.078111+0000","last_clean":"2023-10-01T09:45:19.078111+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:45:19.078111+0000","last_undegraded":"2023-10-01T09:45:19.078111+0000","last_fullsized":"2023-10-01T09:45:19.078111+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":495,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.810651+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:45:18.810651+0000","last_peered":"2023-10-01T09:45:18.810651+0000","last_clean":"2023-10-01T09:45:18.810651+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:45:18.810651+0000","last_undegraded":"2023-10-01T09:45:18.810651+0000","last_fullsized":"2023-10-01T09:45:18.810651+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":694,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.818476+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:45:18.818476+0000","last_peered":"2023-10-01T09:45:18.818476+0000","last_clean":"2023-10-01T09:45:18.818476+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:45:18.818476+0000","last_undegraded":"2023-10-01T09:45:18.818476+0000","last_fullsized":"2023-10-01T09:45:18.818476+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":388,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-10-01T09:45:18.805861+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:45:18.805861+0000","last_peered":"2023-10-01T09:45:18.805861+0000","last_clean":"2023-10-01T09:45:18.805861+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:45:18.805861+0000","last_undegraded":"2023-10-01T09:45:18.805861+0000","last_fullsized":"2023-10-01T09:45:18.805861+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3485073432,"num_objects":53179,"num_object_clones":0,"num_object_copies":159537,"num_objects_missing_on_primary":1760,"num_objects_missing":1760,"num_objects_degraded":6808,"num_objects_misplaced":3235,"num_objects_unfound":0,"num_objects_dirty":53179,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":113261,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8866,"num_bytes_recovered":581042176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10928926720,"data_stored":10928914504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112361,"ondisk_log_size":112361,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430738,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1976856,"kb_used_data":1069648,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":91768296,"statfs":{"total":95995035648,"available":93970735104,"internally_reserved":0,"allocated":1095319552,"data_stored":1094378284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.79300000000000004,"15min":2.6280000000000001},"min":{"1min":0.254,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":3.3849999999999998,"5min":6.8550000000000004,"15min":108.90000000000001},"last":0.371},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.626,"15min":2.7559999999999998},"min":{"1min":0.25600000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.98399999999999999,"5min":5.6639999999999997,"15min":109.187},"last":0.38400000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":2.843,"15min":5.2329999999999997},"min":{"1min":0.314,"5min":0.248,"15min":0.222},"max":{"1min":3.4089999999999998,"5min":229.161,"15min":229.161},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.49099999999999999,"5min":2.7109999999999999,"15min":5.2290000000000001},"min":{"1min":0.29599999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":1.427,"5min":230.126,"15min":230.126},"last":0.34699999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:44:54 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.81399999999999995,"15min":1.6519999999999999},"min":{"1min":0.30399999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.4300000000000002,"5min":5.3090000000000002,"15min":14.901999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.76100000000000001,"15min":1.6299999999999999},"min":{"1min":0.33200000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4630000000000001,"5min":5.6929999999999996,"15min":14.682},"last":0.46800000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.79200000000000004,"15min":2.2810000000000001},"min":{"1min":0.32600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.3149999999999999,"5min":5.7750000000000004,"15min":76.555000000000007},"last":0.498},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.82499999999999996,"15min":2.2709999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.8479999999999999,"5min":6.548,"15min":76.534000000000006},"last":0.42399999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.80200000000000005,"15min":1.0409999999999999},"min":{"1min":0.31,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":3.4609999999999999,"5min":6.8769999999999998,"15min":9.6020000000000003},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.82599999999999996,"15min":0.999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28000000000000003},"max":{"1min":0.63900000000000001,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.27600000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":2.177,"15min":1.5800000000000001},"min":{"1min":0.34399999999999997,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":0.871,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.315},{"interface":"front","average":{"1min":0.61299999999999999,"5min":2.1789999999999998,"15min":1.6619999999999999},"min":{"1min":0.307,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.3319999999999999,"5min":135.339,"15min":135.339},"last":0.40000000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":1.1639999999999999,"15min":1.1359999999999999},"min":{"1min":0.38,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":0.998,"5min":11.58,"15min":11.853999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.94199999999999995,"15min":1.097},"min":{"1min":0.38300000000000001,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":2.298,"5min":13.021000000000001,"15min":14.584},"last":0.35599999999999998}]}]},{"osd":6,"up_from":294,"seq":1262720385249,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1846484,"kb_used_data":1504780,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":91898668,"statfs":{"total":95995035648,"available":94104236032,"internally_reserved":0,"allocated":1540894720,"data_stored":1539957403,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":1.865,"15min":3.9809999999999999},"min":{"1min":0.27100000000000002,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":0.80200000000000005,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.23400000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":2.0640000000000001,"15min":4.0209999999999999},"min":{"1min":0.311,"5min":0.245,"15min":0.221},"max":{"1min":2.1760000000000002,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.25600000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":2.9470000000000001,"15min":4.8600000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.24199999999999999},"max":{"1min":1.9059999999999999,"5min":168.947,"15min":168.947},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.44400000000000001,"5min":2.9980000000000002,"15min":4.8109999999999999},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":0.59899999999999998,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.34200000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.75600000000000001,"15min":1.5249999999999999},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":1.9350000000000001,"5min":8.657,"15min":10.92},"last":0.308},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.76000000000000001,"15min":1.542},"min":{"1min":0.32300000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.2649999999999999,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.40899999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.70599999999999996,"15min":2.0699999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.1909999999999998,"5min":4.149,"15min":89.753},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.875,"15min":2.0179999999999998},"min":{"1min":0.38500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.1379999999999999,"5min":8.5470000000000006,"15min":89.849999999999994},"last":0.40400000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":1.4510000000000001,"15min":1.0680000000000001},"min":{"1min":0.27200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":0.623,"5min":63.134,"15min":63.134},"last":0.98599999999999999},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.4219999999999999,"15min":1.0669999999999999},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":0.73499999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":1.006}]},{"osd":5,"last update":"Sun Oct 1 09:44:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.246,"15min":1.073},"min":{"1min":0.28799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":3.2890000000000001,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.624,"5min":1.3260000000000001,"15min":1.1859999999999999},"min":{"1min":0.29899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.887,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.38400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.67800000000000005,"15min":1.272},"min":{"1min":0.40200000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.83699999999999997,"5min":3.492,"15min":94.245999999999995},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.87,"15min":1.379},"min":{"1min":0.42299999999999999,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":2.202,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.42699999999999999}]}]},{"osd":1,"up_from":94,"seq":403726926226,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1460508,"kb_used_data":646420,"kb_used_omap":7,"kb_used_meta":814072,"kb_avail":92284644,"statfs":{"total":95995035648,"available":94499475456,"internally_reserved":0,"allocated":661934080,"data_stored":659875830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":833610151},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7469999999999999,"5min":0.93400000000000005,"15min":4.101},"min":{"1min":0.189,"5min":0.186,"15min":0.182},"max":{"1min":32.735999999999997,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.373},{"interface":"front","average":{"1min":2.7959999999999998,"5min":1.147,"15min":3.6240000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":32.591999999999999,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.28799999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:44:39 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.42399999999999999,"15min":2.4889999999999999},"min":{"1min":0.23300000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":1.3240000000000001,"5min":2.8740000000000001,"15min":56.573},"last":0.307},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.61199999999999999,"15min":3.1829999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":2.2200000000000002,"5min":6.9710000000000001,"15min":154.197},"last":0.32400000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.752,"15min":3.7629999999999999},"min":{"1min":0.254,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":0.48899999999999999,"5min":18.503,"15min":200.56800000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.70799999999999996,"15min":4.484},"min":{"1min":0.26300000000000001,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":2.7360000000000002,"5min":16.515000000000001,"15min":200.63800000000001},"last":0.33600000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.73299999999999998,"15min":2.2160000000000002},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":2.9209999999999998,"5min":4.5789999999999997,"15min":154.101},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.58199999999999996,"15min":2.3420000000000001},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.56200000000000006,"5min":2.7290000000000001,"15min":154.56999999999999},"last":0.40600000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:44:39 2023","interfaces":[{"interface":"back","average":{"1min":3.593,"5min":1.272,"15min":2.2759999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":59.561,"5min":59.561,"15min":154.22999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":3.5979999999999999,"5min":1.24,"15min":2.5680000000000001},"min":{"1min":0.35699999999999998,"5min":0.309,"15min":0.307},"max":{"1min":59.536000000000001,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.46500000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.99399999999999999,"15min":2.5409999999999999},"min":{"1min":0.33500000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.86299999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.96999999999999997,"15min":2.419},"min":{"1min":0.38200000000000001,"5min":0.34599999999999997,"15min":0.32700000000000001},"max":{"1min":2.194,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.45400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.94999999999999996,"15min":2.5529999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.33900000000000002},"max":{"1min":2.331,"5min":9.2460000000000004,"15min":154.20699999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.82999999999999996,"15min":2.609},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":1.9750000000000001,"5min":5.0110000000000001,"15min":154.51599999999999},"last":0.436}]}]},{"osd":0,"up_from":89,"seq":382252089760,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3722688,"kb_used_data":2801208,"kb_used_omap":8,"kb_used_meta":921463,"kb_avail":90022464,"statfs":{"total":95995035648,"available":92183003136,"internally_reserved":0,"allocated":2868436992,"data_stored":2866411119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":943579074},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.58499999999999996,"15min":4.0869999999999997},"min":{"1min":0.18099999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":4.2919999999999998,"5min":8.4939999999999998,"15min":216.821},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.59099999999999997,"15min":4.8499999999999996},"min":{"1min":0.20999999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":3.9620000000000002,"5min":8.6229999999999993,"15min":217.02699999999999},"last":0.218}]},{"osd":2,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.69999999999999996,"15min":2.0339999999999998},"min":{"1min":0.29299999999999998,"5min":0.246,"15min":0.20100000000000001},"max":{"1min":4.2430000000000003,"5min":4.9459999999999997,"15min":19.788},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.70799999999999996,"15min":2.1200000000000001},"min":{"1min":0.30399999999999999,"5min":0.23000000000000001,"15min":0.189},"max":{"1min":3.4940000000000002,"5min":4.5359999999999996,"15min":25.844999999999999},"last":0.34499999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.63100000000000001,"15min":3.3969999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.73799999999999999,"15min":3.27},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.9199999999999999,"5min":5.8419999999999996,"15min":222.227},"last":0.22900000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.341,"15min":2.028},"min":{"1min":0.34000000000000002,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":4.048,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.64700000000000002,"5min":1.3140000000000001,"15min":2.0880000000000001},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":4,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.35899999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.76300000000000001,"15min":1.8069999999999999},"min":{"1min":0.33200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.476,"5min":5.8769999999999998,"15min":16.808},"last":3.988},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.90300000000000002,"15min":1.708},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":4.2729999999999997,"5min":5.0629999999999997,"15min":17.507000000000001},"last":0.27700000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:44:27 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.95599999999999996,"15min":1.732},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.81399999999999995,"5min":6.5,"15min":17.535},"last":0.437},{"interface":"front","average":{"1min":0.504,"5min":1.0920000000000001,"15min":1.8300000000000001},"min":{"1min":0.38700000000000001,"5min":0.32400000000000001,"15min":0.32100000000000001},"max":{"1min":0.86199999999999999,"5min":10.446,"15min":15.651999999999999},"last":0.45100000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.86199999999999999,"15min":2.0579999999999998},"min":{"1min":0.375,"5min":0.374,"15min":0.374},"max":{"1min":4.3360000000000003,"5min":4.976,"15min":48.895000000000003},"last":0.374},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.88900000000000001,"15min":2.1179999999999999},"min":{"1min":0.439,"5min":0.371,"15min":0.35099999999999998},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":48.274000000000001},"last":4.125}]}]},{"osd":2,"up_from":0,"seq":425201762700,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599173,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1488468,"kb_used_data":656332,"kb_used_omap":3,"kb_used_meta":832124,"kb_avail":92256684,"statfs":{"total":95995035648,"available":94470844416,"internally_reserved":0,"allocated":672083968,"data_stored":669975239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":852095719},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":1.6839999999999999,"15min":4.843},"min":{"1min":0.20399999999999999,"5min":0.183,"15min":0.17899999999999999},"max":{"1min":0.877,"5min":64.504000000000005,"15min":137.34399999999999},"last":0.307},{"interface":"front","average":{"1min":0.34599999999999997,"5min":1.6000000000000001,"15min":4.9329999999999998},"min":{"1min":0.20599999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.80200000000000005,"5min":64.448999999999998,"15min":137.73400000000001},"last":1.2849999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.67900000000000005,"15min":5.0720000000000001},"min":{"1min":0.252,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":0.60499999999999998,"5min":14.917999999999999,"15min":162.22999999999999},"last":1.3500000000000001},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.60699999999999998,"15min":5.125},"min":{"1min":0.26400000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.57399999999999995,"5min":15.074,"15min":162.70699999999999},"last":0.34300000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.60099999999999998,"15min":1.6859999999999999},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.20799999999999999},"max":{"1min":0.99199999999999999,"5min":7.141,"15min":14.254},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.432,"5min":0.55900000000000005,"15min":1.77},"min":{"1min":0.27000000000000002,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":1.111,"5min":4.5049999999999999,"15min":15.523},"last":0.56999999999999995}]},{"osd":4,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":1.371,"15min":1.7769999999999999},"min":{"1min":0.33200000000000002,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":0.64200000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.4570000000000001,"15min":2.0409999999999999},"min":{"1min":0.35799999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.5609999999999999,"5min":76.646000000000001,"15min":76.646000000000001},"last":1.409}]},{"osd":5,"last update":"Sun Oct 1 09:44:32 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":1.0780000000000001,"15min":2.3530000000000002},"min":{"1min":0.32000000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.196,"5min":9.4320000000000004,"15min":28.117999999999999},"last":1.3819999999999999},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.073,"15min":2.4609999999999999},"min":{"1min":0.33600000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.92300000000000004,"5min":10.798999999999999,"15min":28.34},"last":0.45700000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:44:28 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":1.7350000000000001,"15min":1.8999999999999999},"min":{"1min":0.37,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":1.1000000000000001,"5min":73.887,"15min":73.887},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.952,"15min":1.974},"min":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":1.294,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.61799999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.96299999999999997,"15min":1.8360000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":3.0299999999999998,"5min":7.0620000000000003,"15min":13.085000000000001},"last":3.6480000000000001},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.86899999999999999,"15min":1.716},"min":{"1min":0.40000000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.0150000000000001,"5min":4.5810000000000004,"15min":13.191000000000001},"last":3.7120000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019423,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3567188,"kb_used_data":2681100,"kb_used_omap":0,"kb_used_meta":886080,"kb_avail":90177964,"statfs":{"total":95995035648,"available":92342235136,"internally_reserved":0,"allocated":2745446400,"data_stored":2744508638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":907345920},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.4950000000000001,"15min":4.1369999999999996},"min":{"1min":0.22500000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.9889999999999999,"5min":79.837000000000003,"15min":155.315},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.39500000000000002,"5min":1.6519999999999999,"15min":4.5110000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.77600000000000002,"5min":78.983000000000004,"15min":155.334},"last":0.40500000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":1.5489999999999999,"15min":6.298},"min":{"1min":0.33000000000000002,"5min":0.252,"15min":0.224},"max":{"1min":0.67700000000000005,"5min":77.953000000000003,"15min":190.61199999999999},"last":0.315},{"interface":"front","average":{"1min":0.754,"5min":1.468,"15min":7.0140000000000002},"min":{"1min":0.35299999999999998,"5min":0.26700000000000002,"15min":0.25600000000000001},"max":{"1min":3.1339999999999999,"5min":77.962999999999994,"15min":253.273},"last":0.38600000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:44:50 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.77200000000000002,"15min":2.375},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.2329999999999997,"5min":5.702,"15min":170.45400000000001},"last":0.442},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.755,"15min":2.133},"min":{"1min":0.33100000000000002,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":0.71099999999999997,"5min":5.6150000000000002,"15min":58.771000000000001},"last":0.59299999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.77200000000000002,"15min":2.1379999999999999},"min":{"1min":0.31,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.0299999999999998,"5min":5.5279999999999996,"15min":140.881},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.749,"5min":0.68300000000000005,"15min":2.327},"min":{"1min":0.312,"5min":0.312,"15min":0.311},"max":{"1min":3.161,"5min":3.786,"15min":140.911},"last":0.46600000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":1.929,"15min":1.52},"min":{"1min":0.33800000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":1.7509999999999999,"5min":110.003,"15min":110.003},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.84499999999999997,"5min":1.9239999999999999,"15min":1.405},"min":{"1min":0.38400000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.2909999999999999,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.433}]},{"osd":6,"last update":"Sun Oct 1 09:45:14 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.76000000000000001,"15min":0.94199999999999995},"min":{"1min":0.32800000000000001,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":3.0539999999999998,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.498},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.79000000000000004,"15min":1.0569999999999999},"min":{"1min":0.35899999999999999,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":1.028,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.622}]},{"osd":7,"last update":"Sun Oct 1 09:44:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.65000000000000002,"15min":1.1399999999999999},"min":{"1min":0.36399999999999999,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":3.0830000000000002,"5min":3.8580000000000001,"15min":38.034999999999997},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.91600000000000004,"15min":1.4350000000000001},"min":{"1min":0.39900000000000002,"5min":0.36699999999999999,"15min":0.308},"max":{"1min":0.80000000000000004,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.65400000000000003}]}]},{"osd":5,"up_from":379,"seq":1627792605330,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1752448,"kb_used_data":1379704,"kb_used_omap":13,"kb_used_meta":372722,"kb_avail":91992704,"statfs":{"total":95995035648,"available":94200528896,"internally_reserved":0,"allocated":1412816896,"data_stored":1411895038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":381667826},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":1.244,"15min":2.2229999999999999},"min":{"1min":0.24299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":27.584,"5min":27.584,"15min":84.340999999999994},"last":2.1400000000000001},{"interface":"front","average":{"1min":1.9430000000000001,"5min":1.1200000000000001,"15min":2.1280000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23300000000000001},"max":{"1min":27.669,"5min":27.669,"15min":84.356999999999999},"last":4.2300000000000004}]},{"osd":1,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":3.4430000000000001,"15min":5.6900000000000004},"min":{"1min":0.28699999999999998,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":3.7799999999999998,"5min":149.94900000000001,"15min":295.71300000000002},"last":4.3289999999999997},{"interface":"front","average":{"1min":0.71199999999999997,"5min":3.7149999999999999,"15min":5.6859999999999999},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":3.9580000000000002,"5min":149.41999999999999,"15min":299.59399999999999},"last":2.169}]},{"osd":2,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.83799999999999997,"15min":1.552},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.7029999999999998,"5min":5.3470000000000004,"15min":14.199999999999999},"last":4.2670000000000003},{"interface":"front","average":{"1min":0.84599999999999997,"5min":1.056,"15min":1.5309999999999999},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":3.944,"5min":6.984,"15min":14.430999999999999},"last":4.3719999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.876,"15min":1.46},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.0209999999999999,"5min":10.022,"15min":14.004},"last":2.1880000000000002},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.91300000000000003,"15min":1.5940000000000001},"min":{"1min":0.34300000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.8610000000000002,"5min":5.4400000000000004,"15min":13.991},"last":4.2910000000000004}]},{"osd":4,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":2.085,"15min":1.417},"min":{"1min":0.28599999999999998,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":4.0839999999999996,"5min":122.971,"15min":122.971},"last":4.1929999999999996},{"interface":"front","average":{"1min":0.86299999999999999,"5min":2.1949999999999998,"15min":1.6699999999999999},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.0810000000000004,"5min":122.89100000000001,"15min":122.89100000000001},"last":4.3869999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":1.0169999999999999,"15min":1.0269999999999999},"min":{"1min":0.32500000000000001,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":4.3300000000000001,"5min":10.500999999999999,"15min":12.545},"last":2.2069999999999999},{"interface":"front","average":{"1min":0.93899999999999995,"5min":0.97199999999999998,"15min":1.264},"min":{"1min":0.314,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":4.0599999999999996,"5min":6.4729999999999999,"15min":14.193},"last":4.3200000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:44:30 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.0529999999999999,"15min":1.232},"min":{"1min":0.378,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.0999999999999996,"5min":5.758,"15min":10.813000000000001},"last":4.3559999999999999},{"interface":"front","average":{"1min":0.996,"5min":1.1339999999999999,"15min":1.0840000000000001},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":4.2000000000000002,"5min":10.039,"15min":12.500999999999999},"last":2.226}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2852392960,"data_stored":2852388888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":650641408,"data_stored":650641408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":665518080,"data_stored":665518080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2739802112,"data_stored":2739798040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1404502016,"data_stored":1404502016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1530597376,"data_stored":1530593304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1085472768,"data_stored":1085472768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:45:33.345 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:45:33.367 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:33.398 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:33.444 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:33.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:33.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:33.466 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:45:33.466 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:45:33.466 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:45:33.466 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:45:33.466 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:45:33.485 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:33.556 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:33.587 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:33.624 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:33 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6769/157542 objects degraded (4.297%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:33.625 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:33 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6769/157542 objects degraded (4.297%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:33.635 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:33.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:33.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:33.741 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:33.769 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:33.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:33 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6769/157542 objects degraded (4.297%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:34.140 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:34.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:34.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:34.163 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:45:34.245 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:34.271 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:34.315 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:34.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:34.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:34.424 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:34.467 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:34.495 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:45:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:34 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1607487075' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:45:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:34 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:45:34.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:34 smithi190 ceph-mon[160435]: pgmap v1584: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 185 op/s; 6769/157365 objects degraded (4.301%); 3176/157365 objects misplaced (2.018%); 742 KiB/s, 11 objects/s recovering 2023-10-01T09:45:34.643 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:34 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1607487075' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:45:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:34 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:45:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:34 smithi138 ceph-mon[192513]: pgmap v1584: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 185 op/s; 6769/157365 objects degraded (4.301%); 3176/157365 objects misplaced (2.018%); 742 KiB/s, 11 objects/s recovering 2023-10-01T09:45:34.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:34 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1607487075' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:45:34.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:34 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:45:34.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:34 smithi138 ceph-mon[185886]: pgmap v1584: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 185 op/s; 6769/157365 objects degraded (4.301%); 3176/157365 objects misplaced (2.018%); 742 KiB/s, 11 objects/s recovering 2023-10-01T09:45:34.934 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:34.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:34.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:35.038 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:35.067 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:35.128 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:35.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:35.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:35.235 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:35.262 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:35.308 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:35.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:35.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:35.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:45:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:45:34] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.33.4" 2023-10-01T09:45:35.412 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:35.442 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:35.493 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:35.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:35.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:35.601 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:35.631 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:35.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:35 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:35 smithi190 ceph-mon[160435]: failed to encode map e538 with expected crc 2023-10-01T09:45:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:35 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:45:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:35 smithi190 ceph-mon[160435]: osdmap e538: 8 total, 8 up, 7 in 2023-10-01T09:45:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:35 smithi190 ceph-mon[160435]: failed to encode map e538 with expected crc 2023-10-01T09:45:35.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:35 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:35 smithi138 ceph-mon[185886]: failed to encode map e538 with expected crc 2023-10-01T09:45:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:35 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:45:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:35 smithi138 ceph-mon[185886]: osdmap e538: 8 total, 8 up, 7 in 2023-10-01T09:45:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:35 smithi138 ceph-mon[185886]: failed to encode map e538 with expected crc 2023-10-01T09:45:35.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:35 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:35.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:35 smithi138 ceph-mon[192513]: failed to encode map e538 with expected crc 2023-10-01T09:45:35.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:35 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:45:35.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:35 smithi138 ceph-mon[192513]: osdmap e538: 8 total, 8 up, 7 in 2023-10-01T09:45:35.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:35 smithi138 ceph-mon[192513]: failed to encode map e538 with expected crc 2023-10-01T09:45:36.178 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:36.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:36.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:36.283 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:36.320 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:36.627 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:45:36.646 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:45:36.665 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:45:36.687 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:45:36.706 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:45:36.726 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:45:36.749 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:45:36.784 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:36.787 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:36 smithi138 ceph-mon[192513]: pgmap v1586: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 182 op/s; 6769/156156 objects degraded (4.335%); 3176/156156 objects misplaced (2.034%); 742 KiB/s, 11 objects/s recovering 2023-10-01T09:45:36.787 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:36 smithi138 ceph-mon[185886]: pgmap v1586: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 182 op/s; 6769/156156 objects degraded (4.335%); 3176/156156 objects misplaced (2.034%); 742 KiB/s, 11 objects/s recovering 2023-10-01T09:45:36.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:36.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:36.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:36 smithi190 ceph-mon[160435]: pgmap v1586: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 182 op/s; 6769/156156 objects degraded (4.335%); 3176/156156 objects misplaced (2.034%); 742 KiB/s, 11 objects/s recovering 2023-10-01T09:45:36.884 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:45:36.884 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:45:36.884 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:45:36.884 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:45:36.884 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:45:36.884 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:45:36.885 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:45:36.885 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:45:36.885 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:45:36.889 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:36.919 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:37.489 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:37 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1743544733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:37.489 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:37 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:37.490 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:37 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:37.490 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:37 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:45:37.490 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:37 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:45:37.589 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:37.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:37.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:37.689 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:37.691 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:37.731 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:37.791 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:37.794 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:37 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:37.795 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:37 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1743544733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:37.795 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:37 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:37.795 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:37 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:37.795 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:37 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:45:37.795 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:37 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:45:37.795 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:37 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:37.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:37.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:37.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:37 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1743544733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:37 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:37 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:37 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:45:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:37 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:45:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:37 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:45:37.897 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:37.931 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:38.185 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:38.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:38.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:38.290 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:38.323 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:38.351 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:45:38.371 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:38.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:38.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:38.458 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:45:38.458 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 6769/156156 objects degraded (4.335%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2159,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":538,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":52052,"data_bytes":3414169547,"bytes_used":15944523776,"bytes_avail":656020725760,"bytes_total":671965249536,"degraded_objects":6769,"degraded_total":156156,"degraded_ratio":0.043347677963062575,"misplaced_objects":3176,"misplaced_total":156156,"misplaced_ratio":0.020338635723251109,"recovering_objects_per_sec":11,"recovering_bytes_per_sec":760009,"recovering_keys_per_sec":0,"num_objects_recovered":116,"num_bytes_recovered":7602176,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":182},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":34,"modified":"2023-10-01T09:43:40.311181+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:45:38.479 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:38.506 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:38.550 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:38.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:38.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:38.656 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:38.682 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:38.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:38 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6769/156156 objects degraded (4.335%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:38 smithi190 ceph-mon[160435]: pgmap v1587: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 6701/154671 objects degraded (4.332%); 3131/154671 objects misplaced (2.024%); 704 KiB/s, 10 objects/s recovering 2023-10-01T09:45:38.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1571969752' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:45:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:38 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6769/156156 objects degraded (4.335%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:38 smithi138 ceph-mon[185886]: pgmap v1587: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 6701/154671 objects degraded (4.332%); 3131/154671 objects misplaced (2.024%); 704 KiB/s, 10 objects/s recovering 2023-10-01T09:45:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1571969752' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:45:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:38 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6769/156156 objects degraded (4.335%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:38 smithi138 ceph-mon[192513]: pgmap v1587: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 6701/154671 objects degraded (4.332%); 3131/154671 objects misplaced (2.024%); 704 KiB/s, 10 objects/s recovering 2023-10-01T09:45:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1571969752' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:45:39.329 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:39.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:39.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:39.432 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:39.458 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:39.502 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:39.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:39.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:39.545 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:45:39.545 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:45:39.608 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:39.645 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:39.716 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:45:39.721 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:39.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:39.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:39.828 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:39.862 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:39.879 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:39.912 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:39.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:39.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:40.001 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:45:40.001 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:45:40.019 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:40.044 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:40.088 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:40.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:40.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:40.194 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:40.226 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:40.339 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:40.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:45:40.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:45:40.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:45:40.981 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:40.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:40.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:41.085 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:41.115 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:41.156 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:41.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:41.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:41.261 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:41.289 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:41.373 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:45:41.391 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:41.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:41.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:41.496 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:41.530 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:41 smithi138 ceph-mon[192513]: pgmap v1588: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 173 op/s; 6701/154404 objects degraded (4.340%); 3131/154404 objects misplaced (2.028%); 704 KiB/s, 10 objects/s recovering 2023-10-01T09:45:41.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:41 smithi138 ceph-mon[185886]: pgmap v1588: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 173 op/s; 6701/154404 objects degraded (4.340%); 3131/154404 objects misplaced (2.028%); 704 KiB/s, 10 objects/s recovering 2023-10-01T09:45:41.605 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:41.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:41.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:41.663 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:41.714 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:41.740 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:41 smithi190 ceph-mon[160435]: pgmap v1588: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 173 op/s; 6701/154404 objects degraded (4.340%); 3131/154404 objects misplaced (2.028%); 704 KiB/s, 10 objects/s recovering 2023-10-01T09:45:41.743 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:41.812 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:45:41.852 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:41.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:41.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:41.958 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:41.985 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:42.036 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:42.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:42.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:42.140 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:42.174 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:42.523 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:42.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:42.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:42.629 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:42.658 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:42.723 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:42.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:42.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:42.830 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:42.876 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:42.920 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:42.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:42.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:43.024 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:43.054 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:43.109 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:43.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:43.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:43.214 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:43.240 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:43.415 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:45:43.502 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:43 smithi138 ceph-mon[185886]: pgmap v1589: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 6625/152244 objects degraded (4.352%); 3089/152244 objects misplaced (2.029%); 755 KiB/s, 11 objects/s recovering 2023-10-01T09:45:43.503 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:43 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2830208218' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:43.503 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:43 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:43.503 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:43 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6625/152244 objects degraded (4.352%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:43.503 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:43 smithi138 ceph-mon[192513]: pgmap v1589: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 6625/152244 objects degraded (4.352%); 3089/152244 objects misplaced (2.029%); 755 KiB/s, 11 objects/s recovering 2023-10-01T09:45:43.503 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:43 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2830208218' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:43.503 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:43 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:43.503 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:43 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6625/152244 objects degraded (4.352%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:43.535 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:43.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:43.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:43.639 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:43.682 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:43 smithi190 ceph-mon[160435]: pgmap v1589: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 176 op/s; 6625/152244 objects degraded (4.352%); 3089/152244 objects misplaced (2.029%); 755 KiB/s, 11 objects/s recovering 2023-10-01T09:45:43.682 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:43 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2830208218' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:43.682 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:43 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:43.682 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:43 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6625/152244 objects degraded (4.352%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:43.685 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:43.754 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:43.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:43.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:43.859 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:43.903 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:43.962 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:45:43.962 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:45:44.112 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:45:44.196 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:45:44.305 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:45:44.305 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:45:44.306 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:45:44.306 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:45:44.315 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:45:44.335 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:45:44.353 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:45:44.371 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:45:44.373 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:45:44.374 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:45:44.374 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:45:44.374 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:45:44.374 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:45:44.374 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:45:44.375 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:45:44.375 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:45:44.732 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:44 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:44.732 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:44 smithi190 ceph-mon[160435]: failed to encode map e539 with expected crc 2023-10-01T09:45:44.733 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:44 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:45:44.733 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:44 smithi190 ceph-mon[160435]: osdmap e539: 8 total, 8 up, 7 in 2023-10-01T09:45:44.733 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:44 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3967102571' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:44.747 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:45:44.781 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:44 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:44.781 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:44 smithi138 ceph-mon[192513]: failed to encode map e539 with expected crc 2023-10-01T09:45:44.782 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:44 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:45:44.782 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:44 smithi138 ceph-mon[192513]: osdmap e539: 8 total, 8 up, 7 in 2023-10-01T09:45:44.782 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:44 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3967102571' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:44.782 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:44 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:44.782 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:44 smithi138 ceph-mon[185886]: failed to encode map e539 with expected crc 2023-10-01T09:45:44.782 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:44 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:45:44.782 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:44 smithi138 ceph-mon[185886]: osdmap e539: 8 total, 8 up, 7 in 2023-10-01T09:45:44.782 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:44 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3967102571' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:45.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:45:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:45:44] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.33.4" 2023-10-01T09:45:45.201 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:45:45.472 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:45.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:45.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:45.540 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:45 smithi138 ceph-mon[185886]: from='client.37152 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:45.540 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:45 smithi138 ceph-mon[185886]: pgmap v1591: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 179 op/s; 6625/152070 objects degraded (4.357%); 3089/152070 objects misplaced (2.031%); 766 KiB/s, 11 objects/s recovering 2023-10-01T09:45:45.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:45 smithi138 ceph-mon[192513]: from='client.37152 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:45.541 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:45 smithi138 ceph-mon[192513]: pgmap v1591: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 179 op/s; 6625/152070 objects degraded (4.357%); 3089/152070 objects misplaced (2.031%); 766 KiB/s, 11 objects/s recovering 2023-10-01T09:45:45.576 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:45.607 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:45.739 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:45.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:45.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:45.745 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1590,"stamp":"2023-10-01T09:45:43.408409+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3324909515,"num_objects":50690,"num_object_clones":0,"num_object_copies":152070,"num_objects_missing_on_primary":1593,"num_objects_missing":1593,"num_objects_degraded":6625,"num_objects_misplaced":3089,"num_objects_unfound":0,"num_objects_dirty":50690,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":116884,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9549,"num_bytes_recovered":612664335,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115048,"ondisk_log_size":115048,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15340780,"kb_used_data":10279988,"kb_used_omap":39,"kb_used_meta":5060632,"kb_avail":640875284,"statfs":{"total":671965249536,"available":656256290816,"internally_reserved":0,"allocated":10526707712,"data_stored":10516770564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5182087791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-115671040,"num_objects":-1765,"num_object_clones":0,"num_object_copies":-5295,"num_objects_missing_on_primary":-113,"num_objects_missing":-113,"num_objects_degraded":-144,"num_objects_misplaced":-87,"num_objects_unfound":0,"num_objects_dirty":-1765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1765,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":7733248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.062497"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":314,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.507943+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:45:25.507943+0000","last_peered":"2023-10-01T09:45:25.507943+0000","last_clean":"2023-10-01T09:45:25.507943+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:45:25.507943+0000","last_undegraded":"2023-10-01T09:45:25.507943+0000","last_fullsized":"2023-10-01T09:45:25.507943+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":158,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.504580+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:45:25.504580+0000","last_peered":"2023-10-01T09:45:25.504580+0000","last_clean":"2023-10-01T09:45:25.504580+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:45:25.504580+0000","last_undegraded":"2023-10-01T09:45:25.504580+0000","last_fullsized":"2023-10-01T09:45:25.504580+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":538,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.504487+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:45:25.504487+0000","last_peered":"2023-10-01T09:45:25.504487+0000","last_clean":"2023-10-01T09:45:25.504487+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:45:25.504487+0000","last_undegraded":"2023-10-01T09:45:25.504487+0000","last_fullsized":"2023-10-01T09:45:25.504487+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":597,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.501183+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:45:25.501183+0000","last_peered":"2023-10-01T09:45:25.501183+0000","last_clean":"2023-10-01T09:45:25.501183+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:45:25.501183+0000","last_undegraded":"2023-10-01T09:45:25.501183+0000","last_fullsized":"2023-10-01T09:45:25.501183+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":478,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499912+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:45:25.499912+0000","last_peered":"2023-10-01T09:45:25.499912+0000","last_clean":"2023-10-01T09:45:25.499912+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:45:25.499912+0000","last_undegraded":"2023-10-01T09:45:25.499912+0000","last_fullsized":"2023-10-01T09:45:25.499912+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":219,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.603197+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:45:25.603197+0000","last_peered":"2023-10-01T09:45:25.603197+0000","last_clean":"2023-10-01T09:45:25.603197+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:45:25.603197+0000","last_undegraded":"2023-10-01T09:45:25.603197+0000","last_fullsized":"2023-10-01T09:45:25.603197+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":595,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.603259+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:45:25.603259+0000","last_peered":"2023-10-01T09:45:25.603259+0000","last_clean":"2023-10-01T09:45:25.603259+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:45:25.603259+0000","last_undegraded":"2023-10-01T09:45:25.603259+0000","last_fullsized":"2023-10-01T09:45:25.603259+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":525,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.496369+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:45:25.496369+0000","last_peered":"2023-10-01T09:45:25.496369+0000","last_clean":"2023-10-01T09:45:25.496369+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:45:25.496369+0000","last_undegraded":"2023-10-01T09:45:25.496369+0000","last_fullsized":"2023-10-01T09:45:25.496369+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":250,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.503314+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:45:25.503314+0000","last_peered":"2023-10-01T09:45:25.503314+0000","last_clean":"2023-10-01T09:45:25.503314+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:45:25.503314+0000","last_undegraded":"2023-10-01T09:45:25.503314+0000","last_fullsized":"2023-10-01T09:45:25.503314+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":57,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499648+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:45:25.499648+0000","last_peered":"2023-10-01T09:45:25.499648+0000","last_clean":"2023-10-01T09:45:25.499648+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:45:25.499648+0000","last_undegraded":"2023-10-01T09:45:25.499648+0000","last_fullsized":"2023-10-01T09:45:25.499648+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":63,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499782+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:45:25.499782+0000","last_peered":"2023-10-01T09:45:25.499782+0000","last_clean":"2023-10-01T09:45:25.499782+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:45:25.499782+0000","last_undegraded":"2023-10-01T09:45:25.499782+0000","last_fullsized":"2023-10-01T09:45:25.499782+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":62,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.504625+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:45:25.504625+0000","last_peered":"2023-10-01T09:45:25.504625+0000","last_clean":"2023-10-01T09:45:25.504625+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:45:25.504625+0000","last_undegraded":"2023-10-01T09:45:25.504625+0000","last_fullsized":"2023-10-01T09:45:25.504625+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":439,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.508960+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:45:25.508960+0000","last_peered":"2023-10-01T09:45:25.508960+0000","last_clean":"2023-10-01T09:45:25.508960+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:45:25.508960+0000","last_undegraded":"2023-10-01T09:45:25.508960+0000","last_fullsized":"2023-10-01T09:45:25.508960+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":320,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.508381+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:45:25.508381+0000","last_peered":"2023-10-01T09:45:25.508381+0000","last_clean":"2023-10-01T09:45:25.508381+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:45:25.508381+0000","last_undegraded":"2023-10-01T09:45:25.508381+0000","last_fullsized":"2023-10-01T09:45:25.508381+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":656,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499456+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:45:25.499456+0000","last_peered":"2023-10-01T09:45:25.499456+0000","last_clean":"2023-10-01T09:45:25.499456+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:45:25.499456+0000","last_undegraded":"2023-10-01T09:45:25.499456+0000","last_fullsized":"2023-10-01T09:45:25.499456+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":393,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499771+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:45:25.499771+0000","last_peered":"2023-10-01T09:45:25.499771+0000","last_clean":"2023-10-01T09:45:25.499771+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:45:25.499771+0000","last_undegraded":"2023-10-01T09:45:25.499771+0000","last_fullsized":"2023-10-01T09:45:25.499771+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":66,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499292+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:45:25.499292+0000","last_peered":"2023-10-01T09:45:25.499292+0000","last_clean":"2023-10-01T09:45:25.499292+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:45:25.499292+0000","last_undegraded":"2023-10-01T09:45:25.499292+0000","last_fullsized":"2023-10-01T09:45:25.499292+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":52,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.503063+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:45:25.503063+0000","last_peered":"2023-10-01T09:45:25.503063+0000","last_clean":"2023-10-01T09:45:25.503063+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:45:25.503063+0000","last_undegraded":"2023-10-01T09:45:25.503063+0000","last_fullsized":"2023-10-01T09:45:25.503063+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":549,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.501329+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:45:25.501329+0000","last_peered":"2023-10-01T09:45:25.501329+0000","last_clean":"2023-10-01T09:45:25.501329+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:45:25.501329+0000","last_undegraded":"2023-10-01T09:45:25.501329+0000","last_fullsized":"2023-10-01T09:45:25.501329+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":561,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.501237+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:45:25.501237+0000","last_peered":"2023-10-01T09:45:25.501237+0000","last_clean":"2023-10-01T09:45:25.501237+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:45:25.501237+0000","last_undegraded":"2023-10-01T09:45:25.501237+0000","last_fullsized":"2023-10-01T09:45:25.501237+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":417,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499721+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:45:25.499721+0000","last_peered":"2023-10-01T09:45:25.499721+0000","last_clean":"2023-10-01T09:45:25.499721+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:45:25.499721+0000","last_undegraded":"2023-10-01T09:45:25.499721+0000","last_fullsized":"2023-10-01T09:45:25.499721+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":66,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499446+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:45:25.499446+0000","last_peered":"2023-10-01T09:45:25.499446+0000","last_clean":"2023-10-01T09:45:25.499446+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:45:25.499446+0000","last_undegraded":"2023-10-01T09:45:25.499446+0000","last_fullsized":"2023-10-01T09:45:25.499446+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"538'7268","reported_seq":7339,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:41.659285+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:45:41.659285+0000","last_peered":"2023-10-01T09:45:41.659285+0000","last_clean":"2023-10-01T09:45:41.659285+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:45:41.659285+0000","last_undegraded":"2023-10-01T09:45:41.659285+0000","last_fullsized":"2023-10-01T09:45:41.659285+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7268,"log_dups_size":0,"ondisk_log_size":7268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204734464,"num_objects":3124,"num_object_clones":0,"num_object_copies":9372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7268,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":61,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499269+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:45:25.499269+0000","last_peered":"2023-10-01T09:45:25.499269+0000","last_clean":"2023-10-01T09:45:25.499269+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:45:25.499269+0000","last_undegraded":"2023-10-01T09:45:25.499269+0000","last_fullsized":"2023-10-01T09:45:25.499269+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":258,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.503439+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:45:25.503439+0000","last_peered":"2023-10-01T09:45:25.503439+0000","last_clean":"2023-10-01T09:45:25.503439+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:45:25.503439+0000","last_undegraded":"2023-10-01T09:45:25.503439+0000","last_fullsized":"2023-10-01T09:45:25.503439+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"538'7295","reported_seq":13673,"reported_epoch":538,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:45:42.148587+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:45:42.148587+0000","last_peered":"2023-10-01T09:45:42.148587+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:45:42.148587+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7295,"log_dups_size":0,"ondisk_log_size":7295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203882496,"num_objects":3111,"num_object_clones":0,"num_object_copies":9333,"num_objects_missing_on_primary":1593,"num_objects_missing":1593,"num_objects_degraded":4635,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7295,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2093,"num_bytes_recovered":137166848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1449},{"shards":"0,4,6","objects":1593},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":532,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499430+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:45:25.499430+0000","last_peered":"2023-10-01T09:45:25.499430+0000","last_clean":"2023-10-01T09:45:25.499430+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:45:25.499430+0000","last_undegraded":"2023-10-01T09:45:25.499430+0000","last_fullsized":"2023-10-01T09:45:25.499430+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":594,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.504456+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:45:25.504456+0000","last_peered":"2023-10-01T09:45:25.504456+0000","last_clean":"2023-10-01T09:45:25.504456+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:45:25.504456+0000","last_undegraded":"2023-10-01T09:45:25.504456+0000","last_fullsized":"2023-10-01T09:45:25.504456+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"538'7256","reported_seq":7330,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:41.803170+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:45:41.803170+0000","last_peered":"2023-10-01T09:45:41.803170+0000","last_clean":"2023-10-01T09:45:41.803170+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:45:41.803170+0000","last_undegraded":"2023-10-01T09:45:41.803170+0000","last_fullsized":"2023-10-01T09:45:41.803170+0000","mapping_epoch":475,"log_start":"480'200","ondisk_log_start":"480'200","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7056,"log_dups_size":0,"ondisk_log_size":7056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208142336,"num_objects":3176,"num_object_clones":0,"num_object_copies":9528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7256,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":618,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499843+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:45:25.499843+0000","last_peered":"2023-10-01T09:45:25.499843+0000","last_clean":"2023-10-01T09:45:25.499843+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:45:25.499843+0000","last_undegraded":"2023-10-01T09:45:25.499843+0000","last_fullsized":"2023-10-01T09:45:25.499843+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":414,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.504828+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:45:25.504828+0000","last_peered":"2023-10-01T09:45:25.504828+0000","last_clean":"2023-10-01T09:45:25.504828+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:45:25.504828+0000","last_undegraded":"2023-10-01T09:45:25.504828+0000","last_fullsized":"2023-10-01T09:45:25.504828+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"538'7273","reported_seq":13952,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:41.595353+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:45:41.595353+0000","last_peered":"2023-10-01T09:45:41.595353+0000","last_clean":"2023-10-01T09:45:41.595353+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:45:41.595353+0000","last_undegraded":"2023-10-01T09:45:41.595353+0000","last_fullsized":"2023-10-01T09:45:41.595353+0000","mapping_epoch":505,"log_start":"480'200","ondisk_log_start":"480'200","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7073,"log_dups_size":0,"ondisk_log_size":7073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205062144,"num_objects":3129,"num_object_clones":0,"num_object_copies":9387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7273,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":258,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.503015+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:45:25.503015+0000","last_peered":"2023-10-01T09:45:25.503015+0000","last_clean":"2023-10-01T09:45:25.503015+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:45:25.503015+0000","last_undegraded":"2023-10-01T09:45:25.503015+0000","last_fullsized":"2023-10-01T09:45:25.503015+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":517,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.500213+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:45:25.500213+0000","last_peered":"2023-10-01T09:45:25.500213+0000","last_clean":"2023-10-01T09:45:25.500213+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:45:25.500213+0000","last_undegraded":"2023-10-01T09:45:25.500213+0000","last_fullsized":"2023-10-01T09:45:25.500213+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"538'7264","reported_seq":7338,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:40.190388+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:45:40.190388+0000","last_peered":"2023-10-01T09:45:40.190388+0000","last_clean":"2023-10-01T09:45:40.190388+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:45:40.190388+0000","last_undegraded":"2023-10-01T09:45:40.190388+0000","last_fullsized":"2023-10-01T09:45:40.190388+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7264,"log_dups_size":0,"ondisk_log_size":7264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212598784,"num_objects":3244,"num_object_clones":0,"num_object_copies":9732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7264,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":118,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.508568+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:45:25.508568+0000","last_peered":"2023-10-01T09:45:25.508568+0000","last_clean":"2023-10-01T09:45:25.508568+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:45:25.508568+0000","last_undegraded":"2023-10-01T09:45:25.508568+0000","last_fullsized":"2023-10-01T09:45:25.508568+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":192,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.509045+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:45:25.509045+0000","last_peered":"2023-10-01T09:45:25.509045+0000","last_clean":"2023-10-01T09:45:25.509045+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:45:25.509045+0000","last_undegraded":"2023-10-01T09:45:25.509045+0000","last_fullsized":"2023-10-01T09:45:25.509045+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"538'7215","reported_seq":7398,"reported_epoch":538,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:45:41.813875+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:45:41.813875+0000","last_peered":"2023-10-01T09:45:41.813875+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:45:41.813875+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7215,"log_dups_size":0,"ondisk_log_size":7215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206635008,"num_objects":3153,"num_object_clones":0,"num_object_copies":9459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1980,"num_objects_misplaced":3089,"num_objects_unfound":0,"num_objects_dirty":3153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7215,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":3407872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":3089},{"shards":"4,5","objects":-1916}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":436,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.508079+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:45:25.508079+0000","last_peered":"2023-10-01T09:45:25.508079+0000","last_clean":"2023-10-01T09:45:25.508079+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:45:25.508079+0000","last_undegraded":"2023-10-01T09:45:25.508079+0000","last_fullsized":"2023-10-01T09:45:25.508079+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":597,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.505515+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:45:25.505515+0000","last_peered":"2023-10-01T09:45:25.505515+0000","last_clean":"2023-10-01T09:45:25.505515+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:45:25.505515+0000","last_undegraded":"2023-10-01T09:45:25.505515+0000","last_fullsized":"2023-10-01T09:45:25.505515+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"538'7208","reported_seq":7288,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:42.172162+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:45:42.172162+0000","last_peered":"2023-10-01T09:45:42.172162+0000","last_clean":"2023-10-01T09:45:42.172162+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:45:42.172162+0000","last_undegraded":"2023-10-01T09:45:42.172162+0000","last_fullsized":"2023-10-01T09:45:42.172162+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7208,"log_dups_size":0,"ondisk_log_size":7208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207749120,"num_objects":3170,"num_object_clones":0,"num_object_copies":9510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7208,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":432,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.507859+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:45:25.507859+0000","last_peered":"2023-10-01T09:45:25.507859+0000","last_clean":"2023-10-01T09:45:25.507859+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:45:25.507859+0000","last_undegraded":"2023-10-01T09:45:25.507859+0000","last_fullsized":"2023-10-01T09:45:25.507859+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":273,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.509405+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:45:25.509405+0000","last_peered":"2023-10-01T09:45:25.509405+0000","last_clean":"2023-10-01T09:45:25.509405+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:45:25.509405+0000","last_undegraded":"2023-10-01T09:45:25.509405+0000","last_fullsized":"2023-10-01T09:45:25.509405+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"538'7326","reported_seq":7397,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:41.201018+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:45:41.201018+0000","last_peered":"2023-10-01T09:45:41.201018+0000","last_clean":"2023-10-01T09:45:41.201018+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:45:41.201018+0000","last_undegraded":"2023-10-01T09:45:41.201018+0000","last_fullsized":"2023-10-01T09:45:41.201018+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7326,"log_dups_size":0,"ondisk_log_size":7326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207880192,"num_objects":3172,"num_object_clones":0,"num_object_copies":9516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7326,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":401,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.504549+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:45:25.504549+0000","last_peered":"2023-10-01T09:45:25.504549+0000","last_clean":"2023-10-01T09:45:25.504549+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:45:25.504549+0000","last_undegraded":"2023-10-01T09:45:25.504549+0000","last_fullsized":"2023-10-01T09:45:25.504549+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":449,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:34.506734+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:45:34.506734+0000","last_peered":"2023-10-01T09:45:34.506734+0000","last_clean":"2023-10-01T09:45:34.506734+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:45:34.506734+0000","last_undegraded":"2023-10-01T09:45:34.506734+0000","last_fullsized":"2023-10-01T09:45:34.506734+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"538'7091","reported_seq":7162,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:41.028475+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:45:41.028475+0000","last_peered":"2023-10-01T09:45:41.028475+0000","last_clean":"2023-10-01T09:45:41.028475+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:45:41.028475+0000","last_undegraded":"2023-10-01T09:45:41.028475+0000","last_fullsized":"2023-10-01T09:45:41.028475+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7091,"log_dups_size":0,"ondisk_log_size":7091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202309632,"num_objects":3087,"num_object_clones":0,"num_object_copies":9261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7091,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":931,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499624+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:45:25.499624+0000","last_peered":"2023-10-01T09:45:25.499624+0000","last_clean":"2023-10-01T09:45:25.499624+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:45:25.499624+0000","last_undegraded":"2023-10-01T09:45:25.499624+0000","last_fullsized":"2023-10-01T09:45:25.499624+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":268,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.503281+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:45:25.503281+0000","last_peered":"2023-10-01T09:45:25.503281+0000","last_clean":"2023-10-01T09:45:25.503281+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:45:25.503281+0000","last_undegraded":"2023-10-01T09:45:25.503281+0000","last_fullsized":"2023-10-01T09:45:25.503281+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":62,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.504700+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:45:25.504700+0000","last_peered":"2023-10-01T09:45:25.504700+0000","last_clean":"2023-10-01T09:45:25.504700+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:45:25.504700+0000","last_undegraded":"2023-10-01T09:45:25.504700+0000","last_fullsized":"2023-10-01T09:45:25.504700+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"538'7177","reported_seq":7251,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:41.823036+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:45:41.823036+0000","last_peered":"2023-10-01T09:45:41.823036+0000","last_clean":"2023-10-01T09:45:41.823036+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:45:41.823036+0000","last_undegraded":"2023-10-01T09:45:41.823036+0000","last_fullsized":"2023-10-01T09:45:41.823036+0000","mapping_epoch":475,"log_start":"480'200","ondisk_log_start":"480'200","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6977,"log_dups_size":0,"ondisk_log_size":6977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209256448,"num_objects":3193,"num_object_clones":0,"num_object_copies":9579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7177,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1203,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.502966+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:45:25.502966+0000","last_peered":"2023-10-01T09:45:25.502966+0000","last_clean":"2023-10-01T09:45:25.502966+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:45:25.502966+0000","last_undegraded":"2023-10-01T09:45:25.502966+0000","last_fullsized":"2023-10-01T09:45:25.502966+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":413,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.508491+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:45:25.508491+0000","last_peered":"2023-10-01T09:45:25.508491+0000","last_clean":"2023-10-01T09:45:25.508491+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:45:25.508491+0000","last_undegraded":"2023-10-01T09:45:25.508491+0000","last_fullsized":"2023-10-01T09:45:25.508491+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":241,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.503367+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:45:25.503367+0000","last_peered":"2023-10-01T09:45:25.503367+0000","last_clean":"2023-10-01T09:45:25.503367+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:45:25.503367+0000","last_undegraded":"2023-10-01T09:45:25.503367+0000","last_fullsized":"2023-10-01T09:45:25.503367+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"538'7249","reported_seq":14541,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:42.118445+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:45:42.118445+0000","last_peered":"2023-10-01T09:45:42.118445+0000","last_clean":"2023-10-01T09:45:42.118445+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:45:42.118445+0000","last_undegraded":"2023-10-01T09:45:42.118445+0000","last_fullsized":"2023-10-01T09:45:42.118445+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7249,"log_dups_size":0,"ondisk_log_size":7249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204931072,"num_objects":3127,"num_object_clones":0,"num_object_copies":9381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7249,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":884,"reported_epoch":537,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:45:25.499695+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:45:25.499695+0000","last_peered":"2023-10-01T09:45:25.499695+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:45:25.499695+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:45:25.499695+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":601,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.504630+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:45:25.504630+0000","last_peered":"2023-10-01T09:45:25.504630+0000","last_clean":"2023-10-01T09:45:25.504630+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:45:25.504630+0000","last_undegraded":"2023-10-01T09:45:25.504630+0000","last_fullsized":"2023-10-01T09:45:25.504630+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":62,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.505247+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:45:25.505247+0000","last_peered":"2023-10-01T09:45:25.505247+0000","last_clean":"2023-10-01T09:45:25.505247+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:45:25.505247+0000","last_undegraded":"2023-10-01T09:45:25.505247+0000","last_fullsized":"2023-10-01T09:45:25.505247+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"538'7103","reported_seq":7177,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:40.220462+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:45:40.220462+0000","last_peered":"2023-10-01T09:45:40.220462+0000","last_clean":"2023-10-01T09:45:40.220462+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:45:40.220462+0000","last_undegraded":"2023-10-01T09:45:40.220462+0000","last_fullsized":"2023-10-01T09:45:40.220462+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7103,"log_dups_size":0,"ondisk_log_size":7103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207683584,"num_objects":3169,"num_object_clones":0,"num_object_copies":9507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7103,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1654,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.509653+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:45:25.509653+0000","last_peered":"2023-10-01T09:45:25.509653+0000","last_clean":"2023-10-01T09:45:25.509653+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:45:25.509653+0000","last_undegraded":"2023-10-01T09:45:25.509653+0000","last_fullsized":"2023-10-01T09:45:25.509653+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":568,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499060+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:45:25.499060+0000","last_peered":"2023-10-01T09:45:25.499060+0000","last_clean":"2023-10-01T09:45:25.499060+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:45:25.499060+0000","last_undegraded":"2023-10-01T09:45:25.499060+0000","last_fullsized":"2023-10-01T09:45:25.499060+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":346,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.503046+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:45:25.503046+0000","last_peered":"2023-10-01T09:45:25.503046+0000","last_clean":"2023-10-01T09:45:25.503046+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:45:25.503046+0000","last_undegraded":"2023-10-01T09:45:25.503046+0000","last_fullsized":"2023-10-01T09:45:25.503046+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":336,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.508745+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:45:25.508745+0000","last_peered":"2023-10-01T09:45:25.508745+0000","last_clean":"2023-10-01T09:45:25.508745+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:45:25.508745+0000","last_undegraded":"2023-10-01T09:45:25.508745+0000","last_fullsized":"2023-10-01T09:45:25.508745+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"538'7323","reported_seq":7394,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:41.188982+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:45:41.188982+0000","last_peered":"2023-10-01T09:45:41.188982+0000","last_clean":"2023-10-01T09:45:41.188982+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:45:41.188982+0000","last_undegraded":"2023-10-01T09:45:41.188982+0000","last_fullsized":"2023-10-01T09:45:41.188982+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7323,"log_dups_size":0,"ondisk_log_size":7323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210305024,"num_objects":3209,"num_object_clones":0,"num_object_copies":9627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7323,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":745,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499145+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:45:25.499145+0000","last_peered":"2023-10-01T09:45:25.499145+0000","last_clean":"2023-10-01T09:45:25.499145+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:45:25.499145+0000","last_undegraded":"2023-10-01T09:45:25.499145+0000","last_fullsized":"2023-10-01T09:45:25.499145+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":578,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499837+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:45:25.499837+0000","last_peered":"2023-10-01T09:45:25.499837+0000","last_clean":"2023-10-01T09:45:25.499837+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:45:25.499837+0000","last_undegraded":"2023-10-01T09:45:25.499837+0000","last_fullsized":"2023-10-01T09:45:25.499837+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":578,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:34.500904+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:45:34.500904+0000","last_peered":"2023-10-01T09:45:34.500904+0000","last_clean":"2023-10-01T09:45:34.500904+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:45:34.500904+0000","last_undegraded":"2023-10-01T09:45:34.500904+0000","last_fullsized":"2023-10-01T09:45:34.500904+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"538'7244","reported_seq":7319,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:41.798526+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:45:41.798526+0000","last_peered":"2023-10-01T09:45:41.798526+0000","last_clean":"2023-10-01T09:45:41.798526+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:45:41.798526+0000","last_undegraded":"2023-10-01T09:45:41.798526+0000","last_fullsized":"2023-10-01T09:45:41.798526+0000","mapping_epoch":475,"log_start":"480'200","ondisk_log_start":"480'200","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7044,"log_dups_size":0,"ondisk_log_size":7044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210698264,"num_objects":3216,"num_object_clones":0,"num_object_copies":9648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3216,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7244,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":976,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499911+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:45:25.499911+0000","last_peered":"2023-10-01T09:45:25.499911+0000","last_clean":"2023-10-01T09:45:25.499911+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:45:25.499911+0000","last_undegraded":"2023-10-01T09:45:25.499911+0000","last_fullsized":"2023-10-01T09:45:25.499911+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":595,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.503092+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:45:25.503092+0000","last_peered":"2023-10-01T09:45:25.503092+0000","last_clean":"2023-10-01T09:45:25.503092+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:45:25.503092+0000","last_undegraded":"2023-10-01T09:45:25.503092+0000","last_fullsized":"2023-10-01T09:45:25.503092+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":448,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.509698+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:45:25.509698+0000","last_peered":"2023-10-01T09:45:25.509698+0000","last_clean":"2023-10-01T09:45:25.509698+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:45:25.509698+0000","last_undegraded":"2023-10-01T09:45:25.509698+0000","last_fullsized":"2023-10-01T09:45:25.509698+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"538'7196","reported_seq":7275,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:40.817577+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:45:40.817577+0000","last_peered":"2023-10-01T09:45:40.817577+0000","last_clean":"2023-10-01T09:45:40.817577+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:45:40.817577+0000","last_undegraded":"2023-10-01T09:45:40.817577+0000","last_fullsized":"2023-10-01T09:45:40.817577+0000","mapping_epoch":479,"log_start":"481'500","ondisk_log_start":"481'500","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6696,"log_dups_size":0,"ondisk_log_size":6696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207093760,"num_objects":3160,"num_object_clones":0,"num_object_copies":9480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7196,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":477,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499531+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:45:25.499531+0000","last_peered":"2023-10-01T09:45:25.499531+0000","last_clean":"2023-10-01T09:45:25.499531+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:45:25.499531+0000","last_undegraded":"2023-10-01T09:45:25.499531+0000","last_fullsized":"2023-10-01T09:45:25.499531+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1171,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.505024+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:45:25.505024+0000","last_peered":"2023-10-01T09:45:25.505024+0000","last_clean":"2023-10-01T09:45:25.505024+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:45:25.505024+0000","last_undegraded":"2023-10-01T09:45:25.505024+0000","last_fullsized":"2023-10-01T09:45:25.505024+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499003+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:45:25.499003+0000","last_peered":"2023-10-01T09:45:25.499003+0000","last_clean":"2023-10-01T09:45:25.499003+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:45:25.499003+0000","last_undegraded":"2023-10-01T09:45:25.499003+0000","last_fullsized":"2023-10-01T09:45:25.499003+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"538'7290","reported_seq":7369,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:41.261267+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:45:41.261267+0000","last_peered":"2023-10-01T09:45:41.261267+0000","last_clean":"2023-10-01T09:45:41.261267+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:45:41.261267+0000","last_undegraded":"2023-10-01T09:45:41.261267+0000","last_fullsized":"2023-10-01T09:45:41.261267+0000","mapping_epoch":479,"log_start":"481'600","ondisk_log_start":"481'600","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6690,"log_dups_size":0,"ondisk_log_size":6690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211156992,"num_objects":3222,"num_object_clones":0,"num_object_copies":9666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7290,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":70,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499371+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:45:25.499371+0000","last_peered":"2023-10-01T09:45:25.499371+0000","last_clean":"2023-10-01T09:45:25.499371+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:45:25.499371+0000","last_undegraded":"2023-10-01T09:45:25.499371+0000","last_fullsized":"2023-10-01T09:45:25.499371+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":556,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:34.500994+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:45:34.500994+0000","last_peered":"2023-10-01T09:45:34.500994+0000","last_clean":"2023-10-01T09:45:34.500994+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:45:34.500994+0000","last_undegraded":"2023-10-01T09:45:34.500994+0000","last_fullsized":"2023-10-01T09:45:34.500994+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":261,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.503033+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:45:25.503033+0000","last_peered":"2023-10-01T09:45:25.503033+0000","last_clean":"2023-10-01T09:45:25.503033+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:45:25.503033+0000","last_undegraded":"2023-10-01T09:45:25.503033+0000","last_fullsized":"2023-10-01T09:45:25.503033+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":504,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.500348+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:45:25.500348+0000","last_peered":"2023-10-01T09:45:25.500348+0000","last_clean":"2023-10-01T09:45:25.500348+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:45:25.500348+0000","last_undegraded":"2023-10-01T09:45:25.500348+0000","last_fullsized":"2023-10-01T09:45:25.500348+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":618,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.504764+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:45:25.504764+0000","last_peered":"2023-10-01T09:45:25.504764+0000","last_clean":"2023-10-01T09:45:25.504764+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:45:25.504764+0000","last_undegraded":"2023-10-01T09:45:25.504764+0000","last_fullsized":"2023-10-01T09:45:25.504764+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":273,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.502925+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:45:25.502925+0000","last_peered":"2023-10-01T09:45:25.502925+0000","last_clean":"2023-10-01T09:45:25.502925+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:45:25.502925+0000","last_undegraded":"2023-10-01T09:45:25.502925+0000","last_fullsized":"2023-10-01T09:45:25.502925+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":61,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.505786+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:45:25.505786+0000","last_peered":"2023-10-01T09:45:25.505786+0000","last_clean":"2023-10-01T09:45:25.505786+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:45:25.505786+0000","last_undegraded":"2023-10-01T09:45:25.505786+0000","last_fullsized":"2023-10-01T09:45:25.505786+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":273,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.503212+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:45:25.503212+0000","last_peered":"2023-10-01T09:45:25.503212+0000","last_clean":"2023-10-01T09:45:25.503212+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:45:25.503212+0000","last_undegraded":"2023-10-01T09:45:25.503212+0000","last_fullsized":"2023-10-01T09:45:25.503212+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":451,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:34.499989+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:45:34.499989+0000","last_peered":"2023-10-01T09:45:34.499989+0000","last_clean":"2023-10-01T09:45:34.499989+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:45:34.499989+0000","last_undegraded":"2023-10-01T09:45:34.499989+0000","last_fullsized":"2023-10-01T09:45:34.499989+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":79,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.504829+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:45:25.504829+0000","last_peered":"2023-10-01T09:45:25.504829+0000","last_clean":"2023-10-01T09:45:25.504829+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:45:25.504829+0000","last_undegraded":"2023-10-01T09:45:25.504829+0000","last_fullsized":"2023-10-01T09:45:25.504829+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":496,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.499738+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:45:25.499738+0000","last_peered":"2023-10-01T09:45:25.499738+0000","last_clean":"2023-10-01T09:45:25.499738+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:45:25.499738+0000","last_undegraded":"2023-10-01T09:45:25.499738+0000","last_fullsized":"2023-10-01T09:45:25.499738+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":695,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.500048+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:45:25.500048+0000","last_peered":"2023-10-01T09:45:25.500048+0000","last_clean":"2023-10-01T09:45:25.500048+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:45:25.500048+0000","last_undegraded":"2023-10-01T09:45:25.500048+0000","last_fullsized":"2023-10-01T09:45:25.500048+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":389,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-10-01T09:45:25.507705+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:45:25.507705+0000","last_peered":"2023-10-01T09:45:25.507705+0000","last_clean":"2023-10-01T09:45:25.507705+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:45:25.507705+0000","last_undegraded":"2023-10-01T09:45:25.507705+0000","last_fullsized":"2023-10-01T09:45:25.507705+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3320119320,"num_objects":50662,"num_object_clones":0,"num_object_copies":151986,"num_objects_missing_on_primary":1593,"num_objects_missing":1593,"num_objects_degraded":6615,"num_objects_misplaced":3089,"num_objects_unfound":0,"num_objects_dirty":50662,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":115778,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9038,"num_bytes_recovered":592314368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10459230208,"data_stored":10459217992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":113878,"ondisk_log_size":113878,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430741,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1911968,"kb_used_data":1017048,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":91833184,"statfs":{"total":95995035648,"available":94037180416,"internally_reserved":0,"allocated":1041457152,"data_stored":1040515560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.79300000000000004,"15min":2.6280000000000001},"min":{"1min":0.254,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":3.3849999999999998,"5min":6.8550000000000004,"15min":108.90000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.626,"15min":2.7559999999999998},"min":{"1min":0.25600000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.98399999999999999,"5min":5.6639999999999997,"15min":109.187},"last":0.28699999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":2.843,"15min":5.2329999999999997},"min":{"1min":0.314,"5min":0.248,"15min":0.222},"max":{"1min":3.4089999999999998,"5min":229.161,"15min":229.161},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.49099999999999999,"5min":2.7109999999999999,"15min":5.2290000000000001},"min":{"1min":0.29599999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":1.427,"5min":230.126,"15min":230.126},"last":0.439}]},{"osd":2,"last update":"Sun Oct 1 09:44:54 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.81399999999999995,"15min":1.6519999999999999},"min":{"1min":0.30399999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.4300000000000002,"5min":5.3090000000000002,"15min":14.901999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.76100000000000001,"15min":1.6299999999999999},"min":{"1min":0.33200000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4630000000000001,"5min":5.6929999999999996,"15min":14.682},"last":0.498}]},{"osd":3,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.79200000000000004,"15min":2.2810000000000001},"min":{"1min":0.32600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.3149999999999999,"5min":5.7750000000000004,"15min":76.555000000000007},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.82499999999999996,"15min":2.2709999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.8479999999999999,"5min":6.548,"15min":76.534000000000006},"last":0.53900000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.80200000000000005,"15min":1.0409999999999999},"min":{"1min":0.31,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":3.4609999999999999,"5min":6.8769999999999998,"15min":9.6020000000000003},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.82599999999999996,"15min":0.999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28000000000000003},"max":{"1min":0.63900000000000001,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.47499999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":2.0960000000000001,"15min":1.548},"min":{"1min":0.315,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":0.80000000000000004,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":2.0830000000000002,"15min":1.6459999999999999},"min":{"1min":0.35699999999999998,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":2.8849999999999998,"5min":135.339,"15min":135.339},"last":0.58199999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.075,"15min":1.133},"min":{"1min":0.38,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":0.754,"5min":11.58,"15min":11.853999999999999},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.86699999999999999,"15min":1.111},"min":{"1min":0.35599999999999998,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":3.7810000000000001,"5min":13.021000000000001,"15min":14.584},"last":0.60499999999999998}]}]},{"osd":6,"up_from":294,"seq":1262720385252,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1757148,"kb_used_data":1427732,"kb_used_omap":6,"kb_used_meta":329401,"kb_avail":91988004,"statfs":{"total":95995035648,"available":94195716096,"internally_reserved":0,"allocated":1461997568,"data_stored":1461059927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":337306669},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":1.865,"15min":3.9809999999999999},"min":{"1min":0.27100000000000002,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":0.80200000000000005,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":2.0640000000000001,"15min":4.0209999999999999},"min":{"1min":0.311,"5min":0.245,"15min":0.221},"max":{"1min":2.1760000000000002,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.379}]},{"osd":1,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":2.9470000000000001,"15min":4.8600000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.24199999999999999},"max":{"1min":1.9059999999999999,"5min":168.947,"15min":168.947},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":2.9980000000000002,"15min":4.8109999999999999},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":0.59899999999999998,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.57199999999999995}]},{"osd":2,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.75600000000000001,"15min":1.5249999999999999},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":1.9350000000000001,"5min":8.657,"15min":10.92},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.76000000000000001,"15min":1.542},"min":{"1min":0.32300000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.2649999999999999,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.52100000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.70599999999999996,"15min":2.0699999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.1909999999999998,"5min":4.149,"15min":89.753},"last":0.313},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.875,"15min":2.0179999999999998},"min":{"1min":0.38500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.1379999999999999,"5min":8.5470000000000006,"15min":89.849999999999994},"last":0.33700000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":1.4510000000000001,"15min":1.0680000000000001},"min":{"1min":0.27200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":0.623,"5min":63.134,"15min":63.134},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.4219999999999999,"15min":1.0669999999999999},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":0.73499999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.49199999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:45:31 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.1599999999999999,"15min":1.0369999999999999},"min":{"1min":0.307,"5min":0.26700000000000002,"15min":0.251},"max":{"1min":0.82199999999999995,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.2849999999999999,"15min":1.155},"min":{"1min":0.32500000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.6800000000000002,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.59399999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.67800000000000005,"15min":1.272},"min":{"1min":0.40200000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.83699999999999997,"5min":3.492,"15min":94.245999999999995},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.87,"15min":1.379},"min":{"1min":0.42299999999999999,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":2.202,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.434}]}]},{"osd":1,"up_from":94,"seq":403726926228,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1440868,"kb_used_data":626780,"kb_used_omap":7,"kb_used_meta":814072,"kb_avail":92304284,"statfs":{"total":95995035648,"available":94519586816,"internally_reserved":0,"allocated":641822720,"data_stored":639764146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":833610151},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7469999999999999,"5min":0.93400000000000005,"15min":4.101},"min":{"1min":0.189,"5min":0.186,"15min":0.182},"max":{"1min":32.735999999999997,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":2.7959999999999998,"5min":1.147,"15min":3.6240000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":32.591999999999999,"5min":32.591999999999999,"15min":95.236999999999995},"last":2.8330000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:44:39 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.42399999999999999,"15min":2.4889999999999999},"min":{"1min":0.23300000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":1.3240000000000001,"5min":2.8740000000000001,"15min":56.573},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.61199999999999999,"15min":3.1829999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":2.2200000000000002,"5min":6.9710000000000001,"15min":154.197},"last":2.9089999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.752,"15min":3.7629999999999999},"min":{"1min":0.254,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":0.48899999999999999,"5min":18.503,"15min":200.56800000000001},"last":2.8780000000000001},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.70799999999999996,"15min":4.484},"min":{"1min":0.26300000000000001,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":2.7360000000000002,"5min":16.515000000000001,"15min":200.63800000000001},"last":0.51300000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.73299999999999998,"15min":2.2160000000000002},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":2.9209999999999998,"5min":4.5789999999999997,"15min":154.101},"last":1.5309999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.58199999999999996,"15min":2.3420000000000001},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.56200000000000006,"5min":2.7290000000000001,"15min":154.56999999999999},"last":0.34300000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:44:39 2023","interfaces":[{"interface":"back","average":{"1min":3.593,"5min":1.272,"15min":2.2759999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":59.561,"5min":59.561,"15min":154.22999999999999},"last":1.583},{"interface":"front","average":{"1min":3.5979999999999999,"5min":1.24,"15min":2.5680000000000001},"min":{"1min":0.35699999999999998,"5min":0.309,"15min":0.307},"max":{"1min":59.536000000000001,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.374}]},{"osd":6,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.99399999999999999,"15min":2.5409999999999999},"min":{"1min":0.33500000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.86299999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":2.944},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.96999999999999997,"15min":2.419},"min":{"1min":0.38200000000000001,"5min":0.34599999999999997,"15min":0.32700000000000001},"max":{"1min":2.194,"5min":23.382000000000001,"15min":154.16999999999999},"last":1.5580000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.94999999999999996,"15min":2.5529999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.33900000000000002},"max":{"1min":2.331,"5min":9.2460000000000004,"15min":154.20699999999999},"last":1.4950000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.82999999999999996,"15min":2.609},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":1.9750000000000001,"5min":5.0110000000000001,"15min":154.51599999999999},"last":0.40300000000000002}]}]},{"osd":0,"up_from":89,"seq":382252089763,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3626056,"kb_used_data":2698752,"kb_used_omap":8,"kb_used_meta":927223,"kb_avail":90119096,"statfs":{"total":95995035648,"available":92281954304,"internally_reserved":0,"allocated":2763522048,"data_stored":2761495851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":949477314},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.58499999999999996,"15min":4.0869999999999997},"min":{"1min":0.18099999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":4.2919999999999998,"5min":8.4939999999999998,"15min":216.821},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.59099999999999997,"15min":4.8499999999999996},"min":{"1min":0.20999999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":3.9620000000000002,"5min":8.6229999999999993,"15min":217.02699999999999},"last":0.50700000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.69999999999999996,"15min":2.0339999999999998},"min":{"1min":0.29299999999999998,"5min":0.246,"15min":0.20100000000000001},"max":{"1min":4.2430000000000003,"5min":4.9459999999999997,"15min":19.788},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.70799999999999996,"15min":2.1200000000000001},"min":{"1min":0.30399999999999999,"5min":0.23000000000000001,"15min":0.189},"max":{"1min":3.4940000000000002,"5min":4.5359999999999996,"15min":25.844999999999999},"last":0.45200000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.63100000000000001,"15min":3.3969999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.314},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.73799999999999999,"15min":3.27},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.9199999999999999,"5min":5.8419999999999996,"15min":222.227},"last":2.1379999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.341,"15min":2.028},"min":{"1min":0.34000000000000002,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":4.048,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.64700000000000002,"5min":1.3140000000000001,"15min":2.0880000000000001},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":4,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.47799999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:44:37 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.76300000000000001,"15min":1.8069999999999999},"min":{"1min":0.33200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.476,"5min":5.8769999999999998,"15min":16.808},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.90300000000000002,"15min":1.708},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":4.2729999999999997,"5min":5.0629999999999997,"15min":17.507000000000001},"last":0.52100000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.89600000000000002,"15min":1.738},"min":{"1min":0.38200000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.3129999999999997,"5min":5.2110000000000003,"15min":17.535},"last":0.498},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.85799999999999998,"15min":1.8089999999999999},"min":{"1min":0.43099999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.2050000000000001,"5min":4.2640000000000002,"15min":15.651999999999999},"last":0.72999999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.86199999999999999,"15min":2.0579999999999998},"min":{"1min":0.375,"5min":0.374,"15min":0.374},"max":{"1min":4.3360000000000003,"5min":4.976,"15min":48.895000000000003},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.88900000000000001,"15min":2.1179999999999999},"min":{"1min":0.439,"5min":0.371,"15min":0.35099999999999998},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.76100000000000001}]}]},{"osd":2,"up_from":0,"seq":425201762702,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599176,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1460000,"kb_used_data":623768,"kb_used_omap":3,"kb_used_meta":836220,"kb_avail":92285152,"statfs":{"total":95995035648,"available":94499995648,"internally_reserved":0,"allocated":638738432,"data_stored":636625556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":856290023},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":1.6839999999999999,"15min":4.843},"min":{"1min":0.20399999999999999,"5min":0.183,"15min":0.17899999999999999},"max":{"1min":0.877,"5min":64.504000000000005,"15min":137.34399999999999},"last":0.253},{"interface":"front","average":{"1min":0.34599999999999997,"5min":1.6000000000000001,"15min":4.9329999999999998},"min":{"1min":0.20599999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.80200000000000005,"5min":64.448999999999998,"15min":137.73400000000001},"last":0.22600000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.67900000000000005,"15min":5.0720000000000001},"min":{"1min":0.252,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":0.60499999999999998,"5min":14.917999999999999,"15min":162.22999999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.60699999999999998,"15min":5.125},"min":{"1min":0.26400000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.57399999999999995,"5min":15.074,"15min":162.70699999999999},"last":0.20899999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.60099999999999998,"15min":1.6859999999999999},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.20799999999999999},"max":{"1min":0.99199999999999999,"5min":7.141,"15min":14.254},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.432,"5min":0.55900000000000005,"15min":1.77},"min":{"1min":0.27000000000000002,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":1.111,"5min":4.5049999999999999,"15min":15.523},"last":0.314}]},{"osd":4,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":1.371,"15min":1.7769999999999999},"min":{"1min":0.33200000000000002,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":0.64200000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.4570000000000001,"15min":2.0409999999999999},"min":{"1min":0.35799999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.5609999999999999,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.35099999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.82599999999999996,"15min":2.1419999999999999},"min":{"1min":0.314,"5min":0.29399999999999998,"15min":0.27500000000000002},"max":{"1min":2.895,"5min":6.0659999999999998,"15min":28.117999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.76700000000000002,"15min":2.2450000000000001},"min":{"1min":0.34899999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.0720000000000001,"5min":7.1559999999999997,"15min":28.34},"last":0.41799999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:45:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.512,"15min":1.879},"min":{"1min":0.33900000000000002,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":2.9100000000000001,"5min":73.887,"15min":73.887},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.6559999999999999,"15min":1.9530000000000001},"min":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":1.1379999999999999,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.51200000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.96299999999999997,"15min":1.8360000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":3.0299999999999998,"5min":7.0620000000000003,"15min":13.085000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.86899999999999999,"15min":1.716},"min":{"1min":0.40000000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.0150000000000001,"5min":4.5810000000000004,"15min":13.191000000000001},"last":0.38800000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019426,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3463964,"kb_used_data":2569684,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":90281188,"statfs":{"total":95995035648,"available":92447936512,"internally_reserved":0,"allocated":2631356416,"data_stored":2630418330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.4950000000000001,"15min":4.1369999999999996},"min":{"1min":0.22500000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.9889999999999999,"5min":79.837000000000003,"15min":155.315},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.39500000000000002,"5min":1.6519999999999999,"15min":4.5110000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.77600000000000002,"5min":78.983000000000004,"15min":155.334},"last":0.48299999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":1.5489999999999999,"15min":6.298},"min":{"1min":0.33000000000000002,"5min":0.252,"15min":0.224},"max":{"1min":0.67700000000000005,"5min":77.953000000000003,"15min":190.61199999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.754,"5min":1.468,"15min":7.0140000000000002},"min":{"1min":0.35299999999999998,"5min":0.26700000000000002,"15min":0.25600000000000001},"max":{"1min":3.1339999999999999,"5min":77.962999999999994,"15min":253.273},"last":0.41199999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:44:50 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.77200000000000002,"15min":2.375},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.2329999999999997,"5min":5.702,"15min":170.45400000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.755,"15min":2.133},"min":{"1min":0.33100000000000002,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":0.71099999999999997,"5min":5.6150000000000002,"15min":58.771000000000001},"last":0.51200000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.77200000000000002,"15min":2.1379999999999999},"min":{"1min":0.31,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.0299999999999998,"5min":5.5279999999999996,"15min":140.881},"last":1.119},{"interface":"front","average":{"1min":0.749,"5min":0.68300000000000005,"15min":2.327},"min":{"1min":0.312,"5min":0.312,"15min":0.311},"max":{"1min":3.161,"5min":3.786,"15min":140.911},"last":0.47099999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:45:39 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.9279999999999999,"15min":1.4870000000000001},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":2.9649999999999999,"5min":110.003,"15min":110.003},"last":0.309},{"interface":"front","average":{"1min":0.62,"5min":1.8779999999999999,"15min":1.3959999999999999},"min":{"1min":0.34999999999999998,"5min":0.29899999999999999,"15min":0.28899999999999998},"max":{"1min":2.9910000000000001,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.36899999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:45:14 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.76000000000000001,"15min":0.94199999999999995},"min":{"1min":0.32800000000000001,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":3.0539999999999998,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.79000000000000004,"15min":1.0569999999999999},"min":{"1min":0.35899999999999999,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":1.028,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.42399999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:44:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.65000000000000002,"15min":1.1399999999999999},"min":{"1min":0.36399999999999999,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":3.0830000000000002,"5min":3.8580000000000001,"15min":38.034999999999997},"last":0.495},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.91600000000000004,"15min":1.4350000000000001},"min":{"1min":0.39900000000000002,"5min":0.36699999999999999,"15min":0.308},"max":{"1min":0.80000000000000004,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.44800000000000001}]}]},{"osd":5,"up_from":379,"seq":1627792605333,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1680776,"kb_used_data":1316224,"kb_used_omap":13,"kb_used_meta":364530,"kb_avail":92064376,"statfs":{"total":95995035648,"available":94273921024,"internally_reserved":0,"allocated":1347813376,"data_stored":1346891194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":373279218},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":1.004,"15min":2.157},"min":{"1min":0.28699999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.1400000000000001,"5min":27.584,"15min":84.340999999999994},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.97699999999999998,"15min":2.0910000000000002},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":4.2300000000000004,"5min":27.669,"15min":84.356999999999999},"last":0.38}]},{"osd":1,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":2.0430000000000001,"15min":5.6200000000000001},"min":{"1min":0.26600000000000001,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":4.3289999999999997,"5min":106.56999999999999,"15min":295.71300000000002},"last":1.337},{"interface":"front","average":{"1min":0.495,"5min":2.0630000000000002,"15min":5.6260000000000003},"min":{"1min":0.27200000000000002,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":2.169,"5min":106.596,"15min":299.59399999999999},"last":0.35099999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.77000000000000002,"15min":1.5089999999999999},"min":{"1min":0.28799999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.2670000000000003,"5min":5.3470000000000004,"15min":14.199999999999999},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.89800000000000002,"15min":1.468},"min":{"1min":0.315,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":4.3719999999999999,"5min":5.782,"15min":14.430999999999999},"last":1.252}]},{"osd":3,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.71799999999999997,"15min":1.417},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.1880000000000002,"5min":4.0209999999999999,"15min":14.004},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.84399999999999997,"15min":1.5629999999999999},"min":{"1min":0.36599999999999999,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":4.2910000000000004,"5min":5.4400000000000004,"15min":13.991},"last":0.47799999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.86599999999999999,"15min":1.3959999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.1929999999999996,"5min":5.9459999999999997,"15min":122.971},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.02,"15min":1.605},"min":{"1min":0.30399999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.3869999999999996,"5min":12.445,"15min":122.89100000000001},"last":0.371}]},{"osd":6,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.86899999999999999,"15min":0.99399999999999999},"min":{"1min":0.314,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":2.2069999999999999,"5min":6.7750000000000004,"15min":12.545},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.91200000000000003,"15min":1.2130000000000001},"min":{"1min":0.35199999999999998,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":4.3200000000000003,"5min":5.4160000000000004,"15min":14.193},"last":0.44600000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.0349999999999999,"15min":1.1890000000000001},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.3559999999999999,"5min":5.758,"15min":10.813000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.99399999999999999,"15min":1.0660000000000001},"min":{"1min":0.39400000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":2.226,"5min":5.2300000000000004,"15min":12.500999999999999},"last":0.434}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2747928576,"data_stored":2747924504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":629669888,"data_stored":629669888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":631635968,"data_stored":631635968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2623475712,"data_stored":2623471640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1341587456,"data_stored":1341587456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1453723648,"data_stored":1453719576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1031208960,"data_stored":1031208960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:45:45.803 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:45 smithi190 ceph-mon[160435]: from='client.37152 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:45.804 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:45 smithi190 ceph-mon[160435]: pgmap v1591: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 179 op/s; 6625/152070 objects degraded (4.357%); 3089/152070 objects misplaced (2.031%); 766 KiB/s, 11 objects/s recovering 2023-10-01T09:45:45.845 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:45.872 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:45.914 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:45.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:45.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:46.022 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:46.495 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:46.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:46.617 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:46.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:46.718 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:46.796 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:46.867 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:46.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:46.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:46.914 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:45:46.915 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:45:46.915 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:45:46.915 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:45:46.916 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:45:46.973 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:47.001 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:47.024 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:45:47.045 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:47.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:47.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:47.152 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:47.177 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:47.229 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:47.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:47.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:47.335 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:47.363 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:47.413 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:47.415 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:47.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:47.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:47.522 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:47.543 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:45:47.551 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:47.634 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:47 smithi138 ceph-mon[185886]: pgmap v1592: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 6625/150882 objects degraded (4.391%); 3089/150882 objects misplaced (2.047%); 755 KiB/s, 11 objects/s recovering 2023-10-01T09:45:47.634 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:47 smithi138 ceph-mon[192513]: pgmap v1592: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 6625/150882 objects degraded (4.391%); 3089/150882 objects misplaced (2.047%); 755 KiB/s, 11 objects/s recovering 2023-10-01T09:45:47.676 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:47 smithi190 ceph-mon[160435]: pgmap v1592: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 175 op/s; 6625/150882 objects degraded (4.391%); 3089/150882 objects misplaced (2.047%); 755 KiB/s, 11 objects/s recovering 2023-10-01T09:45:47.697 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:45:47.700 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:47.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:47.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:47.805 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:47.839 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:47.889 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:47.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:47.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:48.001 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:48.033 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:48.087 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:48.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:48.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:48.122 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:45:48.197 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:48.226 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:48.268 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:48.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:48.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:48.374 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:48.401 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:48.539 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:48 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6625/150882 objects degraded (4.391%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:48.539 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:48 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6625/150882 objects degraded (4.391%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:48.560 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:48.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:48.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:48.668 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:48.702 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:48 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6625/150882 objects degraded (4.391%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:48.704 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:48.713 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:48.753 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:45:48.780 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:48.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:48.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:48.888 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:48.916 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:49.022 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:49.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:49.126 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:49.154 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:49.393 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:49.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:49.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:49.501 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:49.530 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:49.577 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:49.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:49.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:49.682 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:49.706 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:49 smithi138 ceph-mon[185886]: pgmap v1593: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 180 op/s; 6579/149250 objects degraded (4.408%); 3024/149250 objects misplaced (2.026%); 749 KiB/s, 11 objects/s recovering 2023-10-01T09:45:49.707 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1989850' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:45:49.707 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:49 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:45:49.707 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:49 smithi138 ceph-mon[192513]: pgmap v1593: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 180 op/s; 6579/149250 objects degraded (4.408%); 3024/149250 objects misplaced (2.026%); 749 KiB/s, 11 objects/s recovering 2023-10-01T09:45:49.707 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1989850' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:45:49.707 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:49 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:45:49.710 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:49.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:49 smithi190 ceph-mon[160435]: pgmap v1593: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 180 op/s; 6579/149250 objects degraded (4.408%); 3024/149250 objects misplaced (2.026%); 749 KiB/s, 11 objects/s recovering 2023-10-01T09:45:49.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1989850' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:45:49.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:49 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:45:49.812 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:45:49.848 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:45:49.848 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:45:49.848 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:45:49.848 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:45:49.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.848 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:13046 10.e 10:719b4a1e:::benchmark_data_smithi138_573643_object13045:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e481)", 2023-10-01T09:45:49.848 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:17.662365+0000", 2023-10-01T09:45:49.848 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 332.18265669099998, 2023-10-01T09:45:49.849 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27669893600000001, 2023-10-01T09:45:49.849 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.849 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.849 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:45:49.849 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:45:49.849 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:49.849 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 13046 2023-10-01T09:45:49.849 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.849 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.849 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.849 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.850 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.662365+0000", 2023-10-01T09:45:49.850 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.850 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.850 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.850 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.850 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.662365+0000", 2023-10-01T09:45:49.850 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:45:49.850 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.850 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.850 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.850 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.662364+0000", 2023-10-01T09:45:49.850 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9302000000000002e-05 2023-10-01T09:45:49.851 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.851 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.851 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.851 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.662393+0000", 2023-10-01T09:45:49.851 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6069999999999997e-06 2023-10-01T09:45:49.851 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.851 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.851 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.851 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.662403+0000", 2023-10-01T09:45:49.851 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6609999999999997e-06 2023-10-01T09:45:49.851 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.852 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.852 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.852 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.662408+0000", 2023-10-01T09:45:49.852 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00048125799999999998 2023-10-01T09:45:49.852 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.852 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.852 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.852 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.662890+0000", 2023-10-01T09:45:49.852 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000111477 2023-10-01T09:45:49.852 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.852 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.852 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.853 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.663001+0000", 2023-10-01T09:45:49.853 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014086900000000001 2023-10-01T09:45:49.853 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.853 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.853 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 5,6", 2023-10-01T09:45:49.853 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.663142+0000", 2023-10-01T09:45:49.853 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0022233600000000002 2023-10-01T09:45:49.853 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.854 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.854 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:45:49.854 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.665365+0000", 2023-10-01T09:45:49.854 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001377479 2023-10-01T09:45:49.854 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.854 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.854 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.854 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.666743+0000", 2023-10-01T09:45:49.854 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27222058700000001 2023-10-01T09:45:49.854 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.854 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.855 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.855 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.938963+0000", 2023-10-01T09:45:49.855 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7479000000000003e-05 2023-10-01T09:45:49.855 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.855 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.855 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.855 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.939021+0000", 2023-10-01T09:45:49.855 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2858000000000002e-05 2023-10-01T09:45:49.855 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.855 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.855 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.855 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.939064+0000", 2023-10-01T09:45:49.856 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27669893600000001 2023-10-01T09:45:49.856 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.856 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.856 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.856 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.856 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.856 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:13053 10.e 10:7f9fadc7:::benchmark_data_smithi138_573643_object13052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e481)", 2023-10-01T09:45:49.857 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:17.663380+0000", 2023-10-01T09:45:49.857 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 332.18164173399998, 2023-10-01T09:45:49.857 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.277059838, 2023-10-01T09:45:49.857 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.857 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.857 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:45:49.857 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:45:49.857 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:49.857 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 13053 2023-10-01T09:45:49.857 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.857 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.858 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.858 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.858 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.663380+0000", 2023-10-01T09:45:49.858 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.858 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.858 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.858 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.858 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.663380+0000", 2023-10-01T09:45:49.858 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:45:49.858 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.858 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.858 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.859 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.663379+0000", 2023-10-01T09:45:49.859 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1572000000000002e-05 2023-10-01T09:45:49.859 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.859 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.859 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.859 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.663410+0000", 2023-10-01T09:45:49.859 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.1079999999999999e-06 2023-10-01T09:45:49.859 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.859 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.859 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.859 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.663419+0000", 2023-10-01T09:45:49.859 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.0880000000000002e-06 2023-10-01T09:45:49.860 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.861 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.861 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.861 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.663424+0000", 2023-10-01T09:45:49.861 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00086873400000000002 2023-10-01T09:45:49.861 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.861 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.861 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.861 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.664293+0000", 2023-10-01T09:45:49.861 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013548999999999999 2023-10-01T09:45:49.861 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.861 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.862 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.862 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.664429+0000", 2023-10-01T09:45:49.862 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016613700000000001 2023-10-01T09:45:49.862 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.862 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.862 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 5,6", 2023-10-01T09:45:49.862 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.664595+0000", 2023-10-01T09:45:49.862 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0021553079999999999 2023-10-01T09:45:49.862 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.862 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.862 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:45:49.863 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.666750+0000", 2023-10-01T09:45:49.863 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00060291299999999995 2023-10-01T09:45:49.863 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.863 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.863 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.863 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.667353+0000", 2023-10-01T09:45:49.863 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27300628700000001 2023-10-01T09:45:49.863 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.864 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.864 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.865 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.940359+0000", 2023-10-01T09:45:49.865 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4694000000000003e-05 2023-10-01T09:45:49.865 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.865 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.865 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.865 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.940404+0000", 2023-10-01T09:45:49.866 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5641e-05 2023-10-01T09:45:49.866 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.866 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.866 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.866 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.940440+0000", 2023-10-01T09:45:49.866 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.277059838 2023-10-01T09:45:49.866 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.866 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.867 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.867 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.867 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.867 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:13061 10.f 10:fec0ec3e:::benchmark_data_smithi138_573643_object13060:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e481)", 2023-10-01T09:45:49.867 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:17.666913+0000", 2023-10-01T09:45:49.867 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 332.178108189, 2023-10-01T09:45:49.868 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27252544400000001, 2023-10-01T09:45:49.868 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.868 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.868 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:45:49.868 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:45:49.868 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:49.868 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 13061 2023-10-01T09:45:49.868 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.868 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.868 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.868 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.868 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.666913+0000", 2023-10-01T09:45:49.869 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.869 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.869 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.869 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.869 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.666913+0000", 2023-10-01T09:45:49.869 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:45:49.869 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.869 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.869 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.869 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.666912+0000", 2023-10-01T09:45:49.869 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1195000000000001e-05 2023-10-01T09:45:49.869 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.870 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.870 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.870 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.666953+0000", 2023-10-01T09:45:49.870 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1739999999999999e-05 2023-10-01T09:45:49.870 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.870 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.870 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.870 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.666965+0000", 2023-10-01T09:45:49.870 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.387e-06 2023-10-01T09:45:49.870 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.870 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.871 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.871 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.666971+0000", 2023-10-01T09:45:49.871 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.3968999999999999e-05 2023-10-01T09:45:49.871 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.871 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.871 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.871 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.667015+0000", 2023-10-01T09:45:49.871 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011157700000000001 2023-10-01T09:45:49.871 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.871 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.871 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.871 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.667127+0000", 2023-10-01T09:45:49.872 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000145119 2023-10-01T09:45:49.872 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.872 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.872 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:45:49.872 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.667272+0000", 2023-10-01T09:45:49.872 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001260129 2023-10-01T09:45:49.872 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.872 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.872 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:45:49.872 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.668532+0000", 2023-10-01T09:45:49.872 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00040600100000000002 2023-10-01T09:45:49.872 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.873 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.873 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.873 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.668938+0000", 2023-10-01T09:45:49.873 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27039790600000002 2023-10-01T09:45:49.873 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.873 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.873 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.873 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.939336+0000", 2023-10-01T09:45:49.873 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2158999999999999e-05 2023-10-01T09:45:49.873 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.873 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.873 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.874 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.939398+0000", 2023-10-01T09:45:49.874 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0905999999999998e-05 2023-10-01T09:45:49.874 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.874 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.874 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.874 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.939439+0000", 2023-10-01T09:45:49.874 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27252544400000001 2023-10-01T09:45:49.874 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.874 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.874 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.874 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.874 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.875 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:13070 10.e 10:73d3b1d2:::benchmark_data_smithi138_573643_object13069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e481)", 2023-10-01T09:45:49.875 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:17.670196+0000", 2023-10-01T09:45:49.875 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 332.17482534999999, 2023-10-01T09:45:49.875 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27034502799999999, 2023-10-01T09:45:49.875 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.875 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.875 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:45:49.875 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:45:49.875 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:49.875 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 13070 2023-10-01T09:45:49.875 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.875 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.876 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.876 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.876 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670196+0000", 2023-10-01T09:45:49.876 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.876 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.876 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.876 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.876 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670196+0000", 2023-10-01T09:45:49.876 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:45:49.876 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.876 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.876 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.877 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670195+0000", 2023-10-01T09:45:49.877 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1433e-05 2023-10-01T09:45:49.877 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.877 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.877 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.877 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670246+0000", 2023-10-01T09:45:49.877 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4396000000000001e-05 2023-10-01T09:45:49.877 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.877 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.877 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.877 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670260+0000", 2023-10-01T09:45:49.877 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.8689999999999998e-06 2023-10-01T09:45:49.878 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.878 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.878 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.878 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670267+0000", 2023-10-01T09:45:49.878 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0586000000000001e-05 2023-10-01T09:45:49.878 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.878 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.878 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.878 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670308+0000", 2023-10-01T09:45:49.878 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013956400000000001 2023-10-01T09:45:49.878 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.879 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.879 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.879 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670447+0000", 2023-10-01T09:45:49.879 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000172509 2023-10-01T09:45:49.879 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.879 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.879 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 5,6", 2023-10-01T09:45:49.879 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670620+0000", 2023-10-01T09:45:49.879 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001575449 2023-10-01T09:45:49.879 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.879 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.879 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.880 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.672195+0000", 2023-10-01T09:45:49.880 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1440000000000001e-05 2023-10-01T09:45:49.880 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.880 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.880 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:45:49.880 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.672207+0000", 2023-10-01T09:45:49.880 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26826493899999998 2023-10-01T09:45:49.880 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.880 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.881 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.881 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.940472+0000", 2023-10-01T09:45:49.881 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8254999999999999e-05 2023-10-01T09:45:49.881 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.881 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.881 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.881 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.940510+0000", 2023-10-01T09:45:49.881 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1236999999999999e-05 2023-10-01T09:45:49.881 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.881 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.881 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.882 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.940541+0000", 2023-10-01T09:45:49.882 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27034502799999999 2023-10-01T09:45:49.882 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.882 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.882 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.882 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.882 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.882 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:13071 10.7 10:e7bf23c2:::benchmark_data_smithi138_573643_object13070:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e481)", 2023-10-01T09:45:49.882 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:17.670532+0000", 2023-10-01T09:45:49.882 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 332.17448983999998, 2023-10-01T09:45:49.883 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26907466800000002, 2023-10-01T09:45:49.883 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.883 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.883 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:45:49.883 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:45:49.883 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:49.883 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 13071 2023-10-01T09:45:49.883 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.883 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.883 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.883 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.883 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670532+0000", 2023-10-01T09:45:49.884 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.884 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.884 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.884 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.884 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670532+0000", 2023-10-01T09:45:49.884 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:45:49.884 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.884 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.884 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.884 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670531+0000", 2023-10-01T09:45:49.884 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2860999999999993e-05 2023-10-01T09:45:49.884 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.885 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.885 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.885 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670594+0000", 2023-10-01T09:45:49.885 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2357e-05 2023-10-01T09:45:49.885 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.885 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.885 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.885 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670606+0000", 2023-10-01T09:45:49.885 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.7649999999999996e-06 2023-10-01T09:45:49.886 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.886 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.886 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.886 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670613+0000", 2023-10-01T09:45:49.886 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4332000000000002e-05 2023-10-01T09:45:49.886 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.886 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.886 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.886 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670657+0000", 2023-10-01T09:45:49.886 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000121322 2023-10-01T09:45:49.886 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.887 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.887 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.887 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670778+0000", 2023-10-01T09:45:49.887 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000105289 2023-10-01T09:45:49.887 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.887 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.887 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:45:49.887 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670884+0000", 2023-10-01T09:45:49.887 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000862976 2023-10-01T09:45:49.887 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.887 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.887 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:45:49.888 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.671747+0000", 2023-10-01T09:45:49.889 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000178956 2023-10-01T09:45:49.889 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.889 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.889 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.889 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.671926+0000", 2023-10-01T09:45:49.889 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26759136100000003 2023-10-01T09:45:49.889 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.889 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.889 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.890 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.939517+0000", 2023-10-01T09:45:49.890 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1588000000000001e-05 2023-10-01T09:45:49.890 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.890 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.890 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.890 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.939568+0000", 2023-10-01T09:45:49.890 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.7883000000000003e-05 2023-10-01T09:45:49.890 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.891 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.891 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.891 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.939606+0000", 2023-10-01T09:45:49.891 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26907466800000002 2023-10-01T09:45:49.891 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.891 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.891 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.891 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.891 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.891 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:13072 10.e 10:7583c829:::benchmark_data_smithi138_573643_object13071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e481)", 2023-10-01T09:45:49.891 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:17.670740+0000", 2023-10-01T09:45:49.892 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 332.17428186500001, 2023-10-01T09:45:49.892 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27174748500000001, 2023-10-01T09:45:49.892 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.892 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.892 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:45:49.892 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:45:49.892 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:49.892 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 13072 2023-10-01T09:45:49.892 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.892 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.892 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.892 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.893 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670740+0000", 2023-10-01T09:45:49.893 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.893 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.893 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.893 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.893 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670740+0000", 2023-10-01T09:45:49.893 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:45:49.893 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.893 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.893 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.893 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670739+0000", 2023-10-01T09:45:49.893 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.622e-05 2023-10-01T09:45:49.894 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.894 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.894 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.894 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670765+0000", 2023-10-01T09:45:49.894 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0412e-05 2023-10-01T09:45:49.894 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.894 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.894 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.894 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670775+0000", 2023-10-01T09:45:49.894 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1739999999999997e-06 2023-10-01T09:45:49.894 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.894 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.895 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.895 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.670782+0000", 2023-10-01T09:45:49.895 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00058707099999999999 2023-10-01T09:45:49.895 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.895 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.895 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.895 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.671369+0000", 2023-10-01T09:45:49.895 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00011574399999999999 2023-10-01T09:45:49.895 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.895 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.895 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.895 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.671484+0000", 2023-10-01T09:45:49.896 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014510799999999999 2023-10-01T09:45:49.896 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.896 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.896 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 5,6", 2023-10-01T09:45:49.896 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.671629+0000", 2023-10-01T09:45:49.896 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00067996300000000001 2023-10-01T09:45:49.896 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.896 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.896 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:45:49.896 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.672309+0000", 2023-10-01T09:45:49.896 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00031506999999999998 2023-10-01T09:45:49.897 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.897 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.897 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.897 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.672624+0000", 2023-10-01T09:45:49.897 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26976457799999998 2023-10-01T09:45:49.897 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.897 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.897 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.897 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.942389+0000", 2023-10-01T09:45:49.897 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5648e-05 2023-10-01T09:45:49.897 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.897 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.898 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.898 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.942445+0000", 2023-10-01T09:45:49.898 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2410999999999998e-05 2023-10-01T09:45:49.898 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.898 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.898 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.898 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:17.942487+0000", 2023-10-01T09:45:49.898 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27174748500000001 2023-10-01T09:45:49.898 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.898 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.898 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.898 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.899 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.899 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:43787 10.e 10:7673dfc9:::benchmark_data_smithi138_573643_object43786:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-10-01T09:45:49.899 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:39.487803+0000", 2023-10-01T09:45:49.899 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 310.35721837599999, 2023-10-01T09:45:49.899 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28991972799999999, 2023-10-01T09:45:49.899 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.899 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.899 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:45:49.899 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:45:49.899 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:49.899 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 43787 2023-10-01T09:45:49.899 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.899 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.900 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.900 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.900 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487803+0000", 2023-10-01T09:45:49.900 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.900 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.900 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.900 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.900 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487803+0000", 2023-10-01T09:45:49.900 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:45:49.900 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.900 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.900 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.901 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487801+0000", 2023-10-01T09:45:49.901 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6554999999999997e-05 2023-10-01T09:45:49.901 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.901 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.901 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.901 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487848+0000", 2023-10-01T09:45:49.901 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2105e-05 2023-10-01T09:45:49.901 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.901 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.901 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.901 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487860+0000", 2023-10-01T09:45:49.901 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6630000000000004e-06 2023-10-01T09:45:49.902 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.902 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.902 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.902 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487867+0000", 2023-10-01T09:45:49.902 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1038000000000002e-05 2023-10-01T09:45:49.902 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.902 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.902 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.902 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.487908+0000", 2023-10-01T09:45:49.902 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013976 2023-10-01T09:45:49.902 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.902 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.903 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.903 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.488048+0000", 2023-10-01T09:45:49.903 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00016087499999999999 2023-10-01T09:45:49.903 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.903 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.903 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 5,6", 2023-10-01T09:45:49.903 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.488208+0000", 2023-10-01T09:45:49.903 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019672679999999999 2023-10-01T09:45:49.903 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.903 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.903 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.903 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.490176+0000", 2023-10-01T09:45:49.904 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019025120000000001 2023-10-01T09:45:49.904 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.904 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.904 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:45:49.904 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.492078+0000", 2023-10-01T09:45:49.904 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28557051300000003 2023-10-01T09:45:49.904 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.904 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.904 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.904 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.777649+0000", 2023-10-01T09:45:49.904 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8287999999999997e-05 2023-10-01T09:45:49.904 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.905 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.905 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.905 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.777687+0000", 2023-10-01T09:45:49.905 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.5837000000000003e-05 2023-10-01T09:45:49.905 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.905 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.905 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.905 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:39.777723+0000", 2023-10-01T09:45:49.905 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28991972799999999 2023-10-01T09:45:49.905 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.905 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.905 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.906 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.906 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.906 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.36456.0:46583 10.6 e488/475 10:6449e8e3:::benchmark_data_smithi138_573643_object46582:head v 488'2912, mlcod=488'2907)", 2023-10-01T09:45:49.906 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.272764+0000", 2023-10-01T09:45:49.906 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 307.57225713999998, 2023-10-01T09:45:49.906 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27705594, 2023-10-01T09:45:49.906 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.906 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.906 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.906 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.906 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.906 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.272764+0000", 2023-10-01T09:45:49.906 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.907 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.907 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.907 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.907 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.272764+0000", 2023-10-01T09:45:49.907 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:45:49.907 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.907 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.909 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.909 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.272764+0000", 2023-10-01T09:45:49.909 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.355e-05 2023-10-01T09:45:49.909 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.909 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.909 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.909 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.272787+0000", 2023-10-01T09:45:49.910 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.012e-05 2023-10-01T09:45:49.910 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.910 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.910 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.910 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.272797+0000", 2023-10-01T09:45:49.910 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5779999999999998e-06 2023-10-01T09:45:49.910 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.910 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.910 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.910 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.272803+0000", 2023-10-01T09:45:49.910 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001638811 2023-10-01T09:45:49.910 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.911 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.911 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.911 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274442+0000", 2023-10-01T09:45:49.911 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0653e-05 2023-10-01T09:45:49.911 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.911 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.911 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.911 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274462+0000", 2023-10-01T09:45:49.911 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.004304637 2023-10-01T09:45:49.911 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.911 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.911 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.912 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278767+0000", 2023-10-01T09:45:49.912 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.271053343 2023-10-01T09:45:49.912 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.912 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.912 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.912 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.549820+0000", 2023-10-01T09:45:49.912 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27705594 2023-10-01T09:45:49.912 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.912 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.912 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.912 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.912 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.913 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.36456.0:46587 10.5 e488/475 10:ae263ab1:::benchmark_data_smithi138_573643_object46586:head v 488'2952, mlcod=488'2950)", 2023-10-01T09:45:49.913 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.273382+0000", 2023-10-01T09:45:49.913 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 307.57163919499999, 2023-10-01T09:45:49.913 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28007110000000002, 2023-10-01T09:45:49.913 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.913 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.913 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.913 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.913 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.913 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.273382+0000", 2023-10-01T09:45:49.913 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.914 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.914 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.914 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.914 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.273382+0000", 2023-10-01T09:45:49.914 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:45:49.914 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.914 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.914 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.914 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.273382+0000", 2023-10-01T09:45:49.914 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9792999999999997e-05 2023-10-01T09:45:49.915 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.915 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.915 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.915 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.273422+0000", 2023-10-01T09:45:49.915 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0628000000000001e-05 2023-10-01T09:45:49.915 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.915 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.915 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.915 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.273432+0000", 2023-10-01T09:45:49.915 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9209999999999997e-06 2023-10-01T09:45:49.915 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.916 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.916 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.916 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.273438+0000", 2023-10-01T09:45:49.916 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0016111400000000001 2023-10-01T09:45:49.916 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.916 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.916 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.916 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.275049+0000", 2023-10-01T09:45:49.916 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7697e-05 2023-10-01T09:45:49.916 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.916 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.917 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.917 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.275067+0000", 2023-10-01T09:45:49.917 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27835695399999999 2023-10-01T09:45:49.917 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.917 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.917 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.917 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.553424+0000", 2023-10-01T09:45:49.917 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9573e-05 2023-10-01T09:45:49.917 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.917 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.917 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.917 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.553453+0000", 2023-10-01T09:45:49.918 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.28007110000000002 2023-10-01T09:45:49.918 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.918 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.918 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.918 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.918 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.918 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.36456.0:46589 10.2 e488/475 10:4b693c7e:::benchmark_data_smithi138_573643_object46588:head v 488'2893, mlcod=488'2891)", 2023-10-01T09:45:49.918 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.273678+0000", 2023-10-01T09:45:49.918 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 307.57134352899999, 2023-10-01T09:45:49.918 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27637898999999999, 2023-10-01T09:45:49.918 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.918 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.919 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.919 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.919 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.919 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.273678+0000", 2023-10-01T09:45:49.919 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.919 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.919 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.919 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.919 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.273678+0000", 2023-10-01T09:45:49.919 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:45:49.919 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.919 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.920 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.920 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.273677+0000", 2023-10-01T09:45:49.920 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0488000000000001e-05 2023-10-01T09:45:49.920 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.920 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.920 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.920 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.273738+0000", 2023-10-01T09:45:49.920 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.687e-06 2023-10-01T09:45:49.920 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.920 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.920 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.920 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.273746+0000", 2023-10-01T09:45:49.921 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.6589999999999998e-06 2023-10-01T09:45:49.921 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.921 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.921 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.921 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.273751+0000", 2023-10-01T09:45:49.921 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4804000000000001e-05 2023-10-01T09:45:49.921 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.921 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.921 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.921 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.273786+0000", 2023-10-01T09:45:49.921 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3229e-05 2023-10-01T09:45:49.921 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.922 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.922 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.922 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.273799+0000", 2023-10-01T09:45:49.922 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049413829999999997 2023-10-01T09:45:49.922 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.922 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.922 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.922 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278741+0000", 2023-10-01T09:45:49.922 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27131645599999998 2023-10-01T09:45:49.922 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.922 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.922 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.923 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.550057+0000", 2023-10-01T09:45:49.923 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27637898999999999 2023-10-01T09:45:49.923 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.923 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.923 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.923 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.923 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.923 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:46591 10.e 10:71bb7fcc:::benchmark_data_smithi138_573643_object46590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e488)", 2023-10-01T09:45:49.923 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.274336+0000", 2023-10-01T09:45:49.923 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 307.57068533500001, 2023-10-01T09:45:49.923 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27761666499999998, 2023-10-01T09:45:49.923 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.923 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.924 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:45:49.924 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:45:49.924 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:49.924 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 46591 2023-10-01T09:45:49.924 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.924 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.924 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.924 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.924 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274336+0000", 2023-10-01T09:45:49.924 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.925 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.925 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.925 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.925 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274336+0000", 2023-10-01T09:45:49.925 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:45:49.925 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.925 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.925 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.925 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274334+0000", 2023-10-01T09:45:49.925 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5813999999999998e-05 2023-10-01T09:45:49.926 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.926 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.926 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.926 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274390+0000", 2023-10-01T09:45:49.926 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3324e-05 2023-10-01T09:45:49.926 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.926 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.926 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.926 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274403+0000", 2023-10-01T09:45:49.926 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5499999999999997e-06 2023-10-01T09:45:49.926 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.926 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.927 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.927 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274411+0000", 2023-10-01T09:45:49.927 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5798999999999999e-05 2023-10-01T09:45:49.927 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.927 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.927 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.927 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274466+0000", 2023-10-01T09:45:49.927 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000117713 2023-10-01T09:45:49.927 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.928 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.928 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.928 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274584+0000", 2023-10-01T09:45:49.928 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014348099999999999 2023-10-01T09:45:49.928 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.928 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.928 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 5,6", 2023-10-01T09:45:49.929 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274728+0000", 2023-10-01T09:45:49.929 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0034707509999999998 2023-10-01T09:45:49.929 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.929 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.929 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.929 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278198+0000", 2023-10-01T09:45:49.929 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000115369 2023-10-01T09:45:49.929 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.929 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.929 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.929 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278314+0000", 2023-10-01T09:45:49.929 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27354470600000003 2023-10-01T09:45:49.929 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.930 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.930 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:45:49.930 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.551858+0000", 2023-10-01T09:45:49.930 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.5247999999999997e-05 2023-10-01T09:45:49.930 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.930 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.930 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.930 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.551914+0000", 2023-10-01T09:45:49.930 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9201e-05 2023-10-01T09:45:49.930 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.930 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.931 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.931 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.551953+0000", 2023-10-01T09:45:49.931 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27761666499999998 2023-10-01T09:45:49.931 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.931 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.931 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.931 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.931 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.931 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:46592 10.7 10:ee376add:::benchmark_data_smithi138_573643_object46591:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e488)", 2023-10-01T09:45:49.931 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.274434+0000", 2023-10-01T09:45:49.931 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 307.57058781799998, 2023-10-01T09:45:49.931 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27965479700000001, 2023-10-01T09:45:49.931 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.932 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.932 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:45:49.932 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:45:49.932 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:49.932 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 46592 2023-10-01T09:45:49.932 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.932 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.932 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.932 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.932 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274434+0000", 2023-10-01T09:45:49.932 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.933 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.933 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.933 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.933 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274434+0000", 2023-10-01T09:45:49.933 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:45:49.933 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.933 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.933 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.933 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274433+0000", 2023-10-01T09:45:49.933 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1563999999999998e-05 2023-10-01T09:45:49.933 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.933 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.934 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.934 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274464+0000", 2023-10-01T09:45:49.934 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2400000000000007e-06 2023-10-01T09:45:49.934 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.934 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.934 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.934 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274473+0000", 2023-10-01T09:45:49.934 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3619999999999997e-06 2023-10-01T09:45:49.934 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.934 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.934 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.935 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274478+0000", 2023-10-01T09:45:49.935 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9255000000000002e-05 2023-10-01T09:45:49.935 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.935 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.935 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.935 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274527+0000", 2023-10-01T09:45:49.935 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000110253 2023-10-01T09:45:49.935 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.935 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.935 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.935 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274638+0000", 2023-10-01T09:45:49.935 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013565100000000001 2023-10-01T09:45:49.936 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.936 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.936 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:45:49.936 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.274773+0000", 2023-10-01T09:45:49.936 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0036622289999999999 2023-10-01T09:45:49.936 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.936 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.936 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.936 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278435+0000", 2023-10-01T09:45:49.936 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27366394300000002 2023-10-01T09:45:49.936 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.936 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.937 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:45:49.937 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.552099+0000", 2023-10-01T09:45:49.937 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0018961169999999999 2023-10-01T09:45:49.937 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.937 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.937 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.937 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.553996+0000", 2023-10-01T09:45:49.937 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.4462999999999997e-05 2023-10-01T09:45:49.937 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.937 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.937 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.937 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.554050+0000", 2023-10-01T09:45:49.938 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.8523999999999999e-05 2023-10-01T09:45:49.938 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.938 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.938 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.938 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.554088+0000", 2023-10-01T09:45:49.938 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27965479700000001 2023-10-01T09:45:49.938 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.938 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.938 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.938 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.938 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.938 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.36456.0:46588 10.6 e488/475 10:6f2bc14e:::benchmark_data_smithi138_573643_object46587:head v 488'2913, mlcod=488'2907)", 2023-10-01T09:45:49.939 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.275401+0000", 2023-10-01T09:45:49.939 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 307.569620579, 2023-10-01T09:45:49.939 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27658738100000002, 2023-10-01T09:45:49.939 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.939 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.939 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.939 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.939 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.939 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.275401+0000", 2023-10-01T09:45:49.939 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.939 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.940 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.940 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.940 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.275401+0000", 2023-10-01T09:45:49.940 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:45:49.940 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.940 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.940 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.940 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.275401+0000", 2023-10-01T09:45:49.940 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1963e-05 2023-10-01T09:45:49.940 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.940 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.940 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.940 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.275423+0000", 2023-10-01T09:45:49.941 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3969999999999999e-06 2023-10-01T09:45:49.941 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.941 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.941 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.941 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.275429+0000", 2023-10-01T09:45:49.941 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.42e-06 2023-10-01T09:45:49.941 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.941 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.941 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.941 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.275433+0000", 2023-10-01T09:45:49.941 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00087765300000000005 2023-10-01T09:45:49.942 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.942 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.942 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.942 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276311+0000", 2023-10-01T09:45:49.942 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6509999999999999e-05 2023-10-01T09:45:49.942 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.942 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.942 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.942 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276328+0000", 2023-10-01T09:45:49.942 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27563045000000003 2023-10-01T09:45:49.942 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.943 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.943 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.943 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.551958+0000", 2023-10-01T09:45:49.943 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.0273e-05 2023-10-01T09:45:49.943 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.943 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.943 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.943 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.551988+0000", 2023-10-01T09:45:49.943 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27658738100000002 2023-10-01T09:45:49.943 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.943 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.943 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.943 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.944 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.944 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:46593 10.f 10:f075a679:::benchmark_data_smithi138_573643_object46592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e488)", 2023-10-01T09:45:49.944 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.276537+0000", 2023-10-01T09:45:49.944 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 307.56848465299998, 2023-10-01T09:45:49.944 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27757499000000002, 2023-10-01T09:45:49.944 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.944 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.944 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:45:49.944 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:45:49.944 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:49.944 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 46593 2023-10-01T09:45:49.944 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.945 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.945 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.945 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.945 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276537+0000", 2023-10-01T09:45:49.945 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.945 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.945 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.945 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.945 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276537+0000", 2023-10-01T09:45:49.945 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:45:49.946 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.946 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.946 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.946 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276535+0000", 2023-10-01T09:45:49.946 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4068e-05 2023-10-01T09:45:49.946 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.946 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.946 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.946 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276570+0000", 2023-10-01T09:45:49.946 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.6059999999999992e-06 2023-10-01T09:45:49.946 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.946 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.947 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.947 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276579+0000", 2023-10-01T09:45:49.947 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5770000000000004e-06 2023-10-01T09:45:49.947 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.947 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.947 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.947 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276584+0000", 2023-10-01T09:45:49.947 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000112353 2023-10-01T09:45:49.947 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.947 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.947 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.947 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276696+0000", 2023-10-01T09:45:49.948 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012723699999999999 2023-10-01T09:45:49.948 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.948 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.948 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.948 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276823+0000", 2023-10-01T09:45:49.948 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015360800000000001 2023-10-01T09:45:49.948 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.948 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.948 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:45:49.948 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276977+0000", 2023-10-01T09:45:49.948 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27696646699999999 2023-10-01T09:45:49.948 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.949 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.949 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:45:49.949 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.553943+0000", 2023-10-01T09:45:49.949 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2008000000000007e-05 2023-10-01T09:45:49.949 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.949 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.949 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.949 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.554005+0000", 2023-10-01T09:45:49.949 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7152e-05 2023-10-01T09:45:49.949 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.949 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.949 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.950 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.554033+0000", 2023-10-01T09:45:49.950 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2886000000000002e-05 2023-10-01T09:45:49.950 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.950 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.950 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.950 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.554075+0000", 2023-10-01T09:45:49.950 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6436000000000001e-05 2023-10-01T09:45:49.950 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.950 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.951 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.951 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.554112+0000", 2023-10-01T09:45:49.951 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27757499000000002 2023-10-01T09:45:49.951 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.951 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.951 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.951 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.951 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.951 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:46594 10.f 10:fae56fed:::benchmark_data_smithi138_573643_object46593:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e488)", 2023-10-01T09:45:49.951 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.276603+0000", 2023-10-01T09:45:49.951 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 307.568418065, 2023-10-01T09:45:49.952 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27787024999999999, 2023-10-01T09:45:49.952 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.952 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.952 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:45:49.952 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:45:49.952 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:49.952 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 46594 2023-10-01T09:45:49.952 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.952 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.952 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.952 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.952 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276603+0000", 2023-10-01T09:45:49.952 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.953 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.953 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.953 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.953 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276603+0000", 2023-10-01T09:45:49.953 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:45:49.953 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.953 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.953 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.953 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276603+0000", 2023-10-01T09:45:49.953 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.6681000000000001e-05 2023-10-01T09:45:49.953 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.953 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.954 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.954 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276629+0000", 2023-10-01T09:45:49.954 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.2940000000000006e-06 2023-10-01T09:45:49.954 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.954 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.954 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.954 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276638+0000", 2023-10-01T09:45:49.954 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9450000000000001e-06 2023-10-01T09:45:49.954 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.954 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.954 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.955 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276643+0000", 2023-10-01T09:45:49.955 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001767355 2023-10-01T09:45:49.955 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.955 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.955 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.955 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278410+0000", 2023-10-01T09:45:49.955 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012397100000000001 2023-10-01T09:45:49.955 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.955 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.955 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.955 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278534+0000", 2023-10-01T09:45:49.955 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015497699999999999 2023-10-01T09:45:49.956 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.956 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.956 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:45:49.956 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278689+0000", 2023-10-01T09:45:49.956 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27526164400000003 2023-10-01T09:45:49.956 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.956 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.956 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:45:49.956 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.553951+0000", 2023-10-01T09:45:49.956 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00020521600000000001 2023-10-01T09:45:49.956 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.956 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.957 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.957 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.554156+0000", 2023-10-01T09:45:49.957 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00023954799999999999 2023-10-01T09:45:49.957 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.957 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.957 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.957 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.554395+0000", 2023-10-01T09:45:49.957 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.2654000000000002e-05 2023-10-01T09:45:49.957 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.957 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.957 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.957 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.554438+0000", 2023-10-01T09:45:49.958 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.57e-05 2023-10-01T09:45:49.958 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.958 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.958 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.958 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.554474+0000", 2023-10-01T09:45:49.958 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27787024999999999 2023-10-01T09:45:49.958 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.958 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.958 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.958 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.959 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.959 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:46596 10.f 10:f832de69:::benchmark_data_smithi138_573643_object46595:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e488)", 2023-10-01T09:45:49.959 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.276810+0000", 2023-10-01T09:45:49.959 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 307.56821121600001, 2023-10-01T09:45:49.959 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.43371495399999999, 2023-10-01T09:45:49.959 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.959 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.959 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:45:49.959 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:45:49.959 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:49.959 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 46596 2023-10-01T09:45:49.960 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.960 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.960 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.960 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.960 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276810+0000", 2023-10-01T09:45:49.960 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.960 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.960 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.960 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.960 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276810+0000", 2023-10-01T09:45:49.960 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:45:49.960 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.961 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.961 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.961 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276809+0000", 2023-10-01T09:45:49.961 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3258999999999999e-05 2023-10-01T09:45:49.961 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.961 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.961 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.961 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276873+0000", 2023-10-01T09:45:49.961 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1403000000000001e-05 2023-10-01T09:45:49.961 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.961 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.961 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.962 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276884+0000", 2023-10-01T09:45:49.962 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6579999999999996e-06 2023-10-01T09:45:49.962 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.962 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.962 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.962 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.276891+0000", 2023-10-01T09:45:49.962 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27497587600000001 2023-10-01T09:45:49.962 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.962 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.962 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.962 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.551867+0000", 2023-10-01T09:45:49.962 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014749699999999999 2023-10-01T09:45:49.963 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.963 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.963 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.963 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.552014+0000", 2023-10-01T09:45:49.963 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000185515 2023-10-01T09:45:49.963 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.963 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.963 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:45:49.963 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.552200+0000", 2023-10-01T09:45:49.963 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0054756620000000001 2023-10-01T09:45:49.963 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.963 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.964 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:45:49.964 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.557675+0000", 2023-10-01T09:45:49.964 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0021370429999999999 2023-10-01T09:45:49.964 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.964 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.964 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.964 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.559812+0000", 2023-10-01T09:45:49.964 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15061901799999999 2023-10-01T09:45:49.964 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.964 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.964 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.964 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.710431+0000", 2023-10-01T09:45:49.965 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.2615999999999998e-05 2023-10-01T09:45:49.965 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.965 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.965 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.965 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.710494+0000", 2023-10-01T09:45:49.965 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1359000000000002e-05 2023-10-01T09:45:49.965 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.965 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.965 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.965 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.710525+0000", 2023-10-01T09:45:49.965 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.43371495399999999 2023-10-01T09:45:49.965 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.966 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.966 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.966 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.966 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.966 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.36456.0:46595 10.c e488/475 10:3f59f942:::benchmark_data_smithi138_573643_object46594:head v 488'2875, mlcod=488'2873)", 2023-10-01T09:45:49.966 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.278305+0000", 2023-10-01T09:45:49.966 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 307.56671695799997, 2023-10-01T09:45:49.966 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27375945000000002, 2023-10-01T09:45:49.966 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.966 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.966 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.966 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.967 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.967 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278305+0000", 2023-10-01T09:45:49.967 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.967 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.967 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.967 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.967 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278305+0000", 2023-10-01T09:45:49.967 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:45:49.967 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.967 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.967 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.968 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278304+0000", 2023-10-01T09:45:49.968 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2697e-05 2023-10-01T09:45:49.968 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.968 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.968 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.968 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278327+0000", 2023-10-01T09:45:49.968 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3520000000000007e-06 2023-10-01T09:45:49.968 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.968 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.968 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.968 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278335+0000", 2023-10-01T09:45:49.968 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.8400000000000007e-06 2023-10-01T09:45:49.968 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.969 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.969 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.969 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278345+0000", 2023-10-01T09:45:49.969 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1408000000000001e-05 2023-10-01T09:45:49.969 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.969 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.969 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.969 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278386+0000", 2023-10-01T09:45:49.969 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8250999999999999e-05 2023-10-01T09:45:49.969 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.969 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.970 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.970 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.278405+0000", 2023-10-01T09:45:49.970 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27363084999999998 2023-10-01T09:45:49.970 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.970 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.970 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.970 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.552035+0000", 2023-10-01T09:45:49.970 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8580000000000001e-05 2023-10-01T09:45:49.970 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.970 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.970 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.970 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.552064+0000", 2023-10-01T09:45:49.971 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27375945000000002 2023-10-01T09:45:49.971 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.971 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.971 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.971 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.971 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.971 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.36456.0:46599 10.1 e488/475 10:879a0c70:::benchmark_data_smithi138_573643_object46598:head v 488'2945, mlcod=488'2943)", 2023-10-01T09:45:49.971 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.279700+0000", 2023-10-01T09:45:49.971 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 307.56532108699997, 2023-10-01T09:45:49.971 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27437677500000002, 2023-10-01T09:45:49.971 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.971 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.971 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.972 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.972 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.972 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.279700+0000", 2023-10-01T09:45:49.972 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.972 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.972 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.972 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.972 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.279700+0000", 2023-10-01T09:45:49.972 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:45:49.972 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.972 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.973 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.973 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.279700+0000", 2023-10-01T09:45:49.973 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3510999999999999e-05 2023-10-01T09:45:49.973 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.973 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.973 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.973 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.279723+0000", 2023-10-01T09:45:49.973 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26980655999999997 2023-10-01T09:45:49.973 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.973 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.973 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.973 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.549530+0000", 2023-10-01T09:45:49.974 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2678e-05 2023-10-01T09:45:49.974 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.974 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.974 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.974 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.549542+0000", 2023-10-01T09:45:49.974 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00038930099999999999 2023-10-01T09:45:49.974 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.974 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.974 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.974 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.549932+0000", 2023-10-01T09:45:49.974 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3003999999999999e-05 2023-10-01T09:45:49.974 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.975 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.975 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.975 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.549955+0000", 2023-10-01T09:45:49.975 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0040880489999999998 2023-10-01T09:45:49.975 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.975 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.975 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.975 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.554043+0000", 2023-10-01T09:45:49.975 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.4384999999999997e-05 2023-10-01T09:45:49.975 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.975 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.975 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.976 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.554077+0000", 2023-10-01T09:45:49.976 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27437677500000002 2023-10-01T09:45:49.976 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.976 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.976 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.976 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.976 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.976 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.36456.0:46600 10.2 e488/475 10:4c6a84b2:::benchmark_data_smithi138_573643_object46599:head v 488'2894, mlcod=488'2891)", 2023-10-01T09:45:49.976 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.282847+0000", 2023-10-01T09:45:49.976 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 307.56217406299999, 2023-10-01T09:45:49.976 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27053920999999997, 2023-10-01T09:45:49.977 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.977 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.977 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.977 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.977 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.977 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.282847+0000", 2023-10-01T09:45:49.977 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.977 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.977 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.977 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.977 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.282847+0000", 2023-10-01T09:45:49.977 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:45:49.978 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.978 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.978 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.978 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.282847+0000", 2023-10-01T09:45:49.978 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9839999999999997e-05 2023-10-01T09:45:49.978 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.978 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.978 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.978 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.282896+0000", 2023-10-01T09:45:49.978 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.26655454899999997 2023-10-01T09:45:49.978 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.978 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.979 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.979 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.549451+0000", 2023-10-01T09:45:49.979 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.7309999999999999e-05 2023-10-01T09:45:49.979 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.979 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.979 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.979 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.549468+0000", 2023-10-01T09:45:49.979 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00062575800000000002 2023-10-01T09:45:49.979 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.979 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.979 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.979 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.550094+0000", 2023-10-01T09:45:49.980 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1169e-05 2023-10-01T09:45:49.980 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.980 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.980 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.980 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.550115+0000", 2023-10-01T09:45:49.980 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003231275 2023-10-01T09:45:49.980 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.980 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.980 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.980 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.553346+0000", 2023-10-01T09:45:49.980 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0206000000000001e-05 2023-10-01T09:45:49.980 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.981 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.981 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.981 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.553387+0000", 2023-10-01T09:45:49.981 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27053920999999997 2023-10-01T09:45:49.981 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.981 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.981 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.981 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.981 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.981 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:46605 10.e 10:73ea0908:::benchmark_data_smithi138_573643_object46604:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e488)", 2023-10-01T09:45:49.981 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:40:42.286599+0000", 2023-10-01T09:45:49.981 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 307.55842296499998, 2023-10-01T09:45:49.982 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27320306799999999, 2023-10-01T09:45:49.982 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:45:49.982 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:45:49.982 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:45:49.982 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:45:49.982 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:45:49.982 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 46605 2023-10-01T09:45:49.982 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.982 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:45:49.982 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.982 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:45:49.982 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.286599+0000", 2023-10-01T09:45:49.983 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:45:49.983 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.983 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.983 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:45:49.983 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.286599+0000", 2023-10-01T09:45:49.983 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:45:49.983 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.983 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.983 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:45:49.983 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.286597+0000", 2023-10-01T09:45:49.983 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6217999999999998e-05 2023-10-01T09:45:49.984 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.984 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.984 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:45:49.984 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.286633+0000", 2023-10-01T09:45:49.984 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.263354961 2023-10-01T09:45:49.984 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.984 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.984 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:45:49.984 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.549988+0000", 2023-10-01T09:45:49.984 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.154e-05 2023-10-01T09:45:49.984 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.984 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.985 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:45:49.985 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.550000+0000", 2023-10-01T09:45:49.985 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0023999199999999998 2023-10-01T09:45:49.985 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.985 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.985 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:45:49.985 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.552400+0000", 2023-10-01T09:45:49.985 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.9219000000000001e-05 2023-10-01T09:45:49.985 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.985 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.985 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:45:49.985 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.552489+0000", 2023-10-01T09:45:49.986 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000127859 2023-10-01T09:45:49.986 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.986 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.986 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 5,6", 2023-10-01T09:45:49.986 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.552617+0000", 2023-10-01T09:45:49.986 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0069857850000000004 2023-10-01T09:45:49.986 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.986 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.986 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.986 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.559602+0000", 2023-10-01T09:45:49.986 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.2753999999999998e-05 2023-10-01T09:45:49.986 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.986 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.987 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:45:49.987 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.559635+0000", 2023-10-01T09:45:49.987 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3226000000000002e-05 2023-10-01T09:45:49.987 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.987 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.987 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:45:49.987 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.559688+0000", 2023-10-01T09:45:49.987 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5111999999999996e-05 2023-10-01T09:45:49.987 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.987 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.987 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:45:49.988 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.559754+0000", 2023-10-01T09:45:49.988 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8063999999999999e-05 2023-10-01T09:45:49.988 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:45:49.988 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:45:49.988 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:45:49.989 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:40:42.559802+0000", 2023-10-01T09:45:49.989 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.27320306799999999 2023-10-01T09:45:49.989 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.989 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.989 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.990 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:45:49.990 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:45:49.990 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:45:49.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:49.992 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:49.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:50.092 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:50.121 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:50.268 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:50.479 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:50.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:50.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:50.583 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:50.614 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:50.682 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:50 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:50.682 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:50 smithi138 ceph-mon[192513]: failed to encode map e540 with expected crc 2023-10-01T09:45:50.682 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:50 smithi138 ceph-mon[192513]: failed to encode map e540 with expected crc 2023-10-01T09:45:50.682 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:50 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:45:50.683 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:50 smithi138 ceph-mon[192513]: osdmap e540: 8 total, 8 up, 7 in 2023-10-01T09:45:50.683 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:50 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:50.683 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:50 smithi190 ceph-mon[160435]: failed to encode map e540 with expected crc 2023-10-01T09:45:50.683 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:50 smithi190 ceph-mon[160435]: failed to encode map e540 with expected crc 2023-10-01T09:45:50.683 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:50 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:45:50.684 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:50 smithi190 ceph-mon[160435]: osdmap e540: 8 total, 8 up, 7 in 2023-10-01T09:45:50.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:50 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:45:50.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:50 smithi138 ceph-mon[185886]: failed to encode map e540 with expected crc 2023-10-01T09:45:50.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:50 smithi138 ceph-mon[185886]: failed to encode map e540 with expected crc 2023-10-01T09:45:50.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:50 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:45:50.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:50 smithi138 ceph-mon[185886]: osdmap e540: 8 total, 8 up, 7 in 2023-10-01T09:45:50.685 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:45:50.888 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:50.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:50.921 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:45:50.939 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:45:50.958 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:45:50.980 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:45:50.994 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:51.005 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:45:51.023 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:45:51.026 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:51.045 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:45:51.071 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:45:51.072 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:45:51.072 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:45:51.072 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:45:51.072 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:45:51.072 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:45:51.073 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:45:51.073 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:45:51.073 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:45:51.084 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:51.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:51.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:51.100 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:45:51.188 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:51.217 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:51.688 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:51 smithi190 ceph-mon[160435]: pgmap v1595: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 6579/148920 objects degraded (4.418%); 3024/148920 objects misplaced (2.031%); 440 KiB/s, 6 objects/s recovering 2023-10-01T09:45:51.689 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:51 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3141911023' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:51.764 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:51 smithi138 ceph-mon[192513]: pgmap v1595: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 6579/148920 objects degraded (4.418%); 3024/148920 objects misplaced (2.031%); 440 KiB/s, 6 objects/s recovering 2023-10-01T09:45:51.764 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:51 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3141911023' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:51.764 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:51 smithi138 ceph-mon[185886]: pgmap v1595: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 138 op/s; 6579/148920 objects degraded (4.418%); 3024/148920 objects misplaced (2.031%); 440 KiB/s, 6 objects/s recovering 2023-10-01T09:45:51.765 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:51 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3141911023' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:52.230 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:52.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:52.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:52.338 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:52.376 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:52.447 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:52.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:52.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:52.553 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:52.583 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:52.622 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:45:52.732 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:52.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:52.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:52.789 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:45:52.790 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:45:52.836 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:52.894 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:52.908 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:45:52.913 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:45:53.115 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:53.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:53.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:53.219 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:53.250 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:53.344 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:53.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:53.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:53.447 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:53.476 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:53.563 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:45:53.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:53 smithi138 ceph-mon[192513]: pgmap v1596: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 6515/146868 objects degraded (4.436%); 2973/146868 objects misplaced (2.024%); 809 KiB/s, 12 objects/s recovering 2023-10-01T09:45:53.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:53 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6515/146868 objects degraded (4.436%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:53.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:53 smithi138 ceph-mon[185886]: pgmap v1596: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 6515/146868 objects degraded (4.436%); 2973/146868 objects misplaced (2.024%); 809 KiB/s, 12 objects/s recovering 2023-10-01T09:45:53.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:53 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6515/146868 objects degraded (4.436%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:53.575 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:53.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:53.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:53.680 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:53.707 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:53.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:53 smithi190 ceph-mon[160435]: pgmap v1596: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 193 op/s; 6515/146868 objects degraded (4.436%); 2973/146868 objects misplaced (2.024%); 809 KiB/s, 12 objects/s recovering 2023-10-01T09:45:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:53 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6515/146868 objects degraded (4.436%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:53.815 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:53.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:53.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:53.854 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:53.872 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1596,"stamp":"2023-10-01T09:45:52.350750+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3211270091,"num_objects":48956,"num_object_clones":0,"num_object_copies":146868,"num_objects_missing_on_primary":1480,"num_objects_missing":1480,"num_objects_degraded":6515,"num_objects_misplaced":2973,"num_objects_unfound":0,"num_objects_dirty":48956,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":118618,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9662,"num_bytes_recovered":620069903,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116082,"ondisk_log_size":116082,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15073864,"kb_used_data":9988560,"kb_used_omap":39,"kb_used_meta":5085208,"kb_avail":641142200,"statfs":{"total":671965249536,"available":656529612800,"internally_reserved":0,"allocated":10228285440,"data_stored":10218340905,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5207253615},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,3,1,3,0,4,5,5],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-113639424,"num_objects":-1734,"num_object_clones":0,"num_object_copies":-5202,"num_objects_missing_on_primary":-113,"num_objects_missing":-113,"num_objects_degraded":-110,"num_objects_misplaced":-116,"num_objects_unfound":0,"num_objects_dirty":-1734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1734,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":113,"num_bytes_recovered":7405568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.942340"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":316,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.463708+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:45:43.463708+0000","last_peered":"2023-10-01T09:45:43.463708+0000","last_clean":"2023-10-01T09:45:43.463708+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:45:43.463708+0000","last_undegraded":"2023-10-01T09:45:43.463708+0000","last_fullsized":"2023-10-01T09:45:43.463708+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":160,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.471242+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:45:43.471242+0000","last_peered":"2023-10-01T09:45:43.471242+0000","last_clean":"2023-10-01T09:45:43.471242+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:45:43.471242+0000","last_undegraded":"2023-10-01T09:45:43.471242+0000","last_fullsized":"2023-10-01T09:45:43.471242+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":540,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.471109+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:45:43.471109+0000","last_peered":"2023-10-01T09:45:43.471109+0000","last_clean":"2023-10-01T09:45:43.471109+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:45:43.471109+0000","last_undegraded":"2023-10-01T09:45:43.471109+0000","last_fullsized":"2023-10-01T09:45:43.471109+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":599,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.415181+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:45:43.415181+0000","last_peered":"2023-10-01T09:45:43.415181+0000","last_clean":"2023-10-01T09:45:43.415181+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:45:43.415181+0000","last_undegraded":"2023-10-01T09:45:43.415181+0000","last_fullsized":"2023-10-01T09:45:43.415181+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":480,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.414271+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:45:43.414271+0000","last_peered":"2023-10-01T09:45:43.414271+0000","last_clean":"2023-10-01T09:45:43.414271+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:45:43.414271+0000","last_undegraded":"2023-10-01T09:45:43.414271+0000","last_fullsized":"2023-10-01T09:45:43.414271+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":221,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.472464+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:45:43.472464+0000","last_peered":"2023-10-01T09:45:43.472464+0000","last_clean":"2023-10-01T09:45:43.472464+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:45:43.472464+0000","last_undegraded":"2023-10-01T09:45:43.472464+0000","last_fullsized":"2023-10-01T09:45:43.472464+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":597,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.472573+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:45:43.472573+0000","last_peered":"2023-10-01T09:45:43.472573+0000","last_clean":"2023-10-01T09:45:43.472573+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:45:43.472573+0000","last_undegraded":"2023-10-01T09:45:43.472573+0000","last_fullsized":"2023-10-01T09:45:43.472573+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":526,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-10-01T09:45:34.490947+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:45:34.490947+0000","last_peered":"2023-10-01T09:45:34.490947+0000","last_clean":"2023-10-01T09:45:34.490947+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:45:34.490947+0000","last_undegraded":"2023-10-01T09:45:34.490947+0000","last_fullsized":"2023-10-01T09:45:34.490947+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":252,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.447607+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:45:43.447607+0000","last_peered":"2023-10-01T09:45:43.447607+0000","last_clean":"2023-10-01T09:45:43.447607+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:45:43.447607+0000","last_undegraded":"2023-10-01T09:45:43.447607+0000","last_fullsized":"2023-10-01T09:45:43.447607+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":59,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461974+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:45:43.461974+0000","last_peered":"2023-10-01T09:45:43.461974+0000","last_clean":"2023-10-01T09:45:43.461974+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:45:43.461974+0000","last_undegraded":"2023-10-01T09:45:43.461974+0000","last_fullsized":"2023-10-01T09:45:43.461974+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":65,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461301+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:45:43.461301+0000","last_peered":"2023-10-01T09:45:43.461301+0000","last_clean":"2023-10-01T09:45:43.461301+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:45:43.461301+0000","last_undegraded":"2023-10-01T09:45:43.461301+0000","last_fullsized":"2023-10-01T09:45:43.461301+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":64,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.470986+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:45:43.470986+0000","last_peered":"2023-10-01T09:45:43.470986+0000","last_clean":"2023-10-01T09:45:43.470986+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:45:43.470986+0000","last_undegraded":"2023-10-01T09:45:43.470986+0000","last_fullsized":"2023-10-01T09:45:43.470986+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":441,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.462787+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:45:43.462787+0000","last_peered":"2023-10-01T09:45:43.462787+0000","last_clean":"2023-10-01T09:45:43.462787+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:45:43.462787+0000","last_undegraded":"2023-10-01T09:45:43.462787+0000","last_fullsized":"2023-10-01T09:45:43.462787+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":322,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460961+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:45:43.460961+0000","last_peered":"2023-10-01T09:45:43.460961+0000","last_clean":"2023-10-01T09:45:43.460961+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:45:43.460961+0000","last_undegraded":"2023-10-01T09:45:43.460961+0000","last_fullsized":"2023-10-01T09:45:43.460961+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":658,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460497+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:45:43.460497+0000","last_peered":"2023-10-01T09:45:43.460497+0000","last_clean":"2023-10-01T09:45:43.460497+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:45:43.460497+0000","last_undegraded":"2023-10-01T09:45:43.460497+0000","last_fullsized":"2023-10-01T09:45:43.460497+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":395,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461889+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:45:43.461889+0000","last_peered":"2023-10-01T09:45:43.461889+0000","last_clean":"2023-10-01T09:45:43.461889+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:45:43.461889+0000","last_undegraded":"2023-10-01T09:45:43.461889+0000","last_fullsized":"2023-10-01T09:45:43.461889+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":68,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460310+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:45:43.460310+0000","last_peered":"2023-10-01T09:45:43.460310+0000","last_clean":"2023-10-01T09:45:43.460310+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:45:43.460310+0000","last_undegraded":"2023-10-01T09:45:43.460310+0000","last_fullsized":"2023-10-01T09:45:43.460310+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":54,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.447388+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:45:43.447388+0000","last_peered":"2023-10-01T09:45:43.447388+0000","last_clean":"2023-10-01T09:45:43.447388+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:45:43.447388+0000","last_undegraded":"2023-10-01T09:45:43.447388+0000","last_fullsized":"2023-10-01T09:45:43.447388+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":551,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.463361+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:45:43.463361+0000","last_peered":"2023-10-01T09:45:43.463361+0000","last_clean":"2023-10-01T09:45:43.463361+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:45:43.463361+0000","last_undegraded":"2023-10-01T09:45:43.463361+0000","last_fullsized":"2023-10-01T09:45:43.463361+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":563,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.415231+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:45:43.415231+0000","last_peered":"2023-10-01T09:45:43.415231+0000","last_clean":"2023-10-01T09:45:43.415231+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:45:43.415231+0000","last_undegraded":"2023-10-01T09:45:43.415231+0000","last_fullsized":"2023-10-01T09:45:43.415231+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":419,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.414170+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:45:43.414170+0000","last_peered":"2023-10-01T09:45:43.414170+0000","last_clean":"2023-10-01T09:45:43.414170+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:45:43.414170+0000","last_undegraded":"2023-10-01T09:45:43.414170+0000","last_fullsized":"2023-10-01T09:45:43.414170+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":68,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461054+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:45:43.461054+0000","last_peered":"2023-10-01T09:45:43.461054+0000","last_clean":"2023-10-01T09:45:43.461054+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:45:43.461054+0000","last_undegraded":"2023-10-01T09:45:43.461054+0000","last_fullsized":"2023-10-01T09:45:43.461054+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"540'7386","reported_seq":7459,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:52.112672+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:45:52.112672+0000","last_peered":"2023-10-01T09:45:52.112672+0000","last_clean":"2023-10-01T09:45:52.112672+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:45:52.112672+0000","last_undegraded":"2023-10-01T09:45:52.112672+0000","last_fullsized":"2023-10-01T09:45:52.112672+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7386,"log_dups_size":0,"ondisk_log_size":7386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197001216,"num_objects":3006,"num_object_clones":0,"num_object_copies":9018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7386,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":63,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460903+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:45:43.460903+0000","last_peered":"2023-10-01T09:45:43.460903+0000","last_clean":"2023-10-01T09:45:43.460903+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:45:43.460903+0000","last_undegraded":"2023-10-01T09:45:43.460903+0000","last_fullsized":"2023-10-01T09:45:43.460903+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":260,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.447003+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:45:43.447003+0000","last_peered":"2023-10-01T09:45:43.447003+0000","last_clean":"2023-10-01T09:45:43.447003+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:45:43.447003+0000","last_undegraded":"2023-10-01T09:45:43.447003+0000","last_fullsized":"2023-10-01T09:45:43.447003+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"540'7406","reported_seq":14119,"reported_epoch":540,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:45:52.210131+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:45:52.210131+0000","last_peered":"2023-10-01T09:45:52.210131+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:45:52.210131+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7406,"log_dups_size":0,"ondisk_log_size":7406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196608000,"num_objects":3000,"num_object_clones":0,"num_object_copies":9000,"num_objects_missing_on_primary":1480,"num_objects_missing":1480,"num_objects_degraded":4412,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7406,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2203,"num_bytes_recovered":144375808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1452},{"shards":"0,4,6","objects":1480},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":534,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460809+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:45:43.460809+0000","last_peered":"2023-10-01T09:45:43.460809+0000","last_clean":"2023-10-01T09:45:43.460809+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:45:43.460809+0000","last_undegraded":"2023-10-01T09:45:43.460809+0000","last_fullsized":"2023-10-01T09:45:43.460809+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":596,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.471014+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:45:43.471014+0000","last_peered":"2023-10-01T09:45:43.471014+0000","last_clean":"2023-10-01T09:45:43.471014+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:45:43.471014+0000","last_undegraded":"2023-10-01T09:45:43.471014+0000","last_fullsized":"2023-10-01T09:45:43.471014+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"540'7363","reported_seq":7439,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:51.736591+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:45:51.736591+0000","last_peered":"2023-10-01T09:45:51.736591+0000","last_clean":"2023-10-01T09:45:51.736591+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:45:51.736591+0000","last_undegraded":"2023-10-01T09:45:51.736591+0000","last_fullsized":"2023-10-01T09:45:51.736591+0000","mapping_epoch":475,"log_start":"481'300","ondisk_log_start":"481'300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7063,"log_dups_size":0,"ondisk_log_size":7063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201129984,"num_objects":3069,"num_object_clones":0,"num_object_copies":9207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7363,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":620,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461389+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:45:43.461389+0000","last_peered":"2023-10-01T09:45:43.461389+0000","last_clean":"2023-10-01T09:45:43.461389+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:45:43.461389+0000","last_undegraded":"2023-10-01T09:45:43.461389+0000","last_fullsized":"2023-10-01T09:45:43.461389+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":416,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.471101+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:45:43.471101+0000","last_peered":"2023-10-01T09:45:43.471101+0000","last_clean":"2023-10-01T09:45:43.471101+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:45:43.471101+0000","last_undegraded":"2023-10-01T09:45:43.471101+0000","last_fullsized":"2023-10-01T09:45:43.471101+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"540'7390","reported_seq":14071,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:51.661429+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:45:51.661429+0000","last_peered":"2023-10-01T09:45:51.661429+0000","last_clean":"2023-10-01T09:45:51.661429+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:45:51.661429+0000","last_undegraded":"2023-10-01T09:45:51.661429+0000","last_fullsized":"2023-10-01T09:45:51.661429+0000","mapping_epoch":505,"log_start":"481'400","ondisk_log_start":"481'400","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6990,"log_dups_size":0,"ondisk_log_size":6990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197394432,"num_objects":3012,"num_object_clones":0,"num_object_copies":9036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7390,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":260,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.447239+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:45:43.447239+0000","last_peered":"2023-10-01T09:45:43.447239+0000","last_clean":"2023-10-01T09:45:43.447239+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:45:43.447239+0000","last_undegraded":"2023-10-01T09:45:43.447239+0000","last_fullsized":"2023-10-01T09:45:43.447239+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":519,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.414374+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:45:43.414374+0000","last_peered":"2023-10-01T09:45:43.414374+0000","last_clean":"2023-10-01T09:45:43.414374+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:45:43.414374+0000","last_undegraded":"2023-10-01T09:45:43.414374+0000","last_fullsized":"2023-10-01T09:45:43.414374+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"540'7387","reported_seq":7463,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:50.084055+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:45:50.084055+0000","last_peered":"2023-10-01T09:45:50.084055+0000","last_clean":"2023-10-01T09:45:50.084055+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:45:50.084055+0000","last_undegraded":"2023-10-01T09:45:50.084055+0000","last_fullsized":"2023-10-01T09:45:50.084055+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7387,"log_dups_size":0,"ondisk_log_size":7387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204537856,"num_objects":3121,"num_object_clones":0,"num_object_copies":9363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7387,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":120,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461297+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:45:43.461297+0000","last_peered":"2023-10-01T09:45:43.461297+0000","last_clean":"2023-10-01T09:45:43.461297+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:45:43.461297+0000","last_undegraded":"2023-10-01T09:45:43.461297+0000","last_fullsized":"2023-10-01T09:45:43.461297+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":194,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.462910+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:45:43.462910+0000","last_peered":"2023-10-01T09:45:43.462910+0000","last_clean":"2023-10-01T09:45:43.462910+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:45:43.462910+0000","last_undegraded":"2023-10-01T09:45:43.462910+0000","last_fullsized":"2023-10-01T09:45:43.462910+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"540'7331","reported_seq":7522,"reported_epoch":540,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:45:52.047531+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:45:52.047531+0000","last_peered":"2023-10-01T09:45:52.047531+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:45:52.047531+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7331,"log_dups_size":0,"ondisk_log_size":7331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199032832,"num_objects":3037,"num_object_clones":0,"num_object_copies":9111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2093,"num_objects_misplaced":2973,"num_objects_unfound":0,"num_objects_dirty":3037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7331,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":3604480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2973},{"shards":"4,5","objects":-2029}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":438,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.463858+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:45:43.463858+0000","last_peered":"2023-10-01T09:45:43.463858+0000","last_clean":"2023-10-01T09:45:43.463858+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:45:43.463858+0000","last_undegraded":"2023-10-01T09:45:43.463858+0000","last_fullsized":"2023-10-01T09:45:43.463858+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":599,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.471272+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:45:43.471272+0000","last_peered":"2023-10-01T09:45:43.471272+0000","last_clean":"2023-10-01T09:45:43.471272+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:45:43.471272+0000","last_undegraded":"2023-10-01T09:45:43.471272+0000","last_fullsized":"2023-10-01T09:45:43.471272+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"539'7271","reported_seq":7352,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:47.269822+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:45:47.269822+0000","last_peered":"2023-10-01T09:45:47.269822+0000","last_clean":"2023-10-01T09:45:47.269822+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:45:47.269822+0000","last_undegraded":"2023-10-01T09:45:47.269822+0000","last_fullsized":"2023-10-01T09:45:47.269822+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7271,"log_dups_size":0,"ondisk_log_size":7271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203620352,"num_objects":3107,"num_object_clones":0,"num_object_copies":9321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7271,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":434,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.463200+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:45:43.463200+0000","last_peered":"2023-10-01T09:45:43.463200+0000","last_clean":"2023-10-01T09:45:43.463200+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:45:43.463200+0000","last_undegraded":"2023-10-01T09:45:43.463200+0000","last_fullsized":"2023-10-01T09:45:43.463200+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":275,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460967+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:45:43.460967+0000","last_peered":"2023-10-01T09:45:43.460967+0000","last_clean":"2023-10-01T09:45:43.460967+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:45:43.460967+0000","last_undegraded":"2023-10-01T09:45:43.460967+0000","last_fullsized":"2023-10-01T09:45:43.460967+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"540'7426","reported_seq":7499,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:50.794297+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:45:50.794297+0000","last_peered":"2023-10-01T09:45:50.794297+0000","last_clean":"2023-10-01T09:45:50.794297+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:45:50.794297+0000","last_undegraded":"2023-10-01T09:45:50.794297+0000","last_fullsized":"2023-10-01T09:45:50.794297+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7426,"log_dups_size":0,"ondisk_log_size":7426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201326592,"num_objects":3072,"num_object_clones":0,"num_object_copies":9216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7426,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":403,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.473212+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:45:43.473212+0000","last_peered":"2023-10-01T09:45:43.473212+0000","last_clean":"2023-10-01T09:45:43.473212+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:45:43.473212+0000","last_undegraded":"2023-10-01T09:45:43.473212+0000","last_fullsized":"2023-10-01T09:45:43.473212+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":451,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.471413+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:45:43.471413+0000","last_peered":"2023-10-01T09:45:43.471413+0000","last_clean":"2023-10-01T09:45:43.471413+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:45:43.471413+0000","last_undegraded":"2023-10-01T09:45:43.471413+0000","last_fullsized":"2023-10-01T09:45:43.471413+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"540'7203","reported_seq":7276,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:51.005065+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:45:51.005065+0000","last_peered":"2023-10-01T09:45:51.005065+0000","last_clean":"2023-10-01T09:45:51.005065+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:45:51.005065+0000","last_undegraded":"2023-10-01T09:45:51.005065+0000","last_fullsized":"2023-10-01T09:45:51.005065+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7203,"log_dups_size":0,"ondisk_log_size":7203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194969600,"num_objects":2975,"num_object_clones":0,"num_object_copies":8925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7203,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":933,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461748+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:45:43.461748+0000","last_peered":"2023-10-01T09:45:43.461748+0000","last_clean":"2023-10-01T09:45:43.461748+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:45:43.461748+0000","last_undegraded":"2023-10-01T09:45:43.461748+0000","last_fullsized":"2023-10-01T09:45:43.461748+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":270,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.447472+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:45:43.447472+0000","last_peered":"2023-10-01T09:45:43.447472+0000","last_clean":"2023-10-01T09:45:43.447472+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:45:43.447472+0000","last_undegraded":"2023-10-01T09:45:43.447472+0000","last_fullsized":"2023-10-01T09:45:43.447472+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":64,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.471022+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:45:43.471022+0000","last_peered":"2023-10-01T09:45:43.471022+0000","last_clean":"2023-10-01T09:45:43.471022+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:45:43.471022+0000","last_undegraded":"2023-10-01T09:45:43.471022+0000","last_fullsized":"2023-10-01T09:45:43.471022+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"540'7286","reported_seq":7362,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:51.513994+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:45:51.513994+0000","last_peered":"2023-10-01T09:45:51.513994+0000","last_clean":"2023-10-01T09:45:51.513994+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:45:51.513994+0000","last_undegraded":"2023-10-01T09:45:51.513994+0000","last_fullsized":"2023-10-01T09:45:51.513994+0000","mapping_epoch":475,"log_start":"481'300","ondisk_log_start":"481'300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6986,"log_dups_size":0,"ondisk_log_size":6986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202113024,"num_objects":3084,"num_object_clones":0,"num_object_copies":9252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7286,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1205,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.447111+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:45:43.447111+0000","last_peered":"2023-10-01T09:45:43.447111+0000","last_clean":"2023-10-01T09:45:43.447111+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:45:43.447111+0000","last_undegraded":"2023-10-01T09:45:43.447111+0000","last_fullsized":"2023-10-01T09:45:43.447111+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":415,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461155+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:45:43.461155+0000","last_peered":"2023-10-01T09:45:43.461155+0000","last_clean":"2023-10-01T09:45:43.461155+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:45:43.461155+0000","last_undegraded":"2023-10-01T09:45:43.461155+0000","last_fullsized":"2023-10-01T09:45:43.461155+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":243,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.447347+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:45:43.447347+0000","last_peered":"2023-10-01T09:45:43.447347+0000","last_clean":"2023-10-01T09:45:43.447347+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:45:43.447347+0000","last_undegraded":"2023-10-01T09:45:43.447347+0000","last_fullsized":"2023-10-01T09:45:43.447347+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"540'7366","reported_seq":14660,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:51.746713+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:45:51.746713+0000","last_peered":"2023-10-01T09:45:51.746713+0000","last_clean":"2023-10-01T09:45:51.746713+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:45:51.746713+0000","last_undegraded":"2023-10-01T09:45:51.746713+0000","last_fullsized":"2023-10-01T09:45:51.746713+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7366,"log_dups_size":0,"ondisk_log_size":7366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197263360,"num_objects":3010,"num_object_clones":0,"num_object_copies":9030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7366,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":886,"reported_epoch":539,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:45:43.413993+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:45:43.413993+0000","last_peered":"2023-10-01T09:45:43.413993+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:45:43.413993+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:45:43.413993+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":603,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.473213+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:45:43.473213+0000","last_peered":"2023-10-01T09:45:43.473213+0000","last_clean":"2023-10-01T09:45:43.473213+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:45:43.473213+0000","last_undegraded":"2023-10-01T09:45:43.473213+0000","last_fullsized":"2023-10-01T09:45:43.473213+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":64,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.471527+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:45:43.471527+0000","last_peered":"2023-10-01T09:45:43.471527+0000","last_clean":"2023-10-01T09:45:43.471527+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:45:43.471527+0000","last_undegraded":"2023-10-01T09:45:43.471527+0000","last_fullsized":"2023-10-01T09:45:43.471527+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"540'7212","reported_seq":7288,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.808472+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:45:49.808472+0000","last_peered":"2023-10-01T09:45:49.808472+0000","last_clean":"2023-10-01T09:45:49.808472+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:45:49.808472+0000","last_undegraded":"2023-10-01T09:45:49.808472+0000","last_fullsized":"2023-10-01T09:45:49.808472+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7212,"log_dups_size":0,"ondisk_log_size":7212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200540160,"num_objects":3060,"num_object_clones":0,"num_object_copies":9180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7212,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1656,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.447190+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:45:43.447190+0000","last_peered":"2023-10-01T09:45:43.447190+0000","last_clean":"2023-10-01T09:45:43.447190+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:45:43.447190+0000","last_undegraded":"2023-10-01T09:45:43.447190+0000","last_fullsized":"2023-10-01T09:45:43.447190+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":570,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.462196+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:45:43.462196+0000","last_peered":"2023-10-01T09:45:43.462196+0000","last_clean":"2023-10-01T09:45:43.462196+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:45:43.462196+0000","last_undegraded":"2023-10-01T09:45:43.462196+0000","last_fullsized":"2023-10-01T09:45:43.462196+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":348,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.447152+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:45:43.447152+0000","last_peered":"2023-10-01T09:45:43.447152+0000","last_clean":"2023-10-01T09:45:43.447152+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:45:43.447152+0000","last_undegraded":"2023-10-01T09:45:43.447152+0000","last_fullsized":"2023-10-01T09:45:43.447152+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":338,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460456+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:45:43.460456+0000","last_peered":"2023-10-01T09:45:43.460456+0000","last_clean":"2023-10-01T09:45:43.460456+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:45:43.460456+0000","last_undegraded":"2023-10-01T09:45:43.460456+0000","last_fullsized":"2023-10-01T09:45:43.460456+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"540'7417","reported_seq":7490,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:50.995709+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:45:50.995709+0000","last_peered":"2023-10-01T09:45:50.995709+0000","last_clean":"2023-10-01T09:45:50.995709+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:45:50.995709+0000","last_undegraded":"2023-10-01T09:45:50.995709+0000","last_fullsized":"2023-10-01T09:45:50.995709+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7417,"log_dups_size":0,"ondisk_log_size":7417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204144640,"num_objects":3115,"num_object_clones":0,"num_object_copies":9345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7417,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":747,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.462366+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:45:43.462366+0000","last_peered":"2023-10-01T09:45:43.462366+0000","last_clean":"2023-10-01T09:45:43.462366+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:45:43.462366+0000","last_undegraded":"2023-10-01T09:45:43.462366+0000","last_fullsized":"2023-10-01T09:45:43.462366+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":580,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.414269+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:45:43.414269+0000","last_peered":"2023-10-01T09:45:43.414269+0000","last_clean":"2023-10-01T09:45:43.414269+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:45:43.414269+0000","last_undegraded":"2023-10-01T09:45:43.414269+0000","last_fullsized":"2023-10-01T09:45:43.414269+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":580,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.415106+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:45:43.415106+0000","last_peered":"2023-10-01T09:45:43.415106+0000","last_clean":"2023-10-01T09:45:43.415106+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:45:43.415106+0000","last_undegraded":"2023-10-01T09:45:43.415106+0000","last_fullsized":"2023-10-01T09:45:43.415106+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"540'7363","reported_seq":7440,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:51.547750+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:45:51.547750+0000","last_peered":"2023-10-01T09:45:51.547750+0000","last_clean":"2023-10-01T09:45:51.547750+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:45:51.547750+0000","last_undegraded":"2023-10-01T09:45:51.547750+0000","last_fullsized":"2023-10-01T09:45:51.547750+0000","mapping_epoch":475,"log_start":"481'300","ondisk_log_start":"481'300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7063,"log_dups_size":0,"ondisk_log_size":7063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202899480,"num_objects":3097,"num_object_clones":0,"num_object_copies":9291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3097,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7363,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":978,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461469+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:45:43.461469+0000","last_peered":"2023-10-01T09:45:43.461469+0000","last_clean":"2023-10-01T09:45:43.461469+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:45:43.461469+0000","last_undegraded":"2023-10-01T09:45:43.461469+0000","last_fullsized":"2023-10-01T09:45:43.461469+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":597,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.447245+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:45:43.447245+0000","last_peered":"2023-10-01T09:45:43.447245+0000","last_clean":"2023-10-01T09:45:43.447245+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:45:43.447245+0000","last_undegraded":"2023-10-01T09:45:43.447245+0000","last_fullsized":"2023-10-01T09:45:43.447245+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":450,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.447274+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:45:43.447274+0000","last_peered":"2023-10-01T09:45:43.447274+0000","last_clean":"2023-10-01T09:45:43.447274+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:45:43.447274+0000","last_undegraded":"2023-10-01T09:45:43.447274+0000","last_fullsized":"2023-10-01T09:45:43.447274+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"540'7308","reported_seq":7389,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:51.037535+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:45:51.037535+0000","last_peered":"2023-10-01T09:45:51.037535+0000","last_clean":"2023-10-01T09:45:51.037535+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:45:51.037535+0000","last_undegraded":"2023-10-01T09:45:51.037535+0000","last_fullsized":"2023-10-01T09:45:51.037535+0000","mapping_epoch":479,"log_start":"481'600","ondisk_log_start":"481'600","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6708,"log_dups_size":0,"ondisk_log_size":6708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199753728,"num_objects":3048,"num_object_clones":0,"num_object_copies":9144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7308,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":479,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461139+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:45:43.461139+0000","last_peered":"2023-10-01T09:45:43.461139+0000","last_clean":"2023-10-01T09:45:43.461139+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:45:43.461139+0000","last_undegraded":"2023-10-01T09:45:43.461139+0000","last_fullsized":"2023-10-01T09:45:43.461139+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1173,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.471408+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:45:43.471408+0000","last_peered":"2023-10-01T09:45:43.471408+0000","last_clean":"2023-10-01T09:45:43.471408+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:45:43.471408+0000","last_undegraded":"2023-10-01T09:45:43.471408+0000","last_fullsized":"2023-10-01T09:45:43.471408+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.462113+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:45:43.462113+0000","last_peered":"2023-10-01T09:45:43.462113+0000","last_clean":"2023-10-01T09:45:43.462113+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:45:43.462113+0000","last_undegraded":"2023-10-01T09:45:43.462113+0000","last_fullsized":"2023-10-01T09:45:43.462113+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"540'7397","reported_seq":7478,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:51.083437+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:45:51.083437+0000","last_peered":"2023-10-01T09:45:51.083437+0000","last_clean":"2023-10-01T09:45:51.083437+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:45:51.083437+0000","last_undegraded":"2023-10-01T09:45:51.083437+0000","last_fullsized":"2023-10-01T09:45:51.083437+0000","mapping_epoch":479,"log_start":"481'700","ondisk_log_start":"481'700","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204144640,"num_objects":3115,"num_object_clones":0,"num_object_copies":9345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7397,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":72,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460982+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:45:43.460982+0000","last_peered":"2023-10-01T09:45:43.460982+0000","last_clean":"2023-10-01T09:45:43.460982+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:45:43.460982+0000","last_undegraded":"2023-10-01T09:45:43.460982+0000","last_fullsized":"2023-10-01T09:45:43.460982+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":558,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.415160+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:45:43.415160+0000","last_peered":"2023-10-01T09:45:43.415160+0000","last_clean":"2023-10-01T09:45:43.415160+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:45:43.415160+0000","last_undegraded":"2023-10-01T09:45:43.415160+0000","last_fullsized":"2023-10-01T09:45:43.415160+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":263,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.447079+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:45:43.447079+0000","last_peered":"2023-10-01T09:45:43.447079+0000","last_clean":"2023-10-01T09:45:43.447079+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:45:43.447079+0000","last_undegraded":"2023-10-01T09:45:43.447079+0000","last_fullsized":"2023-10-01T09:45:43.447079+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":506,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.414528+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:45:43.414528+0000","last_peered":"2023-10-01T09:45:43.414528+0000","last_clean":"2023-10-01T09:45:43.414528+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:45:43.414528+0000","last_undegraded":"2023-10-01T09:45:43.414528+0000","last_fullsized":"2023-10-01T09:45:43.414528+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":620,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.471198+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:45:43.471198+0000","last_peered":"2023-10-01T09:45:43.471198+0000","last_clean":"2023-10-01T09:45:43.471198+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:45:43.471198+0000","last_undegraded":"2023-10-01T09:45:43.471198+0000","last_fullsized":"2023-10-01T09:45:43.471198+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":275,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.447112+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:45:43.447112+0000","last_peered":"2023-10-01T09:45:43.447112+0000","last_clean":"2023-10-01T09:45:43.447112+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:45:43.447112+0000","last_undegraded":"2023-10-01T09:45:43.447112+0000","last_fullsized":"2023-10-01T09:45:43.447112+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":64,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:45.272168+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:45:45.272168+0000","last_peered":"2023-10-01T09:45:45.272168+0000","last_clean":"2023-10-01T09:45:45.272168+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:45:45.272168+0000","last_undegraded":"2023-10-01T09:45:45.272168+0000","last_fullsized":"2023-10-01T09:45:45.272168+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":275,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.447492+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:45:43.447492+0000","last_peered":"2023-10-01T09:45:43.447492+0000","last_clean":"2023-10-01T09:45:43.447492+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:45:43.447492+0000","last_undegraded":"2023-10-01T09:45:43.447492+0000","last_fullsized":"2023-10-01T09:45:43.447492+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":453,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.414502+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:45:43.414502+0000","last_peered":"2023-10-01T09:45:43.414502+0000","last_clean":"2023-10-01T09:45:43.414502+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:45:43.414502+0000","last_undegraded":"2023-10-01T09:45:43.414502+0000","last_fullsized":"2023-10-01T09:45:43.414502+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":81,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.471312+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:45:43.471312+0000","last_peered":"2023-10-01T09:45:43.471312+0000","last_clean":"2023-10-01T09:45:43.471312+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:45:43.471312+0000","last_undegraded":"2023-10-01T09:45:43.471312+0000","last_fullsized":"2023-10-01T09:45:43.471312+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":498,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.414162+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:45:43.414162+0000","last_peered":"2023-10-01T09:45:43.414162+0000","last_clean":"2023-10-01T09:45:43.414162+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:45:43.414162+0000","last_undegraded":"2023-10-01T09:45:43.414162+0000","last_fullsized":"2023-10-01T09:45:43.414162+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":697,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.415326+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:45:43.415326+0000","last_peered":"2023-10-01T09:45:43.415326+0000","last_clean":"2023-10-01T09:45:43.415326+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:45:43.415326+0000","last_undegraded":"2023-10-01T09:45:43.415326+0000","last_fullsized":"2023-10-01T09:45:43.415326+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":391,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461164+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:45:43.461164+0000","last_peered":"2023-10-01T09:45:43.461164+0000","last_clean":"2023-10-01T09:45:43.461164+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:45:43.461164+0000","last_undegraded":"2023-10-01T09:45:43.461164+0000","last_fullsized":"2023-10-01T09:45:43.461164+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3206479896,"num_objects":48928,"num_object_clones":0,"num_object_copies":146784,"num_objects_missing_on_primary":1480,"num_objects_missing":1480,"num_objects_degraded":6505,"num_objects_misplaced":2973,"num_objects_unfound":0,"num_objects_dirty":48928,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":117512,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9151,"num_bytes_recovered":599719936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10161631232,"data_stored":10161619016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114912,"ondisk_log_size":114912,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430743,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1882160,"kb_used_data":983144,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":91862992,"statfs":{"total":95995035648,"available":94067703808,"internally_reserved":0,"allocated":1006739456,"data_stored":1005797216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.79300000000000004,"15min":2.6280000000000001},"min":{"1min":0.254,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":3.3849999999999998,"5min":6.8550000000000004,"15min":108.90000000000001},"last":0.377},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.626,"15min":2.7559999999999998},"min":{"1min":0.25600000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.98399999999999999,"5min":5.6639999999999997,"15min":109.187},"last":1.2090000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":2.843,"15min":5.2329999999999997},"min":{"1min":0.314,"5min":0.248,"15min":0.222},"max":{"1min":3.4089999999999998,"5min":229.161,"15min":229.161},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.49099999999999999,"5min":2.7109999999999999,"15min":5.2290000000000001},"min":{"1min":0.29599999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":1.427,"5min":230.126,"15min":230.126},"last":1.123}]},{"osd":2,"last update":"Sun Oct 1 09:44:54 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.81399999999999995,"15min":1.6519999999999999},"min":{"1min":0.30399999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.4300000000000002,"5min":5.3090000000000002,"15min":14.901999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.76100000000000001,"15min":1.6299999999999999},"min":{"1min":0.33200000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4630000000000001,"5min":5.6929999999999996,"15min":14.682},"last":1.0009999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.79200000000000004,"15min":2.2810000000000001},"min":{"1min":0.32600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.3149999999999999,"5min":5.7750000000000004,"15min":76.555000000000007},"last":2.476},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.82499999999999996,"15min":2.2709999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.8479999999999999,"5min":6.548,"15min":76.534000000000006},"last":2.4529999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.80200000000000005,"15min":1.0409999999999999},"min":{"1min":0.31,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":3.4609999999999999,"5min":6.8769999999999998,"15min":9.6020000000000003},"last":1.276},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.82599999999999996,"15min":0.999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28000000000000003},"max":{"1min":0.63900000000000001,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.97399999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":2.0960000000000001,"15min":1.548},"min":{"1min":0.315,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":0.80000000000000004,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.56499999999999995,"5min":2.0830000000000002,"15min":1.6459999999999999},"min":{"1min":0.35699999999999998,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":2.8849999999999998,"5min":135.339,"15min":135.339},"last":0.502}]},{"osd":6,"last update":"Sun Oct 1 09:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.075,"15min":1.133},"min":{"1min":0.38,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":0.754,"5min":11.58,"15min":11.853999999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.86699999999999999,"15min":1.111},"min":{"1min":0.35599999999999998,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":3.7810000000000001,"5min":13.021000000000001,"15min":14.584},"last":0.629}]}]},{"osd":6,"up_from":294,"seq":1262720385253,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1733284,"kb_used_data":1399772,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":92011868,"statfs":{"total":95995035648,"available":94220152832,"internally_reserved":0,"allocated":1433366528,"data_stored":1432428563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":1.865,"15min":3.9809999999999999},"min":{"1min":0.27100000000000002,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":0.80200000000000005,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":2.0640000000000001,"15min":4.0209999999999999},"min":{"1min":0.311,"5min":0.245,"15min":0.221},"max":{"1min":2.1760000000000002,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.52200000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":2.9470000000000001,"15min":4.8600000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.24199999999999999},"max":{"1min":1.9059999999999999,"5min":168.947,"15min":168.947},"last":0.497},{"interface":"front","average":{"1min":0.44400000000000001,"5min":2.9980000000000002,"15min":4.8109999999999999},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":0.59899999999999998,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.46200000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.75600000000000001,"15min":1.5249999999999999},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":1.9350000000000001,"5min":8.657,"15min":10.92},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.76000000000000001,"15min":1.542},"min":{"1min":0.32300000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.2649999999999999,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.48699999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.70599999999999996,"15min":2.0699999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.1909999999999998,"5min":4.149,"15min":89.753},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.875,"15min":2.0179999999999998},"min":{"1min":0.38500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.1379999999999999,"5min":8.5470000000000006,"15min":89.849999999999994},"last":0.46999999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":1.4510000000000001,"15min":1.0680000000000001},"min":{"1min":0.27200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":0.623,"5min":63.134,"15min":63.134},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.4219999999999999,"15min":1.0669999999999999},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":0.73499999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.40699999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:45:31 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.1599999999999999,"15min":1.0369999999999999},"min":{"1min":0.307,"5min":0.26700000000000002,"15min":0.251},"max":{"1min":0.82199999999999995,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.2849999999999999,"15min":1.155},"min":{"1min":0.32500000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.6800000000000002,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.50600000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.67800000000000005,"15min":1.272},"min":{"1min":0.40200000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.83699999999999997,"5min":3.492,"15min":94.245999999999995},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.87,"15min":1.379},"min":{"1min":0.42299999999999999,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":2.202,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.55400000000000005}]}]},{"osd":1,"up_from":94,"seq":403726926230,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1422188,"kb_used_data":604004,"kb_used_omap":7,"kb_used_meta":818168,"kb_avail":92322964,"statfs":{"total":95995035648,"available":94538715136,"internally_reserved":0,"allocated":618500096,"data_stored":616441198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":837804455},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7469999999999999,"5min":0.93400000000000005,"15min":4.101},"min":{"1min":0.189,"5min":0.186,"15min":0.182},"max":{"1min":32.735999999999997,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":2.7959999999999998,"5min":1.147,"15min":3.6240000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":32.591999999999999,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.27800000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.39800000000000002,"15min":2.2669999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.51100000000000001,"5min":2.8740000000000001,"15min":51.640000000000001},"last":0.309},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.61799999999999999,"15min":2.9500000000000002},"min":{"1min":0.245,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":2.9089999999999998,"5min":6.9710000000000001,"15min":154.197},"last":0.38100000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.752,"15min":3.7629999999999999},"min":{"1min":0.254,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":0.48899999999999999,"5min":18.503,"15min":200.56800000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.70799999999999996,"15min":4.484},"min":{"1min":0.26300000000000001,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":2.7360000000000002,"5min":16.515000000000001,"15min":200.63800000000001},"last":0.39200000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.73299999999999998,"15min":2.2160000000000002},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":2.9209999999999998,"5min":4.5789999999999997,"15min":154.101},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.58199999999999996,"15min":2.3420000000000001},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.56200000000000006,"5min":2.7290000000000001,"15min":154.56999999999999},"last":0.46500000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.246,"15min":2.2530000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.30199999999999999},"max":{"1min":1.696,"5min":59.561,"15min":154.22999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.1970000000000001,"15min":2.4649999999999999},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.307},"max":{"1min":0.67300000000000004,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.441}]},{"osd":6,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.99399999999999999,"15min":2.5409999999999999},"min":{"1min":0.33500000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.86299999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.502},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.96999999999999997,"15min":2.419},"min":{"1min":0.38200000000000001,"5min":0.34599999999999997,"15min":0.32700000000000001},"max":{"1min":2.194,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.432}]},{"osd":7,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.94999999999999996,"15min":2.5529999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.33900000000000002},"max":{"1min":2.331,"5min":9.2460000000000004,"15min":154.20699999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.82999999999999996,"15min":2.609},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":1.9750000000000001,"5min":5.0110000000000001,"15min":154.51599999999999},"last":0.54900000000000004}]}]},{"osd":0,"up_from":89,"seq":382252089765,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3560216,"kb_used_data":2628880,"kb_used_omap":8,"kb_used_meta":931319,"kb_avail":90184936,"statfs":{"total":95995035648,"available":92349374464,"internally_reserved":0,"allocated":2691973120,"data_stored":2689946275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":953671618},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.58499999999999996,"15min":4.0869999999999997},"min":{"1min":0.18099999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":4.2919999999999998,"5min":8.4939999999999998,"15min":216.821},"last":0.44},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.59099999999999997,"15min":4.8499999999999996},"min":{"1min":0.20999999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":3.9620000000000002,"5min":8.6229999999999993,"15min":217.02699999999999},"last":0.56599999999999995}]},{"osd":2,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.64600000000000002,"15min":1.9910000000000001},"min":{"1min":0.25,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":4.0110000000000001,"5min":4.9459999999999997,"15min":19.788},"last":1.3100000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.65500000000000003,"15min":2.0310000000000001},"min":{"1min":0.28499999999999998,"5min":0.27200000000000002,"15min":0.189},"max":{"1min":3.8719999999999999,"5min":3.8719999999999999,"15min":25.844999999999999},"last":1.3300000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.63100000000000001,"15min":3.3969999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.73799999999999999,"15min":3.27},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.9199999999999999,"5min":5.8419999999999996,"15min":222.227},"last":1.5589999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.341,"15min":2.028},"min":{"1min":0.34000000000000002,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":4.048,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.64700000000000002,"5min":1.3140000000000001,"15min":2.0880000000000001},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":4,"5min":47.854999999999997,"15min":62.073999999999998},"last":1.5189999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.76800000000000002,"15min":1.744},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":4.0350000000000001,"5min":5.8769999999999998,"15min":16.808},"last":1.5920000000000001},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.84499999999999997,"15min":1.6499999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.1269999999999998,"5min":4.8689999999999998,"15min":17.507000000000001},"last":0.68300000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.89600000000000002,"15min":1.738},"min":{"1min":0.38200000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.3129999999999997,"5min":5.2110000000000003,"15min":17.535},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.85799999999999998,"15min":1.8089999999999999},"min":{"1min":0.43099999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.2050000000000001,"5min":4.2640000000000002,"15min":15.651999999999999},"last":0.78900000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.86199999999999999,"15min":2.0579999999999998},"min":{"1min":0.375,"5min":0.374,"15min":0.374},"max":{"1min":4.3360000000000003,"5min":4.976,"15min":48.895000000000003},"last":0.751},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.88900000000000001,"15min":2.1179999999999999},"min":{"1min":0.439,"5min":0.371,"15min":0.35099999999999998},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.81200000000000006}]}]},{"osd":2,"up_from":0,"seq":425201762704,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599177,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1447724,"kb_used_data":611492,"kb_used_omap":3,"kb_used_meta":836220,"kb_avail":92297428,"statfs":{"total":95995035648,"available":94512566272,"internally_reserved":0,"allocated":626167808,"data_stored":624050785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":856290023},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":1.6839999999999999,"15min":4.843},"min":{"1min":0.20399999999999999,"5min":0.183,"15min":0.17899999999999999},"max":{"1min":0.877,"5min":64.504000000000005,"15min":137.34399999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.34599999999999997,"5min":1.6000000000000001,"15min":4.9329999999999998},"min":{"1min":0.20599999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.80200000000000005,"5min":64.448999999999998,"15min":137.73400000000001},"last":0.42899999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.67900000000000005,"15min":5.0720000000000001},"min":{"1min":0.252,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":0.60499999999999998,"5min":14.917999999999999,"15min":162.22999999999999},"last":0.25},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.60699999999999998,"15min":5.125},"min":{"1min":0.26400000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.57399999999999995,"5min":15.074,"15min":162.70699999999999},"last":0.77600000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.60099999999999998,"15min":1.6859999999999999},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.20799999999999999},"max":{"1min":0.99199999999999999,"5min":7.141,"15min":14.254},"last":0.379},{"interface":"front","average":{"1min":0.432,"5min":0.55900000000000005,"15min":1.77},"min":{"1min":0.27000000000000002,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":1.111,"5min":4.5049999999999999,"15min":15.523},"last":0.80400000000000005}]},{"osd":4,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":1.371,"15min":1.7769999999999999},"min":{"1min":0.33200000000000002,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":0.64200000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.874},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.4570000000000001,"15min":2.0409999999999999},"min":{"1min":0.35799999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.5609999999999999,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.40200000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.82599999999999996,"15min":2.1419999999999999},"min":{"1min":0.314,"5min":0.29399999999999998,"15min":0.27500000000000002},"max":{"1min":2.895,"5min":6.0659999999999998,"15min":28.117999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.76700000000000002,"15min":2.2450000000000001},"min":{"1min":0.34899999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.0720000000000001,"5min":7.1559999999999997,"15min":28.34},"last":0.57399999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:45:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.512,"15min":1.879},"min":{"1min":0.33900000000000002,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":2.9100000000000001,"5min":73.887,"15min":73.887},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.6559999999999999,"15min":1.9530000000000001},"min":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":1.1379999999999999,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.60499999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.96299999999999997,"15min":1.8360000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":3.0299999999999998,"5min":7.0620000000000003,"15min":13.085000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.86899999999999999,"15min":1.716},"min":{"1min":0.40000000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.0150000000000001,"5min":4.5810000000000004,"15min":13.191000000000001},"last":0.84699999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019428,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3387308,"kb_used_data":2488932,"kb_used_omap":0,"kb_used_meta":898368,"kb_avail":90357844,"statfs":{"total":95995035648,"available":92526432256,"internally_reserved":0,"allocated":2548666368,"data_stored":2547727634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919928832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.4950000000000001,"15min":4.1369999999999996},"min":{"1min":0.22500000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.9889999999999999,"5min":79.837000000000003,"15min":155.315},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.39500000000000002,"5min":1.6519999999999999,"15min":4.5110000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.77600000000000002,"5min":78.983000000000004,"15min":155.334},"last":0.41799999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":1.5489999999999999,"15min":6.298},"min":{"1min":0.33000000000000002,"5min":0.252,"15min":0.224},"max":{"1min":0.67700000000000005,"5min":77.953000000000003,"15min":190.61199999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.754,"5min":1.468,"15min":7.0140000000000002},"min":{"1min":0.35299999999999998,"5min":0.26700000000000002,"15min":0.25600000000000001},"max":{"1min":3.1339999999999999,"5min":77.962999999999994,"15min":253.273},"last":0.38500000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:44:50 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.77200000000000002,"15min":2.375},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.2329999999999997,"5min":5.702,"15min":170.45400000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.755,"15min":2.133},"min":{"1min":0.33100000000000002,"5min":0.309,"15min":0.25800000000000001},"max":{"1min":0.71099999999999997,"5min":5.6150000000000002,"15min":58.771000000000001},"last":0.497}]},{"osd":3,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.77200000000000002,"15min":2.1379999999999999},"min":{"1min":0.31,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.0299999999999998,"5min":5.5279999999999996,"15min":140.881},"last":0.628},{"interface":"front","average":{"1min":0.749,"5min":0.68300000000000005,"15min":2.327},"min":{"1min":0.312,"5min":0.312,"15min":0.311},"max":{"1min":3.161,"5min":3.786,"15min":140.911},"last":0.53000000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:45:39 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.9279999999999999,"15min":1.4870000000000001},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":2.9649999999999999,"5min":110.003,"15min":110.003},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.62,"5min":1.8779999999999999,"15min":1.3959999999999999},"min":{"1min":0.34999999999999998,"5min":0.29899999999999999,"15min":0.28899999999999998},"max":{"1min":2.9910000000000001,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.47699999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:45:14 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.76000000000000001,"15min":0.94199999999999995},"min":{"1min":0.32800000000000001,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":3.0539999999999998,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.79000000000000004,"15min":1.0569999999999999},"min":{"1min":0.35899999999999999,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":1.028,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.58199999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:44:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.65000000000000002,"15min":1.1399999999999999},"min":{"1min":0.36399999999999999,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":3.0830000000000002,"5min":3.8580000000000001,"15min":38.034999999999997},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.91600000000000004,"15min":1.4350000000000001},"min":{"1min":0.39900000000000002,"5min":0.36699999999999999,"15min":0.308},"max":{"1min":0.80000000000000004,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.61099999999999999}]}]},{"osd":5,"up_from":379,"seq":1627792605335,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1640984,"kb_used_data":1272336,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":92104168,"statfs":{"total":95995035648,"available":94314668032,"internally_reserved":0,"allocated":1302872064,"data_stored":1301949234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,3,1,3,0,4,5,5],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":1.004,"15min":2.157},"min":{"1min":0.28699999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.1400000000000001,"5min":27.584,"15min":84.340999999999994},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.97699999999999998,"15min":2.0910000000000002},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":4.2300000000000004,"5min":27.669,"15min":84.356999999999999},"last":0.41899999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":2.0430000000000001,"15min":5.6200000000000001},"min":{"1min":0.26600000000000001,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":4.3289999999999997,"5min":106.56999999999999,"15min":295.71300000000002},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.495,"5min":2.0630000000000002,"15min":5.6260000000000003},"min":{"1min":0.27200000000000002,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":2.169,"5min":106.596,"15min":299.59399999999999},"last":0.46500000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.77000000000000002,"15min":1.5089999999999999},"min":{"1min":0.28799999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.2670000000000003,"5min":5.3470000000000004,"15min":14.199999999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.89800000000000002,"15min":1.468},"min":{"1min":0.315,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":4.3719999999999999,"5min":5.782,"15min":14.430999999999999},"last":0.45400000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.71799999999999997,"15min":1.417},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.1880000000000002,"5min":4.0209999999999999,"15min":14.004},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.84399999999999997,"15min":1.5629999999999999},"min":{"1min":0.36599999999999999,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":4.2910000000000004,"5min":5.4400000000000004,"15min":13.991},"last":0.44600000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.86599999999999999,"15min":1.3959999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.1929999999999996,"5min":5.9459999999999997,"15min":122.971},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.02,"15min":1.605},"min":{"1min":0.30399999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.3869999999999996,"5min":12.445,"15min":122.89100000000001},"last":0.39700000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.86899999999999999,"15min":0.99399999999999999},"min":{"1min":0.314,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":2.2069999999999999,"5min":6.7750000000000004,"15min":12.545},"last":0.497},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.91200000000000003,"15min":1.2130000000000001},"min":{"1min":0.35199999999999998,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":4.3200000000000003,"5min":5.4160000000000004,"15min":14.193},"last":0.54800000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.0349999999999999,"15min":1.1890000000000001},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.3559999999999999,"5min":5.758,"15min":10.813000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.99399999999999999,"15min":1.0660000000000001},"min":{"1min":0.39400000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":2.226,"5min":5.2300000000000004,"15min":12.500999999999999},"last":0.65400000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2674135040,"data_stored":2674130968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":606863360,"data_stored":606863360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":619511808,"data_stored":619511808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2542866432,"data_stored":2542862360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1296433152,"data_stored":1296433152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1425215488,"data_stored":1425211416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":996605952,"data_stored":996605952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:45:53.879 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:45:53.919 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:53.976 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:45:53.983 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:54.081 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:54.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:54.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:54.184 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:54.225 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:54.253 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:45:54.284 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:54.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:54.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:54.388 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:54.420 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:54.473 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:54.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:54.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:54 smithi190 ceph-mon[160435]: from='client.47210 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:54.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:54 smithi138 ceph-mon[192513]: from='client.47210 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:54.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:54 smithi138 ceph-mon[185886]: from='client.47210 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:45:54.622 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:54.652 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:54.935 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:54.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:54.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:55.038 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:55.070 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:55.092 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:55.172 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:55.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:55.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:55.248 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:45:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:45:54] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.33.4" 2023-10-01T09:45:55.276 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:45:55.306 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:55.402 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:55.409 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:45:55.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:55.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:45:55.515 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:55.520 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:55 smithi138 ceph-mon[192513]: pgmap v1597: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 6515/146700 objects degraded (4.441%); 2973/146700 objects misplaced (2.027%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:45:55.520 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:45:55.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:55 smithi138 ceph-mon[185886]: pgmap v1597: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 6515/146700 objects degraded (4.441%); 2973/146700 objects misplaced (2.027%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:45:55.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:45:55.559 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:55.623 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:45:55.623 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:45:55.628 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:55.631 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:55 smithi190 ceph-mon[160435]: pgmap v1597: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 6515/146700 objects degraded (4.441%); 2973/146700 objects misplaced (2.027%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:45:55.631 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:45:55.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:55.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:55.733 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:55.764 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:56.351 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:45:56.369 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:56.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:56.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:56.464 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:45:56.473 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:45:56.501 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:56.554 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:45:56.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:56.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:45:56.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:56 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/433269348' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:56.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:56 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:56 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/433269348' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:56.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:56 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:56.660 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:56.687 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:56 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/433269348' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:56.687 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:56 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:45:56.689 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:56.737 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:56.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:56.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:56.760 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:45:56.842 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:45:56.874 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:57.146 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:45:57.146 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 6515/145671 objects degraded (4.472%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2177,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":541,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":48557,"data_bytes":3185121227,"bytes_used":15345946624,"bytes_avail":656619302912,"bytes_total":671965249536,"degraded_objects":6515,"degraded_total":145671,"degraded_ratio":0.044724069993341151,"misplaced_objects":2973,"misplaced_total":145671,"misplaced_ratio":0.020409003851144015,"recovering_objects_per_sec":11,"recovering_bytes_per_sec":740361,"recovering_keys_per_sec":0,"num_objects_recovered":113,"num_bytes_recovered":7405568,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":173},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":34,"modified":"2023-10-01T09:43:40.311181+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:45:57.279 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:45:57.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:57.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:45:57.383 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:57.416 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:57.445 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:57 smithi138 ceph-mon[185886]: pgmap v1598: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 173 op/s; 6515/145671 objects degraded (4.472%); 2973/145671 objects misplaced (2.041%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:45:57.445 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:57 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:45:57.445 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:57 smithi138 ceph-mon[185886]: failed to encode map e541 with expected crc 2023-10-01T09:45:57.446 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:57 smithi138 ceph-mon[185886]: failed to encode map e541 with expected crc 2023-10-01T09:45:57.446 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:57 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:45:57.446 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:57 smithi138 ceph-mon[185886]: osdmap e541: 8 total, 8 up, 7 in 2023-10-01T09:45:57.446 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/353895737' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:45:57.470 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:57.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:57.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:57.575 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:45:57.604 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:57.744 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:57 smithi138 ceph-mon[192513]: pgmap v1598: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 173 op/s; 6515/145671 objects degraded (4.472%); 2973/145671 objects misplaced (2.041%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:45:57.744 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:57 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:45:57.744 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:57 smithi138 ceph-mon[192513]: failed to encode map e541 with expected crc 2023-10-01T09:45:57.744 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:57 smithi138 ceph-mon[192513]: failed to encode map e541 with expected crc 2023-10-01T09:45:57.744 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:57 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:45:57.744 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:57 smithi138 ceph-mon[192513]: osdmap e541: 8 total, 8 up, 7 in 2023-10-01T09:45:57.744 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/353895737' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:45:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:57 smithi190 ceph-mon[160435]: pgmap v1598: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 173 op/s; 6515/145671 objects degraded (4.472%); 2973/145671 objects misplaced (2.041%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:45:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:57 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:45:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:57 smithi190 ceph-mon[160435]: failed to encode map e541 with expected crc 2023-10-01T09:45:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:57 smithi190 ceph-mon[160435]: failed to encode map e541 with expected crc 2023-10-01T09:45:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:57 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:45:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:57 smithi190 ceph-mon[160435]: osdmap e541: 8 total, 8 up, 7 in 2023-10-01T09:45:57.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/353895737' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:45:57.862 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:45:57.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:57.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:45:57.966 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:57.995 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:58.080 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:58.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:58.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:58.185 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:45:58.220 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:58.382 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:45:58.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:58.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:45:58.402 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:45:58.402 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:45:58.447 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:58 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6515/145671 objects degraded (4.472%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:58.458 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:45:58.458 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:45:58.470 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:58 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6515/145671 objects degraded (4.472%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:58.471 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:58 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6515/145671 objects degraded (4.472%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:45:58.486 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:58.533 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:58.647 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:58.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:58.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:58.704 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:45:58.725 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:45:58.750 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:45:58.752 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:58.771 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:45:58.778 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:58.791 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:45:58.820 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:58.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:58.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:58.843 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:45:58.924 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:58.926 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:45:58.951 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:45:58.958 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:58.976 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:45:58.977 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:45:58.977 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:45:58.977 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:45:58.978 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:45:58.978 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:45:58.978 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:45:58.978 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:45:58.978 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:45:59.075 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:59.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:59.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:45:59.180 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:45:59.207 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:59.277 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:45:59.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:59.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:45:59.338 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:45:59.383 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:45:59.414 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:59.479 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:45:59.486 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:59 smithi190 ceph-mon[160435]: pgmap v1600: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 186 op/s; 6445/144468 objects degraded (4.461%); 2929/144468 objects misplaced (2.027%); 876 KiB/s, 13 objects/s recovering 2023-10-01T09:45:59.487 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:45:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4167620985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:59.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:59.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:45:59.590 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:45:59.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:59 smithi138 ceph-mon[185886]: pgmap v1600: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 186 op/s; 6445/144468 objects degraded (4.461%); 2929/144468 objects misplaced (2.027%); 876 KiB/s, 13 objects/s recovering 2023-10-01T09:45:59.618 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:45:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4167620985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:59.618 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:59 smithi138 ceph-mon[192513]: pgmap v1600: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 186 op/s; 6445/144468 objects degraded (4.461%); 2929/144468 objects misplaced (2.027%); 876 KiB/s, 13 objects/s recovering 2023-10-01T09:45:59.618 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:45:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4167620985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:45:59.620 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:45:59.768 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:45:59.816 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:45:59.919 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:45:59.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:45:59.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:00.022 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:00.058 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:00.364 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:00.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:00.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:00.469 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:00.500 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:00.560 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:00.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:00.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:00.573 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:00.665 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:00.696 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:00.839 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:00.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:00.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:00.943 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:46:00.977 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:01.089 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:46:01.095 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:46:01.327 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:46:01.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:01.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:46:01.379 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:46:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:46:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 0 2023-10-01T09:46:01.380 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:46:01.431 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:01.460 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:01.504 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:01.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:01.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:01.602 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:01 smithi138 ceph-mon[185886]: pgmap v1601: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 160 op/s; 6445/144114 objects degraded (4.472%); 2929/144114 objects misplaced (2.032%); 749 KiB/s, 11 objects/s recovering 2023-10-01T09:46:01.603 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:01 smithi138 ceph-mon[192513]: pgmap v1601: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 160 op/s; 6445/144114 objects degraded (4.472%); 2929/144114 objects misplaced (2.032%); 749 KiB/s, 11 objects/s recovering 2023-10-01T09:46:01.608 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:01.638 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:01.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:01 smithi190 ceph-mon[160435]: pgmap v1601: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 160 op/s; 6445/144114 objects degraded (4.472%); 2929/144114 objects misplaced (2.032%); 749 KiB/s, 11 objects/s recovering 2023-10-01T09:46:02.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:46:02.140 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:46:02.189 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1601,"stamp":"2023-10-01T09:46:00.352998+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3151108043,"num_objects":48038,"num_object_clones":0,"num_object_copies":144114,"num_objects_missing_on_primary":1425,"num_objects_missing":1425,"num_objects_degraded":6445,"num_objects_misplaced":2929,"num_objects_unfound":0,"num_objects_dirty":48038,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":119536,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9721,"num_bytes_recovered":623936527,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116800,"ondisk_log_size":116800,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14909896,"kb_used_data":9816400,"kb_used_omap":39,"kb_used_meta":5093400,"kb_avail":641306168,"statfs":{"total":671965249536,"available":656697516032,"internally_reserved":0,"allocated":10051993600,"data_stored":10042039151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5215642223},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-104988672,"num_objects":-1602,"num_object_clones":0,"num_object_copies":-4806,"num_objects_missing_on_primary":-112,"num_objects_missing":-112,"num_objects_degraded":-134,"num_objects_misplaced":-95,"num_objects_unfound":0,"num_objects_dirty":-1602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1602,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":7667712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002687"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":316,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.463708+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:45:43.463708+0000","last_peered":"2023-10-01T09:45:43.463708+0000","last_clean":"2023-10-01T09:45:43.463708+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:45:43.463708+0000","last_undegraded":"2023-10-01T09:45:43.463708+0000","last_fullsized":"2023-10-01T09:45:43.463708+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":161,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.807727+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:45:49.807727+0000","last_peered":"2023-10-01T09:45:49.807727+0000","last_clean":"2023-10-01T09:45:49.807727+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:45:49.807727+0000","last_undegraded":"2023-10-01T09:45:49.807727+0000","last_fullsized":"2023-10-01T09:45:49.807727+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":541,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.807710+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:45:49.807710+0000","last_peered":"2023-10-01T09:45:49.807710+0000","last_clean":"2023-10-01T09:45:49.807710+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:45:49.807710+0000","last_undegraded":"2023-10-01T09:45:49.807710+0000","last_fullsized":"2023-10-01T09:45:49.807710+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":600,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.811680+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:45:49.811680+0000","last_peered":"2023-10-01T09:45:49.811680+0000","last_clean":"2023-10-01T09:45:49.811680+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:45:49.811680+0000","last_undegraded":"2023-10-01T09:45:49.811680+0000","last_fullsized":"2023-10-01T09:45:49.811680+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":481,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.810931+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:45:49.810931+0000","last_peered":"2023-10-01T09:45:49.810931+0000","last_clean":"2023-10-01T09:45:49.810931+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:45:49.810931+0000","last_undegraded":"2023-10-01T09:45:49.810931+0000","last_fullsized":"2023-10-01T09:45:49.810931+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":222,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.877997+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:45:49.877997+0000","last_peered":"2023-10-01T09:45:49.877997+0000","last_clean":"2023-10-01T09:45:49.877997+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:45:49.877997+0000","last_undegraded":"2023-10-01T09:45:49.877997+0000","last_fullsized":"2023-10-01T09:45:49.877997+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":598,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.891163+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:45:49.891163+0000","last_peered":"2023-10-01T09:45:49.891163+0000","last_clean":"2023-10-01T09:45:49.891163+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:45:49.891163+0000","last_undegraded":"2023-10-01T09:45:49.891163+0000","last_fullsized":"2023-10-01T09:45:49.891163+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":528,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.873074+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:45:49.873074+0000","last_peered":"2023-10-01T09:45:49.873074+0000","last_clean":"2023-10-01T09:45:49.873074+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:45:49.873074+0000","last_undegraded":"2023-10-01T09:45:49.873074+0000","last_fullsized":"2023-10-01T09:45:49.873074+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":253,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.836441+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:45:49.836441+0000","last_peered":"2023-10-01T09:45:49.836441+0000","last_clean":"2023-10-01T09:45:49.836441+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:45:49.836441+0000","last_undegraded":"2023-10-01T09:45:49.836441+0000","last_fullsized":"2023-10-01T09:45:49.836441+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":59,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461974+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:45:43.461974+0000","last_peered":"2023-10-01T09:45:43.461974+0000","last_clean":"2023-10-01T09:45:43.461974+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:45:43.461974+0000","last_undegraded":"2023-10-01T09:45:43.461974+0000","last_fullsized":"2023-10-01T09:45:43.461974+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":65,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461301+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:45:43.461301+0000","last_peered":"2023-10-01T09:45:43.461301+0000","last_clean":"2023-10-01T09:45:43.461301+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:45:43.461301+0000","last_undegraded":"2023-10-01T09:45:43.461301+0000","last_fullsized":"2023-10-01T09:45:43.461301+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":65,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.808049+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:45:49.808049+0000","last_peered":"2023-10-01T09:45:49.808049+0000","last_clean":"2023-10-01T09:45:49.808049+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:45:49.808049+0000","last_undegraded":"2023-10-01T09:45:49.808049+0000","last_fullsized":"2023-10-01T09:45:49.808049+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":441,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.462787+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:45:43.462787+0000","last_peered":"2023-10-01T09:45:43.462787+0000","last_clean":"2023-10-01T09:45:43.462787+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:45:43.462787+0000","last_undegraded":"2023-10-01T09:45:43.462787+0000","last_fullsized":"2023-10-01T09:45:43.462787+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":322,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460961+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:45:43.460961+0000","last_peered":"2023-10-01T09:45:43.460961+0000","last_clean":"2023-10-01T09:45:43.460961+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:45:43.460961+0000","last_undegraded":"2023-10-01T09:45:43.460961+0000","last_fullsized":"2023-10-01T09:45:43.460961+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":658,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460497+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:45:43.460497+0000","last_peered":"2023-10-01T09:45:43.460497+0000","last_clean":"2023-10-01T09:45:43.460497+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:45:43.460497+0000","last_undegraded":"2023-10-01T09:45:43.460497+0000","last_fullsized":"2023-10-01T09:45:43.460497+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":395,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461889+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:45:43.461889+0000","last_peered":"2023-10-01T09:45:43.461889+0000","last_clean":"2023-10-01T09:45:43.461889+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:45:43.461889+0000","last_undegraded":"2023-10-01T09:45:43.461889+0000","last_fullsized":"2023-10-01T09:45:43.461889+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":68,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460310+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:45:43.460310+0000","last_peered":"2023-10-01T09:45:43.460310+0000","last_clean":"2023-10-01T09:45:43.460310+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:45:43.460310+0000","last_undegraded":"2023-10-01T09:45:43.460310+0000","last_fullsized":"2023-10-01T09:45:43.460310+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":55,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.835976+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:45:49.835976+0000","last_peered":"2023-10-01T09:45:49.835976+0000","last_clean":"2023-10-01T09:45:49.835976+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:45:49.835976+0000","last_undegraded":"2023-10-01T09:45:49.835976+0000","last_fullsized":"2023-10-01T09:45:49.835976+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":551,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.463361+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:45:43.463361+0000","last_peered":"2023-10-01T09:45:43.463361+0000","last_clean":"2023-10-01T09:45:43.463361+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:45:43.463361+0000","last_undegraded":"2023-10-01T09:45:43.463361+0000","last_fullsized":"2023-10-01T09:45:43.463361+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":564,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.811717+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:45:49.811717+0000","last_peered":"2023-10-01T09:45:49.811717+0000","last_clean":"2023-10-01T09:45:49.811717+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:45:49.811717+0000","last_undegraded":"2023-10-01T09:45:49.811717+0000","last_fullsized":"2023-10-01T09:45:49.811717+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":420,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.810871+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:45:49.810871+0000","last_peered":"2023-10-01T09:45:49.810871+0000","last_clean":"2023-10-01T09:45:49.810871+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:45:49.810871+0000","last_undegraded":"2023-10-01T09:45:49.810871+0000","last_fullsized":"2023-10-01T09:45:49.810871+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":68,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461054+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:45:43.461054+0000","last_peered":"2023-10-01T09:45:43.461054+0000","last_clean":"2023-10-01T09:45:43.461054+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:45:43.461054+0000","last_undegraded":"2023-10-01T09:45:43.461054+0000","last_fullsized":"2023-10-01T09:45:43.461054+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"541'7436","reported_seq":7510,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:57.258483+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:45:57.258483+0000","last_peered":"2023-10-01T09:45:57.258483+0000","last_clean":"2023-10-01T09:45:57.258483+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:45:57.258483+0000","last_undegraded":"2023-10-01T09:45:57.258483+0000","last_fullsized":"2023-10-01T09:45:57.258483+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7436,"log_dups_size":0,"ondisk_log_size":7436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193724416,"num_objects":2956,"num_object_clones":0,"num_object_copies":8868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7436,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":63,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460903+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:45:43.460903+0000","last_peered":"2023-10-01T09:45:43.460903+0000","last_clean":"2023-10-01T09:45:43.460903+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:45:43.460903+0000","last_undegraded":"2023-10-01T09:45:43.460903+0000","last_fullsized":"2023-10-01T09:45:43.460903+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":261,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.835745+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:45:49.835745+0000","last_peered":"2023-10-01T09:45:49.835745+0000","last_clean":"2023-10-01T09:45:49.835745+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:45:49.835745+0000","last_undegraded":"2023-10-01T09:45:49.835745+0000","last_fullsized":"2023-10-01T09:45:49.835745+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"541'7464","reported_seq":14349,"reported_epoch":541,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:45:57.234118+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:45:57.234118+0000","last_peered":"2023-10-01T09:45:57.234118+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:45:57.234118+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7464,"log_dups_size":0,"ondisk_log_size":7464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192806912,"num_objects":2942,"num_object_clones":0,"num_object_copies":8826,"num_objects_missing_on_primary":1425,"num_objects_missing":1425,"num_objects_degraded":4299,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7464,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2261,"num_bytes_recovered":148176896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1449},{"shards":"0,4,6","objects":1425},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":534,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460809+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:45:43.460809+0000","last_peered":"2023-10-01T09:45:43.460809+0000","last_clean":"2023-10-01T09:45:43.460809+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:45:43.460809+0000","last_undegraded":"2023-10-01T09:45:43.460809+0000","last_fullsized":"2023-10-01T09:45:43.460809+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":597,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.807674+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:45:49.807674+0000","last_peered":"2023-10-01T09:45:49.807674+0000","last_clean":"2023-10-01T09:45:49.807674+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:45:49.807674+0000","last_undegraded":"2023-10-01T09:45:49.807674+0000","last_fullsized":"2023-10-01T09:45:49.807674+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"541'7406","reported_seq":7483,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.461991+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:45:56.461991+0000","last_peered":"2023-10-01T09:45:56.461991+0000","last_clean":"2023-10-01T09:45:56.461991+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:45:56.461991+0000","last_undegraded":"2023-10-01T09:45:56.461991+0000","last_fullsized":"2023-10-01T09:45:56.461991+0000","mapping_epoch":475,"log_start":"481'400","ondisk_log_start":"481'400","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7006,"log_dups_size":0,"ondisk_log_size":7006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198311936,"num_objects":3026,"num_object_clones":0,"num_object_copies":9078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7406,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":620,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461389+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:45:43.461389+0000","last_peered":"2023-10-01T09:45:43.461389+0000","last_clean":"2023-10-01T09:45:43.461389+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:45:43.461389+0000","last_undegraded":"2023-10-01T09:45:43.461389+0000","last_fullsized":"2023-10-01T09:45:43.461389+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":417,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.808157+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:45:49.808157+0000","last_peered":"2023-10-01T09:45:49.808157+0000","last_clean":"2023-10-01T09:45:49.808157+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:45:49.808157+0000","last_undegraded":"2023-10-01T09:45:49.808157+0000","last_fullsized":"2023-10-01T09:45:49.808157+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"541'7433","reported_seq":14115,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.589435+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:45:56.589435+0000","last_peered":"2023-10-01T09:45:56.589435+0000","last_clean":"2023-10-01T09:45:56.589435+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:45:56.589435+0000","last_undegraded":"2023-10-01T09:45:56.589435+0000","last_fullsized":"2023-10-01T09:45:56.589435+0000","mapping_epoch":505,"log_start":"481'400","ondisk_log_start":"481'400","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7033,"log_dups_size":0,"ondisk_log_size":7033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194576384,"num_objects":2969,"num_object_clones":0,"num_object_copies":8907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7433,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":261,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.836183+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:45:49.836183+0000","last_peered":"2023-10-01T09:45:49.836183+0000","last_clean":"2023-10-01T09:45:49.836183+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:45:49.836183+0000","last_undegraded":"2023-10-01T09:45:49.836183+0000","last_fullsized":"2023-10-01T09:45:49.836183+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":520,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.811091+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:45:49.811091+0000","last_peered":"2023-10-01T09:45:49.811091+0000","last_clean":"2023-10-01T09:45:49.811091+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:45:49.811091+0000","last_undegraded":"2023-10-01T09:45:49.811091+0000","last_fullsized":"2023-10-01T09:45:49.811091+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"541'7491","reported_seq":7568,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:46:00.217984+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:46:00.217984+0000","last_peered":"2023-10-01T09:46:00.217984+0000","last_clean":"2023-10-01T09:46:00.217984+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:46:00.217984+0000","last_undegraded":"2023-10-01T09:46:00.217984+0000","last_fullsized":"2023-10-01T09:46:00.217984+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7491,"log_dups_size":0,"ondisk_log_size":7491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197722112,"num_objects":3017,"num_object_clones":0,"num_object_copies":9051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7491,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":120,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461297+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:45:43.461297+0000","last_peered":"2023-10-01T09:45:43.461297+0000","last_clean":"2023-10-01T09:45:43.461297+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:45:43.461297+0000","last_undegraded":"2023-10-01T09:45:43.461297+0000","last_fullsized":"2023-10-01T09:45:43.461297+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":194,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.462910+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:45:43.462910+0000","last_peered":"2023-10-01T09:45:43.462910+0000","last_clean":"2023-10-01T09:45:43.462910+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:45:43.462910+0000","last_undegraded":"2023-10-01T09:45:43.462910+0000","last_fullsized":"2023-10-01T09:45:43.462910+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"541'7375","reported_seq":7569,"reported_epoch":541,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:45:57.238082+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:45:57.238082+0000","last_peered":"2023-10-01T09:45:57.238082+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:45:57.238082+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7375,"log_dups_size":0,"ondisk_log_size":7375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196149248,"num_objects":2993,"num_object_clones":0,"num_object_copies":8979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2136,"num_objects_misplaced":2929,"num_objects_unfound":0,"num_objects_dirty":2993,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7375,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":56,"num_bytes_recovered":3670016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2929},{"shards":"4,5","objects":-2072}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":438,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.463858+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:45:43.463858+0000","last_peered":"2023-10-01T09:45:43.463858+0000","last_clean":"2023-10-01T09:45:43.463858+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:45:43.463858+0000","last_undegraded":"2023-10-01T09:45:43.463858+0000","last_fullsized":"2023-10-01T09:45:43.463858+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":600,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.808005+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:45:49.808005+0000","last_peered":"2023-10-01T09:45:49.808005+0000","last_clean":"2023-10-01T09:45:49.808005+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:45:49.808005+0000","last_undegraded":"2023-10-01T09:45:49.808005+0000","last_fullsized":"2023-10-01T09:45:49.808005+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"541'7373","reported_seq":7456,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:57.500790+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:45:57.500790+0000","last_peered":"2023-10-01T09:45:57.500790+0000","last_clean":"2023-10-01T09:45:57.500790+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:45:57.500790+0000","last_undegraded":"2023-10-01T09:45:57.500790+0000","last_fullsized":"2023-10-01T09:45:57.500790+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7373,"log_dups_size":0,"ondisk_log_size":7373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196935680,"num_objects":3005,"num_object_clones":0,"num_object_copies":9015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7373,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":434,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.463200+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:45:43.463200+0000","last_peered":"2023-10-01T09:45:43.463200+0000","last_clean":"2023-10-01T09:45:43.463200+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:45:43.463200+0000","last_undegraded":"2023-10-01T09:45:43.463200+0000","last_fullsized":"2023-10-01T09:45:43.463200+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":275,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460967+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:45:43.460967+0000","last_peered":"2023-10-01T09:45:43.460967+0000","last_clean":"2023-10-01T09:45:43.460967+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:45:43.460967+0000","last_undegraded":"2023-10-01T09:45:43.460967+0000","last_fullsized":"2023-10-01T09:45:43.460967+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"540'7471","reported_seq":7544,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.106891+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:45:56.106891+0000","last_peered":"2023-10-01T09:45:56.106891+0000","last_clean":"2023-10-01T09:45:56.106891+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:45:56.106891+0000","last_undegraded":"2023-10-01T09:45:56.106891+0000","last_fullsized":"2023-10-01T09:45:56.106891+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7471,"log_dups_size":0,"ondisk_log_size":7471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198377472,"num_objects":3027,"num_object_clones":0,"num_object_copies":9081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7471,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":405,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:55.273387+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:45:55.273387+0000","last_peered":"2023-10-01T09:45:55.273387+0000","last_clean":"2023-10-01T09:45:55.273387+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:45:55.273387+0000","last_undegraded":"2023-10-01T09:45:55.273387+0000","last_fullsized":"2023-10-01T09:45:55.273387+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":452,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.808733+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:45:49.808733+0000","last_peered":"2023-10-01T09:45:49.808733+0000","last_clean":"2023-10-01T09:45:49.808733+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:45:49.808733+0000","last_undegraded":"2023-10-01T09:45:49.808733+0000","last_fullsized":"2023-10-01T09:45:49.808733+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"540'7246","reported_seq":7319,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.229375+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:45:56.229375+0000","last_peered":"2023-10-01T09:45:56.229375+0000","last_clean":"2023-10-01T09:45:56.229375+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:45:56.229375+0000","last_undegraded":"2023-10-01T09:45:56.229375+0000","last_fullsized":"2023-10-01T09:45:56.229375+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7246,"log_dups_size":0,"ondisk_log_size":7246,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192151552,"num_objects":2932,"num_object_clones":0,"num_object_copies":8796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7246,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":933,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461748+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:45:43.461748+0000","last_peered":"2023-10-01T09:45:43.461748+0000","last_clean":"2023-10-01T09:45:43.461748+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:45:43.461748+0000","last_undegraded":"2023-10-01T09:45:43.461748+0000","last_fullsized":"2023-10-01T09:45:43.461748+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":271,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.836124+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:45:49.836124+0000","last_peered":"2023-10-01T09:45:49.836124+0000","last_clean":"2023-10-01T09:45:49.836124+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:45:49.836124+0000","last_undegraded":"2023-10-01T09:45:49.836124+0000","last_fullsized":"2023-10-01T09:45:49.836124+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":65,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.808088+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:45:49.808088+0000","last_peered":"2023-10-01T09:45:49.808088+0000","last_clean":"2023-10-01T09:45:49.808088+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:45:49.808088+0000","last_undegraded":"2023-10-01T09:45:49.808088+0000","last_fullsized":"2023-10-01T09:45:49.808088+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"540'7316","reported_seq":7393,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:55.747554+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:45:55.747554+0000","last_peered":"2023-10-01T09:45:55.747554+0000","last_clean":"2023-10-01T09:45:55.747554+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:45:55.747554+0000","last_undegraded":"2023-10-01T09:45:55.747554+0000","last_fullsized":"2023-10-01T09:45:55.747554+0000","mapping_epoch":475,"log_start":"481'300","ondisk_log_start":"481'300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7016,"log_dups_size":0,"ondisk_log_size":7016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200146944,"num_objects":3054,"num_object_clones":0,"num_object_copies":9162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7316,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1206,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.835892+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:45:49.835892+0000","last_peered":"2023-10-01T09:45:49.835892+0000","last_clean":"2023-10-01T09:45:49.835892+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:45:49.835892+0000","last_undegraded":"2023-10-01T09:45:49.835892+0000","last_fullsized":"2023-10-01T09:45:49.835892+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":415,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461155+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:45:43.461155+0000","last_peered":"2023-10-01T09:45:43.461155+0000","last_clean":"2023-10-01T09:45:43.461155+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:45:43.461155+0000","last_undegraded":"2023-10-01T09:45:43.461155+0000","last_fullsized":"2023-10-01T09:45:43.461155+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":244,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.836300+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:45:49.836300+0000","last_peered":"2023-10-01T09:45:49.836300+0000","last_clean":"2023-10-01T09:45:49.836300+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:45:49.836300+0000","last_undegraded":"2023-10-01T09:45:49.836300+0000","last_fullsized":"2023-10-01T09:45:49.836300+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"541'7410","reported_seq":14705,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:57.230972+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:45:57.230972+0000","last_peered":"2023-10-01T09:45:57.230972+0000","last_clean":"2023-10-01T09:45:57.230972+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:45:57.230972+0000","last_undegraded":"2023-10-01T09:45:57.230972+0000","last_fullsized":"2023-10-01T09:45:57.230972+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7410,"log_dups_size":0,"ondisk_log_size":7410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194379776,"num_objects":2966,"num_object_clones":0,"num_object_copies":8898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7410,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":887,"reported_epoch":540,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:45:49.810733+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:45:49.810733+0000","last_peered":"2023-10-01T09:45:49.810733+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:45:49.810733+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:45:49.810733+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":604,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.807697+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:45:49.807697+0000","last_peered":"2023-10-01T09:45:49.807697+0000","last_clean":"2023-10-01T09:45:49.807697+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:45:49.807697+0000","last_undegraded":"2023-10-01T09:45:49.807697+0000","last_fullsized":"2023-10-01T09:45:49.807697+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":65,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.808693+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:45:49.808693+0000","last_peered":"2023-10-01T09:45:49.808693+0000","last_clean":"2023-10-01T09:45:49.808693+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:45:49.808693+0000","last_undegraded":"2023-10-01T09:45:49.808693+0000","last_fullsized":"2023-10-01T09:45:49.808693+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"541'7324","reported_seq":7401,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:46:00.213583+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:46:00.213583+0000","last_peered":"2023-10-01T09:46:00.213583+0000","last_clean":"2023-10-01T09:46:00.213583+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:46:00.213583+0000","last_undegraded":"2023-10-01T09:46:00.213583+0000","last_fullsized":"2023-10-01T09:46:00.213583+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7324,"log_dups_size":0,"ondisk_log_size":7324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193200128,"num_objects":2948,"num_object_clones":0,"num_object_copies":8844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7324,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1657,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.836442+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:45:49.836442+0000","last_peered":"2023-10-01T09:45:49.836442+0000","last_clean":"2023-10-01T09:45:49.836442+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:45:49.836442+0000","last_undegraded":"2023-10-01T09:45:49.836442+0000","last_fullsized":"2023-10-01T09:45:49.836442+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":570,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.462196+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:45:43.462196+0000","last_peered":"2023-10-01T09:45:43.462196+0000","last_clean":"2023-10-01T09:45:43.462196+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:45:43.462196+0000","last_undegraded":"2023-10-01T09:45:43.462196+0000","last_fullsized":"2023-10-01T09:45:43.462196+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":349,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.835978+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:45:49.835978+0000","last_peered":"2023-10-01T09:45:49.835978+0000","last_clean":"2023-10-01T09:45:49.835978+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:45:49.835978+0000","last_undegraded":"2023-10-01T09:45:49.835978+0000","last_fullsized":"2023-10-01T09:45:49.835978+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":338,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460456+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:45:43.460456+0000","last_peered":"2023-10-01T09:45:43.460456+0000","last_clean":"2023-10-01T09:45:43.460456+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:45:43.460456+0000","last_undegraded":"2023-10-01T09:45:43.460456+0000","last_fullsized":"2023-10-01T09:45:43.460456+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"540'7477","reported_seq":7550,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.212457+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:45:56.212457+0000","last_peered":"2023-10-01T09:45:56.212457+0000","last_clean":"2023-10-01T09:45:56.212457+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:45:56.212457+0000","last_undegraded":"2023-10-01T09:45:56.212457+0000","last_fullsized":"2023-10-01T09:45:56.212457+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7477,"log_dups_size":0,"ondisk_log_size":7477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200212480,"num_objects":3055,"num_object_clones":0,"num_object_copies":9165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7477,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":747,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.462366+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:45:43.462366+0000","last_peered":"2023-10-01T09:45:43.462366+0000","last_clean":"2023-10-01T09:45:43.462366+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:45:43.462366+0000","last_undegraded":"2023-10-01T09:45:43.462366+0000","last_fullsized":"2023-10-01T09:45:43.462366+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":581,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.810942+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:45:49.810942+0000","last_peered":"2023-10-01T09:45:49.810942+0000","last_clean":"2023-10-01T09:45:49.810942+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:45:49.810942+0000","last_undegraded":"2023-10-01T09:45:49.810942+0000","last_fullsized":"2023-10-01T09:45:49.810942+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":581,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.811573+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:45:49.811573+0000","last_peered":"2023-10-01T09:45:49.811573+0000","last_clean":"2023-10-01T09:45:49.811573+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:45:49.811573+0000","last_undegraded":"2023-10-01T09:45:49.811573+0000","last_fullsized":"2023-10-01T09:45:49.811573+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"541'7406","reported_seq":7484,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.462121+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:45:56.462121+0000","last_peered":"2023-10-01T09:45:56.462121+0000","last_clean":"2023-10-01T09:45:56.462121+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:45:56.462121+0000","last_undegraded":"2023-10-01T09:45:56.462121+0000","last_fullsized":"2023-10-01T09:45:56.462121+0000","mapping_epoch":475,"log_start":"481'400","ondisk_log_start":"481'400","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7006,"log_dups_size":0,"ondisk_log_size":7006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200081432,"num_objects":3054,"num_object_clones":0,"num_object_copies":9162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3054,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7406,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":978,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461469+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:45:43.461469+0000","last_peered":"2023-10-01T09:45:43.461469+0000","last_clean":"2023-10-01T09:45:43.461469+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:45:43.461469+0000","last_undegraded":"2023-10-01T09:45:43.461469+0000","last_fullsized":"2023-10-01T09:45:43.461469+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":598,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.836262+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:45:49.836262+0000","last_peered":"2023-10-01T09:45:49.836262+0000","last_clean":"2023-10-01T09:45:49.836262+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:45:49.836262+0000","last_undegraded":"2023-10-01T09:45:49.836262+0000","last_fullsized":"2023-10-01T09:45:49.836262+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":451,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.836524+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:45:49.836524+0000","last_peered":"2023-10-01T09:45:49.836524+0000","last_clean":"2023-10-01T09:45:49.836524+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:45:49.836524+0000","last_undegraded":"2023-10-01T09:45:49.836524+0000","last_fullsized":"2023-10-01T09:45:49.836524+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"540'7355","reported_seq":7436,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.224548+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:45:56.224548+0000","last_peered":"2023-10-01T09:45:56.224548+0000","last_clean":"2023-10-01T09:45:56.224548+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:45:56.224548+0000","last_undegraded":"2023-10-01T09:45:56.224548+0000","last_fullsized":"2023-10-01T09:45:56.224548+0000","mapping_epoch":479,"log_start":"481'600","ondisk_log_start":"481'600","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6755,"log_dups_size":0,"ondisk_log_size":6755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196673536,"num_objects":3001,"num_object_clones":0,"num_object_copies":9003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7355,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":479,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461139+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:45:43.461139+0000","last_peered":"2023-10-01T09:45:43.461139+0000","last_clean":"2023-10-01T09:45:43.461139+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:45:43.461139+0000","last_undegraded":"2023-10-01T09:45:43.461139+0000","last_fullsized":"2023-10-01T09:45:43.461139+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1174,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.808701+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:45:49.808701+0000","last_peered":"2023-10-01T09:45:49.808701+0000","last_clean":"2023-10-01T09:45:49.808701+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:45:49.808701+0000","last_undegraded":"2023-10-01T09:45:49.808701+0000","last_fullsized":"2023-10-01T09:45:49.808701+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.462113+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:45:43.462113+0000","last_peered":"2023-10-01T09:45:43.462113+0000","last_clean":"2023-10-01T09:45:43.462113+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:45:43.462113+0000","last_undegraded":"2023-10-01T09:45:43.462113+0000","last_fullsized":"2023-10-01T09:45:43.462113+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"540'7447","reported_seq":7528,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.082206+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:45:56.082206+0000","last_peered":"2023-10-01T09:45:56.082206+0000","last_clean":"2023-10-01T09:45:56.082206+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:45:56.082206+0000","last_undegraded":"2023-10-01T09:45:56.082206+0000","last_fullsized":"2023-10-01T09:45:56.082206+0000","mapping_epoch":479,"log_start":"481'700","ondisk_log_start":"481'700","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6747,"log_dups_size":0,"ondisk_log_size":6747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200867840,"num_objects":3065,"num_object_clones":0,"num_object_copies":9195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7447,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":72,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.460982+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:45:43.460982+0000","last_peered":"2023-10-01T09:45:43.460982+0000","last_clean":"2023-10-01T09:45:43.460982+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:45:43.460982+0000","last_undegraded":"2023-10-01T09:45:43.460982+0000","last_fullsized":"2023-10-01T09:45:43.460982+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":559,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.811639+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:45:49.811639+0000","last_peered":"2023-10-01T09:45:49.811639+0000","last_clean":"2023-10-01T09:45:49.811639+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:45:49.811639+0000","last_undegraded":"2023-10-01T09:45:49.811639+0000","last_fullsized":"2023-10-01T09:45:49.811639+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":264,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.835873+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:45:49.835873+0000","last_peered":"2023-10-01T09:45:49.835873+0000","last_clean":"2023-10-01T09:45:49.835873+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:45:49.835873+0000","last_undegraded":"2023-10-01T09:45:49.835873+0000","last_fullsized":"2023-10-01T09:45:49.835873+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":508,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.462273+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:45:56.462273+0000","last_peered":"2023-10-01T09:45:56.462273+0000","last_clean":"2023-10-01T09:45:56.462273+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:45:56.462273+0000","last_undegraded":"2023-10-01T09:45:56.462273+0000","last_fullsized":"2023-10-01T09:45:56.462273+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":621,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.807874+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:45:49.807874+0000","last_peered":"2023-10-01T09:45:49.807874+0000","last_clean":"2023-10-01T09:45:49.807874+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:45:49.807874+0000","last_undegraded":"2023-10-01T09:45:49.807874+0000","last_fullsized":"2023-10-01T09:45:49.807874+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":276,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.836099+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:45:49.836099+0000","last_peered":"2023-10-01T09:45:49.836099+0000","last_clean":"2023-10-01T09:45:49.836099+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:45:49.836099+0000","last_undegraded":"2023-10-01T09:45:49.836099+0000","last_fullsized":"2023-10-01T09:45:49.836099+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":65,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.808388+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:45:49.808388+0000","last_peered":"2023-10-01T09:45:49.808388+0000","last_clean":"2023-10-01T09:45:49.808388+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:45:49.808388+0000","last_undegraded":"2023-10-01T09:45:49.808388+0000","last_fullsized":"2023-10-01T09:45:49.808388+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":276,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.836210+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:45:49.836210+0000","last_peered":"2023-10-01T09:45:49.836210+0000","last_clean":"2023-10-01T09:45:49.836210+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:45:49.836210+0000","last_undegraded":"2023-10-01T09:45:49.836210+0000","last_fullsized":"2023-10-01T09:45:49.836210+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":454,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.810971+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:45:49.810971+0000","last_peered":"2023-10-01T09:45:49.810971+0000","last_clean":"2023-10-01T09:45:49.810971+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:45:49.810971+0000","last_undegraded":"2023-10-01T09:45:49.810971+0000","last_fullsized":"2023-10-01T09:45:49.810971+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":82,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.808399+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:45:49.808399+0000","last_peered":"2023-10-01T09:45:49.808399+0000","last_clean":"2023-10-01T09:45:49.808399+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:45:49.808399+0000","last_undegraded":"2023-10-01T09:45:49.808399+0000","last_fullsized":"2023-10-01T09:45:49.808399+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":499,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.810911+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:45:49.810911+0000","last_peered":"2023-10-01T09:45:49.810911+0000","last_clean":"2023-10-01T09:45:49.810911+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:45:49.810911+0000","last_undegraded":"2023-10-01T09:45:49.810911+0000","last_fullsized":"2023-10-01T09:45:49.810911+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":698,"reported_epoch":540,"state":"active+clean","last_fresh":"2023-10-01T09:45:49.811427+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:45:49.811427+0000","last_peered":"2023-10-01T09:45:49.811427+0000","last_clean":"2023-10-01T09:45:49.811427+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:45:49.811427+0000","last_undegraded":"2023-10-01T09:45:49.811427+0000","last_fullsized":"2023-10-01T09:45:49.811427+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":391,"reported_epoch":539,"state":"active+clean","last_fresh":"2023-10-01T09:45:43.461164+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:45:43.461164+0000","last_peered":"2023-10-01T09:45:43.461164+0000","last_clean":"2023-10-01T09:45:43.461164+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:45:43.461164+0000","last_undegraded":"2023-10-01T09:45:43.461164+0000","last_fullsized":"2023-10-01T09:45:43.461164+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3146317848,"num_objects":48010,"num_object_clones":0,"num_object_copies":144030,"num_objects_missing_on_primary":1425,"num_objects_missing":1425,"num_objects_degraded":6435,"num_objects_misplaced":2929,"num_objects_unfound":0,"num_objects_dirty":48010,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":118430,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9210,"num_bytes_recovered":603586560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9980227584,"data_stored":9980215368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115630,"ondisk_log_size":115630,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430744,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1866608,"kb_used_data":967592,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":91878544,"statfs":{"total":95995035648,"available":94083629056,"internally_reserved":0,"allocated":990814208,"data_stored":989871968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.79300000000000004,"15min":2.6280000000000001},"min":{"1min":0.254,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":3.3849999999999998,"5min":6.8550000000000004,"15min":108.90000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.626,"15min":2.7559999999999998},"min":{"1min":0.25600000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.98399999999999999,"5min":5.6639999999999997,"15min":109.187},"last":0.25700000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":2.843,"15min":5.2329999999999997},"min":{"1min":0.314,"5min":0.248,"15min":0.222},"max":{"1min":3.4089999999999998,"5min":229.161,"15min":229.161},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.49099999999999999,"5min":2.7109999999999999,"15min":5.2290000000000001},"min":{"1min":0.29599999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":1.427,"5min":230.126,"15min":230.126},"last":0.36099999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:44:54 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.81399999999999995,"15min":1.6519999999999999},"min":{"1min":0.30399999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.4300000000000002,"5min":5.3090000000000002,"15min":14.901999999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.76100000000000001,"15min":1.6299999999999999},"min":{"1min":0.33200000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.4630000000000001,"5min":5.6929999999999996,"15min":14.682},"last":0.33700000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.79200000000000004,"15min":2.2810000000000001},"min":{"1min":0.32600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.3149999999999999,"5min":5.7750000000000004,"15min":76.555000000000007},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.82499999999999996,"15min":2.2709999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.8479999999999999,"5min":6.548,"15min":76.534000000000006},"last":0.43099999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.80200000000000005,"15min":1.0409999999999999},"min":{"1min":0.31,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":3.4609999999999999,"5min":6.8769999999999998,"15min":9.6020000000000003},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.82599999999999996,"15min":0.999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28000000000000003},"max":{"1min":0.63900000000000001,"5min":8.4949999999999992,"15min":8.4949999999999992},"last":0.61399999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":2.0960000000000001,"15min":1.548},"min":{"1min":0.315,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":0.80000000000000004,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.56499999999999995,"5min":2.0830000000000002,"15min":1.6459999999999999},"min":{"1min":0.35699999999999998,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":2.8849999999999998,"5min":135.339,"15min":135.339},"last":0.58599999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.075,"15min":1.133},"min":{"1min":0.38,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":0.754,"5min":11.58,"15min":11.853999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.86699999999999999,"15min":1.111},"min":{"1min":0.35599999999999998,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":3.7810000000000001,"5min":13.021000000000001,"15min":14.584},"last":0.46300000000000002}]}]},{"osd":6,"up_from":294,"seq":1262720385255,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1691636,"kb_used_data":1358124,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":92053516,"statfs":{"total":95995035648,"available":94262800384,"internally_reserved":0,"allocated":1390718976,"data_stored":1389780363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":1.865,"15min":3.9809999999999999},"min":{"1min":0.27100000000000002,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":0.80200000000000005,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":2.0640000000000001,"15min":4.0209999999999999},"min":{"1min":0.311,"5min":0.245,"15min":0.221},"max":{"1min":2.1760000000000002,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.40500000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":2.9470000000000001,"15min":4.8600000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.24199999999999999},"max":{"1min":1.9059999999999999,"5min":168.947,"15min":168.947},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.44400000000000001,"5min":2.9980000000000002,"15min":4.8109999999999999},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":0.59899999999999998,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.29499999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.75600000000000001,"15min":1.5249999999999999},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":1.9350000000000001,"5min":8.657,"15min":10.92},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.76000000000000001,"15min":1.542},"min":{"1min":0.32300000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.2649999999999999,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.38900000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.70599999999999996,"15min":2.0699999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.1909999999999998,"5min":4.149,"15min":89.753},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.875,"15min":2.0179999999999998},"min":{"1min":0.38500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.1379999999999999,"5min":8.5470000000000006,"15min":89.849999999999994},"last":0.48799999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":1.4510000000000001,"15min":1.0680000000000001},"min":{"1min":0.27200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":0.623,"5min":63.134,"15min":63.134},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.4219999999999999,"15min":1.0669999999999999},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":0.73499999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.53000000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:45:31 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.1599999999999999,"15min":1.0369999999999999},"min":{"1min":0.307,"5min":0.26700000000000002,"15min":0.251},"max":{"1min":0.82199999999999995,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.2849999999999999,"15min":1.155},"min":{"1min":0.32500000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.6800000000000002,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.64700000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.67800000000000005,"15min":1.272},"min":{"1min":0.40200000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.83699999999999997,"5min":3.492,"15min":94.245999999999995},"last":0.504},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.87,"15min":1.379},"min":{"1min":0.42299999999999999,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":2.202,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.59799999999999998}]}]},{"osd":1,"up_from":94,"seq":403726926232,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1402564,"kb_used_data":584380,"kb_used_omap":7,"kb_used_meta":818168,"kb_avail":92342588,"statfs":{"total":95995035648,"available":94558810112,"internally_reserved":0,"allocated":598405120,"data_stored":596337928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":837804455},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7469999999999999,"5min":0.93400000000000005,"15min":4.101},"min":{"1min":0.189,"5min":0.186,"15min":0.182},"max":{"1min":32.735999999999997,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.85699999999999998},{"interface":"front","average":{"1min":2.7959999999999998,"5min":1.147,"15min":3.6240000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":32.591999999999999,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.42599999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.39800000000000002,"15min":2.2669999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.51100000000000001,"5min":2.8740000000000001,"15min":51.640000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.61799999999999999,"15min":2.9500000000000002},"min":{"1min":0.245,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":2.9089999999999998,"5min":6.9710000000000001,"15min":154.197},"last":0.48999999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.752,"15min":3.7629999999999999},"min":{"1min":0.254,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":0.48899999999999999,"5min":18.503,"15min":200.56800000000001},"last":1.012},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.70799999999999996,"15min":4.484},"min":{"1min":0.26300000000000001,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":2.7360000000000002,"5min":16.515000000000001,"15min":200.63800000000001},"last":0.95699999999999996}]},{"osd":4,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.73299999999999998,"15min":2.2160000000000002},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":2.9209999999999998,"5min":4.5789999999999997,"15min":154.101},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.58199999999999996,"15min":2.3420000000000001},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.56200000000000006,"5min":2.7290000000000001,"15min":154.56999999999999},"last":0.58499999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.246,"15min":2.2530000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.30199999999999999},"max":{"1min":1.696,"5min":59.561,"15min":154.22999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.1970000000000001,"15min":2.4649999999999999},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.307},"max":{"1min":0.67300000000000004,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.63100000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.99399999999999999,"15min":2.5409999999999999},"min":{"1min":0.33500000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.86299999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.96999999999999997,"15min":2.419},"min":{"1min":0.38200000000000001,"5min":0.34599999999999997,"15min":0.32700000000000001},"max":{"1min":2.194,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.55000000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.94999999999999996,"15min":2.5529999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.33900000000000002},"max":{"1min":2.331,"5min":9.2460000000000004,"15min":154.20699999999999},"last":3.6160000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.82999999999999996,"15min":2.609},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":1.9750000000000001,"5min":5.0110000000000001,"15min":154.51599999999999},"last":3.7490000000000001}]}]},{"osd":0,"up_from":89,"seq":382252089766,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3535128,"kb_used_data":2599696,"kb_used_omap":8,"kb_used_meta":935415,"kb_avail":90210024,"statfs":{"total":95995035648,"available":92375064576,"internally_reserved":0,"allocated":2662088704,"data_stored":2660061859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":957865922},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.58499999999999996,"15min":4.0869999999999997},"min":{"1min":0.18099999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":4.2919999999999998,"5min":8.4939999999999998,"15min":216.821},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.59099999999999997,"15min":4.8499999999999996},"min":{"1min":0.20999999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":3.9620000000000002,"5min":8.6229999999999993,"15min":217.02699999999999},"last":0.47899999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.64600000000000002,"15min":1.9910000000000001},"min":{"1min":0.25,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":4.0110000000000001,"5min":4.9459999999999997,"15min":19.788},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.65500000000000003,"15min":2.0310000000000001},"min":{"1min":0.28499999999999998,"5min":0.27200000000000002,"15min":0.189},"max":{"1min":3.8719999999999999,"5min":3.8719999999999999,"15min":25.844999999999999},"last":0.54000000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.63100000000000001,"15min":3.3969999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.73799999999999999,"15min":3.27},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.9199999999999999,"5min":5.8419999999999996,"15min":222.227},"last":0.38900000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.341,"15min":2.028},"min":{"1min":0.34000000000000002,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":4.048,"5min":48.116999999999997,"15min":62.206000000000003},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.64700000000000002,"5min":1.3140000000000001,"15min":2.0880000000000001},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":4,"5min":47.854999999999997,"15min":62.073999999999998},"last":0.29199999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.76800000000000002,"15min":1.744},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":4.0350000000000001,"5min":5.8769999999999998,"15min":16.808},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.84499999999999997,"15min":1.6499999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.1269999999999998,"5min":4.8689999999999998,"15min":17.507000000000001},"last":0.30599999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.89600000000000002,"15min":1.738},"min":{"1min":0.38200000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.3129999999999997,"5min":5.2110000000000003,"15min":17.535},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.85799999999999998,"15min":1.8089999999999999},"min":{"1min":0.43099999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.2050000000000001,"5min":4.2640000000000002,"15min":15.651999999999999},"last":0.68400000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.86199999999999999,"15min":2.0579999999999998},"min":{"1min":0.375,"5min":0.374,"15min":0.374},"max":{"1min":4.3360000000000003,"5min":4.976,"15min":48.895000000000003},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.88900000000000001,"15min":2.1179999999999999},"min":{"1min":0.439,"5min":0.371,"15min":0.35099999999999998},"max":{"1min":7.1580000000000004,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.70699999999999996}]}]},{"osd":2,"up_from":0,"seq":425201762706,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599179,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1429564,"kb_used_data":593332,"kb_used_omap":3,"kb_used_meta":836220,"kb_avail":92315588,"statfs":{"total":95995035648,"available":94531162112,"internally_reserved":0,"allocated":607571968,"data_stored":605454297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":856290023},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":1.6839999999999999,"15min":4.843},"min":{"1min":0.20399999999999999,"5min":0.183,"15min":0.17899999999999999},"max":{"1min":0.877,"5min":64.504000000000005,"15min":137.34399999999999},"last":0.217},{"interface":"front","average":{"1min":0.34599999999999997,"5min":1.6000000000000001,"15min":4.9329999999999998},"min":{"1min":0.20599999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.80200000000000005,"5min":64.448999999999998,"15min":137.73400000000001},"last":0.29499999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.67900000000000005,"15min":5.0720000000000001},"min":{"1min":0.252,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":0.60499999999999998,"5min":14.917999999999999,"15min":162.22999999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.60699999999999998,"15min":5.125},"min":{"1min":0.26400000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.57399999999999995,"5min":15.074,"15min":162.70699999999999},"last":0.248}]},{"osd":2,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.60099999999999998,"15min":1.6859999999999999},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.20799999999999999},"max":{"1min":0.99199999999999999,"5min":7.141,"15min":14.254},"last":1.222},{"interface":"front","average":{"1min":0.432,"5min":0.55900000000000005,"15min":1.77},"min":{"1min":0.27000000000000002,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":1.111,"5min":4.5049999999999999,"15min":15.523},"last":0.41999999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":1.371,"15min":1.7769999999999999},"min":{"1min":0.33200000000000002,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":0.64200000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.4570000000000001,"15min":2.0409999999999999},"min":{"1min":0.35799999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.5609999999999999,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.32400000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.82599999999999996,"15min":2.1419999999999999},"min":{"1min":0.314,"5min":0.29399999999999998,"15min":0.27500000000000002},"max":{"1min":2.895,"5min":6.0659999999999998,"15min":28.117999999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.76700000000000002,"15min":2.2450000000000001},"min":{"1min":0.34899999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.0720000000000001,"5min":7.1559999999999997,"15min":28.34},"last":0.432}]},{"osd":6,"last update":"Sun Oct 1 09:45:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.512,"15min":1.879},"min":{"1min":0.33900000000000002,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":2.9100000000000001,"5min":73.887,"15min":73.887},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.6559999999999999,"15min":1.9530000000000001},"min":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":1.1379999999999999,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.51200000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.96299999999999997,"15min":1.8360000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":3.0299999999999998,"5min":7.0620000000000003,"15min":13.085000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.86899999999999999,"15min":1.716},"min":{"1min":0.40000000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.0150000000000001,"5min":4.5810000000000004,"15min":13.191000000000001},"last":0.495}]}]},{"osd":4,"up_from":30,"seq":128849019429,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3358124,"kb_used_data":2455652,"kb_used_omap":0,"kb_used_meta":902464,"kb_avail":90387028,"statfs":{"total":95995035648,"available":92556316672,"internally_reserved":0,"allocated":2514587648,"data_stored":2513648914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924123136},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.4950000000000001,"15min":4.1369999999999996},"min":{"1min":0.22500000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":2.9889999999999999,"5min":79.837000000000003,"15min":155.315},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":1.6519999999999999,"15min":4.5110000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.77600000000000002,"5min":78.983000000000004,"15min":155.334},"last":0.41099999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":1.5489999999999999,"15min":6.298},"min":{"1min":0.33000000000000002,"5min":0.252,"15min":0.224},"max":{"1min":0.67700000000000005,"5min":77.953000000000003,"15min":190.61199999999999},"last":0.441},{"interface":"front","average":{"1min":0.754,"5min":1.468,"15min":7.0140000000000002},"min":{"1min":0.35299999999999998,"5min":0.26700000000000002,"15min":0.25600000000000001},"max":{"1min":3.1339999999999999,"5min":77.962999999999994,"15min":253.273},"last":0.433}]},{"osd":2,"last update":"Sun Oct 1 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.74299999999999999,"15min":2.1150000000000002},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.1639999999999999,"5min":5.702,"15min":170.45400000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.748,"15min":1.889},"min":{"1min":0.38600000000000001,"5min":0.314,"15min":0.25800000000000001},"max":{"1min":2.121,"5min":5.6150000000000002,"15min":36.515999999999998},"last":0.35399999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:44:58 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.77200000000000002,"15min":2.1379999999999999},"min":{"1min":0.31,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.0299999999999998,"5min":5.5279999999999996,"15min":140.881},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.749,"5min":0.68300000000000005,"15min":2.327},"min":{"1min":0.312,"5min":0.312,"15min":0.311},"max":{"1min":3.161,"5min":3.786,"15min":140.911},"last":0.38100000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:45:39 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.9279999999999999,"15min":1.4870000000000001},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":2.9649999999999999,"5min":110.003,"15min":110.003},"last":0.497},{"interface":"front","average":{"1min":0.62,"5min":1.8779999999999999,"15min":1.3959999999999999},"min":{"1min":0.34999999999999998,"5min":0.29899999999999999,"15min":0.28899999999999998},"max":{"1min":2.9910000000000001,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.45300000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:45:14 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.76000000000000001,"15min":0.94199999999999995},"min":{"1min":0.32800000000000001,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":3.0539999999999998,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.79000000000000004,"15min":1.0569999999999999},"min":{"1min":0.35899999999999999,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":1.028,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.47499999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.66800000000000004,"15min":1.139},"min":{"1min":0.33400000000000002,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":2.98,"5min":3.8580000000000001,"15min":38.034999999999997},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.91400000000000003,"15min":1.4350000000000001},"min":{"1min":0.40899999999999997,"5min":0.36699999999999999,"15min":0.308},"max":{"1min":3.113,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.51900000000000002}]}]},{"osd":5,"up_from":379,"seq":1627792605336,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1626272,"kb_used_data":1257624,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":92118880,"statfs":{"total":95995035648,"available":94329733120,"internally_reserved":0,"allocated":1287806976,"data_stored":1286883822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":1.004,"15min":2.157},"min":{"1min":0.28699999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.1400000000000001,"5min":27.584,"15min":84.340999999999994},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.97699999999999998,"15min":2.0910000000000002},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":4.2300000000000004,"5min":27.669,"15min":84.356999999999999},"last":0.378}]},{"osd":1,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":2.0430000000000001,"15min":5.6200000000000001},"min":{"1min":0.26600000000000001,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":4.3289999999999997,"5min":106.56999999999999,"15min":295.71300000000002},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.495,"5min":2.0630000000000002,"15min":5.6260000000000003},"min":{"1min":0.27200000000000002,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":2.169,"5min":106.596,"15min":299.59399999999999},"last":0.42599999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.77000000000000002,"15min":1.5089999999999999},"min":{"1min":0.28799999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.2670000000000003,"5min":5.3470000000000004,"15min":14.199999999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.89800000000000002,"15min":1.468},"min":{"1min":0.315,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":4.3719999999999999,"5min":5.782,"15min":14.430999999999999},"last":0.48299999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.71799999999999997,"15min":1.417},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.1880000000000002,"5min":4.0209999999999999,"15min":14.004},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.84399999999999997,"15min":1.5629999999999999},"min":{"1min":0.36599999999999999,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":4.2910000000000004,"5min":5.4400000000000004,"15min":13.991},"last":0.504}]},{"osd":4,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.86599999999999999,"15min":1.3959999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.1929999999999996,"5min":5.9459999999999997,"15min":122.971},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.02,"15min":1.605},"min":{"1min":0.30399999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.3869999999999996,"5min":12.445,"15min":122.89100000000001},"last":0.53000000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.86899999999999999,"15min":0.99399999999999999},"min":{"1min":0.314,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":2.2069999999999999,"5min":6.7750000000000004,"15min":12.545},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.91200000000000003,"15min":1.2130000000000001},"min":{"1min":0.35199999999999998,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":4.3200000000000003,"5min":5.4160000000000004,"15min":14.193},"last":0.47799999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.0349999999999999,"15min":1.1890000000000001},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.3559999999999999,"5min":5.758,"15min":10.813000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.99399999999999999,"15min":1.0660000000000001},"min":{"1min":0.39400000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":2.226,"5min":5.2300000000000004,"15min":12.500999999999999},"last":0.63100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2647134208,"data_stored":2647130136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":585564160,"data_stored":585564160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":599982080,"data_stored":599982080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2508460032,"data_stored":2508455960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1278803968,"data_stored":1278803968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1379733504,"data_stored":1379729432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":980549632,"data_stored":980549632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:46:02.237 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:02.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:02.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:02.345 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:46:02.375 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:02.456 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:02 smithi190 ceph-mon[160435]: from='client.37200 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:02.541 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:46:02.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:02.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:46:02.551 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:46:02.644 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:02.672 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:02.735 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:02.772 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:02.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:02.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:02.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:02 smithi138 ceph-mon[192513]: from='client.37200 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:02.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:02 smithi138 ceph-mon[185886]: from='client.37200 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:02.876 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:02.909 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:03.028 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:03.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:03.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:03.133 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:03.176 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:03.235 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:03.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:03.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:03.340 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:03.369 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:03.418 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:03.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:03.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:03.522 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:03.548 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:03.684 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:03 smithi190 ceph-mon[160435]: pgmap v1602: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 6387/141972 objects degraded (4.499%); 2877/141972 objects misplaced (2.026%); 729 KiB/s, 11 objects/s recovering 2023-10-01T09:46:03.685 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:03 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6387/141972 objects degraded (4.499%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:03.685 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:03 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1445489306' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:46:03.706 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:03.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:03.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:03.813 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:03.817 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:03 smithi138 ceph-mon[192513]: pgmap v1602: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 6387/141972 objects degraded (4.499%); 2877/141972 objects misplaced (2.026%); 729 KiB/s, 11 objects/s recovering 2023-10-01T09:46:03.817 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:03 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6387/141972 objects degraded (4.499%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:03.818 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:03 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1445489306' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:46:03.818 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:46:03.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:03 smithi138 ceph-mon[185886]: pgmap v1602: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 163 op/s; 6387/141972 objects degraded (4.499%); 2877/141972 objects misplaced (2.026%); 729 KiB/s, 11 objects/s recovering 2023-10-01T09:46:03.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:03 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6387/141972 objects degraded (4.499%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:03.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:03 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1445489306' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:46:03.848 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:04.161 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:46:04.161 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:46:04.161 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:46:04.161 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:46:04.162 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:46:04.162 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:04.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:04.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:04.266 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:04.310 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:04.420 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:04.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:04.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:04.524 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:04.555 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:04.610 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:04.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:04.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:04.717 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:04.746 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:04 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:04.747 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:04 smithi138 ceph-mon[192513]: failed to encode map e542 with expected crc 2023-10-01T09:46:04.747 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:04 smithi138 ceph-mon[192513]: failed to encode map e542 with expected crc 2023-10-01T09:46:04.747 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:04 smithi138 ceph-mon[192513]: failed to encode map e542 with expected crc 2023-10-01T09:46:04.747 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1445489306' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:46:04.747 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:04 smithi138 ceph-mon[192513]: osdmap e542: 8 total, 8 up, 7 in 2023-10-01T09:46:04.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:04 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:04.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:04 smithi138 ceph-mon[185886]: failed to encode map e542 with expected crc 2023-10-01T09:46:04.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:04 smithi138 ceph-mon[185886]: failed to encode map e542 with expected crc 2023-10-01T09:46:04.747 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:04 smithi138 ceph-mon[185886]: failed to encode map e542 with expected crc 2023-10-01T09:46:04.748 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1445489306' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:46:04.748 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:04 smithi138 ceph-mon[185886]: osdmap e542: 8 total, 8 up, 7 in 2023-10-01T09:46:04.750 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:04.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:04 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:04 smithi190 ceph-mon[160435]: failed to encode map e542 with expected crc 2023-10-01T09:46:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:04 smithi190 ceph-mon[160435]: failed to encode map e542 with expected crc 2023-10-01T09:46:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:04 smithi190 ceph-mon[160435]: failed to encode map e542 with expected crc 2023-10-01T09:46:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1445489306' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:46:04.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:04 smithi190 ceph-mon[160435]: osdmap e542: 8 total, 8 up, 7 in 2023-10-01T09:46:05.001 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:46:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:46:04] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.33.4" 2023-10-01T09:46:05.027 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:46:05.151 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:05.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:05.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:05.195 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:46:05.197 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:46:05.220 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:46:05.242 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:46:05.255 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:05.268 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:46:05.286 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:46:05.288 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:05.305 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:46:05.338 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:46:05.340 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:05.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:05.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:05.361 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:46:05.363 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:46:05.363 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:46:05.363 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:46:05.363 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:46:05.363 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:46:05.363 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:46:05.363 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:46:05.363 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:46:05.415 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:46:05.447 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:05.482 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:05.538 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:05 smithi138 ceph-mon[192513]: pgmap v1604: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 6387/141789 objects degraded (4.505%); 2877/141789 objects misplaced (2.029%); 912 KiB/s, 14 objects/s recovering 2023-10-01T09:46:05.538 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:05 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/679825335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:05.538 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:05 smithi138 ceph-mon[185886]: pgmap v1604: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 6387/141789 objects degraded (4.505%); 2877/141789 objects misplaced (2.029%); 912 KiB/s, 14 objects/s recovering 2023-10-01T09:46:05.538 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:05 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/679825335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:05.553 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:05.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:05.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:05.657 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:05.688 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:05 smithi190 ceph-mon[160435]: pgmap v1604: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 6387/141789 objects degraded (4.505%); 2877/141789 objects misplaced (2.029%); 912 KiB/s, 14 objects/s recovering 2023-10-01T09:46:05.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:05 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/679825335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:06.041 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:06.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:06.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:06.145 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:06.172 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:06.219 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:06.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:06.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:06.228 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:06.326 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:06.363 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:06.599 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:06.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:06.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:06.706 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:06.746 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:06.901 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:06.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:06.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:07.007 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:07.049 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:46:07.058 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:07.070 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:46:07.071 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:46:07.113 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:07.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:07.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:07.218 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:07.247 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:07.343 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:07.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:07.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:07.447 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:07.470 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:07.483 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:07.537 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:07.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:07.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:07.644 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:07.670 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:07 smithi138 ceph-mon[185886]: pgmap v1605: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 166 op/s; 6387/140724 objects degraded (4.539%); 2877/140724 objects misplaced (2.044%); 737 KiB/s, 11 objects/s recovering 2023-10-01T09:46:07.670 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:07 smithi138 ceph-mon[192513]: pgmap v1605: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 166 op/s; 6387/140724 objects degraded (4.539%); 2877/140724 objects misplaced (2.044%); 737 KiB/s, 11 objects/s recovering 2023-10-01T09:46:07.673 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:07.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:07 smithi190 ceph-mon[160435]: pgmap v1605: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 166 op/s; 6387/140724 objects degraded (4.539%); 2877/140724 objects misplaced (2.044%); 737 KiB/s, 11 objects/s recovering 2023-10-01T09:46:07.811 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:07.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:07.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:07.866 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:46:07.919 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:07.958 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:08.116 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:08.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:08.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:08.148 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:46:08.219 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:08.256 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:08.297 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:46:08.319 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:08.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:08.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:08.399 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:46:08.401 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:46:08.424 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:08.452 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:08.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:08 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6387/140724 objects degraded (4.539%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:08.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:08 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6387/140724 objects degraded (4.539%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:08.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:08 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6387/140724 objects degraded (4.539%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:08.878 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:08.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:08.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:08.985 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:09.012 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:09.321 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:09.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:09.387 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1606,"stamp":"2023-10-01T09:46:08.355165+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3044480971,"num_objects":46411,"num_object_clones":0,"num_object_copies":139233,"num_objects_missing_on_primary":1313,"num_objects_missing":1313,"num_objects_degraded":6328,"num_objects_misplaced":2821,"num_objects_unfound":0,"num_objects_dirty":46411,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":121163,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9834,"num_bytes_recovered":631342095,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117827,"ondisk_log_size":117827,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14598168,"kb_used_data":9512864,"kb_used_omap":39,"kb_used_meta":5085208,"kb_avail":641617896,"statfs":{"total":671965249536,"available":657016725504,"internally_reserved":0,"allocated":9741172736,"data_stored":9731215047,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5207253615},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,0,1,3,0,1,2,7,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-114360320,"num_objects":-1745,"num_object_clones":0,"num_object_copies":-5235,"num_objects_missing_on_primary":-112,"num_objects_missing":-112,"num_objects_degraded":-117,"num_objects_misplaced":-108,"num_objects_unfound":0,"num_objects_dirty":-1745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1745,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":113,"num_bytes_recovered":7405568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002772"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":318,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.461917+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:45:56.461917+0000","last_peered":"2023-10-01T09:45:56.461917+0000","last_clean":"2023-10-01T09:45:56.461917+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:45:56.461917+0000","last_undegraded":"2023-10-01T09:45:56.461917+0000","last_fullsized":"2023-10-01T09:45:56.461917+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":162,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.458865+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:45:56.458865+0000","last_peered":"2023-10-01T09:45:56.458865+0000","last_clean":"2023-10-01T09:45:56.458865+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:45:56.458865+0000","last_undegraded":"2023-10-01T09:45:56.458865+0000","last_fullsized":"2023-10-01T09:45:56.458865+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":543,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.819003+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:46:03.819003+0000","last_peered":"2023-10-01T09:46:03.819003+0000","last_clean":"2023-10-01T09:46:03.819003+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:46:03.819003+0000","last_undegraded":"2023-10-01T09:46:03.819003+0000","last_fullsized":"2023-10-01T09:46:03.819003+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":601,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.462461+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:45:56.462461+0000","last_peered":"2023-10-01T09:45:56.462461+0000","last_clean":"2023-10-01T09:45:56.462461+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:45:56.462461+0000","last_undegraded":"2023-10-01T09:45:56.462461+0000","last_fullsized":"2023-10-01T09:45:56.462461+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":482,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.461884+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:45:56.461884+0000","last_peered":"2023-10-01T09:45:56.461884+0000","last_clean":"2023-10-01T09:45:56.461884+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:45:56.461884+0000","last_undegraded":"2023-10-01T09:45:56.461884+0000","last_fullsized":"2023-10-01T09:45:56.461884+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":223,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.561064+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:45:56.561064+0000","last_peered":"2023-10-01T09:45:56.561064+0000","last_clean":"2023-10-01T09:45:56.561064+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:45:56.561064+0000","last_undegraded":"2023-10-01T09:45:56.561064+0000","last_fullsized":"2023-10-01T09:45:56.561064+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":599,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.561130+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:45:56.561130+0000","last_peered":"2023-10-01T09:45:56.561130+0000","last_clean":"2023-10-01T09:45:56.561130+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:45:56.561130+0000","last_undegraded":"2023-10-01T09:45:56.561130+0000","last_fullsized":"2023-10-01T09:45:56.561130+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":529,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.499100+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:45:56.499100+0000","last_peered":"2023-10-01T09:45:56.499100+0000","last_clean":"2023-10-01T09:45:56.499100+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:45:56.499100+0000","last_undegraded":"2023-10-01T09:45:56.499100+0000","last_fullsized":"2023-10-01T09:45:56.499100+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":254,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.464723+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:45:56.464723+0000","last_peered":"2023-10-01T09:45:56.464723+0000","last_clean":"2023-10-01T09:45:56.464723+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:45:56.464723+0000","last_undegraded":"2023-10-01T09:45:56.464723+0000","last_fullsized":"2023-10-01T09:45:56.464723+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":61,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.519414+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:45:56.519414+0000","last_peered":"2023-10-01T09:45:56.519414+0000","last_clean":"2023-10-01T09:45:56.519414+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:45:56.519414+0000","last_undegraded":"2023-10-01T09:45:56.519414+0000","last_fullsized":"2023-10-01T09:45:56.519414+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":67,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.519425+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:45:56.519425+0000","last_peered":"2023-10-01T09:45:56.519425+0000","last_clean":"2023-10-01T09:45:56.519425+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:45:56.519425+0000","last_undegraded":"2023-10-01T09:45:56.519425+0000","last_fullsized":"2023-10-01T09:45:56.519425+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":66,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.458634+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:45:56.458634+0000","last_peered":"2023-10-01T09:45:56.458634+0000","last_clean":"2023-10-01T09:45:56.458634+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:45:56.458634+0000","last_undegraded":"2023-10-01T09:45:56.458634+0000","last_fullsized":"2023-10-01T09:45:56.458634+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":443,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.462138+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:45:56.462138+0000","last_peered":"2023-10-01T09:45:56.462138+0000","last_clean":"2023-10-01T09:45:56.462138+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:45:56.462138+0000","last_undegraded":"2023-10-01T09:45:56.462138+0000","last_fullsized":"2023-10-01T09:45:56.462138+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":324,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.461646+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:45:56.461646+0000","last_peered":"2023-10-01T09:45:56.461646+0000","last_clean":"2023-10-01T09:45:56.461646+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:45:56.461646+0000","last_undegraded":"2023-10-01T09:45:56.461646+0000","last_fullsized":"2023-10-01T09:45:56.461646+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":660,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.519226+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:45:56.519226+0000","last_peered":"2023-10-01T09:45:56.519226+0000","last_clean":"2023-10-01T09:45:56.519226+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:45:56.519226+0000","last_undegraded":"2023-10-01T09:45:56.519226+0000","last_fullsized":"2023-10-01T09:45:56.519226+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":397,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.519675+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:45:56.519675+0000","last_peered":"2023-10-01T09:45:56.519675+0000","last_clean":"2023-10-01T09:45:56.519675+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:45:56.519675+0000","last_undegraded":"2023-10-01T09:45:56.519675+0000","last_fullsized":"2023-10-01T09:45:56.519675+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":70,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.519234+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:45:56.519234+0000","last_peered":"2023-10-01T09:45:56.519234+0000","last_clean":"2023-10-01T09:45:56.519234+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:45:56.519234+0000","last_undegraded":"2023-10-01T09:45:56.519234+0000","last_fullsized":"2023-10-01T09:45:56.519234+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":56,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.464523+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:45:56.464523+0000","last_peered":"2023-10-01T09:45:56.464523+0000","last_clean":"2023-10-01T09:45:56.464523+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:45:56.464523+0000","last_undegraded":"2023-10-01T09:45:56.464523+0000","last_fullsized":"2023-10-01T09:45:56.464523+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":553,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.522822+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:45:56.522822+0000","last_peered":"2023-10-01T09:45:56.522822+0000","last_clean":"2023-10-01T09:45:56.522822+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:45:56.522822+0000","last_undegraded":"2023-10-01T09:45:56.522822+0000","last_fullsized":"2023-10-01T09:45:56.522822+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":565,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.462501+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:45:56.462501+0000","last_peered":"2023-10-01T09:45:56.462501+0000","last_clean":"2023-10-01T09:45:56.462501+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:45:56.462501+0000","last_undegraded":"2023-10-01T09:45:56.462501+0000","last_fullsized":"2023-10-01T09:45:56.462501+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":421,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.461801+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:45:56.461801+0000","last_peered":"2023-10-01T09:45:56.461801+0000","last_clean":"2023-10-01T09:45:56.461801+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:45:56.461801+0000","last_undegraded":"2023-10-01T09:45:56.461801+0000","last_fullsized":"2023-10-01T09:45:56.461801+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":70,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.519617+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:45:56.519617+0000","last_peered":"2023-10-01T09:45:56.519617+0000","last_clean":"2023-10-01T09:45:56.519617+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:45:56.519617+0000","last_undegraded":"2023-10-01T09:45:56.519617+0000","last_fullsized":"2023-10-01T09:45:56.519617+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"542'7550","reported_seq":7625,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:07.044875+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:46:07.044875+0000","last_peered":"2023-10-01T09:46:07.044875+0000","last_clean":"2023-10-01T09:46:07.044875+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:46:07.044875+0000","last_undegraded":"2023-10-01T09:46:07.044875+0000","last_fullsized":"2023-10-01T09:46:07.044875+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7550,"log_dups_size":0,"ondisk_log_size":7550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186253312,"num_objects":2842,"num_object_clones":0,"num_object_copies":8526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7550,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":65,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.519437+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:45:56.519437+0000","last_peered":"2023-10-01T09:45:56.519437+0000","last_clean":"2023-10-01T09:45:56.519437+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:45:56.519437+0000","last_undegraded":"2023-10-01T09:45:56.519437+0000","last_fullsized":"2023-10-01T09:45:56.519437+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":262,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.464301+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:45:56.464301+0000","last_peered":"2023-10-01T09:45:56.464301+0000","last_clean":"2023-10-01T09:45:56.464301+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:45:56.464301+0000","last_undegraded":"2023-10-01T09:45:56.464301+0000","last_fullsized":"2023-10-01T09:45:56.464301+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"542'7574","reported_seq":14792,"reported_epoch":542,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:46:07.302125+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:46:07.302125+0000","last_peered":"2023-10-01T09:46:07.302125+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:46:07.302125+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7574,"log_dups_size":0,"ondisk_log_size":7574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185597952,"num_objects":2832,"num_object_clones":0,"num_object_copies":8496,"num_objects_missing_on_primary":1313,"num_objects_missing":1313,"num_objects_degraded":4076,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7574,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2372,"num_bytes_recovered":155451392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1450},{"shards":"0,4,6","objects":1313},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":536,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.519350+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:45:56.519350+0000","last_peered":"2023-10-01T09:45:56.519350+0000","last_clean":"2023-10-01T09:45:56.519350+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:45:56.519350+0000","last_undegraded":"2023-10-01T09:45:56.519350+0000","last_fullsized":"2023-10-01T09:45:56.519350+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":598,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.458527+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:45:56.458527+0000","last_peered":"2023-10-01T09:45:56.458527+0000","last_clean":"2023-10-01T09:45:56.458527+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:45:56.458527+0000","last_undegraded":"2023-10-01T09:45:56.458527+0000","last_fullsized":"2023-10-01T09:45:56.458527+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"542'7532","reported_seq":7610,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:06.747296+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:46:06.747296+0000","last_peered":"2023-10-01T09:46:06.747296+0000","last_clean":"2023-10-01T09:46:06.747296+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:46:06.747296+0000","last_undegraded":"2023-10-01T09:46:06.747296+0000","last_fullsized":"2023-10-01T09:46:06.747296+0000","mapping_epoch":475,"log_start":"481'500","ondisk_log_start":"481'500","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7032,"log_dups_size":0,"ondisk_log_size":7032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190054400,"num_objects":2900,"num_object_clones":0,"num_object_copies":8700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7532,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":622,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.519529+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:45:56.519529+0000","last_peered":"2023-10-01T09:45:56.519529+0000","last_clean":"2023-10-01T09:45:56.519529+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:45:56.519529+0000","last_undegraded":"2023-10-01T09:45:56.519529+0000","last_fullsized":"2023-10-01T09:45:56.519529+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":418,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.458844+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:45:56.458844+0000","last_peered":"2023-10-01T09:45:56.458844+0000","last_clean":"2023-10-01T09:45:56.458844+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:45:56.458844+0000","last_undegraded":"2023-10-01T09:45:56.458844+0000","last_fullsized":"2023-10-01T09:45:56.458844+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"542'7545","reported_seq":14228,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:06.809987+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:46:06.809987+0000","last_peered":"2023-10-01T09:46:06.809987+0000","last_clean":"2023-10-01T09:46:06.809987+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:46:06.809987+0000","last_undegraded":"2023-10-01T09:46:06.809987+0000","last_fullsized":"2023-10-01T09:46:06.809987+0000","mapping_epoch":505,"log_start":"481'500","ondisk_log_start":"481'500","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7045,"log_dups_size":0,"ondisk_log_size":7045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187236352,"num_objects":2857,"num_object_clones":0,"num_object_copies":8571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7545,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":262,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.464431+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:45:56.464431+0000","last_peered":"2023-10-01T09:45:56.464431+0000","last_clean":"2023-10-01T09:45:56.464431+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:45:56.464431+0000","last_undegraded":"2023-10-01T09:45:56.464431+0000","last_fullsized":"2023-10-01T09:45:56.464431+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":521,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.462180+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:45:56.462180+0000","last_peered":"2023-10-01T09:45:56.462180+0000","last_clean":"2023-10-01T09:45:56.462180+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:45:56.462180+0000","last_undegraded":"2023-10-01T09:45:56.462180+0000","last_fullsized":"2023-10-01T09:45:56.462180+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"542'7534","reported_seq":7612,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:04.844087+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:46:04.844087+0000","last_peered":"2023-10-01T09:46:04.844087+0000","last_clean":"2023-10-01T09:46:04.844087+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:46:04.844087+0000","last_undegraded":"2023-10-01T09:46:04.844087+0000","last_fullsized":"2023-10-01T09:46:04.844087+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7534,"log_dups_size":0,"ondisk_log_size":7534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194904064,"num_objects":2974,"num_object_clones":0,"num_object_copies":8922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7534,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":122,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.461833+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:45:56.461833+0000","last_peered":"2023-10-01T09:45:56.461833+0000","last_clean":"2023-10-01T09:45:56.461833+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:45:56.461833+0000","last_undegraded":"2023-10-01T09:45:56.461833+0000","last_fullsized":"2023-10-01T09:45:56.461833+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":196,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.462218+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:45:56.462218+0000","last_peered":"2023-10-01T09:45:56.462218+0000","last_clean":"2023-10-01T09:45:56.462218+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:45:56.462218+0000","last_undegraded":"2023-10-01T09:45:56.462218+0000","last_fullsized":"2023-10-01T09:45:56.462218+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"542'7483","reported_seq":7682,"reported_epoch":542,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:46:07.052039+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:46:07.052039+0000","last_peered":"2023-10-01T09:46:07.052039+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:46:07.052039+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7483,"log_dups_size":0,"ondisk_log_size":7483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189071360,"num_objects":2885,"num_object_clones":0,"num_object_copies":8655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2242,"num_objects_misplaced":2821,"num_objects_unfound":0,"num_objects_dirty":2885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7483,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":3801088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2821},{"shards":"4,5","objects":-2178}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":440,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.461987+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:45:56.461987+0000","last_peered":"2023-10-01T09:45:56.461987+0000","last_clean":"2023-10-01T09:45:56.461987+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:45:56.461987+0000","last_undegraded":"2023-10-01T09:45:56.461987+0000","last_fullsized":"2023-10-01T09:45:56.461987+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":601,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.458829+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:45:56.458829+0000","last_peered":"2023-10-01T09:45:56.458829+0000","last_clean":"2023-10-01T09:45:56.458829+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:45:56.458829+0000","last_undegraded":"2023-10-01T09:45:56.458829+0000","last_fullsized":"2023-10-01T09:45:56.458829+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"542'7493","reported_seq":7577,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:07.494188+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:46:07.494188+0000","last_peered":"2023-10-01T09:46:07.494188+0000","last_clean":"2023-10-01T09:46:07.494188+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:46:07.494188+0000","last_undegraded":"2023-10-01T09:46:07.494188+0000","last_fullsized":"2023-10-01T09:46:07.494188+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7493,"log_dups_size":0,"ondisk_log_size":7493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189071360,"num_objects":2885,"num_object_clones":0,"num_object_copies":8655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7493,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":436,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.461390+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:45:56.461390+0000","last_peered":"2023-10-01T09:45:56.461390+0000","last_clean":"2023-10-01T09:45:56.461390+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:45:56.461390+0000","last_undegraded":"2023-10-01T09:45:56.461390+0000","last_fullsized":"2023-10-01T09:45:56.461390+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":277,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.462494+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:45:56.462494+0000","last_peered":"2023-10-01T09:45:56.462494+0000","last_clean":"2023-10-01T09:45:56.462494+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:45:56.462494+0000","last_undegraded":"2023-10-01T09:45:56.462494+0000","last_fullsized":"2023-10-01T09:45:56.462494+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"542'7587","reported_seq":7662,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:06.309526+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:46:06.309526+0000","last_peered":"2023-10-01T09:46:06.309526+0000","last_clean":"2023-10-01T09:46:06.309526+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:46:06.309526+0000","last_undegraded":"2023-10-01T09:46:06.309526+0000","last_fullsized":"2023-10-01T09:46:06.309526+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7587,"log_dups_size":0,"ondisk_log_size":7587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190775296,"num_objects":2911,"num_object_clones":0,"num_object_copies":8733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2911,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7587,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":406,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.459027+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:45:56.459027+0000","last_peered":"2023-10-01T09:45:56.459027+0000","last_clean":"2023-10-01T09:45:56.459027+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:45:56.459027+0000","last_undegraded":"2023-10-01T09:45:56.459027+0000","last_fullsized":"2023-10-01T09:45:56.459027+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":453,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.458938+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:45:56.458938+0000","last_peered":"2023-10-01T09:45:56.458938+0000","last_clean":"2023-10-01T09:45:56.458938+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:45:56.458938+0000","last_undegraded":"2023-10-01T09:45:56.458938+0000","last_fullsized":"2023-10-01T09:45:56.458938+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"542'7361","reported_seq":7436,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:06.104190+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:46:06.104190+0000","last_peered":"2023-10-01T09:46:06.104190+0000","last_clean":"2023-10-01T09:46:06.104190+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:46:06.104190+0000","last_undegraded":"2023-10-01T09:46:06.104190+0000","last_fullsized":"2023-10-01T09:46:06.104190+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7361,"log_dups_size":0,"ondisk_log_size":7361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184614912,"num_objects":2817,"num_object_clones":0,"num_object_copies":8451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7361,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":935,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.519537+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:45:56.519537+0000","last_peered":"2023-10-01T09:45:56.519537+0000","last_clean":"2023-10-01T09:45:56.519537+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:45:56.519537+0000","last_undegraded":"2023-10-01T09:45:56.519537+0000","last_fullsized":"2023-10-01T09:45:56.519537+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":272,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.464606+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:45:56.464606+0000","last_peered":"2023-10-01T09:45:56.464606+0000","last_clean":"2023-10-01T09:45:56.464606+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:45:56.464606+0000","last_undegraded":"2023-10-01T09:45:56.464606+0000","last_fullsized":"2023-10-01T09:45:56.464606+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":66,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.458716+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:45:56.458716+0000","last_peered":"2023-10-01T09:45:56.458716+0000","last_clean":"2023-10-01T09:45:56.458716+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:45:56.458716+0000","last_undegraded":"2023-10-01T09:45:56.458716+0000","last_fullsized":"2023-10-01T09:45:56.458716+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"542'7418","reported_seq":7496,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:06.827605+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:46:06.827605+0000","last_peered":"2023-10-01T09:46:06.827605+0000","last_clean":"2023-10-01T09:46:06.827605+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:46:06.827605+0000","last_undegraded":"2023-10-01T09:46:06.827605+0000","last_fullsized":"2023-10-01T09:46:06.827605+0000","mapping_epoch":475,"log_start":"481'400","ondisk_log_start":"481'400","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193462272,"num_objects":2952,"num_object_clones":0,"num_object_copies":8856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7418,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1207,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.464972+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:45:56.464972+0000","last_peered":"2023-10-01T09:45:56.464972+0000","last_clean":"2023-10-01T09:45:56.464972+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:45:56.464972+0000","last_undegraded":"2023-10-01T09:45:56.464972+0000","last_fullsized":"2023-10-01T09:45:56.464972+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":417,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.461757+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:45:56.461757+0000","last_peered":"2023-10-01T09:45:56.461757+0000","last_clean":"2023-10-01T09:45:56.461757+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:45:56.461757+0000","last_undegraded":"2023-10-01T09:45:56.461757+0000","last_fullsized":"2023-10-01T09:45:56.461757+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":245,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.464531+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:45:56.464531+0000","last_peered":"2023-10-01T09:45:56.464531+0000","last_clean":"2023-10-01T09:45:56.464531+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:45:56.464531+0000","last_undegraded":"2023-10-01T09:45:56.464531+0000","last_fullsized":"2023-10-01T09:45:56.464531+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"542'7510","reported_seq":14806,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:07.084213+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:46:07.084213+0000","last_peered":"2023-10-01T09:46:07.084213+0000","last_clean":"2023-10-01T09:46:07.084213+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:46:07.084213+0000","last_undegraded":"2023-10-01T09:46:07.084213+0000","last_fullsized":"2023-10-01T09:46:07.084213+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7510,"log_dups_size":0,"ondisk_log_size":7510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187826176,"num_objects":2866,"num_object_clones":0,"num_object_copies":8598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7510,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":888,"reported_epoch":541,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:45:56.461657+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:45:56.461657+0000","last_peered":"2023-10-01T09:45:56.461657+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:45:56.461657+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:45:56.461657+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":605,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.459091+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:45:56.459091+0000","last_peered":"2023-10-01T09:45:56.459091+0000","last_clean":"2023-10-01T09:45:56.459091+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:45:56.459091+0000","last_undegraded":"2023-10-01T09:45:56.459091+0000","last_fullsized":"2023-10-01T09:45:56.459091+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":66,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.458797+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:45:56.458797+0000","last_peered":"2023-10-01T09:45:56.458797+0000","last_clean":"2023-10-01T09:45:56.458797+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:45:56.458797+0000","last_undegraded":"2023-10-01T09:45:56.458797+0000","last_fullsized":"2023-10-01T09:45:56.458797+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"542'7374","reported_seq":7452,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:05.216887+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:46:05.216887+0000","last_peered":"2023-10-01T09:46:05.216887+0000","last_clean":"2023-10-01T09:46:05.216887+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:46:05.216887+0000","last_undegraded":"2023-10-01T09:46:05.216887+0000","last_fullsized":"2023-10-01T09:46:05.216887+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7374,"log_dups_size":0,"ondisk_log_size":7374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189923328,"num_objects":2898,"num_object_clones":0,"num_object_copies":8694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7374,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1658,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.464971+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:45:56.464971+0000","last_peered":"2023-10-01T09:45:56.464971+0000","last_clean":"2023-10-01T09:45:56.464971+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:45:56.464971+0000","last_undegraded":"2023-10-01T09:45:56.464971+0000","last_fullsized":"2023-10-01T09:45:56.464971+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":572,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.518938+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:45:56.518938+0000","last_peered":"2023-10-01T09:45:56.518938+0000","last_clean":"2023-10-01T09:45:56.518938+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:45:56.518938+0000","last_undegraded":"2023-10-01T09:45:56.518938+0000","last_fullsized":"2023-10-01T09:45:56.518938+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":350,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.464970+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:45:56.464970+0000","last_peered":"2023-10-01T09:45:56.464970+0000","last_clean":"2023-10-01T09:45:56.464970+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:45:56.464970+0000","last_undegraded":"2023-10-01T09:45:56.464970+0000","last_fullsized":"2023-10-01T09:45:56.464970+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":340,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.462278+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:45:56.462278+0000","last_peered":"2023-10-01T09:45:56.462278+0000","last_clean":"2023-10-01T09:45:56.462278+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:45:56.462278+0000","last_undegraded":"2023-10-01T09:45:56.462278+0000","last_fullsized":"2023-10-01T09:45:56.462278+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"542'7583","reported_seq":7658,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:06.283254+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:46:06.283254+0000","last_peered":"2023-10-01T09:46:06.283254+0000","last_clean":"2023-10-01T09:46:06.283254+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:46:06.283254+0000","last_undegraded":"2023-10-01T09:46:06.283254+0000","last_fullsized":"2023-10-01T09:46:06.283254+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7583,"log_dups_size":0,"ondisk_log_size":7583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193265664,"num_objects":2949,"num_object_clones":0,"num_object_copies":8847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7583,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":749,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.519184+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:45:56.519184+0000","last_peered":"2023-10-01T09:45:56.519184+0000","last_clean":"2023-10-01T09:45:56.519184+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:45:56.519184+0000","last_undegraded":"2023-10-01T09:45:56.519184+0000","last_fullsized":"2023-10-01T09:45:56.519184+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":582,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.461923+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:45:56.461923+0000","last_peered":"2023-10-01T09:45:56.461923+0000","last_clean":"2023-10-01T09:45:56.461923+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:45:56.461923+0000","last_undegraded":"2023-10-01T09:45:56.461923+0000","last_fullsized":"2023-10-01T09:45:56.461923+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":582,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.462343+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:45:56.462343+0000","last_peered":"2023-10-01T09:45:56.462343+0000","last_clean":"2023-10-01T09:45:56.462343+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:45:56.462343+0000","last_undegraded":"2023-10-01T09:45:56.462343+0000","last_fullsized":"2023-10-01T09:45:56.462343+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"542'7514","reported_seq":7593,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:06.849348+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:46:06.849348+0000","last_peered":"2023-10-01T09:46:06.849348+0000","last_clean":"2023-10-01T09:46:06.849348+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:46:06.849348+0000","last_undegraded":"2023-10-01T09:46:06.849348+0000","last_fullsized":"2023-10-01T09:46:06.849348+0000","mapping_epoch":475,"log_start":"481'500","ondisk_log_start":"481'500","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7014,"log_dups_size":0,"ondisk_log_size":7014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193003544,"num_objects":2946,"num_object_clones":0,"num_object_copies":8838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2946,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7514,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":980,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.519635+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:45:56.519635+0000","last_peered":"2023-10-01T09:45:56.519635+0000","last_clean":"2023-10-01T09:45:56.519635+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:45:56.519635+0000","last_undegraded":"2023-10-01T09:45:56.519635+0000","last_fullsized":"2023-10-01T09:45:56.519635+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":599,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.464534+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:45:56.464534+0000","last_peered":"2023-10-01T09:45:56.464534+0000","last_clean":"2023-10-01T09:45:56.464534+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:45:56.464534+0000","last_undegraded":"2023-10-01T09:45:56.464534+0000","last_fullsized":"2023-10-01T09:45:56.464534+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":452,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.465044+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:45:56.465044+0000","last_peered":"2023-10-01T09:45:56.465044+0000","last_clean":"2023-10-01T09:45:56.465044+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:45:56.465044+0000","last_undegraded":"2023-10-01T09:45:56.465044+0000","last_fullsized":"2023-10-01T09:45:56.465044+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"542'7442","reported_seq":7525,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:06.047310+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:46:06.047310+0000","last_peered":"2023-10-01T09:46:06.047310+0000","last_clean":"2023-10-01T09:46:06.047310+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:46:06.047310+0000","last_undegraded":"2023-10-01T09:46:06.047310+0000","last_fullsized":"2023-10-01T09:46:06.047310+0000","mapping_epoch":479,"log_start":"481'700","ondisk_log_start":"481'700","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6742,"log_dups_size":0,"ondisk_log_size":6742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190971904,"num_objects":2914,"num_object_clones":0,"num_object_copies":8742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7442,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":481,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.519705+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:45:56.519705+0000","last_peered":"2023-10-01T09:45:56.519705+0000","last_clean":"2023-10-01T09:45:56.519705+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:45:56.519705+0000","last_undegraded":"2023-10-01T09:45:56.519705+0000","last_fullsized":"2023-10-01T09:45:56.519705+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1175,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.459023+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:45:56.459023+0000","last_peered":"2023-10-01T09:45:56.459023+0000","last_clean":"2023-10-01T09:45:56.459023+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:45:56.459023+0000","last_undegraded":"2023-10-01T09:45:56.459023+0000","last_fullsized":"2023-10-01T09:45:56.459023+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.518885+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:45:56.518885+0000","last_peered":"2023-10-01T09:45:56.518885+0000","last_clean":"2023-10-01T09:45:56.518885+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:45:56.518885+0000","last_undegraded":"2023-10-01T09:45:56.518885+0000","last_fullsized":"2023-10-01T09:45:56.518885+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"542'7557","reported_seq":7640,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:06.198185+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:46:06.198185+0000","last_peered":"2023-10-01T09:46:06.198185+0000","last_clean":"2023-10-01T09:46:06.198185+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:46:06.198185+0000","last_undegraded":"2023-10-01T09:46:06.198185+0000","last_fullsized":"2023-10-01T09:46:06.198185+0000","mapping_epoch":479,"log_start":"481'800","ondisk_log_start":"481'800","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6757,"log_dups_size":0,"ondisk_log_size":6757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193658880,"num_objects":2955,"num_object_clones":0,"num_object_copies":8865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7557,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":74,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.519526+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:45:56.519526+0000","last_peered":"2023-10-01T09:45:56.519526+0000","last_clean":"2023-10-01T09:45:56.519526+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:45:56.519526+0000","last_undegraded":"2023-10-01T09:45:56.519526+0000","last_fullsized":"2023-10-01T09:45:56.519526+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":560,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.462418+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:45:56.462418+0000","last_peered":"2023-10-01T09:45:56.462418+0000","last_clean":"2023-10-01T09:45:56.462418+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:45:56.462418+0000","last_undegraded":"2023-10-01T09:45:56.462418+0000","last_fullsized":"2023-10-01T09:45:56.462418+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":265,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.464440+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:45:56.464440+0000","last_peered":"2023-10-01T09:45:56.464440+0000","last_clean":"2023-10-01T09:45:56.464440+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:45:56.464440+0000","last_undegraded":"2023-10-01T09:45:56.464440+0000","last_fullsized":"2023-10-01T09:45:56.464440+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":509,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.885313+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:45:56.885313+0000","last_peered":"2023-10-01T09:45:56.885313+0000","last_clean":"2023-10-01T09:45:56.885313+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:45:56.885313+0000","last_undegraded":"2023-10-01T09:45:56.885313+0000","last_fullsized":"2023-10-01T09:45:56.885313+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":622,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.458717+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:45:56.458717+0000","last_peered":"2023-10-01T09:45:56.458717+0000","last_clean":"2023-10-01T09:45:56.458717+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:45:56.458717+0000","last_undegraded":"2023-10-01T09:45:56.458717+0000","last_fullsized":"2023-10-01T09:45:56.458717+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":277,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.464425+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:45:56.464425+0000","last_peered":"2023-10-01T09:45:56.464425+0000","last_clean":"2023-10-01T09:45:56.464425+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:45:56.464425+0000","last_undegraded":"2023-10-01T09:45:56.464425+0000","last_fullsized":"2023-10-01T09:45:56.464425+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":66,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.458579+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:45:56.458579+0000","last_peered":"2023-10-01T09:45:56.458579+0000","last_clean":"2023-10-01T09:45:56.458579+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:45:56.458579+0000","last_undegraded":"2023-10-01T09:45:56.458579+0000","last_fullsized":"2023-10-01T09:45:56.458579+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":277,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.464616+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:45:56.464616+0000","last_peered":"2023-10-01T09:45:56.464616+0000","last_clean":"2023-10-01T09:45:56.464616+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:45:56.464616+0000","last_undegraded":"2023-10-01T09:45:56.464616+0000","last_fullsized":"2023-10-01T09:45:56.464616+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":455,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.462010+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:45:56.462010+0000","last_peered":"2023-10-01T09:45:56.462010+0000","last_clean":"2023-10-01T09:45:56.462010+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:45:56.462010+0000","last_undegraded":"2023-10-01T09:45:56.462010+0000","last_fullsized":"2023-10-01T09:45:56.462010+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":84,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.819486+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:46:03.819486+0000","last_peered":"2023-10-01T09:46:03.819486+0000","last_clean":"2023-10-01T09:46:03.819486+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:46:03.819486+0000","last_undegraded":"2023-10-01T09:46:03.819486+0000","last_fullsized":"2023-10-01T09:46:03.819486+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":500,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.461831+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:45:56.461831+0000","last_peered":"2023-10-01T09:45:56.461831+0000","last_clean":"2023-10-01T09:45:56.461831+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:45:56.461831+0000","last_undegraded":"2023-10-01T09:45:56.461831+0000","last_fullsized":"2023-10-01T09:45:56.461831+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":699,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.462625+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:45:56.462625+0000","last_peered":"2023-10-01T09:45:56.462625+0000","last_clean":"2023-10-01T09:45:56.462625+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:45:56.462625+0000","last_undegraded":"2023-10-01T09:45:56.462625+0000","last_fullsized":"2023-10-01T09:45:56.462625+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":393,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-10-01T09:45:56.461216+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:45:56.461216+0000","last_peered":"2023-10-01T09:45:56.461216+0000","last_clean":"2023-10-01T09:45:56.461216+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:45:56.461216+0000","last_undegraded":"2023-10-01T09:45:56.461216+0000","last_fullsized":"2023-10-01T09:45:56.461216+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3039690776,"num_objects":46383,"num_object_clones":0,"num_object_copies":139149,"num_objects_missing_on_primary":1313,"num_objects_missing":1313,"num_objects_degraded":6318,"num_objects_misplaced":2821,"num_objects_unfound":0,"num_objects_dirty":46383,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":120057,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9323,"num_bytes_recovered":610992128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9670963200,"data_stored":9670950984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116657,"ondisk_log_size":116657,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430746,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1835904,"kb_used_data":932792,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":91909248,"statfs":{"total":95995035648,"available":94115069952,"internally_reserved":0,"allocated":955179008,"data_stored":954236120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.74399999999999999,"15min":2.5840000000000001},"min":{"1min":0.27700000000000002,"5min":0.246,"15min":0.20999999999999999},"max":{"1min":1.081,"5min":6.8550000000000004,"15min":108.90000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.52500000000000002,"15min":2.7050000000000001},"min":{"1min":0.25700000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.2090000000000001,"5min":5.6639999999999997,"15min":109.187},"last":0.56699999999999995}]},{"osd":1,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":2.843,"15min":5.2329999999999997},"min":{"1min":0.314,"5min":0.248,"15min":0.222},"max":{"1min":3.4089999999999998,"5min":229.161,"15min":229.161},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.49099999999999999,"5min":2.7109999999999999,"15min":5.2290000000000001},"min":{"1min":0.29599999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":1.427,"5min":230.126,"15min":230.126},"last":0.71399999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.72799999999999998,"15min":1.617},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.1259999999999999,"5min":5.3090000000000002,"15min":14.901999999999999},"last":0.624},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.68500000000000005,"15min":1.6100000000000001},"min":{"1min":0.33700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.0009999999999999,"5min":5.6929999999999996,"15min":14.682},"last":0.38200000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.79200000000000004,"15min":2.2810000000000001},"min":{"1min":0.32600000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.3149999999999999,"5min":5.7750000000000004,"15min":76.555000000000007},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.82499999999999996,"15min":2.2709999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.8479999999999999,"5min":6.548,"15min":76.534000000000006},"last":0.48899999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.70499999999999996,"15min":1.042},"min":{"1min":0.28299999999999997,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.276,"5min":6.8769999999999998,"15min":9.6020000000000003},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.68500000000000005,"15min":0.997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.97399999999999998,"5min":5.7439999999999998,"15min":8.4949999999999992},"last":0.60699999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":2.0960000000000001,"15min":1.548},"min":{"1min":0.315,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":0.80000000000000004,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.56499999999999995,"5min":2.0830000000000002,"15min":1.6459999999999999},"min":{"1min":0.35699999999999998,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":2.8849999999999998,"5min":135.339,"15min":135.339},"last":0.58999999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.075,"15min":1.133},"min":{"1min":0.38,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":0.754,"5min":11.58,"15min":11.853999999999999},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.86699999999999999,"15min":1.111},"min":{"1min":0.35599999999999998,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":3.7810000000000001,"5min":13.021000000000001,"15min":14.584},"last":0.63800000000000001}]}]},{"osd":6,"up_from":294,"seq":1262720385257,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1646716,"kb_used_data":1309108,"kb_used_omap":6,"kb_used_meta":337593,"kb_avail":92098436,"statfs":{"total":95995035648,"available":94308798464,"internally_reserved":0,"allocated":1340526592,"data_stored":1339587655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":345695277},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":1.865,"15min":3.9809999999999999},"min":{"1min":0.27100000000000002,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":0.80200000000000005,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.498},{"interface":"front","average":{"1min":0.58999999999999997,"5min":2.0640000000000001,"15min":4.0209999999999999},"min":{"1min":0.311,"5min":0.245,"15min":0.221},"max":{"1min":2.1760000000000002,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.53100000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":2.9470000000000001,"15min":4.8600000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.24199999999999999},"max":{"1min":1.9059999999999999,"5min":168.947,"15min":168.947},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.44400000000000001,"5min":2.9980000000000002,"15min":4.8109999999999999},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":0.59899999999999998,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.32700000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.75600000000000001,"15min":1.5249999999999999},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":1.9350000000000001,"5min":8.657,"15min":10.92},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.76000000000000001,"15min":1.542},"min":{"1min":0.32300000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.2649999999999999,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.35999999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.70599999999999996,"15min":2.0699999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.1909999999999998,"5min":4.149,"15min":89.753},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.875,"15min":2.0179999999999998},"min":{"1min":0.38500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.1379999999999999,"5min":8.5470000000000006,"15min":89.849999999999994},"last":4.2050000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":1.4510000000000001,"15min":1.0680000000000001},"min":{"1min":0.27200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":0.623,"5min":63.134,"15min":63.134},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.4219999999999999,"15min":1.0669999999999999},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":0.73499999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.46200000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:45:31 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.1599999999999999,"15min":1.0369999999999999},"min":{"1min":0.307,"5min":0.26700000000000002,"15min":0.251},"max":{"1min":0.82199999999999995,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.93799999999999994},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.2849999999999999,"15min":1.155},"min":{"1min":0.32500000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.6800000000000002,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.41199999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.67800000000000005,"15min":1.272},"min":{"1min":0.40200000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.83699999999999997,"5min":3.492,"15min":94.245999999999995},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.87,"15min":1.379},"min":{"1min":0.42299999999999999,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":2.202,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.627}]}]},{"osd":1,"up_from":94,"seq":403726926233,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1392460,"kb_used_data":574276,"kb_used_omap":7,"kb_used_meta":818168,"kb_avail":92352692,"statfs":{"total":95995035648,"available":94569156608,"internally_reserved":0,"allocated":588058624,"data_stored":585991108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":837804455},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7469999999999999,"5min":0.93400000000000005,"15min":4.101},"min":{"1min":0.189,"5min":0.186,"15min":0.182},"max":{"1min":32.735999999999997,"5min":32.735999999999997,"15min":154.21299999999999},"last":2.726},{"interface":"front","average":{"1min":2.7959999999999998,"5min":1.147,"15min":3.6240000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":32.591999999999999,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.56100000000000005}]},{"osd":2,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.39800000000000002,"15min":2.2669999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.51100000000000001,"5min":2.8740000000000001,"15min":51.640000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.61799999999999999,"15min":2.9500000000000002},"min":{"1min":0.245,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":2.9089999999999998,"5min":6.9710000000000001,"15min":154.197},"last":0.33800000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.752,"15min":3.7629999999999999},"min":{"1min":0.254,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":0.48899999999999999,"5min":18.503,"15min":200.56800000000001},"last":0.307},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.70799999999999996,"15min":4.484},"min":{"1min":0.26300000000000001,"5min":0.23899999999999999,"15min":0.216},"max":{"1min":2.7360000000000002,"5min":16.515000000000001,"15min":200.63800000000001},"last":0.39900000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.73299999999999998,"15min":2.2160000000000002},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":2.9209999999999998,"5min":4.5789999999999997,"15min":154.101},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.58199999999999996,"15min":2.3420000000000001},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.56200000000000006,"5min":2.7290000000000001,"15min":154.56999999999999},"last":0.65500000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.246,"15min":2.2530000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.30199999999999999},"max":{"1min":1.696,"5min":59.561,"15min":154.22999999999999},"last":2.8929999999999998},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.1970000000000001,"15min":2.4649999999999999},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.307},"max":{"1min":0.67300000000000004,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.89600000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:45:12 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.99399999999999999,"15min":2.5409999999999999},"min":{"1min":0.33500000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.86299999999999999,"5min":23.672999999999998,"15min":154.12200000000001},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.96999999999999997,"15min":2.419},"min":{"1min":0.38200000000000001,"5min":0.34599999999999997,"15min":0.32700000000000001},"max":{"1min":2.194,"5min":23.382000000000001,"15min":154.16999999999999},"last":0.76100000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.94999999999999996,"15min":2.5529999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.33900000000000002},"max":{"1min":2.331,"5min":9.2460000000000004,"15min":154.20699999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.82999999999999996,"15min":2.609},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":1.9750000000000001,"5min":5.0110000000000001,"15min":154.51599999999999},"last":0.56999999999999995}]}]},{"osd":0,"up_from":89,"seq":382252089768,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3451944,"kb_used_data":2528800,"kb_used_omap":8,"kb_used_meta":923127,"kb_avail":90293208,"statfs":{"total":95995035648,"available":92460244992,"internally_reserved":0,"allocated":2589491200,"data_stored":2587463707,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":945283010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.58499999999999996,"15min":4.0869999999999997},"min":{"1min":0.18099999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":4.2919999999999998,"5min":8.4939999999999998,"15min":216.821},"last":0.309},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.59099999999999997,"15min":4.8499999999999996},"min":{"1min":0.20999999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":3.9620000000000002,"5min":8.6229999999999993,"15min":217.02699999999999},"last":0.38800000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.64600000000000002,"15min":1.9910000000000001},"min":{"1min":0.25,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":4.0110000000000001,"5min":4.9459999999999997,"15min":19.788},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.65500000000000003,"15min":2.0310000000000001},"min":{"1min":0.28499999999999998,"5min":0.27200000000000002,"15min":0.189},"max":{"1min":3.8719999999999999,"5min":3.8719999999999999,"15min":25.844999999999999},"last":0.41399999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.63100000000000001,"15min":3.3969999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.73799999999999999,"15min":3.27},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.9199999999999999,"5min":5.8419999999999996,"15min":222.227},"last":0.36399999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.59499999999999997,"15min":1.925},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":0.84299999999999997,"5min":4.048,"15min":62.206000000000003},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.63600000000000001,"15min":2.0089999999999999},"min":{"1min":0.29199999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":1.5189999999999999,"5min":4,"15min":62.073999999999998},"last":0.33100000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.76800000000000002,"15min":1.744},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":4.0350000000000001,"5min":5.8769999999999998,"15min":16.808},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.84499999999999997,"15min":1.6499999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.1269999999999998,"5min":4.8689999999999998,"15min":17.507000000000001},"last":0.375}]},{"osd":6,"last update":"Sun Oct 1 09:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.89600000000000002,"15min":1.738},"min":{"1min":0.38200000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.3129999999999997,"5min":5.2110000000000003,"15min":17.535},"last":0.625},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.85799999999999998,"15min":1.8089999999999999},"min":{"1min":0.43099999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.2050000000000001,"5min":4.2640000000000002,"15min":15.651999999999999},"last":0.58799999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.79500000000000004,"15min":2.028},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":2.1000000000000001,"5min":4.976,"15min":48.895000000000003},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.89900000000000002,"15min":2.048},"min":{"1min":0.376,"5min":0.376,"15min":0.35099999999999998},"max":{"1min":4.125,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.44800000000000001}]}]},{"osd":2,"up_from":0,"seq":425201762707,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599181,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1411268,"kb_used_data":570940,"kb_used_omap":3,"kb_used_meta":840316,"kb_avail":92333884,"statfs":{"total":95995035648,"available":94549897216,"internally_reserved":0,"allocated":584642560,"data_stored":582524565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":860484327},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":1.6839999999999999,"15min":4.843},"min":{"1min":0.20399999999999999,"5min":0.183,"15min":0.17899999999999999},"max":{"1min":0.877,"5min":64.504000000000005,"15min":137.34399999999999},"last":0.308},{"interface":"front","average":{"1min":0.34599999999999997,"5min":1.6000000000000001,"15min":4.9329999999999998},"min":{"1min":0.20599999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.80200000000000005,"5min":64.448999999999998,"15min":137.73400000000001},"last":0.32800000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.67900000000000005,"15min":5.0720000000000001},"min":{"1min":0.252,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":0.60499999999999998,"5min":14.917999999999999,"15min":162.22999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.60699999999999998,"15min":5.125},"min":{"1min":0.26400000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.57399999999999995,"5min":15.074,"15min":162.70699999999999},"last":0.432}]},{"osd":2,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.60099999999999998,"15min":1.6859999999999999},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.20799999999999999},"max":{"1min":0.99199999999999999,"5min":7.141,"15min":14.254},"last":0.373},{"interface":"front","average":{"1min":0.432,"5min":0.55900000000000005,"15min":1.77},"min":{"1min":0.27000000000000002,"5min":0.247,"15min":0.20100000000000001},"max":{"1min":1.111,"5min":4.5049999999999999,"15min":15.523},"last":0.35199999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":1.371,"15min":1.7769999999999999},"min":{"1min":0.33200000000000002,"5min":0.27400000000000002,"15min":0.26200000000000001},"max":{"1min":0.64200000000000002,"5min":77.793000000000006,"15min":77.793000000000006},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.4570000000000001,"15min":2.0409999999999999},"min":{"1min":0.35799999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.5609999999999999,"5min":76.646000000000001,"15min":76.646000000000001},"last":0.56299999999999994}]},{"osd":5,"last update":"Sun Oct 1 09:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.82599999999999996,"15min":2.1419999999999999},"min":{"1min":0.314,"5min":0.29399999999999998,"15min":0.27500000000000002},"max":{"1min":2.895,"5min":6.0659999999999998,"15min":28.117999999999999},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.76700000000000002,"15min":2.2450000000000001},"min":{"1min":0.34899999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.0720000000000001,"5min":7.1559999999999997,"15min":28.34},"last":0.45800000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:45:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.512,"15min":1.879},"min":{"1min":0.33900000000000002,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":2.9100000000000001,"5min":73.887,"15min":73.887},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.6559999999999999,"15min":1.9530000000000001},"min":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":1.1379999999999999,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.58999999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.96299999999999997,"15min":1.8360000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":3.0299999999999998,"5min":7.0620000000000003,"15min":13.085000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.86899999999999999,"15min":1.716},"min":{"1min":0.40000000000000002,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.0150000000000001,"5min":4.5810000000000004,"15min":13.191000000000001},"last":0.57399999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019431,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3269884,"kb_used_data":2379700,"kb_used_omap":0,"kb_used_meta":890176,"kb_avail":90475268,"statfs":{"total":95995035648,"available":92646674432,"internally_reserved":0,"allocated":2436812800,"data_stored":2435873418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911540224},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.623,"15min":3.9359999999999999},"min":{"1min":0.22600000000000001,"5min":0.22500000000000001,"15min":0.21199999999999999},"max":{"1min":0.51400000000000001,"5min":9.6690000000000005,"15min":155.315},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.371,"5min":0.72499999999999998,"15min":4.2910000000000004},"min":{"1min":0.253,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.499,"5min":6.1459999999999999,"15min":155.334},"last":0.40600000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.72399999999999998,"15min":5.2430000000000003},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":0.57799999999999996,"5min":5.944,"15min":190.61199999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.63400000000000001,"15min":5.9699999999999998},"min":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.25600000000000001},"max":{"1min":1.679,"5min":3.7469999999999999,"15min":253.273},"last":0.38500000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.74299999999999999,"15min":2.1150000000000002},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.1639999999999999,"5min":5.702,"15min":170.45400000000001},"last":0.373},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.748,"15min":1.889},"min":{"1min":0.38600000000000001,"5min":0.314,"15min":0.25800000000000001},"max":{"1min":2.121,"5min":5.6150000000000002,"15min":36.515999999999998},"last":0.44700000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.69699999999999995,"15min":2.125},"min":{"1min":0.29199999999999998,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":2.0379999999999998,"5min":5.5279999999999996,"15min":140.881},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.68500000000000005,"15min":2.2749999999999999},"min":{"1min":0.32300000000000001,"5min":0.312,"15min":0.311},"max":{"1min":6.2969999999999997,"5min":6.2969999999999997,"15min":140.911},"last":0.41699999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:45:39 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.9279999999999999,"15min":1.4870000000000001},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":2.9649999999999999,"5min":110.003,"15min":110.003},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.62,"5min":1.8779999999999999,"15min":1.3959999999999999},"min":{"1min":0.34999999999999998,"5min":0.29899999999999999,"15min":0.28899999999999998},"max":{"1min":2.9910000000000001,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.34100000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:45:14 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.76000000000000001,"15min":0.94199999999999995},"min":{"1min":0.32800000000000001,"5min":0.27700000000000002,"15min":0.27200000000000002},"max":{"1min":3.0539999999999998,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.79000000000000004,"15min":1.0569999999999999},"min":{"1min":0.35899999999999999,"5min":0.317,"15min":0.28999999999999998},"max":{"1min":1.028,"5min":6.1660000000000004,"15min":10.076000000000001},"last":0.32800000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.66800000000000004,"15min":1.139},"min":{"1min":0.33400000000000002,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":2.98,"5min":3.8580000000000001,"15min":38.034999999999997},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.91400000000000003,"15min":1.4350000000000001},"min":{"1min":0.40899999999999997,"5min":0.36699999999999999,"15min":0.308},"max":{"1min":3.113,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.432}]}]},{"osd":5,"up_from":379,"seq":1627792605338,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1589992,"kb_used_data":1217248,"kb_used_omap":13,"kb_used_meta":372722,"kb_avail":92155160,"statfs":{"total":95995035648,"available":94366883840,"internally_reserved":0,"allocated":1246461952,"data_stored":1245538474,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":381667826},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,3,0,1,2,7,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":1.004,"15min":2.157},"min":{"1min":0.28699999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.1400000000000001,"5min":27.584,"15min":84.340999999999994},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.97699999999999998,"15min":2.0910000000000002},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":4.2300000000000004,"5min":27.669,"15min":84.356999999999999},"last":0.28399999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":2.0430000000000001,"15min":5.6200000000000001},"min":{"1min":0.26600000000000001,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":4.3289999999999997,"5min":106.56999999999999,"15min":295.71300000000002},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.495,"5min":2.0630000000000002,"15min":5.6260000000000003},"min":{"1min":0.27200000000000002,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":2.169,"5min":106.596,"15min":299.59399999999999},"last":0.27000000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.77000000000000002,"15min":1.5089999999999999},"min":{"1min":0.28799999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.2670000000000003,"5min":5.3470000000000004,"15min":14.199999999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.89800000000000002,"15min":1.468},"min":{"1min":0.315,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":4.3719999999999999,"5min":5.782,"15min":14.430999999999999},"last":0.372}]},{"osd":3,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.71799999999999997,"15min":1.417},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.1880000000000002,"5min":4.0209999999999999,"15min":14.004},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.84399999999999997,"15min":1.5629999999999999},"min":{"1min":0.36599999999999999,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":4.2910000000000004,"5min":5.4400000000000004,"15min":13.991},"last":0.48699999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.86599999999999999,"15min":1.3959999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.1929999999999996,"5min":5.9459999999999997,"15min":122.971},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.02,"15min":1.605},"min":{"1min":0.30399999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.3869999999999996,"5min":12.445,"15min":122.89100000000001},"last":0.313}]},{"osd":6,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.86899999999999999,"15min":0.99399999999999999},"min":{"1min":0.314,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":2.2069999999999999,"5min":6.7750000000000004,"15min":12.545},"last":0.307},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.91200000000000003,"15min":1.2130000000000001},"min":{"1min":0.35199999999999998,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":4.3200000000000003,"5min":5.4160000000000004,"15min":14.193},"last":0.45800000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.0349999999999999,"15min":1.1890000000000001},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.3559999999999999,"5min":5.758,"15min":10.813000000000001},"last":0.879},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.99399999999999999,"15min":1.0660000000000001},"min":{"1min":0.39400000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":2.226,"5min":5.2300000000000004,"15min":12.500999999999999},"last":0.89200000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2571833344,"data_stored":2571829272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":576323584,"data_stored":576323584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":578093056,"data_stored":578093056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2430799872,"data_stored":2430795800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1238564864,"data_stored":1238564864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1330581504,"data_stored":1330577432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":944766976,"data_stored":944766976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:46:09.397 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:46:09.426 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:09.475 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:09.586 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:09.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:09.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:09.596 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:09 smithi138 ceph-mon[185886]: pgmap v1606: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 6328/139233 objects degraded (4.545%); 2821/139233 objects misplaced (2.026%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:46:09.596 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:09 smithi138 ceph-mon[185886]: from='client.47249 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:09.596 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:09 smithi138 ceph-mon[192513]: pgmap v1606: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 6328/139233 objects degraded (4.545%); 2821/139233 objects misplaced (2.026%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:46:09.596 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:09 smithi138 ceph-mon[192513]: from='client.47249 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:09.697 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:09.716 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:09.728 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:09 smithi190 ceph-mon[160435]: pgmap v1606: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 6328/139233 objects degraded (4.545%); 2821/139233 objects misplaced (2.026%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:46:09.729 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:09 smithi190 ceph-mon[160435]: from='client.47249 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:09.736 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:09.800 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:09.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:09.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:09.894 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:09.905 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:09.922 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:46:09.922 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:46:09.948 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:10.129 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:10.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:10.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:10.234 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:10.267 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:10.485 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:46:10.521 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:10 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/555821244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:10.521 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:46:10.521 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:10 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/555821244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:10.521 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:46:10.709 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T09:46:10.736 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:10 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/555821244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:10.736 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:46:11.124 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:46:11.529 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:11.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:11.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:11.638 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:11.665 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:11 smithi190 ceph-mon[160435]: pgmap v1607: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 6328/138864 objects degraded (4.557%); 2821/138864 objects misplaced (2.031%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:46:11.665 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:11 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:11.666 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:11 smithi190 ceph-mon[160435]: failed to encode map e543 with expected crc 2023-10-01T09:46:11.666 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/555821244' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:46:11.666 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:11 smithi190 ceph-mon[160435]: osdmap e543: 8 total, 8 up, 7 in 2023-10-01T09:46:11.666 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:11 smithi190 ceph-mon[160435]: failed to encode map e543 with expected crc 2023-10-01T09:46:11.668 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:11.774 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:11.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:11.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:11.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:11 smithi138 ceph-mon[192513]: pgmap v1607: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 6328/138864 objects degraded (4.557%); 2821/138864 objects misplaced (2.031%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:46:11.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:11 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:11.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:11 smithi138 ceph-mon[192513]: failed to encode map e543 with expected crc 2023-10-01T09:46:11.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/555821244' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:46:11.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:11 smithi138 ceph-mon[192513]: osdmap e543: 8 total, 8 up, 7 in 2023-10-01T09:46:11.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:11 smithi138 ceph-mon[192513]: failed to encode map e543 with expected crc 2023-10-01T09:46:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:11 smithi138 ceph-mon[185886]: pgmap v1607: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 6328/138864 objects degraded (4.557%); 2821/138864 objects misplaced (2.031%); 723 KiB/s, 11 objects/s recovering 2023-10-01T09:46:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:11 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:11 smithi138 ceph-mon[185886]: failed to encode map e543 with expected crc 2023-10-01T09:46:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/555821244' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:46:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:11 smithi138 ceph-mon[185886]: osdmap e543: 8 total, 8 up, 7 in 2023-10-01T09:46:11.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:11 smithi138 ceph-mon[185886]: failed to encode map e543 with expected crc 2023-10-01T09:46:11.878 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:11.904 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:11.946 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:11.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:11.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:12.049 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:12.079 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:12.132 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:12.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:12.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:12.236 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:12.265 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:12.392 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:12.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:12.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:12.474 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:46:12.479 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:12 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2664732561' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:12.479 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:12 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2664732561' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:12.497 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:12.498 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:46:12.522 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:46:12.534 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:12.548 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:46:12.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:12 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2664732561' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:12.572 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:46:12.598 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:46:12.623 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:46:12.646 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:46:12.646 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:12.647 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:46:12.647 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:46:12.647 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:46:12.647 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:46:12.648 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:46:12.648 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:46:12.648 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:46:12.648 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:46:12.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:12.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:12.753 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:12.785 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:12.836 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:12.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:12.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:12.889 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:46:12.889 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:46:12.889 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:46:12.890 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:46:12.890 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:46:12.940 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:12.967 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:13.012 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:13.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:13.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:13.021 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:46:13.021 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 6328/137235 objects degraded (4.611%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2193,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":543,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":45745,"data_bytes":3000833995,"bytes_used":14836940800,"bytes_avail":657128308736,"bytes_total":671965249536,"degraded_objects":6328,"degraded_total":137235,"degraded_ratio":0.046110686049477184,"misplaced_objects":2821,"misplaced_total":137235,"misplaced_ratio":0.020555980617189492,"recovering_objects_per_sec":6,"recovering_bytes_per_sec":444724,"recovering_keys_per_sec":0,"num_objects_recovered":58,"num_bytes_recovered":3801088,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":177},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":34,"modified":"2023-10-01T09:43:40.311181+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:46:13.115 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:13.148 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:13.203 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:13.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:13.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:13.307 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:13.334 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:13.380 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:13.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:13.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:13.465 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:46:13.484 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:13.538 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:13.732 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:13 smithi190 ceph-mon[160435]: pgmap v1609: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 6328/137235 objects degraded (4.611%); 2821/137235 objects misplaced (2.056%); 434 KiB/s, 6 objects/s recovering 2023-10-01T09:46:13.732 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:13 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6328/137235 objects degraded (4.611%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:13.732 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:13 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3915164028' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:46:13.733 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T09:46:13.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:13 smithi138 ceph-mon[192513]: pgmap v1609: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 6328/137235 objects degraded (4.611%); 2821/137235 objects misplaced (2.056%); 434 KiB/s, 6 objects/s recovering 2023-10-01T09:46:13.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:13 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6328/137235 objects degraded (4.611%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:13.807 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:13 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3915164028' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:46:13.807 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:13 smithi138 ceph-mon[185886]: pgmap v1609: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 6328/137235 objects degraded (4.611%); 2821/137235 objects misplaced (2.056%); 434 KiB/s, 6 objects/s recovering 2023-10-01T09:46:13.807 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:13 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6328/137235 objects degraded (4.611%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:13.807 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:13 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3915164028' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:46:13.826 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:13.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:13.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:13.832 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:46:13.832 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:46:13.906 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:13.932 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:13.962 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:14.012 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:14.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:14.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:14.119 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:14.134 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:46:14.154 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:14.212 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:14.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:14.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:14.317 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:14.352 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:14.496 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:14.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:14.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:14.600 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:14.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:14 smithi190 ceph-mon[160435]: pgmap v1610: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 184 op/s; 6268/136254 objects degraded (4.600%); 2750/136254 objects misplaced (2.018%); 845 KiB/s, 13 objects/s recovering 2023-10-01T09:46:14.636 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:14.750 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:14.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:14.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:14.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:14 smithi138 ceph-mon[185886]: pgmap v1610: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 184 op/s; 6268/136254 objects degraded (4.600%); 2750/136254 objects misplaced (2.018%); 845 KiB/s, 13 objects/s recovering 2023-10-01T09:46:14.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:14 smithi138 ceph-mon[192513]: pgmap v1610: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 184 op/s; 6268/136254 objects degraded (4.600%); 2750/136254 objects misplaced (2.018%); 845 KiB/s, 13 objects/s recovering 2023-10-01T09:46:14.852 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:14.881 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:14.928 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:14.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:14.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:14.949 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:15.032 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:15.061 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:15.163 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:15.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:15.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:15.192 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:46:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:46:14] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-01T09:46:15.267 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:46:15.297 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:15.345 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:46:15.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:15.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:46:15.449 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:15.482 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:15.582 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:15.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:15.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:15.626 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:46:15.687 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:15.716 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:15.835 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:15.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:15.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:15.898 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:46:15.899 INFO:teuthology.orchestra.run.smithi190.stdout: "size": 20, 2023-10-01T09:46:15.899 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 600, 2023-10-01T09:46:15.899 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:46:15.899 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.899 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:83590 10.7 10:e3cab6b8:::benchmark_data_smithi138_573643_object368:head [delete] snapc 0=[] ondisk+write+known_if_redirected e507)", 2023-10-01T09:46:15.899 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:41:45.208406+0000", 2023-10-01T09:46:15.899 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 270.686474455, 2023-10-01T09:46:15.899 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3554332389999999, 2023-10-01T09:46:15.899 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:15.900 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:15.900 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:15.900 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:15.900 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:15.900 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 83590 2023-10-01T09:46:15.900 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.900 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:15.900 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.900 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:15.901 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.208406+0000", 2023-10-01T09:46:15.901 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:15.901 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.901 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.901 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:15.901 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.208406+0000", 2023-10-01T09:46:15.901 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:15.901 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.901 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.901 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:15.902 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.208405+0000", 2023-10-01T09:46:15.902 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7559999999999996e-06 2023-10-01T09:46:15.902 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.902 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.902 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:15.902 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.208411+0000", 2023-10-01T09:46:15.902 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1364e-05 2023-10-01T09:46:15.902 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.902 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.903 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:15.903 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.208422+0000", 2023-10-01T09:46:15.903 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1250000000000003e-06 2023-10-01T09:46:15.903 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.903 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.903 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:15.903 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.208426+0000", 2023-10-01T09:46:15.903 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010424242 2023-10-01T09:46:15.903 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.903 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.904 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.904 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.218851+0000", 2023-10-01T09:46:15.904 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0091560159999999995 2023-10-01T09:46:15.904 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.904 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.904 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:15.904 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.228007+0000", 2023-10-01T09:46:15.904 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.110518031 2023-10-01T09:46:15.904 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.905 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.905 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.905 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.338525+0000", 2023-10-01T09:46:15.905 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015410404000000001 2023-10-01T09:46:15.905 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.905 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.905 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:15.905 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.353935+0000", 2023-10-01T09:46:15.905 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1074809280000002 2023-10-01T09:46:15.905 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.906 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.906 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.906 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.461416+0000", 2023-10-01T09:46:15.906 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0055877849999999996 2023-10-01T09:46:15.906 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.906 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.906 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:15.906 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.467004+0000", 2023-10-01T09:46:15.906 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011485334 2023-10-01T09:46:15.907 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.907 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.907 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:15.907 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.478489+0000", 2023-10-01T09:46:15.907 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.034527074999999997 2023-10-01T09:46:15.907 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.907 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.907 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:15.907 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.513016+0000", 2023-10-01T09:46:15.907 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.023680901000000001 2023-10-01T09:46:15.908 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.908 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.908 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.908 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.536697+0000", 2023-10-01T09:46:15.908 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002173318 2023-10-01T09:46:15.908 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.908 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.908 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.908 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.538870+0000", 2023-10-01T09:46:15.908 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003598286 2023-10-01T09:46:15.909 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.909 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.909 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:15.909 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.542469+0000", 2023-10-01T09:46:15.909 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021370970999999999 2023-10-01T09:46:15.909 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.909 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.909 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:15.909 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.563840+0000", 2023-10-01T09:46:15.909 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3554332389999999 2023-10-01T09:46:15.910 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.910 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:15.910 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.910 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.910 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.910 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:83591 10.7 10:ea69c9ce:::benchmark_data_smithi138_573643_object369:head [delete] snapc 0=[] ondisk+write+known_if_redirected e507)", 2023-10-01T09:46:15.910 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:41:45.440915+0000", 2023-10-01T09:46:15.910 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 270.453966186, 2023-10-01T09:46:15.910 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3340868129999999, 2023-10-01T09:46:15.910 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:15.911 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:15.911 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:15.911 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:15.911 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:15.911 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 83591 2023-10-01T09:46:15.911 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.911 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:15.911 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.911 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:15.912 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.440915+0000", 2023-10-01T09:46:15.912 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:15.912 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.912 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.912 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:15.912 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.440915+0000", 2023-10-01T09:46:15.912 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999976 2023-10-01T09:46:15.912 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.912 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.912 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:15.913 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.440913+0000", 2023-10-01T09:46:15.913 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9959999999999993e-06 2023-10-01T09:46:15.913 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.913 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.913 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:15.913 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.440921+0000", 2023-10-01T09:46:15.913 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3448999999999999e-05 2023-10-01T09:46:15.913 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.913 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.913 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:15.914 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.440934+0000", 2023-10-01T09:46:15.914 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.7509999999999997e-06 2023-10-01T09:46:15.914 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.914 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.914 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:15.914 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.440940+0000", 2023-10-01T09:46:15.914 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.069774713000000002 2023-10-01T09:46:15.914 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.914 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.914 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.915 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.510714+0000", 2023-10-01T09:46:15.915 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010408116 2023-10-01T09:46:15.915 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.915 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.915 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:15.915 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.521123+0000", 2023-10-01T09:46:15.915 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.049019355000000001 2023-10-01T09:46:15.915 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.915 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.915 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.916 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.570142+0000", 2023-10-01T09:46:15.916 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0070280059999999998 2023-10-01T09:46:15.916 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.916 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.916 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:15.916 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.577170+0000", 2023-10-01T09:46:15.916 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1383411699999999 2023-10-01T09:46:15.916 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.916 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.916 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.917 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.715511+0000", 2023-10-01T09:46:15.917 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0057394810000000003 2023-10-01T09:46:15.917 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.917 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.917 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:15.917 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.721251+0000", 2023-10-01T09:46:15.917 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.01201491 2023-10-01T09:46:15.917 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.917 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.917 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:15.918 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.733265+0000", 2023-10-01T09:46:15.918 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020794109000000002 2023-10-01T09:46:15.918 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.918 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.918 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.918 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.754060+0000", 2023-10-01T09:46:15.918 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00076177299999999999 2023-10-01T09:46:15.918 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.918 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.918 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:15.919 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.754821+0000", 2023-10-01T09:46:15.919 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0022316010000000002 2023-10-01T09:46:15.919 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.919 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.919 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.919 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.757053+0000", 2023-10-01T09:46:15.919 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003341045 2023-10-01T09:46:15.919 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.919 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.919 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:15.920 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.760394+0000", 2023-10-01T09:46:15.920 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014607508999999999 2023-10-01T09:46:15.920 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.920 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.920 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:15.920 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:47.775001+0000", 2023-10-01T09:46:15.920 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3340868129999999 2023-10-01T09:46:15.920 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.920 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:15.921 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.921 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.921 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.921 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:83613 10.7 10:edbcb369:::benchmark_data_smithi138_573643_object391:head [delete] snapc 0=[] ondisk+write+known_if_redirected e507)", 2023-10-01T09:46:15.921 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:41:45.724197+0000", 2023-10-01T09:46:15.921 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 270.17068411299999, 2023-10-01T09:46:15.921 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3772443320000001, 2023-10-01T09:46:15.921 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:15.921 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:15.921 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:15.922 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:15.922 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:15.922 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 83613 2023-10-01T09:46:15.922 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.922 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:15.922 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.922 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:15.922 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.724197+0000", 2023-10-01T09:46:15.922 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:15.923 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.923 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.923 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:15.923 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.724197+0000", 2023-10-01T09:46:15.923 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:46:15.923 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.923 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.923 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:15.923 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.724195+0000", 2023-10-01T09:46:15.923 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.4139999999999994e-06 2023-10-01T09:46:15.924 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.924 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.924 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:15.924 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.724203+0000", 2023-10-01T09:46:15.924 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4229e-05 2023-10-01T09:46:15.924 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.924 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.924 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:15.924 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.724217+0000", 2023-10-01T09:46:15.924 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.2520000000000005e-06 2023-10-01T09:46:15.925 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.925 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.925 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:15.925 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.724227+0000", 2023-10-01T09:46:15.925 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.060588154999999998 2023-10-01T09:46:15.925 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.925 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.925 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.925 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.784815+0000", 2023-10-01T09:46:15.925 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0072383899999999999 2023-10-01T09:46:15.926 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.926 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.926 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:15.926 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.792053+0000", 2023-10-01T09:46:15.926 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.103984359 2023-10-01T09:46:15.926 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.926 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.926 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.926 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.896038+0000", 2023-10-01T09:46:15.927 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013116803 2023-10-01T09:46:15.927 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.927 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.927 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:15.927 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.909154+0000", 2023-10-01T09:46:15.927 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1076123369999999 2023-10-01T09:46:15.927 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.927 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.927 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.927 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.016767+0000", 2023-10-01T09:46:15.928 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0052064360000000001 2023-10-01T09:46:15.928 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.928 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.928 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:15.928 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.021973+0000", 2023-10-01T09:46:15.928 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011294665000000001 2023-10-01T09:46:15.928 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.928 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.928 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:15.928 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.033268+0000", 2023-10-01T09:46:15.929 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015390724 2023-10-01T09:46:15.929 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.929 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.929 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.929 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.048659+0000", 2023-10-01T09:46:15.929 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033475240000000002 2023-10-01T09:46:15.929 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.929 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.929 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.929 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.052006+0000", 2023-10-01T09:46:15.930 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.031843712000000003 2023-10-01T09:46:15.930 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.930 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.930 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:15.930 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.083850+0000", 2023-10-01T09:46:15.930 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033814079999999998 2023-10-01T09:46:15.930 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.930 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.930 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:15.931 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.087231+0000", 2023-10-01T09:46:15.931 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014209908 2023-10-01T09:46:15.931 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.931 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.931 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:15.931 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.101441+0000", 2023-10-01T09:46:15.931 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3772443320000001 2023-10-01T09:46:15.931 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.931 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:15.931 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.932 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.932 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.932 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:83645 10.7 10:ef1639f2:::benchmark_data_smithi138_573643_object423:head [delete] snapc 0=[] ondisk+write+known_if_redirected e507)", 2023-10-01T09:46:15.932 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:41:45.994469+0000", 2023-10-01T09:46:15.932 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 269.90041217200002, 2023-10-01T09:46:15.932 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.409882804, 2023-10-01T09:46:15.932 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:15.932 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:15.932 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:15.933 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:15.933 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:15.933 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 83645 2023-10-01T09:46:15.933 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.933 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:15.933 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.933 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:15.933 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.994469+0000", 2023-10-01T09:46:15.933 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:15.934 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.934 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.934 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:15.934 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.994469+0000", 2023-10-01T09:46:15.934 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:15.934 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.934 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.934 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:15.934 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.994467+0000", 2023-10-01T09:46:15.934 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0709999999999997e-06 2023-10-01T09:46:15.935 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.935 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.935 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:15.935 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.994474+0000", 2023-10-01T09:46:15.935 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2948999999999999e-05 2023-10-01T09:46:15.935 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.935 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.935 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:15.935 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.994487+0000", 2023-10-01T09:46:15.935 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0730000000000001e-06 2023-10-01T09:46:15.936 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.936 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.936 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:15.936 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:45.994493+0000", 2023-10-01T09:46:15.936 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.080919399000000003 2023-10-01T09:46:15.936 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.936 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.936 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.936 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:46.075413+0000", 2023-10-01T09:46:15.936 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0077341270000000004 2023-10-01T09:46:15.937 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.937 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.937 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:15.937 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:46.083147+0000", 2023-10-01T09:46:15.937 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.043462848999999998 2023-10-01T09:46:15.937 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.937 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.937 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.937 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:46.126610+0000", 2023-10-01T09:46:15.938 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0070537430000000003 2023-10-01T09:46:15.938 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.938 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.938 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:15.938 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:46.133663+0000", 2023-10-01T09:46:15.938 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1613628199999999 2023-10-01T09:46:15.938 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.938 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.938 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.938 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.295026+0000", 2023-10-01T09:46:15.939 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0060056950000000001 2023-10-01T09:46:15.939 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.939 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.939 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:15.939 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.301032+0000", 2023-10-01T09:46:15.939 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011394835000000001 2023-10-01T09:46:15.939 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.939 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.939 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:15.940 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.312427+0000", 2023-10-01T09:46:15.940 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.069707785999999994 2023-10-01T09:46:15.940 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.940 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.940 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.940 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.382135+0000", 2023-10-01T09:46:15.940 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00067693199999999999 2023-10-01T09:46:15.941 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.941 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.941 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:15.941 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.382812+0000", 2023-10-01T09:46:15.941 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001947655 2023-10-01T09:46:15.941 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.941 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.942 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.942 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.384759+0000", 2023-10-01T09:46:15.942 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0032300620000000001 2023-10-01T09:46:15.942 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.942 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.942 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:15.942 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.387989+0000", 2023-10-01T09:46:15.943 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016362247999999999 2023-10-01T09:46:15.943 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.943 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.943 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:15.943 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:48.404351+0000", 2023-10-01T09:46:15.943 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.409882804 2023-10-01T09:46:15.943 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.944 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:15.944 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.944 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.944 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.944 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:84286 10.7 10:e57119dc:::benchmark_data_smithi138_573643_object1064:head [delete] snapc 0=[] ondisk+write+known_if_redirected e507)", 2023-10-01T09:46:15.944 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:41:51.192166+0000", 2023-10-01T09:46:15.944 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 264.702714582, 2023-10-01T09:46:15.944 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3637229199999998, 2023-10-01T09:46:15.945 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:15.945 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:15.945 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:15.945 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:15.945 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:15.945 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 84286 2023-10-01T09:46:15.945 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.945 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:15.945 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.945 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:15.946 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.192166+0000", 2023-10-01T09:46:15.946 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:15.946 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.946 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.946 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:15.946 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.192166+0000", 2023-10-01T09:46:15.946 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:15.946 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.946 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.946 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:15.947 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.192165+0000", 2023-10-01T09:46:15.947 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9340000000000003e-06 2023-10-01T09:46:15.947 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.947 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.947 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:15.947 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.192171+0000", 2023-10-01T09:46:15.947 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3386e-05 2023-10-01T09:46:15.947 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.947 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.947 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:15.948 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.192184+0000", 2023-10-01T09:46:15.948 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1749999999999996e-06 2023-10-01T09:46:15.948 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.948 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.948 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:15.948 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.192189+0000", 2023-10-01T09:46:15.948 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011353696999999999 2023-10-01T09:46:15.948 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.948 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.949 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.949 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.203543+0000", 2023-10-01T09:46:15.949 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0066603629999999999 2023-10-01T09:46:15.949 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.949 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.949 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:15.949 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.210204+0000", 2023-10-01T09:46:15.949 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.115214728 2023-10-01T09:46:15.949 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.949 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.950 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.950 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.325418+0000", 2023-10-01T09:46:15.950 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0066214999999999998 2023-10-01T09:46:15.950 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.950 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.950 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:15.950 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.332040+0000", 2023-10-01T09:46:15.950 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.08533043 2023-10-01T09:46:15.950 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.950 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.951 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.951 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.417370+0000", 2023-10-01T09:46:15.951 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0054169769999999999 2023-10-01T09:46:15.951 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.951 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.951 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:15.951 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.422787+0000", 2023-10-01T09:46:15.951 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011116684 2023-10-01T09:46:15.951 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.951 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.952 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:15.952 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.433904+0000", 2023-10-01T09:46:15.952 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.069259295999999998 2023-10-01T09:46:15.952 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.952 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.952 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:15.952 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.503163+0000", 2023-10-01T09:46:15.952 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.029698714000000001 2023-10-01T09:46:15.952 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.952 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.953 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.953 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.532862+0000", 2023-10-01T09:46:15.953 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00226429 2023-10-01T09:46:15.953 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.953 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.953 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.953 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.535126+0000", 2023-10-01T09:46:15.953 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033188900000000001 2023-10-01T09:46:15.953 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.953 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.954 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:15.954 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.538445+0000", 2023-10-01T09:46:15.954 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017444158000000001 2023-10-01T09:46:15.954 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.954 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.954 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:15.954 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.555889+0000", 2023-10-01T09:46:15.954 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3637229199999998 2023-10-01T09:46:15.954 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.954 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:15.955 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.955 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.955 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.955 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:84388 10.7 10:e79161cb:::benchmark_data_smithi138_573643_object1166:head [delete] snapc 0=[] ondisk+write+known_if_redirected e507)", 2023-10-01T09:46:15.955 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:41:51.873028+0000", 2023-10-01T09:46:15.955 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 264.02185322399998, 2023-10-01T09:46:15.955 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.396121457, 2023-10-01T09:46:15.955 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:15.955 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:15.956 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:15.956 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:15.956 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:15.956 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 84388 2023-10-01T09:46:15.956 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.956 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:15.956 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.956 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:15.956 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.873028+0000", 2023-10-01T09:46:15.956 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:15.957 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.957 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.957 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:15.957 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.873028+0000", 2023-10-01T09:46:15.957 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:46:15.957 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.957 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.957 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:15.957 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.873026+0000", 2023-10-01T09:46:15.957 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9079999999999995e-06 2023-10-01T09:46:15.958 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.958 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.958 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:15.958 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.873034+0000", 2023-10-01T09:46:15.958 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6517000000000001e-05 2023-10-01T09:46:15.958 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.958 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.958 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:15.958 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.873050+0000", 2023-10-01T09:46:15.959 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.5560000000000002e-06 2023-10-01T09:46:15.959 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.959 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.959 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:15.959 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.873058+0000", 2023-10-01T09:46:15.959 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.101309838 2023-10-01T09:46:15.959 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.959 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.959 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.959 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.974368+0000", 2023-10-01T09:46:15.960 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0073214120000000002 2023-10-01T09:46:15.960 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.960 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.960 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:15.960 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.981689+0000", 2023-10-01T09:46:15.960 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.067913568999999993 2023-10-01T09:46:15.960 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.960 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.960 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.960 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.049603+0000", 2023-10-01T09:46:15.961 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0066668489999999999 2023-10-01T09:46:15.961 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.961 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.961 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:15.961 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.056269+0000", 2023-10-01T09:46:15.961 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.091550088 2023-10-01T09:46:15.961 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.961 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.961 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.961 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.147820+0000", 2023-10-01T09:46:15.962 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050517330000000001 2023-10-01T09:46:15.962 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.962 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.962 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:15.962 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.152871+0000", 2023-10-01T09:46:15.962 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011052024000000001 2023-10-01T09:46:15.962 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.962 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.962 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:15.963 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.163923+0000", 2023-10-01T09:46:15.963 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.01714154 2023-10-01T09:46:15.963 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.963 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.963 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.963 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.181065+0000", 2023-10-01T09:46:15.963 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002111468 2023-10-01T09:46:15.963 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.963 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.964 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.964 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.183176+0000", 2023-10-01T09:46:15.964 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.065504517999999998 2023-10-01T09:46:15.964 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.964 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.964 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:15.964 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.248681+0000", 2023-10-01T09:46:15.964 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00341347 2023-10-01T09:46:15.964 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.964 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.965 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:15.965 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.252094+0000", 2023-10-01T09:46:15.965 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017054791999999999 2023-10-01T09:46:15.965 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.965 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.965 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:15.965 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.269149+0000", 2023-10-01T09:46:15.966 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.396121457 2023-10-01T09:46:15.966 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.966 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:15.966 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.966 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.966 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.966 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:84389 10.7 10:e90dcdbb:::benchmark_data_smithi138_573643_object1167:head [delete] snapc 0=[] ondisk+write+known_if_redirected e507)", 2023-10-01T09:46:15.966 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:41:51.917688+0000", 2023-10-01T09:46:15.966 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 263.97719258799998, 2023-10-01T09:46:15.966 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.384396798, 2023-10-01T09:46:15.967 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:15.967 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:15.967 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:15.967 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:15.967 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:15.967 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 84389 2023-10-01T09:46:15.967 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.968 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:15.968 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.968 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:15.968 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.917688+0000", 2023-10-01T09:46:15.968 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:15.968 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.968 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.968 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:15.968 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.917688+0000", 2023-10-01T09:46:15.969 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:15.969 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.969 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.969 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:15.969 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.917687+0000", 2023-10-01T09:46:15.969 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.8259999999999999e-06 2023-10-01T09:46:15.969 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.969 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.970 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:15.970 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.917695+0000", 2023-10-01T09:46:15.970 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3471e-05 2023-10-01T09:46:15.970 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.970 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.970 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:15.970 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.917709+0000", 2023-10-01T09:46:15.970 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.7110000000000001e-06 2023-10-01T09:46:15.970 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.971 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.971 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:15.971 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:51.917718+0000", 2023-10-01T09:46:15.971 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.14136868899999999 2023-10-01T09:46:15.971 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.971 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.971 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.971 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.059086+0000", 2023-10-01T09:46:15.971 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0066567579999999996 2023-10-01T09:46:15.972 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.972 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.972 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:15.972 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.065743+0000", 2023-10-01T09:46:15.972 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.037641165999999997 2023-10-01T09:46:15.972 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.972 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.972 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.972 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.103384+0000", 2023-10-01T09:46:15.972 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0066515949999999997 2023-10-01T09:46:15.973 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.973 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.973 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:15.973 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.110036+0000", 2023-10-01T09:46:15.973 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1033112119999999 2023-10-01T09:46:15.973 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.973 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.973 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.973 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.213347+0000", 2023-10-01T09:46:15.974 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053204439999999997 2023-10-01T09:46:15.974 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.974 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.974 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:15.974 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.218667+0000", 2023-10-01T09:46:15.974 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011333654 2023-10-01T09:46:15.974 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.974 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.974 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:15.974 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.230001+0000", 2023-10-01T09:46:15.975 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017911955 2023-10-01T09:46:15.975 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.975 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.975 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.975 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.247913+0000", 2023-10-01T09:46:15.975 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.02331948 2023-10-01T09:46:15.975 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.975 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.975 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.975 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.271233+0000", 2023-10-01T09:46:15.976 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00095119599999999996 2023-10-01T09:46:15.976 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.976 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.976 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:15.976 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.272184+0000", 2023-10-01T09:46:15.976 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0041481410000000002 2023-10-01T09:46:15.976 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.976 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.976 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:15.976 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.276332+0000", 2023-10-01T09:46:15.977 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.025753161 2023-10-01T09:46:15.977 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.977 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.977 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:15.977 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.302085+0000", 2023-10-01T09:46:15.977 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.384396798 2023-10-01T09:46:15.977 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.977 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:15.977 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.978 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.978 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.978 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:84395 10.7 10:e1c3a557:::benchmark_data_smithi138_573643_object1173:head [delete] snapc 0=[] ondisk+write+known_if_redirected e507)", 2023-10-01T09:46:15.978 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:41:52.267643+0000", 2023-10-01T09:46:15.978 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 263.62723749600002, 2023-10-01T09:46:15.978 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3845004649999999, 2023-10-01T09:46:15.978 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:15.978 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:15.978 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:15.978 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:15.979 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:15.979 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 84395 2023-10-01T09:46:15.979 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.979 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:15.979 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.979 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:15.979 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.267643+0000", 2023-10-01T09:46:15.979 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:15.979 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.979 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.980 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:15.980 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.267643+0000", 2023-10-01T09:46:15.980 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:46:15.980 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.980 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.980 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:15.980 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.267642+0000", 2023-10-01T09:46:15.980 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1279999999999999e-06 2023-10-01T09:46:15.980 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.980 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.981 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:15.981 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.267647+0000", 2023-10-01T09:46:15.981 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1406000000000001e-05 2023-10-01T09:46:15.981 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.981 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.981 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:15.981 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.267659+0000", 2023-10-01T09:46:15.981 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3650000000000003e-06 2023-10-01T09:46:15.981 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.981 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.982 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:15.982 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.267664+0000", 2023-10-01T09:46:15.982 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.022232548000000001 2023-10-01T09:46:15.982 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.982 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.982 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.982 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.289897+0000", 2023-10-01T09:46:15.982 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0069180439999999999 2023-10-01T09:46:15.982 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.983 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.983 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:15.983 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.296815+0000", 2023-10-01T09:46:15.983 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.079320033999999998 2023-10-01T09:46:15.983 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.983 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.983 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.983 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.376135+0000", 2023-10-01T09:46:15.983 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0089283620000000005 2023-10-01T09:46:15.983 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.984 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.984 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:15.984 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:52.385063+0000", 2023-10-01T09:46:15.984 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1459848030000002 2023-10-01T09:46:15.984 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.984 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.984 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.984 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.531048+0000", 2023-10-01T09:46:15.984 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0059038470000000003 2023-10-01T09:46:15.984 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.985 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.985 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:15.985 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.536952+0000", 2023-10-01T09:46:15.985 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013953999 2023-10-01T09:46:15.985 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.985 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.985 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:15.985 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.550906+0000", 2023-10-01T09:46:15.985 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.075835562999999995 2023-10-01T09:46:15.985 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.986 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.986 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.986 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.626741+0000", 2023-10-01T09:46:15.986 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00070105999999999999 2023-10-01T09:46:15.986 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.986 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.986 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:15.986 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.627442+0000", 2023-10-01T09:46:15.986 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0024119829999999999 2023-10-01T09:46:15.986 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.987 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.987 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.987 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.629854+0000", 2023-10-01T09:46:15.987 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003653987 2023-10-01T09:46:15.987 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.987 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.987 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:15.987 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.633508+0000", 2023-10-01T09:46:15.987 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018635448999999998 2023-10-01T09:46:15.987 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.988 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.988 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:15.988 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:54.652144+0000", 2023-10-01T09:46:15.988 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3845004649999999 2023-10-01T09:46:15.988 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.988 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:15.988 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.988 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.988 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.988 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:84590 10.7 10:e60c6a10:::benchmark_data_smithi138_573643_object1368:head [delete] snapc 0=[] ondisk+write+known_if_redirected e507)", 2023-10-01T09:46:15.989 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:41:53.107743+0000", 2023-10-01T09:46:15.989 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 262.78713763299999, 2023-10-01T09:46:15.989 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.361172818, 2023-10-01T09:46:15.989 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:15.989 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:15.989 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:15.989 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:15.989 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:15.989 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 84590 2023-10-01T09:46:15.990 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.990 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:15.990 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.990 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:15.990 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.107743+0000", 2023-10-01T09:46:15.990 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:15.990 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.990 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.990 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:15.991 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.107743+0000", 2023-10-01T09:46:15.991 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:46:15.991 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.991 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.991 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:15.991 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.107742+0000", 2023-10-01T09:46:15.991 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.9629999999999998e-06 2023-10-01T09:46:15.991 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.991 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.991 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:15.992 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.107750+0000", 2023-10-01T09:46:15.992 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5370999999999999e-05 2023-10-01T09:46:15.992 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.992 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.992 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:15.992 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.107765+0000", 2023-10-01T09:46:15.992 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6039999999999998e-06 2023-10-01T09:46:15.992 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.992 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.992 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:15.993 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.107771+0000", 2023-10-01T09:46:15.993 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.040748682000000001 2023-10-01T09:46:15.993 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.993 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.993 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.993 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.148520+0000", 2023-10-01T09:46:15.993 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0072026099999999999 2023-10-01T09:46:15.993 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.993 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.994 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:15.994 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.155723+0000", 2023-10-01T09:46:15.994 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.080724618999999997 2023-10-01T09:46:15.994 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.994 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.994 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.994 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.236447+0000", 2023-10-01T09:46:15.994 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0067676810000000002 2023-10-01T09:46:15.994 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.994 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.995 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:15.995 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:53.243215+0000", 2023-10-01T09:46:15.995 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.047975536 2023-10-01T09:46:15.995 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.995 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.995 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:15.995 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:55.291191+0000", 2023-10-01T09:46:15.995 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049557760000000003 2023-10-01T09:46:15.995 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.995 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.996 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:15.996 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:55.296146+0000", 2023-10-01T09:46:15.996 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010559013000000001 2023-10-01T09:46:15.996 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.996 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.996 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:15.996 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:55.306705+0000", 2023-10-01T09:46:15.996 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.113153753 2023-10-01T09:46:15.996 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.997 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.997 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.997 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:55.419859+0000", 2023-10-01T09:46:15.997 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001990669 2023-10-01T09:46:15.997 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.997 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.997 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:15.997 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:55.421850+0000", 2023-10-01T09:46:15.997 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.027924740999999999 2023-10-01T09:46:15.997 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.998 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.998 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:15.998 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:55.449775+0000", 2023-10-01T09:46:15.998 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0032274729999999998 2023-10-01T09:46:15.998 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.998 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.998 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:15.998 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:55.453002+0000", 2023-10-01T09:46:15.998 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015914006000000001 2023-10-01T09:46:15.999 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.999 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:15.999 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:15.999 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:41:55.468916+0000", 2023-10-01T09:46:15.999 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.361172818 2023-10-01T09:46:15.999 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.999 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:15.999 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:15.999 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:15.999 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.000 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:89177 10.7 10:eea504cb:::benchmark_data_smithi138_573643_object5955:head [delete] snapc 0=[] ondisk+write+known_if_redirected e515)", 2023-10-01T09:46:16.000 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:31.631446+0000", 2023-10-01T09:46:16.000 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 224.26343495399999, 2023-10-01T09:46:16.000 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3654005319999998, 2023-10-01T09:46:16.000 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:16.000 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:16.000 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:16.000 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:16.000 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:16.001 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 89177 2023-10-01T09:46:16.001 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.001 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:16.001 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.001 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:16.001 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:31.631446+0000", 2023-10-01T09:46:16.001 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:16.001 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.001 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.001 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:16.002 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:31.631446+0000", 2023-10-01T09:46:16.002 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:46:16.002 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.002 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.002 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:16.002 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:31.631445+0000", 2023-10-01T09:46:16.002 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7690000000000004e-06 2023-10-01T09:46:16.002 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.002 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.002 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:16.003 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:31.631450+0000", 2023-10-01T09:46:16.003 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.8950000000000004e-06 2023-10-01T09:46:16.003 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.003 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.003 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:16.003 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:31.631458+0000", 2023-10-01T09:46:16.003 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.4970000000000003e-06 2023-10-01T09:46:16.003 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.003 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.003 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:16.004 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:31.631463+0000", 2023-10-01T09:46:16.004 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.253742617 2023-10-01T09:46:16.004 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.004 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.004 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.004 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:31.885206+0000", 2023-10-01T09:46:16.004 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015710719000000001 2023-10-01T09:46:16.004 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.004 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.004 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:16.005 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:31.900916+0000", 2023-10-01T09:46:16.005 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.13736179000000001 2023-10-01T09:46:16.005 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.005 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.005 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.005 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.038278+0000", 2023-10-01T09:46:16.005 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0066879269999999998 2023-10-01T09:46:16.005 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.005 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.005 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:16.006 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.044966+0000", 2023-10-01T09:46:16.006 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.8368142540000001 2023-10-01T09:46:16.006 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.006 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.006 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.006 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.881780+0000", 2023-10-01T09:46:16.006 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0064896520000000003 2023-10-01T09:46:16.006 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.006 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.007 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:16.007 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.888270+0000", 2023-10-01T09:46:16.007 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017476897000000002 2023-10-01T09:46:16.007 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.007 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.007 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:16.007 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.905747+0000", 2023-10-01T09:46:16.007 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.042333185000000002 2023-10-01T09:46:16.007 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.007 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.008 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:16.008 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.948080+0000", 2023-10-01T09:46:16.008 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.024815071000000001 2023-10-01T09:46:16.008 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.008 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.008 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.008 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.972895+0000", 2023-10-01T09:46:16.008 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0026232920000000002 2023-10-01T09:46:16.008 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.009 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.009 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.009 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.975518+0000", 2023-10-01T09:46:16.009 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003067635 2023-10-01T09:46:16.009 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.009 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.009 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:16.009 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.978586+0000", 2023-10-01T09:46:16.009 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018260506999999999 2023-10-01T09:46:16.009 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.010 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.010 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:16.010 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.996846+0000", 2023-10-01T09:46:16.010 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3654005319999998 2023-10-01T09:46:16.010 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.010 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:16.010 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.010 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.010 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.011 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:89271 10.7 10:ed09a8e8:::benchmark_data_smithi138_573643_object6049:head [delete] snapc 0=[] ondisk+write+known_if_redirected e515)", 2023-10-01T09:46:16.011 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:32.043023+0000", 2023-10-01T09:46:16.011 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 223.851858158, 2023-10-01T09:46:16.011 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3710276119999998, 2023-10-01T09:46:16.011 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:16.011 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:16.011 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:16.011 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:16.011 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:16.011 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 89271 2023-10-01T09:46:16.012 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.012 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:16.012 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.012 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:16.012 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.043023+0000", 2023-10-01T09:46:16.012 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:16.012 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.012 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.012 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:16.013 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.043023+0000", 2023-10-01T09:46:16.013 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:46:16.013 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.013 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.013 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:16.013 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.043022+0000", 2023-10-01T09:46:16.013 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1950000000000003e-06 2023-10-01T09:46:16.013 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.013 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.013 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:16.014 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.043030+0000", 2023-10-01T09:46:16.014 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3599e-05 2023-10-01T09:46:16.014 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.014 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.014 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:16.014 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.043043+0000", 2023-10-01T09:46:16.014 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9909999999999999e-06 2023-10-01T09:46:16.014 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.014 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.015 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:16.015 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.043049+0000", 2023-10-01T09:46:16.015 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021080551999999999 2023-10-01T09:46:16.015 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.015 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.015 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.015 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.064130+0000", 2023-10-01T09:46:16.015 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0064048600000000001 2023-10-01T09:46:16.015 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.015 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.016 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:16.016 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.070535+0000", 2023-10-01T09:46:16.016 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.088875039000000003 2023-10-01T09:46:16.016 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.016 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.016 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.016 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.159410+0000", 2023-10-01T09:46:16.016 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0061216200000000004 2023-10-01T09:46:16.016 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.016 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.017 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:16.017 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.165531+0000", 2023-10-01T09:46:16.017 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1084257179999999 2023-10-01T09:46:16.017 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.017 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.017 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.017 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:34.273957+0000", 2023-10-01T09:46:16.017 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0052392139999999999 2023-10-01T09:46:16.017 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.017 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.018 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:16.018 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:34.279196+0000", 2023-10-01T09:46:16.018 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010649965000000001 2023-10-01T09:46:16.018 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.018 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.018 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:16.018 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:34.289846+0000", 2023-10-01T09:46:16.018 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020544723000000001 2023-10-01T09:46:16.018 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.018 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.019 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.019 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:34.310391+0000", 2023-10-01T09:46:16.019 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0020843649999999999 2023-10-01T09:46:16.019 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.019 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.019 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.019 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:34.312475+0000", 2023-10-01T09:46:16.019 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00060018400000000003 2023-10-01T09:46:16.019 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.019 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.020 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:16.020 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:34.313076+0000", 2023-10-01T09:46:16.020 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0030971739999999998 2023-10-01T09:46:16.020 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.020 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.020 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:16.020 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:34.316173+0000", 2023-10-01T09:46:16.020 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.097877597999999996 2023-10-01T09:46:16.020 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.021 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.021 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:16.021 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:34.414050+0000", 2023-10-01T09:46:16.021 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3710276119999998 2023-10-01T09:46:16.021 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.021 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:16.021 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.021 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.021 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.021 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:89396 10.7 10:e1125be3:::benchmark_data_smithi138_573643_object6174:head [delete] snapc 0=[] ondisk+write+known_if_redirected e515)", 2023-10-01T09:46:16.022 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:32.824667+0000", 2023-10-01T09:46:16.022 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 223.070213741, 2023-10-01T09:46:16.022 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.347798166, 2023-10-01T09:46:16.022 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:16.022 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:16.022 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:16.022 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:16.022 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:16.022 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 89396 2023-10-01T09:46:16.023 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.023 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:16.023 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.023 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:16.023 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.824667+0000", 2023-10-01T09:46:16.023 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:16.023 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.023 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.023 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:16.023 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.824667+0000", 2023-10-01T09:46:16.024 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:46:16.024 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.024 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.024 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:16.024 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.824666+0000", 2023-10-01T09:46:16.024 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.709e-06 2023-10-01T09:46:16.024 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.024 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.024 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:16.024 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.824670+0000", 2023-10-01T09:46:16.025 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3810000000000004e-06 2023-10-01T09:46:16.025 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.025 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.025 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:16.025 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.824676+0000", 2023-10-01T09:46:16.025 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.6370000000000001e-06 2023-10-01T09:46:16.025 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.025 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.025 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:16.025 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.824680+0000", 2023-10-01T09:46:16.026 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.053135703999999999 2023-10-01T09:46:16.026 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.026 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.026 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.026 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.877816+0000", 2023-10-01T09:46:16.026 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0061187979999999999 2023-10-01T09:46:16.026 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.026 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.026 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:16.026 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.883934+0000", 2023-10-01T09:46:16.027 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.035601094999999999 2023-10-01T09:46:16.027 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.027 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.027 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.027 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.919536+0000", 2023-10-01T09:46:16.027 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.006011644 2023-10-01T09:46:16.027 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.027 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.027 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:16.027 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.925547+0000", 2023-10-01T09:46:16.028 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.142748359 2023-10-01T09:46:16.028 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.028 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.028 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.028 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.068296+0000", 2023-10-01T09:46:16.028 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0060258730000000002 2023-10-01T09:46:16.028 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.028 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.028 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:16.029 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.074321+0000", 2023-10-01T09:46:16.029 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010844006 2023-10-01T09:46:16.029 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.029 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.029 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:16.029 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.085165+0000", 2023-10-01T09:46:16.029 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.038660732000000003 2023-10-01T09:46:16.029 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.029 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.029 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:16.030 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.123826+0000", 2023-10-01T09:46:16.030 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.026945373000000002 2023-10-01T09:46:16.030 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.030 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.030 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.030 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.150771+0000", 2023-10-01T09:46:16.030 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001841306 2023-10-01T09:46:16.030 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.030 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.031 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.031 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.152613+0000", 2023-10-01T09:46:16.031 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0030554810000000001 2023-10-01T09:46:16.031 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.031 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.031 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:16.031 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.155668+0000", 2023-10-01T09:46:16.031 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.016797000999999999 2023-10-01T09:46:16.031 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.031 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.032 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:16.032 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.172465+0000", 2023-10-01T09:46:16.032 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.347798166 2023-10-01T09:46:16.032 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.032 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:16.032 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.032 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.032 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.032 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:89397 10.7 10:ee22ca8a:::benchmark_data_smithi138_573643_object6175:head [delete] snapc 0=[] ondisk+write+known_if_redirected e515)", 2023-10-01T09:46:16.032 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:32.835289+0000", 2023-10-01T09:46:16.033 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 223.05959172600001, 2023-10-01T09:46:16.033 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3648484750000001, 2023-10-01T09:46:16.033 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:16.033 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:16.033 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:16.033 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:16.033 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:16.033 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 89397 2023-10-01T09:46:16.033 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.034 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:16.034 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.034 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:16.034 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.835289+0000", 2023-10-01T09:46:16.034 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:16.034 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.034 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.034 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:16.034 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.835289+0000", 2023-10-01T09:46:16.034 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:16.035 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.035 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.035 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:16.035 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.835288+0000", 2023-10-01T09:46:16.035 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.6080000000000003e-06 2023-10-01T09:46:16.035 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.035 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.035 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:16.035 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.835293+0000", 2023-10-01T09:46:16.036 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1090000000000001e-05 2023-10-01T09:46:16.036 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.036 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.036 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:16.036 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.835305+0000", 2023-10-01T09:46:16.036 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.9899999999999997e-06 2023-10-01T09:46:16.036 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.036 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.036 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:16.037 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.835310+0000", 2023-10-01T09:46:16.037 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.049359487000000001 2023-10-01T09:46:16.037 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.037 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.037 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.037 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.884669+0000", 2023-10-01T09:46:16.037 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0059782100000000003 2023-10-01T09:46:16.037 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.037 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.037 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:16.038 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.890647+0000", 2023-10-01T09:46:16.038 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.058386782999999998 2023-10-01T09:46:16.038 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.038 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.038 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.038 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.949034+0000", 2023-10-01T09:46:16.038 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0060751240000000003 2023-10-01T09:46:16.038 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.038 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.039 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:16.039 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:32.955109+0000", 2023-10-01T09:46:16.039 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1695743599999999 2023-10-01T09:46:16.039 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.039 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.039 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.039 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.124683+0000", 2023-10-01T09:46:16.039 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050604760000000004 2023-10-01T09:46:16.039 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.039 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.040 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:16.040 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.129744+0000", 2023-10-01T09:46:16.040 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010753151000000001 2023-10-01T09:46:16.040 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.040 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.040 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:16.040 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.140497+0000", 2023-10-01T09:46:16.040 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.03398205 2023-10-01T09:46:16.040 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.041 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.041 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.041 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.174479+0000", 2023-10-01T09:46:16.041 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0020049260000000002 2023-10-01T09:46:16.041 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.041 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.041 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.041 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.176484+0000", 2023-10-01T09:46:16.041 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00065794699999999998 2023-10-01T09:46:16.041 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.042 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.042 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:16.042 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.177142+0000", 2023-10-01T09:46:16.042 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003463219 2023-10-01T09:46:16.042 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.042 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.042 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:16.042 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.180605+0000", 2023-10-01T09:46:16.042 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019532352999999999 2023-10-01T09:46:16.042 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.043 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.043 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:16.043 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.200138+0000", 2023-10-01T09:46:16.043 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3648484750000001 2023-10-01T09:46:16.043 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.043 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:16.043 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.043 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.043 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.044 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:89400 10.7 10:eb54713e:::benchmark_data_smithi138_573643_object6178:head [delete] snapc 0=[] ondisk+write+known_if_redirected e515)", 2023-10-01T09:46:16.044 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:33.050115+0000", 2023-10-01T09:46:16.044 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 222.844766279, 2023-10-01T09:46:16.044 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3737178779999999, 2023-10-01T09:46:16.044 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:16.044 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:16.044 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:16.044 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:16.044 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:16.044 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 89400 2023-10-01T09:46:16.045 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.045 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:16.045 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.045 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:16.045 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.050115+0000", 2023-10-01T09:46:16.045 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:16.045 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.045 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.046 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:16.046 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.050115+0000", 2023-10-01T09:46:16.046 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:16.046 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.046 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.046 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:16.046 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.050113+0000", 2023-10-01T09:46:16.046 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.994e-06 2023-10-01T09:46:16.046 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.046 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.047 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:16.047 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.050121+0000", 2023-10-01T09:46:16.047 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4549000000000001e-05 2023-10-01T09:46:16.047 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.047 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.047 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:16.047 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.050136+0000", 2023-10-01T09:46:16.047 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3500000000000002e-06 2023-10-01T09:46:16.047 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.047 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.048 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:16.048 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.050142+0000", 2023-10-01T09:46:16.048 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.057114188000000003 2023-10-01T09:46:16.048 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.048 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.048 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.048 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.107256+0000", 2023-10-01T09:46:16.048 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0063063980000000004 2023-10-01T09:46:16.048 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.048 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.049 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:16.049 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.113563+0000", 2023-10-01T09:46:16.049 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.049916266000000001 2023-10-01T09:46:16.049 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.049 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.049 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.049 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.163479+0000", 2023-10-01T09:46:16.049 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0063858379999999996 2023-10-01T09:46:16.049 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.049 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.050 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:16.050 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.169865+0000", 2023-10-01T09:46:16.050 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1537792119999999 2023-10-01T09:46:16.050 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.050 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.050 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.050 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.323644+0000", 2023-10-01T09:46:16.050 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0054735599999999997 2023-10-01T09:46:16.050 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.050 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.051 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:16.051 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.329117+0000", 2023-10-01T09:46:16.051 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011439875 2023-10-01T09:46:16.051 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.051 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.051 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:16.051 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.340557+0000", 2023-10-01T09:46:16.051 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.062158501999999997 2023-10-01T09:46:16.051 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.051 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.052 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:16.052 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.402716+0000", 2023-10-01T09:46:16.052 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019346630000000001 2023-10-01T09:46:16.052 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.052 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.052 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.052 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.404650+0000", 2023-10-01T09:46:16.052 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.001870249 2023-10-01T09:46:16.052 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.052 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.053 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.053 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.406521+0000", 2023-10-01T09:46:16.053 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003162593 2023-10-01T09:46:16.053 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.053 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.053 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:16.053 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.409683+0000", 2023-10-01T09:46:16.053 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014149201 2023-10-01T09:46:16.053 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.053 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.054 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:16.054 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.423832+0000", 2023-10-01T09:46:16.054 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3737178779999999 2023-10-01T09:46:16.054 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.054 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:16.054 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.054 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.054 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.054 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:89447 10.7 10:edf282a1:::benchmark_data_smithi138_573643_object6225:head [delete] snapc 0=[] ondisk+write+known_if_redirected e515)", 2023-10-01T09:46:16.054 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:33.439669+0000", 2023-10-01T09:46:16.055 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 222.45521146300001, 2023-10-01T09:46:16.055 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4431008699999999, 2023-10-01T09:46:16.055 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:16.055 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:16.055 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:16.055 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:16.055 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:16.055 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 89447 2023-10-01T09:46:16.055 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.055 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:16.056 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.056 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:16.056 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.439669+0000", 2023-10-01T09:46:16.056 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:16.056 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.056 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.056 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:16.056 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.439669+0000", 2023-10-01T09:46:16.056 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:16.056 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.057 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.057 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:16.057 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.439668+0000", 2023-10-01T09:46:16.057 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.9450000000000004e-06 2023-10-01T09:46:16.057 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.057 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.057 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:16.057 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.439675+0000", 2023-10-01T09:46:16.057 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1780000000000001e-05 2023-10-01T09:46:16.058 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.058 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.058 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:16.058 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.439687+0000", 2023-10-01T09:46:16.058 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1229999999999997e-06 2023-10-01T09:46:16.058 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.058 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.058 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:16.058 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.439693+0000", 2023-10-01T09:46:16.059 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.073314367000000005 2023-10-01T09:46:16.059 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.059 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.059 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.059 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.513007+0000", 2023-10-01T09:46:16.059 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011878135999999999 2023-10-01T09:46:16.059 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.059 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.059 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:16.059 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.524885+0000", 2023-10-01T09:46:16.060 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.11593255600000001 2023-10-01T09:46:16.060 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.060 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.060 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.060 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.640818+0000", 2023-10-01T09:46:16.060 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0068465610000000001 2023-10-01T09:46:16.060 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.060 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.060 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:16.060 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.647665+0000", 2023-10-01T09:46:16.061 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1131527029999999 2023-10-01T09:46:16.061 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.061 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.061 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.061 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.760817+0000", 2023-10-01T09:46:16.061 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0048512980000000004 2023-10-01T09:46:16.061 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.061 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.061 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:16.061 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.765669+0000", 2023-10-01T09:46:16.062 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010077525 2023-10-01T09:46:16.062 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.062 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.062 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:16.062 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.775746+0000", 2023-10-01T09:46:16.062 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.084089967000000002 2023-10-01T09:46:16.062 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.062 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.062 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.063 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.859836+0000", 2023-10-01T09:46:16.063 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0026252419999999999 2023-10-01T09:46:16.063 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.063 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.063 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.063 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.862461+0000", 2023-10-01T09:46:16.063 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0025979390000000001 2023-10-01T09:46:16.063 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.063 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.063 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:16.064 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.865059+0000", 2023-10-01T09:46:16.064 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0031410549999999998 2023-10-01T09:46:16.064 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.064 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.064 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:16.064 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.868200+0000", 2023-10-01T09:46:16.064 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014570025 2023-10-01T09:46:16.064 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.064 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.064 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:16.065 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.882770+0000", 2023-10-01T09:46:16.065 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4431008699999999 2023-10-01T09:46:16.065 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.065 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:16.065 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.065 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.065 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.065 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:89474 10.7 10:e1ad9d28:::benchmark_data_smithi138_573643_object6252:head [delete] snapc 0=[] ondisk+write+known_if_redirected e515)", 2023-10-01T09:46:16.065 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:33.512185+0000", 2023-10-01T09:46:16.065 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 222.382696229, 2023-10-01T09:46:16.066 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3938316560000001, 2023-10-01T09:46:16.066 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:16.066 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:16.066 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:16.066 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:16.066 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:16.066 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 89474 2023-10-01T09:46:16.066 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.066 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:16.067 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.067 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:16.067 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.512185+0000", 2023-10-01T09:46:16.067 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:16.067 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.067 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.067 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:16.067 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.512185+0000", 2023-10-01T09:46:16.067 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:16.067 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.068 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.068 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:16.068 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.512183+0000", 2023-10-01T09:46:16.068 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3699999999999995e-06 2023-10-01T09:46:16.068 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.068 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.068 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:16.068 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.512191+0000", 2023-10-01T09:46:16.068 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.1129e-05 2023-10-01T09:46:16.069 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.069 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.069 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:16.069 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.512202+0000", 2023-10-01T09:46:16.069 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6390000000000003e-06 2023-10-01T09:46:16.069 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.069 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.069 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:16.069 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.512209+0000", 2023-10-01T09:46:16.069 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.205913346 2023-10-01T09:46:16.070 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.070 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.070 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.070 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.718122+0000", 2023-10-01T09:46:16.070 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0084707830000000008 2023-10-01T09:46:16.070 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.070 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.070 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:16.070 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.726593+0000", 2023-10-01T09:46:16.071 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.054261982 2023-10-01T09:46:16.071 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.071 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.071 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.071 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.780855+0000", 2023-10-01T09:46:16.071 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0073837190000000004 2023-10-01T09:46:16.071 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.071 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.071 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:16.072 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.788239+0000", 2023-10-01T09:46:16.072 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0424057210000002 2023-10-01T09:46:16.072 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.072 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.072 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.072 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.830645+0000", 2023-10-01T09:46:16.072 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0065571279999999997 2023-10-01T09:46:16.072 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.072 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.072 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:16.073 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.837202+0000", 2023-10-01T09:46:16.073 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.012561968 2023-10-01T09:46:16.073 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.073 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.073 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:16.073 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.849764+0000", 2023-10-01T09:46:16.073 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014658833 2023-10-01T09:46:16.073 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.073 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.073 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.074 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.864423+0000", 2023-10-01T09:46:16.074 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018985061000000001 2023-10-01T09:46:16.074 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.074 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.074 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:16.074 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.883408+0000", 2023-10-01T09:46:16.074 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0019760559999999999 2023-10-01T09:46:16.074 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.074 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.074 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.075 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.885384+0000", 2023-10-01T09:46:16.075 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033052480000000002 2023-10-01T09:46:16.075 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.075 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.075 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:16.075 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.888689+0000", 2023-10-01T09:46:16.075 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017327312000000001 2023-10-01T09:46:16.075 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.075 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.075 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:16.076 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.906016+0000", 2023-10-01T09:46:16.076 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3938316560000001 2023-10-01T09:46:16.076 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.076 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:16.076 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.076 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.076 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.076 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:89499 10.7 10:e91ab4b2:::benchmark_data_smithi138_573643_object6277:head [delete] snapc 0=[] ondisk+write+known_if_redirected e515)", 2023-10-01T09:46:16.076 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:33.619904+0000", 2023-10-01T09:46:16.076 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 222.27497708199999, 2023-10-01T09:46:16.077 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5473014350000001, 2023-10-01T09:46:16.077 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:16.077 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:16.077 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:16.077 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:16.077 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:16.077 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 89499 2023-10-01T09:46:16.077 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.077 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:16.077 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.078 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:16.078 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.619904+0000", 2023-10-01T09:46:16.078 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:16.078 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.078 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.078 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:16.078 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.619904+0000", 2023-10-01T09:46:16.078 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:46:16.078 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.079 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.079 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:16.079 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.619902+0000", 2023-10-01T09:46:16.079 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3939999999999996e-06 2023-10-01T09:46:16.079 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.079 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.079 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:16.079 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.619910+0000", 2023-10-01T09:46:16.079 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.5328e-05 2023-10-01T09:46:16.079 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.080 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.080 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:16.080 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.619925+0000", 2023-10-01T09:46:16.080 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6720000000000004e-06 2023-10-01T09:46:16.080 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.080 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.080 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:16.080 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.619933+0000", 2023-10-01T09:46:16.080 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.16911494199999999 2023-10-01T09:46:16.080 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.081 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.081 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.081 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.789048+0000", 2023-10-01T09:46:16.081 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0063828660000000001 2023-10-01T09:46:16.081 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.081 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.081 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:16.081 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.795431+0000", 2023-10-01T09:46:16.081 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.032653173000000001 2023-10-01T09:46:16.082 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.082 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.082 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.082 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.828084+0000", 2023-10-01T09:46:16.082 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0074494390000000004 2023-10-01T09:46:16.082 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.082 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.082 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:16.082 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:33.835533+0000", 2023-10-01T09:46:16.083 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1295964230000002 2023-10-01T09:46:16.083 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.083 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.083 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.083 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.965130+0000", 2023-10-01T09:46:16.083 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0056800949999999996 2023-10-01T09:46:16.083 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.083 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.083 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:16.083 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.970810+0000", 2023-10-01T09:46:16.084 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011739575 2023-10-01T09:46:16.084 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.084 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.084 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:16.084 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:35.982550+0000", 2023-10-01T09:46:16.084 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.11112636200000001 2023-10-01T09:46:16.084 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.084 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.084 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:16.085 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:36.093676+0000", 2023-10-01T09:46:16.085 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.044613610999999997 2023-10-01T09:46:16.085 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.085 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.085 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.085 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:36.138290+0000", 2023-10-01T09:46:16.085 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0049160799999999998 2023-10-01T09:46:16.085 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.085 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.085 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.086 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:36.143206+0000", 2023-10-01T09:46:16.086 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0057269879999999997 2023-10-01T09:46:16.086 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.086 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.086 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:16.086 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:36.148933+0000", 2023-10-01T09:46:16.086 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018272620999999999 2023-10-01T09:46:16.086 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.086 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.086 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:16.087 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:36.167205+0000", 2023-10-01T09:46:16.087 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.5473014350000001 2023-10-01T09:46:16.087 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.087 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:16.087 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.087 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.087 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.087 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:90747 10.7 10:e813022c:::benchmark_data_smithi138_573643_object7525:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:46:16.087 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:42.017599+0000", 2023-10-01T09:46:16.087 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 213.87728207200001, 2023-10-01T09:46:16.088 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4074143929999998, 2023-10-01T09:46:16.088 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:16.088 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:16.088 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:16.088 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:16.088 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:16.088 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 90747 2023-10-01T09:46:16.088 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.088 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:16.088 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.089 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:16.089 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.017599+0000", 2023-10-01T09:46:16.089 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:16.089 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.089 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.089 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:16.089 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.017599+0000", 2023-10-01T09:46:16.089 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:16.089 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.089 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.090 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:16.090 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.017598+0000", 2023-10-01T09:46:16.090 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9020000000000004e-06 2023-10-01T09:46:16.090 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.090 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.090 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:16.090 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.017603+0000", 2023-10-01T09:46:16.090 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3377000000000001e-05 2023-10-01T09:46:16.090 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.090 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.091 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:16.091 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.017617+0000", 2023-10-01T09:46:16.091 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3740000000000003e-06 2023-10-01T09:46:16.091 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.091 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.091 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:16.091 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.017623+0000", 2023-10-01T09:46:16.091 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.058251325 2023-10-01T09:46:16.091 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.091 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.092 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.092 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.075874+0000", 2023-10-01T09:46:16.092 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.007801166 2023-10-01T09:46:16.092 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.092 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.092 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:16.092 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.083676+0000", 2023-10-01T09:46:16.092 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.071429468999999995 2023-10-01T09:46:16.092 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.092 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.093 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.093 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.155105+0000", 2023-10-01T09:46:16.093 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0091262259999999994 2023-10-01T09:46:16.093 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.093 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.093 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:16.093 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:42.164231+0000", 2023-10-01T09:46:16.094 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0423366860000001 2023-10-01T09:46:16.094 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.094 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.094 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.094 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.206568+0000", 2023-10-01T09:46:16.094 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050223919999999997 2023-10-01T09:46:16.094 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.094 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.094 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:16.095 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.211590+0000", 2023-10-01T09:46:16.095 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010629347000000001 2023-10-01T09:46:16.095 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.095 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.095 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:16.095 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.222220+0000", 2023-10-01T09:46:16.095 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015753208000000001 2023-10-01T09:46:16.095 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.096 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.096 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.096 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.237973+0000", 2023-10-01T09:46:16.096 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0017956230000000001 2023-10-01T09:46:16.096 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.096 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.096 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.096 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.239769+0000", 2023-10-01T09:46:16.096 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.15847092199999999 2023-10-01T09:46:16.097 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.097 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.097 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:16.097 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.398240+0000", 2023-10-01T09:46:16.097 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0029837710000000001 2023-10-01T09:46:16.097 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.097 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.098 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:16.098 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.401223+0000", 2023-10-01T09:46:16.098 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.023789879 2023-10-01T09:46:16.098 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.098 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.098 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:16.098 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.425013+0000", 2023-10-01T09:46:16.098 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.4074143929999998 2023-10-01T09:46:16.098 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.098 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:16.099 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.099 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.099 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.099 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:91010 10.7 10:e24cec84:::benchmark_data_smithi138_573643_object7788:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:46:16.099 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:44.115177+0000", 2023-10-01T09:46:16.099 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 211.779704278, 2023-10-01T09:46:16.099 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3729735769999998, 2023-10-01T09:46:16.099 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:16.099 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:16.099 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:16.100 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:16.100 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:16.100 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 91010 2023-10-01T09:46:16.100 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.100 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:16.100 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.100 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:16.100 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.115177+0000", 2023-10-01T09:46:16.100 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:16.101 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.101 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.101 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:16.101 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.115177+0000", 2023-10-01T09:46:16.101 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:46:16.101 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.101 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.101 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:16.101 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.115175+0000", 2023-10-01T09:46:16.101 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1519999999999992e-06 2023-10-01T09:46:16.102 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.102 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.102 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:16.102 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.115183+0000", 2023-10-01T09:46:16.102 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0869e-05 2023-10-01T09:46:16.102 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.102 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.102 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:16.102 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.115194+0000", 2023-10-01T09:46:16.103 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.6340000000000003e-06 2023-10-01T09:46:16.103 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.103 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.103 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:16.103 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.115201+0000", 2023-10-01T09:46:16.103 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.114763752 2023-10-01T09:46:16.103 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.103 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.103 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.104 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.229964+0000", 2023-10-01T09:46:16.104 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0060933430000000002 2023-10-01T09:46:16.104 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.104 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.104 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:16.104 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.236058+0000", 2023-10-01T09:46:16.104 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.027097646999999999 2023-10-01T09:46:16.104 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.104 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.105 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.105 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.263155+0000", 2023-10-01T09:46:16.105 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0061927689999999999 2023-10-01T09:46:16.105 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.105 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.105 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:16.105 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.269348+0000", 2023-10-01T09:46:16.105 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1374383020000001 2023-10-01T09:46:16.105 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.105 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.106 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.106 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:46.406786+0000", 2023-10-01T09:46:16.106 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0056290510000000004 2023-10-01T09:46:16.106 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.106 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.106 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:16.106 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:46.412415+0000", 2023-10-01T09:46:16.106 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010594042 2023-10-01T09:46:16.106 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.107 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.107 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:16.107 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:46.423009+0000", 2023-10-01T09:46:16.107 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.034031065999999999 2023-10-01T09:46:16.107 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.107 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.107 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:16.107 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:46.457041+0000", 2023-10-01T09:46:16.107 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0093942149999999992 2023-10-01T09:46:16.107 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.108 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.108 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.108 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:46.466435+0000", 2023-10-01T09:46:16.108 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0038113349999999999 2023-10-01T09:46:16.108 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.108 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.108 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.108 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:46.470246+0000", 2023-10-01T09:46:16.108 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0029677430000000001 2023-10-01T09:46:16.108 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.109 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.109 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:16.109 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:46.473214+0000", 2023-10-01T09:46:16.109 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014936342999999999 2023-10-01T09:46:16.109 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.109 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.109 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:16.109 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:46.488150+0000", 2023-10-01T09:46:16.109 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3729735769999998 2023-10-01T09:46:16.110 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.110 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:16.110 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.110 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.110 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.110 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:91163 10.7 10:e362cda5:::benchmark_data_smithi138_573643_object7941:head [delete] snapc 0=[] ondisk+write+known_if_redirected e517)", 2023-10-01T09:46:16.110 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:42:44.944821+0000", 2023-10-01T09:46:16.110 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 210.950059439, 2023-10-01T09:46:16.110 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3391028170000001, 2023-10-01T09:46:16.111 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:46:16.111 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:16.111 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:46:16.111 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:46:16.111 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:16.111 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 91163 2023-10-01T09:46:16.111 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.111 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:46:16.111 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.111 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:46:16.112 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.944821+0000", 2023-10-01T09:46:16.112 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:46:16.112 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.112 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.112 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:46:16.112 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.944821+0000", 2023-10-01T09:46:16.112 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999971 2023-10-01T09:46:16.112 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.112 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.112 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:46:16.113 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.944819+0000", 2023-10-01T09:46:16.113 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.3142999999999999e-05 2023-10-01T09:46:16.113 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.113 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.113 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:46:16.113 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.944832+0000", 2023-10-01T09:46:16.113 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9040999999999999e-05 2023-10-01T09:46:16.113 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.113 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.113 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:46:16.114 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.944851+0000", 2023-10-01T09:46:16.114 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.9620000000000005e-06 2023-10-01T09:46:16.114 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.114 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.114 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:46:16.114 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:44.944861+0000", 2023-10-01T09:46:16.114 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.075597414000000002 2023-10-01T09:46:16.114 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.114 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.114 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.115 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:45.020458+0000", 2023-10-01T09:46:16.115 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0064397860000000003 2023-10-01T09:46:16.115 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.115 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.115 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for missing object", 2023-10-01T09:46:16.115 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:45.026898+0000", 2023-10-01T09:46:16.115 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.085934413000000001 2023-10-01T09:46:16.115 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.115 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.116 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.116 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:45.112832+0000", 2023-10-01T09:46:16.116 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0061216839999999996 2023-10-01T09:46:16.116 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.116 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.116 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:46:16.116 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:45.118954+0000", 2023-10-01T09:46:16.116 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.9978237829999999 2023-10-01T09:46:16.116 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.117 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.117 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:46:16.117 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:47.116778+0000", 2023-10-01T09:46:16.117 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0051836950000000003 2023-10-01T09:46:16.117 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.117 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.117 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:46:16.117 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:47.121962+0000", 2023-10-01T09:46:16.117 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010597521 2023-10-01T09:46:16.117 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.118 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.118 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:46:16.118 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:47.132559+0000", 2023-10-01T09:46:16.118 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.063133931000000004 2023-10-01T09:46:16.118 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.118 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.118 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:46:16.118 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:47.195693+0000", 2023-10-01T09:46:16.118 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.064310223999999999 2023-10-01T09:46:16.118 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.119 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.119 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.119 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:47.260003+0000", 2023-10-01T09:46:16.119 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0027281689999999999 2023-10-01T09:46:16.119 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.119 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.119 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:16.119 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:47.262731+0000", 2023-10-01T09:46:16.119 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0035288730000000001 2023-10-01T09:46:16.119 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.120 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.120 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:46:16.120 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:47.266260+0000", 2023-10-01T09:46:16.120 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017663907999999999 2023-10-01T09:46:16.120 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:46:16.120 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:46:16.120 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "done", 2023-10-01T09:46:16.120 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:42:47.283924+0000", 2023-10-01T09:46:16.120 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3391028170000001 2023-10-01T09:46:16.120 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.121 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:16.121 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.121 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:46:16.121 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:46:16.121 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:46:16.121 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:16.149 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:16.194 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:16.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:16.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:16.241 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:46:16.298 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:16.330 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:16.442 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:16 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/319755888' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:46:16.443 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:16 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/319755888' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:46:16.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:16 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/319755888' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:46:16.691 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:46:16.935 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:16.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:16.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:17.038 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:17.069 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:17.122 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:17.173 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:17.233 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:17.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:17.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:17.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:17 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:17.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:17 smithi138 ceph-mon[192513]: failed to encode map e544 with expected crc 2023-10-01T09:46:17.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:17 smithi138 ceph-mon[192513]: failed to encode map e544 with expected crc 2023-10-01T09:46:17.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:17 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/319755888' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:46:17.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:17 smithi138 ceph-mon[192513]: osdmap e544: 8 total, 8 up, 7 in 2023-10-01T09:46:17.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:17 smithi138 ceph-mon[192513]: pgmap v1612: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 172 op/s; 6268/135099 objects degraded (4.640%); 2750/135099 objects misplaced (2.036%); 592 KiB/s, 9 objects/s recovering 2023-10-01T09:46:17.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:17 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:17.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:17 smithi138 ceph-mon[185886]: failed to encode map e544 with expected crc 2023-10-01T09:46:17.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:17 smithi138 ceph-mon[185886]: failed to encode map e544 with expected crc 2023-10-01T09:46:17.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:17 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/319755888' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:46:17.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:17 smithi138 ceph-mon[185886]: osdmap e544: 8 total, 8 up, 7 in 2023-10-01T09:46:17.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:17 smithi138 ceph-mon[185886]: pgmap v1612: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 172 op/s; 6268/135099 objects degraded (4.640%); 2750/135099 objects misplaced (2.036%); 592 KiB/s, 9 objects/s recovering 2023-10-01T09:46:17.346 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:17.374 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:17.385 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:46:17.386 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:46:17.437 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:17.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:17.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:17.544 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:17.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:17 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:17.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:17 smithi190 ceph-mon[160435]: failed to encode map e544 with expected crc 2023-10-01T09:46:17.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:17 smithi190 ceph-mon[160435]: failed to encode map e544 with expected crc 2023-10-01T09:46:17.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:17 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/319755888' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:46:17.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:17 smithi190 ceph-mon[160435]: osdmap e544: 8 total, 8 up, 7 in 2023-10-01T09:46:17.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:17 smithi190 ceph-mon[160435]: pgmap v1612: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 172 op/s; 6268/135099 objects degraded (4.640%); 2750/135099 objects misplaced (2.036%); 592 KiB/s, 9 objects/s recovering 2023-10-01T09:46:17.573 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:17.625 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:17.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:17.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:17.729 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:17.757 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:17.822 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:17.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:17.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:17.928 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:46:17.957 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:18.152 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:46:18.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:18.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:46:18.251 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:18 smithi138 ceph-mon[185886]: from='client.47279 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:18.251 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:18 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6268/135099 objects degraded (4.640%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:18.252 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:18 smithi138 ceph-mon[192513]: from='client.47279 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:18.252 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:18 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6268/135099 objects degraded (4.640%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:18.256 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:18.286 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:18.338 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:18.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:18.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:18.443 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:18.469 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:18.472 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1612,"stamp":"2023-10-01T09:46:16.357027+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2954172363,"num_objects":45033,"num_object_clones":0,"num_object_copies":135099,"num_objects_missing_on_primary":1256,"num_objects_missing":1256,"num_objects_degraded":6268,"num_objects_misplaced":2750,"num_objects_unfound":0,"num_objects_dirty":45033,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":122541,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9908,"num_bytes_recovered":636191759,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118605,"ondisk_log_size":118605,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14368276,"kb_used_data":9266588,"kb_used_omap":39,"kb_used_meta":5101592,"kb_avail":641847788,"statfs":{"total":671965249536,"available":657252134912,"internally_reserved":0,"allocated":9488986112,"data_stored":9479022332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5224030831},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-90308608,"num_objects":-1378,"num_object_clones":0,"num_object_copies":-4134,"num_objects_missing_on_primary":-57,"num_objects_missing":-57,"num_objects_degraded":-60,"num_objects_misplaced":-71,"num_objects_unfound":0,"num_objects_dirty":-1378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1378,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":74,"num_bytes_recovered":4849664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001862"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":320,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.488306+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:46:10.488306+0000","last_peered":"2023-10-01T09:46:10.488306+0000","last_clean":"2023-10-01T09:46:10.488306+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:46:10.488306+0000","last_undegraded":"2023-10-01T09:46:10.488306+0000","last_fullsized":"2023-10-01T09:46:10.488306+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":163,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.818746+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:46:03.818746+0000","last_peered":"2023-10-01T09:46:03.818746+0000","last_clean":"2023-10-01T09:46:03.818746+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:46:03.818746+0000","last_undegraded":"2023-10-01T09:46:03.818746+0000","last_fullsized":"2023-10-01T09:46:03.818746+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":544,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:05.274557+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:46:05.274557+0000","last_peered":"2023-10-01T09:46:05.274557+0000","last_clean":"2023-10-01T09:46:05.274557+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:46:05.274557+0000","last_undegraded":"2023-10-01T09:46:05.274557+0000","last_fullsized":"2023-10-01T09:46:05.274557+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":602,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.815661+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:46:03.815661+0000","last_peered":"2023-10-01T09:46:03.815661+0000","last_clean":"2023-10-01T09:46:03.815661+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:46:03.815661+0000","last_undegraded":"2023-10-01T09:46:03.815661+0000","last_fullsized":"2023-10-01T09:46:03.815661+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":483,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.815762+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:46:03.815762+0000","last_peered":"2023-10-01T09:46:03.815762+0000","last_clean":"2023-10-01T09:46:03.815762+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:46:03.815762+0000","last_undegraded":"2023-10-01T09:46:03.815762+0000","last_fullsized":"2023-10-01T09:46:03.815762+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":224,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.828057+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:46:03.828057+0000","last_peered":"2023-10-01T09:46:03.828057+0000","last_clean":"2023-10-01T09:46:03.828057+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:46:03.828057+0000","last_undegraded":"2023-10-01T09:46:03.828057+0000","last_fullsized":"2023-10-01T09:46:03.828057+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":600,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.828163+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:46:03.828163+0000","last_peered":"2023-10-01T09:46:03.828163+0000","last_clean":"2023-10-01T09:46:03.828163+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:46:03.828163+0000","last_undegraded":"2023-10-01T09:46:03.828163+0000","last_fullsized":"2023-10-01T09:46:03.828163+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":530,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.995142+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:46:03.995142+0000","last_peered":"2023-10-01T09:46:03.995142+0000","last_clean":"2023-10-01T09:46:03.995142+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:46:03.995142+0000","last_undegraded":"2023-10-01T09:46:03.995142+0000","last_fullsized":"2023-10-01T09:46:03.995142+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":255,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.816702+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:46:03.816702+0000","last_peered":"2023-10-01T09:46:03.816702+0000","last_clean":"2023-10-01T09:46:03.816702+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:46:03.816702+0000","last_undegraded":"2023-10-01T09:46:03.816702+0000","last_fullsized":"2023-10-01T09:46:03.816702+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500521+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:46:10.500521+0000","last_peered":"2023-10-01T09:46:10.500521+0000","last_clean":"2023-10-01T09:46:10.500521+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:46:10.500521+0000","last_undegraded":"2023-10-01T09:46:10.500521+0000","last_fullsized":"2023-10-01T09:46:10.500521+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":69,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500936+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:46:10.500936+0000","last_peered":"2023-10-01T09:46:10.500936+0000","last_clean":"2023-10-01T09:46:10.500936+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:46:10.500936+0000","last_undegraded":"2023-10-01T09:46:10.500936+0000","last_fullsized":"2023-10-01T09:46:10.500936+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":67,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.820526+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:46:03.820526+0000","last_peered":"2023-10-01T09:46:03.820526+0000","last_clean":"2023-10-01T09:46:03.820526+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:46:03.820526+0000","last_undegraded":"2023-10-01T09:46:03.820526+0000","last_fullsized":"2023-10-01T09:46:03.820526+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":445,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.487199+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:46:10.487199+0000","last_peered":"2023-10-01T09:46:10.487199+0000","last_clean":"2023-10-01T09:46:10.487199+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:46:10.487199+0000","last_undegraded":"2023-10-01T09:46:10.487199+0000","last_fullsized":"2023-10-01T09:46:10.487199+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":326,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.487790+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:46:10.487790+0000","last_peered":"2023-10-01T09:46:10.487790+0000","last_clean":"2023-10-01T09:46:10.487790+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:46:10.487790+0000","last_undegraded":"2023-10-01T09:46:10.487790+0000","last_fullsized":"2023-10-01T09:46:10.487790+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":662,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500744+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:46:10.500744+0000","last_peered":"2023-10-01T09:46:10.500744+0000","last_clean":"2023-10-01T09:46:10.500744+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:46:10.500744+0000","last_undegraded":"2023-10-01T09:46:10.500744+0000","last_fullsized":"2023-10-01T09:46:10.500744+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":399,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500981+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:46:10.500981+0000","last_peered":"2023-10-01T09:46:10.500981+0000","last_clean":"2023-10-01T09:46:10.500981+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:46:10.500981+0000","last_undegraded":"2023-10-01T09:46:10.500981+0000","last_fullsized":"2023-10-01T09:46:10.500981+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":72,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500663+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:46:10.500663+0000","last_peered":"2023-10-01T09:46:10.500663+0000","last_clean":"2023-10-01T09:46:10.500663+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:46:10.500663+0000","last_undegraded":"2023-10-01T09:46:10.500663+0000","last_fullsized":"2023-10-01T09:46:10.500663+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":57,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.819585+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:46:03.819585+0000","last_peered":"2023-10-01T09:46:03.819585+0000","last_clean":"2023-10-01T09:46:03.819585+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:46:03.819585+0000","last_undegraded":"2023-10-01T09:46:03.819585+0000","last_fullsized":"2023-10-01T09:46:03.819585+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":555,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500644+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:46:10.500644+0000","last_peered":"2023-10-01T09:46:10.500644+0000","last_clean":"2023-10-01T09:46:10.500644+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:46:10.500644+0000","last_undegraded":"2023-10-01T09:46:10.500644+0000","last_fullsized":"2023-10-01T09:46:10.500644+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":566,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.815719+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:46:03.815719+0000","last_peered":"2023-10-01T09:46:03.815719+0000","last_clean":"2023-10-01T09:46:03.815719+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:46:03.815719+0000","last_undegraded":"2023-10-01T09:46:03.815719+0000","last_fullsized":"2023-10-01T09:46:03.815719+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":422,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.815260+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:46:03.815260+0000","last_peered":"2023-10-01T09:46:03.815260+0000","last_clean":"2023-10-01T09:46:03.815260+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:46:03.815260+0000","last_undegraded":"2023-10-01T09:46:03.815260+0000","last_fullsized":"2023-10-01T09:46:03.815260+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.501228+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:46:10.501228+0000","last_peered":"2023-10-01T09:46:10.501228+0000","last_clean":"2023-10-01T09:46:10.501228+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:46:10.501228+0000","last_undegraded":"2023-10-01T09:46:10.501228+0000","last_fullsized":"2023-10-01T09:46:10.501228+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"543'7627","reported_seq":7703,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:12.053925+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:46:12.053925+0000","last_peered":"2023-10-01T09:46:12.053925+0000","last_clean":"2023-10-01T09:46:12.053925+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:46:12.053925+0000","last_undegraded":"2023-10-01T09:46:12.053925+0000","last_fullsized":"2023-10-01T09:46:12.053925+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7627,"log_dups_size":0,"ondisk_log_size":7627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181207040,"num_objects":2765,"num_object_clones":0,"num_object_copies":8295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7627,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":67,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.501055+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:46:10.501055+0000","last_peered":"2023-10-01T09:46:10.501055+0000","last_clean":"2023-10-01T09:46:10.501055+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:46:10.501055+0000","last_undegraded":"2023-10-01T09:46:10.501055+0000","last_fullsized":"2023-10-01T09:46:10.501055+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":263,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.814029+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:46:03.814029+0000","last_peered":"2023-10-01T09:46:03.814029+0000","last_clean":"2023-10-01T09:46:03.814029+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:46:03.814029+0000","last_undegraded":"2023-10-01T09:46:03.814029+0000","last_fullsized":"2023-10-01T09:46:03.814029+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"543'7631","reported_seq":15020,"reported_epoch":543,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:46:12.322187+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:46:12.322187+0000","last_peered":"2023-10-01T09:46:12.322187+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:46:12.322187+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7631,"log_dups_size":0,"ondisk_log_size":7631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181862400,"num_objects":2775,"num_object_clones":0,"num_object_copies":8325,"num_objects_missing_on_primary":1256,"num_objects_missing":1256,"num_objects_degraded":3963,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7631,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2428,"num_bytes_recovered":159121408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1451},{"shards":"0,4,6","objects":1256},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":538,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.501206+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:46:10.501206+0000","last_peered":"2023-10-01T09:46:10.501206+0000","last_clean":"2023-10-01T09:46:10.501206+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:46:10.501206+0000","last_undegraded":"2023-10-01T09:46:10.501206+0000","last_fullsized":"2023-10-01T09:46:10.501206+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":599,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.818928+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:46:03.818928+0000","last_peered":"2023-10-01T09:46:03.818928+0000","last_clean":"2023-10-01T09:46:03.818928+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:46:03.818928+0000","last_undegraded":"2023-10-01T09:46:03.818928+0000","last_fullsized":"2023-10-01T09:46:03.818928+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"543'7579","reported_seq":7658,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:11.521899+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:46:11.521899+0000","last_peered":"2023-10-01T09:46:11.521899+0000","last_clean":"2023-10-01T09:46:11.521899+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:46:11.521899+0000","last_undegraded":"2023-10-01T09:46:11.521899+0000","last_fullsized":"2023-10-01T09:46:11.521899+0000","mapping_epoch":475,"log_start":"481'600","ondisk_log_start":"481'600","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186974208,"num_objects":2853,"num_object_clones":0,"num_object_copies":8559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7579,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":624,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.501039+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:46:10.501039+0000","last_peered":"2023-10-01T09:46:10.501039+0000","last_clean":"2023-10-01T09:46:10.501039+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:46:10.501039+0000","last_undegraded":"2023-10-01T09:46:10.501039+0000","last_fullsized":"2023-10-01T09:46:10.501039+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":420,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.483319+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:46:10.483319+0000","last_peered":"2023-10-01T09:46:10.483319+0000","last_clean":"2023-10-01T09:46:10.483319+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:46:10.483319+0000","last_undegraded":"2023-10-01T09:46:10.483319+0000","last_fullsized":"2023-10-01T09:46:10.483319+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"543'7613","reported_seq":14297,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:11.846278+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:46:11.846278+0000","last_peered":"2023-10-01T09:46:11.846278+0000","last_clean":"2023-10-01T09:46:11.846278+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:46:11.846278+0000","last_undegraded":"2023-10-01T09:46:11.846278+0000","last_fullsized":"2023-10-01T09:46:11.846278+0000","mapping_epoch":505,"log_start":"481'600","ondisk_log_start":"481'600","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7013,"log_dups_size":0,"ondisk_log_size":7013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182779904,"num_objects":2789,"num_object_clones":0,"num_object_copies":8367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7613,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":263,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.819280+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:46:03.819280+0000","last_peered":"2023-10-01T09:46:03.819280+0000","last_clean":"2023-10-01T09:46:03.819280+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:46:03.819280+0000","last_undegraded":"2023-10-01T09:46:03.819280+0000","last_fullsized":"2023-10-01T09:46:03.819280+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":522,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.815450+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:46:03.815450+0000","last_peered":"2023-10-01T09:46:03.815450+0000","last_clean":"2023-10-01T09:46:03.815450+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:46:03.815450+0000","last_undegraded":"2023-10-01T09:46:03.815450+0000","last_fullsized":"2023-10-01T09:46:03.815450+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"543'7640","reported_seq":7719,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:14.687786+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:46:14.687786+0000","last_peered":"2023-10-01T09:46:14.687786+0000","last_clean":"2023-10-01T09:46:14.687786+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:46:14.687786+0000","last_undegraded":"2023-10-01T09:46:14.687786+0000","last_fullsized":"2023-10-01T09:46:14.687786+0000","mapping_epoch":475,"log_start":"480'100","ondisk_log_start":"480'100","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7540,"log_dups_size":0,"ondisk_log_size":7540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187957248,"num_objects":2868,"num_object_clones":0,"num_object_copies":8604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7640,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":124,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.488106+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:46:10.488106+0000","last_peered":"2023-10-01T09:46:10.488106+0000","last_clean":"2023-10-01T09:46:10.488106+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:46:10.488106+0000","last_undegraded":"2023-10-01T09:46:10.488106+0000","last_fullsized":"2023-10-01T09:46:10.488106+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":198,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.487342+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:46:10.487342+0000","last_peered":"2023-10-01T09:46:10.487342+0000","last_clean":"2023-10-01T09:46:10.487342+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:46:10.487342+0000","last_undegraded":"2023-10-01T09:46:10.487342+0000","last_fullsized":"2023-10-01T09:46:10.487342+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"543'7550","reported_seq":7786,"reported_epoch":543,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:46:12.098415+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:46:12.098415+0000","last_peered":"2023-10-01T09:46:12.098415+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:46:12.098415+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7550,"log_dups_size":0,"ondisk_log_size":7550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184680448,"num_objects":2818,"num_object_clones":0,"num_object_copies":8454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2295,"num_objects_misplaced":2750,"num_objects_unfound":0,"num_objects_dirty":2818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7550,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2750},{"shards":"4,5","objects":-2227}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":442,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.488432+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:46:10.488432+0000","last_peered":"2023-10-01T09:46:10.488432+0000","last_clean":"2023-10-01T09:46:10.488432+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:46:10.488432+0000","last_undegraded":"2023-10-01T09:46:10.488432+0000","last_fullsized":"2023-10-01T09:46:10.488432+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":602,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.818824+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:46:03.818824+0000","last_peered":"2023-10-01T09:46:03.818824+0000","last_clean":"2023-10-01T09:46:03.818824+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:46:03.818824+0000","last_undegraded":"2023-10-01T09:46:03.818824+0000","last_fullsized":"2023-10-01T09:46:03.818824+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"543'7565","reported_seq":7650,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:12.369653+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:46:12.369653+0000","last_peered":"2023-10-01T09:46:12.369653+0000","last_clean":"2023-10-01T09:46:12.369653+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:46:12.369653+0000","last_undegraded":"2023-10-01T09:46:12.369653+0000","last_fullsized":"2023-10-01T09:46:12.369653+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7565,"log_dups_size":0,"ondisk_log_size":7565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184352768,"num_objects":2813,"num_object_clones":0,"num_object_copies":8439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7565,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":438,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.489076+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:46:10.489076+0000","last_peered":"2023-10-01T09:46:10.489076+0000","last_clean":"2023-10-01T09:46:10.489076+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:46:10.489076+0000","last_undegraded":"2023-10-01T09:46:10.489076+0000","last_fullsized":"2023-10-01T09:46:10.489076+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":279,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.488813+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:46:10.488813+0000","last_peered":"2023-10-01T09:46:10.488813+0000","last_clean":"2023-10-01T09:46:10.488813+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:46:10.488813+0000","last_undegraded":"2023-10-01T09:46:10.488813+0000","last_fullsized":"2023-10-01T09:46:10.488813+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"543'7706","reported_seq":7783,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:16.216493+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:46:16.216493+0000","last_peered":"2023-10-01T09:46:16.216493+0000","last_clean":"2023-10-01T09:46:16.216493+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:46:16.216493+0000","last_undegraded":"2023-10-01T09:46:16.216493+0000","last_fullsized":"2023-10-01T09:46:16.216493+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7706,"log_dups_size":0,"ondisk_log_size":7706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182976512,"num_objects":2792,"num_object_clones":0,"num_object_copies":8376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7706,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":407,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.818518+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:46:03.818518+0000","last_peered":"2023-10-01T09:46:03.818518+0000","last_clean":"2023-10-01T09:46:03.818518+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:46:03.818518+0000","last_undegraded":"2023-10-01T09:46:03.818518+0000","last_fullsized":"2023-10-01T09:46:03.818518+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":454,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.820008+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:46:03.820008+0000","last_peered":"2023-10-01T09:46:03.820008+0000","last_clean":"2023-10-01T09:46:03.820008+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:46:03.820008+0000","last_undegraded":"2023-10-01T09:46:03.820008+0000","last_fullsized":"2023-10-01T09:46:03.820008+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"543'7473","reported_seq":7549,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:16.093393+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:46:16.093393+0000","last_peered":"2023-10-01T09:46:16.093393+0000","last_clean":"2023-10-01T09:46:16.093393+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:46:16.093393+0000","last_undegraded":"2023-10-01T09:46:16.093393+0000","last_fullsized":"2023-10-01T09:46:16.093393+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7473,"log_dups_size":0,"ondisk_log_size":7473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177274880,"num_objects":2705,"num_object_clones":0,"num_object_copies":8115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7473,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":937,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500881+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:46:10.500881+0000","last_peered":"2023-10-01T09:46:10.500881+0000","last_clean":"2023-10-01T09:46:10.500881+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:46:10.500881+0000","last_undegraded":"2023-10-01T09:46:10.500881+0000","last_fullsized":"2023-10-01T09:46:10.500881+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":273,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.819774+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:46:03.819774+0000","last_peered":"2023-10-01T09:46:03.819774+0000","last_clean":"2023-10-01T09:46:03.819774+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:46:03.819774+0000","last_undegraded":"2023-10-01T09:46:03.819774+0000","last_fullsized":"2023-10-01T09:46:03.819774+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":67,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.820569+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:46:03.820569+0000","last_peered":"2023-10-01T09:46:03.820569+0000","last_clean":"2023-10-01T09:46:03.820569+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:46:03.820569+0000","last_undegraded":"2023-10-01T09:46:03.820569+0000","last_fullsized":"2023-10-01T09:46:03.820569+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"543'7488","reported_seq":7567,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:11.826315+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:46:11.826315+0000","last_peered":"2023-10-01T09:46:11.826315+0000","last_clean":"2023-10-01T09:46:11.826315+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:46:11.826315+0000","last_undegraded":"2023-10-01T09:46:11.826315+0000","last_fullsized":"2023-10-01T09:46:11.826315+0000","mapping_epoch":475,"log_start":"481'500","ondisk_log_start":"481'500","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6988,"log_dups_size":0,"ondisk_log_size":6988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188874752,"num_objects":2882,"num_object_clones":0,"num_object_copies":8646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7488,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1208,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.814212+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:46:03.814212+0000","last_peered":"2023-10-01T09:46:03.814212+0000","last_clean":"2023-10-01T09:46:03.814212+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:46:03.814212+0000","last_undegraded":"2023-10-01T09:46:03.814212+0000","last_fullsized":"2023-10-01T09:46:03.814212+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":419,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.487971+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:46:10.487971+0000","last_peered":"2023-10-01T09:46:10.487971+0000","last_clean":"2023-10-01T09:46:10.487971+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:46:10.487971+0000","last_undegraded":"2023-10-01T09:46:10.487971+0000","last_fullsized":"2023-10-01T09:46:10.487971+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":246,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.819470+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:46:03.819470+0000","last_peered":"2023-10-01T09:46:03.819470+0000","last_clean":"2023-10-01T09:46:03.819470+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:46:03.819470+0000","last_undegraded":"2023-10-01T09:46:03.819470+0000","last_fullsized":"2023-10-01T09:46:03.819470+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"543'7564","reported_seq":14861,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:12.080421+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:46:12.080421+0000","last_peered":"2023-10-01T09:46:12.080421+0000","last_clean":"2023-10-01T09:46:12.080421+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:46:12.080421+0000","last_undegraded":"2023-10-01T09:46:12.080421+0000","last_fullsized":"2023-10-01T09:46:12.080421+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7564,"log_dups_size":0,"ondisk_log_size":7564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184287232,"num_objects":2812,"num_object_clones":0,"num_object_copies":8436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7564,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":889,"reported_epoch":542,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:46:03.815068+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:46:03.815068+0000","last_peered":"2023-10-01T09:46:03.815068+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:46:03.815068+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:46:03.815068+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":607,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482849+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:46:10.482849+0000","last_peered":"2023-10-01T09:46:10.482849+0000","last_clean":"2023-10-01T09:46:10.482849+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:46:10.482849+0000","last_undegraded":"2023-10-01T09:46:10.482849+0000","last_fullsized":"2023-10-01T09:46:10.482849+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":67,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.819888+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:46:03.819888+0000","last_peered":"2023-10-01T09:46:03.819888+0000","last_clean":"2023-10-01T09:46:03.819888+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:46:03.819888+0000","last_undegraded":"2023-10-01T09:46:03.819888+0000","last_fullsized":"2023-10-01T09:46:03.819888+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"543'7513","reported_seq":7592,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:15.018700+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:46:15.018700+0000","last_peered":"2023-10-01T09:46:15.018700+0000","last_clean":"2023-10-01T09:46:15.018700+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:46:15.018700+0000","last_undegraded":"2023-10-01T09:46:15.018700+0000","last_fullsized":"2023-10-01T09:46:15.018700+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7513,"log_dups_size":0,"ondisk_log_size":7513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180813824,"num_objects":2759,"num_object_clones":0,"num_object_copies":8277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7513,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1659,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.814375+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:46:03.814375+0000","last_peered":"2023-10-01T09:46:03.814375+0000","last_clean":"2023-10-01T09:46:03.814375+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:46:03.814375+0000","last_undegraded":"2023-10-01T09:46:03.814375+0000","last_fullsized":"2023-10-01T09:46:03.814375+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":574,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500560+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:46:10.500560+0000","last_peered":"2023-10-01T09:46:10.500560+0000","last_clean":"2023-10-01T09:46:10.500560+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:46:10.500560+0000","last_undegraded":"2023-10-01T09:46:10.500560+0000","last_fullsized":"2023-10-01T09:46:10.500560+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":351,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.814284+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:46:03.814284+0000","last_peered":"2023-10-01T09:46:03.814284+0000","last_clean":"2023-10-01T09:46:03.814284+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:46:03.814284+0000","last_undegraded":"2023-10-01T09:46:03.814284+0000","last_fullsized":"2023-10-01T09:46:03.814284+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":342,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.487448+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:46:10.487448+0000","last_peered":"2023-10-01T09:46:10.487448+0000","last_clean":"2023-10-01T09:46:10.487448+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:46:10.487448+0000","last_undegraded":"2023-10-01T09:46:10.487448+0000","last_fullsized":"2023-10-01T09:46:10.487448+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"543'7699","reported_seq":7776,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:16.211663+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:46:16.211663+0000","last_peered":"2023-10-01T09:46:16.211663+0000","last_clean":"2023-10-01T09:46:16.211663+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:46:16.211663+0000","last_undegraded":"2023-10-01T09:46:16.211663+0000","last_fullsized":"2023-10-01T09:46:16.211663+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7699,"log_dups_size":0,"ondisk_log_size":7699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185663488,"num_objects":2833,"num_object_clones":0,"num_object_copies":8499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7699,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":751,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500799+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:46:10.500799+0000","last_peered":"2023-10-01T09:46:10.500799+0000","last_clean":"2023-10-01T09:46:10.500799+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:46:10.500799+0000","last_undegraded":"2023-10-01T09:46:10.500799+0000","last_fullsized":"2023-10-01T09:46:10.500799+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":583,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.815430+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:46:03.815430+0000","last_peered":"2023-10-01T09:46:03.815430+0000","last_clean":"2023-10-01T09:46:03.815430+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:46:03.815430+0000","last_undegraded":"2023-10-01T09:46:03.815430+0000","last_fullsized":"2023-10-01T09:46:03.815430+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":583,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.815613+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:46:03.815613+0000","last_peered":"2023-10-01T09:46:03.815613+0000","last_clean":"2023-10-01T09:46:03.815613+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:46:03.815613+0000","last_undegraded":"2023-10-01T09:46:03.815613+0000","last_fullsized":"2023-10-01T09:46:03.815613+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"543'7574","reported_seq":7654,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:11.630177+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:46:11.630177+0000","last_peered":"2023-10-01T09:46:11.630177+0000","last_clean":"2023-10-01T09:46:11.630177+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:46:11.630177+0000","last_undegraded":"2023-10-01T09:46:11.630177+0000","last_fullsized":"2023-10-01T09:46:11.630177+0000","mapping_epoch":475,"log_start":"481'500","ondisk_log_start":"481'500","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7074,"log_dups_size":0,"ondisk_log_size":7074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189071384,"num_objects":2886,"num_object_clones":0,"num_object_copies":8658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2886,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7574,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":982,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.501122+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:46:10.501122+0000","last_peered":"2023-10-01T09:46:10.501122+0000","last_clean":"2023-10-01T09:46:10.501122+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:46:10.501122+0000","last_undegraded":"2023-10-01T09:46:10.501122+0000","last_fullsized":"2023-10-01T09:46:10.501122+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":600,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.819376+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:46:03.819376+0000","last_peered":"2023-10-01T09:46:03.819376+0000","last_clean":"2023-10-01T09:46:03.819376+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:46:03.819376+0000","last_undegraded":"2023-10-01T09:46:03.819376+0000","last_fullsized":"2023-10-01T09:46:03.819376+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":453,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.814443+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:46:03.814443+0000","last_peered":"2023-10-01T09:46:03.814443+0000","last_clean":"2023-10-01T09:46:03.814443+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:46:03.814443+0000","last_undegraded":"2023-10-01T09:46:03.814443+0000","last_fullsized":"2023-10-01T09:46:03.814443+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"543'7556","reported_seq":7641,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:16.200895+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:46:16.200895+0000","last_peered":"2023-10-01T09:46:16.200895+0000","last_clean":"2023-10-01T09:46:16.200895+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:46:16.200895+0000","last_undegraded":"2023-10-01T09:46:16.200895+0000","last_fullsized":"2023-10-01T09:46:16.200895+0000","mapping_epoch":479,"log_start":"481'800","ondisk_log_start":"481'800","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6756,"log_dups_size":0,"ondisk_log_size":6756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183500800,"num_objects":2800,"num_object_clones":0,"num_object_copies":8400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7556,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":483,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.501315+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:46:10.501315+0000","last_peered":"2023-10-01T09:46:10.501315+0000","last_clean":"2023-10-01T09:46:10.501315+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:46:10.501315+0000","last_undegraded":"2023-10-01T09:46:10.501315+0000","last_fullsized":"2023-10-01T09:46:10.501315+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1176,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.819671+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:46:03.819671+0000","last_peered":"2023-10-01T09:46:03.819671+0000","last_clean":"2023-10-01T09:46:03.819671+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:46:03.819671+0000","last_undegraded":"2023-10-01T09:46:03.819671+0000","last_fullsized":"2023-10-01T09:46:03.819671+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":72,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500474+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:46:10.500474+0000","last_peered":"2023-10-01T09:46:10.500474+0000","last_clean":"2023-10-01T09:46:10.500474+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:46:10.500474+0000","last_undegraded":"2023-10-01T09:46:10.500474+0000","last_fullsized":"2023-10-01T09:46:10.500474+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"543'7657","reported_seq":7742,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:16.206770+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:46:16.206770+0000","last_peered":"2023-10-01T09:46:16.206770+0000","last_clean":"2023-10-01T09:46:16.206770+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:46:16.206770+0000","last_undegraded":"2023-10-01T09:46:16.206770+0000","last_fullsized":"2023-10-01T09:46:16.206770+0000","mapping_epoch":479,"log_start":"481'900","ondisk_log_start":"481'900","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6757,"log_dups_size":0,"ondisk_log_size":6757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187105280,"num_objects":2855,"num_object_clones":0,"num_object_copies":8565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7657,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":76,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.501146+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:46:10.501146+0000","last_peered":"2023-10-01T09:46:10.501146+0000","last_clean":"2023-10-01T09:46:10.501146+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:46:10.501146+0000","last_undegraded":"2023-10-01T09:46:10.501146+0000","last_fullsized":"2023-10-01T09:46:10.501146+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":561,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.815690+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:46:03.815690+0000","last_peered":"2023-10-01T09:46:03.815690+0000","last_clean":"2023-10-01T09:46:03.815690+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:46:03.815690+0000","last_undegraded":"2023-10-01T09:46:03.815690+0000","last_fullsized":"2023-10-01T09:46:03.815690+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":266,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.813747+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:46:03.813747+0000","last_peered":"2023-10-01T09:46:03.813747+0000","last_clean":"2023-10-01T09:46:03.813747+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:46:03.813747+0000","last_undegraded":"2023-10-01T09:46:03.813747+0000","last_fullsized":"2023-10-01T09:46:03.813747+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":510,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.815508+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:46:03.815508+0000","last_peered":"2023-10-01T09:46:03.815508+0000","last_clean":"2023-10-01T09:46:03.815508+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:46:03.815508+0000","last_undegraded":"2023-10-01T09:46:03.815508+0000","last_fullsized":"2023-10-01T09:46:03.815508+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":623,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.819259+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:46:03.819259+0000","last_peered":"2023-10-01T09:46:03.819259+0000","last_clean":"2023-10-01T09:46:03.819259+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:46:03.819259+0000","last_undegraded":"2023-10-01T09:46:03.819259+0000","last_fullsized":"2023-10-01T09:46:03.819259+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":278,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.819190+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:46:03.819190+0000","last_peered":"2023-10-01T09:46:03.819190+0000","last_clean":"2023-10-01T09:46:03.819190+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:46:03.819190+0000","last_undegraded":"2023-10-01T09:46:03.819190+0000","last_fullsized":"2023-10-01T09:46:03.819190+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":67,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.820250+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:46:03.820250+0000","last_peered":"2023-10-01T09:46:03.820250+0000","last_clean":"2023-10-01T09:46:03.820250+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:46:03.820250+0000","last_undegraded":"2023-10-01T09:46:03.820250+0000","last_fullsized":"2023-10-01T09:46:03.820250+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":278,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.819893+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:46:03.819893+0000","last_peered":"2023-10-01T09:46:03.819893+0000","last_clean":"2023-10-01T09:46:03.819893+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:46:03.819893+0000","last_undegraded":"2023-10-01T09:46:03.819893+0000","last_fullsized":"2023-10-01T09:46:03.819893+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":456,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.815848+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:46:03.815848+0000","last_peered":"2023-10-01T09:46:03.815848+0000","last_clean":"2023-10-01T09:46:03.815848+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:46:03.815848+0000","last_undegraded":"2023-10-01T09:46:03.815848+0000","last_fullsized":"2023-10-01T09:46:03.815848+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":85,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:05.274698+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:46:05.274698+0000","last_peered":"2023-10-01T09:46:05.274698+0000","last_clean":"2023-10-01T09:46:05.274698+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:46:05.274698+0000","last_undegraded":"2023-10-01T09:46:05.274698+0000","last_fullsized":"2023-10-01T09:46:05.274698+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":501,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.815328+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:46:03.815328+0000","last_peered":"2023-10-01T09:46:03.815328+0000","last_clean":"2023-10-01T09:46:03.815328+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:46:03.815328+0000","last_undegraded":"2023-10-01T09:46:03.815328+0000","last_fullsized":"2023-10-01T09:46:03.815328+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":700,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-10-01T09:46:03.815820+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:46:03.815820+0000","last_peered":"2023-10-01T09:46:03.815820+0000","last_clean":"2023-10-01T09:46:03.815820+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:46:03.815820+0000","last_undegraded":"2023-10-01T09:46:03.815820+0000","last_fullsized":"2023-10-01T09:46:03.815820+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":395,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.489642+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:46:10.489642+0000","last_peered":"2023-10-01T09:46:10.489642+0000","last_clean":"2023-10-01T09:46:10.489642+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:46:10.489642+0000","last_undegraded":"2023-10-01T09:46:10.489642+0000","last_fullsized":"2023-10-01T09:46:10.489642+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2949382168,"num_objects":45005,"num_object_clones":0,"num_object_copies":135015,"num_objects_missing_on_primary":1256,"num_objects_missing":1256,"num_objects_degraded":6258,"num_objects_misplaced":2750,"num_objects_unfound":0,"num_objects_dirty":45005,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":121435,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9397,"num_bytes_recovered":615841792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9419108352,"data_stored":9419096136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117435,"ondisk_log_size":117435,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430748,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1798920,"kb_used_data":895808,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":91946232,"statfs":{"total":95995035648,"available":94152941568,"internally_reserved":0,"allocated":917307392,"data_stored":916364180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.74399999999999999,"15min":2.5840000000000001},"min":{"1min":0.27700000000000002,"5min":0.246,"15min":0.20999999999999999},"max":{"1min":1.081,"5min":6.8550000000000004,"15min":108.90000000000001},"last":0.496},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.52500000000000002,"15min":2.7050000000000001},"min":{"1min":0.25700000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.2090000000000001,"5min":5.6639999999999997,"15min":109.187},"last":0.48399999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.66400000000000003,"15min":5.1520000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.222},"max":{"1min":1.0780000000000001,"5min":6.8289999999999997,"15min":229.161},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.59799999999999998,"15min":5.1779999999999999},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":3.0640000000000001,"5min":5.7240000000000002,"15min":230.126},"last":0.44600000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.72799999999999998,"15min":1.617},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.1259999999999999,"5min":5.3090000000000002,"15min":14.901999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.68500000000000005,"15min":1.6100000000000001},"min":{"1min":0.33700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.0009999999999999,"5min":5.6929999999999996,"15min":14.682},"last":0.52500000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.65800000000000003,"15min":2.2000000000000002},"min":{"1min":0.35699999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.476,"5min":5.7750000000000004,"15min":76.555000000000007},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.68100000000000005,"15min":2.2010000000000001},"min":{"1min":0.32800000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.4529999999999998,"5min":6.548,"15min":76.534000000000006},"last":0.59299999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.70499999999999996,"15min":1.042},"min":{"1min":0.28299999999999997,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.276,"5min":6.8769999999999998,"15min":9.6020000000000003},"last":0.628},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.68500000000000005,"15min":0.997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.97399999999999998,"5min":5.7439999999999998,"15min":8.4949999999999992},"last":0.55300000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":2.0960000000000001,"15min":1.548},"min":{"1min":0.315,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":0.80000000000000004,"5min":135.40000000000001,"15min":135.40000000000001},"last":2.2400000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":2.0830000000000002,"15min":1.6459999999999999},"min":{"1min":0.35699999999999998,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":2.8849999999999998,"5min":135.339,"15min":135.339},"last":0.46899999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.075,"15min":1.133},"min":{"1min":0.38,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":0.754,"5min":11.58,"15min":11.853999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.86699999999999999,"15min":1.111},"min":{"1min":0.35599999999999998,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":3.7810000000000001,"5min":13.021000000000001,"15min":14.584},"last":0.51500000000000001}]}]},{"osd":6,"up_from":294,"seq":1262720385258,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1621316,"kb_used_data":1279612,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":92123836,"statfs":{"total":95995035648,"available":94334808064,"internally_reserved":0,"allocated":1310322688,"data_stored":1309383427,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":1.865,"15min":3.9809999999999999},"min":{"1min":0.27100000000000002,"5min":0.219,"15min":0.20599999999999999},"max":{"1min":0.80200000000000005,"5min":158.48599999999999,"15min":158.48599999999999},"last":0.248},{"interface":"front","average":{"1min":0.58999999999999997,"5min":2.0640000000000001,"15min":4.0209999999999999},"min":{"1min":0.311,"5min":0.245,"15min":0.221},"max":{"1min":2.1760000000000002,"5min":158.37700000000001,"15min":158.37700000000001},"last":0.33000000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":2.9470000000000001,"15min":4.8600000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.24199999999999999},"max":{"1min":1.9059999999999999,"5min":168.947,"15min":168.947},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.44400000000000001,"5min":2.9980000000000002,"15min":4.8109999999999999},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":0.59899999999999998,"5min":169.07900000000001,"15min":169.07900000000001},"last":0.38800000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.75600000000000001,"15min":1.5249999999999999},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.26900000000000002},"max":{"1min":1.9350000000000001,"5min":8.657,"15min":10.92},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.76000000000000001,"15min":1.542},"min":{"1min":0.32300000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.2649999999999999,"5min":12.247999999999999,"15min":21.704999999999998},"last":0.44500000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.70599999999999996,"15min":2.0699999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.1909999999999998,"5min":4.149,"15min":89.753},"last":0.436},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.875,"15min":2.0179999999999998},"min":{"1min":0.38500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.1379999999999999,"5min":8.5470000000000006,"15min":89.849999999999994},"last":0.53900000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":1.4510000000000001,"15min":1.0680000000000001},"min":{"1min":0.27200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":0.623,"5min":63.134,"15min":63.134},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.4219999999999999,"15min":1.0669999999999999},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":0.73499999999999999,"5min":62.973999999999997,"15min":62.973999999999997},"last":0.45700000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:45:31 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.1599999999999999,"15min":1.0369999999999999},"min":{"1min":0.307,"5min":0.26700000000000002,"15min":0.251},"max":{"1min":0.82199999999999995,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.2849999999999999,"15min":1.155},"min":{"1min":0.32500000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.6800000000000002,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.59299999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:45:15 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.67800000000000005,"15min":1.272},"min":{"1min":0.40200000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.83699999999999997,"5min":3.492,"15min":94.245999999999995},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.87,"15min":1.379},"min":{"1min":0.42299999999999999,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":2.202,"5min":8.6899999999999995,"15min":94.715000000000003},"last":0.503}]}]},{"osd":1,"up_from":94,"seq":403726926235,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1370964,"kb_used_data":548684,"kb_used_omap":7,"kb_used_meta":822264,"kb_avail":92374188,"statfs":{"total":95995035648,"available":94591168512,"internally_reserved":0,"allocated":561852416,"data_stored":559784576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":841998759},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4410000000000001,"5min":1.1140000000000001,"15min":3.8010000000000002},"min":{"1min":0.217,"5min":0.186,"15min":0.182},"max":{"1min":18.556999999999999,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.29099999999999998},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.302,"15min":3.3359999999999999},"min":{"1min":0.23499999999999999,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":18.704000000000001,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.223}]},{"osd":2,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.39800000000000002,"15min":2.2669999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.51100000000000001,"5min":2.8740000000000001,"15min":51.640000000000001},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.61799999999999999,"15min":2.9500000000000002},"min":{"1min":0.245,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":2.9089999999999998,"5min":6.9710000000000001,"15min":154.197},"last":0.36799999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.45700000000000002,"15min":2.9540000000000002},"min":{"1min":0.28199999999999997,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":2.8780000000000001,"5min":3.1949999999999998,"15min":95.305000000000007},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.432,"5min":0.51200000000000001,"15min":3.6720000000000002},"min":{"1min":0.27300000000000002,"5min":0.245,"15min":0.216},"max":{"1min":0.95699999999999996,"5min":4.7839999999999998,"15min":154.483},"last":0.35499999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.59099999999999997,"15min":2.1829999999999998},"min":{"1min":0.312,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":1.5309999999999999,"5min":3.0550000000000002,"15min":154.101},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.49099999999999999,"15min":2.2869999999999999},"min":{"1min":0.34300000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65500000000000003,"5min":2.4209999999999998,"15min":154.56999999999999},"last":0.40699999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.246,"15min":2.2530000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.30199999999999999},"max":{"1min":1.696,"5min":59.561,"15min":154.22999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.1970000000000001,"15min":2.4649999999999999},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.307},"max":{"1min":0.67300000000000004,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.50600000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.65700000000000003,"15min":2.448},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.944,"5min":3.3660000000000001,"15min":154.12200000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64600000000000002,"15min":2.3450000000000002},"min":{"1min":0.374,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":1.7549999999999999,"5min":2.5630000000000002,"15min":154.16999999999999},"last":1.7549999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.74299999999999999,"15min":2.5230000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.33900000000000002},"max":{"1min":3.6160000000000001,"5min":5.2169999999999996,"15min":154.20699999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.70499999999999996,"15min":2.5470000000000002},"min":{"1min":0.40300000000000002,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":3.7490000000000001,"5min":3.8260000000000001,"15min":154.51599999999999},"last":0.435}]}]},{"osd":0,"up_from":89,"seq":382252089769,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3416816,"kb_used_data":2489576,"kb_used_omap":8,"kb_used_meta":927223,"kb_avail":90328336,"statfs":{"total":95995035648,"available":92496216064,"internally_reserved":0,"allocated":2549325824,"data_stored":2547298007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":949477314},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.58499999999999996,"15min":4.0869999999999997},"min":{"1min":0.18099999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":4.2919999999999998,"5min":8.4939999999999998,"15min":216.821},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.59099999999999997,"15min":4.8499999999999996},"min":{"1min":0.20999999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":3.9620000000000002,"5min":8.6229999999999993,"15min":217.02699999999999},"last":0.26000000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.64600000000000002,"15min":1.9910000000000001},"min":{"1min":0.25,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":4.0110000000000001,"5min":4.9459999999999997,"15min":19.788},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.65500000000000003,"15min":2.0310000000000001},"min":{"1min":0.28499999999999998,"5min":0.27200000000000002,"15min":0.189},"max":{"1min":3.8719999999999999,"5min":3.8719999999999999,"15min":25.844999999999999},"last":0.32300000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.63100000000000001,"15min":3.3969999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.73799999999999999,"15min":3.27},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.9199999999999999,"5min":5.8419999999999996,"15min":222.227},"last":0.311}]},{"osd":4,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.59499999999999997,"15min":1.925},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":0.84299999999999997,"5min":4.048,"15min":62.206000000000003},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.63600000000000001,"15min":2.0089999999999999},"min":{"1min":0.29199999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":1.5189999999999999,"5min":4,"15min":62.073999999999998},"last":0.56100000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.76800000000000002,"15min":1.744},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":4.0350000000000001,"5min":5.8769999999999998,"15min":16.808},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.84499999999999997,"15min":1.6499999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.1269999999999998,"5min":4.8689999999999998,"15min":17.507000000000001},"last":0.442}]},{"osd":6,"last update":"Sun Oct 1 09:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.89600000000000002,"15min":1.738},"min":{"1min":0.38200000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.3129999999999997,"5min":5.2110000000000003,"15min":17.535},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.85799999999999998,"15min":1.8089999999999999},"min":{"1min":0.43099999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.2050000000000001,"5min":4.2640000000000002,"15min":15.651999999999999},"last":0.50700000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.79500000000000004,"15min":2.028},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":2.1000000000000001,"5min":4.976,"15min":48.895000000000003},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.89900000000000002,"15min":2.048},"min":{"1min":0.376,"5min":0.376,"15min":0.35099999999999998},"max":{"1min":4.125,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.52700000000000002}]}]},{"osd":2,"up_from":0,"seq":425201762709,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599182,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1400976,"kb_used_data":560648,"kb_used_omap":3,"kb_used_meta":840316,"kb_avail":92344176,"statfs":{"total":95995035648,"available":94560436224,"internally_reserved":0,"allocated":574103552,"data_stored":571981410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":860484327},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":1.4870000000000001,"15min":4.6909999999999998},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":0.65000000000000002,"5min":64.504000000000005,"15min":137.34399999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":0.41499999999999998,"5min":1.4870000000000001,"15min":4.6959999999999997},"min":{"1min":0.219,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":1.2849999999999999,"5min":64.448999999999998,"15min":137.73400000000001},"last":0.46000000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.51700000000000002,"15min":4.8719999999999999},"min":{"1min":0.25,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":1.3500000000000001,"5min":6.0250000000000004,"15min":162.22999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.40600000000000003,"15min":4.9199999999999999},"min":{"1min":0.20899999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.77600000000000002,"5min":2.931,"15min":162.70699999999999},"last":0.27400000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56599999999999995,"15min":1.5800000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.20799999999999999},"max":{"1min":2.9049999999999998,"5min":7.141,"15min":14.254},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.52300000000000002,"15min":1.5860000000000001},"min":{"1min":0.314,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":0.80400000000000005,"5min":4.5049999999999999,"15min":15.523},"last":0.32700000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.54600000000000004,"15min":1.71},"min":{"1min":0.34999999999999998,"5min":0.312,"15min":0.26200000000000001},"max":{"1min":0.874,"5min":6.8890000000000002,"15min":77.793000000000006},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.58699999999999997,"15min":1.8919999999999999},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.409,"5min":6.0940000000000003,"15min":76.646000000000001},"last":0.439}]},{"osd":5,"last update":"Sun Oct 1 09:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.82599999999999996,"15min":2.1419999999999999},"min":{"1min":0.314,"5min":0.29399999999999998,"15min":0.27500000000000002},"max":{"1min":2.895,"5min":6.0659999999999998,"15min":28.117999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.76700000000000002,"15min":2.2450000000000001},"min":{"1min":0.34899999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.0720000000000001,"5min":7.1559999999999997,"15min":28.34},"last":0.34200000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:45:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.512,"15min":1.879},"min":{"1min":0.33900000000000002,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":2.9100000000000001,"5min":73.887,"15min":73.887},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.6559999999999999,"15min":1.9530000000000001},"min":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":1.1379999999999999,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.48899999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.86299999999999999,"15min":1.7809999999999999},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":3.6480000000000001,"5min":7.0620000000000003,"15min":13.085000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.746,"5min":0.77200000000000002,"15min":1.633},"min":{"1min":0.38800000000000001,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.7120000000000002,"5min":4.1779999999999999,"15min":13.191000000000001},"last":0.44900000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019433,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3195012,"kb_used_data":2300732,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":90550140,"statfs":{"total":95995035648,"available":92723343360,"internally_reserved":0,"allocated":2355949568,"data_stored":2355009862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.623,"15min":3.9359999999999999},"min":{"1min":0.22600000000000001,"5min":0.22500000000000001,"15min":0.21199999999999999},"max":{"1min":0.51400000000000001,"5min":9.6690000000000005,"15min":155.315},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.371,"5min":0.72499999999999998,"15min":4.2910000000000004},"min":{"1min":0.253,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.499,"5min":6.1459999999999999,"15min":155.334},"last":0.26100000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.72399999999999998,"15min":5.2430000000000003},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":0.57799999999999996,"5min":5.944,"15min":190.61199999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.63400000000000001,"15min":5.9699999999999998},"min":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.25600000000000001},"max":{"1min":1.679,"5min":3.7469999999999999,"15min":253.273},"last":0.67400000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.74299999999999999,"15min":2.1150000000000002},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.1639999999999999,"5min":5.702,"15min":170.45400000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.748,"15min":1.889},"min":{"1min":0.38600000000000001,"5min":0.314,"15min":0.25800000000000001},"max":{"1min":2.121,"5min":5.6150000000000002,"15min":36.515999999999998},"last":0.374}]},{"osd":3,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.69699999999999995,"15min":2.125},"min":{"1min":0.29199999999999998,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":2.0379999999999998,"5min":5.5279999999999996,"15min":140.881},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.68500000000000005,"15min":2.2749999999999999},"min":{"1min":0.32300000000000001,"5min":0.312,"15min":0.311},"max":{"1min":6.2969999999999997,"5min":6.2969999999999997,"15min":140.911},"last":0.70499999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:45:39 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.9279999999999999,"15min":1.4870000000000001},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":2.9649999999999999,"5min":110.003,"15min":110.003},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.62,"5min":1.8779999999999999,"15min":1.3959999999999999},"min":{"1min":0.34999999999999998,"5min":0.29899999999999999,"15min":0.28899999999999998},"max":{"1min":2.9910000000000001,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.63100000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.623,"15min":0.91900000000000004},"min":{"1min":0.34599999999999997,"5min":0.28799999999999998,"15min":0.27200000000000002},"max":{"1min":0.63700000000000001,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.63400000000000001,"15min":1.0289999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28999999999999998},"max":{"1min":1.3340000000000001,"5min":4.0220000000000002,"15min":10.076000000000001},"last":0.45000000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.66800000000000004,"15min":1.139},"min":{"1min":0.33400000000000002,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":2.98,"5min":3.8580000000000001,"15min":38.034999999999997},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.91400000000000003,"15min":1.4350000000000001},"min":{"1min":0.40899999999999997,"5min":0.36699999999999999,"15min":0.308},"max":{"1min":3.113,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.42599999999999999}]}]},{"osd":5,"up_from":379,"seq":1627792605339,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1564272,"kb_used_data":1191528,"kb_used_omap":13,"kb_used_meta":372722,"kb_avail":92180880,"statfs":{"total":95995035648,"available":94393221120,"internally_reserved":0,"allocated":1220124672,"data_stored":1219200870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":381667826},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":1.004,"15min":2.157},"min":{"1min":0.28699999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.1400000000000001,"5min":27.584,"15min":84.340999999999994},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.97699999999999998,"15min":2.0910000000000002},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":4.2300000000000004,"5min":27.669,"15min":84.356999999999999},"last":0.53700000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":2.0430000000000001,"15min":5.6200000000000001},"min":{"1min":0.26600000000000001,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":4.3289999999999997,"5min":106.56999999999999,"15min":295.71300000000002},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.495,"5min":2.0630000000000002,"15min":5.6260000000000003},"min":{"1min":0.27200000000000002,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":2.169,"5min":106.596,"15min":299.59399999999999},"last":0.35599999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.77000000000000002,"15min":1.5089999999999999},"min":{"1min":0.28799999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.2670000000000003,"5min":5.3470000000000004,"15min":14.199999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.89800000000000002,"15min":1.468},"min":{"1min":0.315,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":4.3719999999999999,"5min":5.782,"15min":14.430999999999999},"last":0.54200000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.71799999999999997,"15min":1.417},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.1880000000000002,"5min":4.0209999999999999,"15min":14.004},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.84399999999999997,"15min":1.5629999999999999},"min":{"1min":0.36599999999999999,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":4.2910000000000004,"5min":5.4400000000000004,"15min":13.991},"last":0.55200000000000005}]},{"osd":4,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.86599999999999999,"15min":1.3959999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.1929999999999996,"5min":5.9459999999999997,"15min":122.971},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.02,"15min":1.605},"min":{"1min":0.30399999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.3869999999999996,"5min":12.445,"15min":122.89100000000001},"last":0.34300000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.86899999999999999,"15min":0.99399999999999999},"min":{"1min":0.314,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":2.2069999999999999,"5min":6.7750000000000004,"15min":12.545},"last":0.307},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.91200000000000003,"15min":1.2130000000000001},"min":{"1min":0.35199999999999998,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":4.3200000000000003,"5min":5.4160000000000004,"15min":14.193},"last":0.56899999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.0349999999999999,"15min":1.1890000000000001},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.3559999999999999,"5min":5.758,"15min":10.813000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.99399999999999999,"15min":1.0660000000000001},"min":{"1min":0.39400000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":2.226,"5min":5.2300000000000004,"15min":12.500999999999999},"last":0.42899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2528514048,"data_stored":2528509976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":551354368,"data_stored":551354368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":567214080,"data_stored":567214080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2349993984,"data_stored":2349989912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1212809216,"data_stored":1212809216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1301942272,"data_stored":1301938200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":907280384,"data_stored":907280384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:46:18.512 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:18.515 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:18 smithi190 ceph-mon[160435]: from='client.47279 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:18.515 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:18 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6268/135099 objects degraded (4.640%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:18.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:18.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:18.617 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:18.645 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:18.996 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:46:19.256 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:19 smithi138 ceph-mon[192513]: pgmap v1613: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 214 op/s; 6196/133707 objects degraded (4.634%); 2699/133707 objects misplaced (2.019%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:46:19.256 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:19 smithi138 ceph-mon[185886]: pgmap v1613: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 214 op/s; 6196/133707 objects degraded (4.634%); 2699/133707 objects misplaced (2.019%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:46:19.434 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:19.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:19.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:19.538 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:19.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:19 smithi190 ceph-mon[160435]: pgmap v1613: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 214 op/s; 6196/133707 objects degraded (4.634%); 2699/133707 objects misplaced (2.019%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:46:19.566 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:19.614 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:19.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:19.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:19.637 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:19.720 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:19.748 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:19.807 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:19.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:19.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:19.914 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:19.948 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:20.131 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:46:20.132 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:46:20.132 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-10-01T09:46:20.132 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:46:20.173 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:20.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:20.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:20.245 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:46:20.267 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:46:20.277 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:20.288 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:46:20.304 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:20.307 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:46:20.325 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:46:20.344 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:46:20.349 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:20.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:20.363 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:46:20.385 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:46:20.389 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:46:20.389 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:46:20.389 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:46:20.389 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:46:20.389 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:46:20.389 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:46:20.389 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:46:20.389 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:46:20.413 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:20 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2913343799' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:20.414 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:20 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2913343799' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:20.459 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:20.490 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:20.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:20 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2913343799' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:20.979 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:46:20.991 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:20.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:20.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:21.095 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:21.288 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:21.350 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:21.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:21.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:21.457 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:46:21.479 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:46:21.482 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:21 smithi138 ceph-mon[185886]: pgmap v1614: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 183 op/s; 6196/133434 objects degraded (4.643%); 2699/133434 objects misplaced (2.023%); 887 KiB/s, 13 objects/s recovering 2023-10-01T09:46:21.482 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:21 smithi138 ceph-mon[192513]: pgmap v1614: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 183 op/s; 6196/133434 objects degraded (4.643%); 2699/133434 objects misplaced (2.023%); 887 KiB/s, 13 objects/s recovering 2023-10-01T09:46:21.484 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:21.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:21 smithi190 ceph-mon[160435]: pgmap v1614: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 183 op/s; 6196/133434 objects degraded (4.643%); 2699/133434 objects misplaced (2.023%); 887 KiB/s, 13 objects/s recovering 2023-10-01T09:46:21.721 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:46:21.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:21.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:46:21.829 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:21.863 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:21.933 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:21.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:21.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:22.052 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:22.085 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:22.140 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:22.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:22.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:22.248 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:22.254 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:22.275 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:22.290 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:22.321 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:22.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:22.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:22.426 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:22.459 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:22.527 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:22.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:22.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:22.634 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:22.670 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:22.734 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:22.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:22.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:22.840 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:22.868 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:22.918 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:22.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:22.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:23.026 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:23.054 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:23.098 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:23.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:23.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:23.206 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:46:23.238 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:23.384 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:46:23.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:23.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:46:23.488 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:23.519 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:23.561 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:23 smithi138 ceph-mon[185886]: pgmap v1615: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 6196/131907 objects degraded (4.697%); 2699/131907 objects misplaced (2.046%); 877 KiB/s, 13 objects/s recovering 2023-10-01T09:46:23.561 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:23 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6196/131907 objects degraded (4.697%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:23.561 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:23 smithi138 ceph-mon[192513]: pgmap v1615: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 6196/131907 objects degraded (4.697%); 2699/131907 objects misplaced (2.046%); 877 KiB/s, 13 objects/s recovering 2023-10-01T09:46:23.561 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:23 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6196/131907 objects degraded (4.697%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:23.575 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:23.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:23.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:23.681 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:23.714 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:23 smithi190 ceph-mon[160435]: pgmap v1615: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 177 op/s; 6196/131907 objects degraded (4.697%); 2699/131907 objects misplaced (2.046%); 877 KiB/s, 13 objects/s recovering 2023-10-01T09:46:23.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:23 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6196/131907 objects degraded (4.697%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:23.917 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:23.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:23.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:23.956 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:46:24.021 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:24.048 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:24.111 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:24.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:24.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:24.217 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:24.244 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:24.290 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:24.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:24.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:24.398 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:24.433 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:24.437 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:46:24.493 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:24.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:24.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:24.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1959058888' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:24.605 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:24.638 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:24.704 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:24.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:24.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:24.815 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:24.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1959058888' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:24.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1959058888' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:24.845 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:25.268 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:46:25.268 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:46:25.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:46:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:46:24] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-01T09:46:25.543 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:25.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:25.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:25.647 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:25.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[185886]: pgmap v1616: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 173 op/s; 6118/131052 objects degraded (4.668%); 2641/131052 objects misplaced (2.015%); 845 KiB/s, 13 objects/s recovering 2023-10-01T09:46:25.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:46:25.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[185886]: failed to encode map e545 with expected crc 2023-10-01T09:46:25.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[185886]: failed to encode map e545 with expected crc 2023-10-01T09:46:25.675 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1959058888' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:46:25.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[185886]: osdmap e545: 8 total, 8 up, 7 in 2023-10-01T09:46:25.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:25.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:46:25.676 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[192513]: pgmap v1616: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 173 op/s; 6118/131052 objects degraded (4.668%); 2641/131052 objects misplaced (2.015%); 845 KiB/s, 13 objects/s recovering 2023-10-01T09:46:25.676 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:46:25.676 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[192513]: failed to encode map e545 with expected crc 2023-10-01T09:46:25.676 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[192513]: failed to encode map e545 with expected crc 2023-10-01T09:46:25.676 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1959058888' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:46:25.677 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[192513]: osdmap e545: 8 total, 8 up, 7 in 2023-10-01T09:46:25.677 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:25.677 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:46:25.679 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:25 smithi190 ceph-mon[160435]: pgmap v1616: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 173 op/s; 6118/131052 objects degraded (4.668%); 2641/131052 objects misplaced (2.015%); 845 KiB/s, 13 objects/s recovering 2023-10-01T09:46:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:25 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:46:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:25 smithi190 ceph-mon[160435]: failed to encode map e545 with expected crc 2023-10-01T09:46:25.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:25 smithi190 ceph-mon[160435]: failed to encode map e545 with expected crc 2023-10-01T09:46:25.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:25 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1959058888' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:46:25.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:25 smithi190 ceph-mon[160435]: osdmap e545: 8 total, 8 up, 7 in 2023-10-01T09:46:25.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:25.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:46:26.084 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:26.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:26.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:26.109 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:46:26.110 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:46:26.125 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:26.147 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:46:26.189 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:26.224 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:26.324 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:26.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:26.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:26.429 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:26.462 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:26.817 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:26.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:26.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:26.907 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1617,"stamp":"2023-10-01T09:46:24.422023+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2865764299,"num_objects":43684,"num_object_clones":0,"num_object_copies":131052,"num_objects_missing_on_primary":1129,"num_objects_missing":1129,"num_objects_degraded":6118,"num_objects_misplaced":2641,"num_objects_unfound":0,"num_objects_dirty":43684,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":123890,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10040,"num_bytes_recovered":644842511,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119254,"ondisk_log_size":119254,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14112980,"kb_used_data":9031708,"kb_used_omap":39,"kb_used_meta":5081112,"kb_avail":642103084,"statfs":{"total":671965249536,"available":657513558016,"internally_reserved":0,"allocated":9248468992,"data_stored":9238495298,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5203059311},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,0,0,0,0,0,0,0,2],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-105644032,"num_objects":-1612,"num_object_clones":0,"num_object_copies":-4836,"num_objects_missing_on_primary":-127,"num_objects_missing":-127,"num_objects_degraded":-150,"num_objects_misplaced":-109,"num_objects_unfound":0,"num_objects_dirty":-1612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1612,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":132,"num_bytes_recovered":8650752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.193188"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":320,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.488306+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:46:10.488306+0000","last_peered":"2023-10-01T09:46:10.488306+0000","last_clean":"2023-10-01T09:46:10.488306+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:46:10.488306+0000","last_undegraded":"2023-10-01T09:46:10.488306+0000","last_fullsized":"2023-10-01T09:46:10.488306+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":164,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482869+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:46:10.482869+0000","last_peered":"2023-10-01T09:46:10.482869+0000","last_clean":"2023-10-01T09:46:10.482869+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:46:10.482869+0000","last_undegraded":"2023-10-01T09:46:10.482869+0000","last_fullsized":"2023-10-01T09:46:10.482869+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":545,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482462+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:46:10.482462+0000","last_peered":"2023-10-01T09:46:10.482462+0000","last_clean":"2023-10-01T09:46:10.482462+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:46:10.482462+0000","last_undegraded":"2023-10-01T09:46:10.482462+0000","last_fullsized":"2023-10-01T09:46:10.482462+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":603,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482763+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:46:10.482763+0000","last_peered":"2023-10-01T09:46:10.482763+0000","last_clean":"2023-10-01T09:46:10.482763+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:46:10.482763+0000","last_undegraded":"2023-10-01T09:46:10.482763+0000","last_fullsized":"2023-10-01T09:46:10.482763+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":484,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482275+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:46:10.482275+0000","last_peered":"2023-10-01T09:46:10.482275+0000","last_clean":"2023-10-01T09:46:10.482275+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:46:10.482275+0000","last_undegraded":"2023-10-01T09:46:10.482275+0000","last_fullsized":"2023-10-01T09:46:10.482275+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":225,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.542863+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:46:10.542863+0000","last_peered":"2023-10-01T09:46:10.542863+0000","last_clean":"2023-10-01T09:46:10.542863+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:46:10.542863+0000","last_undegraded":"2023-10-01T09:46:10.542863+0000","last_fullsized":"2023-10-01T09:46:10.542863+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":601,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.542979+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:46:10.542979+0000","last_peered":"2023-10-01T09:46:10.542979+0000","last_clean":"2023-10-01T09:46:10.542979+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:46:10.542979+0000","last_undegraded":"2023-10-01T09:46:10.542979+0000","last_fullsized":"2023-10-01T09:46:10.542979+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":531,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.478289+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:46:10.478289+0000","last_peered":"2023-10-01T09:46:10.478289+0000","last_clean":"2023-10-01T09:46:10.478289+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:46:10.478289+0000","last_undegraded":"2023-10-01T09:46:10.478289+0000","last_fullsized":"2023-10-01T09:46:10.478289+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":256,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.489687+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:46:10.489687+0000","last_peered":"2023-10-01T09:46:10.489687+0000","last_clean":"2023-10-01T09:46:10.489687+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:46:10.489687+0000","last_undegraded":"2023-10-01T09:46:10.489687+0000","last_fullsized":"2023-10-01T09:46:10.489687+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":63,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500521+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:46:10.500521+0000","last_peered":"2023-10-01T09:46:10.500521+0000","last_clean":"2023-10-01T09:46:10.500521+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:46:10.500521+0000","last_undegraded":"2023-10-01T09:46:10.500521+0000","last_fullsized":"2023-10-01T09:46:10.500521+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":69,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500936+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:46:10.500936+0000","last_peered":"2023-10-01T09:46:10.500936+0000","last_clean":"2023-10-01T09:46:10.500936+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:46:10.500936+0000","last_undegraded":"2023-10-01T09:46:10.500936+0000","last_fullsized":"2023-10-01T09:46:10.500936+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.483109+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:46:10.483109+0000","last_peered":"2023-10-01T09:46:10.483109+0000","last_clean":"2023-10-01T09:46:10.483109+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:46:10.483109+0000","last_undegraded":"2023-10-01T09:46:10.483109+0000","last_fullsized":"2023-10-01T09:46:10.483109+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":445,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.487199+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:46:10.487199+0000","last_peered":"2023-10-01T09:46:10.487199+0000","last_clean":"2023-10-01T09:46:10.487199+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:46:10.487199+0000","last_undegraded":"2023-10-01T09:46:10.487199+0000","last_fullsized":"2023-10-01T09:46:10.487199+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":326,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.487790+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:46:10.487790+0000","last_peered":"2023-10-01T09:46:10.487790+0000","last_clean":"2023-10-01T09:46:10.487790+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:46:10.487790+0000","last_undegraded":"2023-10-01T09:46:10.487790+0000","last_fullsized":"2023-10-01T09:46:10.487790+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":662,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500744+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:46:10.500744+0000","last_peered":"2023-10-01T09:46:10.500744+0000","last_clean":"2023-10-01T09:46:10.500744+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:46:10.500744+0000","last_undegraded":"2023-10-01T09:46:10.500744+0000","last_fullsized":"2023-10-01T09:46:10.500744+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":399,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500981+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:46:10.500981+0000","last_peered":"2023-10-01T09:46:10.500981+0000","last_clean":"2023-10-01T09:46:10.500981+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:46:10.500981+0000","last_undegraded":"2023-10-01T09:46:10.500981+0000","last_fullsized":"2023-10-01T09:46:10.500981+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":72,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500663+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:46:10.500663+0000","last_peered":"2023-10-01T09:46:10.500663+0000","last_clean":"2023-10-01T09:46:10.500663+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:46:10.500663+0000","last_undegraded":"2023-10-01T09:46:10.500663+0000","last_fullsized":"2023-10-01T09:46:10.500663+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":58,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.486193+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:46:10.486193+0000","last_peered":"2023-10-01T09:46:10.486193+0000","last_clean":"2023-10-01T09:46:10.486193+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:46:10.486193+0000","last_undegraded":"2023-10-01T09:46:10.486193+0000","last_fullsized":"2023-10-01T09:46:10.486193+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":555,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500644+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:46:10.500644+0000","last_peered":"2023-10-01T09:46:10.500644+0000","last_clean":"2023-10-01T09:46:10.500644+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:46:10.500644+0000","last_undegraded":"2023-10-01T09:46:10.500644+0000","last_fullsized":"2023-10-01T09:46:10.500644+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":567,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482834+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:46:10.482834+0000","last_peered":"2023-10-01T09:46:10.482834+0000","last_clean":"2023-10-01T09:46:10.482834+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:46:10.482834+0000","last_undegraded":"2023-10-01T09:46:10.482834+0000","last_fullsized":"2023-10-01T09:46:10.482834+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":423,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482117+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:46:10.482117+0000","last_peered":"2023-10-01T09:46:10.482117+0000","last_clean":"2023-10-01T09:46:10.482117+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:46:10.482117+0000","last_undegraded":"2023-10-01T09:46:10.482117+0000","last_fullsized":"2023-10-01T09:46:10.482117+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.501228+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:46:10.501228+0000","last_peered":"2023-10-01T09:46:10.501228+0000","last_clean":"2023-10-01T09:46:10.501228+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:46:10.501228+0000","last_undegraded":"2023-10-01T09:46:10.501228+0000","last_fullsized":"2023-10-01T09:46:10.501228+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"544'7720","reported_seq":7797,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:22.106629+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:46:22.106629+0000","last_peered":"2023-10-01T09:46:22.106629+0000","last_clean":"2023-10-01T09:46:22.106629+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:46:22.106629+0000","last_undegraded":"2023-10-01T09:46:22.106629+0000","last_fullsized":"2023-10-01T09:46:22.106629+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7720,"log_dups_size":0,"ondisk_log_size":7720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175112192,"num_objects":2672,"num_object_clones":0,"num_object_copies":8016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7720,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":67,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.501055+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:46:10.501055+0000","last_peered":"2023-10-01T09:46:10.501055+0000","last_clean":"2023-10-01T09:46:10.501055+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:46:10.501055+0000","last_undegraded":"2023-10-01T09:46:10.501055+0000","last_fullsized":"2023-10-01T09:46:10.501055+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":264,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.486317+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:46:10.486317+0000","last_peered":"2023-10-01T09:46:10.486317+0000","last_clean":"2023-10-01T09:46:10.486317+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:46:10.486317+0000","last_undegraded":"2023-10-01T09:46:10.486317+0000","last_fullsized":"2023-10-01T09:46:10.486317+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"544'7758","reported_seq":15530,"reported_epoch":544,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:46:22.401736+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:46:22.401736+0000","last_peered":"2023-10-01T09:46:22.401736+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:46:22.401736+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7758,"log_dups_size":0,"ondisk_log_size":7758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173539328,"num_objects":2648,"num_object_clones":0,"num_object_copies":7944,"num_objects_missing_on_primary":1129,"num_objects_missing":1129,"num_objects_degraded":3708,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7758,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2556,"num_bytes_recovered":167510016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1450},{"shards":"0,4,6","objects":1129},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":538,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.501206+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:46:10.501206+0000","last_peered":"2023-10-01T09:46:10.501206+0000","last_clean":"2023-10-01T09:46:10.501206+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:46:10.501206+0000","last_undegraded":"2023-10-01T09:46:10.501206+0000","last_fullsized":"2023-10-01T09:46:10.501206+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":600,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482379+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:46:10.482379+0000","last_peered":"2023-10-01T09:46:10.482379+0000","last_clean":"2023-10-01T09:46:10.482379+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:46:10.482379+0000","last_undegraded":"2023-10-01T09:46:10.482379+0000","last_fullsized":"2023-10-01T09:46:10.482379+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"544'7688","reported_seq":7768,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:21.786959+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:46:21.786959+0000","last_peered":"2023-10-01T09:46:21.786959+0000","last_clean":"2023-10-01T09:46:21.786959+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:46:21.786959+0000","last_undegraded":"2023-10-01T09:46:21.786959+0000","last_fullsized":"2023-10-01T09:46:21.786959+0000","mapping_epoch":475,"log_start":"481'700","ondisk_log_start":"481'700","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6988,"log_dups_size":0,"ondisk_log_size":6988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179830784,"num_objects":2744,"num_object_clones":0,"num_object_copies":8232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7688,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":624,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.501039+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:46:10.501039+0000","last_peered":"2023-10-01T09:46:10.501039+0000","last_clean":"2023-10-01T09:46:10.501039+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:46:10.501039+0000","last_undegraded":"2023-10-01T09:46:10.501039+0000","last_fullsized":"2023-10-01T09:46:10.501039+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":421,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:15.275682+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:46:15.275682+0000","last_peered":"2023-10-01T09:46:15.275682+0000","last_clean":"2023-10-01T09:46:15.275682+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:46:15.275682+0000","last_undegraded":"2023-10-01T09:46:15.275682+0000","last_fullsized":"2023-10-01T09:46:15.275682+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"544'7732","reported_seq":14417,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:21.799101+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:46:21.799101+0000","last_peered":"2023-10-01T09:46:21.799101+0000","last_clean":"2023-10-01T09:46:21.799101+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:46:21.799101+0000","last_undegraded":"2023-10-01T09:46:21.799101+0000","last_fullsized":"2023-10-01T09:46:21.799101+0000","mapping_epoch":505,"log_start":"481'700","ondisk_log_start":"481'700","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7032,"log_dups_size":0,"ondisk_log_size":7032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174981120,"num_objects":2670,"num_object_clones":0,"num_object_copies":8010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7732,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":264,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.486480+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:46:10.486480+0000","last_peered":"2023-10-01T09:46:10.486480+0000","last_clean":"2023-10-01T09:46:10.486480+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:46:10.486480+0000","last_undegraded":"2023-10-01T09:46:10.486480+0000","last_fullsized":"2023-10-01T09:46:10.486480+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":523,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482222+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:46:10.482222+0000","last_peered":"2023-10-01T09:46:10.482222+0000","last_clean":"2023-10-01T09:46:10.482222+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:46:10.482222+0000","last_undegraded":"2023-10-01T09:46:10.482222+0000","last_fullsized":"2023-10-01T09:46:10.482222+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"544'7694","reported_seq":7774,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:20.070517+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:46:20.070517+0000","last_peered":"2023-10-01T09:46:20.070517+0000","last_clean":"2023-10-01T09:46:20.070517+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:46:20.070517+0000","last_undegraded":"2023-10-01T09:46:20.070517+0000","last_fullsized":"2023-10-01T09:46:20.070517+0000","mapping_epoch":475,"log_start":"480'100","ondisk_log_start":"480'100","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7594,"log_dups_size":0,"ondisk_log_size":7594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184418304,"num_objects":2814,"num_object_clones":0,"num_object_copies":8442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7694,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":124,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.488106+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:46:10.488106+0000","last_peered":"2023-10-01T09:46:10.488106+0000","last_clean":"2023-10-01T09:46:10.488106+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:46:10.488106+0000","last_undegraded":"2023-10-01T09:46:10.488106+0000","last_fullsized":"2023-10-01T09:46:10.488106+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":198,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.487342+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:46:10.487342+0000","last_peered":"2023-10-01T09:46:10.487342+0000","last_clean":"2023-10-01T09:46:10.487342+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:46:10.487342+0000","last_undegraded":"2023-10-01T09:46:10.487342+0000","last_fullsized":"2023-10-01T09:46:10.487342+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"544'7663","reported_seq":7908,"reported_epoch":544,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:46:22.148088+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:46:22.148088+0000","last_peered":"2023-10-01T09:46:22.148088+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:46:22.148088+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7663,"log_dups_size":0,"ondisk_log_size":7663,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177274880,"num_objects":2705,"num_object_clones":0,"num_object_copies":8115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2400,"num_objects_misplaced":2641,"num_objects_unfound":0,"num_objects_dirty":2705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7663,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":5242880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2641},{"shards":"4,5","objects":-2336}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":442,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.488432+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:46:10.488432+0000","last_peered":"2023-10-01T09:46:10.488432+0000","last_clean":"2023-10-01T09:46:10.488432+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:46:10.488432+0000","last_undegraded":"2023-10-01T09:46:10.488432+0000","last_fullsized":"2023-10-01T09:46:10.488432+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":603,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482951+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:46:10.482951+0000","last_peered":"2023-10-01T09:46:10.482951+0000","last_clean":"2023-10-01T09:46:10.482951+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:46:10.482951+0000","last_undegraded":"2023-10-01T09:46:10.482951+0000","last_fullsized":"2023-10-01T09:46:10.482951+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"544'7668","reported_seq":7754,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:22.462689+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:46:22.462689+0000","last_peered":"2023-10-01T09:46:22.462689+0000","last_clean":"2023-10-01T09:46:22.462689+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:46:22.462689+0000","last_undegraded":"2023-10-01T09:46:22.462689+0000","last_fullsized":"2023-10-01T09:46:22.462689+0000","mapping_epoch":483,"log_start":"480'100","ondisk_log_start":"480'100","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7568,"log_dups_size":0,"ondisk_log_size":7568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177602560,"num_objects":2710,"num_object_clones":0,"num_object_copies":8130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7668,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":438,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.489076+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:46:10.489076+0000","last_peered":"2023-10-01T09:46:10.489076+0000","last_clean":"2023-10-01T09:46:10.489076+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:46:10.489076+0000","last_undegraded":"2023-10-01T09:46:10.489076+0000","last_fullsized":"2023-10-01T09:46:10.489076+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":279,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.488813+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:46:10.488813+0000","last_peered":"2023-10-01T09:46:10.488813+0000","last_clean":"2023-10-01T09:46:10.488813+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:46:10.488813+0000","last_undegraded":"2023-10-01T09:46:10.488813+0000","last_fullsized":"2023-10-01T09:46:10.488813+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"544'7754","reported_seq":7831,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:21.248285+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:46:21.248285+0000","last_peered":"2023-10-01T09:46:21.248285+0000","last_clean":"2023-10-01T09:46:21.248285+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:46:21.248285+0000","last_undegraded":"2023-10-01T09:46:21.248285+0000","last_fullsized":"2023-10-01T09:46:21.248285+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7754,"log_dups_size":0,"ondisk_log_size":7754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179830784,"num_objects":2744,"num_object_clones":0,"num_object_copies":8232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7754,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":408,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482774+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:46:10.482774+0000","last_peered":"2023-10-01T09:46:10.482774+0000","last_clean":"2023-10-01T09:46:10.482774+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:46:10.482774+0000","last_undegraded":"2023-10-01T09:46:10.482774+0000","last_fullsized":"2023-10-01T09:46:10.482774+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":455,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.483160+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:46:10.483160+0000","last_peered":"2023-10-01T09:46:10.483160+0000","last_clean":"2023-10-01T09:46:10.483160+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:46:10.483160+0000","last_undegraded":"2023-10-01T09:46:10.483160+0000","last_fullsized":"2023-10-01T09:46:10.483160+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"544'7527","reported_seq":7604,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:21.272806+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:46:21.272806+0000","last_peered":"2023-10-01T09:46:21.272806+0000","last_clean":"2023-10-01T09:46:21.272806+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:46:21.272806+0000","last_undegraded":"2023-10-01T09:46:21.272806+0000","last_fullsized":"2023-10-01T09:46:21.272806+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7527,"log_dups_size":0,"ondisk_log_size":7527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173735936,"num_objects":2651,"num_object_clones":0,"num_object_copies":7953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7527,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":937,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500881+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:46:10.500881+0000","last_peered":"2023-10-01T09:46:10.500881+0000","last_clean":"2023-10-01T09:46:10.500881+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:46:10.500881+0000","last_undegraded":"2023-10-01T09:46:10.500881+0000","last_fullsized":"2023-10-01T09:46:10.500881+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":274,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.489499+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:46:10.489499+0000","last_peered":"2023-10-01T09:46:10.489499+0000","last_clean":"2023-10-01T09:46:10.489499+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:46:10.489499+0000","last_undegraded":"2023-10-01T09:46:10.489499+0000","last_fullsized":"2023-10-01T09:46:10.489499+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":68,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.483182+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:46:10.483182+0000","last_peered":"2023-10-01T09:46:10.483182+0000","last_clean":"2023-10-01T09:46:10.483182+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:46:10.483182+0000","last_undegraded":"2023-10-01T09:46:10.483182+0000","last_fullsized":"2023-10-01T09:46:10.483182+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"544'7592","reported_seq":7672,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:21.824822+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:46:21.824822+0000","last_peered":"2023-10-01T09:46:21.824822+0000","last_clean":"2023-10-01T09:46:21.824822+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:46:21.824822+0000","last_undegraded":"2023-10-01T09:46:21.824822+0000","last_fullsized":"2023-10-01T09:46:21.824822+0000","mapping_epoch":475,"log_start":"481'600","ondisk_log_start":"481'600","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182059008,"num_objects":2778,"num_object_clones":0,"num_object_copies":8334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7592,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1209,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.486803+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:46:10.486803+0000","last_peered":"2023-10-01T09:46:10.486803+0000","last_clean":"2023-10-01T09:46:10.486803+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:46:10.486803+0000","last_undegraded":"2023-10-01T09:46:10.486803+0000","last_fullsized":"2023-10-01T09:46:10.486803+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":419,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.487971+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:46:10.487971+0000","last_peered":"2023-10-01T09:46:10.487971+0000","last_clean":"2023-10-01T09:46:10.487971+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:46:10.487971+0000","last_undegraded":"2023-10-01T09:46:10.487971+0000","last_fullsized":"2023-10-01T09:46:10.487971+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":247,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.486615+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:46:10.486615+0000","last_peered":"2023-10-01T09:46:10.486615+0000","last_clean":"2023-10-01T09:46:10.486615+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:46:10.486615+0000","last_undegraded":"2023-10-01T09:46:10.486615+0000","last_fullsized":"2023-10-01T09:46:10.486615+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"544'7682","reported_seq":14980,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:22.114458+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:46:22.114458+0000","last_peered":"2023-10-01T09:46:22.114458+0000","last_clean":"2023-10-01T09:46:22.114458+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:46:22.114458+0000","last_undegraded":"2023-10-01T09:46:22.114458+0000","last_fullsized":"2023-10-01T09:46:22.114458+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7682,"log_dups_size":0,"ondisk_log_size":7682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176553984,"num_objects":2694,"num_object_clones":0,"num_object_copies":8082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7682,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":890,"reported_epoch":543,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:46:10.481871+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:46:10.481871+0000","last_peered":"2023-10-01T09:46:10.481871+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:46:10.481871+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:46:10.481871+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":608,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:15.275644+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:46:15.275644+0000","last_peered":"2023-10-01T09:46:15.275644+0000","last_clean":"2023-10-01T09:46:15.275644+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:46:15.275644+0000","last_undegraded":"2023-10-01T09:46:15.275644+0000","last_fullsized":"2023-10-01T09:46:15.275644+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":68,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.483052+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:46:10.483052+0000","last_peered":"2023-10-01T09:46:10.483052+0000","last_clean":"2023-10-01T09:46:10.483052+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:46:10.483052+0000","last_undegraded":"2023-10-01T09:46:10.483052+0000","last_fullsized":"2023-10-01T09:46:10.483052+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"544'7550","reported_seq":7630,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:19.523948+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:46:19.523948+0000","last_peered":"2023-10-01T09:46:19.523948+0000","last_clean":"2023-10-01T09:46:19.523948+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:46:19.523948+0000","last_undegraded":"2023-10-01T09:46:19.523948+0000","last_fullsized":"2023-10-01T09:46:19.523948+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7550,"log_dups_size":0,"ondisk_log_size":7550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178388992,"num_objects":2722,"num_object_clones":0,"num_object_copies":8166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7550,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1660,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.487160+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:46:10.487160+0000","last_peered":"2023-10-01T09:46:10.487160+0000","last_clean":"2023-10-01T09:46:10.487160+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:46:10.487160+0000","last_undegraded":"2023-10-01T09:46:10.487160+0000","last_fullsized":"2023-10-01T09:46:10.487160+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":574,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500560+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:46:10.500560+0000","last_peered":"2023-10-01T09:46:10.500560+0000","last_clean":"2023-10-01T09:46:10.500560+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:46:10.500560+0000","last_undegraded":"2023-10-01T09:46:10.500560+0000","last_fullsized":"2023-10-01T09:46:10.500560+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":352,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.486874+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:46:10.486874+0000","last_peered":"2023-10-01T09:46:10.486874+0000","last_clean":"2023-10-01T09:46:10.486874+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:46:10.486874+0000","last_undegraded":"2023-10-01T09:46:10.486874+0000","last_fullsized":"2023-10-01T09:46:10.486874+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":342,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.487448+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:46:10.487448+0000","last_peered":"2023-10-01T09:46:10.487448+0000","last_clean":"2023-10-01T09:46:10.487448+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:46:10.487448+0000","last_undegraded":"2023-10-01T09:46:10.487448+0000","last_fullsized":"2023-10-01T09:46:10.487448+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"544'7752","reported_seq":7829,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:21.243520+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:46:21.243520+0000","last_peered":"2023-10-01T09:46:21.243520+0000","last_clean":"2023-10-01T09:46:21.243520+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:46:21.243520+0000","last_undegraded":"2023-10-01T09:46:21.243520+0000","last_fullsized":"2023-10-01T09:46:21.243520+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7752,"log_dups_size":0,"ondisk_log_size":7752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182190080,"num_objects":2780,"num_object_clones":0,"num_object_copies":8340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7752,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":751,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500799+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:46:10.500799+0000","last_peered":"2023-10-01T09:46:10.500799+0000","last_clean":"2023-10-01T09:46:10.500799+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:46:10.500799+0000","last_undegraded":"2023-10-01T09:46:10.500799+0000","last_fullsized":"2023-10-01T09:46:10.500799+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":585,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:16.232392+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:46:16.232392+0000","last_peered":"2023-10-01T09:46:16.232392+0000","last_clean":"2023-10-01T09:46:16.232392+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:46:16.232392+0000","last_undegraded":"2023-10-01T09:46:16.232392+0000","last_fullsized":"2023-10-01T09:46:16.232392+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":584,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482607+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:46:10.482607+0000","last_peered":"2023-10-01T09:46:10.482607+0000","last_clean":"2023-10-01T09:46:10.482607+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:46:10.482607+0000","last_undegraded":"2023-10-01T09:46:10.482607+0000","last_fullsized":"2023-10-01T09:46:10.482607+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"544'7669","reported_seq":7750,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:21.782376+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:46:21.782376+0000","last_peered":"2023-10-01T09:46:21.782376+0000","last_clean":"2023-10-01T09:46:21.782376+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:46:21.782376+0000","last_undegraded":"2023-10-01T09:46:21.782376+0000","last_fullsized":"2023-10-01T09:46:21.782376+0000","mapping_epoch":475,"log_start":"481'600","ondisk_log_start":"481'600","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7069,"log_dups_size":0,"ondisk_log_size":7069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182845464,"num_objects":2791,"num_object_clones":0,"num_object_copies":8373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2791,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7669,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":982,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.501122+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:46:10.501122+0000","last_peered":"2023-10-01T09:46:10.501122+0000","last_clean":"2023-10-01T09:46:10.501122+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:46:10.501122+0000","last_undegraded":"2023-10-01T09:46:10.501122+0000","last_fullsized":"2023-10-01T09:46:10.501122+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":601,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.486550+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:46:10.486550+0000","last_peered":"2023-10-01T09:46:10.486550+0000","last_clean":"2023-10-01T09:46:10.486550+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:46:10.486550+0000","last_undegraded":"2023-10-01T09:46:10.486550+0000","last_fullsized":"2023-10-01T09:46:10.486550+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":454,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.487227+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:46:10.487227+0000","last_peered":"2023-10-01T09:46:10.487227+0000","last_clean":"2023-10-01T09:46:10.487227+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:46:10.487227+0000","last_undegraded":"2023-10-01T09:46:10.487227+0000","last_fullsized":"2023-10-01T09:46:10.487227+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"544'7616","reported_seq":7701,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:21.229024+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:46:21.229024+0000","last_peered":"2023-10-01T09:46:21.229024+0000","last_clean":"2023-10-01T09:46:21.229024+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:46:21.229024+0000","last_undegraded":"2023-10-01T09:46:21.229024+0000","last_fullsized":"2023-10-01T09:46:21.229024+0000","mapping_epoch":479,"log_start":"481'900","ondisk_log_start":"481'900","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6716,"log_dups_size":0,"ondisk_log_size":6716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179568640,"num_objects":2740,"num_object_clones":0,"num_object_copies":8220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7616,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":483,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.501315+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:46:10.501315+0000","last_peered":"2023-10-01T09:46:10.501315+0000","last_clean":"2023-10-01T09:46:10.501315+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:46:10.501315+0000","last_undegraded":"2023-10-01T09:46:10.501315+0000","last_fullsized":"2023-10-01T09:46:10.501315+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1177,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482883+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:46:10.482883+0000","last_peered":"2023-10-01T09:46:10.482883+0000","last_clean":"2023-10-01T09:46:10.482883+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:46:10.482883+0000","last_undegraded":"2023-10-01T09:46:10.482883+0000","last_fullsized":"2023-10-01T09:46:10.482883+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":72,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.500474+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:46:10.500474+0000","last_peered":"2023-10-01T09:46:10.500474+0000","last_clean":"2023-10-01T09:46:10.500474+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:46:10.500474+0000","last_undegraded":"2023-10-01T09:46:10.500474+0000","last_fullsized":"2023-10-01T09:46:10.500474+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"544'7719","reported_seq":7804,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:21.256449+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:46:21.256449+0000","last_peered":"2023-10-01T09:46:21.256449+0000","last_clean":"2023-10-01T09:46:21.256449+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:46:21.256449+0000","last_undegraded":"2023-10-01T09:46:21.256449+0000","last_fullsized":"2023-10-01T09:46:21.256449+0000","mapping_epoch":479,"log_start":"481'1000","ondisk_log_start":"481'1000","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6719,"log_dups_size":0,"ondisk_log_size":6719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183042048,"num_objects":2793,"num_object_clones":0,"num_object_copies":8379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7719,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":76,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.501146+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:46:10.501146+0000","last_peered":"2023-10-01T09:46:10.501146+0000","last_clean":"2023-10-01T09:46:10.501146+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:46:10.501146+0000","last_undegraded":"2023-10-01T09:46:10.501146+0000","last_fullsized":"2023-10-01T09:46:10.501146+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":562,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482690+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:46:10.482690+0000","last_peered":"2023-10-01T09:46:10.482690+0000","last_clean":"2023-10-01T09:46:10.482690+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:46:10.482690+0000","last_undegraded":"2023-10-01T09:46:10.482690+0000","last_fullsized":"2023-10-01T09:46:10.482690+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":267,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.487079+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:46:10.487079+0000","last_peered":"2023-10-01T09:46:10.487079+0000","last_clean":"2023-10-01T09:46:10.487079+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:46:10.487079+0000","last_undegraded":"2023-10-01T09:46:10.487079+0000","last_fullsized":"2023-10-01T09:46:10.487079+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":511,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482360+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:46:10.482360+0000","last_peered":"2023-10-01T09:46:10.482360+0000","last_clean":"2023-10-01T09:46:10.482360+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:46:10.482360+0000","last_undegraded":"2023-10-01T09:46:10.482360+0000","last_fullsized":"2023-10-01T09:46:10.482360+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":624,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482721+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:46:10.482721+0000","last_peered":"2023-10-01T09:46:10.482721+0000","last_clean":"2023-10-01T09:46:10.482721+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:46:10.482721+0000","last_undegraded":"2023-10-01T09:46:10.482721+0000","last_fullsized":"2023-10-01T09:46:10.482721+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":279,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.486410+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:46:10.486410+0000","last_peered":"2023-10-01T09:46:10.486410+0000","last_clean":"2023-10-01T09:46:10.486410+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:46:10.486410+0000","last_undegraded":"2023-10-01T09:46:10.486410+0000","last_fullsized":"2023-10-01T09:46:10.486410+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":68,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482311+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:46:10.482311+0000","last_peered":"2023-10-01T09:46:10.482311+0000","last_clean":"2023-10-01T09:46:10.482311+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:46:10.482311+0000","last_undegraded":"2023-10-01T09:46:10.482311+0000","last_fullsized":"2023-10-01T09:46:10.482311+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":279,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.489581+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:46:10.489581+0000","last_peered":"2023-10-01T09:46:10.489581+0000","last_clean":"2023-10-01T09:46:10.489581+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:46:10.489581+0000","last_undegraded":"2023-10-01T09:46:10.489581+0000","last_fullsized":"2023-10-01T09:46:10.489581+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":457,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.483398+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:46:10.483398+0000","last_peered":"2023-10-01T09:46:10.483398+0000","last_clean":"2023-10-01T09:46:10.483398+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:46:10.483398+0000","last_undegraded":"2023-10-01T09:46:10.483398+0000","last_fullsized":"2023-10-01T09:46:10.483398+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":86,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482679+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:46:10.482679+0000","last_peered":"2023-10-01T09:46:10.482679+0000","last_clean":"2023-10-01T09:46:10.482679+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:46:10.482679+0000","last_undegraded":"2023-10-01T09:46:10.482679+0000","last_fullsized":"2023-10-01T09:46:10.482679+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":502,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.482090+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:46:10.482090+0000","last_peered":"2023-10-01T09:46:10.482090+0000","last_clean":"2023-10-01T09:46:10.482090+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:46:10.482090+0000","last_undegraded":"2023-10-01T09:46:10.482090+0000","last_fullsized":"2023-10-01T09:46:10.482090+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":702,"reported_epoch":544,"state":"active+clean","last_fresh":"2023-10-01T09:46:16.233805+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:46:16.233805+0000","last_peered":"2023-10-01T09:46:16.233805+0000","last_clean":"2023-10-01T09:46:16.233805+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:46:16.233805+0000","last_undegraded":"2023-10-01T09:46:16.233805+0000","last_fullsized":"2023-10-01T09:46:16.233805+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":395,"reported_epoch":543,"state":"active+clean","last_fresh":"2023-10-01T09:46:10.489642+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:46:10.489642+0000","last_peered":"2023-10-01T09:46:10.489642+0000","last_clean":"2023-10-01T09:46:10.489642+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:46:10.489642+0000","last_undegraded":"2023-10-01T09:46:10.489642+0000","last_fullsized":"2023-10-01T09:46:10.489642+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2860974104,"num_objects":43656,"num_object_clones":0,"num_object_copies":130968,"num_objects_missing_on_primary":1129,"num_objects_missing":1129,"num_objects_degraded":6108,"num_objects_misplaced":2641,"num_objects_unfound":0,"num_objects_dirty":43656,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":122784,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9529,"num_bytes_recovered":624492544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9170071552,"data_stored":9170059336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118084,"ondisk_log_size":118084,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430749,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1781904,"kb_used_data":878792,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":91963248,"statfs":{"total":95995035648,"available":94170365952,"internally_reserved":0,"allocated":899883008,"data_stored":898939472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.74399999999999999,"15min":2.5840000000000001},"min":{"1min":0.27700000000000002,"5min":0.246,"15min":0.20999999999999999},"max":{"1min":1.081,"5min":6.8550000000000004,"15min":108.90000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.52500000000000002,"15min":2.7050000000000001},"min":{"1min":0.25700000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.2090000000000001,"5min":5.6639999999999997,"15min":109.187},"last":0.33800000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.66400000000000003,"15min":5.1520000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.222},"max":{"1min":1.0780000000000001,"5min":6.8289999999999997,"15min":229.161},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.59799999999999998,"15min":5.1779999999999999},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":3.0640000000000001,"5min":5.7240000000000002,"15min":230.126},"last":0.28699999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.72799999999999998,"15min":1.617},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.1259999999999999,"5min":5.3090000000000002,"15min":14.901999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.68500000000000005,"15min":1.6100000000000001},"min":{"1min":0.33700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.0009999999999999,"5min":5.6929999999999996,"15min":14.682},"last":0.375}]},{"osd":3,"last update":"Sun Oct 1 09:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.65800000000000003,"15min":2.2000000000000002},"min":{"1min":0.35699999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.476,"5min":5.7750000000000004,"15min":76.555000000000007},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.68100000000000005,"15min":2.2010000000000001},"min":{"1min":0.32800000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.4529999999999998,"5min":6.548,"15min":76.534000000000006},"last":0.45000000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.70499999999999996,"15min":1.042},"min":{"1min":0.28299999999999997,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.276,"5min":6.8769999999999998,"15min":9.6020000000000003},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.68500000000000005,"15min":0.997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.97399999999999998,"5min":5.7439999999999998,"15min":8.4949999999999992},"last":0.435}]},{"osd":5,"last update":"Sun Oct 1 09:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":2.0960000000000001,"15min":1.548},"min":{"1min":0.315,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":0.80000000000000004,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":2.0830000000000002,"15min":1.6459999999999999},"min":{"1min":0.35699999999999998,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":2.8849999999999998,"5min":135.339,"15min":135.339},"last":0.47499999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.075,"15min":1.133},"min":{"1min":0.38,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":0.754,"5min":11.58,"15min":11.853999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.86699999999999999,"15min":1.111},"min":{"1min":0.35599999999999998,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":3.7810000000000001,"5min":13.021000000000001,"15min":14.584},"last":0.5}]}]},{"osd":6,"up_from":294,"seq":1262720385260,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1573324,"kb_used_data":1231556,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":92171828,"statfs":{"total":95995035648,"available":94383951872,"internally_reserved":0,"allocated":1261113344,"data_stored":1260173759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.45200000000000001,"15min":3.681},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20599999999999999},"max":{"1min":0.51700000000000002,"5min":3.3340000000000001,"15min":158.48599999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.55500000000000005,"15min":3.746},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.221},"max":{"1min":3.3519999999999999,"5min":3.3519999999999999,"15min":158.37700000000001},"last":0.41299999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.58499999999999996,"15min":4.5739999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":3.3069999999999999,"5min":3.3069999999999999,"15min":168.947},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.60299999999999998,"15min":4.532},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":3.7469999999999999,"5min":3.7469999999999999,"15min":169.07900000000001},"last":0.40200000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.59999999999999998,"15min":1.4490000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":3.3849999999999998,"5min":3.3849999999999998,"15min":10.92},"last":0.311},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":1.4690000000000001},"min":{"1min":0.32100000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.7160000000000002,"5min":3.7160000000000002,"15min":21.704999999999998},"last":0.32500000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.57999999999999996,"15min":1.8100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.63800000000000001,"5min":3.3759999999999999,"15min":89.753},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.69499999999999995,"15min":1.754},"min":{"1min":0.309,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.2050000000000001,"5min":4.2050000000000001,"15min":89.849999999999994},"last":0.51500000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58899999999999997,"15min":1.069},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.6349999999999998,"5min":4.0250000000000004,"15min":63.134},"last":0.37},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.65800000000000003,"15min":1.0680000000000001},"min":{"1min":0.36599999999999999,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":1.006,"5min":7.4039999999999999,"15min":62.973999999999997},"last":0.43099999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:45:31 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.1599999999999999,"15min":1.0369999999999999},"min":{"1min":0.307,"5min":0.26700000000000002,"15min":0.251},"max":{"1min":0.82199999999999995,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.504},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.2849999999999999,"15min":1.155},"min":{"1min":0.32500000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.6800000000000002,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.497}]},{"osd":7,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.56699999999999995,"15min":1.2729999999999999},"min":{"1min":0.377,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.73399999999999999,"5min":3.339,"15min":94.245999999999995},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.67200000000000004,"15min":1.369},"min":{"1min":0.33100000000000002,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":3.653,"5min":3.653,"15min":94.715000000000003},"last":0.53400000000000003}]}]},{"osd":1,"up_from":94,"seq":403726926236,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1363296,"kb_used_data":541016,"kb_used_omap":7,"kb_used_meta":822264,"kb_avail":92381856,"statfs":{"total":95995035648,"available":94599020544,"internally_reserved":0,"allocated":554000384,"data_stored":551928397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":841998759},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4410000000000001,"5min":1.1140000000000001,"15min":3.8010000000000002},"min":{"1min":0.217,"5min":0.186,"15min":0.182},"max":{"1min":18.556999999999999,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.20599999999999999},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.302,"15min":3.3359999999999999},"min":{"1min":0.23499999999999999,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":18.704000000000001,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.25}]},{"osd":2,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.39800000000000002,"15min":2.2669999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.51100000000000001,"5min":2.8740000000000001,"15min":51.640000000000001},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.61799999999999999,"15min":2.9500000000000002},"min":{"1min":0.245,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":2.9089999999999998,"5min":6.9710000000000001,"15min":154.197},"last":0.32100000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.45700000000000002,"15min":2.9540000000000002},"min":{"1min":0.28199999999999997,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":2.8780000000000001,"5min":3.1949999999999998,"15min":95.305000000000007},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.432,"5min":0.51200000000000001,"15min":3.6720000000000002},"min":{"1min":0.27300000000000002,"5min":0.245,"15min":0.216},"max":{"1min":0.95699999999999996,"5min":4.7839999999999998,"15min":154.483},"last":0.30499999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.59099999999999997,"15min":2.1829999999999998},"min":{"1min":0.312,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":1.5309999999999999,"5min":3.0550000000000002,"15min":154.101},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.49099999999999999,"15min":2.2869999999999999},"min":{"1min":0.34300000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65500000000000003,"5min":2.4209999999999998,"15min":154.56999999999999},"last":0.38}]},{"osd":5,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.246,"15min":2.2530000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.30199999999999999},"max":{"1min":1.696,"5min":59.561,"15min":154.22999999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.1970000000000001,"15min":2.4649999999999999},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.307},"max":{"1min":0.67300000000000004,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.41399999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.65700000000000003,"15min":2.448},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.944,"5min":3.3660000000000001,"15min":154.12200000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64600000000000002,"15min":2.3450000000000002},"min":{"1min":0.374,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":1.7549999999999999,"5min":2.5630000000000002,"15min":154.16999999999999},"last":0.39900000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.74299999999999999,"15min":2.5230000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.33900000000000002},"max":{"1min":3.6160000000000001,"5min":5.2169999999999996,"15min":154.20699999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.70499999999999996,"15min":2.5470000000000002},"min":{"1min":0.40300000000000002,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":3.7490000000000001,"5min":3.8260000000000001,"15min":154.51599999999999},"last":0.45000000000000001}]}]},{"osd":0,"up_from":89,"seq":382252089771,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3355708,"kb_used_data":2424372,"kb_used_omap":8,"kb_used_meta":931319,"kb_avail":90389444,"statfs":{"total":95995035648,"available":92558790656,"internally_reserved":0,"allocated":2482556928,"data_stored":2480524964,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":953671618},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.311,"5min":0.42399999999999999,"15min":3.988},"min":{"1min":0.20699999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.83699999999999997,"5min":4.2919999999999998,"15min":216.821},"last":0.315},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.44800000000000001,"15min":4.6740000000000004},"min":{"1min":0.218,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.56799999999999995,"5min":3.9620000000000002,"15min":217.02699999999999},"last":0.495}]},{"osd":2,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.64600000000000002,"15min":1.9910000000000001},"min":{"1min":0.25,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":4.0110000000000001,"5min":4.9459999999999997,"15min":19.788},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.65500000000000003,"15min":2.0310000000000001},"min":{"1min":0.28499999999999998,"5min":0.27200000000000002,"15min":0.189},"max":{"1min":3.8719999999999999,"5min":3.8719999999999999,"15min":25.844999999999999},"last":0.46999999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.504,"15min":3.153},"min":{"1min":0.24099999999999999,"5min":0.22,"15min":0.22},"max":{"1min":1.9039999999999999,"5min":6.5670000000000002,"15min":222.16499999999999},"last":1.9039999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.68200000000000005,"15min":3.0019999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.1379999999999999,"5min":4.9199999999999999,"15min":222.227},"last":0.61299999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.59499999999999997,"15min":1.925},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":0.84299999999999997,"5min":4.048,"15min":62.206000000000003},"last":0.434},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.63600000000000001,"15min":2.0089999999999999},"min":{"1min":0.29199999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":1.5189999999999999,"5min":4,"15min":62.073999999999998},"last":0.39500000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.76800000000000002,"15min":1.744},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":4.0350000000000001,"5min":5.8769999999999998,"15min":16.808},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.84499999999999997,"15min":1.6499999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.1269999999999998,"5min":4.8689999999999998,"15min":17.507000000000001},"last":1.8100000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.89600000000000002,"15min":1.738},"min":{"1min":0.38200000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":4.3129999999999997,"5min":5.2110000000000003,"15min":17.535},"last":1.8460000000000001},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.85799999999999998,"15min":1.8089999999999999},"min":{"1min":0.43099999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.2050000000000001,"5min":4.2640000000000002,"15min":15.651999999999999},"last":0.64400000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.79500000000000004,"15min":2.028},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":2.1000000000000001,"5min":4.976,"15min":48.895000000000003},"last":1.8740000000000001},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.89900000000000002,"15min":2.048},"min":{"1min":0.376,"5min":0.376,"15min":0.35099999999999998},"max":{"1min":4.125,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.67600000000000005}]}]},{"osd":2,"up_from":0,"seq":425201762711,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599184,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1366424,"kb_used_data":538384,"kb_used_omap":3,"kb_used_meta":828028,"kb_avail":92378728,"statfs":{"total":95995035648,"available":94595817472,"internally_reserved":0,"allocated":551305216,"data_stored":549182750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":847901415},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":1.4870000000000001,"15min":4.6909999999999998},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":0.65000000000000002,"5min":64.504000000000005,"15min":137.34399999999999},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.41499999999999998,"5min":1.4870000000000001,"15min":4.6959999999999997},"min":{"1min":0.219,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":1.2849999999999999,"5min":64.448999999999998,"15min":137.73400000000001},"last":0.33000000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.51700000000000002,"15min":4.8719999999999999},"min":{"1min":0.25,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":1.3500000000000001,"5min":6.0250000000000004,"15min":162.22999999999999},"last":0.373},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.40600000000000003,"15min":4.9199999999999999},"min":{"1min":0.20899999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.77600000000000002,"5min":2.931,"15min":162.70699999999999},"last":0.39400000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56599999999999995,"15min":1.5800000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.20799999999999999},"max":{"1min":2.9049999999999998,"5min":7.141,"15min":14.254},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.52300000000000002,"15min":1.5860000000000001},"min":{"1min":0.314,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":0.80400000000000005,"5min":4.5049999999999999,"15min":15.523},"last":0.54300000000000004}]},{"osd":4,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.54600000000000004,"15min":1.71},"min":{"1min":0.34999999999999998,"5min":0.312,"15min":0.26200000000000001},"max":{"1min":0.874,"5min":6.8890000000000002,"15min":77.793000000000006},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.58699999999999997,"15min":1.8919999999999999},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.409,"5min":6.0940000000000003,"15min":76.646000000000001},"last":0.53400000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.82599999999999996,"15min":2.1419999999999999},"min":{"1min":0.314,"5min":0.29399999999999998,"15min":0.27500000000000002},"max":{"1min":2.895,"5min":6.0659999999999998,"15min":28.117999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.76700000000000002,"15min":2.2450000000000001},"min":{"1min":0.34899999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.0720000000000001,"5min":7.1559999999999997,"15min":28.34},"last":0.42499999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:45:30 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.512,"15min":1.879},"min":{"1min":0.33900000000000002,"5min":0.315,"15min":0.28599999999999998},"max":{"1min":2.9100000000000001,"5min":73.887,"15min":73.887},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.6559999999999999,"15min":1.9530000000000001},"min":{"1min":0.40500000000000003,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":1.1379999999999999,"5min":74.016999999999996,"15min":74.016999999999996},"last":0.46600000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.86299999999999999,"15min":1.7809999999999999},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":3.6480000000000001,"5min":7.0620000000000003,"15min":13.085000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.746,"5min":0.77200000000000002,"15min":1.633},"min":{"1min":0.38800000000000001,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.7120000000000002,"5min":4.1779999999999999,"15min":13.191000000000001},"last":0.57299999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019434,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3160268,"kb_used_data":2265988,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":90584884,"statfs":{"total":95995035648,"available":92758921216,"internally_reserved":0,"allocated":2320371712,"data_stored":2319431682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.623,"15min":3.9359999999999999},"min":{"1min":0.22600000000000001,"5min":0.22500000000000001,"15min":0.21199999999999999},"max":{"1min":0.51400000000000001,"5min":9.6690000000000005,"15min":155.315},"last":0.376},{"interface":"front","average":{"1min":0.371,"5min":0.72499999999999998,"15min":4.2910000000000004},"min":{"1min":0.253,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.499,"5min":6.1459999999999999,"15min":155.334},"last":0.33800000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.72399999999999998,"15min":5.2430000000000003},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":0.57799999999999996,"5min":5.944,"15min":190.61199999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.63400000000000001,"15min":5.9699999999999998},"min":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.25600000000000001},"max":{"1min":1.679,"5min":3.7469999999999999,"15min":253.273},"last":0.32400000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.74299999999999999,"15min":2.1150000000000002},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.1639999999999999,"5min":5.702,"15min":170.45400000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.748,"15min":1.889},"min":{"1min":0.38600000000000001,"5min":0.314,"15min":0.25800000000000001},"max":{"1min":2.121,"5min":5.6150000000000002,"15min":36.515999999999998},"last":0.39800000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.69699999999999995,"15min":2.125},"min":{"1min":0.29199999999999998,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":2.0379999999999998,"5min":5.5279999999999996,"15min":140.881},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.68500000000000005,"15min":2.2749999999999999},"min":{"1min":0.32300000000000001,"5min":0.312,"15min":0.311},"max":{"1min":6.2969999999999997,"5min":6.2969999999999997,"15min":140.911},"last":0.35899999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:45:39 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.9279999999999999,"15min":1.4870000000000001},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":2.9649999999999999,"5min":110.003,"15min":110.003},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.62,"5min":1.8779999999999999,"15min":1.3959999999999999},"min":{"1min":0.34999999999999998,"5min":0.29899999999999999,"15min":0.28899999999999998},"max":{"1min":2.9910000000000001,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.40799999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.623,"15min":0.91900000000000004},"min":{"1min":0.34599999999999997,"5min":0.28799999999999998,"15min":0.27200000000000002},"max":{"1min":0.63700000000000001,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.63400000000000001,"15min":1.0289999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28999999999999998},"max":{"1min":1.3340000000000001,"5min":4.0220000000000002,"15min":10.076000000000001},"last":0.42499999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.66800000000000004,"15min":1.139},"min":{"1min":0.33400000000000002,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":2.98,"5min":3.8580000000000001,"15min":38.034999999999997},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.91400000000000003,"15min":1.4350000000000001},"min":{"1min":0.40899999999999997,"5min":0.36699999999999999,"15min":0.308},"max":{"1min":3.113,"5min":12.092000000000001,"15min":37.723999999999997},"last":2.5990000000000002}]}]},{"osd":5,"up_from":379,"seq":1627792605341,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1512056,"kb_used_data":1151600,"kb_used_omap":13,"kb_used_meta":360434,"kb_avail":92233096,"statfs":{"total":95995035648,"available":94446690304,"internally_reserved":0,"allocated":1179238400,"data_stored":1178314274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":369084914},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,2],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":1.004,"15min":2.157},"min":{"1min":0.28699999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.1400000000000001,"5min":27.584,"15min":84.340999999999994},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.97699999999999998,"15min":2.0910000000000002},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":4.2300000000000004,"5min":27.669,"15min":84.356999999999999},"last":0.54100000000000004}]},{"osd":1,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":2.0430000000000001,"15min":5.6200000000000001},"min":{"1min":0.26600000000000001,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":4.3289999999999997,"5min":106.56999999999999,"15min":295.71300000000002},"last":4.2130000000000001},{"interface":"front","average":{"1min":0.495,"5min":2.0630000000000002,"15min":5.6260000000000003},"min":{"1min":0.27200000000000002,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":2.169,"5min":106.596,"15min":299.59399999999999},"last":0.35099999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.77000000000000002,"15min":1.5089999999999999},"min":{"1min":0.28799999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.2670000000000003,"5min":5.3470000000000004,"15min":14.199999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.89800000000000002,"15min":1.468},"min":{"1min":0.315,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":4.3719999999999999,"5min":5.782,"15min":14.430999999999999},"last":4.2910000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.71799999999999997,"15min":1.417},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.1880000000000002,"5min":4.0209999999999999,"15min":14.004},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.84399999999999997,"15min":1.5629999999999999},"min":{"1min":0.36599999999999999,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":4.2910000000000004,"5min":5.4400000000000004,"15min":13.991},"last":0.57599999999999996}]},{"osd":4,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.86599999999999999,"15min":1.3959999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.1929999999999996,"5min":5.9459999999999997,"15min":122.971},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.02,"15min":1.605},"min":{"1min":0.30399999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.3869999999999996,"5min":12.445,"15min":122.89100000000001},"last":4.2619999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.86899999999999999,"15min":0.99399999999999999},"min":{"1min":0.314,"5min":0.312,"15min":0.28299999999999997},"max":{"1min":2.2069999999999999,"5min":6.7750000000000004,"15min":12.545},"last":0.432},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.91200000000000003,"15min":1.2130000000000001},"min":{"1min":0.35199999999999998,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":4.3200000000000003,"5min":5.4160000000000004,"15min":14.193},"last":0.61599999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:45:32 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.0349999999999999,"15min":1.1890000000000001},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.3559999999999999,"5min":5.758,"15min":10.813000000000001},"last":4.3170000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.99399999999999999,"15min":1.0660000000000001},"min":{"1min":0.39400000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":2.226,"5min":5.2300000000000004,"15min":12.500999999999999},"last":0.89800000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2462519296,"data_stored":2462515224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":542048256,"data_stored":542048256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":544538624,"data_stored":544538624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2311393280,"data_stored":2311389208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1169293312,"data_stored":1169293312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1251217408,"data_stored":1251213336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":889061376,"data_stored":889061376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:46:26.914 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:46:26.921 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:26.974 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:27.040 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:27.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:27.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:27.145 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:27.174 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:27.219 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:27.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:27.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:27.339 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:46:27.377 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:27.392 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:46:27.419 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:27 smithi138 ceph-mon[192513]: from='client.47297 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:27.419 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:27 smithi138 ceph-mon[192513]: pgmap v1618: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 6118/129750 objects degraded (4.715%); 2641/129750 objects misplaced (2.035%); 845 KiB/s, 13 objects/s recovering 2023-10-01T09:46:27.560 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:27 smithi190 ceph-mon[160435]: from='client.47297 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:27.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:27 smithi190 ceph-mon[160435]: pgmap v1618: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 6118/129750 objects degraded (4.715%); 2641/129750 objects misplaced (2.035%); 845 KiB/s, 13 objects/s recovering 2023-10-01T09:46:27.645 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:46:27.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:27.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:46:27.730 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:27 smithi138 ceph-mon[185886]: from='client.47297 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:27.731 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:27 smithi138 ceph-mon[185886]: pgmap v1618: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 6118/129750 objects degraded (4.715%); 2641/129750 objects misplaced (2.035%); 845 KiB/s, 13 objects/s recovering 2023-10-01T09:46:27.750 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:27.782 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:27.782 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:27.832 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:27.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:27.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:27.911 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:27.937 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:27.964 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:28.014 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:28.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:28.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:28.122 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:28.160 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:28.209 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:46:28.228 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:46:28.246 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:28.248 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:46:28.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:28.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:28.268 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:46:28.285 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:46:28.305 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:46:28.331 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:46:28.351 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:46:28.352 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:28.354 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:46:28.357 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:46:28.357 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:46:28.357 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:46:28.357 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:46:28.357 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:46:28.357 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:46:28.358 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:46:28.358 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:46:28.403 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:28.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:28 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6118/129750 objects degraded (4.715%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:28.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2580852448' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:28.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:28 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6118/129750 objects degraded (4.715%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:28.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2580852448' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:28 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6118/129750 objects degraded (4.715%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2580852448' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:29.132 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:46:29.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:29.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:46:29.237 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:29.271 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:29.419 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:29 smithi138 ceph-mon[185886]: pgmap v1619: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 188 op/s; 6037/128061 objects degraded (4.714%); 2586/128061 objects misplaced (2.019%); 902 KiB/s, 14 objects/s recovering 2023-10-01T09:46:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:29 smithi190 ceph-mon[160435]: pgmap v1619: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 188 op/s; 6037/128061 objects degraded (4.714%); 2586/128061 objects misplaced (2.019%); 902 KiB/s, 14 objects/s recovering 2023-10-01T09:46:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:29 smithi138 ceph-mon[192513]: pgmap v1619: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 188 op/s; 6037/128061 objects degraded (4.714%); 2586/128061 objects misplaced (2.019%); 902 KiB/s, 14 objects/s recovering 2023-10-01T09:46:29.835 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:29.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:29.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:29.939 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:29.970 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:30.623 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:30.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:30.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:30.727 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:30.756 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:30.802 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:30.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:30.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:30.915 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:30.944 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:31.189 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:46:31.189 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:46:31.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:46:31.190 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:46:31.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.190 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58507 10.7 10:e26da5e8:::benchmark_data_smithi138_573643_object58506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:46:31.190 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.384942+0000", 2023-10-01T09:46:31.190 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 335.48712074399998, 2023-10-01T09:46:31.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1937820219999999, 2023-10-01T09:46:31.190 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.190 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.190 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:31.191 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:31.191 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:31.191 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58507 2023-10-01T09:46:31.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.191 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.384942+0000", 2023-10-01T09:46:31.191 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.384942+0000", 2023-10-01T09:46:31.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.384940+0000", 2023-10-01T09:46:31.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:46:31.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.384966+0000", 2023-10-01T09:46:31.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6027999999999999e-05 2023-10-01T09:46:31.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.384977+0000", 2023-10-01T09:46:31.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.132e-05 2023-10-01T09:46:31.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.384987+0000", 2023-10-01T09:46:31.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9210000000000004e-06 2023-10-01T09:46:31.194 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.194 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.194 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.194 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385047+0000", 2023-10-01T09:46:31.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0436000000000002e-05 2023-10-01T09:46:31.194 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.194 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.194 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:46:31.194 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385059+0000", 2023-10-01T09:46:31.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1185e-05 2023-10-01T09:46:31.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.561399+0000", 2023-10-01T09:46:31.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.17634086 2023-10-01T09:46:31.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.561597+0000", 2023-10-01T09:46:31.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000197326 2023-10-01T09:46:31.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:46:31.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.561770+0000", 2023-10-01T09:46:31.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000173506 2023-10-01T09:46:31.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.567454+0000", 2023-10-01T09:46:31.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0056839639999999997 2023-10-01T09:46:31.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.198 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:31.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.567469+0000", 2023-10-01T09:46:31.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4489e-05 2023-10-01T09:46:31.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.199 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.567521+0000", 2023-10-01T09:46:31.199 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2615999999999999e-05 2023-10-01T09:46:31.199 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.200 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.576645+0000", 2023-10-01T09:46:31.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0091240330000000001 2023-10-01T09:46:31.200 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.200 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.200 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.200 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.576711+0000", 2023-10-01T09:46:31.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5662999999999998e-05 2023-10-01T09:46:31.200 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.201 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.201 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.201 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.578724+0000", 2023-10-01T09:46:31.201 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0020126079999999999 2023-10-01T09:46:31.201 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.202 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.202 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.202 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.202 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.202 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58516 10.7 10:ead56eb9:::benchmark_data_smithi138_573643_object58515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:46:31.203 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385002+0000", 2023-10-01T09:46:31.203 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 335.48706031900002, 2023-10-01T09:46:31.203 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2021478640000001, 2023-10-01T09:46:31.203 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.203 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.203 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:31.204 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:31.204 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:31.204 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58516 2023-10-01T09:46:31.204 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.204 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385002+0000", 2023-10-01T09:46:31.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.205 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385002+0000", 2023-10-01T09:46:31.206 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.206 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.206 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.206 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.206 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385002+0000", 2023-10-01T09:46:31.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:46:31.207 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.207 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.207 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.207 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385030+0000", 2023-10-01T09:46:31.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8259000000000002e-05 2023-10-01T09:46:31.208 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.208 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.208 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.208 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385037+0000", 2023-10-01T09:46:31.208 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9589999999999998e-06 2023-10-01T09:46:31.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.209 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.209 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385042+0000", 2023-10-01T09:46:31.209 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1919999999999996e-06 2023-10-01T09:46:31.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.210 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.210 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385080+0000", 2023-10-01T09:46:31.210 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8325000000000003e-05 2023-10-01T09:46:31.210 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.210 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.210 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:46:31.210 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385089+0000", 2023-10-01T09:46:31.210 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4880000000000007e-06 2023-10-01T09:46:31.210 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.210 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.210 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.211 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.565074+0000", 2023-10-01T09:46:31.211 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1799852770000001 2023-10-01T09:46:31.211 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.211 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.211 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.211 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.565200+0000", 2023-10-01T09:46:31.211 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012553799999999999 2023-10-01T09:46:31.211 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.211 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.211 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:46:31.211 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.565406+0000", 2023-10-01T09:46:31.211 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020614099999999999 2023-10-01T09:46:31.212 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.212 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.212 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:31.212 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.571877+0000", 2023-10-01T09:46:31.212 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0064711639999999997 2023-10-01T09:46:31.212 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.212 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.212 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.213 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572124+0000", 2023-10-01T09:46:31.213 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00024668299999999998 2023-10-01T09:46:31.213 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.213 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.213 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.213 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572181+0000", 2023-10-01T09:46:31.213 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7689999999999998e-05 2023-10-01T09:46:31.213 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.213 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.213 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.213 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.585282+0000", 2023-10-01T09:46:31.214 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013100799 2023-10-01T09:46:31.214 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.214 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.214 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.585327+0000", 2023-10-01T09:46:31.214 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4759999999999998e-05 2023-10-01T09:46:31.214 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.214 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.214 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.214 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.587150+0000", 2023-10-01T09:46:31.214 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001823046 2023-10-01T09:46:31.214 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.215 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.215 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.215 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.215 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.215 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58518 10.7 10:e3c213cc:::benchmark_data_smithi138_573643_object58517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:46:31.215 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385051+0000", 2023-10-01T09:46:31.215 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 335.48701109299998, 2023-10-01T09:46:31.215 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.204210961, 2023-10-01T09:46:31.215 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.215 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.215 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:31.216 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:31.216 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:31.216 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58518 2023-10-01T09:46:31.216 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.216 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.216 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.216 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.216 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385051+0000", 2023-10-01T09:46:31.216 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.216 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.216 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.217 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.217 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385051+0000", 2023-10-01T09:46:31.217 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.217 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.217 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.217 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.217 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385051+0000", 2023-10-01T09:46:31.217 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:46:31.217 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.217 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.217 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.218 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385072+0000", 2023-10-01T09:46:31.218 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1389000000000001e-05 2023-10-01T09:46:31.218 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.218 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.218 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.218 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385078+0000", 2023-10-01T09:46:31.218 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0240000000000003e-06 2023-10-01T09:46:31.218 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.218 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.218 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.219 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385083+0000", 2023-10-01T09:46:31.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5140000000000003e-06 2023-10-01T09:46:31.219 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.219 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.219 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.219 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385111+0000", 2023-10-01T09:46:31.219 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8581999999999998e-05 2023-10-01T09:46:31.219 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.219 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.219 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:46:31.219 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385118+0000", 2023-10-01T09:46:31.220 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8340000000000001e-06 2023-10-01T09:46:31.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.220 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.220 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.567540+0000", 2023-10-01T09:46:31.220 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1824218449999999 2023-10-01T09:46:31.220 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.220 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.221 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.567733+0000", 2023-10-01T09:46:31.221 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000192898 2023-10-01T09:46:31.221 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.221 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.221 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:46:31.221 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.567867+0000", 2023-10-01T09:46:31.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013439499999999999 2023-10-01T09:46:31.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.222 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.222 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:31.222 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572194+0000", 2023-10-01T09:46:31.222 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043261869999999996 2023-10-01T09:46:31.222 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.223 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.223 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572238+0000", 2023-10-01T09:46:31.223 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4169999999999999e-05 2023-10-01T09:46:31.223 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.223 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.223 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.223 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572290+0000", 2023-10-01T09:46:31.224 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2352000000000003e-05 2023-10-01T09:46:31.224 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.224 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.224 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.224 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.587219+0000", 2023-10-01T09:46:31.224 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014928784 2023-10-01T09:46:31.225 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.225 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.225 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.225 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.587486+0000", 2023-10-01T09:46:31.225 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00026707599999999999 2023-10-01T09:46:31.225 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.226 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.226 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.226 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589262+0000", 2023-10-01T09:46:31.226 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0017762780000000001 2023-10-01T09:46:31.226 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.226 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.227 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.227 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.227 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.227 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58527 10.7 10:ea9f70c7:::benchmark_data_smithi138_573643_object58526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:46:31.227 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385089+0000", 2023-10-01T09:46:31.227 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 335.48697361799998, 2023-10-01T09:46:31.228 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.209989129, 2023-10-01T09:46:31.228 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.228 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.229 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:31.229 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:31.229 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:31.231 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58527 2023-10-01T09:46:31.231 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.231 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.231 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.232 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385089+0000", 2023-10-01T09:46:31.232 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.232 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.232 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.232 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385089+0000", 2023-10-01T09:46:31.233 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.233 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.233 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.233 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.233 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385088+0000", 2023-10-01T09:46:31.234 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:46:31.234 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.234 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.234 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.234 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385113+0000", 2023-10-01T09:46:31.234 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4671000000000001e-05 2023-10-01T09:46:31.235 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.235 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.235 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.235 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385119+0000", 2023-10-01T09:46:31.235 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8320000000000002e-06 2023-10-01T09:46:31.235 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.236 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.236 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.236 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385123+0000", 2023-10-01T09:46:31.236 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6179999999999997e-06 2023-10-01T09:46:31.236 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.236 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.236 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.237 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385165+0000", 2023-10-01T09:46:31.237 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1640000000000001e-05 2023-10-01T09:46:31.237 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.237 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.237 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:46:31.237 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385173+0000", 2023-10-01T09:46:31.238 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3189999999999995e-06 2023-10-01T09:46:31.238 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.238 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.238 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.238 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.569904+0000", 2023-10-01T09:46:31.238 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1847305889999999 2023-10-01T09:46:31.239 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.239 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.239 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.239 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.570077+0000", 2023-10-01T09:46:31.239 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017342599999999999 2023-10-01T09:46:31.239 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.240 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:46:31.240 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.570269+0000", 2023-10-01T09:46:31.240 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000191447 2023-10-01T09:46:31.240 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.240 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.241 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:31.241 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572301+0000", 2023-10-01T09:46:31.241 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002032467 2023-10-01T09:46:31.241 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.241 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.241 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.576561+0000", 2023-10-01T09:46:31.242 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0042592810000000002 2023-10-01T09:46:31.242 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.242 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.242 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.242 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.576591+0000", 2023-10-01T09:46:31.242 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9870999999999999e-05 2023-10-01T09:46:31.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.243 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.243 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.243 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.592534+0000", 2023-10-01T09:46:31.243 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015943704 2023-10-01T09:46:31.243 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.244 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.244 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.592603+0000", 2023-10-01T09:46:31.244 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9125e-05 2023-10-01T09:46:31.244 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.244 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.245 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.245 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.595078+0000", 2023-10-01T09:46:31.245 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024745240000000001 2023-10-01T09:46:31.245 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.245 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.245 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.246 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.246 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.246 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58532 10.7 10:ecf17a99:::benchmark_data_smithi138_573643_object58531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:46:31.246 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385131+0000", 2023-10-01T09:46:31.246 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 335.48693133099999, 2023-10-01T09:46:31.246 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2138979649999999, 2023-10-01T09:46:31.247 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.247 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.247 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:31.247 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:31.247 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:31.247 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58532 2023-10-01T09:46:31.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.248 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.248 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.248 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.248 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385131+0000", 2023-10-01T09:46:31.248 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.248 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.249 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.249 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385131+0000", 2023-10-01T09:46:31.249 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.249 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.249 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.250 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385131+0000", 2023-10-01T09:46:31.250 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:46:31.250 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.250 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.250 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.251 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385150+0000", 2023-10-01T09:46:31.251 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9346000000000001e-05 2023-10-01T09:46:31.251 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.251 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.251 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.251 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385155+0000", 2023-10-01T09:46:31.251 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3550000000000004e-06 2023-10-01T09:46:31.252 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.252 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.252 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.252 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385160+0000", 2023-10-01T09:46:31.252 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1500000000000001e-06 2023-10-01T09:46:31.253 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.253 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.253 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.253 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385194+0000", 2023-10-01T09:46:31.253 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4592999999999999e-05 2023-10-01T09:46:31.253 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.254 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.254 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:46:31.254 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385201+0000", 2023-10-01T09:46:31.254 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8809999999999998e-06 2023-10-01T09:46:31.254 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.254 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.255 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.255 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572324+0000", 2023-10-01T09:46:31.255 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.187123296 2023-10-01T09:46:31.255 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.255 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.255 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.256 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572429+0000", 2023-10-01T09:46:31.256 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010501 2023-10-01T09:46:31.256 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.256 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.256 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:46:31.256 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.572536+0000", 2023-10-01T09:46:31.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106325 2023-10-01T09:46:31.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.257 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.257 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:31.257 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.576600+0000", 2023-10-01T09:46:31.257 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040639450000000002 2023-10-01T09:46:31.257 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.258 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.258 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.258 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.578792+0000", 2023-10-01T09:46:31.258 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021927990000000001 2023-10-01T09:46:31.258 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.258 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.259 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.578829+0000", 2023-10-01T09:46:31.259 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6913999999999999e-05 2023-10-01T09:46:31.259 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.259 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.259 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.596616+0000", 2023-10-01T09:46:31.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017786626999999999 2023-10-01T09:46:31.260 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.260 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.260 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.260 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.596688+0000", 2023-10-01T09:46:31.260 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2444e-05 2023-10-01T09:46:31.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.261 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.261 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.599029+0000", 2023-10-01T09:46:31.261 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002340623 2023-10-01T09:46:31.261 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.261 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.261 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.261 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.261 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.261 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58541 10.7 10:ec507a14:::benchmark_data_smithi138_573643_object58540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:46:31.262 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385164+0000", 2023-10-01T09:46:31.262 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 335.48689856300001, 2023-10-01T09:46:31.262 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.216211062, 2023-10-01T09:46:31.262 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.262 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.262 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:31.262 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:31.262 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:31.262 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58541 2023-10-01T09:46:31.262 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.262 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385164+0000", 2023-10-01T09:46:31.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.263 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.263 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385164+0000", 2023-10-01T09:46:31.263 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.263 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.263 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385163+0000", 2023-10-01T09:46:31.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:46:31.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.264 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385182+0000", 2023-10-01T09:46:31.264 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8678999999999999e-05 2023-10-01T09:46:31.264 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.264 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.264 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385188+0000", 2023-10-01T09:46:31.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4550000000000003e-06 2023-10-01T09:46:31.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385192+0000", 2023-10-01T09:46:31.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4549999999999997e-06 2023-10-01T09:46:31.265 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.265 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.265 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.265 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385238+0000", 2023-10-01T09:46:31.265 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5664000000000002e-05 2023-10-01T09:46:31.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:46:31.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385245+0000", 2023-10-01T09:46:31.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1860000000000003e-06 2023-10-01T09:46:31.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.266 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.266 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.266 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.574514+0000", 2023-10-01T09:46:31.266 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.189269406 2023-10-01T09:46:31.266 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.574676+0000", 2023-10-01T09:46:31.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000161853 2023-10-01T09:46:31.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.267 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:46:31.267 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.574876+0000", 2023-10-01T09:46:31.267 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019950799999999999 2023-10-01T09:46:31.267 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.267 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.578875+0000", 2023-10-01T09:46:31.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003999598 2023-10-01T09:46:31.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:31.268 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.578887+0000", 2023-10-01T09:46:31.268 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1678000000000001e-05 2023-10-01T09:46:31.268 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.268 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.268 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.578923+0000", 2023-10-01T09:46:31.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6443999999999999e-05 2023-10-01T09:46:31.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.599150+0000", 2023-10-01T09:46:31.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.020226289000000001 2023-10-01T09:46:31.269 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.269 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.269 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.269 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.599226+0000", 2023-10-01T09:46:31.269 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6650000000000006e-05 2023-10-01T09:46:31.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.270 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.270 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.601375+0000", 2023-10-01T09:46:31.270 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021485469999999998 2023-10-01T09:46:31.270 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.270 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.270 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.270 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.270 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.270 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58579 10.7 10:e45d4eec:::benchmark_data_smithi138_573643_object58578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:46:31.271 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385196+0000", 2023-10-01T09:46:31.271 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 335.48686619, 2023-10-01T09:46:31.271 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.218245164, 2023-10-01T09:46:31.271 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.271 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.271 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:31.271 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:31.271 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:31.271 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58579 2023-10-01T09:46:31.271 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.271 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.272 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.272 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385196+0000", 2023-10-01T09:46:31.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.272 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.272 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385196+0000", 2023-10-01T09:46:31.272 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.272 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.272 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.272 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385196+0000", 2023-10-01T09:46:31.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:46:31.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385233+0000", 2023-10-01T09:46:31.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6854999999999999e-05 2023-10-01T09:46:31.273 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.273 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.273 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.273 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385239+0000", 2023-10-01T09:46:31.273 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9950000000000002e-06 2023-10-01T09:46:31.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385243+0000", 2023-10-01T09:46:31.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4959999999999998e-06 2023-10-01T09:46:31.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.274 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.274 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.274 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385266+0000", 2023-10-01T09:46:31.274 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2819999999999998e-05 2023-10-01T09:46:31.274 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:46:31.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385273+0000", 2023-10-01T09:46:31.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1550000000000001e-06 2023-10-01T09:46:31.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.275 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.578945+0000", 2023-10-01T09:46:31.275 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.193672235 2023-10-01T09:46:31.275 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.275 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.275 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.579103+0000", 2023-10-01T09:46:31.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015765000000000001 2023-10-01T09:46:31.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:46:31.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.579233+0000", 2023-10-01T09:46:31.276 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013000800000000001 2023-10-01T09:46:31.276 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.276 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.276 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:31.276 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589288+0000", 2023-10-01T09:46:31.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010055364000000001 2023-10-01T09:46:31.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589351+0000", 2023-10-01T09:46:31.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2792000000000005e-05 2023-10-01T09:46:31.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.277 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.277 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.277 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589444+0000", 2023-10-01T09:46:31.277 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3288000000000005e-05 2023-10-01T09:46:31.277 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.601431+0000", 2023-10-01T09:46:31.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011986939 2023-10-01T09:46:31.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.278 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.278 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.601491+0000", 2023-10-01T09:46:31.278 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9592000000000001e-05 2023-10-01T09:46:31.278 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.278 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.279 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.279 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.603441+0000", 2023-10-01T09:46:31.279 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001950301 2023-10-01T09:46:31.279 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.279 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.279 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.279 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.279 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.279 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58611 10.7 10:e059d00b:::benchmark_data_smithi138_573643_object58610:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:46:31.279 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385248+0000", 2023-10-01T09:46:31.279 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 335.48681468299998, 2023-10-01T09:46:31.280 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.219992653, 2023-10-01T09:46:31.280 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.280 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.280 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:31.280 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:31.280 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:31.280 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58611 2023-10-01T09:46:31.280 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.280 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.280 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.280 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385248+0000", 2023-10-01T09:46:31.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385248+0000", 2023-10-01T09:46:31.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.281 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.281 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.281 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.281 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385247+0000", 2023-10-01T09:46:31.281 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:46:31.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.282 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385268+0000", 2023-10-01T09:46:31.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1151000000000001e-05 2023-10-01T09:46:31.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.282 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.282 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385274+0000", 2023-10-01T09:46:31.282 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4670000000000003e-06 2023-10-01T09:46:31.282 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.282 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385278+0000", 2023-10-01T09:46:31.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.34e-06 2023-10-01T09:46:31.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385308+0000", 2023-10-01T09:46:31.283 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9634000000000001e-05 2023-10-01T09:46:31.283 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.283 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.283 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:46:31.283 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385315+0000", 2023-10-01T09:46:31.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4250000000000001e-06 2023-10-01T09:46:31.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.580981+0000", 2023-10-01T09:46:31.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.195665966 2023-10-01T09:46:31.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.284 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.284 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.284 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.581114+0000", 2023-10-01T09:46:31.284 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013232900000000001 2023-10-01T09:46:31.284 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:46:31.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.581238+0000", 2023-10-01T09:46:31.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000123955 2023-10-01T09:46:31.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:31.285 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589297+0000", 2023-10-01T09:46:31.285 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0080591729999999993 2023-10-01T09:46:31.285 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.285 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.285 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589402+0000", 2023-10-01T09:46:31.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010554 2023-10-01T09:46:31.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589470+0000", 2023-10-01T09:46:31.286 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7218000000000006e-05 2023-10-01T09:46:31.286 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.286 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.286 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.286 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.603490+0000", 2023-10-01T09:46:31.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014020566 2023-10-01T09:46:31.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.603542+0000", 2023-10-01T09:46:31.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2351000000000001e-05 2023-10-01T09:46:31.287 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.287 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.287 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.287 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.605240+0000", 2023-10-01T09:46:31.287 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0016979 2023-10-01T09:46:31.288 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.288 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.288 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.288 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.288 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.288 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58625 10.7 10:eb5c5627:::benchmark_data_smithi138_573643_object58624:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:46:31.288 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385285+0000", 2023-10-01T09:46:31.288 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 335.48677704699998, 2023-10-01T09:46:31.288 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.221588224, 2023-10-01T09:46:31.288 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.288 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.288 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:31.289 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:31.289 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:31.289 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58625 2023-10-01T09:46:31.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.289 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.289 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385285+0000", 2023-10-01T09:46:31.289 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.289 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.289 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.289 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385285+0000", 2023-10-01T09:46:31.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385285+0000", 2023-10-01T09:46:31.290 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:46:31.290 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.290 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.290 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.290 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385327+0000", 2023-10-01T09:46:31.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1712999999999999e-05 2023-10-01T09:46:31.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385343+0000", 2023-10-01T09:46:31.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6050999999999999e-05 2023-10-01T09:46:31.291 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.291 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.291 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.291 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385350+0000", 2023-10-01T09:46:31.291 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3950000000000003e-06 2023-10-01T09:46:31.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.292 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385383+0000", 2023-10-01T09:46:31.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2758e-05 2023-10-01T09:46:31.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.292 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.292 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:46:31.292 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385391+0000", 2023-10-01T09:46:31.292 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6960000000000005e-06 2023-10-01T09:46:31.292 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.583192+0000", 2023-10-01T09:46:31.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.197801355 2023-10-01T09:46:31.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.293 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.583333+0000", 2023-10-01T09:46:31.293 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000141426 2023-10-01T09:46:31.293 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.293 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.293 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T09:46:31.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.583441+0000", 2023-10-01T09:46:31.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000107808 2023-10-01T09:46:31.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:31.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589360+0000", 2023-10-01T09:46:31.294 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059190650000000003 2023-10-01T09:46:31.294 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.294 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.294 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.294 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589489+0000", 2023-10-01T09:46:31.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000128369 2023-10-01T09:46:31.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.295 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589511+0000", 2023-10-01T09:46:31.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2337999999999999e-05 2023-10-01T09:46:31.295 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.295 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.295 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.295 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.605294+0000", 2023-10-01T09:46:31.295 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015783177999999998 2023-10-01T09:46:31.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.296 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.296 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.605345+0000", 2023-10-01T09:46:31.296 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0424999999999999e-05 2023-10-01T09:46:31.296 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.296 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.297 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.297 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.606874+0000", 2023-10-01T09:46:31.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015289909999999999 2023-10-01T09:46:31.298 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.298 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.298 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.298 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.298 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.298 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:90813 10.4 10:200bcc97:::benchmark_data_smithi138_573643_object7591:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:46:31.298 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:42:42.508517+0000", 2023-10-01T09:46:31.298 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 228.363545324, 2023-10-01T09:46:31.298 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11063846099999999, 2023-10-01T09:46:31.298 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.298 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.299 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:31.299 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:31.299 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:31.299 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 90813 2023-10-01T09:46:31.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.299 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.299 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.299 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508517+0000", 2023-10-01T09:46:31.299 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.299 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.299 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508517+0000", 2023-10-01T09:46:31.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508516+0000", 2023-10-01T09:46:31.300 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:46:31.300 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.300 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.300 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.300 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508521+0000", 2023-10-01T09:46:31.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9660000000000004e-06 2023-10-01T09:46:31.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.301 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.301 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.301 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508528+0000", 2023-10-01T09:46:31.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0400000000000004e-06 2023-10-01T09:46:31.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.301 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.301 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.301 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508533+0000", 2023-10-01T09:46:31.301 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6110000000000004e-06 2023-10-01T09:46:31.301 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.302 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.302 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508566+0000", 2023-10-01T09:46:31.302 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3018000000000001e-05 2023-10-01T09:46:31.302 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.302 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.302 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508642+0000", 2023-10-01T09:46:31.302 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.6259000000000003e-05 2023-10-01T09:46:31.302 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.302 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.302 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:46:31.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.508706+0000", 2023-10-01T09:46:31.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3503000000000005e-05 2023-10-01T09:46:31.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.303 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.303 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:31.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511112+0000", 2023-10-01T09:46:31.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00240596 2023-10-01T09:46:31.303 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.303 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.303 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.303 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511312+0000", 2023-10-01T09:46:31.303 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020054400000000001 2023-10-01T09:46:31.304 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.304 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.304 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.304 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511439+0000", 2023-10-01T09:46:31.304 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000127346 2023-10-01T09:46:31.304 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.304 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.304 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.304 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511468+0000", 2023-10-01T09:46:31.304 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8252e-05 2023-10-01T09:46:31.304 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.304 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.305 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.305 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.619155+0000", 2023-10-01T09:46:31.305 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.10768778599999999 2023-10-01T09:46:31.305 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.305 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.305 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.305 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.305 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.305 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.36456.0:90813 10.4 e516/475 ondisk, result = 0)", 2023-10-01T09:46:31.305 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:42:42.511386+0000", 2023-10-01T09:46:31.305 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 228.36067616599999, 2023-10-01T09:46:31.305 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.107776915, 2023-10-01T09:46:31.306 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.306 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:46:31.306 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.306 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.306 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.306 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511386+0000", 2023-10-01T09:46:31.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.306 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.306 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.306 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.306 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511386+0000", 2023-10-01T09:46:31.306 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.307 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511386+0000", 2023-10-01T09:46:31.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:46:31.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.307 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.307 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511393+0000", 2023-10-01T09:46:31.307 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9700000000000002e-06 2023-10-01T09:46:31.307 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.307 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511401+0000", 2023-10-01T09:46:31.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1969999999999996e-06 2023-10-01T09:46:31.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511405+0000", 2023-10-01T09:46:31.308 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2409999999999997e-06 2023-10-01T09:46:31.308 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.308 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.308 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.308 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511431+0000", 2023-10-01T09:46:31.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.588e-05 2023-10-01T09:46:31.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511436+0000", 2023-10-01T09:46:31.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1719999999999998e-06 2023-10-01T09:46:31.309 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.309 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.309 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.309 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.619163+0000", 2023-10-01T09:46:31.309 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.107727068 2023-10-01T09:46:31.309 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.310 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.310 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.310 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.310 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.310 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:90814 10.5 10:a1f50e2f:::benchmark_data_smithi138_573643_object7592:head [delete] snapc 0=[] ondisk+write+known_if_redirected e516)", 2023-10-01T09:46:31.310 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:42:42.511584+0000", 2023-10-01T09:46:31.310 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 228.36047813499999, 2023-10-01T09:46:31.310 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.111412285, 2023-10-01T09:46:31.310 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.310 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.310 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:31.311 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:31.311 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:31.311 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 90814 2023-10-01T09:46:31.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.311 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.311 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.311 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511584+0000", 2023-10-01T09:46:31.311 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.311 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.311 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511584+0000", 2023-10-01T09:46:31.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511583+0000", 2023-10-01T09:46:31.312 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:46:31.312 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.312 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.312 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.312 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.511588+0000", 2023-10-01T09:46:31.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8770000000000001e-06 2023-10-01T09:46:31.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.619145+0000", 2023-10-01T09:46:31.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.107557183 2023-10-01T09:46:31.313 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.313 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.313 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.313 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.619157+0000", 2023-10-01T09:46:31.313 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2109e-05 2023-10-01T09:46:31.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.619213+0000", 2023-10-01T09:46:31.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5448000000000002e-05 2023-10-01T09:46:31.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.314 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.314 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.619352+0000", 2023-10-01T09:46:31.314 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013952999999999999 2023-10-01T09:46:31.314 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.314 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:46:31.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.619468+0000", 2023-10-01T09:46:31.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011534199999999999 2023-10-01T09:46:31.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:31.315 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.622489+0000", 2023-10-01T09:46:31.315 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030212780000000001 2023-10-01T09:46:31.315 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.315 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.315 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.316 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.622708+0000", 2023-10-01T09:46:31.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021842899999999999 2023-10-01T09:46:31.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.316 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.316 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.622889+0000", 2023-10-01T09:46:31.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018121500000000001 2023-10-01T09:46:31.316 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.316 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.316 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.316 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.622948+0000", 2023-10-01T09:46:31.316 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9582e-05 2023-10-01T09:46:31.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.317 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.317 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:42:42.622996+0000", 2023-10-01T09:46:31.317 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8189000000000002e-05 2023-10-01T09:46:31.317 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.317 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.317 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.317 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.317 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.317 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:103952 10.8 e529/475 10:1c93a04e:::benchmark_data_smithi138_573643_object20730:head v 529'6554, mlcod=529'6552)", 2023-10-01T09:46:31.318 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:44:27.616960+0000", 2023-10-01T09:46:31.318 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 123.255102346, 2023-10-01T09:46:31.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11918105399999999, 2023-10-01T09:46:31.318 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.318 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.318 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.318 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.318 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.318 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.616960+0000", 2023-10-01T09:46:31.318 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.318 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.616960+0000", 2023-10-01T09:46:31.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.319 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.319 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.616953+0000", 2023-10-01T09:46:31.319 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999928 2023-10-01T09:46:31.319 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.319 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.616973+0000", 2023-10-01T09:46:31.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0029000000000001e-05 2023-10-01T09:46:31.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.616995+0000", 2023-10-01T09:46:31.320 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1849999999999999e-05 2023-10-01T09:46:31.320 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.320 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.320 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.320 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617008+0000", 2023-10-01T09:46:31.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3519999999999999e-05 2023-10-01T09:46:31.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617091+0000", 2023-10-01T09:46:31.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2974999999999998e-05 2023-10-01T09:46:31.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.321 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.321 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.321 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617127+0000", 2023-10-01T09:46:31.321 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5456000000000001e-05 2023-10-01T09:46:31.321 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.736078+0000", 2023-10-01T09:46:31.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11895101399999999 2023-10-01T09:46:31.322 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.322 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.322 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.322 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.736141+0000", 2023-10-01T09:46:31.322 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3173e-05 2023-10-01T09:46:31.322 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.322 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.322 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.323 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.323 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:103953 10.2 e529/475 10:414c62ad:::benchmark_data_smithi138_573643_object20731:head v 529'6517, mlcod=529'6515)", 2023-10-01T09:46:31.323 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:44:27.617965+0000", 2023-10-01T09:46:31.323 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 123.25409762300001, 2023-10-01T09:46:31.323 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.121350888, 2023-10-01T09:46:31.323 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.323 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.323 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.323 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.323 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.324 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617965+0000", 2023-10-01T09:46:31.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.324 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.324 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617965+0000", 2023-10-01T09:46:31.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.324 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.324 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.324 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.324 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617964+0000", 2023-10-01T09:46:31.324 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:46:31.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617970+0000", 2023-10-01T09:46:31.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.494e-06 2023-10-01T09:46:31.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.325 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.325 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617980+0000", 2023-10-01T09:46:31.325 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8959999999999998e-06 2023-10-01T09:46:31.325 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.325 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617987+0000", 2023-10-01T09:46:31.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.263e-06 2023-10-01T09:46:31.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.326 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.618028+0000", 2023-10-01T09:46:31.326 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1325999999999997e-05 2023-10-01T09:46:31.326 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.326 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.326 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.618042+0000", 2023-10-01T09:46:31.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3721e-05 2023-10-01T09:46:31.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.739290+0000", 2023-10-01T09:46:31.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.121248288 2023-10-01T09:46:31.327 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.327 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.327 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.327 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.739316+0000", 2023-10-01T09:46:31.327 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.5629000000000001e-05 2023-10-01T09:46:31.328 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.328 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.328 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.328 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.328 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.328 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:103956 10.6 e529/475 10:6e8702de:::benchmark_data_smithi138_573643_object20734:head v 529'6521, mlcod=529'6519)", 2023-10-01T09:46:31.328 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:44:27.630692+0000", 2023-10-01T09:46:31.328 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 123.24137054400001, 2023-10-01T09:46:31.328 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.108410692, 2023-10-01T09:46:31.328 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.328 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.328 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.329 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.630692+0000", 2023-10-01T09:46:31.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.329 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.329 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.630692+0000", 2023-10-01T09:46:31.329 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.329 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.329 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.630685+0000", 2023-10-01T09:46:31.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999928 2023-10-01T09:46:31.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.330 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.630706+0000", 2023-10-01T09:46:31.330 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1769000000000001e-05 2023-10-01T09:46:31.330 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.330 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.330 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.736062+0000", 2023-10-01T09:46:31.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.105355559 2023-10-01T09:46:31.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.736081+0000", 2023-10-01T09:46:31.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8751999999999999e-05 2023-10-01T09:46:31.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.331 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.331 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.331 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.736124+0000", 2023-10-01T09:46:31.331 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.295e-05 2023-10-01T09:46:31.331 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.736150+0000", 2023-10-01T09:46:31.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6084999999999998e-05 2023-10-01T09:46:31.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.332 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.739067+0000", 2023-10-01T09:46:31.332 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002917546 2023-10-01T09:46:31.332 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.332 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.332 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.333 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.739102+0000", 2023-10-01T09:46:31.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5247000000000003e-05 2023-10-01T09:46:31.333 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.333 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.333 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.333 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.333 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.333 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:113499 10.a 10:5afdbd95:::benchmark_data_smithi138_573643_object30277:head [delete] snapc 0=[] ondisk+write+known_if_redirected e537)", 2023-10-01T09:46:31.333 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:45:28.360369+0000", 2023-10-01T09:46:31.333 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 62.511693553000001, 2023-10-01T09:46:31.333 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11252944600000001, 2023-10-01T09:46:31.334 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.334 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.334 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:31.334 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:31.334 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:31.334 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 113499 2023-10-01T09:46:31.334 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.334 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.334 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.334 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.334 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.360369+0000", 2023-10-01T09:46:31.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.360369+0000", 2023-10-01T09:46:31.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.335 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.335 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.335 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.360367+0000", 2023-10-01T09:46:31.335 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:31.335 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.360374+0000", 2023-10-01T09:46:31.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0060000000000003e-06 2023-10-01T09:46:31.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.336 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.336 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.360386+0000", 2023-10-01T09:46:31.336 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1569000000000001e-05 2023-10-01T09:46:31.336 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.336 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.360393+0000", 2023-10-01T09:46:31.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.1500000000000002e-06 2023-10-01T09:46:31.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.337 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.360429+0000", 2023-10-01T09:46:31.337 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5967999999999999e-05 2023-10-01T09:46:31.337 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.337 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.337 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.360509+0000", 2023-10-01T09:46:31.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0037999999999995e-05 2023-10-01T09:46:31.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 5,6", 2023-10-01T09:46:31.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.360580+0000", 2023-10-01T09:46:31.338 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0512000000000005e-05 2023-10-01T09:46:31.338 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.338 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.338 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:31.338 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.363277+0000", 2023-10-01T09:46:31.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002697746 2023-10-01T09:46:31.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472778+0000", 2023-10-01T09:46:31.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.10950048800000001 2023-10-01T09:46:31.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.339 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.339 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.339 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472826+0000", 2023-10-01T09:46:31.339 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8488000000000001e-05 2023-10-01T09:46:31.339 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472863+0000", 2023-10-01T09:46:31.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6300000000000001e-05 2023-10-01T09:46:31.340 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.340 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.340 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.340 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472898+0000", 2023-10-01T09:46:31.340 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5602000000000002e-05 2023-10-01T09:46:31.340 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.340 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.340 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.341 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.341 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.341 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:113500 10.d 10:b26c9c6d:::benchmark_data_smithi138_573643_object30278:head [delete] snapc 0=[] ondisk+write+known_if_redirected e537)", 2023-10-01T09:46:31.341 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:45:28.362695+0000", 2023-10-01T09:46:31.341 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 62.509367050999998, 2023-10-01T09:46:31.341 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.110190366, 2023-10-01T09:46:31.341 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.341 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.341 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:31.341 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:31.341 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:31.342 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 113500 2023-10-01T09:46:31.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.342 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.362695+0000", 2023-10-01T09:46:31.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.342 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.342 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.342 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.342 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.362695+0000", 2023-10-01T09:46:31.342 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.362694+0000", 2023-10-01T09:46:31.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:46:31.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.343 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.343 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.343 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.362700+0000", 2023-10-01T09:46:31.343 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3600000000000004e-06 2023-10-01T09:46:31.343 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.362711+0000", 2023-10-01T09:46:31.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1141000000000001e-05 2023-10-01T09:46:31.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.344 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.362718+0000", 2023-10-01T09:46:31.344 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2239999999999998e-06 2023-10-01T09:46:31.344 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.344 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.344 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.362757+0000", 2023-10-01T09:46:31.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9351000000000003e-05 2023-10-01T09:46:31.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.362835+0000", 2023-10-01T09:46:31.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7829000000000003e-05 2023-10-01T09:46:31.345 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.345 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.345 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:46:31.345 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.363318+0000", 2023-10-01T09:46:31.345 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00048258399999999999 2023-10-01T09:46:31.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:31.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472746+0000", 2023-10-01T09:46:31.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.109428203 2023-10-01T09:46:31.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.346 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.346 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472801+0000", 2023-10-01T09:46:31.346 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4518000000000002e-05 2023-10-01T09:46:31.346 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.346 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:31.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472825+0000", 2023-10-01T09:46:31.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4569000000000001e-05 2023-10-01T09:46:31.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.347 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472857+0000", 2023-10-01T09:46:31.347 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2258999999999997e-05 2023-10-01T09:46:31.347 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.347 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.347 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.348 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472886+0000", 2023-10-01T09:46:31.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8229999999999999e-05 2023-10-01T09:46:31.348 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.348 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.348 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.348 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.348 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.348 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:113498 10.c e537/475 10:3b43a86d:::benchmark_data_smithi138_573643_object30276:head v 537'6965, mlcod=537'6962)", 2023-10-01T09:46:31.348 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:45:28.364238+0000", 2023-10-01T09:46:31.348 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 62.507824192000001, 2023-10-01T09:46:31.348 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.114613566, 2023-10-01T09:46:31.348 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.349 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:31.349 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.364238+0000", 2023-10-01T09:46:31.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.349 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.349 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.349 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.349 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.364238+0000", 2023-10-01T09:46:31.349 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.364238+0000", 2023-10-01T09:46:31.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:46:31.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.350 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.350 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.350 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.364246+0000", 2023-10-01T09:46:31.350 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3990000000000004e-06 2023-10-01T09:46:31.350 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.364256+0000", 2023-10-01T09:46:31.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9760000000000007e-06 2023-10-01T09:46:31.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.351 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.364263+0000", 2023-10-01T09:46:31.351 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9029999999999998e-06 2023-10-01T09:46:31.351 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.351 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.351 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.364296+0000", 2023-10-01T09:46:31.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3674999999999999e-05 2023-10-01T09:46:31.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.364309+0000", 2023-10-01T09:46:31.352 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2074e-05 2023-10-01T09:46:31.352 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.352 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.352 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:31.352 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.478817+0000", 2023-10-01T09:46:31.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.114508627 2023-10-01T09:46:31.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.353 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.353 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.478852+0000", 2023-10-01T09:46:31.353 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.455e-05 2023-10-01T09:46:31.353 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.353 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.353 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.353 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.353 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.353 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.36456.0:113499 10.a e537/475 ondisk, result = 0)", 2023-10-01T09:46:31.354 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:45:28.365142+0000", 2023-10-01T09:46:31.354 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 62.506920340000001, 2023-10-01T09:46:31.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.107648177, 2023-10-01T09:46:31.354 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.354 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:46:31.354 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.354 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.354 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.354 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.365142+0000", 2023-10-01T09:46:31.354 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.354 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.365142+0000", 2023-10-01T09:46:31.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.355 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.355 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.365141+0000", 2023-10-01T09:46:31.355 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:46:31.355 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.355 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.365145+0000", 2023-10-01T09:46:31.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8990000000000003e-06 2023-10-01T09:46:31.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472697+0000", 2023-10-01T09:46:31.356 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.10755118 2023-10-01T09:46:31.356 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.356 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.356 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.356 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472708+0000", 2023-10-01T09:46:31.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1e-05 2023-10-01T09:46:31.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472760+0000", 2023-10-01T09:46:31.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2094e-05 2023-10-01T09:46:31.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.357 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.357 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.357 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472770+0000", 2023-10-01T09:46:31.357 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0529e-05 2023-10-01T09:46:31.357 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.358 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.358 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472790+0000", 2023-10-01T09:46:31.358 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0044999999999999e-05 2023-10-01T09:46:31.358 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.358 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.358 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.358 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.358 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.358 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop_reply(client.36456.0:113500 10.d e537/475 ondisk, result = 0)", 2023-10-01T09:46:31.359 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:45:28.368260+0000", 2023-10-01T09:46:31.359 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 62.503802290000003, 2023-10-01T09:46:31.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.104543845, 2023-10-01T09:46:31.359 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:31.359 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:46:31.359 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:31.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.359 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:31.359 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.368260+0000", 2023-10-01T09:46:31.359 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.359 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.359 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:31.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.368260+0000", 2023-10-01T09:46:31.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:31.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:31.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.368259+0000", 2023-10-01T09:46:31.360 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:46:31.360 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.360 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.360 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:31.360 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.368275+0000", 2023-10-01T09:46:31.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5928999999999999e-05 2023-10-01T09:46:31.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:31.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472697+0000", 2023-10-01T09:46:31.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.104421457 2023-10-01T09:46:31.361 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.361 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.361 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:31.361 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472707+0000", 2023-10-01T09:46:31.361 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0206e-05 2023-10-01T09:46:31.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:31.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472791+0000", 2023-10-01T09:46:31.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3976999999999999e-05 2023-10-01T09:46:31.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.362 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.362 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:31.362 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472797+0000", 2023-10-01T09:46:31.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0859999999999998e-06 2023-10-01T09:46:31.362 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:31.363 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:31.363 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:31.363 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:28.472804+0000", 2023-10-01T09:46:31.363 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7780000000000002e-06 2023-10-01T09:46:31.363 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.363 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.363 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.363 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:31.363 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:31.363 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:46:31.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:31.364 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:31.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:31.465 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:31.493 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:31.539 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:31.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:31.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:31.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:31 smithi138 ceph-mon[192513]: pgmap v1620: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 192 op/s; 6037/127653 objects degraded (4.729%); 2586/127653 objects misplaced (2.026%); 902 KiB/s, 14 objects/s recovering 2023-10-01T09:46:31.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:31 smithi138 ceph-mon[185886]: pgmap v1620: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 192 op/s; 6037/127653 objects degraded (4.729%); 2586/127653 objects misplaced (2.026%); 902 KiB/s, 14 objects/s recovering 2023-10-01T09:46:31.643 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:31.671 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:31.732 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:31.736 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:31 smithi190 ceph-mon[160435]: pgmap v1620: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 192 op/s; 6037/127653 objects degraded (4.729%); 2586/127653 objects misplaced (2.026%); 902 KiB/s, 14 objects/s recovering 2023-10-01T09:46:31.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:31.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:31.838 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:31.866 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:31.922 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:46:31.933 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:31.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:31.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:32.035 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:46:32.036 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:32.068 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:32.345 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:32.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:32.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:32.454 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:32.487 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:32.549 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:32.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:32.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:32.580 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:32.637 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:32.659 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:46:32.695 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:32.751 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:46:32.751 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 6037/125949 objects degraded (4.793%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2213,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":545,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":41983,"data_bytes":2754287563,"bytes_used":14139867136,"bytes_avail":657825382400,"bytes_total":671965249536,"degraded_objects":6037,"degraded_total":125949,"degraded_ratio":0.047932099500591512,"misplaced_objects":2586,"misplaced_total":125949,"misplaced_ratio":0.020532120143867755,"recovering_objects_per_sec":14,"recovering_bytes_per_sec":923794,"recovering_keys_per_sec":0,"num_objects_recovered":141,"num_bytes_recovered":9240576,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":198},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":34,"modified":"2023-10-01T09:43:40.311181+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:46:33.118 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:46:33.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:33.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:46:33.224 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:33.258 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:33.310 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:33.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:33.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:33.416 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:33.434 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:46:33.447 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:33.510 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:33.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:33.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:33.518 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:33 smithi190 ceph-mon[160435]: pgmap v1621: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 198 op/s; 6037/125949 objects degraded (4.793%); 2586/125949 objects misplaced (2.053%); 902 KiB/s, 14 objects/s recovering 2023-10-01T09:46:33.518 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:33 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4265324917' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:46:33.518 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:33 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4192832998' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:46:33.518 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:33 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 6037/125949 objects degraded (4.793%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:33.618 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:33.663 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:33.737 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:33.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:33.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:33.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:33 smithi138 ceph-mon[185886]: pgmap v1621: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 198 op/s; 6037/125949 objects degraded (4.793%); 2586/125949 objects misplaced (2.053%); 902 KiB/s, 14 objects/s recovering 2023-10-01T09:46:33.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:33 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4265324917' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:46:33.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:33 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4192832998' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:46:33.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:33 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 6037/125949 objects degraded (4.793%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:33.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:33 smithi138 ceph-mon[192513]: pgmap v1621: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 198 op/s; 6037/125949 objects degraded (4.793%); 2586/125949 objects misplaced (2.053%); 902 KiB/s, 14 objects/s recovering 2023-10-01T09:46:33.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:33 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4265324917' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:46:33.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:33 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4192832998' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:46:33.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:33 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 6037/125949 objects degraded (4.793%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:33.850 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:46:33.879 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:34.579 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:46:34.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:34.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:46:34.684 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:34.715 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:34 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:34.715 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:34 smithi190 ceph-mon[160435]: failed to encode map e546 with expected crc 2023-10-01T09:46:34.715 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:34 smithi190 ceph-mon[160435]: failed to encode map e546 with expected crc 2023-10-01T09:46:34.715 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:34 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4192832998' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:46:34.715 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:34 smithi190 ceph-mon[160435]: osdmap e546: 8 total, 8 up, 7 in 2023-10-01T09:46:34.716 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:34.787 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:34.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:34.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:34.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:34 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:34.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:34 smithi138 ceph-mon[185886]: failed to encode map e546 with expected crc 2023-10-01T09:46:34.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:34 smithi138 ceph-mon[185886]: failed to encode map e546 with expected crc 2023-10-01T09:46:34.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:34 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4192832998' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:46:34.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:34 smithi138 ceph-mon[185886]: osdmap e546: 8 total, 8 up, 7 in 2023-10-01T09:46:34.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:34 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:34 smithi138 ceph-mon[192513]: failed to encode map e546 with expected crc 2023-10-01T09:46:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:34 smithi138 ceph-mon[192513]: failed to encode map e546 with expected crc 2023-10-01T09:46:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:34 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4192832998' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:46:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:34 smithi138 ceph-mon[192513]: osdmap e546: 8 total, 8 up, 7 in 2023-10-01T09:46:34.821 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:46:34.822 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:46:34.894 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:34.925 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:34.969 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:34.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:34.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:35.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:46:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:46:34] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-01T09:46:35.074 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:35.101 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:35.441 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:35 smithi138 ceph-mon[192513]: pgmap v1623: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 5965/124995 objects degraded (4.772%); 2517/124995 objects misplaced (2.014%); 946 KiB/s, 14 objects/s recovering 2023-10-01T09:46:35.561 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:46:35.561 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:46:35.571 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:46:35.593 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:46:35.621 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:46:35.646 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:46:35.674 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:46:35.698 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:35.705 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:35 smithi138 ceph-mon[185886]: pgmap v1623: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 5965/124995 objects degraded (4.772%); 2517/124995 objects misplaced (2.014%); 946 KiB/s, 14 objects/s recovering 2023-10-01T09:46:35.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:35.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:35.707 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:46:35.708 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:46:35.708 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:46:35.708 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:46:35.708 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:46:35.709 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:46:35.709 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:46:35.709 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:46:35.709 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:46:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:35 smithi190 ceph-mon[160435]: pgmap v1623: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 203 op/s; 5965/124995 objects degraded (4.772%); 2517/124995 objects misplaced (2.014%); 946 KiB/s, 14 objects/s recovering 2023-10-01T09:46:35.807 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:35.838 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:35.933 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:35.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:35.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:36.038 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:36.063 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:36.123 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:36.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:36.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:36.231 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:36.259 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:36.306 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:36.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:36.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:36.410 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:36.454 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:36.512 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:36.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:36.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:36.519 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:36 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1173243926' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:36.520 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:36 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1173243926' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:36.598 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:36.618 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:46:36.659 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:36.670 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:46:36.670 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:46:36.786 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:46:36.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:36.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:46:36.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:36 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1173243926' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:36.889 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:36.919 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:37.078 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:37.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:37.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:37.182 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:37.209 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:37.259 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:37.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:37.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:37.365 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:37.400 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:37.461 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:37.468 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:37 smithi190 ceph-mon[160435]: pgmap v1624: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 209 op/s; 5965/123471 objects degraded (4.831%); 2517/123471 objects misplaced (2.039%); 941 KiB/s, 14 objects/s recovering 2023-10-01T09:46:37.469 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:37 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:46:37.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:37.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:37.570 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:37.599 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:37.637 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:46:37.639 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:37 smithi138 ceph-mon[192513]: pgmap v1624: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 209 op/s; 5965/123471 objects degraded (4.831%); 2517/123471 objects misplaced (2.039%); 941 KiB/s, 14 objects/s recovering 2023-10-01T09:46:37.640 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:37 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:46:37.640 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:37 smithi138 ceph-mon[185886]: pgmap v1624: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 209 op/s; 5965/123471 objects degraded (4.831%); 2517/123471 objects misplaced (2.039%); 941 KiB/s, 14 objects/s recovering 2023-10-01T09:46:37.640 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:37 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:46:37.653 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:37.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:37.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:37.762 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:37.797 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:37.864 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:37.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:37.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:37.974 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:38.001 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:38.125 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:38.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:38.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:38.151 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:46:38.234 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:38.262 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:38.346 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:38.364 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:38.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:38.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:38.474 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:38.510 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:38.521 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:46:38.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:38 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5965/123471 objects degraded (4.831%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:38.587 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:38.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:38.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:38.693 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:38.731 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:38.793 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:38 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5965/123471 objects degraded (4.831%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:38.793 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:38 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5965/123471 objects degraded (4.831%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:38.848 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:38.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:38.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:38.906 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:46:38.909 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:46:38.957 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:38.993 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:39.251 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:39.756 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:39 smithi138 ceph-mon[192513]: pgmap v1625: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 211 op/s; 5887/121707 objects degraded (4.837%); 2445/121707 objects misplaced (2.009%); 985 KiB/s, 15 objects/s recovering 2023-10-01T09:46:39.757 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:39 smithi138 ceph-mon[185886]: pgmap v1625: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 211 op/s; 5887/121707 objects degraded (4.837%); 2445/121707 objects misplaced (2.009%); 985 KiB/s, 15 objects/s recovering 2023-10-01T09:46:39.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:39 smithi190 ceph-mon[160435]: pgmap v1625: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 211 op/s; 5887/121707 objects degraded (4.837%); 2445/121707 objects misplaced (2.009%); 985 KiB/s, 15 objects/s recovering 2023-10-01T09:46:40.105 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:40.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:40.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:40.208 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:40.215 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1625,"stamp":"2023-10-01T09:46:38.363145+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2661619659,"num_objects":40569,"num_object_clones":0,"num_object_copies":121707,"num_objects_missing_on_primary":928,"num_objects_missing":928,"num_objects_degraded":5887,"num_objects_misplaced":2445,"num_objects_unfound":0,"num_objects_dirty":40569,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":127005,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10266,"num_bytes_recovered":659653647,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120469,"ondisk_log_size":120469,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13554764,"kb_used_data":8463700,"kb_used_omap":39,"kb_used_meta":5090968,"kb_avail":642661300,"statfs":{"total":671965249536,"available":658085171200,"internally_reserved":0,"allocated":8666828800,"data_stored":8656842924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5213151855},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,7,2],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-138805248,"num_objects":-2118,"num_object_clones":0,"num_object_copies":-6354,"num_objects_missing_on_primary":-137,"num_objects_missing":-137,"num_objects_degraded":-150,"num_objects_misplaced":-141,"num_objects_unfound":0,"num_objects_dirty":-2118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2118,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":154,"num_bytes_recovered":10092544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002756"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":322,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.426000+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:46:24.426000+0000","last_peered":"2023-10-01T09:46:24.426000+0000","last_clean":"2023-10-01T09:46:24.426000+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:46:24.426000+0000","last_undegraded":"2023-10-01T09:46:24.426000+0000","last_fullsized":"2023-10-01T09:46:24.426000+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":166,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.446699+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:46:24.446699+0000","last_peered":"2023-10-01T09:46:24.446699+0000","last_clean":"2023-10-01T09:46:24.446699+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:46:24.446699+0000","last_undegraded":"2023-10-01T09:46:24.446699+0000","last_fullsized":"2023-10-01T09:46:24.446699+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":547,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.446448+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:46:24.446448+0000","last_peered":"2023-10-01T09:46:24.446448+0000","last_clean":"2023-10-01T09:46:24.446448+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:46:24.446448+0000","last_undegraded":"2023-10-01T09:46:24.446448+0000","last_fullsized":"2023-10-01T09:46:24.446448+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":605,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.429179+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:46:24.429179+0000","last_peered":"2023-10-01T09:46:24.429179+0000","last_clean":"2023-10-01T09:46:24.429179+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:46:24.429179+0000","last_undegraded":"2023-10-01T09:46:24.429179+0000","last_fullsized":"2023-10-01T09:46:24.429179+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":486,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.431919+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:46:24.431919+0000","last_peered":"2023-10-01T09:46:24.431919+0000","last_clean":"2023-10-01T09:46:24.431919+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:46:24.431919+0000","last_undegraded":"2023-10-01T09:46:24.431919+0000","last_fullsized":"2023-10-01T09:46:24.431919+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":227,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.564005+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:46:24.564005+0000","last_peered":"2023-10-01T09:46:24.564005+0000","last_clean":"2023-10-01T09:46:24.564005+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:46:24.564005+0000","last_undegraded":"2023-10-01T09:46:24.564005+0000","last_fullsized":"2023-10-01T09:46:24.564005+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":603,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.564103+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:46:24.564103+0000","last_peered":"2023-10-01T09:46:24.564103+0000","last_clean":"2023-10-01T09:46:24.564103+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:46:24.564103+0000","last_undegraded":"2023-10-01T09:46:24.564103+0000","last_fullsized":"2023-10-01T09:46:24.564103+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":533,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.425845+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:46:24.425845+0000","last_peered":"2023-10-01T09:46:24.425845+0000","last_clean":"2023-10-01T09:46:24.425845+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:46:24.425845+0000","last_undegraded":"2023-10-01T09:46:24.425845+0000","last_fullsized":"2023-10-01T09:46:24.425845+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":258,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428958+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:46:24.428958+0000","last_peered":"2023-10-01T09:46:24.428958+0000","last_clean":"2023-10-01T09:46:24.428958+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:46:24.428958+0000","last_undegraded":"2023-10-01T09:46:24.428958+0000","last_fullsized":"2023-10-01T09:46:24.428958+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.432821+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:46:24.432821+0000","last_peered":"2023-10-01T09:46:24.432821+0000","last_clean":"2023-10-01T09:46:24.432821+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:46:24.432821+0000","last_undegraded":"2023-10-01T09:46:24.432821+0000","last_fullsized":"2023-10-01T09:46:24.432821+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":71,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.432486+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:46:24.432486+0000","last_peered":"2023-10-01T09:46:24.432486+0000","last_clean":"2023-10-01T09:46:24.432486+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:46:24.432486+0000","last_undegraded":"2023-10-01T09:46:24.432486+0000","last_fullsized":"2023-10-01T09:46:24.432486+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":70,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.446589+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:46:24.446589+0000","last_peered":"2023-10-01T09:46:24.446589+0000","last_clean":"2023-10-01T09:46:24.446589+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:46:24.446589+0000","last_undegraded":"2023-10-01T09:46:24.446589+0000","last_fullsized":"2023-10-01T09:46:24.446589+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":447,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.426329+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:46:24.426329+0000","last_peered":"2023-10-01T09:46:24.426329+0000","last_clean":"2023-10-01T09:46:24.426329+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:46:24.426329+0000","last_undegraded":"2023-10-01T09:46:24.426329+0000","last_fullsized":"2023-10-01T09:46:24.426329+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":328,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.426439+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:46:24.426439+0000","last_peered":"2023-10-01T09:46:24.426439+0000","last_clean":"2023-10-01T09:46:24.426439+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:46:24.426439+0000","last_undegraded":"2023-10-01T09:46:24.426439+0000","last_fullsized":"2023-10-01T09:46:24.426439+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":664,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.431987+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:46:24.431987+0000","last_peered":"2023-10-01T09:46:24.431987+0000","last_clean":"2023-10-01T09:46:24.431987+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:46:24.431987+0000","last_undegraded":"2023-10-01T09:46:24.431987+0000","last_fullsized":"2023-10-01T09:46:24.431987+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":401,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.433288+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:46:24.433288+0000","last_peered":"2023-10-01T09:46:24.433288+0000","last_clean":"2023-10-01T09:46:24.433288+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:46:24.433288+0000","last_undegraded":"2023-10-01T09:46:24.433288+0000","last_fullsized":"2023-10-01T09:46:24.433288+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":74,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.431959+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:46:24.431959+0000","last_peered":"2023-10-01T09:46:24.431959+0000","last_clean":"2023-10-01T09:46:24.431959+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:46:24.431959+0000","last_undegraded":"2023-10-01T09:46:24.431959+0000","last_fullsized":"2023-10-01T09:46:24.431959+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":60,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428413+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:46:24.428413+0000","last_peered":"2023-10-01T09:46:24.428413+0000","last_clean":"2023-10-01T09:46:24.428413+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:46:24.428413+0000","last_undegraded":"2023-10-01T09:46:24.428413+0000","last_fullsized":"2023-10-01T09:46:24.428413+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":557,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.433024+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:46:24.433024+0000","last_peered":"2023-10-01T09:46:24.433024+0000","last_clean":"2023-10-01T09:46:24.433024+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:46:24.433024+0000","last_undegraded":"2023-10-01T09:46:24.433024+0000","last_fullsized":"2023-10-01T09:46:24.433024+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":569,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.429223+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:46:24.429223+0000","last_peered":"2023-10-01T09:46:24.429223+0000","last_clean":"2023-10-01T09:46:24.429223+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:46:24.429223+0000","last_undegraded":"2023-10-01T09:46:24.429223+0000","last_fullsized":"2023-10-01T09:46:24.429223+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":425,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.431831+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:46:24.431831+0000","last_peered":"2023-10-01T09:46:24.431831+0000","last_clean":"2023-10-01T09:46:24.431831+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:46:24.431831+0000","last_undegraded":"2023-10-01T09:46:24.431831+0000","last_fullsized":"2023-10-01T09:46:24.431831+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":74,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.432228+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:46:24.432228+0000","last_peered":"2023-10-01T09:46:24.432228+0000","last_clean":"2023-10-01T09:46:24.432228+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:46:24.432228+0000","last_undegraded":"2023-10-01T09:46:24.432228+0000","last_fullsized":"2023-10-01T09:46:24.432228+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"546'7875","reported_seq":7954,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:37.458822+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:46:37.458822+0000","last_peered":"2023-10-01T09:46:37.458822+0000","last_clean":"2023-10-01T09:46:37.458822+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:46:37.458822+0000","last_undegraded":"2023-10-01T09:46:37.458822+0000","last_fullsized":"2023-10-01T09:46:37.458822+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7875,"log_dups_size":0,"ondisk_log_size":7875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164954112,"num_objects":2517,"num_object_clones":0,"num_object_copies":7551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7875,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":69,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.432083+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:46:24.432083+0000","last_peered":"2023-10-01T09:46:24.432083+0000","last_clean":"2023-10-01T09:46:24.432083+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:46:24.432083+0000","last_undegraded":"2023-10-01T09:46:24.432083+0000","last_fullsized":"2023-10-01T09:46:24.432083+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":266,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.427831+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:46:24.427831+0000","last_peered":"2023-10-01T09:46:24.427831+0000","last_clean":"2023-10-01T09:46:24.427831+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:46:24.427831+0000","last_undegraded":"2023-10-01T09:46:24.427831+0000","last_fullsized":"2023-10-01T09:46:24.427831+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"546'7959","reported_seq":16336,"reported_epoch":546,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:46:37.479181+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:46:37.479181+0000","last_peered":"2023-10-01T09:46:37.479181+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:46:37.479181+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7959,"log_dups_size":0,"ondisk_log_size":7959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160366592,"num_objects":2447,"num_object_clones":0,"num_object_copies":7341,"num_objects_missing_on_primary":928,"num_objects_missing":928,"num_objects_degraded":3307,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7959,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2756,"num_bytes_recovered":180617216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1451},{"shards":"0,4,6","objects":928},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":540,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.432943+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:46:24.432943+0000","last_peered":"2023-10-01T09:46:24.432943+0000","last_clean":"2023-10-01T09:46:24.432943+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:46:24.432943+0000","last_undegraded":"2023-10-01T09:46:24.432943+0000","last_fullsized":"2023-10-01T09:46:24.432943+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":602,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.446362+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:46:24.446362+0000","last_peered":"2023-10-01T09:46:24.446362+0000","last_clean":"2023-10-01T09:46:24.446362+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:46:24.446362+0000","last_undegraded":"2023-10-01T09:46:24.446362+0000","last_fullsized":"2023-10-01T09:46:24.446362+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"546'7895","reported_seq":7978,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:36.522671+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:46:36.522671+0000","last_peered":"2023-10-01T09:46:36.522671+0000","last_clean":"2023-10-01T09:46:36.522671+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:46:36.522671+0000","last_undegraded":"2023-10-01T09:46:36.522671+0000","last_fullsized":"2023-10-01T09:46:36.522671+0000","mapping_epoch":475,"log_start":"481'900","ondisk_log_start":"481'900","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166264832,"num_objects":2537,"num_object_clones":0,"num_object_copies":7611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7895,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":626,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.432569+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:46:24.432569+0000","last_peered":"2023-10-01T09:46:24.432569+0000","last_clean":"2023-10-01T09:46:24.432569+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:46:24.432569+0000","last_undegraded":"2023-10-01T09:46:24.432569+0000","last_fullsized":"2023-10-01T09:46:24.432569+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":423,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.446578+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:46:24.446578+0000","last_peered":"2023-10-01T09:46:24.446578+0000","last_clean":"2023-10-01T09:46:24.446578+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:46:24.446578+0000","last_undegraded":"2023-10-01T09:46:24.446578+0000","last_fullsized":"2023-10-01T09:46:24.446578+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"546'7916","reported_seq":14604,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:36.896010+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:46:36.896010+0000","last_peered":"2023-10-01T09:46:36.896010+0000","last_clean":"2023-10-01T09:46:36.896010+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:46:36.896010+0000","last_undegraded":"2023-10-01T09:46:36.896010+0000","last_fullsized":"2023-10-01T09:46:36.896010+0000","mapping_epoch":505,"log_start":"481'900","ondisk_log_start":"481'900","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7016,"log_dups_size":0,"ondisk_log_size":7016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162922496,"num_objects":2486,"num_object_clones":0,"num_object_copies":7458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7916,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":266,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428431+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:46:24.428431+0000","last_peered":"2023-10-01T09:46:24.428431+0000","last_clean":"2023-10-01T09:46:24.428431+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:46:24.428431+0000","last_undegraded":"2023-10-01T09:46:24.428431+0000","last_fullsized":"2023-10-01T09:46:24.428431+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":525,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428730+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:46:24.428730+0000","last_peered":"2023-10-01T09:46:24.428730+0000","last_clean":"2023-10-01T09:46:24.428730+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:46:24.428730+0000","last_undegraded":"2023-10-01T09:46:24.428730+0000","last_fullsized":"2023-10-01T09:46:24.428730+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"546'7898","reported_seq":7981,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:34.810864+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:46:34.810864+0000","last_peered":"2023-10-01T09:46:34.810864+0000","last_clean":"2023-10-01T09:46:34.810864+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:46:34.810864+0000","last_undegraded":"2023-10-01T09:46:34.810864+0000","last_fullsized":"2023-10-01T09:46:34.810864+0000","mapping_epoch":475,"log_start":"481'300","ondisk_log_start":"481'300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7598,"log_dups_size":0,"ondisk_log_size":7598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171048960,"num_objects":2610,"num_object_clones":0,"num_object_copies":7830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7898,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":126,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.426763+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:46:24.426763+0000","last_peered":"2023-10-01T09:46:24.426763+0000","last_clean":"2023-10-01T09:46:24.426763+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:46:24.426763+0000","last_undegraded":"2023-10-01T09:46:24.426763+0000","last_fullsized":"2023-10-01T09:46:24.426763+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":200,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.426353+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:46:24.426353+0000","last_peered":"2023-10-01T09:46:24.426353+0000","last_clean":"2023-10-01T09:46:24.426353+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:46:24.426353+0000","last_undegraded":"2023-10-01T09:46:24.426353+0000","last_fullsized":"2023-10-01T09:46:24.426353+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"546'7859","reported_seq":8158,"reported_epoch":546,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:46:37.370977+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:46:37.370977+0000","last_peered":"2023-10-01T09:46:37.370977+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:46:37.370977+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7859,"log_dups_size":0,"ondisk_log_size":7859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164429824,"num_objects":2509,"num_object_clones":0,"num_object_copies":7527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2570,"num_objects_misplaced":2445,"num_objects_unfound":0,"num_objects_dirty":2509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7859,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":106,"num_bytes_recovered":6946816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2445},{"shards":"4,5","objects":-2506}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":444,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.426170+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:46:24.426170+0000","last_peered":"2023-10-01T09:46:24.426170+0000","last_clean":"2023-10-01T09:46:24.426170+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:46:24.426170+0000","last_undegraded":"2023-10-01T09:46:24.426170+0000","last_fullsized":"2023-10-01T09:46:24.426170+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":605,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.446779+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:46:24.446779+0000","last_peered":"2023-10-01T09:46:24.446779+0000","last_clean":"2023-10-01T09:46:24.446779+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:46:24.446779+0000","last_undegraded":"2023-10-01T09:46:24.446779+0000","last_fullsized":"2023-10-01T09:46:24.446779+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"546'7846","reported_seq":7934,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:37.109164+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:46:37.109164+0000","last_peered":"2023-10-01T09:46:37.109164+0000","last_clean":"2023-10-01T09:46:37.109164+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:46:37.109164+0000","last_undegraded":"2023-10-01T09:46:37.109164+0000","last_fullsized":"2023-10-01T09:46:37.109164+0000","mapping_epoch":483,"log_start":"481'300","ondisk_log_start":"481'300","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7546,"log_dups_size":0,"ondisk_log_size":7546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165937152,"num_objects":2532,"num_object_clones":0,"num_object_copies":7596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7846,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":440,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.425912+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:46:24.425912+0000","last_peered":"2023-10-01T09:46:24.425912+0000","last_clean":"2023-10-01T09:46:24.425912+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:46:24.425912+0000","last_undegraded":"2023-10-01T09:46:24.425912+0000","last_fullsized":"2023-10-01T09:46:24.425912+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":281,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.427252+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:46:24.427252+0000","last_peered":"2023-10-01T09:46:24.427252+0000","last_clean":"2023-10-01T09:46:24.427252+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:46:24.427252+0000","last_undegraded":"2023-10-01T09:46:24.427252+0000","last_fullsized":"2023-10-01T09:46:24.427252+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"546'7943","reported_seq":8022,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:35.973241+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:46:35.973241+0000","last_peered":"2023-10-01T09:46:35.973241+0000","last_clean":"2023-10-01T09:46:35.973241+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:46:35.973241+0000","last_undegraded":"2023-10-01T09:46:35.973241+0000","last_fullsized":"2023-10-01T09:46:35.973241+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7943,"log_dups_size":0,"ondisk_log_size":7943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167444480,"num_objects":2555,"num_object_clones":0,"num_object_copies":7665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7943,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":410,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.448763+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:46:24.448763+0000","last_peered":"2023-10-01T09:46:24.448763+0000","last_clean":"2023-10-01T09:46:24.448763+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:46:24.448763+0000","last_undegraded":"2023-10-01T09:46:24.448763+0000","last_fullsized":"2023-10-01T09:46:24.448763+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":457,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.448325+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:46:24.448325+0000","last_peered":"2023-10-01T09:46:24.448325+0000","last_clean":"2023-10-01T09:46:24.448325+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:46:24.448325+0000","last_undegraded":"2023-10-01T09:46:24.448325+0000","last_fullsized":"2023-10-01T09:46:24.448325+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"546'7714","reported_seq":7793,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:36.235711+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:46:36.235711+0000","last_peered":"2023-10-01T09:46:36.235711+0000","last_clean":"2023-10-01T09:46:36.235711+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:46:36.235711+0000","last_undegraded":"2023-10-01T09:46:36.235711+0000","last_fullsized":"2023-10-01T09:46:36.235711+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7714,"log_dups_size":0,"ondisk_log_size":7714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161480704,"num_objects":2464,"num_object_clones":0,"num_object_copies":7392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7714,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":939,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.433147+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:46:24.433147+0000","last_peered":"2023-10-01T09:46:24.433147+0000","last_clean":"2023-10-01T09:46:24.433147+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:46:24.433147+0000","last_undegraded":"2023-10-01T09:46:24.433147+0000","last_fullsized":"2023-10-01T09:46:24.433147+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":276,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428689+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:46:24.428689+0000","last_peered":"2023-10-01T09:46:24.428689+0000","last_clean":"2023-10-01T09:46:24.428689+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:46:24.428689+0000","last_undegraded":"2023-10-01T09:46:24.428689+0000","last_fullsized":"2023-10-01T09:46:24.428689+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.446451+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:46:24.446451+0000","last_peered":"2023-10-01T09:46:24.446451+0000","last_clean":"2023-10-01T09:46:24.446451+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:46:24.446451+0000","last_undegraded":"2023-10-01T09:46:24.446451+0000","last_fullsized":"2023-10-01T09:46:24.446451+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"546'7791","reported_seq":7873,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:36.662478+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:46:36.662478+0000","last_peered":"2023-10-01T09:46:36.662478+0000","last_clean":"2023-10-01T09:46:36.662478+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:46:36.662478+0000","last_undegraded":"2023-10-01T09:46:36.662478+0000","last_fullsized":"2023-10-01T09:46:36.662478+0000","mapping_epoch":475,"log_start":"481'800","ondisk_log_start":"481'800","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6991,"log_dups_size":0,"ondisk_log_size":6991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169017344,"num_objects":2579,"num_object_clones":0,"num_object_copies":7737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7791,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1211,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428121+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:46:24.428121+0000","last_peered":"2023-10-01T09:46:24.428121+0000","last_clean":"2023-10-01T09:46:24.428121+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:46:24.428121+0000","last_undegraded":"2023-10-01T09:46:24.428121+0000","last_fullsized":"2023-10-01T09:46:24.428121+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":421,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.426660+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:46:24.426660+0000","last_peered":"2023-10-01T09:46:24.426660+0000","last_clean":"2023-10-01T09:46:24.426660+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:46:24.426660+0000","last_undegraded":"2023-10-01T09:46:24.426660+0000","last_fullsized":"2023-10-01T09:46:24.426660+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":249,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428519+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:46:24.428519+0000","last_peered":"2023-10-01T09:46:24.428519+0000","last_clean":"2023-10-01T09:46:24.428519+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:46:24.428519+0000","last_undegraded":"2023-10-01T09:46:24.428519+0000","last_fullsized":"2023-10-01T09:46:24.428519+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"546'7851","reported_seq":15151,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:36.951355+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:46:36.951355+0000","last_peered":"2023-10-01T09:46:36.951355+0000","last_clean":"2023-10-01T09:46:36.951355+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:46:36.951355+0000","last_undegraded":"2023-10-01T09:46:36.951355+0000","last_fullsized":"2023-10-01T09:46:36.951355+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7851,"log_dups_size":0,"ondisk_log_size":7851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165478400,"num_objects":2525,"num_object_clones":0,"num_object_copies":7575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7851,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":892,"reported_epoch":545,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:46:24.428438+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:46:24.428438+0000","last_peered":"2023-10-01T09:46:24.428438+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:46:24.428438+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:46:24.428438+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":610,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.448791+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:46:24.448791+0000","last_peered":"2023-10-01T09:46:24.448791+0000","last_clean":"2023-10-01T09:46:24.448791+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:46:24.448791+0000","last_undegraded":"2023-10-01T09:46:24.448791+0000","last_fullsized":"2023-10-01T09:46:24.448791+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":70,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.446962+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:46:24.446962+0000","last_peered":"2023-10-01T09:46:24.446962+0000","last_clean":"2023-10-01T09:46:24.446962+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:46:24.446962+0000","last_undegraded":"2023-10-01T09:46:24.446962+0000","last_fullsized":"2023-10-01T09:46:24.446962+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"546'7756","reported_seq":7838,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:34.932048+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:46:34.932048+0000","last_peered":"2023-10-01T09:46:34.932048+0000","last_clean":"2023-10-01T09:46:34.932048+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:46:34.932048+0000","last_undegraded":"2023-10-01T09:46:34.932048+0000","last_fullsized":"2023-10-01T09:46:34.932048+0000","mapping_epoch":475,"log_start":"480'200","ondisk_log_start":"480'200","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7556,"log_dups_size":0,"ondisk_log_size":7556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164888576,"num_objects":2516,"num_object_clones":0,"num_object_copies":7548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7756,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1662,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428616+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:46:24.428616+0000","last_peered":"2023-10-01T09:46:24.428616+0000","last_clean":"2023-10-01T09:46:24.428616+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:46:24.428616+0000","last_undegraded":"2023-10-01T09:46:24.428616+0000","last_fullsized":"2023-10-01T09:46:24.428616+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":576,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.431840+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:46:24.431840+0000","last_peered":"2023-10-01T09:46:24.431840+0000","last_clean":"2023-10-01T09:46:24.431840+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:46:24.431840+0000","last_undegraded":"2023-10-01T09:46:24.431840+0000","last_fullsized":"2023-10-01T09:46:24.431840+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":354,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428210+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:46:24.428210+0000","last_peered":"2023-10-01T09:46:24.428210+0000","last_clean":"2023-10-01T09:46:24.428210+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:46:24.428210+0000","last_undegraded":"2023-10-01T09:46:24.428210+0000","last_fullsized":"2023-10-01T09:46:24.428210+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":344,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.426570+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:46:24.426570+0000","last_peered":"2023-10-01T09:46:24.426570+0000","last_clean":"2023-10-01T09:46:24.426570+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:46:24.426570+0000","last_undegraded":"2023-10-01T09:46:24.426570+0000","last_fullsized":"2023-10-01T09:46:24.426570+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"546'7947","reported_seq":8026,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:36.261685+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:46:36.261685+0000","last_peered":"2023-10-01T09:46:36.261685+0000","last_clean":"2023-10-01T09:46:36.261685+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:46:36.261685+0000","last_undegraded":"2023-10-01T09:46:36.261685+0000","last_fullsized":"2023-10-01T09:46:36.261685+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7947,"log_dups_size":0,"ondisk_log_size":7947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169410560,"num_objects":2585,"num_object_clones":0,"num_object_copies":7755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7947,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":753,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.432549+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:46:24.432549+0000","last_peered":"2023-10-01T09:46:24.432549+0000","last_clean":"2023-10-01T09:46:24.432549+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:46:24.432549+0000","last_undegraded":"2023-10-01T09:46:24.432549+0000","last_fullsized":"2023-10-01T09:46:24.432549+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":587,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428696+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:46:24.428696+0000","last_peered":"2023-10-01T09:46:24.428696+0000","last_clean":"2023-10-01T09:46:24.428696+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:46:24.428696+0000","last_undegraded":"2023-10-01T09:46:24.428696+0000","last_fullsized":"2023-10-01T09:46:24.428696+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":586,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.429050+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:46:24.429050+0000","last_peered":"2023-10-01T09:46:24.429050+0000","last_clean":"2023-10-01T09:46:24.429050+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:46:24.429050+0000","last_undegraded":"2023-10-01T09:46:24.429050+0000","last_fullsized":"2023-10-01T09:46:24.429050+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"546'7886","reported_seq":7969,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:36.585928+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:46:36.585928+0000","last_peered":"2023-10-01T09:46:36.585928+0000","last_clean":"2023-10-01T09:46:36.585928+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:46:36.585928+0000","last_undegraded":"2023-10-01T09:46:36.585928+0000","last_fullsized":"2023-10-01T09:46:36.585928+0000","mapping_epoch":475,"log_start":"481'900","ondisk_log_start":"481'900","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6986,"log_dups_size":0,"ondisk_log_size":6986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168624152,"num_objects":2574,"num_object_clones":0,"num_object_copies":7722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2574,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7886,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":984,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.432644+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:46:24.432644+0000","last_peered":"2023-10-01T09:46:24.432644+0000","last_clean":"2023-10-01T09:46:24.432644+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:46:24.432644+0000","last_undegraded":"2023-10-01T09:46:24.432644+0000","last_fullsized":"2023-10-01T09:46:24.432644+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":603,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428457+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:46:24.428457+0000","last_peered":"2023-10-01T09:46:24.428457+0000","last_clean":"2023-10-01T09:46:24.428457+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:46:24.428457+0000","last_undegraded":"2023-10-01T09:46:24.428457+0000","last_fullsized":"2023-10-01T09:46:24.428457+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":456,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428702+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:46:24.428702+0000","last_peered":"2023-10-01T09:46:24.428702+0000","last_clean":"2023-10-01T09:46:24.428702+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:46:24.428702+0000","last_undegraded":"2023-10-01T09:46:24.428702+0000","last_fullsized":"2023-10-01T09:46:24.428702+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"546'7831","reported_seq":7918,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:35.993604+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:46:35.993604+0000","last_peered":"2023-10-01T09:46:35.993604+0000","last_clean":"2023-10-01T09:46:35.993604+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:46:35.993604+0000","last_undegraded":"2023-10-01T09:46:35.993604+0000","last_fullsized":"2023-10-01T09:46:35.993604+0000","mapping_epoch":479,"log_start":"481'1100","ondisk_log_start":"481'1100","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6731,"log_dups_size":0,"ondisk_log_size":6731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165478400,"num_objects":2525,"num_object_clones":0,"num_object_copies":7575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7831,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":485,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.432310+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:46:24.432310+0000","last_peered":"2023-10-01T09:46:24.432310+0000","last_clean":"2023-10-01T09:46:24.432310+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:46:24.432310+0000","last_undegraded":"2023-10-01T09:46:24.432310+0000","last_fullsized":"2023-10-01T09:46:24.432310+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1179,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.446989+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:46:24.446989+0000","last_peered":"2023-10-01T09:46:24.446989+0000","last_clean":"2023-10-01T09:46:24.446989+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:46:24.446989+0000","last_undegraded":"2023-10-01T09:46:24.446989+0000","last_fullsized":"2023-10-01T09:46:24.446989+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":74,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.431765+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:46:24.431765+0000","last_peered":"2023-10-01T09:46:24.431765+0000","last_clean":"2023-10-01T09:46:24.431765+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:46:24.431765+0000","last_undegraded":"2023-10-01T09:46:24.431765+0000","last_fullsized":"2023-10-01T09:46:24.431765+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"546'7932","reported_seq":8019,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:36.153613+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:46:36.153613+0000","last_peered":"2023-10-01T09:46:36.153613+0000","last_clean":"2023-10-01T09:46:36.153613+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:46:36.153613+0000","last_undegraded":"2023-10-01T09:46:36.153613+0000","last_fullsized":"2023-10-01T09:46:36.153613+0000","mapping_epoch":479,"log_start":"481'1200","ondisk_log_start":"481'1200","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6732,"log_dups_size":0,"ondisk_log_size":6732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169082880,"num_objects":2580,"num_object_clones":0,"num_object_copies":7740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7932,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":78,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.432160+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:46:24.432160+0000","last_peered":"2023-10-01T09:46:24.432160+0000","last_clean":"2023-10-01T09:46:24.432160+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:46:24.432160+0000","last_undegraded":"2023-10-01T09:46:24.432160+0000","last_fullsized":"2023-10-01T09:46:24.432160+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":564,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.429131+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:46:24.429131+0000","last_peered":"2023-10-01T09:46:24.429131+0000","last_clean":"2023-10-01T09:46:24.429131+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:46:24.429131+0000","last_undegraded":"2023-10-01T09:46:24.429131+0000","last_fullsized":"2023-10-01T09:46:24.429131+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":269,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.427937+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:46:24.427937+0000","last_peered":"2023-10-01T09:46:24.427937+0000","last_clean":"2023-10-01T09:46:24.427937+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:46:24.427937+0000","last_undegraded":"2023-10-01T09:46:24.427937+0000","last_fullsized":"2023-10-01T09:46:24.427937+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":513,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428989+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:46:24.428989+0000","last_peered":"2023-10-01T09:46:24.428989+0000","last_clean":"2023-10-01T09:46:24.428989+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:46:24.428989+0000","last_undegraded":"2023-10-01T09:46:24.428989+0000","last_fullsized":"2023-10-01T09:46:24.428989+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":627,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:25.277810+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:46:25.277810+0000","last_peered":"2023-10-01T09:46:25.277810+0000","last_clean":"2023-10-01T09:46:25.277810+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:46:25.277810+0000","last_undegraded":"2023-10-01T09:46:25.277810+0000","last_fullsized":"2023-10-01T09:46:25.277810+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":281,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428381+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:46:24.428381+0000","last_peered":"2023-10-01T09:46:24.428381+0000","last_clean":"2023-10-01T09:46:24.428381+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:46:24.428381+0000","last_undegraded":"2023-10-01T09:46:24.428381+0000","last_fullsized":"2023-10-01T09:46:24.428381+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":70,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.446430+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:46:24.446430+0000","last_peered":"2023-10-01T09:46:24.446430+0000","last_clean":"2023-10-01T09:46:24.446430+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:46:24.446430+0000","last_undegraded":"2023-10-01T09:46:24.446430+0000","last_fullsized":"2023-10-01T09:46:24.446430+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":281,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428701+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:46:24.428701+0000","last_peered":"2023-10-01T09:46:24.428701+0000","last_clean":"2023-10-01T09:46:24.428701+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:46:24.428701+0000","last_undegraded":"2023-10-01T09:46:24.428701+0000","last_fullsized":"2023-10-01T09:46:24.428701+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":459,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.429770+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:46:24.429770+0000","last_peered":"2023-10-01T09:46:24.429770+0000","last_clean":"2023-10-01T09:46:24.429770+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:46:24.429770+0000","last_undegraded":"2023-10-01T09:46:24.429770+0000","last_fullsized":"2023-10-01T09:46:24.429770+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":88,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.446905+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:46:24.446905+0000","last_peered":"2023-10-01T09:46:24.446905+0000","last_clean":"2023-10-01T09:46:24.446905+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:46:24.446905+0000","last_undegraded":"2023-10-01T09:46:24.446905+0000","last_fullsized":"2023-10-01T09:46:24.446905+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":504,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428609+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:46:24.428609+0000","last_peered":"2023-10-01T09:46:24.428609+0000","last_clean":"2023-10-01T09:46:24.428609+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:46:24.428609+0000","last_undegraded":"2023-10-01T09:46:24.428609+0000","last_fullsized":"2023-10-01T09:46:24.428609+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":704,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.429415+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:46:24.429415+0000","last_peered":"2023-10-01T09:46:24.429415+0000","last_clean":"2023-10-01T09:46:24.429415+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:46:24.429415+0000","last_undegraded":"2023-10-01T09:46:24.429415+0000","last_fullsized":"2023-10-01T09:46:24.429415+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":397,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.425882+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:46:24.425882+0000","last_peered":"2023-10-01T09:46:24.425882+0000","last_clean":"2023-10-01T09:46:24.425882+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:46:24.425882+0000","last_undegraded":"2023-10-01T09:46:24.425882+0000","last_fullsized":"2023-10-01T09:46:24.425882+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2656829464,"num_objects":40541,"num_object_clones":0,"num_object_copies":121623,"num_objects_missing_on_primary":928,"num_objects_missing":928,"num_objects_degraded":5877,"num_objects_misplaced":2445,"num_objects_unfound":0,"num_objects_dirty":40541,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":125899,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9755,"num_bytes_recovered":639303680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8594796544,"data_stored":8594784328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119299,"ondisk_log_size":119299,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430752,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1725920,"kb_used_data":818712,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":92019232,"statfs":{"total":95995035648,"available":94227693568,"internally_reserved":0,"allocated":838361088,"data_stored":837416904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.74399999999999999,"15min":2.5840000000000001},"min":{"1min":0.27700000000000002,"5min":0.246,"15min":0.20999999999999999},"max":{"1min":1.081,"5min":6.8550000000000004,"15min":108.90000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.52500000000000002,"15min":2.7050000000000001},"min":{"1min":0.25700000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.2090000000000001,"5min":5.6639999999999997,"15min":109.187},"last":0.314}]},{"osd":1,"last update":"Sun Oct 1 09:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.66400000000000003,"15min":5.1520000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.222},"max":{"1min":1.0780000000000001,"5min":6.8289999999999997,"15min":229.161},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.59799999999999998,"15min":5.1779999999999999},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":3.0640000000000001,"5min":5.7240000000000002,"15min":230.126},"last":0.35799999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.72799999999999998,"15min":1.617},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.1259999999999999,"5min":5.3090000000000002,"15min":14.901999999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.68500000000000005,"15min":1.6100000000000001},"min":{"1min":0.33700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.0009999999999999,"5min":5.6929999999999996,"15min":14.682},"last":0.45700000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.65800000000000003,"15min":2.2000000000000002},"min":{"1min":0.35699999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.476,"5min":5.7750000000000004,"15min":76.555000000000007},"last":0.434},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.68100000000000005,"15min":2.2010000000000001},"min":{"1min":0.32800000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.4529999999999998,"5min":6.548,"15min":76.534000000000006},"last":0.44600000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.70499999999999996,"15min":1.042},"min":{"1min":0.28299999999999997,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.276,"5min":6.8769999999999998,"15min":9.6020000000000003},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.68500000000000005,"15min":0.997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.97399999999999998,"5min":5.7439999999999998,"15min":8.4949999999999992},"last":0.39700000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":2.0960000000000001,"15min":1.548},"min":{"1min":0.315,"5min":0.311,"15min":0.29599999999999999},"max":{"1min":0.80000000000000004,"5min":135.40000000000001,"15min":135.40000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":2.0830000000000002,"15min":1.6459999999999999},"min":{"1min":0.35699999999999998,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":2.8849999999999998,"5min":135.339,"15min":135.339},"last":0.48299999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.075,"15min":1.133},"min":{"1min":0.38,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":0.754,"5min":11.58,"15min":11.853999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.86699999999999999,"15min":1.111},"min":{"1min":0.35599999999999998,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":3.7810000000000001,"5min":13.021000000000001,"15min":14.584},"last":0.51000000000000001}]}]},{"osd":6,"up_from":294,"seq":1262720385263,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1473884,"kb_used_data":1140372,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":92271268,"statfs":{"total":95995035648,"available":94485778432,"internally_reserved":0,"allocated":1167740928,"data_stored":1166800695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.45200000000000001,"15min":3.681},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20599999999999999},"max":{"1min":0.51700000000000002,"5min":3.3340000000000001,"15min":158.48599999999999},"last":34.07},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.55500000000000005,"15min":3.746},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.221},"max":{"1min":3.3519999999999999,"5min":3.3519999999999999,"15min":158.37700000000001},"last":34.088000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.58499999999999996,"15min":4.5739999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":3.3069999999999999,"5min":3.3069999999999999,"15min":168.947},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.60299999999999998,"15min":4.532},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":3.7469999999999999,"5min":3.7469999999999999,"15min":169.07900000000001},"last":0.437}]},{"osd":2,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.59999999999999998,"15min":1.4490000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":3.3849999999999998,"5min":3.3849999999999998,"15min":10.92},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":1.4690000000000001},"min":{"1min":0.32100000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.7160000000000002,"5min":3.7160000000000002,"15min":21.704999999999998},"last":0.46200000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.57999999999999996,"15min":1.8100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.63800000000000001,"5min":3.3759999999999999,"15min":89.753},"last":0.307},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.69499999999999995,"15min":1.754},"min":{"1min":0.309,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.2050000000000001,"5min":4.2050000000000001,"15min":89.849999999999994},"last":0.36199999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58899999999999997,"15min":1.069},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.6349999999999998,"5min":4.0250000000000004,"15min":63.134},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.65800000000000003,"15min":1.0680000000000001},"min":{"1min":0.36599999999999999,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":1.006,"5min":7.4039999999999999,"15min":62.973999999999997},"last":0.34499999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:46:37 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.88500000000000001,"15min":0.999},"min":{"1min":0.33100000000000002,"5min":0.26700000000000002,"15min":0.251},"max":{"1min":0.93799999999999994,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.378},{"interface":"front","average":{"1min":0.875,"5min":1.022,"15min":1.1379999999999999},"min":{"1min":0.315,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.181,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.47899999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.56699999999999995,"15min":1.2729999999999999},"min":{"1min":0.377,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.73399999999999999,"5min":3.339,"15min":94.245999999999995},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.67200000000000004,"15min":1.369},"min":{"1min":0.33100000000000002,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":3.653,"5min":3.653,"15min":94.715000000000003},"last":0.44600000000000001}]}]},{"osd":1,"up_from":94,"seq":403726926239,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1325744,"kb_used_data":501800,"kb_used_omap":7,"kb_used_meta":823928,"kb_avail":92419408,"statfs":{"total":95995035648,"available":94637473792,"internally_reserved":0,"allocated":513843200,"data_stored":511770565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":843702695},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4410000000000001,"5min":1.1140000000000001,"15min":3.8010000000000002},"min":{"1min":0.217,"5min":0.186,"15min":0.182},"max":{"1min":18.556999999999999,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.44},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.302,"15min":3.3359999999999999},"min":{"1min":0.23499999999999999,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":18.704000000000001,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.36499999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.39800000000000002,"15min":2.2669999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.51100000000000001,"5min":2.8740000000000001,"15min":51.640000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.61799999999999999,"15min":2.9500000000000002},"min":{"1min":0.245,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":2.9089999999999998,"5min":6.9710000000000001,"15min":154.197},"last":0.46400000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.45700000000000002,"15min":2.9540000000000002},"min":{"1min":0.28199999999999997,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":2.8780000000000001,"5min":3.1949999999999998,"15min":95.305000000000007},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.432,"5min":0.51200000000000001,"15min":3.6720000000000002},"min":{"1min":0.27300000000000002,"5min":0.245,"15min":0.216},"max":{"1min":0.95699999999999996,"5min":4.7839999999999998,"15min":154.483},"last":0.65200000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.59099999999999997,"15min":2.1829999999999998},"min":{"1min":0.312,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":1.5309999999999999,"5min":3.0550000000000002,"15min":154.101},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.49099999999999999,"15min":2.2869999999999999},"min":{"1min":0.34300000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65500000000000003,"5min":2.4209999999999998,"15min":154.56999999999999},"last":0.74199999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.246,"15min":2.2530000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.30199999999999999},"max":{"1min":1.696,"5min":59.561,"15min":154.22999999999999},"last":0.628},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.1970000000000001,"15min":2.4649999999999999},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.307},"max":{"1min":0.67300000000000004,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.69199999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.65700000000000003,"15min":2.448},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.944,"5min":3.3660000000000001,"15min":154.12200000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64600000000000002,"15min":2.3450000000000002},"min":{"1min":0.374,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":1.7549999999999999,"5min":2.5630000000000002,"15min":154.16999999999999},"last":0.71099999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.74299999999999999,"15min":2.5230000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.33900000000000002},"max":{"1min":3.6160000000000001,"5min":5.2169999999999996,"15min":154.20699999999999},"last":0.748},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.70499999999999996,"15min":2.5470000000000002},"min":{"1min":0.40300000000000002,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":3.7490000000000001,"5min":3.8260000000000001,"15min":154.51599999999999},"last":0.72599999999999998}]}]},{"osd":0,"up_from":89,"seq":382252089774,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3231756,"kb_used_data":2296324,"kb_used_omap":8,"kb_used_meta":935415,"kb_avail":90513396,"statfs":{"total":95995035648,"available":92685717504,"internally_reserved":0,"allocated":2351435776,"data_stored":2349403164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":957865922},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.311,"5min":0.42399999999999999,"15min":3.988},"min":{"1min":0.20699999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.83699999999999997,"5min":4.2919999999999998,"15min":216.821},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.44800000000000001,"15min":4.6740000000000004},"min":{"1min":0.218,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.56799999999999995,"5min":3.9620000000000002,"15min":217.02699999999999},"last":0.42199999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.64600000000000002,"15min":1.9910000000000001},"min":{"1min":0.25,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":4.0110000000000001,"5min":4.9459999999999997,"15min":19.788},"last":0.376},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.65500000000000003,"15min":2.0310000000000001},"min":{"1min":0.28499999999999998,"5min":0.27200000000000002,"15min":0.189},"max":{"1min":3.8719999999999999,"5min":3.8719999999999999,"15min":25.844999999999999},"last":0.31900000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.504,"15min":3.153},"min":{"1min":0.24099999999999999,"5min":0.22,"15min":0.22},"max":{"1min":1.9039999999999999,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.308},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.68200000000000005,"15min":3.0019999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.1379999999999999,"5min":4.9199999999999999,"15min":222.227},"last":0.46800000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.59499999999999997,"15min":1.925},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":0.84299999999999997,"5min":4.048,"15min":62.206000000000003},"last":0.624},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.63600000000000001,"15min":2.0089999999999999},"min":{"1min":0.29199999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":1.5189999999999999,"5min":4,"15min":62.073999999999998},"last":0.435}]},{"osd":5,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.76800000000000002,"15min":1.744},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":4.0350000000000001,"5min":5.8769999999999998,"15min":16.808},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.84499999999999997,"15min":1.6499999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.1269999999999998,"5min":4.8689999999999998,"15min":17.507000000000001},"last":0.48699999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:46:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.63600000000000001,"15min":1.6990000000000001},"min":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.1970000000000001,"5min":4.3129999999999997,"15min":17.535},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65900000000000003,"15min":1.732},"min":{"1min":0.372,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.218,"5min":4.2050000000000001,"15min":15.651999999999999},"last":0.60699999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.79500000000000004,"15min":2.028},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":2.1000000000000001,"5min":4.976,"15min":48.895000000000003},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.89900000000000002,"15min":2.048},"min":{"1min":0.376,"5min":0.376,"15min":0.35099999999999998},"max":{"1min":4.125,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.64000000000000001}]}]},{"osd":2,"up_from":0,"seq":425201762713,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599187,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1332592,"kb_used_data":500456,"kb_used_omap":3,"kb_used_meta":832124,"kb_avail":92412560,"statfs":{"total":95995035648,"available":94630461440,"internally_reserved":0,"allocated":512466944,"data_stored":510336184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":852095719},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":1.4870000000000001,"15min":4.6909999999999998},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":0.65000000000000002,"5min":64.504000000000005,"15min":137.34399999999999},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":1.4870000000000001,"15min":4.6959999999999997},"min":{"1min":0.219,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":1.2849999999999999,"5min":64.448999999999998,"15min":137.73400000000001},"last":0.51000000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.51700000000000002,"15min":4.8719999999999999},"min":{"1min":0.25,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":1.3500000000000001,"5min":6.0250000000000004,"15min":162.22999999999999},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.40600000000000003,"15min":4.9199999999999999},"min":{"1min":0.20899999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.77600000000000002,"5min":2.931,"15min":162.70699999999999},"last":0.30399999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56599999999999995,"15min":1.5800000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.20799999999999999},"max":{"1min":2.9049999999999998,"5min":7.141,"15min":14.254},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.52300000000000002,"15min":1.5860000000000001},"min":{"1min":0.314,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":0.80400000000000005,"5min":4.5049999999999999,"15min":15.523},"last":0.441}]},{"osd":4,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.54600000000000004,"15min":1.71},"min":{"1min":0.34999999999999998,"5min":0.312,"15min":0.26200000000000001},"max":{"1min":0.874,"5min":6.8890000000000002,"15min":77.793000000000006},"last":2.0910000000000002},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.58699999999999997,"15min":1.8919999999999999},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.409,"5min":6.0940000000000003,"15min":76.646000000000001},"last":0.35499999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.58399999999999996,"15min":1.9119999999999999},"min":{"1min":0.32500000000000001,"5min":0.314,"15min":0.27500000000000002},"max":{"1min":0.61399999999999999,"5min":4.3099999999999996,"15min":28.117999999999999},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51800000000000002,"15min":2.0310000000000001},"min":{"1min":0.33800000000000002,"5min":0.33600000000000002,"15min":0.27700000000000002},"max":{"1min":1.9930000000000001,"5min":2.512,"15min":28.34},"last":1.9930000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.55500000000000005,"15min":1.764},"min":{"1min":0.379,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":0.56899999999999995,"5min":2.9100000000000001,"15min":73.887},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.58499999999999996,"15min":1.8560000000000001},"min":{"1min":0.379,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":0.60499999999999998,"5min":2.6000000000000001,"15min":74.016999999999996},"last":0.498}]},{"osd":7,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.86299999999999999,"15min":1.7809999999999999},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":3.6480000000000001,"5min":7.0620000000000003,"15min":13.085000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.746,"5min":0.77200000000000002,"15min":1.633},"min":{"1min":0.38800000000000001,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.7120000000000002,"5min":4.1779999999999999,"15min":13.191000000000001},"last":0.48199999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019437,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3017756,"kb_used_data":2127572,"kb_used_omap":0,"kb_used_meta":890176,"kb_avail":90727396,"statfs":{"total":95995035648,"available":92904853504,"internally_reserved":0,"allocated":2178633728,"data_stored":2177693050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911540224},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.623,"15min":3.9359999999999999},"min":{"1min":0.22600000000000001,"5min":0.22500000000000001,"15min":0.21199999999999999},"max":{"1min":0.51400000000000001,"5min":9.6690000000000005,"15min":155.315},"last":41.963999999999999},{"interface":"front","average":{"1min":0.371,"5min":0.72499999999999998,"15min":4.2910000000000004},"min":{"1min":0.253,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.499,"5min":6.1459999999999999,"15min":155.334},"last":41.890999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.72399999999999998,"15min":5.2430000000000003},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":0.57799999999999996,"5min":5.944,"15min":190.61199999999999},"last":0.432},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.63400000000000001,"15min":5.9699999999999998},"min":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.25600000000000001},"max":{"1min":1.679,"5min":3.7469999999999999,"15min":253.273},"last":0.29199999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.74299999999999999,"15min":2.1150000000000002},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.1639999999999999,"5min":5.702,"15min":170.45400000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.748,"15min":1.889},"min":{"1min":0.38600000000000001,"5min":0.314,"15min":0.25800000000000001},"max":{"1min":2.121,"5min":5.6150000000000002,"15min":36.515999999999998},"last":0.39900000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.69699999999999995,"15min":2.125},"min":{"1min":0.29199999999999998,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":2.0379999999999998,"5min":5.5279999999999996,"15min":140.881},"last":0.307},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.68500000000000005,"15min":2.2749999999999999},"min":{"1min":0.32300000000000001,"5min":0.312,"15min":0.311},"max":{"1min":6.2969999999999997,"5min":6.2969999999999997,"15min":140.911},"last":0.33100000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:45:39 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.9279999999999999,"15min":1.4870000000000001},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":2.9649999999999999,"5min":110.003,"15min":110.003},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.62,"5min":1.8779999999999999,"15min":1.3959999999999999},"min":{"1min":0.34999999999999998,"5min":0.29899999999999999,"15min":0.28899999999999998},"max":{"1min":2.9910000000000001,"5min":108.56399999999999,"15min":108.56399999999999},"last":0.36599999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.623,"15min":0.91900000000000004},"min":{"1min":0.34599999999999997,"5min":0.28799999999999998,"15min":0.27200000000000002},"max":{"1min":0.63700000000000001,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.63400000000000001,"15min":1.0289999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28999999999999998},"max":{"1min":1.3340000000000001,"5min":4.0220000000000002,"15min":10.076000000000001},"last":0.47199999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.66800000000000004,"15min":1.139},"min":{"1min":0.33400000000000002,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":2.98,"5min":3.8580000000000001,"15min":38.034999999999997},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.91400000000000003,"15min":1.4350000000000001},"min":{"1min":0.40899999999999997,"5min":0.36699999999999999,"15min":0.308},"max":{"1min":3.113,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.49099999999999999}]}]},{"osd":5,"up_from":379,"seq":1627792605344,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1447112,"kb_used_data":1078464,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":92298040,"statfs":{"total":95995035648,"available":94513192960,"internally_reserved":0,"allocated":1104347136,"data_stored":1103422362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,7,2],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.749,"15min":2.0830000000000002},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.21099999999999999},"max":{"1min":0.77600000000000002,"5min":27.584,"15min":84.340999999999994},"last":0.377},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.76900000000000002,"15min":2.0369999999999999},"min":{"1min":0.28399999999999997,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.749,"5min":27.669,"15min":84.356999999999999},"last":0.45200000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.55200000000000005,"15min":5.5519999999999996},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.2130000000000001,"5min":4.3289999999999997,"15min":295.71300000000002},"last":0.44},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.51300000000000001,"15min":5.5609999999999999},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":0.70399999999999996,"5min":3.9580000000000002,"15min":299.59399999999999},"last":0.42399999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.52400000000000002,"15min":1.45},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78600000000000003,"5min":4.2670000000000003,"15min":14.199999999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.65100000000000002,"15min":1.4139999999999999},"min":{"1min":0.29599999999999999,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":4.2910000000000004,"5min":4.3719999999999999,"15min":14.430999999999999},"last":0.49399999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.56699999999999995,"15min":1.3720000000000001},"min":{"1min":0.32800000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.75,"5min":4.0209999999999999,"15min":14.004},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.5,"5min":0.58799999999999997,"15min":1.518},"min":{"1min":0.40100000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":0.83299999999999996,"5min":4.2910000000000004,"15min":13.991},"last":0.47999999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.53300000000000003,"15min":1.365},"min":{"1min":0.29499999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.0620000000000001,"5min":4.1929999999999996,"15min":122.971},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.623,"15min":1.5389999999999999},"min":{"1min":0.28899999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2619999999999996,"5min":4.3869999999999996,"15min":122.89100000000001},"last":0.47099999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.54900000000000004,"15min":0.95399999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":0.82799999999999996,"5min":4.3300000000000001,"15min":12.545},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.64100000000000001,"15min":1.1459999999999999},"min":{"1min":0.36499999999999999,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":0.81899999999999995,"5min":4.3200000000000003,"15min":14.193},"last":0.32800000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.68300000000000005,"15min":1.1499999999999999},"min":{"1min":0.35399999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.3170000000000002,"5min":4.3559999999999999,"15min":10.813000000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.67200000000000004,"15min":1.0469999999999999},"min":{"1min":0.39500000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":1.5369999999999999,"5min":4.2000000000000002,"15min":12.500999999999999},"last":0.46200000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2333937664,"data_stored":2333933592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":503316480,"data_stored":503316480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":505217024,"data_stored":505217024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2170949632,"data_stored":2170945560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1095434240,"data_stored":1095434240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1158352896,"data_stored":1158348824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":827588608,"data_stored":827588608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:46:40.239 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:40.296 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:40.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:40.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:40.403 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:40.438 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:40.502 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:40.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:40.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:40.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:40 smithi190 ceph-mon[160435]: from='client.37290 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:40.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:46:40.607 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:40.639 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:40.705 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:40 smithi138 ceph-mon[185886]: from='client.37290 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:40.705 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:40.705 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:46:40.705 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:40 smithi138 ceph-mon[192513]: from='client.37290 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:40.706 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:40.706 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:46:40.735 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:40.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:40.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:40.839 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:40.890 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:40.971 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:40.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:40.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:41.082 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:41.126 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:41.178 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:41.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:41.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:41.237 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:41.283 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:41.315 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:41.454 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:41 smithi190 ceph-mon[160435]: pgmap v1626: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 5887/121395 objects degraded (4.849%); 2445/121395 objects misplaced (2.014%); 985 KiB/s, 15 objects/s recovering 2023-10-01T09:46:41.454 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:41.454 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:41.486 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:41.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:41.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:41.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:41 smithi138 ceph-mon[185886]: pgmap v1626: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 5887/121395 objects degraded (4.849%); 2445/121395 objects misplaced (2.014%); 985 KiB/s, 15 objects/s recovering 2023-10-01T09:46:41.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:41.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:41 smithi138 ceph-mon[192513]: pgmap v1626: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 5887/121395 objects degraded (4.849%); 2445/121395 objects misplaced (2.014%); 985 KiB/s, 15 objects/s recovering 2023-10-01T09:46:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:41.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:41.590 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:41.624 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:41.734 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:41.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:41.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:41.839 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:41.870 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:42.014 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:46:42.014 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:46:42.469 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:46:42.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:42 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2713572926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:42.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:42 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:42 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2713572926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:42 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:42.581 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:42.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:42.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:42.688 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:42.731 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:42 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2713572926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:42.732 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:42 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:42.734 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:42.836 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:42.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:42.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:42.949 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:42.997 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:43.073 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:43.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:43.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:43.178 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:43.180 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:46:43.216 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:43.222 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:46:43.266 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:43.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:43.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:43.372 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:43.408 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:43.471 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:43.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:43.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:43 smithi190 ceph-mon[160435]: pgmap v1627: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 5887/119883 objects degraded (4.911%); 2445/119883 objects misplaced (2.039%); 985 KiB/s, 15 objects/s recovering 2023-10-01T09:46:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:43 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:43 smithi190 ceph-mon[160435]: failed to encode map e547 with expected crc 2023-10-01T09:46:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:43 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:46:43.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:43 smithi190 ceph-mon[160435]: osdmap e547: 8 total, 8 up, 7 in 2023-10-01T09:46:43.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:43 smithi190 ceph-mon[160435]: failed to encode map e547 with expected crc 2023-10-01T09:46:43.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:43 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5887/119883 objects degraded (4.911%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:43.580 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:43.622 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:43.678 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:43.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:43.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:43.744 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:43 smithi138 ceph-mon[192513]: pgmap v1627: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 5887/119883 objects degraded (4.911%); 2445/119883 objects misplaced (2.039%); 985 KiB/s, 15 objects/s recovering 2023-10-01T09:46:43.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:43 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:43.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:43 smithi138 ceph-mon[192513]: failed to encode map e547 with expected crc 2023-10-01T09:46:43.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:43 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:46:43.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:43 smithi138 ceph-mon[192513]: osdmap e547: 8 total, 8 up, 7 in 2023-10-01T09:46:43.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:43 smithi138 ceph-mon[192513]: failed to encode map e547 with expected crc 2023-10-01T09:46:43.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:43 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5887/119883 objects degraded (4.911%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:43.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:43 smithi138 ceph-mon[185886]: pgmap v1627: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 5887/119883 objects degraded (4.911%); 2445/119883 objects misplaced (2.039%); 985 KiB/s, 15 objects/s recovering 2023-10-01T09:46:43.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:43 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:43.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:43 smithi138 ceph-mon[185886]: failed to encode map e547 with expected crc 2023-10-01T09:46:43.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:43 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:46:43.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:43 smithi138 ceph-mon[185886]: osdmap e547: 8 total, 8 up, 7 in 2023-10-01T09:46:43.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:43 smithi138 ceph-mon[185886]: failed to encode map e547 with expected crc 2023-10-01T09:46:43.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:43 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5887/119883 objects degraded (4.911%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:43.784 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:43.817 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:44.070 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:44.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:44.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:44.170 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:44.174 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:44.217 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:44.303 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:46:44.329 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:46:44.335 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:44.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:44.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:44.357 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:46:44.360 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:44.383 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:46:44.408 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:46:44.426 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:46:44.441 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:44.445 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:46:44.464 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:46:44.465 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:46:44.465 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:46:44.465 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:46:44.465 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:46:44.466 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:46:44.466 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:46:44.466 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:46:44.466 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:46:44.469 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:44.516 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:44.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:44.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:44.623 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:44.649 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:44.702 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:44.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:44.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:44.805 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:44 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/556293524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:44.805 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:44 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/556293524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:44.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:44 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/556293524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:44.808 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:46:44.842 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:45.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:46:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:46:44] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-01T09:46:45.775 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:46:45.788 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:45 smithi138 ceph-mon[185886]: pgmap v1629: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 196 op/s; 5798/119088 objects degraded (4.869%); 2396/119088 objects misplaced (2.012%); 966 KiB/s, 15 objects/s recovering 2023-10-01T09:46:45.788 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:45 smithi138 ceph-mon[192513]: pgmap v1629: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 196 op/s; 5798/119088 objects degraded (4.869%); 2396/119088 objects misplaced (2.012%); 966 KiB/s, 15 objects/s recovering 2023-10-01T09:46:45.795 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:46:45.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:45.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:46:45.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:45 smithi190 ceph-mon[160435]: pgmap v1629: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 196 op/s; 5798/119088 objects degraded (4.869%); 2396/119088 objects misplaced (2.012%); 966 KiB/s, 15 objects/s recovering 2023-10-01T09:46:45.899 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:45.925 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:46.029 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:46.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:46.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:46.136 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:46:46.166 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:46.847 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:46:46.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:46.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:46:46.955 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:46.982 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:46.986 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:47.035 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:47.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:47.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:47.141 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:47.179 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:47.386 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:47.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:47.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:47.464 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:47 smithi138 ceph-mon[185886]: pgmap v1630: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 194 op/s; 5798/117636 objects degraded (4.929%); 2396/117636 objects misplaced (2.037%); 966 KiB/s, 15 objects/s recovering 2023-10-01T09:46:47.487 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:46:47.487 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:46:47.487 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:46:47.487 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:46:47.488 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.488 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:147923 9.a 9:5afed6f2:::benchmark_data_smithi138_497829_object45538:head [delete] snapc 0=[] ondisk+write+known_if_redirected e444)", 2023-10-01T09:46:47.488 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:22.282541+0000", 2023-10-01T09:46:47.488 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 565.20082750899996, 2023-10-01T09:46:47.489 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.55728740300000001, 2023-10-01T09:46:47.489 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.489 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.489 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.489 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:46:47.489 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:46:47.490 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 147923 2023-10-01T09:46:47.490 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.490 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.490 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282541+0000", 2023-10-01T09:46:47.491 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.491 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.491 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.491 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.491 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282541+0000", 2023-10-01T09:46:47.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.492 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.492 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.492 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282541+0000", 2023-10-01T09:46:47.492 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:46:47.492 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.493 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.493 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282546+0000", 2023-10-01T09:46:47.493 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.972e-06 2023-10-01T09:46:47.493 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.493 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.494 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282560+0000", 2023-10-01T09:46:47.494 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4630999999999999e-05 2023-10-01T09:46:47.494 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.494 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.494 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.282567+0000", 2023-10-01T09:46:47.495 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3010000000000001e-06 2023-10-01T09:46:47.495 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.495 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.495 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.495 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.285815+0000", 2023-10-01T09:46:47.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003247963 2023-10-01T09:46:47.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.496 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.496 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:46:47.496 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289067+0000", 2023-10-01T09:46:47.496 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032514169999999999 2023-10-01T09:46:47.496 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.307845+0000", 2023-10-01T09:46:47.497 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.018777933 2023-10-01T09:46:47.497 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.497 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.497 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:46:47.497 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.307857+0000", 2023-10-01T09:46:47.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.254e-05 2023-10-01T09:46:47.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.498 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.498 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.498 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.805080+0000", 2023-10-01T09:46:47.498 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.49722238000000002 2023-10-01T09:46:47.498 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for degraded object", 2023-10-01T09:46:47.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.805434+0000", 2023-10-01T09:46:47.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00035419299999999999 2023-10-01T09:46:47.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.499 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.499 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.815559+0000", 2023-10-01T09:46:47.499 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010125265 2023-10-01T09:46:47.499 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.499 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.815641+0000", 2023-10-01T09:46:47.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1499e-05 2023-10-01T09:46:47.500 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.500 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.500 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:46:47.500 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.815740+0000", 2023-10-01T09:46:47.500 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9504000000000001e-05 2023-10-01T09:46:47.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.827989+0000", 2023-10-01T09:46:47.501 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01224926 2023-10-01T09:46:47.501 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.501 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.501 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.501 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.828043+0000", 2023-10-01T09:46:47.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3876999999999999e-05 2023-10-01T09:46:47.502 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.502 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.502 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.502 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.836681+0000", 2023-10-01T09:46:47.502 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0086381300000000008 2023-10-01T09:46:47.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.836733+0000", 2023-10-01T09:46:47.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1399e-05 2023-10-01T09:46:47.503 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.503 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.503 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.503 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.839829+0000", 2023-10-01T09:46:47.503 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030958539999999999 2023-10-01T09:46:47.504 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.504 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.504 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.504 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.504 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.504 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:98428 10.2 10:465b81c4:::benchmark_data_smithi138_573643_object15206:head [delete] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-01T09:46:47.504 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:43:45.525522+0000", 2023-10-01T09:46:47.504 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 181.95784652200001, 2023-10-01T09:46:47.504 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010810827, 2023-10-01T09:46:47.505 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.505 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.505 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.505 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.505 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.505 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 98428 2023-10-01T09:46:47.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.506 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525522+0000", 2023-10-01T09:46:47.506 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.506 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.506 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.506 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.506 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525522+0000", 2023-10-01T09:46:47.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525521+0000", 2023-10-01T09:46:47.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:46:47.507 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.507 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.507 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.507 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525527+0000", 2023-10-01T09:46:47.507 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9970000000000004e-06 2023-10-01T09:46:47.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525540+0000", 2023-10-01T09:46:47.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.344e-05 2023-10-01T09:46:47.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.508 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.508 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525550+0000", 2023-10-01T09:46:47.508 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6409999999999997e-06 2023-10-01T09:46:47.508 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.508 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525599+0000", 2023-10-01T09:46:47.509 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9360999999999997e-05 2023-10-01T09:46:47.509 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.509 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.509 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.509 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525771+0000", 2023-10-01T09:46:47.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000171807 2023-10-01T09:46:47.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.510 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:46:47.510 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.525859+0000", 2023-10-01T09:46:47.510 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7646999999999996e-05 2023-10-01T09:46:47.510 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.510 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.511 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.511 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.529423+0000", 2023-10-01T09:46:47.511 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035644040000000002 2023-10-01T09:46:47.511 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.511 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.533158+0000", 2023-10-01T09:46:47.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0037345740000000001 2023-10-01T09:46:47.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.512 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.512 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.512 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.536237+0000", 2023-10-01T09:46:47.512 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003079217 2023-10-01T09:46:47.512 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.536292+0000", 2023-10-01T09:46:47.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4551e-05 2023-10-01T09:46:47.513 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.513 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.513 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.513 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:45.536333+0000", 2023-10-01T09:46:47.513 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1378000000000003e-05 2023-10-01T09:46:47.514 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.514 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.514 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.514 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.514 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.514 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:98554 10.2 10:449bd2ed:::benchmark_data_smithi138_573643_object15332:head [delete] snapc 0=[] ondisk+write+known_if_redirected e524)", 2023-10-01T09:46:47.514 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:43:46.357532+0000", 2023-10-01T09:46:47.514 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 181.125836932, 2023-10-01T09:46:47.514 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.079442952999999997, 2023-10-01T09:46:47.515 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.515 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.515 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.515 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.515 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.515 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 98554 2023-10-01T09:46:47.515 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.515 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.515 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.515 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357532+0000", 2023-10-01T09:46:47.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.516 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.516 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.516 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357532+0000", 2023-10-01T09:46:47.516 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.516 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.517 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.517 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357530+0000", 2023-10-01T09:46:47.517 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:47.517 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.517 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.518 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357537+0000", 2023-10-01T09:46:47.518 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.2639999999999997e-06 2023-10-01T09:46:47.518 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.518 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.518 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.519 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357548+0000", 2023-10-01T09:46:47.519 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1671000000000001e-05 2023-10-01T09:46:47.519 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.519 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.519 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357556+0000", 2023-10-01T09:46:47.520 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3769999999999998e-06 2023-10-01T09:46:47.520 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.520 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.520 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.520 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357594+0000", 2023-10-01T09:46:47.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.8191000000000001e-05 2023-10-01T09:46:47.521 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.521 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.521 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.521 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357675+0000", 2023-10-01T09:46:47.521 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0648000000000004e-05 2023-10-01T09:46:47.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.522 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.522 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:46:47.522 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.357738+0000", 2023-10-01T09:46:47.522 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3931999999999997e-05 2023-10-01T09:46:47.522 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.523 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.523 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.360298+0000", 2023-10-01T09:46:47.523 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025591089999999999 2023-10-01T09:46:47.523 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.523 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.524 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.524 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.361416+0000", 2023-10-01T09:46:47.524 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001118747 2023-10-01T09:46:47.524 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.524 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.525 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.436873+0000", 2023-10-01T09:46:47.525 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.075457065000000004 2023-10-01T09:46:47.525 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.525 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.525 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.526 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.436928+0000", 2023-10-01T09:46:47.526 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4527999999999997e-05 2023-10-01T09:46:47.526 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.526 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.526 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.527 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:46.436975+0000", 2023-10-01T09:46:47.527 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6888000000000002e-05 2023-10-01T09:46:47.527 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.527 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.527 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.527 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.528 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.528 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:103729 10.b 10:db22441b:::benchmark_data_smithi138_573643_object20507:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:46:47.528 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:44:26.352243+0000", 2023-10-01T09:46:47.528 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 141.13112557700001, 2023-10-01T09:46:47.528 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011887605000000001, 2023-10-01T09:46:47.529 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.529 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.529 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.529 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.529 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.529 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 103729 2023-10-01T09:46:47.530 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.530 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.530 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.530 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.530 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352243+0000", 2023-10-01T09:46:47.530 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.531 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352243+0000", 2023-10-01T09:46:47.531 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.531 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.531 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.531 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352241+0000", 2023-10-01T09:46:47.532 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:46:47.532 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.532 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.532 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.532 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352252+0000", 2023-10-01T09:46:47.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.096e-05 2023-10-01T09:46:47.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.533 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.533 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352275+0000", 2023-10-01T09:46:47.533 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2866000000000001e-05 2023-10-01T09:46:47.533 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.533 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.534 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352286+0000", 2023-10-01T09:46:47.534 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.111e-05 2023-10-01T09:46:47.534 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.534 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.534 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.535 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352352+0000", 2023-10-01T09:46:47.535 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6421999999999995e-05 2023-10-01T09:46:47.535 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.535 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.535 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352529+0000", 2023-10-01T09:46:47.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000176742 2023-10-01T09:46:47.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.536 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.536 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:46:47.536 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.352672+0000", 2023-10-01T09:46:47.536 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014320399999999999 2023-10-01T09:46:47.536 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.356654+0000", 2023-10-01T09:46:47.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039821409999999998 2023-10-01T09:46:47.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.537 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.537 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.361739+0000", 2023-10-01T09:46:47.537 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050842980000000001 2023-10-01T09:46:47.537 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.537 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.364028+0000", 2023-10-01T09:46:47.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00228911 2023-10-01T09:46:47.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.538 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.364086+0000", 2023-10-01T09:46:47.538 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8260000000000001e-05 2023-10-01T09:46:47.538 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.538 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.538 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.539 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:26.364131+0000", 2023-10-01T09:46:47.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4733e-05 2023-10-01T09:46:47.539 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.539 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.539 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.539 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.539 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.539 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:103953 10.2 10:414c62ad:::benchmark_data_smithi138_573643_object20731:head [delete] snapc 0=[] ondisk+write+known_if_redirected e529)", 2023-10-01T09:46:47.539 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:44:27.617662+0000", 2023-10-01T09:46:47.539 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 139.86570634700001, 2023-10-01T09:46:47.539 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.121890762, 2023-10-01T09:46:47.540 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.540 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.540 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.540 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 103953 2023-10-01T09:46:47.540 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.540 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.540 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.540 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.540 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617662+0000", 2023-10-01T09:46:47.540 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617662+0000", 2023-10-01T09:46:47.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.541 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.541 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.541 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.541 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617661+0000", 2023-10-01T09:46:47.541 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:47.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617669+0000", 2023-10-01T09:46:47.542 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1100000000000003e-06 2023-10-01T09:46:47.542 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.542 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.542 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.542 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617684+0000", 2023-10-01T09:46:47.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4935e-05 2023-10-01T09:46:47.543 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.543 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.543 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.543 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617691+0000", 2023-10-01T09:46:47.543 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0480000000000001e-06 2023-10-01T09:46:47.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.544 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.544 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.544 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617736+0000", 2023-10-01T09:46:47.544 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4866e-05 2023-10-01T09:46:47.544 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.545 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.545 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617830+0000", 2023-10-01T09:46:47.545 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4351999999999994e-05 2023-10-01T09:46:47.545 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.545 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:46:47.546 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.617902+0000", 2023-10-01T09:46:47.546 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2039000000000006e-05 2023-10-01T09:46:47.546 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.546 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.546 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.620668+0000", 2023-10-01T09:46:47.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0027652670000000001 2023-10-01T09:46:47.547 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.547 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.547 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.547 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.620958+0000", 2023-10-01T09:46:47.547 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00029073499999999997 2023-10-01T09:46:47.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.548 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.548 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.739468+0000", 2023-10-01T09:46:47.548 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.11850937 2023-10-01T09:46:47.548 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.548 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.739506+0000", 2023-10-01T09:46:47.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7892999999999998e-05 2023-10-01T09:46:47.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.549 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.549 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.549 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:44:27.739553+0000", 2023-10-01T09:46:47.549 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7629000000000001e-05 2023-10-01T09:46:47.549 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.549 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.549 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.549 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.550 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.550 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:110972 10.b 10:d168e84a:::benchmark_data_smithi138_573643_object27750:head [delete] snapc 0=[] ondisk+write+known_if_redirected e535)", 2023-10-01T09:46:47.550 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:45:12.872479+0000", 2023-10-01T09:46:47.550 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 94.610889767000003, 2023-10-01T09:46:47.550 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.088036003000000002, 2023-10-01T09:46:47.550 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.550 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.550 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.550 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.550 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.551 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 110972 2023-10-01T09:46:47.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.551 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:12.872479+0000", 2023-10-01T09:46:47.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.551 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.551 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.551 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:12.872479+0000", 2023-10-01T09:46:47.551 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.551 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:12.872478+0000", 2023-10-01T09:46:47.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:47.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.552 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:12.872484+0000", 2023-10-01T09:46:47.552 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5289999999999997e-06 2023-10-01T09:46:47.552 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.552 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.552 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:12.872498+0000", 2023-10-01T09:46:47.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3553000000000001e-05 2023-10-01T09:46:47.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:12.872507+0000", 2023-10-01T09:46:47.553 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3440000000000007e-06 2023-10-01T09:46:47.553 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.553 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.553 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.553 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:12.872573+0000", 2023-10-01T09:46:47.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6020999999999997e-05 2023-10-01T09:46:47.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:12.872660+0000", 2023-10-01T09:46:47.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6507000000000004e-05 2023-10-01T09:46:47.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.554 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.554 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:46:47.554 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:12.872727+0000", 2023-10-01T09:46:47.554 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7231999999999996e-05 2023-10-01T09:46:47.554 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:12.875538+0000", 2023-10-01T09:46:47.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028111339999999999 2023-10-01T09:46:47.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.555 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:12.876066+0000", 2023-10-01T09:46:47.555 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00052843499999999995 2023-10-01T09:46:47.555 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.555 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.555 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:12.960419+0000", 2023-10-01T09:46:47.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.084352493000000001 2023-10-01T09:46:47.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:12.960472+0000", 2023-10-01T09:46:47.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.3403999999999999e-05 2023-10-01T09:46:47.556 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.556 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.556 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.556 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:12.960515+0000", 2023-10-01T09:46:47.556 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2660000000000002e-05 2023-10-01T09:46:47.557 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.557 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.557 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.557 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.557 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.557 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:111375 10.b 10:dd60c1ba:::benchmark_data_smithi138_573643_object28153:head [delete] snapc 0=[] ondisk+write+known_if_redirected e535)", 2023-10-01T09:46:47.557 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:45:14.892184+0000", 2023-10-01T09:46:47.557 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 92.591184765999998, 2023-10-01T09:46:47.557 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.046499048000000001, 2023-10-01T09:46:47.558 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.558 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.558 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.558 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.558 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.558 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 111375 2023-10-01T09:46:47.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.559 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.559 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.559 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.559 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:14.892184+0000", 2023-10-01T09:46:47.559 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.559 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.560 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:14.892184+0000", 2023-10-01T09:46:47.560 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.560 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.560 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.560 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.561 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:14.892183+0000", 2023-10-01T09:46:47.561 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:47.561 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.561 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.561 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:14.892190+0000", 2023-10-01T09:46:47.562 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.08e-06 2023-10-01T09:46:47.562 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.562 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.562 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.562 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:14.892213+0000", 2023-10-01T09:46:47.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3819999999999999e-05 2023-10-01T09:46:47.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.563 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.563 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.563 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:14.892221+0000", 2023-10-01T09:46:47.563 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.7090000000000002e-06 2023-10-01T09:46:47.563 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.564 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:14.892265+0000", 2023-10-01T09:46:47.564 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3336e-05 2023-10-01T09:46:47.564 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.564 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.564 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:14.892350+0000", 2023-10-01T09:46:47.565 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5458999999999996e-05 2023-10-01T09:46:47.565 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.565 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.565 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:46:47.565 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:14.892417+0000", 2023-10-01T09:46:47.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.7045000000000006e-05 2023-10-01T09:46:47.566 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.566 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.566 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.566 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:14.895236+0000", 2023-10-01T09:46:47.566 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0028186330000000001 2023-10-01T09:46:47.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.567 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.567 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:14.895376+0000", 2023-10-01T09:46:47.567 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014062600000000001 2023-10-01T09:46:47.567 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.567 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.568 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:14.938611+0000", 2023-10-01T09:46:47.568 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.043234795999999999 2023-10-01T09:46:47.568 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.568 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.568 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:14.938652+0000", 2023-10-01T09:46:47.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.1017999999999999e-05 2023-10-01T09:46:47.569 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.569 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.569 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.569 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:14.938683+0000", 2023-10-01T09:46:47.569 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0939000000000003e-05 2023-10-01T09:46:47.570 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.570 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.570 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.570 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.570 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.570 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:114298 10.b 10:debf9cf0:::benchmark_data_smithi138_573643_object31076:head [delete] snapc 0=[] ondisk+write+known_if_redirected e537)", 2023-10-01T09:46:47.570 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:45:32.769568+0000", 2023-10-01T09:46:47.571 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 74.713800601000003, 2023-10-01T09:46:47.571 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012212979000000001, 2023-10-01T09:46:47.571 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.571 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.571 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.571 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.571 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.571 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 114298 2023-10-01T09:46:47.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.572 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.572 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.572 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.572 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:32.769568+0000", 2023-10-01T09:46:47.572 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.572 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.573 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:32.769568+0000", 2023-10-01T09:46:47.573 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.573 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.573 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.573 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:32.769566+0000", 2023-10-01T09:46:47.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:46:47.574 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.574 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.574 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.574 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:32.769577+0000", 2023-10-01T09:46:47.574 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0152999999999999e-05 2023-10-01T09:46:47.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.575 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.575 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:32.769598+0000", 2023-10-01T09:46:47.575 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1423000000000001e-05 2023-10-01T09:46:47.575 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.575 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.576 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:32.769608+0000", 2023-10-01T09:46:47.576 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0202e-05 2023-10-01T09:46:47.576 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.576 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.576 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:32.769666+0000", 2023-10-01T09:46:47.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7438000000000001e-05 2023-10-01T09:46:47.577 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.577 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.577 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.577 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:32.769773+0000", 2023-10-01T09:46:47.577 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010772600000000001 2023-10-01T09:46:47.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:46:47.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:32.769854+0000", 2023-10-01T09:46:47.578 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1064999999999997e-05 2023-10-01T09:46:47.578 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.578 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.578 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.578 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:32.773776+0000", 2023-10-01T09:46:47.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039216119999999997 2023-10-01T09:46:47.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.579 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.579 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:32.773996+0000", 2023-10-01T09:46:47.579 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021995599999999999 2023-10-01T09:46:47.579 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.579 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:32.781615+0000", 2023-10-01T09:46:47.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0076192170000000002 2023-10-01T09:46:47.580 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.580 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.580 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.580 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:32.781708+0000", 2023-10-01T09:46:47.580 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.2571000000000005e-05 2023-10-01T09:46:47.581 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.581 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.581 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.581 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:32.781781+0000", 2023-10-01T09:46:47.581 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3425999999999998e-05 2023-10-01T09:46:47.581 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.581 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.582 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.582 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.582 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.582 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:114875 10.b 10:d4fa2f0f:::benchmark_data_smithi138_573643_object31653:head [delete] snapc 0=[] ondisk+write+known_if_redirected e538)", 2023-10-01T09:46:47.582 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:45:36.327665+0000", 2023-10-01T09:46:47.582 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 71.155703324000001, 2023-10-01T09:46:47.583 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011791474, 2023-10-01T09:46:47.583 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.583 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.583 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.583 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.583 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.583 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 114875 2023-10-01T09:46:47.583 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.584 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:36.327665+0000", 2023-10-01T09:46:47.584 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.584 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.584 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.584 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.584 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:36.327665+0000", 2023-10-01T09:46:47.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.585 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.585 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.585 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:36.327664+0000", 2023-10-01T09:46:47.585 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:47.585 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.586 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:36.327672+0000", 2023-10-01T09:46:47.586 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0749999999999998e-06 2023-10-01T09:46:47.586 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.586 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.586 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.587 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:36.327688+0000", 2023-10-01T09:46:47.587 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5863999999999999e-05 2023-10-01T09:46:47.587 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.587 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:36.327696+0000", 2023-10-01T09:46:47.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6780000000000006e-06 2023-10-01T09:46:47.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.588 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.588 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.588 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:36.327776+0000", 2023-10-01T09:46:47.588 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9907000000000006e-05 2023-10-01T09:46:47.588 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:36.327928+0000", 2023-10-01T09:46:47.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000151779 2023-10-01T09:46:47.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.589 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.589 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:46:47.589 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:36.328056+0000", 2023-10-01T09:46:47.589 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012831600000000001 2023-10-01T09:46:47.589 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:36.332060+0000", 2023-10-01T09:46:47.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040032150000000001 2023-10-01T09:46:47.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.590 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.590 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.590 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:36.335733+0000", 2023-10-01T09:46:47.590 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036733960000000002 2023-10-01T09:46:47.590 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:36.339370+0000", 2023-10-01T09:46:47.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0036367729999999998 2023-10-01T09:46:47.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.591 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.591 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:36.339412+0000", 2023-10-01T09:46:47.591 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2315999999999999e-05 2023-10-01T09:46:47.591 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.591 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.592 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.592 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:36.339457+0000", 2023-10-01T09:46:47.592 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4749999999999997e-05 2023-10-01T09:46:47.592 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.592 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.592 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.592 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.592 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.592 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:116408 10.b 10:dfccc8ab:::benchmark_data_smithi138_573643_object33186:head [delete] snapc 0=[] ondisk+write+known_if_redirected e539)", 2023-10-01T09:46:47.592 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:45:44.859812+0000", 2023-10-01T09:46:47.592 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 62.623556907000001, 2023-10-01T09:46:47.593 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.051431136000000002, 2023-10-01T09:46:47.593 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.593 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.593 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.593 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.593 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.593 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 116408 2023-10-01T09:46:47.593 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.593 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.593 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.593 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.593 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.859812+0000", 2023-10-01T09:46:47.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.594 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.859812+0000", 2023-10-01T09:46:47.594 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.594 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.594 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.594 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.859811+0000", 2023-10-01T09:46:47.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:46:47.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.859817+0000", 2023-10-01T09:46:47.595 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0100000000000001e-06 2023-10-01T09:46:47.595 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.595 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.595 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.595 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.859826+0000", 2023-10-01T09:46:47.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3649999999999993e-06 2023-10-01T09:46:47.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.859832+0000", 2023-10-01T09:46:47.596 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7919999999999998e-06 2023-10-01T09:46:47.596 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.596 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.596 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.596 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.859875+0000", 2023-10-01T09:46:47.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2756000000000002e-05 2023-10-01T09:46:47.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.859977+0000", 2023-10-01T09:46:47.597 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000102698 2023-10-01T09:46:47.597 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.597 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.597 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:46:47.597 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.860051+0000", 2023-10-01T09:46:47.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4138000000000001e-05 2023-10-01T09:46:47.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.863075+0000", 2023-10-01T09:46:47.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0030234260000000001 2023-10-01T09:46:47.598 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.598 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.598 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.598 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.863404+0000", 2023-10-01T09:46:47.598 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00032933099999999998 2023-10-01T09:46:47.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.911170+0000", 2023-10-01T09:46:47.599 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.047766173000000002 2023-10-01T09:46:47.599 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.599 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.599 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.599 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.911218+0000", 2023-10-01T09:46:47.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8068000000000001e-05 2023-10-01T09:46:47.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.600 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.600 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.911243+0000", 2023-10-01T09:46:47.600 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4522e-05 2023-10-01T09:46:47.600 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.600 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.600 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.600 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.600 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.601 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:116410 10.b 10:d089ea37:::benchmark_data_smithi138_573643_object33188:head [delete] snapc 0=[] ondisk+write+known_if_redirected e539)", 2023-10-01T09:46:47.601 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:45:44.870756+0000", 2023-10-01T09:46:47.601 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 62.612612566000003, 2023-10-01T09:46:47.601 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.048656181, 2023-10-01T09:46:47.601 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.601 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.601 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.601 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.602 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.602 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 116410 2023-10-01T09:46:47.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.602 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.602 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.602 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.870756+0000", 2023-10-01T09:46:47.602 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.602 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.602 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.870756+0000", 2023-10-01T09:46:47.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.603 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.870755+0000", 2023-10-01T09:46:47.603 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:46:47.603 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.603 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.603 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.870760+0000", 2023-10-01T09:46:47.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1460000000000004e-06 2023-10-01T09:46:47.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.604 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.604 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.870772+0000", 2023-10-01T09:46:47.604 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1919999999999999e-05 2023-10-01T09:46:47.604 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.604 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.870778+0000", 2023-10-01T09:46:47.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5099999999999998e-06 2023-10-01T09:46:47.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.605 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.605 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.605 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.870821+0000", 2023-10-01T09:46:47.605 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.2781999999999998e-05 2023-10-01T09:46:47.605 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.870914+0000", 2023-10-01T09:46:47.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3734999999999996e-05 2023-10-01T09:46:47.606 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.606 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.606 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:46:47.606 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.870987+0000", 2023-10-01T09:46:47.606 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2590999999999997e-05 2023-10-01T09:46:47.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.874283+0000", 2023-10-01T09:46:47.607 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032957519999999999 2023-10-01T09:46:47.607 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.607 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.607 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.607 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.874397+0000", 2023-10-01T09:46:47.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000114302 2023-10-01T09:46:47.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.919352+0000", 2023-10-01T09:46:47.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.044955465999999999 2023-10-01T09:46:47.608 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.608 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.608 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.608 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.919385+0000", 2023-10-01T09:46:47.608 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2806999999999999e-05 2023-10-01T09:46:47.609 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.609 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.609 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.609 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.919412+0000", 2023-10-01T09:46:47.609 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7229e-05 2023-10-01T09:46:47.609 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.609 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.609 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.610 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.610 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.610 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:116411 10.b 10:d2eb9956:::benchmark_data_smithi138_573643_object33189:head [delete] snapc 0=[] ondisk+write+known_if_redirected e539)", 2023-10-01T09:46:47.610 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:45:44.887740+0000", 2023-10-01T09:46:47.610 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 62.595629013999996, 2023-10-01T09:46:47.610 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.033475440000000002, 2023-10-01T09:46:47.610 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.610 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.610 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.610 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.610 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.610 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 116411 2023-10-01T09:46:47.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.611 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.887740+0000", 2023-10-01T09:46:47.611 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.611 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.611 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.611 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.611 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.887740+0000", 2023-10-01T09:46:47.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.887739+0000", 2023-10-01T09:46:47.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:46:47.612 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.612 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.612 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.612 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.887744+0000", 2023-10-01T09:46:47.612 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4989999999999996e-06 2023-10-01T09:46:47.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.887755+0000", 2023-10-01T09:46:47.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1599e-05 2023-10-01T09:46:47.613 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.613 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.613 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.613 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.887761+0000", 2023-10-01T09:46:47.613 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4620000000000004e-06 2023-10-01T09:46:47.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.887794+0000", 2023-10-01T09:46:47.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.3788000000000002e-05 2023-10-01T09:46:47.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.614 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.614 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.614 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.887881+0000", 2023-10-01T09:46:47.614 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.6360000000000007e-05 2023-10-01T09:46:47.614 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:46:47.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.887951+0000", 2023-10-01T09:46:47.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0241999999999996e-05 2023-10-01T09:46:47.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.615 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.615 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.889866+0000", 2023-10-01T09:46:47.615 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0019151699999999999 2023-10-01T09:46:47.615 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.615 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.891392+0000", 2023-10-01T09:46:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0015259399999999999 2023-10-01T09:46:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.921152+0000", 2023-10-01T09:46:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.029760205000000001 2023-10-01T09:46:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.616 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.921184+0000", 2023-10-01T09:46:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1872999999999998e-05 2023-10-01T09:46:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:45:44.921215+0000", 2023-10-01T09:46:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1031999999999997e-05 2023-10-01T09:46:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.617 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.618 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.618 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.618 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:119320 10.b 10:d96d31f8:::benchmark_data_smithi138_573643_object36098:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-01T09:46:47.618 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:46:02.250018+0000", 2023-10-01T09:46:47.618 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 45.233350753000003, 2023-10-01T09:46:47.618 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014390165999999999, 2023-10-01T09:46:47.618 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.618 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.618 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.618 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.619 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.619 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 119320 2023-10-01T09:46:47.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.619 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.619 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.250018+0000", 2023-10-01T09:46:47.619 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.619 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.619 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.619 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.250018+0000", 2023-10-01T09:46:47.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.620 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.620 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.250017+0000", 2023-10-01T09:46:47.620 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:46:47.620 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.620 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.621 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.250022+0000", 2023-10-01T09:46:47.621 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8319999999999996e-06 2023-10-01T09:46:47.621 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.621 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.621 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.250032+0000", 2023-10-01T09:46:47.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9480000000000003e-06 2023-10-01T09:46:47.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.622 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.250039+0000", 2023-10-01T09:46:47.622 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6309999999999997e-06 2023-10-01T09:46:47.622 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.622 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.622 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.253816+0000", 2023-10-01T09:46:47.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00377729 2023-10-01T09:46:47.623 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.623 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.623 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.623 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.253981+0000", 2023-10-01T09:46:47.623 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016522299999999999 2023-10-01T09:46:47.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:46:47.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.254108+0000", 2023-10-01T09:46:47.624 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012660899999999999 2023-10-01T09:46:47.624 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.624 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.624 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.624 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.258211+0000", 2023-10-01T09:46:47.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041032509999999996 2023-10-01T09:46:47.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.625 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.258445+0000", 2023-10-01T09:46:47.625 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000233809 2023-10-01T09:46:47.625 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.625 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.625 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.264311+0000", 2023-10-01T09:46:47.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0058664310000000001 2023-10-01T09:46:47.626 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.626 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.626 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.626 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.264364+0000", 2023-10-01T09:46:47.626 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2278000000000003e-05 2023-10-01T09:46:47.627 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.627 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.627 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.627 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.264408+0000", 2023-10-01T09:46:47.627 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4629999999999998e-05 2023-10-01T09:46:47.628 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.628 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.628 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.628 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.628 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.628 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:119408 10.b 10:dc4d6105:::benchmark_data_smithi138_573643_object36186:head [delete] snapc 0=[] ondisk+write+known_if_redirected e541)", 2023-10-01T09:46:47.628 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:46:02.962422+0000", 2023-10-01T09:46:47.628 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 44.520946383000002, 2023-10-01T09:46:47.629 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013581058, 2023-10-01T09:46:47.629 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.629 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.629 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.629 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.630 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.630 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 119408 2023-10-01T09:46:47.630 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.630 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.630 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.630 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.630 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.962422+0000", 2023-10-01T09:46:47.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.631 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.631 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.631 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.631 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.962422+0000", 2023-10-01T09:46:47.631 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.632 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.632 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.632 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.962421+0000", 2023-10-01T09:46:47.632 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:47.632 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.633 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.962428+0000", 2023-10-01T09:46:47.633 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.404e-06 2023-10-01T09:46:47.633 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.633 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.633 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.962441+0000", 2023-10-01T09:46:47.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3101999999999999e-05 2023-10-01T09:46:47.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.634 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.634 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.634 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.962449+0000", 2023-10-01T09:46:47.634 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0269999999999995e-06 2023-10-01T09:46:47.634 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.962508+0000", 2023-10-01T09:46:47.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9707999999999998e-05 2023-10-01T09:46:47.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.635 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.962659+0000", 2023-10-01T09:46:47.635 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000150898 2023-10-01T09:46:47.635 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.635 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.635 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:46:47.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.962801+0000", 2023-10-01T09:46:47.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014183800000000001 2023-10-01T09:46:47.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.966986+0000", 2023-10-01T09:46:47.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0041847170000000001 2023-10-01T09:46:47.636 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.636 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.636 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.636 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.974902+0000", 2023-10-01T09:46:47.636 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0079164649999999993 2023-10-01T09:46:47.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.975921+0000", 2023-10-01T09:46:47.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001018489 2023-10-01T09:46:47.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.637 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.637 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.637 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.975967+0000", 2023-10-01T09:46:47.637 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5883000000000001e-05 2023-10-01T09:46:47.637 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.638 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.638 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:02.976003+0000", 2023-10-01T09:46:47.638 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6748000000000001e-05 2023-10-01T09:46:47.638 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.638 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.638 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.638 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.638 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.638 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:125016 10.b 10:d23f23af:::benchmark_data_smithi138_573643_object41794:head [delete] snapc 0=[] ondisk+write+known_if_redirected e545)", 2023-10-01T09:46:47.638 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:46:32.365850+0000", 2023-10-01T09:46:47.638 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 15.117518808, 2023-10-01T09:46:47.639 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011544775, 2023-10-01T09:46:47.639 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.639 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.639 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.639 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.639 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.639 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 125016 2023-10-01T09:46:47.639 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.639 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.365850+0000", 2023-10-01T09:46:47.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.640 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.640 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.640 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.365850+0000", 2023-10-01T09:46:47.640 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.640 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.641 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.641 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.365848+0000", 2023-10-01T09:46:47.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:46:47.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.365858+0000", 2023-10-01T09:46:47.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6679999999999996e-06 2023-10-01T09:46:47.642 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.642 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.642 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.642 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.365880+0000", 2023-10-01T09:46:47.642 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2138e-05 2023-10-01T09:46:47.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.365890+0000", 2023-10-01T09:46:47.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0022e-05 2023-10-01T09:46:47.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.643 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.643 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.365937+0000", 2023-10-01T09:46:47.643 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7012999999999999e-05 2023-10-01T09:46:47.643 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.643 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.366093+0000", 2023-10-01T09:46:47.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000156241 2023-10-01T09:46:47.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:46:47.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.366175+0000", 2023-10-01T09:46:47.644 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.2062000000000001e-05 2023-10-01T09:46:47.644 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.644 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.644 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.644 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.374341+0000", 2023-10-01T09:46:47.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0081659360000000004 2023-10-01T09:46:47.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.374359+0000", 2023-10-01T09:46:47.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8093000000000001e-05 2023-10-01T09:46:47.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.645 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.645 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.645 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.377286+0000", 2023-10-01T09:46:47.645 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0029267429999999999 2023-10-01T09:46:47.645 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.377347+0000", 2023-10-01T09:46:47.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1026999999999997e-05 2023-10-01T09:46:47.646 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.646 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.646 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.646 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.377395+0000", 2023-10-01T09:46:47.646 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7542e-05 2023-10-01T09:46:47.646 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.646 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.647 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.647 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.647 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.647 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:125017 10.b 10:d728f595:::benchmark_data_smithi138_573643_object41795:head [delete] snapc 0=[] ondisk+write+known_if_redirected e545)", 2023-10-01T09:46:47.647 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:46:32.368638+0000", 2023-10-01T09:46:47.647 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 15.114730870000001, 2023-10-01T09:46:47.647 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.01352365, 2023-10-01T09:46:47.647 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.647 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.647 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.647 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.647 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.648 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 125017 2023-10-01T09:46:47.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.648 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.368638+0000", 2023-10-01T09:46:47.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.648 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.648 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.648 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.648 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.368638+0000", 2023-10-01T09:46:47.648 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.368636+0000", 2023-10-01T09:46:47.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:46:47.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.649 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.649 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.368645+0000", 2023-10-01T09:46:47.649 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9949999999999994e-06 2023-10-01T09:46:47.649 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.649 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.368664+0000", 2023-10-01T09:46:47.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9395e-05 2023-10-01T09:46:47.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.368675+0000", 2023-10-01T09:46:47.650 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1042e-05 2023-10-01T09:46:47.650 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.650 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.650 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.650 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.370205+0000", 2023-10-01T09:46:47.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001529987 2023-10-01T09:46:47.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.370345+0000", 2023-10-01T09:46:47.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000139333 2023-10-01T09:46:47.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.651 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.651 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:46:47.651 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.370438+0000", 2023-10-01T09:46:47.651 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.3278999999999999e-05 2023-10-01T09:46:47.651 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.375975+0000", 2023-10-01T09:46:47.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0055372540000000001 2023-10-01T09:46:47.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.652 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.376065+0000", 2023-10-01T09:46:47.652 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9777000000000005e-05 2023-10-01T09:46:47.652 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.652 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.652 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.382065+0000", 2023-10-01T09:46:47.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0060000330000000001 2023-10-01T09:46:47.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.382115+0000", 2023-10-01T09:46:47.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0064e-05 2023-10-01T09:46:47.653 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.653 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.653 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.653 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.382162+0000", 2023-10-01T09:46:47.653 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6393000000000001e-05 2023-10-01T09:46:47.654 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.654 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.654 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.654 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.654 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.654 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:125069 10.b 10:de8adbd2:::benchmark_data_smithi138_573643_object41847:head [delete] snapc 0=[] ondisk+write+known_if_redirected e545)", 2023-10-01T09:46:47.654 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:46:32.547644+0000", 2023-10-01T09:46:47.654 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 14.935724797000001, 2023-10-01T09:46:47.654 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.012912493000000001, 2023-10-01T09:46:47.654 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.655 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.655 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.655 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.655 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.655 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 125069 2023-10-01T09:46:47.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.655 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.655 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.655 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.655 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.547644+0000", 2023-10-01T09:46:47.655 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.655 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.547644+0000", 2023-10-01T09:46:47.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.656 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.547642+0000", 2023-10-01T09:46:47.656 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:46:47.656 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.656 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.656 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.547653+0000", 2023-10-01T09:46:47.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1046999999999999e-05 2023-10-01T09:46:47.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.547678+0000", 2023-10-01T09:46:47.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.4862e-05 2023-10-01T09:46:47.657 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.657 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.657 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.657 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.547690+0000", 2023-10-01T09:46:47.657 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2160999999999999e-05 2023-10-01T09:46:47.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.547756+0000", 2023-10-01T09:46:47.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.6222000000000004e-05 2023-10-01T09:46:47.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.658 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.658 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.547908+0000", 2023-10-01T09:46:47.658 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015123599999999999 2023-10-01T09:46:47.658 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.658 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:46:47.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.548058+0000", 2023-10-01T09:46:47.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00015081400000000001 2023-10-01T09:46:47.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.551613+0000", 2023-10-01T09:46:47.659 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0035540910000000001 2023-10-01T09:46:47.659 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.659 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.659 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.659 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.557928+0000", 2023-10-01T09:46:47.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0063149240000000004 2023-10-01T09:46:47.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.560435+0000", 2023-10-01T09:46:47.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025078570000000001 2023-10-01T09:46:47.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.660 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.660 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.660 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.560499+0000", 2023-10-01T09:46:47.660 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3230000000000003e-05 2023-10-01T09:46:47.660 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.661 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.661 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:32.560556+0000", 2023-10-01T09:46:47.661 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7873e-05 2023-10-01T09:46:47.661 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.661 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.661 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.661 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.661 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.661 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:125265 10.b 10:d152a144:::benchmark_data_smithi138_573643_object42043:head [delete] snapc 0=[] ondisk+write+known_if_redirected e545)", 2023-10-01T09:46:47.661 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:46:33.376927+0000", 2023-10-01T09:46:47.661 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 14.106441388, 2023-10-01T09:46:47.662 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.073908033999999997, 2023-10-01T09:46:47.662 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.662 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.662 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.662 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.662 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.662 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 125265 2023-10-01T09:46:47.662 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.662 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.662 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.662 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.662 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:33.376927+0000", 2023-10-01T09:46:47.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:33.376927+0000", 2023-10-01T09:46:47.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.663 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.663 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.663 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.663 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:33.376926+0000", 2023-10-01T09:46:47.663 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:46:47.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:33.376931+0000", 2023-10-01T09:46:47.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.797e-06 2023-10-01T09:46:47.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.664 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.664 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.664 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:33.376942+0000", 2023-10-01T09:46:47.664 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1406999999999999e-05 2023-10-01T09:46:47.664 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:33.376952+0000", 2023-10-01T09:46:47.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.7510000000000003e-06 2023-10-01T09:46:47.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.665 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:33.377002+0000", 2023-10-01T09:46:47.665 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9570999999999997e-05 2023-10-01T09:46:47.665 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.665 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.665 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:33.377107+0000", 2023-10-01T09:46:47.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000105562 2023-10-01T09:46:47.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,6", 2023-10-01T09:46:47.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:33.377189+0000", 2023-10-01T09:46:47.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.1546999999999997e-05 2023-10-01T09:46:47.666 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.666 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.666 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.666 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:33.380403+0000", 2023-10-01T09:46:47.666 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0032142310000000001 2023-10-01T09:46:47.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:33.380447+0000", 2023-10-01T09:46:47.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.3801000000000003e-05 2023-10-01T09:46:47.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.667 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.667 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:33.450767+0000", 2023-10-01T09:46:47.667 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.070319725999999999 2023-10-01T09:46:47.667 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.667 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:33.450807+0000", 2023-10-01T09:46:47.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.0683999999999999e-05 2023-10-01T09:46:47.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.668 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.668 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.668 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:33.450835+0000", 2023-10-01T09:46:47.668 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8104000000000001e-05 2023-10-01T09:46:47.668 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.668 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.668 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.668 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.669 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.669 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:127731 10.2 10:41cfd779:::benchmark_data_smithi138_573643_object44509:head [delete] snapc 0=[] ondisk+write+known_if_redirected e547)", 2023-10-01T09:46:47.669 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:46:45.978267+0000", 2023-10-01T09:46:47.669 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.505101713, 2023-10-01T09:46:47.669 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014417944, 2023-10-01T09:46:47.669 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.669 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.669 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.669 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.669 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.669 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127731 2023-10-01T09:46:47.669 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.670 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:45.978267+0000", 2023-10-01T09:46:47.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.670 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.670 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:45.978267+0000", 2023-10-01T09:46:47.670 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.670 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.670 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:45.978266+0000", 2023-10-01T09:46:47.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:46:47.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.671 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:45.978271+0000", 2023-10-01T09:46:47.671 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0320000000000003e-06 2023-10-01T09:46:47.671 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.671 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.671 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:45.978280+0000", 2023-10-01T09:46:47.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3669999999999997e-06 2023-10-01T09:46:47.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:45.978285+0000", 2023-10-01T09:46:47.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.4110000000000004e-06 2023-10-01T09:46:47.672 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.672 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.672 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.672 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:45.982335+0000", 2023-10-01T09:46:47.672 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040501790000000001 2023-10-01T09:46:47.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:45.982424+0000", 2023-10-01T09:46:47.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8867000000000001e-05 2023-10-01T09:46:47.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.673 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.673 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:46:47.673 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:45.982534+0000", 2023-10-01T09:46:47.673 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109682 2023-10-01T09:46:47.673 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:45.985688+0000", 2023-10-01T09:46:47.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.003154525 2023-10-01T09:46:47.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.674 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.674 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.674 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:45.990085+0000", 2023-10-01T09:46:47.674 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0043971449999999999 2023-10-01T09:46:47.674 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:45.992581+0000", 2023-10-01T09:46:47.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024957640000000001 2023-10-01T09:46:47.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.675 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.675 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:45.992638+0000", 2023-10-01T09:46:47.675 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6558999999999998e-05 2023-10-01T09:46:47.675 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.675 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.676 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.676 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:45.992685+0000", 2023-10-01T09:46:47.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7212000000000001e-05 2023-10-01T09:46:47.676 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.676 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.676 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.676 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.676 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.676 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:127900 10.2 10:4e058da5:::benchmark_data_smithi138_573643_object44678:head [delete] snapc 0=[] ondisk+write+known_if_redirected e547)", 2023-10-01T09:46:47.676 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:46:46.590074+0000", 2023-10-01T09:46:47.676 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.89329492200000005, 2023-10-01T09:46:47.677 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011225977999999999, 2023-10-01T09:46:47.677 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:46:47.677 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:46:47.677 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:46:47.677 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:46:47.677 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:46:47.677 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 127900 2023-10-01T09:46:47.677 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.677 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:46:47.677 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.677 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:46:47.677 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:46.590074+0000", 2023-10-01T09:46:47.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:46:47.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:46.590074+0000", 2023-10-01T09:46:47.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:46:47.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.678 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.678 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:46:47.678 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:46.590073+0000", 2023-10-01T09:46:47.678 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:46:47.678 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:46:47.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:46.590078+0000", 2023-10-01T09:46:47.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.9869999999999999e-06 2023-10-01T09:46:47.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:46:47.679 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:46.590087+0000", 2023-10-01T09:46:47.679 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.5079999999999994e-06 2023-10-01T09:46:47.679 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.679 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.679 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:46:47.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:46.590093+0000", 2023-10-01T09:46:47.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3820000000000001e-06 2023-10-01T09:46:47.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:46:47.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:46.590140+0000", 2023-10-01T09:46:47.680 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6408e-05 2023-10-01T09:46:47.680 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.680 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.680 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:46:47.680 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:46.590266+0000", 2023-10-01T09:46:47.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012575299999999999 2023-10-01T09:46:47.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 0,4", 2023-10-01T09:46:47.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:46.590370+0000", 2023-10-01T09:46:47.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000104003 2023-10-01T09:46:47.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.681 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.681 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:46.601111+0000", 2023-10-01T09:46:47.681 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010741671 2023-10-01T09:46:47.681 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.681 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:46:47.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:46.601140+0000", 2023-10-01T09:46:47.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.849e-05 2023-10-01T09:46:47.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:46:47.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:46.601178+0000", 2023-10-01T09:46:47.682 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7712000000000001e-05 2023-10-01T09:46:47.682 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.682 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.682 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:46:47.682 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:46.601236+0000", 2023-10-01T09:46:47.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8885999999999998e-05 2023-10-01T09:46:47.683 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:46:47.683 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:46:47.683 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:46:47.683 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:46.601300+0000", 2023-10-01T09:46:47.683 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3409000000000002e-05 2023-10-01T09:46:47.683 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.683 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.683 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.683 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:46:47.683 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:46:47.683 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:46:47.684 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:47.684 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:46:47.688 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:46:47.689 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:46:47.717 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:47 smithi190 ceph-mon[160435]: pgmap v1630: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 194 op/s; 5798/117636 objects degraded (4.929%); 2396/117636 objects misplaced (2.037%); 966 KiB/s, 15 objects/s recovering 2023-10-01T09:46:47.720 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:47.757 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:47 smithi138 ceph-mon[192513]: pgmap v1630: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 194 op/s; 5798/117636 objects degraded (4.929%); 2396/117636 objects misplaced (2.037%); 966 KiB/s, 15 objects/s recovering 2023-10-01T09:46:47.775 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:47.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:47.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:47.891 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:47.924 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:47.968 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:47.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:47.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:48.074 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:48.108 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:48.510 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:46:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:48 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:48 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:48 smithi138 ceph-mon[192513]: from='client.47339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:48 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5798/117636 objects degraded (4.929%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:48.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:48 smithi138 ceph-mon[192513]: pgmap v1631: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 5726/115644 objects degraded (4.951%); 2317/115644 objects misplaced (2.004%); 973 KiB/s, 15 objects/s recovering 2023-10-01T09:46:48.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:48 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:48.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:48 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:48.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:48 smithi138 ceph-mon[185886]: from='client.47339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:48.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:48 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5798/117636 objects degraded (4.929%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:48.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:48 smithi138 ceph-mon[185886]: pgmap v1631: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 5726/115644 objects degraded (4.951%); 2317/115644 objects misplaced (2.004%); 973 KiB/s, 15 objects/s recovering 2023-10-01T09:46:49.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:48 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:49.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:48 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:49.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:48 smithi190 ceph-mon[160435]: from='client.47339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:46:49.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:48 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5798/117636 objects degraded (4.929%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:49.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:48 smithi190 ceph-mon[160435]: pgmap v1631: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 202 op/s; 5726/115644 objects degraded (4.951%); 2317/115644 objects misplaced (2.004%); 973 KiB/s, 15 objects/s recovering 2023-10-01T09:46:49.618 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:49.623 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1630,"stamp":"2023-10-01T09:46:46.365421+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2572687307,"num_objects":39212,"num_object_clones":0,"num_object_copies":117636,"num_objects_missing_on_primary":862,"num_objects_missing":862,"num_objects_degraded":5798,"num_objects_misplaced":2396,"num_objects_unfound":0,"num_objects_dirty":39212,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":128362,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10338,"num_bytes_recovered":664372239,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121326,"ondisk_log_size":121326,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13308132,"kb_used_data":8229356,"kb_used_omap":39,"kb_used_meta":5078680,"kb_avail":642907932,"statfs":{"total":671965249536,"available":658337722368,"internally_reserved":0,"allocated":8426860544,"data_stored":8416873696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5200568943},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-127467520,"num_objects":-1945,"num_object_clones":0,"num_object_copies":-5835,"num_objects_missing_on_primary":-137,"num_objects_missing":-137,"num_objects_degraded":-167,"num_objects_misplaced":-121,"num_objects_unfound":0,"num_objects_dirty":-1945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1945,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":151,"num_bytes_recovered":9895936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002816"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":323,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.433661+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:46:33.433661+0000","last_peered":"2023-10-01T09:46:33.433661+0000","last_clean":"2023-10-01T09:46:33.433661+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:46:33.433661+0000","last_undegraded":"2023-10-01T09:46:33.433661+0000","last_fullsized":"2023-10-01T09:46:33.433661+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":167,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.452605+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:46:33.452605+0000","last_peered":"2023-10-01T09:46:33.452605+0000","last_clean":"2023-10-01T09:46:33.452605+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:46:33.452605+0000","last_undegraded":"2023-10-01T09:46:33.452605+0000","last_fullsized":"2023-10-01T09:46:33.452605+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":548,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.452272+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:46:33.452272+0000","last_peered":"2023-10-01T09:46:33.452272+0000","last_clean":"2023-10-01T09:46:33.452272+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:46:33.452272+0000","last_undegraded":"2023-10-01T09:46:33.452272+0000","last_fullsized":"2023-10-01T09:46:33.452272+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":605,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.429179+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:46:24.429179+0000","last_peered":"2023-10-01T09:46:24.429179+0000","last_clean":"2023-10-01T09:46:24.429179+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:46:24.429179+0000","last_undegraded":"2023-10-01T09:46:24.429179+0000","last_fullsized":"2023-10-01T09:46:24.429179+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":486,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.431919+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:46:24.431919+0000","last_peered":"2023-10-01T09:46:24.431919+0000","last_clean":"2023-10-01T09:46:24.431919+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:46:24.431919+0000","last_undegraded":"2023-10-01T09:46:24.431919+0000","last_fullsized":"2023-10-01T09:46:24.431919+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":227,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.564005+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:46:24.564005+0000","last_peered":"2023-10-01T09:46:24.564005+0000","last_clean":"2023-10-01T09:46:24.564005+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:46:24.564005+0000","last_undegraded":"2023-10-01T09:46:24.564005+0000","last_fullsized":"2023-10-01T09:46:24.564005+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":603,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.564103+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:46:24.564103+0000","last_peered":"2023-10-01T09:46:24.564103+0000","last_clean":"2023-10-01T09:46:24.564103+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:46:24.564103+0000","last_undegraded":"2023-10-01T09:46:24.564103+0000","last_fullsized":"2023-10-01T09:46:24.564103+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":534,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.429234+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:46:33.429234+0000","last_peered":"2023-10-01T09:46:33.429234+0000","last_clean":"2023-10-01T09:46:33.429234+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:46:33.429234+0000","last_undegraded":"2023-10-01T09:46:33.429234+0000","last_fullsized":"2023-10-01T09:46:33.429234+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":259,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.438773+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:46:33.438773+0000","last_peered":"2023-10-01T09:46:33.438773+0000","last_clean":"2023-10-01T09:46:33.438773+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:46:33.438773+0000","last_undegraded":"2023-10-01T09:46:33.438773+0000","last_fullsized":"2023-10-01T09:46:33.438773+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":66,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.456056+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:46:33.456056+0000","last_peered":"2023-10-01T09:46:33.456056+0000","last_clean":"2023-10-01T09:46:33.456056+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:46:33.456056+0000","last_undegraded":"2023-10-01T09:46:33.456056+0000","last_fullsized":"2023-10-01T09:46:33.456056+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":72,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.459154+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:46:33.459154+0000","last_peered":"2023-10-01T09:46:33.459154+0000","last_clean":"2023-10-01T09:46:33.459154+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:46:33.459154+0000","last_undegraded":"2023-10-01T09:46:33.459154+0000","last_fullsized":"2023-10-01T09:46:33.459154+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":71,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.452200+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:46:33.452200+0000","last_peered":"2023-10-01T09:46:33.452200+0000","last_clean":"2023-10-01T09:46:33.452200+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:46:33.452200+0000","last_undegraded":"2023-10-01T09:46:33.452200+0000","last_fullsized":"2023-10-01T09:46:33.452200+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":448,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.434074+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:46:33.434074+0000","last_peered":"2023-10-01T09:46:33.434074+0000","last_clean":"2023-10-01T09:46:33.434074+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:46:33.434074+0000","last_undegraded":"2023-10-01T09:46:33.434074+0000","last_fullsized":"2023-10-01T09:46:33.434074+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":329,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.433554+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:46:33.433554+0000","last_peered":"2023-10-01T09:46:33.433554+0000","last_clean":"2023-10-01T09:46:33.433554+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:46:33.433554+0000","last_undegraded":"2023-10-01T09:46:33.433554+0000","last_fullsized":"2023-10-01T09:46:33.433554+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":665,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.455876+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:46:33.455876+0000","last_peered":"2023-10-01T09:46:33.455876+0000","last_clean":"2023-10-01T09:46:33.455876+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:46:33.455876+0000","last_undegraded":"2023-10-01T09:46:33.455876+0000","last_fullsized":"2023-10-01T09:46:33.455876+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":402,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.456286+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:46:33.456286+0000","last_peered":"2023-10-01T09:46:33.456286+0000","last_clean":"2023-10-01T09:46:33.456286+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:46:33.456286+0000","last_undegraded":"2023-10-01T09:46:33.456286+0000","last_fullsized":"2023-10-01T09:46:33.456286+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":75,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.455706+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:46:33.455706+0000","last_peered":"2023-10-01T09:46:33.455706+0000","last_clean":"2023-10-01T09:46:33.455706+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:46:33.455706+0000","last_undegraded":"2023-10-01T09:46:33.455706+0000","last_fullsized":"2023-10-01T09:46:33.455706+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":61,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.437821+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:46:33.437821+0000","last_peered":"2023-10-01T09:46:33.437821+0000","last_clean":"2023-10-01T09:46:33.437821+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:46:33.437821+0000","last_undegraded":"2023-10-01T09:46:33.437821+0000","last_fullsized":"2023-10-01T09:46:33.437821+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":558,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.456302+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:46:33.456302+0000","last_peered":"2023-10-01T09:46:33.456302+0000","last_clean":"2023-10-01T09:46:33.456302+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:46:33.456302+0000","last_undegraded":"2023-10-01T09:46:33.456302+0000","last_fullsized":"2023-10-01T09:46:33.456302+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":569,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.429223+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:46:24.429223+0000","last_peered":"2023-10-01T09:46:24.429223+0000","last_clean":"2023-10-01T09:46:24.429223+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:46:24.429223+0000","last_undegraded":"2023-10-01T09:46:24.429223+0000","last_fullsized":"2023-10-01T09:46:24.429223+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":425,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.431831+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:46:24.431831+0000","last_peered":"2023-10-01T09:46:24.431831+0000","last_clean":"2023-10-01T09:46:24.431831+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:46:24.431831+0000","last_undegraded":"2023-10-01T09:46:24.431831+0000","last_fullsized":"2023-10-01T09:46:24.431831+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":75,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.456637+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:46:33.456637+0000","last_peered":"2023-10-01T09:46:33.456637+0000","last_clean":"2023-10-01T09:46:33.456637+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:46:33.456637+0000","last_undegraded":"2023-10-01T09:46:33.456637+0000","last_fullsized":"2023-10-01T09:46:33.456637+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"547'7923","reported_seq":8003,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463829+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:46:42.463829+0000","last_peered":"2023-10-01T09:46:42.463829+0000","last_clean":"2023-10-01T09:46:42.463829+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:46:42.463829+0000","last_undegraded":"2023-10-01T09:46:42.463829+0000","last_fullsized":"2023-10-01T09:46:42.463829+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7923,"log_dups_size":0,"ondisk_log_size":7923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161808384,"num_objects":2469,"num_object_clones":0,"num_object_copies":7407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7923,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":70,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.456521+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:46:33.456521+0000","last_peered":"2023-10-01T09:46:33.456521+0000","last_clean":"2023-10-01T09:46:33.456521+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:46:33.456521+0000","last_undegraded":"2023-10-01T09:46:33.456521+0000","last_fullsized":"2023-10-01T09:46:33.456521+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":267,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.437775+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:46:33.437775+0000","last_peered":"2023-10-01T09:46:33.437775+0000","last_clean":"2023-10-01T09:46:33.437775+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:46:33.437775+0000","last_undegraded":"2023-10-01T09:46:33.437775+0000","last_fullsized":"2023-10-01T09:46:33.437775+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"546'8028","reported_seq":16608,"reported_epoch":546,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:46:42.522224+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:46:42.522224+0000","last_peered":"2023-10-01T09:46:42.522224+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:46:42.522224+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8028,"log_dups_size":0,"ondisk_log_size":8028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155844608,"num_objects":2378,"num_object_clones":0,"num_object_copies":7134,"num_objects_missing_on_primary":862,"num_objects_missing":862,"num_objects_degraded":3171,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8028,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1447},{"shards":"0,4,6","objects":862},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":541,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.456576+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:46:33.456576+0000","last_peered":"2023-10-01T09:46:33.456576+0000","last_clean":"2023-10-01T09:46:33.456576+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:46:33.456576+0000","last_undegraded":"2023-10-01T09:46:33.456576+0000","last_fullsized":"2023-10-01T09:46:33.456576+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":603,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.452170+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:46:33.452170+0000","last_peered":"2023-10-01T09:46:33.452170+0000","last_clean":"2023-10-01T09:46:33.452170+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:46:33.452170+0000","last_undegraded":"2023-10-01T09:46:33.452170+0000","last_fullsized":"2023-10-01T09:46:33.452170+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"546'7940","reported_seq":8023,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:41.450063+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:46:41.450063+0000","last_peered":"2023-10-01T09:46:41.450063+0000","last_clean":"2023-10-01T09:46:41.450063+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:46:41.450063+0000","last_undegraded":"2023-10-01T09:46:41.450063+0000","last_fullsized":"2023-10-01T09:46:41.450063+0000","mapping_epoch":475,"log_start":"481'900","ondisk_log_start":"481'900","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7040,"log_dups_size":0,"ondisk_log_size":7040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163315712,"num_objects":2492,"num_object_clones":0,"num_object_copies":7476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7940,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":627,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.459228+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:46:33.459228+0000","last_peered":"2023-10-01T09:46:33.459228+0000","last_clean":"2023-10-01T09:46:33.459228+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:46:33.459228+0000","last_undegraded":"2023-10-01T09:46:33.459228+0000","last_fullsized":"2023-10-01T09:46:33.459228+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":424,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.452402+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:46:33.452402+0000","last_peered":"2023-10-01T09:46:33.452402+0000","last_clean":"2023-10-01T09:46:33.452402+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:46:33.452402+0000","last_undegraded":"2023-10-01T09:46:33.452402+0000","last_fullsized":"2023-10-01T09:46:33.452402+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"546'7975","reported_seq":14663,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:41.558402+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:46:41.558402+0000","last_peered":"2023-10-01T09:46:41.558402+0000","last_clean":"2023-10-01T09:46:41.558402+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:46:41.558402+0000","last_undegraded":"2023-10-01T09:46:41.558402+0000","last_fullsized":"2023-10-01T09:46:41.558402+0000","mapping_epoch":505,"log_start":"481'900","ondisk_log_start":"481'900","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7075,"log_dups_size":0,"ondisk_log_size":7075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159055872,"num_objects":2427,"num_object_clones":0,"num_object_copies":7281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7975,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":267,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.438279+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:46:33.438279+0000","last_peered":"2023-10-01T09:46:33.438279+0000","last_clean":"2023-10-01T09:46:33.438279+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:46:33.438279+0000","last_undegraded":"2023-10-01T09:46:33.438279+0000","last_fullsized":"2023-10-01T09:46:33.438279+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":525,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428730+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:46:24.428730+0000","last_peered":"2023-10-01T09:46:24.428730+0000","last_clean":"2023-10-01T09:46:24.428730+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:46:24.428730+0000","last_undegraded":"2023-10-01T09:46:24.428730+0000","last_fullsized":"2023-10-01T09:46:24.428730+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"547'8010","reported_seq":8094,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:44.942467+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:46:44.942467+0000","last_peered":"2023-10-01T09:46:44.942467+0000","last_clean":"2023-10-01T09:46:44.942467+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:46:44.942467+0000","last_undegraded":"2023-10-01T09:46:44.942467+0000","last_fullsized":"2023-10-01T09:46:44.942467+0000","mapping_epoch":475,"log_start":"481'500","ondisk_log_start":"481'500","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7510,"log_dups_size":0,"ondisk_log_size":7510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163708928,"num_objects":2498,"num_object_clones":0,"num_object_copies":7494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8010,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":127,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.433917+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:46:33.433917+0000","last_peered":"2023-10-01T09:46:33.433917+0000","last_clean":"2023-10-01T09:46:33.433917+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:46:33.433917+0000","last_undegraded":"2023-10-01T09:46:33.433917+0000","last_fullsized":"2023-10-01T09:46:33.433917+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":201,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.434151+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:46:33.434151+0000","last_peered":"2023-10-01T09:46:33.434151+0000","last_clean":"2023-10-01T09:46:33.434151+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:46:33.434151+0000","last_undegraded":"2023-10-01T09:46:33.434151+0000","last_fullsized":"2023-10-01T09:46:33.434151+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"546'7908","reported_seq":8212,"reported_epoch":546,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:46:42.111370+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:46:42.111370+0000","last_peered":"2023-10-01T09:46:42.111370+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:46:42.111370+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7908,"log_dups_size":0,"ondisk_log_size":7908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161218560,"num_objects":2460,"num_object_clones":0,"num_object_copies":7380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2617,"num_objects_misplaced":2396,"num_objects_unfound":0,"num_objects_dirty":2460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7908,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":7077888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2396},{"shards":"4,5","objects":-2553}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":445,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.433817+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:46:33.433817+0000","last_peered":"2023-10-01T09:46:33.433817+0000","last_clean":"2023-10-01T09:46:33.433817+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:46:33.433817+0000","last_undegraded":"2023-10-01T09:46:33.433817+0000","last_fullsized":"2023-10-01T09:46:33.433817+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":607,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.504468+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:46:42.504468+0000","last_peered":"2023-10-01T09:46:42.504468+0000","last_clean":"2023-10-01T09:46:42.504468+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:46:42.504468+0000","last_undegraded":"2023-10-01T09:46:42.504468+0000","last_fullsized":"2023-10-01T09:46:42.504468+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"547'7899","reported_seq":7988,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.562627+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:46:42.562627+0000","last_peered":"2023-10-01T09:46:42.562627+0000","last_clean":"2023-10-01T09:46:42.562627+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:46:42.562627+0000","last_undegraded":"2023-10-01T09:46:42.562627+0000","last_fullsized":"2023-10-01T09:46:42.562627+0000","mapping_epoch":483,"log_start":"481'300","ondisk_log_start":"481'300","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7599,"log_dups_size":0,"ondisk_log_size":7599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162463744,"num_objects":2479,"num_object_clones":0,"num_object_copies":7437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7899,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":441,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.433472+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:46:33.433472+0000","last_peered":"2023-10-01T09:46:33.433472+0000","last_clean":"2023-10-01T09:46:33.433472+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:46:33.433472+0000","last_undegraded":"2023-10-01T09:46:33.433472+0000","last_fullsized":"2023-10-01T09:46:33.433472+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":282,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.434333+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:46:33.434333+0000","last_peered":"2023-10-01T09:46:33.434333+0000","last_clean":"2023-10-01T09:46:33.434333+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:46:33.434333+0000","last_undegraded":"2023-10-01T09:46:33.434333+0000","last_fullsized":"2023-10-01T09:46:33.434333+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"547'8077","reported_seq":8157,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:46.216366+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:46:46.216366+0000","last_peered":"2023-10-01T09:46:46.216366+0000","last_clean":"2023-10-01T09:46:46.216366+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:46:46.216366+0000","last_undegraded":"2023-10-01T09:46:46.216366+0000","last_fullsized":"2023-10-01T09:46:46.216366+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8077,"log_dups_size":0,"ondisk_log_size":8077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158662656,"num_objects":2421,"num_object_clones":0,"num_object_copies":7263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8077,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":411,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.451842+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:46:33.451842+0000","last_peered":"2023-10-01T09:46:33.451842+0000","last_clean":"2023-10-01T09:46:33.451842+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:46:33.451842+0000","last_undegraded":"2023-10-01T09:46:33.451842+0000","last_fullsized":"2023-10-01T09:46:33.451842+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":458,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.453385+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:46:33.453385+0000","last_peered":"2023-10-01T09:46:33.453385+0000","last_clean":"2023-10-01T09:46:33.453385+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:46:33.453385+0000","last_undegraded":"2023-10-01T09:46:33.453385+0000","last_fullsized":"2023-10-01T09:46:33.453385+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"547'7835","reported_seq":7915,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:46.171418+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:46:46.171418+0000","last_peered":"2023-10-01T09:46:46.171418+0000","last_clean":"2023-10-01T09:46:46.171418+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:46:46.171418+0000","last_undegraded":"2023-10-01T09:46:46.171418+0000","last_fullsized":"2023-10-01T09:46:46.171418+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7835,"log_dups_size":0,"ondisk_log_size":7835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153550848,"num_objects":2343,"num_object_clones":0,"num_object_copies":7029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7835,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":940,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.455406+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:46:33.455406+0000","last_peered":"2023-10-01T09:46:33.455406+0000","last_clean":"2023-10-01T09:46:33.455406+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:46:33.455406+0000","last_undegraded":"2023-10-01T09:46:33.455406+0000","last_fullsized":"2023-10-01T09:46:33.455406+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":277,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.438637+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:46:33.438637+0000","last_peered":"2023-10-01T09:46:33.438637+0000","last_clean":"2023-10-01T09:46:33.438637+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:46:33.438637+0000","last_undegraded":"2023-10-01T09:46:33.438637+0000","last_fullsized":"2023-10-01T09:46:33.438637+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":71,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.452277+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:46:33.452277+0000","last_peered":"2023-10-01T09:46:33.452277+0000","last_clean":"2023-10-01T09:46:33.452277+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:46:33.452277+0000","last_undegraded":"2023-10-01T09:46:33.452277+0000","last_fullsized":"2023-10-01T09:46:33.452277+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"546'7841","reported_seq":7924,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:41.521657+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:46:41.521657+0000","last_peered":"2023-10-01T09:46:41.521657+0000","last_clean":"2023-10-01T09:46:41.521657+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:46:41.521657+0000","last_undegraded":"2023-10-01T09:46:41.521657+0000","last_fullsized":"2023-10-01T09:46:41.521657+0000","mapping_epoch":475,"log_start":"481'800","ondisk_log_start":"481'800","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7041,"log_dups_size":0,"ondisk_log_size":7041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165740544,"num_objects":2529,"num_object_clones":0,"num_object_copies":7587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7841,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1212,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.438093+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:46:33.438093+0000","last_peered":"2023-10-01T09:46:33.438093+0000","last_clean":"2023-10-01T09:46:33.438093+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:46:33.438093+0000","last_undegraded":"2023-10-01T09:46:33.438093+0000","last_fullsized":"2023-10-01T09:46:33.438093+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":422,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.433768+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:46:33.433768+0000","last_peered":"2023-10-01T09:46:33.433768+0000","last_clean":"2023-10-01T09:46:33.433768+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:46:33.433768+0000","last_undegraded":"2023-10-01T09:46:33.433768+0000","last_fullsized":"2023-10-01T09:46:33.433768+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":250,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.438458+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:46:33.438458+0000","last_peered":"2023-10-01T09:46:33.438458+0000","last_clean":"2023-10-01T09:46:33.438458+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:46:33.438458+0000","last_undegraded":"2023-10-01T09:46:33.438458+0000","last_fullsized":"2023-10-01T09:46:33.438458+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"546'7897","reported_seq":15198,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.222087+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:46:42.222087+0000","last_peered":"2023-10-01T09:46:42.222087+0000","last_clean":"2023-10-01T09:46:42.222087+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:46:42.222087+0000","last_undegraded":"2023-10-01T09:46:42.222087+0000","last_fullsized":"2023-10-01T09:46:42.222087+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7897,"log_dups_size":0,"ondisk_log_size":7897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162463744,"num_objects":2479,"num_object_clones":0,"num_object_copies":7437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7897,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":892,"reported_epoch":545,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:46:24.428438+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:46:24.428438+0000","last_peered":"2023-10-01T09:46:24.428438+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:46:24.428438+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:46:24.428438+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":611,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.451922+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:46:33.451922+0000","last_peered":"2023-10-01T09:46:33.451922+0000","last_clean":"2023-10-01T09:46:33.451922+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:46:33.451922+0000","last_undegraded":"2023-10-01T09:46:33.451922+0000","last_fullsized":"2023-10-01T09:46:33.451922+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":71,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.453250+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:46:33.453250+0000","last_peered":"2023-10-01T09:46:33.453250+0000","last_clean":"2023-10-01T09:46:33.453250+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:46:33.453250+0000","last_undegraded":"2023-10-01T09:46:33.453250+0000","last_fullsized":"2023-10-01T09:46:33.453250+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"547'7870","reported_seq":7953,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:45.204547+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:46:45.204547+0000","last_peered":"2023-10-01T09:46:45.204547+0000","last_clean":"2023-10-01T09:46:45.204547+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:46:45.204547+0000","last_undegraded":"2023-10-01T09:46:45.204547+0000","last_fullsized":"2023-10-01T09:46:45.204547+0000","mapping_epoch":475,"log_start":"481'300","ondisk_log_start":"481'300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7570,"log_dups_size":0,"ondisk_log_size":7570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157417472,"num_objects":2402,"num_object_clones":0,"num_object_copies":7206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7870,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1663,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.439447+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:46:33.439447+0000","last_peered":"2023-10-01T09:46:33.439447+0000","last_clean":"2023-10-01T09:46:33.439447+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:46:33.439447+0000","last_undegraded":"2023-10-01T09:46:33.439447+0000","last_fullsized":"2023-10-01T09:46:33.439447+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":577,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.455741+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:46:33.455741+0000","last_peered":"2023-10-01T09:46:33.455741+0000","last_clean":"2023-10-01T09:46:33.455741+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:46:33.455741+0000","last_undegraded":"2023-10-01T09:46:33.455741+0000","last_fullsized":"2023-10-01T09:46:33.455741+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":355,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.438114+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:46:33.438114+0000","last_peered":"2023-10-01T09:46:33.438114+0000","last_clean":"2023-10-01T09:46:33.438114+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:46:33.438114+0000","last_undegraded":"2023-10-01T09:46:33.438114+0000","last_fullsized":"2023-10-01T09:46:33.438114+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":345,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.433781+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:46:33.433781+0000","last_peered":"2023-10-01T09:46:33.433781+0000","last_clean":"2023-10-01T09:46:33.433781+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:46:33.433781+0000","last_undegraded":"2023-10-01T09:46:33.433781+0000","last_fullsized":"2023-10-01T09:46:33.433781+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"547'8101","reported_seq":8181,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:46.190822+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:46:46.190822+0000","last_peered":"2023-10-01T09:46:46.190822+0000","last_clean":"2023-10-01T09:46:46.190822+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:46:46.190822+0000","last_undegraded":"2023-10-01T09:46:46.190822+0000","last_fullsized":"2023-10-01T09:46:46.190822+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8101,"log_dups_size":0,"ondisk_log_size":8101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159318016,"num_objects":2431,"num_object_clones":0,"num_object_copies":7293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8101,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":754,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.455901+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:46:33.455901+0000","last_peered":"2023-10-01T09:46:33.455901+0000","last_clean":"2023-10-01T09:46:33.455901+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:46:33.455901+0000","last_undegraded":"2023-10-01T09:46:33.455901+0000","last_fullsized":"2023-10-01T09:46:33.455901+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":587,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428696+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:46:24.428696+0000","last_peered":"2023-10-01T09:46:24.428696+0000","last_clean":"2023-10-01T09:46:24.428696+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:46:24.428696+0000","last_undegraded":"2023-10-01T09:46:24.428696+0000","last_fullsized":"2023-10-01T09:46:24.428696+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":586,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.429050+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:46:24.429050+0000","last_peered":"2023-10-01T09:46:24.429050+0000","last_clean":"2023-10-01T09:46:24.429050+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:46:24.429050+0000","last_undegraded":"2023-10-01T09:46:24.429050+0000","last_fullsized":"2023-10-01T09:46:24.429050+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"546'7945","reported_seq":8028,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:41.563634+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:46:41.563634+0000","last_peered":"2023-10-01T09:46:41.563634+0000","last_clean":"2023-10-01T09:46:41.563634+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:46:41.563634+0000","last_undegraded":"2023-10-01T09:46:41.563634+0000","last_fullsized":"2023-10-01T09:46:41.563634+0000","mapping_epoch":475,"log_start":"481'900","ondisk_log_start":"481'900","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7045,"log_dups_size":0,"ondisk_log_size":7045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164757528,"num_objects":2515,"num_object_clones":0,"num_object_copies":7545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2515,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7945,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":985,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.459241+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:46:33.459241+0000","last_peered":"2023-10-01T09:46:33.459241+0000","last_clean":"2023-10-01T09:46:33.459241+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:46:33.459241+0000","last_undegraded":"2023-10-01T09:46:33.459241+0000","last_fullsized":"2023-10-01T09:46:33.459241+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":604,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.438379+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:46:33.438379+0000","last_peered":"2023-10-01T09:46:33.438379+0000","last_clean":"2023-10-01T09:46:33.438379+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:46:33.438379+0000","last_undegraded":"2023-10-01T09:46:33.438379+0000","last_fullsized":"2023-10-01T09:46:33.438379+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":457,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.439530+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:46:33.439530+0000","last_peered":"2023-10-01T09:46:33.439530+0000","last_clean":"2023-10-01T09:46:33.439530+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:46:33.439530+0000","last_undegraded":"2023-10-01T09:46:33.439530+0000","last_fullsized":"2023-10-01T09:46:33.439530+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"547'7964","reported_seq":8052,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:46.153852+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:46:46.153852+0000","last_peered":"2023-10-01T09:46:46.153852+0000","last_clean":"2023-10-01T09:46:46.153852+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:46:46.153852+0000","last_undegraded":"2023-10-01T09:46:46.153852+0000","last_fullsized":"2023-10-01T09:46:46.153852+0000","mapping_epoch":479,"log_start":"481'1200","ondisk_log_start":"481'1200","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6764,"log_dups_size":0,"ondisk_log_size":6764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156762112,"num_objects":2392,"num_object_clones":0,"num_object_copies":7176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7964,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":486,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.456666+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:46:33.456666+0000","last_peered":"2023-10-01T09:46:33.456666+0000","last_clean":"2023-10-01T09:46:33.456666+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:46:33.456666+0000","last_undegraded":"2023-10-01T09:46:33.456666+0000","last_fullsized":"2023-10-01T09:46:33.456666+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1180,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.452829+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:46:33.452829+0000","last_peered":"2023-10-01T09:46:33.452829+0000","last_clean":"2023-10-01T09:46:33.452829+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:46:33.452829+0000","last_undegraded":"2023-10-01T09:46:33.452829+0000","last_fullsized":"2023-10-01T09:46:33.452829+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":75,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.455661+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:46:33.455661+0000","last_peered":"2023-10-01T09:46:33.455661+0000","last_clean":"2023-10-01T09:46:33.455661+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:46:33.455661+0000","last_undegraded":"2023-10-01T09:46:33.455661+0000","last_fullsized":"2023-10-01T09:46:33.455661+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"547'8043","reported_seq":8131,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:46.248081+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:46:46.248081+0000","last_peered":"2023-10-01T09:46:46.248081+0000","last_clean":"2023-10-01T09:46:46.248081+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:46:46.248081+0000","last_undegraded":"2023-10-01T09:46:46.248081+0000","last_fullsized":"2023-10-01T09:46:46.248081+0000","mapping_epoch":479,"log_start":"482'1300","ondisk_log_start":"482'1300","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6743,"log_dups_size":0,"ondisk_log_size":6743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161808384,"num_objects":2469,"num_object_clones":0,"num_object_copies":7407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8043,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":79,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.456541+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:46:33.456541+0000","last_peered":"2023-10-01T09:46:33.456541+0000","last_clean":"2023-10-01T09:46:33.456541+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:46:33.456541+0000","last_undegraded":"2023-10-01T09:46:33.456541+0000","last_fullsized":"2023-10-01T09:46:33.456541+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":564,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.429131+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:46:24.429131+0000","last_peered":"2023-10-01T09:46:24.429131+0000","last_clean":"2023-10-01T09:46:24.429131+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:46:24.429131+0000","last_undegraded":"2023-10-01T09:46:24.429131+0000","last_fullsized":"2023-10-01T09:46:24.429131+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":270,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.437963+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:46:33.437963+0000","last_peered":"2023-10-01T09:46:33.437963+0000","last_clean":"2023-10-01T09:46:33.437963+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:46:33.437963+0000","last_undegraded":"2023-10-01T09:46:33.437963+0000","last_fullsized":"2023-10-01T09:46:33.437963+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":513,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428989+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:46:24.428989+0000","last_peered":"2023-10-01T09:46:24.428989+0000","last_clean":"2023-10-01T09:46:24.428989+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:46:24.428989+0000","last_undegraded":"2023-10-01T09:46:24.428989+0000","last_fullsized":"2023-10-01T09:46:24.428989+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":628,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.452492+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:46:33.452492+0000","last_peered":"2023-10-01T09:46:33.452492+0000","last_clean":"2023-10-01T09:46:33.452492+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:46:33.452492+0000","last_undegraded":"2023-10-01T09:46:33.452492+0000","last_fullsized":"2023-10-01T09:46:33.452492+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":282,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.438203+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:46:33.438203+0000","last_peered":"2023-10-01T09:46:33.438203+0000","last_clean":"2023-10-01T09:46:33.438203+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:46:33.438203+0000","last_undegraded":"2023-10-01T09:46:33.438203+0000","last_fullsized":"2023-10-01T09:46:33.438203+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":71,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.452358+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:46:33.452358+0000","last_peered":"2023-10-01T09:46:33.452358+0000","last_clean":"2023-10-01T09:46:33.452358+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:46:33.452358+0000","last_undegraded":"2023-10-01T09:46:33.452358+0000","last_fullsized":"2023-10-01T09:46:33.452358+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":282,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.438718+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:46:33.438718+0000","last_peered":"2023-10-01T09:46:33.438718+0000","last_clean":"2023-10-01T09:46:33.438718+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:46:33.438718+0000","last_undegraded":"2023-10-01T09:46:33.438718+0000","last_fullsized":"2023-10-01T09:46:33.438718+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":459,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.429770+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:46:24.429770+0000","last_peered":"2023-10-01T09:46:24.429770+0000","last_clean":"2023-10-01T09:46:24.429770+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:46:24.429770+0000","last_undegraded":"2023-10-01T09:46:24.429770+0000","last_fullsized":"2023-10-01T09:46:24.429770+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":89,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.452731+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:46:33.452731+0000","last_peered":"2023-10-01T09:46:33.452731+0000","last_clean":"2023-10-01T09:46:33.452731+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:46:33.452731+0000","last_undegraded":"2023-10-01T09:46:33.452731+0000","last_fullsized":"2023-10-01T09:46:33.452731+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":504,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.428609+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:46:24.428609+0000","last_peered":"2023-10-01T09:46:24.428609+0000","last_clean":"2023-10-01T09:46:24.428609+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:46:24.428609+0000","last_undegraded":"2023-10-01T09:46:24.428609+0000","last_fullsized":"2023-10-01T09:46:24.428609+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":704,"reported_epoch":545,"state":"active+clean","last_fresh":"2023-10-01T09:46:24.429415+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:46:24.429415+0000","last_peered":"2023-10-01T09:46:24.429415+0000","last_clean":"2023-10-01T09:46:24.429415+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:46:24.429415+0000","last_undegraded":"2023-10-01T09:46:24.429415+0000","last_fullsized":"2023-10-01T09:46:24.429415+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":398,"reported_epoch":546,"state":"active+clean","last_fresh":"2023-10-01T09:46:33.433408+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:46:33.433408+0000","last_peered":"2023-10-01T09:46:33.433408+0000","last_clean":"2023-10-01T09:46:33.433408+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:46:33.433408+0000","last_undegraded":"2023-10-01T09:46:33.433408+0000","last_fullsized":"2023-10-01T09:46:33.433408+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2567897112,"num_objects":39184,"num_object_clones":0,"num_object_copies":117552,"num_objects_missing_on_primary":862,"num_objects_missing":862,"num_objects_degraded":5788,"num_objects_misplaced":2396,"num_objects_unfound":0,"num_objects_dirty":39184,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":127256,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9827,"num_bytes_recovered":644022272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8352641024,"data_stored":8352628808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120156,"ondisk_log_size":120156,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430754,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1674024,"kb_used_data":779104,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":92071128,"statfs":{"total":95995035648,"available":94280835072,"internally_reserved":0,"allocated":797802496,"data_stored":796857988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.74399999999999999,"15min":2.5840000000000001},"min":{"1min":0.27700000000000002,"5min":0.246,"15min":0.20999999999999999},"max":{"1min":1.081,"5min":6.8550000000000004,"15min":108.90000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.52500000000000002,"15min":2.7050000000000001},"min":{"1min":0.25700000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.2090000000000001,"5min":5.6639999999999997,"15min":109.187},"last":0.46899999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.66400000000000003,"15min":5.1520000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.222},"max":{"1min":1.0780000000000001,"5min":6.8289999999999997,"15min":229.161},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.59799999999999998,"15min":5.1779999999999999},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":3.0640000000000001,"5min":5.7240000000000002,"15min":230.126},"last":0.624}]},{"osd":2,"last update":"Sun Oct 1 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.72799999999999998,"15min":1.617},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.1259999999999999,"5min":5.3090000000000002,"15min":14.901999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.68500000000000005,"15min":1.6100000000000001},"min":{"1min":0.33700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.0009999999999999,"5min":5.6929999999999996,"15min":14.682},"last":0.53000000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.65800000000000003,"15min":2.2000000000000002},"min":{"1min":0.35699999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.476,"5min":5.7750000000000004,"15min":76.555000000000007},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.68100000000000005,"15min":2.2010000000000001},"min":{"1min":0.32800000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.4529999999999998,"5min":6.548,"15min":76.534000000000006},"last":0.376}]},{"osd":4,"last update":"Sun Oct 1 09:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.70499999999999996,"15min":1.042},"min":{"1min":0.28299999999999997,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.276,"5min":6.8769999999999998,"15min":9.6020000000000003},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.68500000000000005,"15min":0.997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.97399999999999998,"5min":5.7439999999999998,"15min":8.4949999999999992},"last":0.502}]},{"osd":5,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.52500000000000002,"15min":1.5269999999999999},"min":{"1min":0.35399999999999998,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":2.2400000000000002,"5min":2.2400000000000002,"15min":135.40000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.54500000000000004,"15min":1.627},"min":{"1min":0.34999999999999998,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":0.70699999999999996,"5min":2.8849999999999998,"15min":135.339},"last":0.33500000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.58999999999999997,"15min":1.1200000000000001},"min":{"1min":0.40500000000000003,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":1.7829999999999999,"5min":2.448,"15min":11.853999999999999},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.59699999999999998,"15min":1.093},"min":{"1min":0.39700000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":3.1909999999999998,"5min":3.7810000000000001,"15min":14.584},"last":0.40500000000000003}]}]},{"osd":6,"up_from":294,"seq":1262720385264,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1450076,"kb_used_data":1116564,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":92295076,"statfs":{"total":95995035648,"available":94510157824,"internally_reserved":0,"allocated":1143361536,"data_stored":1142421303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.45200000000000001,"15min":3.681},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20599999999999999},"max":{"1min":0.51700000000000002,"5min":3.3340000000000001,"15min":158.48599999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.55500000000000005,"15min":3.746},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.221},"max":{"1min":3.3519999999999999,"5min":3.3519999999999999,"15min":158.37700000000001},"last":0.33900000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.58499999999999996,"15min":4.5739999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":3.3069999999999999,"5min":3.3069999999999999,"15min":168.947},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.60299999999999998,"15min":4.532},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":3.7469999999999999,"5min":3.7469999999999999,"15min":169.07900000000001},"last":0.45500000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.59999999999999998,"15min":1.4490000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":3.3849999999999998,"5min":3.3849999999999998,"15min":10.92},"last":0.379},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":1.4690000000000001},"min":{"1min":0.32100000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.7160000000000002,"5min":3.7160000000000002,"15min":21.704999999999998},"last":0.50700000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.57999999999999996,"15min":1.8100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.63800000000000001,"5min":3.3759999999999999,"15min":89.753},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.69499999999999995,"15min":1.754},"min":{"1min":0.309,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.2050000000000001,"5min":4.2050000000000001,"15min":89.849999999999994},"last":0.503}]},{"osd":4,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58899999999999997,"15min":1.069},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.6349999999999998,"5min":4.0250000000000004,"15min":63.134},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.65800000000000003,"15min":1.0680000000000001},"min":{"1min":0.36599999999999999,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":1.006,"5min":7.4039999999999999,"15min":62.973999999999997},"last":0.48299999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:46:37 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.88500000000000001,"15min":0.999},"min":{"1min":0.33100000000000002,"5min":0.26700000000000002,"15min":0.251},"max":{"1min":0.93799999999999994,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.875,"5min":1.022,"15min":1.1379999999999999},"min":{"1min":0.315,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.181,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.35799999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.56699999999999995,"15min":1.2729999999999999},"min":{"1min":0.377,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.73399999999999999,"5min":3.339,"15min":94.245999999999995},"last":0.439},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.67200000000000004,"15min":1.369},"min":{"1min":0.33100000000000002,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":3.653,"5min":3.653,"15min":94.715000000000003},"last":0.46999999999999997}]}]},{"osd":1,"up_from":94,"seq":403726926241,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1308408,"kb_used_data":480368,"kb_used_omap":7,"kb_used_meta":828024,"kb_avail":92436744,"statfs":{"total":95995035648,"available":94655225856,"internally_reserved":0,"allocated":491896832,"data_stored":489823873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":847896999},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4410000000000001,"5min":1.1140000000000001,"15min":3.8010000000000002},"min":{"1min":0.217,"5min":0.186,"15min":0.182},"max":{"1min":18.556999999999999,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.82799999999999996},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.302,"15min":3.3359999999999999},"min":{"1min":0.23499999999999999,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":18.704000000000001,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.68200000000000005}]},{"osd":2,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.32100000000000001,"5min":0.39800000000000002,"15min":2.2669999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20000000000000001},"max":{"1min":0.51100000000000001,"5min":2.8740000000000001,"15min":51.640000000000001},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.61799999999999999,"15min":2.9500000000000002},"min":{"1min":0.245,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":2.9089999999999998,"5min":6.9710000000000001,"15min":154.197},"last":0.436}]},{"osd":3,"last update":"Sun Oct 1 09:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.45700000000000002,"15min":2.9540000000000002},"min":{"1min":0.28199999999999997,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":2.8780000000000001,"5min":3.1949999999999998,"15min":95.305000000000007},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.432,"5min":0.51200000000000001,"15min":3.6720000000000002},"min":{"1min":0.27300000000000002,"5min":0.245,"15min":0.216},"max":{"1min":0.95699999999999996,"5min":4.7839999999999998,"15min":154.483},"last":0.56399999999999995}]},{"osd":4,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.59099999999999997,"15min":2.1829999999999998},"min":{"1min":0.312,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":1.5309999999999999,"5min":3.0550000000000002,"15min":154.101},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.49099999999999999,"15min":2.2869999999999999},"min":{"1min":0.34300000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65500000000000003,"5min":2.4209999999999998,"15min":154.56999999999999},"last":0.63600000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.246,"15min":2.2530000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.30199999999999999},"max":{"1min":1.696,"5min":59.561,"15min":154.22999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.1970000000000001,"15min":2.4649999999999999},"min":{"1min":0.34799999999999998,"5min":0.31,"15min":0.307},"max":{"1min":0.67300000000000004,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.60299999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.65700000000000003,"15min":2.448},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.944,"5min":3.3660000000000001,"15min":154.12200000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64600000000000002,"15min":2.3450000000000002},"min":{"1min":0.374,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":1.7549999999999999,"5min":2.5630000000000002,"15min":154.16999999999999},"last":0.67000000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.74299999999999999,"15min":2.5230000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.33900000000000002},"max":{"1min":3.6160000000000001,"5min":5.2169999999999996,"15min":154.20699999999999},"last":0.62},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.70499999999999996,"15min":2.5470000000000002},"min":{"1min":0.40300000000000002,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":3.7490000000000001,"5min":3.8260000000000001,"15min":154.51599999999999},"last":0.66000000000000003}]}]},{"osd":0,"up_from":89,"seq":382252089775,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3184524,"kb_used_data":2261380,"kb_used_omap":8,"kb_used_meta":923127,"kb_avail":90560628,"statfs":{"total":95995035648,"available":92734083072,"internally_reserved":0,"allocated":2315653120,"data_stored":2313620508,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":945283010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.311,"5min":0.42399999999999999,"15min":3.988},"min":{"1min":0.20699999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.83699999999999997,"5min":4.2919999999999998,"15min":216.821},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.44800000000000001,"15min":4.6740000000000004},"min":{"1min":0.218,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.56799999999999995,"5min":3.9620000000000002,"15min":217.02699999999999},"last":0.54100000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.64600000000000002,"15min":1.9910000000000001},"min":{"1min":0.25,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":4.0110000000000001,"5min":4.9459999999999997,"15min":19.788},"last":0.996},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.65500000000000003,"15min":2.0310000000000001},"min":{"1min":0.28499999999999998,"5min":0.27200000000000002,"15min":0.189},"max":{"1min":3.8719999999999999,"5min":3.8719999999999999,"15min":25.844999999999999},"last":1.0289999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.504,"15min":3.153},"min":{"1min":0.24099999999999999,"5min":0.22,"15min":0.22},"max":{"1min":1.9039999999999999,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.68200000000000005,"15min":3.0019999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.1379999999999999,"5min":4.9199999999999999,"15min":222.227},"last":0.45300000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.59499999999999997,"15min":1.925},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":0.84299999999999997,"5min":4.048,"15min":62.206000000000003},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.63600000000000001,"15min":2.0089999999999999},"min":{"1min":0.29199999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":1.5189999999999999,"5min":4,"15min":62.073999999999998},"last":0.49099999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:45:42 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.76800000000000002,"15min":1.744},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":4.0350000000000001,"5min":5.8769999999999998,"15min":16.808},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.84499999999999997,"15min":1.6499999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.1269999999999998,"5min":4.8689999999999998,"15min":17.507000000000001},"last":0.95999999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:46:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.63600000000000001,"15min":1.6990000000000001},"min":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.1970000000000001,"5min":4.3129999999999997,"15min":17.535},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65900000000000003,"15min":1.732},"min":{"1min":0.372,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.218,"5min":4.2050000000000001,"15min":15.651999999999999},"last":0.65700000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.79500000000000004,"15min":2.028},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":2.1000000000000001,"5min":4.976,"15min":48.895000000000003},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.89900000000000002,"15min":2.048},"min":{"1min":0.376,"5min":0.376,"15min":0.35099999999999998},"max":{"1min":4.125,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.56999999999999995}]}]},{"osd":2,"up_from":0,"seq":425201762715,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599188,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1326768,"kb_used_data":490536,"kb_used_omap":3,"kb_used_meta":836220,"kb_avail":92418384,"statfs":{"total":95995035648,"available":94636425216,"internally_reserved":0,"allocated":502308864,"data_stored":500178104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":856290023},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":1.4870000000000001,"15min":4.6909999999999998},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":0.65000000000000002,"5min":64.504000000000005,"15min":137.34399999999999},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.41499999999999998,"5min":1.4870000000000001,"15min":4.6959999999999997},"min":{"1min":0.219,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":1.2849999999999999,"5min":64.448999999999998,"15min":137.73400000000001},"last":0.32900000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.51700000000000002,"15min":4.8719999999999999},"min":{"1min":0.25,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":1.3500000000000001,"5min":6.0250000000000004,"15min":162.22999999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.40600000000000003,"15min":4.9199999999999999},"min":{"1min":0.20899999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.77600000000000002,"5min":2.931,"15min":162.70699999999999},"last":0.34200000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56599999999999995,"15min":1.5800000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.20799999999999999},"max":{"1min":2.9049999999999998,"5min":7.141,"15min":14.254},"last":0.316},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.52300000000000002,"15min":1.5860000000000001},"min":{"1min":0.314,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":0.80400000000000005,"5min":4.5049999999999999,"15min":15.523},"last":0.29499999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.54600000000000004,"15min":1.71},"min":{"1min":0.34999999999999998,"5min":0.312,"15min":0.26200000000000001},"max":{"1min":0.874,"5min":6.8890000000000002,"15min":77.793000000000006},"last":0.373},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.58699999999999997,"15min":1.8919999999999999},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.409,"5min":6.0940000000000003,"15min":76.646000000000001},"last":0.36499999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.58399999999999996,"15min":1.9119999999999999},"min":{"1min":0.32500000000000001,"5min":0.314,"15min":0.27500000000000002},"max":{"1min":0.61399999999999999,"5min":4.3099999999999996,"15min":28.117999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51800000000000002,"15min":2.0310000000000001},"min":{"1min":0.33800000000000002,"5min":0.33600000000000002,"15min":0.27700000000000002},"max":{"1min":1.9930000000000001,"5min":2.512,"15min":28.34},"last":0.40799999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.55500000000000005,"15min":1.764},"min":{"1min":0.379,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":0.56899999999999995,"5min":2.9100000000000001,"15min":73.887},"last":1.0549999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.58499999999999996,"15min":1.8560000000000001},"min":{"1min":0.379,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":0.60499999999999998,"5min":2.6000000000000001,"15min":74.016999999999996},"last":0.501}]},{"osd":7,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.86299999999999999,"15min":1.7809999999999999},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":3.6480000000000001,"5min":7.0620000000000003,"15min":13.085000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.746,"5min":0.77200000000000002,"15min":1.633},"min":{"1min":0.38800000000000001,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.7120000000000002,"5min":4.1779999999999999,"15min":13.191000000000001},"last":0.432}]}]},{"osd":4,"up_from":30,"seq":128849019439,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2937700,"kb_used_data":2043420,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":90807452,"statfs":{"total":95995035648,"available":92986830848,"internally_reserved":0,"allocated":2092462080,"data_stored":2091521078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.623,"15min":3.9359999999999999},"min":{"1min":0.22600000000000001,"5min":0.22500000000000001,"15min":0.21199999999999999},"max":{"1min":0.51400000000000001,"5min":9.6690000000000005,"15min":155.315},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.371,"5min":0.72499999999999998,"15min":4.2910000000000004},"min":{"1min":0.253,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.499,"5min":6.1459999999999999,"15min":155.334},"last":0.33500000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.72399999999999998,"15min":5.2430000000000003},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":0.57799999999999996,"5min":5.944,"15min":190.61199999999999},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.63400000000000001,"15min":5.9699999999999998},"min":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.25600000000000001},"max":{"1min":1.679,"5min":3.7469999999999999,"15min":253.273},"last":0.45300000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.74299999999999999,"15min":2.1150000000000002},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.1639999999999999,"5min":5.702,"15min":170.45400000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.748,"15min":1.889},"min":{"1min":0.38600000000000001,"5min":0.314,"15min":0.25800000000000001},"max":{"1min":2.121,"5min":5.6150000000000002,"15min":36.515999999999998},"last":0.39400000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.69699999999999995,"15min":2.125},"min":{"1min":0.29199999999999998,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":2.0379999999999998,"5min":5.5279999999999996,"15min":140.881},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.68500000000000005,"15min":2.2749999999999999},"min":{"1min":0.32300000000000001,"5min":0.312,"15min":0.311},"max":{"1min":6.2969999999999997,"5min":6.2969999999999997,"15min":140.911},"last":0.47799999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:46:41 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.56999999999999995,"15min":1.454},"min":{"1min":0.27200000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":1.8500000000000001,"5min":3.052,"15min":110.003},"last":1.9019999999999999},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.63400000000000001,"15min":1.3859999999999999},"min":{"1min":0.34100000000000003,"5min":0.317,"15min":0.28899999999999998},"max":{"1min":1.754,"5min":3.2909999999999999,"15min":108.56399999999999},"last":1.8500000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.623,"15min":0.91900000000000004},"min":{"1min":0.34599999999999997,"5min":0.28799999999999998,"15min":0.27200000000000002},"max":{"1min":0.63700000000000001,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.63400000000000001,"15min":1.0289999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28999999999999998},"max":{"1min":1.3340000000000001,"5min":4.0220000000000002,"15min":10.076000000000001},"last":0.36899999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.66800000000000004,"15min":1.139},"min":{"1min":0.33400000000000002,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":2.98,"5min":3.8580000000000001,"15min":38.034999999999997},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.91400000000000003,"15min":1.4350000000000001},"min":{"1min":0.40899999999999997,"5min":0.36699999999999999,"15min":0.308},"max":{"1min":3.113,"5min":12.092000000000001,"15min":37.723999999999997},"last":0.42099999999999999}]}]},{"osd":5,"up_from":379,"seq":1627792605345,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1426632,"kb_used_data":1057984,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":92318520,"statfs":{"total":95995035648,"available":94534164480,"internally_reserved":0,"allocated":1083375616,"data_stored":1082450842,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.749,"15min":2.0830000000000002},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.21099999999999999},"max":{"1min":0.77600000000000002,"5min":27.584,"15min":84.340999999999994},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.76900000000000002,"15min":2.0369999999999999},"min":{"1min":0.28399999999999997,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.749,"5min":27.669,"15min":84.356999999999999},"last":0.32000000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.55200000000000005,"15min":5.5519999999999996},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.2130000000000001,"5min":4.3289999999999997,"15min":295.71300000000002},"last":0.501},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.51300000000000001,"15min":5.5609999999999999},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":0.70399999999999996,"5min":3.9580000000000002,"15min":299.59399999999999},"last":0.35299999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.52400000000000002,"15min":1.45},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78600000000000003,"5min":4.2670000000000003,"15min":14.199999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.65100000000000002,"15min":1.4139999999999999},"min":{"1min":0.29599999999999999,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":4.2910000000000004,"5min":4.3719999999999999,"15min":14.430999999999999},"last":0.53200000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.56699999999999995,"15min":1.3720000000000001},"min":{"1min":0.32800000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.75,"5min":4.0209999999999999,"15min":14.004},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.58799999999999997,"15min":1.518},"min":{"1min":0.40100000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":0.83299999999999996,"5min":4.2910000000000004,"15min":13.991},"last":0.58299999999999996}]},{"osd":4,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.53300000000000003,"15min":1.365},"min":{"1min":0.29499999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.0620000000000001,"5min":4.1929999999999996,"15min":122.971},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.623,"15min":1.5389999999999999},"min":{"1min":0.28899999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2619999999999996,"5min":4.3869999999999996,"15min":122.89100000000001},"last":0.56000000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.54900000000000004,"15min":0.95399999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":0.82799999999999996,"5min":4.3300000000000001,"15min":12.545},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.5,"5min":0.64100000000000001,"15min":1.1459999999999999},"min":{"1min":0.36499999999999999,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":0.81899999999999995,"5min":4.3200000000000003,"15min":14.193},"last":0.63400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.68300000000000005,"15min":1.1499999999999999},"min":{"1min":0.35399999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.3170000000000002,"5min":4.3559999999999999,"15min":10.813000000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.67200000000000004,"15min":1.0469999999999999},"min":{"1min":0.39500000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":1.5369999999999999,"5min":4.2000000000000002,"15min":12.500999999999999},"last":0.748}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2300448768,"data_stored":2300444696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":480706560,"data_stored":480706560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":494993408,"data_stored":494993408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2082410496,"data_stored":2082406424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1075445760,"data_stored":1075445760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1133318144,"data_stored":1133314072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":785317888,"data_stored":785317888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:46:49.631 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:46:49.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:49.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:49.711 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:49.733 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:49.736 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:49.770 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:49.816 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:46:49.816 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:46:49.816 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:49 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:49.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:46:49.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:46:49.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:49 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:49.825 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:49.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:49.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:49.930 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:49.960 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:50.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:46:50.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:46:50.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:49 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:46:50.106 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:50.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:50.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:50.210 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:50.250 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:50.310 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:50.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:50.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:50.424 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:50.456 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:50.506 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:50.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:50.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:50.576 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:46:50.610 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:50.647 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:50.704 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:50.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:50.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:50.810 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:50.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:50 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/993415283' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:46:50.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:50 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:46:50.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:50 smithi138 ceph-mon[185886]: pgmap v1632: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 5726/115140 objects degraded (4.973%); 2317/115140 objects misplaced (2.012%); 973 KiB/s, 15 objects/s recovering 2023-10-01T09:46:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:50 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/993415283' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:46:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:50 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:46:50.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:50 smithi138 ceph-mon[192513]: pgmap v1632: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 5726/115140 objects degraded (4.973%); 2317/115140 objects misplaced (2.012%); 973 KiB/s, 15 objects/s recovering 2023-10-01T09:46:50.852 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:50 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/993415283' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:46:50.852 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:50 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:46:50.852 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:50 smithi190 ceph-mon[160435]: pgmap v1632: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 208 op/s; 5726/115140 objects degraded (4.973%); 2317/115140 objects misplaced (2.012%); 973 KiB/s, 15 objects/s recovering 2023-10-01T09:46:50.852 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:50.939 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:50.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:50.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:51.044 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:51.079 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:51.696 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:51.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:51.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:51.799 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:51 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:51 smithi138 ceph-mon[185886]: failed to encode map e548 with expected crc 2023-10-01T09:46:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:51 smithi138 ceph-mon[185886]: failed to encode map e548 with expected crc 2023-10-01T09:46:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:51 smithi138 ceph-mon[185886]: failed to encode map e548 with expected crc 2023-10-01T09:46:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:51 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:46:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:51 smithi138 ceph-mon[185886]: osdmap e548: 8 total, 8 up, 7 in 2023-10-01T09:46:51.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:51 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:51.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:51 smithi138 ceph-mon[192513]: failed to encode map e548 with expected crc 2023-10-01T09:46:51.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:51 smithi138 ceph-mon[192513]: failed to encode map e548 with expected crc 2023-10-01T09:46:51.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:51 smithi138 ceph-mon[192513]: failed to encode map e548 with expected crc 2023-10-01T09:46:51.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:51 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:46:51.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:51 smithi138 ceph-mon[192513]: osdmap e548: 8 total, 8 up, 7 in 2023-10-01T09:46:51.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:51 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:46:51.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:51 smithi190 ceph-mon[160435]: failed to encode map e548 with expected crc 2023-10-01T09:46:51.823 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:51 smithi190 ceph-mon[160435]: failed to encode map e548 with expected crc 2023-10-01T09:46:51.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:51 smithi190 ceph-mon[160435]: failed to encode map e548 with expected crc 2023-10-01T09:46:51.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:51 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:46:51.824 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:51 smithi190 ceph-mon[160435]: osdmap e548: 8 total, 8 up, 7 in 2023-10-01T09:46:51.826 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:51.869 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:51.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:51.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:51.974 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:52.002 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:52.051 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:52.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:52.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:52.158 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:52.190 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:52.240 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:52.440 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:52.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:52.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:52.568 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:52.609 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:52.635 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:46:52.658 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:46:52.673 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:52.679 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:46:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:52.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:52.706 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:46:52.716 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:46:52.717 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:46:52.729 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:46:52.742 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:52 smithi138 ceph-mon[192513]: pgmap v1634: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 5726/112758 objects degraded (5.078%); 2317/112758 objects misplaced (2.055%); 982 KiB/s, 15 objects/s recovering 2023-10-01T09:46:52.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:52 smithi138 ceph-mon[185886]: pgmap v1634: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 5726/112758 objects degraded (5.078%); 2317/112758 objects misplaced (2.055%); 982 KiB/s, 15 objects/s recovering 2023-10-01T09:46:52.753 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:46:52.775 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:46:52.782 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:52.802 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:46:52.803 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:46:52.803 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:46:52.803 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:46:52.803 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:46:52.803 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:46:52.804 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:46:52.804 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:46:52.804 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:46:52.810 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:52.872 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:52.881 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:52 smithi190 ceph-mon[160435]: pgmap v1634: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 239 op/s; 5726/112758 objects degraded (5.078%); 2317/112758 objects misplaced (2.055%); 982 KiB/s, 15 objects/s recovering 2023-10-01T09:46:52.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:52.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:52.983 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:53.023 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:54.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/390093133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:54.058 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:53 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5726/112758 objects degraded (5.078%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:54.062 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/390093133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:54.062 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:53 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5726/112758 objects degraded (5.078%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:54.062 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/390093133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:46:54.062 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:53 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5726/112758 objects degraded (5.078%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:54.268 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:46:54.437 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:54.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:54.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:54.541 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:54.569 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:46:54.588 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:54.711 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:54 smithi138 ceph-mon[185886]: pgmap v1635: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 251 op/s; 5642/111540 objects degraded (5.058%); 2245/111540 objects misplaced (2.013%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T09:46:54.711 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:54 smithi138 ceph-mon[192513]: pgmap v1635: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 251 op/s; 5642/111540 objects degraded (5.058%); 2245/111540 objects misplaced (2.013%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T09:46:54.736 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:54.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:54.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:54.841 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:54.877 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:54.910 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:54.932 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:54.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:54.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:55.037 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:55.039 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:55.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:54 smithi190 ceph-mon[160435]: pgmap v1635: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 251 op/s; 5642/111540 objects degraded (5.058%); 2245/111540 objects misplaced (2.013%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T09:46:55.068 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:55.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:46:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:46:54] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-01T09:46:55.118 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:55.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:55.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:55.223 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:46:55.262 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:55.284 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:46:55.284 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5642/111540 objects degraded (5.058%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2236,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":548,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":37180,"data_bytes":2439518155,"bytes_used":13291474944,"bytes_avail":658673774592,"bytes_total":671965249536,"degraded_objects":5642,"degraded_total":111540,"degraded_ratio":0.05058275058275058,"misplaced_objects":2245,"misplaced_total":111540,"misplaced_ratio":0.020127308588847051,"recovering_objects_per_sec":16,"recovering_bytes_per_sec":1087565,"recovering_keys_per_sec":0,"num_objects_recovered":166,"num_bytes_recovered":10878976,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":251},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":34,"modified":"2023-10-01T09:43:40.311181+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:46:55.876 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:55 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/972076624' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:46:55.876 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:46:55.877 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:55 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/972076624' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:46:55.877 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:46:56.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:55 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/972076624' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:46:56.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:46:56.522 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:46:56.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:56 smithi138 ceph-mon[192513]: pgmap v1636: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 259 op/s; 5642/109857 objects degraded (5.136%); 2245/109857 objects misplaced (2.044%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T09:46:56.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:56 smithi138 ceph-mon[185886]: pgmap v1636: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 259 op/s; 5642/109857 objects degraded (5.136%); 2245/109857 objects misplaced (2.044%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T09:46:57.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:56 smithi190 ceph-mon[160435]: pgmap v1636: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 259 op/s; 5642/109857 objects degraded (5.136%); 2245/109857 objects misplaced (2.044%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T09:46:57.079 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:46:57.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:57.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:46:57.108 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:46:57.109 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:46:57.184 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:57.210 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:57.262 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:57.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:57.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:57.372 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:46:57.401 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:57.449 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:46:57.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:57.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:46:57.557 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:57.585 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:57.629 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:57.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:57.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:57.737 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:57.767 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:57.829 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:57.844 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:46:57.844 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:46:57.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:57.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:57.947 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:46:57.961 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:58.015 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:58.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:57 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5642/109857 objects degraded (5.136%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:58.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:57 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5642/109857 objects degraded (5.136%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:58.115 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:46:58.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:58.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:46:58.220 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:58.252 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:57 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5642/109857 objects degraded (5.136%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:46:58.255 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:58.304 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:58.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:58.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:58.408 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:46:58.438 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:58.612 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:46:58.893 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:46:59.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:58 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2970773923' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:59.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:58 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:59.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:58 smithi190 ceph-mon[160435]: pgmap v1637: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 5571/107634 objects degraded (5.176%); 2149/107634 objects misplaced (1.997%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:46:59.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:58 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2970773923' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:59.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:58 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:59.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:58 smithi138 ceph-mon[185886]: pgmap v1637: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 5571/107634 objects degraded (5.176%); 2149/107634 objects misplaced (1.997%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:46:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:58 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2970773923' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:58 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:46:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:58 smithi138 ceph-mon[192513]: pgmap v1637: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 5571/107634 objects degraded (5.176%); 2149/107634 objects misplaced (1.997%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:46:59.373 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:46:59.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:59.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:46:59.407 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:46:59.478 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:46:59.509 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:59.561 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:46:59.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:59.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:46:59.670 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:46:59.706 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:59.778 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:46:59.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:59.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:46:59.884 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:46:59.911 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:46:59.954 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:46:59.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:46:59.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:46:59.988 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:59 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:46:59.988 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:59 smithi138 ceph-mon[192513]: failed to encode map e549 with expected crc 2023-10-01T09:46:59.988 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:59 smithi138 ceph-mon[192513]: failed to encode map e549 with expected crc 2023-10-01T09:46:59.988 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:59 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:46:59.988 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:59 smithi138 ceph-mon[192513]: osdmap e549: 8 total, 8 up, 7 in 2023-10-01T09:46:59.988 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:46:59 smithi138 ceph-mon[192513]: failed to encode map e549 with expected crc 2023-10-01T09:46:59.988 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:59 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:46:59.988 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:59 smithi138 ceph-mon[185886]: failed to encode map e549 with expected crc 2023-10-01T09:46:59.989 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:59 smithi138 ceph-mon[185886]: failed to encode map e549 with expected crc 2023-10-01T09:46:59.989 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:59 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:46:59.989 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:59 smithi138 ceph-mon[185886]: osdmap e549: 8 total, 8 up, 7 in 2023-10-01T09:46:59.989 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:46:59 smithi138 ceph-mon[185886]: failed to encode map e549 with expected crc 2023-10-01T09:47:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:59 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:47:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:59 smithi190 ceph-mon[160435]: failed to encode map e549 with expected crc 2023-10-01T09:47:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:59 smithi190 ceph-mon[160435]: failed to encode map e549 with expected crc 2023-10-01T09:47:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:59 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:47:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:59 smithi190 ceph-mon[160435]: osdmap e549: 8 total, 8 up, 7 in 2023-10-01T09:47:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:46:59 smithi190 ceph-mon[160435]: failed to encode map e549 with expected crc 2023-10-01T09:47:00.061 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:00.101 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:00.942 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:00.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:00.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:01.016 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:00 smithi138 ceph-mon[192513]: pgmap v1639: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 272 op/s; 5571/107124 objects degraded (5.201%); 2149/107124 objects misplaced (2.006%); 1.1 MiB/s, 18 objects/s recovering 2023-10-01T09:47:01.016 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:00 smithi138 ceph-mon[185886]: pgmap v1639: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 272 op/s; 5571/107124 objects degraded (5.201%); 2149/107124 objects misplaced (2.006%); 1.1 MiB/s, 18 objects/s recovering 2023-10-01T09:47:01.048 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:01.083 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:01.132 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:01.192 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:01.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:01.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:01.296 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:01.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:00 smithi190 ceph-mon[160435]: pgmap v1639: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 272 op/s; 5571/107124 objects degraded (5.201%); 2149/107124 objects misplaced (2.006%); 1.1 MiB/s, 18 objects/s recovering 2023-10-01T09:47:01.332 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:01.382 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:01.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:01.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:01.476 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:47:01.478 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:47:01.486 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:01.519 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:01.578 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:47:01.611 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:47:01.640 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:47:01.662 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:47:01.684 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:47:01.702 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:47:01.722 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:47:01.858 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:47:01.859 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:47:01.859 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:47:01.859 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:47:01.859 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:47:01.859 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:47:01.859 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:47:01.859 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:47:01.859 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:47:02.277 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:01 smithi138 ceph-mon[192513]: from='client.37338 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:02.277 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/402073307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:02.277 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:01 smithi138 ceph-mon[185886]: from='client.37338 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:02.277 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/402073307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:02.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:01 smithi190 ceph-mon[160435]: from='client.37338 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:02.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/402073307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:02.745 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:47:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:02 smithi138 ceph-mon[192513]: pgmap v1640: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 269 op/s; 5571/104673 objects degraded (5.322%); 2149/104673 objects misplaced (2.053%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:47:03.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:02 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5571/107124 objects degraded (5.201%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:03.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:02 smithi138 ceph-mon[185886]: pgmap v1640: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 269 op/s; 5571/104673 objects degraded (5.322%); 2149/104673 objects misplaced (2.053%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:47:03.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:02 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5571/107124 objects degraded (5.201%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:03.136 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:03.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:03.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:03.223 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1639,"stamp":"2023-10-01T09:47:00.369305+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2343049163,"num_objects":35708,"num_object_clones":0,"num_object_copies":107124,"num_objects_missing_on_primary":646,"num_objects_missing":646,"num_objects_degraded":5571,"num_objects_misplaced":2149,"num_objects_unfound":0,"num_objects_dirty":35708,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":131866,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10596,"num_bytes_recovered":681280527,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122430,"ondisk_log_size":122430,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12732620,"kb_used_data":7611220,"kb_used_omap":39,"kb_used_meta":5121304,"kb_avail":643483444,"statfs":{"total":671965249536,"available":658927046656,"internally_reserved":0,"allocated":7793889280,"data_stored":7783890898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5244215919},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-175112192,"num_objects":-2672,"num_object_clones":0,"num_object_copies":-8016,"num_objects_missing_on_primary":-145,"num_objects_missing":-145,"num_objects_degraded":-155,"num_objects_misplaced":-168,"num_objects_unfound":0,"num_objects_dirty":-2672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2672,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":178,"num_bytes_recovered":11665408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.802089"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":324,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.508111+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:46:42.508111+0000","last_peered":"2023-10-01T09:46:42.508111+0000","last_clean":"2023-10-01T09:46:42.508111+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:46:42.508111+0000","last_undegraded":"2023-10-01T09:46:42.508111+0000","last_fullsized":"2023-10-01T09:46:42.508111+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":169,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.617577+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:46:50.617577+0000","last_peered":"2023-10-01T09:46:50.617577+0000","last_clean":"2023-10-01T09:46:50.617577+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:46:50.617577+0000","last_undegraded":"2023-10-01T09:46:50.617577+0000","last_fullsized":"2023-10-01T09:46:50.617577+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":550,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.617396+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:46:50.617396+0000","last_peered":"2023-10-01T09:46:50.617396+0000","last_clean":"2023-10-01T09:46:50.617396+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:46:50.617396+0000","last_undegraded":"2023-10-01T09:46:50.617396+0000","last_fullsized":"2023-10-01T09:46:50.617396+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":607,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.467793+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:46:42.467793+0000","last_peered":"2023-10-01T09:46:42.467793+0000","last_clean":"2023-10-01T09:46:42.467793+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:46:42.467793+0000","last_undegraded":"2023-10-01T09:46:42.467793+0000","last_fullsized":"2023-10-01T09:46:42.467793+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":488,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.467375+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:46:42.467375+0000","last_peered":"2023-10-01T09:46:42.467375+0000","last_clean":"2023-10-01T09:46:42.467375+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:46:42.467375+0000","last_undegraded":"2023-10-01T09:46:42.467375+0000","last_fullsized":"2023-10-01T09:46:42.467375+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":229,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.604514+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:46:42.604514+0000","last_peered":"2023-10-01T09:46:42.604514+0000","last_clean":"2023-10-01T09:46:42.604514+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:46:42.604514+0000","last_undegraded":"2023-10-01T09:46:42.604514+0000","last_fullsized":"2023-10-01T09:46:42.604514+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":605,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.604602+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:46:42.604602+0000","last_peered":"2023-10-01T09:46:42.604602+0000","last_clean":"2023-10-01T09:46:42.604602+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:46:42.604602+0000","last_undegraded":"2023-10-01T09:46:42.604602+0000","last_fullsized":"2023-10-01T09:46:42.604602+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":535,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.468069+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:46:42.468069+0000","last_peered":"2023-10-01T09:46:42.468069+0000","last_clean":"2023-10-01T09:46:42.468069+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:46:42.468069+0000","last_undegraded":"2023-10-01T09:46:42.468069+0000","last_fullsized":"2023-10-01T09:46:42.468069+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":260,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463906+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:46:42.463906+0000","last_peered":"2023-10-01T09:46:42.463906+0000","last_clean":"2023-10-01T09:46:42.463906+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:46:42.463906+0000","last_undegraded":"2023-10-01T09:46:42.463906+0000","last_fullsized":"2023-10-01T09:46:42.463906+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":67,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.514097+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:46:42.514097+0000","last_peered":"2023-10-01T09:46:42.514097+0000","last_clean":"2023-10-01T09:46:42.514097+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:46:42.514097+0000","last_undegraded":"2023-10-01T09:46:42.514097+0000","last_fullsized":"2023-10-01T09:46:42.514097+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":73,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.516211+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:46:42.516211+0000","last_peered":"2023-10-01T09:46:42.516211+0000","last_clean":"2023-10-01T09:46:42.516211+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:46:42.516211+0000","last_undegraded":"2023-10-01T09:46:42.516211+0000","last_fullsized":"2023-10-01T09:46:42.516211+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":73,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.617399+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:46:50.617399+0000","last_peered":"2023-10-01T09:46:50.617399+0000","last_clean":"2023-10-01T09:46:50.617399+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:46:50.617399+0000","last_undegraded":"2023-10-01T09:46:50.617399+0000","last_fullsized":"2023-10-01T09:46:50.617399+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":449,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.508393+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:46:42.508393+0000","last_peered":"2023-10-01T09:46:42.508393+0000","last_clean":"2023-10-01T09:46:42.508393+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:46:42.508393+0000","last_undegraded":"2023-10-01T09:46:42.508393+0000","last_fullsized":"2023-10-01T09:46:42.508393+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":330,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.508237+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:46:42.508237+0000","last_peered":"2023-10-01T09:46:42.508237+0000","last_clean":"2023-10-01T09:46:42.508237+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:46:42.508237+0000","last_undegraded":"2023-10-01T09:46:42.508237+0000","last_fullsized":"2023-10-01T09:46:42.508237+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":666,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.514763+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:46:42.514763+0000","last_peered":"2023-10-01T09:46:42.514763+0000","last_clean":"2023-10-01T09:46:42.514763+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:46:42.514763+0000","last_undegraded":"2023-10-01T09:46:42.514763+0000","last_fullsized":"2023-10-01T09:46:42.514763+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":403,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.514650+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:46:42.514650+0000","last_peered":"2023-10-01T09:46:42.514650+0000","last_clean":"2023-10-01T09:46:42.514650+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:46:42.514650+0000","last_undegraded":"2023-10-01T09:46:42.514650+0000","last_fullsized":"2023-10-01T09:46:42.514650+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":76,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.514264+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:46:42.514264+0000","last_peered":"2023-10-01T09:46:42.514264+0000","last_clean":"2023-10-01T09:46:42.514264+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:46:42.514264+0000","last_undegraded":"2023-10-01T09:46:42.514264+0000","last_fullsized":"2023-10-01T09:46:42.514264+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":62,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463675+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:46:42.463675+0000","last_peered":"2023-10-01T09:46:42.463675+0000","last_clean":"2023-10-01T09:46:42.463675+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:46:42.463675+0000","last_undegraded":"2023-10-01T09:46:42.463675+0000","last_fullsized":"2023-10-01T09:46:42.463675+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":559,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.514377+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:46:42.514377+0000","last_peered":"2023-10-01T09:46:42.514377+0000","last_clean":"2023-10-01T09:46:42.514377+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:46:42.514377+0000","last_undegraded":"2023-10-01T09:46:42.514377+0000","last_fullsized":"2023-10-01T09:46:42.514377+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":571,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.467839+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:46:42.467839+0000","last_peered":"2023-10-01T09:46:42.467839+0000","last_clean":"2023-10-01T09:46:42.467839+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:46:42.467839+0000","last_undegraded":"2023-10-01T09:46:42.467839+0000","last_fullsized":"2023-10-01T09:46:42.467839+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":427,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.467284+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:46:42.467284+0000","last_peered":"2023-10-01T09:46:42.467284+0000","last_clean":"2023-10-01T09:46:42.467284+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:46:42.467284+0000","last_undegraded":"2023-10-01T09:46:42.467284+0000","last_fullsized":"2023-10-01T09:46:42.467284+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":76,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.514448+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:46:42.514448+0000","last_peered":"2023-10-01T09:46:42.514448+0000","last_clean":"2023-10-01T09:46:42.514448+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:46:42.514448+0000","last_undegraded":"2023-10-01T09:46:42.514448+0000","last_fullsized":"2023-10-01T09:46:42.514448+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"548'8165","reported_seq":8246,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:57.379392+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:46:57.379392+0000","last_peered":"2023-10-01T09:46:57.379392+0000","last_clean":"2023-10-01T09:46:57.379392+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:46:57.379392+0000","last_undegraded":"2023-10-01T09:46:57.379392+0000","last_fullsized":"2023-10-01T09:46:57.379392+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8165,"log_dups_size":0,"ondisk_log_size":8165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145948672,"num_objects":2227,"num_object_clones":0,"num_object_copies":6681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8165,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":71,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.514278+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:46:42.514278+0000","last_peered":"2023-10-01T09:46:42.514278+0000","last_clean":"2023-10-01T09:46:42.514278+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:46:42.514278+0000","last_undegraded":"2023-10-01T09:46:42.514278+0000","last_fullsized":"2023-10-01T09:46:42.514278+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":268,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463460+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:46:42.463460+0000","last_peered":"2023-10-01T09:46:42.463460+0000","last_clean":"2023-10-01T09:46:42.463460+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:46:42.463460+0000","last_undegraded":"2023-10-01T09:46:42.463460+0000","last_fullsized":"2023-10-01T09:46:42.463460+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"548'8241","reported_seq":17468,"reported_epoch":548,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:46:57.545559+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:46:57.545559+0000","last_peered":"2023-10-01T09:46:57.545559+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:46:57.545559+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8241,"log_dups_size":0,"ondisk_log_size":8241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141885440,"num_objects":2165,"num_object_clones":0,"num_object_copies":6495,"num_objects_missing_on_primary":646,"num_objects_missing":646,"num_objects_degraded":2742,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8241,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3039,"num_bytes_recovered":199163904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1450},{"shards":"0,4,6","objects":646},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":542,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.514561+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:46:42.514561+0000","last_peered":"2023-10-01T09:46:42.514561+0000","last_clean":"2023-10-01T09:46:42.514561+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:46:42.514561+0000","last_undegraded":"2023-10-01T09:46:42.514561+0000","last_fullsized":"2023-10-01T09:46:42.514561+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":605,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.617386+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:46:50.617386+0000","last_peered":"2023-10-01T09:46:50.617386+0000","last_clean":"2023-10-01T09:46:50.617386+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:46:50.617386+0000","last_undegraded":"2023-10-01T09:46:50.617386+0000","last_fullsized":"2023-10-01T09:46:50.617386+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"548'8179","reported_seq":8264,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:56.848504+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:46:56.848504+0000","last_peered":"2023-10-01T09:46:56.848504+0000","last_clean":"2023-10-01T09:46:56.848504+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:46:56.848504+0000","last_undegraded":"2023-10-01T09:46:56.848504+0000","last_fullsized":"2023-10-01T09:46:56.848504+0000","mapping_epoch":475,"log_start":"481'1200","ondisk_log_start":"481'1200","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147652608,"num_objects":2253,"num_object_clones":0,"num_object_copies":6759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8179,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":628,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.516265+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:46:42.516265+0000","last_peered":"2023-10-01T09:46:42.516265+0000","last_clean":"2023-10-01T09:46:42.516265+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:46:42.516265+0000","last_undegraded":"2023-10-01T09:46:42.516265+0000","last_fullsized":"2023-10-01T09:46:42.516265+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":426,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.617516+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:46:50.617516+0000","last_peered":"2023-10-01T09:46:50.617516+0000","last_clean":"2023-10-01T09:46:50.617516+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:46:50.617516+0000","last_undegraded":"2023-10-01T09:46:50.617516+0000","last_fullsized":"2023-10-01T09:46:50.617516+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"548'8211","reported_seq":14901,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:56.832430+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:46:56.832430+0000","last_peered":"2023-10-01T09:46:56.832430+0000","last_clean":"2023-10-01T09:46:56.832430+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:46:56.832430+0000","last_undegraded":"2023-10-01T09:46:56.832430+0000","last_fullsized":"2023-10-01T09:46:56.832430+0000","mapping_epoch":505,"log_start":"481'1200","ondisk_log_start":"481'1200","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7011,"log_dups_size":0,"ondisk_log_size":7011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143589376,"num_objects":2191,"num_object_clones":0,"num_object_copies":6573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8211,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":268,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463762+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:46:42.463762+0000","last_peered":"2023-10-01T09:46:42.463762+0000","last_clean":"2023-10-01T09:46:42.463762+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:46:42.463762+0000","last_undegraded":"2023-10-01T09:46:42.463762+0000","last_fullsized":"2023-10-01T09:46:42.463762+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":528,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.574795+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:46:50.574795+0000","last_peered":"2023-10-01T09:46:50.574795+0000","last_clean":"2023-10-01T09:46:50.574795+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:46:50.574795+0000","last_undegraded":"2023-10-01T09:46:50.574795+0000","last_fullsized":"2023-10-01T09:46:50.574795+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"549'8259","reported_seq":8345,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:47:00.096897+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:47:00.096897+0000","last_peered":"2023-10-01T09:47:00.096897+0000","last_clean":"2023-10-01T09:47:00.096897+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:47:00.096897+0000","last_undegraded":"2023-10-01T09:47:00.096897+0000","last_fullsized":"2023-10-01T09:47:00.096897+0000","mapping_epoch":475,"log_start":"481'700","ondisk_log_start":"481'700","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7559,"log_dups_size":0,"ondisk_log_size":7559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147390464,"num_objects":2249,"num_object_clones":0,"num_object_copies":6747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8259,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":128,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.508565+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:46:42.508565+0000","last_peered":"2023-10-01T09:46:42.508565+0000","last_clean":"2023-10-01T09:46:42.508565+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:46:42.508565+0000","last_undegraded":"2023-10-01T09:46:42.508565+0000","last_fullsized":"2023-10-01T09:46:42.508565+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":202,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.508463+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:46:42.508463+0000","last_peered":"2023-10-01T09:46:42.508463+0000","last_clean":"2023-10-01T09:46:42.508463+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:46:42.508463+0000","last_undegraded":"2023-10-01T09:46:42.508463+0000","last_fullsized":"2023-10-01T09:46:42.508463+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"548'8155","reported_seq":8550,"reported_epoch":548,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:46:57.484847+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:46:57.484847+0000","last_peered":"2023-10-01T09:46:57.484847+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:46:57.484847+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8155,"log_dups_size":0,"ondisk_log_size":8155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145031168,"num_objects":2213,"num_object_clones":0,"num_object_copies":6639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2819,"num_objects_misplaced":2149,"num_objects_unfound":0,"num_objects_dirty":2213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8155,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2149},{"shards":"4,5","objects":-2755}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":446,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.508213+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:46:42.508213+0000","last_peered":"2023-10-01T09:46:42.508213+0000","last_clean":"2023-10-01T09:46:42.508213+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:46:42.508213+0000","last_undegraded":"2023-10-01T09:46:42.508213+0000","last_fullsized":"2023-10-01T09:46:42.508213+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":609,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.617685+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:46:50.617685+0000","last_peered":"2023-10-01T09:46:50.617685+0000","last_clean":"2023-10-01T09:46:50.617685+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:46:50.617685+0000","last_undegraded":"2023-10-01T09:46:50.617685+0000","last_fullsized":"2023-10-01T09:46:50.617685+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"548'8158","reported_seq":8248,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:57.417714+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:46:57.417714+0000","last_peered":"2023-10-01T09:46:57.417714+0000","last_clean":"2023-10-01T09:46:57.417714+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:46:57.417714+0000","last_undegraded":"2023-10-01T09:46:57.417714+0000","last_fullsized":"2023-10-01T09:46:57.417714+0000","mapping_epoch":483,"log_start":"481'600","ondisk_log_start":"481'600","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7558,"log_dups_size":0,"ondisk_log_size":7558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145489920,"num_objects":2220,"num_object_clones":0,"num_object_copies":6660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8158,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":442,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.508077+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:46:42.508077+0000","last_peered":"2023-10-01T09:46:42.508077+0000","last_clean":"2023-10-01T09:46:42.508077+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:46:42.508077+0000","last_undegraded":"2023-10-01T09:46:42.508077+0000","last_fullsized":"2023-10-01T09:46:42.508077+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":283,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.508616+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:46:42.508616+0000","last_peered":"2023-10-01T09:46:42.508616+0000","last_clean":"2023-10-01T09:46:42.508616+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:46:42.508616+0000","last_undegraded":"2023-10-01T09:46:42.508616+0000","last_fullsized":"2023-10-01T09:46:42.508616+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"548'8247","reported_seq":8328,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:56.075605+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:46:56.075605+0000","last_peered":"2023-10-01T09:46:56.075605+0000","last_clean":"2023-10-01T09:46:56.075605+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:46:56.075605+0000","last_undegraded":"2023-10-01T09:46:56.075605+0000","last_fullsized":"2023-10-01T09:46:56.075605+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8247,"log_dups_size":0,"ondisk_log_size":8247,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147521536,"num_objects":2251,"num_object_clones":0,"num_object_copies":6753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8247,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":413,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.617343+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:46:50.617343+0000","last_peered":"2023-10-01T09:46:50.617343+0000","last_clean":"2023-10-01T09:46:50.617343+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:46:50.617343+0000","last_undegraded":"2023-10-01T09:46:50.617343+0000","last_fullsized":"2023-10-01T09:46:50.617343+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":460,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.617815+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:46:50.617815+0000","last_peered":"2023-10-01T09:46:50.617815+0000","last_clean":"2023-10-01T09:46:50.617815+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:46:50.617815+0000","last_undegraded":"2023-10-01T09:46:50.617815+0000","last_fullsized":"2023-10-01T09:46:50.617815+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"548'8003","reported_seq":8084,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:56.080740+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:46:56.080740+0000","last_peered":"2023-10-01T09:46:56.080740+0000","last_clean":"2023-10-01T09:46:56.080740+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:46:56.080740+0000","last_undegraded":"2023-10-01T09:46:56.080740+0000","last_fullsized":"2023-10-01T09:46:56.080740+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8003,"log_dups_size":0,"ondisk_log_size":8003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142540800,"num_objects":2175,"num_object_clones":0,"num_object_copies":6525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8003,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":941,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.514442+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:46:42.514442+0000","last_peered":"2023-10-01T09:46:42.514442+0000","last_clean":"2023-10-01T09:46:42.514442+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:46:42.514442+0000","last_undegraded":"2023-10-01T09:46:42.514442+0000","last_fullsized":"2023-10-01T09:46:42.514442+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":278,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463817+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:46:42.463817+0000","last_peered":"2023-10-01T09:46:42.463817+0000","last_clean":"2023-10-01T09:46:42.463817+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:46:42.463817+0000","last_undegraded":"2023-10-01T09:46:42.463817+0000","last_fullsized":"2023-10-01T09:46:42.463817+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":74,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.903624+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:46:58.903624+0000","last_peered":"2023-10-01T09:46:58.903624+0000","last_clean":"2023-10-01T09:46:58.903624+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:46:58.903624+0000","last_undegraded":"2023-10-01T09:46:58.903624+0000","last_fullsized":"2023-10-01T09:46:58.903624+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"548'8095","reported_seq":8180,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:56.901259+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:46:56.901259+0000","last_peered":"2023-10-01T09:46:56.901259+0000","last_clean":"2023-10-01T09:46:56.901259+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:46:56.901259+0000","last_undegraded":"2023-10-01T09:46:56.901259+0000","last_fullsized":"2023-10-01T09:46:56.901259+0000","mapping_epoch":475,"log_start":"481'1100","ondisk_log_start":"481'1100","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149094400,"num_objects":2275,"num_object_clones":0,"num_object_copies":6825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8095,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1213,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463583+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:46:42.463583+0000","last_peered":"2023-10-01T09:46:42.463583+0000","last_clean":"2023-10-01T09:46:42.463583+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:46:42.463583+0000","last_undegraded":"2023-10-01T09:46:42.463583+0000","last_fullsized":"2023-10-01T09:46:42.463583+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":423,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.508555+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:46:42.508555+0000","last_peered":"2023-10-01T09:46:42.508555+0000","last_clean":"2023-10-01T09:46:42.508555+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:46:42.508555+0000","last_undegraded":"2023-10-01T09:46:42.508555+0000","last_fullsized":"2023-10-01T09:46:42.508555+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":251,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463912+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:46:42.463912+0000","last_peered":"2023-10-01T09:46:42.463912+0000","last_clean":"2023-10-01T09:46:42.463912+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:46:42.463912+0000","last_undegraded":"2023-10-01T09:46:42.463912+0000","last_fullsized":"2023-10-01T09:46:42.463912+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"548'8125","reported_seq":15427,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:56.954586+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:46:56.954586+0000","last_peered":"2023-10-01T09:46:56.954586+0000","last_clean":"2023-10-01T09:46:56.954586+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:46:56.954586+0000","last_undegraded":"2023-10-01T09:46:56.954586+0000","last_fullsized":"2023-10-01T09:46:56.954586+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8125,"log_dups_size":0,"ondisk_log_size":8125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147521536,"num_objects":2251,"num_object_clones":0,"num_object_copies":6753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8125,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":894,"reported_epoch":547,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:46:42.466984+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:46:42.466984+0000","last_peered":"2023-10-01T09:46:42.466984+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:46:42.466984+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:46:42.466984+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":613,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.617405+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:46:50.617405+0000","last_peered":"2023-10-01T09:46:50.617405+0000","last_clean":"2023-10-01T09:46:50.617405+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:46:50.617405+0000","last_undegraded":"2023-10-01T09:46:50.617405+0000","last_fullsized":"2023-10-01T09:46:50.617405+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":73,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.617155+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:46:50.617155+0000","last_peered":"2023-10-01T09:46:50.617155+0000","last_clean":"2023-10-01T09:46:50.617155+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:46:50.617155+0000","last_undegraded":"2023-10-01T09:46:50.617155+0000","last_fullsized":"2023-10-01T09:46:50.617155+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"549'8134","reported_seq":8220,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:47:00.247846+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:47:00.247846+0000","last_peered":"2023-10-01T09:47:00.247846+0000","last_clean":"2023-10-01T09:47:00.247846+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:47:00.247846+0000","last_undegraded":"2023-10-01T09:47:00.247846+0000","last_fullsized":"2023-10-01T09:47:00.247846+0000","mapping_epoch":475,"log_start":"481'600","ondisk_log_start":"481'600","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7534,"log_dups_size":0,"ondisk_log_size":7534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140115968,"num_objects":2138,"num_object_clones":0,"num_object_copies":6414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8134,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1664,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463628+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:46:42.463628+0000","last_peered":"2023-10-01T09:46:42.463628+0000","last_clean":"2023-10-01T09:46:42.463628+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:46:42.463628+0000","last_undegraded":"2023-10-01T09:46:42.463628+0000","last_fullsized":"2023-10-01T09:46:42.463628+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":578,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.514097+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:46:42.514097+0000","last_peered":"2023-10-01T09:46:42.514097+0000","last_clean":"2023-10-01T09:46:42.514097+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:46:42.514097+0000","last_undegraded":"2023-10-01T09:46:42.514097+0000","last_fullsized":"2023-10-01T09:46:42.514097+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":356,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463572+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:46:42.463572+0000","last_peered":"2023-10-01T09:46:42.463572+0000","last_clean":"2023-10-01T09:46:42.463572+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:46:42.463572+0000","last_undegraded":"2023-10-01T09:46:42.463572+0000","last_fullsized":"2023-10-01T09:46:42.463572+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":346,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.508741+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:46:42.508741+0000","last_peered":"2023-10-01T09:46:42.508741+0000","last_clean":"2023-10-01T09:46:42.508741+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:46:42.508741+0000","last_undegraded":"2023-10-01T09:46:42.508741+0000","last_fullsized":"2023-10-01T09:46:42.508741+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"548'8266","reported_seq":8347,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:56.102279+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:46:56.102279+0000","last_peered":"2023-10-01T09:46:56.102279+0000","last_clean":"2023-10-01T09:46:56.102279+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:46:56.102279+0000","last_undegraded":"2023-10-01T09:46:56.102279+0000","last_fullsized":"2023-10-01T09:46:56.102279+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8266,"log_dups_size":0,"ondisk_log_size":8266,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148504576,"num_objects":2266,"num_object_clones":0,"num_object_copies":6798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8266,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":755,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.514189+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:46:42.514189+0000","last_peered":"2023-10-01T09:46:42.514189+0000","last_clean":"2023-10-01T09:46:42.514189+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:46:42.514189+0000","last_undegraded":"2023-10-01T09:46:42.514189+0000","last_fullsized":"2023-10-01T09:46:42.514189+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":589,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.467413+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:46:42.467413+0000","last_peered":"2023-10-01T09:46:42.467413+0000","last_clean":"2023-10-01T09:46:42.467413+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:46:42.467413+0000","last_undegraded":"2023-10-01T09:46:42.467413+0000","last_fullsized":"2023-10-01T09:46:42.467413+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":588,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.467644+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:46:42.467644+0000","last_peered":"2023-10-01T09:46:42.467644+0000","last_clean":"2023-10-01T09:46:42.467644+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:46:42.467644+0000","last_undegraded":"2023-10-01T09:46:42.467644+0000","last_fullsized":"2023-10-01T09:46:42.467644+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"548'8184","reported_seq":8270,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:56.907973+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:46:56.907973+0000","last_peered":"2023-10-01T09:46:56.907973+0000","last_clean":"2023-10-01T09:46:56.907973+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:46:56.907973+0000","last_undegraded":"2023-10-01T09:46:56.907973+0000","last_fullsized":"2023-10-01T09:46:56.907973+0000","mapping_epoch":475,"log_start":"481'1200","ondisk_log_start":"481'1200","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149094424,"num_objects":2276,"num_object_clones":0,"num_object_copies":6828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2276,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8184,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":986,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.516304+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:46:42.516304+0000","last_peered":"2023-10-01T09:46:42.516304+0000","last_clean":"2023-10-01T09:46:42.516304+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:46:42.516304+0000","last_undegraded":"2023-10-01T09:46:42.516304+0000","last_fullsized":"2023-10-01T09:46:42.516304+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":605,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463839+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:46:42.463839+0000","last_peered":"2023-10-01T09:46:42.463839+0000","last_clean":"2023-10-01T09:46:42.463839+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:46:42.463839+0000","last_undegraded":"2023-10-01T09:46:42.463839+0000","last_fullsized":"2023-10-01T09:46:42.463839+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":458,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463723+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:46:42.463723+0000","last_peered":"2023-10-01T09:46:42.463723+0000","last_clean":"2023-10-01T09:46:42.463723+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:46:42.463723+0000","last_undegraded":"2023-10-01T09:46:42.463723+0000","last_fullsized":"2023-10-01T09:46:42.463723+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"548'8130","reported_seq":8219,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:56.032403+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:46:56.032403+0000","last_peered":"2023-10-01T09:46:56.032403+0000","last_clean":"2023-10-01T09:46:56.032403+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:46:56.032403+0000","last_undegraded":"2023-10-01T09:46:56.032403+0000","last_fullsized":"2023-10-01T09:46:56.032403+0000","mapping_epoch":479,"log_start":"482'1400","ondisk_log_start":"482'1400","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6730,"log_dups_size":0,"ondisk_log_size":6730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145883136,"num_objects":2226,"num_object_clones":0,"num_object_copies":6678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8130,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":487,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.514531+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:46:42.514531+0000","last_peered":"2023-10-01T09:46:42.514531+0000","last_clean":"2023-10-01T09:46:42.514531+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:46:42.514531+0000","last_undegraded":"2023-10-01T09:46:42.514531+0000","last_fullsized":"2023-10-01T09:46:42.514531+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1183,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.617805+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:46:50.617805+0000","last_peered":"2023-10-01T09:46:50.617805+0000","last_clean":"2023-10-01T09:46:50.617805+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:46:50.617805+0000","last_undegraded":"2023-10-01T09:46:50.617805+0000","last_fullsized":"2023-10-01T09:46:50.617805+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":76,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.514032+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:46:42.514032+0000","last_peered":"2023-10-01T09:46:42.514032+0000","last_clean":"2023-10-01T09:46:42.514032+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:46:42.514032+0000","last_undegraded":"2023-10-01T09:46:42.514032+0000","last_fullsized":"2023-10-01T09:46:42.514032+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"548'8208","reported_seq":8297,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:56.003434+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:46:56.003434+0000","last_peered":"2023-10-01T09:46:56.003434+0000","last_clean":"2023-10-01T09:46:56.003434+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:46:56.003434+0000","last_undegraded":"2023-10-01T09:46:56.003434+0000","last_fullsized":"2023-10-01T09:46:56.003434+0000","mapping_epoch":479,"log_start":"482'1500","ondisk_log_start":"482'1500","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6708,"log_dups_size":0,"ondisk_log_size":6708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150994944,"num_objects":2304,"num_object_clones":0,"num_object_copies":6912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8208,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":80,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.514369+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:46:42.514369+0000","last_peered":"2023-10-01T09:46:42.514369+0000","last_clean":"2023-10-01T09:46:42.514369+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:46:42.514369+0000","last_undegraded":"2023-10-01T09:46:42.514369+0000","last_fullsized":"2023-10-01T09:46:42.514369+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":566,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.467724+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:46:42.467724+0000","last_peered":"2023-10-01T09:46:42.467724+0000","last_clean":"2023-10-01T09:46:42.467724+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:46:42.467724+0000","last_undegraded":"2023-10-01T09:46:42.467724+0000","last_fullsized":"2023-10-01T09:46:42.467724+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":271,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463491+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:46:42.463491+0000","last_peered":"2023-10-01T09:46:42.463491+0000","last_clean":"2023-10-01T09:46:42.463491+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:46:42.463491+0000","last_undegraded":"2023-10-01T09:46:42.463491+0000","last_fullsized":"2023-10-01T09:46:42.463491+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":515,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.468159+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:46:42.468159+0000","last_peered":"2023-10-01T09:46:42.468159+0000","last_clean":"2023-10-01T09:46:42.468159+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:46:42.468159+0000","last_undegraded":"2023-10-01T09:46:42.468159+0000","last_fullsized":"2023-10-01T09:46:42.468159+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":630,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.617611+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:46:50.617611+0000","last_peered":"2023-10-01T09:46:50.617611+0000","last_clean":"2023-10-01T09:46:50.617611+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:46:50.617611+0000","last_undegraded":"2023-10-01T09:46:50.617611+0000","last_fullsized":"2023-10-01T09:46:50.617611+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":283,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463684+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:46:42.463684+0000","last_peered":"2023-10-01T09:46:42.463684+0000","last_clean":"2023-10-01T09:46:42.463684+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:46:42.463684+0000","last_undegraded":"2023-10-01T09:46:42.463684+0000","last_fullsized":"2023-10-01T09:46:42.463684+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":73,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.617469+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:46:50.617469+0000","last_peered":"2023-10-01T09:46:50.617469+0000","last_clean":"2023-10-01T09:46:50.617469+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:46:50.617469+0000","last_undegraded":"2023-10-01T09:46:50.617469+0000","last_fullsized":"2023-10-01T09:46:50.617469+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":283,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.463906+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:46:42.463906+0000","last_peered":"2023-10-01T09:46:42.463906+0000","last_clean":"2023-10-01T09:46:42.463906+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:46:42.463906+0000","last_undegraded":"2023-10-01T09:46:42.463906+0000","last_fullsized":"2023-10-01T09:46:42.463906+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":461,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.467499+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:46:42.467499+0000","last_peered":"2023-10-01T09:46:42.467499+0000","last_clean":"2023-10-01T09:46:42.467499+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:46:42.467499+0000","last_undegraded":"2023-10-01T09:46:42.467499+0000","last_fullsized":"2023-10-01T09:46:42.467499+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":91,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.617713+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:46:50.617713+0000","last_peered":"2023-10-01T09:46:50.617713+0000","last_clean":"2023-10-01T09:46:50.617713+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:46:50.617713+0000","last_undegraded":"2023-10-01T09:46:50.617713+0000","last_fullsized":"2023-10-01T09:46:50.617713+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":506,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.467883+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:46:42.467883+0000","last_peered":"2023-10-01T09:46:42.467883+0000","last_clean":"2023-10-01T09:46:42.467883+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:46:42.467883+0000","last_undegraded":"2023-10-01T09:46:42.467883+0000","last_fullsized":"2023-10-01T09:46:42.467883+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":706,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.467781+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:46:42.467781+0000","last_peered":"2023-10-01T09:46:42.467781+0000","last_clean":"2023-10-01T09:46:42.467781+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:46:42.467781+0000","last_undegraded":"2023-10-01T09:46:42.467781+0000","last_fullsized":"2023-10-01T09:46:42.467781+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":399,"reported_epoch":547,"state":"active+clean","last_fresh":"2023-10-01T09:46:42.507997+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:46:42.507997+0000","last_peered":"2023-10-01T09:46:42.507997+0000","last_clean":"2023-10-01T09:46:42.507997+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:46:42.507997+0000","last_undegraded":"2023-10-01T09:46:42.507997+0000","last_fullsized":"2023-10-01T09:46:42.507997+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2338258968,"num_objects":35680,"num_object_clones":0,"num_object_copies":107040,"num_objects_missing_on_primary":646,"num_objects_missing":646,"num_objects_degraded":5561,"num_objects_misplaced":2149,"num_objects_unfound":0,"num_objects_dirty":35680,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":130760,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10085,"num_bytes_recovered":660930560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7711567872,"data_stored":7711555656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121260,"ondisk_log_size":121260,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430756,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1625008,"kb_used_data":725992,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":92120144,"statfs":{"total":95995035648,"available":94331027456,"internally_reserved":0,"allocated":743415808,"data_stored":742470976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.74399999999999999,"15min":2.5840000000000001},"min":{"1min":0.27700000000000002,"5min":0.246,"15min":0.20999999999999999},"max":{"1min":1.081,"5min":6.8550000000000004,"15min":108.90000000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.52500000000000002,"15min":2.7050000000000001},"min":{"1min":0.25700000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.2090000000000001,"5min":5.6639999999999997,"15min":109.187},"last":0.42199999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.66400000000000003,"15min":5.1520000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.222},"max":{"1min":1.0780000000000001,"5min":6.8289999999999997,"15min":229.161},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.59799999999999998,"15min":5.1779999999999999},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":3.0640000000000001,"5min":5.7240000000000002,"15min":230.126},"last":0.44800000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.72799999999999998,"15min":1.617},"min":{"1min":0.30099999999999999,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":1.1259999999999999,"5min":5.3090000000000002,"15min":14.901999999999999},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.68500000000000005,"15min":1.6100000000000001},"min":{"1min":0.33700000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.0009999999999999,"5min":5.6929999999999996,"15min":14.682},"last":0.49099999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.65800000000000003,"15min":2.2000000000000002},"min":{"1min":0.35699999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.476,"5min":5.7750000000000004,"15min":76.555000000000007},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.68100000000000005,"15min":2.2010000000000001},"min":{"1min":0.32800000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.4529999999999998,"5min":6.548,"15min":76.534000000000006},"last":0.36299999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.70499999999999996,"15min":1.042},"min":{"1min":0.28299999999999997,"5min":0.26800000000000002,"15min":0.26700000000000002},"max":{"1min":1.276,"5min":6.8769999999999998,"15min":9.6020000000000003},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.68500000000000005,"15min":0.997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.97399999999999998,"5min":5.7439999999999998,"15min":8.4949999999999992},"last":0.28699999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.52500000000000002,"15min":1.5269999999999999},"min":{"1min":0.35399999999999998,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":2.2400000000000002,"5min":2.2400000000000002,"15min":135.40000000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.54500000000000004,"15min":1.627},"min":{"1min":0.34999999999999998,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":0.70699999999999996,"5min":2.8849999999999998,"15min":135.339},"last":0.30299999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.58999999999999997,"15min":1.1200000000000001},"min":{"1min":0.40500000000000003,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":1.7829999999999999,"5min":2.448,"15min":11.853999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.59699999999999998,"15min":1.093},"min":{"1min":0.39700000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":3.1909999999999998,"5min":3.7810000000000001,"15min":14.584},"last":0.438}]}]},{"osd":6,"up_from":294,"seq":1262720385267,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1351276,"kb_used_data":1009572,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":92393876,"statfs":{"total":95995035648,"available":94611329024,"internally_reserved":0,"allocated":1033801728,"data_stored":1032860847,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.45200000000000001,"15min":3.681},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20599999999999999},"max":{"1min":0.51700000000000002,"5min":3.3340000000000001,"15min":158.48599999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.55500000000000005,"15min":3.746},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.221},"max":{"1min":3.3519999999999999,"5min":3.3519999999999999,"15min":158.37700000000001},"last":0.46800000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.58499999999999996,"15min":4.5739999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":3.3069999999999999,"5min":3.3069999999999999,"15min":168.947},"last":0.497},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.60299999999999998,"15min":4.532},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":3.7469999999999999,"5min":3.7469999999999999,"15min":169.07900000000001},"last":0.434}]},{"osd":2,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.59999999999999998,"15min":1.4490000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":3.3849999999999998,"5min":3.3849999999999998,"15min":10.92},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":1.4690000000000001},"min":{"1min":0.32100000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.7160000000000002,"5min":3.7160000000000002,"15min":21.704999999999998},"last":0.40899999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.57999999999999996,"15min":1.8100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.63800000000000001,"5min":3.3759999999999999,"15min":89.753},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.69499999999999995,"15min":1.754},"min":{"1min":0.309,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.2050000000000001,"5min":4.2050000000000001,"15min":89.849999999999994},"last":0.55400000000000005}]},{"osd":4,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58899999999999997,"15min":1.069},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.6349999999999998,"5min":4.0250000000000004,"15min":63.134},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.65800000000000003,"15min":1.0680000000000001},"min":{"1min":0.36599999999999999,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":1.006,"5min":7.4039999999999999,"15min":62.973999999999997},"last":0.44800000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:46:37 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.88500000000000001,"15min":0.999},"min":{"1min":0.33100000000000002,"5min":0.26700000000000002,"15min":0.251},"max":{"1min":0.93799999999999994,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.875,"5min":1.022,"15min":1.1379999999999999},"min":{"1min":0.315,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.181,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.372}]},{"osd":7,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.56699999999999995,"15min":1.2729999999999999},"min":{"1min":0.377,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.73399999999999999,"5min":3.339,"15min":94.245999999999995},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.67200000000000004,"15min":1.369},"min":{"1min":0.33100000000000002,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":3.653,"5min":3.653,"15min":94.715000000000003},"last":0.65400000000000003}]}]},{"osd":1,"up_from":94,"seq":403726926244,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1258832,"kb_used_data":430792,"kb_used_omap":7,"kb_used_meta":828024,"kb_avail":92486320,"statfs":{"total":95995035648,"available":94705991680,"internally_reserved":0,"allocated":441131008,"data_stored":439049755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":847896999},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4410000000000001,"5min":1.1140000000000001,"15min":3.8010000000000002},"min":{"1min":0.217,"5min":0.186,"15min":0.182},"max":{"1min":18.556999999999999,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.20100000000000001},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.302,"15min":3.3359999999999999},"min":{"1min":0.23499999999999999,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":18.704000000000001,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.379}]},{"osd":2,"last update":"Sun Oct 1 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.35899999999999999,"15min":2.0179999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.20000000000000001},"max":{"1min":0.78500000000000003,"5min":1.3240000000000001,"15min":51.640000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.44700000000000001,"15min":2.7570000000000001},"min":{"1min":0.29199999999999998,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":0.69499999999999995,"5min":2.9089999999999998,"15min":154.197},"last":0.41199999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.45700000000000002,"15min":2.9540000000000002},"min":{"1min":0.28199999999999997,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":2.8780000000000001,"5min":3.1949999999999998,"15min":95.305000000000007},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.432,"5min":0.51200000000000001,"15min":3.6720000000000002},"min":{"1min":0.27300000000000002,"5min":0.245,"15min":0.216},"max":{"1min":0.95699999999999996,"5min":4.7839999999999998,"15min":154.483},"last":0.32000000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.59099999999999997,"15min":2.1829999999999998},"min":{"1min":0.312,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":1.5309999999999999,"5min":3.0550000000000002,"15min":154.101},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.49099999999999999,"15min":2.2869999999999999},"min":{"1min":0.34300000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65500000000000003,"5min":2.4209999999999998,"15min":154.56999999999999},"last":0.42499999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.143,"15min":2.23},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.8929999999999998,"5min":59.561,"15min":154.22999999999999},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.53500000000000003,"5min":1.137,"15min":2.3660000000000001},"min":{"1min":0.32900000000000001,"5min":0.31,"15min":0.307},"max":{"1min":0.89600000000000002,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.39800000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.65700000000000003,"15min":2.448},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.944,"5min":3.3660000000000001,"15min":154.12200000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64600000000000002,"15min":2.3450000000000002},"min":{"1min":0.374,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":1.7549999999999999,"5min":2.5630000000000002,"15min":154.16999999999999},"last":0.58199999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.74299999999999999,"15min":2.5230000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.33900000000000002},"max":{"1min":3.6160000000000001,"5min":5.2169999999999996,"15min":154.20699999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.70499999999999996,"15min":2.5470000000000002},"min":{"1min":0.40300000000000002,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":3.7490000000000001,"5min":3.8260000000000001,"15min":154.51599999999999},"last":0.45000000000000001}]}]},{"osd":0,"up_from":89,"seq":382252089778,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3037532,"kb_used_data":2106196,"kb_used_omap":8,"kb_used_meta":931319,"kb_avail":90707620,"statfs":{"total":95995035648,"available":92884602880,"internally_reserved":0,"allocated":2156744704,"data_stored":2154711444,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":953671618},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.311,"5min":0.42399999999999999,"15min":3.988},"min":{"1min":0.20699999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.83699999999999997,"5min":4.2919999999999998,"15min":216.821},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.44800000000000001,"15min":4.6740000000000004},"min":{"1min":0.218,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.56799999999999995,"5min":3.9620000000000002,"15min":217.02699999999999},"last":0.38}]},{"osd":2,"last update":"Sun Oct 1 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.47499999999999998,"15min":1.9059999999999999},"min":{"1min":0.29399999999999998,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":1.3100000000000001,"5min":4.2430000000000003,"15min":19.788},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54700000000000004,"15min":1.9239999999999999},"min":{"1min":0.29299999999999998,"5min":0.27200000000000002,"15min":0.215},"max":{"1min":1.3300000000000001,"5min":3.8719999999999999,"15min":25.844999999999999},"last":0.32100000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.504,"15min":3.153},"min":{"1min":0.24099999999999999,"5min":0.22,"15min":0.22},"max":{"1min":1.9039999999999999,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.68200000000000005,"15min":3.0019999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.1379999999999999,"5min":4.9199999999999999,"15min":222.227},"last":2.0259999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.59499999999999997,"15min":1.925},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":0.84299999999999997,"5min":4.048,"15min":62.206000000000003},"last":2.1160000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.63600000000000001,"15min":2.0089999999999999},"min":{"1min":0.29199999999999998,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":1.5189999999999999,"5min":4,"15min":62.073999999999998},"last":0.34499999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.59999999999999998,"15min":1.6699999999999999},"min":{"1min":0.34899999999999998,"5min":0.312,"15min":0.312},"max":{"1min":1.5920000000000001,"5min":4.0350000000000001,"15min":16.808},"last":1.9650000000000001},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.59899999999999998,"15min":1.5820000000000001},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.8100000000000001,"5min":4.2729999999999997,"15min":17.507000000000001},"last":0.71199999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:46:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.63600000000000001,"15min":1.6990000000000001},"min":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.1970000000000001,"5min":4.3129999999999997,"15min":17.535},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65900000000000003,"15min":1.732},"min":{"1min":0.372,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.218,"5min":4.2050000000000001,"15min":15.651999999999999},"last":0.35899999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.79500000000000004,"15min":2.028},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":2.1000000000000001,"5min":4.976,"15min":48.895000000000003},"last":2.2290000000000001},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.89900000000000002,"15min":2.048},"min":{"1min":0.376,"5min":0.376,"15min":0.35099999999999998},"max":{"1min":4.125,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.39500000000000002}]}]},{"osd":2,"up_from":0,"seq":425201762718,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599191,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1303872,"kb_used_data":445496,"kb_used_omap":3,"kb_used_meta":858364,"kb_avail":92441280,"statfs":{"total":95995035648,"available":94659870720,"internally_reserved":0,"allocated":456187904,"data_stored":454056496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":878965479},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":1.4870000000000001,"15min":4.6909999999999998},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":0.65000000000000002,"5min":64.504000000000005,"15min":137.34399999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":1.4870000000000001,"15min":4.6959999999999997},"min":{"1min":0.219,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":1.2849999999999999,"5min":64.448999999999998,"15min":137.73400000000001},"last":0.23899999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.51700000000000002,"15min":4.8719999999999999},"min":{"1min":0.25,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":1.3500000000000001,"5min":6.0250000000000004,"15min":162.22999999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.40600000000000003,"15min":4.9199999999999999},"min":{"1min":0.20899999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.77600000000000002,"5min":2.931,"15min":162.70699999999999},"last":0.27600000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56599999999999995,"15min":1.5800000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.20799999999999999},"max":{"1min":2.9049999999999998,"5min":7.141,"15min":14.254},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.52300000000000002,"15min":1.5860000000000001},"min":{"1min":0.314,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":0.80400000000000005,"5min":4.5049999999999999,"15min":15.523},"last":0.438}]},{"osd":4,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.54600000000000004,"15min":1.71},"min":{"1min":0.34999999999999998,"5min":0.312,"15min":0.26200000000000001},"max":{"1min":0.874,"5min":6.8890000000000002,"15min":77.793000000000006},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.58699999999999997,"15min":1.8919999999999999},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.409,"5min":6.0940000000000003,"15min":76.646000000000001},"last":0.502}]},{"osd":5,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.58399999999999996,"15min":1.9119999999999999},"min":{"1min":0.32500000000000001,"5min":0.314,"15min":0.27500000000000002},"max":{"1min":0.61399999999999999,"5min":4.3099999999999996,"15min":28.117999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51800000000000002,"15min":2.0310000000000001},"min":{"1min":0.33800000000000002,"5min":0.33600000000000002,"15min":0.27700000000000002},"max":{"1min":1.9930000000000001,"5min":2.512,"15min":28.34},"last":0.47899999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.55500000000000005,"15min":1.764},"min":{"1min":0.379,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":0.56899999999999995,"5min":2.9100000000000001,"15min":73.887},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.58499999999999996,"15min":1.8560000000000001},"min":{"1min":0.379,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":0.60499999999999998,"5min":2.6000000000000001,"15min":74.016999999999996},"last":0.45000000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.86299999999999999,"15min":1.7809999999999999},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":3.6480000000000001,"5min":7.0620000000000003,"15min":13.085000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.746,"5min":0.77200000000000002,"15min":1.633},"min":{"1min":0.38800000000000001,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.7120000000000002,"5min":4.1779999999999999,"15min":13.191000000000001},"last":0.48899999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019441,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2829996,"kb_used_data":1927524,"kb_used_omap":0,"kb_used_meta":902464,"kb_avail":90915156,"statfs":{"total":95995035648,"available":93097119744,"internally_reserved":0,"allocated":1973784576,"data_stored":1972843250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924123136},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.623,"15min":3.9359999999999999},"min":{"1min":0.22600000000000001,"5min":0.22500000000000001,"15min":0.21199999999999999},"max":{"1min":0.51400000000000001,"5min":9.6690000000000005,"15min":155.315},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.371,"5min":0.72499999999999998,"15min":4.2910000000000004},"min":{"1min":0.253,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.499,"5min":6.1459999999999999,"15min":155.334},"last":1.117}]},{"osd":1,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.72399999999999998,"15min":5.2430000000000003},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":0.57799999999999996,"5min":5.944,"15min":190.61199999999999},"last":1.133},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.63400000000000001,"15min":5.9699999999999998},"min":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.25600000000000001},"max":{"1min":1.679,"5min":3.7469999999999999,"15min":253.273},"last":0.51000000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.621,"15min":2.0499999999999998},"min":{"1min":0.30499999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.343,"5min":4.2329999999999997,"15min":170.45400000000001},"last":1.1659999999999999},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.61499999999999999,"15min":1.8069999999999999},"min":{"1min":0.35399999999999998,"5min":0.314,"15min":0.309},"max":{"1min":3.1389999999999998,"5min":3.1389999999999998,"15min":36.515999999999998},"last":1.242}]},{"osd":3,"last update":"Sun Oct 1 09:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.69699999999999995,"15min":2.125},"min":{"1min":0.29199999999999998,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":2.0379999999999998,"5min":5.5279999999999996,"15min":140.881},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.68500000000000005,"15min":2.2749999999999999},"min":{"1min":0.32300000000000001,"5min":0.312,"15min":0.311},"max":{"1min":6.2969999999999997,"5min":6.2969999999999997,"15min":140.911},"last":0.93799999999999994}]},{"osd":5,"last update":"Sun Oct 1 09:46:41 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.56999999999999995,"15min":1.454},"min":{"1min":0.27200000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":1.8500000000000001,"5min":3.052,"15min":110.003},"last":1.0860000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.63400000000000001,"15min":1.3859999999999999},"min":{"1min":0.34100000000000003,"5min":0.317,"15min":0.28899999999999998},"max":{"1min":1.754,"5min":3.2909999999999999,"15min":108.56399999999999},"last":0.48099999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.623,"15min":0.91900000000000004},"min":{"1min":0.34599999999999997,"5min":0.28799999999999998,"15min":0.27200000000000002},"max":{"1min":0.63700000000000001,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.318},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.63400000000000001,"15min":1.0289999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28999999999999998},"max":{"1min":1.3340000000000001,"5min":4.0220000000000002,"15min":10.076000000000001},"last":2.008}]},{"osd":7,"last update":"Sun Oct 1 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.63400000000000001,"15min":1.149},"min":{"1min":0.39200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.3079999999999998,"5min":3.3079999999999998,"15min":38.034999999999997},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.70899999999999996,"15min":1.3839999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.308},"max":{"1min":3.0329999999999999,"5min":3.113,"15min":37.723999999999997},"last":1.149}]}]},{"osd":5,"up_from":379,"seq":1627792605348,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1326104,"kb_used_data":965648,"kb_used_omap":13,"kb_used_meta":360434,"kb_avail":92419048,"statfs":{"total":95995035648,"available":94637105152,"internally_reserved":0,"allocated":988823552,"data_stored":987898130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":369084914},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.749,"15min":2.0830000000000002},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.21099999999999999},"max":{"1min":0.77600000000000002,"5min":27.584,"15min":84.340999999999994},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.76900000000000002,"15min":2.0369999999999999},"min":{"1min":0.28399999999999997,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.749,"5min":27.669,"15min":84.356999999999999},"last":0.40400000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.55200000000000005,"15min":5.5519999999999996},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.2130000000000001,"5min":4.3289999999999997,"15min":295.71300000000002},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.51300000000000001,"15min":5.5609999999999999},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":0.70399999999999996,"5min":3.9580000000000002,"15min":299.59399999999999},"last":0.38400000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.52400000000000002,"15min":1.45},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78600000000000003,"5min":4.2670000000000003,"15min":14.199999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.65100000000000002,"15min":1.4139999999999999},"min":{"1min":0.29599999999999999,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":4.2910000000000004,"5min":4.3719999999999999,"15min":14.430999999999999},"last":0.44400000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.56699999999999995,"15min":1.3720000000000001},"min":{"1min":0.32800000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.75,"5min":4.0209999999999999,"15min":14.004},"last":0.309},{"interface":"front","average":{"1min":0.5,"5min":0.58799999999999997,"15min":1.518},"min":{"1min":0.40100000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":0.83299999999999996,"5min":4.2910000000000004,"15min":13.991},"last":0.436}]},{"osd":4,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.53300000000000003,"15min":1.365},"min":{"1min":0.29499999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.0620000000000001,"5min":4.1929999999999996,"15min":122.971},"last":0.37},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.623,"15min":1.5389999999999999},"min":{"1min":0.28899999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2619999999999996,"5min":4.3869999999999996,"15min":122.89100000000001},"last":0.54700000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.54900000000000004,"15min":0.95399999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":0.82799999999999996,"5min":4.3300000000000001,"15min":12.545},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.5,"5min":0.64100000000000001,"15min":1.1459999999999999},"min":{"1min":0.36499999999999999,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":0.81899999999999995,"5min":4.3200000000000003,"15min":14.193},"last":0.48799999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.68300000000000005,"15min":1.1499999999999999},"min":{"1min":0.35399999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.3170000000000002,"5min":4.3559999999999999,"15min":10.813000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.67200000000000004,"15min":1.0469999999999999},"min":{"1min":0.39500000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":1.5369999999999999,"5min":4.2000000000000002,"15min":12.500999999999999},"last":0.49399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2139754496,"data_stored":2139750424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":430178304,"data_stored":430178304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":447545344,"data_stored":447545344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1962479616,"data_stored":1962475544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":978452480,"data_stored":978452480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1022234624,"data_stored":1022230552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":730923008,"data_stored":730923008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:47:03.240 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:03.277 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:02 smithi190 ceph-mon[160435]: pgmap v1640: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 269 op/s; 5571/104673 objects degraded (5.322%); 2149/104673 objects misplaced (2.053%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:47:03.278 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:02 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5571/107124 objects degraded (5.201%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:03.280 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:03.338 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:03.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:03.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:03.443 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:03.477 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:03.566 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:03.575 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:03.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:03.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:03.676 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:47:03.676 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:47:03.676 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:47:03.677 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [] 2023-10-01T09:47:03.677 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:47:03.679 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:47:03.707 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:04.941 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:47:05.103 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:47:05.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:05.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:47:05.207 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:05.234 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:05.246 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:47:05.270 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:47:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:47:04] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-01T09:47:05.282 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:05.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:05.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:05.388 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:05.461 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:05.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:05 smithi190 ceph-mon[160435]: pgmap v1641: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 271 op/s; 5498/103386 objects degraded (5.318%); 2056/103386 objects misplaced (1.989%); 1.1 MiB/s, 18 objects/s recovering 2023-10-01T09:47:05.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:05 smithi138 ceph-mon[185886]: pgmap v1641: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 271 op/s; 5498/103386 objects degraded (5.318%); 2056/103386 objects misplaced (1.989%); 1.1 MiB/s, 18 objects/s recovering 2023-10-01T09:47:05.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:05 smithi138 ceph-mon[192513]: pgmap v1641: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 271 op/s; 5498/103386 objects degraded (5.318%); 2056/103386 objects misplaced (1.989%); 1.1 MiB/s, 18 objects/s recovering 2023-10-01T09:47:05.576 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:05.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:05.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:05.683 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:05.700 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:05.734 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:05.762 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:05.875 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:05.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:05.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:05.979 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:06.008 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:06.056 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:06.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:06.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:06.165 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:06.199 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:06.232 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:47:06.259 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:06.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:06.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:06.367 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:06.400 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:06.459 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:47:06.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:06 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1333087948' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:47:06.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:06 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:47:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:06 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1333087948' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:47:06.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:06 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:47:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:06 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1333087948' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:47:06.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:06 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:47:07.068 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:07.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:07.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:07.173 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:07.208 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:07.267 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:07.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:07.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:07.374 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:07.405 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:07 smithi190 ceph-mon[160435]: pgmap v1642: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 273 op/s; 5498/101661 objects degraded (5.408%); 2056/101661 objects misplaced (2.022%); 1.1 MiB/s, 18 objects/s recovering 2023-10-01T09:47:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:07 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:07 smithi190 ceph-mon[160435]: failed to encode map e550 with expected crc 2023-10-01T09:47:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:07 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:47:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:07 smithi190 ceph-mon[160435]: osdmap e550: 8 total, 8 up, 7 in 2023-10-01T09:47:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:07 smithi190 ceph-mon[160435]: failed to encode map e550 with expected crc 2023-10-01T09:47:07.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:07 smithi190 ceph-mon[160435]: failed to encode map e550 with expected crc 2023-10-01T09:47:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:07 smithi138 ceph-mon[192513]: pgmap v1642: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 273 op/s; 5498/101661 objects degraded (5.408%); 2056/101661 objects misplaced (2.022%); 1.1 MiB/s, 18 objects/s recovering 2023-10-01T09:47:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:07 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:07 smithi138 ceph-mon[192513]: failed to encode map e550 with expected crc 2023-10-01T09:47:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:07 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:47:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:07 smithi138 ceph-mon[192513]: osdmap e550: 8 total, 8 up, 7 in 2023-10-01T09:47:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:07 smithi138 ceph-mon[192513]: failed to encode map e550 with expected crc 2023-10-01T09:47:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:07 smithi138 ceph-mon[192513]: failed to encode map e550 with expected crc 2023-10-01T09:47:07.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:07 smithi138 ceph-mon[185886]: pgmap v1642: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 273 op/s; 5498/101661 objects degraded (5.408%); 2056/101661 objects misplaced (2.022%); 1.1 MiB/s, 18 objects/s recovering 2023-10-01T09:47:07.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:07 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:07.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:07 smithi138 ceph-mon[185886]: failed to encode map e550 with expected crc 2023-10-01T09:47:07.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:07 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:47:07.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:07 smithi138 ceph-mon[185886]: osdmap e550: 8 total, 8 up, 7 in 2023-10-01T09:47:07.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:07 smithi138 ceph-mon[185886]: failed to encode map e550 with expected crc 2023-10-01T09:47:07.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:07 smithi138 ceph-mon[185886]: failed to encode map e550 with expected crc 2023-10-01T09:47:07.953 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:07.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:07.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:08.057 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:08.102 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:08.272 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:08.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:08.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:08.382 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:08.402 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:08.422 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:08.496 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:08 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5498/101661 objects degraded (5.408%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:08.496 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:08 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5498/101661 objects degraded (5.408%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:08.507 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:47:08.531 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:47:08.535 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:08.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:08.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:08.552 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:47:08.576 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:47:08.590 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:47:08.609 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:47:08.628 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:47:08.640 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:08.657 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:47:08.665 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:47:08.665 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:47:08.665 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:47:08.665 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:47:08.666 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:47:08.666 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:47:08.666 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:47:08.666 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:47:08.684 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:08.764 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:47:08.764 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:47:08.793 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:08.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:08.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:08.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:08 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5498/101661 objects degraded (5.408%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:08.897 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:08.927 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:08.978 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:08.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:08.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:09.094 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:09.123 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:09.166 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:09.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:09.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:09.275 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:09.311 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:09.383 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:09.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:09.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:09.491 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:09.517 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:09 smithi138 ceph-mon[192513]: pgmap v1644: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 5413/99498 objects degraded (5.440%); 1959/99498 objects misplaced (1.969%); 1.3 MiB/s, 20 objects/s recovering 2023-10-01T09:47:09.517 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:09 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2321601259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:09.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:09 smithi138 ceph-mon[185886]: pgmap v1644: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 5413/99498 objects degraded (5.440%); 1959/99498 objects misplaced (1.969%); 1.3 MiB/s, 20 objects/s recovering 2023-10-01T09:47:09.517 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:09 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2321601259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:09.519 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:09.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:09 smithi190 ceph-mon[160435]: pgmap v1644: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 5413/99498 objects degraded (5.440%); 1959/99498 objects misplaced (1.969%); 1.3 MiB/s, 20 objects/s recovering 2023-10-01T09:47:09.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:09 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2321601259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:10.050 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:47:10.095 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:47:10.123 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:10.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:10.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:10.227 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:10.260 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:10.310 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:10.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:10.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:10.415 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:10.440 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:10.488 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:10.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:10.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:10.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:47:10.594 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:10.621 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:10.669 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:10.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:10.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:10.774 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:10.776 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:10.801 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:10.806 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:47:10.806 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:47:10.811 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:11.021 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:11.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:11.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:11.125 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:11.175 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:11.241 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:11.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:11.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:11.352 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:11.376 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:47:11.377 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:47:11.389 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:11.452 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:11 smithi190 ceph-mon[160435]: pgmap v1645: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 272 op/s; 5413/98961 objects degraded (5.470%); 1959/98961 objects misplaced (1.980%); 1.2 MiB/s, 19 objects/s recovering 2023-10-01T09:47:11.458 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:11.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:11.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:11.564 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:47:11.595 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:11.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:11 smithi138 ceph-mon[192513]: pgmap v1645: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 272 op/s; 5413/98961 objects degraded (5.470%); 1959/98961 objects misplaced (1.980%); 1.2 MiB/s, 19 objects/s recovering 2023-10-01T09:47:11.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:11 smithi138 ceph-mon[185886]: pgmap v1645: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 272 op/s; 5413/98961 objects degraded (5.470%); 1959/98961 objects misplaced (1.980%); 1.2 MiB/s, 19 objects/s recovering 2023-10-01T09:47:12.487 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1645,"stamp":"2023-10-01T09:47:10.372002+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2164725707,"num_objects":32987,"num_object_clones":0,"num_object_copies":98961,"num_objects_missing_on_primary":491,"num_objects_missing":491,"num_objects_degraded":5413,"num_objects_misplaced":1959,"num_objects_unfound":0,"num_objects_dirty":32987,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":134587,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10789,"num_bytes_recovered":693928975,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123551,"ondisk_log_size":123551,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12203816,"kb_used_data":7115184,"kb_used_omap":39,"kb_used_meta":5088536,"kb_avail":644012248,"statfs":{"total":671965249536,"available":659468541952,"internally_reserved":0,"allocated":7285948416,"data_stored":7275935325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5210661487},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,3,3,0,0,2,5,6,2,9,1],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-178323456,"num_objects":-2721,"num_object_clones":0,"num_object_copies":-8163,"num_objects_missing_on_primary":-155,"num_objects_missing":-155,"num_objects_degraded":-158,"num_objects_misplaced":-190,"num_objects_unfound":0,"num_objects_dirty":-2721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2721,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":193,"num_bytes_recovered":12648448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002697"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":325,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.579002+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:46:50.579002+0000","last_peered":"2023-10-01T09:46:50.579002+0000","last_clean":"2023-10-01T09:46:50.579002+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:46:50.579002+0000","last_undegraded":"2023-10-01T09:46:50.579002+0000","last_fullsized":"2023-10-01T09:46:50.579002+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":170,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.903741+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:46:58.903741+0000","last_peered":"2023-10-01T09:46:58.903741+0000","last_clean":"2023-10-01T09:46:58.903741+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:46:58.903741+0000","last_undegraded":"2023-10-01T09:46:58.903741+0000","last_fullsized":"2023-10-01T09:46:58.903741+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":551,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.903548+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:46:58.903548+0000","last_peered":"2023-10-01T09:46:58.903548+0000","last_clean":"2023-10-01T09:46:58.903548+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:46:58.903548+0000","last_undegraded":"2023-10-01T09:46:58.903548+0000","last_fullsized":"2023-10-01T09:46:58.903548+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":610,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:47:01.912494+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:47:01.912494+0000","last_peered":"2023-10-01T09:47:01.912494+0000","last_clean":"2023-10-01T09:47:01.912494+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:47:01.912494+0000","last_undegraded":"2023-10-01T09:47:01.912494+0000","last_fullsized":"2023-10-01T09:47:01.912494+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":490,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.887212+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:46:58.887212+0000","last_peered":"2023-10-01T09:46:58.887212+0000","last_clean":"2023-10-01T09:46:58.887212+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:46:58.887212+0000","last_undegraded":"2023-10-01T09:46:58.887212+0000","last_fullsized":"2023-10-01T09:46:58.887212+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":231,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.959356+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:46:58.959356+0000","last_peered":"2023-10-01T09:46:58.959356+0000","last_clean":"2023-10-01T09:46:58.959356+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:46:58.959356+0000","last_undegraded":"2023-10-01T09:46:58.959356+0000","last_fullsized":"2023-10-01T09:46:58.959356+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":607,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.959413+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:46:58.959413+0000","last_peered":"2023-10-01T09:46:58.959413+0000","last_clean":"2023-10-01T09:46:58.959413+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:46:58.959413+0000","last_undegraded":"2023-10-01T09:46:58.959413+0000","last_fullsized":"2023-10-01T09:46:58.959413+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":537,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.890392+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:46:58.890392+0000","last_peered":"2023-10-01T09:46:58.890392+0000","last_clean":"2023-10-01T09:46:58.890392+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:46:58.890392+0000","last_undegraded":"2023-10-01T09:46:58.890392+0000","last_fullsized":"2023-10-01T09:46:58.890392+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":262,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.897125+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:46:58.897125+0000","last_peered":"2023-10-01T09:46:58.897125+0000","last_clean":"2023-10-01T09:46:58.897125+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:46:58.897125+0000","last_undegraded":"2023-10-01T09:46:58.897125+0000","last_fullsized":"2023-10-01T09:46:58.897125+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":68,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.575630+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:46:50.575630+0000","last_peered":"2023-10-01T09:46:50.575630+0000","last_clean":"2023-10-01T09:46:50.575630+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:46:50.575630+0000","last_undegraded":"2023-10-01T09:46:50.575630+0000","last_fullsized":"2023-10-01T09:46:50.575630+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":74,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.576456+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:46:50.576456+0000","last_peered":"2023-10-01T09:46:50.576456+0000","last_clean":"2023-10-01T09:46:50.576456+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:46:50.576456+0000","last_undegraded":"2023-10-01T09:46:50.576456+0000","last_fullsized":"2023-10-01T09:46:50.576456+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":74,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.903571+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:46:58.903571+0000","last_peered":"2023-10-01T09:46:58.903571+0000","last_clean":"2023-10-01T09:46:58.903571+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:46:58.903571+0000","last_undegraded":"2023-10-01T09:46:58.903571+0000","last_fullsized":"2023-10-01T09:46:58.903571+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":450,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.581298+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:46:50.581298+0000","last_peered":"2023-10-01T09:46:50.581298+0000","last_clean":"2023-10-01T09:46:50.581298+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:46:50.581298+0000","last_undegraded":"2023-10-01T09:46:50.581298+0000","last_fullsized":"2023-10-01T09:46:50.581298+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":331,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.579208+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:46:50.579208+0000","last_peered":"2023-10-01T09:46:50.579208+0000","last_clean":"2023-10-01T09:46:50.579208+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:46:50.579208+0000","last_undegraded":"2023-10-01T09:46:50.579208+0000","last_fullsized":"2023-10-01T09:46:50.579208+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":667,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.575877+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:46:50.575877+0000","last_peered":"2023-10-01T09:46:50.575877+0000","last_clean":"2023-10-01T09:46:50.575877+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:46:50.575877+0000","last_undegraded":"2023-10-01T09:46:50.575877+0000","last_fullsized":"2023-10-01T09:46:50.575877+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":404,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.576315+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:46:50.576315+0000","last_peered":"2023-10-01T09:46:50.576315+0000","last_clean":"2023-10-01T09:46:50.576315+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:46:50.576315+0000","last_undegraded":"2023-10-01T09:46:50.576315+0000","last_fullsized":"2023-10-01T09:46:50.576315+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":77,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.576850+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:46:50.576850+0000","last_peered":"2023-10-01T09:46:50.576850+0000","last_clean":"2023-10-01T09:46:50.576850+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:46:50.576850+0000","last_undegraded":"2023-10-01T09:46:50.576850+0000","last_fullsized":"2023-10-01T09:46:50.576850+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":64,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.897114+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:46:58.897114+0000","last_peered":"2023-10-01T09:46:58.897114+0000","last_clean":"2023-10-01T09:46:58.897114+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:46:58.897114+0000","last_undegraded":"2023-10-01T09:46:58.897114+0000","last_fullsized":"2023-10-01T09:46:58.897114+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":560,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.575888+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:46:50.575888+0000","last_peered":"2023-10-01T09:46:50.575888+0000","last_clean":"2023-10-01T09:46:50.575888+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:46:50.575888+0000","last_undegraded":"2023-10-01T09:46:50.575888+0000","last_fullsized":"2023-10-01T09:46:50.575888+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":573,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.887531+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:46:58.887531+0000","last_peered":"2023-10-01T09:46:58.887531+0000","last_clean":"2023-10-01T09:46:58.887531+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:46:58.887531+0000","last_undegraded":"2023-10-01T09:46:58.887531+0000","last_fullsized":"2023-10-01T09:46:58.887531+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":430,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:47:01.912393+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:47:01.912393+0000","last_peered":"2023-10-01T09:47:01.912393+0000","last_clean":"2023-10-01T09:47:01.912393+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:47:01.912393+0000","last_undegraded":"2023-10-01T09:47:01.912393+0000","last_fullsized":"2023-10-01T09:47:01.912393+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":77,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.576228+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:46:50.576228+0000","last_peered":"2023-10-01T09:46:50.576228+0000","last_clean":"2023-10-01T09:46:50.576228+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:46:50.576228+0000","last_undegraded":"2023-10-01T09:46:50.576228+0000","last_fullsized":"2023-10-01T09:46:50.576228+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"550'8323","reported_seq":8406,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:07.498978+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:47:07.498978+0000","last_peered":"2023-10-01T09:47:07.498978+0000","last_clean":"2023-10-01T09:47:07.498978+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:47:07.498978+0000","last_undegraded":"2023-10-01T09:47:07.498978+0000","last_fullsized":"2023-10-01T09:47:07.498978+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8323,"log_dups_size":0,"ondisk_log_size":8323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135593984,"num_objects":2069,"num_object_clones":0,"num_object_copies":6207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8323,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":72,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.576081+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:46:50.576081+0000","last_peered":"2023-10-01T09:46:50.576081+0000","last_clean":"2023-10-01T09:46:50.576081+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:46:50.576081+0000","last_undegraded":"2023-10-01T09:46:50.576081+0000","last_fullsized":"2023-10-01T09:46:50.576081+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":270,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.899387+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:46:58.899387+0000","last_peered":"2023-10-01T09:46:58.899387+0000","last_clean":"2023-10-01T09:46:58.899387+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:46:58.899387+0000","last_undegraded":"2023-10-01T09:46:58.899387+0000","last_fullsized":"2023-10-01T09:46:58.899387+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"550'8395","reported_seq":18087,"reported_epoch":550,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:47:07.614841+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:47:07.614841+0000","last_peered":"2023-10-01T09:47:07.614841+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:47:07.614841+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8395,"log_dups_size":0,"ondisk_log_size":8395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131792896,"num_objects":2011,"num_object_clones":0,"num_object_copies":6033,"num_objects_missing_on_primary":491,"num_objects_missing":491,"num_objects_degraded":2433,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8395,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3193,"num_bytes_recovered":209256448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1451},{"shards":"0,4,6","objects":491},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":543,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.576044+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:46:50.576044+0000","last_peered":"2023-10-01T09:46:50.576044+0000","last_clean":"2023-10-01T09:46:50.576044+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:46:50.576044+0000","last_undegraded":"2023-10-01T09:46:50.576044+0000","last_fullsized":"2023-10-01T09:46:50.576044+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":607,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.448379+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:47:06.448379+0000","last_peered":"2023-10-01T09:47:06.448379+0000","last_clean":"2023-10-01T09:47:06.448379+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:47:06.448379+0000","last_undegraded":"2023-10-01T09:47:06.448379+0000","last_fullsized":"2023-10-01T09:47:06.448379+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"550'8332","reported_seq":8419,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.870645+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:47:06.870645+0000","last_peered":"2023-10-01T09:47:06.870645+0000","last_clean":"2023-10-01T09:47:06.870645+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:47:06.870645+0000","last_undegraded":"2023-10-01T09:47:06.870645+0000","last_fullsized":"2023-10-01T09:47:06.870645+0000","mapping_epoch":475,"log_start":"482'1300","ondisk_log_start":"482'1300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7032,"log_dups_size":0,"ondisk_log_size":7032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137625600,"num_objects":2100,"num_object_clones":0,"num_object_copies":6300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8332,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":629,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.576529+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:46:50.576529+0000","last_peered":"2023-10-01T09:46:50.576529+0000","last_clean":"2023-10-01T09:46:50.576529+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:46:50.576529+0000","last_undegraded":"2023-10-01T09:46:50.576529+0000","last_fullsized":"2023-10-01T09:46:50.576529+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":427,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.903719+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:46:58.903719+0000","last_peered":"2023-10-01T09:46:58.903719+0000","last_clean":"2023-10-01T09:46:58.903719+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:46:58.903719+0000","last_undegraded":"2023-10-01T09:46:58.903719+0000","last_fullsized":"2023-10-01T09:46:58.903719+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"550'8379","reported_seq":15071,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.880448+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:47:06.880448+0000","last_peered":"2023-10-01T09:47:06.880448+0000","last_clean":"2023-10-01T09:47:06.880448+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:47:06.880448+0000","last_undegraded":"2023-10-01T09:47:06.880448+0000","last_fullsized":"2023-10-01T09:47:06.880448+0000","mapping_epoch":505,"log_start":"482'1400","ondisk_log_start":"482'1400","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132579328,"num_objects":2023,"num_object_clones":0,"num_object_copies":6069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8379,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":270,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.896943+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:46:58.896943+0000","last_peered":"2023-10-01T09:46:58.896943+0000","last_clean":"2023-10-01T09:46:58.896943+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:46:58.896943+0000","last_undegraded":"2023-10-01T09:46:58.896943+0000","last_fullsized":"2023-10-01T09:46:58.896943+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":530,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.887754+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:46:58.887754+0000","last_peered":"2023-10-01T09:46:58.887754+0000","last_clean":"2023-10-01T09:46:58.887754+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:46:58.887754+0000","last_undegraded":"2023-10-01T09:46:58.887754+0000","last_fullsized":"2023-10-01T09:46:58.887754+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"550'8424","reported_seq":8511,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:09.929391+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:47:09.929391+0000","last_peered":"2023-10-01T09:47:09.929391+0000","last_clean":"2023-10-01T09:47:09.929391+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:47:09.929391+0000","last_undegraded":"2023-10-01T09:47:09.929391+0000","last_fullsized":"2023-10-01T09:47:09.929391+0000","mapping_epoch":475,"log_start":"481'900","ondisk_log_start":"481'900","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7524,"log_dups_size":0,"ondisk_log_size":7524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136577024,"num_objects":2084,"num_object_clones":0,"num_object_copies":6252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8424,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":129,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.579558+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:46:50.579558+0000","last_peered":"2023-10-01T09:46:50.579558+0000","last_clean":"2023-10-01T09:46:50.579558+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:46:50.579558+0000","last_undegraded":"2023-10-01T09:46:50.579558+0000","last_fullsized":"2023-10-01T09:46:50.579558+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":203,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.581375+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:46:50.581375+0000","last_peered":"2023-10-01T09:46:50.581375+0000","last_clean":"2023-10-01T09:46:50.581375+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:46:50.581375+0000","last_undegraded":"2023-10-01T09:46:50.581375+0000","last_fullsized":"2023-10-01T09:46:50.581375+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"550'8345","reported_seq":8820,"reported_epoch":550,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:47:07.470695+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:47:07.470695+0000","last_peered":"2023-10-01T09:47:07.470695+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:47:07.470695+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8345,"log_dups_size":0,"ondisk_log_size":8345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132579328,"num_objects":2023,"num_object_clones":0,"num_object_copies":6069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2970,"num_objects_misplaced":1959,"num_objects_unfound":0,"num_objects_dirty":2023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8345,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":192,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1959},{"shards":"4,5","objects":-2906}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":447,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.578806+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:46:50.578806+0000","last_peered":"2023-10-01T09:46:50.578806+0000","last_clean":"2023-10-01T09:46:50.578806+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:46:50.578806+0000","last_undegraded":"2023-10-01T09:46:50.578806+0000","last_fullsized":"2023-10-01T09:46:50.578806+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":610,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.903793+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:46:58.903793+0000","last_peered":"2023-10-01T09:46:58.903793+0000","last_clean":"2023-10-01T09:46:58.903793+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:46:58.903793+0000","last_undegraded":"2023-10-01T09:46:58.903793+0000","last_fullsized":"2023-10-01T09:46:58.903793+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"550'8323","reported_seq":8415,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:07.374958+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:47:07.374958+0000","last_peered":"2023-10-01T09:47:07.374958+0000","last_clean":"2023-10-01T09:47:07.374958+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:47:07.374958+0000","last_undegraded":"2023-10-01T09:47:07.374958+0000","last_fullsized":"2023-10-01T09:47:07.374958+0000","mapping_epoch":483,"log_start":"481'800","ondisk_log_start":"481'800","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7523,"log_dups_size":0,"ondisk_log_size":7523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134676480,"num_objects":2055,"num_object_clones":0,"num_object_copies":6165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8323,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":443,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.579059+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:46:50.579059+0000","last_peered":"2023-10-01T09:46:50.579059+0000","last_clean":"2023-10-01T09:46:50.579059+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:46:50.579059+0000","last_undegraded":"2023-10-01T09:46:50.579059+0000","last_fullsized":"2023-10-01T09:46:50.579059+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":284,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.579371+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:46:50.579371+0000","last_peered":"2023-10-01T09:46:50.579371+0000","last_clean":"2023-10-01T09:46:50.579371+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:46:50.579371+0000","last_undegraded":"2023-10-01T09:46:50.579371+0000","last_fullsized":"2023-10-01T09:46:50.579371+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"549'8419","reported_seq":8501,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.190644+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:47:06.190644+0000","last_peered":"2023-10-01T09:47:06.190644+0000","last_clean":"2023-10-01T09:47:06.190644+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:47:06.190644+0000","last_undegraded":"2023-10-01T09:47:06.190644+0000","last_fullsized":"2023-10-01T09:47:06.190644+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8419,"log_dups_size":0,"ondisk_log_size":8419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136249344,"num_objects":2079,"num_object_clones":0,"num_object_copies":6237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8419,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":414,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.903535+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:46:58.903535+0000","last_peered":"2023-10-01T09:46:58.903535+0000","last_clean":"2023-10-01T09:46:58.903535+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:46:58.903535+0000","last_undegraded":"2023-10-01T09:46:58.903535+0000","last_fullsized":"2023-10-01T09:46:58.903535+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":461,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.904118+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:46:58.904118+0000","last_peered":"2023-10-01T09:46:58.904118+0000","last_clean":"2023-10-01T09:46:58.904118+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:46:58.904118+0000","last_undegraded":"2023-10-01T09:46:58.904118+0000","last_fullsized":"2023-10-01T09:46:58.904118+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"549'8173","reported_seq":8255,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.291844+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:47:06.291844+0000","last_peered":"2023-10-01T09:47:06.291844+0000","last_clean":"2023-10-01T09:47:06.291844+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:47:06.291844+0000","last_undegraded":"2023-10-01T09:47:06.291844+0000","last_fullsized":"2023-10-01T09:47:06.291844+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8173,"log_dups_size":0,"ondisk_log_size":8173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131399680,"num_objects":2005,"num_object_clones":0,"num_object_copies":6015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8173,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":942,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.576196+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:46:50.576196+0000","last_peered":"2023-10-01T09:46:50.576196+0000","last_clean":"2023-10-01T09:46:50.576196+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:46:50.576196+0000","last_undegraded":"2023-10-01T09:46:50.576196+0000","last_fullsized":"2023-10-01T09:46:50.576196+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":280,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.897248+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:46:58.897248+0000","last_peered":"2023-10-01T09:46:58.897248+0000","last_clean":"2023-10-01T09:46:58.897248+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:46:58.897248+0000","last_undegraded":"2023-10-01T09:46:58.897248+0000","last_fullsized":"2023-10-01T09:46:58.897248+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":75,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:47:00.283071+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:47:00.283071+0000","last_peered":"2023-10-01T09:47:00.283071+0000","last_clean":"2023-10-01T09:47:00.283071+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:47:00.283071+0000","last_undegraded":"2023-10-01T09:47:00.283071+0000","last_fullsized":"2023-10-01T09:47:00.283071+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"550'8261","reported_seq":8348,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.881303+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:47:06.881303+0000","last_peered":"2023-10-01T09:47:06.881303+0000","last_clean":"2023-10-01T09:47:06.881303+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:47:06.881303+0000","last_undegraded":"2023-10-01T09:47:06.881303+0000","last_fullsized":"2023-10-01T09:47:06.881303+0000","mapping_epoch":475,"log_start":"481'1200","ondisk_log_start":"481'1200","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7061,"log_dups_size":0,"ondisk_log_size":7061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138215424,"num_objects":2109,"num_object_clones":0,"num_object_copies":6327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8261,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1215,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.896855+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:46:58.896855+0000","last_peered":"2023-10-01T09:46:58.896855+0000","last_clean":"2023-10-01T09:46:58.896855+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:46:58.896855+0000","last_undegraded":"2023-10-01T09:46:58.896855+0000","last_fullsized":"2023-10-01T09:46:58.896855+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":424,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.579421+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:46:50.579421+0000","last_peered":"2023-10-01T09:46:50.579421+0000","last_clean":"2023-10-01T09:46:50.579421+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:46:50.579421+0000","last_undegraded":"2023-10-01T09:46:50.579421+0000","last_fullsized":"2023-10-01T09:46:50.579421+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":253,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.897040+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:46:58.897040+0000","last_peered":"2023-10-01T09:46:58.897040+0000","last_clean":"2023-10-01T09:46:58.897040+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:46:58.897040+0000","last_undegraded":"2023-10-01T09:46:58.897040+0000","last_fullsized":"2023-10-01T09:46:58.897040+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"550'8312","reported_seq":15616,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:07.551388+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:47:07.551388+0000","last_peered":"2023-10-01T09:47:07.551388+0000","last_clean":"2023-10-01T09:47:07.551388+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:47:07.551388+0000","last_undegraded":"2023-10-01T09:47:07.551388+0000","last_fullsized":"2023-10-01T09:47:07.551388+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8312,"log_dups_size":0,"ondisk_log_size":8312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135266304,"num_objects":2064,"num_object_clones":0,"num_object_copies":6192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8312,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":896,"reported_epoch":549,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:46:58.887347+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:46:58.887347+0000","last_peered":"2023-10-01T09:46:58.887347+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:46:58.887347+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:46:58.887347+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":614,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.903538+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:46:58.903538+0000","last_peered":"2023-10-01T09:46:58.903538+0000","last_clean":"2023-10-01T09:46:58.903538+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:46:58.903538+0000","last_undegraded":"2023-10-01T09:46:58.903538+0000","last_fullsized":"2023-10-01T09:46:58.903538+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":74,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.903949+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:46:58.903949+0000","last_peered":"2023-10-01T09:46:58.903949+0000","last_clean":"2023-10-01T09:46:58.903949+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:46:58.903949+0000","last_undegraded":"2023-10-01T09:46:58.903949+0000","last_fullsized":"2023-10-01T09:46:58.903949+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"550'8311","reported_seq":8398,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:10.164993+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:47:10.164993+0000","last_peered":"2023-10-01T09:47:10.164993+0000","last_clean":"2023-10-01T09:47:10.164993+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:47:10.164993+0000","last_undegraded":"2023-10-01T09:47:10.164993+0000","last_fullsized":"2023-10-01T09:47:10.164993+0000","mapping_epoch":475,"log_start":"481'800","ondisk_log_start":"481'800","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7511,"log_dups_size":0,"ondisk_log_size":7511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128516096,"num_objects":1961,"num_object_clones":0,"num_object_copies":5883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8311,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1666,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.896951+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:46:58.896951+0000","last_peered":"2023-10-01T09:46:58.896951+0000","last_clean":"2023-10-01T09:46:58.896951+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:46:58.896951+0000","last_undegraded":"2023-10-01T09:46:58.896951+0000","last_fullsized":"2023-10-01T09:46:58.896951+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":579,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.575715+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:46:50.575715+0000","last_peered":"2023-10-01T09:46:50.575715+0000","last_clean":"2023-10-01T09:46:50.575715+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:46:50.575715+0000","last_undegraded":"2023-10-01T09:46:50.575715+0000","last_fullsized":"2023-10-01T09:46:50.575715+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":358,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.896862+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:46:58.896862+0000","last_peered":"2023-10-01T09:46:58.896862+0000","last_clean":"2023-10-01T09:46:58.896862+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:46:58.896862+0000","last_undegraded":"2023-10-01T09:46:58.896862+0000","last_fullsized":"2023-10-01T09:46:58.896862+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":347,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.579469+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:46:50.579469+0000","last_peered":"2023-10-01T09:46:50.579469+0000","last_clean":"2023-10-01T09:46:50.579469+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:46:50.579469+0000","last_undegraded":"2023-10-01T09:46:50.579469+0000","last_fullsized":"2023-10-01T09:46:50.579469+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"549'8436","reported_seq":8518,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.282376+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:47:06.282376+0000","last_peered":"2023-10-01T09:47:06.282376+0000","last_clean":"2023-10-01T09:47:06.282376+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:47:06.282376+0000","last_undegraded":"2023-10-01T09:47:06.282376+0000","last_fullsized":"2023-10-01T09:47:06.282376+0000","mapping_epoch":475,"log_start":"480'100","ondisk_log_start":"480'100","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8336,"log_dups_size":0,"ondisk_log_size":8336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137363456,"num_objects":2096,"num_object_clones":0,"num_object_copies":6288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8436,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":756,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.576533+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:46:50.576533+0000","last_peered":"2023-10-01T09:46:50.576533+0000","last_clean":"2023-10-01T09:46:50.576533+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:46:50.576533+0000","last_undegraded":"2023-10-01T09:46:50.576533+0000","last_fullsized":"2023-10-01T09:46:50.576533+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":591,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.887554+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:46:58.887554+0000","last_peered":"2023-10-01T09:46:58.887554+0000","last_clean":"2023-10-01T09:46:58.887554+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:46:58.887554+0000","last_undegraded":"2023-10-01T09:46:58.887554+0000","last_fullsized":"2023-10-01T09:46:58.887554+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":590,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.887352+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:46:58.887352+0000","last_peered":"2023-10-01T09:46:58.887352+0000","last_clean":"2023-10-01T09:46:58.887352+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:46:58.887352+0000","last_undegraded":"2023-10-01T09:46:58.887352+0000","last_fullsized":"2023-10-01T09:46:58.887352+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"550'8350","reported_seq":8438,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.790740+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:47:06.790740+0000","last_peered":"2023-10-01T09:47:06.790740+0000","last_clean":"2023-10-01T09:47:06.790740+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:47:06.790740+0000","last_undegraded":"2023-10-01T09:47:06.790740+0000","last_fullsized":"2023-10-01T09:47:06.790740+0000","mapping_epoch":475,"log_start":"482'1300","ondisk_log_start":"482'1300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7050,"log_dups_size":0,"ondisk_log_size":7050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138215448,"num_objects":2110,"num_object_clones":0,"num_object_copies":6330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2110,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8350,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":987,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.576601+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:46:50.576601+0000","last_peered":"2023-10-01T09:46:50.576601+0000","last_clean":"2023-10-01T09:46:50.576601+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:46:50.576601+0000","last_undegraded":"2023-10-01T09:46:50.576601+0000","last_fullsized":"2023-10-01T09:46:50.576601+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":607,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.897020+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:46:58.897020+0000","last_peered":"2023-10-01T09:46:58.897020+0000","last_clean":"2023-10-01T09:46:58.897020+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:46:58.897020+0000","last_undegraded":"2023-10-01T09:46:58.897020+0000","last_fullsized":"2023-10-01T09:46:58.897020+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":460,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.897149+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:46:58.897149+0000","last_peered":"2023-10-01T09:46:58.897149+0000","last_clean":"2023-10-01T09:46:58.897149+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:46:58.897149+0000","last_undegraded":"2023-10-01T09:46:58.897149+0000","last_fullsized":"2023-10-01T09:46:58.897149+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"549'8298","reported_seq":8388,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.265348+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:47:06.265348+0000","last_peered":"2023-10-01T09:47:06.265348+0000","last_clean":"2023-10-01T09:47:06.265348+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:47:06.265348+0000","last_undegraded":"2023-10-01T09:47:06.265348+0000","last_fullsized":"2023-10-01T09:47:06.265348+0000","mapping_epoch":479,"log_start":"482'1600","ondisk_log_start":"482'1600","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6698,"log_dups_size":0,"ondisk_log_size":6698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134873088,"num_objects":2058,"num_object_clones":0,"num_object_copies":6174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8298,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":488,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.576311+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:46:50.576311+0000","last_peered":"2023-10-01T09:46:50.576311+0000","last_clean":"2023-10-01T09:46:50.576311+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:46:50.576311+0000","last_undegraded":"2023-10-01T09:46:50.576311+0000","last_fullsized":"2023-10-01T09:46:50.576311+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1184,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.903969+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:46:58.903969+0000","last_peered":"2023-10-01T09:46:58.903969+0000","last_clean":"2023-10-01T09:46:58.903969+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:46:58.903969+0000","last_undegraded":"2023-10-01T09:46:58.903969+0000","last_fullsized":"2023-10-01T09:46:58.903969+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":77,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.575635+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:46:50.575635+0000","last_peered":"2023-10-01T09:46:50.575635+0000","last_clean":"2023-10-01T09:46:50.575635+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:46:50.575635+0000","last_undegraded":"2023-10-01T09:46:50.575635+0000","last_fullsized":"2023-10-01T09:46:50.575635+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"549'8400","reported_seq":8490,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.139451+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:47:06.139451+0000","last_peered":"2023-10-01T09:47:06.139451+0000","last_clean":"2023-10-01T09:47:06.139451+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:47:06.139451+0000","last_undegraded":"2023-10-01T09:47:06.139451+0000","last_fullsized":"2023-10-01T09:47:06.139451+0000","mapping_epoch":479,"log_start":"484'1700","ondisk_log_start":"484'1700","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6700,"log_dups_size":0,"ondisk_log_size":6700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138412032,"num_objects":2112,"num_object_clones":0,"num_object_copies":6336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8400,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":81,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.576159+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:46:50.576159+0000","last_peered":"2023-10-01T09:46:50.576159+0000","last_clean":"2023-10-01T09:46:50.576159+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:46:50.576159+0000","last_undegraded":"2023-10-01T09:46:50.576159+0000","last_fullsized":"2023-10-01T09:46:50.576159+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":568,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.887436+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:46:58.887436+0000","last_peered":"2023-10-01T09:46:58.887436+0000","last_clean":"2023-10-01T09:46:58.887436+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:46:58.887436+0000","last_undegraded":"2023-10-01T09:46:58.887436+0000","last_fullsized":"2023-10-01T09:46:58.887436+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":273,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.896811+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:46:58.896811+0000","last_peered":"2023-10-01T09:46:58.896811+0000","last_clean":"2023-10-01T09:46:58.896811+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:46:58.896811+0000","last_undegraded":"2023-10-01T09:46:58.896811+0000","last_fullsized":"2023-10-01T09:46:58.896811+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":517,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.887883+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:46:58.887883+0000","last_peered":"2023-10-01T09:46:58.887883+0000","last_clean":"2023-10-01T09:46:58.887883+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:46:58.887883+0000","last_undegraded":"2023-10-01T09:46:58.887883+0000","last_fullsized":"2023-10-01T09:46:58.887883+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":631,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.903813+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:46:58.903813+0000","last_peered":"2023-10-01T09:46:58.903813+0000","last_clean":"2023-10-01T09:46:58.903813+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:46:58.903813+0000","last_undegraded":"2023-10-01T09:46:58.903813+0000","last_fullsized":"2023-10-01T09:46:58.903813+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":285,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.896926+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:46:58.896926+0000","last_peered":"2023-10-01T09:46:58.896926+0000","last_clean":"2023-10-01T09:46:58.896926+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:46:58.896926+0000","last_undegraded":"2023-10-01T09:46:58.896926+0000","last_fullsized":"2023-10-01T09:46:58.896926+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":74,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.903581+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:46:58.903581+0000","last_peered":"2023-10-01T09:46:58.903581+0000","last_clean":"2023-10-01T09:46:58.903581+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:46:58.903581+0000","last_undegraded":"2023-10-01T09:46:58.903581+0000","last_fullsized":"2023-10-01T09:46:58.903581+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":285,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.897321+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:46:58.897321+0000","last_peered":"2023-10-01T09:46:58.897321+0000","last_clean":"2023-10-01T09:46:58.897321+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:46:58.897321+0000","last_undegraded":"2023-10-01T09:46:58.897321+0000","last_fullsized":"2023-10-01T09:46:58.897321+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":463,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.887866+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:46:58.887866+0000","last_peered":"2023-10-01T09:46:58.887866+0000","last_clean":"2023-10-01T09:46:58.887866+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:46:58.887866+0000","last_undegraded":"2023-10-01T09:46:58.887866+0000","last_fullsized":"2023-10-01T09:46:58.887866+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":92,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.903892+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:46:58.903892+0000","last_peered":"2023-10-01T09:46:58.903892+0000","last_clean":"2023-10-01T09:46:58.903892+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:46:58.903892+0000","last_undegraded":"2023-10-01T09:46:58.903892+0000","last_fullsized":"2023-10-01T09:46:58.903892+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":508,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.887482+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:46:58.887482+0000","last_peered":"2023-10-01T09:46:58.887482+0000","last_clean":"2023-10-01T09:46:58.887482+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:46:58.887482+0000","last_undegraded":"2023-10-01T09:46:58.887482+0000","last_fullsized":"2023-10-01T09:46:58.887482+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":708,"reported_epoch":549,"state":"active+clean","last_fresh":"2023-10-01T09:46:58.888365+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:46:58.888365+0000","last_peered":"2023-10-01T09:46:58.888365+0000","last_clean":"2023-10-01T09:46:58.888365+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:46:58.888365+0000","last_undegraded":"2023-10-01T09:46:58.888365+0000","last_fullsized":"2023-10-01T09:46:58.888365+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":400,"reported_epoch":548,"state":"active+clean","last_fresh":"2023-10-01T09:46:50.578873+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:46:50.578873+0000","last_peered":"2023-10-01T09:46:50.578873+0000","last_clean":"2023-10-01T09:46:50.578873+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:46:50.578873+0000","last_undegraded":"2023-10-01T09:46:50.578873+0000","last_fullsized":"2023-10-01T09:46:50.578873+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2159935512,"num_objects":32959,"num_object_clones":0,"num_object_copies":98877,"num_objects_missing_on_primary":491,"num_objects_missing":491,"num_objects_degraded":5403,"num_objects_misplaced":1959,"num_objects_unfound":0,"num_objects_dirty":32959,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":133481,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10278,"num_bytes_recovered":673579008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7207202816,"data_stored":7207190600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122381,"ondisk_log_size":122381,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430758,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1574136,"kb_used_data":671024,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":92171016,"statfs":{"total":95995035648,"available":94383120384,"internally_reserved":0,"allocated":687128576,"data_stored":686183420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:02 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.495,"15min":2.4870000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":1.3600000000000001,"5min":3.3849999999999998,"15min":108.90000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.46899999999999997,"15min":2.621},"min":{"1min":0.23300000000000001,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":1.7410000000000001,"5min":1.7410000000000001,"15min":109.187},"last":1.6060000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.66400000000000003,"15min":5.1520000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.222},"max":{"1min":1.0780000000000001,"5min":6.8289999999999997,"15min":229.161},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.59799999999999998,"15min":5.1779999999999999},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":3.0640000000000001,"5min":5.7240000000000002,"15min":230.126},"last":0.44600000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:46:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.56299999999999994,"15min":1.591},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":1.6359999999999999,"5min":3.4300000000000002,"15min":14.901999999999999},"last":0.438},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.55700000000000005,"15min":1.48},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.27000000000000002},"max":{"1min":1.8100000000000001,"5min":1.8100000000000001,"15min":14.682},"last":0.46899999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.65800000000000003,"15min":2.2000000000000002},"min":{"1min":0.35699999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.476,"5min":5.7750000000000004,"15min":76.555000000000007},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.68100000000000005,"15min":2.2010000000000001},"min":{"1min":0.32800000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.4529999999999998,"5min":6.548,"15min":76.534000000000006},"last":0.45700000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:47:02 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.52100000000000002,"15min":1.0249999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.4470000000000001,"5min":3.4609999999999999,"15min":9.6020000000000003},"last":0.316},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.52500000000000002,"15min":0.99399999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.837,"5min":1.837,"15min":8.4949999999999992},"last":0.48699999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.52500000000000002,"15min":1.5269999999999999},"min":{"1min":0.35399999999999998,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":2.2400000000000002,"5min":2.2400000000000002,"15min":135.40000000000001},"last":0.505},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.54500000000000004,"15min":1.627},"min":{"1min":0.34999999999999998,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":0.70699999999999996,"5min":2.8849999999999998,"15min":135.339},"last":0.373}]},{"osd":6,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.58999999999999997,"15min":1.1200000000000001},"min":{"1min":0.40500000000000003,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":1.7829999999999999,"5min":2.448,"15min":11.853999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.59699999999999998,"15min":1.093},"min":{"1min":0.39700000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":3.1909999999999998,"5min":3.7810000000000001,"15min":14.584},"last":0.40300000000000002}]}]},{"osd":6,"up_from":294,"seq":1262720385269,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1262460,"kb_used_data":933044,"kb_used_omap":6,"kb_used_meta":329401,"kb_avail":92482692,"statfs":{"total":95995035648,"available":94702276608,"internally_reserved":0,"allocated":955437056,"data_stored":954495527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":337306669},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.45200000000000001,"15min":3.681},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20599999999999999},"max":{"1min":0.51700000000000002,"5min":3.3340000000000001,"15min":158.48599999999999},"last":0.23100000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.55500000000000005,"15min":3.746},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.221},"max":{"1min":3.3519999999999999,"5min":3.3519999999999999,"15min":158.37700000000001},"last":8.202}]},{"osd":1,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.58499999999999996,"15min":4.5739999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":3.3069999999999999,"5min":3.3069999999999999,"15min":168.947},"last":8.2409999999999997},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.60299999999999998,"15min":4.532},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":3.7469999999999999,"5min":3.7469999999999999,"15min":169.07900000000001},"last":0.37}]},{"osd":2,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.59999999999999998,"15min":1.4490000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":3.3849999999999998,"5min":3.3849999999999998,"15min":10.92},"last":8.2829999999999995},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":1.4690000000000001},"min":{"1min":0.32100000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.7160000000000002,"5min":3.7160000000000002,"15min":21.704999999999998},"last":0.40600000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.57999999999999996,"15min":1.8100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.63800000000000001,"5min":3.3759999999999999,"15min":89.753},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.69499999999999995,"15min":1.754},"min":{"1min":0.309,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.2050000000000001,"5min":4.2050000000000001,"15min":89.849999999999994},"last":8.3109999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58899999999999997,"15min":1.069},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.6349999999999998,"5min":4.0250000000000004,"15min":63.134},"last":0.434},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.65800000000000003,"15min":1.0680000000000001},"min":{"1min":0.36599999999999999,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":1.006,"5min":7.4039999999999999,"15min":62.973999999999997},"last":0.72299999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:46:37 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.88500000000000001,"15min":0.999},"min":{"1min":0.33100000000000002,"5min":0.26700000000000002,"15min":0.251},"max":{"1min":0.93799999999999994,"5min":34.094999999999999,"15min":34.094999999999999},"last":1.8979999999999999},{"interface":"front","average":{"1min":0.875,"5min":1.022,"15min":1.1379999999999999},"min":{"1min":0.315,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.181,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.67200000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.56699999999999995,"15min":1.2729999999999999},"min":{"1min":0.377,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.73399999999999999,"5min":3.339,"15min":94.245999999999995},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.67200000000000004,"15min":1.369},"min":{"1min":0.33100000000000002,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":3.653,"5min":3.653,"15min":94.715000000000003},"last":8.3399999999999999}]}]},{"osd":1,"up_from":94,"seq":403726926246,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1230172,"kb_used_data":398036,"kb_used_omap":7,"kb_used_meta":832120,"kb_avail":92514980,"statfs":{"total":95995035648,"available":94735339520,"internally_reserved":0,"allocated":407588864,"data_stored":405503464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":852091303},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":1.0349999999999999,"15min":3.5819999999999999},"min":{"1min":0.19500000000000001,"5min":0.186,"15min":0.182},"max":{"1min":0.82799999999999996,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.0860000000000001,"15min":3.1030000000000002},"min":{"1min":0.222,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":2.0859999999999999,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.30199999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.35899999999999999,"15min":2.0179999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.20000000000000001},"max":{"1min":0.78500000000000003,"5min":1.3240000000000001,"15min":51.640000000000001},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.44700000000000001,"15min":2.7570000000000001},"min":{"1min":0.29199999999999998,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":0.69499999999999995,"5min":2.9089999999999998,"15min":154.197},"last":0.34399999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.45700000000000002,"15min":2.9540000000000002},"min":{"1min":0.28199999999999997,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":2.8780000000000001,"5min":3.1949999999999998,"15min":95.305000000000007},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.432,"5min":0.51200000000000001,"15min":3.6720000000000002},"min":{"1min":0.27300000000000002,"5min":0.245,"15min":0.216},"max":{"1min":0.95699999999999996,"5min":4.7839999999999998,"15min":154.483},"last":0.27100000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.56499999999999995,"15min":2.1389999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":1.204,"5min":3.0550000000000002,"15min":154.101},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.46800000000000003,"15min":2.1469999999999998},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.27700000000000002},"max":{"1min":0.77200000000000002,"5min":0.83199999999999996,"15min":154.56999999999999},"last":0.44}]},{"osd":5,"last update":"Sun Oct 1 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.143,"15min":2.23},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.8929999999999998,"5min":59.561,"15min":154.22999999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.53500000000000003,"5min":1.137,"15min":2.3660000000000001},"min":{"1min":0.32900000000000001,"5min":0.31,"15min":0.307},"max":{"1min":0.89600000000000002,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.46300000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.65700000000000003,"15min":2.448},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.944,"5min":3.3660000000000001,"15min":154.12200000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64600000000000002,"15min":2.3450000000000002},"min":{"1min":0.374,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":1.7549999999999999,"5min":2.5630000000000002,"15min":154.16999999999999},"last":0.39200000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.73299999999999998,"15min":2.4420000000000002},"min":{"1min":0.38900000000000001,"5min":0.36399999999999999,"15min":0.33900000000000002},"max":{"1min":4.3730000000000002,"5min":4.3730000000000002,"15min":154.20699999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.71999999999999997,"15min":2.4460000000000002},"min":{"1min":0.435,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":4.3879999999999999,"5min":4.3879999999999999,"15min":154.51599999999999},"last":0.48999999999999999}]}]},{"osd":0,"up_from":89,"seq":382252089780,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2922920,"kb_used_data":1999776,"kb_used_omap":8,"kb_used_meta":923127,"kb_avail":90822232,"statfs":{"total":95995035648,"available":93001965568,"internally_reserved":0,"allocated":2047770624,"data_stored":2045733217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":945283010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.311,"5min":0.42399999999999999,"15min":3.988},"min":{"1min":0.20699999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.83699999999999997,"5min":4.2919999999999998,"15min":216.821},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.44800000000000001,"15min":4.6740000000000004},"min":{"1min":0.218,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.56799999999999995,"5min":3.9620000000000002,"15min":217.02699999999999},"last":0.214}]},{"osd":2,"last update":"Sun Oct 1 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.47499999999999998,"15min":1.9059999999999999},"min":{"1min":0.29399999999999998,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":1.3100000000000001,"5min":4.2430000000000003,"15min":19.788},"last":1.4399999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54700000000000004,"15min":1.9239999999999999},"min":{"1min":0.29299999999999998,"5min":0.27200000000000002,"15min":0.215},"max":{"1min":1.3300000000000001,"5min":3.8719999999999999,"15min":25.844999999999999},"last":1.45}]},{"osd":3,"last update":"Sun Oct 1 09:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.504,"15min":3.153},"min":{"1min":0.24099999999999999,"5min":0.22,"15min":0.22},"max":{"1min":1.9039999999999999,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.68200000000000005,"15min":3.0019999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.1379999999999999,"5min":4.9199999999999999,"15min":222.227},"last":1.5700000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57199999999999995,"15min":1.8500000000000001},"min":{"1min":0.308,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":2.1160000000000001,"5min":4.048,"15min":62.206000000000003},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.52500000000000002,"15min":1.9359999999999999},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":1.3380000000000001,"5min":4,"15min":62.073999999999998},"last":1.597}]},{"osd":5,"last update":"Sun Oct 1 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.59999999999999998,"15min":1.6699999999999999},"min":{"1min":0.34899999999999998,"5min":0.312,"15min":0.312},"max":{"1min":1.5920000000000001,"5min":4.0350000000000001,"15min":16.808},"last":1.6279999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.59899999999999998,"15min":1.5820000000000001},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.8100000000000001,"5min":4.2729999999999997,"15min":17.507000000000001},"last":0.38400000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:46:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.63600000000000001,"15min":1.6990000000000001},"min":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.1970000000000001,"5min":4.3129999999999997,"15min":17.535},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65900000000000003,"15min":1.732},"min":{"1min":0.372,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.218,"5min":4.2050000000000001,"15min":15.651999999999999},"last":0.39500000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.66300000000000003,"15min":1.927},"min":{"1min":0.39700000000000002,"5min":0.374,"15min":0.374},"max":{"1min":2.2290000000000001,"5min":4.3360000000000003,"15min":48.895000000000003},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.79400000000000004,"15min":1.97},"min":{"1min":0.39300000000000002,"5min":0.376,"15min":0.35299999999999998},"max":{"1min":0.78300000000000003,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.436}]}]},{"osd":2,"up_from":0,"seq":425201762720,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599193,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1257940,"kb_used_data":411852,"kb_used_omap":3,"kb_used_meta":846076,"kb_avail":92487212,"statfs":{"total":95995035648,"available":94706905088,"internally_reserved":0,"allocated":421736448,"data_stored":419600569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":866382567},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":1.4870000000000001,"15min":4.6909999999999998},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":0.65000000000000002,"5min":64.504000000000005,"15min":137.34399999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.41499999999999998,"5min":1.4870000000000001,"15min":4.6959999999999997},"min":{"1min":0.219,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":1.2849999999999999,"5min":64.448999999999998,"15min":137.73400000000001},"last":0.66800000000000004}]},{"osd":1,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.51700000000000002,"15min":4.8719999999999999},"min":{"1min":0.25,"5min":0.214,"15min":0.20399999999999999},"max":{"1min":1.3500000000000001,"5min":6.0250000000000004,"15min":162.22999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.40600000000000003,"15min":4.9199999999999999},"min":{"1min":0.20899999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.77600000000000002,"5min":2.931,"15min":162.70699999999999},"last":0.33600000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56599999999999995,"15min":1.5800000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.20799999999999999},"max":{"1min":2.9049999999999998,"5min":7.141,"15min":14.254},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.52300000000000002,"15min":1.5860000000000001},"min":{"1min":0.314,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":0.80400000000000005,"5min":4.5049999999999999,"15min":15.523},"last":0.44}]},{"osd":4,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.54600000000000004,"15min":1.71},"min":{"1min":0.34999999999999998,"5min":0.312,"15min":0.26200000000000001},"max":{"1min":0.874,"5min":6.8890000000000002,"15min":77.793000000000006},"last":0.376},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.58699999999999997,"15min":1.8919999999999999},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":1.409,"5min":6.0940000000000003,"15min":76.646000000000001},"last":0.56999999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.58399999999999996,"15min":1.9119999999999999},"min":{"1min":0.32500000000000001,"5min":0.314,"15min":0.27500000000000002},"max":{"1min":0.61399999999999999,"5min":4.3099999999999996,"15min":28.117999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51800000000000002,"15min":2.0310000000000001},"min":{"1min":0.33800000000000002,"5min":0.33600000000000002,"15min":0.27700000000000002},"max":{"1min":1.9930000000000001,"5min":2.512,"15min":28.34},"last":0.49099999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.55500000000000005,"15min":1.764},"min":{"1min":0.379,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":0.56899999999999995,"5min":2.9100000000000001,"15min":73.887},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.58499999999999996,"15min":1.8560000000000001},"min":{"1min":0.379,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":0.60499999999999998,"5min":2.6000000000000001,"15min":74.016999999999996},"last":0.55500000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:46:10 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.86299999999999999,"15min":1.7809999999999999},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":3.6480000000000001,"5min":7.0620000000000003,"15min":13.085000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.746,"5min":0.77200000000000002,"15min":1.633},"min":{"1min":0.38800000000000001,"5min":0.33300000000000002,"15min":0.311},"max":{"1min":3.7120000000000002,"5min":4.1779999999999999,"15min":13.191000000000001},"last":0.53500000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019443,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2698612,"kb_used_data":1808428,"kb_used_omap":0,"kb_used_meta":890176,"kb_avail":91046540,"statfs":{"total":95995035648,"available":93231656960,"internally_reserved":0,"allocated":1851830272,"data_stored":1850888622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911540224},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7480000000000002,"5min":0.89400000000000002,"15min":3.5819999999999999},"min":{"1min":0.24399999999999999,"5min":0.22500000000000001,"15min":0.21199999999999999},"max":{"1min":41.963999999999999,"5min":41.963999999999999,"15min":155.315},"last":41.741999999999997},{"interface":"front","average":{"1min":2.7530000000000001,"5min":0.90100000000000002,"15min":3.9140000000000001},"min":{"1min":0.26100000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":41.890999999999998,"5min":41.890999999999998,"15min":155.334},"last":41.713999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.54000000000000004,"15min":5.1449999999999996},"min":{"1min":0.25600000000000001,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":3.3929999999999998,"5min":3.3929999999999998,"15min":190.61199999999999},"last":0.24299999999999999},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.54900000000000004,"15min":5.8639999999999999},"min":{"1min":0.29199999999999998,"5min":0.27500000000000002,"15min":0.25600000000000001},"max":{"1min":3.3839999999999999,"5min":3.3839999999999999,"15min":253.273},"last":0.35699999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.621,"15min":2.0499999999999998},"min":{"1min":0.30499999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.343,"5min":4.2329999999999997,"15min":170.45400000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.61499999999999999,"15min":1.8069999999999999},"min":{"1min":0.35399999999999998,"5min":0.314,"15min":0.309},"max":{"1min":3.1389999999999998,"5min":3.1389999999999998,"15min":36.515999999999998},"last":0.37}]},{"osd":3,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.61199999999999999,"15min":1.9850000000000001},"min":{"1min":0.307,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":3.1539999999999999,"5min":3.1539999999999999,"15min":140.881},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.61399999999999999,"15min":2.0579999999999998},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.311},"max":{"1min":1.3049999999999999,"5min":6.2969999999999997,"15min":140.911},"last":0.40100000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:46:41 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.56999999999999995,"15min":1.454},"min":{"1min":0.27200000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":1.8500000000000001,"5min":3.052,"15min":110.003},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.63400000000000001,"15min":1.3859999999999999},"min":{"1min":0.34100000000000003,"5min":0.317,"15min":0.28899999999999998},"max":{"1min":1.754,"5min":3.2909999999999999,"15min":108.56399999999999},"last":0.44400000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:46:14 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.623,"15min":0.91900000000000004},"min":{"1min":0.34599999999999997,"5min":0.28799999999999998,"15min":0.27200000000000002},"max":{"1min":0.63700000000000001,"5min":9.9849999999999994,"15min":11.358000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.63400000000000001,"15min":1.0289999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28999999999999998},"max":{"1min":1.3340000000000001,"5min":4.0220000000000002,"15min":10.076000000000001},"last":0.41399999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.63400000000000001,"15min":1.149},"min":{"1min":0.39200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.3079999999999998,"5min":3.3079999999999998,"15min":38.034999999999997},"last":0.433},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.70899999999999996,"15min":1.3839999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.308},"max":{"1min":3.0329999999999999,"5min":3.113,"15min":37.723999999999997},"last":0.51000000000000001}]}]},{"osd":5,"up_from":379,"seq":1627792605350,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1257576,"kb_used_data":893024,"kb_used_omap":13,"kb_used_meta":364530,"kb_avail":92487576,"statfs":{"total":95995035648,"available":94707277824,"internally_reserved":0,"allocated":914456576,"data_stored":913530506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":373279218},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,0,0,2,5,6,2,9,1],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.749,"15min":2.0830000000000002},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.21099999999999999},"max":{"1min":0.77600000000000002,"5min":27.584,"15min":84.340999999999994},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.76900000000000002,"15min":2.0369999999999999},"min":{"1min":0.28399999999999997,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.749,"5min":27.669,"15min":84.356999999999999},"last":0.49099999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.55200000000000005,"15min":5.5519999999999996},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.2130000000000001,"5min":4.3289999999999997,"15min":295.71300000000002},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.51300000000000001,"15min":5.5609999999999999},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":0.70399999999999996,"5min":3.9580000000000002,"15min":299.59399999999999},"last":0.70299999999999996}]},{"osd":2,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.52400000000000002,"15min":1.45},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78600000000000003,"5min":4.2670000000000003,"15min":14.199999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.65100000000000002,"15min":1.4139999999999999},"min":{"1min":0.29599999999999999,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":4.2910000000000004,"5min":4.3719999999999999,"15min":14.430999999999999},"last":0.38500000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.56699999999999995,"15min":1.3720000000000001},"min":{"1min":0.32800000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.75,"5min":4.0209999999999999,"15min":14.004},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.5,"5min":0.58799999999999997,"15min":1.518},"min":{"1min":0.40100000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":0.83299999999999996,"5min":4.2910000000000004,"15min":13.991},"last":0.55800000000000005}]},{"osd":4,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.53300000000000003,"15min":1.365},"min":{"1min":0.29499999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.0620000000000001,"5min":4.1929999999999996,"15min":122.971},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.623,"15min":1.5389999999999999},"min":{"1min":0.28899999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2619999999999996,"5min":4.3869999999999996,"15min":122.89100000000001},"last":0.56399999999999995}]},{"osd":6,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.54900000000000004,"15min":0.95399999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":0.82799999999999996,"5min":4.3300000000000001,"15min":12.545},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.64100000000000001,"15min":1.1459999999999999},"min":{"1min":0.36499999999999999,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":0.81899999999999995,"5min":4.3200000000000003,"15min":14.193},"last":3.746}]},{"osd":7,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.68300000000000005,"15min":1.1499999999999999},"min":{"1min":0.35399999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.3170000000000002,"5min":4.3559999999999999,"15min":10.813000000000001},"last":1.8080000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.67200000000000004,"15min":1.0469999999999999},"min":{"1min":0.39500000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":1.5369999999999999,"5min":4.2000000000000002,"15min":12.500999999999999},"last":1.7949999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2030440448,"data_stored":2030436376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":396361728,"data_stored":396361728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":413466624,"data_stored":413466624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1840320512,"data_stored":1840316440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":909312000,"data_stored":909312000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":942346240,"data_stored":942342168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":674955264,"data_stored":674955264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:47:12.493 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:47:12.546 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:47:12.553 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:12 smithi138 ceph-mon[185886]: from='client.47393 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:12.553 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:12 smithi138 ceph-mon[192513]: from='client.47393 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:12.737 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:47:12.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:12.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:47:12.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:12 smithi190 ceph-mon[160435]: from='client.47393 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:12.842 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:12.880 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:12.957 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:12.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:12.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:13.066 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:13.095 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:13.171 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:13.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:13.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:13.287 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:13.330 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:13.394 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:13.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:13.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:13.496 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:13.500 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:47:13.535 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:13.557 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:13.619 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:47:13.620 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:47:13.643 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:13 smithi138 ceph-mon[185886]: pgmap v1646: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 5413/96633 objects degraded (5.602%); 1959/96633 objects misplaced (2.027%); 1.2 MiB/s, 19 objects/s recovering 2023-10-01T09:47:13.643 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:13 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5413/96633 objects degraded (5.602%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:13.643 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:13 smithi138 ceph-mon[192513]: pgmap v1646: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 5413/96633 objects degraded (5.602%); 1959/96633 objects misplaced (2.027%); 1.2 MiB/s, 19 objects/s recovering 2023-10-01T09:47:13.643 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:13 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5413/96633 objects degraded (5.602%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:13.649 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:47:13.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:13.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:47:13.753 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:13.790 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:13 smithi190 ceph-mon[160435]: pgmap v1646: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 5413/96633 objects degraded (5.602%); 1959/96633 objects misplaced (2.027%); 1.2 MiB/s, 19 objects/s recovering 2023-10-01T09:47:13.790 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:13 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5413/96633 objects degraded (5.602%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:13.794 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:13.861 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:13.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:13.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:13.969 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:14.003 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:14.545 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:47:14.632 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:14.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:14.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:14.738 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:14.770 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:15.275 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:47:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:47:14] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-01T09:47:15.297 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:15.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:15.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:15.401 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:15.436 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:15.476 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:47:15.541 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:15.545 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:15 smithi138 ceph-mon[192513]: pgmap v1647: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 269 op/s; 5336/95292 objects degraded (5.600%); 1858/95292 objects misplaced (1.950%); 1.3 MiB/s, 20 objects/s recovering 2023-10-01T09:47:15.545 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:15 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3167954309' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:15.546 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:15 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:15.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:15.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:15.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:15 smithi138 ceph-mon[185886]: pgmap v1647: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 269 op/s; 5336/95292 objects degraded (5.600%); 1858/95292 objects misplaced (1.950%); 1.3 MiB/s, 20 objects/s recovering 2023-10-01T09:47:15.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:15 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3167954309' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:15.548 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:15 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:15.648 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:15.694 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:15.782 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:15.789 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:15 smithi190 ceph-mon[160435]: pgmap v1647: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 269 op/s; 5336/95292 objects degraded (5.600%); 1858/95292 objects misplaced (1.950%); 1.3 MiB/s, 20 objects/s recovering 2023-10-01T09:47:15.789 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:15 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3167954309' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:15.789 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:15 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:15.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:15.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:15.891 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:47:15.916 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:16.006 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:47:16.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:16.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:47:16.123 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:47:16.155 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:16.652 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:47:16.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:16.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:47:16.742 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:16 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:16.742 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:16 smithi138 ceph-mon[185886]: failed to encode map e551 with expected crc 2023-10-01T09:47:16.742 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:16 smithi138 ceph-mon[185886]: failed to encode map e551 with expected crc 2023-10-01T09:47:16.742 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:16 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:47:16.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:16 smithi138 ceph-mon[185886]: osdmap e551: 8 total, 8 up, 7 in 2023-10-01T09:47:16.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:16 smithi138 ceph-mon[185886]: failed to encode map e551 with expected crc 2023-10-01T09:47:16.743 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:16 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:16.743 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:16 smithi138 ceph-mon[192513]: failed to encode map e551 with expected crc 2023-10-01T09:47:16.743 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:16 smithi138 ceph-mon[192513]: failed to encode map e551 with expected crc 2023-10-01T09:47:16.743 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:16 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:47:16.743 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:16 smithi138 ceph-mon[192513]: osdmap e551: 8 total, 8 up, 7 in 2023-10-01T09:47:16.743 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:16 smithi138 ceph-mon[192513]: failed to encode map e551 with expected crc 2023-10-01T09:47:16.756 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:16.783 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:16.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:16 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:16.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:16 smithi190 ceph-mon[160435]: failed to encode map e551 with expected crc 2023-10-01T09:47:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:16 smithi190 ceph-mon[160435]: failed to encode map e551 with expected crc 2023-10-01T09:47:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:16 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:47:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:16 smithi190 ceph-mon[160435]: osdmap e551: 8 total, 8 up, 7 in 2023-10-01T09:47:16.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:16 smithi190 ceph-mon[160435]: failed to encode map e551 with expected crc 2023-10-01T09:47:16.884 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:16.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:16.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:16.987 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:17.005 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:17.014 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:17.089 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:17.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:17.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:17.203 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:17.235 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:17.294 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:47:17.295 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5336/93378 objects degraded (5.714%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2258,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":551,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":31126,"data_bytes":2042763211,"bytes_used":12163825664,"bytes_avail":659801423872,"bytes_total":671965249536,"degraded_objects":5336,"degraded_total":93378,"degraded_ratio":0.057144081046927545,"misplaced_objects":1858,"misplaced_total":93378,"misplaced_ratio":0.019897620424511125,"recovering_objects_per_sec":20,"recovering_bytes_per_sec":1346199,"recovering_keys_per_sec":0,"num_objects_recovered":204,"num_bytes_recovered":13369344,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":278},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":34,"modified":"2023-10-01T09:43:40.311181+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:47:17.295 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:17.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:17.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:17.402 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:17.435 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:17.439 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:47:17.461 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:47:17.481 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:17.482 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:47:17.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:17.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:17.503 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:47:17.521 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:47:17.538 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:47:17.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:17 smithi190 ceph-mon[160435]: pgmap v1649: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 5336/93378 objects degraded (5.714%); 1858/93378 objects misplaced (1.990%); 1.3 MiB/s, 20 objects/s recovering 2023-10-01T09:47:17.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:17 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/881130193' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:47:17.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:17 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4043506354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:17.557 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:47:17.576 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:47:17.577 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:47:17.578 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:47:17.578 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:47:17.578 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:47:17.578 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:47:17.578 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:47:17.578 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:47:17.578 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:47:17.587 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:47:17.615 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:17.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:17 smithi138 ceph-mon[192513]: pgmap v1649: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 5336/93378 objects degraded (5.714%); 1858/93378 objects misplaced (1.990%); 1.3 MiB/s, 20 objects/s recovering 2023-10-01T09:47:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:17 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/881130193' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:47:17.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:17 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4043506354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:17.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:17 smithi138 ceph-mon[185886]: pgmap v1649: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 5336/93378 objects degraded (5.714%); 1858/93378 objects misplaced (1.990%); 1.3 MiB/s, 20 objects/s recovering 2023-10-01T09:47:17.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:17 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/881130193' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:47:17.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:17 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4043506354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:18.416 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:47:18.416 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:47:18.470 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:47:18.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:18.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:47:18.574 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:18.597 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:18 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5336/93378 objects degraded (5.714%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:18.598 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:18 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5336/93378 objects degraded (5.714%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:18.601 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:18.660 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:18.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:18.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:18.745 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:47:18.771 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:18.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:18 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5336/93378 objects degraded (5.714%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:18.820 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:18.868 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:18.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:18.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:18.975 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:19.003 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:19.408 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:19.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:19.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:19.515 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:19.529 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:19.546 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:19.556 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:19.633 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:19 smithi138 ceph-mon[192513]: pgmap v1650: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 292 op/s; 5236/90708 objects degraded (5.772%); 1762/90708 objects misplaced (1.942%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:47:19.633 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:19 smithi138 ceph-mon[185886]: pgmap v1650: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 292 op/s; 5236/90708 objects degraded (5.772%); 1762/90708 objects misplaced (1.942%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:47:19.658 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:19.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:19.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:19.767 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:19.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:19 smithi190 ceph-mon[160435]: pgmap v1650: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 292 op/s; 5236/90708 objects degraded (5.772%); 1762/90708 objects misplaced (1.942%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:47:19.818 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:19.871 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:47:19.871 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:47:19.871 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:47:19.871 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:47:19.871 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.871 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:147927 9.1 9:88c25810:::benchmark_data_smithi138_497829_object45542:head [delete] snapc 0=[] ondisk+write+known_if_redirected e444)", 2023-10-01T09:47:19.871 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:22.269510+0000", 2023-10-01T09:47:19.872 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 597.59826778000001, 2023-10-01T09:47:19.872 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.42430222200000001, 2023-10-01T09:47:19.872 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.872 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "reached pg", 2023-10-01T09:47:19.872 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:47:19.872 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:47:19.872 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:47:19.872 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 147927 2023-10-01T09:47:19.872 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.872 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.872 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269510+0000", 2023-10-01T09:47:19.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269510+0000", 2023-10-01T09:47:19.873 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.873 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.873 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.873 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.873 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269508+0000", 2023-10-01T09:47:19.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:47:19.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269517+0000", 2023-10-01T09:47:19.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8389999999999996e-06 2023-10-01T09:47:19.874 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.874 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.874 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.874 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269531+0000", 2023-10-01T09:47:19.874 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3882e-05 2023-10-01T09:47:19.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.875 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.875 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269540+0000", 2023-10-01T09:47:19.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4409999999999993e-06 2023-10-01T09:47:19.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.875 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.875 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269579+0000", 2023-10-01T09:47:19.875 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9886999999999999e-05 2023-10-01T09:47:19.875 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.875 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269697+0000", 2023-10-01T09:47:19.876 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00011801100000000001 2023-10-01T09:47:19.876 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 2,4,6", 2023-10-01T09:47:19.876 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.269781+0000", 2023-10-01T09:47:19.876 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3877999999999999e-05 2023-10-01T09:47:19.876 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.876 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.876 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.877 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.273704+0000", 2023-10-01T09:47:19.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0039223030000000002 2023-10-01T09:47:19.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.877 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.877 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:47:19.877 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.279487+0000", 2023-10-01T09:47:19.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0057834059999999996 2023-10-01T09:47:19.877 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.877 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.877 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.877 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.286463+0000", 2023-10-01T09:47:19.877 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0069760109999999998 2023-10-01T09:47:19.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:47:19.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.286474+0000", 2023-10-01T09:47:19.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0564000000000001e-05 2023-10-01T09:47:19.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.878 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.878 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.693718+0000", 2023-10-01T09:47:19.878 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.40724410300000002 2023-10-01T09:47:19.878 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.878 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.879 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.879 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.693812+0000", 2023-10-01T09:47:19.879 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4624999999999996e-05 2023-10-01T09:47:19.879 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.879 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.879 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.879 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.879 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.879 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:147929 9.1 9:84c1980f:::benchmark_data_smithi138_497829_object45544:head [delete] snapc 0=[] ondisk+write+known_if_redirected e444)", 2023-10-01T09:47:19.879 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:22.289771+0000", 2023-10-01T09:47:19.879 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 597.57800658600001, 2023-10-01T09:47:19.880 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.44011142599999997, 2023-10-01T09:47:19.880 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.880 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.880 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:47:19.880 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:47:19.880 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:47:19.880 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 147929 2023-10-01T09:47:19.880 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.880 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.880 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.880 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.880 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289771+0000", 2023-10-01T09:47:19.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289771+0000", 2023-10-01T09:47:19.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.881 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.881 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.881 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289770+0000", 2023-10-01T09:47:19.881 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:47:19.881 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.882 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.882 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289776+0000", 2023-10-01T09:47:19.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8130000000000001e-06 2023-10-01T09:47:19.882 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.882 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.882 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289786+0000", 2023-10-01T09:47:19.882 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0533999999999999e-05 2023-10-01T09:47:19.882 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.882 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.289793+0000", 2023-10-01T09:47:19.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.9110000000000004e-06 2023-10-01T09:47:19.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.290885+0000", 2023-10-01T09:47:19.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0010919930000000001 2023-10-01T09:47:19.883 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.883 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.883 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:47:19.883 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.290895+0000", 2023-10-01T09:47:19.883 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9359999999999994e-06 2023-10-01T09:47:19.884 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.884 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.884 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.884 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.694724+0000", 2023-10-01T09:47:19.884 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.40382902199999998 2023-10-01T09:47:19.884 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.884 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.884 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.884 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.694835+0000", 2023-10-01T09:47:19.884 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111291 2023-10-01T09:47:19.884 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.885 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:47:19.885 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.695529+0000", 2023-10-01T09:47:19.885 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00069338699999999995 2023-10-01T09:47:19.885 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.885 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:47:19.885 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.711318+0000", 2023-10-01T09:47:19.885 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015789381000000002 2023-10-01T09:47:19.885 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.885 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.885 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.886 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.720238+0000", 2023-10-01T09:47:19.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0089196480000000005 2023-10-01T09:47:19.886 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.886 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.886 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.886 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.729790+0000", 2023-10-01T09:47:19.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0095524689999999992 2023-10-01T09:47:19.886 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.886 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.886 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.886 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.729838+0000", 2023-10-01T09:47:19.886 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.7947e-05 2023-10-01T09:47:19.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.887 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.887 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.729883+0000", 2023-10-01T09:47:19.887 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.4607000000000002e-05 2023-10-01T09:47:19.887 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.887 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.887 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.887 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.887 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.887 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:147933 9.1 9:89b639aa:::benchmark_data_smithi138_497829_object45548:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:47:19.887 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:22.305679+0000", 2023-10-01T09:47:19.888 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 597.56209931599994, 2023-10-01T09:47:19.888 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.424334777, 2023-10-01T09:47:19.888 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.888 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.888 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:47:19.888 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:47:19.888 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:47:19.888 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 147933 2023-10-01T09:47:19.888 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.888 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.888 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.889 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305679+0000", 2023-10-01T09:47:19.889 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.889 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.889 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305679+0000", 2023-10-01T09:47:19.889 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.889 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.889 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.889 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.889 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305677+0000", 2023-10-01T09:47:19.890 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:47:19.890 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.890 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.890 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305685+0000", 2023-10-01T09:47:19.890 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.5050000000000001e-06 2023-10-01T09:47:19.890 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.890 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.890 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.890 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305702+0000", 2023-10-01T09:47:19.890 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7244999999999998e-05 2023-10-01T09:47:19.891 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.891 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.891 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.891 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305712+0000", 2023-10-01T09:47:19.891 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9410000000000002e-06 2023-10-01T09:47:19.891 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.891 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.891 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.891 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305776+0000", 2023-10-01T09:47:19.891 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4111999999999999e-05 2023-10-01T09:47:19.891 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.891 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.892 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:47:19.892 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.305787+0000", 2023-10-01T09:47:19.892 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1102e-05 2023-10-01T09:47:19.892 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.892 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.892 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.892 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.703627+0000", 2023-10-01T09:47:19.892 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.397840311 2023-10-01T09:47:19.892 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.892 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.892 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.892 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.703767+0000", 2023-10-01T09:47:19.893 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013993999999999999 2023-10-01T09:47:19.893 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.893 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:47:19.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.703898+0000", 2023-10-01T09:47:19.893 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013054399999999999 2023-10-01T09:47:19.893 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.893 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.893 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:47:19.893 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.715395+0000", 2023-10-01T09:47:19.893 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011496908 2023-10-01T09:47:19.893 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.894 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.894 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.729920+0000", 2023-10-01T09:47:19.894 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014524789999999999 2023-10-01T09:47:19.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.894 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.894 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.729948+0000", 2023-10-01T09:47:19.894 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7948000000000001e-05 2023-10-01T09:47:19.894 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.894 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.895 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.895 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.729982+0000", 2023-10-01T09:47:19.895 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4576999999999998e-05 2023-10-01T09:47:19.895 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.895 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.895 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.895 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.730013+0000", 2023-10-01T09:47:19.895 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.1300999999999997e-05 2023-10-01T09:47:19.895 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.895 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.895 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.895 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.896 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.896 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:147961 9.1 9:8c87c16e:::benchmark_data_smithi138_497829_object45576:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:47:19.896 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:22.469797+0000", 2023-10-01T09:47:19.896 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 597.39798112200003, 2023-10-01T09:47:19.896 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.260341398, 2023-10-01T09:47:19.896 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.896 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.896 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:47:19.896 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:47:19.896 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:47:19.896 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 147961 2023-10-01T09:47:19.896 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.897 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.897 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469797+0000", 2023-10-01T09:47:19.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.897 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.897 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.897 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469797+0000", 2023-10-01T09:47:19.897 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.897 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.897 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.898 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469795+0000", 2023-10-01T09:47:19.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:47:19.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.898 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469806+0000", 2023-10-01T09:47:19.898 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1085e-05 2023-10-01T09:47:19.898 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.898 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.898 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.899 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469828+0000", 2023-10-01T09:47:19.899 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2636000000000001e-05 2023-10-01T09:47:19.899 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.899 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.899 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.899 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469840+0000", 2023-10-01T09:47:19.899 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.136e-05 2023-10-01T09:47:19.899 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.899 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.899 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.899 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469896+0000", 2023-10-01T09:47:19.900 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6166999999999999e-05 2023-10-01T09:47:19.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.900 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:47:19.900 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.469909+0000", 2023-10-01T09:47:19.900 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2568999999999999e-05 2023-10-01T09:47:19.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.900 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.900 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.900 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.707773+0000", 2023-10-01T09:47:19.900 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23786492100000001 2023-10-01T09:47:19.900 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.707891+0000", 2023-10-01T09:47:19.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000117571 2023-10-01T09:47:19.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:47:19.901 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.707971+0000", 2023-10-01T09:47:19.901 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9968000000000004e-05 2023-10-01T09:47:19.901 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.901 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.901 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:47:19.902 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.715402+0000", 2023-10-01T09:47:19.902 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0074309370000000003 2023-10-01T09:47:19.902 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.902 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.902 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.730048+0000", 2023-10-01T09:47:19.902 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014646144 2023-10-01T09:47:19.902 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.902 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.902 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.902 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.730077+0000", 2023-10-01T09:47:19.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8615e-05 2023-10-01T09:47:19.903 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.903 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.903 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.903 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.730109+0000", 2023-10-01T09:47:19.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2013e-05 2023-10-01T09:47:19.903 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.903 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.903 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.903 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.730138+0000", 2023-10-01T09:47:19.903 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.9527e-05 2023-10-01T09:47:19.903 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.904 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.904 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.904 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.904 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.904 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.46067.0:147973 9.1 9:85c71572:::benchmark_data_smithi138_497829_object45588:head [delete] snapc 0=[] ondisk+write+known_if_redirected e445)", 2023-10-01T09:47:19.904 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:37:22.511296+0000", 2023-10-01T09:47:19.904 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 597.356481605, 2023-10-01T09:47:19.904 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.23218191699999999, 2023-10-01T09:47:19.904 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.904 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.904 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:47:19.905 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.46067", 2023-10-01T09:47:19.905 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/3873020665", 2023-10-01T09:47:19.905 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 147973 2023-10-01T09:47:19.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.905 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.905 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511296+0000", 2023-10-01T09:47:19.905 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.905 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.905 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.905 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.906 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511296+0000", 2023-10-01T09:47:19.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.906 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.906 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.906 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511294+0000", 2023-10-01T09:47:19.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999976 2023-10-01T09:47:19.906 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.906 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.906 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.906 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511305+0000", 2023-10-01T09:47:19.906 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0614999999999999e-05 2023-10-01T09:47:19.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.907 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.907 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511328+0000", 2023-10-01T09:47:19.907 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3229000000000001e-05 2023-10-01T09:47:19.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.907 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.907 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511338+0000", 2023-10-01T09:47:19.907 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.012e-05 2023-10-01T09:47:19.907 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.907 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511512+0000", 2023-10-01T09:47:19.908 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00017422800000000001 2023-10-01T09:47:19.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.908 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:47:19.908 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.511524+0000", 2023-10-01T09:47:19.908 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2063e-05 2023-10-01T09:47:19.908 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.908 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.908 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.711357+0000", 2023-10-01T09:47:19.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.19983310900000001 2023-10-01T09:47:19.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.909 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.711498+0000", 2023-10-01T09:47:19.909 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00014025499999999999 2023-10-01T09:47:19.909 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.909 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.909 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,6", 2023-10-01T09:47:19.909 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.711635+0000", 2023-10-01T09:47:19.910 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013784999999999999 2023-10-01T09:47:19.910 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.910 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:47:19.910 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.715508+0000", 2023-10-01T09:47:19.910 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0038724050000000002 2023-10-01T09:47:19.910 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.910 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.910 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.910 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.730227+0000", 2023-10-01T09:47:19.910 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.014719603 2023-10-01T09:47:19.910 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.743379+0000", 2023-10-01T09:47:19.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013151949 2023-10-01T09:47:19.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.911 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.911 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.743431+0000", 2023-10-01T09:47:19.911 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2051e-05 2023-10-01T09:47:19.911 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.911 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.912 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.912 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:37:22.743478+0000", 2023-10-01T09:47:19.912 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6755999999999998e-05 2023-10-01T09:47:19.912 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.912 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.912 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.912 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.912 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.912 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58520 10.9 10:9985a324:::benchmark_data_smithi138_573643_object58519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:47:19.912 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385572+0000", 2023-10-01T09:47:19.912 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 384.48220605699998, 2023-10-01T09:47:19.912 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2149010790000001, 2023-10-01T09:47:19.913 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.913 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.913 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:47:19.913 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:47:19.913 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:47:19.913 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58520 2023-10-01T09:47:19.913 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.913 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.913 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.913 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.913 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385572+0000", 2023-10-01T09:47:19.913 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.914 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.914 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.914 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.914 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385572+0000", 2023-10-01T09:47:19.914 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.914 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.914 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.914 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.914 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385570+0000", 2023-10-01T09:47:19.914 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:47:19.914 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.915 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.915 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.915 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385593+0000", 2023-10-01T09:47:19.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3112000000000002e-05 2023-10-01T09:47:19.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.915 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.915 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.915 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385606+0000", 2023-10-01T09:47:19.915 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2847000000000001e-05 2023-10-01T09:47:19.915 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.915 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.916 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.916 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385613+0000", 2023-10-01T09:47:19.916 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3849999999999999e-06 2023-10-01T09:47:19.916 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.916 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.916 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.916 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385721+0000", 2023-10-01T09:47:19.916 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000108593 2023-10-01T09:47:19.916 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.916 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.916 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:47:19.916 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385731+0000", 2023-10-01T09:47:19.916 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.6539999999999994e-06 2023-10-01T09:47:19.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.575851+0000", 2023-10-01T09:47:19.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1901203300000001 2023-10-01T09:47:19.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.917 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.917 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.577232+0000", 2023-10-01T09:47:19.917 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.001380537 2023-10-01T09:47:19.917 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.917 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,7", 2023-10-01T09:47:19.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.577415+0000", 2023-10-01T09:47:19.918 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018354800000000001 2023-10-01T09:47:19.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:47:19.918 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.586834+0000", 2023-10-01T09:47:19.918 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0094188629999999995 2023-10-01T09:47:19.918 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.918 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.918 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.586890+0000", 2023-10-01T09:47:19.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6181999999999998e-05 2023-10-01T09:47:19.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.919 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.587055+0000", 2023-10-01T09:47:19.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000164146 2023-10-01T09:47:19.919 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.919 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.919 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.919 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.597960+0000", 2023-10-01T09:47:19.919 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010905521 2023-10-01T09:47:19.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.598019+0000", 2023-10-01T09:47:19.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8686e-05 2023-10-01T09:47:19.920 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.920 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.920 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.920 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.600473+0000", 2023-10-01T09:47:19.920 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024542159999999999 2023-10-01T09:47:19.920 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.921 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.921 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.921 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.921 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.921 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58531 10.9 10:9bb2758a:::benchmark_data_smithi138_573643_object58530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:47:19.921 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385619+0000", 2023-10-01T09:47:19.921 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 384.482159174, 2023-10-01T09:47:19.921 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.217814583, 2023-10-01T09:47:19.921 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.921 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.921 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:47:19.921 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:47:19.922 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:47:19.922 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58531 2023-10-01T09:47:19.922 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.922 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.922 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.922 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.922 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385619+0000", 2023-10-01T09:47:19.922 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.922 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.922 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.922 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.923 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385619+0000", 2023-10-01T09:47:19.923 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.923 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.923 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.923 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.923 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385618+0000", 2023-10-01T09:47:19.923 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:47:19.923 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.923 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.923 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.923 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385639+0000", 2023-10-01T09:47:19.923 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0242999999999999e-05 2023-10-01T09:47:19.924 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.924 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.924 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385643+0000", 2023-10-01T09:47:19.924 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5360000000000003e-06 2023-10-01T09:47:19.924 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.924 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.924 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385646+0000", 2023-10-01T09:47:19.924 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.0800000000000002e-06 2023-10-01T09:47:19.924 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.924 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.925 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.925 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385753+0000", 2023-10-01T09:47:19.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000106791 2023-10-01T09:47:19.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.925 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.925 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:47:19.925 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385772+0000", 2023-10-01T09:47:19.925 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8834e-05 2023-10-01T09:47:19.925 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.925 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.925 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.925 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.581773+0000", 2023-10-01T09:47:19.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.196001447 2023-10-01T09:47:19.926 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.926 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.926 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.581957+0000", 2023-10-01T09:47:19.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018389799999999999 2023-10-01T09:47:19.926 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.926 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.926 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,7", 2023-10-01T09:47:19.926 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.582136+0000", 2023-10-01T09:47:19.926 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0001783 2023-10-01T09:47:19.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:47:19.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.587009+0000", 2023-10-01T09:47:19.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0048730400000000004 2023-10-01T09:47:19.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.927 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.927 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.587118+0000", 2023-10-01T09:47:19.927 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000109125 2023-10-01T09:47:19.927 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.927 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.592124+0000", 2023-10-01T09:47:19.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050063060000000003 2023-10-01T09:47:19.928 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.928 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.601115+0000", 2023-10-01T09:47:19.928 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0089907430000000007 2023-10-01T09:47:19.928 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.928 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.928 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.928 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.601177+0000", 2023-10-01T09:47:19.929 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.1853000000000006e-05 2023-10-01T09:47:19.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.929 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.929 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.929 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.603433+0000", 2023-10-01T09:47:19.929 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022567500000000001 2023-10-01T09:47:19.929 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.929 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.929 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.929 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.929 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.929 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58551 10.9 10:9c541330:::benchmark_data_smithi138_573643_object58550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:47:19.930 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385649+0000", 2023-10-01T09:47:19.930 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 384.482128837, 2023-10-01T09:47:19.930 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2200250429999999, 2023-10-01T09:47:19.930 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.930 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.930 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:47:19.930 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:47:19.930 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:47:19.930 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58551 2023-10-01T09:47:19.930 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.930 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.931 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.931 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385649+0000", 2023-10-01T09:47:19.931 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.931 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.931 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.931 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385649+0000", 2023-10-01T09:47:19.931 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.931 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.931 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.931 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.932 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385649+0000", 2023-10-01T09:47:19.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:47:19.932 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.932 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.932 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.932 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385670+0000", 2023-10-01T09:47:19.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.1498e-05 2023-10-01T09:47:19.932 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.932 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.932 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.932 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385676+0000", 2023-10-01T09:47:19.932 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.0950000000000001e-06 2023-10-01T09:47:19.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385682+0000", 2023-10-01T09:47:19.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.1189999999999996e-06 2023-10-01T09:47:19.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.933 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.933 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385793+0000", 2023-10-01T09:47:19.933 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000111908 2023-10-01T09:47:19.933 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.933 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:47:19.934 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385801+0000", 2023-10-01T09:47:19.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2400000000000001e-06 2023-10-01T09:47:19.934 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.934 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.584310+0000", 2023-10-01T09:47:19.934 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1985098169999999 2023-10-01T09:47:19.934 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.934 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.934 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.584480+0000", 2023-10-01T09:47:19.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00016994199999999999 2023-10-01T09:47:19.935 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.935 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,7", 2023-10-01T09:47:19.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.584661+0000", 2023-10-01T09:47:19.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018100000000000001 2023-10-01T09:47:19.935 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.935 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.935 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:47:19.935 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.587131+0000", 2023-10-01T09:47:19.935 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0024693720000000001 2023-10-01T09:47:19.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.592184+0000", 2023-10-01T09:47:19.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0050534120000000002 2023-10-01T09:47:19.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.936 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.936 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.592257+0000", 2023-10-01T09:47:19.936 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2509000000000007e-05 2023-10-01T09:47:19.936 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.936 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.603500+0000", 2023-10-01T09:47:19.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011243474999999999 2023-10-01T09:47:19.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.937 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.603566+0000", 2023-10-01T09:47:19.937 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5840000000000007e-05 2023-10-01T09:47:19.937 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.937 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.937 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.938 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.605674+0000", 2023-10-01T09:47:19.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0021080880000000001 2023-10-01T09:47:19.938 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.938 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.938 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.938 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.938 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.938 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58583 10.9 10:9ad37353:::benchmark_data_smithi138_573643_object58582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:47:19.938 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385686+0000", 2023-10-01T09:47:19.938 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 384.48209186700001, 2023-10-01T09:47:19.938 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.222197339, 2023-10-01T09:47:19.938 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.939 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.939 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:47:19.939 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:47:19.939 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:47:19.939 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58583 2023-10-01T09:47:19.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.939 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.939 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.939 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.939 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385686+0000", 2023-10-01T09:47:19.939 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.939 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385686+0000", 2023-10-01T09:47:19.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.940 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385686+0000", 2023-10-01T09:47:19.940 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:47:19.940 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.940 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.940 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385713+0000", 2023-10-01T09:47:19.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7521999999999998e-05 2023-10-01T09:47:19.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385720+0000", 2023-10-01T09:47:19.941 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.0280000000000003e-06 2023-10-01T09:47:19.941 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.941 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.941 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.941 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385726+0000", 2023-10-01T09:47:19.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.66e-06 2023-10-01T09:47:19.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.942 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.942 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385821+0000", 2023-10-01T09:47:19.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4901999999999994e-05 2023-10-01T09:47:19.942 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.942 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.942 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:47:19.942 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385828+0000", 2023-10-01T09:47:19.942 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.8900000000000001e-06 2023-10-01T09:47:19.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.587174+0000", 2023-10-01T09:47:19.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.2013461409999999 2023-10-01T09:47:19.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.943 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.943 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.587384+0000", 2023-10-01T09:47:19.943 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021059 2023-10-01T09:47:19.943 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.943 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.944 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,7", 2023-10-01T09:47:19.944 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.587578+0000", 2023-10-01T09:47:19.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019330500000000001 2023-10-01T09:47:19.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.944 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:47:19.944 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.592132+0000", 2023-10-01T09:47:19.944 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0045539300000000003 2023-10-01T09:47:19.944 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.944 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.944 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.944 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.597825+0000", 2023-10-01T09:47:19.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.005693488 2023-10-01T09:47:19.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.597883+0000", 2023-10-01T09:47:19.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.7493e-05 2023-10-01T09:47:19.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.945 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.945 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.945 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.605740+0000", 2023-10-01T09:47:19.945 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0078573970000000003 2023-10-01T09:47:19.945 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.605804+0000", 2023-10-01T09:47:19.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4410000000000002e-05 2023-10-01T09:47:19.946 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.946 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.946 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.946 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.607883+0000", 2023-10-01T09:47:19.946 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002078978 2023-10-01T09:47:19.946 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.946 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.946 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.947 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.947 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.947 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58601 10.9 10:99ed6712:::benchmark_data_smithi138_573643_object58600:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:47:19.947 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385730+0000", 2023-10-01T09:47:19.947 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 384.482047502, 2023-10-01T09:47:19.947 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.226336528, 2023-10-01T09:47:19.947 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.947 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.947 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:47:19.947 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:47:19.947 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:47:19.948 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58601 2023-10-01T09:47:19.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.948 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385730+0000", 2023-10-01T09:47:19.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.948 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.948 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.948 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.948 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385730+0000", 2023-10-01T09:47:19.948 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385730+0000", 2023-10-01T09:47:19.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:47:19.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.949 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.949 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385749+0000", 2023-10-01T09:47:19.949 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8672e-05 2023-10-01T09:47:19.949 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.949 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385755+0000", 2023-10-01T09:47:19.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5060000000000001e-06 2023-10-01T09:47:19.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.950 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.950 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385760+0000", 2023-10-01T09:47:19.950 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.8409999999999999e-06 2023-10-01T09:47:19.950 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.950 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.950 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385852+0000", 2023-10-01T09:47:19.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1588999999999993e-05 2023-10-01T09:47:19.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:47:19.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385860+0000", 2023-10-01T09:47:19.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3089999999999996e-06 2023-10-01T09:47:19.951 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.951 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.951 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.951 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.589917+0000", 2023-10-01T09:47:19.951 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.204056899 2023-10-01T09:47:19.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.590112+0000", 2023-10-01T09:47:19.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000194749 2023-10-01T09:47:19.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.952 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.952 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,7", 2023-10-01T09:47:19.952 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.590328+0000", 2023-10-01T09:47:19.952 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00021647999999999999 2023-10-01T09:47:19.952 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:47:19.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.597841+0000", 2023-10-01T09:47:19.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0075124969999999999 2023-10-01T09:47:19.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.953 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.597924+0000", 2023-10-01T09:47:19.953 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.3517999999999996e-05 2023-10-01T09:47:19.953 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.953 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.953 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.600528+0000", 2023-10-01T09:47:19.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0026043619999999998 2023-10-01T09:47:19.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.609486+0000", 2023-10-01T09:47:19.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0089572599999999999 2023-10-01T09:47:19.954 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.954 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.954 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.954 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.609545+0000", 2023-10-01T09:47:19.954 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.9070000000000002e-05 2023-10-01T09:47:19.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.955 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.955 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.612067+0000", 2023-10-01T09:47:19.955 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0025222030000000002 2023-10-01T09:47:19.955 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.955 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.955 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.955 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.955 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.955 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58618 10.9 10:9a8c1a55:::benchmark_data_smithi138_573643_object58617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:47:19.956 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385764+0000", 2023-10-01T09:47:19.956 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 384.48201407099998, 2023-10-01T09:47:19.956 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.228793872, 2023-10-01T09:47:19.956 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.956 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.956 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:47:19.956 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:47:19.956 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:47:19.956 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58618 2023-10-01T09:47:19.956 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.956 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.956 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385764+0000", 2023-10-01T09:47:19.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.957 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385764+0000", 2023-10-01T09:47:19.957 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.957 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.957 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.957 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385764+0000", 2023-10-01T09:47:19.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:47:19.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385784+0000", 2023-10-01T09:47:19.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0100999999999999e-05 2023-10-01T09:47:19.958 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.958 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.958 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385790+0000", 2023-10-01T09:47:19.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3450000000000002e-06 2023-10-01T09:47:19.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.959 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.959 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385795+0000", 2023-10-01T09:47:19.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.5739999999999997e-06 2023-10-01T09:47:19.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.959 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.959 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385882+0000", 2023-10-01T09:47:19.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.7119000000000004e-05 2023-10-01T09:47:19.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:47:19.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385901+0000", 2023-10-01T09:47:19.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9517999999999998e-05 2023-10-01T09:47:19.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.592304+0000", 2023-10-01T09:47:19.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.206402626 2023-10-01T09:47:19.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.592507+0000", 2023-10-01T09:47:19.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00020330500000000001 2023-10-01T09:47:19.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,7", 2023-10-01T09:47:19.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.592706+0000", 2023-10-01T09:47:19.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00019918599999999999 2023-10-01T09:47:19.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:47:19.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.597891+0000", 2023-10-01T09:47:19.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051844960000000002 2023-10-01T09:47:19.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.600572+0000", 2023-10-01T09:47:19.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0026814349999999998 2023-10-01T09:47:19.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.600608+0000", 2023-10-01T09:47:19.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5259999999999998e-05 2023-10-01T09:47:19.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.612125+0000", 2023-10-01T09:47:19.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.011517947000000001 2023-10-01T09:47:19.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.963 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.612181+0000", 2023-10-01T09:47:19.963 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5234e-05 2023-10-01T09:47:19.963 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.963 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.963 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.964 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.614558+0000", 2023-10-01T09:47:19.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0023770430000000001 2023-10-01T09:47:19.964 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.964 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.964 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.964 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.964 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.964 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:58620 10.9 10:9173f148:::benchmark_data_smithi138_573643_object58619:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e496)", 2023-10-01T09:47:19.964 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:40:55.385799+0000", 2023-10-01T09:47:19.964 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 384.48197933300003, 2023-10-01T09:47:19.964 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.231120153, 2023-10-01T09:47:19.964 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.965 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.965 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:47:19.965 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:47:19.965 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:47:19.965 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 58620 2023-10-01T09:47:19.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.965 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.965 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.965 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.965 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385799+0000", 2023-10-01T09:47:19.965 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.965 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.966 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385799+0000", 2023-10-01T09:47:19.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.966 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.966 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385798+0000", 2023-10-01T09:47:19.966 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:47:19.966 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.966 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385819+0000", 2023-10-01T09:47:19.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.0421999999999999e-05 2023-10-01T09:47:19.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385828+0000", 2023-10-01T09:47:19.967 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.9210000000000004e-06 2023-10-01T09:47:19.967 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.967 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.967 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.967 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385834+0000", 2023-10-01T09:47:19.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8440000000000003e-06 2023-10-01T09:47:19.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385924+0000", 2023-10-01T09:47:19.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.9350000000000003e-05 2023-10-01T09:47:19.968 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.968 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.968 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for peered", 2023-10-01T09:47:19.968 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:55.385931+0000", 2023-10-01T09:47:19.968 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3939999999999998e-06 2023-10-01T09:47:19.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.595068+0000", 2023-10-01T09:47:19.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.209136912 2023-10-01T09:47:19.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.969 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.969 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.595252+0000", 2023-10-01T09:47:19.969 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018431400000000001 2023-10-01T09:47:19.969 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.969 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 4,5,7", 2023-10-01T09:47:19.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.595435+0000", 2023-10-01T09:47:19.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00018283000000000001 2023-10-01T09:47:19.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:47:19.970 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.600537+0000", 2023-10-01T09:47:19.970 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0051021319999999997 2023-10-01T09:47:19.970 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.970 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.970 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.601045+0000", 2023-10-01T09:47:19.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00050768099999999995 2023-10-01T09:47:19.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.601082+0000", 2023-10-01T09:47:19.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7333000000000003e-05 2023-10-01T09:47:19.971 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.971 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.971 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:47:19.971 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.614627+0000", 2023-10-01T09:47:19.971 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.013545117000000001 2023-10-01T09:47:19.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.614690+0000", 2023-10-01T09:47:19.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3077999999999995e-05 2023-10-01T09:47:19.972 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.972 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.972 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.972 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:40:56.616919+0000", 2023-10-01T09:47:19.972 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0022282629999999999 2023-10-01T09:47:19.972 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.973 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.973 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.973 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.973 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:100084 10.3 e525/503 10:c7b16d54:::benchmark_data_smithi138_573643_object16862:head v 525'6238, mlcod=481'963)", 2023-10-01T09:47:19.973 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:43:58.015618+0000", 2023-10-01T09:47:19.973 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 201.85216027999999, 2023-10-01T09:47:19.973 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017502734999999998, 2023-10-01T09:47:19.973 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.973 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.973 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.973 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:58.015618+0000", 2023-10-01T09:47:19.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.974 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:58.015618+0000", 2023-10-01T09:47:19.974 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.974 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.974 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.974 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.975 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:58.015616+0000", 2023-10-01T09:47:19.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:47:19.975 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.975 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.975 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:58.015626+0000", 2023-10-01T09:47:19.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.0237999999999999e-05 2023-10-01T09:47:19.975 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.975 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.975 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.975 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:58.015645+0000", 2023-10-01T09:47:19.975 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8811999999999999e-05 2023-10-01T09:47:19.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:58.015654+0000", 2023-10-01T09:47:19.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.8580000000000006e-06 2023-10-01T09:47:19.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.976 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.976 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.976 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:58.015720+0000", 2023-10-01T09:47:19.976 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5718999999999999e-05 2023-10-01T09:47:19.976 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:58.015755+0000", 2023-10-01T09:47:19.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.5500999999999998e-05 2023-10-01T09:47:19.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.977 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:58.033068+0000", 2023-10-01T09:47:19.977 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017312962000000001 2023-10-01T09:47:19.977 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.977 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.977 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.978 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:43:58.033120+0000", 2023-10-01T09:47:19.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.2367000000000002e-05 2023-10-01T09:47:19.978 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.978 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.978 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.978 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.978 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.978 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:128565 10.f e547/479 10:f658f7b9:::benchmark_data_smithi138_573643_object45343:head v 547'8085, mlcod=547'8083)", 2023-10-01T09:47:19.978 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:46:48.856952+0000", 2023-10-01T09:47:19.978 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 31.010825593, 2023-10-01T09:47:19.978 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.029451267, 2023-10-01T09:47:19.979 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.979 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.979 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.856952+0000", 2023-10-01T09:47:19.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.979 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.979 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.979 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.979 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.856952+0000", 2023-10-01T09:47:19.979 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.856951+0000", 2023-10-01T09:47:19.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:47:19.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.980 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.980 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.980 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.856961+0000", 2023-10-01T09:47:19.980 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.1169999999999993e-06 2023-10-01T09:47:19.980 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.856970+0000", 2023-10-01T09:47:19.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4900000000000006e-06 2023-10-01T09:47:19.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.981 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.856976+0000", 2023-10-01T09:47:19.981 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6529999999999999e-06 2023-10-01T09:47:19.981 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.981 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.981 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.880381+0000", 2023-10-01T09:47:19.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.023405103999999999 2023-10-01T09:47:19.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.982 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.880453+0000", 2023-10-01T09:47:19.982 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2236000000000004e-05 2023-10-01T09:47:19.982 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.982 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.982 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.982 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.886375+0000", 2023-10-01T09:47:19.983 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0059217640000000004 2023-10-01T09:47:19.983 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.983 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.983 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.983 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.886404+0000", 2023-10-01T09:47:19.983 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.8844e-05 2023-10-01T09:47:19.983 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.983 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.983 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.983 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.983 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.983 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:128566 10.4 e547/475 10:2ebcff4c:::benchmark_data_smithi138_573643_object45344:head v 547'8048, mlcod=547'8046)", 2023-10-01T09:47:19.984 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:46:48.858160+0000", 2023-10-01T09:47:19.984 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 31.009618278000001, 2023-10-01T09:47:19.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.028000691000000001, 2023-10-01T09:47:19.984 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.984 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.984 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.984 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.984 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.984 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.858160+0000", 2023-10-01T09:47:19.984 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.984 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.858160+0000", 2023-10-01T09:47:19.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.985 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.858159+0000", 2023-10-01T09:47:19.985 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:47:19.985 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.985 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.985 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.986 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.858167+0000", 2023-10-01T09:47:19.986 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.0269999999999995e-06 2023-10-01T09:47:19.986 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.986 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.986 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.880350+0000", 2023-10-01T09:47:19.986 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.022182725 2023-10-01T09:47:19.986 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.986 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.986 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.986 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.880373+0000", 2023-10-01T09:47:19.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.3734999999999999e-05 2023-10-01T09:47:19.987 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.987 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.987 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.880477+0000", 2023-10-01T09:47:19.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000103745 2023-10-01T09:47:19.987 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.987 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.987 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.987 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.880535+0000", 2023-10-01T09:47:19.987 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.8102e-05 2023-10-01T09:47:19.987 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.886123+0000", 2023-10-01T09:47:19.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0055879299999999996 2023-10-01T09:47:19.988 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.988 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.988 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.988 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.886160+0000", 2023-10-01T09:47:19.988 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.7222999999999999e-05 2023-10-01T09:47:19.988 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.988 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.988 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.989 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.989 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:128576 10.3 e547/503 10:c1a3471d:::benchmark_data_smithi138_573643_object45354:head v 547'8013, mlcod=487'2814)", 2023-10-01T09:47:19.989 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:46:48.889671+0000", 2023-10-01T09:47:19.989 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 30.978107222999999, 2023-10-01T09:47:19.989 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.023357942, 2023-10-01T09:47:19.989 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.989 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.989 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.989 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.989 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.989 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.889671+0000", 2023-10-01T09:47:19.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.889671+0000", 2023-10-01T09:47:19.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.990 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.990 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.990 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.990 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.889670+0000", 2023-10-01T09:47:19.990 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:47:19.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.991 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.991 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.889681+0000", 2023-10-01T09:47:19.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.084e-05 2023-10-01T09:47:19.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.991 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.991 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.991 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.889696+0000", 2023-10-01T09:47:19.991 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5586999999999998e-05 2023-10-01T09:47:19.991 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.889703+0000", 2023-10-01T09:47:19.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.238e-06 2023-10-01T09:47:19.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.992 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.889738+0000", 2023-10-01T09:47:19.992 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.4696999999999997e-05 2023-10-01T09:47:19.992 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.992 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.992 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.889765+0000", 2023-10-01T09:47:19.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6625999999999999e-05 2023-10-01T09:47:19.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.912989+0000", 2023-10-01T09:47:19.993 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.023224468000000002 2023-10-01T09:47:19.993 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.993 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.993 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.993 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.913029+0000", 2023-10-01T09:47:19.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.9459999999999998e-05 2023-10-01T09:47:19.994 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.994 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.994 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.994 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.994 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.994 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:128580 10.3 e547/503 10:c9f20f64:::benchmark_data_smithi138_573643_object45358:head v 547'8014, mlcod=487'2814)", 2023-10-01T09:47:19.994 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:46:48.893256+0000", 2023-10-01T09:47:19.994 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 30.974522362999998, 2023-10-01T09:47:19.994 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.019792047, 2023-10-01T09:47:19.994 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:19.995 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:19.995 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:19.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:19.995 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.893256+0000", 2023-10-01T09:47:19.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.995 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.995 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.995 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:19.995 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.893256+0000", 2023-10-01T09:47:19.995 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:19.995 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:19.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.893255+0000", 2023-10-01T09:47:19.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:47:19.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.996 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:19.996 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.893264+0000", 2023-10-01T09:47:19.996 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.0310000000000004e-06 2023-10-01T09:47:19.996 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.996 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:19.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.893278+0000", 2023-10-01T09:47:19.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3861e-05 2023-10-01T09:47:19.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:19.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.893285+0000", 2023-10-01T09:47:19.997 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.9459999999999998e-06 2023-10-01T09:47:19.997 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.997 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.997 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:19.997 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.897697+0000", 2023-10-01T09:47:19.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0044115530000000003 2023-10-01T09:47:19.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:19.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.897715+0000", 2023-10-01T09:47:19.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8249000000000001e-05 2023-10-01T09:47:19.998 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.998 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.998 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:19.998 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.913033+0000", 2023-10-01T09:47:19.998 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015317594 2023-10-01T09:47:19.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.999 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:19.999 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:46:48.913048+0000", 2023-10-01T09:47:19.999 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.4761e-05 2023-10-01T09:47:19.999 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.999 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:19.999 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:19.999 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:19.999 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:19.999 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:133788 10.3 e550/503 10:c0aea19b:::benchmark_data_smithi138_573643_object50566:head v 550'8356, mlcod=489'3168)", 2023-10-01T09:47:19.999 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:47:08.085914+0000", 2023-10-01T09:47:20.000 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 11.781864085, 2023-10-01T09:47:20.000 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017911318999999998, 2023-10-01T09:47:20.000 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:20.000 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:20.000 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:20.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.000 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:20.000 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:08.085914+0000", 2023-10-01T09:47:20.000 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:20.000 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.000 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:20.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:08.085914+0000", 2023-10-01T09:47:20.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:20.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:20.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:08.085912+0000", 2023-10-01T09:47:20.001 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999981 2023-10-01T09:47:20.001 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.001 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.001 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:20.001 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:08.085928+0000", 2023-10-01T09:47:20.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5795e-05 2023-10-01T09:47:20.002 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:20.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:08.085955+0000", 2023-10-01T09:47:20.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.7447999999999999e-05 2023-10-01T09:47:20.002 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.002 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.002 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:20.002 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:08.085969+0000", 2023-10-01T09:47:20.002 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3361e-05 2023-10-01T09:47:20.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.003 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:20.003 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:08.093362+0000", 2023-10-01T09:47:20.003 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0073932119999999997 2023-10-01T09:47:20.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.003 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:20.003 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:08.093382+0000", 2023-10-01T09:47:20.003 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.9834e-05 2023-10-01T09:47:20.003 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.003 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.004 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:20.004 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:08.103779+0000", 2023-10-01T09:47:20.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010397262000000001 2023-10-01T09:47:20.004 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.004 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.004 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:20.004 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:08.103825+0000", 2023-10-01T09:47:20.004 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4.6075999999999997e-05 2023-10-01T09:47:20.004 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:20.004 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:20.004 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:20.005 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.005 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "MOSDPGPush(10.7 551/503 [PushOp(10:e84dc4c6:::benchmark_data_smithi138_573643_object53370:head, version: 492'3374, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:e84dc4c6:::benchmark_data_smithi138_573643_object53370:head@492'3374, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-10-01T09:47:20.005 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:47:18.375294+0000", 2023-10-01T09:47:20.005 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.49248398, 2023-10-01T09:47:20.005 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.017690065000000001, 2023-10-01T09:47:20.005 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:20.005 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:47:20.005 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:20.005 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.005 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:20.005 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.375294+0000", 2023-10-01T09:47:20.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:20.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:20.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.375294+0000", 2023-10-01T09:47:20.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:20.006 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.006 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.006 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:20.006 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.375293+0000", 2023-10-01T09:47:20.006 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:47:20.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.007 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:20.007 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.375372+0000", 2023-10-01T09:47:20.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.8832999999999996e-05 2023-10-01T09:47:20.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.007 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:20.007 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.375389+0000", 2023-10-01T09:47:20.007 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.7331000000000001e-05 2023-10-01T09:47:20.007 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.007 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:20.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.375406+0000", 2023-10-01T09:47:20.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.6526999999999999e-05 2023-10-01T09:47:20.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:20.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.382183+0000", 2023-10-01T09:47:20.008 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0067770010000000004 2023-10-01T09:47:20.008 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.008 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.008 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:20.008 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.382197+0000", 2023-10-01T09:47:20.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3648999999999999e-05 2023-10-01T09:47:20.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.009 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:20.009 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.392984+0000", 2023-10-01T09:47:20.009 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.010787374000000001 2023-10-01T09:47:20.009 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:20.009 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:20.009 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:20.009 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.009 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.010 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:136760 10.3 e551/503 10:c8acea78:::benchmark_data_smithi138_573643_object53538:head v 551'8563, mlcod=493'3380)", 2023-10-01T09:47:20.010 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:47:18.685606+0000", 2023-10-01T09:47:20.010 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 1.1821716170000001, 2023-10-01T09:47:20.010 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.016504408000000002, 2023-10-01T09:47:20.010 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:47:20.010 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:47:20.010 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:47:20.010 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.010 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:47:20.010 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.685606+0000", 2023-10-01T09:47:20.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:20.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:47:20.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.685606+0000", 2023-10-01T09:47:20.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:47:20.011 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.011 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.011 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:47:20.011 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.685605+0000", 2023-10-01T09:47:20.011 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:47:20.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.012 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:47:20.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.685614+0000", 2023-10-01T09:47:20.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4419999999999998e-06 2023-10-01T09:47:20.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.012 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:47:20.012 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.685630+0000", 2023-10-01T09:47:20.012 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.5647000000000001e-05 2023-10-01T09:47:20.012 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.012 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:47:20.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.685637+0000", 2023-10-01T09:47:20.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.4499999999999998e-06 2023-10-01T09:47:20.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.013 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:47:20.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.686724+0000", 2023-10-01T09:47:20.013 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00108658 2023-10-01T09:47:20.013 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.013 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.013 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:47:20.013 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.686750+0000", 2023-10-01T09:47:20.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.6302e-05 2023-10-01T09:47:20.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "commit_sent", 2023-10-01T09:47:20.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.702078+0000", 2023-10-01T09:47:20.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.015328085 2023-10-01T09:47:20.014 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:47:20.014 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:47:20.014 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "done", 2023-10-01T09:47:20.014 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:47:18.702111+0000", 2023-10-01T09:47:20.014 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.2774999999999997e-05 2023-10-01T09:47:20.015 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:20.015 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:20.015 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:20.015 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:47:20.015 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:47:20.015 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:47:20.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:20.016 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:20.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:20.117 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:20.143 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:20.192 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:20.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:20.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:20.297 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:20.327 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:20.624 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:47:20.830 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:20.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:20.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:20.936 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:20.961 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:20.990 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:47:21.005 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:21.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:21.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:21.111 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:21.152 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:21.204 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:21.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:21.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:21.312 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:21.340 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:21.394 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:47:21.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:21 smithi190 ceph-mon[160435]: pgmap v1651: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 292 op/s; 5236/90174 objects degraded (5.807%); 1762/90174 objects misplaced (1.954%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:47:21.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:21 smithi138 ceph-mon[192513]: pgmap v1651: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 292 op/s; 5236/90174 objects degraded (5.807%); 1762/90174 objects misplaced (1.954%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:47:21.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:21 smithi138 ceph-mon[185886]: pgmap v1651: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 292 op/s; 5236/90174 objects degraded (5.807%); 1762/90174 objects misplaced (1.954%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:47:21.836 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:21.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:21.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:21.941 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:21.955 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:21.985 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:22.125 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:22.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:22.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:22.230 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:22.264 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:22.315 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:22.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:22.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:22.425 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:22.487 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:22.588 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:22.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:22.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:22.692 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:22.725 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:22.828 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:22.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:22.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:22.893 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:47:22.895 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:47:22.932 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:22.961 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:23.008 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:23.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:23.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:23.113 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:23.140 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:23.407 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:23.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:23.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:23.493 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:47:23.511 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:23.545 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:23.595 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:23.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:23.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:23.675 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:23 smithi138 ceph-mon[192513]: pgmap v1652: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 292 op/s; 5236/87858 objects degraded (5.960%); 1762/87858 objects misplaced (2.006%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:47:23.676 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:23 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1689248391' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:47:23.676 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:23 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:47:23.676 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:23 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5236/87858 objects degraded (5.960%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:23.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:23 smithi138 ceph-mon[185886]: pgmap v1652: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 292 op/s; 5236/87858 objects degraded (5.960%); 1762/87858 objects misplaced (2.006%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:47:23.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:23 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1689248391' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:47:23.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:23 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:47:23.676 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:23 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5236/87858 objects degraded (5.960%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:23.681 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:47:23.681 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:47:23.681 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:23.682 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.36456.0:138100 10.9 e552/509 10:981f4837:::benchmark_data_smithi138_573643_object54878:head v 552'8591, mlcod=551'8589)", 2023-10-01T09:47:23.682 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:47:23.676753+0000", 2023-10-01T09:47:23.682 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0039950480000000002, 2023-10-01T09:47:23.682 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0040304390000000002, 2023-10-01T09:47:23.682 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:47:23.682 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:47:23.682 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:47:23.682 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:23.682 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:47:23.682 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:23.676753+0000", 2023-10-01T09:47:23.682 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:47:23.683 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:23.683 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:23.683 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:47:23.683 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:23.676753+0000", 2023-10-01T09:47:23.683 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:47:23.683 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:23.683 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:23.683 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:47:23.683 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:23.676753+0000", 2023-10-01T09:47:23.683 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.8879999999999997e-06 2023-10-01T09:47:23.683 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:23.683 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:23.683 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:47:23.684 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:23.676760+0000", 2023-10-01T09:47:23.684 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4105e-05 2023-10-01T09:47:23.684 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:23.684 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:23.684 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:47:23.684 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:23.676775+0000", 2023-10-01T09:47:23.684 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9730000000000002e-06 2023-10-01T09:47:23.684 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:23.684 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:23.684 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:47:23.684 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:23.676781+0000", 2023-10-01T09:47:23.684 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0609999999999999e-05 2023-10-01T09:47:23.685 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:23.685 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:23.685 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:47:23.685 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:23.676821+0000", 2023-10-01T09:47:23.685 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.1370999999999999e-05 2023-10-01T09:47:23.685 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:23.685 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:23.685 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:47:23.685 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:23.676842+0000", 2023-10-01T09:47:23.685 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.9133000000000002e-05 2023-10-01T09:47:23.685 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:47:23.685 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:47:23.686 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:47:23.686 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:47:23.686 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:47:23.686 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 1 2023-10-01T09:47:23.686 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:47:23.699 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:23.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:23 smithi190 ceph-mon[160435]: pgmap v1652: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 292 op/s; 5236/87858 objects degraded (5.960%); 1762/87858 objects misplaced (2.006%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:47:23.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:23 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1689248391' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:47:23.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:23 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:47:23.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:23 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5236/87858 objects degraded (5.960%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:23.727 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:23.884 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:23.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:23.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:23.988 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:24.024 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:24.071 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:24.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:24.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:24.174 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:24.199 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:24.286 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1652,"stamp":"2023-10-01T09:47:22.375158+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1922176971,"num_objects":29286,"num_object_clones":0,"num_object_copies":87858,"num_objects_missing_on_primary":331,"num_objects_missing":331,"num_objects_degraded":5236,"num_objects_misplaced":1762,"num_objects_unfound":0,"num_objects_dirty":29286,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":138288,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11003,"num_bytes_recovered":707953679,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124652,"ondisk_log_size":124652,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11560760,"kb_used_data":6443456,"kb_used_omap":39,"kb_used_meta":5117208,"kb_avail":644655304,"statfs":{"total":671965249536,"available":660127031296,"internally_reserved":0,"allocated":6598098944,"data_stored":6588082613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5240021615},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-191692800,"num_objects":-2925,"num_object_clones":0,"num_object_copies":-8775,"num_objects_missing_on_primary":-160,"num_objects_missing":-160,"num_objects_degraded":-177,"num_objects_misplaced":-197,"num_objects_unfound":0,"num_objects_dirty":-2925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2925,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":214,"num_bytes_recovered":14024704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002716"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":327,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.457608+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:47:06.457608+0000","last_peered":"2023-10-01T09:47:06.457608+0000","last_clean":"2023-10-01T09:47:06.457608+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:47:06.457608+0000","last_undegraded":"2023-10-01T09:47:06.457608+0000","last_fullsized":"2023-10-01T09:47:06.457608+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":171,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447448+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:47:06.447448+0000","last_peered":"2023-10-01T09:47:06.447448+0000","last_clean":"2023-10-01T09:47:06.447448+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:47:06.447448+0000","last_undegraded":"2023-10-01T09:47:06.447448+0000","last_fullsized":"2023-10-01T09:47:06.447448+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":552,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.448390+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:47:06.448390+0000","last_peered":"2023-10-01T09:47:06.448390+0000","last_clean":"2023-10-01T09:47:06.448390+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:47:06.448390+0000","last_undegraded":"2023-10-01T09:47:06.448390+0000","last_fullsized":"2023-10-01T09:47:06.448390+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":611,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.452243+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:47:06.452243+0000","last_peered":"2023-10-01T09:47:06.452243+0000","last_clean":"2023-10-01T09:47:06.452243+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:47:06.452243+0000","last_undegraded":"2023-10-01T09:47:06.452243+0000","last_fullsized":"2023-10-01T09:47:06.452243+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":491,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.454053+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:47:06.454053+0000","last_peered":"2023-10-01T09:47:06.454053+0000","last_clean":"2023-10-01T09:47:06.454053+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:47:06.454053+0000","last_undegraded":"2023-10-01T09:47:06.454053+0000","last_fullsized":"2023-10-01T09:47:06.454053+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":232,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.491419+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:47:06.491419+0000","last_peered":"2023-10-01T09:47:06.491419+0000","last_clean":"2023-10-01T09:47:06.491419+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:47:06.491419+0000","last_undegraded":"2023-10-01T09:47:06.491419+0000","last_fullsized":"2023-10-01T09:47:06.491419+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":608,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.502664+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:47:06.502664+0000","last_peered":"2023-10-01T09:47:06.502664+0000","last_clean":"2023-10-01T09:47:06.502664+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:47:06.502664+0000","last_undegraded":"2023-10-01T09:47:06.502664+0000","last_fullsized":"2023-10-01T09:47:06.502664+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":538,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.455667+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:47:06.455667+0000","last_peered":"2023-10-01T09:47:06.455667+0000","last_clean":"2023-10-01T09:47:06.455667+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:47:06.455667+0000","last_undegraded":"2023-10-01T09:47:06.455667+0000","last_fullsized":"2023-10-01T09:47:06.455667+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":263,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.451907+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:47:06.451907+0000","last_peered":"2023-10-01T09:47:06.451907+0000","last_clean":"2023-10-01T09:47:06.451907+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:47:06.451907+0000","last_undegraded":"2023-10-01T09:47:06.451907+0000","last_fullsized":"2023-10-01T09:47:06.451907+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":70,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447037+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:47:06.447037+0000","last_peered":"2023-10-01T09:47:06.447037+0000","last_clean":"2023-10-01T09:47:06.447037+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:47:06.447037+0000","last_undegraded":"2023-10-01T09:47:06.447037+0000","last_fullsized":"2023-10-01T09:47:06.447037+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":76,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447694+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:47:06.447694+0000","last_peered":"2023-10-01T09:47:06.447694+0000","last_clean":"2023-10-01T09:47:06.447694+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:47:06.447694+0000","last_undegraded":"2023-10-01T09:47:06.447694+0000","last_fullsized":"2023-10-01T09:47:06.447694+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":75,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447189+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:47:06.447189+0000","last_peered":"2023-10-01T09:47:06.447189+0000","last_clean":"2023-10-01T09:47:06.447189+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:47:06.447189+0000","last_undegraded":"2023-10-01T09:47:06.447189+0000","last_fullsized":"2023-10-01T09:47:06.447189+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":452,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.459157+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:47:06.459157+0000","last_peered":"2023-10-01T09:47:06.459157+0000","last_clean":"2023-10-01T09:47:06.459157+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:47:06.459157+0000","last_undegraded":"2023-10-01T09:47:06.459157+0000","last_fullsized":"2023-10-01T09:47:06.459157+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":333,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.457576+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:47:06.457576+0000","last_peered":"2023-10-01T09:47:06.457576+0000","last_clean":"2023-10-01T09:47:06.457576+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:47:06.457576+0000","last_undegraded":"2023-10-01T09:47:06.457576+0000","last_fullsized":"2023-10-01T09:47:06.457576+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":669,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447405+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:47:06.447405+0000","last_peered":"2023-10-01T09:47:06.447405+0000","last_clean":"2023-10-01T09:47:06.447405+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:47:06.447405+0000","last_undegraded":"2023-10-01T09:47:06.447405+0000","last_fullsized":"2023-10-01T09:47:06.447405+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":406,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447688+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:47:06.447688+0000","last_peered":"2023-10-01T09:47:06.447688+0000","last_clean":"2023-10-01T09:47:06.447688+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:47:06.447688+0000","last_undegraded":"2023-10-01T09:47:06.447688+0000","last_fullsized":"2023-10-01T09:47:06.447688+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":79,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447227+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:47:06.447227+0000","last_peered":"2023-10-01T09:47:06.447227+0000","last_clean":"2023-10-01T09:47:06.447227+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:47:06.447227+0000","last_undegraded":"2023-10-01T09:47:06.447227+0000","last_fullsized":"2023-10-01T09:47:06.447227+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":65,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.451385+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:47:06.451385+0000","last_peered":"2023-10-01T09:47:06.451385+0000","last_clean":"2023-10-01T09:47:06.451385+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:47:06.451385+0000","last_undegraded":"2023-10-01T09:47:06.451385+0000","last_fullsized":"2023-10-01T09:47:06.451385+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":562,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447149+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:47:06.447149+0000","last_peered":"2023-10-01T09:47:06.447149+0000","last_clean":"2023-10-01T09:47:06.447149+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:47:06.447149+0000","last_undegraded":"2023-10-01T09:47:06.447149+0000","last_fullsized":"2023-10-01T09:47:06.447149+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":575,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:11.916962+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:47:11.916962+0000","last_peered":"2023-10-01T09:47:11.916962+0000","last_clean":"2023-10-01T09:47:11.916962+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:47:11.916962+0000","last_undegraded":"2023-10-01T09:47:11.916962+0000","last_fullsized":"2023-10-01T09:47:11.916962+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":431,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.453938+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:47:06.453938+0000","last_peered":"2023-10-01T09:47:06.453938+0000","last_clean":"2023-10-01T09:47:06.453938+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:47:06.453938+0000","last_undegraded":"2023-10-01T09:47:06.453938+0000","last_fullsized":"2023-10-01T09:47:06.453938+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":79,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.448183+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:47:06.448183+0000","last_peered":"2023-10-01T09:47:06.448183+0000","last_clean":"2023-10-01T09:47:06.448183+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:47:06.448183+0000","last_undegraded":"2023-10-01T09:47:06.448183+0000","last_fullsized":"2023-10-01T09:47:06.448183+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"551'8504","reported_seq":8588,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:17.265908+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:47:17.265908+0000","last_peered":"2023-10-01T09:47:17.265908+0000","last_clean":"2023-10-01T09:47:17.265908+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:47:17.265908+0000","last_undegraded":"2023-10-01T09:47:17.265908+0000","last_fullsized":"2023-10-01T09:47:17.265908+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8504,"log_dups_size":0,"ondisk_log_size":8504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123731968,"num_objects":1888,"num_object_clones":0,"num_object_copies":5664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8504,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":74,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447802+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:47:06.447802+0000","last_peered":"2023-10-01T09:47:06.447802+0000","last_clean":"2023-10-01T09:47:06.447802+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:47:06.447802+0000","last_undegraded":"2023-10-01T09:47:06.447802+0000","last_fullsized":"2023-10-01T09:47:06.447802+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":271,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.454342+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:47:06.454342+0000","last_peered":"2023-10-01T09:47:06.454342+0000","last_clean":"2023-10-01T09:47:06.454342+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:47:06.454342+0000","last_undegraded":"2023-10-01T09:47:06.454342+0000","last_fullsized":"2023-10-01T09:47:06.454342+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"551'8558","reported_seq":18736,"reported_epoch":551,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:47:17.622767+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:47:17.622767+0000","last_peered":"2023-10-01T09:47:17.622767+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:47:17.622767+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8558,"log_dups_size":0,"ondisk_log_size":8558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121110528,"num_objects":1848,"num_object_clones":0,"num_object_copies":5544,"num_objects_missing_on_primary":331,"num_objects_missing":331,"num_objects_degraded":2110,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8558,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3356,"num_bytes_recovered":219938816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1448},{"shards":"0,4,6","objects":331},{"shards":"5,6","objects":69}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":545,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447881+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:47:06.447881+0000","last_peered":"2023-10-01T09:47:06.447881+0000","last_clean":"2023-10-01T09:47:06.447881+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:47:06.447881+0000","last_undegraded":"2023-10-01T09:47:06.447881+0000","last_fullsized":"2023-10-01T09:47:06.447881+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":608,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:10.284162+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:47:10.284162+0000","last_peered":"2023-10-01T09:47:10.284162+0000","last_clean":"2023-10-01T09:47:10.284162+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:47:10.284162+0000","last_undegraded":"2023-10-01T09:47:10.284162+0000","last_fullsized":"2023-10-01T09:47:10.284162+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"551'8624","reported_seq":8712,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:21.880369+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:47:21.880369+0000","last_peered":"2023-10-01T09:47:21.880369+0000","last_clean":"2023-10-01T09:47:21.880369+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:47:21.880369+0000","last_undegraded":"2023-10-01T09:47:21.880369+0000","last_fullsized":"2023-10-01T09:47:21.880369+0000","mapping_epoch":475,"log_start":"482'1600","ondisk_log_start":"482'1600","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7024,"log_dups_size":0,"ondisk_log_size":7024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118489088,"num_objects":1808,"num_object_clones":0,"num_object_copies":5424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8624,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":631,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447745+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:47:06.447745+0000","last_peered":"2023-10-01T09:47:06.447745+0000","last_clean":"2023-10-01T09:47:06.447745+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:47:06.447745+0000","last_undegraded":"2023-10-01T09:47:06.447745+0000","last_fullsized":"2023-10-01T09:47:06.447745+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":428,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447427+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:47:06.447427+0000","last_peered":"2023-10-01T09:47:06.447427+0000","last_clean":"2023-10-01T09:47:06.447427+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:47:06.447427+0000","last_undegraded":"2023-10-01T09:47:06.447427+0000","last_fullsized":"2023-10-01T09:47:06.447427+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"551'8642","reported_seq":15335,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:21.603503+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:47:21.603503+0000","last_peered":"2023-10-01T09:47:21.603503+0000","last_clean":"2023-10-01T09:47:21.603503+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:47:21.603503+0000","last_undegraded":"2023-10-01T09:47:21.603503+0000","last_fullsized":"2023-10-01T09:47:21.603503+0000","mapping_epoch":505,"log_start":"482'1600","ondisk_log_start":"482'1600","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7042,"log_dups_size":0,"ondisk_log_size":7042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115343360,"num_objects":1760,"num_object_clones":0,"num_object_copies":5280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8642,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":271,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.451241+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:47:06.451241+0000","last_peered":"2023-10-01T09:47:06.451241+0000","last_clean":"2023-10-01T09:47:06.451241+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:47:06.451241+0000","last_undegraded":"2023-10-01T09:47:06.451241+0000","last_fullsized":"2023-10-01T09:47:06.451241+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":531,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.453239+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:47:06.453239+0000","last_peered":"2023-10-01T09:47:06.453239+0000","last_clean":"2023-10-01T09:47:06.453239+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:47:06.453239+0000","last_undegraded":"2023-10-01T09:47:06.453239+0000","last_fullsized":"2023-10-01T09:47:06.453239+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"551'8605","reported_seq":8693,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:20.024929+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:47:20.024929+0000","last_peered":"2023-10-01T09:47:20.024929+0000","last_clean":"2023-10-01T09:47:20.024929+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:47:20.024929+0000","last_undegraded":"2023-10-01T09:47:20.024929+0000","last_fullsized":"2023-10-01T09:47:20.024929+0000","mapping_epoch":475,"log_start":"481'1100","ondisk_log_start":"481'1100","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7505,"log_dups_size":0,"ondisk_log_size":7505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124715008,"num_objects":1903,"num_object_clones":0,"num_object_copies":5709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8605,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":131,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.457905+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:47:06.457905+0000","last_peered":"2023-10-01T09:47:06.457905+0000","last_clean":"2023-10-01T09:47:06.457905+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:47:06.457905+0000","last_undegraded":"2023-10-01T09:47:06.457905+0000","last_fullsized":"2023-10-01T09:47:06.457905+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":205,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.459395+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:47:06.459395+0000","last_peered":"2023-10-01T09:47:06.459395+0000","last_clean":"2023-10-01T09:47:06.459395+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:47:06.459395+0000","last_undegraded":"2023-10-01T09:47:06.459395+0000","last_fullsized":"2023-10-01T09:47:06.459395+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"551'8542","reported_seq":9120,"reported_epoch":551,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:47:17.605596+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:47:17.605596+0000","last_peered":"2023-10-01T09:47:17.605596+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:47:17.605596+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8542,"log_dups_size":0,"ondisk_log_size":8542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119668736,"num_objects":1826,"num_object_clones":0,"num_object_copies":5478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3116,"num_objects_misplaced":1762,"num_objects_unfound":0,"num_objects_dirty":1826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8542,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":243,"num_bytes_recovered":15925248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1762},{"shards":"4,5","objects":-3052}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":449,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.457760+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:47:06.457760+0000","last_peered":"2023-10-01T09:47:06.457760+0000","last_clean":"2023-10-01T09:47:06.457760+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:47:06.457760+0000","last_undegraded":"2023-10-01T09:47:06.457760+0000","last_fullsized":"2023-10-01T09:47:06.457760+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":611,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447517+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:47:06.447517+0000","last_peered":"2023-10-01T09:47:06.447517+0000","last_clean":"2023-10-01T09:47:06.447517+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:47:06.447517+0000","last_undegraded":"2023-10-01T09:47:06.447517+0000","last_fullsized":"2023-10-01T09:47:06.447517+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"551'8514","reported_seq":8607,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:17.272013+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:47:17.272013+0000","last_peered":"2023-10-01T09:47:17.272013+0000","last_clean":"2023-10-01T09:47:17.272013+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:47:17.272013+0000","last_undegraded":"2023-10-01T09:47:17.272013+0000","last_fullsized":"2023-10-01T09:47:17.272013+0000","mapping_epoch":483,"log_start":"481'1000","ondisk_log_start":"481'1000","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7514,"log_dups_size":0,"ondisk_log_size":7514,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122159104,"num_objects":1864,"num_object_clones":0,"num_object_copies":5592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8514,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":445,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.459369+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:47:06.459369+0000","last_peered":"2023-10-01T09:47:06.459369+0000","last_clean":"2023-10-01T09:47:06.459369+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:47:06.459369+0000","last_undegraded":"2023-10-01T09:47:06.459369+0000","last_fullsized":"2023-10-01T09:47:06.459369+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":286,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.458455+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:47:06.458455+0000","last_peered":"2023-10-01T09:47:06.458455+0000","last_clean":"2023-10-01T09:47:06.458455+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:47:06.458455+0000","last_undegraded":"2023-10-01T09:47:06.458455+0000","last_fullsized":"2023-10-01T09:47:06.458455+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"551'8711","reported_seq":8795,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:21.152909+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:47:21.152909+0000","last_peered":"2023-10-01T09:47:21.152909+0000","last_clean":"2023-10-01T09:47:21.152909+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:47:21.152909+0000","last_undegraded":"2023-10-01T09:47:21.152909+0000","last_fullsized":"2023-10-01T09:47:21.152909+0000","mapping_epoch":475,"log_start":"480'300","ondisk_log_start":"480'300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8411,"log_dups_size":0,"ondisk_log_size":8411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117112832,"num_objects":1787,"num_object_clones":0,"num_object_copies":5361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8711,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":415,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447241+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:47:06.447241+0000","last_peered":"2023-10-01T09:47:06.447241+0000","last_clean":"2023-10-01T09:47:06.447241+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:47:06.447241+0000","last_undegraded":"2023-10-01T09:47:06.447241+0000","last_fullsized":"2023-10-01T09:47:06.447241+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":462,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447931+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:47:06.447931+0000","last_peered":"2023-10-01T09:47:06.447931+0000","last_clean":"2023-10-01T09:47:06.447931+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:47:06.447931+0000","last_undegraded":"2023-10-01T09:47:06.447931+0000","last_fullsized":"2023-10-01T09:47:06.447931+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"551'8431","reported_seq":8515,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:21.127794+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:47:21.127794+0000","last_peered":"2023-10-01T09:47:21.127794+0000","last_clean":"2023-10-01T09:47:21.127794+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:47:21.127794+0000","last_undegraded":"2023-10-01T09:47:21.127794+0000","last_fullsized":"2023-10-01T09:47:21.127794+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8431,"log_dups_size":0,"ondisk_log_size":8431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114491392,"num_objects":1747,"num_object_clones":0,"num_object_copies":5241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8431,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":944,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447620+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:47:06.447620+0000","last_peered":"2023-10-01T09:47:06.447620+0000","last_clean":"2023-10-01T09:47:06.447620+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:47:06.447620+0000","last_undegraded":"2023-10-01T09:47:06.447620+0000","last_fullsized":"2023-10-01T09:47:06.447620+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":281,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.451463+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:47:06.451463+0000","last_peered":"2023-10-01T09:47:06.451463+0000","last_clean":"2023-10-01T09:47:06.451463+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:47:06.451463+0000","last_undegraded":"2023-10-01T09:47:06.451463+0000","last_fullsized":"2023-10-01T09:47:06.451463+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":76,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447298+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:47:06.447298+0000","last_peered":"2023-10-01T09:47:06.447298+0000","last_clean":"2023-10-01T09:47:06.447298+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:47:06.447298+0000","last_undegraded":"2023-10-01T09:47:06.447298+0000","last_fullsized":"2023-10-01T09:47:06.447298+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"551'8550","reported_seq":8638,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:21.863139+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:47:21.863139+0000","last_peered":"2023-10-01T09:47:21.863139+0000","last_clean":"2023-10-01T09:47:21.863139+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:47:21.863139+0000","last_undegraded":"2023-10-01T09:47:21.863139+0000","last_fullsized":"2023-10-01T09:47:21.863139+0000","mapping_epoch":475,"log_start":"482'1500","ondisk_log_start":"482'1500","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7050,"log_dups_size":0,"ondisk_log_size":7050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119275520,"num_objects":1820,"num_object_clones":0,"num_object_copies":5460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8550,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1216,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.451090+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:47:06.451090+0000","last_peered":"2023-10-01T09:47:06.451090+0000","last_clean":"2023-10-01T09:47:06.451090+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:47:06.451090+0000","last_undegraded":"2023-10-01T09:47:06.451090+0000","last_fullsized":"2023-10-01T09:47:06.451090+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":426,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.457769+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:47:06.457769+0000","last_peered":"2023-10-01T09:47:06.457769+0000","last_clean":"2023-10-01T09:47:06.457769+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:47:06.457769+0000","last_undegraded":"2023-10-01T09:47:06.457769+0000","last_fullsized":"2023-10-01T09:47:06.457769+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":254,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.451324+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:47:06.451324+0000","last_peered":"2023-10-01T09:47:06.451324+0000","last_clean":"2023-10-01T09:47:06.451324+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:47:06.451324+0000","last_undegraded":"2023-10-01T09:47:06.451324+0000","last_fullsized":"2023-10-01T09:47:06.451324+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"551'8500","reported_seq":15805,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:17.597337+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:47:17.597337+0000","last_peered":"2023-10-01T09:47:17.597337+0000","last_clean":"2023-10-01T09:47:17.597337+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:47:17.597337+0000","last_undegraded":"2023-10-01T09:47:17.597337+0000","last_fullsized":"2023-10-01T09:47:17.597337+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8500,"log_dups_size":0,"ondisk_log_size":8500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122945536,"num_objects":1876,"num_object_clones":0,"num_object_copies":5628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8500,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":898,"reported_epoch":551,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:47:15.471622+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:47:15.471622+0000","last_peered":"2023-10-01T09:47:15.471622+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:47:15.471622+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:47:15.471622+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":615,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447320+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:47:06.447320+0000","last_peered":"2023-10-01T09:47:06.447320+0000","last_clean":"2023-10-01T09:47:06.447320+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:47:06.447320+0000","last_undegraded":"2023-10-01T09:47:06.447320+0000","last_fullsized":"2023-10-01T09:47:06.447320+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":76,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.284753+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:47:15.284753+0000","last_peered":"2023-10-01T09:47:15.284753+0000","last_clean":"2023-10-01T09:47:15.284753+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:47:15.284753+0000","last_undegraded":"2023-10-01T09:47:15.284753+0000","last_fullsized":"2023-10-01T09:47:15.284753+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"551'8465","reported_seq":8553,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:20.020138+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:47:20.020138+0000","last_peered":"2023-10-01T09:47:20.020138+0000","last_clean":"2023-10-01T09:47:20.020138+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:47:20.020138+0000","last_undegraded":"2023-10-01T09:47:20.020138+0000","last_fullsized":"2023-10-01T09:47:20.020138+0000","mapping_epoch":475,"log_start":"481'900","ondisk_log_start":"481'900","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7565,"log_dups_size":0,"ondisk_log_size":7565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118423552,"num_objects":1807,"num_object_clones":0,"num_object_copies":5421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8465,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1667,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.454424+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:47:06.454424+0000","last_peered":"2023-10-01T09:47:06.454424+0000","last_clean":"2023-10-01T09:47:06.454424+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:47:06.454424+0000","last_undegraded":"2023-10-01T09:47:06.454424+0000","last_fullsized":"2023-10-01T09:47:06.454424+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":581,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447032+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:47:06.447032+0000","last_peered":"2023-10-01T09:47:06.447032+0000","last_clean":"2023-10-01T09:47:06.447032+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:47:06.447032+0000","last_undegraded":"2023-10-01T09:47:06.447032+0000","last_fullsized":"2023-10-01T09:47:06.447032+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":359,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.451139+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:47:06.451139+0000","last_peered":"2023-10-01T09:47:06.451139+0000","last_clean":"2023-10-01T09:47:06.451139+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:47:06.451139+0000","last_undegraded":"2023-10-01T09:47:06.451139+0000","last_fullsized":"2023-10-01T09:47:06.451139+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":349,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.459133+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:47:06.459133+0000","last_peered":"2023-10-01T09:47:06.459133+0000","last_clean":"2023-10-01T09:47:06.459133+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:47:06.459133+0000","last_undegraded":"2023-10-01T09:47:06.459133+0000","last_fullsized":"2023-10-01T09:47:06.459133+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"551'8683","reported_seq":8767,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:21.168429+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:47:21.168429+0000","last_peered":"2023-10-01T09:47:21.168429+0000","last_clean":"2023-10-01T09:47:21.168429+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:47:21.168429+0000","last_undegraded":"2023-10-01T09:47:21.168429+0000","last_fullsized":"2023-10-01T09:47:21.168429+0000","mapping_epoch":475,"log_start":"481'300","ondisk_log_start":"481'300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8383,"log_dups_size":0,"ondisk_log_size":8383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121176064,"num_objects":1849,"num_object_clones":0,"num_object_copies":5547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8683,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":758,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447211+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:47:06.447211+0000","last_peered":"2023-10-01T09:47:06.447211+0000","last_clean":"2023-10-01T09:47:06.447211+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:47:06.447211+0000","last_undegraded":"2023-10-01T09:47:06.447211+0000","last_fullsized":"2023-10-01T09:47:06.447211+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":592,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.452169+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:47:06.452169+0000","last_peered":"2023-10-01T09:47:06.452169+0000","last_clean":"2023-10-01T09:47:06.452169+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:47:06.452169+0000","last_undegraded":"2023-10-01T09:47:06.452169+0000","last_fullsized":"2023-10-01T09:47:06.452169+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":591,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.452080+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:47:06.452080+0000","last_peered":"2023-10-01T09:47:06.452080+0000","last_clean":"2023-10-01T09:47:06.452080+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:47:06.452080+0000","last_undegraded":"2023-10-01T09:47:06.452080+0000","last_fullsized":"2023-10-01T09:47:06.452080+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"551'8630","reported_seq":8719,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:21.784837+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:47:21.784837+0000","last_peered":"2023-10-01T09:47:21.784837+0000","last_clean":"2023-10-01T09:47:21.784837+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:47:21.784837+0000","last_undegraded":"2023-10-01T09:47:21.784837+0000","last_fullsized":"2023-10-01T09:47:21.784837+0000","mapping_epoch":475,"log_start":"482'1600","ondisk_log_start":"482'1600","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7030,"log_dups_size":0,"ondisk_log_size":7030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119865368,"num_objects":1830,"num_object_clones":0,"num_object_copies":5490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1830,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8630,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":989,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447786+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:47:06.447786+0000","last_peered":"2023-10-01T09:47:06.447786+0000","last_clean":"2023-10-01T09:47:06.447786+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:47:06.447786+0000","last_undegraded":"2023-10-01T09:47:06.447786+0000","last_fullsized":"2023-10-01T09:47:06.447786+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":608,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.451284+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:47:06.451284+0000","last_peered":"2023-10-01T09:47:06.451284+0000","last_clean":"2023-10-01T09:47:06.451284+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:47:06.451284+0000","last_undegraded":"2023-10-01T09:47:06.451284+0000","last_fullsized":"2023-10-01T09:47:06.451284+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":461,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.454524+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:47:06.454524+0000","last_peered":"2023-10-01T09:47:06.454524+0000","last_clean":"2023-10-01T09:47:06.454524+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:47:06.454524+0000","last_undegraded":"2023-10-01T09:47:06.454524+0000","last_fullsized":"2023-10-01T09:47:06.454524+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"551'8575","reported_seq":8667,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:21.142929+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:47:21.142929+0000","last_peered":"2023-10-01T09:47:21.142929+0000","last_clean":"2023-10-01T09:47:21.142929+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:47:21.142929+0000","last_undegraded":"2023-10-01T09:47:21.142929+0000","last_fullsized":"2023-10-01T09:47:21.142929+0000","mapping_epoch":479,"log_start":"484'1900","ondisk_log_start":"484'1900","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6675,"log_dups_size":0,"ondisk_log_size":6675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116719616,"num_objects":1781,"num_object_clones":0,"num_object_copies":5343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8575,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":490,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.448244+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:47:06.448244+0000","last_peered":"2023-10-01T09:47:06.448244+0000","last_clean":"2023-10-01T09:47:06.448244+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:47:06.448244+0000","last_undegraded":"2023-10-01T09:47:06.448244+0000","last_fullsized":"2023-10-01T09:47:06.448244+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1185,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447745+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:47:06.447745+0000","last_peered":"2023-10-01T09:47:06.447745+0000","last_clean":"2023-10-01T09:47:06.447745+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:47:06.447745+0000","last_undegraded":"2023-10-01T09:47:06.447745+0000","last_fullsized":"2023-10-01T09:47:06.447745+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":79,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.446986+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:47:06.446986+0000","last_peered":"2023-10-01T09:47:06.446986+0000","last_clean":"2023-10-01T09:47:06.446986+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:47:06.446986+0000","last_undegraded":"2023-10-01T09:47:06.446986+0000","last_fullsized":"2023-10-01T09:47:06.446986+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"551'8648","reported_seq":8740,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:21.226889+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:47:21.226889+0000","last_peered":"2023-10-01T09:47:21.226889+0000","last_clean":"2023-10-01T09:47:21.226889+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:47:21.226889+0000","last_undegraded":"2023-10-01T09:47:21.226889+0000","last_fullsized":"2023-10-01T09:47:21.226889+0000","mapping_epoch":479,"log_start":"484'1900","ondisk_log_start":"484'1900","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6748,"log_dups_size":0,"ondisk_log_size":6748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122159104,"num_objects":1864,"num_object_clones":0,"num_object_copies":5592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8648,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":83,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.448173+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:47:06.448173+0000","last_peered":"2023-10-01T09:47:06.448173+0000","last_clean":"2023-10-01T09:47:06.448173+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:47:06.448173+0000","last_undegraded":"2023-10-01T09:47:06.448173+0000","last_fullsized":"2023-10-01T09:47:06.448173+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":569,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.452168+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:47:06.452168+0000","last_peered":"2023-10-01T09:47:06.452168+0000","last_clean":"2023-10-01T09:47:06.452168+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:47:06.452168+0000","last_undegraded":"2023-10-01T09:47:06.452168+0000","last_fullsized":"2023-10-01T09:47:06.452168+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":274,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.450958+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:47:06.450958+0000","last_peered":"2023-10-01T09:47:06.450958+0000","last_clean":"2023-10-01T09:47:06.450958+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:47:06.450958+0000","last_undegraded":"2023-10-01T09:47:06.450958+0000","last_fullsized":"2023-10-01T09:47:06.450958+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":518,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.453403+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:47:06.453403+0000","last_peered":"2023-10-01T09:47:06.453403+0000","last_clean":"2023-10-01T09:47:06.453403+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:47:06.453403+0000","last_undegraded":"2023-10-01T09:47:06.453403+0000","last_fullsized":"2023-10-01T09:47:06.453403+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":632,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447554+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:47:06.447554+0000","last_peered":"2023-10-01T09:47:06.447554+0000","last_clean":"2023-10-01T09:47:06.447554+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:47:06.447554+0000","last_undegraded":"2023-10-01T09:47:06.447554+0000","last_fullsized":"2023-10-01T09:47:06.447554+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":286,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.451198+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:47:06.451198+0000","last_peered":"2023-10-01T09:47:06.451198+0000","last_clean":"2023-10-01T09:47:06.451198+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:47:06.451198+0000","last_undegraded":"2023-10-01T09:47:06.451198+0000","last_fullsized":"2023-10-01T09:47:06.451198+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":75,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.449381+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:47:06.449381+0000","last_peered":"2023-10-01T09:47:06.449381+0000","last_clean":"2023-10-01T09:47:06.449381+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:47:06.449381+0000","last_undegraded":"2023-10-01T09:47:06.449381+0000","last_fullsized":"2023-10-01T09:47:06.449381+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":286,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.451507+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:47:06.451507+0000","last_peered":"2023-10-01T09:47:06.451507+0000","last_clean":"2023-10-01T09:47:06.451507+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:47:06.451507+0000","last_undegraded":"2023-10-01T09:47:06.451507+0000","last_fullsized":"2023-10-01T09:47:06.451507+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":464,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.452966+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:47:06.452966+0000","last_peered":"2023-10-01T09:47:06.452966+0000","last_clean":"2023-10-01T09:47:06.452966+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:47:06.452966+0000","last_undegraded":"2023-10-01T09:47:06.452966+0000","last_fullsized":"2023-10-01T09:47:06.452966+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":93,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.447642+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:47:06.447642+0000","last_peered":"2023-10-01T09:47:06.447642+0000","last_clean":"2023-10-01T09:47:06.447642+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:47:06.447642+0000","last_undegraded":"2023-10-01T09:47:06.447642+0000","last_fullsized":"2023-10-01T09:47:06.447642+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":509,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.452078+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:47:06.452078+0000","last_peered":"2023-10-01T09:47:06.452078+0000","last_clean":"2023-10-01T09:47:06.452078+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:47:06.452078+0000","last_undegraded":"2023-10-01T09:47:06.452078+0000","last_fullsized":"2023-10-01T09:47:06.452078+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":709,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.452576+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:47:06.452576+0000","last_peered":"2023-10-01T09:47:06.452576+0000","last_clean":"2023-10-01T09:47:06.452576+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:47:06.452576+0000","last_undegraded":"2023-10-01T09:47:06.452576+0000","last_fullsized":"2023-10-01T09:47:06.452576+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":402,"reported_epoch":550,"state":"active+clean","last_fresh":"2023-10-01T09:47:06.457427+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:47:06.457427+0000","last_peered":"2023-10-01T09:47:06.457427+0000","last_clean":"2023-10-01T09:47:06.457427+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:47:06.457427+0000","last_undegraded":"2023-10-01T09:47:06.457427+0000","last_fullsized":"2023-10-01T09:47:06.457427+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1917386776,"num_objects":29258,"num_object_clones":0,"num_object_copies":87774,"num_objects_missing_on_primary":331,"num_objects_missing":331,"num_objects_degraded":5226,"num_objects_misplaced":1762,"num_objects_unfound":0,"num_objects_dirty":29258,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137182,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10492,"num_bytes_recovered":687603712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6522482688,"data_stored":6522470472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123482,"ondisk_log_size":123482,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430761,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1490888,"kb_used_data":583680,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":92254264,"statfs":{"total":95995035648,"available":94468366336,"internally_reserved":0,"allocated":597688320,"data_stored":596742516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:02 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.495,"15min":2.4870000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":1.3600000000000001,"5min":3.3849999999999998,"15min":108.90000000000001},"last":0.308},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.46899999999999997,"15min":2.621},"min":{"1min":0.23300000000000001,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":1.7410000000000001,"5min":1.7410000000000001,"15min":109.187},"last":0.433}]},{"osd":1,"last update":"Sun Oct 1 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.58299999999999996,"15min":5.0190000000000001},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.23300000000000001},"max":{"1min":7.7140000000000004,"5min":7.7140000000000004,"15min":229.161},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.53100000000000003,"15min":5.0449999999999999},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":1.972,"5min":3.0640000000000001,"15min":230.126},"last":0.52700000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:46:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.56299999999999994,"15min":1.591},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":1.6359999999999999,"5min":3.4300000000000002,"15min":14.901999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.55700000000000005,"15min":1.48},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.27000000000000002},"max":{"1min":1.8100000000000001,"5min":1.8100000000000001,"15min":14.682},"last":0.45900000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.59299999999999997,"15min":2.1059999999999999},"min":{"1min":0.34999999999999998,"5min":0.30199999999999999,"15min":0.29199999999999998},"max":{"1min":2.1749999999999998,"5min":2.476,"15min":76.555000000000007},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.58399999999999996,"15min":2.1120000000000001},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.1880000000000002,"5min":2.8479999999999999,"15min":76.534000000000006},"last":0.44500000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:47:02 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.52100000000000002,"15min":1.0249999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.4470000000000001,"5min":3.4609999999999999,"15min":9.6020000000000003},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.52500000000000002,"15min":0.99399999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.837,"5min":1.837,"15min":8.4949999999999992},"last":0.47899999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.52500000000000002,"15min":1.5269999999999999},"min":{"1min":0.35399999999999998,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":2.2400000000000002,"5min":2.2400000000000002,"15min":135.40000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.54500000000000004,"15min":1.627},"min":{"1min":0.34999999999999998,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":0.70699999999999996,"5min":2.8849999999999998,"15min":135.339},"last":0.49199999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.58999999999999997,"15min":1.1200000000000001},"min":{"1min":0.40500000000000003,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":1.7829999999999999,"5min":2.448,"15min":11.853999999999999},"last":0.502},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.59699999999999998,"15min":1.093},"min":{"1min":0.39700000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":3.1909999999999998,"5min":3.7810000000000001,"15min":14.584},"last":0.47199999999999998}]}]},{"osd":6,"up_from":294,"seq":1262720385271,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1184580,"kb_used_data":851068,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":92560572,"statfs":{"total":95995035648,"available":94782025728,"internally_reserved":0,"allocated":871493632,"data_stored":870551779,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.45200000000000001,"15min":3.681},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20599999999999999},"max":{"1min":0.51700000000000002,"5min":3.3340000000000001,"15min":158.48599999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.55500000000000005,"15min":3.746},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.221},"max":{"1min":3.3519999999999999,"5min":3.3519999999999999,"15min":158.37700000000001},"last":0.28199999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.58499999999999996,"15min":4.5739999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":3.3069999999999999,"5min":3.3069999999999999,"15min":168.947},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.60299999999999998,"15min":4.532},"min":{"1min":0.28100000000000003,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":3.7469999999999999,"5min":3.7469999999999999,"15min":169.07900000000001},"last":0.318}]},{"osd":2,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.59999999999999998,"15min":1.4490000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":3.3849999999999998,"5min":3.3849999999999998,"15min":10.92},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":1.4690000000000001},"min":{"1min":0.32100000000000001,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":3.7160000000000002,"5min":3.7160000000000002,"15min":21.704999999999998},"last":0.41999999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.57999999999999996,"15min":1.8100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.63800000000000001,"5min":3.3759999999999999,"15min":89.753},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.69499999999999995,"15min":1.754},"min":{"1min":0.309,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.2050000000000001,"5min":4.2050000000000001,"15min":89.849999999999994},"last":0.443}]},{"osd":4,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58899999999999997,"15min":1.069},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.6349999999999998,"5min":4.0250000000000004,"15min":63.134},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.65800000000000003,"15min":1.0680000000000001},"min":{"1min":0.36599999999999999,"5min":0.32700000000000001,"15min":0.28899999999999998},"max":{"1min":1.006,"5min":7.4039999999999999,"15min":62.973999999999997},"last":0.55900000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:46:37 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.88500000000000001,"15min":0.999},"min":{"1min":0.33100000000000002,"5min":0.26700000000000002,"15min":0.251},"max":{"1min":0.93799999999999994,"5min":34.094999999999999,"15min":34.094999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.875,"5min":1.022,"15min":1.1379999999999999},"min":{"1min":0.315,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.181,"5min":34.072000000000003,"15min":34.072000000000003},"last":0.5}]},{"osd":7,"last update":"Sun Oct 1 09:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.56699999999999995,"15min":1.2729999999999999},"min":{"1min":0.377,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.73399999999999999,"5min":3.339,"15min":94.245999999999995},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.67200000000000004,"15min":1.369},"min":{"1min":0.33100000000000002,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":3.653,"5min":3.653,"15min":94.715000000000003},"last":0.52300000000000002}]}]},{"osd":1,"up_from":94,"seq":403726926248,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1196900,"kb_used_data":364764,"kb_used_omap":7,"kb_used_meta":832120,"kb_avail":92548252,"statfs":{"total":95995035648,"available":94769410048,"internally_reserved":0,"allocated":373518336,"data_stored":371432612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":852091303},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":1.0349999999999999,"15min":3.5819999999999999},"min":{"1min":0.19500000000000001,"5min":0.186,"15min":0.182},"max":{"1min":0.82799999999999996,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.217},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.0860000000000001,"15min":3.1030000000000002},"min":{"1min":0.222,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":2.0859999999999999,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.439}]},{"osd":2,"last update":"Sun Oct 1 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.35899999999999999,"15min":2.0179999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.20000000000000001},"max":{"1min":0.78500000000000003,"5min":1.3240000000000001,"15min":51.640000000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.44700000000000001,"15min":2.7570000000000001},"min":{"1min":0.29199999999999998,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":0.69499999999999995,"5min":2.9089999999999998,"15min":154.197},"last":0.40600000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45000000000000001,"15min":2.706},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.219},"max":{"1min":2.149,"5min":2.8780000000000001,"15min":95.305000000000007},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.497,"5min":0.47399999999999998,"15min":3.3210000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.994,"5min":2.7360000000000002,"15min":154.483},"last":0.51200000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.56499999999999995,"15min":2.1389999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":1.204,"5min":3.0550000000000002,"15min":154.101},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.46800000000000003,"15min":2.1469999999999998},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.27700000000000002},"max":{"1min":0.77200000000000002,"5min":0.83199999999999996,"15min":154.56999999999999},"last":0.30099999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.143,"15min":2.23},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.8929999999999998,"5min":59.561,"15min":154.22999999999999},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.53500000000000003,"5min":1.137,"15min":2.3660000000000001},"min":{"1min":0.32900000000000001,"5min":0.31,"15min":0.307},"max":{"1min":0.89600000000000002,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.32800000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.65700000000000003,"15min":2.2770000000000001},"min":{"1min":0.33400000000000002,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":4.5069999999999997,"5min":4.5069999999999997,"15min":154.12200000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.63200000000000001,"15min":2.218},"min":{"1min":0.35899999999999999,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":1.5900000000000001,"5min":2.5630000000000002,"15min":154.16999999999999},"last":0.622}]},{"osd":7,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.73299999999999998,"15min":2.4420000000000002},"min":{"1min":0.38900000000000001,"5min":0.36399999999999999,"15min":0.33900000000000002},"max":{"1min":4.3730000000000002,"5min":4.3730000000000002,"15min":154.20699999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.71999999999999997,"15min":2.4460000000000002},"min":{"1min":0.435,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":4.3879999999999999,"5min":4.3879999999999999,"15min":154.51599999999999},"last":0.47799999999999998}]}]},{"osd":0,"up_from":89,"seq":382252089783,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2763512,"kb_used_data":1828080,"kb_used_omap":8,"kb_used_meta":935415,"kb_avail":90981640,"statfs":{"total":95995035648,"available":93165199360,"internally_reserved":0,"allocated":1871953920,"data_stored":1869915865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":957865922},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.40500000000000003,"15min":3.6469999999999998},"min":{"1min":0.20799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.69099999999999995,"5min":4.2919999999999998,"15min":216.821},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.434,"15min":4.2300000000000004},"min":{"1min":0.20599999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.72499999999999998,"5min":3.9620000000000002,"15min":217.02699999999999},"last":1.5669999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.47499999999999998,"15min":1.9059999999999999},"min":{"1min":0.29399999999999998,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":1.3100000000000001,"5min":4.2430000000000003,"15min":19.788},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54700000000000004,"15min":1.9239999999999999},"min":{"1min":0.29299999999999998,"5min":0.27200000000000002,"15min":0.215},"max":{"1min":1.3300000000000001,"5min":3.8719999999999999,"15min":25.844999999999999},"last":0.56999999999999995}]},{"osd":3,"last update":"Sun Oct 1 09:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.47899999999999998,"15min":2.778},"min":{"1min":0.24299999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.68000000000000005,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.91600000000000004},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.626,"15min":2.6200000000000001},"min":{"1min":0.25700000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.0259999999999998,"5min":4.9199999999999999,"15min":222.227},"last":0.5}]},{"osd":4,"last update":"Sun Oct 1 09:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57199999999999995,"15min":1.8500000000000001},"min":{"1min":0.308,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":2.1160000000000001,"5min":4.048,"15min":62.206000000000003},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.52500000000000002,"15min":1.9359999999999999},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":1.3380000000000001,"5min":4,"15min":62.073999999999998},"last":0.52800000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.59999999999999998,"15min":1.6699999999999999},"min":{"1min":0.34899999999999998,"5min":0.312,"15min":0.312},"max":{"1min":1.5920000000000001,"5min":4.0350000000000001,"15min":16.808},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.59899999999999998,"15min":1.5820000000000001},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.8100000000000001,"5min":4.2729999999999997,"15min":17.507000000000001},"last":0.58199999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:46:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.63600000000000001,"15min":1.6990000000000001},"min":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.1970000000000001,"5min":4.3129999999999997,"15min":17.535},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65900000000000003,"15min":1.732},"min":{"1min":0.372,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.218,"5min":4.2050000000000001,"15min":15.651999999999999},"last":0.51400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.66300000000000003,"15min":1.927},"min":{"1min":0.39700000000000002,"5min":0.374,"15min":0.374},"max":{"1min":2.2290000000000001,"5min":4.3360000000000003,"15min":48.895000000000003},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.79400000000000004,"15min":1.97},"min":{"1min":0.39300000000000002,"5min":0.376,"15min":0.35299999999999998},"max":{"1min":0.78300000000000003,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.54700000000000004}]}]},{"osd":2,"up_from":0,"seq":425201762722,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599195,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1228380,"kb_used_data":378196,"kb_used_omap":3,"kb_used_meta":850172,"kb_avail":92516772,"statfs":{"total":95995035648,"available":94737174528,"internally_reserved":0,"allocated":387272704,"data_stored":385136501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":870576871},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.26900000000000002,"5min":0.34300000000000003,"15min":3.5009999999999999},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":0.504,"5min":2.7989999999999999,"15min":137.34399999999999},"last":0.251},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.35899999999999999,"15min":3.5459999999999998},"min":{"1min":0.23400000000000001,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":0.66800000000000004,"5min":1.2849999999999999,"15min":137.73400000000001},"last":0.41499999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.40200000000000002,"15min":4.7430000000000003},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":0.63500000000000001,"5min":2.7250000000000001,"15min":162.22999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.371,"15min":4.7370000000000001},"min":{"1min":0.27200000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.3440000000000001,"5min":1.4119999999999999,"15min":162.70699999999999},"last":0.34499999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.442,"15min":1.552},"min":{"1min":0.251,"5min":0.24199999999999999,"15min":0.20799999999999999},"max":{"1min":0.46000000000000002,"5min":2.9049999999999998,"15min":14.254},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.499,"15min":1.4350000000000001},"min":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":2.5230000000000001,"5min":4.5049999999999999,"15min":15.523},"last":0.311}]},{"osd":4,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.53400000000000003,"15min":1.653},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.26200000000000001},"max":{"1min":4.5010000000000003,"5min":4.5010000000000003,"15min":77.793000000000006},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.499,"5min":0.49399999999999999,"15min":1.8120000000000001},"min":{"1min":0.35299999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":0.72299999999999998,"5min":1.5609999999999999,"15min":76.646000000000001},"last":0.69899999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.58399999999999996,"15min":1.9119999999999999},"min":{"1min":0.32500000000000001,"5min":0.314,"15min":0.27500000000000002},"max":{"1min":0.61399999999999999,"5min":4.3099999999999996,"15min":28.117999999999999},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51800000000000002,"15min":2.0310000000000001},"min":{"1min":0.33800000000000002,"5min":0.33600000000000002,"15min":0.27700000000000002},"max":{"1min":1.9930000000000001,"5min":2.512,"15min":28.34},"last":0.40500000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.55500000000000005,"15min":1.764},"min":{"1min":0.379,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":0.56899999999999995,"5min":2.9100000000000001,"15min":73.887},"last":0.497},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.58499999999999996,"15min":1.8560000000000001},"min":{"1min":0.379,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":0.60499999999999998,"5min":2.6000000000000001,"15min":74.016999999999996},"last":0.44600000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":0.85999999999999999,"15min":1.7729999999999999},"min":{"1min":0.42799999999999999,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":4.1989999999999998,"5min":4.1989999999999998,"15min":13.085000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.2030000000000001,"5min":0.80500000000000005,"15min":1.5529999999999999},"min":{"1min":0.432,"5min":0.38800000000000001,"15min":0.311},"max":{"1min":4.3280000000000003,"5min":4.3280000000000003,"15min":13.191000000000001},"last":0.436}]}]},{"osd":4,"up_from":30,"seq":128849019446,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2501316,"kb_used_data":1615228,"kb_used_omap":0,"kb_used_meta":886080,"kb_avail":91243836,"statfs":{"total":95995035648,"available":93433688064,"internally_reserved":0,"allocated":1653993472,"data_stored":1653051174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":907345920},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7480000000000002,"5min":0.89400000000000002,"15min":3.5819999999999999},"min":{"1min":0.24399999999999999,"5min":0.22500000000000001,"15min":0.21199999999999999},"max":{"1min":41.963999999999999,"5min":41.963999999999999,"15min":155.315},"last":0.25800000000000001},{"interface":"front","average":{"1min":2.7530000000000001,"5min":0.90100000000000002,"15min":3.9140000000000001},"min":{"1min":0.26100000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":41.890999999999998,"5min":41.890999999999998,"15min":155.334},"last":4.8579999999999997}]},{"osd":1,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.54000000000000004,"15min":5.1449999999999996},"min":{"1min":0.25600000000000001,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":3.3929999999999998,"5min":3.3929999999999998,"15min":190.61199999999999},"last":4.9539999999999997},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.54900000000000004,"15min":5.8639999999999999},"min":{"1min":0.29199999999999998,"5min":0.27500000000000002,"15min":0.25600000000000001},"max":{"1min":3.3839999999999999,"5min":3.3839999999999999,"15min":253.273},"last":0.378}]},{"osd":2,"last update":"Sun Oct 1 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.621,"15min":2.0499999999999998},"min":{"1min":0.30499999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.343,"5min":4.2329999999999997,"15min":170.45400000000001},"last":4.9859999999999998},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.61499999999999999,"15min":1.8069999999999999},"min":{"1min":0.35399999999999998,"5min":0.314,"15min":0.309},"max":{"1min":3.1389999999999998,"5min":3.1389999999999998,"15min":36.515999999999998},"last":4.9210000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.61199999999999999,"15min":1.9850000000000001},"min":{"1min":0.307,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":3.1539999999999999,"5min":3.1539999999999999,"15min":140.881},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.61399999999999999,"15min":2.0579999999999998},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.311},"max":{"1min":1.3049999999999999,"5min":6.2969999999999997,"15min":140.911},"last":0.436}]},{"osd":5,"last update":"Sun Oct 1 09:46:41 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.56999999999999995,"15min":1.454},"min":{"1min":0.27200000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":1.8500000000000001,"5min":3.052,"15min":110.003},"last":8.0749999999999993},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.63400000000000001,"15min":1.3859999999999999},"min":{"1min":0.34100000000000003,"5min":0.317,"15min":0.28899999999999998},"max":{"1min":1.754,"5min":3.2909999999999999,"15min":108.56399999999999},"last":0.40699999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:47:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.56499999999999995,"15min":0.91700000000000004},"min":{"1min":0.318,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":3.3029999999999999,"5min":3.3029999999999999,"15min":11.358000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.60999999999999999,"15min":1.046},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.28999999999999998},"max":{"1min":2.2669999999999999,"5min":2.9239999999999999,"15min":10.076000000000001},"last":5.4219999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.63400000000000001,"15min":1.149},"min":{"1min":0.39200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.3079999999999998,"5min":3.3079999999999998,"15min":38.034999999999997},"last":0.67200000000000004},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.70899999999999996,"15min":1.3839999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.308},"max":{"1min":3.0329999999999999,"5min":3.113,"15min":37.723999999999997},"last":8.0190000000000001}]}]},{"osd":5,"up_from":379,"seq":1627792605352,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1195184,"kb_used_data":822440,"kb_used_omap":13,"kb_used_meta":372722,"kb_avail":92549968,"statfs":{"total":95995035648,"available":94771167232,"internally_reserved":0,"allocated":842178560,"data_stored":841252166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":381667826},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.749,"15min":2.0830000000000002},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.21099999999999999},"max":{"1min":0.77600000000000002,"5min":27.584,"15min":84.340999999999994},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.76900000000000002,"15min":2.0369999999999999},"min":{"1min":0.28399999999999997,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.749,"5min":27.669,"15min":84.356999999999999},"last":0.23899999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.55200000000000005,"15min":5.5519999999999996},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.2130000000000001,"5min":4.3289999999999997,"15min":295.71300000000002},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.51300000000000001,"15min":5.5609999999999999},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":0.70399999999999996,"5min":3.9580000000000002,"15min":299.59399999999999},"last":0.47399999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.52400000000000002,"15min":1.45},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78600000000000003,"5min":4.2670000000000003,"15min":14.199999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.65100000000000002,"15min":1.4139999999999999},"min":{"1min":0.29599999999999999,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":4.2910000000000004,"5min":4.3719999999999999,"15min":14.430999999999999},"last":0.53000000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.56699999999999995,"15min":1.3720000000000001},"min":{"1min":0.32800000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.75,"5min":4.0209999999999999,"15min":14.004},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.5,"5min":0.58799999999999997,"15min":1.518},"min":{"1min":0.40100000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":0.83299999999999996,"5min":4.2910000000000004,"15min":13.991},"last":0.56299999999999994}]},{"osd":4,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.53300000000000003,"15min":1.365},"min":{"1min":0.29499999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.0620000000000001,"5min":4.1929999999999996,"15min":122.971},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.623,"15min":1.5389999999999999},"min":{"1min":0.28899999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2619999999999996,"5min":4.3869999999999996,"15min":122.89100000000001},"last":0.48499999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.54900000000000004,"15min":0.95399999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":0.82799999999999996,"5min":4.3300000000000001,"15min":12.545},"last":0.438},{"interface":"front","average":{"1min":0.5,"5min":0.64100000000000001,"15min":1.1459999999999999},"min":{"1min":0.36499999999999999,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":0.81899999999999995,"5min":4.3200000000000003,"15min":14.193},"last":0.44700000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.68300000000000005,"15min":1.1499999999999999},"min":{"1min":0.35399999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.3170000000000002,"5min":4.3559999999999999,"15min":10.813000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.67200000000000004,"15min":1.0469999999999999},"min":{"1min":0.39500000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":1.5369999999999999,"5min":4.2000000000000002,"15min":12.500999999999999},"last":0.59399999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1853427712,"data_stored":1853423640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":362741760,"data_stored":362741760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":379125760,"data_stored":379125760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1642926080,"data_stored":1642922008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":835780608,"data_stored":835780608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":860950528,"data_stored":860946456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":587530240,"data_stored":587530240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:47:24.452 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:24.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:24.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:24.546 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:47:24.561 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:24.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:24 smithi138 ceph-mon[185886]: from='client.47423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:24.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:24 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:24.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:24 smithi138 ceph-mon[185886]: failed to encode map e552 with expected crc 2023-10-01T09:47:24.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:24 smithi138 ceph-mon[185886]: failed to encode map e552 with expected crc 2023-10-01T09:47:24.567 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:24 smithi138 ceph-mon[185886]: failed to encode map e552 with expected crc 2023-10-01T09:47:24.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:24 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:47:24.568 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:24 smithi138 ceph-mon[185886]: osdmap e552: 8 total, 8 up, 7 in 2023-10-01T09:47:24.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:24 smithi138 ceph-mon[192513]: from='client.47423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:24.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:24 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:24.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:24 smithi138 ceph-mon[192513]: failed to encode map e552 with expected crc 2023-10-01T09:47:24.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:24 smithi138 ceph-mon[192513]: failed to encode map e552 with expected crc 2023-10-01T09:47:24.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:24 smithi138 ceph-mon[192513]: failed to encode map e552 with expected crc 2023-10-01T09:47:24.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:24 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:47:24.568 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:24 smithi138 ceph-mon[192513]: osdmap e552: 8 total, 8 up, 7 in 2023-10-01T09:47:24.667 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:24.753 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:24.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:24.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:24.760 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:24 smithi190 ceph-mon[160435]: from='client.47423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:24.760 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:24 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:24.760 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:24 smithi190 ceph-mon[160435]: failed to encode map e552 with expected crc 2023-10-01T09:47:24.760 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:24 smithi190 ceph-mon[160435]: failed to encode map e552 with expected crc 2023-10-01T09:47:24.761 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:24 smithi190 ceph-mon[160435]: failed to encode map e552 with expected crc 2023-10-01T09:47:24.761 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:24 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:47:24.761 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:24 smithi190 ceph-mon[160435]: osdmap e552: 8 total, 8 up, 7 in 2023-10-01T09:47:24.861 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:47:24.893 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:24.931 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:47:24.952 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:47:24.965 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:47:24.972 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:47:24.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:24.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:47:24.992 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:47:25.011 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:47:25.023 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:47:25.033 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:47:25.054 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:47:25.076 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:47:25.077 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:25.078 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:47:25.078 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:47:25.078 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:47:25.078 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:47:25.079 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:47:25.079 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:47:25.079 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:47:25.079 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:47:25.109 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:25.248 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:47:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:47:24] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.33.4" 2023-10-01T09:47:25.258 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:25.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:25.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:25.362 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:25.388 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:25.435 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:25.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:25.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:25.540 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:25 smithi190 ceph-mon[160435]: pgmap v1654: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 307 op/s; 5118/86592 objects degraded (5.910%); 1680/86592 objects misplaced (1.940%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:47:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:25 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/991503813' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:25.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:47:25.565 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:25 smithi138 ceph-mon[185886]: pgmap v1654: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 307 op/s; 5118/86592 objects degraded (5.910%); 1680/86592 objects misplaced (1.940%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:47:25.565 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:25 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/991503813' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:25.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:47:25.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:25 smithi138 ceph-mon[192513]: pgmap v1654: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 307 op/s; 5118/86592 objects degraded (5.910%); 1680/86592 objects misplaced (1.940%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:47:25.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:25 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/991503813' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:25.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:47:25.568 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:25.763 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:25.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:25.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:25.869 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:47:25.896 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:25.944 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:47:25.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:25.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:47:26.047 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:26.080 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:26.129 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:26.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:26.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:26.235 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:26.262 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:26.315 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:26.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:26.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:26.423 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:26.450 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:26.504 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:26.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:26.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:26.585 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:47:26.610 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:26.620 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:26 smithi190 ceph-mon[160435]: pgmap v1655: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 5118/84918 objects degraded (6.027%); 1680/84918 objects misplaced (1.978%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:47:26.683 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:26.810 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:26.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:26.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:26.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:26 smithi138 ceph-mon[192513]: pgmap v1655: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 5118/84918 objects degraded (6.027%); 1680/84918 objects misplaced (1.978%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:47:26.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:26 smithi138 ceph-mon[185886]: pgmap v1655: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 5118/84918 objects degraded (6.027%); 1680/84918 objects misplaced (1.978%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:47:26.895 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:47:26.895 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:47:26.918 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:26.945 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:26.971 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:27.003 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:27.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:27.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:27.109 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:27.147 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:27.213 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:27.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:27.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:27.294 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:47:27.323 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:27.348 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:27.390 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:27.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:27.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:27.494 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:27.524 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:27.755 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:47:27.895 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:47:27.965 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:27.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:27.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:28.069 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:47:28.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:27 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5118/84918 objects degraded (6.027%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:28.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:27 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5118/84918 objects degraded (6.027%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:28.108 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:28.214 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:47:28.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:28.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:47:28.283 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:28.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:27 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5118/84918 objects degraded (6.027%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:28.311 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:28.318 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:28.363 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:28.400 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:47:28.429 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:28.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:28.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:28.545 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:28.582 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:28.747 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:28.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:28.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:28.851 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:28.887 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:28.913 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:47:28.934 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:28 smithi138 ceph-mon[192513]: pgmap v1656: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 5003/82725 objects degraded (6.048%); 1601/82725 objects misplaced (1.935%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:47:28.935 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1592640943' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:28.935 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:28 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:28.935 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:28 smithi138 ceph-mon[185886]: pgmap v1656: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 5003/82725 objects degraded (6.048%); 1601/82725 objects misplaced (1.935%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:47:28.935 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1592640943' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:28.935 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:28 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:29.051 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:28 smithi190 ceph-mon[160435]: pgmap v1656: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 266 op/s; 5003/82725 objects degraded (6.048%); 1601/82725 objects misplaced (1.935%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:47:29.051 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1592640943' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:29.051 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:28 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:29.121 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:29.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:29.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:29.230 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:29.258 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:29.310 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:29.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:29.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:29.419 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:47:29.473 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:29.736 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:47:29.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:29.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:47:29.853 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:47:29.884 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:30.000 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:29 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:47:30.000 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:29 smithi190 ceph-mon[160435]: failed to encode map e553 with expected crc 2023-10-01T09:47:30.000 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:29 smithi190 ceph-mon[160435]: failed to encode map e553 with expected crc 2023-10-01T09:47:30.001 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:29 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:47:30.001 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:29 smithi190 ceph-mon[160435]: osdmap e553: 8 total, 8 up, 7 in 2023-10-01T09:47:30.069 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:29 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:47:30.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:29 smithi138 ceph-mon[185886]: failed to encode map e553 with expected crc 2023-10-01T09:47:30.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:29 smithi138 ceph-mon[185886]: failed to encode map e553 with expected crc 2023-10-01T09:47:30.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:29 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:47:30.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:29 smithi138 ceph-mon[185886]: osdmap e553: 8 total, 8 up, 7 in 2023-10-01T09:47:30.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:29 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:47:30.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:29 smithi138 ceph-mon[192513]: failed to encode map e553 with expected crc 2023-10-01T09:47:30.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:29 smithi138 ceph-mon[192513]: failed to encode map e553 with expected crc 2023-10-01T09:47:30.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:29 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:47:30.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:29 smithi138 ceph-mon[192513]: osdmap e553: 8 total, 8 up, 7 in 2023-10-01T09:47:30.244 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:47:30.244 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:47:30.551 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:47:30.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:30.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:47:30.657 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:30.687 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:30.915 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:30.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:30.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:30.999 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:47:31.002 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:30 smithi138 ceph-mon[192513]: pgmap v1658: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 236 op/s; 5003/82182 objects degraded (6.088%); 1601/82182 objects misplaced (1.948%); 1.8 MiB/s, 28 objects/s recovering 2023-10-01T09:47:31.002 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:30 smithi138 ceph-mon[185886]: pgmap v1658: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 236 op/s; 5003/82182 objects degraded (6.088%); 1601/82182 objects misplaced (1.948%); 1.8 MiB/s, 28 objects/s recovering 2023-10-01T09:47:31.019 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:31.024 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:47:31.050 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:47:31.054 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:31.077 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:47:31.102 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:47:31.126 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:47:31.145 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:47:31.154 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:47:31.171 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:31.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:31.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:31.182 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:47:31.184 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:47:31.184 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:47:31.184 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:47:31.184 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:47:31.184 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:47:31.184 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:47:31.184 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:47:31.185 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:47:31.186 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:30 smithi190 ceph-mon[160435]: pgmap v1658: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 236 op/s; 5003/82182 objects degraded (6.088%); 1601/82182 objects misplaced (1.948%); 1.8 MiB/s, 28 objects/s recovering 2023-10-01T09:47:31.275 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:31.310 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:31.373 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:31.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:31.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:31.450 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:47:31.452 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:47:31.478 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:47:31.511 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:31.615 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:47:31.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:31.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:47:31.679 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:47:31.728 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:31.758 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:32.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4214782147' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:32.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:31 smithi138 ceph-mon[192513]: from='client.37416 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:32.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4214782147' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:32.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:31 smithi138 ceph-mon[185886]: from='client.37416 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:32.279 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4214782147' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:32.279 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:31 smithi190 ceph-mon[160435]: from='client.37416 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:32.423 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:32.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:32.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:32.484 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1658,"stamp":"2023-10-01T09:47:30.377626+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1798182859,"num_objects":27394,"num_object_clones":0,"num_object_copies":82182,"num_objects_missing_on_primary":163,"num_objects_missing":163,"num_objects_degraded":5003,"num_objects_misplaced":1601,"num_objects_unfound":0,"num_objects_dirty":27394,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":140180,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11229,"num_bytes_recovered":722764815,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125344,"ondisk_log_size":125344,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11242616,"kb_used_data":6121216,"kb_used_omap":39,"kb_used_meta":5121304,"kb_avail":644973448,"statfs":{"total":671965249536,"available":660452810752,"internally_reserved":0,"allocated":6268125184,"data_stored":6258106261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5244215919},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1,0,3,1,0,5,7],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-123994112,"num_objects":-1892,"num_object_clones":0,"num_object_copies":-5676,"num_objects_missing_on_primary":-168,"num_objects_missing":-168,"num_objects_degraded":-233,"num_objects_misplaced":-161,"num_objects_unfound":0,"num_objects_dirty":-1892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1892,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":226,"num_bytes_recovered":14811136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002467"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":328,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.496716+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:47:15.496716+0000","last_peered":"2023-10-01T09:47:15.496716+0000","last_clean":"2023-10-01T09:47:15.496716+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:47:15.496716+0000","last_undegraded":"2023-10-01T09:47:15.496716+0000","last_fullsized":"2023-10-01T09:47:15.496716+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":173,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498919+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:47:23.498919+0000","last_peered":"2023-10-01T09:47:23.498919+0000","last_clean":"2023-10-01T09:47:23.498919+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:47:23.498919+0000","last_undegraded":"2023-10-01T09:47:23.498919+0000","last_fullsized":"2023-10-01T09:47:23.498919+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":554,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498509+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:47:23.498509+0000","last_peered":"2023-10-01T09:47:23.498509+0000","last_clean":"2023-10-01T09:47:23.498509+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:47:23.498509+0000","last_undegraded":"2023-10-01T09:47:23.498509+0000","last_fullsized":"2023-10-01T09:47:23.498509+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":612,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.472015+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:47:15.472015+0000","last_peered":"2023-10-01T09:47:15.472015+0000","last_clean":"2023-10-01T09:47:15.472015+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:47:15.472015+0000","last_undegraded":"2023-10-01T09:47:15.472015+0000","last_fullsized":"2023-10-01T09:47:15.472015+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":492,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.472939+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:47:15.472939+0000","last_peered":"2023-10-01T09:47:15.472939+0000","last_clean":"2023-10-01T09:47:15.472939+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:47:15.472939+0000","last_undegraded":"2023-10-01T09:47:15.472939+0000","last_fullsized":"2023-10-01T09:47:15.472939+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":233,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.559245+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:47:15.559245+0000","last_peered":"2023-10-01T09:47:15.559245+0000","last_clean":"2023-10-01T09:47:15.559245+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:47:15.559245+0000","last_undegraded":"2023-10-01T09:47:15.559245+0000","last_fullsized":"2023-10-01T09:47:15.559245+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":609,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.559308+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:47:15.559308+0000","last_peered":"2023-10-01T09:47:15.559308+0000","last_clean":"2023-10-01T09:47:15.559308+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:47:15.559308+0000","last_undegraded":"2023-10-01T09:47:15.559308+0000","last_fullsized":"2023-10-01T09:47:15.559308+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":540,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498771+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:47:23.498771+0000","last_peered":"2023-10-01T09:47:23.498771+0000","last_clean":"2023-10-01T09:47:23.498771+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:47:23.498771+0000","last_undegraded":"2023-10-01T09:47:23.498771+0000","last_fullsized":"2023-10-01T09:47:23.498771+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":264,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.464491+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:47:15.464491+0000","last_peered":"2023-10-01T09:47:15.464491+0000","last_clean":"2023-10-01T09:47:15.464491+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:47:15.464491+0000","last_undegraded":"2023-10-01T09:47:15.464491+0000","last_fullsized":"2023-10-01T09:47:15.464491+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":71,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465289+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:47:15.465289+0000","last_peered":"2023-10-01T09:47:15.465289+0000","last_clean":"2023-10-01T09:47:15.465289+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:47:15.465289+0000","last_undegraded":"2023-10-01T09:47:15.465289+0000","last_fullsized":"2023-10-01T09:47:15.465289+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":77,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465696+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:47:15.465696+0000","last_peered":"2023-10-01T09:47:15.465696+0000","last_clean":"2023-10-01T09:47:15.465696+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:47:15.465696+0000","last_undegraded":"2023-10-01T09:47:15.465696+0000","last_fullsized":"2023-10-01T09:47:15.465696+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":77,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498639+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:47:23.498639+0000","last_peered":"2023-10-01T09:47:23.498639+0000","last_clean":"2023-10-01T09:47:23.498639+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:47:23.498639+0000","last_undegraded":"2023-10-01T09:47:23.498639+0000","last_fullsized":"2023-10-01T09:47:23.498639+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":453,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.497169+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:47:15.497169+0000","last_peered":"2023-10-01T09:47:15.497169+0000","last_clean":"2023-10-01T09:47:15.497169+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:47:15.497169+0000","last_undegraded":"2023-10-01T09:47:15.497169+0000","last_fullsized":"2023-10-01T09:47:15.497169+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":334,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.496827+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:47:15.496827+0000","last_peered":"2023-10-01T09:47:15.496827+0000","last_clean":"2023-10-01T09:47:15.496827+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:47:15.496827+0000","last_undegraded":"2023-10-01T09:47:15.496827+0000","last_fullsized":"2023-10-01T09:47:15.496827+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":670,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465464+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:47:15.465464+0000","last_peered":"2023-10-01T09:47:15.465464+0000","last_clean":"2023-10-01T09:47:15.465464+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:47:15.465464+0000","last_undegraded":"2023-10-01T09:47:15.465464+0000","last_fullsized":"2023-10-01T09:47:15.465464+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":407,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465637+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:47:15.465637+0000","last_peered":"2023-10-01T09:47:15.465637+0000","last_clean":"2023-10-01T09:47:15.465637+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:47:15.465637+0000","last_undegraded":"2023-10-01T09:47:15.465637+0000","last_fullsized":"2023-10-01T09:47:15.465637+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":80,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465369+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:47:15.465369+0000","last_peered":"2023-10-01T09:47:15.465369+0000","last_clean":"2023-10-01T09:47:15.465369+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:47:15.465369+0000","last_undegraded":"2023-10-01T09:47:15.465369+0000","last_fullsized":"2023-10-01T09:47:15.465369+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":66,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.464326+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:47:15.464326+0000","last_peered":"2023-10-01T09:47:15.464326+0000","last_clean":"2023-10-01T09:47:15.464326+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:47:15.464326+0000","last_undegraded":"2023-10-01T09:47:15.464326+0000","last_fullsized":"2023-10-01T09:47:15.464326+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":563,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465521+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:47:15.465521+0000","last_peered":"2023-10-01T09:47:15.465521+0000","last_clean":"2023-10-01T09:47:15.465521+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:47:15.465521+0000","last_undegraded":"2023-10-01T09:47:15.465521+0000","last_fullsized":"2023-10-01T09:47:15.465521+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":576,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.472083+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:47:15.472083+0000","last_peered":"2023-10-01T09:47:15.472083+0000","last_clean":"2023-10-01T09:47:15.472083+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:47:15.472083+0000","last_undegraded":"2023-10-01T09:47:15.472083+0000","last_fullsized":"2023-10-01T09:47:15.472083+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":432,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.472229+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:47:15.472229+0000","last_peered":"2023-10-01T09:47:15.472229+0000","last_clean":"2023-10-01T09:47:15.472229+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:47:15.472229+0000","last_undegraded":"2023-10-01T09:47:15.472229+0000","last_fullsized":"2023-10-01T09:47:15.472229+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":80,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.466070+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:47:15.466070+0000","last_peered":"2023-10-01T09:47:15.466070+0000","last_clean":"2023-10-01T09:47:15.466070+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:47:15.466070+0000","last_undegraded":"2023-10-01T09:47:15.466070+0000","last_fullsized":"2023-10-01T09:47:15.466070+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"552'8688","reported_seq":8773,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:27.176618+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:47:27.176618+0000","last_peered":"2023-10-01T09:47:27.176618+0000","last_clean":"2023-10-01T09:47:27.176618+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:47:27.176618+0000","last_undegraded":"2023-10-01T09:47:27.176618+0000","last_fullsized":"2023-10-01T09:47:27.176618+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8688,"log_dups_size":0,"ondisk_log_size":8688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111673344,"num_objects":1704,"num_object_clones":0,"num_object_copies":5112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8688,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":75,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465907+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:47:15.465907+0000","last_peered":"2023-10-01T09:47:15.465907+0000","last_clean":"2023-10-01T09:47:15.465907+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:47:15.465907+0000","last_undegraded":"2023-10-01T09:47:15.465907+0000","last_fullsized":"2023-10-01T09:47:15.465907+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":272,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.464200+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:47:15.464200+0000","last_peered":"2023-10-01T09:47:15.464200+0000","last_clean":"2023-10-01T09:47:15.464200+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:47:15.464200+0000","last_undegraded":"2023-10-01T09:47:15.464200+0000","last_fullsized":"2023-10-01T09:47:15.464200+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"552'8728","reported_seq":19411,"reported_epoch":552,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:47:27.658432+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:47:27.658432+0000","last_peered":"2023-10-01T09:47:27.658432+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:47:27.658432+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8728,"log_dups_size":0,"ondisk_log_size":8728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109969408,"num_objects":1678,"num_object_clones":0,"num_object_copies":5034,"num_objects_missing_on_primary":163,"num_objects_missing":163,"num_objects_degraded":1773,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8728,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3525,"num_bytes_recovered":231014400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1447},{"shards":"0,4,6","objects":163},{"shards":"5,6","objects":68}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":546,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465959+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:47:15.465959+0000","last_peered":"2023-10-01T09:47:15.465959+0000","last_clean":"2023-10-01T09:47:15.465959+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:47:15.465959+0000","last_undegraded":"2023-10-01T09:47:15.465959+0000","last_fullsized":"2023-10-01T09:47:15.465959+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":610,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498576+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:47:23.498576+0000","last_peered":"2023-10-01T09:47:23.498576+0000","last_clean":"2023-10-01T09:47:23.498576+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:47:23.498576+0000","last_undegraded":"2023-10-01T09:47:23.498576+0000","last_fullsized":"2023-10-01T09:47:23.498576+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"552'8709","reported_seq":8798,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:26.822791+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:47:26.822791+0000","last_peered":"2023-10-01T09:47:26.822791+0000","last_clean":"2023-10-01T09:47:26.822791+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:47:26.822791+0000","last_undegraded":"2023-10-01T09:47:26.822791+0000","last_fullsized":"2023-10-01T09:47:26.822791+0000","mapping_epoch":475,"log_start":"484'1700","ondisk_log_start":"484'1700","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7009,"log_dups_size":0,"ondisk_log_size":7009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112918528,"num_objects":1723,"num_object_clones":0,"num_object_copies":5169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8709,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":632,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465728+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:47:15.465728+0000","last_peered":"2023-10-01T09:47:15.465728+0000","last_clean":"2023-10-01T09:47:15.465728+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:47:15.465728+0000","last_undegraded":"2023-10-01T09:47:15.465728+0000","last_fullsized":"2023-10-01T09:47:15.465728+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":430,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498760+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:47:23.498760+0000","last_peered":"2023-10-01T09:47:23.498760+0000","last_clean":"2023-10-01T09:47:23.498760+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:47:23.498760+0000","last_undegraded":"2023-10-01T09:47:23.498760+0000","last_fullsized":"2023-10-01T09:47:23.498760+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"552'8719","reported_seq":15413,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:26.659580+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:47:26.659580+0000","last_peered":"2023-10-01T09:47:26.659580+0000","last_clean":"2023-10-01T09:47:26.659580+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:47:26.659580+0000","last_undegraded":"2023-10-01T09:47:26.659580+0000","last_fullsized":"2023-10-01T09:47:26.659580+0000","mapping_epoch":505,"log_start":"484'1700","ondisk_log_start":"484'1700","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7019,"log_dups_size":0,"ondisk_log_size":7019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110297088,"num_objects":1683,"num_object_clones":0,"num_object_copies":5049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8719,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":272,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.464505+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:47:15.464505+0000","last_peered":"2023-10-01T09:47:15.464505+0000","last_clean":"2023-10-01T09:47:15.464505+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:47:15.464505+0000","last_undegraded":"2023-10-01T09:47:15.464505+0000","last_fullsized":"2023-10-01T09:47:15.464505+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":532,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.472718+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:47:15.472718+0000","last_peered":"2023-10-01T09:47:15.472718+0000","last_clean":"2023-10-01T09:47:15.472718+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:47:15.472718+0000","last_undegraded":"2023-10-01T09:47:15.472718+0000","last_fullsized":"2023-10-01T09:47:15.472718+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"553'8759","reported_seq":8849,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:30.243174+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:47:30.243174+0000","last_peered":"2023-10-01T09:47:30.243174+0000","last_clean":"2023-10-01T09:47:30.243174+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:47:30.243174+0000","last_undegraded":"2023-10-01T09:47:30.243174+0000","last_fullsized":"2023-10-01T09:47:30.243174+0000","mapping_epoch":475,"log_start":"481'1200","ondisk_log_start":"481'1200","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7559,"log_dups_size":0,"ondisk_log_size":7559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114622464,"num_objects":1749,"num_object_clones":0,"num_object_copies":5247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8759,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":132,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.497056+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:47:15.497056+0000","last_peered":"2023-10-01T09:47:15.497056+0000","last_clean":"2023-10-01T09:47:15.497056+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:47:15.497056+0000","last_undegraded":"2023-10-01T09:47:15.497056+0000","last_fullsized":"2023-10-01T09:47:15.497056+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":206,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.497253+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:47:15.497253+0000","last_peered":"2023-10-01T09:47:15.497253+0000","last_clean":"2023-10-01T09:47:15.497253+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:47:15.497253+0000","last_undegraded":"2023-10-01T09:47:15.497253+0000","last_fullsized":"2023-10-01T09:47:15.497253+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"552'8703","reported_seq":9396,"reported_epoch":552,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:47:27.276487+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:47:27.276487+0000","last_peered":"2023-10-01T09:47:27.276487+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:47:27.276487+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8703,"log_dups_size":0,"ondisk_log_size":8703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109117440,"num_objects":1665,"num_object_clones":0,"num_object_copies":4995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3220,"num_objects_misplaced":1601,"num_objects_unfound":0,"num_objects_dirty":1665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8703,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":300,"num_bytes_recovered":19660800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1601},{"shards":"4,5","objects":-3156}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":450,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.496831+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:47:15.496831+0000","last_peered":"2023-10-01T09:47:15.496831+0000","last_clean":"2023-10-01T09:47:15.496831+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:47:15.496831+0000","last_undegraded":"2023-10-01T09:47:15.496831+0000","last_fullsized":"2023-10-01T09:47:15.496831+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":613,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498990+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:47:23.498990+0000","last_peered":"2023-10-01T09:47:23.498990+0000","last_clean":"2023-10-01T09:47:23.498990+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:47:23.498990+0000","last_undegraded":"2023-10-01T09:47:23.498990+0000","last_fullsized":"2023-10-01T09:47:23.498990+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"552'8683","reported_seq":8777,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:27.191539+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:47:27.191539+0000","last_peered":"2023-10-01T09:47:27.191539+0000","last_clean":"2023-10-01T09:47:27.191539+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:47:27.191539+0000","last_undegraded":"2023-10-01T09:47:27.191539+0000","last_fullsized":"2023-10-01T09:47:27.191539+0000","mapping_epoch":483,"log_start":"481'1100","ondisk_log_start":"481'1100","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7583,"log_dups_size":0,"ondisk_log_size":7583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111083520,"num_objects":1695,"num_object_clones":0,"num_object_copies":5085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8683,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":446,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.496712+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:47:15.496712+0000","last_peered":"2023-10-01T09:47:15.496712+0000","last_clean":"2023-10-01T09:47:15.496712+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:47:15.496712+0000","last_undegraded":"2023-10-01T09:47:15.496712+0000","last_fullsized":"2023-10-01T09:47:15.496712+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":287,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.497249+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:47:15.497249+0000","last_peered":"2023-10-01T09:47:15.497249+0000","last_clean":"2023-10-01T09:47:15.497249+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:47:15.497249+0000","last_undegraded":"2023-10-01T09:47:15.497249+0000","last_fullsized":"2023-10-01T09:47:15.497249+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"552'8793","reported_seq":8878,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:26.323820+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:47:26.323820+0000","last_peered":"2023-10-01T09:47:26.323820+0000","last_clean":"2023-10-01T09:47:26.323820+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:47:26.323820+0000","last_undegraded":"2023-10-01T09:47:26.323820+0000","last_fullsized":"2023-10-01T09:47:26.323820+0000","mapping_epoch":475,"log_start":"481'400","ondisk_log_start":"481'400","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8393,"log_dups_size":0,"ondisk_log_size":8393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111738880,"num_objects":1705,"num_object_clones":0,"num_object_copies":5115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8793,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":417,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498442+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:47:23.498442+0000","last_peered":"2023-10-01T09:47:23.498442+0000","last_clean":"2023-10-01T09:47:23.498442+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:47:23.498442+0000","last_undegraded":"2023-10-01T09:47:23.498442+0000","last_fullsized":"2023-10-01T09:47:23.498442+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":464,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.499314+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:47:23.499314+0000","last_peered":"2023-10-01T09:47:23.499314+0000","last_clean":"2023-10-01T09:47:23.499314+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:47:23.499314+0000","last_undegraded":"2023-10-01T09:47:23.499314+0000","last_fullsized":"2023-10-01T09:47:23.499314+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"552'8517","reported_seq":8602,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:26.295008+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:47:26.295008+0000","last_peered":"2023-10-01T09:47:26.295008+0000","last_clean":"2023-10-01T09:47:26.295008+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:47:26.295008+0000","last_undegraded":"2023-10-01T09:47:26.295008+0000","last_fullsized":"2023-10-01T09:47:26.295008+0000","mapping_epoch":475,"log_start":"480'100","ondisk_log_start":"480'100","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8417,"log_dups_size":0,"ondisk_log_size":8417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108855296,"num_objects":1661,"num_object_clones":0,"num_object_copies":4983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8517,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":945,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465560+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:47:15.465560+0000","last_peered":"2023-10-01T09:47:15.465560+0000","last_clean":"2023-10-01T09:47:15.465560+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:47:15.465560+0000","last_undegraded":"2023-10-01T09:47:15.465560+0000","last_fullsized":"2023-10-01T09:47:15.465560+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":282,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.464482+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:47:15.464482+0000","last_peered":"2023-10-01T09:47:15.464482+0000","last_clean":"2023-10-01T09:47:15.464482+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:47:15.464482+0000","last_undegraded":"2023-10-01T09:47:15.464482+0000","last_fullsized":"2023-10-01T09:47:15.464482+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":78,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498685+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:47:23.498685+0000","last_peered":"2023-10-01T09:47:23.498685+0000","last_clean":"2023-10-01T09:47:23.498685+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:47:23.498685+0000","last_undegraded":"2023-10-01T09:47:23.498685+0000","last_fullsized":"2023-10-01T09:47:23.498685+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"552'8620","reported_seq":8709,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:26.785448+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:47:26.785448+0000","last_peered":"2023-10-01T09:47:26.785448+0000","last_clean":"2023-10-01T09:47:26.785448+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:47:26.785448+0000","last_undegraded":"2023-10-01T09:47:26.785448+0000","last_fullsized":"2023-10-01T09:47:26.785448+0000","mapping_epoch":475,"log_start":"484'1600","ondisk_log_start":"484'1600","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7020,"log_dups_size":0,"ondisk_log_size":7020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114688000,"num_objects":1750,"num_object_clones":0,"num_object_copies":5250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8620,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1217,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.464395+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:47:15.464395+0000","last_peered":"2023-10-01T09:47:15.464395+0000","last_clean":"2023-10-01T09:47:15.464395+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:47:15.464395+0000","last_undegraded":"2023-10-01T09:47:15.464395+0000","last_fullsized":"2023-10-01T09:47:15.464395+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":427,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.496995+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:47:15.496995+0000","last_peered":"2023-10-01T09:47:15.496995+0000","last_clean":"2023-10-01T09:47:15.496995+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:47:15.496995+0000","last_undegraded":"2023-10-01T09:47:15.496995+0000","last_fullsized":"2023-10-01T09:47:15.496995+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":255,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.464587+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:47:15.464587+0000","last_peered":"2023-10-01T09:47:15.464587+0000","last_clean":"2023-10-01T09:47:15.464587+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:47:15.464587+0000","last_undegraded":"2023-10-01T09:47:15.464587+0000","last_fullsized":"2023-10-01T09:47:15.464587+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"552'8652","reported_seq":15958,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:27.290742+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:47:27.290742+0000","last_peered":"2023-10-01T09:47:27.290742+0000","last_clean":"2023-10-01T09:47:27.290742+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:47:27.290742+0000","last_undegraded":"2023-10-01T09:47:27.290742+0000","last_fullsized":"2023-10-01T09:47:27.290742+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8652,"log_dups_size":0,"ondisk_log_size":8652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112984064,"num_objects":1724,"num_object_clones":0,"num_object_copies":5172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8652,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":899,"reported_epoch":551,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:47:21.921691+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:47:21.921691+0000","last_peered":"2023-10-01T09:47:21.921691+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:47:21.921691+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:47:21.921691+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":617,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498476+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:47:23.498476+0000","last_peered":"2023-10-01T09:47:23.498476+0000","last_clean":"2023-10-01T09:47:23.498476+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:47:23.498476+0000","last_undegraded":"2023-10-01T09:47:23.498476+0000","last_fullsized":"2023-10-01T09:47:23.498476+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.499179+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:47:23.499179+0000","last_peered":"2023-10-01T09:47:23.499179+0000","last_clean":"2023-10-01T09:47:23.499179+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:47:23.499179+0000","last_undegraded":"2023-10-01T09:47:23.499179+0000","last_fullsized":"2023-10-01T09:47:23.499179+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"553'8642","reported_seq":8732,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:29.938760+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:47:29.938760+0000","last_peered":"2023-10-01T09:47:29.938760+0000","last_clean":"2023-10-01T09:47:29.938760+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:47:29.938760+0000","last_undegraded":"2023-10-01T09:47:29.938760+0000","last_fullsized":"2023-10-01T09:47:29.938760+0000","mapping_epoch":475,"log_start":"481'1100","ondisk_log_start":"481'1100","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7542,"log_dups_size":0,"ondisk_log_size":7542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106823680,"num_objects":1630,"num_object_clones":0,"num_object_copies":4890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8642,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1668,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.464648+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:47:15.464648+0000","last_peered":"2023-10-01T09:47:15.464648+0000","last_clean":"2023-10-01T09:47:15.464648+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:47:15.464648+0000","last_undegraded":"2023-10-01T09:47:15.464648+0000","last_fullsized":"2023-10-01T09:47:15.464648+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":582,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465087+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:47:15.465087+0000","last_peered":"2023-10-01T09:47:15.465087+0000","last_clean":"2023-10-01T09:47:15.465087+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:47:15.465087+0000","last_undegraded":"2023-10-01T09:47:15.465087+0000","last_fullsized":"2023-10-01T09:47:15.465087+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":360,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.464671+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:47:15.464671+0000","last_peered":"2023-10-01T09:47:15.464671+0000","last_clean":"2023-10-01T09:47:15.464671+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:47:15.464671+0000","last_undegraded":"2023-10-01T09:47:15.464671+0000","last_fullsized":"2023-10-01T09:47:15.464671+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":350,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.497124+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:47:15.497124+0000","last_peered":"2023-10-01T09:47:15.497124+0000","last_clean":"2023-10-01T09:47:15.497124+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:47:15.497124+0000","last_undegraded":"2023-10-01T09:47:15.497124+0000","last_fullsized":"2023-10-01T09:47:15.497124+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"552'8775","reported_seq":8860,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:26.207923+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:47:26.207923+0000","last_peered":"2023-10-01T09:47:26.207923+0000","last_clean":"2023-10-01T09:47:26.207923+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:47:26.207923+0000","last_undegraded":"2023-10-01T09:47:26.207923+0000","last_fullsized":"2023-10-01T09:47:26.207923+0000","mapping_epoch":475,"log_start":"481'400","ondisk_log_start":"481'400","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8375,"log_dups_size":0,"ondisk_log_size":8375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115146752,"num_objects":1757,"num_object_clones":0,"num_object_copies":5271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8775,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":759,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465264+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:47:15.465264+0000","last_peered":"2023-10-01T09:47:15.465264+0000","last_clean":"2023-10-01T09:47:15.465264+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:47:15.465264+0000","last_undegraded":"2023-10-01T09:47:15.465264+0000","last_fullsized":"2023-10-01T09:47:15.465264+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":593,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.471944+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:47:15.471944+0000","last_peered":"2023-10-01T09:47:15.471944+0000","last_clean":"2023-10-01T09:47:15.471944+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:47:15.471944+0000","last_undegraded":"2023-10-01T09:47:15.471944+0000","last_fullsized":"2023-10-01T09:47:15.471944+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":592,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.471861+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:47:15.471861+0000","last_peered":"2023-10-01T09:47:15.471861+0000","last_clean":"2023-10-01T09:47:15.471861+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:47:15.471861+0000","last_undegraded":"2023-10-01T09:47:15.471861+0000","last_fullsized":"2023-10-01T09:47:15.471861+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"552'8715","reported_seq":8805,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:26.781810+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:47:26.781810+0000","last_peered":"2023-10-01T09:47:26.781810+0000","last_clean":"2023-10-01T09:47:26.781810+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:47:26.781810+0000","last_undegraded":"2023-10-01T09:47:26.781810+0000","last_fullsized":"2023-10-01T09:47:26.781810+0000","mapping_epoch":475,"log_start":"484'1700","ondisk_log_start":"484'1700","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7015,"log_dups_size":0,"ondisk_log_size":7015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114294808,"num_objects":1745,"num_object_clones":0,"num_object_copies":5235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1745,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8715,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":990,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465781+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:47:15.465781+0000","last_peered":"2023-10-01T09:47:15.465781+0000","last_clean":"2023-10-01T09:47:15.465781+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:47:15.465781+0000","last_undegraded":"2023-10-01T09:47:15.465781+0000","last_fullsized":"2023-10-01T09:47:15.465781+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":609,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.464540+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:47:15.464540+0000","last_peered":"2023-10-01T09:47:15.464540+0000","last_clean":"2023-10-01T09:47:15.464540+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:47:15.464540+0000","last_undegraded":"2023-10-01T09:47:15.464540+0000","last_fullsized":"2023-10-01T09:47:15.464540+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":462,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.464724+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:47:15.464724+0000","last_peered":"2023-10-01T09:47:15.464724+0000","last_clean":"2023-10-01T09:47:15.464724+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:47:15.464724+0000","last_undegraded":"2023-10-01T09:47:15.464724+0000","last_fullsized":"2023-10-01T09:47:15.464724+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"552'8651","reported_seq":8744,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:26.260882+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:47:26.260882+0000","last_peered":"2023-10-01T09:47:26.260882+0000","last_clean":"2023-10-01T09:47:26.260882+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:47:26.260882+0000","last_undegraded":"2023-10-01T09:47:26.260882+0000","last_fullsized":"2023-10-01T09:47:26.260882+0000","mapping_epoch":479,"log_start":"484'1900","ondisk_log_start":"484'1900","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6751,"log_dups_size":0,"ondisk_log_size":6751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111738880,"num_objects":1705,"num_object_clones":0,"num_object_copies":5115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8651,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":491,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.466154+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:47:15.466154+0000","last_peered":"2023-10-01T09:47:15.466154+0000","last_clean":"2023-10-01T09:47:15.466154+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:47:15.466154+0000","last_undegraded":"2023-10-01T09:47:15.466154+0000","last_fullsized":"2023-10-01T09:47:15.466154+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1187,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.499214+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:47:23.499214+0000","last_peered":"2023-10-01T09:47:23.499214+0000","last_clean":"2023-10-01T09:47:23.499214+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:47:23.499214+0000","last_undegraded":"2023-10-01T09:47:23.499214+0000","last_fullsized":"2023-10-01T09:47:23.499214+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":80,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465045+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:47:15.465045+0000","last_peered":"2023-10-01T09:47:15.465045+0000","last_clean":"2023-10-01T09:47:15.465045+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:47:15.465045+0000","last_undegraded":"2023-10-01T09:47:15.465045+0000","last_fullsized":"2023-10-01T09:47:15.465045+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"552'8720","reported_seq":8813,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:26.210218+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:47:26.210218+0000","last_peered":"2023-10-01T09:47:26.210218+0000","last_clean":"2023-10-01T09:47:26.210218+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:47:26.210218+0000","last_undegraded":"2023-10-01T09:47:26.210218+0000","last_fullsized":"2023-10-01T09:47:26.210218+0000","mapping_epoch":479,"log_start":"484'2000","ondisk_log_start":"484'2000","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6720,"log_dups_size":0,"ondisk_log_size":6720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117440512,"num_objects":1792,"num_object_clones":0,"num_object_copies":5376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8720,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":84,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.465995+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:47:15.465995+0000","last_peered":"2023-10-01T09:47:15.465995+0000","last_clean":"2023-10-01T09:47:15.465995+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:47:15.465995+0000","last_undegraded":"2023-10-01T09:47:15.465995+0000","last_fullsized":"2023-10-01T09:47:15.465995+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":570,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.471946+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:47:15.471946+0000","last_peered":"2023-10-01T09:47:15.471946+0000","last_clean":"2023-10-01T09:47:15.471946+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:47:15.471946+0000","last_undegraded":"2023-10-01T09:47:15.471946+0000","last_fullsized":"2023-10-01T09:47:15.471946+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":275,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.464344+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:47:15.464344+0000","last_peered":"2023-10-01T09:47:15.464344+0000","last_clean":"2023-10-01T09:47:15.464344+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:47:15.464344+0000","last_undegraded":"2023-10-01T09:47:15.464344+0000","last_fullsized":"2023-10-01T09:47:15.464344+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":519,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.472874+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:47:15.472874+0000","last_peered":"2023-10-01T09:47:15.472874+0000","last_clean":"2023-10-01T09:47:15.472874+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:47:15.472874+0000","last_undegraded":"2023-10-01T09:47:15.472874+0000","last_fullsized":"2023-10-01T09:47:15.472874+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":634,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498762+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:47:23.498762+0000","last_peered":"2023-10-01T09:47:23.498762+0000","last_clean":"2023-10-01T09:47:23.498762+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:47:23.498762+0000","last_undegraded":"2023-10-01T09:47:23.498762+0000","last_fullsized":"2023-10-01T09:47:23.498762+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":287,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.464462+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:47:15.464462+0000","last_peered":"2023-10-01T09:47:15.464462+0000","last_clean":"2023-10-01T09:47:15.464462+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:47:15.464462+0000","last_undegraded":"2023-10-01T09:47:15.464462+0000","last_fullsized":"2023-10-01T09:47:15.464462+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":77,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.499071+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:47:23.499071+0000","last_peered":"2023-10-01T09:47:23.499071+0000","last_clean":"2023-10-01T09:47:23.499071+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:47:23.499071+0000","last_undegraded":"2023-10-01T09:47:23.499071+0000","last_fullsized":"2023-10-01T09:47:23.499071+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":287,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.464566+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:47:15.464566+0000","last_peered":"2023-10-01T09:47:15.464566+0000","last_clean":"2023-10-01T09:47:15.464566+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:47:15.464566+0000","last_undegraded":"2023-10-01T09:47:15.464566+0000","last_fullsized":"2023-10-01T09:47:15.464566+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":465,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.473230+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:47:15.473230+0000","last_peered":"2023-10-01T09:47:15.473230+0000","last_clean":"2023-10-01T09:47:15.473230+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:47:15.473230+0000","last_undegraded":"2023-10-01T09:47:15.473230+0000","last_fullsized":"2023-10-01T09:47:15.473230+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":95,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.499123+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:47:23.499123+0000","last_peered":"2023-10-01T09:47:23.499123+0000","last_clean":"2023-10-01T09:47:23.499123+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:47:23.499123+0000","last_undegraded":"2023-10-01T09:47:23.499123+0000","last_fullsized":"2023-10-01T09:47:23.499123+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":510,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.471861+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:47:15.471861+0000","last_peered":"2023-10-01T09:47:15.471861+0000","last_clean":"2023-10-01T09:47:15.471861+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:47:15.471861+0000","last_undegraded":"2023-10-01T09:47:15.471861+0000","last_fullsized":"2023-10-01T09:47:15.471861+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":710,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.472413+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:47:15.472413+0000","last_peered":"2023-10-01T09:47:15.472413+0000","last_clean":"2023-10-01T09:47:15.472413+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:47:15.472413+0000","last_undegraded":"2023-10-01T09:47:15.472413+0000","last_fullsized":"2023-10-01T09:47:15.472413+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":403,"reported_epoch":551,"state":"active+clean","last_fresh":"2023-10-01T09:47:15.496622+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:47:15.496622+0000","last_peered":"2023-10-01T09:47:15.496622+0000","last_clean":"2023-10-01T09:47:15.496622+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:47:15.496622+0000","last_undegraded":"2023-10-01T09:47:15.496622+0000","last_fullsized":"2023-10-01T09:47:15.496622+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1793392664,"num_objects":27366,"num_object_clones":0,"num_object_copies":82098,"num_objects_missing_on_primary":163,"num_objects_missing":163,"num_objects_degraded":4993,"num_objects_misplaced":1601,"num_objects_unfound":0,"num_objects_dirty":27366,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":139074,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10718,"num_bytes_recovered":702414848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6181892096,"data_stored":6181879880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124174,"ondisk_log_size":124174,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430762,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1467664,"kb_used_data":560456,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":92277488,"statfs":{"total":95995035648,"available":94492147712,"internally_reserved":0,"allocated":573906944,"data_stored":572960816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:02 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.495,"15min":2.4870000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":1.3600000000000001,"5min":3.3849999999999998,"15min":108.90000000000001},"last":2.512},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.46899999999999997,"15min":2.621},"min":{"1min":0.23300000000000001,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":1.7410000000000001,"5min":1.7410000000000001,"15min":109.187},"last":0.63100000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.58299999999999996,"15min":5.0190000000000001},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.23300000000000001},"max":{"1min":7.7140000000000004,"5min":7.7140000000000004,"15min":229.161},"last":2.5430000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.53100000000000003,"15min":5.0449999999999999},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":1.972,"5min":3.0640000000000001,"15min":230.126},"last":0.29999999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:46:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.56299999999999994,"15min":1.591},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":1.6359999999999999,"5min":3.4300000000000002,"15min":14.901999999999999},"last":2.5680000000000001},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.55700000000000005,"15min":1.48},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.27000000000000002},"max":{"1min":1.8100000000000001,"5min":1.8100000000000001,"15min":14.682},"last":0.60299999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.59299999999999997,"15min":2.1059999999999999},"min":{"1min":0.34999999999999998,"5min":0.30199999999999999,"15min":0.29199999999999998},"max":{"1min":2.1749999999999998,"5min":2.476,"15min":76.555000000000007},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.58399999999999996,"15min":2.1120000000000001},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.1880000000000002,"5min":2.8479999999999999,"15min":76.534000000000006},"last":0.437}]},{"osd":4,"last update":"Sun Oct 1 09:47:02 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.52100000000000002,"15min":1.0249999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.4470000000000001,"5min":3.4609999999999999,"15min":9.6020000000000003},"last":2.5910000000000002},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.52500000000000002,"15min":0.99399999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.837,"5min":1.837,"15min":8.4949999999999992},"last":0.33700000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.52500000000000002,"15min":1.5269999999999999},"min":{"1min":0.35399999999999998,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":2.2400000000000002,"5min":2.2400000000000002,"15min":135.40000000000001},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.54500000000000004,"15min":1.627},"min":{"1min":0.34999999999999998,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":0.70699999999999996,"5min":2.8849999999999998,"15min":135.339},"last":0.51000000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.58999999999999997,"15min":1.1200000000000001},"min":{"1min":0.40500000000000003,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":1.7829999999999999,"5min":2.448,"15min":11.853999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.59699999999999998,"15min":1.093},"min":{"1min":0.39700000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":3.1909999999999998,"5min":3.7810000000000001,"15min":14.584},"last":0.38800000000000001}]}]},{"osd":6,"up_from":294,"seq":1262720385273,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1116428,"kb_used_data":778820,"kb_used_omap":6,"kb_used_meta":337593,"kb_avail":92628724,"statfs":{"total":95995035648,"available":94851813376,"internally_reserved":0,"allocated":797511680,"data_stored":796569503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":345695277},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":4.5839999999999996,"5min":1.2829999999999999,"15min":2.665},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20599999999999999},"max":{"1min":65.599000000000004,"5min":65.599000000000004,"15min":158.48599999999999},"last":0.32900000000000001},{"interface":"front","average":{"1min":5.4089999999999998,"5min":1.484,"15min":2.7909999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":65.388999999999996,"5min":65.388999999999996,"15min":158.37700000000001},"last":0.27400000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":0.70599999999999996,"15min":3.8839999999999999},"min":{"1min":0.34399999999999997,"5min":0.222,"15min":0.222},"max":{"1min":8.2409999999999997,"5min":8.2409999999999997,"15min":168.947},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.625,"5min":0.56999999999999995,"15min":3.8210000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.25600000000000001},"max":{"1min":2.6669999999999998,"5min":3.7469999999999999,"15min":169.07900000000001},"last":0.35999999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":0.73799999999999999,"15min":1.429},"min":{"1min":0.311,"5min":0.254,"15min":0.254},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":10.92},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.748,"5min":0.57199999999999995,"15min":1.3899999999999999},"min":{"1min":0.314,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":2.6480000000000001,"5min":3.7160000000000002,"15min":21.704999999999998},"last":0.33900000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55000000000000004,"15min":1.647},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.9710000000000001,"5min":3.3759999999999999,"15min":89.753},"last":0.378},{"interface":"front","average":{"1min":1.51,"5min":0.78200000000000003,"15min":1.7589999999999999},"min":{"1min":0.34399999999999997,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.3109999999999999,"5min":8.3109999999999999,"15min":89.849999999999994},"last":0.39100000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.505,"15min":1.0609999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.7559999999999998,"5min":3.6349999999999998,"15min":63.134},"last":0.307},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.54400000000000004,"15min":1.0429999999999999},"min":{"1min":0.34499999999999997,"5min":0.33600000000000002,"15min":0.28899999999999998},"max":{"1min":2.778,"5min":3.2519999999999998,"15min":62.973999999999997},"last":0.29399999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:46:37 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.88500000000000001,"15min":0.999},"min":{"1min":0.33100000000000002,"5min":0.26700000000000002,"15min":0.251},"max":{"1min":0.93799999999999994,"5min":34.094999999999999,"15min":34.094999999999999},"last":136.535},{"interface":"front","average":{"1min":0.875,"5min":1.022,"15min":1.1379999999999999},"min":{"1min":0.315,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.181,"5min":34.072000000000003,"15min":34.072000000000003},"last":136.47300000000001}]},{"osd":7,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.56100000000000005,"15min":1.2609999999999999},"min":{"1min":0.38300000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.3159999999999998,"5min":3.339,"15min":94.245999999999995},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.76700000000000002,"15min":1.421},"min":{"1min":0.37,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":8.3399999999999999,"5min":8.3399999999999999,"15min":94.715000000000003},"last":0.32300000000000001}]}]},{"osd":1,"up_from":94,"seq":403726926250,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1168244,"kb_used_data":332012,"kb_used_omap":7,"kb_used_meta":836216,"kb_avail":92576908,"statfs":{"total":95995035648,"available":94798753792,"internally_reserved":0,"allocated":339980288,"data_stored":337893916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":856285607},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":1.0349999999999999,"15min":3.5819999999999999},"min":{"1min":0.19500000000000001,"5min":0.186,"15min":0.182},"max":{"1min":0.82799999999999996,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.0860000000000001,"15min":3.1030000000000002},"min":{"1min":0.222,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":2.0859999999999999,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.27800000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.35899999999999999,"15min":2.0179999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.20000000000000001},"max":{"1min":0.78500000000000003,"5min":1.3240000000000001,"15min":51.640000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.44700000000000001,"15min":2.7570000000000001},"min":{"1min":0.29199999999999998,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":0.69499999999999995,"5min":2.9089999999999998,"15min":154.197},"last":0.32400000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45000000000000001,"15min":2.706},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.219},"max":{"1min":2.149,"5min":2.8780000000000001,"15min":95.305000000000007},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.497,"5min":0.47399999999999998,"15min":3.3210000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.994,"5min":2.7360000000000002,"15min":154.483},"last":0.40799999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.56499999999999995,"15min":2.1389999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":1.204,"5min":3.0550000000000002,"15min":154.101},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.46800000000000003,"15min":2.1469999999999998},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.27700000000000002},"max":{"1min":0.77200000000000002,"5min":0.83199999999999996,"15min":154.56999999999999},"last":0.46700000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.143,"15min":2.23},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.8929999999999998,"5min":59.561,"15min":154.22999999999999},"last":0.377},{"interface":"front","average":{"1min":0.53500000000000003,"5min":1.137,"15min":2.3660000000000001},"min":{"1min":0.32900000000000001,"5min":0.31,"15min":0.307},"max":{"1min":0.89600000000000002,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.439}]},{"osd":6,"last update":"Sun Oct 1 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.65700000000000003,"15min":2.2770000000000001},"min":{"1min":0.33400000000000002,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":4.5069999999999997,"5min":4.5069999999999997,"15min":154.12200000000001},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.63200000000000001,"15min":2.218},"min":{"1min":0.35899999999999999,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":1.5900000000000001,"5min":2.5630000000000002,"15min":154.16999999999999},"last":0.51800000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.73299999999999998,"15min":2.4420000000000002},"min":{"1min":0.38900000000000001,"5min":0.36399999999999999,"15min":0.33900000000000002},"max":{"1min":4.3730000000000002,"5min":4.3730000000000002,"15min":154.20699999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.71999999999999997,"15min":2.4460000000000002},"min":{"1min":0.435,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":4.3879999999999999,"5min":4.3879999999999999,"15min":154.51599999999999},"last":0.495}]}]},{"osd":0,"up_from":89,"seq":382252089784,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2713216,"kb_used_data":1777784,"kb_used_omap":8,"kb_used_meta":935415,"kb_avail":91031936,"statfs":{"total":95995035648,"available":93216702464,"internally_reserved":0,"allocated":1820450816,"data_stored":1818412437,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":957865922},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.40500000000000003,"15min":3.6469999999999998},"min":{"1min":0.20799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.69099999999999995,"5min":4.2919999999999998,"15min":216.821},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.434,"15min":4.2300000000000004},"min":{"1min":0.20599999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.72499999999999998,"5min":3.9620000000000002,"15min":217.02699999999999},"last":0.32900000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.47499999999999998,"15min":1.9059999999999999},"min":{"1min":0.29399999999999998,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":1.3100000000000001,"5min":4.2430000000000003,"15min":19.788},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54700000000000004,"15min":1.9239999999999999},"min":{"1min":0.29299999999999998,"5min":0.27200000000000002,"15min":0.215},"max":{"1min":1.3300000000000001,"5min":3.8719999999999999,"15min":25.844999999999999},"last":0.35899999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.47899999999999998,"15min":2.778},"min":{"1min":0.24299999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.68000000000000005,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.626,"15min":2.6200000000000001},"min":{"1min":0.25700000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.0259999999999998,"5min":4.9199999999999999,"15min":222.227},"last":0.40500000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57199999999999995,"15min":1.8500000000000001},"min":{"1min":0.308,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":2.1160000000000001,"5min":4.048,"15min":62.206000000000003},"last":0.376},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.52500000000000002,"15min":1.9359999999999999},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":1.3380000000000001,"5min":4,"15min":62.073999999999998},"last":0.47399999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.59999999999999998,"15min":1.6699999999999999},"min":{"1min":0.34899999999999998,"5min":0.312,"15min":0.312},"max":{"1min":1.5920000000000001,"5min":4.0350000000000001,"15min":16.808},"last":0.434},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.59899999999999998,"15min":1.5820000000000001},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.8100000000000001,"5min":4.2729999999999997,"15min":17.507000000000001},"last":0.41799999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:46:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.63600000000000001,"15min":1.6990000000000001},"min":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.1970000000000001,"5min":4.3129999999999997,"15min":17.535},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65900000000000003,"15min":1.732},"min":{"1min":0.372,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.218,"5min":4.2050000000000001,"15min":15.651999999999999},"last":0.46500000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.66300000000000003,"15min":1.927},"min":{"1min":0.39700000000000002,"5min":0.374,"15min":0.374},"max":{"1min":2.2290000000000001,"5min":4.3360000000000003,"15min":48.895000000000003},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.79400000000000004,"15min":1.97},"min":{"1min":0.39300000000000002,"5min":0.376,"15min":0.35299999999999998},"max":{"1min":0.78300000000000003,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.42499999999999999}]}]},{"osd":2,"up_from":0,"seq":425201762724,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599197,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1200420,"kb_used_data":346140,"kb_used_omap":3,"kb_used_meta":854268,"kb_avail":92544732,"statfs":{"total":95995035648,"available":94765805568,"internally_reserved":0,"allocated":354447360,"data_stored":352310833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":874771175},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.26900000000000002,"5min":0.34300000000000003,"15min":3.5009999999999999},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":0.504,"5min":2.7989999999999999,"15min":137.34399999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.35899999999999999,"15min":3.5459999999999998},"min":{"1min":0.23400000000000001,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":0.66800000000000004,"5min":1.2849999999999999,"15min":137.73400000000001},"last":0.439}]},{"osd":1,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.40200000000000002,"15min":4.7430000000000003},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":0.63500000000000001,"5min":2.7250000000000001,"15min":162.22999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.371,"15min":4.7370000000000001},"min":{"1min":0.27200000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.3440000000000001,"5min":1.4119999999999999,"15min":162.70699999999999},"last":0.53500000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.442,"15min":1.552},"min":{"1min":0.251,"5min":0.24199999999999999,"15min":0.20799999999999999},"max":{"1min":0.46000000000000002,"5min":2.9049999999999998,"15min":14.254},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.499,"15min":1.4350000000000001},"min":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":2.5230000000000001,"5min":4.5049999999999999,"15min":15.523},"last":0.45900000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.53400000000000003,"15min":1.653},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.26200000000000001},"max":{"1min":4.5010000000000003,"5min":4.5010000000000003,"15min":77.793000000000006},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.499,"5min":0.49399999999999999,"15min":1.8120000000000001},"min":{"1min":0.35299999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":0.72299999999999998,"5min":1.5609999999999999,"15min":76.646000000000001},"last":0.47399999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.58399999999999996,"15min":1.9119999999999999},"min":{"1min":0.32500000000000001,"5min":0.314,"15min":0.27500000000000002},"max":{"1min":0.61399999999999999,"5min":4.3099999999999996,"15min":28.117999999999999},"last":0.501},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51800000000000002,"15min":2.0310000000000001},"min":{"1min":0.33800000000000002,"5min":0.33600000000000002,"15min":0.27700000000000002},"max":{"1min":1.9930000000000001,"5min":2.512,"15min":28.34},"last":0.55400000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:46:30 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.55500000000000005,"15min":1.764},"min":{"1min":0.379,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":0.56899999999999995,"5min":2.9100000000000001,"15min":73.887},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.58499999999999996,"15min":1.8560000000000001},"min":{"1min":0.379,"5min":0.35499999999999998,"15min":0.32800000000000001},"max":{"1min":0.60499999999999998,"5min":2.6000000000000001,"15min":74.016999999999996},"last":0.58599999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":0.85999999999999999,"15min":1.7729999999999999},"min":{"1min":0.42799999999999999,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":4.1989999999999998,"5min":4.1989999999999998,"15min":13.085000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.2030000000000001,"5min":0.80500000000000005,"15min":1.5529999999999999},"min":{"1min":0.432,"5min":0.38800000000000001,"15min":0.311},"max":{"1min":4.3280000000000003,"5min":4.3280000000000003,"15min":13.191000000000001},"last":0.61099999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019447,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2453708,"kb_used_data":1563524,"kb_used_omap":0,"kb_used_meta":890176,"kb_avail":91291444,"statfs":{"total":95995035648,"available":93482438656,"internally_reserved":0,"allocated":1601048576,"data_stored":1600105954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911540224},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7480000000000002,"5min":0.89400000000000002,"15min":3.5819999999999999},"min":{"1min":0.24399999999999999,"5min":0.22500000000000001,"15min":0.21199999999999999},"max":{"1min":41.963999999999999,"5min":41.963999999999999,"15min":155.315},"last":0.22700000000000001},{"interface":"front","average":{"1min":2.7530000000000001,"5min":0.90100000000000002,"15min":3.9140000000000001},"min":{"1min":0.26100000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":41.890999999999998,"5min":41.890999999999998,"15min":155.334},"last":0.44500000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.54000000000000004,"15min":5.1449999999999996},"min":{"1min":0.25600000000000001,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":3.3929999999999998,"5min":3.3929999999999998,"15min":190.61199999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.54900000000000004,"15min":5.8639999999999999},"min":{"1min":0.29199999999999998,"5min":0.27500000000000002,"15min":0.25600000000000001},"max":{"1min":3.3839999999999999,"5min":3.3839999999999999,"15min":253.273},"last":0.28000000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.621,"15min":2.0499999999999998},"min":{"1min":0.30499999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.343,"5min":4.2329999999999997,"15min":170.45400000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.61499999999999999,"15min":1.8069999999999999},"min":{"1min":0.35399999999999998,"5min":0.314,"15min":0.309},"max":{"1min":3.1389999999999998,"5min":3.1389999999999998,"15min":36.515999999999998},"last":0.499}]},{"osd":3,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.61199999999999999,"15min":1.9850000000000001},"min":{"1min":0.307,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":3.1539999999999999,"5min":3.1539999999999999,"15min":140.881},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.61399999999999999,"15min":2.0579999999999998},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.311},"max":{"1min":1.3049999999999999,"5min":6.2969999999999997,"15min":140.911},"last":0.32300000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:46:41 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.56999999999999995,"15min":1.454},"min":{"1min":0.27200000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":1.8500000000000001,"5min":3.052,"15min":110.003},"last":10.460000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.63400000000000001,"15min":1.3859999999999999},"min":{"1min":0.34100000000000003,"5min":0.317,"15min":0.28899999999999998},"max":{"1min":1.754,"5min":3.2909999999999999,"15min":108.56399999999999},"last":5.891}]},{"osd":6,"last update":"Sun Oct 1 09:47:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.56499999999999995,"15min":0.91700000000000004},"min":{"1min":0.318,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":3.3029999999999999,"5min":3.3029999999999999,"15min":11.358000000000001},"last":1.97},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.60999999999999999,"15min":1.046},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.28999999999999998},"max":{"1min":2.2669999999999999,"5min":2.9239999999999999,"15min":10.076000000000001},"last":0.52800000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.63400000000000001,"15min":1.149},"min":{"1min":0.39200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.3079999999999998,"5min":3.3079999999999998,"15min":38.034999999999997},"last":7.0270000000000001},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.70899999999999996,"15min":1.3839999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.308},"max":{"1min":3.0329999999999999,"5min":3.113,"15min":37.723999999999997},"last":0.51500000000000001}]}]},{"osd":5,"up_from":379,"seq":1627792605354,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1122936,"kb_used_data":762480,"kb_used_omap":13,"kb_used_meta":360434,"kb_avail":92622216,"statfs":{"total":95995035648,"available":94845149184,"internally_reserved":0,"allocated":780779520,"data_stored":779852802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":369084914},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1,0,3,1,0,5,7],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.749,"15min":2.0830000000000002},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.21099999999999999},"max":{"1min":0.77600000000000002,"5min":27.584,"15min":84.340999999999994},"last":99.609999999999999},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.76900000000000002,"15min":2.0369999999999999},"min":{"1min":0.28399999999999997,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.749,"5min":27.669,"15min":84.356999999999999},"last":98.995999999999995}]},{"osd":1,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.55200000000000005,"15min":5.5519999999999996},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.2130000000000001,"5min":4.3289999999999997,"15min":295.71300000000002},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.51300000000000001,"15min":5.5609999999999999},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.24199999999999999},"max":{"1min":0.70399999999999996,"5min":3.9580000000000002,"15min":299.59399999999999},"last":0.46500000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.52400000000000002,"15min":1.45},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78600000000000003,"5min":4.2670000000000003,"15min":14.199999999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.65100000000000002,"15min":1.4139999999999999},"min":{"1min":0.29599999999999999,"5min":0.27900000000000003,"15min":0.27100000000000002},"max":{"1min":4.2910000000000004,"5min":4.3719999999999999,"15min":14.430999999999999},"last":0.379}]},{"osd":3,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.56699999999999995,"15min":1.3720000000000001},"min":{"1min":0.32800000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.75,"5min":4.0209999999999999,"15min":14.004},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.5,"5min":0.58799999999999997,"15min":1.518},"min":{"1min":0.40100000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":0.83299999999999996,"5min":4.2910000000000004,"15min":13.991},"last":0.47599999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.53300000000000003,"15min":1.365},"min":{"1min":0.29499999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.0620000000000001,"5min":4.1929999999999996,"15min":122.971},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.623,"15min":1.5389999999999999},"min":{"1min":0.28899999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2619999999999996,"5min":4.3869999999999996,"15min":122.89100000000001},"last":0.54700000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.54900000000000004,"15min":0.95399999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.28299999999999997},"max":{"1min":0.82799999999999996,"5min":4.3300000000000001,"15min":12.545},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.5,"5min":0.64100000000000001,"15min":1.1459999999999999},"min":{"1min":0.36499999999999999,"5min":0.314,"15min":0.30499999999999999},"max":{"1min":0.81899999999999995,"5min":4.3200000000000003,"15min":14.193},"last":0.56000000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.68300000000000005,"15min":1.1499999999999999},"min":{"1min":0.35399999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.3170000000000002,"5min":4.3559999999999999,"15min":10.813000000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.67200000000000004,"15min":1.0469999999999999},"min":{"1min":0.39500000000000002,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":1.5369999999999999,"5min":4.2000000000000002,"15min":12.500999999999999},"last":0.60599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1799294976,"data_stored":1799290904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":328925184,"data_stored":328925184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":347537408,"data_stored":347537408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1585975296,"data_stored":1585971224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":771686400,"data_stored":771686400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":787484672,"data_stored":787480600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":560988160,"data_stored":560988160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:47:32.490 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:47:32.527 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:32.549 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:47:32.555 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:32.606 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:32.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:32.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:32.710 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:32.743 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:32.908 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:32.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:32.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:32.950 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:33.012 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:33.012 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:33.054 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:33.178 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:32 smithi138 ceph-mon[192513]: pgmap v1659: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 5003/79782 objects degraded (6.271%); 1601/79782 objects misplaced (2.007%); 828 KiB/s, 12 objects/s recovering 2023-10-01T09:47:33.178 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:32 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5003/82182 objects degraded (6.088%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:33.178 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:32 smithi138 ceph-mon[185886]: pgmap v1659: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 5003/79782 objects degraded (6.271%); 1601/79782 objects misplaced (2.007%); 828 KiB/s, 12 objects/s recovering 2023-10-01T09:47:33.178 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:32 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5003/82182 objects degraded (6.088%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:33.220 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:32 smithi190 ceph-mon[160435]: pgmap v1659: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 255 op/s; 5003/79782 objects degraded (6.271%); 1601/79782 objects misplaced (2.007%); 828 KiB/s, 12 objects/s recovering 2023-10-01T09:47:33.220 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:32 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5003/82182 objects degraded (6.088%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:33.228 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:33.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:33.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:33.339 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:33.370 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:33.499 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:47:33.499 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:47:33.499 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 0 2023-10-01T09:47:33.499 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:47:33.525 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:33.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:33.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:33.628 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:33.657 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:34.215 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:47:34.363 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:47:34.515 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:34.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:34.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:34.619 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:34.666 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:34.736 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:34.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:34.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:34.753 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:34.833 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:47:34.840 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:34.879 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:35.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:47:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:47:34] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.33.4" 2023-10-01T09:47:35.092 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:35.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:35.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:35.197 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:47:35.237 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:35.416 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:47:35.416 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 4895/78402 objects degraded (6.243%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2276,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":553,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":26134,"data_bytes":1715607499,"bytes_used":11249233920,"bytes_avail":660716015616,"bytes_total":671965249536,"degraded_objects":4895,"degraded_total":78402,"degraded_ratio":0.062434631769597716,"misplaced_objects":1504,"misplaced_total":78402,"misplaced_ratio":0.019183184102446366,"recovering_objects_per_sec":23,"recovering_bytes_per_sec":1513455,"recovering_keys_per_sec":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":272},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":34,"modified":"2023-10-01T09:43:40.311181+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:47:35.439 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:35 smithi190 ceph-mon[160435]: pgmap v1660: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 272 op/s; 4895/78402 objects degraded (6.243%); 1504/78402 objects misplaced (1.918%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:47:35.439 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1477129683' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:47:35.521 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:47:35.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:35.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:47:35.627 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:35.654 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:35.697 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:35.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:35.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:35.801 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:35 smithi138 ceph-mon[185886]: pgmap v1660: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 272 op/s; 4895/78402 objects degraded (6.243%); 1504/78402 objects misplaced (1.918%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:47:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1477129683' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:47:35.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:35 smithi138 ceph-mon[192513]: pgmap v1660: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 272 op/s; 4895/78402 objects degraded (6.243%); 1504/78402 objects misplaced (1.918%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:47:35.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1477129683' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:47:35.846 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:36.101 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:36.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:36.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:36.206 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:36.236 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:36.452 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:47:36.487 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:36.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:36 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1365670221' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:47:36.489 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:36 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:47:36.490 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:36 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1365670221' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:47:36.490 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:36 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:47:36.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:36.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:36.497 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:47:36.498 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:47:36.576 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:47:36.576 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:47:36.576 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:47:36.576 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:47:36.576 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:47:36.592 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:47:36.624 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:36 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1365670221' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:47:36.624 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:36 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:47:36.632 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:36.812 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:47:36.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:36.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:47:36.917 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:47:36.954 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:37.162 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:47:37.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:37.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:47:37.266 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:47:37.320 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:37.539 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:47:37.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:37.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:47:37.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[192513]: pgmap v1661: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 4895/76563 objects degraded (6.393%); 1504/76563 objects misplaced (1.964%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:47:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[192513]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[192513]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:47:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[192513]: osdmap e554: 8 total, 8 up, 7 in 2023-10-01T09:47:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[192513]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[192513]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[192513]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[185886]: pgmap v1661: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 4895/76563 objects degraded (6.393%); 1504/76563 objects misplaced (1.964%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:47:37.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:37.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[185886]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[185886]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:47:37.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[185886]: osdmap e554: 8 total, 8 up, 7 in 2023-10-01T09:47:37.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[185886]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[185886]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.572 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:37 smithi138 ceph-mon[185886]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.614 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:37.622 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:47:37.643 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:37.661 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:47:37.698 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:47:37.707 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:37.711 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:37 smithi190 ceph-mon[160435]: pgmap v1661: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 4895/76563 objects degraded (6.393%); 1504/76563 objects misplaced (1.964%); 1.4 MiB/s, 23 objects/s recovering 2023-10-01T09:47:37.711 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:37 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:37.711 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:37 smithi190 ceph-mon[160435]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.711 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:37 smithi190 ceph-mon[160435]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.711 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:37 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:47:37.711 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:37 smithi190 ceph-mon[160435]: osdmap e554: 8 total, 8 up, 7 in 2023-10-01T09:47:37.711 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:37 smithi190 ceph-mon[160435]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:37 smithi190 ceph-mon[160435]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.712 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:37 smithi190 ceph-mon[160435]: failed to encode map e554 with expected crc 2023-10-01T09:47:37.731 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:47:37.765 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:47:37.798 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:47:37.818 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:37.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:37.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:37.832 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:47:37.866 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:47:37.893 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:47:37.905 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:47:37.905 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:47:37.905 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:47:37.905 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:47:37.906 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:47:37.906 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:47:37.906 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:47:37.906 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:47:37.924 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:37.980 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:38.274 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:47:38.738 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4095678583' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:38.738 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:38 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 4895/76563 objects degraded (6.393%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4095678583' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:38.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:38 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 4895/76563 objects degraded (6.393%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4095678583' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:38 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 4895/76563 objects degraded (6.393%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:38.843 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:38.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:38.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:38.948 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:39.051 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:39.207 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:39.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:39.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:39.329 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:39.418 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:47:39.423 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:39.529 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:39.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:39.633 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:39.681 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:39 smithi190 ceph-mon[160435]: pgmap v1663: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 4786/73950 objects degraded (6.472%); 1404/73950 objects misplaced (1.899%); 1.7 MiB/s, 26 objects/s recovering 2023-10-01T09:47:39.689 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:39.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:39 smithi138 ceph-mon[185886]: pgmap v1663: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 4786/73950 objects degraded (6.472%); 1404/73950 objects misplaced (1.899%); 1.7 MiB/s, 26 objects/s recovering 2023-10-01T09:47:39.778 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:39 smithi138 ceph-mon[192513]: pgmap v1663: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 4786/73950 objects degraded (6.472%); 1404/73950 objects misplaced (1.899%); 1.7 MiB/s, 26 objects/s recovering 2023-10-01T09:47:39.863 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:39.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:39.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:39.978 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:40.044 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:40.069 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:40.140 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:40.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:40.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:40.253 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:40.309 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:40.369 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:47:40.373 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:47:40.388 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:40.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:40.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:40.499 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:40.538 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:40.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:47:40.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:47:40.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:47:40.611 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:40.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:40.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:40.717 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:40.749 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:40.956 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:47:40.964 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:47:40.964 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:47:41.523 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:41.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:41.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:41.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:41 smithi138 ceph-mon[185886]: from='client.47459 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:41.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:41 smithi138 ceph-mon[185886]: pgmap v1664: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 4786/72936 objects degraded (6.562%); 1404/72936 objects misplaced (1.925%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:47:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:41 smithi138 ceph-mon[192513]: from='client.47459 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:41.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:41 smithi138 ceph-mon[192513]: pgmap v1664: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 4786/72936 objects degraded (6.562%); 1404/72936 objects misplaced (1.925%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:47:41.630 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:41.660 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:41.705 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:41.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:41.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:41.808 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:41.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:41 smithi190 ceph-mon[160435]: from='client.47459 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:41.811 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:41 smithi190 ceph-mon[160435]: pgmap v1664: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 4786/72936 objects degraded (6.562%); 1404/72936 objects misplaced (1.925%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:47:41.858 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:41.883 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1663,"stamp":"2023-10-01T09:47:38.379810+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1618352075,"num_objects":24650,"num_object_clones":0,"num_object_copies":73950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4786,"num_objects_misplaced":1404,"num_objects_unfound":0,"num_objects_dirty":24650,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":142924,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11480,"num_bytes_recovered":739214351,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126088,"ondisk_log_size":126088,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10744072,"kb_used_data":5622672,"kb_used_omap":39,"kb_used_meta":5121304,"kb_avail":645471992,"statfs":{"total":671965249536,"available":660963319808,"internally_reserved":0,"allocated":5757616128,"data_stored":5747586643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5244215919},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,8,1,2,1,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-191692800,"num_objects":-2925,"num_object_clones":0,"num_object_copies":-8775,"num_objects_missing_on_primary":-163,"num_objects_missing":-163,"num_objects_degraded":-217,"num_objects_misplaced":-197,"num_objects_unfound":0,"num_objects_dirty":-2925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2925,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":251,"num_bytes_recovered":16449536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.476736"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":329,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498533+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:47:23.498533+0000","last_peered":"2023-10-01T09:47:23.498533+0000","last_clean":"2023-10-01T09:47:23.498533+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:47:23.498533+0000","last_undegraded":"2023-10-01T09:47:23.498533+0000","last_fullsized":"2023-10-01T09:47:23.498533+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":173,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498919+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:47:23.498919+0000","last_peered":"2023-10-01T09:47:23.498919+0000","last_clean":"2023-10-01T09:47:23.498919+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:47:23.498919+0000","last_undegraded":"2023-10-01T09:47:23.498919+0000","last_fullsized":"2023-10-01T09:47:23.498919+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":554,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498509+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:47:23.498509+0000","last_peered":"2023-10-01T09:47:23.498509+0000","last_clean":"2023-10-01T09:47:23.498509+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:47:23.498509+0000","last_undegraded":"2023-10-01T09:47:23.498509+0000","last_fullsized":"2023-10-01T09:47:23.498509+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":614,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912300+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:47:28.912300+0000","last_peered":"2023-10-01T09:47:28.912300+0000","last_clean":"2023-10-01T09:47:28.912300+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:47:28.912300+0000","last_undegraded":"2023-10-01T09:47:28.912300+0000","last_fullsized":"2023-10-01T09:47:28.912300+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":494,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.911992+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:47:28.911992+0000","last_peered":"2023-10-01T09:47:28.911992+0000","last_clean":"2023-10-01T09:47:28.911992+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:47:28.911992+0000","last_undegraded":"2023-10-01T09:47:28.911992+0000","last_fullsized":"2023-10-01T09:47:28.911992+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":235,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.994903+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:47:28.994903+0000","last_peered":"2023-10-01T09:47:28.994903+0000","last_clean":"2023-10-01T09:47:28.994903+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:47:28.994903+0000","last_undegraded":"2023-10-01T09:47:28.994903+0000","last_fullsized":"2023-10-01T09:47:28.994903+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":611,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.994969+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:47:28.994969+0000","last_peered":"2023-10-01T09:47:28.994969+0000","last_clean":"2023-10-01T09:47:28.994969+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:47:28.994969+0000","last_undegraded":"2023-10-01T09:47:28.994969+0000","last_fullsized":"2023-10-01T09:47:28.994969+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":542,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.906733+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:47:28.906733+0000","last_peered":"2023-10-01T09:47:28.906733+0000","last_clean":"2023-10-01T09:47:28.906733+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:47:28.906733+0000","last_undegraded":"2023-10-01T09:47:28.906733+0000","last_fullsized":"2023-10-01T09:47:28.906733+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":266,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.918624+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:47:28.918624+0000","last_peered":"2023-10-01T09:47:28.918624+0000","last_clean":"2023-10-01T09:47:28.918624+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:47:28.918624+0000","last_undegraded":"2023-10-01T09:47:28.918624+0000","last_fullsized":"2023-10-01T09:47:28.918624+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":72,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.492873+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:47:23.492873+0000","last_peered":"2023-10-01T09:47:23.492873+0000","last_clean":"2023-10-01T09:47:23.492873+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:47:23.492873+0000","last_undegraded":"2023-10-01T09:47:23.492873+0000","last_fullsized":"2023-10-01T09:47:23.492873+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":78,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.493390+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:47:23.493390+0000","last_peered":"2023-10-01T09:47:23.493390+0000","last_clean":"2023-10-01T09:47:23.493390+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:47:23.493390+0000","last_undegraded":"2023-10-01T09:47:23.493390+0000","last_fullsized":"2023-10-01T09:47:23.493390+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":77,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498639+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:47:23.498639+0000","last_peered":"2023-10-01T09:47:23.498639+0000","last_clean":"2023-10-01T09:47:23.498639+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:47:23.498639+0000","last_undegraded":"2023-10-01T09:47:23.498639+0000","last_fullsized":"2023-10-01T09:47:23.498639+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":454,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498895+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:47:23.498895+0000","last_peered":"2023-10-01T09:47:23.498895+0000","last_clean":"2023-10-01T09:47:23.498895+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:47:23.498895+0000","last_undegraded":"2023-10-01T09:47:23.498895+0000","last_fullsized":"2023-10-01T09:47:23.498895+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":335,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498637+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:47:23.498637+0000","last_peered":"2023-10-01T09:47:23.498637+0000","last_clean":"2023-10-01T09:47:23.498637+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:47:23.498637+0000","last_undegraded":"2023-10-01T09:47:23.498637+0000","last_fullsized":"2023-10-01T09:47:23.498637+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":671,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.493094+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:47:23.493094+0000","last_peered":"2023-10-01T09:47:23.493094+0000","last_clean":"2023-10-01T09:47:23.493094+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:47:23.493094+0000","last_undegraded":"2023-10-01T09:47:23.493094+0000","last_fullsized":"2023-10-01T09:47:23.493094+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":408,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.493325+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:47:23.493325+0000","last_peered":"2023-10-01T09:47:23.493325+0000","last_clean":"2023-10-01T09:47:23.493325+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:47:23.493325+0000","last_undegraded":"2023-10-01T09:47:23.493325+0000","last_fullsized":"2023-10-01T09:47:23.493325+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":81,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.492991+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:47:23.492991+0000","last_peered":"2023-10-01T09:47:23.492991+0000","last_clean":"2023-10-01T09:47:23.492991+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:47:23.492991+0000","last_undegraded":"2023-10-01T09:47:23.492991+0000","last_fullsized":"2023-10-01T09:47:23.492991+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.916010+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:47:28.916010+0000","last_peered":"2023-10-01T09:47:28.916010+0000","last_clean":"2023-10-01T09:47:28.916010+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:47:28.916010+0000","last_undegraded":"2023-10-01T09:47:28.916010+0000","last_fullsized":"2023-10-01T09:47:28.916010+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":564,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.492990+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:47:23.492990+0000","last_peered":"2023-10-01T09:47:23.492990+0000","last_clean":"2023-10-01T09:47:23.492990+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:47:23.492990+0000","last_undegraded":"2023-10-01T09:47:23.492990+0000","last_fullsized":"2023-10-01T09:47:23.492990+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":578,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912302+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:47:28.912302+0000","last_peered":"2023-10-01T09:47:28.912302+0000","last_clean":"2023-10-01T09:47:28.912302+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:47:28.912302+0000","last_undegraded":"2023-10-01T09:47:28.912302+0000","last_fullsized":"2023-10-01T09:47:28.912302+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":434,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.911877+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:47:28.911877+0000","last_peered":"2023-10-01T09:47:28.911877+0000","last_clean":"2023-10-01T09:47:28.911877+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:47:28.911877+0000","last_undegraded":"2023-10-01T09:47:28.911877+0000","last_fullsized":"2023-10-01T09:47:28.911877+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":81,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.493585+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:47:23.493585+0000","last_peered":"2023-10-01T09:47:23.493585+0000","last_clean":"2023-10-01T09:47:23.493585+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:47:23.493585+0000","last_undegraded":"2023-10-01T09:47:23.493585+0000","last_fullsized":"2023-10-01T09:47:23.493585+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"554'8901","reported_seq":8988,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:37.576308+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:47:37.576308+0000","last_peered":"2023-10-01T09:47:37.576308+0000","last_clean":"2023-10-01T09:47:37.576308+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:47:37.576308+0000","last_undegraded":"2023-10-01T09:47:37.576308+0000","last_fullsized":"2023-10-01T09:47:37.576308+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8901,"log_dups_size":0,"ondisk_log_size":8901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97714176,"num_objects":1491,"num_object_clones":0,"num_object_copies":4473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8901,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":76,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.493475+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:47:23.493475+0000","last_peered":"2023-10-01T09:47:23.493475+0000","last_clean":"2023-10-01T09:47:23.493475+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:47:23.493475+0000","last_undegraded":"2023-10-01T09:47:23.493475+0000","last_fullsized":"2023-10-01T09:47:23.493475+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":274,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915813+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:47:28.915813+0000","last_peered":"2023-10-01T09:47:28.915813+0000","last_clean":"2023-10-01T09:47:28.915813+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:47:28.915813+0000","last_undegraded":"2023-10-01T09:47:28.915813+0000","last_fullsized":"2023-10-01T09:47:28.915813+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"554'8933","reported_seq":20130,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:47:37.585620+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:47:37.585620+0000","last_peered":"2023-10-01T09:47:37.585620+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:47:37.585620+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8933,"log_dups_size":0,"ondisk_log_size":8933,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96534528,"num_objects":1473,"num_object_clones":0,"num_object_copies":4419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1437,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8933,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3698,"num_bytes_recovered":242352128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1405},{"shards":"5,6","objects":36}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":547,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.493611+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:47:23.493611+0000","last_peered":"2023-10-01T09:47:23.493611+0000","last_clean":"2023-10-01T09:47:23.493611+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:47:23.493611+0000","last_undegraded":"2023-10-01T09:47:23.493611+0000","last_fullsized":"2023-10-01T09:47:23.493611+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":610,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498576+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:47:23.498576+0000","last_peered":"2023-10-01T09:47:23.498576+0000","last_clean":"2023-10-01T09:47:23.498576+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:47:23.498576+0000","last_undegraded":"2023-10-01T09:47:23.498576+0000","last_fullsized":"2023-10-01T09:47:23.498576+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"554'8871","reported_seq":8962,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.895116+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:47:36.895116+0000","last_peered":"2023-10-01T09:47:36.895116+0000","last_clean":"2023-10-01T09:47:36.895116+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:47:36.895116+0000","last_undegraded":"2023-10-01T09:47:36.895116+0000","last_fullsized":"2023-10-01T09:47:36.895116+0000","mapping_epoch":475,"log_start":"484'1800","ondisk_log_start":"484'1800","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7071,"log_dups_size":0,"ondisk_log_size":7071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102301696,"num_objects":1561,"num_object_clones":0,"num_object_copies":4683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8871,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":633,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.493402+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:47:23.493402+0000","last_peered":"2023-10-01T09:47:23.493402+0000","last_clean":"2023-10-01T09:47:23.493402+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:47:23.493402+0000","last_undegraded":"2023-10-01T09:47:23.493402+0000","last_fullsized":"2023-10-01T09:47:23.493402+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":430,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498760+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:47:23.498760+0000","last_peered":"2023-10-01T09:47:23.498760+0000","last_clean":"2023-10-01T09:47:23.498760+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:47:23.498760+0000","last_undegraded":"2023-10-01T09:47:23.498760+0000","last_fullsized":"2023-10-01T09:47:23.498760+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"554'8892","reported_seq":15588,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.880288+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:47:36.880288+0000","last_peered":"2023-10-01T09:47:36.880288+0000","last_clean":"2023-10-01T09:47:36.880288+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:47:36.880288+0000","last_undegraded":"2023-10-01T09:47:36.880288+0000","last_fullsized":"2023-10-01T09:47:36.880288+0000","mapping_epoch":505,"log_start":"484'1900","ondisk_log_start":"484'1900","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98959360,"num_objects":1510,"num_object_clones":0,"num_object_copies":4530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8892,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":274,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.919741+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:47:28.919741+0000","last_peered":"2023-10-01T09:47:28.919741+0000","last_clean":"2023-10-01T09:47:28.919741+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:47:28.919741+0000","last_undegraded":"2023-10-01T09:47:28.919741+0000","last_fullsized":"2023-10-01T09:47:28.919741+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":534,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912024+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:47:28.912024+0000","last_peered":"2023-10-01T09:47:28.912024+0000","last_clean":"2023-10-01T09:47:28.912024+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:47:28.912024+0000","last_undegraded":"2023-10-01T09:47:28.912024+0000","last_fullsized":"2023-10-01T09:47:28.912024+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"553'8865","reported_seq":8955,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:35.214957+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:47:35.214957+0000","last_peered":"2023-10-01T09:47:35.214957+0000","last_clean":"2023-10-01T09:47:35.214957+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:47:35.214957+0000","last_undegraded":"2023-10-01T09:47:35.214957+0000","last_fullsized":"2023-10-01T09:47:35.214957+0000","mapping_epoch":475,"log_start":"482'1300","ondisk_log_start":"482'1300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7565,"log_dups_size":0,"ondisk_log_size":7565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107675648,"num_objects":1643,"num_object_clones":0,"num_object_copies":4929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8865,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":133,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.499063+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:47:23.499063+0000","last_peered":"2023-10-01T09:47:23.499063+0000","last_clean":"2023-10-01T09:47:23.499063+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:47:23.499063+0000","last_undegraded":"2023-10-01T09:47:23.499063+0000","last_fullsized":"2023-10-01T09:47:23.499063+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":207,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498977+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:47:23.498977+0000","last_peered":"2023-10-01T09:47:23.498977+0000","last_clean":"2023-10-01T09:47:23.498977+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:47:23.498977+0000","last_undegraded":"2023-10-01T09:47:23.498977+0000","last_fullsized":"2023-10-01T09:47:23.498977+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"554'8899","reported_seq":9750,"reported_epoch":554,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:47:37.680924+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:47:37.680924+0000","last_peered":"2023-10-01T09:47:37.680924+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:47:37.680924+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8899,"log_dups_size":0,"ondisk_log_size":8899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96272384,"num_objects":1469,"num_object_clones":0,"num_object_copies":4407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3339,"num_objects_misplaced":1404,"num_objects_unfound":0,"num_objects_dirty":1469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8899,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":378,"num_bytes_recovered":24772608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1404},{"shards":"4,5","objects":-3274}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":451,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498621+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:47:23.498621+0000","last_peered":"2023-10-01T09:47:23.498621+0000","last_clean":"2023-10-01T09:47:23.498621+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:47:23.498621+0000","last_undegraded":"2023-10-01T09:47:23.498621+0000","last_fullsized":"2023-10-01T09:47:23.498621+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":613,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498990+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:47:23.498990+0000","last_peered":"2023-10-01T09:47:23.498990+0000","last_clean":"2023-10-01T09:47:23.498990+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:47:23.498990+0000","last_undegraded":"2023-10-01T09:47:23.498990+0000","last_fullsized":"2023-10-01T09:47:23.498990+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"554'8889","reported_seq":8985,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:37.575186+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:47:37.575186+0000","last_peered":"2023-10-01T09:47:37.575186+0000","last_clean":"2023-10-01T09:47:37.575186+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:47:37.575186+0000","last_undegraded":"2023-10-01T09:47:37.575186+0000","last_fullsized":"2023-10-01T09:47:37.575186+0000","mapping_epoch":483,"log_start":"482'1300","ondisk_log_start":"482'1300","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7589,"log_dups_size":0,"ondisk_log_size":7589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97583104,"num_objects":1489,"num_object_clones":0,"num_object_copies":4467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8889,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":447,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498490+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:47:23.498490+0000","last_peered":"2023-10-01T09:47:23.498490+0000","last_clean":"2023-10-01T09:47:23.498490+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:47:23.498490+0000","last_undegraded":"2023-10-01T09:47:23.498490+0000","last_fullsized":"2023-10-01T09:47:23.498490+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":288,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.499026+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:47:23.499026+0000","last_peered":"2023-10-01T09:47:23.499026+0000","last_clean":"2023-10-01T09:47:23.499026+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:47:23.499026+0000","last_undegraded":"2023-10-01T09:47:23.499026+0000","last_fullsized":"2023-10-01T09:47:23.499026+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"553'8968","reported_seq":9054,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.190686+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:47:36.190686+0000","last_peered":"2023-10-01T09:47:36.190686+0000","last_clean":"2023-10-01T09:47:36.190686+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:47:36.190686+0000","last_undegraded":"2023-10-01T09:47:36.190686+0000","last_fullsized":"2023-10-01T09:47:36.190686+0000","mapping_epoch":475,"log_start":"481'600","ondisk_log_start":"481'600","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8368,"log_dups_size":0,"ondisk_log_size":8368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100270080,"num_objects":1530,"num_object_clones":0,"num_object_copies":4590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8968,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":417,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498442+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:47:23.498442+0000","last_peered":"2023-10-01T09:47:23.498442+0000","last_clean":"2023-10-01T09:47:23.498442+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:47:23.498442+0000","last_undegraded":"2023-10-01T09:47:23.498442+0000","last_fullsized":"2023-10-01T09:47:23.498442+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":464,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.499314+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:47:23.499314+0000","last_peered":"2023-10-01T09:47:23.499314+0000","last_clean":"2023-10-01T09:47:23.499314+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:47:23.499314+0000","last_undegraded":"2023-10-01T09:47:23.499314+0000","last_fullsized":"2023-10-01T09:47:23.499314+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"553'8702","reported_seq":8788,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.223915+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:47:36.223915+0000","last_peered":"2023-10-01T09:47:36.223915+0000","last_clean":"2023-10-01T09:47:36.223915+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:47:36.223915+0000","last_undegraded":"2023-10-01T09:47:36.223915+0000","last_fullsized":"2023-10-01T09:47:36.223915+0000","mapping_epoch":475,"log_start":"481'300","ondisk_log_start":"481'300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8402,"log_dups_size":0,"ondisk_log_size":8402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96731136,"num_objects":1476,"num_object_clones":0,"num_object_copies":4428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8702,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":946,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.493172+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:47:23.493172+0000","last_peered":"2023-10-01T09:47:23.493172+0000","last_clean":"2023-10-01T09:47:23.493172+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:47:23.493172+0000","last_undegraded":"2023-10-01T09:47:23.493172+0000","last_fullsized":"2023-10-01T09:47:23.493172+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":284,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915884+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:47:28.915884+0000","last_peered":"2023-10-01T09:47:28.915884+0000","last_clean":"2023-10-01T09:47:28.915884+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:47:28.915884+0000","last_undegraded":"2023-10-01T09:47:28.915884+0000","last_fullsized":"2023-10-01T09:47:28.915884+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":78,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498685+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:47:23.498685+0000","last_peered":"2023-10-01T09:47:23.498685+0000","last_clean":"2023-10-01T09:47:23.498685+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:47:23.498685+0000","last_undegraded":"2023-10-01T09:47:23.498685+0000","last_fullsized":"2023-10-01T09:47:23.498685+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"554'8795","reported_seq":8886,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.888896+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:47:36.888896+0000","last_peered":"2023-10-01T09:47:36.888896+0000","last_clean":"2023-10-01T09:47:36.888896+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:47:36.888896+0000","last_undegraded":"2023-10-01T09:47:36.888896+0000","last_fullsized":"2023-10-01T09:47:36.888896+0000","mapping_epoch":475,"log_start":"484'1800","ondisk_log_start":"484'1800","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103219200,"num_objects":1575,"num_object_clones":0,"num_object_copies":4725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8795,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1219,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.916160+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:47:28.916160+0000","last_peered":"2023-10-01T09:47:28.916160+0000","last_clean":"2023-10-01T09:47:28.916160+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:47:28.916160+0000","last_undegraded":"2023-10-01T09:47:28.916160+0000","last_fullsized":"2023-10-01T09:47:28.916160+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":428,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498898+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:47:23.498898+0000","last_peered":"2023-10-01T09:47:23.498898+0000","last_clean":"2023-10-01T09:47:23.498898+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:47:23.498898+0000","last_undegraded":"2023-10-01T09:47:23.498898+0000","last_fullsized":"2023-10-01T09:47:23.498898+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":257,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915953+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:47:28.915953+0000","last_peered":"2023-10-01T09:47:28.915953+0000","last_clean":"2023-10-01T09:47:28.915953+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:47:28.915953+0000","last_undegraded":"2023-10-01T09:47:28.915953+0000","last_fullsized":"2023-10-01T09:47:28.915953+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"554'8822","reported_seq":16130,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:37.660617+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:47:37.660617+0000","last_peered":"2023-10-01T09:47:37.660617+0000","last_clean":"2023-10-01T09:47:37.660617+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:47:37.660617+0000","last_undegraded":"2023-10-01T09:47:37.660617+0000","last_fullsized":"2023-10-01T09:47:37.660617+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8822,"log_dups_size":0,"ondisk_log_size":8822,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101842944,"num_objects":1554,"num_object_clones":0,"num_object_copies":4662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8822,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":901,"reported_epoch":553,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:47:28.911725+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:47:28.911725+0000","last_peered":"2023-10-01T09:47:28.911725+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:47:28.911725+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:47:28.911725+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":617,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498476+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:47:23.498476+0000","last_peered":"2023-10-01T09:47:23.498476+0000","last_clean":"2023-10-01T09:47:23.498476+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:47:23.498476+0000","last_undegraded":"2023-10-01T09:47:23.498476+0000","last_fullsized":"2023-10-01T09:47:23.498476+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.499179+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:47:23.499179+0000","last_peered":"2023-10-01T09:47:23.499179+0000","last_clean":"2023-10-01T09:47:23.499179+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:47:23.499179+0000","last_undegraded":"2023-10-01T09:47:23.499179+0000","last_fullsized":"2023-10-01T09:47:23.499179+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"553'8725","reported_seq":8815,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:35.241456+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:47:35.241456+0000","last_peered":"2023-10-01T09:47:35.241456+0000","last_clean":"2023-10-01T09:47:35.241456+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:47:35.241456+0000","last_undegraded":"2023-10-01T09:47:35.241456+0000","last_fullsized":"2023-10-01T09:47:35.241456+0000","mapping_epoch":475,"log_start":"481'1200","ondisk_log_start":"481'1200","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7525,"log_dups_size":0,"ondisk_log_size":7525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101384192,"num_objects":1547,"num_object_clones":0,"num_object_copies":4641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8725,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1670,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915829+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:47:28.915829+0000","last_peered":"2023-10-01T09:47:28.915829+0000","last_clean":"2023-10-01T09:47:28.915829+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:47:28.915829+0000","last_undegraded":"2023-10-01T09:47:28.915829+0000","last_fullsized":"2023-10-01T09:47:28.915829+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":583,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.492857+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:47:23.492857+0000","last_peered":"2023-10-01T09:47:23.492857+0000","last_clean":"2023-10-01T09:47:23.492857+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:47:23.492857+0000","last_undegraded":"2023-10-01T09:47:23.492857+0000","last_fullsized":"2023-10-01T09:47:23.492857+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":362,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.916215+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:47:28.916215+0000","last_peered":"2023-10-01T09:47:28.916215+0000","last_clean":"2023-10-01T09:47:28.916215+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:47:28.916215+0000","last_undegraded":"2023-10-01T09:47:28.916215+0000","last_fullsized":"2023-10-01T09:47:28.916215+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":351,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498908+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:47:23.498908+0000","last_peered":"2023-10-01T09:47:23.498908+0000","last_clean":"2023-10-01T09:47:23.498908+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:47:23.498908+0000","last_undegraded":"2023-10-01T09:47:23.498908+0000","last_fullsized":"2023-10-01T09:47:23.498908+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"553'8960","reported_seq":9046,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.145960+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:47:36.145960+0000","last_peered":"2023-10-01T09:47:36.145960+0000","last_clean":"2023-10-01T09:47:36.145960+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:47:36.145960+0000","last_undegraded":"2023-10-01T09:47:36.145960+0000","last_fullsized":"2023-10-01T09:47:36.145960+0000","mapping_epoch":475,"log_start":"481'600","ondisk_log_start":"481'600","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8360,"log_dups_size":0,"ondisk_log_size":8360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103022592,"num_objects":1572,"num_object_clones":0,"num_object_copies":4716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8960,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":760,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.493039+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:47:23.493039+0000","last_peered":"2023-10-01T09:47:23.493039+0000","last_clean":"2023-10-01T09:47:23.493039+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:47:23.493039+0000","last_undegraded":"2023-10-01T09:47:23.493039+0000","last_fullsized":"2023-10-01T09:47:23.493039+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":595,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.911970+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:47:28.911970+0000","last_peered":"2023-10-01T09:47:28.911970+0000","last_clean":"2023-10-01T09:47:28.911970+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:47:28.911970+0000","last_undegraded":"2023-10-01T09:47:28.911970+0000","last_fullsized":"2023-10-01T09:47:28.911970+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":594,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912194+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:47:28.912194+0000","last_peered":"2023-10-01T09:47:28.912194+0000","last_clean":"2023-10-01T09:47:28.912194+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:47:28.912194+0000","last_undegraded":"2023-10-01T09:47:28.912194+0000","last_fullsized":"2023-10-01T09:47:28.912194+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"554'8864","reported_seq":8956,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.879930+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:47:36.879930+0000","last_peered":"2023-10-01T09:47:36.879930+0000","last_clean":"2023-10-01T09:47:36.879930+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:47:36.879930+0000","last_undegraded":"2023-10-01T09:47:36.879930+0000","last_fullsized":"2023-10-01T09:47:36.879930+0000","mapping_epoch":475,"log_start":"484'1800","ondisk_log_start":"484'1800","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7064,"log_dups_size":0,"ondisk_log_size":7064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104529944,"num_objects":1596,"num_object_clones":0,"num_object_copies":4788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1596,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8864,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":991,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.493482+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:47:23.493482+0000","last_peered":"2023-10-01T09:47:23.493482+0000","last_clean":"2023-10-01T09:47:23.493482+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:47:23.493482+0000","last_undegraded":"2023-10-01T09:47:23.493482+0000","last_fullsized":"2023-10-01T09:47:23.493482+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":611,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915877+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:47:28.915877+0000","last_peered":"2023-10-01T09:47:28.915877+0000","last_clean":"2023-10-01T09:47:28.915877+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:47:28.915877+0000","last_undegraded":"2023-10-01T09:47:28.915877+0000","last_fullsized":"2023-10-01T09:47:28.915877+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":464,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915917+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:47:28.915917+0000","last_peered":"2023-10-01T09:47:28.915917+0000","last_clean":"2023-10-01T09:47:28.915917+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:47:28.915917+0000","last_undegraded":"2023-10-01T09:47:28.915917+0000","last_fullsized":"2023-10-01T09:47:28.915917+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"553'8814","reported_seq":8908,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:35.957810+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:47:35.957810+0000","last_peered":"2023-10-01T09:47:35.957810+0000","last_clean":"2023-10-01T09:47:35.957810+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:47:35.957810+0000","last_undegraded":"2023-10-01T09:47:35.957810+0000","last_fullsized":"2023-10-01T09:47:35.957810+0000","mapping_epoch":479,"log_start":"485'2100","ondisk_log_start":"485'2100","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6714,"log_dups_size":0,"ondisk_log_size":6714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101056512,"num_objects":1542,"num_object_clones":0,"num_object_copies":4626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8814,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":492,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.493610+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:47:23.493610+0000","last_peered":"2023-10-01T09:47:23.493610+0000","last_clean":"2023-10-01T09:47:23.493610+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:47:23.493610+0000","last_undegraded":"2023-10-01T09:47:23.493610+0000","last_fullsized":"2023-10-01T09:47:23.493610+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1187,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.499214+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:47:23.499214+0000","last_peered":"2023-10-01T09:47:23.499214+0000","last_clean":"2023-10-01T09:47:23.499214+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:47:23.499214+0000","last_undegraded":"2023-10-01T09:47:23.499214+0000","last_fullsized":"2023-10-01T09:47:23.499214+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":81,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.492831+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:47:23.492831+0000","last_peered":"2023-10-01T09:47:23.492831+0000","last_clean":"2023-10-01T09:47:23.492831+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:47:23.492831+0000","last_undegraded":"2023-10-01T09:47:23.492831+0000","last_fullsized":"2023-10-01T09:47:23.492831+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"553'8918","reported_seq":9012,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.206608+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:47:36.206608+0000","last_peered":"2023-10-01T09:47:36.206608+0000","last_clean":"2023-10-01T09:47:36.206608+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:47:36.206608+0000","last_undegraded":"2023-10-01T09:47:36.206608+0000","last_fullsized":"2023-10-01T09:47:36.206608+0000","mapping_epoch":479,"log_start":"485'2200","ondisk_log_start":"485'2200","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6718,"log_dups_size":0,"ondisk_log_size":6718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104464384,"num_objects":1594,"num_object_clones":0,"num_object_copies":4782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8918,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":85,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.493489+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:47:23.493489+0000","last_peered":"2023-10-01T09:47:23.493489+0000","last_clean":"2023-10-01T09:47:23.493489+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:47:23.493489+0000","last_undegraded":"2023-10-01T09:47:23.493489+0000","last_fullsized":"2023-10-01T09:47:23.493489+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":572,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912222+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:47:28.912222+0000","last_peered":"2023-10-01T09:47:28.912222+0000","last_clean":"2023-10-01T09:47:28.912222+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:47:28.912222+0000","last_undegraded":"2023-10-01T09:47:28.912222+0000","last_fullsized":"2023-10-01T09:47:28.912222+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":277,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915787+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:47:28.915787+0000","last_peered":"2023-10-01T09:47:28.915787+0000","last_clean":"2023-10-01T09:47:28.915787+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:47:28.915787+0000","last_undegraded":"2023-10-01T09:47:28.915787+0000","last_fullsized":"2023-10-01T09:47:28.915787+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":521,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912194+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:47:28.912194+0000","last_peered":"2023-10-01T09:47:28.912194+0000","last_clean":"2023-10-01T09:47:28.912194+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:47:28.912194+0000","last_undegraded":"2023-10-01T09:47:28.912194+0000","last_fullsized":"2023-10-01T09:47:28.912194+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":634,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498762+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:47:23.498762+0000","last_peered":"2023-10-01T09:47:23.498762+0000","last_clean":"2023-10-01T09:47:23.498762+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:47:23.498762+0000","last_undegraded":"2023-10-01T09:47:23.498762+0000","last_fullsized":"2023-10-01T09:47:23.498762+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":289,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915788+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:47:28.915788+0000","last_peered":"2023-10-01T09:47:28.915788+0000","last_clean":"2023-10-01T09:47:28.915788+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:47:28.915788+0000","last_undegraded":"2023-10-01T09:47:28.915788+0000","last_fullsized":"2023-10-01T09:47:28.915788+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":77,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.499071+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:47:23.499071+0000","last_peered":"2023-10-01T09:47:23.499071+0000","last_clean":"2023-10-01T09:47:23.499071+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:47:23.499071+0000","last_undegraded":"2023-10-01T09:47:23.499071+0000","last_fullsized":"2023-10-01T09:47:23.499071+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":289,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915940+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:47:28.915940+0000","last_peered":"2023-10-01T09:47:28.915940+0000","last_clean":"2023-10-01T09:47:28.915940+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:47:28.915940+0000","last_undegraded":"2023-10-01T09:47:28.915940+0000","last_fullsized":"2023-10-01T09:47:28.915940+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":467,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912113+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:47:28.912113+0000","last_peered":"2023-10-01T09:47:28.912113+0000","last_clean":"2023-10-01T09:47:28.912113+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:47:28.912113+0000","last_undegraded":"2023-10-01T09:47:28.912113+0000","last_fullsized":"2023-10-01T09:47:28.912113+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":95,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.499123+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:47:23.499123+0000","last_peered":"2023-10-01T09:47:23.499123+0000","last_clean":"2023-10-01T09:47:23.499123+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:47:23.499123+0000","last_undegraded":"2023-10-01T09:47:23.499123+0000","last_fullsized":"2023-10-01T09:47:23.499123+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":512,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.911891+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:47:28.911891+0000","last_peered":"2023-10-01T09:47:28.911891+0000","last_clean":"2023-10-01T09:47:28.911891+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:47:28.911891+0000","last_undegraded":"2023-10-01T09:47:28.911891+0000","last_fullsized":"2023-10-01T09:47:28.911891+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":712,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912460+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:47:28.912460+0000","last_peered":"2023-10-01T09:47:28.912460+0000","last_clean":"2023-10-01T09:47:28.912460+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:47:28.912460+0000","last_undegraded":"2023-10-01T09:47:28.912460+0000","last_fullsized":"2023-10-01T09:47:28.912460+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":404,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-10-01T09:47:23.498394+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:47:23.498394+0000","last_peered":"2023-10-01T09:47:23.498394+0000","last_clean":"2023-10-01T09:47:23.498394+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:47:23.498394+0000","last_undegraded":"2023-10-01T09:47:23.498394+0000","last_fullsized":"2023-10-01T09:47:23.498394+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1613561880,"num_objects":24622,"num_object_clones":0,"num_object_copies":73866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4776,"num_objects_misplaced":1404,"num_objects_unfound":0,"num_objects_dirty":24622,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":141818,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10969,"num_bytes_recovered":718864384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5652688896,"data_stored":5652676680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124918,"ondisk_log_size":124918,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430764,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1397400,"kb_used_data":502480,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":92347752,"statfs":{"total":95995035648,"available":94564098048,"internally_reserved":0,"allocated":514539520,"data_stored":513593068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:02 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.495,"15min":2.4870000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":1.3600000000000001,"5min":3.3849999999999998,"15min":108.90000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.46899999999999997,"15min":2.621},"min":{"1min":0.23300000000000001,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":1.7410000000000001,"5min":1.7410000000000001,"15min":109.187},"last":0.47699999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.58299999999999996,"15min":5.0190000000000001},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.23300000000000001},"max":{"1min":7.7140000000000004,"5min":7.7140000000000004,"15min":229.161},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.53100000000000003,"15min":5.0449999999999999},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":1.972,"5min":3.0640000000000001,"15min":230.126},"last":0.50600000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:46:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.56299999999999994,"15min":1.591},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":1.6359999999999999,"5min":3.4300000000000002,"15min":14.901999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.55700000000000005,"15min":1.48},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.27000000000000002},"max":{"1min":1.8100000000000001,"5min":1.8100000000000001,"15min":14.682},"last":0.54700000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.59299999999999997,"15min":2.1059999999999999},"min":{"1min":0.34999999999999998,"5min":0.30199999999999999,"15min":0.29199999999999998},"max":{"1min":2.1749999999999998,"5min":2.476,"15min":76.555000000000007},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.58399999999999996,"15min":2.1120000000000001},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.1880000000000002,"5min":2.8479999999999999,"15min":76.534000000000006},"last":0.53400000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:47:02 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.52100000000000002,"15min":1.0249999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.4470000000000001,"5min":3.4609999999999999,"15min":9.6020000000000003},"last":0.44},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.52500000000000002,"15min":0.99399999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.837,"5min":1.837,"15min":8.4949999999999992},"last":0.57099999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.52500000000000002,"15min":1.5269999999999999},"min":{"1min":0.35399999999999998,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":2.2400000000000002,"5min":2.2400000000000002,"15min":135.40000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.54500000000000004,"15min":1.627},"min":{"1min":0.34999999999999998,"5min":0.307,"15min":0.26100000000000001},"max":{"1min":0.70699999999999996,"5min":2.8849999999999998,"15min":135.339},"last":0.58599999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.58999999999999997,"15min":1.1200000000000001},"min":{"1min":0.40500000000000003,"5min":0.313,"15min":0.30299999999999999},"max":{"1min":1.7829999999999999,"5min":2.448,"15min":11.853999999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.59699999999999998,"15min":1.093},"min":{"1min":0.39700000000000002,"5min":0.33900000000000002,"15min":0.32300000000000001},"max":{"1min":3.1909999999999998,"5min":3.7810000000000001,"15min":14.584},"last":0.56200000000000006}]}]},{"osd":6,"up_from":294,"seq":1262720385275,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1041052,"kb_used_data":699348,"kb_used_omap":6,"kb_used_meta":341689,"kb_avail":92704100,"statfs":{"total":95995035648,"available":94928998400,"internally_reserved":0,"allocated":716132352,"data_stored":715189527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":349889581},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":4.5839999999999996,"5min":1.2829999999999999,"15min":2.665},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20599999999999999},"max":{"1min":65.599000000000004,"5min":65.599000000000004,"15min":158.48599999999999},"last":4.79},{"interface":"front","average":{"1min":5.4089999999999998,"5min":1.484,"15min":2.7909999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":65.388999999999996,"5min":65.388999999999996,"15min":158.37700000000001},"last":0.441}]},{"osd":1,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":0.70599999999999996,"15min":3.8839999999999999},"min":{"1min":0.34399999999999997,"5min":0.222,"15min":0.222},"max":{"1min":8.2409999999999997,"5min":8.2409999999999997,"15min":168.947},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.625,"5min":0.56999999999999995,"15min":3.8210000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.25600000000000001},"max":{"1min":2.6669999999999998,"5min":3.7469999999999999,"15min":169.07900000000001},"last":0.44800000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":0.73799999999999999,"15min":1.429},"min":{"1min":0.311,"5min":0.254,"15min":0.254},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":10.92},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.748,"5min":0.57199999999999995,"15min":1.3899999999999999},"min":{"1min":0.314,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":2.6480000000000001,"5min":3.7160000000000002,"15min":21.704999999999998},"last":0.505}]},{"osd":3,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55000000000000004,"15min":1.647},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.9710000000000001,"5min":3.3759999999999999,"15min":89.753},"last":4.835},{"interface":"front","average":{"1min":1.51,"5min":0.78200000000000003,"15min":1.7589999999999999},"min":{"1min":0.34399999999999997,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.3109999999999999,"5min":8.3109999999999999,"15min":89.849999999999994},"last":0.52000000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.505,"15min":1.0609999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.7559999999999998,"5min":3.6349999999999998,"15min":63.134},"last":4.1699999999999999},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.54400000000000004,"15min":1.0429999999999999},"min":{"1min":0.34499999999999997,"5min":0.33600000000000002,"15min":0.28899999999999998},"max":{"1min":2.778,"5min":3.2519999999999998,"15min":62.973999999999997},"last":4.9509999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:47:37 2023","interfaces":[{"interface":"back","average":{"1min":6.5650000000000004,"5min":2.089,"15min":1.367},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":136.535,"5min":136.535,"15min":136.535},"last":4.9800000000000004},{"interface":"front","average":{"1min":6.3049999999999997,"5min":2.1499999999999999,"15min":1.4830000000000001},"min":{"1min":0.316,"5min":0.29899999999999999,"15min":0.28100000000000003},"max":{"1min":136.47300000000001,"5min":136.47300000000001,"15min":136.47300000000001},"last":0.64400000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.56100000000000005,"15min":1.2609999999999999},"min":{"1min":0.38300000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.3159999999999998,"5min":3.339,"15min":94.245999999999995},"last":4.8639999999999999},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.76700000000000002,"15min":1.421},"min":{"1min":0.37,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":8.3399999999999999,"5min":8.3399999999999999,"15min":94.715000000000003},"last":0.49399999999999999}]}]},{"osd":1,"up_from":94,"seq":403726926251,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1150772,"kb_used_data":314540,"kb_used_omap":7,"kb_used_meta":836216,"kb_avail":92594380,"statfs":{"total":95995035648,"available":94816645120,"internally_reserved":0,"allocated":322088960,"data_stored":320002588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":856285607},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":1.0349999999999999,"15min":3.5819999999999999},"min":{"1min":0.19500000000000001,"5min":0.186,"15min":0.182},"max":{"1min":0.82799999999999996,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.314},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.0860000000000001,"15min":3.1030000000000002},"min":{"1min":0.222,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":2.0859999999999999,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.16800000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.35899999999999999,"15min":2.0179999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.20000000000000001},"max":{"1min":0.78500000000000003,"5min":1.3240000000000001,"15min":51.640000000000001},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.44700000000000001,"15min":2.7570000000000001},"min":{"1min":0.29199999999999998,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":0.69499999999999995,"5min":2.9089999999999998,"15min":154.197},"last":0.25900000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45000000000000001,"15min":2.706},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.219},"max":{"1min":2.149,"5min":2.8780000000000001,"15min":95.305000000000007},"last":0.222},{"interface":"front","average":{"1min":0.497,"5min":0.47399999999999998,"15min":3.3210000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.994,"5min":2.7360000000000002,"15min":154.483},"last":0.28599999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.56499999999999995,"15min":2.1389999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":1.204,"5min":3.0550000000000002,"15min":154.101},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.46800000000000003,"15min":2.1469999999999998},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.27700000000000002},"max":{"1min":0.77200000000000002,"5min":0.83199999999999996,"15min":154.56999999999999},"last":0.34899999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.143,"15min":2.23},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.8929999999999998,"5min":59.561,"15min":154.22999999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":1.137,"15min":2.3660000000000001},"min":{"1min":0.32900000000000001,"5min":0.31,"15min":0.307},"max":{"1min":0.89600000000000002,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.39400000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.65700000000000003,"15min":2.2770000000000001},"min":{"1min":0.33400000000000002,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":4.5069999999999997,"5min":4.5069999999999997,"15min":154.12200000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.63200000000000001,"15min":2.218},"min":{"1min":0.35899999999999999,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":1.5900000000000001,"5min":2.5630000000000002,"15min":154.16999999999999},"last":0.30299999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.73299999999999998,"15min":2.4420000000000002},"min":{"1min":0.38900000000000001,"5min":0.36399999999999999,"15min":0.33900000000000002},"max":{"1min":4.3730000000000002,"5min":4.3730000000000002,"15min":154.20699999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.71999999999999997,"15min":2.4460000000000002},"min":{"1min":0.435,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":4.3879999999999999,"5min":4.3879999999999999,"15min":154.51599999999999},"last":0.36799999999999999}]}]},{"osd":0,"up_from":89,"seq":382252089786,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2586952,"kb_used_data":1659712,"kb_used_omap":8,"kb_used_meta":927223,"kb_avail":91158200,"statfs":{"total":95995035648,"available":93345996800,"internally_reserved":0,"allocated":1699545088,"data_stored":1697506385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":949477314},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.40500000000000003,"15min":3.6469999999999998},"min":{"1min":0.20799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.69099999999999995,"5min":4.2919999999999998,"15min":216.821},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.434,"15min":4.2300000000000004},"min":{"1min":0.20599999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.72499999999999998,"5min":3.9620000000000002,"15min":217.02699999999999},"last":0.28999999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.47499999999999998,"15min":1.9059999999999999},"min":{"1min":0.29399999999999998,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":1.3100000000000001,"5min":4.2430000000000003,"15min":19.788},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54700000000000004,"15min":1.9239999999999999},"min":{"1min":0.29299999999999998,"5min":0.27200000000000002,"15min":0.215},"max":{"1min":1.3300000000000001,"5min":3.8719999999999999,"15min":25.844999999999999},"last":0.34799999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.47899999999999998,"15min":2.778},"min":{"1min":0.24299999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.68000000000000005,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.626,"15min":2.6200000000000001},"min":{"1min":0.25700000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.0259999999999998,"5min":4.9199999999999999,"15min":222.227},"last":0.379}]},{"osd":4,"last update":"Sun Oct 1 09:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57199999999999995,"15min":1.8500000000000001},"min":{"1min":0.308,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":2.1160000000000001,"5min":4.048,"15min":62.206000000000003},"last":0.437},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.52500000000000002,"15min":1.9359999999999999},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":1.3380000000000001,"5min":4,"15min":62.073999999999998},"last":0.307}]},{"osd":5,"last update":"Sun Oct 1 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.59999999999999998,"15min":1.6699999999999999},"min":{"1min":0.34899999999999998,"5min":0.312,"15min":0.312},"max":{"1min":1.5920000000000001,"5min":4.0350000000000001,"15min":16.808},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.59899999999999998,"15min":1.5820000000000001},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.8100000000000001,"5min":4.2729999999999997,"15min":17.507000000000001},"last":0.627}]},{"osd":6,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.64400000000000002,"15min":1.51},"min":{"1min":0.35699999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.589,"5min":4.3129999999999997,"15min":17.535},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.66700000000000004,"15min":1.571},"min":{"1min":0.35899999999999999,"5min":0.32600000000000001,"15min":0.32400000000000001},"max":{"1min":2.5960000000000001,"5min":4.2050000000000001,"15min":15.651999999999999},"last":0.48499999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.66300000000000003,"15min":1.927},"min":{"1min":0.39700000000000002,"5min":0.374,"15min":0.374},"max":{"1min":2.2290000000000001,"5min":4.3360000000000003,"15min":48.895000000000003},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.79400000000000004,"15min":1.97},"min":{"1min":0.39300000000000002,"5min":0.376,"15min":0.35299999999999998},"max":{"1min":0.78300000000000003,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.503}]}]},{"osd":2,"up_from":0,"seq":425201762725,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599199,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1170492,"kb_used_data":312116,"kb_used_omap":3,"kb_used_meta":858364,"kb_avail":92574660,"statfs":{"total":95995035648,"available":94796451840,"internally_reserved":0,"allocated":319606784,"data_stored":317461963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":878965479},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.26900000000000002,"5min":0.34300000000000003,"15min":3.5009999999999999},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":0.504,"5min":2.7989999999999999,"15min":137.34399999999999},"last":0.254},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.35899999999999999,"15min":3.5459999999999998},"min":{"1min":0.23400000000000001,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":0.66800000000000004,"5min":1.2849999999999999,"15min":137.73400000000001},"last":0.36099999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.40200000000000002,"15min":4.7430000000000003},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":0.63500000000000001,"5min":2.7250000000000001,"15min":162.22999999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.371,"15min":4.7370000000000001},"min":{"1min":0.27200000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.3440000000000001,"5min":1.4119999999999999,"15min":162.70699999999999},"last":0.34799999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.442,"15min":1.552},"min":{"1min":0.251,"5min":0.24199999999999999,"15min":0.20799999999999999},"max":{"1min":0.46000000000000002,"5min":2.9049999999999998,"15min":14.254},"last":0.505},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.499,"15min":1.4350000000000001},"min":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":2.5230000000000001,"5min":4.5049999999999999,"15min":15.523},"last":0.29399999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.53400000000000003,"15min":1.653},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.26200000000000001},"max":{"1min":4.5010000000000003,"5min":4.5010000000000003,"15min":77.793000000000006},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.499,"5min":0.49399999999999999,"15min":1.8120000000000001},"min":{"1min":0.35299999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":0.72299999999999998,"5min":1.5609999999999999,"15min":76.646000000000001},"last":4.8390000000000004}]},{"osd":5,"last update":"Sun Oct 1 09:47:37 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.67200000000000004,"15min":1.7150000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27500000000000002},"max":{"1min":3.1459999999999999,"5min":4.3099999999999996,"15min":28.117999999999999},"last":1.6000000000000001},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.502,"15min":1.8140000000000001},"min":{"1min":0.34300000000000003,"5min":0.33600000000000002,"15min":0.27700000000000002},"max":{"1min":0.56899999999999995,"5min":2.512,"15min":28.34},"last":0.48099999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:47:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54500000000000004,"15min":1.758},"min":{"1min":0.45500000000000002,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":1.0549999999999999,"5min":2.9100000000000001,"15min":73.887},"last":1.452},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.65000000000000002,"15min":1.869},"min":{"1min":0.38100000000000001,"5min":0.35499999999999998,"15min":0.33600000000000002},"max":{"1min":3.1099999999999999,"5min":3.1099999999999999,"15min":74.016999999999996},"last":1.7649999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":0.85999999999999999,"15min":1.7729999999999999},"min":{"1min":0.42799999999999999,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":4.1989999999999998,"5min":4.1989999999999998,"15min":13.085000000000001},"last":0.77900000000000003},{"interface":"front","average":{"1min":1.2030000000000001,"5min":0.80500000000000005,"15min":1.5529999999999999},"min":{"1min":0.432,"5min":0.38800000000000001,"15min":0.311},"max":{"1min":4.3280000000000003,"5min":4.3280000000000003,"15min":13.191000000000001},"last":0.79000000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019449,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2335892,"kb_used_data":1437516,"kb_used_omap":0,"kb_used_meta":898368,"kb_avail":91409260,"statfs":{"total":95995035648,"available":93603082240,"internally_reserved":0,"allocated":1472016384,"data_stored":1471073438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919928832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7480000000000002,"5min":0.89400000000000002,"15min":3.5819999999999999},"min":{"1min":0.24399999999999999,"5min":0.22500000000000001,"15min":0.21199999999999999},"max":{"1min":41.963999999999999,"5min":41.963999999999999,"15min":155.315},"last":0.223},{"interface":"front","average":{"1min":2.7530000000000001,"5min":0.90100000000000002,"15min":3.9140000000000001},"min":{"1min":0.26100000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":41.890999999999998,"5min":41.890999999999998,"15min":155.334},"last":0.83099999999999996}]},{"osd":1,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.54000000000000004,"15min":5.1449999999999996},"min":{"1min":0.25600000000000001,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":3.3929999999999998,"5min":3.3929999999999998,"15min":190.61199999999999},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.54900000000000004,"15min":5.8639999999999999},"min":{"1min":0.29199999999999998,"5min":0.27500000000000002,"15min":0.25600000000000001},"max":{"1min":3.3839999999999999,"5min":3.3839999999999999,"15min":253.273},"last":0.255}]},{"osd":2,"last update":"Sun Oct 1 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.621,"15min":2.0499999999999998},"min":{"1min":0.30499999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.343,"5min":4.2329999999999997,"15min":170.45400000000001},"last":0.93200000000000005},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.61499999999999999,"15min":1.8069999999999999},"min":{"1min":0.35399999999999998,"5min":0.314,"15min":0.309},"max":{"1min":3.1389999999999998,"5min":3.1389999999999998,"15min":36.515999999999998},"last":0.91300000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.61199999999999999,"15min":1.9850000000000001},"min":{"1min":0.307,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":3.1539999999999999,"5min":3.1539999999999999,"15min":140.881},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.61399999999999999,"15min":2.0579999999999998},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.311},"max":{"1min":1.3049999999999999,"5min":6.2969999999999997,"15min":140.911},"last":0.29199999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:46:41 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.56999999999999995,"15min":1.454},"min":{"1min":0.27200000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":1.8500000000000001,"5min":3.052,"15min":110.003},"last":0.95299999999999996},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.63400000000000001,"15min":1.3859999999999999},"min":{"1min":0.34100000000000003,"5min":0.317,"15min":0.28899999999999998},"max":{"1min":1.754,"5min":3.2909999999999999,"15min":108.56399999999999},"last":0.90200000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:47:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.56499999999999995,"15min":0.91700000000000004},"min":{"1min":0.318,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":3.3029999999999999,"5min":3.3029999999999999,"15min":11.358000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.60999999999999999,"15min":1.046},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.28999999999999998},"max":{"1min":2.2669999999999999,"5min":2.9239999999999999,"15min":10.076000000000001},"last":0.89100000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.63400000000000001,"15min":1.149},"min":{"1min":0.39200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.3079999999999998,"5min":3.3079999999999998,"15min":38.034999999999997},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.70899999999999996,"15min":1.3839999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.308},"max":{"1min":3.0329999999999999,"5min":3.113,"15min":37.723999999999997},"last":1.7709999999999999}]}]},{"osd":5,"up_from":379,"seq":1627792605356,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1061512,"kb_used_data":696960,"kb_used_omap":13,"kb_used_meta":364530,"kb_avail":92683640,"statfs":{"total":95995035648,"available":94908047360,"internally_reserved":0,"allocated":713687040,"data_stored":712759674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":373279218},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6,0,1,1,0,0,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":4.9749999999999996,"5min":1.6719999999999999,"15min":2.3140000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.21099999999999999},"max":{"1min":99.609999999999999,"5min":99.609999999999999,"15min":99.609999999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":5.117,"5min":1.7130000000000001,"15min":2.2959999999999998},"min":{"1min":0.215,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":98.995999999999995,"5min":98.995999999999995,"15min":98.995999999999995},"last":2.21}]},{"osd":1,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":5.4790000000000001},"min":{"1min":0.28399999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.157,"5min":4.3289999999999997,"15min":295.71300000000002},"last":0.374},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.56699999999999995,"15min":5.5129999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":3.665,"5min":3.9580000000000002,"15min":299.59399999999999},"last":0.30299999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.60399999999999998,"15min":1.4179999999999999},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.9129999999999998,"5min":4.2670000000000003,"15min":14.199999999999999},"last":3.0760000000000001},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.70899999999999996,"15min":1.357},"min":{"1min":0.31900000000000001,"5min":0.29599999999999999,"15min":0.27100000000000002},"max":{"1min":4.1909999999999998,"5min":4.3719999999999999,"15min":14.430999999999999},"last":0.41599999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":0.70599999999999996,"15min":1.3759999999999999},"min":{"1min":0.30599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.141,"5min":7.141,"15min":14.004},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.66600000000000004,"15min":1.5029999999999999},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":3.1139999999999999,"5min":4.2910000000000004,"15min":13.991},"last":3.1160000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.66400000000000003,"15min":1.375},"min":{"1min":0.27900000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.0659999999999998,"5min":4.1929999999999996,"15min":122.971},"last":6.7560000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.66500000000000004,"15min":1.472},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2469999999999999,"5min":4.3869999999999996,"15min":122.89100000000001},"last":0.53500000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.63800000000000001,"15min":0.94599999999999995},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28299999999999997},"max":{"1min":3.5750000000000002,"5min":4.3300000000000001,"15min":12.545},"last":0.64800000000000002},{"interface":"front","average":{"1min":1.097,"5min":0.76300000000000001,"15min":1.119},"min":{"1min":0.31,"5min":0.31,"15min":0.30499999999999999},"max":{"1min":4.484,"5min":4.484,"15min":14.193},"last":3.1379999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.74399999999999999,"15min":1.1120000000000001},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.2199999999999998,"5min":4.3559999999999999,"15min":10.813000000000001},"last":0.83199999999999996},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.75600000000000001,"15min":1.0509999999999999},"min":{"1min":0.40799999999999997,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":3.609,"5min":4.2000000000000002,"15min":12.500999999999999},"last":0.81000000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1682640896,"data_stored":1682636824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":310706176,"data_stored":310706176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":308674560,"data_stored":308674560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1458507776,"data_stored":1458503704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":691535872,"data_stored":691535872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":696455168,"data_stored":696451096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":504168448,"data_stored":504168448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:47:41.989 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:42.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:42.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:42.071 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:42.107 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:42.228 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:42.233 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:47:42.384 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:42.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:42.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:42.497 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:42.561 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:42.691 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:42.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:42.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:42.799 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:42.886 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:43.001 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:43.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:43.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:43.065 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:47:43.112 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:47:43.144 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:43.445 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:47:43.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:43.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:47:43.524 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:47:43.551 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:43.684 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:43 smithi190 ceph-mon[160435]: pgmap v1665: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 4786/67872 objects degraded (7.052%); 1404/67872 objects misplaced (2.069%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:47:43.684 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:43 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1143978907' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:43.684 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:43 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 4786/67872 objects degraded (7.052%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:43.694 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:43.757 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:43 smithi138 ceph-mon[192513]: pgmap v1665: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 4786/67872 objects degraded (7.052%); 1404/67872 objects misplaced (2.069%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:47:43.758 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:43 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1143978907' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:43.758 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:43 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 4786/67872 objects degraded (7.052%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:43.758 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:43 smithi138 ceph-mon[185886]: pgmap v1665: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 4786/67872 objects degraded (7.052%); 1404/67872 objects misplaced (2.069%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:47:43.758 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:43 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1143978907' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:43.758 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:43 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 4786/67872 objects degraded (7.052%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:44.004 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:44.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:44.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:44.109 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:44.144 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:44.472 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:44.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:44.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:44.540 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:47:44.570 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:47:44.582 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:47:44.600 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:47:44.632 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:47:44.637 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:44.691 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:47:44.697 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:47:44.699 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:44 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:44.699 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:44 smithi138 ceph-mon[192513]: failed to encode map e555 with expected crc 2023-10-01T09:47:44.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:44 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1143978907' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:47:44.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:44 smithi138 ceph-mon[192513]: osdmap e555: 8 total, 8 up, 7 in 2023-10-01T09:47:44.700 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:44 smithi138 ceph-mon[192513]: failed to encode map e555 with expected crc 2023-10-01T09:47:44.700 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:44 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:44.700 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:44 smithi138 ceph-mon[185886]: failed to encode map e555 with expected crc 2023-10-01T09:47:44.700 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:44 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1143978907' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:47:44.700 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:44 smithi138 ceph-mon[185886]: osdmap e555: 8 total, 8 up, 7 in 2023-10-01T09:47:44.701 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:44 smithi138 ceph-mon[185886]: failed to encode map e555 with expected crc 2023-10-01T09:47:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:44.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:47:44.721 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:47:44.758 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:47:44.802 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:44.804 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:47:44.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:44 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:44.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:44 smithi190 ceph-mon[160435]: failed to encode map e555 with expected crc 2023-10-01T09:47:44.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:44 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1143978907' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:47:44.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:44 smithi190 ceph-mon[160435]: osdmap e555: 8 total, 8 up, 7 in 2023-10-01T09:47:44.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:44 smithi190 ceph-mon[160435]: failed to encode map e555 with expected crc 2023-10-01T09:47:44.820 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:47:44.820 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:47:44.820 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:47:44.820 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:47:44.820 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:47:44.820 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:47:44.820 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:47:44.820 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:47:44.887 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:44.891 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:47:44.996 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:47:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:47:44] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-01T09:47:45.128 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:45.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:45.238 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:45.282 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:45.496 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:45 smithi190 ceph-mon[160435]: pgmap v1667: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 4868/64899 objects degraded (7.501%); 1217/64899 objects misplaced (1.875%); 1.9 MiB/s, 29 objects/s recovering 2023-10-01T09:47:45.496 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:45 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/452591476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:45.553 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:45 smithi138 ceph-mon[192513]: pgmap v1667: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 4868/64899 objects degraded (7.501%); 1217/64899 objects misplaced (1.875%); 1.9 MiB/s, 29 objects/s recovering 2023-10-01T09:47:45.553 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:45 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/452591476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:45.556 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:45 smithi138 ceph-mon[185886]: pgmap v1667: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 4868/64899 objects degraded (7.501%); 1217/64899 objects misplaced (1.875%); 1.9 MiB/s, 29 objects/s recovering 2023-10-01T09:47:45.557 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:45 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/452591476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:45.631 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:45.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:45.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:45.688 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:45.737 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:45.783 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:45.845 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:47:45.845 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:47:45.845 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.845 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.36456.0:147487 10.7 e555/503 10:e5b77bb7:::benchmark_data_smithi138_573643_object64265:head v 555'9265, mlcod=497'3704)", 2023-10-01T09:47:45.845 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:47:45.823112+0000", 2023-10-01T09:47:45.845 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.018045491, 2023-10-01T09:47:45.846 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.018076390000000001, 2023-10-01T09:47:45.846 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:47:45.846 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:47:45.846 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:47:45.846 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.846 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:47:45.846 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.823112+0000", 2023-10-01T09:47:45.846 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:47:45.846 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.846 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.846 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:47:45.847 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.823112+0000", 2023-10-01T09:47:45.847 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:47:45.847 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.847 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.847 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:47:45.847 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.823111+0000", 2023-10-01T09:47:45.847 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0443e-05 2023-10-01T09:47:45.847 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.847 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.847 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:47:45.847 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.823122+0000", 2023-10-01T09:47:45.847 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.6866000000000001e-05 2023-10-01T09:47:45.847 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.848 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:47:45.848 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.823138+0000", 2023-10-01T09:47:45.848 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.3599999999999998e-06 2023-10-01T09:47:45.848 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.848 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:47:45.848 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.823146+0000", 2023-10-01T09:47:45.848 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6285000000000006e-05 2023-10-01T09:47:45.848 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.848 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.848 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:47:45.849 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.823222+0000", 2023-10-01T09:47:45.849 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.9495000000000001e-05 2023-10-01T09:47:45.849 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.849 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.849 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:47:45.849 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.823252+0000", 2023-10-01T09:47:45.849 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013948000000000001 2023-10-01T09:47:45.849 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:47:45.849 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:47:45.849 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:47:45.849 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.849 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.850 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.36456.0:147495 10.f e555/479 10:f082c605:::benchmark_data_smithi138_573643_object64273:head v 555'9295, mlcod=555'9293)", 2023-10-01T09:47:45.850 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:47:45.830357+0000", 2023-10-01T09:47:45.850 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.01080098, 2023-10-01T09:47:45.850 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010908597000000001, 2023-10-01T09:47:45.850 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:47:45.850 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:47:45.850 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:47:45.850 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.850 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:47:45.850 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.830357+0000", 2023-10-01T09:47:45.850 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:47:45.850 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.851 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.851 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:47:45.851 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.830357+0000", 2023-10-01T09:47:45.851 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:47:45.851 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.851 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.851 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:47:45.851 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.830355+0000", 2023-10-01T09:47:45.851 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2330000000000001e-05 2023-10-01T09:47:45.851 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.851 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.851 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:47:45.852 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.830378+0000", 2023-10-01T09:47:45.852 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.0996e-05 2023-10-01T09:47:45.852 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.852 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.852 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:47:45.852 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.830399+0000", 2023-10-01T09:47:45.852 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.1510000000000004e-06 2023-10-01T09:47:45.852 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.852 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.852 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:47:45.852 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.830407+0000", 2023-10-01T09:47:45.852 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0079769679999999992 2023-10-01T09:47:45.853 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.853 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.853 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:47:45.853 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.838384+0000", 2023-10-01T09:47:45.853 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.2056e-05 2023-10-01T09:47:45.853 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.853 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.853 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:47:45.853 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.838406+0000", 2023-10-01T09:47:45.853 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0080491739999999992 2023-10-01T09:47:45.853 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:47:45.853 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:47:45.854 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:47:45.854 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.854 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.854 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.36456.0:147492 10.7 e555/503)", 2023-10-01T09:47:45.854 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:47:45.833949+0000", 2023-10-01T09:47:45.854 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0072083340000000003, 2023-10-01T09:47:45.854 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0073839200000000004, 2023-10-01T09:47:45.854 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:47:45.854 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:47:45.854 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:47:45.854 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.855 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:47:45.855 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.833949+0000", 2023-10-01T09:47:45.855 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:47:45.855 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.855 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.855 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:47:45.855 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.833949+0000", 2023-10-01T09:47:45.855 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:47:45.855 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.855 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.855 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:47:45.855 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.833948+0000", 2023-10-01T09:47:45.856 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0118999999999999e-05 2023-10-01T09:47:45.856 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.856 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.856 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:47:45.856 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.833959+0000", 2023-10-01T09:47:45.856 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2517999999999999e-05 2023-10-01T09:47:45.856 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.856 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.856 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:47:45.856 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.833971+0000", 2023-10-01T09:47:45.856 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.3260000000000001e-06 2023-10-01T09:47:45.856 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.857 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.857 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:47:45.857 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.833977+0000", 2023-10-01T09:47:45.857 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8098000000000001e-05 2023-10-01T09:47:45.857 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:47:45.857 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:47:45.857 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:47:45.857 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.857 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.857 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop(client.36456.0:147498 10.7 e555/503)", 2023-10-01T09:47:45.857 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:47:45.841098+0000", 2023-10-01T09:47:45.857 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 5.9302000000000002e-05, 2023-10-01T09:47:45.858 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00029278600000000001, 2023-10-01T09:47:45.858 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:47:45.858 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:47:45.858 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:47:45.858 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.858 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:47:45.858 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.841098+0000", 2023-10-01T09:47:45.858 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:47:45.858 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.858 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.858 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:47:45.858 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.841098+0000", 2023-10-01T09:47:45.859 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.999999 2023-10-01T09:47:45.859 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.859 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.859 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:47:45.859 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.841097+0000", 2023-10-01T09:47:45.859 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.7650000000000005e-06 2023-10-01T09:47:45.859 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.859 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.859 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:47:45.859 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.841107+0000", 2023-10-01T09:47:45.859 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2337999999999999e-05 2023-10-01T09:47:45.859 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.860 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.860 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:47:45.860 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.841119+0000", 2023-10-01T09:47:45.860 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1019999999999999e-06 2023-10-01T09:47:45.860 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:47:45.860 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:47:45.860 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:47:45.860 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:47:45.841126+0000", 2023-10-01T09:47:45.860 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.8045e-05 2023-10-01T09:47:45.860 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:47:45.860 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:47:45.860 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:47:45.861 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:47:45.861 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:47:45.861 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 4 2023-10-01T09:47:45.861 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:47:45.861 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:45.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:45.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:45.966 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:46.015 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:46.079 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:46.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:46.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:46.183 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:47:46.286 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:46.397 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:47:46.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:46.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:47:46.435 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:47:46.501 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:46.529 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:46.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:46 smithi190 ceph-mon[160435]: pgmap v1668: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 4868/60756 objects degraded (8.012%); 1217/60756 objects misplaced (2.003%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T09:47:46.607 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:46.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:46.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:46.712 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:46.746 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:46 smithi138 ceph-mon[192513]: pgmap v1668: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 4868/60756 objects degraded (8.012%); 1217/60756 objects misplaced (2.003%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T09:47:46.751 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:46 smithi138 ceph-mon[185886]: pgmap v1668: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 4868/60756 objects degraded (8.012%); 1217/60756 objects misplaced (2.003%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T09:47:46.752 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:46.899 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:46.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:46.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:47.005 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:47.032 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:47:47.045 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:47.122 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:47.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:47.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:47.138 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:47.226 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:47.340 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:47.522 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:47.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:47.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:47.579 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:47:47.637 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:47.716 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:47.786 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:47:47.805 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:47:47.820 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:47.823 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:47:47.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:47.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:47.929 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:47:47.962 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:48.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:47 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 4868/60756 objects degraded (8.012%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:48.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:47 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 4868/60756 objects degraded (8.012%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:48.178 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:47 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 4868/60756 objects degraded (8.012%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:48.813 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:47:48.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:48.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:47:48.910 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:48 smithi190 ceph-mon[160435]: from='client.47474 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:48.910 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:48 smithi190 ceph-mon[160435]: pgmap v1669: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 4939/55233 objects degraded (8.942%); 1032/55233 objects misplaced (1.868%); 1.4 MiB/s, 21 objects/s recovering 2023-10-01T09:47:48.910 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:48 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:47:48.926 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:47:48.983 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1668,"stamp":"2023-10-01T09:47:46.381777+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1330124747,"num_objects":20252,"num_object_clones":0,"num_object_copies":60756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4868,"num_objects_misplaced":1217,"num_objects_unfound":0,"num_objects_dirty":20252,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":147322,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11585,"num_bytes_recovered":746095631,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126786,"ondisk_log_size":126786,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9954376,"kb_used_data":4849296,"kb_used_omap":39,"kb_used_meta":5104920,"kb_avail":646261688,"statfs":{"total":671965249536,"available":661771968512,"internally_reserved":0,"allocated":4965679104,"data_stored":4955639705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5227438703},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-345309184,"num_objects":-5269,"num_object_clones":0,"num_object_copies":-15807,"num_objects_missing_on_primary":-74,"num_objects_missing":-74,"num_objects_degraded":-27,"num_objects_misplaced":-287,"num_objects_unfound":0,"num_objects_dirty":-5269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5269,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":240,"num_bytes_recovered":15728640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.939163"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":331,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.448671+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:47:36.448671+0000","last_peered":"2023-10-01T09:47:36.448671+0000","last_clean":"2023-10-01T09:47:36.448671+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:47:36.448671+0000","last_undegraded":"2023-10-01T09:47:36.448671+0000","last_fullsized":"2023-10-01T09:47:36.448671+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":176,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.515779+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:47:43.515779+0000","last_peered":"2023-10-01T09:47:43.515779+0000","last_clean":"2023-10-01T09:47:43.515779+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:47:43.515779+0000","last_undegraded":"2023-10-01T09:47:43.515779+0000","last_fullsized":"2023-10-01T09:47:43.515779+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":556,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.444002+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:47:36.444002+0000","last_peered":"2023-10-01T09:47:36.444002+0000","last_clean":"2023-10-01T09:47:36.444002+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:47:36.444002+0000","last_undegraded":"2023-10-01T09:47:36.444002+0000","last_fullsized":"2023-10-01T09:47:36.444002+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":614,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912300+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:47:28.912300+0000","last_peered":"2023-10-01T09:47:28.912300+0000","last_clean":"2023-10-01T09:47:28.912300+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:47:28.912300+0000","last_undegraded":"2023-10-01T09:47:28.912300+0000","last_fullsized":"2023-10-01T09:47:28.912300+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":495,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.450150+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:47:36.450150+0000","last_peered":"2023-10-01T09:47:36.450150+0000","last_clean":"2023-10-01T09:47:36.450150+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:47:36.450150+0000","last_undegraded":"2023-10-01T09:47:36.450150+0000","last_fullsized":"2023-10-01T09:47:36.450150+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":235,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.994903+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:47:28.994903+0000","last_peered":"2023-10-01T09:47:28.994903+0000","last_clean":"2023-10-01T09:47:28.994903+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:47:28.994903+0000","last_undegraded":"2023-10-01T09:47:28.994903+0000","last_fullsized":"2023-10-01T09:47:28.994903+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":611,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.994969+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:47:28.994969+0000","last_peered":"2023-10-01T09:47:28.994969+0000","last_clean":"2023-10-01T09:47:28.994969+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:47:28.994969+0000","last_undegraded":"2023-10-01T09:47:28.994969+0000","last_fullsized":"2023-10-01T09:47:28.994969+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":542,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.906733+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:47:28.906733+0000","last_peered":"2023-10-01T09:47:28.906733+0000","last_clean":"2023-10-01T09:47:28.906733+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:47:28.906733+0000","last_undegraded":"2023-10-01T09:47:28.906733+0000","last_fullsized":"2023-10-01T09:47:28.906733+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":266,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.918624+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:47:28.918624+0000","last_peered":"2023-10-01T09:47:28.918624+0000","last_clean":"2023-10-01T09:47:28.918624+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:47:28.918624+0000","last_undegraded":"2023-10-01T09:47:28.918624+0000","last_fullsized":"2023-10-01T09:47:28.918624+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":74,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.455820+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:47:36.455820+0000","last_peered":"2023-10-01T09:47:36.455820+0000","last_clean":"2023-10-01T09:47:36.455820+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:47:36.455820+0000","last_undegraded":"2023-10-01T09:47:36.455820+0000","last_fullsized":"2023-10-01T09:47:36.455820+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":80,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.456488+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:47:36.456488+0000","last_peered":"2023-10-01T09:47:36.456488+0000","last_clean":"2023-10-01T09:47:36.456488+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:47:36.456488+0000","last_undegraded":"2023-10-01T09:47:36.456488+0000","last_fullsized":"2023-10-01T09:47:36.456488+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":80,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.515254+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:47:43.515254+0000","last_peered":"2023-10-01T09:47:43.515254+0000","last_clean":"2023-10-01T09:47:43.515254+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:47:43.515254+0000","last_undegraded":"2023-10-01T09:47:43.515254+0000","last_fullsized":"2023-10-01T09:47:43.515254+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":456,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.449822+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:47:36.449822+0000","last_peered":"2023-10-01T09:47:36.449822+0000","last_clean":"2023-10-01T09:47:36.449822+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:47:36.449822+0000","last_undegraded":"2023-10-01T09:47:36.449822+0000","last_fullsized":"2023-10-01T09:47:36.449822+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":337,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.448768+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:47:36.448768+0000","last_peered":"2023-10-01T09:47:36.448768+0000","last_clean":"2023-10-01T09:47:36.448768+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:47:36.448768+0000","last_undegraded":"2023-10-01T09:47:36.448768+0000","last_fullsized":"2023-10-01T09:47:36.448768+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":673,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.455796+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:47:36.455796+0000","last_peered":"2023-10-01T09:47:36.455796+0000","last_clean":"2023-10-01T09:47:36.455796+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:47:36.455796+0000","last_undegraded":"2023-10-01T09:47:36.455796+0000","last_fullsized":"2023-10-01T09:47:36.455796+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":410,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.457067+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:47:36.457067+0000","last_peered":"2023-10-01T09:47:36.457067+0000","last_clean":"2023-10-01T09:47:36.457067+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:47:36.457067+0000","last_undegraded":"2023-10-01T09:47:36.457067+0000","last_fullsized":"2023-10-01T09:47:36.457067+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":83,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.455648+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:47:36.455648+0000","last_peered":"2023-10-01T09:47:36.455648+0000","last_clean":"2023-10-01T09:47:36.455648+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:47:36.455648+0000","last_undegraded":"2023-10-01T09:47:36.455648+0000","last_fullsized":"2023-10-01T09:47:36.455648+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.916010+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:47:28.916010+0000","last_peered":"2023-10-01T09:47:28.916010+0000","last_clean":"2023-10-01T09:47:28.916010+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:47:28.916010+0000","last_undegraded":"2023-10-01T09:47:28.916010+0000","last_fullsized":"2023-10-01T09:47:28.916010+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":566,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.458161+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:47:36.458161+0000","last_peered":"2023-10-01T09:47:36.458161+0000","last_clean":"2023-10-01T09:47:36.458161+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:47:36.458161+0000","last_undegraded":"2023-10-01T09:47:36.458161+0000","last_fullsized":"2023-10-01T09:47:36.458161+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":578,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912302+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:47:28.912302+0000","last_peered":"2023-10-01T09:47:28.912302+0000","last_clean":"2023-10-01T09:47:28.912302+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:47:28.912302+0000","last_undegraded":"2023-10-01T09:47:28.912302+0000","last_fullsized":"2023-10-01T09:47:28.912302+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":434,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.911877+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:47:28.911877+0000","last_peered":"2023-10-01T09:47:28.911877+0000","last_clean":"2023-10-01T09:47:28.911877+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:47:28.911877+0000","last_undegraded":"2023-10-01T09:47:28.911877+0000","last_fullsized":"2023-10-01T09:47:28.911877+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.456179+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:47:36.456179+0000","last_peered":"2023-10-01T09:47:36.456179+0000","last_clean":"2023-10-01T09:47:36.456179+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:47:36.456179+0000","last_undegraded":"2023-10-01T09:47:36.456179+0000","last_fullsized":"2023-10-01T09:47:36.456179+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"554'9112","reported_seq":9199,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:42.681998+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:47:42.681998+0000","last_peered":"2023-10-01T09:47:42.681998+0000","last_clean":"2023-10-01T09:47:42.681998+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:47:42.681998+0000","last_undegraded":"2023-10-01T09:47:42.681998+0000","last_fullsized":"2023-10-01T09:47:42.681998+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":9112,"log_dups_size":0,"ondisk_log_size":9112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83886080,"num_objects":1280,"num_object_clones":0,"num_object_copies":3840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9112,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":78,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.456040+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:47:36.456040+0000","last_peered":"2023-10-01T09:47:36.456040+0000","last_clean":"2023-10-01T09:47:36.456040+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:47:36.456040+0000","last_undegraded":"2023-10-01T09:47:36.456040+0000","last_fullsized":"2023-10-01T09:47:36.456040+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":274,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915813+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:47:28.915813+0000","last_peered":"2023-10-01T09:47:28.915813+0000","last_clean":"2023-10-01T09:47:28.915813+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:47:28.915813+0000","last_undegraded":"2023-10-01T09:47:28.915813+0000","last_fullsized":"2023-10-01T09:47:28.915813+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"554'9140","reported_seq":20337,"reported_epoch":554,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:47:42.701393+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:47:42.701393+0000","last_peered":"2023-10-01T09:47:42.701393+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:47:42.701393+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":9140,"log_dups_size":0,"ondisk_log_size":9140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82968576,"num_objects":1266,"num_object_clones":0,"num_object_copies":3798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1437,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9140,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3698,"num_bytes_recovered":242352128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":1198},{"shards":"5,6","objects":-171}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":549,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.458480+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:47:36.458480+0000","last_peered":"2023-10-01T09:47:36.458480+0000","last_clean":"2023-10-01T09:47:36.458480+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:47:36.458480+0000","last_undegraded":"2023-10-01T09:47:36.458480+0000","last_fullsized":"2023-10-01T09:47:36.458480+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":612,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.443919+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:47:36.443919+0000","last_peered":"2023-10-01T09:47:36.443919+0000","last_clean":"2023-10-01T09:47:36.443919+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:47:36.443919+0000","last_undegraded":"2023-10-01T09:47:36.443919+0000","last_fullsized":"2023-10-01T09:47:36.443919+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"554'9051","reported_seq":9142,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:41.837895+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:47:41.837895+0000","last_peered":"2023-10-01T09:47:41.837895+0000","last_clean":"2023-10-01T09:47:41.837895+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:47:41.837895+0000","last_undegraded":"2023-10-01T09:47:41.837895+0000","last_fullsized":"2023-10-01T09:47:41.837895+0000","mapping_epoch":475,"log_start":"484'2000","ondisk_log_start":"484'2000","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7051,"log_dups_size":0,"ondisk_log_size":7051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90505216,"num_objects":1381,"num_object_clones":0,"num_object_copies":4143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9051,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":635,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.456568+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:47:36.456568+0000","last_peered":"2023-10-01T09:47:36.456568+0000","last_clean":"2023-10-01T09:47:36.456568+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:47:36.456568+0000","last_undegraded":"2023-10-01T09:47:36.456568+0000","last_fullsized":"2023-10-01T09:47:36.456568+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":432,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.444293+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:47:36.444293+0000","last_peered":"2023-10-01T09:47:36.444293+0000","last_clean":"2023-10-01T09:47:36.444293+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:47:36.444293+0000","last_undegraded":"2023-10-01T09:47:36.444293+0000","last_fullsized":"2023-10-01T09:47:36.444293+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"554'9084","reported_seq":15780,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:41.920282+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:47:41.920282+0000","last_peered":"2023-10-01T09:47:41.920282+0000","last_clean":"2023-10-01T09:47:41.920282+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:47:41.920282+0000","last_undegraded":"2023-10-01T09:47:41.920282+0000","last_fullsized":"2023-10-01T09:47:41.920282+0000","mapping_epoch":505,"log_start":"485'2100","ondisk_log_start":"485'2100","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86376448,"num_objects":1318,"num_object_clones":0,"num_object_copies":3954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9084,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":274,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.919741+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:47:28.919741+0000","last_peered":"2023-10-01T09:47:28.919741+0000","last_clean":"2023-10-01T09:47:28.919741+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:47:28.919741+0000","last_undegraded":"2023-10-01T09:47:28.919741+0000","last_fullsized":"2023-10-01T09:47:28.919741+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":534,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912024+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:47:28.912024+0000","last_peered":"2023-10-01T09:47:28.912024+0000","last_clean":"2023-10-01T09:47:28.912024+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:47:28.912024+0000","last_undegraded":"2023-10-01T09:47:28.912024+0000","last_fullsized":"2023-10-01T09:47:28.912024+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"555'9248","reported_seq":9340,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:45.268886+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:47:45.268886+0000","last_peered":"2023-10-01T09:47:45.268886+0000","last_clean":"2023-10-01T09:47:45.268886+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:47:45.268886+0000","last_undegraded":"2023-10-01T09:47:45.268886+0000","last_fullsized":"2023-10-01T09:47:45.268886+0000","mapping_epoch":475,"log_start":"484'1700","ondisk_log_start":"484'1700","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7548,"log_dups_size":0,"ondisk_log_size":7548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82575360,"num_objects":1260,"num_object_clones":0,"num_object_copies":3780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9248,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":135,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.449051+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:47:36.449051+0000","last_peered":"2023-10-01T09:47:36.449051+0000","last_clean":"2023-10-01T09:47:36.449051+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:47:36.449051+0000","last_undegraded":"2023-10-01T09:47:36.449051+0000","last_fullsized":"2023-10-01T09:47:36.449051+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":209,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.449943+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:47:36.449943+0000","last_peered":"2023-10-01T09:47:36.449943+0000","last_clean":"2023-10-01T09:47:36.449943+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:47:36.449943+0000","last_undegraded":"2023-10-01T09:47:36.449943+0000","last_fullsized":"2023-10-01T09:47:36.449943+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"554'9087","reported_seq":10148,"reported_epoch":554,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:47:42.702835+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:47:42.702835+0000","last_peered":"2023-10-01T09:47:42.702835+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:47:42.702835+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":9087,"log_dups_size":0,"ondisk_log_size":9087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83951616,"num_objects":1281,"num_object_clones":0,"num_object_copies":3843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3421,"num_objects_misplaced":1217,"num_objects_unfound":0,"num_objects_dirty":1281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9087,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":483,"num_bytes_recovered":31653888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1217},{"shards":"4,5","objects":-3357}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":453,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.448751+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:47:36.448751+0000","last_peered":"2023-10-01T09:47:36.448751+0000","last_clean":"2023-10-01T09:47:36.448751+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:47:36.448751+0000","last_undegraded":"2023-10-01T09:47:36.448751+0000","last_fullsized":"2023-10-01T09:47:36.448751+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":615,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.444466+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:47:36.444466+0000","last_peered":"2023-10-01T09:47:36.444466+0000","last_clean":"2023-10-01T09:47:36.444466+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:47:36.444466+0000","last_undegraded":"2023-10-01T09:47:36.444466+0000","last_fullsized":"2023-10-01T09:47:36.444466+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"554'9090","reported_seq":9186,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:42.584009+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:47:42.584009+0000","last_peered":"2023-10-01T09:47:42.584009+0000","last_clean":"2023-10-01T09:47:42.584009+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:47:42.584009+0000","last_undegraded":"2023-10-01T09:47:42.584009+0000","last_fullsized":"2023-10-01T09:47:42.584009+0000","mapping_epoch":483,"log_start":"482'1500","ondisk_log_start":"482'1500","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7590,"log_dups_size":0,"ondisk_log_size":7590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84410368,"num_objects":1288,"num_object_clones":0,"num_object_copies":3864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9090,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":449,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.450745+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:47:36.450745+0000","last_peered":"2023-10-01T09:47:36.450745+0000","last_clean":"2023-10-01T09:47:36.450745+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:47:36.450745+0000","last_undegraded":"2023-10-01T09:47:36.450745+0000","last_fullsized":"2023-10-01T09:47:36.450745+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":290,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.449622+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:47:36.449622+0000","last_peered":"2023-10-01T09:47:36.449622+0000","last_clean":"2023-10-01T09:47:36.449622+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:47:36.449622+0000","last_undegraded":"2023-10-01T09:47:36.449622+0000","last_fullsized":"2023-10-01T09:47:36.449622+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"555'9362","reported_seq":9450,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:46.225751+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:47:46.225751+0000","last_peered":"2023-10-01T09:47:46.225751+0000","last_clean":"2023-10-01T09:47:46.225751+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:47:46.225751+0000","last_undegraded":"2023-10-01T09:47:46.225751+0000","last_fullsized":"2023-10-01T09:47:46.225751+0000","mapping_epoch":475,"log_start":"481'1000","ondisk_log_start":"481'1000","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8362,"log_dups_size":0,"ondisk_log_size":8362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74448896,"num_objects":1136,"num_object_clones":0,"num_object_copies":3408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9362,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":419,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.443919+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:47:36.443919+0000","last_peered":"2023-10-01T09:47:36.443919+0000","last_clean":"2023-10-01T09:47:36.443919+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:47:36.443919+0000","last_undegraded":"2023-10-01T09:47:36.443919+0000","last_fullsized":"2023-10-01T09:47:36.443919+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":466,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.444968+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:47:36.444968+0000","last_peered":"2023-10-01T09:47:36.444968+0000","last_clean":"2023-10-01T09:47:36.444968+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:47:36.444968+0000","last_undegraded":"2023-10-01T09:47:36.444968+0000","last_fullsized":"2023-10-01T09:47:36.444968+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"555'9060","reported_seq":9148,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:46.338038+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:47:46.338038+0000","last_peered":"2023-10-01T09:47:46.338038+0000","last_clean":"2023-10-01T09:47:46.338038+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:47:46.338038+0000","last_undegraded":"2023-10-01T09:47:46.338038+0000","last_fullsized":"2023-10-01T09:47:46.338038+0000","mapping_epoch":475,"log_start":"481'700","ondisk_log_start":"481'700","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8360,"log_dups_size":0,"ondisk_log_size":8360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73269248,"num_objects":1118,"num_object_clones":0,"num_object_copies":3354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9060,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":948,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.456936+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:47:36.456936+0000","last_peered":"2023-10-01T09:47:36.456936+0000","last_clean":"2023-10-01T09:47:36.456936+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:47:36.456936+0000","last_undegraded":"2023-10-01T09:47:36.456936+0000","last_fullsized":"2023-10-01T09:47:36.456936+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":284,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915884+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:47:28.915884+0000","last_peered":"2023-10-01T09:47:28.915884+0000","last_clean":"2023-10-01T09:47:28.915884+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:47:28.915884+0000","last_undegraded":"2023-10-01T09:47:28.915884+0000","last_fullsized":"2023-10-01T09:47:28.915884+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":80,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.444171+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:47:36.444171+0000","last_peered":"2023-10-01T09:47:36.444171+0000","last_clean":"2023-10-01T09:47:36.444171+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:47:36.444171+0000","last_undegraded":"2023-10-01T09:47:36.444171+0000","last_fullsized":"2023-10-01T09:47:36.444171+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"554'8992","reported_seq":9083,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:41.900607+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:47:41.900607+0000","last_peered":"2023-10-01T09:47:41.900607+0000","last_clean":"2023-10-01T09:47:41.900607+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:47:41.900607+0000","last_undegraded":"2023-10-01T09:47:41.900607+0000","last_fullsized":"2023-10-01T09:47:41.900607+0000","mapping_epoch":475,"log_start":"485'2000","ondisk_log_start":"485'2000","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90308608,"num_objects":1378,"num_object_clones":0,"num_object_copies":4134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8992,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1219,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.916160+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:47:28.916160+0000","last_peered":"2023-10-01T09:47:28.916160+0000","last_clean":"2023-10-01T09:47:28.916160+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:47:28.916160+0000","last_undegraded":"2023-10-01T09:47:28.916160+0000","last_fullsized":"2023-10-01T09:47:28.916160+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":430,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.448971+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:47:36.448971+0000","last_peered":"2023-10-01T09:47:36.448971+0000","last_clean":"2023-10-01T09:47:36.448971+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:47:36.448971+0000","last_undegraded":"2023-10-01T09:47:36.448971+0000","last_fullsized":"2023-10-01T09:47:36.448971+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":257,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915953+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:47:28.915953+0000","last_peered":"2023-10-01T09:47:28.915953+0000","last_clean":"2023-10-01T09:47:28.915953+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:47:28.915953+0000","last_undegraded":"2023-10-01T09:47:28.915953+0000","last_fullsized":"2023-10-01T09:47:28.915953+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"554'9006","reported_seq":16314,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:42.639467+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:47:42.639467+0000","last_peered":"2023-10-01T09:47:42.639467+0000","last_clean":"2023-10-01T09:47:42.639467+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:47:42.639467+0000","last_undegraded":"2023-10-01T09:47:42.639467+0000","last_fullsized":"2023-10-01T09:47:42.639467+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":9006,"log_dups_size":0,"ondisk_log_size":9006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89784320,"num_objects":1370,"num_object_clones":0,"num_object_copies":4110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9006,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":901,"reported_epoch":553,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:47:28.911725+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:47:28.911725+0000","last_peered":"2023-10-01T09:47:28.911725+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:47:28.911725+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:47:28.911725+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":619,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.443981+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:47:36.443981+0000","last_peered":"2023-10-01T09:47:36.443981+0000","last_clean":"2023-10-01T09:47:36.443981+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:47:36.443981+0000","last_undegraded":"2023-10-01T09:47:36.443981+0000","last_fullsized":"2023-10-01T09:47:36.443981+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":80,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.444440+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:47:36.444440+0000","last_peered":"2023-10-01T09:47:36.444440+0000","last_clean":"2023-10-01T09:47:36.444440+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:47:36.444440+0000","last_undegraded":"2023-10-01T09:47:36.444440+0000","last_fullsized":"2023-10-01T09:47:36.444440+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"555'9087","reported_seq":9179,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:45.262315+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:47:45.262315+0000","last_peered":"2023-10-01T09:47:45.262315+0000","last_clean":"2023-10-01T09:47:45.262315+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:47:45.262315+0000","last_undegraded":"2023-10-01T09:47:45.262315+0000","last_fullsized":"2023-10-01T09:47:45.262315+0000","mapping_epoch":475,"log_start":"482'1500","ondisk_log_start":"482'1500","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7587,"log_dups_size":0,"ondisk_log_size":7587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77660160,"num_objects":1185,"num_object_clones":0,"num_object_copies":3555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9087,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1670,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915829+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:47:28.915829+0000","last_peered":"2023-10-01T09:47:28.915829+0000","last_clean":"2023-10-01T09:47:28.915829+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:47:28.915829+0000","last_undegraded":"2023-10-01T09:47:28.915829+0000","last_fullsized":"2023-10-01T09:47:28.915829+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":585,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.455517+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:47:36.455517+0000","last_peered":"2023-10-01T09:47:36.455517+0000","last_clean":"2023-10-01T09:47:36.455517+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:47:36.455517+0000","last_undegraded":"2023-10-01T09:47:36.455517+0000","last_fullsized":"2023-10-01T09:47:36.455517+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":362,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.916215+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:47:28.916215+0000","last_peered":"2023-10-01T09:47:28.916215+0000","last_clean":"2023-10-01T09:47:28.916215+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:47:28.916215+0000","last_undegraded":"2023-10-01T09:47:28.916215+0000","last_fullsized":"2023-10-01T09:47:28.916215+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":353,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.450181+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:47:36.450181+0000","last_peered":"2023-10-01T09:47:36.450181+0000","last_clean":"2023-10-01T09:47:36.450181+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:47:36.450181+0000","last_undegraded":"2023-10-01T09:47:36.450181+0000","last_fullsized":"2023-10-01T09:47:36.450181+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"555'9365","reported_seq":9453,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:46.291032+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:47:46.291032+0000","last_peered":"2023-10-01T09:47:46.291032+0000","last_clean":"2023-10-01T09:47:46.291032+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:47:46.291032+0000","last_undegraded":"2023-10-01T09:47:46.291032+0000","last_fullsized":"2023-10-01T09:47:46.291032+0000","mapping_epoch":475,"log_start":"481'1000","ondisk_log_start":"481'1000","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8365,"log_dups_size":0,"ondisk_log_size":8365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76480512,"num_objects":1167,"num_object_clones":0,"num_object_copies":3501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9365,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":762,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.455678+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:47:36.455678+0000","last_peered":"2023-10-01T09:47:36.455678+0000","last_clean":"2023-10-01T09:47:36.455678+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:47:36.455678+0000","last_undegraded":"2023-10-01T09:47:36.455678+0000","last_fullsized":"2023-10-01T09:47:36.455678+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":595,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.911970+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:47:28.911970+0000","last_peered":"2023-10-01T09:47:28.911970+0000","last_clean":"2023-10-01T09:47:28.911970+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:47:28.911970+0000","last_undegraded":"2023-10-01T09:47:28.911970+0000","last_fullsized":"2023-10-01T09:47:28.911970+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":594,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912194+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:47:28.912194+0000","last_peered":"2023-10-01T09:47:28.912194+0000","last_clean":"2023-10-01T09:47:28.912194+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:47:28.912194+0000","last_undegraded":"2023-10-01T09:47:28.912194+0000","last_fullsized":"2023-10-01T09:47:28.912194+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"554'9047","reported_seq":9139,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:41.915115+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:47:41.915115+0000","last_peered":"2023-10-01T09:47:41.915115+0000","last_clean":"2023-10-01T09:47:41.915115+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:47:41.915115+0000","last_undegraded":"2023-10-01T09:47:41.915115+0000","last_fullsized":"2023-10-01T09:47:41.915115+0000","mapping_epoch":475,"log_start":"485'2000","ondisk_log_start":"485'2000","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7047,"log_dups_size":0,"ondisk_log_size":7047,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92536856,"num_objects":1413,"num_object_clones":0,"num_object_copies":4239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1413,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9047,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":993,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.456643+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:47:36.456643+0000","last_peered":"2023-10-01T09:47:36.456643+0000","last_clean":"2023-10-01T09:47:36.456643+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:47:36.456643+0000","last_undegraded":"2023-10-01T09:47:36.456643+0000","last_fullsized":"2023-10-01T09:47:36.456643+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":611,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915877+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:47:28.915877+0000","last_peered":"2023-10-01T09:47:28.915877+0000","last_clean":"2023-10-01T09:47:28.915877+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:47:28.915877+0000","last_undegraded":"2023-10-01T09:47:28.915877+0000","last_fullsized":"2023-10-01T09:47:28.915877+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":464,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915917+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:47:28.915917+0000","last_peered":"2023-10-01T09:47:28.915917+0000","last_clean":"2023-10-01T09:47:28.915917+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:47:28.915917+0000","last_undegraded":"2023-10-01T09:47:28.915917+0000","last_fullsized":"2023-10-01T09:47:28.915917+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"555'9176","reported_seq":9271,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:46.301794+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:47:46.301794+0000","last_peered":"2023-10-01T09:47:46.301794+0000","last_clean":"2023-10-01T09:47:46.301794+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:47:46.301794+0000","last_undegraded":"2023-10-01T09:47:46.301794+0000","last_fullsized":"2023-10-01T09:47:46.301794+0000","mapping_epoch":479,"log_start":"486'2500","ondisk_log_start":"486'2500","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6676,"log_dups_size":0,"ondisk_log_size":6676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77332480,"num_objects":1180,"num_object_clones":0,"num_object_copies":3540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9176,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":494,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.456255+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:47:36.456255+0000","last_peered":"2023-10-01T09:47:36.456255+0000","last_clean":"2023-10-01T09:47:36.456255+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:47:36.456255+0000","last_undegraded":"2023-10-01T09:47:36.456255+0000","last_fullsized":"2023-10-01T09:47:36.456255+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1189,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.444788+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:47:36.444788+0000","last_peered":"2023-10-01T09:47:36.444788+0000","last_clean":"2023-10-01T09:47:36.444788+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:47:36.444788+0000","last_undegraded":"2023-10-01T09:47:36.444788+0000","last_fullsized":"2023-10-01T09:47:36.444788+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":83,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.455434+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:47:36.455434+0000","last_peered":"2023-10-01T09:47:36.455434+0000","last_clean":"2023-10-01T09:47:36.455434+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:47:36.455434+0000","last_undegraded":"2023-10-01T09:47:36.455434+0000","last_fullsized":"2023-10-01T09:47:36.455434+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"555'9309","reported_seq":9405,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:46.262758+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:47:46.262758+0000","last_peered":"2023-10-01T09:47:46.262758+0000","last_clean":"2023-10-01T09:47:46.262758+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:47:46.262758+0000","last_undegraded":"2023-10-01T09:47:46.262758+0000","last_fullsized":"2023-10-01T09:47:46.262758+0000","mapping_epoch":479,"log_start":"486'2600","ondisk_log_start":"486'2600","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6709,"log_dups_size":0,"ondisk_log_size":6709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78839808,"num_objects":1203,"num_object_clones":0,"num_object_copies":3609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9309,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":87,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.456113+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:47:36.456113+0000","last_peered":"2023-10-01T09:47:36.456113+0000","last_clean":"2023-10-01T09:47:36.456113+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:47:36.456113+0000","last_undegraded":"2023-10-01T09:47:36.456113+0000","last_fullsized":"2023-10-01T09:47:36.456113+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":572,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912222+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:47:28.912222+0000","last_peered":"2023-10-01T09:47:28.912222+0000","last_clean":"2023-10-01T09:47:28.912222+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:47:28.912222+0000","last_undegraded":"2023-10-01T09:47:28.912222+0000","last_fullsized":"2023-10-01T09:47:28.912222+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":277,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915787+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:47:28.915787+0000","last_peered":"2023-10-01T09:47:28.915787+0000","last_clean":"2023-10-01T09:47:28.915787+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:47:28.915787+0000","last_undegraded":"2023-10-01T09:47:28.915787+0000","last_fullsized":"2023-10-01T09:47:28.915787+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":521,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912194+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:47:28.912194+0000","last_peered":"2023-10-01T09:47:28.912194+0000","last_clean":"2023-10-01T09:47:28.912194+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:47:28.912194+0000","last_undegraded":"2023-10-01T09:47:28.912194+0000","last_fullsized":"2023-10-01T09:47:28.912194+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":636,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.444544+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:47:36.444544+0000","last_peered":"2023-10-01T09:47:36.444544+0000","last_clean":"2023-10-01T09:47:36.444544+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:47:36.444544+0000","last_undegraded":"2023-10-01T09:47:36.444544+0000","last_fullsized":"2023-10-01T09:47:36.444544+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":289,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915788+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:47:28.915788+0000","last_peered":"2023-10-01T09:47:28.915788+0000","last_clean":"2023-10-01T09:47:28.915788+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:47:28.915788+0000","last_undegraded":"2023-10-01T09:47:28.915788+0000","last_fullsized":"2023-10-01T09:47:28.915788+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":79,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.444071+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:47:36.444071+0000","last_peered":"2023-10-01T09:47:36.444071+0000","last_clean":"2023-10-01T09:47:36.444071+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:47:36.444071+0000","last_undegraded":"2023-10-01T09:47:36.444071+0000","last_fullsized":"2023-10-01T09:47:36.444071+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":289,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.915940+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:47:28.915940+0000","last_peered":"2023-10-01T09:47:28.915940+0000","last_clean":"2023-10-01T09:47:28.915940+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:47:28.915940+0000","last_undegraded":"2023-10-01T09:47:28.915940+0000","last_fullsized":"2023-10-01T09:47:28.915940+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":467,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912113+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:47:28.912113+0000","last_peered":"2023-10-01T09:47:28.912113+0000","last_clean":"2023-10-01T09:47:28.912113+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:47:28.912113+0000","last_undegraded":"2023-10-01T09:47:28.912113+0000","last_fullsized":"2023-10-01T09:47:28.912113+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":97,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.444619+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:47:36.444619+0000","last_peered":"2023-10-01T09:47:36.444619+0000","last_clean":"2023-10-01T09:47:36.444619+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:47:36.444619+0000","last_undegraded":"2023-10-01T09:47:36.444619+0000","last_fullsized":"2023-10-01T09:47:36.444619+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":513,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.447098+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:47:36.447098+0000","last_peered":"2023-10-01T09:47:36.447098+0000","last_clean":"2023-10-01T09:47:36.447098+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:47:36.447098+0000","last_undegraded":"2023-10-01T09:47:36.447098+0000","last_fullsized":"2023-10-01T09:47:36.447098+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":712,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-10-01T09:47:28.912460+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:47:28.912460+0000","last_peered":"2023-10-01T09:47:28.912460+0000","last_clean":"2023-10-01T09:47:28.912460+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:47:28.912460+0000","last_undegraded":"2023-10-01T09:47:28.912460+0000","last_fullsized":"2023-10-01T09:47:28.912460+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":406,"reported_epoch":554,"state":"active+clean","last_fresh":"2023-10-01T09:47:36.448560+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:47:36.448560+0000","last_peered":"2023-10-01T09:47:36.448560+0000","last_clean":"2023-10-01T09:47:36.448560+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:47:36.448560+0000","last_undegraded":"2023-10-01T09:47:36.448560+0000","last_fullsized":"2023-10-01T09:47:36.448560+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1325334552,"num_objects":20224,"num_object_clones":0,"num_object_copies":60672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4858,"num_objects_misplaced":1217,"num_objects_unfound":0,"num_objects_dirty":20224,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":146216,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11074,"num_bytes_recovered":725745664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4836634624,"data_stored":4836622408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125616,"ondisk_log_size":125616,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430766,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1284968,"kb_used_data":381792,"kb_used_omap":0,"kb_used_meta":903104,"kb_avail":92460184,"statfs":{"total":95995035648,"available":94679228416,"internally_reserved":0,"allocated":390955008,"data_stored":390007908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924778496},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:02 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.495,"15min":2.4870000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":1.3600000000000001,"5min":3.3849999999999998,"15min":108.90000000000001},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.46899999999999997,"15min":2.621},"min":{"1min":0.23300000000000001,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":1.7410000000000001,"5min":1.7410000000000001,"15min":109.187},"last":0.42799999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.58299999999999996,"15min":5.0190000000000001},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.23300000000000001},"max":{"1min":7.7140000000000004,"5min":7.7140000000000004,"15min":229.161},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.53100000000000003,"15min":5.0449999999999999},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":1.972,"5min":3.0640000000000001,"15min":230.126},"last":4.6890000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:46:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.56299999999999994,"15min":1.591},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":1.6359999999999999,"5min":3.4300000000000002,"15min":14.901999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.55700000000000005,"15min":1.48},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.27000000000000002},"max":{"1min":1.8100000000000001,"5min":1.8100000000000001,"15min":14.682},"last":0.45300000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.59299999999999997,"15min":2.1059999999999999},"min":{"1min":0.34999999999999998,"5min":0.30199999999999999,"15min":0.29199999999999998},"max":{"1min":2.1749999999999998,"5min":2.476,"15min":76.555000000000007},"last":1.2230000000000001},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.58399999999999996,"15min":2.1120000000000001},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.1880000000000002,"5min":2.8479999999999999,"15min":76.534000000000006},"last":9.2599999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:47:02 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.52100000000000002,"15min":1.0249999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.4470000000000001,"5min":3.4609999999999999,"15min":9.6020000000000003},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.52500000000000002,"15min":0.99399999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.837,"5min":1.837,"15min":8.4949999999999992},"last":0.57899999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":0.66600000000000004,"15min":1.5369999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.9370000000000003,"5min":7.9370000000000003,"15min":135.40000000000001},"last":4.6539999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.65100000000000002,"15min":1.6379999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":7.0629999999999997,"5min":7.0629999999999997,"15min":135.339},"last":11.895}]},{"osd":6,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.64900000000000002,"15min":1.1519999999999999},"min":{"1min":0.39600000000000002,"5min":0.377,"15min":0.307},"max":{"1min":3.984,"5min":3.984,"15min":11.853999999999999},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.64100000000000001,"15min":1.0880000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32300000000000001},"max":{"1min":4.3470000000000004,"5min":4.3470000000000004,"15min":14.584},"last":9.3100000000000005}]}]},{"osd":6,"up_from":294,"seq":1262720385276,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":947164,"kb_used_data":613652,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":92797988,"statfs":{"total":95995035648,"available":95025139712,"internally_reserved":0,"allocated":628379648,"data_stored":627436823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":4.5839999999999996,"5min":1.2829999999999999,"15min":2.665},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20599999999999999},"max":{"1min":65.599000000000004,"5min":65.599000000000004,"15min":158.48599999999999},"last":0.26900000000000002},{"interface":"front","average":{"1min":5.4089999999999998,"5min":1.484,"15min":2.7909999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":65.388999999999996,"5min":65.388999999999996,"15min":158.37700000000001},"last":0.28699999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":0.70599999999999996,"15min":3.8839999999999999},"min":{"1min":0.34399999999999997,"5min":0.222,"15min":0.222},"max":{"1min":8.2409999999999997,"5min":8.2409999999999997,"15min":168.947},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.625,"5min":0.56999999999999995,"15min":3.8210000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.25600000000000001},"max":{"1min":2.6669999999999998,"5min":3.7469999999999999,"15min":169.07900000000001},"last":0.51500000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":0.73799999999999999,"15min":1.429},"min":{"1min":0.311,"5min":0.254,"15min":0.254},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":10.92},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.748,"5min":0.57199999999999995,"15min":1.3899999999999999},"min":{"1min":0.314,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":2.6480000000000001,"5min":3.7160000000000002,"15min":21.704999999999998},"last":0.54000000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55000000000000004,"15min":1.647},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.9710000000000001,"5min":3.3759999999999999,"15min":89.753},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.51,"5min":0.78200000000000003,"15min":1.7589999999999999},"min":{"1min":0.34399999999999997,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.3109999999999999,"5min":8.3109999999999999,"15min":89.849999999999994},"last":1.1459999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.505,"15min":1.0609999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.7559999999999998,"5min":3.6349999999999998,"15min":63.134},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.54400000000000004,"15min":1.0429999999999999},"min":{"1min":0.34499999999999997,"5min":0.33600000000000002,"15min":0.28899999999999998},"max":{"1min":2.778,"5min":3.2519999999999998,"15min":62.973999999999997},"last":0.80700000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:47:37 2023","interfaces":[{"interface":"back","average":{"1min":6.5650000000000004,"5min":2.089,"15min":1.367},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":136.535,"5min":136.535,"15min":136.535},"last":2.9260000000000002},{"interface":"front","average":{"1min":6.3049999999999997,"5min":2.1499999999999999,"15min":1.4830000000000001},"min":{"1min":0.316,"5min":0.29899999999999999,"15min":0.28100000000000003},"max":{"1min":136.47300000000001,"5min":136.47300000000001,"15min":136.47300000000001},"last":1.2989999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.56100000000000005,"15min":1.2609999999999999},"min":{"1min":0.38300000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.3159999999999998,"5min":3.339,"15min":94.245999999999995},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.76700000000000002,"15min":1.421},"min":{"1min":0.37,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":8.3399999999999999,"5min":8.3399999999999999,"15min":94.715000000000003},"last":0.505}]}]},{"osd":1,"up_from":94,"seq":403726926253,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1066568,"kb_used_data":242624,"kb_used_omap":7,"kb_used_meta":823928,"kb_avail":92678584,"statfs":{"total":95995035648,"available":94902870016,"internally_reserved":0,"allocated":248446976,"data_stored":246356133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":843702695},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":1.0349999999999999,"15min":3.5819999999999999},"min":{"1min":0.19500000000000001,"5min":0.186,"15min":0.182},"max":{"1min":0.82799999999999996,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.0860000000000001,"15min":3.1030000000000002},"min":{"1min":0.222,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":2.0859999999999999,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.624}]},{"osd":2,"last update":"Sun Oct 1 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.35899999999999999,"15min":2.0179999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.20000000000000001},"max":{"1min":0.78500000000000003,"5min":1.3240000000000001,"15min":51.640000000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.44700000000000001,"15min":2.7570000000000001},"min":{"1min":0.29199999999999998,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":0.69499999999999995,"5min":2.9089999999999998,"15min":154.197},"last":1.151}]},{"osd":3,"last update":"Sun Oct 1 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45000000000000001,"15min":2.706},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.219},"max":{"1min":2.149,"5min":2.8780000000000001,"15min":95.305000000000007},"last":0.378},{"interface":"front","average":{"1min":0.497,"5min":0.47399999999999998,"15min":3.3210000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.994,"5min":2.7360000000000002,"15min":154.483},"last":1.2130000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.56499999999999995,"15min":2.1389999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":1.204,"5min":3.0550000000000002,"15min":154.101},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.46800000000000003,"15min":2.1469999999999998},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.27700000000000002},"max":{"1min":0.77200000000000002,"5min":0.83199999999999996,"15min":154.56999999999999},"last":0.878}]},{"osd":5,"last update":"Sun Oct 1 09:46:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.143,"15min":2.23},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.8929999999999998,"5min":59.561,"15min":154.22999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.53500000000000003,"5min":1.137,"15min":2.3660000000000001},"min":{"1min":0.32900000000000001,"5min":0.31,"15min":0.307},"max":{"1min":0.89600000000000002,"5min":59.536000000000001,"15min":154.54400000000001},"last":5.0819999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.65700000000000003,"15min":2.2770000000000001},"min":{"1min":0.33400000000000002,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":4.5069999999999997,"5min":4.5069999999999997,"15min":154.12200000000001},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.63200000000000001,"15min":2.218},"min":{"1min":0.35899999999999999,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":1.5900000000000001,"5min":2.5630000000000002,"15min":154.16999999999999},"last":0.59299999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.73299999999999998,"15min":2.4420000000000002},"min":{"1min":0.38900000000000001,"5min":0.36399999999999999,"15min":0.33900000000000002},"max":{"1min":4.3730000000000002,"5min":4.3730000000000002,"15min":154.20699999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.71999999999999997,"15min":2.4460000000000002},"min":{"1min":0.435,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":4.3879999999999999,"5min":4.3879999999999999,"15min":154.51599999999999},"last":0.52100000000000002}]}]},{"osd":0,"up_from":89,"seq":382252089787,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2475476,"kb_used_data":1540044,"kb_used_omap":8,"kb_used_meta":935415,"kb_avail":91269676,"statfs":{"total":95995035648,"available":93460148224,"internally_reserved":0,"allocated":1577005056,"data_stored":1574962206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":957865922},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.40500000000000003,"15min":3.6469999999999998},"min":{"1min":0.20799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.69099999999999995,"5min":4.2919999999999998,"15min":216.821},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.434,"15min":4.2300000000000004},"min":{"1min":0.20599999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.72499999999999998,"5min":3.9620000000000002,"15min":217.02699999999999},"last":0.22}]},{"osd":2,"last update":"Sun Oct 1 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.47499999999999998,"15min":1.9059999999999999},"min":{"1min":0.29399999999999998,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":1.3100000000000001,"5min":4.2430000000000003,"15min":19.788},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54700000000000004,"15min":1.9239999999999999},"min":{"1min":0.29299999999999998,"5min":0.27200000000000002,"15min":0.215},"max":{"1min":1.3300000000000001,"5min":3.8719999999999999,"15min":25.844999999999999},"last":0.28100000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.47899999999999998,"15min":2.778},"min":{"1min":0.24299999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.68000000000000005,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.245},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.626,"15min":2.6200000000000001},"min":{"1min":0.25700000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.0259999999999998,"5min":4.9199999999999999,"15min":222.227},"last":0.28699999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57199999999999995,"15min":1.8500000000000001},"min":{"1min":0.308,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":2.1160000000000001,"5min":4.048,"15min":62.206000000000003},"last":138.84100000000001},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.52500000000000002,"15min":1.9359999999999999},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":1.3380000000000001,"5min":4,"15min":62.073999999999998},"last":135.94800000000001}]},{"osd":5,"last update":"Sun Oct 1 09:46:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.59999999999999998,"15min":1.6699999999999999},"min":{"1min":0.34899999999999998,"5min":0.312,"15min":0.312},"max":{"1min":1.5920000000000001,"5min":4.0350000000000001,"15min":16.808},"last":156.69999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.59899999999999998,"15min":1.5820000000000001},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.8100000000000001,"5min":4.2729999999999997,"15min":17.507000000000001},"last":156.773}]},{"osd":6,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.64400000000000002,"15min":1.51},"min":{"1min":0.35699999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.589,"5min":4.3129999999999997,"15min":17.535},"last":135.321},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.66700000000000004,"15min":1.571},"min":{"1min":0.35899999999999999,"5min":0.32600000000000001,"15min":0.32400000000000001},"max":{"1min":2.5960000000000001,"5min":4.2050000000000001,"15min":15.651999999999999},"last":135.29900000000001}]},{"osd":7,"last update":"Sun Oct 1 09:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.66300000000000003,"15min":1.927},"min":{"1min":0.39700000000000002,"5min":0.374,"15min":0.374},"max":{"1min":2.2290000000000001,"5min":4.3360000000000003,"15min":48.895000000000003},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.79400000000000004,"15min":1.97},"min":{"1min":0.39300000000000002,"5min":0.376,"15min":0.35299999999999998},"max":{"1min":0.78300000000000003,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.34999999999999998}]}]},{"osd":2,"up_from":0,"seq":425201762727,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599200,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1121148,"kb_used_data":275060,"kb_used_omap":3,"kb_used_meta":846076,"kb_avail":92624004,"statfs":{"total":95995035648,"available":94846980096,"internally_reserved":0,"allocated":281661440,"data_stored":279516619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":866382567},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.26900000000000002,"5min":0.34300000000000003,"15min":3.5009999999999999},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":0.504,"5min":2.7989999999999999,"15min":137.34399999999999},"last":4.0309999999999997},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.35899999999999999,"15min":3.5459999999999998},"min":{"1min":0.23400000000000001,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":0.66800000000000004,"5min":1.2849999999999999,"15min":137.73400000000001},"last":5.157}]},{"osd":1,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.40200000000000002,"15min":4.7430000000000003},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":0.63500000000000001,"5min":2.7250000000000001,"15min":162.22999999999999},"last":1.7110000000000001},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.371,"15min":4.7370000000000001},"min":{"1min":0.27200000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.3440000000000001,"5min":1.4119999999999999,"15min":162.70699999999999},"last":4.008}]},{"osd":2,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.442,"15min":1.552},"min":{"1min":0.251,"5min":0.24199999999999999,"15min":0.20799999999999999},"max":{"1min":0.46000000000000002,"5min":2.9049999999999998,"15min":14.254},"last":1.1319999999999999},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.499,"15min":1.4350000000000001},"min":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":2.5230000000000001,"5min":4.5049999999999999,"15min":15.523},"last":3.9809999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.53400000000000003,"15min":1.653},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.26200000000000001},"max":{"1min":4.5010000000000003,"5min":4.5010000000000003,"15min":77.793000000000006},"last":3.9420000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.49399999999999999,"15min":1.8120000000000001},"min":{"1min":0.35299999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":0.72299999999999998,"5min":1.5609999999999999,"15min":76.646000000000001},"last":1.7450000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:47:37 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.67200000000000004,"15min":1.7150000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27500000000000002},"max":{"1min":3.1459999999999999,"5min":4.3099999999999996,"15min":28.117999999999999},"last":1.7689999999999999},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.502,"15min":1.8140000000000001},"min":{"1min":0.34300000000000003,"5min":0.33600000000000002,"15min":0.27700000000000002},"max":{"1min":0.56899999999999995,"5min":2.512,"15min":28.34},"last":1.071}]},{"osd":6,"last update":"Sun Oct 1 09:47:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54500000000000004,"15min":1.758},"min":{"1min":0.45500000000000002,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":1.0549999999999999,"5min":2.9100000000000001,"15min":73.887},"last":1.0960000000000001},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.65000000000000002,"15min":1.869},"min":{"1min":0.38100000000000001,"5min":0.35499999999999998,"15min":0.33600000000000002},"max":{"1min":3.1099999999999999,"5min":3.1099999999999999,"15min":74.016999999999996},"last":1.0349999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":0.85999999999999999,"15min":1.7729999999999999},"min":{"1min":0.42799999999999999,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":4.1989999999999998,"5min":4.1989999999999998,"15min":13.085000000000001},"last":3.2069999999999999},{"interface":"front","average":{"1min":1.2030000000000001,"5min":0.80500000000000005,"15min":1.5529999999999999},"min":{"1min":0.432,"5min":0.38800000000000001,"15min":0.311},"max":{"1min":4.3280000000000003,"5min":4.3280000000000003,"15min":13.191000000000001},"last":4.0529999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019451,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2077924,"kb_used_data":1183644,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":91667228,"statfs":{"total":95995035648,"available":93867241472,"internally_reserved":0,"allocated":1212051456,"data_stored":1211107862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7480000000000002,"5min":0.89400000000000002,"15min":3.5819999999999999},"min":{"1min":0.24399999999999999,"5min":0.22500000000000001,"15min":0.21199999999999999},"max":{"1min":41.963999999999999,"5min":41.963999999999999,"15min":155.315},"last":3.609},{"interface":"front","average":{"1min":2.7530000000000001,"5min":0.90100000000000002,"15min":3.9140000000000001},"min":{"1min":0.26100000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":41.890999999999998,"5min":41.890999999999998,"15min":155.334},"last":0.25700000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.54000000000000004,"15min":5.1449999999999996},"min":{"1min":0.25600000000000001,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":3.3929999999999998,"5min":3.3929999999999998,"15min":190.61199999999999},"last":0.313},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.54900000000000004,"15min":5.8639999999999999},"min":{"1min":0.29199999999999998,"5min":0.27500000000000002,"15min":0.25600000000000001},"max":{"1min":3.3839999999999999,"5min":3.3839999999999999,"15min":253.273},"last":0.54100000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.621,"15min":2.0499999999999998},"min":{"1min":0.30499999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.343,"5min":4.2329999999999997,"15min":170.45400000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.61499999999999999,"15min":1.8069999999999999},"min":{"1min":0.35399999999999998,"5min":0.314,"15min":0.309},"max":{"1min":3.1389999999999998,"5min":3.1389999999999998,"15min":36.515999999999998},"last":0.28799999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.61199999999999999,"15min":1.9850000000000001},"min":{"1min":0.307,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":3.1539999999999999,"5min":3.1539999999999999,"15min":140.881},"last":2.7330000000000001},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.61399999999999999,"15min":2.0579999999999998},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.311},"max":{"1min":1.3049999999999999,"5min":6.2969999999999997,"15min":140.911},"last":0.56899999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.0369999999999999,"15min":1.5740000000000001},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.26600000000000001},"max":{"1min":21.364999999999998,"5min":21.364999999999998,"15min":110.003},"last":21.364999999999998},{"interface":"front","average":{"1min":1.9510000000000001,"5min":0.93200000000000005,"15min":1.466},"min":{"1min":0.313,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":21.559999999999999,"5min":21.559999999999999,"15min":108.56399999999999},"last":21.559999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:47:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.56499999999999995,"15min":0.91700000000000004},"min":{"1min":0.318,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":3.3029999999999999,"5min":3.3029999999999999,"15min":11.358000000000001},"last":1.569},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.60999999999999999,"15min":1.046},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.28999999999999998},"max":{"1min":2.2669999999999999,"5min":2.9239999999999999,"15min":10.076000000000001},"last":0.64200000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.63400000000000001,"15min":1.149},"min":{"1min":0.39200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.3079999999999998,"5min":3.3079999999999998,"15min":38.034999999999997},"last":1.6859999999999999},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.70899999999999996,"15min":1.3839999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.308},"max":{"1min":3.0329999999999999,"5min":3.113,"15min":37.723999999999997},"last":0.42399999999999999}]}]},{"osd":5,"up_from":379,"seq":1627792605357,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":981128,"kb_used_data":612480,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":92764024,"statfs":{"total":95995035648,"available":94990360576,"internally_reserved":0,"allocated":627179520,"data_stored":626252154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":4.9749999999999996,"5min":1.6719999999999999,"15min":2.3140000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.21099999999999999},"max":{"1min":99.609999999999999,"5min":99.609999999999999,"15min":99.609999999999999},"last":0.70299999999999996},{"interface":"front","average":{"1min":5.117,"5min":1.7130000000000001,"15min":2.2959999999999998},"min":{"1min":0.215,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":98.995999999999995,"5min":98.995999999999995,"15min":98.995999999999995},"last":0.501}]},{"osd":1,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":5.4790000000000001},"min":{"1min":0.28399999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.157,"5min":4.3289999999999997,"15min":295.71300000000002},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.56699999999999995,"15min":5.5129999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":3.665,"5min":3.9580000000000002,"15min":299.59399999999999},"last":0.76300000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.60399999999999998,"15min":1.4179999999999999},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.9129999999999998,"5min":4.2670000000000003,"15min":14.199999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.70899999999999996,"15min":1.357},"min":{"1min":0.31900000000000001,"5min":0.29599999999999999,"15min":0.27100000000000002},"max":{"1min":4.1909999999999998,"5min":4.3719999999999999,"15min":14.430999999999999},"last":0.64400000000000002}]},{"osd":3,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":0.70599999999999996,"15min":1.3759999999999999},"min":{"1min":0.30599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.141,"5min":7.141,"15min":14.004},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.66600000000000004,"15min":1.5029999999999999},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":3.1139999999999999,"5min":4.2910000000000004,"15min":13.991},"last":0.68899999999999995}]},{"osd":4,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.66400000000000003,"15min":1.375},"min":{"1min":0.27900000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.0659999999999998,"5min":4.1929999999999996,"15min":122.971},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.66500000000000004,"15min":1.472},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2469999999999999,"5min":4.3869999999999996,"15min":122.89100000000001},"last":0.68000000000000005}]},{"osd":6,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.63800000000000001,"15min":0.94599999999999995},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28299999999999997},"max":{"1min":3.5750000000000002,"5min":4.3300000000000001,"15min":12.545},"last":0.91800000000000004},{"interface":"front","average":{"1min":1.097,"5min":0.76300000000000001,"15min":1.119},"min":{"1min":0.31,"5min":0.31,"15min":0.30499999999999999},"max":{"1min":4.484,"5min":4.484,"15min":14.193},"last":0.61499999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.74399999999999999,"15min":1.1120000000000001},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.2199999999999998,"5min":4.3559999999999999,"15min":10.813000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.75600000000000001,"15min":1.0509999999999999},"min":{"1min":0.40799999999999997,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":3.609,"5min":4.2000000000000002,"15min":12.500999999999999},"last":0.51900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1556287488,"data_stored":1556283416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":236584960,"data_stored":236584960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":270925824,"data_stored":270925824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1181028352,"data_stored":1181024280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":604307456,"data_stored":604307456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":609882112,"data_stored":609878040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":377618432,"data_stored":377618432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:47:48.989 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:47:49.037 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:49.134 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:49.157 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:47:49.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:49.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:47:49.225 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:48 smithi138 ceph-mon[192513]: from='client.47474 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:49.225 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:48 smithi138 ceph-mon[192513]: pgmap v1669: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 4939/55233 objects degraded (8.942%); 1032/55233 objects misplaced (1.868%); 1.4 MiB/s, 21 objects/s recovering 2023-10-01T09:47:49.225 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:48 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:47:49.226 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:48 smithi138 ceph-mon[185886]: from='client.47474 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:47:49.226 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:48 smithi138 ceph-mon[185886]: pgmap v1669: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 4939/55233 objects degraded (8.942%); 1032/55233 objects misplaced (1.868%); 1.4 MiB/s, 21 objects/s recovering 2023-10-01T09:47:49.227 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:48 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:47:49.263 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:49.342 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:49.489 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:49.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:49.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:49.597 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:47:49.672 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:49.802 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:47:49.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:49.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:47:49.914 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:49.959 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:50.023 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:47:50.034 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:50 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3236011151' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:47:50.034 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:50 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:47:50.034 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:50 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:50.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:50 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3236011151' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:47:50.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:50 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:47:50.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:50 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:50.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:50 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3236011151' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:47:50.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:50 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:47:50.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:50 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:47:50.374 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:47:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:47:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:47:50.374 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:47:50.374 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:47:50.653 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:50.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:50.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:50.762 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:50.893 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:51.042 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:51.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:51.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:51.157 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:51.159 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:47:51.184 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:51 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:47:51.184 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:51 smithi190 ceph-mon[160435]: osdmap e556: 8 total, 8 up, 7 in 2023-10-01T09:47:51.184 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:51 smithi190 ceph-mon[160435]: failed to encode map e556 with expected crc 2023-10-01T09:47:51.184 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:51 smithi190 ceph-mon[160435]: failed to encode map e556 with expected crc 2023-10-01T09:47:51.184 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:51 smithi190 ceph-mon[160435]: failed to encode map e556 with expected crc 2023-10-01T09:47:51.184 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:51 smithi190 ceph-mon[160435]: failed to encode map e556 with expected crc 2023-10-01T09:47:51.184 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:51 smithi190 ceph-mon[160435]: pgmap v1671: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 4939/54069 objects degraded (9.135%); 1032/54069 objects misplaced (1.909%); 1.7 MiB/s, 27 objects/s recovering 2023-10-01T09:47:51.186 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:51.234 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:47:51.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:51.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:47:51.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:51 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:47:51.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:51 smithi138 ceph-mon[192513]: osdmap e556: 8 total, 8 up, 7 in 2023-10-01T09:47:51.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:51 smithi138 ceph-mon[192513]: failed to encode map e556 with expected crc 2023-10-01T09:47:51.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:51 smithi138 ceph-mon[192513]: failed to encode map e556 with expected crc 2023-10-01T09:47:51.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:51 smithi138 ceph-mon[192513]: failed to encode map e556 with expected crc 2023-10-01T09:47:51.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:51 smithi138 ceph-mon[192513]: failed to encode map e556 with expected crc 2023-10-01T09:47:51.321 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:51 smithi138 ceph-mon[192513]: pgmap v1671: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 4939/54069 objects degraded (9.135%); 1032/54069 objects misplaced (1.909%); 1.7 MiB/s, 27 objects/s recovering 2023-10-01T09:47:51.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:51 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:47:51.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:51 smithi138 ceph-mon[185886]: osdmap e556: 8 total, 8 up, 7 in 2023-10-01T09:47:51.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:51 smithi138 ceph-mon[185886]: failed to encode map e556 with expected crc 2023-10-01T09:47:51.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:51 smithi138 ceph-mon[185886]: failed to encode map e556 with expected crc 2023-10-01T09:47:51.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:51 smithi138 ceph-mon[185886]: failed to encode map e556 with expected crc 2023-10-01T09:47:51.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:51 smithi138 ceph-mon[185886]: failed to encode map e556 with expected crc 2023-10-01T09:47:51.322 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:51 smithi138 ceph-mon[185886]: pgmap v1671: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 4939/54069 objects degraded (9.135%); 1032/54069 objects misplaced (1.909%); 1.7 MiB/s, 27 objects/s recovering 2023-10-01T09:47:51.336 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:47:51.410 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:51.721 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:47:51.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:51.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:47:51.827 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:47:51.902 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:47:51.928 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:47:51.935 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:51.952 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:47:51.977 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:47:51.978 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:47:52.028 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:47:52.048 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:47:52.058 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:47:52.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:52.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:47:52.079 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:47:52.114 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:47:52.115 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:47:52.116 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:47:52.116 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:47:52.116 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:47:52.116 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:47:52.116 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:47:52.116 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:47:52.117 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:47:52.169 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:47:52.198 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:52.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:52 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1923840075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:52.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:52 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1923840075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:52.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:52 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1923840075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:47:52.332 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:47:52.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:52.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:47:52.452 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:52.540 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:52.715 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:52.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:52.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:52.834 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:52.921 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:53.038 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:47:53.056 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:53 smithi138 ceph-mon[185886]: pgmap v1672: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 4939/49059 objects degraded (10.067%); 1032/49059 objects misplaced (2.104%); 821 KiB/s, 12 objects/s recovering 2023-10-01T09:47:53.056 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:53 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 4939/54069 objects degraded (9.135%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:53.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:53 smithi190 ceph-mon[160435]: pgmap v1672: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 4939/49059 objects degraded (10.067%); 1032/49059 objects misplaced (2.104%); 821 KiB/s, 12 objects/s recovering 2023-10-01T09:47:53.068 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:53 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 4939/54069 objects degraded (9.135%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:53.111 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:53.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:53.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:53.190 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:47:53.228 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:53.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:53 smithi138 ceph-mon[192513]: pgmap v1672: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 594 op/s; 4939/49059 objects degraded (10.067%); 1032/49059 objects misplaced (2.104%); 821 KiB/s, 12 objects/s recovering 2023-10-01T09:47:53.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:53 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 4939/54069 objects degraded (9.135%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:53.338 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:53.462 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:53.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:53.574 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:53.615 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:53.703 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:53.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:53.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:53.808 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:47:53.863 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:53.873 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:54.164 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:47:54.165 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 4939/49059 objects degraded (10.067%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2294,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":556,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":16353,"data_bytes":1074599883,"bytes_used":9498013696,"bytes_avail":662467235840,"bytes_total":671965249536,"degraded_objects":4939,"degraded_total":49059,"degraded_ratio":0.10067469781283761,"misplaced_objects":1032,"misplaced_total":49059,"misplaced_ratio":0.021035895554332537,"recovering_objects_per_sec":12,"recovering_bytes_per_sec":840912,"recovering_keys_per_sec":0,"num_objects_recovered":114,"num_bytes_recovered":7471104,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":594},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":34,"modified":"2023-10-01T09:43:40.311181+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:47:54.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:54 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3624247081' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:47:54.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:54 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3624247081' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:47:54.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:54 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3624247081' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:47:54.936 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:47:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:47:54] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.33.4" 2023-10-01T09:47:55.045 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:47:55.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:55.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:47:55.156 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:55.192 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:55.309 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:55 smithi190 ceph-mon[160435]: pgmap v1673: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 973 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 5006/46593 objects degraded (10.744%); 851/46593 objects misplaced (1.826%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:47:55.399 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:47:55.399 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:47:55.473 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:55 smithi138 ceph-mon[185886]: pgmap v1673: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 973 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 5006/46593 objects degraded (10.744%); 851/46593 objects misplaced (1.826%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:47:55.473 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:55 smithi138 ceph-mon[192513]: pgmap v1673: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 973 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 5006/46593 objects degraded (10.744%); 851/46593 objects misplaced (1.826%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:47:55.494 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:47:55.644 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:55.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:55.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:55.756 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:55.855 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:56.016 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:56.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:56.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:56.075 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:47:56.075 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:47:56.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:56.176 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:56.305 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:47:56.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:56 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:47:56.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:56.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:47:56.411 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:47:56.500 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:56.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:56 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:47:56.573 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:56 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:47:56.584 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:56.677 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:47:57.261 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:47:57.293 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:57 smithi138 ceph-mon[192513]: pgmap v1674: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 898 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 5006/42975 objects degraded (11.649%); 851/42975 objects misplaced (1.980%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:47:57.293 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:57 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2567911547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:57.293 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:57 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:57.294 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:57 smithi138 ceph-mon[185886]: pgmap v1674: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 898 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 5006/42975 objects degraded (11.649%); 851/42975 objects misplaced (1.980%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:47:57.294 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:57 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2567911547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:57.295 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:57 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:57.465 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:47:57.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:57.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:47:57.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:57 smithi190 ceph-mon[160435]: pgmap v1674: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 898 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 5006/42975 objects degraded (11.649%); 851/42975 objects misplaced (1.980%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:47:57.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:57 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2567911547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:57.561 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:57 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:47:57.579 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:57.635 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:57.708 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:47:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:58 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:47:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:58 smithi190 ceph-mon[160435]: failed to encode map e557 with expected crc 2023-10-01T09:47:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:58 smithi190 ceph-mon[160435]: failed to encode map e557 with expected crc 2023-10-01T09:47:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:58 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:47:58.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:58 smithi190 ceph-mon[160435]: osdmap e557: 8 total, 8 up, 7 in 2023-10-01T09:47:58.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:58 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5006/42975 objects degraded (11.649%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:58.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:58 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:47:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:58 smithi138 ceph-mon[185886]: failed to encode map e557 with expected crc 2023-10-01T09:47:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:58 smithi138 ceph-mon[185886]: failed to encode map e557 with expected crc 2023-10-01T09:47:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:58 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:47:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:58 smithi138 ceph-mon[185886]: osdmap e557: 8 total, 8 up, 7 in 2023-10-01T09:47:58.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:58 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5006/42975 objects degraded (11.649%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:58.572 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:58 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:47:58.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:58 smithi138 ceph-mon[192513]: failed to encode map e557 with expected crc 2023-10-01T09:47:58.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:58 smithi138 ceph-mon[192513]: failed to encode map e557 with expected crc 2023-10-01T09:47:58.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:58 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:47:58.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:58 smithi138 ceph-mon[192513]: osdmap e557: 8 total, 8 up, 7 in 2023-10-01T09:47:58.573 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:58 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5006/42975 objects degraded (11.649%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:47:58.575 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:47:58.699 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:58.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:58.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:58.806 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:47:58.913 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:59.141 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:47:59.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:59.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:47:59.247 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:47:59.308 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:47:59.444 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:47:59.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:47:59.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:47:59.496 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:59 smithi190 ceph-mon[160435]: pgmap v1676: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 792 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 5086/37905 objects degraded (13.418%); 664/37905 objects misplaced (1.752%); 1.7 MiB/s, 26 objects/s recovering 2023-10-01T09:47:59.496 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:47:59.496 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:47:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:47:59.550 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:47:59.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:59 smithi138 ceph-mon[192513]: pgmap v1676: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 792 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 5086/37905 objects degraded (13.418%); 664/37905 objects misplaced (1.752%); 1.7 MiB/s, 26 objects/s recovering 2023-10-01T09:47:59.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:47:59.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:47:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:47:59.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:59 smithi138 ceph-mon[185886]: pgmap v1676: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 792 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 5086/37905 objects degraded (13.418%); 664/37905 objects misplaced (1.752%); 1.7 MiB/s, 26 objects/s recovering 2023-10-01T09:47:59.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:47:59.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:47:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:47:59.584 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:00.037 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:48:00.048 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:00.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:00.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:00.098 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:48:00.129 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:48:00.153 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:00.161 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:48:00.180 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:00.217 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:48:00.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:00.243 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:00.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:00.254 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:48:00.259 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:48:00.301 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:48:00.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:00 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/450074654' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:00.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:00.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:00 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:00.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:00 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/450074654' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:00.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:00.324 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:00 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:00.344 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:00.351 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:48:00.399 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:48:00.408 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:48:00.408 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:48:00.408 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:48:00.408 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:48:00.408 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:48:00.408 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:48:00.408 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:48:00.408 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:48:00.420 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:00.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:00 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/450074654' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:00.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:00.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:00 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:00.622 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:00.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:00.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:00.726 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:48:00.767 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:00.913 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:48:00.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:00.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:48:01.030 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:01.145 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:01.247 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:01.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:01.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:01.325 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:48:01.341 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:48:01.341 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [ 2023-10-01T09:48:01.341 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.341 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:156504 10.7 10:e4e0ba80:::benchmark_data_smithi138_573643_object73282:head [delete] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-01T09:48:01.341 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:48:01.309169+0000", 2023-10-01T09:48:01.341 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.021681132999999998, 2023-10-01T09:48:01.341 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021713639999999999, 2023-10-01T09:48:01.342 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:48:01.342 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T09:48:01.342 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:48:01.342 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:48:01.342 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:48:01.342 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 156504 2023-10-01T09:48:01.342 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.342 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:48:01.342 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.342 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:48:01.342 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309169+0000", 2023-10-01T09:48:01.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:48:01.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:48:01.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309169+0000", 2023-10-01T09:48:01.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:48:01.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.343 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:48:01.343 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309167+0000", 2023-10-01T09:48:01.343 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3079999999999998e-06 2023-10-01T09:48:01.343 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.343 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:48:01.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309173+0000", 2023-10-01T09:48:01.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.3329999999999997e-06 2023-10-01T09:48:01.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:48:01.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309181+0000", 2023-10-01T09:48:01.344 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.8539999999999996e-06 2023-10-01T09:48:01.344 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.344 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.344 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:48:01.344 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309186+0000", 2023-10-01T09:48:01.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014732800000000001 2023-10-01T09:48:01.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.345 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:48:01.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309333+0000", 2023-10-01T09:48:01.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000182296 2023-10-01T09:48:01.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.345 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:48:01.345 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309515+0000", 2023-10-01T09:48:01.345 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00017949400000000001 2023-10-01T09:48:01.345 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.345 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:48:01.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309695+0000", 2023-10-01T09:48:01.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.015910174999999999 2023-10-01T09:48:01.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:48:01.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.325605+0000", 2023-10-01T09:48:01.346 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0014828840000000001 2023-10-01T09:48:01.346 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.346 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.346 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:48:01.346 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.327088+0000", 2023-10-01T09:48:01.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.003272489 2023-10-01T09:48:01.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:48:01.347 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.330361+0000", 2023-10-01T09:48:01.347 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.9243999999999999e-05 2023-10-01T09:48:01.347 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.347 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.347 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "commit_sent", 2023-10-01T09:48:01.348 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.330400+0000", 2023-10-01T09:48:01.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021231092999999999 2023-10-01T09:48:01.348 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.348 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:48:01.348 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.348 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.348 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.348 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:156505 10.7 10:e483b8e5:::benchmark_data_smithi138_573643_object73283:head [delete] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-01T09:48:01.348 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:48:01.309201+0000", 2023-10-01T09:48:01.348 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.021648948000000001, 2023-10-01T09:48:01.348 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021798358, 2023-10-01T09:48:01.348 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:48:01.348 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:48:01.349 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:48:01.349 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:48:01.349 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:48:01.349 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 156505 2023-10-01T09:48:01.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.349 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:48:01.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.349 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:48:01.349 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309201+0000", 2023-10-01T09:48:01.349 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:48:01.349 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.349 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:48:01.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309201+0000", 2023-10-01T09:48:01.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:48:01.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:48:01.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309200+0000", 2023-10-01T09:48:01.350 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.7329999999999999e-06 2023-10-01T09:48:01.350 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.350 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.350 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:48:01.350 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309203+0000", 2023-10-01T09:48:01.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.3140000000000003e-06 2023-10-01T09:48:01.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:48:01.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309209+0000", 2023-10-01T09:48:01.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.743e-06 2023-10-01T09:48:01.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.351 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.351 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:48:01.351 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309212+0000", 2023-10-01T09:48:01.351 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053992789999999999 2023-10-01T09:48:01.351 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.352 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:48:01.352 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.314612+0000", 2023-10-01T09:48:01.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014409700000000001 2023-10-01T09:48:01.352 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.352 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:48:01.352 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.314756+0000", 2023-10-01T09:48:01.352 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00015664100000000001 2023-10-01T09:48:01.352 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.352 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.352 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:48:01.352 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.314912+0000", 2023-10-01T09:48:01.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010762605 2023-10-01T09:48:01.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.353 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:48:01.353 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.325675+0000", 2023-10-01T09:48:01.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0033611779999999998 2023-10-01T09:48:01.353 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.353 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.353 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:48:01.353 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.329036+0000", 2023-10-01T09:48:01.353 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019835174000000001 2023-10-01T09:48:01.353 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.354 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:48:01.354 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.354 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.354 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.354 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:156506 10.3 10:c90ce3ce:::benchmark_data_smithi138_573643_object73284:head [delete] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-01T09:48:01.354 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:48:01.309774+0000", 2023-10-01T09:48:01.354 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.021075980000000001, 2023-10-01T09:48:01.354 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021324988, 2023-10-01T09:48:01.354 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:48:01.354 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:48:01.354 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:48:01.354 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:48:01.354 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:48:01.355 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 156506 2023-10-01T09:48:01.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.355 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:48:01.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:48:01.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309774+0000", 2023-10-01T09:48:01.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:48:01.355 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.355 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.355 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:48:01.355 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309774+0000", 2023-10-01T09:48:01.355 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:48:01.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:48:01.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309773+0000", 2023-10-01T09:48:01.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.1600000000000001e-06 2023-10-01T09:48:01.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.356 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:48:01.356 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309780+0000", 2023-10-01T09:48:01.356 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.296e-05 2023-10-01T09:48:01.356 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.356 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:48:01.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309793+0000", 2023-10-01T09:48:01.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.6140000000000003e-06 2023-10-01T09:48:01.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:48:01.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309800+0000", 2023-10-01T09:48:01.357 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.002579591 2023-10-01T09:48:01.357 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.357 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.357 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:48:01.357 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.312380+0000", 2023-10-01T09:48:01.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012745399999999999 2023-10-01T09:48:01.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:48:01.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.312507+0000", 2023-10-01T09:48:01.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000155799 2023-10-01T09:48:01.358 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.358 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.358 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:48:01.358 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.312663+0000", 2023-10-01T09:48:01.358 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0028893339999999999 2023-10-01T09:48:01.358 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.359 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:48:01.359 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.359 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.359 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.359 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:156507 10.7 10:e9dabe3f:::benchmark_data_smithi138_573643_object73285:head [delete] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-01T09:48:01.359 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:48:01.309810+0000", 2023-10-01T09:48:01.359 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.021039407, 2023-10-01T09:48:01.359 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021366785999999999, 2023-10-01T09:48:01.359 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:48:01.359 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:48:01.359 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:48:01.359 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:48:01.360 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:48:01.360 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 156507 2023-10-01T09:48:01.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.360 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:48:01.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.360 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:48:01.360 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309810+0000", 2023-10-01T09:48:01.360 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:48:01.360 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.360 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.360 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:48:01.360 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309810+0000", 2023-10-01T09:48:01.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:48:01.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:48:01.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309810+0000", 2023-10-01T09:48:01.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.016e-06 2023-10-01T09:48:01.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.361 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.361 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:48:01.361 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309813+0000", 2023-10-01T09:48:01.361 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.1979999999999998e-06 2023-10-01T09:48:01.361 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:48:01.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309819+0000", 2023-10-01T09:48:01.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2599999999999996e-06 2023-10-01T09:48:01.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:48:01.362 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309824+0000", 2023-10-01T09:48:01.362 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.010777640999999999 2023-10-01T09:48:01.362 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.362 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.362 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:48:01.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.320602+0000", 2023-10-01T09:48:01.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014988100000000001 2023-10-01T09:48:01.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:48:01.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.320752+0000", 2023-10-01T09:48:01.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000153716 2023-10-01T09:48:01.363 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.363 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.363 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,6", 2023-10-01T09:48:01.363 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.320906+0000", 2023-10-01T09:48:01.363 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0081361090000000007 2023-10-01T09:48:01.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:48:01.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.329042+0000", 2023-10-01T09:48:01.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00074486699999999997 2023-10-01T09:48:01.364 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.364 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.364 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:48:01.364 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.329787+0000", 2023-10-01T09:48:01.364 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.019976207999999999 2023-10-01T09:48:01.364 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.364 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:48:01.365 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.365 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.365 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.365 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:156508 10.3 10:cc4d90bf:::benchmark_data_smithi138_573643_object73286:head [delete] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-01T09:48:01.365 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:48:01.309831+0000", 2023-10-01T09:48:01.365 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.021019282, 2023-10-01T09:48:01.365 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.021431517000000001, 2023-10-01T09:48:01.365 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:48:01.365 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:48:01.365 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:48:01.365 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:48:01.365 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:48:01.366 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 156508 2023-10-01T09:48:01.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.366 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:48:01.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:48:01.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309831+0000", 2023-10-01T09:48:01.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:48:01.366 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.366 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.366 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:48:01.366 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309831+0000", 2023-10-01T09:48:01.366 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:48:01.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.367 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:48:01.367 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309830+0000", 2023-10-01T09:48:01.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.92e-06 2023-10-01T09:48:01.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.367 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:48:01.367 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309833+0000", 2023-10-01T09:48:01.367 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5679999999999996e-06 2023-10-01T09:48:01.367 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.367 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.367 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:48:01.368 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309840+0000", 2023-10-01T09:48:01.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1590000000000003e-06 2023-10-01T09:48:01.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.368 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:48:01.368 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.309844+0000", 2023-10-01T09:48:01.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078857590000000009 2023-10-01T09:48:01.368 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.368 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.368 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:48:01.368 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.317729+0000", 2023-10-01T09:48:01.368 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000167145 2023-10-01T09:48:01.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:48:01.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.317897+0000", 2023-10-01T09:48:01.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000181337 2023-10-01T09:48:01.369 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.369 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.369 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:48:01.369 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.318078+0000", 2023-10-01T09:48:01.369 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0082474290000000006 2023-10-01T09:48:01.369 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.369 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:48:01.370 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.370 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.370 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.370 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:156513 10.9 10:99404447:::benchmark_data_smithi138_573643_object73291:head [delete] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-01T09:48:01.370 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:48:01.319482+0000", 2023-10-01T09:48:01.370 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.01136819, 2023-10-01T09:48:01.370 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.011852592, 2023-10-01T09:48:01.370 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:48:01.370 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:48:01.370 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:48:01.370 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:48:01.370 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:48:01.371 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 156513 2023-10-01T09:48:01.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.371 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:48:01.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:48:01.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.319482+0000", 2023-10-01T09:48:01.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:48:01.371 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.371 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.371 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:48:01.371 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.319482+0000", 2023-10-01T09:48:01.371 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:48:01.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:48:01.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.319480+0000", 2023-10-01T09:48:01.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 7.0029999999999997e-06 2023-10-01T09:48:01.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:48:01.372 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.319487+0000", 2023-10-01T09:48:01.372 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.4277000000000001e-05 2023-10-01T09:48:01.372 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.372 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.372 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:48:01.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.319502+0000", 2023-10-01T09:48:01.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.122e-06 2023-10-01T09:48:01.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:48:01.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.319508+0000", 2023-10-01T09:48:01.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.9069e-05 2023-10-01T09:48:01.373 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.373 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.373 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:48:01.373 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.319567+0000", 2023-10-01T09:48:01.373 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000105556 2023-10-01T09:48:01.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:48:01.374 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.319672+0000", 2023-10-01T09:48:01.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000113385 2023-10-01T09:48:01.374 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.374 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.374 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 4,7", 2023-10-01T09:48:01.374 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.319786+0000", 2023-10-01T09:48:01.374 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000304171 2023-10-01T09:48:01.374 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.374 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:48:01.375 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.375 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.375 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.375 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:156515 10.9 10:92c0e54f:::benchmark_data_smithi138_573643_object73293:head [delete] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-01T09:48:01.375 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:48:01.324453+0000", 2023-10-01T09:48:01.375 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0063971260000000004, 2023-10-01T09:48:01.375 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0069636780000000001, 2023-10-01T09:48:01.375 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:48:01.375 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:48:01.375 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:48:01.375 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:48:01.375 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:48:01.376 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 156515 2023-10-01T09:48:01.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.376 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:48:01.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.376 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:48:01.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324453+0000", 2023-10-01T09:48:01.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:48:01.376 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.376 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.376 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:48:01.376 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324453+0000", 2023-10-01T09:48:01.376 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999981 2023-10-01T09:48:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:48:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324451+0000", 2023-10-01T09:48:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.064e-06 2023-10-01T09:48:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:48:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324460+0000", 2023-10-01T09:48:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.2309e-05 2023-10-01T09:48:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.377 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:48:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324472+0000", 2023-10-01T09:48:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.5860000000000001e-06 2023-10-01T09:48:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:48:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324479+0000", 2023-10-01T09:48:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00050422599999999998 2023-10-01T09:48:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:48:01.378 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324983+0000", 2023-10-01T09:48:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000117527 2023-10-01T09:48:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:48:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.325100+0000", 2023-10-01T09:48:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012687399999999999 2023-10-01T09:48:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 4,7", 2023-10-01T09:48:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.325227+0000", 2023-10-01T09:48:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00077465599999999998 2023-10-01T09:48:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.379 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:48:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:156516 10.9 10:9814d83b:::benchmark_data_smithi138_573643_object73294:head [delete] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-01T09:48:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:48:01.324802+0000", 2023-10-01T09:48:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0060474810000000004, 2023-10-01T09:48:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0066859939999999998, 2023-10-01T09:48:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:48:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:48:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:48:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:48:01.380 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:48:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 156516 2023-10-01T09:48:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:48:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:48:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324802+0000", 2023-10-01T09:48:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:48:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:48:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324802+0000", 2023-10-01T09:48:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:48:01.381 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:48:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324802+0000", 2023-10-01T09:48:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0459999999999999e-06 2023-10-01T09:48:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:48:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324806+0000", 2023-10-01T09:48:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 1.0414000000000001e-05 2023-10-01T09:48:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.382 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:48:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324816+0000", 2023-10-01T09:48:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.938e-06 2023-10-01T09:48:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:48:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324821+0000", 2023-10-01T09:48:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0050871980000000002 2023-10-01T09:48:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:48:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.329908+0000", 2023-10-01T09:48:01.383 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012152200000000001 2023-10-01T09:48:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:48:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.330030+0000", 2023-10-01T09:48:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000131891 2023-10-01T09:48:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 4,7", 2023-10-01T09:48:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.330162+0000", 2023-10-01T09:48:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0053594330000000003 2023-10-01T09:48:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.384 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:48:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop_reply(client.36456.0:156506 10.3 e557/503)", 2023-10-01T09:48:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:48:01.324860+0000", 2023-10-01T09:48:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0059895879999999997, 2023-10-01T09:48:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0066984970000000003, 2023-10-01T09:48:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:48:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "queued for pg", 2023-10-01T09:48:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:48:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:48:01.385 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324860+0000", 2023-10-01T09:48:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:48:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:48:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324860+0000", 2023-10-01T09:48:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967296 2023-10-01T09:48:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:48:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324860+0000", 2023-10-01T09:48:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.362e-06 2023-10-01T09:48:01.386 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:48:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324862+0000", 2023-10-01T09:48:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.5920000000000002e-06 2023-10-01T09:48:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:48:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324867+0000", 2023-10-01T09:48:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1240000000000001e-06 2023-10-01T09:48:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.387 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:48:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.324870+0000", 2023-10-01T09:48:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 9.876e-06 2023-10-01T09:48:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:48:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:156495 10.9 10:943669f1:::benchmark_data_smithi138_573643_object73273:head [delete] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-01T09:48:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:48:01.284439+0000", 2023-10-01T09:48:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.046410895000000001, 2023-10-01T09:48:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.047171288999999998, 2023-10-01T09:48:01.388 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:48:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:48:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:48:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:48:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:48:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 156495 2023-10-01T09:48:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:48:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:48:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284439+0000", 2023-10-01T09:48:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:48:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.389 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:48:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284439+0000", 2023-10-01T09:48:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:48:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:48:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284439+0000", 2023-10-01T09:48:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.328e-06 2023-10-01T09:48:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.390 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:48:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284441+0000", 2023-10-01T09:48:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.1319999999999998e-06 2023-10-01T09:48:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:48:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284444+0000", 2023-10-01T09:48:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3659999999999999e-06 2023-10-01T09:48:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:48:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284447+0000", 2023-10-01T09:48:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.1913999999999999e-05 2023-10-01T09:48:01.391 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:48:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284488+0000", 2023-10-01T09:48:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012547599999999999 2023-10-01T09:48:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:48:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284614+0000", 2023-10-01T09:48:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00014113299999999999 2023-10-01T09:48:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.392 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 4,7", 2023-10-01T09:48:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284755+0000", 2023-10-01T09:48:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.013133114 2023-10-01T09:48:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:48:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.297888+0000", 2023-10-01T09:48:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.020729362000000001 2023-10-01T09:48:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:48:01.393 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.318618+0000", 2023-10-01T09:48:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.034178578000000001 2023-10-01T09:48:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:48:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:156499 10.9 10:9ce33993:::benchmark_data_smithi138_573643_object73277:head [delete] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-01T09:48:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:48:01.284457+0000", 2023-10-01T09:48:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.0463931, 2023-10-01T09:48:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.047239624000000001, 2023-10-01T09:48:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:48:01.394 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:48:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:48:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:48:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:48:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 156499 2023-10-01T09:48:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:48:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:48:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284457+0000", 2023-10-01T09:48:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:48:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.395 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:48:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284457+0000", 2023-10-01T09:48:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999995 2023-10-01T09:48:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:48:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284456+0000", 2023-10-01T09:48:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.695e-06 2023-10-01T09:48:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:48:01.396 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284459+0000", 2023-10-01T09:48:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 6.0440000000000001e-06 2023-10-01T09:48:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:48:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284465+0000", 2023-10-01T09:48:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.0579999999999999e-06 2023-10-01T09:48:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:48:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.284469+0000", 2023-10-01T09:48:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0089361579999999996 2023-10-01T09:48:01.397 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:48:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.293405+0000", 2023-10-01T09:48:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00012636799999999999 2023-10-01T09:48:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:48:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.293532+0000", 2023-10-01T09:48:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00013863999999999999 2023-10-01T09:48:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.398 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 4,7", 2023-10-01T09:48:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.293670+0000", 2023-10-01T09:48:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0078615950000000007 2023-10-01T09:48:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "op_commit", 2023-10-01T09:48:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.301532+0000", 2023-10-01T09:48:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.017485434000000001 2023-10-01T09:48:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:48:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.319017+0000", 2023-10-01T09:48:01.399 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.034560601000000003 2023-10-01T09:48:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:48:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_repop_reply(client.36456.0:156505 10.7 e557/503 ondisk, result = 0)", 2023-10-01T09:48:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:48:01.330320+0000", 2023-10-01T09:48:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.00053029200000000002, 2023-10-01T09:48:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0014577870000000001, 2023-10-01T09:48:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:48:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "started", 2023-10-01T09:48:01.400 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:48:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:48:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.330320+0000", 2023-10-01T09:48:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:48:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:48:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.330320+0000", 2023-10-01T09:48:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967296 2023-10-01T09:48:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.401 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:48:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.330319+0000", 2023-10-01T09:48:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 2.3350000000000001e-06 2023-10-01T09:48:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:48:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.330322+0000", 2023-10-01T09:48:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.1800000000000004e-06 2023-10-01T09:48:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:48:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.330327+0000", 2023-10-01T09:48:01.402 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 3.3689999999999998e-06 2023-10-01T09:48:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:48:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.330330+0000", 2023-10-01T09:48:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0012379279999999999 2023-10-01T09:48:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:48:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.331568+0000", 2023-10-01T09:48:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0269999999999995e-06 2023-10-01T09:48:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.403 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:48:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.331576+0000", 2023-10-01T09:48:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0012566089999999999 2023-10-01T09:48:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:48:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: "description": "osd_op(client.36456.0:156501 10.3 10:c2f4bc21:::benchmark_data_smithi138_573643_object73279:head [delete] snapc 0=[] ondisk+write+known_if_redirected e557)", 2023-10-01T09:48:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: "initiated_at": "2023-10-01T09:48:01.302633+0000", 2023-10-01T09:48:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: "age": 0.028216419999999999, 2023-10-01T09:48:01.404 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.029208791000000001, 2023-10-01T09:48:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "type_data": { 2023-10-01T09:48:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:48:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "client_info": { 2023-10-01T09:48:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "client": "client.36456", 2023-10-01T09:48:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:48:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "tid": 156501 2023-10-01T09:48:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "events": [ 2023-10-01T09:48:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "initiated", 2023-10-01T09:48:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.302633+0000", 2023-10-01T09:48:01.405 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0 2023-10-01T09:48:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "throttled", 2023-10-01T09:48:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.302633+0000", 2023-10-01T09:48:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4294967295.9999986 2023-10-01T09:48:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "header_read", 2023-10-01T09:48:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.302632+0000", 2023-10-01T09:48:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 5.2530000000000004e-06 2023-10-01T09:48:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.406 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "all_read", 2023-10-01T09:48:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.302637+0000", 2023-10-01T09:48:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.0050000000000004e-06 2023-10-01T09:48:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "dispatched", 2023-10-01T09:48:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.302645+0000", 2023-10-01T09:48:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 4.7589999999999997e-06 2023-10-01T09:48:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "queued_for_pg", 2023-10-01T09:48:01.407 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.302650+0000", 2023-10-01T09:48:01.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.0095978469999999996 2023-10-01T09:48:01.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:48:01.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.312248+0000", 2023-10-01T09:48:01.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.00010915499999999999 2023-10-01T09:48:01.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.408 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.408 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for rw locks", 2023-10-01T09:48:01.408 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.312357+0000", 2023-10-01T09:48:01.408 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.014470930999999999 2023-10-01T09:48:01.408 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "reached_pg", 2023-10-01T09:48:01.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.326828+0000", 2023-10-01T09:48:01.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 8.8547000000000005e-05 2023-10-01T09:48:01.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "started", 2023-10-01T09:48:01.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.326916+0000", 2023-10-01T09:48:01.409 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.000151552 2023-10-01T09:48:01.409 INFO:teuthology.orchestra.run.smithi190.stdout: }, 2023-10-01T09:48:01.409 INFO:teuthology.orchestra.run.smithi190.stdout: { 2023-10-01T09:48:01.409 INFO:teuthology.orchestra.run.smithi190.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:48:01.409 INFO:teuthology.orchestra.run.smithi190.stdout: "time": "2023-10-01T09:48:01.327068+0000", 2023-10-01T09:48:01.410 INFO:teuthology.orchestra.run.smithi190.stdout: "duration": 0.024434593000000001 2023-10-01T09:48:01.410 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.410 INFO:teuthology.orchestra.run.smithi190.stdout: ] 2023-10-01T09:48:01.410 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.410 INFO:teuthology.orchestra.run.smithi190.stdout: } 2023-10-01T09:48:01.410 INFO:teuthology.orchestra.run.smithi190.stdout: ], 2023-10-01T09:48:01.410 INFO:teuthology.orchestra.run.smithi190.stdout: "num_ops": 13 2023-10-01T09:48:01.410 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:48:01.410 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:01.442 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:01 smithi138 ceph-mon[192513]: from='client.47501 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:48:01.443 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:01 smithi138 ceph-mon[192513]: pgmap v1677: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 768 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 5086/36714 objects degraded (13.853%); 664/36714 objects misplaced (1.809%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:48:01.443 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:01 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:48:01.443 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:01 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:48:01.443 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:01 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:01.443 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:01 smithi138 ceph-mon[185886]: from='client.47501 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:48:01.443 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:01 smithi138 ceph-mon[185886]: pgmap v1677: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 768 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 5086/36714 objects degraded (13.853%); 664/36714 objects misplaced (1.809%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:48:01.443 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:01 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:48:01.443 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:01 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:48:01.443 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:01 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:01.445 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:01.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:01 smithi190 ceph-mon[160435]: from='client.47501 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:48:01.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:01 smithi190 ceph-mon[160435]: pgmap v1677: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 768 MiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 5086/36714 objects degraded (13.853%); 664/36714 objects misplaced (1.809%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:48:01.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:01 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:48:01.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:01 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:48:01.563 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:01 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:01.625 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:01.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:01.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:01.640 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1676,"stamp":"2023-10-01T09:47:58.384858+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":830937035,"num_objects":12635,"num_object_clones":0,"num_object_copies":37905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5086,"num_objects_misplaced":664,"num_objects_unfound":0,"num_objects_dirty":12635,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":154939,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11920,"num_bytes_recovered":768050191,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129103,"ondisk_log_size":129103,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8524272,"kb_used_data":3422456,"kb_used_omap":39,"kb_used_meta":5100824,"kb_avail":647691792,"statfs":{"total":671965249536,"available":663236395008,"internally_reserved":0,"allocated":3504594944,"data_stored":3494544011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5223244399},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-378535936,"num_objects":-5776,"num_object_clones":0,"num_object_copies":-17328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":147,"num_objects_misplaced":-368,"num_objects_unfound":0,"num_objects_dirty":-5776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5776,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":221,"num_bytes_recovered":14483456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.368125"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":332,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.524151+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:47:43.524151+0000","last_peered":"2023-10-01T09:47:43.524151+0000","last_clean":"2023-10-01T09:47:43.524151+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:47:43.524151+0000","last_undegraded":"2023-10-01T09:47:43.524151+0000","last_fullsized":"2023-10-01T09:47:43.524151+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":177,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:45.289364+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:47:45.289364+0000","last_peered":"2023-10-01T09:47:45.289364+0000","last_clean":"2023-10-01T09:47:45.289364+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:47:45.289364+0000","last_undegraded":"2023-10-01T09:47:45.289364+0000","last_fullsized":"2023-10-01T09:47:45.289364+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":557,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.515153+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:47:43.515153+0000","last_peered":"2023-10-01T09:47:43.515153+0000","last_clean":"2023-10-01T09:47:43.515153+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:47:43.515153+0000","last_undegraded":"2023-10-01T09:47:43.515153+0000","last_fullsized":"2023-10-01T09:47:43.515153+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":616,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.517462+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:47:43.517462+0000","last_peered":"2023-10-01T09:47:43.517462+0000","last_clean":"2023-10-01T09:47:43.517462+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:47:43.517462+0000","last_undegraded":"2023-10-01T09:47:43.517462+0000","last_fullsized":"2023-10-01T09:47:43.517462+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":497,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.519502+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:47:43.519502+0000","last_peered":"2023-10-01T09:47:43.519502+0000","last_clean":"2023-10-01T09:47:43.519502+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:47:43.519502+0000","last_undegraded":"2023-10-01T09:47:43.519502+0000","last_fullsized":"2023-10-01T09:47:43.519502+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":238,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.049444+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:47:50.049444+0000","last_peered":"2023-10-01T09:47:50.049444+0000","last_clean":"2023-10-01T09:47:50.049444+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:47:50.049444+0000","last_undegraded":"2023-10-01T09:47:50.049444+0000","last_fullsized":"2023-10-01T09:47:50.049444+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":614,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.049449+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:47:50.049449+0000","last_peered":"2023-10-01T09:47:50.049449+0000","last_clean":"2023-10-01T09:47:50.049449+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:47:50.049449+0000","last_undegraded":"2023-10-01T09:47:50.049449+0000","last_fullsized":"2023-10-01T09:47:50.049449+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":545,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.018366+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:47:50.018366+0000","last_peered":"2023-10-01T09:47:50.018366+0000","last_clean":"2023-10-01T09:47:50.018366+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:47:50.018366+0000","last_undegraded":"2023-10-01T09:47:50.018366+0000","last_fullsized":"2023-10-01T09:47:50.018366+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":269,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.046314+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:47:50.046314+0000","last_peered":"2023-10-01T09:47:50.046314+0000","last_clean":"2023-10-01T09:47:50.046314+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:47:50.046314+0000","last_undegraded":"2023-10-01T09:47:50.046314+0000","last_fullsized":"2023-10-01T09:47:50.046314+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":75,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.520921+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:47:43.520921+0000","last_peered":"2023-10-01T09:47:43.520921+0000","last_clean":"2023-10-01T09:47:43.520921+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:47:43.520921+0000","last_undegraded":"2023-10-01T09:47:43.520921+0000","last_fullsized":"2023-10-01T09:47:43.520921+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":81,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.523031+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:47:43.523031+0000","last_peered":"2023-10-01T09:47:43.523031+0000","last_clean":"2023-10-01T09:47:43.523031+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:47:43.523031+0000","last_undegraded":"2023-10-01T09:47:43.523031+0000","last_fullsized":"2023-10-01T09:47:43.523031+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":81,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:45.289282+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:47:45.289282+0000","last_peered":"2023-10-01T09:47:45.289282+0000","last_clean":"2023-10-01T09:47:45.289282+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:47:45.289282+0000","last_undegraded":"2023-10-01T09:47:45.289282+0000","last_fullsized":"2023-10-01T09:47:45.289282+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":457,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.524029+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:47:43.524029+0000","last_peered":"2023-10-01T09:47:43.524029+0000","last_clean":"2023-10-01T09:47:43.524029+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:47:43.524029+0000","last_undegraded":"2023-10-01T09:47:43.524029+0000","last_fullsized":"2023-10-01T09:47:43.524029+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":338,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.523465+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:47:43.523465+0000","last_peered":"2023-10-01T09:47:43.523465+0000","last_clean":"2023-10-01T09:47:43.523465+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:47:43.523465+0000","last_undegraded":"2023-10-01T09:47:43.523465+0000","last_fullsized":"2023-10-01T09:47:43.523465+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":674,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.521014+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:47:43.521014+0000","last_peered":"2023-10-01T09:47:43.521014+0000","last_clean":"2023-10-01T09:47:43.521014+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:47:43.521014+0000","last_undegraded":"2023-10-01T09:47:43.521014+0000","last_fullsized":"2023-10-01T09:47:43.521014+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":411,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.521268+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:47:43.521268+0000","last_peered":"2023-10-01T09:47:43.521268+0000","last_clean":"2023-10-01T09:47:43.521268+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:47:43.521268+0000","last_undegraded":"2023-10-01T09:47:43.521268+0000","last_fullsized":"2023-10-01T09:47:43.521268+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":84,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.520862+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:47:43.520862+0000","last_peered":"2023-10-01T09:47:43.520862+0000","last_clean":"2023-10-01T09:47:43.520862+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:47:43.520862+0000","last_undegraded":"2023-10-01T09:47:43.520862+0000","last_fullsized":"2023-10-01T09:47:43.520862+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":71,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.045705+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:47:50.045705+0000","last_peered":"2023-10-01T09:47:50.045705+0000","last_clean":"2023-10-01T09:47:50.045705+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:47:50.045705+0000","last_undegraded":"2023-10-01T09:47:50.045705+0000","last_fullsized":"2023-10-01T09:47:50.045705+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":567,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.521013+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:47:43.521013+0000","last_peered":"2023-10-01T09:47:43.521013+0000","last_clean":"2023-10-01T09:47:43.521013+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:47:43.521013+0000","last_undegraded":"2023-10-01T09:47:43.521013+0000","last_fullsized":"2023-10-01T09:47:43.521013+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":580,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.517528+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:47:43.517528+0000","last_peered":"2023-10-01T09:47:43.517528+0000","last_clean":"2023-10-01T09:47:43.517528+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:47:43.517528+0000","last_undegraded":"2023-10-01T09:47:43.517528+0000","last_fullsized":"2023-10-01T09:47:43.517528+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":436,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.519374+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:47:43.519374+0000","last_peered":"2023-10-01T09:47:43.519374+0000","last_clean":"2023-10-01T09:47:43.519374+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:47:43.519374+0000","last_undegraded":"2023-10-01T09:47:43.519374+0000","last_fullsized":"2023-10-01T09:47:43.519374+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":84,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.524596+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:47:43.524596+0000","last_peered":"2023-10-01T09:47:43.524596+0000","last_clean":"2023-10-01T09:47:43.524596+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:47:43.524596+0000","last_undegraded":"2023-10-01T09:47:43.524596+0000","last_fullsized":"2023-10-01T09:47:43.524596+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"557'9640","reported_seq":9729,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.722821+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:47:57.722821+0000","last_peered":"2023-10-01T09:47:57.722821+0000","last_clean":"2023-10-01T09:47:57.722821+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:47:57.722821+0000","last_undegraded":"2023-10-01T09:47:57.722821+0000","last_fullsized":"2023-10-01T09:47:57.722821+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":9640,"log_dups_size":0,"ondisk_log_size":9640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49283072,"num_objects":752,"num_object_clones":0,"num_object_copies":2256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9640,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":79,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.524448+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:47:43.524448+0000","last_peered":"2023-10-01T09:47:43.524448+0000","last_clean":"2023-10-01T09:47:43.524448+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:47:43.524448+0000","last_undegraded":"2023-10-01T09:47:43.524448+0000","last_fullsized":"2023-10-01T09:47:43.524448+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":277,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.045545+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:47:50.045545+0000","last_peered":"2023-10-01T09:47:50.045545+0000","last_clean":"2023-10-01T09:47:50.045545+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:47:50.045545+0000","last_undegraded":"2023-10-01T09:47:50.045545+0000","last_fullsized":"2023-10-01T09:47:50.045545+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"557'9697","reported_seq":20897,"reported_epoch":557,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:47:57.720494+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:47:57.720494+0000","last_peered":"2023-10-01T09:47:57.720494+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:47:57.720494+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":9697,"log_dups_size":0,"ondisk_log_size":9697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46465024,"num_objects":709,"num_object_clones":0,"num_object_copies":2127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1437,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9697,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3698,"num_bytes_recovered":242352128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":641},{"shards":"5,6","objects":-728}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":550,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.523476+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:47:43.523476+0000","last_peered":"2023-10-01T09:47:43.523476+0000","last_clean":"2023-10-01T09:47:43.523476+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:47:43.523476+0000","last_undegraded":"2023-10-01T09:47:43.523476+0000","last_fullsized":"2023-10-01T09:47:43.523476+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":613,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.515040+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:47:43.515040+0000","last_peered":"2023-10-01T09:47:43.515040+0000","last_clean":"2023-10-01T09:47:43.515040+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:47:43.515040+0000","last_undegraded":"2023-10-01T09:47:43.515040+0000","last_fullsized":"2023-10-01T09:47:43.515040+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"556'9642","reported_seq":9735,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:56.917390+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:47:56.917390+0000","last_peered":"2023-10-01T09:47:56.917390+0000","last_clean":"2023-10-01T09:47:56.917390+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:47:56.917390+0000","last_undegraded":"2023-10-01T09:47:56.917390+0000","last_fullsized":"2023-10-01T09:47:56.917390+0000","mapping_epoch":475,"log_start":"486'2600","ondisk_log_start":"486'2600","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7042,"log_dups_size":0,"ondisk_log_size":7042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51773440,"num_objects":790,"num_object_clones":0,"num_object_copies":2370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9642,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":636,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.523092+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:47:43.523092+0000","last_peered":"2023-10-01T09:47:43.523092+0000","last_clean":"2023-10-01T09:47:43.523092+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:47:43.523092+0000","last_undegraded":"2023-10-01T09:47:43.523092+0000","last_fullsized":"2023-10-01T09:47:43.523092+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":433,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.515476+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:47:43.515476+0000","last_peered":"2023-10-01T09:47:43.515476+0000","last_clean":"2023-10-01T09:47:43.515476+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:47:43.515476+0000","last_undegraded":"2023-10-01T09:47:43.515476+0000","last_fullsized":"2023-10-01T09:47:43.515476+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"556'9658","reported_seq":16356,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:56.929857+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:47:56.929857+0000","last_peered":"2023-10-01T09:47:56.929857+0000","last_clean":"2023-10-01T09:47:56.929857+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:47:56.929857+0000","last_undegraded":"2023-10-01T09:47:56.929857+0000","last_fullsized":"2023-10-01T09:47:56.929857+0000","mapping_epoch":505,"log_start":"486'2600","ondisk_log_start":"486'2600","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7058,"log_dups_size":0,"ondisk_log_size":7058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48758784,"num_objects":744,"num_object_clones":0,"num_object_copies":2232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9658,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":277,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.045994+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:47:50.045994+0000","last_peered":"2023-10-01T09:47:50.045994+0000","last_clean":"2023-10-01T09:47:50.045994+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:47:50.045994+0000","last_undegraded":"2023-10-01T09:47:50.045994+0000","last_fullsized":"2023-10-01T09:47:50.045994+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":536,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.518881+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:47:43.518881+0000","last_peered":"2023-10-01T09:47:43.518881+0000","last_clean":"2023-10-01T09:47:43.518881+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:47:43.518881+0000","last_undegraded":"2023-10-01T09:47:43.518881+0000","last_fullsized":"2023-10-01T09:47:43.518881+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"556'9610","reported_seq":9703,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:55.132853+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:47:55.132853+0000","last_peered":"2023-10-01T09:47:55.132853+0000","last_clean":"2023-10-01T09:47:55.132853+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:47:55.132853+0000","last_undegraded":"2023-10-01T09:47:55.132853+0000","last_fullsized":"2023-10-01T09:47:55.132853+0000","mapping_epoch":475,"log_start":"485'2100","ondisk_log_start":"485'2100","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7510,"log_dups_size":0,"ondisk_log_size":7510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58851328,"num_objects":898,"num_object_clones":0,"num_object_copies":2694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9610,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":136,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.523780+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:47:43.523780+0000","last_peered":"2023-10-01T09:47:43.523780+0000","last_clean":"2023-10-01T09:47:43.523780+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:47:43.523780+0000","last_undegraded":"2023-10-01T09:47:43.523780+0000","last_fullsized":"2023-10-01T09:47:43.523780+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":210,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.524163+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:47:43.524163+0000","last_peered":"2023-10-01T09:47:43.524163+0000","last_clean":"2023-10-01T09:47:43.524163+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:47:43.524163+0000","last_undegraded":"2023-10-01T09:47:43.524163+0000","last_fullsized":"2023-10-01T09:47:43.524163+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"557'9639","reported_seq":11373,"reported_epoch":557,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:47:57.744308+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:47:57.744308+0000","last_peered":"2023-10-01T09:47:57.744308+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:47:57.744308+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":9639,"log_dups_size":0,"ondisk_log_size":9639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47775744,"num_objects":729,"num_object_clones":0,"num_object_copies":2187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3639,"num_objects_misplaced":664,"num_objects_unfound":0,"num_objects_dirty":729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9639,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":818,"num_bytes_recovered":53608448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":664},{"shards":"4,5","objects":-3574}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":454,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.524283+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:47:43.524283+0000","last_peered":"2023-10-01T09:47:43.524283+0000","last_clean":"2023-10-01T09:47:43.524283+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:47:43.524283+0000","last_undegraded":"2023-10-01T09:47:43.524283+0000","last_fullsized":"2023-10-01T09:47:43.524283+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":616,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.515859+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:47:43.515859+0000","last_peered":"2023-10-01T09:47:43.515859+0000","last_clean":"2023-10-01T09:47:43.515859+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:47:43.515859+0000","last_undegraded":"2023-10-01T09:47:43.515859+0000","last_fullsized":"2023-10-01T09:47:43.515859+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"557'9675","reported_seq":9774,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.638788+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:47:57.638788+0000","last_peered":"2023-10-01T09:47:57.638788+0000","last_clean":"2023-10-01T09:47:57.638788+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:47:57.638788+0000","last_undegraded":"2023-10-01T09:47:57.638788+0000","last_fullsized":"2023-10-01T09:47:57.638788+0000","mapping_epoch":483,"log_start":"485'2100","ondisk_log_start":"485'2100","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7575,"log_dups_size":0,"ondisk_log_size":7575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46071808,"num_objects":703,"num_object_clones":0,"num_object_copies":2109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9675,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":450,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.525277+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:47:43.525277+0000","last_peered":"2023-10-01T09:47:43.525277+0000","last_clean":"2023-10-01T09:47:43.525277+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:47:43.525277+0000","last_undegraded":"2023-10-01T09:47:43.525277+0000","last_fullsized":"2023-10-01T09:47:43.525277+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":291,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.524777+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:47:43.524777+0000","last_peered":"2023-10-01T09:47:43.524777+0000","last_clean":"2023-10-01T09:47:43.524777+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:47:43.524777+0000","last_undegraded":"2023-10-01T09:47:43.524777+0000","last_fullsized":"2023-10-01T09:47:43.524777+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"556'9721","reported_seq":9810,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:56.298468+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:47:56.298468+0000","last_peered":"2023-10-01T09:47:56.298468+0000","last_clean":"2023-10-01T09:47:56.298468+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:47:56.298468+0000","last_undegraded":"2023-10-01T09:47:56.298468+0000","last_fullsized":"2023-10-01T09:47:56.298468+0000","mapping_epoch":475,"log_start":"482'1300","ondisk_log_start":"482'1300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8421,"log_dups_size":0,"ondisk_log_size":8421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50921472,"num_objects":777,"num_object_clones":0,"num_object_copies":2331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9721,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":420,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.515154+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:47:43.515154+0000","last_peered":"2023-10-01T09:47:43.515154+0000","last_clean":"2023-10-01T09:47:43.515154+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:47:43.515154+0000","last_undegraded":"2023-10-01T09:47:43.515154+0000","last_fullsized":"2023-10-01T09:47:43.515154+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":468,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.047144+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:47:50.047144+0000","last_peered":"2023-10-01T09:47:50.047144+0000","last_clean":"2023-10-01T09:47:50.047144+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:47:50.047144+0000","last_undegraded":"2023-10-01T09:47:50.047144+0000","last_fullsized":"2023-10-01T09:47:50.047144+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"556'9418","reported_seq":9507,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:56.314658+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:47:56.314658+0000","last_peered":"2023-10-01T09:47:56.314658+0000","last_clean":"2023-10-01T09:47:56.314658+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:47:56.314658+0000","last_undegraded":"2023-10-01T09:47:56.314658+0000","last_fullsized":"2023-10-01T09:47:56.314658+0000","mapping_epoch":475,"log_start":"481'1000","ondisk_log_start":"481'1000","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8418,"log_dups_size":0,"ondisk_log_size":8418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49807360,"num_objects":760,"num_object_clones":0,"num_object_copies":2280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9418,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":949,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.521194+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:47:43.521194+0000","last_peered":"2023-10-01T09:47:43.521194+0000","last_clean":"2023-10-01T09:47:43.521194+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:47:43.521194+0000","last_undegraded":"2023-10-01T09:47:43.521194+0000","last_fullsized":"2023-10-01T09:47:43.521194+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":287,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.045841+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:47:50.045841+0000","last_peered":"2023-10-01T09:47:50.045841+0000","last_clean":"2023-10-01T09:47:50.045841+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:47:50.045841+0000","last_undegraded":"2023-10-01T09:47:50.045841+0000","last_fullsized":"2023-10-01T09:47:50.045841+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":81,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.515334+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:47:43.515334+0000","last_peered":"2023-10-01T09:47:43.515334+0000","last_clean":"2023-10-01T09:47:43.515334+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:47:43.515334+0000","last_undegraded":"2023-10-01T09:47:43.515334+0000","last_fullsized":"2023-10-01T09:47:43.515334+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"556'9576","reported_seq":9668,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:56.918142+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:47:56.918142+0000","last_peered":"2023-10-01T09:47:56.918142+0000","last_clean":"2023-10-01T09:47:56.918142+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:47:56.918142+0000","last_undegraded":"2023-10-01T09:47:56.918142+0000","last_fullsized":"2023-10-01T09:47:56.918142+0000","mapping_epoch":475,"log_start":"486'2500","ondisk_log_start":"486'2500","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52035584,"num_objects":794,"num_object_clones":0,"num_object_copies":2382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9576,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1222,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.045690+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:47:50.045690+0000","last_peered":"2023-10-01T09:47:50.045690+0000","last_clean":"2023-10-01T09:47:50.045690+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:47:50.045690+0000","last_undegraded":"2023-10-01T09:47:50.045690+0000","last_fullsized":"2023-10-01T09:47:50.045690+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":431,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.523646+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:47:43.523646+0000","last_peered":"2023-10-01T09:47:43.523646+0000","last_clean":"2023-10-01T09:47:43.523646+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:47:43.523646+0000","last_undegraded":"2023-10-01T09:47:43.523646+0000","last_fullsized":"2023-10-01T09:47:43.523646+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":260,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.046128+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:47:50.046128+0000","last_peered":"2023-10-01T09:47:50.046128+0000","last_clean":"2023-10-01T09:47:50.046128+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:47:50.046128+0000","last_undegraded":"2023-10-01T09:47:50.046128+0000","last_fullsized":"2023-10-01T09:47:50.046128+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"557'9546","reported_seq":16855,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.706190+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:47:57.706190+0000","last_peered":"2023-10-01T09:47:57.706190+0000","last_clean":"2023-10-01T09:47:57.706190+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:47:57.706190+0000","last_undegraded":"2023-10-01T09:47:57.706190+0000","last_fullsized":"2023-10-01T09:47:57.706190+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":9546,"log_dups_size":0,"ondisk_log_size":9546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54394880,"num_objects":830,"num_object_clones":0,"num_object_copies":2490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9546,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":903,"reported_epoch":555,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:47:43.516740+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:47:43.516740+0000","last_peered":"2023-10-01T09:47:43.516740+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:47:43.516740+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:47:43.516740+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":620,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.515253+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:47:43.515253+0000","last_peered":"2023-10-01T09:47:43.515253+0000","last_clean":"2023-10-01T09:47:43.515253+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:47:43.515253+0000","last_undegraded":"2023-10-01T09:47:43.515253+0000","last_fullsized":"2023-10-01T09:47:43.515253+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":81,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.515431+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:47:43.515431+0000","last_peered":"2023-10-01T09:47:43.515431+0000","last_clean":"2023-10-01T09:47:43.515431+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:47:43.515431+0000","last_undegraded":"2023-10-01T09:47:43.515431+0000","last_fullsized":"2023-10-01T09:47:43.515431+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"556'9434","reported_seq":9527,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:55.093314+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:47:55.093314+0000","last_peered":"2023-10-01T09:47:55.093314+0000","last_clean":"2023-10-01T09:47:55.093314+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:47:55.093314+0000","last_undegraded":"2023-10-01T09:47:55.093314+0000","last_fullsized":"2023-10-01T09:47:55.093314+0000","mapping_epoch":475,"log_start":"484'1900","ondisk_log_start":"484'1900","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7534,"log_dups_size":0,"ondisk_log_size":7534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54919168,"num_objects":838,"num_object_clones":0,"num_object_copies":2514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9434,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1673,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.046947+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:47:50.046947+0000","last_peered":"2023-10-01T09:47:50.046947+0000","last_clean":"2023-10-01T09:47:50.046947+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:47:50.046947+0000","last_undegraded":"2023-10-01T09:47:50.046947+0000","last_fullsized":"2023-10-01T09:47:50.046947+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":586,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.520742+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:47:43.520742+0000","last_peered":"2023-10-01T09:47:43.520742+0000","last_clean":"2023-10-01T09:47:43.520742+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:47:43.520742+0000","last_undegraded":"2023-10-01T09:47:43.520742+0000","last_fullsized":"2023-10-01T09:47:43.520742+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":365,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.045771+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:47:50.045771+0000","last_peered":"2023-10-01T09:47:50.045771+0000","last_clean":"2023-10-01T09:47:50.045771+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:47:50.045771+0000","last_undegraded":"2023-10-01T09:47:50.045771+0000","last_fullsized":"2023-10-01T09:47:50.045771+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":354,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.524088+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:47:43.524088+0000","last_peered":"2023-10-01T09:47:43.524088+0000","last_clean":"2023-10-01T09:47:43.524088+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:47:43.524088+0000","last_undegraded":"2023-10-01T09:47:43.524088+0000","last_fullsized":"2023-10-01T09:47:43.524088+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"556'9742","reported_seq":9831,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:56.287802+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:47:56.287802+0000","last_peered":"2023-10-01T09:47:56.287802+0000","last_clean":"2023-10-01T09:47:56.287802+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:47:56.287802+0000","last_undegraded":"2023-10-01T09:47:56.287802+0000","last_fullsized":"2023-10-01T09:47:56.287802+0000","mapping_epoch":475,"log_start":"482'1400","ondisk_log_start":"482'1400","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8342,"log_dups_size":0,"ondisk_log_size":8342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51773440,"num_objects":790,"num_object_clones":0,"num_object_copies":2370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9742,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":763,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.520828+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:47:43.520828+0000","last_peered":"2023-10-01T09:47:43.520828+0000","last_clean":"2023-10-01T09:47:43.520828+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:47:43.520828+0000","last_undegraded":"2023-10-01T09:47:43.520828+0000","last_fullsized":"2023-10-01T09:47:43.520828+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":597,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.516986+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:47:43.516986+0000","last_peered":"2023-10-01T09:47:43.516986+0000","last_clean":"2023-10-01T09:47:43.516986+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:47:43.516986+0000","last_undegraded":"2023-10-01T09:47:43.516986+0000","last_fullsized":"2023-10-01T09:47:43.516986+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":597,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:46.934170+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:47:46.934170+0000","last_peered":"2023-10-01T09:47:46.934170+0000","last_clean":"2023-10-01T09:47:46.934170+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:47:46.934170+0000","last_undegraded":"2023-10-01T09:47:46.934170+0000","last_fullsized":"2023-10-01T09:47:46.934170+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"556'9621","reported_seq":9715,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:56.874993+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:47:56.874993+0000","last_peered":"2023-10-01T09:47:56.874993+0000","last_clean":"2023-10-01T09:47:56.874993+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:47:56.874993+0000","last_undegraded":"2023-10-01T09:47:56.874993+0000","last_fullsized":"2023-10-01T09:47:56.874993+0000","mapping_epoch":475,"log_start":"486'2600","ondisk_log_start":"486'2600","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7021,"log_dups_size":0,"ondisk_log_size":7021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54919192,"num_objects":839,"num_object_clones":0,"num_object_copies":2517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":839,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9621,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":994,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.523185+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:47:43.523185+0000","last_peered":"2023-10-01T09:47:43.523185+0000","last_clean":"2023-10-01T09:47:43.523185+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:47:43.523185+0000","last_undegraded":"2023-10-01T09:47:43.523185+0000","last_fullsized":"2023-10-01T09:47:43.523185+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":614,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.046063+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:47:50.046063+0000","last_peered":"2023-10-01T09:47:50.046063+0000","last_clean":"2023-10-01T09:47:50.046063+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:47:50.046063+0000","last_undegraded":"2023-10-01T09:47:50.046063+0000","last_fullsized":"2023-10-01T09:47:50.046063+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":467,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.047032+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:47:50.047032+0000","last_peered":"2023-10-01T09:47:50.047032+0000","last_clean":"2023-10-01T09:47:50.047032+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:47:50.047032+0000","last_undegraded":"2023-10-01T09:47:50.047032+0000","last_fullsized":"2023-10-01T09:47:50.047032+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"556'9542","reported_seq":9639,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:56.303588+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:47:56.303588+0000","last_peered":"2023-10-01T09:47:56.303588+0000","last_clean":"2023-10-01T09:47:56.303588+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:47:56.303588+0000","last_undegraded":"2023-10-01T09:47:56.303588+0000","last_fullsized":"2023-10-01T09:47:56.303588+0000","mapping_epoch":479,"log_start":"487'2800","ondisk_log_start":"487'2800","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6742,"log_dups_size":0,"ondisk_log_size":6742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53346304,"num_objects":814,"num_object_clones":0,"num_object_copies":2442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9542,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":495,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.524673+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:47:43.524673+0000","last_peered":"2023-10-01T09:47:43.524673+0000","last_clean":"2023-10-01T09:47:43.524673+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:47:43.524673+0000","last_undegraded":"2023-10-01T09:47:43.524673+0000","last_fullsized":"2023-10-01T09:47:43.524673+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1190,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.516252+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:47:43.516252+0000","last_peered":"2023-10-01T09:47:43.516252+0000","last_clean":"2023-10-01T09:47:43.516252+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:47:43.516252+0000","last_undegraded":"2023-10-01T09:47:43.516252+0000","last_fullsized":"2023-10-01T09:47:43.516252+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":84,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.520684+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:47:43.520684+0000","last_peered":"2023-10-01T09:47:43.520684+0000","last_clean":"2023-10-01T09:47:43.520684+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:47:43.520684+0000","last_undegraded":"2023-10-01T09:47:43.520684+0000","last_fullsized":"2023-10-01T09:47:43.520684+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"556'9672","reported_seq":9769,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:56.329255+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:47:56.329255+0000","last_peered":"2023-10-01T09:47:56.329255+0000","last_clean":"2023-10-01T09:47:56.329255+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:47:56.329255+0000","last_undegraded":"2023-10-01T09:47:56.329255+0000","last_fullsized":"2023-10-01T09:47:56.329255+0000","mapping_epoch":479,"log_start":"489'3000","ondisk_log_start":"489'3000","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6672,"log_dups_size":0,"ondisk_log_size":6672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55050240,"num_objects":840,"num_object_clones":0,"num_object_copies":2520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9672,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":88,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.524528+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:47:43.524528+0000","last_peered":"2023-10-01T09:47:43.524528+0000","last_clean":"2023-10-01T09:47:43.524528+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:47:43.524528+0000","last_undegraded":"2023-10-01T09:47:43.524528+0000","last_fullsized":"2023-10-01T09:47:43.524528+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":575,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:46.934195+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:47:46.934195+0000","last_peered":"2023-10-01T09:47:46.934195+0000","last_clean":"2023-10-01T09:47:46.934195+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:47:46.934195+0000","last_undegraded":"2023-10-01T09:47:46.934195+0000","last_fullsized":"2023-10-01T09:47:46.934195+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":280,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.045686+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:47:50.045686+0000","last_peered":"2023-10-01T09:47:50.045686+0000","last_clean":"2023-10-01T09:47:50.045686+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:47:50.045686+0000","last_undegraded":"2023-10-01T09:47:50.045686+0000","last_fullsized":"2023-10-01T09:47:50.045686+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":523,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.519026+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:47:43.519026+0000","last_peered":"2023-10-01T09:47:43.519026+0000","last_clean":"2023-10-01T09:47:43.519026+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:47:43.519026+0000","last_undegraded":"2023-10-01T09:47:43.519026+0000","last_fullsized":"2023-10-01T09:47:43.519026+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":637,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.515622+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:47:43.515622+0000","last_peered":"2023-10-01T09:47:43.515622+0000","last_clean":"2023-10-01T09:47:43.515622+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:47:43.515622+0000","last_undegraded":"2023-10-01T09:47:43.515622+0000","last_fullsized":"2023-10-01T09:47:43.515622+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":292,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.045551+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:47:50.045551+0000","last_peered":"2023-10-01T09:47:50.045551+0000","last_clean":"2023-10-01T09:47:50.045551+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:47:50.045551+0000","last_undegraded":"2023-10-01T09:47:50.045551+0000","last_fullsized":"2023-10-01T09:47:50.045551+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":80,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.516312+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:47:43.516312+0000","last_peered":"2023-10-01T09:47:43.516312+0000","last_clean":"2023-10-01T09:47:43.516312+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:47:43.516312+0000","last_undegraded":"2023-10-01T09:47:43.516312+0000","last_fullsized":"2023-10-01T09:47:43.516312+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":292,"reported_epoch":556,"state":"active+clean","last_fresh":"2023-10-01T09:47:50.045916+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:47:50.045916+0000","last_peered":"2023-10-01T09:47:50.045916+0000","last_clean":"2023-10-01T09:47:50.045916+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:47:50.045916+0000","last_undegraded":"2023-10-01T09:47:50.045916+0000","last_fullsized":"2023-10-01T09:47:50.045916+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":469,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.516991+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:47:43.516991+0000","last_peered":"2023-10-01T09:47:43.516991+0000","last_clean":"2023-10-01T09:47:43.516991+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:47:43.516991+0000","last_undegraded":"2023-10-01T09:47:43.516991+0000","last_fullsized":"2023-10-01T09:47:43.516991+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":98,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.516023+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:47:43.516023+0000","last_peered":"2023-10-01T09:47:43.516023+0000","last_clean":"2023-10-01T09:47:43.516023+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:47:43.516023+0000","last_undegraded":"2023-10-01T09:47:43.516023+0000","last_fullsized":"2023-10-01T09:47:43.516023+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":515,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.516892+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:47:43.516892+0000","last_peered":"2023-10-01T09:47:43.516892+0000","last_clean":"2023-10-01T09:47:43.516892+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:47:43.516892+0000","last_undegraded":"2023-10-01T09:47:43.516892+0000","last_fullsized":"2023-10-01T09:47:43.516892+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":714,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.517791+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:47:43.517791+0000","last_peered":"2023-10-01T09:47:43.517791+0000","last_clean":"2023-10-01T09:47:43.517791+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:47:43.517791+0000","last_undegraded":"2023-10-01T09:47:43.517791+0000","last_fullsized":"2023-10-01T09:47:43.517791+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":407,"reported_epoch":555,"state":"active+clean","last_fresh":"2023-10-01T09:47:43.530363+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:47:43.530363+0000","last_peered":"2023-10-01T09:47:43.530363+0000","last_clean":"2023-10-01T09:47:43.530363+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:47:43.530363+0000","last_undegraded":"2023-10-01T09:47:43.530363+0000","last_fullsized":"2023-10-01T09:47:43.530363+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":826146840,"num_objects":12607,"num_object_clones":0,"num_object_copies":37821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5076,"num_objects_misplaced":664,"num_objects_unfound":0,"num_objects_dirty":12607,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153833,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11409,"num_bytes_recovered":747700224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3378393088,"data_stored":3378380872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127933,"ondisk_log_size":127933,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430768,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1176240,"kb_used_data":268904,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":92568912,"statfs":{"total":95995035648,"available":94790565888,"internally_reserved":0,"allocated":275357696,"data_stored":274410272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:02 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.495,"15min":2.4870000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20999999999999999},"max":{"1min":1.3600000000000001,"5min":3.3849999999999998,"15min":108.90000000000001},"last":1.0549999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.46899999999999997,"15min":2.621},"min":{"1min":0.23300000000000001,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":1.7410000000000001,"5min":1.7410000000000001,"15min":109.187},"last":1.2450000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.58299999999999996,"15min":5.0190000000000001},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.23300000000000001},"max":{"1min":7.7140000000000004,"5min":7.7140000000000004,"15min":229.161},"last":0.90300000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.53100000000000003,"15min":5.0449999999999999},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":1.972,"5min":3.0640000000000001,"15min":230.126},"last":3.0099999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:46:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.56299999999999994,"15min":1.591},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":1.6359999999999999,"5min":3.4300000000000002,"15min":14.901999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.55700000000000005,"15min":1.48},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.27000000000000002},"max":{"1min":1.8100000000000001,"5min":1.8100000000000001,"15min":14.682},"last":0.372}]},{"osd":3,"last update":"Sun Oct 1 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.59299999999999997,"15min":2.1059999999999999},"min":{"1min":0.34999999999999998,"5min":0.30199999999999999,"15min":0.29199999999999998},"max":{"1min":2.1749999999999998,"5min":2.476,"15min":76.555000000000007},"last":2.9649999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.58399999999999996,"15min":2.1120000000000001},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.1880000000000002,"5min":2.8479999999999999,"15min":76.534000000000006},"last":1.843}]},{"osd":4,"last update":"Sun Oct 1 09:47:02 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.52100000000000002,"15min":1.0249999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.4470000000000001,"5min":3.4609999999999999,"15min":9.6020000000000003},"last":0.443},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.52500000000000002,"15min":0.99399999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.837,"5min":1.837,"15min":8.4949999999999992},"last":0.46600000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":0.66600000000000004,"15min":1.5369999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.9370000000000003,"5min":7.9370000000000003,"15min":135.40000000000001},"last":1.996},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.65100000000000002,"15min":1.6379999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":7.0629999999999997,"5min":7.0629999999999997,"15min":135.339},"last":1.7849999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.64900000000000002,"15min":1.1519999999999999},"min":{"1min":0.39600000000000002,"5min":0.377,"15min":0.307},"max":{"1min":3.984,"5min":3.984,"15min":11.853999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.64100000000000001,"15min":1.0880000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32300000000000001},"max":{"1min":4.3470000000000004,"5min":4.3470000000000004,"15min":14.584},"last":1.8149999999999999}]}]},{"osd":6,"up_from":294,"seq":1262720385279,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":672884,"kb_used_data":343468,"kb_used_omap":6,"kb_used_meta":329401,"kb_avail":93072268,"statfs":{"total":95995035648,"available":95306002432,"internally_reserved":0,"allocated":351711232,"data_stored":350767435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":337306669},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":4.5839999999999996,"5min":1.2829999999999999,"15min":2.665},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20599999999999999},"max":{"1min":65.599000000000004,"5min":65.599000000000004,"15min":158.48599999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":5.4089999999999998,"5min":1.484,"15min":2.7909999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":65.388999999999996,"5min":65.388999999999996,"15min":158.37700000000001},"last":0.38500000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":0.70599999999999996,"15min":3.8839999999999999},"min":{"1min":0.34399999999999997,"5min":0.222,"15min":0.222},"max":{"1min":8.2409999999999997,"5min":8.2409999999999997,"15min":168.947},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.625,"5min":0.56999999999999995,"15min":3.8210000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.25600000000000001},"max":{"1min":2.6669999999999998,"5min":3.7469999999999999,"15min":169.07900000000001},"last":0.54400000000000004}]},{"osd":2,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":0.73799999999999999,"15min":1.429},"min":{"1min":0.311,"5min":0.254,"15min":0.254},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":10.92},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.748,"5min":0.57199999999999995,"15min":1.3899999999999999},"min":{"1min":0.314,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":2.6480000000000001,"5min":3.7160000000000002,"15min":21.704999999999998},"last":0.56499999999999995}]},{"osd":3,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55000000000000004,"15min":1.647},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.9710000000000001,"5min":3.3759999999999999,"15min":89.753},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.51,"5min":0.78200000000000003,"15min":1.7589999999999999},"min":{"1min":0.34399999999999997,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.3109999999999999,"5min":8.3109999999999999,"15min":89.849999999999994},"last":0.443}]},{"osd":4,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.505,"15min":1.0609999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.7559999999999998,"5min":3.6349999999999998,"15min":63.134},"last":9.8369999999999997},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.54400000000000004,"15min":1.0429999999999999},"min":{"1min":0.34499999999999997,"5min":0.33600000000000002,"15min":0.28899999999999998},"max":{"1min":2.778,"5min":3.2519999999999998,"15min":62.973999999999997},"last":10.006}]},{"osd":5,"last update":"Sun Oct 1 09:47:37 2023","interfaces":[{"interface":"back","average":{"1min":6.5650000000000004,"5min":2.089,"15min":1.367},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":136.535,"5min":136.535,"15min":136.535},"last":13.978999999999999},{"interface":"front","average":{"1min":6.3049999999999997,"5min":2.1499999999999999,"15min":1.4830000000000001},"min":{"1min":0.316,"5min":0.29899999999999999,"15min":0.28100000000000003},"max":{"1min":136.47300000000001,"5min":136.47300000000001,"15min":136.47300000000001},"last":12.496}]},{"osd":7,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.56100000000000005,"15min":1.2609999999999999},"min":{"1min":0.38300000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.3159999999999998,"5min":3.339,"15min":94.245999999999995},"last":5.8579999999999997},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.76700000000000002,"15min":1.421},"min":{"1min":0.37,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":8.3399999999999999,"5min":8.3399999999999999,"15min":94.715000000000003},"last":3.3490000000000002}]}]},{"osd":1,"up_from":94,"seq":403726926255,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1001424,"kb_used_data":173384,"kb_used_omap":7,"kb_used_meta":828024,"kb_avail":92743728,"statfs":{"total":95995035648,"available":94969577472,"internally_reserved":0,"allocated":177545216,"data_stored":175454049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":847896999},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":1.0349999999999999,"15min":3.5819999999999999},"min":{"1min":0.19500000000000001,"5min":0.186,"15min":0.182},"max":{"1min":0.82799999999999996,"5min":32.735999999999997,"15min":154.21299999999999},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.0860000000000001,"15min":3.1030000000000002},"min":{"1min":0.222,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":2.0859999999999999,"5min":32.591999999999999,"15min":95.236999999999995},"last":0.89800000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:47:45 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.42199999999999999,"15min":1.8109999999999999},"min":{"1min":0.23699999999999999,"5min":0.223,"15min":0.20000000000000001},"max":{"1min":3.6019999999999999,"5min":3.6019999999999999,"15min":51.640000000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.3049999999999999,"5min":0.63500000000000001,"15min":2.5219999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20499999999999999},"max":{"1min":12.018000000000001,"5min":12.018000000000001,"15min":154.197},"last":0.92600000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45000000000000001,"15min":2.706},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.219},"max":{"1min":2.149,"5min":2.8780000000000001,"15min":95.305000000000007},"last":0.96799999999999997},{"interface":"front","average":{"1min":0.497,"5min":0.47399999999999998,"15min":3.3210000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.994,"5min":2.7360000000000002,"15min":154.483},"last":0.314}]},{"osd":4,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.56499999999999995,"15min":2.1389999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":1.204,"5min":3.0550000000000002,"15min":154.101},"last":1.8600000000000001},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.46800000000000003,"15min":2.1469999999999998},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.27700000000000002},"max":{"1min":0.77200000000000002,"5min":0.83199999999999996,"15min":154.56999999999999},"last":1.8720000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:47:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.339,"15min":2.2679999999999998},"min":{"1min":0.31900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":18.158000000000001,"5min":59.561,"15min":154.22999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.296,"15min":2.323},"min":{"1min":0.32800000000000001,"5min":0.31,"15min":0.307},"max":{"1min":6.5789999999999997,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.56200000000000006}]},{"osd":6,"last update":"Sun Oct 1 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.65700000000000003,"15min":2.2770000000000001},"min":{"1min":0.33400000000000002,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":4.5069999999999997,"5min":4.5069999999999997,"15min":154.12200000000001},"last":2.327},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.63200000000000001,"15min":2.218},"min":{"1min":0.35899999999999999,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":1.5900000000000001,"5min":2.5630000000000002,"15min":154.16999999999999},"last":0.432}]},{"osd":7,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.73299999999999998,"15min":2.4420000000000002},"min":{"1min":0.38900000000000001,"5min":0.36399999999999999,"15min":0.33900000000000002},"max":{"1min":4.3730000000000002,"5min":4.3730000000000002,"15min":154.20699999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.71999999999999997,"15min":2.4460000000000002},"min":{"1min":0.435,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":4.3879999999999999,"5min":4.3879999999999999,"15min":154.51599999999999},"last":0.60299999999999998}]}]},{"osd":0,"up_from":89,"seq":382252089790,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2105828,"kb_used_data":1182684,"kb_used_omap":8,"kb_used_meta":923127,"kb_avail":91639324,"statfs":{"total":95995035648,"available":93838667776,"internally_reserved":0,"allocated":1211068416,"data_stored":1209024918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":945283010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.40500000000000003,"15min":3.6469999999999998},"min":{"1min":0.20799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.69099999999999995,"5min":4.2919999999999998,"15min":216.821},"last":0.44},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.434,"15min":4.2300000000000004},"min":{"1min":0.20599999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.72499999999999998,"5min":3.9620000000000002,"15min":217.02699999999999},"last":3.6659999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:47:49 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.50800000000000001,"15min":1.7210000000000001},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":1.6919999999999999,"5min":4.2430000000000003,"15min":19.788},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.60899999999999999,"15min":1.714},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.215},"max":{"1min":2.952,"5min":3.8719999999999999,"15min":25.844999999999999},"last":0.55000000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.47899999999999998,"15min":2.778},"min":{"1min":0.24299999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.68000000000000005,"5min":6.5670000000000002,"15min":222.16499999999999},"last":3.3849999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.626,"15min":2.6200000000000001},"min":{"1min":0.25700000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.0259999999999998,"5min":4.9199999999999999,"15min":222.227},"last":3.8199999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57199999999999995,"15min":1.8500000000000001},"min":{"1min":0.308,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":2.1160000000000001,"5min":4.048,"15min":62.206000000000003},"last":4.3019999999999996},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.52500000000000002,"15min":1.9359999999999999},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":1.3380000000000001,"5min":4,"15min":62.073999999999998},"last":2.3090000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:47:49 2023","interfaces":[{"interface":"back","average":{"1min":6.8460000000000001,"5min":1.869,"15min":2.0089999999999999},"min":{"1min":0.33700000000000002,"5min":0.312,"15min":0.312},"max":{"1min":156.69999999999999,"5min":156.69999999999999,"15min":156.69999999999999},"last":1.9199999999999999},{"interface":"front","average":{"1min":6.7249999999999996,"5min":1.8500000000000001,"15min":1.921},"min":{"1min":0.38400000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":156.773,"5min":156.773,"15min":156.773},"last":0.73699999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.64400000000000002,"15min":1.51},"min":{"1min":0.35699999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.589,"5min":4.3129999999999997,"15min":17.535},"last":2.1190000000000002},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.66700000000000004,"15min":1.571},"min":{"1min":0.35899999999999999,"5min":0.32600000000000001,"15min":0.32400000000000001},"max":{"1min":2.5960000000000001,"5min":4.2050000000000001,"15min":15.651999999999999},"last":3.395}]},{"osd":7,"last update":"Sun Oct 1 09:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.66300000000000003,"15min":1.927},"min":{"1min":0.39700000000000002,"5min":0.374,"15min":0.374},"max":{"1min":2.2290000000000001,"5min":4.3360000000000003,"15min":48.895000000000003},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.79400000000000004,"15min":1.97},"min":{"1min":0.39300000000000002,"5min":0.376,"15min":0.35299999999999998},"max":{"1min":0.78300000000000003,"5min":7.1580000000000004,"15min":48.274000000000001},"last":1.8069999999999999}]}]},{"osd":2,"up_from":0,"seq":425201762729,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599203,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1021716,"kb_used_data":163212,"kb_used_omap":3,"kb_used_meta":858364,"kb_avail":92723436,"statfs":{"total":95995035648,"available":94948798464,"internally_reserved":0,"allocated":167129088,"data_stored":164975973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":878965479},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.26900000000000002,"5min":0.34300000000000003,"15min":3.5009999999999999},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":0.504,"5min":2.7989999999999999,"15min":137.34399999999999},"last":0.19800000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.35899999999999999,"15min":3.5459999999999998},"min":{"1min":0.23400000000000001,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":0.66800000000000004,"5min":1.2849999999999999,"15min":137.73400000000001},"last":0.217}]},{"osd":1,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.40200000000000002,"15min":4.7430000000000003},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":0.63500000000000001,"5min":2.7250000000000001,"15min":162.22999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.371,"15min":4.7370000000000001},"min":{"1min":0.27200000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.3440000000000001,"5min":1.4119999999999999,"15min":162.70699999999999},"last":0.27600000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.442,"15min":1.552},"min":{"1min":0.251,"5min":0.24199999999999999,"15min":0.20799999999999999},"max":{"1min":0.46000000000000002,"5min":2.9049999999999998,"15min":14.254},"last":0.435},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.499,"15min":1.4350000000000001},"min":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":2.5230000000000001,"5min":4.5049999999999999,"15min":15.523},"last":0.33700000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.53400000000000003,"15min":1.653},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.26200000000000001},"max":{"1min":4.5010000000000003,"5min":4.5010000000000003,"15min":77.793000000000006},"last":0.312},{"interface":"front","average":{"1min":0.499,"5min":0.49399999999999999,"15min":1.8120000000000001},"min":{"1min":0.35299999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":0.72299999999999998,"5min":1.5609999999999999,"15min":76.646000000000001},"last":0.378}]},{"osd":5,"last update":"Sun Oct 1 09:47:37 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.67200000000000004,"15min":1.7150000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27500000000000002},"max":{"1min":3.1459999999999999,"5min":4.3099999999999996,"15min":28.117999999999999},"last":46.256999999999998},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.502,"15min":1.8140000000000001},"min":{"1min":0.34300000000000003,"5min":0.33600000000000002,"15min":0.27700000000000002},"max":{"1min":0.56899999999999995,"5min":2.512,"15min":28.34},"last":46.284999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:47:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54500000000000004,"15min":1.758},"min":{"1min":0.45500000000000002,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":1.0549999999999999,"5min":2.9100000000000001,"15min":73.887},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.65000000000000002,"15min":1.869},"min":{"1min":0.38100000000000001,"5min":0.35499999999999998,"15min":0.33600000000000002},"max":{"1min":3.1099999999999999,"5min":3.1099999999999999,"15min":74.016999999999996},"last":0.39500000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":0.85999999999999999,"15min":1.7729999999999999},"min":{"1min":0.42799999999999999,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":4.1989999999999998,"5min":4.1989999999999998,"15min":13.085000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.2030000000000001,"5min":0.80500000000000005,"15min":1.5529999999999999},"min":{"1min":0.432,"5min":0.38800000000000001,"15min":0.311},"max":{"1min":4.3280000000000003,"5min":4.3280000000000003,"15min":13.191000000000001},"last":0.36699999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019453,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1819884,"kb_used_data":929252,"kb_used_omap":0,"kb_used_meta":890176,"kb_avail":91925268,"statfs":{"total":95995035648,"available":94131474432,"internally_reserved":0,"allocated":951554048,"data_stored":950610130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911540224},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7480000000000002,"5min":0.89400000000000002,"15min":3.5819999999999999},"min":{"1min":0.24399999999999999,"5min":0.22500000000000001,"15min":0.21199999999999999},"max":{"1min":41.963999999999999,"5min":41.963999999999999,"15min":155.315},"last":11.590999999999999},{"interface":"front","average":{"1min":2.7530000000000001,"5min":0.90100000000000002,"15min":3.9140000000000001},"min":{"1min":0.26100000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":41.890999999999998,"5min":41.890999999999998,"15min":155.334},"last":7.6840000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.54000000000000004,"15min":5.1449999999999996},"min":{"1min":0.25600000000000001,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":3.3929999999999998,"5min":3.3929999999999998,"15min":190.61199999999999},"last":7.5979999999999999},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.54900000000000004,"15min":5.8639999999999999},"min":{"1min":0.29199999999999998,"5min":0.27500000000000002,"15min":0.25600000000000001},"max":{"1min":3.3839999999999999,"5min":3.3839999999999999,"15min":253.273},"last":3.9860000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.621,"15min":2.0499999999999998},"min":{"1min":0.30499999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.343,"5min":4.2329999999999997,"15min":170.45400000000001},"last":7.6390000000000002},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.61499999999999999,"15min":1.8069999999999999},"min":{"1min":0.35399999999999998,"5min":0.314,"15min":0.309},"max":{"1min":3.1389999999999998,"5min":3.1389999999999998,"15min":36.515999999999998},"last":7.6619999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.61199999999999999,"15min":1.9850000000000001},"min":{"1min":0.307,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":3.1539999999999999,"5min":3.1539999999999999,"15min":140.881},"last":11.667},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.61399999999999999,"15min":2.0579999999999998},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.311},"max":{"1min":1.3049999999999999,"5min":6.2969999999999997,"15min":140.911},"last":4.0540000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.0369999999999999,"15min":1.5740000000000001},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.26600000000000001},"max":{"1min":21.364999999999998,"5min":21.364999999999998,"15min":110.003},"last":7.7590000000000003},{"interface":"front","average":{"1min":1.9510000000000001,"5min":0.93200000000000005,"15min":1.466},"min":{"1min":0.313,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":21.559999999999999,"5min":21.559999999999999,"15min":108.56399999999999},"last":4.0810000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:47:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.56499999999999995,"15min":0.91700000000000004},"min":{"1min":0.318,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":3.3029999999999999,"5min":3.3029999999999999,"15min":11.358000000000001},"last":11.635999999999999},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.60999999999999999,"15min":1.046},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.28999999999999998},"max":{"1min":2.2669999999999999,"5min":2.9239999999999999,"15min":10.076000000000001},"last":7.7149999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:46:53 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.63400000000000001,"15min":1.149},"min":{"1min":0.39200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.3079999999999998,"5min":3.3079999999999998,"15min":38.034999999999997},"last":11.694000000000001},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.70899999999999996,"15min":1.3839999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.308},"max":{"1min":3.0329999999999999,"5min":3.113,"15min":37.723999999999997},"last":7.7370000000000001}]}]},{"osd":5,"up_from":379,"seq":1627792605360,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":726296,"kb_used_data":361552,"kb_used_omap":13,"kb_used_meta":364530,"kb_avail":93018856,"statfs":{"total":95995035648,"available":95251308544,"internally_reserved":0,"allocated":370229248,"data_stored":369301234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":373279218},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":4.9749999999999996,"5min":1.6719999999999999,"15min":2.3140000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.21099999999999999},"max":{"1min":99.609999999999999,"5min":99.609999999999999,"15min":99.609999999999999},"last":6.9139999999999997},{"interface":"front","average":{"1min":5.117,"5min":1.7130000000000001,"15min":2.2959999999999998},"min":{"1min":0.215,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":98.995999999999995,"5min":98.995999999999995,"15min":98.995999999999995},"last":2.7069999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":5.4790000000000001},"min":{"1min":0.28399999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.157,"5min":4.3289999999999997,"15min":295.71300000000002},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.56699999999999995,"15min":5.5129999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":3.665,"5min":3.9580000000000002,"15min":299.59399999999999},"last":0.35799999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.60399999999999998,"15min":1.4179999999999999},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.9129999999999998,"5min":4.2670000000000003,"15min":14.199999999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.70899999999999996,"15min":1.357},"min":{"1min":0.31900000000000001,"5min":0.29599999999999999,"15min":0.27100000000000002},"max":{"1min":4.1909999999999998,"5min":4.3719999999999999,"15min":14.430999999999999},"last":0.68300000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":0.70599999999999996,"15min":1.3759999999999999},"min":{"1min":0.30599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.141,"5min":7.141,"15min":14.004},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.66600000000000004,"15min":1.5029999999999999},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":3.1139999999999999,"5min":4.2910000000000004,"15min":13.991},"last":0.67300000000000004}]},{"osd":4,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.66400000000000003,"15min":1.375},"min":{"1min":0.27900000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.0659999999999998,"5min":4.1929999999999996,"15min":122.971},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.66500000000000004,"15min":1.472},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2469999999999999,"5min":4.3869999999999996,"15min":122.89100000000001},"last":0.60299999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.63800000000000001,"15min":0.94599999999999995},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28299999999999997},"max":{"1min":3.5750000000000002,"5min":4.3300000000000001,"15min":12.545},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.097,"5min":0.76300000000000001,"15min":1.119},"min":{"1min":0.31,"5min":0.31,"15min":0.30499999999999999},"max":{"1min":4.484,"5min":4.484,"15min":14.193},"last":0.33900000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.74399999999999999,"15min":1.1120000000000001},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.2199999999999998,"5min":4.3559999999999999,"15min":10.813000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.75600000000000001,"15min":1.0509999999999999},"min":{"1min":0.40799999999999997,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":3.609,"5min":4.2000000000000002,"15min":12.500999999999999},"last":0.63}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1181552640,"data_stored":1181548568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":166723584,"data_stored":166723584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":156434432,"data_stored":156434432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":922488832,"data_stored":922484760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":346030080,"data_stored":346030080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":345051136,"data_stored":345047064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":260112384,"data_stored":260112384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:48:01.734 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:01.784 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:01.858 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:01.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:01.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:01.905 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:01.964 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:01.991 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:02.135 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:02.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:02.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:02.246 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:02.327 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:02.515 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:02.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:02.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:02.563 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T09:48:02.634 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:02.662 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:02.714 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:02.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:02.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:02.818 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:02.909 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:03.008 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:03.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:03.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:03.120 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:03.152 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:03.211 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:03.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:03.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:03.319 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:03.383 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:48:03.408 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:03.524 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:03.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:03.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:03 smithi138 ceph-mon[192513]: pgmap v1678: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 664 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 5086/31755 objects degraded (16.016%); 664/31755 objects misplaced (2.091%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:48:03.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:03 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5086/31755 objects degraded (16.016%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:03.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:03 smithi138 ceph-mon[185886]: pgmap v1678: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 664 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 5086/31755 objects degraded (16.016%); 664/31755 objects misplaced (2.091%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:48:03.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:03 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5086/31755 objects degraded (16.016%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:03.639 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:03.681 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:03 smithi190 ceph-mon[160435]: pgmap v1678: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 664 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 5086/31755 objects degraded (16.016%); 664/31755 objects misplaced (2.091%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T09:48:03.681 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:03 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5086/31755 objects degraded (16.016%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:03.691 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:03.704 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:48:03.757 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:03.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:03.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:03.867 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:03.947 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:04.167 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:04.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:04.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:04.195 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:04.276 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:04.368 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:04.471 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:48:04.483 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:04.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:04.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:04.594 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:04.622 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:04.647 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:48:04.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4230971315' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:48:04.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:04 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:48:04.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4230971315' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:48:04.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:04 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:48:04.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4230971315' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:48:04.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:04 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:48:05.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:48:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:48:04] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.33.4" 2023-10-01T09:48:05.476 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:05.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:05.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:05.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:05 smithi190 ceph-mon[160435]: pgmap v1679: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 612 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 5132/29223 objects degraded (17.562%); 507/29223 objects misplaced (1.735%); 1.4 MiB/s, 21 objects/s recovering 2023-10-01T09:48:05.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:05 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:48:05.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:05 smithi190 ceph-mon[160435]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:05 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:48:05.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:05 smithi190 ceph-mon[160435]: osdmap e558: 8 total, 8 up, 7 in 2023-10-01T09:48:05.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:05 smithi190 ceph-mon[160435]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:05 smithi190 ceph-mon[160435]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:05 smithi190 ceph-mon[160435]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:05 smithi190 ceph-mon[160435]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:05 smithi190 ceph-mon[160435]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:05 smithi190 ceph-mon[160435]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.559 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:05 smithi190 ceph-mon[160435]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.582 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:05.684 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[192513]: pgmap v1679: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 612 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 5132/29223 objects degraded (17.562%); 507/29223 objects misplaced (1.735%); 1.4 MiB/s, 21 objects/s recovering 2023-10-01T09:48:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:48:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[192513]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:48:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[192513]: osdmap e558: 8 total, 8 up, 7 in 2023-10-01T09:48:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[192513]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[192513]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[192513]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[192513]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[192513]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[192513]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[192513]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[185886]: pgmap v1679: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 612 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 5132/29223 objects degraded (17.562%); 507/29223 objects misplaced (1.735%); 1.4 MiB/s, 21 objects/s recovering 2023-10-01T09:48:05.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:48:05.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[185886]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:48:05.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[185886]: osdmap e558: 8 total, 8 up, 7 in 2023-10-01T09:48:05.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[185886]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[185886]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[185886]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[185886]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[185886]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[185886]: failed to encode map e558 with expected crc 2023-10-01T09:48:05.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:05 smithi138 ceph-mon[185886]: failed to encode map e558 with expected crc 2023-10-01T09:48:06.215 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:06.250 INFO:teuthology.orchestra.run.smithi190.stdout:{ 2023-10-01T09:48:06.250 INFO:teuthology.orchestra.run.smithi190.stdout: "ops": [], 2023-10-01T09:48:06.251 INFO:teuthology.orchestra.run.smithi190.stdout: "complaint_time": 30, 2023-10-01T09:48:06.251 INFO:teuthology.orchestra.run.smithi190.stdout: "num_blocked_ops": 0 2023-10-01T09:48:06.251 INFO:teuthology.orchestra.run.smithi190.stdout:} 2023-10-01T09:48:06.289 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:06.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:06.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:06.401 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:06.506 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:06.535 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:48:06.563 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:48:06.587 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:48:06.615 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:48:06.642 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:48:06.646 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:06.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:06.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:06.749 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:48:06.796 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:06.804 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:48:06.832 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:48:06.868 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:48:06.871 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:48:06.871 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:48:06.871 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:48:06.871 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:48:06.872 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:48:06.872 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:48:06.872 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:48:06.872 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:48:07.099 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T09:48:07.119 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:48:07.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:07.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:48:07.225 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:07.260 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:07.487 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:07 smithi190 ceph-mon[160435]: pgmap v1681: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 535 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 5132/25545 objects degraded (20.090%); 507/25545 objects misplaced (1.985%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:48:07.487 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:07 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/946179079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:07.517 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:07.520 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:07 smithi138 ceph-mon[192513]: pgmap v1681: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 535 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 5132/25545 objects degraded (20.090%); 507/25545 objects misplaced (1.985%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:48:07.520 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:07 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/946179079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:07.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:07.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:07.521 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:07 smithi138 ceph-mon[185886]: pgmap v1681: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 535 MiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 5132/25545 objects degraded (20.090%); 507/25545 objects misplaced (1.985%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T09:48:07.521 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:07 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/946179079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:07.621 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:07.679 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:07.739 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:07.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:07.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:07.769 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:48:07.843 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:07.869 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:08.010 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:48:08.021 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:08.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:08.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:08.133 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:08.182 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:08.328 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:08.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:08.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:08.432 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:08.477 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:08.485 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:08 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5132/25545 objects degraded (20.090%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:08.619 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:08.621 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:08 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5132/25545 objects degraded (20.090%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:08.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:08.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:08.623 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:08 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5132/25545 objects degraded (20.090%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:08.656 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:08.724 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:08.826 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:08.881 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:08.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:08.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:08.945 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:48:08.947 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:48:08.983 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:48:09.009 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:09.050 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:48:09.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:09.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:48:09.154 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:09.207 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:09.498 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:09 smithi138 ceph-mon[185886]: pgmap v1682: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 437 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 5195/20850 objects degraded (24.916%); 341/20850 objects misplaced (1.635%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:48:09.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:09 smithi190 ceph-mon[160435]: pgmap v1682: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 437 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 5195/20850 objects degraded (24.916%); 341/20850 objects misplaced (1.635%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:48:09.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:09 smithi138 ceph-mon[192513]: pgmap v1682: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 437 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 5195/20850 objects degraded (24.916%); 341/20850 objects misplaced (1.635%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T09:48:10.264 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1682,"stamp":"2023-10-01T09:48:08.389721+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":458364875,"num_objects":6950,"num_object_clones":0,"num_object_copies":20850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5195,"num_objects_misplaced":341,"num_objects_unfound":0,"num_objects_dirty":6950,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":160624,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12134,"num_bytes_recovered":782074895,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130288,"ondisk_log_size":130288,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7508452,"kb_used_data":2378540,"kb_used_omap":39,"kb_used_meta":5129496,"kb_avail":648707612,"statfs":{"total":671965249536,"available":664276594688,"internally_reserved":0,"allocated":2435624960,"data_stored":2425558346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5252604527},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,3,3,9,6],"upper_bound":64},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-372572160,"num_objects":-5685,"num_object_clones":0,"num_object_copies":-17055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":109,"num_objects_misplaced":-323,"num_objects_unfound":0,"num_objects_dirty":-5685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5685,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":214,"num_bytes_recovered":14024704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004863"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":334,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.266351+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:47:57.266351+0000","last_peered":"2023-10-01T09:47:57.266351+0000","last_clean":"2023-10-01T09:47:57.266351+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:47:57.266351+0000","last_undegraded":"2023-10-01T09:47:57.266351+0000","last_fullsized":"2023-10-01T09:47:57.266351+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":179,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.277113+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:47:57.277113+0000","last_peered":"2023-10-01T09:47:57.277113+0000","last_clean":"2023-10-01T09:47:57.277113+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:47:57.277113+0000","last_undegraded":"2023-10-01T09:47:57.277113+0000","last_fullsized":"2023-10-01T09:47:57.277113+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":559,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.276868+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:47:57.276868+0000","last_peered":"2023-10-01T09:47:57.276868+0000","last_clean":"2023-10-01T09:47:57.276868+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:47:57.276868+0000","last_undegraded":"2023-10-01T09:47:57.276868+0000","last_fullsized":"2023-10-01T09:47:57.276868+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":618,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.275605+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:47:57.275605+0000","last_peered":"2023-10-01T09:47:57.275605+0000","last_clean":"2023-10-01T09:47:57.275605+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:47:57.275605+0000","last_undegraded":"2023-10-01T09:47:57.275605+0000","last_fullsized":"2023-10-01T09:47:57.275605+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":499,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.276077+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:47:57.276077+0000","last_peered":"2023-10-01T09:47:57.276077+0000","last_clean":"2023-10-01T09:47:57.276077+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:47:57.276077+0000","last_undegraded":"2023-10-01T09:47:57.276077+0000","last_fullsized":"2023-10-01T09:47:57.276077+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":239,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.264000+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:47:57.264000+0000","last_peered":"2023-10-01T09:47:57.264000+0000","last_clean":"2023-10-01T09:47:57.264000+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:47:57.264000+0000","last_undegraded":"2023-10-01T09:47:57.264000+0000","last_fullsized":"2023-10-01T09:47:57.264000+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":615,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.264097+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:47:57.264097+0000","last_peered":"2023-10-01T09:47:57.264097+0000","last_clean":"2023-10-01T09:47:57.264097+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:47:57.264097+0000","last_undegraded":"2023-10-01T09:47:57.264097+0000","last_fullsized":"2023-10-01T09:47:57.264097+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":546,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.257784+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:47:57.257784+0000","last_peered":"2023-10-01T09:47:57.257784+0000","last_clean":"2023-10-01T09:47:57.257784+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:47:57.257784+0000","last_undegraded":"2023-10-01T09:47:57.257784+0000","last_fullsized":"2023-10-01T09:47:57.257784+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":270,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.261444+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:47:57.261444+0000","last_peered":"2023-10-01T09:47:57.261444+0000","last_clean":"2023-10-01T09:47:57.261444+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:47:57.261444+0000","last_undegraded":"2023-10-01T09:47:57.261444+0000","last_fullsized":"2023-10-01T09:47:57.261444+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":77,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.258109+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:47:57.258109+0000","last_peered":"2023-10-01T09:47:57.258109+0000","last_clean":"2023-10-01T09:47:57.258109+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:47:57.258109+0000","last_undegraded":"2023-10-01T09:47:57.258109+0000","last_fullsized":"2023-10-01T09:47:57.258109+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":83,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.259220+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:47:57.259220+0000","last_peered":"2023-10-01T09:47:57.259220+0000","last_clean":"2023-10-01T09:47:57.259220+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:47:57.259220+0000","last_undegraded":"2023-10-01T09:47:57.259220+0000","last_fullsized":"2023-10-01T09:47:57.259220+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":83,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.279676+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:47:57.279676+0000","last_peered":"2023-10-01T09:47:57.279676+0000","last_clean":"2023-10-01T09:47:57.279676+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:47:57.279676+0000","last_undegraded":"2023-10-01T09:47:57.279676+0000","last_fullsized":"2023-10-01T09:47:57.279676+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":459,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.271270+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:47:57.271270+0000","last_peered":"2023-10-01T09:47:57.271270+0000","last_clean":"2023-10-01T09:47:57.271270+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:47:57.271270+0000","last_undegraded":"2023-10-01T09:47:57.271270+0000","last_fullsized":"2023-10-01T09:47:57.271270+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":340,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.266673+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:47:57.266673+0000","last_peered":"2023-10-01T09:47:57.266673+0000","last_clean":"2023-10-01T09:47:57.266673+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:47:57.266673+0000","last_undegraded":"2023-10-01T09:47:57.266673+0000","last_fullsized":"2023-10-01T09:47:57.266673+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":676,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.256429+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:47:57.256429+0000","last_peered":"2023-10-01T09:47:57.256429+0000","last_clean":"2023-10-01T09:47:57.256429+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:47:57.256429+0000","last_undegraded":"2023-10-01T09:47:57.256429+0000","last_fullsized":"2023-10-01T09:47:57.256429+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":413,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.257449+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:47:57.257449+0000","last_peered":"2023-10-01T09:47:57.257449+0000","last_clean":"2023-10-01T09:47:57.257449+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:47:57.257449+0000","last_undegraded":"2023-10-01T09:47:57.257449+0000","last_fullsized":"2023-10-01T09:47:57.257449+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":86,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.256249+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:47:57.256249+0000","last_peered":"2023-10-01T09:47:57.256249+0000","last_clean":"2023-10-01T09:47:57.256249+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:47:57.256249+0000","last_undegraded":"2023-10-01T09:47:57.256249+0000","last_fullsized":"2023-10-01T09:47:57.256249+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":72,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.261865+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:47:57.261865+0000","last_peered":"2023-10-01T09:47:57.261865+0000","last_clean":"2023-10-01T09:47:57.261865+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:47:57.261865+0000","last_undegraded":"2023-10-01T09:47:57.261865+0000","last_fullsized":"2023-10-01T09:47:57.261865+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":569,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.258323+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:47:57.258323+0000","last_peered":"2023-10-01T09:47:57.258323+0000","last_clean":"2023-10-01T09:47:57.258323+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:47:57.258323+0000","last_undegraded":"2023-10-01T09:47:57.258323+0000","last_fullsized":"2023-10-01T09:47:57.258323+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":582,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.275674+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:47:57.275674+0000","last_peered":"2023-10-01T09:47:57.275674+0000","last_clean":"2023-10-01T09:47:57.275674+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:47:57.275674+0000","last_undegraded":"2023-10-01T09:47:57.275674+0000","last_fullsized":"2023-10-01T09:47:57.275674+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":438,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.275932+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:47:57.275932+0000","last_peered":"2023-10-01T09:47:57.275932+0000","last_clean":"2023-10-01T09:47:57.275932+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:47:57.275932+0000","last_undegraded":"2023-10-01T09:47:57.275932+0000","last_fullsized":"2023-10-01T09:47:57.275932+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.257057+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:47:57.257057+0000","last_peered":"2023-10-01T09:47:57.257057+0000","last_clean":"2023-10-01T09:47:57.257057+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:47:57.257057+0000","last_undegraded":"2023-10-01T09:47:57.257057+0000","last_fullsized":"2023-10-01T09:47:57.257057+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"558'9980","reported_seq":10071,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:07.667322+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:48:07.667322+0000","last_peered":"2023-10-01T09:48:07.667322+0000","last_clean":"2023-10-01T09:48:07.667322+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:48:07.667322+0000","last_undegraded":"2023-10-01T09:48:07.667322+0000","last_fullsized":"2023-10-01T09:48:07.667322+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":9980,"log_dups_size":0,"ondisk_log_size":9980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27000832,"num_objects":412,"num_object_clones":0,"num_object_copies":1236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9980,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":81,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.256914+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:47:57.256914+0000","last_peered":"2023-10-01T09:47:57.256914+0000","last_clean":"2023-10-01T09:47:57.256914+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:47:57.256914+0000","last_undegraded":"2023-10-01T09:47:57.256914+0000","last_fullsized":"2023-10-01T09:47:57.256914+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":278,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.261474+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:47:57.261474+0000","last_peered":"2023-10-01T09:47:57.261474+0000","last_clean":"2023-10-01T09:47:57.261474+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:47:57.261474+0000","last_undegraded":"2023-10-01T09:47:57.261474+0000","last_fullsized":"2023-10-01T09:47:57.261474+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"558'10025","reported_seq":21226,"reported_epoch":558,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:48:07.731291+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:48:07.731291+0000","last_peered":"2023-10-01T09:48:07.731291+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:48:07.731291+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":10025,"log_dups_size":0,"ondisk_log_size":10025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24969216,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1437,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10025,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3698,"num_bytes_recovered":242352128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":313},{"shards":"5,6","objects":-1056}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":552,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.256820+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:47:57.256820+0000","last_peered":"2023-10-01T09:47:57.256820+0000","last_clean":"2023-10-01T09:47:57.256820+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:47:57.256820+0000","last_undegraded":"2023-10-01T09:47:57.256820+0000","last_fullsized":"2023-10-01T09:47:57.256820+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":615,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.276791+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:47:57.276791+0000","last_peered":"2023-10-01T09:47:57.276791+0000","last_clean":"2023-10-01T09:47:57.276791+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:47:57.276791+0000","last_undegraded":"2023-10-01T09:47:57.276791+0000","last_fullsized":"2023-10-01T09:47:57.276791+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"558'10008","reported_seq":10103,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:06.926495+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:48:06.926495+0000","last_peered":"2023-10-01T09:48:06.926495+0000","last_clean":"2023-10-01T09:48:06.926495+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:48:06.926495+0000","last_undegraded":"2023-10-01T09:48:06.926495+0000","last_fullsized":"2023-10-01T09:48:06.926495+0000","mapping_epoch":475,"log_start":"489'3000","ondisk_log_start":"489'3000","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7008,"log_dups_size":0,"ondisk_log_size":7008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27787264,"num_objects":424,"num_object_clones":0,"num_object_copies":1272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10008,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":638,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.259303+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:47:57.259303+0000","last_peered":"2023-10-01T09:47:57.259303+0000","last_clean":"2023-10-01T09:47:57.259303+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:47:57.259303+0000","last_undegraded":"2023-10-01T09:47:57.259303+0000","last_fullsized":"2023-10-01T09:47:57.259303+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":435,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.279923+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:47:57.279923+0000","last_peered":"2023-10-01T09:47:57.279923+0000","last_clean":"2023-10-01T09:47:57.279923+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:47:57.279923+0000","last_undegraded":"2023-10-01T09:47:57.279923+0000","last_fullsized":"2023-10-01T09:47:57.279923+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"558'9985","reported_seq":16685,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:06.924571+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:48:06.924571+0000","last_peered":"2023-10-01T09:48:06.924571+0000","last_clean":"2023-10-01T09:48:06.924571+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:48:06.924571+0000","last_undegraded":"2023-10-01T09:48:06.924571+0000","last_fullsized":"2023-10-01T09:48:06.924571+0000","mapping_epoch":505,"log_start":"489'3000","ondisk_log_start":"489'3000","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6985,"log_dups_size":0,"ondisk_log_size":6985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27328512,"num_objects":417,"num_object_clones":0,"num_object_copies":1251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9985,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":278,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.261645+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:47:57.261645+0000","last_peered":"2023-10-01T09:47:57.261645+0000","last_clean":"2023-10-01T09:47:57.261645+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:47:57.261645+0000","last_undegraded":"2023-10-01T09:47:57.261645+0000","last_fullsized":"2023-10-01T09:47:57.261645+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":538,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.275866+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:47:57.275866+0000","last_peered":"2023-10-01T09:47:57.275866+0000","last_clean":"2023-10-01T09:47:57.275866+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:47:57.275866+0000","last_undegraded":"2023-10-01T09:47:57.275866+0000","last_fullsized":"2023-10-01T09:47:57.275866+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"558'9991","reported_seq":10086,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:05.213850+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:48:05.213850+0000","last_peered":"2023-10-01T09:48:05.213850+0000","last_clean":"2023-10-01T09:48:05.213850+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:48:05.213850+0000","last_undegraded":"2023-10-01T09:48:05.213850+0000","last_fullsized":"2023-10-01T09:48:05.213850+0000","mapping_epoch":475,"log_start":"485'2400","ondisk_log_start":"485'2400","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7591,"log_dups_size":0,"ondisk_log_size":7591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33882112,"num_objects":517,"num_object_clones":0,"num_object_copies":1551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9991,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":138,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.267033+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:47:57.267033+0000","last_peered":"2023-10-01T09:47:57.267033+0000","last_clean":"2023-10-01T09:47:57.267033+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:47:57.267033+0000","last_undegraded":"2023-10-01T09:47:57.267033+0000","last_fullsized":"2023-10-01T09:47:57.267033+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":212,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.271410+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:47:57.271410+0000","last_peered":"2023-10-01T09:47:57.271410+0000","last_clean":"2023-10-01T09:47:57.271410+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:47:57.271410+0000","last_undegraded":"2023-10-01T09:47:57.271410+0000","last_fullsized":"2023-10-01T09:47:57.271410+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"558'9958","reported_seq":12120,"reported_epoch":558,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:48:07.738861+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:48:07.738861+0000","last_peered":"2023-10-01T09:48:07.738861+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:48:07.738861+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":9958,"log_dups_size":0,"ondisk_log_size":9958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26869760,"num_objects":410,"num_object_clones":0,"num_object_copies":1230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3748,"num_objects_misplaced":341,"num_objects_unfound":0,"num_objects_dirty":410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9958,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1032,"num_bytes_recovered":67633152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":341},{"shards":"4,5","objects":-3679}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":456,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.266493+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:47:57.266493+0000","last_peered":"2023-10-01T09:47:57.266493+0000","last_clean":"2023-10-01T09:47:57.266493+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:47:57.266493+0000","last_undegraded":"2023-10-01T09:47:57.266493+0000","last_fullsized":"2023-10-01T09:47:57.266493+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":618,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.277188+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:47:57.277188+0000","last_peered":"2023-10-01T09:47:57.277188+0000","last_clean":"2023-10-01T09:47:57.277188+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:47:57.277188+0000","last_undegraded":"2023-10-01T09:47:57.277188+0000","last_fullsized":"2023-10-01T09:47:57.277188+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"558'10011","reported_seq":10111,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:07.627510+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:48:07.627510+0000","last_peered":"2023-10-01T09:48:07.627510+0000","last_clean":"2023-10-01T09:48:07.627510+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:48:07.627510+0000","last_undegraded":"2023-10-01T09:48:07.627510+0000","last_fullsized":"2023-10-01T09:48:07.627510+0000","mapping_epoch":483,"log_start":"486'2500","ondisk_log_start":"486'2500","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7511,"log_dups_size":0,"ondisk_log_size":7511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24051712,"num_objects":367,"num_object_clones":0,"num_object_copies":1101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10011,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":452,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.269615+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:47:57.269615+0000","last_peered":"2023-10-01T09:47:57.269615+0000","last_clean":"2023-10-01T09:47:57.269615+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:47:57.269615+0000","last_undegraded":"2023-10-01T09:47:57.269615+0000","last_fullsized":"2023-10-01T09:47:57.269615+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":293,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.269839+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:47:57.269839+0000","last_peered":"2023-10-01T09:47:57.269839+0000","last_clean":"2023-10-01T09:47:57.269839+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:47:57.269839+0000","last_undegraded":"2023-10-01T09:47:57.269839+0000","last_fullsized":"2023-10-01T09:47:57.269839+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"558'10064","reported_seq":10155,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:06.353706+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:48:06.353706+0000","last_peered":"2023-10-01T09:48:06.353706+0000","last_clean":"2023-10-01T09:48:06.353706+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:48:06.353706+0000","last_undegraded":"2023-10-01T09:48:06.353706+0000","last_fullsized":"2023-10-01T09:48:06.353706+0000","mapping_epoch":475,"log_start":"484'1700","ondisk_log_start":"484'1700","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8364,"log_dups_size":0,"ondisk_log_size":8364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28442624,"num_objects":434,"num_object_clones":0,"num_object_copies":1302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10064,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":423,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.480936+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:48:04.480936+0000","last_peered":"2023-10-01T09:48:04.480936+0000","last_clean":"2023-10-01T09:48:04.480936+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:48:04.480936+0000","last_undegraded":"2023-10-01T09:48:04.480936+0000","last_fullsized":"2023-10-01T09:48:04.480936+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":470,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.279166+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:47:57.279166+0000","last_peered":"2023-10-01T09:47:57.279166+0000","last_clean":"2023-10-01T09:47:57.279166+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:47:57.279166+0000","last_undegraded":"2023-10-01T09:47:57.279166+0000","last_fullsized":"2023-10-01T09:47:57.279166+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"558'9738","reported_seq":9829,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:06.287875+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:48:06.287875+0000","last_peered":"2023-10-01T09:48:06.287875+0000","last_clean":"2023-10-01T09:48:06.287875+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:48:06.287875+0000","last_undegraded":"2023-10-01T09:48:06.287875+0000","last_fullsized":"2023-10-01T09:48:06.287875+0000","mapping_epoch":475,"log_start":"482'1400","ondisk_log_start":"482'1400","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8338,"log_dups_size":0,"ondisk_log_size":8338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28835840,"num_objects":440,"num_object_clones":0,"num_object_copies":1320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9738,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":951,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.257318+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:47:57.257318+0000","last_peered":"2023-10-01T09:47:57.257318+0000","last_clean":"2023-10-01T09:47:57.257318+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:47:57.257318+0000","last_undegraded":"2023-10-01T09:47:57.257318+0000","last_fullsized":"2023-10-01T09:47:57.257318+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":288,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.261989+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:47:57.261989+0000","last_peered":"2023-10-01T09:47:57.261989+0000","last_clean":"2023-10-01T09:47:57.261989+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:47:57.261989+0000","last_undegraded":"2023-10-01T09:47:57.261989+0000","last_fullsized":"2023-10-01T09:47:57.261989+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":83,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.279760+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:47:57.279760+0000","last_peered":"2023-10-01T09:47:57.279760+0000","last_clean":"2023-10-01T09:47:57.279760+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:47:57.279760+0000","last_undegraded":"2023-10-01T09:47:57.279760+0000","last_fullsized":"2023-10-01T09:47:57.279760+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"558'9944","reported_seq":10039,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:06.932818+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:48:06.932818+0000","last_peered":"2023-10-01T09:48:06.932818+0000","last_clean":"2023-10-01T09:48:06.932818+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:48:06.932818+0000","last_undegraded":"2023-10-01T09:48:06.932818+0000","last_fullsized":"2023-10-01T09:48:06.932818+0000","mapping_epoch":475,"log_start":"489'2900","ondisk_log_start":"489'2900","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7044,"log_dups_size":0,"ondisk_log_size":7044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27918336,"num_objects":426,"num_object_clones":0,"num_object_copies":1278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9944,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1223,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.262302+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:47:57.262302+0000","last_peered":"2023-10-01T09:47:57.262302+0000","last_clean":"2023-10-01T09:47:57.262302+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:47:57.262302+0000","last_undegraded":"2023-10-01T09:47:57.262302+0000","last_fullsized":"2023-10-01T09:47:57.262302+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":433,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.266869+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:47:57.266869+0000","last_peered":"2023-10-01T09:47:57.266869+0000","last_clean":"2023-10-01T09:47:57.266869+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:47:57.266869+0000","last_undegraded":"2023-10-01T09:47:57.266869+0000","last_fullsized":"2023-10-01T09:47:57.266869+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":261,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.261782+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:47:57.261782+0000","last_peered":"2023-10-01T09:47:57.261782+0000","last_clean":"2023-10-01T09:47:57.261782+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:47:57.261782+0000","last_undegraded":"2023-10-01T09:47:57.261782+0000","last_fullsized":"2023-10-01T09:47:57.261782+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"558'9932","reported_seq":17243,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:07.744646+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:48:07.744646+0000","last_peered":"2023-10-01T09:48:07.744646+0000","last_clean":"2023-10-01T09:48:07.744646+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:48:07.744646+0000","last_undegraded":"2023-10-01T09:48:07.744646+0000","last_fullsized":"2023-10-01T09:48:07.744646+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":9932,"log_dups_size":0,"ondisk_log_size":9932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29097984,"num_objects":444,"num_object_clones":0,"num_object_copies":1332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9932,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":905,"reported_epoch":557,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:47:57.275127+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:47:57.275127+0000","last_peered":"2023-10-01T09:47:57.275127+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:47:57.275127+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:47:57.275127+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":622,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.276818+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:47:57.276818+0000","last_peered":"2023-10-01T09:47:57.276818+0000","last_clean":"2023-10-01T09:47:57.276818+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:47:57.276818+0000","last_undegraded":"2023-10-01T09:47:57.276818+0000","last_fullsized":"2023-10-01T09:47:57.276818+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":83,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.279038+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:47:57.279038+0000","last_peered":"2023-10-01T09:47:57.279038+0000","last_clean":"2023-10-01T09:47:57.279038+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:47:57.279038+0000","last_undegraded":"2023-10-01T09:47:57.279038+0000","last_fullsized":"2023-10-01T09:47:57.279038+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"558'9813","reported_seq":9908,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:05.272938+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:48:05.272938+0000","last_peered":"2023-10-01T09:48:05.272938+0000","last_clean":"2023-10-01T09:48:05.272938+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:48:05.272938+0000","last_undegraded":"2023-10-01T09:48:05.272938+0000","last_fullsized":"2023-10-01T09:48:05.272938+0000","mapping_epoch":475,"log_start":"485'2300","ondisk_log_start":"485'2300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7513,"log_dups_size":0,"ondisk_log_size":7513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30081024,"num_objects":459,"num_object_clones":0,"num_object_copies":1377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9813,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1674,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.262779+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:47:57.262779+0000","last_peered":"2023-10-01T09:47:57.262779+0000","last_clean":"2023-10-01T09:47:57.262779+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:47:57.262779+0000","last_undegraded":"2023-10-01T09:47:57.262779+0000","last_fullsized":"2023-10-01T09:47:57.262779+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":588,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.257122+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:47:57.257122+0000","last_peered":"2023-10-01T09:47:57.257122+0000","last_clean":"2023-10-01T09:47:57.257122+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:47:57.257122+0000","last_undegraded":"2023-10-01T09:47:57.257122+0000","last_fullsized":"2023-10-01T09:47:57.257122+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":366,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.262380+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:47:57.262380+0000","last_peered":"2023-10-01T09:47:57.262380+0000","last_clean":"2023-10-01T09:47:57.262380+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:47:57.262380+0000","last_undegraded":"2023-10-01T09:47:57.262380+0000","last_fullsized":"2023-10-01T09:47:57.262380+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":356,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.266902+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:47:57.266902+0000","last_peered":"2023-10-01T09:47:57.266902+0000","last_clean":"2023-10-01T09:47:57.266902+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:47:57.266902+0000","last_undegraded":"2023-10-01T09:47:57.266902+0000","last_fullsized":"2023-10-01T09:47:57.266902+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"558'10087","reported_seq":10178,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:06.286928+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:48:06.286928+0000","last_peered":"2023-10-01T09:48:06.286928+0000","last_clean":"2023-10-01T09:48:06.286928+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:48:06.286928+0000","last_undegraded":"2023-10-01T09:48:06.286928+0000","last_fullsized":"2023-10-01T09:48:06.286928+0000","mapping_epoch":475,"log_start":"484'1700","ondisk_log_start":"484'1700","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8387,"log_dups_size":0,"ondisk_log_size":8387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29163520,"num_objects":445,"num_object_clones":0,"num_object_copies":1335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10087,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":765,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.257232+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:47:57.257232+0000","last_peered":"2023-10-01T09:47:57.257232+0000","last_clean":"2023-10-01T09:47:57.257232+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:47:57.257232+0000","last_undegraded":"2023-10-01T09:47:57.257232+0000","last_fullsized":"2023-10-01T09:47:57.257232+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":599,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.278762+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:47:57.278762+0000","last_peered":"2023-10-01T09:47:57.278762+0000","last_clean":"2023-10-01T09:47:57.278762+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:47:57.278762+0000","last_undegraded":"2023-10-01T09:47:57.278762+0000","last_fullsized":"2023-10-01T09:47:57.278762+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":599,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.275433+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:47:57.275433+0000","last_peered":"2023-10-01T09:47:57.275433+0000","last_clean":"2023-10-01T09:47:57.275433+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:47:57.275433+0000","last_undegraded":"2023-10-01T09:47:57.275433+0000","last_fullsized":"2023-10-01T09:47:57.275433+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"558'10013","reported_seq":10109,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:06.938563+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:48:06.938563+0000","last_peered":"2023-10-01T09:48:06.938563+0000","last_clean":"2023-10-01T09:48:06.938563+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:48:06.938563+0000","last_undegraded":"2023-10-01T09:48:06.938563+0000","last_fullsized":"2023-10-01T09:48:06.938563+0000","mapping_epoch":475,"log_start":"489'3000","ondisk_log_start":"489'3000","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7013,"log_dups_size":0,"ondisk_log_size":7013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29229080,"num_objects":447,"num_object_clones":0,"num_object_copies":1341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":447,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10013,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":996,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.259398+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:47:57.259398+0000","last_peered":"2023-10-01T09:47:57.259398+0000","last_clean":"2023-10-01T09:47:57.259398+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:47:57.259398+0000","last_undegraded":"2023-10-01T09:47:57.259398+0000","last_fullsized":"2023-10-01T09:47:57.259398+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":615,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.261716+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:47:57.261716+0000","last_peered":"2023-10-01T09:47:57.261716+0000","last_clean":"2023-10-01T09:47:57.261716+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:47:57.261716+0000","last_undegraded":"2023-10-01T09:47:57.261716+0000","last_fullsized":"2023-10-01T09:47:57.261716+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":468,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.265687+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:47:57.265687+0000","last_peered":"2023-10-01T09:47:57.265687+0000","last_clean":"2023-10-01T09:47:57.265687+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:47:57.265687+0000","last_undegraded":"2023-10-01T09:47:57.265687+0000","last_fullsized":"2023-10-01T09:47:57.265687+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"558'9912","reported_seq":10011,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:06.294444+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:48:06.294444+0000","last_peered":"2023-10-01T09:48:06.294444+0000","last_clean":"2023-10-01T09:48:06.294444+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:48:06.294444+0000","last_undegraded":"2023-10-01T09:48:06.294444+0000","last_fullsized":"2023-10-01T09:48:06.294444+0000","mapping_epoch":479,"log_start":"489'3200","ondisk_log_start":"489'3200","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6712,"log_dups_size":0,"ondisk_log_size":6712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29097984,"num_objects":444,"num_object_clones":0,"num_object_copies":1332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9912,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":497,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.257135+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:47:57.257135+0000","last_peered":"2023-10-01T09:47:57.257135+0000","last_clean":"2023-10-01T09:47:57.257135+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:47:57.257135+0000","last_undegraded":"2023-10-01T09:47:57.257135+0000","last_fullsized":"2023-10-01T09:47:57.257135+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1192,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.277550+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:47:57.277550+0000","last_peered":"2023-10-01T09:47:57.277550+0000","last_clean":"2023-10-01T09:47:57.277550+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:47:57.277550+0000","last_undegraded":"2023-10-01T09:47:57.277550+0000","last_fullsized":"2023-10-01T09:47:57.277550+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":86,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.257073+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:47:57.257073+0000","last_peered":"2023-10-01T09:47:57.257073+0000","last_clean":"2023-10-01T09:47:57.257073+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:47:57.257073+0000","last_undegraded":"2023-10-01T09:47:57.257073+0000","last_fullsized":"2023-10-01T09:47:57.257073+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"558'10057","reported_seq":10156,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:06.311572+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:48:06.311572+0000","last_peered":"2023-10-01T09:48:06.311572+0000","last_clean":"2023-10-01T09:48:06.311572+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:48:06.311572+0000","last_undegraded":"2023-10-01T09:48:06.311572+0000","last_fullsized":"2023-10-01T09:48:06.311572+0000","mapping_epoch":479,"log_start":"492'3300","ondisk_log_start":"492'3300","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6757,"log_dups_size":0,"ondisk_log_size":6757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29818880,"num_objects":455,"num_object_clones":0,"num_object_copies":1365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10057,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":90,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.256988+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:47:57.256988+0000","last_peered":"2023-10-01T09:47:57.256988+0000","last_clean":"2023-10-01T09:47:57.256988+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:47:57.256988+0000","last_undegraded":"2023-10-01T09:47:57.256988+0000","last_fullsized":"2023-10-01T09:47:57.256988+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":577,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.275526+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:47:57.275526+0000","last_peered":"2023-10-01T09:47:57.275526+0000","last_clean":"2023-10-01T09:47:57.275526+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:47:57.275526+0000","last_undegraded":"2023-10-01T09:47:57.275526+0000","last_fullsized":"2023-10-01T09:47:57.275526+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":281,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.262724+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:47:57.262724+0000","last_peered":"2023-10-01T09:47:57.262724+0000","last_clean":"2023-10-01T09:47:57.262724+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:47:57.262724+0000","last_undegraded":"2023-10-01T09:47:57.262724+0000","last_fullsized":"2023-10-01T09:47:57.262724+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":525,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.276004+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:47:57.276004+0000","last_peered":"2023-10-01T09:47:57.276004+0000","last_clean":"2023-10-01T09:47:57.276004+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:47:57.276004+0000","last_undegraded":"2023-10-01T09:47:57.276004+0000","last_fullsized":"2023-10-01T09:47:57.276004+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":639,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.277103+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:47:57.277103+0000","last_peered":"2023-10-01T09:47:57.277103+0000","last_clean":"2023-10-01T09:47:57.277103+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:47:57.277103+0000","last_undegraded":"2023-10-01T09:47:57.277103+0000","last_fullsized":"2023-10-01T09:47:57.277103+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":293,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.261573+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:47:57.261573+0000","last_peered":"2023-10-01T09:47:57.261573+0000","last_clean":"2023-10-01T09:47:57.261573+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:47:57.261573+0000","last_undegraded":"2023-10-01T09:47:57.261573+0000","last_fullsized":"2023-10-01T09:47:57.261573+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":82,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.279356+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:47:57.279356+0000","last_peered":"2023-10-01T09:47:57.279356+0000","last_clean":"2023-10-01T09:47:57.279356+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:47:57.279356+0000","last_undegraded":"2023-10-01T09:47:57.279356+0000","last_fullsized":"2023-10-01T09:47:57.279356+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":293,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.262063+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:47:57.262063+0000","last_peered":"2023-10-01T09:47:57.262063+0000","last_clean":"2023-10-01T09:47:57.262063+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:47:57.262063+0000","last_undegraded":"2023-10-01T09:47:57.262063+0000","last_fullsized":"2023-10-01T09:47:57.262063+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":471,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.276372+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:47:57.276372+0000","last_peered":"2023-10-01T09:47:57.276372+0000","last_clean":"2023-10-01T09:47:57.276372+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:47:57.276372+0000","last_undegraded":"2023-10-01T09:47:57.276372+0000","last_fullsized":"2023-10-01T09:47:57.276372+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":101,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.482499+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:48:04.482499+0000","last_peered":"2023-10-01T09:48:04.482499+0000","last_clean":"2023-10-01T09:48:04.482499+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:48:04.482499+0000","last_undegraded":"2023-10-01T09:48:04.482499+0000","last_fullsized":"2023-10-01T09:48:04.482499+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":517,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.278680+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:47:57.278680+0000","last_peered":"2023-10-01T09:47:57.278680+0000","last_clean":"2023-10-01T09:47:57.278680+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:47:57.278680+0000","last_undegraded":"2023-10-01T09:47:57.278680+0000","last_fullsized":"2023-10-01T09:47:57.278680+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":716,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.279122+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:47:57.279122+0000","last_peered":"2023-10-01T09:47:57.279122+0000","last_clean":"2023-10-01T09:47:57.279122+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:47:57.279122+0000","last_undegraded":"2023-10-01T09:47:57.279122+0000","last_fullsized":"2023-10-01T09:47:57.279122+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":409,"reported_epoch":557,"state":"active+clean","last_fresh":"2023-10-01T09:47:57.266483+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:47:57.266483+0000","last_peered":"2023-10-01T09:47:57.266483+0000","last_clean":"2023-10-01T09:47:57.266483+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:47:57.266483+0000","last_undegraded":"2023-10-01T09:47:57.266483+0000","last_fullsized":"2023-10-01T09:47:57.266483+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":453574680,"num_objects":6922,"num_object_clones":0,"num_object_copies":20766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5185,"num_objects_misplaced":341,"num_objects_unfound":0,"num_objects_dirty":6922,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":159518,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11623,"num_bytes_recovered":761724928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2311729152,"data_stored":2311716936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129118,"ondisk_log_size":129118,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430770,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1055168,"kb_used_data":156152,"kb_used_omap":0,"kb_used_meta":899008,"kb_avail":92689984,"statfs":{"total":95995035648,"available":94914543616,"internally_reserved":0,"allocated":159899648,"data_stored":158951576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":920584192},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":0.67500000000000004,"15min":2.4630000000000001},"min":{"1min":0.252,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":7.6900000000000004,"5min":7.6900000000000004,"15min":108.90000000000001},"last":1.4850000000000001},{"interface":"front","average":{"1min":1.196,"5min":0.60999999999999999,"15min":2.5680000000000001},"min":{"1min":0.32100000000000001,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":6.5199999999999996,"5min":6.5199999999999996,"15min":109.187},"last":0.51900000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.58299999999999996,"15min":5.0190000000000001},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.23300000000000001},"max":{"1min":7.7140000000000004,"5min":7.7140000000000004,"15min":229.161},"last":1.27},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.53100000000000003,"15min":5.0449999999999999},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":1.972,"5min":3.0640000000000001,"15min":230.126},"last":10.127000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":0.86099999999999999,"15min":1.54},"min":{"1min":0.33300000000000002,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":12.565,"5min":12.565,"15min":14.901999999999999},"last":1.625},{"interface":"front","average":{"1min":1.121,"5min":0.68200000000000005,"15min":1.429},"min":{"1min":0.372,"5min":0.32000000000000001,"15min":0.27000000000000002},"max":{"1min":6.4550000000000001,"5min":6.4550000000000001,"15min":14.682},"last":1.891}]},{"osd":3,"last update":"Sun Oct 1 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.59299999999999997,"15min":2.1059999999999999},"min":{"1min":0.34999999999999998,"5min":0.30199999999999999,"15min":0.29199999999999998},"max":{"1min":2.1749999999999998,"5min":2.476,"15min":76.555000000000007},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.58399999999999996,"15min":2.1120000000000001},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.1880000000000002,"5min":2.8479999999999999,"15min":76.534000000000006},"last":1.0840000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:48:03 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":0.79700000000000004,"15min":1.04},"min":{"1min":0.316,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.069000000000001,"5min":11.069000000000001,"15min":11.069000000000001},"last":0.76600000000000001},{"interface":"front","average":{"1min":1.3700000000000001,"5min":0.70099999999999996,"15min":1.002},"min":{"1min":0.33700000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.4809999999999999,"5min":6.4809999999999999,"15min":8.4949999999999992},"last":0.90800000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":0.66600000000000004,"15min":1.5369999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.9370000000000003,"5min":7.9370000000000003,"15min":135.40000000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.65100000000000002,"15min":1.6379999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":7.0629999999999997,"5min":7.0629999999999997,"15min":135.339},"last":1.05}]},{"osd":6,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.64900000000000002,"15min":1.1519999999999999},"min":{"1min":0.39600000000000002,"5min":0.377,"15min":0.307},"max":{"1min":3.984,"5min":3.984,"15min":11.853999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.64100000000000001,"15min":1.0880000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32300000000000001},"max":{"1min":4.3470000000000004,"5min":4.3470000000000004,"15min":14.584},"last":1.111}]}]},{"osd":6,"up_from":294,"seq":1262720385281,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":533884,"kb_used_data":196148,"kb_used_omap":6,"kb_used_meta":337593,"kb_avail":93211268,"statfs":{"total":95995035648,"available":95448338432,"internally_reserved":0,"allocated":200855552,"data_stored":199911431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":345695277},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":4.5839999999999996,"5min":1.2829999999999999,"15min":2.665},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20599999999999999},"max":{"1min":65.599000000000004,"5min":65.599000000000004,"15min":158.48599999999999},"last":3.7599999999999998},{"interface":"front","average":{"1min":5.4089999999999998,"5min":1.484,"15min":2.7909999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":65.388999999999996,"5min":65.388999999999996,"15min":158.37700000000001},"last":3.4140000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":0.70599999999999996,"15min":3.8839999999999999},"min":{"1min":0.34399999999999997,"5min":0.222,"15min":0.222},"max":{"1min":8.2409999999999997,"5min":8.2409999999999997,"15min":168.947},"last":3.4500000000000002},{"interface":"front","average":{"1min":0.625,"5min":0.56999999999999995,"15min":3.8210000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.25600000000000001},"max":{"1min":2.6669999999999998,"5min":3.7469999999999999,"15min":169.07900000000001},"last":4.4139999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":0.73799999999999999,"15min":1.429},"min":{"1min":0.311,"5min":0.254,"15min":0.254},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":10.92},"last":3.532},{"interface":"front","average":{"1min":0.748,"5min":0.57199999999999995,"15min":1.3899999999999999},"min":{"1min":0.314,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":2.6480000000000001,"5min":3.7160000000000002,"15min":21.704999999999998},"last":4.3879999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55000000000000004,"15min":1.647},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.9710000000000001,"5min":3.3759999999999999,"15min":89.753},"last":3.8039999999999998},{"interface":"front","average":{"1min":1.51,"5min":0.78200000000000003,"15min":1.7589999999999999},"min":{"1min":0.34399999999999997,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.3109999999999999,"5min":8.3109999999999999,"15min":89.849999999999994},"last":3.5089999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.505,"15min":1.0609999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.7559999999999998,"5min":3.6349999999999998,"15min":63.134},"last":4.4370000000000003},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.54400000000000004,"15min":1.0429999999999999},"min":{"1min":0.34499999999999997,"5min":0.33600000000000002,"15min":0.28899999999999998},"max":{"1min":2.778,"5min":3.2519999999999998,"15min":62.973999999999997},"last":3.8319999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:47:37 2023","interfaces":[{"interface":"back","average":{"1min":6.5650000000000004,"5min":2.089,"15min":1.367},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":136.535,"5min":136.535,"15min":136.535},"last":3.8559999999999999},{"interface":"front","average":{"1min":6.3049999999999997,"5min":2.1499999999999999,"15min":1.4830000000000001},"min":{"1min":0.316,"5min":0.29899999999999999,"15min":0.28100000000000003},"max":{"1min":136.47300000000001,"5min":136.47300000000001,"15min":136.47300000000001},"last":4.3490000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.56100000000000005,"15min":1.2609999999999999},"min":{"1min":0.38300000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.3159999999999998,"5min":3.339,"15min":94.245999999999995},"last":4.0300000000000002},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.76700000000000002,"15min":1.421},"min":{"1min":0.37,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":8.3399999999999999,"5min":8.3399999999999999,"15min":94.715000000000003},"last":3.4820000000000002}]}]},{"osd":1,"up_from":94,"seq":403726926257,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":933856,"kb_used_data":101720,"kb_used_omap":7,"kb_used_meta":832120,"kb_avail":92811296,"statfs":{"total":95995035648,"available":95038767104,"internally_reserved":0,"allocated":104161280,"data_stored":102069465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":852091303},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":1.0349999999999999,"15min":3.5819999999999999},"min":{"1min":0.19500000000000001,"5min":0.186,"15min":0.182},"max":{"1min":0.82799999999999996,"5min":32.735999999999997,"15min":154.21299999999999},"last":25.465},{"interface":"front","average":{"1min":0.44600000000000001,"5min":1.0860000000000001,"15min":3.1030000000000002},"min":{"1min":0.222,"5min":0.20499999999999999,"15min":0.19500000000000001},"max":{"1min":2.0859999999999999,"5min":32.591999999999999,"15min":95.236999999999995},"last":24.800999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:47:45 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.42199999999999999,"15min":1.8109999999999999},"min":{"1min":0.23699999999999999,"5min":0.223,"15min":0.20000000000000001},"max":{"1min":3.6019999999999999,"5min":3.6019999999999999,"15min":51.640000000000001},"last":0.375},{"interface":"front","average":{"1min":1.3049999999999999,"5min":0.63500000000000001,"15min":2.5219999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20499999999999999},"max":{"1min":12.018000000000001,"5min":12.018000000000001,"15min":154.197},"last":0.36299999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45000000000000001,"15min":2.706},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.219},"max":{"1min":2.149,"5min":2.8780000000000001,"15min":95.305000000000007},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.497,"5min":0.47399999999999998,"15min":3.3210000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.994,"5min":2.7360000000000002,"15min":154.483},"last":0.33300000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.56499999999999995,"15min":2.1389999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":1.204,"5min":3.0550000000000002,"15min":154.101},"last":24.539999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.46800000000000003,"15min":2.1469999999999998},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.27700000000000002},"max":{"1min":0.77200000000000002,"5min":0.83199999999999996,"15min":154.56999999999999},"last":0.42199999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:47:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.339,"15min":2.2679999999999998},"min":{"1min":0.31900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":18.158000000000001,"5min":59.561,"15min":154.22999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.296,"15min":2.323},"min":{"1min":0.32800000000000001,"5min":0.31,"15min":0.307},"max":{"1min":6.5789999999999997,"5min":59.536000000000001,"15min":154.54400000000001},"last":24.405000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:47:13 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.65700000000000003,"15min":2.2770000000000001},"min":{"1min":0.33400000000000002,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":4.5069999999999997,"5min":4.5069999999999997,"15min":154.12200000000001},"last":24.321000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.63200000000000001,"15min":2.218},"min":{"1min":0.35899999999999999,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":1.5900000000000001,"5min":2.5630000000000002,"15min":154.16999999999999},"last":0.39300000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:47:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.73299999999999998,"15min":2.4420000000000002},"min":{"1min":0.38900000000000001,"5min":0.36399999999999999,"15min":0.33900000000000002},"max":{"1min":4.3730000000000002,"5min":4.3730000000000002,"15min":154.20699999999999},"last":1.2010000000000001},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.71999999999999997,"15min":2.4460000000000002},"min":{"1min":0.435,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":4.3879999999999999,"5min":4.3879999999999999,"15min":154.51599999999999},"last":1.2170000000000001}]}]},{"osd":0,"up_from":89,"seq":382252089792,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1888824,"kb_used_data":953392,"kb_used_omap":8,"kb_used_meta":935415,"kb_avail":91856328,"statfs":{"total":95995035648,"available":94060879872,"internally_reserved":0,"allocated":976273408,"data_stored":974225439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":957865922},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.40500000000000003,"15min":3.6469999999999998},"min":{"1min":0.20799999999999999,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.69099999999999995,"5min":4.2919999999999998,"15min":216.821},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.434,"15min":4.2300000000000004},"min":{"1min":0.20599999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.72499999999999998,"5min":3.9620000000000002,"15min":217.02699999999999},"last":0.29599999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:47:49 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.50800000000000001,"15min":1.7210000000000001},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":1.6919999999999999,"5min":4.2430000000000003,"15min":19.788},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.60899999999999999,"15min":1.714},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.215},"max":{"1min":2.952,"5min":3.8719999999999999,"15min":25.844999999999999},"last":0.316}]},{"osd":3,"last update":"Sun Oct 1 09:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.47899999999999998,"15min":2.778},"min":{"1min":0.24299999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.68000000000000005,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.626,"15min":2.6200000000000001},"min":{"1min":0.25700000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.0259999999999998,"5min":4.9199999999999999,"15min":222.227},"last":3.3620000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:48:04 2023","interfaces":[{"interface":"back","average":{"1min":7.1349999999999998,"5min":1.8819999999999999,"15min":2.1699999999999999},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":138.84100000000001,"5min":138.84100000000001,"15min":138.84100000000001},"last":11.566000000000001},{"interface":"front","average":{"1min":6.8760000000000003,"5min":1.8109999999999999,"15min":2.246},"min":{"1min":0.307,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":135.94800000000001,"5min":135.94800000000001,"15min":135.94800000000001},"last":3.4020000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:47:49 2023","interfaces":[{"interface":"back","average":{"1min":6.8460000000000001,"5min":1.869,"15min":2.0089999999999999},"min":{"1min":0.33700000000000002,"5min":0.312,"15min":0.312},"max":{"1min":156.69999999999999,"5min":156.69999999999999,"15min":156.69999999999999},"last":8.9819999999999993},{"interface":"front","average":{"1min":6.7249999999999996,"5min":1.8500000000000001,"15min":1.921},"min":{"1min":0.38400000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":156.773,"5min":156.773,"15min":156.773},"last":3.508}]},{"osd":6,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.64400000000000002,"15min":1.51},"min":{"1min":0.35699999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.589,"5min":4.3129999999999997,"15min":17.535},"last":1.8080000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.66700000000000004,"15min":1.571},"min":{"1min":0.35899999999999999,"5min":0.32600000000000001,"15min":0.32400000000000001},"max":{"1min":2.5960000000000001,"5min":4.2050000000000001,"15min":15.651999999999999},"last":1.7909999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:48:04 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":0.79300000000000004,"15min":1.8089999999999999},"min":{"1min":0.40100000000000002,"5min":0.374,"15min":0.374},"max":{"1min":4.1280000000000001,"5min":4.3360000000000003,"15min":48.895000000000003},"last":2.9710000000000001},{"interface":"front","average":{"1min":1.736,"5min":0.97399999999999998,"15min":1.9350000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.8490000000000002,"5min":7.1580000000000004,"15min":48.274000000000001},"last":7.4199999999999999}]}]},{"osd":2,"up_from":0,"seq":425201762731,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599205,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":945964,"kb_used_data":95780,"kb_used_omap":3,"kb_used_meta":850172,"kb_avail":92799188,"statfs":{"total":95995035648,"available":95026368512,"internally_reserved":0,"allocated":98078720,"data_stored":95917311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":870576871},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.26900000000000002,"5min":0.34300000000000003,"15min":3.5009999999999999},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.17899999999999999},"max":{"1min":0.504,"5min":2.7989999999999999,"15min":137.34399999999999},"last":0.21299999999999999},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.35899999999999999,"15min":3.5459999999999998},"min":{"1min":0.23400000000000001,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":0.66800000000000004,"5min":1.2849999999999999,"15min":137.73400000000001},"last":0.64100000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.40200000000000002,"15min":4.7430000000000003},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":0.63500000000000001,"5min":2.7250000000000001,"15min":162.22999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.371,"15min":4.7370000000000001},"min":{"1min":0.27200000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.3440000000000001,"5min":1.4119999999999999,"15min":162.70699999999999},"last":0.56399999999999995}]},{"osd":2,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.442,"15min":1.552},"min":{"1min":0.251,"5min":0.24199999999999999,"15min":0.20799999999999999},"max":{"1min":0.46000000000000002,"5min":2.9049999999999998,"15min":14.254},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.499,"15min":1.4350000000000001},"min":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":2.5230000000000001,"5min":4.5049999999999999,"15min":15.523},"last":0.60599999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.53400000000000003,"15min":1.653},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.26200000000000001},"max":{"1min":4.5010000000000003,"5min":4.5010000000000003,"15min":77.793000000000006},"last":1.5780000000000001},{"interface":"front","average":{"1min":0.499,"5min":0.49399999999999999,"15min":1.8120000000000001},"min":{"1min":0.35299999999999998,"5min":0.28799999999999998,"15min":0.27900000000000003},"max":{"1min":0.72299999999999998,"5min":1.5609999999999999,"15min":76.646000000000001},"last":3.1600000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:47:37 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.67200000000000004,"15min":1.7150000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27500000000000002},"max":{"1min":3.1459999999999999,"5min":4.3099999999999996,"15min":28.117999999999999},"last":7.4210000000000003},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.502,"15min":1.8140000000000001},"min":{"1min":0.34300000000000003,"5min":0.33600000000000002,"15min":0.27700000000000002},"max":{"1min":0.56899999999999995,"5min":2.512,"15min":28.34},"last":7.1959999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:47:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54500000000000004,"15min":1.758},"min":{"1min":0.45500000000000002,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":1.0549999999999999,"5min":2.9100000000000001,"15min":73.887},"last":6.391},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.65000000000000002,"15min":1.869},"min":{"1min":0.38100000000000001,"5min":0.35499999999999998,"15min":0.33600000000000002},"max":{"1min":3.1099999999999999,"5min":3.1099999999999999,"15min":74.016999999999996},"last":0.55200000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:47:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":0.85999999999999999,"15min":1.7729999999999999},"min":{"1min":0.42799999999999999,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":4.1989999999999998,"5min":4.1989999999999998,"15min":13.085000000000001},"last":3.1120000000000001},{"interface":"front","average":{"1min":1.2030000000000001,"5min":0.80500000000000005,"15min":1.5529999999999999},"min":{"1min":0.432,"5min":0.38800000000000001,"15min":0.311},"max":{"1min":4.3280000000000003,"5min":4.3280000000000003,"15min":13.191000000000001},"last":1.706}]}]},{"osd":4,"up_from":30,"seq":128849019455,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1575612,"kb_used_data":673140,"kb_used_omap":0,"kb_used_meta":902464,"kb_avail":92169540,"statfs":{"total":95995035648,"available":94381608960,"internally_reserved":0,"allocated":689295360,"data_stored":688350794,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":924123136},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:02 2023","interfaces":[{"interface":"back","average":{"1min":8.1419999999999995,"5min":2.4369999999999998,"15min":3.8559999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":95.548000000000002,"5min":95.548000000000002,"15min":155.315},"last":6.8789999999999996},{"interface":"front","average":{"1min":8.2460000000000004,"5min":2.4420000000000002,"15min":4.125},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":95.253,"5min":95.253,"15min":155.334},"last":1.133}]},{"osd":1,"last update":"Sun Oct 1 09:48:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2709999999999999,"5min":0.67900000000000005,"15min":5.0679999999999996},"min":{"1min":0.24299999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":7.5979999999999999,"5min":7.5979999999999999,"15min":190.61199999999999},"last":1.0820000000000001},{"interface":"front","average":{"1min":1.089,"5min":0.67500000000000004,"15min":5.7759999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":6.3490000000000002,"5min":6.3490000000000002,"15min":253.273},"last":1.117}]},{"osd":2,"last update":"Sun Oct 1 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":0.80200000000000005,"15min":2.0510000000000002},"min":{"1min":0.32200000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.6390000000000002,"5min":7.6390000000000002,"15min":170.45400000000001},"last":1.107},{"interface":"front","average":{"1min":1.284,"5min":0.754,"15min":1.6779999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.6619999999999999,"5min":7.6619999999999999,"15min":36.515999999999998},"last":1.1559999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:48:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":0.80500000000000005,"15min":1.9450000000000001},"min":{"1min":0.28599999999999998,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":11.667,"5min":11.667,"15min":140.881},"last":6.9119999999999999},{"interface":"front","average":{"1min":1.141,"5min":0.748,"15min":2.0089999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.3769999999999998,"5min":6.3769999999999998,"15min":140.911},"last":1.0680000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.0369999999999999,"15min":1.5740000000000001},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.26600000000000001},"max":{"1min":21.364999999999998,"5min":21.364999999999998,"15min":110.003},"last":5.7510000000000003},{"interface":"front","average":{"1min":1.9510000000000001,"5min":0.93200000000000005,"15min":1.466},"min":{"1min":0.313,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":21.559999999999999,"5min":21.559999999999999,"15min":108.56399999999999},"last":6.5609999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:47:16 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.56499999999999995,"15min":0.91700000000000004},"min":{"1min":0.318,"5min":0.28799999999999998,"15min":0.27700000000000002},"max":{"1min":3.3029999999999999,"5min":3.3029999999999999,"15min":11.358000000000001},"last":6.9349999999999996},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.60999999999999999,"15min":1.046},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.28999999999999998},"max":{"1min":2.2669999999999999,"5min":2.9239999999999999,"15min":10.076000000000001},"last":3.0670000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":0.88600000000000001,"15min":1.1950000000000001},"min":{"1min":0.40600000000000003,"5min":0.313,"15min":0.313},"max":{"1min":11.694000000000001,"5min":11.694000000000001,"15min":38.034999999999997},"last":6.9560000000000004},{"interface":"front","average":{"1min":2.2130000000000001,"5min":1.0269999999999999,"15min":1.4750000000000001},"min":{"1min":0.35799999999999998,"5min":0.35599999999999998,"15min":0.308},"max":{"1min":12.228,"5min":12.228,"15min":37.723999999999997},"last":1.0489999999999999}]}]},{"osd":5,"up_from":379,"seq":1627792605362,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":575144,"kb_used_data":202208,"kb_used_omap":13,"kb_used_meta":372722,"kb_avail":93170008,"statfs":{"total":95995035648,"available":95406088192,"internally_reserved":0,"allocated":207060992,"data_stored":206132330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":381667826},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,3,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":4.9749999999999996,"5min":1.6719999999999999,"15min":2.3140000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.21099999999999999},"max":{"1min":99.609999999999999,"5min":99.609999999999999,"15min":99.609999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":5.117,"5min":1.7130000000000001,"15min":2.2959999999999998},"min":{"1min":0.215,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":98.995999999999995,"5min":98.995999999999995,"15min":98.995999999999995},"last":3.54}]},{"osd":1,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":5.4790000000000001},"min":{"1min":0.28399999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.157,"5min":4.3289999999999997,"15min":295.71300000000002},"last":5.5970000000000004},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.56699999999999995,"15min":5.5129999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":3.665,"5min":3.9580000000000002,"15min":299.59399999999999},"last":0.44700000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.60399999999999998,"15min":1.4179999999999999},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.9129999999999998,"5min":4.2670000000000003,"15min":14.199999999999999},"last":12.946999999999999},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.70899999999999996,"15min":1.357},"min":{"1min":0.31900000000000001,"5min":0.29599999999999999,"15min":0.27100000000000002},"max":{"1min":4.1909999999999998,"5min":4.3719999999999999,"15min":14.430999999999999},"last":6.5609999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":0.70599999999999996,"15min":1.3759999999999999},"min":{"1min":0.30599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.141,"5min":7.141,"15min":14.004},"last":11.983000000000001},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.66600000000000004,"15min":1.5029999999999999},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":3.1139999999999999,"5min":4.2910000000000004,"15min":13.991},"last":1.782}]},{"osd":4,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.66400000000000003,"15min":1.375},"min":{"1min":0.27900000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.0659999999999998,"5min":4.1929999999999996,"15min":122.971},"last":1.748},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.66500000000000004,"15min":1.472},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2469999999999999,"5min":4.3869999999999996,"15min":122.89100000000001},"last":6.5919999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.63800000000000001,"15min":0.94599999999999995},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28299999999999997},"max":{"1min":3.5750000000000002,"5min":4.3300000000000001,"15min":12.545},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.097,"5min":0.76300000000000001,"15min":1.119},"min":{"1min":0.31,"5min":0.31,"15min":0.30499999999999999},"max":{"1min":4.484,"5min":4.484,"15min":14.193},"last":1.9139999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.74399999999999999,"15min":1.1120000000000001},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.2199999999999998,"5min":4.3559999999999999,"15min":10.813000000000001},"last":13.897},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.75600000000000001,"15min":1.0509999999999999},"min":{"1min":0.40799999999999997,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":3.609,"5min":4.2000000000000002,"15min":12.500999999999999},"last":0.874}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":953290752,"data_stored":953286680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":92209152,"data_stored":92209152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":88801280,"data_stored":88801280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":661655552,"data_stored":661651480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":186056704,"data_stored":186056704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":182259712,"data_stored":182255640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":147456000,"data_stored":147456000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:48:10.270 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph status --format=json 2023-10-01T09:48:10.290 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:10.293 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:48:10.294 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:48:10.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:10.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:10.341 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:48:10.400 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:10.457 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:10.524 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:10 smithi138 ceph-mon[185886]: from='client.47519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:48:10.524 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:48:10.524 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:10 smithi138 ceph-mon[185886]: pgmap v1683: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 403 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 5209/19194 objects degraded (27.139%); 290/19194 objects misplaced (1.511%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:48:10.649 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:10.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:10.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:10.759 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:10.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:10 smithi190 ceph-mon[160435]: from='client.47519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:48:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:48:10.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:10 smithi190 ceph-mon[160435]: pgmap v1683: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 403 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 5209/19194 objects degraded (27.139%); 290/19194 objects misplaced (1.511%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:48:10.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:10 smithi138 ceph-mon[192513]: from='client.47519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:48:10.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:48:10.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:10 smithi138 ceph-mon[192513]: pgmap v1683: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 403 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 5209/19194 objects degraded (27.139%); 290/19194 objects misplaced (1.511%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:48:10.853 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:10.914 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:10.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:10.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:10.939 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:11.019 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:11.068 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:11.074 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:11.213 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:11.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:11.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:11.322 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:11.376 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:11.524 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T09:48:11.542 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:48:11.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3431805998' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:48:11.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3431805998' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:48:11.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3431805998' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:48:12.370 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:12.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:12.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:12.475 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:12.568 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:12.648 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:12 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:48:12.649 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:12 smithi138 ceph-mon[192513]: failed to encode map e559 with expected crc 2023-10-01T09:48:12.649 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:12 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3431805998' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:48:12.649 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:12 smithi138 ceph-mon[192513]: osdmap e559: 8 total, 8 up, 7 in 2023-10-01T09:48:12.649 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:12 smithi138 ceph-mon[192513]: failed to encode map e559 with expected crc 2023-10-01T09:48:12.649 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:12 smithi138 ceph-mon[192513]: failed to encode map e559 with expected crc 2023-10-01T09:48:12.649 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:12 smithi138 ceph-mon[192513]: pgmap v1685: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 305 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 5209/14499 objects degraded (35.927%); 290/14499 objects misplaced (2.000%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:48:12.649 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:12 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:48:12.649 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:12 smithi138 ceph-mon[185886]: failed to encode map e559 with expected crc 2023-10-01T09:48:12.650 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:12 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3431805998' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:48:12.650 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:12 smithi138 ceph-mon[185886]: osdmap e559: 8 total, 8 up, 7 in 2023-10-01T09:48:12.650 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:12 smithi138 ceph-mon[185886]: failed to encode map e559 with expected crc 2023-10-01T09:48:12.650 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:12 smithi138 ceph-mon[185886]: failed to encode map e559 with expected crc 2023-10-01T09:48:12.650 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:12 smithi138 ceph-mon[185886]: pgmap v1685: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 305 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 5209/14499 objects degraded (35.927%); 290/14499 objects misplaced (2.000%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:48:12.690 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:12.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:12.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:12.805 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:12.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:12 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:48:12.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:12 smithi190 ceph-mon[160435]: failed to encode map e559 with expected crc 2023-10-01T09:48:12.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:12 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3431805998' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:48:12.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:12 smithi190 ceph-mon[160435]: osdmap e559: 8 total, 8 up, 7 in 2023-10-01T09:48:12.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:12 smithi190 ceph-mon[160435]: failed to encode map e559 with expected crc 2023-10-01T09:48:12.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:12 smithi190 ceph-mon[160435]: failed to encode map e559 with expected crc 2023-10-01T09:48:12.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:12 smithi190 ceph-mon[160435]: pgmap v1685: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 305 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 5209/14499 objects degraded (35.927%); 290/14499 objects misplaced (2.000%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T09:48:12.854 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:12.995 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:12.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:12.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:13.100 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:13.154 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:13.265 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:13.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:13.371 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:13.409 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:13.478 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:13.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:13.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:13.585 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:13.616 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:13.741 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:13.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:13.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:13.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:13 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5209/14499 objects degraded (35.927%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:13.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:13 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5209/14499 objects degraded (35.927%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:13.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:13 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5209/14499 objects degraded (35.927%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:13.847 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:13.913 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:14.023 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:14.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:14.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:14.089 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:14.140 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:14.218 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:14.381 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:14.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:14.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:14.498 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:14.547 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:14.578 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:48:14.578 INFO:teuthology.orchestra.run.smithi138.stdout:{"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5209/14052 objects degraded (37.069%), 3 pgs degraded, 2 pgs undersized","count":5},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2315,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":559,"num_osds":8,"num_up_osds":8,"osd_up_since":1696152790,"num_in_osds":7,"osd_in_since":1696153200,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":89,"num_pools":5,"num_objects":4684,"data_bytes":309860299,"bytes_used":7227035648,"bytes_avail":664738213888,"bytes_total":671965249536,"degraded_objects":5209,"degraded_total":14052,"degraded_ratio":0.37069456305152293,"misplaced_objects":290,"misplaced_total":14052,"misplaced_ratio":0.020637631653857104,"recovering_objects_per_sec":14,"recovering_bytes_per_sec":923763,"recovering_keys_per_sec":0,"num_objects_recovered":140,"num_bytes_recovered":9175040,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":509},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi138.brhzod","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.138:8443/","prometheus":"http://172.21.15.138:9283/"}},"servicemap":{"epoch":34,"modified":"2023-10-01T09:43:40.311181+0000","services":{}},"progress_events":{"a14d6880-b0ef-4ecc-8670-e4761227bd84":{"message":"Upgrade to 18.2.0-497-gb5682219 (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"db436fff-9387-41ce-8305-bd1869cd9c77":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 2s)","progress":0.95505619049072266,"add_to_ceph_s":true}}} 2023-10-01T09:48:14.600 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:48:14.626 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:48:14.655 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:48:14.694 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:48:14.696 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:14.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:14.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:14.737 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:48:14.775 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:48:14.808 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:48:14.808 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:14.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:14 smithi190 ceph-mon[160435]: pgmap v1686: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 296 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 5209/14052 objects degraded (37.069%); 290/14052 objects misplaced (2.064%); 902 KiB/s, 14 objects/s recovering 2023-10-01T09:48:14.812 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:14 smithi138 ceph-mon[192513]: pgmap v1686: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 296 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 5209/14052 objects degraded (37.069%); 290/14052 objects misplaced (2.064%); 902 KiB/s, 14 objects/s recovering 2023-10-01T09:48:14.813 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:14 smithi138 ceph-mon[185886]: pgmap v1686: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 296 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 5209/14052 objects degraded (37.069%); 290/14052 objects misplaced (2.064%); 902 KiB/s, 14 objects/s recovering 2023-10-01T09:48:14.881 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:14.922 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:48:14.964 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:48:14.966 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:48:14.966 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:48:14.966 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:48:14.967 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:48:14.967 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:48:14.967 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:48:14.967 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:48:14.967 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:48:15.014 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:15.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:15.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:15.069 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:48:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:48:14] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-01T09:48:15.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:15.159 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:15.641 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:15.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:15.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:15.746 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:48:15.772 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:15.816 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:48:15.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:15.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:48:15.921 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:15.970 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:15 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2187734729' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:15.970 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:15 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2795533276' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:48:15.971 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:15 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2187734729' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:15.971 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:15 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2795533276' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:48:15.973 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:16.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:15 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2187734729' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:16.062 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:15 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2795533276' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-10-01T09:48:16.493 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:16.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:16.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:16.514 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-10-01T09:48:16.514 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:48:16.599 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:16.682 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:16.747 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:16.823 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:16 smithi138 ceph-mon[185886]: pgmap v1687: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 156 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 5311/7338 objects degraded (72.377%); 76/7338 objects misplaced (1.036%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:48:16.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:16 smithi138 ceph-mon[192513]: pgmap v1687: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 156 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 5311/7338 objects degraded (72.377%); 76/7338 objects misplaced (1.036%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:48:16.835 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:16.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:16.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:16.940 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:16.975 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:17.059 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:16 smithi190 ceph-mon[160435]: pgmap v1687: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 156 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 5311/7338 objects degraded (72.377%); 76/7338 objects misplaced (1.036%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T09:48:17.134 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:17.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:17.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:17.190 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:48:17.190 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:48:17.190 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.190 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:165798 10.5 10:a654d579:::benchmark_data_smithi138_573643_object82576:head [delete] snapc 0=[] ondisk+write+known_if_redirected e559)", 2023-10-01T09:48:17.190 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:48:17.181390+0000", 2023-10-01T09:48:17.190 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.006231828, 2023-10-01T09:48:17.190 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0062642380000000001, 2023-10-01T09:48:17.191 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:48:17.191 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:48:17.191 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:48:17.191 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:48:17.191 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:48:17.191 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165798 2023-10-01T09:48:17.191 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.191 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:48:17.191 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.191 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:48:17.191 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.181390+0000", 2023-10-01T09:48:17.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:48:17.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:48:17.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.181390+0000", 2023-10-01T09:48:17.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:48:17.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.192 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.192 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:48:17.192 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.181389+0000", 2023-10-01T09:48:17.192 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:48:17.192 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:48:17.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.181396+0000", 2023-10-01T09:48:17.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.4779999999999996e-06 2023-10-01T09:48:17.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.193 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:48:17.193 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.181405+0000", 2023-10-01T09:48:17.193 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.8789999999999998e-06 2023-10-01T09:48:17.193 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.193 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.194 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:48:17.194 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.181412+0000", 2023-10-01T09:48:17.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3550000000000001e-06 2023-10-01T09:48:17.194 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.194 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.194 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:48:17.194 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.181467+0000", 2023-10-01T09:48:17.194 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.5303999999999998e-05 2023-10-01T09:48:17.194 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.194 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.194 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:48:17.194 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.181601+0000", 2023-10-01T09:48:17.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00013375099999999999 2023-10-01T09:48:17.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:48:17.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.181730+0000", 2023-10-01T09:48:17.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012921300000000001 2023-10-01T09:48:17.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.195 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.195 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "op_commit", 2023-10-01T09:48:17.195 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.185819+0000", 2023-10-01T09:48:17.195 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.0040892970000000004 2023-10-01T09:48:17.195 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.196 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "sub_op_commit_rec", 2023-10-01T09:48:17.196 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.186544+0000", 2023-10-01T09:48:17.196 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00072510000000000001 2023-10-01T09:48:17.196 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:48:17.196 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:48:17.196 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:48:17.196 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.196 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.196 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.36456.0:165796 10.8 e559/475 10:1fb41fc7:::benchmark_data_smithi138_573643_object82574:head v 559'10495, mlcod=559'10492)", 2023-10-01T09:48:17.196 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:48:17.185246+0000", 2023-10-01T09:48:17.196 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.0023760180000000001, 2023-10-01T09:48:17.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.002515434, 2023-10-01T09:48:17.197 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:48:17.197 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:48:17.197 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:48:17.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:48:17.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.185246+0000", 2023-10-01T09:48:17.197 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:48:17.197 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.197 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.197 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:48:17.197 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.185246+0000", 2023-10-01T09:48:17.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:48:17.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.198 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:48:17.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.185245+0000", 2023-10-01T09:48:17.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:48:17.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.198 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.198 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:48:17.198 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.185255+0000", 2023-10-01T09:48:17.198 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 9.4919999999999999e-06 2023-10-01T09:48:17.198 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:48:17.199 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.185273+0000", 2023-10-01T09:48:17.199 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8343e-05 2023-10-01T09:48:17.199 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:48:17.199 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.185281+0000", 2023-10-01T09:48:17.199 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.4290000000000001e-06 2023-10-01T09:48:17.199 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.199 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.199 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:48:17.199 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.185345+0000", 2023-10-01T09:48:17.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.3359000000000001e-05 2023-10-01T09:48:17.200 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.200 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.200 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:48:17.200 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.185368+0000", 2023-10-01T09:48:17.200 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 2.2915e-05 2023-10-01T09:48:17.200 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:48:17.200 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:48:17.200 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:48:17.200 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.200 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.200 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:165801 10.4 10:2a069470:::benchmark_data_smithi138_573643_object82579:head [delete] snapc 0=[] ondisk+write+known_if_redirected e559)", 2023-10-01T09:48:17.201 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:48:17.187353+0000", 2023-10-01T09:48:17.201 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.00026892399999999998, 2023-10-01T09:48:17.201 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00047354, 2023-10-01T09:48:17.201 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:48:17.201 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:48:17.201 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:48:17.201 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:48:17.201 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:48:17.201 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165801 2023-10-01T09:48:17.201 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.201 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:48:17.201 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.202 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:48:17.202 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187353+0000", 2023-10-01T09:48:17.202 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:48:17.202 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.202 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.202 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:48:17.202 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187353+0000", 2023-10-01T09:48:17.202 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:48:17.202 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.202 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.202 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:48:17.202 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187352+0000", 2023-10-01T09:48:17.203 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999986 2023-10-01T09:48:17.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.203 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.203 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:48:17.203 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187359+0000", 2023-10-01T09:48:17.203 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3599999999999998e-06 2023-10-01T09:48:17.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.203 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.203 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:48:17.203 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187372+0000", 2023-10-01T09:48:17.203 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.3077e-05 2023-10-01T09:48:17.203 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.204 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:48:17.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187381+0000", 2023-10-01T09:48:17.204 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 8.5580000000000001e-06 2023-10-01T09:48:17.204 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.204 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:48:17.204 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187432+0000", 2023-10-01T09:48:17.204 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.0899999999999997e-05 2023-10-01T09:48:17.204 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.204 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.204 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:48:17.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187576+0000", 2023-10-01T09:48:17.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000144475 2023-10-01T09:48:17.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.205 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.205 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,4", 2023-10-01T09:48:17.205 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187699+0000", 2023-10-01T09:48:17.205 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00012248900000000001 2023-10-01T09:48:17.205 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:48:17.205 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:48:17.205 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:48:17.205 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.205 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.205 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_op(client.36456.0:165803 10.5 10:ac632be5:::benchmark_data_smithi138_573643_object82581:head [delete] snapc 0=[] ondisk+write+known_if_redirected e559)", 2023-10-01T09:48:17.206 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:48:17.187400+0000", 2023-10-01T09:48:17.206 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.00022184799999999999, 2023-10-01T09:48:17.206 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00050694300000000004, 2023-10-01T09:48:17.206 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:48:17.206 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "waiting for sub ops", 2023-10-01T09:48:17.206 INFO:teuthology.orchestra.run.smithi138.stdout: "client_info": { 2023-10-01T09:48:17.206 INFO:teuthology.orchestra.run.smithi138.stdout: "client": "client.36456", 2023-10-01T09:48:17.206 INFO:teuthology.orchestra.run.smithi138.stdout: "client_addr": "172.21.15.138:0/1563497739", 2023-10-01T09:48:17.206 INFO:teuthology.orchestra.run.smithi138.stdout: "tid": 165803 2023-10-01T09:48:17.206 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.206 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:48:17.207 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.207 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:48:17.207 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187400+0000", 2023-10-01T09:48:17.207 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:48:17.207 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.207 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.207 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:48:17.208 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187400+0000", 2023-10-01T09:48:17.208 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:48:17.208 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.208 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.208 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:48:17.208 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187400+0000", 2023-10-01T09:48:17.208 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.9999995 2023-10-01T09:48:17.208 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.208 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.208 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:48:17.208 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187403+0000", 2023-10-01T09:48:17.209 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 3.6169999999999998e-06 2023-10-01T09:48:17.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.209 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:48:17.209 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187411+0000", 2023-10-01T09:48:17.209 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.199e-06 2023-10-01T09:48:17.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.209 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.209 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:48:17.209 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187416+0000", 2023-10-01T09:48:17.209 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6350000000000003e-06 2023-10-01T09:48:17.209 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.210 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.210 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:48:17.210 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187472+0000", 2023-10-01T09:48:17.210 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 5.6165999999999997e-05 2023-10-01T09:48:17.210 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.210 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.210 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:48:17.210 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187589+0000", 2023-10-01T09:48:17.210 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000116119 2023-10-01T09:48:17.210 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:17.210 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:17.210 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "waiting for subops from 3,7", 2023-10-01T09:48:17.211 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:17.187698+0000", 2023-10-01T09:48:17.211 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.00010949499999999999 2023-10-01T09:48:17.211 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:48:17.211 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:48:17.211 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:48:17.211 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:48:17.211 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:48:17.211 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 4 2023-10-01T09:48:17.211 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:48:17.241 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:48:17.327 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:17.494 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:48:17.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:17.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:48:17.608 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:17.642 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:18.145 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Removed 83219 objects 2023-10-01T09:48:18.145 INFO:tasks.radosbench.radosbench.0.smithi138.stdout:Clean up completed and total clean up time :395.511 2023-10-01T09:48:18.198 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:17 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5311/7338 objects degraded (72.377%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:18.237 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:17 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5311/7338 objects degraded (72.377%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:18.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:17 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5311/7338 objects degraded (72.377%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:18.408 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:18.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:18.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:18.480 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T09:48:18.511 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:18.549 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:18.583 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:48:18.736 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:18.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:18.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:18.762 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:18.841 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:18.870 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:18.969 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:18.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:18.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:19.036 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:18 smithi138 ceph-mon[192513]: pgmap v1688: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 93 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 5311/4353 objects degraded (122.008%); 76/4353 objects misplaced (1.746%); 953 KiB/s, 14 objects/s recovering 2023-10-01T09:48:19.037 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:18 smithi138 ceph-mon[185886]: pgmap v1688: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 93 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 5311/4353 objects degraded (122.008%); 76/4353 objects misplaced (1.746%); 953 KiB/s, 14 objects/s recovering 2023-10-01T09:48:19.073 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:19.102 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:19.253 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:19.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:19.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:19.279 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:19.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:18 smithi190 ceph-mon[160435]: pgmap v1688: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 93 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 5311/4353 objects degraded (122.008%); 76/4353 objects misplaced (1.746%); 953 KiB/s, 14 objects/s recovering 2023-10-01T09:48:19.357 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:19.385 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:19.525 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:19.634 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:19.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:19.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:19.714 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-10-01T09:48:19.714 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-10-01T09:48:19.739 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:19.784 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:19.908 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:19 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4236605737' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:48:19.923 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:48:20.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:19 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4236605737' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:48:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:19 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:48:20.319 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4236605737' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:48:20.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:19 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:48:20.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:19 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:48:20.543 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:20.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:20.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:20.647 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:20.675 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:20.928 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:20 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:48:20.929 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:20 smithi138 ceph-mon[185886]: failed to encode map e560 with expected crc 2023-10-01T09:48:20.929 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:20 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:48:20.929 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:20 smithi138 ceph-mon[185886]: osdmap e560: 8 total, 8 up, 7 in 2023-10-01T09:48:20.929 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:20 smithi138 ceph-mon[185886]: pgmap v1690: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 41 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 5273/1845 objects degraded (285.799%); 1.1 MiB/s, 25 objects/s recovering 2023-10-01T09:48:20.929 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:20 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:48:20.929 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:20 smithi138 ceph-mon[192513]: failed to encode map e560 with expected crc 2023-10-01T09:48:20.930 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:20 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:48:20.930 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:20 smithi138 ceph-mon[192513]: osdmap e560: 8 total, 8 up, 7 in 2023-10-01T09:48:20.930 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:20 smithi138 ceph-mon[192513]: pgmap v1690: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 41 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 5273/1845 objects degraded (285.799%); 1.1 MiB/s, 25 objects/s recovering 2023-10-01T09:48:21.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:20 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:48:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:20 smithi190 ceph-mon[160435]: failed to encode map e560 with expected crc 2023-10-01T09:48:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:20 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:48:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:20 smithi190 ceph-mon[160435]: osdmap e560: 8 total, 8 up, 7 in 2023-10-01T09:48:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:20 smithi190 ceph-mon[160435]: pgmap v1690: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 41 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 5273/1845 objects degraded (285.799%); 1.1 MiB/s, 25 objects/s recovering 2023-10-01T09:48:21.875 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:21.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:21.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:21.979 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:22.004 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:22.045 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:22.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:22.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:22.151 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:22.177 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:22.218 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:22.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:22.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:22.323 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:22.349 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:22.389 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:22.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:22.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:22.459 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:22.496 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:22.516 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:48:22.518 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:48:22.522 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:22.537 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:48:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:48:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "complaint_time": 30, 2023-10-01T09:48:22.538 INFO:teuthology.orchestra.run.smithi138.stdout: "num_blocked_ops": 0 2023-10-01T09:48:22.538 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:48:22.563 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:22.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:22.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:22.668 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:22.694 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:22.736 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:22.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:22.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:22.763 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:48:22.782 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:48:22.801 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:48:22.820 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:48:22.838 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:48:22.841 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:48:22.857 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:48:22.867 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:22.876 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:48:22.893 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:48:22.894 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:48:22.894 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:48:22.894 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:48:22.894 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:48:22.894 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:48:22.894 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:48:22.895 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:48:22.895 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:48:22.910 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:48:22.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:22.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:48:23.015 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:23.051 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:23.745 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:23.748 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:23 smithi138 ceph-mon[185886]: pgmap v1691: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.9 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 5273/147 objects degraded (3587.075%); 992 KiB/s, 22 objects/s recovering 2023-10-01T09:48:23.748 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:23 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1024345243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:23.748 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:23 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 5273/147 objects degraded (3587.075%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:23.748 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:23 smithi138 ceph-mon[192513]: pgmap v1691: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.9 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 5273/147 objects degraded (3587.075%); 992 KiB/s, 22 objects/s recovering 2023-10-01T09:48:23.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:23 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1024345243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:23.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:23 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 5273/147 objects degraded (3587.075%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:23.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:23.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:23.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:23 smithi190 ceph-mon[160435]: pgmap v1691: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 5.9 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 5273/147 objects degraded (3587.075%); 992 KiB/s, 22 objects/s recovering 2023-10-01T09:48:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:23 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1024345243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:23 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 5273/147 objects degraded (3587.075%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T09:48:23.820 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1691,"stamp":"2023-10-01T09:48:22.393609+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6166451,"num_objects":49,"num_object_clones":0,"num_object_copies":147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5273,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":49,"num_whiteouts":0,"num_read":6260,"num_read_kb":12660,"num_write":167525,"num_write_kb":5341158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12397,"num_bytes_recovered":794657807,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130889,"ondisk_log_size":130889,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6229884,"kb_used_data":1073924,"kb_used_omap":39,"kb_used_meta":5155736,"kb_avail":649986180,"statfs":{"total":671965249536,"available":665585848320,"internally_reserved":0,"allocated":1099698176,"data_stored":1089628496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5279474287},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-313458712,"num_objects":-4784,"num_object_clones":0,"num_object_copies":-14352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":64,"num_objects_misplaced":-290,"num_objects_unfound":0,"num_objects_dirty":-4784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4784,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":226,"num_bytes_recovered":10158080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002662"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":336,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.547029+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:48:11.547029+0000","last_peered":"2023-10-01T09:48:11.547029+0000","last_clean":"2023-10-01T09:48:11.547029+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:48:11.547029+0000","last_undegraded":"2023-10-01T09:48:11.547029+0000","last_fullsized":"2023-10-01T09:48:11.547029+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":181,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.561642+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:48:11.561642+0000","last_peered":"2023-10-01T09:48:11.561642+0000","last_clean":"2023-10-01T09:48:11.561642+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:48:11.561642+0000","last_undegraded":"2023-10-01T09:48:11.561642+0000","last_fullsized":"2023-10-01T09:48:11.561642+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":561,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.561573+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:48:11.561573+0000","last_peered":"2023-10-01T09:48:11.561573+0000","last_clean":"2023-10-01T09:48:11.561573+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:48:11.561573+0000","last_undegraded":"2023-10-01T09:48:11.561573+0000","last_fullsized":"2023-10-01T09:48:11.561573+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":620,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.549329+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:48:11.549329+0000","last_peered":"2023-10-01T09:48:11.549329+0000","last_clean":"2023-10-01T09:48:11.549329+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:48:11.549329+0000","last_undegraded":"2023-10-01T09:48:11.549329+0000","last_fullsized":"2023-10-01T09:48:11.549329+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":501,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.548938+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:48:11.548938+0000","last_peered":"2023-10-01T09:48:11.548938+0000","last_clean":"2023-10-01T09:48:11.548938+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:48:11.548938+0000","last_undegraded":"2023-10-01T09:48:11.548938+0000","last_fullsized":"2023-10-01T09:48:11.548938+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":240,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.489230+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:48:04.489230+0000","last_peered":"2023-10-01T09:48:04.489230+0000","last_clean":"2023-10-01T09:48:04.489230+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:48:04.489230+0000","last_undegraded":"2023-10-01T09:48:04.489230+0000","last_fullsized":"2023-10-01T09:48:04.489230+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":616,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.489316+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:48:04.489316+0000","last_peered":"2023-10-01T09:48:04.489316+0000","last_clean":"2023-10-01T09:48:04.489316+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:48:04.489316+0000","last_undegraded":"2023-10-01T09:48:04.489316+0000","last_fullsized":"2023-10-01T09:48:04.489316+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":547,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.472360+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:48:04.472360+0000","last_peered":"2023-10-01T09:48:04.472360+0000","last_clean":"2023-10-01T09:48:04.472360+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:48:04.472360+0000","last_undegraded":"2023-10-01T09:48:04.472360+0000","last_fullsized":"2023-10-01T09:48:04.472360+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":271,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.486852+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:48:04.486852+0000","last_peered":"2023-10-01T09:48:04.486852+0000","last_clean":"2023-10-01T09:48:04.486852+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:48:04.486852+0000","last_undegraded":"2023-10-01T09:48:04.486852+0000","last_fullsized":"2023-10-01T09:48:04.486852+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.541597+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:48:11.541597+0000","last_peered":"2023-10-01T09:48:11.541597+0000","last_clean":"2023-10-01T09:48:11.541597+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:48:11.541597+0000","last_undegraded":"2023-10-01T09:48:11.541597+0000","last_fullsized":"2023-10-01T09:48:11.541597+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":85,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.542958+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:48:11.542958+0000","last_peered":"2023-10-01T09:48:11.542958+0000","last_clean":"2023-10-01T09:48:11.542958+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:48:11.542958+0000","last_undegraded":"2023-10-01T09:48:11.542958+0000","last_fullsized":"2023-10-01T09:48:11.542958+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":85,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.558727+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:48:11.558727+0000","last_peered":"2023-10-01T09:48:11.558727+0000","last_clean":"2023-10-01T09:48:11.558727+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:48:11.558727+0000","last_undegraded":"2023-10-01T09:48:11.558727+0000","last_fullsized":"2023-10-01T09:48:11.558727+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":461,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.549179+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:48:11.549179+0000","last_peered":"2023-10-01T09:48:11.549179+0000","last_clean":"2023-10-01T09:48:11.549179+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:48:11.549179+0000","last_undegraded":"2023-10-01T09:48:11.549179+0000","last_fullsized":"2023-10-01T09:48:11.549179+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":342,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.546458+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:48:11.546458+0000","last_peered":"2023-10-01T09:48:11.546458+0000","last_clean":"2023-10-01T09:48:11.546458+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:48:11.546458+0000","last_undegraded":"2023-10-01T09:48:11.546458+0000","last_fullsized":"2023-10-01T09:48:11.546458+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":678,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.542000+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:48:11.542000+0000","last_peered":"2023-10-01T09:48:11.542000+0000","last_clean":"2023-10-01T09:48:11.542000+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:48:11.542000+0000","last_undegraded":"2023-10-01T09:48:11.542000+0000","last_fullsized":"2023-10-01T09:48:11.542000+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":415,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.543534+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:48:11.543534+0000","last_peered":"2023-10-01T09:48:11.543534+0000","last_clean":"2023-10-01T09:48:11.543534+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:48:11.543534+0000","last_undegraded":"2023-10-01T09:48:11.543534+0000","last_fullsized":"2023-10-01T09:48:11.543534+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":88,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.541834+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:48:11.541834+0000","last_peered":"2023-10-01T09:48:11.541834+0000","last_clean":"2023-10-01T09:48:11.541834+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:48:11.541834+0000","last_undegraded":"2023-10-01T09:48:11.541834+0000","last_fullsized":"2023-10-01T09:48:11.541834+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":73,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.484301+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:48:04.484301+0000","last_peered":"2023-10-01T09:48:04.484301+0000","last_clean":"2023-10-01T09:48:04.484301+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:48:04.484301+0000","last_undegraded":"2023-10-01T09:48:04.484301+0000","last_fullsized":"2023-10-01T09:48:04.484301+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":571,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.541845+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:48:11.541845+0000","last_peered":"2023-10-01T09:48:11.541845+0000","last_clean":"2023-10-01T09:48:11.541845+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:48:11.541845+0000","last_undegraded":"2023-10-01T09:48:11.541845+0000","last_fullsized":"2023-10-01T09:48:11.541845+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":584,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.549384+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:48:11.549384+0000","last_peered":"2023-10-01T09:48:11.549384+0000","last_clean":"2023-10-01T09:48:11.549384+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:48:11.549384+0000","last_undegraded":"2023-10-01T09:48:11.549384+0000","last_fullsized":"2023-10-01T09:48:11.549384+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":440,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.548862+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:48:11.548862+0000","last_peered":"2023-10-01T09:48:11.548862+0000","last_clean":"2023-10-01T09:48:11.548862+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:48:11.548862+0000","last_undegraded":"2023-10-01T09:48:11.548862+0000","last_fullsized":"2023-10-01T09:48:11.548862+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":88,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.542710+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:48:11.542710+0000","last_peered":"2023-10-01T09:48:11.542710+0000","last_clean":"2023-10-01T09:48:11.542710+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:48:11.542710+0000","last_undegraded":"2023-10-01T09:48:11.542710+0000","last_fullsized":"2023-10-01T09:48:11.542710+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"559'10392","reported_seq":10484,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:18.137605+0000","last_change":"2023-10-01T09:40:06.130786+0000","last_active":"2023-10-01T09:48:18.137605+0000","last_peered":"2023-10-01T09:48:18.137605+0000","last_clean":"2023-10-01T09:48:18.137605+0000","last_became_active":"2023-10-01T09:40:06.130512+0000","last_became_peered":"2023-10-01T09:40:06.130512+0000","last_unstale":"2023-10-01T09:48:18.137605+0000","last_undegraded":"2023-10-01T09:48:18.137605+0000","last_fullsized":"2023-10-01T09:48:18.137605+0000","mapping_epoch":475,"log_start":"481'300","ondisk_log_start":"481'300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":10092,"log_dups_size":0,"ondisk_log_size":10092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10392,"num_write_kb":332544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":83,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.542551+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:48:11.542551+0000","last_peered":"2023-10-01T09:48:11.542551+0000","last_clean":"2023-10-01T09:48:11.542551+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:48:11.542551+0000","last_undegraded":"2023-10-01T09:48:11.542551+0000","last_fullsized":"2023-10-01T09:48:11.542551+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":279,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.483957+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:48:04.483957+0000","last_peered":"2023-10-01T09:48:04.483957+0000","last_clean":"2023-10-01T09:48:04.483957+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:48:04.483957+0000","last_undegraded":"2023-10-01T09:48:04.483957+0000","last_fullsized":"2023-10-01T09:48:04.483957+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"559'10406","reported_seq":21608,"reported_epoch":559,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T09:48:18.121941+0000","last_change":"2023-10-01T09:41:38.398844+0000","last_active":"2023-10-01T09:48:18.121941+0000","last_peered":"2023-10-01T09:48:18.121941+0000","last_clean":"2023-10-01T09:40:54.372590+0000","last_became_active":"2023-10-01T09:41:38.144679+0000","last_became_peered":"2023-10-01T09:41:38.144679+0000","last_unstale":"2023-10-01T09:48:18.121941+0000","last_undegraded":"2023-10-01T09:41:34.629037+0000","last_fullsized":"2023-10-01T09:41:23.759108+0000","mapping_epoch":503,"log_start":"481'400","ondisk_log_start":"481'400","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":10006,"log_dups_size":0,"ondisk_log_size":10006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1437,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10406,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3698,"num_bytes_recovered":242352128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6],"avail_no_missing":["5","0","4","6"],"object_location_counts":[{"shards":"5,6","objects":-1437}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":554,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.542447+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:48:11.542447+0000","last_peered":"2023-10-01T09:48:11.542447+0000","last_clean":"2023-10-01T09:48:11.542447+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:48:11.542447+0000","last_undegraded":"2023-10-01T09:48:11.542447+0000","last_fullsized":"2023-10-01T09:48:11.542447+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":617,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.561490+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:48:11.561490+0000","last_peered":"2023-10-01T09:48:11.561490+0000","last_clean":"2023-10-01T09:48:11.561490+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:48:11.561490+0000","last_undegraded":"2023-10-01T09:48:11.561490+0000","last_fullsized":"2023-10-01T09:48:11.561490+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"559'10432","reported_seq":10529,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:18.138701+0000","last_change":"2023-10-01T09:40:06.129529+0000","last_active":"2023-10-01T09:48:18.138701+0000","last_peered":"2023-10-01T09:48:18.138701+0000","last_clean":"2023-10-01T09:48:18.138701+0000","last_became_active":"2023-10-01T09:40:06.129219+0000","last_became_peered":"2023-10-01T09:40:06.129219+0000","last_unstale":"2023-10-01T09:48:18.138701+0000","last_undegraded":"2023-10-01T09:48:18.138701+0000","last_fullsized":"2023-10-01T09:48:18.138701+0000","mapping_epoch":475,"log_start":"493'3400","ondisk_log_start":"493'3400","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7032,"log_dups_size":0,"ondisk_log_size":7032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:28:00.367414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10432,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":640,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.543039+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:48:11.543039+0000","last_peered":"2023-10-01T09:48:11.543039+0000","last_clean":"2023-10-01T09:48:11.543039+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:48:11.543039+0000","last_undegraded":"2023-10-01T09:48:11.543039+0000","last_fullsized":"2023-10-01T09:48:11.543039+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":437,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.558931+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:48:11.558931+0000","last_peered":"2023-10-01T09:48:11.558931+0000","last_clean":"2023-10-01T09:48:11.558931+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:48:11.558931+0000","last_undegraded":"2023-10-01T09:48:11.558931+0000","last_fullsized":"2023-10-01T09:48:11.558931+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"559'10402","reported_seq":17104,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:18.137957+0000","last_change":"2023-10-01T09:41:30.777755+0000","last_active":"2023-10-01T09:48:18.137957+0000","last_peered":"2023-10-01T09:48:18.137957+0000","last_clean":"2023-10-01T09:48:18.137957+0000","last_became_active":"2023-10-01T09:41:30.774100+0000","last_became_peered":"2023-10-01T09:41:30.774100+0000","last_unstale":"2023-10-01T09:48:18.137957+0000","last_undegraded":"2023-10-01T09:48:18.137957+0000","last_fullsized":"2023-10-01T09:48:18.137957+0000","mapping_epoch":505,"log_start":"493'3400","ondisk_log_start":"493'3400","created":475,"last_epoch_clean":506,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7002,"log_dups_size":0,"ondisk_log_size":7002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:12.867373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10402,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3291,"num_bytes_recovered":215678976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":279,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.484088+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:48:04.484088+0000","last_peered":"2023-10-01T09:48:04.484088+0000","last_clean":"2023-10-01T09:48:04.484088+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:48:04.484088+0000","last_undegraded":"2023-10-01T09:48:04.484088+0000","last_fullsized":"2023-10-01T09:48:04.484088+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":540,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.548847+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:48:11.548847+0000","last_peered":"2023-10-01T09:48:11.548847+0000","last_clean":"2023-10-01T09:48:11.548847+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:48:11.548847+0000","last_undegraded":"2023-10-01T09:48:11.548847+0000","last_fullsized":"2023-10-01T09:48:11.548847+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"559'10508","reported_seq":10605,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:18.115853+0000","last_change":"2023-10-01T09:40:06.128687+0000","last_active":"2023-10-01T09:48:18.115853+0000","last_peered":"2023-10-01T09:48:18.115853+0000","last_clean":"2023-10-01T09:48:18.115853+0000","last_became_active":"2023-10-01T09:40:06.128346+0000","last_became_peered":"2023-10-01T09:40:06.128346+0000","last_unstale":"2023-10-01T09:48:18.115853+0000","last_undegraded":"2023-10-01T09:48:18.115853+0000","last_fullsized":"2023-10-01T09:48:18.115853+0000","mapping_epoch":475,"log_start":"489'3000","ondisk_log_start":"489'3000","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7508,"log_dups_size":0,"ondisk_log_size":7508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:26.691295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10508,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":140,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.546791+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:48:11.546791+0000","last_peered":"2023-10-01T09:48:11.546791+0000","last_clean":"2023-10-01T09:48:11.546791+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:48:11.546791+0000","last_undegraded":"2023-10-01T09:48:11.546791+0000","last_fullsized":"2023-10-01T09:48:11.546791+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":214,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.549310+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:48:11.549310+0000","last_peered":"2023-10-01T09:48:11.549310+0000","last_clean":"2023-10-01T09:48:11.549310+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:48:11.549310+0000","last_undegraded":"2023-10-01T09:48:11.549310+0000","last_fullsized":"2023-10-01T09:48:11.549310+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"559'10368","reported_seq":13058,"reported_epoch":559,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T09:48:19.551592+0000","last_change":"2023-10-01T09:41:54.775903+0000","last_active":"2023-10-01T09:48:19.551592+0000","last_peered":"2023-10-01T09:48:19.551592+0000","last_clean":"2023-10-01T09:41:22.359386+0000","last_became_active":"2023-10-01T09:41:23.919979+0000","last_became_peered":"2023-10-01T09:41:23.919979+0000","last_unstale":"2023-10-01T09:48:19.551592+0000","last_undegraded":"2023-10-01T09:41:23.789486+0000","last_fullsized":"2023-10-01T09:41:23.759401+0000","mapping_epoch":503,"log_start":"481'300","ondisk_log_start":"481'300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":10068,"log_dups_size":0,"ondisk_log_size":10068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3826,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10368,"num_write_kb":331776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1295,"num_bytes_recovered":80216064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"4,5","objects":-3826}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":458,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.547155+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:48:11.547155+0000","last_peered":"2023-10-01T09:48:11.547155+0000","last_clean":"2023-10-01T09:48:11.547155+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:48:11.547155+0000","last_undegraded":"2023-10-01T09:48:11.547155+0000","last_fullsized":"2023-10-01T09:48:11.547155+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":620,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.561724+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:48:11.561724+0000","last_peered":"2023-10-01T09:48:11.561724+0000","last_clean":"2023-10-01T09:48:11.561724+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:48:11.561724+0000","last_undegraded":"2023-10-01T09:48:11.561724+0000","last_fullsized":"2023-10-01T09:48:11.561724+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.0","version":"559'10357","reported_seq":10458,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:17.584027+0000","last_change":"2023-10-01T09:40:25.379906+0000","last_active":"2023-10-01T09:48:17.584027+0000","last_peered":"2023-10-01T09:48:17.584027+0000","last_clean":"2023-10-01T09:48:17.584027+0000","last_became_active":"2023-10-01T09:40:25.366360+0000","last_became_peered":"2023-10-01T09:40:25.366360+0000","last_unstale":"2023-10-01T09:48:17.584027+0000","last_undegraded":"2023-10-01T09:48:17.584027+0000","last_fullsized":"2023-10-01T09:48:17.584027+0000","mapping_epoch":483,"log_start":"487'2800","ondisk_log_start":"487'2800","created":475,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7557,"log_dups_size":0,"ondisk_log_size":7557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1376256,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10357,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":454,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.550095+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:48:11.550095+0000","last_peered":"2023-10-01T09:48:11.550095+0000","last_clean":"2023-10-01T09:48:11.550095+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:48:11.550095+0000","last_undegraded":"2023-10-01T09:48:11.550095+0000","last_fullsized":"2023-10-01T09:48:11.550095+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":295,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.554866+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:48:11.554866+0000","last_peered":"2023-10-01T09:48:11.554866+0000","last_clean":"2023-10-01T09:48:11.554866+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:48:11.554866+0000","last_undegraded":"2023-10-01T09:48:11.554866+0000","last_fullsized":"2023-10-01T09:48:11.554866+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"559'10498","reported_seq":10591,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:18.140714+0000","last_change":"2023-10-01T09:40:06.128411+0000","last_active":"2023-10-01T09:48:18.140714+0000","last_peered":"2023-10-01T09:48:18.140714+0000","last_clean":"2023-10-01T09:48:18.140714+0000","last_became_active":"2023-10-01T09:40:06.128168+0000","last_became_peered":"2023-10-01T09:40:06.128168+0000","last_unstale":"2023-10-01T09:48:18.140714+0000","last_undegraded":"2023-10-01T09:48:18.140714+0000","last_fullsized":"2023-10-01T09:48:18.140714+0000","mapping_epoch":475,"log_start":"485'2100","ondisk_log_start":"485'2100","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8398,"log_dups_size":0,"ondisk_log_size":8398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10498,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":425,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.561132+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:48:11.561132+0000","last_peered":"2023-10-01T09:48:11.561132+0000","last_clean":"2023-10-01T09:48:11.561132+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:48:11.561132+0000","last_undegraded":"2023-10-01T09:48:11.561132+0000","last_fullsized":"2023-10-01T09:48:11.561132+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":472,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.561640+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:48:11.561640+0000","last_peered":"2023-10-01T09:48:11.561640+0000","last_clean":"2023-10-01T09:48:11.561640+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:48:11.561640+0000","last_undegraded":"2023-10-01T09:48:11.561640+0000","last_fullsized":"2023-10-01T09:48:11.561640+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"559'10178","reported_seq":10271,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:18.138633+0000","last_change":"2023-10-01T09:40:06.129149+0000","last_active":"2023-10-01T09:48:18.138633+0000","last_peered":"2023-10-01T09:48:18.138633+0000","last_clean":"2023-10-01T09:48:18.138633+0000","last_became_active":"2023-10-01T09:40:06.128860+0000","last_became_peered":"2023-10-01T09:40:06.128860+0000","last_unstale":"2023-10-01T09:48:18.138633+0000","last_undegraded":"2023-10-01T09:48:18.138633+0000","last_fullsized":"2023-10-01T09:48:18.138633+0000","mapping_epoch":475,"log_start":"484'1800","ondisk_log_start":"484'1800","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8378,"log_dups_size":0,"ondisk_log_size":8378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10178,"num_write_kb":325696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":953,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.543394+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:48:11.543394+0000","last_peered":"2023-10-01T09:48:11.543394+0000","last_clean":"2023-10-01T09:48:11.543394+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:48:11.543394+0000","last_undegraded":"2023-10-01T09:48:11.543394+0000","last_fullsized":"2023-10-01T09:48:11.543394+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":289,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.484439+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:48:04.484439+0000","last_peered":"2023-10-01T09:48:04.484439+0000","last_clean":"2023-10-01T09:48:04.484439+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:48:04.484439+0000","last_undegraded":"2023-10-01T09:48:04.484439+0000","last_fullsized":"2023-10-01T09:48:04.484439+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":85,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.558801+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:48:11.558801+0000","last_peered":"2023-10-01T09:48:11.558801+0000","last_clean":"2023-10-01T09:48:11.558801+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:48:11.558801+0000","last_undegraded":"2023-10-01T09:48:11.558801+0000","last_fullsized":"2023-10-01T09:48:11.558801+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"559'10370","reported_seq":10467,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:18.137600+0000","last_change":"2023-10-01T09:40:05.736854+0000","last_active":"2023-10-01T09:48:18.137600+0000","last_peered":"2023-10-01T09:48:18.137600+0000","last_clean":"2023-10-01T09:48:18.137600+0000","last_became_active":"2023-10-01T09:40:05.736612+0000","last_became_peered":"2023-10-01T09:40:05.736612+0000","last_unstale":"2023-10-01T09:48:18.137600+0000","last_undegraded":"2023-10-01T09:48:18.137600+0000","last_fullsized":"2023-10-01T09:48:18.137600+0000","mapping_epoch":475,"log_start":"493'3300","ondisk_log_start":"493'3300","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7070,"log_dups_size":0,"ondisk_log_size":7070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:48:41.853664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10370,"num_write_kb":331840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1224,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.486783+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:48:04.486783+0000","last_peered":"2023-10-01T09:48:04.486783+0000","last_clean":"2023-10-01T09:48:04.486783+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:48:04.486783+0000","last_undegraded":"2023-10-01T09:48:04.486783+0000","last_fullsized":"2023-10-01T09:48:04.486783+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":435,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.546653+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:48:11.546653+0000","last_peered":"2023-10-01T09:48:11.546653+0000","last_clean":"2023-10-01T09:48:11.546653+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:48:11.546653+0000","last_undegraded":"2023-10-01T09:48:11.546653+0000","last_fullsized":"2023-10-01T09:48:11.546653+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":262,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.484224+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:48:04.484224+0000","last_peered":"2023-10-01T09:48:04.484224+0000","last_clean":"2023-10-01T09:48:04.484224+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:48:04.484224+0000","last_undegraded":"2023-10-01T09:48:04.484224+0000","last_fullsized":"2023-10-01T09:48:04.484224+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.9","version":"559'10376","reported_seq":17689,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:18.144295+0000","last_change":"2023-10-01T09:41:56.782961+0000","last_active":"2023-10-01T09:48:18.144295+0000","last_peered":"2023-10-01T09:48:18.144295+0000","last_clean":"2023-10-01T09:48:18.144295+0000","last_became_active":"2023-10-01T09:41:56.773138+0000","last_became_peered":"2023-10-01T09:41:56.773138+0000","last_unstale":"2023-10-01T09:48:18.144295+0000","last_undegraded":"2023-10-01T09:48:18.144295+0000","last_fullsized":"2023-10-01T09:48:18.144295+0000","mapping_epoch":509,"log_start":"481'300","ondisk_log_start":"481'300","created":475,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":10076,"log_dups_size":0,"ondisk_log_size":10076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10376,"num_write_kb":332032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":236060672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":907,"reported_epoch":559,"state":"active+recovery_wait+degraded","last_fresh":"2023-10-01T09:48:11.549951+0000","last_change":"2023-10-01T09:41:22.763216+0000","last_active":"2023-10-01T09:48:11.549951+0000","last_peered":"2023-10-01T09:48:11.549951+0000","last_clean":"2023-10-01T09:41:14.390621+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:48:11.549951+0000","last_undegraded":"2023-10-01T09:41:22.745804+0000","last_fullsized":"2023-10-01T09:48:11.549951+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":625,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:15.293324+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:48:15.293324+0000","last_peered":"2023-10-01T09:48:15.293324+0000","last_clean":"2023-10-01T09:48:15.293324+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:48:15.293324+0000","last_undegraded":"2023-10-01T09:48:15.293324+0000","last_fullsized":"2023-10-01T09:48:15.293324+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":85,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.561516+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:48:11.561516+0000","last_peered":"2023-10-01T09:48:11.561516+0000","last_clean":"2023-10-01T09:48:11.561516+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:48:11.561516+0000","last_undegraded":"2023-10-01T09:48:11.561516+0000","last_fullsized":"2023-10-01T09:48:11.561516+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"559'10272","reported_seq":10369,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:18.115393+0000","last_change":"2023-10-01T09:40:05.735376+0000","last_active":"2023-10-01T09:48:18.115393+0000","last_peered":"2023-10-01T09:48:18.115393+0000","last_clean":"2023-10-01T09:48:18.115393+0000","last_became_active":"2023-10-01T09:40:05.735096+0000","last_became_peered":"2023-10-01T09:40:05.735096+0000","last_unstale":"2023-10-01T09:48:18.115393+0000","last_undegraded":"2023-10-01T09:48:18.115393+0000","last_fullsized":"2023-10-01T09:48:18.115393+0000","mapping_epoch":475,"log_start":"487'2700","ondisk_log_start":"487'2700","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7572,"log_dups_size":0,"ondisk_log_size":7572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:13.921861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10272,"num_write_kb":328704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1675,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.491321+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:48:04.491321+0000","last_peered":"2023-10-01T09:48:04.491321+0000","last_clean":"2023-10-01T09:48:04.491321+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:48:04.491321+0000","last_undegraded":"2023-10-01T09:48:04.491321+0000","last_fullsized":"2023-10-01T09:48:04.491321+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":590,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.544415+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:48:11.544415+0000","last_peered":"2023-10-01T09:48:11.544415+0000","last_clean":"2023-10-01T09:48:11.544415+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:48:11.544415+0000","last_undegraded":"2023-10-01T09:48:11.544415+0000","last_fullsized":"2023-10-01T09:48:11.544415+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":367,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.486877+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:48:04.486877+0000","last_peered":"2023-10-01T09:48:04.486877+0000","last_clean":"2023-10-01T09:48:04.486877+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:48:04.486877+0000","last_undegraded":"2023-10-01T09:48:04.486877+0000","last_fullsized":"2023-10-01T09:48:04.486877+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":358,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.546507+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:48:11.546507+0000","last_peered":"2023-10-01T09:48:11.546507+0000","last_clean":"2023-10-01T09:48:11.546507+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:48:11.546507+0000","last_undegraded":"2023-10-01T09:48:11.546507+0000","last_fullsized":"2023-10-01T09:48:11.546507+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"559'10532","reported_seq":10625,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:18.139637+0000","last_change":"2023-10-01T09:40:05.737371+0000","last_active":"2023-10-01T09:48:18.139637+0000","last_peered":"2023-10-01T09:48:18.139637+0000","last_clean":"2023-10-01T09:48:18.139637+0000","last_became_active":"2023-10-01T09:40:05.737072+0000","last_became_peered":"2023-10-01T09:40:05.737072+0000","last_unstale":"2023-10-01T09:48:18.139637+0000","last_undegraded":"2023-10-01T09:48:18.139637+0000","last_fullsized":"2023-10-01T09:48:18.139637+0000","mapping_epoch":475,"log_start":"485'2100","ondisk_log_start":"485'2100","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":8432,"log_dups_size":0,"ondisk_log_size":8432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10532,"num_write_kb":337024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":767,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.544615+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:48:11.544615+0000","last_peered":"2023-10-01T09:48:11.544615+0000","last_clean":"2023-10-01T09:48:11.544615+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:48:11.544615+0000","last_undegraded":"2023-10-01T09:48:11.544615+0000","last_fullsized":"2023-10-01T09:48:11.544615+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":601,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.550156+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:48:11.550156+0000","last_peered":"2023-10-01T09:48:11.550156+0000","last_clean":"2023-10-01T09:48:11.550156+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:48:11.550156+0000","last_undegraded":"2023-10-01T09:48:11.550156+0000","last_fullsized":"2023-10-01T09:48:11.550156+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":601,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.549224+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:48:11.549224+0000","last_peered":"2023-10-01T09:48:11.549224+0000","last_clean":"2023-10-01T09:48:11.549224+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:48:11.549224+0000","last_undegraded":"2023-10-01T09:48:11.549224+0000","last_fullsized":"2023-10-01T09:48:11.549224+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"559'10460","reported_seq":10558,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:18.150384+0000","last_change":"2023-10-01T09:40:06.128713+0000","last_active":"2023-10-01T09:48:18.150384+0000","last_peered":"2023-10-01T09:48:18.150384+0000","last_clean":"2023-10-01T09:48:18.150384+0000","last_became_active":"2023-10-01T09:40:06.128397+0000","last_became_peered":"2023-10-01T09:40:06.128397+0000","last_unstale":"2023-10-01T09:48:18.150384+0000","last_undegraded":"2023-10-01T09:48:18.150384+0000","last_fullsized":"2023-10-01T09:48:18.150384+0000","mapping_epoch":475,"log_start":"493'3400","ondisk_log_start":"493'3400","created":475,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":7060,"log_dups_size":0,"ondisk_log_size":7060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:08:14.738684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10460,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":998,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.543117+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:48:11.543117+0000","last_peered":"2023-10-01T09:48:11.543117+0000","last_clean":"2023-10-01T09:48:11.543117+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:48:11.543117+0000","last_undegraded":"2023-10-01T09:48:11.543117+0000","last_fullsized":"2023-10-01T09:48:11.543117+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":616,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.484158+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:48:04.484158+0000","last_peered":"2023-10-01T09:48:04.484158+0000","last_clean":"2023-10-01T09:48:04.484158+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:48:04.484158+0000","last_undegraded":"2023-10-01T09:48:04.484158+0000","last_fullsized":"2023-10-01T09:48:04.484158+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":469,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.491416+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:48:04.491416+0000","last_peered":"2023-10-01T09:48:04.491416+0000","last_clean":"2023-10-01T09:48:04.491416+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:48:04.491416+0000","last_undegraded":"2023-10-01T09:48:04.491416+0000","last_fullsized":"2023-10-01T09:48:04.491416+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"559'10356","reported_seq":10457,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:18.142606+0000","last_change":"2023-10-01T09:40:13.190830+0000","last_active":"2023-10-01T09:48:18.142606+0000","last_peered":"2023-10-01T09:48:18.142606+0000","last_clean":"2023-10-01T09:48:18.142606+0000","last_became_active":"2023-10-01T09:40:13.190422+0000","last_became_peered":"2023-10-01T09:40:13.190422+0000","last_unstale":"2023-10-01T09:48:18.142606+0000","last_undegraded":"2023-10-01T09:48:18.142606+0000","last_fullsized":"2023-10-01T09:48:18.142606+0000","mapping_epoch":479,"log_start":"494'3600","ondisk_log_start":"494'3600","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6756,"log_dups_size":0,"ondisk_log_size":6756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10356,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":499,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.542796+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:48:11.542796+0000","last_peered":"2023-10-01T09:48:11.542796+0000","last_clean":"2023-10-01T09:48:11.542796+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:48:11.542796+0000","last_undegraded":"2023-10-01T09:48:11.542796+0000","last_fullsized":"2023-10-01T09:48:11.542796+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1194,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.562055+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:48:11.562055+0000","last_peered":"2023-10-01T09:48:11.562055+0000","last_clean":"2023-10-01T09:48:11.562055+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:48:11.562055+0000","last_undegraded":"2023-10-01T09:48:11.562055+0000","last_fullsized":"2023-10-01T09:48:11.562055+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":88,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.544299+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:48:11.544299+0000","last_peered":"2023-10-01T09:48:11.544299+0000","last_clean":"2023-10-01T09:48:11.544299+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:48:11.544299+0000","last_undegraded":"2023-10-01T09:48:11.544299+0000","last_fullsized":"2023-10-01T09:48:11.544299+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"559'10512","reported_seq":10613,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:18.122033+0000","last_change":"2023-10-01T09:40:13.195492+0000","last_active":"2023-10-01T09:48:18.122033+0000","last_peered":"2023-10-01T09:48:18.122033+0000","last_clean":"2023-10-01T09:48:18.122033+0000","last_became_active":"2023-10-01T09:40:13.195021+0000","last_became_peered":"2023-10-01T09:40:13.195021+0000","last_unstale":"2023-10-01T09:48:18.122033+0000","last_undegraded":"2023-10-01T09:48:18.122033+0000","last_fullsized":"2023-10-01T09:48:18.122033+0000","mapping_epoch":479,"log_start":"497'3800","ondisk_log_start":"497'3800","created":475,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:40:04.717450+0000","last_clean_scrub_stamp":"2023-10-01T09:40:04.717450+0000","objects_scrubbed":0,"log_size":6712,"log_dups_size":0,"ondisk_log_size":6712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10512,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":92,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.542635+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:48:11.542635+0000","last_peered":"2023-10-01T09:48:11.542635+0000","last_clean":"2023-10-01T09:48:11.542635+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:48:11.542635+0000","last_undegraded":"2023-10-01T09:48:11.542635+0000","last_fullsized":"2023-10-01T09:48:11.542635+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":579,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.549314+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:48:11.549314+0000","last_peered":"2023-10-01T09:48:11.549314+0000","last_clean":"2023-10-01T09:48:11.549314+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:48:11.549314+0000","last_undegraded":"2023-10-01T09:48:11.549314+0000","last_fullsized":"2023-10-01T09:48:11.549314+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":282,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.487879+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:48:04.487879+0000","last_peered":"2023-10-01T09:48:04.487879+0000","last_clean":"2023-10-01T09:48:04.487879+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:48:04.487879+0000","last_undegraded":"2023-10-01T09:48:04.487879+0000","last_fullsized":"2023-10-01T09:48:04.487879+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":527,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.548941+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:48:11.548941+0000","last_peered":"2023-10-01T09:48:11.548941+0000","last_clean":"2023-10-01T09:48:11.548941+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:48:11.548941+0000","last_undegraded":"2023-10-01T09:48:11.548941+0000","last_fullsized":"2023-10-01T09:48:11.548941+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":641,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.561477+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:48:11.561477+0000","last_peered":"2023-10-01T09:48:11.561477+0000","last_clean":"2023-10-01T09:48:11.561477+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:48:11.561477+0000","last_undegraded":"2023-10-01T09:48:11.561477+0000","last_fullsized":"2023-10-01T09:48:11.561477+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":294,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.484012+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:48:04.484012+0000","last_peered":"2023-10-01T09:48:04.484012+0000","last_clean":"2023-10-01T09:48:04.484012+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:48:04.484012+0000","last_undegraded":"2023-10-01T09:48:04.484012+0000","last_fullsized":"2023-10-01T09:48:04.484012+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:15.293409+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:48:15.293409+0000","last_peered":"2023-10-01T09:48:15.293409+0000","last_clean":"2023-10-01T09:48:15.293409+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:48:15.293409+0000","last_undegraded":"2023-10-01T09:48:15.293409+0000","last_fullsized":"2023-10-01T09:48:15.293409+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":294,"reported_epoch":558,"state":"active+clean","last_fresh":"2023-10-01T09:48:04.484513+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:48:04.484513+0000","last_peered":"2023-10-01T09:48:04.484513+0000","last_clean":"2023-10-01T09:48:04.484513+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:48:04.484513+0000","last_undegraded":"2023-10-01T09:48:04.484513+0000","last_fullsized":"2023-10-01T09:48:04.484513+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":474,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:12.098450+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:48:12.098450+0000","last_peered":"2023-10-01T09:48:12.098450+0000","last_clean":"2023-10-01T09:48:12.098450+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:48:12.098450+0000","last_undegraded":"2023-10-01T09:48:12.098450+0000","last_fullsized":"2023-10-01T09:48:12.098450+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":103,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.561869+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:48:11.561869+0000","last_peered":"2023-10-01T09:48:11.561869+0000","last_clean":"2023-10-01T09:48:11.561869+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:48:11.561869+0000","last_undegraded":"2023-10-01T09:48:11.561869+0000","last_fullsized":"2023-10-01T09:48:11.561869+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":519,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.550081+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:48:11.550081+0000","last_peered":"2023-10-01T09:48:11.550081+0000","last_clean":"2023-10-01T09:48:11.550081+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:48:11.550081+0000","last_undegraded":"2023-10-01T09:48:11.550081+0000","last_fullsized":"2023-10-01T09:48:11.550081+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":718,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.549387+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:48:11.549387+0000","last_peered":"2023-10-01T09:48:11.549387+0000","last_clean":"2023-10-01T09:48:11.549387+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:48:11.549387+0000","last_undegraded":"2023-10-01T09:48:11.549387+0000","last_fullsized":"2023-10-01T09:48:11.549387+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":411,"reported_epoch":559,"state":"active+clean","last_fresh":"2023-10-01T09:48:11.546236+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:48:11.546236+0000","last_peered":"2023-10-01T09:48:11.546236+0000","last_clean":"2023-10-01T09:48:11.546236+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:48:11.546236+0000","last_undegraded":"2023-10-01T09:48:11.546236+0000","last_fullsized":"2023-10-01T09:48:11.546236+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1376256,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5263,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":166419,"num_write_kb":5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11886,"num_bytes_recovered":774307840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1025249280,"data_stored":1025245208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129719,"ondisk_log_size":129719,"up":48,"acting":46,"num_store_stats":7},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":372,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430773,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":916944,"kb_used_data":9736,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":92828208,"statfs":{"total":95995035648,"available":95056084992,"internally_reserved":0,"allocated":9969664,"data_stored":9020944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":0.67500000000000004,"15min":2.4630000000000001},"min":{"1min":0.252,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":7.6900000000000004,"5min":7.6900000000000004,"15min":108.90000000000001},"last":5.8399999999999999},{"interface":"front","average":{"1min":1.196,"5min":0.60999999999999999,"15min":2.5680000000000001},"min":{"1min":0.32100000000000001,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":6.5199999999999996,"5min":6.5199999999999996,"15min":109.187},"last":0.64300000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.58299999999999996,"15min":5.0190000000000001},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.23300000000000001},"max":{"1min":7.7140000000000004,"5min":7.7140000000000004,"15min":229.161},"last":5.875},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.53100000000000003,"15min":5.0449999999999999},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":1.972,"5min":3.0640000000000001,"15min":230.126},"last":0.71599999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":0.86099999999999999,"15min":1.54},"min":{"1min":0.33300000000000002,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":12.565,"5min":12.565,"15min":14.901999999999999},"last":5.8979999999999997},{"interface":"front","average":{"1min":1.121,"5min":0.68200000000000005,"15min":1.429},"min":{"1min":0.372,"5min":0.32000000000000001,"15min":0.27000000000000002},"max":{"1min":6.4550000000000001,"5min":6.4550000000000001,"15min":14.682},"last":0.73899999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:47:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.59299999999999997,"15min":2.1059999999999999},"min":{"1min":0.34999999999999998,"5min":0.30199999999999999,"15min":0.29199999999999998},"max":{"1min":2.1749999999999998,"5min":2.476,"15min":76.555000000000007},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.58399999999999996,"15min":2.1120000000000001},"min":{"1min":0.315,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.1880000000000002,"5min":2.8479999999999999,"15min":76.534000000000006},"last":0.33200000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:48:03 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":0.79700000000000004,"15min":1.04},"min":{"1min":0.316,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.069000000000001,"5min":11.069000000000001,"15min":11.069000000000001},"last":5.9240000000000004},{"interface":"front","average":{"1min":1.3700000000000001,"5min":0.70099999999999996,"15min":1.002},"min":{"1min":0.33700000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.4809999999999999,"5min":6.4809999999999999,"15min":8.4949999999999992},"last":0.69699999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":0.66600000000000004,"15min":1.5369999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.9370000000000003,"5min":7.9370000000000003,"15min":135.40000000000001},"last":3.0800000000000001},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.65100000000000002,"15min":1.6379999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":7.0629999999999997,"5min":7.0629999999999997,"15min":135.339},"last":3.2599999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.64900000000000002,"15min":1.1519999999999999},"min":{"1min":0.39600000000000002,"5min":0.377,"15min":0.307},"max":{"1min":3.984,"5min":3.984,"15min":11.853999999999999},"last":0.98399999999999999},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.64100000000000001,"15min":1.0880000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32300000000000001},"max":{"1min":4.3470000000000004,"5min":4.3470000000000004,"15min":14.584},"last":3.3330000000000002}]}]},{"osd":6,"up_from":294,"seq":1262720385283,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":357444,"kb_used_data":27900,"kb_used_omap":6,"kb_used_meta":329401,"kb_avail":93387708,"statfs":{"total":95995035648,"available":95629012992,"internally_reserved":0,"allocated":28569600,"data_stored":27625155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":337306669},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":4.5839999999999996,"5min":1.2829999999999999,"15min":2.665},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20599999999999999},"max":{"1min":65.599000000000004,"5min":65.599000000000004,"15min":158.48599999999999},"last":0.223},{"interface":"front","average":{"1min":5.4089999999999998,"5min":1.484,"15min":2.7909999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":65.388999999999996,"5min":65.388999999999996,"15min":158.37700000000001},"last":0.44600000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":0.70599999999999996,"15min":3.8839999999999999},"min":{"1min":0.34399999999999997,"5min":0.222,"15min":0.222},"max":{"1min":8.2409999999999997,"5min":8.2409999999999997,"15min":168.947},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.625,"5min":0.56999999999999995,"15min":3.8210000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.25600000000000001},"max":{"1min":2.6669999999999998,"5min":3.7469999999999999,"15min":169.07900000000001},"last":0.315}]},{"osd":2,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":0.73799999999999999,"15min":1.429},"min":{"1min":0.311,"5min":0.254,"15min":0.254},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":10.92},"last":0.502},{"interface":"front","average":{"1min":0.748,"5min":0.57199999999999995,"15min":1.3899999999999999},"min":{"1min":0.314,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":2.6480000000000001,"5min":3.7160000000000002,"15min":21.704999999999998},"last":0.34799999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55000000000000004,"15min":1.647},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.9710000000000001,"5min":3.3759999999999999,"15min":89.753},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.51,"5min":0.78200000000000003,"15min":1.7589999999999999},"min":{"1min":0.34399999999999997,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.3109999999999999,"5min":8.3109999999999999,"15min":89.849999999999994},"last":0.52600000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.505,"15min":1.0609999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.7559999999999998,"5min":3.6349999999999998,"15min":63.134},"last":2.2480000000000002},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.54400000000000004,"15min":1.0429999999999999},"min":{"1min":0.34499999999999997,"5min":0.33600000000000002,"15min":0.28899999999999998},"max":{"1min":2.778,"5min":3.2519999999999998,"15min":62.973999999999997},"last":2.2570000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:47:37 2023","interfaces":[{"interface":"back","average":{"1min":6.5650000000000004,"5min":2.089,"15min":1.367},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":136.535,"5min":136.535,"15min":136.535},"last":6.0919999999999996},{"interface":"front","average":{"1min":6.3049999999999997,"5min":2.1499999999999999,"15min":1.4830000000000001},"min":{"1min":0.316,"5min":0.29899999999999999,"15min":0.28100000000000003},"max":{"1min":136.47300000000001,"5min":136.47300000000001,"15min":136.47300000000001},"last":7.1689999999999996}]},{"osd":7,"last update":"Sun Oct 1 09:47:21 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.56100000000000005,"15min":1.2609999999999999},"min":{"1min":0.38300000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.3159999999999998,"5min":3.339,"15min":94.245999999999995},"last":2.121},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.76700000000000002,"15min":1.421},"min":{"1min":0.37,"5min":0.32400000000000001,"15min":0.28799999999999998},"max":{"1min":8.3399999999999999,"5min":8.3399999999999999,"15min":94.715000000000003},"last":0.55000000000000004}]}]},{"osd":1,"up_from":94,"seq":403726926260,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":850744,"kb_used_data":10416,"kb_used_omap":7,"kb_used_meta":840312,"kb_avail":92894408,"statfs":{"total":95995035648,"available":95123873792,"internally_reserved":0,"allocated":10665984,"data_stored":8565875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":860479911},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":11.182,"5min":3.2109999999999999,"15min":4.0659999999999998},"min":{"1min":0.217,"5min":0.186,"15min":0.182},"max":{"1min":114.33499999999999,"5min":114.33499999999999,"15min":154.21299999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":11.305,"5min":3.2799999999999998,"15min":3.637},"min":{"1min":0.16800000000000001,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":114.099,"5min":114.099,"15min":114.099},"last":0.28699999999999998}]},{"osd":2,"last update":"Sun Oct 1 09:47:45 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.42199999999999999,"15min":1.8109999999999999},"min":{"1min":0.23699999999999999,"5min":0.223,"15min":0.20000000000000001},"max":{"1min":3.6019999999999999,"5min":3.6019999999999999,"15min":51.640000000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.3049999999999999,"5min":0.63500000000000001,"15min":2.5219999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20499999999999999},"max":{"1min":12.018000000000001,"5min":12.018000000000001,"15min":154.197},"last":0.38100000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:48:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":0.66900000000000004,"15min":2.5379999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.219},"max":{"1min":8.4320000000000004,"5min":8.4320000000000004,"15min":95.305000000000007},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.143,"5min":0.624,"15min":3.1499999999999999},"min":{"1min":0.27100000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.8559999999999999,"5min":4.8559999999999999,"15min":154.483},"last":0.34200000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":3.839,"5min":1.2310000000000001,"15min":2.2269999999999999},"min":{"1min":0.27100000000000002,"5min":0.245,"15min":0.245},"max":{"1min":24.539999999999999,"5min":24.539999999999999,"15min":154.101},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.222,"5min":0.61899999999999999,"15min":2.0339999999999998},"min":{"1min":0.29999999999999999,"5min":0.29499999999999998,"15min":0.27700000000000002},"max":{"1min":4.9139999999999997,"5min":4.9139999999999997,"15min":154.56999999999999},"last":0.48999999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:47:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.339,"15min":2.2679999999999998},"min":{"1min":0.31900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":18.158000000000001,"5min":59.561,"15min":154.22999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.296,"15min":2.323},"min":{"1min":0.32800000000000001,"5min":0.31,"15min":0.307},"max":{"1min":6.5789999999999997,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.39500000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:48:15 2023","interfaces":[{"interface":"back","average":{"1min":3.8490000000000002,"5min":1.304,"15min":2.3330000000000002},"min":{"1min":0.34000000000000002,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":24.321000000000002,"5min":24.321000000000002,"15min":154.12200000000001},"last":0.437},{"interface":"front","average":{"1min":2.7280000000000002,"5min":1.079,"15min":2.2349999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.2149999999999999,"5min":9.2149999999999999,"15min":154.16999999999999},"last":0.46100000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":1.1439999999999999,"15min":2.3639999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":15.009,"5min":15.009,"15min":154.20699999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.7689999999999999,"5min":0.95599999999999996,"15min":2.3799999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.34300000000000003},"max":{"1min":6.6180000000000003,"5min":6.6180000000000003,"15min":154.51599999999999},"last":0.42499999999999999}]}]},{"osd":0,"up_from":89,"seq":382252089795,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1620740,"kb_used_data":667260,"kb_used_omap":8,"kb_used_meta":953463,"kb_avail":92124412,"statfs":{"total":95995035648,"available":94335397888,"internally_reserved":0,"allocated":683274240,"data_stored":681229695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":976347074},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":4.4480000000000004,"5min":1.216,"15min":3.7320000000000002},"min":{"1min":0.219,"5min":0.18099999999999999,"15min":0.17799999999999999},"max":{"1min":90.185000000000002,"5min":90.185000000000002,"15min":216.821},"last":1.925},{"interface":"front","average":{"1min":5.6059999999999999,"5min":1.45,"15min":4.3899999999999997},"min":{"1min":0.22,"5min":0.20599999999999999,"15min":0.19600000000000001},"max":{"1min":90.102999999999994,"5min":90.102999999999994,"15min":217.02699999999999},"last":0.39800000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:47:49 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.50800000000000001,"15min":1.7210000000000001},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":1.6919999999999999,"5min":4.2430000000000003,"15min":19.788},"last":1.5009999999999999},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.60899999999999999,"15min":1.714},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.215},"max":{"1min":2.952,"5min":3.8719999999999999,"15min":25.844999999999999},"last":0.35099999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.54900000000000004,"15min":2.516},"min":{"1min":0.245,"5min":0.22,"15min":0.22},"max":{"1min":3.3849999999999998,"5min":6.5670000000000002,"15min":222.16499999999999},"last":1.5429999999999999},{"interface":"front","average":{"1min":1.502,"5min":0.80900000000000005,"15min":2.5710000000000002},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.0650000000000004,"5min":4.9199999999999999,"15min":222.227},"last":1.603}]},{"osd":4,"last update":"Sun Oct 1 09:48:04 2023","interfaces":[{"interface":"back","average":{"1min":7.1349999999999998,"5min":1.8819999999999999,"15min":2.1699999999999999},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":138.84100000000001,"5min":138.84100000000001,"15min":138.84100000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":6.8760000000000003,"5min":1.8109999999999999,"15min":2.246},"min":{"1min":0.307,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":135.94800000000001,"5min":135.94800000000001,"15min":135.94800000000001},"last":3.8969999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:47:49 2023","interfaces":[{"interface":"back","average":{"1min":6.8460000000000001,"5min":1.869,"15min":2.0089999999999999},"min":{"1min":0.33700000000000002,"5min":0.312,"15min":0.312},"max":{"1min":156.69999999999999,"5min":156.69999999999999,"15min":156.69999999999999},"last":3.8559999999999999},{"interface":"front","average":{"1min":6.7249999999999996,"5min":1.8500000000000001,"15min":1.921},"min":{"1min":0.38400000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":156.773,"5min":156.773,"15min":156.773},"last":2.073}]},{"osd":6,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.64400000000000002,"15min":1.51},"min":{"1min":0.35699999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.589,"5min":4.3129999999999997,"15min":17.535},"last":1.7450000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.66700000000000004,"15min":1.571},"min":{"1min":0.35899999999999999,"5min":0.32600000000000001,"15min":0.32400000000000001},"max":{"1min":2.5960000000000001,"5min":4.2050000000000001,"15min":15.651999999999999},"last":0.42199999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:48:04 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":0.79300000000000004,"15min":1.8089999999999999},"min":{"1min":0.40100000000000002,"5min":0.374,"15min":0.374},"max":{"1min":4.1280000000000001,"5min":4.3360000000000003,"15min":48.895000000000003},"last":1.6259999999999999},{"interface":"front","average":{"1min":1.736,"5min":0.97399999999999998,"15min":1.9350000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.8490000000000002,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.59699999999999998}]}]},{"osd":2,"up_from":0,"seq":425201762734,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599207,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":878260,"kb_used_data":19884,"kb_used_omap":3,"kb_used_meta":858364,"kb_avail":92866892,"statfs":{"total":95995035648,"available":95095697408,"internally_reserved":0,"allocated":20361216,"data_stored":18199483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":878965479},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":5.0350000000000001,"5min":1.264,"15min":3.4929999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17899999999999999},"max":{"1min":74.707999999999998,"5min":74.707999999999998,"15min":137.34399999999999},"last":41.162999999999997},{"interface":"front","average":{"1min":5.5439999999999996,"5min":1.4099999999999999,"15min":3.577},"min":{"1min":0.217,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":74.555000000000007,"5min":74.555000000000007,"15min":137.73400000000001},"last":37.274000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":0.52400000000000002,"15min":4.25},"min":{"1min":0.23699999999999999,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":5.9009999999999998,"5min":5.9009999999999998,"15min":162.22999999999999},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.47399999999999998,"15min":4.2800000000000002},"min":{"1min":0.23300000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.008,"5min":4.008,"15min":162.70699999999999},"last":0.25900000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.501,"15min":1.508},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.22800000000000001},"max":{"1min":3.5059999999999998,"5min":3.5059999999999998,"15min":14.254},"last":0.23200000000000001},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.60299999999999998,"15min":1.3029999999999999},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":3.9809999999999999,"5min":4.5049999999999999,"15min":15.523},"last":0.317}]},{"osd":4,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":1.591,"5min":0.76700000000000002,"15min":1.639},"min":{"1min":0.312,"5min":0.312,"15min":0.26200000000000001},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":77.793000000000006},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.5219999999999998,"5min":0.91300000000000003,"15min":1.8680000000000001},"min":{"1min":0.35499999999999998,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":9.7370000000000001,"5min":9.7370000000000001,"15min":76.646000000000001},"last":0.41799999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:47:37 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.67200000000000004,"15min":1.7150000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27500000000000002},"max":{"1min":3.1459999999999999,"5min":4.3099999999999996,"15min":28.117999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.502,"15min":1.8140000000000001},"min":{"1min":0.34300000000000003,"5min":0.33600000000000002,"15min":0.27700000000000002},"max":{"1min":0.56899999999999995,"5min":2.512,"15min":28.34},"last":0.307}]},{"osd":6,"last update":"Sun Oct 1 09:47:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54500000000000004,"15min":1.758},"min":{"1min":0.45500000000000002,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":1.0549999999999999,"5min":2.9100000000000001,"15min":73.887},"last":1.1870000000000001},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.65000000000000002,"15min":1.869},"min":{"1min":0.38100000000000001,"5min":0.35499999999999998,"15min":0.33600000000000002},"max":{"1min":3.1099999999999999,"5min":3.1099999999999999,"15min":74.016999999999996},"last":1.123}]},{"osd":7,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":2.194,"5min":1.1579999999999999,"15min":1.8180000000000001},"min":{"1min":0.47499999999999998,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":10.574,"5min":10.574,"15min":13.085000000000001},"last":3.0270000000000001},{"interface":"front","average":{"1min":2.3330000000000002,"5min":1.1279999999999999,"15min":1.5860000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.311},"max":{"1min":10.609,"5min":10.609,"15min":13.191000000000001},"last":3.0139999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019458,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1232072,"kb_used_data":333696,"kb_used_omap":0,"kb_used_meta":898368,"kb_avail":92513080,"statfs":{"total":95995035648,"available":94733393920,"internally_reserved":0,"allocated":341704704,"data_stored":340763562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919928832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:02 2023","interfaces":[{"interface":"back","average":{"1min":8.1419999999999995,"5min":2.4369999999999998,"15min":3.8559999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":95.548000000000002,"5min":95.548000000000002,"15min":155.315},"last":0.42699999999999999},{"interface":"front","average":{"1min":8.2460000000000004,"5min":2.4420000000000002,"15min":4.125},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":95.253,"5min":95.253,"15min":155.334},"last":0.33000000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:48:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2709999999999999,"5min":0.67900000000000005,"15min":5.0679999999999996},"min":{"1min":0.24299999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":7.5979999999999999,"5min":7.5979999999999999,"15min":190.61199999999999},"last":0.435},{"interface":"front","average":{"1min":1.089,"5min":0.67500000000000004,"15min":5.7759999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":6.3490000000000002,"5min":6.3490000000000002,"15min":253.273},"last":0.33900000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":0.80200000000000005,"15min":2.0510000000000002},"min":{"1min":0.32200000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.6390000000000002,"5min":7.6390000000000002,"15min":170.45400000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.284,"5min":0.754,"15min":1.6779999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.6619999999999999,"5min":7.6619999999999999,"15min":36.515999999999998},"last":0.38100000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:48:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":0.80500000000000005,"15min":1.9450000000000001},"min":{"1min":0.28599999999999998,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":11.667,"5min":11.667,"15min":140.881},"last":0.313},{"interface":"front","average":{"1min":1.141,"5min":0.748,"15min":2.0089999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.3769999999999998,"5min":6.3769999999999998,"15min":140.911},"last":0.40100000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.0369999999999999,"15min":1.5740000000000001},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.26600000000000001},"max":{"1min":21.364999999999998,"5min":21.364999999999998,"15min":110.003},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.9510000000000001,"5min":0.93200000000000005,"15min":1.466},"min":{"1min":0.313,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":21.559999999999999,"5min":21.559999999999999,"15min":108.56399999999999},"last":0.45000000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":0.84499999999999997,"15min":0.92000000000000004},"min":{"1min":0.35199999999999998,"5min":0.318,"15min":0.27700000000000002},"max":{"1min":11.635999999999999,"5min":11.635999999999999,"15min":11.635999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.9590000000000001,"5min":0.88300000000000001,"15min":1.079},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":7.7149999999999999,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":0.49099999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":0.88600000000000001,"15min":1.1950000000000001},"min":{"1min":0.40600000000000003,"5min":0.313,"15min":0.313},"max":{"1min":11.694000000000001,"5min":11.694000000000001,"15min":38.034999999999997},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.2130000000000001,"5min":1.0269999999999999,"15min":1.4750000000000001},"min":{"1min":0.35799999999999998,"5min":0.35599999999999998,"15min":0.308},"max":{"1min":12.228,"5min":12.228,"15min":37.723999999999997},"last":0.51100000000000001}]}]},{"osd":5,"up_from":379,"seq":1627792605365,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":373680,"kb_used_data":5032,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":93371472,"statfs":{"total":95995035648,"available":95612387328,"internally_reserved":0,"allocated":5152768,"data_stored":4223782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":4.9749999999999996,"5min":1.6719999999999999,"15min":2.3140000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.21099999999999999},"max":{"1min":99.609999999999999,"5min":99.609999999999999,"15min":99.609999999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":5.117,"5min":1.7130000000000001,"15min":2.2959999999999998},"min":{"1min":0.215,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":98.995999999999995,"5min":98.995999999999995,"15min":98.995999999999995},"last":0.39700000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":5.4790000000000001},"min":{"1min":0.28399999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.157,"5min":4.3289999999999997,"15min":295.71300000000002},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.56699999999999995,"15min":5.5129999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":3.665,"5min":3.9580000000000002,"15min":299.59399999999999},"last":0.33400000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.60399999999999998,"15min":1.4179999999999999},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.9129999999999998,"5min":4.2670000000000003,"15min":14.199999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.70899999999999996,"15min":1.357},"min":{"1min":0.31900000000000001,"5min":0.29599999999999999,"15min":0.27100000000000002},"max":{"1min":4.1909999999999998,"5min":4.3719999999999999,"15min":14.430999999999999},"last":0.433}]},{"osd":3,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":0.70599999999999996,"15min":1.3759999999999999},"min":{"1min":0.30599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.141,"5min":7.141,"15min":14.004},"last":0.442},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.66600000000000004,"15min":1.5029999999999999},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":3.1139999999999999,"5min":4.2910000000000004,"15min":13.991},"last":0.377}]},{"osd":4,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.66400000000000003,"15min":1.375},"min":{"1min":0.27900000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.0659999999999998,"5min":4.1929999999999996,"15min":122.971},"last":0.314},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.66500000000000004,"15min":1.472},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2469999999999999,"5min":4.3869999999999996,"15min":122.89100000000001},"last":0.35899999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.63800000000000001,"15min":0.94599999999999995},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28299999999999997},"max":{"1min":3.5750000000000002,"5min":4.3300000000000001,"15min":12.545},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.097,"5min":0.76300000000000001,"15min":1.119},"min":{"1min":0.31,"5min":0.31,"15min":0.30499999999999999},"max":{"1min":4.484,"5min":4.484,"15min":14.193},"last":0.42699999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.74399999999999999,"15min":1.1120000000000001},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.2199999999999998,"5min":4.3559999999999999,"15min":10.813000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.75600000000000001,"15min":1.0509999999999999},"min":{"1min":0.40799999999999997,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":3.609,"5min":4.2000000000000002,"15min":12.500999999999999},"last":0.47299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":672071680,"data_stored":672071680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":7864320,"data_stored":7864320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":336527360,"data_stored":336527360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8785920,"data_stored":8781848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:48:23.853 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:23.857 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:48:23.925 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:24.753 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:24 smithi138 ceph-mon[192513]: from='client.47549 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:48:24.754 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:24 smithi138 ceph-mon[185886]: from='client.47549 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:48:24.773 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T09:48:24.775 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:24.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:24.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:24.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:24 smithi190 ceph-mon[160435]: from='client.47549 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:48:24.879 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:24.907 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:25.007 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:25.012 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:48:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:48:24] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.33.4" 2023-10-01T09:48:25.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:25.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:25.036 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:25.114 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:25.140 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:25.182 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:25.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:25.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:25.256 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:25.287 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:25.313 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:25.355 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:25.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:25.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:25.461 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:25.474 INFO:teuthology.orchestra.run.smithi138.stderr:pool 'unique_pool_5' removed 2023-10-01T09:48:25.523 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:25.594 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:25.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:25.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:25.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:25 smithi138 ceph-mon[192513]: pgmap v1692: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 5273/84 objects degraded (6277.381%); 992 KiB/s, 22 objects/s recovering 2023-10-01T09:48:25.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:48:25.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:25 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3547905986' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:48:25.664 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:25 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:48:25.666 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:25 smithi138 ceph-mon[185886]: pgmap v1692: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 5273/84 objects degraded (6277.381%); 992 KiB/s, 22 objects/s recovering 2023-10-01T09:48:25.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:48:25.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:25 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3547905986' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:48:25.667 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:25 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:48:25.701 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:25.794 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:25 smithi190 ceph-mon[160435]: pgmap v1692: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 5273/84 objects degraded (6277.381%); 992 KiB/s, 22 objects/s recovering 2023-10-01T09:48:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:48:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:25 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3547905986' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:48:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:25 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T09:48:25.855 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:48:26.342 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:26.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:26.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:26.445 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:26.477 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:26.522 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:26.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:26.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:26.627 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:26.651 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:26 smithi138 ceph-mon[185886]: failed to encode map e561 with expected crc 2023-10-01T09:48:26.651 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:26 smithi138 ceph-mon[185886]: failed to encode map e561 with expected crc 2023-10-01T09:48:26.651 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:26 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:48:26.651 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:26 smithi138 ceph-mon[185886]: osdmap e561: 8 total, 8 up, 7 in 2023-10-01T09:48:26.651 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:26 smithi138 ceph-mon[185886]: failed to encode map e561 with expected crc 2023-10-01T09:48:26.652 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:26 smithi138 ceph-mon[185886]: failed to encode map e561 with expected crc 2023-10-01T09:48:26.652 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:26 smithi138 ceph-mon[192513]: failed to encode map e561 with expected crc 2023-10-01T09:48:26.652 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:26 smithi138 ceph-mon[192513]: failed to encode map e561 with expected crc 2023-10-01T09:48:26.652 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:26 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:48:26.652 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:26 smithi138 ceph-mon[192513]: osdmap e561: 8 total, 8 up, 7 in 2023-10-01T09:48:26.652 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:26 smithi138 ceph-mon[192513]: failed to encode map e561 with expected crc 2023-10-01T09:48:26.652 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:26 smithi138 ceph-mon[192513]: failed to encode map e561 with expected crc 2023-10-01T09:48:26.654 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:26.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:26 smithi190 ceph-mon[160435]: failed to encode map e561 with expected crc 2023-10-01T09:48:26.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:26 smithi190 ceph-mon[160435]: failed to encode map e561 with expected crc 2023-10-01T09:48:26.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:26 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T09:48:26.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:26 smithi190 ceph-mon[160435]: osdmap e561: 8 total, 8 up, 7 in 2023-10-01T09:48:26.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:26 smithi190 ceph-mon[160435]: failed to encode map e561 with expected crc 2023-10-01T09:48:26.808 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:26 smithi190 ceph-mon[160435]: failed to encode map e561 with expected crc 2023-10-01T09:48:26.827 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:48:27.725 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:27.727 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:27 smithi138 ceph-mon[192513]: pgmap v1694: 73 pgs: 1 active+recovery_wait+degraded, 72 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 10/84 objects degraded (11.905%) 2023-10-01T09:48:27.728 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:27 smithi138 ceph-mon[185886]: pgmap v1694: 73 pgs: 1 active+recovery_wait+degraded, 72 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 10/84 objects degraded (11.905%) 2023-10-01T09:48:27.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:27.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:27.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:27 smithi190 ceph-mon[160435]: pgmap v1694: 73 pgs: 1 active+recovery_wait+degraded, 72 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 10/84 objects degraded (11.905%) 2023-10-01T09:48:27.829 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:27.861 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:27.892 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-01T09:48:27.892 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-01T09:48:27.893 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-01T09:48:27.893 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-01T09:48:27.893 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-01T09:48:27.894 DEBUG:teuthology.orchestra.run.smithi138:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-01T09:48:27.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T09:48:27.953 INFO:teuthology.orchestra.run.smithi138.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-01T09:48:27.954 DEBUG:teuthology.orchestra.run.smithi138:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-01T09:48:27.957 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:27.966 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:27.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:27.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:27.993 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-01T09:48:27.993 DEBUG:teuthology.orchestra.run.smithi138:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-01T09:48:28.050 INFO:tasks.workunit:timeout=3h 2023-10-01T09:48:28.050 INFO:tasks.workunit:cleanup=True 2023-10-01T09:48:28.050 DEBUG:teuthology.orchestra.run.smithi138:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-01T09:48:28.066 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:48:28.066 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:48:28.070 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:28.131 INFO:tasks.workunit.client.0.smithi138.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-01T09:48:28.140 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:28.241 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:28.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:28.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:28.345 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:28.377 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:28.426 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:28.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:28.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:28.482 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:48:28.508 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:28.531 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:28.569 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:28.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/42664090' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:48:28.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:28 smithi138 ceph-mon[192513]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:48:28.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/42664090' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:48:28.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:28 smithi138 ceph-mon[185886]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:48:28.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/42664090' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:48:28.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:28 smithi190 ceph-mon[160435]: Health check update: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded (PG_DEGRADED) 2023-10-01T09:48:29.096 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:29.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:29.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:29.200 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:29.233 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:29.281 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:29.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:29.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:29.389 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:48:29.418 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:29.464 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:48:29.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:29.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:48:29.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:29 smithi190 ceph-mon[160435]: pgmap v1695: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:29.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:29 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:48:29.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:29 smithi190 ceph-mon[160435]: failed to encode map e562 with expected crc 2023-10-01T09:48:29.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:29 smithi190 ceph-mon[160435]: failed to encode map e562 with expected crc 2023-10-01T09:48:29.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:29 smithi190 ceph-mon[160435]: failed to encode map e562 with expected crc 2023-10-01T09:48:29.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:29 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/42664090' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:48:29.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:29 smithi190 ceph-mon[160435]: osdmap e562: 8 total, 8 up, 7 in 2023-10-01T09:48:29.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:29 smithi190 ceph-mon[160435]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded) 2023-10-01T09:48:29.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:29 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:48:29.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:29 smithi190 ceph-mon[160435]: failed to encode map e562 with expected crc 2023-10-01T09:48:29.571 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:29.597 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:29.638 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:29.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:29.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:29.717 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T09:48:29.741 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[192513]: pgmap v1695: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:29.741 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:48:29.742 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[192513]: failed to encode map e562 with expected crc 2023-10-01T09:48:29.742 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[192513]: failed to encode map e562 with expected crc 2023-10-01T09:48:29.742 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[192513]: failed to encode map e562 with expected crc 2023-10-01T09:48:29.742 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/42664090' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:48:29.742 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[192513]: osdmap e562: 8 total, 8 up, 7 in 2023-10-01T09:48:29.742 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[192513]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded) 2023-10-01T09:48:29.742 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:48:29.742 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[192513]: failed to encode map e562 with expected crc 2023-10-01T09:48:29.742 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[185886]: pgmap v1695: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:29.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:48:29.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[185886]: failed to encode map e562 with expected crc 2023-10-01T09:48:29.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[185886]: failed to encode map e562 with expected crc 2023-10-01T09:48:29.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[185886]: failed to encode map e562 with expected crc 2023-10-01T09:48:29.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/42664090' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:48:29.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[185886]: osdmap e562: 8 total, 8 up, 7 in 2023-10-01T09:48:29.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[185886]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 1 pg degraded) 2023-10-01T09:48:29.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:48:29.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:29 smithi138 ceph-mon[185886]: failed to encode map e562 with expected crc 2023-10-01T09:48:29.743 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:29.769 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:29.957 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:29.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:29.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:30.060 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:30.089 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:30.133 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:30.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:30.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:30.238 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:30.265 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:30.306 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:30.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:30.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:30.412 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:30.444 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:30.610 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:48:30.629 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:48:30.642 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:30.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:30.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:30.648 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:48:30.667 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:48:30.685 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:48:30.717 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:48:30.747 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:30.775 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:30.823 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:30.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:30.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:30.899 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:48:30.899 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:48:30.900 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:48:30.900 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:48:30.900 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:48:30.900 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:48:30.900 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:48:30.900 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:48:30.900 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:48:30.900 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:48:30.927 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:30.955 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:30.999 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:31.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:31.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:31.106 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:31.132 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:31.173 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:31.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:31.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:31.280 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:31.307 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:31.402 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:31.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:31.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:31.506 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:31.534 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:31 smithi138 ceph-mon[192513]: pgmap v1697: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1077960513' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:31.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:31 smithi138 ceph-mon[185886]: pgmap v1697: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:31.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1077960513' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:31.577 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:31.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:31.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:31.683 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:31.711 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:31.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:31 smithi190 ceph-mon[160435]: pgmap v1697: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1077960513' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:31.980 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T09:48:32.023 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:32.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:32.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:32.127 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:32.155 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:32.156 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:32.262 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:32.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:32.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:32.371 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:32.403 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:32.450 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:32.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:32.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:32.558 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:32.587 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:32.677 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:48:32.679 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:48:32.679 DEBUG:teuthology.orchestra.run.smithi138:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-01T09:48:32.689 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:32.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:32.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:32.707 INFO:tasks.workunit.client.0.smithi138.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-01T09:48:32.710 INFO:tasks.workunit.client.0.smithi138.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-01T09:48:32.710 INFO:tasks.workunit.client.0.smithi138.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-01T09:48:32.793 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:32.821 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:32.863 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:32.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:32.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:32.929 INFO:tasks.workunit.client.0.smithi138.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-01T09:48:32.967 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:32.999 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:33.053 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:33.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:33.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:33.072 INFO:tasks.workunit.client.0.smithi138.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-01T09:48:33.159 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:33.165 INFO:tasks.workunit.client.0.smithi138.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-01T09:48:33.166 INFO:tasks.workunit.client.0.smithi138.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-01T09:48:33.167 INFO:tasks.workunit.client.0.smithi138.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-01T09:48:33.187 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:33.232 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:33.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:33.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:33.266 INFO:tasks.workunit.client.0.smithi138.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-01T09:48:33.340 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:33.367 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:33.411 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:33.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:33.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:33.415 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T09:48:33.416 DEBUG:teuthology.orchestra.run.smithi138:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-01T09:48:33.516 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:33.533 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-01T09:48:33.535 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-01T09:48:33.535 DEBUG:teuthology.orchestra.run.smithi138:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-01T09:48:33.544 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:33.588 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:33.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:33.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:33.694 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:48:33.721 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:33.762 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:48:33.766 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:33 smithi190 ceph-mon[160435]: pgmap v1698: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:33.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:33.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:48:33.794 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:33 smithi138 ceph-mon[185886]: pgmap v1698: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:33.794 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:33 smithi138 ceph-mon[192513]: pgmap v1698: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:33.820 INFO:tasks.workunit.client.0.smithi138.stdout:Running main() from gmock_main.cc 2023-10-01T09:48:33.820 INFO:tasks.workunit.client.0.smithi138.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-01T09:48:33.820 INFO:tasks.workunit.client.0.smithi138.stdout:[----------] Global test environment set-up. 2023-10-01T09:48:33.820 INFO:tasks.workunit.client.0.smithi138.stdout:[----------] 70 tests from TestClsRbd 2023-10-01T09:48:33.869 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:33.894 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:33.936 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:33.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:33.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:34.042 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:48:34.068 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:34.109 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:48:34.113 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:48:34.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:34.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:48:34.215 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:34.242 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:34.283 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:34.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:34.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:34.389 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:34.424 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:34.485 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:34 smithi190 ceph-mon[160435]: from='client.37530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:48:34.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:34 smithi138 ceph-mon[192513]: from='client.37530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:48:34.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:34 smithi138 ceph-mon[185886]: from='client.37530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:48:35.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:48:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:48:34] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.33.4" 2023-10-01T09:48:35.487 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-01T09:48:35.605 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.get_all_features (119 ms) 2023-10-01T09:48:35.606 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.copyup 2023-10-01T09:48:35.797 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.copyup (191 ms) 2023-10-01T09:48:35.798 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-01T09:48:35.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:35 smithi190 ceph-mon[160435]: pgmap v1699: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:48:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:35 smithi190 ceph-mon[160435]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:35 smithi190 ceph-mon[160435]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:35 smithi190 ceph-mon[160435]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:35 smithi190 ceph-mon[160435]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:35 smithi190 ceph-mon[160435]: osdmap e563: 8 total, 8 up, 7 in 2023-10-01T09:48:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:35 smithi190 ceph-mon[160435]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:35 smithi190 ceph-mon[160435]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:35 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3289143473' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi138-722318-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:48:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:35 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi138-722318-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:48:35.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:35 smithi190 ceph-mon[160435]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.814 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.sparse_copyup (17 ms) 2023-10-01T09:48:35.814 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-01T09:48:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[185886]: pgmap v1699: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:48:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[185886]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[185886]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[185886]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[185886]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[185886]: osdmap e563: 8 total, 8 up, 7 in 2023-10-01T09:48:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[185886]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[185886]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3289143473' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi138-722318-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:48:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi138-722318-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:48:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[185886]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[192513]: pgmap v1699: 73 pgs: 73 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 1 objects/s recovering 2023-10-01T09:48:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[192513]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[192513]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[192513]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[192513]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[192513]: osdmap e563: 8 total, 8 up, 7 in 2023-10-01T09:48:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[192513]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[192513]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3289143473' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi138-722318-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:48:35.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi138-722318-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-01T09:48:35.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:35 smithi138 ceph-mon[192513]: failed to encode map e563 with expected crc 2023-10-01T09:48:35.822 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.get_and_set_id (9 ms) 2023-10-01T09:48:35.822 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-01T09:48:35.834 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.add_remove_child (11 ms) 2023-10-01T09:48:35.834 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-01T09:48:35.868 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.directory_methods (34 ms) 2023-10-01T09:48:35.868 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.create 2023-10-01T09:48:35.882 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.create (14 ms) 2023-10-01T09:48:35.882 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.get_features 2023-10-01T09:48:35.884 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2023-10-01T09:48:35.884 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-01T09:48:35.886 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-10-01T09:48:35.886 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-01T09:48:35.887 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-10-01T09:48:35.887 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-01T09:48:35.889 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-10-01T09:48:35.889 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-01T09:48:35.890 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2023-10-01T09:48:35.890 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-01T09:48:35.894 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2023-10-01T09:48:35.894 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.get_size 2023-10-01T09:48:35.898 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2023-10-01T09:48:35.898 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.set_size 2023-10-01T09:48:35.903 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.set_size (6 ms) 2023-10-01T09:48:35.903 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-01T09:48:35.922 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.protection_status (19 ms) 2023-10-01T09:48:35.922 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-01T09:48:35.931 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.snapshot_limits (8 ms) 2023-10-01T09:48:35.931 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-01T09:48:35.971 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.parents_v1 (41 ms) 2023-10-01T09:48:35.972 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-01T09:48:36.001 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.parents_v2 (29 ms) 2023-10-01T09:48:36.001 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-01T09:48:36.030 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.snapshots (29 ms) 2023-10-01T09:48:36.030 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-01T09:48:36.035 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2023-10-01T09:48:36.035 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-01T09:48:36.136 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.stripingv2 (101 ms) 2023-10-01T09:48:36.136 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-01T09:48:36.138 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.object_map_save (3 ms) 2023-10-01T09:48:36.138 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-01T09:48:36.146 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.object_map_resize (7 ms) 2023-10-01T09:48:36.146 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-01T09:48:36.151 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.object_map_update (5 ms) 2023-10-01T09:48:36.151 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-01T09:48:36.152 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2023-10-01T09:48:36.152 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-01T09:48:36.161 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.object_map_snap_add (7 ms) 2023-10-01T09:48:36.161 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-01T09:48:36.166 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.object_map_snap_remove (8 ms) 2023-10-01T09:48:36.167 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.flags 2023-10-01T09:48:36.171 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.flags (4 ms) 2023-10-01T09:48:36.171 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.metadata 2023-10-01T09:48:36.196 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.metadata (26 ms) 2023-10-01T09:48:36.197 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.set_features 2023-10-01T09:48:36.203 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2023-10-01T09:48:36.203 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.mirror 2023-10-01T09:48:36.239 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.mirror (36 ms) 2023-10-01T09:48:36.239 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-01T09:48:36.253 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.mirror_image (15 ms) 2023-10-01T09:48:36.254 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-01T09:48:36.326 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1698,"stamp":"2023-10-01T09:48:32.395981+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6259,"num_read_kb":12659,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":20349967,"num_keys_recovered":100,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":222,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6165524,"kb_used_data":1003932,"kb_used_omap":39,"kb_used_meta":5161496,"kb_avail":650050540,"statfs":{"total":671965249536,"available":665651752960,"internally_reserved":0,"allocated":1028026368,"data_stored":1017917161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5285372527},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":37,"apply_latency_ms":37,"commit_latency_ns":37000000,"apply_latency_ns":37000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-5273,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-166440,"num_write_kb":-5326017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-11876,"num_bytes_recovered":-774307840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001876"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":338,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.474781+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:48:25.474781+0000","last_peered":"2023-10-01T09:48:25.474781+0000","last_clean":"2023-10-01T09:48:25.474781+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:48:25.474781+0000","last_undegraded":"2023-10-01T09:48:25.474781+0000","last_fullsized":"2023-10-01T09:48:25.474781+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":183,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.726736+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:48:25.726736+0000","last_peered":"2023-10-01T09:48:25.726736+0000","last_clean":"2023-10-01T09:48:25.726736+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:48:25.726736+0000","last_undegraded":"2023-10-01T09:48:25.726736+0000","last_fullsized":"2023-10-01T09:48:25.726736+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":564,"reported_epoch":562,"state":"active+clean","last_fresh":"2023-10-01T09:48:28.622152+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:48:28.622152+0000","last_peered":"2023-10-01T09:48:28.622152+0000","last_clean":"2023-10-01T09:48:28.622152+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:48:28.622152+0000","last_undegraded":"2023-10-01T09:48:28.622152+0000","last_fullsized":"2023-10-01T09:48:28.622152+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":622,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.472112+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:48:25.472112+0000","last_peered":"2023-10-01T09:48:25.472112+0000","last_clean":"2023-10-01T09:48:25.472112+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:48:25.472112+0000","last_undegraded":"2023-10-01T09:48:25.472112+0000","last_fullsized":"2023-10-01T09:48:25.472112+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":503,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.467190+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:48:25.467190+0000","last_peered":"2023-10-01T09:48:25.467190+0000","last_clean":"2023-10-01T09:48:25.467190+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:48:25.467190+0000","last_undegraded":"2023-10-01T09:48:25.467190+0000","last_fullsized":"2023-10-01T09:48:25.467190+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":243,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.626197+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:48:25.626197+0000","last_peered":"2023-10-01T09:48:25.626197+0000","last_clean":"2023-10-01T09:48:25.626197+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:48:25.626197+0000","last_undegraded":"2023-10-01T09:48:25.626197+0000","last_fullsized":"2023-10-01T09:48:25.626197+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":619,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.626288+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:48:25.626288+0000","last_peered":"2023-10-01T09:48:25.626288+0000","last_clean":"2023-10-01T09:48:25.626288+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:48:25.626288+0000","last_undegraded":"2023-10-01T09:48:25.626288+0000","last_fullsized":"2023-10-01T09:48:25.626288+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":549,"reported_epoch":560,"state":"active+clean","last_fresh":"2023-10-01T09:48:19.934703+0000","last_change":"2023-10-01T09:40:25.365269+0000","last_active":"2023-10-01T09:48:19.934703+0000","last_peered":"2023-10-01T09:48:19.934703+0000","last_clean":"2023-10-01T09:48:19.934703+0000","last_became_active":"2023-10-01T09:40:25.364936+0000","last_became_peered":"2023-10-01T09:40:25.364936+0000","last_unstale":"2023-10-01T09:48:19.934703+0000","last_undegraded":"2023-10-01T09:48:19.934703+0000","last_fullsized":"2023-10-01T09:48:19.934703+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:03:06.259960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":274,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.619743+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:48:25.619743+0000","last_peered":"2023-10-01T09:48:25.619743+0000","last_clean":"2023-10-01T09:48:25.619743+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:48:25.619743+0000","last_undegraded":"2023-10-01T09:48:25.619743+0000","last_fullsized":"2023-10-01T09:48:25.619743+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":81,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.623087+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:48:25.623087+0000","last_peered":"2023-10-01T09:48:25.623087+0000","last_clean":"2023-10-01T09:48:25.623087+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:48:25.623087+0000","last_undegraded":"2023-10-01T09:48:25.623087+0000","last_fullsized":"2023-10-01T09:48:25.623087+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":87,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.625006+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:48:25.625006+0000","last_peered":"2023-10-01T09:48:25.625006+0000","last_clean":"2023-10-01T09:48:25.625006+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:48:25.625006+0000","last_undegraded":"2023-10-01T09:48:25.625006+0000","last_fullsized":"2023-10-01T09:48:25.625006+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":87,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.727673+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:48:25.727673+0000","last_peered":"2023-10-01T09:48:25.727673+0000","last_clean":"2023-10-01T09:48:25.727673+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:48:25.727673+0000","last_undegraded":"2023-10-01T09:48:25.727673+0000","last_fullsized":"2023-10-01T09:48:25.727673+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":463,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.476437+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:48:25.476437+0000","last_peered":"2023-10-01T09:48:25.476437+0000","last_clean":"2023-10-01T09:48:25.476437+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:48:25.476437+0000","last_undegraded":"2023-10-01T09:48:25.476437+0000","last_fullsized":"2023-10-01T09:48:25.476437+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":344,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.481574+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:48:25.481574+0000","last_peered":"2023-10-01T09:48:25.481574+0000","last_clean":"2023-10-01T09:48:25.481574+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:48:25.481574+0000","last_undegraded":"2023-10-01T09:48:25.481574+0000","last_fullsized":"2023-10-01T09:48:25.481574+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":680,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.622459+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:48:25.622459+0000","last_peered":"2023-10-01T09:48:25.622459+0000","last_clean":"2023-10-01T09:48:25.622459+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:48:25.622459+0000","last_undegraded":"2023-10-01T09:48:25.622459+0000","last_fullsized":"2023-10-01T09:48:25.622459+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":417,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.623010+0000","last_change":"2023-10-01T09:40:42.563807+0000","last_active":"2023-10-01T09:48:25.623010+0000","last_peered":"2023-10-01T09:48:25.623010+0000","last_clean":"2023-10-01T09:48:25.623010+0000","last_became_active":"2023-10-01T09:40:42.563449+0000","last_became_peered":"2023-10-01T09:40:42.563449+0000","last_unstale":"2023-10-01T09:48:25.623010+0000","last_undegraded":"2023-10-01T09:48:25.623010+0000","last_fullsized":"2023-10-01T09:48:25.623010+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":90,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.620512+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:48:25.620512+0000","last_peered":"2023-10-01T09:48:25.620512+0000","last_clean":"2023-10-01T09:48:25.620512+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:48:25.620512+0000","last_undegraded":"2023-10-01T09:48:25.620512+0000","last_fullsized":"2023-10-01T09:48:25.620512+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":76,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.619306+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:48:25.619306+0000","last_peered":"2023-10-01T09:48:25.619306+0000","last_clean":"2023-10-01T09:48:25.619306+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:48:25.619306+0000","last_undegraded":"2023-10-01T09:48:25.619306+0000","last_fullsized":"2023-10-01T09:48:25.619306+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":573,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.625395+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:48:25.625395+0000","last_peered":"2023-10-01T09:48:25.625395+0000","last_clean":"2023-10-01T09:48:25.625395+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:48:25.625395+0000","last_undegraded":"2023-10-01T09:48:25.625395+0000","last_fullsized":"2023-10-01T09:48:25.625395+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":586,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.472190+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:48:25.472190+0000","last_peered":"2023-10-01T09:48:25.472190+0000","last_clean":"2023-10-01T09:48:25.472190+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:48:25.472190+0000","last_undegraded":"2023-10-01T09:48:25.472190+0000","last_fullsized":"2023-10-01T09:48:25.472190+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":442,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.467044+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:48:25.467044+0000","last_peered":"2023-10-01T09:48:25.467044+0000","last_clean":"2023-10-01T09:48:25.467044+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:48:25.467044+0000","last_undegraded":"2023-10-01T09:48:25.467044+0000","last_fullsized":"2023-10-01T09:48:25.467044+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.620913+0000","last_change":"2023-10-01T09:41:22.756213+0000","last_active":"2023-10-01T09:48:25.620913+0000","last_peered":"2023-10-01T09:48:25.620913+0000","last_clean":"2023-10-01T09:48:25.620913+0000","last_became_active":"2023-10-01T09:41:22.755855+0000","last_became_peered":"2023-10-01T09:41:22.755855+0000","last_unstale":"2023-10-01T09:48:25.620913+0000","last_undegraded":"2023-10-01T09:48:25.620913+0000","last_fullsized":"2023-10-01T09:48:25.620913+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":85,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.620767+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:48:25.620767+0000","last_peered":"2023-10-01T09:48:25.620767+0000","last_clean":"2023-10-01T09:48:25.620767+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:48:25.620767+0000","last_undegraded":"2023-10-01T09:48:25.620767+0000","last_fullsized":"2023-10-01T09:48:25.620767+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":282,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.615880+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:48:25.615880+0000","last_peered":"2023-10-01T09:48:25.615880+0000","last_clean":"2023-10-01T09:48:25.615880+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:48:25.615880+0000","last_undegraded":"2023-10-01T09:48:25.615880+0000","last_fullsized":"2023-10-01T09:48:25.615880+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":556,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.620671+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:48:25.620671+0000","last_peered":"2023-10-01T09:48:25.620671+0000","last_clean":"2023-10-01T09:48:25.620671+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:48:25.620671+0000","last_undegraded":"2023-10-01T09:48:25.620671+0000","last_fullsized":"2023-10-01T09:48:25.620671+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":619,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.727247+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:48:25.727247+0000","last_peered":"2023-10-01T09:48:25.727247+0000","last_clean":"2023-10-01T09:48:25.727247+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:48:25.727247+0000","last_undegraded":"2023-10-01T09:48:25.727247+0000","last_fullsized":"2023-10-01T09:48:25.727247+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":642,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.625095+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:48:25.625095+0000","last_peered":"2023-10-01T09:48:25.625095+0000","last_clean":"2023-10-01T09:48:25.625095+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:48:25.625095+0000","last_undegraded":"2023-10-01T09:48:25.625095+0000","last_fullsized":"2023-10-01T09:48:25.625095+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":440,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.727979+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:48:25.727979+0000","last_peered":"2023-10-01T09:48:25.727979+0000","last_clean":"2023-10-01T09:48:25.727979+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:48:25.727979+0000","last_undegraded":"2023-10-01T09:48:25.727979+0000","last_fullsized":"2023-10-01T09:48:25.727979+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":282,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.615357+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:48:25.615357+0000","last_peered":"2023-10-01T09:48:25.615357+0000","last_clean":"2023-10-01T09:48:25.615357+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:48:25.615357+0000","last_undegraded":"2023-10-01T09:48:25.615357+0000","last_fullsized":"2023-10-01T09:48:25.615357+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":542,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.467907+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:48:25.467907+0000","last_peered":"2023-10-01T09:48:25.467907+0000","last_clean":"2023-10-01T09:48:25.467907+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:48:25.467907+0000","last_undegraded":"2023-10-01T09:48:25.467907+0000","last_fullsized":"2023-10-01T09:48:25.467907+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":142,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.481869+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:48:25.481869+0000","last_peered":"2023-10-01T09:48:25.481869+0000","last_clean":"2023-10-01T09:48:25.481869+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:48:25.481869+0000","last_undegraded":"2023-10-01T09:48:25.481869+0000","last_fullsized":"2023-10-01T09:48:25.481869+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":216,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.476565+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:48:25.476565+0000","last_peered":"2023-10-01T09:48:25.476565+0000","last_clean":"2023-10-01T09:48:25.476565+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:48:25.476565+0000","last_undegraded":"2023-10-01T09:48:25.476565+0000","last_fullsized":"2023-10-01T09:48:25.476565+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":460,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.474903+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:48:25.474903+0000","last_peered":"2023-10-01T09:48:25.474903+0000","last_clean":"2023-10-01T09:48:25.474903+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:48:25.474903+0000","last_undegraded":"2023-10-01T09:48:25.474903+0000","last_fullsized":"2023-10-01T09:48:25.474903+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":622,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.726598+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:48:25.726598+0000","last_peered":"2023-10-01T09:48:25.726598+0000","last_clean":"2023-10-01T09:48:25.726598+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:48:25.726598+0000","last_undegraded":"2023-10-01T09:48:25.726598+0000","last_fullsized":"2023-10-01T09:48:25.726598+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":456,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.480601+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:48:25.480601+0000","last_peered":"2023-10-01T09:48:25.480601+0000","last_clean":"2023-10-01T09:48:25.480601+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:48:25.480601+0000","last_undegraded":"2023-10-01T09:48:25.480601+0000","last_fullsized":"2023-10-01T09:48:25.480601+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":297,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.480588+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:48:25.480588+0000","last_peered":"2023-10-01T09:48:25.480588+0000","last_clean":"2023-10-01T09:48:25.480588+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:48:25.480588+0000","last_undegraded":"2023-10-01T09:48:25.480588+0000","last_fullsized":"2023-10-01T09:48:25.480588+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":427,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.727809+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:48:25.727809+0000","last_peered":"2023-10-01T09:48:25.727809+0000","last_clean":"2023-10-01T09:48:25.727809+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:48:25.727809+0000","last_undegraded":"2023-10-01T09:48:25.727809+0000","last_fullsized":"2023-10-01T09:48:25.727809+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":474,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.726847+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:48:25.726847+0000","last_peered":"2023-10-01T09:48:25.726847+0000","last_clean":"2023-10-01T09:48:25.726847+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:48:25.726847+0000","last_undegraded":"2023-10-01T09:48:25.726847+0000","last_fullsized":"2023-10-01T09:48:25.726847+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"77'211","reported_seq":955,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.622874+0000","last_change":"2023-10-01T09:40:25.358207+0000","last_active":"2023-10-01T09:48:25.622874+0000","last_peered":"2023-10-01T09:48:25.622874+0000","last_clean":"2023-10-01T09:48:25.622874+0000","last_became_active":"2023-10-01T09:40:25.357580+0000","last_became_peered":"2023-10-01T09:40:25.357580+0000","last_unstale":"2023-10-01T09:48:25.622874+0000","last_undegraded":"2023-10-01T09:48:25.622874+0000","last_fullsized":"2023-10-01T09:48:25.622874+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":211,"log_dups_size":0,"ondisk_log_size":211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1374,"num_read_kb":1856,"num_write":181,"num_write_kb":1420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":292,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.619475+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:48:25.619475+0000","last_peered":"2023-10-01T09:48:25.619475+0000","last_clean":"2023-10-01T09:48:25.619475+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:48:25.619475+0000","last_undegraded":"2023-10-01T09:48:25.619475+0000","last_fullsized":"2023-10-01T09:48:25.619475+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":87,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.727859+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:48:25.727859+0000","last_peered":"2023-10-01T09:48:25.727859+0000","last_clean":"2023-10-01T09:48:25.727859+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:48:25.727859+0000","last_undegraded":"2023-10-01T09:48:25.727859+0000","last_fullsized":"2023-10-01T09:48:25.727859+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"77'100","reported_seq":1227,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.615732+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:48:25.615732+0000","last_peered":"2023-10-01T09:48:25.615732+0000","last_clean":"2023-10-01T09:48:25.615732+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:48:25.615732+0000","last_undegraded":"2023-10-01T09:48:25.615732+0000","last_fullsized":"2023-10-01T09:48:25.615732+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1190,"num_read_kb":1085,"num_write":71,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":437,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.481730+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:48:25.481730+0000","last_peered":"2023-10-01T09:48:25.481730+0000","last_clean":"2023-10-01T09:48:25.481730+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:48:25.481730+0000","last_undegraded":"2023-10-01T09:48:25.481730+0000","last_fullsized":"2023-10-01T09:48:25.481730+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":265,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.615506+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:48:25.615506+0000","last_peered":"2023-10-01T09:48:25.615506+0000","last_clean":"2023-10-01T09:48:25.615506+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:48:25.615506+0000","last_undegraded":"2023-10-01T09:48:25.615506+0000","last_fullsized":"2023-10-01T09:48:25.615506+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"77'83","reported_seq":933,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.679793+0000","last_change":"2023-10-01T09:48:25.678391+0000","last_active":"2023-10-01T09:48:25.679793+0000","last_peered":"2023-10-01T09:48:25.679793+0000","last_clean":"2023-10-01T09:48:25.679793+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:48:25.679793+0000","last_undegraded":"2023-10-01T09:48:25.679793+0000","last_fullsized":"2023-10-01T09:48:25.679793+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":384,"num_read_kb":1794,"num_write":78,"num_write_kb":1321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":627,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.726469+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:48:25.726469+0000","last_peered":"2023-10-01T09:48:25.726469+0000","last_clean":"2023-10-01T09:48:25.726469+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:48:25.726469+0000","last_undegraded":"2023-10-01T09:48:25.726469+0000","last_fullsized":"2023-10-01T09:48:25.726469+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":87,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.726720+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:48:25.726720+0000","last_peered":"2023-10-01T09:48:25.726720+0000","last_clean":"2023-10-01T09:48:25.726720+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:48:25.726720+0000","last_undegraded":"2023-10-01T09:48:25.726720+0000","last_fullsized":"2023-10-01T09:48:25.726720+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"77'233","reported_seq":1678,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.620121+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:48:25.620121+0000","last_peered":"2023-10-01T09:48:25.620121+0000","last_clean":"2023-10-01T09:48:25.620121+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:48:25.620121+0000","last_undegraded":"2023-10-01T09:48:25.620121+0000","last_fullsized":"2023-10-01T09:48:25.620121+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":233,"log_dups_size":0,"ondisk_log_size":233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1415,"num_read_kb":1617,"num_write":197,"num_write_kb":297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":592,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.621096+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:48:25.621096+0000","last_peered":"2023-10-01T09:48:25.621096+0000","last_clean":"2023-10-01T09:48:25.621096+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:48:25.621096+0000","last_undegraded":"2023-10-01T09:48:25.621096+0000","last_fullsized":"2023-10-01T09:48:25.621096+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":370,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.615734+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:48:25.615734+0000","last_peered":"2023-10-01T09:48:25.615734+0000","last_clean":"2023-10-01T09:48:25.615734+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:48:25.615734+0000","last_undegraded":"2023-10-01T09:48:25.615734+0000","last_fullsized":"2023-10-01T09:48:25.615734+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":360,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.481831+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:48:25.481831+0000","last_peered":"2023-10-01T09:48:25.481831+0000","last_clean":"2023-10-01T09:48:25.481831+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:48:25.481831+0000","last_undegraded":"2023-10-01T09:48:25.481831+0000","last_fullsized":"2023-10-01T09:48:25.481831+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"77'58","reported_seq":769,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.621167+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:48:25.621167+0000","last_peered":"2023-10-01T09:48:25.621167+0000","last_clean":"2023-10-01T09:48:25.621167+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:48:25.621167+0000","last_undegraded":"2023-10-01T09:48:25.621167+0000","last_fullsized":"2023-10-01T09:48:25.621167+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":58,"log_dups_size":0,"ondisk_log_size":58,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":2164,"num_write":63,"num_write_kb":6180,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":604,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:27.103315+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:48:27.103315+0000","last_peered":"2023-10-01T09:48:27.103315+0000","last_clean":"2023-10-01T09:48:27.103315+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:48:27.103315+0000","last_undegraded":"2023-10-01T09:48:27.103315+0000","last_fullsized":"2023-10-01T09:48:27.103315+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":603,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.471929+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:48:25.471929+0000","last_peered":"2023-10-01T09:48:25.471929+0000","last_clean":"2023-10-01T09:48:25.471929+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:48:25.471929+0000","last_undegraded":"2023-10-01T09:48:25.471929+0000","last_fullsized":"2023-10-01T09:48:25.471929+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"77'113","reported_seq":1000,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.625178+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:48:25.625178+0000","last_peered":"2023-10-01T09:48:25.625178+0000","last_clean":"2023-10-01T09:48:25.625178+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:48:25.625178+0000","last_undegraded":"2023-10-01T09:48:25.625178+0000","last_fullsized":"2023-10-01T09:48:25.625178+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":113,"log_dups_size":0,"ondisk_log_size":113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":704,"num_read_kb":1000,"num_write":109,"num_write_kb":277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":619,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.615436+0000","last_change":"2023-10-01T09:40:25.364778+0000","last_active":"2023-10-01T09:48:25.615436+0000","last_peered":"2023-10-01T09:48:25.615436+0000","last_clean":"2023-10-01T09:48:25.615436+0000","last_became_active":"2023-10-01T09:40:25.364465+0000","last_became_peered":"2023-10-01T09:40:25.364465+0000","last_unstale":"2023-10-01T09:48:25.615436+0000","last_undegraded":"2023-10-01T09:48:25.615436+0000","last_fullsized":"2023-10-01T09:48:25.615436+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":472,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.620190+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:48:25.620190+0000","last_peered":"2023-10-01T09:48:25.620190+0000","last_clean":"2023-10-01T09:48:25.620190+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:48:25.620190+0000","last_undegraded":"2023-10-01T09:48:25.620190+0000","last_fullsized":"2023-10-01T09:48:25.620190+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":501,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.620995+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:48:25.620995+0000","last_peered":"2023-10-01T09:48:25.620995+0000","last_clean":"2023-10-01T09:48:25.620995+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:48:25.620995+0000","last_undegraded":"2023-10-01T09:48:25.620995+0000","last_fullsized":"2023-10-01T09:48:25.620995+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"77'150","reported_seq":1196,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.727641+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:48:25.727641+0000","last_peered":"2023-10-01T09:48:25.727641+0000","last_clean":"2023-10-01T09:48:25.727641+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:48:25.727641+0000","last_undegraded":"2023-10-01T09:48:25.727641+0000","last_fullsized":"2023-10-01T09:48:25.727641+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":700,"num_read_kb":551,"num_write":115,"num_write_kb":1085,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":90,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.621052+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:48:25.621052+0000","last_peered":"2023-10-01T09:48:25.621052+0000","last_clean":"2023-10-01T09:48:25.621052+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:48:25.621052+0000","last_undegraded":"2023-10-01T09:48:25.621052+0000","last_fullsized":"2023-10-01T09:48:25.621052+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"77'28","reported_seq":94,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.620843+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:48:25.620843+0000","last_peered":"2023-10-01T09:48:25.620843+0000","last_clean":"2023-10-01T09:48:25.620843+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:48:25.620843+0000","last_undegraded":"2023-10-01T09:48:25.620843+0000","last_fullsized":"2023-10-01T09:48:25.620843+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":62,"num_read_kb":2092,"num_write":32,"num_write_kb":2060,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":581,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.472028+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:48:25.472028+0000","last_peered":"2023-10-01T09:48:25.472028+0000","last_clean":"2023-10-01T09:48:25.472028+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:48:25.472028+0000","last_undegraded":"2023-10-01T09:48:25.472028+0000","last_fullsized":"2023-10-01T09:48:25.472028+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":285,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.620005+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:48:25.620005+0000","last_peered":"2023-10-01T09:48:25.620005+0000","last_clean":"2023-10-01T09:48:25.620005+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:48:25.620005+0000","last_undegraded":"2023-10-01T09:48:25.620005+0000","last_fullsized":"2023-10-01T09:48:25.620005+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":530,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:27.103352+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:48:27.103352+0000","last_peered":"2023-10-01T09:48:27.103352+0000","last_clean":"2023-10-01T09:48:27.103352+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:48:27.103352+0000","last_undegraded":"2023-10-01T09:48:27.103352+0000","last_fullsized":"2023-10-01T09:48:27.103352+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":643,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.726998+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:48:25.726998+0000","last_peered":"2023-10-01T09:48:25.726998+0000","last_clean":"2023-10-01T09:48:25.726998+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:48:25.726998+0000","last_undegraded":"2023-10-01T09:48:25.726998+0000","last_fullsized":"2023-10-01T09:48:25.726998+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":297,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.615265+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:48:25.615265+0000","last_peered":"2023-10-01T09:48:25.615265+0000","last_clean":"2023-10-01T09:48:25.615265+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:48:25.615265+0000","last_undegraded":"2023-10-01T09:48:25.615265+0000","last_fullsized":"2023-10-01T09:48:25.615265+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":87,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.726934+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:48:25.726934+0000","last_peered":"2023-10-01T09:48:25.726934+0000","last_clean":"2023-10-01T09:48:25.726934+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:48:25.726934+0000","last_undegraded":"2023-10-01T09:48:25.726934+0000","last_fullsized":"2023-10-01T09:48:25.726934+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":297,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.619558+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:48:25.619558+0000","last_peered":"2023-10-01T09:48:25.619558+0000","last_clean":"2023-10-01T09:48:25.619558+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:48:25.619558+0000","last_undegraded":"2023-10-01T09:48:25.619558+0000","last_fullsized":"2023-10-01T09:48:25.619558+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":476,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.467134+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:48:25.467134+0000","last_peered":"2023-10-01T09:48:25.467134+0000","last_clean":"2023-10-01T09:48:25.467134+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:48:25.467134+0000","last_undegraded":"2023-10-01T09:48:25.467134+0000","last_fullsized":"2023-10-01T09:48:25.467134+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":105,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.727417+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:48:25.727417+0000","last_peered":"2023-10-01T09:48:25.727417+0000","last_clean":"2023-10-01T09:48:25.727417+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:48:25.727417+0000","last_undegraded":"2023-10-01T09:48:25.727417+0000","last_fullsized":"2023-10-01T09:48:25.727417+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":521,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.466713+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:48:25.466713+0000","last_peered":"2023-10-01T09:48:25.466713+0000","last_clean":"2023-10-01T09:48:25.466713+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:48:25.466713+0000","last_undegraded":"2023-10-01T09:48:25.466713+0000","last_fullsized":"2023-10-01T09:48:25.466713+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":721,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:27.103419+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:48:27.103419+0000","last_peered":"2023-10-01T09:48:27.103419+0000","last_clean":"2023-10-01T09:48:27.103419+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:48:27.103419+0000","last_undegraded":"2023-10-01T09:48:27.103419+0000","last_fullsized":"2023-10-01T09:48:27.103419+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":413,"reported_epoch":561,"state":"active+clean","last_fresh":"2023-10-01T09:48:25.474161+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:48:25.474161+0000","last_peered":"2023-10-01T09:48:25.474161+0000","last_clean":"2023-10-01T09:48:25.474161+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:48:25.474161+0000","last_undegraded":"2023-10-01T09:48:25.474161+0000","last_fullsized":"2023-10-01T09:48:25.474161+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5965,"num_read_kb":12159,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":382,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430775,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":916964,"kb_used_data":9756,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":92828188,"statfs":{"total":95995035648,"available":95056064512,"internally_reserved":0,"allocated":9990144,"data_stored":9036199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":0.67500000000000004,"15min":2.4630000000000001},"min":{"1min":0.252,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":7.6900000000000004,"5min":7.6900000000000004,"15min":108.90000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.196,"5min":0.60999999999999999,"15min":2.5680000000000001},"min":{"1min":0.32100000000000001,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":6.5199999999999996,"5min":6.5199999999999996,"15min":109.187},"last":0.41899999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:48:22 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":0.99099999999999999,"15min":4.258},"min":{"1min":0.34200000000000003,"5min":0.24199999999999999,"15min":0.23300000000000001},"max":{"1min":12.522,"5min":12.522,"15min":229.161},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.2690000000000001,"5min":0.89000000000000001,"15min":4.3109999999999999},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.24399999999999999},"max":{"1min":10.127000000000001,"5min":10.127000000000001,"15min":230.126},"last":0.51300000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:47:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":0.86099999999999999,"15min":1.54},"min":{"1min":0.33300000000000002,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":12.565,"5min":12.565,"15min":14.901999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.121,"5min":0.68200000000000005,"15min":1.429},"min":{"1min":0.372,"5min":0.32000000000000001,"15min":0.27000000000000002},"max":{"1min":6.4550000000000001,"5min":6.4550000000000001,"15min":14.682},"last":0.54300000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":0.83699999999999997,"15min":1.8009999999999999},"min":{"1min":0.42099999999999999,"5min":0.32600000000000001,"15min":0.29199999999999998},"max":{"1min":7.4699999999999998,"5min":7.4699999999999998,"15min":76.555000000000007},"last":0.59699999999999998},{"interface":"front","average":{"1min":2.008,"5min":0.88500000000000001,"15min":1.9039999999999999},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.2599999999999998,"5min":9.2599999999999998,"15min":76.534000000000006},"last":0.46100000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:48:03 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":0.79700000000000004,"15min":1.04},"min":{"1min":0.316,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.069000000000001,"5min":11.069000000000001,"15min":11.069000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.3700000000000001,"5min":0.70099999999999996,"15min":1.002},"min":{"1min":0.33700000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.4809999999999999,"5min":6.4809999999999999,"15min":8.4949999999999992},"last":0.47399999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":0.66600000000000004,"15min":1.5369999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.9370000000000003,"5min":7.9370000000000003,"15min":135.40000000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.65100000000000002,"15min":1.6379999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":7.0629999999999997,"5min":7.0629999999999997,"15min":135.339},"last":0.39000000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.64900000000000002,"15min":1.1519999999999999},"min":{"1min":0.39600000000000002,"5min":0.377,"15min":0.307},"max":{"1min":3.984,"5min":3.984,"15min":11.853999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.64100000000000001,"15min":1.0880000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32300000000000001},"max":{"1min":4.3470000000000004,"5min":4.3470000000000004,"15min":14.584},"last":0.504}]}]},{"osd":6,"up_from":294,"seq":1262720385285,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":335060,"kb_used_data":5644,"kb_used_omap":6,"kb_used_meta":329401,"kb_avail":93410092,"statfs":{"total":95995035648,"available":95651934208,"internally_reserved":0,"allocated":5779456,"data_stored":4834281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":337306669},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:24 2023","interfaces":[{"interface":"back","average":{"1min":5.0350000000000001,"5min":2.173,"15min":2.7240000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.20599999999999999},"max":{"1min":77.837000000000003,"5min":77.837000000000003,"15min":158.48599999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":4.6109999999999998,"5min":2.3210000000000002,"15min":2.8639999999999999},"min":{"1min":0.27400000000000002,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":77.971000000000004,"5min":77.971000000000004,"15min":158.37700000000001},"last":0.49399999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:48:24 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":0.82099999999999995,"15min":3.7250000000000001},"min":{"1min":0.34300000000000003,"5min":0.222,"15min":0.222},"max":{"1min":6.5860000000000003,"5min":8.2409999999999997,"15min":168.947},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.2370000000000001,"5min":0.71099999999999997,"15min":3.6400000000000001},"min":{"1min":0.315,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.8390000000000004,"5min":6.8390000000000004,"15min":169.07900000000001},"last":0.51700000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:48:24 2023","interfaces":[{"interface":"back","average":{"1min":1.5229999999999999,"5min":0.94699999999999995,"15min":1.331},"min":{"1min":0.34200000000000003,"5min":0.254,"15min":0.254},"max":{"1min":6.6070000000000002,"5min":8.2829999999999995,"15min":9.657},"last":0.438},{"interface":"front","average":{"1min":1.607,"5min":0.80900000000000005,"15min":1.2949999999999999},"min":{"1min":0.33900000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":6.8159999999999998,"5min":6.8159999999999998,"15min":21.704999999999998},"last":0.42399999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:48:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":0.75900000000000001,"15min":1.5960000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.3920000000000003,"5min":6.3920000000000003,"15min":89.753},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.2090000000000001,"5min":0.92400000000000004,"15min":1.657},"min":{"1min":0.39100000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.6470000000000002,"5min":8.3109999999999999,"15min":89.849999999999994},"last":0.55800000000000005}]},{"osd":4,"last update":"Sun Oct 1 09:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2130000000000001,"5min":1.071,"15min":1.1850000000000001},"min":{"1min":0.254,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":12.326000000000001,"5min":12.326000000000001,"15min":63.134},"last":0.54700000000000004},{"interface":"front","average":{"1min":3.9569999999999999,"5min":1.226,"15min":1.1879999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28899999999999998},"max":{"1min":13.641,"5min":13.641,"15min":62.973999999999997},"last":0.56499999999999995}]},{"osd":5,"last update":"Sun Oct 1 09:47:37 2023","interfaces":[{"interface":"back","average":{"1min":6.5650000000000004,"5min":2.089,"15min":1.367},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":136.535,"5min":136.535,"15min":136.535},"last":0.503},{"interface":"front","average":{"1min":6.3049999999999997,"5min":2.1499999999999999,"15min":1.4830000000000001},"min":{"1min":0.316,"5min":0.29899999999999999,"15min":0.28100000000000003},"max":{"1min":136.47300000000001,"5min":136.47300000000001,"15min":136.47300000000001},"last":0.47899999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.956,"5min":1.252,"15min":1.397},"min":{"1min":0.34899999999999998,"5min":0.32700000000000001,"15min":0.29899999999999999},"max":{"1min":15.407999999999999,"5min":15.407999999999999,"15min":94.245999999999995},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.893,"5min":1.048,"15min":1.4910000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.308},"max":{"1min":11.326000000000001,"5min":11.326000000000001,"15min":94.715000000000003},"last":0.57799999999999996}]}]},{"osd":1,"up_from":94,"seq":403726926262,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":850764,"kb_used_data":10436,"kb_used_omap":7,"kb_used_meta":840312,"kb_avail":92894388,"statfs":{"total":95995035648,"available":95123853312,"internally_reserved":0,"allocated":10686464,"data_stored":8581130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":860479911},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":11.182,"5min":3.2109999999999999,"15min":4.0659999999999998},"min":{"1min":0.217,"5min":0.186,"15min":0.182},"max":{"1min":114.33499999999999,"5min":114.33499999999999,"15min":154.21299999999999},"last":0.29899999999999999},{"interface":"front","average":{"1min":11.305,"5min":3.2799999999999998,"15min":3.637},"min":{"1min":0.16800000000000001,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":114.099,"5min":114.099,"15min":114.099},"last":0.23000000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:47:45 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.42199999999999999,"15min":1.8109999999999999},"min":{"1min":0.23699999999999999,"5min":0.223,"15min":0.20000000000000001},"max":{"1min":3.6019999999999999,"5min":3.6019999999999999,"15min":51.640000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.3049999999999999,"5min":0.63500000000000001,"15min":2.5219999999999998},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20499999999999999},"max":{"1min":12.018000000000001,"5min":12.018000000000001,"15min":154.197},"last":0.28399999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:48:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":0.66900000000000004,"15min":2.5379999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.219},"max":{"1min":8.4320000000000004,"5min":8.4320000000000004,"15min":95.305000000000007},"last":0.314},{"interface":"front","average":{"1min":1.143,"5min":0.624,"15min":3.1499999999999999},"min":{"1min":0.27100000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.8559999999999999,"5min":4.8559999999999999,"15min":154.483},"last":0.33000000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":3.839,"5min":1.2310000000000001,"15min":2.2269999999999999},"min":{"1min":0.27100000000000002,"5min":0.245,"15min":0.245},"max":{"1min":24.539999999999999,"5min":24.539999999999999,"15min":154.101},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.222,"5min":0.61899999999999999,"15min":2.0339999999999998},"min":{"1min":0.29999999999999999,"5min":0.29499999999999998,"15min":0.27700000000000002},"max":{"1min":4.9139999999999997,"5min":4.9139999999999997,"15min":154.56999999999999},"last":0.437}]},{"osd":5,"last update":"Sun Oct 1 09:47:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.339,"15min":2.2679999999999998},"min":{"1min":0.31900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":18.158000000000001,"5min":59.561,"15min":154.22999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.296,"15min":2.323},"min":{"1min":0.32800000000000001,"5min":0.31,"15min":0.307},"max":{"1min":6.5789999999999997,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.54300000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:48:15 2023","interfaces":[{"interface":"back","average":{"1min":3.8490000000000002,"5min":1.304,"15min":2.3330000000000002},"min":{"1min":0.34000000000000002,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":24.321000000000002,"5min":24.321000000000002,"15min":154.12200000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.7280000000000002,"5min":1.079,"15min":2.2349999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.2149999999999999,"5min":9.2149999999999999,"15min":154.16999999999999},"last":0.55400000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":1.1439999999999999,"15min":2.3639999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":15.009,"5min":15.009,"15min":154.20699999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.7689999999999999,"5min":0.95599999999999996,"15min":2.3799999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.34300000000000003},"max":{"1min":6.6180000000000003,"5min":6.6180000000000003,"15min":154.51599999999999},"last":0.63}]}]},{"osd":0,"up_from":89,"seq":382252089797,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1597852,"kb_used_data":644372,"kb_used_omap":8,"kb_used_meta":953463,"kb_avail":92147300,"statfs":{"total":95995035648,"available":94358835200,"internally_reserved":0,"allocated":659836928,"data_stored":657783385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":976347074},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":4.4480000000000004,"5min":1.216,"15min":3.7320000000000002},"min":{"1min":0.219,"5min":0.18099999999999999,"15min":0.17799999999999999},"max":{"1min":90.185000000000002,"5min":90.185000000000002,"15min":216.821},"last":0.46999999999999997},{"interface":"front","average":{"1min":5.6059999999999999,"5min":1.45,"15min":4.3899999999999997},"min":{"1min":0.22,"5min":0.20599999999999999,"15min":0.19600000000000001},"max":{"1min":90.102999999999994,"5min":90.102999999999994,"15min":217.02699999999999},"last":0.52500000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:47:49 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.50800000000000001,"15min":1.7210000000000001},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.21299999999999999},"max":{"1min":1.6919999999999999,"5min":4.2430000000000003,"15min":19.788},"last":0.501},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.60899999999999999,"15min":1.714},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.215},"max":{"1min":2.952,"5min":3.8719999999999999,"15min":25.844999999999999},"last":0.56100000000000005}]},{"osd":3,"last update":"Sun Oct 1 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.54900000000000004,"15min":2.516},"min":{"1min":0.245,"5min":0.22,"15min":0.22},"max":{"1min":3.3849999999999998,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.502,"5min":0.80900000000000005,"15min":2.5710000000000002},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.0650000000000004,"5min":4.9199999999999999,"15min":222.227},"last":0.64800000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:48:04 2023","interfaces":[{"interface":"back","average":{"1min":7.1349999999999998,"5min":1.8819999999999999,"15min":2.1699999999999999},"min":{"1min":0.30199999999999999,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":138.84100000000001,"5min":138.84100000000001,"15min":138.84100000000001},"last":0.77800000000000002},{"interface":"front","average":{"1min":6.8760000000000003,"5min":1.8109999999999999,"15min":2.246},"min":{"1min":0.307,"5min":0.28699999999999998,"15min":0.28299999999999997},"max":{"1min":135.94800000000001,"5min":135.94800000000001,"15min":135.94800000000001},"last":0.78600000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:47:49 2023","interfaces":[{"interface":"back","average":{"1min":6.8460000000000001,"5min":1.869,"15min":2.0089999999999999},"min":{"1min":0.33700000000000002,"5min":0.312,"15min":0.312},"max":{"1min":156.69999999999999,"5min":156.69999999999999,"15min":156.69999999999999},"last":0.83899999999999997},{"interface":"front","average":{"1min":6.7249999999999996,"5min":1.8500000000000001,"15min":1.921},"min":{"1min":0.38400000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":156.773,"5min":156.773,"15min":156.773},"last":0.76200000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.64400000000000002,"15min":1.51},"min":{"1min":0.35699999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.589,"5min":4.3129999999999997,"15min":17.535},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.66700000000000004,"15min":1.571},"min":{"1min":0.35899999999999999,"5min":0.32600000000000001,"15min":0.32400000000000001},"max":{"1min":2.5960000000000001,"5min":4.2050000000000001,"15min":15.651999999999999},"last":0.80400000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:48:04 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":0.79300000000000004,"15min":1.8089999999999999},"min":{"1min":0.40100000000000002,"5min":0.374,"15min":0.374},"max":{"1min":4.1280000000000001,"5min":4.3360000000000003,"15min":48.895000000000003},"last":0.72699999999999998},{"interface":"front","average":{"1min":1.736,"5min":0.97399999999999998,"15min":1.9350000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.8490000000000002,"5min":7.1580000000000004,"15min":48.274000000000001},"last":0.751}]}]},{"osd":2,"up_from":0,"seq":425201762736,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599209,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":870540,"kb_used_data":6404,"kb_used_omap":3,"kb_used_meta":864124,"kb_avail":92874612,"statfs":{"total":95995035648,"available":95103602688,"internally_reserved":0,"allocated":6557696,"data_stored":4387388,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":884863719},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":5.0350000000000001,"5min":1.264,"15min":3.4929999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17899999999999999},"max":{"1min":74.707999999999998,"5min":74.707999999999998,"15min":137.34399999999999},"last":0.215},{"interface":"front","average":{"1min":5.5439999999999996,"5min":1.4099999999999999,"15min":3.577},"min":{"1min":0.217,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":74.555000000000007,"5min":74.555000000000007,"15min":137.73400000000001},"last":0.28100000000000003}]},{"osd":1,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":0.52400000000000002,"15min":4.25},"min":{"1min":0.23699999999999999,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":5.9009999999999998,"5min":5.9009999999999998,"15min":162.22999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.47399999999999998,"15min":4.2800000000000002},"min":{"1min":0.23300000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.008,"5min":4.008,"15min":162.70699999999999},"last":0.26700000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.501,"15min":1.508},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.22800000000000001},"max":{"1min":3.5059999999999998,"5min":3.5059999999999998,"15min":14.254},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.60299999999999998,"15min":1.3029999999999999},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":3.9809999999999999,"5min":4.5049999999999999,"15min":15.523},"last":0.316}]},{"osd":4,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":1.591,"5min":0.76700000000000002,"15min":1.639},"min":{"1min":0.312,"5min":0.312,"15min":0.26200000000000001},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":77.793000000000006},"last":0.35499999999999998},{"interface":"front","average":{"1min":2.5219999999999998,"5min":0.91300000000000003,"15min":1.8680000000000001},"min":{"1min":0.35499999999999998,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":9.7370000000000001,"5min":9.7370000000000001,"15min":76.646000000000001},"last":0.33200000000000002}]},{"osd":5,"last update":"Sun Oct 1 09:47:37 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.67200000000000004,"15min":1.7150000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27500000000000002},"max":{"1min":3.1459999999999999,"5min":4.3099999999999996,"15min":28.117999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.502,"15min":1.8140000000000001},"min":{"1min":0.34300000000000003,"5min":0.33600000000000002,"15min":0.27700000000000002},"max":{"1min":0.56899999999999995,"5min":2.512,"15min":28.34},"last":0.38300000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:47:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54500000000000004,"15min":1.758},"min":{"1min":0.45500000000000002,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":1.0549999999999999,"5min":2.9100000000000001,"15min":73.887},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.65000000000000002,"15min":1.869},"min":{"1min":0.38100000000000001,"5min":0.35499999999999998,"15min":0.33600000000000002},"max":{"1min":3.1099999999999999,"5min":3.1099999999999999,"15min":74.016999999999996},"last":0.46600000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":2.194,"5min":1.1579999999999999,"15min":1.8180000000000001},"min":{"1min":0.47499999999999998,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":10.574,"5min":10.574,"15min":13.085000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":2.3330000000000002,"5min":1.1279999999999999,"15min":1.5860000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.311},"max":{"1min":10.609,"5min":10.609,"15min":13.191000000000001},"last":0.45500000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019460,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1220636,"kb_used_data":322260,"kb_used_omap":0,"kb_used_meta":898368,"kb_avail":92524516,"statfs":{"total":95995035648,"available":94745104384,"internally_reserved":0,"allocated":329994240,"data_stored":329047873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":919928832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:02 2023","interfaces":[{"interface":"back","average":{"1min":8.1419999999999995,"5min":2.4369999999999998,"15min":3.8559999999999999},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":95.548000000000002,"5min":95.548000000000002,"15min":155.315},"last":0.497},{"interface":"front","average":{"1min":8.2460000000000004,"5min":2.4420000000000002,"15min":4.125},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":95.253,"5min":95.253,"15min":155.334},"last":0.69399999999999995}]},{"osd":1,"last update":"Sun Oct 1 09:48:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2709999999999999,"5min":0.67900000000000005,"15min":5.0679999999999996},"min":{"1min":0.24299999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":7.5979999999999999,"5min":7.5979999999999999,"15min":190.61199999999999},"last":0.629},{"interface":"front","average":{"1min":1.089,"5min":0.67500000000000004,"15min":5.7759999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":6.3490000000000002,"5min":6.3490000000000002,"15min":253.273},"last":0.58999999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":0.80200000000000005,"15min":2.0510000000000002},"min":{"1min":0.32200000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.6390000000000002,"5min":7.6390000000000002,"15min":170.45400000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":1.284,"5min":0.754,"15min":1.6779999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.6619999999999999,"5min":7.6619999999999999,"15min":36.515999999999998},"last":0.54200000000000004}]},{"osd":3,"last update":"Sun Oct 1 09:48:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":0.80500000000000005,"15min":1.9450000000000001},"min":{"1min":0.28599999999999998,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":11.667,"5min":11.667,"15min":140.881},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.141,"5min":0.748,"15min":2.0089999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.3769999999999998,"5min":6.3769999999999998,"15min":140.911},"last":0.41499999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:47:41 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.0369999999999999,"15min":1.5740000000000001},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.26600000000000001},"max":{"1min":21.364999999999998,"5min":21.364999999999998,"15min":110.003},"last":0.71999999999999997},{"interface":"front","average":{"1min":1.9510000000000001,"5min":0.93200000000000005,"15min":1.466},"min":{"1min":0.313,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":21.559999999999999,"5min":21.559999999999999,"15min":108.56399999999999},"last":0.61899999999999999}]},{"osd":6,"last update":"Sun Oct 1 09:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":0.84499999999999997,"15min":0.92000000000000004},"min":{"1min":0.35199999999999998,"5min":0.318,"15min":0.27700000000000002},"max":{"1min":11.635999999999999,"5min":11.635999999999999,"15min":11.635999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.9590000000000001,"5min":0.88300000000000001,"15min":1.079},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":7.7149999999999999,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":0.66300000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:47:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":0.88600000000000001,"15min":1.1950000000000001},"min":{"1min":0.40600000000000003,"5min":0.313,"15min":0.313},"max":{"1min":11.694000000000001,"5min":11.694000000000001,"15min":38.034999999999997},"last":0.70299999999999996},{"interface":"front","average":{"1min":2.2130000000000001,"5min":1.0269999999999999,"15min":1.4750000000000001},"min":{"1min":0.35799999999999998,"5min":0.35599999999999998,"15min":0.308},"max":{"1min":12.228,"5min":12.228,"15min":37.723999999999997},"last":0.76300000000000001}]}]},{"osd":5,"up_from":379,"seq":1627792605367,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":373708,"kb_used_data":5060,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":93371444,"statfs":{"total":95995035648,"available":95612358656,"internally_reserved":0,"allocated":5181440,"data_stored":4246905,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":17,"apply_latency_ms":17,"commit_latency_ns":17000000,"apply_latency_ns":17000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":4.9749999999999996,"5min":1.6719999999999999,"15min":2.3140000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.21099999999999999},"max":{"1min":99.609999999999999,"5min":99.609999999999999,"15min":99.609999999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":5.117,"5min":1.7130000000000001,"15min":2.2959999999999998},"min":{"1min":0.215,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":98.995999999999995,"5min":98.995999999999995,"15min":98.995999999999995},"last":0.48899999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":5.4790000000000001},"min":{"1min":0.28399999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.157,"5min":4.3289999999999997,"15min":295.71300000000002},"last":0.437},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.56699999999999995,"15min":5.5129999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":3.665,"5min":3.9580000000000002,"15min":299.59399999999999},"last":0.30299999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.60399999999999998,"15min":1.4179999999999999},"min":{"1min":0.28999999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.9129999999999998,"5min":4.2670000000000003,"15min":14.199999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.70899999999999996,"15min":1.357},"min":{"1min":0.31900000000000001,"5min":0.29599999999999999,"15min":0.27100000000000002},"max":{"1min":4.1909999999999998,"5min":4.3719999999999999,"15min":14.430999999999999},"last":0.32800000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":0.70599999999999996,"15min":1.3759999999999999},"min":{"1min":0.30599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.141,"5min":7.141,"15min":14.004},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.66600000000000004,"15min":1.5029999999999999},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.32500000000000001},"max":{"1min":3.1139999999999999,"5min":4.2910000000000004,"15min":13.991},"last":0.45800000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.66400000000000003,"15min":1.375},"min":{"1min":0.27900000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.0659999999999998,"5min":4.1929999999999996,"15min":122.971},"last":0.371},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.66500000000000004,"15min":1.472},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.2469999999999999,"5min":4.3869999999999996,"15min":122.89100000000001},"last":0.46600000000000003}]},{"osd":6,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.63800000000000001,"15min":0.94599999999999995},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28299999999999997},"max":{"1min":3.5750000000000002,"5min":4.3300000000000001,"15min":12.545},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.097,"5min":0.76300000000000001,"15min":1.119},"min":{"1min":0.31,"5min":0.31,"15min":0.30499999999999999},"max":{"1min":4.484,"5min":4.484,"15min":14.193},"last":0.51400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.74399999999999999,"15min":1.1120000000000001},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.2199999999999998,"5min":4.3559999999999999,"15min":10.813000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.75600000000000001,"15min":1.0509999999999999},"min":{"1min":0.40799999999999997,"5min":0.36299999999999999,"15min":0.318},"max":{"1min":3.609,"5min":4.2000000000000002,"15min":12.500999999999999},"last":0.47399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:48:36.332 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-01T09:48:36.333 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:48:36.333 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T09:48:36.333 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 1.000000 2023-10-01T09:48:36.333 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-01T09:48:36.337 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:36.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:36.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:36.421 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:36.445 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:36.497 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:36.548 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:36.560 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:36.565 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[185886]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.565 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[185886]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[185886]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi138-722318-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-01T09:48:36.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[185886]: osdmap e564: 8 total, 8 up, 7 in 2023-10-01T09:48:36.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[185886]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[185886]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[185886]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[185886]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.566 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:36.566 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[192513]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.567 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[192513]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.567 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[192513]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.567 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi138-722318-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-01T09:48:36.567 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[192513]: osdmap e564: 8 total, 8 up, 7 in 2023-10-01T09:48:36.567 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[192513]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.567 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[192513]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.567 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[192513]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.567 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[192513]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.567 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:36 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:36.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:36.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:36.669 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:36.696 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:36.741 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:36.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:36.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:36.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:36 smithi190 ceph-mon[160435]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:36 smithi190 ceph-mon[160435]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:36 smithi190 ceph-mon[160435]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:36 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi138-722318-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-01T09:48:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:36 smithi190 ceph-mon[160435]: osdmap e564: 8 total, 8 up, 7 in 2023-10-01T09:48:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:36 smithi190 ceph-mon[160435]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:36 smithi190 ceph-mon[160435]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:36 smithi190 ceph-mon[160435]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:36 smithi190 ceph-mon[160435]: failed to encode map e564 with expected crc 2023-10-01T09:48:36.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:36 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:36.846 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:36.881 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:36.958 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:48:36.958 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [ 2023-10-01T09:48:36.958 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:36.958 INFO:teuthology.orchestra.run.smithi138.stdout: "description": "osd_repop(client.47585.0:1090 11.6 e565/563 11:67cae2ed:::rbd_mirroring:head v 565'596, mlcod=565'594)", 2023-10-01T09:48:36.958 INFO:teuthology.orchestra.run.smithi138.stdout: "initiated_at": "2023-10-01T09:48:36.956848+0000", 2023-10-01T09:48:36.958 INFO:teuthology.orchestra.run.smithi138.stdout: "age": 0.00030874199999999998, 2023-10-01T09:48:36.958 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0.000338074, 2023-10-01T09:48:36.958 INFO:teuthology.orchestra.run.smithi138.stdout: "type_data": { 2023-10-01T09:48:36.958 INFO:teuthology.orchestra.run.smithi138.stdout: "flag_point": "started", 2023-10-01T09:48:36.959 INFO:teuthology.orchestra.run.smithi138.stdout: "events": [ 2023-10-01T09:48:36.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:36.959 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "initiated", 2023-10-01T09:48:36.959 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:36.956848+0000", 2023-10-01T09:48:36.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:48:36.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:36.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:36.959 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "throttled", 2023-10-01T09:48:36.959 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:36.956848+0000", 2023-10-01T09:48:36.959 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 0 2023-10-01T09:48:36.959 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:36.959 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:36.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "header_read", 2023-10-01T09:48:36.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:36.956847+0000", 2023-10-01T09:48:36.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 4294967295.999999 2023-10-01T09:48:36.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:36.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:36.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "all_read", 2023-10-01T09:48:36.960 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:36.956853+0000", 2023-10-01T09:48:36.960 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 6.5679999999999996e-06 2023-10-01T09:48:36.960 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:36.960 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:36.960 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "dispatched", 2023-10-01T09:48:36.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:36.956865+0000", 2023-10-01T09:48:36.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.1741e-05 2023-10-01T09:48:36.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:36.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:36.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "queued_for_pg", 2023-10-01T09:48:36.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:36.956872+0000", 2023-10-01T09:48:36.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.3810000000000001e-06 2023-10-01T09:48:36.961 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:36.961 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:36.961 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "reached_pg", 2023-10-01T09:48:36.961 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:36.956945+0000", 2023-10-01T09:48:36.961 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 7.2305000000000007e-05 2023-10-01T09:48:36.962 INFO:teuthology.orchestra.run.smithi138.stdout: }, 2023-10-01T09:48:36.962 INFO:teuthology.orchestra.run.smithi138.stdout: { 2023-10-01T09:48:36.962 INFO:teuthology.orchestra.run.smithi138.stdout: "event": "started", 2023-10-01T09:48:36.962 INFO:teuthology.orchestra.run.smithi138.stdout: "time": "2023-10-01T09:48:36.956964+0000", 2023-10-01T09:48:36.962 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 1.8850000000000001e-05 2023-10-01T09:48:36.962 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:48:36.962 INFO:teuthology.orchestra.run.smithi138.stdout: ] 2023-10-01T09:48:36.962 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:48:36.962 INFO:teuthology.orchestra.run.smithi138.stdout: } 2023-10-01T09:48:36.962 INFO:teuthology.orchestra.run.smithi138.stdout: ], 2023-10-01T09:48:36.962 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 1 2023-10-01T09:48:36.962 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:48:37.093 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:37.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:37.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:37.198 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:37.232 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:37.512 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:37 smithi138 ceph-mon[185886]: pgmap v1702: 105 pgs: 10 creating+peering, 6 unknown, 89 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 1.0 MiB/s rd, 527 KiB/s wr, 36 op/s 2023-10-01T09:48:37.512 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:37 smithi138 ceph-mon[185886]: failed to encode map e565 with expected crc 2023-10-01T09:48:37.512 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:37 smithi138 ceph-mon[185886]: failed to encode map e565 with expected crc 2023-10-01T09:48:37.512 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:37 smithi138 ceph-mon[185886]: osdmap e565: 8 total, 8 up, 7 in 2023-10-01T09:48:37.512 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:37 smithi138 ceph-mon[185886]: failed to encode map e565 with expected crc 2023-10-01T09:48:37.512 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:37 smithi138 ceph-mon[185886]: failed to encode map e565 with expected crc 2023-10-01T09:48:37.513 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:37 smithi138 ceph-mon[185886]: failed to encode map e565 with expected crc 2023-10-01T09:48:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:37 smithi190 ceph-mon[160435]: pgmap v1702: 105 pgs: 10 creating+peering, 6 unknown, 89 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 1.0 MiB/s rd, 527 KiB/s wr, 36 op/s 2023-10-01T09:48:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:37 smithi190 ceph-mon[160435]: failed to encode map e565 with expected crc 2023-10-01T09:48:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:37 smithi190 ceph-mon[160435]: failed to encode map e565 with expected crc 2023-10-01T09:48:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:37 smithi190 ceph-mon[160435]: osdmap e565: 8 total, 8 up, 7 in 2023-10-01T09:48:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:37 smithi190 ceph-mon[160435]: failed to encode map e565 with expected crc 2023-10-01T09:48:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:37 smithi190 ceph-mon[160435]: failed to encode map e565 with expected crc 2023-10-01T09:48:37.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:37 smithi190 ceph-mon[160435]: failed to encode map e565 with expected crc 2023-10-01T09:48:37.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:37 smithi138 ceph-mon[192513]: pgmap v1702: 105 pgs: 10 creating+peering, 6 unknown, 89 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 1.0 MiB/s rd, 527 KiB/s wr, 36 op/s 2023-10-01T09:48:37.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:37 smithi138 ceph-mon[192513]: failed to encode map e565 with expected crc 2023-10-01T09:48:37.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:37 smithi138 ceph-mon[192513]: failed to encode map e565 with expected crc 2023-10-01T09:48:37.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:37 smithi138 ceph-mon[192513]: osdmap e565: 8 total, 8 up, 7 in 2023-10-01T09:48:37.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:37 smithi138 ceph-mon[192513]: failed to encode map e565 with expected crc 2023-10-01T09:48:37.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:37 smithi138 ceph-mon[192513]: failed to encode map e565 with expected crc 2023-10-01T09:48:37.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:37 smithi138 ceph-mon[192513]: failed to encode map e565 with expected crc 2023-10-01T09:48:38.039 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:38.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:38.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:38.143 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:38.174 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:38.406 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T09:48:39.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:38 smithi190 ceph-mon[160435]: pgmap v1704: 105 pgs: 10 creating+peering, 95 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 864 KiB/s wr, 487 op/s 2023-10-01T09:48:39.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:38 smithi138 ceph-mon[192513]: pgmap v1704: 105 pgs: 10 creating+peering, 95 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 864 KiB/s wr, 487 op/s 2023-10-01T09:48:39.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:38 smithi138 ceph-mon[185886]: pgmap v1704: 105 pgs: 10 creating+peering, 95 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 1.6 MiB/s rd, 864 KiB/s wr, 487 op/s 2023-10-01T09:48:39.339 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:39.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:39.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:39.443 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:39.473 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:39.659 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:39.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:39.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:39.699 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.mirror_image_status (3446 ms) 2023-10-01T09:48:39.699 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-01T09:48:39.762 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:39.765 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:39.804 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.mirror_image_map (104 ms) 2023-10-01T09:48:39.804 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-01T09:48:39.806 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:39.814 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.mirror_instances (10 ms) 2023-10-01T09:48:39.814 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-01T09:48:39.827 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.mirror_snapshot (14 ms) 2023-10-01T09:48:39.828 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-01T09:48:39.830 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-10-01T09:48:39.830 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-01T09:48:39.833 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_dir_add (3 ms) 2023-10-01T09:48:39.833 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-01T09:48:39.837 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.dir_add_already_existing (4 ms) 2023-10-01T09:48:39.837 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-01T09:48:39.843 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_dir_rename (7 ms) 2023-10-01T09:48:39.844 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-01T09:48:39.849 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_dir_remove (5 ms) 2023-10-01T09:48:39.849 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-01T09:48:39.854 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (6 ms) 2023-10-01T09:48:39.855 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-01T09:48:39.857 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2023-10-01T09:48:39.857 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-01T09:48:39.860 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-10-01T09:48:39.860 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-01T09:48:39.863 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_image_list (4 ms) 2023-10-01T09:48:39.864 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-01T09:48:39.868 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2023-10-01T09:48:39.868 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-01T09:48:39.870 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2023-10-01T09:48:39.870 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-01T09:48:39.873 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-10-01T09:48:39.873 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-01T09:48:39.875 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-10-01T09:48:39.875 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-01T09:48:39.877 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-10-01T09:48:39.877 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-01T09:48:39.879 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-10-01T09:48:39.879 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-01T09:48:39.881 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-10-01T09:48:39.881 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-01T09:48:39.885 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-10-01T09:48:39.885 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-01T09:48:39.887 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_snap_set (3 ms) 2023-10-01T09:48:39.888 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-01T09:48:39.893 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_snap_list (5 ms) 2023-10-01T09:48:39.893 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-01T09:48:39.911 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (18 ms) 2023-10-01T09:48:39.912 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-01T09:48:39.947 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:39.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:39.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:40.052 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:48:40.078 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:40.121 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:48:40.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:40.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:48:40.146 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (235 ms) 2023-10-01T09:48:40.146 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-01T09:48:40.150 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2023-10-01T09:48:40.150 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-01T09:48:40.152 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (3 ms) 2023-10-01T09:48:40.152 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-01T09:48:40.161 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.trash_methods (8 ms) 2023-10-01T09:48:40.161 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.op_features 2023-10-01T09:48:40.168 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2023-10-01T09:48:40.168 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-01T09:48:40.192 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.clone_parent (25 ms) 2023-10-01T09:48:40.193 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-01T09:48:40.204 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.clone_parent_ns (11 ms) 2023-10-01T09:48:40.204 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-01T09:48:40.216 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.clone_child (13 ms) 2023-10-01T09:48:40.216 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-01T09:48:40.226 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:40.226 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.namespace_methods (9 ms) 2023-10-01T09:48:40.227 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.migration 2023-10-01T09:48:40.237 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.migration (11 ms) 2023-10-01T09:48:40.237 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-01T09:48:40.248 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.migration_v1 (11 ms) 2023-10-01T09:48:40.248 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-01T09:48:40.265 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:40.364 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:40.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:40.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:40.453 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:48:40.457 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:40.469 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:48:40.473 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:48:40.494 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:48:40.496 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:40.517 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:48:40.537 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:48:40.540 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:48:40.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:40.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:48:40.558 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:48:40.576 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:48:40.593 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:48:40.595 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:48:40.595 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:48:40.595 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:48:40.595 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:48:40.595 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:48:40.596 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:48:40.596 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:48:40.596 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:48:40.643 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:40.673 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:40.721 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:40.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:40.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:40.826 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:40.851 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:40.893 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:40.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:40.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:40.999 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:48:41.025 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:41.066 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:48:41.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:41.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:48:41.173 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:41.199 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:41.240 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:41.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:41.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:41.332 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1084 ms) 2023-10-01T09:48:41.332 INFO:tasks.workunit.client.0.smithi138.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-01T09:48:41.335 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is set 2023-10-01T09:48:41.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[185886]: failed to encode map e566 with expected crc 2023-10-01T09:48:41.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[185886]: osdmap e566: 8 total, 8 up, 7 in 2023-10-01T09:48:41.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[185886]: failed to encode map e566 with expected crc 2023-10-01T09:48:41.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:41.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:48:41.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4115238550' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:48:41.343 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:48:41.344 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[185886]: pgmap v1706: 105 pgs: 105 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 1.7 MiB/s rd, 894 KiB/s wr, 592 op/s 2023-10-01T09:48:41.344 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2798198995' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:41.344 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[185886]: failed to encode map e566 with expected crc 2023-10-01T09:48:41.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[192513]: failed to encode map e566 with expected crc 2023-10-01T09:48:41.344 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[192513]: osdmap e566: 8 total, 8 up, 7 in 2023-10-01T09:48:41.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[192513]: failed to encode map e566 with expected crc 2023-10-01T09:48:41.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:41.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:48:41.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4115238550' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:48:41.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:48:41.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[192513]: pgmap v1706: 105 pgs: 105 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 1.7 MiB/s rd, 894 KiB/s wr, 592 op/s 2023-10-01T09:48:41.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2798198995' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:41.345 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:41 smithi138 ceph-mon[192513]: failed to encode map e566 with expected crc 2023-10-01T09:48:41.346 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:41.372 INFO:tasks.workunit.client.0.smithi138.stdout:[ OK ] TestClsRbd.sparsify (40 ms) 2023-10-01T09:48:41.374 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:41 smithi190 ceph-mon[160435]: failed to encode map e566 with expected crc 2023-10-01T09:48:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:41 smithi190 ceph-mon[160435]: osdmap e566: 8 total, 8 up, 7 in 2023-10-01T09:48:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:41 smithi190 ceph-mon[160435]: failed to encode map e566 with expected crc 2023-10-01T09:48:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:41 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:48:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:41 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4115238550' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:48:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:41 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T09:48:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:41 smithi190 ceph-mon[160435]: pgmap v1706: 105 pgs: 105 active+clean; 4.6 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 1.7 MiB/s rd, 894 KiB/s wr, 592 op/s 2023-10-01T09:48:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:41 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2798198995' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:41.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:41 smithi190 ceph-mon[160435]: failed to encode map e566 with expected crc 2023-10-01T09:48:42.333 INFO:tasks.workunit.client.0.smithi138.stdout:[----------] 70 tests from TestClsRbd (5885 ms total) 2023-10-01T09:48:42.334 INFO:tasks.workunit.client.0.smithi138.stdout: 2023-10-01T09:48:42.334 INFO:tasks.workunit.client.0.smithi138.stdout:[----------] Global test environment tear-down 2023-10-01T09:48:42.334 INFO:tasks.workunit.client.0.smithi138.stdout:[==========] 70 tests from 1 test suite ran. (8514 ms total) 2023-10-01T09:48:42.334 INFO:tasks.workunit.client.0.smithi138.stdout:[ PASSED ] 70 tests. 2023-10-01T09:48:42.336 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-01T09:48:42.336 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-01T09:48:42.401 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:42 smithi138 ceph-mon[192513]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:48:42.401 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:42 smithi138 ceph-mon[192513]: failed to encode map e567 with expected crc 2023-10-01T09:48:42.401 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:42 smithi138 ceph-mon[192513]: failed to encode map e567 with expected crc 2023-10-01T09:48:42.401 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:42 smithi138 ceph-mon[192513]: failed to encode map e567 with expected crc 2023-10-01T09:48:42.402 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:42 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:48:42.402 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:42 smithi138 ceph-mon[192513]: osdmap e567: 8 total, 8 up, 7 in 2023-10-01T09:48:42.402 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:42 smithi138 ceph-mon[192513]: failed to encode map e567 with expected crc 2023-10-01T09:48:42.402 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:42 smithi138 ceph-mon[185886]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:48:42.402 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:42 smithi138 ceph-mon[185886]: failed to encode map e567 with expected crc 2023-10-01T09:48:42.402 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:42 smithi138 ceph-mon[185886]: failed to encode map e567 with expected crc 2023-10-01T09:48:42.402 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:42 smithi138 ceph-mon[185886]: failed to encode map e567 with expected crc 2023-10-01T09:48:42.402 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:42 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:48:42.402 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:42 smithi138 ceph-mon[185886]: osdmap e567: 8 total, 8 up, 7 in 2023-10-01T09:48:42.403 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:42 smithi138 ceph-mon[185886]: failed to encode map e567 with expected crc 2023-10-01T09:48:42.491 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-01T09:48:42.492 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-01T09:48:42.730 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:42.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:42.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:42 smithi190 ceph-mon[160435]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:48:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:42 smithi190 ceph-mon[160435]: failed to encode map e567 with expected crc 2023-10-01T09:48:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:42 smithi190 ceph-mon[160435]: failed to encode map e567 with expected crc 2023-10-01T09:48:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:42 smithi190 ceph-mon[160435]: failed to encode map e567 with expected crc 2023-10-01T09:48:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:42 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T09:48:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:42 smithi190 ceph-mon[160435]: osdmap e567: 8 total, 8 up, 7 in 2023-10-01T09:48:42.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:42 smithi190 ceph-mon[160435]: failed to encode map e567 with expected crc 2023-10-01T09:48:42.810 DEBUG:teuthology.parallel:result is None 2023-10-01T09:48:42.811 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T09:48:42.835 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:42.838 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T09:48:42.838 DEBUG:teuthology.orchestra.run.smithi138:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-01T09:48:42.897 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T09:48:42.897 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-01T09:48:42.897 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-01T09:48:42.897 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-01T09:48:42.897 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-01T09:48:42.898 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-01T09:48:42.898 DEBUG:teuthology.orchestra.run.smithi138:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-01T09:48:42.905 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:42.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T09:48:42.922 INFO:teuthology.orchestra.run.smithi138.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-01T09:48:42.922 DEBUG:teuthology.orchestra.run.smithi138:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-01T09:48:42.979 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-01T09:48:42.979 DEBUG:teuthology.orchestra.run.smithi138:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-01T09:48:43.035 INFO:tasks.workunit:timeout=3h 2023-10-01T09:48:43.036 INFO:tasks.workunit:cleanup=True 2023-10-01T09:48:43.036 DEBUG:teuthology.orchestra.run.smithi138:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-01T09:48:43.094 INFO:tasks.workunit.client.0.smithi138.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-01T09:48:43.362 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:43.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:43.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:43.466 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:43.498 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:43.548 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:43.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:43.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:43.653 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:43.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:43 smithi138 ceph-mon[185886]: failed to encode map e568 with expected crc 2023-10-01T09:48:43.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:43 smithi138 ceph-mon[185886]: failed to encode map e568 with expected crc 2023-10-01T09:48:43.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:43 smithi138 ceph-mon[185886]: failed to encode map e568 with expected crc 2023-10-01T09:48:43.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:43 smithi138 ceph-mon[185886]: osdmap e568: 8 total, 8 up, 7 in 2023-10-01T09:48:43.682 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:43 smithi138 ceph-mon[185886]: pgmap v1709: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:43.682 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:43 smithi138 ceph-mon[192513]: failed to encode map e568 with expected crc 2023-10-01T09:48:43.682 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:43 smithi138 ceph-mon[192513]: failed to encode map e568 with expected crc 2023-10-01T09:48:43.683 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:43 smithi138 ceph-mon[192513]: failed to encode map e568 with expected crc 2023-10-01T09:48:43.683 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:43 smithi138 ceph-mon[192513]: osdmap e568: 8 total, 8 up, 7 in 2023-10-01T09:48:43.683 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:43 smithi138 ceph-mon[192513]: pgmap v1709: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:43.686 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:43.736 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:43.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:43.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:43.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:43 smithi190 ceph-mon[160435]: failed to encode map e568 with expected crc 2023-10-01T09:48:43.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:43 smithi190 ceph-mon[160435]: failed to encode map e568 with expected crc 2023-10-01T09:48:43.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:43 smithi190 ceph-mon[160435]: failed to encode map e568 with expected crc 2023-10-01T09:48:43.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:43 smithi190 ceph-mon[160435]: osdmap e568: 8 total, 8 up, 7 in 2023-10-01T09:48:43.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:43 smithi190 ceph-mon[160435]: pgmap v1709: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:43.842 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:43.870 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:43.921 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:43.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:43.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:43.946 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:48:43.946 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:48:44.026 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:44.057 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:44.106 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:44.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:44.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:44.214 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:44.243 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:44.289 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:44.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:44.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:44.346 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-01T09:48:44.396 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:44.423 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:44.694 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:44 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3639924697' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:48:44.694 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:44 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3639924697' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:48:44.720 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:44.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:44.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:44.744 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:48:44.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:44 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3639924697' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:48:44.824 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:44.850 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:44.910 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T09:48:44.976 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:48:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:48:44] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.33.4" 2023-10-01T09:48:45.120 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:45.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:45.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:45.226 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:45.257 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:45.306 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:45.309 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:45.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:45.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:45.412 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:45.444 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:45.447 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T09:48:45.466 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:45.548 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:45 smithi138 ceph-mon[192513]: failed to encode map e569 with expected crc 2023-10-01T09:48:45.548 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:45 smithi138 ceph-mon[192513]: failed to encode map e569 with expected crc 2023-10-01T09:48:45.548 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:45 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3639924697' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:48:45.548 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:45 smithi138 ceph-mon[192513]: osdmap e569: 8 total, 8 up, 7 in 2023-10-01T09:48:45.548 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:45 smithi138 ceph-mon[192513]: failed to encode map e569 with expected crc 2023-10-01T09:48:45.549 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:45 smithi138 ceph-mon[192513]: pgmap v1711: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:45.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:45 smithi138 ceph-mon[185886]: failed to encode map e569 with expected crc 2023-10-01T09:48:45.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:45 smithi138 ceph-mon[185886]: failed to encode map e569 with expected crc 2023-10-01T09:48:45.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:45 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3639924697' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:48:45.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:45 smithi138 ceph-mon[185886]: osdmap e569: 8 total, 8 up, 7 in 2023-10-01T09:48:45.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:45 smithi138 ceph-mon[185886]: failed to encode map e569 with expected crc 2023-10-01T09:48:45.549 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:45 smithi138 ceph-mon[185886]: pgmap v1711: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:45.550 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:45.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:45.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:45.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:45 smithi190 ceph-mon[160435]: failed to encode map e569 with expected crc 2023-10-01T09:48:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:45 smithi190 ceph-mon[160435]: failed to encode map e569 with expected crc 2023-10-01T09:48:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:45 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3639924697' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:48:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:45 smithi190 ceph-mon[160435]: osdmap e569: 8 total, 8 up, 7 in 2023-10-01T09:48:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:45 smithi190 ceph-mon[160435]: failed to encode map e569 with expected crc 2023-10-01T09:48:45.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:45 smithi190 ceph-mon[160435]: pgmap v1711: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:45.654 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:45.685 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:45.734 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:45.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:45.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:45.839 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:45.866 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:45.909 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:45.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:45.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:46.014 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:46.049 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:46 smithi138 ceph-mon[192513]: failed to encode map e570 with expected crc 2023-10-01T09:48:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:46 smithi138 ceph-mon[192513]: failed to encode map e570 with expected crc 2023-10-01T09:48:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:46 smithi138 ceph-mon[192513]: failed to encode map e570 with expected crc 2023-10-01T09:48:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:46 smithi138 ceph-mon[192513]: osdmap e570: 8 total, 8 up, 7 in 2023-10-01T09:48:46.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:46 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:46.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:46 smithi138 ceph-mon[185886]: failed to encode map e570 with expected crc 2023-10-01T09:48:46.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:46 smithi138 ceph-mon[185886]: failed to encode map e570 with expected crc 2023-10-01T09:48:46.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:46 smithi138 ceph-mon[185886]: failed to encode map e570 with expected crc 2023-10-01T09:48:46.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:46 smithi138 ceph-mon[185886]: osdmap e570: 8 total, 8 up, 7 in 2023-10-01T09:48:46.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:46 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:46.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:46 smithi190 ceph-mon[160435]: failed to encode map e570 with expected crc 2023-10-01T09:48:46.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:46 smithi190 ceph-mon[160435]: failed to encode map e570 with expected crc 2023-10-01T09:48:46.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:46 smithi190 ceph-mon[160435]: failed to encode map e570 with expected crc 2023-10-01T09:48:46.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:46 smithi190 ceph-mon[160435]: osdmap e570: 8 total, 8 up, 7 in 2023-10-01T09:48:46.810 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:46 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:47.403 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:47.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:47.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:47.478 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:47.509 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:47.540 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:47.587 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:47.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:47.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:47.663 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:47 smithi138 ceph-mon[185886]: failed to encode map e570 with expected crc 2023-10-01T09:48:47.663 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:47 smithi138 ceph-mon[185886]: pgmap v1713: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:47.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:47 smithi138 ceph-mon[192513]: failed to encode map e570 with expected crc 2023-10-01T09:48:47.663 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:47 smithi138 ceph-mon[192513]: pgmap v1713: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:47.692 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:47.721 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:47.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:47 smithi190 ceph-mon[160435]: failed to encode map e570 with expected crc 2023-10-01T09:48:47.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:47 smithi190 ceph-mon[160435]: pgmap v1713: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:47.816 DEBUG:teuthology.orchestra.run.smithi138:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-01T09:48:47.819 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:47.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:47.835 INFO:tasks.workunit.client.0.smithi138.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-01T09:48:47.837 INFO:tasks.workunit.client.0.smithi138.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-01T09:48:47.837 INFO:tasks.workunit.client.0.smithi138.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-01T09:48:47.879 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:48:47.879 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:48:47.888 INFO:tasks.workunit.client.0.smithi138.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-01T09:48:47.924 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:47.952 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:47.957 INFO:tasks.workunit.client.0.smithi138.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-01T09:48:48.000 INFO:tasks.workunit.client.0.smithi138.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-01T09:48:48.002 INFO:tasks.workunit.client.0.smithi138.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-01T09:48:48.002 INFO:tasks.workunit.client.0.smithi138.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-01T09:48:48.044 INFO:tasks.workunit.client.0.smithi138.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-01T09:48:48.047 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2023-10-01T09:48:48.048 DEBUG:teuthology.orchestra.run.smithi138:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-01T09:48:48.105 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-01T09:48:48.106 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-01T09:48:48.106 DEBUG:teuthology.orchestra.run.smithi138:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-01T09:48:48.150 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:48.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:48.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:48.166 INFO:tasks.workunit.client.0.smithi138.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-01T09:48:48.166 INFO:tasks.workunit.client.0.smithi138.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-10-01T09:48:48.166 INFO:tasks.workunit.client.0.smithi138.stderr:+ TMPDIR=/tmp/rbd_import_export_726287 2023-10-01T09:48:48.166 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm -rf /tmp/rbd_import_export_726287 2023-10-01T09:48:48.167 INFO:tasks.workunit.client.0.smithi138.stderr:+ mkdir /tmp/rbd_import_export_726287 2023-10-01T09:48:48.167 INFO:tasks.workunit.client.0.smithi138.stderr:+ trap 'rm -rf /tmp/rbd_import_export_726287' INT TERM EXIT 2023-10-01T09:48:48.167 INFO:tasks.workunit.client.0.smithi138.stderr:+ mkdir foo.726287 2023-10-01T09:48:48.168 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import foo.726287 foo.dir 2023-10-01T09:48:48.255 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:48.268 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: cannot import a directory 2023-10-01T09:48:48.268 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 0% complete...failed. 2023-10-01T09:48:48.268 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: import failed: (21) Is a directory 2023-10-01T09:48:48.271 INFO:tasks.workunit.client.0.smithi138.stderr:+ true 2023-10-01T09:48:48.271 INFO:tasks.workunit.client.0.smithi138.stderr:+ rmdir foo.726287 2023-10-01T09:48:48.272 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_726287/img bs=1k count=1 seek=10 2023-10-01T09:48:48.273 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records in 2023-10-01T09:48:48.273 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records out 2023-10-01T09:48:48.274 INFO:tasks.workunit.client.0.smithi138.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 5.2306e-05 s, 19.6 MB/s 2023-10-01T09:48:48.274 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_726287/img bs=1k count=10 seek=100 2023-10-01T09:48:48.274 INFO:tasks.workunit.client.0.smithi138.stderr:10+0 records in 2023-10-01T09:48:48.274 INFO:tasks.workunit.client.0.smithi138.stderr:10+0 records out 2023-10-01T09:48:48.274 INFO:tasks.workunit.client.0.smithi138.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.3657e-05 s, 191 MB/s 2023-10-01T09:48:48.274 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_726287/img bs=1k count=100 seek=1000 2023-10-01T09:48:48.276 INFO:tasks.workunit.client.0.smithi138.stderr:70+1 records in 2023-10-01T09:48:48.276 INFO:tasks.workunit.client.0.smithi138.stderr:70+1 records out 2023-10-01T09:48:48.276 INFO:tasks.workunit.client.0.smithi138.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.00018244 s, 395 MB/s 2023-10-01T09:48:48.276 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_726287/img bs=1k seek=10000 2023-10-01T09:48:48.276 INFO:tasks.workunit.client.0.smithi138.stderr:139+1 records in 2023-10-01T09:48:48.276 INFO:tasks.workunit.client.0.smithi138.stderr:139+1 records out 2023-10-01T09:48:48.277 INFO:tasks.workunit.client.0.smithi138.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000309411 s, 463 MB/s 2023-10-01T09:48:48.277 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_726287/img bs=1k seek=100000 2023-10-01T09:48:48.277 INFO:tasks.workunit.client.0.smithi138.stderr:70+1 records in 2023-10-01T09:48:48.277 INFO:tasks.workunit.client.0.smithi138.stderr:70+1 records out 2023-10-01T09:48:48.278 INFO:tasks.workunit.client.0.smithi138.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.000182733 s, 395 MB/s 2023-10-01T09:48:48.278 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_726287/img bs=1k seek=1000000 2023-10-01T09:48:48.278 INFO:tasks.workunit.client.0.smithi138.stderr:189+1 records in 2023-10-01T09:48:48.278 INFO:tasks.workunit.client.0.smithi138.stderr:189+1 records out 2023-10-01T09:48:48.278 INFO:tasks.workunit.client.0.smithi138.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000495044 s, 391 MB/s 2023-10-01T09:48:48.278 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm testimg 2023-10-01T09:48:48.287 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:48.325 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 0% complete...failed. 2023-10-01T09:48:48.326 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: delete error: (2) No such file or directory 2023-10-01T09:48:48.330 INFO:tasks.workunit.client.0.smithi138.stderr:+ true 2023-10-01T09:48:48.330 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format /tmp/rbd_import_export_726287/img testimg 2023-10-01T09:48:48.443 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:48:48.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:48.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:48:48.551 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:48:48.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:48 smithi138 ceph-mon[185886]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-01T09:48:48.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:48 smithi138 ceph-mon[192513]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-01T09:48:48.583 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:48.631 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:48:48.634 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:48 smithi190 ceph-mon[160435]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-10-01T09:48:48.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:48.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:48:48.736 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:48.769 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:48.950 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-01T09:48:48.950 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:48:48.953 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export testimg /tmp/rbd_import_export_726287/img2 2023-10-01T09:48:49.216 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:48:49.217 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:48:49.217 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:48:49.217 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:48:49.217 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:48:49.217 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:48:49.217 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:48:49.217 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:48:49.217 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:48:49.217 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:48:49.217 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:48:49.218 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:48:49.218 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:48:49.218 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:48:49.218 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:48:49.218 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:48:49.242 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-01T09:48:49.245 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export testimg - 2023-10-01T09:48:49.272 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:49.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:49.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:49.376 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:49.408 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:49.456 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:49.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:49.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:49.563 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:49.569 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:49 smithi138 ceph-mon[192513]: pgmap v1714: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:49.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:49 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/938245016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:49.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:49 smithi138 ceph-mon[185886]: pgmap v1714: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:49.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:49 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/938245016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:49.590 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:49.629 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:49 smithi190 ceph-mon[160435]: pgmap v1714: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:48:49.629 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:49 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/938245016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:48:49.645 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:49.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:49.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:49.750 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:48:49.777 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:49.822 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:48:49.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:49.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:48:49.927 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:49.958 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:50.009 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:50.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:50.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:50.048 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-01T09:48:50.051 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm testimg 2023-10-01T09:48:50.116 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:48:50.137 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-01T09:48:50.139 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp /tmp/rbd_import_export_726287/img /tmp/rbd_import_export_726287/img2 2023-10-01T09:48:50.145 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:50.186 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:48:50.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:50.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:48:50.238 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 3] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:48:50.238 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:48:50.238 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-01T09:48:50.238 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:48:50.293 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:50.321 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:50.367 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:50.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:50.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:50.472 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:50.500 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:50.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:50 smithi138 ceph-mon[185886]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-10-01T09:48:50.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:50 smithi138 ceph-mon[192513]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-10-01T09:48:50.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:50 smithi190 ceph-mon[160435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-10-01T09:48:50.905 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp /tmp/rbd_import_export_726287/img /tmp/rbd_import_export_726287/img3 2023-10-01T09:48:51.350 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/img2 /tmp/rbd_import_export_726287/img3 2023-10-01T09:48:51.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:51 smithi190 ceph-mon[160435]: pgmap v1715: 73 pgs: 73 active+clean; 5.3 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 46 KiB/s rd, 25 KiB/s wr, 28 op/s 2023-10-01T09:48:51.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:51 smithi138 ceph-mon[192513]: pgmap v1715: 73 pgs: 73 active+clean; 5.3 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 46 KiB/s rd, 25 KiB/s wr, 28 op/s 2023-10-01T09:48:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:51 smithi138 ceph-mon[185886]: pgmap v1715: 73 pgs: 73 active+clean; 5.3 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 46 KiB/s rd, 25 KiB/s wr, 28 op/s 2023-10-01T09:48:52.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:52 smithi190 ceph-mon[160435]: pgmap v1716: 73 pgs: 73 active+clean; 5.3 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 121 KiB/s rd, 62 KiB/s wr, 31 op/s 2023-10-01T09:48:52.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:52 smithi138 ceph-mon[192513]: pgmap v1716: 73 pgs: 73 active+clean; 5.3 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 121 KiB/s rd, 62 KiB/s wr, 31 op/s 2023-10-01T09:48:52.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:52 smithi138 ceph-mon[185886]: pgmap v1716: 73 pgs: 73 active+clean; 5.3 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 121 KiB/s rd, 62 KiB/s wr, 31 op/s 2023-10-01T09:48:55.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:48:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:48:54] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.33.4" 2023-10-01T09:48:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:55 smithi190 ceph-mon[160435]: pgmap v1717: 73 pgs: 73 active+clean; 5.3 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 97 KiB/s rd, 50 KiB/s wr, 25 op/s 2023-10-01T09:48:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:48:55.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:55 smithi138 ceph-mon[192513]: pgmap v1717: 73 pgs: 73 active+clean; 5.3 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 97 KiB/s rd, 50 KiB/s wr, 25 op/s 2023-10-01T09:48:55.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:55.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:48:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:55 smithi138 ceph-mon[185886]: pgmap v1717: 73 pgs: 73 active+clean; 5.3 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 97 KiB/s rd, 50 KiB/s wr, 25 op/s 2023-10-01T09:48:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:48:57.368 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format - testimg 2023-10-01T09:48:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:57 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:57.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:57 smithi190 ceph-mon[160435]: pgmap v1718: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 108 KiB/s rd, 46 KiB/s wr, 27 op/s 2023-10-01T09:48:57.569 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:57 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:57.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:57 smithi138 ceph-mon[185886]: pgmap v1718: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 108 KiB/s rd, 46 KiB/s wr, 27 op/s 2023-10-01T09:48:57.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:57 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:48:57.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:57 smithi138 ceph-mon[192513]: pgmap v1718: 73 pgs: 73 active+clean; 4.6 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 108 KiB/s rd, 46 KiB/s wr, 27 op/s 2023-10-01T09:48:57.802 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:48:57.802 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:48:57.805 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export testimg /tmp/rbd_import_export_726287/img2 2023-10-01T09:48:58.130 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-01T09:48:58.133 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export testimg - 2023-10-01T09:48:58.282 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:58.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:58.295 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T09:48:58.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:58.397 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:48:58.430 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-10-01T09:48:58.434 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-10-01T09:48:58.434 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:58.540 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:48:58.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:58.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:48:58.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-10-01T09:48:58.645 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:58.677 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:58.681 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:58.831 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:58.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:58.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:58.936 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:48:58.974 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:58.997 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-01T09:48:59.001 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm testimg 2023-10-01T09:48:59.028 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:48:59.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:59.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:48:59.101 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-01T09:48:59.102 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp /tmp/rbd_import_export_726287/img /tmp/rbd_import_export_726287/img2 2023-10-01T09:48:59.134 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:48:59.149 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:48:59.180 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:59.180 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:59 smithi138 ceph-mon[185886]: failed to encode map e571 with expected crc 2023-10-01T09:48:59.180 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-10-01T09:48:59.180 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:59 smithi138 ceph-mon[185886]: osdmap e571: 8 total, 8 up, 7 in 2023-10-01T09:48:59.180 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:59 smithi138 ceph-mon[185886]: failed to encode map e571 with expected crc 2023-10-01T09:48:59.180 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:59 smithi138 ceph-mon[185886]: pgmap v1720: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-10-01T09:48:59.180 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/196232879' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:48:59.180 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:48:59 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:48:59.181 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is set 2023-10-01T09:48:59.231 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:48:59.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:59.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:48:59.336 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:48:59.368 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:59.520 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:48:59.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:48:59.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:48:59.524 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:59 smithi138 ceph-mon[192513]: failed to encode map e571 with expected crc 2023-10-01T09:48:59.524 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-10-01T09:48:59.524 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:59 smithi138 ceph-mon[192513]: osdmap e571: 8 total, 8 up, 7 in 2023-10-01T09:48:59.524 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:59 smithi138 ceph-mon[192513]: failed to encode map e571 with expected crc 2023-10-01T09:48:59.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:59 smithi138 ceph-mon[192513]: pgmap v1720: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-10-01T09:48:59.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/196232879' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:48:59.525 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:48:59 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:48:59.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:59 smithi190 ceph-mon[160435]: failed to encode map e571 with expected crc 2023-10-01T09:48:59.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-10-01T09:48:59.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:59 smithi190 ceph-mon[160435]: osdmap e571: 8 total, 8 up, 7 in 2023-10-01T09:48:59.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:59 smithi190 ceph-mon[160435]: failed to encode map e571 with expected crc 2023-10-01T09:48:59.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:59 smithi190 ceph-mon[160435]: pgmap v1720: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-10-01T09:48:59.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/196232879' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:48:59.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:48:59 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:48:59.624 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:48:59.654 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:48:59.773 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp /tmp/rbd_import_export_726287/img /tmp/rbd_import_export_726287/img3 2023-10-01T09:49:00.232 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/img /tmp/rbd_import_export_726287/img2 /tmp/rbd_import_export_726287/img3 2023-10-01T09:49:00.323 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:49:00.453 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:49:00.455 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:00 smithi138 ceph-mon[192513]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:49:00.455 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:00 smithi138 ceph-mon[192513]: failed to encode map e572 with expected crc 2023-10-01T09:49:00.455 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:00 smithi138 ceph-mon[192513]: failed to encode map e572 with expected crc 2023-10-01T09:49:00.455 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:00 smithi138 ceph-mon[192513]: failed to encode map e572 with expected crc 2023-10-01T09:49:00.455 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:00 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:49:00.455 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:00 smithi138 ceph-mon[192513]: osdmap e572: 8 total, 8 up, 7 in 2023-10-01T09:49:00.456 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:00 smithi138 ceph-mon[192513]: failed to encode map e572 with expected crc 2023-10-01T09:49:00.456 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:00 smithi138 ceph-mon[185886]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:49:00.456 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:00 smithi138 ceph-mon[185886]: failed to encode map e572 with expected crc 2023-10-01T09:49:00.456 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:00 smithi138 ceph-mon[185886]: failed to encode map e572 with expected crc 2023-10-01T09:49:00.456 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:00 smithi138 ceph-mon[185886]: failed to encode map e572 with expected crc 2023-10-01T09:49:00.456 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:00 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:49:00.456 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:00 smithi138 ceph-mon[185886]: osdmap e572: 8 total, 8 up, 7 in 2023-10-01T09:49:00.456 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:00 smithi138 ceph-mon[185886]: failed to encode map e572 with expected crc 2023-10-01T09:49:00.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:00.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:49:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:00 smithi190 ceph-mon[160435]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:49:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:00 smithi190 ceph-mon[160435]: failed to encode map e572 with expected crc 2023-10-01T09:49:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:00 smithi190 ceph-mon[160435]: failed to encode map e572 with expected crc 2023-10-01T09:49:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:00 smithi190 ceph-mon[160435]: failed to encode map e572 with expected crc 2023-10-01T09:49:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:00 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T09:49:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:00 smithi190 ceph-mon[160435]: osdmap e572: 8 total, 8 up, 7 in 2023-10-01T09:49:00.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:00 smithi190 ceph-mon[160435]: failed to encode map e572 with expected crc 2023-10-01T09:49:00.557 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:49:00.591 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:00.638 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:49:00.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:00.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:49:00.746 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:49:00.774 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:01 smithi190 ceph-mon[160435]: pgmap v1722: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 119 KiB/s rd, 29 KiB/s wr, 65 op/s 2023-10-01T09:49:01.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:01 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:49:01.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:01 smithi138 ceph-mon[192513]: pgmap v1722: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 119 KiB/s rd, 29 KiB/s wr, 65 op/s 2023-10-01T09:49:01.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:01 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:49:01.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:01 smithi138 ceph-mon[185886]: pgmap v1722: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 119 KiB/s rd, 29 KiB/s wr, 65 op/s 2023-10-01T09:49:01.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:01 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:49:03.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:03 smithi190 ceph-mon[160435]: pgmap v1723: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 195 KiB/s rd, 67 KiB/s wr, 68 op/s 2023-10-01T09:49:03.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:03 smithi138 ceph-mon[185886]: pgmap v1723: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 195 KiB/s rd, 67 KiB/s wr, 68 op/s 2023-10-01T09:49:03.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:03 smithi138 ceph-mon[192513]: pgmap v1723: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 195 KiB/s rd, 67 KiB/s wr, 68 op/s 2023-10-01T09:49:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:04 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:04 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:04 smithi190 ceph-mon[160435]: pgmap v1724: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 167 KiB/s rd, 66 KiB/s wr, 62 op/s 2023-10-01T09:49:05.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:04 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:05.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:04 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:05.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:04 smithi138 ceph-mon[192513]: pgmap v1724: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 167 KiB/s rd, 66 KiB/s wr, 62 op/s 2023-10-01T09:49:05.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:04 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:05.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:04 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:05.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:04 smithi138 ceph-mon[185886]: pgmap v1724: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 167 KiB/s rd, 66 KiB/s wr, 62 op/s 2023-10-01T09:49:05.071 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:49:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:49:04] "GET /metrics HTTP/1.1" 200 34038 "" "Prometheus/2.33.4" 2023-10-01T09:49:06.515 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd help export 2023-10-01T09:49:06.515 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep -q export-format 2023-10-01T09:49:06.533 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_726287/img bs=1k count=10 seek=100 2023-10-01T09:49:06.534 INFO:tasks.workunit.client.0.smithi138.stderr:10+0 records in 2023-10-01T09:49:06.534 INFO:tasks.workunit.client.0.smithi138.stderr:10+0 records out 2023-10-01T09:49:06.534 INFO:tasks.workunit.client.0.smithi138.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.7276e-05 s, 152 MB/s 2023-10-01T09:49:06.534 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format /tmp/rbd_import_export_726287/img testimg 2023-10-01T09:49:06.587 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:49:06.587 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:49:06.589 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap create testimg@snap 2023-10-01T09:49:07.480 INFO:tasks.workunit.client.0.smithi138.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-01T09:49:07.486 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd image-meta set testimg key1 value1 2023-10-01T09:49:07.532 INFO:tasks.workunit.client.0.smithi138.stderr:++ rbd image-meta list testimg 2023-10-01T09:49:07.574 INFO:tasks.workunit.client.0.smithi138.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-10-01T09:49:07.574 INFO:tasks.workunit.client.0.smithi138.stderr: 2023-10-01T09:49:07.574 INFO:tasks.workunit.client.0.smithi138.stderr:Key Value 2023-10-01T09:49:07.574 INFO:tasks.workunit.client.0.smithi138.stderr:key1 value1' 2023-10-01T09:49:07.574 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_726287/img_v2 2023-10-01T09:49:07.620 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T09:49:07.623 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_726287/img_v2 testimg_import 2023-10-01T09:49:07.682 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:49:07.684 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:07 smithi138 ceph-mon[185886]: pgmap v1725: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 162 KiB/s rd, 64 KiB/s wr, 60 op/s 2023-10-01T09:49:07.685 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:07 smithi138 ceph-mon[192513]: pgmap v1725: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 162 KiB/s rd, 64 KiB/s wr, 60 op/s 2023-10-01T09:49:07.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:07.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:49:07.787 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:49:07.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:07 smithi190 ceph-mon[160435]: pgmap v1725: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 162 KiB/s rd, 64 KiB/s wr, 60 op/s 2023-10-01T09:49:07.816 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:07.918 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:49:07.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:07.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:49:08.023 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:49:08.055 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:08.079 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:08.125 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:49:08.125 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:49:08.125 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-10-01T09:49:08.125 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:49:08.158 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:49:08.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:08.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:49:08.262 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:49:08.297 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:08.346 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:49:08.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:08.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:49:08.390 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:49:08.391 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:49:08.454 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:49:08.484 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:08.528 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:49:08.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:08.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:49:08.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:08 smithi190 ceph-mon[160435]: failed to encode map e573 with expected crc 2023-10-01T09:49:08.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:08 smithi190 ceph-mon[160435]: osdmap e573: 8 total, 8 up, 7 in 2023-10-01T09:49:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:08 smithi190 ceph-mon[160435]: failed to encode map e573 with expected crc 2023-10-01T09:49:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:08 smithi190 ceph-mon[160435]: failed to encode map e573 with expected crc 2023-10-01T09:49:08.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:08 smithi190 ceph-mon[160435]: failed to encode map e573 with expected crc 2023-10-01T09:49:08.561 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T09:49:08.564 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd info testimg_import 2023-10-01T09:49:08.596 INFO:tasks.workunit.client.0.smithi138.stdout:rbd image 'testimg_import': 2023-10-01T09:49:08.597 INFO:tasks.workunit.client.0.smithi138.stdout: size 110 KiB in 1 objects 2023-10-01T09:49:08.597 INFO:tasks.workunit.client.0.smithi138.stdout: order 22 (4 MiB objects) 2023-10-01T09:49:08.597 INFO:tasks.workunit.client.0.smithi138.stdout: snapshot_count: 1 2023-10-01T09:49:08.597 INFO:tasks.workunit.client.0.smithi138.stdout: id: 931e9ab12465 2023-10-01T09:49:08.597 INFO:tasks.workunit.client.0.smithi138.stdout: block_name_prefix: rbd_data.931e9ab12465 2023-10-01T09:49:08.597 INFO:tasks.workunit.client.0.smithi138.stdout: format: 2 2023-10-01T09:49:08.597 INFO:tasks.workunit.client.0.smithi138.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-01T09:49:08.597 INFO:tasks.workunit.client.0.smithi138.stdout: op_features: 2023-10-01T09:49:08.597 INFO:tasks.workunit.client.0.smithi138.stdout: flags: 2023-10-01T09:49:08.597 INFO:tasks.workunit.client.0.smithi138.stdout: create_timestamp: Sun Oct 1 09:49:07 2023 2023-10-01T09:49:08.597 INFO:tasks.workunit.client.0.smithi138.stdout: access_timestamp: Sun Oct 1 09:49:07 2023 2023-10-01T09:49:08.598 INFO:tasks.workunit.client.0.smithi138.stdout: modify_timestamp: Sun Oct 1 09:49:07 2023 2023-10-01T09:49:08.599 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd info testimg_import@snap 2023-10-01T09:49:08.631 INFO:tasks.workunit.client.0.smithi138.stdout:rbd image 'testimg_import': 2023-10-01T09:49:08.631 INFO:tasks.workunit.client.0.smithi138.stdout: size 110 KiB in 1 objects 2023-10-01T09:49:08.631 INFO:tasks.workunit.client.0.smithi138.stdout: order 22 (4 MiB objects) 2023-10-01T09:49:08.631 INFO:tasks.workunit.client.0.smithi138.stdout: snapshot_count: 1 2023-10-01T09:49:08.631 INFO:tasks.workunit.client.0.smithi138.stdout: id: 931e9ab12465 2023-10-01T09:49:08.631 INFO:tasks.workunit.client.0.smithi138.stdout: block_name_prefix: rbd_data.931e9ab12465 2023-10-01T09:49:08.631 INFO:tasks.workunit.client.0.smithi138.stdout: format: 2 2023-10-01T09:49:08.631 INFO:tasks.workunit.client.0.smithi138.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-01T09:49:08.631 INFO:tasks.workunit.client.0.smithi138.stdout: op_features: 2023-10-01T09:49:08.631 INFO:tasks.workunit.client.0.smithi138.stdout: flags: 2023-10-01T09:49:08.632 INFO:tasks.workunit.client.0.smithi138.stdout: create_timestamp: Sun Oct 1 09:49:07 2023 2023-10-01T09:49:08.632 INFO:tasks.workunit.client.0.smithi138.stdout: access_timestamp: Sun Oct 1 09:49:07 2023 2023-10-01T09:49:08.632 INFO:tasks.workunit.client.0.smithi138.stdout: modify_timestamp: Sun Oct 1 09:49:07 2023 2023-10-01T09:49:08.632 INFO:tasks.workunit.client.0.smithi138.stdout: protected: False 2023-10-01T09:49:08.633 INFO:tasks.workunit.client.0.smithi138.stderr:++ rbd image-meta list testimg_import 2023-10-01T09:49:08.635 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:49:08.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:08 smithi138 ceph-mon[185886]: failed to encode map e573 with expected crc 2023-10-01T09:49:08.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:08 smithi138 ceph-mon[185886]: osdmap e573: 8 total, 8 up, 7 in 2023-10-01T09:49:08.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:08 smithi138 ceph-mon[185886]: failed to encode map e573 with expected crc 2023-10-01T09:49:08.661 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:08 smithi138 ceph-mon[185886]: failed to encode map e573 with expected crc 2023-10-01T09:49:08.662 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:08 smithi138 ceph-mon[185886]: failed to encode map e573 with expected crc 2023-10-01T09:49:08.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:08 smithi138 ceph-mon[192513]: failed to encode map e573 with expected crc 2023-10-01T09:49:08.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:08 smithi138 ceph-mon[192513]: osdmap e573: 8 total, 8 up, 7 in 2023-10-01T09:49:08.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:08 smithi138 ceph-mon[192513]: failed to encode map e573 with expected crc 2023-10-01T09:49:08.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:08 smithi138 ceph-mon[192513]: failed to encode map e573 with expected crc 2023-10-01T09:49:08.662 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:08 smithi138 ceph-mon[192513]: failed to encode map e573 with expected crc 2023-10-01T09:49:08.664 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:08.672 INFO:tasks.workunit.client.0.smithi138.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-10-01T09:49:08.672 INFO:tasks.workunit.client.0.smithi138.stderr: 2023-10-01T09:49:08.672 INFO:tasks.workunit.client.0.smithi138.stderr:Key Value 2023-10-01T09:49:08.672 INFO:tasks.workunit.client.0.smithi138.stderr:key1 value1' 2023-10-01T09:49:08.672 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 'There is 1 metadatum on this image: 2023-10-01T09:49:08.672 INFO:tasks.workunit.client.0.smithi138.stderr: 2023-10-01T09:49:08.673 INFO:tasks.workunit.client.0.smithi138.stderr:Key Value 2023-10-01T09:49:08.673 INFO:tasks.workunit.client.0.smithi138.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-10-01T09:49:08.673 INFO:tasks.workunit.client.0.smithi138.stderr: 2023-10-01T09:49:08.673 INFO:tasks.workunit.client.0.smithi138.stderr:Key Value 2023-10-01T09:49:08.673 INFO:tasks.workunit.client.0.smithi138.stderr:key1 value1' ']' 2023-10-01T09:49:08.673 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export testimg_import /tmp/rbd_import_export_726287/img_import 2023-10-01T09:49:08.708 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete...done. 2023-10-01T09:49:08.711 INFO:tasks.workunit.client.0.smithi138.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_726287/img /tmp/rbd_import_export_726287/img_import 2023-10-01T09:49:08.711 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp -l /tmp/rbd_import_export_726287/img /tmp/rbd_import_export_726287/img_import 2023-10-01T09:49:08.712 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_726287/img --format %b 2023-10-01T09:49:08.713 INFO:tasks.workunit.client.0.smithi138.stderr:+ origsize=24 2023-10-01T09:49:08.713 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_726287/img_import --format %b 2023-10-01T09:49:08.714 INFO:tasks.workunit.client.0.smithi138.stderr:+ exportsize=224 2023-10-01T09:49:08.714 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=200 2023-10-01T09:49:08.714 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=200 2023-10-01T09:49:08.715 INFO:tasks.workunit.client.0.smithi138.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-01T09:49:08.715 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_726287/img_snap 2023-10-01T09:49:08.748 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete...done. 2023-10-01T09:49:08.751 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_726287/img_snap_import 2023-10-01T09:49:08.881 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete...done. 2023-10-01T09:49:08.883 INFO:tasks.workunit.client.0.smithi138.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_726287/img_snap /tmp/rbd_import_export_726287/img_snap_import 2023-10-01T09:49:08.884 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp -l /tmp/rbd_import_export_726287/img_snap /tmp/rbd_import_export_726287/img_snap_import 2023-10-01T09:49:08.884 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_726287/img_snap --format %b 2023-10-01T09:49:08.885 INFO:tasks.workunit.client.0.smithi138.stderr:+ origsize=224 2023-10-01T09:49:08.886 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_726287/img_snap_import --format %b 2023-10-01T09:49:08.886 INFO:tasks.workunit.client.0.smithi138.stderr:+ exportsize=224 2023-10-01T09:49:08.887 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:49:08.887 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:49:08.887 INFO:tasks.workunit.client.0.smithi138.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T09:49:08.887 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/img_v2 2023-10-01T09:49:08.887 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/img_import 2023-10-01T09:49:08.888 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/img_snap 2023-10-01T09:49:08.888 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/img_snap_import 2023-10-01T09:49:08.889 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap rm testimg_import@snap 2023-10-01T09:49:09.022 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T09:49:09.113 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:49:09.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:09.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:49:09.217 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:49:09.245 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:09.287 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:49:09.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:09.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:49:09.395 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:49:09.421 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:09.463 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:49:09.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:09.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:49:09.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:09 smithi190 ceph-mon[160435]: from='client.37668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:49:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:09 smithi190 ceph-mon[160435]: pgmap v1727: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 150 KiB/s rd, 57 KiB/s wr, 61 op/s 2023-10-01T09:49:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:09 smithi190 ceph-mon[160435]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:09 smithi190 ceph-mon[160435]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:09 smithi190 ceph-mon[160435]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:09 smithi190 ceph-mon[160435]: osdmap e574: 8 total, 8 up, 7 in 2023-10-01T09:49:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:09 smithi190 ceph-mon[160435]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:09 smithi190 ceph-mon[160435]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.559 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[192513]: from='client.37668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:49:09.559 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[192513]: pgmap v1727: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 150 KiB/s rd, 57 KiB/s wr, 61 op/s 2023-10-01T09:49:09.559 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[192513]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.559 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[192513]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.559 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[192513]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.559 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[192513]: osdmap e574: 8 total, 8 up, 7 in 2023-10-01T09:49:09.560 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[192513]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.560 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[192513]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.560 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[185886]: from='client.37668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:49:09.560 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[185886]: pgmap v1727: 73 pgs: 73 active+clean; 4.6 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 150 KiB/s rd, 57 KiB/s wr, 61 op/s 2023-10-01T09:49:09.560 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[185886]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.560 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[185886]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.560 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[185886]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.560 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[185886]: osdmap e574: 8 total, 8 up, 7 in 2023-10-01T09:49:09.561 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[185886]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.561 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:09 smithi138 ceph-mon[185886]: failed to encode map e574 with expected crc 2023-10-01T09:49:09.571 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:49:09.599 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:09.600 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:49:09.874 INFO:tasks.workunit.client.0.smithi138.stderr: Removing snap: 100% complete...done. 2023-10-01T09:49:09.882 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg_import 2023-10-01T09:49:09.970 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:49:09.972 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap rm testimg@snap 2023-10-01T09:49:10.413 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:49:10.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:10.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:49:10.480 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1726,"stamp":"2023-10-01T09:49:07.458995+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790211,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":7005,"num_read_kb":15237,"num_write":1232,"num_write_kb":16179,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":20349967,"num_keys_recovered":100,"num_objects_omap":17,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1311,"ondisk_log_size":1311,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":222,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5970364,"kb_used_data":803004,"kb_used_omap":39,"kb_used_meta":5167256,"kb_avail":650245700,"statfs":{"total":671965249536,"available":665851596800,"internally_reserved":0,"allocated":822276096,"data_stored":812041869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40337,"internal_metadata":5291270767},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":16,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":490,"num_read_kb":1387,"num_write":75,"num_write_kb":529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.056619"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":348,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.330803+0000","last_change":"2023-10-01T09:40:13.193799+0000","last_active":"2023-10-01T09:48:58.330803+0000","last_peered":"2023-10-01T09:48:58.330803+0000","last_clean":"2023-10-01T09:48:58.330803+0000","last_became_active":"2023-10-01T09:40:13.193434+0000","last_became_peered":"2023-10-01T09:40:13.193434+0000","last_unstale":"2023-10-01T09:48:58.330803+0000","last_undegraded":"2023-10-01T09:48:58.330803+0000","last_fullsized":"2023-10-01T09:48:58.330803+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":193,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162477+0000","last_change":"2023-10-01T09:37:22.667580+0000","last_active":"2023-10-01T09:48:58.162477+0000","last_peered":"2023-10-01T09:48:58.162477+0000","last_clean":"2023-10-01T09:48:58.162477+0000","last_became_active":"2023-10-01T09:37:22.667241+0000","last_became_peered":"2023-10-01T09:37:22.667241+0000","last_unstale":"2023-10-01T09:48:58.162477+0000","last_undegraded":"2023-10-01T09:48:58.162477+0000","last_fullsized":"2023-10-01T09:48:58.162477+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:21:23.967553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":574,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162819+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:48:58.162819+0000","last_peered":"2023-10-01T09:48:58.162819+0000","last_clean":"2023-10-01T09:48:58.162819+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:48:58.162819+0000","last_undegraded":"2023-10-01T09:48:58.162819+0000","last_fullsized":"2023-10-01T09:48:58.162819+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":633,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-01T09:48:59.238079+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:48:59.238079+0000","last_peered":"2023-10-01T09:48:59.238079+0000","last_clean":"2023-10-01T09:48:59.238079+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:48:59.238079+0000","last_undegraded":"2023-10-01T09:48:59.238079+0000","last_fullsized":"2023-10-01T09:48:59.238079+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":513,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.166057+0000","last_change":"2023-10-01T09:40:42.547196+0000","last_active":"2023-10-01T09:48:58.166057+0000","last_peered":"2023-10-01T09:48:58.166057+0000","last_clean":"2023-10-01T09:48:58.166057+0000","last_became_active":"2023-10-01T09:40:42.546875+0000","last_became_peered":"2023-10-01T09:40:42.546875+0000","last_unstale":"2023-10-01T09:48:58.166057+0000","last_undegraded":"2023-10-01T09:48:58.166057+0000","last_fullsized":"2023-10-01T09:48:58.166057+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:46.918138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":253,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162216+0000","last_change":"2023-10-01T09:35:23.610119+0000","last_active":"2023-10-01T09:48:58.162216+0000","last_peered":"2023-10-01T09:48:58.162216+0000","last_clean":"2023-10-01T09:48:58.162216+0000","last_became_active":"2023-10-01T09:35:23.609803+0000","last_became_peered":"2023-10-01T09:35:23.609803+0000","last_unstale":"2023-10-01T09:48:58.162216+0000","last_undegraded":"2023-10-01T09:48:58.162216+0000","last_fullsized":"2023-10-01T09:48:58.162216+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":629,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162305+0000","last_change":"2023-10-01T09:39:12.701997+0000","last_active":"2023-10-01T09:48:58.162305+0000","last_peered":"2023-10-01T09:48:58.162305+0000","last_clean":"2023-10-01T09:48:58.162305+0000","last_became_active":"2023-10-01T09:39:12.701704+0000","last_became_peered":"2023-10-01T09:39:12.701704+0000","last_unstale":"2023-10-01T09:48:58.162305+0000","last_undegraded":"2023-10-01T09:48:58.162305+0000","last_fullsized":"2023-10-01T09:48:58.162305+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":517,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162244+0000","last_change":"2023-10-01T09:48:45.528871+0000","last_active":"2023-10-01T09:48:58.162244+0000","last_peered":"2023-10-01T09:48:58.162244+0000","last_clean":"2023-10-01T09:48:58.162244+0000","last_became_active":"2023-10-01T09:48:45.528538+0000","last_became_peered":"2023-10-01T09:48:45.528538+0000","last_unstale":"2023-10-01T09:48:58.162244+0000","last_undegraded":"2023-10-01T09:48:58.162244+0000","last_fullsized":"2023-10-01T09:48:58.162244+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:52.935017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":284,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162178+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:48:58.162178+0000","last_peered":"2023-10-01T09:48:58.162178+0000","last_clean":"2023-10-01T09:48:58.162178+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:48:58.162178+0000","last_undegraded":"2023-10-01T09:48:58.162178+0000","last_fullsized":"2023-10-01T09:48:58.162178+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":91,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.160134+0000","last_change":"2023-10-01T09:40:47.738491+0000","last_active":"2023-10-01T09:48:58.160134+0000","last_peered":"2023-10-01T09:48:58.160134+0000","last_clean":"2023-10-01T09:48:58.160134+0000","last_became_active":"2023-10-01T09:40:47.738148+0000","last_became_peered":"2023-10-01T09:40:47.738148+0000","last_unstale":"2023-10-01T09:48:58.160134+0000","last_undegraded":"2023-10-01T09:48:58.160134+0000","last_fullsized":"2023-10-01T09:48:58.160134+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":97,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.160435+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:48:58.160435+0000","last_peered":"2023-10-01T09:48:58.160435+0000","last_clean":"2023-10-01T09:48:58.160435+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:48:58.160435+0000","last_undegraded":"2023-10-01T09:48:58.160435+0000","last_fullsized":"2023-10-01T09:48:58.160435+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":97,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162440+0000","last_change":"2023-10-01T09:40:42.543682+0000","last_active":"2023-10-01T09:48:58.162440+0000","last_peered":"2023-10-01T09:48:58.162440+0000","last_clean":"2023-10-01T09:48:58.162440+0000","last_became_active":"2023-10-01T09:40:42.543357+0000","last_became_peered":"2023-10-01T09:40:42.543357+0000","last_unstale":"2023-10-01T09:48:58.162440+0000","last_undegraded":"2023-10-01T09:48:58.162440+0000","last_fullsized":"2023-10-01T09:48:58.162440+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:41.195703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":473,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.330973+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:48:58.330973+0000","last_peered":"2023-10-01T09:48:58.330973+0000","last_clean":"2023-10-01T09:48:58.330973+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:48:58.330973+0000","last_undegraded":"2023-10-01T09:48:58.330973+0000","last_fullsized":"2023-10-01T09:48:58.330973+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":354,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.330880+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:48:58.330880+0000","last_peered":"2023-10-01T09:48:58.330880+0000","last_clean":"2023-10-01T09:48:58.330880+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:48:58.330880+0000","last_undegraded":"2023-10-01T09:48:58.330880+0000","last_fullsized":"2023-10-01T09:48:58.330880+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":690,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.160343+0000","last_change":"2023-10-01T09:40:01.591151+0000","last_active":"2023-10-01T09:48:58.160343+0000","last_peered":"2023-10-01T09:48:58.160343+0000","last_clean":"2023-10-01T09:48:58.160343+0000","last_became_active":"2023-10-01T09:40:01.499914+0000","last_became_peered":"2023-10-01T09:40:01.499914+0000","last_unstale":"2023-10-01T09:48:58.160343+0000","last_undegraded":"2023-10-01T09:48:58.160343+0000","last_fullsized":"2023-10-01T09:48:58.160343+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":340,"num_keys_recovered":100,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":349,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162150+0000","last_change":"2023-10-01T09:48:45.528897+0000","last_active":"2023-10-01T09:48:58.162150+0000","last_peered":"2023-10-01T09:48:58.162150+0000","last_clean":"2023-10-01T09:48:58.162150+0000","last_became_active":"2023-10-01T09:48:45.528559+0000","last_became_peered":"2023-10-01T09:48:45.528559+0000","last_unstale":"2023-10-01T09:48:58.162150+0000","last_undegraded":"2023-10-01T09:48:58.162150+0000","last_fullsized":"2023-10-01T09:48:58.162150+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:45:32.452712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":100,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.160265+0000","last_change":"2023-10-01T09:41:22.756773+0000","last_active":"2023-10-01T09:48:58.160265+0000","last_peered":"2023-10-01T09:48:58.160265+0000","last_clean":"2023-10-01T09:48:58.160265+0000","last_became_active":"2023-10-01T09:41:22.756285+0000","last_became_peered":"2023-10-01T09:41:22.756285+0000","last_unstale":"2023-10-01T09:48:58.160265+0000","last_undegraded":"2023-10-01T09:48:58.160265+0000","last_fullsized":"2023-10-01T09:48:58.160265+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":86,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162105+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:48:58.162105+0000","last_peered":"2023-10-01T09:48:58.162105+0000","last_clean":"2023-10-01T09:48:58.162105+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:48:58.162105+0000","last_undegraded":"2023-10-01T09:48:58.162105+0000","last_fullsized":"2023-10-01T09:48:58.162105+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":583,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.160159+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:48:58.160159+0000","last_peered":"2023-10-01T09:48:58.160159+0000","last_clean":"2023-10-01T09:48:58.160159+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:48:58.160159+0000","last_undegraded":"2023-10-01T09:48:58.160159+0000","last_fullsized":"2023-10-01T09:48:58.160159+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":596,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.166370+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:48:58.166370+0000","last_peered":"2023-10-01T09:48:58.166370+0000","last_clean":"2023-10-01T09:48:58.166370+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:48:58.166370+0000","last_undegraded":"2023-10-01T09:48:58.166370+0000","last_fullsized":"2023-10-01T09:48:58.166370+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":452,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.165969+0000","last_change":"2023-10-01T09:40:55.395778+0000","last_active":"2023-10-01T09:48:58.165969+0000","last_peered":"2023-10-01T09:48:58.165969+0000","last_clean":"2023-10-01T09:48:58.165969+0000","last_became_active":"2023-10-01T09:40:55.384602+0000","last_became_peered":"2023-10-01T09:40:55.384602+0000","last_unstale":"2023-10-01T09:48:58.165969+0000","last_undegraded":"2023-10-01T09:48:58.165969+0000","last_fullsized":"2023-10-01T09:48:58.165969+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:38:32.527433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":109,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-01T09:48:59.245292+0000","last_change":"2023-10-01T09:48:59.245292+0000","last_active":"2023-10-01T09:48:59.245292+0000","last_peered":"2023-10-01T09:48:59.245292+0000","last_clean":"2023-10-01T09:48:59.245292+0000","last_became_active":"2023-10-01T09:48:59.244922+0000","last_became_peered":"2023-10-01T09:48:59.244922+0000","last_unstale":"2023-10-01T09:48:59.245292+0000","last_undegraded":"2023-10-01T09:48:59.245292+0000","last_fullsized":"2023-10-01T09:48:59.245292+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":95,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.160563+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:48:58.160563+0000","last_peered":"2023-10-01T09:48:58.160563+0000","last_clean":"2023-10-01T09:48:58.160563+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:48:58.160563+0000","last_undegraded":"2023-10-01T09:48:58.160563+0000","last_fullsized":"2023-10-01T09:48:58.160563+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":292,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.161898+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:48:58.161898+0000","last_peered":"2023-10-01T09:48:58.161898+0000","last_clean":"2023-10-01T09:48:58.161898+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:48:58.161898+0000","last_undegraded":"2023-10-01T09:48:58.161898+0000","last_fullsized":"2023-10-01T09:48:58.161898+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":566,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.160620+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:48:58.160620+0000","last_peered":"2023-10-01T09:48:58.160620+0000","last_clean":"2023-10-01T09:48:58.160620+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:48:58.160620+0000","last_undegraded":"2023-10-01T09:48:58.160620+0000","last_fullsized":"2023-10-01T09:48:58.160620+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":629,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162743+0000","last_change":"2023-10-01T09:41:22.762837+0000","last_active":"2023-10-01T09:48:58.162743+0000","last_peered":"2023-10-01T09:48:58.162743+0000","last_clean":"2023-10-01T09:48:58.162743+0000","last_became_active":"2023-10-01T09:41:22.762494+0000","last_became_peered":"2023-10-01T09:41:22.762494+0000","last_unstale":"2023-10-01T09:48:58.162743+0000","last_undegraded":"2023-10-01T09:48:58.162743+0000","last_fullsized":"2023-10-01T09:48:58.162743+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:50:44.215108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":652,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.160511+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:48:58.160511+0000","last_peered":"2023-10-01T09:48:58.160511+0000","last_clean":"2023-10-01T09:48:58.160511+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:48:58.160511+0000","last_undegraded":"2023-10-01T09:48:58.160511+0000","last_fullsized":"2023-10-01T09:48:58.160511+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":450,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162678+0000","last_change":"2023-10-01T09:40:47.735509+0000","last_active":"2023-10-01T09:48:58.162678+0000","last_peered":"2023-10-01T09:48:58.162678+0000","last_clean":"2023-10-01T09:48:58.162678+0000","last_became_active":"2023-10-01T09:40:47.735008+0000","last_became_peered":"2023-10-01T09:40:47.735008+0000","last_unstale":"2023-10-01T09:48:58.162678+0000","last_undegraded":"2023-10-01T09:48:58.162678+0000","last_fullsized":"2023-10-01T09:48:58.162678+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:00.456262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":292,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162172+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:48:58.162172+0000","last_peered":"2023-10-01T09:48:58.162172+0000","last_clean":"2023-10-01T09:48:58.162172+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:48:58.162172+0000","last_undegraded":"2023-10-01T09:48:58.162172+0000","last_fullsized":"2023-10-01T09:48:58.162172+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":552,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.166492+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:48:58.166492+0000","last_peered":"2023-10-01T09:48:58.166492+0000","last_clean":"2023-10-01T09:48:58.166492+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:48:58.166492+0000","last_undegraded":"2023-10-01T09:48:58.166492+0000","last_fullsized":"2023-10-01T09:48:58.166492+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":152,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.331113+0000","last_change":"2023-10-01T09:40:13.196185+0000","last_active":"2023-10-01T09:48:58.331113+0000","last_peered":"2023-10-01T09:48:58.331113+0000","last_clean":"2023-10-01T09:48:58.331113+0000","last_became_active":"2023-10-01T09:40:13.195858+0000","last_became_peered":"2023-10-01T09:40:13.195858+0000","last_unstale":"2023-10-01T09:48:58.331113+0000","last_undegraded":"2023-10-01T09:48:58.331113+0000","last_fullsized":"2023-10-01T09:48:58.331113+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":226,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.331087+0000","last_change":"2023-10-01T09:37:22.680513+0000","last_active":"2023-10-01T09:48:58.331087+0000","last_peered":"2023-10-01T09:48:58.331087+0000","last_clean":"2023-10-01T09:48:58.331087+0000","last_became_active":"2023-10-01T09:37:22.680174+0000","last_became_peered":"2023-10-01T09:37:22.680174+0000","last_unstale":"2023-10-01T09:48:58.331087+0000","last_undegraded":"2023-10-01T09:48:58.331087+0000","last_fullsized":"2023-10-01T09:48:58.331087+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":470,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.330912+0000","last_change":"2023-10-01T09:40:42.562810+0000","last_active":"2023-10-01T09:48:58.330912+0000","last_peered":"2023-10-01T09:48:58.330912+0000","last_clean":"2023-10-01T09:48:58.330912+0000","last_became_active":"2023-10-01T09:40:42.562506+0000","last_became_peered":"2023-10-01T09:40:42.562506+0000","last_unstale":"2023-10-01T09:48:58.330912+0000","last_undegraded":"2023-10-01T09:48:58.330912+0000","last_fullsized":"2023-10-01T09:48:58.330912+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":633,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-01T09:48:59.245446+0000","last_change":"2023-10-01T09:40:01.500075+0000","last_active":"2023-10-01T09:48:59.245446+0000","last_peered":"2023-10-01T09:48:59.245446+0000","last_clean":"2023-10-01T09:48:59.245446+0000","last_became_active":"2023-10-01T09:40:01.499760+0000","last_became_peered":"2023-10-01T09:40:01.499760+0000","last_unstale":"2023-10-01T09:48:59.245446+0000","last_undegraded":"2023-10-01T09:48:59.245446+0000","last_fullsized":"2023-10-01T09:48:59.245446+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:46:35.232617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":466,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.330682+0000","last_change":"2023-10-01T09:40:13.195200+0000","last_active":"2023-10-01T09:48:58.330682+0000","last_peered":"2023-10-01T09:48:58.330682+0000","last_clean":"2023-10-01T09:48:58.330682+0000","last_became_active":"2023-10-01T09:40:13.194870+0000","last_became_peered":"2023-10-01T09:40:13.194870+0000","last_unstale":"2023-10-01T09:48:58.330682+0000","last_undegraded":"2023-10-01T09:48:58.330682+0000","last_fullsized":"2023-10-01T09:48:58.330682+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":307,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.331324+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:48:58.331324+0000","last_peered":"2023-10-01T09:48:58.331324+0000","last_clean":"2023-10-01T09:48:58.331324+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:48:58.331324+0000","last_undegraded":"2023-10-01T09:48:58.331324+0000","last_fullsized":"2023-10-01T09:48:58.331324+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":437,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162236+0000","last_change":"2023-10-01T09:41:22.760008+0000","last_active":"2023-10-01T09:48:58.162236+0000","last_peered":"2023-10-01T09:48:58.162236+0000","last_clean":"2023-10-01T09:48:58.162236+0000","last_became_active":"2023-10-01T09:41:22.759648+0000","last_became_peered":"2023-10-01T09:41:22.759648+0000","last_unstale":"2023-10-01T09:48:58.162236+0000","last_undegraded":"2023-10-01T09:48:58.162236+0000","last_fullsized":"2023-10-01T09:48:58.162236+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:13:51.641873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":484,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162594+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:48:58.162594+0000","last_peered":"2023-10-01T09:48:58.162594+0000","last_clean":"2023-10-01T09:48:58.162594+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:48:58.162594+0000","last_undegraded":"2023-10-01T09:48:58.162594+0000","last_fullsized":"2023-10-01T09:48:58.162594+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"571'242","reported_seq":1363,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:59.097843+0000","last_change":"2023-10-01T09:48:45.360950+0000","last_active":"2023-10-01T09:48:59.097843+0000","last_peered":"2023-10-01T09:48:59.097843+0000","last_clean":"2023-10-01T09:48:59.097843+0000","last_became_active":"2023-10-01T09:48:45.360357+0000","last_became_peered":"2023-10-01T09:48:45.360357+0000","last_unstale":"2023-10-01T09:48:59.097843+0000","last_undegraded":"2023-10-01T09:48:59.097843+0000","last_fullsized":"2023-10-01T09:48:59.097843+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":242,"log_dups_size":0,"ondisk_log_size":242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:46.766586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1468,"num_read_kb":1929,"num_write":218,"num_write_kb":1444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":302,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162247+0000","last_change":"2023-10-01T09:40:01.499841+0000","last_active":"2023-10-01T09:48:58.162247+0000","last_peered":"2023-10-01T09:48:58.162247+0000","last_clean":"2023-10-01T09:48:58.162247+0000","last_became_active":"2023-10-01T09:40:01.499491+0000","last_became_peered":"2023-10-01T09:40:01.499491+0000","last_unstale":"2023-10-01T09:48:58.162247+0000","last_undegraded":"2023-10-01T09:48:58.162247+0000","last_fullsized":"2023-10-01T09:48:58.162247+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":98,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-01T09:48:59.244975+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:48:59.244975+0000","last_peered":"2023-10-01T09:48:59.244975+0000","last_clean":"2023-10-01T09:48:59.244975+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:48:59.244975+0000","last_undegraded":"2023-10-01T09:48:59.244975+0000","last_fullsized":"2023-10-01T09:48:59.244975+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"571'128","reported_seq":1413,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:59.096522+0000","last_change":"2023-10-01T09:33:11.676246+0000","last_active":"2023-10-01T09:48:59.096522+0000","last_peered":"2023-10-01T09:48:59.096522+0000","last_clean":"2023-10-01T09:48:59.096522+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:48:59.096522+0000","last_undegraded":"2023-10-01T09:48:59.096522+0000","last_fullsized":"2023-10-01T09:48:59.096522+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1515,"num_read_kb":1377,"num_write":90,"num_write_kb":66,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":447,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.331005+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:48:58.331005+0000","last_peered":"2023-10-01T09:48:58.331005+0000","last_clean":"2023-10-01T09:48:58.331005+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:48:58.331005+0000","last_undegraded":"2023-10-01T09:48:58.331005+0000","last_fullsized":"2023-10-01T09:48:58.331005+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":275,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162310+0000","last_change":"2023-10-01T09:41:22.756327+0000","last_active":"2023-10-01T09:48:58.162310+0000","last_peered":"2023-10-01T09:48:58.162310+0000","last_clean":"2023-10-01T09:48:58.162310+0000","last_became_active":"2023-10-01T09:41:22.755818+0000","last_became_peered":"2023-10-01T09:41:22.755818+0000","last_unstale":"2023-10-01T09:48:58.162310+0000","last_undegraded":"2023-10-01T09:48:58.162310+0000","last_fullsized":"2023-10-01T09:48:58.162310+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"572'96","reported_seq":998,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-01T09:49:06.626105+0000","last_change":"2023-10-01T09:48:25.678391+0000","last_active":"2023-10-01T09:49:06.626105+0000","last_peered":"2023-10-01T09:49:06.626105+0000","last_clean":"2023-10-01T09:49:06.626105+0000","last_became_active":"2023-10-01T09:41:22.762629+0000","last_became_peered":"2023-10-01T09:41:22.762629+0000","last_unstale":"2023-10-01T09:49:06.626105+0000","last_undegraded":"2023-10-01T09:49:06.626105+0000","last_fullsized":"2023-10-01T09:49:06.626105+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":96,"log_dups_size":0,"ondisk_log_size":96,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:37:24.687604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":468,"num_read_kb":1849,"num_write":87,"num_write_kb":1325,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":637,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162154+0000","last_change":"2023-10-01T09:40:01.496129+0000","last_active":"2023-10-01T09:48:58.162154+0000","last_peered":"2023-10-01T09:48:58.162154+0000","last_clean":"2023-10-01T09:48:58.162154+0000","last_became_active":"2023-10-01T09:40:01.495805+0000","last_became_peered":"2023-10-01T09:40:01.495805+0000","last_unstale":"2023-10-01T09:48:58.162154+0000","last_undegraded":"2023-10-01T09:48:58.162154+0000","last_fullsized":"2023-10-01T09:48:58.162154+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:29:49.408955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":97,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162527+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:48:58.162527+0000","last_peered":"2023-10-01T09:48:58.162527+0000","last_clean":"2023-10-01T09:48:58.162527+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:48:58.162527+0000","last_undegraded":"2023-10-01T09:48:58.162527+0000","last_fullsized":"2023-10-01T09:48:58.162527+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"571'263","reported_seq":1735,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:59.100362+0000","last_change":"2023-10-01T09:40:50.766880+0000","last_active":"2023-10-01T09:48:59.100362+0000","last_peered":"2023-10-01T09:48:59.100362+0000","last_clean":"2023-10-01T09:48:59.100362+0000","last_became_active":"2023-10-01T09:40:50.766281+0000","last_became_peered":"2023-10-01T09:40:50.766281+0000","last_unstale":"2023-10-01T09:48:59.100362+0000","last_undegraded":"2023-10-01T09:48:59.100362+0000","last_fullsized":"2023-10-01T09:48:59.100362+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":263,"log_dups_size":0,"ondisk_log_size":263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1485,"num_read_kb":2431,"num_write":232,"num_write_kb":696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":602,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.160114+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:48:58.160114+0000","last_peered":"2023-10-01T09:48:58.160114+0000","last_clean":"2023-10-01T09:48:58.160114+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:48:58.160114+0000","last_undegraded":"2023-10-01T09:48:58.160114+0000","last_fullsized":"2023-10-01T09:48:58.160114+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":380,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162143+0000","last_change":"2023-10-01T09:41:22.756135+0000","last_active":"2023-10-01T09:48:58.162143+0000","last_peered":"2023-10-01T09:48:58.162143+0000","last_clean":"2023-10-01T09:48:58.162143+0000","last_became_active":"2023-10-01T09:41:22.755823+0000","last_became_peered":"2023-10-01T09:41:22.755823+0000","last_unstale":"2023-10-01T09:48:58.162143+0000","last_undegraded":"2023-10-01T09:48:58.162143+0000","last_fullsized":"2023-10-01T09:48:58.162143+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":370,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.331211+0000","last_change":"2023-10-01T09:40:13.192227+0000","last_active":"2023-10-01T09:48:58.331211+0000","last_peered":"2023-10-01T09:48:58.331211+0000","last_clean":"2023-10-01T09:48:58.331211+0000","last_became_active":"2023-10-01T09:40:13.191929+0000","last_became_peered":"2023-10-01T09:40:13.191929+0000","last_unstale":"2023-10-01T09:48:58.331211+0000","last_undegraded":"2023-10-01T09:48:58.331211+0000","last_fullsized":"2023-10-01T09:48:58.331211+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"570'63","reported_seq":786,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.160222+0000","last_change":"2023-10-01T09:40:42.636367+0000","last_active":"2023-10-01T09:48:58.160222+0000","last_peered":"2023-10-01T09:48:58.160222+0000","last_clean":"2023-10-01T09:48:58.160222+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:48:58.160222+0000","last_undegraded":"2023-10-01T09:48:58.160222+0000","last_fullsized":"2023-10-01T09:48:58.160222+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":63,"log_dups_size":0,"ondisk_log_size":63,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":138,"num_read_kb":2340,"num_write":67,"num_write_kb":6268,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":614,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.166037+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:48:58.166037+0000","last_peered":"2023-10-01T09:48:58.166037+0000","last_clean":"2023-10-01T09:48:58.166037+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:48:58.166037+0000","last_undegraded":"2023-10-01T09:48:58.166037+0000","last_fullsized":"2023-10-01T09:48:58.166037+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":613,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.166194+0000","last_change":"2023-10-01T09:40:01.498702+0000","last_active":"2023-10-01T09:48:58.166194+0000","last_peered":"2023-10-01T09:48:58.166194+0000","last_clean":"2023-10-01T09:48:58.166194+0000","last_became_active":"2023-10-01T09:40:01.498393+0000","last_became_peered":"2023-10-01T09:40:01.498393+0000","last_unstale":"2023-10-01T09:48:58.166194+0000","last_undegraded":"2023-10-01T09:48:58.166194+0000","last_fullsized":"2023-10-01T09:48:58.166194+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:03:51.721295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"571'119","reported_seq":1022,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:59.089332+0000","last_change":"2023-10-01T09:32:30.560685+0000","last_active":"2023-10-01T09:48:59.089332+0000","last_peered":"2023-10-01T09:48:59.089332+0000","last_clean":"2023-10-01T09:48:59.089332+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:48:59.089332+0000","last_undegraded":"2023-10-01T09:48:59.089332+0000","last_fullsized":"2023-10-01T09:48:59.089332+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":710,"num_read_kb":1568,"num_write":115,"num_write_kb":561,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":561,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162440+0000","last_change":"2023-10-01T09:48:45.360906+0000","last_active":"2023-10-01T09:48:58.162440+0000","last_peered":"2023-10-01T09:48:58.162440+0000","last_clean":"2023-10-01T09:48:58.162440+0000","last_became_active":"2023-10-01T09:48:45.360492+0000","last_became_peered":"2023-10-01T09:48:45.360492+0000","last_unstale":"2023-10-01T09:48:58.162440+0000","last_undegraded":"2023-10-01T09:48:58.162440+0000","last_fullsized":"2023-10-01T09:48:58.162440+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:06:22.068058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":482,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162031+0000","last_change":"2023-10-01T09:39:12.701481+0000","last_active":"2023-10-01T09:48:58.162031+0000","last_peered":"2023-10-01T09:48:58.162031+0000","last_clean":"2023-10-01T09:48:58.162031+0000","last_became_active":"2023-10-01T09:39:12.701153+0000","last_became_peered":"2023-10-01T09:39:12.701153+0000","last_unstale":"2023-10-01T09:48:58.162031+0000","last_undegraded":"2023-10-01T09:48:58.162031+0000","last_fullsized":"2023-10-01T09:48:58.162031+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":511,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.161043+0000","last_change":"2023-10-01T09:37:22.701954+0000","last_active":"2023-10-01T09:48:58.161043+0000","last_peered":"2023-10-01T09:48:58.161043+0000","last_clean":"2023-10-01T09:48:58.161043+0000","last_became_active":"2023-10-01T09:37:22.682623+0000","last_became_peered":"2023-10-01T09:37:22.682623+0000","last_unstale":"2023-10-01T09:48:58.161043+0000","last_undegraded":"2023-10-01T09:48:58.161043+0000","last_fullsized":"2023-10-01T09:48:58.161043+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"571'172","reported_seq":1301,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-10-01T09:48:59.246212+0000","last_change":"2023-10-01T09:32:30.544834+0000","last_active":"2023-10-01T09:48:59.246212+0000","last_peered":"2023-10-01T09:48:59.246212+0000","last_clean":"2023-10-01T09:48:59.246212+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:48:59.246212+0000","last_undegraded":"2023-10-01T09:48:59.246212+0000","last_fullsized":"2023-10-01T09:48:59.246212+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":172,"log_dups_size":0,"ondisk_log_size":172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":861,"num_read_kb":695,"num_write":125,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":100,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.160094+0000","last_change":"2023-10-01T09:41:22.756077+0000","last_active":"2023-10-01T09:48:58.160094+0000","last_peered":"2023-10-01T09:48:58.160094+0000","last_clean":"2023-10-01T09:48:58.160094+0000","last_became_active":"2023-10-01T09:41:22.755727+0000","last_became_peered":"2023-10-01T09:41:22.755727+0000","last_unstale":"2023-10-01T09:48:58.160094+0000","last_undegraded":"2023-10-01T09:48:58.160094+0000","last_fullsized":"2023-10-01T09:48:58.160094+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"571'34","reported_seq":114,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:59.089762+0000","last_change":"2023-10-01T09:40:47.757028+0000","last_active":"2023-10-01T09:48:59.089762+0000","last_peered":"2023-10-01T09:48:59.089762+0000","last_clean":"2023-10-01T09:48:59.089762+0000","last_became_active":"2023-10-01T09:40:47.735789+0000","last_became_peered":"2023-10-01T09:40:47.735789+0000","last_unstale":"2023-10-01T09:48:59.089762+0000","last_undegraded":"2023-10-01T09:48:59.089762+0000","last_fullsized":"2023-10-01T09:48:59.089762+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":66,"num_read_kb":2548,"num_write":38,"num_write_kb":2288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":591,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.166260+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:48:58.166260+0000","last_peered":"2023-10-01T09:48:58.166260+0000","last_clean":"2023-10-01T09:48:58.166260+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:48:58.166260+0000","last_undegraded":"2023-10-01T09:48:58.166260+0000","last_fullsized":"2023-10-01T09:48:58.166260+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":295,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.161936+0000","last_change":"2023-10-01T09:33:11.655896+0000","last_active":"2023-10-01T09:48:58.161936+0000","last_peered":"2023-10-01T09:48:58.161936+0000","last_clean":"2023-10-01T09:48:58.161936+0000","last_became_active":"2023-10-01T09:33:11.655596+0000","last_became_peered":"2023-10-01T09:33:11.655596+0000","last_unstale":"2023-10-01T09:48:58.161936+0000","last_undegraded":"2023-10-01T09:48:58.161936+0000","last_fullsized":"2023-10-01T09:48:58.161936+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":540,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.166570+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:48:58.166570+0000","last_peered":"2023-10-01T09:48:58.166570+0000","last_clean":"2023-10-01T09:48:58.166570+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:48:58.166570+0000","last_undegraded":"2023-10-01T09:48:58.166570+0000","last_fullsized":"2023-10-01T09:48:58.166570+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":654,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162534+0000","last_change":"2023-10-01T09:40:01.497651+0000","last_active":"2023-10-01T09:48:58.162534+0000","last_peered":"2023-10-01T09:48:58.162534+0000","last_clean":"2023-10-01T09:48:58.162534+0000","last_became_active":"2023-10-01T09:40:01.496965+0000","last_became_peered":"2023-10-01T09:40:01.496965+0000","last_unstale":"2023-10-01T09:48:58.162534+0000","last_undegraded":"2023-10-01T09:48:58.162534+0000","last_fullsized":"2023-10-01T09:48:58.162534+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:02:15.724376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":307,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162102+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:48:58.162102+0000","last_peered":"2023-10-01T09:48:58.162102+0000","last_clean":"2023-10-01T09:48:58.162102+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:48:58.162102+0000","last_undegraded":"2023-10-01T09:48:58.162102+0000","last_fullsized":"2023-10-01T09:48:58.162102+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":97,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162150+0000","last_change":"2023-10-01T09:40:42.559279+0000","last_active":"2023-10-01T09:48:58.162150+0000","last_peered":"2023-10-01T09:48:58.162150+0000","last_clean":"2023-10-01T09:48:58.162150+0000","last_became_active":"2023-10-01T09:40:42.558971+0000","last_became_peered":"2023-10-01T09:40:42.558971+0000","last_unstale":"2023-10-01T09:48:58.162150+0000","last_undegraded":"2023-10-01T09:48:58.162150+0000","last_fullsized":"2023-10-01T09:48:58.162150+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:53.503356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":307,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162311+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:48:58.162311+0000","last_peered":"2023-10-01T09:48:58.162311+0000","last_clean":"2023-10-01T09:48:58.162311+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:48:58.162311+0000","last_undegraded":"2023-10-01T09:48:58.162311+0000","last_fullsized":"2023-10-01T09:48:58.162311+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":486,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.166055+0000","last_change":"2023-10-01T09:37:22.672522+0000","last_active":"2023-10-01T09:48:58.166055+0000","last_peered":"2023-10-01T09:48:58.166055+0000","last_clean":"2023-10-01T09:48:58.166055+0000","last_became_active":"2023-10-01T09:37:22.672214+0000","last_became_peered":"2023-10-01T09:37:22.672214+0000","last_unstale":"2023-10-01T09:48:58.166055+0000","last_undegraded":"2023-10-01T09:48:58.166055+0000","last_fullsized":"2023-10-01T09:48:58.166055+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:10.668308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":115,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.162844+0000","last_change":"2023-10-01T09:40:01.495710+0000","last_active":"2023-10-01T09:48:58.162844+0000","last_peered":"2023-10-01T09:48:58.162844+0000","last_clean":"2023-10-01T09:48:58.162844+0000","last_became_active":"2023-10-01T09:40:01.494770+0000","last_became_peered":"2023-10-01T09:40:01.494770+0000","last_unstale":"2023-10-01T09:48:58.162844+0000","last_undegraded":"2023-10-01T09:48:58.162844+0000","last_fullsized":"2023-10-01T09:48:58.162844+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:29:09.250524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":531,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.165945+0000","last_change":"2023-10-01T09:40:01.495299+0000","last_active":"2023-10-01T09:48:58.165945+0000","last_peered":"2023-10-01T09:48:58.165945+0000","last_clean":"2023-10-01T09:48:58.165945+0000","last_became_active":"2023-10-01T09:40:01.494985+0000","last_became_peered":"2023-10-01T09:40:01.494985+0000","last_unstale":"2023-10-01T09:48:58.165945+0000","last_undegraded":"2023-10-01T09:48:58.165945+0000","last_fullsized":"2023-10-01T09:48:58.165945+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:18.926557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":731,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.166625+0000","last_change":"2023-10-01T09:40:01.504815+0000","last_active":"2023-10-01T09:48:58.166625+0000","last_peered":"2023-10-01T09:48:58.166625+0000","last_clean":"2023-10-01T09:48:58.166625+0000","last_became_active":"2023-10-01T09:40:01.495741+0000","last_became_peered":"2023-10-01T09:40:01.495741+0000","last_unstale":"2023-10-01T09:48:58.166625+0000","last_undegraded":"2023-10-01T09:48:58.166625+0000","last_fullsized":"2023-10-01T09:48:58.166625+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:37:27.037791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":423,"reported_epoch":571,"state":"active+clean","last_fresh":"2023-10-01T09:48:58.330680+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:48:58.330680+0000","last_peered":"2023-10-01T09:48:58.330680+0000","last_clean":"2023-10-01T09:48:58.330680+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:48:58.330680+0000","last_undegraded":"2023-10-01T09:48:58.330680+0000","last_fullsized":"2023-10-01T09:48:58.330680+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":30486,"num_keys_recovered":100,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194339,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":6711,"num_read_kb":14737,"num_write":972,"num_write_kb":13734,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":382,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":1117,"ondisk_log_size":1117,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430782,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917052,"kb_used_data":9840,"kb_used_omap":0,"kb_used_meta":907200,"kb_avail":92828100,"statfs":{"total":95995035648,"available":95055974400,"internally_reserved":0,"allocated":10076160,"data_stored":9113220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":928972800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:49:05 2023","interfaces":[{"interface":"back","average":{"1min":8.423,"5min":2.2650000000000001,"15min":2.871},"min":{"1min":0.249,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":112.274,"5min":112.274,"15min":112.274},"last":0.46200000000000002},{"interface":"front","average":{"1min":8.1859999999999999,"5min":2.1669999999999998,"15min":2.944},"min":{"1min":0.32700000000000001,"5min":0.23300000000000001,"15min":0.21199999999999999},"max":{"1min":112.44799999999999,"5min":112.44799999999999,"15min":112.44799999999999},"last":0.63500000000000001}]},{"osd":1,"last update":"Sun Oct 1 09:48:22 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":0.99099999999999999,"15min":4.258},"min":{"1min":0.34200000000000003,"5min":0.24199999999999999,"15min":0.23300000000000001},"max":{"1min":12.522,"5min":12.522,"15min":229.161},"last":0.61399999999999999},{"interface":"front","average":{"1min":2.2690000000000001,"5min":0.89000000000000001,"15min":4.3109999999999999},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.24399999999999999},"max":{"1min":10.127000000000001,"5min":10.127000000000001,"15min":230.126},"last":0.58599999999999997}]},{"osd":2,"last update":"Sun Oct 1 09:49:01 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":1.081,"15min":1.4650000000000001},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.5670000000000002,"5min":12.565,"15min":14.901999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.163,"5min":0.80000000000000004,"15min":1.3660000000000001},"min":{"1min":0.36499999999999999,"5min":0.33200000000000002,"15min":0.27000000000000002},"max":{"1min":7.4379999999999997,"5min":7.4379999999999997,"15min":14.682},"last":0.49199999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:48:22 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":0.83699999999999997,"15min":1.8009999999999999},"min":{"1min":0.42099999999999999,"5min":0.32600000000000001,"15min":0.29199999999999998},"max":{"1min":7.4699999999999998,"5min":7.4699999999999998,"15min":76.555000000000007},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.008,"5min":0.88500000000000001,"15min":1.9039999999999999},"min":{"1min":0.33200000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.2599999999999998,"5min":9.2599999999999998,"15min":76.534000000000006},"last":0.68000000000000005}]},{"osd":4,"last update":"Sun Oct 1 09:49:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":0.94899999999999995,"15min":1.0409999999999999},"min":{"1min":0.28899999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.4889999999999999,"5min":11.069000000000001,"15min":11.069000000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.78700000000000003,"15min":0.93799999999999994},"min":{"1min":0.34200000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.5,"5min":7.5,"15min":8.4949999999999992},"last":0.71399999999999997}]},{"osd":5,"last update":"Sun Oct 1 09:48:42 2023","interfaces":[{"interface":"back","average":{"1min":2.766,"5min":1.113,"15min":1.6559999999999999},"min":{"1min":0.43099999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":13.41,"5min":13.41,"15min":135.40000000000001},"last":0.754},{"interface":"front","average":{"1min":2.2709999999999999,"5min":0.98199999999999998,"15min":1.7370000000000001},"min":{"1min":0.39000000000000001,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":11.895,"5min":11.895,"15min":135.339},"last":0.65300000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:48:42 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":0.85799999999999998,"15min":1.139},"min":{"1min":0.40200000000000002,"5min":0.38,"15min":0.313},"max":{"1min":10.417999999999999,"5min":10.417999999999999,"15min":11.853999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":2.2749999999999999,"5min":0.99299999999999999,"15min":1.1459999999999999},"min":{"1min":0.432,"5min":0.32600000000000001,"15min":0.32300000000000001},"max":{"1min":9.3550000000000004,"5min":9.3550000000000004,"15min":14.584},"last":0.60099999999999998}]}]},{"osd":6,"up_from":294,"seq":1262720385292,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":339248,"kb_used_data":5736,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":93405904,"statfs":{"total":95995035648,"available":95647645696,"internally_reserved":0,"allocated":5873664,"data_stored":4918747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:24 2023","interfaces":[{"interface":"back","average":{"1min":5.0350000000000001,"5min":2.173,"15min":2.7240000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.20599999999999999},"max":{"1min":77.837000000000003,"5min":77.837000000000003,"15min":158.48599999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":4.6109999999999998,"5min":2.3210000000000002,"15min":2.8639999999999999},"min":{"1min":0.27400000000000002,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":77.971000000000004,"5min":77.971000000000004,"15min":158.37700000000001},"last":0.31}]},{"osd":1,"last update":"Sun Oct 1 09:48:24 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":0.82099999999999995,"15min":3.7250000000000001},"min":{"1min":0.34300000000000003,"5min":0.222,"15min":0.222},"max":{"1min":6.5860000000000003,"5min":8.2409999999999997,"15min":168.947},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.2370000000000001,"5min":0.71099999999999997,"15min":3.6400000000000001},"min":{"1min":0.315,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":6.8390000000000004,"5min":6.8390000000000004,"15min":169.07900000000001},"last":0.50900000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:48:24 2023","interfaces":[{"interface":"back","average":{"1min":1.5229999999999999,"5min":0.94699999999999995,"15min":1.331},"min":{"1min":0.34200000000000003,"5min":0.254,"15min":0.254},"max":{"1min":6.6070000000000002,"5min":8.2829999999999995,"15min":9.657},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.607,"5min":0.80900000000000005,"15min":1.2949999999999999},"min":{"1min":0.33900000000000002,"5min":0.27500000000000002,"15min":0.27100000000000002},"max":{"1min":6.8159999999999998,"5min":6.8159999999999998,"15min":21.704999999999998},"last":0.36099999999999999}]},{"osd":3,"last update":"Sun Oct 1 09:48:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":0.75900000000000001,"15min":1.5960000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.3920000000000003,"5min":6.3920000000000003,"15min":89.753},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.2090000000000001,"5min":0.92400000000000004,"15min":1.657},"min":{"1min":0.39100000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.6470000000000002,"5min":8.3109999999999999,"15min":89.849999999999994},"last":0.40400000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2130000000000001,"5min":1.071,"15min":1.1850000000000001},"min":{"1min":0.254,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":12.326000000000001,"5min":12.326000000000001,"15min":63.134},"last":0.249},{"interface":"front","average":{"1min":3.9569999999999999,"5min":1.226,"15min":1.1879999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28899999999999998},"max":{"1min":13.641,"5min":13.641,"15min":62.973999999999997},"last":0.55400000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:48:38 2023","interfaces":[{"interface":"back","average":{"1min":3.0960000000000001,"5min":2.254,"15min":1.504},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":13.978999999999999,"5min":136.535,"15min":136.535},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.4300000000000002,"5min":2.1789999999999998,"15min":1.5700000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28100000000000003},"max":{"1min":12.496,"5min":136.47300000000001,"15min":136.47300000000001},"last":0.46700000000000003}]},{"osd":7,"last update":"Sun Oct 1 09:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.956,"5min":1.252,"15min":1.397},"min":{"1min":0.34899999999999998,"5min":0.32700000000000001,"15min":0.29899999999999999},"max":{"1min":15.407999999999999,"5min":15.407999999999999,"15min":94.245999999999995},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.893,"5min":1.048,"15min":1.4910000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.308},"max":{"1min":11.326000000000001,"5min":11.326000000000001,"15min":94.715000000000003},"last":0.57899999999999996}]}]},{"osd":1,"up_from":94,"seq":403726926269,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":860724,"kb_used_data":10536,"kb_used_omap":7,"kb_used_meta":850168,"kb_avail":92884428,"statfs":{"total":95995035648,"available":95113654272,"internally_reserved":0,"allocated":10788864,"data_stored":8660351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":870572455},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":11.182,"5min":3.2109999999999999,"15min":4.0659999999999998},"min":{"1min":0.217,"5min":0.186,"15min":0.182},"max":{"1min":114.33499999999999,"5min":114.33499999999999,"15min":154.21299999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":11.305,"5min":3.2799999999999998,"15min":3.637},"min":{"1min":0.16800000000000001,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":114.099,"5min":114.099,"15min":114.099},"last":0.51100000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:48:49 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":0.60199999999999998,"15min":1.49},"min":{"1min":0.22,"5min":0.22,"15min":0.20000000000000001},"max":{"1min":9.2690000000000001,"5min":9.2690000000000001,"15min":22.891999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.72999999999999998,"15min":2.2000000000000002},"min":{"1min":0.23699999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.7969999999999997,"5min":12.018000000000001,"15min":154.197},"last":0.46700000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:48:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5309999999999999,"5min":0.66900000000000004,"15min":2.5379999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.219},"max":{"1min":8.4320000000000004,"5min":8.4320000000000004,"15min":95.305000000000007},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.143,"5min":0.624,"15min":3.1499999999999999},"min":{"1min":0.27100000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.8559999999999999,"5min":4.8559999999999999,"15min":154.483},"last":0.49199999999999999}]},{"osd":4,"last update":"Sun Oct 1 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":3.839,"5min":1.2310000000000001,"15min":2.2269999999999999},"min":{"1min":0.27100000000000002,"5min":0.245,"15min":0.245},"max":{"1min":24.539999999999999,"5min":24.539999999999999,"15min":154.101},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.222,"5min":0.61899999999999999,"15min":2.0339999999999998},"min":{"1min":0.29999999999999999,"5min":0.29499999999999998,"15min":0.27700000000000002},"max":{"1min":4.9139999999999997,"5min":4.9139999999999997,"15min":154.56999999999999},"last":0.55100000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:48:49 2023","interfaces":[{"interface":"back","average":{"1min":3.1699999999999999,"5min":1.871,"15min":2.419},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":15.942,"5min":59.561,"15min":154.22999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.7160000000000002,"5min":1.742,"15min":2.3700000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.307},"max":{"1min":24.405000000000001,"5min":59.536000000000001,"15min":154.54400000000001},"last":0.59199999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:48:15 2023","interfaces":[{"interface":"back","average":{"1min":3.8490000000000002,"5min":1.304,"15min":2.3330000000000002},"min":{"1min":0.34000000000000002,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":24.321000000000002,"5min":24.321000000000002,"15min":154.12200000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":2.7280000000000002,"5min":1.079,"15min":2.2349999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.2149999999999999,"5min":9.2149999999999999,"15min":154.16999999999999},"last":0.60499999999999998}]},{"osd":7,"last update":"Sun Oct 1 09:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":1.1439999999999999,"15min":2.3639999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":15.009,"5min":15.009,"15min":154.20699999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.7689999999999999,"5min":0.95599999999999996,"15min":2.3799999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.34300000000000003},"max":{"1min":6.6180000000000003,"5min":6.6180000000000003,"15min":154.51599999999999},"last":0.628}]}]},{"osd":0,"up_from":89,"seq":382252089804,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1459468,"kb_used_data":510084,"kb_used_omap":8,"kb_used_meta":949367,"kb_avail":92285684,"statfs":{"total":95995035648,"available":94500540416,"internally_reserved":0,"allocated":522326016,"data_stored":520238562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":972152770},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":4.4480000000000004,"5min":1.216,"15min":3.7320000000000002},"min":{"1min":0.219,"5min":0.18099999999999999,"15min":0.17799999999999999},"max":{"1min":90.185000000000002,"5min":90.185000000000002,"15min":216.821},"last":0.36699999999999999},{"interface":"front","average":{"1min":5.6059999999999999,"5min":1.45,"15min":4.3899999999999997},"min":{"1min":0.22,"5min":0.20599999999999999,"15min":0.19600000000000001},"max":{"1min":90.102999999999994,"5min":90.102999999999994,"15min":217.02699999999999},"last":0.39000000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:48:53 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.60199999999999998,"15min":1.655},"min":{"1min":0.223,"5min":0.223,"15min":0.21299999999999999},"max":{"1min":4.5819999999999999,"5min":4.5819999999999999,"15min":19.788},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.5720000000000001,"5min":0.78100000000000003,"15min":1.679},"min":{"1min":0.30599999999999999,"5min":0.27500000000000002,"15min":0.215},"max":{"1min":10.901,"5min":10.901,"15min":25.844999999999999},"last":0.45100000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:48:19 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.54900000000000004,"15min":2.516},"min":{"1min":0.245,"5min":0.22,"15min":0.22},"max":{"1min":3.3849999999999998,"5min":6.5670000000000002,"15min":222.16499999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.502,"5min":0.80900000000000005,"15min":2.5710000000000002},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.0650000000000004,"5min":4.9199999999999999,"15min":222.227},"last":0.58499999999999996}]},{"osd":4,"last update":"Sun Oct 1 09:49:06 2023","interfaces":[{"interface":"back","average":{"1min":2.1579999999999999,"5min":2.1899999999999999,"15min":1.952},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":11.566000000000001,"5min":138.84100000000001,"15min":138.84100000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":1.3200000000000001,"5min":1.9590000000000001,"15min":1.994},"min":{"1min":0.36099999999999999,"5min":0.29199999999999998,"15min":0.28699999999999998},"max":{"1min":4.5839999999999996,"5min":135.94800000000001,"15min":135.94800000000001},"last":0.63900000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:48:53 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.355,"15min":2.081},"min":{"1min":0.39200000000000002,"5min":0.312,"15min":0.312},"max":{"1min":8.9819999999999993,"5min":156.69999999999999,"15min":156.69999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.7529999999999999,"5min":2.105,"15min":1.929},"min":{"1min":0.39400000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":9.8670000000000009,"5min":156.773,"15min":156.773},"last":0.65100000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:48:38 2023","interfaces":[{"interface":"back","average":{"1min":7.9500000000000002,"5min":2.1230000000000002,"15min":1.8939999999999999},"min":{"1min":0.45100000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":135.321,"5min":135.321,"15min":135.321},"last":0.67500000000000004},{"interface":"front","average":{"1min":8.4870000000000001,"5min":2.1890000000000001,"15min":1.986},"min":{"1min":0.41999999999999998,"5min":0.35899999999999999,"15min":0.32400000000000001},"max":{"1min":135.29900000000001,"5min":135.29900000000001,"15min":135.29900000000001},"last":0.56399999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":0.96399999999999997,"15min":1.698},"min":{"1min":0.499,"5min":0.374,"15min":0.374},"max":{"1min":8.6829999999999998,"5min":8.6829999999999998,"15min":48.895000000000003},"last":0.62},{"interface":"front","average":{"1min":2.1709999999999998,"5min":1.27,"15min":1.8580000000000001},"min":{"1min":0.44900000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.82,"5min":10.82,"15min":48.274000000000001},"last":0.60799999999999998}]}]},{"osd":2,"up_from":0,"seq":425201762743,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":104,"seq":446676599216,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":870656,"kb_used_data":6520,"kb_used_omap":3,"kb_used_meta":864124,"kb_avail":92874496,"statfs":{"total":95995035648,"available":95103483904,"internally_reserved":0,"allocated":6676480,"data_stored":4474969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":884863719},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":5.0350000000000001,"5min":1.264,"15min":3.4929999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17899999999999999},"max":{"1min":74.707999999999998,"5min":74.707999999999998,"15min":137.34399999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":5.5439999999999996,"5min":1.4099999999999999,"15min":3.577},"min":{"1min":0.217,"5min":0.20599999999999999,"15min":0.19800000000000001},"max":{"1min":74.555000000000007,"5min":74.555000000000007,"15min":137.73400000000001},"last":0.39900000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":0.52400000000000002,"15min":4.25},"min":{"1min":0.23699999999999999,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":5.9009999999999998,"5min":5.9009999999999998,"15min":162.22999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.47399999999999998,"15min":4.2800000000000002},"min":{"1min":0.23300000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.008,"5min":4.008,"15min":162.70699999999999},"last":0.36699999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.501,"15min":1.508},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.22800000000000001},"max":{"1min":3.5059999999999998,"5min":3.5059999999999998,"15min":14.254},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.60299999999999998,"15min":1.3029999999999999},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.20100000000000001},"max":{"1min":3.9809999999999999,"5min":4.5049999999999999,"15min":15.523},"last":0.41699999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":1.591,"5min":0.76700000000000002,"15min":1.639},"min":{"1min":0.312,"5min":0.312,"15min":0.26200000000000001},"max":{"1min":6.2370000000000001,"5min":6.2370000000000001,"15min":77.793000000000006},"last":0.52400000000000002},{"interface":"front","average":{"1min":2.5219999999999998,"5min":0.91300000000000003,"15min":1.8680000000000001},"min":{"1min":0.35499999999999998,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":9.7370000000000001,"5min":9.7370000000000001,"15min":76.646000000000001},"last":0.441}]},{"osd":5,"last update":"Sun Oct 1 09:48:41 2023","interfaces":[{"interface":"back","average":{"1min":4.5309999999999997,"5min":1.4159999999999999,"15min":1.758},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.27500000000000002},"max":{"1min":46.256999999999998,"5min":46.256999999999998,"15min":46.256999999999998},"last":0.47499999999999998},{"interface":"front","average":{"1min":4.4279999999999999,"5min":1.268,"15min":1.861},"min":{"1min":0.307,"5min":0.307,"15min":0.27700000000000002},"max":{"1min":46.284999999999997,"5min":46.284999999999997,"15min":46.284999999999997},"last":0.51100000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:48:35 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":0.82799999999999996,"15min":1.772},"min":{"1min":0.40000000000000002,"5min":0.33900000000000002,"15min":0.315},"max":{"1min":8.3230000000000004,"5min":8.3230000000000004,"15min":73.887},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.5780000000000001,"5min":0.83999999999999997,"15min":1.8640000000000001},"min":{"1min":0.39500000000000002,"5min":0.379,"15min":0.33600000000000002},"max":{"1min":4.2169999999999996,"5min":4.2169999999999996,"15min":74.016999999999996},"last":0.48399999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:48:13 2023","interfaces":[{"interface":"back","average":{"1min":2.194,"5min":1.1579999999999999,"15min":1.8180000000000001},"min":{"1min":0.47499999999999998,"5min":0.38900000000000001,"15min":0.30199999999999999},"max":{"1min":10.574,"5min":10.574,"15min":13.085000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.3330000000000002,"5min":1.1279999999999999,"15min":1.5860000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.311},"max":{"1min":10.609,"5min":10.609,"15min":13.191000000000001},"last":0.501}]}]},{"osd":4,"up_from":30,"seq":128849019467,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1149424,"kb_used_data":255144,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":92595728,"statfs":{"total":95995035648,"available":94818025472,"internally_reserved":0,"allocated":261267456,"data_stored":260312094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:49:05 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":2.5270000000000001,"15min":2.6269999999999998},"min":{"1min":0.28100000000000003,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.8789999999999996,"5min":95.548000000000002,"15min":95.548000000000002},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.78400000000000003,"5min":2.5089999999999999,"15min":2.823},"min":{"1min":0.33000000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.069,"5min":95.253,"15min":95.253},"last":0.61199999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:49:05 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.746,"15min":4.1630000000000003},"min":{"1min":0.314,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":5.2350000000000003,"5min":7.5979999999999999,"15min":165.666},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.2110000000000001,"5min":0.83699999999999997,"15min":4.1210000000000004},"min":{"1min":0.33900000000000002,"5min":0.255,"15min":0.255},"max":{"1min":13.023999999999999,"5min":13.023999999999999,"15min":165.70500000000001},"last":0.56499999999999995}]},{"osd":2,"last update":"Sun Oct 1 09:49:02 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":0.84199999999999997,"15min":1.28},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.149,"5min":7.6390000000000002,"15min":10.161},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.80400000000000005,"15min":1.4330000000000001},"min":{"1min":0.34300000000000003,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.5579999999999998,"5min":7.6619999999999999,"15min":12.045},"last":0.63300000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:49:05 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.85299999999999998,"15min":1.823},"min":{"1min":0.313,"5min":0.28599999999999998,"15min":0.27400000000000002},"max":{"1min":6.9119999999999999,"5min":11.667,"15min":140.881},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.21,"5min":0.90700000000000003,"15min":1.9159999999999999},"min":{"1min":0.39500000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":13.176,"5min":13.176,"15min":140.911},"last":0.439}]},{"osd":5,"last update":"Sun Oct 1 09:48:46 2023","interfaces":[{"interface":"back","average":{"1min":4.2519999999999998,"5min":1.7649999999999999,"15min":1.7849999999999999},"min":{"1min":0.30499999999999999,"5min":0.27200000000000002,"15min":0.26600000000000001},"max":{"1min":37.548000000000002,"5min":37.548000000000002,"15min":110.003},"last":0.34899999999999998},{"interface":"front","average":{"1min":4.5259999999999998,"5min":1.708,"15min":1.718},"min":{"1min":0.36199999999999999,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":37.567,"5min":37.567,"15min":108.56399999999999},"last":0.51900000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":0.84499999999999997,"15min":0.92000000000000004},"min":{"1min":0.35199999999999998,"5min":0.318,"15min":0.27700000000000002},"max":{"1min":11.635999999999999,"5min":11.635999999999999,"15min":11.635999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.9590000000000001,"5min":0.88300000000000001,"15min":1.079},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":7.7149999999999999,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":0.48899999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:49:02 2023","interfaces":[{"interface":"back","average":{"1min":1.5569999999999999,"5min":1.105,"15min":1.171},"min":{"1min":0.35499999999999998,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":8.4399999999999995,"5min":11.694000000000001,"15min":38.034999999999997},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.127,"15min":1.4259999999999999},"min":{"1min":0.39900000000000002,"5min":0.35599999999999998,"15min":0.34200000000000003},"max":{"1min":6.5199999999999996,"5min":12.228,"15min":37.723999999999997},"last":0.54900000000000004}]}]},{"osd":5,"up_from":379,"seq":1627792605374,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":373792,"kb_used_data":5144,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":93371360,"statfs":{"total":95995035648,"available":95612272640,"internally_reserved":0,"allocated":5267456,"data_stored":4323926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":3.7240000000000002,"5min":2.3239999999999998,"15min":2.4620000000000002},"min":{"1min":0.26700000000000002,"5min":0.222,"15min":0.21099999999999999},"max":{"1min":46.908000000000001,"5min":99.609999999999999,"15min":99.609999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":7.4459999999999997,"5min":3.1219999999999999,"15min":2.7109999999999999},"min":{"1min":0.27000000000000002,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":104.01600000000001,"5min":104.01600000000001,"15min":104.01600000000001},"last":0.67300000000000004}]},{"osd":1,"last update":"Sun Oct 1 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":0.85499999999999998,"15min":5.4790000000000001},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.6820000000000004,"5min":7.6820000000000004,"15min":295.71300000000002},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.929,"5min":0.84599999999999997,"15min":5.5490000000000004},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.24199999999999999},"max":{"1min":14.859999999999999,"5min":14.859999999999999,"15min":299.59399999999999},"last":0.66400000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":0.86899999999999999,"15min":1.4530000000000001},"min":{"1min":0.30299999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.946999999999999,"5min":12.946999999999999,"15min":14.199999999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.665,"5min":0.95299999999999996,"15min":1.3600000000000001},"min":{"1min":0.32800000000000001,"5min":0.29599999999999999,"15min":0.27100000000000002},"max":{"1min":6.5609999999999999,"5min":6.5609999999999999,"15min":14.430999999999999},"last":0.51100000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.016,"15min":1.4390000000000001},"min":{"1min":0.33400000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":11.983000000000001,"5min":11.983000000000001,"15min":14.004},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.4379999999999999,"5min":0.84999999999999998,"15min":1.52},"min":{"1min":0.377,"5min":0.34300000000000003,"15min":0.32500000000000001},"max":{"1min":12.201000000000001,"5min":12.201000000000001,"15min":13.991},"last":0.70199999999999996}]},{"osd":4,"last update":"Sun Oct 1 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":2.117,"5min":1.002,"15min":1.454},"min":{"1min":0.29899999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":12.266,"5min":12.266,"15min":122.971},"last":0.39100000000000001},{"interface":"front","average":{"1min":2.0369999999999999,"5min":0.98499999999999999,"15min":1.4970000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.27700000000000002},"max":{"1min":16.277999999999999,"5min":16.277999999999999,"15min":122.89100000000001},"last":0.438}]},{"osd":6,"last update":"Sun Oct 1 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":0.96499999999999997,"15min":1.02},"min":{"1min":0.30599999999999999,"5min":0.29399999999999998,"15min":0.28299999999999997},"max":{"1min":14.396000000000001,"5min":14.396000000000001,"15min":14.396000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.3570000000000002,"5min":1.1220000000000001,"15min":1.177},"min":{"1min":0.33900000000000002,"5min":0.31,"15min":0.30499999999999999},"max":{"1min":12.157,"5min":12.157,"15min":14.193},"last":0.52300000000000002}]},{"osd":7,"last update":"Sun Oct 1 09:48:37 2023","interfaces":[{"interface":"back","average":{"1min":3.9460000000000002,"5min":1.4330000000000001,"15min":1.284},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.29699999999999999},"max":{"1min":23.02,"5min":23.02,"15min":23.02},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.4590000000000001,"5min":1.125,"15min":1.155},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":12.121,"5min":12.121,"15min":12.500999999999999},"last":0.497}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:49:10.486 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-01T09:49:10.486 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 31 2023-10-01T09:49:10.516 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:49:10.544 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:10.587 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:49:10.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:10.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:49:10.694 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:49:10.720 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:10.761 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:49:10.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:10.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:49:10.866 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:49:10.878 INFO:tasks.workunit.client.0.smithi138.stderr: Removing snap: 100% complete...done. 2023-10-01T09:49:10.891 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm testimg 2023-10-01T09:49:10.893 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:10.936 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:49:10.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:10.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:49:10.977 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:10.993 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:49:10.997 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --order 20 /tmp/rbd_import_export_726287/img testimg 2023-10-01T09:49:11.041 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:49:11.066 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:11.069 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:10 smithi138 ceph-mon[192513]: failed to encode map e575 with expected crc 2023-10-01T09:49:11.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:10 smithi138 ceph-mon[192513]: failed to encode map e575 with expected crc 2023-10-01T09:49:11.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:10 smithi138 ceph-mon[192513]: osdmap e575: 8 total, 8 up, 7 in 2023-10-01T09:49:11.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:10 smithi138 ceph-mon[192513]: failed to encode map e575 with expected crc 2023-10-01T09:49:11.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:10 smithi138 ceph-mon[192513]: failed to encode map e575 with expected crc 2023-10-01T09:49:11.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:49:11.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:10 smithi138 ceph-mon[192513]: pgmap v1730: 73 pgs: 73 active+clean; 4.8 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 23 KiB/s rd, 4.8 KiB/s wr, 27 op/s 2023-10-01T09:49:11.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:10 smithi138 ceph-mon[185886]: failed to encode map e575 with expected crc 2023-10-01T09:49:11.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:10 smithi138 ceph-mon[185886]: failed to encode map e575 with expected crc 2023-10-01T09:49:11.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:10 smithi138 ceph-mon[185886]: osdmap e575: 8 total, 8 up, 7 in 2023-10-01T09:49:11.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:10 smithi138 ceph-mon[185886]: failed to encode map e575 with expected crc 2023-10-01T09:49:11.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:10 smithi138 ceph-mon[185886]: failed to encode map e575 with expected crc 2023-10-01T09:49:11.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:49:11.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:10 smithi138 ceph-mon[185886]: pgmap v1730: 73 pgs: 73 active+clean; 4.8 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 23 KiB/s rd, 4.8 KiB/s wr, 27 op/s 2023-10-01T09:49:11.073 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:49:11.074 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T09:49:11.077 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_726287/img_v2 2023-10-01T09:49:11.097 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:49:11.108 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:49:11.112 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T09:49:11.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:11.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:49:11.115 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_726287/img_v2 testimg_import 2023-10-01T09:49:11.118 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:49:11.125 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:11.141 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:49:11.173 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:49:11.193 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T09:49:11.196 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd info testimg_import 2023-10-01T09:49:11.196 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep order 2023-10-01T09:49:11.197 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:49:11.197 INFO:tasks.workunit.client.0.smithi138.stderr:+ awk '{print $2}' 2023-10-01T09:49:11.197 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep 20 2023-10-01T09:49:11.214 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:49:11.215 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:49:11.237 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:10 smithi190 ceph-mon[160435]: failed to encode map e575 with expected crc 2023-10-01T09:49:11.238 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:10 smithi190 ceph-mon[160435]: failed to encode map e575 with expected crc 2023-10-01T09:49:11.238 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:10 smithi190 ceph-mon[160435]: osdmap e575: 8 total, 8 up, 7 in 2023-10-01T09:49:11.238 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:10 smithi190 ceph-mon[160435]: failed to encode map e575 with expected crc 2023-10-01T09:49:11.238 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:10 smithi190 ceph-mon[160435]: failed to encode map e575 with expected crc 2023-10-01T09:49:11.238 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:49:11.238 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:10 smithi190 ceph-mon[160435]: pgmap v1730: 73 pgs: 73 active+clean; 4.8 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 23 KiB/s rd, 4.8 KiB/s wr, 27 op/s 2023-10-01T09:49:11.240 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:11.240 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:49:11.241 INFO:tasks.workunit.client.0.smithi138.stdout:20 2023-10-01T09:49:11.241 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/img_v2 2023-10-01T09:49:11.241 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg_import 2023-10-01T09:49:11.261 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:49:11.268 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:49:11.268 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:49:11.268 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:49:11.268 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:49:11.268 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:49:11.268 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:49:11.268 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:49:11.268 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:49:11.281 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:49:11.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:11.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:49:11.340 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:49:11.343 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg 2023-10-01T09:49:11.388 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:49:11.426 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:11.447 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:49:11.451 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_726287/img testimg 2023-10-01T09:49:11.452 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:11.509 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:49:11.512 INFO:tasks.workunit.client.0.smithi138.stderr:++ rbd info testimg 2023-10-01T09:49:11.512 INFO:tasks.workunit.client.0.smithi138.stderr:++ grep features 2023-10-01T09:49:11.547 INFO:tasks.workunit.client.0.smithi138.stderr:+ FEATURES_BEFORE=' features: layering 2023-10-01T09:49:11.547 INFO:tasks.workunit.client.0.smithi138.stderr: op_features: ' 2023-10-01T09:49:11.547 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_726287/img_v2 2023-10-01T09:49:11.583 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T09:49:11.587 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_726287/img_v2 testimg_import 2023-10-01T09:49:11.644 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T09:49:11.648 INFO:tasks.workunit.client.0.smithi138.stderr:++ rbd info testimg_import 2023-10-01T09:49:11.648 INFO:tasks.workunit.client.0.smithi138.stderr:++ grep features 2023-10-01T09:49:11.682 INFO:tasks.workunit.client.0.smithi138.stderr:+ FEATURES_AFTER=' features: layering 2023-10-01T09:49:11.682 INFO:tasks.workunit.client.0.smithi138.stderr: op_features: ' 2023-10-01T09:49:11.682 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' ' features: layering 2023-10-01T09:49:11.682 INFO:tasks.workunit.client.0.smithi138.stderr: op_features: ' '!=' ' features: layering 2023-10-01T09:49:11.682 INFO:tasks.workunit.client.0.smithi138.stderr: op_features: ' ']' 2023-10-01T09:49:11.682 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/img_v2 2023-10-01T09:49:11.682 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg_import 2023-10-01T09:49:11.748 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:49:11.751 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg 2023-10-01T09:49:11.816 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:49:11.819 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_726287/img testimg 2023-10-01T09:49:11.870 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:11 smithi138 ceph-mon[185886]: failed to encode map e576 with expected crc 2023-10-01T09:49:11.870 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:11 smithi138 ceph-mon[185886]: failed to encode map e576 with expected crc 2023-10-01T09:49:11.870 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:11 smithi138 ceph-mon[185886]: failed to encode map e576 with expected crc 2023-10-01T09:49:11.870 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:11 smithi138 ceph-mon[185886]: osdmap e576: 8 total, 8 up, 7 in 2023-10-01T09:49:11.871 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1824213229' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:49:11.880 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:49:11.883 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_726287/img_v2 2023-10-01T09:49:11.922 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T09:49:11.924 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_726287/img_v2 testimg_import 2023-10-01T09:49:11.987 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T09:49:11.990 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd info testimg_import 2023-10-01T09:49:11.990 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep 'stripe unit' 2023-10-01T09:49:11.990 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep -Ei '(4 KiB|4096)' 2023-10-01T09:49:12.026 INFO:tasks.workunit.client.0.smithi138.stdout: stripe unit: 4 KiB 2023-10-01T09:49:12.026 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd info testimg_import 2023-10-01T09:49:12.026 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep 'stripe count' 2023-10-01T09:49:12.027 INFO:tasks.workunit.client.0.smithi138.stderr:+ awk '{print $3}' 2023-10-01T09:49:12.027 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep 1000 2023-10-01T09:49:12.060 INFO:tasks.workunit.client.0.smithi138.stdout:1000 2023-10-01T09:49:12.061 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/img_v2 2023-10-01T09:49:12.061 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg_import 2023-10-01T09:49:12.153 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-01T09:49:12.156 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg 2023-10-01T09:49:12.174 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:11 smithi138 ceph-mon[192513]: failed to encode map e576 with expected crc 2023-10-01T09:49:12.174 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:11 smithi138 ceph-mon[192513]: failed to encode map e576 with expected crc 2023-10-01T09:49:12.174 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:11 smithi138 ceph-mon[192513]: failed to encode map e576 with expected crc 2023-10-01T09:49:12.174 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:11 smithi138 ceph-mon[192513]: osdmap e576: 8 total, 8 up, 7 in 2023-10-01T09:49:12.174 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1824213229' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:49:12.258 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-01T09:49:12.261 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_726287/img testimg 2023-10-01T09:49:12.283 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:49:12.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:12.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:49:12.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:11 smithi190 ceph-mon[160435]: failed to encode map e576 with expected crc 2023-10-01T09:49:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:11 smithi190 ceph-mon[160435]: failed to encode map e576 with expected crc 2023-10-01T09:49:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:11 smithi190 ceph-mon[160435]: failed to encode map e576 with expected crc 2023-10-01T09:49:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:11 smithi190 ceph-mon[160435]: osdmap e576: 8 total, 8 up, 7 in 2023-10-01T09:49:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1824213229' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:49:12.327 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:49:12.330 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap create testimg@snap1 2023-10-01T09:49:12.388 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:49:12.414 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:12.455 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:49:12.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:12.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:49:12.562 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:49:12.591 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:12.885 INFO:tasks.workunit.client.0.smithi138.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-01T09:49:12.891 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap create testimg@snap2 2023-10-01T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:12 smithi138 ceph-mon[192513]: pgmap v1732: 73 pgs: 73 active+clean; 4.9 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 199 KiB/s rd, 18 KiB/s wr, 248 op/s 2023-10-01T09:49:13.203 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:12 smithi138 ceph-mon[185886]: pgmap v1732: 73 pgs: 73 active+clean; 4.9 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 199 KiB/s rd, 18 KiB/s wr, 248 op/s 2023-10-01T09:49:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:12 smithi190 ceph-mon[160435]: pgmap v1732: 73 pgs: 73 active+clean; 4.9 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 199 KiB/s rd, 18 KiB/s wr, 248 op/s 2023-10-01T09:49:13.894 INFO:tasks.workunit.client.0.smithi138.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-01T09:49:13.901 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap protect testimg@snap2 2023-10-01T09:49:13.948 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_726287/snap_protect 2023-10-01T09:49:13.986 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T09:49:13.989 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_726287/snap_protect testimg_import 2023-10-01T09:49:14.290 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:13 smithi138 ceph-mon[185886]: osdmap e577: 8 total, 8 up, 7 in 2023-10-01T09:49:14.291 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:13 smithi138 ceph-mon[185886]: failed to encode map e577 with expected crc 2023-10-01T09:49:14.291 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:13 smithi138 ceph-mon[185886]: failed to encode map e577 with expected crc 2023-10-01T09:49:14.291 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:13 smithi138 ceph-mon[185886]: failed to encode map e577 with expected crc 2023-10-01T09:49:14.291 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:13 smithi138 ceph-mon[185886]: failed to encode map e577 with expected crc 2023-10-01T09:49:14.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:13 smithi138 ceph-mon[192513]: osdmap e577: 8 total, 8 up, 7 in 2023-10-01T09:49:14.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:13 smithi138 ceph-mon[192513]: failed to encode map e577 with expected crc 2023-10-01T09:49:14.291 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:13 smithi138 ceph-mon[192513]: failed to encode map e577 with expected crc 2023-10-01T09:49:14.292 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:13 smithi138 ceph-mon[192513]: failed to encode map e577 with expected crc 2023-10-01T09:49:14.292 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:13 smithi138 ceph-mon[192513]: failed to encode map e577 with expected crc 2023-10-01T09:49:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:13 smithi190 ceph-mon[160435]: osdmap e577: 8 total, 8 up, 7 in 2023-10-01T09:49:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:13 smithi190 ceph-mon[160435]: failed to encode map e577 with expected crc 2023-10-01T09:49:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:13 smithi190 ceph-mon[160435]: failed to encode map e577 with expected crc 2023-10-01T09:49:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:13 smithi190 ceph-mon[160435]: failed to encode map e577 with expected crc 2023-10-01T09:49:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:13 smithi190 ceph-mon[160435]: failed to encode map e577 with expected crc 2023-10-01T09:49:14.796 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:49:14.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:14.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:49:14.891 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:49:14.907 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:49:14.940 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:14.989 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:49:14.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:14.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:49:15.070 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:49:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:49:14] "GET /metrics HTTP/1.1" 200 34037 "" "Prometheus/2.33.4" 2023-10-01T09:49:15.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:14 smithi138 ceph-mon[192513]: failed to encode map e578 with expected crc 2023-10-01T09:49:15.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:14 smithi138 ceph-mon[192513]: failed to encode map e578 with expected crc 2023-10-01T09:49:15.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:14 smithi138 ceph-mon[192513]: osdmap e578: 8 total, 8 up, 7 in 2023-10-01T09:49:15.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:14 smithi138 ceph-mon[192513]: failed to encode map e578 with expected crc 2023-10-01T09:49:15.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2457373059' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:49:15.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:14 smithi138 ceph-mon[192513]: pgmap v1735: 73 pgs: 73 active+clean; 4.9 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 199 KiB/s rd, 14 KiB/s wr, 256 op/s 2023-10-01T09:49:15.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:14 smithi138 ceph-mon[185886]: failed to encode map e578 with expected crc 2023-10-01T09:49:15.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:14 smithi138 ceph-mon[185886]: failed to encode map e578 with expected crc 2023-10-01T09:49:15.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:14 smithi138 ceph-mon[185886]: osdmap e578: 8 total, 8 up, 7 in 2023-10-01T09:49:15.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:14 smithi138 ceph-mon[185886]: failed to encode map e578 with expected crc 2023-10-01T09:49:15.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2457373059' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:49:15.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:14 smithi138 ceph-mon[185886]: pgmap v1735: 73 pgs: 73 active+clean; 4.9 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 199 KiB/s rd, 14 KiB/s wr, 256 op/s 2023-10-01T09:49:15.095 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:49:15.122 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:15.166 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:49:15.169 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:14 smithi190 ceph-mon[160435]: failed to encode map e578 with expected crc 2023-10-01T09:49:15.169 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:14 smithi190 ceph-mon[160435]: failed to encode map e578 with expected crc 2023-10-01T09:49:15.169 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:14 smithi190 ceph-mon[160435]: osdmap e578: 8 total, 8 up, 7 in 2023-10-01T09:49:15.169 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:14 smithi190 ceph-mon[160435]: failed to encode map e578 with expected crc 2023-10-01T09:49:15.169 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2457373059' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:49:15.169 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:14 smithi190 ceph-mon[160435]: pgmap v1735: 73 pgs: 73 active+clean; 4.9 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 199 KiB/s rd, 14 KiB/s wr, 256 op/s 2023-10-01T09:49:15.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:15.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:49:15.255 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T09:49:15.271 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:49:15.328 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:15.433 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:49:15.433 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:49:15.917 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T09:49:15.920 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd info testimg_import@snap1 2023-10-01T09:49:15.920 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep 'protected: False' 2023-10-01T09:49:15.958 INFO:tasks.workunit.client.0.smithi138.stdout: protected: False 2023-10-01T09:49:15.959 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd info testimg_import@snap2 2023-10-01T09:49:15.959 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep 'protected: True' 2023-10-01T09:49:15.995 INFO:tasks.workunit.client.0.smithi138.stdout: protected: True 2023-10-01T09:49:15.996 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/snap_protect 2023-10-01T09:49:15.996 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap unprotect testimg@snap2 2023-10-01T09:49:16.040 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-01T09:49:16.083 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap purge testimg 2023-10-01T09:49:16.174 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:15 smithi138 ceph-mon[185886]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:49:16.174 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:15 smithi138 ceph-mon[185886]: failed to encode map e579 with expected crc 2023-10-01T09:49:16.174 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:15 smithi138 ceph-mon[185886]: failed to encode map e579 with expected crc 2023-10-01T09:49:16.174 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:15 smithi138 ceph-mon[185886]: failed to encode map e579 with expected crc 2023-10-01T09:49:16.174 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:15 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2457373059' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:49:16.174 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:15 smithi138 ceph-mon[185886]: osdmap e579: 8 total, 8 up, 7 in 2023-10-01T09:49:16.174 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:15 smithi138 ceph-mon[192513]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:49:16.175 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:15 smithi138 ceph-mon[192513]: failed to encode map e579 with expected crc 2023-10-01T09:49:16.175 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:15 smithi138 ceph-mon[192513]: failed to encode map e579 with expected crc 2023-10-01T09:49:16.175 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:15 smithi138 ceph-mon[192513]: failed to encode map e579 with expected crc 2023-10-01T09:49:16.175 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:15 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2457373059' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:49:16.175 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:15 smithi138 ceph-mon[192513]: osdmap e579: 8 total, 8 up, 7 in 2023-10-01T09:49:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:15 smithi190 ceph-mon[160435]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T09:49:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:15 smithi190 ceph-mon[160435]: failed to encode map e579 with expected crc 2023-10-01T09:49:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:15 smithi190 ceph-mon[160435]: failed to encode map e579 with expected crc 2023-10-01T09:49:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:15 smithi190 ceph-mon[160435]: failed to encode map e579 with expected crc 2023-10-01T09:49:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:15 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2457373059' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:49:16.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:15 smithi190 ceph-mon[160435]: osdmap e579: 8 total, 8 up, 7 in 2023-10-01T09:49:16.686 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:49:16.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:16.689 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T09:49:16.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:49:16.791 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:49:16.822 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:16.871 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:49:16.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:16.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:49:16.926 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T09:49:16.952 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:16 smithi138 ceph-mon[185886]: failed to encode map e580 with expected crc 2023-10-01T09:49:16.952 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:16 smithi138 ceph-mon[185886]: osdmap e580: 8 total, 8 up, 7 in 2023-10-01T09:49:16.952 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:16 smithi138 ceph-mon[185886]: failed to encode map e580 with expected crc 2023-10-01T09:49:16.952 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:16 smithi138 ceph-mon[185886]: failed to encode map e580 with expected crc 2023-10-01T09:49:16.952 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:16 smithi138 ceph-mon[185886]: pgmap v1738: 73 pgs: 73 active+clean; 4.8 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 294 KiB/s rd, 41 KiB/s wr, 403 op/s 2023-10-01T09:49:16.952 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:16 smithi138 ceph-mon[192513]: failed to encode map e580 with expected crc 2023-10-01T09:49:16.952 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:16 smithi138 ceph-mon[192513]: osdmap e580: 8 total, 8 up, 7 in 2023-10-01T09:49:16.952 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:16 smithi138 ceph-mon[192513]: failed to encode map e580 with expected crc 2023-10-01T09:49:16.953 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:16 smithi138 ceph-mon[192513]: failed to encode map e580 with expected crc 2023-10-01T09:49:16.953 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:16 smithi138 ceph-mon[192513]: pgmap v1738: 73 pgs: 73 active+clean; 4.8 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 294 KiB/s rd, 41 KiB/s wr, 403 op/s 2023-10-01T09:49:16.978 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:49:17.005 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:17.048 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:49:17.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:17.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:49:17.154 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:49:17.182 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:17.227 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:49:17.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:17.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:49:17.250 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:17.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:16 smithi190 ceph-mon[160435]: failed to encode map e580 with expected crc 2023-10-01T09:49:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:16 smithi190 ceph-mon[160435]: osdmap e580: 8 total, 8 up, 7 in 2023-10-01T09:49:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:16 smithi190 ceph-mon[160435]: failed to encode map e580 with expected crc 2023-10-01T09:49:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:16 smithi190 ceph-mon[160435]: failed to encode map e580 with expected crc 2023-10-01T09:49:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:16 smithi190 ceph-mon[160435]: pgmap v1738: 73 pgs: 73 active+clean; 4.8 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 294 KiB/s rd, 41 KiB/s wr, 403 op/s 2023-10-01T09:49:17.331 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:49:17.357 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:17.400 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:49:17.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:17.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:49:17.506 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:49:17.532 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:17.575 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:49:17.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:17.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:49:17.681 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:49:17.709 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:17.870 INFO:tasks.workunit.client.0.smithi138.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-01T09:49:17.877 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd snap purge testimg_import 2023-10-01T09:49:18.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[192513]: failed to encode map e581 with expected crc 2023-10-01T09:49:18.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[192513]: osdmap e581: 8 total, 8 up, 7 in 2023-10-01T09:49:18.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[192513]: failed to encode map e581 with expected crc 2023-10-01T09:49:18.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[192513]: failed to encode map e581 with expected crc 2023-10-01T09:49:18.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[192513]: failed to encode map e581 with expected crc 2023-10-01T09:49:18.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1122055650' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-01T09:49:18.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-01T09:49:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:18.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[192513]: osdmap e582: 8 total, 8 up, 7 in 2023-10-01T09:49:18.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[185886]: failed to encode map e581 with expected crc 2023-10-01T09:49:18.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[185886]: osdmap e581: 8 total, 8 up, 7 in 2023-10-01T09:49:18.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[185886]: failed to encode map e581 with expected crc 2023-10-01T09:49:18.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[185886]: failed to encode map e581 with expected crc 2023-10-01T09:49:18.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[185886]: failed to encode map e581 with expected crc 2023-10-01T09:49:18.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1122055650' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-01T09:49:18.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-01T09:49:18.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:18.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:18.072 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:17 smithi138 ceph-mon[185886]: osdmap e582: 8 total, 8 up, 7 in 2023-10-01T09:49:18.157 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:49:18.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:18.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:49:18.166 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T09:49:18.262 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:49:18.286 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:17 smithi190 ceph-mon[160435]: failed to encode map e581 with expected crc 2023-10-01T09:49:18.286 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:17 smithi190 ceph-mon[160435]: osdmap e581: 8 total, 8 up, 7 in 2023-10-01T09:49:18.286 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:17 smithi190 ceph-mon[160435]: failed to encode map e581 with expected crc 2023-10-01T09:49:18.286 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:17 smithi190 ceph-mon[160435]: failed to encode map e581 with expected crc 2023-10-01T09:49:18.286 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:17 smithi190 ceph-mon[160435]: failed to encode map e581 with expected crc 2023-10-01T09:49:18.287 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:17 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1122055650' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-01T09:49:18.287 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:17 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-01T09:49:18.287 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:17 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:18.287 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:17 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:18.287 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:17 smithi190 ceph-mon[160435]: osdmap e582: 8 total, 8 up, 7 in 2023-10-01T09:49:18.288 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:18.324 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:18.331 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:49:18.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:18.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:49:18.437 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T09:49:18.462 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:18.505 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6 found: no such container 2023-10-01T09:49:18.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:18.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T09:49:18.610 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:49:18.638 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:18.755 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:49:18.911 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:49:18.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:18.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:49:19.016 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:49:19.047 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:18 smithi138 ceph-mon[192513]: failed to encode map e582 with expected crc 2023-10-01T09:49:19.047 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:18 smithi138 ceph-mon[192513]: failed to encode map e582 with expected crc 2023-10-01T09:49:19.047 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:18 smithi138 ceph-mon[192513]: failed to encode map e582 with expected crc 2023-10-01T09:49:19.047 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:18 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:49:19.047 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:18 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:49:19.047 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:18 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:19.047 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:18 smithi138 ceph-mon[192513]: pgmap v1741: 73 pgs: 73 active+clean; 4.8 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 296 KiB/s rd, 41 KiB/s wr, 410 op/s 2023-10-01T09:49:19.048 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:18 smithi138 ceph-mon[185886]: failed to encode map e582 with expected crc 2023-10-01T09:49:19.048 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:18 smithi138 ceph-mon[185886]: failed to encode map e582 with expected crc 2023-10-01T09:49:19.048 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:18 smithi138 ceph-mon[185886]: failed to encode map e582 with expected crc 2023-10-01T09:49:19.048 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:18 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:49:19.048 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:18 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:49:19.048 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:18 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:19.048 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:18 smithi138 ceph-mon[185886]: pgmap v1741: 73 pgs: 73 active+clean; 4.8 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 296 KiB/s rd, 41 KiB/s wr, 410 op/s 2023-10-01T09:49:19.051 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:19.099 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:49:19.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:19.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:49:19.205 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:49:19.234 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:18 smithi190 ceph-mon[160435]: failed to encode map e582 with expected crc 2023-10-01T09:49:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:18 smithi190 ceph-mon[160435]: failed to encode map e582 with expected crc 2023-10-01T09:49:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:18 smithi190 ceph-mon[160435]: failed to encode map e582 with expected crc 2023-10-01T09:49:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:18 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:49:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:18 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:49:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:18 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:18 smithi190 ceph-mon[160435]: pgmap v1741: 73 pgs: 73 active+clean; 4.8 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 296 KiB/s rd, 41 KiB/s wr, 410 op/s 2023-10-01T09:49:19.482 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:49:19.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:19.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:49:19.546 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:49:19.565 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:49:19.584 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:49:19.587 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:49:19.602 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:49:19.615 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:19.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:19.700 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:49:19.700 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:49:19.700 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:49:19.700 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:49:19.700 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T09:49:19.700 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T09:49:19.701 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T09:49:19.701 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T09:49:19.701 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T09:49:19.701 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T09:49:19.701 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T09:49:19.701 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T09:49:19.701 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:49:19.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:49:19.802 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T09:49:19.832 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:19.926 INFO:tasks.workunit.client.0.smithi138.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-01T09:49:19.932 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg 2023-10-01T09:49:20.021 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:49:20.024 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd remove testimg_import 2023-10-01T09:49:20.113 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:49:20.118 INFO:tasks.workunit.client.0.smithi138.stderr:+ tiered=0 2023-10-01T09:49:20.118 INFO:tasks.workunit.client.0.smithi138.stderr:+ ceph osd dump 2023-10-01T09:49:20.118 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep '^pool' 2023-10-01T09:49:20.118 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep ''\''rbd'\''' 2023-10-01T09:49:20.118 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep tier 2023-10-01T09:49:20.174 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2 found: no such container 2023-10-01T09:49:20.176 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:19 smithi138 ceph-mon[192513]: failed to encode map e583 with expected crc 2023-10-01T09:49:20.176 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:19 smithi138 ceph-mon[192513]: osdmap e583: 8 total, 8 up, 7 in 2023-10-01T09:49:20.176 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:19 smithi138 ceph-mon[192513]: failed to encode map e583 with expected crc 2023-10-01T09:49:20.176 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:19 smithi138 ceph-mon[192513]: failed to encode map e583 with expected crc 2023-10-01T09:49:20.176 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:19 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3115565105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:49:20.177 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:19 smithi138 ceph-mon[185886]: failed to encode map e583 with expected crc 2023-10-01T09:49:20.177 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:19 smithi138 ceph-mon[185886]: osdmap e583: 8 total, 8 up, 7 in 2023-10-01T09:49:20.177 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:19 smithi138 ceph-mon[185886]: failed to encode map e583 with expected crc 2023-10-01T09:49:20.177 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:19 smithi138 ceph-mon[185886]: failed to encode map e583 with expected crc 2023-10-01T09:49:20.177 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:19 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3115565105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:49:20.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:20.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T09:49:20.279 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:49:20.305 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:19 smithi190 ceph-mon[160435]: failed to encode map e583 with expected crc 2023-10-01T09:49:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:19 smithi190 ceph-mon[160435]: osdmap e583: 8 total, 8 up, 7 in 2023-10-01T09:49:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:19 smithi190 ceph-mon[160435]: failed to encode map e583 with expected crc 2023-10-01T09:49:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:19 smithi190 ceph-mon[160435]: failed to encode map e583 with expected crc 2023-10-01T09:49:20.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:19 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3115565105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:49:20.473 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:20.730 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:49:20.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:20.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:49:20.836 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T09:49:20.839 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_726287/sparse1 2023-10-01T09:49:20.845 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records in 2023-10-01T09:49:20.845 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records out 2023-10-01T09:49:20.845 INFO:tasks.workunit.client.0.smithi138.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00506507 s, 207 MB/s 2023-10-01T09:49:20.845 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_726287/sparse2 2023-10-01T09:49:20.851 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records in 2023-10-01T09:49:20.851 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records out 2023-10-01T09:49:20.851 INFO:tasks.workunit.client.0.smithi138.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00492725 s, 213 MB/s 2023-10-01T09:49:20.851 INFO:tasks.workunit.client.0.smithi138.stderr:+ truncate /tmp/rbd_import_export_726287/sparse2 -s 2M 2023-10-01T09:49:20.851 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse1 2023-10-01T09:49:20.864 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:20.889 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 0% complete...failed. 2023-10-01T09:49:20.889 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: delete error: (2) No such file or directory 2023-10-01T09:49:20.892 INFO:tasks.workunit.client.0.smithi138.stderr:+ true 2023-10-01T09:49:20.892 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_726287/sparse1 2023-10-01T09:49:20.909 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5 found: no such container 2023-10-01T09:49:20.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:20.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T09:49:20.956 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-01T09:49:20.956 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:49:20.956 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T09:49:20.959 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd ls -l 2023-10-01T09:49:20.959 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep sparse1 2023-10-01T09:49:20.959 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-01T09:49:20.995 INFO:tasks.workunit.client.0.smithi138.stdout:sparse1 2 MiB 2 2023-10-01T09:49:20.995 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects sparse1 2023-10-01T09:49:20.995 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse1 2023-10-01T09:49:20.996 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse1 2023-10-01T09:49:20.996 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep block_name_prefix 2023-10-01T09:49:20.996 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk '{print $NF;}' 2023-10-01T09:49:21.017 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:49:21.023 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:20 smithi138 ceph-mon[185886]: failed to encode map e584 with expected crc 2023-10-01T09:49:21.023 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:20 smithi138 ceph-mon[185886]: osdmap e584: 8 total, 8 up, 7 in 2023-10-01T09:49:21.023 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:20 smithi138 ceph-mon[185886]: failed to encode map e584 with expected crc 2023-10-01T09:49:21.024 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:20 smithi138 ceph-mon[185886]: pgmap v1744: 73 pgs: 73 active+clean; 4.7 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 117 KiB/s rd, 6.5 KiB/s wr, 155 op/s 2023-10-01T09:49:21.024 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:20 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2416355100' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:49:21.024 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:20 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2416355100' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:49:21.024 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:20 smithi138 ceph-mon[192513]: failed to encode map e584 with expected crc 2023-10-01T09:49:21.024 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:20 smithi138 ceph-mon[192513]: osdmap e584: 8 total, 8 up, 7 in 2023-10-01T09:49:21.025 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:20 smithi138 ceph-mon[192513]: failed to encode map e584 with expected crc 2023-10-01T09:49:21.025 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:20 smithi138 ceph-mon[192513]: pgmap v1744: 73 pgs: 73 active+clean; 4.7 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 117 KiB/s rd, 6.5 KiB/s wr, 155 op/s 2023-10-01T09:49:21.025 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:20 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2416355100' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:49:21.025 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:20 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2416355100' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:49:21.034 INFO:tasks.workunit.client.0.smithi138.stderr:++ prefix=rbd_data.945655cbf4f9 2023-10-01T09:49:21.035 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep rbd_data.945655cbf4f9 2023-10-01T09:49:21.035 INFO:tasks.workunit.client.0.smithi138.stderr:++++ get_image_data_pool sparse1 2023-10-01T09:49:21.035 INFO:tasks.workunit.client.0.smithi138.stderr:++++ image=sparse1 2023-10-01T09:49:21.035 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sed -e 's/rbd_data.945655cbf4f9\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-01T09:49:21.036 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sort -u 2023-10-01T09:49:21.036 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ rbd info sparse1 2023-10-01T09:49:21.036 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ grep 'data_pool: ' 2023-10-01T09:49:21.036 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ awk -F: '{ print $NF }' 2023-10-01T09:49:21.054 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:21.076 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool= 2023-10-01T09:49:21.076 INFO:tasks.workunit.client.0.smithi138.stderr:++++ '[' -z ']' 2023-10-01T09:49:21.076 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool=rbd 2023-10-01T09:49:21.076 INFO:tasks.workunit.client.0.smithi138.stderr:++++ echo rbd 2023-10-01T09:49:21.076 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rados ls -p rbd 2023-10-01T09:49:21.109 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects=1 2023-10-01T09:49:21.110 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo 1 2023-10-01T09:49:21.110 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-01T09:49:21.110 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export sparse1 /tmp/rbd_import_export_726287/sparse1.out 2023-10-01T09:49:21.151 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-01T09:49:21.154 INFO:tasks.workunit.client.0.smithi138.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_726287/sparse1 /tmp/rbd_import_export_726287/sparse1.out 2023-10-01T09:49:21.154 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp -l /tmp/rbd_import_export_726287/sparse1 /tmp/rbd_import_export_726287/sparse1.out 2023-10-01T09:49:21.156 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_726287/sparse1 --format %b 2023-10-01T09:49:21.157 INFO:tasks.workunit.client.0.smithi138.stderr:+ origsize=2048 2023-10-01T09:49:21.158 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_726287/sparse1.out --format %b 2023-10-01T09:49:21.159 INFO:tasks.workunit.client.0.smithi138.stderr:+ exportsize=2048 2023-10-01T09:49:21.159 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:49:21.159 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:49:21.159 INFO:tasks.workunit.client.0.smithi138.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T09:49:21.159 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/sparse1.out 2023-10-01T09:49:21.159 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse1 2023-10-01T09:49:21.252 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-01T09:49:21.255 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse2 2023-10-01T09:49:21.294 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 0% complete...failed. 2023-10-01T09:49:21.295 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: delete error: (2) No such file or directory 2023-10-01T09:49:21.297 INFO:tasks.workunit.client.0.smithi138.stderr:+ true 2023-10-01T09:49:21.297 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_726287/sparse2 2023-10-01T09:49:21.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:20 smithi190 ceph-mon[160435]: failed to encode map e584 with expected crc 2023-10-01T09:49:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:20 smithi190 ceph-mon[160435]: osdmap e584: 8 total, 8 up, 7 in 2023-10-01T09:49:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:20 smithi190 ceph-mon[160435]: failed to encode map e584 with expected crc 2023-10-01T09:49:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:20 smithi190 ceph-mon[160435]: pgmap v1744: 73 pgs: 73 active+clean; 4.7 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 117 KiB/s rd, 6.5 KiB/s wr, 155 op/s 2023-10-01T09:49:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:20 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2416355100' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:49:21.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:20 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2416355100' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T09:49:21.358 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-01T09:49:21.359 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:49:21.359 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T09:49:21.361 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd ls -l 2023-10-01T09:49:21.362 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep sparse2 2023-10-01T09:49:21.362 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-10-01T09:49:21.362 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:49:21.362 INFO:teuthology.orchestra.run.smithi138.stdout: "size": 20, 2023-10-01T09:49:21.362 INFO:teuthology.orchestra.run.smithi138.stdout: "duration": 600, 2023-10-01T09:49:21.362 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [] 2023-10-01T09:49:21.363 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:49:21.398 INFO:tasks.workunit.client.0.smithi138.stdout:sparse2 2 MiB 2 2023-10-01T09:49:21.399 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects sparse2 2023-10-01T09:49:21.399 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse2 2023-10-01T09:49:21.399 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse2 2023-10-01T09:49:21.399 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep block_name_prefix 2023-10-01T09:49:21.399 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk '{print $NF;}' 2023-10-01T09:49:21.433 INFO:tasks.workunit.client.0.smithi138.stderr:++ prefix=rbd_data.9486cd596bae 2023-10-01T09:49:21.434 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep rbd_data.9486cd596bae 2023-10-01T09:49:21.434 INFO:tasks.workunit.client.0.smithi138.stderr:++++ get_image_data_pool sparse2 2023-10-01T09:49:21.434 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sed -e 's/rbd_data.9486cd596bae\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-01T09:49:21.435 INFO:tasks.workunit.client.0.smithi138.stderr:++++ image=sparse2 2023-10-01T09:49:21.435 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sort -u 2023-10-01T09:49:21.435 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ rbd info sparse2 2023-10-01T09:49:21.435 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ grep 'data_pool: ' 2023-10-01T09:49:21.435 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ awk -F: '{ print $NF }' 2023-10-01T09:49:21.474 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool= 2023-10-01T09:49:21.474 INFO:tasks.workunit.client.0.smithi138.stderr:++++ '[' -z ']' 2023-10-01T09:49:21.474 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool=rbd 2023-10-01T09:49:21.475 INFO:tasks.workunit.client.0.smithi138.stderr:++++ echo rbd 2023-10-01T09:49:21.475 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rados ls -p rbd 2023-10-01T09:49:21.505 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects=0 2023-10-01T09:49:21.506 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo 0 2023-10-01T09:49:21.506 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-01T09:49:21.506 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export sparse2 /tmp/rbd_import_export_726287/sparse2.out 2023-10-01T09:49:21.540 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-01T09:49:21.543 INFO:tasks.workunit.client.0.smithi138.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_726287/sparse2 /tmp/rbd_import_export_726287/sparse2.out 2023-10-01T09:49:21.543 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp -l /tmp/rbd_import_export_726287/sparse2 /tmp/rbd_import_export_726287/sparse2.out 2023-10-01T09:49:21.545 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_726287/sparse2 --format %b 2023-10-01T09:49:21.546 INFO:tasks.workunit.client.0.smithi138.stderr:+ origsize=2048 2023-10-01T09:49:21.547 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_726287/sparse2.out --format %b 2023-10-01T09:49:21.548 INFO:tasks.workunit.client.0.smithi138.stderr:+ exportsize=2048 2023-10-01T09:49:21.548 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:49:21.548 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:49:21.548 INFO:tasks.workunit.client.0.smithi138.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T09:49:21.548 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/sparse2.out 2023-10-01T09:49:21.549 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse2 2023-10-01T09:49:21.696 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-01T09:49:21.698 INFO:tasks.workunit.client.0.smithi138.stderr:+ truncate /tmp/rbd_import_export_726287/sparse1 -s 10M 2023-10-01T09:49:21.699 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-01T09:49:21.860 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:49:21.861 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:49:21.861 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T09:49:21.863 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd ls -l 2023-10-01T09:49:21.863 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep sparse1 2023-10-01T09:49:21.864 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-10-01T09:49:21.900 INFO:tasks.workunit.client.0.smithi138.stdout:sparse1 10 MiB 2 2023-10-01T09:49:21.901 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects sparse1 2023-10-01T09:49:21.901 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse1 2023-10-01T09:49:21.901 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse1 2023-10-01T09:49:21.901 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep block_name_prefix 2023-10-01T09:49:21.901 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk '{print $NF;}' 2023-10-01T09:49:21.934 INFO:tasks.workunit.client.0.smithi138.stderr:++ prefix=rbd_data.94b31a135e33 2023-10-01T09:49:21.935 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep rbd_data.94b31a135e33 2023-10-01T09:49:21.935 INFO:tasks.workunit.client.0.smithi138.stderr:++++ get_image_data_pool sparse1 2023-10-01T09:49:21.935 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sed -e 's/rbd_data.94b31a135e33\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-01T09:49:21.935 INFO:tasks.workunit.client.0.smithi138.stderr:++++ image=sparse1 2023-10-01T09:49:21.935 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sort -u 2023-10-01T09:49:21.935 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ rbd info sparse1 2023-10-01T09:49:21.936 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ grep 'data_pool: ' 2023-10-01T09:49:21.936 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ awk -F: '{ print $NF }' 2023-10-01T09:49:21.969 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool= 2023-10-01T09:49:21.969 INFO:tasks.workunit.client.0.smithi138.stderr:++++ '[' -z ']' 2023-10-01T09:49:21.969 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool=rbd 2023-10-01T09:49:21.969 INFO:tasks.workunit.client.0.smithi138.stderr:++++ echo rbd 2023-10-01T09:49:21.970 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rados ls -p rbd 2023-10-01T09:49:21.999 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects=1 2023-10-01T09:49:21.999 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo 1 2023-10-01T09:49:21.999 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-01T09:49:21.999 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export sparse1 /tmp/rbd_import_export_726287/sparse1.out 2023-10-01T09:49:22.118 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-01T09:49:22.121 INFO:tasks.workunit.client.0.smithi138.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_726287/sparse1 /tmp/rbd_import_export_726287/sparse1.out 2023-10-01T09:49:22.122 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp -l /tmp/rbd_import_export_726287/sparse1 /tmp/rbd_import_export_726287/sparse1.out 2023-10-01T09:49:22.129 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_726287/sparse1 --format %b 2023-10-01T09:49:22.130 INFO:tasks.workunit.client.0.smithi138.stderr:+ origsize=2048 2023-10-01T09:49:22.130 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_726287/sparse1.out --format %b 2023-10-01T09:49:22.131 INFO:tasks.workunit.client.0.smithi138.stderr:+ exportsize=2048 2023-10-01T09:49:22.131 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:49:22.131 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:49:22.131 INFO:tasks.workunit.client.0.smithi138.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T09:49:22.131 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/sparse1.out 2023-10-01T09:49:22.132 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse1 2023-10-01T09:49:22.216 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-01T09:49:22.219 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_726287/sparse2 oflag=append conv=notrunc 2023-10-01T09:49:22.230 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records in 2023-10-01T09:49:22.230 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records out 2023-10-01T09:49:22.230 INFO:tasks.workunit.client.0.smithi138.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0100651 s, 208 MB/s 2023-10-01T09:49:22.231 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-01T09:49:22.316 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:49:22.317 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:49:22.317 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T09:49:22.319 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd ls -l 2023-10-01T09:49:22.320 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep sparse2 2023-10-01T09:49:22.320 INFO:tasks.workunit.client.0.smithi138.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-10-01T09:49:22.354 INFO:tasks.workunit.client.0.smithi138.stdout:sparse2 4 MiB 2 2023-10-01T09:49:22.355 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects sparse2 2023-10-01T09:49:22.355 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse2 2023-10-01T09:49:22.355 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse2 2023-10-01T09:49:22.355 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep block_name_prefix 2023-10-01T09:49:22.356 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk '{print $NF;}' 2023-10-01T09:49:22.393 INFO:tasks.workunit.client.0.smithi138.stderr:++ prefix=rbd_data.94e0b1078546 2023-10-01T09:49:22.394 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep rbd_data.94e0b1078546 2023-10-01T09:49:22.394 INFO:tasks.workunit.client.0.smithi138.stderr:++++ get_image_data_pool sparse2 2023-10-01T09:49:22.395 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sed -e 's/rbd_data.94e0b1078546\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-01T09:49:22.395 INFO:tasks.workunit.client.0.smithi138.stderr:++++ image=sparse2 2023-10-01T09:49:22.395 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sort -u 2023-10-01T09:49:22.395 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ rbd info sparse2 2023-10-01T09:49:22.395 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ grep 'data_pool: ' 2023-10-01T09:49:22.395 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ awk -F: '{ print $NF }' 2023-10-01T09:49:22.430 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool= 2023-10-01T09:49:22.430 INFO:tasks.workunit.client.0.smithi138.stderr:++++ '[' -z ']' 2023-10-01T09:49:22.430 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool=rbd 2023-10-01T09:49:22.430 INFO:tasks.workunit.client.0.smithi138.stderr:++++ echo rbd 2023-10-01T09:49:22.431 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rados ls -p rbd 2023-10-01T09:49:22.431 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:49:22.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:22.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:49:22.461 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects='0 2023-10-01T09:49:22.461 INFO:tasks.workunit.client.0.smithi138.stderr:2 2023-10-01T09:49:22.461 INFO:tasks.workunit.client.0.smithi138.stderr:3' 2023-10-01T09:49:22.461 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo 0 2 3 2023-10-01T09:49:22.461 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-10-01T09:49:22.461 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export sparse2 /tmp/rbd_import_export_726287/sparse2.out 2023-10-01T09:49:22.500 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-01T09:49:22.505 INFO:tasks.workunit.client.0.smithi138.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_726287/sparse2 /tmp/rbd_import_export_726287/sparse2.out 2023-10-01T09:49:22.505 INFO:tasks.workunit.client.0.smithi138.stderr:+ cmp -l /tmp/rbd_import_export_726287/sparse2 /tmp/rbd_import_export_726287/sparse2.out 2023-10-01T09:49:22.508 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_726287/sparse2 --format %b 2023-10-01T09:49:22.509 INFO:tasks.workunit.client.0.smithi138.stderr:+ origsize=6144 2023-10-01T09:49:22.509 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_726287/sparse2.out --format %b 2023-10-01T09:49:22.510 INFO:tasks.workunit.client.0.smithi138.stderr:+ exportsize=6144 2023-10-01T09:49:22.510 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:49:22.511 INFO:tasks.workunit.client.0.smithi138.stderr:+ difference=0 2023-10-01T09:49:22.511 INFO:tasks.workunit.client.0.smithi138.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T09:49:22.511 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/sparse2.out 2023-10-01T09:49:22.511 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse2 2023-10-01T09:49:22.519 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T09:49:22.534 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:49:22.561 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:22.608 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:49:22.611 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-01T09:49:22.614 INFO:tasks.workunit.client.0.smithi138.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-10-01T09:49:22.615 INFO:tasks.workunit.client.0.smithi138.stdout:partially-sparse file imports to partially-sparse image 2023-10-01T09:49:22.615 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_726287/sparse1 sparse 2023-10-01T09:49:22.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:22.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:49:22.681 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-01T09:49:22.681 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:49:22.681 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T09:49:22.683 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects sparse 2023-10-01T09:49:22.684 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse 2023-10-01T09:49:22.684 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse 2023-10-01T09:49:22.684 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep block_name_prefix 2023-10-01T09:49:22.684 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk '{print $NF;}' 2023-10-01T09:49:22.717 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:49:22.722 INFO:tasks.workunit.client.0.smithi138.stderr:++ prefix=rbd_data.950d65d88659 2023-10-01T09:49:22.722 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep rbd_data.950d65d88659 2023-10-01T09:49:22.723 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sed -e 's/rbd_data.950d65d88659\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-01T09:49:22.723 INFO:tasks.workunit.client.0.smithi138.stderr:++++ get_image_data_pool sparse 2023-10-01T09:49:22.723 INFO:tasks.workunit.client.0.smithi138.stderr:++++ image=sparse 2023-10-01T09:49:22.723 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sort -u 2023-10-01T09:49:22.727 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ rbd info sparse 2023-10-01T09:49:22.728 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ grep 'data_pool: ' 2023-10-01T09:49:22.728 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ awk -F: '{ print $NF }' 2023-10-01T09:49:22.743 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:22.771 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool= 2023-10-01T09:49:22.771 INFO:tasks.workunit.client.0.smithi138.stderr:++++ '[' -z ']' 2023-10-01T09:49:22.771 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool=rbd 2023-10-01T09:49:22.772 INFO:tasks.workunit.client.0.smithi138.stderr:++++ echo rbd 2023-10-01T09:49:22.772 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rados ls -p rbd 2023-10-01T09:49:22.786 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:49:22.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:22.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:49:22.796 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T09:49:22.814 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects=1 2023-10-01T09:49:22.814 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo 1 2023-10-01T09:49:22.814 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-10-01T09:49:22.814 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse 2023-10-01T09:49:22.891 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:49:22.922 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:22.933 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-01T09:49:22.937 INFO:tasks.workunit.client.0.smithi138.stdout:zeros import through stdin to sparse image 2023-10-01T09:49:22.937 INFO:tasks.workunit.client.0.smithi138.stderr:+ echo 'zeros import through stdin to sparse image' 2023-10-01T09:49:22.938 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/dev/zero bs=1M count=4 2023-10-01T09:49:22.938 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd import --new-format - sparse 2023-10-01T09:49:22.980 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:49:22.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:22.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:49:23.000 INFO:tasks.workunit.client.0.smithi138.stderr:4+0 records in 2023-10-01T09:49:23.000 INFO:tasks.workunit.client.0.smithi138.stderr:4+0 records out 2023-10-01T09:49:23.000 INFO:tasks.workunit.client.0.smithi138.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0623935 s, 67.2 MB/s 2023-10-01T09:49:23.016 INFO:tasks.workunit.client.0.smithi138.stderr: Importing image: 100% complete...done. 2023-10-01T09:49:23.017 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:49:23.020 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects sparse 2023-10-01T09:49:23.020 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse 2023-10-01T09:49:23.021 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse 2023-10-01T09:49:23.021 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep block_name_prefix 2023-10-01T09:49:23.021 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk '{print $NF;}' 2023-10-01T09:49:23.052 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:23.066 INFO:tasks.workunit.client.0.smithi138.stderr:++ prefix=rbd_data.e2c9d9f3271f 2023-10-01T09:49:23.066 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep rbd_data.e2c9d9f3271f 2023-10-01T09:49:23.067 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sed -e 's/rbd_data.e2c9d9f3271f\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-01T09:49:23.067 INFO:tasks.workunit.client.0.smithi138.stderr:++++ get_image_data_pool sparse 2023-10-01T09:49:23.067 INFO:tasks.workunit.client.0.smithi138.stderr:++++ image=sparse 2023-10-01T09:49:23.068 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sort -u 2023-10-01T09:49:23.068 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ rbd info sparse 2023-10-01T09:49:23.068 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ grep 'data_pool: ' 2023-10-01T09:49:23.068 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ awk -F: '{ print $NF }' 2023-10-01T09:49:23.085 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T09:49:23.114 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool= 2023-10-01T09:49:23.115 INFO:tasks.workunit.client.0.smithi138.stderr:++++ '[' -z ']' 2023-10-01T09:49:23.115 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool=rbd 2023-10-01T09:49:23.115 INFO:tasks.workunit.client.0.smithi138.stderr:++++ echo rbd 2023-10-01T09:49:23.115 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rados ls -p rbd 2023-10-01T09:49:23.119 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:23.153 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects= 2023-10-01T09:49:23.153 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo 2023-10-01T09:49:23.153 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-10-01T09:49:23.153 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse 2023-10-01T09:49:23.169 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 5, 6, 4, 3] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 7, 6, 5] 2023-10-01T09:49:23.169 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T09:49:23.169 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-01T09:49:23.169 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T09:49:23.169 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T09:49:23.169 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-01T09:49:23.191 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1 found: no such container 2023-10-01T09:49:23.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:23.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T09:49:23.254 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:49:23.256 INFO:tasks.workunit.client.0.smithi138.stdout:zeros export to sparse file 2023-10-01T09:49:23.257 INFO:tasks.workunit.client.0.smithi138.stderr:+ echo 'zeros export to sparse file' 2023-10-01T09:49:23.257 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd create --new-format sparse --size 4 2023-10-01T09:49:23.295 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T09:49:23.299 INFO:tasks.workunit.client.0.smithi138.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T09:49:23.303 INFO:tasks.workunit.client.0.smithi138.stderr:++ rbd info sparse 2023-10-01T09:49:23.303 INFO:tasks.workunit.client.0.smithi138.stderr:++ grep block_name_prefix 2023-10-01T09:49:23.303 INFO:tasks.workunit.client.0.smithi138.stderr:++ awk '{print $NF;}' 2023-10-01T09:49:23.321 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:23.343 INFO:tasks.workunit.client.0.smithi138.stderr:+ prefix=rbd_data.95465d3e1b89 2023-10-01T09:49:23.343 INFO:tasks.workunit.client.0.smithi138.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-01T09:49:23.344 INFO:tasks.workunit.client.0.smithi138.stderr:++ get_image_data_pool sparse 2023-10-01T09:49:23.344 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse 2023-10-01T09:49:23.344 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse 2023-10-01T09:49:23.344 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep 'data_pool: ' 2023-10-01T09:49:23.344 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk -F: '{ print $NF }' 2023-10-01T09:49:23.363 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7 found: no such container 2023-10-01T09:49:23.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:23.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T09:49:23.388 INFO:tasks.workunit.client.0.smithi138.stderr:++ data_pool= 2023-10-01T09:49:23.388 INFO:tasks.workunit.client.0.smithi138.stderr:++ '[' -z ']' 2023-10-01T09:49:23.388 INFO:tasks.workunit.client.0.smithi138.stderr:++ data_pool=rbd 2023-10-01T09:49:23.388 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo rbd 2023-10-01T09:49:23.389 INFO:tasks.workunit.client.0.smithi138.stderr:+ rados -p rbd put rbd_data.95465d3e1b89.000000000000 - 2023-10-01T09:49:23.424 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records in 2023-10-01T09:49:23.424 INFO:tasks.workunit.client.0.smithi138.stderr:1+0 records out 2023-10-01T09:49:23.424 INFO:tasks.workunit.client.0.smithi138.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0802771 s, 52.2 MB/s 2023-10-01T09:49:23.444 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects sparse 2023-10-01T09:49:23.444 INFO:tasks.workunit.client.0.smithi138.stderr:++ image=sparse 2023-10-01T09:49:23.445 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rbd info sparse 2023-10-01T09:49:23.445 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep block_name_prefix 2023-10-01T09:49:23.445 INFO:tasks.workunit.client.0.smithi138.stderr:+++ awk '{print $NF;}' 2023-10-01T09:49:23.469 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:49:23.486 INFO:tasks.workunit.client.0.smithi138.stderr:++ prefix=rbd_data.95465d3e1b89 2023-10-01T09:49:23.487 INFO:tasks.workunit.client.0.smithi138.stderr:+++ grep rbd_data.95465d3e1b89 2023-10-01T09:49:23.487 INFO:tasks.workunit.client.0.smithi138.stderr:++++ get_image_data_pool sparse 2023-10-01T09:49:23.487 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sed -e 's/rbd_data.95465d3e1b89\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-10-01T09:49:23.487 INFO:tasks.workunit.client.0.smithi138.stderr:++++ image=sparse 2023-10-01T09:49:23.487 INFO:tasks.workunit.client.0.smithi138.stderr:+++ sort -u 2023-10-01T09:49:23.489 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ rbd info sparse 2023-10-01T09:49:23.490 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ grep 'data_pool: ' 2023-10-01T09:49:23.490 INFO:tasks.workunit.client.0.smithi138.stderr:+++++ awk -F: '{ print $NF }' 2023-10-01T09:49:23.494 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:23.522 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool= 2023-10-01T09:49:23.523 INFO:tasks.workunit.client.0.smithi138.stderr:++++ '[' -z ']' 2023-10-01T09:49:23.523 INFO:tasks.workunit.client.0.smithi138.stderr:++++ data_pool=rbd 2023-10-01T09:49:23.523 INFO:tasks.workunit.client.0.smithi138.stderr:++++ echo rbd 2023-10-01T09:49:23.523 INFO:tasks.workunit.client.0.smithi138.stderr:+++ rados ls -p rbd 2023-10-01T09:49:23.537 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:49:23.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:23.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:49:23.553 INFO:tasks.workunit.client.0.smithi138.stderr:++ objects=0 2023-10-01T09:49:23.553 INFO:tasks.workunit.client.0.smithi138.stderr:++ echo 0 2023-10-01T09:49:23.553 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-10-01T09:49:23.554 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/sparse 2023-10-01T09:49:23.554 INFO:tasks.workunit.client.0.smithi138.stderr:rm: cannot remove '/tmp/rbd_import_export_726287/sparse': No such file or directory 2023-10-01T09:49:23.554 INFO:tasks.workunit.client.0.smithi138.stderr:+ true 2023-10-01T09:49:23.554 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd export sparse /tmp/rbd_import_export_726287/sparse 2023-10-01T09:49:23.557 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:23.599 INFO:tasks.workunit.client.0.smithi138.stderr: Exporting image: 100% complete...done. 2023-10-01T09:49:23.603 INFO:tasks.workunit.client.0.smithi138.stderr:++ stat /tmp/rbd_import_export_726287/sparse --format=%b 2023-10-01T09:49:23.604 INFO:tasks.workunit.client.0.smithi138.stderr:+ '[' 0 = 0 ']' 2023-10-01T09:49:23.604 INFO:tasks.workunit.client.0.smithi138.stderr:+ rbd rm sparse 2023-10-01T09:49:23.643 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T09:49:23.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:23 smithi138 ceph-mon[185886]: pgmap v1745: 73 pgs: 73 active+clean; 5.7 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 294 KiB/s rd, 381 KiB/s wr, 159 op/s 2023-10-01T09:49:23.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:23 smithi138 ceph-mon[185886]: failed to encode map e585 with expected crc 2023-10-01T09:49:23.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:23 smithi138 ceph-mon[185886]: osdmap e585: 8 total, 8 up, 7 in 2023-10-01T09:49:23.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:23 smithi138 ceph-mon[185886]: failed to encode map e585 with expected crc 2023-10-01T09:49:23.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:23 smithi138 ceph-mon[185886]: failed to encode map e585 with expected crc 2023-10-01T09:49:23.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:23 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2761512010' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:49:23.671 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:23 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:49:23.672 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:23 smithi138 ceph-mon[192513]: pgmap v1745: 73 pgs: 73 active+clean; 5.7 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 294 KiB/s rd, 381 KiB/s wr, 159 op/s 2023-10-01T09:49:23.672 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:23 smithi138 ceph-mon[192513]: failed to encode map e585 with expected crc 2023-10-01T09:49:23.672 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:23 smithi138 ceph-mon[192513]: osdmap e585: 8 total, 8 up, 7 in 2023-10-01T09:49:23.672 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:23 smithi138 ceph-mon[192513]: failed to encode map e585 with expected crc 2023-10-01T09:49:23.672 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:23 smithi138 ceph-mon[192513]: failed to encode map e585 with expected crc 2023-10-01T09:49:23.673 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:23 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2761512010' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:49:23.673 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:23 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:49:23.674 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:23.691 INFO:tasks.workunit.client.0.smithi138.stderr: Removing image: 100% complete...done. 2023-10-01T09:49:23.695 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm /tmp/rbd_import_export_726287/sparse /tmp/rbd_import_export_726287/sparse1 /tmp/rbd_import_export_726287/sparse2 /tmp/rbd_import_export_726287/sparse3 2023-10-01T09:49:23.697 INFO:tasks.workunit.client.0.smithi138.stderr:rm: cannot remove '/tmp/rbd_import_export_726287/sparse3': No such file or directory 2023-10-01T09:49:23.698 INFO:tasks.workunit.client.0.smithi138.stderr:+ true 2023-10-01T09:49:23.698 INFO:tasks.workunit.client.0.smithi138.stderr:+ echo OK 2023-10-01T09:49:23.698 INFO:tasks.workunit.client.0.smithi138.stdout:OK 2023-10-01T09:49:23.701 INFO:tasks.workunit.client.0.smithi138.stderr:+ rm -rf /tmp/rbd_import_export_726287 2023-10-01T09:49:23.701 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-01T09:49:23.701 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-01T09:49:23.724 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3 found: no such container 2023-10-01T09:49:23.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:23.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T09:49:23.751 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-01T09:49:23.751 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-01T09:49:23.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:23 smithi190 ceph-mon[160435]: pgmap v1745: 73 pgs: 73 active+clean; 5.7 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 294 KiB/s rd, 381 KiB/s wr, 159 op/s 2023-10-01T09:49:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:23 smithi190 ceph-mon[160435]: failed to encode map e585 with expected crc 2023-10-01T09:49:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:23 smithi190 ceph-mon[160435]: osdmap e585: 8 total, 8 up, 7 in 2023-10-01T09:49:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:23 smithi190 ceph-mon[160435]: failed to encode map e585 with expected crc 2023-10-01T09:49:23.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:23 smithi190 ceph-mon[160435]: failed to encode map e585 with expected crc 2023-10-01T09:49:23.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:23 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2761512010' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:49:23.809 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:23 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:49:23.828 DEBUG:teuthology.orchestra.run.smithi190:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T09:49:23.853 INFO:teuthology.orchestra.run.smithi190.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:23.877 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:49:23.898 INFO:teuthology.orchestra.run.smithi190.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4 found: no such container 2023-10-01T09:49:23.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:23.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi190 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T09:49:24.005 DEBUG:teuthology.orchestra.run.smithi138:> sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T09:49:24.010 DEBUG:teuthology.parallel:result is None 2023-10-01T09:49:24.010 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T09:49:24.032 INFO:teuthology.orchestra.run.smithi138.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-10-01T09:49:24.059 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T09:49:24.060 DEBUG:teuthology.orchestra.run.smithi138:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-01T09:49:24.117 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T09:49:24.117 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-01T09:49:24.117 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-01T09:49:24.118 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-01T09:49:24.118 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-01T09:49:24.118 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-01T09:49:24.118 DEBUG:teuthology.orchestra.run.smithi138:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-01T09:49:24.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T09:49:24.174 INFO:teuthology.orchestra.run.smithi138.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-01T09:49:24.175 DEBUG:teuthology.orchestra.run.smithi138:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-01T09:49:24.231 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-01T09:49:24.232 DEBUG:teuthology.orchestra.run.smithi138:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-01T09:49:24.289 INFO:tasks.thrashosds:joining thrashosds 2023-10-01T09:49:24.328 INFO:teuthology.orchestra.run.smithi138.stderr:Error: no container with name or ID ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0 found: no such container 2023-10-01T09:49:24.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-10-01T09:49:24.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi138 with status 125: 'sudo docker kill -s 1 ceph-465870a4-6038-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T09:49:24.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:24 smithi138 ceph-mon[192513]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:49:24.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:24 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:49:24.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:24 smithi138 ceph-mon[192513]: failed to encode map e586 with expected crc 2023-10-01T09:49:24.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:24 smithi138 ceph-mon[192513]: failed to encode map e586 with expected crc 2023-10-01T09:49:24.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:24 smithi138 ceph-mon[192513]: failed to encode map e586 with expected crc 2023-10-01T09:49:24.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:24 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:49:24.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:24 smithi138 ceph-mon[192513]: osdmap e586: 8 total, 8 up, 7 in 2023-10-01T09:49:24.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:24 smithi138 ceph-mon[185886]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:49:24.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:24 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:49:24.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:24 smithi138 ceph-mon[185886]: failed to encode map e586 with expected crc 2023-10-01T09:49:24.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:24 smithi138 ceph-mon[185886]: failed to encode map e586 with expected crc 2023-10-01T09:49:24.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:24 smithi138 ceph-mon[185886]: failed to encode map e586 with expected crc 2023-10-01T09:49:24.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:24 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:49:24.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:24 smithi138 ceph-mon[185886]: osdmap e586: 8 total, 8 up, 7 in 2023-10-01T09:49:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:24 smithi190 ceph-mon[160435]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T09:49:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:24 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:49:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:24 smithi190 ceph-mon[160435]: failed to encode map e586 with expected crc 2023-10-01T09:49:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:24 smithi190 ceph-mon[160435]: failed to encode map e586 with expected crc 2023-10-01T09:49:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:24 smithi190 ceph-mon[160435]: failed to encode map e586 with expected crc 2023-10-01T09:49:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:24 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:49:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:24 smithi190 ceph-mon[160435]: osdmap e586: 8 total, 8 up, 7 in 2023-10-01T09:49:25.192 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:49:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:49:24] "GET /metrics HTTP/1.1" 200 34037 "" "Prometheus/2.33.4" 2023-10-01T09:49:25.250 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:25.433 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:49:25.433 INFO:teuthology.orchestra.run.smithi138.stdout: "ops": [], 2023-10-01T09:49:25.433 INFO:teuthology.orchestra.run.smithi138.stdout: "num_ops": 0 2023-10-01T09:49:25.434 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:49:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:25 smithi190 ceph-mon[160435]: pgmap v1748: 73 pgs: 73 active+clean; 5.7 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 293 KiB/s rd, 382 KiB/s wr, 154 op/s 2023-10-01T09:49:25.952 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:25 smithi190 ceph-mon[160435]: failed to encode map e586 with expected crc 2023-10-01T09:49:25.952 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:25.953 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:49:25.953 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:25 smithi138 ceph-mon[185886]: pgmap v1748: 73 pgs: 73 active+clean; 5.7 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 293 KiB/s rd, 382 KiB/s wr, 154 op/s 2023-10-01T09:49:25.953 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:25 smithi138 ceph-mon[185886]: failed to encode map e586 with expected crc 2023-10-01T09:49:25.953 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:25.954 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:49:25.954 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:25 smithi138 ceph-mon[192513]: pgmap v1748: 73 pgs: 73 active+clean; 5.7 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 293 KiB/s rd, 382 KiB/s wr, 154 op/s 2023-10-01T09:49:25.954 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:25 smithi138 ceph-mon[192513]: failed to encode map e586 with expected crc 2023-10-01T09:49:25.954 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:25.954 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:49:26.658 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T09:49:26.947 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T09:49:27.658 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:27.711 INFO:teuthology.orchestra.run.smithi138.stdout:{ 2023-10-01T09:49:27.711 INFO:teuthology.orchestra.run.smithi138.stdout: "success": "" 2023-10-01T09:49:27.711 INFO:teuthology.orchestra.run.smithi138.stdout:} 2023-10-01T09:49:27.722 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:27 smithi138 ceph-mon[192513]: pgmap v1749: 73 pgs: 73 active+clean; 9.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.7 MiB/s wr, 418 op/s 2023-10-01T09:49:27.723 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:27 smithi138 ceph-mon[185886]: pgmap v1749: 73 pgs: 73 active+clean; 9.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.7 MiB/s wr, 418 op/s 2023-10-01T09:49:27.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:27 smithi190 ceph-mon[160435]: pgmap v1749: 73 pgs: 73 active+clean; 9.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.7 MiB/s wr, 418 op/s 2023-10-01T09:49:27.927 INFO:teuthology.orchestra.run.smithi138.stdout:osd.0: {} 2023-10-01T09:49:27.946 INFO:teuthology.orchestra.run.smithi138.stdout:osd.1: {} 2023-10-01T09:49:27.965 INFO:teuthology.orchestra.run.smithi138.stdout:osd.2: {} 2023-10-01T09:49:27.984 INFO:teuthology.orchestra.run.smithi138.stdout:osd.3: {} 2023-10-01T09:49:28.003 INFO:teuthology.orchestra.run.smithi138.stdout:osd.4: {} 2023-10-01T09:49:28.021 INFO:teuthology.orchestra.run.smithi138.stdout:osd.5: {} 2023-10-01T09:49:28.042 INFO:teuthology.orchestra.run.smithi138.stdout:osd.6: {} 2023-10-01T09:49:28.061 INFO:teuthology.orchestra.run.smithi138.stdout:osd.7: {} 2023-10-01T09:49:28.062 INFO:teuthology.orchestra.run.smithi138.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T09:49:28.062 INFO:teuthology.orchestra.run.smithi138.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T09:49:28.062 INFO:teuthology.orchestra.run.smithi138.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T09:49:28.062 INFO:teuthology.orchestra.run.smithi138.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T09:49:28.062 INFO:teuthology.orchestra.run.smithi138.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T09:49:28.062 INFO:teuthology.orchestra.run.smithi138.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T09:49:28.062 INFO:teuthology.orchestra.run.smithi138.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T09:49:28.062 INFO:teuthology.orchestra.run.smithi138.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T09:49:28.813 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:29.271 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:28 smithi138 ceph-mon[192513]: failed to encode map e587 with expected crc 2023-10-01T09:49:29.271 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:28 smithi138 ceph-mon[192513]: osdmap e587: 8 total, 8 up, 7 in 2023-10-01T09:49:29.272 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:28 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1748297239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:49:29.272 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:28 smithi138 ceph-mon[192513]: pgmap v1751: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 375 op/s 2023-10-01T09:49:29.272 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:28 smithi138 ceph-mon[185886]: failed to encode map e587 with expected crc 2023-10-01T09:49:29.272 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:28 smithi138 ceph-mon[185886]: osdmap e587: 8 total, 8 up, 7 in 2023-10-01T09:49:29.272 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:28 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1748297239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:49:29.272 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:28 smithi138 ceph-mon[185886]: pgmap v1751: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 375 op/s 2023-10-01T09:49:29.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:28 smithi190 ceph-mon[160435]: failed to encode map e587 with expected crc 2023-10-01T09:49:29.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:28 smithi190 ceph-mon[160435]: osdmap e587: 8 total, 8 up, 7 in 2023-10-01T09:49:29.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:28 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1748297239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T09:49:29.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:28 smithi190 ceph-mon[160435]: pgmap v1751: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 375 op/s 2023-10-01T09:49:29.961 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:29 smithi138 ceph-mon[192513]: failed to encode map e587 with expected crc 2023-10-01T09:49:29.961 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:29 smithi138 ceph-mon[192513]: failed to encode map e587 with expected crc 2023-10-01T09:49:29.961 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:29 smithi138 ceph-mon[192513]: failed to encode map e587 with expected crc 2023-10-01T09:49:29.962 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:29 smithi138 ceph-mon[185886]: failed to encode map e587 with expected crc 2023-10-01T09:49:29.962 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:29 smithi138 ceph-mon[185886]: failed to encode map e587 with expected crc 2023-10-01T09:49:29.962 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:29 smithi138 ceph-mon[185886]: failed to encode map e587 with expected crc 2023-10-01T09:49:30.203 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:49:30.204 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:49:30.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:29 smithi190 ceph-mon[160435]: failed to encode map e587 with expected crc 2023-10-01T09:49:30.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:29 smithi190 ceph-mon[160435]: failed to encode map e587 with expected crc 2023-10-01T09:49:30.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:29 smithi190 ceph-mon[160435]: failed to encode map e587 with expected crc 2023-10-01T09:49:31.192 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:30 smithi138 ceph-mon[192513]: pgmap v1752: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 909 KiB/s rd, 1.2 MiB/s wr, 298 op/s 2023-10-01T09:49:31.192 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:30 smithi138 ceph-mon[185886]: pgmap v1752: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 909 KiB/s rd, 1.2 MiB/s wr, 298 op/s 2023-10-01T09:49:31.212 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T09:49:31.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:30 smithi190 ceph-mon[160435]: pgmap v1752: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 909 KiB/s rd, 1.2 MiB/s wr, 298 op/s 2023-10-01T09:49:31.687 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:31.897 INFO:teuthology.orchestra.run.smithi138.stderr:noscrub is unset 2023-10-01T09:49:32.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2759740209' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:49:32.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:31 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:49:32.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2759740209' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:49:32.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:31 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:49:32.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2759740209' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:49:32.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:31 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T09:49:32.929 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T09:49:32.955 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:32 smithi138 ceph-mon[192513]: failed to encode map e588 with expected crc 2023-10-01T09:49:32.956 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:32 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:49:32.956 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:32 smithi138 ceph-mon[192513]: osdmap e588: 8 total, 8 up, 7 in 2023-10-01T09:49:32.956 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:32 smithi138 ceph-mon[192513]: pgmap v1754: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 857 KiB/s rd, 1.1 MiB/s wr, 281 op/s 2023-10-01T09:49:32.956 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:32 smithi138 ceph-mon[185886]: failed to encode map e588 with expected crc 2023-10-01T09:49:32.956 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:32 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:49:32.957 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:32 smithi138 ceph-mon[185886]: osdmap e588: 8 total, 8 up, 7 in 2023-10-01T09:49:32.957 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:32 smithi138 ceph-mon[185886]: pgmap v1754: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 857 KiB/s rd, 1.1 MiB/s wr, 281 op/s 2023-10-01T09:49:33.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:32 smithi190 ceph-mon[160435]: failed to encode map e588 with expected crc 2023-10-01T09:49:33.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:32 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T09:49:33.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:32 smithi190 ceph-mon[160435]: osdmap e588: 8 total, 8 up, 7 in 2023-10-01T09:49:33.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:32 smithi190 ceph-mon[160435]: pgmap v1754: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 857 KiB/s rd, 1.1 MiB/s wr, 281 op/s 2023-10-01T09:49:33.341 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:34.022 INFO:teuthology.orchestra.run.smithi138.stdout:ERROR: (22) Invalid argument 2023-10-01T09:49:34.023 INFO:teuthology.orchestra.run.smithi138.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:49:34.711 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-01T09:49:34.711 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-10-01T09:49:34.711 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd in 2 2023-10-01T09:49:34.998 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T09:49:35.193 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:49:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:49:34] "GET /metrics HTTP/1.1" 200 34037 "" "Prometheus/2.33.4" 2023-10-01T09:49:35.299 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:35.359 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:49:35.726 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:35 smithi190 ceph-mon[160435]: pgmap v1755: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 131 KiB/s rd, 639 B/s wr, 6 op/s 2023-10-01T09:49:35.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:35 smithi138 ceph-mon[192513]: pgmap v1755: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 131 KiB/s rd, 639 B/s wr, 6 op/s 2023-10-01T09:49:35.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:35 smithi138 ceph-mon[185886]: pgmap v1755: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 131 KiB/s rd, 639 B/s wr, 6 op/s 2023-10-01T09:49:36.475 INFO:teuthology.orchestra.run.smithi138.stderr:nodeep-scrub is unset 2023-10-01T09:49:36.556 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:36 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2638466548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:49:36.556 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:36 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2638466548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:49:36.802 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:36 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2638466548' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T09:49:37.048 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:49:37.048 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:49:37.471 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:37 smithi138 conmon[185862]: 2023-10-01T09:49:37.291+0000 7f70d64ec700 -1 mon.a@0(leader).osd e589 definitely_dead 0 2023-10-01T09:49:37.487 INFO:teuthology.orchestra.run.smithi138.stderr:marked in osd.2. 2023-10-01T09:49:37.723 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:37 smithi190 ceph-mon[160435]: pgmap v1756: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:49:37.723 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:37 smithi190 ceph-mon[160435]: failed to encode map e589 with expected crc 2023-10-01T09:49:37.723 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:37 smithi190 ceph-mon[160435]: failed to encode map e589 with expected crc 2023-10-01T09:49:37.723 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:37 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2638466548' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:49:37.723 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:37 smithi190 ceph-mon[160435]: osdmap e589: 8 total, 8 up, 7 in 2023-10-01T09:49:37.723 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:37 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3079792281' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:49:37.724 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T09:49:37.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:37 smithi138 ceph-mon[185886]: pgmap v1756: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:49:37.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:37 smithi138 ceph-mon[185886]: failed to encode map e589 with expected crc 2023-10-01T09:49:37.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:37 smithi138 ceph-mon[185886]: failed to encode map e589 with expected crc 2023-10-01T09:49:37.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:37 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2638466548' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:49:37.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:37 smithi138 ceph-mon[185886]: osdmap e589: 8 total, 8 up, 7 in 2023-10-01T09:49:37.822 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:37 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3079792281' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:49:37.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:37 smithi138 ceph-mon[192513]: pgmap v1756: 73 pgs: 73 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:49:37.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:37 smithi138 ceph-mon[192513]: failed to encode map e589 with expected crc 2023-10-01T09:49:37.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:37 smithi138 ceph-mon[192513]: failed to encode map e589 with expected crc 2023-10-01T09:49:37.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:37 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2638466548' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T09:49:37.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:37 smithi138 ceph-mon[192513]: osdmap e589: 8 total, 8 up, 7 in 2023-10-01T09:49:37.823 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:37 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3079792281' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-10-01T09:49:38.066 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-10-01T09:49:38.066 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-01T09:49:38.066 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 31 2023-10-01T09:49:38.089 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:49:38.459 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:38.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:38 smithi190 ceph-mon[160435]: failed to encode map e590 with expected crc 2023-10-01T09:49:38.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:38 smithi190 ceph-mon[160435]: failed to encode map e590 with expected crc 2023-10-01T09:49:38.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:38 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3079792281' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-01T09:49:38.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:38 smithi190 ceph-mon[160435]: osdmap e590: 8 total, 8 up, 8 in 2023-10-01T09:49:38.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:38 smithi190 ceph-mon[160435]: failed to encode map e590 with expected crc 2023-10-01T09:49:38.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:38 smithi190 ceph-mon[160435]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:38 smithi190 ceph-mon[160435]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:38 smithi190 ceph-mon[160435]: osdmap e591: 8 total, 8 up, 8 in 2023-10-01T09:49:38.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:38 smithi190 ceph-mon[160435]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:38 smithi190 ceph-mon[160435]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:38 smithi190 ceph-mon[160435]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.558 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:38 smithi190 ceph-mon[160435]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[185886]: failed to encode map e590 with expected crc 2023-10-01T09:49:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[185886]: failed to encode map e590 with expected crc 2023-10-01T09:49:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3079792281' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-01T09:49:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[185886]: osdmap e590: 8 total, 8 up, 8 in 2023-10-01T09:49:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[185886]: failed to encode map e590 with expected crc 2023-10-01T09:49:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[185886]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[185886]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[185886]: osdmap e591: 8 total, 8 up, 8 in 2023-10-01T09:49:38.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[185886]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[185886]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[185886]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[185886]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[192513]: failed to encode map e590 with expected crc 2023-10-01T09:49:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[192513]: failed to encode map e590 with expected crc 2023-10-01T09:49:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3079792281' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-10-01T09:49:38.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[192513]: osdmap e590: 8 total, 8 up, 8 in 2023-10-01T09:49:38.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[192513]: failed to encode map e590 with expected crc 2023-10-01T09:49:38.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[192513]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[192513]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[192513]: osdmap e591: 8 total, 8 up, 8 in 2023-10-01T09:49:38.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[192513]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[192513]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[192513]: failed to encode map e591 with expected crc 2023-10-01T09:49:38.822 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:38 smithi138 ceph-mon[192513]: failed to encode map e591 with expected crc 2023-10-01T09:49:39.646 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:49:39.646 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:49:39.670 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:39 smithi190 ceph-mon[160435]: pgmap v1760: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:49:39.671 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:39 smithi190 ceph-mon[160435]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:49:39.671 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:39 smithi190 ceph-mon[160435]: failed to encode map e592 with expected crc 2023-10-01T09:49:39.671 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:39 smithi190 ceph-mon[160435]: osdmap e592: 8 total, 8 up, 8 in 2023-10-01T09:49:39.671 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:39 smithi190 ceph-mon[160435]: failed to encode map e592 with expected crc 2023-10-01T09:49:39.671 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:39 smithi190 ceph-mon[160435]: failed to encode map e592 with expected crc 2023-10-01T09:49:39.671 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:39 smithi190 ceph-mon[160435]: failed to encode map e592 with expected crc 2023-10-01T09:49:39.728 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:39 smithi138 ceph-mon[185886]: pgmap v1760: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:49:39.728 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:39 smithi138 ceph-mon[185886]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:49:39.729 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:39 smithi138 ceph-mon[185886]: failed to encode map e592 with expected crc 2023-10-01T09:49:39.729 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:39 smithi138 ceph-mon[185886]: osdmap e592: 8 total, 8 up, 8 in 2023-10-01T09:49:39.729 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:39 smithi138 ceph-mon[185886]: failed to encode map e592 with expected crc 2023-10-01T09:49:39.729 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:39 smithi138 ceph-mon[185886]: failed to encode map e592 with expected crc 2023-10-01T09:49:39.729 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:39 smithi138 ceph-mon[185886]: failed to encode map e592 with expected crc 2023-10-01T09:49:39.729 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:39 smithi138 ceph-mon[192513]: pgmap v1760: 73 pgs: 1 peering, 72 active+clean; 8.6 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail 2023-10-01T09:49:39.729 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:39 smithi138 ceph-mon[192513]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-01T09:49:39.730 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:39 smithi138 ceph-mon[192513]: failed to encode map e592 with expected crc 2023-10-01T09:49:39.730 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:39 smithi138 ceph-mon[192513]: osdmap e592: 8 total, 8 up, 8 in 2023-10-01T09:49:39.730 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:39 smithi138 ceph-mon[192513]: failed to encode map e592 with expected crc 2023-10-01T09:49:39.730 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:39 smithi138 ceph-mon[192513]: failed to encode map e592 with expected crc 2023-10-01T09:49:39.730 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:39 smithi138 ceph-mon[192513]: failed to encode map e592 with expected crc 2023-10-01T09:49:40.118 INFO:teuthology.orchestra.run.smithi138.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T09:49:40.307 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T09:49:40.657 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:49:40.818 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T09:49:40.819 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-10-01T09:49:41.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[185886]: failed to encode map e592 with expected crc 2023-10-01T09:49:41.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[185886]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[185886]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[185886]: osdmap e593: 8 total, 8 up, 8 in 2023-10-01T09:49:41.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[185886]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[185886]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[185886]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4256175391' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-01T09:49:41.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:41.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:49:41.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[185886]: pgmap v1763: 73 pgs: 1 active+recovering+undersized+remapped, 1 peering, 71 active+clean; 8.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 1/87 objects misplaced (1.149%); 0 B/s, 4 objects/s recovering 2023-10-01T09:49:41.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[192513]: failed to encode map e592 with expected crc 2023-10-01T09:49:41.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[192513]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[192513]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[192513]: osdmap e593: 8 total, 8 up, 8 in 2023-10-01T09:49:41.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[192513]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[192513]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[192513]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4256175391' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-01T09:49:41.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:41.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:49:41.072 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:40 smithi138 ceph-mon[192513]: pgmap v1763: 73 pgs: 1 active+recovering+undersized+remapped, 1 peering, 71 active+clean; 8.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 1/87 objects misplaced (1.149%); 0 B/s, 4 objects/s recovering 2023-10-01T09:49:41.176 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:41.241 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:40 smithi190 ceph-mon[160435]: failed to encode map e592 with expected crc 2023-10-01T09:49:41.241 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:40 smithi190 ceph-mon[160435]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.241 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:40 smithi190 ceph-mon[160435]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.241 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:40 smithi190 ceph-mon[160435]: osdmap e593: 8 total, 8 up, 8 in 2023-10-01T09:49:41.241 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:40 smithi190 ceph-mon[160435]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.242 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:40 smithi190 ceph-mon[160435]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.242 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:40 smithi190 ceph-mon[160435]: failed to encode map e593 with expected crc 2023-10-01T09:49:41.242 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:40 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4256175391' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-01T09:49:41.242 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:41.242 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:49:41.242 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:40 smithi190 ceph-mon[160435]: pgmap v1763: 73 pgs: 1 active+recovering+undersized+remapped, 1 peering, 71 active+clean; 8.6 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 1/87 objects misplaced (1.149%); 0 B/s, 4 objects/s recovering 2023-10-01T09:49:41.993 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:41 smithi190 ceph-mon[160435]: failed to encode map e594 with expected crc 2023-10-01T09:49:41.993 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:41 smithi190 ceph-mon[160435]: failed to encode map e594 with expected crc 2023-10-01T09:49:41.993 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:41 smithi190 ceph-mon[160435]: osdmap e594: 8 total, 8 up, 8 in 2023-10-01T09:49:41.994 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:41 smithi190 ceph-mon[160435]: failed to encode map e594 with expected crc 2023-10-01T09:49:41.994 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:41 smithi190 ceph-mon[160435]: failed to encode map e594 with expected crc 2023-10-01T09:49:41.994 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:41 smithi190 ceph-mon[160435]: failed to encode map e594 with expected crc 2023-10-01T09:49:41.994 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:41 smithi190 ceph-mon[160435]: failed to encode map e594 with expected crc 2023-10-01T09:49:41.994 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:41 smithi190 ceph-mon[160435]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[185886]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[185886]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[185886]: osdmap e594: 8 total, 8 up, 8 in 2023-10-01T09:49:42.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[185886]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[185886]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[185886]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[185886]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[185886]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[192513]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[192513]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[192513]: osdmap e594: 8 total, 8 up, 8 in 2023-10-01T09:49:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[192513]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[192513]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[192513]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[192513]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.071 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:41 smithi138 ceph-mon[192513]: failed to encode map e594 with expected crc 2023-10-01T09:49:42.186 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:49:42.186 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:49:42.605 INFO:teuthology.orchestra.run.smithi138.stdout:mon.a: {} 2023-10-01T09:49:42.615 INFO:teuthology.orchestra.run.smithi138.stdout:mon.b: {} 2023-10-01T09:49:42.624 INFO:teuthology.orchestra.run.smithi138.stdout:mon.c: {} 2023-10-01T09:49:42.625 INFO:teuthology.orchestra.run.smithi138.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-01T09:49:42.625 INFO:teuthology.orchestra.run.smithi138.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-01T09:49:42.625 INFO:teuthology.orchestra.run.smithi138.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-01T09:49:42.858 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T09:49:43.214 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:49:43.299 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:42 smithi138 ceph-mon[185886]: pgmap v1765: 73 pgs: 1 active+recovering+undersized+remapped, 1 peering, 71 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1/87 objects misplaced (1.149%); 129 B/s, 2 keys/s, 11 objects/s recovering 2023-10-01T09:49:43.299 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:42 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/328719456' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T09:49:43.300 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:42 smithi138 ceph-mon[192513]: pgmap v1765: 73 pgs: 1 active+recovering+undersized+remapped, 1 peering, 71 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1/87 objects misplaced (1.149%); 129 B/s, 2 keys/s, 11 objects/s recovering 2023-10-01T09:49:43.300 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:42 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/328719456' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T09:49:43.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:42 smithi190 ceph-mon[160435]: pgmap v1765: 73 pgs: 1 active+recovering+undersized+remapped, 1 peering, 71 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1/87 objects misplaced (1.149%); 129 B/s, 2 keys/s, 11 objects/s recovering 2023-10-01T09:49:43.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:42 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/328719456' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T09:49:43.317 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-01T09:49:43.663 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:44.673 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:49:44.673 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:49:44.694 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:44 smithi190 ceph-mon[160435]: pgmap v1766: 73 pgs: 1 active+recovering+undersized+remapped, 72 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1/87 objects misplaced (1.149%); 97 B/s, 1 keys/s, 8 objects/s recovering 2023-10-01T09:49:44.694 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:44 smithi190 ceph-mon[160435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-01T09:49:44.694 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:44 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:49:44.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:44 smithi138 ceph-mon[185886]: pgmap v1766: 73 pgs: 1 active+recovering+undersized+remapped, 72 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1/87 objects misplaced (1.149%); 97 B/s, 1 keys/s, 8 objects/s recovering 2023-10-01T09:49:44.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:44 smithi138 ceph-mon[185886]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-01T09:49:44.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:44 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:49:44.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:44 smithi138 ceph-mon[192513]: pgmap v1766: 73 pgs: 1 active+recovering+undersized+remapped, 72 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 1/87 objects misplaced (1.149%); 97 B/s, 1 keys/s, 8 objects/s recovering 2023-10-01T09:49:44.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:44 smithi138 ceph-mon[192513]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-01T09:49:44.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:44 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:49:45.300 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T09:49:45.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:49:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:49:44] "GET /metrics HTTP/1.1" 200 34038 "" "Prometheus/2.33.4" 2023-10-01T09:49:45.541 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.0 to 1 (10000) 2023-10-01T09:49:45.556 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:45 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/676483290' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:49:45.650 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:49:45.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:45 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/676483290' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:49:45.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:45 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/676483290' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:49:46.201 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-10-01T09:49:46.551 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:46.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:46 smithi138 ceph-mon[192513]: failed to encode map e595 with expected crc 2023-10-01T09:49:46.790 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:46 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/676483290' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:49:46.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:46 smithi138 ceph-mon[192513]: osdmap e595: 8 total, 8 up, 8 in 2023-10-01T09:49:46.791 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:46 smithi138 ceph-mon[192513]: pgmap v1768: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 90 B/s, 1 keys/s, 6 objects/s recovering 2023-10-01T09:49:46.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:46 smithi138 ceph-mon[185886]: failed to encode map e595 with expected crc 2023-10-01T09:49:46.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:46 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/676483290' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:49:46.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:46 smithi138 ceph-mon[185886]: osdmap e595: 8 total, 8 up, 8 in 2023-10-01T09:49:46.791 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:46 smithi138 ceph-mon[185886]: pgmap v1768: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 90 B/s, 1 keys/s, 6 objects/s recovering 2023-10-01T09:49:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:46 smithi190 ceph-mon[160435]: failed to encode map e595 with expected crc 2023-10-01T09:49:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:46 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/676483290' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:49:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:46 smithi190 ceph-mon[160435]: osdmap e595: 8 total, 8 up, 8 in 2023-10-01T09:49:46.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:46 smithi190 ceph-mon[160435]: pgmap v1768: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 90 B/s, 1 keys/s, 6 objects/s recovering 2023-10-01T09:49:47.061 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:49:47.061 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:49:47.690 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T09:49:47.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:47 smithi190 ceph-mon[160435]: failed to encode map e595 with expected crc 2023-10-01T09:49:47.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:47 smithi190 ceph-mon[160435]: failed to encode map e595 with expected crc 2023-10-01T09:49:47.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:47 smithi190 ceph-mon[160435]: failed to encode map e595 with expected crc 2023-10-01T09:49:47.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:47 smithi138 ceph-mon[185886]: failed to encode map e595 with expected crc 2023-10-01T09:49:47.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:47 smithi138 ceph-mon[185886]: failed to encode map e595 with expected crc 2023-10-01T09:49:47.818 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:47 smithi138 ceph-mon[185886]: failed to encode map e595 with expected crc 2023-10-01T09:49:47.818 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:47 smithi138 ceph-mon[192513]: failed to encode map e595 with expected crc 2023-10-01T09:49:47.818 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:47 smithi138 ceph-mon[192513]: failed to encode map e595 with expected crc 2023-10-01T09:49:47.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:47 smithi138 ceph-mon[192513]: failed to encode map e595 with expected crc 2023-10-01T09:49:48.051 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:49:48.884 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-10-01T09:49:49.241 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:48 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3922740935' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:49:49.242 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:48 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:49:49.242 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:48 smithi190 ceph-mon[160435]: pgmap v1769: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 73 B/s, 1 keys/s, 5 objects/s recovering 2023-10-01T09:49:49.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:48 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3922740935' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:49:49.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:48 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:49:49.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:48 smithi138 ceph-mon[192513]: pgmap v1769: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 73 B/s, 1 keys/s, 5 objects/s recovering 2023-10-01T09:49:49.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:48 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3922740935' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:49:49.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:48 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T09:49:49.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:48 smithi138 ceph-mon[185886]: pgmap v1769: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 73 B/s, 1 keys/s, 5 objects/s recovering 2023-10-01T09:49:49.468 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:49:49.469 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:49:49.578 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-01T09:49:49.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:49 smithi190 ceph-mon[160435]: failed to encode map e596 with expected crc 2023-10-01T09:49:49.887 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:49 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:49:49.933 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:50.090 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T09:49:50.150 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:49 smithi138 ceph-mon[192513]: failed to encode map e596 with expected crc 2023-10-01T09:49:50.151 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:49 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:49:50.151 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:49 smithi138 ceph-mon[192513]: osdmap e596: 8 total, 8 up, 8 in 2023-10-01T09:49:50.154 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:49 smithi138 ceph-mon[192513]: failed to encode map e596 with expected crc 2023-10-01T09:49:50.154 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:49 smithi138 ceph-mon[192513]: failed to encode map e596 with expected crc 2023-10-01T09:49:50.154 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:49 smithi138 ceph-mon[192513]: failed to encode map e596 with expected crc 2023-10-01T09:49:50.154 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:49 smithi138 ceph-mon[185886]: failed to encode map e596 with expected crc 2023-10-01T09:49:50.154 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:49 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-01T09:49:50.155 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:49 smithi138 ceph-mon[185886]: osdmap e596: 8 total, 8 up, 8 in 2023-10-01T09:49:50.155 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:49 smithi138 ceph-mon[185886]: failed to encode map e596 with expected crc 2023-10-01T09:49:50.155 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:49 smithi138 ceph-mon[185886]: failed to encode map e596 with expected crc 2023-10-01T09:49:50.155 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:49 smithi138 ceph-mon[185886]: failed to encode map e596 with expected crc 2023-10-01T09:49:50.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:49 smithi190 ceph-mon[160435]: osdmap e596: 8 total, 8 up, 8 in 2023-10-01T09:49:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:49 smithi190 ceph-mon[160435]: failed to encode map e596 with expected crc 2023-10-01T09:49:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:49 smithi190 ceph-mon[160435]: failed to encode map e596 with expected crc 2023-10-01T09:49:50.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:49 smithi190 ceph-mon[160435]: failed to encode map e596 with expected crc 2023-10-01T09:49:50.436 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:49:51.193 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:50 smithi138 ceph-mon[192513]: failed to encode map e597 with expected crc 2023-10-01T09:49:51.193 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:50 smithi138 ceph-mon[192513]: osdmap e597: 8 total, 8 up, 8 in 2023-10-01T09:49:51.193 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:50 smithi138 ceph-mon[192513]: failed to encode map e597 with expected crc 2023-10-01T09:49:51.193 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:50 smithi138 ceph-mon[192513]: failed to encode map e597 with expected crc 2023-10-01T09:49:51.193 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:50 smithi138 ceph-mon[192513]: failed to encode map e597 with expected crc 2023-10-01T09:49:51.193 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:50 smithi138 ceph-mon[192513]: pgmap v1772: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:51.194 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:50 smithi138 ceph-mon[185886]: failed to encode map e597 with expected crc 2023-10-01T09:49:51.194 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:50 smithi138 ceph-mon[185886]: osdmap e597: 8 total, 8 up, 8 in 2023-10-01T09:49:51.194 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:50 smithi138 ceph-mon[185886]: failed to encode map e597 with expected crc 2023-10-01T09:49:51.194 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:50 smithi138 ceph-mon[185886]: failed to encode map e597 with expected crc 2023-10-01T09:49:51.194 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:50 smithi138 ceph-mon[185886]: failed to encode map e597 with expected crc 2023-10-01T09:49:51.194 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:50 smithi138 ceph-mon[185886]: pgmap v1772: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:51.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:50 smithi190 ceph-mon[160435]: failed to encode map e597 with expected crc 2023-10-01T09:49:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:50 smithi190 ceph-mon[160435]: osdmap e597: 8 total, 8 up, 8 in 2023-10-01T09:49:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:50 smithi190 ceph-mon[160435]: failed to encode map e597 with expected crc 2023-10-01T09:49:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:50 smithi190 ceph-mon[160435]: failed to encode map e597 with expected crc 2023-10-01T09:49:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:50 smithi190 ceph-mon[160435]: failed to encode map e597 with expected crc 2023-10-01T09:49:51.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:50 smithi190 ceph-mon[160435]: pgmap v1772: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:51.883 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:49:51.884 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:49:52.165 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.1 to 1 (10000) 2023-10-01T09:49:52.455 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:52 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:52.456 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:52 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2573730448' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:49:52.472 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T09:49:52.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:52 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:52.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:52 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2573730448' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:49:52.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:52 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:52.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:52 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2573730448' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:49:52.774 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-01T09:49:52.825 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:49:53.128 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:53 smithi190 ceph-mon[160435]: failed to encode map e598 with expected crc 2023-10-01T09:49:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:53 smithi190 ceph-mon[160435]: failed to encode map e598 with expected crc 2023-10-01T09:49:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:53 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2573730448' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:49:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:53 smithi190 ceph-mon[160435]: osdmap e598: 8 total, 8 up, 8 in 2023-10-01T09:49:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:53 smithi190 ceph-mon[160435]: failed to encode map e598 with expected crc 2023-10-01T09:49:53.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:53 smithi190 ceph-mon[160435]: pgmap v1774: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:53.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:53 smithi138 ceph-mon[185886]: failed to encode map e598 with expected crc 2023-10-01T09:49:53.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:53 smithi138 ceph-mon[185886]: failed to encode map e598 with expected crc 2023-10-01T09:49:53.518 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:53 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2573730448' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:49:53.519 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:53 smithi138 ceph-mon[185886]: osdmap e598: 8 total, 8 up, 8 in 2023-10-01T09:49:53.519 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:53 smithi138 ceph-mon[185886]: failed to encode map e598 with expected crc 2023-10-01T09:49:53.519 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:53 smithi138 ceph-mon[185886]: pgmap v1774: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:53.519 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:53 smithi138 ceph-mon[192513]: failed to encode map e598 with expected crc 2023-10-01T09:49:53.519 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:53 smithi138 ceph-mon[192513]: failed to encode map e598 with expected crc 2023-10-01T09:49:53.519 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:53 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2573730448' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:49:53.519 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:53 smithi138 ceph-mon[192513]: osdmap e598: 8 total, 8 up, 8 in 2023-10-01T09:49:53.519 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:53 smithi138 ceph-mon[192513]: failed to encode map e598 with expected crc 2023-10-01T09:49:53.520 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:53 smithi138 ceph-mon[192513]: pgmap v1774: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:54.217 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:49:54.217 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:49:54.830 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T09:49:55.180 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:49:55.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:49:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:49:54] "GET /metrics HTTP/1.1" 200 34038 "" "Prometheus/2.33.4" 2023-10-01T09:49:55.479 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-01T09:49:55.714 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:55 smithi190 ceph-mon[160435]: pgmap v1775: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:55.714 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:55 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/638936276' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:49:55.714 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:55.714 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:49:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:55 smithi138 ceph-mon[185886]: pgmap v1775: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:55 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/638936276' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:49:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:55.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:49:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:55 smithi138 ceph-mon[192513]: pgmap v1775: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:55 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/638936276' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T09:49:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:49:55.821 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:49:56.181 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-01T09:49:56.536 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:49:56.609 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:49:56.609 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:49:56.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:56 smithi138 ceph-mon[192513]: failed to encode map e599 with expected crc 2023-10-01T09:49:56.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:56 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/638936276' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:49:56.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:56 smithi138 ceph-mon[192513]: osdmap e599: 8 total, 8 up, 8 in 2023-10-01T09:49:56.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:56 smithi138 ceph-mon[192513]: failed to encode map e599 with expected crc 2023-10-01T09:49:56.749 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:56 smithi138 ceph-mon[192513]: failed to encode map e599 with expected crc 2023-10-01T09:49:56.750 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:56 smithi138 ceph-mon[185886]: failed to encode map e599 with expected crc 2023-10-01T09:49:56.750 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:56 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/638936276' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:49:56.750 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:56 smithi138 ceph-mon[185886]: osdmap e599: 8 total, 8 up, 8 in 2023-10-01T09:49:56.750 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:56 smithi138 ceph-mon[185886]: failed to encode map e599 with expected crc 2023-10-01T09:49:56.750 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:56 smithi138 ceph-mon[185886]: failed to encode map e599 with expected crc 2023-10-01T09:49:56.805 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:56 smithi190 ceph-mon[160435]: failed to encode map e599 with expected crc 2023-10-01T09:49:56.805 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:56 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/638936276' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-01T09:49:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:56 smithi190 ceph-mon[160435]: osdmap e599: 8 total, 8 up, 8 in 2023-10-01T09:49:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:56 smithi190 ceph-mon[160435]: failed to encode map e599 with expected crc 2023-10-01T09:49:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:56 smithi190 ceph-mon[160435]: failed to encode map e599 with expected crc 2023-10-01T09:49:57.330 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T09:49:57.679 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:49:57.774 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:57 smithi138 ceph-mon[192513]: pgmap v1777: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:57.775 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:57 smithi138 ceph-mon[185886]: pgmap v1777: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:57.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:57 smithi190 ceph-mon[160435]: pgmap v1777: 73 pgs: 73 active+clean; 8.6 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:58.892 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.2 to 1 (10000) 2023-10-01T09:49:58.959 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:58 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-01T09:49:58.960 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:58 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/913356794' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:49:58.960 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:58 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:49:58.960 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:58 smithi190 ceph-mon[160435]: pgmap v1778: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:59.154 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:49:59.154 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:49:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:58 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-01T09:49:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:58 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/913356794' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:49:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:58 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:49:59.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:58 smithi138 ceph-mon[192513]: pgmap v1778: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:59.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:58 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-01T09:49:59.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:58 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/913356794' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:49:59.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:58 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:49:59.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:58 smithi138 ceph-mon[185886]: pgmap v1778: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-10-01T09:49:59.597 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-01T09:49:59.805 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T09:49:59.950 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:59 smithi190 ceph-mon[160435]: failed to encode map e600 with expected crc 2023-10-01T09:50:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:59 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-01T09:50:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:59 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:50:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:59 smithi190 ceph-mon[160435]: osdmap e600: 8 total, 8 up, 8 in 2023-10-01T09:50:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:59 smithi190 ceph-mon[160435]: failed to encode map e600 with expected crc 2023-10-01T09:50:00.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:49:59 smithi190 ceph-mon[160435]: failed to encode map e600 with expected crc 2023-10-01T09:50:00.148 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:59 smithi138 ceph-mon[192513]: failed to encode map e600 with expected crc 2023-10-01T09:50:00.148 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:59 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-01T09:50:00.149 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:59 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:50:00.149 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:59 smithi138 ceph-mon[192513]: osdmap e600: 8 total, 8 up, 8 in 2023-10-01T09:50:00.149 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:59 smithi138 ceph-mon[192513]: failed to encode map e600 with expected crc 2023-10-01T09:50:00.149 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:49:59 smithi138 ceph-mon[192513]: failed to encode map e600 with expected crc 2023-10-01T09:50:00.149 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:59 smithi138 ceph-mon[185886]: failed to encode map e600 with expected crc 2023-10-01T09:50:00.149 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:59 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-01T09:50:00.149 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:59 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:50:00.149 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:59 smithi138 ceph-mon[185886]: osdmap e600: 8 total, 8 up, 8 in 2023-10-01T09:50:00.149 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:59 smithi138 ceph-mon[185886]: failed to encode map e600 with expected crc 2023-10-01T09:50:00.150 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:49:59 smithi138 ceph-mon[185886]: failed to encode map e600 with expected crc 2023-10-01T09:50:00.155 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:50:01.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:00 smithi190 ceph-mon[160435]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:00 smithi190 ceph-mon[160435]: osdmap e601: 8 total, 8 up, 8 in 2023-10-01T09:50:01.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:00 smithi190 ceph-mon[160435]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:00 smithi190 ceph-mon[160435]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:00 smithi190 ceph-mon[160435]: overall HEALTH_OK 2023-10-01T09:50:01.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:00 smithi190 ceph-mon[160435]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:00 smithi190 ceph-mon[160435]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:00 smithi190 ceph-mon[160435]: pgmap v1781: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:01.242 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[185886]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.242 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[185886]: osdmap e601: 8 total, 8 up, 8 in 2023-10-01T09:50:01.242 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[185886]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.242 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[185886]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.242 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[185886]: overall HEALTH_OK 2023-10-01T09:50:01.242 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[185886]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.242 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[185886]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.243 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[185886]: pgmap v1781: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:01.243 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[192513]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.243 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[192513]: osdmap e601: 8 total, 8 up, 8 in 2023-10-01T09:50:01.243 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[192513]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.243 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[192513]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.243 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[192513]: overall HEALTH_OK 2023-10-01T09:50:01.243 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[192513]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.243 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[192513]: failed to encode map e601 with expected crc 2023-10-01T09:50:01.243 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:00 smithi138 ceph-mon[192513]: pgmap v1781: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:01.653 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:50:01.653 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:50:01.916 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-01T09:50:02.056 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2480532958' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:50:02.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:01 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:50:02.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2480532958' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:50:02.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:01 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:50:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2480532958' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:50:02.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:01 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T09:50:02.321 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T09:50:02.555 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-01T09:50:02.670 INFO:teuthology.orchestra.run.smithi190.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T09:50:02.921 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:03.171 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:02 smithi138 ceph-mon[185886]: failed to encode map e602 with expected crc 2023-10-01T09:50:03.171 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:02 smithi138 ceph-mon[185886]: failed to encode map e602 with expected crc 2023-10-01T09:50:03.171 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:02 smithi138 ceph-mon[185886]: failed to encode map e602 with expected crc 2023-10-01T09:50:03.171 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:02 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:50:03.171 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:02 smithi138 ceph-mon[185886]: osdmap e602: 8 total, 8 up, 8 in 2023-10-01T09:50:03.172 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:02 smithi138 ceph-mon[185886]: pgmap v1783: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:03.172 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:02 smithi138 ceph-mon[192513]: failed to encode map e602 with expected crc 2023-10-01T09:50:03.172 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:02 smithi138 ceph-mon[192513]: failed to encode map e602 with expected crc 2023-10-01T09:50:03.172 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:02 smithi138 ceph-mon[192513]: failed to encode map e602 with expected crc 2023-10-01T09:50:03.172 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:02 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:50:03.172 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:02 smithi138 ceph-mon[192513]: osdmap e602: 8 total, 8 up, 8 in 2023-10-01T09:50:03.172 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:02 smithi138 ceph-mon[192513]: pgmap v1783: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:03.213 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:02 smithi190 ceph-mon[160435]: failed to encode map e602 with expected crc 2023-10-01T09:50:03.213 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:02 smithi190 ceph-mon[160435]: failed to encode map e602 with expected crc 2023-10-01T09:50:03.213 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:02 smithi190 ceph-mon[160435]: failed to encode map e602 with expected crc 2023-10-01T09:50:03.213 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:02 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-01T09:50:03.213 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:02 smithi190 ceph-mon[160435]: osdmap e602: 8 total, 8 up, 8 in 2023-10-01T09:50:03.213 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:02 smithi190 ceph-mon[160435]: pgmap v1783: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:03.922 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:03 smithi190 ceph-mon[160435]: failed to encode map e603 with expected crc 2023-10-01T09:50:03.922 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:03 smithi190 ceph-mon[160435]: failed to encode map e603 with expected crc 2023-10-01T09:50:03.922 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:03 smithi190 ceph-mon[160435]: failed to encode map e603 with expected crc 2023-10-01T09:50:03.922 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:03 smithi190 ceph-mon[160435]: osdmap e603: 8 total, 8 up, 8 in 2023-10-01T09:50:04.112 INFO:teuthology.orchestra.run.smithi190.stdout:ERROR: (22) Invalid argument 2023-10-01T09:50:04.112 INFO:teuthology.orchestra.run.smithi190.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T09:50:04.165 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:03 smithi138 ceph-mon[192513]: failed to encode map e603 with expected crc 2023-10-01T09:50:04.165 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:03 smithi138 ceph-mon[192513]: failed to encode map e603 with expected crc 2023-10-01T09:50:04.165 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:03 smithi138 ceph-mon[192513]: failed to encode map e603 with expected crc 2023-10-01T09:50:04.165 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:03 smithi138 ceph-mon[192513]: osdmap e603: 8 total, 8 up, 8 in 2023-10-01T09:50:04.166 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:03 smithi138 ceph-mon[185886]: failed to encode map e603 with expected crc 2023-10-01T09:50:04.166 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:03 smithi138 ceph-mon[185886]: failed to encode map e603 with expected crc 2023-10-01T09:50:04.166 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:03 smithi138 ceph-mon[185886]: failed to encode map e603 with expected crc 2023-10-01T09:50:04.166 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:03 smithi138 ceph-mon[185886]: osdmap e603: 8 total, 8 up, 8 in 2023-10-01T09:50:04.925 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.3 to 1 (10000) 2023-10-01T09:50:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:04 smithi190 ceph-mon[160435]: pgmap v1785: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/148590230' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:50:05.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:04 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:50:05.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:04 smithi138 ceph-mon[192513]: pgmap v1785: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:05.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/148590230' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:50:05.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:04 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:50:05.320 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:50:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:50:04] "GET /metrics HTTP/1.1" 200 34037 "" "Prometheus/2.33.4" 2023-10-01T09:50:05.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:04 smithi138 ceph-mon[185886]: pgmap v1785: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:05.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/148590230' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:50:05.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:04 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:50:05.557 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-01T09:50:05.911 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:06.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:05 smithi138 ceph-mon[185886]: failed to encode map e604 with expected crc 2023-10-01T09:50:06.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:05 smithi138 ceph-mon[185886]: failed to encode map e604 with expected crc 2023-10-01T09:50:06.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:05 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:50:06.265 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:05 smithi138 ceph-mon[185886]: osdmap e604: 8 total, 8 up, 8 in 2023-10-01T09:50:06.265 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:05 smithi138 ceph-mon[192513]: failed to encode map e604 with expected crc 2023-10-01T09:50:06.266 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:05 smithi138 ceph-mon[192513]: failed to encode map e604 with expected crc 2023-10-01T09:50:06.266 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:05 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:50:06.266 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:05 smithi138 ceph-mon[192513]: osdmap e604: 8 total, 8 up, 8 in 2023-10-01T09:50:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:05 smithi190 ceph-mon[160435]: failed to encode map e604 with expected crc 2023-10-01T09:50:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:05 smithi190 ceph-mon[160435]: failed to encode map e604 with expected crc 2023-10-01T09:50:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:05 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:50:06.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:05 smithi190 ceph-mon[160435]: osdmap e604: 8 total, 8 up, 8 in 2023-10-01T09:50:07.171 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:06 smithi138 ceph-mon[185886]: failed to encode map e604 with expected crc 2023-10-01T09:50:07.172 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:06 smithi138 ceph-mon[185886]: pgmap v1787: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:07.172 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:06 smithi138 ceph-mon[192513]: failed to encode map e604 with expected crc 2023-10-01T09:50:07.172 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:06 smithi138 ceph-mon[192513]: pgmap v1787: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:06 smithi190 ceph-mon[160435]: failed to encode map e604 with expected crc 2023-10-01T09:50:07.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:06 smithi190 ceph-mon[160435]: pgmap v1787: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:07.942 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-01T09:50:08.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:07 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4018481907' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:50:08.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:07 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4018481907' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:50:08.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:07 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4018481907' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T09:50:08.604 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-01T09:50:08.938 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4018481907' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:50:08.938 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[185886]: failed to encode map e605 with expected crc 2023-10-01T09:50:08.938 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[185886]: failed to encode map e605 with expected crc 2023-10-01T09:50:08.938 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[185886]: osdmap e605: 8 total, 8 up, 8 in 2023-10-01T09:50:08.939 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[185886]: failed to encode map e605 with expected crc 2023-10-01T09:50:08.939 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[185886]: failed to encode map e605 with expected crc 2023-10-01T09:50:08.939 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[185886]: failed to encode map e605 with expected crc 2023-10-01T09:50:08.939 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[185886]: pgmap v1789: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:08.939 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4018481907' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:50:08.939 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[192513]: failed to encode map e605 with expected crc 2023-10-01T09:50:08.940 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[192513]: failed to encode map e605 with expected crc 2023-10-01T09:50:08.940 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[192513]: osdmap e605: 8 total, 8 up, 8 in 2023-10-01T09:50:08.940 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[192513]: failed to encode map e605 with expected crc 2023-10-01T09:50:08.940 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[192513]: failed to encode map e605 with expected crc 2023-10-01T09:50:08.940 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[192513]: failed to encode map e605 with expected crc 2023-10-01T09:50:08.940 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:08 smithi138 ceph-mon[192513]: pgmap v1789: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:08.967 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:08 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4018481907' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-01T09:50:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:08 smithi190 ceph-mon[160435]: failed to encode map e605 with expected crc 2023-10-01T09:50:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:08 smithi190 ceph-mon[160435]: failed to encode map e605 with expected crc 2023-10-01T09:50:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:08 smithi190 ceph-mon[160435]: osdmap e605: 8 total, 8 up, 8 in 2023-10-01T09:50:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:08 smithi190 ceph-mon[160435]: failed to encode map e605 with expected crc 2023-10-01T09:50:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:08 smithi190 ceph-mon[160435]: failed to encode map e605 with expected crc 2023-10-01T09:50:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:08 smithi190 ceph-mon[160435]: failed to encode map e605 with expected crc 2023-10-01T09:50:09.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:08 smithi190 ceph-mon[160435]: pgmap v1789: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:10.198 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:09 smithi138 ceph-mon[185886]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.198 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:09 smithi138 ceph-mon[185886]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.198 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:09 smithi138 ceph-mon[185886]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.198 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:09 smithi138 ceph-mon[185886]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.198 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:09 smithi138 ceph-mon[185886]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.198 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:09 smithi138 ceph-mon[185886]: osdmap e606: 8 total, 8 up, 8 in 2023-10-01T09:50:10.198 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:09 smithi138 ceph-mon[192513]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.199 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:09 smithi138 ceph-mon[192513]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.199 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:09 smithi138 ceph-mon[192513]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.199 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:09 smithi138 ceph-mon[192513]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.199 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:09 smithi138 ceph-mon[192513]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.199 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:09 smithi138 ceph-mon[192513]: osdmap e606: 8 total, 8 up, 8 in 2023-10-01T09:50:10.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:09 smithi190 ceph-mon[160435]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:09 smithi190 ceph-mon[160435]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:09 smithi190 ceph-mon[160435]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:09 smithi190 ceph-mon[160435]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:09 smithi190 ceph-mon[160435]: failed to encode map e606 with expected crc 2023-10-01T09:50:10.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:09 smithi190 ceph-mon[160435]: osdmap e606: 8 total, 8 up, 8 in 2023-10-01T09:50:10.945 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.4 to 1 (10000) 2023-10-01T09:50:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:50:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:10 smithi190 ceph-mon[160435]: pgmap v1791: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:11.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:10 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3345317688' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:50:11.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:50:11.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:10 smithi138 ceph-mon[192513]: pgmap v1791: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:11.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:10 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3345317688' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:50:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:50:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:10 smithi138 ceph-mon[185886]: pgmap v1791: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:11.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:10 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3345317688' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:50:11.571 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-01T09:50:11.920 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:11 smithi190 ceph-mon[160435]: failed to encode map e607 with expected crc 2023-10-01T09:50:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:11 smithi190 ceph-mon[160435]: failed to encode map e607 with expected crc 2023-10-01T09:50:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:11 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3345317688' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:50:12.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:11 smithi190 ceph-mon[160435]: osdmap e607: 8 total, 8 up, 8 in 2023-10-01T09:50:12.307 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:11 smithi138 ceph-mon[185886]: failed to encode map e607 with expected crc 2023-10-01T09:50:12.308 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:11 smithi138 ceph-mon[185886]: failed to encode map e607 with expected crc 2023-10-01T09:50:12.308 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:11 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3345317688' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:50:12.308 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:11 smithi138 ceph-mon[185886]: osdmap e607: 8 total, 8 up, 8 in 2023-10-01T09:50:12.308 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:11 smithi138 ceph-mon[192513]: failed to encode map e607 with expected crc 2023-10-01T09:50:12.308 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:11 smithi138 ceph-mon[192513]: failed to encode map e607 with expected crc 2023-10-01T09:50:12.308 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:11 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3345317688' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:50:12.308 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:11 smithi138 ceph-mon[192513]: osdmap e607: 8 total, 8 up, 8 in 2023-10-01T09:50:13.193 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:12 smithi138 ceph-mon[185886]: failed to encode map e607 with expected crc 2023-10-01T09:50:13.194 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:12 smithi138 ceph-mon[185886]: pgmap v1793: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:13.194 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:12 smithi138 ceph-mon[192513]: failed to encode map e607 with expected crc 2023-10-01T09:50:13.194 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:12 smithi138 ceph-mon[192513]: pgmap v1793: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:12 smithi190 ceph-mon[160435]: failed to encode map e607 with expected crc 2023-10-01T09:50:13.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:12 smithi190 ceph-mon[160435]: pgmap v1793: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:13.958 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-01T09:50:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:13 smithi190 ceph-mon[160435]: failed to encode map e607 with expected crc 2023-10-01T09:50:14.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:13 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/445090516' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:50:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:13 smithi138 ceph-mon[192513]: failed to encode map e607 with expected crc 2023-10-01T09:50:14.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:13 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/445090516' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:50:14.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:13 smithi138 ceph-mon[185886]: failed to encode map e607 with expected crc 2023-10-01T09:50:14.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:13 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/445090516' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T09:50:14.603 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-01T09:50:14.935 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:50:14 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:50:14] "GET /metrics HTTP/1.1" 200 34038 "" "Prometheus/2.33.4" 2023-10-01T09:50:14.947 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:15.207 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:14 smithi138 ceph-mon[192513]: failed to encode map e608 with expected crc 2023-10-01T09:50:15.207 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:14 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/445090516' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:50:15.207 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:14 smithi138 ceph-mon[192513]: osdmap e608: 8 total, 8 up, 8 in 2023-10-01T09:50:15.207 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:14 smithi138 ceph-mon[192513]: failed to encode map e608 with expected crc 2023-10-01T09:50:15.207 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:14 smithi138 ceph-mon[192513]: failed to encode map e608 with expected crc 2023-10-01T09:50:15.208 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:14 smithi138 ceph-mon[192513]: failed to encode map e608 with expected crc 2023-10-01T09:50:15.208 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:14 smithi138 ceph-mon[192513]: pgmap v1795: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:15.208 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:14 smithi138 ceph-mon[185886]: failed to encode map e608 with expected crc 2023-10-01T09:50:15.208 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:14 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/445090516' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:50:15.208 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:14 smithi138 ceph-mon[185886]: osdmap e608: 8 total, 8 up, 8 in 2023-10-01T09:50:15.208 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:14 smithi138 ceph-mon[185886]: failed to encode map e608 with expected crc 2023-10-01T09:50:15.208 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:14 smithi138 ceph-mon[185886]: failed to encode map e608 with expected crc 2023-10-01T09:50:15.208 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:14 smithi138 ceph-mon[185886]: failed to encode map e608 with expected crc 2023-10-01T09:50:15.209 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:14 smithi138 ceph-mon[185886]: pgmap v1795: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:14 smithi190 ceph-mon[160435]: failed to encode map e608 with expected crc 2023-10-01T09:50:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:14 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/445090516' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-01T09:50:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:14 smithi190 ceph-mon[160435]: osdmap e608: 8 total, 8 up, 8 in 2023-10-01T09:50:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:14 smithi190 ceph-mon[160435]: failed to encode map e608 with expected crc 2023-10-01T09:50:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:14 smithi190 ceph-mon[160435]: failed to encode map e608 with expected crc 2023-10-01T09:50:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:14 smithi190 ceph-mon[160435]: failed to encode map e608 with expected crc 2023-10-01T09:50:15.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:14 smithi190 ceph-mon[160435]: pgmap v1795: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:16.214 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:15 smithi138 ceph-mon[185886]: failed to encode map e609 with expected crc 2023-10-01T09:50:16.214 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:15 smithi138 ceph-mon[185886]: failed to encode map e609 with expected crc 2023-10-01T09:50:16.214 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:15 smithi138 ceph-mon[185886]: failed to encode map e609 with expected crc 2023-10-01T09:50:16.215 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:15 smithi138 ceph-mon[185886]: osdmap e609: 8 total, 8 up, 8 in 2023-10-01T09:50:16.215 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:15 smithi138 ceph-mon[185886]: failed to encode map e609 with expected crc 2023-10-01T09:50:16.215 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:15 smithi138 ceph-mon[192513]: failed to encode map e609 with expected crc 2023-10-01T09:50:16.215 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:15 smithi138 ceph-mon[192513]: failed to encode map e609 with expected crc 2023-10-01T09:50:16.215 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:15 smithi138 ceph-mon[192513]: failed to encode map e609 with expected crc 2023-10-01T09:50:16.215 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:15 smithi138 ceph-mon[192513]: osdmap e609: 8 total, 8 up, 8 in 2023-10-01T09:50:16.216 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:15 smithi138 ceph-mon[192513]: failed to encode map e609 with expected crc 2023-10-01T09:50:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:15 smithi190 ceph-mon[160435]: failed to encode map e609 with expected crc 2023-10-01T09:50:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:15 smithi190 ceph-mon[160435]: failed to encode map e609 with expected crc 2023-10-01T09:50:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:15 smithi190 ceph-mon[160435]: failed to encode map e609 with expected crc 2023-10-01T09:50:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:15 smithi190 ceph-mon[160435]: osdmap e609: 8 total, 8 up, 8 in 2023-10-01T09:50:16.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:15 smithi190 ceph-mon[160435]: failed to encode map e609 with expected crc 2023-10-01T09:50:16.972 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.7 to 1 (10000) 2023-10-01T09:50:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:16 smithi190 ceph-mon[160435]: pgmap v1797: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:16 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4169080312' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:50:17.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:16 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:50:17.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:16 smithi138 ceph-mon[192513]: pgmap v1797: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:17.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:16 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4169080312' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:50:17.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:16 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:50:17.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:16 smithi138 ceph-mon[185886]: pgmap v1797: 73 pgs: 73 active+clean; 8.6 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:17.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:16 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4169080312' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:50:17.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:16 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:50:17.605 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-01T09:50:17.957 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:17.966 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:17 smithi138 ceph-mon[185886]: failed to encode map e610 with expected crc 2023-10-01T09:50:18.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:17 smithi138 ceph-mon[185886]: failed to encode map e610 with expected crc 2023-10-01T09:50:18.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:17 smithi138 ceph-mon[185886]: failed to encode map e610 with expected crc 2023-10-01T09:50:18.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:17 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:50:18.222 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:17 smithi138 ceph-mon[185886]: osdmap e610: 8 total, 8 up, 8 in 2023-10-01T09:50:18.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:17 smithi138 ceph-mon[192513]: failed to encode map e610 with expected crc 2023-10-01T09:50:18.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:17 smithi138 ceph-mon[192513]: failed to encode map e610 with expected crc 2023-10-01T09:50:18.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:17 smithi138 ceph-mon[192513]: failed to encode map e610 with expected crc 2023-10-01T09:50:18.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:17 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:50:18.223 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:17 smithi138 ceph-mon[192513]: osdmap e610: 8 total, 8 up, 8 in 2023-10-01T09:50:18.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:17 smithi190 ceph-mon[160435]: failed to encode map e610 with expected crc 2023-10-01T09:50:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:17 smithi190 ceph-mon[160435]: failed to encode map e610 with expected crc 2023-10-01T09:50:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:17 smithi190 ceph-mon[160435]: failed to encode map e610 with expected crc 2023-10-01T09:50:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:17 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:50:18.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:17 smithi190 ceph-mon[160435]: osdmap e610: 8 total, 8 up, 8 in 2023-10-01T09:50:19.248 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:18 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:50:19.248 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:18 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:19.248 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:18 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:19.248 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:18 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:19.248 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:18 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:19.248 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:18 smithi138 ceph-mon[192513]: pgmap v1799: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:19.248 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:18 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:50:19.249 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:18 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:19.249 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:18 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:19.249 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:18 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:19.249 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:18 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:19.249 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:18 smithi138 ceph-mon[185886]: pgmap v1799: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:18 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T09:50:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:18 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:18 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:18 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:18 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:19.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:18 smithi190 ceph-mon[160435]: pgmap v1799: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:20.416 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-01T09:50:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:20 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3974164846' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:50:20.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:20 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:50:20.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:20 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3974164846' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:50:20.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:20 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:50:20.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:20 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3974164846' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:50:20.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:20 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T09:50:21.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:21 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:50:21.672 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:21 smithi138 ceph-mon[185886]: osdmap e611: 8 total, 8 up, 8 in 2023-10-01T09:50:21.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:21 smithi138 ceph-mon[185886]: failed to encode map e611 with expected crc 2023-10-01T09:50:21.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:21 smithi138 ceph-mon[185886]: pgmap v1801: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:21.673 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:21 smithi138 ceph-mon[185886]: failed to encode map e611 with expected crc 2023-10-01T09:50:21.673 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:21 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:50:21.673 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:21 smithi138 ceph-mon[192513]: osdmap e611: 8 total, 8 up, 8 in 2023-10-01T09:50:21.673 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:21 smithi138 ceph-mon[192513]: failed to encode map e611 with expected crc 2023-10-01T09:50:21.673 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:21 smithi138 ceph-mon[192513]: pgmap v1801: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:21.673 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:21 smithi138 ceph-mon[192513]: failed to encode map e611 with expected crc 2023-10-01T09:50:21.674 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-01T09:50:21.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:21 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-01T09:50:21.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:21 smithi190 ceph-mon[160435]: osdmap e611: 8 total, 8 up, 8 in 2023-10-01T09:50:21.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:21 smithi190 ceph-mon[160435]: failed to encode map e611 with expected crc 2023-10-01T09:50:21.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:21 smithi190 ceph-mon[160435]: pgmap v1801: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:21.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:21 smithi190 ceph-mon[160435]: failed to encode map e611 with expected crc 2023-10-01T09:50:22.235 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:22 smithi190 ceph-mon[160435]: failed to encode map e611 with expected crc 2023-10-01T09:50:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:22 smithi190 ceph-mon[160435]: failed to encode map e611 with expected crc 2023-10-01T09:50:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:22.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:22 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:22 smithi138 ceph-mon[192513]: failed to encode map e611 with expected crc 2023-10-01T09:50:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:22 smithi138 ceph-mon[192513]: failed to encode map e611 with expected crc 2023-10-01T09:50:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:22.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:22 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:22.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:22 smithi138 ceph-mon[185886]: failed to encode map e611 with expected crc 2023-10-01T09:50:22.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:22.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:22.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:22 smithi138 ceph-mon[185886]: failed to encode map e611 with expected crc 2023-10-01T09:50:22.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:22.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:22 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:23.631 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:23 smithi138 ceph-mon[192513]: pgmap v1802: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:23.631 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:23 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:50:23.631 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:23 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:50:23.631 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:23 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:23.632 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:23 smithi138 ceph-mon[185886]: pgmap v1802: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:23.632 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:23 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:50:23.632 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:23 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:50:23.632 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:23 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:23 smithi190 ceph-mon[160435]: pgmap v1802: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:23 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T09:50:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:23 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T09:50:23.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:23 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:24.500 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.6 to 1 (10000) 2023-10-01T09:50:24.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:24 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1178524857' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:50:24.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:24 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1178524857' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:50:24.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:24 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1178524857' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:50:25.146 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-01T09:50:25.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:50:24 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:50:24] "GET /metrics HTTP/1.1" 200 34038 "" "Prometheus/2.33.4" 2023-10-01T09:50:25.500 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:25.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:25 smithi138 ceph-mon[192513]: pgmap v1803: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:25.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:25 smithi138 ceph-mon[192513]: failed to encode map e612 with expected crc 2023-10-01T09:50:25.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:25 smithi138 ceph-mon[192513]: failed to encode map e612 with expected crc 2023-10-01T09:50:25.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:25 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1178524857' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:50:25.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:25 smithi138 ceph-mon[192513]: osdmap e612: 8 total, 8 up, 8 in 2023-10-01T09:50:25.745 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:25 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:50:25.745 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:25 smithi138 ceph-mon[185886]: pgmap v1803: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:25.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:25 smithi138 ceph-mon[185886]: failed to encode map e612 with expected crc 2023-10-01T09:50:25.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:25 smithi138 ceph-mon[185886]: failed to encode map e612 with expected crc 2023-10-01T09:50:25.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:25 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1178524857' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:50:25.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:25 smithi138 ceph-mon[185886]: osdmap e612: 8 total, 8 up, 8 in 2023-10-01T09:50:25.746 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:25 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:50:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:25 smithi190 ceph-mon[160435]: pgmap v1803: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:25 smithi190 ceph-mon[160435]: failed to encode map e612 with expected crc 2023-10-01T09:50:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:25 smithi190 ceph-mon[160435]: failed to encode map e612 with expected crc 2023-10-01T09:50:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:25 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1178524857' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:50:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:25 smithi190 ceph-mon[160435]: osdmap e612: 8 total, 8 up, 8 in 2023-10-01T09:50:25.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:25 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:50:27.507 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-01T09:50:27.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:27 smithi190 ceph-mon[160435]: pgmap v1805: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:27.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:27 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3394297048' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:50:27.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:27 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:50:27.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:27 smithi138 ceph-mon[185886]: pgmap v1805: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:27.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:27 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3394297048' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:50:27.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:27 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:50:27.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:27 smithi138 ceph-mon[192513]: pgmap v1805: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:27.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:27 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3394297048' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:50:27.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:27 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T09:50:28.143 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-01T09:50:28.493 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:28.765 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:28 smithi138 ceph-mon[185886]: failed to encode map e613 with expected crc 2023-10-01T09:50:28.766 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:28 smithi138 ceph-mon[185886]: failed to encode map e613 with expected crc 2023-10-01T09:50:28.766 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:28 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:50:28.766 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:28 smithi138 ceph-mon[185886]: osdmap e613: 8 total, 8 up, 8 in 2023-10-01T09:50:28.766 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:28 smithi138 ceph-mon[192513]: failed to encode map e613 with expected crc 2023-10-01T09:50:28.766 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:28 smithi138 ceph-mon[192513]: failed to encode map e613 with expected crc 2023-10-01T09:50:28.767 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:28 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:50:28.767 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:28 smithi138 ceph-mon[192513]: osdmap e613: 8 total, 8 up, 8 in 2023-10-01T09:50:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:28 smithi190 ceph-mon[160435]: failed to encode map e613 with expected crc 2023-10-01T09:50:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:28 smithi190 ceph-mon[160435]: failed to encode map e613 with expected crc 2023-10-01T09:50:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:28 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-01T09:50:28.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:28 smithi190 ceph-mon[160435]: osdmap e613: 8 total, 8 up, 8 in 2023-10-01T09:50:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:29 smithi190 ceph-mon[160435]: pgmap v1807: 73 pgs: 6 peering, 67 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:29 smithi190 ceph-mon[160435]: failed to encode map e614 with expected crc 2023-10-01T09:50:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:29 smithi190 ceph-mon[160435]: failed to encode map e614 with expected crc 2023-10-01T09:50:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:29 smithi190 ceph-mon[160435]: osdmap e614: 8 total, 8 up, 8 in 2023-10-01T09:50:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:29 smithi190 ceph-mon[160435]: failed to encode map e614 with expected crc 2023-10-01T09:50:29.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:29 smithi190 ceph-mon[160435]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-01T09:50:29.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:29 smithi138 ceph-mon[192513]: pgmap v1807: 73 pgs: 6 peering, 67 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:29 smithi138 ceph-mon[192513]: failed to encode map e614 with expected crc 2023-10-01T09:50:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:29 smithi138 ceph-mon[192513]: failed to encode map e614 with expected crc 2023-10-01T09:50:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:29 smithi138 ceph-mon[192513]: osdmap e614: 8 total, 8 up, 8 in 2023-10-01T09:50:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:29 smithi138 ceph-mon[192513]: failed to encode map e614 with expected crc 2023-10-01T09:50:29.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:29 smithi138 ceph-mon[192513]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-01T09:50:29.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:29 smithi138 ceph-mon[185886]: pgmap v1807: 73 pgs: 6 peering, 67 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:29.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:29 smithi138 ceph-mon[185886]: failed to encode map e614 with expected crc 2023-10-01T09:50:29.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:29 smithi138 ceph-mon[185886]: failed to encode map e614 with expected crc 2023-10-01T09:50:29.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:29 smithi138 ceph-mon[185886]: osdmap e614: 8 total, 8 up, 8 in 2023-10-01T09:50:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:29 smithi138 ceph-mon[185886]: failed to encode map e614 with expected crc 2023-10-01T09:50:29.821 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:29 smithi138 ceph-mon[185886]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-01T09:50:30.526 INFO:teuthology.orchestra.run.smithi138.stderr:reweighted osd.5 to 1 (10000) 2023-10-01T09:50:30.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:30 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3947393761' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:50:30.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:30 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3947393761' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:50:30.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:30 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3947393761' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:50:31.284 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-01T09:50:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:31 smithi138 ceph-mon[192513]: pgmap v1809: 73 pgs: 6 peering, 67 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:31 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3947393761' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:50:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:31 smithi138 ceph-mon[192513]: osdmap e615: 8 total, 8 up, 8 in 2023-10-01T09:50:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:31 smithi138 ceph-mon[192513]: failed to encode map e615 with expected crc 2023-10-01T09:50:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:31 smithi138 ceph-mon[192513]: failed to encode map e615 with expected crc 2023-10-01T09:50:31.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:31 smithi138 ceph-mon[192513]: failed to encode map e615 with expected crc 2023-10-01T09:50:31.571 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:31 smithi138 ceph-mon[192513]: failed to encode map e615 with expected crc 2023-10-01T09:50:31.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:31 smithi138 ceph-mon[185886]: pgmap v1809: 73 pgs: 6 peering, 67 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:31.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:31 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3947393761' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:50:31.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:31 smithi138 ceph-mon[185886]: osdmap e615: 8 total, 8 up, 8 in 2023-10-01T09:50:31.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:31 smithi138 ceph-mon[185886]: failed to encode map e615 with expected crc 2023-10-01T09:50:31.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:31 smithi138 ceph-mon[185886]: failed to encode map e615 with expected crc 2023-10-01T09:50:31.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:31 smithi138 ceph-mon[185886]: failed to encode map e615 with expected crc 2023-10-01T09:50:31.571 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:31 smithi138 ceph-mon[185886]: failed to encode map e615 with expected crc 2023-10-01T09:50:31.634 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:31 smithi190 ceph-mon[160435]: pgmap v1809: 73 pgs: 6 peering, 67 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:31 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3947393761' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:50:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:31 smithi190 ceph-mon[160435]: osdmap e615: 8 total, 8 up, 8 in 2023-10-01T09:50:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:31 smithi190 ceph-mon[160435]: failed to encode map e615 with expected crc 2023-10-01T09:50:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:31 smithi190 ceph-mon[160435]: failed to encode map e615 with expected crc 2023-10-01T09:50:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:31 smithi190 ceph-mon[160435]: failed to encode map e615 with expected crc 2023-10-01T09:50:31.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:31 smithi190 ceph-mon[160435]: failed to encode map e615 with expected crc 2023-10-01T09:50:32.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:32 smithi138 ceph-mon[192513]: pgmap v1811: 73 pgs: 6 peering, 67 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:32.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:32 smithi138 ceph-mon[185886]: pgmap v1811: 73 pgs: 6 peering, 67 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:33.057 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:32 smithi190 ceph-mon[160435]: pgmap v1811: 73 pgs: 6 peering, 67 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:33.893 INFO:teuthology.orchestra.run.smithi138.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-01T09:50:34.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:33 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2801494476' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:50:34.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:33 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:50:34.319 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:33 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2801494476' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:50:34.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:33 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:50:34.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:33 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2801494476' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:50:34.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:33 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T09:50:34.618 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-10-01T09:50:34.618 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-10-01T09:50:34.618 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-10-01T09:50:34.618 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-10-01T09:50:34.619 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-01T09:50:34.619 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T09:50:34.976 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:35.185 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:34 smithi138 ceph-mon[192513]: failed to encode map e616 with expected crc 2023-10-01T09:50:35.185 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:34 smithi138 ceph-mon[192513]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:50:35.185 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:34 smithi138 ceph-mon[192513]: osdmap e616: 8 total, 8 up, 8 in 2023-10-01T09:50:35.185 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:34 smithi138 ceph-mon[192513]: pgmap v1813: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:35.185 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:34 smithi138 ceph-mon[185886]: failed to encode map e616 with expected crc 2023-10-01T09:50:35.186 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:34 smithi138 ceph-mon[185886]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:50:35.186 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:34 smithi138 ceph-mon[185886]: osdmap e616: 8 total, 8 up, 8 in 2023-10-01T09:50:35.186 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:34 smithi138 ceph-mon[185886]: pgmap v1813: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:35.186 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:50:34 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:50:34] "GET /metrics HTTP/1.1" 200 34035 "" "Prometheus/2.33.4" 2023-10-01T09:50:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:34 smithi190 ceph-mon[160435]: failed to encode map e616 with expected crc 2023-10-01T09:50:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:34 smithi190 ceph-mon[160435]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-01T09:50:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:34 smithi190 ceph-mon[160435]: osdmap e616: 8 total, 8 up, 8 in 2023-10-01T09:50:35.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:34 smithi190 ceph-mon[160435]: pgmap v1813: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:36.245 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:35 smithi138 ceph-mon[192513]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-10-01T09:50:36.245 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:35 smithi138 ceph-mon[192513]: Cluster is now healthy 2023-10-01T09:50:36.245 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:35 smithi138 ceph-mon[185886]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-10-01T09:50:36.245 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:35 smithi138 ceph-mon[185886]: Cluster is now healthy 2023-10-01T09:50:36.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:35 smithi190 ceph-mon[160435]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-10-01T09:50:36.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:35 smithi190 ceph-mon[160435]: Cluster is now healthy 2023-10-01T09:50:36.629 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:50:36.629 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":616,"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","created":"2023-10-01T08:56:55.039846+0000","modified":"2023-10-01T09:50:33.880160+0000","last_up_change":"2023-10-01T09:33:10.480571+0000","last_in_change":"2023-10-01T09:49:37.470309+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-01T08:59:41.882085+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T09:02:05.272330+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"584","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":584,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-01T09:03:01.537913+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-01T09:03:02.452550+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"12f71e77-6099-48ee-a89e-e931e000e9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":85,"up_from":89,"up_thru":596,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6803","nonce":2668617192}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6805","nonce":2668617192}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6809","nonce":2668617192}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6807","nonce":2668617192}]},"public_addr":"172.21.15.138:6803/2668617192","cluster_addr":"172.21.15.138:6805/2668617192","heartbeat_back_addr":"172.21.15.138:6809/2668617192","heartbeat_front_addr":"172.21.15.138:6807/2668617192","state":["exists","up"]},{"osd":1,"uuid":"525b5e81-4fa5-4b10-9147-83fcb549ca30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":90,"up_from":94,"up_thru":590,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6811","nonce":2021985596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6813","nonce":2021985596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6817","nonce":2021985596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6815","nonce":2021985596}]},"public_addr":"172.21.15.138:6811/2021985596","cluster_addr":"172.21.15.138:6813/2021985596","heartbeat_back_addr":"172.21.15.138:6817/2021985596","heartbeat_front_addr":"172.21.15.138:6815/2021985596","state":["exists","up"]},{"osd":2,"uuid":"97443e05-2513-4832-bb54-f65493df5967","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":95,"up_from":99,"up_thru":602,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6819","nonce":4290419390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6821","nonce":4290419390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6825","nonce":4290419390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6823","nonce":4290419390}]},"public_addr":"172.21.15.138:6819/4290419390","cluster_addr":"172.21.15.138:6821/4290419390","heartbeat_back_addr":"172.21.15.138:6825/4290419390","heartbeat_front_addr":"172.21.15.138:6823/4290419390","state":["exists","up"]},{"osd":3,"uuid":"ac51c165-e007-4390-b95b-26eef984d637","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":100,"up_from":104,"up_thru":605,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6827","nonce":244903146}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6829","nonce":244903146}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6832","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6833","nonce":244903146}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6831","nonce":244903146}]},"public_addr":"172.21.15.138:6827/244903146","cluster_addr":"172.21.15.138:6829/244903146","heartbeat_back_addr":"172.21.15.138:6833/244903146","heartbeat_front_addr":"172.21.15.138:6831/244903146","state":["exists","up"]},{"osd":4,"uuid":"4bef630c-bff4-41a3-a932-de0501541f78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":608,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6801","nonce":743095145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6803","nonce":743095145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6807","nonce":743095145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6805","nonce":743095145}]},"public_addr":"172.21.15.190:6801/743095145","cluster_addr":"172.21.15.190:6803/743095145","heartbeat_back_addr":"172.21.15.190:6807/743095145","heartbeat_front_addr":"172.21.15.190:6805/743095145","state":["exists","up"]},{"osd":5,"uuid":"ac6e16ed-681d-4e9d-87a3-d20d50e01437","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":227,"last_clean_end":368,"up_from":379,"up_thru":593,"down_at":369,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6808","nonce":3401418768},{"type":"v1","addr":"172.21.15.190:6809","nonce":3401418768}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6810","nonce":3401418768},{"type":"v1","addr":"172.21.15.190:6811","nonce":3401418768}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6814","nonce":3401418768},{"type":"v1","addr":"172.21.15.190:6815","nonce":3401418768}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6812","nonce":3401418768},{"type":"v1","addr":"172.21.15.190:6813","nonce":3401418768}]},"public_addr":"172.21.15.190:6809/3401418768","cluster_addr":"172.21.15.190:6811/3401418768","heartbeat_back_addr":"172.21.15.190:6815/3401418768","heartbeat_front_addr":"172.21.15.190:6813/3401418768","state":["exists","up"]},{"osd":6,"uuid":"7a6f65a5-16dc-45de-854d-751d9151e85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":42,"last_clean_end":268,"up_from":294,"up_thru":613,"down_at":269,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6816","nonce":1281430772},{"type":"v1","addr":"172.21.15.190:6817","nonce":1281430772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6818","nonce":1281430772},{"type":"v1","addr":"172.21.15.190:6819","nonce":1281430772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6822","nonce":1281430772},{"type":"v1","addr":"172.21.15.190:6823","nonce":1281430772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6820","nonce":1281430772},{"type":"v1","addr":"172.21.15.190:6821","nonce":1281430772}]},"public_addr":"172.21.15.190:6817/1281430772","cluster_addr":"172.21.15.190:6819/1281430772","heartbeat_back_addr":"172.21.15.190:6823/1281430772","heartbeat_front_addr":"172.21.15.190:6821/1281430772","state":["exists","up"]},{"osd":7,"uuid":"09a5ad5d-d58f-4fcf-8b09-10f395bd01f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":600,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6824","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6825","nonce":185238034}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6826","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6827","nonce":185238034}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6830","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6831","nonce":185238034}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6828","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6829","nonce":185238034}]},"public_addr":"172.21.15.190:6825/185238034","cluster_addr":"172.21.15.190:6827/185238034","heartbeat_back_addr":"172.21.15.190:6831/185238034","heartbeat_front_addr":"172.21.15.190:6829/185238034","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:58:54.313619+0000","dead_epoch":85},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:16.837452+0000","dead_epoch":90},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:39.610865+0000","dead_epoch":95},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:03.127916+0000","dead_epoch":100},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:23.838460+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:45.582295+0000","dead_epoch":368},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:07.078976+0000","dead_epoch":268},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:29.080850+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.138:6801/3849457598":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/319567031":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/2091838076":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/1464782684":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/25071367":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/3940341494":"2023-10-02T08:57:39.299924+0000","172.21.15.190:0/3151728630":"2023-10-02T09:07:17.185267+0000","172.21.15.138:6801/3165627846":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6800/3165627846":"2023-10-02T08:57:18.125987+0000","172.21.15.138:0/3256141027":"2023-10-02T09:07:55.156656+0000","172.21.15.190:6835/2242576319":"2023-10-02T09:07:17.185267+0000","172.21.15.190:0/2286153897":"2023-10-02T09:07:17.185267+0000","172.21.15.138:0/2575047401":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6801/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.190:0/2209516562":"2023-10-02T09:07:17.185267+0000","172.21.15.190:0/3337553974":"2023-10-02T09:07:17.185267+0000","172.21.15.138:0/2018177063":"2023-10-02T08:57:18.125987+0000","172.21.15.138:0/523071753":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/2054656381":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/1836785572":"2023-10-02T09:06:07.582189+0000","172.21.15.138:6800/3849457598":"2023-10-02T09:07:55.156656+0000","172.21.15.138:6800/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.190:6834/2242576319":"2023-10-02T09:07:17.185267+0000","172.21.15.190:0/299211527":"2023-10-02T09:07:17.185267+0000","172.21.15.138:6801/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:0/3533950111":"2023-10-02T09:06:07.582189+0000","172.21.15.190:0/710760207":"2023-10-02T09:07:17.185267+0000","172.21.15.138:6800/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:0/3675584653":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6800/974662983":"2023-10-02T09:06:07.582189+0000","172.21.15.138:6801/974662983":"2023-10-02T09:06:07.582189+0000","172.21.15.138:0/2334114390":"2023-10-02T09:01:44.817357+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T09:50:37.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:36 smithi138 ceph-mon[185886]: pgmap v1814: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:37.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:36 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1863253203' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:50:37.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:36 smithi138 ceph-mon[192513]: pgmap v1814: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:37.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:36 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1863253203' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:50:37.261 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-01T09:50:37.261 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T09:50:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:36 smithi190 ceph-mon[160435]: pgmap v1814: 73 pgs: 73 active+clean; 8.6 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:37.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:36 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1863253203' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:50:37.612 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:39.267 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:50:39.267 INFO:teuthology.orchestra.run.smithi138.stdout:{"epoch":616,"fsid":"465870a4-6038-11ee-8db5-212e2dc638e7","created":"2023-10-01T08:56:55.039846+0000","modified":"2023-10-01T09:50:33.880160+0000","last_up_change":"2023-10-01T09:33:10.480571+0000","last_in_change":"2023-10-01T09:49:37.470309+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-01T08:59:41.882085+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"82","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T09:02:05.272330+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"584","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":584,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-01T09:03:01.537913+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-01T09:03:02.452550+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"12f71e77-6099-48ee-a89e-e931e000e9fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":85,"up_from":89,"up_thru":596,"down_at":86,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6802","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6803","nonce":2668617192}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6804","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6805","nonce":2668617192}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6808","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6809","nonce":2668617192}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6806","nonce":2668617192},{"type":"v1","addr":"172.21.15.138:6807","nonce":2668617192}]},"public_addr":"172.21.15.138:6803/2668617192","cluster_addr":"172.21.15.138:6805/2668617192","heartbeat_back_addr":"172.21.15.138:6809/2668617192","heartbeat_front_addr":"172.21.15.138:6807/2668617192","state":["exists","up"]},{"osd":1,"uuid":"525b5e81-4fa5-4b10-9147-83fcb549ca30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":90,"up_from":94,"up_thru":590,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6810","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6811","nonce":2021985596}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6812","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6813","nonce":2021985596}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6816","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6817","nonce":2021985596}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6814","nonce":2021985596},{"type":"v1","addr":"172.21.15.138:6815","nonce":2021985596}]},"public_addr":"172.21.15.138:6811/2021985596","cluster_addr":"172.21.15.138:6813/2021985596","heartbeat_back_addr":"172.21.15.138:6817/2021985596","heartbeat_front_addr":"172.21.15.138:6815/2021985596","state":["exists","up"]},{"osd":2,"uuid":"97443e05-2513-4832-bb54-f65493df5967","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":95,"up_from":99,"up_thru":602,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6818","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6819","nonce":4290419390}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6820","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6821","nonce":4290419390}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6824","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6825","nonce":4290419390}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6822","nonce":4290419390},{"type":"v1","addr":"172.21.15.138:6823","nonce":4290419390}]},"public_addr":"172.21.15.138:6819/4290419390","cluster_addr":"172.21.15.138:6821/4290419390","heartbeat_back_addr":"172.21.15.138:6825/4290419390","heartbeat_front_addr":"172.21.15.138:6823/4290419390","state":["exists","up"]},{"osd":3,"uuid":"ac51c165-e007-4390-b95b-26eef984d637","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":100,"up_from":104,"up_thru":605,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6826","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6827","nonce":244903146}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6828","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6829","nonce":244903146}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6832","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6833","nonce":244903146}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.138:6830","nonce":244903146},{"type":"v1","addr":"172.21.15.138:6831","nonce":244903146}]},"public_addr":"172.21.15.138:6827/244903146","cluster_addr":"172.21.15.138:6829/244903146","heartbeat_back_addr":"172.21.15.138:6833/244903146","heartbeat_front_addr":"172.21.15.138:6831/244903146","state":["exists","up"]},{"osd":4,"uuid":"4bef630c-bff4-41a3-a932-de0501541f78","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":608,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6801","nonce":743095145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6803","nonce":743095145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6807","nonce":743095145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":743095145},{"type":"v1","addr":"172.21.15.190:6805","nonce":743095145}]},"public_addr":"172.21.15.190:6801/743095145","cluster_addr":"172.21.15.190:6803/743095145","heartbeat_back_addr":"172.21.15.190:6807/743095145","heartbeat_front_addr":"172.21.15.190:6805/743095145","state":["exists","up"]},{"osd":5,"uuid":"ac6e16ed-681d-4e9d-87a3-d20d50e01437","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":227,"last_clean_end":368,"up_from":379,"up_thru":593,"down_at":369,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6808","nonce":3401418768},{"type":"v1","addr":"172.21.15.190:6809","nonce":3401418768}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6810","nonce":3401418768},{"type":"v1","addr":"172.21.15.190:6811","nonce":3401418768}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6814","nonce":3401418768},{"type":"v1","addr":"172.21.15.190:6815","nonce":3401418768}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6812","nonce":3401418768},{"type":"v1","addr":"172.21.15.190:6813","nonce":3401418768}]},"public_addr":"172.21.15.190:6809/3401418768","cluster_addr":"172.21.15.190:6811/3401418768","heartbeat_back_addr":"172.21.15.190:6815/3401418768","heartbeat_front_addr":"172.21.15.190:6813/3401418768","state":["exists","up"]},{"osd":6,"uuid":"7a6f65a5-16dc-45de-854d-751d9151e85b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":42,"last_clean_end":268,"up_from":294,"up_thru":613,"down_at":269,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6816","nonce":1281430772},{"type":"v1","addr":"172.21.15.190:6817","nonce":1281430772}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6818","nonce":1281430772},{"type":"v1","addr":"172.21.15.190:6819","nonce":1281430772}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6822","nonce":1281430772},{"type":"v1","addr":"172.21.15.190:6823","nonce":1281430772}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6820","nonce":1281430772},{"type":"v1","addr":"172.21.15.190:6821","nonce":1281430772}]},"public_addr":"172.21.15.190:6817/1281430772","cluster_addr":"172.21.15.190:6819/1281430772","heartbeat_back_addr":"172.21.15.190:6823/1281430772","heartbeat_front_addr":"172.21.15.190:6821/1281430772","state":["exists","up"]},{"osd":7,"uuid":"09a5ad5d-d58f-4fcf-8b09-10f395bd01f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":600,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6824","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6825","nonce":185238034}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6826","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6827","nonce":185238034}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6830","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6831","nonce":185238034}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6828","nonce":185238034},{"type":"v1","addr":"172.21.15.190:6829","nonce":185238034}]},"public_addr":"172.21.15.190:6825/185238034","cluster_addr":"172.21.15.190:6827/185238034","heartbeat_back_addr":"172.21.15.190:6831/185238034","heartbeat_front_addr":"172.21.15.190:6829/185238034","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:58:54.313619+0000","dead_epoch":85},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:16.837452+0000","dead_epoch":90},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T08:59:39.610865+0000","dead_epoch":95},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:03.127916+0000","dead_epoch":100},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:23.838460+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:00:45.582295+0000","dead_epoch":368},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:07.078976+0000","dead_epoch":268},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T09:01:29.080850+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.138:6801/3849457598":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/319567031":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/2091838076":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/1464782684":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/25071367":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/3940341494":"2023-10-02T08:57:39.299924+0000","172.21.15.190:0/3151728630":"2023-10-02T09:07:17.185267+0000","172.21.15.138:6801/3165627846":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6800/3165627846":"2023-10-02T08:57:18.125987+0000","172.21.15.138:0/3256141027":"2023-10-02T09:07:55.156656+0000","172.21.15.190:6835/2242576319":"2023-10-02T09:07:17.185267+0000","172.21.15.190:0/2286153897":"2023-10-02T09:07:17.185267+0000","172.21.15.138:0/2575047401":"2023-10-02T09:01:44.817357+0000","172.21.15.138:6801/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.190:0/2209516562":"2023-10-02T09:07:17.185267+0000","172.21.15.190:0/3337553974":"2023-10-02T09:07:17.185267+0000","172.21.15.138:0/2018177063":"2023-10-02T08:57:18.125987+0000","172.21.15.138:0/523071753":"2023-10-02T09:07:55.156656+0000","172.21.15.138:0/2054656381":"2023-10-02T08:57:39.299924+0000","172.21.15.138:0/1836785572":"2023-10-02T09:06:07.582189+0000","172.21.15.138:6800/3849457598":"2023-10-02T09:07:55.156656+0000","172.21.15.138:6800/62009114":"2023-10-02T08:57:39.299924+0000","172.21.15.190:6834/2242576319":"2023-10-02T09:07:17.185267+0000","172.21.15.190:0/299211527":"2023-10-02T09:07:17.185267+0000","172.21.15.138:6801/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:0/3533950111":"2023-10-02T09:06:07.582189+0000","172.21.15.190:0/710760207":"2023-10-02T09:07:17.185267+0000","172.21.15.138:6800/4033793848":"2023-10-02T09:01:44.817357+0000","172.21.15.138:0/3675584653":"2023-10-02T08:57:18.125987+0000","172.21.15.138:6800/974662983":"2023-10-02T09:06:07.582189+0000","172.21.15.138:6801/974662983":"2023-10-02T09:06:07.582189+0000","172.21.15.138:0/2334114390":"2023-10-02T09:01:44.817357+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T09:50:39.289 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:38 smithi138 ceph-mon[185886]: pgmap v1815: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:39.289 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:38 smithi138 ceph-mon[192513]: pgmap v1815: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:39.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:38 smithi190 ceph-mon[160435]: pgmap v1815: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:39.874 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-01T09:50:39.874 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-01T09:50:39.875 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-01T09:50:39.875 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-01T09:50:39.876 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-01T09:50:39.876 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-01T09:50:39.876 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-01T09:50:39.877 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-01T09:50:40.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:39 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/2816830109' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:50:40.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:39 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/2816830109' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:50:40.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:39 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/2816830109' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T09:50:41.302 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:41.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:40 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:50:41.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:40 smithi190 ceph-mon[160435]: pgmap v1816: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:41.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:40 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:50:41.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:40 smithi138 ceph-mon[192513]: pgmap v1816: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:41.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:40 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:50:41.321 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:40 smithi138 ceph-mon[185886]: pgmap v1816: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:41.353 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:41.427 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:41.461 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:41.566 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:41.980 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:42.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:42 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:42.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:42 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:42.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:42 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' 2023-10-01T09:50:42.913 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:43.177 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:43 smithi138 ceph-mon[192513]: pgmap v1817: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:43.367 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:43.557 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:43 smithi190 ceph-mon[160435]: pgmap v1817: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:43.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:43 smithi138 ceph-mon[185886]: pgmap v1817: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:44.935 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:50:44 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:50:44] "GET /metrics HTTP/1.1" 200 34038 "" "Prometheus/2.33.4" 2023-10-01T09:50:45.570 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:45 smithi138 ceph-mon[192513]: pgmap v1818: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:45.570 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:45 smithi138 ceph-mon[185886]: pgmap v1818: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:45.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:45 smithi190 ceph-mon[160435]: pgmap v1818: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:47.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:47 smithi190 ceph-mon[160435]: pgmap v1819: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:47.819 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:47 smithi138 ceph-mon[185886]: pgmap v1819: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:47.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:47 smithi138 ceph-mon[192513]: pgmap v1819: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:48.150 INFO:teuthology.orchestra.run.smithi138.stdout:1627792605395 2023-10-01T09:50:48.150 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-01T09:50:49.047 INFO:teuthology.orchestra.run.smithi138.stdout:1262720385313 2023-10-01T09:50:49.047 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-01T09:50:49.151 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:48 smithi138 ceph-mon[185886]: pgmap v1820: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:49.151 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:48 smithi138 ceph-mon[192513]: pgmap v1820: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:49.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:48 smithi190 ceph-mon[160435]: pgmap v1820: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:50.403 INFO:teuthology.orchestra.run.smithi138.stdout:382252089825 2023-10-01T09:50:50.404 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-01T09:50:51.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:51 smithi190 ceph-mon[160435]: pgmap v1821: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:51.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:51 smithi138 ceph-mon[192513]: pgmap v1821: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:51.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:51 smithi138 ceph-mon[185886]: pgmap v1821: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:52.207 INFO:teuthology.orchestra.run.smithi138.stdout:206158430803 2023-10-01T09:50:52.208 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-01T09:50:52.264 INFO:teuthology.orchestra.run.smithi138.stdout:403726926290 2023-10-01T09:50:52.264 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-01T09:50:52.644 INFO:teuthology.orchestra.run.smithi138.stdout:425201762765 2023-10-01T09:50:52.645 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-01T09:50:52.813 INFO:teuthology.orchestra.run.smithi138.stdout:446676599238 2023-10-01T09:50:52.813 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-01T09:50:52.873 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:52.901 INFO:teuthology.orchestra.run.smithi138.stdout:128849019488 2023-10-01T09:50:52.902 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-01T09:50:53.094 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:53.485 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:53 smithi138 ceph-mon[192513]: pgmap v1822: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:53.486 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:53 smithi138 ceph-mon[185886]: pgmap v1822: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:53.710 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:53.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:53 smithi190 ceph-mon[160435]: pgmap v1822: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:54.903 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:54.911 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:55.176 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:55.177 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:55.319 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:50:54 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:50:54] "GET /metrics HTTP/1.1" 200 34038 "" "Prometheus/2.33.4" 2023-10-01T09:50:55.461 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:50:55.742 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:55 smithi138 ceph-mon[192513]: pgmap v1823: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:55.742 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:55 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:50:55.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:55 smithi138 ceph-mon[185886]: pgmap v1823: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:55.743 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:55 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:50:55.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:55 smithi190 ceph-mon[160435]: pgmap v1823: 73 pgs: 73 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:55.807 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:55 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:50:56.157 INFO:teuthology.orchestra.run.smithi138.stdout:403726926291 2023-10-01T09:50:56.699 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:56 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3930593044' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T09:50:56.700 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:56 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3930593044' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T09:50:56.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:56 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3930593044' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T09:50:57.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:57 smithi190 ceph-mon[160435]: pgmap v1824: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:57.819 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:57 smithi138 ceph-mon[192513]: pgmap v1824: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:57.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:57 smithi138 ceph-mon[185886]: pgmap v1824: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:58.876 INFO:teuthology.orchestra.run.smithi138.stdout:1262720385315 2023-10-01T09:50:59.226 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:58 smithi138 ceph-mon[185886]: pgmap v1825: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:59.226 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:58 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4103505599' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T09:50:59.226 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:58 smithi138 ceph-mon[192513]: pgmap v1825: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:59.226 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:58 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4103505599' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T09:50:59.230 INFO:teuthology.orchestra.run.smithi138.stdout:1627792605397 2023-10-01T09:50:59.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:58 smithi190 ceph-mon[160435]: pgmap v1825: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:50:59.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:58 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4103505599' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T09:51:00.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:50:59 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3487255054' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T09:51:00.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:50:59 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3487255054' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T09:51:00.156 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1262720385313 got 1262720385315 for osd.6 2023-10-01T09:51:00.156 DEBUG:teuthology.parallel:result is None 2023-10-01T09:51:00.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:50:59 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3487255054' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T09:51:00.570 INFO:teuthology.orchestra.run.smithi138.stdout:425201762766 2023-10-01T09:51:00.704 INFO:teuthology.orchestra.run.smithi138.stdout:206158430805 2023-10-01T09:51:01.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:00 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3615213275' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T09:51:01.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:00 smithi138 ceph-mon[192513]: pgmap v1826: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:01.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:00 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4167607783' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T09:51:01.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:00 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3615213275' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T09:51:01.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:00 smithi138 ceph-mon[185886]: pgmap v1826: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:01.071 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:00 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4167607783' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T09:51:01.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:00 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3615213275' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T09:51:01.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:00 smithi190 ceph-mon[160435]: pgmap v1826: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:01.308 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:00 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4167607783' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T09:51:01.445 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726926290 got 403726926291 for osd.1 2023-10-01T09:51:01.445 DEBUG:teuthology.parallel:result is None 2023-10-01T09:51:02.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:01 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/4240376424' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T09:51:02.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:01 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/4240376424' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T09:51:02.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:01 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/4240376424' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T09:51:02.907 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:02 smithi138 ceph-mon[185886]: pgmap v1827: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:02.908 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:02 smithi138 ceph-mon[192513]: pgmap v1827: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:02.974 INFO:teuthology.orchestra.run.smithi138.stdout:446676599240 2023-10-01T09:51:03.040 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762765 got 425201762766 for osd.2 2023-10-01T09:51:03.041 DEBUG:teuthology.parallel:result is None 2023-10-01T09:51:03.084 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430803 got 206158430805 for osd.7 2023-10-01T09:51:03.084 DEBUG:teuthology.parallel:result is None 2023-10-01T09:51:03.252 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1627792605395 got 1627792605397 for osd.5 2023-10-01T09:51:03.253 DEBUG:teuthology.parallel:result is None 2023-10-01T09:51:03.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:02 smithi190 ceph-mon[160435]: pgmap v1827: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:03.431 INFO:teuthology.orchestra.run.smithi138.stdout:382252089828 2023-10-01T09:51:04.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:03 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/1329553403' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T09:51:04.320 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:03 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/1329553403' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T09:51:04.320 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:03 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/1329553403' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T09:51:04.414 INFO:tasks.cephadm.ceph_manager.ceph:need seq 446676599238 got 446676599240 for osd.3 2023-10-01T09:51:04.414 DEBUG:teuthology.parallel:result is None 2023-10-01T09:51:04.454 INFO:tasks.cephadm.ceph_manager.ceph:need seq 382252089825 got 382252089828 for osd.0 2023-10-01T09:51:04.454 DEBUG:teuthology.parallel:result is None 2023-10-01T09:51:04.765 INFO:teuthology.orchestra.run.smithi138.stdout:128849019491 2023-10-01T09:51:05.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:04 smithi138 ceph-mon[192513]: pgmap v1828: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:05.070 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:04 smithi138 ceph-mon[192513]: from='client.? 172.21.15.138:0/3342466329' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T09:51:05.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:04 smithi138 ceph-mon[185886]: pgmap v1828: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:05.070 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:04 smithi138 ceph-mon[185886]: from='client.? 172.21.15.138:0/3342466329' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T09:51:05.071 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:04 smithi138 conmon[178630]: ::ffff:172.21.15.190 - - [01/Oct/2023:09:51:04] "GET /metrics HTTP/1.1" 200 34041 "" "Prometheus/2.33.4" 2023-10-01T09:51:05.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:04 smithi190 ceph-mon[160435]: pgmap v1828: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:05.307 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:04 smithi190 ceph-mon[160435]: from='client.? 172.21.15.138:0/3342466329' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T09:51:05.404 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019488 got 128849019491 for osd.4 2023-10-01T09:51:05.404 DEBUG:teuthology.parallel:result is None 2023-10-01T09:51:05.404 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-01T09:51:05.404 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:51:05.765 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:51:07.394 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:51:07.395 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:51:07.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:07 smithi190 ceph-mon[160435]: pgmap v1829: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:07.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:07 smithi138 ceph-mon[192513]: pgmap v1829: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:07.820 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:07 smithi138 ceph-mon[185886]: pgmap v1829: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:08.044 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1829,"stamp":"2023-10-01T09:51:06.431673+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":12257,"num_read_kb":25874,"num_write":1950,"num_write_kb":27835,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":540,"num_bytes_recovered":20350555,"num_keys_recovered":110,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2144,"ondisk_log_size":2144,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":222,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6073292,"kb_used_data":154892,"kb_used_omap":50,"kb_used_meta":5918285,"kb_avail":743887924,"statfs":{"total":767960285184,"available":761741234176,"internally_reserved":0,"allocated":158609408,"data_stored":145629261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51738,"internal_metadata":6060324326},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002596"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":401,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.041117+0000","last_change":"2023-10-01T09:49:37.884604+0000","last_active":"2023-10-01T09:50:31.041117+0000","last_peered":"2023-10-01T09:50:31.041117+0000","last_clean":"2023-10-01T09:50:31.041117+0000","last_became_active":"2023-10-01T09:49:37.884169+0000","last_became_peered":"2023-10-01T09:49:37.884169+0000","last_unstale":"2023-10-01T09:50:31.041117+0000","last_undegraded":"2023-10-01T09:50:31.041117+0000","last_fullsized":"2023-10-01T09:50:31.041117+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":75,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523182+0000","last_change":"2023-10-01T09:50:08.985245+0000","last_active":"2023-10-01T09:50:30.523182+0000","last_peered":"2023-10-01T09:50:30.523182+0000","last_clean":"2023-10-01T09:50:30.523182+0000","last_became_active":"2023-10-01T09:50:08.984895+0000","last_became_peered":"2023-10-01T09:50:08.984895+0000","last_unstale":"2023-10-01T09:50:30.523182+0000","last_undegraded":"2023-10-01T09:50:30.523182+0000","last_fullsized":"2023-10-01T09:50:30.523182+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:17:35.675391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":619,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-10-01T09:50:35.020892+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:50:35.020892+0000","last_peered":"2023-10-01T09:50:35.020892+0000","last_clean":"2023-10-01T09:50:35.020892+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:50:35.020892+0000","last_undegraded":"2023-10-01T09:50:35.020892+0000","last_fullsized":"2023-10-01T09:50:35.020892+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":678,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-10-01T09:50:33.888792+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:50:33.888792+0000","last_peered":"2023-10-01T09:50:33.888792+0000","last_clean":"2023-10-01T09:50:33.888792+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:50:33.888792+0000","last_undegraded":"2023-10-01T09:50:33.888792+0000","last_fullsized":"2023-10-01T09:50:33.888792+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":440,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523460+0000","last_change":"2023-10-01T09:50:08.986396+0000","last_active":"2023-10-01T09:50:30.523460+0000","last_peered":"2023-10-01T09:50:30.523460+0000","last_clean":"2023-10-01T09:50:30.523460+0000","last_became_active":"2023-10-01T09:50:08.985734+0000","last_became_peered":"2023-10-01T09:50:08.985734+0000","last_unstale":"2023-10-01T09:50:30.523460+0000","last_undegraded":"2023-10-01T09:50:30.523460+0000","last_fullsized":"2023-10-01T09:50:30.523460+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:47:50.441209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":149,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.527105+0000","last_change":"2023-10-01T09:50:29.052725+0000","last_active":"2023-10-01T09:50:30.527105+0000","last_peered":"2023-10-01T09:50:30.527105+0000","last_clean":"2023-10-01T09:50:30.527105+0000","last_became_active":"2023-10-01T09:50:29.052221+0000","last_became_peered":"2023-10-01T09:50:29.052221+0000","last_unstale":"2023-10-01T09:50:30.527105+0000","last_undegraded":"2023-10-01T09:50:30.527105+0000","last_fullsized":"2023-10-01T09:50:30.527105+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":563,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.038623+0000","last_change":"2023-10-01T09:49:50.024454+0000","last_active":"2023-10-01T09:50:31.038623+0000","last_peered":"2023-10-01T09:50:31.038623+0000","last_clean":"2023-10-01T09:50:31.038623+0000","last_became_active":"2023-10-01T09:49:50.024154+0000","last_became_peered":"2023-10-01T09:49:50.024154+0000","last_unstale":"2023-10-01T09:50:31.038623+0000","last_undegraded":"2023-10-01T09:50:31.038623+0000","last_fullsized":"2023-10-01T09:50:31.038623+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:57:07.108325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":577,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523463+0000","last_change":"2023-10-01T09:50:08.986145+0000","last_active":"2023-10-01T09:50:30.523463+0000","last_peered":"2023-10-01T09:50:30.523463+0000","last_clean":"2023-10-01T09:50:30.523463+0000","last_became_active":"2023-10-01T09:50:08.984987+0000","last_became_peered":"2023-10-01T09:50:08.984987+0000","last_unstale":"2023-10-01T09:50:30.523463+0000","last_undegraded":"2023-10-01T09:50:30.523463+0000","last_fullsized":"2023-10-01T09:50:30.523463+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:22:03.682298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":328,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.041147+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:50:31.041147+0000","last_peered":"2023-10-01T09:50:31.041147+0000","last_clean":"2023-10-01T09:50:31.041147+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:50:31.041147+0000","last_undegraded":"2023-10-01T09:50:31.041147+0000","last_fullsized":"2023-10-01T09:50:31.041147+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":56,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523545+0000","last_change":"2023-10-01T09:50:08.948042+0000","last_active":"2023-10-01T09:50:30.523545+0000","last_peered":"2023-10-01T09:50:30.523545+0000","last_clean":"2023-10-01T09:50:30.523545+0000","last_became_active":"2023-10-01T09:50:08.947663+0000","last_became_peered":"2023-10-01T09:50:08.947663+0000","last_unstale":"2023-10-01T09:50:30.523545+0000","last_undegraded":"2023-10-01T09:50:30.523545+0000","last_fullsized":"2023-10-01T09:50:30.523545+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:26.568510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":141,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040595+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:50:31.040595+0000","last_peered":"2023-10-01T09:50:31.040595+0000","last_clean":"2023-10-01T09:50:31.040595+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:50:31.040595+0000","last_undegraded":"2023-10-01T09:50:31.040595+0000","last_fullsized":"2023-10-01T09:50:31.040595+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":151,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.026241+0000","last_change":"2023-10-01T09:49:37.882756+0000","last_active":"2023-10-01T09:50:31.026241+0000","last_peered":"2023-10-01T09:50:31.026241+0000","last_clean":"2023-10-01T09:50:31.026241+0000","last_became_active":"2023-10-01T09:49:37.882460+0000","last_became_peered":"2023-10-01T09:49:37.882460+0000","last_unstale":"2023-10-01T09:50:31.026241+0000","last_undegraded":"2023-10-01T09:50:31.026241+0000","last_fullsized":"2023-10-01T09:50:31.026241+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:04:53.618777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":517,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040084+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:50:31.040084+0000","last_peered":"2023-10-01T09:50:31.040084+0000","last_clean":"2023-10-01T09:50:31.040084+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:50:31.040084+0000","last_undegraded":"2023-10-01T09:50:31.040084+0000","last_fullsized":"2023-10-01T09:50:31.040084+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":398,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040376+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:50:31.040376+0000","last_peered":"2023-10-01T09:50:31.040376+0000","last_clean":"2023-10-01T09:50:31.040376+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:50:31.040376+0000","last_undegraded":"2023-10-01T09:50:31.040376+0000","last_fullsized":"2023-10-01T09:50:31.040376+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":749,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040969+0000","last_change":"2023-10-01T09:49:37.973230+0000","last_active":"2023-10-01T09:50:31.040969+0000","last_peered":"2023-10-01T09:50:31.040969+0000","last_clean":"2023-10-01T09:50:31.040969+0000","last_became_active":"2023-10-01T09:49:37.881204+0000","last_became_peered":"2023-10-01T09:49:37.881204+0000","last_unstale":"2023-10-01T09:50:31.040969+0000","last_undegraded":"2023-10-01T09:50:31.040969+0000","last_fullsized":"2023-10-01T09:50:31.040969+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":374,"num_keys_recovered":110,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":449,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523519+0000","last_change":"2023-10-01T09:50:08.945926+0000","last_active":"2023-10-01T09:50:30.523519+0000","last_peered":"2023-10-01T09:50:30.523519+0000","last_clean":"2023-10-01T09:50:30.523519+0000","last_became_active":"2023-10-01T09:50:08.945623+0000","last_became_peered":"2023-10-01T09:50:08.945623+0000","last_unstale":"2023-10-01T09:50:30.523519+0000","last_undegraded":"2023-10-01T09:50:30.523519+0000","last_fullsized":"2023-10-01T09:50:30.523519+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:21:58.878782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":540,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.038371+0000","last_change":"2023-10-01T09:49:49.885502+0000","last_active":"2023-10-01T09:50:31.038371+0000","last_peered":"2023-10-01T09:50:31.038371+0000","last_clean":"2023-10-01T09:50:31.038371+0000","last_became_active":"2023-10-01T09:49:49.885219+0000","last_became_peered":"2023-10-01T09:49:49.885219+0000","last_unstale":"2023-10-01T09:50:31.038371+0000","last_undegraded":"2023-10-01T09:50:31.038371+0000","last_fullsized":"2023-10-01T09:50:31.038371+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:14:36.619704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":130,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040709+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:50:31.040709+0000","last_peered":"2023-10-01T09:50:31.040709+0000","last_clean":"2023-10-01T09:50:31.040709+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:50:31.040709+0000","last_undegraded":"2023-10-01T09:50:31.040709+0000","last_fullsized":"2023-10-01T09:50:31.040709+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":627,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.041042+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:50:31.041042+0000","last_peered":"2023-10-01T09:50:31.041042+0000","last_clean":"2023-10-01T09:50:31.041042+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:50:31.041042+0000","last_undegraded":"2023-10-01T09:50:31.041042+0000","last_fullsized":"2023-10-01T09:50:31.041042+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":641,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.038576+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:50:31.038576+0000","last_peered":"2023-10-01T09:50:31.038576+0000","last_clean":"2023-10-01T09:50:31.038576+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:50:31.038576+0000","last_undegraded":"2023-10-01T09:50:31.038576+0000","last_fullsized":"2023-10-01T09:50:31.038576+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":186,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.526615+0000","last_change":"2023-10-01T09:50:28.522830+0000","last_active":"2023-10-01T09:50:30.526615+0000","last_peered":"2023-10-01T09:50:30.526615+0000","last_clean":"2023-10-01T09:50:30.526615+0000","last_became_active":"2023-10-01T09:50:28.522458+0000","last_became_peered":"2023-10-01T09:50:28.522458+0000","last_unstale":"2023-10-01T09:50:30.526615+0000","last_undegraded":"2023-10-01T09:50:30.526615+0000","last_fullsized":"2023-10-01T09:50:30.526615+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":122,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.522578+0000","last_change":"2023-10-01T09:50:08.985351+0000","last_active":"2023-10-01T09:50:30.522578+0000","last_peered":"2023-10-01T09:50:30.522578+0000","last_clean":"2023-10-01T09:50:30.522578+0000","last_became_active":"2023-10-01T09:50:08.985061+0000","last_became_peered":"2023-10-01T09:50:08.985061+0000","last_unstale":"2023-10-01T09:50:30.522578+0000","last_undegraded":"2023-10-01T09:50:30.522578+0000","last_fullsized":"2023-10-01T09:50:30.522578+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:10:22.545254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":139,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040831+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:50:31.040831+0000","last_peered":"2023-10-01T09:50:31.040831+0000","last_clean":"2023-10-01T09:50:31.040831+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:50:31.040831+0000","last_undegraded":"2023-10-01T09:50:31.040831+0000","last_fullsized":"2023-10-01T09:50:31.040831+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":336,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040528+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:50:31.040528+0000","last_peered":"2023-10-01T09:50:31.040528+0000","last_clean":"2023-10-01T09:50:31.040528+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:50:31.040528+0000","last_undegraded":"2023-10-01T09:50:31.040528+0000","last_fullsized":"2023-10-01T09:50:31.040528+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":610,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040456+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:50:31.040456+0000","last_peered":"2023-10-01T09:50:31.040456+0000","last_clean":"2023-10-01T09:50:31.040456+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:50:31.040456+0000","last_undegraded":"2023-10-01T09:50:31.040456+0000","last_fullsized":"2023-10-01T09:50:31.040456+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":567,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040236+0000","last_change":"2023-10-01T09:50:15.286925+0000","last_active":"2023-10-01T09:50:31.040236+0000","last_peered":"2023-10-01T09:50:31.040236+0000","last_clean":"2023-10-01T09:50:31.040236+0000","last_became_active":"2023-10-01T09:50:15.286576+0000","last_became_peered":"2023-10-01T09:50:15.286576+0000","last_unstale":"2023-10-01T09:50:31.040236+0000","last_undegraded":"2023-10-01T09:50:31.040236+0000","last_fullsized":"2023-10-01T09:50:31.040236+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":696,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040659+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:50:31.040659+0000","last_peered":"2023-10-01T09:50:31.040659+0000","last_clean":"2023-10-01T09:50:31.040659+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:50:31.040659+0000","last_undegraded":"2023-10-01T09:50:31.040659+0000","last_fullsized":"2023-10-01T09:50:31.040659+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":370,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.526794+0000","last_change":"2023-10-01T09:50:29.052720+0000","last_active":"2023-10-01T09:50:30.526794+0000","last_peered":"2023-10-01T09:50:30.526794+0000","last_clean":"2023-10-01T09:50:30.526794+0000","last_became_active":"2023-10-01T09:50:29.052306+0000","last_became_peered":"2023-10-01T09:50:29.052306+0000","last_unstale":"2023-10-01T09:50:30.526794+0000","last_undegraded":"2023-10-01T09:50:30.526794+0000","last_fullsized":"2023-10-01T09:50:30.526794+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":336,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040450+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:50:31.040450+0000","last_peered":"2023-10-01T09:50:31.040450+0000","last_clean":"2023-10-01T09:50:31.040450+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:50:31.040450+0000","last_undegraded":"2023-10-01T09:50:31.040450+0000","last_fullsized":"2023-10-01T09:50:31.040450+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":597,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.038398+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:50:31.038398+0000","last_peered":"2023-10-01T09:50:31.038398+0000","last_clean":"2023-10-01T09:50:31.038398+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:50:31.038398+0000","last_undegraded":"2023-10-01T09:50:31.038398+0000","last_fullsized":"2023-10-01T09:50:31.038398+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":210,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.522266+0000","last_change":"2023-10-01T09:50:08.946873+0000","last_active":"2023-10-01T09:50:30.522266+0000","last_peered":"2023-10-01T09:50:30.522266+0000","last_clean":"2023-10-01T09:50:30.522266+0000","last_became_active":"2023-10-01T09:50:08.946622+0000","last_became_peered":"2023-10-01T09:50:08.946622+0000","last_unstale":"2023-10-01T09:50:30.522266+0000","last_undegraded":"2023-10-01T09:50:30.522266+0000","last_fullsized":"2023-10-01T09:50:30.522266+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:19:11.600014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":276,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040256+0000","last_change":"2023-10-01T09:50:15.286778+0000","last_active":"2023-10-01T09:50:31.040256+0000","last_peered":"2023-10-01T09:50:31.040256+0000","last_clean":"2023-10-01T09:50:31.040256+0000","last_became_active":"2023-10-01T09:50:15.286418+0000","last_became_peered":"2023-10-01T09:50:15.286418+0000","last_unstale":"2023-10-01T09:50:31.040256+0000","last_undegraded":"2023-10-01T09:50:31.040256+0000","last_fullsized":"2023-10-01T09:50:31.040256+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":411,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.526908+0000","last_change":"2023-10-01T09:50:29.052667+0000","last_active":"2023-10-01T09:50:30.526908+0000","last_peered":"2023-10-01T09:50:30.526908+0000","last_clean":"2023-10-01T09:50:30.526908+0000","last_became_active":"2023-10-01T09:50:29.052256+0000","last_became_peered":"2023-10-01T09:50:29.052256+0000","last_unstale":"2023-10-01T09:50:30.526908+0000","last_undegraded":"2023-10-01T09:50:30.526908+0000","last_fullsized":"2023-10-01T09:50:30.526908+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":673,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.041526+0000","last_change":"2023-10-01T09:50:15.097746+0000","last_active":"2023-10-01T09:50:31.041526+0000","last_peered":"2023-10-01T09:50:31.041526+0000","last_clean":"2023-10-01T09:50:31.041526+0000","last_became_active":"2023-10-01T09:50:15.097289+0000","last_became_peered":"2023-10-01T09:50:15.097289+0000","last_unstale":"2023-10-01T09:50:31.041526+0000","last_undegraded":"2023-10-01T09:50:31.041526+0000","last_fullsized":"2023-10-01T09:50:31.041526+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":519,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.039995+0000","last_change":"2023-10-01T09:49:37.885772+0000","last_active":"2023-10-01T09:50:31.039995+0000","last_peered":"2023-10-01T09:50:31.039995+0000","last_clean":"2023-10-01T09:50:31.039995+0000","last_became_active":"2023-10-01T09:49:37.885461+0000","last_became_peered":"2023-10-01T09:49:37.885461+0000","last_unstale":"2023-10-01T09:50:31.039995+0000","last_undegraded":"2023-10-01T09:50:31.039995+0000","last_fullsized":"2023-10-01T09:50:31.039995+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":351,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.041459+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:50:31.041459+0000","last_peered":"2023-10-01T09:50:31.041459+0000","last_clean":"2023-10-01T09:50:31.041459+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:50:31.041459+0000","last_undegraded":"2023-10-01T09:50:31.041459+0000","last_fullsized":"2023-10-01T09:50:31.041459+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":377,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523374+0000","last_change":"2023-10-01T09:50:08.986062+0000","last_active":"2023-10-01T09:50:30.523374+0000","last_peered":"2023-10-01T09:50:30.523374+0000","last_clean":"2023-10-01T09:50:30.523374+0000","last_became_active":"2023-10-01T09:50:08.985419+0000","last_became_peered":"2023-10-01T09:50:08.985419+0000","last_unstale":"2023-10-01T09:50:30.523374+0000","last_undegraded":"2023-10-01T09:50:30.523374+0000","last_fullsized":"2023-10-01T09:50:30.523374+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:08:59.087065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":529,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.026507+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:50:31.026507+0000","last_peered":"2023-10-01T09:50:31.026507+0000","last_clean":"2023-10-01T09:50:31.026507+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:50:31.026507+0000","last_undegraded":"2023-10-01T09:50:31.026507+0000","last_fullsized":"2023-10-01T09:50:31.026507+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"585'392","reported_seq":2103,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-10-01T09:50:35.021491+0000","last_change":"2023-10-01T09:49:19.958909+0000","last_active":"2023-10-01T09:50:35.021491+0000","last_peered":"2023-10-01T09:50:35.021491+0000","last_clean":"2023-10-01T09:50:35.021491+0000","last_became_active":"2023-10-01T09:48:45.360357+0000","last_became_peered":"2023-10-01T09:48:45.360357+0000","last_unstale":"2023-10-01T09:50:35.021491+0000","last_undegraded":"2023-10-01T09:50:35.021491+0000","last_fullsized":"2023-10-01T09:50:35.021491+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":392,"log_dups_size":0,"ondisk_log_size":392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:46.766586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054191000000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2673,"num_read_kb":2980,"num_write":346,"num_write_kb":1499,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":355,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040870+0000","last_change":"2023-10-01T09:49:37.881458+0000","last_active":"2023-10-01T09:50:31.040870+0000","last_peered":"2023-10-01T09:50:31.040870+0000","last_clean":"2023-10-01T09:50:31.040870+0000","last_became_active":"2023-10-01T09:49:37.880898+0000","last_became_peered":"2023-10-01T09:49:37.880898+0000","last_unstale":"2023-10-01T09:50:31.040870+0000","last_undegraded":"2023-10-01T09:50:31.040870+0000","last_fullsized":"2023-10-01T09:50:31.040870+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":142,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.026289+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:50:31.026289+0000","last_peered":"2023-10-01T09:50:31.026289+0000","last_clean":"2023-10-01T09:50:31.026289+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:50:31.026289+0000","last_undegraded":"2023-10-01T09:50:31.026289+0000","last_fullsized":"2023-10-01T09:50:31.026289+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"585'228","reported_seq":1955,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040142+0000","last_change":"2023-10-01T09:49:19.923862+0000","last_active":"2023-10-01T09:50:31.040142+0000","last_peered":"2023-10-01T09:50:31.040142+0000","last_clean":"2023-10-01T09:50:31.040142+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:50:31.040142+0000","last_undegraded":"2023-10-01T09:50:31.040142+0000","last_fullsized":"2023-10-01T09:50:31.040142+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":228,"log_dups_size":0,"ondisk_log_size":228,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2216,"num_read_kb":2036,"num_write":182,"num_write_kb":4239,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":491,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040626+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:50:31.040626+0000","last_peered":"2023-10-01T09:50:31.040626+0000","last_clean":"2023-10-01T09:50:31.040626+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:50:31.040626+0000","last_undegraded":"2023-10-01T09:50:31.040626+0000","last_fullsized":"2023-10-01T09:50:31.040626+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":328,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040604+0000","last_change":"2023-10-01T09:49:37.881294+0000","last_active":"2023-10-01T09:50:31.040604+0000","last_peered":"2023-10-01T09:50:31.040604+0000","last_clean":"2023-10-01T09:50:31.040604+0000","last_became_active":"2023-10-01T09:49:37.880930+0000","last_became_peered":"2023-10-01T09:49:37.880930+0000","last_unstale":"2023-10-01T09:50:31.040604+0000","last_undegraded":"2023-10-01T09:50:31.040604+0000","last_fullsized":"2023-10-01T09:50:31.040604+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"584'227","reported_seq":1509,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.521705+0000","last_change":"2023-10-01T09:50:02.916276+0000","last_active":"2023-10-01T09:50:30.521705+0000","last_peered":"2023-10-01T09:50:30.521705+0000","last_clean":"2023-10-01T09:50:30.521705+0000","last_became_active":"2023-10-01T09:50:02.915490+0000","last_became_peered":"2023-10-01T09:50:02.915490+0000","last_unstale":"2023-10-01T09:50:30.521705+0000","last_undegraded":"2023-10-01T09:50:30.521705+0000","last_fullsized":"2023-10-01T09:50:30.521705+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":227,"log_dups_size":0,"ondisk_log_size":227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:41:44.100291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040081099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1258,"num_read_kb":4518,"num_write":199,"num_write_kb":3425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":691,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.026199+0000","last_change":"2023-10-01T09:49:37.882934+0000","last_active":"2023-10-01T09:50:31.026199+0000","last_peered":"2023-10-01T09:50:31.026199+0000","last_clean":"2023-10-01T09:50:31.026199+0000","last_became_active":"2023-10-01T09:49:37.882670+0000","last_became_peered":"2023-10-01T09:49:37.882670+0000","last_unstale":"2023-10-01T09:50:31.026199+0000","last_undegraded":"2023-10-01T09:50:31.026199+0000","last_fullsized":"2023-10-01T09:50:31.026199+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:46:43.118412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":142,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.026502+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:50:31.026502+0000","last_peered":"2023-10-01T09:50:31.026502+0000","last_clean":"2023-10-01T09:50:31.026502+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:50:31.026502+0000","last_undegraded":"2023-10-01T09:50:31.026502+0000","last_fullsized":"2023-10-01T09:50:31.026502+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"585'397","reported_seq":2235,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.041208+0000","last_change":"2023-10-01T09:49:40.901604+0000","last_active":"2023-10-01T09:50:31.041208+0000","last_peered":"2023-10-01T09:50:31.041208+0000","last_clean":"2023-10-01T09:50:31.041208+0000","last_became_active":"2023-10-01T09:49:40.900950+0000","last_became_peered":"2023-10-01T09:49:40.900950+0000","last_unstale":"2023-10-01T09:50:31.041208+0000","last_undegraded":"2023-10-01T09:50:31.041208+0000","last_fullsized":"2023-10-01T09:50:31.041208+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":397,"log_dups_size":0,"ondisk_log_size":397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2083,"num_read_kb":6022,"num_write":364,"num_write_kb":3847,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":139,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":646,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040250+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:50:31.040250+0000","last_peered":"2023-10-01T09:50:31.040250+0000","last_clean":"2023-10-01T09:50:31.040250+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:50:31.040250+0000","last_undegraded":"2023-10-01T09:50:31.040250+0000","last_fullsized":"2023-10-01T09:50:31.040250+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":324,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.522919+0000","last_change":"2023-10-01T09:50:08.985808+0000","last_active":"2023-10-01T09:50:30.522919+0000","last_peered":"2023-10-01T09:50:30.522919+0000","last_clean":"2023-10-01T09:50:30.522919+0000","last_became_active":"2023-10-01T09:50:08.985521+0000","last_became_peered":"2023-10-01T09:50:08.985521+0000","last_unstale":"2023-10-01T09:50:30.522919+0000","last_undegraded":"2023-10-01T09:50:30.522919+0000","last_fullsized":"2023-10-01T09:50:30.522919+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:02:35.181923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":283,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.526980+0000","last_change":"2023-10-01T09:50:28.523682+0000","last_active":"2023-10-01T09:50:30.526980+0000","last_peered":"2023-10-01T09:50:30.526980+0000","last_clean":"2023-10-01T09:50:30.526980+0000","last_became_active":"2023-10-01T09:50:28.523239+0000","last_became_peered":"2023-10-01T09:50:28.523239+0000","last_unstale":"2023-10-01T09:50:30.526980+0000","last_undegraded":"2023-10-01T09:50:30.526980+0000","last_fullsized":"2023-10-01T09:50:30.526980+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"585'105","reported_seq":987,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040378+0000","last_change":"2023-10-01T09:49:19.925623+0000","last_active":"2023-10-01T09:50:31.040378+0000","last_peered":"2023-10-01T09:50:31.040378+0000","last_clean":"2023-10-01T09:50:31.040378+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:50:31.040378+0000","last_undegraded":"2023-10-01T09:50:31.040378+0000","last_fullsized":"2023-10-01T09:50:31.040378+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":339,"num_read_kb":3531,"num_write":98,"num_write_kb":7299,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":659,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:32.118714+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:50:32.118714+0000","last_peered":"2023-10-01T09:50:32.118714+0000","last_clean":"2023-10-01T09:50:32.118714+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:50:32.118714+0000","last_undegraded":"2023-10-01T09:50:32.118714+0000","last_fullsized":"2023-10-01T09:50:32.118714+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":667,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.038458+0000","last_change":"2023-10-01T09:49:37.884060+0000","last_active":"2023-10-01T09:50:31.038458+0000","last_peered":"2023-10-01T09:50:31.038458+0000","last_clean":"2023-10-01T09:50:31.038458+0000","last_became_active":"2023-10-01T09:49:37.883747+0000","last_became_peered":"2023-10-01T09:49:37.883747+0000","last_unstale":"2023-10-01T09:50:31.038458+0000","last_undegraded":"2023-10-01T09:50:31.038458+0000","last_fullsized":"2023-10-01T09:50:31.038458+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:43:04.596671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"585'205","reported_seq":1365,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040704+0000","last_change":"2023-10-01T09:49:19.926295+0000","last_active":"2023-10-01T09:50:31.040704+0000","last_peered":"2023-10-01T09:50:31.040704+0000","last_clean":"2023-10-01T09:50:31.040704+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:50:31.040704+0000","last_undegraded":"2023-10-01T09:50:31.040704+0000","last_fullsized":"2023-10-01T09:50:31.040704+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":205,"log_dups_size":0,"ondisk_log_size":205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1233,"num_read_kb":2003,"num_write":197,"num_write_kb":615,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":615,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-10-01T09:50:35.021499+0000","last_change":"2023-10-01T09:49:37.884027+0000","last_active":"2023-10-01T09:50:35.021499+0000","last_peered":"2023-10-01T09:50:35.021499+0000","last_clean":"2023-10-01T09:50:35.021499+0000","last_became_active":"2023-10-01T09:49:37.883624+0000","last_became_peered":"2023-10-01T09:49:37.883624+0000","last_unstale":"2023-10-01T09:50:35.021499+0000","last_undegraded":"2023-10-01T09:50:35.021499+0000","last_fullsized":"2023-10-01T09:50:35.021499+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:48:03.636472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":482,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.038521+0000","last_change":"2023-10-01T09:49:50.024251+0000","last_active":"2023-10-01T09:50:31.038521+0000","last_peered":"2023-10-01T09:50:31.038521+0000","last_clean":"2023-10-01T09:50:31.038521+0000","last_became_active":"2023-10-01T09:49:50.023983+0000","last_became_peered":"2023-10-01T09:49:50.023983+0000","last_unstale":"2023-10-01T09:50:31.038521+0000","last_undegraded":"2023-10-01T09:50:31.038521+0000","last_fullsized":"2023-10-01T09:50:31.038521+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:29:09.013661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":394,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.522658+0000","last_change":"2023-10-01T09:50:08.946132+0000","last_active":"2023-10-01T09:50:30.522658+0000","last_peered":"2023-10-01T09:50:30.522658+0000","last_clean":"2023-10-01T09:50:30.522658+0000","last_became_active":"2023-10-01T09:50:08.945792+0000","last_became_peered":"2023-10-01T09:50:08.945792+0000","last_unstale":"2023-10-01T09:50:30.522658+0000","last_undegraded":"2023-10-01T09:50:30.522658+0000","last_fullsized":"2023-10-01T09:50:30.522658+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:54:20.779232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"585'269","reported_seq":1681,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.026140+0000","last_change":"2023-10-01T09:49:19.958887+0000","last_active":"2023-10-01T09:50:31.026140+0000","last_peered":"2023-10-01T09:50:31.026140+0000","last_clean":"2023-10-01T09:50:31.026140+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:50:31.026140+0000","last_undegraded":"2023-10-01T09:50:31.026140+0000","last_fullsized":"2023-10-01T09:50:31.026140+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":269,"log_dups_size":0,"ondisk_log_size":269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024152000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1384,"num_read_kb":1112,"num_write":193,"num_write_kb":2135,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":44,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523354+0000","last_change":"2023-10-01T09:50:08.946005+0000","last_active":"2023-10-01T09:50:30.523354+0000","last_peered":"2023-10-01T09:50:30.523354+0000","last_clean":"2023-10-01T09:50:30.523354+0000","last_became_active":"2023-10-01T09:50:08.945730+0000","last_became_peered":"2023-10-01T09:50:08.945730+0000","last_unstale":"2023-10-01T09:50:30.523354+0000","last_undegraded":"2023-10-01T09:50:30.523354+0000","last_fullsized":"2023-10-01T09:50:30.523354+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:57:48.661109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"584'127","reported_seq":516,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.527118+0000","last_change":"2023-10-01T09:50:28.523996+0000","last_active":"2023-10-01T09:50:30.527118+0000","last_peered":"2023-10-01T09:50:30.527118+0000","last_clean":"2023-10-01T09:50:30.527118+0000","last_became_active":"2023-10-01T09:50:28.523374+0000","last_became_peered":"2023-10-01T09:50:28.523374+0000","last_unstale":"2023-10-01T09:50:30.527118+0000","last_undegraded":"2023-10-01T09:50:30.527118+0000","last_fullsized":"2023-10-01T09:50:30.527118+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":777,"num_read_kb":3172,"num_write":111,"num_write_kb":2331,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":636,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.038499+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:50:31.038499+0000","last_peered":"2023-10-01T09:50:31.038499+0000","last_clean":"2023-10-01T09:50:31.038499+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:50:31.038499+0000","last_undegraded":"2023-10-01T09:50:31.038499+0000","last_fullsized":"2023-10-01T09:50:31.038499+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":113,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.039842+0000","last_change":"2023-10-01T09:50:14.965790+0000","last_active":"2023-10-01T09:50:31.039842+0000","last_peered":"2023-10-01T09:50:31.039842+0000","last_clean":"2023-10-01T09:50:31.039842+0000","last_became_active":"2023-10-01T09:50:14.965430+0000","last_became_peered":"2023-10-01T09:50:14.965430+0000","last_unstale":"2023-10-01T09:50:31.039842+0000","last_undegraded":"2023-10-01T09:50:31.039842+0000","last_fullsized":"2023-10-01T09:50:31.039842+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":585,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-10-01T09:50:33.888686+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:50:33.888686+0000","last_peered":"2023-10-01T09:50:33.888686+0000","last_clean":"2023-10-01T09:50:33.888686+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:50:33.888686+0000","last_undegraded":"2023-10-01T09:50:33.888686+0000","last_fullsized":"2023-10-01T09:50:33.888686+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":698,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.522078+0000","last_change":"2023-10-01T09:50:02.919144+0000","last_active":"2023-10-01T09:50:30.522078+0000","last_peered":"2023-10-01T09:50:30.522078+0000","last_clean":"2023-10-01T09:50:30.522078+0000","last_became_active":"2023-10-01T09:50:02.918856+0000","last_became_peered":"2023-10-01T09:50:02.918856+0000","last_unstale":"2023-10-01T09:50:30.522078+0000","last_undegraded":"2023-10-01T09:50:30.522078+0000","last_fullsized":"2023-10-01T09:50:30.522078+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:34:57.342926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":351,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040348+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:50:31.040348+0000","last_peered":"2023-10-01T09:50:31.040348+0000","last_clean":"2023-10-01T09:50:31.040348+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:50:31.040348+0000","last_undegraded":"2023-10-01T09:50:31.040348+0000","last_fullsized":"2023-10-01T09:50:31.040348+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":286,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040529+0000","last_change":"2023-10-01T09:50:15.286842+0000","last_active":"2023-10-01T09:50:31.040529+0000","last_peered":"2023-10-01T09:50:31.040529+0000","last_clean":"2023-10-01T09:50:31.040529+0000","last_became_active":"2023-10-01T09:50:15.286494+0000","last_became_peered":"2023-10-01T09:50:15.286494+0000","last_unstale":"2023-10-01T09:50:31.040529+0000","last_undegraded":"2023-10-01T09:50:31.040529+0000","last_fullsized":"2023-10-01T09:50:31.040529+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":351,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040969+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:50:31.040969+0000","last_peered":"2023-10-01T09:50:31.040969+0000","last_clean":"2023-10-01T09:50:31.040969+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:50:31.040969+0000","last_undegraded":"2023-10-01T09:50:31.040969+0000","last_fullsized":"2023-10-01T09:50:31.040969+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":367,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.041332+0000","last_change":"2023-10-01T09:50:15.287047+0000","last_active":"2023-10-01T09:50:31.041332+0000","last_peered":"2023-10-01T09:50:31.041332+0000","last_clean":"2023-10-01T09:50:31.041332+0000","last_became_active":"2023-10-01T09:50:15.286668+0000","last_became_peered":"2023-10-01T09:50:15.286668+0000","last_unstale":"2023-10-01T09:50:31.041332+0000","last_undegraded":"2023-10-01T09:50:31.041332+0000","last_fullsized":"2023-10-01T09:50:31.041332+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":77,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.522484+0000","last_change":"2023-10-01T09:50:08.948473+0000","last_active":"2023-10-01T09:50:30.522484+0000","last_peered":"2023-10-01T09:50:30.522484+0000","last_clean":"2023-10-01T09:50:30.522484+0000","last_became_active":"2023-10-01T09:50:08.948199+0000","last_became_peered":"2023-10-01T09:50:08.948199+0000","last_unstale":"2023-10-01T09:50:30.522484+0000","last_undegraded":"2023-10-01T09:50:30.522484+0000","last_fullsized":"2023-10-01T09:50:30.522484+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:53:13.704522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":574,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523602+0000","last_change":"2023-10-01T09:50:08.944753+0000","last_active":"2023-10-01T09:50:30.523602+0000","last_peered":"2023-10-01T09:50:30.523602+0000","last_clean":"2023-10-01T09:50:30.523602+0000","last_became_active":"2023-10-01T09:50:08.944441+0000","last_became_peered":"2023-10-01T09:50:08.944441+0000","last_unstale":"2023-10-01T09:50:30.523602+0000","last_undegraded":"2023-10-01T09:50:30.523602+0000","last_fullsized":"2023-10-01T09:50:30.523602+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:02:44.959539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":791,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:32.118829+0000","last_change":"2023-10-01T09:49:37.975951+0000","last_active":"2023-10-01T09:50:32.118829+0000","last_peered":"2023-10-01T09:50:32.118829+0000","last_clean":"2023-10-01T09:50:32.118829+0000","last_became_active":"2023-10-01T09:49:37.884496+0000","last_became_peered":"2023-10-01T09:49:37.884496+0000","last_unstale":"2023-10-01T09:50:32.118829+0000","last_undegraded":"2023-10-01T09:50:32.118829+0000","last_fullsized":"2023-10-01T09:50:32.118829+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:10:46.008390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":467,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.039939+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:50:31.039939+0000","last_peered":"2023-10-01T09:50:31.039939+0000","last_clean":"2023-10-01T09:50:31.039939+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:50:31.039939+0000","last_undegraded":"2023-10-01T09:50:31.039939+0000","last_fullsized":"2023-10-01T09:50:31.039939+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":31074,"num_keys_recovered":110,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11963,"num_read_kb":25374,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":404,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430807,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905112,"kb_used_data":10192,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":92840040,"statfs":{"total":95995035648,"available":95068200960,"internally_reserved":0,"allocated":10436608,"data_stored":9444579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:50:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":2.2410000000000001,"15min":2.2309999999999999},"min":{"1min":0.32000000000000001,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":0.81299999999999994,"5min":112.274,"15min":112.274},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.66900000000000004,"5min":2.2090000000000001,"15min":2.3319999999999999},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.21199999999999999},"max":{"1min":3.5249999999999999,"5min":112.44799999999999,"15min":112.44799999999999},"last":0.504}]},{"osd":1,"last update":"Sun Oct 1 09:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.97799999999999998,"15min":3.141},"min":{"1min":0.34300000000000003,"5min":0.24199999999999999,"15min":0.23300000000000001},"max":{"1min":0.79100000000000004,"5min":12.522,"15min":229.161},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.92500000000000004,"15min":3.1269999999999998},"min":{"1min":0.318,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":0.83699999999999997,"5min":10.127000000000001,"15min":230.126},"last":0.75900000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":1.089,"15min":1.3939999999999999},"min":{"1min":0.40999999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.77600000000000002,"5min":12.565,"15min":14.901999999999999},"last":0.63},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.81699999999999995,"15min":1.2689999999999999},"min":{"1min":0.375,"5min":0.36499999999999999,"15min":0.27000000000000002},"max":{"1min":0.79100000000000004,"5min":7.4379999999999997,"15min":14.682},"last":0.78700000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.90700000000000003,"15min":1.736},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.29199999999999998},"max":{"1min":0.93500000000000005,"5min":7.4699999999999998,"15min":76.555000000000007},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.89900000000000002,"15min":1.835},"min":{"1min":0.38300000000000001,"5min":0.315,"15min":0.27600000000000002},"max":{"1min":0.75700000000000001,"5min":9.2599999999999998,"15min":76.534000000000006},"last":0.66100000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:50:06 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.90800000000000003,"15min":0.995},"min":{"1min":0.29299999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.73799999999999999,"5min":11.069000000000001,"15min":11.069000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.80500000000000005,"15min":0.90900000000000003},"min":{"1min":0.31900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.84199999999999997,"5min":7.5,"15min":8.4949999999999992},"last":0.73399999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:50:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.153,"15min":1.6080000000000001},"min":{"1min":0.432,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.98999999999999999,"5min":13.41,"15min":135.40000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.95999999999999996,"15min":1.6850000000000001},"min":{"1min":0.34499999999999997,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":0.88600000000000001,"5min":11.895,"15min":135.339},"last":0.59299999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:50:46 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.89800000000000002,"15min":1.0820000000000001},"min":{"1min":0.46400000000000002,"5min":0.39600000000000002,"15min":0.313},"max":{"1min":0.96899999999999997,"5min":10.417999999999999,"15min":11.853999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.93799999999999994,"15min":1.081},"min":{"1min":0.35599999999999998,"5min":0.32600000000000001,"15min":0.32400000000000001},"max":{"1min":0.85699999999999998,"5min":9.3550000000000004,"15min":14.584},"last":0.71599999999999997}]}]},{"osd":6,"up_from":294,"seq":1262720385317,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343688,"kb_used_data":10176,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":93401464,"statfs":{"total":95995035648,"available":95643099136,"internally_reserved":0,"allocated":10420224,"data_stored":9443822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:50:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":2.1930000000000001,"15min":2.4079999999999999},"min":{"1min":0.245,"5min":0.223,"15min":0.219},"max":{"1min":0.58099999999999996,"5min":77.837000000000003,"15min":158.48599999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.53300000000000003,"5min":2.3740000000000001,"15min":2.6560000000000001},"min":{"1min":0.36199999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.86499999999999999,"5min":77.971000000000004,"15min":158.37700000000001},"last":0.42999999999999999}]},{"osd":1,"last update":"Sun Oct 1 09:50:27 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.89500000000000002,"15min":2.9649999999999999},"min":{"1min":0.25700000000000001,"5min":0.222,"15min":0.222},"max":{"1min":0.75,"5min":8.2409999999999997,"15min":168.947},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.73199999999999998,"15min":2.8900000000000001},"min":{"1min":0.36499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.96199999999999997,"5min":6.8390000000000004,"15min":169.07900000000001},"last":0.53200000000000003}]},{"osd":2,"last update":"Sun Oct 1 09:50:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":1.0049999999999999,"15min":1.272},"min":{"1min":0.40000000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.63800000000000001,"5min":8.2829999999999995,"15min":9.657},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.502,"5min":0.79400000000000004,"15min":1.204},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.27100000000000002},"max":{"1min":0.88300000000000001,"5min":6.8159999999999998,"15min":21.704999999999998},"last":0.496}]},{"osd":3,"last update":"Sun Oct 1 09:50:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.746,"15min":1.181},"min":{"1min":0.33500000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.81599999999999995,"5min":6.3920000000000003,"15min":9.5679999999999996},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.97299999999999998,"15min":1.2989999999999999},"min":{"1min":0.311,"5min":0.309,"15min":0.30399999999999999},"max":{"1min":0.90600000000000003,"5min":8.3109999999999999,"15min":10.191000000000001},"last":0.54800000000000004}]},{"osd":4,"last update":"Sun Oct 1 09:50:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":1.0740000000000001,"15min":1.103},"min":{"1min":0.31900000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.85999999999999999,"5min":12.326000000000001,"15min":63.134},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":1.2390000000000001,"15min":1.1200000000000001},"min":{"1min":0.41199999999999998,"5min":0.29399999999999998,"15min":0.28899999999999998},"max":{"1min":0.92600000000000005,"5min":13.641,"15min":62.973999999999997},"last":0.68000000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":2.238,"15min":1.4470000000000001},"min":{"1min":0.34799999999999998,"5min":0.313,"15min":0.251},"max":{"1min":0.71099999999999997,"5min":136.535,"15min":136.535},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.52300000000000002,"5min":2.1539999999999999,"15min":1.4930000000000001},"min":{"1min":0.311,"5min":0.28699999999999998,"15min":0.28100000000000003},"max":{"1min":0.76600000000000001,"5min":136.47300000000001,"15min":136.47300000000001},"last":0.58599999999999997}]},{"osd":7,"last update":"Sun Oct 1 09:50:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":1.256,"15min":1.3480000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.29899999999999999},"max":{"1min":0.81399999999999995,"5min":15.407999999999999,"15min":94.245999999999995},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":1.069,"15min":1.446},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.96299999999999997,"5min":11.326000000000001,"15min":94.715000000000003},"last":0.57199999999999995}]}]},{"osd":1,"up_from":94,"seq":403726926294,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":869360,"kb_used_data":15080,"kb_used_omap":7,"kb_used_meta":854264,"kb_avail":92875792,"statfs":{"total":95995035648,"available":95104811008,"internally_reserved":0,"allocated":15441920,"data_stored":13196764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":874766759},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.316,"5min":2.8199999999999998,"15min":3.6600000000000001},"min":{"1min":0.22600000000000001,"5min":0.19500000000000001,"15min":0.182},"max":{"1min":0.41299999999999998,"5min":114.33499999999999,"15min":154.21299999999999},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.32800000000000001,"5min":2.8090000000000002,"15min":3.222},"min":{"1min":0.214,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":0.50600000000000001,"5min":114.099,"15min":114.099},"last":0.20000000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.65900000000000003,"15min":1.3160000000000001},"min":{"1min":0.249,"5min":0.22,"15min":0.22},"max":{"1min":5.6929999999999996,"5min":9.2690000000000001,"15min":22.891999999999999},"last":0.317},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.77300000000000002,"15min":2.0299999999999998},"min":{"1min":0.27800000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.534000000000001,"5min":12.018000000000001,"15min":154.197},"last":0.23000000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.65800000000000003,"15min":2.1160000000000001},"min":{"1min":0.28199999999999997,"5min":0.222,"15min":0.219},"max":{"1min":0.55300000000000005,"5min":8.4320000000000004,"15min":95.305000000000007},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.57199999999999995,"15min":2.71},"min":{"1min":0.28599999999999998,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.48499999999999999,"5min":4.8559999999999999,"15min":154.483},"last":0.28999999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.2170000000000001,"15min":2.1880000000000002},"min":{"1min":0.30399999999999999,"5min":0.245,"15min":0.245},"max":{"1min":0.93400000000000005,"5min":24.539999999999999,"15min":154.101},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.67400000000000004,"15min":1.9039999999999999},"min":{"1min":0.35099999999999998,"5min":0.29999999999999999,"15min":0.27700000000000002},"max":{"1min":1.069,"5min":4.9139999999999997,"15min":154.56999999999999},"last":0.55800000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":1.2949999999999999,"15min":2.2949999999999999},"min":{"1min":0.39700000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.085,"5min":18.158000000000001,"15min":154.22999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.76200000000000001,"5min":1.1990000000000001,"15min":2.1920000000000002},"min":{"1min":0.46700000000000003,"5min":0.32300000000000001,"15min":0.307},"max":{"1min":4.2199999999999998,"5min":24.405000000000001,"15min":154.54400000000001},"last":0.50800000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":1.3029999999999999,"15min":2.1709999999999998},"min":{"1min":0.35899999999999999,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":1.2230000000000001,"5min":24.321000000000002,"15min":154.12200000000001},"last":0.625},{"interface":"front","average":{"1min":0.56000000000000005,"5min":1.012,"15min":2.153},"min":{"1min":0.33400000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.97799999999999998,"5min":9.2149999999999999,"15min":154.16999999999999},"last":0.54800000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.1299999999999999,"15min":2.2829999999999999},"min":{"1min":0.39900000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.995,"5min":15.009,"15min":154.20699999999999},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.96899999999999997,"15min":2.214},"min":{"1min":0.40699999999999997,"5min":0.36799999999999999,"15min":0.34300000000000003},"max":{"1min":1.149,"5min":6.6180000000000003,"15min":154.51599999999999},"last":0.61699999999999999}]}]},{"osd":0,"up_from":89,"seq":382252089828,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1022440,"kb_used_data":68960,"kb_used_omap":8,"kb_used_meta":953463,"kb_avail":92722712,"statfs":{"total":95995035648,"available":94948057088,"internally_reserved":0,"allocated":70615040,"data_stored":68385238,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":976347074},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:50:24 2023","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":1.159,"15min":3.4529999999999998},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.17799999999999999},"max":{"1min":0.46100000000000002,"5min":90.185000000000002,"15min":216.821},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.34200000000000003,"5min":1.403,"15min":4.0369999999999999},"min":{"1min":0.23400000000000001,"5min":0.20599999999999999,"15min":0.19600000000000001},"max":{"1min":0.47199999999999998,"5min":90.102999999999994,"15min":217.02699999999999},"last":0.438}]},{"osd":2,"last update":"Sun Oct 1 09:50:57 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.53300000000000003,"15min":1.359},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.52100000000000002,"5min":4.5819999999999999,"15min":19.788},"last":0.5},{"interface":"front","average":{"1min":0.372,"5min":0.71099999999999997,"15min":1.427},"min":{"1min":0.28499999999999998,"5min":0.27500000000000002,"15min":0.215},"max":{"1min":0.53500000000000003,"5min":10.901,"15min":25.844999999999999},"last":0.46999999999999997}]},{"osd":3,"last update":"Sun Oct 1 09:50:24 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.47099999999999997,"15min":2.2480000000000002},"min":{"1min":0.29699999999999999,"5min":0.24099999999999999,"15min":0.22},"max":{"1min":0.54800000000000004,"5min":3.3849999999999998,"15min":222.16499999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.372,"5min":0.67500000000000004,"15min":2.2909999999999999},"min":{"1min":0.27200000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.55900000000000005,"5min":4.0650000000000004,"15min":222.227},"last":0.45400000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":2.1720000000000002,"15min":1.9059999999999999},"min":{"1min":0.32000000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.77600000000000002,"5min":138.84100000000001,"15min":138.84100000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.51500000000000001,"5min":1.9319999999999999,"15min":1.9610000000000001},"min":{"1min":0.34999999999999998,"5min":0.29199999999999998,"15min":0.28699999999999998},"max":{"1min":0.71799999999999997,"5min":135.94800000000001,"15min":135.94800000000001},"last":0.63200000000000001}]},{"osd":5,"last update":"Sun Oct 1 09:50:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":2.2869999999999999,"15min":1.8939999999999999},"min":{"1min":0.35699999999999998,"5min":0.308,"15min":0.308},"max":{"1min":0.80700000000000005,"5min":156.69999999999999,"15min":156.69999999999999},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.59999999999999998,"5min":2.0470000000000002,"15min":1.7689999999999999},"min":{"1min":0.46100000000000002,"5min":0.30599999999999999,"15min":0.27700000000000002},"max":{"1min":0.77300000000000002,"5min":156.773,"15min":156.773},"last":0.70899999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:50:41 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":2.0840000000000001,"15min":1.7410000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.73499999999999999,"5min":135.321,"15min":135.321},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":2.181,"15min":1.8200000000000001},"min":{"1min":0.36799999999999999,"5min":0.35899999999999999,"15min":0.32400000000000001},"max":{"1min":0.80200000000000005,"5min":135.29900000000001,"15min":135.29900000000001},"last":0.67000000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.89600000000000002,"15min":1.6619999999999999},"min":{"1min":0.38800000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.77800000000000002,"5min":8.6829999999999998,"15min":48.895000000000003},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.1899999999999999,"15min":1.804},"min":{"1min":0.35699999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.79000000000000004,"5min":10.82,"15min":48.274000000000001},"last":0.73199999999999998}]}]},{"osd":2,"up_from":99,"seq":425201762768,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":757932,"kb_used_data":6884,"kb_used_omap":11,"kb_used_meta":751028,"kb_avail":92987220,"statfs":{"total":95995035648,"available":95218913280,"internally_reserved":0,"allocated":7049216,"data_stored":4809623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":769053559},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:50:02 2023","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":1.069,"15min":3.1659999999999999},"min":{"1min":0.253,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":0.54600000000000004,"5min":59.658999999999999,"15min":137.72900000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.35699999999999998,"5min":1.222,"15min":3.3620000000000001},"min":{"1min":0.24199999999999999,"5min":0.21099999999999999,"15min":0.19600000000000001},"max":{"1min":0.47999999999999998,"5min":59.930999999999997,"15min":138.30600000000001},"last":0.318}]},{"osd":1,"last update":"Sun Oct 1 09:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":2.391,"15min":2.8279999999999998},"min":{"1min":0.27200000000000002,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":0.48399999999999999,"5min":134.94499999999999,"15min":134.94499999999999},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.35399999999999998,"5min":2.625,"15min":3.4159999999999999},"min":{"1min":0.28599999999999998,"5min":0.21299999999999999,"15min":0.19900000000000001},"max":{"1min":0.53200000000000003,"5min":134.965,"15min":134.965},"last":0.34999999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:50:31 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.56699999999999995,"15min":1.617},"min":{"1min":0.28199999999999997,"5min":0.23799999999999999,"15min":0.20899999999999999},"max":{"1min":0.49399999999999999,"5min":4.9009999999999998,"15min":50.289000000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.49299999999999999,"15min":1.3100000000000001},"min":{"1min":0.312,"5min":0.23699999999999999,"15min":0.215},"max":{"1min":0.53500000000000003,"5min":3.016,"15min":54.115000000000002},"last":0.40200000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.333,"15min":1.3160000000000001},"min":{"1min":0.36399999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.85599999999999998,"5min":60.994,"15min":60.994},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.393,"15min":1.4470000000000001},"min":{"1min":0.42699999999999999,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":0.81299999999999994,"5min":60.970999999999997,"15min":60.970999999999997},"last":0.59699999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":2.3679999999999999,"15min":1.716},"min":{"1min":0.36499999999999999,"5min":0.34499999999999997,"15min":0.28799999999999998},"max":{"1min":0.78400000000000003,"5min":111.70999999999999,"15min":111.70999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.56999999999999995,"5min":2.524,"15min":1.919},"min":{"1min":0.44,"5min":0.33300000000000002,"15min":0.32000000000000001},"max":{"1min":0.753,"5min":111.735,"15min":111.735},"last":0.67300000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:50:31 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.96999999999999997,"15min":1.389},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":0.69499999999999995,"5min":8.2959999999999994,"15min":11.526999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.89900000000000002,"15min":1.1359999999999999},"min":{"1min":0.45000000000000001,"5min":0.378,"15min":0.32800000000000001},"max":{"1min":0.78700000000000003,"5min":8.5410000000000004,"15min":12.384},"last":0.55700000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.83499999999999996,"15min":1.7230000000000001},"min":{"1min":0.48399999999999999,"5min":0.39700000000000002,"15min":0.34899999999999998},"max":{"1min":0.88900000000000001,"5min":4.3959999999999999,"15min":139.03299999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.93799999999999994,"15min":1.919},"min":{"1min":0.41699999999999998,"5min":0.36599999999999999,"15min":0.35999999999999999},"max":{"1min":0.94999999999999996,"5min":6.468,"15min":139.065},"last":0.63300000000000001}]}]},{"osd":3,"up_from":104,"seq":446676599241,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":875224,"kb_used_data":6992,"kb_used_omap":3,"kb_used_meta":868220,"kb_avail":92869928,"statfs":{"total":95995035648,"available":95098806272,"internally_reserved":0,"allocated":7159808,"data_stored":4819713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":889058023},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":2.625,"15min":3.105},"min":{"1min":0.20999999999999999,"5min":0.19800000000000001,"15min":0.17899999999999999},"max":{"1min":1.0589999999999999,"5min":93.245999999999995,"15min":118.748},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.38500000000000001,"5min":2.7360000000000002,"15min":3.3290000000000002},"min":{"1min":0.23200000000000001,"5min":0.217,"15min":0.19800000000000001},"max":{"1min":0.97399999999999998,"5min":91.430000000000007,"15min":118.86199999999999},"last":0.35099999999999998}]},{"osd":1,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.56499999999999995,"15min":3.2949999999999999},"min":{"1min":0.28000000000000003,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":1.105,"5min":5.9009999999999998,"15min":153.77000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.48499999999999999,"15min":3.2149999999999999},"min":{"1min":0.25,"5min":0.20899999999999999,"15min":0.20499999999999999},"max":{"1min":1.0980000000000001,"5min":4.008,"15min":151.02199999999999},"last":0.42299999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.51300000000000001,"15min":1.401},"min":{"1min":0.254,"5min":0.22900000000000001,"15min":0.22800000000000001},"max":{"1min":1.5780000000000001,"5min":3.5059999999999998,"15min":14.254},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.621,"15min":1.0580000000000001},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.20100000000000001},"max":{"1min":1.639,"5min":3.9809999999999999,"15min":15.523},"last":0.46700000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.80400000000000005,"15min":1.351},"min":{"1min":0.36599999999999999,"5min":0.312,"15min":0.27400000000000002},"max":{"1min":0.71799999999999997,"5min":6.2370000000000001,"15min":77.793000000000006},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.96599999999999997,"15min":1.611},"min":{"1min":0.40600000000000003,"5min":0.32300000000000001,"15min":0.27900000000000003},"max":{"1min":0.86299999999999999,"5min":9.7370000000000001,"15min":76.646000000000001},"last":0.65500000000000003}]},{"osd":5,"last update":"Sun Oct 1 09:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.409,"15min":1.466},"min":{"1min":0.38700000000000001,"5min":0.30399999999999999,"15min":0.27500000000000002},"max":{"1min":0.70699999999999996,"5min":46.256999999999998,"15min":46.256999999999998},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.3169999999999999,"15min":1.5960000000000001},"min":{"1min":0.41499999999999998,"5min":0.307,"15min":0.27700000000000002},"max":{"1min":0.81599999999999995,"5min":46.284999999999997,"15min":46.284999999999997},"last":0.52700000000000002}]},{"osd":6,"last update":"Sun Oct 1 09:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.82699999999999996,"15min":1.667},"min":{"1min":0.41999999999999998,"5min":0.379,"15min":0.315},"max":{"1min":0.748,"5min":8.3230000000000004,"15min":73.887},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.85099999999999998,"15min":1.7470000000000001},"min":{"1min":0.439,"5min":0.379,"15min":0.33600000000000002},"max":{"1min":0.70599999999999996,"5min":4.2169999999999996,"15min":74.016999999999996},"last":0.62}]},{"osd":7,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.161,"15min":1.706},"min":{"1min":0.45300000000000001,"5min":0.40400000000000003,"15min":0.30199999999999999},"max":{"1min":0.85099999999999998,"5min":10.574,"15min":13.085000000000001},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.63400000000000001,"5min":1.2110000000000001,"15min":1.4039999999999999},"min":{"1min":0.41699999999999998,"5min":0.36699999999999999,"15min":0.33300000000000002},"max":{"1min":0.81599999999999995,"5min":10.609,"15min":11.109999999999999},"last":0.70699999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019492,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":921296,"kb_used_data":27016,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":92823856,"statfs":{"total":95995035648,"available":95051628544,"internally_reserved":0,"allocated":27664384,"data_stored":26679933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:50:08 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":2.4990000000000001,"15min":2.585},"min":{"1min":0.224,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.65100000000000002,"5min":95.548000000000002,"15min":95.548000000000002},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":2.532,"15min":2.7690000000000001},"min":{"1min":0.20699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.65500000000000003,"5min":95.253,"15min":95.253},"last":0.438}]},{"osd":1,"last update":"Sun Oct 1 09:50:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.76200000000000001,"15min":4.0810000000000004},"min":{"1min":0.35399999999999998,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":0.73099999999999998,"5min":7.5979999999999999,"15min":165.666},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.78300000000000003,"15min":4.0270000000000001},"min":{"1min":0.30599999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.71499999999999997,"5min":13.023999999999999,"15min":165.70500000000001},"last":0.48999999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.83099999999999996,"15min":1.181},"min":{"1min":0.34799999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.70599999999999996,"5min":7.6390000000000002,"15min":10.161},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.83099999999999996,"15min":1.3009999999999999},"min":{"1min":0.372,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.222,"5min":7.6619999999999999,"15min":10.625},"last":0.63300000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:50:08 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.82499999999999996,"15min":1.794},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":0.63200000000000001,"5min":11.667,"15min":140.881},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.86599999999999999,"15min":1.885},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.5029999999999999,"5min":13.176,"15min":140.911},"last":0.55300000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":1.746,"15min":1.667},"min":{"1min":0.32300000000000001,"5min":0.27200000000000002,"15min":0.26600000000000001},"max":{"1min":0.77700000000000002,"5min":37.548000000000002,"15min":110.003},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.631,"15min":1.627},"min":{"1min":0.33900000000000002,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":0.66300000000000003,"5min":37.567,"15min":108.56399999999999},"last":0.66600000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.82599999999999996,"15min":0.84199999999999997},"min":{"1min":0.32000000000000001,"5min":0.318,"15min":0.27700000000000002},"max":{"1min":0.83799999999999997,"5min":11.635999999999999,"15min":11.635999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.89600000000000002,"15min":0.98399999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.314},"max":{"1min":0.77900000000000003,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":0.749}]},{"osd":7,"last update":"Sun Oct 1 09:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.0600000000000001,"15min":1.0229999999999999},"min":{"1min":0.35799999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.88800000000000001,"5min":11.694000000000001,"15min":38.034999999999997},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.123,"15min":1.2490000000000001},"min":{"1min":0.42599999999999999,"5min":0.34599999999999997,"15min":0.34200000000000003},"max":{"1min":0.82699999999999996,"5min":12.228,"15min":37.723999999999997},"last":0.77600000000000002}]}]},{"osd":5,"up_from":379,"seq":1627792605399,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":378240,"kb_used_data":9592,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":93366912,"statfs":{"total":95995035648,"available":95607717888,"internally_reserved":0,"allocated":9822208,"data_stored":8849589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":2.02,"15min":2.0289999999999999},"min":{"1min":0.28899999999999998,"5min":0.222,"15min":0.21099999999999999},"max":{"1min":0.72199999999999998,"5min":99.609999999999999,"15min":99.609999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":2.8149999999999999,"15min":2.258},"min":{"1min":0.33400000000000002,"5min":0.215,"15min":0.20399999999999999},"max":{"1min":1.3660000000000001,"5min":104.01600000000001,"15min":104.01600000000001},"last":0.67600000000000005}]},{"osd":1,"last update":"Sun Oct 1 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.81799999999999995,"15min":4.6699999999999999},"min":{"1min":0.30399999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.6789999999999998,"5min":7.6820000000000004,"15min":295.71300000000002},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.81699999999999995,"15min":4.7370000000000001},"min":{"1min":0.34300000000000003,"5min":0.26600000000000001,"15min":0.25700000000000001},"max":{"1min":0.70299999999999996,"5min":14.859999999999999,"15min":299.59399999999999},"last":0.438}]},{"osd":2,"last update":"Sun Oct 1 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.81799999999999995,"15min":1.3100000000000001},"min":{"1min":0.38400000000000001,"5min":0.28999999999999998,"15min":0.27600000000000002},"max":{"1min":0.755,"5min":12.946999999999999,"15min":14.199999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.86499999999999999,"15min":1.264},"min":{"1min":0.35299999999999998,"5min":0.29599999999999999,"15min":0.27100000000000002},"max":{"1min":1.109,"5min":6.5609999999999999,"15min":14.430999999999999},"last":0.53000000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.98599999999999999,"15min":1.3460000000000001},"min":{"1min":0.34899999999999998,"5min":0.30599999999999999,"15min":0.28100000000000003},"max":{"1min":0.82499999999999996,"5min":11.983000000000001,"15min":14.004},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.81299999999999994,"15min":1.4159999999999999},"min":{"1min":0.42299999999999999,"5min":0.377,"15min":0.32500000000000001},"max":{"1min":0.79400000000000004,"5min":12.201000000000001,"15min":13.991},"last":0.71899999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.92400000000000004,"15min":1.363},"min":{"1min":0.29499999999999998,"5min":0.27900000000000003,"15min":0.24399999999999999},"max":{"1min":0.72499999999999998,"5min":12.266,"15min":122.971},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.874,"15min":1.3560000000000001},"min":{"1min":0.30499999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.165,"5min":16.277999999999999,"15min":122.89100000000001},"last":0.59899999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.90900000000000003,"15min":0.95699999999999996},"min":{"1min":0.31900000000000001,"5min":0.29399999999999998,"15min":0.28299999999999997},"max":{"1min":0.80000000000000004,"5min":14.396000000000001,"15min":14.396000000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":1.0149999999999999,"15min":1.0669999999999999},"min":{"1min":0.32800000000000001,"5min":0.31,"15min":0.30499999999999999},"max":{"1min":1.087,"5min":12.157,"15min":14.193},"last":0.76400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.3280000000000001,"15min":1.159},"min":{"1min":0.314,"5min":0.311,"15min":0.29699999999999999},"max":{"1min":1.1459999999999999,"5min":23.02,"15min":23.02},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":1.04,"15min":1.1020000000000001},"min":{"1min":0.435,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":0.77200000000000002,"5min":12.121,"15min":12.500999999999999},"last":0.69399999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:51:08.049 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T09:51:08.402 INFO:teuthology.orchestra.run.smithi138.stderr:Inferring config /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T09:51:08.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:08 smithi190 ceph-mon[160435]: from='client.48572 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:51:08.817 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:08 smithi138 ceph-mon[185886]: from='client.48572 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:51:08.817 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:08 smithi138 ceph-mon[192513]: from='client.48572 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T09:51:09.763 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:09 smithi138 ceph-mon[185886]: pgmap v1830: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:09.764 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:09 smithi138 ceph-mon[192513]: pgmap v1830: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:09.806 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:09 smithi190 ceph-mon[160435]: pgmap v1830: 73 pgs: 73 active+clean; 8.6 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail 2023-10-01T09:51:10.114 INFO:teuthology.orchestra.run.smithi138.stdout: 2023-10-01T09:51:10.116 INFO:teuthology.orchestra.run.smithi138.stderr:dumped all 2023-10-01T09:51:10.496 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:10 smithi138 ceph-mon[192513]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:51:10.496 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:10 smithi138 ceph-mon[185886]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:51:10.778 INFO:teuthology.orchestra.run.smithi138.stdout:{"pg_ready":true,"pg_map":{"version":1830,"stamp":"2023-10-01T09:51:08.432112+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":12257,"num_read_kb":25874,"num_write":1950,"num_write_kb":27835,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":540,"num_bytes_recovered":20350555,"num_keys_recovered":110,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2144,"ondisk_log_size":2144,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":222,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6054092,"kb_used_data":135692,"kb_used_omap":50,"kb_used_meta":5918285,"kb_avail":743907124,"statfs":{"total":767960285184,"available":761760894976,"internally_reserved":0,"allocated":138948608,"data_stored":125968461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51738,"internal_metadata":6060324326},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002563"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":401,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.041117+0000","last_change":"2023-10-01T09:49:37.884604+0000","last_active":"2023-10-01T09:50:31.041117+0000","last_peered":"2023-10-01T09:50:31.041117+0000","last_clean":"2023-10-01T09:50:31.041117+0000","last_became_active":"2023-10-01T09:49:37.884169+0000","last_became_peered":"2023-10-01T09:49:37.884169+0000","last_unstale":"2023-10-01T09:50:31.041117+0000","last_undegraded":"2023-10-01T09:50:31.041117+0000","last_fullsized":"2023-10-01T09:50:31.041117+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":75,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523182+0000","last_change":"2023-10-01T09:50:08.985245+0000","last_active":"2023-10-01T09:50:30.523182+0000","last_peered":"2023-10-01T09:50:30.523182+0000","last_clean":"2023-10-01T09:50:30.523182+0000","last_became_active":"2023-10-01T09:50:08.984895+0000","last_became_peered":"2023-10-01T09:50:08.984895+0000","last_unstale":"2023-10-01T09:50:30.523182+0000","last_undegraded":"2023-10-01T09:50:30.523182+0000","last_fullsized":"2023-10-01T09:50:30.523182+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:17:35.675391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":619,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-10-01T09:50:35.020892+0000","last_change":"2023-10-01T09:37:22.670781+0000","last_active":"2023-10-01T09:50:35.020892+0000","last_peered":"2023-10-01T09:50:35.020892+0000","last_clean":"2023-10-01T09:50:35.020892+0000","last_became_active":"2023-10-01T09:37:22.670397+0000","last_became_peered":"2023-10-01T09:37:22.670397+0000","last_unstale":"2023-10-01T09:50:35.020892+0000","last_undegraded":"2023-10-01T09:50:35.020892+0000","last_fullsized":"2023-10-01T09:50:35.020892+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:51.412664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":678,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-10-01T09:50:33.888792+0000","last_change":"2023-10-01T09:41:22.759014+0000","last_active":"2023-10-01T09:50:33.888792+0000","last_peered":"2023-10-01T09:50:33.888792+0000","last_clean":"2023-10-01T09:50:33.888792+0000","last_became_active":"2023-10-01T09:41:22.758712+0000","last_became_peered":"2023-10-01T09:41:22.758712+0000","last_unstale":"2023-10-01T09:50:33.888792+0000","last_undegraded":"2023-10-01T09:50:33.888792+0000","last_fullsized":"2023-10-01T09:50:33.888792+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:16:08.919664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":440,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523460+0000","last_change":"2023-10-01T09:50:08.986396+0000","last_active":"2023-10-01T09:50:30.523460+0000","last_peered":"2023-10-01T09:50:30.523460+0000","last_clean":"2023-10-01T09:50:30.523460+0000","last_became_active":"2023-10-01T09:50:08.985734+0000","last_became_peered":"2023-10-01T09:50:08.985734+0000","last_unstale":"2023-10-01T09:50:30.523460+0000","last_undegraded":"2023-10-01T09:50:30.523460+0000","last_fullsized":"2023-10-01T09:50:30.523460+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:47:50.441209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":149,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.527105+0000","last_change":"2023-10-01T09:50:29.052725+0000","last_active":"2023-10-01T09:50:30.527105+0000","last_peered":"2023-10-01T09:50:30.527105+0000","last_clean":"2023-10-01T09:50:30.527105+0000","last_became_active":"2023-10-01T09:50:29.052221+0000","last_became_peered":"2023-10-01T09:50:29.052221+0000","last_unstale":"2023-10-01T09:50:30.527105+0000","last_undegraded":"2023-10-01T09:50:30.527105+0000","last_fullsized":"2023-10-01T09:50:30.527105+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":563,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.038623+0000","last_change":"2023-10-01T09:49:50.024454+0000","last_active":"2023-10-01T09:50:31.038623+0000","last_peered":"2023-10-01T09:50:31.038623+0000","last_clean":"2023-10-01T09:50:31.038623+0000","last_became_active":"2023-10-01T09:49:50.024154+0000","last_became_peered":"2023-10-01T09:49:50.024154+0000","last_unstale":"2023-10-01T09:50:31.038623+0000","last_undegraded":"2023-10-01T09:50:31.038623+0000","last_fullsized":"2023-10-01T09:50:31.038623+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:57:07.108325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":577,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523463+0000","last_change":"2023-10-01T09:50:08.986145+0000","last_active":"2023-10-01T09:50:30.523463+0000","last_peered":"2023-10-01T09:50:30.523463+0000","last_clean":"2023-10-01T09:50:30.523463+0000","last_became_active":"2023-10-01T09:50:08.984987+0000","last_became_peered":"2023-10-01T09:50:08.984987+0000","last_unstale":"2023-10-01T09:50:30.523463+0000","last_undegraded":"2023-10-01T09:50:30.523463+0000","last_fullsized":"2023-10-01T09:50:30.523463+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:22:03.682298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":328,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.041147+0000","last_change":"2023-10-01T09:40:42.548938+0000","last_active":"2023-10-01T09:50:31.041147+0000","last_peered":"2023-10-01T09:50:31.041147+0000","last_clean":"2023-10-01T09:50:31.041147+0000","last_became_active":"2023-10-01T09:40:42.548567+0000","last_became_peered":"2023-10-01T09:40:42.548567+0000","last_unstale":"2023-10-01T09:50:31.041147+0000","last_undegraded":"2023-10-01T09:50:31.041147+0000","last_fullsized":"2023-10-01T09:50:31.041147+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":56,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523545+0000","last_change":"2023-10-01T09:50:08.948042+0000","last_active":"2023-10-01T09:50:30.523545+0000","last_peered":"2023-10-01T09:50:30.523545+0000","last_clean":"2023-10-01T09:50:30.523545+0000","last_became_active":"2023-10-01T09:50:08.947663+0000","last_became_peered":"2023-10-01T09:50:08.947663+0000","last_unstale":"2023-10-01T09:50:30.523545+0000","last_undegraded":"2023-10-01T09:50:30.523545+0000","last_fullsized":"2023-10-01T09:50:30.523545+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:26.568510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":141,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040595+0000","last_change":"2023-10-01T09:40:47.753583+0000","last_active":"2023-10-01T09:50:31.040595+0000","last_peered":"2023-10-01T09:50:31.040595+0000","last_clean":"2023-10-01T09:50:31.040595+0000","last_became_active":"2023-10-01T09:40:47.742703+0000","last_became_peered":"2023-10-01T09:40:47.742703+0000","last_unstale":"2023-10-01T09:50:31.040595+0000","last_undegraded":"2023-10-01T09:50:31.040595+0000","last_fullsized":"2023-10-01T09:50:31.040595+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":151,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.026241+0000","last_change":"2023-10-01T09:49:37.882756+0000","last_active":"2023-10-01T09:50:31.026241+0000","last_peered":"2023-10-01T09:50:31.026241+0000","last_clean":"2023-10-01T09:50:31.026241+0000","last_became_active":"2023-10-01T09:49:37.882460+0000","last_became_peered":"2023-10-01T09:49:37.882460+0000","last_unstale":"2023-10-01T09:50:31.026241+0000","last_undegraded":"2023-10-01T09:50:31.026241+0000","last_fullsized":"2023-10-01T09:50:31.026241+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:04:53.618777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"85'22","reported_seq":517,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040084+0000","last_change":"2023-10-01T09:40:13.191848+0000","last_active":"2023-10-01T09:50:31.040084+0000","last_peered":"2023-10-01T09:50:31.040084+0000","last_clean":"2023-10-01T09:50:31.040084+0000","last_became_active":"2023-10-01T09:40:13.191470+0000","last_became_peered":"2023-10-01T09:40:13.191470+0000","last_unstale":"2023-10-01T09:50:31.040084+0000","last_undegraded":"2023-10-01T09:50:31.040084+0000","last_fullsized":"2023-10-01T09:50:31.040084+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":398,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040376+0000","last_change":"2023-10-01T09:40:13.193975+0000","last_active":"2023-10-01T09:50:31.040376+0000","last_peered":"2023-10-01T09:50:31.040376+0000","last_clean":"2023-10-01T09:50:31.040376+0000","last_became_active":"2023-10-01T09:40:13.193554+0000","last_became_peered":"2023-10-01T09:40:13.193554+0000","last_unstale":"2023-10-01T09:50:31.040376+0000","last_undegraded":"2023-10-01T09:50:31.040376+0000","last_fullsized":"2023-10-01T09:50:31.040376+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":749,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040969+0000","last_change":"2023-10-01T09:49:37.973230+0000","last_active":"2023-10-01T09:50:31.040969+0000","last_peered":"2023-10-01T09:50:31.040969+0000","last_clean":"2023-10-01T09:50:31.040969+0000","last_became_active":"2023-10-01T09:49:37.881204+0000","last_became_peered":"2023-10-01T09:49:37.881204+0000","last_unstale":"2023-10-01T09:50:31.040969+0000","last_undegraded":"2023-10-01T09:50:31.040969+0000","last_fullsized":"2023-10-01T09:50:31.040969+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":374,"num_keys_recovered":110,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":449,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523519+0000","last_change":"2023-10-01T09:50:08.945926+0000","last_active":"2023-10-01T09:50:30.523519+0000","last_peered":"2023-10-01T09:50:30.523519+0000","last_clean":"2023-10-01T09:50:30.523519+0000","last_became_active":"2023-10-01T09:50:08.945623+0000","last_became_peered":"2023-10-01T09:50:08.945623+0000","last_unstale":"2023-10-01T09:50:30.523519+0000","last_undegraded":"2023-10-01T09:50:30.523519+0000","last_fullsized":"2023-10-01T09:50:30.523519+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:21:58.878782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":540,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.038371+0000","last_change":"2023-10-01T09:49:49.885502+0000","last_active":"2023-10-01T09:50:31.038371+0000","last_peered":"2023-10-01T09:50:31.038371+0000","last_clean":"2023-10-01T09:50:31.038371+0000","last_became_active":"2023-10-01T09:49:49.885219+0000","last_became_peered":"2023-10-01T09:49:49.885219+0000","last_unstale":"2023-10-01T09:50:31.038371+0000","last_undegraded":"2023-10-01T09:50:31.038371+0000","last_fullsized":"2023-10-01T09:50:31.038371+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:14:36.619704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":130,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040709+0000","last_change":"2023-10-01T09:40:55.385158+0000","last_active":"2023-10-01T09:50:31.040709+0000","last_peered":"2023-10-01T09:50:31.040709+0000","last_clean":"2023-10-01T09:50:31.040709+0000","last_became_active":"2023-10-01T09:40:55.384881+0000","last_became_peered":"2023-10-01T09:40:55.384881+0000","last_unstale":"2023-10-01T09:50:31.040709+0000","last_undegraded":"2023-10-01T09:50:31.040709+0000","last_fullsized":"2023-10-01T09:50:31.040709+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":627,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.041042+0000","last_change":"2023-10-01T09:32:27.450187+0000","last_active":"2023-10-01T09:50:31.041042+0000","last_peered":"2023-10-01T09:50:31.041042+0000","last_clean":"2023-10-01T09:50:31.041042+0000","last_became_active":"2023-10-01T09:32:27.449900+0000","last_became_peered":"2023-10-01T09:32:27.449900+0000","last_unstale":"2023-10-01T09:50:31.041042+0000","last_undegraded":"2023-10-01T09:50:31.041042+0000","last_fullsized":"2023-10-01T09:50:31.041042+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":641,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.038576+0000","last_change":"2023-10-01T09:40:47.736535+0000","last_active":"2023-10-01T09:50:31.038576+0000","last_peered":"2023-10-01T09:50:31.038576+0000","last_clean":"2023-10-01T09:50:31.038576+0000","last_became_active":"2023-10-01T09:40:47.736240+0000","last_became_peered":"2023-10-01T09:40:47.736240+0000","last_unstale":"2023-10-01T09:50:31.038576+0000","last_undegraded":"2023-10-01T09:50:31.038576+0000","last_fullsized":"2023-10-01T09:50:31.038576+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:41:05.404024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":186,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.526615+0000","last_change":"2023-10-01T09:50:28.522830+0000","last_active":"2023-10-01T09:50:30.526615+0000","last_peered":"2023-10-01T09:50:30.526615+0000","last_clean":"2023-10-01T09:50:30.526615+0000","last_became_active":"2023-10-01T09:50:28.522458+0000","last_became_peered":"2023-10-01T09:50:28.522458+0000","last_unstale":"2023-10-01T09:50:30.526615+0000","last_undegraded":"2023-10-01T09:50:30.526615+0000","last_fullsized":"2023-10-01T09:50:30.526615+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":122,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.522578+0000","last_change":"2023-10-01T09:50:08.985351+0000","last_active":"2023-10-01T09:50:30.522578+0000","last_peered":"2023-10-01T09:50:30.522578+0000","last_clean":"2023-10-01T09:50:30.522578+0000","last_became_active":"2023-10-01T09:50:08.985061+0000","last_became_peered":"2023-10-01T09:50:08.985061+0000","last_unstale":"2023-10-01T09:50:30.522578+0000","last_undegraded":"2023-10-01T09:50:30.522578+0000","last_fullsized":"2023-10-01T09:50:30.522578+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:10:22.545254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":139,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040831+0000","last_change":"2023-10-01T09:40:47.755308+0000","last_active":"2023-10-01T09:50:31.040831+0000","last_peered":"2023-10-01T09:50:31.040831+0000","last_clean":"2023-10-01T09:50:31.040831+0000","last_became_active":"2023-10-01T09:40:47.742737+0000","last_became_peered":"2023-10-01T09:40:47.742737+0000","last_unstale":"2023-10-01T09:50:31.040831+0000","last_undegraded":"2023-10-01T09:50:31.040831+0000","last_fullsized":"2023-10-01T09:50:31.040831+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":336,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040528+0000","last_change":"2023-10-01T09:40:47.741416+0000","last_active":"2023-10-01T09:50:31.040528+0000","last_peered":"2023-10-01T09:50:31.040528+0000","last_clean":"2023-10-01T09:50:31.040528+0000","last_became_active":"2023-10-01T09:40:47.741007+0000","last_became_peered":"2023-10-01T09:40:47.741007+0000","last_unstale":"2023-10-01T09:50:31.040528+0000","last_undegraded":"2023-10-01T09:50:31.040528+0000","last_fullsized":"2023-10-01T09:50:31.040528+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":610,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040456+0000","last_change":"2023-10-01T09:32:27.485401+0000","last_active":"2023-10-01T09:50:31.040456+0000","last_peered":"2023-10-01T09:50:31.040456+0000","last_clean":"2023-10-01T09:50:31.040456+0000","last_became_active":"2023-10-01T09:32:27.450218+0000","last_became_peered":"2023-10-01T09:32:27.450218+0000","last_unstale":"2023-10-01T09:50:31.040456+0000","last_undegraded":"2023-10-01T09:50:31.040456+0000","last_fullsized":"2023-10-01T09:50:31.040456+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":567,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040236+0000","last_change":"2023-10-01T09:50:15.286925+0000","last_active":"2023-10-01T09:50:31.040236+0000","last_peered":"2023-10-01T09:50:31.040236+0000","last_clean":"2023-10-01T09:50:31.040236+0000","last_became_active":"2023-10-01T09:50:15.286576+0000","last_became_peered":"2023-10-01T09:50:15.286576+0000","last_unstale":"2023-10-01T09:50:31.040236+0000","last_undegraded":"2023-10-01T09:50:31.040236+0000","last_fullsized":"2023-10-01T09:50:31.040236+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":696,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040659+0000","last_change":"2023-10-01T09:33:11.667104+0000","last_active":"2023-10-01T09:50:31.040659+0000","last_peered":"2023-10-01T09:50:31.040659+0000","last_clean":"2023-10-01T09:50:31.040659+0000","last_became_active":"2023-10-01T09:33:11.666710+0000","last_became_peered":"2023-10-01T09:33:11.666710+0000","last_unstale":"2023-10-01T09:50:31.040659+0000","last_undegraded":"2023-10-01T09:50:31.040659+0000","last_fullsized":"2023-10-01T09:50:31.040659+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":370,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.526794+0000","last_change":"2023-10-01T09:50:29.052720+0000","last_active":"2023-10-01T09:50:30.526794+0000","last_peered":"2023-10-01T09:50:30.526794+0000","last_clean":"2023-10-01T09:50:30.526794+0000","last_became_active":"2023-10-01T09:50:29.052306+0000","last_became_peered":"2023-10-01T09:50:29.052306+0000","last_unstale":"2023-10-01T09:50:30.526794+0000","last_undegraded":"2023-10-01T09:50:30.526794+0000","last_fullsized":"2023-10-01T09:50:30.526794+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":336,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040450+0000","last_change":"2023-10-01T09:37:22.681085+0000","last_active":"2023-10-01T09:50:31.040450+0000","last_peered":"2023-10-01T09:50:31.040450+0000","last_clean":"2023-10-01T09:50:31.040450+0000","last_became_active":"2023-10-01T09:37:22.680555+0000","last_became_peered":"2023-10-01T09:37:22.680555+0000","last_unstale":"2023-10-01T09:50:31.040450+0000","last_undegraded":"2023-10-01T09:50:31.040450+0000","last_fullsized":"2023-10-01T09:50:31.040450+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":597,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.038398+0000","last_change":"2023-10-01T09:32:27.451434+0000","last_active":"2023-10-01T09:50:31.038398+0000","last_peered":"2023-10-01T09:50:31.038398+0000","last_clean":"2023-10-01T09:50:31.038398+0000","last_became_active":"2023-10-01T09:32:27.450267+0000","last_became_peered":"2023-10-01T09:32:27.450267+0000","last_unstale":"2023-10-01T09:50:31.038398+0000","last_undegraded":"2023-10-01T09:50:31.038398+0000","last_fullsized":"2023-10-01T09:50:31.038398+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:05:41.678950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":210,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.522266+0000","last_change":"2023-10-01T09:50:08.946873+0000","last_active":"2023-10-01T09:50:30.522266+0000","last_peered":"2023-10-01T09:50:30.522266+0000","last_clean":"2023-10-01T09:50:30.522266+0000","last_became_active":"2023-10-01T09:50:08.946622+0000","last_became_peered":"2023-10-01T09:50:08.946622+0000","last_unstale":"2023-10-01T09:50:30.522266+0000","last_undegraded":"2023-10-01T09:50:30.522266+0000","last_fullsized":"2023-10-01T09:50:30.522266+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:19:11.600014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":276,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040256+0000","last_change":"2023-10-01T09:50:15.286778+0000","last_active":"2023-10-01T09:50:31.040256+0000","last_peered":"2023-10-01T09:50:31.040256+0000","last_clean":"2023-10-01T09:50:31.040256+0000","last_became_active":"2023-10-01T09:50:15.286418+0000","last_became_peered":"2023-10-01T09:50:15.286418+0000","last_unstale":"2023-10-01T09:50:31.040256+0000","last_undegraded":"2023-10-01T09:50:31.040256+0000","last_fullsized":"2023-10-01T09:50:31.040256+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":411,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.526908+0000","last_change":"2023-10-01T09:50:29.052667+0000","last_active":"2023-10-01T09:50:30.526908+0000","last_peered":"2023-10-01T09:50:30.526908+0000","last_clean":"2023-10-01T09:50:30.526908+0000","last_became_active":"2023-10-01T09:50:29.052256+0000","last_became_peered":"2023-10-01T09:50:29.052256+0000","last_unstale":"2023-10-01T09:50:30.526908+0000","last_undegraded":"2023-10-01T09:50:30.526908+0000","last_fullsized":"2023-10-01T09:50:30.526908+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":673,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.041526+0000","last_change":"2023-10-01T09:50:15.097746+0000","last_active":"2023-10-01T09:50:31.041526+0000","last_peered":"2023-10-01T09:50:31.041526+0000","last_clean":"2023-10-01T09:50:31.041526+0000","last_became_active":"2023-10-01T09:50:15.097289+0000","last_became_peered":"2023-10-01T09:50:15.097289+0000","last_unstale":"2023-10-01T09:50:31.041526+0000","last_undegraded":"2023-10-01T09:50:31.041526+0000","last_fullsized":"2023-10-01T09:50:31.041526+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":519,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.039995+0000","last_change":"2023-10-01T09:49:37.885772+0000","last_active":"2023-10-01T09:50:31.039995+0000","last_peered":"2023-10-01T09:50:31.039995+0000","last_clean":"2023-10-01T09:50:31.039995+0000","last_became_active":"2023-10-01T09:49:37.885461+0000","last_became_peered":"2023-10-01T09:49:37.885461+0000","last_unstale":"2023-10-01T09:50:31.039995+0000","last_undegraded":"2023-10-01T09:50:31.039995+0000","last_fullsized":"2023-10-01T09:50:31.039995+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":351,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.041459+0000","last_change":"2023-10-01T09:40:42.561802+0000","last_active":"2023-10-01T09:50:31.041459+0000","last_peered":"2023-10-01T09:50:31.041459+0000","last_clean":"2023-10-01T09:50:31.041459+0000","last_became_active":"2023-10-01T09:40:42.561532+0000","last_became_peered":"2023-10-01T09:40:42.561532+0000","last_unstale":"2023-10-01T09:50:31.041459+0000","last_undegraded":"2023-10-01T09:50:31.041459+0000","last_fullsized":"2023-10-01T09:50:31.041459+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":377,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523374+0000","last_change":"2023-10-01T09:50:08.986062+0000","last_active":"2023-10-01T09:50:30.523374+0000","last_peered":"2023-10-01T09:50:30.523374+0000","last_clean":"2023-10-01T09:50:30.523374+0000","last_became_active":"2023-10-01T09:50:08.985419+0000","last_became_peered":"2023-10-01T09:50:08.985419+0000","last_unstale":"2023-10-01T09:50:30.523374+0000","last_undegraded":"2023-10-01T09:50:30.523374+0000","last_fullsized":"2023-10-01T09:50:30.523374+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:08:59.087065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":529,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.026507+0000","last_change":"2023-10-01T09:37:22.673387+0000","last_active":"2023-10-01T09:50:31.026507+0000","last_peered":"2023-10-01T09:50:31.026507+0000","last_clean":"2023-10-01T09:50:31.026507+0000","last_became_active":"2023-10-01T09:37:22.673043+0000","last_became_peered":"2023-10-01T09:37:22.673043+0000","last_unstale":"2023-10-01T09:50:31.026507+0000","last_undegraded":"2023-10-01T09:50:31.026507+0000","last_fullsized":"2023-10-01T09:50:31.026507+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:05:56.097027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"585'392","reported_seq":2103,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-10-01T09:50:35.021491+0000","last_change":"2023-10-01T09:49:19.958909+0000","last_active":"2023-10-01T09:50:35.021491+0000","last_peered":"2023-10-01T09:50:35.021491+0000","last_clean":"2023-10-01T09:50:35.021491+0000","last_became_active":"2023-10-01T09:48:45.360357+0000","last_became_peered":"2023-10-01T09:48:45.360357+0000","last_unstale":"2023-10-01T09:50:35.021491+0000","last_undegraded":"2023-10-01T09:50:35.021491+0000","last_fullsized":"2023-10-01T09:50:35.021491+0000","mapping_epoch":569,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":570,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":392,"log_dups_size":0,"ondisk_log_size":392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:08:46.766586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054191000000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2673,"num_read_kb":2980,"num_write":346,"num_write_kb":1499,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":355,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040870+0000","last_change":"2023-10-01T09:49:37.881458+0000","last_active":"2023-10-01T09:50:31.040870+0000","last_peered":"2023-10-01T09:50:31.040870+0000","last_clean":"2023-10-01T09:50:31.040870+0000","last_became_active":"2023-10-01T09:49:37.880898+0000","last_became_peered":"2023-10-01T09:49:37.880898+0000","last_unstale":"2023-10-01T09:50:31.040870+0000","last_undegraded":"2023-10-01T09:50:31.040870+0000","last_fullsized":"2023-10-01T09:50:31.040870+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":142,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.026289+0000","last_change":"2023-10-01T09:40:42.560394+0000","last_active":"2023-10-01T09:50:31.026289+0000","last_peered":"2023-10-01T09:50:31.026289+0000","last_clean":"2023-10-01T09:50:31.026289+0000","last_became_active":"2023-10-01T09:40:42.560094+0000","last_became_peered":"2023-10-01T09:40:42.560094+0000","last_unstale":"2023-10-01T09:50:31.026289+0000","last_undegraded":"2023-10-01T09:50:31.026289+0000","last_fullsized":"2023-10-01T09:50:31.026289+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:55.538456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"585'228","reported_seq":1955,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040142+0000","last_change":"2023-10-01T09:49:19.923862+0000","last_active":"2023-10-01T09:50:31.040142+0000","last_peered":"2023-10-01T09:50:31.040142+0000","last_clean":"2023-10-01T09:50:31.040142+0000","last_became_active":"2023-10-01T09:33:11.662904+0000","last_became_peered":"2023-10-01T09:33:11.662904+0000","last_unstale":"2023-10-01T09:50:31.040142+0000","last_undegraded":"2023-10-01T09:50:31.040142+0000","last_fullsized":"2023-10-01T09:50:31.040142+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":228,"log_dups_size":0,"ondisk_log_size":228,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2216,"num_read_kb":2036,"num_write":182,"num_write_kb":4239,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":491,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040626+0000","last_change":"2023-10-01T09:39:12.701479+0000","last_active":"2023-10-01T09:50:31.040626+0000","last_peered":"2023-10-01T09:50:31.040626+0000","last_clean":"2023-10-01T09:50:31.040626+0000","last_became_active":"2023-10-01T09:39:12.701141+0000","last_became_peered":"2023-10-01T09:39:12.701141+0000","last_unstale":"2023-10-01T09:50:31.040626+0000","last_undegraded":"2023-10-01T09:50:31.040626+0000","last_fullsized":"2023-10-01T09:50:31.040626+0000","mapping_epoch":461,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":462,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":328,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040604+0000","last_change":"2023-10-01T09:49:37.881294+0000","last_active":"2023-10-01T09:50:31.040604+0000","last_peered":"2023-10-01T09:50:31.040604+0000","last_clean":"2023-10-01T09:50:31.040604+0000","last_became_active":"2023-10-01T09:49:37.880930+0000","last_became_peered":"2023-10-01T09:49:37.880930+0000","last_unstale":"2023-10-01T09:50:31.040604+0000","last_undegraded":"2023-10-01T09:50:31.040604+0000","last_fullsized":"2023-10-01T09:50:31.040604+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"584'227","reported_seq":1509,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.521705+0000","last_change":"2023-10-01T09:50:02.916276+0000","last_active":"2023-10-01T09:50:30.521705+0000","last_peered":"2023-10-01T09:50:30.521705+0000","last_clean":"2023-10-01T09:50:30.521705+0000","last_became_active":"2023-10-01T09:50:02.915490+0000","last_became_peered":"2023-10-01T09:50:02.915490+0000","last_unstale":"2023-10-01T09:50:30.521705+0000","last_undegraded":"2023-10-01T09:50:30.521705+0000","last_fullsized":"2023-10-01T09:50:30.521705+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":227,"log_dups_size":0,"ondisk_log_size":227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:41:44.100291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040081099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1258,"num_read_kb":4518,"num_write":199,"num_write_kb":3425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":691,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.026199+0000","last_change":"2023-10-01T09:49:37.882934+0000","last_active":"2023-10-01T09:50:31.026199+0000","last_peered":"2023-10-01T09:50:31.026199+0000","last_clean":"2023-10-01T09:50:31.026199+0000","last_became_active":"2023-10-01T09:49:37.882670+0000","last_became_peered":"2023-10-01T09:49:37.882670+0000","last_unstale":"2023-10-01T09:50:31.026199+0000","last_undegraded":"2023-10-01T09:50:31.026199+0000","last_fullsized":"2023-10-01T09:50:31.026199+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:46:43.118412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":142,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.026502+0000","last_change":"2023-10-01T09:40:42.541105+0000","last_active":"2023-10-01T09:50:31.026502+0000","last_peered":"2023-10-01T09:50:31.026502+0000","last_clean":"2023-10-01T09:50:31.026502+0000","last_became_active":"2023-10-01T09:40:42.540805+0000","last_became_peered":"2023-10-01T09:40:42.540805+0000","last_unstale":"2023-10-01T09:50:31.026502+0000","last_undegraded":"2023-10-01T09:50:31.026502+0000","last_fullsized":"2023-10-01T09:50:31.026502+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:42:41.254928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"585'397","reported_seq":2235,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.041208+0000","last_change":"2023-10-01T09:49:40.901604+0000","last_active":"2023-10-01T09:50:31.041208+0000","last_peered":"2023-10-01T09:50:31.041208+0000","last_clean":"2023-10-01T09:50:31.041208+0000","last_became_active":"2023-10-01T09:49:40.900950+0000","last_became_peered":"2023-10-01T09:49:40.900950+0000","last_unstale":"2023-10-01T09:50:31.041208+0000","last_undegraded":"2023-10-01T09:50:31.041208+0000","last_fullsized":"2023-10-01T09:50:31.041208+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":397,"log_dups_size":0,"ondisk_log_size":397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2083,"num_read_kb":6022,"num_write":364,"num_write_kb":3847,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":139,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"85'139","reported_seq":646,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040250+0000","last_change":"2023-10-01T09:40:50.774357+0000","last_active":"2023-10-01T09:50:31.040250+0000","last_peered":"2023-10-01T09:50:31.040250+0000","last_clean":"2023-10-01T09:50:31.040250+0000","last_became_active":"2023-10-01T09:40:50.773885+0000","last_became_peered":"2023-10-01T09:40:50.773885+0000","last_unstale":"2023-10-01T09:50:31.040250+0000","last_undegraded":"2023-10-01T09:50:31.040250+0000","last_fullsized":"2023-10-01T09:50:31.040250+0000","mapping_epoch":493,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":494,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T08:59:42.603095+0000","last_clean_scrub_stamp":"2023-10-01T08:59:42.603095+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":324,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.522919+0000","last_change":"2023-10-01T09:50:08.985808+0000","last_active":"2023-10-01T09:50:30.522919+0000","last_peered":"2023-10-01T09:50:30.522919+0000","last_clean":"2023-10-01T09:50:30.522919+0000","last_became_active":"2023-10-01T09:50:08.985521+0000","last_became_peered":"2023-10-01T09:50:08.985521+0000","last_unstale":"2023-10-01T09:50:30.522919+0000","last_undegraded":"2023-10-01T09:50:30.522919+0000","last_fullsized":"2023-10-01T09:50:30.522919+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:02:35.181923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":283,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.526980+0000","last_change":"2023-10-01T09:50:28.523682+0000","last_active":"2023-10-01T09:50:30.526980+0000","last_peered":"2023-10-01T09:50:30.526980+0000","last_clean":"2023-10-01T09:50:30.526980+0000","last_became_active":"2023-10-01T09:50:28.523239+0000","last_became_peered":"2023-10-01T09:50:28.523239+0000","last_unstale":"2023-10-01T09:50:30.526980+0000","last_undegraded":"2023-10-01T09:50:30.526980+0000","last_fullsized":"2023-10-01T09:50:30.526980+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"585'105","reported_seq":987,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040378+0000","last_change":"2023-10-01T09:49:19.925623+0000","last_active":"2023-10-01T09:50:31.040378+0000","last_peered":"2023-10-01T09:50:31.040378+0000","last_clean":"2023-10-01T09:50:31.040378+0000","last_became_active":"2023-10-01T09:40:42.563811+0000","last_became_peered":"2023-10-01T09:40:42.563811+0000","last_unstale":"2023-10-01T09:50:31.040378+0000","last_undegraded":"2023-10-01T09:50:31.040378+0000","last_fullsized":"2023-10-01T09:50:31.040378+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":339,"num_read_kb":3531,"num_write":98,"num_write_kb":7299,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":659,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:32.118714+0000","last_change":"2023-10-01T09:41:22.762893+0000","last_active":"2023-10-01T09:50:32.118714+0000","last_peered":"2023-10-01T09:50:32.118714+0000","last_clean":"2023-10-01T09:50:32.118714+0000","last_became_active":"2023-10-01T09:41:22.762564+0000","last_became_peered":"2023-10-01T09:41:22.762564+0000","last_unstale":"2023-10-01T09:50:32.118714+0000","last_undegraded":"2023-10-01T09:50:32.118714+0000","last_fullsized":"2023-10-01T09:50:32.118714+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:47:44.669515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":667,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.038458+0000","last_change":"2023-10-01T09:49:37.884060+0000","last_active":"2023-10-01T09:50:31.038458+0000","last_peered":"2023-10-01T09:50:31.038458+0000","last_clean":"2023-10-01T09:50:31.038458+0000","last_became_active":"2023-10-01T09:49:37.883747+0000","last_became_peered":"2023-10-01T09:49:37.883747+0000","last_unstale":"2023-10-01T09:50:31.038458+0000","last_undegraded":"2023-10-01T09:50:31.038458+0000","last_fullsized":"2023-10-01T09:50:31.038458+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:43:04.596671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"585'205","reported_seq":1365,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040704+0000","last_change":"2023-10-01T09:49:19.926295+0000","last_active":"2023-10-01T09:50:31.040704+0000","last_peered":"2023-10-01T09:50:31.040704+0000","last_clean":"2023-10-01T09:50:31.040704+0000","last_became_active":"2023-10-01T09:32:30.560205+0000","last_became_peered":"2023-10-01T09:32:30.560205+0000","last_unstale":"2023-10-01T09:50:31.040704+0000","last_undegraded":"2023-10-01T09:50:31.040704+0000","last_fullsized":"2023-10-01T09:50:31.040704+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":205,"log_dups_size":0,"ondisk_log_size":205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1233,"num_read_kb":2003,"num_write":197,"num_write_kb":615,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":615,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-10-01T09:50:35.021499+0000","last_change":"2023-10-01T09:49:37.884027+0000","last_active":"2023-10-01T09:50:35.021499+0000","last_peered":"2023-10-01T09:50:35.021499+0000","last_clean":"2023-10-01T09:50:35.021499+0000","last_became_active":"2023-10-01T09:49:37.883624+0000","last_became_peered":"2023-10-01T09:49:37.883624+0000","last_unstale":"2023-10-01T09:50:35.021499+0000","last_undegraded":"2023-10-01T09:50:35.021499+0000","last_fullsized":"2023-10-01T09:50:35.021499+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:48:03.636472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":482,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.038521+0000","last_change":"2023-10-01T09:49:50.024251+0000","last_active":"2023-10-01T09:50:31.038521+0000","last_peered":"2023-10-01T09:50:31.038521+0000","last_clean":"2023-10-01T09:50:31.038521+0000","last_became_active":"2023-10-01T09:49:50.023983+0000","last_became_peered":"2023-10-01T09:49:50.023983+0000","last_unstale":"2023-10-01T09:50:31.038521+0000","last_undegraded":"2023-10-01T09:50:31.038521+0000","last_fullsized":"2023-10-01T09:50:31.038521+0000","mapping_epoch":596,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:29:09.013661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":394,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.522658+0000","last_change":"2023-10-01T09:50:08.946132+0000","last_active":"2023-10-01T09:50:30.522658+0000","last_peered":"2023-10-01T09:50:30.522658+0000","last_clean":"2023-10-01T09:50:30.522658+0000","last_became_active":"2023-10-01T09:50:08.945792+0000","last_became_peered":"2023-10-01T09:50:08.945792+0000","last_unstale":"2023-10-01T09:50:30.522658+0000","last_undegraded":"2023-10-01T09:50:30.522658+0000","last_fullsized":"2023-10-01T09:50:30.522658+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:54:20.779232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"585'269","reported_seq":1681,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.026140+0000","last_change":"2023-10-01T09:49:19.958887+0000","last_active":"2023-10-01T09:50:31.026140+0000","last_peered":"2023-10-01T09:50:31.026140+0000","last_clean":"2023-10-01T09:50:31.026140+0000","last_became_active":"2023-10-01T09:32:30.544297+0000","last_became_peered":"2023-10-01T09:32:30.544297+0000","last_unstale":"2023-10-01T09:50:31.026140+0000","last_undegraded":"2023-10-01T09:50:31.026140+0000","last_fullsized":"2023-10-01T09:50:31.026140+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":269,"log_dups_size":0,"ondisk_log_size":269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:43:09.898513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00024152000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1384,"num_read_kb":1112,"num_write":193,"num_write_kb":2135,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":55,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":44,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523354+0000","last_change":"2023-10-01T09:50:08.946005+0000","last_active":"2023-10-01T09:50:30.523354+0000","last_peered":"2023-10-01T09:50:30.523354+0000","last_clean":"2023-10-01T09:50:30.523354+0000","last_became_active":"2023-10-01T09:50:08.945730+0000","last_became_peered":"2023-10-01T09:50:08.945730+0000","last_unstale":"2023-10-01T09:50:30.523354+0000","last_undegraded":"2023-10-01T09:50:30.523354+0000","last_fullsized":"2023-10-01T09:50:30.523354+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:57:48.661109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"584'127","reported_seq":516,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.527118+0000","last_change":"2023-10-01T09:50:28.523996+0000","last_active":"2023-10-01T09:50:30.527118+0000","last_peered":"2023-10-01T09:50:30.527118+0000","last_clean":"2023-10-01T09:50:30.527118+0000","last_became_active":"2023-10-01T09:50:28.523374+0000","last_became_peered":"2023-10-01T09:50:28.523374+0000","last_unstale":"2023-10-01T09:50:30.527118+0000","last_undegraded":"2023-10-01T09:50:30.527118+0000","last_fullsized":"2023-10-01T09:50:30.527118+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":614,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:02:05.983986+0000","last_clean_scrub_stamp":"2023-10-01T09:02:05.983986+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":777,"num_read_kb":3172,"num_write":111,"num_write_kb":2331,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":636,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.038499+0000","last_change":"2023-10-01T09:32:27.487270+0000","last_active":"2023-10-01T09:50:31.038499+0000","last_peered":"2023-10-01T09:50:31.038499+0000","last_clean":"2023-10-01T09:50:31.038499+0000","last_became_active":"2023-10-01T09:32:27.450051+0000","last_became_peered":"2023-10-01T09:32:27.450051+0000","last_unstale":"2023-10-01T09:50:31.038499+0000","last_undegraded":"2023-10-01T09:50:31.038499+0000","last_fullsized":"2023-10-01T09:50:31.038499+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:43:05.122851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":113,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.039842+0000","last_change":"2023-10-01T09:50:14.965790+0000","last_active":"2023-10-01T09:50:31.039842+0000","last_peered":"2023-10-01T09:50:31.039842+0000","last_clean":"2023-10-01T09:50:31.039842+0000","last_became_active":"2023-10-01T09:50:14.965430+0000","last_became_peered":"2023-10-01T09:50:14.965430+0000","last_unstale":"2023-10-01T09:50:31.039842+0000","last_undegraded":"2023-10-01T09:50:31.039842+0000","last_fullsized":"2023-10-01T09:50:31.039842+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":585,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-10-01T09:50:33.888686+0000","last_change":"2023-10-01T09:32:27.451595+0000","last_active":"2023-10-01T09:50:33.888686+0000","last_peered":"2023-10-01T09:50:33.888686+0000","last_clean":"2023-10-01T09:50:33.888686+0000","last_became_active":"2023-10-01T09:32:27.450634+0000","last_became_peered":"2023-10-01T09:32:27.450634+0000","last_unstale":"2023-10-01T09:50:33.888686+0000","last_undegraded":"2023-10-01T09:50:33.888686+0000","last_fullsized":"2023-10-01T09:50:33.888686+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:32:57.329859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":698,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.522078+0000","last_change":"2023-10-01T09:50:02.919144+0000","last_active":"2023-10-01T09:50:30.522078+0000","last_peered":"2023-10-01T09:50:30.522078+0000","last_clean":"2023-10-01T09:50:30.522078+0000","last_became_active":"2023-10-01T09:50:02.918856+0000","last_became_peered":"2023-10-01T09:50:02.918856+0000","last_unstale":"2023-10-01T09:50:30.522078+0000","last_undegraded":"2023-10-01T09:50:30.522078+0000","last_fullsized":"2023-10-01T09:50:30.522078+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:34:57.342926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":351,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040348+0000","last_change":"2023-10-01T09:33:11.657900+0000","last_active":"2023-10-01T09:50:31.040348+0000","last_peered":"2023-10-01T09:50:31.040348+0000","last_clean":"2023-10-01T09:50:31.040348+0000","last_became_active":"2023-10-01T09:33:11.657584+0000","last_became_peered":"2023-10-01T09:33:11.657584+0000","last_unstale":"2023-10-01T09:50:31.040348+0000","last_undegraded":"2023-10-01T09:50:31.040348+0000","last_fullsized":"2023-10-01T09:50:31.040348+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":286,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040529+0000","last_change":"2023-10-01T09:50:15.286842+0000","last_active":"2023-10-01T09:50:31.040529+0000","last_peered":"2023-10-01T09:50:31.040529+0000","last_clean":"2023-10-01T09:50:31.040529+0000","last_became_active":"2023-10-01T09:50:15.286494+0000","last_became_peered":"2023-10-01T09:50:15.286494+0000","last_unstale":"2023-10-01T09:50:31.040529+0000","last_undegraded":"2023-10-01T09:50:31.040529+0000","last_fullsized":"2023-10-01T09:50:31.040529+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":351,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.040969+0000","last_change":"2023-10-01T09:33:11.662048+0000","last_active":"2023-10-01T09:50:31.040969+0000","last_peered":"2023-10-01T09:50:31.040969+0000","last_clean":"2023-10-01T09:50:31.040969+0000","last_became_active":"2023-10-01T09:33:11.653163+0000","last_became_peered":"2023-10-01T09:33:11.653163+0000","last_unstale":"2023-10-01T09:50:31.040969+0000","last_undegraded":"2023-10-01T09:50:31.040969+0000","last_fullsized":"2023-10-01T09:50:31.040969+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":367,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.041332+0000","last_change":"2023-10-01T09:50:15.287047+0000","last_active":"2023-10-01T09:50:31.041332+0000","last_peered":"2023-10-01T09:50:31.041332+0000","last_clean":"2023-10-01T09:50:31.041332+0000","last_became_active":"2023-10-01T09:50:15.286668+0000","last_became_peered":"2023-10-01T09:50:15.286668+0000","last_unstale":"2023-10-01T09:50:31.041332+0000","last_undegraded":"2023-10-01T09:50:31.041332+0000","last_fullsized":"2023-10-01T09:50:31.041332+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":77,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.522484+0000","last_change":"2023-10-01T09:50:08.948473+0000","last_active":"2023-10-01T09:50:30.522484+0000","last_peered":"2023-10-01T09:50:30.522484+0000","last_clean":"2023-10-01T09:50:30.522484+0000","last_became_active":"2023-10-01T09:50:08.948199+0000","last_became_peered":"2023-10-01T09:50:08.948199+0000","last_unstale":"2023-10-01T09:50:30.522484+0000","last_undegraded":"2023-10-01T09:50:30.522484+0000","last_fullsized":"2023-10-01T09:50:30.522484+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T09:53:13.704522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":574,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:30.523602+0000","last_change":"2023-10-01T09:50:08.944753+0000","last_active":"2023-10-01T09:50:30.523602+0000","last_peered":"2023-10-01T09:50:30.523602+0000","last_clean":"2023-10-01T09:50:30.523602+0000","last_became_active":"2023-10-01T09:50:08.944441+0000","last_became_peered":"2023-10-01T09:50:08.944441+0000","last_unstale":"2023-10-01T09:50:30.523602+0000","last_undegraded":"2023-10-01T09:50:30.523602+0000","last_fullsized":"2023-10-01T09:50:30.523602+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:02:44.959539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":791,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:32.118829+0000","last_change":"2023-10-01T09:49:37.975951+0000","last_active":"2023-10-01T09:50:32.118829+0000","last_peered":"2023-10-01T09:50:32.118829+0000","last_clean":"2023-10-01T09:50:32.118829+0000","last_became_active":"2023-10-01T09:49:37.884496+0000","last_became_peered":"2023-10-01T09:49:37.884496+0000","last_unstale":"2023-10-01T09:50:32.118829+0000","last_undegraded":"2023-10-01T09:50:32.118829+0000","last_fullsized":"2023-10-01T09:50:32.118829+0000","mapping_epoch":590,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":591,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:02.178369+0000","last_clean_scrub_stamp":"2023-10-01T09:03:02.178369+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:10:46.008390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6094,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":467,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-10-01T09:50:31.039939+0000","last_change":"2023-10-01T09:40:13.194041+0000","last_active":"2023-10-01T09:50:31.039939+0000","last_peered":"2023-10-01T09:50:31.039939+0000","last_clean":"2023-10-01T09:50:31.039939+0000","last_became_active":"2023-10-01T09:40:13.193255+0000","last_became_peered":"2023-10-01T09:40:13.193255+0000","last_unstale":"2023-10-01T09:50:31.039939+0000","last_undegraded":"2023-10-01T09:50:31.039939+0000","last_fullsized":"2023-10-01T09:50:31.039939+0000","mapping_epoch":479,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T09:03:03.180189+0000","last_clean_scrub_stamp":"2023-10-01T09:03:03.180189+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":31074,"num_keys_recovered":110,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48310,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11963,"num_read_kb":25374,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":404,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430807,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905112,"kb_used_data":10192,"kb_used_omap":0,"kb_used_meta":894912,"kb_avail":92840040,"statfs":{"total":95995035648,"available":95068200960,"internally_reserved":0,"allocated":10436608,"data_stored":9444579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":916389888},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:50:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":2.2410000000000001,"15min":2.2309999999999999},"min":{"1min":0.32000000000000001,"5min":0.22500000000000001,"15min":0.216},"max":{"1min":0.81299999999999994,"5min":112.274,"15min":112.274},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.66900000000000004,"5min":2.2090000000000001,"15min":2.3319999999999999},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.21199999999999999},"max":{"1min":3.5249999999999999,"5min":112.44799999999999,"15min":112.44799999999999},"last":0.504}]},{"osd":1,"last update":"Sun Oct 1 09:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.97799999999999998,"15min":3.141},"min":{"1min":0.34300000000000003,"5min":0.24199999999999999,"15min":0.23300000000000001},"max":{"1min":0.79100000000000004,"5min":12.522,"15min":229.161},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.92500000000000004,"15min":3.1269999999999998},"min":{"1min":0.318,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":0.83699999999999997,"5min":10.127000000000001,"15min":230.126},"last":0.75900000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":1.089,"15min":1.3939999999999999},"min":{"1min":0.40999999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.77600000000000002,"5min":12.565,"15min":14.901999999999999},"last":0.63},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.81699999999999995,"15min":1.2689999999999999},"min":{"1min":0.375,"5min":0.36499999999999999,"15min":0.27000000000000002},"max":{"1min":0.79100000000000004,"5min":7.4379999999999997,"15min":14.682},"last":0.78700000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.90700000000000003,"15min":1.736},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.29199999999999998},"max":{"1min":0.93500000000000005,"5min":7.4699999999999998,"15min":76.555000000000007},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.89900000000000002,"15min":1.835},"min":{"1min":0.38300000000000001,"5min":0.315,"15min":0.27600000000000002},"max":{"1min":0.75700000000000001,"5min":9.2599999999999998,"15min":76.534000000000006},"last":0.66100000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:50:06 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.90800000000000003,"15min":0.995},"min":{"1min":0.29299999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.73799999999999999,"5min":11.069000000000001,"15min":11.069000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.80500000000000005,"15min":0.90900000000000003},"min":{"1min":0.31900000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.84199999999999997,"5min":7.5,"15min":8.4949999999999992},"last":0.73399999999999999}]},{"osd":5,"last update":"Sun Oct 1 09:50:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.153,"15min":1.6080000000000001},"min":{"1min":0.432,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.98999999999999999,"5min":13.41,"15min":135.40000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.95999999999999996,"15min":1.6850000000000001},"min":{"1min":0.34499999999999997,"5min":0.27000000000000002,"15min":0.26100000000000001},"max":{"1min":0.88600000000000001,"5min":11.895,"15min":135.339},"last":0.59299999999999997}]},{"osd":6,"last update":"Sun Oct 1 09:50:46 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.89800000000000002,"15min":1.0820000000000001},"min":{"1min":0.46400000000000002,"5min":0.39600000000000002,"15min":0.313},"max":{"1min":0.96899999999999997,"5min":10.417999999999999,"15min":11.853999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.93799999999999994,"15min":1.081},"min":{"1min":0.35599999999999998,"5min":0.32600000000000001,"15min":0.32400000000000001},"max":{"1min":0.85699999999999998,"5min":9.3550000000000004,"15min":14.584},"last":0.71599999999999997}]}]},{"osd":6,"up_from":294,"seq":1262720385318,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":343688,"kb_used_data":10176,"kb_used_omap":6,"kb_used_meta":333497,"kb_avail":93401464,"statfs":{"total":95995035648,"available":95643099136,"internally_reserved":0,"allocated":10420224,"data_stored":9443822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":341500973},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:50:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":2.1930000000000001,"15min":2.4079999999999999},"min":{"1min":0.245,"5min":0.223,"15min":0.219},"max":{"1min":0.58099999999999996,"5min":77.837000000000003,"15min":158.48599999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":2.3740000000000001,"15min":2.6560000000000001},"min":{"1min":0.36199999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.86499999999999999,"5min":77.971000000000004,"15min":158.37700000000001},"last":0.52100000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:50:27 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.89500000000000002,"15min":2.9649999999999999},"min":{"1min":0.25700000000000001,"5min":0.222,"15min":0.222},"max":{"1min":0.75,"5min":8.2409999999999997,"15min":168.947},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.73199999999999998,"15min":2.8900000000000001},"min":{"1min":0.36499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.96199999999999997,"5min":6.8390000000000004,"15min":169.07900000000001},"last":0.39700000000000002}]},{"osd":2,"last update":"Sun Oct 1 09:50:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":1.0049999999999999,"15min":1.272},"min":{"1min":0.40000000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.63800000000000001,"5min":8.2829999999999995,"15min":9.657},"last":0.436},{"interface":"front","average":{"1min":0.502,"5min":0.79400000000000004,"15min":1.204},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.27100000000000002},"max":{"1min":0.88300000000000001,"5min":6.8159999999999998,"15min":21.704999999999998},"last":0.502}]},{"osd":3,"last update":"Sun Oct 1 09:50:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.746,"15min":1.181},"min":{"1min":0.33500000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.81599999999999995,"5min":6.3920000000000003,"15min":9.5679999999999996},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.97299999999999998,"15min":1.2989999999999999},"min":{"1min":0.311,"5min":0.309,"15min":0.30399999999999999},"max":{"1min":0.90600000000000003,"5min":8.3109999999999999,"15min":10.191000000000001},"last":0.63100000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:50:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":1.0740000000000001,"15min":1.103},"min":{"1min":0.31900000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.85999999999999999,"5min":12.326000000000001,"15min":63.134},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":1.2390000000000001,"15min":1.1200000000000001},"min":{"1min":0.41199999999999998,"5min":0.29399999999999998,"15min":0.28899999999999998},"max":{"1min":0.92600000000000005,"5min":13.641,"15min":62.973999999999997},"last":0.58299999999999996}]},{"osd":5,"last update":"Sun Oct 1 09:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":2.238,"15min":1.4470000000000001},"min":{"1min":0.34799999999999998,"5min":0.313,"15min":0.251},"max":{"1min":0.71099999999999997,"5min":136.535,"15min":136.535},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.52300000000000002,"5min":2.1539999999999999,"15min":1.4930000000000001},"min":{"1min":0.311,"5min":0.28699999999999998,"15min":0.28100000000000003},"max":{"1min":0.76600000000000001,"5min":136.47300000000001,"15min":136.47300000000001},"last":0.56899999999999995}]},{"osd":7,"last update":"Sun Oct 1 09:50:27 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":1.256,"15min":1.3480000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.29899999999999999},"max":{"1min":0.81399999999999995,"5min":15.407999999999999,"15min":94.245999999999995},"last":0.623},{"interface":"front","average":{"1min":0.54400000000000004,"5min":1.069,"15min":1.446},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.96299999999999997,"5min":11.326000000000001,"15min":94.715000000000003},"last":0.66000000000000003}]}]},{"osd":1,"up_from":94,"seq":403726926294,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":869360,"kb_used_data":15080,"kb_used_omap":7,"kb_used_meta":854264,"kb_avail":92875792,"statfs":{"total":95995035648,"available":95104811008,"internally_reserved":0,"allocated":15441920,"data_stored":13196764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":874766759},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.316,"5min":2.8199999999999998,"15min":3.6600000000000001},"min":{"1min":0.22600000000000001,"5min":0.19500000000000001,"15min":0.182},"max":{"1min":0.41299999999999998,"5min":114.33499999999999,"15min":154.21299999999999},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.32800000000000001,"5min":2.8090000000000002,"15min":3.222},"min":{"1min":0.214,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":0.50600000000000001,"5min":114.099,"15min":114.099},"last":0.20000000000000001}]},{"osd":2,"last update":"Sun Oct 1 09:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.65900000000000003,"15min":1.3160000000000001},"min":{"1min":0.249,"5min":0.22,"15min":0.22},"max":{"1min":5.6929999999999996,"5min":9.2690000000000001,"15min":22.891999999999999},"last":0.317},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.77300000000000002,"15min":2.0299999999999998},"min":{"1min":0.27800000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":10.534000000000001,"5min":12.018000000000001,"15min":154.197},"last":0.23000000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.65800000000000003,"15min":2.1160000000000001},"min":{"1min":0.28199999999999997,"5min":0.222,"15min":0.219},"max":{"1min":0.55300000000000005,"5min":8.4320000000000004,"15min":95.305000000000007},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.57199999999999995,"15min":2.71},"min":{"1min":0.28599999999999998,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.48499999999999999,"5min":4.8559999999999999,"15min":154.483},"last":0.28999999999999998}]},{"osd":4,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.2170000000000001,"15min":2.1880000000000002},"min":{"1min":0.30399999999999999,"5min":0.245,"15min":0.245},"max":{"1min":0.93400000000000005,"5min":24.539999999999999,"15min":154.101},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.67400000000000004,"15min":1.9039999999999999},"min":{"1min":0.35099999999999998,"5min":0.29999999999999999,"15min":0.27700000000000002},"max":{"1min":1.069,"5min":4.9139999999999997,"15min":154.56999999999999},"last":0.55800000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:50:52 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":1.2949999999999999,"15min":2.2949999999999999},"min":{"1min":0.39700000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.085,"5min":18.158000000000001,"15min":154.22999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.76200000000000001,"5min":1.1990000000000001,"15min":2.1920000000000002},"min":{"1min":0.46700000000000003,"5min":0.32300000000000001,"15min":0.307},"max":{"1min":4.2199999999999998,"5min":24.405000000000001,"15min":154.54400000000001},"last":0.50800000000000001}]},{"osd":6,"last update":"Sun Oct 1 09:50:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":1.3029999999999999,"15min":2.1709999999999998},"min":{"1min":0.35899999999999999,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":1.2230000000000001,"5min":24.321000000000002,"15min":154.12200000000001},"last":0.625},{"interface":"front","average":{"1min":0.56000000000000005,"5min":1.012,"15min":2.153},"min":{"1min":0.33400000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.97799999999999998,"5min":9.2149999999999999,"15min":154.16999999999999},"last":0.54800000000000004}]},{"osd":7,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.1299999999999999,"15min":2.2829999999999999},"min":{"1min":0.39900000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.995,"5min":15.009,"15min":154.20699999999999},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.96899999999999997,"15min":2.214},"min":{"1min":0.40699999999999997,"5min":0.36799999999999999,"15min":0.34300000000000003},"max":{"1min":1.149,"5min":6.6180000000000003,"15min":154.51599999999999},"last":0.61699999999999999}]}]},{"osd":0,"up_from":89,"seq":382252089829,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1003240,"kb_used_data":49760,"kb_used_omap":8,"kb_used_meta":953463,"kb_avail":92741912,"statfs":{"total":95995035648,"available":94967717888,"internally_reserved":0,"allocated":50954240,"data_stored":48724438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":976347074},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 09:50:24 2023","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":1.159,"15min":3.4529999999999998},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.17799999999999999},"max":{"1min":0.46100000000000002,"5min":90.185000000000002,"15min":216.821},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.34200000000000003,"5min":1.403,"15min":4.0369999999999999},"min":{"1min":0.23400000000000001,"5min":0.20599999999999999,"15min":0.19600000000000001},"max":{"1min":0.47199999999999998,"5min":90.102999999999994,"15min":217.02699999999999},"last":0.23899999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:50:57 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.53300000000000003,"15min":1.359},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.52100000000000002,"5min":4.5819999999999999,"15min":19.788},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.372,"5min":0.71099999999999997,"15min":1.427},"min":{"1min":0.28499999999999998,"5min":0.27500000000000002,"15min":0.215},"max":{"1min":0.53500000000000003,"5min":10.901,"15min":25.844999999999999},"last":0.312}]},{"osd":3,"last update":"Sun Oct 1 09:50:24 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.47099999999999997,"15min":2.2480000000000002},"min":{"1min":0.29699999999999999,"5min":0.24099999999999999,"15min":0.22},"max":{"1min":0.54800000000000004,"5min":3.3849999999999998,"15min":222.16499999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.372,"5min":0.67500000000000004,"15min":2.2909999999999999},"min":{"1min":0.27200000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.55900000000000005,"5min":4.0650000000000004,"15min":222.227},"last":0.32200000000000001}]},{"osd":4,"last update":"Sun Oct 1 09:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":2.1720000000000002,"15min":1.9059999999999999},"min":{"1min":0.32000000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.77600000000000002,"5min":138.84100000000001,"15min":138.84100000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.51500000000000001,"5min":1.9319999999999999,"15min":1.9610000000000001},"min":{"1min":0.34999999999999998,"5min":0.29199999999999998,"15min":0.28699999999999998},"max":{"1min":0.71799999999999997,"5min":135.94800000000001,"15min":135.94800000000001},"last":0.436}]},{"osd":5,"last update":"Sun Oct 1 09:50:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":2.2869999999999999,"15min":1.8939999999999999},"min":{"1min":0.35699999999999998,"5min":0.308,"15min":0.308},"max":{"1min":0.80700000000000005,"5min":156.69999999999999,"15min":156.69999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.59999999999999998,"5min":2.0470000000000002,"15min":1.7689999999999999},"min":{"1min":0.46100000000000002,"5min":0.30599999999999999,"15min":0.27700000000000002},"max":{"1min":0.77300000000000002,"5min":156.773,"15min":156.773},"last":0.54100000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:50:41 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":2.0840000000000001,"15min":1.7410000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.73499999999999999,"5min":135.321,"15min":135.321},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":2.181,"15min":1.8200000000000001},"min":{"1min":0.36799999999999999,"5min":0.35899999999999999,"15min":0.32400000000000001},"max":{"1min":0.80200000000000005,"5min":135.29900000000001,"15min":135.29900000000001},"last":0.61099999999999999}]},{"osd":7,"last update":"Sun Oct 1 09:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.89600000000000002,"15min":1.6619999999999999},"min":{"1min":0.38800000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.77800000000000002,"5min":8.6829999999999998,"15min":48.895000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.1899999999999999,"15min":1.804},"min":{"1min":0.35699999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.79000000000000004,"5min":10.82,"15min":48.274000000000001},"last":0.47299999999999998}]}]},{"osd":2,"up_from":99,"seq":425201762768,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":757932,"kb_used_data":6884,"kb_used_omap":11,"kb_used_meta":751028,"kb_avail":92987220,"statfs":{"total":95995035648,"available":95218913280,"internally_reserved":0,"allocated":7049216,"data_stored":4809623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11401,"internal_metadata":769053559},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:50:02 2023","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":1.069,"15min":3.1659999999999999},"min":{"1min":0.253,"5min":0.189,"15min":0.17699999999999999},"max":{"1min":0.54600000000000004,"5min":59.658999999999999,"15min":137.72900000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.35699999999999998,"5min":1.222,"15min":3.3620000000000001},"min":{"1min":0.24199999999999999,"5min":0.21099999999999999,"15min":0.19600000000000001},"max":{"1min":0.47999999999999998,"5min":59.930999999999997,"15min":138.30600000000001},"last":0.318}]},{"osd":1,"last update":"Sun Oct 1 09:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":2.391,"15min":2.8279999999999998},"min":{"1min":0.27200000000000002,"5min":0.20999999999999999,"15min":0.19900000000000001},"max":{"1min":0.48399999999999999,"5min":134.94499999999999,"15min":134.94499999999999},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.35399999999999998,"5min":2.625,"15min":3.4159999999999999},"min":{"1min":0.28599999999999998,"5min":0.21299999999999999,"15min":0.19900000000000001},"max":{"1min":0.53200000000000003,"5min":134.965,"15min":134.965},"last":0.34999999999999998}]},{"osd":3,"last update":"Sun Oct 1 09:50:31 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.56699999999999995,"15min":1.617},"min":{"1min":0.28199999999999997,"5min":0.23799999999999999,"15min":0.20899999999999999},"max":{"1min":0.49399999999999999,"5min":4.9009999999999998,"15min":50.289000000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.49299999999999999,"15min":1.3100000000000001},"min":{"1min":0.312,"5min":0.23699999999999999,"15min":0.215},"max":{"1min":0.53500000000000003,"5min":3.016,"15min":54.115000000000002},"last":0.40200000000000002}]},{"osd":4,"last update":"Sun Oct 1 09:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":1.333,"15min":1.3160000000000001},"min":{"1min":0.36399999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.85599999999999998,"5min":60.994,"15min":60.994},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.393,"15min":1.4470000000000001},"min":{"1min":0.42699999999999999,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":0.81299999999999994,"5min":60.970999999999997,"15min":60.970999999999997},"last":0.59699999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":2.3679999999999999,"15min":1.716},"min":{"1min":0.36499999999999999,"5min":0.34499999999999997,"15min":0.28799999999999998},"max":{"1min":0.78400000000000003,"5min":111.70999999999999,"15min":111.70999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.56999999999999995,"5min":2.524,"15min":1.919},"min":{"1min":0.44,"5min":0.33300000000000002,"15min":0.32000000000000001},"max":{"1min":0.753,"5min":111.735,"15min":111.735},"last":0.67300000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:50:31 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.96999999999999997,"15min":1.389},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.30499999999999999},"max":{"1min":0.69499999999999995,"5min":8.2959999999999994,"15min":11.526999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.89900000000000002,"15min":1.1359999999999999},"min":{"1min":0.45000000000000001,"5min":0.378,"15min":0.32800000000000001},"max":{"1min":0.78700000000000003,"5min":8.5410000000000004,"15min":12.384},"last":0.55700000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.83499999999999996,"15min":1.7230000000000001},"min":{"1min":0.48399999999999999,"5min":0.39700000000000002,"15min":0.34899999999999998},"max":{"1min":0.88900000000000001,"5min":4.3959999999999999,"15min":139.03299999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.93799999999999994,"15min":1.919},"min":{"1min":0.41699999999999998,"5min":0.36599999999999999,"15min":0.35999999999999999},"max":{"1min":0.94999999999999996,"5min":6.468,"15min":139.065},"last":0.63300000000000001}]}]},{"osd":3,"up_from":104,"seq":446676599242,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":875224,"kb_used_data":6992,"kb_used_omap":3,"kb_used_meta":868220,"kb_avail":92869928,"statfs":{"total":95995035648,"available":95098806272,"internally_reserved":0,"allocated":7159808,"data_stored":4819713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":889058023},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":2.625,"15min":3.105},"min":{"1min":0.20999999999999999,"5min":0.19800000000000001,"15min":0.17899999999999999},"max":{"1min":1.0589999999999999,"5min":93.245999999999995,"15min":118.748},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":2.7360000000000002,"15min":3.3290000000000002},"min":{"1min":0.23200000000000001,"5min":0.217,"15min":0.19800000000000001},"max":{"1min":0.97399999999999998,"5min":91.430000000000007,"15min":118.86199999999999},"last":0.27000000000000002}]},{"osd":1,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.56499999999999995,"15min":3.2949999999999999},"min":{"1min":0.28000000000000003,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":1.105,"5min":5.9009999999999998,"15min":153.77000000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.48499999999999999,"15min":3.2149999999999999},"min":{"1min":0.25,"5min":0.20899999999999999,"15min":0.20499999999999999},"max":{"1min":1.0980000000000001,"5min":4.008,"15min":151.02199999999999},"last":0.374}]},{"osd":2,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.51300000000000001,"15min":1.401},"min":{"1min":0.254,"5min":0.22900000000000001,"15min":0.22800000000000001},"max":{"1min":1.5780000000000001,"5min":3.5059999999999998,"15min":14.254},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.621,"15min":1.0580000000000001},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.20100000000000001},"max":{"1min":1.639,"5min":3.9809999999999999,"15min":15.523},"last":0.40500000000000003}]},{"osd":4,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.80400000000000005,"15min":1.351},"min":{"1min":0.36599999999999999,"5min":0.312,"15min":0.27400000000000002},"max":{"1min":0.71799999999999997,"5min":6.2370000000000001,"15min":77.793000000000006},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.96599999999999997,"15min":1.611},"min":{"1min":0.40600000000000003,"5min":0.32300000000000001,"15min":0.27900000000000003},"max":{"1min":0.86299999999999999,"5min":9.7370000000000001,"15min":76.646000000000001},"last":0.47499999999999998}]},{"osd":5,"last update":"Sun Oct 1 09:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":1.409,"15min":1.466},"min":{"1min":0.38700000000000001,"5min":0.30399999999999999,"15min":0.27500000000000002},"max":{"1min":0.70699999999999996,"5min":46.256999999999998,"15min":46.256999999999998},"last":0.497},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.3169999999999999,"15min":1.5960000000000001},"min":{"1min":0.41499999999999998,"5min":0.307,"15min":0.27700000000000002},"max":{"1min":0.81599999999999995,"5min":46.284999999999997,"15min":46.284999999999997},"last":0.57999999999999996}]},{"osd":6,"last update":"Sun Oct 1 09:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.82699999999999996,"15min":1.667},"min":{"1min":0.41999999999999998,"5min":0.379,"15min":0.315},"max":{"1min":0.748,"5min":8.3230000000000004,"15min":73.887},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.85099999999999998,"15min":1.7470000000000001},"min":{"1min":0.439,"5min":0.379,"15min":0.33600000000000002},"max":{"1min":0.70599999999999996,"5min":4.2169999999999996,"15min":74.016999999999996},"last":0.68300000000000005}]},{"osd":7,"last update":"Sun Oct 1 09:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.161,"15min":1.706},"min":{"1min":0.45300000000000001,"5min":0.40400000000000003,"15min":0.30199999999999999},"max":{"1min":0.85099999999999998,"5min":10.574,"15min":13.085000000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.63400000000000001,"5min":1.2110000000000001,"15min":1.4039999999999999},"min":{"1min":0.41699999999999998,"5min":0.36699999999999999,"15min":0.33300000000000002},"max":{"1min":0.81599999999999995,"5min":10.609,"15min":11.109999999999999},"last":0.55500000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019492,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":921296,"kb_used_data":27016,"kb_used_omap":0,"kb_used_meta":894272,"kb_avail":92823856,"statfs":{"total":95995035648,"available":95051628544,"internally_reserved":0,"allocated":27664384,"data_stored":26679933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915734528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:50:08 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":2.4990000000000001,"15min":2.585},"min":{"1min":0.224,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.65100000000000002,"5min":95.548000000000002,"15min":95.548000000000002},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":2.532,"15min":2.7690000000000001},"min":{"1min":0.20699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.65500000000000003,"5min":95.253,"15min":95.253},"last":0.438}]},{"osd":1,"last update":"Sun Oct 1 09:50:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.76200000000000001,"15min":4.0810000000000004},"min":{"1min":0.35399999999999998,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":0.73099999999999998,"5min":7.5979999999999999,"15min":165.666},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.78300000000000003,"15min":4.0270000000000001},"min":{"1min":0.30599999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.71499999999999997,"5min":13.023999999999999,"15min":165.70500000000001},"last":0.48999999999999999}]},{"osd":2,"last update":"Sun Oct 1 09:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.83099999999999996,"15min":1.181},"min":{"1min":0.34799999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.70599999999999996,"5min":7.6390000000000002,"15min":10.161},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.83099999999999996,"15min":1.3009999999999999},"min":{"1min":0.372,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.222,"5min":7.6619999999999999,"15min":10.625},"last":0.63300000000000001}]},{"osd":3,"last update":"Sun Oct 1 09:50:08 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.82499999999999996,"15min":1.794},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":0.63200000000000001,"5min":11.667,"15min":140.881},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.86599999999999999,"15min":1.885},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.5029999999999999,"5min":13.176,"15min":140.911},"last":0.55300000000000005}]},{"osd":5,"last update":"Sun Oct 1 09:50:53 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":1.746,"15min":1.667},"min":{"1min":0.32300000000000001,"5min":0.27200000000000002,"15min":0.26600000000000001},"max":{"1min":0.77700000000000002,"5min":37.548000000000002,"15min":110.003},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.631,"15min":1.627},"min":{"1min":0.33900000000000002,"5min":0.313,"15min":0.28899999999999998},"max":{"1min":0.66300000000000003,"5min":37.567,"15min":108.56399999999999},"last":0.66600000000000004}]},{"osd":6,"last update":"Sun Oct 1 09:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.82599999999999996,"15min":0.84199999999999997},"min":{"1min":0.32000000000000001,"5min":0.318,"15min":0.27700000000000002},"max":{"1min":0.83799999999999997,"5min":11.635999999999999,"15min":11.635999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.89600000000000002,"15min":0.98399999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.314},"max":{"1min":0.77900000000000003,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":0.749}]},{"osd":7,"last update":"Sun Oct 1 09:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.0600000000000001,"15min":1.0229999999999999},"min":{"1min":0.35799999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.88800000000000001,"5min":11.694000000000001,"15min":38.034999999999997},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.123,"15min":1.2490000000000001},"min":{"1min":0.42599999999999999,"5min":0.34599999999999997,"15min":0.34200000000000003},"max":{"1min":0.82699999999999996,"5min":12.228,"15min":37.723999999999997},"last":0.77600000000000002}]}]},{"osd":5,"up_from":379,"seq":1627792605399,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":378240,"kb_used_data":9592,"kb_used_omap":13,"kb_used_meta":368626,"kb_avail":93366912,"statfs":{"total":95995035648,"available":95607717888,"internally_reserved":0,"allocated":9822208,"data_stored":8849589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":377473522},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":2.02,"15min":2.0289999999999999},"min":{"1min":0.28899999999999998,"5min":0.222,"15min":0.21099999999999999},"max":{"1min":0.72199999999999998,"5min":99.609999999999999,"15min":99.609999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":2.8149999999999999,"15min":2.258},"min":{"1min":0.33400000000000002,"5min":0.215,"15min":0.20399999999999999},"max":{"1min":1.3660000000000001,"5min":104.01600000000001,"15min":104.01600000000001},"last":0.67600000000000005}]},{"osd":1,"last update":"Sun Oct 1 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.81799999999999995,"15min":4.6699999999999999},"min":{"1min":0.30399999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.6789999999999998,"5min":7.6820000000000004,"15min":295.71300000000002},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.81699999999999995,"15min":4.7370000000000001},"min":{"1min":0.34300000000000003,"5min":0.26600000000000001,"15min":0.25700000000000001},"max":{"1min":0.70299999999999996,"5min":14.859999999999999,"15min":299.59399999999999},"last":0.438}]},{"osd":2,"last update":"Sun Oct 1 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.81799999999999995,"15min":1.3100000000000001},"min":{"1min":0.38400000000000001,"5min":0.28999999999999998,"15min":0.27600000000000002},"max":{"1min":0.755,"5min":12.946999999999999,"15min":14.199999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.86499999999999999,"15min":1.264},"min":{"1min":0.35299999999999998,"5min":0.29599999999999999,"15min":0.27100000000000002},"max":{"1min":1.109,"5min":6.5609999999999999,"15min":14.430999999999999},"last":0.53000000000000003}]},{"osd":3,"last update":"Sun Oct 1 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.98599999999999999,"15min":1.3460000000000001},"min":{"1min":0.34899999999999998,"5min":0.30599999999999999,"15min":0.28100000000000003},"max":{"1min":0.82499999999999996,"5min":11.983000000000001,"15min":14.004},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.81299999999999994,"15min":1.4159999999999999},"min":{"1min":0.42299999999999999,"5min":0.377,"15min":0.32500000000000001},"max":{"1min":0.79400000000000004,"5min":12.201000000000001,"15min":13.991},"last":0.71899999999999997}]},{"osd":4,"last update":"Sun Oct 1 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.92400000000000004,"15min":1.363},"min":{"1min":0.29499999999999998,"5min":0.27900000000000003,"15min":0.24399999999999999},"max":{"1min":0.72499999999999998,"5min":12.266,"15min":122.971},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.874,"15min":1.3560000000000001},"min":{"1min":0.30499999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.165,"5min":16.277999999999999,"15min":122.89100000000001},"last":0.59899999999999998}]},{"osd":6,"last update":"Sun Oct 1 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.90900000000000003,"15min":0.95699999999999996},"min":{"1min":0.31900000000000001,"5min":0.29399999999999998,"15min":0.28299999999999997},"max":{"1min":0.80000000000000004,"5min":14.396000000000001,"15min":14.396000000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":1.0149999999999999,"15min":1.0669999999999999},"min":{"1min":0.32800000000000001,"5min":0.31,"15min":0.30499999999999999},"max":{"1min":1.087,"5min":12.157,"15min":14.193},"last":0.76400000000000001}]},{"osd":7,"last update":"Sun Oct 1 09:50:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.3280000000000001,"15min":1.159},"min":{"1min":0.314,"5min":0.311,"15min":0.29699999999999999},"max":{"1min":1.1459999999999999,"5min":23.02,"15min":23.02},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":1.04,"15min":1.1020000000000001},"min":{"1min":0.435,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":0.77200000000000002,"5min":12.121,"15min":12.500999999999999},"last":0.69399999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2110,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8254,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7769,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7973,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3353,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13838,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7123,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T09:51:10.784 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-01T09:51:10.784 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_fc1e90a395a6a5631142eb70fd9bf3d22740a9da/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-01T09:51:10.856 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=58e49ba47a38463e855bd0c02577a9ee Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_fc1e90a395a6a5631142eb70fd9bf3d22740a9da/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-01T09:51:10.859 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-01T09:51:10.870 INFO:tasks.cephadm:Teardown begin 2023-10-01T09:51:10.870 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-01T09:51:10.876 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:10 smithi190 ceph-mon[160435]: from='mgr.26000 172.21.15.138:0/2676223403' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T09:51:10.916 DEBUG:teuthology.orchestra.run.smithi190:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-01T09:51:10.991 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-01T09:51:10.992 DEBUG:teuthology.orchestra.run.smithi138:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-01T09:51:11.009 DEBUG:teuthology.orchestra.run.smithi190:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-01T09:51:11.027 INFO:tasks.cephadm:Stopping all daemons... 2023-10-01T09:51:11.027 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-01T09:51:11.028 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.a 2023-10-01T09:51:11.332 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:11 smithi138 systemd[1]: Stopping Ceph mon.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:51:11.332 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:11 smithi138 conmon[185862]: 2023-10-01T09:51:11.319+0000 7f70dbcf7700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T09:51:11.332 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:11 smithi138 conmon[185862]: 2023-10-01T09:51:11.319+0000 7f70dbcf7700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-10-01T09:51:11.332 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:11 smithi138 conmon[178630]: [01/Oct/2023:09:51:11] ENGINE Bus STOPPING 2023-10-01T09:51:11.655 INFO:journalctl@ceph.mon.a.smithi138.stdout:Oct 01 09:51:11 smithi138 bash[755713]: 4f0e28d177103453bbf53e18d3a692c64f6a49d2d70fcb622e1b4a995b85be88 2023-10-01T09:51:11.655 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:11 smithi138 conmon[178630]: [01/Oct/2023:09:51:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T09:51:11.655 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:11 smithi138 conmon[178630]: [01/Oct/2023:09:51:11] ENGINE Bus STOPPED 2023-10-01T09:51:11.655 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:11 smithi138 conmon[178630]: [01/Oct/2023:09:51:11] ENGINE Bus STARTING 2023-10-01T09:51:11.656 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:11 smithi138 conmon[178630]: [01/Oct/2023:09:51:11] ENGINE Serving on http://:::9283 2023-10-01T09:51:11.656 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:11 smithi138 conmon[178630]: [01/Oct/2023:09:51:11] ENGINE Bus STARTED 2023-10-01T09:51:12.147 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-01T09:51:12.194 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:51:12.195 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-01T09:51:12.195 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-10-01T09:51:12.195 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.c 2023-10-01T09:51:12.495 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:12 smithi138 systemd[1]: Stopping Ceph mon.c for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:51:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:12 smithi138 conmon[192489]: 2023-10-01T09:51:12.495+0000 7fa9055a3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T09:51:12.820 INFO:journalctl@ceph.mon.c.smithi138.stdout:Oct 01 09:51:12 smithi138 conmon[192489]: 2023-10-01T09:51:12.495+0000 7fa9055a3700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-10-01T09:51:12.820 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:12 smithi138 conmon[178630]: [01/Oct/2023:09:51:12] ENGINE Bus STOPPING 2023-10-01T09:51:12.820 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:12 smithi138 conmon[178630]: [01/Oct/2023:09:51:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T09:51:12.821 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:12 smithi138 conmon[178630]: [01/Oct/2023:09:51:12] ENGINE Bus STOPPED 2023-10-01T09:51:12.821 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:12 smithi138 conmon[178630]: [01/Oct/2023:09:51:12] ENGINE Bus STARTING 2023-10-01T09:51:12.821 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:12 smithi138 conmon[178630]: [01/Oct/2023:09:51:12] ENGINE Serving on http://:::9283 2023-10-01T09:51:12.821 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:12 smithi138 conmon[178630]: [01/Oct/2023:09:51:12] ENGINE Bus STARTED 2023-10-01T09:51:13.341 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-01T09:51:13.378 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:51:13.378 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-10-01T09:51:13.378 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-10-01T09:51:13.379 DEBUG:teuthology.orchestra.run.smithi190:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.b 2023-10-01T09:51:13.659 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:13 smithi190 systemd[1]: Stopping Ceph mon.b for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:51:14.053 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:13 smithi190 conmon[160411]: 2023-10-01T09:51:13.657+0000 7fede2ac3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T09:51:14.053 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:13 smithi190 conmon[160411]: 2023-10-01T09:51:13.657+0000 7fede2ac3700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-10-01T09:51:14.306 INFO:journalctl@ceph.mon.b.smithi190.stdout:Oct 01 09:51:14 smithi190 bash[416860]: c0ac51d9b5e0a27314b8d74e1f40200b80d08c93cf95152b2610076c03645540 2023-10-01T09:51:14.595 DEBUG:teuthology.orchestra.run.smithi190:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-01T09:51:14.636 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:51:14.636 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-10-01T09:51:14.636 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-10-01T09:51:14.637 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.y 2023-10-01T09:51:14.941 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:14 smithi138 systemd[1]: Stopping Ceph mgr.y for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:51:14.941 INFO:journalctl@ceph.mgr.y.smithi138.stdout:Oct 01 09:51:14 smithi138 conmon[178630]: 2023-10-01T09:51:14.911+0000 7fa0401ef700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T09:51:15.730 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.y.service' 2023-10-01T09:51:15.769 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:51:15.769 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-10-01T09:51:15.769 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-01T09:51:15.769 DEBUG:teuthology.orchestra.run.smithi190:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.x 2023-10-01T09:51:16.057 INFO:journalctl@ceph.mgr.x.smithi190.stdout:Oct 01 09:51:15 smithi190 systemd[1]: Stopping Ceph mgr.x for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:51:16.810 DEBUG:teuthology.orchestra.run.smithi190:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-01T09:51:16.846 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:51:16.847 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-01T09:51:16.847 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-01T09:51:16.847 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.0 2023-10-01T09:51:17.137 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:51:16 smithi138 systemd[1]: Stopping Ceph osd.0 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:51:17.569 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:51:17 smithi138 conmon[202683]: 2023-10-01T09:51:17.136+0000 7fdd27047700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T09:51:17.570 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:51:17 smithi138 conmon[202683]: 2023-10-01T09:51:17.136+0000 7fdd27047700 -1 osd.0 616 *** Got signal Terminated *** 2023-10-01T09:51:17.570 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:51:17 smithi138 conmon[202683]: 2023-10-01T09:51:17.136+0000 7fdd27047700 -1 osd.0 616 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:51:23.320 INFO:journalctl@ceph.osd.0.smithi138.stdout:Oct 01 09:51:22 smithi138 bash[756630]: cb94dc6c4a672007131dbce497dc9249e0fdbc74b6e180b986a2d1473de1b895 2023-10-01T09:51:25.511 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-01T09:51:25.552 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:51:25.552 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-01T09:51:25.552 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-01T09:51:25.552 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.1 2023-10-01T09:51:25.820 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:51:25 smithi138 systemd[1]: Stopping Ceph osd.1 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:51:26.320 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:51:25 smithi138 conmon[208402]: 2023-10-01T09:51:25.872+0000 7fb2ca129700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T09:51:26.320 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:51:25 smithi138 conmon[208402]: 2023-10-01T09:51:25.872+0000 7fb2ca129700 -1 osd.1 616 *** Got signal Terminated *** 2023-10-01T09:51:26.320 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:51:25 smithi138 conmon[208402]: 2023-10-01T09:51:25.872+0000 7fb2ca129700 -1 osd.1 616 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:51:30.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:51:30 smithi190 conmon[151158]: ts=2023-10-01T09:51:30.199Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.138:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.138:8765: connect: connection refused" 2023-10-01T09:51:30.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:51:30 smithi190 conmon[151158]: ts=2023-10-01T09:51:30.199Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.138:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.138:8765: connect: connection refused" 2023-10-01T09:51:30.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:51:30 smithi190 conmon[151158]: ts=2023-10-01T09:51:30.202Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.138:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.138:8765: connect: connection refused" 2023-10-01T09:51:30.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:51:30 smithi190 conmon[151158]: ts=2023-10-01T09:51:30.202Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.138:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.138:8765: connect: connection refused" 2023-10-01T09:51:32.070 INFO:journalctl@ceph.osd.1.smithi138.stdout:Oct 01 09:51:31 smithi138 bash[757213]: 6abadff62084753562edc3f5a2adfc3846033b7df91c7009afa0bbee644709e3 2023-10-01T09:51:33.470 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-01T09:51:33.516 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:51:33.517 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-01T09:51:33.517 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-01T09:51:33.517 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.2 2023-10-01T09:51:33.820 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:51:33 smithi138 systemd[1]: Stopping Ceph osd.2 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:51:34.320 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:51:33 smithi138 conmon[213382]: 2023-10-01T09:51:33.841+0000 7f5590543700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T09:51:34.320 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:51:33 smithi138 conmon[213382]: 2023-10-01T09:51:33.841+0000 7f5590543700 -1 osd.2 616 *** Got signal Terminated *** 2023-10-01T09:51:34.320 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:51:33 smithi138 conmon[213382]: 2023-10-01T09:51:33.841+0000 7f5590543700 -1 osd.2 616 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:51:40.070 INFO:journalctl@ceph.osd.2.smithi138.stdout:Oct 01 09:51:39 smithi138 bash[757793]: ef89ad6ba2b692978388fffbcd2eb9c1a9e39efbeb3d77ae54562be88112b3b0 2023-10-01T09:51:41.340 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-01T09:51:41.381 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:51:41.381 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-01T09:51:41.382 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-01T09:51:41.382 DEBUG:teuthology.orchestra.run.smithi138:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.3 2023-10-01T09:51:41.688 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:51:41 smithi138 systemd[1]: Stopping Ceph osd.3 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:51:42.070 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:51:41 smithi138 conmon[218387]: 2023-10-01T09:51:41.686+0000 7ff64b4ae700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-10-01T09:51:42.070 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:51:41 smithi138 conmon[218387]: 2023-10-01T09:51:41.686+0000 7ff64b4ae700 -1 osd.3 616 *** Got signal Terminated *** 2023-10-01T09:51:42.070 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:51:41 smithi138 conmon[218387]: 2023-10-01T09:51:41.686+0000 7ff64b4ae700 -1 osd.3 616 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:51:44.057 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:43 smithi190 conmon[106808]: debug 2023-10-01T09:51:43.672+0000 7fee39977700 -1 osd.4 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.597069+0000 front 2023-10-01T09:51:20.597147+0000 (oldest deadline 2023-10-01T09:51:42.897021+0000) 2023-10-01T09:51:44.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:43 smithi190 conmon[247554]: debug 2023-10-01T09:51:43.906+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:44.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:44 smithi190 conmon[124440]: debug 2023-10-01T09:51:44.233+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:45.056 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:44 smithi190 conmon[247554]: debug 2023-10-01T09:51:44.936+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:45.057 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:44 smithi190 conmon[106808]: debug 2023-10-01T09:51:44.719+0000 7fee39977700 -1 osd.4 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.597069+0000 front 2023-10-01T09:51:20.597147+0000 (oldest deadline 2023-10-01T09:51:42.897021+0000) 2023-10-01T09:51:45.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:44 smithi190 conmon[291240]: debug 2023-10-01T09:51:44.882+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:45.556 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:45 smithi190 conmon[124440]: debug 2023-10-01T09:51:45.246+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:45.925 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:45 smithi190 conmon[106808]: debug 2023-10-01T09:51:45.671+0000 7fee39977700 -1 osd.4 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.597069+0000 front 2023-10-01T09:51:20.597147+0000 (oldest deadline 2023-10-01T09:51:42.897021+0000) 2023-10-01T09:51:45.925 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:45 smithi190 conmon[247554]: debug 2023-10-01T09:51:45.899+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:46.296 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:45 smithi190 conmon[291240]: debug 2023-10-01T09:51:45.924+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:46.556 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:46 smithi190 conmon[124440]: debug 2023-10-01T09:51:46.296+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:46.570 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:51:46 smithi138 conmon[218387]: 2023-10-01T09:51:46.151+0000 7ff64529f700 -1 osd.3 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.833949+0000 front 2023-10-01T09:51:21.833967+0000 (oldest deadline 2023-10-01T09:51:45.334069+0000) 2023-10-01T09:51:47.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:46 smithi190 conmon[247554]: debug 2023-10-01T09:51:46.893+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:47.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:46 smithi190 conmon[291240]: debug 2023-10-01T09:51:46.917+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:47.057 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:46 smithi190 conmon[106808]: debug 2023-10-01T09:51:46.682+0000 7fee39977700 -1 osd.4 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.597069+0000 front 2023-10-01T09:51:20.597147+0000 (oldest deadline 2023-10-01T09:51:42.897021+0000) 2023-10-01T09:51:47.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:47 smithi190 conmon[124440]: debug 2023-10-01T09:51:47.256+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:48.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:47 smithi190 conmon[247554]: debug 2023-10-01T09:51:47.871+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:48.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:47 smithi190 conmon[291240]: debug 2023-10-01T09:51:47.880+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:48.057 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:47 smithi190 conmon[106808]: debug 2023-10-01T09:51:47.678+0000 7fee39977700 -1 osd.4 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.597069+0000 front 2023-10-01T09:51:20.597147+0000 (oldest deadline 2023-10-01T09:51:42.897021+0000) 2023-10-01T09:51:48.069 INFO:journalctl@ceph.osd.3.smithi138.stdout:Oct 01 09:51:47 smithi138 bash[758384]: 4b9563996612ff07dac8428c2fe7524decf3879521dfaeb75d98446e703f8ba9 2023-10-01T09:51:48.556 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:48 smithi190 conmon[124440]: debug 2023-10-01T09:51:48.209+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:49.056 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:48 smithi190 conmon[247554]: debug 2023-10-01T09:51:48.921+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:49.057 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:48 smithi190 conmon[106808]: debug 2023-10-01T09:51:48.727+0000 7fee39977700 -1 osd.4 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.597069+0000 front 2023-10-01T09:51:20.597147+0000 (oldest deadline 2023-10-01T09:51:42.897021+0000) 2023-10-01T09:51:49.058 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:48 smithi190 conmon[291240]: debug 2023-10-01T09:51:48.904+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:49.197 DEBUG:teuthology.orchestra.run.smithi138:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-01T09:51:49.239 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:51:49.239 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-01T09:51:49.239 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-01T09:51:49.240 DEBUG:teuthology.orchestra.run.smithi190:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.4 2023-10-01T09:51:49.530 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:49 smithi190 systemd[1]: Stopping Ceph osd.4 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:51:49.531 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:49 smithi190 conmon[124440]: debug 2023-10-01T09:51:49.185+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:49.806 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:49 smithi190 conmon[106808]: debug 2023-10-01T09:51:49.529+0000 7fee3c1f9700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T09:51:49.807 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:49 smithi190 conmon[106808]: debug 2023-10-01T09:51:49.529+0000 7fee3c1f9700 -1 osd.4 616 *** Got signal Terminated *** 2023-10-01T09:51:49.807 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:49 smithi190 conmon[106808]: debug 2023-10-01T09:51:49.529+0000 7fee3c1f9700 -1 osd.4 616 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:51:49.807 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:49 smithi190 conmon[106808]: debug 2023-10-01T09:51:49.691+0000 7fee39977700 -1 osd.4 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.597069+0000 front 2023-10-01T09:51:20.597147+0000 (oldest deadline 2023-10-01T09:51:42.897021+0000) 2023-10-01T09:51:50.205 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:49 smithi190 conmon[247554]: debug 2023-10-01T09:51:49.884+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:50.206 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:49 smithi190 conmon[291240]: debug 2023-10-01T09:51:49.859+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:50.556 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:50 smithi190 conmon[124440]: debug 2023-10-01T09:51:50.205+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:51.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:50 smithi190 conmon[247554]: debug 2023-10-01T09:51:50.842+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:51.057 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:50 smithi190 conmon[106808]: debug 2023-10-01T09:51:50.690+0000 7fee39977700 -1 osd.4 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.597069+0000 front 2023-10-01T09:51:20.597147+0000 (oldest deadline 2023-10-01T09:51:42.897021+0000) 2023-10-01T09:51:51.058 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:50 smithi190 conmon[291240]: debug 2023-10-01T09:51:50.896+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:51.556 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:51 smithi190 conmon[124440]: debug 2023-10-01T09:51:51.218+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:52.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:51 smithi190 conmon[247554]: debug 2023-10-01T09:51:51.833+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:52.057 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:51 smithi190 conmon[106808]: debug 2023-10-01T09:51:51.730+0000 7fee39977700 -1 osd.4 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.597069+0000 front 2023-10-01T09:51:20.597147+0000 (oldest deadline 2023-10-01T09:51:42.897021+0000) 2023-10-01T09:51:52.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:51 smithi190 conmon[291240]: debug 2023-10-01T09:51:51.861+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:52.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:52 smithi190 conmon[124440]: debug 2023-10-01T09:51:52.203+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:53.057 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:52 smithi190 conmon[106808]: debug 2023-10-01T09:51:52.695+0000 7fee39977700 -1 osd.4 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.597069+0000 front 2023-10-01T09:51:20.597147+0000 (oldest deadline 2023-10-01T09:51:42.897021+0000) 2023-10-01T09:51:53.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:52 smithi190 conmon[291240]: debug 2023-10-01T09:51:52.820+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:53.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:52 smithi190 conmon[247554]: debug 2023-10-01T09:51:52.839+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:53.058 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:52 smithi190 conmon[247554]: debug 2023-10-01T09:51:52.839+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:51:53.556 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:53 smithi190 conmon[124440]: debug 2023-10-01T09:51:53.226+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:54.057 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:53 smithi190 conmon[106808]: debug 2023-10-01T09:51:53.724+0000 7fee39977700 -1 osd.4 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.597069+0000 front 2023-10-01T09:51:20.597147+0000 (oldest deadline 2023-10-01T09:51:42.897021+0000) 2023-10-01T09:51:54.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:53 smithi190 conmon[247554]: debug 2023-10-01T09:51:53.831+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:54.058 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:53 smithi190 conmon[247554]: debug 2023-10-01T09:51:53.831+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:51:54.058 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:53 smithi190 conmon[291240]: debug 2023-10-01T09:51:53.859+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:54.058 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:53 smithi190 conmon[291240]: debug 2023-10-01T09:51:53.859+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.328933+0000 front 2023-10-01T09:51:30.328817+0000 (oldest deadline 2023-10-01T09:51:53.129435+0000) 2023-10-01T09:51:54.556 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:54 smithi190 conmon[124440]: debug 2023-10-01T09:51:54.224+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:55.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:54 smithi190 conmon[247554]: debug 2023-10-01T09:51:54.822+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:55.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:54 smithi190 conmon[247554]: debug 2023-10-01T09:51:54.822+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:51:55.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:54 smithi190 conmon[291240]: debug 2023-10-01T09:51:54.877+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:55.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:54 smithi190 conmon[291240]: debug 2023-10-01T09:51:54.877+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.328933+0000 front 2023-10-01T09:51:30.328817+0000 (oldest deadline 2023-10-01T09:51:53.129435+0000) 2023-10-01T09:51:55.557 INFO:journalctl@ceph.osd.4.smithi190.stdout:Oct 01 09:51:55 smithi190 bash[417471]: cda54a78761408ca41e5cf3ce3db59aba0d23137a1800d78ea660f40aef42682 2023-10-01T09:51:55.558 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:55 smithi190 conmon[124440]: debug 2023-10-01T09:51:55.204+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:56.056 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:55 smithi190 conmon[247554]: debug 2023-10-01T09:51:55.779+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:56.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:55 smithi190 conmon[247554]: debug 2023-10-01T09:51:55.779+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:51:56.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:55 smithi190 conmon[291240]: debug 2023-10-01T09:51:55.892+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:56.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:55 smithi190 conmon[291240]: debug 2023-10-01T09:51:55.892+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.328933+0000 front 2023-10-01T09:51:30.328817+0000 (oldest deadline 2023-10-01T09:51:53.129435+0000) 2023-10-01T09:51:56.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:56 smithi190 conmon[124440]: debug 2023-10-01T09:51:56.243+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:56.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:56 smithi190 conmon[124440]: debug 2023-10-01T09:51:56.243+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:51:56.995 DEBUG:teuthology.orchestra.run.smithi190:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-01T09:51:57.038 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:51:57.038 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-01T09:51:57.038 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-01T09:51:57.038 DEBUG:teuthology.orchestra.run.smithi190:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.5 2023-10-01T09:51:57.102 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:56 smithi190 conmon[247554]: debug 2023-10-01T09:51:56.826+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:57.102 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:56 smithi190 conmon[247554]: debug 2023-10-01T09:51:56.826+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:51:57.103 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:56 smithi190 conmon[291240]: debug 2023-10-01T09:51:56.925+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:57.103 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:56 smithi190 conmon[291240]: debug 2023-10-01T09:51:56.925+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.328933+0000 front 2023-10-01T09:51:30.328817+0000 (oldest deadline 2023-10-01T09:51:53.129435+0000) 2023-10-01T09:51:57.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:57 smithi190 conmon[124440]: debug 2023-10-01T09:51:57.203+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:57.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:57 smithi190 conmon[124440]: debug 2023-10-01T09:51:57.203+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:51:57.557 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:57 smithi190 systemd[1]: Stopping Ceph osd.5 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:51:57.557 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:57 smithi190 conmon[291240]: debug 2023-10-01T09:51:57.333+0000 7fed6ca02700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T09:51:57.557 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:57 smithi190 conmon[291240]: debug 2023-10-01T09:51:57.333+0000 7fed6ca02700 -1 osd.5 616 *** Got signal Terminated *** 2023-10-01T09:51:57.558 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:57 smithi190 conmon[291240]: debug 2023-10-01T09:51:57.333+0000 7fed6ca02700 -1 osd.5 616 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:51:58.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:57 smithi190 conmon[247554]: debug 2023-10-01T09:51:57.804+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:58.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:57 smithi190 conmon[247554]: debug 2023-10-01T09:51:57.804+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:51:58.058 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:57 smithi190 conmon[291240]: debug 2023-10-01T09:51:57.930+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:58.058 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:57 smithi190 conmon[291240]: debug 2023-10-01T09:51:57.930+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.328933+0000 front 2023-10-01T09:51:30.328817+0000 (oldest deadline 2023-10-01T09:51:53.129435+0000) 2023-10-01T09:51:58.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:58 smithi190 conmon[124440]: debug 2023-10-01T09:51:58.242+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:58.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:58 smithi190 conmon[124440]: debug 2023-10-01T09:51:58.242+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:51:59.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:58 smithi190 conmon[247554]: debug 2023-10-01T09:51:58.777+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:51:59.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:58 smithi190 conmon[247554]: debug 2023-10-01T09:51:58.777+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:51:59.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:58 smithi190 conmon[291240]: debug 2023-10-01T09:51:58.980+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:51:59.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:58 smithi190 conmon[291240]: debug 2023-10-01T09:51:58.980+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.328933+0000 front 2023-10-01T09:51:30.328817+0000 (oldest deadline 2023-10-01T09:51:53.129435+0000) 2023-10-01T09:51:59.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:59 smithi190 conmon[124440]: debug 2023-10-01T09:51:59.243+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:51:59.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:51:59 smithi190 conmon[124440]: debug 2023-10-01T09:51:59.243+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:00.223 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:59 smithi190 conmon[291240]: debug 2023-10-01T09:51:59.971+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:52:00.224 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:51:59 smithi190 conmon[291240]: debug 2023-10-01T09:51:59.971+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.328933+0000 front 2023-10-01T09:51:30.328817+0000 (oldest deadline 2023-10-01T09:51:53.129435+0000) 2023-10-01T09:52:00.224 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:59 smithi190 conmon[247554]: debug 2023-10-01T09:51:59.823+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:52:00.224 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:51:59 smithi190 conmon[247554]: debug 2023-10-01T09:51:59.823+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:52:00.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:00 smithi190 conmon[124440]: debug 2023-10-01T09:52:00.223+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:00.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:00 smithi190 conmon[124440]: debug 2023-10-01T09:52:00.223+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:01.211 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:00 smithi190 conmon[247554]: debug 2023-10-01T09:52:00.829+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:52:01.211 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:00 smithi190 conmon[247554]: debug 2023-10-01T09:52:00.829+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:52:01.211 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:52:00 smithi190 conmon[291240]: debug 2023-10-01T09:52:00.961+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:52:01.211 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:52:00 smithi190 conmon[291240]: debug 2023-10-01T09:52:00.961+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.328933+0000 front 2023-10-01T09:51:30.328817+0000 (oldest deadline 2023-10-01T09:51:53.129435+0000) 2023-10-01T09:52:01.212 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:52:00 smithi190 conmon[291240]: debug 2023-10-01T09:52:00.961+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:34.830289+0000 front 2023-10-01T09:51:34.830490+0000 (oldest deadline 2023-10-01T09:52:00.730416+0000) 2023-10-01T09:52:01.556 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:01 smithi190 conmon[124440]: debug 2023-10-01T09:52:01.210+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:01.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:01 smithi190 conmon[124440]: debug 2023-10-01T09:52:01.210+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:01.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:01 smithi190 conmon[124440]: debug 2023-10-01T09:52:01.210+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:02.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:01 smithi190 conmon[247554]: debug 2023-10-01T09:52:01.804+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:52:02.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:01 smithi190 conmon[247554]: debug 2023-10-01T09:52:01.804+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:52:02.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:52:01 smithi190 conmon[291240]: debug 2023-10-01T09:52:01.985+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:20.427377+0000 front 2023-10-01T09:51:20.427434+0000 (oldest deadline 2023-10-01T09:51:43.927384+0000) 2023-10-01T09:52:02.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:52:01 smithi190 conmon[291240]: debug 2023-10-01T09:52:01.985+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.328933+0000 front 2023-10-01T09:51:30.328817+0000 (oldest deadline 2023-10-01T09:51:53.129435+0000) 2023-10-01T09:52:02.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:52:01 smithi190 conmon[291240]: debug 2023-10-01T09:52:01.985+0000 7fed65977700 -1 osd.5 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:34.830289+0000 front 2023-10-01T09:51:34.830490+0000 (oldest deadline 2023-10-01T09:52:00.730416+0000) 2023-10-01T09:52:02.556 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:02 smithi190 conmon[124440]: debug 2023-10-01T09:52:02.184+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:02.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:02 smithi190 conmon[124440]: debug 2023-10-01T09:52:02.184+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:02.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:02 smithi190 conmon[124440]: debug 2023-10-01T09:52:02.184+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:03.057 INFO:journalctl@ceph.osd.5.smithi190.stdout:Oct 01 09:52:02 smithi190 bash[418050]: 6377b4c138ce70222ecbae6de4699f733362da09c749352bd7838513f16e4913 2023-10-01T09:52:03.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:02 smithi190 conmon[247554]: debug 2023-10-01T09:52:02.827+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:52:03.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:02 smithi190 conmon[247554]: debug 2023-10-01T09:52:02.827+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:52:03.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:03 smithi190 conmon[124440]: debug 2023-10-01T09:52:03.159+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:03.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:03 smithi190 conmon[124440]: debug 2023-10-01T09:52:03.159+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:03.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:03 smithi190 conmon[124440]: debug 2023-10-01T09:52:03.159+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:03.937 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:03 smithi190 conmon[247554]: debug 2023-10-01T09:52:03.820+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:52:03.937 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:03 smithi190 conmon[247554]: debug 2023-10-01T09:52:03.820+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:52:04.307 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:04 smithi190 conmon[124440]: debug 2023-10-01T09:52:04.113+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:04.307 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:04 smithi190 conmon[124440]: debug 2023-10-01T09:52:04.113+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:04.307 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:04 smithi190 conmon[124440]: debug 2023-10-01T09:52:04.113+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:04.578 DEBUG:teuthology.orchestra.run.smithi190:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-01T09:52:04.660 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:52:04.660 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-01T09:52:04.660 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-01T09:52:04.661 DEBUG:teuthology.orchestra.run.smithi190:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.6 2023-10-01T09:52:04.979 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:04 smithi190 systemd[1]: Stopping Ceph osd.6 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:52:04.979 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:04 smithi190 conmon[247554]: debug 2023-10-01T09:52:04.783+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:52:04.979 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:04 smithi190 conmon[247554]: debug 2023-10-01T09:52:04.783+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:52:04.980 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:04 smithi190 conmon[247554]: debug 2023-10-01T09:52:04.977+0000 7fcf2156c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T09:52:05.306 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:04 smithi190 conmon[247554]: debug 2023-10-01T09:52:04.977+0000 7fcf2156c700 -1 osd.6 616 *** Got signal Terminated *** 2023-10-01T09:52:05.307 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:04 smithi190 conmon[247554]: debug 2023-10-01T09:52:04.977+0000 7fcf2156c700 -1 osd.6 616 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:52:05.307 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:05 smithi190 conmon[124440]: debug 2023-10-01T09:52:05.095+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:05.307 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:05 smithi190 conmon[124440]: debug 2023-10-01T09:52:05.095+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:05.308 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:05 smithi190 conmon[124440]: debug 2023-10-01T09:52:05.095+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:06.055 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:05 smithi190 conmon[247554]: debug 2023-10-01T09:52:05.764+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:52:06.055 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:05 smithi190 conmon[247554]: debug 2023-10-01T09:52:05.764+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:52:06.056 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:05 smithi190 conmon[247554]: debug 2023-10-01T09:52:05.764+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:35.468818+0000 front 2023-10-01T09:51:35.468862+0000 (oldest deadline 2023-10-01T09:52:04.869247+0000) 2023-10-01T09:52:06.306 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:06 smithi190 conmon[124440]: debug 2023-10-01T09:52:06.054+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:06.307 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:06 smithi190 conmon[124440]: debug 2023-10-01T09:52:06.054+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:06.307 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:06 smithi190 conmon[124440]: debug 2023-10-01T09:52:06.054+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:07.056 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:06 smithi190 conmon[247554]: debug 2023-10-01T09:52:06.719+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:52:07.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:06 smithi190 conmon[247554]: debug 2023-10-01T09:52:06.719+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:52:07.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:06 smithi190 conmon[247554]: debug 2023-10-01T09:52:06.719+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:35.468818+0000 front 2023-10-01T09:51:35.468862+0000 (oldest deadline 2023-10-01T09:52:04.869247+0000) 2023-10-01T09:52:07.556 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:07 smithi190 conmon[124440]: debug 2023-10-01T09:52:07.095+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:07.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:07 smithi190 conmon[124440]: debug 2023-10-01T09:52:07.095+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:07.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:07 smithi190 conmon[124440]: debug 2023-10-01T09:52:07.095+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:08.056 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:07 smithi190 conmon[247554]: debug 2023-10-01T09:52:07.674+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:52:08.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:07 smithi190 conmon[247554]: debug 2023-10-01T09:52:07.674+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:52:08.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:07 smithi190 conmon[247554]: debug 2023-10-01T09:52:07.674+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:35.468818+0000 front 2023-10-01T09:51:35.468862+0000 (oldest deadline 2023-10-01T09:52:04.869247+0000) 2023-10-01T09:52:08.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:08 smithi190 conmon[124440]: debug 2023-10-01T09:52:08.076+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:08.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:08 smithi190 conmon[124440]: debug 2023-10-01T09:52:08.076+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:08.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:08 smithi190 conmon[124440]: debug 2023-10-01T09:52:08.076+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:09.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:08 smithi190 conmon[247554]: debug 2023-10-01T09:52:08.661+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:52:09.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:08 smithi190 conmon[247554]: debug 2023-10-01T09:52:08.661+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:52:09.057 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:08 smithi190 conmon[247554]: debug 2023-10-01T09:52:08.661+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:35.468818+0000 front 2023-10-01T09:51:35.468862+0000 (oldest deadline 2023-10-01T09:52:04.869247+0000) 2023-10-01T09:52:09.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:09 smithi190 conmon[124440]: debug 2023-10-01T09:52:09.061+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:09.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:09 smithi190 conmon[124440]: debug 2023-10-01T09:52:09.061+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:09.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:09 smithi190 conmon[124440]: debug 2023-10-01T09:52:09.061+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:10.058 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:09 smithi190 conmon[247554]: debug 2023-10-01T09:52:09.613+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:21.665785+0000 front 2023-10-01T09:51:21.665873+0000 (oldest deadline 2023-10-01T09:51:43.365811+0000) 2023-10-01T09:52:10.058 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:09 smithi190 conmon[247554]: debug 2023-10-01T09:52:09.613+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:30.367585+0000 front 2023-10-01T09:51:30.367498+0000 (oldest deadline 2023-10-01T09:51:52.067412+0000) 2023-10-01T09:52:10.058 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:09 smithi190 conmon[247554]: debug 2023-10-01T09:52:09.613+0000 7fcf1ecea700 -1 osd.6 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:35.468818+0000 front 2023-10-01T09:51:35.468862+0000 (oldest deadline 2023-10-01T09:52:04.869247+0000) 2023-10-01T09:52:10.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:10 smithi190 conmon[124440]: debug 2023-10-01T09:52:10.059+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:10.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:10 smithi190 conmon[124440]: debug 2023-10-01T09:52:10.059+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:10.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:10 smithi190 conmon[124440]: debug 2023-10-01T09:52:10.059+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:10.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:10 smithi190 conmon[124440]: debug 2023-10-01T09:52:10.059+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6830 osd.3 since back 2023-10-01T09:51:46.556172+0000 front 2023-10-01T09:51:46.556097+0000 (oldest deadline 2023-10-01T09:52:10.055999+0000) 2023-10-01T09:52:11.056 INFO:journalctl@ceph.osd.6.smithi190.stdout:Oct 01 09:52:10 smithi190 bash[418634]: 37f530693c7fc2237ac08f17677754d220c69eabee9d845254186ce0581bc5d0 2023-10-01T09:52:11.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:11 smithi190 conmon[124440]: debug 2023-10-01T09:52:11.094+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:11.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:11 smithi190 conmon[124440]: debug 2023-10-01T09:52:11.094+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:11.558 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:11 smithi190 conmon[124440]: debug 2023-10-01T09:52:11.094+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:11.558 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:11 smithi190 conmon[124440]: debug 2023-10-01T09:52:11.094+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6830 osd.3 since back 2023-10-01T09:51:46.556172+0000 front 2023-10-01T09:51:46.556097+0000 (oldest deadline 2023-10-01T09:52:10.055999+0000) 2023-10-01T09:52:12.201 DEBUG:teuthology.orchestra.run.smithi190:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-01T09:52:12.242 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:52:12.243 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-01T09:52:12.243 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-01T09:52:12.243 DEBUG:teuthology.orchestra.run.smithi190:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.7 2023-10-01T09:52:12.548 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:12 smithi190 conmon[124440]: debug 2023-10-01T09:52:12.090+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:12.548 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:12 smithi190 conmon[124440]: debug 2023-10-01T09:52:12.090+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:12.549 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:12 smithi190 conmon[124440]: debug 2023-10-01T09:52:12.090+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:12.549 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:12 smithi190 conmon[124440]: debug 2023-10-01T09:52:12.090+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6830 osd.3 since back 2023-10-01T09:51:46.556172+0000 front 2023-10-01T09:51:46.556097+0000 (oldest deadline 2023-10-01T09:52:10.055999+0000) 2023-10-01T09:52:12.549 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:12 smithi190 systemd[1]: Stopping Ceph osd.7 for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:52:12.806 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:12 smithi190 conmon[124440]: debug 2023-10-01T09:52:12.546+0000 7fb3b7457700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T09:52:12.807 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:12 smithi190 conmon[124440]: debug 2023-10-01T09:52:12.546+0000 7fb3b7457700 -1 osd.7 616 *** Got signal Terminated *** 2023-10-01T09:52:12.807 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:12 smithi190 conmon[124440]: debug 2023-10-01T09:52:12.546+0000 7fb3b7457700 -1 osd.7 616 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T09:52:13.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:13 smithi190 conmon[124440]: debug 2023-10-01T09:52:13.088+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:13.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:13 smithi190 conmon[124440]: debug 2023-10-01T09:52:13.088+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:13.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:13 smithi190 conmon[124440]: debug 2023-10-01T09:52:13.088+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:13.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:13 smithi190 conmon[124440]: debug 2023-10-01T09:52:13.088+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6830 osd.3 since back 2023-10-01T09:51:46.556172+0000 front 2023-10-01T09:51:46.556097+0000 (oldest deadline 2023-10-01T09:52:10.055999+0000) 2023-10-01T09:52:14.556 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:14 smithi190 conmon[124440]: debug 2023-10-01T09:52:14.101+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:14.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:14 smithi190 conmon[124440]: debug 2023-10-01T09:52:14.101+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:14.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:14 smithi190 conmon[124440]: debug 2023-10-01T09:52:14.101+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:14.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:14 smithi190 conmon[124440]: debug 2023-10-01T09:52:14.101+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6830 osd.3 since back 2023-10-01T09:51:46.556172+0000 front 2023-10-01T09:51:46.556097+0000 (oldest deadline 2023-10-01T09:52:10.055999+0000) 2023-10-01T09:52:15.556 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:15 smithi190 conmon[124440]: debug 2023-10-01T09:52:15.126+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:15.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:15 smithi190 conmon[124440]: debug 2023-10-01T09:52:15.126+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:15.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:15 smithi190 conmon[124440]: debug 2023-10-01T09:52:15.126+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:15.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:15 smithi190 conmon[124440]: debug 2023-10-01T09:52:15.126+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6830 osd.3 since back 2023-10-01T09:51:46.556172+0000 front 2023-10-01T09:51:46.556097+0000 (oldest deadline 2023-10-01T09:52:10.055999+0000) 2023-10-01T09:52:16.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:16 smithi190 conmon[124440]: debug 2023-10-01T09:52:16.151+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:16.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:16 smithi190 conmon[124440]: debug 2023-10-01T09:52:16.151+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:16.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:16 smithi190 conmon[124440]: debug 2023-10-01T09:52:16.151+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:16.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:16 smithi190 conmon[124440]: debug 2023-10-01T09:52:16.151+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6830 osd.3 since back 2023-10-01T09:51:46.556172+0000 front 2023-10-01T09:51:46.556097+0000 (oldest deadline 2023-10-01T09:52:10.055999+0000) 2023-10-01T09:52:16.558 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:16 smithi190 conmon[124440]: debug 2023-10-01T09:52:16.151+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.190:6804 osd.4 since back 2023-10-01T09:51:50.056480+0000 front 2023-10-01T09:51:50.056392+0000 (oldest deadline 2023-10-01T09:52:15.956582+0000) 2023-10-01T09:52:17.556 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:17 smithi190 conmon[124440]: debug 2023-10-01T09:52:17.110+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6806 osd.0 since back 2023-10-01T09:51:18.452502+0000 front 2023-10-01T09:51:18.452638+0000 (oldest deadline 2023-10-01T09:51:43.752390+0000) 2023-10-01T09:52:17.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:17 smithi190 conmon[124440]: debug 2023-10-01T09:52:17.110+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6814 osd.1 since back 2023-10-01T09:51:29.653496+0000 front 2023-10-01T09:51:29.653613+0000 (oldest deadline 2023-10-01T09:51:55.553422+0000) 2023-10-01T09:52:17.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:17 smithi190 conmon[124440]: debug 2023-10-01T09:52:17.110+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6822 osd.2 since back 2023-10-01T09:51:37.254415+0000 front 2023-10-01T09:51:37.254505+0000 (oldest deadline 2023-10-01T09:52:00.754433+0000) 2023-10-01T09:52:17.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:17 smithi190 conmon[124440]: debug 2023-10-01T09:52:17.110+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.138:6830 osd.3 since back 2023-10-01T09:51:46.556172+0000 front 2023-10-01T09:51:46.556097+0000 (oldest deadline 2023-10-01T09:52:10.055999+0000) 2023-10-01T09:52:17.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:17 smithi190 conmon[124440]: debug 2023-10-01T09:52:17.110+0000 7fb3b2bd1700 -1 osd.7 616 heartbeat_check: no reply from 172.21.15.190:6804 osd.4 since back 2023-10-01T09:51:50.056480+0000 front 2023-10-01T09:51:50.056392+0000 (oldest deadline 2023-10-01T09:52:15.956582+0000) 2023-10-01T09:52:18.557 INFO:journalctl@ceph.osd.7.smithi190.stdout:Oct 01 09:52:18 smithi190 bash[419211]: c925b7fc3562399e4d2169542c4f83b06ba326ac1d2fc308cee27d844bd96db3 2023-10-01T09:52:19.749 DEBUG:teuthology.orchestra.run.smithi190:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-01T09:52:19.790 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:52:19.791 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-01T09:52:19.791 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-10-01T09:52:19.791 DEBUG:teuthology.orchestra.run.smithi190:> sudo systemctl stop ceph-465870a4-6038-11ee-8db5-212e2dc638e7@prometheus.a 2023-10-01T09:52:20.057 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:52:19 smithi190 systemd[1]: Stopping Ceph prometheus.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:52:20.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:52:20 smithi190 conmon[151158]: ts=2023-10-01T09:52:20.101Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-01T09:52:20.557 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:52:20 smithi190 conmon[151158]: ts=2023-10-01T09:52:20.101Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-01T09:52:20.558 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:52:20 smithi190 conmon[151158]: ts=2023-10-01T09:52:20.101Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-01T09:52:20.558 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:52:20 smithi190 conmon[151158]: ts=2023-10-01T09:52:20.101Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-01T09:52:20.558 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:52:20 smithi190 conmon[151158]: ts=2023-10-01T09:52:20.101Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-01T09:52:20.558 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:52:20 smithi190 conmon[151158]: ts=2023-10-01T09:52:20.101Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-01T09:52:20.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:52:20 smithi190 conmon[151158]: ts=2023-10-01T09:52:20.101Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-01T09:52:20.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:52:20 smithi190 conmon[151158]: ts=2023-10-01T09:52:20.101Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-01T09:52:20.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:52:20 smithi190 conmon[151158]: ts=2023-10-01T09:52:20.101Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-01T09:52:20.559 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:52:20 smithi190 conmon[151158]: ts=2023-10-01T09:52:20.103Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-01T09:52:20.560 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:52:20 smithi190 conmon[151158]: ts=2023-10-01T09:52:20.103Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-01T09:52:20.560 INFO:journalctl@ceph.prometheus.a.smithi190.stdout:Oct 01 09:52:20 smithi190 conmon[151158]: ts=2023-10-01T09:52:20.103Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-01T09:52:20.805 DEBUG:teuthology.orchestra.run.smithi190:> sudo pkill -f 'journalctl -f -n 0 -u ceph-465870a4-6038-11ee-8db5-212e2dc638e7@prometheus.a.service' 2023-10-01T09:52:20.841 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T09:52:20.842 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-10-01T09:52:20.842 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-01T09:52:22.320 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:52:22 smithi138 systemd[1]: Stopping Ceph node-exporter.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:52:23.070 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:52:22 smithi138 bash[759267]: bd644aa2145be582d032ba451564f02b6818b10b88656fd439d0a01ec214a925 2023-10-01T09:52:23.070 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:52:22 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-01T09:52:23.070 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:52:22 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-01T09:52:23.070 INFO:journalctl@ceph.node-exporter.a.smithi138.stdout:Oct 01 09:52:22 smithi138 systemd[1]: Stopped Ceph node-exporter.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:52:23.459 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:52:23 smithi138 systemd[1]: Stopping Ceph alertmanager.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:52:23.460 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:52:23 smithi138 conmon[177222]: level=info ts=2023-10-01T09:52:23.324Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-01T09:52:24.136 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:52:23 smithi138 bash[759547]: 8debc681d94ead91fb942c886c8eb421520c07ea3848e0423ef359f52b9ce5fe 2023-10-01T09:52:24.136 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:52:24 smithi138 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-01T09:52:24.136 INFO:journalctl@ceph.alertmanager.a.smithi138.stdout:Oct 01 09:52:24 smithi138 systemd[1]: Stopped Ceph alertmanager.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:52:35.404 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-01T09:52:36.807 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:52:36 smithi190 systemd[1]: Stopping Ceph node-exporter.b for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:52:37.557 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:52:37 smithi190 bash[420398]: c08f7a49a78d5374c7265f1711761c92e078aca8742af1fc2fa72a35d3aeae34 2023-10-01T09:52:37.557 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:52:37 smithi190 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-01T09:52:37.557 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:52:37 smithi190 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-01T09:52:37.557 INFO:journalctl@ceph.node-exporter.b.smithi190.stdout:Oct 01 09:52:37 smithi190 systemd[1]: Stopped Ceph node-exporter.b for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:52:37.931 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:52:37 smithi190 systemd[1]: Stopping Ceph grafana.a for 465870a4-6038-11ee-8db5-212e2dc638e7... 2023-10-01T09:52:37.931 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:52:37 smithi190 conmon[146762]: t=2023-10-01T09:52:37+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-01T09:52:38.584 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:52:38 smithi190 bash[420679]: 58ea19a8c949e621b7bb100277b98c89386c960325c4b078c4ab95dcd1119787 2023-10-01T09:52:38.584 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:52:38 smithi190 systemd[1]: ceph-465870a4-6038-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-01T09:52:38.584 INFO:journalctl@ceph.grafana.a.smithi190.stdout:Oct 01 09:52:38 smithi190 systemd[1]: Stopped Ceph grafana.a for 465870a4-6038-11ee-8db5-212e2dc638e7. 2023-10-01T09:52:43.328 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-01T09:52:43.365 DEBUG:teuthology.orchestra.run.smithi190:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-01T09:52:43.395 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-01T09:52:43.398 DEBUG:teuthology.misc:Transferring archived files from smithi138:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407954/remote/smithi138/crash 2023-10-01T09:52:43.398 DEBUG:teuthology.orchestra.run.smithi138:> sudo tar c -f - -C /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/crash -- . 2023-10-01T09:52:43.467 INFO:teuthology.orchestra.run.smithi138.stderr:tar: /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-01T09:52:43.468 INFO:teuthology.orchestra.run.smithi138.stderr:tar: Error is not recoverable: exiting now 2023-10-01T09:52:43.469 DEBUG:teuthology.misc:Transferring archived files from smithi190:/var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407954/remote/smithi190/crash 2023-10-01T09:52:43.470 DEBUG:teuthology.orchestra.run.smithi190:> sudo tar c -f - -C /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/crash -- . 2023-10-01T09:52:43.524 INFO:teuthology.orchestra.run.smithi190.stderr:tar: /var/lib/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-01T09:52:43.524 INFO:teuthology.orchestra.run.smithi190.stderr:tar: Error is not recoverable: exiting now 2023-10-01T09:52:43.525 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-01T09:52:43.525 DEBUG:teuthology.orchestra.run.smithi138:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-01T09:52:43.554 INFO:teuthology.orchestra.run.smithi138.stderr:grep: /var/log/ceph/465870a4-6038-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-01T09:52:43.557 INFO:tasks.cephadm:Compressing logs... 2023-10-01T09:52:43.557 DEBUG:teuthology.orchestra.run.smithi138:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-01T09:52:43.601 DEBUG:teuthology.orchestra.run.smithi190:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-01T09:52:43.625 INFO:teuthology.orchestra.run.smithi138.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-01T09:52:43.637 INFO:teuthology.orchestra.run.smithi190.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-10-01T10:01:42.705 INFO:tasks.cephadm:Archiving logs... 2023-10-01T10:01:42.707 DEBUG:teuthology.misc:Transferring archived files from smithi138:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407954/remote/smithi138/log 2023-10-01T10:01:42.707 DEBUG:teuthology.orchestra.run.smithi138:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-01T10:02:50.945 DEBUG:teuthology.misc:Transferring archived files from smithi190:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407954/remote/smithi190/log 2023-10-01T10:02:50.946 DEBUG:teuthology.orchestra.run.smithi190:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-01T10:03:36.198 INFO:tasks.cephadm:Removing cluster... 2023-10-01T10:03:36.222 DEBUG:teuthology.orchestra.run.smithi138:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 --force 2023-10-01T10:03:37.441 DEBUG:teuthology.orchestra.run.smithi190:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 465870a4-6038-11ee-8db5-212e2dc638e7 --force 2023-10-01T10:03:38.509 INFO:tasks.cephadm:Removing cephadm ... 2023-10-01T10:03:38.509 DEBUG:teuthology.orchestra.run.smithi138:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-01T10:03:38.530 DEBUG:teuthology.orchestra.run.smithi190:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-01T10:03:38.564 INFO:tasks.cephadm:Teardown complete 2023-10-01T10:03:38.564 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-10-01T10:03:38.574 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-10-01T10:03:38.574 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-01T10:03:38.577 DEBUG:teuthology.orchestra.run.smithi190:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-01T10:03:38.707 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-01T10:03:38.717 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-01T10:03:38.718 DEBUG:teuthology.orchestra.run.smithi138:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-01T10:03:38.720 DEBUG:teuthology.orchestra.run.smithi190:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-01T10:03:38.736 INFO:teuthology.orchestra.run.smithi190.stderr:bash: ntpq: command not found 2023-10-01T10:03:38.736 INFO:teuthology.orchestra.run.smithi138.stderr:bash: ntpq: command not found 2023-10-01T10:03:38.761 INFO:teuthology.orchestra.run.smithi138.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-01T10:03:38.761 INFO:teuthology.orchestra.run.smithi138.stdout:=============================================================================== 2023-10-01T10:03:38.761 INFO:teuthology.orchestra.run.smithi138.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 23 +16us[ +16us] +/- 78ms 2023-10-01T10:03:38.761 INFO:teuthology.orchestra.run.smithi138.stdout:^* hv02.front.sepia.ceph.com 3 6 377 24 -343us[ -335us] +/- 58ms 2023-10-01T10:03:38.761 INFO:teuthology.orchestra.run.smithi138.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 24 +532us[ +532us] +/- 91ms 2023-10-01T10:03:38.762 INFO:teuthology.orchestra.run.smithi138.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T10:03:38.763 INFO:teuthology.orchestra.run.smithi190.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-10-01T10:03:38.763 INFO:teuthology.orchestra.run.smithi190.stdout:=============================================================================== 2023-10-01T10:03:38.764 INFO:teuthology.orchestra.run.smithi190.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 21 +71us[ +71us] +/- 78ms 2023-10-01T10:03:38.764 INFO:teuthology.orchestra.run.smithi190.stdout:^* hv02.front.sepia.ceph.com 3 6 377 21 -341us[ -315us] +/- 58ms 2023-10-01T10:03:38.764 INFO:teuthology.orchestra.run.smithi190.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 21 +592us[ +618us] +/- 91ms 2023-10-01T10:03:38.764 INFO:teuthology.orchestra.run.smithi190.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-10-01T10:03:38.764 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-01T10:03:38.774 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-01T10:03:38.774 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-01T10:03:38.799 DEBUG:teuthology.orchestra.run.smithi138:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-01T10:03:38.950 DEBUG:teuthology.orchestra.run.smithi190:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-10-01T10:03:39.069 DEBUG:teuthology.orchestra.run.smithi138:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-01T10:03:39.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:03:39.104 DEBUG:teuthology.orchestra.run.smithi190:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-10-01T10:03:39.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:03:39.135 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-01T10:03:39.145 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-01T10:03:39.154 INFO:teuthology.task.internal:Duration was 5043.575721 seconds 2023-10-01T10:03:39.155 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-01T10:03:39.164 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-01T10:03:39.165 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-01T10:03:39.167 DEBUG:teuthology.orchestra.run.smithi190:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-01T10:03:39.235 INFO:teuthology.orchestra.run.smithi138.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-01T10:03:39.240 INFO:teuthology.orchestra.run.smithi190.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-10-01T10:03:39.595 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-01T10:03:39.595 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi138.front.sepia.ceph.com 2023-10-01T10:03:39.595 DEBUG:teuthology.orchestra.run.smithi138:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-01T10:03:39.626 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi190.front.sepia.ceph.com 2023-10-01T10:03:39.627 DEBUG:teuthology.orchestra.run.smithi190:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-01T10:03:39.668 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-01T10:03:39.668 DEBUG:teuthology.orchestra.run.smithi138:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-01T10:03:39.671 DEBUG:teuthology.orchestra.run.smithi190:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-01T10:03:39.852 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-01T10:03:39.852 DEBUG:teuthology.orchestra.run.smithi138:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-01T10:03:39.855 DEBUG:teuthology.orchestra.run.smithi190:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-01T10:03:40.253 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-01T10:03:40.264 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-01T10:03:40.264 DEBUG:teuthology.orchestra.run.smithi138:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-01T10:03:40.300 DEBUG:teuthology.orchestra.run.smithi190:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-01T10:03:40.344 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-01T10:03:40.355 DEBUG:teuthology.orchestra.run.smithi138:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-01T10:03:40.358 DEBUG:teuthology.orchestra.run.smithi190:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-01T10:03:40.399 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern = core 2023-10-01T10:03:40.417 INFO:teuthology.orchestra.run.smithi190.stdout:kernel.core_pattern = core 2023-10-01T10:03:40.472 DEBUG:teuthology.orchestra.run.smithi138:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-01T10:03:40.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:03:40.489 DEBUG:teuthology.orchestra.run.smithi190:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-01T10:03:40.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:03:40.528 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-01T10:03:40.539 INFO:teuthology.task.internal:Transferring archived files... 2023-10-01T10:03:40.540 DEBUG:teuthology.misc:Transferring archived files from smithi138:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407954/remote/smithi138 2023-10-01T10:03:40.541 DEBUG:teuthology.orchestra.run.smithi138:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-01T10:03:40.606 DEBUG:teuthology.misc:Transferring archived files from smithi190:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407954/remote/smithi190 2023-10-01T10:03:40.606 DEBUG:teuthology.orchestra.run.smithi190:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-01T10:03:40.658 INFO:teuthology.task.internal:Removing archive directory... 2023-10-01T10:03:40.659 DEBUG:teuthology.orchestra.run.smithi138:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-01T10:03:40.661 DEBUG:teuthology.orchestra.run.smithi190:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-01T10:03:40.704 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-01T10:03:40.717 INFO:teuthology.task.internal:Not uploading archives. 2023-10-01T10:03:40.717 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-01T10:03:40.729 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-01T10:03:40.729 DEBUG:teuthology.orchestra.run.smithi138:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-01T10:03:40.732 DEBUG:teuthology.orchestra.run.smithi190:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-01T10:03:40.760 INFO:teuthology.orchestra.run.smithi190.stdout: 262320 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 1 10:03 /home/ubuntu/cephtest 2023-10-01T10:03:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: 262320 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 1 10:03 /home/ubuntu/cephtest 2023-10-01T10:03:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: 4063233 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 1 09:49 /home/ubuntu/cephtest/mnt.0 2023-10-01T10:03:40.791 INFO:teuthology.orchestra.run.smithi138.stdout: 4063234 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 1 09:49 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T10:03:40.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:03:40.801 INFO:teuthology.orchestra.run.smithi138.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-01T10:03:40.802 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi138 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-01T10:03:40.808 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-01T10:03:40.820 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-01T10:03:40.833 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi138 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-01T10:03:40.833 INFO:teuthology.nuke:Checking targets against current locks 2023-10-01T10:03:40.864 DEBUG:teuthology.nuke:shortname: smithi138 2023-10-01T10:03:40.865 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-01T10:03:40.890 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi138.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407954', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-01 08:34:16.615934', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-01T10:03:40.894 DEBUG:teuthology.nuke:shortname: smithi190 2023-10-01T10:03:40.895 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-01T10:03:40.915 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi190.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407954', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-01 08:34:16.614776', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-10-01T10:03:40.947 INFO:teuthology.orchestra.console.smithi138:Power off 2023-10-01T10:03:40.948 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-01T10:03:40.970 DEBUG:teuthology.orchestra.console.smithi138:power off output: Chassis Power Control: Down/Off 2023-10-01T10:03:40.971 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T10:03:40.987 INFO:teuthology.orchestra.console.smithi190:Power off 2023-10-01T10:03:40.988 DEBUG:teuthology.orchestra.console.smithi190:pexpect command: ipmitool -H smithi190.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-01T10:03:40.998 DEBUG:teuthology.orchestra.console.smithi138:check power output: Chassis Power is on 2023-10-01T10:03:41.010 DEBUG:teuthology.orchestra.console.smithi190:power off output: Chassis Power Control: Down/Off 2023-10-01T10:03:41.010 DEBUG:teuthology.orchestra.console.smithi190:pexpect command: ipmitool -H smithi190.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T10:03:41.034 DEBUG:teuthology.orchestra.console.smithi190:check power output: Chassis Power is on 2023-10-01T10:03:44.999 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T10:03:45.035 DEBUG:teuthology.orchestra.console.smithi190:pexpect command: ipmitool -H smithi190.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T10:03:45.110 DEBUG:teuthology.orchestra.console.smithi138:check power output: Chassis Power is on 2023-10-01T10:03:45.145 DEBUG:teuthology.orchestra.console.smithi190:check power output: Chassis Power is on 2023-10-01T10:03:49.114 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T10:03:49.146 DEBUG:teuthology.orchestra.console.smithi190:pexpect command: ipmitool -H smithi190.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T10:03:49.225 DEBUG:teuthology.orchestra.console.smithi138:check power output: Chassis Power is on 2023-10-01T10:03:49.256 DEBUG:teuthology.orchestra.console.smithi190:check power output: Chassis Power is on 2023-10-01T10:03:53.230 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T10:03:53.257 DEBUG:teuthology.orchestra.console.smithi190:pexpect command: ipmitool -H smithi190.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T10:03:53.341 DEBUG:teuthology.orchestra.console.smithi138:check power output: Chassis Power is off 2023-10-01T10:03:53.367 DEBUG:teuthology.orchestra.console.smithi190:check power output: Chassis Power is off 2023-10-01T10:03:53.442 INFO:teuthology.orchestra.console.smithi138:Power off completed 2023-10-01T10:03:53.468 INFO:teuthology.orchestra.console.smithi190:Power off completed 2023-10-01T10:03:53.573 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 5043.575720787048 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=58e49ba47a38463e855bd0c02577a9ee status: fail success: false 2023-10-01T10:03:53.573 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-01T10:03:53.649 INFO:teuthology.run:FAIL